diff --git a/ops/nginx/azure-pipelines.yaml b/ops/nginx/azure-pipelines.yaml index cd1146767..8bdc1bb0e 100644 --- a/ops/nginx/azure-pipelines.yaml +++ b/ops/nginx/azure-pipelines.yaml @@ -1,33 +1,33 @@ trigger: batch: false branches: - include: ["master"] + include: [master] paths: - include: ["ops/nginx/*", "docker-compose.yml", "docker-compose.prod.yml"] + include: [ops/nginx/*, docker-compose.yml, docker-compose.prod.yml] pr: none pool: - vmImage: "ubuntu-latest" + vmImage: ubuntu-latest steps: - task: CopyFiles@2 - displayName: "stage files" + displayName: stage files inputs: - SourceFolder: "$(Build.SourcesDirectory)" + SourceFolder: $(Build.SourcesDirectory) Contents: | ops/nginx/** docker-compose.yml docker-compose.prod.yml - TargetFolder: "$(Build.ArtifactStagingDirectory)" + TargetFolder: $(Build.ArtifactStagingDirectory) CleanTargetFolder: true OverWrite: true - condition: "eq(variables['Build.SourceBranch'], 'refs/heads/master')" + condition: eq(variables['Build.SourceBranch'], 'refs/heads/master') - task: PublishBuildArtifacts@1 - displayName: "publish files" + displayName: publish files inputs: - PathtoPublish: "$(Build.ArtifactStagingDirectory)" - ArtifactName: "drop" - publishLocation: "Container" - condition: "and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))" + PathtoPublish: $(Build.ArtifactStagingDirectory) + ArtifactName: drop + publishLocation: Container + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) diff --git a/server/azure-pipelines.api.yaml b/server/azure-pipelines.api.yaml index 6de3e9faa..66332a8f0 100644 --- a/server/azure-pipelines.api.yaml +++ b/server/azure-pipelines.api.yaml @@ -1,162 +1,172 @@ trigger: batch: false branches: - include: ["master"] + include: [master] paths: include: [ - "server/azure-pipelines.api.yaml", - "server/src/FilterLists.Api/*", - "server/src/FilterLists.Services/*", - "server/src/FilterLists.Data/*", - "server/src/FilterLists.Data.Migrations/*", - "server/tests/FilterLists.Services.Tests/*", - "server/tests/FilterLists.Data.Tests/*", - "server/**/.editorconfig", - ".dockerignore", - "docker-compose.yml", - "docker-compose.prod.yml", - "docker-compose.data.tests.yml", - "docker-compose.services.tests.yml", - "data/*", + server/azure-pipelines.api.yaml, + server/src/FilterLists.Api/*, + server/src/FilterLists.Services/*, + server/src/FilterLists.Data/*, + server/src/FilterLists.Data.Migrations/*, + server/tests/FilterLists.Services.Tests/*, + server/tests/FilterLists.Data.Tests/*, + server/**/.editorconfig, + .dockerignore, + docker-compose.yml, + docker-compose.prod.yml, + docker-compose.data.tests.yml, + docker-compose.services.tests.yml, ] pr: autoCancel: false branches: - include: ["master"] + include: [master] paths: include: [ - "server/azure-pipelines.api.yaml", - "server/src/FilterLists.Api/*", - "server/src/FilterLists.Services/*", - "server/src/FilterLists.Data/*", - "server/src/FilterLists.Data.Migrations/*", - "server/tests/FilterLists.Services.Tests/*", - "server/tests/FilterLists.Data.Tests/*", - "server/**/.editorconfig", - ".dockerignore", - "docker-compose.yml", - "docker-compose.prod.yml", - "docker-compose.data.tests.yml", - "docker-compose.services.tests.yml", - "data/*", + server/azure-pipelines.api.yaml, + server/src/FilterLists.Api/*, + server/src/FilterLists.Services/*, + server/src/FilterLists.Data/*, + server/src/FilterLists.Data.Migrations/*, + server/tests/FilterLists.Services.Tests/*, + server/tests/FilterLists.Data.Tests/*, + server/**/.editorconfig, + .dockerignore, + docker-compose.yml, + docker-compose.prod.yml, + docker-compose.data.tests.yml, + docker-compose.services.tests.yml, ] pool: - vmImage: "ubuntu-latest" + vmImage: ubuntu-latest steps: - - task: Docker@2 - displayName: "create volume test-data-results" + - task: DockerCompose@0 + displayName: build test-data inputs: - command: "volume" - arguments: "create test-data-results" + dockerComposeFile: docker-compose.data.tests.yml + dockerComposeCommand: build api + + - task: Docker@2 + displayName: create volume test-data-results + inputs: + command: volume + arguments: create test-data-results - task: DockerCompose@0 - displayName: "up test-data db" + displayName: up test-data db inputs: - dockerComposeFile: "docker-compose.data.tests.yml" - dockerComposeCommand: "up -d mariadb" + dockerComposeFile: docker-compose.data.tests.yml + dockerComposeCommand: up -d mariadb - task: DockerCompose@0 - displayName: "run test-data" + displayName: run test-data inputs: - dockerComposeFile: "docker-compose.data.tests.yml" - dockerComposeCommand: "run api" - - - task: Docker@2 - displayName: "create volume test-services-results" - inputs: - command: "volume" - arguments: "create test-services-results" + dockerComposeFile: docker-compose.data.tests.yml + dockerComposeCommand: run api - task: DockerCompose@0 - displayName: "run test-services" + displayName: build test-services inputs: - dockerComposeFile: "docker-compose.services.tests.yml" - dockerComposeCommand: "run api" + dockerComposeFile: docker-compose.services.tests.yml + dockerComposeCommand: build api - task: Docker@2 - displayName: "create container test-data-results" + displayName: create volume test-services-results inputs: - command: "container" - arguments: "create --name test-data-results -v test-data-results:/results hello-world" - condition: "succeededOrFailed()" + command: volume + arguments: create test-services-results + + - task: DockerCompose@0 + displayName: run test-services + inputs: + dockerComposeFile: docker-compose.services.tests.yml + dockerComposeCommand: run api - task: Docker@2 - displayName: "copy out test-data results" + displayName: create container test-data-results inputs: - command: "cp" - arguments: "test-data-results:/results $(System.DefaultWorkingDirectory)" - condition: "succeededOrFailed()" + command: container + arguments: create --name test-data-results -v test-data-results:/results hello-world + condition: succeededOrFailed() - task: Docker@2 - displayName: "create container test-services-results" + displayName: copy out test-data results inputs: - command: "container" - arguments: "create --name test-services-results -v test-services-results:/results hello-world" - condition: "succeededOrFailed()" + command: cp + arguments: test-data-results:/results $(System.DefaultWorkingDirectory) + condition: succeededOrFailed() - task: Docker@2 - displayName: "copy out test-services results" + displayName: create container test-services-results inputs: - command: "cp" - arguments: "test-services-results:/results $(System.DefaultWorkingDirectory)" - condition: "succeededOrFailed()" + command: container + arguments: create --name test-services-results -v test-services-results:/results hello-world + condition: succeededOrFailed() + + - task: Docker@2 + displayName: copy out test-services results + inputs: + command: cp + arguments: test-services-results:/results $(System.DefaultWorkingDirectory) + condition: succeededOrFailed() - task: PublishTestResults@2 - displayName: "publish test results" + displayName: publish test results inputs: - testResultsFormat: "VSTest" + testResultsFormat: VSTest testResultsFiles: "**/*.trx" - condition: "succeededOrFailed()" + condition: succeededOrFailed() - task: DockerCompose@0 - displayName: "build final" + displayName: build final inputs: - dockerComposeFile: "docker-compose.yml" - additionalDockerComposeFiles: "docker-compose.prod.yml" - dockerComposeCommand: "build api" - condition: "and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))" + dockerComposeFile: docker-compose.yml + additionalDockerComposeFiles: docker-compose.prod.yml + dockerComposeCommand: build api + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) - task: Docker@2 - displayName: "tag build number" + displayName: tag build number inputs: - containerRegistry: "collinbarrett2-dockerhub" - repository: "collinbarrett2/filterlists.api" - command: "tag" - arguments: "collinbarrett2/filterlists.api:latest collinbarrett2/filterlists.api:$(Build.BuildId)" - condition: "and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))" + containerRegistry: collinbarrett2-dockerhub + repository: collinbarrett2/filterlists.api + command: tag + arguments: collinbarrett2/filterlists.api:latest collinbarrett2/filterlists.api:$(Build.BuildId) + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) - task: Docker@2 - displayName: "push" + displayName: push inputs: - containerRegistry: "collinbarrett2-dockerhub" - repository: "collinbarrett2/filterlists.api" - command: "push" + containerRegistry: collinbarrett2-dockerhub + repository: collinbarrett2/filterlists.api + command: push tags: | latest $(Build.BuildId) - condition: "and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))" + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) - task: CopyFiles@2 - displayName: "stage docker-compose files" + displayName: stage docker-compose files inputs: - SourceFolder: "$(Build.SourcesDirectory)" + SourceFolder: $(Build.SourcesDirectory) Contents: | docker-compose.yml docker-compose.prod.yml - TargetFolder: "$(Build.ArtifactStagingDirectory)" + TargetFolder: $(Build.ArtifactStagingDirectory) CleanTargetFolder: true OverWrite: true - condition: "and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))" + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) - task: PublishBuildArtifacts@1 - displayName: "publish docker-compose files" + displayName: publish docker-compose files inputs: - PathtoPublish: "$(Build.ArtifactStagingDirectory)" - ArtifactName: "drop" - publishLocation: "Container" - condition: "and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))" + PathtoPublish: $(Build.ArtifactStagingDirectory) + ArtifactName: drop + publishLocation: Container + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) diff --git a/server/azure-pipelines.migrate.yaml b/server/azure-pipelines.migrate.yaml new file mode 100644 index 000000000..1f69a0dc5 --- /dev/null +++ b/server/azure-pipelines.migrate.yaml @@ -0,0 +1,78 @@ +trigger: none + +pr: + autoCancel: true + branches: + include: [master] + paths: + include: [data/*] + +pool: + vmImage: ubuntu-latest + +steps: + - checkout: none + + - script: | + git config --global user.name "$(GitHub.Name)" + git config --global user.email "$(GitHub.Email)" + git clone $(Build.Repository.Uri) . + git checkout $(System.PullRequest.SourceBranch) + displayName: checkout source branch + + - bash: | + LASTMSG=$(git log -n 1 --pretty=format:"%s") + echo "Last commit message: $LASTMSG" + if [[ "$LASTMSG" == "migrate PR #$SYSTEM_PULLREQUEST_PULLREQUESTNUMBER" ]] ; then + echo "Just migrated. Exiting..." + echo "##vso[task.setvariable variable=aborted;isOutput=true]true" + fi + displayName: abort if just migrated + name: abortIfJustMigrated + + - task: UseDotNet@2 + displayName: use latest dotnet sdk + inputs: + version: 3.x + condition: ne(variables['abortIfJustMigrated.aborted'], 'true') + + - script: dotnet tool install -g dotnet-ef + displayName: install ef dotnet tool + condition: ne(variables['abortIfJustMigrated.aborted'], 'true') + + - bash: | + MIGLIST=$(dotnet ef migrations list -p FilterLists.Data.Migrations -s FilterLists.Api) + echo "$MIGLIST" + if [[ "$MIGLIST" == *$SYSTEM_PULLREQUEST_PULLREQUESTNUMBER ]] ; then + echo "A migration already exists for PR #$SYSTEM_PULLREQUEST_PULLREQUESTNUMBER, reverting..." + REVERTHASH=$(git log --grep="migrate PR #$SYSTEM_PULLREQUEST_PULLREQUESTNUMBER" -n 1 --pretty=format:"%H") + git revert --no-edit $REVERTHASH + else + echo "No migration exists yet for PR #$SYSTEM_PULLREQUEST_PULLREQUESTNUMBER." + fi + displayName: revert any existing migration for PR + workingDirectory: server/src + condition: ne(variables['abortIfJustMigrated.aborted'], 'true') + + - script: dotnet ef migrations add $(System.PullRequest.PullRequestNumber) -p FilterLists.Data.Migrations -s FilterLists.Api + displayName: add migration + workingDirectory: server/src + condition: ne(variables['abortIfJustMigrated.aborted'], 'true') + + - bash: | + DIFF=$(git diff --numstat | wc -l ) + echo "$DIFF files changed" + if (( $DIFF != 3 )) ; then + echo "noop migration. Effective EF migrations change 3 files (.Designer.cs, .cs, and *ModelSnapshot.cs). Abandoning..." + else + git commit -m "migrate PR #$(System.PullRequest.PullRequestNumber)" + fi + displayName: commit or abandon migration + condition: ne(variables['abortIfJustMigrated.aborted'], 'true') + + - script: | + git config --global credential.helper store + echo "https://$(GitHub.PAT):x-oauth-basic@github.com" >> ~/.git-credentials + git push origin $(System.PullRequest.SourceBranch) + displayName: git push + condition: ne(variables['abortIfJustMigrated.aborted'], 'true') diff --git a/server/src/FilterLists.Api/Dockerfile b/server/src/FilterLists.Api/Dockerfile index b04ba1176..2d78b1f4d 100644 --- a/server/src/FilterLists.Api/Dockerfile +++ b/server/src/FilterLists.Api/Dockerfile @@ -5,7 +5,6 @@ FROM mcr.microsoft.com/dotnet/core/runtime-deps:3.1-alpine as base ENTRYPOINT ["./FilterLists.Api"] EXPOSE 80 -COPY data/. data/ WORKDIR /app # init build diff --git a/server/src/FilterLists.Api/debug.Dockerfile b/server/src/FilterLists.Api/debug.Dockerfile index 7a5b2c61c..06bd17831 100644 --- a/server/src/FilterLists.Api/debug.Dockerfile +++ b/server/src/FilterLists.Api/debug.Dockerfile @@ -4,5 +4,4 @@ # init base debug (for VS) FROM mcr.microsoft.com/dotnet/core/aspnet:3.1-buster-slim AS base EXPOSE 80 -COPY data/. data/ WORKDIR /app diff --git a/web/azure-pipelines.yaml b/web/azure-pipelines.yaml index f37f15b95..25fc9401e 100644 --- a/web/azure-pipelines.yaml +++ b/web/azure-pipelines.yaml @@ -1,61 +1,61 @@ trigger: batch: false branches: - include: ["master"] + include: [master] paths: - include: ["web/*", "docker-compose.yml", "docker-compose.prod.yml"] + include: [web/*, docker-compose.yml, docker-compose.prod.yml] pr: autoCancel: false branches: - include: ["master"] + include: [master] paths: - include: ["web/*", "docker-compose.yml", "docker-compose.prod.yml"] + include: [web/*, docker-compose.yml, docker-compose.prod.yml] pool: - vmImage: "ubuntu-latest" + vmImage: ubuntu-latest steps: - task: Docker@2 - displayName: "build" + displayName: build inputs: - containerRegistry: "collinbarrett2-dockerhub" - repository: "collinbarrett2/filterlists.web" - command: "build" - Dockerfile: "web/Dockerfile" - buildContext: "./web" + containerRegistry: collinbarrett2-dockerhub + repository: collinbarrett2/filterlists.web + command: build + Dockerfile: web/Dockerfile + buildContext: ./web tags: | latest $(Build.BuildId) addPipelineData: false - task: Docker@2 - displayName: "push" + displayName: push inputs: - containerRegistry: "collinbarrett2-dockerhub" - repository: "collinbarrett2/filterlists.web" - command: "push" + containerRegistry: collinbarrett2-dockerhub + repository: collinbarrett2/filterlists.web + command: push tags: | latest $(Build.BuildId) - condition: "and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))" + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) - task: CopyFiles@2 - displayName: "stage docker-compose files" + displayName: stage docker-compose files inputs: - SourceFolder: "$(Build.SourcesDirectory)" + SourceFolder: $(Build.SourcesDirectory) Contents: | docker-compose.yml docker-compose.prod.yml - TargetFolder: "$(Build.ArtifactStagingDirectory)" + TargetFolder: $(Build.ArtifactStagingDirectory) CleanTargetFolder: true OverWrite: true - condition: "and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))" + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) - task: PublishBuildArtifacts@1 - displayName: "publish docker-compose files" + displayName: publish docker-compose files inputs: - PathtoPublish: "$(Build.ArtifactStagingDirectory)" - ArtifactName: "drop" - publishLocation: "Container" - condition: "and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))" + PathtoPublish: $(Build.ArtifactStagingDirectory) + ArtifactName: drop + publishLocation: Container + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))