From 5b35c102b4df31c6a42ebbe3623a4403171d4a26 Mon Sep 17 00:00:00 2001 From: "Collin M. Barrett" Date: Sun, 30 Aug 2020 19:10:22 -0500 Subject: [PATCH] =?UTF-8?q?build:=20=F0=9F=94=A8=F0=9F=9A=A7=20WIP=20port?= =?UTF-8?q?=20forward=20pipeline=20configs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker-compose.prod.yml | 2 +- docker-compose.yml | 2 +- services/directory/azure-pipelines.api.yaml | 84 +++++++ .../directory/azure-pipelines.migrate.yaml | 207 ++++++++++++++++++ web/azure-pipelines.yaml | 61 ++++++ 5 files changed, 354 insertions(+), 2 deletions(-) create mode 100644 services/directory/azure-pipelines.api.yaml create mode 100644 services/directory/azure-pipelines.migrate.yaml create mode 100644 web/azure-pipelines.yaml diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 403afc66a..477dd8c1f 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -13,7 +13,7 @@ services: - ~/cert:/etc/nginx/cert:ro web: - image: ${DOCKER_REGISTRY-}filterlists_web + image: filterlists-web restart: always build: context: ./web diff --git a/docker-compose.yml b/docker-compose.yml index e43ea674f..37617c07e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,7 @@ services: - ./reverse-proxy/svr-includes:/etc/nginx/svr-includes:ro directory.api: - image: ${DOCKER_REGISTRY-}filterlists_directory_api + image: filterlists-directory-api restart: always build: context: ./services diff --git a/services/directory/azure-pipelines.api.yaml b/services/directory/azure-pipelines.api.yaml new file mode 100644 index 000000000..4dfd04b97 --- /dev/null +++ b/services/directory/azure-pipelines.api.yaml @@ -0,0 +1,84 @@ +trigger: + batch: false + branches: + include: [master] + paths: + include: + [ + services/directory/azure-pipelines.api.yaml, + services/directory/src/FilterLists.Directory.Domain/*, + services/directory/src/FilterLists.Directory.Infrastructure/*, + services/directory/src/FilterLists.Directory.Infrastructure.Migrations/*, + services/directory/src/FilterLists.Directory.Data.Application/*, + services/directory/src/FilterLists.Directory.Data.Api/*, + docker-compose.yml, + docker-compose.prod.yml, + ] + +pr: + autoCancel: false + branches: + include: [master] + paths: + include: + [ + services/directory/azure-pipelines.api.yaml, + services/directory/src/FilterLists.Directory.Domain/*, + services/directory/src/FilterLists.Directory.Infrastructure/*, + services/directory/src/FilterLists.Directory.Infrastructure.Migrations/*, + services/directory/src/FilterLists.Directory.Data.Application/*, + services/directory/src/FilterLists.Directory.Data.Api/*, + docker-compose.yml, + docker-compose.prod.yml, + ] + +pool: + vmImage: ubuntu-latest + +steps: + - task: DockerCompose@0 + displayName: build + inputs: + dockerComposeFile: docker-compose.yml + additionalDockerComposeFiles: docker-compose.prod.yml + dockerComposeCommand: build directory.api + + - task: Docker@2 + displayName: tag build number + inputs: + containerRegistry: collinbarrett-dockerhub + repository: collinbarrett/filterlists-directory-api + command: tag + arguments: collinbarrett/filterlists-directory-api:latest collinbarrett/filterlists-directory-api:$(Build.BuildId) + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) + + - task: Docker@2 + displayName: push + inputs: + containerRegistry: collinbarrett-dockerhub + repository: collinbarrett/filterlists-directory-api + command: push + tags: | + latest + $(Build.BuildId) + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) + + - task: CopyFiles@2 + displayName: stage docker-compose files + inputs: + SourceFolder: $(Build.SourcesDirectory) + Contents: | + docker-compose.yml + docker-compose.prod.yml + TargetFolder: $(Build.ArtifactStagingDirectory) + CleanTargetFolder: true + OverWrite: true + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) + + - task: PublishBuildArtifacts@1 + displayName: publish docker-compose files + inputs: + PathtoPublish: $(Build.ArtifactStagingDirectory) + ArtifactName: drop + publishLocation: Container + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) diff --git a/services/directory/azure-pipelines.migrate.yaml b/services/directory/azure-pipelines.migrate.yaml new file mode 100644 index 000000000..5766e05e1 --- /dev/null +++ b/services/directory/azure-pipelines.migrate.yaml @@ -0,0 +1,207 @@ +# Adds an Entity Framework Core migration to a pull request that changes data in data/*.json + +trigger: none + +pr: + autoCancel: true + branches: + include: [master] + paths: + include: + [ + services/directory/data/*, + services/directory/src/FilterLists.Infrastructure.Migrations/Migrations/*, + ] + +pool: + vmImage: ubuntu-latest + +steps: + - checkout: none + + - task: Bash@3 + displayName: checkout source branch + inputs: + targetType: inline + script: | + FORKURI=$(curl -X GET "https://api.github.com/repos/$BUILD_REPOSITORY_NAME/pulls/$SYSTEM_PULLREQUEST_PULLREQUESTNUMBER" | jq -r '.head.repo.clone_url') + git clone --single-branch -b "$SYSTEM_PULLREQUEST_SOURCEBRANCH" "$FORKURI" . + + - task: Bash@3 + displayName: abort if just migrated + name: abortJustMigrated + env: + GITHUBNAME: $(GITHUBNAME) + inputs: + targetType: inline + script: | + LASTAUTHOR=$(git log -n 1 --pretty=format:"%an") + LASTMSG=$(git log -n 1 --pretty=format:"%s") + if [[ $LASTAUTHOR == "$GITHUBNAME" ]] && [[ $LASTMSG == "feat(data): migrate PR #$SYSTEM_PULLREQUEST_PULLREQUESTNUMBER" ]] ; then + echo "Just migrated. Aborting..." + else + echo "##vso[task.setvariable variable=aborted;isOutput=true]false" + fi + + - task: ShellScript@2 + displayName: lint json + condition: and(eq(variables['abortJustMigrated.aborted'], 'false'), succeeded()) + inputs: + scriptPath: services/directory/data/lint.sh + + - task: Bash@3 + displayName: git config + condition: and(eq(variables['abortJustMigrated.aborted'], 'false'), succeeded()) + env: + GITHUBNAME: $(GITHUBNAME) + GITHUBEMAIL: $(GITHUBEMAIL) + inputs: + targetType: inline + script: | + git config --global user.name "$GITHUBNAME" + git config --global user.email "$GITHUBEMAIL" + + - task: Bash@3 + displayName: commit lint results + name: commitLint + condition: and(eq(variables['abortJustMigrated.aborted'], 'false'), succeeded()) + inputs: + targetType: inline + workingDirectory: services/directory/data + script: | + DIFF=$(git status -s -- . | wc -l) + echo "$DIFF file(s) required linting" + if (( $DIFF > 0 )) ; then + git add . + git commit -m "refactor(data): lint PR #$SYSTEM_PULLREQUEST_PULLREQUESTNUMBER" + fi + + - task: UseDotNet@2 + displayName: use latest dotnet sdk + condition: and(eq(variables['abortJustMigrated.aborted'], 'false'), succeeded()) + inputs: + version: 3.x + + - task: CmdLine@2 + displayName: install ef dotnet tool + condition: and(eq(variables['abortJustMigrated.aborted'], 'false'), succeeded()) + inputs: + script: dotnet tool install -g dotnet-ef + + - task: Bash@3 + displayName: revert existing PR migration + condition: and(eq(variables['abortJustMigrated.aborted'], 'false'), succeeded()) + env: + GITHUBNAME: $(GITHUBNAME) + inputs: + targetType: inline + workingDirectory: services/directory/src + script: | + MIGLIST=$(dotnet ef migrations list -p FilterLists.Infrastructure.Migrations -s FilterLists.Directory.Api) + echo "$MIGLIST" + if [[ $MIGLIST == *$SYSTEM_PULLREQUEST_PULLREQUESTNUMBER ]] ; then + echo "A migration already exists for PR #$SYSTEM_PULLREQUEST_PULLREQUESTNUMBER. Reverting..." + REVERTHASH=$(git log -n 1 --author="$GITHUBNAME" --grep="feat(data): migrate PR #$SYSTEM_PULLREQUEST_PULLREQUESTNUMBER" --pretty=format:"%H") + git revert --no-edit "$REVERTHASH" + fi + + - task: Bash@3 + displayName: sync with upstream + condition: and(eq(variables['abortJustMigrated.aborted'], 'false'), succeeded()) + inputs: + targetType: inline + script: | + git remote add upstream https://github.com/collinbarrett/FilterLists.git + git fetch upstream + git merge upstream/master + CONFLICTS=$(git ls-files -u | wc -l) + if [ "$CONFLICTS" -gt 0 ] ; then + echo "Conflicts with upstream repository. Aborting..." + exit 1 + fi + + - task: CmdLine@2 + displayName: add migration + condition: and(eq(variables['abortJustMigrated.aborted'], 'false'), succeeded()) + inputs: + workingDirectory: services/directory/src + script: dotnet ef migrations add $(System.PullRequest.PullRequestNumber) -p FilterLists.Infrastructure.Migrations -s FilterLists.Directory.Api + + - task: Bash@3 + displayName: commit or abandon no-op migration + name: commitOrAbandon + condition: and(eq(variables['abortJustMigrated.aborted'], 'false'), succeeded()) + inputs: + targetType: inline + workingDirectory: services/directory/src/FilterLists.Infrastructure.Migrations + script: | + DIFF=$(git status -s -- . | wc -l) + echo "$DIFF file(s) changed" + if (( $DIFF != 3 )) ; then + echo "No-op migration. Effective EF migrations add/change 3 files (.Designer.cs, .cs, and *ModelSnapshot.cs). Abandoning..." + else + git add . + git commit -m "feat(data): migrate PR #$SYSTEM_PULLREQUEST_PULLREQUESTNUMBER" + echo "##vso[task.setvariable variable=committed;isOutput=true]true" + fi + + # - task: DockerCompose@0 + # displayName: build test-data + # condition: and(eq(variables['abortJustMigrated.aborted'], 'false'), eq(variables['commitOrAbandon.committed'], 'true'), succeeded()) + # inputs: + # dockerComposeFile: docker-compose.data.tests.yml + # dockerComposeCommand: build api + + # - task: Docker@2 + # displayName: create volume test-data-results + # condition: and(eq(variables['abortJustMigrated.aborted'], 'false'), eq(variables['commitOrAbandon.committed'], 'true'), succeeded()) + # inputs: + # command: volume + # arguments: create test-data-results + + # - task: DockerCompose@0 + # displayName: up test-data db + # condition: and(eq(variables['abortJustMigrated.aborted'], 'false'), eq(variables['commitOrAbandon.committed'], 'true'), succeeded()) + # inputs: + # dockerComposeFile: docker-compose.data.tests.yml + # dockerComposeCommand: up -d mariadb + + # - task: DockerCompose@0 + # displayName: run test-data + # condition: and(eq(variables['abortJustMigrated.aborted'], 'false'), eq(variables['commitOrAbandon.committed'], 'true'), succeeded()) + # inputs: + # dockerComposeFile: docker-compose.data.tests.yml + # dockerComposeCommand: run api + + # - task: Docker@2 + # displayName: create container test-data-results + # condition: and(eq(variables['abortJustMigrated.aborted'], 'false'), eq(variables['commitOrAbandon.committed'], 'true'), succeededOrFailed()) + # inputs: + # command: container + # arguments: create --name test-data-results -v test-data-results:/results hello-world + + # - task: Docker@2 + # displayName: copy out test-data results + # condition: and(eq(variables['abortJustMigrated.aborted'], 'false'), eq(variables['commitOrAbandon.committed'], 'true'), succeededOrFailed()) + # inputs: + # command: cp + # arguments: test-data-results:/results $(System.DefaultWorkingDirectory) + + # - task: PublishTestResults@2 + # displayName: publish test results + # condition: and(eq(variables['abortJustMigrated.aborted'], 'false'), eq(variables['commitOrAbandon.committed'], 'true'), succeededOrFailed()) + # inputs: + # testResultsFormat: VSTest + # testResultsFiles: "**/*.trx" + + - task: Bash@3 + displayName: push + condition: and(eq(variables['abortJustMigrated.aborted'], 'false'), succeeded()) + env: + GITHUBPAT: $(GITHUBPAT) + inputs: + targetType: inline + script: | + git config --global credential.helper store + echo "https://$GITHUBPAT:x-oauth-basic@github.com" >> ~/.git-credentials + git push origin "$SYSTEM_PULLREQUEST_SOURCEBRANCH" diff --git a/web/azure-pipelines.yaml b/web/azure-pipelines.yaml new file mode 100644 index 000000000..49df1758a --- /dev/null +++ b/web/azure-pipelines.yaml @@ -0,0 +1,61 @@ +trigger: + batch: false + branches: + include: [master] + paths: + include: [web/*, docker-compose.yml, docker-compose.prod.yml] + +pr: + autoCancel: false + branches: + include: [master] + paths: + include: [web/*, docker-compose.yml, docker-compose.prod.yml] + +pool: + vmImage: ubuntu-latest + +steps: + - task: Docker@2 + displayName: build + inputs: + containerRegistry: collinbarrett-dockerhub + repository: collinbarrett/filterlists-web + command: build + Dockerfile: web/Dockerfile + buildContext: ./web + tags: | + latest + $(Build.BuildId) + addPipelineData: false + + - task: Docker@2 + displayName: push + inputs: + containerRegistry: collinbarrett-dockerhub + repository: collinbarrett/filterlists-web + command: push + tags: | + latest + $(Build.BuildId) + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) + + - task: CopyFiles@2 + displayName: stage docker-compose files + inputs: + SourceFolder: $(Build.SourcesDirectory) + Contents: | + docker-compose.yml + docker-compose.prod.yml + TargetFolder: $(Build.ArtifactStagingDirectory) + CleanTargetFolder: true + OverWrite: true + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) + + - task: PublishBuildArtifacts@1 + displayName: publish docker-compose files + inputs: + PathtoPublish: $(Build.ArtifactStagingDirectory) + ArtifactName: drop + publishLocation: Container + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))