add and push ef migration to PRs containing data changes (#1357)

2026-03-11 09:04:27 +00:00 · 2020-02-12 02:39:10 +00:00 · 2020-02-12 02:39:10 +00:00 · 9bf6bc5d96
commit 9bf6bc5d96
parent a27def2399
6 changed files with 219 additions and 133 deletions
--- a/ops/nginx/azure-pipelines.yaml
+++ b/ops/nginx/azure-pipelines.yaml
@ -1,33 +1,33 @@
 trigger:
  batch: false
  branches:
-    include: ["master"]
+    include: [master]
  paths:
-    include: ["ops/nginx/*", "docker-compose.yml", "docker-compose.prod.yml"]
+    include: [ops/nginx/*, docker-compose.yml, docker-compose.prod.yml]

 pr: none

 pool:
-  vmImage: "ubuntu-latest"
+  vmImage: ubuntu-latest

 steps:
  - task: CopyFiles@2
-    displayName: "stage files"
+    displayName: stage files
    inputs:
-      SourceFolder: "$(Build.SourcesDirectory)"
+      SourceFolder: $(Build.SourcesDirectory)
      Contents: |
        ops/nginx/**
        docker-compose.yml
        docker-compose.prod.yml
-      TargetFolder: "$(Build.ArtifactStagingDirectory)"
+      TargetFolder: $(Build.ArtifactStagingDirectory)
      CleanTargetFolder: true
      OverWrite: true
-    condition: "eq(variables['Build.SourceBranch'], 'refs/heads/master')"
+    condition: eq(variables['Build.SourceBranch'], 'refs/heads/master')

  - task: PublishBuildArtifacts@1
-    displayName: "publish files"
+    displayName: publish files
    inputs:
-      PathtoPublish: "$(Build.ArtifactStagingDirectory)"
-      ArtifactName: "drop"
-      publishLocation: "Container"
-    condition: "and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))"
+      PathtoPublish: $(Build.ArtifactStagingDirectory)
+      ArtifactName: drop
+      publishLocation: Container
+    condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
--- a/server/azure-pipelines.api.yaml
+++ b/server/azure-pipelines.api.yaml
@ -1,162 +1,172 @@
 trigger:
  batch: false
  branches:
-    include: ["master"]
+    include: [master]
  paths:
    include:
      [
-        "server/azure-pipelines.api.yaml",
-        "server/src/FilterLists.Api/*",
-        "server/src/FilterLists.Services/*",
-        "server/src/FilterLists.Data/*",
-        "server/src/FilterLists.Data.Migrations/*",
-        "server/tests/FilterLists.Services.Tests/*",
-        "server/tests/FilterLists.Data.Tests/*",
-        "server/**/.editorconfig",
-        ".dockerignore",
-        "docker-compose.yml",
-        "docker-compose.prod.yml",
-        "docker-compose.data.tests.yml",
-        "docker-compose.services.tests.yml",
-        "data/*",
+        server/azure-pipelines.api.yaml,
+        server/src/FilterLists.Api/*,
+        server/src/FilterLists.Services/*,
+        server/src/FilterLists.Data/*,
+        server/src/FilterLists.Data.Migrations/*,
+        server/tests/FilterLists.Services.Tests/*,
+        server/tests/FilterLists.Data.Tests/*,
+        server/**/.editorconfig,
+        .dockerignore,
+        docker-compose.yml,
+        docker-compose.prod.yml,
+        docker-compose.data.tests.yml,
+        docker-compose.services.tests.yml,
      ]

 pr:
  autoCancel: false
  branches:
-    include: ["master"]
+    include: [master]
  paths:
    include:
      [
-        "server/azure-pipelines.api.yaml",
-        "server/src/FilterLists.Api/*",
-        "server/src/FilterLists.Services/*",
-        "server/src/FilterLists.Data/*",
-        "server/src/FilterLists.Data.Migrations/*",
-        "server/tests/FilterLists.Services.Tests/*",
-        "server/tests/FilterLists.Data.Tests/*",
-        "server/**/.editorconfig",
-        ".dockerignore",
-        "docker-compose.yml",
-        "docker-compose.prod.yml",
-        "docker-compose.data.tests.yml",
-        "docker-compose.services.tests.yml",
-        "data/*",
+        server/azure-pipelines.api.yaml,
+        server/src/FilterLists.Api/*,
+        server/src/FilterLists.Services/*,
+        server/src/FilterLists.Data/*,
+        server/src/FilterLists.Data.Migrations/*,
+        server/tests/FilterLists.Services.Tests/*,
+        server/tests/FilterLists.Data.Tests/*,
+        server/**/.editorconfig,
+        .dockerignore,
+        docker-compose.yml,
+        docker-compose.prod.yml,
+        docker-compose.data.tests.yml,
+        docker-compose.services.tests.yml,
      ]

 pool:
-  vmImage: "ubuntu-latest"
+  vmImage: ubuntu-latest

 steps:
-  - task: Docker@2
-    displayName: "create volume test-data-results"
+  - task: DockerCompose@0
+    displayName: build test-data
    inputs:
-      command: "volume"
-      arguments: "create test-data-results"
+      dockerComposeFile: docker-compose.data.tests.yml
+      dockerComposeCommand: build api
+
+  - task: Docker@2
+    displayName: create volume test-data-results
+    inputs:
+      command: volume
+      arguments: create test-data-results

  - task: DockerCompose@0
-    displayName: "up test-data db"
+    displayName: up test-data db
    inputs:
-      dockerComposeFile: "docker-compose.data.tests.yml"
-      dockerComposeCommand: "up -d mariadb"
+      dockerComposeFile: docker-compose.data.tests.yml
+      dockerComposeCommand: up -d mariadb

  - task: DockerCompose@0
-    displayName: "run test-data"
+    displayName: run test-data
    inputs:
-      dockerComposeFile: "docker-compose.data.tests.yml"
-      dockerComposeCommand: "run api"
-
-  - task: Docker@2
-    displayName: "create volume test-services-results"
-    inputs:
-      command: "volume"
-      arguments: "create test-services-results"
+      dockerComposeFile: docker-compose.data.tests.yml
+      dockerComposeCommand: run api

  - task: DockerCompose@0
-    displayName: "run test-services"
+    displayName: build test-services
    inputs:
-      dockerComposeFile: "docker-compose.services.tests.yml"
-      dockerComposeCommand: "run api"
+      dockerComposeFile: docker-compose.services.tests.yml
+      dockerComposeCommand: build api

  - task: Docker@2
-    displayName: "create container test-data-results"
+    displayName: create volume test-services-results
    inputs:
-      command: "container"
-      arguments: "create --name test-data-results -v test-data-results:/results hello-world"
-    condition: "succeededOrFailed()"
+      command: volume
+      arguments: create test-services-results
+
+  - task: DockerCompose@0
+    displayName: run test-services
+    inputs:
+      dockerComposeFile: docker-compose.services.tests.yml
+      dockerComposeCommand: run api

  - task: Docker@2
-    displayName: "copy out test-data results"
+    displayName: create container test-data-results
    inputs:
-      command: "cp"
-      arguments: "test-data-results:/results $(System.DefaultWorkingDirectory)"
-    condition: "succeededOrFailed()"
+      command: container
+      arguments: create --name test-data-results -v test-data-results:/results hello-world
+    condition: succeededOrFailed()

  - task: Docker@2
-    displayName: "create container test-services-results"
+    displayName: copy out test-data results
    inputs:
-      command: "container"
-      arguments: "create --name test-services-results -v test-services-results:/results hello-world"
-    condition: "succeededOrFailed()"
+      command: cp
+      arguments: test-data-results:/results $(System.DefaultWorkingDirectory)
+    condition: succeededOrFailed()

  - task: Docker@2
-    displayName: "copy out test-services results"
+    displayName: create container test-services-results
    inputs:
-      command: "cp"
-      arguments: "test-services-results:/results $(System.DefaultWorkingDirectory)"
-    condition: "succeededOrFailed()"
+      command: container
+      arguments: create --name test-services-results -v test-services-results:/results hello-world
+    condition: succeededOrFailed()
+
+  - task: Docker@2
+    displayName: copy out test-services results
+    inputs:
+      command: cp
+      arguments: test-services-results:/results $(System.DefaultWorkingDirectory)
+    condition: succeededOrFailed()

  - task: PublishTestResults@2
-    displayName: "publish test results"
+    displayName: publish test results
    inputs:
-      testResultsFormat: "VSTest"
+      testResultsFormat: VSTest
      testResultsFiles: "**/*.trx"
-    condition: "succeededOrFailed()"
+    condition: succeededOrFailed()

  - task: DockerCompose@0
-    displayName: "build final"
+    displayName: build final
    inputs:
-      dockerComposeFile: "docker-compose.yml"
-      additionalDockerComposeFiles: "docker-compose.prod.yml"
-      dockerComposeCommand: "build api"
-    condition: "and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))"
+      dockerComposeFile: docker-compose.yml
+      additionalDockerComposeFiles: docker-compose.prod.yml
+      dockerComposeCommand: build api
+    condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))

  - task: Docker@2
-    displayName: "tag build number"
+    displayName: tag build number
    inputs:
-      containerRegistry: "collinbarrett2-dockerhub"
-      repository: "collinbarrett2/filterlists.api"
-      command: "tag"
-      arguments: "collinbarrett2/filterlists.api:latest collinbarrett2/filterlists.api:$(Build.BuildId)"
-    condition: "and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))"
+      containerRegistry: collinbarrett2-dockerhub
+      repository: collinbarrett2/filterlists.api
+      command: tag
+      arguments: collinbarrett2/filterlists.api:latest collinbarrett2/filterlists.api:$(Build.BuildId)
+    condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))

  - task: Docker@2
-    displayName: "push"
+    displayName: push
    inputs:
-      containerRegistry: "collinbarrett2-dockerhub"
-      repository: "collinbarrett2/filterlists.api"
-      command: "push"
+      containerRegistry: collinbarrett2-dockerhub
+      repository: collinbarrett2/filterlists.api
+      command: push
      tags: |
        latest
        $(Build.BuildId)
-    condition: "and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))"
+    condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))

  - task: CopyFiles@2
-    displayName: "stage docker-compose files"
+    displayName: stage docker-compose files
    inputs:
-      SourceFolder: "$(Build.SourcesDirectory)"
+      SourceFolder: $(Build.SourcesDirectory)
      Contents: |
        docker-compose.yml
        docker-compose.prod.yml
-      TargetFolder: "$(Build.ArtifactStagingDirectory)"
+      TargetFolder: $(Build.ArtifactStagingDirectory)
      CleanTargetFolder: true
      OverWrite: true
-    condition: "and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))"
+    condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))

  - task: PublishBuildArtifacts@1
-    displayName: "publish docker-compose files"
+    displayName: publish docker-compose files
    inputs:
-      PathtoPublish: "$(Build.ArtifactStagingDirectory)"
-      ArtifactName: "drop"
-      publishLocation: "Container"
-    condition: "and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))"
+      PathtoPublish: $(Build.ArtifactStagingDirectory)
+      ArtifactName: drop
+      publishLocation: Container
+    condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
--- a/server/azure-pipelines.migrate.yaml
+++ b/server/azure-pipelines.migrate.yaml
@ -0,0 +1,78 @@
+trigger: none
+
+pr:
+  autoCancel: true
+  branches:
+    include: [master]
+  paths:
+    include: [data/*]
+
+pool:
+  vmImage: ubuntu-latest
+
+steps:
+  - checkout: none
+
+  - script: |
+      git config --global user.name "$(GitHub.Name)"
+      git config --global user.email "$(GitHub.Email)"
+      git clone $(Build.Repository.Uri) .
+      git checkout $(System.PullRequest.SourceBranch)
+    displayName: checkout source branch
+
+  - bash: |
+      LASTMSG=$(git log -n 1 --pretty=format:"%s")
+      echo "Last commit message: $LASTMSG"
+      if [[ "$LASTMSG" == "migrate PR #$SYSTEM_PULLREQUEST_PULLREQUESTNUMBER" ]] ; then
+        echo "Just migrated. Exiting..."
+        echo "##vso[task.setvariable variable=aborted;isOutput=true]true"
+      fi
+    displayName: abort if just migrated
+    name: abortIfJustMigrated
+
+  - task: UseDotNet@2
+    displayName: use latest dotnet sdk
+    inputs:
+      version: 3.x
+    condition: ne(variables['abortIfJustMigrated.aborted'], 'true')
+
+  - script: dotnet tool install -g dotnet-ef
+    displayName: install ef dotnet tool
+    condition: ne(variables['abortIfJustMigrated.aborted'], 'true')
+
+  - bash: |
+      MIGLIST=$(dotnet ef migrations list -p FilterLists.Data.Migrations -s FilterLists.Api)
+      echo "$MIGLIST"
+      if [[ "$MIGLIST" == *$SYSTEM_PULLREQUEST_PULLREQUESTNUMBER ]] ; then
+        echo "A migration already exists for PR #$SYSTEM_PULLREQUEST_PULLREQUESTNUMBER, reverting..."
+        REVERTHASH=$(git log --grep="migrate PR #$SYSTEM_PULLREQUEST_PULLREQUESTNUMBER" -n 1 --pretty=format:"%H")
+        git revert --no-edit $REVERTHASH
+      else
+        echo "No migration exists yet for PR #$SYSTEM_PULLREQUEST_PULLREQUESTNUMBER."
+      fi
+    displayName: revert any existing migration for PR
+    workingDirectory: server/src
+    condition: ne(variables['abortIfJustMigrated.aborted'], 'true')
+
+  - script: dotnet ef migrations add $(System.PullRequest.PullRequestNumber) -p FilterLists.Data.Migrations -s FilterLists.Api
+    displayName: add migration
+    workingDirectory: server/src
+    condition: ne(variables['abortIfJustMigrated.aborted'], 'true')
+
+  - bash: |
+      DIFF=$(git diff --numstat | wc -l )
+      echo "$DIFF files changed"
+      if (( $DIFF != 3 )) ; then
+        echo "noop migration. Effective EF migrations change 3 files (.Designer.cs, .cs, and *ModelSnapshot.cs). Abandoning..."
+      else
+        git commit -m "migrate PR #$(System.PullRequest.PullRequestNumber)"
+      fi
+    displayName: commit or abandon migration
+    condition: ne(variables['abortIfJustMigrated.aborted'], 'true')
+
+  - script: |
+      git config --global credential.helper store
+      echo "https://$(GitHub.PAT):x-oauth-basic@github.com" >> ~/.git-credentials
+      git push origin $(System.PullRequest.SourceBranch)
+    displayName: git push
+    condition: ne(variables['abortIfJustMigrated.aborted'], 'true')
--- a/server/src/FilterLists.Api/Dockerfile
+++ b/server/src/FilterLists.Api/Dockerfile
@ -5,7 +5,6 @@
 FROM mcr.microsoft.com/dotnet/core/runtime-deps:3.1-alpine as base
 ENTRYPOINT ["./FilterLists.Api"]
 EXPOSE 80
-COPY data/. data/
 WORKDIR /app

 # init build
--- a/server/src/FilterLists.Api/debug.Dockerfile
+++ b/server/src/FilterLists.Api/debug.Dockerfile
@ -4,5 +4,4 @@
 # init base debug (for VS)
 FROM mcr.microsoft.com/dotnet/core/aspnet:3.1-buster-slim AS base
 EXPOSE 80
-COPY data/. data/
 WORKDIR /app
--- a/web/azure-pipelines.yaml
+++ b/web/azure-pipelines.yaml
@ -1,61 +1,61 @@
 trigger:
  batch: false
  branches:
-    include: ["master"]
+    include: [master]
  paths:
-    include: ["web/*", "docker-compose.yml", "docker-compose.prod.yml"]
+    include: [web/*, docker-compose.yml, docker-compose.prod.yml]

 pr:
  autoCancel: false
  branches:
-    include: ["master"]
+    include: [master]
  paths:
-    include: ["web/*", "docker-compose.yml", "docker-compose.prod.yml"]
+    include: [web/*, docker-compose.yml, docker-compose.prod.yml]

 pool:
-  vmImage: "ubuntu-latest"
+  vmImage: ubuntu-latest

 steps:
  - task: Docker@2
-    displayName: "build"
+    displayName: build
    inputs:
-      containerRegistry: "collinbarrett2-dockerhub"
-      repository: "collinbarrett2/filterlists.web"
-      command: "build"
-      Dockerfile: "web/Dockerfile"
-      buildContext: "./web"
+      containerRegistry: collinbarrett2-dockerhub
+      repository: collinbarrett2/filterlists.web
+      command: build
+      Dockerfile: web/Dockerfile
+      buildContext: ./web
      tags: |
        latest
        $(Build.BuildId)
      addPipelineData: false

  - task: Docker@2
-    displayName: "push"
+    displayName: push
    inputs:
-      containerRegistry: "collinbarrett2-dockerhub"
-      repository: "collinbarrett2/filterlists.web"
-      command: "push"
+      containerRegistry: collinbarrett2-dockerhub
+      repository: collinbarrett2/filterlists.web
+      command: push
      tags: |
        latest
        $(Build.BuildId)
-    condition: "and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))"
+    condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))

  - task: CopyFiles@2
-    displayName: "stage docker-compose files"
+    displayName: stage docker-compose files
    inputs:
-      SourceFolder: "$(Build.SourcesDirectory)"
+      SourceFolder: $(Build.SourcesDirectory)
      Contents: |
        docker-compose.yml
        docker-compose.prod.yml
-      TargetFolder: "$(Build.ArtifactStagingDirectory)"
+      TargetFolder: $(Build.ArtifactStagingDirectory)
      CleanTargetFolder: true
      OverWrite: true
-    condition: "and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))"
+    condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))

  - task: PublishBuildArtifacts@1
-    displayName: "publish docker-compose files"
+    displayName: publish docker-compose files
    inputs:
-      PathtoPublish: "$(Build.ArtifactStagingDirectory)"
-      ArtifactName: "drop"
-      publishLocation: "Container"
-    condition: "and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))"
+      PathtoPublish: $(Build.ArtifactStagingDirectory)
+      ArtifactName: drop
+      publishLocation: Container
+    condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))