gracefully rollback interrupted snapshots

currently assumes deduplication didn't partially complete (should be revisited at some point)

src/FilterLists.Agent/Program.cs

@@ -60,7 +60,7 @@ private static void CaptureSnapshots(int batchSize)
             var snapshotService = serviceProvider.GetService<SnapshotService>();
 
             Log("Capturing FilterList snapshots...");
-            snapshotService.CaptureSnapshotsAsync(batchSize).Wait();
+            snapshotService.CaptureAsync(batchSize).Wait();
             Log("\nSnapshots captured.");
             telemetryClient.Flush();
         }

src/FilterLists.Api/Migrations/20180210204923_AddIsCompletedFlagToSnapshot.Designer.cs

@@ -0,0 +1,522 @@
+// <auto-generated />
+
+using System;
+using FilterLists.Data;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Metadata;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+namespace FilterLists.Api.Migrations
+{
+    [DbContext(typeof(FilterListsDbContext))]
+    [Migration("20180210204923_AddIsCompletedFlagToSnapshot")]
+    partial class AddIsCompletedFlagToSnapshot
+    {
+        protected override void BuildTargetModel(ModelBuilder modelBuilder)
+        {
+#pragma warning disable 612, 618
+            modelBuilder
+                .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn)
+                .HasAnnotation("ProductVersion", "2.0.1-rtm-125");
+
+            modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
+            {
+                b.Property<int>("Id")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("SMALLINT UNSIGNED");
+
+                b.Property<DateTime>("CreatedDateUtc")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP");
+
+                b.Property<string>("Description")
+                 .HasColumnType("TEXT");
+
+                b.Property<string>("DescriptionSourceUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<DateTime?>("DiscontinuedDate");
+
+                b.Property<string>("DonateUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<string>("EmailAddress")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("VARCHAR(126)")
+                 .HasDefaultValueSql("NULL");
+
+                b.Property<string>("ForumUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<string>("HomeUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<string>("IssuesUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<int?>("LicenseId");
+
+                b.Property<DateTime>("ModifiedDateUtc")
+                 .ValueGeneratedOnAddOrUpdate()
+                 .HasColumnType("TIMESTAMP");
+
+                b.Property<string>("Name")
+                 .IsRequired()
+                 .HasColumnType("VARCHAR(126)");
+
+                b.Property<string>("PolicyUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<string>("SubmissionUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<int?>("SyntaxId");
+
+                b.Property<string>("ViewUrl")
+                 .HasColumnType("TEXT");
+
+                b.HasKey("Id");
+
+                b.HasIndex("LicenseId");
+
+                b.HasIndex("SyntaxId");
+
+                b.ToTable("filterlists");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
+            {
+                b.Property<int>("FilterListId");
+
+                b.Property<int>("LanguageId");
+
+                b.Property<DateTime>("CreatedDateUtc")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP");
+
+                b.HasKey("FilterListId", "LanguageId");
+
+                b.HasIndex("LanguageId");
+
+                b.ToTable("filterlists_languages");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
+            {
+                b.Property<int>("FilterListId");
+
+                b.Property<int>("MaintainerId");
+
+                b.Property<DateTime>("CreatedDateUtc")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP");
+
+                b.HasKey("FilterListId", "MaintainerId");
+
+                b.HasIndex("MaintainerId");
+
+                b.ToTable("filterlists_maintainers");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
+            {
+                b.Property<int>("ForkFilterListId");
+
+                b.Property<int>("UpstreamFilterListId");
+
+                b.Property<DateTime>("CreatedDateUtc")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP");
+
+                b.HasKey("ForkFilterListId", "UpstreamFilterListId");
+
+                b.HasIndex("UpstreamFilterListId");
+
+                b.ToTable("forks");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
+            {
+                b.Property<int>("MergeFilterListId");
+
+                b.Property<int>("UpstreamFilterListId");
+
+                b.Property<DateTime>("CreatedDateUtc")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP");
+
+                b.HasKey("MergeFilterListId", "UpstreamFilterListId");
+
+                b.HasIndex("UpstreamFilterListId");
+
+                b.ToTable("merges");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b =>
+            {
+                b.Property<int>("AddedBySnapshotId");
+
+                b.Property<int>("RuleId");
+
+                b.Property<DateTime>("CreatedDateUtc")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP");
+
+                b.Property<DateTime>("ModifiedDateUtc")
+                 .ValueGeneratedOnAddOrUpdate()
+                 .HasColumnType("TIMESTAMP");
+
+                b.Property<int?>("RemovedBySnapshotId");
+
+                b.HasKey("AddedBySnapshotId", "RuleId");
+
+                b.HasIndex("RemovedBySnapshotId");
+
+                b.HasIndex("RuleId");
+
+                b.ToTable("snapshots_rules");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
+            {
+                b.Property<int>("SyntaxId");
+
+                b.Property<int>("SoftwareId");
+
+                b.Property<DateTime>("CreatedDateUtc")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP");
+
+                b.HasKey("SyntaxId", "SoftwareId");
+
+                b.HasIndex("SoftwareId");
+
+                b.ToTable("software_syntaxes");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Language", b =>
+            {
+                b.Property<int>("Id")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("SMALLINT UNSIGNED");
+
+                b.Property<DateTime>("CreatedDateUtc")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP");
+
+                b.Property<string>("Iso6391")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("VARCHAR(2)")
+                 .HasDefaultValueSql("NULL");
+
+                b.Property<string>("Iso6392")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("VARCHAR(3)")
+                 .HasDefaultValueSql("NULL");
+
+                b.Property<string>("Iso6392B")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("VARCHAR(3)")
+                 .HasDefaultValueSql("NULL");
+
+                b.Property<string>("Iso6392T")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("VARCHAR(3)")
+                 .HasDefaultValueSql("NULL");
+
+                b.Property<string>("Iso6393")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("VARCHAR(3)")
+                 .HasDefaultValueSql("NULL");
+
+                b.Property<string>("LocalName")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("VARCHAR(126)")
+                 .HasDefaultValueSql("NULL");
+
+                b.Property<DateTime>("ModifiedDateUtc")
+                 .ValueGeneratedOnAddOrUpdate()
+                 .HasColumnType("TIMESTAMP");
+
+                b.Property<string>("Name")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("VARCHAR(126)")
+                 .HasDefaultValueSql("NULL");
+
+                b.HasKey("Id");
+
+                b.ToTable("languages");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.License", b =>
+            {
+                b.Property<int>("Id")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TINYINT UNSIGNED");
+
+                b.Property<DateTime>("CreatedDateUtc")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP");
+
+                b.Property<string>("DescriptionUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<DateTime>("ModifiedDateUtc")
+                 .ValueGeneratedOnAddOrUpdate()
+                 .HasColumnType("TIMESTAMP");
+
+                b.Property<string>("Name")
+                 .IsRequired()
+                 .HasColumnType("VARCHAR(126)");
+
+                b.Property<bool>("PermissiveAdaptation");
+
+                b.Property<bool>("PermissiveCommercial");
+
+                b.HasKey("Id");
+
+                b.ToTable("licenses");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Maintainer", b =>
+            {
+                b.Property<int>("Id")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("SMALLINT UNSIGNED");
+
+                b.Property<DateTime>("CreatedDateUtc")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP");
+
+                b.Property<string>("EmailAddress")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("VARCHAR(126)")
+                 .HasDefaultValueSql("NULL");
+
+                b.Property<string>("HomeUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<DateTime>("ModifiedDateUtc")
+                 .ValueGeneratedOnAddOrUpdate()
+                 .HasColumnType("TIMESTAMP");
+
+                b.Property<string>("Name")
+                 .IsRequired()
+                 .HasColumnType("VARCHAR(126)");
+
+                b.Property<string>("TwitterHandle")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("VARCHAR(126)")
+                 .HasDefaultValueSql("NULL");
+
+                b.HasKey("Id");
+
+                b.ToTable("maintainers");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Rule", b =>
+            {
+                b.Property<int>("Id")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("INT UNSIGNED");
+
+                b.Property<DateTime>("CreatedDateUtc")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP");
+
+                b.Property<string>("Raw")
+                 .IsRequired()
+                 .HasColumnType("VARCHAR(8192)");
+
+                b.HasKey("Id");
+
+                b.ToTable("rules");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
+            {
+                b.Property<int>("Id")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("MEDIUMINT UNSIGNED");
+
+                b.Property<DateTime>("CreatedDateUtc")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP");
+
+                b.Property<int>("FilterListId");
+
+                b.Property<string>("HttpStatusCode")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("VARCHAR(3)")
+                 .HasDefaultValueSql("NULL");
+
+                b.Property<bool>("IsCompleted");
+
+                b.HasKey("Id");
+
+                b.HasIndex("FilterListId");
+
+                b.ToTable("snapshots");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Software", b =>
+            {
+                b.Property<int>("Id")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TINYINT UNSIGNED");
+
+                b.Property<DateTime>("CreatedDateUtc")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP");
+
+                b.Property<string>("DownloadUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<string>("HomeUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<DateTime>("ModifiedDateUtc")
+                 .ValueGeneratedOnAddOrUpdate()
+                 .HasColumnType("TIMESTAMP");
+
+                b.Property<string>("Name")
+                 .IsRequired()
+                 .HasColumnType("VARCHAR(126)");
+
+                b.HasKey("Id");
+
+                b.ToTable("software");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Syntax", b =>
+            {
+                b.Property<int>("Id")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TINYINT UNSIGNED");
+
+                b.Property<DateTime>("CreatedDateUtc")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP");
+
+                b.Property<string>("DefinitionUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<DateTime>("ModifiedDateUtc")
+                 .ValueGeneratedOnAddOrUpdate()
+                 .HasColumnType("TIMESTAMP");
+
+                b.Property<string>("Name")
+                 .IsRequired()
+                 .HasColumnType("VARCHAR(126)");
+
+                b.HasKey("Id");
+
+                b.ToTable("syntaxes");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
+            {
+                b.HasOne("FilterLists.Data.Entities.License", "License")
+                 .WithMany("FilterLists")
+                 .HasForeignKey("LicenseId");
+
+                b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax")
+                 .WithMany("FilterLists")
+                 .HasForeignKey("SyntaxId");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
+            {
+                b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+                 .WithMany("FilterListLanguages")
+                 .HasForeignKey("FilterListId")
+                 .OnDelete(DeleteBehavior.Cascade);
+
+                b.HasOne("FilterLists.Data.Entities.Language", "Language")
+                 .WithMany("FilterListLanguages")
+                 .HasForeignKey("LanguageId")
+                 .OnDelete(DeleteBehavior.Cascade);
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
+            {
+                b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+                 .WithMany("FilterListMaintainers")
+                 .HasForeignKey("FilterListId")
+                 .OnDelete(DeleteBehavior.Cascade);
+
+                b.HasOne("FilterLists.Data.Entities.Maintainer", "Maintainer")
+                 .WithMany("FilterListMaintainers")
+                 .HasForeignKey("MaintainerId")
+                 .OnDelete(DeleteBehavior.Cascade);
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
+            {
+                b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList")
+                 .WithMany("ForkFilterLists")
+                 .HasForeignKey("ForkFilterListId")
+                 .OnDelete(DeleteBehavior.Cascade);
+
+                b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
+                 .WithMany("UpstreamForkFilterLists")
+                 .HasForeignKey("UpstreamFilterListId")
+                 .OnDelete(DeleteBehavior.Cascade);
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
+            {
+                b.HasOne("FilterLists.Data.Entities.FilterList", "MergeFilterList")
+                 .WithMany("MergeFilterLists")
+                 .HasForeignKey("MergeFilterListId")
+                 .OnDelete(DeleteBehavior.Cascade);
+
+                b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
+                 .WithMany("UpstreamMergeFilterLists")
+                 .HasForeignKey("UpstreamFilterListId")
+                 .OnDelete(DeleteBehavior.Cascade);
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b =>
+            {
+                b.HasOne("FilterLists.Data.Entities.Snapshot", "AddedBySnapshot")
+                 .WithMany("AddedSnapshotRules")
+                 .HasForeignKey("AddedBySnapshotId")
+                 .OnDelete(DeleteBehavior.Cascade);
+
+                b.HasOne("FilterLists.Data.Entities.Snapshot", "RemovedBySnapshot")
+                 .WithMany("RemovedSnapshotRules")
+                 .HasForeignKey("RemovedBySnapshotId")
+                 .OnDelete(DeleteBehavior.Cascade);
+
+                b.HasOne("FilterLists.Data.Entities.Rule", "Rule")
+                 .WithMany("SnapshotRules")
+                 .HasForeignKey("RuleId")
+                 .OnDelete(DeleteBehavior.Cascade);
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
+            {
+                b.HasOne("FilterLists.Data.Entities.Software", "Software")
+                 .WithMany("SoftwareSyntaxes")
+                 .HasForeignKey("SoftwareId")
+                 .OnDelete(DeleteBehavior.Cascade);
+
+                b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax")
+                 .WithMany("SoftwareSyntaxes")
+                 .HasForeignKey("SyntaxId")
+                 .OnDelete(DeleteBehavior.Cascade);
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
+            {
+                b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+                 .WithMany("Snapshots")
+                 .HasForeignKey("FilterListId")
+                 .OnDelete(DeleteBehavior.Cascade);
+            });
+#pragma warning restore 612, 618
+        }
+    }
+}

src/FilterLists.Api/Migrations/20180210204923_AddIsCompletedFlagToSnapshot.cs

@@ -0,0 +1,23 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+namespace FilterLists.Api.Migrations
+{
+    public partial class AddIsCompletedFlagToSnapshot : Migration
+    {
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.AddColumn<bool>(
+                "IsCompleted",
+                "snapshots",
+                nullable: false,
+                defaultValue: false);
+        }
+
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropColumn(
+                "IsCompleted",
+                "snapshots");
+        }
+    }
+}

src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs

@@ -349,6 +349,8 @@ protected override void BuildModel(ModelBuilder modelBuilder)
                  .HasColumnType("VARCHAR(3)")
                  .HasDefaultValueSql("NULL");
 
+                b.Property<bool>("IsCompleted");
+
                 b.HasKey("Id");
 
                 b.HasIndex("FilterListId");

src/FilterLists.Data/Entities/Snapshot.cs

@@ -8,6 +8,7 @@ public class Snapshot : BaseEntity
         public int FilterListId { get; set; }
         public FilterList FilterList { get; set; }
         public string HttpStatusCode { get; set; }
+        public bool IsCompleted { get; set; }
         public ICollection<SnapshotRule> AddedSnapshotRules { get; set; }
         public ICollection<SnapshotRule> RemovedSnapshotRules { get; set; }
     }

src/FilterLists.Services/SnapshotService/SnapshotDe.cs

@@ -32,6 +32,8 @@ public async Task SaveSnapshotAsync()
                 await SaveSnapshotInBatches(content);
                 await DedupSnapshotRules();
             }
+
+            await SetCompleted();
         }
 
         private async Task<string> CaptureSnapshot()
@@ -136,5 +138,11 @@ private void RemoveDuplicateSnapshotRules(IQueryable<SnapshotRule> existingSnaps
                 existingSnapshotRules.Any(y => y.Rule == x.Rule));
             dbContext.SnapshotRules.RemoveRange(duplicateSnapshotRules);
         }
+
+        private async Task SetCompleted()
+        {
+            snapshot.IsCompleted = true;
+            await dbContext.SaveChangesAsync();
+        }
     }
 }

src/FilterLists.Services/SnapshotService/SnapshotService.cs

@@ -16,14 +16,21 @@ public SnapshotService(FilterListsDbContext dbContext)
             this.dbContext = dbContext;
         }
 
-        public async Task CaptureSnapshotsAsync(int batchSize)
+        public async Task CaptureAsync(int batchSize)
         {
-            //TODO: rollback changes from recent snapshots if they were interrupted
+            RollbackIncomplete();
             var lists = await GetLeastRecentlyCapturedLists(batchSize);
             var snapshots = GetSnapshots(lists);
             await SaveSnapshots(snapshots);
         }
 
+        private void RollbackIncomplete()
+        {
+            var incompleteSnapshots = dbContext.Snapshots.Where(x => x.IsCompleted == false);
+            dbContext.Snapshots.RemoveRange(incompleteSnapshots);
+            //TODO: don't assume that SnapshotDe.DedupSnapshotRules() didn't partially complete
+        }
+
         private async Task<IEnumerable<FilterListViewUrlDto>> GetLeastRecentlyCapturedLists(int batchSize)
         {
             return await dbContext.FilterLists