From 345ff98d24dfb52a01f0a37645666b4600ff3a20 Mon Sep 17 00:00:00 2001 From: "Collin M. Barrett" Date: Sat, 10 Feb 2018 15:06:49 -0600 Subject: [PATCH] gracefully rollback interrupted snapshots currently assumes deduplication didn't partially complete (should be revisited at some point) --- src/FilterLists.Agent/Program.cs | 2 +- ...3_AddIsCompletedFlagToSnapshot.Designer.cs | 522 ++++++++++++++++++ ...0210204923_AddIsCompletedFlagToSnapshot.cs | 23 + .../FilterListsDbContextModelSnapshot.cs | 2 + src/FilterLists.Data/Entities/Snapshot.cs | 1 + .../SnapshotService/SnapshotDe.cs | 8 + .../SnapshotService/SnapshotService.cs | 11 +- 7 files changed, 566 insertions(+), 3 deletions(-) create mode 100644 src/FilterLists.Api/Migrations/20180210204923_AddIsCompletedFlagToSnapshot.Designer.cs create mode 100644 src/FilterLists.Api/Migrations/20180210204923_AddIsCompletedFlagToSnapshot.cs diff --git a/src/FilterLists.Agent/Program.cs b/src/FilterLists.Agent/Program.cs index c0bcdc1e6..cb346366f 100644 --- a/src/FilterLists.Agent/Program.cs +++ b/src/FilterLists.Agent/Program.cs @@ -60,7 +60,7 @@ private static void CaptureSnapshots(int batchSize) var snapshotService = serviceProvider.GetService(); Log("Capturing FilterList snapshots..."); - snapshotService.CaptureSnapshotsAsync(batchSize).Wait(); + snapshotService.CaptureAsync(batchSize).Wait(); Log("\nSnapshots captured."); telemetryClient.Flush(); } diff --git a/src/FilterLists.Api/Migrations/20180210204923_AddIsCompletedFlagToSnapshot.Designer.cs b/src/FilterLists.Api/Migrations/20180210204923_AddIsCompletedFlagToSnapshot.Designer.cs new file mode 100644 index 000000000..6a4347b96 --- /dev/null +++ b/src/FilterLists.Api/Migrations/20180210204923_AddIsCompletedFlagToSnapshot.Designer.cs @@ -0,0 +1,522 @@ +// + +using System; +using FilterLists.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; + +namespace FilterLists.Api.Migrations +{ + [DbContext(typeof(FilterListsDbContext))] + [Migration("20180210204923_AddIsCompletedFlagToSnapshot")] + partial class AddIsCompletedFlagToSnapshot + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn) + .HasAnnotation("ProductVersion", "2.0.1-rtm-125"); + + modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("Description") + .HasColumnType("TEXT"); + + b.Property("DescriptionSourceUrl") + .HasColumnType("TEXT"); + + b.Property("DiscontinuedDate"); + + b.Property("DonateUrl") + .HasColumnType("TEXT"); + + b.Property("EmailAddress") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.Property("ForumUrl") + .HasColumnType("TEXT"); + + b.Property("HomeUrl") + .HasColumnType("TEXT"); + + b.Property("IssuesUrl") + .HasColumnType("TEXT"); + + b.Property("LicenseId"); + + b.Property("ModifiedDateUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.Property("PolicyUrl") + .HasColumnType("TEXT"); + + b.Property("SubmissionUrl") + .HasColumnType("TEXT"); + + b.Property("SyntaxId"); + + b.Property("ViewUrl") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("LicenseId"); + + b.HasIndex("SyntaxId"); + + b.ToTable("filterlists"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b => + { + b.Property("FilterListId"); + + b.Property("LanguageId"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.HasKey("FilterListId", "LanguageId"); + + b.HasIndex("LanguageId"); + + b.ToTable("filterlists_languages"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b => + { + b.Property("FilterListId"); + + b.Property("MaintainerId"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.HasKey("FilterListId", "MaintainerId"); + + b.HasIndex("MaintainerId"); + + b.ToTable("filterlists_maintainers"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b => + { + b.Property("ForkFilterListId"); + + b.Property("UpstreamFilterListId"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.HasKey("ForkFilterListId", "UpstreamFilterListId"); + + b.HasIndex("UpstreamFilterListId"); + + b.ToTable("forks"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b => + { + b.Property("MergeFilterListId"); + + b.Property("UpstreamFilterListId"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.HasKey("MergeFilterListId", "UpstreamFilterListId"); + + b.HasIndex("UpstreamFilterListId"); + + b.ToTable("merges"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b => + { + b.Property("AddedBySnapshotId"); + + b.Property("RuleId"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("ModifiedDateUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP"); + + b.Property("RemovedBySnapshotId"); + + b.HasKey("AddedBySnapshotId", "RuleId"); + + b.HasIndex("RemovedBySnapshotId"); + + b.HasIndex("RuleId"); + + b.ToTable("snapshots_rules"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b => + { + b.Property("SyntaxId"); + + b.Property("SoftwareId"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.HasKey("SyntaxId", "SoftwareId"); + + b.HasIndex("SoftwareId"); + + b.ToTable("software_syntaxes"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Language", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("Iso6391") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(2)") + .HasDefaultValueSql("NULL"); + + b.Property("Iso6392") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); + + b.Property("Iso6392B") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); + + b.Property("Iso6392T") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); + + b.Property("Iso6393") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); + + b.Property("LocalName") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.Property("ModifiedDateUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP"); + + b.Property("Name") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.HasKey("Id"); + + b.ToTable("languages"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.License", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TINYINT UNSIGNED"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("DescriptionUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.Property("PermissiveAdaptation"); + + b.Property("PermissiveCommercial"); + + b.HasKey("Id"); + + b.ToTable("licenses"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Maintainer", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("EmailAddress") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.Property("HomeUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.Property("TwitterHandle") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.HasKey("Id"); + + b.ToTable("maintainers"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Rule", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INT UNSIGNED"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("Raw") + .IsRequired() + .HasColumnType("VARCHAR(8192)"); + + b.HasKey("Id"); + + b.ToTable("rules"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("MEDIUMINT UNSIGNED"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("FilterListId"); + + b.Property("HttpStatusCode") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); + + b.Property("IsCompleted"); + + b.HasKey("Id"); + + b.HasIndex("FilterListId"); + + b.ToTable("snapshots"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Software", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TINYINT UNSIGNED"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("DownloadUrl") + .HasColumnType("TEXT"); + + b.Property("HomeUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.HasKey("Id"); + + b.ToTable("software"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Syntax", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TINYINT UNSIGNED"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("DefinitionUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.HasKey("Id"); + + b.ToTable("syntaxes"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b => + { + b.HasOne("FilterLists.Data.Entities.License", "License") + .WithMany("FilterLists") + .HasForeignKey("LicenseId"); + + b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax") + .WithMany("FilterLists") + .HasForeignKey("SyntaxId"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("FilterListLanguages") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Language", "Language") + .WithMany("FilterListLanguages") + .HasForeignKey("LanguageId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("FilterListMaintainers") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Maintainer", "Maintainer") + .WithMany("FilterListMaintainers") + .HasForeignKey("MaintainerId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList") + .WithMany("ForkFilterLists") + .HasForeignKey("ForkFilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList") + .WithMany("UpstreamForkFilterLists") + .HasForeignKey("UpstreamFilterListId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "MergeFilterList") + .WithMany("MergeFilterLists") + .HasForeignKey("MergeFilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList") + .WithMany("UpstreamMergeFilterLists") + .HasForeignKey("UpstreamFilterListId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b => + { + b.HasOne("FilterLists.Data.Entities.Snapshot", "AddedBySnapshot") + .WithMany("AddedSnapshotRules") + .HasForeignKey("AddedBySnapshotId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Snapshot", "RemovedBySnapshot") + .WithMany("RemovedSnapshotRules") + .HasForeignKey("RemovedBySnapshotId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Rule", "Rule") + .WithMany("SnapshotRules") + .HasForeignKey("RuleId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b => + { + b.HasOne("FilterLists.Data.Entities.Software", "Software") + .WithMany("SoftwareSyntaxes") + .HasForeignKey("SoftwareId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax") + .WithMany("SoftwareSyntaxes") + .HasForeignKey("SyntaxId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("Snapshots") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + }); +#pragma warning restore 612, 618 + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Api/Migrations/20180210204923_AddIsCompletedFlagToSnapshot.cs b/src/FilterLists.Api/Migrations/20180210204923_AddIsCompletedFlagToSnapshot.cs new file mode 100644 index 000000000..029ca3dc1 --- /dev/null +++ b/src/FilterLists.Api/Migrations/20180210204923_AddIsCompletedFlagToSnapshot.cs @@ -0,0 +1,23 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +namespace FilterLists.Api.Migrations +{ + public partial class AddIsCompletedFlagToSnapshot : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + "IsCompleted", + "snapshots", + nullable: false, + defaultValue: false); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + "IsCompleted", + "snapshots"); + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs index 8fc1bd536..48108e14a 100644 --- a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs +++ b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs @@ -349,6 +349,8 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("VARCHAR(3)") .HasDefaultValueSql("NULL"); + b.Property("IsCompleted"); + b.HasKey("Id"); b.HasIndex("FilterListId"); diff --git a/src/FilterLists.Data/Entities/Snapshot.cs b/src/FilterLists.Data/Entities/Snapshot.cs index e1efba0f3..d55f09e16 100644 --- a/src/FilterLists.Data/Entities/Snapshot.cs +++ b/src/FilterLists.Data/Entities/Snapshot.cs @@ -8,6 +8,7 @@ public class Snapshot : BaseEntity public int FilterListId { get; set; } public FilterList FilterList { get; set; } public string HttpStatusCode { get; set; } + public bool IsCompleted { get; set; } public ICollection AddedSnapshotRules { get; set; } public ICollection RemovedSnapshotRules { get; set; } } diff --git a/src/FilterLists.Services/SnapshotService/SnapshotDe.cs b/src/FilterLists.Services/SnapshotService/SnapshotDe.cs index e5c2905ba..80353949f 100644 --- a/src/FilterLists.Services/SnapshotService/SnapshotDe.cs +++ b/src/FilterLists.Services/SnapshotService/SnapshotDe.cs @@ -32,6 +32,8 @@ public async Task SaveSnapshotAsync() await SaveSnapshotInBatches(content); await DedupSnapshotRules(); } + + await SetCompleted(); } private async Task CaptureSnapshot() @@ -136,5 +138,11 @@ private void RemoveDuplicateSnapshotRules(IQueryable existingSnaps existingSnapshotRules.Any(y => y.Rule == x.Rule)); dbContext.SnapshotRules.RemoveRange(duplicateSnapshotRules); } + + private async Task SetCompleted() + { + snapshot.IsCompleted = true; + await dbContext.SaveChangesAsync(); + } } } \ No newline at end of file diff --git a/src/FilterLists.Services/SnapshotService/SnapshotService.cs b/src/FilterLists.Services/SnapshotService/SnapshotService.cs index 768031648..174b88707 100644 --- a/src/FilterLists.Services/SnapshotService/SnapshotService.cs +++ b/src/FilterLists.Services/SnapshotService/SnapshotService.cs @@ -16,14 +16,21 @@ public SnapshotService(FilterListsDbContext dbContext) this.dbContext = dbContext; } - public async Task CaptureSnapshotsAsync(int batchSize) + public async Task CaptureAsync(int batchSize) { - //TODO: rollback changes from recent snapshots if they were interrupted + RollbackIncomplete(); var lists = await GetLeastRecentlyCapturedLists(batchSize); var snapshots = GetSnapshots(lists); await SaveSnapshots(snapshots); } + private void RollbackIncomplete() + { + var incompleteSnapshots = dbContext.Snapshots.Where(x => x.IsCompleted == false); + dbContext.Snapshots.RemoveRange(incompleteSnapshots); + //TODO: don't assume that SnapshotDe.DedupSnapshotRules() didn't partially complete + } + private async Task> GetLeastRecentlyCapturedLists(int batchSize) { return await dbContext.FilterLists