skip storing snapshot on matching checksum

closes #349

src/FilterLists.Api/Migrations/20180830220606_AddMd5Checksum.Designer.cs

@@ -0,0 +1,643 @@
+// <auto-generated />
+
+using System;
+using FilterLists.Data;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Metadata;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+namespace FilterLists.Api.Migrations
+{
+    [DbContext(typeof(FilterListsDbContext))]
+    [Migration("20180830220606_AddMd5Checksum")]
+    partial class AddMd5Checksum
+    {
+        protected override void BuildTargetModel(ModelBuilder modelBuilder)
+        {
+#pragma warning disable 612, 618
+            modelBuilder
+                .HasAnnotation("ProductVersion", "2.1.2-rtm-30932")
+                .HasAnnotation("Relational:MaxIdentifierLength", 64);
+
+            modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
+            {
+                b.Property<ushort>("Id")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("SMALLINT UNSIGNED")
+                 .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+                b.Property<string>("ChatUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<DateTime?>("CreatedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp()");
+
+                b.Property<string>("Description")
+                 .HasColumnType("TEXT");
+
+                b.Property<string>("DescriptionSourceUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<DateTime?>("DiscontinuedDate")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("DATE")
+                 .HasDefaultValueSql("NULL");
+
+                b.Property<string>("DonateUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<string>("EmailAddress")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("VARCHAR(126)")
+                 .HasDefaultValueSql("NULL");
+
+                b.Property<string>("ForumUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<string>("HomeUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<string>("IssuesUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<byte>("LicenseId")
+                 .ValueGeneratedOnAdd()
+                 .HasDefaultValue((byte)5);
+
+                b.Property<DateTime?>("ModifiedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAddOrUpdate()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+                b.Property<string>("Name")
+                 .IsRequired()
+                 .HasColumnType("VARCHAR(126)");
+
+                b.Property<string>("PolicyUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<DateTime?>("PublishedDate")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("DATE")
+                 .HasDefaultValueSql("NULL");
+
+                b.Property<string>("SubmissionUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<byte?>("SyntaxId");
+
+                b.Property<string>("ViewUrl")
+                 .HasColumnType("TEXT");
+
+                b.HasKey("Id");
+
+                b.HasIndex("LicenseId");
+
+                b.HasIndex("SyntaxId");
+
+                b.ToTable("filterlists");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
+            {
+                b.Property<ushort>("FilterListId");
+
+                b.Property<ushort>("LanguageId");
+
+                b.Property<DateTime?>("CreatedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp()");
+
+                b.HasKey("FilterListId", "LanguageId");
+
+                b.HasIndex("LanguageId");
+
+                b.ToTable("filterlists_languages");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
+            {
+                b.Property<ushort>("FilterListId");
+
+                b.Property<ushort>("MaintainerId");
+
+                b.Property<DateTime?>("CreatedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp()");
+
+                b.HasKey("FilterListId", "MaintainerId");
+
+                b.HasIndex("MaintainerId");
+
+                b.ToTable("filterlists_maintainers");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListTag", b =>
+            {
+                b.Property<ushort>("FilterListId");
+
+                b.Property<byte>("TagId");
+
+                b.Property<DateTime?>("CreatedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp()");
+
+                b.HasKey("FilterListId", "TagId");
+
+                b.HasIndex("TagId");
+
+                b.ToTable("filterlists_tags");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
+            {
+                b.Property<ushort>("ForkFilterListId");
+
+                b.Property<ushort>("UpstreamFilterListId");
+
+                b.Property<DateTime?>("CreatedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp()");
+
+                b.HasKey("ForkFilterListId", "UpstreamFilterListId");
+
+                b.HasIndex("UpstreamFilterListId");
+
+                b.ToTable("forks");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
+            {
+                b.Property<ushort>("MergeFilterListId");
+
+                b.Property<ushort>("UpstreamFilterListId");
+
+                b.Property<DateTime?>("CreatedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp()");
+
+                b.HasKey("MergeFilterListId", "UpstreamFilterListId");
+
+                b.HasIndex("UpstreamFilterListId");
+
+                b.ToTable("merges");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b =>
+            {
+                b.Property<uint>("SnapshotId");
+
+                b.Property<uint>("RuleId");
+
+                b.HasKey("SnapshotId", "RuleId");
+
+                b.HasIndex("RuleId");
+
+                b.ToTable("snapshots_rules");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
+            {
+                b.Property<byte>("SyntaxId");
+
+                b.Property<byte>("SoftwareId");
+
+                b.Property<DateTime?>("CreatedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp()");
+
+                b.HasKey("SyntaxId", "SoftwareId");
+
+                b.HasIndex("SoftwareId");
+
+                b.ToTable("software_syntaxes");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Language", b =>
+            {
+                b.Property<ushort>("Id")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("SMALLINT UNSIGNED")
+                 .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+                b.Property<DateTime?>("CreatedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp()");
+
+                b.Property<string>("Iso6391")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("VARCHAR(2)")
+                 .HasDefaultValueSql("NULL");
+
+                b.Property<string>("Iso6392")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("VARCHAR(3)")
+                 .HasDefaultValueSql("NULL");
+
+                b.Property<string>("Iso6392B")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("VARCHAR(3)")
+                 .HasDefaultValueSql("NULL");
+
+                b.Property<string>("Iso6392T")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("VARCHAR(3)")
+                 .HasDefaultValueSql("NULL");
+
+                b.Property<string>("Iso6393")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("VARCHAR(3)")
+                 .HasDefaultValueSql("NULL");
+
+                b.Property<string>("LocalName")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("VARCHAR(126)")
+                 .HasDefaultValueSql("NULL");
+
+                b.Property<DateTime?>("ModifiedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAddOrUpdate()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+                b.Property<string>("Name")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("VARCHAR(126)")
+                 .HasDefaultValueSql("NULL");
+
+                b.HasKey("Id");
+
+                b.ToTable("languages");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.License", b =>
+            {
+                b.Property<byte>("Id")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TINYINT UNSIGNED")
+                 .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+                b.Property<DateTime?>("CreatedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp()");
+
+                b.Property<string>("DescriptionUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<DateTime?>("ModifiedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAddOrUpdate()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+                b.Property<string>("Name")
+                 .IsRequired()
+                 .HasColumnType("VARCHAR(126)");
+
+                b.Property<bool>("PermissiveAdaptation");
+
+                b.Property<bool>("PermissiveCommercial");
+
+                b.HasKey("Id");
+
+                b.ToTable("licenses");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Maintainer", b =>
+            {
+                b.Property<ushort>("Id")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("SMALLINT UNSIGNED")
+                 .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+                b.Property<DateTime?>("CreatedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp()");
+
+                b.Property<string>("EmailAddress")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("VARCHAR(126)")
+                 .HasDefaultValueSql("NULL");
+
+                b.Property<string>("HomeUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<DateTime?>("ModifiedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAddOrUpdate()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+                b.Property<string>("Name")
+                 .IsRequired()
+                 .HasColumnType("VARCHAR(126)");
+
+                b.Property<string>("TwitterHandle")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("VARCHAR(126)")
+                 .HasDefaultValueSql("NULL");
+
+                b.HasKey("Id");
+
+                b.ToTable("maintainers");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Rule", b =>
+            {
+                b.Property<uint>("Id")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("INT UNSIGNED")
+                 .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+                b.Property<DateTime?>("CreatedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp()");
+
+                b.Property<string>("Raw")
+                 .IsRequired()
+                 .HasColumnType("LONGTEXT");
+
+                b.HasKey("Id");
+
+                b.ToTable("rules");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
+            {
+                b.Property<uint>("Id")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("MEDIUMINT UNSIGNED")
+                 .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+                b.Property<DateTime?>("CreatedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp()");
+
+                b.Property<ushort>("FilterListId");
+
+                b.Property<short?>("HttpStatusCode")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("SMALLINT")
+                 .HasDefaultValueSql("NULL");
+
+                b.Property<byte[]>("Md5Checksum")
+                 .HasColumnType("BINARY(16)");
+
+                b.Property<DateTime?>("ModifiedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAddOrUpdate()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+                b.Property<bool>("WasSuccessful");
+
+                b.Property<DateTime?>("WaybackTimestamp")
+                 .HasColumnType("TIMESTAMP");
+
+                b.Property<string>("WaybackUrl")
+                 .HasColumnType("TEXT");
+
+                b.HasKey("Id");
+
+                b.HasIndex("FilterListId");
+
+                b.ToTable("snapshots");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Software", b =>
+            {
+                b.Property<byte>("Id")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TINYINT UNSIGNED")
+                 .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+                b.Property<DateTime?>("CreatedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp()");
+
+                b.Property<string>("DownloadUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<string>("HomeUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<DateTime?>("ModifiedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAddOrUpdate()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+                b.Property<string>("Name")
+                 .IsRequired()
+                 .HasColumnType("VARCHAR(126)");
+
+                b.HasKey("Id");
+
+                b.ToTable("software");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Syntax", b =>
+            {
+                b.Property<byte>("Id")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TINYINT UNSIGNED")
+                 .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+                b.Property<DateTime?>("CreatedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp()");
+
+                b.Property<string>("DefinitionUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<DateTime?>("ModifiedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAddOrUpdate()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+                b.Property<string>("Name")
+                 .IsRequired()
+                 .HasColumnType("VARCHAR(126)");
+
+                b.HasKey("Id");
+
+                b.ToTable("syntaxes");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Tag", b =>
+            {
+                b.Property<byte>("Id")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TINYINT UNSIGNED")
+                 .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+                b.Property<DateTime?>("CreatedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp()");
+
+                b.Property<string>("Description")
+                 .HasColumnType("TEXT");
+
+                b.Property<DateTime?>("ModifiedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAddOrUpdate()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+                b.Property<string>("Name")
+                 .IsRequired()
+                 .HasColumnType("VARCHAR(126)");
+
+                b.HasKey("Id");
+
+                b.ToTable("tags");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
+            {
+                b.HasOne("FilterLists.Data.Entities.License", "License")
+                 .WithMany("FilterLists")
+                 .HasForeignKey("LicenseId")
+                 .OnDelete(DeleteBehavior.Cascade);
+
+                b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax")
+                 .WithMany("FilterLists")
+                 .HasForeignKey("SyntaxId");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
+            {
+                b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+                 .WithMany("FilterListLanguages")
+                 .HasForeignKey("FilterListId")
+                 .OnDelete(DeleteBehavior.Cascade);
+
+                b.HasOne("FilterLists.Data.Entities.Language", "Language")
+                 .WithMany("FilterListLanguages")
+                 .HasForeignKey("LanguageId")
+                 .OnDelete(DeleteBehavior.Cascade);
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
+            {
+                b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+                 .WithMany("FilterListMaintainers")
+                 .HasForeignKey("FilterListId")
+                 .OnDelete(DeleteBehavior.Cascade);
+
+                b.HasOne("FilterLists.Data.Entities.Maintainer", "Maintainer")
+                 .WithMany("FilterListMaintainers")
+                 .HasForeignKey("MaintainerId")
+                 .OnDelete(DeleteBehavior.Cascade);
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListTag", b =>
+            {
+                b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+                 .WithMany("FilterListTags")
+                 .HasForeignKey("FilterListId")
+                 .OnDelete(DeleteBehavior.Cascade);
+
+                b.HasOne("FilterLists.Data.Entities.Tag", "Tag")
+                 .WithMany("FilterListTags")
+                 .HasForeignKey("TagId")
+                 .OnDelete(DeleteBehavior.Cascade);
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
+            {
+                b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList")
+                 .WithMany("ForkFilterLists")
+                 .HasForeignKey("ForkFilterListId")
+                 .OnDelete(DeleteBehavior.Cascade);
+
+                b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
+                 .WithMany("UpstreamForkFilterLists")
+                 .HasForeignKey("UpstreamFilterListId")
+                 .OnDelete(DeleteBehavior.Cascade);
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
+            {
+                b.HasOne("FilterLists.Data.Entities.FilterList", "MergeFilterList")
+                 .WithMany("MergeFilterLists")
+                 .HasForeignKey("MergeFilterListId")
+                 .OnDelete(DeleteBehavior.Cascade);
+
+                b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
+                 .WithMany("UpstreamMergeFilterLists")
+                 .HasForeignKey("UpstreamFilterListId")
+                 .OnDelete(DeleteBehavior.Cascade);
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b =>
+            {
+                b.HasOne("FilterLists.Data.Entities.Rule", "Rule")
+                 .WithMany("SnapshotRules")
+                 .HasForeignKey("RuleId")
+                 .OnDelete(DeleteBehavior.Cascade);
+
+                b.HasOne("FilterLists.Data.Entities.Snapshot", "Snapshot")
+                 .WithMany("SnapshotRules")
+                 .HasForeignKey("SnapshotId")
+                 .OnDelete(DeleteBehavior.Cascade);
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
+            {
+                b.HasOne("FilterLists.Data.Entities.Software", "Software")
+                 .WithMany("SoftwareSyntaxes")
+                 .HasForeignKey("SoftwareId")
+                 .OnDelete(DeleteBehavior.Cascade);
+
+                b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax")
+                 .WithMany("SoftwareSyntaxes")
+                 .HasForeignKey("SyntaxId")
+                 .OnDelete(DeleteBehavior.Cascade);
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
+            {
+                b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+                 .WithMany("Snapshots")
+                 .HasForeignKey("FilterListId")
+                 .OnDelete(DeleteBehavior.Cascade);
+            });
+#pragma warning restore 612, 618
+        }
+    }
+}

src/FilterLists.Api/Migrations/20180830220606_AddMd5Checksum.cs

@@ -0,0 +1,23 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+namespace FilterLists.Api.Migrations
+{
+    public partial class AddMd5Checksum : Migration
+    {
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.AddColumn<byte[]>(
+                "Md5Checksum",
+                "snapshots",
+                "BINARY(16)",
+                nullable: true);
+        }
+
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropColumn(
+                "Md5Checksum",
+                "snapshots");
+        }
+    }
+}

src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs

@@ -405,6 +405,9 @@ protected override void BuildModel(ModelBuilder modelBuilder)
                  .HasColumnType("SMALLINT")
                  .HasDefaultValueSql("NULL");
 
+                b.Property<byte[]>("Md5Checksum")
+                 .HasColumnType("BINARY(16)");
+
                 b.Property<DateTime?>("ModifiedDateUtc")
                  .IsRequired()
                  .ValueGeneratedOnAddOrUpdate()

src/FilterLists.Data/Entities/Snapshot.cs

@@ -9,8 +9,9 @@ public class Snapshot : BaseEntity
         public uint FilterListId { get; set; }
         public FilterList FilterList { get; set; }
         public int? HttpStatusCode { get; set; }
-        public bool WasSuccessful { get; set; }
+        public byte[] Md5Checksum { get; set; }
         public ICollection<SnapshotRule> SnapshotRules { get; set; }
+        public bool WasSuccessful { get; set; }
         public DateTime? WaybackTimestamp { get; set; }
         public string WaybackUrl { get; set; }
     }

src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs

@@ -15,6 +15,8 @@ public override void Configure(EntityTypeBuilder<Snapshot> entityTypeBuilder)
             entityTypeBuilder.Property(x => x.HttpStatusCode)
                              .HasColumnType("SMALLINT")
                              .HasDefaultValueSql("NULL");
+            entityTypeBuilder.Property(x => x.Md5Checksum)
+                             .HasColumnType("BINARY(16)");
             entityTypeBuilder.Property(x => x.WaybackTimestamp)
                              .HasColumnType("TIMESTAMP");
             entityTypeBuilder.Property(x => x.WaybackUrl)

src/FilterLists.Services/Snapshot/Snapshot.cs

@@ -4,6 +4,7 @@
 using System.Linq;
 using System.Net;
 using System.Net.Http;
+using System.Security.Cryptography;
 using System.Threading;
 using System.Threading.Tasks;
 using FilterLists.Data;
@@ -12,6 +13,7 @@
 using FilterLists.Services.Snapshot.Models;
 using JetBrains.Annotations;
 using Microsoft.ApplicationInsights;
+using Microsoft.EntityFrameworkCore;
 using MoreLinq;
 using SharpCompress.Archives.SevenZip;
 
@@ -112,27 +114,52 @@ private async Task GetLines()
                 lines = new HashSet<string>();
                 using (var stream = await response.Content.ReadAsStreamAsync())
                 {
-                    if (ListUrl.EndsWith(".7z"))
-                        await GetLinesFrom7Zip(stream);
-                    else
-                        await GetLinesFromStream(stream);
+                    using (var memoryStream = new MemoryStream())
+                    {
+                        await stream.CopyToAsync(memoryStream);
+                        await SaveChecksum(memoryStream);
+                        if (await IsNewChecksum())
+                        {
+                            memoryStream.Position = 0;
+                            if (ListUrl.EndsWith(".7z"))
+                                await GetLinesFrom7Zip(memoryStream);
+                            else
+                                await GetLinesFromStream(memoryStream);
+                        }
+                    }
                 }
             }
         }
 
+        private async Task SaveChecksum(Stream stream)
+        {
+            using (var md5 = MD5.Create())
+            {
+                SnapEntity.Md5Checksum = md5.ComputeHash(stream);
+            }
+
+            await dbContext.SaveChangesAsync();
+        }
+
+        private async Task<bool> IsNewChecksum() =>
+            !SnapEntity.Md5Checksum.SequenceEqual(await GetPreviousChecksum() ?? Array.Empty<byte>());
+
+        private async Task<byte[]> GetPreviousChecksum() =>
+            await dbContext.Snapshots
+                           .Where(s => s.WasSuccessful && s.FilterListId == List.Id)
+                           .OrderByDescending(s => s.CreatedDateUtc)
+                           .Select(s => s.Md5Checksum)
+                           .FirstOrDefaultAsync();
+
         private async Task GetLinesFrom7Zip(Stream stream)
         {
-            using (var memoryStream = new MemoryStream())
+            using (var archive = SevenZipArchive.Open(stream))
             {
-                stream.CopyTo(memoryStream);
-                using (var archive = SevenZipArchive.Open(memoryStream))
-                {
-                    foreach (var entry in archive.Entries.Where(entry => !entry.IsDirectory))
-                        using (var entryMemoryStream = entry.OpenEntryStream())
-                        {
-                            await GetLinesFromStream(entryMemoryStream);
-                        }
-                }
+                foreach (var entry in archive.Entries.Where(entry => !entry.IsDirectory))
+                    using (var entryMemoryStream = entry.OpenEntryStream())
+                    {
+                        await GetLinesFromStream(entryMemoryStream);
+                    }
             }
         }