diff --git a/src/FilterLists.Api/Migrations/20180830220606_AddMd5Checksum.Designer.cs b/src/FilterLists.Api/Migrations/20180830220606_AddMd5Checksum.Designer.cs
new file mode 100644
index 000000000..be75a8b37
--- /dev/null
+++ b/src/FilterLists.Api/Migrations/20180830220606_AddMd5Checksum.Designer.cs
@@ -0,0 +1,643 @@
+//
+
+using System;
+using FilterLists.Data;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Metadata;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+namespace FilterLists.Api.Migrations
+{
+ [DbContext(typeof(FilterListsDbContext))]
+ [Migration("20180830220606_AddMd5Checksum")]
+ partial class AddMd5Checksum
+ {
+ protected override void BuildTargetModel(ModelBuilder modelBuilder)
+ {
+#pragma warning disable 612, 618
+ modelBuilder
+ .HasAnnotation("ProductVersion", "2.1.2-rtm-30932")
+ .HasAnnotation("Relational:MaxIdentifierLength", 64);
+
+ modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("SMALLINT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("ChatUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("Description")
+ .HasColumnType("TEXT");
+
+ b.Property("DescriptionSourceUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("DiscontinuedDate")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("DATE")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("DonateUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("EmailAddress")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("ForumUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("HomeUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("IssuesUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("LicenseId")
+ .ValueGeneratedOnAdd()
+ .HasDefaultValue((byte)5);
+
+ b.Property("ModifiedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("VARCHAR(126)");
+
+ b.Property("PolicyUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("PublishedDate")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("DATE")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("SubmissionUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("SyntaxId");
+
+ b.Property("ViewUrl")
+ .HasColumnType("TEXT");
+
+ b.HasKey("Id");
+
+ b.HasIndex("LicenseId");
+
+ b.HasIndex("SyntaxId");
+
+ b.ToTable("filterlists");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
+ {
+ b.Property("FilterListId");
+
+ b.Property("LanguageId");
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.HasKey("FilterListId", "LanguageId");
+
+ b.HasIndex("LanguageId");
+
+ b.ToTable("filterlists_languages");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
+ {
+ b.Property("FilterListId");
+
+ b.Property("MaintainerId");
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.HasKey("FilterListId", "MaintainerId");
+
+ b.HasIndex("MaintainerId");
+
+ b.ToTable("filterlists_maintainers");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListTag", b =>
+ {
+ b.Property("FilterListId");
+
+ b.Property("TagId");
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.HasKey("FilterListId", "TagId");
+
+ b.HasIndex("TagId");
+
+ b.ToTable("filterlists_tags");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
+ {
+ b.Property("ForkFilterListId");
+
+ b.Property("UpstreamFilterListId");
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.HasKey("ForkFilterListId", "UpstreamFilterListId");
+
+ b.HasIndex("UpstreamFilterListId");
+
+ b.ToTable("forks");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
+ {
+ b.Property("MergeFilterListId");
+
+ b.Property("UpstreamFilterListId");
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.HasKey("MergeFilterListId", "UpstreamFilterListId");
+
+ b.HasIndex("UpstreamFilterListId");
+
+ b.ToTable("merges");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b =>
+ {
+ b.Property("SnapshotId");
+
+ b.Property("RuleId");
+
+ b.HasKey("SnapshotId", "RuleId");
+
+ b.HasIndex("RuleId");
+
+ b.ToTable("snapshots_rules");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
+ {
+ b.Property("SyntaxId");
+
+ b.Property("SoftwareId");
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.HasKey("SyntaxId", "SoftwareId");
+
+ b.HasIndex("SoftwareId");
+
+ b.ToTable("software_syntaxes");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Language", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("SMALLINT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("Iso6391")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(2)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("Iso6392")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(3)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("Iso6392B")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(3)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("Iso6392T")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(3)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("Iso6393")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(3)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("LocalName")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("ModifiedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+ b.Property("Name")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
+
+ b.HasKey("Id");
+
+ b.ToTable("languages");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.License", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TINYINT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("DescriptionUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("ModifiedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("VARCHAR(126)");
+
+ b.Property("PermissiveAdaptation");
+
+ b.Property("PermissiveCommercial");
+
+ b.HasKey("Id");
+
+ b.ToTable("licenses");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Maintainer", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("SMALLINT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("EmailAddress")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("HomeUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("ModifiedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("VARCHAR(126)");
+
+ b.Property("TwitterHandle")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
+
+ b.HasKey("Id");
+
+ b.ToTable("maintainers");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Rule", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("INT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("Raw")
+ .IsRequired()
+ .HasColumnType("LONGTEXT");
+
+ b.HasKey("Id");
+
+ b.ToTable("rules");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("MEDIUMINT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("FilterListId");
+
+ b.Property("HttpStatusCode")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("SMALLINT")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("Md5Checksum")
+ .HasColumnType("BINARY(16)");
+
+ b.Property("ModifiedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+ b.Property("WasSuccessful");
+
+ b.Property("WaybackTimestamp")
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("WaybackUrl")
+ .HasColumnType("TEXT");
+
+ b.HasKey("Id");
+
+ b.HasIndex("FilterListId");
+
+ b.ToTable("snapshots");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Software", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TINYINT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("DownloadUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("HomeUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("ModifiedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("VARCHAR(126)");
+
+ b.HasKey("Id");
+
+ b.ToTable("software");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Syntax", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TINYINT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("DefinitionUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("ModifiedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("VARCHAR(126)");
+
+ b.HasKey("Id");
+
+ b.ToTable("syntaxes");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Tag", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TINYINT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("Description")
+ .HasColumnType("TEXT");
+
+ b.Property("ModifiedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("VARCHAR(126)");
+
+ b.HasKey("Id");
+
+ b.ToTable("tags");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.License", "License")
+ .WithMany("FilterLists")
+ .HasForeignKey("LicenseId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax")
+ .WithMany("FilterLists")
+ .HasForeignKey("SyntaxId");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+ .WithMany("FilterListLanguages")
+ .HasForeignKey("FilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Language", "Language")
+ .WithMany("FilterListLanguages")
+ .HasForeignKey("LanguageId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+ .WithMany("FilterListMaintainers")
+ .HasForeignKey("FilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Maintainer", "Maintainer")
+ .WithMany("FilterListMaintainers")
+ .HasForeignKey("MaintainerId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListTag", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+ .WithMany("FilterListTags")
+ .HasForeignKey("FilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Tag", "Tag")
+ .WithMany("FilterListTags")
+ .HasForeignKey("TagId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList")
+ .WithMany("ForkFilterLists")
+ .HasForeignKey("ForkFilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
+ .WithMany("UpstreamForkFilterLists")
+ .HasForeignKey("UpstreamFilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "MergeFilterList")
+ .WithMany("MergeFilterLists")
+ .HasForeignKey("MergeFilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
+ .WithMany("UpstreamMergeFilterLists")
+ .HasForeignKey("UpstreamFilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.Rule", "Rule")
+ .WithMany("SnapshotRules")
+ .HasForeignKey("RuleId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Snapshot", "Snapshot")
+ .WithMany("SnapshotRules")
+ .HasForeignKey("SnapshotId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.Software", "Software")
+ .WithMany("SoftwareSyntaxes")
+ .HasForeignKey("SoftwareId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax")
+ .WithMany("SoftwareSyntaxes")
+ .HasForeignKey("SyntaxId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+ .WithMany("Snapshots")
+ .HasForeignKey("FilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+#pragma warning restore 612, 618
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/FilterLists.Api/Migrations/20180830220606_AddMd5Checksum.cs b/src/FilterLists.Api/Migrations/20180830220606_AddMd5Checksum.cs
new file mode 100644
index 000000000..f5edafc81
--- /dev/null
+++ b/src/FilterLists.Api/Migrations/20180830220606_AddMd5Checksum.cs
@@ -0,0 +1,23 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+namespace FilterLists.Api.Migrations
+{
+ public partial class AddMd5Checksum : Migration
+ {
+ protected override void Up(MigrationBuilder migrationBuilder)
+ {
+ migrationBuilder.AddColumn(
+ "Md5Checksum",
+ "snapshots",
+ "BINARY(16)",
+ nullable: true);
+ }
+
+ protected override void Down(MigrationBuilder migrationBuilder)
+ {
+ migrationBuilder.DropColumn(
+ "Md5Checksum",
+ "snapshots");
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs
index 7263e6578..60824e763 100644
--- a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs
+++ b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs
@@ -405,6 +405,9 @@ protected override void BuildModel(ModelBuilder modelBuilder)
.HasColumnType("SMALLINT")
.HasDefaultValueSql("NULL");
+ b.Property("Md5Checksum")
+ .HasColumnType("BINARY(16)");
+
b.Property("ModifiedDateUtc")
.IsRequired()
.ValueGeneratedOnAddOrUpdate()
diff --git a/src/FilterLists.Data/Entities/Snapshot.cs b/src/FilterLists.Data/Entities/Snapshot.cs
index 7772e3689..ab0ebf1c6 100644
--- a/src/FilterLists.Data/Entities/Snapshot.cs
+++ b/src/FilterLists.Data/Entities/Snapshot.cs
@@ -9,8 +9,9 @@ public class Snapshot : BaseEntity
public uint FilterListId { get; set; }
public FilterList FilterList { get; set; }
public int? HttpStatusCode { get; set; }
- public bool WasSuccessful { get; set; }
+ public byte[] Md5Checksum { get; set; }
public ICollection SnapshotRules { get; set; }
+ public bool WasSuccessful { get; set; }
public DateTime? WaybackTimestamp { get; set; }
public string WaybackUrl { get; set; }
}
diff --git a/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs
index e44e38c85..34ae29b27 100644
--- a/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs
+++ b/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs
@@ -15,6 +15,8 @@ public override void Configure(EntityTypeBuilder entityTypeBuilder)
entityTypeBuilder.Property(x => x.HttpStatusCode)
.HasColumnType("SMALLINT")
.HasDefaultValueSql("NULL");
+ entityTypeBuilder.Property(x => x.Md5Checksum)
+ .HasColumnType("BINARY(16)");
entityTypeBuilder.Property(x => x.WaybackTimestamp)
.HasColumnType("TIMESTAMP");
entityTypeBuilder.Property(x => x.WaybackUrl)
diff --git a/src/FilterLists.Services/Snapshot/Snapshot.cs b/src/FilterLists.Services/Snapshot/Snapshot.cs
index f2b59eab9..327f76726 100644
--- a/src/FilterLists.Services/Snapshot/Snapshot.cs
+++ b/src/FilterLists.Services/Snapshot/Snapshot.cs
@@ -4,6 +4,7 @@
using System.Linq;
using System.Net;
using System.Net.Http;
+using System.Security.Cryptography;
using System.Threading;
using System.Threading.Tasks;
using FilterLists.Data;
@@ -12,6 +13,7 @@
using FilterLists.Services.Snapshot.Models;
using JetBrains.Annotations;
using Microsoft.ApplicationInsights;
+using Microsoft.EntityFrameworkCore;
using MoreLinq;
using SharpCompress.Archives.SevenZip;
@@ -112,27 +114,52 @@ private async Task GetLines()
lines = new HashSet();
using (var stream = await response.Content.ReadAsStreamAsync())
{
- if (ListUrl.EndsWith(".7z"))
- await GetLinesFrom7Zip(stream);
- else
- await GetLinesFromStream(stream);
+ using (var memoryStream = new MemoryStream())
+ {
+ await stream.CopyToAsync(memoryStream);
+ await SaveChecksum(memoryStream);
+ if (await IsNewChecksum())
+ {
+ memoryStream.Position = 0;
+ if (ListUrl.EndsWith(".7z"))
+ await GetLinesFrom7Zip(memoryStream);
+ else
+ await GetLinesFromStream(memoryStream);
+ }
+ }
}
}
}
+ private async Task SaveChecksum(Stream stream)
+ {
+ using (var md5 = MD5.Create())
+ {
+ SnapEntity.Md5Checksum = md5.ComputeHash(stream);
+ }
+
+ await dbContext.SaveChangesAsync();
+ }
+
+ private async Task IsNewChecksum() =>
+ !SnapEntity.Md5Checksum.SequenceEqual(await GetPreviousChecksum() ?? Array.Empty());
+
+ private async Task GetPreviousChecksum() =>
+ await dbContext.Snapshots
+ .Where(s => s.WasSuccessful && s.FilterListId == List.Id)
+ .OrderByDescending(s => s.CreatedDateUtc)
+ .Select(s => s.Md5Checksum)
+ .FirstOrDefaultAsync();
+
private async Task GetLinesFrom7Zip(Stream stream)
{
- using (var memoryStream = new MemoryStream())
+ using (var archive = SevenZipArchive.Open(stream))
{
- stream.CopyTo(memoryStream);
- using (var archive = SevenZipArchive.Open(memoryStream))
- {
- foreach (var entry in archive.Entries.Where(entry => !entry.IsDirectory))
- using (var entryMemoryStream = entry.OpenEntryStream())
- {
- await GetLinesFromStream(entryMemoryStream);
- }
- }
+ foreach (var entry in archive.Entries.Where(entry => !entry.IsDirectory))
+ using (var entryMemoryStream = entry.OpenEntryStream())
+ {
+ await GetLinesFromStream(entryMemoryStream);
+ }
}
}