From 3f80da819eca181f24ab9822615be47f6e39c96f Mon Sep 17 00:00:00 2001 From: Collin Barrett Date: Wed, 15 Aug 2018 18:36:18 -0500 Subject: [PATCH] tidy up ef core transaction refactor ref #343 --- cantSnapshot.sql | 5 + src/FilterLists.Agent/Program.cs | 2 +- ...20180815233049_AddCantSnapshot.Designer.cs | 594 ++++++++++++++++++ .../20180815233049_AddCantSnapshot.cs | 33 + .../FilterListsDbContextModelSnapshot.cs | 6 +- src/FilterLists.Data/Entities/FilterList.cs | 1 + src/FilterLists.Data/Entities/Snapshot.cs | 2 +- .../FilterListTypeConfiguration.cs | 4 +- .../ListDetailsDtoMappingProfile.cs | 6 +- .../ListSummaryDtoMappingProfile.cs | 2 +- .../Snapshot/SnapshotDe.cs | 14 +- .../Snapshot/SnapshotService.cs | 18 +- 12 files changed, 654 insertions(+), 33 deletions(-) create mode 100644 cantSnapshot.sql create mode 100644 src/FilterLists.Api/Migrations/20180815233049_AddCantSnapshot.Designer.cs create mode 100644 src/FilterLists.Api/Migrations/20180815233049_AddCantSnapshot.cs diff --git a/cantSnapshot.sql b/cantSnapshot.sql new file mode 100644 index 000000000..cecf99b07 --- /dev/null +++ b/cantSnapshot.sql @@ -0,0 +1,5 @@ +--TODO: https://github.com/collinbarrett/FilterLists/issues/200 +--TODO: https://github.com/collinbarrett/FilterLists/issues/201 + +UPDATE filterlists SET CantSnapshot = 1 +WHERE id IN (48, 149, 173, 185, 186, 187, 188, 189, 352) \ No newline at end of file diff --git a/src/FilterLists.Agent/Program.cs b/src/FilterLists.Agent/Program.cs index 7fc5cc265..ebf6f79a3 100644 --- a/src/FilterLists.Agent/Program.cs +++ b/src/FilterLists.Agent/Program.cs @@ -49,7 +49,7 @@ private static void CaptureSnapshots(int batchSize) var snapshotService = serviceProvider.GetService(); Log("Capturing FilterList snapshots..."); snapshotService.CaptureAsync(batchSize).Wait(); - Log(Environment.NewLine + "Snapshots captured."); + Log("Snapshots captured."); telemetryClient.Flush(); } diff --git a/src/FilterLists.Api/Migrations/20180815233049_AddCantSnapshot.Designer.cs b/src/FilterLists.Api/Migrations/20180815233049_AddCantSnapshot.Designer.cs new file mode 100644 index 000000000..2faf03a7f --- /dev/null +++ b/src/FilterLists.Api/Migrations/20180815233049_AddCantSnapshot.Designer.cs @@ -0,0 +1,594 @@ +// + +using System; +using FilterLists.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; + +namespace FilterLists.Api.Migrations +{ + [DbContext(typeof(FilterListsDbContext))] + [Migration("20180815233049_AddCantSnapshot")] + partial class AddCantSnapshot + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "2.1.1-rtm-30846") + .HasAnnotation("Relational:MaxIdentifierLength", 64); + + modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CantSnapshot"); + + b.Property("ChatUrl") + .HasColumnType("TEXT"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("Description") + .HasColumnType("TEXT"); + + b.Property("DescriptionSourceUrl") + .HasColumnType("TEXT"); + + b.Property("DiscontinuedDate") + .ValueGeneratedOnAdd() + .HasColumnType("DATE") + .HasDefaultValueSql("NULL"); + + b.Property("DonateUrl") + .HasColumnType("TEXT"); + + b.Property("EmailAddress") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.Property("ForumUrl") + .HasColumnType("TEXT"); + + b.Property("HomeUrl") + .HasColumnType("TEXT"); + + b.Property("IssuesUrl") + .HasColumnType("TEXT"); + + b.Property("LicenseId") + .ValueGeneratedOnAdd() + .HasDefaultValue((byte)5); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.Property("PolicyUrl") + .HasColumnType("TEXT"); + + b.Property("PublishedDate") + .ValueGeneratedOnAdd() + .HasColumnType("DATE") + .HasDefaultValueSql("NULL"); + + b.Property("SubmissionUrl") + .HasColumnType("TEXT"); + + b.Property("SyntaxId"); + + b.Property("ViewUrl") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("LicenseId"); + + b.HasIndex("SyntaxId"); + + b.ToTable("filterlists"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b => + { + b.Property("FilterListId"); + + b.Property("LanguageId"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.HasKey("FilterListId", "LanguageId"); + + b.HasIndex("LanguageId"); + + b.ToTable("filterlists_languages"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b => + { + b.Property("FilterListId"); + + b.Property("MaintainerId"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.HasKey("FilterListId", "MaintainerId"); + + b.HasIndex("MaintainerId"); + + b.ToTable("filterlists_maintainers"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b => + { + b.Property("ForkFilterListId"); + + b.Property("UpstreamFilterListId"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.HasKey("ForkFilterListId", "UpstreamFilterListId"); + + b.HasIndex("UpstreamFilterListId"); + + b.ToTable("forks"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b => + { + b.Property("MergeFilterListId"); + + b.Property("UpstreamFilterListId"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.HasKey("MergeFilterListId", "UpstreamFilterListId"); + + b.HasIndex("UpstreamFilterListId"); + + b.ToTable("merges"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b => + { + b.Property("AddedBySnapshotId"); + + b.Property("RuleId"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("RemovedBySnapshotId"); + + b.HasKey("AddedBySnapshotId", "RuleId"); + + b.HasIndex("RemovedBySnapshotId"); + + b.HasIndex("RuleId"); + + b.ToTable("snapshots_rules"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b => + { + b.Property("SyntaxId"); + + b.Property("SoftwareId"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.HasKey("SyntaxId", "SoftwareId"); + + b.HasIndex("SoftwareId"); + + b.ToTable("software_syntaxes"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Language", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("Iso6391") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(2)") + .HasDefaultValueSql("NULL"); + + b.Property("Iso6392") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); + + b.Property("Iso6392B") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); + + b.Property("Iso6392T") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); + + b.Property("Iso6393") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); + + b.Property("LocalName") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("Name") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.HasKey("Id"); + + b.ToTable("languages"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.License", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TINYINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("DescriptionUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.Property("PermissiveAdaptation"); + + b.Property("PermissiveCommercial"); + + b.HasKey("Id"); + + b.ToTable("licenses"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Maintainer", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("EmailAddress") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.Property("HomeUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.Property("TwitterHandle") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.HasKey("Id"); + + b.ToTable("maintainers"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Rule", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("Raw") + .IsRequired() + .HasColumnType("VARCHAR(8192)"); + + b.HasKey("Id"); + + b.ToTable("rules"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("MEDIUMINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("FilterListId"); + + b.Property("HttpStatusCode") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("WasSuccessful"); + + b.HasKey("Id"); + + b.HasIndex("FilterListId"); + + b.ToTable("snapshots"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Software", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TINYINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("DownloadUrl") + .HasColumnType("TEXT"); + + b.Property("HomeUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.HasKey("Id"); + + b.ToTable("software"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Syntax", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TINYINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("DefinitionUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.HasKey("Id"); + + b.ToTable("syntaxes"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b => + { + b.HasOne("FilterLists.Data.Entities.License", "License") + .WithMany("FilterLists") + .HasForeignKey("LicenseId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax") + .WithMany("FilterLists") + .HasForeignKey("SyntaxId"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("FilterListLanguages") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Language", "Language") + .WithMany("FilterListLanguages") + .HasForeignKey("LanguageId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("FilterListMaintainers") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Maintainer", "Maintainer") + .WithMany("FilterListMaintainers") + .HasForeignKey("MaintainerId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList") + .WithMany("ForkFilterLists") + .HasForeignKey("ForkFilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList") + .WithMany("UpstreamForkFilterLists") + .HasForeignKey("UpstreamFilterListId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "MergeFilterList") + .WithMany("MergeFilterLists") + .HasForeignKey("MergeFilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList") + .WithMany("UpstreamMergeFilterLists") + .HasForeignKey("UpstreamFilterListId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b => + { + b.HasOne("FilterLists.Data.Entities.Snapshot", "AddedBySnapshot") + .WithMany("AddedSnapshotRules") + .HasForeignKey("AddedBySnapshotId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Snapshot", "RemovedBySnapshot") + .WithMany("RemovedSnapshotRules") + .HasForeignKey("RemovedBySnapshotId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Rule", "Rule") + .WithMany("SnapshotRules") + .HasForeignKey("RuleId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b => + { + b.HasOne("FilterLists.Data.Entities.Software", "Software") + .WithMany("SoftwareSyntaxes") + .HasForeignKey("SoftwareId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax") + .WithMany("SoftwareSyntaxes") + .HasForeignKey("SyntaxId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("Snapshots") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + }); +#pragma warning restore 612, 618 + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Api/Migrations/20180815233049_AddCantSnapshot.cs b/src/FilterLists.Api/Migrations/20180815233049_AddCantSnapshot.cs new file mode 100644 index 000000000..63a17f1bd --- /dev/null +++ b/src/FilterLists.Api/Migrations/20180815233049_AddCantSnapshot.cs @@ -0,0 +1,33 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +namespace FilterLists.Api.Migrations +{ + public partial class AddCantSnapshot : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.RenameColumn( + "IsCompleted", + "snapshots", + "WasSuccessful"); + + migrationBuilder.AddColumn( + "CantSnapshot", + "filterlists", + nullable: false, + defaultValue: false); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + "CantSnapshot", + "filterlists"); + + migrationBuilder.RenameColumn( + "WasSuccessful", + "snapshots", + "IsCompleted"); + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs index 18ff9a2e3..944ec6e94 100644 --- a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs +++ b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs @@ -25,6 +25,8 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("SMALLINT UNSIGNED") .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + b.Property("CantSnapshot"); + b.Property("ChatUrl") .HasColumnType("TEXT"); @@ -402,14 +404,14 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("VARCHAR(3)") .HasDefaultValueSql("NULL"); - b.Property("IsCompleted"); - b.Property("ModifiedDateUtc") .IsRequired() .ValueGeneratedOnAddOrUpdate() .HasColumnType("TIMESTAMP") .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + b.Property("WasSuccessful"); + b.HasKey("Id"); b.HasIndex("FilterListId"); diff --git a/src/FilterLists.Data/Entities/FilterList.cs b/src/FilterLists.Data/Entities/FilterList.cs index 405bbc5be..56a4ef296 100644 --- a/src/FilterLists.Data/Entities/FilterList.cs +++ b/src/FilterLists.Data/Entities/FilterList.cs @@ -8,6 +8,7 @@ namespace FilterLists.Data.Entities [UsedImplicitly] public class FilterList : BaseEntity { + public bool CantSnapshot { get; set; } public string ChatUrl { get; set; } public string Description { get; set; } public string DescriptionSourceUrl { get; set; } diff --git a/src/FilterLists.Data/Entities/Snapshot.cs b/src/FilterLists.Data/Entities/Snapshot.cs index ce6aaf699..74702a8cb 100644 --- a/src/FilterLists.Data/Entities/Snapshot.cs +++ b/src/FilterLists.Data/Entities/Snapshot.cs @@ -8,7 +8,7 @@ public class Snapshot : BaseEntity public uint FilterListId { get; set; } public FilterList FilterList { get; set; } public string HttpStatusCode { get; set; } - public bool IsCompleted { get; set; } + public bool WasSuccessful { get; set; } public ICollection AddedSnapshotRules { get; set; } public ICollection RemovedSnapshotRules { get; set; } } diff --git a/src/FilterLists.Data/EntityTypeConfigurations/FilterListTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/FilterListTypeConfiguration.cs index e32b4d239..89cf03aba 100644 --- a/src/FilterLists.Data/EntityTypeConfigurations/FilterListTypeConfiguration.cs +++ b/src/FilterLists.Data/EntityTypeConfigurations/FilterListTypeConfiguration.cs @@ -18,7 +18,7 @@ public override void Configure(EntityTypeBuilder entityTypeBuilder) .HasColumnType("TEXT"); entityTypeBuilder.Property(x => x.DiscontinuedDate) .HasColumnType("DATE") - .HasDefaultValueSql("NULL"); ; + .HasDefaultValueSql("NULL"); entityTypeBuilder.Property(x => x.DonateUrl) .HasColumnType("TEXT"); entityTypeBuilder.Property(x => x.EmailAddress) @@ -40,7 +40,7 @@ public override void Configure(EntityTypeBuilder entityTypeBuilder) .HasColumnType("TEXT"); entityTypeBuilder.Property(x => x.PublishedDate) .HasColumnType("DATE") - .HasDefaultValueSql("NULL"); ; + .HasDefaultValueSql("NULL"); entityTypeBuilder.Property(x => x.SubmissionUrl) .HasColumnType("TEXT"); entityTypeBuilder.Property(x => x.ViewUrl) diff --git a/src/FilterLists.Services/FilterList/MappingProfiles/ListDetailsDtoMappingProfile.cs b/src/FilterLists.Services/FilterList/MappingProfiles/ListDetailsDtoMappingProfile.cs index 99f2b6f76..12b886214 100644 --- a/src/FilterLists.Services/FilterList/MappingProfiles/ListDetailsDtoMappingProfile.cs +++ b/src/FilterLists.Services/FilterList/MappingProfiles/ListDetailsDtoMappingProfile.cs @@ -15,15 +15,15 @@ public ListDetailsDtoMappingProfile() => .ForMember(d => d.Maintainers, c => c.MapFrom(l => l.FilterListMaintainers.Select(m => m.Maintainer))) .ForMember(d => d.RuleCount, c => c.MapFrom(l => - l.Snapshots.Where(s => s.IsCompleted && s.HttpStatusCode == "200") + l.Snapshots.Where(s => s.WasSuccessful) .SelectMany(sr => sr.AddedSnapshotRules) .Count() - - l.Snapshots.Where(s => s.IsCompleted && s.HttpStatusCode == "200") + l.Snapshots.Where(s => s.WasSuccessful) .SelectMany(sr => sr.RemovedSnapshotRules) .Count())) .ForMember(d => d.UpdatedDate, c => c.MapFrom(l => - l.Snapshots.Where(s => s.IsCompleted && s.HttpStatusCode == "200") + l.Snapshots.Where(s => s.WasSuccessful) .Where(s => s.AddedSnapshotRules.Count > 0 || s.RemovedSnapshotRules.Count > 0) .OrderByDescending(s => s.CreatedDateUtc) .Select(s => s.CreatedDateUtc) diff --git a/src/FilterLists.Services/FilterList/MappingProfiles/ListSummaryDtoMappingProfile.cs b/src/FilterLists.Services/FilterList/MappingProfiles/ListSummaryDtoMappingProfile.cs index b4e4669a5..eb6d930f3 100644 --- a/src/FilterLists.Services/FilterList/MappingProfiles/ListSummaryDtoMappingProfile.cs +++ b/src/FilterLists.Services/FilterList/MappingProfiles/ListSummaryDtoMappingProfile.cs @@ -13,7 +13,7 @@ public ListSummaryDtoMappingProfile() => .ForMember(d => d.Languages, c => c.MapFrom(l => l.FilterListLanguages.Select(la => la.Language))) .ForMember(d => d.UpdatedDate, c => c.MapFrom(l => - l.Snapshots.Where(s => s.IsCompleted && s.HttpStatusCode == "200") + l.Snapshots.Where(s => s.WasSuccessful) .Where(s => s.AddedSnapshotRules.Count > 0 || s.RemovedSnapshotRules.Count > 0) .OrderByDescending(s => s.CreatedDateUtc) .Select(s => s.CreatedDateUtc) diff --git a/src/FilterLists.Services/Snapshot/SnapshotDe.cs b/src/FilterLists.Services/Snapshot/SnapshotDe.cs index 1d94a8062..92e137111 100644 --- a/src/FilterLists.Services/Snapshot/SnapshotDe.cs +++ b/src/FilterLists.Services/Snapshot/SnapshotDe.cs @@ -46,11 +46,9 @@ public async Task SaveAsync() { await SaveSnapshotInBatches(content); await DedupSnapshotRules(); + await SetSuccessful(); } - //TODO: remove after closed: https://github.com/collinbarrett/FilterLists/issues/344 - await SetCompleted(); - transaction.Commit(); } catch (Exception e) @@ -119,7 +117,11 @@ private async Task SendExceptionEmail(Exception e) var message = new StringBuilder(); message.AppendLine("Snapshot Exception"); message.AppendLine("FilterListId: " + snapshot.FilterListId); - message.AppendLine("Exception: " + e.Message); + message.AppendLine("Exception:"); + message.AppendLine(e.Message); + message.AppendLine(e.StackTrace); + message.AppendLine(e.InnerException?.Message); + message.AppendLine(e.InnerException?.StackTrace); await emailService.SendEmailAsync("Snapshot Exception", message.ToString()); } @@ -176,9 +178,9 @@ private void RemoveDuplicateSnapshotRules(IQueryable existingSnaps dbContext.SnapshotRules.RemoveRange(duplicateSnapshotRules); } - private async Task SetCompleted() + private async Task SetSuccessful() { - snapshot.IsCompleted = true; + snapshot.WasSuccessful = true; await dbContext.SaveChangesAsync(); } } diff --git a/src/FilterLists.Services/Snapshot/SnapshotService.cs b/src/FilterLists.Services/Snapshot/SnapshotService.cs index c53959a93..986161807 100644 --- a/src/FilterLists.Services/Snapshot/SnapshotService.cs +++ b/src/FilterLists.Services/Snapshot/SnapshotService.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Collections.ObjectModel; using System.Linq; using System.Threading.Tasks; using AutoMapper; @@ -15,11 +14,6 @@ namespace FilterLists.Services.Snapshot [UsedImplicitly] public class SnapshotService : Service { - //TODO: https://github.com/collinbarrett/FilterLists/issues/200 - //TODO: https://github.com/collinbarrett/FilterLists/issues/201 - private static readonly IList IgnoreLists = - new ReadOnlyCollection(new List {48, 149, 173, 185, 186, 187, 188, 189, 352}); - private readonly DateTime yesterday = DateTime.UtcNow.AddDays(-1); public SnapshotService(FilterListsDbContext dbContext, IConfigurationProvider configurationProvider, @@ -30,25 +24,15 @@ public SnapshotService(FilterListsDbContext dbContext, IConfigurationProvider co public async Task CaptureAsync(int batchSize) { - //TODO: remove after closed: https://github.com/collinbarrett/FilterLists/issues/344 - await RollbackIncompletedSnapshots(); - var lists = await GetListsToCapture(batchSize); var snapshots = CreateSnapshots(lists); await SaveSnapshots(snapshots); } - private async Task RollbackIncompletedSnapshots() - { - var incompleteSnapshots = DbContext.Snapshots.Where(ss => ss.IsCompleted == false); - DbContext.Snapshots.RemoveRange(incompleteSnapshots); - await DbContext.SaveChangesAsync(); - } - private async Task> GetListsToCapture(int batchSize) => await DbContext .FilterLists - .Where(l => !IgnoreLists.Contains(l.Id) && + .Where(l => !l.CantSnapshot && (!l.Snapshots.Any() || l.Snapshots .Select(s => s.CreatedDateUtc)