From aa5617e938c1cfa23c1d991fbf9388ae515b8ed5 Mon Sep 17 00:00:00 2001 From: "Collin M. Barrett" Date: Thu, 23 Aug 2018 06:13:57 -0500 Subject: [PATCH 1/4] wip --- src/FilterLists.Services/Snapshot/Snapshot.cs | 14 ++------ .../Snapshot/SnapshotBatch.cs | 34 +++++++++++++------ 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/src/FilterLists.Services/Snapshot/Snapshot.cs b/src/FilterLists.Services/Snapshot/Snapshot.cs index 8b1f16923..4cca2b402 100644 --- a/src/FilterLists.Services/Snapshot/Snapshot.cs +++ b/src/FilterLists.Services/Snapshot/Snapshot.cs @@ -71,7 +71,7 @@ private async Task SaveAsync() if (lines != null) { await SaveInBatches(lines); - await DedupSnapshotRules(); + await AddRemovedSnapshotRules(); await SetSuccessful(); } @@ -136,11 +136,10 @@ private static async Task SaveBatches(IEnumerable batches) await batch.SaveAsync(); } - private async Task DedupSnapshotRules() + private async Task AddRemovedSnapshotRules() { var existingSnapshotRules = GetExistingSnapshotRules(); AddRemovedBySnapshots(existingSnapshotRules); - RemoveDuplicateSnapshotRules(existingSnapshotRules); await dbContext.SaveChangesAsync(); } @@ -157,15 +156,6 @@ private void AddRemovedBySnapshots(IQueryable existingSnapshotRule removedSnapshotRules.ForEach(sr => sr.RemovedBySnapshot = snapEntity); } - private void RemoveDuplicateSnapshotRules(IQueryable existingSnapshotRules) - { - var duplicateSnapshotRules = dbContext.SnapshotRules.Where(sr => - sr.AddedBySnapshot == snapEntity && - existingSnapshotRules.Any(e => - e.Rule == sr.Rule && e.RemovedBySnapshot == null)); - dbContext.SnapshotRules.RemoveRange(duplicateSnapshotRules); - } - private async Task SetSuccessful() { snapEntity.WasSuccessful = true; diff --git a/src/FilterLists.Services/Snapshot/SnapshotBatch.cs b/src/FilterLists.Services/Snapshot/SnapshotBatch.cs index a1af7099e..1385e54df 100644 --- a/src/FilterLists.Services/Snapshot/SnapshotBatch.cs +++ b/src/FilterLists.Services/Snapshot/SnapshotBatch.cs @@ -13,6 +13,7 @@ public class SnapshotBatch private readonly FilterListsDbContext dbContext; private readonly IEnumerable lines; private readonly Data.Entities.Snapshot snapEntity; + private IQueryable existingRules; public SnapshotBatch(FilterListsDbContext dbContext, IEnumerable lines, Data.Entities.Snapshot snapEntity) @@ -24,21 +25,32 @@ public SnapshotBatch(FilterListsDbContext dbContext, IEnumerable lines, public async Task SaveAsync() { - var existingRules = GetExistingRules(); - var newRules = CreateNewRules(existingRules); - dbContext.Rules.AddRange(newRules); - var rules = existingRules.Concat(newRules); - AddSnapshotRules(rules); + GetExistingRules(); + AddAddedSnapshotRulesForExistingRules(); + AddAddedSnapshotRulesForNewRules(); await dbContext.SaveChangesAsync(); } - private IQueryable GetExistingRules() => - dbContext.Rules.Join(lines, rule => rule.Raw, line => line, (rule, line) => rule); + private void GetExistingRules() => + existingRules = dbContext.Rules.Join(lines, rule => rule.Raw, line => line, (rule, line) => rule); - private List CreateNewRules(IQueryable existingRules) => - lines.Except(existingRules.Select(r => r.Raw)).Select(r => new Rule {Raw = r}).ToList(); + private void AddAddedSnapshotRulesForExistingRules() + { + var existingSnapshotRules = dbContext.SnapshotRules + .Where(sr => + existingRules.Contains(sr.Rule) && + sr.AddedBySnapshot.FilterListId == snapEntity.FilterListId && + sr.RemovedBySnapshot == null); + var newSnapshotRules = existingRules.Where(r => !existingSnapshotRules.Select(sr => sr.Rule).Contains(r)) + .Select(r => new SnapshotRule {Rule = r}); + snapEntity.AddedSnapshotRules.AddRange(newSnapshotRules); + } - private void AddSnapshotRules(IQueryable rules) => - snapEntity.AddedSnapshotRules.AddRange(rules.Select(r => new SnapshotRule {Rule = r})); + private void AddAddedSnapshotRulesForNewRules() + { + var rules = lines.Except(existingRules.Select(r => r.Raw)).Select(l => new Rule {Raw = l}); + var snapshotRules = rules.Select(r => new SnapshotRule {Rule = r}); + snapEntity.AddedSnapshotRules.AddRange(snapshotRules); + } } } \ No newline at end of file From 6dd9a3049e03eb08131ffad235cbc31d86f6b891 Mon Sep 17 00:00:00 2001 From: "Collin M. Barrett" Date: Thu, 23 Aug 2018 09:26:00 -0500 Subject: [PATCH 2/4] more work on SnapshotBatch --- .../Snapshot/SnapshotBatch.cs | 50 +++++++++++-------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/src/FilterLists.Services/Snapshot/SnapshotBatch.cs b/src/FilterLists.Services/Snapshot/SnapshotBatch.cs index 1385e54df..d88f9cf54 100644 --- a/src/FilterLists.Services/Snapshot/SnapshotBatch.cs +++ b/src/FilterLists.Services/Snapshot/SnapshotBatch.cs @@ -13,7 +13,6 @@ public class SnapshotBatch private readonly FilterListsDbContext dbContext; private readonly IEnumerable lines; private readonly Data.Entities.Snapshot snapEntity; - private IQueryable existingRules; public SnapshotBatch(FilterListsDbContext dbContext, IEnumerable lines, Data.Entities.Snapshot snapEntity) @@ -23,34 +22,45 @@ public SnapshotBatch(FilterListsDbContext dbContext, IEnumerable lines, this.snapEntity = snapEntity; } + private IQueryable ExistingRules => + dbContext.Rules.Join(lines, rule => rule.Raw, line => line, (rule, line) => rule); + public async Task SaveAsync() { - GetExistingRules(); - AddAddedSnapshotRulesForExistingRules(); - AddAddedSnapshotRulesForNewRules(); + AddAddedSnapRulesForExistingRules(); + AddAddedSnapRulesForNewRules(); await dbContext.SaveChangesAsync(); } - private void GetExistingRules() => - existingRules = dbContext.Rules.Join(lines, rule => rule.Raw, line => line, (rule, line) => rule); - - private void AddAddedSnapshotRulesForExistingRules() + private void AddAddedSnapRulesForExistingRules() { - var existingSnapshotRules = dbContext.SnapshotRules - .Where(sr => - existingRules.Contains(sr.Rule) && - sr.AddedBySnapshot.FilterListId == snapEntity.FilterListId && - sr.RemovedBySnapshot == null); - var newSnapshotRules = existingRules.Where(r => !existingSnapshotRules.Select(sr => sr.Rule).Contains(r)) - .Select(r => new SnapshotRule {Rule = r}); - snapEntity.AddedSnapshotRules.AddRange(newSnapshotRules); + var existingAddedSnapRules = GetExistingAddedSnapRules(); + var newAddedSnapRules = CreateNewAddedSnapRules(existingAddedSnapRules); + snapEntity.AddedSnapshotRules.AddRange(newAddedSnapRules); } - private void AddAddedSnapshotRulesForNewRules() + private IQueryable GetExistingAddedSnapRules() => + dbContext.SnapshotRules + .Where(sr => + ExistingRules.Contains(sr.Rule) && + sr.AddedBySnapshot.FilterListId == snapEntity.FilterListId && + sr.RemovedBySnapshot == null); + + private IEnumerable CreateNewAddedSnapRules(IQueryable existingAddedSnapRules) => + ExistingRules.Where(r => !existingAddedSnapRules.Select(sr => sr.Rule).Contains(r)) + .Select(r => new SnapshotRule {Rule = r}); + + private void AddAddedSnapRulesForNewRules() { - var rules = lines.Except(existingRules.Select(r => r.Raw)).Select(l => new Rule {Raw = l}); - var snapshotRules = rules.Select(r => new SnapshotRule {Rule = r}); - snapEntity.AddedSnapshotRules.AddRange(snapshotRules); + var newRules = CreateNewRules(); + var newAddedSnapRules = CreateSnapRules(newRules); + snapEntity.AddedSnapshotRules.AddRange(newAddedSnapRules); } + + private IEnumerable CreateNewRules() => + lines.Except(ExistingRules.Select(r => r.Raw)).Select(l => new Rule {Raw = l}); + + private static IEnumerable CreateSnapRules(IEnumerable rules) => + rules.Select(r => new SnapshotRule {Rule = r}); } } \ No newline at end of file From 3729fa2d272edce8d013ec572f54aa6b3dda933c Mon Sep 17 00:00:00 2001 From: "Collin M. Barrett" Date: Thu, 23 Aug 2018 10:13:25 -0500 Subject: [PATCH 3/4] update snapshot --- src/FilterLists.Services/Snapshot/Snapshot.cs | 58 +++++++------------ 1 file changed, 20 insertions(+), 38 deletions(-) diff --git a/src/FilterLists.Services/Snapshot/Snapshot.cs b/src/FilterLists.Services/Snapshot/Snapshot.cs index 4cca2b402..0c48ad5e6 100644 --- a/src/FilterLists.Services/Snapshot/Snapshot.cs +++ b/src/FilterLists.Services/Snapshot/Snapshot.cs @@ -25,6 +25,7 @@ public class Snapshot private readonly Data.Entities.Snapshot snapEntity; private readonly TelemetryClient telemetryClient; private readonly string userAgentString; + private HashSet lines; public Snapshot(FilterListsDbContext dbContext, EmailService emailService, FilterListViewUrlDto list, string userAgentString) @@ -41,7 +42,6 @@ public Snapshot(FilterListsDbContext dbContext, EmailService emailService, Filte telemetryClient = new TelemetryClient(); } - //TODO: add better compliance with Try/Parse pattern (https://stackoverflow.com/q/37810660/2343739) public async Task TrySaveAsync() { await AddSnapEntity(); @@ -65,50 +65,43 @@ private async Task AddSnapEntity() private async Task SaveAsync() { - using (var transaction = dbContext.Database.BeginTransaction()) + await TryGetLines(); + if (lines != null) { - var lines = await TryGetLines(); - if (lines != null) - { - await SaveInBatches(lines); - await AddRemovedSnapshotRules(); - await SetSuccessful(); - } - - transaction.Commit(); + await SaveInBatches(); + await AddRemovedSnapRules(); + await SetSuccessful(); } } - private async Task> TryGetLines() + private async Task TryGetLines() { try { - return await GetLines(); + await GetLines(); } catch (HttpRequestException hre) { await dbContext.SaveChangesAsync(); await TrackException(hre); - return null; } catch (WebException we) { snapEntity.HttpStatusCode = ((int)((HttpWebResponse)we.Response).StatusCode).ToString(); await dbContext.SaveChangesAsync(); await TrackException(we); - return null; } } - private async Task> GetLines() + private async Task GetLines() { - var lines = new HashSet(); using (var httpClient = new HttpClient()) { httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(userAgentString); var response = await httpClient.GetAsync(list.ViewUrl, HttpCompletionOption.ResponseHeadersRead); snapEntity.HttpStatusCode = ((int)response.StatusCode).ToString(); response.EnsureSuccessStatusCode(); + lines = new HashSet(); using (var stream = await response.Content.ReadAsStreamAsync()) using (var streamReader = new StreamReader(stream)) { @@ -117,17 +110,15 @@ private async Task> GetLines() lines.AddIfNotNullOrEmpty(line.LintLine()); } } - - return lines; } - private async Task SaveInBatches(IEnumerable lines) + private async Task SaveInBatches() { - var snapshotBatches = CreateBatches(lines); + var snapshotBatches = CreateBatches(); await SaveBatches(snapshotBatches); } - private IEnumerable CreateBatches(IEnumerable lines) => + private IEnumerable CreateBatches() => lines.Batch(BatchSize).Select(b => new SnapshotBatch(dbContext, b, snapEntity)); private static async Task SaveBatches(IEnumerable batches) @@ -136,26 +127,17 @@ private static async Task SaveBatches(IEnumerable batches) await batch.SaveAsync(); } - private async Task AddRemovedSnapshotRules() + private async Task AddRemovedSnapRules() { - var existingSnapshotRules = GetExistingSnapshotRules(); - AddRemovedBySnapshots(existingSnapshotRules); + dbContext.SnapshotRules + .Where(sr => + sr.AddedBySnapshot.FilterListId == list.Id && + sr.RemovedBySnapshot == null && + !lines.Contains(sr.Rule.Raw)) + .ForEach(sr => sr.RemovedBySnapshot = snapEntity); await dbContext.SaveChangesAsync(); } - private IQueryable GetExistingSnapshotRules() => - dbContext.SnapshotRules.Where(sr => - sr.AddedBySnapshot.FilterListId == list.Id && - sr.AddedBySnapshot != snapEntity && - sr.RemovedBySnapshot == null); - - private void AddRemovedBySnapshots(IQueryable existingSnapshotRules) - { - var newSnapshotRules = dbContext.SnapshotRules.Where(sr => sr.AddedBySnapshot == snapEntity); - var removedSnapshotRules = existingSnapshotRules.Where(sr => !newSnapshotRules.Any(n => n.Rule == sr.Rule)); - removedSnapshotRules.ForEach(sr => sr.RemovedBySnapshot = snapEntity); - } - private async Task SetSuccessful() { snapEntity.WasSuccessful = true; From 33e0766a34d95b98dbaea989c1df47a650ce6838 Mon Sep 17 00:00:00 2001 From: "Collin M. Barrett" Date: Thu, 23 Aug 2018 13:19:10 -0500 Subject: [PATCH 4/4] re-work SnapshotRule model --- ...4733_SimplifySnapshotRuleModel.Designer.cs | 645 ++++++++++++++++++ ...0180823174733_SimplifySnapshotRuleModel.cs | 111 +++ .../FilterListsDbContextModelSnapshot.cs | 33 +- .../Entities/Junctions/SnapshotRule.cs | 11 +- src/FilterLists.Data/Entities/Snapshot.cs | 5 +- .../SnapshotRuleTypeConfiguration.cs | 17 +- .../SnapshotTypeConfiguration.cs | 2 +- .../Extensions/ConfigureServicesCollection.cs | 1 - .../Extensions/CollectionExtensions.cs | 10 - .../ListDetailsDtoMappingProfile.cs | 14 +- .../ListSummaryDtoMappingProfile.cs | 6 +- src/FilterLists.Services/Snapshot/Snapshot.cs | 68 +- .../Snapshot/SnapshotBatch.cs | 43 +- .../Snapshot/SnapshotService.cs | 22 +- 14 files changed, 814 insertions(+), 174 deletions(-) create mode 100644 src/FilterLists.Api/Migrations/20180823174733_SimplifySnapshotRuleModel.Designer.cs create mode 100644 src/FilterLists.Api/Migrations/20180823174733_SimplifySnapshotRuleModel.cs diff --git a/src/FilterLists.Api/Migrations/20180823174733_SimplifySnapshotRuleModel.Designer.cs b/src/FilterLists.Api/Migrations/20180823174733_SimplifySnapshotRuleModel.Designer.cs new file mode 100644 index 000000000..ef2b3968e --- /dev/null +++ b/src/FilterLists.Api/Migrations/20180823174733_SimplifySnapshotRuleModel.Designer.cs @@ -0,0 +1,645 @@ +// + +using System; +using FilterLists.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; + +namespace FilterLists.Api.Migrations +{ + [DbContext(typeof(FilterListsDbContext))] + [Migration("20180823174733_SimplifySnapshotRuleModel")] + partial class SimplifySnapshotRuleModel + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "2.1.2-rtm-30932") + .HasAnnotation("Relational:MaxIdentifierLength", 64); + + modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CantSnapshot"); + + b.Property("ChatUrl") + .HasColumnType("TEXT"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("Description") + .HasColumnType("TEXT"); + + b.Property("DescriptionSourceUrl") + .HasColumnType("TEXT"); + + b.Property("DiscontinuedDate") + .ValueGeneratedOnAdd() + .HasColumnType("DATE") + .HasDefaultValueSql("NULL"); + + b.Property("DonateUrl") + .HasColumnType("TEXT"); + + b.Property("EmailAddress") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.Property("ForumUrl") + .HasColumnType("TEXT"); + + b.Property("HomeUrl") + .HasColumnType("TEXT"); + + b.Property("IssuesUrl") + .HasColumnType("TEXT"); + + b.Property("LicenseId") + .ValueGeneratedOnAdd() + .HasDefaultValue((byte)5); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.Property("PolicyUrl") + .HasColumnType("TEXT"); + + b.Property("PublishedDate") + .ValueGeneratedOnAdd() + .HasColumnType("DATE") + .HasDefaultValueSql("NULL"); + + b.Property("SubmissionUrl") + .HasColumnType("TEXT"); + + b.Property("SyntaxId"); + + b.Property("ViewUrl") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("LicenseId"); + + b.HasIndex("SyntaxId"); + + b.ToTable("filterlists"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b => + { + b.Property("FilterListId"); + + b.Property("LanguageId"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.HasKey("FilterListId", "LanguageId"); + + b.HasIndex("LanguageId"); + + b.ToTable("filterlists_languages"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b => + { + b.Property("FilterListId"); + + b.Property("MaintainerId"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.HasKey("FilterListId", "MaintainerId"); + + b.HasIndex("MaintainerId"); + + b.ToTable("filterlists_maintainers"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListTag", b => + { + b.Property("FilterListId"); + + b.Property("TagId"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.HasKey("FilterListId", "TagId"); + + b.HasIndex("TagId"); + + b.ToTable("filterlists_tags"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b => + { + b.Property("ForkFilterListId"); + + b.Property("UpstreamFilterListId"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.HasKey("ForkFilterListId", "UpstreamFilterListId"); + + b.HasIndex("UpstreamFilterListId"); + + b.ToTable("forks"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b => + { + b.Property("MergeFilterListId"); + + b.Property("UpstreamFilterListId"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.HasKey("MergeFilterListId", "UpstreamFilterListId"); + + b.HasIndex("UpstreamFilterListId"); + + b.ToTable("merges"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b => + { + b.Property("SnapshotId"); + + b.Property("RuleId"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.HasKey("SnapshotId", "RuleId"); + + b.HasIndex("RuleId"); + + b.ToTable("snapshots_rules"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b => + { + b.Property("SyntaxId"); + + b.Property("SoftwareId"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.HasKey("SyntaxId", "SoftwareId"); + + b.HasIndex("SoftwareId"); + + b.ToTable("software_syntaxes"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Language", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("Iso6391") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(2)") + .HasDefaultValueSql("NULL"); + + b.Property("Iso6392") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); + + b.Property("Iso6392B") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); + + b.Property("Iso6392T") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); + + b.Property("Iso6393") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); + + b.Property("LocalName") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("Name") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.HasKey("Id"); + + b.ToTable("languages"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.License", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TINYINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("DescriptionUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.Property("PermissiveAdaptation"); + + b.Property("PermissiveCommercial"); + + b.HasKey("Id"); + + b.ToTable("licenses"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Maintainer", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("EmailAddress") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.Property("HomeUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.Property("TwitterHandle") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.HasKey("Id"); + + b.ToTable("maintainers"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Rule", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("Raw") + .IsRequired() + .HasColumnType("VARCHAR(768)"); + + b.HasKey("Id"); + + b.HasIndex("Raw") + .IsUnique(); + + b.ToTable("rules"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("MEDIUMINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("FilterListId"); + + b.Property("HttpStatusCode") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED") + .HasDefaultValueSql("NULL"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("WasSuccessful"); + + b.HasKey("Id"); + + b.HasIndex("FilterListId"); + + b.ToTable("snapshots"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Software", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TINYINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("DownloadUrl") + .HasColumnType("TEXT"); + + b.Property("HomeUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.HasKey("Id"); + + b.ToTable("software"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Syntax", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TINYINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("DefinitionUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.HasKey("Id"); + + b.ToTable("syntaxes"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Tag", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TINYINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("Description") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.HasKey("Id"); + + b.ToTable("tags"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b => + { + b.HasOne("FilterLists.Data.Entities.License", "License") + .WithMany("FilterLists") + .HasForeignKey("LicenseId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax") + .WithMany("FilterLists") + .HasForeignKey("SyntaxId"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("FilterListLanguages") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Language", "Language") + .WithMany("FilterListLanguages") + .HasForeignKey("LanguageId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("FilterListMaintainers") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Maintainer", "Maintainer") + .WithMany("FilterListMaintainers") + .HasForeignKey("MaintainerId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListTag", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("FilterListTags") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Tag", "Tag") + .WithMany("FilterListTags") + .HasForeignKey("TagId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList") + .WithMany("ForkFilterLists") + .HasForeignKey("ForkFilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList") + .WithMany("UpstreamForkFilterLists") + .HasForeignKey("UpstreamFilterListId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "MergeFilterList") + .WithMany("MergeFilterLists") + .HasForeignKey("MergeFilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList") + .WithMany("UpstreamMergeFilterLists") + .HasForeignKey("UpstreamFilterListId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b => + { + b.HasOne("FilterLists.Data.Entities.Rule", "Rule") + .WithMany("SnapshotRules") + .HasForeignKey("RuleId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Snapshot", "Snapshot") + .WithMany("SnapshotRules") + .HasForeignKey("SnapshotId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b => + { + b.HasOne("FilterLists.Data.Entities.Software", "Software") + .WithMany("SoftwareSyntaxes") + .HasForeignKey("SoftwareId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax") + .WithMany("SoftwareSyntaxes") + .HasForeignKey("SyntaxId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("Snapshots") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + }); +#pragma warning restore 612, 618 + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Api/Migrations/20180823174733_SimplifySnapshotRuleModel.cs b/src/FilterLists.Api/Migrations/20180823174733_SimplifySnapshotRuleModel.cs new file mode 100644 index 000000000..17ca42463 --- /dev/null +++ b/src/FilterLists.Api/Migrations/20180823174733_SimplifySnapshotRuleModel.cs @@ -0,0 +1,111 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +namespace FilterLists.Api.Migrations +{ + public partial class SimplifySnapshotRuleModel : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropForeignKey( + "FK_snapshots_rules_snapshots_AddedBySnapshotId", + "snapshots_rules"); + + migrationBuilder.DropForeignKey( + "FK_snapshots_rules_snapshots_RemovedBySnapshotId", + "snapshots_rules"); + + migrationBuilder.DropIndex( + "IX_snapshots_rules_RemovedBySnapshotId", + "snapshots_rules"); + + migrationBuilder.DropColumn( + "ModifiedDateUtc", + "snapshots_rules"); + + migrationBuilder.DropColumn( + "RemovedBySnapshotId", + "snapshots_rules"); + + migrationBuilder.RenameColumn( + "AddedBySnapshotId", + "snapshots_rules", + "SnapshotId"); + + migrationBuilder.AlterColumn( + "HttpStatusCode", + "snapshots", + "SMALLINT UNSIGNED", + nullable: true, + defaultValueSql: "NULL", + oldClrType: typeof(string), + oldType: "VARCHAR(3)", + oldNullable: true, + oldDefaultValueSql: "NULL"); + + migrationBuilder.AddForeignKey( + "FK_snapshots_rules_snapshots_SnapshotId", + "snapshots_rules", + "SnapshotId", + "snapshots", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropForeignKey( + "FK_snapshots_rules_snapshots_SnapshotId", + "snapshots_rules"); + + migrationBuilder.RenameColumn( + "SnapshotId", + "snapshots_rules", + "AddedBySnapshotId"); + + migrationBuilder.AddColumn( + "ModifiedDateUtc", + "snapshots_rules", + "TIMESTAMP", + nullable: false, + defaultValueSql: "current_timestamp() ON UPDATE current_timestamp()"); + + migrationBuilder.AddColumn( + "RemovedBySnapshotId", + "snapshots_rules", + nullable: true); + + migrationBuilder.AlterColumn( + "HttpStatusCode", + "snapshots", + "VARCHAR(3)", + nullable: true, + defaultValueSql: "NULL", + oldClrType: typeof(ushort), + oldType: "SMALLINT UNSIGNED", + oldNullable: true, + oldDefaultValueSql: "NULL"); + + migrationBuilder.CreateIndex( + "IX_snapshots_rules_RemovedBySnapshotId", + "snapshots_rules", + "RemovedBySnapshotId"); + + migrationBuilder.AddForeignKey( + "FK_snapshots_rules_snapshots_AddedBySnapshotId", + "snapshots_rules", + "AddedBySnapshotId", + "snapshots", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + + migrationBuilder.AddForeignKey( + "FK_snapshots_rules_snapshots_RemovedBySnapshotId", + "snapshots_rules", + "RemovedBySnapshotId", + "snapshots", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs index da65dc322..971c63ce9 100644 --- a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs +++ b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs @@ -200,7 +200,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b => { - b.Property("AddedBySnapshotId"); + b.Property("SnapshotId"); b.Property("RuleId"); @@ -210,17 +210,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("TIMESTAMP") .HasDefaultValueSql("current_timestamp()"); - b.Property("ModifiedDateUtc") - .IsRequired() - .ValueGeneratedOnAddOrUpdate() - .HasColumnType("TIMESTAMP") - .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); - - b.Property("RemovedBySnapshotId"); - - b.HasKey("AddedBySnapshotId", "RuleId"); - - b.HasIndex("RemovedBySnapshotId"); + b.HasKey("SnapshotId", "RuleId"); b.HasIndex("RuleId"); @@ -421,9 +411,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("FilterListId"); - b.Property("HttpStatusCode") + b.Property("HttpStatusCode") .ValueGeneratedOnAdd() - .HasColumnType("VARCHAR(3)") + .HasColumnType("SMALLINT UNSIGNED") .HasDefaultValueSql("NULL"); b.Property("ModifiedDateUtc") @@ -616,20 +606,15 @@ protected override void BuildModel(ModelBuilder modelBuilder) modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b => { - b.HasOne("FilterLists.Data.Entities.Snapshot", "AddedBySnapshot") - .WithMany("AddedSnapshotRules") - .HasForeignKey("AddedBySnapshotId") - .OnDelete(DeleteBehavior.Cascade); - - b.HasOne("FilterLists.Data.Entities.Snapshot", "RemovedBySnapshot") - .WithMany("RemovedSnapshotRules") - .HasForeignKey("RemovedBySnapshotId") - .OnDelete(DeleteBehavior.Cascade); - b.HasOne("FilterLists.Data.Entities.Rule", "Rule") .WithMany("SnapshotRules") .HasForeignKey("RuleId") .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Snapshot", "Snapshot") + .WithMany("SnapshotRules") + .HasForeignKey("SnapshotId") + .OnDelete(DeleteBehavior.Cascade); }); modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b => diff --git a/src/FilterLists.Data/Entities/Junctions/SnapshotRule.cs b/src/FilterLists.Data/Entities/Junctions/SnapshotRule.cs index 4bc330ecf..088226aa7 100644 --- a/src/FilterLists.Data/Entities/Junctions/SnapshotRule.cs +++ b/src/FilterLists.Data/Entities/Junctions/SnapshotRule.cs @@ -1,14 +1,9 @@ -using System; - -namespace FilterLists.Data.Entities.Junctions +namespace FilterLists.Data.Entities.Junctions { public class SnapshotRule : BaseJunctionEntity { - public DateTime? ModifiedDateUtc { get; set; } - public uint AddedBySnapshotId { get; set; } - public Snapshot AddedBySnapshot { get; set; } - public uint? RemovedBySnapshotId { get; set; } - public Snapshot RemovedBySnapshot { get; set; } + public uint SnapshotId { get; set; } + public Snapshot Snapshot { get; set; } public uint RuleId { get; set; } public Rule Rule { get; set; } } diff --git a/src/FilterLists.Data/Entities/Snapshot.cs b/src/FilterLists.Data/Entities/Snapshot.cs index 74702a8cb..b151b2c23 100644 --- a/src/FilterLists.Data/Entities/Snapshot.cs +++ b/src/FilterLists.Data/Entities/Snapshot.cs @@ -7,9 +7,8 @@ public class Snapshot : BaseEntity { public uint FilterListId { get; set; } public FilterList FilterList { get; set; } - public string HttpStatusCode { get; set; } + public uint? HttpStatusCode { get; set; } public bool WasSuccessful { get; set; } - public ICollection AddedSnapshotRules { get; set; } - public ICollection RemovedSnapshotRules { get; set; } + public ICollection SnapshotRules { get; set; } } } \ No newline at end of file diff --git a/src/FilterLists.Data/EntityTypeConfigurations/Junctions/SnapshotRuleTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/Junctions/SnapshotRuleTypeConfiguration.cs index 68088f440..52df04c6a 100644 --- a/src/FilterLists.Data/EntityTypeConfigurations/Junctions/SnapshotRuleTypeConfiguration.cs +++ b/src/FilterLists.Data/EntityTypeConfigurations/Junctions/SnapshotRuleTypeConfiguration.cs @@ -10,19 +10,10 @@ public override void Configure(EntityTypeBuilder entityTypeBuilder { base.Configure(entityTypeBuilder); entityTypeBuilder.ToTable("snapshots_rules"); - entityTypeBuilder.HasKey(x => new {x.AddedBySnapshotId, x.RuleId}); - entityTypeBuilder.Property(x => x.ModifiedDateUtc) - .HasColumnType("TIMESTAMP") - .ValueGeneratedOnAddOrUpdate() - .IsRequired() - .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); - entityTypeBuilder.HasOne(x => x.AddedBySnapshot) - .WithMany(x => x.AddedSnapshotRules) - .HasForeignKey(x => x.AddedBySnapshotId); - entityTypeBuilder.HasOne(x => x.RemovedBySnapshot) - .WithMany(x => x.RemovedSnapshotRules) - .HasForeignKey(x => x.RemovedBySnapshotId) - .OnDelete(DeleteBehavior.Cascade); + entityTypeBuilder.HasKey(x => new {x.SnapshotId, x.RuleId}); + entityTypeBuilder.HasOne(x => x.Snapshot) + .WithMany(x => x.SnapshotRules) + .HasForeignKey(x => x.SnapshotId); entityTypeBuilder.HasOne(x => x.Rule) .WithMany(x => x.SnapshotRules) .HasForeignKey(x => x.RuleId); diff --git a/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs index 946c4a730..924b1ca02 100644 --- a/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs +++ b/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs @@ -13,7 +13,7 @@ public override void Configure(EntityTypeBuilder entityTypeBuilder) entityTypeBuilder.Property(x => x.Id) .HasColumnType("MEDIUMINT UNSIGNED"); entityTypeBuilder.Property(x => x.HttpStatusCode) - .HasColumnType("VARCHAR(3)") + .HasColumnType("SMALLINT UNSIGNED") .HasDefaultValueSql("NULL"); } } diff --git a/src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs b/src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs index 769134cb7..ee93937d8 100644 --- a/src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs +++ b/src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs @@ -33,7 +33,6 @@ public static void AddFilterListsAgentServices(this IServiceCollection services, o.UseMySql(config.GetConnectionString("FilterListsConnection"), m => m.MigrationsAssembly("FilterLists.Api"))); services.TryAddScoped(); - services.TryAddSingleton(); services.AddAutoMapper(); } } diff --git a/src/FilterLists.Services/Extensions/CollectionExtensions.cs b/src/FilterLists.Services/Extensions/CollectionExtensions.cs index 68c6042a1..2c65a12c0 100644 --- a/src/FilterLists.Services/Extensions/CollectionExtensions.cs +++ b/src/FilterLists.Services/Extensions/CollectionExtensions.cs @@ -4,16 +4,6 @@ namespace FilterLists.Services.Extensions { public static class CollectionExtensions { - //https://stackoverflow.com/a/26360010/2343739 - public static void AddRange(this ICollection destination, IEnumerable source) - { - if (destination is List list) - list.AddRange(source); - else - foreach (var item in source) - destination.Add(item); - } - public static void AddIfNotNullOrEmpty(this ICollection set, string item) { if (!string.IsNullOrEmpty(item)) diff --git a/src/FilterLists.Services/FilterList/MappingProfiles/ListDetailsDtoMappingProfile.cs b/src/FilterLists.Services/FilterList/MappingProfiles/ListDetailsDtoMappingProfile.cs index ea54d1e0f..12b92acf0 100644 --- a/src/FilterLists.Services/FilterList/MappingProfiles/ListDetailsDtoMappingProfile.cs +++ b/src/FilterLists.Services/FilterList/MappingProfiles/ListDetailsDtoMappingProfile.cs @@ -8,6 +8,7 @@ namespace FilterLists.Services.FilterList.MappingProfiles [UsedImplicitly] public class ListDetailsDtoMappingProfile : Profile { + //TODO: fix UpdatedDate to look at most recent snapshot with changes public ListDetailsDtoMappingProfile() => CreateMap() .ForMember(d => d.Languages, c => c.MapFrom(l => l.FilterListLanguages.Select(la => la.Language.Name))) @@ -16,17 +17,14 @@ public ListDetailsDtoMappingProfile() => .ForMember(d => d.RuleCount, c => c.MapFrom(l => l.Snapshots.Where(s => s.WasSuccessful) - .SelectMany(sr => sr.AddedSnapshotRules) - .Count() - - l.Snapshots.Where(s => s.WasSuccessful) - .SelectMany(sr => sr.RemovedSnapshotRules) - .Count())) + .OrderByDescending(s => s.CreatedDateUtc) + .FirstOrDefault() + .SnapshotRules.Count)) .ForMember(d => d.UpdatedDate, c => c.MapFrom(l => l.Snapshots.Where(s => s.WasSuccessful) - .Where(s => s.AddedSnapshotRules.Count > 0 || s.RemovedSnapshotRules.Count > 0) .OrderByDescending(s => s.CreatedDateUtc) - .Select(s => s.CreatedDateUtc) - .FirstOrDefault())); + .FirstOrDefault() + .CreatedDateUtc)); } } \ No newline at end of file diff --git a/src/FilterLists.Services/FilterList/MappingProfiles/ListSummaryDtoMappingProfile.cs b/src/FilterLists.Services/FilterList/MappingProfiles/ListSummaryDtoMappingProfile.cs index 0c9918aaf..9c1a40a1d 100644 --- a/src/FilterLists.Services/FilterList/MappingProfiles/ListSummaryDtoMappingProfile.cs +++ b/src/FilterLists.Services/FilterList/MappingProfiles/ListSummaryDtoMappingProfile.cs @@ -8,6 +8,7 @@ namespace FilterLists.Services.FilterList.MappingProfiles [UsedImplicitly] public class ListSummaryDtoMappingProfile : Profile { + //TODO: fix UpdatedDate to look at most recent snapshot with changes public ListSummaryDtoMappingProfile() => CreateMap() .ForMember(d => d.Languages, c => c.MapFrom(l => l.FilterListLanguages.Select(la => la.Language))) @@ -15,9 +16,8 @@ public ListSummaryDtoMappingProfile() => .ForMember(d => d.UpdatedDate, c => c.MapFrom(l => l.Snapshots.Where(s => s.WasSuccessful) - .Where(s => s.AddedSnapshotRules.Count > 0 || s.RemovedSnapshotRules.Count > 0) .OrderByDescending(s => s.CreatedDateUtc) - .Select(s => s.CreatedDateUtc) - .FirstOrDefault())); + .FirstOrDefault() + .CreatedDateUtc)); } } \ No newline at end of file diff --git a/src/FilterLists.Services/Snapshot/Snapshot.cs b/src/FilterLists.Services/Snapshot/Snapshot.cs index 0c48ad5e6..6340f5079 100644 --- a/src/FilterLists.Services/Snapshot/Snapshot.cs +++ b/src/FilterLists.Services/Snapshot/Snapshot.cs @@ -4,11 +4,9 @@ using System.Linq; using System.Net; using System.Net.Http; -using System.Text; using System.Threading; using System.Threading.Tasks; using FilterLists.Data; -using FilterLists.Data.Entities.Junctions; using FilterLists.Services.Extensions; using FilterLists.Services.Snapshot.Models; using Microsoft.ApplicationInsights; @@ -20,25 +18,21 @@ public class Snapshot { private const int BatchSize = 500; private readonly FilterListsDbContext dbContext; - private readonly EmailService emailService; private readonly FilterListViewUrlDto list; private readonly Data.Entities.Snapshot snapEntity; private readonly TelemetryClient telemetryClient; - private readonly string userAgentString; + private readonly string uaString; private HashSet lines; - public Snapshot(FilterListsDbContext dbContext, EmailService emailService, FilterListViewUrlDto list, - string userAgentString) + public Snapshot(FilterListsDbContext dbContext, FilterListViewUrlDto list, string uaString) { this.dbContext = dbContext; - this.emailService = emailService; this.list = list; + this.uaString = uaString; snapEntity = new Data.Entities.Snapshot { - FilterListId = list.Id, - AddedSnapshotRules = new List() + FilterListId = list.Id }; - this.userAgentString = userAgentString; telemetryClient = new TelemetryClient(); } @@ -53,7 +47,7 @@ public async Task TrySaveAsync() } catch (Exception e) { - await TrackException(e); + TrackException(e); } } @@ -69,7 +63,6 @@ private async Task SaveAsync() if (lines != null) { await SaveInBatches(); - await AddRemovedSnapRules(); await SetSuccessful(); } } @@ -83,13 +76,13 @@ private async Task TryGetLines() catch (HttpRequestException hre) { await dbContext.SaveChangesAsync(); - await TrackException(hre); + TrackException(hre); } catch (WebException we) { - snapEntity.HttpStatusCode = ((int)((HttpWebResponse)we.Response).StatusCode).ToString(); + snapEntity.HttpStatusCode = (uint)((HttpWebResponse)we.Response).StatusCode; await dbContext.SaveChangesAsync(); - await TrackException(we); + TrackException(we); } } @@ -97,9 +90,9 @@ private async Task GetLines() { using (var httpClient = new HttpClient()) { - httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(userAgentString); + httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(uaString); var response = await httpClient.GetAsync(list.ViewUrl, HttpCompletionOption.ResponseHeadersRead); - snapEntity.HttpStatusCode = ((int)response.StatusCode).ToString(); + snapEntity.HttpStatusCode = (uint)response.StatusCode; response.EnsureSuccessStatusCode(); lines = new HashSet(); using (var stream = await response.Content.ReadAsStreamAsync()) @@ -114,8 +107,8 @@ private async Task GetLines() private async Task SaveInBatches() { - var snapshotBatches = CreateBatches(); - await SaveBatches(snapshotBatches); + var snapBatches = CreateBatches(); + await SaveBatches(snapBatches); } private IEnumerable CreateBatches() => @@ -127,48 +120,13 @@ private static async Task SaveBatches(IEnumerable batches) await batch.SaveAsync(); } - private async Task AddRemovedSnapRules() - { - dbContext.SnapshotRules - .Where(sr => - sr.AddedBySnapshot.FilterListId == list.Id && - sr.RemovedBySnapshot == null && - !lines.Contains(sr.Rule.Raw)) - .ForEach(sr => sr.RemovedBySnapshot = snapEntity); - await dbContext.SaveChangesAsync(); - } - private async Task SetSuccessful() { snapEntity.WasSuccessful = true; await dbContext.SaveChangesAsync(); } - private async Task TrackException(Exception e) - { - await SendExceptionEmail(e); - TrackExceptionInApplicationInsights(e); - } - - private async Task SendExceptionEmail(Exception e) - { - if (!IsDeploymentInterrupted(e)) - { - var msg = new StringBuilder(); - msg.AppendLine("FilterListId: " + list.Id); - msg.AppendLine("Exception:"); - msg.AppendLine(e.Message); - msg.AppendLine(e.StackTrace); - msg.AppendLine(e.InnerException?.Message); - msg.AppendLine(e.InnerException?.StackTrace); - await emailService.SendEmailAsync("Snapshot Exception", msg.ToString()); - } - } - - private static bool IsDeploymentInterrupted(Exception e) => - e.Message.Contains("Failed to read the result set."); - - private void TrackExceptionInApplicationInsights(Exception e) + private void TrackException(Exception e) { telemetryClient.TrackException(e); telemetryClient.Flush(); diff --git a/src/FilterLists.Services/Snapshot/SnapshotBatch.cs b/src/FilterLists.Services/Snapshot/SnapshotBatch.cs index d88f9cf54..2581932f1 100644 --- a/src/FilterLists.Services/Snapshot/SnapshotBatch.cs +++ b/src/FilterLists.Services/Snapshot/SnapshotBatch.cs @@ -4,7 +4,6 @@ using FilterLists.Data; using FilterLists.Data.Entities; using FilterLists.Data.Entities.Junctions; -using FilterLists.Services.Extensions; namespace FilterLists.Services.Snapshot { @@ -22,45 +21,15 @@ public SnapshotBatch(FilterListsDbContext dbContext, IEnumerable lines, this.snapEntity = snapEntity; } - private IQueryable ExistingRules => - dbContext.Rules.Join(lines, rule => rule.Raw, line => line, (rule, line) => rule); - public async Task SaveAsync() { - AddAddedSnapRulesForExistingRules(); - AddAddedSnapRulesForNewRules(); + var existingRules = dbContext.Rules.Join(lines, rule => rule.Raw, line => line, (rule, line) => rule); + var newRules = lines.Except(existingRules.Select(r => r.Raw)).Select(l => new Rule { Raw = l }).ToList(); + dbContext.Rules.AddRange(newRules); + var rules = existingRules.Concat(newRules); + var snapshotRules = rules.Select(r => new SnapshotRule { Rule = r }); + snapEntity.SnapshotRules = snapshotRules.ToList(); await dbContext.SaveChangesAsync(); } - - private void AddAddedSnapRulesForExistingRules() - { - var existingAddedSnapRules = GetExistingAddedSnapRules(); - var newAddedSnapRules = CreateNewAddedSnapRules(existingAddedSnapRules); - snapEntity.AddedSnapshotRules.AddRange(newAddedSnapRules); - } - - private IQueryable GetExistingAddedSnapRules() => - dbContext.SnapshotRules - .Where(sr => - ExistingRules.Contains(sr.Rule) && - sr.AddedBySnapshot.FilterListId == snapEntity.FilterListId && - sr.RemovedBySnapshot == null); - - private IEnumerable CreateNewAddedSnapRules(IQueryable existingAddedSnapRules) => - ExistingRules.Where(r => !existingAddedSnapRules.Select(sr => sr.Rule).Contains(r)) - .Select(r => new SnapshotRule {Rule = r}); - - private void AddAddedSnapRulesForNewRules() - { - var newRules = CreateNewRules(); - var newAddedSnapRules = CreateSnapRules(newRules); - snapEntity.AddedSnapshotRules.AddRange(newAddedSnapRules); - } - - private IEnumerable CreateNewRules() => - lines.Except(ExistingRules.Select(r => r.Raw)).Select(l => new Rule {Raw = l}); - - private static IEnumerable CreateSnapRules(IEnumerable rules) => - rules.Select(r => new SnapshotRule {Rule = r}); } } \ No newline at end of file diff --git a/src/FilterLists.Services/Snapshot/SnapshotService.cs b/src/FilterLists.Services/Snapshot/SnapshotService.cs index 87f43102b..18700a7e1 100644 --- a/src/FilterLists.Services/Snapshot/SnapshotService.cs +++ b/src/FilterLists.Services/Snapshot/SnapshotService.cs @@ -12,19 +12,19 @@ namespace FilterLists.Services.Snapshot { public class SnapshotService : Service { - private readonly EmailService emailService; private readonly DateTime yesterday = DateTime.UtcNow.AddDays(-1); - public SnapshotService(FilterListsDbContext dbContext, IConfigurationProvider mapConfig, - EmailService emailService) - : base(dbContext, mapConfig) => this.emailService = emailService; + public SnapshotService(FilterListsDbContext dbContext, IConfigurationProvider mapConfig) + : base(dbContext, mapConfig) + { + } public async Task CaptureAsync(int batchSize) { var lists = await GetListsToCapture(batchSize); var uaString = await UserAgentService.GetMostPopularString(); - var snapshots = CreateSnapshots(lists, uaString); - await SaveSnapshots(snapshots); + var snaps = CreateSnaps(lists, uaString); + await SaveSnaps(snaps); } private async Task> GetListsToCapture(int batchSize) => @@ -49,13 +49,13 @@ await DbContext .ProjectTo(MapConfig) .ToListAsync(); - private IEnumerable CreateSnapshots(IEnumerable lists, string uaString) => - lists.Select(l => new Snapshot(DbContext, emailService, l, uaString)); + private IEnumerable CreateSnaps(IEnumerable lists, string uaString) => + lists.Select(l => new Snapshot(DbContext, l, uaString)); - private static async Task SaveSnapshots(IEnumerable snapshots) + private static async Task SaveSnaps(IEnumerable snaps) { - foreach (var snapshot in snapshots) - await snapshot.TrySaveAsync(); + foreach (var snap in snaps) + await snap.TrySaveAsync(); } } } \ No newline at end of file