From a6728cc8bca8d338ccd92c6ef27deaa9f83814d5 Mon Sep 17 00:00:00 2001 From: "Collin M. Barrett" Date: Sat, 27 Jan 2018 17:35:51 -0600 Subject: [PATCH] further progress on ScrapeService --- ...2215_AddScrapedAndUpdatedDates.Designer.cs | 451 ++++++++++++++++++ ...0180127232215_AddScrapedAndUpdatedDates.cs | 32 ++ .../FilterListsDbContextModelSnapshot.cs | 4 + src/FilterLists.Data/Entities/FilterList.cs | 2 + .../Extensions/SeedFilterListsDbContext.cs | 6 +- .../Services/ScrapeService.cs | 94 ++-- 6 files changed, 559 insertions(+), 30 deletions(-) create mode 100644 src/FilterLists.Api/Migrations/20180127232215_AddScrapedAndUpdatedDates.Designer.cs create mode 100644 src/FilterLists.Api/Migrations/20180127232215_AddScrapedAndUpdatedDates.cs diff --git a/src/FilterLists.Api/Migrations/20180127232215_AddScrapedAndUpdatedDates.Designer.cs b/src/FilterLists.Api/Migrations/20180127232215_AddScrapedAndUpdatedDates.Designer.cs new file mode 100644 index 000000000..6cf92f98c --- /dev/null +++ b/src/FilterLists.Api/Migrations/20180127232215_AddScrapedAndUpdatedDates.Designer.cs @@ -0,0 +1,451 @@ +// +using FilterLists.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage; +using Microsoft.EntityFrameworkCore.Storage.Internal; +using System; + +namespace FilterLists.Api.Migrations +{ + [DbContext(typeof(FilterListsDbContext))] + [Migration("20180127232215_AddScrapedAndUpdatedDates")] + partial class AddScrapedAndUpdatedDates + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn) + .HasAnnotation("ProductVersion", "2.0.1-rtm-125"); + + modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("Description") + .HasColumnType("TEXT"); + + b.Property("DescriptionSourceUrl") + .HasColumnType("TEXT"); + + b.Property("DiscontinuedDate"); + + b.Property("DonateUrl") + .HasColumnType("TEXT"); + + b.Property("EmailAddress") + .HasColumnType("NVARCHAR(126)"); + + b.Property("ForumUrl") + .HasColumnType("TEXT"); + + b.Property("HomeUrl") + .HasColumnType("TEXT"); + + b.Property("IssuesUrl") + .HasColumnType("TEXT"); + + b.Property("LicenseId"); + + b.Property("ModifiedDateUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP"); + + b.Property("Name") + .HasColumnType("NVARCHAR(126) NOT NULL"); + + b.Property("ScrapedDateUtc"); + + b.Property("SubmissionUrl") + .HasColumnType("TEXT"); + + b.Property("SyntaxId"); + + b.Property("UpdatedDateUtc"); + + b.Property("ViewUrl") + .HasColumnType("TEXT NOT NULL"); + + b.HasKey("Id"); + + b.HasIndex("LicenseId"); + + b.HasIndex("SyntaxId"); + + b.ToTable("filterlists"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b => + { + b.Property("FilterListId"); + + b.Property("LanguageId"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.HasKey("FilterListId", "LanguageId"); + + b.HasIndex("LanguageId"); + + b.ToTable("filterlists_languages"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b => + { + b.Property("FilterListId"); + + b.Property("MaintainerId"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.HasKey("FilterListId", "MaintainerId"); + + b.HasIndex("MaintainerId"); + + b.ToTable("filterlists_maintainers"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListRule", b => + { + b.Property("FilterListId"); + + b.Property("RuleId"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.HasKey("FilterListId", "RuleId"); + + b.HasIndex("RuleId"); + + b.ToTable("filterlists_rules"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b => + { + b.Property("ForkFilterListId"); + + b.Property("UpstreamFilterListId"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.HasKey("ForkFilterListId", "UpstreamFilterListId"); + + b.HasIndex("UpstreamFilterListId"); + + b.ToTable("forks"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b => + { + b.Property("MergeFilterListId"); + + b.Property("UpstreamFilterListId"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.HasKey("MergeFilterListId", "UpstreamFilterListId"); + + b.HasIndex("UpstreamFilterListId"); + + b.ToTable("merges"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b => + { + b.Property("SoftwareId"); + + b.Property("SyntaxId"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.HasKey("SoftwareId", "SyntaxId"); + + b.HasIndex("SyntaxId"); + + b.ToTable("software_syntaxes"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Language", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("Iso6391") + .HasColumnType("NVARCHAR(2)"); + + b.Property("Iso6392") + .HasColumnType("NVARCHAR(3)"); + + b.Property("Iso6392B") + .HasColumnType("NVARCHAR(3)"); + + b.Property("Iso6392T") + .HasColumnType("NVARCHAR(3)"); + + b.Property("Iso6393") + .HasColumnType("NVARCHAR(3)"); + + b.Property("LocalName") + .HasColumnType("NVARCHAR(126)"); + + b.Property("ModifiedDateUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP"); + + b.Property("Name") + .HasColumnType("NVARCHAR(126)"); + + b.HasKey("Id"); + + b.ToTable("languages"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.License", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("DescriptionUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP"); + + b.Property("Name") + .HasColumnType("NVARCHAR(126) NOT NULL"); + + b.Property("PermissiveAdaptation"); + + b.Property("PermissiveCommercial"); + + b.HasKey("Id"); + + b.ToTable("licenses"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Maintainer", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("EmailAddress") + .HasColumnType("NVARCHAR(126)"); + + b.Property("HomeUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP"); + + b.Property("Name") + .HasColumnType("NVARCHAR(126) NOT NULL"); + + b.Property("TwitterHandle") + .HasColumnType("NVARCHAR(126)"); + + b.HasKey("Id"); + + b.ToTable("maintainers"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Rule", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("BIGINT UNSIGNED"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("Raw") + .HasColumnType("NVARCHAR(2083) NOT NULL"); + + b.HasKey("Id"); + + b.ToTable("rules"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Software", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("DownloadUrl") + .HasColumnType("TEXT"); + + b.Property("HomeUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP"); + + b.Property("Name") + .HasColumnType("NVARCHAR(126) NOT NULL"); + + b.HasKey("Id"); + + b.ToTable("software"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Syntax", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("DefinitionUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP"); + + b.Property("Name") + .HasColumnType("NVARCHAR(126) NOT NULL"); + + b.HasKey("Id"); + + b.ToTable("syntaxes"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b => + { + b.HasOne("FilterLists.Data.Entities.License") + .WithMany("FilterLists") + .HasForeignKey("LicenseId"); + + b.HasOne("FilterLists.Data.Entities.Syntax") + .WithMany("FilterLists") + .HasForeignKey("SyntaxId"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("FilterListLanguages") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Language", "Language") + .WithMany("FilterListLanguages") + .HasForeignKey("LanguageId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("FilterListMaintainers") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Maintainer", "Maintainer") + .WithMany("FilterListMaintainers") + .HasForeignKey("MaintainerId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListRule", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("FilterListRules") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Rule", "Rule") + .WithMany("FilterListRules") + .HasForeignKey("RuleId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList") + .WithMany() + .HasForeignKey("ForkFilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList") + .WithMany() + .HasForeignKey("UpstreamFilterListId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "MergeFilterList") + .WithMany() + .HasForeignKey("MergeFilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList") + .WithMany() + .HasForeignKey("UpstreamFilterListId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b => + { + b.HasOne("FilterLists.Data.Entities.Software", "Software") + .WithMany("SoftwareSyntaxes") + .HasForeignKey("SoftwareId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax") + .WithMany("SoftwareSyntaxes") + .HasForeignKey("SyntaxId") + .OnDelete(DeleteBehavior.Cascade); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/FilterLists.Api/Migrations/20180127232215_AddScrapedAndUpdatedDates.cs b/src/FilterLists.Api/Migrations/20180127232215_AddScrapedAndUpdatedDates.cs new file mode 100644 index 000000000..027ef4fa1 --- /dev/null +++ b/src/FilterLists.Api/Migrations/20180127232215_AddScrapedAndUpdatedDates.cs @@ -0,0 +1,32 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +namespace FilterLists.Api.Migrations +{ + public partial class AddScrapedAndUpdatedDates : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + "ScrapedDateUtc", + "filterlists", + nullable: true); + + migrationBuilder.AddColumn( + "UpdatedDateUtc", + "filterlists", + nullable: true); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + "ScrapedDateUtc", + "filterlists"); + + migrationBuilder.DropColumn( + "UpdatedDateUtc", + "filterlists"); + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs index 809a83891..4c4d78ead 100644 --- a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs +++ b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs @@ -60,11 +60,15 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Name") .HasColumnType("NVARCHAR(126) NOT NULL"); + b.Property("ScrapedDateUtc"); + b.Property("SubmissionUrl") .HasColumnType("TEXT"); b.Property("SyntaxId"); + b.Property("UpdatedDateUtc"); + b.Property("ViewUrl") .HasColumnType("TEXT NOT NULL"); diff --git a/src/FilterLists.Data/Entities/FilterList.cs b/src/FilterLists.Data/Entities/FilterList.cs index 05a3c2ddb..b151e5d08 100644 --- a/src/FilterLists.Data/Entities/FilterList.cs +++ b/src/FilterLists.Data/Entities/FilterList.cs @@ -20,8 +20,10 @@ public class FilterList public string IssuesUrl { get; set; } public int? LicenseId { get; set; } public string Name { get; set; } + public DateTime? ScrapedDateUtc { get; set; } public string SubmissionUrl { get; set; } public int? SyntaxId { get; set; } + public DateTime? UpdatedDateUtc { get; set; } public string ViewUrl { get; set; } public ICollection FilterListLanguages { get; set; } diff --git a/src/FilterLists.Data/Seed/Extensions/SeedFilterListsDbContext.cs b/src/FilterLists.Data/Seed/Extensions/SeedFilterListsDbContext.cs index 21a51cd61..01ab21ea4 100644 --- a/src/FilterLists.Data/Seed/Extensions/SeedFilterListsDbContext.cs +++ b/src/FilterLists.Data/Seed/Extensions/SeedFilterListsDbContext.cs @@ -47,8 +47,10 @@ private static void InsertOnDuplicateKeyUpdate(this DbContext dbCon private static List GetPropertiesLessValueGeneratedTimestamps(IEntityType entityType) { - return entityType.GetProperties().Where(x => x.Name != "CreatedDateUtc" && x.Name != "ModifiedDateUtc") - .ToList(); + //TODO: filter dynamically from JSON + return entityType.GetProperties().Where(x => + !new List {"CreatedDateUtc", "ModifiedDateUtc", "ScrapedDateUtc", "UpdatedDateUtc"} + .Contains(x.Name)).ToList(); } private static string CreateValues(IReadOnlyCollection properties, string dataPath) diff --git a/src/FilterLists.Services/Services/ScrapeService.cs b/src/FilterLists.Services/Services/ScrapeService.cs index 6a934387b..dd355d067 100644 --- a/src/FilterLists.Services/Services/ScrapeService.cs +++ b/src/FilterLists.Services/Services/ScrapeService.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Linq; using System.Net.Http; using System.Threading.Tasks; @@ -18,56 +19,81 @@ public ScrapeService(FilterListsDbContext filterListsDbContext) this.filterListsDbContext = filterListsDbContext; } - public async Task Scrape() + public async Task Scrape(int numberLists) { - //TEMP: remove Take(1) when algorithm complete - var lists = await filterListsDbContext.FilterLists.Take(1).ToListAsync(); - foreach (var list in lists) await ScrapeList(list); + var lists = filterListsDbContext.FilterLists.OrderBy(x => x.ScrapedDateUtc).Take(numberLists) + .Select(x => new FilterListDto {Id = x.Id, ViewUrl = x.ViewUrl}); + var snapshots = await GetSnapshots(lists); + await AddOrUpdateRules(snapshots); + //TODO: update FilterList.ScrapedDateUtc } - private async Task ScrapeList(FilterList list) + private static async Task> GetSnapshots(IEnumerable lists) + { + var snapshots = new List(); + foreach (var list in lists) + snapshots.Add(new Snapshot {Content = await GetContent(list.ViewUrl), FilterListId = list.Id}); + return snapshots; + } + + //TODO: move to string (URL) extension method + private static async Task GetContent(string url) { - var snapshot = await GetContent(list.ViewUrl); try { - if (snapshot == null) return; + using (var httpClient = new HttpClient()) + { + using (var httpResponseMessage = await httpClient.GetAsync(url)) + { + if (httpResponseMessage.IsSuccessStatusCode) + return await httpResponseMessage.Content.ReadAsStringAsync(); + } + } } catch (Exception) { - //TODO: log exception - return; - } - - await AddRules(list, snapshot); - } - - private static async Task GetContent(string url) - { - using (var httpClient = new HttpClient()) - { - using (var httpResponseMessage = await httpClient.GetAsync(url)) - { - if (httpResponseMessage.IsSuccessStatusCode) - return await httpResponseMessage.Content.ReadAsStringAsync(); - } + //TODO: log exception for analysis + return null; } + //TODO: log httpResponseMessage.StatusCode for analysis return null; } - private async Task AddRules(FilterList list, string snapshot) + private async Task AddOrUpdateRules(IEnumerable snapshots) { - var lines = snapshot.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.None); + foreach (var snapshot in snapshots) + await AddOrUpdateRules(snapshot); + } - foreach (var line in lines) + private async Task AddOrUpdateRules(Snapshot snapshot) + { + var cachedRules = filterListsDbContext.FilterListRules + .Where(x => x.FilterListId == snapshot.FilterListId) + .Select(x => x.Rule); + + IEnumerable currentRulesRaw = + snapshot.Content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.None); + + var existingCurrentRules = filterListsDbContext.Rules.Where(x => currentRulesRaw.Contains(x.Raw)); + + var deletedRules = cachedRules.Except(existingCurrentRules).ToList(); + + filterListsDbContext.FilterListRules.RemoveRange(filterListsDbContext.FilterListRules + .Where(x => deletedRules.Select(y => y.Id).Contains(x.RuleId)) + .Where(x => x.FilterListId == snapshot.FilterListId)); + + //TODO: update FilterList.UpdatedDateUtc + + foreach (var line in currentRulesRaw) { if (string.IsNullOrWhiteSpace(line)) continue; var rule = new Rule {Raw = line}; if (!await filterListsDbContext.Rules.AnyAsync(x => x.Raw == rule.Raw)) filterListsDbContext.Rules.Add(rule); - var filterListRule = new FilterListRule {FilterList = list, Rule = rule}; + var filterListRule = new FilterListRule {FilterListId = snapshot.FilterListId, Rule = rule}; if (!await filterListsDbContext.FilterListRules.AnyAsync(x => - x.RuleId == rule.Id && x.FilterListId == list.Id)) + x.RuleId == rule.Id && x.FilterListId == snapshot.FilterListId)) filterListsDbContext.FilterListRules.Add(filterListRule); filterListsDbContext.FilterListRules.Add(filterListRule); } @@ -81,5 +107,17 @@ private async Task AddRules(FilterList list, string snapshot) //TODO: log exception } } + + private class FilterListDto + { + public int Id { get; set; } + public string ViewUrl { get; set; } + } + + private class Snapshot + { + public string Content { get; set; } + public int FilterListId { get; set; } + } } } \ No newline at end of file