mirror of
https://github.com/collinbarrett/FilterLists.git
synced 2026-03-11 09:04:27 +00:00
further progress on ScrapeService
This commit is contained in:
Collin M. Barrett
parent
ac6513a38f
commit
a6728cc8bc
6 changed files with 559 additions and 30 deletions
451
src/FilterLists.Api/Migrations/20180127232215_AddScrapedAndUpdatedDates.Designer.cs
generated
Normal file
451
src/FilterLists.Api/Migrations/20180127232215_AddScrapedAndUpdatedDates.Designer.cs
generated
Normal file
|
|
@ -0,0 +1,451 @@
|
|||
// <auto-generated />
|
||||
using FilterLists.Data;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||
using Microsoft.EntityFrameworkCore.Metadata;
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
using Microsoft.EntityFrameworkCore.Storage;
|
||||
using Microsoft.EntityFrameworkCore.Storage.Internal;
|
||||
using System;
|
||||
|
||||
namespace FilterLists.Api.Migrations
|
||||
{
|
||||
[DbContext(typeof(FilterListsDbContext))]
|
||||
[Migration("20180127232215_AddScrapedAndUpdatedDates")]
|
||||
partial class AddScrapedAndUpdatedDates
|
||||
{
|
||||
protected override void BuildTargetModel(ModelBuilder modelBuilder)
|
||||
{
|
||||
#pragma warning disable 612, 618
|
||||
modelBuilder
|
||||
.HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn)
|
||||
.HasAnnotation("ProductVersion", "2.0.1-rtm-125");
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("SMALLINT UNSIGNED");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("Description")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<string>("DescriptionSourceUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<DateTime?>("DiscontinuedDate");
|
||||
|
||||
b.Property<string>("DonateUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<string>("EmailAddress")
|
||||
.HasColumnType("NVARCHAR(126)");
|
||||
|
||||
b.Property<string>("ForumUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<string>("HomeUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<string>("IssuesUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<int?>("LicenseId");
|
||||
|
||||
b.Property<DateTime>("ModifiedDateUtc")
|
||||
.ValueGeneratedOnAddOrUpdate()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("Name")
|
||||
.HasColumnType("NVARCHAR(126) NOT NULL");
|
||||
|
||||
b.Property<DateTime?>("ScrapedDateUtc");
|
||||
|
||||
b.Property<string>("SubmissionUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<int?>("SyntaxId");
|
||||
|
||||
b.Property<DateTime?>("UpdatedDateUtc");
|
||||
|
||||
b.Property<string>("ViewUrl")
|
||||
.HasColumnType("TEXT NOT NULL");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.HasIndex("LicenseId");
|
||||
|
||||
b.HasIndex("SyntaxId");
|
||||
|
||||
b.ToTable("filterlists");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
|
||||
{
|
||||
b.Property<int>("FilterListId");
|
||||
|
||||
b.Property<int>("LanguageId");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.HasKey("FilterListId", "LanguageId");
|
||||
|
||||
b.HasIndex("LanguageId");
|
||||
|
||||
b.ToTable("filterlists_languages");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
|
||||
{
|
||||
b.Property<int>("FilterListId");
|
||||
|
||||
b.Property<int>("MaintainerId");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.HasKey("FilterListId", "MaintainerId");
|
||||
|
||||
b.HasIndex("MaintainerId");
|
||||
|
||||
b.ToTable("filterlists_maintainers");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListRule", b =>
|
||||
{
|
||||
b.Property<int>("FilterListId");
|
||||
|
||||
b.Property<int>("RuleId");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.HasKey("FilterListId", "RuleId");
|
||||
|
||||
b.HasIndex("RuleId");
|
||||
|
||||
b.ToTable("filterlists_rules");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
|
||||
{
|
||||
b.Property<int>("ForkFilterListId");
|
||||
|
||||
b.Property<int>("UpstreamFilterListId");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.HasKey("ForkFilterListId", "UpstreamFilterListId");
|
||||
|
||||
b.HasIndex("UpstreamFilterListId");
|
||||
|
||||
b.ToTable("forks");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
|
||||
{
|
||||
b.Property<int>("MergeFilterListId");
|
||||
|
||||
b.Property<int>("UpstreamFilterListId");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.HasKey("MergeFilterListId", "UpstreamFilterListId");
|
||||
|
||||
b.HasIndex("UpstreamFilterListId");
|
||||
|
||||
b.ToTable("merges");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
|
||||
{
|
||||
b.Property<int>("SoftwareId");
|
||||
|
||||
b.Property<int>("SyntaxId");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.HasKey("SoftwareId", "SyntaxId");
|
||||
|
||||
b.HasIndex("SyntaxId");
|
||||
|
||||
b.ToTable("software_syntaxes");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Language", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("SMALLINT UNSIGNED");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("Iso6391")
|
||||
.HasColumnType("NVARCHAR(2)");
|
||||
|
||||
b.Property<string>("Iso6392")
|
||||
.HasColumnType("NVARCHAR(3)");
|
||||
|
||||
b.Property<string>("Iso6392B")
|
||||
.HasColumnType("NVARCHAR(3)");
|
||||
|
||||
b.Property<string>("Iso6392T")
|
||||
.HasColumnType("NVARCHAR(3)");
|
||||
|
||||
b.Property<string>("Iso6393")
|
||||
.HasColumnType("NVARCHAR(3)");
|
||||
|
||||
b.Property<string>("LocalName")
|
||||
.HasColumnType("NVARCHAR(126)");
|
||||
|
||||
b.Property<DateTime>("ModifiedDateUtc")
|
||||
.ValueGeneratedOnAddOrUpdate()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("Name")
|
||||
.HasColumnType("NVARCHAR(126)");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.ToTable("languages");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.License", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("SMALLINT UNSIGNED");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("DescriptionUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<DateTime>("ModifiedDateUtc")
|
||||
.ValueGeneratedOnAddOrUpdate()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("Name")
|
||||
.HasColumnType("NVARCHAR(126) NOT NULL");
|
||||
|
||||
b.Property<bool>("PermissiveAdaptation");
|
||||
|
||||
b.Property<bool>("PermissiveCommercial");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.ToTable("licenses");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Maintainer", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("SMALLINT UNSIGNED");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("EmailAddress")
|
||||
.HasColumnType("NVARCHAR(126)");
|
||||
|
||||
b.Property<string>("HomeUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<DateTime>("ModifiedDateUtc")
|
||||
.ValueGeneratedOnAddOrUpdate()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("Name")
|
||||
.HasColumnType("NVARCHAR(126) NOT NULL");
|
||||
|
||||
b.Property<string>("TwitterHandle")
|
||||
.HasColumnType("NVARCHAR(126)");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.ToTable("maintainers");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Rule", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("BIGINT UNSIGNED");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("Raw")
|
||||
.HasColumnType("NVARCHAR(2083) NOT NULL");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.ToTable("rules");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Software", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("SMALLINT UNSIGNED");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("DownloadUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<string>("HomeUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<DateTime>("ModifiedDateUtc")
|
||||
.ValueGeneratedOnAddOrUpdate()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("Name")
|
||||
.HasColumnType("NVARCHAR(126) NOT NULL");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.ToTable("software");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Syntax", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("SMALLINT UNSIGNED");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("DefinitionUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<DateTime>("ModifiedDateUtc")
|
||||
.ValueGeneratedOnAddOrUpdate()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("Name")
|
||||
.HasColumnType("NVARCHAR(126) NOT NULL");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.ToTable("syntaxes");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.License")
|
||||
.WithMany("FilterLists")
|
||||
.HasForeignKey("LicenseId");
|
||||
|
||||
b.HasOne("FilterLists.Data.Entities.Syntax")
|
||||
.WithMany("FilterLists")
|
||||
.HasForeignKey("SyntaxId");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
|
||||
.WithMany("FilterListLanguages")
|
||||
.HasForeignKey("FilterListId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
|
||||
b.HasOne("FilterLists.Data.Entities.Language", "Language")
|
||||
.WithMany("FilterListLanguages")
|
||||
.HasForeignKey("LanguageId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
|
||||
.WithMany("FilterListMaintainers")
|
||||
.HasForeignKey("FilterListId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
|
||||
b.HasOne("FilterLists.Data.Entities.Maintainer", "Maintainer")
|
||||
.WithMany("FilterListMaintainers")
|
||||
.HasForeignKey("MaintainerId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListRule", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
|
||||
.WithMany("FilterListRules")
|
||||
.HasForeignKey("FilterListId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
|
||||
b.HasOne("FilterLists.Data.Entities.Rule", "Rule")
|
||||
.WithMany("FilterListRules")
|
||||
.HasForeignKey("RuleId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList")
|
||||
.WithMany()
|
||||
.HasForeignKey("ForkFilterListId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
|
||||
b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
|
||||
.WithMany()
|
||||
.HasForeignKey("UpstreamFilterListId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.FilterList", "MergeFilterList")
|
||||
.WithMany()
|
||||
.HasForeignKey("MergeFilterListId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
|
||||
b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
|
||||
.WithMany()
|
||||
.HasForeignKey("UpstreamFilterListId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.Software", "Software")
|
||||
.WithMany("SoftwareSyntaxes")
|
||||
.HasForeignKey("SoftwareId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
|
||||
b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax")
|
||||
.WithMany("SoftwareSyntaxes")
|
||||
.HasForeignKey("SyntaxId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
#pragma warning restore 612, 618
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
using System;
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
|
||||
namespace FilterLists.Api.Migrations
|
||||
{
|
||||
public partial class AddScrapedAndUpdatedDates : Migration
|
||||
{
|
||||
protected override void Up(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.AddColumn<DateTime>(
|
||||
"ScrapedDateUtc",
|
||||
"filterlists",
|
||||
nullable: true);
|
||||
|
||||
migrationBuilder.AddColumn<DateTime>(
|
||||
"UpdatedDateUtc",
|
||||
"filterlists",
|
||||
nullable: true);
|
||||
}
|
||||
|
||||
protected override void Down(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.DropColumn(
|
||||
"ScrapedDateUtc",
|
||||
"filterlists");
|
||||
|
||||
migrationBuilder.DropColumn(
|
||||
"UpdatedDateUtc",
|
||||
"filterlists");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -60,11 +60,15 @@ protected override void BuildModel(ModelBuilder modelBuilder)
|
|||
b.Property<string>("Name")
|
||||
.HasColumnType("NVARCHAR(126) NOT NULL");
|
||||
|
||||
b.Property<DateTime?>("ScrapedDateUtc");
|
||||
|
||||
b.Property<string>("SubmissionUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<int?>("SyntaxId");
|
||||
|
||||
b.Property<DateTime?>("UpdatedDateUtc");
|
||||
|
||||
b.Property<string>("ViewUrl")
|
||||
.HasColumnType("TEXT NOT NULL");
|
||||
|
||||
|
|
|
|||
|
|
@ -20,8 +20,10 @@ public class FilterList
|
|||
public string IssuesUrl { get; set; }
|
||||
public int? LicenseId { get; set; }
|
||||
public string Name { get; set; }
|
||||
public DateTime? ScrapedDateUtc { get; set; }
|
||||
public string SubmissionUrl { get; set; }
|
||||
public int? SyntaxId { get; set; }
|
||||
public DateTime? UpdatedDateUtc { get; set; }
|
||||
public string ViewUrl { get; set; }
|
||||
|
||||
public ICollection<FilterListLanguage> FilterListLanguages { get; set; }
|
||||
|
|
|
|||
|
|
@ -47,8 +47,10 @@ private static void InsertOnDuplicateKeyUpdate<TEntityType>(this DbContext dbCon
|
|||
|
||||
private static List<IProperty> GetPropertiesLessValueGeneratedTimestamps(IEntityType entityType)
|
||||
{
|
||||
return entityType.GetProperties().Where(x => x.Name != "CreatedDateUtc" && x.Name != "ModifiedDateUtc")
|
||||
.ToList();
|
||||
//TODO: filter dynamically from JSON
|
||||
return entityType.GetProperties().Where(x =>
|
||||
!new List<string> {"CreatedDateUtc", "ModifiedDateUtc", "ScrapedDateUtc", "UpdatedDateUtc"}
|
||||
.Contains(x.Name)).ToList();
|
||||
}
|
||||
|
||||
private static string CreateValues<TEntityType>(IReadOnlyCollection<IProperty> properties, string dataPath)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Threading.Tasks;
|
||||
|
|
@ -18,56 +19,81 @@ public ScrapeService(FilterListsDbContext filterListsDbContext)
|
|||
this.filterListsDbContext = filterListsDbContext;
|
||||
}
|
||||
|
||||
public async Task Scrape()
|
||||
public async Task Scrape(int numberLists)
|
||||
{
|
||||
//TEMP: remove Take(1) when algorithm complete
|
||||
var lists = await filterListsDbContext.FilterLists.Take(1).ToListAsync();
|
||||
foreach (var list in lists) await ScrapeList(list);
|
||||
var lists = filterListsDbContext.FilterLists.OrderBy(x => x.ScrapedDateUtc).Take(numberLists)
|
||||
.Select(x => new FilterListDto {Id = x.Id, ViewUrl = x.ViewUrl});
|
||||
var snapshots = await GetSnapshots(lists);
|
||||
await AddOrUpdateRules(snapshots);
|
||||
//TODO: update FilterList.ScrapedDateUtc
|
||||
}
|
||||
|
||||
private async Task ScrapeList(FilterList list)
|
||||
private static async Task<IEnumerable<Snapshot>> GetSnapshots(IEnumerable<FilterListDto> lists)
|
||||
{
|
||||
var snapshots = new List<Snapshot>();
|
||||
foreach (var list in lists)
|
||||
snapshots.Add(new Snapshot {Content = await GetContent(list.ViewUrl), FilterListId = list.Id});
|
||||
return snapshots;
|
||||
}
|
||||
|
||||
//TODO: move to string (URL) extension method
|
||||
private static async Task<string> GetContent(string url)
|
||||
{
|
||||
var snapshot = await GetContent(list.ViewUrl);
|
||||
try
|
||||
{
|
||||
if (snapshot == null) return;
|
||||
using (var httpClient = new HttpClient())
|
||||
{
|
||||
using (var httpResponseMessage = await httpClient.GetAsync(url))
|
||||
{
|
||||
if (httpResponseMessage.IsSuccessStatusCode)
|
||||
return await httpResponseMessage.Content.ReadAsStringAsync();
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
//TODO: log exception
|
||||
return;
|
||||
}
|
||||
|
||||
await AddRules(list, snapshot);
|
||||
}
|
||||
|
||||
private static async Task<string> GetContent(string url)
|
||||
{
|
||||
using (var httpClient = new HttpClient())
|
||||
{
|
||||
using (var httpResponseMessage = await httpClient.GetAsync(url))
|
||||
{
|
||||
if (httpResponseMessage.IsSuccessStatusCode)
|
||||
return await httpResponseMessage.Content.ReadAsStringAsync();
|
||||
}
|
||||
//TODO: log exception for analysis
|
||||
return null;
|
||||
}
|
||||
|
||||
//TODO: log httpResponseMessage.StatusCode for analysis
|
||||
return null;
|
||||
}
|
||||
|
||||
private async Task AddRules(FilterList list, string snapshot)
|
||||
private async Task AddOrUpdateRules(IEnumerable<Snapshot> snapshots)
|
||||
{
|
||||
var lines = snapshot.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.None);
|
||||
foreach (var snapshot in snapshots)
|
||||
await AddOrUpdateRules(snapshot);
|
||||
}
|
||||
|
||||
foreach (var line in lines)
|
||||
private async Task AddOrUpdateRules(Snapshot snapshot)
|
||||
{
|
||||
var cachedRules = filterListsDbContext.FilterListRules
|
||||
.Where(x => x.FilterListId == snapshot.FilterListId)
|
||||
.Select(x => x.Rule);
|
||||
|
||||
IEnumerable<string> currentRulesRaw =
|
||||
snapshot.Content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.None);
|
||||
|
||||
var existingCurrentRules = filterListsDbContext.Rules.Where(x => currentRulesRaw.Contains(x.Raw));
|
||||
|
||||
var deletedRules = cachedRules.Except(existingCurrentRules).ToList();
|
||||
|
||||
filterListsDbContext.FilterListRules.RemoveRange(filterListsDbContext.FilterListRules
|
||||
.Where(x => deletedRules.Select(y => y.Id).Contains(x.RuleId))
|
||||
.Where(x => x.FilterListId == snapshot.FilterListId));
|
||||
|
||||
//TODO: update FilterList.UpdatedDateUtc
|
||||
|
||||
foreach (var line in currentRulesRaw)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(line)) continue;
|
||||
var rule = new Rule {Raw = line};
|
||||
if (!await filterListsDbContext.Rules.AnyAsync(x => x.Raw == rule.Raw))
|
||||
filterListsDbContext.Rules.Add(rule);
|
||||
var filterListRule = new FilterListRule {FilterList = list, Rule = rule};
|
||||
var filterListRule = new FilterListRule {FilterListId = snapshot.FilterListId, Rule = rule};
|
||||
if (!await filterListsDbContext.FilterListRules.AnyAsync(x =>
|
||||
x.RuleId == rule.Id && x.FilterListId == list.Id))
|
||||
x.RuleId == rule.Id && x.FilterListId == snapshot.FilterListId))
|
||||
filterListsDbContext.FilterListRules.Add(filterListRule);
|
||||
filterListsDbContext.FilterListRules.Add(filterListRule);
|
||||
}
|
||||
|
|
@ -81,5 +107,17 @@ private async Task AddRules(FilterList list, string snapshot)
|
|||
//TODO: log exception
|
||||
}
|
||||
}
|
||||
|
||||
private class FilterListDto
|
||||
{
|
||||
public int Id { get; set; }
|
||||
public string ViewUrl { get; set; }
|
||||
}
|
||||
|
||||
private class Snapshot
|
||||
{
|
||||
public string Content { get; set; }
|
||||
public int FilterListId { get; set; }
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue