minor cleanup

This commit is contained in:
Collin Barrett 2018-08-05 16:20:19 -05:00
parent c080673fc1
commit 70aaad38e5
17 changed files with 122 additions and 129 deletions

View file

@ -12,59 +12,53 @@ namespace FilterLists.Agent
public static class Program
{
private const int BatchSize = 1;
private static TelemetryClient _telemetryClient;
private static ServiceProvider _serviceProvider;
private static IConfigurationRoot _configurationRoot;
private const string AiConfigSetting = "ApplicationInsights:InstrumentationKey";
private static IConfigurationRoot configurationRoot;
private static ServiceProvider serviceProvider;
private static TelemetryClient telemetryClient;
public static int Main()
public static void Main()
{
InstantiateConfigurationRoot();
BuildConfigurationRoot();
InstantiateTelemetryClient();
InstantiateServiceProvider();
BuildServiceProvider();
CaptureSnapshots(BatchSize);
return 0;
}
private static void InstantiateConfigurationRoot()
private static void BuildConfigurationRoot()
{
_configurationRoot = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json", false, true)
.Build();
configurationRoot = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json", true)
.Build();
}
private static void InstantiateTelemetryClient()
{
TelemetryConfiguration.Active.InstrumentationKey =
_configurationRoot["ApplicationInsights:InstrumentationKey"];
_telemetryClient = new TelemetryClient();
TelemetryConfiguration.Active.InstrumentationKey = configurationRoot[AiConfigSetting];
telemetryClient = new TelemetryClient();
}
private static void InstantiateServiceProvider()
private static void BuildServiceProvider()
{
var serviceCollection = new ServiceCollection();
ConfigureServices(serviceCollection);
_serviceProvider = serviceCollection.BuildServiceProvider();
}
private static void ConfigureServices(IServiceCollection serviceCollection)
{
serviceCollection.AddFilterListsServices(_configurationRoot);
serviceCollection.AddFilterListsServices(configurationRoot);
serviceProvider = serviceCollection.BuildServiceProvider();
}
private static void CaptureSnapshots(int batchSize)
{
var snapshotService = _serviceProvider.GetService<SnapshotService>();
var snapshotService = serviceProvider.GetService<SnapshotService>();
Log("Capturing FilterList snapshots...");
snapshotService.CaptureAsync(batchSize).Wait();
Log("\nSnapshots captured.");
_telemetryClient.Flush();
Log(Environment.NewLine + "Snapshots captured.");
telemetryClient.Flush();
}
private static void Log(string message)
{
Console.WriteLine(message);
_telemetryClient.TrackTrace(message);
telemetryClient.TrackTrace(message);
}
}
}

View file

@ -9,20 +9,20 @@ namespace FilterLists.Api.V1.Controllers
{
public class ListsController : BaseController
{
private readonly FilterListService _filterListService;
private readonly FilterListService filterListService;
public ListsController(SeedService seedService, FilterListService filterListService) : base(seedService)
{
_filterListService = filterListService;
this.filterListService = filterListService;
}
[HttpGet]
public async Task<IActionResult> Index() => Json(await _filterListService.GetAllSummariesAsync());
public async Task<IActionResult> Index() => Json(await filterListService.GetAllSummariesAsync());
[HttpGet]
[Route("{id}")]
//TODO: respond with appropriate exception if negative id queried
public async Task<IActionResult> GetById(int id) => Json(await _filterListService.GetDetailsAsync((uint)id));
public async Task<IActionResult> GetById(int id) => Json(await filterListService.GetDetailsAsync((uint)id));
[HttpGet("seed")]
public async Task<IActionResult> Seed() => Json(await SeedService.GetAllAsync<FilterList, FilterListSeedDto>());

View file

@ -6,14 +6,14 @@ namespace FilterLists.Api.V1.Controllers
{
public class RulesController : BaseController
{
private readonly RuleService _ruleService;
private readonly RuleService ruleService;
public RulesController(RuleService ruleService)
{
_ruleService = ruleService;
this.ruleService = ruleService;
}
[HttpGet]
public async Task<IActionResult> Index() => Json(await _ruleService.GetCountAll());
public async Task<IActionResult> Index() => Json(await ruleService.GetCountAll());
}
}

View file

@ -2,7 +2,7 @@
namespace FilterLists.Data.Entities
{
public class BaseEntity
public class BaseEntity : IBaseEntity
{
public uint Id { get; set; }
public DateTime CreatedDateUtc { get; set; }

View file

@ -0,0 +1,9 @@
using System;
namespace FilterLists.Data.Entities
{
public interface IBaseEntity
{
DateTime CreatedDateUtc { get; set; }
}
}

View file

@ -2,7 +2,7 @@
namespace FilterLists.Data.Entities.Junctions
{
public class BaseJunction
public class BaseJunctionEntity : IBaseEntity
{
public DateTime CreatedDateUtc { get; set; }
}

View file

@ -1,6 +1,6 @@
namespace FilterLists.Data.Entities.Junctions
{
public class FilterListLanguage : BaseJunction
public class FilterListLanguage : BaseJunctionEntity
{
public uint FilterListId { get; set; }
public FilterList FilterList { get; set; }

View file

@ -1,6 +1,6 @@
namespace FilterLists.Data.Entities.Junctions
{
public class FilterListMaintainer : BaseJunction
public class FilterListMaintainer : BaseJunctionEntity
{
public uint FilterListId { get; set; }
public FilterList FilterList { get; set; }

View file

@ -1,6 +1,6 @@
namespace FilterLists.Data.Entities.Junctions
{
public class Fork : BaseJunction
public class Fork : BaseJunctionEntity
{
public uint ForkFilterListId { get; set; }
public FilterList ForkFilterList { get; set; }

View file

@ -1,6 +1,6 @@
namespace FilterLists.Data.Entities.Junctions
{
public class Merge : BaseJunction
public class Merge : BaseJunctionEntity
{
public uint MergeFilterListId { get; set; }
public FilterList MergeFilterList { get; set; }

View file

@ -2,7 +2,7 @@
namespace FilterLists.Data.Entities.Junctions
{
public class SnapshotRule : BaseJunction
public class SnapshotRule : BaseJunctionEntity
{
public DateTime ModifiedDateUtc { get; set; }
public uint AddedBySnapshotId { get; set; }

View file

@ -1,6 +1,6 @@
namespace FilterLists.Data.Entities.Junctions
{
public class SoftwareSyntax : BaseJunction
public class SoftwareSyntax : BaseJunctionEntity
{
public uint SoftwareId { get; set; }
public Software Software { get; set; }

View file

@ -5,7 +5,7 @@
namespace FilterLists.Data.EntityTypeConfigurations.Junctions
{
public class BaseJunctionTypeConfiguration<TJunction> : IEntityTypeConfiguration<TJunction>
where TJunction : BaseJunction
where TJunction : BaseJunctionEntity
{
public virtual void Configure(EntityTypeBuilder<TJunction> entityTypeBuilder)
{

View file

@ -12,27 +12,6 @@ public FilterListsDbContext(DbContextOptions options) : base(options)
{
}
protected override void OnModelCreating(ModelBuilder modelBuilder)
{
base.OnModelCreating(modelBuilder);
ApplyConfigurationsEntities(modelBuilder);
ApplyConfigurationsJunctions(modelBuilder);
}
#region Entities
private static void ApplyConfigurationsEntities(ModelBuilder modelBuilder)
{
modelBuilder.ApplyConfiguration(new FilterListTypeConfiguration());
modelBuilder.ApplyConfiguration(new LanguageTypeConfiguration());
modelBuilder.ApplyConfiguration(new LicenseTypeConfiguration());
modelBuilder.ApplyConfiguration(new MaintainerTypeConfiguration());
modelBuilder.ApplyConfiguration(new RuleTypeConfiguration());
modelBuilder.ApplyConfiguration(new SnapshotTypeConfiguration());
modelBuilder.ApplyConfiguration(new SoftwareTypeConfiguration());
modelBuilder.ApplyConfiguration(new SyntaxTypeConfiguration());
}
public DbSet<FilterList> FilterLists { get; set; }
public DbSet<Language> Languages { get; set; }
public DbSet<License> Licenses { get; set; }
@ -42,20 +21,6 @@ private static void ApplyConfigurationsEntities(ModelBuilder modelBuilder)
public DbSet<Software> Software { get; set; }
public DbSet<Syntax> Syntaxes { get; set; }
#endregion
#region Junctions
private static void ApplyConfigurationsJunctions(ModelBuilder modelBuilder)
{
modelBuilder.ApplyConfiguration(new FilterListLanguageTypeConfiguration());
modelBuilder.ApplyConfiguration(new FilterListMaintainerTypeConfiguration());
modelBuilder.ApplyConfiguration(new ForkTypeConfiguration());
modelBuilder.ApplyConfiguration(new MergeTypeConfiguration());
modelBuilder.ApplyConfiguration(new SnapshotRuleTypeConfiguration());
modelBuilder.ApplyConfiguration(new SoftwareSyntaxTypeConfiguration());
}
public DbSet<FilterListLanguage> FilterListLanguages { get; set; }
public DbSet<FilterListMaintainer> FilterListMaintainers { get; set; }
public DbSet<Fork> Forks { get; set; }
@ -63,6 +28,29 @@ private static void ApplyConfigurationsJunctions(ModelBuilder modelBuilder)
public DbSet<SnapshotRule> SnapshotRules { get; set; }
public DbSet<SoftwareSyntax> SoftwareSyntaxes { get; set; }
#endregion
protected override void OnModelCreating(ModelBuilder modelBuilder)
{
base.OnModelCreating(modelBuilder);
ApplyConfigurations(modelBuilder);
}
private static void ApplyConfigurations(ModelBuilder modelBuilder)
{
modelBuilder.ApplyConfiguration(new FilterListTypeConfiguration());
modelBuilder.ApplyConfiguration(new LanguageTypeConfiguration());
modelBuilder.ApplyConfiguration(new LicenseTypeConfiguration());
modelBuilder.ApplyConfiguration(new MaintainerTypeConfiguration());
modelBuilder.ApplyConfiguration(new RuleTypeConfiguration());
modelBuilder.ApplyConfiguration(new SnapshotTypeConfiguration());
modelBuilder.ApplyConfiguration(new SoftwareTypeConfiguration());
modelBuilder.ApplyConfiguration(new SyntaxTypeConfiguration());
modelBuilder.ApplyConfiguration(new FilterListLanguageTypeConfiguration());
modelBuilder.ApplyConfiguration(new FilterListMaintainerTypeConfiguration());
modelBuilder.ApplyConfiguration(new ForkTypeConfiguration());
modelBuilder.ApplyConfiguration(new MergeTypeConfiguration());
modelBuilder.ApplyConfiguration(new SnapshotRuleTypeConfiguration());
modelBuilder.ApplyConfiguration(new SoftwareSyntaxTypeConfiguration());
}
}
}

View file

@ -27,12 +27,12 @@ public static void SeedOrUpdate(this FilterListsDbContext dbContext, string data
dbContext.InsertOnDuplicateKeyUpdate<SoftwareSyntax>(dataPath);
}
private static void InsertOnDuplicateKeyUpdate<TEntityType>(this DbContext dbContext, string dataPath)
where TEntityType : class
private static void InsertOnDuplicateKeyUpdate<TEntity>(this DbContext dbContext, string dataPath)
where TEntity : IBaseEntity
{
var entityType = dbContext.Model.FindEntityType(typeof(TEntityType));
var entityType = dbContext.Model.FindEntityType(typeof(TEntity));
var properties = GetPropertiesLessValueGeneratedTimestamps(entityType);
var values = CreateValues<TEntityType>(properties, dataPath);
var values = CreateValues<TEntity>(properties, dataPath);
if (values == "") return;
var columns = string.Join(", ", properties.Select(x => x.Name));
var updates = CreateUpdates(properties);
@ -50,28 +50,30 @@ private static List<IProperty> GetPropertiesLessValueGeneratedTimestamps(IEntity
.ToList();
}
private static string CreateValues<TEntityType>(IReadOnlyCollection<IProperty> properties, string dataPath)
private static string CreateValues<TEntity>(IReadOnlyCollection<IProperty> properties, string dataPath)
where TEntity : IBaseEntity
{
return GetSeedRows<TEntityType>(dataPath)
return GetSeedRows<TEntity>(dataPath)
.Select(row => CreateRowValues(properties, row))
.Aggregate("", (current, rowValues) => current == "" ? rowValues : current + ", " + rowValues);
}
private static List<TEntityType> GetSeedRows<TEntityType>(string dataPath)
private static List<TEntity> GetSeedRows<TEntity>(string dataPath) where TEntity : IBaseEntity
{
try
{
return JsonConvert.DeserializeObject<List<TEntityType>>(
File.ReadAllText(dataPath + Path.DirectorySeparatorChar + typeof(TEntityType).Name + ".json"));
return JsonConvert.DeserializeObject<List<TEntity>>(
File.ReadAllText(dataPath + Path.DirectorySeparatorChar + typeof(TEntity).Name + ".json"));
}
catch (FileNotFoundException e)
{
Console.WriteLine(e.Message);
return new List<TEntityType>();
return new List<TEntity>();
}
}
private static string CreateRowValues<TEntityType>(IEnumerable<IProperty> properties, TEntityType row)
private static string CreateRowValues<TEntity>(IEnumerable<IProperty> properties, TEntity row)
where TEntity : IBaseEntity
{
return (from property in properties
let value = row.GetType().GetProperty(property.Name)?.GetValue(row)

View file

@ -10,43 +10,43 @@ namespace FilterLists.Services.Snapshot
{
public class SnapshotBatchDe
{
private readonly FilterListsDbContext _dbContext;
private readonly IEnumerable<string> _rawRules;
private readonly Data.Entities.Snapshot _snapshot;
private IQueryable<Rule> _rules;
private readonly FilterListsDbContext dbContext;
private readonly IEnumerable<string> rawRules;
private readonly Data.Entities.Snapshot snapshot;
private IQueryable<Rule> rules;
public SnapshotBatchDe(FilterListsDbContext dbContext, Data.Entities.Snapshot snapshot,
IEnumerable<string> rawRules)
{
_dbContext = dbContext;
_snapshot = snapshot;
_rawRules = rawRules;
this.dbContext = dbContext;
this.snapshot = snapshot;
this.rawRules = rawRules;
}
public async Task SaveSnapshotBatchAsync()
{
AddRules();
AddSnapshotRules();
await _dbContext.SaveChangesAsync();
await dbContext.SaveChangesAsync();
}
private void AddRules()
{
var existingRules = _dbContext.Rules.Where(rule => _rawRules.Contains(rule.Raw));
var newRawRules = _rawRules.Except(existingRules.Select(r => r.Raw));
var existingRules = dbContext.Rules.Where(rule => rawRules.Contains(rule.Raw));
var newRawRules = rawRules.Except(existingRules.Select(r => r.Raw));
var newRules = newRawRules.Select(newRawRule => new Rule {Raw = newRawRule}).ToList();
_dbContext.Rules.AddRange(newRules);
_rules = existingRules.Concat(newRules);
dbContext.Rules.AddRange(newRules);
rules = existingRules.Concat(newRules);
}
private void AddSnapshotRules()
{
var snapshotRules = new List<SnapshotRule>();
foreach (var rule in _rules)
foreach (var rule in rules)
snapshotRules.Add(new SnapshotRule {Rule = rule});
if (_snapshot.AddedSnapshotRules == null)
_snapshot.AddedSnapshotRules = new List<SnapshotRule>();
_snapshot.AddedSnapshotRules.AddRange(snapshotRules);
if (snapshot.AddedSnapshotRules == null)
snapshot.AddedSnapshotRules = new List<SnapshotRule>();
snapshot.AddedSnapshotRules.AddRange(snapshotRules);
}
}
}

View file

@ -16,14 +16,14 @@ public class SnapshotDe
@"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36";
private const int BatchSize = 1000;
private readonly FilterListsDbContext _dbContext;
private readonly FilterListViewUrlDto _list;
private Data.Entities.Snapshot _snapshot;
private readonly FilterListsDbContext dbContext;
private readonly FilterListViewUrlDto list;
private Data.Entities.Snapshot snapshot;
public SnapshotDe(FilterListsDbContext dbContext, FilterListViewUrlDto list)
{
_dbContext = dbContext;
_list = list;
this.dbContext = dbContext;
this.list = list;
}
public async Task SaveSnapshotAsync()
@ -42,14 +42,14 @@ private async Task<string> CaptureSnapshot()
{
await AddSnapshot();
var content = await TryGetContent();
await _dbContext.SaveChangesAsync();
await dbContext.SaveChangesAsync();
return content;
}
private async Task AddSnapshot()
{
_snapshot = new Data.Entities.Snapshot {FilterListId = _list.Id};
await _dbContext.Snapshots.AddAsync(_snapshot);
snapshot = new Data.Entities.Snapshot {FilterListId = list.Id};
await dbContext.Snapshots.AddAsync(snapshot);
}
private async Task<string> TryGetContent()
@ -60,13 +60,13 @@ private async Task<string> TryGetContent()
}
catch (WebException we)
{
_snapshot.HttpStatusCode = ((int) ((HttpWebResponse) we.Response).StatusCode).ToString();
snapshot.HttpStatusCode = ((int) ((HttpWebResponse) we.Response).StatusCode).ToString();
return null;
}
catch (Exception)
{
//TODO: log exception (#148)
_snapshot.HttpStatusCode = null;
snapshot.HttpStatusCode = null;
return null;
}
}
@ -76,9 +76,9 @@ private async Task<string> GetContent()
using (var httpClient = new HttpClient())
{
httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentString);
using (var httpResponseMessage = await httpClient.GetAsync(_list.ViewUrl))
using (var httpResponseMessage = await httpClient.GetAsync(list.ViewUrl))
{
_snapshot.HttpStatusCode = ((int) httpResponseMessage.StatusCode).ToString();
snapshot.HttpStatusCode = ((int) httpResponseMessage.StatusCode).ToString();
if (httpResponseMessage.IsSuccessStatusCode)
return await httpResponseMessage.Content.ReadAsStringAsync();
}
@ -105,7 +105,7 @@ private static IEnumerable<string> GetRawRules(string content)
private IEnumerable<SnapshotBatchDe> GetSnapshotBatches(IEnumerable<string> rawRules)
{
return rawRules.Batch(BatchSize)
.Select(rawRuleBatch => new SnapshotBatchDe(_dbContext, _snapshot, rawRuleBatch));
.Select(rawRuleBatch => new SnapshotBatchDe(dbContext, snapshot, rawRuleBatch));
}
private static async Task SaveSnapshotBatches(IEnumerable<SnapshotBatchDe> snapshotBatches)
@ -119,36 +119,36 @@ private async Task DedupSnapshotRules()
var existingSnapshotRules = GetExistingSnapshotRules();
UpdateRemovedSnapshotRules(existingSnapshotRules);
RemoveDuplicateSnapshotRules(existingSnapshotRules);
await _dbContext.SaveChangesAsync();
await dbContext.SaveChangesAsync();
}
private IQueryable<SnapshotRule> GetExistingSnapshotRules()
{
return _dbContext.SnapshotRules.Where(sr =>
sr.AddedBySnapshot.FilterListId == _list.Id && sr.AddedBySnapshot != _snapshot &&
return dbContext.SnapshotRules.Where(sr =>
sr.AddedBySnapshot.FilterListId == list.Id && sr.AddedBySnapshot != snapshot &&
sr.RemovedBySnapshot == null);
}
private void UpdateRemovedSnapshotRules(IQueryable<SnapshotRule> existingSnapshotRules)
{
var newSnapshotRules = _dbContext.SnapshotRules.Where(sr => sr.AddedBySnapshot == _snapshot);
var newSnapshotRules = dbContext.SnapshotRules.Where(sr => sr.AddedBySnapshot == snapshot);
var removedSnapshotRules =
existingSnapshotRules.Where(sr => !newSnapshotRules.Any(nsr => nsr.Rule == sr.Rule));
removedSnapshotRules.ToList().ForEach(sr => sr.RemovedBySnapshot = _snapshot);
removedSnapshotRules.ToList().ForEach(sr => sr.RemovedBySnapshot = snapshot);
}
private void RemoveDuplicateSnapshotRules(IQueryable<SnapshotRule> existingSnapshotRules)
{
var duplicateSnapshotRules = _dbContext.SnapshotRules.Where(sr =>
sr.AddedBySnapshot == _snapshot &&
var duplicateSnapshotRules = dbContext.SnapshotRules.Where(sr =>
sr.AddedBySnapshot == snapshot &&
existingSnapshotRules.Any(esr => esr.Rule == sr.Rule));
_dbContext.SnapshotRules.RemoveRange(duplicateSnapshotRules);
dbContext.SnapshotRules.RemoveRange(duplicateSnapshotRules);
}
private async Task SetCompleted()
{
_snapshot.IsCompleted = true;
await _dbContext.SaveChangesAsync();
snapshot.IsCompleted = true;
await dbContext.SaveChangesAsync();
}
}
}