prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

seed async and sooner

Browse source 
closes #540
closes #541
This commit is contained in:
Collin M. Barrett 2018-10-01 08:40:19 -05:00
parent da8a8c6cd0
commit b20fed904c
5 changed files with 54 additions and 93 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
src/FilterLists.Api/DependencyInjection/Extensions/IWebHostExtension.cs
View file

@ -1,56 +1,21 @@
using System;
using System.Data.SqlClient;
using Microsoft.AspNetCore.Hosting;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Polly;
namespace FilterLists.Api.DependencyInjection.Extensions
{
public static class IWebHostExtension
public static class IWebHostExtension
{
public static IWebHost MigrateDbContext<TContext>(this IWebHost webHost, Action<TContext,IServiceProvider> seeder) where TContext : DbContext
public static IWebHost MigrateAndSeedDbContext<TContext>(this IWebHost webHost,
Action<TContext, IServiceProvider> seeder) where TContext : DbContext
{
using (var scope = webHost.Services.CreateScope())
{
var services = scope.ServiceProvider;
var logger = services.GetRequiredService<ILogger<TContext>>();
var context = services.GetService<TContext>();
try
{
logger.LogInformation($"Migrating database associated with context {typeof(TContext).Name}");
var retry = Policy.Handle<SqlException>()
.WaitAndRetry(new TimeSpan[]
{
TimeSpan.FromSeconds(3),
TimeSpan.FromSeconds(5),
TimeSpan.FromSeconds(8),
});
retry.Execute(() =>
{
//if the sql server container is not created on run docker compose this
//migration can't fail for network related exception. The retry options for DbContext only
//apply to transient exceptions.
context.Database
.Migrate();
seeder(context, services);
});
logger.LogInformation($"Migrated database associated with context {typeof(TContext).Name}");
}
catch (Exception ex)
{
logger.LogError(ex, $"An error occurred while migrating the database used on context {typeof(TContext).Name}");
}
context.Database.Migrate();
seeder(context, services);
}
return webHost;

1
src/FilterLists.Api/FilterLists.Api.csproj
View file

@ -41,7 +41,6 @@
<PackageReference Include="Microsoft.AspNetCore.App" Version="2.1.4" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.Versioning" Version="2.3.0" />
<PackageReference Include="Microsoft.DotNet.Analyzers.Compatibility" Version="0.2.12-alpha" />
<PackageReference Include="Polly" Version="6.1.0" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="3.0.0" />
</ItemGroup>

22
src/FilterLists.Api/Program.cs
View file

@ -1,7 +1,8 @@
using Microsoft.AspNetCore;
using Microsoft.AspNetCore.Hosting;
using FilterLists.Api.DependencyInjection.Extensions;
using FilterLists.Api.DependencyInjection.Extensions;
using FilterLists.Data;
using FilterLists.Data.Seed.Extensions;
using Microsoft.AspNetCore;
using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
@ -11,13 +12,16 @@ public static class Program
{
public static void Main(string[] args)
{
CreateWebHostBuilder(args).Build().MigrateDbContext<Data.FilterListsDbContext>((context,service)=>
{
var dataPath = service.GetService<IConfiguration>()["DataDirectory:Path"].ToString();
new SeedFilterListsDbContext().SeedOrUpdateAsync(context,dataPath).Wait();
}).Run();
CreateWebHostBuilder(args)
.Build()
.MigrateAndSeedDbContext<FilterListsDbContext>((context, service) =>
{
var dataPath = service.GetService<IConfiguration>()["DataDirectory:Path"].ToString();
SeedFilterListsDbContext.SeedOrUpdateAsync(context, dataPath).Wait();
})
.Run();
}
private static IWebHostBuilder CreateWebHostBuilder(string[] args) =>
WebHost.CreateDefaultBuilder(args)
.UseUrls("http://localhost:5000")

7
src/FilterLists.Api/Startup.cs
View file

@ -1,13 +1,10 @@
using System.Linq;
using FilterLists.Api.DependencyInjection.Extensions;
using FilterLists.Data;
using FilterLists.Data.Seed.Extensions;
using FilterLists.Services.DependencyInjection.Extensions;
using JetBrains.Annotations;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.HttpOverrides;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Swashbuckle.AspNetCore.Swagger;
@ -26,7 +23,7 @@ public void ConfigureServices(IServiceCollection services)
{
services.AddFilterListsApiServices(Configuration);
services.AddFilterListsApi();
services.AddSingleton<IConfiguration> (Configuration);
services.AddSingleton(Configuration);
}
[UsedImplicitly]
@ -66,7 +63,6 @@ public void Configure(IApplicationBuilder app, IHostingEnvironment env)
opts.DocumentTitle = "FilterLists API v1";
opts.RoutePrefix = "docs";
});
// MigrateAndSeedDatabase(app);
}
//TODO: remove hack (https://github.com/domaindrivendev/Swashbuckle.AspNetCore/issues/74#issuecomment-386762178)
@ -77,6 +73,5 @@ private static void UseLowercaseControllerNameInSwaggerHack(SwaggerOptions opts)
document.Paths.Clear();
foreach (var pathItem in paths) document.Paths.Add(pathItem.Key, pathItem.Value);
});
}
}

72
src/FilterLists.Data/Seed/Extensions/SeedFilterListsDbContext.cs
View file

@ -12,28 +12,28 @@
namespace FilterLists.Data.Seed.Extensions
{
public class SeedFilterListsDbContext
public static class SeedFilterListsDbContext
{
public async Task SeedOrUpdateAsync(FilterListsDbContext dbContext, string dataPath)
public static async Task SeedOrUpdateAsync(FilterListsDbContext dbContext, string dataPath)
{
await SeedOrUpdateAsync<Language>(dbContext,dataPath);
await SeedOrUpdateAsync<License>(dbContext, dataPath);
await SeedOrUpdateAsync<Maintainer>(dbContext,dataPath);
await SeedOrUpdateAsync<Software>(dbContext,dataPath);
await SeedOrUpdateAsync<Syntax>(dbContext,dataPath);
await SeedOrUpdateAsync<Tag>(dbContext,dataPath);
await SeedOrUpdateAsync<FilterList>(dbContext,dataPath);
await SetDefaultLicenseIdAsync(dbContext);
await SeedOrUpdateAsync<FilterListLanguage>(dbContext,dataPath);
await SeedOrUpdateAsync<FilterListMaintainer>(dbContext,dataPath);
await SeedOrUpdateAsync<FilterListTag>(dbContext,dataPath);
await SeedOrUpdateAsync<Dependent>(dbContext,dataPath);
await SeedOrUpdateAsync<Fork>(dbContext,dataPath);
await SeedOrUpdateAsync<Merge>(dbContext,dataPath);
await SeedOrUpdateAsync<SoftwareSyntax>(dbContext,dataPath);
await SeedOrUpdate<Language>(dbContext, dataPath);
await SeedOrUpdate<License>(dbContext, dataPath);
await SeedOrUpdate<Maintainer>(dbContext, dataPath);
await SeedOrUpdate<Software>(dbContext, dataPath);
await SeedOrUpdate<Syntax>(dbContext, dataPath);
await SeedOrUpdate<Tag>(dbContext, dataPath);
await SeedOrUpdate<FilterList>(dbContext, dataPath);
await SetDefaultLicenseIdAsync(dbContext);
await SeedOrUpdate<FilterListLanguage>(dbContext, dataPath);
await SeedOrUpdate<FilterListMaintainer>(dbContext, dataPath);
await SeedOrUpdate<FilterListTag>(dbContext, dataPath);
await SeedOrUpdate<Dependent>(dbContext, dataPath);
await SeedOrUpdate<Fork>(dbContext, dataPath);
await SeedOrUpdate<Merge>(dbContext, dataPath);
await SeedOrUpdate<SoftwareSyntax>(dbContext, dataPath);
}
private async Task SeedOrUpdateAsync<TEntity>(DbContext dbContext, string dataPath)
private static async Task SeedOrUpdate<TEntity>(DbContext dbContext, string dataPath)
where TEntity : class, IBaseEntity
{
var seed = GetSeed<TEntity>(dataPath);
@ -41,12 +41,12 @@ private async Task SeedOrUpdateAsync<TEntity>(DbContext dbContext, string dataPa
await InsertOnDuplicateKeyUpdate(dbContext, seed);
}
private List<TEntity> GetSeed<TEntity>(string dataPath) where TEntity : IBaseEntity
private static List<TEntity> GetSeed<TEntity>(string dataPath) where TEntity : IBaseEntity
{
try
{
return JsonConvert.DeserializeObject<List<TEntity>>(
File.ReadAllText(Path.Combine(dataPath,typeof(TEntity).Name + ".json")));
File.ReadAllText(Path.Combine(dataPath, typeof(TEntity).Name + ".json")));
}
catch (FileNotFoundException e)
{
@ -55,7 +55,7 @@ private List<TEntity> GetSeed<TEntity>(string dataPath) where TEntity : IBaseEnt
}
}
private async Task ApplyRemovals<TEntity>(DbContext dbContext, IEnumerable<TEntity> seed)
private static async Task ApplyRemovals<TEntity>(DbContext dbContext, IEnumerable<TEntity> seed)
where TEntity : class
{
var removedEntities = GetRemovedEntities(dbContext, seed);
@ -64,7 +64,7 @@ private async Task ApplyRemovals<TEntity>(DbContext dbContext, IEnumerable<TEnti
}
//https://stackoverflow.com/a/52264468/2343739
private IEnumerable<TEntity> GetRemovedEntities<TEntity>(DbContext dbContext, IEnumerable<TEntity> seed)
private static IEnumerable<TEntity> GetRemovedEntities<TEntity>(DbContext dbContext, IEnumerable<TEntity> seed)
where TEntity : class
{
var entityType = dbContext.Model.FindEntityType(typeof(TEntity));
@ -74,20 +74,18 @@ private IEnumerable<TEntity> GetRemovedEntities<TEntity>(DbContext dbContext, IE
return dbContext.Set<TEntity>().Where(notInSeed);
}
private Expression GetMatchesAnyPk<TEntity>(IEnumerable<TEntity> seed, IEntityType entityType,
Expression dbEntity)
where TEntity : class =>
private static Expression GetMatchesAnyPk<TEntity>(IEnumerable<TEntity> seed, IEntityType entityType,
Expression dbEntity) where TEntity : class =>
seed.Select(e => entityType.FindPrimaryKey()
.Properties
.Select(p => Expression.Equal(
Expression.Property(dbEntity, p.PropertyInfo),
Expression.Property(Expression.Constant(e), p.PropertyInfo)))
.Properties.Select(p =>
Expression.Equal(Expression.Property(dbEntity, p.PropertyInfo),
Expression.Property(Expression.Constant(e), p.PropertyInfo)))
.Aggregate(Expression.AndAlso))
.Aggregate(null,
(Func<Expression, BinaryExpression, Expression>)((current, match) =>
current != null ? Expression.OrElse(current, match) : match));
private async Task InsertOnDuplicateKeyUpdate<TEntity>(DbContext dbContext, IEnumerable<TEntity> seed)
private static async Task InsertOnDuplicateKeyUpdate<TEntity>(DbContext dbContext, IEnumerable<TEntity> seed)
where TEntity : IBaseEntity
{
var entityType = dbContext.Model.FindEntityType(typeof(TEntity));
@ -102,24 +100,24 @@ private async Task InsertOnDuplicateKeyUpdate<TEntity>(DbContext dbContext, IEnu
await dbContext.Database.ExecuteSqlCommandAsync(sql);
}
private List<IProperty> GetPropertiesLessValueGeneratedTimestamps(IEntityType entityType) =>
private static List<IProperty> GetPropertiesLessValueGeneratedTimestamps(IEntityType entityType) =>
entityType.GetProperties()
.Where(x => !new List<string> {"CreatedDateUtc", "ModifiedDateUtc"}.Contains(x.Name))
.ToList();
private string CreateValues<TEntity>(IEnumerable<TEntity> seed,
private static string CreateValues<TEntity>(IEnumerable<TEntity> seed,
IReadOnlyCollection<IProperty> properties) where TEntity : IBaseEntity =>
seed.Select(row => CreateRowValues(properties, row))
.Aggregate("", (current, rowValues) => current == "" ? rowValues : current + ", " + rowValues);
private string CreateRowValues<TEntity>(IEnumerable<IProperty> properties, TEntity row)
private static string CreateRowValues<TEntity>(IEnumerable<IProperty> properties, TEntity row)
where TEntity : IBaseEntity =>
(from property in properties
let value = row.GetType().GetProperty(property.Name)?.GetValue(row)
select FormatDataForMySql(property, value)).Aggregate("",
(rowValues, value) => rowValues == "" ? "(" + value : rowValues + ", " + value) + ")";
private object FormatDataForMySql(IProperty property, object value)
private static object FormatDataForMySql(IProperty property, object value)
{
if (value == null)
return "NULL";
@ -132,7 +130,7 @@ private object FormatDataForMySql(IProperty property, object value)
return value;
}
private string CreateUpdates(IReadOnlyCollection<IProperty> properties)
private static string CreateUpdates(IReadOnlyCollection<IProperty> properties)
{
var update =
(from property in properties
@ -144,13 +142,13 @@ private string CreateUpdates(IReadOnlyCollection<IProperty> properties)
return update;
}
private string GetUpdateUnchangedColumnHack(IEnumerable<IProperty> properties)
private static string GetUpdateUnchangedColumnHack(IEnumerable<IProperty> properties)
{
var firstId = properties.First(x => x.IsPrimaryKey()).Name;
return firstId + " = VALUES(" + firstId + ")";
}
private async Task SetDefaultLicenseIdAsync(DbContext dbContext)
private static async Task SetDefaultLicenseIdAsync(DbContext dbContext)
{
var listsWithoutLicense = dbContext.Set<FilterList>()
.Where(l => l.LicenseId == null)