prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

add async seeding in Main()

Browse source
This commit is contained in:
thuanpt 2018-10-01 14:37:51 +07:00
parent 6a627146d9
commit da8a8c6cd0
5 changed files with 111 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
src/FilterLists.Api/DependencyInjection/Extensions/IWebHostExtension.cs Normal file
View file

@ -0,0 +1,59 @@
using System;
using System.Data.SqlClient;
using Microsoft.AspNetCore.Hosting;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Polly;
namespace FilterLists.Api.DependencyInjection.Extensions
{
public static class IWebHostExtension
{
public static IWebHost MigrateDbContext<TContext>(this IWebHost webHost, Action<TContext,IServiceProvider> seeder) where TContext : DbContext
{
using (var scope = webHost.Services.CreateScope())
{
var services = scope.ServiceProvider;
var logger = services.GetRequiredService<ILogger<TContext>>();
var context = services.GetService<TContext>();
try
{
logger.LogInformation($"Migrating database associated with context {typeof(TContext).Name}");
var retry = Policy.Handle<SqlException>()
.WaitAndRetry(new TimeSpan[]
{
TimeSpan.FromSeconds(3),
TimeSpan.FromSeconds(5),
TimeSpan.FromSeconds(8),
});
retry.Execute(() =>
{
//if the sql server container is not created on run docker compose this
//migration can't fail for network related exception. The retry options for DbContext only
//apply to transient exceptions.
context.Database
.Migrate();
seeder(context, services);
});
logger.LogInformation($"Migrated database associated with context {typeof(TContext).Name}");
}
catch (Exception ex)
{
logger.LogError(ex, $"An error occurred while migrating the database used on context {typeof(TContext).Name}");
}
}
return webHost;
}
}
}

1
src/FilterLists.Api/FilterLists.Api.csproj
View file

@ -41,6 +41,7 @@
<PackageReference Include="Microsoft.AspNetCore.App" Version="2.1.4" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.Versioning" Version="2.3.0" />
<PackageReference Include="Microsoft.DotNet.Analyzers.Compatibility" Version="0.2.12-alpha" />
<PackageReference Include="Polly" Version="6.1.0" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="3.0.0" />
</ItemGroup>

13
src/FilterLists.Api/Program.cs
View file

@ -1,12 +1,23 @@
using Microsoft.AspNetCore;
using Microsoft.AspNetCore.Hosting;
using FilterLists.Api.DependencyInjection.Extensions;
using FilterLists.Data.Seed.Extensions;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
namespace FilterLists.Api
{
public static class Program
{
public static void Main(string[] args) => CreateWebHostBuilder(args).Build().Run();
public static void Main(string[] args)
{
CreateWebHostBuilder(args).Build().MigrateDbContext<Data.FilterListsDbContext>((context,service)=>
{
var dataPath = service.GetService<IConfiguration>()["DataDirectory:Path"].ToString();
new SeedFilterListsDbContext().SeedOrUpdateAsync(context,dataPath).Wait();
}).Run();
}
private static IWebHostBuilder CreateWebHostBuilder(string[] args) =>
WebHost.CreateDefaultBuilder(args)
.UseUrls("http://localhost:5000")

13
src/FilterLists.Api/Startup.cs
View file

@ -26,6 +26,7 @@ public void ConfigureServices(IServiceCollection services)
{
services.AddFilterListsApiServices(Configuration);
services.AddFilterListsApi();
services.AddSingleton<IConfiguration> (Configuration);
}
[UsedImplicitly]
@ -65,7 +66,7 @@ public void Configure(IApplicationBuilder app, IHostingEnvironment env)
opts.DocumentTitle = "FilterLists API v1";
opts.RoutePrefix = "docs";
});
MigrateAndSeedDatabase(app);
// MigrateAndSeedDatabase(app);
}
//TODO: remove hack (https://github.com/domaindrivendev/Swashbuckle.AspNetCore/issues/74#issuecomment-386762178)
@ -77,15 +78,5 @@ private static void UseLowercaseControllerNameInSwaggerHack(SwaggerOptions opts)
foreach (var pathItem in paths) document.Paths.Add(pathItem.Key, pathItem.Value);
});
private void MigrateAndSeedDatabase(IApplicationBuilder app)
{
var dataPath = Configuration.GetSection("DataDirectory").GetValue<string>("Path");
using (var serviceScope = app.ApplicationServices.GetRequiredService<IServiceScopeFactory>().CreateScope())
{
var filterListsDbContext = serviceScope.ServiceProvider.GetService<FilterListsDbContext>();
filterListsDbContext.Database.Migrate();
filterListsDbContext.SeedOrUpdate(dataPath);
}
}
}
}

73
src/FilterLists.Data/Seed/Extensions/SeedFilterListsDbContext.cs
View file

@ -3,6 +3,7 @@
using System.IO;
using System.Linq;
using System.Linq.Expressions;
using System.Threading.Tasks;
using FilterLists.Data.Entities;
using FilterLists.Data.Entities.Junctions;
using Microsoft.EntityFrameworkCore;
@ -11,41 +12,41 @@
namespace FilterLists.Data.Seed.Extensions
{
public static class SeedFilterListsDbContext
public class SeedFilterListsDbContext
{
public static void SeedOrUpdate(this FilterListsDbContext dbContext, string dataPath)
public async Task SeedOrUpdateAsync(FilterListsDbContext dbContext, string dataPath)
{
dbContext.SeedOrUpdate<Language>(dataPath);
dbContext.SeedOrUpdate<License>(dataPath);
dbContext.SeedOrUpdate<Maintainer>(dataPath);
dbContext.SeedOrUpdate<Software>(dataPath);
dbContext.SeedOrUpdate<Syntax>(dataPath);
dbContext.SeedOrUpdate<Tag>(dataPath);
dbContext.SeedOrUpdate<FilterList>(dataPath);
dbContext.SetDefaultLicenseId();
dbContext.SeedOrUpdate<FilterListLanguage>(dataPath);
dbContext.SeedOrUpdate<FilterListMaintainer>(dataPath);
dbContext.SeedOrUpdate<FilterListTag>(dataPath);
dbContext.SeedOrUpdate<Dependent>(dataPath);
dbContext.SeedOrUpdate<Fork>(dataPath);
dbContext.SeedOrUpdate<Merge>(dataPath);
dbContext.SeedOrUpdate<SoftwareSyntax>(dataPath);
await SeedOrUpdateAsync<Language>(dbContext,dataPath);
await SeedOrUpdateAsync<License>(dbContext, dataPath);
await SeedOrUpdateAsync<Maintainer>(dbContext,dataPath);
await SeedOrUpdateAsync<Software>(dbContext,dataPath);
await SeedOrUpdateAsync<Syntax>(dbContext,dataPath);
await SeedOrUpdateAsync<Tag>(dbContext,dataPath);
await SeedOrUpdateAsync<FilterList>(dbContext,dataPath);
await SetDefaultLicenseIdAsync(dbContext);
await SeedOrUpdateAsync<FilterListLanguage>(dbContext,dataPath);
await SeedOrUpdateAsync<FilterListMaintainer>(dbContext,dataPath);
await SeedOrUpdateAsync<FilterListTag>(dbContext,dataPath);
await SeedOrUpdateAsync<Dependent>(dbContext,dataPath);
await SeedOrUpdateAsync<Fork>(dbContext,dataPath);
await SeedOrUpdateAsync<Merge>(dbContext,dataPath);
await SeedOrUpdateAsync<SoftwareSyntax>(dbContext,dataPath);
}
private static void SeedOrUpdate<TEntity>(this DbContext dbContext, string dataPath)
private async Task SeedOrUpdateAsync<TEntity>(DbContext dbContext, string dataPath)
where TEntity : class, IBaseEntity
{
var seed = GetSeed<TEntity>(dataPath);
ApplyRemovals(dbContext, seed);
InsertOnDuplicateKeyUpdate(dbContext, seed);
await ApplyRemovals(dbContext, seed);
await InsertOnDuplicateKeyUpdate(dbContext, seed);
}
private static List<TEntity> GetSeed<TEntity>(string dataPath) where TEntity : IBaseEntity
private List<TEntity> GetSeed<TEntity>(string dataPath) where TEntity : IBaseEntity
{
try
{
return JsonConvert.DeserializeObject<List<TEntity>>(
File.ReadAllText(dataPath + Path.DirectorySeparatorChar + typeof(TEntity).Name + ".json"));
File.ReadAllText(Path.Combine(dataPath,typeof(TEntity).Name + ".json")));
}
catch (FileNotFoundException e)
{
@ -54,16 +55,16 @@ private static List<TEntity> GetSeed<TEntity>(string dataPath) where TEntity : I
}
}
private static void ApplyRemovals<TEntity>(DbContext dbContext, IEnumerable<TEntity> seed)
private async Task ApplyRemovals<TEntity>(DbContext dbContext, IEnumerable<TEntity> seed)
where TEntity : class
{
var removedEntities = GetRemovedEntities(dbContext, seed);
dbContext.RemoveRange(removedEntities);
dbContext.SaveChanges();
await dbContext.SaveChangesAsync();
}
//https://stackoverflow.com/a/52264468/2343739
private static IEnumerable<TEntity> GetRemovedEntities<TEntity>(DbContext dbContext, IEnumerable<TEntity> seed)
private IEnumerable<TEntity> GetRemovedEntities<TEntity>(DbContext dbContext, IEnumerable<TEntity> seed)
where TEntity : class
{
var entityType = dbContext.Model.FindEntityType(typeof(TEntity));
@ -73,7 +74,7 @@ private static IEnumerable<TEntity> GetRemovedEntities<TEntity>(DbContext dbCont
return dbContext.Set<TEntity>().Where(notInSeed);
}
private static Expression GetMatchesAnyPk<TEntity>(IEnumerable<TEntity> seed, IEntityType entityType,
private Expression GetMatchesAnyPk<TEntity>(IEnumerable<TEntity> seed, IEntityType entityType,
Expression dbEntity)
where TEntity : class =>
seed.Select(e => entityType.FindPrimaryKey()
@ -86,7 +87,7 @@ private static Expression GetMatchesAnyPk<TEntity>(IEnumerable<TEntity> seed, IE
(Func<Expression, BinaryExpression, Expression>)((current, match) =>
current != null ? Expression.OrElse(current, match) : match));
private static void InsertOnDuplicateKeyUpdate<TEntity>(DbContext dbContext, IEnumerable<TEntity> seed)
private async Task InsertOnDuplicateKeyUpdate<TEntity>(DbContext dbContext, IEnumerable<TEntity> seed)
where TEntity : IBaseEntity
{
var entityType = dbContext.Model.FindEntityType(typeof(TEntity));
@ -98,27 +99,27 @@ private static void InsertOnDuplicateKeyUpdate<TEntity>(DbContext dbContext, IEn
var updates = CreateUpdates(properties);
var sql = "INSERT INTO " + entityType.Relational().TableName + " (" + columns + ") VALUES " + values +
" ON DUPLICATE KEY UPDATE " + updates;
dbContext.Database.ExecuteSqlCommand(sql);
await dbContext.Database.ExecuteSqlCommandAsync(sql);
}
private static List<IProperty> GetPropertiesLessValueGeneratedTimestamps(IEntityType entityType) =>
private List<IProperty> GetPropertiesLessValueGeneratedTimestamps(IEntityType entityType) =>
entityType.GetProperties()
.Where(x => !new List<string> {"CreatedDateUtc", "ModifiedDateUtc"}.Contains(x.Name))
.ToList();
private static string CreateValues<TEntity>(IEnumerable<TEntity> seed,
private string CreateValues<TEntity>(IEnumerable<TEntity> seed,
IReadOnlyCollection<IProperty> properties) where TEntity : IBaseEntity =>
seed.Select(row => CreateRowValues(properties, row))
.Aggregate("", (current, rowValues) => current == "" ? rowValues : current + ", " + rowValues);
private static string CreateRowValues<TEntity>(IEnumerable<IProperty> properties, TEntity row)
private string CreateRowValues<TEntity>(IEnumerable<IProperty> properties, TEntity row)
where TEntity : IBaseEntity =>
(from property in properties
let value = row.GetType().GetProperty(property.Name)?.GetValue(row)
select FormatDataForMySql(property, value)).Aggregate("",
(rowValues, value) => rowValues == "" ? "(" + value : rowValues + ", " + value) + ")";
private static object FormatDataForMySql(IProperty property, object value)
private object FormatDataForMySql(IProperty property, object value)
{
if (value == null)
return "NULL";
@ -131,7 +132,7 @@ private static object FormatDataForMySql(IProperty property, object value)
return value;
}
private static string CreateUpdates(IReadOnlyCollection<IProperty> properties)
private string CreateUpdates(IReadOnlyCollection<IProperty> properties)
{
var update =
(from property in properties
@ -143,13 +144,13 @@ private static string CreateUpdates(IReadOnlyCollection<IProperty> properties)
return update;
}
private static string GetUpdateUnchangedColumnHack(IEnumerable<IProperty> properties)
private string GetUpdateUnchangedColumnHack(IEnumerable<IProperty> properties)
{
var firstId = properties.First(x => x.IsPrimaryKey()).Name;
return firstId + " = VALUES(" + firstId + ")";
}
private static void SetDefaultLicenseId(this DbContext dbContext)
private async Task SetDefaultLicenseIdAsync(DbContext dbContext)
{
var listsWithoutLicense = dbContext.Set<FilterList>()
.Where(l => l.LicenseId == null)
@ -160,7 +161,7 @@ private static void SetDefaultLicenseId(this DbContext dbContext)
return l;
});
dbContext.UpdateRange(listsWithoutLicense);
dbContext.SaveChanges();
await dbContext.SaveChangesAsync();
}
}
}