From da8a8c6cd09b7ad36e0402589fcd81e6164625ad Mon Sep 17 00:00:00 2001 From: thuanpt Date: Mon, 1 Oct 2018 14:37:51 +0700 Subject: [PATCH] add async seeding in Main() --- .../Extensions/IWebHostExtension.cs | 59 +++++++++++++++ src/FilterLists.Api/FilterLists.Api.csproj | 1 + src/FilterLists.Api/Program.cs | 13 +++- src/FilterLists.Api/Startup.cs | 13 +--- .../Extensions/SeedFilterListsDbContext.cs | 73 ++++++++++--------- 5 files changed, 111 insertions(+), 48 deletions(-) create mode 100644 src/FilterLists.Api/DependencyInjection/Extensions/IWebHostExtension.cs diff --git a/src/FilterLists.Api/DependencyInjection/Extensions/IWebHostExtension.cs b/src/FilterLists.Api/DependencyInjection/Extensions/IWebHostExtension.cs new file mode 100644 index 000000000..c7a3c8051 --- /dev/null +++ b/src/FilterLists.Api/DependencyInjection/Extensions/IWebHostExtension.cs @@ -0,0 +1,59 @@ +using System; +using System.Data.SqlClient; +using Microsoft.AspNetCore.Hosting; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Polly; + +namespace FilterLists.Api.DependencyInjection.Extensions +{ + public static class IWebHostExtension + { + public static IWebHost MigrateDbContext(this IWebHost webHost, Action seeder) where TContext : DbContext + { + using (var scope = webHost.Services.CreateScope()) + { + var services = scope.ServiceProvider; + + var logger = services.GetRequiredService>(); + + var context = services.GetService(); + + try + { + logger.LogInformation($"Migrating database associated with context {typeof(TContext).Name}"); + + var retry = Policy.Handle() + .WaitAndRetry(new TimeSpan[] + { + TimeSpan.FromSeconds(3), + TimeSpan.FromSeconds(5), + TimeSpan.FromSeconds(8), + }); + + retry.Execute(() => + { + //if the sql server container is not created on run docker compose this + //migration can't fail for network related exception. The retry options for DbContext only + //apply to transient exceptions. + + context.Database + .Migrate(); + + seeder(context, services); + }); + + + logger.LogInformation($"Migrated database associated with context {typeof(TContext).Name}"); + } + catch (Exception ex) + { + logger.LogError(ex, $"An error occurred while migrating the database used on context {typeof(TContext).Name}"); + } + } + + return webHost; + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Api/FilterLists.Api.csproj b/src/FilterLists.Api/FilterLists.Api.csproj index 1aec86167..9a055004f 100644 --- a/src/FilterLists.Api/FilterLists.Api.csproj +++ b/src/FilterLists.Api/FilterLists.Api.csproj @@ -41,6 +41,7 @@ + diff --git a/src/FilterLists.Api/Program.cs b/src/FilterLists.Api/Program.cs index 08be01525..6945694fb 100644 --- a/src/FilterLists.Api/Program.cs +++ b/src/FilterLists.Api/Program.cs @@ -1,12 +1,23 @@ using Microsoft.AspNetCore; using Microsoft.AspNetCore.Hosting; +using FilterLists.Api.DependencyInjection.Extensions; +using FilterLists.Data.Seed.Extensions; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; namespace FilterLists.Api { public static class Program { - public static void Main(string[] args) => CreateWebHostBuilder(args).Build().Run(); + public static void Main(string[] args) + { + CreateWebHostBuilder(args).Build().MigrateDbContext((context,service)=> + { + var dataPath = service.GetService()["DataDirectory:Path"].ToString(); + new SeedFilterListsDbContext().SeedOrUpdateAsync(context,dataPath).Wait(); + }).Run(); + } private static IWebHostBuilder CreateWebHostBuilder(string[] args) => WebHost.CreateDefaultBuilder(args) .UseUrls("http://localhost:5000") diff --git a/src/FilterLists.Api/Startup.cs b/src/FilterLists.Api/Startup.cs index 9e3cd5d2a..3e22f4671 100644 --- a/src/FilterLists.Api/Startup.cs +++ b/src/FilterLists.Api/Startup.cs @@ -26,6 +26,7 @@ public void ConfigureServices(IServiceCollection services) { services.AddFilterListsApiServices(Configuration); services.AddFilterListsApi(); + services.AddSingleton (Configuration); } [UsedImplicitly] @@ -65,7 +66,7 @@ public void Configure(IApplicationBuilder app, IHostingEnvironment env) opts.DocumentTitle = "FilterLists API v1"; opts.RoutePrefix = "docs"; }); - MigrateAndSeedDatabase(app); + // MigrateAndSeedDatabase(app); } //TODO: remove hack (https://github.com/domaindrivendev/Swashbuckle.AspNetCore/issues/74#issuecomment-386762178) @@ -77,15 +78,5 @@ private static void UseLowercaseControllerNameInSwaggerHack(SwaggerOptions opts) foreach (var pathItem in paths) document.Paths.Add(pathItem.Key, pathItem.Value); }); - private void MigrateAndSeedDatabase(IApplicationBuilder app) - { - var dataPath = Configuration.GetSection("DataDirectory").GetValue("Path"); - using (var serviceScope = app.ApplicationServices.GetRequiredService().CreateScope()) - { - var filterListsDbContext = serviceScope.ServiceProvider.GetService(); - filterListsDbContext.Database.Migrate(); - filterListsDbContext.SeedOrUpdate(dataPath); - } - } } } \ No newline at end of file diff --git a/src/FilterLists.Data/Seed/Extensions/SeedFilterListsDbContext.cs b/src/FilterLists.Data/Seed/Extensions/SeedFilterListsDbContext.cs index 5e3b9e5fa..eeb694db8 100644 --- a/src/FilterLists.Data/Seed/Extensions/SeedFilterListsDbContext.cs +++ b/src/FilterLists.Data/Seed/Extensions/SeedFilterListsDbContext.cs @@ -3,6 +3,7 @@ using System.IO; using System.Linq; using System.Linq.Expressions; +using System.Threading.Tasks; using FilterLists.Data.Entities; using FilterLists.Data.Entities.Junctions; using Microsoft.EntityFrameworkCore; @@ -11,41 +12,41 @@ namespace FilterLists.Data.Seed.Extensions { - public static class SeedFilterListsDbContext + public class SeedFilterListsDbContext { - public static void SeedOrUpdate(this FilterListsDbContext dbContext, string dataPath) + public async Task SeedOrUpdateAsync(FilterListsDbContext dbContext, string dataPath) { - dbContext.SeedOrUpdate(dataPath); - dbContext.SeedOrUpdate(dataPath); - dbContext.SeedOrUpdate(dataPath); - dbContext.SeedOrUpdate(dataPath); - dbContext.SeedOrUpdate(dataPath); - dbContext.SeedOrUpdate(dataPath); - dbContext.SeedOrUpdate(dataPath); - dbContext.SetDefaultLicenseId(); - dbContext.SeedOrUpdate(dataPath); - dbContext.SeedOrUpdate(dataPath); - dbContext.SeedOrUpdate(dataPath); - dbContext.SeedOrUpdate(dataPath); - dbContext.SeedOrUpdate(dataPath); - dbContext.SeedOrUpdate(dataPath); - dbContext.SeedOrUpdate(dataPath); + await SeedOrUpdateAsync(dbContext,dataPath); + await SeedOrUpdateAsync(dbContext, dataPath); + await SeedOrUpdateAsync(dbContext,dataPath); + await SeedOrUpdateAsync(dbContext,dataPath); + await SeedOrUpdateAsync(dbContext,dataPath); + await SeedOrUpdateAsync(dbContext,dataPath); + await SeedOrUpdateAsync(dbContext,dataPath); + await SetDefaultLicenseIdAsync(dbContext); + await SeedOrUpdateAsync(dbContext,dataPath); + await SeedOrUpdateAsync(dbContext,dataPath); + await SeedOrUpdateAsync(dbContext,dataPath); + await SeedOrUpdateAsync(dbContext,dataPath); + await SeedOrUpdateAsync(dbContext,dataPath); + await SeedOrUpdateAsync(dbContext,dataPath); + await SeedOrUpdateAsync(dbContext,dataPath); } - private static void SeedOrUpdate(this DbContext dbContext, string dataPath) + private async Task SeedOrUpdateAsync(DbContext dbContext, string dataPath) where TEntity : class, IBaseEntity { var seed = GetSeed(dataPath); - ApplyRemovals(dbContext, seed); - InsertOnDuplicateKeyUpdate(dbContext, seed); + await ApplyRemovals(dbContext, seed); + await InsertOnDuplicateKeyUpdate(dbContext, seed); } - private static List GetSeed(string dataPath) where TEntity : IBaseEntity + private List GetSeed(string dataPath) where TEntity : IBaseEntity { try { return JsonConvert.DeserializeObject>( - File.ReadAllText(dataPath + Path.DirectorySeparatorChar + typeof(TEntity).Name + ".json")); + File.ReadAllText(Path.Combine(dataPath,typeof(TEntity).Name + ".json"))); } catch (FileNotFoundException e) { @@ -54,16 +55,16 @@ private static List GetSeed(string dataPath) where TEntity : I } } - private static void ApplyRemovals(DbContext dbContext, IEnumerable seed) + private async Task ApplyRemovals(DbContext dbContext, IEnumerable seed) where TEntity : class { var removedEntities = GetRemovedEntities(dbContext, seed); dbContext.RemoveRange(removedEntities); - dbContext.SaveChanges(); + await dbContext.SaveChangesAsync(); } //https://stackoverflow.com/a/52264468/2343739 - private static IEnumerable GetRemovedEntities(DbContext dbContext, IEnumerable seed) + private IEnumerable GetRemovedEntities(DbContext dbContext, IEnumerable seed) where TEntity : class { var entityType = dbContext.Model.FindEntityType(typeof(TEntity)); @@ -73,7 +74,7 @@ private static IEnumerable GetRemovedEntities(DbContext dbCont return dbContext.Set().Where(notInSeed); } - private static Expression GetMatchesAnyPk(IEnumerable seed, IEntityType entityType, + private Expression GetMatchesAnyPk(IEnumerable seed, IEntityType entityType, Expression dbEntity) where TEntity : class => seed.Select(e => entityType.FindPrimaryKey() @@ -86,7 +87,7 @@ private static Expression GetMatchesAnyPk(IEnumerable seed, IE (Func)((current, match) => current != null ? Expression.OrElse(current, match) : match)); - private static void InsertOnDuplicateKeyUpdate(DbContext dbContext, IEnumerable seed) + private async Task InsertOnDuplicateKeyUpdate(DbContext dbContext, IEnumerable seed) where TEntity : IBaseEntity { var entityType = dbContext.Model.FindEntityType(typeof(TEntity)); @@ -98,27 +99,27 @@ private static void InsertOnDuplicateKeyUpdate(DbContext dbContext, IEn var updates = CreateUpdates(properties); var sql = "INSERT INTO " + entityType.Relational().TableName + " (" + columns + ") VALUES " + values + " ON DUPLICATE KEY UPDATE " + updates; - dbContext.Database.ExecuteSqlCommand(sql); + await dbContext.Database.ExecuteSqlCommandAsync(sql); } - private static List GetPropertiesLessValueGeneratedTimestamps(IEntityType entityType) => + private List GetPropertiesLessValueGeneratedTimestamps(IEntityType entityType) => entityType.GetProperties() .Where(x => !new List {"CreatedDateUtc", "ModifiedDateUtc"}.Contains(x.Name)) .ToList(); - private static string CreateValues(IEnumerable seed, + private string CreateValues(IEnumerable seed, IReadOnlyCollection properties) where TEntity : IBaseEntity => seed.Select(row => CreateRowValues(properties, row)) .Aggregate("", (current, rowValues) => current == "" ? rowValues : current + ", " + rowValues); - private static string CreateRowValues(IEnumerable properties, TEntity row) + private string CreateRowValues(IEnumerable properties, TEntity row) where TEntity : IBaseEntity => (from property in properties let value = row.GetType().GetProperty(property.Name)?.GetValue(row) select FormatDataForMySql(property, value)).Aggregate("", (rowValues, value) => rowValues == "" ? "(" + value : rowValues + ", " + value) + ")"; - private static object FormatDataForMySql(IProperty property, object value) + private object FormatDataForMySql(IProperty property, object value) { if (value == null) return "NULL"; @@ -131,7 +132,7 @@ private static object FormatDataForMySql(IProperty property, object value) return value; } - private static string CreateUpdates(IReadOnlyCollection properties) + private string CreateUpdates(IReadOnlyCollection properties) { var update = (from property in properties @@ -143,13 +144,13 @@ private static string CreateUpdates(IReadOnlyCollection properties) return update; } - private static string GetUpdateUnchangedColumnHack(IEnumerable properties) + private string GetUpdateUnchangedColumnHack(IEnumerable properties) { var firstId = properties.First(x => x.IsPrimaryKey()).Name; return firstId + " = VALUES(" + firstId + ")"; } - private static void SetDefaultLicenseId(this DbContext dbContext) + private async Task SetDefaultLicenseIdAsync(DbContext dbContext) { var listsWithoutLicense = dbContext.Set() .Where(l => l.LicenseId == null) @@ -160,7 +161,7 @@ private static void SetDefaultLicenseId(this DbContext dbContext) return l; }); dbContext.UpdateRange(listsWithoutLicense); - dbContext.SaveChanges(); + await dbContext.SaveChangesAsync(); } } } \ No newline at end of file