prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

convert to EF Core seeding

Browse source 
ref #312
This commit is contained in:
Collin M. Barrett 2020-02-09 19:07:07 -06:00
parent b65225a5cd
commit 6b2d8e2482
12 changed files with 145849 additions and 694 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
server/src/FilterLists.Api/DependencyInjection/Extensions/IWebHostExtensions.cs
View file

@ -1,4 +1,3 @@
using System;
using Microsoft.AspNetCore.Hosting;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
@ -8,15 +7,13 @@ namespace FilterLists.Api.DependencyInjection.Extensions
// ReSharper disable once InconsistentNaming
public static class IWebHostExtensions
{
public static IWebHost MigrateAndSeedDbContext<TContext>(this IWebHost webHost,
Action<TContext, IServiceProvider> seeder) where TContext : DbContext
public static IWebHost Migrate<TContext>(this IWebHost webHost) where TContext : DbContext
{
using (var scope = webHost.Services.CreateScope())
{
var services = scope.ServiceProvider;
var context = services.GetService<TContext>();
context.Database.Migrate();
seeder(context, services);
}
return webHost;

4
server/src/FilterLists.Api/FilterLists.Api.csproj
View file

@ -45,6 +45,10 @@
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.DotNet.Analyzers.Compatibility" Version="0.2.12-alpha" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="3.1.1">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.9.10" />
</ItemGroup>

8
server/src/FilterLists.Api/Program.cs
View file

@ -1,6 +1,5 @@
using FilterLists.Api.DependencyInjection.Extensions;
using FilterLists.Data;
using FilterLists.Data.Seed.Extensions;
using Microsoft.AspNetCore;
using Microsoft.AspNetCore.Hosting;
@ -8,16 +7,11 @@ namespace FilterLists.Api
{
public static class Program
{
private const string DataPath = "../data";
public static void Main(string[] args)
{
CreateWebHostBuilder(args)
.Build()
.MigrateAndSeedDbContext<FilterListsDbContext>((context, service) =>
{
SeedFilterListsDbContext.SeedOrUpdateAsync(context, DataPath).Wait();
})
.Migrate<FilterListsDbContext>()
.Run();
}

5
server/src/FilterLists.Api/appsettings.json Normal file
View file

@ -0,0 +1,5 @@
{
"ConnectionStrings": {
"FilterListsConnection": "Server=mariadb;Database=filterlists;Uid=filterlists;Pwd=filterlists;"
}
}

2
server/src/FilterLists.Data/EntityTypeConfigurations/BaseEntityTypeConfiguration.cs
View file

@ -1,4 +1,5 @@
using FilterLists.Data.Entities;
using FilterLists.Data.Seed.Extensions;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Metadata.Builders;
@ -21,6 +22,7 @@ public virtual void Configure(EntityTypeBuilder<TEntity> entityTypeBuilder)
.ValueGeneratedOnAddOrUpdate()
.IsRequired()
.HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
entityTypeBuilder.HasDataJsonFile<TEntity>();
}
}
}

6
server/src/FilterLists.Data/EntityTypeConfigurations/Junctions/BaseJunctionTypeConfiguration.cs
View file

@ -1,4 +1,5 @@
using FilterLists.Data.Entities.Junctions;
using FilterLists.Data.Seed.Extensions;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Metadata.Builders;
@ -7,11 +8,14 @@ namespace FilterLists.Data.EntityTypeConfigurations.Junctions
public class BaseJunctionTypeConfiguration<TJunction> : IEntityTypeConfiguration<TJunction>
where TJunction : BaseJunctionEntity
{
public virtual void Configure(EntityTypeBuilder<TJunction> entityTypeBuilder) =>
public virtual void Configure(EntityTypeBuilder<TJunction> entityTypeBuilder)
{
entityTypeBuilder.Property(x => x.CreatedDateUtc)
.HasColumnType("TIMESTAMP")
.ValueGeneratedOnAdd()
.IsRequired()
.HasDefaultValueSql("current_timestamp()");
entityTypeBuilder.HasDataJsonFile<TJunction>();
}
}
}

1
server/src/FilterLists.Data/FilterLists.Data.csproj
View file

@ -22,6 +22,7 @@
</PackageReference>
<PackageReference Include="Microsoft.DotNet.Analyzers.Compatibility" Version="0.2.12-alpha" />
<PackageReference Include="Pomelo.EntityFrameworkCore.MySql" Version="3.1.1" />
<PackageReference Include="System.Text.Json" Version="4.7.0" />
</ItemGroup>
</Project>

50494
server/src/FilterLists.Data/Migrations/20200210010407_InitialEfCoreSeeding.Designer.cs generated Normal file
View file

File diff suppressed because it is too large Load diff

45028
server/src/FilterLists.Data/Migrations/20200210010407_InitialEfCoreSeeding.cs Normal file
View file

File diff suppressed because it is too large Load diff

50758
server/src/FilterLists.Data/Migrations/FilterListsDbContextModelSnapshot.cs
View file

File diff suppressed because it is too large Load diff

24
server/src/FilterLists.Data/Seed/Extensions/SeedExtension.cs Normal file
View file

@ -0,0 +1,24 @@
using System.Collections.Generic;
using System.IO;
using System.Text.Json;
using Microsoft.EntityFrameworkCore.Metadata.Builders;
namespace FilterLists.Data.Seed.Extensions
{
public static class SeedExtension
{
public static void HasDataJsonFile<TEntity>(this EntityTypeBuilder entityTypeBuilder)
{
string path = Path.Combine("../../../data", $"{typeof(TEntity).Name}.json");
if (File.Exists(path))
{
var entitiesJson = File.ReadAllText(path);
var entities = JsonSerializer.Deserialize<IEnumerable<TEntity>>(entitiesJson, new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
});
entityTypeBuilder.HasData((IEnumerable<object>)entities);
}
}
}
}

208
server/src/FilterLists.Data/Seed/Extensions/SeedFilterListsDbContext.cs
View file

@ -1,208 +0,0 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Linq.Expressions;
using System.Threading.Tasks;
using FilterLists.Data.Entities;
using FilterLists.Data.Entities.Junctions;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Metadata;
using Newtonsoft.Json;
namespace FilterLists.Data.Seed.Extensions
{
public static class SeedFilterListsDbContext
{
public static async Task SeedOrUpdateAsync(FilterListsDbContext dbContext, string dataPath)
{
if (dbContext is null)
throw new ArgumentNullException(nameof(dbContext));
await SeedOrUpdate<Language>(dbContext, dataPath);
await SeedOrUpdate<License>(dbContext, dataPath);
await SeedOrUpdate<Maintainer>(dbContext, dataPath);
await SeedOrUpdate<Software>(dbContext, dataPath);
await SeedOrUpdate<Syntax>(dbContext, dataPath);
await SeedOrUpdate<Tag>(dbContext, dataPath);
await SeedOrUpdate<FilterList>(dbContext, dataPath);
await SetDefaultLicenseIdAsync(dbContext);
await SeedFilterListDates(dbContext, dataPath);
await SeedOrUpdate<FilterListLanguage>(dbContext, dataPath);
await SeedOrUpdate<FilterListMaintainer>(dbContext, dataPath);
await SeedOrUpdate<FilterListTag>(dbContext, dataPath);
await SeedOrUpdate<Dependent>(dbContext, dataPath);
await SeedOrUpdate<Fork>(dbContext, dataPath);
await SeedOrUpdate<Merge>(dbContext, dataPath);
await SeedOrUpdate<SoftwareSyntax>(dbContext, dataPath);
}
private static async Task SeedOrUpdate<TEntity>(DbContext dbContext, string dataPath)
where TEntity : class, IBaseEntity
{
var seed = GetSeed<TEntity>(dataPath);
await ApplyRemovals(dbContext, seed);
await InsertOnDuplicateKeyUpdate(dbContext, seed);
}
private static List<TEntity> GetSeed<TEntity>(string dataPath) where TEntity : IBaseEntity
{
try
{
return JsonConvert.DeserializeObject<List<TEntity>>(
File.ReadAllText(Path.Combine(dataPath, typeof(TEntity).Name + ".json")));
}
catch (FileNotFoundException e)
{
Console.WriteLine(e.Message);
return new List<TEntity>();
}
}
private static async Task ApplyRemovals<TEntity>(DbContext dbContext, IEnumerable<TEntity> seed)
where TEntity : class
{
var removedEntities = GetRemovedEntities(dbContext, seed);
dbContext.RemoveRange(removedEntities);
await dbContext.SaveChangesAsync();
}
//https://stackoverflow.com/a/52264468/2343739
private static IEnumerable<TEntity> GetRemovedEntities<TEntity>(DbContext dbContext, IEnumerable<TEntity> seed)
where TEntity : class
{
var entityType = dbContext.Model.FindEntityType(typeof(TEntity));
var dbEntity = Expression.Parameter(entityType.ClrType, "e");
var matchesAnyPk = GetMatchesAnyPk(seed, entityType, dbEntity);
var notInSeed = Expression.Lambda<Func<TEntity, bool>>(Expression.Not(matchesAnyPk), dbEntity);
return dbContext.Set<TEntity>().Where(notInSeed);
}
private static Expression GetMatchesAnyPk<TEntity>(IEnumerable<TEntity> seed, IEntityType entityType,
Expression dbEntity) where TEntity : class =>
seed.Select(e => entityType.FindPrimaryKey()
.Properties.Select(p =>
Expression.Equal(Expression.Property(dbEntity, p.PropertyInfo),
Expression.Property(Expression.Constant(e), p.PropertyInfo)))
.Aggregate(Expression.AndAlso))
.Aggregate(null,
(Func<Expression, BinaryExpression, Expression>)((current, match) =>
current != null ? Expression.OrElse(current, match) : match));
private static async Task InsertOnDuplicateKeyUpdate<TEntity>(DbContext dbContext, IEnumerable<TEntity> seed)
where TEntity : IBaseEntity
{
var entityType = dbContext.Model.FindEntityType(typeof(TEntity));
var properties = GetPropertiesLessValueGeneratedTimestamps(entityType);
var values = CreateValues(seed, properties);
if (string.IsNullOrEmpty(values))
return;
var columns = string.Join(", ", properties.Select(x => x.Name));
var updates = CreateUpdates(properties);
var sql = "INSERT INTO " + entityType.GetTableName() + " (" + columns + ") VALUES " + values +
" ON DUPLICATE KEY UPDATE " + updates;
await dbContext.Database.ExecuteSqlCommandAsync(sql);
}
private static List<IProperty> GetPropertiesLessValueGeneratedTimestamps(IEntityType entityType) =>
entityType.GetProperties()
.Where(x => !new List<string>
{
"CreatedDateUtc",
"ModifiedDateUtc",
"PublishedDate",
"UpdatedDate"
}.Contains(x.Name))
.ToList();
private static string
CreateValues<TEntity>(IEnumerable<TEntity> seed, IReadOnlyCollection<IProperty> properties)
where TEntity : IBaseEntity => seed.Select(row => CreateRowValues(properties, row))
.Aggregate("",
(current, rowValues) => string.IsNullOrEmpty(current) ? rowValues : current + ", " + rowValues);
private static string CreateRowValues<TEntity>(IEnumerable<IProperty> properties, TEntity row)
where TEntity : IBaseEntity =>
(from property in properties
let value = row.GetType().GetProperty(property.Name)?.GetValue(row)
select FormatDataForMySql(property, value)).Aggregate("",
(rowValues, value) => string.IsNullOrEmpty(rowValues) ? "(" + value : rowValues + ", " + value) + ")";
private static object FormatDataForMySql(IProperty property, object value)
{
if (value == null)
return "NULL";
if (property.ClrType == typeof(string))
return "'" + value.ToString().Replace("'", "''") + "'";
if (property.ClrType == typeof(bool))
return Convert.ToInt32(value, CultureInfo.InvariantCulture);
if (property.ClrType == typeof(DateTime?))
return "'" + ((DateTime)value).ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture) + "'";
return value;
}
private static string CreateUpdates(IReadOnlyCollection<IProperty> properties)
{
var update =
(from property in properties
where !property.IsPrimaryKey()
select property.Name + " = VALUES(" + property.Name + ")").Aggregate("",
(updates, columnUpdates) =>
string.IsNullOrEmpty(updates) ? columnUpdates : updates + ", " + columnUpdates);
if (string.IsNullOrEmpty(update))
update = GetUpdateUnchangedColumnHack(properties);
return update;
}
private static string GetUpdateUnchangedColumnHack(IEnumerable<IProperty> properties)
{
var firstId = properties.First(x => x.IsPrimaryKey()).Name;
return firstId + " = VALUES(" + firstId + ")";
}
private static async Task SetDefaultLicenseIdAsync(DbContext dbContext)
{
var listsWithoutLicense = dbContext.Set<FilterList>()
.Where(l => l.LicenseId == null)
.ToList()
.Select(l =>
{
l.LicenseId = 5;
return l;
});
dbContext.UpdateRange(listsWithoutLicense);
await dbContext.SaveChangesAsync();
}
private static async Task SeedFilterListDates(DbContext dbContext, string dataPath)
{
var seed = GetSeed<FilterList>(dataPath);
var lists = dbContext.Set<FilterList>()
.ToList()
.Select(l =>
{
var seedList = seed.FirstOrDefault(s => s.Id == l.Id);
l.PublishedDate = seedList?.PublishedDate is DateTime seedPublishedDate
? l.PublishedDate is DateTime currentPublishedDate
? seedPublishedDate < currentPublishedDate
? seedPublishedDate
: currentPublishedDate
: seedPublishedDate
: l.PublishedDate;
l.UpdatedDate = seedList?.UpdatedDate is DateTime seedUpdatedDate
? l.UpdatedDate is DateTime currentUpdatedDate
? seedUpdatedDate < currentUpdatedDate
? currentUpdatedDate
: seedUpdatedDate
: seedUpdatedDate
: l.UpdatedDate;
return l;
});
dbContext.UpdateRange(lists);
await dbContext.SaveChangesAsync();
}
}
}