prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Revert "seed via ef core"

Browse source 
This reverts commit b7aaa1ff3c.
This commit is contained in:
Collin M. Barrett 2018-08-28 18:13:21 -05:00
parent b4b86c0b55
commit 7b736d62dc
13 changed files with 155 additions and 27152 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
data/FilterList.json
View file

@ -6635,7 +6635,7 @@
"chatUrl": null,
"description": "This is an AdGuard-specific supplement for the Adblock polskie reguły filter. It specifically removes adverts on Polish language websites.",
"descriptionSourceUrl": "https://kb.adguard.com/en/general/adguard-ad-filters#polish-ads-filter",
"discontinuedDate": "2018-06-26T00:00:00",
"discontinuedDate": "2018-06-26T08:05:33",
"donateUrl": null,
"emailAddress": null,
"forumUrl": "https://forum.adguard.com/index.php?categories/filter-rules.66/",
@ -9461,4 +9461,4 @@
"syntaxId": 3,
"viewUrl": "https://raw.githubusercontent.com/azet12/KAD/gh-pages/assets/other/kadfakewhitelist.txt"
}
]
]

2
data/FilterListLanguage.json
View file

@ -687,4 +687,4 @@
"filterListId": 508,
"languageId": 11
}
]
]

12
data/FilterListTag.json
View file

@ -127,14 +127,6 @@
"filterListId": 223,
"tagId": 2
},
{
"filterListId": 223,
"tagId": 4
},
{
"filterListId": 224,
"tagId": 4
},
{
"filterListId": 224,
"tagId": 8
@ -447,10 +439,6 @@
"filterListId": 507,
"tagId": 17
},
{
"filterListId": 508,
"tagId": 7
},
{
"filterListId": 508,
"tagId": 10

2
data/Tag.json
View file

@ -84,4 +84,4 @@
"description": "Intended for use with Internet Explorer's TPL feature",
"name": "ie"
}
]
]

7634
src/FilterLists.Api/Migrations/20180828204523_InitialEfCoreSeed.Designer.cs generated
View file

File diff suppressed because it is too large Load diff

12429
src/FilterLists.Api/Migrations/20180828204523_InitialEfCoreSeed.cs
View file

File diff suppressed because it is too large Load diff

7038
src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs
View file

File diff suppressed because it is too large Load diff

11
src/FilterLists.Api/Startup.cs
View file

@ -1,7 +1,7 @@
using System.Linq;
using FilterLists.Api.DependencyInjection.Extensions;
using FilterLists.Data;
using FilterLists.Data.Extensions;
using FilterLists.Data.Seed.Extensions;
using FilterLists.Services.DependencyInjection.Extensions;
using JetBrains.Annotations;
using Microsoft.AspNetCore.Builder;
@ -17,14 +17,14 @@ namespace FilterLists.Api
[UsedImplicitly]
public class Startup
{
public Startup(IConfiguration configuration) => Config = configuration;
public Startup(IConfiguration configuration) => Configuration = configuration;
private IConfiguration Config { get; }
private IConfiguration Configuration { get; }
[UsedImplicitly]
public void ConfigureServices(IServiceCollection services)
{
services.AddFilterListsApiServices(Config);
services.AddFilterListsApiServices(Configuration);
services.AddFilterListsApi();
}
@ -79,11 +79,12 @@ private static void UseLowercaseControllerNameInSwaggerHack(SwaggerOptions opts)
private void MigrateAndSeedDatabase(IApplicationBuilder app)
{
ModelBuilderExtensions.DataPath = Config.GetSection("DataDirectory").GetValue<string>("Path");
var dataPath = Configuration.GetSection("DataDirectory").GetValue<string>("Path");
using (var serviceScope = app.ApplicationServices.GetRequiredService<IServiceScopeFactory>().CreateScope())
{
var filterListsDbContext = serviceScope.ServiceProvider.GetService<FilterListsDbContext>();
filterListsDbContext.Database.Migrate();
filterListsDbContext.SeedOrUpdate(dataPath);
}
}
}

2
src/FilterLists.Data/EntityTypeConfigurations/BaseEntityTypeConfiguration.cs
View file

@ -14,10 +14,12 @@ public virtual void Configure(EntityTypeBuilder<TEntity> entityTypeBuilder)
entityTypeBuilder.Property(x => x.CreatedDateUtc)
.HasColumnType("TIMESTAMP")
.ValueGeneratedOnAdd()
.IsRequired()
.HasDefaultValueSql("current_timestamp()");
entityTypeBuilder.Property(x => x.ModifiedDateUtc)
.HasColumnType("TIMESTAMP")
.ValueGeneratedOnAddOrUpdate()
.IsRequired()
.HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
}
}

1
src/FilterLists.Data/EntityTypeConfigurations/Junctions/BaseJunctionTypeConfiguration.cs
View file

@ -11,6 +11,7 @@ public virtual void Configure(EntityTypeBuilder<TJunction> entityTypeBuilder) =>
entityTypeBuilder.Property(x => x.CreatedDateUtc)
.HasColumnType("TIMESTAMP")
.ValueGeneratedOnAdd()
.IsRequired()
.HasDefaultValueSql("current_timestamp()");
}
}

33
src/FilterLists.Data/Extensions/ModelBuilderExtensions.cs
View file

@ -1,33 +0,0 @@
using System;
using System.IO;
using FilterLists.Data.Entities;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json;
namespace FilterLists.Data.Extensions
{
public static class ModelBuilderExtensions
{
public static string DataPath { private get; set; } = "..\\..\\data";
public static void Seed<TEntity>(this ModelBuilder modelBuilder) where TEntity : class, IBaseEntity
{
var entities = GetSeedRows<TEntity>();
modelBuilder.Entity<TEntity>().HasData(entities);
}
private static TEntity[] GetSeedRows<TEntity>() where TEntity : IBaseEntity
{
try
{
return JsonConvert.DeserializeObject<TEntity[]>(
File.ReadAllText(DataPath + Path.DirectorySeparatorChar + typeof(TEntity).Name + ".json"));
}
catch (FileNotFoundException e)
{
Console.WriteLine(e.Message);
return null;
}
}
}
}

20
src/FilterLists.Data/FilterListsDbContext.cs
View file

@ -2,7 +2,6 @@
using FilterLists.Data.Entities.Junctions;
using FilterLists.Data.EntityTypeConfigurations;
using FilterLists.Data.EntityTypeConfigurations.Junctions;
using FilterLists.Data.Extensions;
using Microsoft.EntityFrameworkCore;
namespace FilterLists.Data
@ -35,7 +34,6 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
{
base.OnModelCreating(modelBuilder);
ApplyConfigurations(modelBuilder);
ApplySeed(modelBuilder);
}
private static void ApplyConfigurations(ModelBuilder modelBuilder)
@ -58,23 +56,5 @@ private static void ApplyConfigurations(ModelBuilder modelBuilder)
modelBuilder.ApplyConfiguration(new SnapshotRuleTypeConfiguration());
modelBuilder.ApplyConfiguration(new SoftwareSyntaxTypeConfiguration());
}
private static void ApplySeed(ModelBuilder modelBuilder)
{
modelBuilder.Seed<Language>();
modelBuilder.Seed<License>();
modelBuilder.Seed<Maintainer>();
modelBuilder.Seed<Software>();
modelBuilder.Seed<Syntax>();
modelBuilder.Seed<Tag>();
modelBuilder.Seed<FilterList>();
modelBuilder.Seed<FilterListLanguage>();
modelBuilder.Seed<FilterListMaintainer>();
modelBuilder.Seed<FilterListTag>();
modelBuilder.Seed<Fork>();
modelBuilder.Seed<Merge>();
modelBuilder.Seed<SoftwareSyntax>();
}
}
}

119
src/FilterLists.Data/Seed/Extensions/SeedFilterListsDbContext.cs Normal file
View file

@ -0,0 +1,119 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using FilterLists.Data.Entities;
using FilterLists.Data.Entities.Junctions;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Metadata;
using Newtonsoft.Json;
namespace FilterLists.Data.Seed.Extensions
{
public static class SeedFilterListsDbContext
{
public static void SeedOrUpdate(this FilterListsDbContext dbContext, string dataPath)
{
dbContext.InsertOnDuplicateKeyUpdate<Language>(dataPath);
dbContext.InsertOnDuplicateKeyUpdate<License>(dataPath);
dbContext.InsertOnDuplicateKeyUpdate<Maintainer>(dataPath);
dbContext.InsertOnDuplicateKeyUpdate<Software>(dataPath);
dbContext.InsertOnDuplicateKeyUpdate<Syntax>(dataPath);
dbContext.InsertOnDuplicateKeyUpdate<Tag>(dataPath);
dbContext.InsertOnDuplicateKeyUpdate<FilterList>(dataPath);
dbContext.InsertOnDuplicateKeyUpdate<FilterListLanguage>(dataPath);
dbContext.InsertOnDuplicateKeyUpdate<FilterListMaintainer>(dataPath);
dbContext.InsertOnDuplicateKeyUpdate<FilterListTag>(dataPath);
dbContext.InsertOnDuplicateKeyUpdate<Fork>(dataPath);
dbContext.InsertOnDuplicateKeyUpdate<Merge>(dataPath);
dbContext.InsertOnDuplicateKeyUpdate<SoftwareSyntax>(dataPath);
SetCantSnapshot(dbContext);
}
private static void InsertOnDuplicateKeyUpdate<TEntity>(this DbContext dbContext, string dataPath)
where TEntity : IBaseEntity
{
var entityType = dbContext.Model.FindEntityType(typeof(TEntity));
var properties = GetPropertiesLessValueGeneratedTimestamps(entityType);
var values = CreateValues<TEntity>(properties, dataPath);
if (values == "") return;
var columns = string.Join(", ", properties.Select(x => x.Name));
var updates = CreateUpdates(properties);
var rawSqlString = "INSERT INTO " + entityType.Relational().TableName + " (" + columns + ") VALUES " +
values + " ON DUPLICATE KEY UPDATE " + updates;
dbContext.Database.ExecuteSqlCommand(rawSqlString);
dbContext.SaveChanges();
}
//TODO: get seed properties dynamically from JSON
private static List<IProperty> GetPropertiesLessValueGeneratedTimestamps(IEntityType entityType) =>
entityType.GetProperties()
.Where(x => !new List<string> {"CreatedDateUtc", "ModifiedDateUtc"}.Contains(x.Name))
.ToList();
private static string CreateValues<TEntity>(IReadOnlyCollection<IProperty> properties, string dataPath)
where TEntity : IBaseEntity =>
GetSeedRows<TEntity>(dataPath)
.Select(row => CreateRowValues(properties, row))
.Aggregate("", (current, rowValues) => current == "" ? rowValues : current + ", " + rowValues);
private static List<TEntity> GetSeedRows<TEntity>(string dataPath) where TEntity : IBaseEntity
{
try
{
return JsonConvert.DeserializeObject<List<TEntity>>(
File.ReadAllText(dataPath + Path.DirectorySeparatorChar + typeof(TEntity).Name + ".json"));
}
catch (FileNotFoundException e)
{
Console.WriteLine(e.Message);
return new List<TEntity>();
}
}
private static string CreateRowValues<TEntity>(IEnumerable<IProperty> properties, TEntity row)
where TEntity : IBaseEntity =>
(from property in properties
let value = row.GetType().GetProperty(property.Name)?.GetValue(row)
select FormatDataForMySql(property, value)).Aggregate("",
(rowValues, value) => rowValues == "" ? "(" + value : rowValues + ", " + value) + ")";
private static object FormatDataForMySql(IProperty property, object value)
{
if (value == null) return "NULL";
if (property.ClrType == typeof(string))
return "'" + value.ToString().Replace("'", "''") + "'";
if (property.ClrType == typeof(bool))
return Convert.ToInt32(value);
if (property.ClrType == typeof(DateTime?))
return "'" + ((DateTime)value).ToString("yyyy-MM-dd HH:mm:ss") + "'";
return value;
}
private static string CreateUpdates(IReadOnlyCollection<IProperty> properties)
{
var update =
(from property in properties
where !property.IsPrimaryKey()
select property.Name + " = VALUES(" + property.Name + ")").Aggregate("",
(updates, columnUpdates) => updates == "" ? columnUpdates : updates + ", " + columnUpdates);
if (update == "")
update = GetUpdateUnchangedColumnHack(properties);
return update;
}
private static string GetUpdateUnchangedColumnHack(IEnumerable<IProperty> properties)
{
var firstId = properties.First(x => x.IsPrimaryKey()).Name;
return firstId + " = VALUES(" + firstId + ")";
}
//TODO: https://github.com/collinbarrett/FilterLists/issues/201
private static void SetCantSnapshot(DbContext dbContext)
{
const string rawSqlString = "UPDATE filterlists SET CantSnapshot = 1 WHERE id IN(173, 185, 188, 189)";
dbContext.Database.ExecuteSqlCommand(rawSqlString);
dbContext.SaveChanges();
}
}
}