From 03f93c8175f5544e493badeae8e612cde4ea4601 Mon Sep 17 00:00:00 2001 From: "Collin M. Barrett" Date: Sat, 24 Jun 2017 17:36:38 -0500 Subject: [PATCH] add DataLoad console app and TableService --- FilterLists.sln | 10 ++- src/FilterLists.Data/Models/List.cs | 72 ++++++++++--------- .../Repositories/Contracts/IListRepository.cs | 1 + .../Implementations/ListRepository.cs | 8 ++- src/FilterLists.DataLoad/DataLoad.cs | 19 +++++ .../FilterLists.DataLoad.csproj | 16 +++++ .../Contracts/ITableService.cs | 7 ++ .../Implementations/TableService.cs | 64 +++++++++++++++++ 8 files changed, 160 insertions(+), 37 deletions(-) create mode 100644 src/FilterLists.DataLoad/DataLoad.cs create mode 100644 src/FilterLists.DataLoad/FilterLists.DataLoad.csproj create mode 100644 src/FilterLists.Services/Contracts/ITableService.cs create mode 100644 src/FilterLists.Services/Implementations/TableService.cs diff --git a/FilterLists.sln b/FilterLists.sln index c7e0c69aa..41e26970c 100644 --- a/FilterLists.sln +++ b/FilterLists.sln @@ -1,7 +1,6 @@ - Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio 15 -VisualStudioVersion = 15.0.26403.3 +VisualStudioVersion = 15.0.26430.14 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FilterLists.Api", "src\FilterLists.Api\FilterLists.Api.csproj", "{57E4CE18-41F3-48F6-B142-12947FFBA86C}" EndProject @@ -28,6 +27,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "scripts", "scripts", "{E2F4 scripts\deploy.sh = scripts\deploy.sh EndProjectSection EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FilterLists.DataLoad", "src\FilterLists.DataLoad\FilterLists.DataLoad.csproj", "{B507EE76-B035-45D7-9D26-DF38A5013BC3}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -54,6 +55,10 @@ Global {AE4E5758-92F3-4BFA-B71E-6A05F5E71FAC}.Debug|Any CPU.Build.0 = Debug|Any CPU {AE4E5758-92F3-4BFA-B71E-6A05F5E71FAC}.Release|Any CPU.ActiveCfg = Release|Any CPU {AE4E5758-92F3-4BFA-B71E-6A05F5E71FAC}.Release|Any CPU.Build.0 = Release|Any CPU + {B507EE76-B035-45D7-9D26-DF38A5013BC3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B507EE76-B035-45D7-9D26-DF38A5013BC3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B507EE76-B035-45D7-9D26-DF38A5013BC3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B507EE76-B035-45D7-9D26-DF38A5013BC3}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -65,5 +70,6 @@ Global {39072BBE-268D-408D-878F-1416F9C400A2} = {2AD4EEF3-8372-4AE9-8B1C-25359EC87435} {AE4E5758-92F3-4BFA-B71E-6A05F5E71FAC} = {2AD4EEF3-8372-4AE9-8B1C-25359EC87435} {E2F4C0A9-CA4D-4A11-8B10-899513E777F8} = {6EF78534-FCFD-4918-91AA-316D8AA951AA} + {B507EE76-B035-45D7-9D26-DF38A5013BC3} = {E7590A2B-621D-47EA-B026-315793F0FE50} EndGlobalSection EndGlobal diff --git a/src/FilterLists.Data/Models/List.cs b/src/FilterLists.Data/Models/List.cs index 578982cff..ab5d80f7b 100644 --- a/src/FilterLists.Data/Models/List.cs +++ b/src/FilterLists.Data/Models/List.cs @@ -5,40 +5,6 @@ namespace FilterLists.Data.Models { public class List : BaseEntity { - [MaxLength(1022)] - [UsedImplicitly] - public string Description { get; set; } - - [MaxLength(2083)] - [MinLength(6)] - [UsedImplicitly] - public string DescriptionSourceUrl { get; set; } - - [MaxLength(2083)] - [MinLength(6)] - [UsedImplicitly] - public string DonateUrl { get; set; } - - [MaxLength(126)] - [MinLength(7)] - [UsedImplicitly] - public string Email { get; set; } - - [MaxLength(2083)] - [MinLength(6)] - [UsedImplicitly] - public string ForumUrl { get; set; } - - [MaxLength(2083)] - [MinLength(6)] - [UsedImplicitly] - public string HomeUrl { get; set; } - - [MaxLength(2083)] - [MinLength(6)] - [UsedImplicitly] - public string IssuesUrl { get; set; } - [MaxLength(126)] [UsedImplicitly] public string Name { get; set; } @@ -48,5 +14,43 @@ public class List : BaseEntity [MinLength(6)] [UsedImplicitly] public string ViewUrl { get; set; } + + [MaxLength(2083)] + [MinLength(6)] + [UsedImplicitly] + public string HomeUrl { get; set; } + + [MaxLength(1022)] + [UsedImplicitly] + public string Description { get; set; } + + [MaxLength(2083)] + [MinLength(6)] + [UsedImplicitly] + public string DescriptionSourceUrl { get; set; } + + [MaxLength(126)] + [UsedImplicitly] + public string Author { get; set; } + + [MaxLength(2083)] + [MinLength(6)] + [UsedImplicitly] + public string ForumUrl { get; set; } + + [MaxLength(2083)] + [MinLength(6)] + [UsedImplicitly] + public string IssuesUrl { get; set; } + + [MaxLength(126)] + [MinLength(7)] + [UsedImplicitly] + public string Email { get; set; } + + [MaxLength(2083)] + [MinLength(6)] + [UsedImplicitly] + public string DonateUrl { get; set; } } } \ No newline at end of file diff --git a/src/FilterLists.Data/Repositories/Contracts/IListRepository.cs b/src/FilterLists.Data/Repositories/Contracts/IListRepository.cs index 95f39442a..8a08735ba 100644 --- a/src/FilterLists.Data/Repositories/Contracts/IListRepository.cs +++ b/src/FilterLists.Data/Repositories/Contracts/IListRepository.cs @@ -6,5 +6,6 @@ namespace FilterLists.Data.Repositories.Contracts public interface IListRepository { IEnumerable GetAll(); + IEnumerable UpdateAll(); } } \ No newline at end of file diff --git a/src/FilterLists.Data/Repositories/Implementations/ListRepository.cs b/src/FilterLists.Data/Repositories/Implementations/ListRepository.cs index 7bcde0169..96d9aa175 100644 --- a/src/FilterLists.Data/Repositories/Implementations/ListRepository.cs +++ b/src/FilterLists.Data/Repositories/Implementations/ListRepository.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Linq; using FilterLists.Data.Contexts; using FilterLists.Data.Models; @@ -21,5 +22,10 @@ public IEnumerable GetAll() { return _filterListsDbContext.List.AsEnumerable(); } + + public IEnumerable UpdateAll() + { + throw new NotImplementedException(); + } } } \ No newline at end of file diff --git a/src/FilterLists.DataLoad/DataLoad.cs b/src/FilterLists.DataLoad/DataLoad.cs new file mode 100644 index 000000000..e0ff6afc0 --- /dev/null +++ b/src/FilterLists.DataLoad/DataLoad.cs @@ -0,0 +1,19 @@ +using FilterLists.Services.Contracts; +using FilterLists.Services.Implementations; +using Microsoft.Extensions.DependencyInjection; + +namespace FilterLists.DataLoad +{ + //TODO: trigger on job or ideally via GitHub webhook when any .csv is updated + internal class DataLoad + { + private static void Main() + { + var serviceProvider = new ServiceCollection() + .AddLogging() + .AddSingleton() + .BuildServiceProvider(); + serviceProvider.GetService().UpdateTables(); + } + } +} \ No newline at end of file diff --git a/src/FilterLists.DataLoad/FilterLists.DataLoad.csproj b/src/FilterLists.DataLoad/FilterLists.DataLoad.csproj new file mode 100644 index 000000000..91bbe737c --- /dev/null +++ b/src/FilterLists.DataLoad/FilterLists.DataLoad.csproj @@ -0,0 +1,16 @@ + + + + Exe + netcoreapp1.1 + + + + + + + + + + + \ No newline at end of file diff --git a/src/FilterLists.Services/Contracts/ITableService.cs b/src/FilterLists.Services/Contracts/ITableService.cs new file mode 100644 index 000000000..45e3f9702 --- /dev/null +++ b/src/FilterLists.Services/Contracts/ITableService.cs @@ -0,0 +1,7 @@ +namespace FilterLists.Services.Contracts +{ + public interface ITableService + { + void UpdateTables(); + } +} \ No newline at end of file diff --git a/src/FilterLists.Services/Implementations/TableService.cs b/src/FilterLists.Services/Implementations/TableService.cs new file mode 100644 index 000000000..6ed07a0f9 --- /dev/null +++ b/src/FilterLists.Services/Implementations/TableService.cs @@ -0,0 +1,64 @@ +using System.Net.Http; +using System.Net.Http.Headers; +using System.Threading.Tasks; +using FilterLists.Data.Models; +using FilterLists.Data.Repositories.Contracts; +using FilterLists.Services.Contracts; + +namespace FilterLists.Services.Implementations +{ + public class TableService : ITableService + { + //private readonly IListRepository _listRepository; + + public TableService(/*IListRepository listRepository*/) + { + //_listRepository = listRepository; + } + + /// + /// update all tables via .CSVs from GitHub + /// + public void UpdateTables() + { + //TODO: loop through all CSVs + UpdateTable("List"); + } + + /// + /// update table via .CSV from GitHub + /// + /// name of database table + public void UpdateTable(string tableName) + { + //TODO: fetch CSV URL from db table + const string csvUrl = "https://raw.githubusercontent.com/collinbarrett/FilterLists/dataToCsv/data/List.csv"; + var file = FetchFile(csvUrl).Result; + var csvRows = file.Split('\n'); + var headerCells = csvRows[0].Split(','); + for (var i = 1; i < csvRows.Length; i++) + if (!string.IsNullOrEmpty(csvRows[i])) + { + var cells = csvRows[i].Split(','); + var list = new List + { + Name = cells[0] + //TODO: finish mappings, try to make generic for any table type using headerCells as properties + }; + } + //TODO: update table via repo/EF + } + + /// + /// fetch a file as a string from the internet + /// + /// URL of file to fetch + /// string of file + private static async Task FetchFile(string url) + { + var httpClient = new HttpClient(); + httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("FilterLists", "1.0")); + return await httpClient.GetStringAsync(url); + } + } +} \ No newline at end of file