prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

add DataLoad console app and TableService

Browse source
This commit is contained in:
Collin M. Barrett 2017-06-24 17:36:38 -05:00
parent 6dceefebe0
commit 03f93c8175
8 changed files with 160 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
FilterLists.sln
View file

@ -1,7 +1,6 @@
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 15
VisualStudioVersion = 15.0.26403.3
VisualStudioVersion = 15.0.26430.14
MinimumVisualStudioVersion = 10.0.40219.1
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FilterLists.Api", "src\FilterLists.Api\FilterLists.Api.csproj", "{57E4CE18-41F3-48F6-B142-12947FFBA86C}"
EndProject
@ -28,6 +27,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "scripts", "scripts", "{E2F4
scripts\deploy.sh = scripts\deploy.sh
EndProjectSection
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FilterLists.DataLoad", "src\FilterLists.DataLoad\FilterLists.DataLoad.csproj", "{B507EE76-B035-45D7-9D26-DF38A5013BC3}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -54,6 +55,10 @@ Global
{AE4E5758-92F3-4BFA-B71E-6A05F5E71FAC}.Debug|Any CPU.Build.0 = Debug|Any CPU
{AE4E5758-92F3-4BFA-B71E-6A05F5E71FAC}.Release|Any CPU.ActiveCfg = Release|Any CPU
{AE4E5758-92F3-4BFA-B71E-6A05F5E71FAC}.Release|Any CPU.Build.0 = Release|Any CPU
{B507EE76-B035-45D7-9D26-DF38A5013BC3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{B507EE76-B035-45D7-9D26-DF38A5013BC3}.Debug|Any CPU.Build.0 = Debug|Any CPU
{B507EE76-B035-45D7-9D26-DF38A5013BC3}.Release|Any CPU.ActiveCfg = Release|Any CPU
{B507EE76-B035-45D7-9D26-DF38A5013BC3}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@ -65,5 +70,6 @@ Global
{39072BBE-268D-408D-878F-1416F9C400A2} = {2AD4EEF3-8372-4AE9-8B1C-25359EC87435}
{AE4E5758-92F3-4BFA-B71E-6A05F5E71FAC} = {2AD4EEF3-8372-4AE9-8B1C-25359EC87435}
{E2F4C0A9-CA4D-4A11-8B10-899513E777F8} = {6EF78534-FCFD-4918-91AA-316D8AA951AA}
{B507EE76-B035-45D7-9D26-DF38A5013BC3} = {E7590A2B-621D-47EA-B026-315793F0FE50}
EndGlobalSection
EndGlobal

72
src/FilterLists.Data/Models/List.cs
View file

@ -5,40 +5,6 @@ namespace FilterLists.Data.Models
{
public class List : BaseEntity
{
[MaxLength(1022)]
[UsedImplicitly]
public string Description { get; set; }
[MaxLength(2083)]
[MinLength(6)]
[UsedImplicitly]
public string DescriptionSourceUrl { get; set; }
[MaxLength(2083)]
[MinLength(6)]
[UsedImplicitly]
public string DonateUrl { get; set; }
[MaxLength(126)]
[MinLength(7)]
[UsedImplicitly]
public string Email { get; set; }
[MaxLength(2083)]
[MinLength(6)]
[UsedImplicitly]
public string ForumUrl { get; set; }
[MaxLength(2083)]
[MinLength(6)]
[UsedImplicitly]
public string HomeUrl { get; set; }
[MaxLength(2083)]
[MinLength(6)]
[UsedImplicitly]
public string IssuesUrl { get; set; }
[MaxLength(126)]
[UsedImplicitly]
public string Name { get; set; }
@ -48,5 +14,43 @@ public class List : BaseEntity
[MinLength(6)]
[UsedImplicitly]
public string ViewUrl { get; set; }
[MaxLength(2083)]
[MinLength(6)]
[UsedImplicitly]
public string HomeUrl { get; set; }
[MaxLength(1022)]
[UsedImplicitly]
public string Description { get; set; }
[MaxLength(2083)]
[MinLength(6)]
[UsedImplicitly]
public string DescriptionSourceUrl { get; set; }
[MaxLength(126)]
[UsedImplicitly]
public string Author { get; set; }
[MaxLength(2083)]
[MinLength(6)]
[UsedImplicitly]
public string ForumUrl { get; set; }
[MaxLength(2083)]
[MinLength(6)]
[UsedImplicitly]
public string IssuesUrl { get; set; }
[MaxLength(126)]
[MinLength(7)]
[UsedImplicitly]
public string Email { get; set; }
[MaxLength(2083)]
[MinLength(6)]
[UsedImplicitly]
public string DonateUrl { get; set; }
}
}

1
src/FilterLists.Data/Repositories/Contracts/IListRepository.cs
View file

@ -6,5 +6,6 @@ namespace FilterLists.Data.Repositories.Contracts
public interface IListRepository
{
IEnumerable<List> GetAll();
IEnumerable<List> UpdateAll();
}
}

8
src/FilterLists.Data/Repositories/Implementations/ListRepository.cs
View file

@ -1,4 +1,5 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.Linq;
using FilterLists.Data.Contexts;
using FilterLists.Data.Models;
@ -21,5 +22,10 @@ public IEnumerable<List> GetAll()
{
return _filterListsDbContext.List.AsEnumerable();
}
public IEnumerable<List> UpdateAll()
{
throw new NotImplementedException();
}
}
}

19
src/FilterLists.DataLoad/DataLoad.cs Normal file
View file

@ -0,0 +1,19 @@
using FilterLists.Services.Contracts;
using FilterLists.Services.Implementations;
using Microsoft.Extensions.DependencyInjection;
namespace FilterLists.DataLoad
{
//TODO: trigger on job or ideally via GitHub webhook when any .csv is updated
internal class DataLoad
{
private static void Main()
{
var serviceProvider = new ServiceCollection()
.AddLogging()
.AddSingleton<ITableService, TableService>()
.BuildServiceProvider();
serviceProvider.GetService<ITableService>().UpdateTables();
}
}
}

16
src/FilterLists.DataLoad/FilterLists.DataLoad.csproj Normal file
View file

@ -0,0 +1,16 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp1.1</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="JetBrains.Annotations" Version="10.4.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\FilterLists.Services\FilterLists.Services.csproj" />
</ItemGroup>
</Project>

7
src/FilterLists.Services/Contracts/ITableService.cs Normal file
View file

@ -0,0 +1,7 @@
namespace FilterLists.Services.Contracts
{
public interface ITableService
{
void UpdateTables();
}
}

64
src/FilterLists.Services/Implementations/TableService.cs Normal file
View file

@ -0,0 +1,64 @@
using System.Net.Http;
using System.Net.Http.Headers;
using System.Threading.Tasks;
using FilterLists.Data.Models;
using FilterLists.Data.Repositories.Contracts;
using FilterLists.Services.Contracts;
namespace FilterLists.Services.Implementations
{
public class TableService : ITableService
{
//private readonly IListRepository _listRepository;
public TableService(/*IListRepository listRepository*/)
{
//_listRepository = listRepository;
}
/// <summary>
/// update all tables via .CSVs from GitHub
/// </summary>
public void UpdateTables()
{
//TODO: loop through all CSVs
UpdateTable("List");
}
/// <summary>
/// update table via .CSV from GitHub
/// </summary>
/// <param name="tableName">name of database table</param>
public void UpdateTable(string tableName)
{
//TODO: fetch CSV URL from db table
const string csvUrl = "https://raw.githubusercontent.com/collinbarrett/FilterLists/dataToCsv/data/List.csv";
var file = FetchFile(csvUrl).Result;
var csvRows = file.Split('\n');
var headerCells = csvRows[0].Split(',');
for (var i = 1; i < csvRows.Length; i++)
if (!string.IsNullOrEmpty(csvRows[i]))
{
var cells = csvRows[i].Split(',');
var list = new List
{
Name = cells[0]
//TODO: finish mappings, try to make generic for any table type using headerCells as properties
};
}
//TODO: update table via repo/EF
}
/// <summary>
/// fetch a file as a string from the internet
/// </summary>
/// <param name="url">URL of file to fetch</param>
/// <returns>string of file</returns>
private static async Task<string> FetchFile(string url)
{
var httpClient = new HttpClient();
httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("FilterLists", "1.0"));
return await httpClient.GetStringAsync(url);
}
}
}