prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

convert batch size to config

Browse source 
ref #344
This commit is contained in:
Collin Barrett 2018-08-26 16:24:23 -05:00
parent 87cf5edeab
commit 227c4d5a82
5 changed files with 35 additions and 76 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
scripts/deploy.sh
View file

@ -3,9 +3,6 @@
#deploy seed data
sshpass -p $FTP_PASSWORD scp -o StrictHostKeyChecking=no -r /home/travis/build/collinbarrett/FilterLists/data/ $FTP_USER@$FTP_HOST:$FTP_DIR
#deploy Agent
sshpass -p $FTP_PASSWORD scp -o StrictHostKeyChecking=no -r /home/travis/build/collinbarrett/FilterLists/src/FilterLists.Agent/bin/Release/netcoreapp2.1/ubuntu.18.04-x64/publish/* $FTP_USER@$FTP_HOST:$FTP_DIR
#deploy API
sshpass -p $FTP_PASSWORD ssh -o StrictHostKeyChecking=no $FTP_USER@$FTP_HOST 'sudo systemctl stop filterlists.api.service'
sshpass -p $FTP_PASSWORD scp -o StrictHostKeyChecking=no -r /home/travis/build/collinbarrett/FilterLists/src/FilterLists.Api/bin/Release/netcoreapp2.1/ubuntu.18.04-x64/publish/* $FTP_USER@$FTP_HOST:$FTP_DIR
@ -16,9 +13,8 @@ sshpass -p $FTP_PASSWORD ssh -o StrictHostKeyChecking=no $FTP_USER@$FTP_HOST 'su
sshpass -p $FTP_PASSWORD scp -o StrictHostKeyChecking=no -r /home/travis/build/collinbarrett/FilterLists/src/FilterLists.Web/bin/Release/netcoreapp2.1/ubuntu.18.04-x64/publish/* $FTP_USER@$FTP_HOST:$FTP_DIR
sshpass -p $FTP_PASSWORD ssh -o StrictHostKeyChecking=no $FTP_USER@$FTP_HOST 'sudo systemctl start filterlists.web.service'
#purge CDN
curl -X DELETE "https://api.cloudflare.com/client/v4/zones/$CF_FILTERLISTS_ZONE/purge_cache" -H "X-Auth-Email: $CF_EMAIL" -H "X-Auth-Key: $CF_GLOBAL_API_KEY" -H "Content-Type: application/json" --data '{"purge_everything":true}'
sleep 5
#deploy Agent
sshpass -p $FTP_PASSWORD scp -o StrictHostKeyChecking=no -r /home/travis/build/collinbarrett/FilterLists/src/FilterLists.Agent/bin/Release/netcoreapp2.1/ubuntu.18.04-x64/publish/* $FTP_USER@$FTP_HOST:$FTP_DIR
#prime app
curl https://filterlists.com/api/v1/lists
#purge CDN
#curl -X DELETE "https://api.cloudflare.com/client/v4/zones/$CF_FILTERLISTS_ZONE/purge_cache" -H "X-Auth-Email: $CF_EMAIL" -H "X-Auth-Key: $CF_GLOBAL_API_KEY" -H "Content-Type: application/json" --data '{"purge_everything":true}'

20
src/FilterLists.Agent/appsettings.json
View file

@ -11,6 +11,14 @@
"ConnectionStrings": {
"FilterListsConnection": ""
},
"Logging": {
"IncludeScopes": false,
"LogLevel": {
"Default": "Debug",
"System": "Information",
"Microsoft": "Information"
}
},
"SendGrid": {
"ApiKey": "",
"From": {
@ -22,15 +30,7 @@
"EmailAddress": ""
}
},
"DataDirectory": {
"Path": "..\\..\\data"
},
"Logging": {
"IncludeScopes": false,
"LogLevel": {
"Default": "Debug",
"System": "Information",
"Microsoft": "Information"
}
"Snapshots": {
"BatchSize": "1000"
}
}

22
src/FilterLists.Api/appsettings.json
View file

@ -11,17 +11,6 @@
"ConnectionStrings": {
"FilterListsConnection": ""
},
"SendGrid": {
"ApiKey": "",
"From": {
"Name": "",
"EmailAddress": ""
},
"To": {
"Name": "",
"EmailAddress": ""
}
},
"DataDirectory": {
"Path": "..\\..\\data"
},
@ -32,5 +21,16 @@
"System": "Information",
"Microsoft": "Information"
}
},
"SendGrid": {
"ApiKey": "",
"From": {
"Name": "",
"EmailAddress": ""
},
"To": {
"Name": "",
"EmailAddress": ""
}
}
}

51
src/FilterLists.Services/Snapshot/BatchSizeService.cs
View file

@ -1,55 +1,18 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using FilterLists.Data;
using Microsoft.EntityFrameworkCore;
using FilterLists.Data;
using Microsoft.Extensions.Configuration;
namespace FilterLists.Services.Snapshot
{
public class BatchSizeService : Service
{
private const int DefaultBatchSize = 1000;
private const float PercentMultiplier = 0.05F;
private readonly int batchSize = 1000;
public BatchSizeService(FilterListsDbContext dbContext) : base(dbContext)
public BatchSizeService(FilterListsDbContext dbContext, IConfiguration config) : base(dbContext)
{
var configBatchSize = config.GetSection("Snapshots").GetValue<int>("BatchSize");
batchSize = configBatchSize > 0 ? configBatchSize : batchSize;
}
public async Task<int> GetBatchSize()
{
return DefaultBatchSize;
var recentSnapPerfs = await GetRecentSnapPerfs();
return recentSnapPerfs.Count == 2
? recentSnapPerfs[0].RulesCount > recentSnapPerfs[0].BatchSize
? (recentSnapPerfs[0].RulesPerMs >= recentSnapPerfs[1].RulesPerMs
? (int)Math.Round(recentSnapPerfs[0].BatchSize * (1 + PercentMultiplier))
: (int)Math.Round(recentSnapPerfs[0].BatchSize * (1 - PercentMultiplier)))
: recentSnapPerfs[0].BatchSize
: DefaultBatchSize;
}
private async Task<List<SnapshotPerformance>> GetRecentSnapPerfs() =>
await DbContext.Snapshots
.Where(s => s.WasSuccessful && s.BatchSize.HasValue)
.OrderByDescending(s => s.CreatedDateUtc)
.Take(2)
.Select(s => new SnapshotPerformance
{
BatchSize = s.BatchSize.Value,
CreatedDateUtc = s.CreatedDateUtc.Value,
ModifiedDateUtc = s.ModifiedDateUtc.Value,
RulesCount = s.SnapshotRules.Count
})
.ToListAsync();
private class SnapshotPerformance
{
public int BatchSize { get; set; }
public int RulesCount { get; set; }
public DateTime CreatedDateUtc { private get; set; }
public DateTime ModifiedDateUtc { private get; set; }
public int RulesPerMs => (ModifiedDateUtc - CreatedDateUtc).Milliseconds / RulesCount;
}
public int GetBatchSize() => batchSize;
}
}

6
src/FilterLists.Services/Snapshot/Snapshot.cs
View file

@ -122,13 +122,13 @@ private async Task GetLines()
private async Task SaveInBatches()
{
var snapBatches = await CreateBatches();
var snapBatches = CreateBatches();
await SaveBatches(snapBatches);
}
private async Task<IEnumerable<Batch>> CreateBatches()
private IEnumerable<Batch> CreateBatches()
{
SnapEntity.BatchSize = await batchSizeService.GetBatchSize();
SnapEntity.BatchSize = batchSizeService.GetBatchSize();
return lines.Batch(SnapEntity.BatchSize.Value).Select(b => new Batch(dbContext, b, SnapEntity));
}