From 227c4d5a8245e789db4bdf3d4bcdec1d76c62a2f Mon Sep 17 00:00:00 2001 From: Collin Barrett Date: Sun, 26 Aug 2018 16:24:23 -0500 Subject: [PATCH] convert batch size to config ref #344 --- scripts/deploy.sh | 12 ++--- src/FilterLists.Agent/appsettings.json | 20 ++++---- src/FilterLists.Api/appsettings.json | 22 ++++---- .../Snapshot/BatchSizeService.cs | 51 +++---------------- src/FilterLists.Services/Snapshot/Snapshot.cs | 6 +-- 5 files changed, 35 insertions(+), 76 deletions(-) diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 6dbdd7de3..180622f24 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -3,9 +3,6 @@ #deploy seed data sshpass -p $FTP_PASSWORD scp -o StrictHostKeyChecking=no -r /home/travis/build/collinbarrett/FilterLists/data/ $FTP_USER@$FTP_HOST:$FTP_DIR -#deploy Agent -sshpass -p $FTP_PASSWORD scp -o StrictHostKeyChecking=no -r /home/travis/build/collinbarrett/FilterLists/src/FilterLists.Agent/bin/Release/netcoreapp2.1/ubuntu.18.04-x64/publish/* $FTP_USER@$FTP_HOST:$FTP_DIR - #deploy API sshpass -p $FTP_PASSWORD ssh -o StrictHostKeyChecking=no $FTP_USER@$FTP_HOST 'sudo systemctl stop filterlists.api.service' sshpass -p $FTP_PASSWORD scp -o StrictHostKeyChecking=no -r /home/travis/build/collinbarrett/FilterLists/src/FilterLists.Api/bin/Release/netcoreapp2.1/ubuntu.18.04-x64/publish/* $FTP_USER@$FTP_HOST:$FTP_DIR @@ -16,9 +13,8 @@ sshpass -p $FTP_PASSWORD ssh -o StrictHostKeyChecking=no $FTP_USER@$FTP_HOST 'su sshpass -p $FTP_PASSWORD scp -o StrictHostKeyChecking=no -r /home/travis/build/collinbarrett/FilterLists/src/FilterLists.Web/bin/Release/netcoreapp2.1/ubuntu.18.04-x64/publish/* $FTP_USER@$FTP_HOST:$FTP_DIR sshpass -p $FTP_PASSWORD ssh -o StrictHostKeyChecking=no $FTP_USER@$FTP_HOST 'sudo systemctl start filterlists.web.service' -#purge CDN -curl -X DELETE "https://api.cloudflare.com/client/v4/zones/$CF_FILTERLISTS_ZONE/purge_cache" -H "X-Auth-Email: $CF_EMAIL" -H "X-Auth-Key: $CF_GLOBAL_API_KEY" -H "Content-Type: application/json" --data '{"purge_everything":true}' -sleep 5 +#deploy Agent +sshpass -p $FTP_PASSWORD scp -o StrictHostKeyChecking=no -r /home/travis/build/collinbarrett/FilterLists/src/FilterLists.Agent/bin/Release/netcoreapp2.1/ubuntu.18.04-x64/publish/* $FTP_USER@$FTP_HOST:$FTP_DIR -#prime app -curl https://filterlists.com/api/v1/lists \ No newline at end of file +#purge CDN +#curl -X DELETE "https://api.cloudflare.com/client/v4/zones/$CF_FILTERLISTS_ZONE/purge_cache" -H "X-Auth-Email: $CF_EMAIL" -H "X-Auth-Key: $CF_GLOBAL_API_KEY" -H "Content-Type: application/json" --data '{"purge_everything":true}' \ No newline at end of file diff --git a/src/FilterLists.Agent/appsettings.json b/src/FilterLists.Agent/appsettings.json index 9dfea0c64..b38dcda59 100644 --- a/src/FilterLists.Agent/appsettings.json +++ b/src/FilterLists.Agent/appsettings.json @@ -11,6 +11,14 @@ "ConnectionStrings": { "FilterListsConnection": "" }, + "Logging": { + "IncludeScopes": false, + "LogLevel": { + "Default": "Debug", + "System": "Information", + "Microsoft": "Information" + } + }, "SendGrid": { "ApiKey": "", "From": { @@ -22,15 +30,7 @@ "EmailAddress": "" } }, - "DataDirectory": { - "Path": "..\\..\\data" - }, - "Logging": { - "IncludeScopes": false, - "LogLevel": { - "Default": "Debug", - "System": "Information", - "Microsoft": "Information" - } + "Snapshots": { + "BatchSize": "1000" } } \ No newline at end of file diff --git a/src/FilterLists.Api/appsettings.json b/src/FilterLists.Api/appsettings.json index 9dfea0c64..40219fe34 100644 --- a/src/FilterLists.Api/appsettings.json +++ b/src/FilterLists.Api/appsettings.json @@ -11,17 +11,6 @@ "ConnectionStrings": { "FilterListsConnection": "" }, - "SendGrid": { - "ApiKey": "", - "From": { - "Name": "", - "EmailAddress": "" - }, - "To": { - "Name": "", - "EmailAddress": "" - } - }, "DataDirectory": { "Path": "..\\..\\data" }, @@ -32,5 +21,16 @@ "System": "Information", "Microsoft": "Information" } + }, + "SendGrid": { + "ApiKey": "", + "From": { + "Name": "", + "EmailAddress": "" + }, + "To": { + "Name": "", + "EmailAddress": "" + } } } \ No newline at end of file diff --git a/src/FilterLists.Services/Snapshot/BatchSizeService.cs b/src/FilterLists.Services/Snapshot/BatchSizeService.cs index 5d4cfe7b7..fdfe6009b 100644 --- a/src/FilterLists.Services/Snapshot/BatchSizeService.cs +++ b/src/FilterLists.Services/Snapshot/BatchSizeService.cs @@ -1,55 +1,18 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using FilterLists.Data; -using Microsoft.EntityFrameworkCore; +using FilterLists.Data; +using Microsoft.Extensions.Configuration; namespace FilterLists.Services.Snapshot { public class BatchSizeService : Service { - private const int DefaultBatchSize = 1000; - private const float PercentMultiplier = 0.05F; + private readonly int batchSize = 1000; - public BatchSizeService(FilterListsDbContext dbContext) : base(dbContext) + public BatchSizeService(FilterListsDbContext dbContext, IConfiguration config) : base(dbContext) { + var configBatchSize = config.GetSection("Snapshots").GetValue("BatchSize"); + batchSize = configBatchSize > 0 ? configBatchSize : batchSize; } - public async Task GetBatchSize() - { - return DefaultBatchSize; - var recentSnapPerfs = await GetRecentSnapPerfs(); - return recentSnapPerfs.Count == 2 - ? recentSnapPerfs[0].RulesCount > recentSnapPerfs[0].BatchSize - ? (recentSnapPerfs[0].RulesPerMs >= recentSnapPerfs[1].RulesPerMs - ? (int)Math.Round(recentSnapPerfs[0].BatchSize * (1 + PercentMultiplier)) - : (int)Math.Round(recentSnapPerfs[0].BatchSize * (1 - PercentMultiplier))) - : recentSnapPerfs[0].BatchSize - : DefaultBatchSize; - } - - private async Task> GetRecentSnapPerfs() => - await DbContext.Snapshots - .Where(s => s.WasSuccessful && s.BatchSize.HasValue) - .OrderByDescending(s => s.CreatedDateUtc) - .Take(2) - .Select(s => new SnapshotPerformance - { - BatchSize = s.BatchSize.Value, - CreatedDateUtc = s.CreatedDateUtc.Value, - ModifiedDateUtc = s.ModifiedDateUtc.Value, - RulesCount = s.SnapshotRules.Count - }) - .ToListAsync(); - - private class SnapshotPerformance - { - public int BatchSize { get; set; } - public int RulesCount { get; set; } - public DateTime CreatedDateUtc { private get; set; } - public DateTime ModifiedDateUtc { private get; set; } - public int RulesPerMs => (ModifiedDateUtc - CreatedDateUtc).Milliseconds / RulesCount; - } + public int GetBatchSize() => batchSize; } } \ No newline at end of file diff --git a/src/FilterLists.Services/Snapshot/Snapshot.cs b/src/FilterLists.Services/Snapshot/Snapshot.cs index 3da8a886a..62b0c80c1 100644 --- a/src/FilterLists.Services/Snapshot/Snapshot.cs +++ b/src/FilterLists.Services/Snapshot/Snapshot.cs @@ -122,13 +122,13 @@ private async Task GetLines() private async Task SaveInBatches() { - var snapBatches = await CreateBatches(); + var snapBatches = CreateBatches(); await SaveBatches(snapBatches); } - private async Task> CreateBatches() + private IEnumerable CreateBatches() { - SnapEntity.BatchSize = await batchSizeService.GetBatchSize(); + SnapEntity.BatchSize = batchSizeService.GetBatchSize(); return lines.Batch(SnapEntity.BatchSize.Value).Select(b => new Batch(dbContext, b, SnapEntity)); }