diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 180622f24..6dbdd7de3 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -3,6 +3,9 @@ #deploy seed data sshpass -p $FTP_PASSWORD scp -o StrictHostKeyChecking=no -r /home/travis/build/collinbarrett/FilterLists/data/ $FTP_USER@$FTP_HOST:$FTP_DIR +#deploy Agent +sshpass -p $FTP_PASSWORD scp -o StrictHostKeyChecking=no -r /home/travis/build/collinbarrett/FilterLists/src/FilterLists.Agent/bin/Release/netcoreapp2.1/ubuntu.18.04-x64/publish/* $FTP_USER@$FTP_HOST:$FTP_DIR + #deploy API sshpass -p $FTP_PASSWORD ssh -o StrictHostKeyChecking=no $FTP_USER@$FTP_HOST 'sudo systemctl stop filterlists.api.service' sshpass -p $FTP_PASSWORD scp -o StrictHostKeyChecking=no -r /home/travis/build/collinbarrett/FilterLists/src/FilterLists.Api/bin/Release/netcoreapp2.1/ubuntu.18.04-x64/publish/* $FTP_USER@$FTP_HOST:$FTP_DIR @@ -13,8 +16,9 @@ sshpass -p $FTP_PASSWORD ssh -o StrictHostKeyChecking=no $FTP_USER@$FTP_HOST 'su sshpass -p $FTP_PASSWORD scp -o StrictHostKeyChecking=no -r /home/travis/build/collinbarrett/FilterLists/src/FilterLists.Web/bin/Release/netcoreapp2.1/ubuntu.18.04-x64/publish/* $FTP_USER@$FTP_HOST:$FTP_DIR sshpass -p $FTP_PASSWORD ssh -o StrictHostKeyChecking=no $FTP_USER@$FTP_HOST 'sudo systemctl start filterlists.web.service' -#deploy Agent -sshpass -p $FTP_PASSWORD scp -o StrictHostKeyChecking=no -r /home/travis/build/collinbarrett/FilterLists/src/FilterLists.Agent/bin/Release/netcoreapp2.1/ubuntu.18.04-x64/publish/* $FTP_USER@$FTP_HOST:$FTP_DIR - #purge CDN -#curl -X DELETE "https://api.cloudflare.com/client/v4/zones/$CF_FILTERLISTS_ZONE/purge_cache" -H "X-Auth-Email: $CF_EMAIL" -H "X-Auth-Key: $CF_GLOBAL_API_KEY" -H "Content-Type: application/json" --data '{"purge_everything":true}' \ No newline at end of file +curl -X DELETE "https://api.cloudflare.com/client/v4/zones/$CF_FILTERLISTS_ZONE/purge_cache" -H "X-Auth-Email: $CF_EMAIL" -H "X-Auth-Key: $CF_GLOBAL_API_KEY" -H "Content-Type: application/json" --data '{"purge_everything":true}' +sleep 5 + +#prime app +curl https://filterlists.com/api/v1/lists \ No newline at end of file diff --git a/src/FilterLists.Agent/appsettings.json b/src/FilterLists.Agent/appsettings.json index b38dcda59..9dfea0c64 100644 --- a/src/FilterLists.Agent/appsettings.json +++ b/src/FilterLists.Agent/appsettings.json @@ -11,14 +11,6 @@ "ConnectionStrings": { "FilterListsConnection": "" }, - "Logging": { - "IncludeScopes": false, - "LogLevel": { - "Default": "Debug", - "System": "Information", - "Microsoft": "Information" - } - }, "SendGrid": { "ApiKey": "", "From": { @@ -30,7 +22,15 @@ "EmailAddress": "" } }, - "Snapshots": { - "BatchSize": "1000" + "DataDirectory": { + "Path": "..\\..\\data" + }, + "Logging": { + "IncludeScopes": false, + "LogLevel": { + "Default": "Debug", + "System": "Information", + "Microsoft": "Information" + } } } \ No newline at end of file diff --git a/src/FilterLists.Api/appsettings.json b/src/FilterLists.Api/appsettings.json index 40219fe34..9dfea0c64 100644 --- a/src/FilterLists.Api/appsettings.json +++ b/src/FilterLists.Api/appsettings.json @@ -11,17 +11,6 @@ "ConnectionStrings": { "FilterListsConnection": "" }, - "DataDirectory": { - "Path": "..\\..\\data" - }, - "Logging": { - "IncludeScopes": false, - "LogLevel": { - "Default": "Debug", - "System": "Information", - "Microsoft": "Information" - } - }, "SendGrid": { "ApiKey": "", "From": { @@ -32,5 +21,16 @@ "Name": "", "EmailAddress": "" } + }, + "DataDirectory": { + "Path": "..\\..\\data" + }, + "Logging": { + "IncludeScopes": false, + "LogLevel": { + "Default": "Debug", + "System": "Information", + "Microsoft": "Information" + } } } \ No newline at end of file diff --git a/src/FilterLists.Services/Snapshot/BatchSizeService.cs b/src/FilterLists.Services/Snapshot/BatchSizeService.cs index fdfe6009b..5d4cfe7b7 100644 --- a/src/FilterLists.Services/Snapshot/BatchSizeService.cs +++ b/src/FilterLists.Services/Snapshot/BatchSizeService.cs @@ -1,18 +1,55 @@ -using FilterLists.Data; -using Microsoft.Extensions.Configuration; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using FilterLists.Data; +using Microsoft.EntityFrameworkCore; namespace FilterLists.Services.Snapshot { public class BatchSizeService : Service { - private readonly int batchSize = 1000; + private const int DefaultBatchSize = 1000; + private const float PercentMultiplier = 0.05F; - public BatchSizeService(FilterListsDbContext dbContext, IConfiguration config) : base(dbContext) + public BatchSizeService(FilterListsDbContext dbContext) : base(dbContext) { - var configBatchSize = config.GetSection("Snapshots").GetValue("BatchSize"); - batchSize = configBatchSize > 0 ? configBatchSize : batchSize; } - public int GetBatchSize() => batchSize; + public async Task GetBatchSize() + { + return DefaultBatchSize; + var recentSnapPerfs = await GetRecentSnapPerfs(); + return recentSnapPerfs.Count == 2 + ? recentSnapPerfs[0].RulesCount > recentSnapPerfs[0].BatchSize + ? (recentSnapPerfs[0].RulesPerMs >= recentSnapPerfs[1].RulesPerMs + ? (int)Math.Round(recentSnapPerfs[0].BatchSize * (1 + PercentMultiplier)) + : (int)Math.Round(recentSnapPerfs[0].BatchSize * (1 - PercentMultiplier))) + : recentSnapPerfs[0].BatchSize + : DefaultBatchSize; + } + + private async Task> GetRecentSnapPerfs() => + await DbContext.Snapshots + .Where(s => s.WasSuccessful && s.BatchSize.HasValue) + .OrderByDescending(s => s.CreatedDateUtc) + .Take(2) + .Select(s => new SnapshotPerformance + { + BatchSize = s.BatchSize.Value, + CreatedDateUtc = s.CreatedDateUtc.Value, + ModifiedDateUtc = s.ModifiedDateUtc.Value, + RulesCount = s.SnapshotRules.Count + }) + .ToListAsync(); + + private class SnapshotPerformance + { + public int BatchSize { get; set; } + public int RulesCount { get; set; } + public DateTime CreatedDateUtc { private get; set; } + public DateTime ModifiedDateUtc { private get; set; } + public int RulesPerMs => (ModifiedDateUtc - CreatedDateUtc).Milliseconds / RulesCount; + } } } \ No newline at end of file diff --git a/src/FilterLists.Services/Snapshot/Snapshot.cs b/src/FilterLists.Services/Snapshot/Snapshot.cs index 62b0c80c1..3da8a886a 100644 --- a/src/FilterLists.Services/Snapshot/Snapshot.cs +++ b/src/FilterLists.Services/Snapshot/Snapshot.cs @@ -122,13 +122,13 @@ private async Task GetLines() private async Task SaveInBatches() { - var snapBatches = CreateBatches(); + var snapBatches = await CreateBatches(); await SaveBatches(snapBatches); } - private IEnumerable CreateBatches() + private async Task> CreateBatches() { - SnapEntity.BatchSize = batchSizeService.GetBatchSize(); + SnapEntity.BatchSize = await batchSizeService.GetBatchSize(); return lines.Batch(SnapEntity.BatchSize.Value).Select(b => new Batch(dbContext, b, SnapEntity)); }