prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

set batchsize to 5k

Browse source 
ref #344
This commit is contained in:
Collin Barrett 2018-08-26 18:03:03 -05:00
parent 6dd85f50dd
commit 1f41153d02
4 changed files with 25 additions and 68 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
src/FilterLists.Agent/appsettings.json
View file

@ -11,6 +11,14 @@
"ConnectionStrings": {
"FilterListsConnection": ""
},
"Logging": {
"IncludeScopes": false,
"LogLevel": {
"Default": "Debug",
"System": "Information",
"Microsoft": "Information"
}
},
"SendGrid": {
"ApiKey": "",
"From": {
@ -21,16 +29,5 @@
"Name": "",
"EmailAddress": ""
}
},
"DataDirectory": {
"Path": "..\\..\\data"
},
"Logging": {
"IncludeScopes": false,
"LogLevel": {
"Default": "Debug",
"System": "Information",
"Microsoft": "Information"
}
}
}

22
src/FilterLists.Api/appsettings.json
View file

@ -11,17 +11,6 @@
"ConnectionStrings": {
"FilterListsConnection": ""
},
"SendGrid": {
"ApiKey": "",
"From": {
"Name": "",
"EmailAddress": ""
},
"To": {
"Name": "",
"EmailAddress": ""
}
},
"DataDirectory": {
"Path": "..\\..\\data"
},
@ -32,5 +21,16 @@
"System": "Information",
"Microsoft": "Information"
}
},
"SendGrid": {
"ApiKey": "",
"From": {
"Name": "",
"EmailAddress": ""
},
"To": {
"Name": "",
"EmailAddress": ""
}
}
}

46
src/FilterLists.Services/Snapshot/BatchSizeService.cs
View file

@ -1,55 +1,15 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using FilterLists.Data;
using Microsoft.EntityFrameworkCore;
using FilterLists.Data;
namespace FilterLists.Services.Snapshot
{
public class BatchSizeService : Service
{
private const int DefaultBatchSize = 1000;
private const float PercentMultiplier = 0.05F;
private const int DefaultBatchSize = 5000;
public BatchSizeService(FilterListsDbContext dbContext) : base(dbContext)
{
}
public async Task<int> GetBatchSize()
{
return DefaultBatchSize;
var recentSnapPerfs = await GetRecentSnapPerfs();
return recentSnapPerfs.Count == 2
? recentSnapPerfs[0].RulesCount > recentSnapPerfs[0].BatchSize
? (recentSnapPerfs[0].RulesPerMs >= recentSnapPerfs[1].RulesPerMs
? (int)Math.Round(recentSnapPerfs[0].BatchSize * (1 + PercentMultiplier))
: (int)Math.Round(recentSnapPerfs[0].BatchSize * (1 - PercentMultiplier)))
: recentSnapPerfs[0].BatchSize
: DefaultBatchSize;
}
private async Task<List<SnapshotPerformance>> GetRecentSnapPerfs() =>
await DbContext.Snapshots
.Where(s => s.WasSuccessful && s.BatchSize.HasValue)
.OrderByDescending(s => s.CreatedDateUtc)
.Take(2)
.Select(s => new SnapshotPerformance
{
BatchSize = s.BatchSize.Value,
CreatedDateUtc = s.CreatedDateUtc.Value,
ModifiedDateUtc = s.ModifiedDateUtc.Value,
RulesCount = s.SnapshotRules.Count
})
.ToListAsync();
private class SnapshotPerformance
{
public int BatchSize { get; set; }
public int RulesCount { get; set; }
public DateTime CreatedDateUtc { private get; set; }
public DateTime ModifiedDateUtc { private get; set; }
public int RulesPerMs => (ModifiedDateUtc - CreatedDateUtc).Milliseconds / RulesCount;
}
public static int GetBatchSize() => DefaultBatchSize;
}
}

6
src/FilterLists.Services/Snapshot/Snapshot.cs
View file

@ -122,13 +122,13 @@ private async Task GetLines()
private async Task SaveInBatches()
{
var snapBatches = await CreateBatches();
var snapBatches = CreateBatches();
await SaveBatches(snapBatches);
}
private async Task<IEnumerable<Batch>> CreateBatches()
private IEnumerable<Batch> CreateBatches()
{
SnapEntity.BatchSize = await batchSizeService.GetBatchSize();
SnapEntity.BatchSize = BatchSizeService.GetBatchSize();
return lines.Batch(SnapEntity.BatchSize.Value).Select(b => new Batch(dbContext, b, SnapEntity));
}