From 1f41153d02ceffbd3412f22116483a5fefb673f7 Mon Sep 17 00:00:00 2001
From: Collin Barrett <collinbarrett@users.noreply.github.com>
Date: Sun, 26 Aug 2018 18:03:03 -0500
Subject: [PATCH] set batchsize to 5k

ref #344
---
 src/FilterLists.Agent/appsettings.json        | 19 ++++----
 src/FilterLists.Api/appsettings.json          | 22 ++++-----
 .../Snapshot/BatchSizeService.cs              | 46 ++-----------------
 src/FilterLists.Services/Snapshot/Snapshot.cs |  6 +--
 4 files changed, 25 insertions(+), 68 deletions(-)

diff --git a/src/FilterLists.Agent/appsettings.json b/src/FilterLists.Agent/appsettings.json
index 9dfea0c64..9e5c733bb 100644
--- a/src/FilterLists.Agent/appsettings.json
+++ b/src/FilterLists.Agent/appsettings.json
@@ -11,6 +11,14 @@
   "ConnectionStrings": {
     "FilterListsConnection": ""
   },
+  "Logging": {
+    "IncludeScopes": false,
+    "LogLevel": {
+      "Default": "Debug",
+      "System": "Information",
+      "Microsoft": "Information"
+    }
+  },
   "SendGrid": {
     "ApiKey": "",
     "From": {
@@ -21,16 +29,5 @@
       "Name": "",
       "EmailAddress": ""
     }
-  },
-  "DataDirectory": {
-    "Path": "..\\..\\data"
-  },
-  "Logging": {
-    "IncludeScopes": false,
-    "LogLevel": {
-      "Default": "Debug",
-      "System": "Information",
-      "Microsoft": "Information"
-    }
   }
 }
\ No newline at end of file
diff --git a/src/FilterLists.Api/appsettings.json b/src/FilterLists.Api/appsettings.json
index 9dfea0c64..40219fe34 100644
--- a/src/FilterLists.Api/appsettings.json
+++ b/src/FilterLists.Api/appsettings.json
@@ -11,17 +11,6 @@
   "ConnectionStrings": {
     "FilterListsConnection": ""
   },
-  "SendGrid": {
-    "ApiKey": "",
-    "From": {
-      "Name": "",
-      "EmailAddress": ""
-    },
-    "To": {
-      "Name": "",
-      "EmailAddress": ""
-    }
-  },
   "DataDirectory": {
     "Path": "..\\..\\data"
   },
@@ -32,5 +21,16 @@
       "System": "Information",
       "Microsoft": "Information"
     }
+  },
+  "SendGrid": {
+    "ApiKey": "",
+    "From": {
+      "Name": "",
+      "EmailAddress": ""
+    },
+    "To": {
+      "Name": "",
+      "EmailAddress": ""
+    }
   }
 }
\ No newline at end of file
diff --git a/src/FilterLists.Services/Snapshot/BatchSizeService.cs b/src/FilterLists.Services/Snapshot/BatchSizeService.cs
index 5d4cfe7b7..f562ce294 100644
--- a/src/FilterLists.Services/Snapshot/BatchSizeService.cs
+++ b/src/FilterLists.Services/Snapshot/BatchSizeService.cs
@@ -1,55 +1,15 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
-using FilterLists.Data;
-using Microsoft.EntityFrameworkCore;
+using FilterLists.Data;
 
 namespace FilterLists.Services.Snapshot
 {
     public class BatchSizeService : Service
     {
-        private const int DefaultBatchSize = 1000;
-        private const float PercentMultiplier = 0.05F;
+        private const int DefaultBatchSize = 5000;
 
         public BatchSizeService(FilterListsDbContext dbContext) : base(dbContext)
         {
         }
 
-        public async Task<int> GetBatchSize()
-        {
-            return DefaultBatchSize;
-            var recentSnapPerfs = await GetRecentSnapPerfs();
-            return recentSnapPerfs.Count == 2
-                ? recentSnapPerfs[0].RulesCount > recentSnapPerfs[0].BatchSize
-                    ? (recentSnapPerfs[0].RulesPerMs >= recentSnapPerfs[1].RulesPerMs
-                        ? (int)Math.Round(recentSnapPerfs[0].BatchSize * (1 + PercentMultiplier))
-                        : (int)Math.Round(recentSnapPerfs[0].BatchSize * (1 - PercentMultiplier)))
-                    : recentSnapPerfs[0].BatchSize
-                : DefaultBatchSize;
-        }
-
-        private async Task<List<SnapshotPerformance>> GetRecentSnapPerfs() =>
-            await DbContext.Snapshots
-                           .Where(s => s.WasSuccessful && s.BatchSize.HasValue)
-                           .OrderByDescending(s => s.CreatedDateUtc)
-                           .Take(2)
-                           .Select(s => new SnapshotPerformance
-                           {
-                               BatchSize = s.BatchSize.Value,
-                               CreatedDateUtc = s.CreatedDateUtc.Value,
-                               ModifiedDateUtc = s.ModifiedDateUtc.Value,
-                               RulesCount = s.SnapshotRules.Count
-                           })
-                           .ToListAsync();
-
-        private class SnapshotPerformance
-        {
-            public int BatchSize { get; set; }
-            public int RulesCount { get; set; }
-            public DateTime CreatedDateUtc { private get; set; }
-            public DateTime ModifiedDateUtc { private get; set; }
-            public int RulesPerMs => (ModifiedDateUtc - CreatedDateUtc).Milliseconds / RulesCount;
-        }
+        public static int GetBatchSize() => DefaultBatchSize;
     }
 }
\ No newline at end of file
diff --git a/src/FilterLists.Services/Snapshot/Snapshot.cs b/src/FilterLists.Services/Snapshot/Snapshot.cs
index 3da8a886a..1b33f54e1 100644
--- a/src/FilterLists.Services/Snapshot/Snapshot.cs
+++ b/src/FilterLists.Services/Snapshot/Snapshot.cs
@@ -122,13 +122,13 @@ private async Task GetLines()
 
         private async Task SaveInBatches()
         {
-            var snapBatches = await CreateBatches();
+            var snapBatches = CreateBatches();
             await SaveBatches(snapBatches);
         }
 
-        private async Task<IEnumerable<Batch>> CreateBatches()
+        private IEnumerable<Batch> CreateBatches()
         {
-            SnapEntity.BatchSize = await batchSizeService.GetBatchSize();
+            SnapEntity.BatchSize = BatchSizeService.GetBatchSize();
             return lines.Batch(SnapEntity.BatchSize.Value).Select(b => new Batch(dbContext, b, SnapEntity));
         }