From c753c0ff4001a303561f99b9d15f56704948a3d6 Mon Sep 17 00:00:00 2001
From: "Collin M. Barrett" <collinbarrett@users.noreply.github.com>
Date: Tue, 21 Aug 2018 12:33:32 -0500
Subject: [PATCH] re-add UserAgent spoofing

closes #366
---
 src/FilterLists.Services/Snapshot/Snapshot.cs |  8 ++-
 .../Snapshot/SnapshotService.cs               |  7 +--
 src/FilterLists.Services/UserAgentService.cs  | 49 +++++++++++++++++++
 3 files changed, 59 insertions(+), 5 deletions(-)
 create mode 100644 src/FilterLists.Services/UserAgentService.cs

diff --git a/src/FilterLists.Services/Snapshot/Snapshot.cs b/src/FilterLists.Services/Snapshot/Snapshot.cs
index f4123a524..f9cdc0da3 100644
--- a/src/FilterLists.Services/Snapshot/Snapshot.cs
+++ b/src/FilterLists.Services/Snapshot/Snapshot.cs
@@ -24,8 +24,10 @@ public class Snapshot
         private readonly FilterListViewUrlDto list;
         private readonly Data.Entities.Snapshot snapEntity;
         private readonly TelemetryClient telemetryClient;
+        private readonly string userAgentString;
 
-        public Snapshot(FilterListsDbContext dbContext, EmailService emailService, FilterListViewUrlDto list)
+        public Snapshot(FilterListsDbContext dbContext, EmailService emailService, FilterListViewUrlDto list,
+            string userAgentString)
         {
             this.dbContext = dbContext;
             this.emailService = emailService;
@@ -35,6 +37,7 @@ public Snapshot(FilterListsDbContext dbContext, EmailService emailService, Filte
                 FilterListId = list.Id,
                 AddedSnapshotRules = new List<SnapshotRule>()
             };
+            this.userAgentString = userAgentString;
             telemetryClient = new TelemetryClient();
         }
 
@@ -59,7 +62,7 @@ private async Task AddSnapEntity()
             dbContext.Snapshots.Add(snapEntity);
             await dbContext.SaveChangesAsync();
         }
-        
+
         private bool IsViewUrlValid() =>
             Uri.TryCreate(list.ViewUrl, UriKind.Absolute, out var uriResult) &&
             (uriResult.Scheme == Uri.UriSchemeHttp || uriResult.Scheme == Uri.UriSchemeHttps);
@@ -104,6 +107,7 @@ private async Task<IEnumerable<string>> GetLines()
             var lines = new HashSet<string>();
             using (var httpClient = new HttpClient())
             {
+                httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(userAgentString);
                 var response = await httpClient.GetAsync(list.ViewUrl, HttpCompletionOption.ResponseHeadersRead);
                 snapEntity.HttpStatusCode = ((int)response.StatusCode).ToString();
                 response.EnsureSuccessStatusCode();
diff --git a/src/FilterLists.Services/Snapshot/SnapshotService.cs b/src/FilterLists.Services/Snapshot/SnapshotService.cs
index b77d7eb6a..85a1c90b3 100644
--- a/src/FilterLists.Services/Snapshot/SnapshotService.cs
+++ b/src/FilterLists.Services/Snapshot/SnapshotService.cs
@@ -22,7 +22,8 @@ public SnapshotService(FilterListsDbContext dbContext, IConfigurationProvider ma
         public async Task CaptureAsync(int batchSize)
         {
             var lists = await GetListsToCapture(batchSize);
-            var snapshots = CreateSnapshots(lists);
+            var uaString = await UserAgentService.GetMostPopularString();
+            var snapshots = CreateSnapshots(lists, uaString);
             await SaveSnapshots(snapshots);
         }
 
@@ -44,8 +45,8 @@ await DbContext
                   .ProjectTo<FilterListViewUrlDto>(MapConfig)
                   .ToListAsync();
 
-        private IEnumerable<Snapshot> CreateSnapshots(IEnumerable<FilterListViewUrlDto> lists) =>
-            lists.Select(l => new Snapshot(DbContext, emailService, l));
+        private IEnumerable<Snapshot> CreateSnapshots(IEnumerable<FilterListViewUrlDto> lists, string uaString) =>
+            lists.Select(l => new Snapshot(DbContext, emailService, l, uaString));
 
         private static async Task SaveSnapshots(IEnumerable<Snapshot> snapshots)
         {
diff --git a/src/FilterLists.Services/UserAgentService.cs b/src/FilterLists.Services/UserAgentService.cs
new file mode 100644
index 000000000..cc48a9d5b
--- /dev/null
+++ b/src/FilterLists.Services/UserAgentService.cs
@@ -0,0 +1,49 @@
+using System;
+using System.IO;
+using System.Net.Http;
+using System.Threading.Tasks;
+using JetBrains.Annotations;
+
+namespace FilterLists.Services
+{
+    [UsedImplicitly]
+    public static class UserAgentService
+    {
+        private const string UaStringSource = "https://techblog.willshouse.com/2012/01/03/most-common-user-agents/";
+        private const string UaStringSourceClass = "get-the-list";
+
+        private const string UaStringDefault =
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36";
+
+        public static async Task<string> GetMostPopularString()
+        {
+            using (var httpClient = new HttpClient())
+            {
+                httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(UaStringDefault);
+                var response = await httpClient.GetAsync(UaStringSource, HttpCompletionOption.ResponseHeadersRead);
+                using (var stream = await response.Content.ReadAsStreamAsync())
+                using (var streamReader = new StreamReader(stream))
+                {
+                    string line;
+                    while ((line = await streamReader.ReadLineAsync()) != null)
+                        if (line.Contains(UaStringSourceClass))
+                            return ParseUaString(line);
+                }
+            }
+
+            return UaStringDefault;
+        }
+
+        private static string ParseUaString(string line)
+        {
+            var classIndex = line.IndexOf(UaStringSourceClass, StringComparison.Ordinal);
+            var postClassIndexSubstring = line.Substring(classIndex);
+            var uaStringStartIndex = postClassIndexSubstring.IndexOf(">", StringComparison.Ordinal) + 1;
+            var uaString = postClassIndexSubstring.Substring(uaStringStartIndex).Trim();
+            return IsSane(uaString) ? uaString : UaStringDefault;
+        }
+
+        private static bool IsSane(string uaString) =>
+            uaString.Length > 25 && uaString.Length < 300 && uaString.Contains("Mozilla");
+    }
+}
\ No newline at end of file