prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

re-add UserAgent spoofing

Browse source 
closes #366
This commit is contained in:
Collin M. Barrett 2018-08-21 12:33:32 -05:00
parent e15ef93621
commit a646854695
3 changed files with 59 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
src/FilterLists.Services/Snapshot/Snapshot.cs
View file

@ -24,8 +24,10 @@ public class Snapshot
private readonly FilterListViewUrlDto list;
private readonly Data.Entities.Snapshot snapEntity;
private readonly TelemetryClient telemetryClient;
private readonly string userAgentString;
public Snapshot(FilterListsDbContext dbContext, EmailService emailService, FilterListViewUrlDto list)
public Snapshot(FilterListsDbContext dbContext, EmailService emailService, FilterListViewUrlDto list,
string userAgentString)
{
this.dbContext = dbContext;
this.emailService = emailService;
@ -35,6 +37,7 @@ public Snapshot(FilterListsDbContext dbContext, EmailService emailService, Filte
FilterListId = list.Id,
AddedSnapshotRules = new List<SnapshotRule>()
};
this.userAgentString = userAgentString;
telemetryClient = new TelemetryClient();
}
@ -59,7 +62,7 @@ private async Task AddSnapEntity()
dbContext.Snapshots.Add(snapEntity);
await dbContext.SaveChangesAsync();
}
private bool IsViewUrlValid() =>
Uri.TryCreate(list.ViewUrl, UriKind.Absolute, out var uriResult) &&
(uriResult.Scheme == Uri.UriSchemeHttp || uriResult.Scheme == Uri.UriSchemeHttps);
@ -104,6 +107,7 @@ private async Task<IEnumerable<string>> GetLines()
var lines = new HashSet<string>();
using (var httpClient = new HttpClient())
{
httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(userAgentString);
var response = await httpClient.GetAsync(list.ViewUrl, HttpCompletionOption.ResponseHeadersRead);
snapEntity.HttpStatusCode = ((int)response.StatusCode).ToString();
response.EnsureSuccessStatusCode();

7
src/FilterLists.Services/Snapshot/SnapshotService.cs
View file

@ -22,7 +22,8 @@ public SnapshotService(FilterListsDbContext dbContext, IConfigurationProvider ma
public async Task CaptureAsync(int batchSize)
{
var lists = await GetListsToCapture(batchSize);
var snapshots = CreateSnapshots(lists);
var uaString = await UserAgentService.GetMostPopularString();
var snapshots = CreateSnapshots(lists, uaString);
await SaveSnapshots(snapshots);
}
@ -44,8 +45,8 @@ await DbContext
.ProjectTo<FilterListViewUrlDto>(MapConfig)
.ToListAsync();
private IEnumerable<Snapshot> CreateSnapshots(IEnumerable<FilterListViewUrlDto> lists) =>
lists.Select(l => new Snapshot(DbContext, emailService, l));
private IEnumerable<Snapshot> CreateSnapshots(IEnumerable<FilterListViewUrlDto> lists, string uaString) =>
lists.Select(l => new Snapshot(DbContext, emailService, l, uaString));
private static async Task SaveSnapshots(IEnumerable<Snapshot> snapshots)
{

49
src/FilterLists.Services/UserAgentService.cs Normal file
View file

@ -0,0 +1,49 @@
using System;
using System.IO;
using System.Net.Http;
using System.Threading.Tasks;
using JetBrains.Annotations;
namespace FilterLists.Services
{
[UsedImplicitly]
public static class UserAgentService
{
private const string UaStringSource = "https://techblog.willshouse.com/2012/01/03/most-common-user-agents/";
private const string UaStringSourceClass = "get-the-list";
private const string UaStringDefault =
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36";
public static async Task<string> GetMostPopularString()
{
using (var httpClient = new HttpClient())
{
httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(UaStringDefault);
var response = await httpClient.GetAsync(UaStringSource, HttpCompletionOption.ResponseHeadersRead);
using (var stream = await response.Content.ReadAsStreamAsync())
using (var streamReader = new StreamReader(stream))
{
string line;
while ((line = await streamReader.ReadLineAsync()) != null)
if (line.Contains(UaStringSourceClass))
return ParseUaString(line);
}
}
return UaStringDefault;
}
private static string ParseUaString(string line)
{
var classIndex = line.IndexOf(UaStringSourceClass, StringComparison.Ordinal);
var postClassIndexSubstring = line.Substring(classIndex);
var uaStringStartIndex = postClassIndexSubstring.IndexOf(">", StringComparison.Ordinal) + 1;
var uaString = postClassIndexSubstring.Substring(uaStringStartIndex).Trim();
return IsSane(uaString) ? uaString : UaStringDefault;
}
private static bool IsSane(string uaString) =>
uaString.Length > 25 && uaString.Length < 300 && uaString.Contains("Mozilla");
}
}