prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

simplify snap fail logic

Browse source
This commit is contained in:
Collin M. Barrett 2018-09-01 15:50:46 -05:00
parent ea03226b9c
commit bb672b1e22
3 changed files with 6 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
src/FilterLists.Services/Snapshot/Snapshot.cs
View file

@ -29,7 +29,6 @@ public class Snapshot
private readonly string uaString;
protected string ListUrl;
private HashSet<string> lines;
private bool wasWebException;
public Snapshot()
{
@ -46,7 +45,7 @@ public Snapshot(FilterListsDbContext dbContext, FilterListViewUrlDto list, strin
telemetryClient = new TelemetryClient();
}
public bool RetryMirror => !SnapEntity.WasSuccessful && wasWebException;
public bool WebExcepted { get; private set; }
public virtual async Task TrySaveAsync() => await TrySaveAsyncBase();
@ -89,13 +88,13 @@ private async Task TryGetLines()
}
catch (HttpRequestException hre)
{
wasWebException = true;
WebExcepted = true;
await dbContext.SaveChangesAsync();
TrackException(hre);
}
catch (WebException we)
{
wasWebException = true;
WebExcepted = true;
SnapEntity.HttpStatusCode = (int)((HttpWebResponse)we.Response).StatusCode;
await dbContext.SaveChangesAsync();
TrackException(we);
@ -141,7 +140,8 @@ private void SetChecksum(Stream stream)
private async Task<bool> SaveIsNewChecksum()
{
SnapEntity.WasUpdated = !SnapEntity.Md5Checksum.SequenceEqual(await GetPreviousChecksum() ?? Array.Empty<byte>());
SnapEntity.WasUpdated =
!SnapEntity.Md5Checksum.SequenceEqual(await GetPreviousChecksum() ?? Array.Empty<byte>());
await dbContext.SaveChangesAsync();
return SnapEntity.WasUpdated;
}

2
src/FilterLists.Services/Snapshot/SnapshotService.cs
View file

@ -43,7 +43,7 @@ public async Task CaptureAsync(int batchSize)
uaString = await UserAgentService.GetMostPopularString();
var lists = await GetListsToCapture(batchSize);
var snaps = await CreateAndSaveSnaps<Snapshot>(lists);
var listsToRetry = snaps.Where(s => s.RetryMirror).Select(s => s.List);
var listsToRetry = snaps.Where(s => s.WebExcepted).Select(s => s.List);
await CreateAndSaveSnaps<SnapshotWayback>(listsToRetry);
}

2
src/FilterLists.Services/UserAgentService.cs
View file

@ -2,11 +2,9 @@
using System.IO;
using System.Net.Http;
using System.Threading.Tasks;
using JetBrains.Annotations;
namespace FilterLists.Services
{
[UsedImplicitly]
public static class UserAgentService
{
private const string UaStringSource = "https://techblog.willshouse.com/2012/01/03/most-common-user-agents/";