From 7060bf2c11dffc76dac3681c39519310f98867d9 Mon Sep 17 00:00:00 2001
From: "Collin M. Barrett" <collinbarrett@users.noreply.github.com>
Date: Wed, 26 Jun 2019 21:19:34 -0500
Subject: [PATCH] initial support for parallel archive downloads

---
 docker-compose.yml                            |  3 +++
 .../ListArchiver/CaptureAllLists.cs           |  4 ++--
 .../ListArchiver/CaptureList.cs               | 20 ++++++++++++++++++-
 3 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 65d83fc0d..ec5aa2180 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -62,6 +62,8 @@ services:
       context: .
       dockerfile: src/FilterLists.Agent/Dockerfile
       target: final
+    volumes:
+      - archives:/app/archives
 
 networks:
   reverse-proxy:
@@ -69,3 +71,4 @@ networks:
 
 volumes:
   data:
+  archives:
diff --git a/src/FilterLists.Agent/ListArchiver/CaptureAllLists.cs b/src/FilterLists.Agent/ListArchiver/CaptureAllLists.cs
index 9bb73e45e..5e2750c27 100644
--- a/src/FilterLists.Agent/ListArchiver/CaptureAllLists.cs
+++ b/src/FilterLists.Agent/ListArchiver/CaptureAllLists.cs
@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using FilterLists.Agent.Entities;
@@ -29,8 +30,7 @@ protected override async Task Handle(Command request, CancellationToken cancella
             {
                 var listsRequest = new RestRequest("lists");
                 var lists = await _apiClient.ExecuteAsync<IEnumerable<ListInfo>>(listsRequest);
-                foreach (var list in lists)
-                    await _mediator.Send(new CaptureList.Command(list), cancellationToken);
+                await Task.WhenAll(lists.Select(l => _mediator.Send(new CaptureList.Command(l), cancellationToken)));
             }
         }
     }
diff --git a/src/FilterLists.Agent/ListArchiver/CaptureList.cs b/src/FilterLists.Agent/ListArchiver/CaptureList.cs
index bf3fcc236..389406728 100644
--- a/src/FilterLists.Agent/ListArchiver/CaptureList.cs
+++ b/src/FilterLists.Agent/ListArchiver/CaptureList.cs
@@ -1,4 +1,5 @@
-using System.Net.Http;
+using System.IO;
+using System.Net.Http;
 using System.Threading;
 using System.Threading.Tasks;
 using FilterLists.Agent.Entities;
@@ -29,6 +30,23 @@ public Handler(HttpClient httpClient)
 
             protected override async Task Handle(Command request, CancellationToken cancellationToken)
             {
+                if (Path.GetExtension(request.ListInfo.ViewUrl.AbsolutePath) == ".txt")
+                    try
+                    {
+                        using (var result = await _httpClient.GetAsync(request.ListInfo.ViewUrl, cancellationToken))
+                        {
+                            if (result.IsSuccessStatusCode)
+                                using (Stream output =
+                                    File.OpenWrite(Path.Combine("archives", $"{request.ListInfo.Id}.txt")))
+                                using (var input = await result.Content.ReadAsStreamAsync())
+                                {
+                                    input.CopyTo(output);
+                                }
+                        }
+                    }
+                    catch (HttpRequestException)
+                    {
+                    }
             }
         }
     }