Adding Multithreading (#20)

* Preparing to add multithreading

* Added Multithreading

* autopep8 action fixes (#21)

* Added docs about multithreading

Readme.md

@@ -6,9 +6,10 @@
 pip install itchiodl
 ```
 ## Download All Games in library from account
+**Please Note:** Having too many jobs may cause rate-limiting from some testing 8 works fine but 10 starts giving errors.
 
 ```bash
-
+# via python
 python -m itchiodl.downloader
 
 # via setup-tools entry point
@@ -18,11 +19,11 @@ itch-download
 This uses the same API the itchio app uses to download the files. If you have 2FA enabled, generate an API key [here](https://itch.io/user/settings/api-keys) and run the following instead
 
 ```bash
-# via python
-python -m itchiodl.downloader --api-key=KEYHERE
+# via python (with 4 concurrent downloads)
+python -m itchiodl.downloader --api-key=KEYHERE --jobs=4
 
 # via setup-tools entry point
-itch-download -k KEYHERE
+itch-download -k KEYHERE -j 4
 ```
 
 ## Add All Games in a bundle to your library

itchiodl/downloader/__main__.py

@@ -15,6 +15,13 @@ def main():
         "--api-key",
         help="Use API key instead of username/password")
 
+    parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        default=4,
+        help="Number of concurrent downloads, defaults to 4")
+
     args = parser.parse_args()
 
     l = ""
@@ -26,7 +33,7 @@ def main():
     else:
         l = args.api_key
 
-    lib = itchiodl.Library(l)
+    lib = itchiodl.Library(l, args.jobs)
     lib.load_games()
     lib.download_library()
 

itchiodl/library.py

@@ -1,13 +1,15 @@
 import requests
 import json
+from concurrent.futures import ThreadPoolExecutor
 
 from itchiodl.game import Game
 
 
 class Library:
-    def __init__(self, login):
+    def __init__(self, login, jobs=4):
         self.login = login
         self.games = []
+        self.jobs = jobs
 
     def load_game_page(self, page):
         print("Loading page", page)
@@ -31,5 +33,6 @@ class Library:
             page += 1
 
     def download_library(self):
-        for game in self.games:
-            game.download(self.login)
+        with ThreadPoolExecutor(max_workers=self.jobs) as executor:
+            def dl(g): return g.download(self.login)
+            executor.map(dl, self.games)

itchiodl/utils.py

@@ -38,6 +38,11 @@ def download(url, path, name, file):
             print(f"{filename} exists but is incomplete, downloading again")
 
     with open(f"{path}/{filename}", "wb") as f:
+        for chunk in rsp.iter_content(10240):
+            f.write(chunk)
+
+        """
+        # remove the progress bar output for multiple threads
         for chunk in progress.bar(
             rsp.iter_content(
                 chunk_size=1024), expected_size=(
@@ -45,4 +50,7 @@ def download(url, path, name, file):
             if chunk:
                 f.write(chunk)
                 f.flush()
+        """
+
+    print(f"Downloaded {filename}")
     return f"{path}/{filename}", True