diff --git a/Readme.md b/Readme.md index f82d051..cf9cd42 100644 --- a/Readme.md +++ b/Readme.md @@ -6,9 +6,10 @@ pip install itchiodl ``` ## Download All Games in library from account +**Please Note:** Having too many jobs may cause rate-limiting from some testing 8 works fine but 10 starts giving errors. ```bash - +# via python python -m itchiodl.downloader # via setup-tools entry point @@ -18,11 +19,11 @@ itch-download This uses the same API the itchio app uses to download the files. If you have 2FA enabled, generate an API key [here](https://itch.io/user/settings/api-keys) and run the following instead ```bash -# via python -python -m itchiodl.downloader --api-key=KEYHERE +# via python (with 4 concurrent downloads) +python -m itchiodl.downloader --api-key=KEYHERE --jobs=4 # via setup-tools entry point -itch-download -k KEYHERE +itch-download -k KEYHERE -j 4 ``` ## Add All Games in a bundle to your library diff --git a/itchiodl/downloader/__main__.py b/itchiodl/downloader/__main__.py index da04d48..7770a93 100644 --- a/itchiodl/downloader/__main__.py +++ b/itchiodl/downloader/__main__.py @@ -15,6 +15,13 @@ def main(): "--api-key", help="Use API key instead of username/password") + parser.add_argument( + "-j", + "--jobs", + type=int, + default=4, + help="Number of concurrent downloads, defaults to 4") + args = parser.parse_args() l = "" @@ -26,7 +33,7 @@ def main(): else: l = args.api_key - lib = itchiodl.Library(l) + lib = itchiodl.Library(l, args.jobs) lib.load_games() lib.download_library() diff --git a/itchiodl/library.py b/itchiodl/library.py index a88d11c..5859df2 100644 --- a/itchiodl/library.py +++ b/itchiodl/library.py @@ -1,13 +1,15 @@ import requests import json +from concurrent.futures import ThreadPoolExecutor from itchiodl.game import Game class Library: - def __init__(self, login): + def __init__(self, login, jobs=4): self.login = login self.games = [] + self.jobs = jobs def load_game_page(self, page): print("Loading page", page) @@ -31,5 +33,6 @@ class Library: page += 1 def download_library(self): - for game in self.games: - game.download(self.login) + with ThreadPoolExecutor(max_workers=self.jobs) as executor: + def dl(g): return g.download(self.login) + executor.map(dl, self.games) diff --git a/itchiodl/utils.py b/itchiodl/utils.py index efe0708..4dee575 100644 --- a/itchiodl/utils.py +++ b/itchiodl/utils.py @@ -38,6 +38,11 @@ def download(url, path, name, file): print(f"{filename} exists but is incomplete, downloading again") with open(f"{path}/{filename}", "wb") as f: + for chunk in rsp.iter_content(10240): + f.write(chunk) + + """ + # remove the progress bar output for multiple threads for chunk in progress.bar( rsp.iter_content( chunk_size=1024), expected_size=( @@ -45,4 +50,7 @@ def download(url, path, name, file): if chunk: f.write(chunk) f.flush() + """ + + print(f"Downloaded {filename}") return f"{path}/{filename}", True