From 8e3a678967267c3df7de9bf4cc46562141b15057 Mon Sep 17 00:00:00 2001 From: Peter Taylor Date: Thu, 5 May 2022 11:19:42 +0100 Subject: [PATCH 01/10] Added Progress Option --- itchiodl/library.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/itchiodl/library.py b/itchiodl/library.py index a401025..687f7e7 100644 --- a/itchiodl/library.py +++ b/itchiodl/library.py @@ -1,5 +1,6 @@ import json from concurrent.futures import ThreadPoolExecutor +import re import requests from itchiodl.game import Game @@ -38,9 +39,13 @@ class Library: def download_library(self, platform=None): """Download all games in the library""" - with ThreadPoolExecutor(max_workers=self.jobs) as executor: + with ThreadPoolExecutor(max_workers=self.jobs) as executor: + i = 0 + l = len(self.games) def dl(g): - return g.download(self.login, platform) + x = g.download(self.login, platform) + print(f"Downloaded {i} games of {l}") + return x executor.map(dl, self.games) From f55558cb927e4bccd6ee6285e6484b82bd6651e5 Mon Sep 17 00:00:00 2001 From: Peter Taylor Date: Thu, 5 May 2022 11:22:22 +0100 Subject: [PATCH 02/10] VSCode adding unwanted imports --- itchiodl/library.py | 1 - 1 file changed, 1 deletion(-) diff --git a/itchiodl/library.py b/itchiodl/library.py index 687f7e7..597f1f8 100644 --- a/itchiodl/library.py +++ b/itchiodl/library.py @@ -1,6 +1,5 @@ import json from concurrent.futures import ThreadPoolExecutor -import re import requests from itchiodl.game import Game From e66e81a5c01259059d76580872960bcaf829f820 Mon Sep 17 00:00:00 2001 From: Peter Taylor Date: Thu, 5 May 2022 11:26:19 +0100 Subject: [PATCH 03/10] linting --- itchiodl/library.py | 1 + 1 file changed, 1 insertion(+) diff --git a/itchiodl/library.py b/itchiodl/library.py index 597f1f8..f563901 100644 --- a/itchiodl/library.py +++ b/itchiodl/library.py @@ -42,6 +42,7 @@ class Library: with ThreadPoolExecutor(max_workers=self.jobs) as executor: i = 0 l = len(self.games) + def dl(g): x = g.download(self.login, platform) print(f"Downloaded {i} games of {l}") From be5baca3775eed81e32b14575b065a9ba78e0723 Mon Sep 17 00:00:00 2001 From: Peter Taylor Date: Thu, 5 May 2022 14:43:54 +0100 Subject: [PATCH 04/10] Added individual game downloader --- itchiodl/downloader/__main__.py | 23 ++++++++++++++++++++++- itchiodl/library.py | 18 +++++++++++++++++- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/itchiodl/downloader/__main__.py b/itchiodl/downloader/__main__.py index aada99d..15cde04 100644 --- a/itchiodl/downloader/__main__.py +++ b/itchiodl/downloader/__main__.py @@ -1,5 +1,6 @@ import argparse from getpass import getpass +import re import itchiodl @@ -25,6 +26,18 @@ def main(): help="Number of concurrent downloads, defaults to 4", ) + parser.add_argument( + "--download-publisher", + type=str, + help="Download all games from a specific publisher", + ) + + parser.add_argument( + "--download-game", + type=str, + help="Download a specific game, should be in the format publisher.itch.io/game", + ) + args = parser.parse_args() l = "" @@ -37,7 +50,15 @@ def main(): l = args.api_key lib = itchiodl.Library(l, args.jobs) - lib.load_games() + + if args.download_publisher: + lib.load_game(args.download_publisher) + elif args.download_game: + matches = re.match(r"https://(.+)\.itch\.io/(.+)", args.download_game) + lib.load_game(matches.group(1), matches.group(2)) + else: + lib.load_owned_games() + lib.download_library(args.platform) diff --git a/itchiodl/library.py b/itchiodl/library.py index f563901..d2563c8 100644 --- a/itchiodl/library.py +++ b/itchiodl/library.py @@ -1,6 +1,7 @@ import json from concurrent.futures import ThreadPoolExecutor import requests +from bs4 import BeautifulSoup from itchiodl.game import Game @@ -27,7 +28,7 @@ class Library: return len(j["owned_keys"]) - def load_games(self): + def load_owned_games(self): """Load all games in the library via the API""" page = 1 while True: @@ -36,6 +37,21 @@ class Library: break page += 1 + def load_game(self, publisher, title): + """Load a game by publisher and title""" + requests.get( + f"https://{publisher}.itch.io/{title}/data.json", + headers={"Authorization": self.login}, + ) + self.games.append(Game(publisher, title)) + + def load_games(self, publisher): + """Load all games by publisher""" + rsp = requests.get(f"https://{publisher}.itch.io") + soup = BeautifulSoup(rsp.text, "html.parser") + for link in soup.select("a.game-link"): + self.games.append(Game(link.get("data-label").split(":")[1])) + def download_library(self, platform=None): """Download all games in the library""" From 58d10805eab9904c09d45f12ec81705d7841f774 Mon Sep 17 00:00:00 2001 From: Peter Taylor Date: Thu, 5 May 2022 15:31:55 +0100 Subject: [PATCH 05/10] issue on the list of args being passed --- itchiodl/library.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/itchiodl/library.py b/itchiodl/library.py index d2563c8..452a3bf 100644 --- a/itchiodl/library.py +++ b/itchiodl/library.py @@ -43,7 +43,8 @@ class Library: f"https://{publisher}.itch.io/{title}/data.json", headers={"Authorization": self.login}, ) - self.games.append(Game(publisher, title)) + j = json.loads(r.text) + self.games.append(Game(j["id"])) def load_games(self, publisher): """Load all games by publisher""" From e8ef0f8a5a2afdca33452acfe139f3e6f1e8be5f Mon Sep 17 00:00:00 2001 From: Peter Taylor Date: Thu, 5 May 2022 15:52:27 +0100 Subject: [PATCH 06/10] response --- itchiodl/library.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/itchiodl/library.py b/itchiodl/library.py index 452a3bf..e91b290 100644 --- a/itchiodl/library.py +++ b/itchiodl/library.py @@ -39,11 +39,11 @@ class Library: def load_game(self, publisher, title): """Load a game by publisher and title""" - requests.get( + rsp = requests.get( f"https://{publisher}.itch.io/{title}/data.json", headers={"Authorization": self.login}, ) - j = json.loads(r.text) + j = json.loads(rsp.text) self.games.append(Game(j["id"])) def load_games(self, publisher): From c93b5ddb0c43d26b4748f06b73d1b2a0d3b2ae9a Mon Sep 17 00:00:00 2001 From: Peter Taylor Date: Wed, 18 May 2022 13:44:49 +0100 Subject: [PATCH 07/10] #55 merges (dos2Unix) Co-authored-by: Blaze Marshall --- itchiodl/downloader/__main__.py | 2 +- itchiodl/game.py | 28 ++++++++++++++++++++++------ itchiodl/library.py | 16 +++++++++++++--- 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/itchiodl/downloader/__main__.py b/itchiodl/downloader/__main__.py index 15cde04..70e2223 100644 --- a/itchiodl/downloader/__main__.py +++ b/itchiodl/downloader/__main__.py @@ -52,7 +52,7 @@ def main(): lib = itchiodl.Library(l, args.jobs) if args.download_publisher: - lib.load_game(args.download_publisher) + lib.load_games(args.download_publisher) elif args.download_game: matches = re.match(r"https://(.+)\.itch\.io/(.+)", args.download_game) lib.load_game(matches.group(1), matches.group(2)) diff --git a/itchiodl/game.py b/itchiodl/game.py index 314932d..bbf3e9a 100644 --- a/itchiodl/game.py +++ b/itchiodl/game.py @@ -18,9 +18,13 @@ class Game: self.name = self.data["title"] self.publisher = self.data["user"]["username"] self.link = self.data["url"] - - self.id = data["id"] - self.game_id = data["game_id"] + if "game_id" in self.data: + self.id = self.data["id"] + self.game_id = self.data["game_id"] + else: + self.id = False + self.game_id = self.data["id"] + matches = re.match(r"https://(.+)\.itch\.io/(.+)", self.link) self.game_slug = matches.group(2) @@ -32,10 +36,16 @@ class Game: def load_downloads(self, token): """Load all downloads for this game""" self.downloads = [] - r = requests.get( + if self.id: + r = requests.get( f"https://api.itch.io/games/{self.game_id}/uploads?download_key_id={self.id}", headers={"Authorization": token}, ) + else: + r = requests.get( + f"https://api.itch.io/games/{self.game_id}/uploads", + headers={"Authorization": token}, + ) j = r.json() for d in j["uploads"]: self.downloads.append(d) @@ -129,10 +139,16 @@ class Game: j = r.json() # Download - url = ( + if self.id: + url = ( f"https://api.itch.io/uploads/{d['id']}/" + f"download?api_key={token}&download_key_id={self.id}&uuid={j['uuid']}" - ) + ) + else: + url = ( + f"https://api.itch.io/uploads/{d['id']}/" + + f"download?api_key={token}&uuid={j['uuid']}" + ) # response_code = urllib.request.urlopen(url).getcode() try: itchiodl.utils.download(url, path, self.name, file) diff --git a/itchiodl/library.py b/itchiodl/library.py index e91b290..d9ac40c 100644 --- a/itchiodl/library.py +++ b/itchiodl/library.py @@ -44,14 +44,24 @@ class Library: headers={"Authorization": self.login}, ) j = json.loads(rsp.text) - self.games.append(Game(j["id"])) + game_id = j["id"] + gsp = requests.get( + f"https://api.itch.io/games/{game_id}", + headers={"Authorization": self.login}, + ) + self.games.append(Game(json.loads(gsp.text))) def load_games(self, publisher): """Load all games by publisher""" rsp = requests.get(f"https://{publisher}.itch.io") soup = BeautifulSoup(rsp.text, "html.parser") - for link in soup.select("a.game-link"): - self.games.append(Game(link.get("data-label").split(":")[1])) + for link in soup.select("a.game_link"): + game_id = link.get("data-label").split(":")[1] + gsp = requests.get( + f"https://api.itch.io/games/{game_id}", + headers={"Authorization": self.login}, + ) + self.games.append(Game(json.loads(gsp.text))) def download_library(self, platform=None): """Download all games in the library""" From 8b998e6acd0cab0517c85f243fa0fe04e77fe06e Mon Sep 17 00:00:00 2001 From: Peter Taylor Date: Wed, 18 May 2022 13:46:44 +0100 Subject: [PATCH 08/10] fixes by black & pylint --- itchiodl/game.py | 21 ++++++++++----------- itchiodl/library.py | 2 +- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/itchiodl/game.py b/itchiodl/game.py index bbf3e9a..c080354 100644 --- a/itchiodl/game.py +++ b/itchiodl/game.py @@ -24,7 +24,6 @@ class Game: else: self.id = False self.game_id = self.data["id"] - matches = re.match(r"https://(.+)\.itch\.io/(.+)", self.link) self.game_slug = matches.group(2) @@ -38,14 +37,14 @@ class Game: self.downloads = [] if self.id: r = requests.get( - f"https://api.itch.io/games/{self.game_id}/uploads?download_key_id={self.id}", - headers={"Authorization": token}, - ) + f"https://api.itch.io/games/{self.game_id}/uploads?download_key_id={self.id}", + headers={"Authorization": token}, + ) else: r = requests.get( - f"https://api.itch.io/games/{self.game_id}/uploads", - headers={"Authorization": token}, - ) + f"https://api.itch.io/games/{self.game_id}/uploads", + headers={"Authorization": token}, + ) j = r.json() for d in j["uploads"]: self.downloads.append(d) @@ -141,13 +140,13 @@ class Game: # Download if self.id: url = ( - f"https://api.itch.io/uploads/{d['id']}/" - + f"download?api_key={token}&download_key_id={self.id}&uuid={j['uuid']}" + f"https://api.itch.io/uploads/{d['id']}/" + + f"download?api_key={token}&download_key_id={self.id}&uuid={j['uuid']}" ) else: url = ( - f"https://api.itch.io/uploads/{d['id']}/" - + f"download?api_key={token}&uuid={j['uuid']}" + f"https://api.itch.io/uploads/{d['id']}/" + + f"download?api_key={token}&uuid={j['uuid']}" ) # response_code = urllib.request.urlopen(url).getcode() try: diff --git a/itchiodl/library.py b/itchiodl/library.py index d9ac40c..822100d 100644 --- a/itchiodl/library.py +++ b/itchiodl/library.py @@ -48,7 +48,7 @@ class Library: gsp = requests.get( f"https://api.itch.io/games/{game_id}", headers={"Authorization": self.login}, - ) + ) self.games.append(Game(json.loads(gsp.text))) def load_games(self, publisher): From 1b68b596a2c6ab079c04e714478c22ea731831f0 Mon Sep 17 00:00:00 2001 From: Peter Taylor Date: Wed, 18 May 2022 13:53:01 +0100 Subject: [PATCH 09/10] response changes to bring it inline with style --- itchiodl/library.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/itchiodl/library.py b/itchiodl/library.py index 822100d..df0e8f0 100644 --- a/itchiodl/library.py +++ b/itchiodl/library.py @@ -49,7 +49,8 @@ class Library: f"https://api.itch.io/games/{game_id}", headers={"Authorization": self.login}, ) - self.games.append(Game(json.loads(gsp.text))) + k = json.loads(gsp.text) + self.games.append(Game(k)) def load_games(self, publisher): """Load all games by publisher""" @@ -61,7 +62,8 @@ class Library: f"https://api.itch.io/games/{game_id}", headers={"Authorization": self.login}, ) - self.games.append(Game(json.loads(gsp.text))) + k = json.loads(gsp.text) + self.games.append(Game(k)) def download_library(self, platform=None): """Download all games in the library""" From cc2c4b59d548c62cad8ded7ec0ff5276b3952a5c Mon Sep 17 00:00:00 2001 From: Peter Taylor Date: Wed, 18 May 2022 13:55:24 +0100 Subject: [PATCH 10/10] Version Bump --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7b9fb74..606aeac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "itchiodl" -version = "2.0.0" +version = "2.1.0" description = "Python Scripts for downloading / archiving your itchio library" authors = ["Peter Taylor "] license = "MIT"