From 156b92f8ca3aafd45b5ec07ed005b5413558fd75 Mon Sep 17 00:00:00 2001 From: Mathew Scouten Date: Wed, 15 Mar 2023 21:17:23 -0500 Subject: [PATCH] * Fix handing of windows paths with illegal characters. * Fix issues with inconsistent path handing. * Fix issues with swallow errors. * Fix issues with path cleaning flattening directory structure. --- itchiodl/game.py | 52 ++++++++++++++++++++++----------------------- itchiodl/library.py | 18 ++++++++++++---- itchiodl/login.py | 18 +++++++++------- itchiodl/utils.py | 11 +++++++--- 4 files changed, 57 insertions(+), 42 deletions(-) diff --git a/itchiodl/game.py b/itchiodl/game.py index 1f904b2..c9ea2ab 100644 --- a/itchiodl/game.py +++ b/itchiodl/game.py @@ -1,3 +1,4 @@ +import copy import re import json import os @@ -7,7 +8,7 @@ import shutil import requests -import itchiodl.utils +from itchiodl.utils import clean_path, md5sum, download, NoDownloadError class Game: @@ -31,6 +32,8 @@ class Game: self.files = [] self.downloads = [] + self.path = os.path.abspath( + f"{clean_path(self.publisher_slug)}/{clean_path(self.game_slug)}") def load_downloads(self, token): """Load all downloads for this game""" @@ -45,9 +48,9 @@ class Game: f"https://api.itch.io/games/{self.game_id}/uploads", headers={"Authorization": token}, ) + r.raise_for_status() j = r.json() - for d in j["uploads"]: - self.downloads.append(d) + self.downloads = copy.copy(j["uploads"]) def download(self, token, platform): """Download a singular file""" @@ -59,11 +62,7 @@ class Game: self.load_downloads(token) - if not os.path.exists(self.publisher_slug): - os.mkdir(self.publisher_slug) - - if not os.path.exists(f"{self.publisher_slug}/{self.game_slug}"): - os.mkdir(f"{self.publisher_slug}/{self.game_slug}") + os.makedirs(self.path, exist_ok=True) for d in self.downloads: if ( @@ -75,7 +74,7 @@ class Game: continue self.do_download(d, token) - with open(f"{self.publisher_slug}/{self.game_slug}.json", "w") as f: + with open("{self.path}.json", "w") as f: json.dump( { "name": self.name, @@ -93,14 +92,13 @@ class Game: """Download a single file, checking for existing files""" print(f"Downloading {d['filename']}") - file = itchiodl.utils.clean_path(d["filename"] or d["display_name"] or d["id"]) - path = itchiodl.utils.clean_path(f"{self.publisher_slug}/{self.game_slug}") + file = clean_path(d["filename"] or d["display_name"] or d["id"]) - if os.path.exists(f"{path}/{file}"): + if os.path.exists(f"{self.path}/{file}"): print(f"File Already Exists! {file}") - if os.path.exists(f"{path}/{file}.md5"): + if os.path.exists(f"{self.path}/{file}.md5"): - with open(f"{path}/{file}.md5", "r") as f: + with open(f"{self.path}/{file}.md5", "r") as f: md5 = f.read().strip() if md5 == d["md5_hash"]: @@ -108,27 +106,27 @@ class Game: return print(f"MD5 Mismatch! {file}") else: - md5 = itchiodl.utils.md5sum(f"{path}/{file}") + md5 = md5sum(f"{self.path}/{file}") if md5 == d["md5_hash"]: print(f"Skipping {self.name} - {file}") # Create checksum file - with open(f"{path}/{file}.md5", "w") as f: + with open(f"{self.path}/{file}.md5", "w") as f: f.write(d["md5_hash"]) return # Old Download or corrupted file? corrupted = False if corrupted: - os.remove(f"{path}/{file}") + os.remove(f"{self.path}/{file}") return - if not os.path.exists(f"{path}/old"): - os.mkdir(f"{path}/old") + if not os.path.exists(f"{self.path}/old"): + os.mkdir(f"{self.path}/old") print(f"Moving {file} to old/") timestamp = datetime.datetime.now().strftime("%Y-%m-%d") print(timestamp) - shutil.move(f"{path}/{file}", f"{path}/old/{timestamp}-{file}") + shutil.move(f"{self.path}/{file}", f"{self.path}/old/{timestamp}-{file}") # Get UUID r = requests.post( @@ -150,15 +148,15 @@ class Game: ) # response_code = urllib.request.urlopen(url).getcode() try: - itchiodl.utils.download(url, path, self.name, file) - except itchiodl.utils.NoDownloadError: + actual_destination = download(url, self.path, self.name, file) + except NoDownloadError: print("Http response is not a download, skipping") with open("errors.txt", "a") as f: f.write( f""" Cannot download game/asset: {self.game_slug} Publisher Name: {self.publisher_slug} - Path: {path} + Path: {self.path} File: {file} Request URL: {url} This request failed due to a missing response header @@ -174,7 +172,7 @@ class Game: f.write( f""" Cannot download game/asset: {self.game_slug} Publisher Name: {self.publisher_slug} - Path: {path} + Path: {self.path} File: {file} Request URL: {url} Request Response Code: {e.code} @@ -186,10 +184,10 @@ class Game: return # Verify - if itchiodl.utils.md5sum(f"{path}/{file}") != d["md5_hash"]: - print(f"Failed to verify {file}") + if md5sum(actual_destination) != d["md5_hash"]: + print(f"Failed to verify {file}{d}") return # Create checksum file - with open(f"{path}/{file}.md5", "w") as f: + with open(f"{actual_destination}.md5", "w") as f: f.write(d["md5_hash"]) diff --git a/itchiodl/library.py b/itchiodl/library.py index eeb08ad..4c5c9ae 100644 --- a/itchiodl/library.py +++ b/itchiodl/library.py @@ -1,4 +1,5 @@ import json +import traceback from concurrent.futures import ThreadPoolExecutor import functools import threading @@ -94,10 +95,19 @@ class Library: lock = threading.RLock() def dl(i, g): - x = g.download(self.login, platform) - with lock: - i[0] += 1 + try: + x = g.download(self.login, platform) + with lock: + i[0] += 1 + except Exception as e: + s = f"\nUnable to get '{g.name}' because of unexpected error: \n'{traceback.format_exc()}'\n\n" + with open("errors.txt", "a", encoding='utf-8') as f: + f.write(s) + print(s) + return + print(f"Downloaded {g.name} ({i[0]} of {l})") return x - executor.map(functools.partial(dl, i), self.games) + for _ in executor.map(functools.partial(dl, i), self.games): + pass # exhaust iterator to re-throw exceptions. diff --git a/itchiodl/login.py b/itchiodl/login.py index 0c05854..b8d8ed3 100644 --- a/itchiodl/login.py +++ b/itchiodl/login.py @@ -32,15 +32,17 @@ def LoginWeb(user, password): def LoginAPI(user, password): """Login to itch.io using API""" - r = requests.post( - "https://api.itch.io/login", - {"username": user, "password": password, "source": "desktop"}, - ) - if r.status_code != 200: - print(f"Error: {r.status_code} is not 200") + try: + r = requests.post( + "https://api.itch.io/login", + {"username": user, "password": password, "source": "desktop"}, + ) + r.raise_for_status() + except requests.HTTPError as e: + print(f"Error: {e.response.status_code} is not 200") print(warning) - print(r.text) - raise RuntimeError + print(e.response.text) + raise t = json.loads(r.text) if not t["success"]: diff --git a/itchiodl/utils.py b/itchiodl/utils.py index 78a289f..1e1c82c 100644 --- a/itchiodl/utils.py +++ b/itchiodl/utils.py @@ -13,7 +13,12 @@ def download(url, path, name, file): desc = f"{name} - {file}" print(f"Downloading {desc}") - rsp = requests.get(url, stream=True) + + try: + rsp = requests.get(url, stream=True) + rsp.raise_for_status() + except requests.HTTPError as e: + raise NoDownloadError("Http response is not a download, skipping") from e if ( rsp.headers.get("content-length") is None @@ -34,13 +39,13 @@ def download(url, path, name, file): f.write(chunk) print(f"Downloaded {filename}") - return f"{path}/{filename}", True + return f"{path}/{filename}" def clean_path(path): """Cleans a path on windows""" if sys.platform in ["win32", "cygwin", "msys"]: - path_clean = re.replace(r"[\<\>\:\"\/\\\|\?\*]", "-", path) + path_clean = re.sub(r"[<>:\"/\\|?*]", "-", path) return path_clean return path