From c95b75b085670c572ca077203967b06e8b9b5214 Mon Sep 17 00:00:00 2001 From: 64Core Date: Sat, 18 Mar 2023 21:18:40 -0500 Subject: [PATCH 01/17] Fixes an issue prevents writing paths that end in '...' or similar --- itchiodl/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/itchiodl/utils.py b/itchiodl/utils.py index b972f67..4a7b78a 100644 --- a/itchiodl/utils.py +++ b/itchiodl/utils.py @@ -41,6 +41,8 @@ def clean_path(path): """Cleans a path on windows""" if sys.platform in ["win32", "cygwin", "msys"]: path_clean = re.sub(r"[<>:|?*\"\/\\]", "-", path) + # This checks for strings that end in ... or similar, weird corner case that affects fewer than 0.1% of titles + path_clean = re.sub(r'(.)[.]\1+$', "-", path_clean) return path_clean return path From 2310f0ff66500253865e199d9403eac5cf2bc0b4 Mon Sep 17 00:00:00 2001 From: 64Core Date: Sat, 18 Mar 2023 21:32:04 -0500 Subject: [PATCH 02/17] Add Framework for human readable folder structures --- itchiodl/downloader/__main__.py | 10 ++++++++++ itchiodl/game.py | 17 ++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/itchiodl/downloader/__main__.py b/itchiodl/downloader/__main__.py index 70e2223..f6dd03d 100644 --- a/itchiodl/downloader/__main__.py +++ b/itchiodl/downloader/__main__.py @@ -18,6 +18,16 @@ def main(): help="Platform to download for (default: all), will accept values like 'windows', 'linux', 'osx' and android", ) + parser.add_argument( + "-h", + "--human-folders", + type=bool, + default=False, + const=True, + nargs='?', + help="Download Folders are named based on the full text version of the title instead of the trimmed URL title" + ) + parser.add_argument( "-j", "--jobs", diff --git a/itchiodl/game.py b/itchiodl/game.py index 3937c49..6425d11 100644 --- a/itchiodl/game.py +++ b/itchiodl/game.py @@ -4,6 +4,7 @@ import urllib import datetime from pathlib import Path import requests +from sys import argv from itchiodl import utils @@ -12,6 +13,12 @@ class Game: """Representation of a game download""" def __init__(self, data): + self.args = argv[1:] + if '-h' in self.args or '--human-folders' in self.args: + self.humanFolders = True + else: + self.humanFolders = False + self.data = data["game"] self.name = self.data["title"] self.publisher = self.data["user"]["username"] @@ -25,7 +32,15 @@ class Game: matches = re.match(r"https://(.+)\.itch\.io/(.+)", self.link) self.game_slug = matches.group(2) - self.publisher_slug = matches.group(1) + if self.humanFolders: + self.name = utils.clean_path(self.data["title"]) + self.publisher_slug = self.data.get("user").get("display_name") + # This Branch covers the case that the user has not set a display name, and defaults to their username + if not self.publisher_slug: + self.publisher_slug = self.data.get("user").get("username") + else: + self.name = self.game_slug + self.publisher_slug = matches.group(1) self.files = [] self.downloads = [] From 624eec9b6a06a5026064a6beaeb0917c59d7ed28 Mon Sep 17 00:00:00 2001 From: 64Core Date: Sat, 18 Mar 2023 21:55:15 -0500 Subject: [PATCH 03/17] Removed -h argument due to conflict --- itchiodl/downloader/__main__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/itchiodl/downloader/__main__.py b/itchiodl/downloader/__main__.py index f6dd03d..3830a93 100644 --- a/itchiodl/downloader/__main__.py +++ b/itchiodl/downloader/__main__.py @@ -19,7 +19,6 @@ def main(): ) parser.add_argument( - "-h", "--human-folders", type=bool, default=False, From 4ddff68482456b767979bd234cb4325940e21da1 Mon Sep 17 00:00:00 2001 From: 64Core Date: Sat, 18 Mar 2023 22:02:34 -0500 Subject: [PATCH 04/17] Game Folders / Files now respond to --human-folders --- itchiodl/game.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/itchiodl/game.py b/itchiodl/game.py index 6425d11..01d293d 100644 --- a/itchiodl/game.py +++ b/itchiodl/game.py @@ -33,13 +33,12 @@ class Game: matches = re.match(r"https://(.+)\.itch\.io/(.+)", self.link) self.game_slug = matches.group(2) if self.humanFolders: - self.name = utils.clean_path(self.data["title"]) + self.game_slug = utils.clean_path(self.data["title"]) self.publisher_slug = self.data.get("user").get("display_name") # This Branch covers the case that the user has not set a display name, and defaults to their username if not self.publisher_slug: self.publisher_slug = self.data.get("user").get("username") else: - self.name = self.game_slug self.publisher_slug = matches.group(1) self.files = [] From 767895bfeb82a39f906521ef2a4d1720faffecbb Mon Sep 17 00:00:00 2001 From: 64Core Date: Sat, 18 Mar 2023 22:10:06 -0500 Subject: [PATCH 05/17] Update Readme.md to include new argument --- Readme.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Readme.md b/Readme.md index 5eab20c..18829f4 100644 --- a/Readme.md +++ b/Readme.md @@ -30,6 +30,10 @@ itch-download -k KEYHERE -j 4 # only download osx or cross platform downloads itch-download -p osx + +# folder structure uses display names for users/publishers and game titles +itch-download --human-folders + ``` ## Add All Games in a bundle to your library From ac09641220ff9683ff7f88395460eec99d86e7aa Mon Sep 17 00:00:00 2001 From: 64Core Date: Sun, 19 Mar 2023 16:59:14 -0500 Subject: [PATCH 06/17] Remove -h from arg logic because it's not used and collides with help --- itchiodl/game.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/itchiodl/game.py b/itchiodl/game.py index 01d293d..55b5641 100644 --- a/itchiodl/game.py +++ b/itchiodl/game.py @@ -14,7 +14,7 @@ class Game: def __init__(self, data): self.args = argv[1:] - if '-h' in self.args or '--human-folders' in self.args: + if '--human-folders' in self.args: self.humanFolders = True else: self.humanFolders = False From e7bbe1983978b97a12921fe66a3b07beff9b35ec Mon Sep 17 00:00:00 2001 From: 64Core Date: Mon, 20 Mar 2023 23:21:11 -0500 Subject: [PATCH 07/17] Fixes the errors in filename handling that were shredding unicode characters. I don't remember exactly what the problem is, but I believe it had to do something with how the = operator and string objects don't play perfectly well together when it comes to unicode and need to be handled delicately. This fix is backported from a dead pull request from 2022 that I half remember working on. --- itchiodl/game.py | 53 ++++++++++++++++++++++++----------------------- itchiodl/utils.py | 20 +++++++++--------- 2 files changed, 37 insertions(+), 36 deletions(-) diff --git a/itchiodl/game.py b/itchiodl/game.py index 55b5641..c0643ee 100644 --- a/itchiodl/game.py +++ b/itchiodl/game.py @@ -1,7 +1,9 @@ +import os import re import json import urllib import datetime +from os import path from pathlib import Path import requests from sys import argv @@ -41,13 +43,14 @@ class Game: else: self.publisher_slug = matches.group(1) + self.destination_path = path.normpath(f"{self.publisher_slug}/{self.game_slug}") self.files = [] self.downloads = [] - self.dir = ( - Path(".") - / utils.clean_path(self.publisher_slug) - / utils.clean_path(self.game_slug) - ) + #self.dir = ( + # Path(".") + # / utils.clean_path(self.publisher_slug) + # / utils.clean_path(self.game_slug) + #) def load_downloads(self, token): """Load all downloads for this game""" @@ -76,7 +79,8 @@ class Game: self.load_downloads(token) - self.dir.mkdir(parents=True, exist_ok=True) + if not os.path.exists(self.destination_path): + os.makedirs(self.destination_path) for d in self.downloads: if ( @@ -88,7 +92,7 @@ class Game: continue self.do_download(d, token) - with self.dir.with_suffix(".json").open("w") as f: + with open(f"{self.destination_path}.json", "w") as f: json.dump( { "name": self.name, @@ -106,41 +110,38 @@ class Game: """Download a single file, checking for existing files""" print(f"Downloading {d['filename']}") - filename = d["filename"] or d["display_name"] or d["id"] - - out_file = self.dir / filename - - if out_file.exists(): + filename = utils.clean_path(d["filename"] or d["display_name"] or d["id"]) + pathname = self.destination_path + if path.exists(f"{pathname}/{filename}"): print(f"File Already Exists! {filename}") - md5_file = out_file.with_suffix(".md5") - if md5_file.exists(): - with md5_file.open("r") as f: + if path.exists(f"{pathname}/{filename}.md5"): + with open(f"{pathname}/{filename}.md5", "r") as f: md5 = f.read().strip() if md5 == d["md5_hash"]: print(f"Skipping {self.name} - {filename}") return print(f"MD5 Mismatch! {filename}") else: - md5 = utils.md5sum(str(out_file)) + md5 = utils.md5sum(f"{pathname}/{filename}") if md5 == d["md5_hash"]: print(f"Skipping {self.name} - {filename}") # Create checksum file - with md5_file.open("w") as f: + with open(f"{pathname}/{filename}.md5", "w") as f: f.write(d["md5_hash"]) return # Old Download or corrupted file? corrupted = False if corrupted: - out_file.remove() + filename.remove() return - old_dir = self.dir / "old" - old_dir.mkdir(exist_ok=True) + old_dir = f"{pathname}/old" + os.mkdir(old_dir) print(f"Moving {filename} to old/") timestamp = datetime.datetime.now().strftime("%Y-%m-%d") - out_file.rename(old_dir / f"{timestamp}-{filename}") + filename.rename(old_dir / f"{timestamp}-{filename}") # Get UUID r = requests.post( @@ -162,7 +163,7 @@ class Game: ) # response_code = urllib.request.urlopen(url).getcode() try: - utils.download(url, self.dir, self.name, filename) + utils.download(url, self.destination_path, self.name, filename) except utils.NoDownloadError: print("Http response is not a download, skipping") @@ -170,7 +171,7 @@ class Game: f.write( f""" Cannot download game/asset: {self.game_slug} Publisher Name: {self.publisher_slug} - Path: {out_file} + Path: {pathname} File: {filename} Request URL: {url} This request failed due to a missing response header @@ -186,7 +187,7 @@ class Game: f.write( f""" Cannot download game/asset: {self.game_slug} Publisher Name: {self.publisher_slug} - Path: {out_file} + Path: {pathname} File: {filename} Request URL: {url} Request Response Code: {e.code} @@ -198,10 +199,10 @@ class Game: return # Verify - if utils.md5sum(out_file) != d["md5_hash"]: + if utils.md5sum(f"{pathname}/{filename}") != d["md5_hash"]: print(f"Failed to verify {filename}") return # Create checksum file - with out_file.with_suffix(".md5").open("w") as f: + with open(f"{pathname}/{filename}.md5", "w") as f: f.write(d["md5_hash"]) diff --git a/itchiodl/utils.py b/itchiodl/utils.py index 4a7b78a..722a5e4 100644 --- a/itchiodl/utils.py +++ b/itchiodl/utils.py @@ -23,18 +23,18 @@ def download(url, path, name, file): cd = rsp.headers.get("Content-Disposition") - filename_re = re.search(r'filename="(.+)"', cd) - if filename_re is None: - filename = file - else: - filename = filename_re.group(1) + #filename_re = re.search(r'filename="(.+)"', cd) + #if filename_re is None: + # filename = file + #else: + # filename = filename_re.group(1) - with open(f"{path}/{filename}", "wb") as f: + with open(f"{path}/{file}", "wb") as f: for chunk in rsp.iter_content(10240): f.write(chunk) - print(f"Downloaded {filename}") - return f"{path}/{filename}", True + print(f"Downloaded {file}") + return f"{path}/{file}", True def clean_path(path): @@ -47,10 +47,10 @@ def clean_path(path): return path -def md5sum(path): +def md5sum(pathname): """Returns the md5sum of a file""" md5 = hashlib.md5() - with path.open("rb") as f: + with open(pathname, "rb") as f: for chunk in iter(lambda: f.read(4096), b""): md5.update(chunk) return md5.hexdigest() From 7c7a70d9fe606819698865af9c81fdd4a9ded7b2 Mon Sep 17 00:00:00 2001 From: 64Core Date: Mon, 20 Mar 2023 23:26:19 -0500 Subject: [PATCH 08/17] Remove redundant include --- itchiodl/game.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/itchiodl/game.py b/itchiodl/game.py index c0643ee..6f00590 100644 --- a/itchiodl/game.py +++ b/itchiodl/game.py @@ -1,10 +1,9 @@ -import os import re import json import urllib import datetime from os import path -from pathlib import Path +from os import mkdir import requests from sys import argv @@ -137,7 +136,7 @@ class Game: return old_dir = f"{pathname}/old" - os.mkdir(old_dir) + mkdir(old_dir) print(f"Moving {filename} to old/") timestamp = datetime.datetime.now().strftime("%Y-%m-%d") From 0c1e49d8b2e7ad6f8785f65df29b1e9251314e31 Mon Sep 17 00:00:00 2001 From: 64Core Date: Mon, 20 Mar 2023 23:35:31 -0500 Subject: [PATCH 09/17] Fix includes again --- itchiodl/game.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/itchiodl/game.py b/itchiodl/game.py index 6f00590..aabefe8 100644 --- a/itchiodl/game.py +++ b/itchiodl/game.py @@ -4,6 +4,7 @@ import urllib import datetime from os import path from os import mkdir +from os import makedirs import requests from sys import argv @@ -78,8 +79,8 @@ class Game: self.load_downloads(token) - if not os.path.exists(self.destination_path): - os.makedirs(self.destination_path) + if not path.exists(self.destination_path): + makedirs(self.destination_path) for d in self.downloads: if ( From 1eb2f69693d89f8fc070219b11add7f7e24adbf9 Mon Sep 17 00:00:00 2001 From: 64Core <64core@pm.me> Date: Fri, 24 Mar 2023 18:07:57 -0500 Subject: [PATCH 10/17] Revert "Merge remote-tracking branch 'upstream/main' into bugfix/unicode_support" This reverts commit a61f4c5e9394a9456d734f554b3b76c23adbc333, reversing changes made to 0c1e49d8b2e7ad6f8785f65df29b1e9251314e31. --- itchiodl/downloader/__main__.py | 4 +-- itchiodl/game.py | 62 ++++++++++++++++----------------- itchiodl/utils.py | 25 +++++++------ 3 files changed, 45 insertions(+), 46 deletions(-) diff --git a/itchiodl/downloader/__main__.py b/itchiodl/downloader/__main__.py index 164f9bf..3830a93 100644 --- a/itchiodl/downloader/__main__.py +++ b/itchiodl/downloader/__main__.py @@ -23,8 +23,8 @@ def main(): type=bool, default=False, const=True, - nargs="?", - help="Download Folders are named based on the full text version of the title instead of the trimmed URL title", + nargs='?', + help="Download Folders are named based on the full text version of the title instead of the trimmed URL title" ) parser.add_argument( diff --git a/itchiodl/game.py b/itchiodl/game.py index feaa1d6..aabefe8 100644 --- a/itchiodl/game.py +++ b/itchiodl/game.py @@ -2,9 +2,11 @@ import re import json import urllib import datetime -from pathlib import Path -from sys import argv +from os import path +from os import mkdir +from os import makedirs import requests +from sys import argv from itchiodl import utils @@ -14,7 +16,7 @@ class Game: def __init__(self, data): self.args = argv[1:] - if "--human-folders" in self.args: + if '--human-folders' in self.args: self.humanFolders = True else: self.humanFolders = False @@ -35,20 +37,20 @@ class Game: if self.humanFolders: self.game_slug = utils.clean_path(self.data["title"]) self.publisher_slug = self.data.get("user").get("display_name") - # This Branch covers the case that the user has - # not set a display name, and defaults to their username + # This Branch covers the case that the user has not set a display name, and defaults to their username if not self.publisher_slug: self.publisher_slug = self.data.get("user").get("username") else: self.publisher_slug = matches.group(1) + self.destination_path = path.normpath(f"{self.publisher_slug}/{self.game_slug}") self.files = [] self.downloads = [] - self.dir = ( - Path(".") - / utils.clean_path(self.publisher_slug) - / utils.clean_path(self.game_slug) - ) + #self.dir = ( + # Path(".") + # / utils.clean_path(self.publisher_slug) + # / utils.clean_path(self.game_slug) + #) def load_downloads(self, token): """Load all downloads for this game""" @@ -77,7 +79,8 @@ class Game: self.load_downloads(token) - self.dir.mkdir(parents=True, exist_ok=True) + if not path.exists(self.destination_path): + makedirs(self.destination_path) for d in self.downloads: if ( @@ -89,7 +92,7 @@ class Game: continue self.do_download(d, token) - with self.dir.with_suffix(".json").open("w") as f: + with open(f"{self.destination_path}.json", "w") as f: json.dump( { "name": self.name, @@ -107,41 +110,38 @@ class Game: """Download a single file, checking for existing files""" print(f"Downloading {d['filename']}") - filename = d["filename"] or d["display_name"] or d["id"] - - out_file = self.dir / filename - - if out_file.exists(): + filename = utils.clean_path(d["filename"] or d["display_name"] or d["id"]) + pathname = self.destination_path + if path.exists(f"{pathname}/{filename}"): print(f"File Already Exists! {filename}") - md5_file = out_file.with_suffix(".md5") - if md5_file.exists(): - with md5_file.open("r") as f: + if path.exists(f"{pathname}/{filename}.md5"): + with open(f"{pathname}/{filename}.md5", "r") as f: md5 = f.read().strip() if md5 == d["md5_hash"]: print(f"Skipping {self.name} - {filename}") return print(f"MD5 Mismatch! {filename}") else: - md5 = utils.md5sum(str(out_file)) + md5 = utils.md5sum(f"{pathname}/{filename}") if md5 == d["md5_hash"]: print(f"Skipping {self.name} - {filename}") # Create checksum file - with md5_file.open("w") as f: + with open(f"{pathname}/{filename}.md5", "w") as f: f.write(d["md5_hash"]) return # Old Download or corrupted file? corrupted = False if corrupted: - out_file.remove() + filename.remove() return - old_dir = self.dir / "old" - old_dir.mkdir(exist_ok=True) + old_dir = f"{pathname}/old" + mkdir(old_dir) print(f"Moving {filename} to old/") timestamp = datetime.datetime.now().strftime("%Y-%m-%d") - out_file.rename(old_dir / f"{timestamp}-{filename}") + filename.rename(old_dir / f"{timestamp}-{filename}") # Get UUID r = requests.post( @@ -163,7 +163,7 @@ class Game: ) # response_code = urllib.request.urlopen(url).getcode() try: - utils.download(url, self.dir, self.name, filename) + utils.download(url, self.destination_path, self.name, filename) except utils.NoDownloadError: print("Http response is not a download, skipping") @@ -171,7 +171,7 @@ class Game: f.write( f""" Cannot download game/asset: {self.game_slug} Publisher Name: {self.publisher_slug} - Path: {out_file} + Path: {pathname} File: {filename} Request URL: {url} This request failed due to a missing response header @@ -187,7 +187,7 @@ class Game: f.write( f""" Cannot download game/asset: {self.game_slug} Publisher Name: {self.publisher_slug} - Path: {out_file} + Path: {pathname} File: {filename} Request URL: {url} Request Response Code: {e.code} @@ -199,10 +199,10 @@ class Game: return # Verify - if utils.md5sum(out_file) != d["md5_hash"]: + if utils.md5sum(f"{pathname}/{filename}") != d["md5_hash"]: print(f"Failed to verify {filename}") return # Create checksum file - with out_file.with_suffix(".md5").open("w") as f: + with open(f"{pathname}/{filename}.md5", "w") as f: f.write(d["md5_hash"]) diff --git a/itchiodl/utils.py b/itchiodl/utils.py index b5ee124..722a5e4 100644 --- a/itchiodl/utils.py +++ b/itchiodl/utils.py @@ -23,35 +23,34 @@ def download(url, path, name, file): cd = rsp.headers.get("Content-Disposition") - filename_re = re.search(r'filename="(.+)"', cd) - if filename_re is None: - filename = file - else: - filename = filename_re.group(1) + #filename_re = re.search(r'filename="(.+)"', cd) + #if filename_re is None: + # filename = file + #else: + # filename = filename_re.group(1) - with open(f"{path}/{filename}", "wb") as f: + with open(f"{path}/{file}", "wb") as f: for chunk in rsp.iter_content(10240): f.write(chunk) - print(f"Downloaded {filename}") - return f"{path}/{filename}", True + print(f"Downloaded {file}") + return f"{path}/{file}", True def clean_path(path): """Cleans a path on windows""" if sys.platform in ["win32", "cygwin", "msys"]: path_clean = re.sub(r"[<>:|?*\"\/\\]", "-", path) - # This checks for strings that end in ... or similar, - # weird corner case that affects fewer than 0.1% of titles - path_clean = re.sub(r"(.)[.]\1+$", "-", path_clean) + # This checks for strings that end in ... or similar, weird corner case that affects fewer than 0.1% of titles + path_clean = re.sub(r'(.)[.]\1+$', "-", path_clean) return path_clean return path -def md5sum(path): +def md5sum(pathname): """Returns the md5sum of a file""" md5 = hashlib.md5() - with path.open("rb") as f: + with open(pathname, "rb") as f: for chunk in iter(lambda: f.read(4096), b""): md5.update(chunk) return md5.hexdigest() From e7c28a8ca9199de772f435ef03c1e9da1bd3e3ce Mon Sep 17 00:00:00 2001 From: 64Core <64core@pm.me> Date: Fri, 24 Mar 2023 18:21:18 -0500 Subject: [PATCH 11/17] Fix linting conflicts --- itchiodl/downloader/__main__.py | 4 ++-- itchiodl/game.py | 7 ++++--- itchiodl/utils.py | 3 ++- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/itchiodl/downloader/__main__.py b/itchiodl/downloader/__main__.py index 3830a93..164f9bf 100644 --- a/itchiodl/downloader/__main__.py +++ b/itchiodl/downloader/__main__.py @@ -23,8 +23,8 @@ def main(): type=bool, default=False, const=True, - nargs='?', - help="Download Folders are named based on the full text version of the title instead of the trimmed URL title" + nargs="?", + help="Download Folders are named based on the full text version of the title instead of the trimmed URL title", ) parser.add_argument( diff --git a/itchiodl/game.py b/itchiodl/game.py index aabefe8..24daf9f 100644 --- a/itchiodl/game.py +++ b/itchiodl/game.py @@ -5,8 +5,8 @@ import datetime from os import path from os import mkdir from os import makedirs -import requests from sys import argv +import requests from itchiodl import utils @@ -16,7 +16,7 @@ class Game: def __init__(self, data): self.args = argv[1:] - if '--human-folders' in self.args: + if "--human-folders" in self.args: self.humanFolders = True else: self.humanFolders = False @@ -37,7 +37,8 @@ class Game: if self.humanFolders: self.game_slug = utils.clean_path(self.data["title"]) self.publisher_slug = self.data.get("user").get("display_name") - # This Branch covers the case that the user has not set a display name, and defaults to their username + # This Branch covers the case that the user has + # not set a display name, and defaults to their username if not self.publisher_slug: self.publisher_slug = self.data.get("user").get("username") else: diff --git a/itchiodl/utils.py b/itchiodl/utils.py index 722a5e4..8296fea 100644 --- a/itchiodl/utils.py +++ b/itchiodl/utils.py @@ -41,7 +41,8 @@ def clean_path(path): """Cleans a path on windows""" if sys.platform in ["win32", "cygwin", "msys"]: path_clean = re.sub(r"[<>:|?*\"\/\\]", "-", path) - # This checks for strings that end in ... or similar, weird corner case that affects fewer than 0.1% of titles + # This checks for strings that end in ... or similar, + # weird corner case that affects fewer than 0.1% of titles path_clean = re.sub(r'(.)[.]\1+$', "-", path_clean) return path_clean return path From c0a69ad2a8b6908d8a156778c1dc532088c38d2f Mon Sep 17 00:00:00 2001 From: 64Core <64core@pm.me> Date: Fri, 24 Mar 2023 18:33:35 -0500 Subject: [PATCH 12/17] Fix linting conflicts --- itchiodl/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/itchiodl/utils.py b/itchiodl/utils.py index 8296fea..1a20de8 100644 --- a/itchiodl/utils.py +++ b/itchiodl/utils.py @@ -43,7 +43,7 @@ def clean_path(path): path_clean = re.sub(r"[<>:|?*\"\/\\]", "-", path) # This checks for strings that end in ... or similar, # weird corner case that affects fewer than 0.1% of titles - path_clean = re.sub(r'(.)[.]\1+$', "-", path_clean) + path_clean = re.sub(r"(.)[.]\1+$", "-", path_clean) return path_clean return path From dbc42dd4b2c44e2774e8d1edaaf6ecb558db2215 Mon Sep 17 00:00:00 2001 From: 64Core <64core@pm.me> Date: Wed, 29 Mar 2023 22:38:22 -0500 Subject: [PATCH 13/17] PR #77 Convert path handling back to using pathlib --- itchiodl/game.py | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/itchiodl/game.py b/itchiodl/game.py index 24daf9f..276f071 100644 --- a/itchiodl/game.py +++ b/itchiodl/game.py @@ -2,10 +2,8 @@ import re import json import urllib import datetime -from os import path -from os import mkdir -from os import makedirs from sys import argv +from pathlib import Path import requests from itchiodl import utils @@ -44,14 +42,14 @@ class Game: else: self.publisher_slug = matches.group(1) - self.destination_path = path.normpath(f"{self.publisher_slug}/{self.game_slug}") + self.destination_path = Path(f"{self.publisher_slug}/{self.game_slug}") self.files = [] self.downloads = [] - #self.dir = ( + # self.dir = ( # Path(".") # / utils.clean_path(self.publisher_slug) # / utils.clean_path(self.game_slug) - #) + # ) def load_downloads(self, token): """Load all downloads for this game""" @@ -80,8 +78,7 @@ class Game: self.load_downloads(token) - if not path.exists(self.destination_path): - makedirs(self.destination_path) + self.destination_path.mkdir(parents=True, exist_ok=True) for d in self.downloads: if ( @@ -113,23 +110,26 @@ class Game: filename = utils.clean_path(d["filename"] or d["display_name"] or d["id"]) pathname = self.destination_path - if path.exists(f"{pathname}/{filename}"): + filepath = Path(f"{pathname}/{filename}") + hashpath = Path(f"{pathname}/{filename}.md5") + oldpath = Path(f"{pathname}/old") + if filepath.exists(): print(f"File Already Exists! {filename}") - if path.exists(f"{pathname}/{filename}.md5"): - with open(f"{pathname}/{filename}.md5", "r") as f: - md5 = f.read().strip() + if hashpath.exists(): + with hashpath.open(mode="r") as hashfile: + md5 = hashfile.read().strip() if md5 == d["md5_hash"]: print(f"Skipping {self.name} - {filename}") return print(f"MD5 Mismatch! {filename}") else: - md5 = utils.md5sum(f"{pathname}/{filename}") + md5 = utils.md5sum(filepath) if md5 == d["md5_hash"]: print(f"Skipping {self.name} - {filename}") # Create checksum file - with open(f"{pathname}/{filename}.md5", "w") as f: - f.write(d["md5_hash"]) + with hashpath.open(mode="w") as hashfile: + hashfile.write(d["md5_hash"]) return # Old Download or corrupted file? corrupted = False @@ -137,12 +137,11 @@ class Game: filename.remove() return - old_dir = f"{pathname}/old" - mkdir(old_dir) + oldpath.mkdir() print(f"Moving {filename} to old/") timestamp = datetime.datetime.now().strftime("%Y-%m-%d") - filename.rename(old_dir / f"{timestamp}-{filename}") + filename.rename(oldpath / f"{timestamp}-{filename}") # Get UUID r = requests.post( @@ -200,10 +199,10 @@ class Game: return # Verify - if utils.md5sum(f"{pathname}/{filename}") != d["md5_hash"]: + if utils.md5sum(filepath) != d["md5_hash"]: print(f"Failed to verify {filename}") return # Create checksum file - with open(f"{pathname}/{filename}.md5", "w") as f: - f.write(d["md5_hash"]) + with hashpath.open(mode="w") as hashfile: + hashfile.write(d["md5_hash"]) From 85d705901ca04c1d572b014994cc4ba80206fd8e Mon Sep 17 00:00:00 2001 From: 64Core <64core@pm.me> Date: Wed, 29 Mar 2023 22:42:10 -0500 Subject: [PATCH 14/17] PR #77 Linting Fixes --- itchiodl/game.py | 11 +++++------ itchiodl/utils.py | 8 ++++---- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/itchiodl/game.py b/itchiodl/game.py index 276f071..26d5e5e 100644 --- a/itchiodl/game.py +++ b/itchiodl/game.py @@ -109,10 +109,9 @@ class Game: print(f"Downloading {d['filename']}") filename = utils.clean_path(d["filename"] or d["display_name"] or d["id"]) - pathname = self.destination_path - filepath = Path(f"{pathname}/{filename}") - hashpath = Path(f"{pathname}/{filename}.md5") - oldpath = Path(f"{pathname}/old") + filepath = Path(f"{self.destination_path}/{filename}") + hashpath = Path(f"{self.destination_path}/{filename}.md5") + oldpath = Path(f"{self.destination_path}/old") if filepath.exists(): print(f"File Already Exists! {filename}") if hashpath.exists(): @@ -171,7 +170,7 @@ class Game: f.write( f""" Cannot download game/asset: {self.game_slug} Publisher Name: {self.publisher_slug} - Path: {pathname} + Path: {self.destination_path} File: {filename} Request URL: {url} This request failed due to a missing response header @@ -187,7 +186,7 @@ class Game: f.write( f""" Cannot download game/asset: {self.game_slug} Publisher Name: {self.publisher_slug} - Path: {pathname} + Path: {self.destination_path} File: {filename} Request URL: {url} Request Response Code: {e.code} diff --git a/itchiodl/utils.py b/itchiodl/utils.py index 1a20de8..b03e0ab 100644 --- a/itchiodl/utils.py +++ b/itchiodl/utils.py @@ -21,12 +21,12 @@ def download(url, path, name, file): ): raise NoDownloadError("Http response is not a download, skipping") - cd = rsp.headers.get("Content-Disposition") + # cd = rsp.headers.get("Content-Disposition") - #filename_re = re.search(r'filename="(.+)"', cd) - #if filename_re is None: + # filename_re = re.search(r'filename="(.+)"', cd) + # if filename_re is None: # filename = file - #else: + # else: # filename = filename_re.group(1) with open(f"{path}/{file}", "wb") as f: From cd5f7bb6b2de3be51a819a8ef3c4388d744ae702 Mon Sep 17 00:00:00 2001 From: 64Core <64core@pm.me> Date: Wed, 29 Mar 2023 22:45:20 -0500 Subject: [PATCH 15/17] PR #77 More Linting Fixes, Comment out dead branch that would never be hit. --- itchiodl/game.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/itchiodl/game.py b/itchiodl/game.py index 26d5e5e..c37a561 100644 --- a/itchiodl/game.py +++ b/itchiodl/game.py @@ -131,10 +131,10 @@ class Game: hashfile.write(d["md5_hash"]) return # Old Download or corrupted file? - corrupted = False - if corrupted: - filename.remove() - return + # corrupted = False + # if corrupted: + # filename.remove() + # return oldpath.mkdir() From 6de39b63f50454917d28a91fff9e242a7944ae18 Mon Sep 17 00:00:00 2001 From: 64Core <64core@pm.me> Date: Wed, 29 Mar 2023 22:59:22 -0500 Subject: [PATCH 16/17] PR #77 Tidying Pathlib code --- itchiodl/game.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/itchiodl/game.py b/itchiodl/game.py index c37a561..5ec34f1 100644 --- a/itchiodl/game.py +++ b/itchiodl/game.py @@ -90,7 +90,7 @@ class Game: continue self.do_download(d, token) - with open(f"{self.destination_path}.json", "w") as f: + with self.destination_path.with_suffix(".json").open(mode="w") as f: json.dump( { "name": self.name, @@ -110,8 +110,8 @@ class Game: filename = utils.clean_path(d["filename"] or d["display_name"] or d["id"]) filepath = Path(f"{self.destination_path}/{filename}") - hashpath = Path(f"{self.destination_path}/{filename}.md5") - oldpath = Path(f"{self.destination_path}/old") + hashpath = filepath.with_suffix(".md5") + if filepath.exists(): print(f"File Already Exists! {filename}") if hashpath.exists(): @@ -135,12 +135,12 @@ class Game: # if corrupted: # filename.remove() # return - - oldpath.mkdir() + old_dir = self.destination_path / "old" + old_dir.mkdir(exist_ok=True) print(f"Moving {filename} to old/") timestamp = datetime.datetime.now().strftime("%Y-%m-%d") - filename.rename(oldpath / f"{timestamp}-{filename}") + filename.rename(old_dir / f"{timestamp}-{filename}") # Get UUID r = requests.post( From a5a09d10afc6511502c80c2bf0a6fa397d3c5b70 Mon Sep 17 00:00:00 2001 From: 64Core <64core@pm.me> Date: Fri, 14 Apr 2023 22:39:05 -0500 Subject: [PATCH 17/17] PR #77 address second comment, potential bug regarding renaming old files --- itchiodl/game.py | 16 ++++++++-------- itchiodl/utils.py | 14 +++++++------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/itchiodl/game.py b/itchiodl/game.py index 5ec34f1..52350e8 100644 --- a/itchiodl/game.py +++ b/itchiodl/game.py @@ -42,14 +42,14 @@ class Game: else: self.publisher_slug = matches.group(1) - self.destination_path = Path(f"{self.publisher_slug}/{self.game_slug}") + # self.destination_path = Path(self.publisher_slug / self.game_slug) self.files = [] self.downloads = [] - # self.dir = ( - # Path(".") - # / utils.clean_path(self.publisher_slug) - # / utils.clean_path(self.game_slug) - # ) + self.destination_path = ( + Path(".") + / utils.clean_path(self.publisher_slug) + / utils.clean_path(self.game_slug) + ) def load_downloads(self, token): """Load all downloads for this game""" @@ -109,7 +109,7 @@ class Game: print(f"Downloading {d['filename']}") filename = utils.clean_path(d["filename"] or d["display_name"] or d["id"]) - filepath = Path(f"{self.destination_path}/{filename}") + filepath = self.destination_path / filename hashpath = filepath.with_suffix(".md5") if filepath.exists(): @@ -140,7 +140,7 @@ class Game: print(f"Moving {filename} to old/") timestamp = datetime.datetime.now().strftime("%Y-%m-%d") - filename.rename(old_dir / f"{timestamp}-{filename}") + filepath.rename(old_dir / f"{timestamp}-{filename}") # Get UUID r = requests.post( diff --git a/itchiodl/utils.py b/itchiodl/utils.py index b03e0ab..00a5efa 100644 --- a/itchiodl/utils.py +++ b/itchiodl/utils.py @@ -8,10 +8,10 @@ class NoDownloadError(Exception): """No download found exception""" -def download(url, path, name, file): +def download(url, pathname, name, filename): """Downloads a file from a url and saves it to a path, skips it if it already exists.""" - desc = f"{name} - {file}" + desc = f"{name} - {filename}" print(f"Downloading {desc}") rsp = requests.get(url, stream=True) @@ -29,12 +29,12 @@ def download(url, path, name, file): # else: # filename = filename_re.group(1) - with open(f"{path}/{file}", "wb") as f: + with open(f"{pathname}/{filename}", "wb") as f: for chunk in rsp.iter_content(10240): f.write(chunk) - print(f"Downloaded {file}") - return f"{path}/{file}", True + print(f"Downloaded {filename}") + return f"{pathname}/{filename}", True def clean_path(path): @@ -48,10 +48,10 @@ def clean_path(path): return path -def md5sum(pathname): +def md5sum(path): """Returns the md5sum of a file""" md5 = hashlib.md5() - with open(pathname, "rb") as f: + with path.open("rb") as f: for chunk in iter(lambda: f.read(4096), b""): md5.update(chunk) return md5.hexdigest()