From e7bbe1983978b97a12921fe66a3b07beff9b35ec Mon Sep 17 00:00:00 2001 From: 64Core Date: Mon, 20 Mar 2023 23:21:11 -0500 Subject: [PATCH] Fixes the errors in filename handling that were shredding unicode characters. I don't remember exactly what the problem is, but I believe it had to do something with how the = operator and string objects don't play perfectly well together when it comes to unicode and need to be handled delicately. This fix is backported from a dead pull request from 2022 that I half remember working on. --- itchiodl/game.py | 53 ++++++++++++++++++++++++----------------------- itchiodl/utils.py | 20 +++++++++--------- 2 files changed, 37 insertions(+), 36 deletions(-) diff --git a/itchiodl/game.py b/itchiodl/game.py index 55b5641..c0643ee 100644 --- a/itchiodl/game.py +++ b/itchiodl/game.py @@ -1,7 +1,9 @@ +import os import re import json import urllib import datetime +from os import path from pathlib import Path import requests from sys import argv @@ -41,13 +43,14 @@ class Game: else: self.publisher_slug = matches.group(1) + self.destination_path = path.normpath(f"{self.publisher_slug}/{self.game_slug}") self.files = [] self.downloads = [] - self.dir = ( - Path(".") - / utils.clean_path(self.publisher_slug) - / utils.clean_path(self.game_slug) - ) + #self.dir = ( + # Path(".") + # / utils.clean_path(self.publisher_slug) + # / utils.clean_path(self.game_slug) + #) def load_downloads(self, token): """Load all downloads for this game""" @@ -76,7 +79,8 @@ class Game: self.load_downloads(token) - self.dir.mkdir(parents=True, exist_ok=True) + if not os.path.exists(self.destination_path): + os.makedirs(self.destination_path) for d in self.downloads: if ( @@ -88,7 +92,7 @@ class Game: continue self.do_download(d, token) - with self.dir.with_suffix(".json").open("w") as f: + with open(f"{self.destination_path}.json", "w") as f: json.dump( { "name": self.name, @@ -106,41 +110,38 @@ class Game: """Download a single file, checking for existing files""" print(f"Downloading {d['filename']}") - filename = d["filename"] or d["display_name"] or d["id"] - - out_file = self.dir / filename - - if out_file.exists(): + filename = utils.clean_path(d["filename"] or d["display_name"] or d["id"]) + pathname = self.destination_path + if path.exists(f"{pathname}/{filename}"): print(f"File Already Exists! {filename}") - md5_file = out_file.with_suffix(".md5") - if md5_file.exists(): - with md5_file.open("r") as f: + if path.exists(f"{pathname}/{filename}.md5"): + with open(f"{pathname}/{filename}.md5", "r") as f: md5 = f.read().strip() if md5 == d["md5_hash"]: print(f"Skipping {self.name} - {filename}") return print(f"MD5 Mismatch! {filename}") else: - md5 = utils.md5sum(str(out_file)) + md5 = utils.md5sum(f"{pathname}/{filename}") if md5 == d["md5_hash"]: print(f"Skipping {self.name} - {filename}") # Create checksum file - with md5_file.open("w") as f: + with open(f"{pathname}/{filename}.md5", "w") as f: f.write(d["md5_hash"]) return # Old Download or corrupted file? corrupted = False if corrupted: - out_file.remove() + filename.remove() return - old_dir = self.dir / "old" - old_dir.mkdir(exist_ok=True) + old_dir = f"{pathname}/old" + os.mkdir(old_dir) print(f"Moving {filename} to old/") timestamp = datetime.datetime.now().strftime("%Y-%m-%d") - out_file.rename(old_dir / f"{timestamp}-{filename}") + filename.rename(old_dir / f"{timestamp}-{filename}") # Get UUID r = requests.post( @@ -162,7 +163,7 @@ class Game: ) # response_code = urllib.request.urlopen(url).getcode() try: - utils.download(url, self.dir, self.name, filename) + utils.download(url, self.destination_path, self.name, filename) except utils.NoDownloadError: print("Http response is not a download, skipping") @@ -170,7 +171,7 @@ class Game: f.write( f""" Cannot download game/asset: {self.game_slug} Publisher Name: {self.publisher_slug} - Path: {out_file} + Path: {pathname} File: {filename} Request URL: {url} This request failed due to a missing response header @@ -186,7 +187,7 @@ class Game: f.write( f""" Cannot download game/asset: {self.game_slug} Publisher Name: {self.publisher_slug} - Path: {out_file} + Path: {pathname} File: {filename} Request URL: {url} Request Response Code: {e.code} @@ -198,10 +199,10 @@ class Game: return # Verify - if utils.md5sum(out_file) != d["md5_hash"]: + if utils.md5sum(f"{pathname}/{filename}") != d["md5_hash"]: print(f"Failed to verify {filename}") return # Create checksum file - with out_file.with_suffix(".md5").open("w") as f: + with open(f"{pathname}/{filename}.md5", "w") as f: f.write(d["md5_hash"]) diff --git a/itchiodl/utils.py b/itchiodl/utils.py index 4a7b78a..722a5e4 100644 --- a/itchiodl/utils.py +++ b/itchiodl/utils.py @@ -23,18 +23,18 @@ def download(url, path, name, file): cd = rsp.headers.get("Content-Disposition") - filename_re = re.search(r'filename="(.+)"', cd) - if filename_re is None: - filename = file - else: - filename = filename_re.group(1) + #filename_re = re.search(r'filename="(.+)"', cd) + #if filename_re is None: + # filename = file + #else: + # filename = filename_re.group(1) - with open(f"{path}/{filename}", "wb") as f: + with open(f"{path}/{file}", "wb") as f: for chunk in rsp.iter_content(10240): f.write(chunk) - print(f"Downloaded {filename}") - return f"{path}/{filename}", True + print(f"Downloaded {file}") + return f"{path}/{file}", True def clean_path(path): @@ -47,10 +47,10 @@ def clean_path(path): return path -def md5sum(path): +def md5sum(pathname): """Returns the md5sum of a file""" md5 = hashlib.md5() - with path.open("rb") as f: + with open(pathname, "rb") as f: for chunk in iter(lambda: f.read(4096), b""): md5.update(chunk) return md5.hexdigest()