prehistoric-systems/itchiodl
1
0
Fork 0
mirror of https://github.com/Emersont1/itchio.git synced 2026-03-11 08:54:39 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fixes the errors in filename handling that were shredding unicode characters.

Browse source 
I don't remember exactly what the problem is, but I believe it had to do something with how the = operator and string objects don't play perfectly well together when it comes to unicode and need to be handled delicately. This fix is backported from a dead pull request from 2022 that I half remember working on.
This commit is contained in:
64Core 2023-03-20 23:21:11 -05:00
parent ac09641220
commit e7bbe19839
2 changed files with 37 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

53
itchiodl/game.py
View file

@ -1,7 +1,9 @@
import os
import re
import json
import urllib
import datetime
from os import path
from pathlib import Path
import requests
from sys import argv
@ -41,13 +43,14 @@ class Game:
else:
self.publisher_slug = matches.group(1)
self.destination_path = path.normpath(f"{self.publisher_slug}/{self.game_slug}")
self.files = []
self.downloads = []
self.dir = (
Path(".")
/ utils.clean_path(self.publisher_slug)
/ utils.clean_path(self.game_slug)
)
#self.dir = (
# Path(".")
# / utils.clean_path(self.publisher_slug)
# / utils.clean_path(self.game_slug)
#)
def load_downloads(self, token):
"""Load all downloads for this game"""
@ -76,7 +79,8 @@ class Game:
self.load_downloads(token)
self.dir.mkdir(parents=True, exist_ok=True)
if not os.path.exists(self.destination_path):
os.makedirs(self.destination_path)
for d in self.downloads:
if (
@ -88,7 +92,7 @@ class Game:
continue
self.do_download(d, token)
with self.dir.with_suffix(".json").open("w") as f:
with open(f"{self.destination_path}.json", "w") as f:
json.dump(
{
"name": self.name,
@ -106,41 +110,38 @@ class Game:
"""Download a single file, checking for existing files"""
print(f"Downloading {d['filename']}")
filename = d["filename"] or d["display_name"] or d["id"]
out_file = self.dir / filename
if out_file.exists():
filename = utils.clean_path(d["filename"] or d["display_name"] or d["id"])
pathname = self.destination_path
if path.exists(f"{pathname}/{filename}"):
print(f"File Already Exists! {filename}")
md5_file = out_file.with_suffix(".md5")
if md5_file.exists():
with md5_file.open("r") as f:
if path.exists(f"{pathname}/{filename}.md5"):
with open(f"{pathname}/{filename}.md5", "r") as f:
md5 = f.read().strip()
if md5 == d["md5_hash"]:
print(f"Skipping {self.name} - {filename}")
return
print(f"MD5 Mismatch! {filename}")
else:
md5 = utils.md5sum(str(out_file))
md5 = utils.md5sum(f"{pathname}/{filename}")
if md5 == d["md5_hash"]:
print(f"Skipping {self.name} - {filename}")
# Create checksum file
with md5_file.open("w") as f:
with open(f"{pathname}/{filename}.md5", "w") as f:
f.write(d["md5_hash"])
return
# Old Download or corrupted file?
corrupted = False
if corrupted:
out_file.remove()
filename.remove()
return
old_dir = self.dir / "old"
old_dir.mkdir(exist_ok=True)
old_dir = f"{pathname}/old"
os.mkdir(old_dir)
print(f"Moving {filename} to old/")
timestamp = datetime.datetime.now().strftime("%Y-%m-%d")
out_file.rename(old_dir / f"{timestamp}-{filename}")
filename.rename(old_dir / f"{timestamp}-{filename}")
# Get UUID
r = requests.post(
@ -162,7 +163,7 @@ class Game:
)
# response_code = urllib.request.urlopen(url).getcode()
try:
utils.download(url, self.dir, self.name, filename)
utils.download(url, self.destination_path, self.name, filename)
except utils.NoDownloadError:
print("Http response is not a download, skipping")
@ -170,7 +171,7 @@ class Game:
f.write(
f""" Cannot download game/asset: {self.game_slug}
Publisher Name: {self.publisher_slug}
Path: {out_file}
Path: {pathname}
File: {filename}
Request URL: {url}
This request failed due to a missing response header
@ -186,7 +187,7 @@ class Game:
f.write(
f""" Cannot download game/asset: {self.game_slug}
Publisher Name: {self.publisher_slug}
Path: {out_file}
Path: {pathname}
File: {filename}
Request URL: {url}
Request Response Code: {e.code}
@ -198,10 +199,10 @@ class Game:
return
# Verify
if utils.md5sum(out_file) != d["md5_hash"]:
if utils.md5sum(f"{pathname}/{filename}") != d["md5_hash"]:
print(f"Failed to verify {filename}")
return
# Create checksum file
with out_file.with_suffix(".md5").open("w") as f:
with open(f"{pathname}/{filename}.md5", "w") as f:
f.write(d["md5_hash"])

20
itchiodl/utils.py
View file

@ -23,18 +23,18 @@ def download(url, path, name, file):
cd = rsp.headers.get("Content-Disposition")
filename_re = re.search(r'filename="(.+)"', cd)
if filename_re is None:
filename = file
else:
filename = filename_re.group(1)
#filename_re = re.search(r'filename="(.+)"', cd)
#if filename_re is None:
# filename = file
#else:
# filename = filename_re.group(1)
with open(f"{path}/{filename}", "wb") as f:
with open(f"{path}/{file}", "wb") as f:
for chunk in rsp.iter_content(10240):
f.write(chunk)
print(f"Downloaded {filename}")
return f"{path}/{filename}", True
print(f"Downloaded {file}")
return f"{path}/{file}", True
def clean_path(path):
@ -47,10 +47,10 @@ def clean_path(path):
return path
def md5sum(path):
def md5sum(pathname):
"""Returns the md5sum of a file"""
md5 = hashlib.md5()
with path.open("rb") as f:
with open(pathname, "rb") as f:
for chunk in iter(lambda: f.read(4096), b""):
md5.update(chunk)
return md5.hexdigest()