From 0f645f1923db498d214bb061ef08add1b0499837 Mon Sep 17 00:00:00 2001 From: Peter Taylor Date: Sat, 18 Dec 2021 02:48:27 +0000 Subject: [PATCH] First Commit, added everything for a version 1.0 --- .gitignore | 5 + Readme.md | 21 +++++ poetry.lock | 147 ++++++++++++++++++++++++++++++ pyproject.toml | 18 ++++ src/itchio/__init__.py | 4 + src/itchio/bundle.py | 39 ++++++++ src/itchio/bundle/__main__.py | 11 +++ src/itchio/downloader/__main__.py | 26 ++++++ src/itchio/game.py | 58 ++++++++++++ src/itchio/library.py | 32 +++++++ src/itchio/login.py | 32 +++++++ src/itchio/utils.py | 16 ++++ 12 files changed, 409 insertions(+) create mode 100644 .gitignore create mode 100644 Readme.md create mode 100644 poetry.lock create mode 100644 pyproject.toml create mode 100644 src/itchio/__init__.py create mode 100644 src/itchio/bundle.py create mode 100644 src/itchio/bundle/__main__.py create mode 100644 src/itchio/downloader/__main__.py create mode 100644 src/itchio/game.py create mode 100644 src/itchio/library.py create mode 100644 src/itchio/login.py create mode 100644 src/itchio/utils.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bfc9c93 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +venv +__pycache__ + +src/* +!src/itchio \ No newline at end of file diff --git a/Readme.md b/Readme.md new file mode 100644 index 0000000..d97fd1e --- /dev/null +++ b/Readme.md @@ -0,0 +1,21 @@ +# Itchio Downloader Tool + +## Download All Games in library from account + +```bash +python -m itchio.downloader +``` + +This uses the same API the itchio app uses to download the files. If you have 2FA enabled, generate an API key [here](https://itch.io/user/settings/api-keys) and run the following instead + +```bash +python -m itchio.downloader --api-key=KEYHERE +``` + +## Add All Games in a bundle to your library + +```bash +python -m itchio.downloader +``` + +This is a bit of a bodge, but it works. It essentially goes through and clicks the "Download" link on every item on the bundle's page, which adds it to your itchio library. It does not download any files. You will need the download page's URL (this will be in the bundle's email, and possibly your purchase history). It will not work with 2FA, and I'm unlikely to be able to fix it without making it far more complicated \ No newline at end of file diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..68148ad --- /dev/null +++ b/poetry.lock @@ -0,0 +1,147 @@ +[[package]] +name = "beautifulsoup4" +version = "4.10.0" +description = "Screen-scraping library" +category = "main" +optional = false +python-versions = ">3.0.0" + +[package.dependencies] +soupsieve = ">1.2" + +[package.extras] +html5lib = ["html5lib"] +lxml = ["lxml"] + +[[package]] +name = "certifi" +version = "2021.10.8" +description = "Python package for providing Mozilla's CA Bundle." +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "charset-normalizer" +version = "2.0.9" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" +optional = false +python-versions = ">=3.5.0" + +[package.extras] +unicode_backport = ["unicodedata2"] + +[[package]] +name = "colorama" +version = "0.4.4" +description = "Cross-platform colored terminal text." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "idna" +version = "3.3" +description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" +optional = false +python-versions = ">=3.5" + +[[package]] +name = "requests" +version = "2.26.0" +description = "Python HTTP for Humans." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = {version = ">=2.0.0,<2.1.0", markers = "python_version >= \"3\""} +idna = {version = ">=2.5,<4", markers = "python_version >= \"3\""} +urllib3 = ">=1.21.1,<1.27" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"] + +[[package]] +name = "soupsieve" +version = "2.3.1" +description = "A modern CSS selector implementation for Beautiful Soup." +category = "main" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "tqdm" +version = "4.62.3" +description = "Fast, Extensible Progress Meter" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +dev = ["py-make (>=0.1.0)", "twine", "wheel"] +notebook = ["ipywidgets (>=6)"] +telegram = ["requests"] + +[[package]] +name = "urllib3" +version = "1.26.7" +description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" + +[package.extras] +brotli = ["brotlipy (>=0.6.0)"] +secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[metadata] +lock-version = "1.1" +python-versions = "^3.10" +content-hash = "c4579afeecd957dc271da7282c523c619640819471bde1ca4c41228d0521932c" + +[metadata.files] +beautifulsoup4 = [ + {file = "beautifulsoup4-4.10.0-py3-none-any.whl", hash = "sha256:9a315ce70049920ea4572a4055bc4bd700c940521d36fc858205ad4fcde149bf"}, + {file = "beautifulsoup4-4.10.0.tar.gz", hash = "sha256:c23ad23c521d818955a4151a67d81580319d4bf548d3d49f4223ae041ff98891"}, +] +certifi = [ + {file = "certifi-2021.10.8-py2.py3-none-any.whl", hash = "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"}, + {file = "certifi-2021.10.8.tar.gz", hash = "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872"}, +] +charset-normalizer = [ + {file = "charset-normalizer-2.0.9.tar.gz", hash = "sha256:b0b883e8e874edfdece9c28f314e3dd5badf067342e42fb162203335ae61aa2c"}, + {file = "charset_normalizer-2.0.9-py3-none-any.whl", hash = "sha256:1eecaa09422db5be9e29d7fc65664e6c33bd06f9ced7838578ba40d58bdf3721"}, +] +colorama = [ + {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, + {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, +] +idna = [ + {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, + {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, +] +requests = [ + {file = "requests-2.26.0-py2.py3-none-any.whl", hash = "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24"}, + {file = "requests-2.26.0.tar.gz", hash = "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7"}, +] +soupsieve = [ + {file = "soupsieve-2.3.1-py3-none-any.whl", hash = "sha256:1a3cca2617c6b38c0343ed661b1fa5de5637f257d4fe22bd9f1338010a1efefb"}, + {file = "soupsieve-2.3.1.tar.gz", hash = "sha256:b8d49b1cd4f037c7082a9683dfa1801aa2597fb11c3a1155b7a5b94829b4f1f9"}, +] +tqdm = [ + {file = "tqdm-4.62.3-py2.py3-none-any.whl", hash = "sha256:8dd278a422499cd6b727e6ae4061c40b48fce8b76d1ccbf5d34fca9b7f925b0c"}, + {file = "tqdm-4.62.3.tar.gz", hash = "sha256:d359de7217506c9851b7869f3708d8ee53ed70a1b8edbba4dbcb47442592920d"}, +] +urllib3 = [ + {file = "urllib3-1.26.7-py2.py3-none-any.whl", hash = "sha256:c4fdf4019605b6e5423637e01bc9fe4daef873709a7973e195ceba0a62bbc844"}, + {file = "urllib3-1.26.7.tar.gz", hash = "sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece"}, +] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..b021427 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,18 @@ +[tool.poetry] +name = "itchio" +version = "1.0.0" +description = "Python Scripts for downloading / archiving your itchio library" +authors = ["Peter Taylor "] +license = "MIT" + +[tool.poetry.dependencies] +python = "^3.10" +requests = "beautifulsoup4" +beautifulsoup4 = "^4.10.0" +tqdm = "^4.62.3" + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/src/itchio/__init__.py b/src/itchio/__init__.py new file mode 100644 index 0000000..c1757b0 --- /dev/null +++ b/src/itchio/__init__.py @@ -0,0 +1,4 @@ +from .login import LoginWeb, LoginAPI +from .bundle import Bundle +from .library import Library +from .game import Game \ No newline at end of file diff --git a/src/itchio/bundle.py b/src/itchio/bundle.py new file mode 100644 index 0000000..5a0b301 --- /dev/null +++ b/src/itchio/bundle.py @@ -0,0 +1,39 @@ +import requests +from bs4 import BeautifulSoup as soup + +class Bundle: + def __init__(self, login, url): + self.login = login + self.url = url + + def load_games(self): + i = 1 + + r = self.login.get(self.url) + s = soup(r.text, "html.parser") + pages = int(s.find("span", {"class": "pager_label"}).find_all("a")[-1].text) + while i < pages: + if self.load_game(i): + i += 1 + print(f"Processing Page {i} of {pages}") + + def load_game(self, i): + # Load 1 game. This will refresh the game afterwards, as the csrf token will update + r = self.login.get(f"{self.url}?page={i}") + s = soup(r.text, "html.parser") + for g in s.find_all("div", {"class": "game_row"}): + name = g.find("h2").find("a").text + if f := g.find("form"): + print(f"Processing {name}") + + game_id = f.find("input", {"name":"game_id"})["value"] + csrf_token = f.find("input", {"name":"csrf_token"})["value"] + + data = {"action":"claim", "game_id":game_id, "csrf_token":csrf_token} + + r = self.login.post(f"{self.url}?page={i}", data=data) + return False + #else: + # print(f"Skipping {name} - Already in Library") + return True + diff --git a/src/itchio/bundle/__main__.py b/src/itchio/bundle/__main__.py new file mode 100644 index 0000000..513208d --- /dev/null +++ b/src/itchio/bundle/__main__.py @@ -0,0 +1,11 @@ +from getpass import getpass +import itchio + +user = input("Username: ") +password = getpass("Password: ") + +l = itchio.LoginWeb(user, password) + +url = input("Bundle URL: ") +b = itchio.Bundle(l, url) +b.load_games() \ No newline at end of file diff --git a/src/itchio/downloader/__main__.py b/src/itchio/downloader/__main__.py new file mode 100644 index 0000000..45f7b1c --- /dev/null +++ b/src/itchio/downloader/__main__.py @@ -0,0 +1,26 @@ +import argparse +from getpass import getpass + +import itchio + +parser = argparse.ArgumentParser( + prog='python -m hstp', + description='Build an ' +) + +parser.add_argument("-k", "--api-key", help="Use API key instead of username/password") + +args = parser.parse_args() + +l = "" + +if not args.api_key: + user = input("Username: ") + password = getpass("Password: ") + l = itchio.LoginWeb(user, password) +else: + l = args.api_key + +lib = itchio.Library(l) +lib.load_games() +lib.download_library() diff --git a/src/itchio/game.py b/src/itchio/game.py new file mode 100644 index 0000000..7334a04 --- /dev/null +++ b/src/itchio/game.py @@ -0,0 +1,58 @@ +import re +import requests +import json +import os + +import itchio.utils + +class Game: + def __init__(self, data): + self.data = data["game"] + self.name = self.data["title"] + self.publisher = self.data["user"]["username"] + self.link = self.data["url"] + + self.id = data["id"] + self.game_id = data["game_id"] + + matches = re.match(r"https://(.+)\.itch\.io/(.+)", self.link) + self.game_slug = matches.group(2) + self.publisher_slug = matches.group(1) + + self.files = [] + + def load_downloads(self, token): + self.downloads = [] + r = requests.get(f"https://api.itch.io/games/{self.game_id}/uploads?download_key_id={self.id}", headers={"Authorization": token}) + j = r.json() + for d in j["uploads"]: + self.downloads.append(d) + + def download(self, token): + self.load_downloads(token) + + if not os.path.exists(self.publisher_slug): + os.mkdir(self.publisher_slug) + + if not os.path.exists(f"{self.publisher_slug}/{self.game_slug}"): + os.mkdir(f"{self.publisher_slug}/{self.game_slug}") + + with open(f"{self.publisher_slug}/{self.game_slug}.json", "w") as f: + json.dump({ + "name": self.name, + "publisher": self.publisher, + "link": self.link, + "itch_id": self.id, + "game_id": self.game_id, + "itch_data": self.data, + }, f) + + for d in self.downloads: + # Get UUID + r = requests.post(f"https://api.itch.io/games/{self.game_id}/download-sessions", headers={"Authorization": token}) + j = r.json() + print(j) + + # Download + url = f"https://api.itch.io/uploads/{d['id']}/download?api_key={token}&download_key_id={self.id}&uuid={j['uuid']}" + itchio.utils.download_url(url, f"{self.publisher_slug}/{self.game_slug}/{d['filename']}", self.name +" - "+d["filename"]) diff --git a/src/itchio/library.py b/src/itchio/library.py new file mode 100644 index 0000000..71b57e3 --- /dev/null +++ b/src/itchio/library.py @@ -0,0 +1,32 @@ +import requests +import json + +from itchio.game import Game + +class Library: + def __init__(self, login): + self.login = login + self.games = [] + + def load_game_page(self, page): + print("Loading page", page) + r = requests.get(f"https://api.itch.io/profile/owned-keys?page={page}", headers={"Authorization": self.login}) + j = json.loads(r.text) + + + for s in j["owned_keys"]: + self.games.append(Game(s)) + + return len(j["owned_keys"]) + + def load_games(self): + page = 1 + while True: + n = self.load_game_page(page) + if n == 0: + break + page += 1 + + def download_library(self): + for game in self.games: + game.download(self.login) diff --git a/src/itchio/login.py b/src/itchio/login.py new file mode 100644 index 0000000..3e09ba9 --- /dev/null +++ b/src/itchio/login.py @@ -0,0 +1,32 @@ +import requests +import json +from bs4 import BeautifulSoup as soup +import requests + +def LoginWeb(user, password): + session = requests.Session() + + # GET the page first so we have a valid CSRF token value + login1 = session.get("https://itch.io/login") + s = soup(login1.text, "html.parser") + csrf_token = s.find("input", {"name": "csrf_token"})["value"] + + # Now POST the login + r = session.post("https://itch.io/login", {"username": user, "password":password, "csrf_token": csrf_token}) + + if r.status_code != 200: + raise RuntimeError + + return session + +def LoginAPI(user, password): + r = requests.post("https://api.itch.io/login", {"username": user, "password":password, "source": "desktop"}) + if r.status_code != 200: + raise RuntimeError + t = json.loads(r.text) + + if not t["success"]: + raise RuntimeError + + return t["key"]["key"] + \ No newline at end of file diff --git a/src/itchio/utils.py b/src/itchio/utils.py new file mode 100644 index 0000000..da9052b --- /dev/null +++ b/src/itchio/utils.py @@ -0,0 +1,16 @@ +import urllib.request + +from tqdm import tqdm + + +class DownloadProgressBar(tqdm): + def update_to(self, b=1, bsize=1, tsize=None): + if tsize is not None: + self.total = tsize + self.update(b * bsize - self.n) + + +def download_url(url, output_path, desc): + with DownloadProgressBar(unit='B', unit_scale=True, + miniters=1, desc=desc) as t: + urllib.request.urlretrieve(url, filename=output_path, reporthook=t.update_to)