diff --git a/core/medium_parser/bypass_cloudflare.py b/core/medium_parser/bypass_cloudflare.py new file mode 100644 index 0000000..c4516b2 --- /dev/null +++ b/core/medium_parser/bypass_cloudflare.py @@ -0,0 +1,37 @@ +# source: https://github.com/thorio/KGrabber/issues/35#issuecomment-1164056688 +# curl -L --user-agent "Mozilla/5.0" --cipher AES256-SHA256 --tls-max 1.2 "" + +import sys + +async def main(): + # Create a default SSL context + ssl_context = ssl.create_default_context() + + # Disable TLS versions older than 1.2 + ssl_context.options |= ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1 | ssl.OP_NO_TLSv1_3 # ??? - --tls-max + + # Set cipher suite + ssl_context.set_ciphers('AES256-SHA') + + headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:126.0) Gecko/20100101 Firefox/126.0'} + + # Configure aiohttp ClientSession with custom SSL context and User-Agent + async with aiohttp.ClientSession(headers=headers) as session: + async with session.get('https://github.zhaoqumao.com/veggiemonk/awesome-docker', ssl=ssl_context) as response: + print(await response.text()) + print(response.status) + + +### + +# Port to Python 3.10+ +# https://github.com/fedora-infra/fedora-messaging/pull/250/commits/f3439350de7f55d42f4350e662392e9ffbdde028 + +ssl_context = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH) # ??? - purpose +if sys.version_info >= (3, 7): + ssl_context.minimum_version = ssl.TLSVersion.TLSv1_2 +else: + ssl_context.options |= ssl.OP_NO_SSLv2 + ssl_context.options |= ssl.OP_NO_SSLv3 + ssl_context.options |= ssl.OP_NO_TLSv1 + ssl_context.options |= ssl.OP_NO_TLSv1_1 \ No newline at end of file diff --git a/core/medium_parser/medium_api.py b/core/medium_parser/medium_api.py index 6005174..b594436 100644 --- a/core/medium_parser/medium_api.py +++ b/core/medium_parser/medium_api.py @@ -3,18 +3,22 @@ from typing import Optional import aiohttp import orjson from aiohttp_retry import RetryClient +from aiohttp_socks import ProxyConnector from loguru import logger from . import retry_options from .time import get_unix_ms from .utils import generate_random_sha256_hash +socks_proxy = "socks5h://wgcf1:1080" -# https://gist.github.com/vladar/a4e3afd608cfe8b13e5844d75447f0a4 -async def query_post_by_id(post_id: str, timeout: int = 3, auth_cookies: Optional[str] = None): - logger.debug(f"Starting request contructing for post {post_id}") + +async def query_post_by_id(post_id: str, timeout: int = 3, auth_cookies: Optional[str] = None, use_proxy: bool = True): + logger.debug(f"Starting request construction for post {post_id}") auth_cookies = "" if auth_cookies is None else auth_cookies + connector = ProxyConnector.from_url(socks_proxy) if use_proxy else None + headers = { "X-APOLLO-OPERATION-ID": generate_random_sha256_hash(), "X-APOLLO-OPERATION-NAME": "FullPostQuery", @@ -44,7 +48,7 @@ async def query_post_by_id(post_id: str, timeout: int = 3, auth_cookies: Optiona logger.debug(f"Request started...") - async with aiohttp.ClientSession() as session: + async with aiohttp.ClientSession(connector=connector) as session: async with RetryClient(client_session=session, raise_for_status=False, retry_options=retry_options) as retry_client: async with retry_client.post( "https://medium.com/_/graphql", diff --git a/core/requirements.txt b/core/requirements.txt index e3006b2..a01ed85 100644 --- a/core/requirements.txt +++ b/core/requirements.txt @@ -1,6 +1,6 @@ rl_string_helper==0.1.0 database_lib==0.1.0 - +aiohttp_socks loguru==0.6.0 aiohttp==3.9.4 aiohttp-retry==2.8.3 diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 734a4eb..fa73211 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -26,6 +26,40 @@ services: restart: always stop_grace_period: 2m + wgcf1: + image: neilpang/wgcf-docker:latest + volumes: + # - ./wgcf:/wgcf + - /lib/modules:/lib/modules + - ./scripts/entryWGCF.sh:/entry.sh + - /etc/localtime:/etc/localtime:ro + privileged: true + sysctls: + net.ipv6.conf.all.disable_ipv6: 0 + cap_add: + - NET_ADMIN + # ports: + # - "9681:1080" + restart: always + healthcheck: + test: curl -fs https://www.cloudflare.com/cdn-cgi/trace | grep -q -E 'warp=(on|plus)' && exit 0 || exit 1 + interval: 5s + timeout: 2s + retries: 10 + + dante_1: + image: shturman/dante:1.4.2 + volumes: + - ./scripts/dante.config:/etc/sockd.conf + - /etc/localtime:/etc/localtime:ro + restart: always + environment: + - CFGFILE=/etc/sockd.conf + network_mode: "service:wgcf1" + depends_on: + wgcf1: + condition: service_healthy + web: build: context: ./ diff --git a/scripts/dante.config b/scripts/dante.config new file mode 100644 index 0000000..64ce6de --- /dev/null +++ b/scripts/dante.config @@ -0,0 +1,19 @@ +debug: 0 +logoutput: stderr +internal: 0.0.0.0 port = 1080 +external: wgcf +socksmethod: username none +clientmethod: none +user.privileged: root +user.unprivileged: nobody + +client pass { + from: 0.0.0.0/0 port 1-65535 to: 0.0.0.0/0 + log: error +} + +socks pass { + from: 0.0.0.0/0 to: 0.0.0.0/0 + #socksmethod: username + log: error +} diff --git a/scripts/entryWGCF.sh b/scripts/entryWGCF.sh new file mode 100755 index 0000000..348ec5b --- /dev/null +++ b/scripts/entryWGCF.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash + +set -e + +_downwgcf() { + echo + echo "clean up" + if ! wg-quick down wgcf; then + echo "error down" + fi + echo "clean up done" + exit 0 +} + +#-4|-6 +runwgcf() { + trap '_downwgcf' ERR TERM INT + + _enableV4="1" + if [ "$1" = "-6" ]; then + _enableV4="" + fi + + if [ ! -e "wgcf-account.toml" ]; then + wgcf register --accept-tos + fi + + if [ -e "custom-wgcf-license.conf" ]; then + _license_key=$(cat custom-wgcf-license.conf) + _old_license_key=$(awk -F "['']" '/license_key/{print $2}' wgcf-account.toml) + if [ "$_license_key" != "$_old_license_key" ]; then + echo 'updating license key' + sed -i "s/license_key.*$/license_key = '${_license_key}'/" wgcf-account.toml + wgcf update + fi + fi + + if [ ! -e "wgcf-profile.conf" ]; then + wgcf generate + else + _account_file_private_key=$(awk -F "['']" '/private_key/{print $2}' wgcf-account.toml) + _profile_file_private_key=$(awk -F " = " '/PrivateKey/{print $2}' wgcf-profile.conf) + if [ "$_account_file_private_key" != "$_profile_file_private_key" ]; then + echo 're generate profile' + wgcf generate + fi + fi + + if [ -e "custom-wgcf-endpoint.conf" ]; then + _endpoint=$(cat custom-wgcf-endpoint.conf) + sed -i "s/Endpoint.*$/Endpoint = ${_endpoint}/" wgcf-profile.conf + fi + + cp wgcf-profile.conf /etc/wireguard/wgcf.conf + + DEFAULT_GATEWAY_NETWORK_CARD_NAME=$(route | grep default | awk '{print $8}' | head -1) + DEFAULT_ROUTE_IP=$(ifconfig $DEFAULT_GATEWAY_NETWORK_CARD_NAME | grep "inet " | awk '{print $2}' | sed "s/addr://") + + echo ${DEFAULT_GATEWAY_NETWORK_CARD_NAME} + echo ${DEFAULT_ROUTE_IP} + + sed -i "/\[Interface\]/a PostDown = ip rule delete from $DEFAULT_ROUTE_IP lookup main" /etc/wireguard/wgcf.conf + sed -i "/\[Interface\]/a PostUp = ip rule add from $DEFAULT_ROUTE_IP lookup main" /etc/wireguard/wgcf.conf + + if [ "$1" = "-6" ]; then + sed -i 's/AllowedIPs = 0.0.0.0/#AllowedIPs = 0.0.0.0/' /etc/wireguard/wgcf.conf + elif [ "$1" = "-4" ]; then + sed -i 's/AllowedIPs = ::/#AllowedIPs = ::/' /etc/wireguard/wgcf.conf + fi + + modprobe ip6table_raw + + wg-quick up wgcf + + if [ "$_enableV4" ]; then + _checkV4 + else + _checkV6 + fi + + echo + echo + echo "wgcf status" + wgcf status + + echo + echo "OK, wgcf is up." + + sleep infinity & + wait + +} + +_checkV4() { + echo "Checking network status, please wait...." + while ! curl --max-time 2 ipinfo.io; do + wg-quick down wgcf + echo "Sleep 2 and retry again." + sleep 2 + wg-quick up wgcf + done + +} + +_checkV6() { + echo "Checking network status, please wait...." + while ! curl --max-time 2 -6 ipv6.google.com; do + wg-quick down wgcf + echo "Sleep 2 and retry again." + sleep 2 + wg-quick up wgcf + done + +} + +if [ -z "$@" ] || [[ "$1" = -* ]]; then + runwgcf "$@" +else + exec "$@" +fi diff --git a/server/handlers/reverse_proxy.py b/server/handlers/reverse_proxy.py index e47c938..278e3c0 100644 --- a/server/handlers/reverse_proxy.py +++ b/server/handlers/reverse_proxy.py @@ -29,7 +29,6 @@ async def iframe_proxy(iframe_id): return Response(content=request_content_soup.prettify(), media_type="text/html", headers=IFRAME_HEADERS) -@trace async def miro_proxy(miro_data: str): async with aiohttp.ClientSession() as client: request = await client.get(