proxy: make proxying requests

core/medium_parser/bypass_cloudflare.py

@@ -0,0 +1,37 @@
+# source: https://github.com/thorio/KGrabber/issues/35#issuecomment-1164056688
+# curl -L --user-agent "Mozilla/5.0" --cipher AES256-SHA256 --tls-max 1.2 "<url>"
+
+import sys
+
+async def main():
+    # Create a default SSL context
+    ssl_context = ssl.create_default_context()
+
+    # Disable TLS versions older than 1.2
+    ssl_context.options |= ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1 | ssl.OP_NO_TLSv1_3 # ??? - --tls-max
+
+    # Set cipher suite
+    ssl_context.set_ciphers('AES256-SHA')
+
+    headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:126.0) Gecko/20100101 Firefox/126.0'}
+
+    # Configure aiohttp ClientSession with custom SSL context and User-Agent
+    async with aiohttp.ClientSession(headers=headers) as session:
+        async with session.get('https://github.zhaoqumao.com/veggiemonk/awesome-docker', ssl=ssl_context) as response:
+            print(await response.text())
+            print(response.status)
+
+
+### 
+
+# Port to Python 3.10+
+# https://github.com/fedora-infra/fedora-messaging/pull/250/commits/f3439350de7f55d42f4350e662392e9ffbdde028
+
+ssl_context = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH) # ??? - purpose
+if sys.version_info >= (3, 7):
+    ssl_context.minimum_version = ssl.TLSVersion.TLSv1_2
+else:
+    ssl_context.options |= ssl.OP_NO_SSLv2
+    ssl_context.options |= ssl.OP_NO_SSLv3
+    ssl_context.options |= ssl.OP_NO_TLSv1
+    ssl_context.options |= ssl.OP_NO_TLSv1_1

core/medium_parser/medium_api.py

@@ -3,18 +3,22 @@ from typing import Optional
 import aiohttp
 import orjson
 from aiohttp_retry import RetryClient
+from aiohttp_socks import ProxyConnector
 from loguru import logger
 
 from . import retry_options
 from .time import get_unix_ms
 from .utils import generate_random_sha256_hash
 
+socks_proxy = "socks5h://wgcf1:1080"
 
-# https://gist.github.com/vladar/a4e3afd608cfe8b13e5844d75447f0a4
-async def query_post_by_id(post_id: str, timeout: int = 3, auth_cookies: Optional[str] = None):
-    logger.debug(f"Starting request contructing for post {post_id}")
+
+async def query_post_by_id(post_id: str, timeout: int = 3, auth_cookies: Optional[str] = None, use_proxy: bool = True):
+    logger.debug(f"Starting request construction for post {post_id}")
     auth_cookies = "" if auth_cookies is None else auth_cookies
 
+    connector = ProxyConnector.from_url(socks_proxy) if use_proxy else None
+
     headers = {
         "X-APOLLO-OPERATION-ID": generate_random_sha256_hash(),
         "X-APOLLO-OPERATION-NAME": "FullPostQuery",
@@ -44,7 +48,7 @@ async def query_post_by_id(post_id: str, timeout: int = 3, auth_cookies: Optiona
 
     logger.debug(f"Request started...")
 
-    async with aiohttp.ClientSession() as session:
+    async with aiohttp.ClientSession(connector=connector) as session:
         async with RetryClient(client_session=session, raise_for_status=False, retry_options=retry_options) as retry_client:
             async with retry_client.post(
                 "https://medium.com/_/graphql",

core/requirements.txt

@@ -1,6 +1,6 @@
 rl_string_helper==0.1.0
 database_lib==0.1.0
-
+aiohttp_socks
 loguru==0.6.0
 aiohttp==3.9.4
 aiohttp-retry==2.8.3

docker-compose-dev.yml

@@ -26,6 +26,40 @@ services:
     restart: always
     stop_grace_period: 2m
 
+  wgcf1:
+    image: neilpang/wgcf-docker:latest
+    volumes:
+    #   - ./wgcf:/wgcf
+      - /lib/modules:/lib/modules
+      - ./scripts/entryWGCF.sh:/entry.sh
+      - /etc/localtime:/etc/localtime:ro
+    privileged: true
+    sysctls:
+      net.ipv6.conf.all.disable_ipv6: 0
+    cap_add:
+      - NET_ADMIN
+    # ports:
+    #   - "9681:1080"
+    restart: always
+    healthcheck:
+      test: curl -fs https://www.cloudflare.com/cdn-cgi/trace | grep -q -E 'warp=(on|plus)' && exit 0 || exit 1
+      interval: 5s
+      timeout: 2s
+      retries: 10
+
+  dante_1:
+    image: shturman/dante:1.4.2
+    volumes:
+      - ./scripts/dante.config:/etc/sockd.conf
+      - /etc/localtime:/etc/localtime:ro
+    restart: always
+    environment:
+      - CFGFILE=/etc/sockd.conf
+    network_mode: "service:wgcf1"
+    depends_on:
+      wgcf1:
+        condition: service_healthy
+
   web:
     build:
       context: ./

scripts/dante.config

@@ -0,0 +1,19 @@
+debug: 0
+logoutput: stderr
+internal: 0.0.0.0 port = 1080
+external: wgcf
+socksmethod: username none
+clientmethod: none
+user.privileged: root
+user.unprivileged: nobody
+
+client pass {
+    from: 0.0.0.0/0 port 1-65535 to: 0.0.0.0/0
+    log: error
+}
+
+socks pass {
+    from: 0.0.0.0/0 to: 0.0.0.0/0
+    #socksmethod: username
+    log: error
+}

scripts/entryWGCF.sh

@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+
+set -e
+
+_downwgcf() {
+    echo
+    echo "clean up"
+    if ! wg-quick down wgcf; then
+        echo "error down"
+    fi
+    echo "clean up done"
+    exit 0
+}
+
+#-4|-6
+runwgcf() {
+    trap '_downwgcf' ERR TERM INT
+
+    _enableV4="1"
+    if [ "$1" = "-6" ]; then
+        _enableV4=""
+    fi
+
+    if [ ! -e "wgcf-account.toml" ]; then
+        wgcf register --accept-tos
+    fi
+
+    if [ -e "custom-wgcf-license.conf" ]; then
+        _license_key=$(cat custom-wgcf-license.conf)
+        _old_license_key=$(awk -F "['']" '/license_key/{print $2}' wgcf-account.toml)
+        if [ "$_license_key" != "$_old_license_key" ]; then
+            echo 'updating license key'
+            sed -i "s/license_key.*$/license_key = '${_license_key}'/" wgcf-account.toml
+            wgcf update
+        fi
+    fi
+
+    if [ ! -e "wgcf-profile.conf" ]; then
+        wgcf generate
+    else
+        _account_file_private_key=$(awk -F "['']" '/private_key/{print $2}' wgcf-account.toml)
+        _profile_file_private_key=$(awk -F " = " '/PrivateKey/{print $2}' wgcf-profile.conf)
+        if [ "$_account_file_private_key" != "$_profile_file_private_key" ]; then
+            echo 're generate profile'
+            wgcf generate
+        fi
+    fi
+
+    if [ -e "custom-wgcf-endpoint.conf" ]; then
+        _endpoint=$(cat custom-wgcf-endpoint.conf)
+        sed -i "s/Endpoint.*$/Endpoint = ${_endpoint}/" wgcf-profile.conf
+    fi
+
+    cp wgcf-profile.conf /etc/wireguard/wgcf.conf
+
+    DEFAULT_GATEWAY_NETWORK_CARD_NAME=$(route | grep default | awk '{print $8}' | head -1)
+    DEFAULT_ROUTE_IP=$(ifconfig $DEFAULT_GATEWAY_NETWORK_CARD_NAME | grep "inet " | awk '{print $2}' | sed "s/addr://")
+
+    echo ${DEFAULT_GATEWAY_NETWORK_CARD_NAME}
+    echo ${DEFAULT_ROUTE_IP}
+
+    sed -i "/\[Interface\]/a PostDown = ip rule delete from $DEFAULT_ROUTE_IP  lookup main" /etc/wireguard/wgcf.conf
+    sed -i "/\[Interface\]/a PostUp = ip rule add from $DEFAULT_ROUTE_IP lookup main" /etc/wireguard/wgcf.conf
+
+    if [ "$1" = "-6" ]; then
+        sed -i 's/AllowedIPs = 0.0.0.0/#AllowedIPs = 0.0.0.0/' /etc/wireguard/wgcf.conf
+    elif [ "$1" = "-4" ]; then
+        sed -i 's/AllowedIPs = ::/#AllowedIPs = ::/' /etc/wireguard/wgcf.conf
+    fi
+
+    modprobe ip6table_raw
+
+    wg-quick up wgcf
+
+    if [ "$_enableV4" ]; then
+        _checkV4
+    else
+        _checkV6
+    fi
+
+    echo
+    echo
+    echo "wgcf status"
+    wgcf status
+
+    echo
+    echo "OK, wgcf is up."
+
+    sleep infinity &
+    wait
+
+}
+
+_checkV4() {
+    echo "Checking network status, please wait...."
+    while ! curl --max-time 2 ipinfo.io; do
+        wg-quick down wgcf
+        echo "Sleep 2 and retry again."
+        sleep 2
+        wg-quick up wgcf
+    done
+
+}
+
+_checkV6() {
+    echo "Checking network status, please wait...."
+    while ! curl --max-time 2 -6 ipv6.google.com; do
+        wg-quick down wgcf
+        echo "Sleep 2 and retry again."
+        sleep 2
+        wg-quick up wgcf
+    done
+
+}
+
+if [ -z "$@" ] || [[ "$1" = -* ]]; then
+    runwgcf "$@"
+else
+    exec "$@"
+fi

server/handlers/reverse_proxy.py

@@ -29,7 +29,6 @@ async def iframe_proxy(iframe_id):
     return Response(content=request_content_soup.prettify(), media_type="text/html", headers=IFRAME_HEADERS)
 
 
-@trace
 async def miro_proxy(miro_data: str):
     async with aiohttp.ClientSession() as client:
         request = await client.get(