diff --git a/app/__init__.py b/app/__init__.py index 772d6d3..34dad77 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -53,24 +53,18 @@ app.config['BUILD_FOLDER'] = os.path.join( app.config['STATIC_FOLDER'], 'build') app.config['CACHE_BUSTING_MAP'] = {} app.config['BUNDLE_STATIC'] = read_config_bool('WHOOGLE_BUNDLE_STATIC') -app.config['LANGUAGES'] = json.load(open( - os.path.join(app.config['STATIC_FOLDER'], 'settings/languages.json'), - encoding='utf-8')) -app.config['COUNTRIES'] = json.load(open( - os.path.join(app.config['STATIC_FOLDER'], 'settings/countries.json'), - encoding='utf-8')) -app.config['TIME_PERIODS'] = json.load(open( - os.path.join(app.config['STATIC_FOLDER'], 'settings/time_periods.json'), - encoding='utf-8')) -app.config['TRANSLATIONS'] = json.load(open( - os.path.join(app.config['STATIC_FOLDER'], 'settings/translations.json'), - encoding='utf-8')) -app.config['THEMES'] = json.load(open( - os.path.join(app.config['STATIC_FOLDER'], 'settings/themes.json'), - encoding='utf-8')) -app.config['HEADER_TABS'] = json.load(open( - os.path.join(app.config['STATIC_FOLDER'], 'settings/header_tabs.json'), - encoding='utf-8')) +with open(os.path.join(app.config['STATIC_FOLDER'], 'settings/languages.json'), 'r', encoding='utf-8') as f: + app.config['LANGUAGES'] = json.load(f) +with open(os.path.join(app.config['STATIC_FOLDER'], 'settings/countries.json'), 'r', encoding='utf-8') as f: + app.config['COUNTRIES'] = json.load(f) +with open(os.path.join(app.config['STATIC_FOLDER'], 'settings/time_periods.json'), 'r', encoding='utf-8') as f: + app.config['TIME_PERIODS'] = json.load(f) +with open(os.path.join(app.config['STATIC_FOLDER'], 'settings/translations.json'), 'r', encoding='utf-8') as f: + app.config['TRANSLATIONS'] = json.load(f) +with open(os.path.join(app.config['STATIC_FOLDER'], 'settings/themes.json'), 'r', encoding='utf-8') as f: + app.config['THEMES'] = json.load(f) +with open(os.path.join(app.config['STATIC_FOLDER'], 'settings/header_tabs.json'), 'r', encoding='utf-8') as f: + app.config['HEADER_TABS'] = json.load(f) app.config['CONFIG_PATH'] = os.getenv( 'CONFIG_VOLUME', os.path.join(app.config['STATIC_FOLDER'], 'config')) @@ -117,14 +111,14 @@ if not os.path.exists(app.config['BUILD_FOLDER']): app_key_path = os.path.join(app.config['CONFIG_PATH'], 'whoogle.key') if os.path.exists(app_key_path): try: - app.config['SECRET_KEY'] = open(app_key_path, 'r').read() + with open(app_key_path, 'r', encoding='utf-8') as f: + app.config['SECRET_KEY'] = f.read() except PermissionError: app.config['SECRET_KEY'] = str(b64encode(os.urandom(32))) else: app.config['SECRET_KEY'] = str(b64encode(os.urandom(32))) - with open(app_key_path, 'w') as key_file: + with open(app_key_path, 'w', encoding='utf-8') as key_file: key_file.write(app.config['SECRET_KEY']) - key_file.close() app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(days=365) # NOTE: SESSION_COOKIE_SAMESITE must be set to 'lax' to allow the user's @@ -160,7 +154,8 @@ app.config['CSP'] = 'default-src \'none\';' \ generating_bangs = False if not os.path.exists(app.config['BANG_FILE']): generating_bangs = True - json.dump({}, open(app.config['BANG_FILE'], 'w')) + with open(app.config['BANG_FILE'], 'w', encoding='utf-8') as f: + json.dump({}, f) bangs_thread = threading.Thread( target=gen_bangs_json, args=(app.config['BANG_FILE'],)) @@ -199,13 +194,15 @@ if app.config['BUNDLE_STATIC']: if name.endswith('-theme.css'): continue try: - css_parts.append(open(os.path.join(css_dir, name), 'r', encoding='utf-8').read()) + with open(os.path.join(css_dir, name), 'r', encoding='utf-8') as f: + css_parts.append(f.read()) except Exception: pass css_bundle = '\n'.join(css_parts) if css_bundle: css_tmp = os.path.join(app.config['BUILD_FOLDER'], 'app.css') - open(css_tmp, 'w', encoding='utf-8').write(css_bundle) + with open(css_tmp, 'w', encoding='utf-8') as f: + f.write(css_bundle) css_hashed = gen_file_hash(app.config['BUILD_FOLDER'], 'app.css') os.replace(css_tmp, os.path.join(app.config['BUILD_FOLDER'], css_hashed)) map_path = os.path.join('app/static/build', css_hashed) @@ -218,13 +215,15 @@ if app.config['BUNDLE_STATIC']: if not name.endswith('.js'): continue try: - js_parts.append(open(os.path.join(js_dir, name), 'r', encoding='utf-8').read()) + with open(os.path.join(js_dir, name), 'r', encoding='utf-8') as f: + js_parts.append(f.read()) except Exception: pass js_bundle = '\n;'.join(js_parts) if js_bundle: js_tmp = os.path.join(app.config['BUILD_FOLDER'], 'app.js') - open(js_tmp, 'w', encoding='utf-8').write(js_bundle) + with open(js_tmp, 'w', encoding='utf-8') as f: + f.write(js_bundle) js_hashed = gen_file_hash(app.config['BUILD_FOLDER'], 'app.js') os.replace(js_tmp, os.path.join(app.config['BUILD_FOLDER'], js_hashed)) map_path = os.path.join('app/static/build', js_hashed) diff --git a/app/filter.py b/app/filter.py index 8c40dbe..f079a64 100644 --- a/app/filter.py +++ b/app/filter.py @@ -219,7 +219,7 @@ class Filter: return for d in div.find_all('div', recursive=True): - d_text = d.find(text=True, recursive=False) + d_text = d.find(string=True, recursive=False) # Ensure we're working with tags that contain text content if not d_text or not d.string: @@ -295,7 +295,7 @@ class Filter: return search_string = ' '.join(['-site:' + _ for _ in self.config.block.split(',')]) - selected = soup.body.findAll(text=re.compile(search_string)) + selected = soup.body.find_all(string=re.compile(search_string)) for result in selected: result.string.replace_with(result.string.replace( @@ -362,11 +362,11 @@ class Filter: def pull_child_divs(result_div: BeautifulSoup): try: - return result_div.findChildren( - 'div', recursive=False - )[0].findChildren( - 'div', recursive=False) - except IndexError: + top_level_divs = result_div.find_all('div', recursive=False) + if not top_level_divs: + return [] + return top_level_divs[0].find_all('div', recursive=False) + except Exception: return [] if not self.main_divs: @@ -657,7 +657,7 @@ class Filter: prefix_pattern = re.compile(r'^(?:https?:\/\/)?(?:(?:www|mobile|m)\.)?') # 1) Replace bare domain divs (single token) once, avoiding duplicates - for div in self.soup.find_all('div', text=sites_pattern): + for div in self.soup.find_all('div', string=sites_pattern): if not div or not div.string: continue if len(div.string.split(' ')) != 1: @@ -679,7 +679,7 @@ class Filter: link['href'] = get_site_alt(link['href']) # Find a description text node matching a known site - desc_nodes = link.find_all(text=sites_pattern) + desc_nodes = link.find_all(string=sites_pattern) if not desc_nodes: continue desc_node = desc_nodes[0] diff --git a/app/models/config.py b/app/models/config.py index 2e73f6d..08d0e63 100644 --- a/app/models/config.py +++ b/app/models/config.py @@ -131,10 +131,9 @@ class Config: Returns: str -- the new style """ - style_sheet = cssutils.parseString( - open(os.path.join(current_app.config['STATIC_FOLDER'], - 'css/variables.css')).read() - ) + vars_path = os.path.join(current_app.config['STATIC_FOLDER'], 'css/variables.css') + with open(vars_path, 'r', encoding='utf-8') as f: + style_sheet = cssutils.parseString(f.read()) modified_sheet = cssutils.parseString(self.style_modified) for rule in modified_sheet: diff --git a/app/request.py b/app/request.py index 5f40bbc..734ea95 100644 --- a/app/request.py +++ b/app/request.py @@ -205,9 +205,10 @@ class Request: def __init__(self, normal_ua, root_path, config: Config, http_client=None): self.search_url = 'https://www.google.com/search?gbv=1&num=' + str( os.getenv('WHOOGLE_RESULTS_PER_PAGE', 10)) + '&q=' - # Send heartbeat to Tor, used in determining if the user can or cannot - # enable Tor for future requests - send_tor_signal(Signal.HEARTBEAT) + # Optionally send heartbeat to Tor to determine availability + # Only when Tor is enabled in config to avoid unnecessary socket usage + if config.tor: + send_tor_signal(Signal.HEARTBEAT) self.language = config.lang_search if config.lang_search else '' self.country = config.country if config.country else '' @@ -334,10 +335,12 @@ class Request: # view is suppressed correctly now = datetime.now() - cookies = { - 'CONSENT': 'PENDING+987', - 'SOCS': 'CAESHAgBEhIaAB', - } + consent_cookie = 'CONSENT=PENDING+987; SOCS=CAESHAgBEhIaAB' + # Prefer header-based cookies to avoid httpx per-request cookies deprecation + if 'Cookie' in headers: + headers['Cookie'] += '; ' + consent_cookie + else: + headers['Cookie'] = consent_cookie # Validate Tor conn and request new identity if the last one failed if self.tor and not send_tor_signal( @@ -368,8 +371,7 @@ class Request: try: response = self.http_client.get( (base_url or self.search_url) + query, - headers=headers, - cookies=cookies) + headers=headers) except httpx.HTTPError as e: raise diff --git a/app/routes.py b/app/routes.py index 258abb9..24beea3 100644 --- a/app/routes.py +++ b/app/routes.py @@ -682,7 +682,7 @@ def internal_error(e): fallback_engine = os.environ.get('WHOOGLE_FALLBACK_ENGINE_URL', '') if (fallback_engine): - return redirect(fallback_engine + query) + return redirect(fallback_engine + (query or '')) localization_lang = g.user_config.get_localization_lang() translation = app.config['TRANSLATIONS'][localization_lang] @@ -692,7 +692,7 @@ def internal_error(e): translation=translation, farside='https://farside.link', config=g.user_config, - query=urlparse.unquote(query), + query=urlparse.unquote(query or ''), params=g.user_config.to_params(keys=['preferences'])), 500 diff --git a/app/services/http_client.py b/app/services/http_client.py index 4f9730b..9bb1184 100644 --- a/app/services/http_client.py +++ b/app/services/http_client.py @@ -4,6 +4,8 @@ from typing import Any, Dict, Optional, Tuple import httpx from cachetools import TTLCache +import ssl +import os class HttpxClient: @@ -25,38 +27,81 @@ class HttpxClient: follow_redirects=True) # Prefer future-proof mounts when proxies are provided; fall back to proxies= self._proxies = proxies or {} + self._http2 = http2 + + # Determine verify behavior and initialize client with fallbacks + self._verify = self._determine_verify_setting() + try: + self._client = self._build_client(client_kwargs, self._verify) + except ssl.SSLError: + # Fallback to system trust store + try: + system_ctx = ssl.create_default_context() + self._client = self._build_client(client_kwargs, system_ctx) + self._verify = system_ctx + except ssl.SSLError: + insecure_fallback = os.environ.get('WHOOGLE_INSECURE_FALLBACK', '0').lower() in ('1', 'true', 't', 'yes', 'y') + if insecure_fallback: + self._client = self._build_client(client_kwargs, False) + self._verify = False + else: + raise + self._timeout_seconds = timeout_seconds + self._cache = TTLCache(maxsize=cache_maxsize, ttl=cache_ttl_seconds) + self._cache_lock = threading.Lock() + + def _determine_verify_setting(self): + """Determine SSL verification setting from environment. + + Honors: + - WHOOGLE_CA_BUNDLE: path to CA bundle file + - WHOOGLE_SSL_VERIFY: '0' to disable verification + - WHOOGLE_SSL_BACKEND: 'system' to prefer system trust store + """ + ca_bundle = os.environ.get('WHOOGLE_CA_BUNDLE', '').strip() + if ca_bundle: + return ca_bundle + + verify_env = os.environ.get('WHOOGLE_SSL_VERIFY', '1').lower() + if verify_env in ('0', 'false', 'no', 'n'): + return False + + backend = os.environ.get('WHOOGLE_SSL_BACKEND', '').lower() + if backend == 'system': + return ssl.create_default_context() + + return True + + def _build_client(self, client_kwargs: Dict[str, Any], verify: Any) -> httpx.Client: + """Construct httpx.Client with proxies and provided verify setting.""" + kwargs = dict(client_kwargs) + kwargs['verify'] = verify if self._proxies: - # If both schemes map to the same proxy, try the newer proxy= API first proxy_values = list(self._proxies.values()) single_proxy = proxy_values[0] if proxy_values and all(v == proxy_values[0] for v in proxy_values) else None if single_proxy: try: - self._client = httpx.Client(proxy=single_proxy, **client_kwargs) + return httpx.Client(proxy=single_proxy, **kwargs) except TypeError: - # Older httpx that doesn't support proxy=; try proxies= try: - self._client = httpx.Client(proxies=self._proxies, **client_kwargs) + return httpx.Client(proxies=self._proxies, **kwargs) except TypeError: mounts: Dict[str, httpx.Proxy] = {} for scheme_key, url in self._proxies.items(): prefix = f"{scheme_key}://" mounts[prefix] = httpx.Proxy(url) - self._client = httpx.Client(mounts=mounts, **client_kwargs) + return httpx.Client(mounts=mounts, **kwargs) else: - # Distinct proxies per scheme; use mounts fallback if needed try: - self._client = httpx.Client(proxies=self._proxies, **client_kwargs) + return httpx.Client(proxies=self._proxies, **kwargs) except TypeError: mounts: Dict[str, httpx.Proxy] = {} for scheme_key, url in self._proxies.items(): prefix = f"{scheme_key}://" mounts[prefix] = httpx.Proxy(url) - self._client = httpx.Client(mounts=mounts, **client_kwargs) + return httpx.Client(mounts=mounts, **kwargs) else: - self._client = httpx.Client(**client_kwargs) - self._timeout_seconds = timeout_seconds - self._cache = TTLCache(maxsize=cache_maxsize, ttl=cache_ttl_seconds) - self._cache_lock = threading.Lock() + return httpx.Client(**kwargs) @property def proxies(self) -> Dict[str, str]: @@ -119,34 +164,23 @@ class HttpxClient: # Recreate with same configuration client_kwargs = dict(timeout=self._timeout_seconds, - follow_redirects=True) - - if self._proxies: - proxy_values = list(self._proxies.values()) - single_proxy = proxy_values[0] if proxy_values and all(v == proxy_values[0] for v in proxy_values) else None - if single_proxy: - try: - self._client = httpx.Client(proxy=single_proxy, **client_kwargs) - except TypeError: - try: - self._client = httpx.Client(proxies=self._proxies, **client_kwargs) - except TypeError: - mounts: Dict[str, httpx.Proxy] = {} - for scheme_key, url in self._proxies.items(): - prefix = f"{scheme_key}://" - mounts[prefix] = httpx.Proxy(url) - self._client = httpx.Client(mounts=mounts, **client_kwargs) - else: - try: - self._client = httpx.Client(proxies=self._proxies, **client_kwargs) - except TypeError: - mounts: Dict[str, httpx.Proxy] = {} - for scheme_key, url in self._proxies.items(): - prefix = f"{scheme_key}://" - mounts[prefix] = httpx.Proxy(url) - self._client = httpx.Client(mounts=mounts, **client_kwargs) - else: - self._client = httpx.Client(**client_kwargs) + follow_redirects=True, + http2=self._http2) + + try: + self._client = self._build_client(client_kwargs, self._verify) + except ssl.SSLError: + try: + system_ctx = ssl.create_default_context() + self._client = self._build_client(client_kwargs, system_ctx) + self._verify = system_ctx + except ssl.SSLError: + insecure_fallback = os.environ.get('WHOOGLE_INSECURE_FALLBACK', '0').lower() in ('1', 'true', 't', 'yes', 'y') + if insecure_fallback: + self._client = self._build_client(client_kwargs, False) + self._verify = False + else: + raise def close(self) -> None: self._client.close() diff --git a/app/utils/bangs.py b/app/utils/bangs.py index de9f109..52161fa 100644 --- a/app/utils/bangs.py +++ b/app/utils/bangs.py @@ -43,7 +43,8 @@ def load_all_bangs(ddg_bangs_file: str, ddg_bangs: dict = {}): for i, bang_file in enumerate(bang_files): try: - bangs |= json.load(open(bang_file)) + with open(bang_file, 'r', encoding='utf-8') as f: + bangs |= json.load(f) except json.decoder.JSONDecodeError: # Ignore decoding error only for the ddg bangs file, since this can # occur if file is still being written @@ -80,7 +81,8 @@ def gen_bangs_json(bangs_file: str) -> None: 'suggestion': bang_command + ' (' + row['s'] + ')' } - json.dump(bangs_data, open(bangs_file, 'w')) + with open(bangs_file, 'w', encoding='utf-8') as f: + json.dump(bangs_data, f) print('* Finished creating ddg bangs json') load_all_bangs(bangs_file, bangs_data) diff --git a/app/utils/misc.py b/app/utils/misc.py index 20a2640..d4e49d7 100644 --- a/app/utils/misc.py +++ b/app/utils/misc.py @@ -48,7 +48,8 @@ def fetch_favicon(url: str) -> bytes: def gen_file_hash(path: str, static_file: str) -> str: - file_contents = open(os.path.join(path, static_file), 'rb').read() + with open(os.path.join(path, static_file), 'rb') as f: + file_contents = f.read() file_hash = hashlib.md5(file_contents).hexdigest()[:8] filename_split = os.path.splitext(static_file) diff --git a/app/utils/results.py b/app/utils/results.py index bc7c910..d75d328 100644 --- a/app/utils/results.py +++ b/app/utils/results.py @@ -1,7 +1,8 @@ from app.models.config import Config from app.models.endpoint import Endpoint from app.utils.misc import list_to_dict -from bs4 import BeautifulSoup, NavigableString +from bs4 import BeautifulSoup, NavigableString, MarkupResemblesLocatorWarning +import warnings import copy from flask import current_app import html @@ -9,7 +10,7 @@ import os import urllib.parse as urlparse from urllib.parse import parse_qs import re -import warnings +warnings.filterwarnings('ignore', category=MarkupResemblesLocatorWarning) SKIP_ARGS = ['ref_src', 'utm'] SKIP_PREFIX = ['//www.', '//mobile.', '//m.'] @@ -114,7 +115,7 @@ def bold_search_terms(response: str, query: str) -> BeautifulSoup: for word in re.split(r'\s+(?=[^"]*(?:"[^"]*"[^"]*)*$)', query): word = re.sub(r'[@_!#$%^&*()<>?/\|}{~:]+', '', word) target = response.find_all( - text=re.compile(r'' + re.escape(word), re.I)) + string=re.compile(r'' + re.escape(word), re.I)) for nav_str in target: replace_any_case(nav_str, word) diff --git a/requirements.txt b/requirements.txt index 45865c3..f78ce8d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,36 +1,36 @@ -attrs==22.2.0 -beautifulsoup4==4.11.2 -brotli==1.0.9 -certifi==2024.7.4 -cffi==1.17.1 -click==8.1.3 +attrs==25.3.0 +beautifulsoup4==4.13.5 +brotli==1.1.0 +certifi==2025.8.3 +cffi==2.0.0 +click==8.3.0 cryptography==3.3.2; platform_machine == 'armv7l' -cryptography==45.0.7; platform_machine != 'armv7l' -cssutils==2.7.0 +cryptography==46.0.1; platform_machine != 'armv7l' +cssutils==2.11.1 defusedxml==0.7.1 Flask==2.3.2 -idna==3.7 +idna==3.10 itsdangerous==2.1.2 Jinja2==3.1.6 -MarkupSafe==2.1.2 -more-itertools==9.0.0 -packaging==23.0 -pluggy==1.0.0 -pycodestyle==2.10.0 +MarkupSafe==3.0.2 +more-itertools==10.8.0 +packaging==25.0 +pluggy==1.6.0 +pycodestyle==2.14.0 pycparser==2.22 pyOpenSSL==19.1.0; platform_machine == 'armv7l' pyOpenSSL==25.3.0; platform_machine != 'armv7l' -pyparsing==3.0.9 +pyparsing==3.2.5 pytest==7.2.1 -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 httpx[http2,socks]==0.28.1 -cachetools==5.5.0 -soupsieve==2.4 -stem==1.8.1 +cachetools==6.2.0 +soupsieve==2.8 +stem==1.8.2 httpcore>=1.0.9 h11>=0.16.0 -validators==0.22.0 -waitress==3.0.1 -wcwidth==0.2.6 +validators==0.35.0 +waitress==3.0.2 +wcwidth==0.2.14 Werkzeug==3.0.6 -python-dotenv==0.21.1 +python-dotenv==1.1.1