From be83605c7783dcf09eddf2a5b9833343178c4512 Mon Sep 17 00:00:00 2001 From: Don-Swanson <32144818+Don-Swanson@users.noreply.github.com> Date: Tue, 23 Sep 2025 22:14:41 -0500 Subject: [PATCH] Update dependencies in requirements.txt and refactor file handling in app initialization and utility functions to use context managers for better resource management. Adjust filter logic to utilize 'string' instead of 'text' for BeautifulSoup queries, enhancing compatibility with future versions. --- app/__init__.py | 51 ++++++++-------- app/filter.py | 18 +++--- app/models/config.py | 7 +-- app/request.py | 20 ++++--- app/routes.py | 4 +- app/services/http_client.py | 114 +++++++++++++++++++++++------------- app/utils/bangs.py | 6 +- app/utils/misc.py | 3 +- app/utils/results.py | 7 ++- requirements.txt | 46 +++++++-------- 10 files changed, 157 insertions(+), 119 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 772d6d3..34dad77 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -53,24 +53,18 @@ app.config['BUILD_FOLDER'] = os.path.join( app.config['STATIC_FOLDER'], 'build') app.config['CACHE_BUSTING_MAP'] = {} app.config['BUNDLE_STATIC'] = read_config_bool('WHOOGLE_BUNDLE_STATIC') -app.config['LANGUAGES'] = json.load(open( - os.path.join(app.config['STATIC_FOLDER'], 'settings/languages.json'), - encoding='utf-8')) -app.config['COUNTRIES'] = json.load(open( - os.path.join(app.config['STATIC_FOLDER'], 'settings/countries.json'), - encoding='utf-8')) -app.config['TIME_PERIODS'] = json.load(open( - os.path.join(app.config['STATIC_FOLDER'], 'settings/time_periods.json'), - encoding='utf-8')) -app.config['TRANSLATIONS'] = json.load(open( - os.path.join(app.config['STATIC_FOLDER'], 'settings/translations.json'), - encoding='utf-8')) -app.config['THEMES'] = json.load(open( - os.path.join(app.config['STATIC_FOLDER'], 'settings/themes.json'), - encoding='utf-8')) -app.config['HEADER_TABS'] = json.load(open( - os.path.join(app.config['STATIC_FOLDER'], 'settings/header_tabs.json'), - encoding='utf-8')) +with open(os.path.join(app.config['STATIC_FOLDER'], 'settings/languages.json'), 'r', encoding='utf-8') as f: + app.config['LANGUAGES'] = json.load(f) +with open(os.path.join(app.config['STATIC_FOLDER'], 'settings/countries.json'), 'r', encoding='utf-8') as f: + app.config['COUNTRIES'] = json.load(f) +with open(os.path.join(app.config['STATIC_FOLDER'], 'settings/time_periods.json'), 'r', encoding='utf-8') as f: + app.config['TIME_PERIODS'] = json.load(f) +with open(os.path.join(app.config['STATIC_FOLDER'], 'settings/translations.json'), 'r', encoding='utf-8') as f: + app.config['TRANSLATIONS'] = json.load(f) +with open(os.path.join(app.config['STATIC_FOLDER'], 'settings/themes.json'), 'r', encoding='utf-8') as f: + app.config['THEMES'] = json.load(f) +with open(os.path.join(app.config['STATIC_FOLDER'], 'settings/header_tabs.json'), 'r', encoding='utf-8') as f: + app.config['HEADER_TABS'] = json.load(f) app.config['CONFIG_PATH'] = os.getenv( 'CONFIG_VOLUME', os.path.join(app.config['STATIC_FOLDER'], 'config')) @@ -117,14 +111,14 @@ if not os.path.exists(app.config['BUILD_FOLDER']): app_key_path = os.path.join(app.config['CONFIG_PATH'], 'whoogle.key') if os.path.exists(app_key_path): try: - app.config['SECRET_KEY'] = open(app_key_path, 'r').read() + with open(app_key_path, 'r', encoding='utf-8') as f: + app.config['SECRET_KEY'] = f.read() except PermissionError: app.config['SECRET_KEY'] = str(b64encode(os.urandom(32))) else: app.config['SECRET_KEY'] = str(b64encode(os.urandom(32))) - with open(app_key_path, 'w') as key_file: + with open(app_key_path, 'w', encoding='utf-8') as key_file: key_file.write(app.config['SECRET_KEY']) - key_file.close() app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(days=365) # NOTE: SESSION_COOKIE_SAMESITE must be set to 'lax' to allow the user's @@ -160,7 +154,8 @@ app.config['CSP'] = 'default-src \'none\';' \ generating_bangs = False if not os.path.exists(app.config['BANG_FILE']): generating_bangs = True - json.dump({}, open(app.config['BANG_FILE'], 'w')) + with open(app.config['BANG_FILE'], 'w', encoding='utf-8') as f: + json.dump({}, f) bangs_thread = threading.Thread( target=gen_bangs_json, args=(app.config['BANG_FILE'],)) @@ -199,13 +194,15 @@ if app.config['BUNDLE_STATIC']: if name.endswith('-theme.css'): continue try: - css_parts.append(open(os.path.join(css_dir, name), 'r', encoding='utf-8').read()) + with open(os.path.join(css_dir, name), 'r', encoding='utf-8') as f: + css_parts.append(f.read()) except Exception: pass css_bundle = '\n'.join(css_parts) if css_bundle: css_tmp = os.path.join(app.config['BUILD_FOLDER'], 'app.css') - open(css_tmp, 'w', encoding='utf-8').write(css_bundle) + with open(css_tmp, 'w', encoding='utf-8') as f: + f.write(css_bundle) css_hashed = gen_file_hash(app.config['BUILD_FOLDER'], 'app.css') os.replace(css_tmp, os.path.join(app.config['BUILD_FOLDER'], css_hashed)) map_path = os.path.join('app/static/build', css_hashed) @@ -218,13 +215,15 @@ if app.config['BUNDLE_STATIC']: if not name.endswith('.js'): continue try: - js_parts.append(open(os.path.join(js_dir, name), 'r', encoding='utf-8').read()) + with open(os.path.join(js_dir, name), 'r', encoding='utf-8') as f: + js_parts.append(f.read()) except Exception: pass js_bundle = '\n;'.join(js_parts) if js_bundle: js_tmp = os.path.join(app.config['BUILD_FOLDER'], 'app.js') - open(js_tmp, 'w', encoding='utf-8').write(js_bundle) + with open(js_tmp, 'w', encoding='utf-8') as f: + f.write(js_bundle) js_hashed = gen_file_hash(app.config['BUILD_FOLDER'], 'app.js') os.replace(js_tmp, os.path.join(app.config['BUILD_FOLDER'], js_hashed)) map_path = os.path.join('app/static/build', js_hashed) diff --git a/app/filter.py b/app/filter.py index 8c40dbe..f079a64 100644 --- a/app/filter.py +++ b/app/filter.py @@ -219,7 +219,7 @@ class Filter: return for d in div.find_all('div', recursive=True): - d_text = d.find(text=True, recursive=False) + d_text = d.find(string=True, recursive=False) # Ensure we're working with tags that contain text content if not d_text or not d.string: @@ -295,7 +295,7 @@ class Filter: return search_string = ' '.join(['-site:' + _ for _ in self.config.block.split(',')]) - selected = soup.body.findAll(text=re.compile(search_string)) + selected = soup.body.find_all(string=re.compile(search_string)) for result in selected: result.string.replace_with(result.string.replace( @@ -362,11 +362,11 @@ class Filter: def pull_child_divs(result_div: BeautifulSoup): try: - return result_div.findChildren( - 'div', recursive=False - )[0].findChildren( - 'div', recursive=False) - except IndexError: + top_level_divs = result_div.find_all('div', recursive=False) + if not top_level_divs: + return [] + return top_level_divs[0].find_all('div', recursive=False) + except Exception: return [] if not self.main_divs: @@ -657,7 +657,7 @@ class Filter: prefix_pattern = re.compile(r'^(?:https?:\/\/)?(?:(?:www|mobile|m)\.)?') # 1) Replace bare domain divs (single token) once, avoiding duplicates - for div in self.soup.find_all('div', text=sites_pattern): + for div in self.soup.find_all('div', string=sites_pattern): if not div or not div.string: continue if len(div.string.split(' ')) != 1: @@ -679,7 +679,7 @@ class Filter: link['href'] = get_site_alt(link['href']) # Find a description text node matching a known site - desc_nodes = link.find_all(text=sites_pattern) + desc_nodes = link.find_all(string=sites_pattern) if not desc_nodes: continue desc_node = desc_nodes[0] diff --git a/app/models/config.py b/app/models/config.py index 2e73f6d..08d0e63 100644 --- a/app/models/config.py +++ b/app/models/config.py @@ -131,10 +131,9 @@ class Config: Returns: str -- the new style """ - style_sheet = cssutils.parseString( - open(os.path.join(current_app.config['STATIC_FOLDER'], - 'css/variables.css')).read() - ) + vars_path = os.path.join(current_app.config['STATIC_FOLDER'], 'css/variables.css') + with open(vars_path, 'r', encoding='utf-8') as f: + style_sheet = cssutils.parseString(f.read()) modified_sheet = cssutils.parseString(self.style_modified) for rule in modified_sheet: diff --git a/app/request.py b/app/request.py index 5f40bbc..734ea95 100644 --- a/app/request.py +++ b/app/request.py @@ -205,9 +205,10 @@ class Request: def __init__(self, normal_ua, root_path, config: Config, http_client=None): self.search_url = 'https://www.google.com/search?gbv=1&num=' + str( os.getenv('WHOOGLE_RESULTS_PER_PAGE', 10)) + '&q=' - # Send heartbeat to Tor, used in determining if the user can or cannot - # enable Tor for future requests - send_tor_signal(Signal.HEARTBEAT) + # Optionally send heartbeat to Tor to determine availability + # Only when Tor is enabled in config to avoid unnecessary socket usage + if config.tor: + send_tor_signal(Signal.HEARTBEAT) self.language = config.lang_search if config.lang_search else '' self.country = config.country if config.country else '' @@ -334,10 +335,12 @@ class Request: # view is suppressed correctly now = datetime.now() - cookies = { - 'CONSENT': 'PENDING+987', - 'SOCS': 'CAESHAgBEhIaAB', - } + consent_cookie = 'CONSENT=PENDING+987; SOCS=CAESHAgBEhIaAB' + # Prefer header-based cookies to avoid httpx per-request cookies deprecation + if 'Cookie' in headers: + headers['Cookie'] += '; ' + consent_cookie + else: + headers['Cookie'] = consent_cookie # Validate Tor conn and request new identity if the last one failed if self.tor and not send_tor_signal( @@ -368,8 +371,7 @@ class Request: try: response = self.http_client.get( (base_url or self.search_url) + query, - headers=headers, - cookies=cookies) + headers=headers) except httpx.HTTPError as e: raise diff --git a/app/routes.py b/app/routes.py index 258abb9..24beea3 100644 --- a/app/routes.py +++ b/app/routes.py @@ -682,7 +682,7 @@ def internal_error(e): fallback_engine = os.environ.get('WHOOGLE_FALLBACK_ENGINE_URL', '') if (fallback_engine): - return redirect(fallback_engine + query) + return redirect(fallback_engine + (query or '')) localization_lang = g.user_config.get_localization_lang() translation = app.config['TRANSLATIONS'][localization_lang] @@ -692,7 +692,7 @@ def internal_error(e): translation=translation, farside='https://farside.link', config=g.user_config, - query=urlparse.unquote(query), + query=urlparse.unquote(query or ''), params=g.user_config.to_params(keys=['preferences'])), 500 diff --git a/app/services/http_client.py b/app/services/http_client.py index 4f9730b..9bb1184 100644 --- a/app/services/http_client.py +++ b/app/services/http_client.py @@ -4,6 +4,8 @@ from typing import Any, Dict, Optional, Tuple import httpx from cachetools import TTLCache +import ssl +import os class HttpxClient: @@ -25,38 +27,81 @@ class HttpxClient: follow_redirects=True) # Prefer future-proof mounts when proxies are provided; fall back to proxies= self._proxies = proxies or {} + self._http2 = http2 + + # Determine verify behavior and initialize client with fallbacks + self._verify = self._determine_verify_setting() + try: + self._client = self._build_client(client_kwargs, self._verify) + except ssl.SSLError: + # Fallback to system trust store + try: + system_ctx = ssl.create_default_context() + self._client = self._build_client(client_kwargs, system_ctx) + self._verify = system_ctx + except ssl.SSLError: + insecure_fallback = os.environ.get('WHOOGLE_INSECURE_FALLBACK', '0').lower() in ('1', 'true', 't', 'yes', 'y') + if insecure_fallback: + self._client = self._build_client(client_kwargs, False) + self._verify = False + else: + raise + self._timeout_seconds = timeout_seconds + self._cache = TTLCache(maxsize=cache_maxsize, ttl=cache_ttl_seconds) + self._cache_lock = threading.Lock() + + def _determine_verify_setting(self): + """Determine SSL verification setting from environment. + + Honors: + - WHOOGLE_CA_BUNDLE: path to CA bundle file + - WHOOGLE_SSL_VERIFY: '0' to disable verification + - WHOOGLE_SSL_BACKEND: 'system' to prefer system trust store + """ + ca_bundle = os.environ.get('WHOOGLE_CA_BUNDLE', '').strip() + if ca_bundle: + return ca_bundle + + verify_env = os.environ.get('WHOOGLE_SSL_VERIFY', '1').lower() + if verify_env in ('0', 'false', 'no', 'n'): + return False + + backend = os.environ.get('WHOOGLE_SSL_BACKEND', '').lower() + if backend == 'system': + return ssl.create_default_context() + + return True + + def _build_client(self, client_kwargs: Dict[str, Any], verify: Any) -> httpx.Client: + """Construct httpx.Client with proxies and provided verify setting.""" + kwargs = dict(client_kwargs) + kwargs['verify'] = verify if self._proxies: - # If both schemes map to the same proxy, try the newer proxy= API first proxy_values = list(self._proxies.values()) single_proxy = proxy_values[0] if proxy_values and all(v == proxy_values[0] for v in proxy_values) else None if single_proxy: try: - self._client = httpx.Client(proxy=single_proxy, **client_kwargs) + return httpx.Client(proxy=single_proxy, **kwargs) except TypeError: - # Older httpx that doesn't support proxy=; try proxies= try: - self._client = httpx.Client(proxies=self._proxies, **client_kwargs) + return httpx.Client(proxies=self._proxies, **kwargs) except TypeError: mounts: Dict[str, httpx.Proxy] = {} for scheme_key, url in self._proxies.items(): prefix = f"{scheme_key}://" mounts[prefix] = httpx.Proxy(url) - self._client = httpx.Client(mounts=mounts, **client_kwargs) + return httpx.Client(mounts=mounts, **kwargs) else: - # Distinct proxies per scheme; use mounts fallback if needed try: - self._client = httpx.Client(proxies=self._proxies, **client_kwargs) + return httpx.Client(proxies=self._proxies, **kwargs) except TypeError: mounts: Dict[str, httpx.Proxy] = {} for scheme_key, url in self._proxies.items(): prefix = f"{scheme_key}://" mounts[prefix] = httpx.Proxy(url) - self._client = httpx.Client(mounts=mounts, **client_kwargs) + return httpx.Client(mounts=mounts, **kwargs) else: - self._client = httpx.Client(**client_kwargs) - self._timeout_seconds = timeout_seconds - self._cache = TTLCache(maxsize=cache_maxsize, ttl=cache_ttl_seconds) - self._cache_lock = threading.Lock() + return httpx.Client(**kwargs) @property def proxies(self) -> Dict[str, str]: @@ -119,34 +164,23 @@ class HttpxClient: # Recreate with same configuration client_kwargs = dict(timeout=self._timeout_seconds, - follow_redirects=True) - - if self._proxies: - proxy_values = list(self._proxies.values()) - single_proxy = proxy_values[0] if proxy_values and all(v == proxy_values[0] for v in proxy_values) else None - if single_proxy: - try: - self._client = httpx.Client(proxy=single_proxy, **client_kwargs) - except TypeError: - try: - self._client = httpx.Client(proxies=self._proxies, **client_kwargs) - except TypeError: - mounts: Dict[str, httpx.Proxy] = {} - for scheme_key, url in self._proxies.items(): - prefix = f"{scheme_key}://" - mounts[prefix] = httpx.Proxy(url) - self._client = httpx.Client(mounts=mounts, **client_kwargs) - else: - try: - self._client = httpx.Client(proxies=self._proxies, **client_kwargs) - except TypeError: - mounts: Dict[str, httpx.Proxy] = {} - for scheme_key, url in self._proxies.items(): - prefix = f"{scheme_key}://" - mounts[prefix] = httpx.Proxy(url) - self._client = httpx.Client(mounts=mounts, **client_kwargs) - else: - self._client = httpx.Client(**client_kwargs) + follow_redirects=True, + http2=self._http2) + + try: + self._client = self._build_client(client_kwargs, self._verify) + except ssl.SSLError: + try: + system_ctx = ssl.create_default_context() + self._client = self._build_client(client_kwargs, system_ctx) + self._verify = system_ctx + except ssl.SSLError: + insecure_fallback = os.environ.get('WHOOGLE_INSECURE_FALLBACK', '0').lower() in ('1', 'true', 't', 'yes', 'y') + if insecure_fallback: + self._client = self._build_client(client_kwargs, False) + self._verify = False + else: + raise def close(self) -> None: self._client.close() diff --git a/app/utils/bangs.py b/app/utils/bangs.py index de9f109..52161fa 100644 --- a/app/utils/bangs.py +++ b/app/utils/bangs.py @@ -43,7 +43,8 @@ def load_all_bangs(ddg_bangs_file: str, ddg_bangs: dict = {}): for i, bang_file in enumerate(bang_files): try: - bangs |= json.load(open(bang_file)) + with open(bang_file, 'r', encoding='utf-8') as f: + bangs |= json.load(f) except json.decoder.JSONDecodeError: # Ignore decoding error only for the ddg bangs file, since this can # occur if file is still being written @@ -80,7 +81,8 @@ def gen_bangs_json(bangs_file: str) -> None: 'suggestion': bang_command + ' (' + row['s'] + ')' } - json.dump(bangs_data, open(bangs_file, 'w')) + with open(bangs_file, 'w', encoding='utf-8') as f: + json.dump(bangs_data, f) print('* Finished creating ddg bangs json') load_all_bangs(bangs_file, bangs_data) diff --git a/app/utils/misc.py b/app/utils/misc.py index 20a2640..d4e49d7 100644 --- a/app/utils/misc.py +++ b/app/utils/misc.py @@ -48,7 +48,8 @@ def fetch_favicon(url: str) -> bytes: def gen_file_hash(path: str, static_file: str) -> str: - file_contents = open(os.path.join(path, static_file), 'rb').read() + with open(os.path.join(path, static_file), 'rb') as f: + file_contents = f.read() file_hash = hashlib.md5(file_contents).hexdigest()[:8] filename_split = os.path.splitext(static_file) diff --git a/app/utils/results.py b/app/utils/results.py index bc7c910..d75d328 100644 --- a/app/utils/results.py +++ b/app/utils/results.py @@ -1,7 +1,8 @@ from app.models.config import Config from app.models.endpoint import Endpoint from app.utils.misc import list_to_dict -from bs4 import BeautifulSoup, NavigableString +from bs4 import BeautifulSoup, NavigableString, MarkupResemblesLocatorWarning +import warnings import copy from flask import current_app import html @@ -9,7 +10,7 @@ import os import urllib.parse as urlparse from urllib.parse import parse_qs import re -import warnings +warnings.filterwarnings('ignore', category=MarkupResemblesLocatorWarning) SKIP_ARGS = ['ref_src', 'utm'] SKIP_PREFIX = ['//www.', '//mobile.', '//m.'] @@ -114,7 +115,7 @@ def bold_search_terms(response: str, query: str) -> BeautifulSoup: for word in re.split(r'\s+(?=[^"]*(?:"[^"]*"[^"]*)*$)', query): word = re.sub(r'[@_!#$%^&*()<>?/\|}{~:]+', '', word) target = response.find_all( - text=re.compile(r'' + re.escape(word), re.I)) + string=re.compile(r'' + re.escape(word), re.I)) for nav_str in target: replace_any_case(nav_str, word) diff --git a/requirements.txt b/requirements.txt index 45865c3..f78ce8d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,36 +1,36 @@ -attrs==22.2.0 -beautifulsoup4==4.11.2 -brotli==1.0.9 -certifi==2024.7.4 -cffi==1.17.1 -click==8.1.3 +attrs==25.3.0 +beautifulsoup4==4.13.5 +brotli==1.1.0 +certifi==2025.8.3 +cffi==2.0.0 +click==8.3.0 cryptography==3.3.2; platform_machine == 'armv7l' -cryptography==45.0.7; platform_machine != 'armv7l' -cssutils==2.7.0 +cryptography==46.0.1; platform_machine != 'armv7l' +cssutils==2.11.1 defusedxml==0.7.1 Flask==2.3.2 -idna==3.7 +idna==3.10 itsdangerous==2.1.2 Jinja2==3.1.6 -MarkupSafe==2.1.2 -more-itertools==9.0.0 -packaging==23.0 -pluggy==1.0.0 -pycodestyle==2.10.0 +MarkupSafe==3.0.2 +more-itertools==10.8.0 +packaging==25.0 +pluggy==1.6.0 +pycodestyle==2.14.0 pycparser==2.22 pyOpenSSL==19.1.0; platform_machine == 'armv7l' pyOpenSSL==25.3.0; platform_machine != 'armv7l' -pyparsing==3.0.9 +pyparsing==3.2.5 pytest==7.2.1 -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 httpx[http2,socks]==0.28.1 -cachetools==5.5.0 -soupsieve==2.4 -stem==1.8.1 +cachetools==6.2.0 +soupsieve==2.8 +stem==1.8.2 httpcore>=1.0.9 h11>=0.16.0 -validators==0.22.0 -waitress==3.0.1 -wcwidth==0.2.6 +validators==0.35.0 +waitress==3.0.2 +wcwidth==0.2.14 Werkzeug==3.0.6 -python-dotenv==0.21.1 +python-dotenv==1.1.1