Update dependencies in requirements.txt and refactor file handling in app initialization and utility functions to use context managers for better resource management. Adjust filter logic to utilize 'string' instead of 'text' for BeautifulSoup queries, enhancing compatibility with future versions.

This commit is contained in:
Don-Swanson 2025-09-23 22:14:41 -05:00
parent ffdeeb5f44
commit be83605c77
No known key found for this signature in database
GPG key ID: C6A6ACD574A005E5
10 changed files with 157 additions and 119 deletions

View file

@ -53,24 +53,18 @@ app.config['BUILD_FOLDER'] = os.path.join(
app.config['STATIC_FOLDER'], 'build')
app.config['CACHE_BUSTING_MAP'] = {}
app.config['BUNDLE_STATIC'] = read_config_bool('WHOOGLE_BUNDLE_STATIC')
app.config['LANGUAGES'] = json.load(open(
os.path.join(app.config['STATIC_FOLDER'], 'settings/languages.json'),
encoding='utf-8'))
app.config['COUNTRIES'] = json.load(open(
os.path.join(app.config['STATIC_FOLDER'], 'settings/countries.json'),
encoding='utf-8'))
app.config['TIME_PERIODS'] = json.load(open(
os.path.join(app.config['STATIC_FOLDER'], 'settings/time_periods.json'),
encoding='utf-8'))
app.config['TRANSLATIONS'] = json.load(open(
os.path.join(app.config['STATIC_FOLDER'], 'settings/translations.json'),
encoding='utf-8'))
app.config['THEMES'] = json.load(open(
os.path.join(app.config['STATIC_FOLDER'], 'settings/themes.json'),
encoding='utf-8'))
app.config['HEADER_TABS'] = json.load(open(
os.path.join(app.config['STATIC_FOLDER'], 'settings/header_tabs.json'),
encoding='utf-8'))
with open(os.path.join(app.config['STATIC_FOLDER'], 'settings/languages.json'), 'r', encoding='utf-8') as f:
app.config['LANGUAGES'] = json.load(f)
with open(os.path.join(app.config['STATIC_FOLDER'], 'settings/countries.json'), 'r', encoding='utf-8') as f:
app.config['COUNTRIES'] = json.load(f)
with open(os.path.join(app.config['STATIC_FOLDER'], 'settings/time_periods.json'), 'r', encoding='utf-8') as f:
app.config['TIME_PERIODS'] = json.load(f)
with open(os.path.join(app.config['STATIC_FOLDER'], 'settings/translations.json'), 'r', encoding='utf-8') as f:
app.config['TRANSLATIONS'] = json.load(f)
with open(os.path.join(app.config['STATIC_FOLDER'], 'settings/themes.json'), 'r', encoding='utf-8') as f:
app.config['THEMES'] = json.load(f)
with open(os.path.join(app.config['STATIC_FOLDER'], 'settings/header_tabs.json'), 'r', encoding='utf-8') as f:
app.config['HEADER_TABS'] = json.load(f)
app.config['CONFIG_PATH'] = os.getenv(
'CONFIG_VOLUME',
os.path.join(app.config['STATIC_FOLDER'], 'config'))
@ -117,14 +111,14 @@ if not os.path.exists(app.config['BUILD_FOLDER']):
app_key_path = os.path.join(app.config['CONFIG_PATH'], 'whoogle.key')
if os.path.exists(app_key_path):
try:
app.config['SECRET_KEY'] = open(app_key_path, 'r').read()
with open(app_key_path, 'r', encoding='utf-8') as f:
app.config['SECRET_KEY'] = f.read()
except PermissionError:
app.config['SECRET_KEY'] = str(b64encode(os.urandom(32)))
else:
app.config['SECRET_KEY'] = str(b64encode(os.urandom(32)))
with open(app_key_path, 'w') as key_file:
with open(app_key_path, 'w', encoding='utf-8') as key_file:
key_file.write(app.config['SECRET_KEY'])
key_file.close()
app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(days=365)
# NOTE: SESSION_COOKIE_SAMESITE must be set to 'lax' to allow the user's
@ -160,7 +154,8 @@ app.config['CSP'] = 'default-src \'none\';' \
generating_bangs = False
if not os.path.exists(app.config['BANG_FILE']):
generating_bangs = True
json.dump({}, open(app.config['BANG_FILE'], 'w'))
with open(app.config['BANG_FILE'], 'w', encoding='utf-8') as f:
json.dump({}, f)
bangs_thread = threading.Thread(
target=gen_bangs_json,
args=(app.config['BANG_FILE'],))
@ -199,13 +194,15 @@ if app.config['BUNDLE_STATIC']:
if name.endswith('-theme.css'):
continue
try:
css_parts.append(open(os.path.join(css_dir, name), 'r', encoding='utf-8').read())
with open(os.path.join(css_dir, name), 'r', encoding='utf-8') as f:
css_parts.append(f.read())
except Exception:
pass
css_bundle = '\n'.join(css_parts)
if css_bundle:
css_tmp = os.path.join(app.config['BUILD_FOLDER'], 'app.css')
open(css_tmp, 'w', encoding='utf-8').write(css_bundle)
with open(css_tmp, 'w', encoding='utf-8') as f:
f.write(css_bundle)
css_hashed = gen_file_hash(app.config['BUILD_FOLDER'], 'app.css')
os.replace(css_tmp, os.path.join(app.config['BUILD_FOLDER'], css_hashed))
map_path = os.path.join('app/static/build', css_hashed)
@ -218,13 +215,15 @@ if app.config['BUNDLE_STATIC']:
if not name.endswith('.js'):
continue
try:
js_parts.append(open(os.path.join(js_dir, name), 'r', encoding='utf-8').read())
with open(os.path.join(js_dir, name), 'r', encoding='utf-8') as f:
js_parts.append(f.read())
except Exception:
pass
js_bundle = '\n;'.join(js_parts)
if js_bundle:
js_tmp = os.path.join(app.config['BUILD_FOLDER'], 'app.js')
open(js_tmp, 'w', encoding='utf-8').write(js_bundle)
with open(js_tmp, 'w', encoding='utf-8') as f:
f.write(js_bundle)
js_hashed = gen_file_hash(app.config['BUILD_FOLDER'], 'app.js')
os.replace(js_tmp, os.path.join(app.config['BUILD_FOLDER'], js_hashed))
map_path = os.path.join('app/static/build', js_hashed)

View file

@ -219,7 +219,7 @@ class Filter:
return
for d in div.find_all('div', recursive=True):
d_text = d.find(text=True, recursive=False)
d_text = d.find(string=True, recursive=False)
# Ensure we're working with tags that contain text content
if not d_text or not d.string:
@ -295,7 +295,7 @@ class Filter:
return
search_string = ' '.join(['-site:' +
_ for _ in self.config.block.split(',')])
selected = soup.body.findAll(text=re.compile(search_string))
selected = soup.body.find_all(string=re.compile(search_string))
for result in selected:
result.string.replace_with(result.string.replace(
@ -362,11 +362,11 @@ class Filter:
def pull_child_divs(result_div: BeautifulSoup):
try:
return result_div.findChildren(
'div', recursive=False
)[0].findChildren(
'div', recursive=False)
except IndexError:
top_level_divs = result_div.find_all('div', recursive=False)
if not top_level_divs:
return []
return top_level_divs[0].find_all('div', recursive=False)
except Exception:
return []
if not self.main_divs:
@ -657,7 +657,7 @@ class Filter:
prefix_pattern = re.compile(r'^(?:https?:\/\/)?(?:(?:www|mobile|m)\.)?')
# 1) Replace bare domain divs (single token) once, avoiding duplicates
for div in self.soup.find_all('div', text=sites_pattern):
for div in self.soup.find_all('div', string=sites_pattern):
if not div or not div.string:
continue
if len(div.string.split(' ')) != 1:
@ -679,7 +679,7 @@ class Filter:
link['href'] = get_site_alt(link['href'])
# Find a description text node matching a known site
desc_nodes = link.find_all(text=sites_pattern)
desc_nodes = link.find_all(string=sites_pattern)
if not desc_nodes:
continue
desc_node = desc_nodes[0]

View file

@ -131,10 +131,9 @@ class Config:
Returns:
str -- the new style
"""
style_sheet = cssutils.parseString(
open(os.path.join(current_app.config['STATIC_FOLDER'],
'css/variables.css')).read()
)
vars_path = os.path.join(current_app.config['STATIC_FOLDER'], 'css/variables.css')
with open(vars_path, 'r', encoding='utf-8') as f:
style_sheet = cssutils.parseString(f.read())
modified_sheet = cssutils.parseString(self.style_modified)
for rule in modified_sheet:

View file

@ -205,9 +205,10 @@ class Request:
def __init__(self, normal_ua, root_path, config: Config, http_client=None):
self.search_url = 'https://www.google.com/search?gbv=1&num=' + str(
os.getenv('WHOOGLE_RESULTS_PER_PAGE', 10)) + '&q='
# Send heartbeat to Tor, used in determining if the user can or cannot
# enable Tor for future requests
send_tor_signal(Signal.HEARTBEAT)
# Optionally send heartbeat to Tor to determine availability
# Only when Tor is enabled in config to avoid unnecessary socket usage
if config.tor:
send_tor_signal(Signal.HEARTBEAT)
self.language = config.lang_search if config.lang_search else ''
self.country = config.country if config.country else ''
@ -334,10 +335,12 @@ class Request:
# view is suppressed correctly
now = datetime.now()
cookies = {
'CONSENT': 'PENDING+987',
'SOCS': 'CAESHAgBEhIaAB',
}
consent_cookie = 'CONSENT=PENDING+987; SOCS=CAESHAgBEhIaAB'
# Prefer header-based cookies to avoid httpx per-request cookies deprecation
if 'Cookie' in headers:
headers['Cookie'] += '; ' + consent_cookie
else:
headers['Cookie'] = consent_cookie
# Validate Tor conn and request new identity if the last one failed
if self.tor and not send_tor_signal(
@ -368,8 +371,7 @@ class Request:
try:
response = self.http_client.get(
(base_url or self.search_url) + query,
headers=headers,
cookies=cookies)
headers=headers)
except httpx.HTTPError as e:
raise

View file

@ -682,7 +682,7 @@ def internal_error(e):
fallback_engine = os.environ.get('WHOOGLE_FALLBACK_ENGINE_URL', '')
if (fallback_engine):
return redirect(fallback_engine + query)
return redirect(fallback_engine + (query or ''))
localization_lang = g.user_config.get_localization_lang()
translation = app.config['TRANSLATIONS'][localization_lang]
@ -692,7 +692,7 @@ def internal_error(e):
translation=translation,
farside='https://farside.link',
config=g.user_config,
query=urlparse.unquote(query),
query=urlparse.unquote(query or ''),
params=g.user_config.to_params(keys=['preferences'])), 500

View file

@ -4,6 +4,8 @@ from typing import Any, Dict, Optional, Tuple
import httpx
from cachetools import TTLCache
import ssl
import os
class HttpxClient:
@ -25,38 +27,81 @@ class HttpxClient:
follow_redirects=True)
# Prefer future-proof mounts when proxies are provided; fall back to proxies=
self._proxies = proxies or {}
self._http2 = http2
# Determine verify behavior and initialize client with fallbacks
self._verify = self._determine_verify_setting()
try:
self._client = self._build_client(client_kwargs, self._verify)
except ssl.SSLError:
# Fallback to system trust store
try:
system_ctx = ssl.create_default_context()
self._client = self._build_client(client_kwargs, system_ctx)
self._verify = system_ctx
except ssl.SSLError:
insecure_fallback = os.environ.get('WHOOGLE_INSECURE_FALLBACK', '0').lower() in ('1', 'true', 't', 'yes', 'y')
if insecure_fallback:
self._client = self._build_client(client_kwargs, False)
self._verify = False
else:
raise
self._timeout_seconds = timeout_seconds
self._cache = TTLCache(maxsize=cache_maxsize, ttl=cache_ttl_seconds)
self._cache_lock = threading.Lock()
def _determine_verify_setting(self):
"""Determine SSL verification setting from environment.
Honors:
- WHOOGLE_CA_BUNDLE: path to CA bundle file
- WHOOGLE_SSL_VERIFY: '0' to disable verification
- WHOOGLE_SSL_BACKEND: 'system' to prefer system trust store
"""
ca_bundle = os.environ.get('WHOOGLE_CA_BUNDLE', '').strip()
if ca_bundle:
return ca_bundle
verify_env = os.environ.get('WHOOGLE_SSL_VERIFY', '1').lower()
if verify_env in ('0', 'false', 'no', 'n'):
return False
backend = os.environ.get('WHOOGLE_SSL_BACKEND', '').lower()
if backend == 'system':
return ssl.create_default_context()
return True
def _build_client(self, client_kwargs: Dict[str, Any], verify: Any) -> httpx.Client:
"""Construct httpx.Client with proxies and provided verify setting."""
kwargs = dict(client_kwargs)
kwargs['verify'] = verify
if self._proxies:
# If both schemes map to the same proxy, try the newer proxy= API first
proxy_values = list(self._proxies.values())
single_proxy = proxy_values[0] if proxy_values and all(v == proxy_values[0] for v in proxy_values) else None
if single_proxy:
try:
self._client = httpx.Client(proxy=single_proxy, **client_kwargs)
return httpx.Client(proxy=single_proxy, **kwargs)
except TypeError:
# Older httpx that doesn't support proxy=; try proxies=
try:
self._client = httpx.Client(proxies=self._proxies, **client_kwargs)
return httpx.Client(proxies=self._proxies, **kwargs)
except TypeError:
mounts: Dict[str, httpx.Proxy] = {}
for scheme_key, url in self._proxies.items():
prefix = f"{scheme_key}://"
mounts[prefix] = httpx.Proxy(url)
self._client = httpx.Client(mounts=mounts, **client_kwargs)
return httpx.Client(mounts=mounts, **kwargs)
else:
# Distinct proxies per scheme; use mounts fallback if needed
try:
self._client = httpx.Client(proxies=self._proxies, **client_kwargs)
return httpx.Client(proxies=self._proxies, **kwargs)
except TypeError:
mounts: Dict[str, httpx.Proxy] = {}
for scheme_key, url in self._proxies.items():
prefix = f"{scheme_key}://"
mounts[prefix] = httpx.Proxy(url)
self._client = httpx.Client(mounts=mounts, **client_kwargs)
return httpx.Client(mounts=mounts, **kwargs)
else:
self._client = httpx.Client(**client_kwargs)
self._timeout_seconds = timeout_seconds
self._cache = TTLCache(maxsize=cache_maxsize, ttl=cache_ttl_seconds)
self._cache_lock = threading.Lock()
return httpx.Client(**kwargs)
@property
def proxies(self) -> Dict[str, str]:
@ -119,34 +164,23 @@ class HttpxClient:
# Recreate with same configuration
client_kwargs = dict(timeout=self._timeout_seconds,
follow_redirects=True)
if self._proxies:
proxy_values = list(self._proxies.values())
single_proxy = proxy_values[0] if proxy_values and all(v == proxy_values[0] for v in proxy_values) else None
if single_proxy:
try:
self._client = httpx.Client(proxy=single_proxy, **client_kwargs)
except TypeError:
try:
self._client = httpx.Client(proxies=self._proxies, **client_kwargs)
except TypeError:
mounts: Dict[str, httpx.Proxy] = {}
for scheme_key, url in self._proxies.items():
prefix = f"{scheme_key}://"
mounts[prefix] = httpx.Proxy(url)
self._client = httpx.Client(mounts=mounts, **client_kwargs)
else:
try:
self._client = httpx.Client(proxies=self._proxies, **client_kwargs)
except TypeError:
mounts: Dict[str, httpx.Proxy] = {}
for scheme_key, url in self._proxies.items():
prefix = f"{scheme_key}://"
mounts[prefix] = httpx.Proxy(url)
self._client = httpx.Client(mounts=mounts, **client_kwargs)
else:
self._client = httpx.Client(**client_kwargs)
follow_redirects=True,
http2=self._http2)
try:
self._client = self._build_client(client_kwargs, self._verify)
except ssl.SSLError:
try:
system_ctx = ssl.create_default_context()
self._client = self._build_client(client_kwargs, system_ctx)
self._verify = system_ctx
except ssl.SSLError:
insecure_fallback = os.environ.get('WHOOGLE_INSECURE_FALLBACK', '0').lower() in ('1', 'true', 't', 'yes', 'y')
if insecure_fallback:
self._client = self._build_client(client_kwargs, False)
self._verify = False
else:
raise
def close(self) -> None:
self._client.close()

View file

@ -43,7 +43,8 @@ def load_all_bangs(ddg_bangs_file: str, ddg_bangs: dict = {}):
for i, bang_file in enumerate(bang_files):
try:
bangs |= json.load(open(bang_file))
with open(bang_file, 'r', encoding='utf-8') as f:
bangs |= json.load(f)
except json.decoder.JSONDecodeError:
# Ignore decoding error only for the ddg bangs file, since this can
# occur if file is still being written
@ -80,7 +81,8 @@ def gen_bangs_json(bangs_file: str) -> None:
'suggestion': bang_command + ' (' + row['s'] + ')'
}
json.dump(bangs_data, open(bangs_file, 'w'))
with open(bangs_file, 'w', encoding='utf-8') as f:
json.dump(bangs_data, f)
print('* Finished creating ddg bangs json')
load_all_bangs(bangs_file, bangs_data)

View file

@ -48,7 +48,8 @@ def fetch_favicon(url: str) -> bytes:
def gen_file_hash(path: str, static_file: str) -> str:
file_contents = open(os.path.join(path, static_file), 'rb').read()
with open(os.path.join(path, static_file), 'rb') as f:
file_contents = f.read()
file_hash = hashlib.md5(file_contents).hexdigest()[:8]
filename_split = os.path.splitext(static_file)

View file

@ -1,7 +1,8 @@
from app.models.config import Config
from app.models.endpoint import Endpoint
from app.utils.misc import list_to_dict
from bs4 import BeautifulSoup, NavigableString
from bs4 import BeautifulSoup, NavigableString, MarkupResemblesLocatorWarning
import warnings
import copy
from flask import current_app
import html
@ -9,7 +10,7 @@ import os
import urllib.parse as urlparse
from urllib.parse import parse_qs
import re
import warnings
warnings.filterwarnings('ignore', category=MarkupResemblesLocatorWarning)
SKIP_ARGS = ['ref_src', 'utm']
SKIP_PREFIX = ['//www.', '//mobile.', '//m.']
@ -114,7 +115,7 @@ def bold_search_terms(response: str, query: str) -> BeautifulSoup:
for word in re.split(r'\s+(?=[^"]*(?:"[^"]*"[^"]*)*$)', query):
word = re.sub(r'[@_!#$%^&*()<>?/\|}{~:]+', '', word)
target = response.find_all(
text=re.compile(r'' + re.escape(word), re.I))
string=re.compile(r'' + re.escape(word), re.I))
for nav_str in target:
replace_any_case(nav_str, word)

View file

@ -1,36 +1,36 @@
attrs==22.2.0
beautifulsoup4==4.11.2
brotli==1.0.9
certifi==2024.7.4
cffi==1.17.1
click==8.1.3
attrs==25.3.0
beautifulsoup4==4.13.5
brotli==1.1.0
certifi==2025.8.3
cffi==2.0.0
click==8.3.0
cryptography==3.3.2; platform_machine == 'armv7l'
cryptography==45.0.7; platform_machine != 'armv7l'
cssutils==2.7.0
cryptography==46.0.1; platform_machine != 'armv7l'
cssutils==2.11.1
defusedxml==0.7.1
Flask==2.3.2
idna==3.7
idna==3.10
itsdangerous==2.1.2
Jinja2==3.1.6
MarkupSafe==2.1.2
more-itertools==9.0.0
packaging==23.0
pluggy==1.0.0
pycodestyle==2.10.0
MarkupSafe==3.0.2
more-itertools==10.8.0
packaging==25.0
pluggy==1.6.0
pycodestyle==2.14.0
pycparser==2.22
pyOpenSSL==19.1.0; platform_machine == 'armv7l'
pyOpenSSL==25.3.0; platform_machine != 'armv7l'
pyparsing==3.0.9
pyparsing==3.2.5
pytest==7.2.1
python-dateutil==2.8.2
python-dateutil==2.9.0.post0
httpx[http2,socks]==0.28.1
cachetools==5.5.0
soupsieve==2.4
stem==1.8.1
cachetools==6.2.0
soupsieve==2.8
stem==1.8.2
httpcore>=1.0.9
h11>=0.16.0
validators==0.22.0
waitress==3.0.1
wcwidth==0.2.6
validators==0.35.0
waitress==3.0.2
wcwidth==0.2.14
Werkzeug==3.0.6
python-dotenv==0.21.1
python-dotenv==1.1.1