From 6f6197064d6ef154cbc40c54e8e1eee73a277e28 Mon Sep 17 00:00:00 2001 From: Meinard Francisco <44516102+znarfm@users.noreply.github.com> Date: Mon, 1 Sep 2025 16:25:01 +0800 Subject: [PATCH] Refactor bookmark generator --- make_fmhy_bookmarks.py | 489 ++++++++++++++++++++++------------------- 1 file changed, 262 insertions(+), 227 deletions(-) diff --git a/make_fmhy_bookmarks.py b/make_fmhy_bookmarks.py index d410977..bb91784 100644 --- a/make_fmhy_bookmarks.py +++ b/make_fmhy_bookmarks.py @@ -1,269 +1,304 @@ +"""Generate FMHY bookmark HTML files from FMHY markdown sections.""" + +from __future__ import annotations + +import base64 +import logging +import re +from dataclasses import dataclass +from typing import Dict, List, Tuple + import requests -def addPretext(lines, sectionName, baseURL, subURL): +# Configure logging +logging.basicConfig(level=logging.INFO, format="[%(levelname)s] %(message)s") +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class Config: + """Configuration constants for the FMHY bookmark generator.""" + site_base_url: str = "https://fmhy.net/" + reddit_base_url: str = "https://www.reddit.com/r/FREEMEDIAHECKYEAH/wiki/" + base64_rentry_url: str = "https://rentry.co/FMHYBase64/raw" + github_raw_base: str = "https://raw.githubusercontent.com/fmhy/edit/refs/heads/main/docs/" + folder_name: str = "FMHY" + decode_base64: bool = True + + +@dataclass +class WikiSection: + """Represents a wiki section to be processed.""" + filename: str + icon: str + url_key: str + + +CONFIG = Config() + + +def parse_heading(line: str, sub_url: str) -> Tuple[str, str]: + """Parse heading line and return (subcategory, subsubcategory).""" + if sub_url != "storage": + if line.startswith("# ►"): + return line.replace("# ►", "").strip(), "/" + elif line.startswith("## ▷"): + return "", line.replace("## ▷", "").strip() + else: # storage section uses different heading levels + if line.startswith("## "): + return line.replace("## ", "").strip(), "/" + elif line.startswith("### "): + return "", line.replace("### ", "").strip() + return "", "" + + +def clean_category_name(category: str) -> str: + """Remove URLs from category names.""" + return "" if "http" in category else category + + +def add_hierarchy_prefix( + lines: List[str], section_name: str, sub_url: str +) -> List[str]: + """Add hierarchy prefix to content lines.""" modified_lines = [] - currMdSubheading = "" - currSubCat = "" - currSubSubCat = "" + curr_subcat = "" + curr_subsubcat = "" - #Remove from the lines any line that isnt a heading and doesnt contain the character `⭐` - #lines = [line for line in lines if line.startswith("#") or '⭐' in line] - - #Parse headings for line in lines: - if line.startswith("#"): #Title Lines - if not subURL=="storage": - if line.startswith("# ►"): - currMdSubheading = "#" + line.replace("# ►", "").strip().replace(" / ", "-").replace(" ", "-").lower() - currSubCat = line.replace("# ►", "").strip() - currSubSubCat = "/" - elif line.startswith("## ▷"): - if not subURL=="non-english": #Because non-eng section has multiple subsubcats with same names - currMdSubheading = "#" + line.replace("## ▷", "").strip().replace(" / ", "-").replace(" ", "-").lower() - currSubSubCat = line.replace("## ▷", "").strip() - elif subURL=="storage": - if line.startswith("## "): - currMdSubheading = "#" + line.replace("## ", "").strip().replace(" / ", "-").replace(" ", "-").lower() - currSubCat = line.replace("## ", "").strip() - currSubSubCat = "/" - elif line.startswith("### "): - currMdSubheading = "#" + line.replace("### ", "").strip().replace(" / ", "-").replace(" ", "-").lower() - currSubSubCat = line.replace("### ", "").strip() - - # Remove links from subcategory titles (because the screw the format) - if 'http' in currSubCat: currSubCat = '' - if 'http' in currSubSubCat: currSubSubCat = '' - - elif any(char.isalpha() for char in line): #If line has content - preText = f"{{\"{sectionName.replace(".md", "")}\", \"{currSubCat}\", \"{currSubSubCat}\"}}" - if line.startswith("* "): line = line[2:] - modified_lines.append(preText + line) + if line.startswith("#"): # Heading line + subcat, subsubcat = parse_heading(line, sub_url) + if subcat: + curr_subcat = clean_category_name(subcat) + if subsubcat: + curr_subsubcat = clean_category_name(subsubcat) + elif any(char.isalpha() for char in line): # Content line + prefix = f'{{"{ section_name.replace(".md", "") }", "{ curr_subcat }", "{ curr_subsubcat }"}}' + content = line[2:] if line.startswith("* ") else line + modified_lines.append(prefix + content) return modified_lines -#----------------base64 page processing------------ -import base64 -import re - -doBase64Decoding = True - -def fix_base64_string(encoded_string): +# Base64 processing functions +def fix_base64_padding(encoded_string: str) -> str: + """Fix base64 padding.""" missing_padding = len(encoded_string) % 4 - if missing_padding != 0: - encoded_string += '=' * (4 - missing_padding) + if missing_padding: + encoded_string += "=" * (4 - missing_padding) return encoded_string -def decode_base64_in_backticks(input_string): + +def decode_base64_content(input_string: str) -> str: + """Decode base64 content within backticks.""" + if not CONFIG.decode_base64: + return input_string + def base64_decode(match): - encoded_data = match.group(0)[1:-1] # Extract content within backticks - decoded_bytes = base64.b64decode( fix_base64_string(encoded_data) ) + encoded_data = match.group(0)[1:-1] # Remove backticks + decoded_bytes = base64.b64decode(fix_base64_padding(encoded_data)) return decoded_bytes.decode() - pattern = r"`[^`]+`" # Regex pattern to find substrings within backticks - decoded_string = re.sub(pattern, base64_decode, input_string) - return decoded_string + pattern = r"`[^`]+`" + return re.sub(pattern, base64_decode, input_string) -def remove_empty_lines(text): - lines = text.split('\n') # Split the text into lines - non_empty_lines = [line for line in lines if line.strip()] # Filter out empty lines - return '\n'.join(non_empty_lines) # Join non-empty lines back together -def extract_base64_sections(base64_page): - sections = base64_page.split("***") # Split the input string by "***" to get sections +def process_base64_sections(base64_page: str) -> List[str]: + """Process base64 page sections.""" + sections = base64_page.split("***") formatted_sections = [] + for section in sections: - formatted_section = remove_empty_lines( section.strip().replace("#### ", "").replace("\n\n", " - ").replace("\n", ", ") ) - if doBase64Decoding: formatted_section = decode_base64_in_backticks(formatted_section) - formatted_section = '[🔑Base64](https://rentry.co/FMHYBase64) ► ' + formatted_section + # Clean up section formatting + clean_section = (section.strip() + .replace("#### ", "") + .replace("\n\n", " - ") + .replace("\n", ", ")) + + # Remove empty lines + lines = [line for line in clean_section.split("\n") if line.strip()] + clean_section = "\n".join(lines) + + # Decode base64 if enabled + clean_section = decode_base64_content(clean_section) + + # Add base64 prefix + formatted_section = "[🔑Base64](https://rentry.co/FMHYBase64) ► " + clean_section formatted_sections.append(formatted_section) - lines = formatted_sections - return lines -#----------------base64 page processing------------ + + return formatted_sections - -def dlWikiChunk(fileName, icon, redditSubURL): - - #first, try to get the chunk locally +def download_wiki_content(filename: str) -> List[str]: + """Download and process wiki content.""" try: - #First, try to get it from the local file - print("Loading " + fileName + " from local file...") - with open(fileName.lower(), 'r') as f: - page = f.read() - print("Loaded.\n") - #if not available locally, download the chunk - except: - if not fileName=='base64.md': - print("Local file not found. Downloading " + fileName + " from Github...") - page = requests.get("https://raw.githubusercontent.com/fmhy/FMHYedit/main/docs/" + fileName.lower()).text - elif fileName=='base64.md': - print("Local file not found. Downloading rentry.co/FMHYBase64...") - page = requests.get("https://rentry.co/FMHYBase64/raw").text.replace("\r", "") - print("Downloaded") + with open(filename, "r", encoding="utf-8") as f: + content = f.read() + logger.info("Loaded %s locally", filename) + except FileNotFoundError: + if filename != "base64.md": + url = CONFIG.github_raw_base + filename + try: + resp = requests.get(url, timeout=30) + resp.raise_for_status() + content = resp.text + logger.info("Downloaded %s", filename) + except requests.HTTPError as e: + logger.error("Failed to fetch %s (%s). Skipping.", filename, e) + return [] + else: + logger.info("Downloading Base64 page from rentry...") + resp = requests.get(CONFIG.base64_rentry_url, timeout=30) + resp.raise_for_status() + content = resp.text.replace("\r", "") + logger.info("Downloaded base64 page") - #add a pretext - redditBaseURL = "https://www.reddit.com/r/FREEMEDIAHECKYEAH/wiki/" - siteBaseURL = "https://fmhy.net/" - if not fileName=='base64.md': - pagesDevSiteSubURL = fileName.replace(".md", "").lower() - subURL = pagesDevSiteSubURL - lines = page.split('\n') - lines = addPretext(lines, fileName, siteBaseURL, subURL) - elif fileName=='base64.md': - lines = extract_base64_sections(page) - - return lines - -def cleanLineForSearchMatchChecks(line): - siteBaseURL = "https://fmhy.net/" - redditBaseURL = "https://www.reddit.com/r/FREEMEDIAHECKYEAH/wiki/" - return line.replace(redditBaseURL, '/').replace(siteBaseURL, '/') - -def alternativeWikiIndexing(): - wikiChunks = [ - dlWikiChunk("VideoPiracyGuide.md", "📺", "video"), - dlWikiChunk("AI.md", "🤖", "ai"), - dlWikiChunk("Android-iOSGuide.md", "📱", "android"), - dlWikiChunk("AudioPiracyGuide.md", "🎵", "audio"), - dlWikiChunk("DownloadPiracyGuide.md", "💾", "download"), - dlWikiChunk("EDUPiracyGuide.md", "🧠", "edu"), - dlWikiChunk("GamingPiracyGuide.md", "🎮", "games"), - dlWikiChunk("AdblockVPNGuide.md", "📛", "adblock-vpn-privacy"), - dlWikiChunk("System-Tools.md", "💻", "system-tools"), - dlWikiChunk("File-Tools.md", "🗃️", "file-tools"), - dlWikiChunk("Internet-Tools.md", "🔗", "internet-tools"), - dlWikiChunk("Social-Media-Tools.md", "💬", "social-media"), - dlWikiChunk("Text-Tools.md", "📝", "text-tools"), - dlWikiChunk("Video-Tools.md", "📼", "video-tools"), - dlWikiChunk("MISCGuide.md", "📂", "misc"), - dlWikiChunk("ReadingPiracyGuide.md", "📗", "reading"), - dlWikiChunk("TorrentPiracyGuide.md", "🌀", "torrent"), - dlWikiChunk("img-tools.md", "📷", "img-tools"), - dlWikiChunk("gaming-tools.md", "👾", "gaming-tools"), - dlWikiChunk("LinuxGuide.md", "🐧🍏", "linux"), - dlWikiChunk("DEVTools.md", "🖥️", "dev-tools"), - dlWikiChunk("Non-English.md", "🌏", "non-eng"), - dlWikiChunk("STORAGE.md", "🗄️", "storage"), - #dlWikiChunk("base64.md", "🔑", "base64"), - dlWikiChunk("NSFWPiracy.md", "🌶", "https://saidit.net/s/freemediafuckyeah/wiki/index") - ] - return [item for sublist in wikiChunks for item in sublist] #Flatten a into a -#-------------------------------- + if filename != "base64.md": + sub_url = filename.replace(".md", "").lower() + return add_hierarchy_prefix(content.split("\n"), filename, sub_url) + else: + return process_base64_sections(content) -# Save the result of alternativeWikiIndexing to a .md file -# with open('wiki_adapted.md', 'w') as f: -# for line in alternativeWikiIndexing(): -# f.write(line + '\n') - -# Instead of saving it to a file, save it into a string variable -wiki_adapted_md = '\n'.join(alternativeWikiIndexing()) - -# Remove from the lines in wiki_adapted_md any line that doesnt contain the character `⭐` or '🌟' -wiki_adapted_starred_only_md = '\n'.join([line for line in wiki_adapted_md.split('\n') if '⭐' in line or '🌟' in line]) +# Wiki sections to process +WIKI_SECTIONS = [ + WikiSection("video.md", "📺", "video"), + WikiSection("ai.md", "🤖", "ai"), + WikiSection("mobile.md", "📱", "mobile"), + WikiSection("audio.md", "🎵", "audio"), + WikiSection("downloading.md", "💾", "downloading"), + WikiSection("educational.md", "🧠", "educational"), + WikiSection("gaming.md", "🎮", "gaming"), + WikiSection("privacy.md", "📛", "privacy"), + WikiSection("system-tools.md", "💻", "system-tools"), + WikiSection("file-tools.md", "🗃️", "file-tools"), + WikiSection("internet-tools.md", "🔗", "internet-tools"), + WikiSection("social-media-tools.md", "💬", "social-media-tools"), + WikiSection("text-tools.md", "📝", "text-tools"), + WikiSection("video-tools.md", "📼", "video-tools"), + WikiSection("misc.md", "📂", "misc"), + WikiSection("reading.md", "📗", "reading"), + WikiSection("torrenting.md", "🌀", "torrenting"), + WikiSection("image-tools.md", "📷", "image-tools"), + WikiSection("gaming-tools.md", "👾", "gaming-tools"), + WikiSection("linux-macos.md", "🐧🍏", "linux-macos"), + WikiSection("developer-tools.md", "🖥️", "developer-tools"), + WikiSection("non-english.md", "🌏", "non-english"), + WikiSection("storage.md", "🗄️", "storage"), + WikiSection("base64.md", "🔑", "base64"), + WikiSection("unsafe.md", "🌶", "unsafe"), +] +def collect_all_wiki_content() -> List[str]: + """Collect and process all wiki sections.""" + all_lines = [] + for section in WIKI_SECTIONS: + lines = download_wiki_content(section.filename) + all_lines.extend(lines) + return all_lines -import re -def markdown_to_html_bookmarks(input_md_text, output_file): - # Predefined folder name - folder_name = "FMHY" +def filter_starred_content(content: str) -> str: + """Filter content to only include starred items.""" + return "\n".join( + line for line in content.split("\n") if "⭐" in line or "🌟" in line + ) - # Read the input markdown file - #with open(input_file, 'r', encoding='utf-8') as f: - # markdown_content = f.read() - # Instead of reading from a file, read from a string variable - markdown_content = input_md_text - - # Regex pattern to extract URLs and titles from markdown - url_pattern = re.compile(r'\[([^\]]+)\]\((https?://[^\)]+)\)') - # Regex pattern to extract hierarchy levels +def parse_bookmark_line(line: str, starred_only: bool = False) -> Tuple[str, str, str, List[Tuple[str, str]]]: + """Parse a line to extract hierarchy and bookmarks.""" + url_pattern = re.compile(r"\[([^\]]+)\]\((https?://[^\)]+)\)") hierarchy_pattern = re.compile(r'^\{"([^"]+)", "([^"]+)", "([^"]+)"\}') + + hierarchy_match = hierarchy_pattern.match(line) + if not hierarchy_match: + return "", "", "", [] + + level1, level2, level3 = hierarchy_match.groups() + matches = url_pattern.findall(line) + + if starred_only: + matches = matches[:1] # Only first match for starred content + + # Extract description (text after last URL) + last_paren = line.rfind(")") + description = ( + line[last_paren + 1:].replace("**", "").strip() if last_paren != -1 else "" + ) + + if not description: + description = "- " + (level3 if level3 != "/" else level2 if level2 else level1) + + bookmarks = [(f"{title} {description}".strip(), url) for title, url in matches] + return level1, level2, level3, bookmarks - # Dictionary to hold bookmarks by hierarchy - bookmarks = {} - # Split the content by lines - lines = markdown_content.split('\n') +def generate_bookmark_html(bookmarks_dict: Dict, indent: int = 1) -> str: + """Generate HTML from bookmark dictionary.""" + html = "" + for key, value in bookmarks_dict.items(): + html += " " * indent + f"

{key}

\n" + html += " " * indent + "

\n" + + if isinstance(value, dict): + html += generate_bookmark_html(value, indent + 1) + else: + for title, url in value: + html += (" " * (indent + 1) + + f'

{title}\n') + + html += " " * indent + "

\n" + return html - # Parse each line - for line in lines: - # Find hierarchy levels - hierarchy_match = hierarchy_pattern.match(line) - if not hierarchy_match: + +def create_html_bookmarks(content: str, output_file: str, starred_only: bool = False) -> None: + """Create HTML bookmark file from processed content.""" + bookmarks: Dict[str, Dict[str, Dict[str, List[Tuple[str, str]]]]] = {} + + for line in content.split("\n"): + level1, level2, level3, bookmark_list = parse_bookmark_line(line, starred_only) + if not level1: # Skip lines that don't match hierarchy pattern continue + + # Initialize nested structure + bookmarks.setdefault(level1, {}).setdefault(level2, {}).setdefault(level3, []) + bookmarks[level1][level2][level3].extend(bookmark_list) - level1, level2, level3 = hierarchy_match.groups() + # Generate HTML + html_content = ( + '\n' + '\n' + 'Bookmarks\n' + '

Bookmarks

\n' + '

\n' + f'

{CONFIG.folder_name}

\n' + '

\n' + + generate_bookmark_html(bookmarks) + + '

\n' + '

\n' + ) - # Initialize nested dictionaries for hierarchy levels - if level1 not in bookmarks: - bookmarks[level1] = {} - if level2 not in bookmarks[level1]: - bookmarks[level1][level2] = {} - if level3 not in bookmarks[level1][level2]: - bookmarks[level1][level2][level3] = [] - - # Find all matches in the line for URLs - matches = url_pattern.findall(line) - - # If the input_md_text is wiki_adapted_starred_only_md, only add the first match of url_pattern in each line - if input_md_text == wiki_adapted_starred_only_md: - matches = matches[:1] - - # Extract the description (text after the last match) - last_match_end = line.rfind(')') - description = line[last_match_end+1:].replace('**', '').strip() if last_match_end != -1 else '' - - # When the description is empty, use as description the lowest hierachy level that is not empty - if not description: - description = '- ' + (level3 if level3 != '/' else level2 if level2 else level1) - - # Add matches to the appropriate hierarchy - for title, url in matches: - full_title = f"{title} {description}" if description else title - bookmarks[level1][level2][level3].append((full_title, url)) - - # Function to generate HTML from nested dictionary - def generate_html(bookmarks_dict, indent=1): - html = '' - for key, value in bookmarks_dict.items(): - html += ' ' * indent + f'

{key}

\n' - html += ' ' * indent + '

\n' - if isinstance(value, dict): - html += generate_html(value, indent + 1) - else: - for full_title, url in value: - html += ' ' * (indent + 1) + f'

{full_title}\n' - html += ' ' * indent + '

\n' - return html - - # HTML structure - html_content = ''' - -Bookmarks -

Bookmarks

-

-''' - # Add the main folder - html_content += f'

{folder_name}

\n' - html_content += '

\n' - - # Add bookmarks to HTML content - html_content += generate_html(bookmarks) - - html_content += '

\n' - html_content += '

\n' - - # Write the HTML content to the output file - with open(output_file, 'w', encoding='utf-8') as f: + with open(output_file, "w", encoding="utf-8") as f: f.write(html_content) + + logger.info("Created bookmark file: %s", output_file) - # Print success message - #print(f'Successfully created bookmarks in {output_file}') -# Example usage: -markdown_to_html_bookmarks(wiki_adapted_md, 'fmhy_in_bookmarks.html') -markdown_to_html_bookmarks(wiki_adapted_starred_only_md, 'fmhy_in_bookmarks_starred_only.html') +def main() -> None: + """Main execution function.""" + logger.info("Collecting wiki content...") + all_content = collect_all_wiki_content() + full_content = "\n".join(all_content) + + # Generate both bookmark files + create_html_bookmarks(full_content, "fmhy_in_bookmarks.html") + + starred_content = filter_starred_content(full_content) + create_html_bookmarks(starred_content, "fmhy_in_bookmarks_starred_only.html", starred_only=True) + + logger.info("Bookmark generation complete!") + + +if __name__ == "__main__": + main()