web_to_pdf.py

#!/usr/bin/env python3
"""
web-to-pdf: Scrape multi-page web tutorials and convert to a bookmarked A4 PDF.

Usage:
    python3 web_to_pdf.py <start_url> [options]

Options:
    --output PATH       Output PDF path (default: ./output.pdf)
    --title TITLE       PDF cover title (default: auto-detect from page)
    --no-font-fix       Skip SVG font replacement (for non-CJK sites)
    --delay SECONDS     Delay between page fetches (default: 0.5)
    --no-cover          Skip cover page
    --no-toc            Skip table of contents page
"""

import argparse
import asyncio
import os
import re
import sys
import time
from urllib.parse import urljoin, urlparse

import requests
from bs4 import BeautifulSoup, Comment

# ─── Configuration ───────────────────────────────────────────────────────────

CONTENT_SELECTORS = [
    ("div", {"id": "content"}),
    ("div", {"class_": "article-body"}),
    ("article", {}),
    ("div", {"class_": "article-intro"}),
    ("div", {"id": "article-start"}),
]

AD_SELECTORS = [
    "script", "noscript", "iframe",
    ".google-auto-placed", ".adsbygoogle",
    "#ad-sidebar", "#ad-content",
    ".sidebar-box", ".sidebar",
    "#sidebar", "#leftcolumn",
    ".navigation", "#footer",
    "footer", "header", "nav",
    ".top-header", ".runoob-header",
    "#topnav", ".topnav",
    "#right-ad", "#bottom-ad",
    ".ad", ".ads", ".advertisement",
    ".share-bar", ".social-share",
    ".related-articles", ".related_article",
    "#comments", ".comment-section",
    "#pagepreviousnext",
    ".previous-next-links",
    ".runcode-wrapper",
]

CJK_FONT_REPLACEMENT = '"PingFang SC", "Heiti SC", "Hiragino Sans GB", "Microsoft YaHei", sans-serif'

CJK_FONT_PATTERNS = [
    (r'"Comic Neue"\s*,\s*"Comic Sans MS"\s*,\s*"Noto Sans SC"\s*,\s*cursive\s*,\s*sans-serif',
     CJK_FONT_REPLACEMENT),
    (r'"Noto Sans SC"', '"PingFang SC", "Heiti SC"'),
    (r"font-family:\s*[^;]*\"Comic[^;]*;",
     f'font-family: {CJK_FONT_REPLACEMENT};'),
]

# ─── HTTP Session ────────────────────────────────────────────────────────────

session = requests.Session()
session.headers.update({
    "User-Agent": (
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
        "AppleWebKit/537.36 (KHTML, like Gecko) "
        "Chrome/120.0.0.0 Safari/537.36"
    )
})


# ─── Page Discovery ─────────────────────────────────────────────────────────

def discover_pages(start_url):
    """Fetch the start page and extract all tutorial page links from sidebar/nav."""
    resp = session.get(start_url, timeout=30)
    resp.encoding = "utf-8"
    soup = BeautifulSoup(resp.text, "html.parser")

    parsed = urlparse(start_url)
    base = f"{parsed.scheme}://{parsed.netloc}"
    path_prefix = "/".join(parsed.path.split("/")[:-1]) + "/"

    # Find sidebar or nav links that share the same path prefix
    links = []
    seen = set()
    for a in soup.find_all("a", href=True):
        href = a["href"]
        # Resolve relative URLs
        full = urljoin(start_url, href)
        full_parsed = urlparse(full)
        # Only keep links under the same path prefix
        if full_parsed.netloc == parsed.netloc and full_parsed.path.startswith(path_prefix):
            if full_parsed.path.endswith(".html") or full_parsed.path.endswith(".htm"):
                clean = f"{full_parsed.scheme}://{full_parsed.netloc}{full_parsed.path}"
                if clean not in seen:
                    seen.add(clean)
                    links.append(clean)

    # Ensure start page is first
    start_clean = f"{parsed.scheme}://{parsed.netloc}{parsed.path}"
    if start_clean in seen:
        links.remove(start_clean)
    links.insert(0, start_clean)

    return links, base


# ─── Content Extraction ─────────────────────────────────────────────────────

def extract_content(url, base_url, fix_fonts=True):
    """Fetch a page, extract main content, clean ads, fix SVGs."""
    resp = session.get(url, timeout=30)
    resp.encoding = "utf-8"
    soup = BeautifulSoup(resp.text, "html.parser")

    # Find main content
    article = None
    for tag, attrs in CONTENT_SELECTORS:
        article = soup.find(tag, **attrs)
        if article:
            break

    if not article:
        # Fallback: find div with 'content' or 'article' in class
        for div in soup.find_all("div"):
            cls = " ".join(div.get("class", []))
            if "content" in cls.lower() or "article" in cls.lower():
                article = div
                break

    if not article:
        return "", "Untitled"

    # Get title
    title_el = article.find("h1") or article.find("h2") or soup.find("h1")
    title = title_el.get_text(strip=True) if title_el else "Untitled"

    # Remove ad elements
    for sel in AD_SELECTORS:
        if sel.startswith("."):
            cls_name = sel[1:]
            for el in article.find_all(class_=re.compile(re.escape(cls_name), re.I)):
                el.decompose()
        elif sel.startswith("#"):
            el = article.find(id=sel[1:])
            if el:
                el.decompose()
        else:
            for el in article.find_all(sel):
                el.decompose()

    # Remove HTML comments
    for el in article.find_all(string=lambda t: isinstance(t, Comment)):
        el.extract()

    # Remove hidden elements
    for el in article.find_all(style=re.compile(r"display\s*:\s*none", re.I)):
        el.decompose()

    # Remove ad-related attributes
    for el in article.find_all(attrs={"data-ad": True}):
        el.decompose()
    for el in article.find_all(attrs={"class": re.compile(r"ad[s_-]|sponsor|promo", re.I)}):
        el.decompose()

    # Fix images and inline SVGs
    for img in article.find_all("img"):
        src = img.get("src", "")
        if not src:
            continue
        if src.startswith("/"):
            src = base_url + src
        elif not src.startswith("http"):
            src = base_url + "/" + src
        img["src"] = src

        # Inline SVG images
        if src.lower().endswith(".svg") and fix_fonts:
            try:
                svg_resp = session.get(src, timeout=15)
                svg_resp.encoding = "utf-8"
                svg_text = svg_resp.text

                # Replace CJK fonts
                for pattern, replacement in CJK_FONT_PATTERNS:
                    svg_text = re.sub(pattern, replacement, svg_text)

                svg_soup = BeautifulSoup(svg_text, "html.parser")
                svg_el = svg_soup.find("svg")
                if svg_el:
                    svg_el["style"] = svg_el.get("style", "") + "; max-width: 100%; height: auto;"
                    img.replace_with(svg_el)
            except Exception as e:
                print(f"    SVG inline failed for {src}: {e}")

    # Fix relative links
    for a in article.find_all("a"):
        href = a.get("href", "")
        if href.startswith("/"):
            a["href"] = base_url + href

    return str(article), title


# ─── HTML Builder ────────────────────────────────────────────────────────────

def build_html(pages, title="Tutorial", show_cover=True, show_toc=True):
    """Combine all page contents into a single styled HTML document."""
    toc_items = []
    body_parts = []

    for i, (content, ch_title) in enumerate(pages):
        anchor = f"chapter-{i}"
        toc_items.append(f'<li><a href="#{anchor}">{ch_title}</a></li>')
        body_parts.append(f'''
        <div class="chapter" id="{anchor}">
            <div class="chapter-content">{content}</div>
        </div>
        {'<div class="page-break"></div>' if i < len(pages) - 1 else ''}
        ''')

    cover = ""
    if show_cover:
        cover = f'''
        <div class="cover">
            <h1 style="border: none;">{title}</h1>
            <p class="subtitle">Complete Tutorial</p>
        </div>
        '''

    toc = ""
    if show_toc:
        toc = f'''
        <div class="toc-page">
            <h2 style="border-left: none;">Table of Contents</h2>
            <ul class="toc">{"".join(toc_items)}</ul>
        </div>
        '''

    return f'''<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<title>{title}</title>
<style>
@page {{
    size: A4;
    margin: 2cm 1.8cm;
}}
body {{
    font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", "PingFang SC",
                 "Hiragino Sans GB", "Microsoft YaHei", "Helvetica Neue",
                 Helvetica, Arial, sans-serif;
    font-size: 11pt;
    line-height: 1.7;
    color: #333;
    max-width: 100%;
}}
.cover {{
    text-align: center;
    padding-top: 200px;
    page-break-after: always;
}}
.cover h1 {{
    font-size: 32pt;
    color: #1a1a2e;
    margin-bottom: 20px;
}}
.cover .subtitle {{
    font-size: 14pt;
    color: #666;
}}
.toc-page {{
    page-break-after: always;
}}
.toc-page h2 {{
    font-size: 20pt;
    color: #1a1a2e;
    border-bottom: 2px solid #4a90d9;
    padding-bottom: 10px;
}}
.toc {{
    list-style: none;
    padding: 0;
}}
.toc li {{
    padding: 5px 0;
    border-bottom: 1px dotted #ddd;
}}
.toc li a {{
    color: #333;
    text-decoration: none;
}}
.page-break {{
    page-break-before: always;
}}
h1 {{
    font-size: 20pt;
    color: #1a1a2e;
    border-bottom: 2px solid #4a90d9;
    padding-bottom: 8px;
    margin-top: 10px;
}}
h2 {{
    font-size: 16pt;
    color: #2c3e50;
    margin-top: 20px;
    border-left: 4px solid #4a90d9;
    padding-left: 12px;
}}
h3 {{
    font-size: 13pt;
    color: #34495e;
    margin-top: 15px;
}}
pre {{
    background-color: #f6f8fa;
    border: 1px solid #e1e4e8;
    border-radius: 6px;
    padding: 12px 16px;
    overflow-x: auto;
    font-size: 9.5pt;
    line-height: 1.5;
    white-space: pre-wrap;
    word-wrap: break-word;
}}
code {{
    font-family: "SF Mono", "Fira Code", Menlo, Monaco, Consolas, monospace;
    font-size: 9.5pt;
}}
p > code, li > code, td > code {{
    background-color: #f0f0f0;
    padding: 2px 6px;
    border-radius: 3px;
    color: #c7254e;
}}
table {{
    width: 100%;
    border-collapse: collapse;
    margin: 15px 0;
    font-size: 10pt;
}}
th, td {{
    border: 1px solid #ddd;
    padding: 8px 12px;
    text-align: left;
}}
th {{
    background-color: #f2f2f2;
    font-weight: 600;
}}
tr:nth-child(even) {{
    background-color: #fafafa;
}}
img {{
    max-width: 100%;
    height: auto;
    display: block;
    margin: 10px auto;
}}
blockquote {{
    border-left: 4px solid #4a90d9;
    margin: 15px 0;
    padding: 10px 20px;
    background-color: #f8f9fa;
    color: #555;
}}
a {{
    color: #4a90d9;
    text-decoration: none;
}}
hr {{
    border: none;
    border-top: 1px solid #eee;
    margin: 20px 0;
}}
</style>
</head>
<body>
{cover}
{toc}
{"".join(body_parts)}
</body>
</html>'''


# ─── PDF Generation ──────────────────────────────────────────────────────────

async def generate_pdf(html_path, pdf_path):
    """Render HTML to PDF with Playwright, then add bookmarks with pypdf."""
    from playwright.async_api import async_playwright
    from pypdf import PdfReader, PdfWriter
    from pypdf.generic import Fit

    temp_pdf = pdf_path + ".tmp"

    async with async_playwright() as p:
        browser = await p.chromium.launch()
        page = await browser.new_page()

        print("Loading HTML in headless Chromium...")
        await page.goto(f"file://{html_path}", wait_until="networkidle", timeout=120000)
        await page.wait_for_timeout(3000)

        # Get chapter positions for bookmark calculation
        await page.emulate_media(media="print")
        await page.wait_for_timeout(1000)

        chapters_info = await page.evaluate("""() => {
            const results = [];
            const cover = document.querySelector('.cover');
            if (cover) results.push({ title: 'Cover', top: 0 });
            const toc = document.querySelector('.toc-page');
            if (toc) results.push({ title: 'Table of Contents', top: toc.getBoundingClientRect().top + window.scrollY });
            document.querySelectorAll('.chapter').forEach(ch => {
                const h1 = ch.querySelector('h1');
                if (h1) {
                    results.push({
                        title: h1.textContent.trim(),
                        top: ch.getBoundingClientRect().top + window.scrollY
                    });
                }
            });
            results.push({ title: '__total__', top: document.documentElement.scrollHeight });
            return results;
        }""")

        total_height = next(c["top"] for c in chapters_info if c["title"] == "__total__")
        bookmarks = [c for c in chapters_info if c["title"] != "__total__"]

        print("Rendering PDF...")
        await page.pdf(
            path=temp_pdf,
            format="A4",
            margin={"top": "1.5cm", "bottom": "1.5cm", "left": "1.8cm", "right": "1.8cm"},
            print_background=True,
            display_header_footer=False,
        )
        await browser.close()

    # Add bookmarks
    print("Adding bookmarks...")
    reader = PdfReader(temp_pdf)
    writer = PdfWriter()
    for pg in reader.pages:
        writer.add_page(pg)

    total_pages = len(reader.pages)
    px_per_page = total_height / total_pages if total_pages else 1

    for bm in bookmarks:
        pg = max(0, min(int(bm["top"] / px_per_page), total_pages - 1))
        writer.add_outline_item(bm["title"], pg, fit=Fit.fit_horizontally(top=800))
        print(f"  Page {pg+1:>3d}: {bm['title']}")

    writer.write(pdf_path)
    os.remove(temp_pdf)
    return total_pages


# ─── Main ────────────────────────────────────────────────────────────────────

def main():
    parser = argparse.ArgumentParser(description="Convert web tutorials to bookmarked A4 PDF")
    parser.add_argument("url", help="Start page URL of the tutorial")
    parser.add_argument("--output", "-o", default="./output.pdf", help="Output PDF path")
    parser.add_argument("--title", "-t", default=None, help="PDF cover title")
    parser.add_argument("--no-font-fix", action="store_true", help="Skip SVG font replacement")
    parser.add_argument("--delay", type=float, default=0.5, help="Delay between requests (seconds)")
    parser.add_argument("--no-cover", action="store_true", help="Skip cover page")
    parser.add_argument("--no-toc", action="store_true", help="Skip table of contents")
    args = parser.parse_args()

    # Step 1: Discover pages
    print(f"Discovering pages from {args.url} ...")
    pages_urls, base_url = discover_pages(args.url)
    print(f"Found {len(pages_urls)} pages")

    # Step 2: Fetch and clean each page
    pages_content = []
    for i, url in enumerate(pages_urls):
        print(f"  [{i+1}/{len(pages_urls)}] {urlparse(url).path}")
        try:
            content, title = extract_content(url, base_url, fix_fonts=not args.no_font_fix)
            if content:
                pages_content.append((content, title))
        except Exception as e:
            print(f"    Error: {e}")
        if args.delay > 0:
            time.sleep(args.delay)

    print(f"Fetched {len(pages_content)} pages successfully")

    # Auto-detect title
    pdf_title = args.title or (pages_content[0][1] if pages_content else "Tutorial")

    # Step 3: Build combined HTML
    print("Building combined HTML...")
    html = build_html(pages_content, title=pdf_title,
                      show_cover=not args.no_cover, show_toc=not args.no_toc)

    output_dir = os.path.dirname(os.path.abspath(args.output))
    html_path = os.path.join(output_dir, "._temp_combined.html")
    with open(html_path, "w", encoding="utf-8") as f:
        f.write(html)

    # Step 4: Generate PDF with bookmarks
    total_pages = asyncio.run(generate_pdf(html_path, os.path.abspath(args.output)))
    os.remove(html_path)

    size_mb = os.path.getsize(args.output) / (1024 * 1024)
    print(f"\nDone! {total_pages} pages, {size_mb:.1f} MB")
    print(f"Output: {os.path.abspath(args.output)}")


if __name__ == "__main__":
    main()