dr-shahab-mousavi/tools/clone_page.py

"""Regenerate the page components in src/components/generated from dr-mousavi.com.

Downloads each source page, mirrors any asset it references into public/theme
(preserving the remote paths so the theme CSS keeps resolving), then converts the
markup to JSX with class names, inline styles and AOS attributes intact.

    python3 tools/clone_page.py            # all pages
    python3 tools/clone_page.py services   # a single page

Source HTML is cached under tools/.cache; delete it to pull fresh markup.

Storage-hosted article thumbnails have Persian filenames, so they are stored
under a hash of their URL, which keeps names stable when the article list changes.
"""

import hashlib
import json
import os
import re
import sys
import urllib.parse
import urllib.request
from html.parser import HTMLParser

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
CACHE = os.path.join(ROOT, "tools", ".cache")
THEME = os.path.join(ROOT, "public", "theme")
OUT = os.path.join(ROOT, "src", "components", "generated")

TPL = "https://meshkee-templates.sas.amin.parminstorage.ir/sites/user_themes/dr-mousavi_RsrCTgktvU/assets/"
STORE = "https://meshkee-storage.sas.amin.parminstorage.ir/"
ASSET_EXT = {".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp", ".ttf", ".woff", ".woff2"}

# source page -> (url, component name, section of the document to convert)
PAGES = {
    "home": ("https://www.dr-mousavi.com/", "HomeSections", "main"),
    "about": ("https://www.dr-mousavi.com/about-us", "AboutSections", "main"),
    "contact": ("https://www.dr-mousavi.com/contact-us", "ContactSections", "main"),
    "services": ("https://www.dr-mousavi.com/ourservices", "ServicesSections", "main"),
    "header": ("https://www.dr-mousavi.com/", "SiteHeader", "header"),
    "footer": ("https://www.dr-mousavi.com/", "SiteFooter", "footer"),
}

# Pages that exist in this clone; anything else keeps pointing at the live site.
INTERNAL = {
    "https://www.dr-mousavi.com": "/",
    "https://www.dr-mousavi.com/": "/",
    "http://www.dr-mousavi.com": "/",
    "https://dr-mousavi.com": "/",
    "https://www.dr-mousavi.com/about-us": "/about-us",
    "https://www.dr-mousavi.com/contact-us": "/contact-us",
    "https://www.dr-mousavi.com/ourservices": "/ourservices",
    "https://www.dr-mousavi.com/surgeries": "/surgeries",
    "https://www.dr-mousavi.com/articles": "/articles",
}

# The original points its account icon at an app subdomain; this clone sends
# visitors to the Meshkee customer portal instead, whose URL lives in one place.
CUSTOMER_PORTAL_HOSTS = ("app.dr-mousavi.com", "customer.dr-mousavi.com")
CUSTOMER_URL_TOKEN = "__CUSTOMER_URL__"

VOID = {"img", "input", "br", "hr", "meta", "link", "source", "area", "base", "col", "embed", "track", "wbr"}
DROP_TAGS = {"script", "style", "noscript"}

ATTR_RENAME = {
    "class": "className",
    "for": "htmlFor",
    "colspan": "colSpan",
    "rowspan": "rowSpan",
    "tabindex": "tabIndex",
    "maxlength": "maxLength",
    "allowfullscreen": "allowFullScreen",
    "frameborder": "frameBorder",
    "srcset": "srcSet",
    "novalidate": "noValidate",
    "autocomplete": "autoComplete",
    "readonly": "readOnly",
    "viewbox": "viewBox",
    "preserveaspectratio": "preserveAspectRatio",
    "stroke-dasharray": "strokeDasharray",
    "stroke-dashoffset": "strokeDashoffset",
}

# Builder bookkeeping and Bootstrap JS hooks; behaviour is reimplemented in React.
DROP_ATTR_PREFIX = ("wb-", "data-toggle", "data-target", "data-parent", "aria-expanded",
                    "webkitallowfullscreen", "mozallowfullscreen")
DROP_ATTR = {"onclick", "onsubmit", "onload"}

# The theme's date script reads this attribute; SiteBehaviour does the same.
ATTR_RENAME["date"] = "data-date"


def jstr(value):
    return json.dumps(value, ensure_ascii=False)


class Node:
    def __init__(self, tag=None, attrs=None):
        self.tag = tag
        self.attrs = dict(attrs or [])
        self.children = []
        self.text = None


class Builder(HTMLParser):
    def __init__(self):
        super().__init__(convert_charrefs=False)
        self.root = Node("#root")
        self.stack = [self.root]

    def handle_starttag(self, tag, attrs):
        node = Node(tag, attrs)
        self.stack[-1].children.append(node)
        if tag not in VOID:
            self.stack.append(node)

    def handle_startendtag(self, tag, attrs):
        self.stack[-1].children.append(Node(tag, attrs))

    def handle_endtag(self, tag):
        for i in range(len(self.stack) - 1, 0, -1):
            if self.stack[i].tag == tag:
                del self.stack[i:]
                return

    def _text(self, value):
        node = Node()
        node.text = value
        self.stack[-1].children.append(node)

    def handle_data(self, data):
        self._text(data)

    def handle_entityref(self, name):
        self._text(f"&{name};")

    def handle_charref(self, name):
        self._text(f"&#{name};")


def fetch(url, dest):
    if os.path.exists(dest) and os.path.getsize(dest) > 0:
        return True
    os.makedirs(os.path.dirname(dest), exist_ok=True)
    parts = urllib.parse.urlsplit(url)
    encoded = urllib.parse.urlunsplit(
        (parts.scheme, parts.netloc, urllib.parse.quote(parts.path), parts.query, parts.fragment)
    )
    try:
        request = urllib.request.Request(encoded, headers={"User-Agent": "Mozilla/5.0"})
        with urllib.request.urlopen(request, timeout=60) as response, open(dest, "wb") as handle:
            handle.write(response.read())
        return True
    except Exception as exc:  # noqa: BLE001 - upstream 404s are expected and skippable
        print("  asset failed:", url.rsplit("/", 1)[-1], exc)
        if os.path.exists(dest):
            os.remove(dest)
        return False


def download_pages(selected):
    os.makedirs(CACHE, exist_ok=True)
    html_by_page = {}
    for name in selected:
        url = PAGES[name][0]
        path = os.path.join(CACHE, f"{name}.html")
        if not os.path.exists(path):
            print("downloading", url)
            request = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
            with urllib.request.urlopen(request, timeout=60) as response:
                open(path, "wb").write(response.read())
        html_by_page[name] = open(path, encoding="utf-8", errors="ignore").read()
    return html_by_page


def collect_asset_urls(html):
    """Attribute values and CSS url() targets; article paths may contain parentheses."""
    html = html.replace("\\/", "/")
    urls = set()
    # Anchor url() matches on the file extension: article filenames contain "()".
    image = r"\.(?:png|jpe?g|gif|svg|webp)"
    for pattern in (
        r"(?:src|href|data-src)=\"(https?://[^\"]+)\"",
        rf"url\((https?://.+?{image})\)",
        rf"url\('(https?://.+?{image})'\)",
    ):
        urls.update(re.findall(pattern, html, re.IGNORECASE))
    return {u.split("?")[0] for u in urls if u.startswith(TPL) or u.startswith(STORE)}


def mirror_assets(html_by_page):
    """Return a remote-url -> local-path map, downloading anything missing."""
    urls = set()
    for html in html_by_page.values():
        urls.update(collect_asset_urls(html))

    mapping = {}
    for url in sorted(urls):
        ext = os.path.splitext(url)[1].lower()
        if ext not in ASSET_EXT:
            continue
        if url.startswith(TPL):
            rel = url[len(TPL):]
        else:
            rel = f"articles/{hashlib.md5(url.encode()).hexdigest()[:12]}{ext}"
        if fetch(url, os.path.join(THEME, rel)):
            mapping[url] = f"/theme/{rel}"
    return mapping


def prune_articles(mapping):
    """Drop thumbnails from earlier runs so no stale file shadows a new name."""
    folder = os.path.join(THEME, "articles")
    if not os.path.isdir(folder):
        return
    keep = {os.path.basename(path) for path in mapping.values()}
    for name in os.listdir(folder):
        if name not in keep:
            os.remove(os.path.join(folder, name))


def camel(prop):
    if prop.startswith("--"):
        return prop
    head, *rest = prop.split("-")
    return head + "".join(part.capitalize() for part in rest)


class Converter:
    def __init__(self, assets):
        self.assets = assets
        self.icons = set()
        self.unsupported_important = set()

    def rewrite_urls(self, value):
        for url, local in self.assets.items():
            if url in value:
                value = value.replace(url, local)
        return value

    def rewrite_link(self, href):
        href = self.rewrite_urls(href.strip())
        if any(host in href for host in CUSTOMER_PORTAL_HOSTS):
            return CUSTOMER_URL_TOKEN
        if href in INTERNAL:
            return INTERNAL[href]
        if href.startswith("/assets"):
            return "#"
        if href.startswith("/article-details"):
            return "/articles"
        return href

    def style_object(self, value):
        """Return the JSX style object plus the declarations marked !important.

        React style objects cannot express !important, so the caller has to
        replace those declarations with a class that carries the same weight.
        """
        pairs = []
        important = {}
        for chunk in value.split(";"):
            if ":" not in chunk:
                continue
            prop, val = chunk.split(":", 1)
            prop, val = prop.strip(), val.strip()
            if not prop or not val or val == "undefined":
                continue
            plain = re.sub(r"\s*!\s*important$", "", val)
            if plain != val:
                important[prop] = plain
            pairs.append(f"{jstr(camel(prop))}: {jstr(self.rewrite_urls(plain))}")
        return ("{ " + ", ".join(pairs) + " }" if pairs else None), important

    def important_classes(self, important):
        """Map !important inline declarations onto the helper classes in globals.css."""
        classes = []
        for prop, val in important.items():
            if prop == "direction" and val in ("ltr", "rtl"):
                classes.append(f"wb-force-{val}")
            else:
                self.unsupported_important.add(f"{prop}: {val}")
        return classes

    @staticmethod
    def should_drop(node):
        if node.tag in DROP_TAGS:
            return True
        cls = (node.attrs.get("class") or "").split()
        if "loader-mask" in cls or "copyright-willaarts" in cls:
            return True
        return node.tag == "input" and node.attrs.get("type") == "hidden"

    def serialize_attrs(self, node):
        out = []
        style_obj, important = self.style_object(node.attrs.get("style") or "")
        extra_classes = self.important_classes(important)
        for key, value in node.attrs.items():
            low = key.lower().strip()
            if not low or low in DROP_ATTR or any(low.startswith(p) for p in DROP_ATTR_PREFIX):
                continue
            value = value or ""
            if low == "style":
                if style_obj:
                    out.append(f"style={{{style_obj}}}")
                continue
            if low in ("src", "poster"):
                value = self.rewrite_urls(value)
            elif low == "href":
                value = self.rewrite_link(value)
                if value == CUSTOMER_URL_TOKEN:
                    out.append("href={CUSTOMER_URL}")
                    continue
            elif low == "class":
                value = " ".join(dict.fromkeys(value.split() + extra_classes))
                extra_classes = []
            elif low == "value" and node.tag in ("input", "textarea"):
                out.append(f"defaultValue={jstr(value)}")
                continue
            elif low == "allowfullscreen":
                out.append("allowFullScreen")
                continue
            name = ATTR_RENAME.get(low, low)
            if "-" in name and not name.startswith(("data-", "aria-")):
                head, *rest = name.split("-")
                name = head + "".join(part.capitalize() for part in rest)
            out.append(f"{name}={jstr(value)}")
        if extra_classes:
            out.append(f"className={jstr(' '.join(extra_classes))}")
        if node.tag == "img" and "alt" not in node.attrs:
            out.append('alt=""')
        return out

    def serialize(self, node, indent=0):
        pad = "  " * indent
        if node.text is not None:
            text = node.text.strip()
            return pad + text.replace("{", "{").replace("}", "}") + "\n" if text else ""
        if self.should_drop(node):
            return ""

        self.icons.update(re.findall(r"WMi-[a-zA-Z0-9_-]+", node.attrs.get("class") or ""))
        children = "".join(self.serialize(child, indent + 1) for child in node.children)
        attr_text = " ".join(self.serialize_attrs(node))
        attr_text = f" {attr_text}" if attr_text else ""

        if (node.attrs.get("id") or "").startswith("owl-carousel"):
            margin = 10 if node.attrs["id"] == "owl-carouselone" else 30
            return f"{pad}<Carousel margin={{{margin}}}>\n{children}{pad}</Carousel>\n"

        tag = node.tag
        # A <p> around block markup is invalid HTML: browsers reparent it, which
        # would leave React's hydrated tree different from the server output.
        if tag == "p" and any(
            child.tag in {"div", "figure", "section", "ul", "ol", "table", "form", "p"}
            for child in node.children
        ):
            tag = "div"
        elif tag == "a":
            href = self.rewrite_link(node.attrs.get("href") or "")
            if href.startswith("/") and not href.startswith("//"):
                tag = "Link"

        if tag in VOID:
            return f"{pad}<{tag}{attr_text} />\n"
        if not children:
            return f"{pad}<{tag}{attr_text}></{tag}>\n"
        return f"{pad}<{tag}{attr_text}>\n{children}{pad}</{tag}>\n"


def extract(html, section):
    if section == "header":
        start, end = '<header class="wb-header">', "</header>\n            <main"
    elif section == "footer":
        start, end = '<footer class="wb-footer">', "</footer>"
    else:
        start, end = '<main class="wb-content">', "</main>"
    begin = html.index(start) + len(start)
    return html[begin:html.index(end, begin)]


def main():
    selected = sys.argv[1:] or list(PAGES)
    unknown = [name for name in selected if name not in PAGES]
    if unknown:
        sys.exit(f"unknown page(s): {', '.join(unknown)}\navailable: {', '.join(PAGES)}")

    html_by_page = download_pages(sorted(set(selected) | {"home", "about", "contact", "services"}))
    assets = mirror_assets(html_by_page)
    prune_articles(assets)
    print(f"assets mapped: {len(assets)}")

    os.makedirs(OUT, exist_ok=True)
    for name in selected:
        _, component, section = PAGES[name]
        converter = Converter(assets)
        tree = Builder()
        tree.feed(extract(html_by_page[name], section))
        jsx = "".join(converter.serialize(child, 3) for child in tree.root.children)

        imports = ""
        if "<Link" in jsx:
            imports += 'import Link from "next/link";\n'
        if "<Carousel" in jsx:
            imports += 'import Carousel from "@/components/Carousel";\n'
        if "{CUSTOMER_URL}" in jsx:
            imports += 'import { CUSTOMER_URL } from "@/lib/meshkee";\n'
        if imports:
            imports += "\n"

        path = os.path.join(OUT, f"{component}.tsx")
        with open(path, "w", encoding="utf-8") as handle:
            handle.write(
                f"{imports}export default function {component}() {{\n"
                f"  return (\n    <>\n{jsx}    </>\n  );\n}}\n"
            )
        print(f"wrote {component}.tsx  icons={sorted(converter.icons)}")
        for declaration in sorted(converter.unsupported_important):
            print(f"  !important declaration needs a helper class: {declaration}")


if __name__ == "__main__":
    main()