From e0607bbda379c4252c62c93f365818e9564c5d7b Mon Sep 17 00:00:00 2001 From: squid Date: Sun, 15 Nov 2026 09:56:11 +0100 Subject: [PATCH] chore: add the one-off migration and compare tools --- tools/compare.py | 127 +++++++++++++++++++++ tools/migrate.py | 251 ++++++++++++++++++++++++++++++++++++++++++ tools/test_compare.py | 59 ++++++++++ tools/test_migrate.py | 101 +++++++++++++++++ 4 files changed, 538 insertions(+) create mode 100644 tools/compare.py create mode 100644 tools/migrate.py create mode 100644 tools/test_compare.py create mode 100644 tools/test_migrate.py diff --git a/tools/compare.py b/tools/compare.py new file mode 100644 index 0000000..0f94fea --- /dev/null +++ b/tools/compare.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""Compare the site built by build.py (old) with the one built by motorino (new), page by page. + +Usage: compare.py OLD_DIR NEW_DIR + +The text, the tags, their attributes, the links and the comments of every page must be the same. The +differences that the change wants are ignored: the "Powered by motorino" footer line, the generator meta, the +author and categories after the date of a news item, and its description meta. Old news addresses are moved +to the new dated ones before the comparison. Exit status 1 when anything else differs.""" +import difflib +import html as htmllib +import os +import re +import sys +from html.parser import HTMLParser + + +class Tokens(HTMLParser): + """The stream of (kind, value) tokens of a page: whitespace is collapsed and attribute order does not matter.""" + + def __init__(self): + super().__init__(convert_charrefs=True) + self.items = [] + + def handle_starttag(self, tag, attrs): + self.items.append(("start", tag + "".join(f' {k}="{v}"' for k, v in sorted(attrs)))) + + def handle_startendtag(self, tag, attrs): + self.handle_starttag(tag, attrs) + + def handle_endtag(self, tag): + self.items.append(("end", tag)) + + def handle_data(self, data): + text = " ".join(data.split()) + if text: + if self.items and self.items[-1][0] == "text": + self.items[-1] = ("text", self.items[-1][1] + " " + text) + else: + self.items.append(("text", text)) + + def handle_comment(self, data): + self.items.append(("comment", " ".join(data.split()))) + + +CREDIT = re.compile(r"

\s*Powered by .*?

\s*", re.S) +GENERATOR = re.compile(r']*>\s*') +BYLINE = re.compile(r'()\s*·.*?(\s*

)', re.S) +DESCRIPTION = re.compile(r']*>\s*') + + +def tokens(html, post=False): + html = GENERATOR.sub("", CREDIT.sub("", html)) + if post: + html = DESCRIPTION.sub("", BYLINE.sub(r"\1\2", html, count=1)) + parser = Tokens() + parser.feed(html) + return [f"{kind}: {value}" for kind, value in parser.items] + + +def compare_pages(old, new, post=False): + """The differences between two pages, as lines of a diff; empty when they match.""" + a, b = tokens(old, post), tokens(new, post) + if a == b: + return [] + return [line for line in difflib.unified_diff(a, b, "old", "new", lineterm="", n=2)] + + +def remap(html, mapping): + """Replace old addresses by new ones.""" + for old, new in sorted(mapping.items(), key=lambda kv: -len(kv[0])): + html = html.replace(old, new) + return html + + +def html_files(root): + for folder, _, files in os.walk(root): + for name in files: + if name.endswith(".html"): + yield "/" + os.path.relpath(os.path.join(folder, name), root).replace(os.sep, "/") + + +def read(root, rel): + with open(os.path.join(root, rel.lstrip("/")), encoding="utf-8") as f: + return f.read() + + +def post_mapping(old_dir, new_dir): + """Old news addresses (/news/slug.html) to the new dated ones, found by name in the new output.""" + new_posts = {} + for rel in html_files(new_dir): + m = re.fullmatch(r"/news/(\d{4})/(\d{2})/([a-z0-9-]+)\.html", rel) + if m: + new_posts[m.group(3)] = rel + mapping = {} + for rel in html_files(old_dir): + m = re.fullmatch(r"/news/([a-z0-9-]+)\.html", rel) + if m and m.group(1) in new_posts: + mapping[rel] = new_posts[m.group(1)] + return mapping + + +def main(old_dir, new_dir): + mapping = post_mapping(old_dir, new_dir) + bad = 0 + for rel in sorted(html_files(old_dir)): + new_rel = mapping.get(rel, rel) + if not os.path.exists(os.path.join(new_dir, new_rel.lstrip("/"))): + print(f"MISSING in new output: {new_rel}") + bad += 1 + continue + diffs = compare_pages(remap(read(old_dir, rel), mapping), read(new_dir, new_rel), post=rel in mapping) + if diffs: + bad += 1 + print(f"DIFFERENT: {rel}" + (f" -> {new_rel}" if new_rel != rel else "")) + print("\n".join(diffs[:60])) + extra = sorted(set(html_files(new_dir)) - {mapping.get(r, r) for r in html_files(old_dir)}) + for rel in extra: + print(f"new page: {rel}") + print(f"compared {len(list(html_files(old_dir)))} pages, {bad} differ") + return 1 if bad else 0 + + +if __name__ == "__main__": + if len(sys.argv) != 3: + sys.exit(__doc__) + sys.exit(main(sys.argv[1], sys.argv[2])) diff --git a/tools/migrate.py b/tools/migrate.py new file mode 100644 index 0000000..aad6db7 --- /dev/null +++ b/tools/migrate.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +"""One-off migration of the kmalloc.org site from build.py to motorino. + +Usage: migrate.py [--site DIR] [--old-out DIR] + +It builds the current site with build.py (the reference, saved in --old-out), then writes the new layout in +place: pages as Markdown in content//, news with the new header, the reduced site.json and lang files, +and the new templates. The pages are converted from the HTML that build.py produced, so their text is the +same by construction; HTML that Markdown cannot express (a table, an image, a heading with a class) is kept +as raw HTML in the file. Delete this tool once the new site is live.""" +import argparse +import ast +import json +import os +import re +import shutil +import subprocess +import sys + +# ---------------------------------------------------------------- HTML to Markdown + +BLOCK = re.compile(r"\s*(?:()|<(p|h[1-6]|ul|ol|table|blockquote|div)\b([^>]*)>(.*?))", re.S) +ID_ONLY = re.compile(r'^\s*id="([^"]+)"\s*$') + + +def inline_to_md(inner): + """Inline HTML as Markdown, or None when it has markup that Markdown cannot say.""" + s = inner.strip() + s = re.sub(r'(.*?)', r"[\2](\1)", s, flags=re.S) + s = re.sub(r"(.*?)", r"**\1**", s, flags=re.S) + s = re.sub(r"(.*?)", r"*\1*", s, flags=re.S) + s = re.sub(r"(.*?)", r"`\1`", s, flags=re.S) + if "<" in s: + return None + return "\n".join(line.strip() for line in s.splitlines()) + + +def block_to_md(match): + """One top-level block as Markdown, or as the raw HTML it came from.""" + raw = match.group(0).strip() + comment, tag, attrs, inner = match.groups() + if comment: + return raw + if tag == "p" and not attrs.strip(): + text = inline_to_md(inner) + return text if text is not None else raw + if re.fullmatch(r"h[1-6]", tag): + text = inline_to_md(inner) + heading_id = ID_ONLY.match(attrs) + if text is not None and (not attrs.strip() or heading_id): + line = "#" * int(tag[1]) + " " + text + return line + (" {#" + heading_id.group(1) + "}" if heading_id else "") + return raw + if tag == "ul" and not attrs.strip(): + items = re.findall(r"
  • (.*?)
  • ", inner, flags=re.S) + texts = [inline_to_md(i) for i in items] + if items and all(t is not None for t in texts) and not re.sub(r"
  • .*?
  • |\s", "", inner, flags=re.S): + return "\n".join("- " + t for t in texts) + return raw + + +def html_to_md(html): + """Convert the body of a page: the blocks of build.py output become Markdown, the rest stays HTML.""" + parts, pos = [], 0 + while True: + if not html[pos:].strip(): + break + m = BLOCK.match(html, pos) + if not m: + raise ValueError(f"unconvertible HTML near: {html[pos:pos + 80].strip()!r}") + parts.append(block_to_md(m)) + pos = m.end() + return "\n\n".join(parts) + "\n" + + +def restore(md, values): + """Put {{key}} back where a value of site.json appears as text, longest values first.""" + for key, value in sorted(values.items(), key=lambda kv: -len(kv[1])): + if value: + md = md.replace(value, "{{" + key + "}}") + return md + + +def mark_latest_news(html): + """The news block of the home page becomes the {{latest_news}} placeholder.""" + return re.sub(r'

    .*?\n?', "

    {{latest_news}}

    \n", html, count=1, flags=re.S) + + +def mark_news_list(html): + """The list of the news page becomes the {{news_list}} placeholder.""" + return re.sub(r"
      .*?
    ", "

    {{news_list}}

    ", html, count=1, flags=re.S) + + +# ---------------------------------------------------------------- pages and news + +def extract_content(page_html): + """The part of a built page between the header and the footer.""" + m = re.search(r"\n\n(.*)\n