From e0607bbda379c4252c62c93f365818e9564c5d7b Mon Sep 17 00:00:00 2001
From: squid
Date: Sun, 15 Nov 2026 09:56:11 +0100
Subject: [PATCH 1/3] chore: add the one-off migration and compare tools
---
tools/compare.py | 127 +++++++++++++++++++++
tools/migrate.py | 251 ++++++++++++++++++++++++++++++++++++++++++
tools/test_compare.py | 59 ++++++++++
tools/test_migrate.py | 101 +++++++++++++++++
4 files changed, 538 insertions(+)
create mode 100644 tools/compare.py
create mode 100644 tools/migrate.py
create mode 100644 tools/test_compare.py
create mode 100644 tools/test_migrate.py
diff --git a/tools/compare.py b/tools/compare.py
new file mode 100644
index 0000000..0f94fea
--- /dev/null
+++ b/tools/compare.py
@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+"""Compare the site built by build.py (old) with the one built by motorino (new), page by page.
+
+Usage: compare.py OLD_DIR NEW_DIR
+
+The text, the tags, their attributes, the links and the comments of every page must be the same. The
+differences that the change wants are ignored: the "Powered by motorino" footer line, the generator meta, the
+author and categories after the date of a news item, and its description meta. Old news addresses are moved
+to the new dated ones before the comparison. Exit status 1 when anything else differs."""
+import difflib
+import html as htmllib
+import os
+import re
+import sys
+from html.parser import HTMLParser
+
+
+class Tokens(HTMLParser):
+ """The stream of (kind, value) tokens of a page: whitespace is collapsed and attribute order does not matter."""
+
+ def __init__(self):
+ super().__init__(convert_charrefs=True)
+ self.items = []
+
+ def handle_starttag(self, tag, attrs):
+ self.items.append(("start", tag + "".join(f' {k}="{v}"' for k, v in sorted(attrs))))
+
+ def handle_startendtag(self, tag, attrs):
+ self.handle_starttag(tag, attrs)
+
+ def handle_endtag(self, tag):
+ self.items.append(("end", tag))
+
+ def handle_data(self, data):
+ text = " ".join(data.split())
+ if text:
+ if self.items and self.items[-1][0] == "text":
+ self.items[-1] = ("text", self.items[-1][1] + " " + text)
+ else:
+ self.items.append(("text", text))
+
+ def handle_comment(self, data):
+ self.items.append(("comment", " ".join(data.split())))
+
+
+CREDIT = re.compile(r"
)', re.S)
+DESCRIPTION = re.compile(r']*>\s*')
+
+
+def tokens(html, post=False):
+ html = GENERATOR.sub("", CREDIT.sub("", html))
+ if post:
+ html = DESCRIPTION.sub("", BYLINE.sub(r"\1\2", html, count=1))
+ parser = Tokens()
+ parser.feed(html)
+ return [f"{kind}: {value}" for kind, value in parser.items]
+
+
+def compare_pages(old, new, post=False):
+ """The differences between two pages, as lines of a diff; empty when they match."""
+ a, b = tokens(old, post), tokens(new, post)
+ if a == b:
+ return []
+ return [line for line in difflib.unified_diff(a, b, "old", "new", lineterm="", n=2)]
+
+
+def remap(html, mapping):
+ """Replace old addresses by new ones."""
+ for old, new in sorted(mapping.items(), key=lambda kv: -len(kv[0])):
+ html = html.replace(old, new)
+ return html
+
+
+def html_files(root):
+ for folder, _, files in os.walk(root):
+ for name in files:
+ if name.endswith(".html"):
+ yield "/" + os.path.relpath(os.path.join(folder, name), root).replace(os.sep, "/")
+
+
+def read(root, rel):
+ with open(os.path.join(root, rel.lstrip("/")), encoding="utf-8") as f:
+ return f.read()
+
+
+def post_mapping(old_dir, new_dir):
+ """Old news addresses (/news/slug.html) to the new dated ones, found by name in the new output."""
+ new_posts = {}
+ for rel in html_files(new_dir):
+ m = re.fullmatch(r"/news/(\d{4})/(\d{2})/([a-z0-9-]+)\.html", rel)
+ if m:
+ new_posts[m.group(3)] = rel
+ mapping = {}
+ for rel in html_files(old_dir):
+ m = re.fullmatch(r"/news/([a-z0-9-]+)\.html", rel)
+ if m and m.group(1) in new_posts:
+ mapping[rel] = new_posts[m.group(1)]
+ return mapping
+
+
+def main(old_dir, new_dir):
+ mapping = post_mapping(old_dir, new_dir)
+ bad = 0
+ for rel in sorted(html_files(old_dir)):
+ new_rel = mapping.get(rel, rel)
+ if not os.path.exists(os.path.join(new_dir, new_rel.lstrip("/"))):
+ print(f"MISSING in new output: {new_rel}")
+ bad += 1
+ continue
+ diffs = compare_pages(remap(read(old_dir, rel), mapping), read(new_dir, new_rel), post=rel in mapping)
+ if diffs:
+ bad += 1
+ print(f"DIFFERENT: {rel}" + (f" -> {new_rel}" if new_rel != rel else ""))
+ print("\n".join(diffs[:60]))
+ extra = sorted(set(html_files(new_dir)) - {mapping.get(r, r) for r in html_files(old_dir)})
+ for rel in extra:
+ print(f"new page: {rel}")
+ print(f"compared {len(list(html_files(old_dir)))} pages, {bad} differ")
+ return 1 if bad else 0
+
+
+if __name__ == "__main__":
+ if len(sys.argv) != 3:
+ sys.exit(__doc__)
+ sys.exit(main(sys.argv[1], sys.argv[2]))
diff --git a/tools/migrate.py b/tools/migrate.py
new file mode 100644
index 0000000..aad6db7
--- /dev/null
+++ b/tools/migrate.py
@@ -0,0 +1,251 @@
+#!/usr/bin/env python3
+"""One-off migration of the kmalloc.org site from build.py to motorino.
+
+Usage: migrate.py [--site DIR] [--old-out DIR]
+
+It builds the current site with build.py (the reference, saved in --old-out), then writes the new layout in
+place: pages as Markdown in content//, news with the new header, the reduced site.json and lang files,
+and the new templates. The pages are converted from the HTML that build.py produced, so their text is the
+same by construction; HTML that Markdown cannot express (a table, an image, a heading with a class) is kept
+as raw HTML in the file. Delete this tool once the new site is live."""
+import argparse
+import ast
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+
+# ---------------------------------------------------------------- HTML to Markdown
+
+BLOCK = re.compile(r"\s*(?:()|<(p|h[1-6]|ul|ol|table|blockquote|div)\b([^>]*)>(.*?)\2>)", re.S)
+ID_ONLY = re.compile(r'^\s*id="([^"]+)"\s*$')
+
+
+def inline_to_md(inner):
+ """Inline HTML as Markdown, or None when it has markup that Markdown cannot say."""
+ s = inner.strip()
+ s = re.sub(r'(.*?)', r"[\2](\1)", s, flags=re.S)
+ s = re.sub(r"(.*?)", r"**\1**", s, flags=re.S)
+ s = re.sub(r"(.*?)", r"*\1*", s, flags=re.S)
+ s = re.sub(r"(.*?)", r"`\1`", s, flags=re.S)
+ if "<" in s:
+ return None
+ return "\n".join(line.strip() for line in s.splitlines())
+
+
+def block_to_md(match):
+ """One top-level block as Markdown, or as the raw HTML it came from."""
+ raw = match.group(0).strip()
+ comment, tag, attrs, inner = match.groups()
+ if comment:
+ return raw
+ if tag == "p" and not attrs.strip():
+ text = inline_to_md(inner)
+ return text if text is not None else raw
+ if re.fullmatch(r"h[1-6]", tag):
+ text = inline_to_md(inner)
+ heading_id = ID_ONLY.match(attrs)
+ if text is not None and (not attrs.strip() or heading_id):
+ line = "#" * int(tag[1]) + " " + text
+ return line + (" {#" + heading_id.group(1) + "}" if heading_id else "")
+ return raw
+ if tag == "ul" and not attrs.strip():
+ items = re.findall(r"
(.*?)
", inner, flags=re.S)
+ texts = [inline_to_md(i) for i in items]
+ if items and all(t is not None for t in texts) and not re.sub(r"
.*?
|\s", "", inner, flags=re.S):
+ return "\n".join("- " + t for t in texts)
+ return raw
+
+
+def html_to_md(html):
+ """Convert the body of a page: the blocks of build.py output become Markdown, the rest stays HTML."""
+ parts, pos = [], 0
+ while True:
+ if not html[pos:].strip():
+ break
+ m = BLOCK.match(html, pos)
+ if not m:
+ raise ValueError(f"unconvertible HTML near: {html[pos:pos + 80].strip()!r}")
+ parts.append(block_to_md(m))
+ pos = m.end()
+ return "\n\n".join(parts) + "\n"
+
+
+def restore(md, values):
+ """Put {{key}} back where a value of site.json appears as text, longest values first."""
+ for key, value in sorted(values.items(), key=lambda kv: -len(kv[1])):
+ if value:
+ md = md.replace(value, "{{" + key + "}}")
+ return md
+
+
+def mark_latest_news(html):
+ """The news block of the home page becomes the {{latest_news}} placeholder."""
+ return re.sub(r'
.*?\n?', "
{{latest_news}}
\n", html, count=1, flags=re.S)
+
+
+def mark_news_list(html):
+ """The list of the news page becomes the {{news_list}} placeholder."""
+ return re.sub(r"
.*?
", "
{{news_list}}
", html, count=1, flags=re.S)
+
+
+# ---------------------------------------------------------------- pages and news
+
+def extract_content(page_html):
+ """The part of a built page between the header and the footer."""
+ m = re.search(r"\n\n(.*)\n
diff --git a/templates/news-post.html b/templates/news-post.html
deleted file mode 100644
index ff4d43f..0000000
--- a/templates/news-post.html
+++ /dev/null
@@ -1,7 +0,0 @@
-