kmalloc.org/tools/migrate.py
squid d2f9bba46b feat: move the pages to Markdown and the news to the dated layout
Pages are Markdown files in content/<lang>/ with a Key: value header, the
news carry Title, Author, Date, Categories and Description and are published
as /news/YYYY/MM/name.html, with a page for every category. The templates, the
language files and site.json shrink to the chrome and the shared strings, and
the footer credits motorino. The output matches the one of build.py page by
page, checked with tools/compare.py and the Nu HTML Checker.
2026-11-15 09:58:10 +01:00

256 lines
11 KiB
Python

#!/usr/bin/env python3
"""One-off migration of the kmalloc.org site from build.py to motorino.
Usage: migrate.py [--site DIR] [--old-out DIR]
It builds the current site with build.py (the reference, saved in --old-out), then writes the new layout in
place: pages as Markdown in content/<lang>/, news with the new header, the reduced site.json and lang files,
and the new templates. The pages are converted from the HTML that build.py produced, so their text is the
same by construction; HTML that Markdown cannot express (a table, an image, a heading with a class) is kept
as raw HTML in the file. Delete this tool once the new site is live."""
import argparse
import ast
import json
import os
import re
import shutil
import subprocess
import sys
# ---------------------------------------------------------------- HTML to Markdown
BLOCK = re.compile(r"\s*(?:(<!--.*?-->)|<(p|h[1-6]|ul|ol|table|blockquote|div)\b([^>]*)>(.*?)</\2>)", re.S)
ID_ONLY = re.compile(r'^\s*id="([^"]+)"\s*$')
def inline_to_md(inner):
"""Inline HTML as Markdown, or None when it has markup that Markdown cannot say."""
s = inner.strip()
s = re.sub(r'<a href="([^"]+)">(.*?)</a>', r"[\2](\1)", s, flags=re.S)
s = re.sub(r"<strong>(.*?)</strong>", r"**\1**", s, flags=re.S)
s = re.sub(r"<em>(.*?)</em>", r"*\1*", s, flags=re.S)
s = re.sub(r"<code>(.*?)</code>", r"`\1`", s, flags=re.S)
if "<" in s:
return None
return "\n".join(line.strip() for line in s.splitlines())
def block_to_md(match):
"""One top-level block as Markdown, or as the raw HTML it came from."""
raw = match.group(0).strip()
comment, tag, attrs, inner = match.groups()
if comment:
return raw
if tag == "p" and not attrs.strip():
text = inline_to_md(inner)
return text if text is not None else raw
if re.fullmatch(r"h[1-6]", tag):
text = inline_to_md(inner)
heading_id = ID_ONLY.match(attrs)
if text is not None and (not attrs.strip() or heading_id):
line = "#" * int(tag[1]) + " " + text
return line + (" {#" + heading_id.group(1) + "}" if heading_id else "")
return raw
if tag == "ul" and not attrs.strip():
items = re.findall(r"<li>(.*?)</li>", inner, flags=re.S)
texts = [inline_to_md(i) for i in items]
if items and all(t is not None for t in texts) and not re.sub(r"<li>.*?</li>|\s", "", inner, flags=re.S):
return "\n".join("- " + t for t in texts)
return raw
def html_to_md(html):
"""Convert the body of a page: the blocks of build.py output become Markdown, the rest stays HTML."""
parts, pos = [], 0
while True:
if not html[pos:].strip():
break
m = BLOCK.match(html, pos)
if not m:
raise ValueError(f"unconvertible HTML near: {html[pos:pos + 80].strip()!r}")
parts.append(block_to_md(m))
pos = m.end()
return "\n\n".join(parts) + "\n"
def restore(md, values):
"""Put {{key}} back where a value of site.json appears as text, longest values first."""
for key, value in sorted(values.items(), key=lambda kv: -len(kv[1])):
if value:
md = md.replace(value, "{{" + key + "}}")
return md
def mark_latest_news(html):
"""The news block of the home page becomes the {{latest_news}} placeholder."""
return re.sub(r'<h2 id="news">.*?</ul>\n?', "<p>{{latest_news}}</p>\n", html, count=1, flags=re.S)
def mark_news_list(html):
"""The list of the news page becomes the {{news_list}} placeholder."""
return re.sub(r"<ul>.*?</ul>", "<p>{{news_list}}</p>", html, count=1, flags=re.S)
# ---------------------------------------------------------------- pages and news
def extract_content(page_html):
"""The part of a built page between the header and the footer."""
m = re.search(r"</header>\n\n(.*)\n<footer>", page_html, re.S)
if not m:
raise ValueError("no content between header and footer")
return m.group(1).rstrip("\n") + "\n"
def page_file(path, default_lang, lang):
"""File of a page under content/<lang>/, from its address."""
if lang != default_lang and path.startswith(f"/{lang}/"):
path = path[len(lang) + 1:]
if path.endswith("/"):
return path.lstrip("/") + "index.md"
return path.lstrip("/")[: -len(".html")] + ".md"
def page_header(title, page_id, description, layout, note="", switch=None, manifesto_version="", manifesto_date=""):
"""The header of a page file."""
lines = [f"Title: {title}", f"ID: {page_id}", f"Description: {description}", f"Layout: {layout}"]
if note:
lines.append(f"Note: {note}")
if switch:
lines.append("Switch: " + ", ".join(f"{k}={v}" for k, v in switch.items()))
if manifesto_version:
lines.append(f"ManifestoVersion: {manifesto_version}")
if manifesto_date:
lines.append(f"ManifestoDate: {manifesto_date}")
return "\n".join(lines) + "\n-\n"
def convert_post(filename, text):
"""A news file with the old front matter becomes one with the new header."""
m = re.match(r"(\d{4}-\d{2}-\d{2})-(.+)\.md\Z", filename)
date, slug = m.group(1), m.group(2)
head = re.match(r"---\n(.*?)\n---\n(.*)\Z", text, re.S)
meta = dict(line.split(": ", 1) for line in head.group(1).splitlines())
lines = [f"Title: {meta['title']}", "Author: squid", f"Date: {date}T12:00:00+0100", "Categories: News"]
if meta.get("summary"):
lines.append(f"Description: {meta['summary']}")
return slug + ".md", "\n".join(lines) + "\n-\n" + head.group(2)
# ---------------------------------------------------------------- the migration
def write(path, text):
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w", encoding="utf-8") as f:
f.write(text)
def read(path):
with open(path, encoding="utf-8") as f:
return f.read()
def old_assets(build_py):
"""The ASSETS dict of build.py."""
m = re.search(r"^ASSETS = (\{.*?^\})", build_py, re.S | re.M)
return ast.literal_eval(m.group(1))
def site_values(site):
"""The literal values that go back to placeholders in the page text."""
shown = {k + "_shown": site[k].replace("@", " (at) ") for k in ("contact", "security", "abuse")}
keep = {k: site[k] for k in ("btc", "shop_url", "gpg_fingerprint", "gpg_key", "gpg_keyserver", "issues_url", "mastodon_url")}
return {**shown, **keep}
def migrate_pages(root, old_out, site, langs):
default = site["languages"][0]
values = site_values(site)
for lang in site["languages"]:
for pid, data in langs[lang]["pages"].items():
path = data["path"]
switch = data.get("switch")
if pid == "manifesto":
body = read(os.path.join(root, f"MANIFESTO.{lang}.md"))
body = re.sub(r"\A# .*\n\n?", "", body)
layout, note = "manifesto", data.get("note", "")
else:
out_file = os.path.join(old_out, path.lstrip("/"), "index.html") if path.endswith("/") else os.path.join(old_out, path.lstrip("/"))
html = extract_content(read(out_file))
if pid == "home":
html = mark_latest_news(html)
html = html.replace(f"({langs[lang]['version'].lower()} {site['manifesto_version']})",
"({{version_lc}} {{manifesto_version}})")
if pid == "news":
html = mark_news_list(html)
body = restore(html_to_md(html), values)
layout, note = "bare", ""
write(os.path.join(root, "content", lang, page_file(path, default, lang)),
page_header(data["title"], pid, data["description"], layout, note, switch,
data.get("manifesto_version", ""), data.get("manifesto_date", "")) + body)
def migrate_news(root):
folder = os.path.join(root, "news")
for name in sorted(os.listdir(folder)):
if re.match(r"\d{4}-\d{2}-\d{2}-.+\.md\Z", name):
new_name, text = convert_post(name, read(os.path.join(folder, name)))
write(os.path.join(folder, new_name), text)
os.remove(os.path.join(folder, name))
def migrate_config(root, site, langs, assets):
new_site = {k: v for k, v in site.items() if isinstance(v, str) or k == "languages"}
new_site["motorino_url"] = "https://git.kmalloc.org/squid/motorino"
new_site["assets"] = assets
write(os.path.join(root, "site.json"), json.dumps(new_site, indent=2, ensure_ascii=False) + "\n")
for lang, data in langs.items():
slim = {k: v for k, v in data.items() if k != "pages"}
slim["powered_by"] = "Powered by"
slim["category_h"] = {"en": "Category", "it": "Categoria"}.get(lang, "Category")
write(os.path.join(root, "lang", lang + ".json"), json.dumps(slim, indent=2, ensure_ascii=False) + "\n")
def migrate_templates(root):
tpl = os.path.join(root, "templates")
base = read(os.path.join(tpl, "base.html"))
base = base.replace('{{robots_meta}}<meta name="description"', '<meta name="description"')
base = base.replace('<meta name="color-scheme" content="light dark">\n',
'<meta name="color-scheme" content="light dark">\n<meta name="generator" content="motorino {{motorino_version}}">\n')
write(os.path.join(tpl, "base.html"), base)
footer = read(os.path.join(tpl, "footer.html"))
footer = footer.replace("{{badges}}", '<p>\n{{powered_by}} <a href="{{motorino_url}}">motorino</a>.\n</p>\n{{badges}}')
write(os.path.join(tpl, "footer.html"), footer)
manifesto = read(os.path.join(tpl, "pages", "manifesto.html"))
write(os.path.join(tpl, "pages", "manifesto.html"),
manifesto.replace("{{manifesto_title}}", "{{page_title}}").replace("{{manifesto_body}}", "{{body}}"))
write(os.path.join(tpl, "pages", "bare.html"), "{{body}}\n")
write(os.path.join(tpl, "pages", "page.html"), "<h1>{{page_title}}</h1>\n\n{{body}}\n")
write(os.path.join(tpl, "post.html"),
'<h1>{{post_title}}</h1>\n\n<p>\n<time datetime="{{date}}">{{date}}</time>{{post_byline}}\n</p>\n\n{{post_body}}\n')
write(os.path.join(tpl, "category.html"), "<h1>{{category_h}}: {{category_name}}</h1>\n\n{{category_posts}}\n")
for old in ("pages/home", "pages/people", "pages/services", "pages/news", "pages/history", "news-post"):
os.remove(os.path.join(tpl, old + ".html"))
shutil.rmtree(os.path.join(tpl, "parts"))
def main():
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--site", default=".")
ap.add_argument("--old-out", default="/tmp/kmalloc-old-out")
args = ap.parse_args()
root = os.path.abspath(args.site)
if os.path.exists(args.old_out):
shutil.rmtree(args.old_out)
subprocess.run([sys.executable, os.path.join(root, "build.py"), args.old_out], check=True)
site = json.loads(read(os.path.join(root, "site.json")))
langs = {l: json.loads(read(os.path.join(root, "lang", l + ".json"))) for l in site["languages"]}
assets = old_assets(read(os.path.join(root, "build.py")))
migrate_pages(root, args.old_out, site, langs)
migrate_news(root)
migrate_config(root, site, langs, assets)
migrate_templates(root)
print(f"migrated; the reference output is in {args.old_out}")
if __name__ == "__main__":
main()