Pages are Markdown files in content/<lang>/ with a Key: value header, the news carry Title, Author, Date, Categories and Description and are published as /news/YYYY/MM/name.html, with a page for every category. The templates, the language files and site.json shrink to the chrome and the shared strings, and the footer credits motorino. The output matches the one of build.py page by page, checked with tools/compare.py and the Nu HTML Checker.
256 lines
11 KiB
Python
256 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""One-off migration of the kmalloc.org site from build.py to motorino.
|
|
|
|
Usage: migrate.py [--site DIR] [--old-out DIR]
|
|
|
|
It builds the current site with build.py (the reference, saved in --old-out), then writes the new layout in
|
|
place: pages as Markdown in content/<lang>/, news with the new header, the reduced site.json and lang files,
|
|
and the new templates. The pages are converted from the HTML that build.py produced, so their text is the
|
|
same by construction; HTML that Markdown cannot express (a table, an image, a heading with a class) is kept
|
|
as raw HTML in the file. Delete this tool once the new site is live."""
|
|
import argparse
|
|
import ast
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
|
|
# ---------------------------------------------------------------- HTML to Markdown
|
|
|
|
BLOCK = re.compile(r"\s*(?:(<!--.*?-->)|<(p|h[1-6]|ul|ol|table|blockquote|div)\b([^>]*)>(.*?)</\2>)", re.S)
|
|
ID_ONLY = re.compile(r'^\s*id="([^"]+)"\s*$')
|
|
|
|
|
|
def inline_to_md(inner):
|
|
"""Inline HTML as Markdown, or None when it has markup that Markdown cannot say."""
|
|
s = inner.strip()
|
|
s = re.sub(r'<a href="([^"]+)">(.*?)</a>', r"[\2](\1)", s, flags=re.S)
|
|
s = re.sub(r"<strong>(.*?)</strong>", r"**\1**", s, flags=re.S)
|
|
s = re.sub(r"<em>(.*?)</em>", r"*\1*", s, flags=re.S)
|
|
s = re.sub(r"<code>(.*?)</code>", r"`\1`", s, flags=re.S)
|
|
if "<" in s:
|
|
return None
|
|
return "\n".join(line.strip() for line in s.splitlines())
|
|
|
|
|
|
def block_to_md(match):
|
|
"""One top-level block as Markdown, or as the raw HTML it came from."""
|
|
raw = match.group(0).strip()
|
|
comment, tag, attrs, inner = match.groups()
|
|
if comment:
|
|
return raw
|
|
if tag == "p" and not attrs.strip():
|
|
text = inline_to_md(inner)
|
|
return text if text is not None else raw
|
|
if re.fullmatch(r"h[1-6]", tag):
|
|
text = inline_to_md(inner)
|
|
heading_id = ID_ONLY.match(attrs)
|
|
if text is not None and (not attrs.strip() or heading_id):
|
|
line = "#" * int(tag[1]) + " " + text
|
|
return line + (" {#" + heading_id.group(1) + "}" if heading_id else "")
|
|
return raw
|
|
if tag == "ul" and not attrs.strip():
|
|
items = re.findall(r"<li>(.*?)</li>", inner, flags=re.S)
|
|
texts = [inline_to_md(i) for i in items]
|
|
if items and all(t is not None for t in texts) and not re.sub(r"<li>.*?</li>|\s", "", inner, flags=re.S):
|
|
return "\n".join("- " + t for t in texts)
|
|
return raw
|
|
|
|
|
|
def html_to_md(html):
|
|
"""Convert the body of a page: the blocks of build.py output become Markdown, the rest stays HTML."""
|
|
parts, pos = [], 0
|
|
while True:
|
|
if not html[pos:].strip():
|
|
break
|
|
m = BLOCK.match(html, pos)
|
|
if not m:
|
|
raise ValueError(f"unconvertible HTML near: {html[pos:pos + 80].strip()!r}")
|
|
parts.append(block_to_md(m))
|
|
pos = m.end()
|
|
return "\n\n".join(parts) + "\n"
|
|
|
|
|
|
def restore(md, values):
|
|
"""Put {{key}} back where a value of site.json appears as text, longest values first."""
|
|
for key, value in sorted(values.items(), key=lambda kv: -len(kv[1])):
|
|
if value:
|
|
md = md.replace(value, "{{" + key + "}}")
|
|
return md
|
|
|
|
|
|
def mark_latest_news(html):
|
|
"""The news block of the home page becomes the {{latest_news}} placeholder."""
|
|
return re.sub(r'<h2 id="news">.*?</ul>\n?', "<p>{{latest_news}}</p>\n", html, count=1, flags=re.S)
|
|
|
|
|
|
def mark_news_list(html):
|
|
"""The list of the news page becomes the {{news_list}} placeholder."""
|
|
return re.sub(r"<ul>.*?</ul>", "<p>{{news_list}}</p>", html, count=1, flags=re.S)
|
|
|
|
|
|
# ---------------------------------------------------------------- pages and news
|
|
|
|
def extract_content(page_html):
|
|
"""The part of a built page between the header and the footer."""
|
|
m = re.search(r"</header>\n\n(.*)\n<footer>", page_html, re.S)
|
|
if not m:
|
|
raise ValueError("no content between header and footer")
|
|
return m.group(1).rstrip("\n") + "\n"
|
|
|
|
|
|
def page_file(path, default_lang, lang):
|
|
"""File of a page under content/<lang>/, from its address."""
|
|
if lang != default_lang and path.startswith(f"/{lang}/"):
|
|
path = path[len(lang) + 1:]
|
|
if path.endswith("/"):
|
|
return path.lstrip("/") + "index.md"
|
|
return path.lstrip("/")[: -len(".html")] + ".md"
|
|
|
|
|
|
def page_header(title, page_id, description, layout, note="", switch=None, manifesto_version="", manifesto_date=""):
|
|
"""The header of a page file."""
|
|
lines = [f"Title: {title}", f"ID: {page_id}", f"Description: {description}", f"Layout: {layout}"]
|
|
if note:
|
|
lines.append(f"Note: {note}")
|
|
if switch:
|
|
lines.append("Switch: " + ", ".join(f"{k}={v}" for k, v in switch.items()))
|
|
if manifesto_version:
|
|
lines.append(f"ManifestoVersion: {manifesto_version}")
|
|
if manifesto_date:
|
|
lines.append(f"ManifestoDate: {manifesto_date}")
|
|
return "\n".join(lines) + "\n-\n"
|
|
|
|
|
|
def convert_post(filename, text):
|
|
"""A news file with the old front matter becomes one with the new header."""
|
|
m = re.match(r"(\d{4}-\d{2}-\d{2})-(.+)\.md\Z", filename)
|
|
date, slug = m.group(1), m.group(2)
|
|
head = re.match(r"---\n(.*?)\n---\n(.*)\Z", text, re.S)
|
|
meta = dict(line.split(": ", 1) for line in head.group(1).splitlines())
|
|
lines = [f"Title: {meta['title']}", "Author: squid", f"Date: {date}T12:00:00+0100", "Categories: News"]
|
|
if meta.get("summary"):
|
|
lines.append(f"Description: {meta['summary']}")
|
|
return slug + ".md", "\n".join(lines) + "\n-\n" + head.group(2)
|
|
|
|
|
|
# ---------------------------------------------------------------- the migration
|
|
|
|
def write(path, text):
|
|
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
f.write(text)
|
|
|
|
|
|
def read(path):
|
|
with open(path, encoding="utf-8") as f:
|
|
return f.read()
|
|
|
|
|
|
def old_assets(build_py):
|
|
"""The ASSETS dict of build.py."""
|
|
m = re.search(r"^ASSETS = (\{.*?^\})", build_py, re.S | re.M)
|
|
return ast.literal_eval(m.group(1))
|
|
|
|
|
|
def site_values(site):
|
|
"""The literal values that go back to placeholders in the page text."""
|
|
shown = {k + "_shown": site[k].replace("@", " (at) ") for k in ("contact", "security", "abuse")}
|
|
keep = {k: site[k] for k in ("btc", "shop_url", "gpg_fingerprint", "gpg_key", "gpg_keyserver", "issues_url", "mastodon_url")}
|
|
return {**shown, **keep}
|
|
|
|
|
|
def migrate_pages(root, old_out, site, langs):
|
|
default = site["languages"][0]
|
|
values = site_values(site)
|
|
for lang in site["languages"]:
|
|
for pid, data in langs[lang]["pages"].items():
|
|
path = data["path"]
|
|
switch = data.get("switch")
|
|
if pid == "manifesto":
|
|
body = read(os.path.join(root, f"MANIFESTO.{lang}.md"))
|
|
body = re.sub(r"\A# .*\n\n?", "", body)
|
|
layout, note = "manifesto", data.get("note", "")
|
|
else:
|
|
out_file = os.path.join(old_out, path.lstrip("/"), "index.html") if path.endswith("/") else os.path.join(old_out, path.lstrip("/"))
|
|
html = extract_content(read(out_file))
|
|
if pid == "home":
|
|
html = mark_latest_news(html)
|
|
html = html.replace(f"({langs[lang]['version'].lower()} {site['manifesto_version']})",
|
|
"({{version_lc}} {{manifesto_version}})")
|
|
if pid == "news":
|
|
html = mark_news_list(html)
|
|
body = restore(html_to_md(html), values)
|
|
layout, note = "bare", ""
|
|
write(os.path.join(root, "content", lang, page_file(path, default, lang)),
|
|
page_header(data["title"], pid, data["description"], layout, note, switch,
|
|
data.get("manifesto_version", ""), data.get("manifesto_date", "")) + body)
|
|
|
|
|
|
def migrate_news(root):
|
|
folder = os.path.join(root, "news")
|
|
for name in sorted(os.listdir(folder)):
|
|
if re.match(r"\d{4}-\d{2}-\d{2}-.+\.md\Z", name):
|
|
new_name, text = convert_post(name, read(os.path.join(folder, name)))
|
|
write(os.path.join(folder, new_name), text)
|
|
os.remove(os.path.join(folder, name))
|
|
|
|
|
|
def migrate_config(root, site, langs, assets):
|
|
new_site = {k: v for k, v in site.items() if isinstance(v, str) or k == "languages"}
|
|
new_site["motorino_url"] = "https://git.kmalloc.org/squid/motorino"
|
|
new_site["assets"] = assets
|
|
write(os.path.join(root, "site.json"), json.dumps(new_site, indent=2, ensure_ascii=False) + "\n")
|
|
for lang, data in langs.items():
|
|
slim = {k: v for k, v in data.items() if k != "pages"}
|
|
slim["powered_by"] = "Powered by"
|
|
slim["category_h"] = {"en": "Category", "it": "Categoria"}.get(lang, "Category")
|
|
write(os.path.join(root, "lang", lang + ".json"), json.dumps(slim, indent=2, ensure_ascii=False) + "\n")
|
|
|
|
|
|
def migrate_templates(root):
|
|
tpl = os.path.join(root, "templates")
|
|
base = read(os.path.join(tpl, "base.html"))
|
|
base = base.replace('{{robots_meta}}<meta name="description"', '<meta name="description"')
|
|
base = base.replace('<meta name="color-scheme" content="light dark">\n',
|
|
'<meta name="color-scheme" content="light dark">\n<meta name="generator" content="motorino {{motorino_version}}">\n')
|
|
write(os.path.join(tpl, "base.html"), base)
|
|
footer = read(os.path.join(tpl, "footer.html"))
|
|
footer = footer.replace("{{badges}}", '<p>\n{{powered_by}} <a href="{{motorino_url}}">motorino</a>.\n</p>\n{{badges}}')
|
|
write(os.path.join(tpl, "footer.html"), footer)
|
|
manifesto = read(os.path.join(tpl, "pages", "manifesto.html"))
|
|
write(os.path.join(tpl, "pages", "manifesto.html"),
|
|
manifesto.replace("{{manifesto_title}}", "{{page_title}}").replace("{{manifesto_body}}", "{{body}}"))
|
|
write(os.path.join(tpl, "pages", "bare.html"), "{{body}}\n")
|
|
write(os.path.join(tpl, "pages", "page.html"), "<h1>{{page_title}}</h1>\n\n{{body}}\n")
|
|
write(os.path.join(tpl, "post.html"),
|
|
'<h1>{{post_title}}</h1>\n\n<p>\n<time datetime="{{date}}">{{date}}</time>{{post_byline}}\n</p>\n\n{{post_body}}\n')
|
|
write(os.path.join(tpl, "category.html"), "<h1>{{category_h}}: {{category_name}}</h1>\n\n{{category_posts}}\n")
|
|
for old in ("pages/home", "pages/people", "pages/services", "pages/news", "pages/history", "news-post"):
|
|
os.remove(os.path.join(tpl, old + ".html"))
|
|
shutil.rmtree(os.path.join(tpl, "parts"))
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(description=__doc__)
|
|
ap.add_argument("--site", default=".")
|
|
ap.add_argument("--old-out", default="/tmp/kmalloc-old-out")
|
|
args = ap.parse_args()
|
|
root = os.path.abspath(args.site)
|
|
if os.path.exists(args.old_out):
|
|
shutil.rmtree(args.old_out)
|
|
subprocess.run([sys.executable, os.path.join(root, "build.py"), args.old_out], check=True)
|
|
site = json.loads(read(os.path.join(root, "site.json")))
|
|
langs = {l: json.loads(read(os.path.join(root, "lang", l + ".json"))) for l in site["languages"]}
|
|
assets = old_assets(read(os.path.join(root, "build.py")))
|
|
migrate_pages(root, args.old_out, site, langs)
|
|
migrate_news(root)
|
|
migrate_config(root, site, langs, assets)
|
|
migrate_templates(root)
|
|
print(f"migrated; the reference output is in {args.old_out}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|