251 lines
11 KiB
Python
251 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""One-off migration of the kmalloc.org site from build.py to motorino.
|
|
|
|
Usage: migrate.py [--site DIR] [--old-out DIR]
|
|
|
|
It builds the current site with build.py (the reference, saved in --old-out), then writes the new layout in
|
|
place: pages as Markdown in content/<lang>/, news with the new header, the reduced site.json and lang files,
|
|
and the new templates. The pages are converted from the HTML that build.py produced, so their text is the
|
|
same by construction; HTML that Markdown cannot express (a table, an image, a heading with a class) is kept
|
|
as raw HTML in the file. Delete this tool once the new site is live."""
|
|
import argparse
|
|
import ast
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
|
|
# ---------------------------------------------------------------- HTML to Markdown
|
|
|
|
BLOCK = re.compile(r"\s*(?:(<!--.*?-->)|<(p|h[1-6]|ul|ol|table|blockquote|div)\b([^>]*)>(.*?)</\2>)", re.S)
|
|
ID_ONLY = re.compile(r'^\s*id="([^"]+)"\s*$')
|
|
|
|
|
|
def inline_to_md(inner):
|
|
"""Inline HTML as Markdown, or None when it has markup that Markdown cannot say."""
|
|
s = inner.strip()
|
|
s = re.sub(r'<a href="([^"]+)">(.*?)</a>', r"[\2](\1)", s, flags=re.S)
|
|
s = re.sub(r"<strong>(.*?)</strong>", r"**\1**", s, flags=re.S)
|
|
s = re.sub(r"<em>(.*?)</em>", r"*\1*", s, flags=re.S)
|
|
s = re.sub(r"<code>(.*?)</code>", r"`\1`", s, flags=re.S)
|
|
if "<" in s:
|
|
return None
|
|
return "\n".join(line.strip() for line in s.splitlines())
|
|
|
|
|
|
def block_to_md(match):
|
|
"""One top-level block as Markdown, or as the raw HTML it came from."""
|
|
raw = match.group(0).strip()
|
|
comment, tag, attrs, inner = match.groups()
|
|
if comment:
|
|
return raw
|
|
if tag == "p" and not attrs.strip():
|
|
text = inline_to_md(inner)
|
|
return text if text is not None else raw
|
|
if re.fullmatch(r"h[1-6]", tag):
|
|
text = inline_to_md(inner)
|
|
heading_id = ID_ONLY.match(attrs)
|
|
if text is not None and (not attrs.strip() or heading_id):
|
|
line = "#" * int(tag[1]) + " " + text
|
|
return line + (" {#" + heading_id.group(1) + "}" if heading_id else "")
|
|
return raw
|
|
if tag == "ul" and not attrs.strip():
|
|
items = re.findall(r"<li>(.*?)</li>", inner, flags=re.S)
|
|
texts = [inline_to_md(i) for i in items]
|
|
if items and all(t is not None for t in texts) and not re.sub(r"<li>.*?</li>|\s", "", inner, flags=re.S):
|
|
return "\n".join("- " + t for t in texts)
|
|
return raw
|
|
|
|
|
|
def html_to_md(html):
|
|
"""Convert the body of a page: the blocks of build.py output become Markdown, the rest stays HTML."""
|
|
parts, pos = [], 0
|
|
while True:
|
|
if not html[pos:].strip():
|
|
break
|
|
m = BLOCK.match(html, pos)
|
|
if not m:
|
|
raise ValueError(f"unconvertible HTML near: {html[pos:pos + 80].strip()!r}")
|
|
parts.append(block_to_md(m))
|
|
pos = m.end()
|
|
return "\n\n".join(parts) + "\n"
|
|
|
|
|
|
def restore(md, values):
|
|
"""Put {{key}} back where a value of site.json appears as text, longest values first."""
|
|
for key, value in sorted(values.items(), key=lambda kv: -len(kv[1])):
|
|
if value:
|
|
md = md.replace(value, "{{" + key + "}}")
|
|
return md
|
|
|
|
|
|
def mark_latest_news(html):
|
|
"""The news block of the home page becomes the {{latest_news}} placeholder."""
|
|
return re.sub(r'<h2 id="news">.*?</ul>\n?', "<p>{{latest_news}}</p>\n", html, count=1, flags=re.S)
|
|
|
|
|
|
def mark_news_list(html):
|
|
"""The list of the news page becomes the {{news_list}} placeholder."""
|
|
return re.sub(r"<ul>.*?</ul>", "<p>{{news_list}}</p>", html, count=1, flags=re.S)
|
|
|
|
|
|
# ---------------------------------------------------------------- pages and news
|
|
|
|
def extract_content(page_html):
|
|
"""The part of a built page between the header and the footer."""
|
|
m = re.search(r"</header>\n\n(.*)\n<footer>", page_html, re.S)
|
|
if not m:
|
|
raise ValueError("no content between header and footer")
|
|
return m.group(1).rstrip("\n") + "\n"
|
|
|
|
|
|
def page_file(path, default_lang, lang):
|
|
"""File of a page under content/<lang>/, from its address."""
|
|
if lang != default_lang and path.startswith(f"/{lang}/"):
|
|
path = path[len(lang) + 1:]
|
|
if path.endswith("/"):
|
|
return path.lstrip("/") + "index.md"
|
|
return path.lstrip("/")[: -len(".html")] + ".md"
|
|
|
|
|
|
def page_header(title, page_id, description, layout, note="", switch=None):
|
|
"""The header of a page file."""
|
|
lines = [f"Title: {title}", f"ID: {page_id}", f"Description: {description}", f"Layout: {layout}"]
|
|
if note:
|
|
lines.append(f"Note: {note}")
|
|
if switch:
|
|
lines.append("Switch: " + ", ".join(f"{k}={v}" for k, v in switch.items()))
|
|
return "\n".join(lines) + "\n-\n"
|
|
|
|
|
|
def convert_post(filename, text):
|
|
"""A news file with the old front matter becomes one with the new header."""
|
|
m = re.match(r"(\d{4}-\d{2}-\d{2})-(.+)\.md\Z", filename)
|
|
date, slug = m.group(1), m.group(2)
|
|
head = re.match(r"---\n(.*?)\n---\n(.*)\Z", text, re.S)
|
|
meta = dict(line.split(": ", 1) for line in head.group(1).splitlines())
|
|
lines = [f"Title: {meta['title']}", "Author: squid", f"Date: {date}T12:00:00+0100", "Categories: News"]
|
|
if meta.get("summary"):
|
|
lines.append(f"Description: {meta['summary']}")
|
|
return slug + ".md", "\n".join(lines) + "\n-\n" + head.group(2)
|
|
|
|
|
|
# ---------------------------------------------------------------- the migration
|
|
|
|
def write(path, text):
|
|
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
f.write(text)
|
|
|
|
|
|
def read(path):
|
|
with open(path, encoding="utf-8") as f:
|
|
return f.read()
|
|
|
|
|
|
def old_assets(build_py):
|
|
"""The ASSETS dict of build.py."""
|
|
m = re.search(r"^ASSETS = (\{.*?^\})", build_py, re.S | re.M)
|
|
return ast.literal_eval(m.group(1))
|
|
|
|
|
|
def site_values(site):
|
|
"""The literal values that go back to placeholders in the page text."""
|
|
shown = {k + "_shown": site[k].replace("@", " (at) ") for k in ("contact", "security", "abuse")}
|
|
keep = {k: site[k] for k in ("btc", "shop_url", "gpg_fingerprint", "gpg_key", "gpg_keyserver", "issues_url", "mastodon_url")}
|
|
return {**shown, **keep}
|
|
|
|
|
|
def migrate_pages(root, old_out, site, langs):
|
|
default = site["languages"][0]
|
|
values = site_values(site)
|
|
for lang in site["languages"]:
|
|
for pid, data in langs[lang]["pages"].items():
|
|
path = data["path"]
|
|
switch = data.get("switch")
|
|
if pid == "manifesto":
|
|
body = read(os.path.join(root, f"MANIFESTO.{lang}.md"))
|
|
body = re.sub(r"\A# .*\n\n?", "", body)
|
|
layout, note = "manifesto", data.get("note", "")
|
|
else:
|
|
out_file = os.path.join(old_out, path.lstrip("/"), "index.html") if path.endswith("/") else os.path.join(old_out, path.lstrip("/"))
|
|
html = extract_content(read(out_file))
|
|
if pid == "home":
|
|
html = mark_latest_news(html)
|
|
html = html.replace(f"({langs[lang]['version'].lower()} {site['manifesto_version']})",
|
|
"({{version_lc}} {{manifesto_version}})")
|
|
if pid == "news":
|
|
html = mark_news_list(html)
|
|
body = restore(html_to_md(html), values)
|
|
layout, note = "bare", ""
|
|
write(os.path.join(root, "content", lang, page_file(path, default, lang)),
|
|
page_header(data["title"], pid, data["description"], layout, note, switch) + body)
|
|
|
|
|
|
def migrate_news(root):
|
|
folder = os.path.join(root, "news")
|
|
for name in sorted(os.listdir(folder)):
|
|
if re.match(r"\d{4}-\d{2}-\d{2}-.+\.md\Z", name):
|
|
new_name, text = convert_post(name, read(os.path.join(folder, name)))
|
|
write(os.path.join(folder, new_name), text)
|
|
os.remove(os.path.join(folder, name))
|
|
|
|
|
|
def migrate_config(root, site, langs, assets):
|
|
new_site = {k: v for k, v in site.items() if isinstance(v, str) or k == "languages"}
|
|
new_site["motorino_url"] = "https://git.kmalloc.org/squid/motorino"
|
|
new_site["assets"] = assets
|
|
write(os.path.join(root, "site.json"), json.dumps(new_site, indent=2, ensure_ascii=False) + "\n")
|
|
for lang, data in langs.items():
|
|
slim = {k: v for k, v in data.items() if k != "pages"}
|
|
slim["powered_by"] = "Powered by"
|
|
slim["category_h"] = {"en": "Category", "it": "Categoria"}.get(lang, "Category")
|
|
write(os.path.join(root, "lang", lang + ".json"), json.dumps(slim, indent=2, ensure_ascii=False) + "\n")
|
|
|
|
|
|
def migrate_templates(root):
|
|
tpl = os.path.join(root, "templates")
|
|
base = read(os.path.join(tpl, "base.html"))
|
|
base = base.replace('{{robots_meta}}<meta name="description"', '<meta name="description"')
|
|
base = base.replace('<meta name="color-scheme" content="light dark">\n',
|
|
'<meta name="color-scheme" content="light dark">\n<meta name="generator" content="motorino {{motorino_version}}">\n')
|
|
write(os.path.join(tpl, "base.html"), base)
|
|
footer = read(os.path.join(tpl, "footer.html"))
|
|
footer = footer.replace("{{badges}}", '<p>\n{{powered_by}} <a href="{{motorino_url}}">motorino</a>.\n</p>\n{{badges}}')
|
|
write(os.path.join(tpl, "footer.html"), footer)
|
|
manifesto = read(os.path.join(tpl, "pages", "manifesto.html"))
|
|
write(os.path.join(tpl, "pages", "manifesto.html"),
|
|
manifesto.replace("{{manifesto_title}}", "{{page_title}}").replace("{{manifesto_body}}", "{{body}}"))
|
|
write(os.path.join(tpl, "pages", "bare.html"), "{{body}}\n")
|
|
write(os.path.join(tpl, "pages", "page.html"), "<h1>{{page_title}}</h1>\n\n{{body}}\n")
|
|
write(os.path.join(tpl, "post.html"),
|
|
'<h1>{{post_title}}</h1>\n\n<p>\n<time datetime="{{date}}">{{date}}</time>{{post_byline}}\n</p>\n\n{{post_body}}\n')
|
|
write(os.path.join(tpl, "category.html"), "<h1>{{category_h}}: {{category_name}}</h1>\n\n{{category_posts}}\n")
|
|
for old in ("pages/home", "pages/people", "pages/services", "pages/news", "pages/history", "news-post"):
|
|
os.remove(os.path.join(tpl, old + ".html"))
|
|
shutil.rmtree(os.path.join(tpl, "parts"))
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(description=__doc__)
|
|
ap.add_argument("--site", default=".")
|
|
ap.add_argument("--old-out", default="/tmp/kmalloc-old-out")
|
|
args = ap.parse_args()
|
|
root = os.path.abspath(args.site)
|
|
if os.path.exists(args.old_out):
|
|
shutil.rmtree(args.old_out)
|
|
subprocess.run([sys.executable, os.path.join(root, "build.py"), args.old_out], check=True)
|
|
site = json.loads(read(os.path.join(root, "site.json")))
|
|
langs = {l: json.loads(read(os.path.join(root, "lang", l + ".json"))) for l in site["languages"]}
|
|
assets = old_assets(read(os.path.join(root, "build.py")))
|
|
migrate_pages(root, args.old_out, site, langs)
|
|
migrate_news(root)
|
|
migrate_config(root, site, langs, assets)
|
|
migrate_templates(root)
|
|
print(f"migrated; the reference output is in {args.old_out}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|