kmalloc.org/build.py
squid 6464fac2c7 fix: publish logo.png, icon-512.png and logo-light|dark.svg from the build
These files were copied by hand to the server, so they kept the old logo.
Publish them from the logo set so every public URL serves the new shrike.
2026-11-13 17:48:36 +01:00

437 lines
20 KiB
Python

#!/usr/bin/env python3
"""Build the kmalloc.org static site (plain HTML, no JavaScript, standard library only).
Usage: build.py [output-dir] (default: preview)
Layout:
site.json what every language shares: languages, contact, Bitcoin address, services, people
lang/<lang>.json every text of one language: menu, footer, and one entry per page it has
templates/ page chrome (base, header, footer), page bodies (pages/) and repeated blocks (parts/)
static/style.css the one stylesheet, inlined into every page
MANIFESTO.<lang>.md the manifesto, one Markdown file per language
news/*.md news posts, English only: YYYY-MM-DD-slug.md, front matter, then Markdown
To add a language, add lang/<lang>.json and list it in site.json. A language needs only the pages it has:
a page that is missing from lang/<lang>.json is simply not built, and its language links disappear.
Texts in the JSON files are plain text: the build escapes them. Placeholders {{name}} in the templates are
replaced with values from site.json, the language file and the page; an unknown placeholder stops the build."""
import html, json, os, re, shutil, sys
from datetime import datetime, timezone
from email.utils import format_datetime
from xml.sax.saxutils import escape as xml_escape
HERE = os.path.dirname(os.path.abspath(__file__))
OUT = os.path.abspath(sys.argv[1] if len(sys.argv) > 1 else os.path.join(HERE, "preview"))
NEWS_ON_HOME = 3
NEWS_IN_FEED = 20
POST_NAME = re.compile(r"^(\d{4}-\d{2}-\d{2})-([a-z0-9]+(?:-[a-z0-9]+)*)$")
# Page types: menu entry to highlight and footer lead. The body is templates/pages/<id>.html.
PAGES = {
"home": dict(nav="home", lead="home"),
"manifesto": dict(nav="manifesto", lead="back"),
"people": dict(nav="people", lead="back"),
"services": dict(nav="services", lead="back"),
"news": dict(nav="news", lead="back"),
"history": dict(nav="history", lead="back"),
}
NAV = ("home", "manifesto", "people", "services", "news")
# Only these pages may be indexed by search engines; every other page gets noindex and is blocked in robots.txt.
INDEXABLE = ("home", "manifesto")
NOINDEX = '<meta name="robots" content="noindex, nofollow, noarchive">\n'
# Files copied as they are: published name -> source.
ASSETS = {
"favicon.svg": "logo/svg/favicon-auto.svg",
"favicon-auto.svg": "logo/svg/favicon-auto.svg",
"logo-auto.svg": "logo/svg/mark-auto.svg",
"logo-light.svg": "logo/svg/mark-light.svg",
"logo-dark.svg": "logo/svg/mark-dark.svg",
"logo.png": "logo/png/icon-light-512.png",
"icon-512.png": "logo/png/icon-light-512.png",
"favicon.ico": "logo/web/favicon.ico",
"apple-touch-icon.png": "logo/web/apple-touch-icon.png",
"badges/w3c-html5.png": "badges/w3c-html5.png",
"badges/w3c-css.png": "badges/w3c-css.png",
"squid.asc": "static/squid.asc",
}
def fail(msg):
raise SystemExit(f"build.py: {msg}")
def read(*parts):
with open(os.path.join(HERE, *parts), encoding="utf-8") as f:
return f.read()
def write(rel, text):
path = os.path.join(OUT, rel)
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w", encoding="utf-8") as f:
f.write(text)
def load_json(*parts):
try:
return json.loads(read(*parts))
except ValueError as e:
fail(f"{'/'.join(parts)}: {e}")
def esc(text):
return html.escape(text, quote=False)
def parse(text, name):
"""Split 'front matter, then body' into (dict, body)."""
m = re.match(r"---\n(.*?)\n---\n(.*)\Z", text, re.S)
if not m:
fail(f"{name}: missing front matter")
meta = {}
for line in m.group(1).splitlines():
key, sep, value = line.partition(": ")
if not sep:
fail(f"{name}: bad front matter line {line!r}")
meta[key.strip()] = value.strip()
return meta, m.group(2)
def render(text, ctx, name):
def sub(m):
if m.group(1) not in ctx:
fail(f"{name}: unknown placeholder {{{{{m.group(1)}}}}}")
return ctx[m.group(1)]
return re.sub(r"\{\{(\w+)\}\}", sub, text)
# ---------------------------------------------------------------- Markdown
def no_em_dash(md):
"""The manifesto sources use em dashes; the site does not: headings 'NN — Title' become 'NN. Title',
'Word — Title' becomes 'Word: Title', attribution lines lose the dash, parenthetical dashes become commas."""
md = re.sub(r"^(#+ )(\d\d) — ", r"\1\2. ", md, flags=re.M)
md = re.sub(r"^(#+ [^\d\n][^—\n]*?) — ", r"\1: ", md, flags=re.M)
md = re.sub(r"^> — ", "> ", md, flags=re.M)
for a, b in ((" — where technically feasible — ", ", where technically feasible, "),
(" — dove tecnicamente possibile — ", ", dove tecnicamente possibile, "),
(" — and continues", ", and continues"),
(" — e continua", ", e continua")):
md = md.replace(a, b)
md = md.replace("kmalloc.org — ", "kmalloc.org: ")
assert "—" not in md, [l for l in md.splitlines() if "—" in l]
return md
def inline(s):
s = html.escape(s, quote=False)
s = re.sub(r"`([^`]+)`", r"<code>\1</code>", s)
s = re.sub(r"\[([^\]]+)\]\(([^)\s]+)\)", r'<a href="\2">\1</a>', s)
s = re.sub(r"\*\*(.+?)\*\*", r"<strong>\1</strong>", s)
s = re.sub(r"(?<!\*)\*(?!\s)(.+?)(?<!\s)\*(?!\*)", r"<em>\1</em>", s)
return s
def slug(heading):
m = re.match(r"(\d\d)\b", heading)
if m:
return m.group(1)
h = heading.lower()
if h.startswith(("preamble", "preambolo", "preámbulo", "präambel", "préambule")):
return "preamble"
if h.startswith(("our commitments", "i nostri impegni", "nuestros compromisos", "unsere verpflichtungen", "nos engagements")):
return "commitments"
if h.startswith(("epilogue", "epilogo", "epílogo", "epilog", "épilogue")):
return "epilogue"
return re.sub(r"[^a-z0-9]+", "-", h).strip("-")
def md_to_html(md):
"""Headings (##), paragraphs, quotes, bullet lists and inline markup. Returns (title, toc, html)."""
out, toc = [], []
para, quote, items = [], [], []
def flush():
nonlocal para, quote, items
if para:
out.append("<p>\n" + "\n".join(para) + "\n</p>\n")
para = []
if quote:
out.append("<blockquote>\n<p>\n" + "\n".join(quote) + "\n</p>\n</blockquote>\n")
quote = []
if items:
out.append("<ul>\n" + "\n".join(f"<li>{i}</li>" for i in items) + "\n</ul>\n")
items = []
title = ""
for raw in md.splitlines():
line = raw.rstrip("\n")
br = line.endswith(" ")
s = line.strip()
if not s:
flush()
continue
if s.startswith("# "):
flush()
title = s[2:]
continue
if s.startswith("## "):
flush()
h = s[3:]
sl = slug(h)
toc.append((sl, h))
out.append(f'<h2 id="{sl}">{inline(h)}</h2>\n')
continue
if s.startswith("> "):
if para or items:
flush()
quote.append(inline(s[2:]) + ("<br>" if br else ""))
continue
if s.startswith("- "):
if para or quote:
flush()
items.append(inline(s[2:]))
continue
if quote or items:
flush()
para.append(inline(s))
flush()
return title, toc, "".join(out)
# ---------------------------------------------------------------- site model
def load_posts():
"""News posts from news/*.md (English only)."""
posts = []
folder = os.path.join(HERE, "news")
for fn in sorted(os.listdir(folder)) if os.path.isdir(folder) else []:
if not fn.endswith(".md"):
continue
m = POST_NAME.match(fn[:-3])
if not m:
fail(f"news/{fn}: name must be YYYY-MM-DD-slug.md")
src = f"news/{fn}"
meta, text = parse(read("news", fn), src)
if "title" not in meta:
fail(f"{src}: front matter lacks 'title'")
if "\u2014" in text or "\u2014" in meta["title"]:
fail(f"{src}: no em dashes, please")
posts.append(dict(lang="en", kind="post", id="news/" + fn[:-3], date=m.group(1), nav="news",
path=f"/news/{m.group(2)}.html", body=text, src=src, title=meta["title"],
summary=meta.get("summary", ""), description=meta.get("summary") or meta["title"]))
return sorted(posts, key=lambda p: p["date"], reverse=True)
def rss(L, site, all_posts):
"""RSS 2.0 feed of the news (English only): the latest posts, full text, absolute links."""
base = site["base_url"]
posts = all_posts[:NEWS_IN_FEED]
def stamp(d):
return format_datetime(datetime.strptime(d, "%Y-%m-%d").replace(tzinfo=timezone.utc))
items = []
for p in posts:
text = md_to_html(p["body"])[2].replace('href="/', f'href="{base}/')
items.append(f"""<item>
<title>{xml_escape(p["title"])}</title>
<link>{base}{p["path"]}</link>
<guid isPermaLink="true">{base}{p["path"]}</guid>
<pubDate>{stamp(p["date"])}</pubDate>
<description>{xml_escape(text)}</description>
</item>
""")
built = f"<lastBuildDate>{stamp(posts[0]['date'])}</lastBuildDate>\n" if posts else ""
url = f"{base}{site['news_url']}"
return f"""<?xml version="1.0" encoding="utf-8"?>
<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
<channel>
<title>kmalloc.org: {xml_escape(L["nav"]["news"])}</title>
<link>{url}</link>
<description>{xml_escape(L["news_p"])}</description>
<language>{L["lang"]}</language>
{built}<atom:link href="{url}feed.xml" rel="self" type="application/rss+xml"/>
{"".join(items)}</channel>
</rss>
"""
def retention_row(row, cols):
"""One row of the data and retention table. A 'todo' field becomes an HTML comment, never visible text."""
if "--" in row.get("todo", ""):
fail(f"retention row {row['service']!r}: no '--' inside a TODO comment")
todo = f"<!-- TODO verificare: {esc(row['todo'])} -->\n" if row.get("todo") else ""
cells = "\n".join(f'<td data-label="{html.escape(cols[k], quote=True)}">{esc(row[k])}</td>' for k in ("admin", "not_e2e", "kept"))
return f'{todo}<tr>\n<th scope="row">{esc(row["service"])}</th>\n{cells}\n</tr>'
def robots_txt(paths):
"""Allow the indexable pages (exact paths, '$' anchors the end) and block everything else."""
allow = []
for path in paths:
allow.append(f"Allow: {path}$")
if path == "/":
allow.append("Allow: /?lang=en$") # the English home as linked from the other pages
return "User-agent: *\n" + "\n".join(sorted(set(allow))) + "\nDisallow: /\n"
def build():
site = load_json("site.json")
langs_order = site["languages"]
langs = {l: load_json("lang", l + ".json") for l in langs_order}
css = read("static", "style.css").strip("\n")
tpl = {}
for root, _, files in os.walk(os.path.join(HERE, "templates")):
for fn in files:
if fn.endswith(".html"):
key = os.path.relpath(os.path.join(root, fn), os.path.join(HERE, "templates"))[:-5]
tpl[key] = read("templates", key + ".html")
shared = {k: v for k, v in site.items() if isinstance(v, str)}
for k in ("contact", "security", "abuse"): # no mailto link, against address harvesters
shared[k + "_shown"] = site[k].replace("@", " (at) ")
svc = {s["id"]: s for s in site["services"]}
pages = []
for lang in langs_order:
for pid, spec in PAGES.items():
data = langs[lang]["pages"].get(pid)
if data:
pages.append(dict(lang=lang, kind="page", id=pid, nav=spec["nav"], lead=spec["lead"], path=data["path"],
title=data["title"], description=data["description"], data=data, src=f"lang/{lang}.json: {pid}"))
posts = load_posts()
pages += posts
by_id = {(p["id"], p["lang"]): p for p in pages}
if ("news", "en") not in by_id:
fail("lang/en.json needs a 'news' page")
site["news_url"] = by_id[("news", "en")]["path"]
def url_for(lang, pid):
"""Page in this language, else the English one."""
return (by_id.get((pid, lang)) or by_id[(pid, "en")])["path"]
# Manifesto: one Markdown file per language that has the page.
manifesto = {}
for lang in langs_order:
if ("manifesto", lang) in by_id:
title, toc, md_html = md_to_html(no_em_dash(read(f"MANIFESTO.{lang}.md")))
manifesto[lang] = dict(
manifesto_title=html.escape(title),
manifesto_toc="\n".join(f'<li><a href="#{sl}">{inline(h)}</a></li>' for sl, h in toc),
manifesto_body=md_html)
def news_lists(lang):
mark = "" if lang == "en" else ' hreflang="en" lang="en"'
lines = []
for p in posts:
line = f'<li><time datetime="{p["date"]}">{p["date"]}</time> <a href="{p["path"]}"{mark}>{esc(p["title"])}</a>'
if p["summary"]:
line += ": " + esc(p["summary"])
lines.append(line + "</li>")
full = "<ul>\n" + "\n".join(lines) + "\n</ul>" if posts else f"<p>\n{esc(langs[lang].get('news_empty', ''))}\n</p>"
short = "\n".join(f'<li><time datetime="{x["date"]}">{x["date"]}</time> <a href="{x["path"]}"{mark}>{esc(x["title"])}</a></li>'
for x in posts[:NEWS_ON_HOME])
return full, short
for p in pages:
lang = p["lang"]
L = langs[lang]
name = p["src"]
ctx = dict(shared)
ctx.update({k: esc(v) for k, v in L.items() if isinstance(v, str)})
ctx["lang"] = lang
for k, label in L["nav"].items():
ctx["nav_" + k] = esc(label)
for k in NAV[1:] + ("history",):
ctx["url_" + k] = url_for(lang, k)
ctx["history_attr"] = "" if ("history", lang) in by_id else ' hreflang="en"'
ctx["robots_meta"] = "" if p["kind"] == "page" and p["id"] in INDEXABLE else NOINDEX
ctx["version_lc"] = esc(L["version"].lower())
data = p.get("data", {})
ctx.update({k: esc(v) for k, v in data.items() if isinstance(v, str)})
# A list of plain strings in a page entry becomes list items: {{name}} goes inside <ul>.
ctx.update({k: "\n".join(f"<li>{esc(x)}</li>" for x in v) for k, v in data.items()
if isinstance(v, list) and all(isinstance(x, str) for x in v)})
page_title = p["title"] if p["kind"] == "page" else "kmalloc.org: " + p["title"]
ctx.update(css=css, path=p["path"], title_html=html.escape(page_title), post_title=html.escape(p["title"]),
description_attr=html.escape(p["description"], quote=True), doc_url=site["base_url"] + p["path"],
date=p.get("date", ""))
for k in NAV:
ctx["cur_" + k] = ' aria-current="page"' if p["nav"] == k else ""
# Every other language that has this page; a page without translation (news) points to the other home.
twins = {l: by_id[(p["id"], l)] for l in langs_order if l != lang and (p["id"], l) in by_id}
if not twins:
fallback = "it" if lang == "en" else "en"
twins = {fallback: by_id[("home", fallback)]}
switch = data.get("switch", {})
ctx["lang_links"] = "\n&middot;\n".join(
f'<a href="{switch.get(l, t["path"])}" hreflang="{l}" lang="{l}">{esc(langs[l]["lang_name"])}</a>' for l, t in twins.items())
have = {l: by_id[(p["id"], l)] for l in langs_order if (p["id"], l) in by_id}
ctx["alt_links"] = "\n".join(f'<link rel="alternate" hreflang="{l}" href="{q["path"]}">' for l, q in have.items()) if len(have) > 1 else ""
if p["kind"] == "post":
_, _, post_html = md_to_html(p["body"])
ctx["post_body"] = post_html
content = render(tpl["news-post"], ctx, "templates/news-post.html")
lead = "news"
else:
body_tpl = tpl["pages/" + p["id"]]
if "{{news_list}}" in body_tpl:
ctx["news_list"] = news_lists(lang)[0]
if "{{latest_news}}" in body_tpl:
short = news_lists(lang)[1]
ctx["latest_news"] = render(tpl["latest-news"], dict(ctx, news_items=short), "templates/latest-news.html") if short else ""
if p["id"] == "home":
ctx["services_list"] = "\n".join(
f'<li><a href="{svc[i].get("home_url", svc[i]["url"])}">{svc[i]["label"]}</a>: {esc(data["services"][i])}</li>' for i in site["home_order"])
elif p["id"] == "services":
ctx["service_sections"] = "".join(
render(tpl["parts/service-section"], dict(ctx, id=s["id"], name=esc(data["items"][s["id"]]["name"]),
text=esc(data["items"][s["id"]]["text"]), url=s["url"]),
"templates/parts/service-section.html") for s in site["services"])
ctx.update({"col_" + k: esc(v) for k, v in data["retention_cols"].items()})
ctx["retention_rows"] = "\n".join(retention_row(r, data["retention_cols"]) for r in data["retention_rows"])
elif p["id"] == "people":
cards = []
for person in site["people"]:
handle, nm = person["handle"], person.get("name", "")
entries = ([("home", person["home"], data["personal_page"])] if person.get("home") else [])
entries += [("link", x["url"], x["label"]) for x in person["links"]]
links = "\n".join(f'<li><a href="{u}">{esc(label)}</a></li>' for _, u, label in entries)
if person.get("gpg"):
links += (f'\n<li>{esc(data["gpg_label"])}: <a href="{site["gpg_key"]}">{os.path.basename(site["gpg_key"])}</a>, '
f'{esc(data["gpg_word"])} <code>{site["gpg_fingerprint"]}</code></li>')
cards.append(render(tpl["parts/person-card"], dict(
ctx, handle=esc(handle), title=(esc(nm) + " (" + esc(handle) + ")") if nm else esc(handle),
bio=esc(data["bios"][handle]), links=links), "templates/parts/person-card.html"))
ctx["people_cards"] = "".join(cards)
elif p["id"] == "history":
ctx["history_list"] = "\n".join(
f'<h2 id="v{esc(v["version"])}">{esc(L["version"])} {esc(v["version"])}, <time datetime="{v["date"]}">{v["date"]}</time></h2>\n\n'
+ "<ul>\n" + "\n".join(f"<li>{esc(c)}</li>" for c in v["changes"]) + "\n</ul>\n" for v in data["versions"])
elif p["id"] == "manifesto":
ctx.update(manifesto[lang])
ctx["note_block"] = f'<p>\n{esc(data["note"])}\n</p>\n\n' if data.get("note") else ""
content = render(body_tpl.strip("\n") + "\n", ctx, f"templates/pages/{p['id']}.html")
lead = p["lead"]
ctx["content"] = content
ctx["lead"] = render(tpl["lead-" + lead], ctx, f"templates/lead-{lead}.html")
ctx["feed"] = render(tpl["feed"], ctx, "templates/feed.html")
ctx["badges"] = render(tpl["badges"], dict(ctx, doc_q=ctx["doc_url"].replace(":", "%3A").replace("/", "%2F")), "templates/badges.html")
ctx["header"] = render(tpl[L.get("header", "header")], ctx, "templates/header.html")
ctx["footer"] = render(tpl["footer"], ctx, "templates/footer.html")
rel = p["path"].lstrip("/")
write(rel + "index.html" if rel == "" or rel.endswith("/") else rel, render(tpl["base"], ctx, "templates/base.html"))
write(site["news_url"].lstrip("/") + "feed.xml", rss(langs["en"], site, posts))
for dest, src in ASSETS.items():
target = os.path.join(OUT, dest)
os.makedirs(os.path.dirname(target), exist_ok=True)
shutil.copy(os.path.join(HERE, src), target)
write("robots.txt", robots_txt(p["path"] for p in pages if p["kind"] == "page" and p["id"] in INDEXABLE))
print(f"built {len(pages)} pages into {OUT}")
build()