- the last line of the home motto is set in the accent colour, as in the logo - the header of every page starts with the small shrike mark - og:* and twitter:card meta tags with a 1200x630 preview image (og.png)
438 lines
20 KiB
Python
438 lines
20 KiB
Python
#!/usr/bin/env python3
|
|
"""Build the kmalloc.org static site (plain HTML, no JavaScript, standard library only).
|
|
|
|
Usage: build.py [output-dir] (default: preview)
|
|
|
|
Layout:
|
|
site.json what every language shares: languages, contact, Bitcoin address, services, people
|
|
lang/<lang>.json every text of one language: menu, footer, and one entry per page it has
|
|
templates/ page chrome (base, header, footer), page bodies (pages/) and repeated blocks (parts/)
|
|
static/style.css the one stylesheet, inlined into every page
|
|
MANIFESTO.<lang>.md the manifesto, one Markdown file per language
|
|
news/*.md news posts, English only: YYYY-MM-DD-slug.md, front matter, then Markdown
|
|
|
|
To add a language, add lang/<lang>.json and list it in site.json. A language needs only the pages it has:
|
|
a page that is missing from lang/<lang>.json is simply not built, and its language links disappear.
|
|
Texts in the JSON files are plain text: the build escapes them. Placeholders {{name}} in the templates are
|
|
replaced with values from site.json, the language file and the page; an unknown placeholder stops the build."""
|
|
import html, json, os, re, shutil, sys
|
|
from datetime import datetime, timezone
|
|
from email.utils import format_datetime
|
|
from xml.sax.saxutils import escape as xml_escape
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
OUT = os.path.abspath(sys.argv[1] if len(sys.argv) > 1 else os.path.join(HERE, "preview"))
|
|
NEWS_ON_HOME = 3
|
|
NEWS_IN_FEED = 20
|
|
POST_NAME = re.compile(r"^(\d{4}-\d{2}-\d{2})-([a-z0-9]+(?:-[a-z0-9]+)*)$")
|
|
|
|
# Page types: menu entry to highlight and footer lead. The body is templates/pages/<id>.html.
|
|
PAGES = {
|
|
"home": dict(nav="home", lead="home"),
|
|
"manifesto": dict(nav="manifesto", lead="back"),
|
|
"people": dict(nav="people", lead="back"),
|
|
"services": dict(nav="services", lead="back"),
|
|
"news": dict(nav="news", lead="back"),
|
|
"history": dict(nav="history", lead="back"),
|
|
}
|
|
NAV = ("home", "manifesto", "people", "services", "news")
|
|
# Only these pages may be indexed by search engines; every other page gets noindex and is blocked in robots.txt.
|
|
INDEXABLE = ("home", "manifesto")
|
|
NOINDEX = '<meta name="robots" content="noindex, nofollow, noarchive">\n'
|
|
|
|
# Files copied as they are: published name -> source.
|
|
ASSETS = {
|
|
"favicon.svg": "logo/svg/favicon-auto.svg",
|
|
"favicon-auto.svg": "logo/svg/favicon-auto.svg",
|
|
"logo-auto.svg": "logo/svg/mark-auto.svg",
|
|
"logo-light.svg": "logo/svg/mark-light.svg",
|
|
"logo-dark.svg": "logo/svg/mark-dark.svg",
|
|
"logo.png": "logo/png/icon-light-512.png",
|
|
"icon-512.png": "logo/png/icon-light-512.png",
|
|
"favicon.ico": "logo/web/favicon.ico",
|
|
"apple-touch-icon.png": "logo/web/apple-touch-icon.png",
|
|
"og.png": "logo/png/og-1200x630.png",
|
|
"badges/w3c-html5.png": "badges/w3c-html5.png",
|
|
"badges/w3c-css.png": "badges/w3c-css.png",
|
|
"squid.asc": "static/squid.asc",
|
|
}
|
|
|
|
|
|
def fail(msg):
|
|
raise SystemExit(f"build.py: {msg}")
|
|
|
|
|
|
def read(*parts):
|
|
with open(os.path.join(HERE, *parts), encoding="utf-8") as f:
|
|
return f.read()
|
|
|
|
|
|
def write(rel, text):
|
|
path = os.path.join(OUT, rel)
|
|
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
f.write(text)
|
|
|
|
|
|
def load_json(*parts):
|
|
try:
|
|
return json.loads(read(*parts))
|
|
except ValueError as e:
|
|
fail(f"{'/'.join(parts)}: {e}")
|
|
|
|
|
|
def esc(text):
|
|
return html.escape(text, quote=False)
|
|
|
|
|
|
def parse(text, name):
|
|
"""Split 'front matter, then body' into (dict, body)."""
|
|
m = re.match(r"---\n(.*?)\n---\n(.*)\Z", text, re.S)
|
|
if not m:
|
|
fail(f"{name}: missing front matter")
|
|
meta = {}
|
|
for line in m.group(1).splitlines():
|
|
key, sep, value = line.partition(": ")
|
|
if not sep:
|
|
fail(f"{name}: bad front matter line {line!r}")
|
|
meta[key.strip()] = value.strip()
|
|
return meta, m.group(2)
|
|
|
|
|
|
def render(text, ctx, name):
|
|
def sub(m):
|
|
if m.group(1) not in ctx:
|
|
fail(f"{name}: unknown placeholder {{{{{m.group(1)}}}}}")
|
|
return ctx[m.group(1)]
|
|
return re.sub(r"\{\{(\w+)\}\}", sub, text)
|
|
|
|
|
|
# ---------------------------------------------------------------- Markdown
|
|
|
|
def no_em_dash(md):
|
|
"""The manifesto sources use em dashes; the site does not: headings 'NN — Title' become 'NN. Title',
|
|
'Word — Title' becomes 'Word: Title', attribution lines lose the dash, parenthetical dashes become commas."""
|
|
md = re.sub(r"^(#+ )(\d\d) — ", r"\1\2. ", md, flags=re.M)
|
|
md = re.sub(r"^(#+ [^\d\n][^—\n]*?) — ", r"\1: ", md, flags=re.M)
|
|
md = re.sub(r"^> — ", "> ", md, flags=re.M)
|
|
for a, b in ((" — where technically feasible — ", ", where technically feasible, "),
|
|
(" — dove tecnicamente possibile — ", ", dove tecnicamente possibile, "),
|
|
(" — and continues", ", and continues"),
|
|
(" — e continua", ", e continua")):
|
|
md = md.replace(a, b)
|
|
md = md.replace("kmalloc.org — ", "kmalloc.org: ")
|
|
assert "—" not in md, [l for l in md.splitlines() if "—" in l]
|
|
return md
|
|
|
|
|
|
def inline(s):
|
|
s = html.escape(s, quote=False)
|
|
s = re.sub(r"`([^`]+)`", r"<code>\1</code>", s)
|
|
s = re.sub(r"\[([^\]]+)\]\(([^)\s]+)\)", r'<a href="\2">\1</a>', s)
|
|
s = re.sub(r"\*\*(.+?)\*\*", r"<strong>\1</strong>", s)
|
|
s = re.sub(r"(?<!\*)\*(?!\s)(.+?)(?<!\s)\*(?!\*)", r"<em>\1</em>", s)
|
|
return s
|
|
|
|
|
|
def slug(heading):
|
|
m = re.match(r"(\d\d)\b", heading)
|
|
if m:
|
|
return m.group(1)
|
|
h = heading.lower()
|
|
if h.startswith(("preamble", "preambolo", "preámbulo", "präambel", "préambule")):
|
|
return "preamble"
|
|
if h.startswith(("our commitments", "i nostri impegni", "nuestros compromisos", "unsere verpflichtungen", "nos engagements")):
|
|
return "commitments"
|
|
if h.startswith(("epilogue", "epilogo", "epílogo", "epilog", "épilogue")):
|
|
return "epilogue"
|
|
return re.sub(r"[^a-z0-9]+", "-", h).strip("-")
|
|
|
|
|
|
def md_to_html(md):
|
|
"""Headings (##), paragraphs, quotes, bullet lists and inline markup. Returns (title, toc, html)."""
|
|
out, toc = [], []
|
|
para, quote, items = [], [], []
|
|
|
|
def flush():
|
|
nonlocal para, quote, items
|
|
if para:
|
|
out.append("<p>\n" + "\n".join(para) + "\n</p>\n")
|
|
para = []
|
|
if quote:
|
|
out.append("<blockquote>\n<p>\n" + "\n".join(quote) + "\n</p>\n</blockquote>\n")
|
|
quote = []
|
|
if items:
|
|
out.append("<ul>\n" + "\n".join(f"<li>{i}</li>" for i in items) + "\n</ul>\n")
|
|
items = []
|
|
|
|
title = ""
|
|
for raw in md.splitlines():
|
|
line = raw.rstrip("\n")
|
|
br = line.endswith(" ")
|
|
s = line.strip()
|
|
if not s:
|
|
flush()
|
|
continue
|
|
if s.startswith("# "):
|
|
flush()
|
|
title = s[2:]
|
|
continue
|
|
if s.startswith("## "):
|
|
flush()
|
|
h = s[3:]
|
|
sl = slug(h)
|
|
toc.append((sl, h))
|
|
out.append(f'<h2 id="{sl}">{inline(h)}</h2>\n')
|
|
continue
|
|
if s.startswith("> "):
|
|
if para or items:
|
|
flush()
|
|
quote.append(inline(s[2:]) + ("<br>" if br else ""))
|
|
continue
|
|
if s.startswith("- "):
|
|
if para or quote:
|
|
flush()
|
|
items.append(inline(s[2:]))
|
|
continue
|
|
if quote or items:
|
|
flush()
|
|
para.append(inline(s))
|
|
flush()
|
|
return title, toc, "".join(out)
|
|
|
|
|
|
# ---------------------------------------------------------------- site model
|
|
|
|
def load_posts():
|
|
"""News posts from news/*.md (English only)."""
|
|
posts = []
|
|
folder = os.path.join(HERE, "news")
|
|
for fn in sorted(os.listdir(folder)) if os.path.isdir(folder) else []:
|
|
if not fn.endswith(".md"):
|
|
continue
|
|
m = POST_NAME.match(fn[:-3])
|
|
if not m:
|
|
fail(f"news/{fn}: name must be YYYY-MM-DD-slug.md")
|
|
src = f"news/{fn}"
|
|
meta, text = parse(read("news", fn), src)
|
|
if "title" not in meta:
|
|
fail(f"{src}: front matter lacks 'title'")
|
|
if "\u2014" in text or "\u2014" in meta["title"]:
|
|
fail(f"{src}: no em dashes, please")
|
|
posts.append(dict(lang="en", kind="post", id="news/" + fn[:-3], date=m.group(1), nav="news",
|
|
path=f"/news/{m.group(2)}.html", body=text, src=src, title=meta["title"],
|
|
summary=meta.get("summary", ""), description=meta.get("summary") or meta["title"]))
|
|
return sorted(posts, key=lambda p: p["date"], reverse=True)
|
|
|
|
|
|
def rss(L, site, all_posts):
|
|
"""RSS 2.0 feed of the news (English only): the latest posts, full text, absolute links."""
|
|
base = site["base_url"]
|
|
posts = all_posts[:NEWS_IN_FEED]
|
|
|
|
def stamp(d):
|
|
return format_datetime(datetime.strptime(d, "%Y-%m-%d").replace(tzinfo=timezone.utc))
|
|
items = []
|
|
for p in posts:
|
|
text = md_to_html(p["body"])[2].replace('href="/', f'href="{base}/')
|
|
items.append(f"""<item>
|
|
<title>{xml_escape(p["title"])}</title>
|
|
<link>{base}{p["path"]}</link>
|
|
<guid isPermaLink="true">{base}{p["path"]}</guid>
|
|
<pubDate>{stamp(p["date"])}</pubDate>
|
|
<description>{xml_escape(text)}</description>
|
|
</item>
|
|
""")
|
|
built = f"<lastBuildDate>{stamp(posts[0]['date'])}</lastBuildDate>\n" if posts else ""
|
|
url = f"{base}{site['news_url']}"
|
|
return f"""<?xml version="1.0" encoding="utf-8"?>
|
|
<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
|
|
<channel>
|
|
<title>kmalloc.org: {xml_escape(L["nav"]["news"])}</title>
|
|
<link>{url}</link>
|
|
<description>{xml_escape(L["news_p"])}</description>
|
|
<language>{L["lang"]}</language>
|
|
{built}<atom:link href="{url}feed.xml" rel="self" type="application/rss+xml"/>
|
|
{"".join(items)}</channel>
|
|
</rss>
|
|
"""
|
|
|
|
|
|
def retention_row(row, cols):
|
|
"""One row of the data and retention table. A 'todo' field becomes an HTML comment, never visible text."""
|
|
if "--" in row.get("todo", ""):
|
|
fail(f"retention row {row['service']!r}: no '--' inside a TODO comment")
|
|
todo = f"<!-- TODO verificare: {esc(row['todo'])} -->\n" if row.get("todo") else ""
|
|
cells = "\n".join(f'<td data-label="{html.escape(cols[k], quote=True)}">{esc(row[k])}</td>' for k in ("admin", "not_e2e", "kept"))
|
|
return f'{todo}<tr>\n<th scope="row">{esc(row["service"])}</th>\n{cells}\n</tr>'
|
|
|
|
|
|
def robots_txt(paths):
|
|
"""Allow the indexable pages (exact paths, '$' anchors the end) and block everything else."""
|
|
allow = []
|
|
for path in paths:
|
|
allow.append(f"Allow: {path}$")
|
|
if path == "/":
|
|
allow.append("Allow: /?lang=en$") # the English home as linked from the other pages
|
|
return "User-agent: *\n" + "\n".join(sorted(set(allow))) + "\nDisallow: /\n"
|
|
|
|
|
|
def build():
|
|
site = load_json("site.json")
|
|
langs_order = site["languages"]
|
|
langs = {l: load_json("lang", l + ".json") for l in langs_order}
|
|
css = read("static", "style.css").strip("\n")
|
|
tpl = {}
|
|
for root, _, files in os.walk(os.path.join(HERE, "templates")):
|
|
for fn in files:
|
|
if fn.endswith(".html"):
|
|
key = os.path.relpath(os.path.join(root, fn), os.path.join(HERE, "templates"))[:-5]
|
|
tpl[key] = read("templates", key + ".html")
|
|
shared = {k: v for k, v in site.items() if isinstance(v, str)}
|
|
for k in ("contact", "security", "abuse"): # no mailto link, against address harvesters
|
|
shared[k + "_shown"] = site[k].replace("@", " (at) ")
|
|
svc = {s["id"]: s for s in site["services"]}
|
|
|
|
pages = []
|
|
for lang in langs_order:
|
|
for pid, spec in PAGES.items():
|
|
data = langs[lang]["pages"].get(pid)
|
|
if data:
|
|
pages.append(dict(lang=lang, kind="page", id=pid, nav=spec["nav"], lead=spec["lead"], path=data["path"],
|
|
title=data["title"], description=data["description"], data=data, src=f"lang/{lang}.json: {pid}"))
|
|
posts = load_posts()
|
|
pages += posts
|
|
by_id = {(p["id"], p["lang"]): p for p in pages}
|
|
if ("news", "en") not in by_id:
|
|
fail("lang/en.json needs a 'news' page")
|
|
site["news_url"] = by_id[("news", "en")]["path"]
|
|
|
|
def url_for(lang, pid):
|
|
"""Page in this language, else the English one."""
|
|
return (by_id.get((pid, lang)) or by_id[(pid, "en")])["path"]
|
|
|
|
# Manifesto: one Markdown file per language that has the page.
|
|
manifesto = {}
|
|
for lang in langs_order:
|
|
if ("manifesto", lang) in by_id:
|
|
title, toc, md_html = md_to_html(no_em_dash(read(f"MANIFESTO.{lang}.md")))
|
|
manifesto[lang] = dict(
|
|
manifesto_title=html.escape(title),
|
|
manifesto_toc="\n".join(f'<li><a href="#{sl}">{inline(h)}</a></li>' for sl, h in toc),
|
|
manifesto_body=md_html)
|
|
|
|
def news_lists(lang):
|
|
mark = "" if lang == "en" else ' hreflang="en" lang="en"'
|
|
lines = []
|
|
for p in posts:
|
|
line = f'<li><time datetime="{p["date"]}">{p["date"]}</time> <a href="{p["path"]}"{mark}>{esc(p["title"])}</a>'
|
|
if p["summary"]:
|
|
line += ": " + esc(p["summary"])
|
|
lines.append(line + "</li>")
|
|
full = "<ul>\n" + "\n".join(lines) + "\n</ul>" if posts else f"<p>\n{esc(langs[lang].get('news_empty', ''))}\n</p>"
|
|
short = "\n".join(f'<li><time datetime="{x["date"]}">{x["date"]}</time> <a href="{x["path"]}"{mark}>{esc(x["title"])}</a></li>'
|
|
for x in posts[:NEWS_ON_HOME])
|
|
return full, short
|
|
|
|
for p in pages:
|
|
lang = p["lang"]
|
|
L = langs[lang]
|
|
name = p["src"]
|
|
ctx = dict(shared)
|
|
ctx.update({k: esc(v) for k, v in L.items() if isinstance(v, str)})
|
|
ctx["lang"] = lang
|
|
for k, label in L["nav"].items():
|
|
ctx["nav_" + k] = esc(label)
|
|
for k in NAV[1:] + ("history",):
|
|
ctx["url_" + k] = url_for(lang, k)
|
|
ctx["history_attr"] = "" if ("history", lang) in by_id else ' hreflang="en"'
|
|
ctx["robots_meta"] = "" if p["kind"] == "page" and p["id"] in INDEXABLE else NOINDEX
|
|
ctx["version_lc"] = esc(L["version"].lower())
|
|
data = p.get("data", {})
|
|
ctx.update({k: esc(v) for k, v in data.items() if isinstance(v, str)})
|
|
# A list of plain strings in a page entry becomes list items: {{name}} goes inside <ul>.
|
|
ctx.update({k: "\n".join(f"<li>{esc(x)}</li>" for x in v) for k, v in data.items()
|
|
if isinstance(v, list) and all(isinstance(x, str) for x in v)})
|
|
page_title = p["title"] if p["kind"] == "page" else "kmalloc.org: " + p["title"]
|
|
ctx.update(css=css, path=p["path"], title_html=html.escape(page_title), post_title=html.escape(p["title"]),
|
|
description_attr=html.escape(p["description"], quote=True), doc_url=site["base_url"] + p["path"],
|
|
date=p.get("date", ""))
|
|
for k in NAV:
|
|
ctx["cur_" + k] = ' aria-current="page"' if p["nav"] == k else ""
|
|
|
|
# Every other language that has this page; a page without translation (news) points to the other home.
|
|
twins = {l: by_id[(p["id"], l)] for l in langs_order if l != lang and (p["id"], l) in by_id}
|
|
if not twins:
|
|
fallback = "it" if lang == "en" else "en"
|
|
twins = {fallback: by_id[("home", fallback)]}
|
|
switch = data.get("switch", {})
|
|
ctx["lang_links"] = "\n·\n".join(
|
|
f'<a href="{switch.get(l, t["path"])}" hreflang="{l}" lang="{l}">{esc(langs[l]["lang_name"])}</a>' for l, t in twins.items())
|
|
have = {l: by_id[(p["id"], l)] for l in langs_order if (p["id"], l) in by_id}
|
|
ctx["alt_links"] = "\n".join(f'<link rel="alternate" hreflang="{l}" href="{q["path"]}">' for l, q in have.items()) if len(have) > 1 else ""
|
|
|
|
if p["kind"] == "post":
|
|
_, _, post_html = md_to_html(p["body"])
|
|
ctx["post_body"] = post_html
|
|
content = render(tpl["news-post"], ctx, "templates/news-post.html")
|
|
lead = "news"
|
|
else:
|
|
body_tpl = tpl["pages/" + p["id"]]
|
|
if "{{news_list}}" in body_tpl:
|
|
ctx["news_list"] = news_lists(lang)[0]
|
|
if "{{latest_news}}" in body_tpl:
|
|
short = news_lists(lang)[1]
|
|
ctx["latest_news"] = render(tpl["latest-news"], dict(ctx, news_items=short), "templates/latest-news.html") if short else ""
|
|
if p["id"] == "home":
|
|
ctx["services_list"] = "\n".join(
|
|
f'<li><a href="{svc[i].get("home_url", svc[i]["url"])}">{svc[i]["label"]}</a>: {esc(data["services"][i])}</li>' for i in site["home_order"])
|
|
elif p["id"] == "services":
|
|
ctx["service_sections"] = "".join(
|
|
render(tpl["parts/service-section"], dict(ctx, id=s["id"], name=esc(data["items"][s["id"]]["name"]),
|
|
text=esc(data["items"][s["id"]]["text"]), url=s["url"]),
|
|
"templates/parts/service-section.html") for s in site["services"])
|
|
ctx.update({"col_" + k: esc(v) for k, v in data["retention_cols"].items()})
|
|
ctx["retention_rows"] = "\n".join(retention_row(r, data["retention_cols"]) for r in data["retention_rows"])
|
|
elif p["id"] == "people":
|
|
cards = []
|
|
for person in site["people"]:
|
|
handle, nm = person["handle"], person.get("name", "")
|
|
entries = ([("home", person["home"], data["personal_page"])] if person.get("home") else [])
|
|
entries += [("link", x["url"], x["label"]) for x in person["links"]]
|
|
links = "\n".join(f'<li><a href="{u}">{esc(label)}</a></li>' for _, u, label in entries)
|
|
if person.get("gpg"):
|
|
links += (f'\n<li>{esc(data["gpg_label"])}: <a href="{site["gpg_key"]}">{os.path.basename(site["gpg_key"])}</a>, '
|
|
f'{esc(data["gpg_word"])} <code>{site["gpg_fingerprint"]}</code></li>')
|
|
cards.append(render(tpl["parts/person-card"], dict(
|
|
ctx, handle=esc(handle), title=(esc(nm) + " (" + esc(handle) + ")") if nm else esc(handle),
|
|
bio=esc(data["bios"][handle]), links=links), "templates/parts/person-card.html"))
|
|
ctx["people_cards"] = "".join(cards)
|
|
elif p["id"] == "history":
|
|
ctx["history_list"] = "\n".join(
|
|
f'<h2 id="v{esc(v["version"])}">{esc(L["version"])} {esc(v["version"])}, <time datetime="{v["date"]}">{v["date"]}</time></h2>\n\n'
|
|
+ "<ul>\n" + "\n".join(f"<li>{esc(c)}</li>" for c in v["changes"]) + "\n</ul>\n" for v in data["versions"])
|
|
elif p["id"] == "manifesto":
|
|
ctx.update(manifesto[lang])
|
|
ctx["note_block"] = f'<p>\n{esc(data["note"])}\n</p>\n\n' if data.get("note") else ""
|
|
content = render(body_tpl.strip("\n") + "\n", ctx, f"templates/pages/{p['id']}.html")
|
|
lead = p["lead"]
|
|
ctx["content"] = content
|
|
ctx["lead"] = render(tpl["lead-" + lead], ctx, f"templates/lead-{lead}.html")
|
|
ctx["feed"] = render(tpl["feed"], ctx, "templates/feed.html")
|
|
ctx["badges"] = render(tpl["badges"], dict(ctx, doc_q=ctx["doc_url"].replace(":", "%3A").replace("/", "%2F")), "templates/badges.html")
|
|
ctx["header"] = render(tpl[L.get("header", "header")], ctx, "templates/header.html")
|
|
ctx["footer"] = render(tpl["footer"], ctx, "templates/footer.html")
|
|
rel = p["path"].lstrip("/")
|
|
write(rel + "index.html" if rel == "" or rel.endswith("/") else rel, render(tpl["base"], ctx, "templates/base.html"))
|
|
|
|
write(site["news_url"].lstrip("/") + "feed.xml", rss(langs["en"], site, posts))
|
|
|
|
for dest, src in ASSETS.items():
|
|
target = os.path.join(OUT, dest)
|
|
os.makedirs(os.path.dirname(target), exist_ok=True)
|
|
shutil.copy(os.path.join(HERE, src), target)
|
|
write("robots.txt", robots_txt(p["path"] for p in pages if p["kind"] == "page" and p["id"] in INDEXABLE))
|
|
print(f"built {len(pages)} pages into {OUT}")
|
|
|
|
|
|
build()
|