Open indexing to people, services, the manifesto history, the news list and every news post, besides the home and the manifesto. robots.txt and the robots meta are generated from the same rule. Downloads and the personal pages under /~user stay out. Closes #4
444 lines
20 KiB
Python
444 lines
20 KiB
Python
#!/usr/bin/env python3
|
|
"""Build the kmalloc.org static site (plain HTML, no JavaScript, standard library only).
|
|
|
|
Usage: build.py [output-dir] (default: preview)
|
|
|
|
Layout:
|
|
site.json what every language shares: languages, contact, Bitcoin address, services, people
|
|
lang/<lang>.json every text of one language: menu, footer, and one entry per page it has
|
|
templates/ page chrome (base, header, footer), page bodies (pages/) and repeated blocks (parts/)
|
|
static/style.css the one stylesheet, inlined into every page
|
|
MANIFESTO.<lang>.md the manifesto, one Markdown file per language
|
|
news/*.md news posts, English only: YYYY-MM-DD-slug.md, front matter, then Markdown
|
|
|
|
To add a language, add lang/<lang>.json and list it in site.json. A language needs only the pages it has:
|
|
a page that is missing from lang/<lang>.json is simply not built, and its language links disappear.
|
|
Texts in the JSON files are plain text: the build escapes them. Placeholders {{name}} in the templates are
|
|
replaced with values from site.json, the language file and the page; an unknown placeholder stops the build."""
|
|
import html, json, os, re, shutil, sys
|
|
from datetime import datetime, timezone
|
|
from email.utils import format_datetime
|
|
from xml.sax.saxutils import escape as xml_escape
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
OUT = os.path.abspath(sys.argv[1] if len(sys.argv) > 1 else os.path.join(HERE, "preview"))
|
|
NEWS_ON_HOME = 3
|
|
NEWS_IN_FEED = 20
|
|
POST_NAME = re.compile(r"^(\d{4}-\d{2}-\d{2})-([a-z0-9]+(?:-[a-z0-9]+)*)$")
|
|
|
|
# Page types: menu entry to highlight and footer lead. The body is templates/pages/<id>.html.
|
|
PAGES = {
|
|
"home": dict(nav="home", lead="home"),
|
|
"manifesto": dict(nav="manifesto", lead="back"),
|
|
"people": dict(nav="people", lead="back"),
|
|
"services": dict(nav="services", lead="back"),
|
|
"news": dict(nav="news", lead="back"),
|
|
"history": dict(nav="history", lead="back"),
|
|
}
|
|
NAV = ("home", "manifesto", "people", "services", "news")
|
|
# The pages of the site and the news posts may be indexed by search engines; anything else gets noindex.
|
|
INDEXABLE = ("home", "manifesto", "people", "services", "news", "history")
|
|
NOINDEX = '<meta name="robots" content="noindex, nofollow, noarchive">\n'
|
|
|
|
# Files copied as they are: published name -> source.
|
|
ASSETS = {
|
|
"favicon.svg": "logo/svg/favicon-auto.svg",
|
|
"favicon-auto.svg": "logo/svg/favicon-auto.svg",
|
|
"logo-auto.svg": "logo/svg/mark-auto.svg",
|
|
"logo-light.svg": "logo/svg/mark-light.svg",
|
|
"logo-dark.svg": "logo/svg/mark-dark.svg",
|
|
"logo.png": "logo/png/icon-light-512.png",
|
|
"icon-512.png": "logo/png/icon-light-512.png",
|
|
"favicon.ico": "logo/web/favicon.ico",
|
|
"apple-touch-icon.png": "logo/web/apple-touch-icon.png",
|
|
"og.png": "logo/png/og-1200x630.png",
|
|
"badges/w3c-html5.png": "badges/w3c-html5.png",
|
|
"badges/w3c-css.png": "badges/w3c-css.png",
|
|
"squid.asc": "static/squid.asc",
|
|
"riot_pack.zip": "static/riot_pack.zip",
|
|
}
|
|
|
|
|
|
def fail(msg):
|
|
raise SystemExit(f"build.py: {msg}")
|
|
|
|
|
|
def read(*parts):
|
|
with open(os.path.join(HERE, *parts), encoding="utf-8") as f:
|
|
return f.read()
|
|
|
|
|
|
def write(rel, text):
|
|
path = os.path.join(OUT, rel)
|
|
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
f.write(text)
|
|
|
|
|
|
def load_json(*parts):
|
|
try:
|
|
return json.loads(read(*parts))
|
|
except ValueError as e:
|
|
fail(f"{'/'.join(parts)}: {e}")
|
|
|
|
|
|
def esc(text):
|
|
return html.escape(text, quote=False)
|
|
|
|
|
|
def parse(text, name):
|
|
"""Split 'front matter, then body' into (dict, body)."""
|
|
m = re.match(r"---\n(.*?)\n---\n(.*)\Z", text, re.S)
|
|
if not m:
|
|
fail(f"{name}: missing front matter")
|
|
meta = {}
|
|
for line in m.group(1).splitlines():
|
|
key, sep, value = line.partition(": ")
|
|
if not sep:
|
|
fail(f"{name}: bad front matter line {line!r}")
|
|
meta[key.strip()] = value.strip()
|
|
return meta, m.group(2)
|
|
|
|
|
|
def render(text, ctx, name):
|
|
def sub(m):
|
|
if m.group(1) not in ctx:
|
|
fail(f"{name}: unknown placeholder {{{{{m.group(1)}}}}}")
|
|
return ctx[m.group(1)]
|
|
return re.sub(r"\{\{(\w+)\}\}", sub, text)
|
|
|
|
|
|
# ---------------------------------------------------------------- Markdown
|
|
|
|
def no_em_dash(md):
|
|
"""The manifesto sources use em dashes; the site does not: headings 'NN — Title' become 'NN. Title',
|
|
'Word — Title' becomes 'Word: Title', attribution lines lose the dash, parenthetical dashes become commas."""
|
|
md = re.sub(r"^(#+ )(\d\d) — ", r"\1\2. ", md, flags=re.M)
|
|
md = re.sub(r"^(#+ [^\d\n][^—\n]*?) — ", r"\1: ", md, flags=re.M)
|
|
md = re.sub(r"^> — ", "> ", md, flags=re.M)
|
|
for a, b in ((" — where technically feasible — ", ", where technically feasible, "),
|
|
(" — dove tecnicamente possibile — ", ", dove tecnicamente possibile, "),
|
|
(" — and continues", ", and continues"),
|
|
(" — e continua", ", e continua")):
|
|
md = md.replace(a, b)
|
|
md = md.replace("kmalloc.org — ", "kmalloc.org: ")
|
|
assert "—" not in md, [l for l in md.splitlines() if "—" in l]
|
|
return md
|
|
|
|
|
|
def inline(s):
|
|
s = html.escape(s, quote=False)
|
|
s = re.sub(r"`([^`]+)`", r"<code>\1</code>", s)
|
|
s = re.sub(r"\[([^\]]+)\]\(([^)\s]+)\)", r'<a href="\2">\1</a>', s)
|
|
s = re.sub(r"\*\*(.+?)\*\*", r"<strong>\1</strong>", s)
|
|
s = re.sub(r"(?<!\*)\*(?!\s)(.+?)(?<!\s)\*(?!\*)", r"<em>\1</em>", s)
|
|
return s
|
|
|
|
|
|
def slug(heading):
|
|
m = re.match(r"(\d\d)\b", heading)
|
|
if m:
|
|
return m.group(1)
|
|
h = heading.lower()
|
|
if h.startswith(("preamble", "preambolo", "preámbulo", "präambel", "préambule")):
|
|
return "preamble"
|
|
if h.startswith(("our commitments", "i nostri impegni", "nuestros compromisos", "unsere verpflichtungen", "nos engagements")):
|
|
return "commitments"
|
|
if h.startswith(("epilogue", "epilogo", "epílogo", "epilog", "épilogue")):
|
|
return "epilogue"
|
|
return re.sub(r"[^a-z0-9]+", "-", h).strip("-")
|
|
|
|
|
|
def md_to_html(md):
|
|
"""Headings (##), paragraphs, quotes, bullet lists and inline markup. Returns (title, toc, html)."""
|
|
out, toc = [], []
|
|
para, quote, items = [], [], []
|
|
|
|
def flush():
|
|
nonlocal para, quote, items
|
|
if para:
|
|
out.append("<p>\n" + "\n".join(para) + "\n</p>\n")
|
|
para = []
|
|
if quote:
|
|
out.append("<blockquote>\n<p>\n" + "\n".join(quote) + "\n</p>\n</blockquote>\n")
|
|
quote = []
|
|
if items:
|
|
out.append("<ul>\n" + "\n".join(f"<li>{i}</li>" for i in items) + "\n</ul>\n")
|
|
items = []
|
|
|
|
title = ""
|
|
for raw in md.splitlines():
|
|
line = raw.rstrip("\n")
|
|
br = line.endswith(" ")
|
|
s = line.strip()
|
|
if not s:
|
|
flush()
|
|
continue
|
|
if s.startswith("# "):
|
|
flush()
|
|
title = s[2:]
|
|
continue
|
|
if s.startswith("## "):
|
|
flush()
|
|
h = s[3:]
|
|
sl = slug(h)
|
|
toc.append((sl, h))
|
|
out.append(f'<h2 id="{sl}">{inline(h)}</h2>\n')
|
|
continue
|
|
if s.startswith("> "):
|
|
if para or items:
|
|
flush()
|
|
quote.append(inline(s[2:]) + ("<br>" if br else ""))
|
|
continue
|
|
if s.startswith("- "):
|
|
if para or quote:
|
|
flush()
|
|
items.append(inline(s[2:]))
|
|
continue
|
|
if quote or items:
|
|
flush()
|
|
para.append(inline(s))
|
|
flush()
|
|
return title, toc, "".join(out)
|
|
|
|
|
|
# ---------------------------------------------------------------- site model
|
|
|
|
def load_posts():
|
|
"""News posts from news/*.md (English only)."""
|
|
posts = []
|
|
folder = os.path.join(HERE, "news")
|
|
for fn in sorted(os.listdir(folder)) if os.path.isdir(folder) else []:
|
|
if not fn.endswith(".md"):
|
|
continue
|
|
m = POST_NAME.match(fn[:-3])
|
|
if not m:
|
|
fail(f"news/{fn}: name must be YYYY-MM-DD-slug.md")
|
|
src = f"news/{fn}"
|
|
meta, text = parse(read("news", fn), src)
|
|
if "title" not in meta:
|
|
fail(f"{src}: front matter lacks 'title'")
|
|
if "\u2014" in text or "\u2014" in meta["title"]:
|
|
fail(f"{src}: no em dashes, please")
|
|
posts.append(dict(lang="en", kind="post", id="news/" + fn[:-3], date=m.group(1), nav="news",
|
|
path=f"/news/{m.group(2)}.html", body=text, src=src, title=meta["title"],
|
|
summary=meta.get("summary", ""), description=meta.get("summary") or meta["title"]))
|
|
return sorted(posts, key=lambda p: p["date"], reverse=True)
|
|
|
|
|
|
def rss(L, site, all_posts):
|
|
"""RSS 2.0 feed of the news (English only): the latest posts, full text, absolute links."""
|
|
base = site["base_url"]
|
|
posts = all_posts[:NEWS_IN_FEED]
|
|
|
|
def stamp(d):
|
|
return format_datetime(datetime.strptime(d, "%Y-%m-%d").replace(tzinfo=timezone.utc))
|
|
items = []
|
|
for p in posts:
|
|
text = md_to_html(p["body"])[2].replace('href="/', f'href="{base}/')
|
|
items.append(f"""<item>
|
|
<title>{xml_escape(p["title"])}</title>
|
|
<link>{base}{p["path"]}</link>
|
|
<guid isPermaLink="true">{base}{p["path"]}</guid>
|
|
<pubDate>{stamp(p["date"])}</pubDate>
|
|
<description>{xml_escape(text)}</description>
|
|
</item>
|
|
""")
|
|
built = f"<lastBuildDate>{stamp(posts[0]['date'])}</lastBuildDate>\n" if posts else ""
|
|
url = f"{base}{site['news_url']}"
|
|
return f"""<?xml version="1.0" encoding="utf-8"?>
|
|
<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
|
|
<channel>
|
|
<title>kmalloc.org: {xml_escape(L["nav"]["news"])}</title>
|
|
<link>{url}</link>
|
|
<description>{xml_escape(L["news_p"])}</description>
|
|
<language>{L["lang"]}</language>
|
|
{built}<atom:link href="{url}feed.xml" rel="self" type="application/rss+xml"/>
|
|
{"".join(items)}</channel>
|
|
</rss>
|
|
"""
|
|
|
|
|
|
def retention_row(row, cols):
|
|
"""One row of the data and retention table. A 'todo' field becomes an HTML comment, never visible text."""
|
|
if "--" in row.get("todo", ""):
|
|
fail(f"retention row {row['service']!r}: no '--' inside a TODO comment")
|
|
todo = f"<!-- TODO verificare: {esc(row['todo'])} -->\n" if row.get("todo") else ""
|
|
cells = "\n".join(f'<td data-label="{html.escape(cols[k], quote=True)}">{esc(row[k])}</td>' for k in ("admin", "not_e2e", "kept"))
|
|
return f'{todo}<tr>\n<th scope="row">{esc(row["service"])}</th>\n{cells}\n</tr>'
|
|
|
|
|
|
def indexable(p):
|
|
"""A listed page of the site or a news post."""
|
|
return p["kind"] == "post" or (p["kind"] == "page" and p["id"] in INDEXABLE)
|
|
|
|
|
|
def robots_txt(paths):
|
|
"""Allow the indexable pages (exact paths, '$' anchors the end) and block everything else."""
|
|
allow = []
|
|
for path in paths:
|
|
allow.append(f"Allow: {path}$")
|
|
if path == "/":
|
|
allow.append("Allow: /?lang=en$") # the English home as linked from the other pages
|
|
return "User-agent: *\n" + "\n".join(sorted(set(allow))) + "\nDisallow: /\n"
|
|
|
|
|
|
def build():
|
|
site = load_json("site.json")
|
|
langs_order = site["languages"]
|
|
langs = {l: load_json("lang", l + ".json") for l in langs_order}
|
|
css = read("static", "style.css").strip("\n")
|
|
tpl = {}
|
|
for root, _, files in os.walk(os.path.join(HERE, "templates")):
|
|
for fn in files:
|
|
if fn.endswith(".html"):
|
|
key = os.path.relpath(os.path.join(root, fn), os.path.join(HERE, "templates"))[:-5]
|
|
tpl[key] = read("templates", key + ".html")
|
|
shared = {k: v for k, v in site.items() if isinstance(v, str)}
|
|
for k in ("contact", "security", "abuse"): # no mailto link, against address harvesters
|
|
shared[k + "_shown"] = site[k].replace("@", " (at) ")
|
|
svc = {s["id"]: s for s in site["services"]}
|
|
|
|
pages = []
|
|
for lang in langs_order:
|
|
for pid, spec in PAGES.items():
|
|
data = langs[lang]["pages"].get(pid)
|
|
if data:
|
|
pages.append(dict(lang=lang, kind="page", id=pid, nav=spec["nav"], lead=spec["lead"], path=data["path"],
|
|
title=data["title"], description=data["description"], data=data, src=f"lang/{lang}.json: {pid}"))
|
|
posts = load_posts()
|
|
pages += posts
|
|
by_id = {(p["id"], p["lang"]): p for p in pages}
|
|
if ("news", "en") not in by_id:
|
|
fail("lang/en.json needs a 'news' page")
|
|
site["news_url"] = by_id[("news", "en")]["path"]
|
|
|
|
def url_for(lang, pid):
|
|
"""Page in this language, else the English one."""
|
|
return (by_id.get((pid, lang)) or by_id[(pid, "en")])["path"]
|
|
|
|
# Manifesto: one Markdown file per language that has the page.
|
|
manifesto = {}
|
|
for lang in langs_order:
|
|
if ("manifesto", lang) in by_id:
|
|
title, toc, md_html = md_to_html(no_em_dash(read(f"MANIFESTO.{lang}.md")))
|
|
manifesto[lang] = dict(
|
|
manifesto_title=html.escape(title),
|
|
manifesto_toc="\n".join(f'<li><a href="#{sl}">{inline(h)}</a></li>' for sl, h in toc),
|
|
manifesto_body=md_html)
|
|
|
|
def news_lists(lang):
|
|
mark = "" if lang == "en" else ' hreflang="en" lang="en"'
|
|
lines = []
|
|
for p in posts:
|
|
line = f'<li><time datetime="{p["date"]}">{p["date"]}</time> <a href="{p["path"]}"{mark}>{esc(p["title"])}</a>'
|
|
if p["summary"]:
|
|
line += ": " + esc(p["summary"])
|
|
lines.append(line + "</li>")
|
|
full = "<ul>\n" + "\n".join(lines) + "\n</ul>" if posts else f"<p>\n{esc(langs[lang].get('news_empty', ''))}\n</p>"
|
|
short = "\n".join(f'<li><time datetime="{x["date"]}">{x["date"]}</time> <a href="{x["path"]}"{mark}>{esc(x["title"])}</a></li>'
|
|
for x in posts[:NEWS_ON_HOME])
|
|
return full, short
|
|
|
|
for p in pages:
|
|
lang = p["lang"]
|
|
L = langs[lang]
|
|
name = p["src"]
|
|
ctx = dict(shared)
|
|
ctx.update({k: esc(v) for k, v in L.items() if isinstance(v, str)})
|
|
ctx["lang"] = lang
|
|
for k, label in L["nav"].items():
|
|
ctx["nav_" + k] = esc(label)
|
|
for k in NAV[1:] + ("history",):
|
|
ctx["url_" + k] = url_for(lang, k)
|
|
ctx["history_attr"] = "" if ("history", lang) in by_id else ' hreflang="en"'
|
|
ctx["robots_meta"] = "" if indexable(p) else NOINDEX
|
|
ctx["version_lc"] = esc(L["version"].lower())
|
|
data = p.get("data", {})
|
|
ctx.update({k: esc(v) for k, v in data.items() if isinstance(v, str)})
|
|
# A list of plain strings in a page entry becomes list items: {{name}} goes inside <ul>.
|
|
ctx.update({k: "\n".join(f"<li>{esc(x)}</li>" for x in v) for k, v in data.items()
|
|
if isinstance(v, list) and all(isinstance(x, str) for x in v)})
|
|
page_title = p["title"] if p["kind"] == "page" else "kmalloc.org: " + p["title"]
|
|
ctx.update(css=css, path=p["path"], title_html=html.escape(page_title), post_title=html.escape(p["title"]),
|
|
description_attr=html.escape(p["description"], quote=True), doc_url=site["base_url"] + p["path"],
|
|
date=p.get("date", ""))
|
|
for k in NAV:
|
|
ctx["cur_" + k] = ' aria-current="page"' if p["nav"] == k else ""
|
|
|
|
# Every other language that has this page; a page without translation (news) points to the other home.
|
|
twins = {l: by_id[(p["id"], l)] for l in langs_order if l != lang and (p["id"], l) in by_id}
|
|
if not twins:
|
|
fallback = "it" if lang == "en" else "en"
|
|
twins = {fallback: by_id[("home", fallback)]}
|
|
switch = data.get("switch", {})
|
|
ctx["lang_links"] = "\n·\n".join(
|
|
f'<a href="{switch.get(l, t["path"])}" hreflang="{l}" lang="{l}">{esc(langs[l]["lang_name"])}</a>' for l, t in twins.items())
|
|
have = {l: by_id[(p["id"], l)] for l in langs_order if (p["id"], l) in by_id}
|
|
ctx["alt_links"] = "\n".join(f'<link rel="alternate" hreflang="{l}" href="{q["path"]}">' for l, q in have.items()) if len(have) > 1 else ""
|
|
|
|
if p["kind"] == "post":
|
|
_, _, post_html = md_to_html(p["body"])
|
|
ctx["post_body"] = post_html
|
|
content = render(tpl["news-post"], ctx, "templates/news-post.html")
|
|
lead = "news"
|
|
else:
|
|
body_tpl = tpl["pages/" + p["id"]]
|
|
if "{{news_list}}" in body_tpl:
|
|
ctx["news_list"] = news_lists(lang)[0]
|
|
if "{{latest_news}}" in body_tpl:
|
|
short = news_lists(lang)[1]
|
|
ctx["latest_news"] = render(tpl["latest-news"], dict(ctx, news_items=short), "templates/latest-news.html") if short else ""
|
|
if p["id"] == "home":
|
|
ctx["services_list"] = "\n".join(
|
|
f'<li><a href="{svc[i].get("home_url", svc[i]["url"])}">{svc[i]["label"]}</a>: {esc(data["services"][i])}</li>' for i in site["home_order"])
|
|
elif p["id"] == "services":
|
|
ctx["service_sections"] = "".join(
|
|
render(tpl["parts/service-section"], dict(ctx, id=s["id"], name=esc(data["items"][s["id"]]["name"]),
|
|
text=esc(data["items"][s["id"]]["text"]), url=s["url"]),
|
|
"templates/parts/service-section.html") for s in site["services"])
|
|
ctx.update({"col_" + k: esc(v) for k, v in data["retention_cols"].items()})
|
|
ctx["retention_rows"] = "\n".join(retention_row(r, data["retention_cols"]) for r in data["retention_rows"])
|
|
elif p["id"] == "people":
|
|
cards = []
|
|
for person in site["people"]:
|
|
handle, nm = person["handle"], person.get("name", "")
|
|
entries = ([("home", person["home"], data["personal_page"])] if person.get("home") else [])
|
|
entries += [("link", x["url"], x["label"]) for x in person["links"]]
|
|
links = "\n".join(f'<li><a href="{u}">{esc(label)}</a></li>' for _, u, label in entries)
|
|
if person.get("gpg"):
|
|
links += (f'\n<li>{esc(data["gpg_label"])}: <a href="{site["gpg_key"]}">{os.path.basename(site["gpg_key"])}</a>, '
|
|
f'{esc(data["gpg_word"])} <code>{site["gpg_fingerprint"]}</code></li>')
|
|
cards.append(render(tpl["parts/person-card"], dict(
|
|
ctx, handle=esc(handle), title=(esc(nm) + " (" + esc(handle) + ")") if nm else esc(handle),
|
|
bio=esc(data["bios"][handle]), links=links), "templates/parts/person-card.html"))
|
|
ctx["people_cards"] = "".join(cards)
|
|
elif p["id"] == "history":
|
|
ctx["history_list"] = "\n".join(
|
|
f'<h2 id="v{esc(v["version"])}">{esc(L["version"])} {esc(v["version"])}, <time datetime="{v["date"]}">{v["date"]}</time></h2>\n\n'
|
|
+ "<ul>\n" + "\n".join(f"<li>{esc(c)}</li>" for c in v["changes"]) + "\n</ul>\n" for v in data["versions"])
|
|
elif p["id"] == "manifesto":
|
|
ctx.update(manifesto[lang])
|
|
ctx["note_block"] = f'<p>\n{esc(data["note"])}\n</p>\n\n' if data.get("note") else ""
|
|
content = render(body_tpl.strip("\n") + "\n", ctx, f"templates/pages/{p['id']}.html")
|
|
lead = p["lead"]
|
|
ctx["content"] = content
|
|
ctx["lead"] = render(tpl["lead-" + lead], ctx, f"templates/lead-{lead}.html")
|
|
ctx["feed"] = render(tpl["feed"], ctx, "templates/feed.html")
|
|
ctx["badges"] = render(tpl["badges"], dict(ctx, doc_q=ctx["doc_url"].replace(":", "%3A").replace("/", "%2F")), "templates/badges.html")
|
|
ctx["header"] = render(tpl[L.get("header", "header")], ctx, "templates/header.html")
|
|
ctx["footer"] = render(tpl["footer"], ctx, "templates/footer.html")
|
|
rel = p["path"].lstrip("/")
|
|
write(rel + "index.html" if rel == "" or rel.endswith("/") else rel, render(tpl["base"], ctx, "templates/base.html"))
|
|
|
|
write(site["news_url"].lstrip("/") + "feed.xml", rss(langs["en"], site, posts))
|
|
|
|
for dest, src in ASSETS.items():
|
|
target = os.path.join(OUT, dest)
|
|
os.makedirs(os.path.dirname(target), exist_ok=True)
|
|
shutil.copy(os.path.join(HERE, src), target)
|
|
write("robots.txt", robots_txt(p["path"] for p in pages if indexable(p)))
|
|
print(f"built {len(pages)} pages into {OUT}")
|
|
|
|
|
|
build()
|