#!/usr/bin/env python3 """Build the kmalloc.org static site (plain HTML, no JavaScript, standard library only). Usage: build.py [output-dir] (default: preview) Layout: site.json what every language shares: languages, contact, Bitcoin address, services, people lang/.json every text of one language: menu, footer, and one entry per page it has templates/ page chrome (base, header, footer), page bodies (pages/) and repeated blocks (parts/) static/style.css the one stylesheet, inlined into every page MANIFESTO..md the manifesto, one Markdown file per language news/*.md news posts, English only: YYYY-MM-DD-slug.md, front matter, then Markdown To add a language, add lang/.json and list it in site.json. A language needs only the pages it has: a page that is missing from lang/.json is simply not built, and its language links disappear. Texts in the JSON files are plain text: the build escapes them. Placeholders {{name}} in the templates are replaced with values from site.json, the language file and the page; an unknown placeholder stops the build.""" import html, json, os, re, shutil, sys from datetime import datetime, timezone from email.utils import format_datetime from xml.sax.saxutils import escape as xml_escape HERE = os.path.dirname(os.path.abspath(__file__)) OUT = os.path.abspath(sys.argv[1] if len(sys.argv) > 1 else os.path.join(HERE, "preview")) NEWS_ON_HOME = 3 NEWS_IN_FEED = 20 POST_NAME = re.compile(r"^(\d{4}-\d{2}-\d{2})-([a-z0-9]+(?:-[a-z0-9]+)*)$") # Page types: menu entry to highlight and footer lead. The body is templates/pages/.html. PAGES = { "home": dict(nav="home", lead="home"), "manifesto": dict(nav="manifesto", lead="back"), "people": dict(nav="people", lead="back"), "services": dict(nav="services", lead="back"), "news": dict(nav="news", lead="back"), "history": dict(nav="history", lead="back"), } NAV = ("home", "manifesto", "people", "services", "news") # Only these pages may be indexed by search engines; every other page gets noindex and is blocked in robots.txt. INDEXABLE = ("home", "manifesto") NOINDEX = '\n' # Files copied as they are: published name -> source. ASSETS = { "favicon.svg": "logo/svg/favicon-auto.svg", "favicon-auto.svg": "logo/svg/favicon-auto.svg", "logo-auto.svg": "logo/svg/mark-auto.svg", "favicon.ico": "logo/web/favicon.ico", "apple-touch-icon.png": "logo/web/apple-touch-icon.png", "badges/w3c-html5.png": "badges/w3c-html5.png", "badges/w3c-css.png": "badges/w3c-css.png", "squid.asc": "static/squid.asc", } def fail(msg): raise SystemExit(f"build.py: {msg}") def read(*parts): with open(os.path.join(HERE, *parts), encoding="utf-8") as f: return f.read() def write(rel, text): path = os.path.join(OUT, rel) os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "w", encoding="utf-8") as f: f.write(text) def load_json(*parts): try: return json.loads(read(*parts)) except ValueError as e: fail(f"{'/'.join(parts)}: {e}") def esc(text): return html.escape(text, quote=False) def parse(text, name): """Split 'front matter, then body' into (dict, body).""" m = re.match(r"---\n(.*?)\n---\n(.*)\Z", text, re.S) if not m: fail(f"{name}: missing front matter") meta = {} for line in m.group(1).splitlines(): key, sep, value = line.partition(": ") if not sep: fail(f"{name}: bad front matter line {line!r}") meta[key.strip()] = value.strip() return meta, m.group(2) def render(text, ctx, name): def sub(m): if m.group(1) not in ctx: fail(f"{name}: unknown placeholder {{{{{m.group(1)}}}}}") return ctx[m.group(1)] return re.sub(r"\{\{(\w+)\}\}", sub, text) # ---------------------------------------------------------------- Markdown def no_em_dash(md): """The manifesto sources use em dashes; the site does not: headings 'NN — Title' become 'NN. Title', 'Word — Title' becomes 'Word: Title', attribution lines lose the dash, parenthetical dashes become commas.""" md = re.sub(r"^(#+ )(\d\d) — ", r"\1\2. ", md, flags=re.M) md = re.sub(r"^(#+ [^\d\n][^—\n]*?) — ", r"\1: ", md, flags=re.M) md = re.sub(r"^> — ", "> ", md, flags=re.M) for a, b in ((" — where technically feasible — ", ", where technically feasible, "), (" — dove tecnicamente possibile — ", ", dove tecnicamente possibile, "), (" — and continues", ", and continues"), (" — e continua", ", e continua")): md = md.replace(a, b) md = md.replace("kmalloc.org — ", "kmalloc.org: ") assert "—" not in md, [l for l in md.splitlines() if "—" in l] return md def inline(s): s = html.escape(s, quote=False) s = re.sub(r"`([^`]+)`", r"\1", s) s = re.sub(r"\[([^\]]+)\]\(([^)\s]+)\)", r'\1', s) s = re.sub(r"\*\*(.+?)\*\*", r"\1", s) s = re.sub(r"(?\1", s) return s def slug(heading): m = re.match(r"(\d\d)\b", heading) if m: return m.group(1) h = heading.lower() if h.startswith(("preamble", "preambolo", "preámbulo", "präambel", "préambule")): return "preamble" if h.startswith(("our commitments", "i nostri impegni", "nuestros compromisos", "unsere verpflichtungen", "nos engagements")): return "commitments" if h.startswith(("epilogue", "epilogo", "epílogo", "epilog", "épilogue")): return "epilogue" return re.sub(r"[^a-z0-9]+", "-", h).strip("-") def md_to_html(md): """Headings (##), paragraphs, quotes, bullet lists and inline markup. Returns (title, toc, html).""" out, toc = [], [] para, quote, items = [], [], [] def flush(): nonlocal para, quote, items if para: out.append("

\n" + "\n".join(para) + "\n

\n") para = [] if quote: out.append("
\n

\n" + "\n".join(quote) + "\n

\n
\n") quote = [] if items: out.append("
    \n" + "\n".join(f"
  • {i}
  • " for i in items) + "\n
\n") items = [] title = "" for raw in md.splitlines(): line = raw.rstrip("\n") br = line.endswith(" ") s = line.strip() if not s: flush() continue if s.startswith("# "): flush() title = s[2:] continue if s.startswith("## "): flush() h = s[3:] sl = slug(h) toc.append((sl, h)) out.append(f'

{inline(h)}

\n') continue if s.startswith("> "): if para or items: flush() quote.append(inline(s[2:]) + ("
" if br else "")) continue if s.startswith("- "): if para or quote: flush() items.append(inline(s[2:])) continue if quote or items: flush() para.append(inline(s)) flush() return title, toc, "".join(out) # ---------------------------------------------------------------- site model def load_posts(): """News posts from news/*.md (English only).""" posts = [] folder = os.path.join(HERE, "news") for fn in sorted(os.listdir(folder)) if os.path.isdir(folder) else []: if not fn.endswith(".md"): continue m = POST_NAME.match(fn[:-3]) if not m: fail(f"news/{fn}: name must be YYYY-MM-DD-slug.md") src = f"news/{fn}" meta, text = parse(read("news", fn), src) if "title" not in meta: fail(f"{src}: front matter lacks 'title'") if "\u2014" in text or "\u2014" in meta["title"]: fail(f"{src}: no em dashes, please") posts.append(dict(lang="en", kind="post", id="news/" + fn[:-3], date=m.group(1), nav="news", path=f"/news/{m.group(2)}.html", body=text, src=src, title=meta["title"], summary=meta.get("summary", ""), description=meta.get("summary") or meta["title"])) return sorted(posts, key=lambda p: p["date"], reverse=True) def rss(L, site, all_posts): """RSS 2.0 feed of the news (English only): the latest posts, full text, absolute links.""" base = site["base_url"] posts = all_posts[:NEWS_IN_FEED] def stamp(d): return format_datetime(datetime.strptime(d, "%Y-%m-%d").replace(tzinfo=timezone.utc)) items = [] for p in posts: text = md_to_html(p["body"])[2].replace('href="/', f'href="{base}/') items.append(f""" {xml_escape(p["title"])} {base}{p["path"]} {base}{p["path"]} {stamp(p["date"])} {xml_escape(text)} """) built = f"{stamp(posts[0]['date'])}\n" if posts else "" url = f"{base}{site['news_url']}" return f""" kmalloc.org: {xml_escape(L["nav"]["news"])} {url} {xml_escape(L["news_p"])} {L["lang"]} {built} {"".join(items)} """ def retention_row(row, cols): """One row of the data and retention table. A 'todo' field becomes an HTML comment, never visible text.""" if "--" in row.get("todo", ""): fail(f"retention row {row['service']!r}: no '--' inside a TODO comment") todo = f"\n" if row.get("todo") else "" cells = "\n".join(f'{esc(row[k])}' for k in ("admin", "not_e2e", "kept")) return f'{todo}\n{esc(row["service"])}\n{cells}\n' def robots_txt(paths): """Allow the indexable pages (exact paths, '$' anchors the end) and block everything else.""" allow = [] for path in paths: allow.append(f"Allow: {path}$") if path == "/": allow.append("Allow: /?lang=en$") # the English home as linked from the other pages return "User-agent: *\n" + "\n".join(sorted(set(allow))) + "\nDisallow: /\n" def build(): site = load_json("site.json") langs_order = site["languages"] langs = {l: load_json("lang", l + ".json") for l in langs_order} css = read("static", "style.css").strip("\n") tpl = {} for root, _, files in os.walk(os.path.join(HERE, "templates")): for fn in files: if fn.endswith(".html"): key = os.path.relpath(os.path.join(root, fn), os.path.join(HERE, "templates"))[:-5] tpl[key] = read("templates", key + ".html") shared = {k: v for k, v in site.items() if isinstance(v, str)} for k in ("contact", "security", "abuse"): # no mailto link, against address harvesters shared[k + "_shown"] = site[k].replace("@", " (at) ") svc = {s["id"]: s for s in site["services"]} pages = [] for lang in langs_order: for pid, spec in PAGES.items(): data = langs[lang]["pages"].get(pid) if data: pages.append(dict(lang=lang, kind="page", id=pid, nav=spec["nav"], lead=spec["lead"], path=data["path"], title=data["title"], description=data["description"], data=data, src=f"lang/{lang}.json: {pid}")) posts = load_posts() pages += posts by_id = {(p["id"], p["lang"]): p for p in pages} if ("news", "en") not in by_id: fail("lang/en.json needs a 'news' page") site["news_url"] = by_id[("news", "en")]["path"] def url_for(lang, pid): """Page in this language, else the English one.""" return (by_id.get((pid, lang)) or by_id[(pid, "en")])["path"] # Manifesto: one Markdown file per language that has the page. manifesto = {} for lang in langs_order: if ("manifesto", lang) in by_id: title, toc, md_html = md_to_html(no_em_dash(read(f"MANIFESTO.{lang}.md"))) manifesto[lang] = dict( manifesto_title=html.escape(title), manifesto_toc="\n".join(f'
  • {inline(h)}
  • ' for sl, h in toc), manifesto_body=md_html) def news_lists(lang): mark = "" if lang == "en" else ' hreflang="en" lang="en"' lines = [] for p in posts: line = f'
  • {esc(p["title"])}' if p["summary"]: line += ": " + esc(p["summary"]) lines.append(line + "
  • ") full = "
      \n" + "\n".join(lines) + "\n
    " if posts else f"

    \n{esc(langs[lang].get('news_empty', ''))}\n

    " short = "\n".join(f'
  • {esc(x["title"])}
  • ' for x in posts[:NEWS_ON_HOME]) return full, short for p in pages: lang = p["lang"] L = langs[lang] name = p["src"] ctx = dict(shared) ctx.update({k: esc(v) for k, v in L.items() if isinstance(v, str)}) ctx["lang"] = lang for k, label in L["nav"].items(): ctx["nav_" + k] = esc(label) for k in NAV[1:] + ("history",): ctx["url_" + k] = url_for(lang, k) ctx["history_attr"] = "" if ("history", lang) in by_id else ' hreflang="en"' ctx["robots_meta"] = "" if p["kind"] == "page" and p["id"] in INDEXABLE else NOINDEX ctx["version_lc"] = esc(L["version"].lower()) data = p.get("data", {}) ctx.update({k: esc(v) for k, v in data.items() if isinstance(v, str)}) # A list of plain strings in a page entry becomes list items: {{name}} goes inside
      . ctx.update({k: "\n".join(f"
    • {esc(x)}
    • " for x in v) for k, v in data.items() if isinstance(v, list) and all(isinstance(x, str) for x in v)}) page_title = p["title"] if p["kind"] == "page" else "kmalloc.org: " + p["title"] ctx.update(css=css, path=p["path"], title_html=html.escape(page_title), post_title=html.escape(p["title"]), description_attr=html.escape(p["description"], quote=True), doc_url=site["base_url"] + p["path"], date=p.get("date", "")) for k in NAV: ctx["cur_" + k] = ' aria-current="page"' if p["nav"] == k else "" # Every other language that has this page; a page without translation (news) points to the other home. twins = {l: by_id[(p["id"], l)] for l in langs_order if l != lang and (p["id"], l) in by_id} if not twins: fallback = "it" if lang == "en" else "en" twins = {fallback: by_id[("home", fallback)]} switch = data.get("switch", {}) ctx["lang_links"] = "\n·\n".join( f'{esc(langs[l]["lang_name"])}' for l, t in twins.items()) have = {l: by_id[(p["id"], l)] for l in langs_order if (p["id"], l) in by_id} ctx["alt_links"] = "\n".join(f'' for l, q in have.items()) if len(have) > 1 else "" if p["kind"] == "post": _, _, post_html = md_to_html(p["body"]) ctx["post_body"] = post_html content = render(tpl["news-post"], ctx, "templates/news-post.html") lead = "news" else: body_tpl = tpl["pages/" + p["id"]] if "{{news_list}}" in body_tpl: ctx["news_list"] = news_lists(lang)[0] if "{{latest_news}}" in body_tpl: short = news_lists(lang)[1] ctx["latest_news"] = render(tpl["latest-news"], dict(ctx, news_items=short), "templates/latest-news.html") if short else "" if p["id"] == "home": ctx["services_list"] = "\n".join( f'
    • {svc[i]["label"]}: {esc(data["services"][i])}
    • ' for i in site["home_order"]) elif p["id"] == "services": ctx["service_sections"] = "".join( render(tpl["parts/service-section"], dict(ctx, id=s["id"], name=esc(data["items"][s["id"]]["name"]), text=esc(data["items"][s["id"]]["text"]), url=s["url"]), "templates/parts/service-section.html") for s in site["services"]) ctx.update({"col_" + k: esc(v) for k, v in data["retention_cols"].items()}) ctx["retention_rows"] = "\n".join(retention_row(r, data["retention_cols"]) for r in data["retention_rows"]) elif p["id"] == "people": cards = [] for person in site["people"]: handle, nm = person["handle"], person.get("name", "") entries = ([("home", person["home"], data["personal_page"])] if person.get("home") else []) entries += [("link", x["url"], x["label"]) for x in person["links"]] links = "\n".join(f'
    • {esc(label)}
    • ' for _, u, label in entries) if person.get("gpg"): links += (f'\n
    • {esc(data["gpg_label"])}: {os.path.basename(site["gpg_key"])}, ' f'{esc(data["gpg_word"])} {site["gpg_fingerprint"]}
    • ') cards.append(render(tpl["parts/person-card"], dict( ctx, handle=esc(handle), title=(esc(nm) + " (" + esc(handle) + ")") if nm else esc(handle), bio=esc(data["bios"][handle]), links=links), "templates/parts/person-card.html")) ctx["people_cards"] = "".join(cards) elif p["id"] == "history": ctx["history_list"] = "\n".join( f'

      {esc(L["version"])} {esc(v["version"])},

      \n\n' + "
        \n" + "\n".join(f"
      • {esc(c)}
      • " for c in v["changes"]) + "\n
      \n" for v in data["versions"]) elif p["id"] == "manifesto": ctx.update(manifesto[lang]) ctx["note_block"] = f'

      \n{esc(data["note"])}\n

      \n\n' if data.get("note") else "" content = render(body_tpl.strip("\n") + "\n", ctx, f"templates/pages/{p['id']}.html") lead = p["lead"] ctx["content"] = content ctx["lead"] = render(tpl["lead-" + lead], ctx, f"templates/lead-{lead}.html") ctx["feed"] = render(tpl["feed"], ctx, "templates/feed.html") ctx["badges"] = render(tpl["badges"], dict(ctx, doc_q=ctx["doc_url"].replace(":", "%3A").replace("/", "%2F")), "templates/badges.html") ctx["header"] = render(tpl[L.get("header", "header")], ctx, "templates/header.html") ctx["footer"] = render(tpl["footer"], ctx, "templates/footer.html") rel = p["path"].lstrip("/") write(rel + "index.html" if rel == "" or rel.endswith("/") else rel, render(tpl["base"], ctx, "templates/base.html")) write(site["news_url"].lstrip("/") + "feed.xml", rss(langs["en"], site, posts)) for dest, src in ASSETS.items(): target = os.path.join(OUT, dest) os.makedirs(os.path.dirname(target), exist_ok=True) shutil.copy(os.path.join(HERE, src), target) write("robots.txt", robots_txt(p["path"] for p in pages if p["kind"] == "page" and p["id"] in INDEXABLE)) print(f"built {len(pages)} pages into {OUT}") build()