From 7d12f468b875ce48d425394a4043b7765cf9cd9b Mon Sep 17 00:00:00 2001 From: squid Date: Sat, 14 Nov 2026 20:35:16 +0100 Subject: [PATCH] feat: let search engines index the site pages and the news Open indexing to people, services, the manifesto history, the news list and every news post, besides the home and the manifesto. robots.txt and the robots meta are generated from the same rule. Downloads and the personal pages under /~user stay out. Closes #4 --- build.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/build.py b/build.py index 8b0ffe5..6c59851 100644 --- a/build.py +++ b/build.py @@ -36,8 +36,8 @@ PAGES = { "history": dict(nav="history", lead="back"), } NAV = ("home", "manifesto", "people", "services", "news") -# Only these pages may be indexed by search engines; every other page gets noindex and is blocked in robots.txt. -INDEXABLE = ("home", "manifesto") +# The pages of the site and the news posts may be indexed by search engines; anything else gets noindex. +INDEXABLE = ("home", "manifesto", "people", "services", "news", "history") NOINDEX = '\n' # Files copied as they are: published name -> source. @@ -268,6 +268,11 @@ def retention_row(row, cols): return f'{todo}\n{esc(row["service"])}\n{cells}\n' +def indexable(p): + """A listed page of the site or a news post.""" + return p["kind"] == "post" or (p["kind"] == "page" and p["id"] in INDEXABLE) + + def robots_txt(paths): """Allow the indexable pages (exact paths, '$' anchors the end) and block everything else.""" allow = [] @@ -347,7 +352,7 @@ def build(): for k in NAV[1:] + ("history",): ctx["url_" + k] = url_for(lang, k) ctx["history_attr"] = "" if ("history", lang) in by_id else ' hreflang="en"' - ctx["robots_meta"] = "" if p["kind"] == "page" and p["id"] in INDEXABLE else NOINDEX + ctx["robots_meta"] = "" if indexable(p) else NOINDEX ctx["version_lc"] = esc(L["version"].lower()) data = p.get("data", {}) ctx.update({k: esc(v) for k, v in data.items() if isinstance(v, str)}) @@ -432,7 +437,7 @@ def build(): target = os.path.join(OUT, dest) os.makedirs(os.path.dirname(target), exist_ok=True) shutil.copy(os.path.join(HERE, src), target) - write("robots.txt", robots_txt(p["path"] for p in pages if p["kind"] == "page" and p["id"] in INDEXABLE)) + write("robots.txt", robots_txt(p["path"] for p in pages if indexable(p))) print(f"built {len(pages)} pages into {OUT}")