feat: let search engines index the site pages and the news

Open indexing to people, services, the manifesto history, the news list and
every news post, besides the home and the manifesto. robots.txt and the
robots meta are generated from the same rule. Downloads and the personal
pages under /~user stay out.

Closes #4
This commit is contained in:
squid 2026-11-14 20:35:16 +01:00
commit 7d12f468b8

View file

@ -36,8 +36,8 @@ PAGES = {
"history": dict(nav="history", lead="back"), "history": dict(nav="history", lead="back"),
} }
NAV = ("home", "manifesto", "people", "services", "news") NAV = ("home", "manifesto", "people", "services", "news")
# Only these pages may be indexed by search engines; every other page gets noindex and is blocked in robots.txt. # The pages of the site and the news posts may be indexed by search engines; anything else gets noindex.
INDEXABLE = ("home", "manifesto") INDEXABLE = ("home", "manifesto", "people", "services", "news", "history")
NOINDEX = '<meta name="robots" content="noindex, nofollow, noarchive">\n' NOINDEX = '<meta name="robots" content="noindex, nofollow, noarchive">\n'
# Files copied as they are: published name -> source. # Files copied as they are: published name -> source.
@ -268,6 +268,11 @@ def retention_row(row, cols):
return f'{todo}<tr>\n<th scope="row">{esc(row["service"])}</th>\n{cells}\n</tr>' return f'{todo}<tr>\n<th scope="row">{esc(row["service"])}</th>\n{cells}\n</tr>'
def indexable(p):
"""A listed page of the site or a news post."""
return p["kind"] == "post" or (p["kind"] == "page" and p["id"] in INDEXABLE)
def robots_txt(paths): def robots_txt(paths):
"""Allow the indexable pages (exact paths, '$' anchors the end) and block everything else.""" """Allow the indexable pages (exact paths, '$' anchors the end) and block everything else."""
allow = [] allow = []
@ -347,7 +352,7 @@ def build():
for k in NAV[1:] + ("history",): for k in NAV[1:] + ("history",):
ctx["url_" + k] = url_for(lang, k) ctx["url_" + k] = url_for(lang, k)
ctx["history_attr"] = "" if ("history", lang) in by_id else ' hreflang="en"' ctx["history_attr"] = "" if ("history", lang) in by_id else ' hreflang="en"'
ctx["robots_meta"] = "" if p["kind"] == "page" and p["id"] in INDEXABLE else NOINDEX ctx["robots_meta"] = "" if indexable(p) else NOINDEX
ctx["version_lc"] = esc(L["version"].lower()) ctx["version_lc"] = esc(L["version"].lower())
data = p.get("data", {}) data = p.get("data", {})
ctx.update({k: esc(v) for k, v in data.items() if isinstance(v, str)}) ctx.update({k: esc(v) for k, v in data.items() if isinstance(v, str)})
@ -432,7 +437,7 @@ def build():
target = os.path.join(OUT, dest) target = os.path.join(OUT, dest)
os.makedirs(os.path.dirname(target), exist_ok=True) os.makedirs(os.path.dirname(target), exist_ok=True)
shutil.copy(os.path.join(HERE, src), target) shutil.copy(os.path.join(HERE, src), target)
write("robots.txt", robots_txt(p["path"] for p in pages if p["kind"] == "page" and p["id"] in INDEXABLE)) write("robots.txt", robots_txt(p["path"] for p in pages if indexable(p)))
print(f"built {len(pages)} pages into {OUT}") print(f"built {len(pages)} pages into {OUT}")