feat: let search engines index the site pages and the news
Open indexing to people, services, the manifesto history, the news list and every news post, besides the home and the manifesto. robots.txt and the robots meta are generated from the same rule. Downloads and the personal pages under /~user stay out. Closes #4
This commit is contained in:
parent
b13aa17dff
commit
7d12f468b8
1 changed files with 9 additions and 4 deletions
13
build.py
13
build.py
|
|
@ -36,8 +36,8 @@ PAGES = {
|
||||||
"history": dict(nav="history", lead="back"),
|
"history": dict(nav="history", lead="back"),
|
||||||
}
|
}
|
||||||
NAV = ("home", "manifesto", "people", "services", "news")
|
NAV = ("home", "manifesto", "people", "services", "news")
|
||||||
# Only these pages may be indexed by search engines; every other page gets noindex and is blocked in robots.txt.
|
# The pages of the site and the news posts may be indexed by search engines; anything else gets noindex.
|
||||||
INDEXABLE = ("home", "manifesto")
|
INDEXABLE = ("home", "manifesto", "people", "services", "news", "history")
|
||||||
NOINDEX = '<meta name="robots" content="noindex, nofollow, noarchive">\n'
|
NOINDEX = '<meta name="robots" content="noindex, nofollow, noarchive">\n'
|
||||||
|
|
||||||
# Files copied as they are: published name -> source.
|
# Files copied as they are: published name -> source.
|
||||||
|
|
@ -268,6 +268,11 @@ def retention_row(row, cols):
|
||||||
return f'{todo}<tr>\n<th scope="row">{esc(row["service"])}</th>\n{cells}\n</tr>'
|
return f'{todo}<tr>\n<th scope="row">{esc(row["service"])}</th>\n{cells}\n</tr>'
|
||||||
|
|
||||||
|
|
||||||
|
def indexable(p):
|
||||||
|
"""A listed page of the site or a news post."""
|
||||||
|
return p["kind"] == "post" or (p["kind"] == "page" and p["id"] in INDEXABLE)
|
||||||
|
|
||||||
|
|
||||||
def robots_txt(paths):
|
def robots_txt(paths):
|
||||||
"""Allow the indexable pages (exact paths, '$' anchors the end) and block everything else."""
|
"""Allow the indexable pages (exact paths, '$' anchors the end) and block everything else."""
|
||||||
allow = []
|
allow = []
|
||||||
|
|
@ -347,7 +352,7 @@ def build():
|
||||||
for k in NAV[1:] + ("history",):
|
for k in NAV[1:] + ("history",):
|
||||||
ctx["url_" + k] = url_for(lang, k)
|
ctx["url_" + k] = url_for(lang, k)
|
||||||
ctx["history_attr"] = "" if ("history", lang) in by_id else ' hreflang="en"'
|
ctx["history_attr"] = "" if ("history", lang) in by_id else ' hreflang="en"'
|
||||||
ctx["robots_meta"] = "" if p["kind"] == "page" and p["id"] in INDEXABLE else NOINDEX
|
ctx["robots_meta"] = "" if indexable(p) else NOINDEX
|
||||||
ctx["version_lc"] = esc(L["version"].lower())
|
ctx["version_lc"] = esc(L["version"].lower())
|
||||||
data = p.get("data", {})
|
data = p.get("data", {})
|
||||||
ctx.update({k: esc(v) for k, v in data.items() if isinstance(v, str)})
|
ctx.update({k: esc(v) for k, v in data.items() if isinstance(v, str)})
|
||||||
|
|
@ -432,7 +437,7 @@ def build():
|
||||||
target = os.path.join(OUT, dest)
|
target = os.path.join(OUT, dest)
|
||||||
os.makedirs(os.path.dirname(target), exist_ok=True)
|
os.makedirs(os.path.dirname(target), exist_ok=True)
|
||||||
shutil.copy(os.path.join(HERE, src), target)
|
shutil.copy(os.path.join(HERE, src), target)
|
||||||
write("robots.txt", robots_txt(p["path"] for p in pages if p["kind"] == "page" and p["id"] in INDEXABLE))
|
write("robots.txt", robots_txt(p["path"] for p in pages if indexable(p)))
|
||||||
print(f"built {len(pages)} pages into {OUT}")
|
print(f"built {len(pages)} pages into {OUT}")
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue