chore: add the one-off migration and compare tools
This commit is contained in:
parent
7d12f468b8
commit
e0607bbda3
4 changed files with 538 additions and 0 deletions
127
tools/compare.py
Normal file
127
tools/compare.py
Normal file
|
|
@ -0,0 +1,127 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Compare the site built by build.py (old) with the one built by motorino (new), page by page.
|
||||||
|
|
||||||
|
Usage: compare.py OLD_DIR NEW_DIR
|
||||||
|
|
||||||
|
The text, the tags, their attributes, the links and the comments of every page must be the same. The
|
||||||
|
differences that the change wants are ignored: the "Powered by motorino" footer line, the generator meta, the
|
||||||
|
author and categories after the date of a news item, and its description meta. Old news addresses are moved
|
||||||
|
to the new dated ones before the comparison. Exit status 1 when anything else differs."""
|
||||||
|
import difflib
|
||||||
|
import html as htmllib
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from html.parser import HTMLParser
|
||||||
|
|
||||||
|
|
||||||
|
class Tokens(HTMLParser):
|
||||||
|
"""The stream of (kind, value) tokens of a page: whitespace is collapsed and attribute order does not matter."""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__(convert_charrefs=True)
|
||||||
|
self.items = []
|
||||||
|
|
||||||
|
def handle_starttag(self, tag, attrs):
|
||||||
|
self.items.append(("start", tag + "".join(f' {k}="{v}"' for k, v in sorted(attrs))))
|
||||||
|
|
||||||
|
def handle_startendtag(self, tag, attrs):
|
||||||
|
self.handle_starttag(tag, attrs)
|
||||||
|
|
||||||
|
def handle_endtag(self, tag):
|
||||||
|
self.items.append(("end", tag))
|
||||||
|
|
||||||
|
def handle_data(self, data):
|
||||||
|
text = " ".join(data.split())
|
||||||
|
if text:
|
||||||
|
if self.items and self.items[-1][0] == "text":
|
||||||
|
self.items[-1] = ("text", self.items[-1][1] + " " + text)
|
||||||
|
else:
|
||||||
|
self.items.append(("text", text))
|
||||||
|
|
||||||
|
def handle_comment(self, data):
|
||||||
|
self.items.append(("comment", " ".join(data.split())))
|
||||||
|
|
||||||
|
|
||||||
|
CREDIT = re.compile(r"<p>\s*Powered by .*?</p>\s*", re.S)
|
||||||
|
GENERATOR = re.compile(r'<meta name="generator"[^>]*>\s*')
|
||||||
|
BYLINE = re.compile(r'(<time datetime="[^"]+">[^<]*</time>)\s*·.*?(\s*</p>)', re.S)
|
||||||
|
DESCRIPTION = re.compile(r'<meta (?:name="description"|property="og:description")[^>]*>\s*')
|
||||||
|
|
||||||
|
|
||||||
|
def tokens(html, post=False):
|
||||||
|
html = GENERATOR.sub("", CREDIT.sub("", html))
|
||||||
|
if post:
|
||||||
|
html = DESCRIPTION.sub("", BYLINE.sub(r"\1\2", html, count=1))
|
||||||
|
parser = Tokens()
|
||||||
|
parser.feed(html)
|
||||||
|
return [f"{kind}: {value}" for kind, value in parser.items]
|
||||||
|
|
||||||
|
|
||||||
|
def compare_pages(old, new, post=False):
|
||||||
|
"""The differences between two pages, as lines of a diff; empty when they match."""
|
||||||
|
a, b = tokens(old, post), tokens(new, post)
|
||||||
|
if a == b:
|
||||||
|
return []
|
||||||
|
return [line for line in difflib.unified_diff(a, b, "old", "new", lineterm="", n=2)]
|
||||||
|
|
||||||
|
|
||||||
|
def remap(html, mapping):
|
||||||
|
"""Replace old addresses by new ones."""
|
||||||
|
for old, new in sorted(mapping.items(), key=lambda kv: -len(kv[0])):
|
||||||
|
html = html.replace(old, new)
|
||||||
|
return html
|
||||||
|
|
||||||
|
|
||||||
|
def html_files(root):
|
||||||
|
for folder, _, files in os.walk(root):
|
||||||
|
for name in files:
|
||||||
|
if name.endswith(".html"):
|
||||||
|
yield "/" + os.path.relpath(os.path.join(folder, name), root).replace(os.sep, "/")
|
||||||
|
|
||||||
|
|
||||||
|
def read(root, rel):
|
||||||
|
with open(os.path.join(root, rel.lstrip("/")), encoding="utf-8") as f:
|
||||||
|
return f.read()
|
||||||
|
|
||||||
|
|
||||||
|
def post_mapping(old_dir, new_dir):
|
||||||
|
"""Old news addresses (/news/slug.html) to the new dated ones, found by name in the new output."""
|
||||||
|
new_posts = {}
|
||||||
|
for rel in html_files(new_dir):
|
||||||
|
m = re.fullmatch(r"/news/(\d{4})/(\d{2})/([a-z0-9-]+)\.html", rel)
|
||||||
|
if m:
|
||||||
|
new_posts[m.group(3)] = rel
|
||||||
|
mapping = {}
|
||||||
|
for rel in html_files(old_dir):
|
||||||
|
m = re.fullmatch(r"/news/([a-z0-9-]+)\.html", rel)
|
||||||
|
if m and m.group(1) in new_posts:
|
||||||
|
mapping[rel] = new_posts[m.group(1)]
|
||||||
|
return mapping
|
||||||
|
|
||||||
|
|
||||||
|
def main(old_dir, new_dir):
|
||||||
|
mapping = post_mapping(old_dir, new_dir)
|
||||||
|
bad = 0
|
||||||
|
for rel in sorted(html_files(old_dir)):
|
||||||
|
new_rel = mapping.get(rel, rel)
|
||||||
|
if not os.path.exists(os.path.join(new_dir, new_rel.lstrip("/"))):
|
||||||
|
print(f"MISSING in new output: {new_rel}")
|
||||||
|
bad += 1
|
||||||
|
continue
|
||||||
|
diffs = compare_pages(remap(read(old_dir, rel), mapping), read(new_dir, new_rel), post=rel in mapping)
|
||||||
|
if diffs:
|
||||||
|
bad += 1
|
||||||
|
print(f"DIFFERENT: {rel}" + (f" -> {new_rel}" if new_rel != rel else ""))
|
||||||
|
print("\n".join(diffs[:60]))
|
||||||
|
extra = sorted(set(html_files(new_dir)) - {mapping.get(r, r) for r in html_files(old_dir)})
|
||||||
|
for rel in extra:
|
||||||
|
print(f"new page: {rel}")
|
||||||
|
print(f"compared {len(list(html_files(old_dir)))} pages, {bad} differ")
|
||||||
|
return 1 if bad else 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
if len(sys.argv) != 3:
|
||||||
|
sys.exit(__doc__)
|
||||||
|
sys.exit(main(sys.argv[1], sys.argv[2]))
|
||||||
251
tools/migrate.py
Normal file
251
tools/migrate.py
Normal file
|
|
@ -0,0 +1,251 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""One-off migration of the kmalloc.org site from build.py to motorino.
|
||||||
|
|
||||||
|
Usage: migrate.py [--site DIR] [--old-out DIR]
|
||||||
|
|
||||||
|
It builds the current site with build.py (the reference, saved in --old-out), then writes the new layout in
|
||||||
|
place: pages as Markdown in content/<lang>/, news with the new header, the reduced site.json and lang files,
|
||||||
|
and the new templates. The pages are converted from the HTML that build.py produced, so their text is the
|
||||||
|
same by construction; HTML that Markdown cannot express (a table, an image, a heading with a class) is kept
|
||||||
|
as raw HTML in the file. Delete this tool once the new site is live."""
|
||||||
|
import argparse
|
||||||
|
import ast
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- HTML to Markdown
|
||||||
|
|
||||||
|
BLOCK = re.compile(r"\s*(?:(<!--.*?-->)|<(p|h[1-6]|ul|ol|table|blockquote|div)\b([^>]*)>(.*?)</\2>)", re.S)
|
||||||
|
ID_ONLY = re.compile(r'^\s*id="([^"]+)"\s*$')
|
||||||
|
|
||||||
|
|
||||||
|
def inline_to_md(inner):
|
||||||
|
"""Inline HTML as Markdown, or None when it has markup that Markdown cannot say."""
|
||||||
|
s = inner.strip()
|
||||||
|
s = re.sub(r'<a href="([^"]+)">(.*?)</a>', r"[\2](\1)", s, flags=re.S)
|
||||||
|
s = re.sub(r"<strong>(.*?)</strong>", r"**\1**", s, flags=re.S)
|
||||||
|
s = re.sub(r"<em>(.*?)</em>", r"*\1*", s, flags=re.S)
|
||||||
|
s = re.sub(r"<code>(.*?)</code>", r"`\1`", s, flags=re.S)
|
||||||
|
if "<" in s:
|
||||||
|
return None
|
||||||
|
return "\n".join(line.strip() for line in s.splitlines())
|
||||||
|
|
||||||
|
|
||||||
|
def block_to_md(match):
|
||||||
|
"""One top-level block as Markdown, or as the raw HTML it came from."""
|
||||||
|
raw = match.group(0).strip()
|
||||||
|
comment, tag, attrs, inner = match.groups()
|
||||||
|
if comment:
|
||||||
|
return raw
|
||||||
|
if tag == "p" and not attrs.strip():
|
||||||
|
text = inline_to_md(inner)
|
||||||
|
return text if text is not None else raw
|
||||||
|
if re.fullmatch(r"h[1-6]", tag):
|
||||||
|
text = inline_to_md(inner)
|
||||||
|
heading_id = ID_ONLY.match(attrs)
|
||||||
|
if text is not None and (not attrs.strip() or heading_id):
|
||||||
|
line = "#" * int(tag[1]) + " " + text
|
||||||
|
return line + (" {#" + heading_id.group(1) + "}" if heading_id else "")
|
||||||
|
return raw
|
||||||
|
if tag == "ul" and not attrs.strip():
|
||||||
|
items = re.findall(r"<li>(.*?)</li>", inner, flags=re.S)
|
||||||
|
texts = [inline_to_md(i) for i in items]
|
||||||
|
if items and all(t is not None for t in texts) and not re.sub(r"<li>.*?</li>|\s", "", inner, flags=re.S):
|
||||||
|
return "\n".join("- " + t for t in texts)
|
||||||
|
return raw
|
||||||
|
|
||||||
|
|
||||||
|
def html_to_md(html):
|
||||||
|
"""Convert the body of a page: the blocks of build.py output become Markdown, the rest stays HTML."""
|
||||||
|
parts, pos = [], 0
|
||||||
|
while True:
|
||||||
|
if not html[pos:].strip():
|
||||||
|
break
|
||||||
|
m = BLOCK.match(html, pos)
|
||||||
|
if not m:
|
||||||
|
raise ValueError(f"unconvertible HTML near: {html[pos:pos + 80].strip()!r}")
|
||||||
|
parts.append(block_to_md(m))
|
||||||
|
pos = m.end()
|
||||||
|
return "\n\n".join(parts) + "\n"
|
||||||
|
|
||||||
|
|
||||||
|
def restore(md, values):
|
||||||
|
"""Put {{key}} back where a value of site.json appears as text, longest values first."""
|
||||||
|
for key, value in sorted(values.items(), key=lambda kv: -len(kv[1])):
|
||||||
|
if value:
|
||||||
|
md = md.replace(value, "{{" + key + "}}")
|
||||||
|
return md
|
||||||
|
|
||||||
|
|
||||||
|
def mark_latest_news(html):
|
||||||
|
"""The news block of the home page becomes the {{latest_news}} placeholder."""
|
||||||
|
return re.sub(r'<h2 id="news">.*?</ul>\n?', "<p>{{latest_news}}</p>\n", html, count=1, flags=re.S)
|
||||||
|
|
||||||
|
|
||||||
|
def mark_news_list(html):
|
||||||
|
"""The list of the news page becomes the {{news_list}} placeholder."""
|
||||||
|
return re.sub(r"<ul>.*?</ul>", "<p>{{news_list}}</p>", html, count=1, flags=re.S)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- pages and news
|
||||||
|
|
||||||
|
def extract_content(page_html):
|
||||||
|
"""The part of a built page between the header and the footer."""
|
||||||
|
m = re.search(r"</header>\n\n(.*)\n<footer>", page_html, re.S)
|
||||||
|
if not m:
|
||||||
|
raise ValueError("no content between header and footer")
|
||||||
|
return m.group(1).rstrip("\n") + "\n"
|
||||||
|
|
||||||
|
|
||||||
|
def page_file(path, default_lang, lang):
|
||||||
|
"""File of a page under content/<lang>/, from its address."""
|
||||||
|
if lang != default_lang and path.startswith(f"/{lang}/"):
|
||||||
|
path = path[len(lang) + 1:]
|
||||||
|
if path.endswith("/"):
|
||||||
|
return path.lstrip("/") + "index.md"
|
||||||
|
return path.lstrip("/")[: -len(".html")] + ".md"
|
||||||
|
|
||||||
|
|
||||||
|
def page_header(title, page_id, description, layout, note="", switch=None):
|
||||||
|
"""The header of a page file."""
|
||||||
|
lines = [f"Title: {title}", f"ID: {page_id}", f"Description: {description}", f"Layout: {layout}"]
|
||||||
|
if note:
|
||||||
|
lines.append(f"Note: {note}")
|
||||||
|
if switch:
|
||||||
|
lines.append("Switch: " + ", ".join(f"{k}={v}" for k, v in switch.items()))
|
||||||
|
return "\n".join(lines) + "\n-\n"
|
||||||
|
|
||||||
|
|
||||||
|
def convert_post(filename, text):
|
||||||
|
"""A news file with the old front matter becomes one with the new header."""
|
||||||
|
m = re.match(r"(\d{4}-\d{2}-\d{2})-(.+)\.md\Z", filename)
|
||||||
|
date, slug = m.group(1), m.group(2)
|
||||||
|
head = re.match(r"---\n(.*?)\n---\n(.*)\Z", text, re.S)
|
||||||
|
meta = dict(line.split(": ", 1) for line in head.group(1).splitlines())
|
||||||
|
lines = [f"Title: {meta['title']}", "Author: squid", f"Date: {date}T12:00:00+0100", "Categories: News"]
|
||||||
|
if meta.get("summary"):
|
||||||
|
lines.append(f"Description: {meta['summary']}")
|
||||||
|
return slug + ".md", "\n".join(lines) + "\n-\n" + head.group(2)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- the migration
|
||||||
|
|
||||||
|
def write(path, text):
|
||||||
|
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||||
|
with open(path, "w", encoding="utf-8") as f:
|
||||||
|
f.write(text)
|
||||||
|
|
||||||
|
|
||||||
|
def read(path):
|
||||||
|
with open(path, encoding="utf-8") as f:
|
||||||
|
return f.read()
|
||||||
|
|
||||||
|
|
||||||
|
def old_assets(build_py):
|
||||||
|
"""The ASSETS dict of build.py."""
|
||||||
|
m = re.search(r"^ASSETS = (\{.*?^\})", build_py, re.S | re.M)
|
||||||
|
return ast.literal_eval(m.group(1))
|
||||||
|
|
||||||
|
|
||||||
|
def site_values(site):
|
||||||
|
"""The literal values that go back to placeholders in the page text."""
|
||||||
|
shown = {k + "_shown": site[k].replace("@", " (at) ") for k in ("contact", "security", "abuse")}
|
||||||
|
keep = {k: site[k] for k in ("btc", "shop_url", "gpg_fingerprint", "gpg_key", "gpg_keyserver", "issues_url", "mastodon_url")}
|
||||||
|
return {**shown, **keep}
|
||||||
|
|
||||||
|
|
||||||
|
def migrate_pages(root, old_out, site, langs):
|
||||||
|
default = site["languages"][0]
|
||||||
|
values = site_values(site)
|
||||||
|
for lang in site["languages"]:
|
||||||
|
for pid, data in langs[lang]["pages"].items():
|
||||||
|
path = data["path"]
|
||||||
|
switch = data.get("switch")
|
||||||
|
if pid == "manifesto":
|
||||||
|
body = read(os.path.join(root, f"MANIFESTO.{lang}.md"))
|
||||||
|
body = re.sub(r"\A# .*\n\n?", "", body)
|
||||||
|
layout, note = "manifesto", data.get("note", "")
|
||||||
|
else:
|
||||||
|
out_file = os.path.join(old_out, path.lstrip("/"), "index.html") if path.endswith("/") else os.path.join(old_out, path.lstrip("/"))
|
||||||
|
html = extract_content(read(out_file))
|
||||||
|
if pid == "home":
|
||||||
|
html = mark_latest_news(html)
|
||||||
|
html = html.replace(f"({langs[lang]['version'].lower()} {site['manifesto_version']})",
|
||||||
|
"({{version_lc}} {{manifesto_version}})")
|
||||||
|
if pid == "news":
|
||||||
|
html = mark_news_list(html)
|
||||||
|
body = restore(html_to_md(html), values)
|
||||||
|
layout, note = "bare", ""
|
||||||
|
write(os.path.join(root, "content", lang, page_file(path, default, lang)),
|
||||||
|
page_header(data["title"], pid, data["description"], layout, note, switch) + body)
|
||||||
|
|
||||||
|
|
||||||
|
def migrate_news(root):
|
||||||
|
folder = os.path.join(root, "news")
|
||||||
|
for name in sorted(os.listdir(folder)):
|
||||||
|
if re.match(r"\d{4}-\d{2}-\d{2}-.+\.md\Z", name):
|
||||||
|
new_name, text = convert_post(name, read(os.path.join(folder, name)))
|
||||||
|
write(os.path.join(folder, new_name), text)
|
||||||
|
os.remove(os.path.join(folder, name))
|
||||||
|
|
||||||
|
|
||||||
|
def migrate_config(root, site, langs, assets):
|
||||||
|
new_site = {k: v for k, v in site.items() if isinstance(v, str) or k == "languages"}
|
||||||
|
new_site["motorino_url"] = "https://git.kmalloc.org/squid/motorino"
|
||||||
|
new_site["assets"] = assets
|
||||||
|
write(os.path.join(root, "site.json"), json.dumps(new_site, indent=2, ensure_ascii=False) + "\n")
|
||||||
|
for lang, data in langs.items():
|
||||||
|
slim = {k: v for k, v in data.items() if k != "pages"}
|
||||||
|
slim["powered_by"] = "Powered by"
|
||||||
|
slim["category_h"] = {"en": "Category", "it": "Categoria"}.get(lang, "Category")
|
||||||
|
write(os.path.join(root, "lang", lang + ".json"), json.dumps(slim, indent=2, ensure_ascii=False) + "\n")
|
||||||
|
|
||||||
|
|
||||||
|
def migrate_templates(root):
|
||||||
|
tpl = os.path.join(root, "templates")
|
||||||
|
base = read(os.path.join(tpl, "base.html"))
|
||||||
|
base = base.replace('{{robots_meta}}<meta name="description"', '<meta name="description"')
|
||||||
|
base = base.replace('<meta name="color-scheme" content="light dark">\n',
|
||||||
|
'<meta name="color-scheme" content="light dark">\n<meta name="generator" content="motorino {{motorino_version}}">\n')
|
||||||
|
write(os.path.join(tpl, "base.html"), base)
|
||||||
|
footer = read(os.path.join(tpl, "footer.html"))
|
||||||
|
footer = footer.replace("{{badges}}", '<p>\n{{powered_by}} <a href="{{motorino_url}}">motorino</a>.\n</p>\n{{badges}}')
|
||||||
|
write(os.path.join(tpl, "footer.html"), footer)
|
||||||
|
manifesto = read(os.path.join(tpl, "pages", "manifesto.html"))
|
||||||
|
write(os.path.join(tpl, "pages", "manifesto.html"),
|
||||||
|
manifesto.replace("{{manifesto_title}}", "{{page_title}}").replace("{{manifesto_body}}", "{{body}}"))
|
||||||
|
write(os.path.join(tpl, "pages", "bare.html"), "{{body}}\n")
|
||||||
|
write(os.path.join(tpl, "pages", "page.html"), "<h1>{{page_title}}</h1>\n\n{{body}}\n")
|
||||||
|
write(os.path.join(tpl, "post.html"),
|
||||||
|
'<h1>{{post_title}}</h1>\n\n<p>\n<time datetime="{{date}}">{{date}}</time>{{post_byline}}\n</p>\n\n{{post_body}}\n')
|
||||||
|
write(os.path.join(tpl, "category.html"), "<h1>{{category_h}}: {{category_name}}</h1>\n\n{{category_posts}}\n")
|
||||||
|
for old in ("pages/home", "pages/people", "pages/services", "pages/news", "pages/history", "news-post"):
|
||||||
|
os.remove(os.path.join(tpl, old + ".html"))
|
||||||
|
shutil.rmtree(os.path.join(tpl, "parts"))
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser(description=__doc__)
|
||||||
|
ap.add_argument("--site", default=".")
|
||||||
|
ap.add_argument("--old-out", default="/tmp/kmalloc-old-out")
|
||||||
|
args = ap.parse_args()
|
||||||
|
root = os.path.abspath(args.site)
|
||||||
|
if os.path.exists(args.old_out):
|
||||||
|
shutil.rmtree(args.old_out)
|
||||||
|
subprocess.run([sys.executable, os.path.join(root, "build.py"), args.old_out], check=True)
|
||||||
|
site = json.loads(read(os.path.join(root, "site.json")))
|
||||||
|
langs = {l: json.loads(read(os.path.join(root, "lang", l + ".json"))) for l in site["languages"]}
|
||||||
|
assets = old_assets(read(os.path.join(root, "build.py")))
|
||||||
|
migrate_pages(root, args.old_out, site, langs)
|
||||||
|
migrate_news(root)
|
||||||
|
migrate_config(root, site, langs, assets)
|
||||||
|
migrate_templates(root)
|
||||||
|
print(f"migrated; the reference output is in {args.old_out}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
59
tools/test_compare.py
Normal file
59
tools/test_compare.py
Normal file
|
|
@ -0,0 +1,59 @@
|
||||||
|
"""Tests of the old-versus-new output comparison."""
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
import compare
|
||||||
|
|
||||||
|
|
||||||
|
def page(body, footer="", head=""):
|
||||||
|
return f"<html><head>{head}</head><body>\n<header>h</header>\n\n{body}\n<footer>\n{footer}\n</footer>\n</body></html>"
|
||||||
|
|
||||||
|
|
||||||
|
class ComparePages(unittest.TestCase):
|
||||||
|
def test_identical_pages_have_no_differences(self):
|
||||||
|
html = page('<h1>T</h1>\n<p>\nHello <a href="/x.html" hreflang="en">link</a>.\n</p>')
|
||||||
|
self.assertEqual(compare.compare_pages(html, html), [])
|
||||||
|
|
||||||
|
def test_whitespace_and_attribute_order_do_not_matter(self):
|
||||||
|
old = page('<p>\nHello\n world <a href="/x" lang="en" hreflang="en">l</a>\n</p>')
|
||||||
|
new = page('<p>Hello world <a hreflang="en" href="/x" lang="en">l</a></p>')
|
||||||
|
self.assertEqual(compare.compare_pages(old, new), [])
|
||||||
|
|
||||||
|
def test_a_changed_link_is_reported(self):
|
||||||
|
old = page('<p>\n<a href="/a.html">x</a>\n</p>')
|
||||||
|
new = page('<p>\n<a href="/b.html">x</a>\n</p>')
|
||||||
|
diffs = compare.compare_pages(old, new)
|
||||||
|
self.assertTrue(diffs)
|
||||||
|
self.assertTrue(any("/a.html" in d or "/b.html" in d for d in diffs))
|
||||||
|
|
||||||
|
def test_changed_text_is_reported(self):
|
||||||
|
self.assertTrue(compare.compare_pages(page("<p>one</p>"), page("<p>two</p>")))
|
||||||
|
|
||||||
|
def test_comments_are_compared(self):
|
||||||
|
self.assertTrue(compare.compare_pages(page("<!-- TODO a --><p>x</p>"), page("<p>x</p>")))
|
||||||
|
|
||||||
|
def test_the_footer_credit_and_generator_meta_are_ignored(self):
|
||||||
|
old = page("<p>x</p>", footer="<p>\nlicense\n</p>")
|
||||||
|
new = page("<p>x</p>", footer='<p>\nlicense\n</p>\n<p>\nPowered by <a href="https://git.kmalloc.org/squid/motorino">motorino</a>.\n</p>',
|
||||||
|
head='<meta name="generator" content="motorino 1.0.0">')
|
||||||
|
self.assertEqual(compare.compare_pages(old, new), [])
|
||||||
|
|
||||||
|
def test_post_byline_and_description_are_ignored_but_the_body_is_not(self):
|
||||||
|
old = page('<h1>T</h1>\n\n<p>\n<time datetime="2026-11-14">2026-11-14</time>\n</p>\n\n<p>\nBody.\n</p>',
|
||||||
|
head='<meta name="description" content="Title">')
|
||||||
|
new = page('<h1>T</h1>\n\n<p>\n<time datetime="2026-11-14">2026-11-14</time> · squid · <a href="/news/category/news.html">News</a>\n</p>\n\n<p>\nBody.\n</p>',
|
||||||
|
head='<meta name="description" content="First sentence.">')
|
||||||
|
self.assertEqual(compare.compare_pages(old, new, post=True), [])
|
||||||
|
changed = new.replace("Body.", "Other body.")
|
||||||
|
self.assertTrue(compare.compare_pages(old, changed, post=True))
|
||||||
|
|
||||||
|
|
||||||
|
class PostPaths(unittest.TestCase):
|
||||||
|
def test_old_post_links_are_moved_to_the_new_paths(self):
|
||||||
|
mapping = {"/news/shop.html": "/news/2026/11/shop.html"}
|
||||||
|
html = '<a href="/news/shop.html">shop</a> and https://kmalloc.org/news/shop.html'
|
||||||
|
self.assertEqual(compare.remap(html, mapping),
|
||||||
|
'<a href="/news/2026/11/shop.html">shop</a> and https://kmalloc.org/news/2026/11/shop.html')
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
101
tools/test_migrate.py
Normal file
101
tools/test_migrate.py
Normal file
|
|
@ -0,0 +1,101 @@
|
||||||
|
"""Tests of the one-off migration helpers (run with: python3 -m unittest discover -s tools)."""
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
import migrate
|
||||||
|
|
||||||
|
|
||||||
|
class HtmlToMarkdown(unittest.TestCase):
|
||||||
|
def test_paragraph_with_inline_markup(self):
|
||||||
|
html = '<p>\nHello <strong>world</strong> and <a href="/x.html">a link</a> and <em>this</em> and <code>code</code>.\n</p>'
|
||||||
|
self.assertEqual(migrate.html_to_md(html), "Hello **world** and [a link](/x.html) and *this* and `code`.\n")
|
||||||
|
|
||||||
|
def test_link_around_code(self):
|
||||||
|
html = '<p>\nSend to <a href="bitcoin:bc1q"><code>bc1q</code></a>\n</p>'
|
||||||
|
self.assertEqual(migrate.html_to_md(html), "Send to [`bc1q`](bitcoin:bc1q)\n")
|
||||||
|
|
||||||
|
def test_heading_with_id(self):
|
||||||
|
self.assertEqual(migrate.html_to_md('<h2 id="account">Request an account</h2>'), "## Request an account {#account}\n")
|
||||||
|
self.assertEqual(migrate.html_to_md("<h1>Services</h1>"), "# Services\n")
|
||||||
|
|
||||||
|
def test_list(self):
|
||||||
|
html = '<ul>\n<li>one</li>\n<li><a href="/a">two</a></li>\n</ul>'
|
||||||
|
self.assertEqual(migrate.html_to_md(html), "- one\n- [two](/a)\n")
|
||||||
|
|
||||||
|
def test_blocks_are_separated_by_a_blank_line(self):
|
||||||
|
html = "<h1>T</h1>\n\n<p>\nBody.\n</p>\n<ul>\n<li>x</li>\n</ul>\n"
|
||||||
|
self.assertEqual(migrate.html_to_md(html), "# T\n\nBody.\n\n- x\n")
|
||||||
|
|
||||||
|
def test_table_stays_raw(self):
|
||||||
|
table = '<table class="data">\n<caption>C</caption>\n<tbody>\n<tr>\n<td data-label="Kept">14 days</td>\n</tr>\n</tbody>\n</table>'
|
||||||
|
self.assertEqual(migrate.html_to_md(table), table + "\n")
|
||||||
|
|
||||||
|
def test_comment_stays_raw(self):
|
||||||
|
self.assertEqual(migrate.html_to_md("<!-- TODO verificare: x -->"), "<!-- TODO verificare: x -->\n")
|
||||||
|
|
||||||
|
def test_raw_when_there_are_attributes_or_other_tags(self):
|
||||||
|
cases = [
|
||||||
|
'<p>\n<img class="logo" src="/logo-auto.svg" alt="kmalloc" width="112">\n</p>',
|
||||||
|
'<h1 class="hero">Technology.<br>Freedom.<br><span class="accent">Dissent.</span></h1>',
|
||||||
|
'<h2 id="v1.2">Version 1.2, <time datetime="2026-11-14">2026-11-14</time></h2>',
|
||||||
|
'<p class="note" role="note">\nA note.\n</p>',
|
||||||
|
'<p>\n<a href="https://x.org/" rel="noreferrer">shop</a>.\n</p>',
|
||||||
|
]
|
||||||
|
for html in cases:
|
||||||
|
with self.subTest(html=html):
|
||||||
|
self.assertEqual(migrate.html_to_md(html), html + "\n")
|
||||||
|
|
||||||
|
def test_unknown_text_between_blocks_fails(self):
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
migrate.html_to_md("<p>\nfine\n</p>\nstray text\n<p>\nmore\n</p>")
|
||||||
|
|
||||||
|
|
||||||
|
class Placeholders(unittest.TestCase):
|
||||||
|
def test_restore_replaces_literal_values(self):
|
||||||
|
values = {"btc": "bc1qabc", "contact_shown": "squid (at) kmalloc.org", "gpg_key": "/squid.asc"}
|
||||||
|
md = "Write to **squid (at) kmalloc.org**. [squid.asc](/squid.asc) and bc1qabc."
|
||||||
|
self.assertEqual(migrate.restore(md, values),
|
||||||
|
"Write to **{{contact_shown}}**. [squid.asc]({{gpg_key}}) and {{btc}}.")
|
||||||
|
|
||||||
|
def test_restore_prefers_the_longest_value(self):
|
||||||
|
values = {"a": "https://x.org/", "b": "https://x.org/search?q=1"}
|
||||||
|
self.assertEqual(migrate.restore("see https://x.org/search?q=1", values), "see {{b}}")
|
||||||
|
|
||||||
|
|
||||||
|
class Content(unittest.TestCase):
|
||||||
|
def test_extract_content_between_header_and_footer(self):
|
||||||
|
page = "<body>\n<header>\n<nav>x</nav>\n</header>\n\n<h1>T</h1>\n\n<p>\nB\n</p>\n\n<footer>\nf\n</footer>\n</body>"
|
||||||
|
self.assertEqual(migrate.extract_content(page), "<h1>T</h1>\n\n<p>\nB\n</p>\n")
|
||||||
|
|
||||||
|
def test_page_file_name(self):
|
||||||
|
cases = [("/", "en", "en", "index.md"), ("/manifesto.html", "en", "en", "manifesto.md"),
|
||||||
|
("/news/", "en", "en", "news/index.md"), ("/it/", "en", "it", "index.md"),
|
||||||
|
("/it/servizi.html", "en", "it", "servizi.md")]
|
||||||
|
for path, default, lang, want in cases:
|
||||||
|
with self.subTest(path=path):
|
||||||
|
self.assertEqual(migrate.page_file(path, default, lang), want)
|
||||||
|
|
||||||
|
def test_page_header(self):
|
||||||
|
header = migrate.page_header("kmalloc.org: News", "news", "Updates.", "bare", note="", switch={"en": "/?lang=en"})
|
||||||
|
self.assertEqual(header, "Title: kmalloc.org: News\nID: news\nDescription: Updates.\nLayout: bare\nSwitch: en=/?lang=en\n-\n")
|
||||||
|
|
||||||
|
def test_post_conversion(self):
|
||||||
|
old = "---\ntitle: A shop\nsummary: About the shop.\n---\n\nThere is a shop.\n"
|
||||||
|
name, text = migrate.convert_post("2026-11-14-shop.md", old)
|
||||||
|
self.assertEqual(name, "shop.md")
|
||||||
|
self.assertEqual(text, "Title: A shop\nAuthor: squid\nDate: 2026-11-14T12:00:00+0100\nCategories: News\n"
|
||||||
|
"Description: About the shop.\n-\n\nThere is a shop.\n")
|
||||||
|
|
||||||
|
def test_post_without_summary_has_no_description(self):
|
||||||
|
_, text = migrate.convert_post("2026-11-12-x.md", "---\ntitle: X\n---\nBody.\n")
|
||||||
|
self.assertNotIn("Description:", text)
|
||||||
|
|
||||||
|
def test_block_markers(self):
|
||||||
|
home = '<p>\nA\n</p>\n\n<h2 id="news">News</h2>\n\n<p>\nIntro. <a href="/news/">All</a>.\n</p>\n\n<ul>\n<li>x</li>\n</ul>\n'
|
||||||
|
self.assertEqual(migrate.mark_latest_news(home), '<p>\nA\n</p>\n\n<p>{{latest_news}}</p>\n')
|
||||||
|
news = '<h1>News</h1>\n\n<p>\nIntro.\n</p>\n\n<ul>\n<li>x</li>\n</ul>\n\n<p>\n<a href="/news/feed.xml">RSS feed</a>\n</p>\n'
|
||||||
|
self.assertEqual(migrate.mark_news_list(news),
|
||||||
|
'<h1>News</h1>\n\n<p>\nIntro.\n</p>\n\n<p>{{news_list}}</p>\n\n<p>\n<a href="/news/feed.xml">RSS feed</a>\n</p>\n')
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Loading…
Reference in a new issue