kmalloc.org/tools/compare.py
squid d2f9bba46b feat: move the pages to Markdown and the news to the dated layout
Pages are Markdown files in content/<lang>/ with a Key: value header, the
news carry Title, Author, Date, Categories and Description and are published
as /news/YYYY/MM/name.html, with a page for every category. The templates, the
language files and site.json shrink to the chrome and the shared strings, and
the footer credits motorino. The output matches the one of build.py page by
page, checked with tools/compare.py and the Nu HTML Checker.
2026-11-15 09:58:10 +01:00

127 lines
4.5 KiB
Python

#!/usr/bin/env python3
"""Compare the site built by build.py (old) with the one built by motorino (new), page by page.
Usage: compare.py OLD_DIR NEW_DIR
The text, the tags, their attributes, the links and the comments of every page must be the same. The
differences that the change wants are ignored: the "Powered by motorino" footer line, the generator meta, the
author and categories after the date of a news item, and its description meta. Old news addresses are moved
to the new dated ones before the comparison. Exit status 1 when anything else differs."""
import difflib
import html as htmllib
import os
import re
import sys
from html.parser import HTMLParser
class Tokens(HTMLParser):
"""The stream of (kind, value) tokens of a page: whitespace is collapsed and attribute order does not matter."""
def __init__(self):
super().__init__(convert_charrefs=True)
self.items = []
def handle_starttag(self, tag, attrs):
self.items.append(("start", tag + "".join(f' {k}="{v}"' for k, v in sorted(attrs))))
def handle_startendtag(self, tag, attrs):
self.handle_starttag(tag, attrs)
def handle_endtag(self, tag):
self.items.append(("end", tag))
def handle_data(self, data):
text = " ".join(data.split())
if text:
if self.items and self.items[-1][0] == "text":
self.items[-1] = ("text", self.items[-1][1] + " " + text)
else:
self.items.append(("text", text))
def handle_comment(self, data):
self.items.append(("comment", " ".join(data.split())))
CREDIT = re.compile(r"<p>\s*Powered by .*?</p>\s*", re.S)
GENERATOR = re.compile(r'<meta name="generator"[^>]*>\s*')
BYLINE = re.compile(r'(<time datetime="[^"]+">[^<]*</time>)\s*&middot;.*?(\s*</p>)', re.S)
DESCRIPTION = re.compile(r'<meta (?:name="description"|property="og:description")[^>]*>\s*')
def tokens(html, post=False):
html = GENERATOR.sub("", CREDIT.sub("", html))
if post:
html = DESCRIPTION.sub("", BYLINE.sub(r"\1\2", html, count=1))
parser = Tokens()
parser.feed(html)
return [f"{kind}: {value}" for kind, value in parser.items]
def compare_pages(old, new, post=False):
"""The differences between two pages, as lines of a diff; empty when they match."""
a, b = tokens(old, post), tokens(new, post)
if a == b:
return []
return [line for line in difflib.unified_diff(a, b, "old", "new", lineterm="", n=2)]
def remap(html, mapping):
"""Replace old addresses by new ones."""
for old, new in sorted(mapping.items(), key=lambda kv: -len(kv[0])):
html = html.replace(old, new).replace(old.replace("/", "%2F"), new.replace("/", "%2F"))
return html
def html_files(root):
for folder, _, files in os.walk(root):
for name in files:
if name.endswith(".html"):
yield "/" + os.path.relpath(os.path.join(folder, name), root).replace(os.sep, "/")
def read(root, rel):
with open(os.path.join(root, rel.lstrip("/")), encoding="utf-8") as f:
return f.read()
def post_mapping(old_dir, new_dir):
"""Old news addresses (/news/slug.html) to the new dated ones, found by name in the new output."""
new_posts = {}
for rel in html_files(new_dir):
m = re.fullmatch(r"/news/(\d{4})/(\d{2})/([a-z0-9-]+)\.html", rel)
if m:
new_posts[m.group(3)] = rel
mapping = {}
for rel in html_files(old_dir):
m = re.fullmatch(r"/news/([a-z0-9-]+)\.html", rel)
if m and m.group(1) in new_posts:
mapping[rel] = new_posts[m.group(1)]
return mapping
def main(old_dir, new_dir):
mapping = post_mapping(old_dir, new_dir)
bad = 0
for rel in sorted(html_files(old_dir)):
new_rel = mapping.get(rel, rel)
if not os.path.exists(os.path.join(new_dir, new_rel.lstrip("/"))):
print(f"MISSING in new output: {new_rel}")
bad += 1
continue
diffs = compare_pages(remap(read(old_dir, rel), mapping), read(new_dir, new_rel), post=rel in mapping)
if diffs:
bad += 1
print(f"DIFFERENT: {rel}" + (f" -> {new_rel}" if new_rel != rel else ""))
print("\n".join(diffs[:60]))
extra = sorted(set(html_files(new_dir)) - {mapping.get(r, r) for r in html_files(old_dir)})
for rel in extra:
print(f"new page: {rel}")
print(f"compared {len(list(html_files(old_dir)))} pages, {bad} differ")
return 1 if bad else 0
if __name__ == "__main__":
if len(sys.argv) != 3:
sys.exit(__doc__)
sys.exit(main(sys.argv[1], sys.argv[2]))