Pages are Markdown files in content/<lang>/ with a Key: value header, the news carry Title, Author, Date, Categories and Description and are published as /news/YYYY/MM/name.html, with a page for every category. The templates, the language files and site.json shrink to the chrome and the shared strings, and the footer credits motorino. The output matches the one of build.py page by page, checked with tools/compare.py and the Nu HTML Checker.
127 lines
4.5 KiB
Python
127 lines
4.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Compare the site built by build.py (old) with the one built by motorino (new), page by page.
|
|
|
|
Usage: compare.py OLD_DIR NEW_DIR
|
|
|
|
The text, the tags, their attributes, the links and the comments of every page must be the same. The
|
|
differences that the change wants are ignored: the "Powered by motorino" footer line, the generator meta, the
|
|
author and categories after the date of a news item, and its description meta. Old news addresses are moved
|
|
to the new dated ones before the comparison. Exit status 1 when anything else differs."""
|
|
import difflib
|
|
import html as htmllib
|
|
import os
|
|
import re
|
|
import sys
|
|
from html.parser import HTMLParser
|
|
|
|
|
|
class Tokens(HTMLParser):
|
|
"""The stream of (kind, value) tokens of a page: whitespace is collapsed and attribute order does not matter."""
|
|
|
|
def __init__(self):
|
|
super().__init__(convert_charrefs=True)
|
|
self.items = []
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
self.items.append(("start", tag + "".join(f' {k}="{v}"' for k, v in sorted(attrs))))
|
|
|
|
def handle_startendtag(self, tag, attrs):
|
|
self.handle_starttag(tag, attrs)
|
|
|
|
def handle_endtag(self, tag):
|
|
self.items.append(("end", tag))
|
|
|
|
def handle_data(self, data):
|
|
text = " ".join(data.split())
|
|
if text:
|
|
if self.items and self.items[-1][0] == "text":
|
|
self.items[-1] = ("text", self.items[-1][1] + " " + text)
|
|
else:
|
|
self.items.append(("text", text))
|
|
|
|
def handle_comment(self, data):
|
|
self.items.append(("comment", " ".join(data.split())))
|
|
|
|
|
|
CREDIT = re.compile(r"<p>\s*Powered by .*?</p>\s*", re.S)
|
|
GENERATOR = re.compile(r'<meta name="generator"[^>]*>\s*')
|
|
BYLINE = re.compile(r'(<time datetime="[^"]+">[^<]*</time>)\s*·.*?(\s*</p>)', re.S)
|
|
DESCRIPTION = re.compile(r'<meta (?:name="description"|property="og:description")[^>]*>\s*')
|
|
|
|
|
|
def tokens(html, post=False):
|
|
html = GENERATOR.sub("", CREDIT.sub("", html))
|
|
if post:
|
|
html = DESCRIPTION.sub("", BYLINE.sub(r"\1\2", html, count=1))
|
|
parser = Tokens()
|
|
parser.feed(html)
|
|
return [f"{kind}: {value}" for kind, value in parser.items]
|
|
|
|
|
|
def compare_pages(old, new, post=False):
|
|
"""The differences between two pages, as lines of a diff; empty when they match."""
|
|
a, b = tokens(old, post), tokens(new, post)
|
|
if a == b:
|
|
return []
|
|
return [line for line in difflib.unified_diff(a, b, "old", "new", lineterm="", n=2)]
|
|
|
|
|
|
def remap(html, mapping):
|
|
"""Replace old addresses by new ones."""
|
|
for old, new in sorted(mapping.items(), key=lambda kv: -len(kv[0])):
|
|
html = html.replace(old, new).replace(old.replace("/", "%2F"), new.replace("/", "%2F"))
|
|
return html
|
|
|
|
|
|
def html_files(root):
|
|
for folder, _, files in os.walk(root):
|
|
for name in files:
|
|
if name.endswith(".html"):
|
|
yield "/" + os.path.relpath(os.path.join(folder, name), root).replace(os.sep, "/")
|
|
|
|
|
|
def read(root, rel):
|
|
with open(os.path.join(root, rel.lstrip("/")), encoding="utf-8") as f:
|
|
return f.read()
|
|
|
|
|
|
def post_mapping(old_dir, new_dir):
|
|
"""Old news addresses (/news/slug.html) to the new dated ones, found by name in the new output."""
|
|
new_posts = {}
|
|
for rel in html_files(new_dir):
|
|
m = re.fullmatch(r"/news/(\d{4})/(\d{2})/([a-z0-9-]+)\.html", rel)
|
|
if m:
|
|
new_posts[m.group(3)] = rel
|
|
mapping = {}
|
|
for rel in html_files(old_dir):
|
|
m = re.fullmatch(r"/news/([a-z0-9-]+)\.html", rel)
|
|
if m and m.group(1) in new_posts:
|
|
mapping[rel] = new_posts[m.group(1)]
|
|
return mapping
|
|
|
|
|
|
def main(old_dir, new_dir):
|
|
mapping = post_mapping(old_dir, new_dir)
|
|
bad = 0
|
|
for rel in sorted(html_files(old_dir)):
|
|
new_rel = mapping.get(rel, rel)
|
|
if not os.path.exists(os.path.join(new_dir, new_rel.lstrip("/"))):
|
|
print(f"MISSING in new output: {new_rel}")
|
|
bad += 1
|
|
continue
|
|
diffs = compare_pages(remap(read(old_dir, rel), mapping), read(new_dir, new_rel), post=rel in mapping)
|
|
if diffs:
|
|
bad += 1
|
|
print(f"DIFFERENT: {rel}" + (f" -> {new_rel}" if new_rel != rel else ""))
|
|
print("\n".join(diffs[:60]))
|
|
extra = sorted(set(html_files(new_dir)) - {mapping.get(r, r) for r in html_files(old_dir)})
|
|
for rel in extra:
|
|
print(f"new page: {rel}")
|
|
print(f"compared {len(list(html_files(old_dir)))} pages, {bad} differ")
|
|
return 1 if bad else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if len(sys.argv) != 3:
|
|
sys.exit(__doc__)
|
|
sys.exit(main(sys.argv[1], sys.argv[2]))
|