kmalloc.org/tools/compare.py

127 lines
4.5 KiB
Python

#!/usr/bin/env python3
"""Compare the site built by build.py (old) with the one built by motorino (new), page by page.
Usage: compare.py OLD_DIR NEW_DIR
The text, the tags, their attributes, the links and the comments of every page must be the same. The
differences that the change wants are ignored: the "Powered by motorino" footer line, the generator meta, the
author and categories after the date of a news item, and its description meta. Old news addresses are moved
to the new dated ones before the comparison. Exit status 1 when anything else differs."""
import difflib
import html as htmllib
import os
import re
import sys
from html.parser import HTMLParser
class Tokens(HTMLParser):
"""The stream of (kind, value) tokens of a page: whitespace is collapsed and attribute order does not matter."""
def __init__(self):
super().__init__(convert_charrefs=True)
self.items = []
def handle_starttag(self, tag, attrs):
self.items.append(("start", tag + "".join(f' {k}="{v}"' for k, v in sorted(attrs))))
def handle_startendtag(self, tag, attrs):
self.handle_starttag(tag, attrs)
def handle_endtag(self, tag):
self.items.append(("end", tag))
def handle_data(self, data):
text = " ".join(data.split())
if text:
if self.items and self.items[-1][0] == "text":
self.items[-1] = ("text", self.items[-1][1] + " " + text)
else:
self.items.append(("text", text))
def handle_comment(self, data):
self.items.append(("comment", " ".join(data.split())))
CREDIT = re.compile(r"<p>\s*Powered by .*?</p>\s*", re.S)
GENERATOR = re.compile(r'<meta name="generator"[^>]*>\s*')
BYLINE = re.compile(r'(<time datetime="[^"]+">[^<]*</time>)\s*&middot;.*?(\s*</p>)', re.S)
DESCRIPTION = re.compile(r'<meta (?:name="description"|property="og:description")[^>]*>\s*')
def tokens(html, post=False):
html = GENERATOR.sub("", CREDIT.sub("", html))
if post:
html = DESCRIPTION.sub("", BYLINE.sub(r"\1\2", html, count=1))
parser = Tokens()
parser.feed(html)
return [f"{kind}: {value}" for kind, value in parser.items]
def compare_pages(old, new, post=False):
"""The differences between two pages, as lines of a diff; empty when they match."""
a, b = tokens(old, post), tokens(new, post)
if a == b:
return []
return [line for line in difflib.unified_diff(a, b, "old", "new", lineterm="", n=2)]
def remap(html, mapping):
"""Replace old addresses by new ones."""
for old, new in sorted(mapping.items(), key=lambda kv: -len(kv[0])):
html = html.replace(old, new)
return html
def html_files(root):
for folder, _, files in os.walk(root):
for name in files:
if name.endswith(".html"):
yield "/" + os.path.relpath(os.path.join(folder, name), root).replace(os.sep, "/")
def read(root, rel):
with open(os.path.join(root, rel.lstrip("/")), encoding="utf-8") as f:
return f.read()
def post_mapping(old_dir, new_dir):
"""Old news addresses (/news/slug.html) to the new dated ones, found by name in the new output."""
new_posts = {}
for rel in html_files(new_dir):
m = re.fullmatch(r"/news/(\d{4})/(\d{2})/([a-z0-9-]+)\.html", rel)
if m:
new_posts[m.group(3)] = rel
mapping = {}
for rel in html_files(old_dir):
m = re.fullmatch(r"/news/([a-z0-9-]+)\.html", rel)
if m and m.group(1) in new_posts:
mapping[rel] = new_posts[m.group(1)]
return mapping
def main(old_dir, new_dir):
mapping = post_mapping(old_dir, new_dir)
bad = 0
for rel in sorted(html_files(old_dir)):
new_rel = mapping.get(rel, rel)
if not os.path.exists(os.path.join(new_dir, new_rel.lstrip("/"))):
print(f"MISSING in new output: {new_rel}")
bad += 1
continue
diffs = compare_pages(remap(read(old_dir, rel), mapping), read(new_dir, new_rel), post=rel in mapping)
if diffs:
bad += 1
print(f"DIFFERENT: {rel}" + (f" -> {new_rel}" if new_rel != rel else ""))
print("\n".join(diffs[:60]))
extra = sorted(set(html_files(new_dir)) - {mapping.get(r, r) for r in html_files(old_dir)})
for rel in extra:
print(f"new page: {rel}")
print(f"compared {len(list(html_files(old_dir)))} pages, {bad} differ")
return 1 if bad else 0
if __name__ == "__main__":
if len(sys.argv) != 3:
sys.exit(__doc__)
sys.exit(main(sys.argv[1], sys.argv[2]))