127 lines
4.5 KiB
Python
127 lines
4.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Compare the site built by build.py (old) with the one built by motorino (new), page by page.
|
|
|
|
Usage: compare.py OLD_DIR NEW_DIR
|
|
|
|
The text, the tags, their attributes, the links and the comments of every page must be the same. The
|
|
differences that the change wants are ignored: the "Powered by motorino" footer line, the generator meta, the
|
|
author and categories after the date of a news item, and its description meta. Old news addresses are moved
|
|
to the new dated ones before the comparison. Exit status 1 when anything else differs."""
|
|
import difflib
|
|
import html as htmllib
|
|
import os
|
|
import re
|
|
import sys
|
|
from html.parser import HTMLParser
|
|
|
|
|
|
class Tokens(HTMLParser):
|
|
"""The stream of (kind, value) tokens of a page: whitespace is collapsed and attribute order does not matter."""
|
|
|
|
def __init__(self):
|
|
super().__init__(convert_charrefs=True)
|
|
self.items = []
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
self.items.append(("start", tag + "".join(f' {k}="{v}"' for k, v in sorted(attrs))))
|
|
|
|
def handle_startendtag(self, tag, attrs):
|
|
self.handle_starttag(tag, attrs)
|
|
|
|
def handle_endtag(self, tag):
|
|
self.items.append(("end", tag))
|
|
|
|
def handle_data(self, data):
|
|
text = " ".join(data.split())
|
|
if text:
|
|
if self.items and self.items[-1][0] == "text":
|
|
self.items[-1] = ("text", self.items[-1][1] + " " + text)
|
|
else:
|
|
self.items.append(("text", text))
|
|
|
|
def handle_comment(self, data):
|
|
self.items.append(("comment", " ".join(data.split())))
|
|
|
|
|
|
CREDIT = re.compile(r"<p>\s*Powered by .*?</p>\s*", re.S)
|
|
GENERATOR = re.compile(r'<meta name="generator"[^>]*>\s*')
|
|
BYLINE = re.compile(r'(<time datetime="[^"]+">[^<]*</time>)\s*·.*?(\s*</p>)', re.S)
|
|
DESCRIPTION = re.compile(r'<meta (?:name="description"|property="og:description")[^>]*>\s*')
|
|
|
|
|
|
def tokens(html, post=False):
|
|
html = GENERATOR.sub("", CREDIT.sub("", html))
|
|
if post:
|
|
html = DESCRIPTION.sub("", BYLINE.sub(r"\1\2", html, count=1))
|
|
parser = Tokens()
|
|
parser.feed(html)
|
|
return [f"{kind}: {value}" for kind, value in parser.items]
|
|
|
|
|
|
def compare_pages(old, new, post=False):
|
|
"""The differences between two pages, as lines of a diff; empty when they match."""
|
|
a, b = tokens(old, post), tokens(new, post)
|
|
if a == b:
|
|
return []
|
|
return [line for line in difflib.unified_diff(a, b, "old", "new", lineterm="", n=2)]
|
|
|
|
|
|
def remap(html, mapping):
|
|
"""Replace old addresses by new ones."""
|
|
for old, new in sorted(mapping.items(), key=lambda kv: -len(kv[0])):
|
|
html = html.replace(old, new)
|
|
return html
|
|
|
|
|
|
def html_files(root):
|
|
for folder, _, files in os.walk(root):
|
|
for name in files:
|
|
if name.endswith(".html"):
|
|
yield "/" + os.path.relpath(os.path.join(folder, name), root).replace(os.sep, "/")
|
|
|
|
|
|
def read(root, rel):
|
|
with open(os.path.join(root, rel.lstrip("/")), encoding="utf-8") as f:
|
|
return f.read()
|
|
|
|
|
|
def post_mapping(old_dir, new_dir):
|
|
"""Old news addresses (/news/slug.html) to the new dated ones, found by name in the new output."""
|
|
new_posts = {}
|
|
for rel in html_files(new_dir):
|
|
m = re.fullmatch(r"/news/(\d{4})/(\d{2})/([a-z0-9-]+)\.html", rel)
|
|
if m:
|
|
new_posts[m.group(3)] = rel
|
|
mapping = {}
|
|
for rel in html_files(old_dir):
|
|
m = re.fullmatch(r"/news/([a-z0-9-]+)\.html", rel)
|
|
if m and m.group(1) in new_posts:
|
|
mapping[rel] = new_posts[m.group(1)]
|
|
return mapping
|
|
|
|
|
|
def main(old_dir, new_dir):
|
|
mapping = post_mapping(old_dir, new_dir)
|
|
bad = 0
|
|
for rel in sorted(html_files(old_dir)):
|
|
new_rel = mapping.get(rel, rel)
|
|
if not os.path.exists(os.path.join(new_dir, new_rel.lstrip("/"))):
|
|
print(f"MISSING in new output: {new_rel}")
|
|
bad += 1
|
|
continue
|
|
diffs = compare_pages(remap(read(old_dir, rel), mapping), read(new_dir, new_rel), post=rel in mapping)
|
|
if diffs:
|
|
bad += 1
|
|
print(f"DIFFERENT: {rel}" + (f" -> {new_rel}" if new_rel != rel else ""))
|
|
print("\n".join(diffs[:60]))
|
|
extra = sorted(set(html_files(new_dir)) - {mapping.get(r, r) for r in html_files(old_dir)})
|
|
for rel in extra:
|
|
print(f"new page: {rel}")
|
|
print(f"compared {len(list(html_files(old_dir)))} pages, {bad} differ")
|
|
return 1 if bad else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if len(sys.argv) != 3:
|
|
sys.exit(__doc__)
|
|
sys.exit(main(sys.argv[1], sys.argv[2]))
|