#!/usr/bin/env python3 """Compare the site built by build.py (old) with the one built by motorino (new), page by page. Usage: compare.py OLD_DIR NEW_DIR The text, the tags, their attributes, the links and the comments of every page must be the same. The differences that the change wants are ignored: the "Powered by motorino" footer line, the generator meta, the author and categories after the date of a news item, and its description meta. Old news addresses are moved to the new dated ones before the comparison. Exit status 1 when anything else differs.""" import difflib import html as htmllib import os import re import sys from html.parser import HTMLParser class Tokens(HTMLParser): """The stream of (kind, value) tokens of a page: whitespace is collapsed and attribute order does not matter.""" def __init__(self): super().__init__(convert_charrefs=True) self.items = [] def handle_starttag(self, tag, attrs): self.items.append(("start", tag + "".join(f' {k}="{v}"' for k, v in sorted(attrs)))) def handle_startendtag(self, tag, attrs): self.handle_starttag(tag, attrs) def handle_endtag(self, tag): self.items.append(("end", tag)) def handle_data(self, data): text = " ".join(data.split()) if text: if self.items and self.items[-1][0] == "text": self.items[-1] = ("text", self.items[-1][1] + " " + text) else: self.items.append(("text", text)) def handle_comment(self, data): self.items.append(("comment", " ".join(data.split()))) CREDIT = re.compile(r"
\s*Powered by .*?
\s*", re.S) GENERATOR = re.compile(r']*>\s*') BYLINE = re.compile(r'()\s*·.*?(\s*)', re.S) DESCRIPTION = re.compile(r']*>\s*') def tokens(html, post=False): html = GENERATOR.sub("", CREDIT.sub("", html)) if post: html = DESCRIPTION.sub("", BYLINE.sub(r"\1\2", html, count=1)) parser = Tokens() parser.feed(html) return [f"{kind}: {value}" for kind, value in parser.items] def compare_pages(old, new, post=False): """The differences between two pages, as lines of a diff; empty when they match.""" a, b = tokens(old, post), tokens(new, post) if a == b: return [] return [line for line in difflib.unified_diff(a, b, "old", "new", lineterm="", n=2)] def remap(html, mapping): """Replace old addresses by new ones.""" for old, new in sorted(mapping.items(), key=lambda kv: -len(kv[0])): html = html.replace(old, new).replace(old.replace("/", "%2F"), new.replace("/", "%2F")) return html def html_files(root): for folder, _, files in os.walk(root): for name in files: if name.endswith(".html"): yield "/" + os.path.relpath(os.path.join(folder, name), root).replace(os.sep, "/") def read(root, rel): with open(os.path.join(root, rel.lstrip("/")), encoding="utf-8") as f: return f.read() def post_mapping(old_dir, new_dir): """Old news addresses (/news/slug.html) to the new dated ones, found by name in the new output.""" new_posts = {} for rel in html_files(new_dir): m = re.fullmatch(r"/news/(\d{4})/(\d{2})/([a-z0-9-]+)\.html", rel) if m: new_posts[m.group(3)] = rel mapping = {} for rel in html_files(old_dir): m = re.fullmatch(r"/news/([a-z0-9-]+)\.html", rel) if m and m.group(1) in new_posts: mapping[rel] = new_posts[m.group(1)] return mapping def main(old_dir, new_dir): mapping = post_mapping(old_dir, new_dir) bad = 0 for rel in sorted(html_files(old_dir)): new_rel = mapping.get(rel, rel) if not os.path.exists(os.path.join(new_dir, new_rel.lstrip("/"))): print(f"MISSING in new output: {new_rel}") bad += 1 continue diffs = compare_pages(remap(read(old_dir, rel), mapping), read(new_dir, new_rel), post=rel in mapping) if diffs: bad += 1 print(f"DIFFERENT: {rel}" + (f" -> {new_rel}" if new_rel != rel else "")) print("\n".join(diffs[:60])) extra = sorted(set(html_files(new_dir)) - {mapping.get(r, r) for r in html_files(old_dir)}) for rel in extra: print(f"new page: {rel}") print(f"compared {len(list(html_files(old_dir)))} pages, {bad} differ") return 1 if bad else 0 if __name__ == "__main__": if len(sys.argv) != 3: sys.exit(__doc__) sys.exit(main(sys.argv[1], sys.argv[2]))