diff --git a/tools/compare.py b/tools/compare.py new file mode 100644 index 0000000..0f94fea --- /dev/null +++ b/tools/compare.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""Compare the site built by build.py (old) with the one built by motorino (new), page by page. + +Usage: compare.py OLD_DIR NEW_DIR + +The text, the tags, their attributes, the links and the comments of every page must be the same. The +differences that the change wants are ignored: the "Powered by motorino" footer line, the generator meta, the +author and categories after the date of a news item, and its description meta. Old news addresses are moved +to the new dated ones before the comparison. Exit status 1 when anything else differs.""" +import difflib +import html as htmllib +import os +import re +import sys +from html.parser import HTMLParser + + +class Tokens(HTMLParser): + """The stream of (kind, value) tokens of a page: whitespace is collapsed and attribute order does not matter.""" + + def __init__(self): + super().__init__(convert_charrefs=True) + self.items = [] + + def handle_starttag(self, tag, attrs): + self.items.append(("start", tag + "".join(f' {k}="{v}"' for k, v in sorted(attrs)))) + + def handle_startendtag(self, tag, attrs): + self.handle_starttag(tag, attrs) + + def handle_endtag(self, tag): + self.items.append(("end", tag)) + + def handle_data(self, data): + text = " ".join(data.split()) + if text: + if self.items and self.items[-1][0] == "text": + self.items[-1] = ("text", self.items[-1][1] + " " + text) + else: + self.items.append(("text", text)) + + def handle_comment(self, data): + self.items.append(("comment", " ".join(data.split()))) + + +CREDIT = re.compile(r"
\s*Powered by .*?
\s*", re.S) +GENERATOR = re.compile(r']*>\s*') +BYLINE = re.compile(r'()\s*·.*?(\s*)', re.S) +DESCRIPTION = re.compile(r']*>\s*') + + +def tokens(html, post=False): + html = GENERATOR.sub("", CREDIT.sub("", html)) + if post: + html = DESCRIPTION.sub("", BYLINE.sub(r"\1\2", html, count=1)) + parser = Tokens() + parser.feed(html) + return [f"{kind}: {value}" for kind, value in parser.items] + + +def compare_pages(old, new, post=False): + """The differences between two pages, as lines of a diff; empty when they match.""" + a, b = tokens(old, post), tokens(new, post) + if a == b: + return [] + return [line for line in difflib.unified_diff(a, b, "old", "new", lineterm="", n=2)] + + +def remap(html, mapping): + """Replace old addresses by new ones.""" + for old, new in sorted(mapping.items(), key=lambda kv: -len(kv[0])): + html = html.replace(old, new) + return html + + +def html_files(root): + for folder, _, files in os.walk(root): + for name in files: + if name.endswith(".html"): + yield "/" + os.path.relpath(os.path.join(folder, name), root).replace(os.sep, "/") + + +def read(root, rel): + with open(os.path.join(root, rel.lstrip("/")), encoding="utf-8") as f: + return f.read() + + +def post_mapping(old_dir, new_dir): + """Old news addresses (/news/slug.html) to the new dated ones, found by name in the new output.""" + new_posts = {} + for rel in html_files(new_dir): + m = re.fullmatch(r"/news/(\d{4})/(\d{2})/([a-z0-9-]+)\.html", rel) + if m: + new_posts[m.group(3)] = rel + mapping = {} + for rel in html_files(old_dir): + m = re.fullmatch(r"/news/([a-z0-9-]+)\.html", rel) + if m and m.group(1) in new_posts: + mapping[rel] = new_posts[m.group(1)] + return mapping + + +def main(old_dir, new_dir): + mapping = post_mapping(old_dir, new_dir) + bad = 0 + for rel in sorted(html_files(old_dir)): + new_rel = mapping.get(rel, rel) + if not os.path.exists(os.path.join(new_dir, new_rel.lstrip("/"))): + print(f"MISSING in new output: {new_rel}") + bad += 1 + continue + diffs = compare_pages(remap(read(old_dir, rel), mapping), read(new_dir, new_rel), post=rel in mapping) + if diffs: + bad += 1 + print(f"DIFFERENT: {rel}" + (f" -> {new_rel}" if new_rel != rel else "")) + print("\n".join(diffs[:60])) + extra = sorted(set(html_files(new_dir)) - {mapping.get(r, r) for r in html_files(old_dir)}) + for rel in extra: + print(f"new page: {rel}") + print(f"compared {len(list(html_files(old_dir)))} pages, {bad} differ") + return 1 if bad else 0 + + +if __name__ == "__main__": + if len(sys.argv) != 3: + sys.exit(__doc__) + sys.exit(main(sys.argv[1], sys.argv[2])) diff --git a/tools/migrate.py b/tools/migrate.py new file mode 100644 index 0000000..aad6db7 --- /dev/null +++ b/tools/migrate.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +"""One-off migration of the kmalloc.org site from build.py to motorino. + +Usage: migrate.py [--site DIR] [--old-out DIR] + +It builds the current site with build.py (the reference, saved in --old-out), then writes the new layout in +place: pages as Markdown in content/(.*?)", r"`\1`", s, flags=re.S)
+ if "<" in s:
+ return None
+ return "\n".join(line.strip() for line in s.splitlines())
+
+
+def block_to_md(match):
+ """One top-level block as Markdown, or as the raw HTML it came from."""
+ raw = match.group(0).strip()
+ comment, tag, attrs, inner = match.groups()
+ if comment:
+ return raw
+ if tag == "p" and not attrs.strip():
+ text = inline_to_md(inner)
+ return text if text is not None else raw
+ if re.fullmatch(r"h[1-6]", tag):
+ text = inline_to_md(inner)
+ heading_id = ID_ONLY.match(attrs)
+ if text is not None and (not attrs.strip() or heading_id):
+ line = "#" * int(tag[1]) + " " + text
+ return line + (" {#" + heading_id.group(1) + "}" if heading_id else "")
+ return raw
+ if tag == "ul" and not attrs.strip():
+ items = re.findall(r"{{latest_news}}
\n", html, count=1, flags=re.S) + + +def mark_news_list(html): + """The list of the news page becomes the {{news_list}} placeholder.""" + return re.sub(r"{{news_list}}
", html, count=1, flags=re.S) + + +# ---------------------------------------------------------------- pages and news + +def extract_content(page_html): + """The part of a built page between the header and the footer.""" + m = re.search(r"\n\n(.*)\n