feat: render Markdown with stable heading ids and the em dash rule
This commit is contained in:
parent
27dac18c8f
commit
2c7bca35be
6 changed files with 366 additions and 0 deletions
56
internal/mdx/emdash.go
Normal file
56
internal/mdx/emdash.go
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
package mdx
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// emDash is the character the published site never shows.
|
||||
const emDash = "—"
|
||||
|
||||
// Rewrites of the manifesto headings, compiled once. The manifesto sources use
|
||||
// em dashes: "NN — Title" becomes "NN. Title", "Word — Title" becomes
|
||||
// "Word: Title" and an attribution line loses its dash.
|
||||
var (
|
||||
numberedHeading = regexp.MustCompile(`(?m)^(#+ )(\d\d) ` + emDash + ` `)
|
||||
wordHeading = regexp.MustCompile(`(?m)^(#+ [^\d\n][^` + emDash + `\n]*?) ` + emDash + ` `)
|
||||
attribution = regexp.MustCompile(`(?m)^> ` + emDash + ` `)
|
||||
)
|
||||
|
||||
// parentheticals are the sentences in which a pair of dashes becomes commas.
|
||||
var parentheticals = []struct{ from, to string }{
|
||||
{" " + emDash + " where technically feasible " + emDash + " ", ", where technically feasible, "},
|
||||
{" " + emDash + " dove tecnicamente possibile " + emDash + " ", ", dove tecnicamente possibile, "},
|
||||
{" " + emDash + " and continues", ", and continues"},
|
||||
{" " + emDash + " e continua", ", e continua"},
|
||||
{"kmalloc.org " + emDash + " ", "kmalloc.org: "},
|
||||
}
|
||||
|
||||
// NoEmDash rewrites the em dashes of a manifesto source with the rules above
|
||||
// and returns an error if any em dash is left, so a new one is noticed.
|
||||
func NoEmDash(md string) (string, error) {
|
||||
md = numberedHeading.ReplaceAllString(md, "${1}${2}. ")
|
||||
md = wordHeading.ReplaceAllString(md, "${1}: ")
|
||||
md = attribution.ReplaceAllString(md, "> ")
|
||||
for _, p := range parentheticals {
|
||||
md = strings.ReplaceAll(md, p.from, p.to)
|
||||
}
|
||||
if err := RejectEmDash("manifesto source", md); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return md, nil
|
||||
}
|
||||
|
||||
// RejectEmDash returns an error naming the file and the line of the first em
|
||||
// dash in text, or nil when there is none.
|
||||
func RejectEmDash(name, text string) error {
|
||||
for i, line := range strings.Split(text, "\n") {
|
||||
if strings.Contains(line, emDash) {
|
||||
return fmt.Errorf("%s:%d: no em dashes, please", name, i+1)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
57
internal/mdx/emdash_test.go
Normal file
57
internal/mdx/emdash_test.go
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
package mdx
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestNoEmDash checks every rewrite of the manifesto sources.
|
||||
func TestNoEmDash(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{"numbered heading", "## 01 — Antifascist. No ambiguity.", "## 01. Antifascist. No ambiguity."},
|
||||
{"word heading", "## Preamble — We don't ask permission to think", "## Preamble: We don't ask permission to think"},
|
||||
{"title heading", "# kmalloc.org — Manifesto", "# kmalloc.org: Manifesto"},
|
||||
{"attribution", "> — Funkadelic, Free Your Mind", "> Funkadelic, Free Your Mind"},
|
||||
{"parenthetical en", "modify, and — where technically feasible — install", "modify, and, where technically feasible, install"},
|
||||
{"parenthetical it", "e — dove tecnicamente possibile — installare", "e, dove tecnicamente possibile, installare"},
|
||||
{"and continues", "it — and continues", "it, and continues"},
|
||||
{"e continua", "esso — e continua", "esso, e continua"},
|
||||
{"no dash", "plain text", "plain text"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := NoEmDash(tt.in)
|
||||
if err != nil {
|
||||
t.Fatalf("NoEmDash() error = %v", err)
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Errorf("NoEmDash() = %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestNoEmDashLeftover checks that a dash with no rewrite rule is an error.
|
||||
func TestNoEmDashLeftover(t *testing.T) {
|
||||
_, err := NoEmDash("some words — and then more")
|
||||
if err == nil {
|
||||
t.Fatal("NoEmDash() error = nil, want an error for a leftover em dash")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRejectEmDash checks that the error names the file and the line.
|
||||
func TestRejectEmDash(t *testing.T) {
|
||||
if err := RejectEmDash("content/en/x.md", "clean\ntext\n"); err != nil {
|
||||
t.Errorf("RejectEmDash(clean) = %v", err)
|
||||
}
|
||||
err := RejectEmDash("content/en/x.md", "clean\nbad — line\n")
|
||||
if err == nil || !strings.Contains(err.Error(), "content/en/x.md") || !strings.Contains(err.Error(), "2") {
|
||||
t.Errorf("RejectEmDash() = %v, want an error naming the file and line 2", err)
|
||||
}
|
||||
}
|
||||
85
internal/mdx/mdx.go
Normal file
85
internal/mdx/mdx.go
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// Package mdx renders the Markdown of pages and news with goldmark.
|
||||
package mdx
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/yuin/goldmark"
|
||||
"github.com/yuin/goldmark/ast"
|
||||
"github.com/yuin/goldmark/renderer/html"
|
||||
"github.com/yuin/goldmark/text"
|
||||
)
|
||||
|
||||
// Heading is a heading of a rendered document.
|
||||
type Heading struct {
|
||||
// Level is 1 for "#", 2 for "##", and so on.
|
||||
Level int
|
||||
// ID is the anchor identifier given to the heading.
|
||||
ID string
|
||||
// Text is the heading without markup.
|
||||
Text string
|
||||
}
|
||||
|
||||
// engine converts Markdown to HTML. Raw HTML is passed through because the
|
||||
// pages use it for tables with data-label cells, hidden notes and the hero.
|
||||
var engine = goldmark.New(goldmark.WithRendererOptions(html.WithUnsafe()))
|
||||
|
||||
// Render converts md to HTML and returns the headings it found. Every heading
|
||||
// gets an id attribute made by Slug, with -2, -3 and so on for duplicates.
|
||||
func Render(md string) (string, []Heading, error) {
|
||||
src := []byte(md)
|
||||
doc := engine.Parser().Parse(text.NewReader(src))
|
||||
|
||||
var headings []Heading
|
||||
used := make(map[string]int)
|
||||
err := ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
|
||||
h, ok := n.(*ast.Heading)
|
||||
if !ok || !entering {
|
||||
return ast.WalkContinue, nil
|
||||
}
|
||||
plain := plainText(h, src)
|
||||
id := uniqueID(Slug(plain), used)
|
||||
h.SetAttributeString("id", []byte(id))
|
||||
headings = append(headings, Heading{Level: h.Level, ID: id, Text: plain})
|
||||
return ast.WalkSkipChildren, nil
|
||||
})
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("walk markdown: %w", err)
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := engine.Renderer().Render(&out, src, doc); err != nil {
|
||||
return "", nil, fmt.Errorf("render markdown: %w", err)
|
||||
}
|
||||
return out.String(), headings, nil
|
||||
}
|
||||
|
||||
// uniqueID returns id, or id with a numeric suffix when it was already used.
|
||||
func uniqueID(id string, used map[string]int) string {
|
||||
used[id]++
|
||||
if used[id] == 1 {
|
||||
return id
|
||||
}
|
||||
return id + "-" + strconv.Itoa(used[id])
|
||||
}
|
||||
|
||||
// plainText returns the text of a node and its children without any markup.
|
||||
func plainText(n ast.Node, src []byte) string {
|
||||
var b strings.Builder
|
||||
for c := n.FirstChild(); c != nil; c = c.NextSibling() {
|
||||
switch t := c.(type) {
|
||||
case *ast.Text:
|
||||
b.Write(t.Segment.Value(src))
|
||||
case *ast.String:
|
||||
b.Write(t.Value)
|
||||
default:
|
||||
b.WriteString(plainText(c, src))
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
76
internal/mdx/mdx_test.go
Normal file
76
internal/mdx/mdx_test.go
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
package mdx
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestRenderParagraphAndList checks the basic blocks and inline markup.
|
||||
func TestRenderParagraphAndList(t *testing.T) {
|
||||
html, _, err := Render("Hello **world** and [a link](https://kmalloc.org/).\n\n- one\n- two\n")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"<p>Hello <strong>world</strong> and <a href=\"https://kmalloc.org/\">a link</a>.</p>",
|
||||
"<li>one</li>", "<li>two</li>",
|
||||
} {
|
||||
if !strings.Contains(html, want) {
|
||||
t.Errorf("html %q does not contain %q", html, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestRenderHeadingIDs checks that ids come from Slug and are unique.
|
||||
func TestRenderHeadingIDs(t *testing.T) {
|
||||
src := "## 01. First\n\n## Our commitments\n\n## Same\n\n## Same\n"
|
||||
html, headings, err := Render(src)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantIDs := []string{"01", "commitments", "same", "same-2"}
|
||||
if len(headings) != len(wantIDs) {
|
||||
t.Fatalf("got %d headings, want %d: %+v", len(headings), len(wantIDs), headings)
|
||||
}
|
||||
for i, id := range wantIDs {
|
||||
if headings[i].ID != id || headings[i].Level != 2 {
|
||||
t.Errorf("heading %d = %+v, want level 2 and id %q", i, headings[i], id)
|
||||
}
|
||||
if !strings.Contains(html, `id="`+id+`"`) {
|
||||
t.Errorf("html does not contain id=%q", id)
|
||||
}
|
||||
}
|
||||
if headings[0].Text != "01. First" {
|
||||
t.Errorf("heading text = %q, want %q", headings[0].Text, "01. First")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRenderRawHTMLPasses checks that raw HTML blocks and comments survive,
|
||||
// because pages use them for tables with data-label cells and hidden notes.
|
||||
func TestRenderRawHTMLPasses(t *testing.T) {
|
||||
src := "Intro.\n\n<table class=\"data\">\n<tr><td data-label=\"Kept\">14 days</td></tr>\n</table>\n\n<!-- TODO check: not shown -->\n\nEnd.\n"
|
||||
html, _, err := Render(src)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{
|
||||
`<table class="data">`, `<td data-label="Kept">14 days</td>`, "<!-- TODO check: not shown -->",
|
||||
} {
|
||||
if !strings.Contains(html, want) {
|
||||
t.Errorf("html %q does not contain %q", html, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestRenderHeadingTextIsPlain checks that the heading text drops the markup.
|
||||
func TestRenderHeadingTextIsPlain(t *testing.T) {
|
||||
_, headings, err := Render("## Free *your* mind\n")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(headings) != 1 || headings[0].Text != "Free your mind" {
|
||||
t.Errorf("headings = %+v, want one with plain text %q", headings, "Free your mind")
|
||||
}
|
||||
}
|
||||
45
internal/mdx/slug.go
Normal file
45
internal/mdx/slug.go
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
package mdx
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// leadingNumber matches a heading that starts with a two digit number.
|
||||
var leadingNumber = regexp.MustCompile(`^(\d\d)\b`)
|
||||
|
||||
// nonSlug matches a run of characters that cannot be part of a slug.
|
||||
var nonSlug = regexp.MustCompile(`[^a-z0-9]+`)
|
||||
|
||||
// Slug turns a heading into the identifier used for its anchor. A heading that
|
||||
// starts with a two digit number gets that number ("01. Title" becomes "01"),
|
||||
// the preamble, commitments and epilogue headings of the manifesto get the same
|
||||
// identifier in every language, and any other heading becomes lowercase words
|
||||
// joined by hyphens.
|
||||
func Slug(heading string) string {
|
||||
if m := leadingNumber.FindStringSubmatch(heading); m != nil {
|
||||
return m[1]
|
||||
}
|
||||
lower := strings.ToLower(strings.TrimSpace(heading))
|
||||
for _, group := range wellKnown {
|
||||
for _, prefix := range group.prefixes {
|
||||
if strings.HasPrefix(lower, prefix) {
|
||||
return group.slug
|
||||
}
|
||||
}
|
||||
}
|
||||
return strings.Trim(nonSlug.ReplaceAllString(lower, "-"), "-")
|
||||
}
|
||||
|
||||
// wellKnown lists the heading prefixes of the manifesto, in all its languages,
|
||||
// and the identifier each one gets.
|
||||
var wellKnown = []struct {
|
||||
slug string
|
||||
prefixes []string
|
||||
}{
|
||||
{"preamble", []string{"preamble", "preambolo", "preámbulo", "präambel", "préambule"}},
|
||||
{"commitments", []string{"our commitments", "i nostri impegni", "nuestros compromisos", "unsere verpflichtungen", "nos engagements"}},
|
||||
{"epilogue", []string{"epilogue", "epilogo", "epílogo", "epilog", "épilogue"}},
|
||||
}
|
||||
43
internal/mdx/slug_test.go
Normal file
43
internal/mdx/slug_test.go
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
package mdx
|
||||
|
||||
import "testing"
|
||||
|
||||
// TestSlug checks the heading identifiers against the real manifesto headings
|
||||
// (after the em dash rewrite) and ordinary headings.
|
||||
func TestSlug(t *testing.T) {
|
||||
tests := []struct {
|
||||
heading string
|
||||
want string
|
||||
}{
|
||||
{"01. Antifascist. No ambiguity.", "01"},
|
||||
{"14. Meaning doesn't need a god", "14"},
|
||||
{"01. Antifascisti. Senza ambiguità.", "01"},
|
||||
{"Preamble: We don't ask permission to think", "preamble"},
|
||||
{"Preambolo: Non chiediamo il permesso di pensare", "preamble"},
|
||||
{"Preámbulo: No pedimos permiso", "preamble"},
|
||||
{"Präambel", "preamble"},
|
||||
{"Préambule", "preamble"},
|
||||
{"Our commitments", "commitments"},
|
||||
{"I nostri impegni", "commitments"},
|
||||
{"Nuestros compromisos", "commitments"},
|
||||
{"Unsere Verpflichtungen", "commitments"},
|
||||
{"Nos engagements", "commitments"},
|
||||
{"Epilogue: Free your mind", "epilogue"},
|
||||
{"Epilogo: Libera la mente", "epilogue"},
|
||||
{"Epílogo: Libera tu mente", "epilogue"},
|
||||
{"Epilog", "epilogue"},
|
||||
{"Épilogue", "epilogue"},
|
||||
{"About this text", "about-this-text"},
|
||||
{"Request an account", "request-an-account"},
|
||||
{" Spaces & Symbols! ", "spaces-symbols"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.heading, func(t *testing.T) {
|
||||
if got := Slug(tt.heading); got != tt.want {
|
||||
t.Errorf("Slug(%q) = %q, want %q", tt.heading, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue