diff --git a/go.mod b/go.mod index 58439fe..0509b92 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,5 @@ module git.kmalloc.org/squid/motorino go 1.27 + +require github.com/yuin/goldmark v1.8.7 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..323ce3d --- /dev/null +++ b/go.sum @@ -0,0 +1,2 @@ +github.com/yuin/goldmark v1.8.7 h1:XT/tNqCIbabh0ln675w4x3dv04kQBKi8NwYqi9IJaxI= +github.com/yuin/goldmark v1.8.7/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= diff --git a/internal/mdx/emdash.go b/internal/mdx/emdash.go new file mode 100644 index 0000000..e6e74fa --- /dev/null +++ b/internal/mdx/emdash.go @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package mdx + +import ( + "fmt" + "regexp" + "strings" +) + +// emDash is the character the published site never shows. +const emDash = "—" + +// Rewrites of the manifesto headings, compiled once. The manifesto sources use +// em dashes: "NN — Title" becomes "NN. Title", "Word — Title" becomes +// "Word: Title" and an attribution line loses its dash. +var ( + numberedHeading = regexp.MustCompile(`(?m)^(#+ )(\d\d) ` + emDash + ` `) + wordHeading = regexp.MustCompile(`(?m)^(#+ [^\d\n][^` + emDash + `\n]*?) ` + emDash + ` `) + attribution = regexp.MustCompile(`(?m)^> ` + emDash + ` `) +) + +// parentheticals are the sentences in which a pair of dashes becomes commas. +var parentheticals = []struct{ from, to string }{ + {" " + emDash + " where technically feasible " + emDash + " ", ", where technically feasible, "}, + {" " + emDash + " dove tecnicamente possibile " + emDash + " ", ", dove tecnicamente possibile, "}, + {" " + emDash + " and continues", ", and continues"}, + {" " + emDash + " e continua", ", e continua"}, + {"kmalloc.org " + emDash + " ", "kmalloc.org: "}, +} + +// NoEmDash rewrites the em dashes of a manifesto source with the rules above +// and returns an error if any em dash is left, so a new one is noticed. +func NoEmDash(md string) (string, error) { + md = numberedHeading.ReplaceAllString(md, "${1}${2}. ") + md = wordHeading.ReplaceAllString(md, "${1}: ") + md = attribution.ReplaceAllString(md, "> ") + for _, p := range parentheticals { + md = strings.ReplaceAll(md, p.from, p.to) + } + if err := RejectEmDash("manifesto source", md); err != nil { + return "", err + } + return md, nil +} + +// RejectEmDash returns an error naming the file and the line of the first em +// dash in text, or nil when there is none. +func RejectEmDash(name, text string) error { + for i, line := range strings.Split(text, "\n") { + if strings.Contains(line, emDash) { + return fmt.Errorf("%s:%d: no em dashes, please", name, i+1) + } + } + return nil +} diff --git a/internal/mdx/emdash_test.go b/internal/mdx/emdash_test.go new file mode 100644 index 0000000..0682d9f --- /dev/null +++ b/internal/mdx/emdash_test.go @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package mdx + +import ( + "strings" + "testing" +) + +// TestNoEmDash checks every rewrite of the manifesto sources. +func TestNoEmDash(t *testing.T) { + tests := []struct { + name string + in string + want string + }{ + {"numbered heading", "## 01 — Antifascist. No ambiguity.", "## 01. Antifascist. No ambiguity."}, + {"word heading", "## Preamble — We don't ask permission to think", "## Preamble: We don't ask permission to think"}, + {"title heading", "# kmalloc.org — Manifesto", "# kmalloc.org: Manifesto"}, + {"attribution", "> — Funkadelic, Free Your Mind", "> Funkadelic, Free Your Mind"}, + {"parenthetical en", "modify, and — where technically feasible — install", "modify, and, where technically feasible, install"}, + {"parenthetical it", "e — dove tecnicamente possibile — installare", "e, dove tecnicamente possibile, installare"}, + {"and continues", "it — and continues", "it, and continues"}, + {"e continua", "esso — e continua", "esso, e continua"}, + {"no dash", "plain text", "plain text"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := NoEmDash(tt.in) + if err != nil { + t.Fatalf("NoEmDash() error = %v", err) + } + if got != tt.want { + t.Errorf("NoEmDash() = %q, want %q", got, tt.want) + } + }) + } +} + +// TestNoEmDashLeftover checks that a dash with no rewrite rule is an error. +func TestNoEmDashLeftover(t *testing.T) { + _, err := NoEmDash("some words — and then more") + if err == nil { + t.Fatal("NoEmDash() error = nil, want an error for a leftover em dash") + } +} + +// TestRejectEmDash checks that the error names the file and the line. +func TestRejectEmDash(t *testing.T) { + if err := RejectEmDash("content/en/x.md", "clean\ntext\n"); err != nil { + t.Errorf("RejectEmDash(clean) = %v", err) + } + err := RejectEmDash("content/en/x.md", "clean\nbad — line\n") + if err == nil || !strings.Contains(err.Error(), "content/en/x.md") || !strings.Contains(err.Error(), "2") { + t.Errorf("RejectEmDash() = %v, want an error naming the file and line 2", err) + } +} diff --git a/internal/mdx/mdx.go b/internal/mdx/mdx.go new file mode 100644 index 0000000..41ea41b --- /dev/null +++ b/internal/mdx/mdx.go @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +// Package mdx renders the Markdown of pages and news with goldmark. +package mdx + +import ( + "bytes" + "fmt" + "strconv" + "strings" + + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/renderer/html" + "github.com/yuin/goldmark/text" +) + +// Heading is a heading of a rendered document. +type Heading struct { + // Level is 1 for "#", 2 for "##", and so on. + Level int + // ID is the anchor identifier given to the heading. + ID string + // Text is the heading without markup. + Text string +} + +// engine converts Markdown to HTML. Raw HTML is passed through because the +// pages use it for tables with data-label cells, hidden notes and the hero. +var engine = goldmark.New(goldmark.WithRendererOptions(html.WithUnsafe())) + +// Render converts md to HTML and returns the headings it found. Every heading +// gets an id attribute made by Slug, with -2, -3 and so on for duplicates. +func Render(md string) (string, []Heading, error) { + src := []byte(md) + doc := engine.Parser().Parse(text.NewReader(src)) + + var headings []Heading + used := make(map[string]int) + err := ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) { + h, ok := n.(*ast.Heading) + if !ok || !entering { + return ast.WalkContinue, nil + } + plain := plainText(h, src) + id := uniqueID(Slug(plain), used) + h.SetAttributeString("id", []byte(id)) + headings = append(headings, Heading{Level: h.Level, ID: id, Text: plain}) + return ast.WalkSkipChildren, nil + }) + if err != nil { + return "", nil, fmt.Errorf("walk markdown: %w", err) + } + + var out bytes.Buffer + if err := engine.Renderer().Render(&out, src, doc); err != nil { + return "", nil, fmt.Errorf("render markdown: %w", err) + } + return out.String(), headings, nil +} + +// uniqueID returns id, or id with a numeric suffix when it was already used. +func uniqueID(id string, used map[string]int) string { + used[id]++ + if used[id] == 1 { + return id + } + return id + "-" + strconv.Itoa(used[id]) +} + +// plainText returns the text of a node and its children without any markup. +func plainText(n ast.Node, src []byte) string { + var b strings.Builder + for c := n.FirstChild(); c != nil; c = c.NextSibling() { + switch t := c.(type) { + case *ast.Text: + b.Write(t.Segment.Value(src)) + case *ast.String: + b.Write(t.Value) + default: + b.WriteString(plainText(c, src)) + } + } + return b.String() +} diff --git a/internal/mdx/mdx_test.go b/internal/mdx/mdx_test.go new file mode 100644 index 0000000..a0a58a3 --- /dev/null +++ b/internal/mdx/mdx_test.go @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package mdx + +import ( + "strings" + "testing" +) + +// TestRenderParagraphAndList checks the basic blocks and inline markup. +func TestRenderParagraphAndList(t *testing.T) { + html, _, err := Render("Hello **world** and [a link](https://kmalloc.org/).\n\n- one\n- two\n") + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + "

Hello world and a link.

", + "
  • one
  • ", "
  • two
  • ", + } { + if !strings.Contains(html, want) { + t.Errorf("html %q does not contain %q", html, want) + } + } +} + +// TestRenderHeadingIDs checks that ids come from Slug and are unique. +func TestRenderHeadingIDs(t *testing.T) { + src := "## 01. First\n\n## Our commitments\n\n## Same\n\n## Same\n" + html, headings, err := Render(src) + if err != nil { + t.Fatal(err) + } + wantIDs := []string{"01", "commitments", "same", "same-2"} + if len(headings) != len(wantIDs) { + t.Fatalf("got %d headings, want %d: %+v", len(headings), len(wantIDs), headings) + } + for i, id := range wantIDs { + if headings[i].ID != id || headings[i].Level != 2 { + t.Errorf("heading %d = %+v, want level 2 and id %q", i, headings[i], id) + } + if !strings.Contains(html, `id="`+id+`"`) { + t.Errorf("html does not contain id=%q", id) + } + } + if headings[0].Text != "01. First" { + t.Errorf("heading text = %q, want %q", headings[0].Text, "01. First") + } +} + +// TestRenderRawHTMLPasses checks that raw HTML blocks and comments survive, +// because pages use them for tables with data-label cells and hidden notes. +func TestRenderRawHTMLPasses(t *testing.T) { + src := "Intro.\n\n\n\n
    14 days
    \n\n\n\nEnd.\n" + html, _, err := Render(src) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + ``, ``, "", + } { + if !strings.Contains(html, want) { + t.Errorf("html %q does not contain %q", html, want) + } + } +} + +// TestRenderHeadingTextIsPlain checks that the heading text drops the markup. +func TestRenderHeadingTextIsPlain(t *testing.T) { + _, headings, err := Render("## Free *your* mind\n") + if err != nil { + t.Fatal(err) + } + if len(headings) != 1 || headings[0].Text != "Free your mind" { + t.Errorf("headings = %+v, want one with plain text %q", headings, "Free your mind") + } +} diff --git a/internal/mdx/slug.go b/internal/mdx/slug.go new file mode 100644 index 0000000..47359bf --- /dev/null +++ b/internal/mdx/slug.go @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package mdx + +import ( + "regexp" + "strings" +) + +// leadingNumber matches a heading that starts with a two digit number. +var leadingNumber = regexp.MustCompile(`^(\d\d)\b`) + +// nonSlug matches a run of characters that cannot be part of a slug. +var nonSlug = regexp.MustCompile(`[^a-z0-9]+`) + +// Slug turns a heading into the identifier used for its anchor. A heading that +// starts with a two digit number gets that number ("01. Title" becomes "01"), +// the preamble, commitments and epilogue headings of the manifesto get the same +// identifier in every language, and any other heading becomes lowercase words +// joined by hyphens. +func Slug(heading string) string { + if m := leadingNumber.FindStringSubmatch(heading); m != nil { + return m[1] + } + lower := strings.ToLower(strings.TrimSpace(heading)) + for _, group := range wellKnown { + for _, prefix := range group.prefixes { + if strings.HasPrefix(lower, prefix) { + return group.slug + } + } + } + return strings.Trim(nonSlug.ReplaceAllString(lower, "-"), "-") +} + +// wellKnown lists the heading prefixes of the manifesto, in all its languages, +// and the identifier each one gets. +var wellKnown = []struct { + slug string + prefixes []string +}{ + {"preamble", []string{"preamble", "preambolo", "preámbulo", "präambel", "préambule"}}, + {"commitments", []string{"our commitments", "i nostri impegni", "nuestros compromisos", "unsere verpflichtungen", "nos engagements"}}, + {"epilogue", []string{"epilogue", "epilogo", "epílogo", "epilog", "épilogue"}}, +} diff --git a/internal/mdx/slug_test.go b/internal/mdx/slug_test.go new file mode 100644 index 0000000..a6a0788 --- /dev/null +++ b/internal/mdx/slug_test.go @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package mdx + +import "testing" + +// TestSlug checks the heading identifiers against the real manifesto headings +// (after the em dash rewrite) and ordinary headings. +func TestSlug(t *testing.T) { + tests := []struct { + heading string + want string + }{ + {"01. Antifascist. No ambiguity.", "01"}, + {"14. Meaning doesn't need a god", "14"}, + {"01. Antifascisti. Senza ambiguità.", "01"}, + {"Preamble: We don't ask permission to think", "preamble"}, + {"Preambolo: Non chiediamo il permesso di pensare", "preamble"}, + {"Preámbulo: No pedimos permiso", "preamble"}, + {"Präambel", "preamble"}, + {"Préambule", "preamble"}, + {"Our commitments", "commitments"}, + {"I nostri impegni", "commitments"}, + {"Nuestros compromisos", "commitments"}, + {"Unsere Verpflichtungen", "commitments"}, + {"Nos engagements", "commitments"}, + {"Epilogue: Free your mind", "epilogue"}, + {"Epilogo: Libera la mente", "epilogue"}, + {"Epílogo: Libera tu mente", "epilogue"}, + {"Epilog", "epilogue"}, + {"Épilogue", "epilogue"}, + {"About this text", "about-this-text"}, + {"Request an account", "request-an-account"}, + {" Spaces & Symbols! ", "spaces-symbols"}, + } + for _, tt := range tests { + t.Run(tt.heading, func(t *testing.T) { + if got := Slug(tt.heading); got != tt.want { + t.Errorf("Slug(%q) = %q, want %q", tt.heading, got, tt.want) + } + }) + } +}
    14 days