From 917683afa491c7f0b4331757d6066d215b5f19d9 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Sun, 15 Nov 2026 10:32:19 +0100 Subject: [PATCH] fix: close the minor findings of the first review Require refuses an empty value. Heading ids: an explicit {#id} must be sound and unique, generated ids avoid the explicit ones and each other, and a heading with no ASCII letters gets a fallback. Fenced code blocks keep their placeholders and a backslash keeps one anywhere. The em dash rule covers the header of the manifesto pages and the HTML entities. The feed makes src links absolute, base_url is required, and the news page must be a folder index. A category written twice in a post counts once, a block name in a news title is no longer taken for a placeholder, and build refuses extra arguments. --- README.md | 2 +- cmd/motorino/main.go | 5 ++ cmd/motorino/main_test.go | 16 ++++- internal/build/build.go | 8 ++- internal/build/build_test.go | 49 +++++++++++++++ internal/build/feed.go | 2 +- internal/build/pages.go | 35 +++++++---- internal/content/pages.go | 8 +++ internal/content/pages_test.go | 15 +++++ internal/content/posts.go | 8 ++- internal/content/posts_test.go | 12 ++++ internal/header/header.go | 6 +- internal/header/header_test.go | 15 +++++ internal/mdx/emdash.go | 11 +++- internal/mdx/emdash_test.go | 21 +++++++ internal/mdx/mdx.go | 61 ++++++++++++++++--- internal/mdx/mdx_test.go | 78 ++++++++++++++++++++++++ internal/render/markdown.go | 108 +++++++++++++++++++++++++++++++++ internal/render/render_test.go | 37 +++++++++++ internal/site/site.go | 12 ++++ internal/site/site_test.go | 17 +++++- 21 files changed, 493 insertions(+), 33 deletions(-) create mode 100644 internal/render/markdown.go diff --git a/README.md b/README.md index 81c0168..9ad28fa 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ Pages: `Title`, `ID` (the same in every language, it links the translations), `D ### Placeholders -`{{name}}`, with lowercase letters, digits and underscores, is replaced by a string from `site.json` or the language file. An unknown placeholder stops the build and names the file. `{{news_list}}`, `{{latest_news}}` and `{{categories_list}}` stand alone in a paragraph and expand to generated lists. +`{{name}}`, with lowercase letters, digits and underscores, is replaced by a string from `site.json` or the language file. An unknown placeholder stops the build and names the file. Fenced code blocks are left as written, so an article can show a placeholder there; elsewhere a backslash before the placeholder (`\{{name}}`) keeps it as written, and Markdown drops the backslash. Code spans are substituted, which is how a page shows a value in monospace. `{{news_list}}`, `{{latest_news}}` and `{{categories_list}}` stand alone in a paragraph and expand to generated lists. ## Development diff --git a/cmd/motorino/main.go b/cmd/motorino/main.go index ce40c01..a1997f1 100644 --- a/cmd/motorino/main.go +++ b/cmd/motorino/main.go @@ -15,6 +15,7 @@ import ( "fmt" "io" "os" + "strings" "git.kmalloc.org/squid/motorino/internal/build" "git.kmalloc.org/squid/motorino/internal/version" @@ -59,6 +60,10 @@ func runBuild(args []string, stdout, stderr io.Writer) int { } return 2 } + if flags.NArg() > 0 { + fmt.Fprintf(stderr, "motorino: unexpected arguments: %s\n%s\n", strings.Join(flags.Args(), " "), usage) + return 2 + } result, err := build.Build(build.Options{SiteDir: *site, OutDir: *out, Version: version.String()}) if err != nil { fmt.Fprintf(stderr, "motorino: %v\n", err) diff --git a/cmd/motorino/main_test.go b/cmd/motorino/main_test.go index 8e2ffd6..4090294 100644 --- a/cmd/motorino/main_test.go +++ b/cmd/motorino/main_test.go @@ -30,8 +30,11 @@ func miniSite(t *testing.T) string { } for _, id := range []string{"home", "manifesto", "people", "services", "news", "history"} { name := id - if id == "home" { + switch id { + case "home": name = "index" + case "news": + name = "news/index" } files["content/en/"+name+".md"] = "Title: " + id + "\nID: " + id + "\nDescription: d\n-\nBody of " + id + ".\n" } @@ -101,6 +104,17 @@ func TestRunUnknownCommand(t *testing.T) { } } +// TestRunBuildRejectsExtraArguments checks that arguments that are not flags are not ignored. +func TestRunBuildRejectsExtraArguments(t *testing.T) { + var out, errOut bytes.Buffer + if code := run([]string{"build", "somesite", "someout"}, &out, &errOut); code != 2 { + t.Errorf("run(build somesite someout) = %d, want 2", code) + } + if !strings.Contains(errOut.String(), "unexpected") { + t.Errorf("stderr = %q, want it to say what was unexpected", errOut.String()) + } +} + // TestRunBuildBadFlag checks that an unknown flag is a usage error. func TestRunBuildBadFlag(t *testing.T) { var out, errOut bytes.Buffer diff --git a/internal/build/build.go b/internal/build/build.go index 7975c88..a6d2ea3 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -76,7 +76,10 @@ type Result struct { } // Build reads the site in opts.SiteDir and writes it to opts.OutDir. It renders -// everything in memory first, so a failure leaves the output folder untouched. +// everything in memory first, so a failure in the content leaves the output +// folder untouched. Writing the files is not atomic, and files of an earlier +// build that are no longer produced are not removed: build into a fresh folder +// when that matters, as the deploy does. func Build(opts Options) (Result, error) { b, err := load(opts) if err != nil { @@ -137,6 +140,9 @@ func load(opts Options) (*builder, error) { return nil, fmt.Errorf("content/%s: there is no page with ID %q", b.defaultLang, id) } } + if news := b.byKey[pageKey{"news", b.defaultLang}]; !strings.HasSuffix(news.Path, "/") { + return nil, fmt.Errorf("%s: the news page must be an index (news/index.md), so that the feed has an address next to it", news.Source) + } if b.templates, err = readTemplates(opts.SiteDir); err != nil { return nil, err } diff --git a/internal/build/build_test.go b/internal/build/build_test.go index b46db52..9d88ad0 100644 --- a/internal/build/build_test.go +++ b/internal/build/build_test.go @@ -358,6 +358,55 @@ func TestBuildRejectsSwitchToUnknownLanguage(t *testing.T) { } } +// TestBuildFeedMakesImagesAbsolute checks that src links get the full address like href links. +func TestBuildFeedMakesImagesAbsolute(t *testing.T) { + files := baseFiles() + files["news/pic.md"] = "Title: Pic\nDate: 2026-11-15T12:00:00+0100\n-\n![the logo](/logo.svg)\n" + out, err := runBuild(t, files) + if err != nil { + t.Fatal(err) + } + mustContain(t, "feed", readOut(t, out, "news/feed.xml"), `src="https://kmalloc.org/logo.svg"`) +} + +// TestBuildNewsPageMustBeAFolderIndex checks that the feed has a sound address. +func TestBuildNewsPageMustBeAFolderIndex(t *testing.T) { + files := baseFiles() + delete(files, "content/en/news/index.md") + files["content/en/news.md"] = page("news", "News", "", "{{news_list}}\n") + _, err := runBuild(t, files) + if err == nil || !strings.Contains(err.Error(), "content/en/news.md") || !strings.Contains(err.Error(), "index") { + t.Errorf("Build() error = %v, want it to say that content/en/news.md must be an index", err) + } +} + +// TestBuildBlockNameInATitleIsNotAnError checks that a generated list that shows +// the text of a block placeholder (in a news title) is not mistaken for a +// placeholder that is not alone in its paragraph. +func TestBuildBlockNameInATitleIsNotAnError(t *testing.T) { + files := baseFiles() + files["news/tpl.md"] = "Title: Use {{latest_news}} in a page\nDate: 2026-11-15T12:00:00+0100\n-\nA note.\n" + out, err := runBuild(t, files) + if err != nil { + t.Fatalf("Build() error = %v", err) + } + mustContain(t, "home", readOut(t, out, "index.html"), "Use {{latest_news}} in a page") +} + +// TestBuildPlaceholdersAndCode checks the three ways a page can use a placeholder +// next to code: a value in a code span (the pages show the GPG fingerprint that +// way), a fenced block that shows a placeholder as written, and an escaped one. +func TestBuildPlaceholdersAndCode(t *testing.T) { + files := baseFiles() + files["content/en/people.md"] = page("people", "People", "", "Value `{{lang}}` and escaped \\{{lang}}.\n\n```\n{{lang}} {{nope}}\n```\n\nAnd {{lang}}.\n") + out, err := runBuild(t, files) + if err != nil { + t.Fatalf("Build() error = %v", err) + } + got := readOut(t, out, "people.html") + mustContain(t, "people", got, "en", "escaped {{lang}}.", "{{lang}} {{nope}}", "And en.") +} + // TestBuildAssetsCopied checks the assets listed in site.json. func TestBuildAssetsCopied(t *testing.T) { out, err := runBuild(t, baseFiles()) diff --git a/internal/build/feed.go b/internal/build/feed.go index 485cc8a..719bd6e 100644 --- a/internal/build/feed.go +++ b/internal/build/feed.go @@ -21,7 +21,7 @@ func feed(base string, l site.Lang, newsURL string, posts []renderedPost, limit var items strings.Builder for _, p := range posts { link := base + p.Path - text := strings.ReplaceAll(p.HTML, `href="/`, `href="`+base+`/`) + text := strings.NewReplacer(`href="/`, `href="`+base+`/`, `src="/`, `src="`+base+`/`).Replace(p.HTML) items.WriteString("\n") items.WriteString("" + render.Escape(p.Title) + "\n") items.WriteString("" + link + "\n") diff --git a/internal/build/pages.go b/internal/build/pages.go index 6de74b6..c873d09 100644 --- a/internal/build/pages.go +++ b/internal/build/pages.go @@ -4,6 +4,7 @@ package build import ( "fmt" + "regexp" "strings" "git.kmalloc.org/squid/motorino/internal/content" @@ -61,7 +62,7 @@ func (b *builder) renderPage(p content.Page) error { var toc []string for _, h := range headings { if h.Level == 2 { - toc = append(toc, fmt.Sprintf(`
  • %s
  • `, h.ID, render.Escape(h.Text))) + toc = append(toc, fmt.Sprintf(`
  • %s
  • `, render.EscapeAttr(h.ID), render.Escape(h.Text))) } } ctx["manifesto_toc"] = strings.Join(toc, "\n") @@ -80,7 +81,7 @@ func (b *builder) renderPage(p content.Page) error { // markdown replaces the placeholders of a source, converts it to HTML and // expands the generated lists. The name is used in error messages. func (b *builder) markdown(name, source, lang string, ctx map[string]string) (string, []mdx.Heading, error) { - text, err := render.Substitute(name, source, ctx) + text, err := render.SubstituteMarkdown(name, source, ctx) if err != nil { return "", nil, err } @@ -92,22 +93,32 @@ func (b *builder) markdown(name, source, lang string, ctx map[string]string) (st return html, headings, err } +// codeElement matches a code element, where a placeholder is shown as written. +var codeElement = regexp.MustCompile(`(?s)]*>.*?`) + // expandBlocks replaces each paragraph that holds only a block placeholder by -// the generated HTML. A block placeholder inside other text is an error. +// the generated HTML. A block placeholder inside other text is an error. All +// blocks are checked before any is expanded, so the text of a generated list +// (a news title may say {{latest_news}}) is never taken for a placeholder. func (b *builder) expandBlocks(name, html, lang string, ctx map[string]string) (string, error) { + outside := codeElement.ReplaceAllString(html, "") for _, block := range blockNames { - marker := "

    {{" + block + "}}

    " - if strings.Contains(html, marker) { - generated, err := b.block(block, lang, ctx) - if err != nil { - return "", err - } - html = strings.ReplaceAll(html, marker, generated) - } - if strings.Contains(html, "{{"+block+"}}") { + token := "{{" + block + "}}" + if strings.Count(outside, token) != strings.Count(outside, "

    "+token+"

    ") { return "", fmt.Errorf("%s: {{%s}} must stand alone in a paragraph", name, block) } } + for _, block := range blockNames { + marker := "

    {{" + block + "}}

    " + if !strings.Contains(html, marker) { + continue + } + generated, err := b.block(block, lang, ctx) + if err != nil { + return "", err + } + html = strings.ReplaceAll(html, marker, generated) + } return html, nil } diff --git a/internal/content/pages.go b/internal/content/pages.go index 3c6911b..696dfe3 100644 --- a/internal/content/pages.go +++ b/internal/content/pages.go @@ -142,6 +142,14 @@ func loadPage(file, source, defaultLang, lang, rel string) (Page, error) { if err := mdx.RejectEmDash(source, string(data)); err != nil { return Page{}, err } + } else { + // The body of the manifesto is the owner's text and is rewritten when the + // page is built; the keys of the header are not. + for _, key := range []string{"Title", "Description", "Note"} { + if err := mdx.RejectEmDash(source+" (key "+key+")", h.Get(key)); err != nil { + return Page{}, err + } + } } switches, err := parseSwitch(source, h.Get("Switch")) if err != nil { diff --git a/internal/content/pages_test.go b/internal/content/pages_test.go index a42e2f9..91881fc 100644 --- a/internal/content/pages_test.go +++ b/internal/content/pages_test.go @@ -107,6 +107,21 @@ func TestLoadPagesManifestoKeepsEmDash(t *testing.T) { } } +// TestLoadPagesManifestoHeaderEmDash checks that the exemption of the manifesto +// body does not cover the title, the description and the note. +func TestLoadPagesManifestoHeaderEmDash(t *testing.T) { + for _, field := range []string{"Title", "Description", "Note"} { + head := map[string]string{"Title": "M", "Description": "d", "Note": "n"} + head[field] = "words \u2014 more" + file := "Title: " + head["Title"] + "\nID: manifesto\nDescription: " + head["Description"] + "\nLayout: manifesto\nNote: " + head["Note"] + "\n-\n## 01 \u2014 Body\n" + dir := writeFiles(t, map[string]string{"content/en/m.md": file}) + _, err := LoadPages(dir, []string{"en"}) + if err == nil || !strings.Contains(err.Error(), "content/en/m.md") || !strings.Contains(err.Error(), field) { + t.Errorf("%s with an em dash: LoadPages() error = %v, want it to name content/en/m.md and %s", field, err, field) + } + } +} + // TestLoadPagesDuplicateID checks that two pages cannot share an ID in a language. func TestLoadPagesDuplicateID(t *testing.T) { dir := writeFiles(t, map[string]string{ diff --git a/internal/content/posts.go b/internal/content/posts.go index a51c408..ae5f103 100644 --- a/internal/content/posts.go +++ b/internal/content/posts.go @@ -136,6 +136,7 @@ func parseDate(source, value string) (time.Time, error) { // parseCategories splits a comma separated list into categories. func parseCategories(source, value string) ([]Category, error) { var out []Category + seen := map[string]bool{} for _, name := range strings.Split(value, ",") { name = strings.TrimSpace(name) if name == "" { @@ -145,6 +146,10 @@ func parseCategories(source, value string) ([]Category, error) { if err != nil { return nil, fmt.Errorf("%s: %w", source, err) } + if seen[slug] { + continue + } + seen[slug] = true out = append(out, Category{Name: name, Slug: slug}) } return out, nil @@ -177,7 +182,8 @@ func checkCategories(posts []Post) error { return nil } -// markup matches the Markdown that is dropped from a fallback description. +// markdownLink, markdownMark and sentenceEnd pick the first sentence out of a +// body and drop its Markdown markup, for a description that was not written. var ( markdownLink = regexp.MustCompile(`\[([^\]]*)\]\([^)]*\)`) markdownMark = regexp.MustCompile("[*_`]") diff --git a/internal/content/posts_test.go b/internal/content/posts_test.go index 3905178..c3dfda2 100644 --- a/internal/content/posts_test.go +++ b/internal/content/posts_test.go @@ -171,6 +171,18 @@ func TestLoadPostsCategoryCollision(t *testing.T) { } } +// TestLoadPostsRepeatedCategory checks that a category written twice counts once. +func TestLoadPostsRepeatedCategory(t *testing.T) { + dir := writeFiles(t, map[string]string{"news/a.md": "Title: A\nDate: 2026-01-01T10:00:00+0100\nCategories: News, Linux, news, News\n-\nx\n"}) + posts, err := LoadPosts(dir) + if err != nil { + t.Fatal(err) + } + if len(posts[0].Categories) != 2 || posts[0].Categories[0].Name != "News" || posts[0].Categories[1].Name != "Linux" { + t.Errorf("Categories = %+v, want News and Linux once each", posts[0].Categories) + } +} + // TestLoadPostsNoNewsFolder checks that a site without news is fine. func TestLoadPostsNoNewsFolder(t *testing.T) { posts, err := LoadPosts(t.TempDir()) diff --git a/internal/header/header.go b/internal/header/header.go index 9b8d6aa..78fa8ac 100644 --- a/internal/header/header.go +++ b/internal/header/header.go @@ -78,12 +78,16 @@ func (h Header) Has(key string) bool { return ok } -// Require returns an error naming name and the first missing key of keys. +// Require returns an error naming name and the first key of keys that is +// missing or has an empty value. func (h Header) Require(name string, keys ...string) error { for _, k := range keys { if !h.Has(k) { return fmt.Errorf("%s: missing key %q", name, k) } + if h.Get(k) == "" { + return fmt.Errorf("%s: the key %q is empty", name, k) + } } return nil } diff --git a/internal/header/header_test.go b/internal/header/header_test.go index 30758f6..9e38835 100644 --- a/internal/header/header_test.go +++ b/internal/header/header_test.go @@ -124,6 +124,21 @@ func TestRequire(t *testing.T) { } } +// TestRequireNeedsAValue checks that a key with an empty value counts as missing. +func TestRequireNeedsAValue(t *testing.T) { + h, _, err := Parse("file.md", []byte("Title:\nID: x\n-\n")) + if err != nil { + t.Fatal(err) + } + err = h.Require("file.md", "Title") + if err == nil || !strings.Contains(err.Error(), "Title") || !strings.Contains(err.Error(), "empty") { + t.Errorf("Require(Title) = %v, want an error saying that Title is empty", err) + } + if err := h.Require("file.md", "ID"); err != nil { + t.Errorf("Require(ID) = %v", err) + } +} + // TestOnly checks that a key outside the allowed list is rejected. func TestOnly(t *testing.T) { h, _, err := Parse("file.md", []byte("Title: T\nTitolo: T\n-\n")) diff --git a/internal/mdx/emdash.go b/internal/mdx/emdash.go index e6e74fa..932f980 100644 --- a/internal/mdx/emdash.go +++ b/internal/mdx/emdash.go @@ -38,17 +38,22 @@ func NoEmDash(md string) (string, error) { for _, p := range parentheticals { md = strings.ReplaceAll(md, p.from, p.to) } - if err := RejectEmDash("manifesto source", md); err != nil { + if err := RejectEmDash("the body", md); err != nil { return "", err } return md, nil } +// emDashEntity matches the HTML entities of the em dash, which would put it +// on the page as well. +var emDashEntity = regexp.MustCompile(`(?i)—|—|—`) + // RejectEmDash returns an error naming the file and the line of the first em -// dash in text, or nil when there is none. +// dash in text, written as the character or as an HTML entity, or nil when +// there is none. func RejectEmDash(name, text string) error { for i, line := range strings.Split(text, "\n") { - if strings.Contains(line, emDash) { + if strings.Contains(line, emDash) || emDashEntity.MatchString(line) { return fmt.Errorf("%s:%d: no em dashes, please", name, i+1) } } diff --git a/internal/mdx/emdash_test.go b/internal/mdx/emdash_test.go index 0682d9f..8e13178 100644 --- a/internal/mdx/emdash_test.go +++ b/internal/mdx/emdash_test.go @@ -45,6 +45,27 @@ func TestNoEmDashLeftover(t *testing.T) { } } +// TestRejectEmDashEntities checks that the entities of the em dash are refused too. +func TestRejectEmDashEntities(t *testing.T) { + for _, in := range []string{"a — b", "a &MDASH; b", "a — b", "a — b", "a — b"} { + if err := RejectEmDash("f.md", in); err == nil { + t.Errorf("RejectEmDash(%q) = nil, want an error", in) + } + } + if err := RejectEmDash("f.md", "a – b & c"); err != nil { + t.Errorf("RejectEmDash(ndash) = %v", err) + } +} + +// TestNoEmDashErrorSaysBody checks that a leftover is reported as a line of the +// body, which is how the caller counts it. +func TestNoEmDashErrorSaysBody(t *testing.T) { + _, err := NoEmDash("fine\nsome words \u2014 and then more\n") + if err == nil || !strings.Contains(err.Error(), "the body:2") { + t.Errorf("NoEmDash() error = %v, want it to say the body:2", err) + } +} + // TestRejectEmDash checks that the error names the file and the line. func TestRejectEmDash(t *testing.T) { if err := RejectEmDash("content/en/x.md", "clean\ntext\n"); err != nil { diff --git a/internal/mdx/mdx.go b/internal/mdx/mdx.go index f79f644..4eb323b 100644 --- a/internal/mdx/mdx.go +++ b/internal/mdx/mdx.go @@ -6,6 +6,7 @@ package mdx import ( "bytes" "fmt" + "regexp" "strconv" "strings" @@ -42,9 +43,12 @@ func Render(md string) (string, []Heading, error) { src := []byte(md) doc := engine.Parser().Parse(text.NewReader(src)) + used, err := explicitIDs(doc) + if err != nil { + return "", nil, err + } var headings []Heading - used := make(map[string]int) - err := ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) { + err = ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) { h, ok := n.(*ast.Heading) if !ok || !entering { return ast.WalkContinue, nil @@ -53,9 +57,12 @@ func Render(md string) (string, []Heading, error) { id := "" if explicit, ok := h.AttributeString("id"); ok { id = attributeText(explicit) - used[id]++ } else if h.Level >= 2 { - id = uniqueID(Slug(plain), used) + base := Slug(plain) + if base == "" { + base = "section" + } + id = uniqueID(base, used) h.SetAttributeString("id", []byte(id)) } headings = append(headings, Heading{Level: h.Level, ID: id, Text: plain}) @@ -83,13 +90,47 @@ func attributeText(value any) string { return "" } -// uniqueID returns id, or id with a numeric suffix when it was already used. -func uniqueID(id string, used map[string]int) string { - used[id]++ - if used[id] == 1 { - return id +// explicitID is what an id set with {#id} may look like: it ends up in +// attributes and in addresses. +var explicitID = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.:-]*$`) + +// explicitIDs returns the ids that headings set for themselves with {#id}. A +// bad id or an id used by two headings is an error. Generated ids avoid these, +// whatever the order of the headings. +func explicitIDs(doc ast.Node) (map[string]bool, error) { + used := map[string]bool{} + err := ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) { + h, ok := n.(*ast.Heading) + if !ok || !entering { + return ast.WalkContinue, nil + } + value, ok := h.AttributeString("id") + if !ok { + return ast.WalkSkipChildren, nil + } + id := attributeText(value) + if !explicitID.MatchString(id) { + return ast.WalkStop, fmt.Errorf("the heading id %q may have only letters, digits and _ . : -", id) + } + if used[id] { + return ast.WalkStop, fmt.Errorf("the heading id %q is used by two headings", id) + } + used[id] = true + return ast.WalkSkipChildren, nil + }) + return used, err +} + +// uniqueID returns base, or base with -2, -3 and so on until the id is not +// used, and marks it as used. +func uniqueID(base string, used map[string]bool) string { + id, n := base, 1 + for used[id] { + n++ + id = base + "-" + strconv.Itoa(n) } - return id + "-" + strconv.Itoa(used[id]) + used[id] = true + return id } // plainText returns the text of a node and its children without any markup. diff --git a/internal/mdx/mdx_test.go b/internal/mdx/mdx_test.go index 3173977..c576ca4 100644 --- a/internal/mdx/mdx_test.go +++ b/internal/mdx/mdx_test.go @@ -99,6 +99,84 @@ func TestRenderExplicitHeadingID(t *testing.T) { } } +// TestRenderRejectsRepeatedExplicitID checks that two headings cannot ask for the same id. +func TestRenderRejectsRepeatedExplicitID(t *testing.T) { + for _, src := range []string{"## Foo {#a}\n\n## Bar {#a}\n"} { + _, _, err := Render(src) + if err == nil || !strings.Contains(err.Error(), "id") { + t.Errorf("Render(%q) error = %v, want an error about the repeated id", src, err) + } + } +} + +// TestRenderGeneratedIDAvoidsLaterExplicit checks that a generated id never takes +// one that a later heading asks for by name, in either order of the two. +func TestRenderGeneratedIDAvoidsLaterExplicit(t *testing.T) { + _, headings, err := Render("## Foo\n\n## Bar {#foo-2}\n\n## Foo\n") + if err != nil { + t.Fatal(err) + } + var ids []string + for _, h := range headings { + ids = append(ids, h.ID) + } + if got, want := strings.Join(ids, ","), "foo,foo-2,foo-3"; got != want { + t.Errorf("ids = %s, want %s", got, want) + } +} + +// TestRenderSuffixClash checks that a numbered suffix never repeats an id. +func TestRenderSuffixClash(t *testing.T) { + _, headings, err := Render("## A\n\n## A\n\n## A 2\n") + if err != nil { + t.Fatal(err) + } + seen := map[string]bool{} + for _, h := range headings { + if seen[h.ID] { + t.Errorf("id %q is used twice: %+v", h.ID, headings) + } + seen[h.ID] = true + } +} + +// TestRenderEmptySlugFallback checks that a heading with no ASCII letters or digits still gets an id. +func TestRenderEmptySlugFallback(t *testing.T) { + html, headings, err := Render("## \u4f60\u597d\n\n## \u4e16\u754c\n") + if err != nil { + t.Fatal(err) + } + if strings.Contains(html, `id=""`) { + t.Errorf("html %q has an empty id", html) + } + if len(headings) != 2 || headings[0].ID != "section" || headings[1].ID != "section-2" { + t.Errorf("headings = %+v, want section and section-2", headings) + } +} + +// TestRenderBadExplicitIDCannotInject checks that a malformed {#id} never ends +// up as an id with quotes, spaces or angle brackets, whether goldmark leaves it +// as text or the id check refuses it; and that sound ids pass. +func TestRenderBadExplicitIDCannotInject(t *testing.T) { + for _, src := range []string{"## A {#a\"b}\n", "## A {#a b}\n", "## A {#}\n"} { + html, headings, err := Render(src) + if err != nil { + continue + } + for _, h := range headings { + if !explicitID.MatchString(h.ID) { + t.Errorf("Render(%q) gave the id %q", src, h.ID) + } + } + if strings.Contains(html, `id="a"b`) || strings.Contains(html, `id="a b`) { + t.Errorf("Render(%q) = %q, the id broke out of its attribute", src, html) + } + } + if _, _, err := Render("## A {#v1.2}\n\n## B {#01}\n\n## C {#a_b:c-d}\n"); err != nil { + t.Errorf("Render(valid ids) error = %v", err) + } +} + // TestRenderHeadingTextIsPlain checks that the heading text drops the markup. func TestRenderHeadingTextIsPlain(t *testing.T) { _, headings, err := Render("## Free *your* mind\n") diff --git a/internal/render/markdown.go b/internal/render/markdown.go new file mode 100644 index 0000000..abc1251 --- /dev/null +++ b/internal/render/markdown.go @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package render + +import ( + "fmt" + "strings" +) + +// SubstituteMarkdown replaces the {{name}} placeholders of Markdown text like +// Substitute does, with two exceptions that let an article show a placeholder +// as it is written: fenced code blocks are left untouched (an unknown +// placeholder in one is not an error), and a placeholder with a backslash in +// front of it is kept, backslash included, which Markdown then renders as the +// plain placeholder. Code spans are substituted: the pages use them to show a +// value in monospace. An unknown placeholder elsewhere is an error that names +// the file. +func SubstituteMarkdown(name, md string, ctx map[string]string) (string, error) { + var out strings.Builder + var text strings.Builder // the lines outside fences, waiting to be substituted + flush := func() error { + if text.Len() == 0 { + return nil + } + replaced, err := substituteUnescaped(name, text.String(), ctx) + if err != nil { + return err + } + out.WriteString(replaced) + text.Reset() + return nil + } + + lines := strings.SplitAfter(md, "\n") + for i := 0; i < len(lines); i++ { + marker, length := fenceStart(lines[i]) + if length == 0 { + text.WriteString(lines[i]) + continue + } + if err := flush(); err != nil { + return "", err + } + out.WriteString(lines[i]) + for i++; i < len(lines); i++ { + out.WriteString(lines[i]) + if fenceEnds(lines[i], marker, length) { + break + } + } + } + if err := flush(); err != nil { + return "", err + } + return out.String(), nil +} + +// fenceStart returns the fence character and its length when the line opens a +// fenced code block (three or more backticks or tildes, indented by at most +// three spaces), and a zero length otherwise. +func fenceStart(line string) (marker byte, length int) { + rest := strings.TrimLeft(line, " ") + if len(line)-len(rest) > 3 || rest == "" || (rest[0] != '`' && rest[0] != '~') { + return 0, 0 + } + marker = rest[0] + for length < len(rest) && rest[length] == marker { + length++ + } + if length < 3 { + return 0, 0 + } + return marker, length +} + +// fenceEnds reports whether the line closes a fence opened with length +// characters of marker: at least as many, and nothing else on the line. +func fenceEnds(line string, marker byte, length int) bool { + rest := strings.TrimSpace(line) + if len(rest) < length { + return false + } + return strings.Trim(rest, string(marker)) == "" +} + +// substituteUnescaped is Substitute that leaves alone the placeholders that +// have a backslash right before them. +func substituteUnescaped(name, text string, ctx map[string]string) (string, error) { + var out strings.Builder + last := 0 + for _, loc := range placeholder.FindAllStringSubmatchIndex(text, -1) { + start, end := loc[0], loc[1] + out.WriteString(text[last:start]) + last = end + if start > 0 && text[start-1] == '\\' { + out.WriteString(text[start:end]) + continue + } + key := text[loc[2]:loc[3]] + value, ok := ctx[key] + if !ok { + return "", fmt.Errorf("%s: unknown placeholder {{%s}}", name, key) + } + out.WriteString(value) + } + out.WriteString(text[last:]) + return out.String(), nil +} diff --git a/internal/render/render_test.go b/internal/render/render_test.go index 5d2100a..9e97c33 100644 --- a/internal/render/render_test.go +++ b/internal/render/render_test.go @@ -59,6 +59,43 @@ func TestSubstituteValueIsNotRescanned(t *testing.T) { } } +// TestSubstituteMarkdownFences checks that placeholders in fenced code blocks +// stay as written (an unknown one is no error there), while code spans are +// substituted: the pages use them to show a value in monospace. +func TestSubstituteMarkdownFences(t *testing.T) { + ctx := map[string]string{"lang": "en"} + src := "Outside {{lang}} and `{{lang}}`.\n\n```\n{{lang}} {{nope}}\n```\n\n~~~go\n{{lang}}\n~~~\n\nEnd {{lang}}.\n" + got, err := SubstituteMarkdown("page.md", src, ctx) + if err != nil { + t.Fatalf("SubstituteMarkdown() error = %v", err) + } + want := "Outside en and `en`.\n\n```\n{{lang}} {{nope}}\n```\n\n~~~go\n{{lang}}\n~~~\n\nEnd en.\n" + if got != want { + t.Errorf("SubstituteMarkdown() = %q, want %q", got, want) + } +} + +// TestSubstituteMarkdownEscape checks that a backslash before a placeholder +// keeps it as written: Markdown then drops the backslash. +func TestSubstituteMarkdownEscape(t *testing.T) { + got, err := SubstituteMarkdown("page.md", "Write \\{{lang}} for {{lang}}, and \\{{nope}} is fine.\n", map[string]string{"lang": "en"}) + if err != nil { + t.Fatalf("SubstituteMarkdown() error = %v", err) + } + if want := "Write \\{{lang}} for en, and \\{{nope}} is fine.\n"; got != want { + t.Errorf("SubstituteMarkdown() = %q, want %q", got, want) + } +} + +// TestSubstituteMarkdownUnknownOutsideCode checks that an unknown placeholder +// outside a fence still stops the build and names the file. +func TestSubstituteMarkdownUnknownOutsideCode(t *testing.T) { + _, err := SubstituteMarkdown("news/x.md", "ok {{a}}\n\nbad {{nope}}\n", map[string]string{"a": "1"}) + if err == nil || !strings.Contains(err.Error(), "news/x.md") || !strings.Contains(err.Error(), "nope") { + t.Errorf("SubstituteMarkdown() error = %v, want it to name news/x.md and nope", err) + } +} + // TestEscape checks that text escaping leaves quotes alone. func TestEscape(t *testing.T) { if got, want := Escape(`a & b < c > "d" 'e'`), `a & b < c > "d" 'e'`; got != want { diff --git a/internal/site/site.go b/internal/site/site.go index f610721..8a5ca1e 100644 --- a/internal/site/site.go +++ b/internal/site/site.go @@ -47,6 +47,9 @@ func Load(dir string) (Config, map[string]Lang, error) { return Config{}, nil, err } cfg := Config{Strings: stringValues(raw), Assets: map[string]string{}} + if err := checkBaseURL(cfg.Strings["base_url"]); err != nil { + return Config{}, nil, err + } if err := decodeField(raw, "site.json", "languages", &cfg.Languages); err != nil { return Config{}, nil, err } @@ -76,6 +79,15 @@ func Load(dir string) (Config, map[string]Lang, error) { return cfg, langs, nil } +// checkBaseURL returns an error unless the address of the site is absolute and +// has no trailing slash: page addresses and the feed are built by adding to it. +func checkBaseURL(base string) error { + if !(strings.HasPrefix(base, "https://") || strings.HasPrefix(base, "http://")) || strings.HasSuffix(base, "/") { + return fmt.Errorf("site.json: base_url must be an absolute address without a trailing slash, such as https://kmalloc.org") + } + return nil +} + // languageCode is the shape of a language code: "en", "pt-BR". var languageCode = regexp.MustCompile(`^[a-z]{2,3}(-[A-Za-z0-9]+)?$`) diff --git a/internal/site/site_test.go b/internal/site/site_test.go index 188662a..3ac6eba 100644 --- a/internal/site/site_test.go +++ b/internal/site/site_test.go @@ -132,7 +132,7 @@ func TestLoadRejectsEscapingAssets(t *testing.T) { `{"a.txt": "../../etc/hostname"}`, `{"a.txt": "/etc/hostname"}`, } { files := validFiles() - files["site.json"] = `{"languages": ["en", "it"], "assets": ` + assets + `}` + files["site.json"] = `{"base_url": "https://kmalloc.org", "languages": ["en", "it"], "assets": ` + assets + `}` _, _, err := Load(writeSite(t, files)) if err == nil || !strings.Contains(err.Error(), "asset") { t.Errorf("Load(%s) error = %v, want an error about the asset", assets, err) @@ -144,7 +144,7 @@ func TestLoadRejectsEscapingAssets(t *testing.T) { func TestLoadRejectsBadLanguageCodes(t *testing.T) { for _, codes := range []string{`["en", "../x"]`, `["EN"]`, `["en", "en"]`, `["en", "it/x"]`} { files := validFiles() - files["site.json"] = `{"languages": ` + codes + `}` + files["site.json"] = `{"base_url": "https://kmalloc.org", "languages": ` + codes + `}` _, _, err := Load(writeSite(t, files)) if err == nil || !strings.Contains(err.Error(), "language") { t.Errorf("Load(%s) error = %v, want an error about the language code", codes, err) @@ -152,6 +152,19 @@ func TestLoadRejectsBadLanguageCodes(t *testing.T) { } } +// TestLoadNeedsBaseURL checks that the address of the site is required and absolute, +// because the feed and the page addresses are built from it. +func TestLoadNeedsBaseURL(t *testing.T) { + for _, site := range []string{`{"languages": ["en", "it"]}`, `{"base_url": "kmalloc.org", "languages": ["en", "it"]}`, `{"base_url": "https://kmalloc.org/", "languages": ["en", "it"]}`} { + files := validFiles() + files["site.json"] = site + _, _, err := Load(writeSite(t, files)) + if err == nil || !strings.Contains(err.Error(), "base_url") { + t.Errorf("Load(%s) error = %v, want an error about base_url", site, err) + } + } +} + // TestLoadNeedsLanguages checks that an empty language list is an error. func TestLoadNeedsLanguages(t *testing.T) { files := validFiles()