From 917683afa491c7f0b4331757d6066d215b5f19d9 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Sun, 15 Nov 2026 10:32:19 +0100 Subject: [PATCH 1/2] fix: close the minor findings of the first review Require refuses an empty value. Heading ids: an explicit {#id} must be sound and unique, generated ids avoid the explicit ones and each other, and a heading with no ASCII letters gets a fallback. Fenced code blocks keep their placeholders and a backslash keeps one anywhere. The em dash rule covers the header of the manifesto pages and the HTML entities. The feed makes src links absolute, base_url is required, and the news page must be a folder index. A category written twice in a post counts once, a block name in a news title is no longer taken for a placeholder, and build refuses extra arguments. --- README.md | 2 +- cmd/motorino/main.go | 5 ++ cmd/motorino/main_test.go | 16 ++++- internal/build/build.go | 8 ++- internal/build/build_test.go | 49 +++++++++++++++ internal/build/feed.go | 2 +- internal/build/pages.go | 35 +++++++---- internal/content/pages.go | 8 +++ internal/content/pages_test.go | 15 +++++ internal/content/posts.go | 8 ++- internal/content/posts_test.go | 12 ++++ internal/header/header.go | 6 +- internal/header/header_test.go | 15 +++++ internal/mdx/emdash.go | 11 +++- internal/mdx/emdash_test.go | 21 +++++++ internal/mdx/mdx.go | 61 ++++++++++++++++--- internal/mdx/mdx_test.go | 78 ++++++++++++++++++++++++ internal/render/markdown.go | 108 +++++++++++++++++++++++++++++++++ internal/render/render_test.go | 37 +++++++++++ internal/site/site.go | 12 ++++ internal/site/site_test.go | 17 +++++- 21 files changed, 493 insertions(+), 33 deletions(-) create mode 100644 internal/render/markdown.go diff --git a/README.md b/README.md index 81c0168..9ad28fa 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ Pages: `Title`, `ID` (the same in every language, it links the translations), `D ### Placeholders -`{{name}}`, with lowercase letters, digits and underscores, is replaced by a string from `site.json` or the language file. An unknown placeholder stops the build and names the file. `{{news_list}}`, `{{latest_news}}` and `{{categories_list}}` stand alone in a paragraph and expand to generated lists. +`{{name}}`, with lowercase letters, digits and underscores, is replaced by a string from `site.json` or the language file. An unknown placeholder stops the build and names the file. Fenced code blocks are left as written, so an article can show a placeholder there; elsewhere a backslash before the placeholder (`\{{name}}`) keeps it as written, and Markdown drops the backslash. Code spans are substituted, which is how a page shows a value in monospace. `{{news_list}}`, `{{latest_news}}` and `{{categories_list}}` stand alone in a paragraph and expand to generated lists. ## Development diff --git a/cmd/motorino/main.go b/cmd/motorino/main.go index ce40c01..a1997f1 100644 --- a/cmd/motorino/main.go +++ b/cmd/motorino/main.go @@ -15,6 +15,7 @@ import ( "fmt" "io" "os" + "strings" "git.kmalloc.org/squid/motorino/internal/build" "git.kmalloc.org/squid/motorino/internal/version" @@ -59,6 +60,10 @@ func runBuild(args []string, stdout, stderr io.Writer) int { } return 2 } + if flags.NArg() > 0 { + fmt.Fprintf(stderr, "motorino: unexpected arguments: %s\n%s\n", strings.Join(flags.Args(), " "), usage) + return 2 + } result, err := build.Build(build.Options{SiteDir: *site, OutDir: *out, Version: version.String()}) if err != nil { fmt.Fprintf(stderr, "motorino: %v\n", err) diff --git a/cmd/motorino/main_test.go b/cmd/motorino/main_test.go index 8e2ffd6..4090294 100644 --- a/cmd/motorino/main_test.go +++ b/cmd/motorino/main_test.go @@ -30,8 +30,11 @@ func miniSite(t *testing.T) string { } for _, id := range []string{"home", "manifesto", "people", "services", "news", "history"} { name := id - if id == "home" { + switch id { + case "home": name = "index" + case "news": + name = "news/index" } files["content/en/"+name+".md"] = "Title: " + id + "\nID: " + id + "\nDescription: d\n-\nBody of " + id + ".\n" } @@ -101,6 +104,17 @@ func TestRunUnknownCommand(t *testing.T) { } } +// TestRunBuildRejectsExtraArguments checks that arguments that are not flags are not ignored. +func TestRunBuildRejectsExtraArguments(t *testing.T) { + var out, errOut bytes.Buffer + if code := run([]string{"build", "somesite", "someout"}, &out, &errOut); code != 2 { + t.Errorf("run(build somesite someout) = %d, want 2", code) + } + if !strings.Contains(errOut.String(), "unexpected") { + t.Errorf("stderr = %q, want it to say what was unexpected", errOut.String()) + } +} + // TestRunBuildBadFlag checks that an unknown flag is a usage error. func TestRunBuildBadFlag(t *testing.T) { var out, errOut bytes.Buffer diff --git a/internal/build/build.go b/internal/build/build.go index 7975c88..a6d2ea3 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -76,7 +76,10 @@ type Result struct { } // Build reads the site in opts.SiteDir and writes it to opts.OutDir. It renders -// everything in memory first, so a failure leaves the output folder untouched. +// everything in memory first, so a failure in the content leaves the output +// folder untouched. Writing the files is not atomic, and files of an earlier +// build that are no longer produced are not removed: build into a fresh folder +// when that matters, as the deploy does. func Build(opts Options) (Result, error) { b, err := load(opts) if err != nil { @@ -137,6 +140,9 @@ func load(opts Options) (*builder, error) { return nil, fmt.Errorf("content/%s: there is no page with ID %q", b.defaultLang, id) } } + if news := b.byKey[pageKey{"news", b.defaultLang}]; !strings.HasSuffix(news.Path, "/") { + return nil, fmt.Errorf("%s: the news page must be an index (news/index.md), so that the feed has an address next to it", news.Source) + } if b.templates, err = readTemplates(opts.SiteDir); err != nil { return nil, err } diff --git a/internal/build/build_test.go b/internal/build/build_test.go index b46db52..9d88ad0 100644 --- a/internal/build/build_test.go +++ b/internal/build/build_test.go @@ -358,6 +358,55 @@ func TestBuildRejectsSwitchToUnknownLanguage(t *testing.T) { } } +// TestBuildFeedMakesImagesAbsolute checks that src links get the full address like href links. +func TestBuildFeedMakesImagesAbsolute(t *testing.T) { + files := baseFiles() + files["news/pic.md"] = "Title: Pic\nDate: 2026-11-15T12:00:00+0100\n-\n![the logo](/logo.svg)\n" + out, err := runBuild(t, files) + if err != nil { + t.Fatal(err) + } + mustContain(t, "feed", readOut(t, out, "news/feed.xml"), `src="https://kmalloc.org/logo.svg"`) +} + +// TestBuildNewsPageMustBeAFolderIndex checks that the feed has a sound address. +func TestBuildNewsPageMustBeAFolderIndex(t *testing.T) { + files := baseFiles() + delete(files, "content/en/news/index.md") + files["content/en/news.md"] = page("news", "News", "", "{{news_list}}\n") + _, err := runBuild(t, files) + if err == nil || !strings.Contains(err.Error(), "content/en/news.md") || !strings.Contains(err.Error(), "index") { + t.Errorf("Build() error = %v, want it to say that content/en/news.md must be an index", err) + } +} + +// TestBuildBlockNameInATitleIsNotAnError checks that a generated list that shows +// the text of a block placeholder (in a news title) is not mistaken for a +// placeholder that is not alone in its paragraph. +func TestBuildBlockNameInATitleIsNotAnError(t *testing.T) { + files := baseFiles() + files["news/tpl.md"] = "Title: Use {{latest_news}} in a page\nDate: 2026-11-15T12:00:00+0100\n-\nA note.\n" + out, err := runBuild(t, files) + if err != nil { + t.Fatalf("Build() error = %v", err) + } + mustContain(t, "home", readOut(t, out, "index.html"), "Use {{latest_news}} in a page") +} + +// TestBuildPlaceholdersAndCode checks the three ways a page can use a placeholder +// next to code: a value in a code span (the pages show the GPG fingerprint that +// way), a fenced block that shows a placeholder as written, and an escaped one. +func TestBuildPlaceholdersAndCode(t *testing.T) { + files := baseFiles() + files["content/en/people.md"] = page("people", "People", "", "Value `{{lang}}` and escaped \\{{lang}}.\n\n```\n{{lang}} {{nope}}\n```\n\nAnd {{lang}}.\n") + out, err := runBuild(t, files) + if err != nil { + t.Fatalf("Build() error = %v", err) + } + got := readOut(t, out, "people.html") + mustContain(t, "people", got, "en", "escaped {{lang}}.", "{{lang}} {{nope}}", "And en.") +} + // TestBuildAssetsCopied checks the assets listed in site.json. func TestBuildAssetsCopied(t *testing.T) { out, err := runBuild(t, baseFiles()) diff --git a/internal/build/feed.go b/internal/build/feed.go index 485cc8a..719bd6e 100644 --- a/internal/build/feed.go +++ b/internal/build/feed.go @@ -21,7 +21,7 @@ func feed(base string, l site.Lang, newsURL string, posts []renderedPost, limit var items strings.Builder for _, p := range posts { link := base + p.Path - text := strings.ReplaceAll(p.HTML, `href="/`, `href="`+base+`/`) + text := strings.NewReplacer(`href="/`, `href="`+base+`/`, `src="/`, `src="`+base+`/`).Replace(p.HTML) items.WriteString("\n") items.WriteString("" + render.Escape(p.Title) + "\n") items.WriteString("" + link + "\n") diff --git a/internal/build/pages.go b/internal/build/pages.go index 6de74b6..c873d09 100644 --- a/internal/build/pages.go +++ b/internal/build/pages.go @@ -4,6 +4,7 @@ package build import ( "fmt" + "regexp" "strings" "git.kmalloc.org/squid/motorino/internal/content" @@ -61,7 +62,7 @@ func (b *builder) renderPage(p content.Page) error { var toc []string for _, h := range headings { if h.Level == 2 { - toc = append(toc, fmt.Sprintf(`
  • %s
  • `, h.ID, render.Escape(h.Text))) + toc = append(toc, fmt.Sprintf(`
  • %s
  • `, render.EscapeAttr(h.ID), render.Escape(h.Text))) } } ctx["manifesto_toc"] = strings.Join(toc, "\n") @@ -80,7 +81,7 @@ func (b *builder) renderPage(p content.Page) error { // markdown replaces the placeholders of a source, converts it to HTML and // expands the generated lists. The name is used in error messages. func (b *builder) markdown(name, source, lang string, ctx map[string]string) (string, []mdx.Heading, error) { - text, err := render.Substitute(name, source, ctx) + text, err := render.SubstituteMarkdown(name, source, ctx) if err != nil { return "", nil, err } @@ -92,22 +93,32 @@ func (b *builder) markdown(name, source, lang string, ctx map[string]string) (st return html, headings, err } +// codeElement matches a code element, where a placeholder is shown as written. +var codeElement = regexp.MustCompile(`(?s)]*>.*?`) + // expandBlocks replaces each paragraph that holds only a block placeholder by -// the generated HTML. A block placeholder inside other text is an error. +// the generated HTML. A block placeholder inside other text is an error. All +// blocks are checked before any is expanded, so the text of a generated list +// (a news title may say {{latest_news}}) is never taken for a placeholder. func (b *builder) expandBlocks(name, html, lang string, ctx map[string]string) (string, error) { + outside := codeElement.ReplaceAllString(html, "") for _, block := range blockNames { - marker := "

    {{" + block + "}}

    " - if strings.Contains(html, marker) { - generated, err := b.block(block, lang, ctx) - if err != nil { - return "", err - } - html = strings.ReplaceAll(html, marker, generated) - } - if strings.Contains(html, "{{"+block+"}}") { + token := "{{" + block + "}}" + if strings.Count(outside, token) != strings.Count(outside, "

    "+token+"

    ") { return "", fmt.Errorf("%s: {{%s}} must stand alone in a paragraph", name, block) } } + for _, block := range blockNames { + marker := "

    {{" + block + "}}

    " + if !strings.Contains(html, marker) { + continue + } + generated, err := b.block(block, lang, ctx) + if err != nil { + return "", err + } + html = strings.ReplaceAll(html, marker, generated) + } return html, nil } diff --git a/internal/content/pages.go b/internal/content/pages.go index 3c6911b..696dfe3 100644 --- a/internal/content/pages.go +++ b/internal/content/pages.go @@ -142,6 +142,14 @@ func loadPage(file, source, defaultLang, lang, rel string) (Page, error) { if err := mdx.RejectEmDash(source, string(data)); err != nil { return Page{}, err } + } else { + // The body of the manifesto is the owner's text and is rewritten when the + // page is built; the keys of the header are not. + for _, key := range []string{"Title", "Description", "Note"} { + if err := mdx.RejectEmDash(source+" (key "+key+")", h.Get(key)); err != nil { + return Page{}, err + } + } } switches, err := parseSwitch(source, h.Get("Switch")) if err != nil { diff --git a/internal/content/pages_test.go b/internal/content/pages_test.go index a42e2f9..91881fc 100644 --- a/internal/content/pages_test.go +++ b/internal/content/pages_test.go @@ -107,6 +107,21 @@ func TestLoadPagesManifestoKeepsEmDash(t *testing.T) { } } +// TestLoadPagesManifestoHeaderEmDash checks that the exemption of the manifesto +// body does not cover the title, the description and the note. +func TestLoadPagesManifestoHeaderEmDash(t *testing.T) { + for _, field := range []string{"Title", "Description", "Note"} { + head := map[string]string{"Title": "M", "Description": "d", "Note": "n"} + head[field] = "words \u2014 more" + file := "Title: " + head["Title"] + "\nID: manifesto\nDescription: " + head["Description"] + "\nLayout: manifesto\nNote: " + head["Note"] + "\n-\n## 01 \u2014 Body\n" + dir := writeFiles(t, map[string]string{"content/en/m.md": file}) + _, err := LoadPages(dir, []string{"en"}) + if err == nil || !strings.Contains(err.Error(), "content/en/m.md") || !strings.Contains(err.Error(), field) { + t.Errorf("%s with an em dash: LoadPages() error = %v, want it to name content/en/m.md and %s", field, err, field) + } + } +} + // TestLoadPagesDuplicateID checks that two pages cannot share an ID in a language. func TestLoadPagesDuplicateID(t *testing.T) { dir := writeFiles(t, map[string]string{ diff --git a/internal/content/posts.go b/internal/content/posts.go index a51c408..ae5f103 100644 --- a/internal/content/posts.go +++ b/internal/content/posts.go @@ -136,6 +136,7 @@ func parseDate(source, value string) (time.Time, error) { // parseCategories splits a comma separated list into categories. func parseCategories(source, value string) ([]Category, error) { var out []Category + seen := map[string]bool{} for _, name := range strings.Split(value, ",") { name = strings.TrimSpace(name) if name == "" { @@ -145,6 +146,10 @@ func parseCategories(source, value string) ([]Category, error) { if err != nil { return nil, fmt.Errorf("%s: %w", source, err) } + if seen[slug] { + continue + } + seen[slug] = true out = append(out, Category{Name: name, Slug: slug}) } return out, nil @@ -177,7 +182,8 @@ func checkCategories(posts []Post) error { return nil } -// markup matches the Markdown that is dropped from a fallback description. +// markdownLink, markdownMark and sentenceEnd pick the first sentence out of a +// body and drop its Markdown markup, for a description that was not written. var ( markdownLink = regexp.MustCompile(`\[([^\]]*)\]\([^)]*\)`) markdownMark = regexp.MustCompile("[*_`]") diff --git a/internal/content/posts_test.go b/internal/content/posts_test.go index 3905178..c3dfda2 100644 --- a/internal/content/posts_test.go +++ b/internal/content/posts_test.go @@ -171,6 +171,18 @@ func TestLoadPostsCategoryCollision(t *testing.T) { } } +// TestLoadPostsRepeatedCategory checks that a category written twice counts once. +func TestLoadPostsRepeatedCategory(t *testing.T) { + dir := writeFiles(t, map[string]string{"news/a.md": "Title: A\nDate: 2026-01-01T10:00:00+0100\nCategories: News, Linux, news, News\n-\nx\n"}) + posts, err := LoadPosts(dir) + if err != nil { + t.Fatal(err) + } + if len(posts[0].Categories) != 2 || posts[0].Categories[0].Name != "News" || posts[0].Categories[1].Name != "Linux" { + t.Errorf("Categories = %+v, want News and Linux once each", posts[0].Categories) + } +} + // TestLoadPostsNoNewsFolder checks that a site without news is fine. func TestLoadPostsNoNewsFolder(t *testing.T) { posts, err := LoadPosts(t.TempDir()) diff --git a/internal/header/header.go b/internal/header/header.go index 9b8d6aa..78fa8ac 100644 --- a/internal/header/header.go +++ b/internal/header/header.go @@ -78,12 +78,16 @@ func (h Header) Has(key string) bool { return ok } -// Require returns an error naming name and the first missing key of keys. +// Require returns an error naming name and the first key of keys that is +// missing or has an empty value. func (h Header) Require(name string, keys ...string) error { for _, k := range keys { if !h.Has(k) { return fmt.Errorf("%s: missing key %q", name, k) } + if h.Get(k) == "" { + return fmt.Errorf("%s: the key %q is empty", name, k) + } } return nil } diff --git a/internal/header/header_test.go b/internal/header/header_test.go index 30758f6..9e38835 100644 --- a/internal/header/header_test.go +++ b/internal/header/header_test.go @@ -124,6 +124,21 @@ func TestRequire(t *testing.T) { } } +// TestRequireNeedsAValue checks that a key with an empty value counts as missing. +func TestRequireNeedsAValue(t *testing.T) { + h, _, err := Parse("file.md", []byte("Title:\nID: x\n-\n")) + if err != nil { + t.Fatal(err) + } + err = h.Require("file.md", "Title") + if err == nil || !strings.Contains(err.Error(), "Title") || !strings.Contains(err.Error(), "empty") { + t.Errorf("Require(Title) = %v, want an error saying that Title is empty", err) + } + if err := h.Require("file.md", "ID"); err != nil { + t.Errorf("Require(ID) = %v", err) + } +} + // TestOnly checks that a key outside the allowed list is rejected. func TestOnly(t *testing.T) { h, _, err := Parse("file.md", []byte("Title: T\nTitolo: T\n-\n")) diff --git a/internal/mdx/emdash.go b/internal/mdx/emdash.go index e6e74fa..932f980 100644 --- a/internal/mdx/emdash.go +++ b/internal/mdx/emdash.go @@ -38,17 +38,22 @@ func NoEmDash(md string) (string, error) { for _, p := range parentheticals { md = strings.ReplaceAll(md, p.from, p.to) } - if err := RejectEmDash("manifesto source", md); err != nil { + if err := RejectEmDash("the body", md); err != nil { return "", err } return md, nil } +// emDashEntity matches the HTML entities of the em dash, which would put it +// on the page as well. +var emDashEntity = regexp.MustCompile(`(?i)—|—|—`) + // RejectEmDash returns an error naming the file and the line of the first em -// dash in text, or nil when there is none. +// dash in text, written as the character or as an HTML entity, or nil when +// there is none. func RejectEmDash(name, text string) error { for i, line := range strings.Split(text, "\n") { - if strings.Contains(line, emDash) { + if strings.Contains(line, emDash) || emDashEntity.MatchString(line) { return fmt.Errorf("%s:%d: no em dashes, please", name, i+1) } } diff --git a/internal/mdx/emdash_test.go b/internal/mdx/emdash_test.go index 0682d9f..8e13178 100644 --- a/internal/mdx/emdash_test.go +++ b/internal/mdx/emdash_test.go @@ -45,6 +45,27 @@ func TestNoEmDashLeftover(t *testing.T) { } } +// TestRejectEmDashEntities checks that the entities of the em dash are refused too. +func TestRejectEmDashEntities(t *testing.T) { + for _, in := range []string{"a — b", "a &MDASH; b", "a — b", "a — b", "a — b"} { + if err := RejectEmDash("f.md", in); err == nil { + t.Errorf("RejectEmDash(%q) = nil, want an error", in) + } + } + if err := RejectEmDash("f.md", "a – b & c"); err != nil { + t.Errorf("RejectEmDash(ndash) = %v", err) + } +} + +// TestNoEmDashErrorSaysBody checks that a leftover is reported as a line of the +// body, which is how the caller counts it. +func TestNoEmDashErrorSaysBody(t *testing.T) { + _, err := NoEmDash("fine\nsome words \u2014 and then more\n") + if err == nil || !strings.Contains(err.Error(), "the body:2") { + t.Errorf("NoEmDash() error = %v, want it to say the body:2", err) + } +} + // TestRejectEmDash checks that the error names the file and the line. func TestRejectEmDash(t *testing.T) { if err := RejectEmDash("content/en/x.md", "clean\ntext\n"); err != nil { diff --git a/internal/mdx/mdx.go b/internal/mdx/mdx.go index f79f644..4eb323b 100644 --- a/internal/mdx/mdx.go +++ b/internal/mdx/mdx.go @@ -6,6 +6,7 @@ package mdx import ( "bytes" "fmt" + "regexp" "strconv" "strings" @@ -42,9 +43,12 @@ func Render(md string) (string, []Heading, error) { src := []byte(md) doc := engine.Parser().Parse(text.NewReader(src)) + used, err := explicitIDs(doc) + if err != nil { + return "", nil, err + } var headings []Heading - used := make(map[string]int) - err := ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) { + err = ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) { h, ok := n.(*ast.Heading) if !ok || !entering { return ast.WalkContinue, nil @@ -53,9 +57,12 @@ func Render(md string) (string, []Heading, error) { id := "" if explicit, ok := h.AttributeString("id"); ok { id = attributeText(explicit) - used[id]++ } else if h.Level >= 2 { - id = uniqueID(Slug(plain), used) + base := Slug(plain) + if base == "" { + base = "section" + } + id = uniqueID(base, used) h.SetAttributeString("id", []byte(id)) } headings = append(headings, Heading{Level: h.Level, ID: id, Text: plain}) @@ -83,13 +90,47 @@ func attributeText(value any) string { return "" } -// uniqueID returns id, or id with a numeric suffix when it was already used. -func uniqueID(id string, used map[string]int) string { - used[id]++ - if used[id] == 1 { - return id +// explicitID is what an id set with {#id} may look like: it ends up in +// attributes and in addresses. +var explicitID = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.:-]*$`) + +// explicitIDs returns the ids that headings set for themselves with {#id}. A +// bad id or an id used by two headings is an error. Generated ids avoid these, +// whatever the order of the headings. +func explicitIDs(doc ast.Node) (map[string]bool, error) { + used := map[string]bool{} + err := ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) { + h, ok := n.(*ast.Heading) + if !ok || !entering { + return ast.WalkContinue, nil + } + value, ok := h.AttributeString("id") + if !ok { + return ast.WalkSkipChildren, nil + } + id := attributeText(value) + if !explicitID.MatchString(id) { + return ast.WalkStop, fmt.Errorf("the heading id %q may have only letters, digits and _ . : -", id) + } + if used[id] { + return ast.WalkStop, fmt.Errorf("the heading id %q is used by two headings", id) + } + used[id] = true + return ast.WalkSkipChildren, nil + }) + return used, err +} + +// uniqueID returns base, or base with -2, -3 and so on until the id is not +// used, and marks it as used. +func uniqueID(base string, used map[string]bool) string { + id, n := base, 1 + for used[id] { + n++ + id = base + "-" + strconv.Itoa(n) } - return id + "-" + strconv.Itoa(used[id]) + used[id] = true + return id } // plainText returns the text of a node and its children without any markup. diff --git a/internal/mdx/mdx_test.go b/internal/mdx/mdx_test.go index 3173977..c576ca4 100644 --- a/internal/mdx/mdx_test.go +++ b/internal/mdx/mdx_test.go @@ -99,6 +99,84 @@ func TestRenderExplicitHeadingID(t *testing.T) { } } +// TestRenderRejectsRepeatedExplicitID checks that two headings cannot ask for the same id. +func TestRenderRejectsRepeatedExplicitID(t *testing.T) { + for _, src := range []string{"## Foo {#a}\n\n## Bar {#a}\n"} { + _, _, err := Render(src) + if err == nil || !strings.Contains(err.Error(), "id") { + t.Errorf("Render(%q) error = %v, want an error about the repeated id", src, err) + } + } +} + +// TestRenderGeneratedIDAvoidsLaterExplicit checks that a generated id never takes +// one that a later heading asks for by name, in either order of the two. +func TestRenderGeneratedIDAvoidsLaterExplicit(t *testing.T) { + _, headings, err := Render("## Foo\n\n## Bar {#foo-2}\n\n## Foo\n") + if err != nil { + t.Fatal(err) + } + var ids []string + for _, h := range headings { + ids = append(ids, h.ID) + } + if got, want := strings.Join(ids, ","), "foo,foo-2,foo-3"; got != want { + t.Errorf("ids = %s, want %s", got, want) + } +} + +// TestRenderSuffixClash checks that a numbered suffix never repeats an id. +func TestRenderSuffixClash(t *testing.T) { + _, headings, err := Render("## A\n\n## A\n\n## A 2\n") + if err != nil { + t.Fatal(err) + } + seen := map[string]bool{} + for _, h := range headings { + if seen[h.ID] { + t.Errorf("id %q is used twice: %+v", h.ID, headings) + } + seen[h.ID] = true + } +} + +// TestRenderEmptySlugFallback checks that a heading with no ASCII letters or digits still gets an id. +func TestRenderEmptySlugFallback(t *testing.T) { + html, headings, err := Render("## \u4f60\u597d\n\n## \u4e16\u754c\n") + if err != nil { + t.Fatal(err) + } + if strings.Contains(html, `id=""`) { + t.Errorf("html %q has an empty id", html) + } + if len(headings) != 2 || headings[0].ID != "section" || headings[1].ID != "section-2" { + t.Errorf("headings = %+v, want section and section-2", headings) + } +} + +// TestRenderBadExplicitIDCannotInject checks that a malformed {#id} never ends +// up as an id with quotes, spaces or angle brackets, whether goldmark leaves it +// as text or the id check refuses it; and that sound ids pass. +func TestRenderBadExplicitIDCannotInject(t *testing.T) { + for _, src := range []string{"## A {#a\"b}\n", "## A {#a b}\n", "## A {#}\n"} { + html, headings, err := Render(src) + if err != nil { + continue + } + for _, h := range headings { + if !explicitID.MatchString(h.ID) { + t.Errorf("Render(%q) gave the id %q", src, h.ID) + } + } + if strings.Contains(html, `id="a"b`) || strings.Contains(html, `id="a b`) { + t.Errorf("Render(%q) = %q, the id broke out of its attribute", src, html) + } + } + if _, _, err := Render("## A {#v1.2}\n\n## B {#01}\n\n## C {#a_b:c-d}\n"); err != nil { + t.Errorf("Render(valid ids) error = %v", err) + } +} + // TestRenderHeadingTextIsPlain checks that the heading text drops the markup. func TestRenderHeadingTextIsPlain(t *testing.T) { _, headings, err := Render("## Free *your* mind\n") diff --git a/internal/render/markdown.go b/internal/render/markdown.go new file mode 100644 index 0000000..abc1251 --- /dev/null +++ b/internal/render/markdown.go @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package render + +import ( + "fmt" + "strings" +) + +// SubstituteMarkdown replaces the {{name}} placeholders of Markdown text like +// Substitute does, with two exceptions that let an article show a placeholder +// as it is written: fenced code blocks are left untouched (an unknown +// placeholder in one is not an error), and a placeholder with a backslash in +// front of it is kept, backslash included, which Markdown then renders as the +// plain placeholder. Code spans are substituted: the pages use them to show a +// value in monospace. An unknown placeholder elsewhere is an error that names +// the file. +func SubstituteMarkdown(name, md string, ctx map[string]string) (string, error) { + var out strings.Builder + var text strings.Builder // the lines outside fences, waiting to be substituted + flush := func() error { + if text.Len() == 0 { + return nil + } + replaced, err := substituteUnescaped(name, text.String(), ctx) + if err != nil { + return err + } + out.WriteString(replaced) + text.Reset() + return nil + } + + lines := strings.SplitAfter(md, "\n") + for i := 0; i < len(lines); i++ { + marker, length := fenceStart(lines[i]) + if length == 0 { + text.WriteString(lines[i]) + continue + } + if err := flush(); err != nil { + return "", err + } + out.WriteString(lines[i]) + for i++; i < len(lines); i++ { + out.WriteString(lines[i]) + if fenceEnds(lines[i], marker, length) { + break + } + } + } + if err := flush(); err != nil { + return "", err + } + return out.String(), nil +} + +// fenceStart returns the fence character and its length when the line opens a +// fenced code block (three or more backticks or tildes, indented by at most +// three spaces), and a zero length otherwise. +func fenceStart(line string) (marker byte, length int) { + rest := strings.TrimLeft(line, " ") + if len(line)-len(rest) > 3 || rest == "" || (rest[0] != '`' && rest[0] != '~') { + return 0, 0 + } + marker = rest[0] + for length < len(rest) && rest[length] == marker { + length++ + } + if length < 3 { + return 0, 0 + } + return marker, length +} + +// fenceEnds reports whether the line closes a fence opened with length +// characters of marker: at least as many, and nothing else on the line. +func fenceEnds(line string, marker byte, length int) bool { + rest := strings.TrimSpace(line) + if len(rest) < length { + return false + } + return strings.Trim(rest, string(marker)) == "" +} + +// substituteUnescaped is Substitute that leaves alone the placeholders that +// have a backslash right before them. +func substituteUnescaped(name, text string, ctx map[string]string) (string, error) { + var out strings.Builder + last := 0 + for _, loc := range placeholder.FindAllStringSubmatchIndex(text, -1) { + start, end := loc[0], loc[1] + out.WriteString(text[last:start]) + last = end + if start > 0 && text[start-1] == '\\' { + out.WriteString(text[start:end]) + continue + } + key := text[loc[2]:loc[3]] + value, ok := ctx[key] + if !ok { + return "", fmt.Errorf("%s: unknown placeholder {{%s}}", name, key) + } + out.WriteString(value) + } + out.WriteString(text[last:]) + return out.String(), nil +} diff --git a/internal/render/render_test.go b/internal/render/render_test.go index 5d2100a..9e97c33 100644 --- a/internal/render/render_test.go +++ b/internal/render/render_test.go @@ -59,6 +59,43 @@ func TestSubstituteValueIsNotRescanned(t *testing.T) { } } +// TestSubstituteMarkdownFences checks that placeholders in fenced code blocks +// stay as written (an unknown one is no error there), while code spans are +// substituted: the pages use them to show a value in monospace. +func TestSubstituteMarkdownFences(t *testing.T) { + ctx := map[string]string{"lang": "en"} + src := "Outside {{lang}} and `{{lang}}`.\n\n```\n{{lang}} {{nope}}\n```\n\n~~~go\n{{lang}}\n~~~\n\nEnd {{lang}}.\n" + got, err := SubstituteMarkdown("page.md", src, ctx) + if err != nil { + t.Fatalf("SubstituteMarkdown() error = %v", err) + } + want := "Outside en and `en`.\n\n```\n{{lang}} {{nope}}\n```\n\n~~~go\n{{lang}}\n~~~\n\nEnd en.\n" + if got != want { + t.Errorf("SubstituteMarkdown() = %q, want %q", got, want) + } +} + +// TestSubstituteMarkdownEscape checks that a backslash before a placeholder +// keeps it as written: Markdown then drops the backslash. +func TestSubstituteMarkdownEscape(t *testing.T) { + got, err := SubstituteMarkdown("page.md", "Write \\{{lang}} for {{lang}}, and \\{{nope}} is fine.\n", map[string]string{"lang": "en"}) + if err != nil { + t.Fatalf("SubstituteMarkdown() error = %v", err) + } + if want := "Write \\{{lang}} for en, and \\{{nope}} is fine.\n"; got != want { + t.Errorf("SubstituteMarkdown() = %q, want %q", got, want) + } +} + +// TestSubstituteMarkdownUnknownOutsideCode checks that an unknown placeholder +// outside a fence still stops the build and names the file. +func TestSubstituteMarkdownUnknownOutsideCode(t *testing.T) { + _, err := SubstituteMarkdown("news/x.md", "ok {{a}}\n\nbad {{nope}}\n", map[string]string{"a": "1"}) + if err == nil || !strings.Contains(err.Error(), "news/x.md") || !strings.Contains(err.Error(), "nope") { + t.Errorf("SubstituteMarkdown() error = %v, want it to name news/x.md and nope", err) + } +} + // TestEscape checks that text escaping leaves quotes alone. func TestEscape(t *testing.T) { if got, want := Escape(`a & b < c > "d" 'e'`), `a & b < c > "d" 'e'`; got != want { diff --git a/internal/site/site.go b/internal/site/site.go index f610721..8a5ca1e 100644 --- a/internal/site/site.go +++ b/internal/site/site.go @@ -47,6 +47,9 @@ func Load(dir string) (Config, map[string]Lang, error) { return Config{}, nil, err } cfg := Config{Strings: stringValues(raw), Assets: map[string]string{}} + if err := checkBaseURL(cfg.Strings["base_url"]); err != nil { + return Config{}, nil, err + } if err := decodeField(raw, "site.json", "languages", &cfg.Languages); err != nil { return Config{}, nil, err } @@ -76,6 +79,15 @@ func Load(dir string) (Config, map[string]Lang, error) { return cfg, langs, nil } +// checkBaseURL returns an error unless the address of the site is absolute and +// has no trailing slash: page addresses and the feed are built by adding to it. +func checkBaseURL(base string) error { + if !(strings.HasPrefix(base, "https://") || strings.HasPrefix(base, "http://")) || strings.HasSuffix(base, "/") { + return fmt.Errorf("site.json: base_url must be an absolute address without a trailing slash, such as https://kmalloc.org") + } + return nil +} + // languageCode is the shape of a language code: "en", "pt-BR". var languageCode = regexp.MustCompile(`^[a-z]{2,3}(-[A-Za-z0-9]+)?$`) diff --git a/internal/site/site_test.go b/internal/site/site_test.go index 188662a..3ac6eba 100644 --- a/internal/site/site_test.go +++ b/internal/site/site_test.go @@ -132,7 +132,7 @@ func TestLoadRejectsEscapingAssets(t *testing.T) { `{"a.txt": "../../etc/hostname"}`, `{"a.txt": "/etc/hostname"}`, } { files := validFiles() - files["site.json"] = `{"languages": ["en", "it"], "assets": ` + assets + `}` + files["site.json"] = `{"base_url": "https://kmalloc.org", "languages": ["en", "it"], "assets": ` + assets + `}` _, _, err := Load(writeSite(t, files)) if err == nil || !strings.Contains(err.Error(), "asset") { t.Errorf("Load(%s) error = %v, want an error about the asset", assets, err) @@ -144,7 +144,7 @@ func TestLoadRejectsEscapingAssets(t *testing.T) { func TestLoadRejectsBadLanguageCodes(t *testing.T) { for _, codes := range []string{`["en", "../x"]`, `["EN"]`, `["en", "en"]`, `["en", "it/x"]`} { files := validFiles() - files["site.json"] = `{"languages": ` + codes + `}` + files["site.json"] = `{"base_url": "https://kmalloc.org", "languages": ` + codes + `}` _, _, err := Load(writeSite(t, files)) if err == nil || !strings.Contains(err.Error(), "language") { t.Errorf("Load(%s) error = %v, want an error about the language code", codes, err) @@ -152,6 +152,19 @@ func TestLoadRejectsBadLanguageCodes(t *testing.T) { } } +// TestLoadNeedsBaseURL checks that the address of the site is required and absolute, +// because the feed and the page addresses are built from it. +func TestLoadNeedsBaseURL(t *testing.T) { + for _, site := range []string{`{"languages": ["en", "it"]}`, `{"base_url": "kmalloc.org", "languages": ["en", "it"]}`, `{"base_url": "https://kmalloc.org/", "languages": ["en", "it"]}`} { + files := validFiles() + files["site.json"] = site + _, _, err := Load(writeSite(t, files)) + if err == nil || !strings.Contains(err.Error(), "base_url") { + t.Errorf("Load(%s) error = %v, want an error about base_url", site, err) + } + } +} + // TestLoadNeedsLanguages checks that an empty language list is an error. func TestLoadNeedsLanguages(t *testing.T) { files := validFiles() -- 2.54.0 From 4ab8f8d12223fca0db20ec5afee92217533d058b Mon Sep 17 00:00:00 2001 From: tgiachi Date: Sun, 15 Nov 2026 10:38:56 +0100 Subject: [PATCH 2/2] fix: find code blocks with the Markdown parser and make the escape work everywhere The hand written fence scanner took a line that starts with a code span for a fence and missed fences in quotes and lists. The code blocks are now the ones goldmark finds. A backslash before a placeholder is turned into a marker that survives the Markdown and the list expansion, so it works in a code span and next to a block name, and two backslashes are a literal one. The em dash rule reports the line of the body, refuses padded entities and checks the header keys by name, the feed leaves protocol-relative links alone, base_url needs a host and a language file cannot set it. --- README.md | 2 +- internal/build/build_test.go | 17 ++++++ internal/build/feed.go | 7 ++- internal/build/feed_test.go | 10 +++ internal/build/pages.go | 11 +++- internal/content/pages.go | 4 +- internal/mdx/emdash.go | 16 +++-- internal/mdx/emdash_test.go | 15 ++++- internal/mdx/mdx_test.go | 64 +++++++++++++++++++ internal/mdx/substitute.go | 58 ++++++++++++++++++ internal/render/escape.go | 59 ++++++++++++++++++ internal/render/markdown.go | 108 --------------------------------- internal/render/render_test.go | 58 +++++++++--------- internal/site/site.go | 9 ++- internal/site/site_test.go | 12 +++- 15 files changed, 297 insertions(+), 153 deletions(-) create mode 100644 internal/mdx/substitute.go create mode 100644 internal/render/escape.go delete mode 100644 internal/render/markdown.go diff --git a/README.md b/README.md index 9ad28fa..06b5921 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ Pages: `Title`, `ID` (the same in every language, it links the translations), `D ### Placeholders -`{{name}}`, with lowercase letters, digits and underscores, is replaced by a string from `site.json` or the language file. An unknown placeholder stops the build and names the file. Fenced code blocks are left as written, so an article can show a placeholder there; elsewhere a backslash before the placeholder (`\{{name}}`) keeps it as written, and Markdown drops the backslash. Code spans are substituted, which is how a page shows a value in monospace. `{{news_list}}`, `{{latest_news}}` and `{{categories_list}}` stand alone in a paragraph and expand to generated lists. +`{{name}}`, with lowercase letters, digits and underscores, is replaced by a string from `site.json` or the language file. An unknown placeholder stops the build and names the file. Fenced code blocks (found the way CommonMark finds them, in lists and quotes too) are left as written, so an article can show a placeholder there. Anywhere else a backslash before the placeholder (`\{{name}}`) keeps it as written, in a code span too; two backslashes are one literal backslash followed by the value. Code spans are otherwise substituted, which is how a page shows a value in monospace. The id of a heading is not checked against ids that a template writes itself, such as the `news` of the latest news block. `{{news_list}}`, `{{latest_news}}` and `{{categories_list}}` stand alone in a paragraph and expand to generated lists. ## Development diff --git a/internal/build/build_test.go b/internal/build/build_test.go index 9d88ad0..1d6623d 100644 --- a/internal/build/build_test.go +++ b/internal/build/build_test.go @@ -407,6 +407,23 @@ func TestBuildPlaceholdersAndCode(t *testing.T) { mustContain(t, "people", got, "en", "escaped {{lang}}.", "{{lang}} {{nope}}", "And en.") } +// TestBuildEscapedPlaceholders checks the escape with block placeholders: an +// escaped one inside a sentence or alone in a paragraph is shown, not expanded +// and not an error, and a code span can show a placeholder as written. +func TestBuildEscapedPlaceholders(t *testing.T) { + files := baseFiles() + files["content/en/people.md"] = page("people", "People", "", "See \\{{news_list}} here.\n\n\\{{latest_news}}\n\nAnd `\\{{lang}}`.\n") + out, err := runBuild(t, files) + if err != nil { + t.Fatalf("Build() error = %v", err) + } + got := readOut(t, out, "people.html") + mustContain(t, "people", got, "See {{news_list}} here.", "

    {{latest_news}}

    ", "{{lang}}") + if strings.Contains(got, "
      ") { + t.Error("an escaped block placeholder was expanded") + } +} + // TestBuildAssetsCopied checks the assets listed in site.json. func TestBuildAssetsCopied(t *testing.T) { out, err := runBuild(t, baseFiles()) diff --git a/internal/build/feed.go b/internal/build/feed.go index 719bd6e..d9adb7c 100644 --- a/internal/build/feed.go +++ b/internal/build/feed.go @@ -3,6 +3,7 @@ package build import ( + "regexp" "strings" "time" @@ -10,6 +11,10 @@ import ( "git.kmalloc.org/squid/motorino/internal/site" ) +// siteRelative matches a link that starts at the root of the site, such as +// href="/people.html", but not one that is relative to the protocol (href="//host/x"). +var siteRelative = regexp.MustCompile(`\b(href|src)="/([^/])`) + // feed returns the RSS 2.0 feed of the latest limit posts, which must be sorted // newest first. Each item carries the full text with absolute links. The // channel is described by the language l, and newsURL is the address of the @@ -21,7 +26,7 @@ func feed(base string, l site.Lang, newsURL string, posts []renderedPost, limit var items strings.Builder for _, p := range posts { link := base + p.Path - text := strings.NewReplacer(`href="/`, `href="`+base+`/`, `src="/`, `src="`+base+`/`).Replace(p.HTML) + text := siteRelative.ReplaceAllString(p.HTML, `$1="`+base+`/$2`) items.WriteString("\n") items.WriteString("" + render.Escape(p.Title) + "\n") items.WriteString("" + link + "\n") diff --git a/internal/build/feed_test.go b/internal/build/feed_test.go index 5c508ab..bf3b932 100644 --- a/internal/build/feed_test.go +++ b/internal/build/feed_test.go @@ -49,6 +49,16 @@ func TestFeedItems(t *testing.T) { } } +// TestFeedLeavesProtocolRelativeLinks checks that "//host/x" is not given the site address. +func TestFeedLeavesProtocolRelativeLinks(t *testing.T) { + items := []renderedPost{{Post: testPost("a", "2026-11-14", "A", ""), HTML: ` p`}} + got := feed("https://kmalloc.org", feedLang(), "/news/", items, 5) + mustContain(t, "feed", got, `src="//cdn.example/x.png"`, `href="https://kmalloc.org/p.html"`) + if strings.Contains(got, "https://kmalloc.org//cdn") { + t.Error("a protocol-relative link got the site address") + } +} + // TestFeedWithoutPosts checks that an empty feed is still a valid channel. func TestFeedWithoutPosts(t *testing.T) { got := feed("https://kmalloc.org", feedLang(), "/news/", nil, 20) diff --git a/internal/build/pages.go b/internal/build/pages.go index c873d09..31849d3 100644 --- a/internal/build/pages.go +++ b/internal/build/pages.go @@ -81,7 +81,7 @@ func (b *builder) renderPage(p content.Page) error { // markdown replaces the placeholders of a source, converts it to HTML and // expands the generated lists. The name is used in error messages. func (b *builder) markdown(name, source, lang string, ctx map[string]string) (string, []mdx.Heading, error) { - text, err := render.SubstituteMarkdown(name, source, ctx) + text, err := mdx.SubstituteMarkdown(name, source, ctx) if err != nil { return "", nil, err } @@ -89,8 +89,13 @@ func (b *builder) markdown(name, source, lang string, ctx map[string]string) (st if err != nil { return "", nil, fmt.Errorf("%s: %w", name, err) } - html, err = b.expandBlocks(name, html, lang, ctx) - return html, headings, err + if html, err = b.expandBlocks(name, html, lang, ctx); err != nil { + return "", nil, err + } + for i := range headings { + headings[i].Text = render.Unescape(headings[i].Text) + } + return render.Unescape(html), headings, nil } // codeElement matches a code element, where a placeholder is shown as written. diff --git a/internal/content/pages.go b/internal/content/pages.go index 696dfe3..5931766 100644 --- a/internal/content/pages.go +++ b/internal/content/pages.go @@ -146,8 +146,8 @@ func loadPage(file, source, defaultLang, lang, rel string) (Page, error) { // The body of the manifesto is the owner's text and is rewritten when the // page is built; the keys of the header are not. for _, key := range []string{"Title", "Description", "Note"} { - if err := mdx.RejectEmDash(source+" (key "+key+")", h.Get(key)); err != nil { - return Page{}, err + if mdx.HasEmDash(h.Get(key)) { + return Page{}, fmt.Errorf("%s: the key %s has an em dash, please remove it", source, key) } } } diff --git a/internal/mdx/emdash.go b/internal/mdx/emdash.go index 932f980..2f6b888 100644 --- a/internal/mdx/emdash.go +++ b/internal/mdx/emdash.go @@ -38,22 +38,30 @@ func NoEmDash(md string) (string, error) { for _, p := range parentheticals { md = strings.ReplaceAll(md, p.from, p.to) } - if err := RejectEmDash("the body", md); err != nil { - return "", err + for i, line := range strings.Split(md, "\n") { + if HasEmDash(line) { + return "", fmt.Errorf("line %d of the body: no em dashes, please", i+1) + } } return md, nil } // emDashEntity matches the HTML entities of the em dash, which would put it // on the page as well. -var emDashEntity = regexp.MustCompile(`(?i)—|—|—`) +var emDashEntity = regexp.MustCompile(`(?i)—|�*8212;|�*2014;`) + +// HasEmDash reports whether text has an em dash, written as the character or as +// an HTML entity. +func HasEmDash(text string) bool { + return strings.Contains(text, emDash) || emDashEntity.MatchString(text) +} // RejectEmDash returns an error naming the file and the line of the first em // dash in text, written as the character or as an HTML entity, or nil when // there is none. func RejectEmDash(name, text string) error { for i, line := range strings.Split(text, "\n") { - if strings.Contains(line, emDash) || emDashEntity.MatchString(line) { + if HasEmDash(line) { return fmt.Errorf("%s:%d: no em dashes, please", name, i+1) } } diff --git a/internal/mdx/emdash_test.go b/internal/mdx/emdash_test.go index 8e13178..dcd1267 100644 --- a/internal/mdx/emdash_test.go +++ b/internal/mdx/emdash_test.go @@ -47,7 +47,7 @@ func TestNoEmDashLeftover(t *testing.T) { // TestRejectEmDashEntities checks that the entities of the em dash are refused too. func TestRejectEmDashEntities(t *testing.T) { - for _, in := range []string{"a — b", "a &MDASH; b", "a — b", "a — b", "a — b"} { + for _, in := range []string{"a — b", "a &MDASH; b", "a — b", "a — b", "a — b", "a — b", "a — b"} { if err := RejectEmDash("f.md", in); err == nil { t.Errorf("RejectEmDash(%q) = nil, want an error", in) } @@ -61,8 +61,17 @@ func TestRejectEmDashEntities(t *testing.T) { // body, which is how the caller counts it. func TestNoEmDashErrorSaysBody(t *testing.T) { _, err := NoEmDash("fine\nsome words \u2014 and then more\n") - if err == nil || !strings.Contains(err.Error(), "the body:2") { - t.Errorf("NoEmDash() error = %v, want it to say the body:2", err) + if err == nil || !strings.Contains(err.Error(), "line 2 of the body") { + t.Errorf("NoEmDash() error = %v, want it to say line 2 of the body", err) + } +} + +// TestHasEmDash checks the test that the header keys use. +func TestHasEmDash(t *testing.T) { + for in, want := range map[string]bool{"plain": false, "a \u2014 b": true, "a — b": true, "a – b": false, "": false} { + if got := HasEmDash(in); got != want { + t.Errorf("HasEmDash(%q) = %v, want %v", in, got, want) + } } } diff --git a/internal/mdx/mdx_test.go b/internal/mdx/mdx_test.go index c576ca4..9d47e40 100644 --- a/internal/mdx/mdx_test.go +++ b/internal/mdx/mdx_test.go @@ -5,6 +5,8 @@ package mdx import ( "strings" "testing" + + "git.kmalloc.org/squid/motorino/internal/render" ) // TestRenderParagraphAndList checks the basic blocks and inline markup. @@ -177,6 +179,68 @@ func TestRenderBadExplicitIDCannotInject(t *testing.T) { } } +// TestSubstituteMarkdownFences checks that placeholders in fenced code blocks +// stay as written, wherever CommonMark finds the block, and that an unknown one +// there is no error. +func TestSubstituteMarkdownFences(t *testing.T) { + ctx := map[string]string{"lang": "en"} + tests := []struct{ name, src, want string }{ + {"backtick fence", "A {{lang}}\n\n```\n{{lang}} {{nope}}\n```\n\nB {{lang}}\n", "A en\n\n```\n{{lang}} {{nope}}\n```\n\nB en\n"}, + {"tilde fence with info", "~~~go\n{{lang}}\n~~~\n{{lang}}\n", "~~~go\n{{lang}}\n~~~\nen\n"}, + {"code span is substituted", "Value `{{lang}}`.\n", "Value `en`.\n"}, + {"fence in a quote", "> ```\n> {{nope}}\n> ```\n\n{{lang}}\n", "> ```\n> {{nope}}\n> ```\n\nen\n"}, + {"fence in a list item", "1. item\n\n ```\n {{nope}}\n ```\n\n{{lang}}\n", "1. item\n\n ```\n {{nope}}\n ```\n\nen\n"}, + {"code span at the start of a line is not a fence", "```x``` and {{lang}}\n\nthen {{lang}}\n", "```x``` and en\n\nthen en\n"}, + {"unclosed fence runs to the end", "{{lang}}\n\n```\n{{nope}}\n{{lang}}\n", "en\n\n```\n{{nope}}\n{{lang}}\n"}, + {"a closer indented four spaces does not close", "```\n ```\n{{nope}}\n```\n{{lang}}\n", "```\n ```\n{{nope}}\n```\nen\n"}, + {"a closer may be longer", "```\n{{nope}}\n`````\n{{lang}}\n", "```\n{{nope}}\n`````\nen\n"}, + {"CRLF line endings", "{{lang}}\r\n\r\n```\r\n{{nope}}\r\n```\r\n{{lang}}\r\n", "en\r\n\r\n```\r\n{{nope}}\r\n```\r\nen\r\n"}, + {"empty fence", "```\n```\n{{lang}}\n", "```\n```\nen\n"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := SubstituteMarkdown("page.md", tt.src, ctx) + if err != nil { + t.Fatalf("SubstituteMarkdown() error = %v", err) + } + if got != tt.want { + t.Errorf("SubstituteMarkdown() = %q, want %q", got, tt.want) + } + }) + } +} + +// TestSubstituteMarkdownUnknownOutsideFence checks that an unknown placeholder +// outside a fence still stops the build and names the file. +func TestSubstituteMarkdownUnknownOutsideFence(t *testing.T) { + _, err := SubstituteMarkdown("news/x.md", "```\n{{a}}\n```\n\nbad {{nope}}\n", map[string]string{}) + if err == nil || !strings.Contains(err.Error(), "news/x.md") || !strings.Contains(err.Error(), "nope") { + t.Errorf("SubstituteMarkdown() error = %v, want it to name news/x.md and nope", err) + } +} + +// TestSubstituteMarkdownEscapeEverywhere checks that a backslash keeps a +// placeholder as written in text, in a code span and next to a block name, and +// that the written form comes back through render.Unescape. +func TestSubstituteMarkdownEscapeEverywhere(t *testing.T) { + src := "Text \\{{lang}} and `\\{{lang}}` and \\{{news_list}}.\n" + got, err := SubstituteMarkdown("page.md", src, map[string]string{"lang": "en"}) + if err != nil { + t.Fatal(err) + } + html, _, err := Render(got) + if err != nil { + t.Fatal(err) + } + if strings.Contains(html, "{{") { + t.Errorf("html %q still has braces before Unescape", html) + } + want := "

      Text {{lang}} and {{lang}} and {{news_list}}.

      \n" + if out := render.Unescape(html); out != want { + t.Errorf("Unescape(html) = %q, want %q", out, want) + } +} + // TestRenderHeadingTextIsPlain checks that the heading text drops the markup. func TestRenderHeadingTextIsPlain(t *testing.T) { _, headings, err := Render("## Free *your* mind\n") diff --git a/internal/mdx/substitute.go b/internal/mdx/substitute.go new file mode 100644 index 0000000..6976f97 --- /dev/null +++ b/internal/mdx/substitute.go @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package mdx + +import ( + "strings" + + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/text" + + "git.kmalloc.org/squid/motorino/internal/render" +) + +// SubstituteMarkdown replaces the {{name}} placeholders of Markdown source like +// render.SubstituteEscaped does, but leaves the content of fenced code blocks +// as written, so an article can show a placeholder there. The blocks are found +// by the Markdown parser, so they are the ones CommonMark sees, in lists and in +// quotes too. Code spans are substituted: the pages use them to show a value in +// monospace. The result still has the markers of escaped placeholders; turn +// them back with render.Unescape after the Markdown is rendered. +func SubstituteMarkdown(name, md string, ctx map[string]string) (string, error) { + src := []byte(md) + doc := engine.Parser().Parse(text.NewReader(src)) + + type span struct{ start, stop int } + var fenced []span + _ = ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) { + if block, ok := n.(*ast.FencedCodeBlock); ok && entering { + lines := block.Lines() + for i := 0; i < lines.Len(); i++ { + segment := lines.At(i) + fenced = append(fenced, span{segment.Start, segment.Stop}) + } + } + return ast.WalkContinue, nil + }) + + var out strings.Builder + pos := 0 + for _, s := range fenced { + if s.start < pos { + continue + } + replaced, err := render.SubstituteEscaped(name, md[pos:s.start], ctx) + if err != nil { + return "", err + } + out.WriteString(replaced) + out.WriteString(md[s.start:s.stop]) + pos = s.stop + } + replaced, err := render.SubstituteEscaped(name, md[pos:], ctx) + if err != nil { + return "", err + } + out.WriteString(replaced) + return out.String(), nil +} diff --git a/internal/render/escape.go b/internal/render/escape.go new file mode 100644 index 0000000..19caa0a --- /dev/null +++ b/internal/render/escape.go @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package render + +import ( + "fmt" + "regexp" + "strings" +) + +// The markers that stand for an escaped placeholder while the Markdown is +// rendered. They are private use characters, which neither Markdown nor the +// templates treat in any special way. +const ( + escapeOpen = "" + escapeClose = "" +) + +// escaped matches a marker with the name of the placeholder it stands for. +var escaped = regexp.MustCompile(escapeOpen + `([a-z0-9_]+)` + escapeClose) + +// SubstituteEscaped replaces the {{name}} placeholders of text like Substitute +// does, except those with a backslash right before them (an odd number of +// backslashes: two make a literal backslash). The backslash is dropped and the +// placeholder is replaced by a marker that Unescape turns back into the written +// placeholder once the Markdown has been rendered. An unknown placeholder is an +// error that names the file; an escaped one is not looked up. +func SubstituteEscaped(name, text string, ctx map[string]string) (string, error) { + var out strings.Builder + last := 0 + for _, loc := range placeholder.FindAllStringSubmatchIndex(text, -1) { + start, end := loc[0], loc[1] + key := text[loc[2]:loc[3]] + slashes := 0 + for i := start - 1; i >= last && text[i] == '\\'; i-- { + slashes++ + } + if slashes%2 == 1 { + out.WriteString(text[last : start-1]) + out.WriteString(escapeOpen + key + escapeClose) + } else { + value, ok := ctx[key] + if !ok { + return "", fmt.Errorf("%s: unknown placeholder {{%s}}", name, key) + } + out.WriteString(text[last:start]) + out.WriteString(value) + } + last = end + } + out.WriteString(text[last:]) + return out.String(), nil +} + +// Unescape turns the markers left by SubstituteEscaped back into the +// placeholders as they were written. +func Unescape(s string) string { + return escaped.ReplaceAllString(s, "{{$1}}") +} diff --git a/internal/render/markdown.go b/internal/render/markdown.go deleted file mode 100644 index abc1251..0000000 --- a/internal/render/markdown.go +++ /dev/null @@ -1,108 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -package render - -import ( - "fmt" - "strings" -) - -// SubstituteMarkdown replaces the {{name}} placeholders of Markdown text like -// Substitute does, with two exceptions that let an article show a placeholder -// as it is written: fenced code blocks are left untouched (an unknown -// placeholder in one is not an error), and a placeholder with a backslash in -// front of it is kept, backslash included, which Markdown then renders as the -// plain placeholder. Code spans are substituted: the pages use them to show a -// value in monospace. An unknown placeholder elsewhere is an error that names -// the file. -func SubstituteMarkdown(name, md string, ctx map[string]string) (string, error) { - var out strings.Builder - var text strings.Builder // the lines outside fences, waiting to be substituted - flush := func() error { - if text.Len() == 0 { - return nil - } - replaced, err := substituteUnescaped(name, text.String(), ctx) - if err != nil { - return err - } - out.WriteString(replaced) - text.Reset() - return nil - } - - lines := strings.SplitAfter(md, "\n") - for i := 0; i < len(lines); i++ { - marker, length := fenceStart(lines[i]) - if length == 0 { - text.WriteString(lines[i]) - continue - } - if err := flush(); err != nil { - return "", err - } - out.WriteString(lines[i]) - for i++; i < len(lines); i++ { - out.WriteString(lines[i]) - if fenceEnds(lines[i], marker, length) { - break - } - } - } - if err := flush(); err != nil { - return "", err - } - return out.String(), nil -} - -// fenceStart returns the fence character and its length when the line opens a -// fenced code block (three or more backticks or tildes, indented by at most -// three spaces), and a zero length otherwise. -func fenceStart(line string) (marker byte, length int) { - rest := strings.TrimLeft(line, " ") - if len(line)-len(rest) > 3 || rest == "" || (rest[0] != '`' && rest[0] != '~') { - return 0, 0 - } - marker = rest[0] - for length < len(rest) && rest[length] == marker { - length++ - } - if length < 3 { - return 0, 0 - } - return marker, length -} - -// fenceEnds reports whether the line closes a fence opened with length -// characters of marker: at least as many, and nothing else on the line. -func fenceEnds(line string, marker byte, length int) bool { - rest := strings.TrimSpace(line) - if len(rest) < length { - return false - } - return strings.Trim(rest, string(marker)) == "" -} - -// substituteUnescaped is Substitute that leaves alone the placeholders that -// have a backslash right before them. -func substituteUnescaped(name, text string, ctx map[string]string) (string, error) { - var out strings.Builder - last := 0 - for _, loc := range placeholder.FindAllStringSubmatchIndex(text, -1) { - start, end := loc[0], loc[1] - out.WriteString(text[last:start]) - last = end - if start > 0 && text[start-1] == '\\' { - out.WriteString(text[start:end]) - continue - } - key := text[loc[2]:loc[3]] - value, ok := ctx[key] - if !ok { - return "", fmt.Errorf("%s: unknown placeholder {{%s}}", name, key) - } - out.WriteString(value) - } - out.WriteString(text[last:]) - return out.String(), nil -} diff --git a/internal/render/render_test.go b/internal/render/render_test.go index 9e97c33..c4e9bd0 100644 --- a/internal/render/render_test.go +++ b/internal/render/render_test.go @@ -59,40 +59,42 @@ func TestSubstituteValueIsNotRescanned(t *testing.T) { } } -// TestSubstituteMarkdownFences checks that placeholders in fenced code blocks -// stay as written (an unknown one is no error there), while code spans are -// substituted: the pages use them to show a value in monospace. -func TestSubstituteMarkdownFences(t *testing.T) { +// TestSubstituteEscapedKeepsEscapes checks that a backslash before a placeholder +// keeps it as written: it becomes a marker that Unescape turns back into the +// placeholder after the Markdown is rendered. An unknown escaped one is no error. +func TestSubstituteEscapedKeepsEscapes(t *testing.T) { + got, err := SubstituteEscaped("page.md", "Write \\{{lang}} for {{lang}}, and \\{{nope}}.", map[string]string{"lang": "en"}) + if err != nil { + t.Fatalf("SubstituteEscaped() error = %v", err) + } + if strings.Contains(got, "{{") || strings.Contains(got, "\\") { + t.Errorf("SubstituteEscaped() = %q, want no braces and no backslash left", got) + } + if want := "Write {{lang}} for en, and {{nope}}."; Unescape(got) != want { + t.Errorf("Unescape(%q) = %q, want %q", got, Unescape(got), want) + } +} + +// TestSubstituteEscapedBackslashParity checks that an escaped backslash does not +// escape the placeholder: two backslashes are one literal backslash. +func TestSubstituteEscapedBackslashParity(t *testing.T) { ctx := map[string]string{"lang": "en"} - src := "Outside {{lang}} and `{{lang}}`.\n\n```\n{{lang}} {{nope}}\n```\n\n~~~go\n{{lang}}\n~~~\n\nEnd {{lang}}.\n" - got, err := SubstituteMarkdown("page.md", src, ctx) - if err != nil { - t.Fatalf("SubstituteMarkdown() error = %v", err) + even, err := SubstituteEscaped("f", "\\\\{{lang}}", ctx) + if err != nil || even != "\\\\en" { + t.Errorf("two backslashes: got %q, %v, want the value after both backslashes", even, err) } - want := "Outside en and `en`.\n\n```\n{{lang}} {{nope}}\n```\n\n~~~go\n{{lang}}\n~~~\n\nEnd en.\n" - if got != want { - t.Errorf("SubstituteMarkdown() = %q, want %q", got, want) + odd, err := SubstituteEscaped("f", "\\\\\\{{lang}}", ctx) + if err != nil || Unescape(odd) != "\\\\{{lang}}" { + t.Errorf("three backslashes: got %q, %v, want two backslashes then the placeholder", Unescape(odd), err) } } -// TestSubstituteMarkdownEscape checks that a backslash before a placeholder -// keeps it as written: Markdown then drops the backslash. -func TestSubstituteMarkdownEscape(t *testing.T) { - got, err := SubstituteMarkdown("page.md", "Write \\{{lang}} for {{lang}}, and \\{{nope}} is fine.\n", map[string]string{"lang": "en"}) - if err != nil { - t.Fatalf("SubstituteMarkdown() error = %v", err) - } - if want := "Write \\{{lang}} for en, and \\{{nope}} is fine.\n"; got != want { - t.Errorf("SubstituteMarkdown() = %q, want %q", got, want) - } -} - -// TestSubstituteMarkdownUnknownOutsideCode checks that an unknown placeholder -// outside a fence still stops the build and names the file. -func TestSubstituteMarkdownUnknownOutsideCode(t *testing.T) { - _, err := SubstituteMarkdown("news/x.md", "ok {{a}}\n\nbad {{nope}}\n", map[string]string{"a": "1"}) +// TestSubstituteEscapedUnknown checks that an unknown placeholder still stops the +// build and names the file. +func TestSubstituteEscapedUnknown(t *testing.T) { + _, err := SubstituteEscaped("news/x.md", "ok {{a}} bad {{nope}}", map[string]string{"a": "1"}) if err == nil || !strings.Contains(err.Error(), "news/x.md") || !strings.Contains(err.Error(), "nope") { - t.Errorf("SubstituteMarkdown() error = %v, want it to name news/x.md and nope", err) + t.Errorf("SubstituteEscaped() error = %v, want it to name news/x.md and nope", err) } } diff --git a/internal/site/site.go b/internal/site/site.go index 8a5ca1e..98e5549 100644 --- a/internal/site/site.go +++ b/internal/site/site.go @@ -7,6 +7,7 @@ package site import ( "encoding/json" "fmt" + "net/url" "os" "path/filepath" "regexp" @@ -82,8 +83,9 @@ func Load(dir string) (Config, map[string]Lang, error) { // checkBaseURL returns an error unless the address of the site is absolute and // has no trailing slash: page addresses and the feed are built by adding to it. func checkBaseURL(base string) error { - if !(strings.HasPrefix(base, "https://") || strings.HasPrefix(base, "http://")) || strings.HasSuffix(base, "/") { - return fmt.Errorf("site.json: base_url must be an absolute address without a trailing slash, such as https://kmalloc.org") + u, err := url.Parse(base) + if err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" || strings.HasSuffix(base, "/") { + return fmt.Errorf("site.json: base_url must be an absolute address with a host and no trailing slash, such as https://kmalloc.org") } return nil } @@ -133,6 +135,9 @@ func loadLang(dir, code string) (Lang, error) { if lang.Strings["lang_name"] == "" { return Lang{}, fmt.Errorf("%s: missing lang_name", name) } + if _, ok := lang.Strings["base_url"]; ok { + return Lang{}, fmt.Errorf("%s: base_url is set in site.json only", name) + } if _, ok := raw["nav"]; !ok { return Lang{}, fmt.Errorf("%s: missing nav", name) } diff --git a/internal/site/site_test.go b/internal/site/site_test.go index 3ac6eba..13261c2 100644 --- a/internal/site/site_test.go +++ b/internal/site/site_test.go @@ -155,7 +155,7 @@ func TestLoadRejectsBadLanguageCodes(t *testing.T) { // TestLoadNeedsBaseURL checks that the address of the site is required and absolute, // because the feed and the page addresses are built from it. func TestLoadNeedsBaseURL(t *testing.T) { - for _, site := range []string{`{"languages": ["en", "it"]}`, `{"base_url": "kmalloc.org", "languages": ["en", "it"]}`, `{"base_url": "https://kmalloc.org/", "languages": ["en", "it"]}`} { + for _, site := range []string{`{"languages": ["en", "it"]}`, `{"base_url": "kmalloc.org", "languages": ["en", "it"]}`, `{"base_url": "https://kmalloc.org/", "languages": ["en", "it"]}`, `{"base_url": "https://", "languages": ["en", "it"]}`, `{"base_url": "https:///x", "languages": ["en", "it"]}`} { files := validFiles() files["site.json"] = site _, _, err := Load(writeSite(t, files)) @@ -165,6 +165,16 @@ func TestLoadNeedsBaseURL(t *testing.T) { } } +// TestLoadLanguageCannotOverrideBaseURL checks that the address of the site has one source. +func TestLoadLanguageCannotOverrideBaseURL(t *testing.T) { + files := validFiles() + files["lang/it.json"] = `{"lang_name": "Italiano", "base_url": "https://example.org", "nav": {}}` + _, _, err := Load(writeSite(t, files)) + if err == nil || !strings.Contains(err.Error(), "lang/it.json") || !strings.Contains(err.Error(), "base_url") { + t.Errorf("Load() error = %v, want it to name lang/it.json and base_url", err) + } +} + // TestLoadNeedsLanguages checks that an empty language list is an error. func TestLoadNeedsLanguages(t *testing.T) { files := validFiles() -- 2.54.0