From beb7698a77ae70a3927c5f9acb474f456f355d2a Mon Sep 17 00:00:00 2001 From: tgiachi Date: Sun, 15 Nov 2026 10:18:21 +0100 Subject: [PATCH 01/22] ci: run make check on every push and pull request --- .forgejo/workflows/check.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .forgejo/workflows/check.yml diff --git a/.forgejo/workflows/check.yml b/.forgejo/workflows/check.yml new file mode 100644 index 0000000..538b9f3 --- /dev/null +++ b/.forgejo/workflows/check.yml @@ -0,0 +1,23 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# +# Runs `make check` (gofmt, go vet, staticcheck and the tests) on every push to main and on every +# pull request, on the shared runner of git.kmalloc.org. +name: check + +on: + push: + branches: [main] + pull_request: + +jobs: + check: + runs-on: docker + steps: + - uses: https://code.forgejo.org/actions/checkout@v4 + - uses: https://code.forgejo.org/actions/setup-go@v5 + with: + go-version-file: go.mod + - name: Install staticcheck + run: go install honnef.co/go/tools/cmd/staticcheck@latest + - name: Run the checks + run: PATH="$PATH:$(go env GOPATH)/bin" make check From 8f294d92e2a60d71d23aaa6189aef5b63317395d Mon Sep 17 00:00:00 2001 From: tgiachi Date: Sun, 15 Nov 2026 10:27:25 +0100 Subject: [PATCH 02/22] docs: mention the check workflow in the README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index af5a634..81c0168 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ make build a local binary in bin/motorino make dist the release binary and its SHA-256 in dist/ ``` -Go code has a doc comment on every package and declaration, and is formatted with `gofmt`. Commits follow conventional commits. +Go code has a doc comment on every package and declaration, and is formatted with `gofmt`. Commits follow conventional commits. A Forgejo Actions workflow (`.forgejo/workflows/check.yml`) runs `make check` on every push to `main` and on every pull request, on the shared runner of git.kmalloc.org. ## Release From 917683afa491c7f0b4331757d6066d215b5f19d9 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Sun, 15 Nov 2026 10:32:19 +0100 Subject: [PATCH 03/22] fix: close the minor findings of the first review Require refuses an empty value. Heading ids: an explicit {#id} must be sound and unique, generated ids avoid the explicit ones and each other, and a heading with no ASCII letters gets a fallback. Fenced code blocks keep their placeholders and a backslash keeps one anywhere. The em dash rule covers the header of the manifesto pages and the HTML entities. The feed makes src links absolute, base_url is required, and the news page must be a folder index. A category written twice in a post counts once, a block name in a news title is no longer taken for a placeholder, and build refuses extra arguments. --- README.md | 2 +- cmd/motorino/main.go | 5 ++ cmd/motorino/main_test.go | 16 ++++- internal/build/build.go | 8 ++- internal/build/build_test.go | 49 +++++++++++++++ internal/build/feed.go | 2 +- internal/build/pages.go | 35 +++++++---- internal/content/pages.go | 8 +++ internal/content/pages_test.go | 15 +++++ internal/content/posts.go | 8 ++- internal/content/posts_test.go | 12 ++++ internal/header/header.go | 6 +- internal/header/header_test.go | 15 +++++ internal/mdx/emdash.go | 11 +++- internal/mdx/emdash_test.go | 21 +++++++ internal/mdx/mdx.go | 61 ++++++++++++++++--- internal/mdx/mdx_test.go | 78 ++++++++++++++++++++++++ internal/render/markdown.go | 108 +++++++++++++++++++++++++++++++++ internal/render/render_test.go | 37 +++++++++++ internal/site/site.go | 12 ++++ internal/site/site_test.go | 17 +++++- 21 files changed, 493 insertions(+), 33 deletions(-) create mode 100644 internal/render/markdown.go diff --git a/README.md b/README.md index 81c0168..9ad28fa 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ Pages: `Title`, `ID` (the same in every language, it links the translations), `D ### Placeholders -`{{name}}`, with lowercase letters, digits and underscores, is replaced by a string from `site.json` or the language file. An unknown placeholder stops the build and names the file. `{{news_list}}`, `{{latest_news}}` and `{{categories_list}}` stand alone in a paragraph and expand to generated lists. +`{{name}}`, with lowercase letters, digits and underscores, is replaced by a string from `site.json` or the language file. An unknown placeholder stops the build and names the file. Fenced code blocks are left as written, so an article can show a placeholder there; elsewhere a backslash before the placeholder (`\{{name}}`) keeps it as written, and Markdown drops the backslash. Code spans are substituted, which is how a page shows a value in monospace. `{{news_list}}`, `{{latest_news}}` and `{{categories_list}}` stand alone in a paragraph and expand to generated lists. ## Development diff --git a/cmd/motorino/main.go b/cmd/motorino/main.go index ce40c01..a1997f1 100644 --- a/cmd/motorino/main.go +++ b/cmd/motorino/main.go @@ -15,6 +15,7 @@ import ( "fmt" "io" "os" + "strings" "git.kmalloc.org/squid/motorino/internal/build" "git.kmalloc.org/squid/motorino/internal/version" @@ -59,6 +60,10 @@ func runBuild(args []string, stdout, stderr io.Writer) int { } return 2 } + if flags.NArg() > 0 { + fmt.Fprintf(stderr, "motorino: unexpected arguments: %s\n%s\n", strings.Join(flags.Args(), " "), usage) + return 2 + } result, err := build.Build(build.Options{SiteDir: *site, OutDir: *out, Version: version.String()}) if err != nil { fmt.Fprintf(stderr, "motorino: %v\n", err) diff --git a/cmd/motorino/main_test.go b/cmd/motorino/main_test.go index 8e2ffd6..4090294 100644 --- a/cmd/motorino/main_test.go +++ b/cmd/motorino/main_test.go @@ -30,8 +30,11 @@ func miniSite(t *testing.T) string { } for _, id := range []string{"home", "manifesto", "people", "services", "news", "history"} { name := id - if id == "home" { + switch id { + case "home": name = "index" + case "news": + name = "news/index" } files["content/en/"+name+".md"] = "Title: " + id + "\nID: " + id + "\nDescription: d\n-\nBody of " + id + ".\n" } @@ -101,6 +104,17 @@ func TestRunUnknownCommand(t *testing.T) { } } +// TestRunBuildRejectsExtraArguments checks that arguments that are not flags are not ignored. +func TestRunBuildRejectsExtraArguments(t *testing.T) { + var out, errOut bytes.Buffer + if code := run([]string{"build", "somesite", "someout"}, &out, &errOut); code != 2 { + t.Errorf("run(build somesite someout) = %d, want 2", code) + } + if !strings.Contains(errOut.String(), "unexpected") { + t.Errorf("stderr = %q, want it to say what was unexpected", errOut.String()) + } +} + // TestRunBuildBadFlag checks that an unknown flag is a usage error. func TestRunBuildBadFlag(t *testing.T) { var out, errOut bytes.Buffer diff --git a/internal/build/build.go b/internal/build/build.go index 7975c88..a6d2ea3 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -76,7 +76,10 @@ type Result struct { } // Build reads the site in opts.SiteDir and writes it to opts.OutDir. It renders -// everything in memory first, so a failure leaves the output folder untouched. +// everything in memory first, so a failure in the content leaves the output +// folder untouched. Writing the files is not atomic, and files of an earlier +// build that are no longer produced are not removed: build into a fresh folder +// when that matters, as the deploy does. func Build(opts Options) (Result, error) { b, err := load(opts) if err != nil { @@ -137,6 +140,9 @@ func load(opts Options) (*builder, error) { return nil, fmt.Errorf("content/%s: there is no page with ID %q", b.defaultLang, id) } } + if news := b.byKey[pageKey{"news", b.defaultLang}]; !strings.HasSuffix(news.Path, "/") { + return nil, fmt.Errorf("%s: the news page must be an index (news/index.md), so that the feed has an address next to it", news.Source) + } if b.templates, err = readTemplates(opts.SiteDir); err != nil { return nil, err } diff --git a/internal/build/build_test.go b/internal/build/build_test.go index b46db52..9d88ad0 100644 --- a/internal/build/build_test.go +++ b/internal/build/build_test.go @@ -358,6 +358,55 @@ func TestBuildRejectsSwitchToUnknownLanguage(t *testing.T) { } } +// TestBuildFeedMakesImagesAbsolute checks that src links get the full address like href links. +func TestBuildFeedMakesImagesAbsolute(t *testing.T) { + files := baseFiles() + files["news/pic.md"] = "Title: Pic\nDate: 2026-11-15T12:00:00+0100\n-\n![the logo](/logo.svg)\n" + out, err := runBuild(t, files) + if err != nil { + t.Fatal(err) + } + mustContain(t, "feed", readOut(t, out, "news/feed.xml"), `src="https://kmalloc.org/logo.svg"`) +} + +// TestBuildNewsPageMustBeAFolderIndex checks that the feed has a sound address. +func TestBuildNewsPageMustBeAFolderIndex(t *testing.T) { + files := baseFiles() + delete(files, "content/en/news/index.md") + files["content/en/news.md"] = page("news", "News", "", "{{news_list}}\n") + _, err := runBuild(t, files) + if err == nil || !strings.Contains(err.Error(), "content/en/news.md") || !strings.Contains(err.Error(), "index") { + t.Errorf("Build() error = %v, want it to say that content/en/news.md must be an index", err) + } +} + +// TestBuildBlockNameInATitleIsNotAnError checks that a generated list that shows +// the text of a block placeholder (in a news title) is not mistaken for a +// placeholder that is not alone in its paragraph. +func TestBuildBlockNameInATitleIsNotAnError(t *testing.T) { + files := baseFiles() + files["news/tpl.md"] = "Title: Use {{latest_news}} in a page\nDate: 2026-11-15T12:00:00+0100\n-\nA note.\n" + out, err := runBuild(t, files) + if err != nil { + t.Fatalf("Build() error = %v", err) + } + mustContain(t, "home", readOut(t, out, "index.html"), "Use {{latest_news}} in a page") +} + +// TestBuildPlaceholdersAndCode checks the three ways a page can use a placeholder +// next to code: a value in a code span (the pages show the GPG fingerprint that +// way), a fenced block that shows a placeholder as written, and an escaped one. +func TestBuildPlaceholdersAndCode(t *testing.T) { + files := baseFiles() + files["content/en/people.md"] = page("people", "People", "", "Value `{{lang}}` and escaped \\{{lang}}.\n\n```\n{{lang}} {{nope}}\n```\n\nAnd {{lang}}.\n") + out, err := runBuild(t, files) + if err != nil { + t.Fatalf("Build() error = %v", err) + } + got := readOut(t, out, "people.html") + mustContain(t, "people", got, "en", "escaped {{lang}}.", "{{lang}} {{nope}}", "And en.") +} + // TestBuildAssetsCopied checks the assets listed in site.json. func TestBuildAssetsCopied(t *testing.T) { out, err := runBuild(t, baseFiles()) diff --git a/internal/build/feed.go b/internal/build/feed.go index 485cc8a..719bd6e 100644 --- a/internal/build/feed.go +++ b/internal/build/feed.go @@ -21,7 +21,7 @@ func feed(base string, l site.Lang, newsURL string, posts []renderedPost, limit var items strings.Builder for _, p := range posts { link := base + p.Path - text := strings.ReplaceAll(p.HTML, `href="/`, `href="`+base+`/`) + text := strings.NewReplacer(`href="/`, `href="`+base+`/`, `src="/`, `src="`+base+`/`).Replace(p.HTML) items.WriteString("\n") items.WriteString("" + render.Escape(p.Title) + "\n") items.WriteString("" + link + "\n") diff --git a/internal/build/pages.go b/internal/build/pages.go index 6de74b6..c873d09 100644 --- a/internal/build/pages.go +++ b/internal/build/pages.go @@ -4,6 +4,7 @@ package build import ( "fmt" + "regexp" "strings" "git.kmalloc.org/squid/motorino/internal/content" @@ -61,7 +62,7 @@ func (b *builder) renderPage(p content.Page) error { var toc []string for _, h := range headings { if h.Level == 2 { - toc = append(toc, fmt.Sprintf(`
  • %s
  • `, h.ID, render.Escape(h.Text))) + toc = append(toc, fmt.Sprintf(`
  • %s
  • `, render.EscapeAttr(h.ID), render.Escape(h.Text))) } } ctx["manifesto_toc"] = strings.Join(toc, "\n") @@ -80,7 +81,7 @@ func (b *builder) renderPage(p content.Page) error { // markdown replaces the placeholders of a source, converts it to HTML and // expands the generated lists. The name is used in error messages. func (b *builder) markdown(name, source, lang string, ctx map[string]string) (string, []mdx.Heading, error) { - text, err := render.Substitute(name, source, ctx) + text, err := render.SubstituteMarkdown(name, source, ctx) if err != nil { return "", nil, err } @@ -92,22 +93,32 @@ func (b *builder) markdown(name, source, lang string, ctx map[string]string) (st return html, headings, err } +// codeElement matches a code element, where a placeholder is shown as written. +var codeElement = regexp.MustCompile(`(?s)]*>.*?`) + // expandBlocks replaces each paragraph that holds only a block placeholder by -// the generated HTML. A block placeholder inside other text is an error. +// the generated HTML. A block placeholder inside other text is an error. All +// blocks are checked before any is expanded, so the text of a generated list +// (a news title may say {{latest_news}}) is never taken for a placeholder. func (b *builder) expandBlocks(name, html, lang string, ctx map[string]string) (string, error) { + outside := codeElement.ReplaceAllString(html, "") for _, block := range blockNames { - marker := "

    {{" + block + "}}

    " - if strings.Contains(html, marker) { - generated, err := b.block(block, lang, ctx) - if err != nil { - return "", err - } - html = strings.ReplaceAll(html, marker, generated) - } - if strings.Contains(html, "{{"+block+"}}") { + token := "{{" + block + "}}" + if strings.Count(outside, token) != strings.Count(outside, "

    "+token+"

    ") { return "", fmt.Errorf("%s: {{%s}} must stand alone in a paragraph", name, block) } } + for _, block := range blockNames { + marker := "

    {{" + block + "}}

    " + if !strings.Contains(html, marker) { + continue + } + generated, err := b.block(block, lang, ctx) + if err != nil { + return "", err + } + html = strings.ReplaceAll(html, marker, generated) + } return html, nil } diff --git a/internal/content/pages.go b/internal/content/pages.go index 3c6911b..696dfe3 100644 --- a/internal/content/pages.go +++ b/internal/content/pages.go @@ -142,6 +142,14 @@ func loadPage(file, source, defaultLang, lang, rel string) (Page, error) { if err := mdx.RejectEmDash(source, string(data)); err != nil { return Page{}, err } + } else { + // The body of the manifesto is the owner's text and is rewritten when the + // page is built; the keys of the header are not. + for _, key := range []string{"Title", "Description", "Note"} { + if err := mdx.RejectEmDash(source+" (key "+key+")", h.Get(key)); err != nil { + return Page{}, err + } + } } switches, err := parseSwitch(source, h.Get("Switch")) if err != nil { diff --git a/internal/content/pages_test.go b/internal/content/pages_test.go index a42e2f9..91881fc 100644 --- a/internal/content/pages_test.go +++ b/internal/content/pages_test.go @@ -107,6 +107,21 @@ func TestLoadPagesManifestoKeepsEmDash(t *testing.T) { } } +// TestLoadPagesManifestoHeaderEmDash checks that the exemption of the manifesto +// body does not cover the title, the description and the note. +func TestLoadPagesManifestoHeaderEmDash(t *testing.T) { + for _, field := range []string{"Title", "Description", "Note"} { + head := map[string]string{"Title": "M", "Description": "d", "Note": "n"} + head[field] = "words \u2014 more" + file := "Title: " + head["Title"] + "\nID: manifesto\nDescription: " + head["Description"] + "\nLayout: manifesto\nNote: " + head["Note"] + "\n-\n## 01 \u2014 Body\n" + dir := writeFiles(t, map[string]string{"content/en/m.md": file}) + _, err := LoadPages(dir, []string{"en"}) + if err == nil || !strings.Contains(err.Error(), "content/en/m.md") || !strings.Contains(err.Error(), field) { + t.Errorf("%s with an em dash: LoadPages() error = %v, want it to name content/en/m.md and %s", field, err, field) + } + } +} + // TestLoadPagesDuplicateID checks that two pages cannot share an ID in a language. func TestLoadPagesDuplicateID(t *testing.T) { dir := writeFiles(t, map[string]string{ diff --git a/internal/content/posts.go b/internal/content/posts.go index a51c408..ae5f103 100644 --- a/internal/content/posts.go +++ b/internal/content/posts.go @@ -136,6 +136,7 @@ func parseDate(source, value string) (time.Time, error) { // parseCategories splits a comma separated list into categories. func parseCategories(source, value string) ([]Category, error) { var out []Category + seen := map[string]bool{} for _, name := range strings.Split(value, ",") { name = strings.TrimSpace(name) if name == "" { @@ -145,6 +146,10 @@ func parseCategories(source, value string) ([]Category, error) { if err != nil { return nil, fmt.Errorf("%s: %w", source, err) } + if seen[slug] { + continue + } + seen[slug] = true out = append(out, Category{Name: name, Slug: slug}) } return out, nil @@ -177,7 +182,8 @@ func checkCategories(posts []Post) error { return nil } -// markup matches the Markdown that is dropped from a fallback description. +// markdownLink, markdownMark and sentenceEnd pick the first sentence out of a +// body and drop its Markdown markup, for a description that was not written. var ( markdownLink = regexp.MustCompile(`\[([^\]]*)\]\([^)]*\)`) markdownMark = regexp.MustCompile("[*_`]") diff --git a/internal/content/posts_test.go b/internal/content/posts_test.go index 3905178..c3dfda2 100644 --- a/internal/content/posts_test.go +++ b/internal/content/posts_test.go @@ -171,6 +171,18 @@ func TestLoadPostsCategoryCollision(t *testing.T) { } } +// TestLoadPostsRepeatedCategory checks that a category written twice counts once. +func TestLoadPostsRepeatedCategory(t *testing.T) { + dir := writeFiles(t, map[string]string{"news/a.md": "Title: A\nDate: 2026-01-01T10:00:00+0100\nCategories: News, Linux, news, News\n-\nx\n"}) + posts, err := LoadPosts(dir) + if err != nil { + t.Fatal(err) + } + if len(posts[0].Categories) != 2 || posts[0].Categories[0].Name != "News" || posts[0].Categories[1].Name != "Linux" { + t.Errorf("Categories = %+v, want News and Linux once each", posts[0].Categories) + } +} + // TestLoadPostsNoNewsFolder checks that a site without news is fine. func TestLoadPostsNoNewsFolder(t *testing.T) { posts, err := LoadPosts(t.TempDir()) diff --git a/internal/header/header.go b/internal/header/header.go index 9b8d6aa..78fa8ac 100644 --- a/internal/header/header.go +++ b/internal/header/header.go @@ -78,12 +78,16 @@ func (h Header) Has(key string) bool { return ok } -// Require returns an error naming name and the first missing key of keys. +// Require returns an error naming name and the first key of keys that is +// missing or has an empty value. func (h Header) Require(name string, keys ...string) error { for _, k := range keys { if !h.Has(k) { return fmt.Errorf("%s: missing key %q", name, k) } + if h.Get(k) == "" { + return fmt.Errorf("%s: the key %q is empty", name, k) + } } return nil } diff --git a/internal/header/header_test.go b/internal/header/header_test.go index 30758f6..9e38835 100644 --- a/internal/header/header_test.go +++ b/internal/header/header_test.go @@ -124,6 +124,21 @@ func TestRequire(t *testing.T) { } } +// TestRequireNeedsAValue checks that a key with an empty value counts as missing. +func TestRequireNeedsAValue(t *testing.T) { + h, _, err := Parse("file.md", []byte("Title:\nID: x\n-\n")) + if err != nil { + t.Fatal(err) + } + err = h.Require("file.md", "Title") + if err == nil || !strings.Contains(err.Error(), "Title") || !strings.Contains(err.Error(), "empty") { + t.Errorf("Require(Title) = %v, want an error saying that Title is empty", err) + } + if err := h.Require("file.md", "ID"); err != nil { + t.Errorf("Require(ID) = %v", err) + } +} + // TestOnly checks that a key outside the allowed list is rejected. func TestOnly(t *testing.T) { h, _, err := Parse("file.md", []byte("Title: T\nTitolo: T\n-\n")) diff --git a/internal/mdx/emdash.go b/internal/mdx/emdash.go index e6e74fa..932f980 100644 --- a/internal/mdx/emdash.go +++ b/internal/mdx/emdash.go @@ -38,17 +38,22 @@ func NoEmDash(md string) (string, error) { for _, p := range parentheticals { md = strings.ReplaceAll(md, p.from, p.to) } - if err := RejectEmDash("manifesto source", md); err != nil { + if err := RejectEmDash("the body", md); err != nil { return "", err } return md, nil } +// emDashEntity matches the HTML entities of the em dash, which would put it +// on the page as well. +var emDashEntity = regexp.MustCompile(`(?i)—|—|—`) + // RejectEmDash returns an error naming the file and the line of the first em -// dash in text, or nil when there is none. +// dash in text, written as the character or as an HTML entity, or nil when +// there is none. func RejectEmDash(name, text string) error { for i, line := range strings.Split(text, "\n") { - if strings.Contains(line, emDash) { + if strings.Contains(line, emDash) || emDashEntity.MatchString(line) { return fmt.Errorf("%s:%d: no em dashes, please", name, i+1) } } diff --git a/internal/mdx/emdash_test.go b/internal/mdx/emdash_test.go index 0682d9f..8e13178 100644 --- a/internal/mdx/emdash_test.go +++ b/internal/mdx/emdash_test.go @@ -45,6 +45,27 @@ func TestNoEmDashLeftover(t *testing.T) { } } +// TestRejectEmDashEntities checks that the entities of the em dash are refused too. +func TestRejectEmDashEntities(t *testing.T) { + for _, in := range []string{"a — b", "a &MDASH; b", "a — b", "a — b", "a — b"} { + if err := RejectEmDash("f.md", in); err == nil { + t.Errorf("RejectEmDash(%q) = nil, want an error", in) + } + } + if err := RejectEmDash("f.md", "a – b & c"); err != nil { + t.Errorf("RejectEmDash(ndash) = %v", err) + } +} + +// TestNoEmDashErrorSaysBody checks that a leftover is reported as a line of the +// body, which is how the caller counts it. +func TestNoEmDashErrorSaysBody(t *testing.T) { + _, err := NoEmDash("fine\nsome words \u2014 and then more\n") + if err == nil || !strings.Contains(err.Error(), "the body:2") { + t.Errorf("NoEmDash() error = %v, want it to say the body:2", err) + } +} + // TestRejectEmDash checks that the error names the file and the line. func TestRejectEmDash(t *testing.T) { if err := RejectEmDash("content/en/x.md", "clean\ntext\n"); err != nil { diff --git a/internal/mdx/mdx.go b/internal/mdx/mdx.go index f79f644..4eb323b 100644 --- a/internal/mdx/mdx.go +++ b/internal/mdx/mdx.go @@ -6,6 +6,7 @@ package mdx import ( "bytes" "fmt" + "regexp" "strconv" "strings" @@ -42,9 +43,12 @@ func Render(md string) (string, []Heading, error) { src := []byte(md) doc := engine.Parser().Parse(text.NewReader(src)) + used, err := explicitIDs(doc) + if err != nil { + return "", nil, err + } var headings []Heading - used := make(map[string]int) - err := ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) { + err = ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) { h, ok := n.(*ast.Heading) if !ok || !entering { return ast.WalkContinue, nil @@ -53,9 +57,12 @@ func Render(md string) (string, []Heading, error) { id := "" if explicit, ok := h.AttributeString("id"); ok { id = attributeText(explicit) - used[id]++ } else if h.Level >= 2 { - id = uniqueID(Slug(plain), used) + base := Slug(plain) + if base == "" { + base = "section" + } + id = uniqueID(base, used) h.SetAttributeString("id", []byte(id)) } headings = append(headings, Heading{Level: h.Level, ID: id, Text: plain}) @@ -83,13 +90,47 @@ func attributeText(value any) string { return "" } -// uniqueID returns id, or id with a numeric suffix when it was already used. -func uniqueID(id string, used map[string]int) string { - used[id]++ - if used[id] == 1 { - return id +// explicitID is what an id set with {#id} may look like: it ends up in +// attributes and in addresses. +var explicitID = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.:-]*$`) + +// explicitIDs returns the ids that headings set for themselves with {#id}. A +// bad id or an id used by two headings is an error. Generated ids avoid these, +// whatever the order of the headings. +func explicitIDs(doc ast.Node) (map[string]bool, error) { + used := map[string]bool{} + err := ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) { + h, ok := n.(*ast.Heading) + if !ok || !entering { + return ast.WalkContinue, nil + } + value, ok := h.AttributeString("id") + if !ok { + return ast.WalkSkipChildren, nil + } + id := attributeText(value) + if !explicitID.MatchString(id) { + return ast.WalkStop, fmt.Errorf("the heading id %q may have only letters, digits and _ . : -", id) + } + if used[id] { + return ast.WalkStop, fmt.Errorf("the heading id %q is used by two headings", id) + } + used[id] = true + return ast.WalkSkipChildren, nil + }) + return used, err +} + +// uniqueID returns base, or base with -2, -3 and so on until the id is not +// used, and marks it as used. +func uniqueID(base string, used map[string]bool) string { + id, n := base, 1 + for used[id] { + n++ + id = base + "-" + strconv.Itoa(n) } - return id + "-" + strconv.Itoa(used[id]) + used[id] = true + return id } // plainText returns the text of a node and its children without any markup. diff --git a/internal/mdx/mdx_test.go b/internal/mdx/mdx_test.go index 3173977..c576ca4 100644 --- a/internal/mdx/mdx_test.go +++ b/internal/mdx/mdx_test.go @@ -99,6 +99,84 @@ func TestRenderExplicitHeadingID(t *testing.T) { } } +// TestRenderRejectsRepeatedExplicitID checks that two headings cannot ask for the same id. +func TestRenderRejectsRepeatedExplicitID(t *testing.T) { + for _, src := range []string{"## Foo {#a}\n\n## Bar {#a}\n"} { + _, _, err := Render(src) + if err == nil || !strings.Contains(err.Error(), "id") { + t.Errorf("Render(%q) error = %v, want an error about the repeated id", src, err) + } + } +} + +// TestRenderGeneratedIDAvoidsLaterExplicit checks that a generated id never takes +// one that a later heading asks for by name, in either order of the two. +func TestRenderGeneratedIDAvoidsLaterExplicit(t *testing.T) { + _, headings, err := Render("## Foo\n\n## Bar {#foo-2}\n\n## Foo\n") + if err != nil { + t.Fatal(err) + } + var ids []string + for _, h := range headings { + ids = append(ids, h.ID) + } + if got, want := strings.Join(ids, ","), "foo,foo-2,foo-3"; got != want { + t.Errorf("ids = %s, want %s", got, want) + } +} + +// TestRenderSuffixClash checks that a numbered suffix never repeats an id. +func TestRenderSuffixClash(t *testing.T) { + _, headings, err := Render("## A\n\n## A\n\n## A 2\n") + if err != nil { + t.Fatal(err) + } + seen := map[string]bool{} + for _, h := range headings { + if seen[h.ID] { + t.Errorf("id %q is used twice: %+v", h.ID, headings) + } + seen[h.ID] = true + } +} + +// TestRenderEmptySlugFallback checks that a heading with no ASCII letters or digits still gets an id. +func TestRenderEmptySlugFallback(t *testing.T) { + html, headings, err := Render("## \u4f60\u597d\n\n## \u4e16\u754c\n") + if err != nil { + t.Fatal(err) + } + if strings.Contains(html, `id=""`) { + t.Errorf("html %q has an empty id", html) + } + if len(headings) != 2 || headings[0].ID != "section" || headings[1].ID != "section-2" { + t.Errorf("headings = %+v, want section and section-2", headings) + } +} + +// TestRenderBadExplicitIDCannotInject checks that a malformed {#id} never ends +// up as an id with quotes, spaces or angle brackets, whether goldmark leaves it +// as text or the id check refuses it; and that sound ids pass. +func TestRenderBadExplicitIDCannotInject(t *testing.T) { + for _, src := range []string{"## A {#a\"b}\n", "## A {#a b}\n", "## A {#}\n"} { + html, headings, err := Render(src) + if err != nil { + continue + } + for _, h := range headings { + if !explicitID.MatchString(h.ID) { + t.Errorf("Render(%q) gave the id %q", src, h.ID) + } + } + if strings.Contains(html, `id="a"b`) || strings.Contains(html, `id="a b`) { + t.Errorf("Render(%q) = %q, the id broke out of its attribute", src, html) + } + } + if _, _, err := Render("## A {#v1.2}\n\n## B {#01}\n\n## C {#a_b:c-d}\n"); err != nil { + t.Errorf("Render(valid ids) error = %v", err) + } +} + // TestRenderHeadingTextIsPlain checks that the heading text drops the markup. func TestRenderHeadingTextIsPlain(t *testing.T) { _, headings, err := Render("## Free *your* mind\n") diff --git a/internal/render/markdown.go b/internal/render/markdown.go new file mode 100644 index 0000000..abc1251 --- /dev/null +++ b/internal/render/markdown.go @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package render + +import ( + "fmt" + "strings" +) + +// SubstituteMarkdown replaces the {{name}} placeholders of Markdown text like +// Substitute does, with two exceptions that let an article show a placeholder +// as it is written: fenced code blocks are left untouched (an unknown +// placeholder in one is not an error), and a placeholder with a backslash in +// front of it is kept, backslash included, which Markdown then renders as the +// plain placeholder. Code spans are substituted: the pages use them to show a +// value in monospace. An unknown placeholder elsewhere is an error that names +// the file. +func SubstituteMarkdown(name, md string, ctx map[string]string) (string, error) { + var out strings.Builder + var text strings.Builder // the lines outside fences, waiting to be substituted + flush := func() error { + if text.Len() == 0 { + return nil + } + replaced, err := substituteUnescaped(name, text.String(), ctx) + if err != nil { + return err + } + out.WriteString(replaced) + text.Reset() + return nil + } + + lines := strings.SplitAfter(md, "\n") + for i := 0; i < len(lines); i++ { + marker, length := fenceStart(lines[i]) + if length == 0 { + text.WriteString(lines[i]) + continue + } + if err := flush(); err != nil { + return "", err + } + out.WriteString(lines[i]) + for i++; i < len(lines); i++ { + out.WriteString(lines[i]) + if fenceEnds(lines[i], marker, length) { + break + } + } + } + if err := flush(); err != nil { + return "", err + } + return out.String(), nil +} + +// fenceStart returns the fence character and its length when the line opens a +// fenced code block (three or more backticks or tildes, indented by at most +// three spaces), and a zero length otherwise. +func fenceStart(line string) (marker byte, length int) { + rest := strings.TrimLeft(line, " ") + if len(line)-len(rest) > 3 || rest == "" || (rest[0] != '`' && rest[0] != '~') { + return 0, 0 + } + marker = rest[0] + for length < len(rest) && rest[length] == marker { + length++ + } + if length < 3 { + return 0, 0 + } + return marker, length +} + +// fenceEnds reports whether the line closes a fence opened with length +// characters of marker: at least as many, and nothing else on the line. +func fenceEnds(line string, marker byte, length int) bool { + rest := strings.TrimSpace(line) + if len(rest) < length { + return false + } + return strings.Trim(rest, string(marker)) == "" +} + +// substituteUnescaped is Substitute that leaves alone the placeholders that +// have a backslash right before them. +func substituteUnescaped(name, text string, ctx map[string]string) (string, error) { + var out strings.Builder + last := 0 + for _, loc := range placeholder.FindAllStringSubmatchIndex(text, -1) { + start, end := loc[0], loc[1] + out.WriteString(text[last:start]) + last = end + if start > 0 && text[start-1] == '\\' { + out.WriteString(text[start:end]) + continue + } + key := text[loc[2]:loc[3]] + value, ok := ctx[key] + if !ok { + return "", fmt.Errorf("%s: unknown placeholder {{%s}}", name, key) + } + out.WriteString(value) + } + out.WriteString(text[last:]) + return out.String(), nil +} diff --git a/internal/render/render_test.go b/internal/render/render_test.go index 5d2100a..9e97c33 100644 --- a/internal/render/render_test.go +++ b/internal/render/render_test.go @@ -59,6 +59,43 @@ func TestSubstituteValueIsNotRescanned(t *testing.T) { } } +// TestSubstituteMarkdownFences checks that placeholders in fenced code blocks +// stay as written (an unknown one is no error there), while code spans are +// substituted: the pages use them to show a value in monospace. +func TestSubstituteMarkdownFences(t *testing.T) { + ctx := map[string]string{"lang": "en"} + src := "Outside {{lang}} and `{{lang}}`.\n\n```\n{{lang}} {{nope}}\n```\n\n~~~go\n{{lang}}\n~~~\n\nEnd {{lang}}.\n" + got, err := SubstituteMarkdown("page.md", src, ctx) + if err != nil { + t.Fatalf("SubstituteMarkdown() error = %v", err) + } + want := "Outside en and `en`.\n\n```\n{{lang}} {{nope}}\n```\n\n~~~go\n{{lang}}\n~~~\n\nEnd en.\n" + if got != want { + t.Errorf("SubstituteMarkdown() = %q, want %q", got, want) + } +} + +// TestSubstituteMarkdownEscape checks that a backslash before a placeholder +// keeps it as written: Markdown then drops the backslash. +func TestSubstituteMarkdownEscape(t *testing.T) { + got, err := SubstituteMarkdown("page.md", "Write \\{{lang}} for {{lang}}, and \\{{nope}} is fine.\n", map[string]string{"lang": "en"}) + if err != nil { + t.Fatalf("SubstituteMarkdown() error = %v", err) + } + if want := "Write \\{{lang}} for en, and \\{{nope}} is fine.\n"; got != want { + t.Errorf("SubstituteMarkdown() = %q, want %q", got, want) + } +} + +// TestSubstituteMarkdownUnknownOutsideCode checks that an unknown placeholder +// outside a fence still stops the build and names the file. +func TestSubstituteMarkdownUnknownOutsideCode(t *testing.T) { + _, err := SubstituteMarkdown("news/x.md", "ok {{a}}\n\nbad {{nope}}\n", map[string]string{"a": "1"}) + if err == nil || !strings.Contains(err.Error(), "news/x.md") || !strings.Contains(err.Error(), "nope") { + t.Errorf("SubstituteMarkdown() error = %v, want it to name news/x.md and nope", err) + } +} + // TestEscape checks that text escaping leaves quotes alone. func TestEscape(t *testing.T) { if got, want := Escape(`a & b < c > "d" 'e'`), `a & b < c > "d" 'e'`; got != want { diff --git a/internal/site/site.go b/internal/site/site.go index f610721..8a5ca1e 100644 --- a/internal/site/site.go +++ b/internal/site/site.go @@ -47,6 +47,9 @@ func Load(dir string) (Config, map[string]Lang, error) { return Config{}, nil, err } cfg := Config{Strings: stringValues(raw), Assets: map[string]string{}} + if err := checkBaseURL(cfg.Strings["base_url"]); err != nil { + return Config{}, nil, err + } if err := decodeField(raw, "site.json", "languages", &cfg.Languages); err != nil { return Config{}, nil, err } @@ -76,6 +79,15 @@ func Load(dir string) (Config, map[string]Lang, error) { return cfg, langs, nil } +// checkBaseURL returns an error unless the address of the site is absolute and +// has no trailing slash: page addresses and the feed are built by adding to it. +func checkBaseURL(base string) error { + if !(strings.HasPrefix(base, "https://") || strings.HasPrefix(base, "http://")) || strings.HasSuffix(base, "/") { + return fmt.Errorf("site.json: base_url must be an absolute address without a trailing slash, such as https://kmalloc.org") + } + return nil +} + // languageCode is the shape of a language code: "en", "pt-BR". var languageCode = regexp.MustCompile(`^[a-z]{2,3}(-[A-Za-z0-9]+)?$`) diff --git a/internal/site/site_test.go b/internal/site/site_test.go index 188662a..3ac6eba 100644 --- a/internal/site/site_test.go +++ b/internal/site/site_test.go @@ -132,7 +132,7 @@ func TestLoadRejectsEscapingAssets(t *testing.T) { `{"a.txt": "../../etc/hostname"}`, `{"a.txt": "/etc/hostname"}`, } { files := validFiles() - files["site.json"] = `{"languages": ["en", "it"], "assets": ` + assets + `}` + files["site.json"] = `{"base_url": "https://kmalloc.org", "languages": ["en", "it"], "assets": ` + assets + `}` _, _, err := Load(writeSite(t, files)) if err == nil || !strings.Contains(err.Error(), "asset") { t.Errorf("Load(%s) error = %v, want an error about the asset", assets, err) @@ -144,7 +144,7 @@ func TestLoadRejectsEscapingAssets(t *testing.T) { func TestLoadRejectsBadLanguageCodes(t *testing.T) { for _, codes := range []string{`["en", "../x"]`, `["EN"]`, `["en", "en"]`, `["en", "it/x"]`} { files := validFiles() - files["site.json"] = `{"languages": ` + codes + `}` + files["site.json"] = `{"base_url": "https://kmalloc.org", "languages": ` + codes + `}` _, _, err := Load(writeSite(t, files)) if err == nil || !strings.Contains(err.Error(), "language") { t.Errorf("Load(%s) error = %v, want an error about the language code", codes, err) @@ -152,6 +152,19 @@ func TestLoadRejectsBadLanguageCodes(t *testing.T) { } } +// TestLoadNeedsBaseURL checks that the address of the site is required and absolute, +// because the feed and the page addresses are built from it. +func TestLoadNeedsBaseURL(t *testing.T) { + for _, site := range []string{`{"languages": ["en", "it"]}`, `{"base_url": "kmalloc.org", "languages": ["en", "it"]}`, `{"base_url": "https://kmalloc.org/", "languages": ["en", "it"]}`} { + files := validFiles() + files["site.json"] = site + _, _, err := Load(writeSite(t, files)) + if err == nil || !strings.Contains(err.Error(), "base_url") { + t.Errorf("Load(%s) error = %v, want an error about base_url", site, err) + } + } +} + // TestLoadNeedsLanguages checks that an empty language list is an error. func TestLoadNeedsLanguages(t *testing.T) { files := validFiles() From 4ab8f8d12223fca0db20ec5afee92217533d058b Mon Sep 17 00:00:00 2001 From: tgiachi Date: Sun, 15 Nov 2026 10:38:56 +0100 Subject: [PATCH 04/22] fix: find code blocks with the Markdown parser and make the escape work everywhere The hand written fence scanner took a line that starts with a code span for a fence and missed fences in quotes and lists. The code blocks are now the ones goldmark finds. A backslash before a placeholder is turned into a marker that survives the Markdown and the list expansion, so it works in a code span and next to a block name, and two backslashes are a literal one. The em dash rule reports the line of the body, refuses padded entities and checks the header keys by name, the feed leaves protocol-relative links alone, base_url needs a host and a language file cannot set it. --- README.md | 2 +- internal/build/build_test.go | 17 ++++++ internal/build/feed.go | 7 ++- internal/build/feed_test.go | 10 +++ internal/build/pages.go | 11 +++- internal/content/pages.go | 4 +- internal/mdx/emdash.go | 16 +++-- internal/mdx/emdash_test.go | 15 ++++- internal/mdx/mdx_test.go | 64 +++++++++++++++++++ internal/mdx/substitute.go | 58 ++++++++++++++++++ internal/render/escape.go | 59 ++++++++++++++++++ internal/render/markdown.go | 108 --------------------------------- internal/render/render_test.go | 58 +++++++++--------- internal/site/site.go | 9 ++- internal/site/site_test.go | 12 +++- 15 files changed, 297 insertions(+), 153 deletions(-) create mode 100644 internal/mdx/substitute.go create mode 100644 internal/render/escape.go delete mode 100644 internal/render/markdown.go diff --git a/README.md b/README.md index 9ad28fa..06b5921 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ Pages: `Title`, `ID` (the same in every language, it links the translations), `D ### Placeholders -`{{name}}`, with lowercase letters, digits and underscores, is replaced by a string from `site.json` or the language file. An unknown placeholder stops the build and names the file. Fenced code blocks are left as written, so an article can show a placeholder there; elsewhere a backslash before the placeholder (`\{{name}}`) keeps it as written, and Markdown drops the backslash. Code spans are substituted, which is how a page shows a value in monospace. `{{news_list}}`, `{{latest_news}}` and `{{categories_list}}` stand alone in a paragraph and expand to generated lists. +`{{name}}`, with lowercase letters, digits and underscores, is replaced by a string from `site.json` or the language file. An unknown placeholder stops the build and names the file. Fenced code blocks (found the way CommonMark finds them, in lists and quotes too) are left as written, so an article can show a placeholder there. Anywhere else a backslash before the placeholder (`\{{name}}`) keeps it as written, in a code span too; two backslashes are one literal backslash followed by the value. Code spans are otherwise substituted, which is how a page shows a value in monospace. The id of a heading is not checked against ids that a template writes itself, such as the `news` of the latest news block. `{{news_list}}`, `{{latest_news}}` and `{{categories_list}}` stand alone in a paragraph and expand to generated lists. ## Development diff --git a/internal/build/build_test.go b/internal/build/build_test.go index 9d88ad0..1d6623d 100644 --- a/internal/build/build_test.go +++ b/internal/build/build_test.go @@ -407,6 +407,23 @@ func TestBuildPlaceholdersAndCode(t *testing.T) { mustContain(t, "people", got, "en", "escaped {{lang}}.", "{{lang}} {{nope}}", "And en.") } +// TestBuildEscapedPlaceholders checks the escape with block placeholders: an +// escaped one inside a sentence or alone in a paragraph is shown, not expanded +// and not an error, and a code span can show a placeholder as written. +func TestBuildEscapedPlaceholders(t *testing.T) { + files := baseFiles() + files["content/en/people.md"] = page("people", "People", "", "See \\{{news_list}} here.\n\n\\{{latest_news}}\n\nAnd `\\{{lang}}`.\n") + out, err := runBuild(t, files) + if err != nil { + t.Fatalf("Build() error = %v", err) + } + got := readOut(t, out, "people.html") + mustContain(t, "people", got, "See {{news_list}} here.", "

    {{latest_news}}

    ", "{{lang}}") + if strings.Contains(got, "
      ") { + t.Error("an escaped block placeholder was expanded") + } +} + // TestBuildAssetsCopied checks the assets listed in site.json. func TestBuildAssetsCopied(t *testing.T) { out, err := runBuild(t, baseFiles()) diff --git a/internal/build/feed.go b/internal/build/feed.go index 719bd6e..d9adb7c 100644 --- a/internal/build/feed.go +++ b/internal/build/feed.go @@ -3,6 +3,7 @@ package build import ( + "regexp" "strings" "time" @@ -10,6 +11,10 @@ import ( "git.kmalloc.org/squid/motorino/internal/site" ) +// siteRelative matches a link that starts at the root of the site, such as +// href="/people.html", but not one that is relative to the protocol (href="//host/x"). +var siteRelative = regexp.MustCompile(`\b(href|src)="/([^/])`) + // feed returns the RSS 2.0 feed of the latest limit posts, which must be sorted // newest first. Each item carries the full text with absolute links. The // channel is described by the language l, and newsURL is the address of the @@ -21,7 +26,7 @@ func feed(base string, l site.Lang, newsURL string, posts []renderedPost, limit var items strings.Builder for _, p := range posts { link := base + p.Path - text := strings.NewReplacer(`href="/`, `href="`+base+`/`, `src="/`, `src="`+base+`/`).Replace(p.HTML) + text := siteRelative.ReplaceAllString(p.HTML, `$1="`+base+`/$2`) items.WriteString("\n") items.WriteString("" + render.Escape(p.Title) + "\n") items.WriteString("" + link + "\n") diff --git a/internal/build/feed_test.go b/internal/build/feed_test.go index 5c508ab..bf3b932 100644 --- a/internal/build/feed_test.go +++ b/internal/build/feed_test.go @@ -49,6 +49,16 @@ func TestFeedItems(t *testing.T) { } } +// TestFeedLeavesProtocolRelativeLinks checks that "//host/x" is not given the site address. +func TestFeedLeavesProtocolRelativeLinks(t *testing.T) { + items := []renderedPost{{Post: testPost("a", "2026-11-14", "A", ""), HTML: ` p`}} + got := feed("https://kmalloc.org", feedLang(), "/news/", items, 5) + mustContain(t, "feed", got, `src="//cdn.example/x.png"`, `href="https://kmalloc.org/p.html"`) + if strings.Contains(got, "https://kmalloc.org//cdn") { + t.Error("a protocol-relative link got the site address") + } +} + // TestFeedWithoutPosts checks that an empty feed is still a valid channel. func TestFeedWithoutPosts(t *testing.T) { got := feed("https://kmalloc.org", feedLang(), "/news/", nil, 20) diff --git a/internal/build/pages.go b/internal/build/pages.go index c873d09..31849d3 100644 --- a/internal/build/pages.go +++ b/internal/build/pages.go @@ -81,7 +81,7 @@ func (b *builder) renderPage(p content.Page) error { // markdown replaces the placeholders of a source, converts it to HTML and // expands the generated lists. The name is used in error messages. func (b *builder) markdown(name, source, lang string, ctx map[string]string) (string, []mdx.Heading, error) { - text, err := render.SubstituteMarkdown(name, source, ctx) + text, err := mdx.SubstituteMarkdown(name, source, ctx) if err != nil { return "", nil, err } @@ -89,8 +89,13 @@ func (b *builder) markdown(name, source, lang string, ctx map[string]string) (st if err != nil { return "", nil, fmt.Errorf("%s: %w", name, err) } - html, err = b.expandBlocks(name, html, lang, ctx) - return html, headings, err + if html, err = b.expandBlocks(name, html, lang, ctx); err != nil { + return "", nil, err + } + for i := range headings { + headings[i].Text = render.Unescape(headings[i].Text) + } + return render.Unescape(html), headings, nil } // codeElement matches a code element, where a placeholder is shown as written. diff --git a/internal/content/pages.go b/internal/content/pages.go index 696dfe3..5931766 100644 --- a/internal/content/pages.go +++ b/internal/content/pages.go @@ -146,8 +146,8 @@ func loadPage(file, source, defaultLang, lang, rel string) (Page, error) { // The body of the manifesto is the owner's text and is rewritten when the // page is built; the keys of the header are not. for _, key := range []string{"Title", "Description", "Note"} { - if err := mdx.RejectEmDash(source+" (key "+key+")", h.Get(key)); err != nil { - return Page{}, err + if mdx.HasEmDash(h.Get(key)) { + return Page{}, fmt.Errorf("%s: the key %s has an em dash, please remove it", source, key) } } } diff --git a/internal/mdx/emdash.go b/internal/mdx/emdash.go index 932f980..2f6b888 100644 --- a/internal/mdx/emdash.go +++ b/internal/mdx/emdash.go @@ -38,22 +38,30 @@ func NoEmDash(md string) (string, error) { for _, p := range parentheticals { md = strings.ReplaceAll(md, p.from, p.to) } - if err := RejectEmDash("the body", md); err != nil { - return "", err + for i, line := range strings.Split(md, "\n") { + if HasEmDash(line) { + return "", fmt.Errorf("line %d of the body: no em dashes, please", i+1) + } } return md, nil } // emDashEntity matches the HTML entities of the em dash, which would put it // on the page as well. -var emDashEntity = regexp.MustCompile(`(?i)—|—|—`) +var emDashEntity = regexp.MustCompile(`(?i)—|�*8212;|�*2014;`) + +// HasEmDash reports whether text has an em dash, written as the character or as +// an HTML entity. +func HasEmDash(text string) bool { + return strings.Contains(text, emDash) || emDashEntity.MatchString(text) +} // RejectEmDash returns an error naming the file and the line of the first em // dash in text, written as the character or as an HTML entity, or nil when // there is none. func RejectEmDash(name, text string) error { for i, line := range strings.Split(text, "\n") { - if strings.Contains(line, emDash) || emDashEntity.MatchString(line) { + if HasEmDash(line) { return fmt.Errorf("%s:%d: no em dashes, please", name, i+1) } } diff --git a/internal/mdx/emdash_test.go b/internal/mdx/emdash_test.go index 8e13178..dcd1267 100644 --- a/internal/mdx/emdash_test.go +++ b/internal/mdx/emdash_test.go @@ -47,7 +47,7 @@ func TestNoEmDashLeftover(t *testing.T) { // TestRejectEmDashEntities checks that the entities of the em dash are refused too. func TestRejectEmDashEntities(t *testing.T) { - for _, in := range []string{"a — b", "a &MDASH; b", "a — b", "a — b", "a — b"} { + for _, in := range []string{"a — b", "a &MDASH; b", "a — b", "a — b", "a — b", "a — b", "a — b"} { if err := RejectEmDash("f.md", in); err == nil { t.Errorf("RejectEmDash(%q) = nil, want an error", in) } @@ -61,8 +61,17 @@ func TestRejectEmDashEntities(t *testing.T) { // body, which is how the caller counts it. func TestNoEmDashErrorSaysBody(t *testing.T) { _, err := NoEmDash("fine\nsome words \u2014 and then more\n") - if err == nil || !strings.Contains(err.Error(), "the body:2") { - t.Errorf("NoEmDash() error = %v, want it to say the body:2", err) + if err == nil || !strings.Contains(err.Error(), "line 2 of the body") { + t.Errorf("NoEmDash() error = %v, want it to say line 2 of the body", err) + } +} + +// TestHasEmDash checks the test that the header keys use. +func TestHasEmDash(t *testing.T) { + for in, want := range map[string]bool{"plain": false, "a \u2014 b": true, "a — b": true, "a – b": false, "": false} { + if got := HasEmDash(in); got != want { + t.Errorf("HasEmDash(%q) = %v, want %v", in, got, want) + } } } diff --git a/internal/mdx/mdx_test.go b/internal/mdx/mdx_test.go index c576ca4..9d47e40 100644 --- a/internal/mdx/mdx_test.go +++ b/internal/mdx/mdx_test.go @@ -5,6 +5,8 @@ package mdx import ( "strings" "testing" + + "git.kmalloc.org/squid/motorino/internal/render" ) // TestRenderParagraphAndList checks the basic blocks and inline markup. @@ -177,6 +179,68 @@ func TestRenderBadExplicitIDCannotInject(t *testing.T) { } } +// TestSubstituteMarkdownFences checks that placeholders in fenced code blocks +// stay as written, wherever CommonMark finds the block, and that an unknown one +// there is no error. +func TestSubstituteMarkdownFences(t *testing.T) { + ctx := map[string]string{"lang": "en"} + tests := []struct{ name, src, want string }{ + {"backtick fence", "A {{lang}}\n\n```\n{{lang}} {{nope}}\n```\n\nB {{lang}}\n", "A en\n\n```\n{{lang}} {{nope}}\n```\n\nB en\n"}, + {"tilde fence with info", "~~~go\n{{lang}}\n~~~\n{{lang}}\n", "~~~go\n{{lang}}\n~~~\nen\n"}, + {"code span is substituted", "Value `{{lang}}`.\n", "Value `en`.\n"}, + {"fence in a quote", "> ```\n> {{nope}}\n> ```\n\n{{lang}}\n", "> ```\n> {{nope}}\n> ```\n\nen\n"}, + {"fence in a list item", "1. item\n\n ```\n {{nope}}\n ```\n\n{{lang}}\n", "1. item\n\n ```\n {{nope}}\n ```\n\nen\n"}, + {"code span at the start of a line is not a fence", "```x``` and {{lang}}\n\nthen {{lang}}\n", "```x``` and en\n\nthen en\n"}, + {"unclosed fence runs to the end", "{{lang}}\n\n```\n{{nope}}\n{{lang}}\n", "en\n\n```\n{{nope}}\n{{lang}}\n"}, + {"a closer indented four spaces does not close", "```\n ```\n{{nope}}\n```\n{{lang}}\n", "```\n ```\n{{nope}}\n```\nen\n"}, + {"a closer may be longer", "```\n{{nope}}\n`````\n{{lang}}\n", "```\n{{nope}}\n`````\nen\n"}, + {"CRLF line endings", "{{lang}}\r\n\r\n```\r\n{{nope}}\r\n```\r\n{{lang}}\r\n", "en\r\n\r\n```\r\n{{nope}}\r\n```\r\nen\r\n"}, + {"empty fence", "```\n```\n{{lang}}\n", "```\n```\nen\n"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := SubstituteMarkdown("page.md", tt.src, ctx) + if err != nil { + t.Fatalf("SubstituteMarkdown() error = %v", err) + } + if got != tt.want { + t.Errorf("SubstituteMarkdown() = %q, want %q", got, tt.want) + } + }) + } +} + +// TestSubstituteMarkdownUnknownOutsideFence checks that an unknown placeholder +// outside a fence still stops the build and names the file. +func TestSubstituteMarkdownUnknownOutsideFence(t *testing.T) { + _, err := SubstituteMarkdown("news/x.md", "```\n{{a}}\n```\n\nbad {{nope}}\n", map[string]string{}) + if err == nil || !strings.Contains(err.Error(), "news/x.md") || !strings.Contains(err.Error(), "nope") { + t.Errorf("SubstituteMarkdown() error = %v, want it to name news/x.md and nope", err) + } +} + +// TestSubstituteMarkdownEscapeEverywhere checks that a backslash keeps a +// placeholder as written in text, in a code span and next to a block name, and +// that the written form comes back through render.Unescape. +func TestSubstituteMarkdownEscapeEverywhere(t *testing.T) { + src := "Text \\{{lang}} and `\\{{lang}}` and \\{{news_list}}.\n" + got, err := SubstituteMarkdown("page.md", src, map[string]string{"lang": "en"}) + if err != nil { + t.Fatal(err) + } + html, _, err := Render(got) + if err != nil { + t.Fatal(err) + } + if strings.Contains(html, "{{") { + t.Errorf("html %q still has braces before Unescape", html) + } + want := "

      Text {{lang}} and {{lang}} and {{news_list}}.

      \n" + if out := render.Unescape(html); out != want { + t.Errorf("Unescape(html) = %q, want %q", out, want) + } +} + // TestRenderHeadingTextIsPlain checks that the heading text drops the markup. func TestRenderHeadingTextIsPlain(t *testing.T) { _, headings, err := Render("## Free *your* mind\n") diff --git a/internal/mdx/substitute.go b/internal/mdx/substitute.go new file mode 100644 index 0000000..6976f97 --- /dev/null +++ b/internal/mdx/substitute.go @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package mdx + +import ( + "strings" + + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/text" + + "git.kmalloc.org/squid/motorino/internal/render" +) + +// SubstituteMarkdown replaces the {{name}} placeholders of Markdown source like +// render.SubstituteEscaped does, but leaves the content of fenced code blocks +// as written, so an article can show a placeholder there. The blocks are found +// by the Markdown parser, so they are the ones CommonMark sees, in lists and in +// quotes too. Code spans are substituted: the pages use them to show a value in +// monospace. The result still has the markers of escaped placeholders; turn +// them back with render.Unescape after the Markdown is rendered. +func SubstituteMarkdown(name, md string, ctx map[string]string) (string, error) { + src := []byte(md) + doc := engine.Parser().Parse(text.NewReader(src)) + + type span struct{ start, stop int } + var fenced []span + _ = ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) { + if block, ok := n.(*ast.FencedCodeBlock); ok && entering { + lines := block.Lines() + for i := 0; i < lines.Len(); i++ { + segment := lines.At(i) + fenced = append(fenced, span{segment.Start, segment.Stop}) + } + } + return ast.WalkContinue, nil + }) + + var out strings.Builder + pos := 0 + for _, s := range fenced { + if s.start < pos { + continue + } + replaced, err := render.SubstituteEscaped(name, md[pos:s.start], ctx) + if err != nil { + return "", err + } + out.WriteString(replaced) + out.WriteString(md[s.start:s.stop]) + pos = s.stop + } + replaced, err := render.SubstituteEscaped(name, md[pos:], ctx) + if err != nil { + return "", err + } + out.WriteString(replaced) + return out.String(), nil +} diff --git a/internal/render/escape.go b/internal/render/escape.go new file mode 100644 index 0000000..19caa0a --- /dev/null +++ b/internal/render/escape.go @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package render + +import ( + "fmt" + "regexp" + "strings" +) + +// The markers that stand for an escaped placeholder while the Markdown is +// rendered. They are private use characters, which neither Markdown nor the +// templates treat in any special way. +const ( + escapeOpen = "" + escapeClose = "" +) + +// escaped matches a marker with the name of the placeholder it stands for. +var escaped = regexp.MustCompile(escapeOpen + `([a-z0-9_]+)` + escapeClose) + +// SubstituteEscaped replaces the {{name}} placeholders of text like Substitute +// does, except those with a backslash right before them (an odd number of +// backslashes: two make a literal backslash). The backslash is dropped and the +// placeholder is replaced by a marker that Unescape turns back into the written +// placeholder once the Markdown has been rendered. An unknown placeholder is an +// error that names the file; an escaped one is not looked up. +func SubstituteEscaped(name, text string, ctx map[string]string) (string, error) { + var out strings.Builder + last := 0 + for _, loc := range placeholder.FindAllStringSubmatchIndex(text, -1) { + start, end := loc[0], loc[1] + key := text[loc[2]:loc[3]] + slashes := 0 + for i := start - 1; i >= last && text[i] == '\\'; i-- { + slashes++ + } + if slashes%2 == 1 { + out.WriteString(text[last : start-1]) + out.WriteString(escapeOpen + key + escapeClose) + } else { + value, ok := ctx[key] + if !ok { + return "", fmt.Errorf("%s: unknown placeholder {{%s}}", name, key) + } + out.WriteString(text[last:start]) + out.WriteString(value) + } + last = end + } + out.WriteString(text[last:]) + return out.String(), nil +} + +// Unescape turns the markers left by SubstituteEscaped back into the +// placeholders as they were written. +func Unescape(s string) string { + return escaped.ReplaceAllString(s, "{{$1}}") +} diff --git a/internal/render/markdown.go b/internal/render/markdown.go deleted file mode 100644 index abc1251..0000000 --- a/internal/render/markdown.go +++ /dev/null @@ -1,108 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -package render - -import ( - "fmt" - "strings" -) - -// SubstituteMarkdown replaces the {{name}} placeholders of Markdown text like -// Substitute does, with two exceptions that let an article show a placeholder -// as it is written: fenced code blocks are left untouched (an unknown -// placeholder in one is not an error), and a placeholder with a backslash in -// front of it is kept, backslash included, which Markdown then renders as the -// plain placeholder. Code spans are substituted: the pages use them to show a -// value in monospace. An unknown placeholder elsewhere is an error that names -// the file. -func SubstituteMarkdown(name, md string, ctx map[string]string) (string, error) { - var out strings.Builder - var text strings.Builder // the lines outside fences, waiting to be substituted - flush := func() error { - if text.Len() == 0 { - return nil - } - replaced, err := substituteUnescaped(name, text.String(), ctx) - if err != nil { - return err - } - out.WriteString(replaced) - text.Reset() - return nil - } - - lines := strings.SplitAfter(md, "\n") - for i := 0; i < len(lines); i++ { - marker, length := fenceStart(lines[i]) - if length == 0 { - text.WriteString(lines[i]) - continue - } - if err := flush(); err != nil { - return "", err - } - out.WriteString(lines[i]) - for i++; i < len(lines); i++ { - out.WriteString(lines[i]) - if fenceEnds(lines[i], marker, length) { - break - } - } - } - if err := flush(); err != nil { - return "", err - } - return out.String(), nil -} - -// fenceStart returns the fence character and its length when the line opens a -// fenced code block (three or more backticks or tildes, indented by at most -// three spaces), and a zero length otherwise. -func fenceStart(line string) (marker byte, length int) { - rest := strings.TrimLeft(line, " ") - if len(line)-len(rest) > 3 || rest == "" || (rest[0] != '`' && rest[0] != '~') { - return 0, 0 - } - marker = rest[0] - for length < len(rest) && rest[length] == marker { - length++ - } - if length < 3 { - return 0, 0 - } - return marker, length -} - -// fenceEnds reports whether the line closes a fence opened with length -// characters of marker: at least as many, and nothing else on the line. -func fenceEnds(line string, marker byte, length int) bool { - rest := strings.TrimSpace(line) - if len(rest) < length { - return false - } - return strings.Trim(rest, string(marker)) == "" -} - -// substituteUnescaped is Substitute that leaves alone the placeholders that -// have a backslash right before them. -func substituteUnescaped(name, text string, ctx map[string]string) (string, error) { - var out strings.Builder - last := 0 - for _, loc := range placeholder.FindAllStringSubmatchIndex(text, -1) { - start, end := loc[0], loc[1] - out.WriteString(text[last:start]) - last = end - if start > 0 && text[start-1] == '\\' { - out.WriteString(text[start:end]) - continue - } - key := text[loc[2]:loc[3]] - value, ok := ctx[key] - if !ok { - return "", fmt.Errorf("%s: unknown placeholder {{%s}}", name, key) - } - out.WriteString(value) - } - out.WriteString(text[last:]) - return out.String(), nil -} diff --git a/internal/render/render_test.go b/internal/render/render_test.go index 9e97c33..c4e9bd0 100644 --- a/internal/render/render_test.go +++ b/internal/render/render_test.go @@ -59,40 +59,42 @@ func TestSubstituteValueIsNotRescanned(t *testing.T) { } } -// TestSubstituteMarkdownFences checks that placeholders in fenced code blocks -// stay as written (an unknown one is no error there), while code spans are -// substituted: the pages use them to show a value in monospace. -func TestSubstituteMarkdownFences(t *testing.T) { +// TestSubstituteEscapedKeepsEscapes checks that a backslash before a placeholder +// keeps it as written: it becomes a marker that Unescape turns back into the +// placeholder after the Markdown is rendered. An unknown escaped one is no error. +func TestSubstituteEscapedKeepsEscapes(t *testing.T) { + got, err := SubstituteEscaped("page.md", "Write \\{{lang}} for {{lang}}, and \\{{nope}}.", map[string]string{"lang": "en"}) + if err != nil { + t.Fatalf("SubstituteEscaped() error = %v", err) + } + if strings.Contains(got, "{{") || strings.Contains(got, "\\") { + t.Errorf("SubstituteEscaped() = %q, want no braces and no backslash left", got) + } + if want := "Write {{lang}} for en, and {{nope}}."; Unescape(got) != want { + t.Errorf("Unescape(%q) = %q, want %q", got, Unescape(got), want) + } +} + +// TestSubstituteEscapedBackslashParity checks that an escaped backslash does not +// escape the placeholder: two backslashes are one literal backslash. +func TestSubstituteEscapedBackslashParity(t *testing.T) { ctx := map[string]string{"lang": "en"} - src := "Outside {{lang}} and `{{lang}}`.\n\n```\n{{lang}} {{nope}}\n```\n\n~~~go\n{{lang}}\n~~~\n\nEnd {{lang}}.\n" - got, err := SubstituteMarkdown("page.md", src, ctx) - if err != nil { - t.Fatalf("SubstituteMarkdown() error = %v", err) + even, err := SubstituteEscaped("f", "\\\\{{lang}}", ctx) + if err != nil || even != "\\\\en" { + t.Errorf("two backslashes: got %q, %v, want the value after both backslashes", even, err) } - want := "Outside en and `en`.\n\n```\n{{lang}} {{nope}}\n```\n\n~~~go\n{{lang}}\n~~~\n\nEnd en.\n" - if got != want { - t.Errorf("SubstituteMarkdown() = %q, want %q", got, want) + odd, err := SubstituteEscaped("f", "\\\\\\{{lang}}", ctx) + if err != nil || Unescape(odd) != "\\\\{{lang}}" { + t.Errorf("three backslashes: got %q, %v, want two backslashes then the placeholder", Unescape(odd), err) } } -// TestSubstituteMarkdownEscape checks that a backslash before a placeholder -// keeps it as written: Markdown then drops the backslash. -func TestSubstituteMarkdownEscape(t *testing.T) { - got, err := SubstituteMarkdown("page.md", "Write \\{{lang}} for {{lang}}, and \\{{nope}} is fine.\n", map[string]string{"lang": "en"}) - if err != nil { - t.Fatalf("SubstituteMarkdown() error = %v", err) - } - if want := "Write \\{{lang}} for en, and \\{{nope}} is fine.\n"; got != want { - t.Errorf("SubstituteMarkdown() = %q, want %q", got, want) - } -} - -// TestSubstituteMarkdownUnknownOutsideCode checks that an unknown placeholder -// outside a fence still stops the build and names the file. -func TestSubstituteMarkdownUnknownOutsideCode(t *testing.T) { - _, err := SubstituteMarkdown("news/x.md", "ok {{a}}\n\nbad {{nope}}\n", map[string]string{"a": "1"}) +// TestSubstituteEscapedUnknown checks that an unknown placeholder still stops the +// build and names the file. +func TestSubstituteEscapedUnknown(t *testing.T) { + _, err := SubstituteEscaped("news/x.md", "ok {{a}} bad {{nope}}", map[string]string{"a": "1"}) if err == nil || !strings.Contains(err.Error(), "news/x.md") || !strings.Contains(err.Error(), "nope") { - t.Errorf("SubstituteMarkdown() error = %v, want it to name news/x.md and nope", err) + t.Errorf("SubstituteEscaped() error = %v, want it to name news/x.md and nope", err) } } diff --git a/internal/site/site.go b/internal/site/site.go index 8a5ca1e..98e5549 100644 --- a/internal/site/site.go +++ b/internal/site/site.go @@ -7,6 +7,7 @@ package site import ( "encoding/json" "fmt" + "net/url" "os" "path/filepath" "regexp" @@ -82,8 +83,9 @@ func Load(dir string) (Config, map[string]Lang, error) { // checkBaseURL returns an error unless the address of the site is absolute and // has no trailing slash: page addresses and the feed are built by adding to it. func checkBaseURL(base string) error { - if !(strings.HasPrefix(base, "https://") || strings.HasPrefix(base, "http://")) || strings.HasSuffix(base, "/") { - return fmt.Errorf("site.json: base_url must be an absolute address without a trailing slash, such as https://kmalloc.org") + u, err := url.Parse(base) + if err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" || strings.HasSuffix(base, "/") { + return fmt.Errorf("site.json: base_url must be an absolute address with a host and no trailing slash, such as https://kmalloc.org") } return nil } @@ -133,6 +135,9 @@ func loadLang(dir, code string) (Lang, error) { if lang.Strings["lang_name"] == "" { return Lang{}, fmt.Errorf("%s: missing lang_name", name) } + if _, ok := lang.Strings["base_url"]; ok { + return Lang{}, fmt.Errorf("%s: base_url is set in site.json only", name) + } if _, ok := raw["nav"]; !ok { return Lang{}, fmt.Errorf("%s: missing nav", name) } diff --git a/internal/site/site_test.go b/internal/site/site_test.go index 3ac6eba..13261c2 100644 --- a/internal/site/site_test.go +++ b/internal/site/site_test.go @@ -155,7 +155,7 @@ func TestLoadRejectsBadLanguageCodes(t *testing.T) { // TestLoadNeedsBaseURL checks that the address of the site is required and absolute, // because the feed and the page addresses are built from it. func TestLoadNeedsBaseURL(t *testing.T) { - for _, site := range []string{`{"languages": ["en", "it"]}`, `{"base_url": "kmalloc.org", "languages": ["en", "it"]}`, `{"base_url": "https://kmalloc.org/", "languages": ["en", "it"]}`} { + for _, site := range []string{`{"languages": ["en", "it"]}`, `{"base_url": "kmalloc.org", "languages": ["en", "it"]}`, `{"base_url": "https://kmalloc.org/", "languages": ["en", "it"]}`, `{"base_url": "https://", "languages": ["en", "it"]}`, `{"base_url": "https:///x", "languages": ["en", "it"]}`} { files := validFiles() files["site.json"] = site _, _, err := Load(writeSite(t, files)) @@ -165,6 +165,16 @@ func TestLoadNeedsBaseURL(t *testing.T) { } } +// TestLoadLanguageCannotOverrideBaseURL checks that the address of the site has one source. +func TestLoadLanguageCannotOverrideBaseURL(t *testing.T) { + files := validFiles() + files["lang/it.json"] = `{"lang_name": "Italiano", "base_url": "https://example.org", "nav": {}}` + _, _, err := Load(writeSite(t, files)) + if err == nil || !strings.Contains(err.Error(), "lang/it.json") || !strings.Contains(err.Error(), "base_url") { + t.Errorf("Load() error = %v, want it to name lang/it.json and base_url", err) + } +} + // TestLoadNeedsLanguages checks that an empty language list is an error. func TestLoadNeedsLanguages(t *testing.T) { files := validFiles() From f92d5e8cb44d9c643f09786cb5929ed2ca9058c1 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sun, 15 Nov 2026 10:40:28 +0100 Subject: [PATCH 05/22] chore(release): 1.0.1 [skip ci] ## [1.0.1](https://git.kmalloc.org/squid/motorino/compare/v1.0.0...v1.0.1) (2026-11-15) ### Bug Fixes * close the minor findings of the first review ([917683a](https://git.kmalloc.org/squid/motorino/commit/917683afa491c7f0b4331757d6066d215b5f19d9)) * find code blocks with the Markdown parser and make the escape work everywhere ([4ab8f8d](https://git.kmalloc.org/squid/motorino/commit/4ab8f8d12223fca0db20ec5afee92217533d058b)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77eb11c..66ddd0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [1.0.1](https://git.kmalloc.org/squid/motorino/compare/v1.0.0...v1.0.1) (2026-11-15) + +### Bug Fixes + +* close the minor findings of the first review ([917683a](https://git.kmalloc.org/squid/motorino/commit/917683afa491c7f0b4331757d6066d215b5f19d9)) +* find code blocks with the Markdown parser and make the escape work everywhere ([4ab8f8d](https://git.kmalloc.org/squid/motorino/commit/4ab8f8d12223fca0db20ec5afee92217533d058b)) + ## 1.0.0 (2026-11-15) ### Features From 1505e5d7820a3b6f24a0f2a2ea8c8dc29219436d Mon Sep 17 00:00:00 2001 From: tgiachi Date: Sun, 15 Nov 2026 17:44:40 +0100 Subject: [PATCH 06/22] feat(nav): add an optional projects page to the menu kmalloc.org wants a "Projects" entry between People and Services, linking to a page listing the software it currently builds. Adds "projects" to the menu, the linked pages and the required pages, and keeps the test fixtures (and the main_test page count) in sync. --- cmd/motorino/main_test.go | 4 ++-- internal/build/build.go | 2 +- internal/build/context.go | 4 ++-- internal/build/fixture_test.go | 2 ++ 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/cmd/motorino/main_test.go b/cmd/motorino/main_test.go index 4090294..cd8f08a 100644 --- a/cmd/motorino/main_test.go +++ b/cmd/motorino/main_test.go @@ -28,7 +28,7 @@ func miniSite(t *testing.T) string { "templates/pages/page.html": "

      {{page_title}}

      {{body}}\n", "templates/post.html": "{{post_body}}\n", "templates/category.html": "{{category_posts}}\n", } - for _, id := range []string{"home", "manifesto", "people", "services", "news", "history"} { + for _, id := range []string{"home", "manifesto", "people", "projects", "services", "news", "history"} { name := id switch id { case "home": @@ -71,7 +71,7 @@ func TestRunBuildSuccess(t *testing.T) { if code != 0 { t.Fatalf("run(build) = %d, stderr %q", code, errOut.String()) } - if want := "built 6 pages into " + outDir; !strings.Contains(out.String(), want) { + if want := "built 7 pages into " + outDir; !strings.Contains(out.String(), want) { t.Errorf("output %q does not contain %q", out.String(), want) } if _, err := os.Stat(filepath.Join(outDir, "index.html")); err != nil { diff --git a/internal/build/build.go b/internal/build/build.go index a6d2ea3..d43a902 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -28,7 +28,7 @@ type Options struct { // requiredPages are the IDs every site must have in its default language, // because the menu and the footer link to them. -var requiredPages = []string{"home", "manifesto", "people", "services", "news", "history"} +var requiredPages = []string{"home", "manifesto", "people", "projects", "services", "news", "history"} // newsOnHome is how many news the home page lists; feedLimit is how many the feed has. const ( diff --git a/internal/build/context.go b/internal/build/context.go index 0b4d7df..a46a049 100644 --- a/internal/build/context.go +++ b/internal/build/context.go @@ -10,10 +10,10 @@ import ( ) // menuPages are the IDs of the pages in the menu; the current one is marked. -var menuPages = []string{"home", "manifesto", "people", "services", "news"} +var menuPages = []string{"home", "manifesto", "people", "projects", "services", "news"} // linkedPages are the IDs the templates link to with url_. -var linkedPages = []string{"manifesto", "people", "services", "news", "history"} +var linkedPages = []string{"manifesto", "people", "projects", "services", "news", "history"} // blockNames are the placeholders that expand to generated HTML. Until they // are expanded they stand for themselves, so a page can hold them. diff --git a/internal/build/fixture_test.go b/internal/build/fixture_test.go index 815888e..e02e0ac 100644 --- a/internal/build/fixture_test.go +++ b/internal/build/fixture_test.go @@ -94,12 +94,14 @@ func baseFiles() map[string]string { "content/en/manifesto.md": page("manifesto", "Manifesto", "Layout: manifesto\n", "## 01 — Antifascist\n\nText one.\n\n## Our commitments\n\nText two.\n"), "content/en/manifesto-history.md": page("history", "Manifesto history", "", "Versions.\n"), "content/en/people.md": page("people", "People", "", "## squid\n\nI run it.\n"), + "content/en/projects.md": page("projects", "Projects", "", "## motorino\n\nThis.\n"), "content/en/services.md": page("services", "Services", "Switch: it=/it/servizi.html\n", "Write to **{{contact_shown}}**.\n"), "content/en/news/index.md": page("news", "News", "", "{{news_list}}\n\n{{categories_list}}\n"), "content/it/index.md": page("home", "kmalloc.org", "Layout: bare\n", "{{latest_news}}\n"), "content/it/manifesto.md": page("manifesto", "Manifesto", "Layout: manifesto\n", "## 01 — Antifascisti\n\nTesto.\n"), "content/it/cronologia-manifesto.md": page("history", "Cronologia", "", "Versioni.\n"), "content/it/persone.md": page("people", "Persone", "", "Io.\n"), + "content/it/progetti.md": page("projects", "Progetti", "", "## motorino\n\nQuesto.\n"), "content/it/servizi.md": page("services", "Servizi", "", "Scrivi a {{contact_shown}}.\n"), "content/es/manifesto.md": page("manifesto", "Manifiesto", "Layout: manifesto\nNote: Traduccion de la version 1.0.\nManifestoVersion: 1.0\nManifestoDate: 2026-11-12\n", "## 01 — Antifascistas\n\nTexto.\n"), From d7f076659ae7b9c7a53ccd1b00a846d66134f09b Mon Sep 17 00:00:00 2001 From: tgiachi Date: Sun, 15 Nov 2026 17:44:50 +0100 Subject: [PATCH 07/22] chore(release): 1.1.0 [skip ci] ## [1.1.0](https://git.kmalloc.org/squid/motorino/compare/v1.0.1...v1.1.0) (2026-11-15) ### Features * add an optional projects page to the menu (1505e5d) --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66ddd0a..2e89b5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## [1.1.0](https://git.kmalloc.org/squid/motorino/compare/v1.0.1...v1.1.0) (2026-11-15) + +### Features + +* add an optional projects page to the menu ([1505e5d](https://git.kmalloc.org/squid/motorino/commit/1505e5d)) + ## [1.0.1](https://git.kmalloc.org/squid/motorino/compare/v1.0.0...v1.0.1) (2026-11-15) ### Bug Fixes From eb2a8dc9b4437040d2192d7c8af69dc5eaa697f0 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Sun, 15 Nov 2026 18:08:42 +0100 Subject: [PATCH 08/22] feat: take the menu from the site, not from the code The menu entries were three lists in Go (the menu, the linked pages and the required pages), so a new menu entry meant a new motorino release. Now every page ID gets url_ and cur_ for the templates, and header.html alone decides what is in the menu and in which order. Only the home page and the news section stay required, since motorino builds them itself. Output of the real kmalloc.org site is byte for byte the same as 1.1.0. --- internal/build/build.go | 13 +++++++++++-- internal/build/build_test.go | 30 ++++++++++++++++++++++++++++++ internal/build/context.go | 20 ++++++++------------ internal/build/pages.go | 14 +------------- 4 files changed, 50 insertions(+), 27 deletions(-) diff --git a/internal/build/build.go b/internal/build/build.go index d43a902..a38c938 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -27,8 +27,10 @@ type Options struct { } // requiredPages are the IDs every site must have in its default language, -// because the menu and the footer link to them. -var requiredPages = []string{"home", "manifesto", "people", "projects", "services", "news", "history"} +// because motorino builds the home page and the news section itself. The +// other pages, and the menu that links to them, are the site's own: a template +// can link to any page with url_ and mark it with cur_. +var requiredPages = []string{"home", "news"} // newsOnHome is how many news the home page lists; feedLimit is how many the feed has. const ( @@ -51,6 +53,8 @@ type builder struct { templates map[string]string css string byKey map[pageKey]content.Page + // pageIDs lists the ID of every page, once, whatever the language. + pageIDs []string // files holds the rendered output: relative path to content. files map[string]string // owners names, for each output file, what produces it. @@ -127,8 +131,13 @@ func load(opts Options) (*builder, error) { if b.posts, err = content.LoadPosts(opts.SiteDir); err != nil { return nil, err } + seenID := map[string]bool{} for _, p := range b.pages { b.byKey[pageKey{p.ID, p.Lang}] = p + if !seenID[p.ID] { + seenID[p.ID] = true + b.pageIDs = append(b.pageIDs, p.ID) + } for lang := range p.Switch { if _, known := langs[lang]; !known { return nil, fmt.Errorf("%s: Switch names the language %q, which is not in site.json", p.Source, lang) diff --git a/internal/build/build_test.go b/internal/build/build_test.go index 1d6623d..2d750ba 100644 --- a/internal/build/build_test.go +++ b/internal/build/build_test.go @@ -466,3 +466,33 @@ func TestBuildErrorsOnMissingTemplate(t *testing.T) { t.Errorf("Build() error = %v, want it to name pages/manifesto", err) } } + +// TestBuildMenuComesFromTheSite checks that a page with an ID motorino has never +// heard of can be linked and marked from a template, with no change to the code. +func TestBuildMenuComesFromTheSite(t *testing.T) { + files := baseFiles() + files["content/en/labs.md"] = page("labs", "Labs", "", "Experiments.\n") + files["templates/header.html"] = `
      +Labs +News +
      +` + out, err := runBuild(t, files) + if err != nil { + t.Fatal(err) + } + mustContain(t, "labs.html", readOut(t, out, "labs.html"), + `Labs`, `News`) + mustContain(t, "index.html", readOut(t, out, "index.html"), `Labs`) +} + +// TestBuildPagesBesidesHomeAndNewsAreOptional checks that only the home page and +// the news section are required: the rest of the menu is the site's choice. +func TestBuildPagesBesidesHomeAndNewsAreOptional(t *testing.T) { + files := baseFiles() + delete(files, "content/en/people.md") + delete(files, "content/it/persone.md") + if _, err := runBuild(t, files); err != nil { + t.Errorf("Build() without a people page: %v", err) + } +} diff --git a/internal/build/context.go b/internal/build/context.go index a46a049..1ab2e25 100644 --- a/internal/build/context.go +++ b/internal/build/context.go @@ -9,12 +9,6 @@ import ( "git.kmalloc.org/squid/motorino/internal/render" ) -// menuPages are the IDs of the pages in the menu; the current one is marked. -var menuPages = []string{"home", "manifesto", "people", "projects", "services", "news"} - -// linkedPages are the IDs the templates link to with url_. -var linkedPages = []string{"manifesto", "people", "projects", "services", "news", "history"} - // blockNames are the placeholders that expand to generated HTML. Until they // are expanded they stand for themselves, so a page can hold them. var blockNames = []string{"news_list", "latest_news", "categories_list"} @@ -78,8 +72,10 @@ func (b *builder) languageContext(lang string) map[string]string { for k, label := range l.Nav { ctx["nav_"+k] = render.Escape(label) } - for _, id := range linkedPages { - ctx["url_"+id] = b.pageAddress(lang, id) + for _, id := range b.pageIDs { + if _, ok := b.byKey[pageKey{id, b.defaultLang}]; ok { + ctx["url_"+id] = b.pageAddress(lang, id) + } } ctx["history_attr"] = "" if _, ok := b.byKey[pageKey{"history", lang}]; !ok { @@ -110,10 +106,10 @@ func (b *builder) documentContext(d document) map[string]string { ctx["title_html"] = render.EscapeAttr(title) ctx["page_title"] = render.Escape(d.title) ctx["description_attr"] = render.EscapeAttr(d.description) - for _, k := range menuPages { - ctx["cur_"+k] = "" - if d.nav == k { - ctx["cur_"+k] = ` aria-current="page"` + for _, id := range b.pageIDs { + ctx["cur_"+id] = "" + if d.nav == id { + ctx["cur_"+id] = ` aria-current="page"` } } ctx["lang_links"], ctx["alt_links"] = b.languageLinks(d) diff --git a/internal/build/pages.go b/internal/build/pages.go index 31849d3..b161e8e 100644 --- a/internal/build/pages.go +++ b/internal/build/pages.go @@ -29,9 +29,7 @@ func (b *builder) renderPage(p content.Page) error { lang: p.Lang, id: p.ID, lead: "back", path: p.Path, title: p.Title, description: p.Description, switches: p.Switch, source: p.Source, } - if contains(menuPages, p.ID) { - d.nav = p.ID - } + d.nav = p.ID if p.ID == "home" { d.lead = "home" } @@ -237,13 +235,3 @@ func (b *builder) renderFeed() error { text := feed(b.cfg.Strings["base_url"], b.langs[b.defaultLang], newsPath, b.rendered, feedLimit) return b.publish(outputPath(newsPath+"feed.xml"), "the feed", text) } - -// contains reports whether list has s. -func contains(list []string, s string) bool { - for _, item := range list { - if item == s { - return true - } - } - return false -} From b2f43c34bbb495f77fb0af8105969985d0163673 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Sun, 15 Nov 2026 18:08:42 +0100 Subject: [PATCH 09/22] chore(release): 1.2.0 [skip ci] --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e89b5e..05d521d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## [1.2.0](https://git.kmalloc.org/squid/motorino/compare/v1.1.0...v1.2.0) (2026-11-15) + +### Features + +* take the menu from the site, not from the code ([eb2a8dc](https://git.kmalloc.org/squid/motorino/commit/eb2a8dc)) + ## [1.1.0](https://git.kmalloc.org/squid/motorino/compare/v1.0.1...v1.1.0) (2026-11-15) ### Features From 26edf78449dfd66da49b0e05b5bd79869c2701ca Mon Sep 17 00:00:00 2001 From: tgiachi Date: Mon, 16 Nov 2026 12:04:50 +0100 Subject: [PATCH 10/22] feat(news): hide drafts and future posts, give the news one language - Draft: true and a date in the future keep a post out of the pages, the lists, the categories and the feed. motorino build -drafts (next commit) shows them. - news_language in site.json, and an optional Language key in a post, choose the language of the news pages and of the interface strings they use. The news still exist once, not once per language. - New header keys Mastodon (an https address), Draft and Language; a post knows its reading time and whether its description was written. Refs #9 --- internal/build/blog_test.go | 129 +++++++++++++++++++++++++++++++++ internal/build/build.go | 105 +++++++++++++++++++++++---- internal/build/date.go | 51 +++++++++++++ internal/build/date_test.go | 73 +++++++++++++++++++ internal/build/fixture_test.go | 6 ++ internal/build/news.go | 9 ++- internal/build/news_test.go | 2 +- internal/build/pages.go | 15 ++-- internal/content/posts.go | 82 ++++++++++++++++++++- internal/content/posts_test.go | 105 +++++++++++++++++++++++++++ 10 files changed, 546 insertions(+), 31 deletions(-) create mode 100644 internal/build/blog_test.go create mode 100644 internal/build/date.go create mode 100644 internal/build/date_test.go diff --git a/internal/build/blog_test.go b/internal/build/blog_test.go new file mode 100644 index 0000000..392675f --- /dev/null +++ b/internal/build/blog_test.go @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package build + +import ( + "path/filepath" + "strings" + "testing" + "time" +) + +// fixedNow is the moment the blog tests build at, so that drafts and future +// dates do not depend on the clock. +var fixedNow = time.Date(2026, 11, 15, 12, 0, 0, 0, time.FixedZone("CET", 3600)) + +// withFiles returns the base site with files added or replaced; an empty value removes the file. +func withFiles(extra map[string]string) map[string]string { + files := baseFiles() + for name, content := range extra { + if content == "" { + delete(files, name) + continue + } + files[name] = content + } + return files +} + +// buildWith builds the site made of files with the options changed by tune. +func buildWith(t *testing.T, files map[string]string, tune func(*Options)) (string, error) { + t.Helper() + out := filepath.Join(t.TempDir(), "public") + opts := Options{SiteDir: writeFiles(t, files), OutDir: out, Version: "1.2.3", Now: fixedNow} + if tune != nil { + tune(&opts) + } + _, err := Build(opts) + return out, err +} + +// exists reports whether the output has the file rel. +func exists(t *testing.T, out, rel string) bool { + t.Helper() + _, err := readOutMaybe(out, rel) + return err == nil +} + +// postNews returns the news files used by the draft and future tests. +func postNews() map[string]string { + return map[string]string{ + "news/a-draft.md": "Title: A draft\nDate: 2026-11-10T12:00:00+0100\nCategories: Secret\nDraft: true\n-\nNot yet.\n", + "news/in-future.md": "Title: In the future\nDate: 2026-12-24T12:00:00+0100\nCategories: Later\n-\nSoon.\n", + } +} + +// TestDraftsAndFutureStayHidden checks that a draft and a post dated in the +// future are not published anywhere, and that no category exists only for them. +func TestDraftsAndFutureStayHidden(t *testing.T) { + out, err := buildWith(t, withFiles(postNews()), nil) + if err != nil { + t.Fatalf("Build() error = %v", err) + } + for _, rel := range []string{"news/2026/11/a-draft.html", "news/2026/12/in-future.html", "news/category/secret.html", "news/category/later.html"} { + if exists(t, out, rel) { + t.Errorf("%s is published, but the post is hidden", rel) + } + } + for _, rel := range []string{"news/index.html", "news/feed.xml", "index.html"} { + got := readOut(t, out, rel) + for _, hidden := range []string{"A draft", "In the future", "Secret", "Later"} { + if strings.Contains(got, hidden) { + t.Errorf("%s mentions %q, but the post is hidden", rel, hidden) + } + } + } + mustContain(t, "news index", readOut(t, out, "news/index.html"), "Status page") +} + +// TestDraftsFlagShowsHiddenPosts checks the preview build. +func TestDraftsFlagShowsHiddenPosts(t *testing.T) { + out, err := buildWith(t, withFiles(postNews()), func(o *Options) { o.Drafts = true }) + if err != nil { + t.Fatalf("Build() error = %v", err) + } + for _, rel := range []string{"news/2026/11/a-draft.html", "news/2026/12/in-future.html", "news/category/secret.html"} { + if !exists(t, out, rel) { + t.Errorf("%s is missing from a build with drafts", rel) + } + } +} + +// TestFuturePostAppearsWhenItsDateComes checks the moment of publication. +func TestFuturePostAppearsWhenItsDateComes(t *testing.T) { + later := time.Date(2026, 12, 24, 12, 0, 0, 0, time.FixedZone("CET", 3600)) + out, err := buildWith(t, withFiles(postNews()), func(o *Options) { o.Now = later }) + if err != nil { + t.Fatalf("Build() error = %v", err) + } + if !exists(t, out, "news/2026/12/in-future.html") { + t.Error("a post is published from the moment of its date") + } + if exists(t, out, "news/2026/11/a-draft.html") { + t.Error("a draft stays hidden whatever the date") + } +} + +// TestNewsLanguage checks the language of the news and its errors. +func TestNewsLanguage(t *testing.T) { + tests := []struct { + name string + extra map[string]string + want string + }{ + {"unknown language in site.json", map[string]string{ + "site.json": strings.Replace(baseFiles()["site.json"], `"languages"`, `"news_language": "xx", "languages"`, 1)}, `news_language "xx"`}, + {"no news page in that language", map[string]string{ + "site.json": strings.Replace(baseFiles()["site.json"], `"languages"`, `"news_language": "it", "languages"`, 1)}, `content/it`}, + {"post in a language the site does not have", map[string]string{ + "news/tedesco.md": "Title: Deutsch\nDate: 2026-11-01T12:00:00+0100\nLanguage: de\n-\nText.\n"}, `Language "de"`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := buildWith(t, withFiles(tt.extra), nil) + if err == nil || !strings.Contains(err.Error(), tt.want) { + t.Errorf("error = %v, want one that contains %q", err, tt.want) + } + }) + } +} diff --git a/internal/build/build.go b/internal/build/build.go index a38c938..7044965 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -9,6 +9,7 @@ import ( "path/filepath" "sort" "strings" + "time" "git.kmalloc.org/squid/motorino/internal/content" "git.kmalloc.org/squid/motorino/internal/site" @@ -24,12 +25,19 @@ type Options struct { OutDir string // Version is the version of motorino, shown in the generator meta. Version string + // Now is the moment the site is built at: a post dated later is not + // published yet. The zero value means the current time. + Now time.Time + // Drafts publishes the drafts and the posts dated in the future too, for a + // preview of the site. + Drafts bool } -// requiredPages are the IDs every site must have in its default language, -// because motorino builds the home page and the news section itself. The -// other pages, and the menu that links to them, are the site's own: a template -// can link to any page with url_ and mark it with cur_. +// requiredPages are the IDs every site must have, because motorino builds the +// home page and the news section itself: the home in the default language and +// the news page in the language of the news. The other pages, and the menu +// that links to them, are the site's own: a template can link to any page with +// url_ and mark it with cur_. var requiredPages = []string{"home", "news"} // newsOnHome is how many news the home page lists; feedLimit is how many the feed has. @@ -47,12 +55,20 @@ type builder struct { cfg site.Config langs map[string]site.Lang defaultLang string - pages []content.Page - posts []content.Post - categories []categoryEntry - templates map[string]string - css string - byKey map[pageKey]content.Page + // newsLang is the language the news are written in, and so the language of + // the news pages: the posts, the index, the archives, the categories and + // the feed exist once, in this language. A post can say another one. + newsLang string + // newsPath is the address of the news index, such as "/news/". + newsPath string + // now is the moment the site is built at. + now time.Time + pages []content.Page + posts []content.Post + categories []categoryEntry + templates map[string]string + css string + byKey map[pageKey]content.Page // pageIDs lists the ID of every page, once, whatever the language. pageIDs []string // files holds the rendered output: relative path to content. @@ -121,14 +137,24 @@ func load(opts Options) (*builder, error) { return nil, err } b := &builder{ - opts: opts, cfg: cfg, langs: langs, defaultLang: cfg.Languages[0], + opts: opts, cfg: cfg, langs: langs, defaultLang: cfg.Languages[0], now: opts.Now, byKey: map[pageKey]content.Page{}, files: map[string]string{}, owners: map[string]string{}, langCtx: map[string]map[string]string{}, } + if b.now.IsZero() { + b.now = time.Now() + } + if b.newsLang, err = newsLanguage(cfg); err != nil { + return nil, err + } if b.pages, err = content.LoadPages(opts.SiteDir, cfg.Languages); err != nil { return nil, err } - if b.posts, err = content.LoadPosts(opts.SiteDir); err != nil { + all, err := content.LoadPosts(opts.SiteDir) + if err != nil { + return nil, err + } + if b.posts, err = b.publishedPosts(all); err != nil { return nil, err } seenID := map[string]bool{} @@ -145,13 +171,19 @@ func load(opts Options) (*builder, error) { } } for _, id := range requiredPages { - if _, ok := b.byKey[pageKey{id, b.defaultLang}]; !ok { - return nil, fmt.Errorf("content/%s: there is no page with ID %q", b.defaultLang, id) + lang := b.defaultLang + if id == "news" { + lang = b.newsLang + } + if _, ok := b.byKey[pageKey{id, lang}]; !ok { + return nil, fmt.Errorf("content/%s: there is no page with ID %q", lang, id) } } - if news := b.byKey[pageKey{"news", b.defaultLang}]; !strings.HasSuffix(news.Path, "/") { + news := b.byKey[pageKey{"news", b.newsLang}] + if !strings.HasSuffix(news.Path, "/") { return nil, fmt.Errorf("%s: the news page must be an index (news/index.md), so that the feed has an address next to it", news.Source) } + b.newsPath = news.Path if b.templates, err = readTemplates(opts.SiteDir); err != nil { return nil, err } @@ -164,6 +196,49 @@ func load(opts Options) (*builder, error) { return b, nil } +// newsLanguage returns the language of the news: news_language in site.json, +// else English when the site has it, else the default language. +func newsLanguage(cfg site.Config) (string, error) { + listed := func(code string) bool { + for _, l := range cfg.Languages { + if l == code { + return true + } + } + return false + } + switch code := cfg.Strings["news_language"]; { + case code == "" && listed("en"): + return "en", nil + case code == "": + return cfg.Languages[0], nil + case !listed(code): + return "", fmt.Errorf("site.json: news_language %q is not one of the languages of the site", code) + default: + return code, nil + } +} + +// publishedPosts gives every post the language of the news when it names none, +// checks that the language is one of the site, and keeps the posts that are +// published at the moment of the build: not a draft and not dated in the +// future, unless the build is a preview with drafts. +func (b *builder) publishedPosts(all []content.Post) ([]content.Post, error) { + var out []content.Post + for _, p := range all { + if p.Language == "" { + p.Language = b.newsLang + } + if _, ok := b.langs[p.Language]; !ok { + return nil, fmt.Errorf("%s: Language %q is not one of the languages of site.json", p.Source, p.Language) + } + if b.opts.Drafts || p.Visible(b.now) { + out = append(out, p) + } + } + return out, nil +} + // groupCategories lists the categories of the posts, sorted by name, each with // its posts. func groupCategories(posts []content.Post) []categoryEntry { diff --git a/internal/build/date.go b/internal/build/date.go new file mode 100644 index 0000000..cdf9173 --- /dev/null +++ b/internal/build/date.go @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package build + +import ( + "fmt" + "strconv" + "strings" + "time" + + "git.kmalloc.org/squid/motorino/internal/site" +) + +// longDate writes a date for a reader, in the language l, from the strings +// date_long (a layout with {d}, {month} and {y}) and month_01 to month_12. The +// day, the month and the year are those of the date as written, not converted +// to another time zone. A language without the layout, or without the name of +// the month, gets the ISO date. +func longDate(date time.Time, l site.Lang) string { + layout := l.Strings["date_long"] + month := l.Strings[fmt.Sprintf("month_%02d", int(date.Month()))] + if layout == "" || month == "" { + return date.Format(dateOnly) + } + return strings.NewReplacer( + "{d}", strconv.Itoa(date.Day()), "{month}", month, "{y}", strconv.Itoa(date.Year()), + ).Replace(layout) +} + +// monthYear is the label of an archive month in the language l, from the +// strings date_month_year (a layout with {month} and {y}) and month_01 to +// month_12; without them it is the year and the month, "2026-11". +func monthYear(year, month int, l site.Lang) string { + layout := l.Strings["date_month_year"] + name := l.Strings[fmt.Sprintf("month_%02d", month)] + if layout == "" || name == "" { + return fmt.Sprintf("%04d-%02d", year, month) + } + return strings.NewReplacer("{month}", name, "{y}", strconv.Itoa(year)).Replace(layout) +} + +// readingText is the reading time of a post in the language l, from the string +// reading_time (a sentence with {n}); without the string it is empty and no +// reading time is shown. +func readingText(minutes int, l site.Lang) string { + layout := l.Strings["reading_time"] + if layout == "" { + return "" + } + return strings.ReplaceAll(layout, "{n}", strconv.Itoa(minutes)) +} diff --git a/internal/build/date_test.go b/internal/build/date_test.go new file mode 100644 index 0000000..5aa09d9 --- /dev/null +++ b/internal/build/date_test.go @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package build + +import ( + "testing" + "time" + + "git.kmalloc.org/squid/motorino/internal/site" +) + +// dateLang returns a language with the strings that write a date. +func dateLang(layout string, months ...string) site.Lang { + l := site.Lang{Code: "xx", Strings: map[string]string{}} + if layout != "" { + l.Strings["date_long"] = layout + } + for i, m := range months { + l.Strings["month_"+string(rune('0'+(i+1)/10))+string(rune('0'+(i+1)%10))] = m + } + return l +} + +// allMonths lists twelve month names for a test language. +var allMonths = []string{"gennaio", "febbraio", "marzo", "aprile", "maggio", "giugno", "luglio", "agosto", "settembre", "ottobre", "novembre", "dicembre"} + +// TestLongDate checks the long date in a language, as written in the date. +func TestLongDate(t *testing.T) { + date, err := time.Parse("2006-01-02T15:04:05-0700", "2026-11-05T23:30:00-0500") + if err != nil { + t.Fatal(err) + } + if got := longDate(date, dateLang("{d} {month} {y}", allMonths...)); got != "5 novembre 2026" { + t.Errorf("longDate() = %q, want the day without a zero, the month name and the year", got) + } + if got := longDate(date, dateLang("{month} {d}, {y}", allMonths...)); got != "novembre 5, 2026" { + t.Errorf("longDate() with another layout = %q", got) + } +} + +// TestLongDateFallsBack checks the ISO date when the language has no layout or a month. +func TestLongDateFallsBack(t *testing.T) { + date, _ := time.Parse("2006-01-02T15:04:05-0700", "2026-11-14T12:00:00+0100") + if got := longDate(date, dateLang("")); got != "2026-11-14" { + t.Errorf("no layout: longDate() = %q, want the ISO date", got) + } + if got := longDate(date, dateLang("{d} {month} {y}", "uno", "due")); got != "2026-11-14" { + t.Errorf("a missing month name: longDate() = %q, want the ISO date", got) + } +} + +// TestMonthYear checks the label of an archive month. +func TestMonthYear(t *testing.T) { + l := dateLang("{d} {month} {y}", allMonths...) + l.Strings["date_month_year"] = "{month} {y}" + if got := monthYear(2026, 11, l); got != "novembre 2026" { + t.Errorf("monthYear() = %q", got) + } + if got := monthYear(2026, 3, dateLang("")); got != "2026-03" { + t.Errorf("monthYear() without strings = %q, want 2026-03", got) + } +} + +// TestReadingText checks the reading time sentence and its absence. +func TestReadingText(t *testing.T) { + l := site.Lang{Strings: map[string]string{"reading_time": "{n} min di lettura"}} + if got := readingText(3, l); got != "3 min di lettura" { + t.Errorf("readingText() = %q", got) + } + if got := readingText(3, site.Lang{Strings: map[string]string{}}); got != "" { + t.Errorf("readingText() without a string = %q, want empty", got) + } +} diff --git a/internal/build/fixture_test.go b/internal/build/fixture_test.go index e02e0ac..cf5d456 100644 --- a/internal/build/fixture_test.go +++ b/internal/build/fixture_test.go @@ -46,6 +46,12 @@ func readOut(t *testing.T, out, rel string) string { return string(data) } +// readOutMaybe returns the content of a file of the output, or the error when it is missing. +func readOutMaybe(out, rel string) (string, error) { + data, err := os.ReadFile(filepath.Join(out, filepath.FromSlash(rel))) + return string(data), err +} + // mustContain fails the test for each want that is not in got. func mustContain(t *testing.T, name, got string, wants ...string) { t.Helper() diff --git a/internal/build/news.go b/internal/build/news.go index a9f6e89..93ea37f 100644 --- a/internal/build/news.go +++ b/internal/build/news.go @@ -24,12 +24,13 @@ type renderedPost struct { } // newsItem returns the
    • of a post for a list: the date, the title as a -// link and, when withDescription is set and there is one, the description. News -// are written in English, so on any other language the link says so. +// link and, when withDescription is set and there is one, the description. A +// post is written in its own language, so on a page in another language the +// link says which one. func newsItem(p content.Post, lang string, withDescription bool) string { mark := "" - if lang != "en" { - mark = ` hreflang="en" lang="en"` + if p.Language != "" && p.Language != lang { + mark = fmt.Sprintf(` hreflang="%s" lang="%s"`, p.Language, p.Language) } day := p.Date.Format(dateOnly) item := fmt.Sprintf(`
    • %s`, diff --git a/internal/build/news_test.go b/internal/build/news_test.go index 39bddfe..3960f21 100644 --- a/internal/build/news_test.go +++ b/internal/build/news_test.go @@ -23,7 +23,7 @@ func testPost(slug, day, title, description string, cats ...string) content.Post } return content.Post{ Slug: slug, Path: content.PostPath(date, slug), Title: title, Description: description, - Date: date, Categories: categories, Source: "news/" + slug + ".md", + Language: "en", Date: date, Categories: categories, Source: "news/" + slug + ".md", } } diff --git a/internal/build/pages.go b/internal/build/pages.go index b161e8e..e00812c 100644 --- a/internal/build/pages.go +++ b/internal/build/pages.go @@ -158,11 +158,11 @@ func (b *builder) renderPosts() error { } for _, p := range b.posts { d := document{ - lang: b.defaultLang, nav: "news", lead: "news", path: p.Path, title: p.Title, + lang: p.Language, nav: "news", lead: "news", path: p.Path, title: p.Title, prefixed: true, description: p.Description, source: p.Source, } ctx := b.documentContext(d) - html, _, err := b.markdown(p.Source, p.Body, b.defaultLang, ctx) + html, _, err := b.markdown(p.Source, p.Body, p.Language, ctx) if err != nil { return err } @@ -211,13 +211,13 @@ func (b *builder) renderCategories() error { } for _, c := range b.categories { d := document{ - lang: b.defaultLang, nav: "news", lead: "news", path: categoryPath(c.Slug), title: c.Name, prefixed: true, - description: b.langs[b.defaultLang].Strings["category_h"] + ": " + c.Name, + lang: b.newsLang, nav: "news", lead: "news", path: categoryPath(c.Slug), title: c.Name, prefixed: true, + description: b.langs[b.newsLang].Strings["category_h"] + ": " + c.Name, source: fmt.Sprintf("the page of the category %q", c.Name), } ctx := b.documentContext(d) ctx["category_name"] = render.Escape(c.Name) - ctx["category_posts"] = newsList(c.posts, b.defaultLang) + ctx["category_posts"] = newsList(c.posts, b.newsLang) body, err := render.Substitute("templates/category.html", tmpl, ctx) if err != nil { return err @@ -231,7 +231,6 @@ func (b *builder) renderCategories() error { // renderFeed writes the feed next to the news page. func (b *builder) renderFeed() error { - newsPath := b.byKey[pageKey{"news", b.defaultLang}].Path - text := feed(b.cfg.Strings["base_url"], b.langs[b.defaultLang], newsPath, b.rendered, feedLimit) - return b.publish(outputPath(newsPath+"feed.xml"), "the feed", text) + text := feed(b.cfg.Strings["base_url"], b.langs[b.newsLang], b.newsPath, b.rendered, feedLimit) + return b.publish(outputPath(b.newsPath+"feed.xml"), "the feed", text) } diff --git a/internal/content/posts.go b/internal/content/posts.go index ae5f103..b28b9c8 100644 --- a/internal/content/posts.go +++ b/internal/content/posts.go @@ -4,6 +4,7 @@ package content import ( "fmt" + "net/url" "os" "path/filepath" "regexp" @@ -28,18 +29,54 @@ type Post struct { Date time.Time // Categories are the categories of the post, in the order written. Categories []Category + // HasDescription is true when the header has a Description; without one, + // Description is the first sentence of the body and a page should not show + // it as a summary, because the text starts with it. + HasDescription bool + // Mastodon is the address of the post on Mastodon, where a reader can reply; + // empty when the header has none. + Mastodon string + // Draft is true for a post that is not to be published yet. + Draft bool + // Language is the language the post is written in, from the Language key; + // empty when the header has none, and the builder then uses the language of + // the news. + Language string + // Words is the number of words of the body. + Words int // Source is the file the post was read from, relative to the site. Source string // Body is the Markdown after the header. Body string } +// wordsPerMinute is the reading speed the reading time is worked out with. +const wordsPerMinute = 200 + +// ReadingMinutes returns how many minutes the post takes to read, at least 1. +func (p Post) ReadingMinutes() int { + minutes := (p.Words + wordsPerMinute - 1) / wordsPerMinute + if minutes < 1 { + return 1 + } + return minutes +} + +// Visible reports whether the post is published at the moment now: it is not a +// draft and its date has come. +func (p Post) Visible(now time.Time) bool { + return !p.Draft && !p.Date.After(now) +} + // postKeys are the keys a news header may have. -var postKeys = []string{"Title", "Author", "Date", "Categories", "Description"} +var postKeys = []string{"Title", "Author", "Date", "Categories", "Description", "Mastodon", "Draft", "Language"} // dateLayouts are the accepted spellings of Date, ISO 8601 with an offset. var dateLayouts = []string{"2006-01-02T15:04:05-0700", "2006-01-02T15:04:05Z07:00"} +// LanguageCode is the shape of a language code: "en", "pt-BR". +var LanguageCode = regexp.MustCompile(`^[a-z]{2,3}(-[A-Za-z0-9]+)?$`) + // slugName matches a file name made of lowercase words joined by hyphens. var slugName = regexp.MustCompile(`^[a-z0-9]+(?:-[a-z0-9]+)*$`) @@ -113,16 +150,55 @@ func loadPost(file string) (Post, error) { if err != nil { return Post{}, err } + mastodon, err := parseMastodon(source, h.Get("Mastodon")) + if err != nil { + return Post{}, err + } + draft, err := parseDraft(source, h.Get("Draft")) + if err != nil { + return Post{}, err + } + language := h.Get("Language") + if language != "" && !LanguageCode.MatchString(language) { + return Post{}, fmt.Errorf("%s: Language %q is not a language code like \"en\" or \"pt-BR\"", source, language) + } description := h.Get("Description") - if description == "" { + hasDescription := description != "" + if !hasDescription { description = firstSentence(body) } return Post{ Slug: slug, Path: PostPath(date, slug), Title: h.Get("Title"), Author: h.Get("Author"), - Description: description, Date: date, Categories: categories, Source: source, Body: body, + Description: description, HasDescription: hasDescription, Mastodon: mastodon, Draft: draft, Language: language, + Words: len(strings.Fields(body)), Date: date, Categories: categories, Source: source, Body: body, }, nil } +// parseMastodon accepts an empty value or an https address with a host and no +// spaces, because the address becomes the target of a link. +func parseMastodon(source, value string) (string, error) { + if value == "" { + return "", nil + } + u, err := url.Parse(value) + if err != nil || u.Scheme != "https" || u.Host == "" || strings.ContainsAny(value, " \t\"'<>") { + return "", fmt.Errorf("%s: Mastodon %q is not an https address, such as https://social.kmalloc.org/@squid/115000000000000000", source, value) + } + return value, nil +} + +// parseDraft reads the Draft key: true or yes make a draft; false, no or no key +// do not. +func parseDraft(source, value string) (bool, error) { + switch strings.ToLower(value) { + case "", "false", "no": + return false, nil + case "true", "yes": + return true, nil + } + return false, fmt.Errorf("%s: Draft %q must be true or false", source, value) +} + // parseDate reads Date in any of the accepted layouts. func parseDate(source, value string) (time.Time, error) { for _, layout := range dateLayouts { diff --git a/internal/content/posts_test.go b/internal/content/posts_test.go index c3dfda2..9e60ab9 100644 --- a/internal/content/posts_test.go +++ b/internal/content/posts_test.go @@ -190,3 +190,108 @@ func TestLoadPostsNoNewsFolder(t *testing.T) { t.Errorf("LoadPosts() = %v, %v, want no posts and no error", posts, err) } } + +// postFile returns a news file with the given extra header lines. +func postFile(extra string) string { + return "Title: A post\nDate: 2026-11-14T12:00:00+0100\n" + extra + "-\nOne two three four five.\n" +} + +// loadOne loads a single news file and returns its post. +func loadOne(t *testing.T, file string) (Post, error) { + t.Helper() + posts, err := LoadPosts(writeFiles(t, map[string]string{"news/a-post.md": file})) + if err != nil { + return Post{}, err + } + return posts[0], nil +} + +// TestPostMastodon checks the optional reply address and its validation. +func TestPostMastodon(t *testing.T) { + p, err := loadOne(t, postFile("Mastodon: https://social.kmalloc.org/@squid/115000000000000000\n")) + if err != nil || p.Mastodon != "https://social.kmalloc.org/@squid/115000000000000000" { + t.Fatalf("Mastodon = %q, err = %v", p.Mastodon, err) + } + if p, _ = loadOne(t, postFile("")); p.Mastodon != "" { + t.Errorf("a post without the key has Mastodon %q", p.Mastodon) + } + for _, bad := range []string{"http://social.kmalloc.org/@squid/1", "not an address", "https://", "https://a b/c", "javascript:alert(1)"} { + if _, err := loadOne(t, postFile("Mastodon: "+bad+"\n")); err == nil || !strings.Contains(err.Error(), "Mastodon") { + t.Errorf("Mastodon %q: error = %v, want one about Mastodon", bad, err) + } + } +} + +// TestPostDraft checks the Draft key. +func TestPostDraft(t *testing.T) { + for value, want := range map[string]bool{"true": true, "True": true, "yes": true, "false": false, "no": false} { + p, err := loadOne(t, postFile("Draft: "+value+"\n")) + if err != nil || p.Draft != want { + t.Errorf("Draft %q: got %v, err %v, want %v", value, p.Draft, err, want) + } + } + if p, _ := loadOne(t, postFile("")); p.Draft { + t.Error("a post without Draft must not be a draft") + } + if _, err := loadOne(t, postFile("Draft: maybe\n")); err == nil || !strings.Contains(err.Error(), "Draft") { + t.Errorf("Draft maybe: error = %v, want one about Draft", err) + } +} + +// TestPostVisible checks that drafts and future dates stay hidden. +func TestPostVisible(t *testing.T) { + now, _ := time.Parse("2006-01-02T15:04:05-0700", "2026-11-15T12:00:00+0100") + past, _ := loadOne(t, postFile("")) + draft, _ := loadOne(t, postFile("Draft: true\n")) + future, _ := loadOne(t, "Title: Later\nDate: 2026-12-01T09:00:00+0100\n-\nSoon.\n") + if !past.Visible(now) || draft.Visible(now) || future.Visible(now) { + t.Errorf("Visible: past %v draft %v future %v, want true false false", past.Visible(now), draft.Visible(now), future.Visible(now)) + } + if !future.Visible(future.Date) { + t.Error("a post is visible from the moment of its date") + } +} + +// TestPostHasDescription tells a written description from the fallback. +func TestPostHasDescription(t *testing.T) { + written, _ := loadOne(t, postFile("Description: Written by hand.\n")) + fallback, _ := loadOne(t, postFile("")) + if !written.HasDescription || written.Description != "Written by hand." { + t.Errorf("written = %+v", written) + } + if fallback.HasDescription || fallback.Description != "One two three four five." { + t.Errorf("fallback: HasDescription %v, Description %q", fallback.HasDescription, fallback.Description) + } +} + +// TestPostReadingMinutes checks the word count and the reading time. +func TestPostReadingMinutes(t *testing.T) { + p, _ := loadOne(t, postFile("")) + if p.Words != 5 || p.ReadingMinutes() != 1 { + t.Errorf("Words %d, ReadingMinutes %d, want 5 and 1", p.Words, p.ReadingMinutes()) + } + long := strings.Repeat("word ", 401) + "\n" + p, _ = loadOne(t, "Title: Long\nDate: 2026-11-14T12:00:00+0100\n-\n"+long) + if p.Words != 401 || p.ReadingMinutes() != 3 { + t.Errorf("Words %d, ReadingMinutes %d, want 401 and 3", p.Words, p.ReadingMinutes()) + } + if (Post{}).ReadingMinutes() != 1 { + t.Error("an empty post still reads in 1 minute") + } +} + +// TestPostLanguage checks the optional Language key and its validation. +func TestPostLanguage(t *testing.T) { + p, err := loadOne(t, postFile("Language: it\n")) + if err != nil || p.Language != "it" { + t.Fatalf("Language = %q, err = %v", p.Language, err) + } + if p, _ = loadOne(t, postFile("")); p.Language != "" { + t.Errorf("a post without the key has Language %q, want empty (the builder gives it the language of the news)", p.Language) + } + for _, bad := range []string{"Italian", "IT", "i", "it_IT", "it fr"} { + if _, err := loadOne(t, postFile("Language: "+bad+"\n")); err == nil || !strings.Contains(err.Error(), "Language") { + t.Errorf("Language %q: error = %v, want one about Language", bad, err) + } + } +} From bf50b2e1c2d835340415c776886e1c533f047765 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Mon, 16 Nov 2026 12:06:55 +0100 Subject: [PATCH 11/22] feat(news): a post page with date, author, categories, summary and links New placeholders for templates/post.html, written in the language of the post: date_long, post_summary, post_meta, post_reading, post_mastodon and post_nav (older, newer and all news), plus og_type and article_meta for Open Graph, which every other page sets to website. A post with a Mastodon address needs the reply_mastodon string; the other strings are optional and a feature without its string is left out. Refs #9 --- internal/build/blog_test.go | 172 ++++++++++++++++++++++++++++++++++++ internal/build/context.go | 2 + internal/build/pages.go | 13 ++- internal/build/post.go | 113 +++++++++++++++++++++++ 4 files changed, 293 insertions(+), 7 deletions(-) create mode 100644 internal/build/post.go diff --git a/internal/build/blog_test.go b/internal/build/blog_test.go index 392675f..6cedf98 100644 --- a/internal/build/blog_test.go +++ b/internal/build/blog_test.go @@ -127,3 +127,175 @@ func TestNewsLanguage(t *testing.T) { }) } } + +// monthsJSON writes the month_01..month_12 strings of a test language. +func monthsJSON(names ...string) string { + var b strings.Builder + for i, n := range names { + b.WriteString(`"month_` + string(rune('0'+(i+1)/10)) + string(rune('0'+(i+1)%10)) + `": "` + n + `", `) + } + return b.String() +} + +// blogLangEn and blogLangIt are the English and Italian language files with the strings the blog features need. +var ( + blogLangEn = `{"lang_name": "English", "sections": "Sections", "home_link": "/", "nojs": "No JavaScript.", + "powered_by": "Powered by", "back": "kmalloc.org", "version": "Version", "toc": "Contents", "history_a": "Manifesto history", + "news_p": "What is new.", "news_all": "All news", "news_empty": "No news yet.", "category_h": "Category", + "date_long": "{d} {month} {y}", "date_month_year": "{month} {y}", "reading_time": "{n} min read", + ` + monthsJSON("January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December") + ` + "post_older": "Older", "post_newer": "Newer", "post_nav": "Other news", "reply_mastodon": "Reply on Mastodon", + "read_more": "Read more", "pager_label": "News pages", "pager_newer": "Newer news", "pager_older": "Older news", "pager_page": "Page {n} of {total}", + "archive_h": "Archive", "archive_year_h": "Year", "archive_up": "Archive of the year", + "nav": {"manifesto": "Manifesto", "people": "People", "services": "Services", "news": "News"}}` + blogLangIt = `{"lang_name": "Italiano", "sections": "Sezioni", "home_link": "/it/", "nojs": "Niente JavaScript.", + "powered_by": "Powered by", "back": "kmalloc.org", "version": "Versione", "toc": "Indice", "history_a": "Cronologia", + "news_p": "Le novita.", "news_all": "Tutte le news", "news_empty": "Nessuna news.", "category_h": "Categoria", + "date_long": "{d} {month} {y}", "date_month_year": "{month} {y}", "reading_time": "{n} min di lettura", + ` + monthsJSON("gennaio", "febbraio", "marzo", "aprile", "maggio", "giugno", "luglio", "agosto", "settembre", "ottobre", "novembre", "dicembre") + ` + "post_older": "Piu vecchia", "post_newer": "Piu recente", "post_nav": "Altre news", "reply_mastodon": "Rispondi su Mastodon", + "read_more": "Leggi tutto", "pager_label": "Pagine delle news", "pager_newer": "News piu recenti", "pager_older": "News piu vecchie", "pager_page": "Pagina {n} di {total}", + "archive_h": "Archivio", "archive_year_h": "Anno", "archive_up": "Archivio dell'anno", + "nav": {"manifesto": "Manifesto", "people": "Persone", "services": "Servizi", "news": "News"}}` +) + +// blogPostTemplate is a post page that uses every placeholder of the blog. +const blogPostTemplate = `

      {{post_title}}

      + +{{post_summary}} +{{post_meta}} +{{post_body}} +{{post_mastodon}} +{{post_nav}} +` + +// blogBaseTemplate is the base template with the Open Graph placeholders. +const blogBaseTemplate = ` + + + +{{title_html}} + + +{{article_meta}} + +{{alt_links}} + + + +{{header}} + +{{content}} +{{footer}} + + +` + +// blogSite is the base site with the blog language strings and templates. +func blogSite(extra map[string]string) map[string]string { + files := map[string]string{ + "lang/en.json": blogLangEn, "lang/it.json": blogLangIt, + "templates/post.html": blogPostTemplate, "templates/base.html": blogBaseTemplate, + "news/shop.md": "Title: A shop\nAuthor: squid\nDate: 2026-11-14T12:00:00+0100\nCategories: FLUG\nDescription: About the shop.\nMastodon: https://social.kmalloc.org/@squid/115\n-\nThere is a shop.\n", + } + for k, v := range extra { + files[k] = v + } + return withFiles(files) +} + +// TestPostPageHasTheBlogFields checks the long date, the byline, the summary, +// the reading time, the Mastodon link and the links to the other news. +func TestPostPageHasTheBlogFields(t *testing.T) { + out, err := buildWith(t, blogSite(nil), nil) + if err != nil { + t.Fatalf("Build() error = %v", err) + } + shop := readOut(t, out, "news/2026/11/shop.html") + mustContain(t, "shop post", shop, + `

      About the shop.

      `, + `

      · squid · FLUG · 1 min read

      `, + `

      Reply on Mastodon

      `, + `