From 6987541f88246f092b4325654f4c6789773afa53 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Mon, 16 Nov 2026 12:48:17 +0100 Subject: [PATCH 1/2] fix(news): group the archive by the month as written, drop the empty lists - groupArchive compared each post with the last group only, so two posts near midnight at the end of a month, written with different offsets, listed the same month twice and the build failed with "both published as". Posts are grouped by their own year and month, newest first. - {{news_archive}} is empty when there are no posts, so the index says there is no news once, and the heading of an empty {{news_archive}} or {{categories_list}} goes with it. - tests for exactly ten posts, eleven posts, a single post, a title with quotes, an ampersand, a tag and non Latin letters, and a neighbour written in another language Refs #9 --- internal/build/blog.go | 45 ++++++--- internal/build/blog_test.go | 178 ++++++++++++++++++++++++++++++++++++ internal/build/pages.go | 33 ++++++- 3 files changed, 243 insertions(+), 13 deletions(-) diff --git a/internal/build/blog.go b/internal/build/blog.go index 1395840..0a8a7a5 100644 --- a/internal/build/blog.go +++ b/internal/build/blog.go @@ -4,6 +4,7 @@ package build import ( "fmt" + "sort" "strconv" "strings" @@ -109,24 +110,43 @@ type ( } ) -// groupArchive groups posts, newest first, by year and by month, as written in -// their dates. The years and the months are newest first too. +// groupArchive groups posts by year and by month, as written in their dates. +// The years and the months are newest first, and the posts of a group keep the +// order they were given. A post is put in its group by its own year and month, +// not by the group before it: two posts near midnight at the end of a month, +// written with different offsets, can be in the order of their instants and in +// the opposite order of their months. func groupArchive(posts []content.Post) []archiveYear { - var years []archiveYear + type key struct{ year, month int } + years := map[int]*archiveYear{} + months := map[key]*archiveMonth{} for _, p := range posts { y, m := p.Date.Year(), int(p.Date.Month()) - if len(years) == 0 || years[len(years)-1].year != y { - years = append(years, archiveYear{year: y}) + year := years[y] + if year == nil { + year = &archiveYear{year: y} + years[y] = year } - year := &years[len(years)-1] year.posts = append(year.posts, p) - if len(year.months) == 0 || year.months[len(year.months)-1].month != m { - year.months = append(year.months, archiveMonth{year: y, month: m}) + month := months[key{y, m}] + if month == nil { + month = &archiveMonth{year: y, month: m} + months[key{y, m}] = month } - month := &year.months[len(year.months)-1] month.posts = append(month.posts, p) } - return years + var out []archiveYear + for _, year := range years { + for k, month := range months { + if k.year == year.year { + year.months = append(year.months, *month) + } + } + sort.Slice(year.months, func(i, j int) bool { return year.months[i].month > year.months[j].month }) + out = append(out, *year) + } + sort.Slice(out, func(i, j int) bool { return out[i].year > out[j].year }) + return out } // yearPath and monthPath are the addresses of an archive year and month. @@ -151,14 +171,15 @@ func (b *builder) monthList(y archiveYear, lang string) string { } // newsArchive returns the nested list of the years and the months that have -// posts, or the news_empty text when there are none. It needs the archive pages. +// posts, or nothing when there are none: the blog list says there is no news. It +// needs the archive pages. func (b *builder) newsArchive(lang string, ctx map[string]string) (string, error) { if _, err := b.template("archive"); err != nil { return "", fmt.Errorf("the {{news_archive}} block needs the archive pages: %w", err) } years := groupArchive(b.posts) if len(years) == 0 { - return "

\n" + ctx["news_empty"] + "\n

", nil + return "", nil } items := make([]string, len(years)) for i, y := range years { diff --git a/internal/build/blog_test.go b/internal/build/blog_test.go index 2835a32..fe21863 100644 --- a/internal/build/blog_test.go +++ b/internal/build/blog_test.go @@ -3,6 +3,7 @@ package build import ( + "encoding/xml" "path/filepath" "strings" "testing" @@ -526,3 +527,180 @@ func TestPanelOfAPostInAnotherLanguage(t *testing.T) { t.Error("an English post is marked as written in Italian") } } + +// TestArchiveGroupsByTheDateAsWritten checks that two posts either side of midnight, written with different +// offsets, land in their own months and that no month is listed twice, whatever the order of the instants. +func TestArchiveGroupsByTheDateAsWritten(t *testing.T) { + files := blogSite(map[string]string{"content/en/news/index.md": blogIndex, "templates/archive.html": archiveTemplate, + "news/late.md": "Title: Late\nDate: 2026-11-30T23:45:00+0000\n-\nText.\n", + "news/early.md": "Title: Early\nDate: 2026-12-01T00:30:00+0100\n-\nText.\n", + "news/mid.md": "Title: Mid\nDate: 2026-11-29T12:00:00+0100\n-\nText.\n"}) + out, err := buildWith(t, files, func(o *Options) { o.Now = time.Date(2026, 12, 20, 12, 0, 0, 0, time.UTC) }) + if err != nil { + t.Fatalf("Build() error = %v", err) + } + index := readOut(t, out, "news/index.html") + mustContain(t, "news index", index, `2026 (6)`) + dec, nov, sep := strings.Index(index, ">December 2026 (1)"), strings.Index(index, ">November 2026 (4)"), strings.Index(index, ">September 2026 (1)") + if dec < 0 || nov < dec || sep < nov { + t.Errorf("the months are not December, November, September, one each:\n%s", index) + } + if strings.Count(index, "November 2026") != 1 { + t.Error("November is listed more than once") + } + month := readOut(t, out, "news/2026/11/index.html") + late, mid, shop := strings.Index(month, "Late."), strings.Index(month, "Mid."), strings.Index(month, "A shop.") + if late < 0 || mid < late || shop < mid || strings.Contains(month, "Early.") { + t.Errorf("the month of November is wrong:\n%s", month) + } + mustContain(t, "month of December", readOut(t, out, "news/2026/12/index.html"), "Early.") +} + +// TestNoNewsAtAll checks that a site without posts says so once, and that the empty lists take their headings with them. +func TestNoNewsAtAll(t *testing.T) { + index := "Title: News\nID: news\nDescription: About News.\n-\n# News\n\n{{news_blog}}\n\n## Categories\n\n{{categories_list}}\n\n## Archive\n\n{{news_archive}}\n" + files := blogSite(map[string]string{"content/en/news/index.md": index, "templates/archive.html": archiveTemplate, + "news/shop.md": "", "news/gnu-linux-day-2026.md": "", "news/status.md": ""}) + out, err := buildWith(t, files, nil) + if err != nil { + t.Fatalf("Build() error = %v", err) + } + got := readOut(t, out, "news/index.html") + if n := strings.Count(got, "No news yet."); n != 1 { + t.Errorf("the index says there is no news %d times, want once:\n%s", n, got) + } + for _, heading := range []string{">Categories", ">Archive"} { + if strings.Contains(got, heading) { + t.Errorf("the index keeps the heading %q of an empty list", heading) + } + } + mustContain(t, "news index", got, "

News

") +} + +// TestHomeKeepsItsHeadingWithoutNews checks that the heading before the latest news is the page's own and stays. +func TestHomeKeepsItsHeadingWithoutNews(t *testing.T) { + files := blogSite(map[string]string{"news/shop.md": "", "news/gnu-linux-day-2026.md": "", "news/status.md": ""}) + out, err := buildWith(t, files, nil) + if err != nil { + t.Fatalf("Build() error = %v", err) + } + mustContain(t, "home", readOut(t, out, "index.html"), `

Technology.

`) +} + +// TestHeadingStaysWithAListThatHasItems checks that the heading of a generated list is dropped only when the list is empty. +func TestHeadingStaysWithAListThatHasItems(t *testing.T) { + index := "Title: News\nID: news\nDescription: About News.\n-\n# News\n\n## Categories\n\n{{categories_list}}\n\n## Archive\n\n{{news_archive}}\n" + out, err := buildWith(t, blogSite(map[string]string{"content/en/news/index.md": index, "templates/archive.html": archiveTemplate}), nil) + if err != nil { + t.Fatalf("Build() error = %v", err) + } + mustContain(t, "news index", readOut(t, out, "news/index.html"), ">Categories", ">Archive") +} + +// listFiles returns the site of the pagination tests: the three base posts and extra ones, so that n posts in all. +func listFiles(t *testing.T, n int) map[string]string { + t.Helper() + files := blogSite(map[string]string{"content/en/news/index.md": blogIndex, "templates/archive.html": archiveTemplate}) + return manyPosts(files, n-3) +} + +// TestExactlyOnePageOfPosts checks that ten posts make one page with no pager and no second page. +func TestExactlyOnePageOfPosts(t *testing.T) { + out, err := buildWith(t, listFiles(t, 10), nil) + if err != nil { + t.Fatalf("Build() error = %v", err) + } + index := readOut(t, out, "news/index.html") + if n := strings.Count(index, `role="note"`); n != 10 { + t.Errorf("the index has %d panels, want 10", n) + } + if strings.Contains(index, "News pages") || strings.Contains(index, "Page 1 of") || exists(t, out, "news/page/2/index.html") { + t.Error("ten posts have a pager or a second page") + } +} + +// TestEleventhPostIsAloneOnPageTwo checks the page that holds the one post that does not fit. +func TestEleventhPostIsAloneOnPageTwo(t *testing.T) { + out, err := buildWith(t, listFiles(t, 11), nil) + if err != nil { + t.Fatalf("Build() error = %v", err) + } + first := readOut(t, out, "news/index.html") + mustContain(t, "page 1", first, "Page 1 of 2", ``) + if n := strings.Count(first, `role="note"`); n != 10 || strings.Contains(first, "GNU/Linux Day 2026.") { + t.Errorf("page 1 has %d panels or the oldest post", n) + } + second := readOut(t, out, "news/page/2/index.html") + mustContain(t, "page 2", second, "Page 2 of 2", ``, "GNU/Linux Day 2026.") + if n := strings.Count(second, `role="note"`); n != 1 || strings.Contains(second, `rel="next"`) { + t.Errorf("page 2 has %d panels or a link to a third page", n) + } + if exists(t, out, "news/page/3/index.html") { + t.Error("eleven posts have a third page") + } +} + +// TestOnlyOnePost checks that the one post has no link to a newer or an older post anywhere, and no pager. +func TestOnlyOnePost(t *testing.T) { + files := blogSite(map[string]string{"content/en/news/index.md": blogIndex, "templates/archive.html": archiveTemplate, + "news/gnu-linux-day-2026.md": "", "news/status.md": ""}) + out, err := buildWith(t, files, nil) + if err != nil { + t.Fatalf("Build() error = %v", err) + } + for _, rel := range []string{"news/2026/11/shop.html", "news/index.html", "news/2026/11/index.html"} { + got := readOut(t, out, rel) + for _, absent := range []string{"Older", "Newer", `rel="prev"`, `rel="next"`, "News pages"} { + if strings.Contains(got, absent) { + t.Errorf("%s has %q, but there is one post", rel, absent) + } + } + } + mustContain(t, "the post", readOut(t, out, "news/2026/11/shop.html"), `
  • All news
  • `) +} + +// TestUnusualTitle checks that a title with quotes, an ampersand, a tag and non Latin letters is escaped everywhere. +func TestUnusualTitle(t *testing.T) { + title := `Say "hi" & go ünï 日本語` + files := blogSite(map[string]string{"content/en/news/index.md": blogIndex, "templates/archive.html": archiveTemplate, + "news/odd.md": "Title: " + title + "\nDate: 2026-11-15T09:00:00+0100\nCategories: Odd\n-\nText.\n"}) + out, err := buildWith(t, files, nil) + if err != nil { + t.Fatalf("Build() error = %v", err) + } + escaped := `Say "hi" & <b>go</b> ünï 日本語` + attr := `Say "hi" & <b>go</b> ünï 日本語` + for _, rel := range []string{"news/index.html", "news/category/odd.html", "news/2026/11/index.html"} { + mustContain(t, rel, readOut(t, out, rel), ""+escaped+".", `aria-label="Read more: `+attr+`"`) + } + mustContain(t, "the post", readOut(t, out, "news/2026/11/odd.html"), "

    "+escaped+"

    ") + mustContain(t, "the post below", readOut(t, out, "news/2026/11/shop.html"), `Newer: `+escaped) + for _, rel := range []string{"news/index.html", "news/category/odd.html", "news/2026/11/odd.html", "news/feed.xml"} { + if strings.Contains(readOut(t, out, rel), "go") { + t.Errorf("%s has the tag of the title as markup", rel) + } + } + var feed struct { + Channel struct { + Items []struct { + Title string `xml:"title"` + } `xml:"item"` + } `xml:"channel"` + } + if err := xml.Unmarshal([]byte(readOut(t, out, "news/feed.xml")), &feed); err != nil { + t.Fatalf("the feed is not well formed: %v", err) + } + if len(feed.Channel.Items) == 0 || feed.Channel.Items[0].Title != title { + t.Errorf("the first item of the feed is %+v, want the title %q", feed.Channel.Items, title) + } +} + +// TestNeighbourInAnotherLanguageIsMarked checks that the link to an older post in another language names the language. +func TestNeighbourInAnotherLanguageIsMarked(t *testing.T) { + out, err := buildWith(t, blogSite(map[string]string{"news/ciao.md": "Title: Ciao\nDate: 2026-11-05T09:00:00+0100\nLanguage: it\n-\nUn testo.\n"}), nil) + if err != nil { + t.Fatalf("Build() error = %v", err) + } + mustContain(t, "status post", readOut(t, out, "news/2026/11/status.html"), `rel="prev">Older: Ciao`) + mustContain(t, "italian post", readOut(t, out, "news/2026/11/ciao.html"), `rel="next">Piu recente: Status page`) +} diff --git a/internal/build/pages.go b/internal/build/pages.go index 04d04bc..45cb424 100644 --- a/internal/build/pages.go +++ b/internal/build/pages.go @@ -134,7 +134,8 @@ func (b *builder) markdown(name, source, lang string, ctx map[string]string) (st var codeElement = regexp.MustCompile(`(?s)]*>.*?`) // expandBlocks replaces each paragraph that holds only a block placeholder by -// the generated HTML. A block placeholder inside other text is an error. All +// the generated HTML. An empty list of categories or of the archive goes with the +// heading right before it. A block placeholder inside other text is an error. All // blocks are checked before any is expanded, so the text of a generated list // (a news title may say {{latest_news}}) is never taken for a placeholder. func (b *builder) expandBlocks(name, html, lang string, ctx map[string]string) (string, error) { @@ -154,11 +155,41 @@ func (b *builder) expandBlocks(name, html, lang string, ctx map[string]string) ( if err != nil { return "", err } + if generated == "" && headedBlocks[block] { + html = removeWithHeading(html, marker) + continue + } html = strings.ReplaceAll(html, marker, generated) } return html, nil } +// headedBlocks are the lists a page puts under a heading of its own, which goes +// when the list is empty. The latest news bring their heading with them, and a +// heading before them is the page's. +var headedBlocks = map[string]bool{"categories_list": true, "news_archive": true} + +// headingEnd matches the end of a heading element at the end of a text. +var headingEnd = regexp.MustCompile(`$`) + +// removeWithHeading removes every marker of an empty block and the heading right +// before it, which would be left with nothing under it. +func removeWithHeading(html, marker string) string { + for { + at := strings.Index(html, marker) + if at < 0 { + return html + } + before := strings.TrimRight(html[:at], " \t\r\n") + if m := headingEnd.FindStringSubmatchIndex(before); m != nil { + if open := strings.LastIndex(before[:m[0]], "= 0 { + before = before[:open] + } + } + html = before + "\n" + strings.TrimLeft(html[at+len(marker):], "\n") + } +} + // block returns the HTML of a generated list. func (b *builder) block(name, lang string, ctx map[string]string) (string, error) { switch name { From 52edb15b64d95959e6acdec9bc194cd2cc6aba7e Mon Sep 17 00:00:00 2001 From: tgiachi Date: Mon, 16 Nov 2026 12:49:17 +0100 Subject: [PATCH 2/2] feat(build): report the old files of the output folder, add -clean A build never emptied the output folder, so the pages of a list that got shorter stayed behind. The build now returns the files it did not write and the command prints a warning that names them. -clean empties the folder first, only when it is new, empty or holds the .motorino-out file that an earlier -clean left, and never the folder of the site or one that holds it. The default is unchanged: no extra file in a folder built without it, so the deploy (a new folder published with rsync --delete) is not affected. Refs #9 --- README.md | 3 + cmd/motorino/main.go | 21 ++++- cmd/motorino/main_test.go | 54 ++++++++++++ internal/build/build.go | 40 +++++---- internal/build/output.go | 105 ++++++++++++++++++++++ internal/build/output_test.go | 162 ++++++++++++++++++++++++++++++++++ 6 files changed, 363 insertions(+), 22 deletions(-) create mode 100644 internal/build/output.go create mode 100644 internal/build/output_test.go diff --git a/README.md b/README.md index 1b58d2c..d0cbb14 100644 --- a/README.md +++ b/README.md @@ -9,12 +9,15 @@ Licence: GPL-3.0-or-later, see `LICENSE`. ``` motorino build -site DIR -out DIR build the site in DIR into the output folder (defaults: . and public) motorino build -drafts ... the same, with the drafts and the posts dated in the future, for a preview +motorino build -clean ... empty the output folder first, if it is new, empty or holds .motorino-out motorino new -site DIR "A title" write news/a-title.md, a draft with the header ready, and print its path motorino version print the version ``` The binary is a single static Go program. The version comes from semantic-release. +A build writes the files it makes and leaves the others in the output folder, so the pages of a list that got shorter (`news/page/3/`) stay behind. When that happens the command prints a warning that names the old files. The deploy never has them: it builds into a new folder and publishes it with `rsync --delete`. For a folder you reuse, `-clean` empties it first. It does that only for a folder that is new, empty or holds the `.motorino-out` file that an earlier `-clean` left, never for any other folder with files in it and never for the folder of the site or one that holds it. A build without `-clean` writes no extra file. + ## What a site looks like ``` diff --git a/cmd/motorino/main.go b/cmd/motorino/main.go index 88ca4e9..7443176 100644 --- a/cmd/motorino/main.go +++ b/cmd/motorino/main.go @@ -5,7 +5,7 @@ // // Usage: // -// motorino build [-site DIR] [-out DIR] [-drafts] +// motorino build [-site DIR] [-out DIR] [-drafts] [-clean] // motorino new [-site DIR] "title" // motorino version package main @@ -26,7 +26,7 @@ import ( ) // usage is the text printed for a missing or unknown command. -const usage = "usage: motorino build [-site DIR] [-out DIR] [-drafts] | motorino new [-site DIR] \"title\" | motorino version" +const usage = "usage: motorino build [-site DIR] [-out DIR] [-drafts] [-clean] | motorino new [-site DIR] \"title\" | motorino version" // main runs the command line and exits with its status code. func main() { @@ -61,6 +61,7 @@ func runBuild(args []string, stdout, stderr io.Writer) int { site := flags.String("site", ".", "the folder of the site") out := flags.String("out", "public", "the folder the HTML is written to") drafts := flags.Bool("drafts", false, "publish the drafts and the posts dated in the future, for a preview") + clean := flags.Bool("clean", false, "empty the output folder first, if it is new, empty or was emptied this way before") if err := flags.Parse(args); err != nil { if errors.Is(err, flag.ErrHelp) { return 0 @@ -71,15 +72,29 @@ func runBuild(args []string, stdout, stderr io.Writer) int { fmt.Fprintf(stderr, "motorino: unexpected arguments: %s\n%s\n", strings.Join(flags.Args(), " "), usage) return 2 } - result, err := build.Build(build.Options{SiteDir: *site, OutDir: *out, Version: version.String(), Drafts: *drafts}) + result, err := build.Build(build.Options{SiteDir: *site, OutDir: *out, Version: version.String(), Drafts: *drafts, Clean: *clean}) if err != nil { fmt.Fprintf(stderr, "motorino: %v\n", err) return 1 } fmt.Fprintf(stdout, "built %d pages into %s\n", result.Pages, *out) + warnStale(stderr, *out, result.Stale) return 0 } +// warnStale tells the files of the output folder that the build did not write, +// which stay there unless -clean is used. +func warnStale(stderr io.Writer, out string, stale []string) { + if len(stale) == 0 { + return + } + noun := "files" + if len(stale) == 1 { + noun = "file" + } + fmt.Fprintf(stderr, "motorino: warning: %d %s in %s were not written by this build, such as %s; build into a new folder or use -clean\n", len(stale), noun, out, stale[0]) +} + // runNew parses the flags of the new command and writes the file of a new post. // The author is the author key of site.json, when it has one. func runNew(args []string, stdout, stderr io.Writer) int { diff --git a/cmd/motorino/main_test.go b/cmd/motorino/main_test.go index 8798844..6ecd765 100644 --- a/cmd/motorino/main_test.go +++ b/cmd/motorino/main_test.go @@ -187,3 +187,57 @@ func TestRunBuildDraftsFlag(t *testing.T) { t.Errorf("a draft is missing from the build with -drafts: %v", err) } } + +// TestRunBuildWarnsAboutOldFiles checks that old files in the output folder are named in a warning and left alone. +func TestRunBuildWarnsAboutOldFiles(t *testing.T) { + site := miniSite(t) + outDir := t.TempDir() + if err := os.WriteFile(filepath.Join(outDir, "old.html"), []byte("old"), 0o644); err != nil { + t.Fatal(err) + } + var out, errOut bytes.Buffer + if code := run([]string{"build", "-site", site, "-out", outDir}, &out, &errOut); code != 0 { + t.Fatalf("run(build) = %d, stderr %q", code, errOut.String()) + } + for _, want := range []string{"warning", "1 file", "old.html", "-clean"} { + if !strings.Contains(errOut.String(), want) { + t.Errorf("stderr %q does not contain %q", errOut.String(), want) + } + } + if _, err := os.Stat(filepath.Join(outDir, "old.html")); err != nil { + t.Error("a build without -clean removed the old file") + } + errOut.Reset() + if code := run([]string{"build", "-site", site, "-out", filepath.Join(t.TempDir(), "new")}, &out, &errOut); code != 0 || errOut.Len() != 0 { + t.Errorf("a build into a new folder: code %d, stderr %q, want 0 and nothing", code, errOut.String()) + } +} + +// TestRunBuildClean checks -clean on a folder that is marked and on one that is not. +func TestRunBuildClean(t *testing.T) { + site := miniSite(t) + var out, errOut bytes.Buffer + outDir := filepath.Join(t.TempDir(), "public") + if code := run([]string{"build", "-clean", "-site", site, "-out", outDir}, &out, &errOut); code != 0 { + t.Fatalf("run(build -clean) into a new folder = %d, stderr %q", code, errOut.String()) + } + if err := os.WriteFile(filepath.Join(outDir, "old.html"), []byte("old"), 0o644); err != nil { + t.Fatal(err) + } + if code := run([]string{"build", "-clean", "-site", site, "-out", outDir}, &out, &errOut); code != 0 || errOut.Len() != 0 { + t.Fatalf("run(build -clean) again = %d, stderr %q", code, errOut.String()) + } + if _, err := os.Stat(filepath.Join(outDir, "old.html")); err == nil { + t.Error("-clean left the old file") + } + other := t.TempDir() + if err := os.WriteFile(filepath.Join(other, "mine.txt"), []byte("mine"), 0o644); err != nil { + t.Fatal(err) + } + if code := run([]string{"build", "-clean", "-site", site, "-out", other}, &out, &errOut); code != 1 || !strings.Contains(errOut.String(), ".motorino-out") { + t.Errorf("run(build -clean) into a folder that is not motorino's = %d, stderr %q, want 1 and the marker named", code, errOut.String()) + } + if _, err := os.Stat(filepath.Join(other, "mine.txt")); err != nil { + t.Error("-clean removed a file of a folder that is not motorino's") + } +} diff --git a/internal/build/build.go b/internal/build/build.go index fa57268..f38ea46 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -31,6 +31,10 @@ type Options struct { // Drafts publishes the drafts and the posts dated in the future too, for a // preview of the site. Drafts bool + // Clean empties OutDir before the site is written, when it is new, empty or + // was emptied this way before (it holds the file .motorino-out); any other + // folder with files in it is an error. Without it old files stay. + Clean bool } // requiredPages are the IDs every site must have, because motorino builds the @@ -93,13 +97,18 @@ type categoryEntry struct { type Result struct { // Pages is the number of HTML pages written: pages, news and categories. Pages int + // Stale lists the files that were already in the output folder and that this + // build did not write, such as the pages of a list that got shorter. They are + // left alone unless Clean is set, which removes them first. + Stale []string } // Build reads the site in opts.SiteDir and writes it to opts.OutDir. It renders // everything in memory first, so a failure in the content leaves the output // folder untouched. Writing the files is not atomic, and files of an earlier -// build that are no longer produced are not removed: build into a fresh folder -// when that matters, as the deploy does. +// build that are no longer produced are not removed, but listed in Result.Stale: +// build into a fresh folder when that matters, as the deploy does (it publishes +// the new folder with rsync --delete), or set Options.Clean. func Build(opts Options) (Result, error) { b, err := load(opts) if err != nil { @@ -127,10 +136,18 @@ func Build(opts Options) (Result, error) { if err != nil { return Result{}, err } - if err := b.write(assets); err != nil { + files := make(map[string][]byte, len(b.files)+len(assets)) + for rel, text := range b.files { + files[rel] = []byte(text) + } + for rel, data := range assets { + files[rel] = data + } + stale, err := writeOutput(opts.OutDir, opts.SiteDir, opts.Clean, files) + if err != nil { return Result{}, err } - return Result{Pages: len(b.indexable)}, nil + return Result{Pages: len(b.indexable), Stale: stale}, nil } // load reads the configuration, the content, the templates and the style. @@ -331,21 +348,6 @@ func (b *builder) readAssets() (map[string][]byte, error) { return out, nil } -// write creates the output folder and writes the pages and the assets. -func (b *builder) write(assets map[string][]byte) error { - for rel, text := range b.files { - if err := writeFile(b.opts.OutDir, rel, []byte(text)); err != nil { - return err - } - } - for rel, data := range assets { - if err := writeFile(b.opts.OutDir, rel, data); err != nil { - return err - } - } - return nil -} - // writeFile writes data to rel under dir, creating the folders. func writeFile(dir, rel string, data []byte) error { path := filepath.Join(dir, filepath.FromSlash(rel)) diff --git a/internal/build/output.go b/internal/build/output.go new file mode 100644 index 0000000..0dbe133 --- /dev/null +++ b/internal/build/output.go @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package build + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +// motorinoMarker is the file motorino leaves in a folder it emptied with the +// clean option. A folder that has it, or has nothing in it, may be emptied again; +// any other folder never is. It is only written when the option is used, so the +// folders of a build without it hold the site and nothing else. +const motorinoMarker = ".motorino-out" + +// writeOutput writes the files (path with slashes, relative to out) into out and +// returns the files that were already in out and are not among them: the pages +// of an earlier build that this one no longer makes. With clean set it first +// empties out, but only a folder that is new, empty or marked by an earlier clean +// build, and never the folder of the site or one that holds it. It does not +// touch the folder before every file is known, so a content error leaves it as +// it was. +func writeOutput(out, siteDir string, clean bool, files map[string][]byte) ([]string, error) { + if clean { + if err := emptyOutput(out, siteDir); err != nil { + return nil, err + } + files = withMarker(files) + } + for rel, data := range files { + if err := writeFile(out, rel, data); err != nil { + return nil, err + } + } + return staleFiles(out, files) +} + +// withMarker returns the files and the marker. +func withMarker(files map[string][]byte) map[string][]byte { + marked := make(map[string][]byte, len(files)+1) + for rel, data := range files { + marked[rel] = data + } + marked[motorinoMarker] = []byte("This folder is written by motorino and emptied by motorino build -clean and motorino gemini -clean.\n") + return marked +} + +// emptyOutput removes what is in out, after checking that it may be removed. +func emptyOutput(out, siteDir string) error { + absOut, err := filepath.Abs(out) + if err != nil { + return err + } + absSite, err := filepath.Abs(siteDir) + if err != nil { + return err + } + if rel, err := filepath.Rel(absOut, absSite); err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return fmt.Errorf("refusing to empty %s: it is the folder of the site or holds it", out) + } + entries, err := os.ReadDir(out) + if os.IsNotExist(err) || (err == nil && len(entries) == 0) { + return nil + } + if err != nil { + return fmt.Errorf("read %s: %w", out, err) + } + if info, err := os.Lstat(filepath.Join(out, motorinoMarker)); err != nil || !info.Mode().IsRegular() { + return fmt.Errorf("refusing to empty %s: it has files and no %s, which motorino leaves in a folder it emptied; empty it yourself this first time", out, motorinoMarker) + } + for _, entry := range entries { + if err := os.RemoveAll(filepath.Join(out, entry.Name())); err != nil { + return fmt.Errorf("empty %s: %w", out, err) + } + } + return nil +} + +// staleFiles lists the files under out that are not among the written ones, with +// slashes, sorted. The marker is never one of them. +func staleFiles(out string, written map[string][]byte) ([]string, error) { + var stale []string + err := filepath.WalkDir(out, func(path string, entry os.DirEntry, err error) error { + if err != nil || entry.IsDir() { + return err + } + rel, err := filepath.Rel(out, path) + if err != nil { + return err + } + rel = filepath.ToSlash(rel) + if _, ok := written[rel]; !ok && rel != motorinoMarker { + stale = append(stale, rel) + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("read %s: %w", out, err) + } + sort.Strings(stale) + return stale, nil +} diff --git a/internal/build/output_test.go b/internal/build/output_test.go new file mode 100644 index 0000000..8fd0efa --- /dev/null +++ b/internal/build/output_test.go @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package build + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// put writes a file of a test folder. +func put(t *testing.T, dir, rel, text string) { + t.Helper() + path := filepath.Join(dir, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(text), 0o644); err != nil { + t.Fatal(err) + } +} + +// extraBuilders are the other outputs the tests of the output folder run against, by name: each takes the +// folder of a site and returns a function that builds it into a folder. +var extraBuilders = map[string]func(site string) func(out string, clean bool) ([]string, error){} + +// builders are the outputs, each as a function that builds the site made of files into a folder and returns +// the files of the folder that the build did not write. +func builders(t *testing.T, files map[string]string) map[string]func(out string, clean bool) ([]string, error) { + t.Helper() + site := writeFiles(t, files) + all := map[string]func(string, bool) ([]string, error){ + "html": func(out string, clean bool) ([]string, error) { + r, err := Build(Options{SiteDir: site, OutDir: out, Version: "1.2.3", Now: fixedNow, Clean: clean}) + return r.Stale, err + }, + } + for name, newBuilder := range extraBuilders { + all[name] = newBuilder(site) + } + return all +} + +// TestStaleFilesAreReported checks that a build into a folder with files it did not write reports them and leaves them. +func TestStaleFilesAreReported(t *testing.T) { + for name, run := range builders(t, withFiles(nil)) { + t.Run(name, func(t *testing.T) { + out := t.TempDir() + put(t, out, "news/page/2/index.html", "old") + put(t, out, "old.txt", "old") + stale, err := run(out, false) + if err != nil { + t.Fatalf("build error = %v", err) + } + if strings.Join(stale, ",") != "news/page/2/index.html,old.txt" { + t.Errorf("stale = %v, want the two old files", stale) + } + if !exists(t, out, "old.txt") { + t.Error("a build without -clean removed a file") + } + if exists(t, out, motorinoMarker) { + t.Error("a build without -clean wrote the marker into the folder") + } + }) + } +} + +// TestFreshFolderHasNothingStale checks that a build into a new folder reports nothing and writes only the site. +func TestFreshFolderHasNothingStale(t *testing.T) { + for name, run := range builders(t, withFiles(nil)) { + t.Run(name, func(t *testing.T) { + stale, err := run(filepath.Join(t.TempDir(), "new"), false) + if err != nil || len(stale) != 0 { + t.Errorf("stale = %v, error = %v", stale, err) + } + }) + } +} + +// TestCleanEmptiesAFolderItMarked checks that -clean removes the old files of a folder that has the marker, +// the pages of a list that got shorter included, and writes the marker again. +func TestCleanEmptiesAFolderItMarked(t *testing.T) { + for name, run := range builders(t, withFiles(nil)) { + t.Run(name, func(t *testing.T) { + out := t.TempDir() + put(t, out, motorinoMarker, "x") + put(t, out, "news/page/2/index.html", "old") + put(t, out, "old/deep/file.txt", "old") + stale, err := run(out, true) + if err != nil { + t.Fatalf("build error = %v", err) + } + if len(stale) != 0 { + t.Errorf("stale = %v after a clean build", stale) + } + for _, gone := range []string{"news/page/2/index.html", "old/deep/file.txt"} { + if exists(t, out, gone) { + t.Errorf("%s is still there after -clean", gone) + } + } + if !exists(t, out, motorinoMarker) { + t.Error("the marker is gone after -clean") + } + }) + } +} + +// TestCleanTakesAnEmptyOrNewFolder checks that -clean needs no marker for a folder with nothing in it, and marks it. +func TestCleanTakesAnEmptyOrNewFolder(t *testing.T) { + for name, run := range builders(t, withFiles(nil)) { + t.Run(name, func(t *testing.T) { + for _, out := range []string{t.TempDir(), filepath.Join(t.TempDir(), "new", "deeper")} { + if _, err := run(out, true); err != nil { + t.Fatalf("build into %s error = %v", out, err) + } + if !exists(t, out, motorinoMarker) { + t.Errorf("no marker in %s", out) + } + } + }) + } +} + +// TestCleanRefusesAFolderThatIsNotMotorinos checks that -clean leaves a folder with files and no marker alone. +func TestCleanRefusesAFolderThatIsNotMotorinos(t *testing.T) { + for name, run := range builders(t, withFiles(nil)) { + t.Run(name, func(t *testing.T) { + out := t.TempDir() + put(t, out, "precious.txt", "keep me") + _, err := run(out, true) + if err == nil || !strings.Contains(err.Error(), motorinoMarker) { + t.Fatalf("error = %v, want one that names %s", err, motorinoMarker) + } + if got := readOut(t, out, "precious.txt"); got != "keep me" { + t.Errorf("precious.txt = %q", got) + } + if exists(t, out, "index.html") || exists(t, out, "index.gmi") || exists(t, out, "en/index.gmi") { + t.Error("the build wrote into a folder it refused to empty") + } + }) + } +} + +// TestCleanRefusesAFolderThatHoldsTheSite checks that -clean never empties the folder of the site or one above it, +// even when it has the marker. +func TestCleanRefusesAFolderThatHoldsTheSite(t *testing.T) { + parent := t.TempDir() + site := filepath.Join(parent, "site") + for rel, text := range withFiles(nil) { + put(t, site, rel, text) + } + put(t, parent, motorinoMarker, "x") + for _, out := range []string{site, parent} { + if _, err := Build(Options{SiteDir: site, OutDir: out, Now: fixedNow, Clean: true}); err == nil || !strings.Contains(err.Error(), "site") { + t.Errorf("Build into %s: error = %v, want one about the site", out, err) + } + } + if !exists(t, site, "site.json") { + t.Error("the site is gone") + } +}