From 52edb15b64d95959e6acdec9bc194cd2cc6aba7e Mon Sep 17 00:00:00 2001 From: tgiachi Date: Mon, 16 Nov 2026 12:49:17 +0100 Subject: [PATCH] feat(build): report the old files of the output folder, add -clean A build never emptied the output folder, so the pages of a list that got shorter stayed behind. The build now returns the files it did not write and the command prints a warning that names them. -clean empties the folder first, only when it is new, empty or holds the .motorino-out file that an earlier -clean left, and never the folder of the site or one that holds it. The default is unchanged: no extra file in a folder built without it, so the deploy (a new folder published with rsync --delete) is not affected. Refs #9 --- README.md | 3 + cmd/motorino/main.go | 21 ++++- cmd/motorino/main_test.go | 54 ++++++++++++ internal/build/build.go | 40 +++++---- internal/build/output.go | 105 ++++++++++++++++++++++ internal/build/output_test.go | 162 ++++++++++++++++++++++++++++++++++ 6 files changed, 363 insertions(+), 22 deletions(-) create mode 100644 internal/build/output.go create mode 100644 internal/build/output_test.go diff --git a/README.md b/README.md index 1b58d2c..d0cbb14 100644 --- a/README.md +++ b/README.md @@ -9,12 +9,15 @@ Licence: GPL-3.0-or-later, see `LICENSE`. ``` motorino build -site DIR -out DIR build the site in DIR into the output folder (defaults: . and public) motorino build -drafts ... the same, with the drafts and the posts dated in the future, for a preview +motorino build -clean ... empty the output folder first, if it is new, empty or holds .motorino-out motorino new -site DIR "A title" write news/a-title.md, a draft with the header ready, and print its path motorino version print the version ``` The binary is a single static Go program. The version comes from semantic-release. +A build writes the files it makes and leaves the others in the output folder, so the pages of a list that got shorter (`news/page/3/`) stay behind. When that happens the command prints a warning that names the old files. The deploy never has them: it builds into a new folder and publishes it with `rsync --delete`. For a folder you reuse, `-clean` empties it first. It does that only for a folder that is new, empty or holds the `.motorino-out` file that an earlier `-clean` left, never for any other folder with files in it and never for the folder of the site or one that holds it. A build without `-clean` writes no extra file. + ## What a site looks like ``` diff --git a/cmd/motorino/main.go b/cmd/motorino/main.go index 88ca4e9..7443176 100644 --- a/cmd/motorino/main.go +++ b/cmd/motorino/main.go @@ -5,7 +5,7 @@ // // Usage: // -// motorino build [-site DIR] [-out DIR] [-drafts] +// motorino build [-site DIR] [-out DIR] [-drafts] [-clean] // motorino new [-site DIR] "title" // motorino version package main @@ -26,7 +26,7 @@ import ( ) // usage is the text printed for a missing or unknown command. -const usage = "usage: motorino build [-site DIR] [-out DIR] [-drafts] | motorino new [-site DIR] \"title\" | motorino version" +const usage = "usage: motorino build [-site DIR] [-out DIR] [-drafts] [-clean] | motorino new [-site DIR] \"title\" | motorino version" // main runs the command line and exits with its status code. func main() { @@ -61,6 +61,7 @@ func runBuild(args []string, stdout, stderr io.Writer) int { site := flags.String("site", ".", "the folder of the site") out := flags.String("out", "public", "the folder the HTML is written to") drafts := flags.Bool("drafts", false, "publish the drafts and the posts dated in the future, for a preview") + clean := flags.Bool("clean", false, "empty the output folder first, if it is new, empty or was emptied this way before") if err := flags.Parse(args); err != nil { if errors.Is(err, flag.ErrHelp) { return 0 @@ -71,15 +72,29 @@ func runBuild(args []string, stdout, stderr io.Writer) int { fmt.Fprintf(stderr, "motorino: unexpected arguments: %s\n%s\n", strings.Join(flags.Args(), " "), usage) return 2 } - result, err := build.Build(build.Options{SiteDir: *site, OutDir: *out, Version: version.String(), Drafts: *drafts}) + result, err := build.Build(build.Options{SiteDir: *site, OutDir: *out, Version: version.String(), Drafts: *drafts, Clean: *clean}) if err != nil { fmt.Fprintf(stderr, "motorino: %v\n", err) return 1 } fmt.Fprintf(stdout, "built %d pages into %s\n", result.Pages, *out) + warnStale(stderr, *out, result.Stale) return 0 } +// warnStale tells the files of the output folder that the build did not write, +// which stay there unless -clean is used. +func warnStale(stderr io.Writer, out string, stale []string) { + if len(stale) == 0 { + return + } + noun := "files" + if len(stale) == 1 { + noun = "file" + } + fmt.Fprintf(stderr, "motorino: warning: %d %s in %s were not written by this build, such as %s; build into a new folder or use -clean\n", len(stale), noun, out, stale[0]) +} + // runNew parses the flags of the new command and writes the file of a new post. // The author is the author key of site.json, when it has one. func runNew(args []string, stdout, stderr io.Writer) int { diff --git a/cmd/motorino/main_test.go b/cmd/motorino/main_test.go index 8798844..6ecd765 100644 --- a/cmd/motorino/main_test.go +++ b/cmd/motorino/main_test.go @@ -187,3 +187,57 @@ func TestRunBuildDraftsFlag(t *testing.T) { t.Errorf("a draft is missing from the build with -drafts: %v", err) } } + +// TestRunBuildWarnsAboutOldFiles checks that old files in the output folder are named in a warning and left alone. +func TestRunBuildWarnsAboutOldFiles(t *testing.T) { + site := miniSite(t) + outDir := t.TempDir() + if err := os.WriteFile(filepath.Join(outDir, "old.html"), []byte("old"), 0o644); err != nil { + t.Fatal(err) + } + var out, errOut bytes.Buffer + if code := run([]string{"build", "-site", site, "-out", outDir}, &out, &errOut); code != 0 { + t.Fatalf("run(build) = %d, stderr %q", code, errOut.String()) + } + for _, want := range []string{"warning", "1 file", "old.html", "-clean"} { + if !strings.Contains(errOut.String(), want) { + t.Errorf("stderr %q does not contain %q", errOut.String(), want) + } + } + if _, err := os.Stat(filepath.Join(outDir, "old.html")); err != nil { + t.Error("a build without -clean removed the old file") + } + errOut.Reset() + if code := run([]string{"build", "-site", site, "-out", filepath.Join(t.TempDir(), "new")}, &out, &errOut); code != 0 || errOut.Len() != 0 { + t.Errorf("a build into a new folder: code %d, stderr %q, want 0 and nothing", code, errOut.String()) + } +} + +// TestRunBuildClean checks -clean on a folder that is marked and on one that is not. +func TestRunBuildClean(t *testing.T) { + site := miniSite(t) + var out, errOut bytes.Buffer + outDir := filepath.Join(t.TempDir(), "public") + if code := run([]string{"build", "-clean", "-site", site, "-out", outDir}, &out, &errOut); code != 0 { + t.Fatalf("run(build -clean) into a new folder = %d, stderr %q", code, errOut.String()) + } + if err := os.WriteFile(filepath.Join(outDir, "old.html"), []byte("old"), 0o644); err != nil { + t.Fatal(err) + } + if code := run([]string{"build", "-clean", "-site", site, "-out", outDir}, &out, &errOut); code != 0 || errOut.Len() != 0 { + t.Fatalf("run(build -clean) again = %d, stderr %q", code, errOut.String()) + } + if _, err := os.Stat(filepath.Join(outDir, "old.html")); err == nil { + t.Error("-clean left the old file") + } + other := t.TempDir() + if err := os.WriteFile(filepath.Join(other, "mine.txt"), []byte("mine"), 0o644); err != nil { + t.Fatal(err) + } + if code := run([]string{"build", "-clean", "-site", site, "-out", other}, &out, &errOut); code != 1 || !strings.Contains(errOut.String(), ".motorino-out") { + t.Errorf("run(build -clean) into a folder that is not motorino's = %d, stderr %q, want 1 and the marker named", code, errOut.String()) + } + if _, err := os.Stat(filepath.Join(other, "mine.txt")); err != nil { + t.Error("-clean removed a file of a folder that is not motorino's") + } +} diff --git a/internal/build/build.go b/internal/build/build.go index fa57268..f38ea46 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -31,6 +31,10 @@ type Options struct { // Drafts publishes the drafts and the posts dated in the future too, for a // preview of the site. Drafts bool + // Clean empties OutDir before the site is written, when it is new, empty or + // was emptied this way before (it holds the file .motorino-out); any other + // folder with files in it is an error. Without it old files stay. + Clean bool } // requiredPages are the IDs every site must have, because motorino builds the @@ -93,13 +97,18 @@ type categoryEntry struct { type Result struct { // Pages is the number of HTML pages written: pages, news and categories. Pages int + // Stale lists the files that were already in the output folder and that this + // build did not write, such as the pages of a list that got shorter. They are + // left alone unless Clean is set, which removes them first. + Stale []string } // Build reads the site in opts.SiteDir and writes it to opts.OutDir. It renders // everything in memory first, so a failure in the content leaves the output // folder untouched. Writing the files is not atomic, and files of an earlier -// build that are no longer produced are not removed: build into a fresh folder -// when that matters, as the deploy does. +// build that are no longer produced are not removed, but listed in Result.Stale: +// build into a fresh folder when that matters, as the deploy does (it publishes +// the new folder with rsync --delete), or set Options.Clean. func Build(opts Options) (Result, error) { b, err := load(opts) if err != nil { @@ -127,10 +136,18 @@ func Build(opts Options) (Result, error) { if err != nil { return Result{}, err } - if err := b.write(assets); err != nil { + files := make(map[string][]byte, len(b.files)+len(assets)) + for rel, text := range b.files { + files[rel] = []byte(text) + } + for rel, data := range assets { + files[rel] = data + } + stale, err := writeOutput(opts.OutDir, opts.SiteDir, opts.Clean, files) + if err != nil { return Result{}, err } - return Result{Pages: len(b.indexable)}, nil + return Result{Pages: len(b.indexable), Stale: stale}, nil } // load reads the configuration, the content, the templates and the style. @@ -331,21 +348,6 @@ func (b *builder) readAssets() (map[string][]byte, error) { return out, nil } -// write creates the output folder and writes the pages and the assets. -func (b *builder) write(assets map[string][]byte) error { - for rel, text := range b.files { - if err := writeFile(b.opts.OutDir, rel, []byte(text)); err != nil { - return err - } - } - for rel, data := range assets { - if err := writeFile(b.opts.OutDir, rel, data); err != nil { - return err - } - } - return nil -} - // writeFile writes data to rel under dir, creating the folders. func writeFile(dir, rel string, data []byte) error { path := filepath.Join(dir, filepath.FromSlash(rel)) diff --git a/internal/build/output.go b/internal/build/output.go new file mode 100644 index 0000000..0dbe133 --- /dev/null +++ b/internal/build/output.go @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package build + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +// motorinoMarker is the file motorino leaves in a folder it emptied with the +// clean option. A folder that has it, or has nothing in it, may be emptied again; +// any other folder never is. It is only written when the option is used, so the +// folders of a build without it hold the site and nothing else. +const motorinoMarker = ".motorino-out" + +// writeOutput writes the files (path with slashes, relative to out) into out and +// returns the files that were already in out and are not among them: the pages +// of an earlier build that this one no longer makes. With clean set it first +// empties out, but only a folder that is new, empty or marked by an earlier clean +// build, and never the folder of the site or one that holds it. It does not +// touch the folder before every file is known, so a content error leaves it as +// it was. +func writeOutput(out, siteDir string, clean bool, files map[string][]byte) ([]string, error) { + if clean { + if err := emptyOutput(out, siteDir); err != nil { + return nil, err + } + files = withMarker(files) + } + for rel, data := range files { + if err := writeFile(out, rel, data); err != nil { + return nil, err + } + } + return staleFiles(out, files) +} + +// withMarker returns the files and the marker. +func withMarker(files map[string][]byte) map[string][]byte { + marked := make(map[string][]byte, len(files)+1) + for rel, data := range files { + marked[rel] = data + } + marked[motorinoMarker] = []byte("This folder is written by motorino and emptied by motorino build -clean and motorino gemini -clean.\n") + return marked +} + +// emptyOutput removes what is in out, after checking that it may be removed. +func emptyOutput(out, siteDir string) error { + absOut, err := filepath.Abs(out) + if err != nil { + return err + } + absSite, err := filepath.Abs(siteDir) + if err != nil { + return err + } + if rel, err := filepath.Rel(absOut, absSite); err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return fmt.Errorf("refusing to empty %s: it is the folder of the site or holds it", out) + } + entries, err := os.ReadDir(out) + if os.IsNotExist(err) || (err == nil && len(entries) == 0) { + return nil + } + if err != nil { + return fmt.Errorf("read %s: %w", out, err) + } + if info, err := os.Lstat(filepath.Join(out, motorinoMarker)); err != nil || !info.Mode().IsRegular() { + return fmt.Errorf("refusing to empty %s: it has files and no %s, which motorino leaves in a folder it emptied; empty it yourself this first time", out, motorinoMarker) + } + for _, entry := range entries { + if err := os.RemoveAll(filepath.Join(out, entry.Name())); err != nil { + return fmt.Errorf("empty %s: %w", out, err) + } + } + return nil +} + +// staleFiles lists the files under out that are not among the written ones, with +// slashes, sorted. The marker is never one of them. +func staleFiles(out string, written map[string][]byte) ([]string, error) { + var stale []string + err := filepath.WalkDir(out, func(path string, entry os.DirEntry, err error) error { + if err != nil || entry.IsDir() { + return err + } + rel, err := filepath.Rel(out, path) + if err != nil { + return err + } + rel = filepath.ToSlash(rel) + if _, ok := written[rel]; !ok && rel != motorinoMarker { + stale = append(stale, rel) + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("read %s: %w", out, err) + } + sort.Strings(stale) + return stale, nil +} diff --git a/internal/build/output_test.go b/internal/build/output_test.go new file mode 100644 index 0000000..8fd0efa --- /dev/null +++ b/internal/build/output_test.go @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package build + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// put writes a file of a test folder. +func put(t *testing.T, dir, rel, text string) { + t.Helper() + path := filepath.Join(dir, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(text), 0o644); err != nil { + t.Fatal(err) + } +} + +// extraBuilders are the other outputs the tests of the output folder run against, by name: each takes the +// folder of a site and returns a function that builds it into a folder. +var extraBuilders = map[string]func(site string) func(out string, clean bool) ([]string, error){} + +// builders are the outputs, each as a function that builds the site made of files into a folder and returns +// the files of the folder that the build did not write. +func builders(t *testing.T, files map[string]string) map[string]func(out string, clean bool) ([]string, error) { + t.Helper() + site := writeFiles(t, files) + all := map[string]func(string, bool) ([]string, error){ + "html": func(out string, clean bool) ([]string, error) { + r, err := Build(Options{SiteDir: site, OutDir: out, Version: "1.2.3", Now: fixedNow, Clean: clean}) + return r.Stale, err + }, + } + for name, newBuilder := range extraBuilders { + all[name] = newBuilder(site) + } + return all +} + +// TestStaleFilesAreReported checks that a build into a folder with files it did not write reports them and leaves them. +func TestStaleFilesAreReported(t *testing.T) { + for name, run := range builders(t, withFiles(nil)) { + t.Run(name, func(t *testing.T) { + out := t.TempDir() + put(t, out, "news/page/2/index.html", "old") + put(t, out, "old.txt", "old") + stale, err := run(out, false) + if err != nil { + t.Fatalf("build error = %v", err) + } + if strings.Join(stale, ",") != "news/page/2/index.html,old.txt" { + t.Errorf("stale = %v, want the two old files", stale) + } + if !exists(t, out, "old.txt") { + t.Error("a build without -clean removed a file") + } + if exists(t, out, motorinoMarker) { + t.Error("a build without -clean wrote the marker into the folder") + } + }) + } +} + +// TestFreshFolderHasNothingStale checks that a build into a new folder reports nothing and writes only the site. +func TestFreshFolderHasNothingStale(t *testing.T) { + for name, run := range builders(t, withFiles(nil)) { + t.Run(name, func(t *testing.T) { + stale, err := run(filepath.Join(t.TempDir(), "new"), false) + if err != nil || len(stale) != 0 { + t.Errorf("stale = %v, error = %v", stale, err) + } + }) + } +} + +// TestCleanEmptiesAFolderItMarked checks that -clean removes the old files of a folder that has the marker, +// the pages of a list that got shorter included, and writes the marker again. +func TestCleanEmptiesAFolderItMarked(t *testing.T) { + for name, run := range builders(t, withFiles(nil)) { + t.Run(name, func(t *testing.T) { + out := t.TempDir() + put(t, out, motorinoMarker, "x") + put(t, out, "news/page/2/index.html", "old") + put(t, out, "old/deep/file.txt", "old") + stale, err := run(out, true) + if err != nil { + t.Fatalf("build error = %v", err) + } + if len(stale) != 0 { + t.Errorf("stale = %v after a clean build", stale) + } + for _, gone := range []string{"news/page/2/index.html", "old/deep/file.txt"} { + if exists(t, out, gone) { + t.Errorf("%s is still there after -clean", gone) + } + } + if !exists(t, out, motorinoMarker) { + t.Error("the marker is gone after -clean") + } + }) + } +} + +// TestCleanTakesAnEmptyOrNewFolder checks that -clean needs no marker for a folder with nothing in it, and marks it. +func TestCleanTakesAnEmptyOrNewFolder(t *testing.T) { + for name, run := range builders(t, withFiles(nil)) { + t.Run(name, func(t *testing.T) { + for _, out := range []string{t.TempDir(), filepath.Join(t.TempDir(), "new", "deeper")} { + if _, err := run(out, true); err != nil { + t.Fatalf("build into %s error = %v", out, err) + } + if !exists(t, out, motorinoMarker) { + t.Errorf("no marker in %s", out) + } + } + }) + } +} + +// TestCleanRefusesAFolderThatIsNotMotorinos checks that -clean leaves a folder with files and no marker alone. +func TestCleanRefusesAFolderThatIsNotMotorinos(t *testing.T) { + for name, run := range builders(t, withFiles(nil)) { + t.Run(name, func(t *testing.T) { + out := t.TempDir() + put(t, out, "precious.txt", "keep me") + _, err := run(out, true) + if err == nil || !strings.Contains(err.Error(), motorinoMarker) { + t.Fatalf("error = %v, want one that names %s", err, motorinoMarker) + } + if got := readOut(t, out, "precious.txt"); got != "keep me" { + t.Errorf("precious.txt = %q", got) + } + if exists(t, out, "index.html") || exists(t, out, "index.gmi") || exists(t, out, "en/index.gmi") { + t.Error("the build wrote into a folder it refused to empty") + } + }) + } +} + +// TestCleanRefusesAFolderThatHoldsTheSite checks that -clean never empties the folder of the site or one above it, +// even when it has the marker. +func TestCleanRefusesAFolderThatHoldsTheSite(t *testing.T) { + parent := t.TempDir() + site := filepath.Join(parent, "site") + for rel, text := range withFiles(nil) { + put(t, site, rel, text) + } + put(t, parent, motorinoMarker, "x") + for _, out := range []string{site, parent} { + if _, err := Build(Options{SiteDir: site, OutDir: out, Now: fixedNow, Clean: true}); err == nil || !strings.Contains(err.Error(), "site") { + t.Errorf("Build into %s: error = %v, want one about the site", out, err) + } + } + if !exists(t, site, "site.json") { + t.Error("the site is gone") + } +}