feat(build): report the old files of the output folder, add -clean
All checks were successful
check / check (pull_request) Successful in 29s

A build never emptied the output folder, so the pages of a list that got shorter stayed
behind. The build now returns the files it did not write and the command prints a
warning that names them. -clean empties the folder first, only when it is new, empty or
holds the .motorino-out file that an earlier -clean left, and never the folder of the site
or one that holds it. The default is unchanged: no extra file in a folder built without it,
so the deploy (a new folder published with rsync --delete) is not affected.

Refs #9
This commit is contained in:
tgiachi 2026-11-16 12:49:17 +01:00
commit 52edb15b64
6 changed files with 363 additions and 22 deletions

View file

@ -9,12 +9,15 @@ Licence: GPL-3.0-or-later, see `LICENSE`.
``` ```
motorino build -site DIR -out DIR build the site in DIR into the output folder (defaults: . and public) motorino build -site DIR -out DIR build the site in DIR into the output folder (defaults: . and public)
motorino build -drafts ... the same, with the drafts and the posts dated in the future, for a preview motorino build -drafts ... the same, with the drafts and the posts dated in the future, for a preview
motorino build -clean ... empty the output folder first, if it is new, empty or holds .motorino-out
motorino new -site DIR "A title" write news/a-title.md, a draft with the header ready, and print its path motorino new -site DIR "A title" write news/a-title.md, a draft with the header ready, and print its path
motorino version print the version motorino version print the version
``` ```
The binary is a single static Go program. The version comes from semantic-release. The binary is a single static Go program. The version comes from semantic-release.
A build writes the files it makes and leaves the others in the output folder, so the pages of a list that got shorter (`news/page/3/`) stay behind. When that happens the command prints a warning that names the old files. The deploy never has them: it builds into a new folder and publishes it with `rsync --delete`. For a folder you reuse, `-clean` empties it first. It does that only for a folder that is new, empty or holds the `.motorino-out` file that an earlier `-clean` left, never for any other folder with files in it and never for the folder of the site or one that holds it. A build without `-clean` writes no extra file.
## What a site looks like ## What a site looks like
``` ```

View file

@ -5,7 +5,7 @@
// //
// Usage: // Usage:
// //
// motorino build [-site DIR] [-out DIR] [-drafts] // motorino build [-site DIR] [-out DIR] [-drafts] [-clean]
// motorino new [-site DIR] "title" // motorino new [-site DIR] "title"
// motorino version // motorino version
package main package main
@ -26,7 +26,7 @@ import (
) )
// usage is the text printed for a missing or unknown command. // usage is the text printed for a missing or unknown command.
const usage = "usage: motorino build [-site DIR] [-out DIR] [-drafts] | motorino new [-site DIR] \"title\" | motorino version" const usage = "usage: motorino build [-site DIR] [-out DIR] [-drafts] [-clean] | motorino new [-site DIR] \"title\" | motorino version"
// main runs the command line and exits with its status code. // main runs the command line and exits with its status code.
func main() { func main() {
@ -61,6 +61,7 @@ func runBuild(args []string, stdout, stderr io.Writer) int {
site := flags.String("site", ".", "the folder of the site") site := flags.String("site", ".", "the folder of the site")
out := flags.String("out", "public", "the folder the HTML is written to") out := flags.String("out", "public", "the folder the HTML is written to")
drafts := flags.Bool("drafts", false, "publish the drafts and the posts dated in the future, for a preview") drafts := flags.Bool("drafts", false, "publish the drafts and the posts dated in the future, for a preview")
clean := flags.Bool("clean", false, "empty the output folder first, if it is new, empty or was emptied this way before")
if err := flags.Parse(args); err != nil { if err := flags.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) { if errors.Is(err, flag.ErrHelp) {
return 0 return 0
@ -71,15 +72,29 @@ func runBuild(args []string, stdout, stderr io.Writer) int {
fmt.Fprintf(stderr, "motorino: unexpected arguments: %s\n%s\n", strings.Join(flags.Args(), " "), usage) fmt.Fprintf(stderr, "motorino: unexpected arguments: %s\n%s\n", strings.Join(flags.Args(), " "), usage)
return 2 return 2
} }
result, err := build.Build(build.Options{SiteDir: *site, OutDir: *out, Version: version.String(), Drafts: *drafts}) result, err := build.Build(build.Options{SiteDir: *site, OutDir: *out, Version: version.String(), Drafts: *drafts, Clean: *clean})
if err != nil { if err != nil {
fmt.Fprintf(stderr, "motorino: %v\n", err) fmt.Fprintf(stderr, "motorino: %v\n", err)
return 1 return 1
} }
fmt.Fprintf(stdout, "built %d pages into %s\n", result.Pages, *out) fmt.Fprintf(stdout, "built %d pages into %s\n", result.Pages, *out)
warnStale(stderr, *out, result.Stale)
return 0 return 0
} }
// warnStale tells the files of the output folder that the build did not write,
// which stay there unless -clean is used.
func warnStale(stderr io.Writer, out string, stale []string) {
if len(stale) == 0 {
return
}
noun := "files"
if len(stale) == 1 {
noun = "file"
}
fmt.Fprintf(stderr, "motorino: warning: %d %s in %s were not written by this build, such as %s; build into a new folder or use -clean\n", len(stale), noun, out, stale[0])
}
// runNew parses the flags of the new command and writes the file of a new post. // runNew parses the flags of the new command and writes the file of a new post.
// The author is the author key of site.json, when it has one. // The author is the author key of site.json, when it has one.
func runNew(args []string, stdout, stderr io.Writer) int { func runNew(args []string, stdout, stderr io.Writer) int {

View file

@ -187,3 +187,57 @@ func TestRunBuildDraftsFlag(t *testing.T) {
t.Errorf("a draft is missing from the build with -drafts: %v", err) t.Errorf("a draft is missing from the build with -drafts: %v", err)
} }
} }
// TestRunBuildWarnsAboutOldFiles checks that old files in the output folder are named in a warning and left alone.
func TestRunBuildWarnsAboutOldFiles(t *testing.T) {
site := miniSite(t)
outDir := t.TempDir()
if err := os.WriteFile(filepath.Join(outDir, "old.html"), []byte("old"), 0o644); err != nil {
t.Fatal(err)
}
var out, errOut bytes.Buffer
if code := run([]string{"build", "-site", site, "-out", outDir}, &out, &errOut); code != 0 {
t.Fatalf("run(build) = %d, stderr %q", code, errOut.String())
}
for _, want := range []string{"warning", "1 file", "old.html", "-clean"} {
if !strings.Contains(errOut.String(), want) {
t.Errorf("stderr %q does not contain %q", errOut.String(), want)
}
}
if _, err := os.Stat(filepath.Join(outDir, "old.html")); err != nil {
t.Error("a build without -clean removed the old file")
}
errOut.Reset()
if code := run([]string{"build", "-site", site, "-out", filepath.Join(t.TempDir(), "new")}, &out, &errOut); code != 0 || errOut.Len() != 0 {
t.Errorf("a build into a new folder: code %d, stderr %q, want 0 and nothing", code, errOut.String())
}
}
// TestRunBuildClean checks -clean on a folder that is marked and on one that is not.
func TestRunBuildClean(t *testing.T) {
site := miniSite(t)
var out, errOut bytes.Buffer
outDir := filepath.Join(t.TempDir(), "public")
if code := run([]string{"build", "-clean", "-site", site, "-out", outDir}, &out, &errOut); code != 0 {
t.Fatalf("run(build -clean) into a new folder = %d, stderr %q", code, errOut.String())
}
if err := os.WriteFile(filepath.Join(outDir, "old.html"), []byte("old"), 0o644); err != nil {
t.Fatal(err)
}
if code := run([]string{"build", "-clean", "-site", site, "-out", outDir}, &out, &errOut); code != 0 || errOut.Len() != 0 {
t.Fatalf("run(build -clean) again = %d, stderr %q", code, errOut.String())
}
if _, err := os.Stat(filepath.Join(outDir, "old.html")); err == nil {
t.Error("-clean left the old file")
}
other := t.TempDir()
if err := os.WriteFile(filepath.Join(other, "mine.txt"), []byte("mine"), 0o644); err != nil {
t.Fatal(err)
}
if code := run([]string{"build", "-clean", "-site", site, "-out", other}, &out, &errOut); code != 1 || !strings.Contains(errOut.String(), ".motorino-out") {
t.Errorf("run(build -clean) into a folder that is not motorino's = %d, stderr %q, want 1 and the marker named", code, errOut.String())
}
if _, err := os.Stat(filepath.Join(other, "mine.txt")); err != nil {
t.Error("-clean removed a file of a folder that is not motorino's")
}
}

View file

@ -31,6 +31,10 @@ type Options struct {
// Drafts publishes the drafts and the posts dated in the future too, for a // Drafts publishes the drafts and the posts dated in the future too, for a
// preview of the site. // preview of the site.
Drafts bool Drafts bool
// Clean empties OutDir before the site is written, when it is new, empty or
// was emptied this way before (it holds the file .motorino-out); any other
// folder with files in it is an error. Without it old files stay.
Clean bool
} }
// requiredPages are the IDs every site must have, because motorino builds the // requiredPages are the IDs every site must have, because motorino builds the
@ -93,13 +97,18 @@ type categoryEntry struct {
type Result struct { type Result struct {
// Pages is the number of HTML pages written: pages, news and categories. // Pages is the number of HTML pages written: pages, news and categories.
Pages int Pages int
// Stale lists the files that were already in the output folder and that this
// build did not write, such as the pages of a list that got shorter. They are
// left alone unless Clean is set, which removes them first.
Stale []string
} }
// Build reads the site in opts.SiteDir and writes it to opts.OutDir. It renders // Build reads the site in opts.SiteDir and writes it to opts.OutDir. It renders
// everything in memory first, so a failure in the content leaves the output // everything in memory first, so a failure in the content leaves the output
// folder untouched. Writing the files is not atomic, and files of an earlier // folder untouched. Writing the files is not atomic, and files of an earlier
// build that are no longer produced are not removed: build into a fresh folder // build that are no longer produced are not removed, but listed in Result.Stale:
// when that matters, as the deploy does. // build into a fresh folder when that matters, as the deploy does (it publishes
// the new folder with rsync --delete), or set Options.Clean.
func Build(opts Options) (Result, error) { func Build(opts Options) (Result, error) {
b, err := load(opts) b, err := load(opts)
if err != nil { if err != nil {
@ -127,10 +136,18 @@ func Build(opts Options) (Result, error) {
if err != nil { if err != nil {
return Result{}, err return Result{}, err
} }
if err := b.write(assets); err != nil { files := make(map[string][]byte, len(b.files)+len(assets))
for rel, text := range b.files {
files[rel] = []byte(text)
}
for rel, data := range assets {
files[rel] = data
}
stale, err := writeOutput(opts.OutDir, opts.SiteDir, opts.Clean, files)
if err != nil {
return Result{}, err return Result{}, err
} }
return Result{Pages: len(b.indexable)}, nil return Result{Pages: len(b.indexable), Stale: stale}, nil
} }
// load reads the configuration, the content, the templates and the style. // load reads the configuration, the content, the templates and the style.
@ -331,21 +348,6 @@ func (b *builder) readAssets() (map[string][]byte, error) {
return out, nil return out, nil
} }
// write creates the output folder and writes the pages and the assets.
func (b *builder) write(assets map[string][]byte) error {
for rel, text := range b.files {
if err := writeFile(b.opts.OutDir, rel, []byte(text)); err != nil {
return err
}
}
for rel, data := range assets {
if err := writeFile(b.opts.OutDir, rel, data); err != nil {
return err
}
}
return nil
}
// writeFile writes data to rel under dir, creating the folders. // writeFile writes data to rel under dir, creating the folders.
func writeFile(dir, rel string, data []byte) error { func writeFile(dir, rel string, data []byte) error {
path := filepath.Join(dir, filepath.FromSlash(rel)) path := filepath.Join(dir, filepath.FromSlash(rel))

105
internal/build/output.go Normal file
View file

@ -0,0 +1,105 @@
// SPDX-License-Identifier: GPL-3.0-or-later
package build
import (
"fmt"
"os"
"path/filepath"
"sort"
"strings"
)
// motorinoMarker is the file motorino leaves in a folder it emptied with the
// clean option. A folder that has it, or has nothing in it, may be emptied again;
// any other folder never is. It is only written when the option is used, so the
// folders of a build without it hold the site and nothing else.
const motorinoMarker = ".motorino-out"
// writeOutput writes the files (path with slashes, relative to out) into out and
// returns the files that were already in out and are not among them: the pages
// of an earlier build that this one no longer makes. With clean set it first
// empties out, but only a folder that is new, empty or marked by an earlier clean
// build, and never the folder of the site or one that holds it. It does not
// touch the folder before every file is known, so a content error leaves it as
// it was.
func writeOutput(out, siteDir string, clean bool, files map[string][]byte) ([]string, error) {
if clean {
if err := emptyOutput(out, siteDir); err != nil {
return nil, err
}
files = withMarker(files)
}
for rel, data := range files {
if err := writeFile(out, rel, data); err != nil {
return nil, err
}
}
return staleFiles(out, files)
}
// withMarker returns the files and the marker.
func withMarker(files map[string][]byte) map[string][]byte {
marked := make(map[string][]byte, len(files)+1)
for rel, data := range files {
marked[rel] = data
}
marked[motorinoMarker] = []byte("This folder is written by motorino and emptied by motorino build -clean and motorino gemini -clean.\n")
return marked
}
// emptyOutput removes what is in out, after checking that it may be removed.
func emptyOutput(out, siteDir string) error {
absOut, err := filepath.Abs(out)
if err != nil {
return err
}
absSite, err := filepath.Abs(siteDir)
if err != nil {
return err
}
if rel, err := filepath.Rel(absOut, absSite); err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
return fmt.Errorf("refusing to empty %s: it is the folder of the site or holds it", out)
}
entries, err := os.ReadDir(out)
if os.IsNotExist(err) || (err == nil && len(entries) == 0) {
return nil
}
if err != nil {
return fmt.Errorf("read %s: %w", out, err)
}
if info, err := os.Lstat(filepath.Join(out, motorinoMarker)); err != nil || !info.Mode().IsRegular() {
return fmt.Errorf("refusing to empty %s: it has files and no %s, which motorino leaves in a folder it emptied; empty it yourself this first time", out, motorinoMarker)
}
for _, entry := range entries {
if err := os.RemoveAll(filepath.Join(out, entry.Name())); err != nil {
return fmt.Errorf("empty %s: %w", out, err)
}
}
return nil
}
// staleFiles lists the files under out that are not among the written ones, with
// slashes, sorted. The marker is never one of them.
func staleFiles(out string, written map[string][]byte) ([]string, error) {
var stale []string
err := filepath.WalkDir(out, func(path string, entry os.DirEntry, err error) error {
if err != nil || entry.IsDir() {
return err
}
rel, err := filepath.Rel(out, path)
if err != nil {
return err
}
rel = filepath.ToSlash(rel)
if _, ok := written[rel]; !ok && rel != motorinoMarker {
stale = append(stale, rel)
}
return nil
})
if err != nil {
return nil, fmt.Errorf("read %s: %w", out, err)
}
sort.Strings(stale)
return stale, nil
}

View file

@ -0,0 +1,162 @@
// SPDX-License-Identifier: GPL-3.0-or-later
package build
import (
"os"
"path/filepath"
"strings"
"testing"
)
// put writes a file of a test folder.
func put(t *testing.T, dir, rel, text string) {
t.Helper()
path := filepath.Join(dir, filepath.FromSlash(rel))
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte(text), 0o644); err != nil {
t.Fatal(err)
}
}
// extraBuilders are the other outputs the tests of the output folder run against, by name: each takes the
// folder of a site and returns a function that builds it into a folder.
var extraBuilders = map[string]func(site string) func(out string, clean bool) ([]string, error){}
// builders are the outputs, each as a function that builds the site made of files into a folder and returns
// the files of the folder that the build did not write.
func builders(t *testing.T, files map[string]string) map[string]func(out string, clean bool) ([]string, error) {
t.Helper()
site := writeFiles(t, files)
all := map[string]func(string, bool) ([]string, error){
"html": func(out string, clean bool) ([]string, error) {
r, err := Build(Options{SiteDir: site, OutDir: out, Version: "1.2.3", Now: fixedNow, Clean: clean})
return r.Stale, err
},
}
for name, newBuilder := range extraBuilders {
all[name] = newBuilder(site)
}
return all
}
// TestStaleFilesAreReported checks that a build into a folder with files it did not write reports them and leaves them.
func TestStaleFilesAreReported(t *testing.T) {
for name, run := range builders(t, withFiles(nil)) {
t.Run(name, func(t *testing.T) {
out := t.TempDir()
put(t, out, "news/page/2/index.html", "old")
put(t, out, "old.txt", "old")
stale, err := run(out, false)
if err != nil {
t.Fatalf("build error = %v", err)
}
if strings.Join(stale, ",") != "news/page/2/index.html,old.txt" {
t.Errorf("stale = %v, want the two old files", stale)
}
if !exists(t, out, "old.txt") {
t.Error("a build without -clean removed a file")
}
if exists(t, out, motorinoMarker) {
t.Error("a build without -clean wrote the marker into the folder")
}
})
}
}
// TestFreshFolderHasNothingStale checks that a build into a new folder reports nothing and writes only the site.
func TestFreshFolderHasNothingStale(t *testing.T) {
for name, run := range builders(t, withFiles(nil)) {
t.Run(name, func(t *testing.T) {
stale, err := run(filepath.Join(t.TempDir(), "new"), false)
if err != nil || len(stale) != 0 {
t.Errorf("stale = %v, error = %v", stale, err)
}
})
}
}
// TestCleanEmptiesAFolderItMarked checks that -clean removes the old files of a folder that has the marker,
// the pages of a list that got shorter included, and writes the marker again.
func TestCleanEmptiesAFolderItMarked(t *testing.T) {
for name, run := range builders(t, withFiles(nil)) {
t.Run(name, func(t *testing.T) {
out := t.TempDir()
put(t, out, motorinoMarker, "x")
put(t, out, "news/page/2/index.html", "old")
put(t, out, "old/deep/file.txt", "old")
stale, err := run(out, true)
if err != nil {
t.Fatalf("build error = %v", err)
}
if len(stale) != 0 {
t.Errorf("stale = %v after a clean build", stale)
}
for _, gone := range []string{"news/page/2/index.html", "old/deep/file.txt"} {
if exists(t, out, gone) {
t.Errorf("%s is still there after -clean", gone)
}
}
if !exists(t, out, motorinoMarker) {
t.Error("the marker is gone after -clean")
}
})
}
}
// TestCleanTakesAnEmptyOrNewFolder checks that -clean needs no marker for a folder with nothing in it, and marks it.
func TestCleanTakesAnEmptyOrNewFolder(t *testing.T) {
for name, run := range builders(t, withFiles(nil)) {
t.Run(name, func(t *testing.T) {
for _, out := range []string{t.TempDir(), filepath.Join(t.TempDir(), "new", "deeper")} {
if _, err := run(out, true); err != nil {
t.Fatalf("build into %s error = %v", out, err)
}
if !exists(t, out, motorinoMarker) {
t.Errorf("no marker in %s", out)
}
}
})
}
}
// TestCleanRefusesAFolderThatIsNotMotorinos checks that -clean leaves a folder with files and no marker alone.
func TestCleanRefusesAFolderThatIsNotMotorinos(t *testing.T) {
for name, run := range builders(t, withFiles(nil)) {
t.Run(name, func(t *testing.T) {
out := t.TempDir()
put(t, out, "precious.txt", "keep me")
_, err := run(out, true)
if err == nil || !strings.Contains(err.Error(), motorinoMarker) {
t.Fatalf("error = %v, want one that names %s", err, motorinoMarker)
}
if got := readOut(t, out, "precious.txt"); got != "keep me" {
t.Errorf("precious.txt = %q", got)
}
if exists(t, out, "index.html") || exists(t, out, "index.gmi") || exists(t, out, "en/index.gmi") {
t.Error("the build wrote into a folder it refused to empty")
}
})
}
}
// TestCleanRefusesAFolderThatHoldsTheSite checks that -clean never empties the folder of the site or one above it,
// even when it has the marker.
func TestCleanRefusesAFolderThatHoldsTheSite(t *testing.T) {
parent := t.TempDir()
site := filepath.Join(parent, "site")
for rel, text := range withFiles(nil) {
put(t, site, rel, text)
}
put(t, parent, motorinoMarker, "x")
for _, out := range []string{site, parent} {
if _, err := Build(Options{SiteDir: site, OutDir: out, Now: fixedNow, Clean: true}); err == nil || !strings.Contains(err.Error(), "site") {
t.Errorf("Build into %s: error = %v, want one about the site", out, err)
}
}
if !exists(t, site, "site.json") {
t.Error("the site is gone")
}
}