From 2f169b8d9f79a49dcefd3ddc48e5db1404dd7e17 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Mon, 16 Nov 2026 12:28:10 +0100 Subject: [PATCH] feat(gemini): write the site as a Gemini capsule `motorino gemini [-site DIR] [-out DIR] [-lang CODE] [-drafts]` turns the pages and the news into gemtext with the new gemtext package. - the language served by default sits at the root (-lang, then gemini_language in site.json, then the default language) and the others under // - the news exist once, under /news/: index, posts, categories, archives and a gemlog style feed (news/feed.gmi, a date before each title) - every page ends with the navigation of its language, the language links and, on translations, the machine translation note - .meta for Agate (--central-conf): two lines per language other than the root one and two for the news when they are in another language, never linked - addresses are mapped to the capsule, a fragment is dropped, a missing target stops the build, assets are copied only when a page points to them, gemini_web_paths keeps paths served by other means as links to the website - blocks of generated lists become gemtext with the strings of the language file - the block replacement of a page is shared with the HTML build Refs #10 --- README.md | 14 +- cmd/motorino/main.go | 33 ++- cmd/motorino/main_test.go | 46 +++ internal/build/context.go | 43 ++- internal/build/gemini.go | 479 +++++++++++++++++++++++++++++++ internal/build/gemini_news.go | 248 ++++++++++++++++ internal/build/gemini_test.go | 399 +++++++++++++++++++++++++ internal/build/pages.go | 18 +- internal/gemtext/gemtext.go | 9 + internal/gemtext/gemtext_test.go | 16 ++ 10 files changed, 1283 insertions(+), 22 deletions(-) create mode 100644 internal/build/gemini.go create mode 100644 internal/build/gemini_news.go create mode 100644 internal/build/gemini_test.go diff --git a/README.md b/README.md index 96cf016..41f30fe 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # motorino -The static site generator of [kmalloc.org](https://kmalloc.org/). It turns Markdown pages, news, templates and language files into plain HTML with no JavaScript. It is made for that one site and is not a general-purpose generator. +The static site generator of [kmalloc.org](https://kmalloc.org/). It turns Markdown pages, news, templates and language files into plain HTML with no JavaScript, and into a Gemini capsule. It is made for that one site and is not a general-purpose generator. Licence: GPL-3.0-or-later, see `LICENSE`. @@ -9,6 +9,8 @@ Licence: GPL-3.0-or-later, see `LICENSE`. ``` motorino build -site DIR -out DIR build the site in DIR into the output folder (defaults: . and public) motorino build -drafts ... the same, with the drafts and the posts dated in the future, for a preview +motorino gemini -site DIR -out DIR write the same site as gemtext, for a Gemini server (defaults: . and capsule) +motorino gemini -lang CODE ... the language at the root of the capsule; -drafts works as for build motorino new -site DIR "A title" write news/a-title.md, a draft with the header ready, and print its path motorino version print the version ``` @@ -77,6 +79,16 @@ Pages: `Title`, `ID` (the same in every language, it links the translations), `D `{{name}}`, with lowercase letters, digits and underscores, is replaced by a string from `site.json` or the language file. An unknown placeholder stops the build and names the file. Fenced code blocks (found the way CommonMark finds them, in lists and quotes too) are left as written, so an article can show a placeholder there. Anywhere else a backslash before the placeholder (`\{{name}}`) keeps it as written, in a code span too; two backslashes are one literal backslash followed by the value. Code spans are otherwise substituted, which is how a page shows a value in monospace. The id of a heading is not checked against ids that a template writes itself, such as the `news` of the latest news block. `{{news_list}}`, `{{latest_news}}` and `{{categories_list}}` stand alone in a paragraph and expand to generated lists. +## Gemini + +`motorino gemini` reads the same site and writes gemtext: the pages of each language, the news once and a `.meta` file for the server. It does not read the templates, the style or the layouts: a page is its title, its text and a few navigation links. + +- **Language at the root.** The language served by default sits at the root and the others under `//`, the default language of the site included. It is set with `-lang`, else `gemini_language` in `site.json`, else the default language of the site. The server must be started with the same default (Agate: `--lang`). It must be a language that has a home page. +- **`.meta`.** The server reads the language of each document from `/.meta`, which it never serves and no page links to. For every language other than the root one there are two lines, `en/*:;lang=en` and `en/**/*:;lang=en`, because a `*` does not cross folders, and the same two lines for `news/` when the news are not in the root language. It is for Agate run with `--central-conf`. A post that names another `Language` than the news is not given its own line. +- **News.** They exist once, under `/news/`, as in the website: the index, the posts, the categories, the archives and `news/feed.gmi`, a list of `=> /news/2026/11/post.gmi 2026-11-14 Title` lines, newest first, which feed readers subscribe to. Every page ends with the navigation of its language, which has the news. `{{news_blog}}` lists all the posts, without pages of ten. +- **Conversion.** A heading is a heading (three levels), a paragraph is a line, the links of a paragraph, list, quote or table follow it on lines of their own, a list is bullets (a list of only links is link lines), a quote or a note is a quote, code is a preformatted block, a table is a preformatted block of aligned columns, an image with an alt text is a link line, and emphasis is dropped. A page has one first level heading: its title; a first heading that says the title is dropped and the others go one level down. The blocks (`{{news_list}}`, `{{latest_news}}` and the others) become generated lines, with the texts of the language file. +- **Addresses.** An address of the site becomes the address of its page in the capsule (`/people.html` is `/en/people.gmi`), and a fragment is dropped. A file listed in `assets` is copied when a page points to it, and only then. `http`, `https`, `mailto`, `gemini`, `xmpp`, `irc`, `ircs`, `matrix`, `bitcoin`, `magnet`, `tel` and `sms` links stay as they are. A path the website serves by other means (`/~squid/`) becomes a link to the website when it starts with one of the comma separated prefixes of `gemini_web_paths` in `site.json`. Any other address that leads nowhere stops the build and names the page. + ## Development ``` diff --git a/cmd/motorino/main.go b/cmd/motorino/main.go index 88ca4e9..dae0be3 100644 --- a/cmd/motorino/main.go +++ b/cmd/motorino/main.go @@ -6,6 +6,7 @@ // Usage: // // motorino build [-site DIR] [-out DIR] [-drafts] +// motorino gemini [-site DIR] [-out DIR] [-lang CODE] [-drafts] // motorino new [-site DIR] "title" // motorino version package main @@ -26,7 +27,7 @@ import ( ) // usage is the text printed for a missing or unknown command. -const usage = "usage: motorino build [-site DIR] [-out DIR] [-drafts] | motorino new [-site DIR] \"title\" | motorino version" +const usage = "usage: motorino build [-site DIR] [-out DIR] [-drafts] | motorino gemini [-site DIR] [-out DIR] [-lang CODE] [-drafts] | motorino new [-site DIR] \"title\" | motorino version" // main runs the command line and exits with its status code. func main() { @@ -46,6 +47,8 @@ func run(args []string, stdout, stderr io.Writer) int { return 0 case "build": return runBuild(args[1:], stdout, stderr) + case "gemini": + return runGemini(args[1:], stdout, stderr) case "new": return runNew(args[1:], stdout, stderr) default: @@ -80,6 +83,34 @@ func runBuild(args []string, stdout, stderr io.Writer) int { return 0 } +// runGemini parses the flags of the gemini command and writes the site as a +// Gemini capsule. +func runGemini(args []string, stdout, stderr io.Writer) int { + flags := flag.NewFlagSet("gemini", flag.ContinueOnError) + flags.SetOutput(stderr) + site := flags.String("site", ".", "the folder of the site") + out := flags.String("out", "capsule", "the folder the gemtext is written to") + lang := flags.String("lang", "", "the language at the root of the capsule (default: gemini_language of site.json, then the default language)") + drafts := flags.Bool("drafts", false, "publish the drafts and the posts dated in the future, for a preview") + if err := flags.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + if flags.NArg() > 0 { + fmt.Fprintf(stderr, "motorino: unexpected arguments: %s\n%s\n", strings.Join(flags.Args(), " "), usage) + return 2 + } + result, err := build.BuildGemini(build.GeminiOptions{SiteDir: *site, OutDir: *out, Version: version.String(), Drafts: *drafts, Language: *lang}) + if err != nil { + fmt.Fprintf(stderr, "motorino: %v\n", err) + return 1 + } + fmt.Fprintf(stdout, "wrote %d gemtext files into %s\n", result.Pages, *out) + return 0 +} + // runNew parses the flags of the new command and writes the file of a new post. // The author is the author key of site.json, when it has one. func runNew(args []string, stdout, stderr io.Writer) int { diff --git a/cmd/motorino/main_test.go b/cmd/motorino/main_test.go index 8798844..5743d54 100644 --- a/cmd/motorino/main_test.go +++ b/cmd/motorino/main_test.go @@ -187,3 +187,49 @@ func TestRunBuildDraftsFlag(t *testing.T) { t.Errorf("a draft is missing from the build with -drafts: %v", err) } } + +// TestRunGemini checks the gemini command: the capsule, the language flag and the drafts flag. +func TestRunGemini(t *testing.T) { + dir := miniSite(t) + if err := os.MkdirAll(filepath.Join(dir, "news"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "news", "wip.md"), []byte("Title: Work in progress\nDate: 2026-01-01T12:00:00+0100\nDraft: true\n-\nSoon.\n"), 0o644); err != nil { + t.Fatal(err) + } + var out, errOut bytes.Buffer + capsule := filepath.Join(t.TempDir(), "capsule") + if code := run([]string{"gemini", "-site", dir, "-out", capsule, "-lang", "en"}, &out, &errOut); code != 0 { + t.Fatalf("run(gemini) = %d, stderr %q", code, errOut.String()) + } + if want := "wrote 8 gemtext files into " + capsule; !strings.Contains(out.String(), want) { + t.Errorf("output %q does not contain %q", out.String(), want) + } + for _, rel := range []string{"index.gmi", "news/index.gmi", "news/feed.gmi", ".meta"} { + if _, err := os.Stat(filepath.Join(capsule, filepath.FromSlash(rel))); err != nil { + t.Errorf("no %s in the capsule: %v", rel, err) + } + } + if _, err := os.Stat(filepath.Join(capsule, "news", "2026", "01", "wip.gmi")); err == nil { + t.Error("a draft is published without -drafts") + } + preview := filepath.Join(t.TempDir(), "preview") + if code := run([]string{"gemini", "-drafts", "-site", dir, "-out", preview}, &out, &errOut); code != 0 { + t.Fatalf("run(gemini -drafts) = %d, stderr %q", code, errOut.String()) + } + if _, err := os.Stat(filepath.Join(preview, "news", "2026", "01", "wip.gmi")); err != nil { + t.Errorf("a draft is missing from the capsule with -drafts: %v", err) + } +} + +// TestRunGeminiErrors checks the exit codes: 1 for a failed build, 2 for a mistake in the command line. +func TestRunGeminiErrors(t *testing.T) { + var out, errOut bytes.Buffer + if code := run([]string{"gemini", "-site", miniSite(t), "-out", filepath.Join(t.TempDir(), "c"), "-lang", "xx"}, &out, &errOut); code != 1 || !strings.Contains(errOut.String(), `"xx"`) { + t.Errorf("an unknown language: code %d, stderr %q, want 1 and a message that names it", code, errOut.String()) + } + errOut.Reset() + if code := run([]string{"gemini", "extra"}, &out, &errOut); code != 2 || !strings.Contains(errOut.String(), "unexpected arguments") { + t.Errorf("an extra argument: code %d, stderr %q, want 2", code, errOut.String()) + } +} diff --git a/internal/build/context.go b/internal/build/context.go index 3708e26..d21f894 100644 --- a/internal/build/context.go +++ b/internal/build/context.go @@ -13,6 +13,10 @@ import ( // are expanded they stand for themselves, so a page can hold them. var blockNames = []string{"news_list", "latest_news", "categories_list", "news_blog", "news_archive"} +// sitePrefix goes before the title of a page that has no title of its own in the +// browser, such as a post. +const sitePrefix = "kmalloc.org: " + // document describes one output page for the shared page frame. type document struct { // lang is the language code of the page. @@ -101,7 +105,7 @@ func (b *builder) documentContext(d document) map[string]string { ctx["doc_q"] = strings.NewReplacer(":", "%3A", "/", "%2F").Replace(docURL) title := d.title if d.prefixed { - title = "kmalloc.org: " + title + title = sitePrefix + title } ctx["title_html"] = render.EscapeAttr(title) ctx["page_title"] = render.Escape(d.title) @@ -118,34 +122,45 @@ func (b *builder) documentContext(d document) map[string]string { return ctx } -// languageLinks returns the links to the other languages for the menu and the -// alternate links for the head. A page without translations links to the home -// of the first other language that has one. -func (b *builder) languageLinks(d document) (links, alternates string) { - type target struct{ lang, path string } - var twins []target +// langTarget is a page in another language that a page links to. +type langTarget struct{ lang, path string } + +// languageTargets returns the pages in the other languages that a page links +// to: its translations, with the address a Switch key gives instead of the +// translation, or, when it has none, the home of the first other language that +// has one. +func (b *builder) languageTargets(d document) []langTarget { + var twins []langTarget if d.id != "" { for _, l := range b.cfg.Languages { if p, ok := b.byKey[pageKey{d.id, l}]; ok && l != d.lang { - twins = append(twins, target{l, p.Path}) + twins = append(twins, langTarget{l, p.Path}) } } } if len(twins) == 0 { for _, l := range b.cfg.Languages { if p, ok := b.byKey[pageKey{"home", l}]; ok && l != d.lang { - twins = append(twins, target{l, p.Path}) + twins = append(twins, langTarget{l, p.Path}) break } } } + for i, t := range twins { + if override, ok := d.switches[t.lang]; ok { + twins[i].path = override + } + } + return twins +} + +// languageLinks returns the links to the other languages for the menu and the +// alternate links for the head. +func (b *builder) languageLinks(d document) (links, alternates string) { + twins := b.languageTargets(d) anchors := make([]string, len(twins)) for i, t := range twins { - href := t.path - if override, ok := d.switches[t.lang]; ok { - href = override - } - anchors[i] = fmt.Sprintf(`%s`, render.EscapeAttr(href), t.lang, t.lang, render.Escape(b.langs[t.lang].Strings["lang_name"])) + anchors[i] = fmt.Sprintf(`%s`, render.EscapeAttr(t.path), t.lang, t.lang, render.Escape(b.langs[t.lang].Strings["lang_name"])) } links = strings.Join(anchors, "\n·\n") diff --git a/internal/build/gemini.go b/internal/build/gemini.go new file mode 100644 index 0000000..5f3c85d --- /dev/null +++ b/internal/build/gemini.go @@ -0,0 +1,479 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package build + +import ( + "errors" + "fmt" + "os" + "path" + "path/filepath" + "regexp" + "sort" + "strings" + "time" + + "git.kmalloc.org/squid/motorino/internal/content" + "git.kmalloc.org/squid/motorino/internal/gemtext" + "git.kmalloc.org/squid/motorino/internal/mdx" + "git.kmalloc.org/squid/motorino/internal/render" +) + +// GeminiOptions tell BuildGemini where the site is and where to write the capsule. +type GeminiOptions struct { + // SiteDir is the folder of the site, the same one Build reads. + SiteDir string + // OutDir is the folder the gemtext is written to; it is created when missing. + OutDir string + // Version is the version of motorino. + Version string + // Now is the moment the capsule is built at, as in Options. + Now time.Time + // Drafts publishes the drafts and the posts dated in the future too. + Drafts bool + // Language is the language served at the root of the capsule, the others + // living under //. It is the language the server serves by default + // (Agate's --lang). Empty means gemini_language of site.json, then the + // default language of the site. + Language string +} + +// GeminiResult summarises a Gemini build. +type GeminiResult struct { + // Pages is the number of gemtext files written. + Pages int +} + +// gemNews is the address of the news in the capsule. They exist once, whatever +// the language of the root, so it is always /news/. +const gemNews = "/news/" + +// metaName is the file that tells the server the language of each folder; the +// server reads it and never serves it. +const metaName = ".meta" + +// gemini holds one Gemini build, on top of the loaded site. +type gemini struct { + *builder + // root is the language served at the root. + root string + // pageAddr is the address of every page in the capsule, by ID and language. + pageAddr map[pageKey]string + // addr maps an address of the website to the address in the capsule. + addr map[string]string + // usedAssets lists the assets some page points to: only those are copied. + usedAssets map[string]bool + // gemFiles holds the output, and gemOwners what produces each file. + gemFiles map[string]string + gemOwners map[string]string +} + +// BuildGemini reads the site in opts.SiteDir and writes it as a Gemini capsule +// to opts.OutDir: the pages of each language, the news once under /news/ and a +// .meta file for the server. Everything is rendered in memory first, so a +// failure in the content leaves the folder untouched. +func BuildGemini(opts GeminiOptions) (GeminiResult, error) { + b, err := load(Options{SiteDir: opts.SiteDir, OutDir: opts.OutDir, Version: opts.Version, Now: opts.Now, Drafts: opts.Drafts}) + if err != nil { + return GeminiResult{}, err + } + g := &gemini{ + builder: b, pageAddr: map[pageKey]string{}, addr: map[string]string{}, usedAssets: map[string]bool{}, + gemFiles: map[string]string{}, gemOwners: map[string]string{}, + } + if g.root, err = g.rootLanguage(opts.Language); err != nil { + return GeminiResult{}, err + } + g.mapAddresses() + for _, p := range b.pages { + if err := g.renderPage(p); err != nil { + return GeminiResult{}, err + } + } + steps := []func() error{g.renderPosts, g.renderCategories, g.renderArchives, g.renderFeed} + for _, step := range steps { + if err := step(); err != nil { + return GeminiResult{}, err + } + } + pages := len(g.gemFiles) + if err := g.publish(metaName, "the language file of the server", g.meta()); err != nil { + return GeminiResult{}, err + } + assets, err := g.assets() + if err != nil { + return GeminiResult{}, err + } + for rel, text := range g.gemFiles { + if err := writeFile(opts.OutDir, rel, []byte(text)); err != nil { + return GeminiResult{}, err + } + } + for rel, data := range assets { + if err := writeFile(opts.OutDir, rel, data); err != nil { + return GeminiResult{}, err + } + } + return GeminiResult{Pages: pages}, nil +} + +// rootLanguage returns the language at the root of the capsule: the option, +// else gemini_language of site.json, else the default language. It must be a +// language of the site that has a home page. +func (g *gemini) rootLanguage(option string) (string, error) { + code, from := option, "the language option" + if code == "" { + code, from = g.cfg.Strings["gemini_language"], "gemini_language in site.json" + } + if code == "" { + return g.defaultLang, nil + } + if _, ok := g.langs[code]; !ok { + return "", fmt.Errorf("%s: %q is not one of the languages of the site", from, code) + } + if _, ok := g.byKey[pageKey{"home", code}]; !ok { + return "", fmt.Errorf("%s: the language %q has no home page, so it cannot be the root of the capsule", from, code) + } + return code, nil +} + +// meta returns the content of the .meta file: for every language other than +// the one at the root, and for the news when they are in another language, the +// lines that give the language of the documents. A glob does not cross folders, +// so each folder needs a line for itself and one for what is below it. +func (g *gemini) meta() string { + var out strings.Builder + line := func(folder, lang string) { + fmt.Fprintf(&out, "%s/*:;lang=%s\n%s/**/*:;lang=%s\n", folder, lang, folder, lang) + } + for _, lang := range g.cfg.Languages { + if lang != g.root { + line(lang, lang) + } + } + if g.newsLang != g.root { + line(strings.Trim(gemNews, "/"), g.newsLang) + } + return out.String() +} + +// pageGemAddress returns the address in the capsule of a page of lang whose +// address in the website is html: the root language at the root, the others +// under their code, .gmi for .html. +func (g *gemini) pageGemAddress(lang, html string) string { + p := html + if lang != g.defaultLang { + p = strings.TrimPrefix(p, "/"+lang) + } + if strings.HasSuffix(p, ".html") { + p = strings.TrimSuffix(p, ".html") + ".gmi" + } + if lang != g.root { + p = "/" + lang + p + } + return p +} + +// gemPostAddress, gemCategoryAddress, gemYearAddress and gemMonthAddress are the +// addresses in the capsule of a post, a category and the archives. +func gemPostAddress(p content.Post) string { return strings.TrimSuffix(p.Path, ".html") + ".gmi" } +func gemCategoryAddress(slug string) string { return gemNews + "category/" + slug + ".gmi" } +func gemYearAddress(year int) string { return fmt.Sprintf("%s%04d/", gemNews, year) } +func gemMonthAddress(year, month int) string { + return fmt.Sprintf("%s%04d/%02d/", gemNews, year, month) +} + +// mapAddresses fills the maps from the addresses of the website to those of the capsule. +func (g *gemini) mapAddresses() { + for _, p := range g.pages { + address := g.pageGemAddress(p.Lang, p.Path) + if p.ID == "news" && p.Lang == g.newsLang { + address = gemNews + } + g.pageAddr[pageKey{p.ID, p.Lang}] = address + g.addr[p.Path] = address + } + for _, p := range g.posts { + g.addr[p.Path] = gemPostAddress(p) + } + for _, c := range g.categories { + g.addr[categoryPath(c.Slug)] = gemCategoryAddress(c.Slug) + } + for _, y := range groupArchive(g.posts) { + g.addr[g.yearPath(y.year)] = gemYearAddress(y.year) + for _, m := range y.months { + g.addr[g.monthPath(m.year, m.month)] = gemMonthAddress(m.year, m.month) + } + } + g.addr[g.newsPath+"feed.xml"] = gemNews + "feed.gmi" +} + +// publish stores a file of the capsule; source names what produced it. +func (g *gemini) publish(rel, source, text string) error { + if other, taken := g.gemOwners[rel]; taken { + return fmt.Errorf("%s and %s are both published as %s", other, source, rel) + } + g.gemOwners[rel] = source + g.gemFiles[rel] = text + return nil +} + +// gemFile returns the file of an address of the capsule: a folder gets index.gmi. +func gemFile(address string) string { + rel := strings.TrimPrefix(address, "/") + if rel == "" || strings.HasSuffix(rel, "/") { + rel += "index.gmi" + } + return rel +} + +// scheme matches the start of an address that names a protocol. +var scheme = regexp.MustCompile(`^([A-Za-z][A-Za-z0-9+.-]*):`) + +// externalSchemes are the protocols a link may use to leave the capsule. +var externalSchemes = map[string]bool{ + "http": true, "https": true, "mailto": true, "gemini": true, "xmpp": true, "irc": true, "ircs": true, "matrix": true, + "bitcoin": true, "magnet": true, "tel": true, "sms": true, +} + +// resolve turns an address found in the HTML of the page at base into the +// address to write in the capsule. An address with a protocol stays as it is +// when the protocol is a known one, and is an error otherwise; an address in the +// website is mapped to its page or file, and a file is copied by the end of the +// build. An address that starts with one of the paths of gemini_web_paths in +// site.json is one the website serves by other means: it becomes a link to the +// website. A fragment alone is dropped, and so is the fragment of an address: a +// gemini link cannot point inside a document. +func (g *gemini) resolve(href, base string) (string, error) { + if strings.HasPrefix(href, "#") { + return "", nil + } + if m := scheme.FindStringSubmatch(href); m != nil { + if !externalSchemes[strings.ToLower(m[1])] { + return "", fmt.Errorf("the protocol %q is not supported", m[1]) + } + return href, nil + } + if strings.HasPrefix(href, "//") { + return "", errors.New("an address that starts with // is not supported") + } + target, _, _ := strings.Cut(href, "#") + if !strings.HasPrefix(target, "/") { + folder := path.Dir(base) + if strings.HasSuffix(base, "/") { + folder = strings.TrimSuffix(base, "/") + } + joined := path.Join("/", folder, target) + if strings.HasSuffix(target, "/") && joined != "/" { + joined += "/" + } + target = joined + } + if address, ok := g.addr[target]; ok { + return address, nil + } + name := strings.TrimPrefix(target, "/") + for asset := range g.cfg.Assets { + if path.Clean(asset) == name { + g.usedAssets[asset] = true + return target, nil + } + } + for _, prefix := range strings.Split(g.cfg.Strings["gemini_web_paths"], ",") { + if prefix = strings.TrimSpace(prefix); prefix != "" && strings.HasPrefix(target, prefix) { + return g.cfg.Strings["base_url"] + target, nil + } + } + return "", errors.New("no page or file of the site has this address (a path the website serves by other means goes in gemini_web_paths of site.json)") +} + +// convert converts the HTML of a page or post to gemtext. base is the address +// of the page in the website, lang its language. +func (g *gemini) convert(name, html, base, lang string) (string, error) { + text, err := gemtext.Convert(html, gemtext.Options{ + Link: func(href string) (string, error) { return g.resolve(href, base) }, + Block: func(block string) (string, error) { return g.block(block, lang) }, + }) + if err != nil { + return "", fmt.Errorf("%s: %w", name, err) + } + return strings.TrimRight(text, "\n"), nil +} + +// markdown replaces the placeholders of a source, converts it to HTML and puts +// the element where a generated list goes, for convert to expand. +func (g *gemini) markdown(name, source string, ctx map[string]string) (string, error) { + text, err := mdx.SubstituteMarkdown(name, source, ctx) + if err != nil { + return "", err + } + html, _, err := mdx.Render(text) + if err != nil { + return "", fmt.Errorf("%s: %w", name, err) + } + html, err = replaceBlocks(name, html, func(block string) (string, error) { + return ``, nil + }) + if err != nil { + return "", err + } + return render.Unescape(html), nil +} + +// assets reads the files some page points to. +func (g *gemini) assets() (map[string][]byte, error) { + names := make([]string, 0, len(g.usedAssets)) + for name := range g.usedAssets { + names = append(names, name) + } + sort.Strings(names) + out := map[string][]byte{} + for _, name := range names { + source := g.cfg.Assets[name] + data, err := os.ReadFile(filepath.Join(g.opts.SiteDir, filepath.FromSlash(source))) + if err != nil { + return nil, fmt.Errorf("asset %q: %w", name, err) + } + rel := path.Clean(name) + if other, taken := g.gemOwners[rel]; taken { + return nil, fmt.Errorf("the asset %q and %s are both published as %s", name, other, rel) + } + out[rel] = data + } + return out, nil +} + +// heading is the first level heading of a page. +func heading(title string) string { return "# " + strings.Join(strings.Fields(title), " ") } + +// withoutTitle prepares a body converted to gemtext to go under the heading of +// the title. A leading heading that says the title is dropped, since the page +// has it already, and the other first level headings go one level down: a page +// has one. +func withoutTitle(title, body string) string { + lines := strings.Split(body, "\n") + if first, ok := strings.CutPrefix(lines[0], "# "); ok && (first == title || strings.HasSuffix(title, ": "+first)) { + lines = lines[1:] + } + fenced := false + for i, line := range lines { + switch { + case strings.HasPrefix(line, "```"): + fenced = !fenced + case !fenced && strings.HasPrefix(line, "# "): + lines[i] = "#" + line + } + } + return strings.Join(lines, "\n") +} + +// navigation returns the links every page ends with: the home and the entries +// of the menu of the language, in the order its file lists them, then the +// other languages. An entry whose page the language lacks is left out. +func (g *gemini) navigation(d document) string { + l := g.langs[d.lang] + var menu []string + if home, ok := g.pageAddr[pageKey{"home", d.lang}]; ok { + label := l.Strings["back"] + if label == "" { + label = g.byKey[pageKey{"home", d.lang}].Title + } + menu = append(menu, gemtext.Link(home, label)) + } + for _, key := range l.NavOrder { + address, ok := g.pageAddr[pageKey{key, d.lang}] + if !ok && key == "news" { + address, ok = gemNews, true + } + if ok { + menu = append(menu, gemtext.Link(address, l.Nav[key])) + } + } + parts := []string{strings.Join(menu, "\n")} + var others []string + for _, t := range g.languageTargets(d) { + if address, err := g.resolve(t.path, d.path); err == nil && address != "" { + others = append(others, gemtext.Link(address, g.langs[t.lang].Strings["lang_name"])) + } + } + if len(others) > 0 { + parts = append(parts, strings.Join(others, "\n")) + } + return strings.Join(parts, "\n\n") +} + +// translationNote returns the line, and the link to report a problem, that a +// machine translated language puts at the top of its pages; empty for the others. +func (g *gemini) translationNote(lang string) string { + l := g.langs[lang] + note := l.Strings["translation_note"] + if note == "" { + return "" + } + label, issues := l.Strings["translation_note_a"], g.cfg.Strings["issues_url"] + if label == "" || issues == "" { + return gemtext.Text(note) + } + return gemtext.Text(note+" "+label+".") + "\n" + gemtext.Link(issues, label) +} + +// compose joins the parts of a page with a blank line between them, leaving out +// the empty ones, and ends the text with a line break. +func compose(parts ...string) string { + var kept []string + for _, part := range parts { + if part = strings.Trim(part, "\n"); part != "" { + kept = append(kept, part) + } + } + return strings.Join(kept, "\n\n") + "\n" +} + +// renderPage converts a page and writes it with the navigation. +func (g *gemini) renderPage(p content.Page) error { + d := document{ + lang: p.Lang, id: p.ID, nav: p.ID, path: p.Path, title: p.Title, + description: p.Description, switches: p.Switch, source: p.Source, + } + ctx := g.documentContext(d) + version, date := g.cfg.Strings["manifesto_version"], g.cfg.Strings["manifesto_date"] + if p.ManifestoVersion != "" { + version = p.ManifestoVersion + } + if p.ManifestoDate != "" { + date = p.ManifestoDate + } + ctx["manifesto_version"], ctx["manifesto_date"] = render.Escape(version), render.Escape(date) + + source := p.Body + if p.Layout == "manifesto" { + var err error + if source, err = mdx.NoEmDash(source); err != nil { + return fmt.Errorf("%s: %w", p.Source, err) + } + } + html, err := g.markdown(p.Source, source, ctx) + if err != nil { + return err + } + body, err := g.convert(p.Source, html, p.Path, p.Lang) + if err != nil { + return err + } + var intro []string + if p.Layout == "manifesto" { + intro = []string{g.manifestoLine(p.Lang, version, date), gemtext.Text(p.Note)} + } + parts := append([]string{heading(p.Title), g.translationNote(p.Lang)}, intro...) + parts = append(parts, withoutTitle(p.Title, body), g.navigation(d)) + return g.publish(gemFile(g.pageAddr[pageKey{p.ID, p.Lang}]), p.Source, compose(parts...)) +} + +// manifestoLine is the line that says the version and the date of the manifesto. +func (g *gemini) manifestoLine(lang, version, date string) string { + line := strings.TrimSpace(g.langs[lang].Strings["version"] + " " + version) + if date != "" { + line += ", " + date + } + return gemtext.Text(line) +} diff --git a/internal/build/gemini_news.go b/internal/build/gemini_news.go new file mode 100644 index 0000000..7b9939d --- /dev/null +++ b/internal/build/gemini_news.go @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package build + +import ( + "fmt" + "strings" + + "git.kmalloc.org/squid/motorino/internal/content" + "git.kmalloc.org/squid/motorino/internal/gemtext" +) + +// newsLines returns the lines of a list of posts: a link line for each, with +// its date in front of the title, the format feed readers know, and the +// description under it when asked for. +func newsLines(posts []content.Post, withDescription bool) string { + var lines []string + for _, p := range posts { + lines = append(lines, gemtext.Link(gemPostAddress(p), p.Date.Format(dateOnly)+" "+p.Title)) + if withDescription && p.Description != "" { + lines = append(lines, gemtext.Text(p.Description)) + } + } + return strings.Join(lines, "\n") +} + +// block returns the gemtext of a generated list. The blog list is the whole list +// of the posts: a capsule has no pages of ten. +func (g *gemini) block(name, lang string) (string, error) { + l := g.langs[lang] + empty := gemtext.Text(l.Strings["news_empty"]) + switch name { + case "news_list", "news_blog": + if len(g.posts) == 0 { + return empty, nil + } + return newsLines(g.posts, true), nil + case "latest_news": + return g.latestNews(lang), nil + case "categories_list": + lines := make([]string, len(g.categories)) + for i, c := range g.categories { + lines[i] = gemtext.Link(gemCategoryAddress(c.Slug), fmt.Sprintf("%s (%d)", c.Name, len(c.posts))) + } + return strings.Join(lines, "\n"), nil + default: + return g.archiveLines(lang, empty), nil + } +} + +// latestNews returns the block of the latest news of the home page: a heading +// with the name of the news in the menu, the sentence news_p, the first posts +// and the link to all of them. It is empty without posts. +func (g *gemini) latestNews(lang string) string { + if len(g.posts) == 0 { + return "" + } + l := g.langs[lang] + var lines []string + if name := l.Nav["news"]; name != "" { + lines = append(lines, "#"+heading(name)+"\n") + } + if text := gemtext.Text(l.Strings["news_p"]); text != "" { + lines = append(lines, text) + } + n := newsOnHome + if n > len(g.posts) { + n = len(g.posts) + } + lines = append(lines, newsLines(g.posts[:n], false)) + label := l.Strings["news_all"] + if label == "" { + label = l.Nav["news"] + } + return strings.Join(append(lines, gemtext.Link(gemNews, label)), "\n") +} + +// archiveLines returns the years that have posts, each followed by its months. +func (g *gemini) archiveLines(lang, empty string) string { + years := groupArchive(g.posts) + if len(years) == 0 { + return empty + } + var lines []string + for _, y := range years { + lines = append(lines, gemtext.Link(gemYearAddress(y.year), fmt.Sprintf("%d (%d)", y.year, len(y.posts)))) + lines = append(lines, g.monthLines(y, lang)...) + } + return strings.Join(lines, "\n") +} + +// monthLines returns the link lines of the months of a year. +func (g *gemini) monthLines(y archiveYear, lang string) []string { + lines := make([]string, len(y.months)) + for i, m := range y.months { + lines[i] = gemtext.Link(gemMonthAddress(m.year, m.month), fmt.Sprintf("%s (%d)", monthYear(m.year, m.month, g.langs[lang]), len(m.posts))) + } + return lines +} + +// newsDocument describes a page of the news for the navigation. +func (g *gemini) newsDocument(lang, path, title, source string) document { + return document{lang: lang, nav: "news", path: path, title: title, source: source} +} + +// renderPosts converts every post. +func (g *gemini) renderPosts() error { + for i, p := range g.posts { + if err := g.renderPost(i, p); err != nil { + return err + } + } + return nil +} + +// renderPost converts a post: the title, a line with the date, the author, the +// categories and the reading time, the summary when it was written, the text, +// the link to reply on Mastodon, and the links to the older and the newer post +// and to all the news. +func (g *gemini) renderPost(i int, p content.Post) error { + l := g.langs[p.Language] + d := g.newsDocument(p.Language, p.Path, p.Title, p.Source) + ctx := g.documentContext(d) + html, err := g.markdown(p.Source, p.Body, ctx) + if err != nil { + return err + } + body, err := g.convert(p.Source, html, p.Path, p.Language) + if err != nil { + return err + } + + parts := []string{longDate(p.Date, l)} + if p.Author != "" { + parts = append(parts, p.Author) + } + if len(p.Categories) > 0 { + names := make([]string, len(p.Categories)) + for i, c := range p.Categories { + names[i] = c.Name + } + parts = append(parts, strings.Join(names, ", ")) + } + if reading := readingText(p.ReadingMinutes(), l); reading != "" { + parts = append(parts, reading) + } + meta := []string{gemtext.Text(strings.Join(parts, " · "))} + for _, c := range p.Categories { + meta = append(meta, gemtext.Link(gemCategoryAddress(c.Slug), c.Name)) + } + + summary := "" + if p.HasDescription { + summary = gemtext.Text(p.Description) + } + mastodon := "" + if p.Mastodon != "" { + label := l.Strings["reply_mastodon"] + if label == "" { + return fmt.Errorf("%s: the post has a Mastodon address, but lang/%s.json has no reply_mastodon to label the link", p.Source, p.Language) + } + mastodon = gemtext.Link(p.Mastodon, label) + } + out := compose(heading(p.Title), strings.Join(meta, "\n"), summary, withoutTitle(p.Title, body), mastodon, g.postNavigation(i, l.Code), g.navigation(d)) + return g.publish(gemFile(gemPostAddress(p)), p.Source, out) +} + +// postNavigation returns the links to the newer and the older post and to all +// the news. A link whose label the language lacks is left out. +func (g *gemini) postNavigation(i int, lang string) string { + l := g.langs[lang] + var lines []string + if label := l.Strings["post_newer"]; label != "" && i > 0 { + lines = append(lines, gemtext.Link(gemPostAddress(g.posts[i-1]), label+": "+g.posts[i-1].Title)) + } + if label := l.Strings["post_older"]; label != "" && i+1 < len(g.posts) { + lines = append(lines, gemtext.Link(gemPostAddress(g.posts[i+1]), label+": "+g.posts[i+1].Title)) + } + if label := l.Strings["news_all"]; label != "" { + lines = append(lines, gemtext.Link(gemNews, label)) + } + return strings.Join(lines, "\n") +} + +// renderCategories writes the page of every category. +func (g *gemini) renderCategories() error { + l := g.langs[g.newsLang] + for _, c := range g.categories { + address := gemCategoryAddress(c.Slug) + d := g.newsDocument(g.newsLang, categoryPath(c.Slug), c.Name, fmt.Sprintf("the page of the category %q", c.Name)) + title := c.Name + if h := l.Strings["category_h"]; h != "" { + title = h + ": " + c.Name + } + text := compose(heading(title), newsLines(c.posts, true), g.navigation(d)) + if err := g.publish(gemFile(address), d.source, text); err != nil { + return err + } + } + return nil +} + +// renderArchives writes the page of every year and every month that has posts. +func (g *gemini) renderArchives() error { + lang := g.newsLang + l := g.langs[lang] + page := func(address, htmlPath, name string, posts []content.Post, links []string, source string) error { + title := name + if h := l.Strings["archive_h"]; h != "" { + title = h + ": " + name + } + d := g.newsDocument(lang, htmlPath, name, source) + text := compose(heading(title), strings.Join(links, "\n"), newsLines(posts, true), g.navigation(d)) + return g.publish(gemFile(address), source, text) + } + for _, y := range groupArchive(g.posts) { + year := fmt.Sprint(y.year) + if err := page(gemYearAddress(y.year), g.yearPath(y.year), year, y.posts, g.monthLines(y, lang), "the archive of "+year); err != nil { + return err + } + var up []string + if text := l.Strings["archive_up"]; text != "" { + up = []string{gemtext.Link(gemYearAddress(y.year), text+" "+year)} + } + for _, m := range y.months { + name := monthYear(m.year, m.month, l) + if err := page(gemMonthAddress(m.year, m.month), g.monthPath(m.year, m.month), name, m.posts, up, "the archive of "+name); err != nil { + return err + } + } + } + return nil +} + +// renderFeed writes the list of the latest posts, newest first, a date before +// each title: the format that Gemini feed readers subscribe to. +func (g *gemini) renderFeed() error { + title := g.byKey[pageKey{"news", g.newsLang}].Title + if !strings.HasPrefix(title, sitePrefix) { + title = sitePrefix + title + } + posts := g.posts + if len(posts) > feedLimit { + posts = posts[:feedLimit] + } + text := compose(heading(title), newsLines(posts, false)) + return g.publish(gemFile(gemNews+"feed.gmi"), "the feed", text) +} diff --git a/internal/build/gemini_test.go b/internal/build/gemini_test.go new file mode 100644 index 0000000..2d6d3ec --- /dev/null +++ b/internal/build/gemini_test.go @@ -0,0 +1,399 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package build + +import ( + "io/fs" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "testing" +) + +// buildGem builds the Gemini output of the site made of files, with the options changed by tune. +func buildGem(t *testing.T, files map[string]string, tune func(*GeminiOptions)) (string, error) { + t.Helper() + out := filepath.Join(t.TempDir(), "capsule") + opts := GeminiOptions{SiteDir: writeFiles(t, files), OutDir: out, Version: "1.2.3", Now: fixedNow, Language: "it"} + if tune != nil { + tune(&opts) + } + _, err := BuildGemini(opts) + return out, err +} + +// gemSite is the base site with a services page that has a table and an image, a Spanish news page left out and +// the news index of the English pages. +func gemSite(extra map[string]string) map[string]string { + files := withFiles(map[string]string{ + "lang/en.json": blogLangEn, "lang/it.json": blogLangIt, + "content/en/services.md": page("services", "Services", "", "Write to **{{contact_shown}}**.\n\n\n\n\n
ServiceKept
Mail30 days
\n\n![The mark of kmalloc](/logo.svg)\n"), + "content/en/news/index.md": page("news", "News", "Layout: bare\n", "# News\n\n{{news_list}}\n\n## Categories\n\n{{categories_list}}\n\n## Archive\n\n{{news_archive}}\n\n[Feed](/news/feed.xml)\n"), + "news/shop.md": "Title: A shop\nAuthor: squid\nDate: 2026-11-14T12:00:00+0100\nCategories: FLUG\nDescription: About the shop.\nMastodon: https://social.kmalloc.org/@squid/115\n-\nThere is a shop. Mail me at squid@kmalloc.org or visit [the site](https://kmalloc.org/).\n", + }) + for k, v := range extra { + files[k] = v + } + return files +} + +// gemFiles returns every file of the output, as slash paths relative to it. +func gemFiles(t *testing.T, out string) []string { + t.Helper() + var files []string + err := filepath.WalkDir(out, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + rel, _ := filepath.Rel(out, path) + files = append(files, filepath.ToSlash(rel)) + return nil + }) + if err != nil { + t.Fatal(err) + } + sort.Strings(files) + return files +} + +// TestGeminiTree checks which files the capsule has, with Italian at the root. +func TestGeminiTree(t *testing.T) { + out, err := buildGem(t, gemSite(nil), nil) + if err != nil { + t.Fatalf("BuildGemini() error = %v", err) + } + have := map[string]bool{} + for _, f := range gemFiles(t, out) { + have[f] = true + } + for _, want := range []string{ + "index.gmi", "manifesto.gmi", "persone.gmi", "servizi.gmi", "cronologia-manifesto.gmi", "progetti.gmi", + "en/index.gmi", "en/manifesto.gmi", "en/people.gmi", "en/services.gmi", "es/manifesto.gmi", + "news/index.gmi", "news/feed.gmi", "news/2026/11/shop.gmi", "news/2026/11/status.gmi", "news/2026/09/gnu-linux-day-2026.gmi", + "news/category/flug.gmi", "news/category/linux-day.gmi", "news/2026/index.gmi", "news/2026/11/index.gmi", "news/2026/09/index.gmi", + ".meta", "logo.svg", + } { + if !have[want] { + t.Errorf("the capsule has no %s", want) + } + } + for f := range have { + if strings.HasSuffix(f, ".html") || strings.HasPrefix(f, "it/") || strings.HasPrefix(f, "en/news/") || strings.HasPrefix(f, "es/news/") || strings.HasSuffix(f, "robots.txt") { + t.Errorf("the capsule has %s, which it should not", f) + } + } +} + +// TestGeminiRootLanguage checks the language at the root: the option, site.json, then the default language. +func TestGeminiRootLanguage(t *testing.T) { + plain, err := buildGem(t, gemSite(nil), func(o *GeminiOptions) { o.Language = "" }) + if err != nil { + t.Fatal(err) + } + mustContain(t, "index.gmi", readOut(t, plain, "index.gmi"), "# kmalloc.org") + if !exists(t, plain, "it/index.gmi") || exists(t, plain, "en/index.gmi") { + t.Error("without a choice the default language is at the root and the others under their code") + } + files := gemSite(map[string]string{"site.json": strings.Replace(baseFiles()["site.json"], `"languages"`, `"gemini_language": "it", "languages"`, 1)}) + fromSite, err := buildGem(t, files, func(o *GeminiOptions) { o.Language = "" }) + if err != nil { + t.Fatal(err) + } + if !exists(t, fromSite, "en/index.gmi") || exists(t, fromSite, "it/index.gmi") { + t.Error("gemini_language in site.json does not put Italian at the root") + } + for name, tune := range map[string]func(*GeminiOptions){"xx": func(o *GeminiOptions) { o.Language = "xx" }, "es has no home": func(o *GeminiOptions) { o.Language = "es" }} { + if _, err := buildGem(t, gemSite(nil), tune); err == nil { + t.Errorf("language %s: no error", name) + } + } +} + +// TestGeminiMeta checks the .meta file that sets the language of the documents. +func TestGeminiMeta(t *testing.T) { + out, err := buildGem(t, gemSite(nil), nil) + if err != nil { + t.Fatal(err) + } + want := "en/*:;lang=en\nen/**/*:;lang=en\nes/*:;lang=es\nes/**/*:;lang=es\nnews/*:;lang=en\nnews/**/*:;lang=en\n" + if got := readOut(t, out, ".meta"); got != want { + t.Errorf(".meta =\n%q\nwant\n%q", got, want) + } + plain, err := buildGem(t, gemSite(nil), func(o *GeminiOptions) { o.Language = "en" }) + if err != nil { + t.Fatal(err) + } + want = "it/*:;lang=it\nit/**/*:;lang=it\nes/*:;lang=es\nes/**/*:;lang=es\n" + if got := readOut(t, plain, ".meta"); got != want { + t.Errorf(".meta with English at the root =\n%q\nwant\n%q (the news are in the language of the root: no lines for them)", got, want) + } + for _, f := range gemFiles(t, out) { + if strings.HasSuffix(f, ".gmi") && strings.Contains(readOut(t, out, f), ".meta") { + t.Errorf("%s mentions .meta", f) + } + } +} + +// TestGeminiPages checks the headings, the layouts and the navigation of a page. +func TestGeminiPages(t *testing.T) { + out, err := buildGem(t, gemSite(nil), nil) + if err != nil { + t.Fatal(err) + } + services := readOut(t, out, "en/services.gmi") + mustContain(t, "en/services.gmi", services, "# Services\n", "Write to squid (at) kmalloc.org.\n", + "```\nService Kept\n------- -------\nMail 30 days\n```\n", "=> /logo.svg The mark of kmalloc\n") + if strings.Contains(services, "<") || strings.Contains(services, "**") { + t.Errorf("the page has markup left:\n%s", services) + } + mustContain(t, "en/services.gmi navigation", services, + "=> /en/ kmalloc.org\n", "=> /en/manifesto.gmi Manifesto\n", "=> /en/people.gmi People\n", "=> /en/services.gmi Services\n", "=> /news/ News\n", + "=> /servizi.gmi Italiano\n") + manifesto := readOut(t, out, "es/manifesto.gmi") + mustContain(t, "es/manifesto.gmi", manifesto, "# Manifiesto\n", "Version 1.0, 2026-11-12\n", "Traduccion de la version 1.0.\n", "=> /es/manifesto.gmi Manifiesto\n") + home := readOut(t, out, "index.gmi") + mustContain(t, "index.gmi", home, "=> /news/ Tutte le news\n", "=> /news/2026/11/shop.gmi 2026-11-14 A shop\n", "=> /en/ English\n") + if strings.HasPrefix(home, "# ") && !strings.Contains(home, "kmalloc.org") { + t.Errorf("home has no title:\n%s", home) + } +} + +// TestGeminiHomesLinkToTheNews checks that every home links to the one copy of the news. +func TestGeminiHomesLinkToTheNews(t *testing.T) { + out, err := buildGem(t, gemSite(nil), nil) + if err != nil { + t.Fatal(err) + } + for _, rel := range []string{"index.gmi", "en/index.gmi"} { + mustContain(t, rel, readOut(t, out, rel), "=> /news/ ", "=> /news/2026/11/shop.gmi 2026-11-14 A shop\n") + } +} + +// TestGeminiNews checks a post, the index, a category, the archive and the feed. +func TestGeminiNews(t *testing.T) { + out, err := buildGem(t, blogGem(), nil) + if err != nil { + t.Fatal(err) + } + post := readOut(t, out, "news/2026/11/shop.gmi") + mustContain(t, "post", post, "# A shop\n", "14 November 2026 · squid · FLUG · 1 min read\n", "About the shop.\n", + "There is a shop. Mail me at squid@kmalloc.org or visit the site.\n", "=> https://kmalloc.org/ the site\n", + "=> /news/category/flug.gmi FLUG\n", "=> https://social.kmalloc.org/@squid/115 Reply on Mastodon\n", + "=> /news/2026/11/status.gmi Older: Status page\n", "=> /news/ All news\n", "=> /en/manifesto.gmi Manifesto\n") + if strings.Contains(post, "Newer:") { + t.Error("the newest post has a link to a newer one") + } + gnu := readOut(t, out, "news/2026/09/gnu-linux-day-2026.gmi") + mustContain(t, "older post", gnu, "=> /en/people.gmi people\n", "=> /news/2026/11/status.gmi Newer: Status page\n") + index := readOut(t, out, "news/index.gmi") + mustContain(t, "index", index, "# News\n", + "=> /news/2026/11/shop.gmi 2026-11-14 A shop\nAbout the shop.\n", "=> /news/2026/11/status.gmi 2026-11-12 Status page\n", + "=> /news/category/flug.gmi FLUG (2)\n", "=> /news/2026/ 2026 (3)\n", "=> /news/2026/11/ November 2026 (2)\n", "=> /news/feed.gmi Feed\n") + category := readOut(t, out, "news/category/flug.gmi") + mustContain(t, "category", category, "# Category: FLUG\n", "=> /news/2026/11/shop.gmi 2026-11-14 A shop\n", "=> /news/2026/09/gnu-linux-day-2026.gmi 2026-09-18 GNU/Linux Day 2026\n") + month := readOut(t, out, "news/2026/11/index.gmi") + mustContain(t, "month", month, "# Archive: November 2026\n", "=> /news/2026/ Archive of the year 2026\n", "A shop") + if strings.Contains(month, "GNU/Linux") { + t.Error("the archive of November lists a post of September") + } + year := readOut(t, out, "news/2026/index.gmi") + mustContain(t, "year", year, "# Archive: 2026\n", "=> /news/2026/09/ September 2026 (1)\n", "=> /news/2026/11/ November 2026 (2)\n") +} + +// blogGem is the site of the tests of the news pages. +func blogGem() map[string]string { return gemSite(nil) } + +// TestGeminiFeed checks the feed: newest first, a date before each title. +func TestGeminiFeed(t *testing.T) { + out, err := buildGem(t, blogGem(), nil) + if err != nil { + t.Fatal(err) + } + feed := readOut(t, out, "news/feed.gmi") + want := "=> /news/2026/11/shop.gmi 2026-11-14 A shop\n=> /news/2026/11/status.gmi 2026-11-12 Status page\n=> /news/2026/09/gnu-linux-day-2026.gmi 2026-09-18 GNU/Linux Day 2026\n" + mustContain(t, "feed", feed, "# kmalloc.org: News\n", want) + if first := strings.Index(feed, "A shop"); first > strings.Index(feed, "Status page") { + t.Error("the feed is not newest first") + } +} + +// TestGeminiNeverHasBrokenLinksOrEmptyFiles walks the whole capsule: no empty file, and every +// link that stays in the capsule leads to a file. +func TestGeminiNeverHasBrokenLinksOrEmptyFiles(t *testing.T) { + out, err := buildGem(t, blogGem(), nil) + if err != nil { + t.Fatal(err) + } + link := regexp.MustCompile(`(?m)^=> (\S+)`) + for _, f := range gemFiles(t, out) { + data, err := os.ReadFile(filepath.Join(out, filepath.FromSlash(f))) + if err != nil { + t.Fatal(err) + } + if len(strings.TrimSpace(string(data))) == 0 && f != ".meta" { + t.Errorf("%s is empty", f) + } + if !strings.HasSuffix(f, ".gmi") { + continue + } + for _, m := range link.FindAllStringSubmatch(string(data), -1) { + target := m[1] + if strings.Contains(target, ":") || !strings.HasPrefix(target, "/") { + if !strings.Contains(target, ":") { + t.Errorf("%s has the relative link %s", f, target) + } + continue + } + rel := strings.TrimPrefix(target, "/") + if rel == "" || strings.HasSuffix(rel, "/") { + rel += "index.gmi" + } + if !exists(t, out, rel) { + t.Errorf("%s links to %s, which is not in the capsule", f, target) + } + } + } +} + +// TestGeminiLinksMustResolve checks that a link to nothing stops the build and names the page and the address. +func TestGeminiLinksMustResolve(t *testing.T) { + for name, extra := range map[string]map[string]string{ + "page": {"content/en/people.md": page("people", "People", "", "See [nowhere](/nowhere.html).\n")}, + "post": {"news/broken.md": "Title: Broken\nDate: 2026-11-01T12:00:00+0100\n-\nSee [nowhere](/nowhere.html).\n"}, + "unsafe scheme": {"content/en/people.md": page("people", "People", "", "[x](javascript:alert(1))\n")}, + } { + t.Run(name, func(t *testing.T) { + _, err := buildGem(t, gemSite(extra), nil) + if err == nil || !strings.Contains(err.Error(), "nowhere.html") && !strings.Contains(err.Error(), "javascript") { + t.Errorf("error = %v, want one that names the address", err) + } + }) + } +} + +// TestGeminiOnlyReferencedAssets checks that an asset is copied when a page uses it and not otherwise. +func TestGeminiOnlyReferencedAssets(t *testing.T) { + out, err := buildGem(t, gemSite(map[string]string{"content/en/services.md": page("services", "Services", "", "No image.\n")}), nil) + if err != nil { + t.Fatal(err) + } + if exists(t, out, "logo.svg") { + t.Error("an asset no page uses is copied") + } +} + +// TestGeminiDraftsAndFuture checks that hidden posts stay out of the capsule, as in the site. +func TestGeminiDraftsAndFuture(t *testing.T) { + out, err := buildGem(t, gemSite(postNews()), nil) + if err != nil { + t.Fatal(err) + } + for _, f := range gemFiles(t, out) { + if strings.Contains(f, "a-draft") || strings.Contains(f, "in-future") || strings.Contains(f, "secret") { + t.Errorf("the capsule has %s, but the post is hidden", f) + } + } + with, err := buildGem(t, gemSite(postNews()), func(o *GeminiOptions) { o.Drafts = true }) + if err != nil { + t.Fatal(err) + } + if !exists(t, with, "news/2026/11/a-draft.gmi") { + t.Error("a build with drafts lacks the draft") + } +} + +// TestGeminiBlockMustStandAlone checks the rule of the generated blocks. +func TestGeminiBlockMustStandAlone(t *testing.T) { + _, err := buildGem(t, gemSite(map[string]string{"content/en/people.md": page("people", "People", "", "Text {{news_list}} text.\n")}), nil) + if err == nil || !strings.Contains(err.Error(), "news_list") { + t.Errorf("error = %v, want one about {{news_list}}", err) + } +} + +// TestGeminiWebPaths checks that a path the website serves by other means can stay a link to the website. +func TestGeminiWebPaths(t *testing.T) { + files := gemSite(map[string]string{ + "site.json": strings.Replace(baseFiles()["site.json"], `"languages"`, `"gemini_web_paths": "/~, /pastes/", "languages"`, 1), + "content/en/people.md": page("people", "People", "", "Pages of the [people](/~squid/) and the [pastes](/pastes/x).\n"), + }) + out, err := buildGem(t, files, nil) + if err != nil { + t.Fatalf("BuildGemini() error = %v", err) + } + mustContain(t, "en/people.gmi", readOut(t, out, "en/people.gmi"), + "=> https://kmalloc.org/~squid/ people\n", "=> https://kmalloc.org/pastes/x pastes\n") + if _, err := buildGem(t, gemSite(map[string]string{"content/en/people.md": page("people", "People", "", "[x](/~squid/)\n")}), nil); err == nil { + t.Error("a path of the website is a link when the site did not say so") + } +} + +// TestGeminiHeadings checks that a page has one first level heading and that code is left alone. +func TestGeminiHeadings(t *testing.T) { + out, err := buildGem(t, gemSite(map[string]string{ + "content/en/people.md": page("people", "People", "", "# People\n\n# Other\n\n```\n# a comment in code\n```\n"), + }), nil) + if err != nil { + t.Fatal(err) + } + got := readOut(t, out, "en/people.gmi") + mustContain(t, "en/people.gmi", got, "# People\n\n## Other\n\n```\n# a comment in code\n```\n") + if strings.Count(got, "# People") != 1 { + t.Errorf("the title is repeated:\n%s", got) + } + home := readOut(t, out, "en/index.gmi") + mustContain(t, "en/index.gmi", home, "# kmalloc.org\n\n## Technology.\n") +} + +// TestGeminiKeepsLinksToOtherProtocols checks the protocols that leave the capsule as they are. +func TestGeminiKeepsLinksToOtherProtocols(t *testing.T) { + out, err := buildGem(t, gemSite(map[string]string{"content/en/people.md": page("people", "People", "", + "[Mail](mailto:squid@kmalloc.org) [Pay](bitcoin:bc1qxyz) [Chat](xmpp:squid@kmalloc.org) [Capsule](gemini://example.org/)\n")}), nil) + if err != nil { + t.Fatal(err) + } + mustContain(t, "en/people.gmi", readOut(t, out, "en/people.gmi"), + "=> mailto:squid@kmalloc.org Mail\n", "=> bitcoin:bc1qxyz Pay\n", "=> xmpp:squid@kmalloc.org Chat\n", "=> gemini://example.org/ Capsule\n") +} + +// TestGeminiTitleThatSaysTheHeading checks that a heading the title already says is not repeated, as in +// the real site where the page title is "kmalloc.org: Services" and the text starts with "# Services". +func TestGeminiTitleThatSaysTheHeading(t *testing.T) { + out, err := buildGem(t, gemSite(map[string]string{ + "content/en/people.md": page("people", "kmalloc.org: People", "", "# People\n\nText.\n"), + }), nil) + if err != nil { + t.Fatal(err) + } + got := readOut(t, out, "en/people.gmi") + if !strings.HasPrefix(got, "# kmalloc.org: People\n\nText.\n") { + t.Errorf("en/people.gmi starts with\n%q", got) + } +} + +// TestGeminiFeedTitleIsNotPrefixedTwice checks the feed title when the news page title has the prefix already. +func TestGeminiFeedTitleIsNotPrefixedTwice(t *testing.T) { + out, err := buildGem(t, gemSite(map[string]string{ + "content/en/news/index.md": page("news", "kmalloc.org: News", "", "{{news_list}}\n"), + }), nil) + if err != nil { + t.Fatal(err) + } + if got := readOut(t, out, "news/feed.gmi"); !strings.HasPrefix(got, "# kmalloc.org: News\n") { + t.Errorf("news/feed.gmi starts with %q", got) + } +} + +// TestGeminiTranslationNote checks that the note of a machine translated language reads as one sentence. +func TestGeminiTranslationNote(t *testing.T) { + files := gemSite(map[string]string{ + "lang/es.json": strings.Replace(baseFiles()["lang/es.json"], `"translation_note": "Traduccion automatica."`, `"translation_note": "Traduccion automatica, avisa si ves un error:"`, 1), + }) + out, err := buildGem(t, files, nil) + if err != nil { + t.Fatal(err) + } + mustContain(t, "es/manifesto.gmi", readOut(t, out, "es/manifesto.gmi"), + "Traduccion automatica, avisa si ves un error: avisar.\n=> https://git.kmalloc.org/squid/kmalloc.org/issues avisar\n") +} diff --git a/internal/build/pages.go b/internal/build/pages.go index 0136168..5506fe8 100644 --- a/internal/build/pages.go +++ b/internal/build/pages.go @@ -133,11 +133,12 @@ func (b *builder) markdown(name, source, lang string, ctx map[string]string) (st // codeElement matches a code element, where a placeholder is shown as written. var codeElement = regexp.MustCompile(`(?s)]*>.*?
`) -// expandBlocks replaces each paragraph that holds only a block placeholder by -// the generated HTML. A block placeholder inside other text is an error. All -// blocks are checked before any is expanded, so the text of a generated list -// (a news title may say {{latest_news}}) is never taken for a placeholder. -func (b *builder) expandBlocks(name, html, lang string, ctx map[string]string) (string, error) { +// replaceBlocks replaces each paragraph that holds only a block placeholder by +// what replace returns for the name of the block. A block placeholder inside +// other text is an error. All blocks are checked before any is replaced, so the +// text of a generated list (a news title may say {{latest_news}}) is never +// taken for a placeholder. +func replaceBlocks(name, html string, replace func(block string) (string, error)) (string, error) { outside := codeElement.ReplaceAllString(html, "") for _, block := range blockNames { token := "{{" + block + "}}" @@ -150,7 +151,7 @@ func (b *builder) expandBlocks(name, html, lang string, ctx map[string]string) ( if !strings.Contains(html, marker) { continue } - generated, err := b.block(block, lang, ctx) + generated, err := replace(block) if err != nil { return "", err } @@ -159,6 +160,11 @@ func (b *builder) expandBlocks(name, html, lang string, ctx map[string]string) ( return html, nil } +// expandBlocks replaces the block placeholders of a page by the generated HTML. +func (b *builder) expandBlocks(name, html, lang string, ctx map[string]string) (string, error) { + return replaceBlocks(name, html, func(block string) (string, error) { return b.block(block, lang, ctx) }) +} + // block returns the HTML of a generated list. func (b *builder) block(name, lang string, ctx map[string]string) (string, error) { switch name { diff --git a/internal/gemtext/gemtext.go b/internal/gemtext/gemtext.go index 395c942..4fd4ed5 100644 --- a/internal/gemtext/gemtext.go +++ b/internal/gemtext/gemtext.go @@ -88,6 +88,15 @@ func (b block) String() string { return strings.Join(lines, "\n") } +// Text returns plain text as one line of gemtext: white space collapsed, and a +// leading space when the text would be read as another kind of line. +func Text(s string) string { return protect(strings.Join(strings.Fields(s), " ")) } + +// Link returns the link line of an address with a label. +func Link(target, label string) string { + return linkLine(link{target, strings.Join(strings.Fields(label), " ")}) +} + // linkLine writes a link on its own line. func linkLine(l link) string { if l.label == "" || l.label == l.target { diff --git a/internal/gemtext/gemtext_test.go b/internal/gemtext/gemtext_test.go index d028076..1abe9fe 100644 --- a/internal/gemtext/gemtext_test.go +++ b/internal/gemtext/gemtext_test.go @@ -117,3 +117,19 @@ func TestConvertRejectsBrokenHTML(t *testing.T) { t.Errorf("error = %v, want one about the HTML", err) } } + +// TestTextAndLink checks the helpers that write single lines. +func TestTextAndLink(t *testing.T) { + if got := Text(" two \n words "); got != "two words" { + t.Errorf("Text() = %q", got) + } + if got := Text("# not a heading"); got != " # not a heading" { + t.Errorf("Text() = %q, want a leading space", got) + } + if got := Link("/a.gmi", " A page "); got != "=> /a.gmi A page" { + t.Errorf("Link() = %q", got) + } + if got := Link("/a.gmi", ""); got != "=> /a.gmi" { + t.Errorf("Link() without a label = %q", got) + } +}