motorino/internal/build/gemini.go
tgiachi 2f169b8d9f feat(gemini): write the site as a Gemini capsule
`motorino gemini [-site DIR] [-out DIR] [-lang CODE] [-drafts]` turns the pages and the
news into gemtext with the new gemtext package.

- the language served by default sits at the root (-lang, then gemini_language in
  site.json, then the default language) and the others under /<code>/
- the news exist once, under /news/: index, posts, categories, archives and a
  gemlog style feed (news/feed.gmi, a date before each title)
- every page ends with the navigation of its language, the language links and, on
  translations, the machine translation note
- .meta for Agate (--central-conf): two lines per language other than the root one
  and two for the news when they are in another language, never linked
- addresses are mapped to the capsule, a fragment is dropped, a missing target stops
  the build, assets are copied only when a page points to them, gemini_web_paths
  keeps paths served by other means as links to the website
- blocks of generated lists become gemtext with the strings of the language file
- the block replacement of a page is shared with the HTML build

Refs #10
2026-11-16 12:28:10 +01:00

479 lines
16 KiB
Go

// SPDX-License-Identifier: GPL-3.0-or-later
package build
import (
"errors"
"fmt"
"os"
"path"
"path/filepath"
"regexp"
"sort"
"strings"
"time"
"git.kmalloc.org/squid/motorino/internal/content"
"git.kmalloc.org/squid/motorino/internal/gemtext"
"git.kmalloc.org/squid/motorino/internal/mdx"
"git.kmalloc.org/squid/motorino/internal/render"
)
// GeminiOptions tell BuildGemini where the site is and where to write the capsule.
type GeminiOptions struct {
// SiteDir is the folder of the site, the same one Build reads.
SiteDir string
// OutDir is the folder the gemtext is written to; it is created when missing.
OutDir string
// Version is the version of motorino.
Version string
// Now is the moment the capsule is built at, as in Options.
Now time.Time
// Drafts publishes the drafts and the posts dated in the future too.
Drafts bool
// Language is the language served at the root of the capsule, the others
// living under /<code>/. It is the language the server serves by default
// (Agate's --lang). Empty means gemini_language of site.json, then the
// default language of the site.
Language string
}
// GeminiResult summarises a Gemini build.
type GeminiResult struct {
// Pages is the number of gemtext files written.
Pages int
}
// gemNews is the address of the news in the capsule. They exist once, whatever
// the language of the root, so it is always /news/.
const gemNews = "/news/"
// metaName is the file that tells the server the language of each folder; the
// server reads it and never serves it.
const metaName = ".meta"
// gemini holds one Gemini build, on top of the loaded site.
type gemini struct {
*builder
// root is the language served at the root.
root string
// pageAddr is the address of every page in the capsule, by ID and language.
pageAddr map[pageKey]string
// addr maps an address of the website to the address in the capsule.
addr map[string]string
// usedAssets lists the assets some page points to: only those are copied.
usedAssets map[string]bool
// gemFiles holds the output, and gemOwners what produces each file.
gemFiles map[string]string
gemOwners map[string]string
}
// BuildGemini reads the site in opts.SiteDir and writes it as a Gemini capsule
// to opts.OutDir: the pages of each language, the news once under /news/ and a
// .meta file for the server. Everything is rendered in memory first, so a
// failure in the content leaves the folder untouched.
func BuildGemini(opts GeminiOptions) (GeminiResult, error) {
b, err := load(Options{SiteDir: opts.SiteDir, OutDir: opts.OutDir, Version: opts.Version, Now: opts.Now, Drafts: opts.Drafts})
if err != nil {
return GeminiResult{}, err
}
g := &gemini{
builder: b, pageAddr: map[pageKey]string{}, addr: map[string]string{}, usedAssets: map[string]bool{},
gemFiles: map[string]string{}, gemOwners: map[string]string{},
}
if g.root, err = g.rootLanguage(opts.Language); err != nil {
return GeminiResult{}, err
}
g.mapAddresses()
for _, p := range b.pages {
if err := g.renderPage(p); err != nil {
return GeminiResult{}, err
}
}
steps := []func() error{g.renderPosts, g.renderCategories, g.renderArchives, g.renderFeed}
for _, step := range steps {
if err := step(); err != nil {
return GeminiResult{}, err
}
}
pages := len(g.gemFiles)
if err := g.publish(metaName, "the language file of the server", g.meta()); err != nil {
return GeminiResult{}, err
}
assets, err := g.assets()
if err != nil {
return GeminiResult{}, err
}
for rel, text := range g.gemFiles {
if err := writeFile(opts.OutDir, rel, []byte(text)); err != nil {
return GeminiResult{}, err
}
}
for rel, data := range assets {
if err := writeFile(opts.OutDir, rel, data); err != nil {
return GeminiResult{}, err
}
}
return GeminiResult{Pages: pages}, nil
}
// rootLanguage returns the language at the root of the capsule: the option,
// else gemini_language of site.json, else the default language. It must be a
// language of the site that has a home page.
func (g *gemini) rootLanguage(option string) (string, error) {
code, from := option, "the language option"
if code == "" {
code, from = g.cfg.Strings["gemini_language"], "gemini_language in site.json"
}
if code == "" {
return g.defaultLang, nil
}
if _, ok := g.langs[code]; !ok {
return "", fmt.Errorf("%s: %q is not one of the languages of the site", from, code)
}
if _, ok := g.byKey[pageKey{"home", code}]; !ok {
return "", fmt.Errorf("%s: the language %q has no home page, so it cannot be the root of the capsule", from, code)
}
return code, nil
}
// meta returns the content of the .meta file: for every language other than
// the one at the root, and for the news when they are in another language, the
// lines that give the language of the documents. A glob does not cross folders,
// so each folder needs a line for itself and one for what is below it.
func (g *gemini) meta() string {
var out strings.Builder
line := func(folder, lang string) {
fmt.Fprintf(&out, "%s/*:;lang=%s\n%s/**/*:;lang=%s\n", folder, lang, folder, lang)
}
for _, lang := range g.cfg.Languages {
if lang != g.root {
line(lang, lang)
}
}
if g.newsLang != g.root {
line(strings.Trim(gemNews, "/"), g.newsLang)
}
return out.String()
}
// pageGemAddress returns the address in the capsule of a page of lang whose
// address in the website is html: the root language at the root, the others
// under their code, .gmi for .html.
func (g *gemini) pageGemAddress(lang, html string) string {
p := html
if lang != g.defaultLang {
p = strings.TrimPrefix(p, "/"+lang)
}
if strings.HasSuffix(p, ".html") {
p = strings.TrimSuffix(p, ".html") + ".gmi"
}
if lang != g.root {
p = "/" + lang + p
}
return p
}
// gemPostAddress, gemCategoryAddress, gemYearAddress and gemMonthAddress are the
// addresses in the capsule of a post, a category and the archives.
func gemPostAddress(p content.Post) string { return strings.TrimSuffix(p.Path, ".html") + ".gmi" }
func gemCategoryAddress(slug string) string { return gemNews + "category/" + slug + ".gmi" }
func gemYearAddress(year int) string { return fmt.Sprintf("%s%04d/", gemNews, year) }
func gemMonthAddress(year, month int) string {
return fmt.Sprintf("%s%04d/%02d/", gemNews, year, month)
}
// mapAddresses fills the maps from the addresses of the website to those of the capsule.
func (g *gemini) mapAddresses() {
for _, p := range g.pages {
address := g.pageGemAddress(p.Lang, p.Path)
if p.ID == "news" && p.Lang == g.newsLang {
address = gemNews
}
g.pageAddr[pageKey{p.ID, p.Lang}] = address
g.addr[p.Path] = address
}
for _, p := range g.posts {
g.addr[p.Path] = gemPostAddress(p)
}
for _, c := range g.categories {
g.addr[categoryPath(c.Slug)] = gemCategoryAddress(c.Slug)
}
for _, y := range groupArchive(g.posts) {
g.addr[g.yearPath(y.year)] = gemYearAddress(y.year)
for _, m := range y.months {
g.addr[g.monthPath(m.year, m.month)] = gemMonthAddress(m.year, m.month)
}
}
g.addr[g.newsPath+"feed.xml"] = gemNews + "feed.gmi"
}
// publish stores a file of the capsule; source names what produced it.
func (g *gemini) publish(rel, source, text string) error {
if other, taken := g.gemOwners[rel]; taken {
return fmt.Errorf("%s and %s are both published as %s", other, source, rel)
}
g.gemOwners[rel] = source
g.gemFiles[rel] = text
return nil
}
// gemFile returns the file of an address of the capsule: a folder gets index.gmi.
func gemFile(address string) string {
rel := strings.TrimPrefix(address, "/")
if rel == "" || strings.HasSuffix(rel, "/") {
rel += "index.gmi"
}
return rel
}
// scheme matches the start of an address that names a protocol.
var scheme = regexp.MustCompile(`^([A-Za-z][A-Za-z0-9+.-]*):`)
// externalSchemes are the protocols a link may use to leave the capsule.
var externalSchemes = map[string]bool{
"http": true, "https": true, "mailto": true, "gemini": true, "xmpp": true, "irc": true, "ircs": true, "matrix": true,
"bitcoin": true, "magnet": true, "tel": true, "sms": true,
}
// resolve turns an address found in the HTML of the page at base into the
// address to write in the capsule. An address with a protocol stays as it is
// when the protocol is a known one, and is an error otherwise; an address in the
// website is mapped to its page or file, and a file is copied by the end of the
// build. An address that starts with one of the paths of gemini_web_paths in
// site.json is one the website serves by other means: it becomes a link to the
// website. A fragment alone is dropped, and so is the fragment of an address: a
// gemini link cannot point inside a document.
func (g *gemini) resolve(href, base string) (string, error) {
if strings.HasPrefix(href, "#") {
return "", nil
}
if m := scheme.FindStringSubmatch(href); m != nil {
if !externalSchemes[strings.ToLower(m[1])] {
return "", fmt.Errorf("the protocol %q is not supported", m[1])
}
return href, nil
}
if strings.HasPrefix(href, "//") {
return "", errors.New("an address that starts with // is not supported")
}
target, _, _ := strings.Cut(href, "#")
if !strings.HasPrefix(target, "/") {
folder := path.Dir(base)
if strings.HasSuffix(base, "/") {
folder = strings.TrimSuffix(base, "/")
}
joined := path.Join("/", folder, target)
if strings.HasSuffix(target, "/") && joined != "/" {
joined += "/"
}
target = joined
}
if address, ok := g.addr[target]; ok {
return address, nil
}
name := strings.TrimPrefix(target, "/")
for asset := range g.cfg.Assets {
if path.Clean(asset) == name {
g.usedAssets[asset] = true
return target, nil
}
}
for _, prefix := range strings.Split(g.cfg.Strings["gemini_web_paths"], ",") {
if prefix = strings.TrimSpace(prefix); prefix != "" && strings.HasPrefix(target, prefix) {
return g.cfg.Strings["base_url"] + target, nil
}
}
return "", errors.New("no page or file of the site has this address (a path the website serves by other means goes in gemini_web_paths of site.json)")
}
// convert converts the HTML of a page or post to gemtext. base is the address
// of the page in the website, lang its language.
func (g *gemini) convert(name, html, base, lang string) (string, error) {
text, err := gemtext.Convert(html, gemtext.Options{
Link: func(href string) (string, error) { return g.resolve(href, base) },
Block: func(block string) (string, error) { return g.block(block, lang) },
})
if err != nil {
return "", fmt.Errorf("%s: %w", name, err)
}
return strings.TrimRight(text, "\n"), nil
}
// markdown replaces the placeholders of a source, converts it to HTML and puts
// the element <gemblock> where a generated list goes, for convert to expand.
func (g *gemini) markdown(name, source string, ctx map[string]string) (string, error) {
text, err := mdx.SubstituteMarkdown(name, source, ctx)
if err != nil {
return "", err
}
html, _, err := mdx.Render(text)
if err != nil {
return "", fmt.Errorf("%s: %w", name, err)
}
html, err = replaceBlocks(name, html, func(block string) (string, error) {
return `<gemblock name="` + block + `"></gemblock>`, nil
})
if err != nil {
return "", err
}
return render.Unescape(html), nil
}
// assets reads the files some page points to.
func (g *gemini) assets() (map[string][]byte, error) {
names := make([]string, 0, len(g.usedAssets))
for name := range g.usedAssets {
names = append(names, name)
}
sort.Strings(names)
out := map[string][]byte{}
for _, name := range names {
source := g.cfg.Assets[name]
data, err := os.ReadFile(filepath.Join(g.opts.SiteDir, filepath.FromSlash(source)))
if err != nil {
return nil, fmt.Errorf("asset %q: %w", name, err)
}
rel := path.Clean(name)
if other, taken := g.gemOwners[rel]; taken {
return nil, fmt.Errorf("the asset %q and %s are both published as %s", name, other, rel)
}
out[rel] = data
}
return out, nil
}
// heading is the first level heading of a page.
func heading(title string) string { return "# " + strings.Join(strings.Fields(title), " ") }
// withoutTitle prepares a body converted to gemtext to go under the heading of
// the title. A leading heading that says the title is dropped, since the page
// has it already, and the other first level headings go one level down: a page
// has one.
func withoutTitle(title, body string) string {
lines := strings.Split(body, "\n")
if first, ok := strings.CutPrefix(lines[0], "# "); ok && (first == title || strings.HasSuffix(title, ": "+first)) {
lines = lines[1:]
}
fenced := false
for i, line := range lines {
switch {
case strings.HasPrefix(line, "```"):
fenced = !fenced
case !fenced && strings.HasPrefix(line, "# "):
lines[i] = "#" + line
}
}
return strings.Join(lines, "\n")
}
// navigation returns the links every page ends with: the home and the entries
// of the menu of the language, in the order its file lists them, then the
// other languages. An entry whose page the language lacks is left out.
func (g *gemini) navigation(d document) string {
l := g.langs[d.lang]
var menu []string
if home, ok := g.pageAddr[pageKey{"home", d.lang}]; ok {
label := l.Strings["back"]
if label == "" {
label = g.byKey[pageKey{"home", d.lang}].Title
}
menu = append(menu, gemtext.Link(home, label))
}
for _, key := range l.NavOrder {
address, ok := g.pageAddr[pageKey{key, d.lang}]
if !ok && key == "news" {
address, ok = gemNews, true
}
if ok {
menu = append(menu, gemtext.Link(address, l.Nav[key]))
}
}
parts := []string{strings.Join(menu, "\n")}
var others []string
for _, t := range g.languageTargets(d) {
if address, err := g.resolve(t.path, d.path); err == nil && address != "" {
others = append(others, gemtext.Link(address, g.langs[t.lang].Strings["lang_name"]))
}
}
if len(others) > 0 {
parts = append(parts, strings.Join(others, "\n"))
}
return strings.Join(parts, "\n\n")
}
// translationNote returns the line, and the link to report a problem, that a
// machine translated language puts at the top of its pages; empty for the others.
func (g *gemini) translationNote(lang string) string {
l := g.langs[lang]
note := l.Strings["translation_note"]
if note == "" {
return ""
}
label, issues := l.Strings["translation_note_a"], g.cfg.Strings["issues_url"]
if label == "" || issues == "" {
return gemtext.Text(note)
}
return gemtext.Text(note+" "+label+".") + "\n" + gemtext.Link(issues, label)
}
// compose joins the parts of a page with a blank line between them, leaving out
// the empty ones, and ends the text with a line break.
func compose(parts ...string) string {
var kept []string
for _, part := range parts {
if part = strings.Trim(part, "\n"); part != "" {
kept = append(kept, part)
}
}
return strings.Join(kept, "\n\n") + "\n"
}
// renderPage converts a page and writes it with the navigation.
func (g *gemini) renderPage(p content.Page) error {
d := document{
lang: p.Lang, id: p.ID, nav: p.ID, path: p.Path, title: p.Title,
description: p.Description, switches: p.Switch, source: p.Source,
}
ctx := g.documentContext(d)
version, date := g.cfg.Strings["manifesto_version"], g.cfg.Strings["manifesto_date"]
if p.ManifestoVersion != "" {
version = p.ManifestoVersion
}
if p.ManifestoDate != "" {
date = p.ManifestoDate
}
ctx["manifesto_version"], ctx["manifesto_date"] = render.Escape(version), render.Escape(date)
source := p.Body
if p.Layout == "manifesto" {
var err error
if source, err = mdx.NoEmDash(source); err != nil {
return fmt.Errorf("%s: %w", p.Source, err)
}
}
html, err := g.markdown(p.Source, source, ctx)
if err != nil {
return err
}
body, err := g.convert(p.Source, html, p.Path, p.Lang)
if err != nil {
return err
}
var intro []string
if p.Layout == "manifesto" {
intro = []string{g.manifestoLine(p.Lang, version, date), gemtext.Text(p.Note)}
}
parts := append([]string{heading(p.Title), g.translationNote(p.Lang)}, intro...)
parts = append(parts, withoutTitle(p.Title, body), g.navigation(d))
return g.publish(gemFile(g.pageAddr[pageKey{p.ID, p.Lang}]), p.Source, compose(parts...))
}
// manifestoLine is the line that says the version and the date of the manifesto.
func (g *gemini) manifestoLine(lang, version, date string) string {
line := strings.TrimSpace(g.langs[lang].Strings["version"] + " " + version)
if date != "" {
line += ", " + date
}
return gemtext.Text(line)
}