// SPDX-License-Identifier: GPL-3.0-or-later package build import ( "errors" "fmt" "os" "path" "path/filepath" "regexp" "sort" "strings" "time" "git.kmalloc.org/squid/motorino/internal/content" "git.kmalloc.org/squid/motorino/internal/gemtext" "git.kmalloc.org/squid/motorino/internal/mdx" "git.kmalloc.org/squid/motorino/internal/render" ) // GeminiOptions tell BuildGemini where the site is and where to write the capsule. type GeminiOptions struct { // SiteDir is the folder of the site, the same one Build reads. SiteDir string // OutDir is the folder the gemtext is written to; it is created when missing. OutDir string // Version is the version of motorino. Version string // Now is the moment the capsule is built at, as in Options. Now time.Time // Drafts publishes the drafts and the posts dated in the future too. Drafts bool // Clean empties OutDir before the capsule is written, as Options.Clean does. Clean bool // Language is the language served at the root of the capsule, the others // living under //. It is the language the server serves by default // (Agate's --lang). Empty means gemini_language of site.json, then the // default language of the site. Language string } // GeminiResult summarises a Gemini build. type GeminiResult struct { // Pages is the number of gemtext files written. Pages int // Stale lists the files that were already in the output folder and that this // build did not write; see Result.Stale. Stale []string } // gemNews is the address of the news in the capsule. They exist once, whatever // the language of the root, so it is always /news/. const gemNews = "/news/" // metaName is the file that tells the server the language of each folder; the // server reads it and never serves it. const metaName = ".meta" // gemini holds one Gemini build, on top of the loaded site. type gemini struct { *builder // root is the language served at the root. root string // pageAddr is the address of every page in the capsule, by ID and language. pageAddr map[pageKey]string // addr maps an address of the website to the address in the capsule. addr map[string]string // usedAssets lists the assets some page points to: only those are copied. usedAssets map[string]bool // gemFiles holds the output, and gemOwners what produces each file. gemFiles map[string]string gemOwners map[string]string } // BuildGemini reads the site in opts.SiteDir and writes it as a Gemini capsule // to opts.OutDir: the pages of each language, the news once under /news/ and a // .meta file for the server. Everything is rendered in memory first, so a // failure in the content leaves the folder untouched. func BuildGemini(opts GeminiOptions) (GeminiResult, error) { b, err := load(Options{SiteDir: opts.SiteDir, OutDir: opts.OutDir, Version: opts.Version, Now: opts.Now, Drafts: opts.Drafts}) if err != nil { return GeminiResult{}, err } g := &gemini{ builder: b, pageAddr: map[pageKey]string{}, addr: map[string]string{}, usedAssets: map[string]bool{}, gemFiles: map[string]string{}, gemOwners: map[string]string{}, } if g.root, err = g.rootLanguage(opts.Language); err != nil { return GeminiResult{}, err } g.mapAddresses() for _, p := range b.pages { if err := g.renderPage(p); err != nil { return GeminiResult{}, err } } steps := []func() error{g.renderPosts, g.renderCategories, g.renderArchives, g.renderFeed} for _, step := range steps { if err := step(); err != nil { return GeminiResult{}, err } } pages := len(g.gemFiles) if err := g.publish(metaName, "the language file of the server", g.meta()); err != nil { return GeminiResult{}, err } assets, err := g.assets() if err != nil { return GeminiResult{}, err } files := make(map[string][]byte, len(g.gemFiles)+len(assets)) for rel, text := range g.gemFiles { files[rel] = []byte(text) } for rel, data := range assets { files[rel] = data } stale, err := writeOutput(opts.OutDir, opts.SiteDir, opts.Clean, files) if err != nil { return GeminiResult{}, err } return GeminiResult{Pages: pages, Stale: stale}, nil } // rootLanguage returns the language at the root of the capsule: the option, // else gemini_language of site.json, else the default language. It must be a // language of the site that has a home page. func (g *gemini) rootLanguage(option string) (string, error) { code, from := option, "the language option" if code == "" { code, from = g.cfg.Strings["gemini_language"], "gemini_language in site.json" } if code == "" { return g.defaultLang, nil } if _, ok := g.langs[code]; !ok { return "", fmt.Errorf("%s: %q is not one of the languages of the site", from, code) } if _, ok := g.byKey[pageKey{"home", code}]; !ok { return "", fmt.Errorf("%s: the language %q has no home page, so it cannot be the root of the capsule", from, code) } return code, nil } // meta returns the content of the .meta file: for every language other than // the one at the root, and for the news when they are in another language, the // lines that give the language of the documents. A glob does not cross folders, // so each folder needs a line for itself and one for what is below it. func (g *gemini) meta() string { var out strings.Builder line := func(folder, lang string) { fmt.Fprintf(&out, "%s/*:;lang=%s\n%s/**/*:;lang=%s\n", folder, lang, folder, lang) } for _, lang := range g.cfg.Languages { if lang != g.root { line(lang, lang) } } if g.newsLang != g.root { line(strings.Trim(gemNews, "/"), g.newsLang) } return out.String() } // pageGemAddress returns the address in the capsule of a page of lang whose // address in the website is html: the root language at the root, the others // under their code, .gmi for .html. func (g *gemini) pageGemAddress(lang, html string) string { p := html if lang != g.defaultLang { p = strings.TrimPrefix(p, "/"+lang) } if strings.HasSuffix(p, ".html") { p = strings.TrimSuffix(p, ".html") + ".gmi" } if lang != g.root { p = "/" + lang + p } return p } // gemPostAddress, gemCategoryAddress, gemYearAddress and gemMonthAddress are the // addresses in the capsule of a post, a category and the archives. func gemPostAddress(p content.Post) string { return strings.TrimSuffix(p.Path, ".html") + ".gmi" } func gemCategoryAddress(slug string) string { return gemNews + "category/" + slug + ".gmi" } func gemYearAddress(year int) string { return fmt.Sprintf("%s%04d/", gemNews, year) } func gemMonthAddress(year, month int) string { return fmt.Sprintf("%s%04d/%02d/", gemNews, year, month) } // mapAddresses fills the maps from the addresses of the website to those of the capsule. func (g *gemini) mapAddresses() { for _, p := range g.pages { address := g.pageGemAddress(p.Lang, p.Path) if p.ID == "news" && p.Lang == g.newsLang { address = gemNews } g.pageAddr[pageKey{p.ID, p.Lang}] = address g.addr[p.Path] = address } for _, p := range g.posts { g.addr[p.Path] = gemPostAddress(p) } for _, c := range g.categories { g.addr[categoryPath(c.Slug)] = gemCategoryAddress(c.Slug) } for _, y := range groupArchive(g.posts) { g.addr[g.yearPath(y.year)] = gemYearAddress(y.year) for _, m := range y.months { g.addr[g.monthPath(m.year, m.month)] = gemMonthAddress(m.year, m.month) } } g.addr[g.newsPath+"feed.xml"] = gemNews + "feed.gmi" } // publish stores a file of the capsule; source names what produced it. func (g *gemini) publish(rel, source, text string) error { if other, taken := g.gemOwners[rel]; taken { return fmt.Errorf("%s and %s are both published as %s", other, source, rel) } g.gemOwners[rel] = source g.gemFiles[rel] = text return nil } // gemFile returns the file of an address of the capsule: a folder gets index.gmi. func gemFile(address string) string { rel := strings.TrimPrefix(address, "/") if rel == "" || strings.HasSuffix(rel, "/") { rel += "index.gmi" } return rel } // scheme matches the start of an address that names a protocol. var scheme = regexp.MustCompile(`^([A-Za-z][A-Za-z0-9+.-]*):`) // externalSchemes are the protocols a link may use to leave the capsule. var externalSchemes = map[string]bool{ "http": true, "https": true, "mailto": true, "gemini": true, "xmpp": true, "irc": true, "ircs": true, "matrix": true, "bitcoin": true, "magnet": true, "tel": true, "sms": true, } // resolve turns an address found in the HTML of the page at base into the // address to write in the capsule. An address with a protocol stays as it is // when the protocol is a known one, and is an error otherwise; an address in the // website is mapped to its page or file, and a file is copied by the end of the // build. An address that starts with one of the paths of gemini_web_paths in // site.json is one the website serves by other means: it becomes a link to the // website. A fragment alone is dropped, and so is the fragment of an address: a // gemini link cannot point inside a document. func (g *gemini) resolve(href, base string) (string, error) { if strings.HasPrefix(href, "#") { return "", nil } if m := scheme.FindStringSubmatch(href); m != nil { if !externalSchemes[strings.ToLower(m[1])] { return "", fmt.Errorf("the protocol %q is not supported", m[1]) } return href, nil } if strings.HasPrefix(href, "//") { return "", errors.New("an address that starts with // is not supported") } target, _, _ := strings.Cut(href, "#") if !strings.HasPrefix(target, "/") { folder := path.Dir(base) if strings.HasSuffix(base, "/") { folder = strings.TrimSuffix(base, "/") } joined := path.Join("/", folder, target) if strings.HasSuffix(target, "/") && joined != "/" { joined += "/" } target = joined } if address, ok := g.addr[target]; ok { return address, nil } name := strings.TrimPrefix(target, "/") for asset := range g.cfg.Assets { if path.Clean(asset) == name { g.usedAssets[asset] = true return target, nil } } for _, prefix := range strings.Split(g.cfg.Strings["gemini_web_paths"], ",") { if prefix = strings.TrimSpace(prefix); prefix != "" && strings.HasPrefix(target, prefix) { return g.cfg.Strings["base_url"] + target, nil } } return "", errors.New("no page or file of the site has this address (a path the website serves by other means goes in gemini_web_paths of site.json)") } // convert converts the HTML of a page or post to gemtext. base is the address // of the page in the website, lang its language. func (g *gemini) convert(name, html, base, lang string) (string, error) { text, err := gemtext.Convert(html, gemtext.Options{ Link: func(href string) (string, error) { return g.resolve(href, base) }, Block: func(block string) (string, error) { return g.block(block, lang) }, }) if err != nil { return "", fmt.Errorf("%s: %w", name, err) } return strings.TrimRight(text, "\n"), nil } // markdown replaces the placeholders of a source, converts it to HTML and puts // the element where a generated list goes, for convert to expand. func (g *gemini) markdown(name, source string, ctx map[string]string) (string, error) { text, err := mdx.SubstituteMarkdown(name, source, ctx) if err != nil { return "", err } html, _, err := mdx.Render(text) if err != nil { return "", fmt.Errorf("%s: %w", name, err) } html, err = replaceBlocks(name, html, func(block string) (string, error) { return ``, nil }) if err != nil { return "", err } return render.Unescape(html), nil } // assets reads the files some page points to. func (g *gemini) assets() (map[string][]byte, error) { names := make([]string, 0, len(g.usedAssets)) for name := range g.usedAssets { names = append(names, name) } sort.Strings(names) out := map[string][]byte{} for _, name := range names { source := g.cfg.Assets[name] data, err := os.ReadFile(filepath.Join(g.opts.SiteDir, filepath.FromSlash(source))) if err != nil { return nil, fmt.Errorf("asset %q: %w", name, err) } rel := path.Clean(name) if other, taken := g.gemOwners[rel]; taken { return nil, fmt.Errorf("the asset %q and %s are both published as %s", name, other, rel) } out[rel] = data } return out, nil } // heading is the first level heading of a page. func heading(title string) string { return "# " + strings.Join(strings.Fields(title), " ") } // withoutTitle prepares a body converted to gemtext to go under the heading of // the title. A leading heading that says the title is dropped, since the page // has it already, and the other first level headings go one level down: a page // has one. func withoutTitle(title, body string) string { lines := strings.Split(body, "\n") if first, ok := strings.CutPrefix(lines[0], "# "); ok && (first == title || strings.HasSuffix(title, ": "+first)) { lines = lines[1:] } fenced := false for i, line := range lines { switch { case strings.HasPrefix(line, "```"): fenced = !fenced case !fenced && strings.HasPrefix(line, "# "): lines[i] = "#" + line } } return strings.Join(lines, "\n") } // navigation returns the links every page ends with: the home and the entries // of the menu of the language, in the order its file lists them, then the // other languages. An entry whose page the language lacks is left out. func (g *gemini) navigation(d document) string { l := g.langs[d.lang] var menu []string if home, ok := g.pageAddr[pageKey{"home", d.lang}]; ok { label := l.Strings["back"] if label == "" { label = g.byKey[pageKey{"home", d.lang}].Title } menu = append(menu, gemtext.Link(home, label)) } for _, key := range l.NavOrder { address, ok := g.pageAddr[pageKey{key, d.lang}] if !ok && key == "news" { address, ok = gemNews, true } if ok { menu = append(menu, gemtext.Link(address, l.Nav[key])) } } parts := []string{strings.Join(menu, "\n")} var others []string for _, t := range g.languageTargets(d) { if address, err := g.resolve(t.path, d.path); err == nil && address != "" { others = append(others, gemtext.Link(address, g.langs[t.lang].Strings["lang_name"])) } } if len(others) > 0 { parts = append(parts, strings.Join(others, "\n")) } return strings.Join(parts, "\n\n") } // translationNote returns the line, and the link to report a problem, that a // machine translated language puts at the top of its pages; empty for the others. func (g *gemini) translationNote(lang string) string { l := g.langs[lang] note := l.Strings["translation_note"] if note == "" { return "" } label, issues := l.Strings["translation_note_a"], g.cfg.Strings["issues_url"] if label == "" || issues == "" { return gemtext.Text(note) } return gemtext.Text(note+" "+label+".") + "\n" + gemtext.Link(issues, label) } // compose joins the parts of a page with a blank line between them, leaving out // the empty ones, and ends the text with a line break. func compose(parts ...string) string { var kept []string for _, part := range parts { if part = strings.Trim(part, "\n"); part != "" { kept = append(kept, part) } } return strings.Join(kept, "\n\n") + "\n" } // renderPage converts a page and writes it with the navigation. func (g *gemini) renderPage(p content.Page) error { d := document{ lang: p.Lang, id: p.ID, nav: p.ID, path: p.Path, title: p.Title, description: p.Description, switches: p.Switch, source: p.Source, } ctx := g.documentContext(d) version, date := g.cfg.Strings["manifesto_version"], g.cfg.Strings["manifesto_date"] if p.ManifestoVersion != "" { version = p.ManifestoVersion } if p.ManifestoDate != "" { date = p.ManifestoDate } ctx["manifesto_version"], ctx["manifesto_date"] = render.Escape(version), render.Escape(date) source := p.Body if p.Layout == "manifesto" { var err error if source, err = mdx.NoEmDash(source); err != nil { return fmt.Errorf("%s: %w", p.Source, err) } } html, err := g.markdown(p.Source, source, ctx) if err != nil { return err } body, err := g.convert(p.Source, html, p.Path, p.Lang) if err != nil { return err } var intro []string if p.Layout == "manifesto" { intro = []string{g.manifestoLine(p.Lang, version, date), gemtext.Text(p.Note)} } parts := append([]string{heading(p.Title), g.translationNote(p.Lang)}, intro...) parts = append(parts, withoutTitle(p.Title, body), g.navigation(d)) return g.publish(gemFile(g.pageAddr[pageKey{p.ID, p.Lang}]), p.Source, compose(parts...)) } // manifestoLine is the line that says the version and the date of the manifesto. func (g *gemini) manifestoLine(lang, version, date string) string { line := strings.TrimSpace(g.langs[lang].Strings["version"] + " " + version) if date != "" { line += ", " + date } return gemtext.Text(line) }