485 lines
16 KiB
Go
485 lines
16 KiB
Go
// SPDX-License-Identifier: GPL-3.0-or-later
|
|
|
|
package build
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.kmalloc.org/squid/motorino/internal/content"
|
|
"git.kmalloc.org/squid/motorino/internal/gemtext"
|
|
"git.kmalloc.org/squid/motorino/internal/mdx"
|
|
"git.kmalloc.org/squid/motorino/internal/render"
|
|
)
|
|
|
|
// GeminiOptions tell BuildGemini where the site is and where to write the capsule.
|
|
type GeminiOptions struct {
|
|
// SiteDir is the folder of the site, the same one Build reads.
|
|
SiteDir string
|
|
// OutDir is the folder the gemtext is written to; it is created when missing.
|
|
OutDir string
|
|
// Version is the version of motorino.
|
|
Version string
|
|
// Now is the moment the capsule is built at, as in Options.
|
|
Now time.Time
|
|
// Drafts publishes the drafts and the posts dated in the future too.
|
|
Drafts bool
|
|
// Clean empties OutDir before the capsule is written, as Options.Clean does.
|
|
Clean bool
|
|
// Language is the language served at the root of the capsule, the others
|
|
// living under /<code>/. It is the language the server serves by default
|
|
// (Agate's --lang). Empty means gemini_language of site.json, then the
|
|
// default language of the site.
|
|
Language string
|
|
}
|
|
|
|
// GeminiResult summarises a Gemini build.
|
|
type GeminiResult struct {
|
|
// Pages is the number of gemtext files written.
|
|
Pages int
|
|
// Stale lists the files that were already in the output folder and that this
|
|
// build did not write; see Result.Stale.
|
|
Stale []string
|
|
}
|
|
|
|
// gemNews is the address of the news in the capsule. They exist once, whatever
|
|
// the language of the root, so it is always /news/.
|
|
const gemNews = "/news/"
|
|
|
|
// metaName is the file that tells the server the language of each folder; the
|
|
// server reads it and never serves it.
|
|
const metaName = ".meta"
|
|
|
|
// gemini holds one Gemini build, on top of the loaded site.
|
|
type gemini struct {
|
|
*builder
|
|
// root is the language served at the root.
|
|
root string
|
|
// pageAddr is the address of every page in the capsule, by ID and language.
|
|
pageAddr map[pageKey]string
|
|
// addr maps an address of the website to the address in the capsule.
|
|
addr map[string]string
|
|
// usedAssets lists the assets some page points to: only those are copied.
|
|
usedAssets map[string]bool
|
|
// gemFiles holds the output, and gemOwners what produces each file.
|
|
gemFiles map[string]string
|
|
gemOwners map[string]string
|
|
}
|
|
|
|
// BuildGemini reads the site in opts.SiteDir and writes it as a Gemini capsule
|
|
// to opts.OutDir: the pages of each language, the news once under /news/ and a
|
|
// .meta file for the server. Everything is rendered in memory first, so a
|
|
// failure in the content leaves the folder untouched.
|
|
func BuildGemini(opts GeminiOptions) (GeminiResult, error) {
|
|
b, err := load(Options{SiteDir: opts.SiteDir, OutDir: opts.OutDir, Version: opts.Version, Now: opts.Now, Drafts: opts.Drafts})
|
|
if err != nil {
|
|
return GeminiResult{}, err
|
|
}
|
|
g := &gemini{
|
|
builder: b, pageAddr: map[pageKey]string{}, addr: map[string]string{}, usedAssets: map[string]bool{},
|
|
gemFiles: map[string]string{}, gemOwners: map[string]string{},
|
|
}
|
|
if g.root, err = g.rootLanguage(opts.Language); err != nil {
|
|
return GeminiResult{}, err
|
|
}
|
|
g.mapAddresses()
|
|
for _, p := range b.pages {
|
|
if err := g.renderPage(p); err != nil {
|
|
return GeminiResult{}, err
|
|
}
|
|
}
|
|
steps := []func() error{g.renderPosts, g.renderCategories, g.renderArchives, g.renderFeed}
|
|
for _, step := range steps {
|
|
if err := step(); err != nil {
|
|
return GeminiResult{}, err
|
|
}
|
|
}
|
|
pages := len(g.gemFiles)
|
|
if err := g.publish(metaName, "the language file of the server", g.meta()); err != nil {
|
|
return GeminiResult{}, err
|
|
}
|
|
assets, err := g.assets()
|
|
if err != nil {
|
|
return GeminiResult{}, err
|
|
}
|
|
files := make(map[string][]byte, len(g.gemFiles)+len(assets))
|
|
for rel, text := range g.gemFiles {
|
|
files[rel] = []byte(text)
|
|
}
|
|
for rel, data := range assets {
|
|
files[rel] = data
|
|
}
|
|
stale, err := writeOutput(opts.OutDir, opts.SiteDir, opts.Clean, files)
|
|
if err != nil {
|
|
return GeminiResult{}, err
|
|
}
|
|
return GeminiResult{Pages: pages, Stale: stale}, nil
|
|
}
|
|
|
|
// rootLanguage returns the language at the root of the capsule: the option,
|
|
// else gemini_language of site.json, else the default language. It must be a
|
|
// language of the site that has a home page.
|
|
func (g *gemini) rootLanguage(option string) (string, error) {
|
|
code, from := option, "the language option"
|
|
if code == "" {
|
|
code, from = g.cfg.Strings["gemini_language"], "gemini_language in site.json"
|
|
}
|
|
if code == "" {
|
|
return g.defaultLang, nil
|
|
}
|
|
if _, ok := g.langs[code]; !ok {
|
|
return "", fmt.Errorf("%s: %q is not one of the languages of the site", from, code)
|
|
}
|
|
if _, ok := g.byKey[pageKey{"home", code}]; !ok {
|
|
return "", fmt.Errorf("%s: the language %q has no home page, so it cannot be the root of the capsule", from, code)
|
|
}
|
|
return code, nil
|
|
}
|
|
|
|
// meta returns the content of the .meta file: for every language other than
|
|
// the one at the root, and for the news when they are in another language, the
|
|
// lines that give the language of the documents. A glob does not cross folders,
|
|
// so each folder needs a line for itself and one for what is below it.
|
|
func (g *gemini) meta() string {
|
|
var out strings.Builder
|
|
line := func(folder, lang string) {
|
|
fmt.Fprintf(&out, "%s/*:;lang=%s\n%s/**/*:;lang=%s\n", folder, lang, folder, lang)
|
|
}
|
|
for _, lang := range g.cfg.Languages {
|
|
if lang != g.root {
|
|
line(lang, lang)
|
|
}
|
|
}
|
|
if g.newsLang != g.root {
|
|
line(strings.Trim(gemNews, "/"), g.newsLang)
|
|
}
|
|
return out.String()
|
|
}
|
|
|
|
// pageGemAddress returns the address in the capsule of a page of lang whose
|
|
// address in the website is html: the root language at the root, the others
|
|
// under their code, .gmi for .html.
|
|
func (g *gemini) pageGemAddress(lang, html string) string {
|
|
p := html
|
|
if lang != g.defaultLang {
|
|
p = strings.TrimPrefix(p, "/"+lang)
|
|
}
|
|
if strings.HasSuffix(p, ".html") {
|
|
p = strings.TrimSuffix(p, ".html") + ".gmi"
|
|
}
|
|
if lang != g.root {
|
|
p = "/" + lang + p
|
|
}
|
|
return p
|
|
}
|
|
|
|
// gemPostAddress, gemCategoryAddress, gemYearAddress and gemMonthAddress are the
|
|
// addresses in the capsule of a post, a category and the archives.
|
|
func gemPostAddress(p content.Post) string { return strings.TrimSuffix(p.Path, ".html") + ".gmi" }
|
|
func gemCategoryAddress(slug string) string { return gemNews + "category/" + slug + ".gmi" }
|
|
func gemYearAddress(year int) string { return fmt.Sprintf("%s%04d/", gemNews, year) }
|
|
func gemMonthAddress(year, month int) string {
|
|
return fmt.Sprintf("%s%04d/%02d/", gemNews, year, month)
|
|
}
|
|
|
|
// mapAddresses fills the maps from the addresses of the website to those of the capsule.
|
|
func (g *gemini) mapAddresses() {
|
|
for _, p := range g.pages {
|
|
address := g.pageGemAddress(p.Lang, p.Path)
|
|
if p.ID == "news" && p.Lang == g.newsLang {
|
|
address = gemNews
|
|
}
|
|
g.pageAddr[pageKey{p.ID, p.Lang}] = address
|
|
g.addr[p.Path] = address
|
|
}
|
|
for _, p := range g.posts {
|
|
g.addr[p.Path] = gemPostAddress(p)
|
|
}
|
|
for _, c := range g.categories {
|
|
g.addr[categoryPath(c.Slug)] = gemCategoryAddress(c.Slug)
|
|
}
|
|
for _, y := range groupArchive(g.posts) {
|
|
g.addr[g.yearPath(y.year)] = gemYearAddress(y.year)
|
|
for _, m := range y.months {
|
|
g.addr[g.monthPath(m.year, m.month)] = gemMonthAddress(m.year, m.month)
|
|
}
|
|
}
|
|
g.addr[g.newsPath+"feed.xml"] = gemNews + "feed.gmi"
|
|
}
|
|
|
|
// publish stores a file of the capsule; source names what produced it.
|
|
func (g *gemini) publish(rel, source, text string) error {
|
|
if other, taken := g.gemOwners[rel]; taken {
|
|
return fmt.Errorf("%s and %s are both published as %s", other, source, rel)
|
|
}
|
|
g.gemOwners[rel] = source
|
|
g.gemFiles[rel] = text
|
|
return nil
|
|
}
|
|
|
|
// gemFile returns the file of an address of the capsule: a folder gets index.gmi.
|
|
func gemFile(address string) string {
|
|
rel := strings.TrimPrefix(address, "/")
|
|
if rel == "" || strings.HasSuffix(rel, "/") {
|
|
rel += "index.gmi"
|
|
}
|
|
return rel
|
|
}
|
|
|
|
// scheme matches the start of an address that names a protocol.
|
|
var scheme = regexp.MustCompile(`^([A-Za-z][A-Za-z0-9+.-]*):`)
|
|
|
|
// externalSchemes are the protocols a link may use to leave the capsule.
|
|
var externalSchemes = map[string]bool{
|
|
"http": true, "https": true, "mailto": true, "gemini": true, "xmpp": true, "irc": true, "ircs": true, "matrix": true,
|
|
"bitcoin": true, "magnet": true, "tel": true, "sms": true,
|
|
}
|
|
|
|
// resolve turns an address found in the HTML of the page at base into the
|
|
// address to write in the capsule. An address with a protocol stays as it is
|
|
// when the protocol is a known one, and is an error otherwise; an address in the
|
|
// website is mapped to its page or file, and a file is copied by the end of the
|
|
// build. An address that starts with one of the paths of gemini_web_paths in
|
|
// site.json is one the website serves by other means: it becomes a link to the
|
|
// website. A fragment alone is dropped, and so is the fragment of an address: a
|
|
// gemini link cannot point inside a document.
|
|
func (g *gemini) resolve(href, base string) (string, error) {
|
|
if strings.HasPrefix(href, "#") {
|
|
return "", nil
|
|
}
|
|
if m := scheme.FindStringSubmatch(href); m != nil {
|
|
if !externalSchemes[strings.ToLower(m[1])] {
|
|
return "", fmt.Errorf("the protocol %q is not supported", m[1])
|
|
}
|
|
return href, nil
|
|
}
|
|
if strings.HasPrefix(href, "//") {
|
|
return "", errors.New("an address that starts with // is not supported")
|
|
}
|
|
target, _, _ := strings.Cut(href, "#")
|
|
if !strings.HasPrefix(target, "/") {
|
|
folder := path.Dir(base)
|
|
if strings.HasSuffix(base, "/") {
|
|
folder = strings.TrimSuffix(base, "/")
|
|
}
|
|
joined := path.Join("/", folder, target)
|
|
if strings.HasSuffix(target, "/") && joined != "/" {
|
|
joined += "/"
|
|
}
|
|
target = joined
|
|
}
|
|
if address, ok := g.addr[target]; ok {
|
|
return address, nil
|
|
}
|
|
name := strings.TrimPrefix(target, "/")
|
|
for asset := range g.cfg.Assets {
|
|
if path.Clean(asset) == name {
|
|
g.usedAssets[asset] = true
|
|
return target, nil
|
|
}
|
|
}
|
|
for _, prefix := range strings.Split(g.cfg.Strings["gemini_web_paths"], ",") {
|
|
if prefix = strings.TrimSpace(prefix); prefix != "" && strings.HasPrefix(target, prefix) {
|
|
return g.cfg.Strings["base_url"] + target, nil
|
|
}
|
|
}
|
|
return "", errors.New("no page or file of the site has this address (a path the website serves by other means goes in gemini_web_paths of site.json)")
|
|
}
|
|
|
|
// convert converts the HTML of a page or post to gemtext. base is the address
|
|
// of the page in the website, lang its language.
|
|
func (g *gemini) convert(name, html, base, lang string) (string, error) {
|
|
text, err := gemtext.Convert(html, gemtext.Options{
|
|
Link: func(href string) (string, error) { return g.resolve(href, base) },
|
|
Block: func(block string) (string, error) { return g.block(block, lang) },
|
|
})
|
|
if err != nil {
|
|
return "", fmt.Errorf("%s: %w", name, err)
|
|
}
|
|
return strings.TrimRight(text, "\n"), nil
|
|
}
|
|
|
|
// markdown replaces the placeholders of a source, converts it to HTML and puts
|
|
// the element <gemblock> where a generated list goes, for convert to expand.
|
|
func (g *gemini) markdown(name, source string, ctx map[string]string) (string, error) {
|
|
text, err := mdx.SubstituteMarkdown(name, source, ctx)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
html, _, err := mdx.Render(text)
|
|
if err != nil {
|
|
return "", fmt.Errorf("%s: %w", name, err)
|
|
}
|
|
html, err = replaceBlocks(name, html, func(block string) (string, error) {
|
|
return `<gemblock name="` + block + `"></gemblock>`, nil
|
|
})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return render.Unescape(html), nil
|
|
}
|
|
|
|
// assets reads the files some page points to.
|
|
func (g *gemini) assets() (map[string][]byte, error) {
|
|
names := make([]string, 0, len(g.usedAssets))
|
|
for name := range g.usedAssets {
|
|
names = append(names, name)
|
|
}
|
|
sort.Strings(names)
|
|
out := map[string][]byte{}
|
|
for _, name := range names {
|
|
source := g.cfg.Assets[name]
|
|
data, err := os.ReadFile(filepath.Join(g.opts.SiteDir, filepath.FromSlash(source)))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("asset %q: %w", name, err)
|
|
}
|
|
rel := path.Clean(name)
|
|
if other, taken := g.gemOwners[rel]; taken {
|
|
return nil, fmt.Errorf("the asset %q and %s are both published as %s", name, other, rel)
|
|
}
|
|
out[rel] = data
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// heading is the first level heading of a page.
|
|
func heading(title string) string { return "# " + strings.Join(strings.Fields(title), " ") }
|
|
|
|
// withoutTitle prepares a body converted to gemtext to go under the heading of
|
|
// the title. A leading heading that says the title is dropped, since the page
|
|
// has it already, and the other first level headings go one level down: a page
|
|
// has one.
|
|
func withoutTitle(title, body string) string {
|
|
lines := strings.Split(body, "\n")
|
|
if first, ok := strings.CutPrefix(lines[0], "# "); ok && (first == title || strings.HasSuffix(title, ": "+first)) {
|
|
lines = lines[1:]
|
|
}
|
|
fenced := false
|
|
for i, line := range lines {
|
|
switch {
|
|
case strings.HasPrefix(line, "```"):
|
|
fenced = !fenced
|
|
case !fenced && strings.HasPrefix(line, "# "):
|
|
lines[i] = "#" + line
|
|
}
|
|
}
|
|
return strings.Join(lines, "\n")
|
|
}
|
|
|
|
// navigation returns the links every page ends with: the home and the entries
|
|
// of the menu of the language, in the order its file lists them, then the
|
|
// other languages. An entry whose page the language lacks is left out.
|
|
func (g *gemini) navigation(d document) string {
|
|
l := g.langs[d.lang]
|
|
var menu []string
|
|
if home, ok := g.pageAddr[pageKey{"home", d.lang}]; ok {
|
|
label := l.Strings["back"]
|
|
if label == "" {
|
|
label = g.byKey[pageKey{"home", d.lang}].Title
|
|
}
|
|
menu = append(menu, gemtext.Link(home, label))
|
|
}
|
|
for _, key := range l.NavOrder {
|
|
address, ok := g.pageAddr[pageKey{key, d.lang}]
|
|
if !ok && key == "news" {
|
|
address, ok = gemNews, true
|
|
}
|
|
if ok {
|
|
menu = append(menu, gemtext.Link(address, l.Nav[key]))
|
|
}
|
|
}
|
|
parts := []string{strings.Join(menu, "\n")}
|
|
var others []string
|
|
for _, t := range g.languageTargets(d) {
|
|
if address, err := g.resolve(t.path, d.path); err == nil && address != "" {
|
|
others = append(others, gemtext.Link(address, g.langs[t.lang].Strings["lang_name"]))
|
|
}
|
|
}
|
|
if len(others) > 0 {
|
|
parts = append(parts, strings.Join(others, "\n"))
|
|
}
|
|
return strings.Join(parts, "\n\n")
|
|
}
|
|
|
|
// translationNote returns the line, and the link to report a problem, that a
|
|
// machine translated language puts at the top of its pages; empty for the others.
|
|
func (g *gemini) translationNote(lang string) string {
|
|
l := g.langs[lang]
|
|
note := l.Strings["translation_note"]
|
|
if note == "" {
|
|
return ""
|
|
}
|
|
label, issues := l.Strings["translation_note_a"], g.cfg.Strings["issues_url"]
|
|
if label == "" || issues == "" {
|
|
return gemtext.Text(note)
|
|
}
|
|
return gemtext.Text(note+" "+label+".") + "\n" + gemtext.Link(issues, label)
|
|
}
|
|
|
|
// compose joins the parts of a page with a blank line between them, leaving out
|
|
// the empty ones, and ends the text with a line break.
|
|
func compose(parts ...string) string {
|
|
var kept []string
|
|
for _, part := range parts {
|
|
if part = strings.Trim(part, "\n"); part != "" {
|
|
kept = append(kept, part)
|
|
}
|
|
}
|
|
return strings.Join(kept, "\n\n") + "\n"
|
|
}
|
|
|
|
// renderPage converts a page and writes it with the navigation.
|
|
func (g *gemini) renderPage(p content.Page) error {
|
|
d := document{
|
|
lang: p.Lang, id: p.ID, nav: p.ID, path: p.Path, title: p.Title,
|
|
description: p.Description, switches: p.Switch, source: p.Source,
|
|
}
|
|
ctx := g.documentContext(d)
|
|
version, date := g.cfg.Strings["manifesto_version"], g.cfg.Strings["manifesto_date"]
|
|
if p.ManifestoVersion != "" {
|
|
version = p.ManifestoVersion
|
|
}
|
|
if p.ManifestoDate != "" {
|
|
date = p.ManifestoDate
|
|
}
|
|
ctx["manifesto_version"], ctx["manifesto_date"] = render.Escape(version), render.Escape(date)
|
|
|
|
source := p.Body
|
|
if p.Layout == "manifesto" {
|
|
var err error
|
|
if source, err = mdx.NoEmDash(source); err != nil {
|
|
return fmt.Errorf("%s: %w", p.Source, err)
|
|
}
|
|
}
|
|
html, err := g.markdown(p.Source, source, ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
body, err := g.convert(p.Source, html, p.Path, p.Lang)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var intro []string
|
|
if p.Layout == "manifesto" {
|
|
intro = []string{g.manifestoLine(p.Lang, version, date), gemtext.Text(p.Note)}
|
|
}
|
|
parts := append([]string{heading(p.Title), g.translationNote(p.Lang)}, intro...)
|
|
parts = append(parts, withoutTitle(p.Title, body), g.navigation(d))
|
|
return g.publish(gemFile(g.pageAddr[pageKey{p.ID, p.Lang}]), p.Source, compose(parts...))
|
|
}
|
|
|
|
// manifestoLine is the line that says the version and the date of the manifesto.
|
|
func (g *gemini) manifestoLine(lang, version, date string) string {
|
|
line := strings.TrimSpace(g.langs[lang].Strings["version"] + " " + version)
|
|
if date != "" {
|
|
line += ", " + date
|
|
}
|
|
return gemtext.Text(line)
|
|
}
|