motorino/internal/gemtext/gemtext.go
tgiachi 2f169b8d9f feat(gemini): write the site as a Gemini capsule
`motorino gemini [-site DIR] [-out DIR] [-lang CODE] [-drafts]` turns the pages and the
news into gemtext with the new gemtext package.

- the language served by default sits at the root (-lang, then gemini_language in
  site.json, then the default language) and the others under /<code>/
- the news exist once, under /news/: index, posts, categories, archives and a
  gemlog style feed (news/feed.gmi, a date before each title)
- every page ends with the navigation of its language, the language links and, on
  translations, the machine translation note
- .meta for Agate (--central-conf): two lines per language other than the root one
  and two for the news when they are in another language, never linked
- addresses are mapped to the capsule, a fragment is dropped, a missing target stops
  the build, assets are copied only when a page points to them, gemini_web_paths
  keeps paths served by other means as links to the website
- blocks of generated lists become gemtext with the strings of the language file
- the block replacement of a page is shared with the HTML build

Refs #10
2026-11-16 12:28:10 +01:00

646 lines
17 KiB
Go

// SPDX-License-Identifier: GPL-3.0-or-later
// Package gemtext turns the HTML of a page into gemtext, the format of the
// Gemini protocol. Gemtext has headings, one-line paragraphs, bullet items,
// quotes, preformatted blocks and links on lines of their own, and nothing
// else: no emphasis, no links inside text, no tables, no images.
package gemtext
import (
"encoding/xml"
"fmt"
"io"
"regexp"
"strconv"
"strings"
"unicode/utf8"
)
// Options tell Convert how to treat the addresses and the generated blocks.
type Options struct {
// Link turns an address found in the HTML (a link or an image) into the
// address to write. An empty result drops the link and keeps its text, as
// for "#section"; an error stops the conversion and names the address. A
// nil Link leaves every address as it is.
Link func(href string) (string, error)
// Block returns the gemtext of a block placeholder, the element
// <gemblock name="...">. A nil Block makes such an element an error.
Block func(name string) (string, error)
}
// node is an element or a piece of text of the parsed HTML.
type node struct {
tag string
text string
attrs map[string]string
kids []*node
}
// link is an address with the text that labels it.
type link struct{ target, label string }
// block is a part of the output: lines of text followed by link lines.
type block struct {
text []string
links []link
}
// converter holds the options of one conversion.
type converter struct{ opts Options }
// Convert converts html, the body of a page, to gemtext. Blocks are separated
// by a blank line; the links of a paragraph, a list, a quote or a table come
// right after it, each on a line of its own, and the output ends with a line
// break unless it is empty.
func Convert(html string, opts Options) (string, error) {
root, err := parse(html)
if err != nil {
return "", err
}
c := &converter{opts: opts}
blocks, err := c.blocks(root.kids)
if err != nil {
return "", err
}
parts := make([]string, 0, len(blocks))
for _, b := range blocks {
if text := b.String(); text != "" {
parts = append(parts, text)
}
}
if len(parts) == 0 {
return "", nil
}
return strings.Join(parts, "\n\n") + "\n", nil
}
// String returns the lines of the block, text first, then the link lines.
func (b block) String() string {
lines := append([]string(nil), b.text...)
seen := map[link]bool{}
for _, l := range b.links {
if seen[l] {
continue
}
seen[l] = true
lines = append(lines, linkLine(l))
}
return strings.Join(lines, "\n")
}
// Text returns plain text as one line of gemtext: white space collapsed, and a
// leading space when the text would be read as another kind of line.
func Text(s string) string { return protect(strings.Join(strings.Fields(s), " ")) }
// Link returns the link line of an address with a label.
func Link(target, label string) string {
return linkLine(link{target, strings.Join(strings.Fields(label), " ")})
}
// linkLine writes a link on its own line.
func linkLine(l link) string {
if l.label == "" || l.label == l.target {
return "=> " + l.target
}
return "=> " + l.target + " " + l.label
}
// voidTags are the elements that have no end tag.
var voidTags = map[string]bool{
"area": true, "base": true, "br": true, "col": true, "hr": true, "img": true, "input": true,
"link": true, "meta": true, "param": true, "source": true, "basefont": true, "frame": true, "isindex": true,
}
// parse reads a fragment of HTML into a tree. The decoder is the lenient mode
// of encoding/xml with the HTML void elements and entities: the HTML of a page
// comes from a Markdown renderer and from the author's own raw HTML, which are
// well formed apart from the void elements.
func parse(html string) (*node, error) {
source := "<root>" + html + "</root>"
decoder := xml.NewDecoder(strings.NewReader(source))
decoder.Strict = false
decoder.AutoClose = xml.HTMLAutoClose
decoder.Entity = xml.HTMLEntity
root := &node{tag: "root"}
stack := []*node{root}
for {
token, err := decoder.Token()
if err == io.EOF {
break
}
if err != nil {
return nil, fmt.Errorf("read the HTML: %w", err)
}
top := stack[len(stack)-1]
switch t := token.(type) {
case xml.StartElement:
n := &node{tag: strings.ToLower(t.Name.Local), attrs: map[string]string{}}
for _, a := range t.Attr {
n.attrs[strings.ToLower(a.Name.Local)] = a.Value
}
top.kids = append(top.kids, n)
stack = append(stack, n)
case xml.EndElement:
// The lenient mode closes by itself what the text left open; that is
// a mistake in the page, so an end that the text does not have is an error.
name := strings.ToLower(t.Name.Local)
written := strings.ToLower(source[:decoder.InputOffset()])
if !voidTags[name] && !strings.HasSuffix(written, "</"+name+">") && !strings.HasSuffix(written, "/>") {
return nil, fmt.Errorf("read the HTML: the element <%s> is not closed", name)
}
if len(stack) > 1 {
stack = stack[:len(stack)-1]
}
case xml.CharData:
top.kids = append(top.kids, &node{text: string(t)})
}
}
if len(stack) != 1 || len(root.kids) != 1 {
return nil, fmt.Errorf("read the HTML: the element <%s> is not closed", stack[len(stack)-1].tag)
}
return root.kids[0], nil
}
// blockTags are the elements that start a block of their own.
var blockTags = map[string]bool{
"h1": true, "h2": true, "h3": true, "h4": true, "h5": true, "h6": true, "p": true,
"ul": true, "ol": true, "blockquote": true, "pre": true, "hr": true, "table": true,
"div": true, "section": true, "article": true, "main": true, "header": true, "footer": true,
"nav": true, "aside": true, "figure": true, "figcaption": true, "details": true, "summary": true,
"dl": true, "dt": true, "dd": true, "form": true, "fieldset": true, "address": true, "gemblock": true,
}
// skippedTags are the elements that have nothing to read.
var skippedTags = map[string]bool{
"script": true, "style": true, "svg": true, "noscript": true, "template": true,
"head": true, "title": true, "meta": true, "link": true,
}
// skipped reports whether an element is left out: one with nothing to read, or
// one that is hidden.
func skipped(n *node) bool {
_, hidden := n.attrs["hidden"]
return skippedTags[n.tag] || hidden || n.attrs["aria-hidden"] == "true"
}
// blocks converts a list of nodes into blocks: block elements make their own,
// and the text and the inline elements between them make paragraphs.
func (c *converter) blocks(nodes []*node) ([]block, error) {
var out []block
var run []*node
flush := func() error {
if len(run) == 0 {
return nil
}
b, err := c.paragraph(run)
run = nil
if err != nil {
return err
}
out = append(out, b)
return nil
}
for _, n := range nodes {
if n.tag != "" && skipped(n) {
continue
}
if n.tag == "" || !blockTags[n.tag] {
run = append(run, n)
continue
}
if err := flush(); err != nil {
return nil, err
}
got, err := c.element(n)
if err != nil {
return nil, err
}
out = append(out, got...)
}
if err := flush(); err != nil {
return nil, err
}
return out, nil
}
// element converts one block element.
func (c *converter) element(n *node) ([]block, error) {
switch n.tag {
case "h1", "h2", "h3", "h4", "h5", "h6":
return c.heading(n)
case "p", "summary", "figcaption", "dt", "dd":
if strings.Contains(" "+n.attrs["class"]+" ", " note ") || n.attrs["role"] == "note" {
return c.quote(n.kids)
}
b, err := c.paragraph(n.kids)
return []block{b}, err
case "ul", "ol":
b, err := c.list(n)
return []block{b}, err
case "blockquote":
return c.quote(n.kids)
case "pre":
return []block{preformatted(n)}, nil
case "hr":
return nil, nil
case "table":
b, err := c.table(n)
return []block{b}, err
case "gemblock":
return c.generated(n)
default:
return c.blocks(n.kids)
}
}
// generated asks Options.Block for the gemtext of a block placeholder.
func (c *converter) generated(n *node) ([]block, error) {
name := n.attrs["name"]
if c.opts.Block == nil {
return nil, fmt.Errorf("the block %q has no generator", name)
}
text, err := c.opts.Block(name)
if err != nil {
return nil, fmt.Errorf("block %q: %w", name, err)
}
text = strings.Trim(text, "\n")
if text == "" {
return nil, nil
}
return []block{{text: strings.Split(text, "\n")}}, nil
}
// heading converts h1 to h6; gemtext has three levels.
func (c *converter) heading(n *node) ([]block, error) {
text, links, err := c.inline(n.kids, " ")
if err != nil {
return nil, err
}
text = strings.Join(strings.Fields(text), " ")
if text == "" {
return nil, nil
}
level := int(n.tag[1] - '0')
if level > 3 {
level = 3
}
return []block{{text: []string{strings.Repeat("#", level) + " " + text}, links: links}}, nil
}
// paragraph converts a run of text and inline elements. Each line break is a
// line of its own, and the links come after. A paragraph that is only a link
// is only the link line.
func (c *converter) paragraph(nodes []*node) (block, error) {
text, links, err := c.inline(nodes, "\n")
if err != nil {
return block{}, err
}
lines := cleanLines(text)
if len(lines) == 1 && len(links) == 1 && lines[0] == links[0].label {
return block{links: links}, nil
}
for i := range lines {
lines[i] = protect(lines[i])
}
return block{text: lines, links: links}, nil
}
// cleanLines splits text at its line breaks, collapses the white space of every
// line and drops the empty ones.
func cleanLines(text string) []string {
var lines []string
for _, line := range strings.Split(text, "\n") {
if line = strings.Join(strings.Fields(line), " "); line != "" {
lines = append(lines, line)
}
}
return lines
}
// protect keeps a line of text from being read as another kind of line: a
// leading space does not change what it says and gemtext has no escape.
func protect(line string) string {
for _, prefix := range []string{"=>", "#", ">", "```", "* "} {
if strings.HasPrefix(line, prefix) {
return " " + line
}
}
return line
}
// space matches a run of white space, the no-break space included.
var space = regexp.MustCompile(`[\s\x{00a0}]+`)
// inline returns the text of the nodes, the links in them and the addresses of
// their images. lineBreak is what a <br> becomes.
func (c *converter) inline(nodes []*node, lineBreak string) (string, []link, error) {
var text strings.Builder
var links []link
var walk func(n *node) error
walk = func(n *node) error {
if n.tag == "" {
text.WriteString(space.ReplaceAllString(n.text, " "))
return nil
}
if skipped(n) {
return nil
}
switch n.tag {
case "br":
text.WriteString(lineBreak)
return nil
case "img":
alt := strings.Join(strings.Fields(n.attrs["alt"]), " ")
if alt == "" || n.attrs["src"] == "" {
return nil
}
target, err := c.address(n.attrs["src"])
if err != nil {
return err
}
if target != "" {
links = append(links, link{target, alt})
}
return nil
case "a":
label, inner, err := c.inline(n.kids, " ")
if err != nil {
return err
}
text.WriteString(label)
links = append(links, inner...)
if href := n.attrs["href"]; href != "" {
target, err := c.address(href)
if err != nil {
return err
}
if target != "" {
links = append(links, link{target, strings.Join(strings.Fields(label), " ")})
}
}
return nil
}
for _, kid := range n.kids {
if err := walk(kid); err != nil {
return err
}
}
return nil
}
for _, n := range nodes {
if err := walk(n); err != nil {
return "", nil, err
}
}
return text.String(), links, nil
}
// address maps an address through Options.Link.
func (c *converter) address(href string) (string, error) {
if c.opts.Link == nil {
return href, nil
}
target, err := c.opts.Link(href)
if err != nil {
return "", fmt.Errorf("the address %q: %w", href, err)
}
return target, nil
}
// item is one bullet of a list.
type item struct {
text string
links []link
}
// list converts a list: bullets, with the links after them. A list in which
// every item is only a link is a list of link lines. A list inside an item is
// flattened into the same list. An ordered list numbers its items.
func (c *converter) list(n *node) (block, error) {
var items []item
if err := c.items(n, &items); err != nil {
return block{}, err
}
linksOnly := len(items) > 0
var links []link
for _, it := range items {
links = append(links, it.links...)
if len(it.links) != 1 || it.text != it.links[0].label {
linksOnly = false
}
}
if linksOnly {
return block{links: links}, nil
}
lines := make([]string, len(items))
for i, it := range items {
prefix := "* "
if n.tag == "ol" {
prefix += strconv.Itoa(i+1) + ". "
}
lines[i] = prefix + it.text
}
return block{text: lines, links: links}, nil
}
// items collects the items of a list and of the lists inside it.
func (c *converter) items(list *node, out *[]item) error {
for _, li := range list.kids {
if li.tag != "li" || skipped(li) {
continue
}
var own, nested []*node
for _, kid := range li.kids {
if kid.tag == "ul" || kid.tag == "ol" {
nested = append(nested, kid)
} else {
own = append(own, kid)
}
}
text, links, err := c.inline(own, " ")
if err != nil {
return err
}
if text = strings.Join(strings.Fields(text), " "); text != "" || len(links) > 0 {
*out = append(*out, item{text: text, links: links})
}
for _, inner := range nested {
if err := c.items(inner, out); err != nil {
return err
}
}
}
return nil
}
// quote converts the content of a quote: every line starts with "> ", the
// blocks are separated by an empty quote line, and the links come after.
func (c *converter) quote(nodes []*node) ([]block, error) {
inner, err := c.blocks(nodes)
if err != nil {
return nil, err
}
var lines []string
var links []link
for _, b := range inner {
if len(b.text) == 0 {
links = append(links, b.links...)
continue
}
if len(lines) > 0 {
lines = append(lines, ">")
}
for _, line := range b.text {
lines = append(lines, "> "+strings.TrimPrefix(line, " "))
}
links = append(links, b.links...)
}
if len(lines) == 0 && len(links) == 0 {
return nil, nil
}
return []block{{text: lines, links: links}}, nil
}
// preformatted converts a <pre>: its text as it is between two fence lines. A
// line that would close the block early gets a space in front.
func preformatted(n *node) block {
var text strings.Builder
var walk func(n *node)
walk = func(n *node) {
if n.tag == "" {
text.WriteString(n.text)
return
}
if n.tag == "br" {
text.WriteString("\n")
return
}
for _, kid := range n.kids {
walk(kid)
}
}
walk(n)
lines := strings.Split(strings.TrimSuffix(text.String(), "\n"), "\n")
out := []string{"```"}
for _, line := range lines {
if strings.HasPrefix(line, "```") {
line = " " + line
}
out = append(out, strings.TrimRight(line, " \t"))
}
return block{text: append(out, "```")}
}
// table converts a table to a preformatted block of aligned columns, with a
// line under a header row, and the caption as a line of text before it. The
// links of the cells come after the block.
func (c *converter) table(n *node) (block, error) {
var rows [][]string
var links []link
var caption string
headerRow := false
var walk func(n *node) error
walk = func(n *node) error {
for _, kid := range n.kids {
if kid.tag == "" || skipped(kid) {
continue
}
switch kid.tag {
case "caption":
text, inner, err := c.inline(kid.kids, " ")
if err != nil {
return err
}
caption = strings.Join(strings.Fields(text), " ")
links = append(links, inner...)
case "tr":
var cells []string
onlyHeaders := true
for _, cell := range kid.kids {
if cell.tag != "td" && cell.tag != "th" {
continue
}
text, inner, err := c.inline(cell.kids, " ")
if err != nil {
return err
}
cells = append(cells, strings.Join(strings.Fields(text), " "))
links = append(links, inner...)
if cell.tag != "th" {
onlyHeaders = false
}
}
if len(cells) > 0 {
if len(rows) == 0 && onlyHeaders {
headerRow = true
}
rows = append(rows, cells)
}
default:
if err := walk(kid); err != nil {
return err
}
}
}
return nil
}
if err := walk(n); err != nil {
return block{}, err
}
var text []string
if caption != "" {
text = append(text, protect(caption))
}
if len(rows) > 0 {
text = append(text, "```")
text = append(text, alignRows(rows, headerRow)...)
text = append(text, "```")
}
return block{text: text, links: links}, nil
}
// alignRows writes rows of cells as columns two spaces apart, each as wide as
// its widest cell, with a line of dashes under the first row when it is a header.
func alignRows(rows [][]string, header bool) []string {
columns := 0
for _, r := range rows {
if len(r) > columns {
columns = len(r)
}
}
widths := make([]int, columns)
for _, r := range rows {
for i, cell := range r {
if w := utf8.RuneCountInString(cell); w > widths[i] {
widths[i] = w
}
}
}
write := func(r []string) string {
cells := make([]string, columns)
for i := range cells {
cell := ""
if i < len(r) {
cell = r[i]
}
cells[i] = cell + strings.Repeat(" ", widths[i]-utf8.RuneCountInString(cell))
}
line := strings.TrimRight(strings.Join(cells, " "), " ")
if strings.HasPrefix(line, "```") {
line = " " + line
}
return line
}
var lines []string
for i, r := range rows {
lines = append(lines, write(r))
if i == 0 && header {
dashes := make([]string, columns)
for j := range dashes {
dashes[j] = strings.Repeat("-", widths[j])
}
lines = append(lines, strings.Join(dashes, " "))
}
}
return lines
}