motorino/internal/gemtext/gemtext.go
tgiachi 28745d452f feat(gemtext): convert the HTML of a page to gemtext
A new package, with no new dependency, that reads the HTML a page renders to (the
lenient mode of encoding/xml with the HTML void elements and entities) and writes
gemtext: three heading levels, one line paragraphs, the links of a paragraph, a list,
a quote or a table on lines of their own after it, bullets (a list of only links is
link lines), quotes (a note is a quote), preformatted blocks, tables as aligned
preformatted blocks, images as link lines, no emphasis, and lines that look like
gemtext are protected. Options.Link maps or drops an address and its error stops
the conversion; Options.Block expands the placeholders of generated blocks.

The language files also keep the order of the menu now (Lang.NavOrder), which the
Gemini output needs.

Refs #10
2026-11-16 12:19:38 +01:00

637 lines
17 KiB
Go

// SPDX-License-Identifier: GPL-3.0-or-later
// Package gemtext turns the HTML of a page into gemtext, the format of the
// Gemini protocol. Gemtext has headings, one-line paragraphs, bullet items,
// quotes, preformatted blocks and links on lines of their own, and nothing
// else: no emphasis, no links inside text, no tables, no images.
package gemtext
import (
"encoding/xml"
"fmt"
"io"
"regexp"
"strconv"
"strings"
"unicode/utf8"
)
// Options tell Convert how to treat the addresses and the generated blocks.
type Options struct {
// Link turns an address found in the HTML (a link or an image) into the
// address to write. An empty result drops the link and keeps its text, as
// for "#section"; an error stops the conversion and names the address. A
// nil Link leaves every address as it is.
Link func(href string) (string, error)
// Block returns the gemtext of a block placeholder, the element
// <gemblock name="...">. A nil Block makes such an element an error.
Block func(name string) (string, error)
}
// node is an element or a piece of text of the parsed HTML.
type node struct {
tag string
text string
attrs map[string]string
kids []*node
}
// link is an address with the text that labels it.
type link struct{ target, label string }
// block is a part of the output: lines of text followed by link lines.
type block struct {
text []string
links []link
}
// converter holds the options of one conversion.
type converter struct{ opts Options }
// Convert converts html, the body of a page, to gemtext. Blocks are separated
// by a blank line; the links of a paragraph, a list, a quote or a table come
// right after it, each on a line of its own, and the output ends with a line
// break unless it is empty.
func Convert(html string, opts Options) (string, error) {
root, err := parse(html)
if err != nil {
return "", err
}
c := &converter{opts: opts}
blocks, err := c.blocks(root.kids)
if err != nil {
return "", err
}
parts := make([]string, 0, len(blocks))
for _, b := range blocks {
if text := b.String(); text != "" {
parts = append(parts, text)
}
}
if len(parts) == 0 {
return "", nil
}
return strings.Join(parts, "\n\n") + "\n", nil
}
// String returns the lines of the block, text first, then the link lines.
func (b block) String() string {
lines := append([]string(nil), b.text...)
seen := map[link]bool{}
for _, l := range b.links {
if seen[l] {
continue
}
seen[l] = true
lines = append(lines, linkLine(l))
}
return strings.Join(lines, "\n")
}
// linkLine writes a link on its own line.
func linkLine(l link) string {
if l.label == "" || l.label == l.target {
return "=> " + l.target
}
return "=> " + l.target + " " + l.label
}
// voidTags are the elements that have no end tag.
var voidTags = map[string]bool{
"area": true, "base": true, "br": true, "col": true, "hr": true, "img": true, "input": true,
"link": true, "meta": true, "param": true, "source": true, "basefont": true, "frame": true, "isindex": true,
}
// parse reads a fragment of HTML into a tree. The decoder is the lenient mode
// of encoding/xml with the HTML void elements and entities: the HTML of a page
// comes from a Markdown renderer and from the author's own raw HTML, which are
// well formed apart from the void elements.
func parse(html string) (*node, error) {
source := "<root>" + html + "</root>"
decoder := xml.NewDecoder(strings.NewReader(source))
decoder.Strict = false
decoder.AutoClose = xml.HTMLAutoClose
decoder.Entity = xml.HTMLEntity
root := &node{tag: "root"}
stack := []*node{root}
for {
token, err := decoder.Token()
if err == io.EOF {
break
}
if err != nil {
return nil, fmt.Errorf("read the HTML: %w", err)
}
top := stack[len(stack)-1]
switch t := token.(type) {
case xml.StartElement:
n := &node{tag: strings.ToLower(t.Name.Local), attrs: map[string]string{}}
for _, a := range t.Attr {
n.attrs[strings.ToLower(a.Name.Local)] = a.Value
}
top.kids = append(top.kids, n)
stack = append(stack, n)
case xml.EndElement:
// The lenient mode closes by itself what the text left open; that is
// a mistake in the page, so an end that the text does not have is an error.
name := strings.ToLower(t.Name.Local)
written := strings.ToLower(source[:decoder.InputOffset()])
if !voidTags[name] && !strings.HasSuffix(written, "</"+name+">") && !strings.HasSuffix(written, "/>") {
return nil, fmt.Errorf("read the HTML: the element <%s> is not closed", name)
}
if len(stack) > 1 {
stack = stack[:len(stack)-1]
}
case xml.CharData:
top.kids = append(top.kids, &node{text: string(t)})
}
}
if len(stack) != 1 || len(root.kids) != 1 {
return nil, fmt.Errorf("read the HTML: the element <%s> is not closed", stack[len(stack)-1].tag)
}
return root.kids[0], nil
}
// blockTags are the elements that start a block of their own.
var blockTags = map[string]bool{
"h1": true, "h2": true, "h3": true, "h4": true, "h5": true, "h6": true, "p": true,
"ul": true, "ol": true, "blockquote": true, "pre": true, "hr": true, "table": true,
"div": true, "section": true, "article": true, "main": true, "header": true, "footer": true,
"nav": true, "aside": true, "figure": true, "figcaption": true, "details": true, "summary": true,
"dl": true, "dt": true, "dd": true, "form": true, "fieldset": true, "address": true, "gemblock": true,
}
// skippedTags are the elements that have nothing to read.
var skippedTags = map[string]bool{
"script": true, "style": true, "svg": true, "noscript": true, "template": true,
"head": true, "title": true, "meta": true, "link": true,
}
// skipped reports whether an element is left out: one with nothing to read, or
// one that is hidden.
func skipped(n *node) bool {
_, hidden := n.attrs["hidden"]
return skippedTags[n.tag] || hidden || n.attrs["aria-hidden"] == "true"
}
// blocks converts a list of nodes into blocks: block elements make their own,
// and the text and the inline elements between them make paragraphs.
func (c *converter) blocks(nodes []*node) ([]block, error) {
var out []block
var run []*node
flush := func() error {
if len(run) == 0 {
return nil
}
b, err := c.paragraph(run)
run = nil
if err != nil {
return err
}
out = append(out, b)
return nil
}
for _, n := range nodes {
if n.tag != "" && skipped(n) {
continue
}
if n.tag == "" || !blockTags[n.tag] {
run = append(run, n)
continue
}
if err := flush(); err != nil {
return nil, err
}
got, err := c.element(n)
if err != nil {
return nil, err
}
out = append(out, got...)
}
if err := flush(); err != nil {
return nil, err
}
return out, nil
}
// element converts one block element.
func (c *converter) element(n *node) ([]block, error) {
switch n.tag {
case "h1", "h2", "h3", "h4", "h5", "h6":
return c.heading(n)
case "p", "summary", "figcaption", "dt", "dd":
if strings.Contains(" "+n.attrs["class"]+" ", " note ") || n.attrs["role"] == "note" {
return c.quote(n.kids)
}
b, err := c.paragraph(n.kids)
return []block{b}, err
case "ul", "ol":
b, err := c.list(n)
return []block{b}, err
case "blockquote":
return c.quote(n.kids)
case "pre":
return []block{preformatted(n)}, nil
case "hr":
return nil, nil
case "table":
b, err := c.table(n)
return []block{b}, err
case "gemblock":
return c.generated(n)
default:
return c.blocks(n.kids)
}
}
// generated asks Options.Block for the gemtext of a block placeholder.
func (c *converter) generated(n *node) ([]block, error) {
name := n.attrs["name"]
if c.opts.Block == nil {
return nil, fmt.Errorf("the block %q has no generator", name)
}
text, err := c.opts.Block(name)
if err != nil {
return nil, fmt.Errorf("block %q: %w", name, err)
}
text = strings.Trim(text, "\n")
if text == "" {
return nil, nil
}
return []block{{text: strings.Split(text, "\n")}}, nil
}
// heading converts h1 to h6; gemtext has three levels.
func (c *converter) heading(n *node) ([]block, error) {
text, links, err := c.inline(n.kids, " ")
if err != nil {
return nil, err
}
text = strings.Join(strings.Fields(text), " ")
if text == "" {
return nil, nil
}
level := int(n.tag[1] - '0')
if level > 3 {
level = 3
}
return []block{{text: []string{strings.Repeat("#", level) + " " + text}, links: links}}, nil
}
// paragraph converts a run of text and inline elements. Each line break is a
// line of its own, and the links come after. A paragraph that is only a link
// is only the link line.
func (c *converter) paragraph(nodes []*node) (block, error) {
text, links, err := c.inline(nodes, "\n")
if err != nil {
return block{}, err
}
lines := cleanLines(text)
if len(lines) == 1 && len(links) == 1 && lines[0] == links[0].label {
return block{links: links}, nil
}
for i := range lines {
lines[i] = protect(lines[i])
}
return block{text: lines, links: links}, nil
}
// cleanLines splits text at its line breaks, collapses the white space of every
// line and drops the empty ones.
func cleanLines(text string) []string {
var lines []string
for _, line := range strings.Split(text, "\n") {
if line = strings.Join(strings.Fields(line), " "); line != "" {
lines = append(lines, line)
}
}
return lines
}
// protect keeps a line of text from being read as another kind of line: a
// leading space does not change what it says and gemtext has no escape.
func protect(line string) string {
for _, prefix := range []string{"=>", "#", ">", "```", "* "} {
if strings.HasPrefix(line, prefix) {
return " " + line
}
}
return line
}
// space matches a run of white space, the no-break space included.
var space = regexp.MustCompile(`[\s\x{00a0}]+`)
// inline returns the text of the nodes, the links in them and the addresses of
// their images. lineBreak is what a <br> becomes.
func (c *converter) inline(nodes []*node, lineBreak string) (string, []link, error) {
var text strings.Builder
var links []link
var walk func(n *node) error
walk = func(n *node) error {
if n.tag == "" {
text.WriteString(space.ReplaceAllString(n.text, " "))
return nil
}
if skipped(n) {
return nil
}
switch n.tag {
case "br":
text.WriteString(lineBreak)
return nil
case "img":
alt := strings.Join(strings.Fields(n.attrs["alt"]), " ")
if alt == "" || n.attrs["src"] == "" {
return nil
}
target, err := c.address(n.attrs["src"])
if err != nil {
return err
}
if target != "" {
links = append(links, link{target, alt})
}
return nil
case "a":
label, inner, err := c.inline(n.kids, " ")
if err != nil {
return err
}
text.WriteString(label)
links = append(links, inner...)
if href := n.attrs["href"]; href != "" {
target, err := c.address(href)
if err != nil {
return err
}
if target != "" {
links = append(links, link{target, strings.Join(strings.Fields(label), " ")})
}
}
return nil
}
for _, kid := range n.kids {
if err := walk(kid); err != nil {
return err
}
}
return nil
}
for _, n := range nodes {
if err := walk(n); err != nil {
return "", nil, err
}
}
return text.String(), links, nil
}
// address maps an address through Options.Link.
func (c *converter) address(href string) (string, error) {
if c.opts.Link == nil {
return href, nil
}
target, err := c.opts.Link(href)
if err != nil {
return "", fmt.Errorf("the address %q: %w", href, err)
}
return target, nil
}
// item is one bullet of a list.
type item struct {
text string
links []link
}
// list converts a list: bullets, with the links after them. A list in which
// every item is only a link is a list of link lines. A list inside an item is
// flattened into the same list. An ordered list numbers its items.
func (c *converter) list(n *node) (block, error) {
var items []item
if err := c.items(n, &items); err != nil {
return block{}, err
}
linksOnly := len(items) > 0
var links []link
for _, it := range items {
links = append(links, it.links...)
if len(it.links) != 1 || it.text != it.links[0].label {
linksOnly = false
}
}
if linksOnly {
return block{links: links}, nil
}
lines := make([]string, len(items))
for i, it := range items {
prefix := "* "
if n.tag == "ol" {
prefix += strconv.Itoa(i+1) + ". "
}
lines[i] = prefix + it.text
}
return block{text: lines, links: links}, nil
}
// items collects the items of a list and of the lists inside it.
func (c *converter) items(list *node, out *[]item) error {
for _, li := range list.kids {
if li.tag != "li" || skipped(li) {
continue
}
var own, nested []*node
for _, kid := range li.kids {
if kid.tag == "ul" || kid.tag == "ol" {
nested = append(nested, kid)
} else {
own = append(own, kid)
}
}
text, links, err := c.inline(own, " ")
if err != nil {
return err
}
if text = strings.Join(strings.Fields(text), " "); text != "" || len(links) > 0 {
*out = append(*out, item{text: text, links: links})
}
for _, inner := range nested {
if err := c.items(inner, out); err != nil {
return err
}
}
}
return nil
}
// quote converts the content of a quote: every line starts with "> ", the
// blocks are separated by an empty quote line, and the links come after.
func (c *converter) quote(nodes []*node) ([]block, error) {
inner, err := c.blocks(nodes)
if err != nil {
return nil, err
}
var lines []string
var links []link
for _, b := range inner {
if len(b.text) == 0 {
links = append(links, b.links...)
continue
}
if len(lines) > 0 {
lines = append(lines, ">")
}
for _, line := range b.text {
lines = append(lines, "> "+strings.TrimPrefix(line, " "))
}
links = append(links, b.links...)
}
if len(lines) == 0 && len(links) == 0 {
return nil, nil
}
return []block{{text: lines, links: links}}, nil
}
// preformatted converts a <pre>: its text as it is between two fence lines. A
// line that would close the block early gets a space in front.
func preformatted(n *node) block {
var text strings.Builder
var walk func(n *node)
walk = func(n *node) {
if n.tag == "" {
text.WriteString(n.text)
return
}
if n.tag == "br" {
text.WriteString("\n")
return
}
for _, kid := range n.kids {
walk(kid)
}
}
walk(n)
lines := strings.Split(strings.TrimSuffix(text.String(), "\n"), "\n")
out := []string{"```"}
for _, line := range lines {
if strings.HasPrefix(line, "```") {
line = " " + line
}
out = append(out, strings.TrimRight(line, " \t"))
}
return block{text: append(out, "```")}
}
// table converts a table to a preformatted block of aligned columns, with a
// line under a header row, and the caption as a line of text before it. The
// links of the cells come after the block.
func (c *converter) table(n *node) (block, error) {
var rows [][]string
var links []link
var caption string
headerRow := false
var walk func(n *node) error
walk = func(n *node) error {
for _, kid := range n.kids {
if kid.tag == "" || skipped(kid) {
continue
}
switch kid.tag {
case "caption":
text, inner, err := c.inline(kid.kids, " ")
if err != nil {
return err
}
caption = strings.Join(strings.Fields(text), " ")
links = append(links, inner...)
case "tr":
var cells []string
onlyHeaders := true
for _, cell := range kid.kids {
if cell.tag != "td" && cell.tag != "th" {
continue
}
text, inner, err := c.inline(cell.kids, " ")
if err != nil {
return err
}
cells = append(cells, strings.Join(strings.Fields(text), " "))
links = append(links, inner...)
if cell.tag != "th" {
onlyHeaders = false
}
}
if len(cells) > 0 {
if len(rows) == 0 && onlyHeaders {
headerRow = true
}
rows = append(rows, cells)
}
default:
if err := walk(kid); err != nil {
return err
}
}
}
return nil
}
if err := walk(n); err != nil {
return block{}, err
}
var text []string
if caption != "" {
text = append(text, protect(caption))
}
if len(rows) > 0 {
text = append(text, "```")
text = append(text, alignRows(rows, headerRow)...)
text = append(text, "```")
}
return block{text: text, links: links}, nil
}
// alignRows writes rows of cells as columns two spaces apart, each as wide as
// its widest cell, with a line of dashes under the first row when it is a header.
func alignRows(rows [][]string, header bool) []string {
columns := 0
for _, r := range rows {
if len(r) > columns {
columns = len(r)
}
}
widths := make([]int, columns)
for _, r := range rows {
for i, cell := range r {
if w := utf8.RuneCountInString(cell); w > widths[i] {
widths[i] = w
}
}
}
write := func(r []string) string {
cells := make([]string, columns)
for i := range cells {
cell := ""
if i < len(r) {
cell = r[i]
}
cells[i] = cell + strings.Repeat(" ", widths[i]-utf8.RuneCountInString(cell))
}
line := strings.TrimRight(strings.Join(cells, " "), " ")
if strings.HasPrefix(line, "```") {
line = " " + line
}
return line
}
var lines []string
for i, r := range rows {
lines = append(lines, write(r))
if i == 0 && header {
dashes := make([]string, columns)
for j := range dashes {
dashes[j] = strings.Repeat("-", widths[j])
}
lines = append(lines, strings.Join(dashes, " "))
}
}
return lines
}