// SPDX-License-Identifier: GPL-3.0-or-later // Package gemtext turns the HTML of a page into gemtext, the format of the // Gemini protocol. Gemtext has headings, one-line paragraphs, bullet items, // quotes, preformatted blocks and links on lines of their own, and nothing // else: no emphasis, no links inside text, no tables, no images. package gemtext import ( "encoding/xml" "fmt" "io" "regexp" "strconv" "strings" "unicode/utf8" ) // Options tell Convert how to treat the addresses and the generated blocks. type Options struct { // Link turns an address found in the HTML (a link or an image) into the // address to write. An empty result drops the link and keeps its text, as // for "#section"; an error stops the conversion and names the address. A // nil Link leaves every address as it is. Link func(href string) (string, error) // Block returns the gemtext of a block placeholder, the element // . A nil Block makes such an element an error. Block func(name string) (string, error) } // node is an element or a piece of text of the parsed HTML. type node struct { tag string text string attrs map[string]string kids []*node } // link is an address with the text that labels it. type link struct{ target, label string } // block is a part of the output: lines of text followed by link lines. type block struct { text []string links []link } // converter holds the options of one conversion. type converter struct{ opts Options } // Convert converts html, the body of a page, to gemtext. Blocks are separated // by a blank line; the links of a paragraph, a list, a quote or a table come // right after it, each on a line of its own, and the output ends with a line // break unless it is empty. func Convert(html string, opts Options) (string, error) { root, err := parse(html) if err != nil { return "", err } c := &converter{opts: opts} blocks, err := c.blocks(root.kids) if err != nil { return "", err } parts := make([]string, 0, len(blocks)) for _, b := range blocks { if text := b.String(); text != "" { parts = append(parts, text) } } if len(parts) == 0 { return "", nil } return strings.Join(parts, "\n\n") + "\n", nil } // String returns the lines of the block, text first, then the link lines. func (b block) String() string { lines := append([]string(nil), b.text...) seen := map[link]bool{} for _, l := range b.links { if seen[l] { continue } seen[l] = true lines = append(lines, linkLine(l)) } return strings.Join(lines, "\n") } // Text returns plain text as one line of gemtext: white space collapsed, and a // leading space when the text would be read as another kind of line. func Text(s string) string { return protect(strings.Join(strings.Fields(s), " ")) } // Link returns the link line of an address with a label. func Link(target, label string) string { return linkLine(link{target, strings.Join(strings.Fields(label), " ")}) } // linkLine writes a link on its own line. func linkLine(l link) string { if l.label == "" || l.label == l.target { return "=> " + l.target } return "=> " + l.target + " " + l.label } // voidTags are the elements that have no end tag. var voidTags = map[string]bool{ "area": true, "base": true, "br": true, "col": true, "hr": true, "img": true, "input": true, "link": true, "meta": true, "param": true, "source": true, "basefont": true, "frame": true, "isindex": true, } // parse reads a fragment of HTML into a tree. The decoder is the lenient mode // of encoding/xml with the HTML void elements and entities: the HTML of a page // comes from a Markdown renderer and from the author's own raw HTML, which are // well formed apart from the void elements. func parse(html string) (*node, error) { source := "" + html + "" decoder := xml.NewDecoder(strings.NewReader(source)) decoder.Strict = false decoder.AutoClose = xml.HTMLAutoClose decoder.Entity = xml.HTMLEntity root := &node{tag: "root"} stack := []*node{root} for { token, err := decoder.Token() if err == io.EOF { break } if err != nil { return nil, fmt.Errorf("read the HTML: %w", err) } top := stack[len(stack)-1] switch t := token.(type) { case xml.StartElement: n := &node{tag: strings.ToLower(t.Name.Local), attrs: map[string]string{}} for _, a := range t.Attr { n.attrs[strings.ToLower(a.Name.Local)] = a.Value } top.kids = append(top.kids, n) stack = append(stack, n) case xml.EndElement: // The lenient mode closes by itself what the text left open; that is // a mistake in the page, so an end that the text does not have is an error. name := strings.ToLower(t.Name.Local) written := strings.ToLower(source[:decoder.InputOffset()]) if !voidTags[name] && !strings.HasSuffix(written, "") && !strings.HasSuffix(written, "/>") { return nil, fmt.Errorf("read the HTML: the element <%s> is not closed", name) } if len(stack) > 1 { stack = stack[:len(stack)-1] } case xml.CharData: top.kids = append(top.kids, &node{text: string(t)}) } } if len(stack) != 1 || len(root.kids) != 1 { return nil, fmt.Errorf("read the HTML: the element <%s> is not closed", stack[len(stack)-1].tag) } return root.kids[0], nil } // blockTags are the elements that start a block of their own. var blockTags = map[string]bool{ "h1": true, "h2": true, "h3": true, "h4": true, "h5": true, "h6": true, "p": true, "ul": true, "ol": true, "blockquote": true, "pre": true, "hr": true, "table": true, "div": true, "section": true, "article": true, "main": true, "header": true, "footer": true, "nav": true, "aside": true, "figure": true, "figcaption": true, "details": true, "summary": true, "dl": true, "dt": true, "dd": true, "form": true, "fieldset": true, "address": true, "gemblock": true, } // skippedTags are the elements that have nothing to read. var skippedTags = map[string]bool{ "script": true, "style": true, "svg": true, "noscript": true, "template": true, "head": true, "title": true, "meta": true, "link": true, } // skipped reports whether an element is left out: one with nothing to read, or // one that is hidden. func skipped(n *node) bool { _, hidden := n.attrs["hidden"] return skippedTags[n.tag] || hidden || n.attrs["aria-hidden"] == "true" } // blocks converts a list of nodes into blocks: block elements make their own, // and the text and the inline elements between them make paragraphs. func (c *converter) blocks(nodes []*node) ([]block, error) { var out []block var run []*node flush := func() error { if len(run) == 0 { return nil } b, err := c.paragraph(run) run = nil if err != nil { return err } out = append(out, b) return nil } for _, n := range nodes { if n.tag != "" && skipped(n) { continue } if n.tag == "" || !blockTags[n.tag] { run = append(run, n) continue } if err := flush(); err != nil { return nil, err } got, err := c.element(n) if err != nil { return nil, err } out = append(out, got...) } if err := flush(); err != nil { return nil, err } return out, nil } // element converts one block element. func (c *converter) element(n *node) ([]block, error) { switch n.tag { case "h1", "h2", "h3", "h4", "h5", "h6": return c.heading(n) case "p", "summary", "figcaption", "dt", "dd": if strings.Contains(" "+n.attrs["class"]+" ", " note ") || n.attrs["role"] == "note" { return c.quote(n.kids) } b, err := c.paragraph(n.kids) return []block{b}, err case "ul", "ol": b, err := c.list(n) return []block{b}, err case "blockquote": return c.quote(n.kids) case "pre": return []block{preformatted(n)}, nil case "hr": return nil, nil case "table": b, err := c.table(n) return []block{b}, err case "gemblock": return c.generated(n) default: return c.blocks(n.kids) } } // generated asks Options.Block for the gemtext of a block placeholder. func (c *converter) generated(n *node) ([]block, error) { name := n.attrs["name"] if c.opts.Block == nil { return nil, fmt.Errorf("the block %q has no generator", name) } text, err := c.opts.Block(name) if err != nil { return nil, fmt.Errorf("block %q: %w", name, err) } text = strings.Trim(text, "\n") if text == "" { return nil, nil } return []block{{text: strings.Split(text, "\n")}}, nil } // heading converts h1 to h6; gemtext has three levels. func (c *converter) heading(n *node) ([]block, error) { text, links, err := c.inline(n.kids, " ") if err != nil { return nil, err } text = strings.Join(strings.Fields(text), " ") if text == "" { return nil, nil } level := int(n.tag[1] - '0') if level > 3 { level = 3 } return []block{{text: []string{strings.Repeat("#", level) + " " + text}, links: links}}, nil } // paragraph converts a run of text and inline elements. Each line break is a // line of its own, and the links come after. A paragraph that is only a link // is only the link line. func (c *converter) paragraph(nodes []*node) (block, error) { text, links, err := c.inline(nodes, "\n") if err != nil { return block{}, err } lines := cleanLines(text) if len(lines) == 1 && len(links) == 1 && lines[0] == links[0].label { return block{links: links}, nil } for i := range lines { lines[i] = protect(lines[i]) } return block{text: lines, links: links}, nil } // cleanLines splits text at its line breaks, collapses the white space of every // line and drops the empty ones. func cleanLines(text string) []string { var lines []string for _, line := range strings.Split(text, "\n") { if line = strings.Join(strings.Fields(line), " "); line != "" { lines = append(lines, line) } } return lines } // protect keeps a line of text from being read as another kind of line: a // leading space does not change what it says and gemtext has no escape. func protect(line string) string { for _, prefix := range []string{"=>", "#", ">", "```", "* "} { if strings.HasPrefix(line, prefix) { return " " + line } } return line } // space matches a run of white space, the no-break space included. var space = regexp.MustCompile(`[\s\x{00a0}]+`) // inline returns the text of the nodes, the links in them and the addresses of // their images. lineBreak is what a
becomes. func (c *converter) inline(nodes []*node, lineBreak string) (string, []link, error) { var text strings.Builder var links []link var walk func(n *node) error walk = func(n *node) error { if n.tag == "" { text.WriteString(space.ReplaceAllString(n.text, " ")) return nil } if skipped(n) { return nil } switch n.tag { case "br": text.WriteString(lineBreak) return nil case "img": alt := strings.Join(strings.Fields(n.attrs["alt"]), " ") if alt == "" || n.attrs["src"] == "" { return nil } target, err := c.address(n.attrs["src"]) if err != nil { return err } if target != "" { links = append(links, link{target, alt}) } return nil case "a": label, inner, err := c.inline(n.kids, " ") if err != nil { return err } text.WriteString(label) links = append(links, inner...) if href := n.attrs["href"]; href != "" { target, err := c.address(href) if err != nil { return err } if target != "" { links = append(links, link{target, strings.Join(strings.Fields(label), " ")}) } } return nil } for _, kid := range n.kids { if err := walk(kid); err != nil { return err } } return nil } for _, n := range nodes { if err := walk(n); err != nil { return "", nil, err } } return text.String(), links, nil } // address maps an address through Options.Link. func (c *converter) address(href string) (string, error) { if c.opts.Link == nil { return href, nil } target, err := c.opts.Link(href) if err != nil { return "", fmt.Errorf("the address %q: %w", href, err) } return target, nil } // item is one bullet of a list. type item struct { text string links []link } // list converts a list: bullets, with the links after them. A list in which // every item is only a link is a list of link lines. A list inside an item is // flattened into the same list. An ordered list numbers its items. func (c *converter) list(n *node) (block, error) { var items []item if err := c.items(n, &items); err != nil { return block{}, err } linksOnly := len(items) > 0 var links []link for _, it := range items { links = append(links, it.links...) if len(it.links) != 1 || it.text != it.links[0].label { linksOnly = false } } if linksOnly { return block{links: links}, nil } lines := make([]string, len(items)) for i, it := range items { prefix := "* " if n.tag == "ol" { prefix += strconv.Itoa(i+1) + ". " } lines[i] = prefix + it.text } return block{text: lines, links: links}, nil } // items collects the items of a list and of the lists inside it. func (c *converter) items(list *node, out *[]item) error { for _, li := range list.kids { if li.tag != "li" || skipped(li) { continue } var own, nested []*node for _, kid := range li.kids { if kid.tag == "ul" || kid.tag == "ol" { nested = append(nested, kid) } else { own = append(own, kid) } } text, links, err := c.inline(own, " ") if err != nil { return err } if text = strings.Join(strings.Fields(text), " "); text != "" || len(links) > 0 { *out = append(*out, item{text: text, links: links}) } for _, inner := range nested { if err := c.items(inner, out); err != nil { return err } } } return nil } // quote converts the content of a quote: every line starts with "> ", the // blocks are separated by an empty quote line, and the links come after. func (c *converter) quote(nodes []*node) ([]block, error) { inner, err := c.blocks(nodes) if err != nil { return nil, err } var lines []string var links []link for _, b := range inner { if len(b.text) == 0 { links = append(links, b.links...) continue } if len(lines) > 0 { lines = append(lines, ">") } for _, line := range b.text { lines = append(lines, "> "+strings.TrimPrefix(line, " ")) } links = append(links, b.links...) } if len(lines) == 0 && len(links) == 0 { return nil, nil } return []block{{text: lines, links: links}}, nil } // preformatted converts a
: its text as it is between two fence lines. A
// line that would close the block early gets a space in front.
func preformatted(n *node) block {
	var text strings.Builder
	var walk func(n *node)
	walk = func(n *node) {
		if n.tag == "" {
			text.WriteString(n.text)
			return
		}
		if n.tag == "br" {
			text.WriteString("\n")
			return
		}
		for _, kid := range n.kids {
			walk(kid)
		}
	}
	walk(n)
	lines := strings.Split(strings.TrimSuffix(text.String(), "\n"), "\n")
	out := []string{"```"}
	for _, line := range lines {
		if strings.HasPrefix(line, "```") {
			line = " " + line
		}
		out = append(out, strings.TrimRight(line, " \t"))
	}
	return block{text: append(out, "```")}
}

// table converts a table to a preformatted block of aligned columns, with a
// line under a header row, and the caption as a line of text before it. The
// links of the cells come after the block.
func (c *converter) table(n *node) (block, error) {
	var rows [][]string
	var links []link
	var caption string
	headerRow := false
	var walk func(n *node) error
	walk = func(n *node) error {
		for _, kid := range n.kids {
			if kid.tag == "" || skipped(kid) {
				continue
			}
			switch kid.tag {
			case "caption":
				text, inner, err := c.inline(kid.kids, " ")
				if err != nil {
					return err
				}
				caption = strings.Join(strings.Fields(text), " ")
				links = append(links, inner...)
			case "tr":
				var cells []string
				onlyHeaders := true
				for _, cell := range kid.kids {
					if cell.tag != "td" && cell.tag != "th" {
						continue
					}
					text, inner, err := c.inline(cell.kids, " ")
					if err != nil {
						return err
					}
					cells = append(cells, strings.Join(strings.Fields(text), " "))
					links = append(links, inner...)
					if cell.tag != "th" {
						onlyHeaders = false
					}
				}
				if len(cells) > 0 {
					if len(rows) == 0 && onlyHeaders {
						headerRow = true
					}
					rows = append(rows, cells)
				}
			default:
				if err := walk(kid); err != nil {
					return err
				}
			}
		}
		return nil
	}
	if err := walk(n); err != nil {
		return block{}, err
	}
	var text []string
	if caption != "" {
		text = append(text, protect(caption))
	}
	switch {
	case len(rows) == 0:
	case tableWidth(rows) <= maxTableWidth:
		text = append(text, "```")
		text = append(text, alignRows(rows, headerRow)...)
		text = append(text, "```")
	default:
		if caption != "" {
			text = append(text, "")
		}
		text = append(text, records(rows, headerRow)...)
	}
	return block{text: text, links: links}, nil
}

// maxTableWidth is the widest a table can be and stay a preformatted block: lines
// of a preformatted block are not wrapped by the readers.
const maxTableWidth = 80

// tableWidth returns the width of the rows written as columns two spaces apart.
func tableWidth(rows [][]string) int {
	widest := 0
	for _, line := range alignRows(rows, false) {
		if w := utf8.RuneCountInString(line); w > widest {
			widest = w
		}
	}
	return widest
}

// records writes the rows of a wide table as a list. With a header row each
// row is a heading, its first cell, and a bullet for each other cell that is not
// empty, named by the header of its column. Without one each row is a bullet
// with the cells between bars.
func records(rows [][]string, header bool) []string {
	var lines []string
	if !header {
		for _, r := range rows {
			lines = append(lines, "* "+strings.Join(r, " | "))
		}
		return lines
	}
	names := rows[0]
	for i, r := range rows[1:] {
		if i > 0 {
			lines = append(lines, "")
		}
		title := ""
		if len(r) > 0 {
			title = r[0]
		}
		lines = append(lines, "### "+title)
		for j := 1; j < len(r); j++ {
			if r[j] == "" {
				continue
			}
			name := ""
			if j < len(names) {
				name = names[j] + ": "
			}
			lines = append(lines, "* "+name+r[j])
		}
	}
	return lines
}

// alignRows writes rows of cells as columns two spaces apart, each as wide as
// its widest cell, with a line of dashes under the first row when it is a header.
func alignRows(rows [][]string, header bool) []string {
	columns := 0
	for _, r := range rows {
		if len(r) > columns {
			columns = len(r)
		}
	}
	widths := make([]int, columns)
	for _, r := range rows {
		for i, cell := range r {
			if w := utf8.RuneCountInString(cell); w > widths[i] {
				widths[i] = w
			}
		}
	}
	write := func(r []string) string {
		cells := make([]string, columns)
		for i := range cells {
			cell := ""
			if i < len(r) {
				cell = r[i]
			}
			cells[i] = cell + strings.Repeat(" ", widths[i]-utf8.RuneCountInString(cell))
		}
		line := strings.TrimRight(strings.Join(cells, "  "), " ")
		if strings.HasPrefix(line, "```") {
			line = " " + line
		}
		return line
	}
	var lines []string
	for i, r := range rows {
		lines = append(lines, write(r))
		if i == 0 && header {
			dashes := make([]string, columns)
			for j := range dashes {
				dashes[j] = strings.Repeat("-", widths[j])
			}
			lines = append(lines, strings.Join(dashes, "  "))
		}
	}
	return lines
}