feat(gemtext): convert the HTML of a page to gemtext
A new package, with no new dependency, that reads the HTML a page renders to (the lenient mode of encoding/xml with the HTML void elements and entities) and writes gemtext: three heading levels, one line paragraphs, the links of a paragraph, a list, a quote or a table on lines of their own after it, bullets (a list of only links is link lines), quotes (a note is a quote), preformatted blocks, tables as aligned preformatted blocks, images as link lines, no emphasis, and lines that look like gemtext are protected. Options.Link maps or drops an address and its error stops the conversion; Options.Block expands the placeholders of generated blocks. The language files also keep the order of the menu now (Lang.NavOrder), which the Gemini output needs. Refs #10
This commit is contained in:
parent
633da1c46b
commit
28745d452f
4 changed files with 805 additions and 0 deletions
637
internal/gemtext/gemtext.go
Normal file
637
internal/gemtext/gemtext.go
Normal file
|
|
@ -0,0 +1,637 @@
|
||||||
|
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||||
|
|
||||||
|
// Package gemtext turns the HTML of a page into gemtext, the format of the
|
||||||
|
// Gemini protocol. Gemtext has headings, one-line paragraphs, bullet items,
|
||||||
|
// quotes, preformatted blocks and links on lines of their own, and nothing
|
||||||
|
// else: no emphasis, no links inside text, no tables, no images.
|
||||||
|
package gemtext
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/xml"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"regexp"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"unicode/utf8"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Options tell Convert how to treat the addresses and the generated blocks.
|
||||||
|
type Options struct {
|
||||||
|
// Link turns an address found in the HTML (a link or an image) into the
|
||||||
|
// address to write. An empty result drops the link and keeps its text, as
|
||||||
|
// for "#section"; an error stops the conversion and names the address. A
|
||||||
|
// nil Link leaves every address as it is.
|
||||||
|
Link func(href string) (string, error)
|
||||||
|
// Block returns the gemtext of a block placeholder, the element
|
||||||
|
// <gemblock name="...">. A nil Block makes such an element an error.
|
||||||
|
Block func(name string) (string, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// node is an element or a piece of text of the parsed HTML.
|
||||||
|
type node struct {
|
||||||
|
tag string
|
||||||
|
text string
|
||||||
|
attrs map[string]string
|
||||||
|
kids []*node
|
||||||
|
}
|
||||||
|
|
||||||
|
// link is an address with the text that labels it.
|
||||||
|
type link struct{ target, label string }
|
||||||
|
|
||||||
|
// block is a part of the output: lines of text followed by link lines.
|
||||||
|
type block struct {
|
||||||
|
text []string
|
||||||
|
links []link
|
||||||
|
}
|
||||||
|
|
||||||
|
// converter holds the options of one conversion.
|
||||||
|
type converter struct{ opts Options }
|
||||||
|
|
||||||
|
// Convert converts html, the body of a page, to gemtext. Blocks are separated
|
||||||
|
// by a blank line; the links of a paragraph, a list, a quote or a table come
|
||||||
|
// right after it, each on a line of its own, and the output ends with a line
|
||||||
|
// break unless it is empty.
|
||||||
|
func Convert(html string, opts Options) (string, error) {
|
||||||
|
root, err := parse(html)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
c := &converter{opts: opts}
|
||||||
|
blocks, err := c.blocks(root.kids)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
parts := make([]string, 0, len(blocks))
|
||||||
|
for _, b := range blocks {
|
||||||
|
if text := b.String(); text != "" {
|
||||||
|
parts = append(parts, text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(parts) == 0 {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
return strings.Join(parts, "\n\n") + "\n", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// String returns the lines of the block, text first, then the link lines.
|
||||||
|
func (b block) String() string {
|
||||||
|
lines := append([]string(nil), b.text...)
|
||||||
|
seen := map[link]bool{}
|
||||||
|
for _, l := range b.links {
|
||||||
|
if seen[l] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[l] = true
|
||||||
|
lines = append(lines, linkLine(l))
|
||||||
|
}
|
||||||
|
return strings.Join(lines, "\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
// linkLine writes a link on its own line.
|
||||||
|
func linkLine(l link) string {
|
||||||
|
if l.label == "" || l.label == l.target {
|
||||||
|
return "=> " + l.target
|
||||||
|
}
|
||||||
|
return "=> " + l.target + " " + l.label
|
||||||
|
}
|
||||||
|
|
||||||
|
// voidTags are the elements that have no end tag.
|
||||||
|
var voidTags = map[string]bool{
|
||||||
|
"area": true, "base": true, "br": true, "col": true, "hr": true, "img": true, "input": true,
|
||||||
|
"link": true, "meta": true, "param": true, "source": true, "basefont": true, "frame": true, "isindex": true,
|
||||||
|
}
|
||||||
|
|
||||||
|
// parse reads a fragment of HTML into a tree. The decoder is the lenient mode
|
||||||
|
// of encoding/xml with the HTML void elements and entities: the HTML of a page
|
||||||
|
// comes from a Markdown renderer and from the author's own raw HTML, which are
|
||||||
|
// well formed apart from the void elements.
|
||||||
|
func parse(html string) (*node, error) {
|
||||||
|
source := "<root>" + html + "</root>"
|
||||||
|
decoder := xml.NewDecoder(strings.NewReader(source))
|
||||||
|
decoder.Strict = false
|
||||||
|
decoder.AutoClose = xml.HTMLAutoClose
|
||||||
|
decoder.Entity = xml.HTMLEntity
|
||||||
|
root := &node{tag: "root"}
|
||||||
|
stack := []*node{root}
|
||||||
|
for {
|
||||||
|
token, err := decoder.Token()
|
||||||
|
if err == io.EOF {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("read the HTML: %w", err)
|
||||||
|
}
|
||||||
|
top := stack[len(stack)-1]
|
||||||
|
switch t := token.(type) {
|
||||||
|
case xml.StartElement:
|
||||||
|
n := &node{tag: strings.ToLower(t.Name.Local), attrs: map[string]string{}}
|
||||||
|
for _, a := range t.Attr {
|
||||||
|
n.attrs[strings.ToLower(a.Name.Local)] = a.Value
|
||||||
|
}
|
||||||
|
top.kids = append(top.kids, n)
|
||||||
|
stack = append(stack, n)
|
||||||
|
case xml.EndElement:
|
||||||
|
// The lenient mode closes by itself what the text left open; that is
|
||||||
|
// a mistake in the page, so an end that the text does not have is an error.
|
||||||
|
name := strings.ToLower(t.Name.Local)
|
||||||
|
written := strings.ToLower(source[:decoder.InputOffset()])
|
||||||
|
if !voidTags[name] && !strings.HasSuffix(written, "</"+name+">") && !strings.HasSuffix(written, "/>") {
|
||||||
|
return nil, fmt.Errorf("read the HTML: the element <%s> is not closed", name)
|
||||||
|
}
|
||||||
|
if len(stack) > 1 {
|
||||||
|
stack = stack[:len(stack)-1]
|
||||||
|
}
|
||||||
|
case xml.CharData:
|
||||||
|
top.kids = append(top.kids, &node{text: string(t)})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(stack) != 1 || len(root.kids) != 1 {
|
||||||
|
return nil, fmt.Errorf("read the HTML: the element <%s> is not closed", stack[len(stack)-1].tag)
|
||||||
|
}
|
||||||
|
return root.kids[0], nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// blockTags are the elements that start a block of their own.
|
||||||
|
var blockTags = map[string]bool{
|
||||||
|
"h1": true, "h2": true, "h3": true, "h4": true, "h5": true, "h6": true, "p": true,
|
||||||
|
"ul": true, "ol": true, "blockquote": true, "pre": true, "hr": true, "table": true,
|
||||||
|
"div": true, "section": true, "article": true, "main": true, "header": true, "footer": true,
|
||||||
|
"nav": true, "aside": true, "figure": true, "figcaption": true, "details": true, "summary": true,
|
||||||
|
"dl": true, "dt": true, "dd": true, "form": true, "fieldset": true, "address": true, "gemblock": true,
|
||||||
|
}
|
||||||
|
|
||||||
|
// skippedTags are the elements that have nothing to read.
|
||||||
|
var skippedTags = map[string]bool{
|
||||||
|
"script": true, "style": true, "svg": true, "noscript": true, "template": true,
|
||||||
|
"head": true, "title": true, "meta": true, "link": true,
|
||||||
|
}
|
||||||
|
|
||||||
|
// skipped reports whether an element is left out: one with nothing to read, or
|
||||||
|
// one that is hidden.
|
||||||
|
func skipped(n *node) bool {
|
||||||
|
_, hidden := n.attrs["hidden"]
|
||||||
|
return skippedTags[n.tag] || hidden || n.attrs["aria-hidden"] == "true"
|
||||||
|
}
|
||||||
|
|
||||||
|
// blocks converts a list of nodes into blocks: block elements make their own,
|
||||||
|
// and the text and the inline elements between them make paragraphs.
|
||||||
|
func (c *converter) blocks(nodes []*node) ([]block, error) {
|
||||||
|
var out []block
|
||||||
|
var run []*node
|
||||||
|
flush := func() error {
|
||||||
|
if len(run) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
b, err := c.paragraph(run)
|
||||||
|
run = nil
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
out = append(out, b)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, n := range nodes {
|
||||||
|
if n.tag != "" && skipped(n) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if n.tag == "" || !blockTags[n.tag] {
|
||||||
|
run = append(run, n)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := flush(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
got, err := c.element(n)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out = append(out, got...)
|
||||||
|
}
|
||||||
|
if err := flush(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// element converts one block element.
|
||||||
|
func (c *converter) element(n *node) ([]block, error) {
|
||||||
|
switch n.tag {
|
||||||
|
case "h1", "h2", "h3", "h4", "h5", "h6":
|
||||||
|
return c.heading(n)
|
||||||
|
case "p", "summary", "figcaption", "dt", "dd":
|
||||||
|
if strings.Contains(" "+n.attrs["class"]+" ", " note ") || n.attrs["role"] == "note" {
|
||||||
|
return c.quote(n.kids)
|
||||||
|
}
|
||||||
|
b, err := c.paragraph(n.kids)
|
||||||
|
return []block{b}, err
|
||||||
|
case "ul", "ol":
|
||||||
|
b, err := c.list(n)
|
||||||
|
return []block{b}, err
|
||||||
|
case "blockquote":
|
||||||
|
return c.quote(n.kids)
|
||||||
|
case "pre":
|
||||||
|
return []block{preformatted(n)}, nil
|
||||||
|
case "hr":
|
||||||
|
return nil, nil
|
||||||
|
case "table":
|
||||||
|
b, err := c.table(n)
|
||||||
|
return []block{b}, err
|
||||||
|
case "gemblock":
|
||||||
|
return c.generated(n)
|
||||||
|
default:
|
||||||
|
return c.blocks(n.kids)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// generated asks Options.Block for the gemtext of a block placeholder.
|
||||||
|
func (c *converter) generated(n *node) ([]block, error) {
|
||||||
|
name := n.attrs["name"]
|
||||||
|
if c.opts.Block == nil {
|
||||||
|
return nil, fmt.Errorf("the block %q has no generator", name)
|
||||||
|
}
|
||||||
|
text, err := c.opts.Block(name)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("block %q: %w", name, err)
|
||||||
|
}
|
||||||
|
text = strings.Trim(text, "\n")
|
||||||
|
if text == "" {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return []block{{text: strings.Split(text, "\n")}}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// heading converts h1 to h6; gemtext has three levels.
|
||||||
|
func (c *converter) heading(n *node) ([]block, error) {
|
||||||
|
text, links, err := c.inline(n.kids, " ")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
text = strings.Join(strings.Fields(text), " ")
|
||||||
|
if text == "" {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
level := int(n.tag[1] - '0')
|
||||||
|
if level > 3 {
|
||||||
|
level = 3
|
||||||
|
}
|
||||||
|
return []block{{text: []string{strings.Repeat("#", level) + " " + text}, links: links}}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// paragraph converts a run of text and inline elements. Each line break is a
|
||||||
|
// line of its own, and the links come after. A paragraph that is only a link
|
||||||
|
// is only the link line.
|
||||||
|
func (c *converter) paragraph(nodes []*node) (block, error) {
|
||||||
|
text, links, err := c.inline(nodes, "\n")
|
||||||
|
if err != nil {
|
||||||
|
return block{}, err
|
||||||
|
}
|
||||||
|
lines := cleanLines(text)
|
||||||
|
if len(lines) == 1 && len(links) == 1 && lines[0] == links[0].label {
|
||||||
|
return block{links: links}, nil
|
||||||
|
}
|
||||||
|
for i := range lines {
|
||||||
|
lines[i] = protect(lines[i])
|
||||||
|
}
|
||||||
|
return block{text: lines, links: links}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// cleanLines splits text at its line breaks, collapses the white space of every
|
||||||
|
// line and drops the empty ones.
|
||||||
|
func cleanLines(text string) []string {
|
||||||
|
var lines []string
|
||||||
|
for _, line := range strings.Split(text, "\n") {
|
||||||
|
if line = strings.Join(strings.Fields(line), " "); line != "" {
|
||||||
|
lines = append(lines, line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return lines
|
||||||
|
}
|
||||||
|
|
||||||
|
// protect keeps a line of text from being read as another kind of line: a
|
||||||
|
// leading space does not change what it says and gemtext has no escape.
|
||||||
|
func protect(line string) string {
|
||||||
|
for _, prefix := range []string{"=>", "#", ">", "```", "* "} {
|
||||||
|
if strings.HasPrefix(line, prefix) {
|
||||||
|
return " " + line
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return line
|
||||||
|
}
|
||||||
|
|
||||||
|
// space matches a run of white space, the no-break space included.
|
||||||
|
var space = regexp.MustCompile(`[\s\x{00a0}]+`)
|
||||||
|
|
||||||
|
// inline returns the text of the nodes, the links in them and the addresses of
|
||||||
|
// their images. lineBreak is what a <br> becomes.
|
||||||
|
func (c *converter) inline(nodes []*node, lineBreak string) (string, []link, error) {
|
||||||
|
var text strings.Builder
|
||||||
|
var links []link
|
||||||
|
var walk func(n *node) error
|
||||||
|
walk = func(n *node) error {
|
||||||
|
if n.tag == "" {
|
||||||
|
text.WriteString(space.ReplaceAllString(n.text, " "))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if skipped(n) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
switch n.tag {
|
||||||
|
case "br":
|
||||||
|
text.WriteString(lineBreak)
|
||||||
|
return nil
|
||||||
|
case "img":
|
||||||
|
alt := strings.Join(strings.Fields(n.attrs["alt"]), " ")
|
||||||
|
if alt == "" || n.attrs["src"] == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
target, err := c.address(n.attrs["src"])
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if target != "" {
|
||||||
|
links = append(links, link{target, alt})
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
case "a":
|
||||||
|
label, inner, err := c.inline(n.kids, " ")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
text.WriteString(label)
|
||||||
|
links = append(links, inner...)
|
||||||
|
if href := n.attrs["href"]; href != "" {
|
||||||
|
target, err := c.address(href)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if target != "" {
|
||||||
|
links = append(links, link{target, strings.Join(strings.Fields(label), " ")})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, kid := range n.kids {
|
||||||
|
if err := walk(kid); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, n := range nodes {
|
||||||
|
if err := walk(n); err != nil {
|
||||||
|
return "", nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return text.String(), links, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// address maps an address through Options.Link.
|
||||||
|
func (c *converter) address(href string) (string, error) {
|
||||||
|
if c.opts.Link == nil {
|
||||||
|
return href, nil
|
||||||
|
}
|
||||||
|
target, err := c.opts.Link(href)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("the address %q: %w", href, err)
|
||||||
|
}
|
||||||
|
return target, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// item is one bullet of a list.
|
||||||
|
type item struct {
|
||||||
|
text string
|
||||||
|
links []link
|
||||||
|
}
|
||||||
|
|
||||||
|
// list converts a list: bullets, with the links after them. A list in which
|
||||||
|
// every item is only a link is a list of link lines. A list inside an item is
|
||||||
|
// flattened into the same list. An ordered list numbers its items.
|
||||||
|
func (c *converter) list(n *node) (block, error) {
|
||||||
|
var items []item
|
||||||
|
if err := c.items(n, &items); err != nil {
|
||||||
|
return block{}, err
|
||||||
|
}
|
||||||
|
linksOnly := len(items) > 0
|
||||||
|
var links []link
|
||||||
|
for _, it := range items {
|
||||||
|
links = append(links, it.links...)
|
||||||
|
if len(it.links) != 1 || it.text != it.links[0].label {
|
||||||
|
linksOnly = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if linksOnly {
|
||||||
|
return block{links: links}, nil
|
||||||
|
}
|
||||||
|
lines := make([]string, len(items))
|
||||||
|
for i, it := range items {
|
||||||
|
prefix := "* "
|
||||||
|
if n.tag == "ol" {
|
||||||
|
prefix += strconv.Itoa(i+1) + ". "
|
||||||
|
}
|
||||||
|
lines[i] = prefix + it.text
|
||||||
|
}
|
||||||
|
return block{text: lines, links: links}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// items collects the items of a list and of the lists inside it.
|
||||||
|
func (c *converter) items(list *node, out *[]item) error {
|
||||||
|
for _, li := range list.kids {
|
||||||
|
if li.tag != "li" || skipped(li) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var own, nested []*node
|
||||||
|
for _, kid := range li.kids {
|
||||||
|
if kid.tag == "ul" || kid.tag == "ol" {
|
||||||
|
nested = append(nested, kid)
|
||||||
|
} else {
|
||||||
|
own = append(own, kid)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
text, links, err := c.inline(own, " ")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if text = strings.Join(strings.Fields(text), " "); text != "" || len(links) > 0 {
|
||||||
|
*out = append(*out, item{text: text, links: links})
|
||||||
|
}
|
||||||
|
for _, inner := range nested {
|
||||||
|
if err := c.items(inner, out); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// quote converts the content of a quote: every line starts with "> ", the
|
||||||
|
// blocks are separated by an empty quote line, and the links come after.
|
||||||
|
func (c *converter) quote(nodes []*node) ([]block, error) {
|
||||||
|
inner, err := c.blocks(nodes)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var lines []string
|
||||||
|
var links []link
|
||||||
|
for _, b := range inner {
|
||||||
|
if len(b.text) == 0 {
|
||||||
|
links = append(links, b.links...)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if len(lines) > 0 {
|
||||||
|
lines = append(lines, ">")
|
||||||
|
}
|
||||||
|
for _, line := range b.text {
|
||||||
|
lines = append(lines, "> "+strings.TrimPrefix(line, " "))
|
||||||
|
}
|
||||||
|
links = append(links, b.links...)
|
||||||
|
}
|
||||||
|
if len(lines) == 0 && len(links) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return []block{{text: lines, links: links}}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// preformatted converts a <pre>: its text as it is between two fence lines. A
|
||||||
|
// line that would close the block early gets a space in front.
|
||||||
|
func preformatted(n *node) block {
|
||||||
|
var text strings.Builder
|
||||||
|
var walk func(n *node)
|
||||||
|
walk = func(n *node) {
|
||||||
|
if n.tag == "" {
|
||||||
|
text.WriteString(n.text)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if n.tag == "br" {
|
||||||
|
text.WriteString("\n")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, kid := range n.kids {
|
||||||
|
walk(kid)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
walk(n)
|
||||||
|
lines := strings.Split(strings.TrimSuffix(text.String(), "\n"), "\n")
|
||||||
|
out := []string{"```"}
|
||||||
|
for _, line := range lines {
|
||||||
|
if strings.HasPrefix(line, "```") {
|
||||||
|
line = " " + line
|
||||||
|
}
|
||||||
|
out = append(out, strings.TrimRight(line, " \t"))
|
||||||
|
}
|
||||||
|
return block{text: append(out, "```")}
|
||||||
|
}
|
||||||
|
|
||||||
|
// table converts a table to a preformatted block of aligned columns, with a
|
||||||
|
// line under a header row, and the caption as a line of text before it. The
|
||||||
|
// links of the cells come after the block.
|
||||||
|
func (c *converter) table(n *node) (block, error) {
|
||||||
|
var rows [][]string
|
||||||
|
var links []link
|
||||||
|
var caption string
|
||||||
|
headerRow := false
|
||||||
|
var walk func(n *node) error
|
||||||
|
walk = func(n *node) error {
|
||||||
|
for _, kid := range n.kids {
|
||||||
|
if kid.tag == "" || skipped(kid) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
switch kid.tag {
|
||||||
|
case "caption":
|
||||||
|
text, inner, err := c.inline(kid.kids, " ")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
caption = strings.Join(strings.Fields(text), " ")
|
||||||
|
links = append(links, inner...)
|
||||||
|
case "tr":
|
||||||
|
var cells []string
|
||||||
|
onlyHeaders := true
|
||||||
|
for _, cell := range kid.kids {
|
||||||
|
if cell.tag != "td" && cell.tag != "th" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
text, inner, err := c.inline(cell.kids, " ")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
cells = append(cells, strings.Join(strings.Fields(text), " "))
|
||||||
|
links = append(links, inner...)
|
||||||
|
if cell.tag != "th" {
|
||||||
|
onlyHeaders = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(cells) > 0 {
|
||||||
|
if len(rows) == 0 && onlyHeaders {
|
||||||
|
headerRow = true
|
||||||
|
}
|
||||||
|
rows = append(rows, cells)
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
if err := walk(kid); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := walk(n); err != nil {
|
||||||
|
return block{}, err
|
||||||
|
}
|
||||||
|
var text []string
|
||||||
|
if caption != "" {
|
||||||
|
text = append(text, protect(caption))
|
||||||
|
}
|
||||||
|
if len(rows) > 0 {
|
||||||
|
text = append(text, "```")
|
||||||
|
text = append(text, alignRows(rows, headerRow)...)
|
||||||
|
text = append(text, "```")
|
||||||
|
}
|
||||||
|
return block{text: text, links: links}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// alignRows writes rows of cells as columns two spaces apart, each as wide as
|
||||||
|
// its widest cell, with a line of dashes under the first row when it is a header.
|
||||||
|
func alignRows(rows [][]string, header bool) []string {
|
||||||
|
columns := 0
|
||||||
|
for _, r := range rows {
|
||||||
|
if len(r) > columns {
|
||||||
|
columns = len(r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
widths := make([]int, columns)
|
||||||
|
for _, r := range rows {
|
||||||
|
for i, cell := range r {
|
||||||
|
if w := utf8.RuneCountInString(cell); w > widths[i] {
|
||||||
|
widths[i] = w
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
write := func(r []string) string {
|
||||||
|
cells := make([]string, columns)
|
||||||
|
for i := range cells {
|
||||||
|
cell := ""
|
||||||
|
if i < len(r) {
|
||||||
|
cell = r[i]
|
||||||
|
}
|
||||||
|
cells[i] = cell + strings.Repeat(" ", widths[i]-utf8.RuneCountInString(cell))
|
||||||
|
}
|
||||||
|
line := strings.TrimRight(strings.Join(cells, " "), " ")
|
||||||
|
if strings.HasPrefix(line, "```") {
|
||||||
|
line = " " + line
|
||||||
|
}
|
||||||
|
return line
|
||||||
|
}
|
||||||
|
var lines []string
|
||||||
|
for i, r := range rows {
|
||||||
|
lines = append(lines, write(r))
|
||||||
|
if i == 0 && header {
|
||||||
|
dashes := make([]string, columns)
|
||||||
|
for j := range dashes {
|
||||||
|
dashes[j] = strings.Repeat("-", widths[j])
|
||||||
|
}
|
||||||
|
lines = append(lines, strings.Join(dashes, " "))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return lines
|
||||||
|
}
|
||||||
119
internal/gemtext/gemtext_test.go
Normal file
119
internal/gemtext/gemtext_test.go
Normal file
|
|
@ -0,0 +1,119 @@
|
||||||
|
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||||
|
|
||||||
|
package gemtext
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// identity leaves every address as it is.
|
||||||
|
func identity(href string) (string, error) { return href, nil }
|
||||||
|
|
||||||
|
// convert converts html with addresses left as they are and no blocks.
|
||||||
|
func convert(t *testing.T, html string) string {
|
||||||
|
t.Helper()
|
||||||
|
got, err := Convert(html, Options{Link: identity})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Convert(%q) error = %v", html, err)
|
||||||
|
}
|
||||||
|
return got
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestConvert checks the gemtext of every kind of element the site uses.
|
||||||
|
func TestConvert(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name, html, want string
|
||||||
|
}{
|
||||||
|
{"headings", `<h1>Title</h1><h2 id="a">Sub</h2><h4>Deep</h4>`, "# Title\n\n## Sub\n\n### Deep\n"},
|
||||||
|
{"hero with line breaks", `<h1 class="hero">Technology.<br>Freedom.<br><span class="accent">Dissent.</span></h1>`, "# Technology. Freedom. Dissent.\n"},
|
||||||
|
{"links go after the paragraph",
|
||||||
|
`<p>Read the <a href="/a.html">first</a> and <strong>the <a href="https://x.org/b">second</a></strong> page.</p>`,
|
||||||
|
"Read the first and the second page.\n=> /a.html first\n=> https://x.org/b second\n"},
|
||||||
|
{"the same link once", `<p><a href="/a">A</a> and <a href="/a">A</a> again.</p>`, "A and A again.\n=> /a A\n"},
|
||||||
|
{"a paragraph that is only a link", `<p><a href="/x.html">All news</a></p>`, "=> /x.html All news\n"},
|
||||||
|
{"line breaks", `<p>One<br>Two<br />Three</p>`, "One\nTwo\nThree\n"},
|
||||||
|
{"whitespace is collapsed", "<p>a\n b\t c</p>", "a b c\n"},
|
||||||
|
{"entities", `<p>Tom & Jerry © <ok></p>`, "Tom & Jerry © <ok>\n"},
|
||||||
|
{"unordered list", `<ul><li>One</li><li>Two <a href="/c">link</a></li></ul>`, "* One\n* Two link\n=> /c link\n"},
|
||||||
|
{"a list of only links has no bullets", `<ul><li><a href="/a">A</a></li><li><a href="/b">B</a></li></ul>`, "=> /a A\n=> /b B\n"},
|
||||||
|
{"nested list is flattened", `<ul><li>A<ul><li>A1</li></ul></li><li>B</li></ul>`, "* A\n* A1\n* B\n"},
|
||||||
|
{"ordered list", `<ol><li>First</li><li>Second</li></ol>`, "* 1. First\n* 2. Second\n"},
|
||||||
|
{"quote", `<blockquote><p>One</p><p>Two <a href="/x">x</a></p></blockquote>`, "> One\n>\n> Two x\n=> /x x\n"},
|
||||||
|
{"note is a quote", `<p class="note" role="note">Careful</p>`, "> Careful\n"},
|
||||||
|
{"preformatted", "<pre><code>line1\n line2\n```\n</code></pre>", "```\nline1\n line2\n ```\n```\n"},
|
||||||
|
{"table with a header",
|
||||||
|
`<table class="data"><caption>Data</caption><thead><tr><th>Service</th><th>Kept</th></tr></thead><tbody><tr><td data-label="Service">Mail</td><td>30 days</td></tr><tr><td>Chat</td><td>1 year</td></tr></tbody></table>`,
|
||||||
|
"Data\n```\nService Kept\n------- -------\nMail 30 days\nChat 1 year\n```\n"},
|
||||||
|
{"table without a header", `<table><tr><td>a</td><td>bb</td></tr><tr><td>ccc</td><td>d</td></tr></table>`, "```\na bb\nccc d\n```\n"},
|
||||||
|
{"table links", `<table><tr><td><a href="/m">Mail</a></td><td>x</td></tr></table>`, "```\nMail x\n```\n=> /m Mail\n"},
|
||||||
|
{"image with a description", `<p><img src="/logo.svg" alt="kmalloc"></p>`, "=> /logo.svg kmalloc\n"},
|
||||||
|
{"image without a description is skipped", `<p><img src="/logo.svg" alt=""></p>`, ""},
|
||||||
|
{"lines that look like gemtext are protected", `<p># no</p><p>=> no</p><p>* no</p><p>> no</p>`, " # no\n\n => no\n\n * no\n\n > no\n"},
|
||||||
|
{"hidden and scripted things are skipped", `<script>x</script><style>y</style><p hidden>z</p><p aria-hidden="true">w</p><svg><path/></svg><p>kept</p>`, "kept\n"},
|
||||||
|
{"containers are transparent", `<div><section><p>A</p></section></div><div>Loose <em>text</em></div>`, "A\n\nLoose text\n"},
|
||||||
|
{"time and span are inline", `<p><time datetime="2026-11-14">14 November 2026</time> <span>by</span> squid</p>`, "14 November 2026 by squid\n"},
|
||||||
|
{"empty input", ``, ""},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
if got := convert(t, tt.html); got != tt.want {
|
||||||
|
t.Errorf("Convert() =\n%q\nwant\n%q", got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestConvertLinkMapping checks that Link rewrites and drops addresses, and that its error stops the conversion.
|
||||||
|
func TestConvertLinkMapping(t *testing.T) {
|
||||||
|
link := func(href string) (string, error) {
|
||||||
|
switch {
|
||||||
|
case strings.HasPrefix(href, "#"):
|
||||||
|
return "", nil
|
||||||
|
case href == "/broken.html":
|
||||||
|
return "", errors.New("no such page")
|
||||||
|
}
|
||||||
|
target, _, _ := strings.Cut(href, "#")
|
||||||
|
return strings.Replace(target, ".html", ".gmi", 1), nil
|
||||||
|
}
|
||||||
|
got, err := Convert(`<p>See <a href="/a.html#x">a</a> and <a href="#top">top</a>.</p>`, Options{Link: link})
|
||||||
|
if err != nil || got != "See a and top.\n=> /a.gmi a\n" {
|
||||||
|
t.Errorf("Convert() = %q, %v", got, err)
|
||||||
|
}
|
||||||
|
_, err = Convert(`<p><a href="/broken.html">x</a></p>`, Options{Link: link})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "/broken.html") || !strings.Contains(err.Error(), "no such page") {
|
||||||
|
t.Errorf("error = %v, want one that names the address and the cause", err)
|
||||||
|
}
|
||||||
|
_, err = Convert(`<p><img src="/broken.html" alt="pic"></p>`, Options{Link: link})
|
||||||
|
if err == nil {
|
||||||
|
t.Error("a broken image address is not an error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestConvertBlocks checks the placeholders of generated blocks.
|
||||||
|
func TestConvertBlocks(t *testing.T) {
|
||||||
|
block := func(name string) (string, error) {
|
||||||
|
if name != "news_list" {
|
||||||
|
return "", errors.New("unknown block " + name)
|
||||||
|
}
|
||||||
|
return "=> /a 2026-01-01 A\n=> /b 2025-12-31 B", nil
|
||||||
|
}
|
||||||
|
got, err := Convert(`<p>intro</p><gemblock name="news_list"></gemblock><p>end</p>`, Options{Link: identity, Block: block})
|
||||||
|
if err != nil || got != "intro\n\n=> /a 2026-01-01 A\n=> /b 2025-12-31 B\n\nend\n" {
|
||||||
|
t.Errorf("Convert() = %q, %v", got, err)
|
||||||
|
}
|
||||||
|
if _, err := Convert(`<gemblock name="other"></gemblock>`, Options{Link: identity, Block: block}); err == nil {
|
||||||
|
t.Error("an unknown block is not an error")
|
||||||
|
}
|
||||||
|
if _, err := Convert(`<gemblock name="news_list"></gemblock>`, Options{Link: identity}); err == nil {
|
||||||
|
t.Error("a block without a Block function is not an error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestConvertRejectsBrokenHTML checks that markup that cannot be read is an error.
|
||||||
|
func TestConvertRejectsBrokenHTML(t *testing.T) {
|
||||||
|
if _, err := Convert(`<p>unclosed`, Options{Link: identity}); err == nil || !strings.Contains(err.Error(), "HTML") {
|
||||||
|
t.Errorf("error = %v, want one about the HTML", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -35,6 +35,8 @@ type Lang struct {
|
||||||
Strings map[string]string
|
Strings map[string]string
|
||||||
// Nav maps a menu entry to its label in this language.
|
// Nav maps a menu entry to its label in this language.
|
||||||
Nav map[string]string
|
Nav map[string]string
|
||||||
|
// NavOrder lists the menu entries in the order the file writes them.
|
||||||
|
NavOrder []string
|
||||||
}
|
}
|
||||||
|
|
||||||
// shownKeys are the address keys that get a harvest-proof "_shown" twin.
|
// shownKeys are the address keys that get a harvest-proof "_shown" twin.
|
||||||
|
|
@ -144,9 +146,35 @@ func loadLang(dir, code string) (Lang, error) {
|
||||||
if err := decodeField(raw, name, "nav", &lang.Nav); err != nil {
|
if err := decodeField(raw, name, "nav", &lang.Nav); err != nil {
|
||||||
return Lang{}, err
|
return Lang{}, err
|
||||||
}
|
}
|
||||||
|
order, err := keyOrder(raw["nav"])
|
||||||
|
if err != nil {
|
||||||
|
return Lang{}, fmt.Errorf("%s: nav: %w", name, err)
|
||||||
|
}
|
||||||
|
lang.NavOrder = order
|
||||||
return lang, nil
|
return lang, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// keyOrder returns the keys of a JSON object in the order they are written.
|
||||||
|
func keyOrder(object json.RawMessage) ([]string, error) {
|
||||||
|
decoder := json.NewDecoder(strings.NewReader(string(object)))
|
||||||
|
if _, err := decoder.Token(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var keys []string
|
||||||
|
for decoder.More() {
|
||||||
|
key, err := decoder.Token()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
keys = append(keys, key.(string))
|
||||||
|
var skipped json.RawMessage
|
||||||
|
if err := decoder.Decode(&skipped); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return keys, nil
|
||||||
|
}
|
||||||
|
|
||||||
// readObject reads a JSON file that holds one object and returns its fields
|
// readObject reads a JSON file that holds one object and returns its fields
|
||||||
// undecoded. Errors name the file.
|
// undecoded. Errors name the file.
|
||||||
func readObject(dir, name string) (map[string]json.RawMessage, error) {
|
func readObject(dir, name string) (map[string]json.RawMessage, error) {
|
||||||
|
|
|
||||||
|
|
@ -184,3 +184,24 @@ func TestLoadNeedsLanguages(t *testing.T) {
|
||||||
t.Errorf("Load() error = %v, want it to mention languages", err)
|
t.Errorf("Load() error = %v, want it to mention languages", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestLoadKeepsTheOrderOfTheMenu checks that the entries of nav are listed in the
|
||||||
|
// order the language file writes them, which a map cannot tell.
|
||||||
|
func TestLoadKeepsTheOrderOfTheMenu(t *testing.T) {
|
||||||
|
files := validFiles()
|
||||||
|
files["lang/en.json"] = `{"lang_name": "English", "nav": {"manifesto": "Manifesto", "people": "People", "news": "News", "services": "Services"}}`
|
||||||
|
_, langs, err := Load(writeSite(t, files))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() error = %v", err)
|
||||||
|
}
|
||||||
|
want := []string{"manifesto", "people", "news", "services"}
|
||||||
|
got := langs["en"].NavOrder
|
||||||
|
if len(got) != len(want) {
|
||||||
|
t.Fatalf("NavOrder = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
for i := range want {
|
||||||
|
if got[i] != want[i] {
|
||||||
|
t.Fatalf("NavOrder = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue