From 28745d452f923081f68983679a92772fe948d53e Mon Sep 17 00:00:00 2001 From: tgiachi Date: Mon, 16 Nov 2026 12:19:38 +0100 Subject: [PATCH] feat(gemtext): convert the HTML of a page to gemtext A new package, with no new dependency, that reads the HTML a page renders to (the lenient mode of encoding/xml with the HTML void elements and entities) and writes gemtext: three heading levels, one line paragraphs, the links of a paragraph, a list, a quote or a table on lines of their own after it, bullets (a list of only links is link lines), quotes (a note is a quote), preformatted blocks, tables as aligned preformatted blocks, images as link lines, no emphasis, and lines that look like gemtext are protected. Options.Link maps or drops an address and its error stops the conversion; Options.Block expands the placeholders of generated blocks. The language files also keep the order of the menu now (Lang.NavOrder), which the Gemini output needs. Refs #10 --- internal/gemtext/gemtext.go | 637 +++++++++++++++++++++++++++++++ internal/gemtext/gemtext_test.go | 119 ++++++ internal/site/site.go | 28 ++ internal/site/site_test.go | 21 + 4 files changed, 805 insertions(+) create mode 100644 internal/gemtext/gemtext.go create mode 100644 internal/gemtext/gemtext_test.go diff --git a/internal/gemtext/gemtext.go b/internal/gemtext/gemtext.go new file mode 100644 index 0000000..395c942 --- /dev/null +++ b/internal/gemtext/gemtext.go @@ -0,0 +1,637 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +// Package gemtext turns the HTML of a page into gemtext, the format of the +// Gemini protocol. Gemtext has headings, one-line paragraphs, bullet items, +// quotes, preformatted blocks and links on lines of their own, and nothing +// else: no emphasis, no links inside text, no tables, no images. +package gemtext + +import ( + "encoding/xml" + "fmt" + "io" + "regexp" + "strconv" + "strings" + "unicode/utf8" +) + +// Options tell Convert how to treat the addresses and the generated blocks. +type Options struct { + // Link turns an address found in the HTML (a link or an image) into the + // address to write. An empty result drops the link and keeps its text, as + // for "#section"; an error stops the conversion and names the address. A + // nil Link leaves every address as it is. + Link func(href string) (string, error) + // Block returns the gemtext of a block placeholder, the element + // . A nil Block makes such an element an error. + Block func(name string) (string, error) +} + +// node is an element or a piece of text of the parsed HTML. +type node struct { + tag string + text string + attrs map[string]string + kids []*node +} + +// link is an address with the text that labels it. +type link struct{ target, label string } + +// block is a part of the output: lines of text followed by link lines. +type block struct { + text []string + links []link +} + +// converter holds the options of one conversion. +type converter struct{ opts Options } + +// Convert converts html, the body of a page, to gemtext. Blocks are separated +// by a blank line; the links of a paragraph, a list, a quote or a table come +// right after it, each on a line of its own, and the output ends with a line +// break unless it is empty. +func Convert(html string, opts Options) (string, error) { + root, err := parse(html) + if err != nil { + return "", err + } + c := &converter{opts: opts} + blocks, err := c.blocks(root.kids) + if err != nil { + return "", err + } + parts := make([]string, 0, len(blocks)) + for _, b := range blocks { + if text := b.String(); text != "" { + parts = append(parts, text) + } + } + if len(parts) == 0 { + return "", nil + } + return strings.Join(parts, "\n\n") + "\n", nil +} + +// String returns the lines of the block, text first, then the link lines. +func (b block) String() string { + lines := append([]string(nil), b.text...) + seen := map[link]bool{} + for _, l := range b.links { + if seen[l] { + continue + } + seen[l] = true + lines = append(lines, linkLine(l)) + } + return strings.Join(lines, "\n") +} + +// linkLine writes a link on its own line. +func linkLine(l link) string { + if l.label == "" || l.label == l.target { + return "=> " + l.target + } + return "=> " + l.target + " " + l.label +} + +// voidTags are the elements that have no end tag. +var voidTags = map[string]bool{ + "area": true, "base": true, "br": true, "col": true, "hr": true, "img": true, "input": true, + "link": true, "meta": true, "param": true, "source": true, "basefont": true, "frame": true, "isindex": true, +} + +// parse reads a fragment of HTML into a tree. The decoder is the lenient mode +// of encoding/xml with the HTML void elements and entities: the HTML of a page +// comes from a Markdown renderer and from the author's own raw HTML, which are +// well formed apart from the void elements. +func parse(html string) (*node, error) { + source := "" + html + "" + decoder := xml.NewDecoder(strings.NewReader(source)) + decoder.Strict = false + decoder.AutoClose = xml.HTMLAutoClose + decoder.Entity = xml.HTMLEntity + root := &node{tag: "root"} + stack := []*node{root} + for { + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return nil, fmt.Errorf("read the HTML: %w", err) + } + top := stack[len(stack)-1] + switch t := token.(type) { + case xml.StartElement: + n := &node{tag: strings.ToLower(t.Name.Local), attrs: map[string]string{}} + for _, a := range t.Attr { + n.attrs[strings.ToLower(a.Name.Local)] = a.Value + } + top.kids = append(top.kids, n) + stack = append(stack, n) + case xml.EndElement: + // The lenient mode closes by itself what the text left open; that is + // a mistake in the page, so an end that the text does not have is an error. + name := strings.ToLower(t.Name.Local) + written := strings.ToLower(source[:decoder.InputOffset()]) + if !voidTags[name] && !strings.HasSuffix(written, "") && !strings.HasSuffix(written, "/>") { + return nil, fmt.Errorf("read the HTML: the element <%s> is not closed", name) + } + if len(stack) > 1 { + stack = stack[:len(stack)-1] + } + case xml.CharData: + top.kids = append(top.kids, &node{text: string(t)}) + } + } + if len(stack) != 1 || len(root.kids) != 1 { + return nil, fmt.Errorf("read the HTML: the element <%s> is not closed", stack[len(stack)-1].tag) + } + return root.kids[0], nil +} + +// blockTags are the elements that start a block of their own. +var blockTags = map[string]bool{ + "h1": true, "h2": true, "h3": true, "h4": true, "h5": true, "h6": true, "p": true, + "ul": true, "ol": true, "blockquote": true, "pre": true, "hr": true, "table": true, + "div": true, "section": true, "article": true, "main": true, "header": true, "footer": true, + "nav": true, "aside": true, "figure": true, "figcaption": true, "details": true, "summary": true, + "dl": true, "dt": true, "dd": true, "form": true, "fieldset": true, "address": true, "gemblock": true, +} + +// skippedTags are the elements that have nothing to read. +var skippedTags = map[string]bool{ + "script": true, "style": true, "svg": true, "noscript": true, "template": true, + "head": true, "title": true, "meta": true, "link": true, +} + +// skipped reports whether an element is left out: one with nothing to read, or +// one that is hidden. +func skipped(n *node) bool { + _, hidden := n.attrs["hidden"] + return skippedTags[n.tag] || hidden || n.attrs["aria-hidden"] == "true" +} + +// blocks converts a list of nodes into blocks: block elements make their own, +// and the text and the inline elements between them make paragraphs. +func (c *converter) blocks(nodes []*node) ([]block, error) { + var out []block + var run []*node + flush := func() error { + if len(run) == 0 { + return nil + } + b, err := c.paragraph(run) + run = nil + if err != nil { + return err + } + out = append(out, b) + return nil + } + for _, n := range nodes { + if n.tag != "" && skipped(n) { + continue + } + if n.tag == "" || !blockTags[n.tag] { + run = append(run, n) + continue + } + if err := flush(); err != nil { + return nil, err + } + got, err := c.element(n) + if err != nil { + return nil, err + } + out = append(out, got...) + } + if err := flush(); err != nil { + return nil, err + } + return out, nil +} + +// element converts one block element. +func (c *converter) element(n *node) ([]block, error) { + switch n.tag { + case "h1", "h2", "h3", "h4", "h5", "h6": + return c.heading(n) + case "p", "summary", "figcaption", "dt", "dd": + if strings.Contains(" "+n.attrs["class"]+" ", " note ") || n.attrs["role"] == "note" { + return c.quote(n.kids) + } + b, err := c.paragraph(n.kids) + return []block{b}, err + case "ul", "ol": + b, err := c.list(n) + return []block{b}, err + case "blockquote": + return c.quote(n.kids) + case "pre": + return []block{preformatted(n)}, nil + case "hr": + return nil, nil + case "table": + b, err := c.table(n) + return []block{b}, err + case "gemblock": + return c.generated(n) + default: + return c.blocks(n.kids) + } +} + +// generated asks Options.Block for the gemtext of a block placeholder. +func (c *converter) generated(n *node) ([]block, error) { + name := n.attrs["name"] + if c.opts.Block == nil { + return nil, fmt.Errorf("the block %q has no generator", name) + } + text, err := c.opts.Block(name) + if err != nil { + return nil, fmt.Errorf("block %q: %w", name, err) + } + text = strings.Trim(text, "\n") + if text == "" { + return nil, nil + } + return []block{{text: strings.Split(text, "\n")}}, nil +} + +// heading converts h1 to h6; gemtext has three levels. +func (c *converter) heading(n *node) ([]block, error) { + text, links, err := c.inline(n.kids, " ") + if err != nil { + return nil, err + } + text = strings.Join(strings.Fields(text), " ") + if text == "" { + return nil, nil + } + level := int(n.tag[1] - '0') + if level > 3 { + level = 3 + } + return []block{{text: []string{strings.Repeat("#", level) + " " + text}, links: links}}, nil +} + +// paragraph converts a run of text and inline elements. Each line break is a +// line of its own, and the links come after. A paragraph that is only a link +// is only the link line. +func (c *converter) paragraph(nodes []*node) (block, error) { + text, links, err := c.inline(nodes, "\n") + if err != nil { + return block{}, err + } + lines := cleanLines(text) + if len(lines) == 1 && len(links) == 1 && lines[0] == links[0].label { + return block{links: links}, nil + } + for i := range lines { + lines[i] = protect(lines[i]) + } + return block{text: lines, links: links}, nil +} + +// cleanLines splits text at its line breaks, collapses the white space of every +// line and drops the empty ones. +func cleanLines(text string) []string { + var lines []string + for _, line := range strings.Split(text, "\n") { + if line = strings.Join(strings.Fields(line), " "); line != "" { + lines = append(lines, line) + } + } + return lines +} + +// protect keeps a line of text from being read as another kind of line: a +// leading space does not change what it says and gemtext has no escape. +func protect(line string) string { + for _, prefix := range []string{"=>", "#", ">", "```", "* "} { + if strings.HasPrefix(line, prefix) { + return " " + line + } + } + return line +} + +// space matches a run of white space, the no-break space included. +var space = regexp.MustCompile(`[\s\x{00a0}]+`) + +// inline returns the text of the nodes, the links in them and the addresses of +// their images. lineBreak is what a
becomes. +func (c *converter) inline(nodes []*node, lineBreak string) (string, []link, error) { + var text strings.Builder + var links []link + var walk func(n *node) error + walk = func(n *node) error { + if n.tag == "" { + text.WriteString(space.ReplaceAllString(n.text, " ")) + return nil + } + if skipped(n) { + return nil + } + switch n.tag { + case "br": + text.WriteString(lineBreak) + return nil + case "img": + alt := strings.Join(strings.Fields(n.attrs["alt"]), " ") + if alt == "" || n.attrs["src"] == "" { + return nil + } + target, err := c.address(n.attrs["src"]) + if err != nil { + return err + } + if target != "" { + links = append(links, link{target, alt}) + } + return nil + case "a": + label, inner, err := c.inline(n.kids, " ") + if err != nil { + return err + } + text.WriteString(label) + links = append(links, inner...) + if href := n.attrs["href"]; href != "" { + target, err := c.address(href) + if err != nil { + return err + } + if target != "" { + links = append(links, link{target, strings.Join(strings.Fields(label), " ")}) + } + } + return nil + } + for _, kid := range n.kids { + if err := walk(kid); err != nil { + return err + } + } + return nil + } + for _, n := range nodes { + if err := walk(n); err != nil { + return "", nil, err + } + } + return text.String(), links, nil +} + +// address maps an address through Options.Link. +func (c *converter) address(href string) (string, error) { + if c.opts.Link == nil { + return href, nil + } + target, err := c.opts.Link(href) + if err != nil { + return "", fmt.Errorf("the address %q: %w", href, err) + } + return target, nil +} + +// item is one bullet of a list. +type item struct { + text string + links []link +} + +// list converts a list: bullets, with the links after them. A list in which +// every item is only a link is a list of link lines. A list inside an item is +// flattened into the same list. An ordered list numbers its items. +func (c *converter) list(n *node) (block, error) { + var items []item + if err := c.items(n, &items); err != nil { + return block{}, err + } + linksOnly := len(items) > 0 + var links []link + for _, it := range items { + links = append(links, it.links...) + if len(it.links) != 1 || it.text != it.links[0].label { + linksOnly = false + } + } + if linksOnly { + return block{links: links}, nil + } + lines := make([]string, len(items)) + for i, it := range items { + prefix := "* " + if n.tag == "ol" { + prefix += strconv.Itoa(i+1) + ". " + } + lines[i] = prefix + it.text + } + return block{text: lines, links: links}, nil +} + +// items collects the items of a list and of the lists inside it. +func (c *converter) items(list *node, out *[]item) error { + for _, li := range list.kids { + if li.tag != "li" || skipped(li) { + continue + } + var own, nested []*node + for _, kid := range li.kids { + if kid.tag == "ul" || kid.tag == "ol" { + nested = append(nested, kid) + } else { + own = append(own, kid) + } + } + text, links, err := c.inline(own, " ") + if err != nil { + return err + } + if text = strings.Join(strings.Fields(text), " "); text != "" || len(links) > 0 { + *out = append(*out, item{text: text, links: links}) + } + for _, inner := range nested { + if err := c.items(inner, out); err != nil { + return err + } + } + } + return nil +} + +// quote converts the content of a quote: every line starts with "> ", the +// blocks are separated by an empty quote line, and the links come after. +func (c *converter) quote(nodes []*node) ([]block, error) { + inner, err := c.blocks(nodes) + if err != nil { + return nil, err + } + var lines []string + var links []link + for _, b := range inner { + if len(b.text) == 0 { + links = append(links, b.links...) + continue + } + if len(lines) > 0 { + lines = append(lines, ">") + } + for _, line := range b.text { + lines = append(lines, "> "+strings.TrimPrefix(line, " ")) + } + links = append(links, b.links...) + } + if len(lines) == 0 && len(links) == 0 { + return nil, nil + } + return []block{{text: lines, links: links}}, nil +} + +// preformatted converts a
: its text as it is between two fence lines. A
+// line that would close the block early gets a space in front.
+func preformatted(n *node) block {
+	var text strings.Builder
+	var walk func(n *node)
+	walk = func(n *node) {
+		if n.tag == "" {
+			text.WriteString(n.text)
+			return
+		}
+		if n.tag == "br" {
+			text.WriteString("\n")
+			return
+		}
+		for _, kid := range n.kids {
+			walk(kid)
+		}
+	}
+	walk(n)
+	lines := strings.Split(strings.TrimSuffix(text.String(), "\n"), "\n")
+	out := []string{"```"}
+	for _, line := range lines {
+		if strings.HasPrefix(line, "```") {
+			line = " " + line
+		}
+		out = append(out, strings.TrimRight(line, " \t"))
+	}
+	return block{text: append(out, "```")}
+}
+
+// table converts a table to a preformatted block of aligned columns, with a
+// line under a header row, and the caption as a line of text before it. The
+// links of the cells come after the block.
+func (c *converter) table(n *node) (block, error) {
+	var rows [][]string
+	var links []link
+	var caption string
+	headerRow := false
+	var walk func(n *node) error
+	walk = func(n *node) error {
+		for _, kid := range n.kids {
+			if kid.tag == "" || skipped(kid) {
+				continue
+			}
+			switch kid.tag {
+			case "caption":
+				text, inner, err := c.inline(kid.kids, " ")
+				if err != nil {
+					return err
+				}
+				caption = strings.Join(strings.Fields(text), " ")
+				links = append(links, inner...)
+			case "tr":
+				var cells []string
+				onlyHeaders := true
+				for _, cell := range kid.kids {
+					if cell.tag != "td" && cell.tag != "th" {
+						continue
+					}
+					text, inner, err := c.inline(cell.kids, " ")
+					if err != nil {
+						return err
+					}
+					cells = append(cells, strings.Join(strings.Fields(text), " "))
+					links = append(links, inner...)
+					if cell.tag != "th" {
+						onlyHeaders = false
+					}
+				}
+				if len(cells) > 0 {
+					if len(rows) == 0 && onlyHeaders {
+						headerRow = true
+					}
+					rows = append(rows, cells)
+				}
+			default:
+				if err := walk(kid); err != nil {
+					return err
+				}
+			}
+		}
+		return nil
+	}
+	if err := walk(n); err != nil {
+		return block{}, err
+	}
+	var text []string
+	if caption != "" {
+		text = append(text, protect(caption))
+	}
+	if len(rows) > 0 {
+		text = append(text, "```")
+		text = append(text, alignRows(rows, headerRow)...)
+		text = append(text, "```")
+	}
+	return block{text: text, links: links}, nil
+}
+
+// alignRows writes rows of cells as columns two spaces apart, each as wide as
+// its widest cell, with a line of dashes under the first row when it is a header.
+func alignRows(rows [][]string, header bool) []string {
+	columns := 0
+	for _, r := range rows {
+		if len(r) > columns {
+			columns = len(r)
+		}
+	}
+	widths := make([]int, columns)
+	for _, r := range rows {
+		for i, cell := range r {
+			if w := utf8.RuneCountInString(cell); w > widths[i] {
+				widths[i] = w
+			}
+		}
+	}
+	write := func(r []string) string {
+		cells := make([]string, columns)
+		for i := range cells {
+			cell := ""
+			if i < len(r) {
+				cell = r[i]
+			}
+			cells[i] = cell + strings.Repeat(" ", widths[i]-utf8.RuneCountInString(cell))
+		}
+		line := strings.TrimRight(strings.Join(cells, "  "), " ")
+		if strings.HasPrefix(line, "```") {
+			line = " " + line
+		}
+		return line
+	}
+	var lines []string
+	for i, r := range rows {
+		lines = append(lines, write(r))
+		if i == 0 && header {
+			dashes := make([]string, columns)
+			for j := range dashes {
+				dashes[j] = strings.Repeat("-", widths[j])
+			}
+			lines = append(lines, strings.Join(dashes, "  "))
+		}
+	}
+	return lines
+}
diff --git a/internal/gemtext/gemtext_test.go b/internal/gemtext/gemtext_test.go
new file mode 100644
index 0000000..d028076
--- /dev/null
+++ b/internal/gemtext/gemtext_test.go
@@ -0,0 +1,119 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package gemtext
+
+import (
+	"errors"
+	"strings"
+	"testing"
+)
+
+// identity leaves every address as it is.
+func identity(href string) (string, error) { return href, nil }
+
+// convert converts html with addresses left as they are and no blocks.
+func convert(t *testing.T, html string) string {
+	t.Helper()
+	got, err := Convert(html, Options{Link: identity})
+	if err != nil {
+		t.Fatalf("Convert(%q) error = %v", html, err)
+	}
+	return got
+}
+
+// TestConvert checks the gemtext of every kind of element the site uses.
+func TestConvert(t *testing.T) {
+	tests := []struct {
+		name, html, want string
+	}{
+		{"headings", `

Title

Sub

Deep

`, "# Title\n\n## Sub\n\n### Deep\n"}, + {"hero with line breaks", `

Technology.
Freedom.
Dissent.

`, "# Technology. Freedom. Dissent.\n"}, + {"links go after the paragraph", + `

Read the first and the second page.

`, + "Read the first and the second page.\n=> /a.html first\n=> https://x.org/b second\n"}, + {"the same link once", `

A and A again.

`, "A and A again.\n=> /a A\n"}, + {"a paragraph that is only a link", `

All news

`, "=> /x.html All news\n"}, + {"line breaks", `

One
Two
Three

`, "One\nTwo\nThree\n"}, + {"whitespace is collapsed", "

a\n b\t c

", "a b c\n"}, + {"entities", `

Tom & Jerry © <ok>

`, "Tom & Jerry © \n"}, + {"unordered list", ``, "* One\n* Two link\n=> /c link\n"}, + {"a list of only links has no bullets", ``, "=> /a A\n=> /b B\n"}, + {"nested list is flattened", `
  • A
    • A1
  • B
`, "* A\n* A1\n* B\n"}, + {"ordered list", `
  1. First
  2. Second
`, "* 1. First\n* 2. Second\n"}, + {"quote", `

One

Two x

`, "> One\n>\n> Two x\n=> /x x\n"}, + {"note is a quote", `

Careful

`, "> Careful\n"}, + {"preformatted", "
line1\n  line2\n```\n
", "```\nline1\n line2\n ```\n```\n"}, + {"table with a header", + `
Data
ServiceKept
Mail30 days
Chat1 year
`, + "Data\n```\nService Kept\n------- -------\nMail 30 days\nChat 1 year\n```\n"}, + {"table without a header", `
abb
cccd
`, "```\na bb\nccc d\n```\n"}, + {"table links", `
Mailx
`, "```\nMail x\n```\n=> /m Mail\n"}, + {"image with a description", `

kmalloc

`, "=> /logo.svg kmalloc\n"}, + {"image without a description is skipped", `

`, ""}, + {"lines that look like gemtext are protected", `

# no

=> no

* no

> no

`, " # no\n\n => no\n\n * no\n\n > no\n"}, + {"hidden and scripted things are skipped", `

kept

`, "kept\n"}, + {"containers are transparent", `

A

Loose text
`, "A\n\nLoose text\n"}, + {"time and span are inline", `

by squid

`, "14 November 2026 by squid\n"}, + {"empty input", ``, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := convert(t, tt.html); got != tt.want { + t.Errorf("Convert() =\n%q\nwant\n%q", got, tt.want) + } + }) + } +} + +// TestConvertLinkMapping checks that Link rewrites and drops addresses, and that its error stops the conversion. +func TestConvertLinkMapping(t *testing.T) { + link := func(href string) (string, error) { + switch { + case strings.HasPrefix(href, "#"): + return "", nil + case href == "/broken.html": + return "", errors.New("no such page") + } + target, _, _ := strings.Cut(href, "#") + return strings.Replace(target, ".html", ".gmi", 1), nil + } + got, err := Convert(`

See a and top.

`, Options{Link: link}) + if err != nil || got != "See a and top.\n=> /a.gmi a\n" { + t.Errorf("Convert() = %q, %v", got, err) + } + _, err = Convert(`

x

`, Options{Link: link}) + if err == nil || !strings.Contains(err.Error(), "/broken.html") || !strings.Contains(err.Error(), "no such page") { + t.Errorf("error = %v, want one that names the address and the cause", err) + } + _, err = Convert(`

pic

`, Options{Link: link}) + if err == nil { + t.Error("a broken image address is not an error") + } +} + +// TestConvertBlocks checks the placeholders of generated blocks. +func TestConvertBlocks(t *testing.T) { + block := func(name string) (string, error) { + if name != "news_list" { + return "", errors.New("unknown block " + name) + } + return "=> /a 2026-01-01 A\n=> /b 2025-12-31 B", nil + } + got, err := Convert(`

intro

end

`, Options{Link: identity, Block: block}) + if err != nil || got != "intro\n\n=> /a 2026-01-01 A\n=> /b 2025-12-31 B\n\nend\n" { + t.Errorf("Convert() = %q, %v", got, err) + } + if _, err := Convert(``, Options{Link: identity, Block: block}); err == nil { + t.Error("an unknown block is not an error") + } + if _, err := Convert(``, Options{Link: identity}); err == nil { + t.Error("a block without a Block function is not an error") + } +} + +// TestConvertRejectsBrokenHTML checks that markup that cannot be read is an error. +func TestConvertRejectsBrokenHTML(t *testing.T) { + if _, err := Convert(`

unclosed`, Options{Link: identity}); err == nil || !strings.Contains(err.Error(), "HTML") { + t.Errorf("error = %v, want one about the HTML", err) + } +} diff --git a/internal/site/site.go b/internal/site/site.go index 98e5549..586b225 100644 --- a/internal/site/site.go +++ b/internal/site/site.go @@ -35,6 +35,8 @@ type Lang struct { Strings map[string]string // Nav maps a menu entry to its label in this language. Nav map[string]string + // NavOrder lists the menu entries in the order the file writes them. + NavOrder []string } // shownKeys are the address keys that get a harvest-proof "_shown" twin. @@ -144,9 +146,35 @@ func loadLang(dir, code string) (Lang, error) { if err := decodeField(raw, name, "nav", &lang.Nav); err != nil { return Lang{}, err } + order, err := keyOrder(raw["nav"]) + if err != nil { + return Lang{}, fmt.Errorf("%s: nav: %w", name, err) + } + lang.NavOrder = order return lang, nil } +// keyOrder returns the keys of a JSON object in the order they are written. +func keyOrder(object json.RawMessage) ([]string, error) { + decoder := json.NewDecoder(strings.NewReader(string(object))) + if _, err := decoder.Token(); err != nil { + return nil, err + } + var keys []string + for decoder.More() { + key, err := decoder.Token() + if err != nil { + return nil, err + } + keys = append(keys, key.(string)) + var skipped json.RawMessage + if err := decoder.Decode(&skipped); err != nil { + return nil, err + } + } + return keys, nil +} + // readObject reads a JSON file that holds one object and returns its fields // undecoded. Errors name the file. func readObject(dir, name string) (map[string]json.RawMessage, error) { diff --git a/internal/site/site_test.go b/internal/site/site_test.go index 13261c2..b65d28a 100644 --- a/internal/site/site_test.go +++ b/internal/site/site_test.go @@ -184,3 +184,24 @@ func TestLoadNeedsLanguages(t *testing.T) { t.Errorf("Load() error = %v, want it to mention languages", err) } } + +// TestLoadKeepsTheOrderOfTheMenu checks that the entries of nav are listed in the +// order the language file writes them, which a map cannot tell. +func TestLoadKeepsTheOrderOfTheMenu(t *testing.T) { + files := validFiles() + files["lang/en.json"] = `{"lang_name": "English", "nav": {"manifesto": "Manifesto", "people": "People", "news": "News", "services": "Services"}}` + _, langs, err := Load(writeSite(t, files)) + if err != nil { + t.Fatalf("Load() error = %v", err) + } + want := []string{"manifesto", "people", "news", "services"} + got := langs["en"].NavOrder + if len(got) != len(want) { + t.Fatalf("NavOrder = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("NavOrder = %v, want %v", got, want) + } + } +}