motorino/internal/build/robots.go

27 lines
691 B
Go

// SPDX-License-Identifier: GPL-3.0-or-later
package build
import (
"sort"
"strings"
)
// robots returns robots.txt: it allows each address exactly ("$" anchors the
// end) and blocks everything else, so downloads and assets are left out.
func robots(addresses []string) string {
allow := map[string]bool{}
for _, a := range addresses {
allow["Allow: "+a+"$"] = true
if a == "/" {
// The English home as the other languages link to it.
allow["Allow: /?lang=en$"] = true
}
}
lines := make([]string, 0, len(allow))
for line := range allow {
lines = append(lines, line)
}
sort.Strings(lines)
return "User-agent: *\n" + strings.Join(lines, "\n") + "\nDisallow: /\n"
}