27 lines
691 B
Go
27 lines
691 B
Go
// SPDX-License-Identifier: GPL-3.0-or-later
|
|
|
|
package build
|
|
|
|
import (
|
|
"sort"
|
|
"strings"
|
|
)
|
|
|
|
// robots returns robots.txt: it allows each address exactly ("$" anchors the
|
|
// end) and blocks everything else, so downloads and assets are left out.
|
|
func robots(addresses []string) string {
|
|
allow := map[string]bool{}
|
|
for _, a := range addresses {
|
|
allow["Allow: "+a+"$"] = true
|
|
if a == "/" {
|
|
// The English home as the other languages link to it.
|
|
allow["Allow: /?lang=en$"] = true
|
|
}
|
|
}
|
|
lines := make([]string, 0, len(allow))
|
|
for line := range allow {
|
|
lines = append(lines, line)
|
|
}
|
|
sort.Strings(lines)
|
|
return "User-agent: *\n" + strings.Join(lines, "\n") + "\nDisallow: /\n"
|
|
}
|