// Package providers manages the external services Sonda can ask about artists // (Last.fm, MusicBrainz) and the rules for talking to them. // // Nothing here makes a request on its own. Every provider is off until an admin // turns it on, and the settings (including any API key) live in the database. package providers import ( "context" "errors" "fmt" "net/url" "regexp" "slices" "strings" "sync" "time" "unicode" "unicode/utf8" ) // The providers Sonda knows. const ( LastFM = "lastfm" MusicBrainz = "musicbrainz" Wikipedia = "wikipedia" Commons = "commons" CoverArtArchive = "coverartarchive" Lidarr = "lidarr" Anthropic = "anthropic" ) // Errors returned by the service. var ( ErrUnknownProvider = errors.New("unknown provider") ErrInvalidSettings = errors.New("invalid provider settings") ErrNoTest = errors.New("this provider has no test") ) // What a checker says went wrong, so that the result of a test does not depend // on the client of any one provider. var ( ErrKeyRefused = errors.New("the provider refused the API key") ErrRateLimited = errors.New("the provider's rate limit was reached") ErrUnreachable = errors.New("the provider could not be reached") ) const ( maxContactLen = 200 maxURLLen = 300 maxFolderLen = 500 ) var lastfmKeyRe = regexp.MustCompile(`^[0-9a-f]{32}$`) // An Anthropic API key: sk-ant-, then letters, digits, - and _ (the part after the prefix is long). var anthropicKeyRe = regexp.MustCompile(`^sk-ant-[A-Za-z0-9_-]{20,300}$`) // A Lidarr API key is 32 hexadecimal characters too. var lidarrKeyRe = lastfmKeyRe // LidarrMonitors are the values of the monitor option that Lidarr accepts when an artist is added. var LidarrMonitors = []string{"all", "future", "missing", "existing", "latest", "first", "none"} // LidarrOptions are the choices an admin makes once for what is sent to Lidarr when an artist // is added. The profiles and the folder are Lidarr's own: the admin reads them from // GET /providers/lidarr/options and picks. type LidarrOptions struct { QualityProfileID int MetadataProfileID int RootFolderPath string Monitor string // one of LidarrMonitors SearchMissing bool // Lidarr starts looking for the albums at once } // Complete says whether an artist can be added with these options: Lidarr needs both profiles and a // folder, and Sonda does not guess them. func (o LidarrOptions) Complete() bool { return o.QualityProfileID > 0 && o.MetadataProfileID > 0 && o.RootFolderPath != "" } // withDefaults is what a setup that was never saved starts from. func (o LidarrOptions) withDefaults() LidarrOptions { if o.Monitor == "" { o.Monitor, o.SearchMissing = "all", true } return o } // Settings are the stored settings of one provider, API key included. They are // only for code that talks to the provider; the API never returns the key. type Settings struct { Provider string Enabled bool APIKey string Contact string // goes into the User-Agent; empty means the project address URL string // where the provider is, for those that are the user's own (Lidarr) Lidarr LidarrOptions UpdatedAt time.Time } // Info is what the API shows about a provider: the settings without the key, // plus a plain statement of what leaves the server. type Info struct { Name string Enabled bool NeedsAPIKey bool NeedsURL bool HasAPIKey bool Contact string URL string // only for a provider that has one Lidarr LidarrOptions // only for Lidarr; what an unsaved setup starts from until it is set Sends string // what is sent to the provider, and what never is Notes string // terms of use and attribution UpdatedAt time.Time // zero when the provider was never configured } // Patch changes some settings. A nil field is left alone. An empty APIKey, Contact or URL // clears it. Lidarr replaces all the options of Lidarr at once. type Patch struct { Enabled *bool APIKey *string Contact *string URL *string Lidarr *LidarrOptions } // Store persists provider settings. type Store interface { // Get returns found=false when the provider was never configured. Get(ctx context.Context, name string) (s Settings, found bool, err error) List(ctx context.Context) ([]Settings, error) Save(ctx context.Context, s Settings) error } type meta struct { name string needsKey bool needsURL bool sends, note string } // known lists the providers in the order the API shows them. var known = []meta{ { name: LastFM, needsKey: true, sends: "The artist name (or MusicBrainz id) you look up, your API key, your IP address and the " + "User-Agent. Never your users, your listens, track titles or file paths.", note: "Last.fm data is for non-commercial use only. Credit Last.fm when you show it. " + "Images are not licensed and are never stored.", }, { name: MusicBrainz, sends: "The MusicBrainz id of the artist you look up (never an artist name: Sonda does not search for artists). " + "Only when the Cover Art Archive provider is on too and an album has no picture and no id in its files: the name " + "of the album's artist and its title, to find the album (a result is taken only when both match exactly). " + "Your IP address and the User-Agent, which carries the contact below. Never your users, your listens " + "or file paths.", note: "Core data (artist names and ids) is public domain. Tags and genres are CC BY-NC-SA 3.0: " + "non-commercial, with attribution to MusicBrainz.", }, { name: Wikipedia, sends: "It needs the MusicBrainz provider on too. The MusicBrainz id of the artist you look up goes to MusicBrainz " + "(never a name: Sonda does not search), with the contact set for MusicBrainz; the Wikidata item that " + "MusicBrainz links to it goes to Wikidata, and the title of the Wikipedia page that Wikidata names goes to " + "Wikipedia, with the contact below in the User-Agent. Your IP address goes to all three. Never your users, " + "your listens, track titles or file paths.", note: "Wikipedia text is CC BY-SA 4.0: show the name of the page, a link to it and the licence next to " + "the text, and say if you changed it.", }, { name: Commons, sends: "It needs the MusicBrainz provider on too. The MusicBrainz id of the artist you look up goes to MusicBrainz " + "(never a name: Sonda does not search), with the contact set for MusicBrainz; the Wikidata item that " + "MusicBrainz links to it goes to Wikidata, the name of the picture Wikidata gives it goes to Wikimedia " + "Commons, and a copy of the picture is downloaded from Wikimedia, with the contact below in the User-Agent. " + "Your IP address goes to all of them. Never your users, your listens, track titles or file paths.", note: "Only pictures under free licences are kept (public domain, CC0, CC BY, CC BY-SA); the author, the licence " + "and the page of the file are stored with the picture and must be shown next to it.", }, { name: CoverArtArchive, sends: "Only when an album has no picture in your own files, when someone opens its cover or an admin starts the " + "covers job (which goes through every album): the MusicBrainz release group id that the tags of " + "its files carry, your IP address and the User-Agent, which carries the contact below. When the files have no id " + "and the MusicBrainz provider is on too, the name of the album's artist and its title go to MusicBrainz to find " + "the id, and the id found goes to the archive. Redirects are followed only to archive.org, over https. " + "Never your users, your listens or file paths.", note: "Covers are uploaded by MusicBrainz users and have no licence of their own: Sonda keeps a copy only for " + "the albums of your own library, for your use, and never shows it as free.", }, { name: Lidarr, needsKey: true, needsURL: true, sends: "Only when an admin sends an artist to Lidarr, and for the admin's setup: the MusicBrainz id of the artist " + "(never a name: Sonda does not search) and your API key, to the address you set, which is your own Lidarr; " + "asking Lidarr what it offers (profiles and folders) and testing the key send nothing but the key. Lidarr " + "then looks for the music through its own indexers and download clients, which Sonda does not control. " + "Never your users, your listens, track titles or file paths.", note: "Sonda does not download, search for or choose music: Lidarr does, with the quality profile, the metadata " + "profile and the folder an admin picked, and only for the artists an admin sends. Nothing is sent by itself.", }, { name: Anthropic, needsKey: true, sends: "Only when someone asks for a name for a playlist, a mix or a smart playlist (and the same facts were not asked " + "before: the names are kept): up to 5 of its genres, up to 8 of its artist names, how many songs it has, for a " + "smart playlist a few words that say what its rules are, the language the name should be in, your API key, " + "your IP address and a User-Agent that says what the software is. Never your users, your listens, track titles " + "or file paths. Without this provider, or when it fails, a name is made on this server from the genres and the " + "artists, and nothing leaves it.", note: "The names are made by Claude, an AI model from Anthropic, under your own agreement with Anthropic, which bills the " + "API key you give; each name costs a fraction of a cent. What Anthropic keeps of the facts above depends on that " + "agreement. Sonda keeps the names it got, to ask less. Nothing is asked by itself.", }, } func lookup(name string) (meta, bool) { for _, m := range known { if m.name == name { return m, true } } return meta{}, false } // Service reads and changes provider settings. type Service struct { store Store now func() time.Time mu sync.Mutex // an update reads then writes: keep two of them apart checkers map[string]Checker purger Purger } // NewService builds a Service. func NewService(store Store) *Service { return &Service{store: store, now: time.Now} } func info(m meta, s Settings) Info { in := Info{ Name: m.name, NeedsAPIKey: m.needsKey, NeedsURL: m.needsURL, Sends: m.sends, Notes: m.note, Enabled: s.Enabled, HasAPIKey: s.APIKey != "", Contact: s.Contact, UpdatedAt: s.UpdatedAt, } if m.needsURL { in.URL = s.URL in.Lidarr = s.Lidarr.withDefaults() } return in } // List returns every known provider, configured or not, in a fixed order. func (s *Service) List(ctx context.Context) ([]Info, error) { rows, err := s.store.List(ctx) if err != nil { return nil, fmt.Errorf("list provider settings: %w", err) } byName := map[string]Settings{} for _, r := range rows { byName[r.Provider] = r } out := make([]Info, 0, len(known)) for _, m := range known { out = append(out, info(m, byName[m.name])) } return out, nil } // Update applies a patch and returns the provider as it is afterwards. func (s *Service) Update(ctx context.Context, name string, p Patch) (Info, error) { m, ok := lookup(name) if !ok { return Info{}, ErrUnknownProvider } if p.Enabled == nil && p.APIKey == nil && p.Contact == nil && p.URL == nil && p.Lidarr == nil { return Info{}, fmt.Errorf("%w: nothing to change", ErrInvalidSettings) } s.mu.Lock() defer s.mu.Unlock() cur, _, err := s.store.Get(ctx, name) if err != nil { return Info{}, fmt.Errorf("read provider settings: %w", err) } cur.Provider = name if p.APIKey != nil { key, err := validateKey(m, *p.APIKey) if err != nil { return Info{}, err } cur.APIKey = key if key == "" && m.needsKey { cur.Enabled = false // a provider that needs a key is off without one } } if p.URL != nil { u, err := validateURL(m, *p.URL) if err != nil { return Info{}, err } cur.URL = u if u == "" && m.needsURL { cur.Enabled = false // and so is one that needs an address } } if p.Lidarr != nil { if !m.needsURL { return Info{}, fmt.Errorf("%w: %s has no Lidarr options", ErrInvalidSettings, m.name) } if err := validateLidarrOptions(*p.Lidarr); err != nil { return Info{}, err } cur.Lidarr = *p.Lidarr } if p.Contact != nil { contact, err := validateContact(*p.Contact) if err != nil { return Info{}, err } cur.Contact = contact } if p.Enabled != nil { if *p.Enabled && m.needsKey && cur.APIKey == "" { return Info{}, fmt.Errorf("%w: %s needs an API key before it can be enabled", ErrInvalidSettings, name) } if *p.Enabled && m.needsURL && cur.URL == "" { return Info{}, fmt.Errorf("%w: %s needs an address before it can be enabled", ErrInvalidSettings, name) } cur.Enabled = *p.Enabled } if m.needsURL { // What the list shows is what is stored: a setup that never chose a monitor option has the default. cur.Lidarr = cur.Lidarr.withDefaults() } cur.UpdatedAt = s.now() if err := s.store.Save(ctx, cur); err != nil { return Info{}, fmt.Errorf("save provider settings: %w", err) } return info(m, cur), nil } // Active returns the settings of a provider when it can be used right now: it // is on and, if it needs a key, has one. Provider clients call this before // every outbound request; false means "do not contact anyone". func (s *Service) Active(ctx context.Context, name string) (Settings, bool, error) { m, ok := lookup(name) if !ok { return Settings{}, false, ErrUnknownProvider } cur, found, err := s.store.Get(ctx, name) if err != nil { return Settings{}, false, fmt.Errorf("read provider settings: %w", err) } if !found || !cur.Enabled || (m.needsKey && cur.APIKey == "") || (m.needsURL && cur.URL == "") { return Settings{}, false, nil } return cur, true, nil } // Stored returns the stored settings of a provider, key included, whether it is on or not; the zero // Settings when it was never configured. It is for what an admin does to finish setting a provider up // (testing a key, reading what Lidarr offers). Anything that talks to a provider for someone else asks // Active instead. func (s *Service) Stored(ctx context.Context, name string) (Settings, error) { if _, ok := lookup(name); !ok { return Settings{}, ErrUnknownProvider } cur, _, err := s.store.Get(ctx, name) if err != nil { return Settings{}, fmt.Errorf("read provider settings: %w", err) } return cur, nil } func validateKey(m meta, raw string) (string, error) { key := strings.TrimSpace(raw) if key == "" { return "", nil } if !m.needsKey { return "", fmt.Errorf("%w: %s does not use an API key", ErrInvalidSettings, m.name) } if m.name == Anthropic { // A key is case-sensitive: it is kept as it was typed. if !anthropicKeyRe.MatchString(key) { return "", fmt.Errorf("%w: an Anthropic API key starts with sk-ant- and has only letters, digits, - and _", ErrInvalidSettings) } return key, nil } key = strings.ToLower(key) if m.name == LastFM && !lastfmKeyRe.MatchString(key) { return "", fmt.Errorf("%w: a Last.fm API key is 32 hexadecimal characters", ErrInvalidSettings) } if m.name == Lidarr && !lidarrKeyRe.MatchString(key) { return "", fmt.Errorf("%w: a Lidarr API key is 32 hexadecimal characters", ErrInvalidSettings) } return key, nil } // validateURL accepts the address of a provider that is the user's own: http or https, a host, an // optional port and path (Lidarr can live under a URL base), and nothing that could carry a secret or // change where a request goes: no user and password, no query, no fragment. The closing slash goes. func validateURL(m meta, raw string) (string, error) { if !m.needsURL { return "", fmt.Errorf("%w: %s does not have an address", ErrInvalidSettings, m.name) } raw = strings.TrimSpace(raw) if raw == "" { return "", nil } bad := func(why string) (string, error) { return "", fmt.Errorf("%w: the address %s", ErrInvalidSettings, why) } if utf8.RuneCountInString(raw) > maxURLLen { return bad(fmt.Sprintf("is longer than %d characters", maxURLLen)) } if strings.ContainsFunc(raw, func(r rune) bool { return unicode.IsSpace(r) || unicode.IsControl(r) }) { return bad("cannot contain spaces or control characters") } if strings.ContainsAny(raw, "?#") { return bad("cannot have a query or a fragment") } u, err := url.Parse(raw) if err != nil { return bad("is not a valid address") } if u.Scheme != "http" && u.Scheme != "https" { return bad("must start with http:// or https://") } if u.Hostname() == "" { return bad("has no host") } if u.User != nil { return bad("cannot carry a user or a password: the key is set apart") } out := u.Scheme + "://" + u.Host + strings.TrimRight(u.EscapedPath(), "/") if back, err := url.Parse(out); err != nil || back.Host != u.Host { return bad("cannot be used as it is (an IPv6 zone must be left out)") // what is stored must read back } return out, nil } func validateLidarrOptions(o LidarrOptions) error { if o.QualityProfileID < 0 || o.MetadataProfileID < 0 { return fmt.Errorf("%w: a profile id cannot be negative", ErrInvalidSettings) } if !slices.Contains(LidarrMonitors, o.Monitor) { return fmt.Errorf("%w: monitor must be one of %s", ErrInvalidSettings, strings.Join(LidarrMonitors, ", ")) } if utf8.RuneCountInString(o.RootFolderPath) > maxFolderLen { return fmt.Errorf("%w: the root folder is longer than %d characters", ErrInvalidSettings, maxFolderLen) } for _, r := range o.RootFolderPath { if unicode.IsControl(r) { return fmt.Errorf("%w: the root folder cannot contain control characters", ErrInvalidSettings) } } return nil } // validateContact accepts what MusicBrainz asks for in a User-Agent: a URL or an // e-mail address. Parentheses are refused because they delimit it there. func validateContact(raw string) (string, error) { c := strings.TrimSpace(raw) if utf8.RuneCountInString(c) > maxContactLen { return "", fmt.Errorf("%w: contact is longer than %d characters", ErrInvalidSettings, maxContactLen) } for _, r := range c { if unicode.IsControl(r) || r == '(' || r == ')' { return "", fmt.Errorf("%w: contact cannot contain control characters or parentheses", ErrInvalidSettings) } } return c, nil } // ProjectURL is the contact used in the User-Agent when no other is set. const ProjectURL = "https://git.kmalloc.org/squid/sonda" // UserAgent builds the header sent to providers, in the shape MusicBrainz asks // for: "Application/version ( contact )". The version and the contact are // cleaned, so the result is always a single safe line. func UserAgent(version, contact string) string { clean := func(s string) string { var b strings.Builder for _, r := range s { if unicode.IsControl(r) || r == '(' || r == ')' { continue } b.WriteRune(r) } return strings.TrimSpace(b.String()) } v := strings.ReplaceAll(clean(version), " ", "") if v == "" { v = "dev" } c := clean(contact) if c == "" { c = ProjectURL } return fmt.Sprintf("Sonda/%s ( %s )", v, c) } // Checker asks a provider one cheap question with a key, to find out whether the // key works. It returns nil, or one of ErrKeyRefused, ErrRateLimited and // ErrUnreachable (any other error counts as an unknown failure). type Checker func(ctx context.Context, key string) error // SetChecker registers the test of a provider. Call it before the service is used. func (s *Service) SetChecker(name string, c Checker) { if s.checkers == nil { s.checkers = map[string]Checker{} } s.checkers[name] = c } // Results of a test. const ( TestOK = "ok" TestNoKey = "no_api_key" TestNoURL = "no_address" TestKeyRefused = "key_refused" TestRateLimited = "rate_limited" TestUnreachable = "unreachable" TestFailed = "error" ) // TestResult is the outcome of Test. Status is one of the Test constants. type TestResult struct { OK bool Status string } // Test checks a provider with the stored key. It works whether the provider is // on or off, because an admin wants to know before switching it on; it is the one // request Sonda makes on demand of an admin and not of a listener, and it sends // nothing but the key. The key is never part of the result. func (s *Service) Test(ctx context.Context, name string) (TestResult, error) { m, ok := lookup(name) if !ok { return TestResult{}, ErrUnknownProvider } check := s.checkers[name] if check == nil { return TestResult{}, ErrNoTest } cur, _, err := s.store.Get(ctx, name) if err != nil { return TestResult{}, fmt.Errorf("read provider settings: %w", err) } if m.needsKey && cur.APIKey == "" { return TestResult{Status: TestNoKey}, nil } if m.needsURL && cur.URL == "" { return TestResult{Status: TestNoURL}, nil } switch err := check(ctx, cur.APIKey); { case err == nil: return TestResult{OK: true, Status: TestOK}, nil case errors.Is(err, ErrKeyRefused): return TestResult{Status: TestKeyRefused}, nil case errors.Is(err, ErrRateLimited): return TestResult{Status: TestRateLimited}, nil case errors.Is(err, ErrUnreachable): return TestResult{Status: TestUnreachable}, nil default: return TestResult{Status: TestFailed}, nil } } // UserAgentFor returns a function that builds the User-Agent for a provider from // the contact stored now, so a change by an admin applies to the next request. // If the settings cannot be read it uses the project address: a request is never // held up or sent with a broken header because of it. func (s *Service) UserAgentFor(name, version string) func() string { return func() string { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() cur, _, err := s.store.Get(ctx, name) if err != nil { return UserAgent(version, "") } return UserAgent(version, cur.Contact) } } // Purged says how much a purge removed. type Purged struct { SimilarArtists int // lists of similar artists Tags int // tags Descriptions int // descriptions Images int // pictures of artists Artists int // artists that only the removed data named } // Purger removes everything a provider gave, and reports how much. type Purger func(ctx context.Context, name string) (Purged, error) // SetPurger registers how provider data is removed. Call it before the service is used. func (s *Service) SetPurger(p Purger) { s.purger = p } // Purge removes everything a provider gave: what it said about artists, and the // artists that only that data named and that nobody has files for. It works // whether the provider is on or off and does not change that; a provider that is // still on will be asked again when someone asks. Last.fm's terms ask for exactly // this at the end of an agreement. The user's music, artists and listens stay. func (s *Service) Purge(ctx context.Context, name string) (Purged, error) { if _, ok := lookup(name); !ok { return Purged{}, ErrUnknownProvider } if s.purger == nil { return Purged{}, errors.New("no way to remove provider data was set up") } return s.purger(ctx, name) }