From 9f4da2e77e25616d46deed6dff084b04ce4588e6 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Wed, 11 Nov 2026 16:07:12 +0100 Subject: [PATCH 1/2] feat(providers): provider settings in a table and a careful outbound HTTP client - feat(storage): provider_settings table (provider, enabled, api_key, contact); migration 0007 for both drivers. A provider with no row, or turned off, is never contacted - feat(providers): a service that validates and applies settings: a Last.fm key is 32 hexadecimal characters, enabling a provider that needs a key without one is refused, clearing the key turns the provider off, a contact cannot contain parentheses or control characters. Each provider states what it sends and under which terms, and what it never sends - feat(api): admin-only GET /providers and PUT /providers/{name}. The API key is write-only: responses carry only has_api_key, errors never repeat it and the request log never contains it (tested) - feat(providers/outbound): the only way Sonda talks to the outside: plain GET with an identifiable User-Agent, no cookies, no Referer, no Authorization, no redirects, bounded answers, a timeout per attempt, a minimum interval between requests, retries with exponential backoff that honour Retry-After without waiting past the caller's deadline or for an absurd time, and a circuit breaker that makes a failing provider fail fast - feat(providers): UserAgent builds "Sonda/version ( contact )" as MusicBrainz asks, defaulting to the project address so no personal address is sent - breaking(config): providers.lastfm and SONDA_LASTFM_API_KEY are gone; an old config, or the variable, fails with a hint instead of being ignored - docs: OpenAPI operations and schemas, README section on the providers and on what leaves the server - test: fake servers and a fake clock only; no network in the suite --- README.md | 29 +- cmd/sondad/main.go | 2 + internal/api/http/api_test.go | 2 + internal/api/http/contract_test.go | 18 + internal/api/http/handlers_providers.go | 81 +++ internal/api/http/providers_test.go | 171 ++++++ internal/api/http/router.go | 10 + internal/api/openapi/openapi.yaml | 101 ++++ internal/config/config.go | 24 +- internal/config/config_test.go | 38 +- internal/providers/outbound/outbound.go | 376 +++++++++++++ internal/providers/outbound/outbound_test.go | 498 ++++++++++++++++++ internal/providers/settings.go | 266 ++++++++++ internal/providers/settings_test.go | 261 +++++++++ internal/storage/providers.go | 81 +++ internal/storage/providers_test.go | 72 +++ internal/storage/storage_test.go | 6 +- .../postgres/0007_provider_settings.sql | 11 + migrations/sqlite/0007_provider_settings.sql | 11 + sonda.example.toml | 5 +- 20 files changed, 2032 insertions(+), 31 deletions(-) create mode 100644 internal/api/http/handlers_providers.go create mode 100644 internal/api/http/providers_test.go create mode 100644 internal/providers/outbound/outbound.go create mode 100644 internal/providers/outbound/outbound_test.go create mode 100644 internal/providers/settings.go create mode 100644 internal/providers/settings_test.go create mode 100644 internal/storage/providers.go create mode 100644 internal/storage/providers_test.go create mode 100644 migrations/postgres/0007_provider_settings.sql create mode 100644 migrations/sqlite/0007_provider_settings.sql diff --git a/README.md b/README.md index 6891ed8..c7ab97d 100644 --- a/README.md +++ b/README.md @@ -14,11 +14,11 @@ It is a single executable, one database (SQLite by default, PostgreSQL if you pr - **Catalog.** Browse and search artists, albums and tracks, with paging, by library, artist or album. Only tracks you can actually play are listed. - **Streaming.** `GET` and `HEAD` with `Range` support, so players can seek. Opening or prefetching a stream does not record a listen. - **API documentation.** The full API is described in OpenAPI and browsable with Swagger UI, served by Sonda itself. The UI is embedded in the binary, so opening it contacts no other server. -- **Listening history.** Clients report explicit `play`, `complete` and `skip` events. An event counts as a listen when it is `complete`, or when it was played for at least 20 seconds (`listening.min_listen_seconds`); the rule is applied when the history is read, so changing it re-reads the past. You can read your own history, with a `counted` flag on every event. It stays in your database; nothing is sent anywhere. Sonda makes no outbound network requests at all. +- **Listening history.** Clients report explicit `play`, `complete` and `skip` events. An event counts as a listen when it is `complete`, or when it was played for at least 20 seconds (`listening.min_listen_seconds`); the rule is applied when the history is read, so changing it re-reads the past. You can read your own history, with a `counted` flag on every event. It stays in your database and is never sent anywhere. Today Sonda makes no outbound network requests at all. ## What is not there yet -Similar-artist discovery, recommendations and radio queues, a web client, OpenSubsonic compatibility, transcoding, and Last.fm/MusicBrainz/ListenBrainz integrations. The `[providers.lastfm]` section of the config is reserved for a later milestone and is not used yet. +Similar-artist discovery, recommendations and radio queues, a web client, OpenSubsonic compatibility, transcoding, and the ListenBrainz integration. Last.fm and MusicBrainz can already be set up (see [External providers](#external-providers)), but nothing asks them anything yet. ## Requirements @@ -58,9 +58,8 @@ Precedence is: built-in defaults, then the file, then environment variables. An | `auth.access_ttl_minutes` | | `15` | Lifetime of an access token. | | `auth.refresh_ttl_hours` | | `720` | Lifetime of a refresh token (30 days). | | `listening.min_listen_seconds` | | `20` | A play counts as a listen after this long (1 to 3600), or when it is `complete`. | -| `providers.lastfm.api_key` | `SONDA_LASTFM_API_KEY` | | Reserved, not used yet. | -Libraries are not in the config: you add them through the API (see the demo). +Libraries and the external providers are not in the config: you set them up through the API (see the demo and [External providers](#external-providers)). ### Using PostgreSQL @@ -174,6 +173,25 @@ AUTH="Authorization: Bearer $TOKEN" curl -s -X POST $BASE/users -H "$AUTH" -d '{"username":"bob","password":"another long password"}' | jq ``` +## External providers + +Sonda can ask Last.fm and MusicBrainz about artists. This is **off by default**: until an admin turns a provider on, nothing leaves your server. Today a provider can be set up but nothing uses it yet. + +- **Last.fm** needs an API key. Create one for your own use at `last.fm/api` (it needs a Last.fm account). No user name or password is involved and the "shared secret" is not needed. Last.fm data is for non-commercial use only, and you should credit Last.fm when you show it. +- **MusicBrainz** needs no key, but it asks every client to identify itself with a contact (a URL or an e-mail address) in the `User-Agent`. By default Sonda uses the address of the project, so no personal address is sent unless you choose to. + +Set them up with `PUT /api/v1/providers/{name}` (admin), for example: + +```bash +curl -s -X PUT $BASE/providers/lastfm -H "$AUTH" -d '{"api_key":""}' +curl -s -X PUT $BASE/providers/lastfm -H "$AUTH" -d '{"enabled":true}' +curl -s $BASE/providers -H "$AUTH" | jq +``` + +The key is stored in the database, in clear text, and is never returned by the API or written to the log: the list only says `has_api_key`. Because the database holds it, treat database backups like secrets. Removing the key turns the provider off, and each provider describes in the list exactly what it sends. + +**What leaves your server, when a provider is on:** the artist name (or MusicBrainz id) being looked up, your API key for Last.fm, your IP address and the `User-Agent`. Never your users, your listens, track titles or file paths. Requests only happen when someone asks for them, never as a bulk pass over your library, and they are spaced out, retried with backoff, and stopped by a circuit breaker if a provider is down. + ## The API at a glance Everything lives under `/api/v1`. Bodies are JSON. Errors always look like `{"error":{"code":"...","message":"..."}}`. @@ -193,6 +211,7 @@ Everything lives under `/api/v1`. Bodies are JSON. Errors always look like `{"er | signed in | `GET /library/scan` | Scan status and the last result. | | admin | `GET`, `POST /libraries`; `PATCH`, `DELETE /libraries/{id}` | Manage libraries. Deleting one removes its index entries, never your files. | | admin | `POST /library/scan`, `POST /libraries/{id}/scan` | Scan everything, or one library. | +| admin | `GET /providers`, `PUT /providers/{name}` | See and set up Last.fm and MusicBrainz. The API key is write-only. | | admin | `GET`, `POST /users`; `PATCH`, `DELETE /users/{id}`; `PUT /users/{id}/password` | Manage users and the admin role. The last admin cannot be demoted or deleted. | A library path must be an absolute, existing folder, never the filesystem root, and never inside or around another library. Search is a case-insensitive substring match: `BJÖRK` finds `Björk`, but accents are kept, so `bjork` does not. @@ -203,6 +222,7 @@ A library path must be an absolute, existing folder, never the filesystem root, - Passwords are hashed with Argon2id. Refresh tokens are stored only as hashes. - Music folders are read-only to Sonda. Streaming opens files through `os.Root`, so a path can never leave its library, even if a symlink is swapped in later. - File paths are never returned by the catalog, and audio file paths are not logged. +- Provider API keys are write-only through the API and never logged. They sit in the database, so protect your backups. ## Back up @@ -238,6 +258,7 @@ internal/api/openapi/ the OpenAPI spec, embedded in the binary internal/auth/ users, roles, passwords, access and refresh tokens internal/catalog/ browsing and searching artists, albums, tracks internal/library/ libraries, the scanner, the event hub, the tag reader +internal/providers/ external providers: settings, and the careful outbound HTTP client internal/listening/ listen events internal/storage/ SQLite and PostgreSQL through bun, the repositories internal/streaming/ opening a track's file safely diff --git a/cmd/sondad/main.go b/cmd/sondad/main.go index 4df7ed5..22e8f5f 100644 --- a/cmd/sondad/main.go +++ b/cmd/sondad/main.go @@ -20,6 +20,7 @@ import ( "codeberg.org/squidd/sonda/internal/config" "codeberg.org/squidd/sonda/internal/library" "codeberg.org/squidd/sonda/internal/listening" + "codeberg.org/squidd/sonda/internal/providers" "codeberg.org/squidd/sonda/internal/storage" "codeberg.org/squidd/sonda/internal/streaming" "codeberg.org/squidd/sonda/migrations" @@ -105,6 +106,7 @@ func run() error { Scan: scanControl{ctx: ctx, m: scans}, Libraries: libraries, Catalog: catalog.NewService(storage.NewCatalogRepo(db)), + Providers: providers.NewService(storage.NewProviderRepo(db)), Streams: streaming.NewService(libraryRepo), }) diff --git a/internal/api/http/api_test.go b/internal/api/http/api_test.go index 0643598..3cc4109 100644 --- a/internal/api/http/api_test.go +++ b/internal/api/http/api_test.go @@ -18,6 +18,7 @@ import ( "codeberg.org/squidd/sonda/internal/auth" "codeberg.org/squidd/sonda/internal/library" "codeberg.org/squidd/sonda/internal/listening" + "codeberg.org/squidd/sonda/internal/providers" "codeberg.org/squidd/sonda/internal/storage" "codeberg.org/squidd/sonda/internal/streaming" "codeberg.org/squidd/sonda/migrations" @@ -80,6 +81,7 @@ func newTestEnvDB(t *testing.T, mutate func(*Deps, *bun.DB)) (http.Handler, *bun Listening: listening.NewService(storage.NewListeningRepo(db), 20*time.Second), Scan: &fakeScan{}, Libraries: library.NewLibraries(storage.NewLibraryRepo(db)), + Providers: providers.NewService(storage.NewProviderRepo(db)), Streams: streaming.NewService(noFiles{}), } mutate(&deps, db) diff --git a/internal/api/http/contract_test.go b/internal/api/http/contract_test.go index 8ea7978..5689c1a 100644 --- a/internal/api/http/contract_test.go +++ b/internal/api/http/contract_test.go @@ -277,6 +277,7 @@ func TestResponsesMatchTheSpec(t *testing.T) { c.expect(c.call("POST", "/users", token, map[string]any{"username": "x", "password": "bobs long pass"}), 400, "create invalid", "invalid_input") bobToken := c.expect(c.call("POST", "/auth/login", "", map[string]string{"username": "bob", "password": "bobs long pass"}), 200, "bob logs in")["access_token"].(string) c.expect(c.call("GET", "/users", bobToken, nil), 403, "users as non-admin", "admin_required") + c.expect(c.call("GET", "/providers", bobToken, nil), 403, "providers as non-admin", "admin_required") c.expect(c.call("GET", "/users", token, nil), 200, "list users") bobID := idOf(bob) c.expect(c.call("PATCH", "/users/"+bobID, token, map[string]bool{"is_admin": true}), 200, "promote") @@ -371,6 +372,23 @@ func TestResponsesMatchTheSpec(t *testing.T) { t.Fatalf("track.album = %#v, want an explicit null", v) } + // ---- providers ---- + listed := c.expect(c.call("GET", "/providers", token, nil), 200, "list providers")["providers"].([]any) + if len(listed) != 2 || listed[0].(map[string]any)["enabled"] != false || listed[1].(map[string]any)["enabled"] != false { + t.Fatalf("providers = %v, want two, both off", listed) + } + c.expect(c.call("PUT", "/providers/lastfm", token, map[string]any{"enabled": true}), 400, "enable without a key", "invalid_input") + c.expect(c.call("PUT", "/providers/lastfm", token, map[string]any{"api_key": "short"}), 400, "a bad key", "invalid_input") + c.expect(c.call("PUT", "/providers/spotify", token, map[string]any{"enabled": true}), 404, "unknown provider", "provider_not_found") + saved := c.expect(c.call("PUT", "/providers/lastfm", token, map[string]any{"api_key": "0123456789abcdef0123456789abcdef"}), 200, "save a key") + if saved["has_api_key"] != true || saved["updated_at"] == nil { + t.Fatalf("saved = %v", saved) + } + if on := c.expect(c.call("PUT", "/providers/lastfm", token, map[string]any{"enabled": true}), 200, "enable"); on["enabled"] != true { + t.Fatalf("enabled = %v", on) + } + c.expect(c.call("PUT", "/providers/musicbrainz", token, map[string]any{"enabled": true, "contact": "https://example.org/me"}), 200, "set up musicbrainz") + // ---- playback ---- stream := "/tracks/" + trackID + "/stream" c.expect(c.call("GET", stream, "", nil), 401, "stream without token", "unauthorized") diff --git a/internal/api/http/handlers_providers.go b/internal/api/http/handlers_providers.go new file mode 100644 index 0000000..3ff7c1b --- /dev/null +++ b/internal/api/http/handlers_providers.go @@ -0,0 +1,81 @@ +package httpapi + +import ( + "errors" + "net/http" + + "github.com/gin-gonic/gin" + + "codeberg.org/squidd/sonda/internal/providers" +) + +// providersHandlers are the admin-only routes that set up the external +// providers. The API key can be written here but is never returned. +type providersHandlers struct{ svc providerService } + +type providerJSON struct { + Name string `json:"name"` + Enabled bool `json:"enabled"` + NeedsAPIKey bool `json:"needs_api_key"` + HasAPIKey bool `json:"has_api_key"` + Contact string `json:"contact"` + Sends string `json:"sends"` + Notes string `json:"notes"` + UpdatedAt *string `json:"updated_at"` +} + +func toProviderJSON(in providers.Info) providerJSON { + out := providerJSON{ + Name: in.Name, Enabled: in.Enabled, NeedsAPIKey: in.NeedsAPIKey, HasAPIKey: in.HasAPIKey, + Contact: in.Contact, Sends: in.Sends, Notes: in.Notes, + } + if !in.UpdatedAt.IsZero() { + out.UpdatedAt = rfc3339(in.UpdatedAt) + } + return out +} + +func writeProviderError(c *gin.Context, err error) { + switch { + case errors.Is(err, providers.ErrUnknownProvider): + writeError(c, http.StatusNotFound, "provider_not_found", "no such provider") + case errors.Is(err, providers.ErrInvalidSettings): + writeError(c, http.StatusBadRequest, "invalid_input", err.Error()) + default: + _ = c.Error(err) + writeError(c, http.StatusInternalServerError, "internal", "internal error") + } +} + +func (h *providersHandlers) list(c *gin.Context) { + infos, err := h.svc.List(c.Request.Context()) + if err != nil { + writeProviderError(c, err) + return + } + out := make([]providerJSON, len(infos)) + for i, in := range infos { + out[i] = toProviderJSON(in) + } + c.JSON(http.StatusOK, gin.H{"providers": out}) +} + +type updateProviderRequest struct { + Enabled *bool `json:"enabled"` // pointers: an absent field is left alone + APIKey *string `json:"api_key"` + Contact *string `json:"contact"` +} + +func (h *providersHandlers) update(c *gin.Context) { + var req updateProviderRequest + if !bindJSON(c, &req) { + return + } + info, err := h.svc.Update(c.Request.Context(), c.Param("name"), + providers.Patch{Enabled: req.Enabled, APIKey: req.APIKey, Contact: req.Contact}) + if err != nil { + writeProviderError(c, err) + return + } + c.JSON(http.StatusOK, toProviderJSON(info)) +} diff --git a/internal/api/http/providers_test.go b/internal/api/http/providers_test.go new file mode 100644 index 0000000..3bcca17 --- /dev/null +++ b/internal/api/http/providers_test.go @@ -0,0 +1,171 @@ +package httpapi + +import ( + "encoding/json" + "net/http" + "strings" + "testing" + + "github.com/uptrace/bun" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" +) + +const lastfmKey = "0123456789abcdef0123456789abcdef" + +func TestProviderRoutesAreAdminOnly(t *testing.T) { + h := newTestAPI(t) + admin := registerAndLogin(t, h) + _, bob := addUser(t, h, admin, "bob", false) + + for _, c := range []struct{ method, path string }{ + {"GET", "/api/v1/providers"}, + {"PUT", "/api/v1/providers/lastfm"}, + } { + if r := do(t, h, c.method, c.path, "", nil); r.Code != http.StatusUnauthorized { + t.Errorf("%s %s without a token: %d", c.method, c.path, r.Code) + } + r := do(t, h, c.method, c.path, bob, map[string]any{"enabled": true}) + if r.Code != http.StatusForbidden || errCode(r) != "admin_required" { + t.Errorf("%s %s as a normal user: %d %v", c.method, c.path, r.Code, r.Body) + } + } +} + +func TestProvidersStartOffAndSayWhatTheySend(t *testing.T) { + h := newTestAPI(t) + admin := registerAndLogin(t, h) + + r := do(t, h, "GET", "/api/v1/providers", admin, nil) + if r.Code != http.StatusOK { + t.Fatalf("list: %d %v", r.Code, r.Body) + } + list := r.Body["providers"].([]any) + if len(list) != 2 { + t.Fatalf("providers = %v", list) + } + for _, it := range list { + p := it.(map[string]any) + sameKeys(t, "provider", p, "name", "enabled", "needs_api_key", "has_api_key", "contact", "sends", "notes", "updated_at") + if p["enabled"] != false || p["has_api_key"] != false { + t.Errorf("%v: a provider must start off and without a key", p["name"]) + } + if v, present := p["updated_at"]; !present || v != nil { + t.Errorf("%v: updated_at = %#v, want an explicit null before any change", p["name"], v) + } + if !strings.Contains(p["sends"].(string), "Never") { + t.Errorf("%v: sends = %q", p["name"], p["sends"]) + } + } + if list[0].(map[string]any)["name"] != "lastfm" || list[0].(map[string]any)["needs_api_key"] != true || + list[1].(map[string]any)["name"] != "musicbrainz" || list[1].(map[string]any)["needs_api_key"] != false { + t.Errorf("order or needs_api_key wrong: %v", list) + } +} + +func TestSettingAProviderUp(t *testing.T) { + h := newTestAPI(t) + admin := registerAndLogin(t, h) + + // Enabling before there is a key is refused. + r := do(t, h, "PUT", "/api/v1/providers/lastfm", admin, map[string]any{"enabled": true}) + if r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" { + t.Fatalf("enable without a key: %d %v", r.Code, r.Body) + } + + // The key is written, not read back. + r = do(t, h, "PUT", "/api/v1/providers/lastfm", admin, map[string]any{"api_key": lastfmKey}) + if r.Code != http.StatusOK || r.Body["has_api_key"] != true || r.Body["enabled"] != false { + t.Fatalf("save the key: %d %v", r.Code, r.Body) + } + if r.Body["updated_at"] == nil { + t.Error("updated_at should be set after a change") + } + raw, _ := json.Marshal(r.Body) + if strings.Contains(string(raw), lastfmKey) { + t.Fatal("the response contains the API key") + } + + r = do(t, h, "PUT", "/api/v1/providers/lastfm", admin, map[string]any{"enabled": true}) + if r.Code != http.StatusOK || r.Body["enabled"] != true { + t.Fatalf("enable: %d %v", r.Code, r.Body) + } + + listed := do(t, h, "GET", "/api/v1/providers", admin, nil) + rawList, _ := json.Marshal(listed.Body) + if strings.Contains(string(rawList), lastfmKey) { + t.Fatal("the list contains the API key") + } + first := listed.Body["providers"].([]any)[0].(map[string]any) + if first["enabled"] != true || first["has_api_key"] != true { + t.Errorf("listed = %v", first) + } + + // Clearing the key turns the provider off. + r = do(t, h, "PUT", "/api/v1/providers/lastfm", admin, map[string]any{"api_key": ""}) + if r.Code != http.StatusOK || r.Body["has_api_key"] != false || r.Body["enabled"] != false { + t.Fatalf("clear the key: %d %v", r.Code, r.Body) + } +} + +func TestMusicBrainzTakesAContactAndNoKey(t *testing.T) { + h := newTestAPI(t) + admin := registerAndLogin(t, h) + + r := do(t, h, "PUT", "/api/v1/providers/musicbrainz", admin, map[string]any{"enabled": true, "contact": "https://example.org/me"}) + if r.Code != http.StatusOK || r.Body["enabled"] != true || r.Body["contact"] != "https://example.org/me" { + t.Fatalf("%d %v", r.Code, r.Body) + } + r = do(t, h, "PUT", "/api/v1/providers/musicbrainz", admin, map[string]any{"api_key": lastfmKey}) + if r.Code != http.StatusBadRequest { + t.Errorf("a key for a provider without keys: %d", r.Code) + } +} + +func TestProviderUpdateRejectsBadInputWithoutEchoingTheKey(t *testing.T) { + h := newTestAPI(t) + admin := registerAndLogin(t, h) + badKey := "not-a-valid-key-but-secret-looking-12345" + + for name, tt := range map[string]struct { + path string + body any + code int + err string + }{ + "empty body": {"/api/v1/providers/lastfm", map[string]any{}, 400, "invalid_input"}, + "malformed json": {"/api/v1/providers/lastfm", "{oops", 400, "invalid_input"}, + "bad key format": {"/api/v1/providers/lastfm", map[string]any{"api_key": badKey}, 400, "invalid_input"}, + "unknown": {"/api/v1/providers/spotify", map[string]any{"enabled": true}, 404, "provider_not_found"}, + "contact with ()": {"/api/v1/providers/musicbrainz", map[string]any{"contact": "a (b)"}, 400, "invalid_input"}, + } { + r := do(t, h, "PUT", tt.path, admin, tt.body) + if r.Code != tt.code || errCode(r) != tt.err { + t.Errorf("%s: %d %v, want %d %s", name, r.Code, r.Body, tt.code, tt.err) + } + raw, _ := json.Marshal(r.Body) + if strings.Contains(string(raw), badKey) { + t.Errorf("%s: the error repeats the key", name) + } + } +} + +// The key is a secret: it must not appear in the request log either. +func TestTheKeyNeverReachesTheLog(t *testing.T) { + core, logs := observer.New(zap.DebugLevel) + h, _ := newTestEnvDB(t, func(d *Deps, _ *bun.DB) { d.Logger = zap.New(core) }) + admin := registerAndLogin(t, h) + + do(t, h, "PUT", "/api/v1/providers/lastfm", admin, map[string]any{"api_key": lastfmKey}) + do(t, h, "PUT", "/api/v1/providers/lastfm", admin, map[string]any{"api_key": "bad" + lastfmKey}) + + if logs.Len() == 0 { + t.Fatal("the test saw no log entries, so it proves nothing") + } + for _, e := range logs.All() { + dump, _ := json.Marshal(map[string]any{"message": e.Message, "fields": e.ContextMap()}) + if strings.Contains(string(dump), lastfmKey) { + t.Fatalf("a log entry contains the key: %s", dump) + } + } +} diff --git a/internal/api/http/router.go b/internal/api/http/router.go index b5da7f2..dda5db4 100644 --- a/internal/api/http/router.go +++ b/internal/api/http/router.go @@ -14,6 +14,7 @@ import ( "codeberg.org/squidd/sonda/internal/catalog" "codeberg.org/squidd/sonda/internal/library" "codeberg.org/squidd/sonda/internal/listening" + "codeberg.org/squidd/sonda/internal/providers" "codeberg.org/squidd/sonda/internal/streaming" ) @@ -53,6 +54,11 @@ type scanService interface { Status() library.Status } +type providerService interface { + List(ctx context.Context) ([]providers.Info, error) + Update(ctx context.Context, name string, p providers.Patch) (providers.Info, error) +} + type catalogService interface { Artists(ctx context.Context, f catalog.Filter) (catalog.Page[catalog.Artist], error) Albums(ctx context.Context, f catalog.Filter) (catalog.Page[catalog.Album], error) @@ -78,6 +84,7 @@ type Deps struct { Scan scanService Libraries libraryService Catalog catalogService + Providers providerService Streams streamService } @@ -133,6 +140,9 @@ func NewRouter(d Deps) *gin.Engine { admin.PATCH("/libraries/:id", libs.update) admin.DELETE("/libraries/:id", libs.remove) admin.POST("/libraries/:id/scan", lib.startLibraryScan) + prov := &providersHandlers{svc: d.Providers} + admin.GET("/providers", prov.list) + admin.PUT("/providers/:name", prov.update) u := &usersHandlers{svc: d.Auth} admin.GET("/users", u.list) admin.POST("/users", u.create) diff --git a/internal/api/openapi/openapi.yaml b/internal/api/openapi/openapi.yaml index 8c6f68f..3af3ad9 100644 --- a/internal/api/openapi/openapi.yaml +++ b/internal/api/openapi/openapi.yaml @@ -33,6 +33,8 @@ tags: description: Browsing and searching what can be played. - name: Playback description: Streaming and listening history. + - name: Providers + description: External services Sonda can ask about artists, set up by an admin. paths: /health: @@ -751,6 +753,70 @@ paths: '412': {description: '`If-Unmodified-Since` names a date before the last change of the file.'} '416': {description: The range cannot be satisfied.} + /providers: + get: + tags: [Providers] + operationId: listProviders + summary: List the external providers + description: | + Admin only. Every provider Sonda knows (Last.fm, MusicBrainz), with whether it is on and + what it would send. Providers are off by default: nothing leaves the server until an admin + enables one. The API key is never returned; `has_api_key` says whether one is stored. + security: [{bearerAuth: []}] + responses: + '200': + description: The providers, in a fixed order. + content: + application/json: + schema: + type: object + additionalProperties: false + required: [providers] + properties: + providers: + type: array + items: {$ref: '#/components/schemas/Provider'} + '401': {$ref: '#/components/responses/Unauthorized'} + '403': {$ref: '#/components/responses/AdminRequired'} + + /providers/{name}: + parameters: + - name: name + in: path + required: true + schema: {type: string, enum: [lastfm, musicbrainz]} + put: + tags: [Providers] + operationId: updateProvider + summary: Set up a provider + description: | + Admin only. Send any of `enabled`, `api_key` and `contact`; fields you leave out are kept. + + The key is write-only: it is never returned and never logged. Sending an empty `api_key` + clears it and turns the provider off. A provider that needs a key (Last.fm) cannot be + enabled without one. `contact` goes into the User-Agent sent to the provider (MusicBrainz + asks for a URL or an e-mail address); empty means the project address. + security: [{bearerAuth: []}] + requestBody: + required: true + content: + application/json: + schema: {$ref: '#/components/schemas/ProviderUpdate'} + responses: + '200': + description: The provider as it is now. + content: + application/json: + schema: {$ref: '#/components/schemas/Provider'} + '400': {$ref: '#/components/responses/InvalidInput'} + '401': {$ref: '#/components/responses/Unauthorized'} + '403': {$ref: '#/components/responses/AdminRequired'} + '404': + description: No such provider (`provider_not_found`). + content: + application/json: + schema: {$ref: '#/components/schemas/Error'} + /listens: get: tags: [Playback] @@ -1168,6 +1234,41 @@ components: description: Milliseconds actually played. Up to twice the track length plus 10 seconds when that length is known. idempotency_key: {type: string, description: "One per play, 1 to 128 bytes (UTF-8). A key that was already used is ignored."} + Provider: + type: object + additionalProperties: false + required: [name, enabled, needs_api_key, has_api_key, contact, sends, notes, updated_at] + properties: + name: {type: string, enum: [lastfm, musicbrainz]} + enabled: {type: boolean, description: Whether Sonda may contact this provider.} + needs_api_key: {type: boolean} + has_api_key: {type: boolean, description: Whether a key is stored. The key itself is never returned.} + contact: {type: string, description: Contact in the User-Agent; empty means the project address.} + sends: {type: string, description: "Plain statement of what is sent to the provider, and what never is."} + notes: {type: string, description: Terms of use and attribution that apply to the provider's data.} + updated_at: + type: string + format: date-time + nullable: true + description: When the settings last changed; null if they never did. + + ProviderUpdate: + type: object + anyOf: + - required: [enabled] + - required: [api_key] + - required: [contact] + properties: + enabled: {type: boolean} + api_key: + type: string + format: password + description: A Last.fm key is 32 hexadecimal characters. Empty clears it. + contact: + type: string + maxLength: 200 + description: A URL or e-mail address, without parentheses. Empty restores the default. + ListenResult: type: object additionalProperties: false diff --git a/internal/config/config.go b/internal/config/config.go index 1bd64b1..624beeb 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -23,7 +23,6 @@ type Config struct { Log Log `toml:"log"` Auth Auth `toml:"auth"` Listening Listening `toml:"listening"` - Providers Providers `toml:"providers"` } // Server holds HTTP listener settings. @@ -81,16 +80,6 @@ func (l Listening) MinListen() time.Duration { return time.Duration(l.MinListenSeconds) * time.Second } -// Providers holds external provider settings. -type Providers struct { - LastFM LastFM `toml:"lastfm"` -} - -// LastFM holds the Last.fm API settings. -type LastFM struct { - APIKey string `toml:"api_key"` -} - // Default returns the configuration used when nothing else is set. func Default() Config { return Config{ @@ -106,6 +95,12 @@ func Default() Config { // Load builds a Config from defaults, the TOML file at path, and environment // variables. An empty path skips the file. Unknown keys in the file are errors. func Load(path string) (Config, error) { + // Provider settings live in the database now. A variable that used to carry + // the Last.fm key must not be ignored in silence. + if os.Getenv("SONDA_LASTFM_API_KEY") != "" { + return Config{}, errors.New("SONDA_LASTFM_API_KEY is no longer read: set the key with " + + "PUT /api/v1/providers/lastfm and unset the variable") + } cfg := Default() if path != "" { @@ -143,9 +138,6 @@ func applyEnv(cfg *Config) { if v := os.Getenv("SONDA_JWT_SECRET"); v != "" { cfg.Auth.JWTSecret = v } - if v := os.Getenv("SONDA_LASTFM_API_KEY"); v != "" { - cfg.Providers.LastFM.APIKey = v - } } // Validate checks that the configuration is usable. @@ -196,6 +188,10 @@ func explainDecodeError(path string, err error) error { if errors.As(err, &missing) { for _, e := range missing.Errors { key := e.Key() + if len(key) > 0 && key[0] == "providers" { + return fmt.Errorf("parse config %s: the providers section was removed; "+ + "remove it and set the settings with PUT /api/v1/providers/{name}", path) + } if len(key) == 2 && key[0] == "library" && key[1] == "roots" { return fmt.Errorf("parse config %s: library.roots was removed; "+ "remove it and add each directory as a library with POST /api/v1/libraries", path) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 873cceb..2c9169e 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -57,8 +57,6 @@ listen = "0.0.0.0:9000" [library] scan_workers = 9 -[providers.lastfm] -api_key = "k" `) cfg, err := Load(p) if err != nil { @@ -70,9 +68,6 @@ api_key = "k" if cfg.Library.ScanWorkers != 9 { t.Errorf("scan workers = %d", cfg.Library.ScanWorkers) } - if cfg.Providers.LastFM.APIKey != "k" { - t.Errorf("api key = %q", cfg.Providers.LastFM.APIKey) - } // untouched keys keep their defaults if cfg.Database.Path != "sonda.db" { t.Errorf("db path = %q", cfg.Database.Path) @@ -87,14 +82,13 @@ func TestLoadEnvOverridesFile(t *testing.T) { t.Setenv("SONDA_DB_PATH", "x.db") t.Setenv("SONDA_DB_DRIVER", "postgres") t.Setenv("SONDA_DB_DSN", "postgres://u:p@h/db") - t.Setenv("SONDA_LASTFM_API_KEY", "envkey") t.Setenv("SONDA_JWT_SECRET", "envsecret") cfg, err := Load(p) if err != nil { t.Fatal(err) } if cfg.Log.Env != "dev" || cfg.Server.Listen != "127.0.0.1:1" || - cfg.Database.Path != "x.db" || cfg.Providers.LastFM.APIKey != "envkey" || + cfg.Database.Path != "x.db" || cfg.Auth.JWTSecret != "envsecret" || cfg.Database.Driver != "postgres" || cfg.Database.DSN != "postgres://u:p@h/db" { t.Errorf("env not applied: %+v", cfg) @@ -186,3 +180,33 @@ func TestListeningSettingIsReadFromTheFile(t *testing.T) { t.Errorf("min listen = %v", cfg.Listening.MinListen()) } } + +// Provider settings moved into the database. An old config must fail with a +// hint, never be ignored in silence. +func TestLoadExplainsRemovedProviderSettings(t *testing.T) { + p := writeFile(t, t.TempDir(), "[providers.lastfm]\napi_key = \"0123456789abcdef0123456789abcdef\"\n") + _, err := Load(p) + if err == nil { + t.Fatal("expected an error for the providers section") + } + if !strings.Contains(err.Error(), "providers") || !strings.Contains(err.Error(), "/api/v1/providers") { + t.Errorf("error = %v, want a pointer to the providers API", err) + } + if strings.Contains(err.Error(), "0123456789abcdef") { + t.Errorf("the error repeats the key: %v", err) + } +} + +func TestLoadRefusesTheOldLastFMEnvironmentVariable(t *testing.T) { + t.Setenv("SONDA_LASTFM_API_KEY", "0123456789abcdef0123456789abcdef") + _, err := Load("") + if err == nil { + t.Fatal("expected an error: the variable is no longer read, so it must not be ignored silently") + } + if !strings.Contains(err.Error(), "SONDA_LASTFM_API_KEY") || !strings.Contains(err.Error(), "/api/v1/providers") { + t.Errorf("error = %v", err) + } + if strings.Contains(err.Error(), "0123456789abcdef") { + t.Errorf("the error repeats the key: %v", err) + } +} diff --git a/internal/providers/outbound/outbound.go b/internal/providers/outbound/outbound.go new file mode 100644 index 0000000..a4ac20c --- /dev/null +++ b/internal/providers/outbound/outbound.go @@ -0,0 +1,376 @@ +// Package outbound is the only way Sonda talks to the outside world. +// +// It is built for a server that must not misbehave toward the services it asks +// and must not leak anything about its users: +// +// - every request is a plain GET with an identifiable User-Agent and nothing +// else that identifies anyone: no cookies are kept, no Referer, no +// Authorization header, and redirects are never followed; +// - answers are size-bounded and every attempt has a timeout; +// - requests to one provider are spaced by a minimum interval; +// - temporary failures (429, 502, 503, 504, dropped connections) are retried +// with exponential backoff, honouring Retry-After, never waiting past the +// caller's deadline and never waiting an unreasonable time; +// - after repeated failures a circuit breaker answers at once instead of +// asking a provider that is down, so a dead provider cannot slow Sonda. +package outbound + +import ( + "context" + "errors" + "fmt" + "io" + "math/rand/v2" + "net/http" + "net/url" + "strconv" + "sync" + "time" +) + +// Errors returned by the client. +var ( + ErrCircuitOpen = errors.New("provider keeps failing: not asking again yet") + ErrResponseTooLarge = errors.New("response is larger than the allowed size") + ErrRedirect = errors.New("provider answered with a redirect, which is never followed") + ErrBadURL = errors.New("not an http(s) URL") +) + +// RateLimitedError means the provider asked us to wait longer than we are +// willing to, so the request was not retried. +type RateLimitedError struct{ RetryAfter time.Duration } + +func (e *RateLimitedError) Error() string { + return fmt.Sprintf("provider asked us to wait %v", e.RetryAfter) +} + +// NoRetries as Options.Retries disables retrying (zero means "the default"). +const NoRetries = -1 + +// Clock lets tests control time. The zero Options use the real one. +type Clock interface { + Now() time.Time + Sleep(ctx context.Context, d time.Duration) error +} + +type realClock struct{} + +func (realClock) Now() time.Time { return time.Now() } + +func (realClock) Sleep(ctx context.Context, d time.Duration) error { + t := time.NewTimer(d) + defer t.Stop() + select { + case <-t.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// Options configure a Client. Zero values mean sensible defaults. +type Options struct { + UserAgent string + Timeout time.Duration // per attempt; default 10s + + MinInterval time.Duration // minimum time between requests; 0 = no limit + MaxBody int64 // largest accepted answer; default 4 MiB + + Retries int // retries after the first attempt; 0 = default (2), NoRetries = none + BackoffBase time.Duration // first backoff; doubles each retry; default 1s + MaxRetryAfter time.Duration // longest Retry-After we will wait for; default 30s + + BreakerThreshold int // consecutive failures that open the breaker; default 5 + BreakerOpenFor time.Duration // first open window; doubles on each new failure; default 30s + BreakerMaxOpen time.Duration // longest open window; default 10 min + + Transport http.RoundTripper // default: http.DefaultTransport + Clock Clock // default: the real clock + Jitter func(max time.Duration) time.Duration // random extra backoff; default: uniform in [0, max] +} + +// Response is a completed exchange. Any HTTP status can be returned: it is up +// to the caller to read the status (and the provider's own error format). +type Response struct { + Status int + Header http.Header + Body []byte +} + +// Client talks to one provider. It is safe for concurrent use. +type Client struct { + userAgent string + timeout time.Duration + minInterval time.Duration + maxBody int64 + retries int + backoffBase time.Duration + maxRetryAfter time.Duration + breakerThreshold int + breakerOpenFor time.Duration + breakerMaxOpen time.Duration + + hc *http.Client + clock Clock + jitter func(time.Duration) time.Duration + + mu sync.Mutex + next time.Time // earliest start of the next request + failures int // consecutive failures + openFor time.Duration + openUntil time.Time +} + +// New builds a Client. +func New(o Options) *Client { + def := func(v, d time.Duration) time.Duration { + if v <= 0 { + return d + } + return v + } + c := &Client{ + userAgent: o.UserAgent, + timeout: def(o.Timeout, 10*time.Second), + minInterval: o.MinInterval, + maxBody: o.MaxBody, + retries: o.Retries, + backoffBase: def(o.BackoffBase, time.Second), + maxRetryAfter: def(o.MaxRetryAfter, 30*time.Second), + breakerThreshold: o.BreakerThreshold, + breakerOpenFor: def(o.BreakerOpenFor, 30*time.Second), + breakerMaxOpen: def(o.BreakerMaxOpen, 10*time.Minute), + clock: o.Clock, + jitter: o.Jitter, + } + if c.maxBody <= 0 { + c.maxBody = 4 << 20 + } + switch { + case c.retries == 0: + c.retries = 2 + case c.retries < 0: + c.retries = 0 + } + if c.breakerThreshold <= 0 { + c.breakerThreshold = 5 + } + if c.clock == nil { + c.clock = realClock{} + } + if c.jitter == nil { + c.jitter = func(max time.Duration) time.Duration { + if max <= 0 { + return 0 + } + return time.Duration(rand.Int64N(int64(max) + 1)) + } + } + + transport := o.Transport + if transport == nil { + transport = http.DefaultTransport + } + c.hc = &http.Client{ + Transport: transport, // no cookie jar: nothing a provider sets is ever sent back + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse // never follow: a redirect could lead anywhere + }, + } + return c +} + +// Get fetches rawURL. It returns a Response for any HTTP status; an error means +// no usable answer (network failure, timeout, breaker open, answer too large, +// redirect, or a Retry-After we refuse to wait for). +func (c *Client) Get(ctx context.Context, rawURL string) (*Response, error) { + u, err := url.Parse(rawURL) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return nil, ErrBadURL + } + if err := ctx.Err(); err != nil { + return nil, err + } + if err := c.allow(); err != nil { + return nil, err + } + + var lastResp *Response + var lastErr error + for attempt := 0; ; attempt++ { + if err := c.throttle(ctx); err != nil { + return nil, err + } + resp, err := c.once(ctx, rawURL) + switch { + case errors.Is(err, ErrRedirect), errors.Is(err, ErrResponseTooLarge): + return nil, err + case err != nil: + if ctx.Err() != nil { + return nil, ctx.Err() + } + lastResp, lastErr = nil, err + case retryable(resp.Status): + lastResp, lastErr = resp, nil + default: + // An answer that is not worth retrying. A server error still counts + // against the provider; anything else (including 403 and 404) shows + // the provider is alive. + if resp.Status >= 500 { + c.failed() + } else { + c.succeeded() + } + return resp, nil + } + + if attempt >= c.retries { + break + } + delay := c.backoff(attempt) + if lastResp != nil { + if wait, ok := retryAfter(lastResp.Header, c.clock.Now()); ok { + if wait > c.maxRetryAfter { + c.failed() + return nil, &RateLimitedError{RetryAfter: wait} + } + if wait > delay { + delay = wait + } + } + } + // Do not sleep through the caller's own deadline: give back what we have. + if dl, ok := ctx.Deadline(); ok && time.Until(dl) < delay { + break + } + if err := c.clock.Sleep(ctx, delay); err != nil { + return nil, err + } + } + + // Only temporary failures get here: retried out, so the provider is struggling. + c.failed() + if lastResp != nil { + return lastResp, nil + } + return nil, lastErr +} + +// once makes a single attempt. +func (c *Client) once(ctx context.Context, rawURL string) (*Response, error) { + actx, cancel := context.WithTimeout(ctx, c.timeout) + defer cancel() + + req, err := http.NewRequestWithContext(actx, http.MethodGet, rawURL, nil) + if err != nil { + return nil, ErrBadURL + } + req.Header.Set("User-Agent", c.userAgent) + req.Header.Set("Accept", "application/json") + + resp, err := c.hc.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode >= 300 && resp.StatusCode < 400 { + return nil, ErrRedirect + } + body, err := io.ReadAll(io.LimitReader(resp.Body, c.maxBody+1)) + if err != nil { + return nil, err + } + if int64(len(body)) > c.maxBody { + return nil, ErrResponseTooLarge + } + return &Response{Status: resp.StatusCode, Header: resp.Header.Clone(), Body: body}, nil +} + +func retryable(status int) bool { + switch status { + case http.StatusTooManyRequests, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout: + return true + } + return false +} + +// backoff is base * 2^attempt plus a little random spread, capped. +func (c *Client) backoff(attempt int) time.Duration { + d := c.backoffBase << attempt + if d > c.maxRetryAfter || d <= 0 { + d = c.maxRetryAfter + } + return d + c.jitter(c.backoffBase/4) +} + +// retryAfter reads the Retry-After header: seconds, or an HTTP date. +func retryAfter(h http.Header, now time.Time) (time.Duration, bool) { + v := h.Get("Retry-After") + if v == "" { + return 0, false + } + if secs, err := strconv.Atoi(v); err == nil && secs >= 0 { + return time.Duration(secs) * time.Second, true + } + if t, err := http.ParseTime(v); err == nil { + if d := t.Sub(now); d > 0 { + return d, true + } + return 0, true + } + return 0, false +} + +// throttle reserves the next free slot and waits for it. +func (c *Client) throttle(ctx context.Context) error { + if c.minInterval <= 0 { + return nil + } + c.mu.Lock() + now := c.clock.Now() + at := c.next + if at.Before(now) { + at = now + } + c.next = at.Add(c.minInterval) + c.mu.Unlock() + + if wait := at.Sub(now); wait > 0 { + return c.clock.Sleep(ctx, wait) + } + return nil +} + +// allow refuses the request while the breaker is open. +func (c *Client) allow() error { + c.mu.Lock() + defer c.mu.Unlock() + if c.clock.Now().Before(c.openUntil) { + return ErrCircuitOpen + } + return nil +} + +func (c *Client) succeeded() { + c.mu.Lock() + defer c.mu.Unlock() + c.failures, c.openFor, c.openUntil = 0, 0, time.Time{} +} + +// failed records a failure. Once the threshold is reached the breaker opens; +// each further failure (a probe after the window) reopens it for twice as long. +func (c *Client) failed() { + c.mu.Lock() + defer c.mu.Unlock() + c.failures++ + if c.failures < c.breakerThreshold { + return + } + if c.openFor == 0 { + c.openFor = c.breakerOpenFor + } else if c.openFor *= 2; c.openFor > c.breakerMaxOpen { + c.openFor = c.breakerMaxOpen + } + c.openUntil = c.clock.Now().Add(c.openFor) +} diff --git a/internal/providers/outbound/outbound_test.go b/internal/providers/outbound/outbound_test.go new file mode 100644 index 0000000..8fac1f4 --- /dev/null +++ b/internal/providers/outbound/outbound_test.go @@ -0,0 +1,498 @@ +package outbound + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +// fakeClock never really sleeps: it moves its own time forward and remembers by +// how much, so retry and rate-limit timing can be asserted exactly. +type fakeClock struct { + mu sync.Mutex + now time.Time + slept []time.Duration +} + +func newClock() *fakeClock { return &fakeClock{now: time.Unix(1_700_000_000, 0)} } + +func (f *fakeClock) Now() time.Time { + f.mu.Lock() + defer f.mu.Unlock() + return f.now +} + +func (f *fakeClock) Sleep(ctx context.Context, d time.Duration) error { + if err := ctx.Err(); err != nil { + return err + } + f.mu.Lock() + defer f.mu.Unlock() + f.slept = append(f.slept, d) + f.now = f.now.Add(d) + return nil +} + +func (f *fakeClock) advance(d time.Duration) { + f.mu.Lock() + f.now = f.now.Add(d) + f.mu.Unlock() +} + +func (f *fakeClock) sleeps() []time.Duration { + f.mu.Lock() + defer f.mu.Unlock() + return append([]time.Duration(nil), f.slept...) +} + +func noJitter(time.Duration) time.Duration { return 0 } + +func newClient(clock *fakeClock, mutate func(*Options)) *Client { + o := Options{UserAgent: "Sonda/test ( https://example.org/sonda )", Clock: clock, Jitter: noJitter} + if mutate != nil { + mutate(&o) + } + return New(o) +} + +// server counts the requests it receives and answers with the next handler of a list. +func server(t *testing.T, handlers ...http.HandlerFunc) (*httptest.Server, *atomic.Int64) { + t.Helper() + var n atomic.Int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + i := int(n.Add(1)) - 1 + if i >= len(handlers) { + i = len(handlers) - 1 + } + handlers[i](w, r) + })) + t.Cleanup(srv.Close) + return srv, &n +} + +func status(code int) http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(code) } +} + +func ok(body string) http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte(body)) } +} + +// ---- what leaves the server ---- + +func TestRequestCarriesOnlyWhatItShould(t *testing.T) { + var got http.Header + var method string + srv, _ := server(t, func(w http.ResponseWriter, r *http.Request) { + got, method = r.Header.Clone(), r.Method + _, _ = w.Write([]byte("{}")) + }) + c := newClient(newClock(), nil) + + if _, err := c.Get(context.Background(), srv.URL+"/x?artist=Creed"); err != nil { + t.Fatal(err) + } + if method != http.MethodGet { + t.Errorf("method = %s", method) + } + if ua := got.Get("User-Agent"); ua != "Sonda/test ( https://example.org/sonda )" { + t.Errorf("User-Agent = %q", ua) + } + if got.Get("Accept") != "application/json" { + t.Errorf("Accept = %q", got.Get("Accept")) + } + for _, h := range []string{"Cookie", "Referer", "Authorization", "Origin", "X-Forwarded-For", "Proxy-Authorization"} { + if v := got.Get(h); v != "" { + t.Errorf("header %s = %q must never be sent to a provider", h, v) + } + } +} + +func TestCookiesAreNeverRemembered(t *testing.T) { + var second http.Header + srv, n := server(t, + func(w http.ResponseWriter, _ *http.Request) { + http.SetCookie(w, &http.Cookie{Name: "track", Value: "me"}) + _, _ = w.Write([]byte("{}")) + }, + func(w http.ResponseWriter, r *http.Request) { + second = r.Header.Clone() + _, _ = w.Write([]byte("{}")) + }) + c := newClient(newClock(), nil) + for i := 0; i < 2; i++ { + if _, err := c.Get(context.Background(), srv.URL); err != nil { + t.Fatal(err) + } + } + if n.Load() != 2 || second.Get("Cookie") != "" { + t.Errorf("a cookie set by the provider came back: %q", second.Get("Cookie")) + } +} + +func TestRedirectsAreRefused(t *testing.T) { + elsewhere, hits := server(t, ok("you should never see this")) + srv, _ := server(t, func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, elsewhere.URL+"/steal", http.StatusFound) + }) + c := newClient(newClock(), nil) + + _, err := c.Get(context.Background(), srv.URL+"/x") + if !errors.Is(err, ErrRedirect) { + t.Fatalf("err = %v, want ErrRedirect", err) + } + if hits.Load() != 0 { + t.Fatal("the client followed a redirect to another host") + } +} + +func TestOnlyHTTPURLsAreAccepted(t *testing.T) { + c := newClient(newClock(), nil) + for _, u := range []string{"", "ftp://example.org/x", "file:///etc/passwd", "http:///nohost", "://bad", "javascript:alert(1)"} { + if _, err := c.Get(context.Background(), u); !errors.Is(err, ErrBadURL) { + t.Errorf("%q: err = %v, want ErrBadURL", u, err) + } + } +} + +// ---- bounds ---- + +func TestResponseBodyIsBounded(t *testing.T) { + srv, _ := server(t, ok("0123456789")) + c := newClient(newClock(), func(o *Options) { o.MaxBody = 9 }) + if _, err := c.Get(context.Background(), srv.URL); !errors.Is(err, ErrResponseTooLarge) { + t.Fatalf("10 bytes over a limit of 9: err = %v", err) + } + c = newClient(newClock(), func(o *Options) { o.MaxBody = 10 }) + resp, err := c.Get(context.Background(), srv.URL) + if err != nil || string(resp.Body) != "0123456789" { + t.Fatalf("exactly the limit: %v %v", resp, err) + } +} + +func TestEachAttemptHasATimeout(t *testing.T) { + srv, n := server(t, func(w http.ResponseWriter, r *http.Request) { + select { + case <-time.After(2 * time.Second): + case <-r.Context().Done(): + } + }) + c := newClient(newClock(), func(o *Options) { o.Timeout = 50 * time.Millisecond; o.Retries = 1 }) + start := time.Now() + if _, err := c.Get(context.Background(), srv.URL); err == nil { + t.Fatal("a stuck server must end in an error") + } + if elapsed := time.Since(start); elapsed > time.Second { + t.Errorf("took %v: the per-attempt timeout did not apply", elapsed) + } + if n.Load() != 2 { + t.Errorf("attempts = %d, want 2 (one try and one retry)", n.Load()) + } +} + +func TestACancelledContextSendsNothing(t *testing.T) { + srv, n := server(t, ok("{}")) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := newClient(newClock(), nil).Get(ctx, srv.URL); !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v, want context.Canceled", err) + } + if n.Load() != 0 { + t.Error("a request was sent for a cancelled context") + } +} + +// ---- rate limit ---- + +func TestRequestsAreSpacedByTheMinimumInterval(t *testing.T) { + srv, _ := server(t, ok("{}")) + clock := newClock() + c := newClient(clock, func(o *Options) { o.MinInterval = time.Second }) + + for i := 0; i < 3; i++ { + if _, err := c.Get(context.Background(), srv.URL); err != nil { + t.Fatal(err) + } + } + // The first goes at once; each next one waits for the interval. + if got := clock.sleeps(); len(got) != 2 || got[0] != time.Second || got[1] != time.Second { + t.Errorf("sleeps = %v, want [1s 1s]", got) + } +} + +func TestSlotsAreNotWastedWhenTimePasses(t *testing.T) { + srv, _ := server(t, ok("{}")) + clock := newClock() + c := newClient(clock, func(o *Options) { o.MinInterval = time.Second }) + _, _ = c.Get(context.Background(), srv.URL) + clock.advance(5 * time.Second) // plenty of time passed + _, _ = c.Get(context.Background(), srv.URL) + if got := clock.sleeps(); len(got) != 0 { + t.Errorf("sleeps = %v, want none: the interval had already passed", got) + } +} + +// Many goroutines at once must still be spaced, with a real clock. +func TestConcurrentRequestsStaySpaced(t *testing.T) { + var mu sync.Mutex + var times []time.Time + srv, _ := server(t, func(w http.ResponseWriter, _ *http.Request) { + mu.Lock() + times = append(times, time.Now()) + mu.Unlock() + _, _ = w.Write([]byte("{}")) + }) + const gap = 20 * time.Millisecond + c := New(Options{UserAgent: "t", MinInterval: gap, Jitter: noJitter}) + + var wg sync.WaitGroup + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if _, err := c.Get(context.Background(), srv.URL); err != nil { + t.Error(err) + } + }() + } + wg.Wait() + + mu.Lock() + defer mu.Unlock() + if len(times) != 10 { + t.Fatalf("%d requests arrived, want 10", len(times)) + } + for i := 1; i < len(times); i++ { + for j := 0; j < i; j++ { // order of arrival may differ: compare every pair + d := times[i].Sub(times[j]) + if d < 0 { + d = -d + } + if d < gap/2 { + t.Fatalf("two requests arrived %v apart, want at least about %v", d, gap) + } + } + } +} + +// ---- retries ---- + +func TestRetriesTemporaryFailuresWithBackoff(t *testing.T) { + srv, n := server(t, status(503), status(502), ok("fine")) + clock := newClock() + c := newClient(clock, func(o *Options) { o.BackoffBase = time.Second }) + + resp, err := c.Get(context.Background(), srv.URL) + if err != nil || resp.Status != 200 || string(resp.Body) != "fine" { + t.Fatalf("resp=%+v err=%v", resp, err) + } + if n.Load() != 3 { + t.Errorf("attempts = %d, want 3", n.Load()) + } + if got := clock.sleeps(); len(got) != 2 || got[0] != time.Second || got[1] != 2*time.Second { + t.Errorf("backoff = %v, want [1s 2s]", got) + } +} + +func TestGivesUpAfterTheRetriesAndReturnsTheLastAnswer(t *testing.T) { + srv, n := server(t, status(503)) + c := newClient(newClock(), func(o *Options) { o.Retries = 2 }) + resp, err := c.Get(context.Background(), srv.URL) + if err != nil || resp.Status != 503 { + t.Fatalf("resp=%+v err=%v, want the final 503 as an answer", resp, err) + } + if n.Load() != 3 { + t.Errorf("attempts = %d, want 3 (one try, two retries)", n.Load()) + } +} + +func TestTooManyRequestsIsRetried(t *testing.T) { + srv, n := server(t, status(429), ok("{}")) + resp, err := newClient(newClock(), nil).Get(context.Background(), srv.URL) + if err != nil || resp.Status != 200 || n.Load() != 2 { + t.Fatalf("resp=%+v err=%v attempts=%d", resp, err, n.Load()) + } +} + +func TestClientErrorsAreNotRetried(t *testing.T) { + for _, code := range []int{400, 403, 404} { + srv, n := server(t, status(code)) + resp, err := newClient(newClock(), nil).Get(context.Background(), srv.URL) + if err != nil || resp.Status != code || n.Load() != 1 { + t.Errorf("%d: resp=%+v err=%v attempts=%d, want one attempt and the answer", code, resp, err, n.Load()) + } + } +} + +func TestRetryAfterIsHonoured(t *testing.T) { + srv, _ := server(t, + func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Retry-After", "5") + w.WriteHeader(503) + }, ok("{}")) + clock := newClock() + c := newClient(clock, func(o *Options) { o.BackoffBase = time.Second }) + if _, err := c.Get(context.Background(), srv.URL); err != nil { + t.Fatal(err) + } + if got := clock.sleeps(); len(got) != 1 || got[0] != 5*time.Second { + t.Errorf("sleeps = %v, want the 5 s the server asked for (more than the 1 s backoff)", got) + } +} + +func TestRetryAfterAsADate(t *testing.T) { + clock := newClock() + srv, _ := server(t, + func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Retry-After", clock.Now().Add(7*time.Second).UTC().Format(http.TimeFormat)) + w.WriteHeader(503) + }, ok("{}")) + c := newClient(clock, nil) + if _, err := c.Get(context.Background(), srv.URL); err != nil { + t.Fatal(err) + } + if got := clock.sleeps(); len(got) != 1 || got[0] < 6*time.Second || got[0] > 7*time.Second { + t.Errorf("sleeps = %v, want about 7 s", got) + } +} + +func TestAnAbsurdRetryAfterIsNotWaitedFor(t *testing.T) { + srv, n := server(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Retry-After", "3600") + w.WriteHeader(429) + }) + clock := newClock() + c := newClient(clock, func(o *Options) { o.MaxRetryAfter = 30 * time.Second }) + + _, err := c.Get(context.Background(), srv.URL) + var rl *RateLimitedError + if !errors.As(err, &rl) || rl.RetryAfter != time.Hour { + t.Fatalf("err = %v, want RateLimitedError with RetryAfter 1h", err) + } + if n.Load() != 1 || len(clock.sleeps()) != 0 { + t.Errorf("attempts=%d sleeps=%v: it must give up at once, not wait an hour", n.Load(), clock.sleeps()) + } +} + +func TestDoesNotWaitPastTheCallersDeadline(t *testing.T) { + srv, n := server(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Retry-After", "20") + w.WriteHeader(503) + }) + clock := newClock() + c := newClient(clock, nil) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) // real time: the fake clock cannot see it + defer cancel() + + start := time.Now() + resp, err := c.Get(ctx, srv.URL) + if time.Since(start) > time.Second { + t.Errorf("took %v: it waited although the deadline was 2 s away and the server asked for 20 s", time.Since(start)) + } + if n.Load() != 1 || len(clock.sleeps()) != 0 { + t.Errorf("attempts=%d sleeps=%v, want a single attempt and no wait", n.Load(), clock.sleeps()) + } + if err == nil && resp.Status != 503 { + t.Errorf("resp=%+v", resp) + } +} + +func TestATransportErrorIsRetried(t *testing.T) { + var n atomic.Int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if n.Add(1) == 1 { + conn, _, _ := w.(http.Hijacker).Hijack() + _ = conn.Close() // drop the connection without an answer + return + } + _, _ = w.Write([]byte("{}")) + })) + defer srv.Close() + + resp, err := newClient(newClock(), nil).Get(context.Background(), srv.URL) + if err != nil || resp.Status != 200 || n.Load() != 2 { + t.Fatalf("resp=%+v err=%v attempts=%d", resp, err, n.Load()) + } +} + +// ---- circuit breaker ---- + +func TestABrokenProviderFailsFast(t *testing.T) { + srv, n := server(t, status(500)) + clock := newClock() + c := newClient(clock, func(o *Options) { + o.Retries, o.BreakerThreshold, o.BreakerOpenFor = NoRetries, 3, 30*time.Second + }) + ctx := context.Background() + + for i := 0; i < 3; i++ { + if resp, err := c.Get(ctx, srv.URL); err != nil || resp.Status != 500 { + t.Fatalf("failure %d: %+v %v", i, resp, err) + } + } + // Open: no network at all. + before := n.Load() + if _, err := c.Get(ctx, srv.URL); !errors.Is(err, ErrCircuitOpen) { + t.Fatalf("err = %v, want ErrCircuitOpen", err) + } + if n.Load() != before { + t.Error("a request reached the server while the breaker was open") + } + + // After the window one request is let through; failing again reopens it for longer. + clock.advance(31 * time.Second) + if resp, err := c.Get(ctx, srv.URL); err != nil || resp.Status != 500 { + t.Fatalf("probe: %+v %v", resp, err) + } + clock.advance(31 * time.Second) // the window is now 60 s, so this is not enough + if _, err := c.Get(ctx, srv.URL); !errors.Is(err, ErrCircuitOpen) { + t.Fatalf("after a failed probe the window must double: err = %v", err) + } + clock.advance(30 * time.Second) + if _, err := c.Get(ctx, srv.URL); errors.Is(err, ErrCircuitOpen) { + t.Fatal("still open after the doubled window") + } +} + +func TestASuccessClosesTheBreaker(t *testing.T) { + srv, _ := server(t, status(500), status(500), ok("{}"), status(500), status(500), ok("{}")) + clock := newClock() + c := newClient(clock, func(o *Options) { o.Retries, o.BreakerThreshold = NoRetries, 3 }) + ctx := context.Background() + + // Two failures, a success, two failures: never three in a row, so it stays closed. + for i := 0; i < 6; i++ { + if _, err := c.Get(ctx, srv.URL); errors.Is(err, ErrCircuitOpen) { + t.Fatalf("request %d: the breaker opened although failures were never consecutive", i) + } + } +} + +func TestClientErrorsDoNotOpenTheBreaker(t *testing.T) { + srv, n := server(t, status(404)) + c := newClient(newClock(), func(o *Options) { o.Retries, o.BreakerThreshold = NoRetries, 2 }) + for i := 0; i < 6; i++ { + if _, err := c.Get(context.Background(), srv.URL); err != nil { + t.Fatalf("request %d: %v", i, err) + } + } + if n.Load() != 6 { + t.Errorf("attempts = %d: 404 is an answer, not a failure of the provider", n.Load()) + } +} + +func TestDefaultsAreSane(t *testing.T) { + c := New(Options{UserAgent: "x"}) + if c.timeout <= 0 || c.maxBody <= 0 || c.retries != 2 || c.breakerThreshold <= 0 { + t.Errorf("defaults: timeout=%v maxBody=%d retries=%d threshold=%d", c.timeout, c.maxBody, c.retries, c.breakerThreshold) + } + if !strings.Contains(strings.ToLower(c.userAgent), "x") { + t.Errorf("user agent = %q", c.userAgent) + } +} diff --git a/internal/providers/settings.go b/internal/providers/settings.go new file mode 100644 index 0000000..33edd99 --- /dev/null +++ b/internal/providers/settings.go @@ -0,0 +1,266 @@ +// Package providers manages the external services Sonda can ask about artists +// (Last.fm, MusicBrainz) and the rules for talking to them. +// +// Nothing here makes a request on its own. Every provider is off until an admin +// turns it on, and the settings (including any API key) live in the database. +package providers + +import ( + "context" + "errors" + "fmt" + "regexp" + "strings" + "sync" + "time" + "unicode" + "unicode/utf8" +) + +// The providers Sonda knows. +const ( + LastFM = "lastfm" + MusicBrainz = "musicbrainz" +) + +// Errors returned by the service. +var ( + ErrUnknownProvider = errors.New("unknown provider") + ErrInvalidSettings = errors.New("invalid provider settings") +) + +const maxContactLen = 200 + +var lastfmKeyRe = regexp.MustCompile(`^[0-9a-f]{32}$`) + +// Settings are the stored settings of one provider, API key included. They are +// only for code that talks to the provider; the API never returns the key. +type Settings struct { + Provider string + Enabled bool + APIKey string + Contact string // goes into the User-Agent; empty means the project address + UpdatedAt time.Time +} + +// Info is what the API shows about a provider: the settings without the key, +// plus a plain statement of what leaves the server. +type Info struct { + Name string + Enabled bool + NeedsAPIKey bool + HasAPIKey bool + Contact string + Sends string // what is sent to the provider, and what never is + Notes string // terms of use and attribution + UpdatedAt time.Time // zero when the provider was never configured +} + +// Patch changes some settings. A nil field is left alone. An empty APIKey or +// Contact clears it. +type Patch struct { + Enabled *bool + APIKey *string + Contact *string +} + +// Store persists provider settings. +type Store interface { + // Get returns found=false when the provider was never configured. + Get(ctx context.Context, name string) (s Settings, found bool, err error) + List(ctx context.Context) ([]Settings, error) + Save(ctx context.Context, s Settings) error +} + +type meta struct { + name string + needsKey bool + sends, note string +} + +// known lists the providers in the order the API shows them. +var known = []meta{ + { + name: LastFM, + needsKey: true, + sends: "The artist name (or MusicBrainz id) you look up, your API key, your IP address and the " + + "User-Agent. Never your users, your listens, track titles or file paths.", + note: "Last.fm data is for non-commercial use only. Credit Last.fm when you show it. " + + "Images are not licensed and are never stored.", + }, + { + name: MusicBrainz, + sends: "The artist name or MusicBrainz id you look up, your IP address and the User-Agent, " + + "which carries the contact below. Never your users, your listens, track titles or file paths.", + note: "Core data (artist names and ids) is public domain. Tags and genres are CC BY-NC-SA 3.0: " + + "non-commercial, with attribution to MusicBrainz.", + }, +} + +func lookup(name string) (meta, bool) { + for _, m := range known { + if m.name == name { + return m, true + } + } + return meta{}, false +} + +// Service reads and changes provider settings. +type Service struct { + store Store + now func() time.Time + mu sync.Mutex // an update reads then writes: keep two of them apart +} + +// NewService builds a Service. +func NewService(store Store) *Service { return &Service{store: store, now: time.Now} } + +func info(m meta, s Settings) Info { + return Info{ + Name: m.name, NeedsAPIKey: m.needsKey, Sends: m.sends, Notes: m.note, + Enabled: s.Enabled, HasAPIKey: s.APIKey != "", Contact: s.Contact, UpdatedAt: s.UpdatedAt, + } +} + +// List returns every known provider, configured or not, in a fixed order. +func (s *Service) List(ctx context.Context) ([]Info, error) { + rows, err := s.store.List(ctx) + if err != nil { + return nil, fmt.Errorf("list provider settings: %w", err) + } + byName := map[string]Settings{} + for _, r := range rows { + byName[r.Provider] = r + } + out := make([]Info, 0, len(known)) + for _, m := range known { + out = append(out, info(m, byName[m.name])) + } + return out, nil +} + +// Update applies a patch and returns the provider as it is afterwards. +func (s *Service) Update(ctx context.Context, name string, p Patch) (Info, error) { + m, ok := lookup(name) + if !ok { + return Info{}, ErrUnknownProvider + } + if p.Enabled == nil && p.APIKey == nil && p.Contact == nil { + return Info{}, fmt.Errorf("%w: nothing to change", ErrInvalidSettings) + } + + s.mu.Lock() + defer s.mu.Unlock() + + cur, _, err := s.store.Get(ctx, name) + if err != nil { + return Info{}, fmt.Errorf("read provider settings: %w", err) + } + cur.Provider = name + + if p.APIKey != nil { + key, err := validateKey(m, *p.APIKey) + if err != nil { + return Info{}, err + } + cur.APIKey = key + if key == "" { + cur.Enabled = false // a provider that needs a key is off without one + } + } + if p.Contact != nil { + contact, err := validateContact(*p.Contact) + if err != nil { + return Info{}, err + } + cur.Contact = contact + } + if p.Enabled != nil { + if *p.Enabled && m.needsKey && cur.APIKey == "" { + return Info{}, fmt.Errorf("%w: %s needs an API key before it can be enabled", ErrInvalidSettings, name) + } + cur.Enabled = *p.Enabled + } + + cur.UpdatedAt = s.now() + if err := s.store.Save(ctx, cur); err != nil { + return Info{}, fmt.Errorf("save provider settings: %w", err) + } + return info(m, cur), nil +} + +// Active returns the settings of a provider when it can be used right now: it +// is on and, if it needs a key, has one. Provider clients call this before +// every outbound request; false means "do not contact anyone". +func (s *Service) Active(ctx context.Context, name string) (Settings, bool, error) { + m, ok := lookup(name) + if !ok { + return Settings{}, false, ErrUnknownProvider + } + cur, found, err := s.store.Get(ctx, name) + if err != nil { + return Settings{}, false, fmt.Errorf("read provider settings: %w", err) + } + if !found || !cur.Enabled || (m.needsKey && cur.APIKey == "") { + return Settings{}, false, nil + } + return cur, true, nil +} + +func validateKey(m meta, raw string) (string, error) { + key := strings.ToLower(strings.TrimSpace(raw)) + if key == "" { + return "", nil + } + if !m.needsKey { + return "", fmt.Errorf("%w: %s does not use an API key", ErrInvalidSettings, m.name) + } + if m.name == LastFM && !lastfmKeyRe.MatchString(key) { + return "", fmt.Errorf("%w: a Last.fm API key is 32 hexadecimal characters", ErrInvalidSettings) + } + return key, nil +} + +// validateContact accepts what MusicBrainz asks for in a User-Agent: a URL or an +// e-mail address. Parentheses are refused because they delimit it there. +func validateContact(raw string) (string, error) { + c := strings.TrimSpace(raw) + if utf8.RuneCountInString(c) > maxContactLen { + return "", fmt.Errorf("%w: contact is longer than %d characters", ErrInvalidSettings, maxContactLen) + } + for _, r := range c { + if unicode.IsControl(r) || r == '(' || r == ')' { + return "", fmt.Errorf("%w: contact cannot contain control characters or parentheses", ErrInvalidSettings) + } + } + return c, nil +} + +// ProjectURL is the contact used in the User-Agent when no other is set. +const ProjectURL = "https://codeberg.org/squidd/sonda" + +// UserAgent builds the header sent to providers, in the shape MusicBrainz asks +// for: "Application/version ( contact )". The version and the contact are +// cleaned, so the result is always a single safe line. +func UserAgent(version, contact string) string { + clean := func(s string) string { + var b strings.Builder + for _, r := range s { + if unicode.IsControl(r) || r == '(' || r == ')' { + continue + } + b.WriteRune(r) + } + return strings.TrimSpace(b.String()) + } + v := strings.ReplaceAll(clean(version), " ", "") + if v == "" { + v = "dev" + } + c := clean(contact) + if c == "" { + c = ProjectURL + } + return fmt.Sprintf("Sonda/%s ( %s )", v, c) +} diff --git a/internal/providers/settings_test.go b/internal/providers/settings_test.go new file mode 100644 index 0000000..20cb1fe --- /dev/null +++ b/internal/providers/settings_test.go @@ -0,0 +1,261 @@ +package providers + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + "time" +) + +// memStore is an in-memory Store. +type memStore struct { + rows map[string]Settings + err error +} + +func newMemStore() *memStore { return &memStore{rows: map[string]Settings{}} } + +func (m *memStore) Get(_ context.Context, name string) (Settings, bool, error) { + s, ok := m.rows[name] + return s, ok, m.err +} + +func (m *memStore) List(context.Context) ([]Settings, error) { + var out []Settings + for _, s := range m.rows { + out = append(out, s) + } + return out, m.err +} + +func (m *memStore) Save(_ context.Context, s Settings) error { + if m.err != nil { + return m.err + } + m.rows[s.Provider] = s + return nil +} + +const goodKey = "0123456789abcdef0123456789abcdef" + +func boolp(b bool) *bool { return &b } +func strp(s string) *string { return &s } +func newSvc() (*Service, *memStore) { + st := newMemStore() + return NewService(st), st +} + +func TestEveryProviderIsOffByDefault(t *testing.T) { + s, _ := newSvc() + infos, err := s.List(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(infos) != 2 || infos[0].Name != LastFM || infos[1].Name != MusicBrainz { + t.Fatalf("providers = %+v, want lastfm then musicbrainz", infos) + } + for _, in := range infos { + if in.Enabled || in.HasAPIKey { + t.Errorf("%s is on or has a key before anything was configured: %+v", in.Name, in) + } + if strings.TrimSpace(in.Sends) == "" || strings.TrimSpace(in.Notes) == "" { + t.Errorf("%s does not say what it sends or under which terms", in.Name) + } + } + if !infos[0].NeedsAPIKey || infos[1].NeedsAPIKey { + t.Errorf("needs_api_key: lastfm=%v musicbrainz=%v, want true/false", infos[0].NeedsAPIKey, infos[1].NeedsAPIKey) + } +} + +func TestWhatEachProviderSendsNeverMentionsPrivateData(t *testing.T) { + s, _ := newSvc() + infos, _ := s.List(context.Background()) + for _, in := range infos { + if !strings.Contains(in.Sends, "Never") { + t.Errorf("%s: the description of what is sent must say what is never sent: %q", in.Name, in.Sends) + } + } +} + +func TestInfoNeverCarriesTheKey(t *testing.T) { + typ := reflect.TypeOf(Info{}) + for i := 0; i < typ.NumField(); i++ { + if name := strings.ToLower(typ.Field(i).Name); name == "apikey" || strings.Contains(name, "secret") || name == "key" { + t.Fatalf("Info has a field %q that could carry the key", typ.Field(i).Name) + } + } + s, _ := newSvc() + info, err := s.Update(context.Background(), LastFM, Patch{APIKey: strp(goodKey)}) + if err != nil { + t.Fatal(err) + } + if !info.HasAPIKey { + t.Error("HasAPIKey should be true after saving a key") + } +} + +func TestSettingAndEnablingLastFM(t *testing.T) { + s, st := newSvc() + ctx := context.Background() + + // Enabling without a key is refused. + if _, err := s.Update(ctx, LastFM, Patch{Enabled: boolp(true)}); !errors.Is(err, ErrInvalidSettings) { + t.Fatalf("enable without a key: err = %v", err) + } + if _, ok := st.rows[LastFM]; ok { + t.Error("a refused update left a row behind") + } + + info, err := s.Update(ctx, LastFM, Patch{APIKey: strp(" " + strings.ToUpper(goodKey) + " ")}) + if err != nil { + t.Fatal(err) + } + if !info.HasAPIKey || info.Enabled { + t.Errorf("after saving a key: %+v, want a key and still off", info) + } + if st.rows[LastFM].APIKey != goodKey { + t.Errorf("stored key = %q, want it trimmed and lower-cased", st.rows[LastFM].APIKey) + } + + if info, err = s.Update(ctx, LastFM, Patch{Enabled: boolp(true)}); err != nil || !info.Enabled { + t.Fatalf("enable: %+v err=%v", info, err) + } + + // Clearing the key turns the provider off with it. + info, err = s.Update(ctx, LastFM, Patch{APIKey: strp("")}) + if err != nil || info.HasAPIKey || info.Enabled { + t.Fatalf("clear the key: %+v err=%v, want no key and off", info, err) + } + if st.rows[LastFM].APIKey != "" || st.rows[LastFM].Enabled { + t.Errorf("stored row after clearing: %+v", st.rows[LastFM]) + } +} + +func TestUpdateRejectsBadInput(t *testing.T) { + ctx := context.Background() + tests := []struct { + name string + provider string + patch Patch + }{ + {"empty patch", LastFM, Patch{}}, + {"key too short", LastFM, Patch{APIKey: strp("abc123")}}, + {"key not hex", LastFM, Patch{APIKey: strp(strings.Repeat("z", 32))}}, + {"key too long", LastFM, Patch{APIKey: strp(goodKey + "0")}}, + {"a key for a provider without keys", MusicBrainz, Patch{APIKey: strp(goodKey)}}, + {"contact with parentheses", MusicBrainz, Patch{Contact: strp("me (at) example.org")}}, + {"contact with a line break", MusicBrainz, Patch{Contact: strp("a\nb")}}, + {"contact too long", MusicBrainz, Patch{Contact: strp(strings.Repeat("x", 201))}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s, st := newSvc() + if _, err := s.Update(ctx, tt.provider, tt.patch); !errors.Is(err, ErrInvalidSettings) { + t.Fatalf("err = %v, want ErrInvalidSettings", err) + } + if len(st.rows) != 0 { + t.Error("a rejected update was stored") + } + }) + } +} + +func TestUnknownProvider(t *testing.T) { + s, _ := newSvc() + if _, err := s.Update(context.Background(), "spotify", Patch{Enabled: boolp(true)}); !errors.Is(err, ErrUnknownProvider) { + t.Fatalf("err = %v, want ErrUnknownProvider", err) + } + if _, _, err := s.Active(context.Background(), "spotify"); !errors.Is(err, ErrUnknownProvider) { + t.Fatalf("Active: err = %v, want ErrUnknownProvider", err) + } +} + +func TestMusicBrainzNeedsNoKeyAndTakesAContact(t *testing.T) { + s, st := newSvc() + ctx := context.Background() + info, err := s.Update(ctx, MusicBrainz, Patch{Enabled: boolp(true), Contact: strp(" https://example.org/me ")}) + if err != nil || !info.Enabled || info.Contact != "https://example.org/me" { + t.Fatalf("%+v err=%v", info, err) + } + if st.rows[MusicBrainz].Contact != "https://example.org/me" { + t.Errorf("stored contact = %q", st.rows[MusicBrainz].Contact) + } + // An empty contact goes back to the default (the project's address). + info, err = s.Update(ctx, MusicBrainz, Patch{Contact: strp("")}) + if err != nil || info.Contact != "" || !info.Enabled { + t.Fatalf("clear the contact: %+v err=%v", info, err) + } +} + +func TestActiveOnlyReturnsAUsableProvider(t *testing.T) { + s, _ := newSvc() + ctx := context.Background() + + if _, ok, err := s.Active(ctx, LastFM); err != nil || ok { + t.Fatalf("nothing configured: ok=%v err=%v", ok, err) + } + _, _ = s.Update(ctx, LastFM, Patch{APIKey: strp(goodKey)}) + if _, ok, _ := s.Active(ctx, LastFM); ok { + t.Error("a provider with a key but turned off was reported active") + } + _, _ = s.Update(ctx, LastFM, Patch{Enabled: boolp(true)}) + cfg, ok, err := s.Active(ctx, LastFM) + if err != nil || !ok || cfg.APIKey != goodKey { + t.Fatalf("active: %+v ok=%v err=%v", cfg, ok, err) + } + _, _ = s.Update(ctx, LastFM, Patch{Enabled: boolp(false)}) + if _, ok, _ := s.Active(ctx, LastFM); ok { + t.Error("a provider turned off was reported active") + } + + // MusicBrainz is usable as soon as it is on. + _, _ = s.Update(ctx, MusicBrainz, Patch{Enabled: boolp(true)}) + if _, ok, _ := s.Active(ctx, MusicBrainz); !ok { + t.Error("musicbrainz on, no key needed: should be active") + } +} + +func TestListMergesStoredRowsInAFixedOrder(t *testing.T) { + s, st := newSvc() + st.rows[MusicBrainz] = Settings{Provider: MusicBrainz, Enabled: true, Contact: "c", UpdatedAt: time.Unix(1, 0)} + st.rows["stale-unknown"] = Settings{Provider: "stale-unknown", Enabled: true} // a row of a provider that no longer exists + infos, err := s.List(context.Background()) + if err != nil || len(infos) != 2 { + t.Fatalf("infos = %+v err=%v, want exactly the known providers", infos, err) + } + if infos[0].Name != LastFM || infos[1].Name != MusicBrainz || !infos[1].Enabled || infos[1].Contact != "c" { + t.Errorf("infos = %+v", infos) + } +} + +func TestStoreErrorsPassThrough(t *testing.T) { + boom := errors.New("db down") + s := NewService(&memStore{rows: map[string]Settings{}, err: boom}) + if _, err := s.List(context.Background()); !errors.Is(err, boom) { + t.Errorf("List: %v", err) + } + if _, err := s.Update(context.Background(), LastFM, Patch{APIKey: strp(goodKey)}); !errors.Is(err, boom) { + t.Errorf("Update: %v", err) + } +} + +func TestUserAgentIsIdentifiableAndSafe(t *testing.T) { + if got := UserAgent("1.2.3", ""); got != "Sonda/1.2.3 ( https://codeberg.org/squidd/sonda )" { + t.Errorf("default contact: %q", got) + } + if got := UserAgent("1.2.3", " https://example.org/me "); got != "Sonda/1.2.3 ( https://example.org/me )" { + t.Errorf("with a contact: %q", got) + } + if got := UserAgent("", ""); !strings.HasPrefix(got, "Sonda/dev ( ") { + t.Errorf("no version: %q", got) + } + // Whatever is passed in, the header stays one line and keeps its shape. + for _, evil := range []string{"a\r\nX-Evil: 1", "a ) b ( c", "\x00\x07"} { + ua := UserAgent("1.0\r\nX: y", evil) + if strings.ContainsAny(ua, "\r\n\x00") || strings.Count(ua, "(") != 1 || strings.Count(ua, ")") != 1 { + t.Errorf("UserAgent(%q) = %q is not a safe single-line header", evil, ua) + } + } +} diff --git a/internal/storage/providers.go b/internal/storage/providers.go new file mode 100644 index 0000000..27c5687 --- /dev/null +++ b/internal/storage/providers.go @@ -0,0 +1,81 @@ +package storage + +import ( + "context" + "database/sql" + "errors" + "fmt" + "time" + + "github.com/uptrace/bun" + + "codeberg.org/squidd/sonda/internal/providers" +) + +type providerRow struct { + bun.BaseModel `bun:"table:provider_settings"` + + Provider string `bun:"provider,pk"` + Enabled bool `bun:"enabled,notnull"` + APIKey string `bun:"api_key,notnull"` + Contact string `bun:"contact,notnull"` + UpdatedAt int64 `bun:"updated_at,notnull"` +} + +func (r providerRow) toSettings() providers.Settings { + return providers.Settings{ + Provider: r.Provider, Enabled: r.Enabled, APIKey: r.APIKey, Contact: r.Contact, + UpdatedAt: time.Unix(r.UpdatedAt, 0), + } +} + +// ProviderRepo stores provider settings. It implements providers.Store. +type ProviderRepo struct{ db *bun.DB } + +// NewProviderRepo builds a ProviderRepo. +func NewProviderRepo(db *bun.DB) *ProviderRepo { return &ProviderRepo{db: db} } + +// Get returns the settings of one provider, with found=false if there are none. +func (r *ProviderRepo) Get(ctx context.Context, name string) (providers.Settings, bool, error) { + var row providerRow + err := r.db.NewSelect().Model(&row).Where("provider = ?", name).Limit(1).Scan(ctx) + if errors.Is(err, sql.ErrNoRows) { + return providers.Settings{}, false, nil + } + if err != nil { + return providers.Settings{}, false, fmt.Errorf("select provider settings: %w", err) + } + return row.toSettings(), true, nil +} + +// List returns every stored row. +func (r *ProviderRepo) List(ctx context.Context) ([]providers.Settings, error) { + var rows []providerRow + if err := r.db.NewSelect().Model(&rows).OrderExpr("provider").Scan(ctx); err != nil { + return nil, fmt.Errorf("list provider settings: %w", err) + } + out := make([]providers.Settings, len(rows)) + for i, row := range rows { + out[i] = row.toSettings() + } + return out, nil +} + +// Save inserts or updates the row of a provider. +func (r *ProviderRepo) Save(ctx context.Context, s providers.Settings) error { + row := &providerRow{ + Provider: s.Provider, Enabled: s.Enabled, APIKey: s.APIKey, + Contact: s.Contact, UpdatedAt: s.UpdatedAt.Unix(), + } + _, err := r.db.NewInsert().Model(row). + On("CONFLICT (provider) DO UPDATE"). + Set("enabled = EXCLUDED.enabled"). + Set("api_key = EXCLUDED.api_key"). + Set("contact = EXCLUDED.contact"). + Set("updated_at = EXCLUDED.updated_at"). + Exec(ctx) + if err != nil { + return fmt.Errorf("save provider settings: %w", err) + } + return nil +} diff --git a/internal/storage/providers_test.go b/internal/storage/providers_test.go new file mode 100644 index 0000000..87a70e1 --- /dev/null +++ b/internal/storage/providers_test.go @@ -0,0 +1,72 @@ +package storage + +import ( + "context" + "testing" + "time" + + "github.com/uptrace/bun" + + "codeberg.org/squidd/sonda/internal/providers" +) + +func TestProviderRepoSaveGetList(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + ctx := context.Background() + repo := NewProviderRepo(db) + + if _, found, err := repo.Get(ctx, providers.LastFM); err != nil || found { + t.Fatalf("never configured: found=%v err=%v", found, err) + } + if rows, err := repo.List(ctx); err != nil || len(rows) != 0 { + t.Fatalf("empty list: %v err=%v", rows, err) + } + + at := time.Unix(1_700_000_000, 0) + want := providers.Settings{Provider: providers.LastFM, Enabled: true, APIKey: "0123456789abcdef0123456789abcdef", Contact: "x", UpdatedAt: at} + if err := repo.Save(ctx, want); err != nil { + t.Fatal(err) + } + got, found, err := repo.Get(ctx, providers.LastFM) + if err != nil || !found { + t.Fatalf("Get: found=%v err=%v", found, err) + } + if got.Provider != want.Provider || !got.Enabled || got.APIKey != want.APIKey || got.Contact != "x" || !got.UpdatedAt.Equal(at) { + t.Errorf("got %+v, want %+v", got, want) + } + + // Saving again updates the same row. + want.Enabled, want.APIKey, want.Contact = false, "", "" + want.UpdatedAt = at.Add(time.Hour) + if err := repo.Save(ctx, want); err != nil { + t.Fatal(err) + } + got, _, _ = repo.Get(ctx, providers.LastFM) + if got.Enabled || got.APIKey != "" || got.Contact != "" || !got.UpdatedAt.Equal(at.Add(time.Hour)) { + t.Errorf("after the second save: %+v", got) + } + if n := count(t, db, "SELECT COUNT(*) FROM provider_settings"); n != 1 { + t.Errorf("rows = %d, want 1: a save must update, not duplicate", n) + } + + _ = repo.Save(ctx, providers.Settings{Provider: providers.MusicBrainz, Enabled: true, UpdatedAt: at}) + rows, err := repo.List(ctx) + if err != nil || len(rows) != 2 { + t.Fatalf("list: %v err=%v", rows, err) + } + }) +} + +// A provider that was never configured must not appear as enabled: the table +// default is "off". +func TestProviderTableDefaultsToOff(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + if _, err := db.Exec("INSERT INTO provider_settings (provider, updated_at) VALUES ('lastfm', 1)"); err != nil { + t.Fatal(err) + } + got, found, err := NewProviderRepo(db).Get(context.Background(), providers.LastFM) + if err != nil || !found || got.Enabled || got.APIKey != "" || got.Contact != "" { + t.Fatalf("defaults: %+v found=%v err=%v", got, found, err) + } + }) +} diff --git a/internal/storage/storage_test.go b/internal/storage/storage_test.go index ef05f23..a4407ea 100644 --- a/internal/storage/storage_test.go +++ b/internal/storage/storage_test.go @@ -147,7 +147,7 @@ func TestMigrateCreatesSchemaAndIsIdempotent(t *testing.T) { if err := Migrate(context.Background(), db, fsys); err != nil { t.Fatalf("second Migrate: %v", err) } - for _, table := range []string{"artists", "albums", "tracks", "track_artists", "audio_files", "users", "listening_events", "refresh_tokens", "libraries"} { + for _, table := range []string{"artists", "albums", "tracks", "track_artists", "audio_files", "users", "listening_events", "refresh_tokens", "libraries", "provider_settings"} { if !tableExists(t, db, driver, table) { t.Errorf("table %s missing", table) } @@ -156,8 +156,8 @@ func TestMigrateCreatesSchemaAndIsIdempotent(t *testing.T) { if err := db.QueryRow("SELECT COUNT(*) FROM schema_migrations").Scan(&applied); err != nil { t.Fatal(err) } - if applied != 6 { - t.Errorf("applied migrations = %d, want 6", applied) + if applied != 7 { + t.Errorf("applied migrations = %d, want 7", applied) } }) } diff --git a/migrations/postgres/0007_provider_settings.sql b/migrations/postgres/0007_provider_settings.sql new file mode 100644 index 0000000..d51da50 --- /dev/null +++ b/migrations/postgres/0007_provider_settings.sql @@ -0,0 +1,11 @@ +-- Settings of the external providers (Last.fm, MusicBrainz). A provider with no +-- row, or with enabled = false, is never contacted. The API key is stored as given +-- and is never returned by the API. + +CREATE TABLE provider_settings ( + provider TEXT PRIMARY KEY, + enabled BOOLEAN NOT NULL DEFAULT FALSE, + api_key TEXT NOT NULL DEFAULT '', + contact TEXT NOT NULL DEFAULT '', -- goes into the User-Agent; '' = the project address + updated_at BIGINT NOT NULL -- unix seconds +); diff --git a/migrations/sqlite/0007_provider_settings.sql b/migrations/sqlite/0007_provider_settings.sql new file mode 100644 index 0000000..a500d08 --- /dev/null +++ b/migrations/sqlite/0007_provider_settings.sql @@ -0,0 +1,11 @@ +-- Settings of the external providers (Last.fm, MusicBrainz). A provider with no +-- row, or with enabled = 0, is never contacted. The API key is stored as given +-- and is never returned by the API. + +CREATE TABLE provider_settings ( + provider TEXT PRIMARY KEY, + enabled INTEGER NOT NULL DEFAULT 0, + api_key TEXT NOT NULL DEFAULT '', + contact TEXT NOT NULL DEFAULT '', -- goes into the User-Agent; '' = the project address + updated_at INTEGER NOT NULL -- unix seconds +); diff --git a/sonda.example.toml b/sonda.example.toml index 1829a72..b4f0b76 100644 --- a/sonda.example.toml +++ b/sonda.example.toml @@ -1,7 +1,7 @@ # Copy to sonda.toml and adjust. sonda.toml is git-ignored. # Precedence: defaults < this file < environment variables # (SONDA_ENV, SONDA_LISTEN, SONDA_DB_DRIVER, SONDA_DB_PATH, SONDA_DB_DSN, -# SONDA_JWT_SECRET, SONDA_LASTFM_API_KEY). +# SONDA_JWT_SECRET). [server] listen = "127.0.0.1:8080" # localhost by default @@ -26,5 +26,4 @@ refresh_ttl_hours = 720 # lifetime of a refresh token (30 days) [listening] min_listen_seconds = 20 # a play counts as a listen after this long, or when it completes -[providers.lastfm] -api_key = "" +# Last.fm and MusicBrainz are set up through the API, not here: PUT /api/v1/providers/{name}. From e2318601194bda1e0e527cbc77c949b84f992687 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Wed, 11 Nov 2026 16:11:53 +0100 Subject: [PATCH 2/2] fix(providers): keep the key out of errors and make the outbound client robust - fix: errors from the outbound client no longer carry the query string. Go puts the whole URL into a transport error, and a Last.fm key travels in the query, so any log of such an error would have written the key. The host and path stay for debugging and the cause can still be matched with errors.Is - fix: a request that gives up while waiting for its turn gives the turn back, and a turn is not taken at all when the wait would pass the caller's deadline. Before, a burst of impatient callers pushed the queue further away until no request ever reached the provider - fix: failures of requests that were already in flight when the provider died no longer double the breaker window each; one outage opens it once - fix: redirects and oversized answers count as failures, so a provider that keeps answering that way stops being asked - fix: sending an empty api_key to a provider that needs none (MusicBrainz) no longer turns it off - test: the query string in transport errors, refused connections and timeouts; cancelling during the wait; the deadline shorter than the wait; twelve concurrent failures; redirects and big answers; the empty key. Checked by removing each fix on purpose --- internal/api/openapi/openapi.yaml | 4 +- internal/providers/outbound/outbound.go | 52 ++++- internal/providers/outbound/outbound_test.go | 202 +++++++++++++++++++ internal/providers/settings.go | 2 +- internal/providers/settings_test.go | 16 ++ 5 files changed, 268 insertions(+), 8 deletions(-) diff --git a/internal/api/openapi/openapi.yaml b/internal/api/openapi/openapi.yaml index 3af3ad9..e05d1c8 100644 --- a/internal/api/openapi/openapi.yaml +++ b/internal/api/openapi/openapi.yaml @@ -793,8 +793,8 @@ paths: Admin only. Send any of `enabled`, `api_key` and `contact`; fields you leave out are kept. The key is write-only: it is never returned and never logged. Sending an empty `api_key` - clears it and turns the provider off. A provider that needs a key (Last.fm) cannot be - enabled without one. `contact` goes into the User-Agent sent to the provider (MusicBrainz + clears it, and turns a provider that needs a key (Last.fm) off; a provider that needs none + (MusicBrainz) is not affected. A provider that needs a key cannot be enabled without one. `contact` goes into the User-Agent sent to the provider (MusicBrainz asks for a URL or an e-mail address); empty means the project address. security: [{bearerAuth: []}] requestBody: diff --git a/internal/providers/outbound/outbound.go b/internal/providers/outbound/outbound.go index a4ac20c..200a32d 100644 --- a/internal/providers/outbound/outbound.go +++ b/internal/providers/outbound/outbound.go @@ -204,6 +204,9 @@ func (c *Client) Get(ctx context.Context, rawURL string) (*Response, error) { resp, err := c.once(ctx, rawURL) switch { case errors.Is(err, ErrRedirect), errors.Is(err, ErrResponseTooLarge): + // Not worth retrying, but a provider that keeps answering like this + // must stop being asked. + c.failed() return nil, err case err != nil: if ctx.Err() != nil { @@ -270,7 +273,7 @@ func (c *Client) once(ctx context.Context, rawURL string) (*Response, error) { resp, err := c.hc.Do(req) if err != nil { - return nil, err + return nil, redactQuery(err) } defer resp.Body.Close() @@ -287,6 +290,23 @@ func (c *Client) once(ctx context.Context, rawURL string) (*Response, error) { return &Response{Status: resp.StatusCode, Header: resp.Header.Clone(), Body: body}, nil } +// redactQuery removes the query string, fragment and credentials from the URL +// that net/http puts into its errors. A Last.fm key travels in the query, and +// an error that reaches a log must not carry it. The host and path stay, for +// debugging, and the error can still be matched with errors.Is. +func redactQuery(err error) error { + var ue *url.Error + if !errors.As(err, &ue) { + return err + } + if u, perr := url.Parse(ue.URL); perr == nil { + ue.URL = u.Scheme + "://" + u.Host + u.Path + } else { + ue.URL = "(unreadable URL)" + } + return err +} + func retryable(status int) bool { switch status { case http.StatusTooManyRequests, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout: @@ -322,7 +342,10 @@ func retryAfter(h http.Header, now time.Time) (time.Duration, bool) { return 0, false } -// throttle reserves the next free slot and waits for it. +// throttle reserves the next free slot and waits for it. A slot is only taken +// when the caller can use it before its deadline, and it is given back when the +// caller gives up while waiting. Otherwise a burst of impatient callers would +// push the queue further and further away until nobody ever reached the provider. func (c *Client) throttle(ctx context.Context) error { if c.minInterval <= 0 { return nil @@ -333,11 +356,24 @@ func (c *Client) throttle(ctx context.Context) error { if at.Before(now) { at = now } - c.next = at.Add(c.minInterval) + wait := at.Sub(now) + if dl, ok := ctx.Deadline(); ok && wait > 0 && time.Until(dl) < wait { + c.mu.Unlock() + return fmt.Errorf("%w: waiting %v for our turn would pass the deadline", context.DeadlineExceeded, wait) + } + taken := at.Add(c.minInterval) + c.next = taken c.mu.Unlock() - if wait := at.Sub(now); wait > 0 { - return c.clock.Sleep(ctx, wait) + if wait > 0 { + if err := c.clock.Sleep(ctx, wait); err != nil { + c.mu.Lock() + if c.next.Equal(taken) { // nobody queued behind us: hand the turn back + c.next = at + } + c.mu.Unlock() + return err + } } return nil } @@ -367,6 +403,12 @@ func (c *Client) failed() { if c.failures < c.breakerThreshold { return } + // A failure that arrives while the breaker is already open belongs to the + // requests that were in flight when the provider died: the same outage, not + // a new reason to wait longer. Only a probe after the window may extend it. + if c.clock.Now().Before(c.openUntil) { + return + } if c.openFor == 0 { c.openFor = c.breakerOpenFor } else if c.openFor *= 2; c.openFor > c.breakerMaxOpen { diff --git a/internal/providers/outbound/outbound_test.go b/internal/providers/outbound/outbound_test.go index 8fac1f4..b74d3df 100644 --- a/internal/providers/outbound/outbound_test.go +++ b/internal/providers/outbound/outbound_test.go @@ -496,3 +496,205 @@ func TestDefaultsAreSane(t *testing.T) { t.Errorf("user agent = %q", c.userAgent) } } + +// ---- review fixes ---- + +// failingTransport makes every request fail the way a dead network does. +type failingTransport struct{} + +func (failingTransport) RoundTrip(*http.Request) (*http.Response, error) { + return nil, errors.New("boom: network is down") +} + +// A Last.fm key travels in the query string. Go puts the whole URL into a +// transport error, so the client must cut the query out before it goes anywhere. +func TestTransportErrorsNeverCarryTheQueryString(t *testing.T) { + const secret = "SECRETKEY0123456789abcdef" + url := "https://ws.example.org/2.0/?method=artist.getSimilar&artist=Creed&api_key=" + secret + + check := func(name string, err error) { + t.Helper() + if err == nil { + t.Fatalf("%s: expected an error", name) + } + msg := err.Error() + if strings.Contains(msg, secret) || strings.Contains(msg, "artist=Creed") || strings.Contains(msg, "api_key") { + t.Errorf("%s: the error repeats the query string: %q", name, msg) + } + if !strings.Contains(msg, "ws.example.org") { + t.Errorf("%s: the host should stay, to debug: %q", name, msg) + } + } + + c := newClient(newClock(), func(o *Options) { o.Transport = failingTransport{}; o.Retries = NoRetries }) + _, err := c.Get(context.Background(), url) + check("failing transport", err) + if !strings.Contains(err.Error(), "boom") { + t.Errorf("the underlying cause was lost: %q", err.Error()) + } + + // A refused connection on a real port. + srv := httptest.NewServer(http.NotFoundHandler()) + dead := srv.URL + srv.Close() + c = newClient(newClock(), func(o *Options) { o.Retries = NoRetries }) + _, err = c.Get(context.Background(), dead+"/x?api_key="+secret) + if err == nil || strings.Contains(err.Error(), secret) { + t.Errorf("refused connection: %v", err) + } + + // A timeout. + slow, _ := server(t, func(w http.ResponseWriter, r *http.Request) { <-r.Context().Done() }) + c = newClient(newClock(), func(o *Options) { o.Timeout = 30 * time.Millisecond; o.Retries = NoRetries }) + _, err = c.Get(context.Background(), slow.URL+"/x?api_key="+secret) + if err == nil || strings.Contains(err.Error(), secret) { + t.Errorf("timeout: %v", err) + } + if !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("the timeout must still be recognisable: %v", err) + } +} + +// A request that gives up while waiting for its turn must give the turn back, +// otherwise a burst of impatient callers makes everybody wait for nobody. +type cancellingClock struct { + *fakeClock + cancel func() +} + +func (c *cancellingClock) Sleep(ctx context.Context, d time.Duration) error { + c.cancel() // the caller gives up during the wait + return ctx.Err() +} + +func TestAbandonedRequestsGiveTheirTurnBack(t *testing.T) { + srv, _ := server(t, ok("{}")) + clock := newClock() + c := newClient(clock, func(o *Options) { o.MinInterval = time.Second }) + if _, err := c.Get(context.Background(), srv.URL); err != nil { // takes the first slot + t.Fatal(err) + } + + // Ten impatient callers, each cancelled while it waits. + for i := 0; i < 10; i++ { + ctx, cancel := context.WithCancel(context.Background()) + c.clock = &cancellingClock{fakeClock: clock, cancel: cancel} + if _, err := c.Get(ctx, srv.URL); !errors.Is(err, context.Canceled) { + t.Fatalf("caller %d: err = %v, want context.Canceled", i, err) + } + } + + // The next real caller waits one interval, not eleven. + c.clock = clock + if _, err := c.Get(context.Background(), srv.URL); err != nil { + t.Fatal(err) + } + got := clock.sleeps() + if len(got) != 1 || got[0] != time.Second { + t.Errorf("sleeps = %v, want a single 1s wait: the abandoned turns were not returned", got) + } +} + +func TestDoesNotQueueWhenTheWaitWouldPassTheDeadline(t *testing.T) { + srv, n := server(t, ok("{}")) + clock := newClock() + c := newClient(clock, func(o *Options) { o.MinInterval = time.Second }) + if _, err := c.Get(context.Background(), srv.URL); err != nil { + t.Fatal(err) + } + before := n.Load() + + // Our turn is a second away but the caller only has 100 ms. + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + if _, err := c.Get(ctx, srv.URL); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("err = %v, want a deadline error without waiting", err) + } + if n.Load() != before || len(clock.sleeps()) != 0 { + t.Errorf("it sent a request or slept although it could not make it in time") + } + + // And it took no turn: the next caller waits one interval, not two. + if _, err := c.Get(context.Background(), srv.URL); err != nil { + t.Fatal(err) + } + if got := clock.sleeps(); len(got) != 1 || got[0] != time.Second { + t.Errorf("sleeps = %v, want [1s]", got) + } +} + +// Requests already in flight when the provider dies all fail together. That is +// one outage, not fifteen: the breaker must open for the first window only. +func TestConcurrentFailuresOpenTheBreakerOnce(t *testing.T) { + var arrived atomic.Int64 + var healthy atomic.Bool + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if healthy.Load() { + _, _ = w.Write([]byte("{}")) + return + } + arrived.Add(1) + <-release + w.WriteHeader(500) + })) + defer srv.Close() + + clock := newClock() + c := newClient(clock, func(o *Options) { + o.Retries, o.BreakerThreshold, o.BreakerOpenFor = NoRetries, 3, 30*time.Second + }) + + const inFlight = 12 + var wg sync.WaitGroup + for i := 0; i < inFlight; i++ { + wg.Add(1) + go func() { + defer wg.Done() + _, _ = c.Get(context.Background(), srv.URL) + }() + } + deadline := time.Now().Add(5 * time.Second) + for arrived.Load() < inFlight && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + } + if arrived.Load() != inFlight { + t.Fatalf("only %d of %d requests were in flight", arrived.Load(), inFlight) + } + close(release) + wg.Wait() + + healthy.Store(true) + clock.advance(31 * time.Second) // just past the first window + if _, err := c.Get(context.Background(), srv.URL); errors.Is(err, ErrCircuitOpen) { + t.Fatal("the window grew with every failure of the same outage: it should be the first 30 s") + } +} + +func TestRedirectsAndOversizedAnswersCountAsFailures(t *testing.T) { + for name, handler := range map[string]http.HandlerFunc{ + "redirect": func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "https://elsewhere.example/", http.StatusFound) + }, + "too big": ok("0123456789"), + } { + t.Run(name, func(t *testing.T) { + srv, n := server(t, handler) + c := newClient(newClock(), func(o *Options) { + o.Retries, o.BreakerThreshold, o.MaxBody = NoRetries, 2, 5 + }) + for i := 0; i < 2; i++ { + if _, err := c.Get(context.Background(), srv.URL); err == nil { + t.Fatalf("attempt %d should fail", i) + } + } + before := n.Load() + if _, err := c.Get(context.Background(), srv.URL); !errors.Is(err, ErrCircuitOpen) { + t.Fatalf("err = %v, want ErrCircuitOpen: a provider that keeps misbehaving this way must stop being asked", err) + } + if n.Load() != before { + t.Error("a request reached the provider while the breaker was open") + } + }) + } +} diff --git a/internal/providers/settings.go b/internal/providers/settings.go index 33edd99..4423b4d 100644 --- a/internal/providers/settings.go +++ b/internal/providers/settings.go @@ -165,7 +165,7 @@ func (s *Service) Update(ctx context.Context, name string, p Patch) (Info, error return Info{}, err } cur.APIKey = key - if key == "" { + if key == "" && m.needsKey { cur.Enabled = false // a provider that needs a key is off without one } } diff --git a/internal/providers/settings_test.go b/internal/providers/settings_test.go index 20cb1fe..f78a20f 100644 --- a/internal/providers/settings_test.go +++ b/internal/providers/settings_test.go @@ -259,3 +259,19 @@ func TestUserAgentIsIdentifiableAndSafe(t *testing.T) { } } } + +// An empty api_key on a provider that has no keys must not switch it off. +func TestAnEmptyKeyDoesNotTurnOffAProviderThatNeedsNone(t *testing.T) { + s, _ := newSvc() + ctx := context.Background() + if _, err := s.Update(ctx, MusicBrainz, Patch{Enabled: boolp(true)}); err != nil { + t.Fatal(err) + } + info, err := s.Update(ctx, MusicBrainz, Patch{Contact: strp("https://example.org/me"), APIKey: strp("")}) + if err != nil || !info.Enabled { + t.Fatalf("%+v err=%v: musicbrainz needs no key, so an empty one must leave it on", info, err) + } + if _, ok, _ := s.Active(ctx, MusicBrainz); !ok { + t.Error("musicbrainz stopped being active") + } +}