From 63fee3227102f4420e64ae585c13a2f485420d33 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Thu, 12 Nov 2026 11:52:20 +0100 Subject: [PATCH 1/2] feat(discover): one Discover at a time for each listener, three at once on the server A cold Discover can make dozens of provider requests, and nothing stopped one listener, or a client that keeps opening and cancelling requests, from lining up work for the providers and slowing everyone else's. - A listener who already has a Discover running gets 429 `discover_busy` with Retry-After, at once. - At most three Discovers run at the same time, whoever asks; the others wait for their turn, and one that stops waiting does no work and keeps no place. - The place and the turn are given back when a Discover fails or ends. - Spec and README describe it; the fakes of the tests are safe for parallel runs. --- README.md | 2 +- internal/api/http/discover_busy_test.go | 35 +++++++ internal/api/http/handlers_discover.go | 4 + internal/api/openapi/openapi.yaml | 13 +++ internal/discover/service.go | 44 ++++++++- internal/discover/service_test.go | 120 ++++++++++++++++++++++++ 6 files changed, 216 insertions(+), 2 deletions(-) create mode 100644 internal/api/http/discover_busy_test.go diff --git a/README.md b/README.md index 44e2696..80d4fd2 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ It is a single executable, one database (SQLite by default, PostgreSQL if you pr - **API documentation.** The full API is described in OpenAPI and browsable with Swagger UI, served by Sonda itself. The UI is embedded in the binary, so opening it contacts no other server. - **Listening history.** Clients report explicit `play`, `complete` and `skip` events. An event counts as a listen when it is `complete`, or when it was played for at least 20 seconds (`listening.min_listen_seconds`); the rule is applied when the history is read, so changing it re-reads the past. You can read your own history, with a `counted` flag on every event. It stays in your database and is never sent anywhere. - **Your taste.** `GET /me/taste` shows what Sonda has learned from your own listening: the artists you like and the kinds of music (tags) they have in common. A complete listen counts for more than a short one, a skip counts against the artist, and older listens weigh less (they halve every 90 days, `taste.half_life_days`; only the last ten half-lives are read). It is worked out on every request from data already in your database; nothing is sent to any provider. -- **Discover.** `GET /discover` suggests artists you may like, in your library or not, each with a score from 0 to 1 and the parts it is made of: how close it is to the artists you like (affinity), how close its tags are to yours (genre), what your listening says about it (preference) and a bonus for what you have not played (discovery). The weights are in `[scoring]` in the config. A part with no data is left out and the rest rescaled, and every answer says which seeds led to each suggestion. It starts from the artists you name with `?seed=`, or, only if you turned on `discover_from_listening`, from the artists you play most. It needs Last.fm to be switched on to ask for anything new: when Last.fm is off and a seed's similar artists are not stored yet, the answer says `disabled` (what was stored earlier is still used). See [External providers](#external-providers). +- **Discover.** `GET /discover` suggests artists you may like, in your library or not, each with a score from 0 to 1 and the parts it is made of: how close it is to the artists you like (affinity), how close its tags are to yours (genre), what your listening says about it (preference) and a bonus for what you have not played (discovery). The weights are in `[scoring]` in the config. A part with no data is left out and the rest rescaled, and every answer says which seeds led to each suggestion. It starts from the artists you name with `?seed=`, or, only if you turned on `discover_from_listening`, from the artists you play most. It needs Last.fm to be switched on to ask for anything new: when Last.fm is off and a seed's similar artists are not stored yet, the answer says `disabled` (what was stored earlier is still used). See [External providers](#external-providers). A listener has one Discover at a time (a second one answers `429` with `Retry-After`), and at most three run at once on the server, so the providers are never lined up with work; one that stops waiting does none. - **Artist descriptions.** `GET /artists/{id}/description` shows what Wikipedia says about an artist: a short text, with the page it comes from, a link to it and the licence (CC BY-SA 4.0), which you must show next to it. With the `wikipedia` provider and the `musicbrainz` one switched on (the link to Wikidata comes from MusicBrainz), Sonda follows ids and never a name: the artist's MusicBrainz id, then the Wikidata item that MusicBrainz links to it, then the Wikipedia page that Wikidata names in the first of your languages (`[wikipedia] languages`, the code of that Wikipedia, for example `["it", "en"]`) that has one. An artist without a MusicBrainz id has no description. It is kept for 30 days and removed with the provider's data. To read the descriptions of the whole library in one go, an admin starts a background job like a scan: `POST /library/descriptions` (follow it with `GET /library/descriptions`, stop it with `DELETE /library/descriptions`; `?refresh=true` reads every artist again). It only goes through artists that have a MusicBrainz id, never replaces a description by a failure, and ends at once, saying so, when a provider it needs is off. See [External providers](#external-providers). - **Artist photos.** `GET /artists/{id}/image` serves a free picture of an artist from Wikimedia Commons, and `GET /artists/{id}/image/credit` says whose it is and under which licence, which you must show next to it. Only free licences are kept (public domain, CC0, CC BY, CC BY-SA): non-commercial, no-derivatives, fair-use and unknown ones are refused, because Sonda stores a copy (`?size=small` is 300 pixels, `large` 1000) in the data folder. With the `commons` provider and the `musicbrainz` one switched on, Sonda follows ids and never a name: the artist's MusicBrainz id, the Wikidata item MusicBrainz links to it, the picture Wikidata names for it, and Commons for its licence. Many artists will have none. Kept for 30 days (a week when there is none) and removed, files included, with the provider's data. To look for the pictures of the whole library in one go, an admin starts a background job like a scan: `POST /library/artist-images` (follow it with `GET /library/artist-images`, stop it with `DELETE /library/artist-images`; `?refresh=true` searches every artist again). It only goes through artists that have a MusicBrainz id, never replaces a picture by a failure, and ends at once, saying so, when a provider it needs is off. See [External providers](#external-providers). - **Album covers.** `GET /albums/{id}/cover` serves the cover of an album from your own files: the picture embedded in them, or else a `cover`, `folder`, `front`, `album` or `albumart` image in the album's folder. It is built the first time it is asked for (a few at a time), scaled (`?size=small` is 300 pixels, `large` 1000) and kept in the data folder (`storage.data_dir`, you can delete it: covers are built again). A picture you replace in a folder is noticed within a week, or at once after a full rescan. When your files have no picture at all and an admin switched on the `coverartarchive` provider, the cover is asked from the Cover Art Archive by the MusicBrainz release group id that the tags of the album's files carry (`MUSICBRAINZ_RELEASEGROUPID`; run a full rescan to read it for music you already indexed); your own files always come first, and only that id leaves the server. For an album split in `CD1`/`Disc 2` folders, the picture in the folder above is used. Nothing is sent anywhere unless you switched on the `coverartarchive` provider (see [External providers](#external-providers)). It also takes `?access_token=`, so an `` element can use it. To build the covers of the whole library in one go instead of one album at a time, an admin starts a background job like a scan: `POST /library/covers` (follow it with `GET /library/covers`, stop it with `DELETE /library/covers`; `?refresh=true` searches every album again; it never replaces a cover by "no cover" because a disk was not mounted, and keeps the covers that came from the archive if you switched it off). diff --git a/internal/api/http/discover_busy_test.go b/internal/api/http/discover_busy_test.go new file mode 100644 index 0000000..a682408 --- /dev/null +++ b/internal/api/http/discover_busy_test.go @@ -0,0 +1,35 @@ +package httpapi + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/uptrace/bun" + + "codeberg.org/squidd/sonda/internal/discover" +) + +type busyDiscover struct{ err error } + +func (b busyDiscover) Discover(context.Context, int64, discover.Query) (discover.Result, error) { + return discover.Result{}, b.err +} + +func TestADiscoverThatIsAlreadyRunningIsTurnedAwayWithAWaitAndAKnownCode(t *testing.T) { + h, _ := newTestEnvDB(t, func(d *Deps, _ *bun.DB) { d.Discover = busyDiscover{err: discover.ErrBusy} }) + admin := registerAndLogin(t, h) + + req := httptest.NewRequest("GET", "/api/v1/discover?seed=1", nil) + req.Header.Set("Authorization", "Bearer "+admin) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != http.StatusTooManyRequests || !strings.Contains(rec.Body.String(), `"discover_busy"`) { + t.Fatalf("%d %s, want 429 discover_busy", rec.Code, rec.Body.String()) + } + if rec.Header().Get("Retry-After") == "" { + t.Error("no Retry-After: the client does not know when to ask again") + } +} diff --git a/internal/api/http/handlers_discover.go b/internal/api/http/handlers_discover.go index a8aace2..559d159 100644 --- a/internal/api/http/handlers_discover.go +++ b/internal/api/http/handlers_discover.go @@ -100,6 +100,10 @@ func (h *discoverHandlers) discover(c *gin.Context) { case errors.Is(err, discover.ErrInvalidQuery): writeError(c, http.StatusBadRequest, "invalid_input", err.Error()) return + case errors.Is(err, discover.ErrBusy): + c.Header("Retry-After", "2") + writeError(c, http.StatusTooManyRequests, "discover_busy", err.Error()) + return case errors.Is(err, context.Canceled): c.AbortWithStatus(statusClientClosed) // the client left; nobody is waiting return diff --git a/internal/api/openapi/openapi.yaml b/internal/api/openapi/openapi.yaml index 88737c5..6b05676 100644 --- a/internal/api/openapi/openapi.yaml +++ b/internal/api/openapi/openapi.yaml @@ -930,6 +930,10 @@ paths: is still used, so the list can be empty. With every seed's data stored and fresh the status is `ok` whether Last.fm is on or not. It is `error` when Last.fm failed for some seed (the rest is still returned) and `ok` otherwise. Show `attribution` where you show the data. + + **How many at once.** One Discover at a time for each listener (a second one answers 429 + `discover_busy`), and at most three at the same time on the server: a request that finds them + busy waits for its turn, and leaves without doing any work if the client gives up. security: [{bearerAuth: []}] parameters: - name: seed @@ -958,6 +962,15 @@ paths: application/json: schema: {$ref: '#/components/schemas/Error'} '401': {$ref: '#/components/responses/Unauthorized'} + '429': + description: "You already have a Discover running (`discover_busy`): one at a time for each listener. Ask again after `Retry-After` seconds." + headers: + Retry-After: + description: Seconds to wait before asking again. + schema: {type: integer} + content: + application/json: + schema: {$ref: '#/components/schemas/Error'} /discover/dismissed: get: diff --git a/internal/discover/service.go b/internal/discover/service.go index 2efedeb..5b34d9a 100644 --- a/internal/discover/service.go +++ b/internal/discover/service.go @@ -30,6 +30,9 @@ const ( // minPool is how many candidates, best affinity first, are scored whatever the // limit: the genre can reorder them, and each one without tags costs a request. minPool = 20 + // MaxConcurrent is how many Discovers run at the same time, whoever asks. A request that finds them + // all busy waits for its turn; one that stops waiting does no work. + MaxConcurrent = 3 // MaxBecause is how many seeds are named as the reason for a suggestion. MaxBecause = 3 ) @@ -40,6 +43,9 @@ var ( // ErrSeedRequired is returned when no seed was named and the listener did not // allow Discover to start from their listening. ErrSeedRequired = errors.New("name a seed artist, or allow Discover to start from your listening") + // ErrBusy is returned when the same listener already has a Discover running: one at a time, so that + // a client that keeps asking cannot line up work for the providers. + ErrBusy = errors.New("a Discover of yours is already running") ) // Taster says what the listener likes. @@ -125,11 +131,42 @@ type Service struct { settings Settings marks Marks weights Weights + + mu sync.Mutex + running map[int64]bool // the listeners who have a Discover running or waiting for its turn + turns chan struct{} // MaxConcurrent places } // NewService builds a Service. func NewService(t Taster, sim Similar, tags TagFetcher, store Store, settings Settings, marks Marks, w Weights) *Service { - return &Service{taste: t, similar: sim, tags: tags, store: store, settings: settings, marks: marks, weights: w} + return &Service{ + taste: t, similar: sim, tags: tags, store: store, settings: settings, marks: marks, weights: w, + running: map[int64]bool{}, turns: make(chan struct{}, MaxConcurrent), + } +} + +// enter takes the place of the listener and a turn, and returns what gives them back. A listener who +// has a Discover already is refused at once; one who leaves while waiting for a turn gives up the place. +func (s *Service) enter(ctx context.Context, userID int64) (func(), error) { + s.mu.Lock() + if s.running[userID] { + s.mu.Unlock() + return nil, ErrBusy + } + s.running[userID] = true + s.mu.Unlock() + free := func() { + s.mu.Lock() + delete(s.running, userID) + s.mu.Unlock() + } + select { + case s.turns <- struct{}{}: + return func() { <-s.turns; free() }, nil + case <-ctx.Done(): + free() + return nil, ctx.Err() + } } // Discover suggests artists. Seeds are the artists the listener names, or, when none @@ -142,6 +179,11 @@ func (s *Service) Discover(ctx context.Context, userID int64, q Query) (Result, if err != nil { return Result{}, err } + leave, err := s.enter(ctx, userID) + if err != nil { + return Result{}, err + } + defer leave() named, err := s.checkSeeds(ctx, q.Seeds) if err != nil { return Result{}, err diff --git a/internal/discover/service_test.go b/internal/discover/service_test.go index f36324f..5d4117c 100644 --- a/internal/discover/service_test.go +++ b/internal/discover/service_test.go @@ -16,6 +16,7 @@ import ( ) type fakeTaste struct { + mu sync.Mutex // Discovers of several listeners run together learned taste.Learned err error user int64 @@ -23,6 +24,8 @@ type fakeTaste struct { } func (f *fakeTaste) Learn(_ context.Context, userID int64) (taste.Learned, error) { + f.mu.Lock() + defer f.mu.Unlock() f.calls++ f.user = userID return f.learned, f.err @@ -148,12 +151,15 @@ func (f fakeSettings) Get(context.Context, int64) (usersettings.Settings, error) } type fakeMarks struct { + mu sync.Mutex // Discovers of several listeners run together marks map[int64]feedback.Kind err error user int64 } func (f *fakeMarks) Marks(_ context.Context, userID int64) (map[int64]feedback.Kind, error) { + f.mu.Lock() + defer f.mu.Unlock() f.user = userID return f.marks, f.err } @@ -762,3 +768,117 @@ func TestMarksErrorsPass(t *testing.T) { t.Errorf("err = %v", err) } } + +// ---- how many Discovers run ---- + +// waitCalled waits until the providers were asked n times: that many Discovers are running. +func waitCalled(t *testing.T, f *fakeSimilar, n int) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for len(f.called()) < n { + if time.Now().After(deadline) { + t.Fatalf("the providers were asked %d times, want %d", len(f.called()), n) + } + time.Sleep(2 * time.Millisecond) + } +} + +func TestAListenerHasOneDiscoverAtATime(t *testing.T) { + r := newRig() + r.similar.delay = 150 * time.Millisecond + first := make(chan error, 1) + go func() { _, err := r.svc.Discover(context.Background(), 7, Query{Seeds: []int64{1}}); first <- err }() + waitCalled(t, r.similar, 1) + + start := time.Now() + if _, err := r.svc.Discover(context.Background(), 7, Query{Seeds: []int64{1}}); !errors.Is(err, ErrBusy) { + t.Errorf("a second one of the same listener: err = %v, want ErrBusy", err) + } + if time.Since(start) > 50*time.Millisecond { + t.Errorf("a refused one waited %v", time.Since(start)) + } + if _, err := r.svc.Discover(context.Background(), 8, Query{Seeds: []int64{1}}); err != nil { + t.Errorf("another listener is not held up: %v", err) + } + if err := <-first; err != nil { + t.Fatal(err) + } + if _, err := r.svc.Discover(context.Background(), 7, Query{Seeds: []int64{1}}); err != nil { + t.Errorf("after the first ended: %v", err) + } +} + +func TestOnlyAFewDiscoversRunAtTheSameTimeAndTheRestWait(t *testing.T) { + r := newRig() + r.similar.delay = 40 * time.Millisecond + var wg sync.WaitGroup + for user := int64(1); user <= 9; user++ { + wg.Add(1) + go func() { + defer wg.Done() + if _, err := r.svc.Discover(context.Background(), user, Query{Seeds: []int64{1}}); err != nil { + t.Errorf("listener %d: %v", user, err) + } + }() + } + wg.Wait() + // One seed means one provider request for each Discover, so this is how many ran together. + if m := atomic.LoadInt32(&r.similar.maxSeen); m > MaxConcurrent || m < 2 { + t.Errorf("%d Discovers ran together, want at most %d (and more than one)", m, MaxConcurrent) + } + if n := len(r.similar.called()); n != 9 { + t.Errorf("the providers were asked %d times, want 9: all of them were done", n) + } +} + +func TestAListenerWhoLeavesWhileWaitingForATurnDoesNoWorkAndKeepsNoPlace(t *testing.T) { + r := newRig() + r.similar.delay = 250 * time.Millisecond + var wg sync.WaitGroup + for user := int64(1); user <= MaxConcurrent; user++ { + wg.Add(1) + go func() { + defer wg.Done() + _, _ = r.svc.Discover(context.Background(), user, Query{Seeds: []int64{1}}) + }() + } + waitCalled(t, r.similar, MaxConcurrent) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) + defer cancel() + start := time.Now() + if _, err := r.svc.Discover(ctx, 50, Query{Seeds: []int64{1}}); !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("err = %v, want the deadline of the request", err) + } + if time.Since(start) > 150*time.Millisecond { + t.Errorf("it waited %v for a turn it gave up", time.Since(start)) + } + wg.Wait() + if n := len(r.similar.called()); n != MaxConcurrent { + t.Errorf("the providers were asked %d times: the one that left did work", n) + } + // Neither a turn nor the place of that listener was kept. + r.similar.delay = 0 + if _, err := r.svc.Discover(context.Background(), 50, Query{Seeds: []int64{1}}); err != nil { + t.Errorf("the listener who left cannot come back: %v", err) + } +} + +func TestADiscoverThatFailsFreesTheListenerAndTheTurn(t *testing.T) { + r := newRig() + for i := 0; i < 3*MaxConcurrent; i++ { + if _, err := r.svc.Discover(context.Background(), 7, Query{Seeds: []int64{99999}}); !errors.Is(err, ErrInvalidQuery) { + t.Fatalf("round %d: err = %v, want ErrInvalidQuery and never ErrBusy", i, err) + } + } + r.taste.err = errors.New("boom") + for i := 0; i < 3*MaxConcurrent; i++ { + if _, err := r.svc.Discover(context.Background(), int64(i), Query{Seeds: []int64{1}}); errors.Is(err, ErrBusy) { + t.Fatalf("round %d: busy after a failure", i) + } + } + r.taste.err = nil + if _, err := r.svc.Discover(context.Background(), 7, Query{Seeds: []int64{1}}); err != nil { + t.Errorf("after the failures: %v", err) + } +} From 69da67065df973e0225017a6f9d311c280b5752f Mon Sep 17 00:00:00 2001 From: tgiachi Date: Thu, 12 Nov 2026 12:00:18 +0100 Subject: [PATCH 2/2] fix(discover): refusals do not wait for a turn, and waiting for one has an end - The gate now comes after what only the database decides (a seed that is not an artist, no seed and no permission), so those answers are not held up by three slow Discovers and do not make the listener "busy". - A request that waits more than 30 seconds for a turn answers 429 instead of hanging for as long as the providers are slow. - The spec and the README say that a waiting request counts as running. --- README.md | 2 +- internal/api/openapi/openapi.yaml | 5 +-- internal/discover/service.go | 33 ++++++++++++------ internal/discover/service_test.go | 58 +++++++++++++++++++++++++++++++ 4 files changed, 85 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 80d4fd2..33f3929 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ It is a single executable, one database (SQLite by default, PostgreSQL if you pr - **API documentation.** The full API is described in OpenAPI and browsable with Swagger UI, served by Sonda itself. The UI is embedded in the binary, so opening it contacts no other server. - **Listening history.** Clients report explicit `play`, `complete` and `skip` events. An event counts as a listen when it is `complete`, or when it was played for at least 20 seconds (`listening.min_listen_seconds`); the rule is applied when the history is read, so changing it re-reads the past. You can read your own history, with a `counted` flag on every event. It stays in your database and is never sent anywhere. - **Your taste.** `GET /me/taste` shows what Sonda has learned from your own listening: the artists you like and the kinds of music (tags) they have in common. A complete listen counts for more than a short one, a skip counts against the artist, and older listens weigh less (they halve every 90 days, `taste.half_life_days`; only the last ten half-lives are read). It is worked out on every request from data already in your database; nothing is sent to any provider. -- **Discover.** `GET /discover` suggests artists you may like, in your library or not, each with a score from 0 to 1 and the parts it is made of: how close it is to the artists you like (affinity), how close its tags are to yours (genre), what your listening says about it (preference) and a bonus for what you have not played (discovery). The weights are in `[scoring]` in the config. A part with no data is left out and the rest rescaled, and every answer says which seeds led to each suggestion. It starts from the artists you name with `?seed=`, or, only if you turned on `discover_from_listening`, from the artists you play most. It needs Last.fm to be switched on to ask for anything new: when Last.fm is off and a seed's similar artists are not stored yet, the answer says `disabled` (what was stored earlier is still used). See [External providers](#external-providers). A listener has one Discover at a time (a second one answers `429` with `Retry-After`), and at most three run at once on the server, so the providers are never lined up with work; one that stops waiting does none. +- **Discover.** `GET /discover` suggests artists you may like, in your library or not, each with a score from 0 to 1 and the parts it is made of: how close it is to the artists you like (affinity), how close its tags are to yours (genre), what your listening says about it (preference) and a bonus for what you have not played (discovery). The weights are in `[scoring]` in the config. A part with no data is left out and the rest rescaled, and every answer says which seeds led to each suggestion. It starts from the artists you name with `?seed=`, or, only if you turned on `discover_from_listening`, from the artists you play most. It needs Last.fm to be switched on to ask for anything new: when Last.fm is off and a seed's similar artists are not stored yet, the answer says `disabled` (what was stored earlier is still used). See [External providers](#external-providers). A listener has one Discover at a time (a second one answers `429` with `Retry-After`), and at most three run at once on the server, so the providers are never lined up with work: one that waits more than 30 seconds for its turn answers `429` too, and one that stops waiting does none. - **Artist descriptions.** `GET /artists/{id}/description` shows what Wikipedia says about an artist: a short text, with the page it comes from, a link to it and the licence (CC BY-SA 4.0), which you must show next to it. With the `wikipedia` provider and the `musicbrainz` one switched on (the link to Wikidata comes from MusicBrainz), Sonda follows ids and never a name: the artist's MusicBrainz id, then the Wikidata item that MusicBrainz links to it, then the Wikipedia page that Wikidata names in the first of your languages (`[wikipedia] languages`, the code of that Wikipedia, for example `["it", "en"]`) that has one. An artist without a MusicBrainz id has no description. It is kept for 30 days and removed with the provider's data. To read the descriptions of the whole library in one go, an admin starts a background job like a scan: `POST /library/descriptions` (follow it with `GET /library/descriptions`, stop it with `DELETE /library/descriptions`; `?refresh=true` reads every artist again). It only goes through artists that have a MusicBrainz id, never replaces a description by a failure, and ends at once, saying so, when a provider it needs is off. See [External providers](#external-providers). - **Artist photos.** `GET /artists/{id}/image` serves a free picture of an artist from Wikimedia Commons, and `GET /artists/{id}/image/credit` says whose it is and under which licence, which you must show next to it. Only free licences are kept (public domain, CC0, CC BY, CC BY-SA): non-commercial, no-derivatives, fair-use and unknown ones are refused, because Sonda stores a copy (`?size=small` is 300 pixels, `large` 1000) in the data folder. With the `commons` provider and the `musicbrainz` one switched on, Sonda follows ids and never a name: the artist's MusicBrainz id, the Wikidata item MusicBrainz links to it, the picture Wikidata names for it, and Commons for its licence. Many artists will have none. Kept for 30 days (a week when there is none) and removed, files included, with the provider's data. To look for the pictures of the whole library in one go, an admin starts a background job like a scan: `POST /library/artist-images` (follow it with `GET /library/artist-images`, stop it with `DELETE /library/artist-images`; `?refresh=true` searches every artist again). It only goes through artists that have a MusicBrainz id, never replaces a picture by a failure, and ends at once, saying so, when a provider it needs is off. See [External providers](#external-providers). - **Album covers.** `GET /albums/{id}/cover` serves the cover of an album from your own files: the picture embedded in them, or else a `cover`, `folder`, `front`, `album` or `albumart` image in the album's folder. It is built the first time it is asked for (a few at a time), scaled (`?size=small` is 300 pixels, `large` 1000) and kept in the data folder (`storage.data_dir`, you can delete it: covers are built again). A picture you replace in a folder is noticed within a week, or at once after a full rescan. When your files have no picture at all and an admin switched on the `coverartarchive` provider, the cover is asked from the Cover Art Archive by the MusicBrainz release group id that the tags of the album's files carry (`MUSICBRAINZ_RELEASEGROUPID`; run a full rescan to read it for music you already indexed); your own files always come first, and only that id leaves the server. For an album split in `CD1`/`Disc 2` folders, the picture in the folder above is used. Nothing is sent anywhere unless you switched on the `coverartarchive` provider (see [External providers](#external-providers)). It also takes `?access_token=`, so an `` element can use it. To build the covers of the whole library in one go instead of one album at a time, an admin starts a background job like a scan: `POST /library/covers` (follow it with `GET /library/covers`, stop it with `DELETE /library/covers`; `?refresh=true` searches every album again; it never replaces a cover by "no cover" because a disk was not mounted, and keeps the covers that came from the archive if you switched it off). diff --git a/internal/api/openapi/openapi.yaml b/internal/api/openapi/openapi.yaml index 6b05676..588587c 100644 --- a/internal/api/openapi/openapi.yaml +++ b/internal/api/openapi/openapi.yaml @@ -933,7 +933,8 @@ paths: **How many at once.** One Discover at a time for each listener (a second one answers 429 `discover_busy`), and at most three at the same time on the server: a request that finds them - busy waits for its turn, and leaves without doing any work if the client gives up. + busy waits for its turn (at most 30 seconds, then it answers 429 as well), and leaves without + doing any work if the client gives up. A request that waits for its turn counts as running. security: [{bearerAuth: []}] parameters: - name: seed @@ -963,7 +964,7 @@ paths: schema: {$ref: '#/components/schemas/Error'} '401': {$ref: '#/components/responses/Unauthorized'} '429': - description: "You already have a Discover running (`discover_busy`): one at a time for each listener. Ask again after `Retry-After` seconds." + description: "You already have a Discover running or waiting for its turn, or the server was too busy to give it one in 30 seconds (`discover_busy`). Ask again after `Retry-After` seconds." headers: Retry-After: description: Seconds to wait before asking again. diff --git a/internal/discover/service.go b/internal/discover/service.go index 5b34d9a..4db52be 100644 --- a/internal/discover/service.go +++ b/internal/discover/service.go @@ -6,6 +6,7 @@ import ( "fmt" "sort" "sync" + "time" "codeberg.org/squidd/sonda/internal/discovery" "codeberg.org/squidd/sonda/internal/feedback" @@ -33,6 +34,9 @@ const ( // MaxConcurrent is how many Discovers run at the same time, whoever asks. A request that finds them // all busy waits for its turn; one that stops waiting does no work. MaxConcurrent = 3 + // TurnWait is the longest a request waits for a turn: after that it is told it is busy, so that + // slow providers cannot hold a queue of requests open for as long as they like. + TurnWait = 30 * time.Second // MaxBecause is how many seeds are named as the reason for a suggestion. MaxBecause = 3 ) @@ -132,21 +136,23 @@ type Service struct { marks Marks weights Weights - mu sync.Mutex - running map[int64]bool // the listeners who have a Discover running or waiting for its turn - turns chan struct{} // MaxConcurrent places + mu sync.Mutex + running map[int64]bool // the listeners who have a Discover running or waiting for its turn + turns chan struct{} // MaxConcurrent places + turnWait time.Duration // how long a request waits for a turn before it is told it is busy } // NewService builds a Service. func NewService(t Taster, sim Similar, tags TagFetcher, store Store, settings Settings, marks Marks, w Weights) *Service { return &Service{ taste: t, similar: sim, tags: tags, store: store, settings: settings, marks: marks, weights: w, - running: map[int64]bool{}, turns: make(chan struct{}, MaxConcurrent), + running: map[int64]bool{}, turns: make(chan struct{}, MaxConcurrent), turnWait: TurnWait, } } // enter takes the place of the listener and a turn, and returns what gives them back. A listener who -// has a Discover already is refused at once; one who leaves while waiting for a turn gives up the place. +// has a Discover already is refused at once; one who leaves while waiting for a turn, or waits for +// TurnWait without getting one, gives up the place. func (s *Service) enter(ctx context.Context, userID int64) (func(), error) { s.mu.Lock() if s.running[userID] { @@ -160,12 +166,17 @@ func (s *Service) enter(ctx context.Context, userID int64) (func(), error) { delete(s.running, userID) s.mu.Unlock() } + wait := time.NewTimer(s.turnWait) + defer wait.Stop() select { case s.turns <- struct{}{}: return func() { <-s.turns; free() }, nil case <-ctx.Done(): free() return nil, ctx.Err() + case <-wait.C: + free() + return nil, ErrBusy } } @@ -179,11 +190,6 @@ func (s *Service) Discover(ctx context.Context, userID int64, q Query) (Result, if err != nil { return Result{}, err } - leave, err := s.enter(ctx, userID) - if err != nil { - return Result{}, err - } - defer leave() named, err := s.checkSeeds(ctx, q.Seeds) if err != nil { return Result{}, err @@ -203,6 +209,13 @@ func (s *Service) Discover(ctx context.Context, userID int64, q Query) (Result, res.FromListening = true } + // What is refused above costs the database and nothing else, so it does not wait for a turn. From + // here on the providers may be asked. + leave, err := s.enter(ctx, userID) + if err != nil { + return Result{}, err + } + defer leave() marks, err := s.marks.Marks(ctx, userID) if err != nil { return Result{}, fmt.Errorf("read marks: %w", err) diff --git a/internal/discover/service_test.go b/internal/discover/service_test.go index 5d4117c..9e946c1 100644 --- a/internal/discover/service_test.go +++ b/internal/discover/service_test.go @@ -882,3 +882,61 @@ func TestADiscoverThatFailsFreesTheListenerAndTheTurn(t *testing.T) { t.Errorf("after the failures: %v", err) } } + +func TestARequestThatIsRefusedForWhatItAsksDoesNotWaitForATurn(t *testing.T) { + r := newRig() + r.similar.delay = 300 * time.Millisecond + var wg sync.WaitGroup + for user := int64(1); user <= MaxConcurrent; user++ { + wg.Add(1) + go func() { + defer wg.Done() + _, _ = r.svc.Discover(context.Background(), user, Query{Seeds: []int64{1}}) + }() + } + waitCalled(t, r.similar, MaxConcurrent) + + start := time.Now() + if _, err := r.svc.Discover(context.Background(), 50, Query{Seeds: []int64{99999}}); !errors.Is(err, ErrInvalidQuery) { + t.Errorf("a seed that is not an artist: %v", err) + } + if _, err := r.svc.Discover(context.Background(), 50, Query{}); !errors.Is(err, ErrSeedRequired) { // listening not allowed + t.Errorf("no seed and no permission: %v", err) + } + if time.Since(start) > 100*time.Millisecond { + t.Errorf("refusals took %v: they waited for a turn that they do not need", time.Since(start)) + } + // And that listener is not "busy" because of them. + wg.Wait() + r.similar.delay = 0 + if _, err := r.svc.Discover(context.Background(), 50, Query{Seeds: []int64{1}}); err != nil { + t.Errorf("the listener was held back by its own refusals: %v", err) + } +} + +func TestWaitingForATurnHasAnEndAndTheAnswerIsBusy(t *testing.T) { + r := newRig() + r.svc.turnWait = 40 * time.Millisecond + r.similar.delay = 300 * time.Millisecond + var wg sync.WaitGroup + for user := int64(1); user <= MaxConcurrent; user++ { + wg.Add(1) + go func() { + defer wg.Done() + _, _ = r.svc.Discover(context.Background(), user, Query{Seeds: []int64{1}}) + }() + } + waitCalled(t, r.similar, MaxConcurrent) + start := time.Now() + if _, err := r.svc.Discover(context.Background(), 50, Query{Seeds: []int64{1}}); !errors.Is(err, ErrBusy) { + t.Errorf("err = %v, want ErrBusy after the wait", err) + } + if el := time.Since(start); el < 30*time.Millisecond || el > 200*time.Millisecond { + t.Errorf("it waited %v, want about the time allowed", el) + } + wg.Wait() + r.similar.delay = 0 + if _, err := r.svc.Discover(context.Background(), 50, Query{Seeds: []int64{1}}); err != nil { + t.Errorf("no place was kept by the one who gave up: %v", err) + } +}