Merge pull request 'feat(jobs): background jobs for artist photos and descriptions' (#48) from feat/artist-jobs into main
This commit is contained in:
commit
bea1158e68
27 changed files with 2341 additions and 730 deletions
10
README.md
10
README.md
|
|
@ -17,8 +17,8 @@ It is a single executable, one database (SQLite by default, PostgreSQL if you pr
|
|||
- **Listening history.** Clients report explicit `play`, `complete` and `skip` events. An event counts as a listen when it is `complete`, or when it was played for at least 20 seconds (`listening.min_listen_seconds`); the rule is applied when the history is read, so changing it re-reads the past. You can read your own history, with a `counted` flag on every event. It stays in your database and is never sent anywhere.
|
||||
- **Your taste.** `GET /me/taste` shows what Sonda has learned from your own listening: the artists you like and the kinds of music (tags) they have in common. A complete listen counts for more than a short one, a skip counts against the artist, and older listens weigh less (they halve every 90 days, `taste.half_life_days`; only the last ten half-lives are read). It is worked out on every request from data already in your database; nothing is sent to any provider.
|
||||
- **Discover.** `GET /discover` suggests artists you may like, in your library or not, each with a score from 0 to 1 and the parts it is made of: how close it is to the artists you like (affinity), how close its tags are to yours (genre), what your listening says about it (preference) and a bonus for what you have not played (discovery). The weights are in `[scoring]` in the config. A part with no data is left out and the rest rescaled, and every answer says which seeds led to each suggestion. It starts from the artists you name with `?seed=`, or, only if you turned on `discover_from_listening`, from the artists you play most. It needs Last.fm to be switched on to ask for anything new: when Last.fm is off and a seed's similar artists are not stored yet, the answer says `disabled` (what was stored earlier is still used). See [External providers](#external-providers).
|
||||
- **Artist descriptions.** `GET /artists/{id}/description` shows what Wikipedia says about an artist: a short text, with the page it comes from, a link to it and the licence (CC BY-SA 4.0), which you must show next to it. With the `wikipedia` provider and the `musicbrainz` one switched on (the link to Wikidata comes from MusicBrainz), Sonda follows ids and never a name: the artist's MusicBrainz id, then the Wikidata item that MusicBrainz links to it, then the Wikipedia page that Wikidata names in the first of your languages (`[wikipedia] languages`, the code of that Wikipedia, for example `["it", "en"]`) that has one. An artist without a MusicBrainz id has no description. It is kept for 30 days and removed with the provider's data. See [External providers](#external-providers).
|
||||
- **Artist photos.** `GET /artists/{id}/image` serves a free picture of an artist from Wikimedia Commons, and `GET /artists/{id}/image/credit` says whose it is and under which licence, which you must show next to it. Only free licences are kept (public domain, CC0, CC BY, CC BY-SA): non-commercial, no-derivatives, fair-use and unknown ones are refused, because Sonda stores a copy (`?size=small` is 300 pixels, `large` 1000) in the data folder. With the `commons` provider and the `musicbrainz` one switched on, Sonda follows ids and never a name: the artist's MusicBrainz id, the Wikidata item MusicBrainz links to it, the picture Wikidata names for it, and Commons for its licence. Many artists will have none. Kept for 30 days (a week when there is none) and removed, files included, with the provider's data. See [External providers](#external-providers).
|
||||
- **Artist descriptions.** `GET /artists/{id}/description` shows what Wikipedia says about an artist: a short text, with the page it comes from, a link to it and the licence (CC BY-SA 4.0), which you must show next to it. With the `wikipedia` provider and the `musicbrainz` one switched on (the link to Wikidata comes from MusicBrainz), Sonda follows ids and never a name: the artist's MusicBrainz id, then the Wikidata item that MusicBrainz links to it, then the Wikipedia page that Wikidata names in the first of your languages (`[wikipedia] languages`, the code of that Wikipedia, for example `["it", "en"]`) that has one. An artist without a MusicBrainz id has no description. It is kept for 30 days and removed with the provider's data. To read the descriptions of the whole library in one go, an admin starts a background job like a scan: `POST /library/descriptions` (follow it with `GET /library/descriptions`, stop it with `DELETE /library/descriptions`; `?refresh=true` reads every artist again). It only goes through artists that have a MusicBrainz id, never replaces a description by a failure, and ends at once, saying so, when a provider it needs is off. See [External providers](#external-providers).
|
||||
- **Artist photos.** `GET /artists/{id}/image` serves a free picture of an artist from Wikimedia Commons, and `GET /artists/{id}/image/credit` says whose it is and under which licence, which you must show next to it. Only free licences are kept (public domain, CC0, CC BY, CC BY-SA): non-commercial, no-derivatives, fair-use and unknown ones are refused, because Sonda stores a copy (`?size=small` is 300 pixels, `large` 1000) in the data folder. With the `commons` provider and the `musicbrainz` one switched on, Sonda follows ids and never a name: the artist's MusicBrainz id, the Wikidata item MusicBrainz links to it, the picture Wikidata names for it, and Commons for its licence. Many artists will have none. Kept for 30 days (a week when there is none) and removed, files included, with the provider's data. To look for the pictures of the whole library in one go, an admin starts a background job like a scan: `POST /library/artist-images` (follow it with `GET /library/artist-images`, stop it with `DELETE /library/artist-images`; `?refresh=true` searches every artist again). It only goes through artists that have a MusicBrainz id, never replaces a picture by a failure, and ends at once, saying so, when a provider it needs is off. See [External providers](#external-providers).
|
||||
- **Album covers.** `GET /albums/{id}/cover` serves the cover of an album from your own files: the picture embedded in them, or else a `cover`, `folder`, `front`, `album` or `albumart` image in the album's folder. It is built the first time it is asked for (a few at a time), scaled (`?size=small` is 300 pixels, `large` 1000) and kept in the data folder (`storage.data_dir`, you can delete it: covers are built again). A picture you replace in a folder is noticed within a week, or at once after a full rescan. When your files have no picture at all and an admin switched on the `coverartarchive` provider, the cover is asked from the Cover Art Archive by the MusicBrainz release group id that the tags of the album's files carry (`MUSICBRAINZ_RELEASEGROUPID`; run a full rescan to read it for music you already indexed); your own files always come first, and only that id leaves the server. For an album split in `CD1`/`Disc 2` folders, the picture in the folder above is used. Nothing is sent anywhere unless you switched on the `coverartarchive` provider (see [External providers](#external-providers)). It also takes `?access_token=`, so an `<img>` element can use it. To build the covers of the whole library in one go instead of one album at a time, an admin starts a background job like a scan: `POST /library/covers` (follow it with `GET /library/covers`, stop it with `DELETE /library/covers`; `?refresh=true` searches every album again; it never replaces a cover by "no cover" because a disk was not mounted, and keeps the covers that came from the archive if you switched it off).
|
||||
- **Wishlist and "not interested".** `PUT /wishlist/{artist}` puts an artist on your wishlist (`GET /wishlist` lists it, `DELETE` takes it off), and `POST /discover/{artist}/dismiss` hides an artist from your Discover for good (`DELETE` brings it back, `GET /discover/dismissed` lists them). An artist is wished or dismissed, never both, and both lists are yours alone. Discover marks the wished artists it shows (`wishlisted`) and never suggests a dismissed one. Nothing is downloaded or sent anywhere: the wishlist is a list for you, and for a download client you may set up later. An artist that only Last.fm named is kept while you have it marked (your lists still show it, but other users cannot see that it exists), and goes with the rest of Last.fm's data if an admin removes Last.fm's data. Removing MusicBrainz's data does not touch it.
|
||||
- **Your own settings.** `GET` and `PATCH /me/settings` hold choices that belong to you, not to the server. Everyone starts private: `discover_from_listening` is off, so this setting never sends your listening to a provider. When you turn it on, Discover may start from the artists you listen to most, and their names are sent to Last.fm (only if an admin switched it on). What Last.fm answers is kept in the shared database like any other lookup, so another local user can see, from `retrieved_at` of `GET /artists/{id}/similar`, that someone looked up an artist, but not who.
|
||||
|
|
@ -205,7 +205,7 @@ curl -s -X PUT $BASE/providers/musicbrainz -H "$AUTH" -d '{"enabled":true,"conta
|
|||
|
||||
The key is stored in the database, in clear text, and is never returned by the API or written to the log: the list only says `has_api_key`. Because the database holds it, treat database backups like secrets. Removing the key turns the provider off, and each provider describes in the list exactly what it sends.
|
||||
|
||||
**What leaves your server, when a provider is on:** to Last.fm, the artist name (or MusicBrainz id) being looked up, your API key, your IP address and the `User-Agent`; to MusicBrainz, only the MusicBrainz id of the artist, your IP address and a `User-Agent` that carries your contact. Never your users, your listens, track titles or file paths. Requests only happen when someone asks for them, never as a bulk pass over your library (the one exception is the covers job, which an admin starts on purpose with `POST /library/covers`: with the Cover Art Archive on, it sends the release group id of every album that has no picture in its files, one after the other, spaced out, and it stops when the archive keeps failing), and they are spaced out, retried with backoff, and stopped by a circuit breaker if a provider is down. **Descriptions** (the Wikipedia provider, which also needs MusicBrainz on) send the MusicBrainz id to MusicBrainz, the Wikidata item to Wikidata and the title of the page to Wikipedia, with your IP address and a `User-Agent` that carries a contact (the one set for each provider; Wikimedia refuses requests without one), and never a name. **Photos** (the Commons provider, which also needs MusicBrainz on) send the MusicBrainz id to MusicBrainz, the Wikidata item to Wikidata, the name of the picture to Commons and the address of a copy to Wikimedia, with your IP address and a `User-Agent` with a contact, and never a name. **Covers** (the Cover Art Archive provider) send only the MusicBrainz release group id of an album, read from the tags of its own files and never a title, and only when the files have no picture; the redirects of the archive are followed only to archive.org, over https. Its covers have no licence of their own: Sonda keeps a copy for your library and never treats it as free. **Discover** asks Last.fm for the similar artists of each seed (the seed's name or MusicBrainz id) and, for the best suggestions that have no tags yet, for the tags of that suggestion (its name); never MusicBrainz. Seeds you name are sent when you ask. Seeds taken from your listening, which are the names of the artists you play most, are sent only if you turned on `discover_from_listening` yourself; otherwise Discover refuses to run without a `seed`. Besides the covers job, the other exception to "only when someone asks" is `POST /providers/{name}/test`, which an admin sends to check a key; it sends nothing but the key.
|
||||
**What leaves your server, when a provider is on:** to Last.fm, the artist name (or MusicBrainz id) being looked up, your API key, your IP address and the `User-Agent`; to MusicBrainz, only the MusicBrainz id of the artist, your IP address and a `User-Agent` that carries your contact. Never your users, your listens, track titles or file paths. Requests only happen when someone asks for them, never as a bulk pass over your library (the exceptions are the three jobs that an admin starts on purpose, each of them one request that goes through the whole library, one item after the other, spaced out, and each stops when its providers keep failing: `POST /library/covers`, with the Cover Art Archive on, sends the release group id of every album that has no picture in its files; `POST /library/artist-images`, with Commons and MusicBrainz on, sends the MusicBrainz id of every artist that has one, to MusicBrainz, Wikidata and Commons; `POST /library/descriptions`, with Wikipedia and MusicBrainz on, sends it to MusicBrainz, Wikidata and Wikipedia), and they are spaced out, retried with backoff, and stopped by a circuit breaker if a provider is down. **Descriptions** (the Wikipedia provider, which also needs MusicBrainz on) send the MusicBrainz id to MusicBrainz, the Wikidata item to Wikidata and the title of the page to Wikipedia, with your IP address and a `User-Agent` that carries a contact (the one set for each provider; Wikimedia refuses requests without one), and never a name. **Photos** (the Commons provider, which also needs MusicBrainz on) send the MusicBrainz id to MusicBrainz, the Wikidata item to Wikidata, the name of the picture to Commons and the address of a copy to Wikimedia, with your IP address and a `User-Agent` with a contact, and never a name. **Covers** (the Cover Art Archive provider) send only the MusicBrainz release group id of an album, read from the tags of its own files and never a title, and only when the files have no picture; the redirects of the archive are followed only to archive.org, over https. Its covers have no licence of their own: Sonda keeps a copy for your library and never treats it as free. **Discover** asks Last.fm for the similar artists of each seed (the seed's name or MusicBrainz id) and, for the best suggestions that have no tags yet, for the tags of that suggestion (its name); never MusicBrainz. Seeds you name are sent when you ask. Seeds taken from your listening, which are the names of the artists you play most, are sent only if you turned on `discover_from_listening` yourself; otherwise Discover refuses to run without a `seed`. Besides the three jobs, the other exception to "only when someone asks" is `POST /providers/{name}/test`, which an admin sends to check a key; it sends nothing but the key.
|
||||
|
||||
**Similar artists.** `GET /api/v1/artists/{id}/similar` asks Last.fm for the artists similar to an artist (one of yours, or one Last.fm named earlier) and keeps the answer in your database: 7 days, or 1 day when Last.fm does not know the artist. Until then a second look costs nothing and sends nothing, and if Last.fm is down you still get the last answer, marked `stale`. If the provider is off the answer says `disabled` instead of failing. Each similar artist says whether you have it (`in_library`); the ones you do not have are stored as artists without files. They can be opened with `GET /artists/{id}` but are never part of your artist lists, and if you later add their music, the scan finds them there. Last.fm's `score` is relative to each answer (the best one is usually 1), so compare artists only inside one answer.
|
||||
|
||||
|
|
@ -245,10 +245,10 @@ Everything lives under `/api/v1`. Bodies are JSON. Errors always look like `{"er
|
|||
| signed in | `GET /discover/dismissed`, `POST /discover/{id}/dismiss`, `DELETE /discover/{id}/dismiss` | The artists you are not interested in: list them, dismiss one, bring it back. |
|
||||
| signed in | `GET /me/taste` | The artists and tags Sonda learned you like, from your own listening. Query: `limit` (default 20, max 50). |
|
||||
| signed in | `GET /me/settings`, `PATCH /me/settings` | Your own settings (today: `discover_from_listening`, off by default). |
|
||||
| signed in | `GET /library/covers` | Progress of the job that builds the covers, and how the last one ended. |
|
||||
| signed in | `GET /library/covers`, `GET /library/artist-images`, `GET /library/descriptions` | Progress of the job that builds the covers, looks for the artist pictures or for the artist descriptions, and how the last one ended. |
|
||||
| signed in | `GET /library/scan` | Scan status and the last result. |
|
||||
| admin | `GET`, `POST /libraries`; `PATCH`, `DELETE /libraries/{id}` | Manage libraries. Deleting one removes its index entries, never your files. |
|
||||
| admin | `POST /library/covers`, `DELETE /library/covers` | Start, in the background, a job that builds the cover of every album (`?refresh=true` for all), or stop it. |
|
||||
| admin | `POST`, `DELETE /library/covers`; `POST`, `DELETE /library/artist-images`; `POST`, `DELETE /library/descriptions` | Start, in the background, a job that builds the cover of every album, looks for the picture of every artist or for the description of every artist (`?refresh=true` for all), or stop it. One of each at a time. |
|
||||
| admin | `POST /library/scan`, `POST /libraries/{id}/scan` | Scan everything, or one library. `?full=true` reads every file again. |
|
||||
| admin | `GET /providers`, `PUT /providers/{name}`, `POST /providers/{name}/test`, `DELETE /providers/{name}/data` | See, set up and check Last.fm and MusicBrainz, and remove what they gave. The API key is write-only. |
|
||||
| admin | `GET`, `POST /users`; `PATCH`, `DELETE /users/{id}`; `PUT /users/{id}/password` | Manage users and the admin role. The last admin cannot be demoted or deleted. |
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import (
|
|||
"codeberg.org/squidd/sonda/internal/discovery"
|
||||
"codeberg.org/squidd/sonda/internal/feedback"
|
||||
"codeberg.org/squidd/sonda/internal/images"
|
||||
"codeberg.org/squidd/sonda/internal/jobs"
|
||||
"codeberg.org/squidd/sonda/internal/library"
|
||||
"codeberg.org/squidd/sonda/internal/listening"
|
||||
"codeberg.org/squidd/sonda/internal/providers"
|
||||
|
|
@ -224,6 +225,8 @@ func run() error {
|
|||
}
|
||||
return discoveryRepo.Purge(ctx, name)
|
||||
})
|
||||
descriptions := about.NewService(discoveryRepo, about.WikipediaSource{MusicBrainz: musicbrainzClient, Wikipedia: wikipediaClient},
|
||||
providerSettings, cfg.Wikipedia.Languages, logger)
|
||||
discoveries := discovery.NewService(discoveryRepo, lastfmClient, providerSettings, logger)
|
||||
tags := discovery.NewTagService(discoveryRepo, map[string]discovery.TagSource{
|
||||
providers.LastFM: discovery.LastfmTags{Client: lastfmClient},
|
||||
|
|
@ -240,26 +243,27 @@ func run() error {
|
|||
}
|
||||
|
||||
router := httpapi.NewRouter(httpapi.Deps{
|
||||
Logger: logger,
|
||||
Auth: auth.NewService(storage.NewUserRepo(db), storage.NewRefreshRepo(db), tokens, cfg.Auth.RefreshTTL()),
|
||||
Tokens: tokens,
|
||||
Listening: listening.NewService(storage.NewListeningRepo(db), cfg.Listening.MinListen()),
|
||||
Scan: scanControl{ctx: ctx, m: scans},
|
||||
Libraries: libraries,
|
||||
Catalog: catalog.NewService(storage.NewCatalogRepo(db)),
|
||||
Providers: providerSettings,
|
||||
Discovery: discoveries,
|
||||
Tags: tags,
|
||||
Taste: tasteService,
|
||||
Discover: discover.NewService(tasteService, discoveries, tags, tasteRepo, userSettings, feedbacks, weights),
|
||||
Feedback: feedbacks,
|
||||
Images: pictures,
|
||||
CoversJob: coversControl{ctx: ctx, job: images.NewJob(pictures, imageRepo)},
|
||||
ArtistImages: pictures,
|
||||
About: about.NewService(discoveryRepo, about.WikipediaSource{MusicBrainz: musicbrainzClient, Wikipedia: wikipediaClient},
|
||||
providerSettings, cfg.Wikipedia.Languages, logger),
|
||||
Settings: userSettings,
|
||||
Streams: streaming.NewService(libraryRepo),
|
||||
Logger: logger,
|
||||
Auth: auth.NewService(storage.NewUserRepo(db), storage.NewRefreshRepo(db), tokens, cfg.Auth.RefreshTTL()),
|
||||
Tokens: tokens,
|
||||
Listening: listening.NewService(storage.NewListeningRepo(db), cfg.Listening.MinListen()),
|
||||
Scan: scanControl{ctx: ctx, m: scans},
|
||||
Libraries: libraries,
|
||||
Catalog: catalog.NewService(storage.NewCatalogRepo(db)),
|
||||
Providers: providerSettings,
|
||||
Discovery: discoveries,
|
||||
Tags: tags,
|
||||
Taste: tasteService,
|
||||
Discover: discover.NewService(tasteService, discoveries, tags, tasteRepo, userSettings, feedbacks, weights),
|
||||
Feedback: feedbacks,
|
||||
Images: pictures,
|
||||
CoversJob: jobs.Runner{Ctx: ctx, Job: images.NewCoversJob(pictures, imageRepo)},
|
||||
ArtistImagesJob: jobs.Runner{Ctx: ctx, Job: images.NewArtistImagesJob(pictures, imageRepo)},
|
||||
DescriptionsJob: jobs.Runner{Ctx: ctx, Job: about.NewJob(descriptions, discoveryRepo)},
|
||||
ArtistImages: pictures,
|
||||
About: descriptions,
|
||||
Settings: userSettings,
|
||||
Streams: streaming.NewService(libraryRepo),
|
||||
})
|
||||
|
||||
// No WriteTimeout: audio streaming (later) needs long responses.
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ import (
|
|||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"codeberg.org/squidd/sonda/internal/images"
|
||||
"codeberg.org/squidd/sonda/internal/library"
|
||||
)
|
||||
|
||||
|
|
@ -62,17 +61,3 @@ func warnAboutLibraries(ctx context.Context, logger *zap.Logger, libs *library.L
|
|||
logger.Warn("library problem: review GET /api/v1/libraries", zap.String("problem", p))
|
||||
}
|
||||
}
|
||||
|
||||
// coversControl lets the HTTP layer start the covers job on a context that lives as long as
|
||||
// the application, not as long as a single request.
|
||||
type coversControl struct {
|
||||
ctx context.Context
|
||||
job *images.Job
|
||||
}
|
||||
|
||||
func (c coversControl) Start(refresh bool) error {
|
||||
return c.job.Start(c.ctx, images.JobOptions{Refresh: refresh})
|
||||
}
|
||||
|
||||
func (c coversControl) Stop() bool { return c.job.Stop() }
|
||||
func (c coversControl) Status() images.JobStatus { return c.job.Status() }
|
||||
|
|
|
|||
|
|
@ -142,6 +142,18 @@ func NewService(store Store, source Source, settings discovery.Settings, languag
|
|||
// Description returns the description of an artist. A provider that is off, cannot be
|
||||
// asked or fails is not an error: the answer says so and carries what was stored.
|
||||
func (s *Service) Description(ctx context.Context, artistID int64) (Result, error) {
|
||||
return s.describe(ctx, artistID, false)
|
||||
}
|
||||
|
||||
// Ensure is Description for the job that goes through the library: with refresh it reads
|
||||
// again even a description that is still good. Whatever goes wrong is in the status of the
|
||||
// answer, which then still carries what was stored: a refresh never replaces a description
|
||||
// by a failure.
|
||||
func (s *Service) Ensure(ctx context.Context, artistID int64, refresh bool) (Result, error) {
|
||||
return s.describe(ctx, artistID, refresh)
|
||||
}
|
||||
|
||||
func (s *Service) describe(ctx context.Context, artistID int64, force bool) (Result, error) {
|
||||
if artistID <= 0 {
|
||||
return Result{}, discovery.ErrNotFound
|
||||
}
|
||||
|
|
@ -153,7 +165,7 @@ func (s *Service) Description(ctx context.Context, artistID int64) (Result, erro
|
|||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
fresh := snap.Fetched && s.fresh(snap)
|
||||
fresh := !force && snap.Fetched && s.fresh(snap)
|
||||
active, err := s.active(ctx)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
|
|
|
|||
60
internal/about/job.go
Normal file
60
internal/about/job.go
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
package about
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"codeberg.org/squidd/sonda/internal/discovery"
|
||||
"codeberg.org/squidd/sonda/internal/jobs"
|
||||
)
|
||||
|
||||
// Outcomes of the descriptions job, as the summary counts them.
|
||||
const (
|
||||
OutcomeFound = jobs.Found // a description was found
|
||||
OutcomeNone = jobs.None // Wikipedia has none for the artist
|
||||
)
|
||||
|
||||
// Lister lists the artists that need their description looked for, so the job does not read
|
||||
// them all at once.
|
||||
type Lister interface {
|
||||
// DescriptionsToBuild returns up to limit ids greater than after, in order, of the artists
|
||||
// of the library that have a MusicBrainz id and no answer about their description that is
|
||||
// still good (or all of them with refresh). Artists with no id are never listed: nothing
|
||||
// is looked up by name.
|
||||
DescriptionsToBuild(ctx context.Context, after int64, limit int, refresh bool, now time.Time) ([]int64, error)
|
||||
// CountDescriptionsToBuild says how many there are.
|
||||
CountDescriptionsToBuild(ctx context.Context, refresh bool, now time.Time) (int, error)
|
||||
}
|
||||
|
||||
// NewJob builds the job that looks for the description of every artist in the background. It
|
||||
// does for each artist what asking for the description does, and leaves alone those whose
|
||||
// answer is still good. It ends at once when a provider it needs is off.
|
||||
func NewJob(svc *Service, artists Lister) *jobs.Job {
|
||||
return jobs.New(jobs.Task{
|
||||
Noun: "artists",
|
||||
Count: artists.CountDescriptionsToBuild,
|
||||
List: artists.DescriptionsToBuild,
|
||||
Do: func(ctx context.Context, id int64, refresh bool) (string, error) {
|
||||
res, err := svc.Ensure(ctx, id, refresh)
|
||||
switch {
|
||||
case errors.Is(err, discovery.ErrNotFound):
|
||||
return "", jobs.ErrSkip
|
||||
case err != nil:
|
||||
return "", err
|
||||
}
|
||||
switch res.Status {
|
||||
case discovery.StatusDisabled:
|
||||
return "", jobs.Off("the wikipedia or the musicbrainz provider is off")
|
||||
case discovery.StatusNoID:
|
||||
return "", jobs.ErrSkip // the artist lost its id since it was listed
|
||||
case discovery.StatusError:
|
||||
return "", errors.New("Wikipedia could not be reached")
|
||||
}
|
||||
if res.Description == nil {
|
||||
return OutcomeNone, nil
|
||||
}
|
||||
return OutcomeFound, nil
|
||||
},
|
||||
}, svc.now)
|
||||
}
|
||||
163
internal/about/job_test.go
Normal file
163
internal/about/job_test.go
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
package about
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"codeberg.org/squidd/sonda/internal/discovery"
|
||||
"codeberg.org/squidd/sonda/internal/jobs"
|
||||
)
|
||||
|
||||
func TestEnsuringAFreshDescriptionAsksNobodyUnlessARefreshIsWanted(t *testing.T) {
|
||||
r := newRig(true)
|
||||
for i := 0; i < 2; i++ {
|
||||
res, err := r.svc.Ensure(context.Background(), 1, false)
|
||||
if err != nil || res.Status != discovery.StatusOK || res.Description == nil {
|
||||
t.Fatalf("result = %+v, err = %v", res, err)
|
||||
}
|
||||
}
|
||||
calls := r.src.total()
|
||||
if calls != 3 { // the id, the pages and one summary, once
|
||||
t.Fatalf("the source was asked %d times, want 3", calls)
|
||||
}
|
||||
r.src.summaries["it"] = standard("it", "Q44190", "Un testo nuovo.")
|
||||
res, err := r.svc.Ensure(context.Background(), 1, true)
|
||||
if err != nil || res.Description == nil || res.Description.Text != "Un testo nuovo." {
|
||||
t.Fatalf("refresh: result = %+v, err = %v", res, err)
|
||||
}
|
||||
if got := r.src.total(); got != 2*calls {
|
||||
t.Errorf("a refresh asked the source %d times in all, want %d", got, 2*calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARefreshThatFailsKeepsTheStoredDescription(t *testing.T) {
|
||||
r := newRig(true)
|
||||
r.get(t, 1)
|
||||
r.src.qidErr = errors.New("wikidata is down")
|
||||
res, err := r.svc.Ensure(context.Background(), 1, true)
|
||||
if err != nil || res.Status != discovery.StatusError || res.Description == nil || res.Description.Language != "it" {
|
||||
t.Errorf("result = %+v, err = %v, want an error status that still carries the stored text", res, err)
|
||||
}
|
||||
if got := r.store.snaps[1]; got.None || got.Description.Language != "it" {
|
||||
t.Errorf("stored = %+v, want it untouched", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARefreshWithTheProviderOffAsksNobodyAndKeepsTheDescription(t *testing.T) {
|
||||
r := newRig(true)
|
||||
r.get(t, 1)
|
||||
r.sw.set("wikipedia", false)
|
||||
before := r.src.total()
|
||||
res, err := r.svc.Ensure(context.Background(), 1, true)
|
||||
if err != nil || res.Status != discovery.StatusDisabled || res.Description == nil {
|
||||
t.Errorf("result = %+v, err = %v", res, err)
|
||||
}
|
||||
if r.src.total() != before {
|
||||
t.Error("the source was asked although the provider is off")
|
||||
}
|
||||
}
|
||||
|
||||
// ---- the job ----
|
||||
|
||||
type fakeLister struct {
|
||||
mu sync.Mutex
|
||||
ids []int64
|
||||
}
|
||||
|
||||
func (f *fakeLister) DescriptionsToBuild(_ context.Context, after int64, limit int, _ bool, _ time.Time) ([]int64, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
var out []int64
|
||||
for _, id := range f.ids {
|
||||
if id > after && len(out) < limit {
|
||||
out = append(out, id)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (f *fakeLister) CountDescriptionsToBuild(context.Context, bool, time.Time) (int, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return len(f.ids), nil
|
||||
}
|
||||
|
||||
func waitDone(t *testing.T, j *jobs.Job) jobs.Status {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(20 * time.Second)
|
||||
for {
|
||||
if st := j.Status(); !st.Running {
|
||||
return st
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("the job did not finish")
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func jobRig(t *testing.T, ids ...int64) (*rig, *jobs.Job) {
|
||||
t.Helper()
|
||||
r := newRig(true)
|
||||
for _, id := range ids {
|
||||
r.store.artists[id] = discovery.Artist{ID: id, Name: "Artist", MBID: mbid}
|
||||
}
|
||||
return r, NewJob(r.svc, &fakeLister{ids: ids})
|
||||
}
|
||||
|
||||
func TestTheDescriptionsJobFindsTheDescriptionsAndCountsWhatHasNone(t *testing.T) {
|
||||
r, job := jobRig(t, 11, 12, 13)
|
||||
r.src.summaries["it"] = Summary{Type: "disambiguation"}
|
||||
r.src.summaries["en"] = Summary{Type: "disambiguation"}
|
||||
// The pages are not articles, so there is no description for any of them.
|
||||
if err := job.Start(context.Background(), jobs.Options{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st := waitDone(t, job)
|
||||
if st.Last == nil || st.Last.Checked != 3 || st.Last.Outcomes[OutcomeNone] != 3 || st.Last.Failed != 0 {
|
||||
t.Fatalf("status = %+v", st)
|
||||
}
|
||||
_, job2 := jobRig(t, 11, 12)
|
||||
_ = job2.Start(context.Background(), jobs.Options{})
|
||||
st = waitDone(t, job2)
|
||||
if st.Last.Outcomes[OutcomeFound] != 2 {
|
||||
t.Errorf("summary = %+v, want two found", *st.Last)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheDescriptionsJobEndsAtOnceWhenAProviderIsOff(t *testing.T) {
|
||||
r, job := jobRig(t, 11, 12, 13)
|
||||
r.sw.set("musicbrainz", false)
|
||||
_ = job.Start(context.Background(), jobs.Options{})
|
||||
st := waitDone(t, job)
|
||||
if st.LastError == "" || st.Last.Checked != 0 || st.Last.Failed != 0 {
|
||||
t.Errorf("status = %+v, want it to end saying a provider is off", st)
|
||||
}
|
||||
if r.src.total() != 0 {
|
||||
t.Error("the source was asked although a provider is off")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheDescriptionsJobGivesUpWhenWikipediaKeepsFailing(t *testing.T) {
|
||||
r, job := jobRig(t, 11, 12, 13, 14, 15, 16, 17)
|
||||
r.src.qidErr = errors.New("down")
|
||||
_ = job.Start(context.Background(), jobs.Options{})
|
||||
st := waitDone(t, job)
|
||||
if st.Last.Failed != jobs.MaxConsecutiveFailures || st.LastError == "" {
|
||||
t.Errorf("status = %+v", st)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnArtistThatWentOrLostItsIdIsSkippedByTheDescriptionsJob(t *testing.T) {
|
||||
r, job := jobRig(t, 11, 12)
|
||||
delete(r.store.artists, 11)
|
||||
r.store.artists[12] = discovery.Artist{ID: 12, Name: "No Id Any More"}
|
||||
_ = job.Start(context.Background(), jobs.Options{})
|
||||
st := waitDone(t, job)
|
||||
if st.Last.Skipped != 2 || st.Last.Failed != 0 || st.LastError != "" {
|
||||
t.Errorf("status = %+v", st)
|
||||
}
|
||||
}
|
||||
|
|
@ -22,6 +22,7 @@ import (
|
|||
"codeberg.org/squidd/sonda/internal/discovery"
|
||||
"codeberg.org/squidd/sonda/internal/feedback"
|
||||
"codeberg.org/squidd/sonda/internal/images"
|
||||
"codeberg.org/squidd/sonda/internal/jobs"
|
||||
"codeberg.org/squidd/sonda/internal/library"
|
||||
"codeberg.org/squidd/sonda/internal/listening"
|
||||
"codeberg.org/squidd/sonda/internal/providers"
|
||||
|
|
@ -103,11 +104,15 @@ func newTestEnvDB(t *testing.T, mutate func(*Deps, *bun.DB)) (http.Handler, *bun
|
|||
imageRepo := storage.NewImageRepo(db)
|
||||
imageSvc := images.NewService(filepath.Join(t.TempDir(), "data"), imageRepo, imageRepo)
|
||||
deps.Images, deps.ArtistImages = imageSvc, imageSvc
|
||||
deps.CoversJob = &jobControl{ctx: context.Background(), job: images.NewJob(imageSvc, imageRepo)}
|
||||
deps.CoversJob = jobs.Runner{Ctx: context.Background(), Job: images.NewCoversJob(imageSvc, imageRepo)}
|
||||
deps.ArtistImagesJob = jobs.Runner{Ctx: context.Background(), Job: images.NewArtistImagesJob(imageSvc, imageRepo)}
|
||||
mutate(&deps, db)
|
||||
if deps.About == nil {
|
||||
deps.About = about.NewService(storage.NewDiscoveryRepo(db), noWikipedia{}, prov, []string{"en"}, zap.NewNop())
|
||||
}
|
||||
if deps.DescriptionsJob == nil {
|
||||
deps.DescriptionsJob = jobs.Runner{Ctx: context.Background(), Job: about.NewJob(about.NewService(storage.NewDiscoveryRepo(db), noWikipedia{}, prov, []string{"en"}, zap.NewNop()), storage.NewDiscoveryRepo(db))}
|
||||
}
|
||||
// Built from the final services, so a test that swaps one gets a Discover that uses it.
|
||||
if deps.Discover == nil {
|
||||
deps.Discover = discover.NewService(deps.Taste.(discover.Taster), deps.Discovery.(discover.Similar),
|
||||
|
|
@ -117,18 +122,6 @@ func newTestEnvDB(t *testing.T, mutate func(*Deps, *bun.DB)) (http.Handler, *bun
|
|||
return NewRouter(deps), db
|
||||
}
|
||||
|
||||
// jobControl lets the API start the covers job on a context that outlives the request.
|
||||
type jobControl struct {
|
||||
ctx context.Context
|
||||
job *images.Job
|
||||
}
|
||||
|
||||
func (c *jobControl) Start(refresh bool) error {
|
||||
return c.job.Start(c.ctx, images.JobOptions{Refresh: refresh})
|
||||
}
|
||||
func (c *jobControl) Stop() bool { return c.job.Stop() }
|
||||
func (c *jobControl) Status() images.JobStatus { return c.job.Status() }
|
||||
|
||||
// noWikipedia is a Wikipedia that knows nothing, for tests that do not care about it.
|
||||
type noWikipedia struct{}
|
||||
|
||||
|
|
|
|||
|
|
@ -578,6 +578,30 @@ func TestResponsesMatchTheSpec(t *testing.T) {
|
|||
}
|
||||
c.expect(c.call("DELETE", "/library/covers", token, nil), 204, "stop the covers job when none runs")
|
||||
|
||||
// ---- the artist jobs ----
|
||||
for _, job := range []struct{ path, what string }{{"/library/artist-images", "artist pictures"}, {"/library/descriptions", "descriptions"}} {
|
||||
if idle := c.expect(c.call("GET", job.path, token, nil), 200, "the "+job.what+" job before it ran"); idle["running"] != false {
|
||||
t.Fatalf("%s job = %v", job.what, idle)
|
||||
}
|
||||
c.expect(c.call("POST", job.path+"?refresh=maybe", token, nil), 400, "a "+job.what+" job with a bad refresh", "invalid_input")
|
||||
c.expect(c.call("POST", job.path, "", nil), 401, "a "+job.what+" job without a token", "unauthorized")
|
||||
c.expect(c.call("POST", job.path, token, nil), 202, "start the "+job.what+" job")
|
||||
for i := 0; ; i++ {
|
||||
st := c.expect(c.call("GET", job.path, token, nil), 200, "the "+job.what+" job running")
|
||||
if st["running"] == false {
|
||||
if last, _ := st["last"].(map[string]any); last == nil || last["checked"] == nil {
|
||||
t.Fatalf("%s job = %v, want a summary", job.what, st)
|
||||
}
|
||||
break
|
||||
}
|
||||
if i > 400 {
|
||||
t.Fatalf("the %s job did not finish", job.what)
|
||||
}
|
||||
time.Sleep(25 * time.Millisecond)
|
||||
}
|
||||
c.expect(c.call("DELETE", job.path, token, nil), 204, "stop the "+job.what+" job when none runs")
|
||||
}
|
||||
|
||||
// ---- covers ----
|
||||
cover := "/albums/" + albumID + "/cover"
|
||||
big := c.call("GET", cover, token, nil)
|
||||
|
|
|
|||
|
|
@ -1,164 +0,0 @@
|
|||
package httpapi
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/uptrace/bun"
|
||||
|
||||
"codeberg.org/squidd/sonda/internal/images"
|
||||
"codeberg.org/squidd/sonda/internal/storage"
|
||||
)
|
||||
|
||||
type fakeCoversJob struct {
|
||||
started []bool
|
||||
err error
|
||||
running bool
|
||||
stopped int
|
||||
status images.JobStatus
|
||||
}
|
||||
|
||||
func (f *fakeCoversJob) Start(refresh bool) error {
|
||||
if f.err != nil {
|
||||
return f.err
|
||||
}
|
||||
f.started = append(f.started, refresh)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeCoversJob) Stop() bool { f.stopped++; return f.running }
|
||||
func (f *fakeCoversJob) Status() images.JobStatus { return f.status }
|
||||
|
||||
func newJobAPI(t *testing.T) (http.Handler, *fakeCoversJob, string, string) {
|
||||
t.Helper()
|
||||
fake := &fakeCoversJob{}
|
||||
h, _ := newTestEnvDB(t, func(d *Deps, _ *bun.DB) { d.CoversJob = fake })
|
||||
admin := registerAndLogin(t, h)
|
||||
// A second user who is not an admin.
|
||||
cred := map[string]any{"username": "bob", "password": "longenough"}
|
||||
if r := do(t, h, "POST", "/api/v1/users", admin, cred); r.Code != http.StatusCreated {
|
||||
t.Fatalf("create bob: %d %v", r.Code, r.Body)
|
||||
}
|
||||
bob, _ := do(t, h, "POST", "/api/v1/auth/login", "", cred).Body["access_token"].(string)
|
||||
return h, fake, admin, bob
|
||||
}
|
||||
|
||||
func TestOnlyAnAdminStartsOrStopsTheCoversJob(t *testing.T) {
|
||||
h, fake, admin, bob := newJobAPI(t)
|
||||
for _, m := range []string{"POST", "DELETE"} {
|
||||
if r := do(t, h, m, "/api/v1/library/covers", "", nil); r.Code != http.StatusUnauthorized {
|
||||
t.Errorf("%s without a token: %d", m, r.Code)
|
||||
}
|
||||
if r := do(t, h, m, "/api/v1/library/covers", bob, nil); r.Code != http.StatusForbidden || errCode(r) != "admin_required" {
|
||||
t.Errorf("%s as a user: %d %v", m, r.Code, r.Body)
|
||||
}
|
||||
}
|
||||
if len(fake.started) != 0 || fake.stopped != 0 {
|
||||
t.Error("something was started or stopped by someone who is not an admin")
|
||||
}
|
||||
if r := do(t, h, "GET", "/api/v1/library/covers", "", nil); r.Code != http.StatusUnauthorized {
|
||||
t.Errorf("status without a token: %d", r.Code)
|
||||
}
|
||||
if r := do(t, h, "GET", "/api/v1/library/covers", bob, nil); r.Code != http.StatusOK {
|
||||
t.Errorf("status for a user (like the scan status): %d", r.Code)
|
||||
}
|
||||
if r := do(t, h, "POST", "/api/v1/library/covers", admin, nil); r.Code != http.StatusAccepted || r.Body["status"] != "started" {
|
||||
t.Errorf("start: %d %v", r.Code, r.Body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheCoversJobTakesAStrictRefreshParameter(t *testing.T) {
|
||||
h, fake, admin, _ := newJobAPI(t)
|
||||
for q, want := range map[string]bool{"": false, "?refresh=false": false, "?refresh=true": true} {
|
||||
if r := do(t, h, "POST", "/api/v1/library/covers"+q, admin, nil); r.Code != http.StatusAccepted {
|
||||
t.Fatalf("%q: %d %v", q, r.Code, r.Body)
|
||||
}
|
||||
if got := fake.started[len(fake.started)-1]; got != want {
|
||||
t.Errorf("%q: refresh = %v, want %v", q, got, want)
|
||||
}
|
||||
}
|
||||
n := len(fake.started)
|
||||
for _, q := range []string{"?refresh=yes", "?refresh=1", "?refresh=true&refresh=false", "?refresh="} {
|
||||
if r := do(t, h, "POST", "/api/v1/library/covers"+q, admin, nil); r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" {
|
||||
t.Errorf("%q: %d %v, want 400", q, r.Code, r.Body)
|
||||
}
|
||||
}
|
||||
if len(fake.started) != n {
|
||||
t.Error("a request that was refused started a job")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheCoversJobIsOneAtATime(t *testing.T) {
|
||||
h, fake, admin, _ := newJobAPI(t)
|
||||
fake.err = images.ErrJobRunning
|
||||
if r := do(t, h, "POST", "/api/v1/library/covers", admin, nil); r.Code != http.StatusConflict || errCode(r) != "covers_job_running" {
|
||||
t.Errorf("%d %v, want 409 covers_job_running", r.Code, r.Body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoppingTheCoversJobIsAlwaysFine(t *testing.T) {
|
||||
h, fake, admin, _ := newJobAPI(t)
|
||||
for _, running := range []bool{true, false} {
|
||||
fake.running = running
|
||||
if r := do(t, h, "DELETE", "/api/v1/library/covers", admin, nil); r.Code != http.StatusNoContent {
|
||||
t.Errorf("running=%v: %d %v", running, r.Code, r.Body)
|
||||
}
|
||||
}
|
||||
if fake.stopped != 2 {
|
||||
t.Errorf("stopped %d times", fake.stopped)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheCoversJobStatusSaysWhatItIsDoingAndHowTheLastOneEnded(t *testing.T) {
|
||||
h, fake, admin, _ := newJobAPI(t)
|
||||
r := do(t, h, "GET", "/api/v1/library/covers", admin, nil)
|
||||
if r.Code != http.StatusOK || r.Body["running"] != false {
|
||||
t.Fatalf("idle: %d %v", r.Code, r.Body)
|
||||
}
|
||||
sameKeys(t, "idle status", r.Body, "running")
|
||||
|
||||
started := time.Date(2026, 11, 12, 10, 0, 0, 0, time.UTC)
|
||||
fake.status = images.JobStatus{Running: true, StartedAt: started, Progress: &images.Progress{Done: 3, Total: 10}}
|
||||
r = do(t, h, "GET", "/api/v1/library/covers", admin, nil)
|
||||
sameKeys(t, "running status", r.Body, "running", "started_at", "progress")
|
||||
p, _ := r.Body["progress"].(map[string]any)
|
||||
if r.Body["running"] != true || r.Body["started_at"] != "2026-11-12T10:00:00Z" || p["done"] != float64(3) || p["total"] != float64(10) {
|
||||
t.Errorf("running = %v", r.Body)
|
||||
}
|
||||
|
||||
fake.status = images.JobStatus{StartedAt: started, LastFinishedAt: started.Add(time.Minute), LastError: "gave up after 5 albums in a row failed",
|
||||
Last: &images.JobSummary{Checked: 5, Failed: 5, Stopped: false}}
|
||||
r = do(t, h, "GET", "/api/v1/library/covers", admin, nil)
|
||||
sameKeys(t, "finished status", r.Body, "running", "started_at", "last_finished_at", "last", "last_error")
|
||||
last, _ := r.Body["last"].(map[string]any)
|
||||
sameKeys(t, "last", last, "checked", "from_files", "from_archive", "none", "skipped", "failed", "stopped")
|
||||
if last["checked"] != float64(5) || last["failed"] != float64(5) || last["stopped"] != false || r.Body["last_error"] == "" {
|
||||
t.Errorf("last = %v", r.Body)
|
||||
}
|
||||
}
|
||||
|
||||
// With the real job on the real stack: the albums of a library get their covers.
|
||||
func TestTheCoversJobBuildsTheCoversOfTheLibrary(t *testing.T) {
|
||||
e := newCoversEnv(t) // "With Art" has a cover.jpg in its folder, "Without" has nothing
|
||||
repo := storage.NewImageRepo(e.db)
|
||||
job := images.NewJob(e.svc, repo)
|
||||
if err := job.Start(t.Context(), images.JobOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
deadline := time.Now().Add(20 * time.Second)
|
||||
for job.Status().Running {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("the job did not finish")
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
st := job.Status()
|
||||
if st.Last == nil || st.Last.Checked != 2 || st.Last.FromFiles != 1 || st.Last.None != 1 {
|
||||
t.Errorf("last = %+v, want two albums: one cover from the files, one with none", st.Last)
|
||||
}
|
||||
// The cover is there without anybody having asked for it.
|
||||
if rec, ok, _ := repo.Get(t.Context(), "album", e.withArt); !ok || rec.Status != images.StatusOK {
|
||||
t.Errorf("record of the album with a picture: %+v ok=%v", rec, ok)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,95 +0,0 @@
|
|||
package httpapi
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"codeberg.org/squidd/sonda/internal/images"
|
||||
)
|
||||
|
||||
type coversJobHandlers struct{ job coversJobService }
|
||||
|
||||
type coversSummaryJSON struct {
|
||||
Checked int `json:"checked"`
|
||||
FromFiles int `json:"from_files"`
|
||||
FromArchive int `json:"from_archive"`
|
||||
None int `json:"none"`
|
||||
Skipped int `json:"skipped"`
|
||||
Failed int `json:"failed"`
|
||||
Stopped bool `json:"stopped"`
|
||||
}
|
||||
|
||||
type progressJSON struct {
|
||||
Done int `json:"done"`
|
||||
Total int `json:"total"`
|
||||
}
|
||||
|
||||
type coversStatusJSON struct {
|
||||
Running bool `json:"running"`
|
||||
StartedAt *string `json:"started_at,omitempty"`
|
||||
LastFinishedAt *string `json:"last_finished_at,omitempty"`
|
||||
Progress *progressJSON `json:"progress,omitempty"`
|
||||
Last *coversSummaryJSON `json:"last,omitempty"`
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
}
|
||||
|
||||
// refreshParam reads ?refresh=: only "true" and "false", once. It answers 400 itself otherwise:
|
||||
// a typo must not quietly start a job of another kind.
|
||||
func refreshParam(c *gin.Context) (refresh, ok bool) {
|
||||
values := c.QueryArray("refresh")
|
||||
switch {
|
||||
case len(values) == 0, len(values) == 1 && values[0] == "false":
|
||||
return false, true
|
||||
case len(values) == 1 && values[0] == "true":
|
||||
return true, true
|
||||
}
|
||||
writeError(c, http.StatusBadRequest, "invalid_input", "refresh must be true or false")
|
||||
return false, false
|
||||
}
|
||||
|
||||
// start launches the background job that builds every cover.
|
||||
func (h *coversJobHandlers) start(c *gin.Context) {
|
||||
refresh, ok := refreshParam(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
switch err := h.job.Start(refresh); {
|
||||
case errors.Is(err, images.ErrJobRunning):
|
||||
writeError(c, http.StatusConflict, "covers_job_running", err.Error())
|
||||
case err != nil:
|
||||
_ = c.Error(err)
|
||||
writeError(c, http.StatusInternalServerError, "internal", "internal error")
|
||||
default:
|
||||
c.JSON(http.StatusAccepted, gin.H{"status": "started"})
|
||||
}
|
||||
}
|
||||
|
||||
// stop asks a running job to end. It answers 204 whether or not one was running.
|
||||
func (h *coversJobHandlers) stop(c *gin.Context) {
|
||||
h.job.Stop()
|
||||
c.Status(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// status says whether the job is running, how far it is, and how the last one ended.
|
||||
func (h *coversJobHandlers) status(c *gin.Context) {
|
||||
st := h.job.Status()
|
||||
out := coversStatusJSON{Running: st.Running, LastError: st.LastError}
|
||||
if !st.StartedAt.IsZero() {
|
||||
out.StartedAt = rfc3339(st.StartedAt)
|
||||
}
|
||||
if !st.LastFinishedAt.IsZero() {
|
||||
out.LastFinishedAt = rfc3339(st.LastFinishedAt)
|
||||
}
|
||||
if st.Progress != nil {
|
||||
out.Progress = &progressJSON{Done: st.Progress.Done, Total: st.Progress.Total}
|
||||
}
|
||||
if l := st.Last; l != nil {
|
||||
out.Last = &coversSummaryJSON{
|
||||
Checked: l.Checked, FromFiles: l.FromFiles, FromArchive: l.FromArchive, None: l.None,
|
||||
Skipped: l.Skipped, Failed: l.Failed, Stopped: l.Stopped,
|
||||
}
|
||||
}
|
||||
c.JSON(http.StatusOK, out)
|
||||
}
|
||||
125
internal/api/http/handlers_jobs.go
Normal file
125
internal/api/http/handlers_jobs.go
Normal file
|
|
@ -0,0 +1,125 @@
|
|||
package httpapi
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"codeberg.org/squidd/sonda/internal/images"
|
||||
"codeberg.org/squidd/sonda/internal/jobs"
|
||||
)
|
||||
|
||||
// jobHandlers serve one of the jobs that go through the library: they all start, stop and
|
||||
// report the same way, and differ in what their summary counts and in the error code that says
|
||||
// the job is already running.
|
||||
type jobHandlers struct {
|
||||
job jobService
|
||||
runningCode string
|
||||
summary func(*jobs.Summary) any
|
||||
}
|
||||
|
||||
type coversSummaryJSON struct {
|
||||
Checked int `json:"checked"`
|
||||
FromFiles int `json:"from_files"`
|
||||
FromArchive int `json:"from_archive"`
|
||||
None int `json:"none"`
|
||||
Skipped int `json:"skipped"`
|
||||
Failed int `json:"failed"`
|
||||
Stopped bool `json:"stopped"`
|
||||
}
|
||||
|
||||
type artistSummaryJSON struct {
|
||||
Checked int `json:"checked"`
|
||||
Found int `json:"found"`
|
||||
None int `json:"none"`
|
||||
Skipped int `json:"skipped"`
|
||||
Failed int `json:"failed"`
|
||||
Stopped bool `json:"stopped"`
|
||||
}
|
||||
|
||||
func coversSummary(s *jobs.Summary) any {
|
||||
return coversSummaryJSON{
|
||||
Checked: s.Checked, FromFiles: s.Outcomes[images.OutcomeFromFiles], FromArchive: s.Outcomes[images.OutcomeFromArchive],
|
||||
None: s.Outcomes[images.OutcomeNone], Skipped: s.Skipped, Failed: s.Failed, Stopped: s.Stopped,
|
||||
}
|
||||
}
|
||||
|
||||
// artistSummary is the summary of the jobs that look for something about each artist; the
|
||||
// pictures and the descriptions count the same outcomes.
|
||||
func artistSummary(s *jobs.Summary) any {
|
||||
return artistSummaryJSON{
|
||||
Checked: s.Checked, Found: s.Outcomes[jobs.Found], None: s.Outcomes[jobs.None],
|
||||
Skipped: s.Skipped, Failed: s.Failed, Stopped: s.Stopped,
|
||||
}
|
||||
}
|
||||
|
||||
type progressJSON struct {
|
||||
Done int `json:"done"`
|
||||
Total int `json:"total"`
|
||||
}
|
||||
|
||||
type jobStatusJSON struct {
|
||||
Running bool `json:"running"`
|
||||
StartedAt *string `json:"started_at,omitempty"`
|
||||
LastFinishedAt *string `json:"last_finished_at,omitempty"`
|
||||
Progress *progressJSON `json:"progress,omitempty"`
|
||||
Last any `json:"last,omitempty"`
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
}
|
||||
|
||||
// refreshParam reads ?refresh=: only "true" and "false", once. It answers 400 itself otherwise:
|
||||
// a typo must not quietly start a job of another kind.
|
||||
func refreshParam(c *gin.Context) (refresh, ok bool) {
|
||||
values := c.QueryArray("refresh")
|
||||
switch {
|
||||
case len(values) == 0, len(values) == 1 && values[0] == "false":
|
||||
return false, true
|
||||
case len(values) == 1 && values[0] == "true":
|
||||
return true, true
|
||||
}
|
||||
writeError(c, http.StatusBadRequest, "invalid_input", "refresh must be true or false")
|
||||
return false, false
|
||||
}
|
||||
|
||||
// start launches the background job.
|
||||
func (h *jobHandlers) start(c *gin.Context) {
|
||||
refresh, ok := refreshParam(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
switch err := h.job.Start(refresh); {
|
||||
case errors.Is(err, jobs.ErrRunning):
|
||||
writeError(c, http.StatusConflict, h.runningCode, err.Error())
|
||||
case err != nil:
|
||||
_ = c.Error(err)
|
||||
writeError(c, http.StatusInternalServerError, "internal", "internal error")
|
||||
default:
|
||||
c.JSON(http.StatusAccepted, gin.H{"status": "started"})
|
||||
}
|
||||
}
|
||||
|
||||
// stop asks a running job to end. It answers 204 whether or not one was running.
|
||||
func (h *jobHandlers) stop(c *gin.Context) {
|
||||
h.job.Stop()
|
||||
c.Status(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// status says whether the job is running, how far it is, and how the last one ended.
|
||||
func (h *jobHandlers) status(c *gin.Context) {
|
||||
st := h.job.Status()
|
||||
out := jobStatusJSON{Running: st.Running, LastError: st.LastError}
|
||||
if !st.StartedAt.IsZero() {
|
||||
out.StartedAt = rfc3339(st.StartedAt)
|
||||
}
|
||||
if !st.LastFinishedAt.IsZero() {
|
||||
out.LastFinishedAt = rfc3339(st.LastFinishedAt)
|
||||
}
|
||||
if st.Progress != nil {
|
||||
out.Progress = &progressJSON{Done: st.Progress.Done, Total: st.Progress.Total}
|
||||
}
|
||||
if l := st.Last; l != nil {
|
||||
out.Last = h.summary(l)
|
||||
}
|
||||
c.JSON(http.StatusOK, out)
|
||||
}
|
||||
205
internal/api/http/jobs_test.go
Normal file
205
internal/api/http/jobs_test.go
Normal file
|
|
@ -0,0 +1,205 @@
|
|||
package httpapi
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/uptrace/bun"
|
||||
|
||||
"codeberg.org/squidd/sonda/internal/images"
|
||||
"codeberg.org/squidd/sonda/internal/jobs"
|
||||
"codeberg.org/squidd/sonda/internal/storage"
|
||||
)
|
||||
|
||||
type fakeJob struct {
|
||||
started []bool
|
||||
err error
|
||||
running bool
|
||||
stopped int
|
||||
status jobs.Status
|
||||
}
|
||||
|
||||
func (f *fakeJob) Start(refresh bool) error {
|
||||
if f.err != nil {
|
||||
return f.err
|
||||
}
|
||||
f.started = append(f.started, refresh)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeJob) Stop() bool { f.stopped++; return f.running }
|
||||
func (f *fakeJob) Status() jobs.Status { return f.status }
|
||||
|
||||
// jobCase is one of the jobs that go through the library: they all behave the same.
|
||||
type jobCase struct {
|
||||
name string
|
||||
path string
|
||||
running string // the error code when it is already running
|
||||
keys []string // the counters of a summary
|
||||
count string // the key of the outcomes the test fills in
|
||||
outcome string // the outcome that this key counts
|
||||
set func(*Deps, jobService)
|
||||
}
|
||||
|
||||
var jobCases = []jobCase{
|
||||
{"covers", "/api/v1/library/covers", "covers_job_running",
|
||||
[]string{"checked", "from_files", "from_archive", "none", "skipped", "failed", "stopped"},
|
||||
"from_files", images.OutcomeFromFiles, func(d *Deps, j jobService) { d.CoversJob = j }},
|
||||
{"artist images", "/api/v1/library/artist-images", "artist_images_job_running",
|
||||
[]string{"checked", "found", "none", "skipped", "failed", "stopped"},
|
||||
"found", images.OutcomeFound, func(d *Deps, j jobService) { d.ArtistImagesJob = j }},
|
||||
{"descriptions", "/api/v1/library/descriptions", "descriptions_job_running",
|
||||
[]string{"checked", "found", "none", "skipped", "failed", "stopped"},
|
||||
"found", images.OutcomeFound, func(d *Deps, j jobService) { d.DescriptionsJob = j }},
|
||||
}
|
||||
|
||||
func newJobAPI(t *testing.T, jc jobCase) (http.Handler, *fakeJob, string, string) {
|
||||
t.Helper()
|
||||
fake := &fakeJob{}
|
||||
h, _ := newTestEnvDB(t, func(d *Deps, _ *bun.DB) { jc.set(d, fake) })
|
||||
admin := registerAndLogin(t, h)
|
||||
// A second user who is not an admin.
|
||||
cred := map[string]any{"username": "bob", "password": "longenough"}
|
||||
if r := do(t, h, "POST", "/api/v1/users", admin, cred); r.Code != http.StatusCreated {
|
||||
t.Fatalf("create bob: %d %v", r.Code, r.Body)
|
||||
}
|
||||
bob, _ := do(t, h, "POST", "/api/v1/auth/login", "", cred).Body["access_token"].(string)
|
||||
return h, fake, admin, bob
|
||||
}
|
||||
|
||||
func eachJob(t *testing.T, f func(t *testing.T, jc jobCase)) {
|
||||
for _, jc := range jobCases {
|
||||
t.Run(jc.name, func(t *testing.T) { f(t, jc) })
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlyAnAdminStartsOrStopsAJob(t *testing.T) {
|
||||
eachJob(t, func(t *testing.T, jc jobCase) {
|
||||
h, fake, admin, bob := newJobAPI(t, jc)
|
||||
for _, m := range []string{"POST", "DELETE"} {
|
||||
if r := do(t, h, m, jc.path, "", nil); r.Code != http.StatusUnauthorized {
|
||||
t.Errorf("%s without a token: %d", m, r.Code)
|
||||
}
|
||||
if r := do(t, h, m, jc.path, bob, nil); r.Code != http.StatusForbidden || errCode(r) != "admin_required" {
|
||||
t.Errorf("%s as a user: %d %v", m, r.Code, r.Body)
|
||||
}
|
||||
}
|
||||
if len(fake.started) != 0 || fake.stopped != 0 {
|
||||
t.Error("something was started or stopped by someone who is not an admin")
|
||||
}
|
||||
if r := do(t, h, "GET", jc.path, "", nil); r.Code != http.StatusUnauthorized {
|
||||
t.Errorf("status without a token: %d", r.Code)
|
||||
}
|
||||
if r := do(t, h, "GET", jc.path, bob, nil); r.Code != http.StatusOK {
|
||||
t.Errorf("status for a user (like the scan status): %d", r.Code)
|
||||
}
|
||||
if r := do(t, h, "POST", jc.path, admin, nil); r.Code != http.StatusAccepted || r.Body["status"] != "started" {
|
||||
t.Errorf("start: %d %v", r.Code, r.Body)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestAJobTakesAStrictRefreshParameter(t *testing.T) {
|
||||
eachJob(t, func(t *testing.T, jc jobCase) {
|
||||
h, fake, admin, _ := newJobAPI(t, jc)
|
||||
for q, want := range map[string]bool{"": false, "?refresh=false": false, "?refresh=true": true} {
|
||||
if r := do(t, h, "POST", jc.path+q, admin, nil); r.Code != http.StatusAccepted {
|
||||
t.Fatalf("%q: %d %v", q, r.Code, r.Body)
|
||||
}
|
||||
if got := fake.started[len(fake.started)-1]; got != want {
|
||||
t.Errorf("%q: refresh = %v, want %v", q, got, want)
|
||||
}
|
||||
}
|
||||
n := len(fake.started)
|
||||
for _, q := range []string{"?refresh=yes", "?refresh=1", "?refresh=true&refresh=false", "?refresh="} {
|
||||
if r := do(t, h, "POST", jc.path+q, admin, nil); r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" {
|
||||
t.Errorf("%q: %d %v, want 400", q, r.Code, r.Body)
|
||||
}
|
||||
}
|
||||
if len(fake.started) != n {
|
||||
t.Error("a request that was refused started a job")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestAJobIsOneAtATime(t *testing.T) {
|
||||
eachJob(t, func(t *testing.T, jc jobCase) {
|
||||
h, fake, admin, _ := newJobAPI(t, jc)
|
||||
fake.err = jobs.ErrRunning
|
||||
if r := do(t, h, "POST", jc.path, admin, nil); r.Code != http.StatusConflict || errCode(r) != jc.running {
|
||||
t.Errorf("%d %v, want 409 %s", r.Code, r.Body, jc.running)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestStoppingAJobIsAlwaysFine(t *testing.T) {
|
||||
eachJob(t, func(t *testing.T, jc jobCase) {
|
||||
h, fake, admin, _ := newJobAPI(t, jc)
|
||||
for _, running := range []bool{true, false} {
|
||||
fake.running = running
|
||||
if r := do(t, h, "DELETE", jc.path, admin, nil); r.Code != http.StatusNoContent {
|
||||
t.Errorf("running=%v: %d %v", running, r.Code, r.Body)
|
||||
}
|
||||
}
|
||||
if fake.stopped != 2 {
|
||||
t.Errorf("stopped %d times", fake.stopped)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestTheStatusOfAJobSaysWhatItIsDoingAndHowTheLastOneEnded(t *testing.T) {
|
||||
eachJob(t, func(t *testing.T, jc jobCase) {
|
||||
h, fake, admin, _ := newJobAPI(t, jc)
|
||||
r := do(t, h, "GET", jc.path, admin, nil)
|
||||
if r.Code != http.StatusOK || r.Body["running"] != false {
|
||||
t.Fatalf("idle: %d %v", r.Code, r.Body)
|
||||
}
|
||||
sameKeys(t, "idle status", r.Body, "running")
|
||||
|
||||
started := time.Date(2026, 11, 12, 10, 0, 0, 0, time.UTC)
|
||||
fake.status = jobs.Status{Running: true, StartedAt: started, Progress: &jobs.Progress{Done: 3, Total: 10}}
|
||||
r = do(t, h, "GET", jc.path, admin, nil)
|
||||
sameKeys(t, "running status", r.Body, "running", "started_at", "progress")
|
||||
p, _ := r.Body["progress"].(map[string]any)
|
||||
if r.Body["running"] != true || r.Body["started_at"] != "2026-11-12T10:00:00Z" || p["done"] != float64(3) || p["total"] != float64(10) {
|
||||
t.Errorf("running = %v", r.Body)
|
||||
}
|
||||
|
||||
fake.status = jobs.Status{StartedAt: started, LastFinishedAt: started.Add(time.Minute), LastError: "gave up after 5 in a row failed",
|
||||
Last: &jobs.Summary{Checked: 7, Failed: 5, Skipped: 1, Outcomes: map[string]int{jc.outcome: 1}}}
|
||||
r = do(t, h, "GET", jc.path, admin, nil)
|
||||
sameKeys(t, "finished status", r.Body, "running", "started_at", "last_finished_at", "last", "last_error")
|
||||
last, _ := r.Body["last"].(map[string]any)
|
||||
sameKeys(t, "last", last, jc.keys...)
|
||||
if last["checked"] != float64(7) || last["failed"] != float64(5) || last["skipped"] != float64(1) || last["stopped"] != false ||
|
||||
last[jc.count] != float64(1) || r.Body["last_error"] == "" {
|
||||
t.Errorf("last = %v", r.Body)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// With the real job on the real stack: the albums of a library get their covers.
|
||||
func TestTheCoversJobBuildsTheCoversOfTheLibrary(t *testing.T) {
|
||||
e := newCoversEnv(t) // "With Art" has a cover.jpg in its folder, "Without" has nothing
|
||||
repo := storage.NewImageRepo(e.db)
|
||||
job := images.NewCoversJob(e.svc, repo)
|
||||
if err := job.Start(t.Context(), jobs.Options{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
deadline := time.Now().Add(20 * time.Second)
|
||||
for job.Status().Running {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("the job did not finish")
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
st := job.Status()
|
||||
if st.Last == nil || st.Last.Checked != 2 || st.Last.Outcomes[images.OutcomeFromFiles] != 1 || st.Last.Outcomes[images.OutcomeNone] != 1 {
|
||||
t.Errorf("last = %+v, want two albums: one cover from the files, one with none", st.Last)
|
||||
}
|
||||
// The cover is there without anybody having asked for it.
|
||||
if rec, ok, _ := repo.Get(t.Context(), "album", e.withArt); !ok || rec.Status != images.StatusOK {
|
||||
t.Errorf("record of the album with a picture: %+v ok=%v", rec, ok)
|
||||
}
|
||||
}
|
||||
|
|
@ -19,6 +19,7 @@ import (
|
|||
"codeberg.org/squidd/sonda/internal/discovery"
|
||||
"codeberg.org/squidd/sonda/internal/feedback"
|
||||
"codeberg.org/squidd/sonda/internal/images"
|
||||
"codeberg.org/squidd/sonda/internal/jobs"
|
||||
"codeberg.org/squidd/sonda/internal/library"
|
||||
"codeberg.org/squidd/sonda/internal/listening"
|
||||
"codeberg.org/squidd/sonda/internal/providers"
|
||||
|
|
@ -96,10 +97,11 @@ type aboutService interface {
|
|||
Description(ctx context.Context, artistID int64) (about.Result, error)
|
||||
}
|
||||
|
||||
type coversJobService interface {
|
||||
// jobService is one of the jobs that go through the library: jobs.Runner.
|
||||
type jobService interface {
|
||||
Start(refresh bool) error
|
||||
Stop() bool
|
||||
Status() images.JobStatus
|
||||
Status() jobs.Status
|
||||
}
|
||||
|
||||
type coverService interface {
|
||||
|
|
@ -133,25 +135,27 @@ type libraryService interface {
|
|||
|
||||
// Deps are the services the API needs.
|
||||
type Deps struct {
|
||||
Logger *zap.Logger
|
||||
Auth authService
|
||||
Tokens tokenParser
|
||||
Listening listenService
|
||||
Scan scanService
|
||||
Libraries libraryService
|
||||
Catalog catalogService
|
||||
Providers providerService
|
||||
Discovery discoveryService
|
||||
Tags tagService
|
||||
Taste tasteService
|
||||
Discover discoverService
|
||||
Feedback feedbackService
|
||||
Images coverService
|
||||
CoversJob coversJobService
|
||||
About aboutService
|
||||
ArtistImages artistImageService
|
||||
Settings userSettingsService
|
||||
Streams streamService
|
||||
Logger *zap.Logger
|
||||
Auth authService
|
||||
Tokens tokenParser
|
||||
Listening listenService
|
||||
Scan scanService
|
||||
Libraries libraryService
|
||||
Catalog catalogService
|
||||
Providers providerService
|
||||
Discovery discoveryService
|
||||
Tags tagService
|
||||
Taste tasteService
|
||||
Discover discoverService
|
||||
Feedback feedbackService
|
||||
Images coverService
|
||||
CoversJob jobService
|
||||
ArtistImagesJob jobService
|
||||
DescriptionsJob jobService
|
||||
About aboutService
|
||||
ArtistImages artistImageService
|
||||
Settings userSettingsService
|
||||
Streams streamService
|
||||
}
|
||||
|
||||
// missingDeps names the services of d that were not set. Every field but the logger is an
|
||||
|
|
@ -217,8 +221,12 @@ func NewRouter(d Deps) *gin.Engine {
|
|||
|
||||
lib := &libraryHandlers{scan: d.Scan}
|
||||
protected.GET("/library/scan", lib.scanStatus)
|
||||
cj := &coversJobHandlers{job: d.CoversJob}
|
||||
cj := &jobHandlers{job: d.CoversJob, runningCode: "covers_job_running", summary: coversSummary}
|
||||
protected.GET("/library/covers", cj.status)
|
||||
aj := &jobHandlers{job: d.ArtistImagesJob, runningCode: "artist_images_job_running", summary: artistSummary}
|
||||
protected.GET("/library/artist-images", aj.status)
|
||||
dj := &jobHandlers{job: d.DescriptionsJob, runningCode: "descriptions_job_running", summary: artistSummary}
|
||||
protected.GET("/library/descriptions", dj.status)
|
||||
|
||||
cat := &catalogHandlers{svc: d.Catalog}
|
||||
protected.GET("/artists", cat.listArtists)
|
||||
|
|
@ -254,6 +262,10 @@ func NewRouter(d Deps) *gin.Engine {
|
|||
admin.POST("/library/scan", lib.startScan)
|
||||
admin.POST("/library/covers", cj.start)
|
||||
admin.DELETE("/library/covers", cj.stop)
|
||||
admin.POST("/library/artist-images", aj.start)
|
||||
admin.DELETE("/library/artist-images", aj.stop)
|
||||
admin.POST("/library/descriptions", dj.start)
|
||||
admin.DELETE("/library/descriptions", dj.stop)
|
||||
libs := &librariesHandlers{svc: d.Libraries}
|
||||
admin.GET("/libraries", libs.list)
|
||||
admin.POST("/libraries", libs.create)
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ func TestNewRouterRefusesAMissingDependency(t *testing.T) {
|
|||
|
||||
func TestNewRouterNamesEveryMissingDependency(t *testing.T) {
|
||||
missing := missingDeps(Deps{Logger: zap.NewNop()})
|
||||
for _, want := range []string{"Auth", "Tokens", "Listening", "CoversJob", "ArtistImages", "About", "Streams"} {
|
||||
for _, want := range []string{"Auth", "Tokens", "Listening", "CoversJob", "ArtistImagesJob", "DescriptionsJob", "ArtistImages", "About", "Streams"} {
|
||||
found := false
|
||||
for _, m := range missing {
|
||||
found = found || m == want
|
||||
|
|
|
|||
|
|
@ -508,7 +508,7 @@ paths:
|
|||
found one for a week, "no cover" for a day) is skipped; send `refresh=true` to search every
|
||||
album again.
|
||||
|
||||
**This is the one request that goes through the whole library.** With the archive on, it sends
|
||||
**This goes through the whole library.** With the archive on, it sends
|
||||
the release group id of every album that has no picture in its files, one after the other and
|
||||
spaced out. It gives up, and says why in `last_error`, when the provider fails for five albums
|
||||
in a row, and also when the files cannot be read (an unmounted disk): it never turns that into
|
||||
|
|
@ -550,7 +550,7 @@ paths:
|
|||
operationId: stopCoversJob
|
||||
summary: Stop the job that builds the covers
|
||||
description: |
|
||||
Admin only. Asks a running job to end after the album it is on; the covers already built stay.
|
||||
Admin only. Asks a running job to end after the album it is on (its requests to the provider may still finish); the covers already built stay.
|
||||
It answers the same when no job is running.
|
||||
security: [{bearerAuth: []}]
|
||||
responses:
|
||||
|
|
@ -558,6 +558,163 @@ paths:
|
|||
'401': {$ref: '#/components/responses/Unauthorized'}
|
||||
'403': {$ref: '#/components/responses/AdminRequired'}
|
||||
|
||||
/library/artist-images:
|
||||
get:
|
||||
tags: [Scan]
|
||||
operationId: getArtistImagesJob
|
||||
summary: Progress of the job that looks for the artist pictures
|
||||
description: |
|
||||
Whether the job is running, how far it is (`progress`, only while it runs), and how the last one
|
||||
ended. The job is described at `POST /library/artist-images`.
|
||||
security: [{bearerAuth: []}]
|
||||
responses:
|
||||
'200':
|
||||
description: The job.
|
||||
content:
|
||||
application/json:
|
||||
schema: {$ref: '#/components/schemas/ArtistJobStatus'}
|
||||
'401': {$ref: '#/components/responses/Unauthorized'}
|
||||
post:
|
||||
tags: [Scan]
|
||||
operationId: startArtistImagesJob
|
||||
summary: Look for all the artist pictures in the background
|
||||
description: |
|
||||
Admin only. Starts, in the background and one at a time like a scan, a job that looks for the
|
||||
picture of every artist of the library that has a MusicBrainz id and no answer about its
|
||||
picture that is still good, instead of waiting for each artist to be opened; follow it with
|
||||
`GET /library/artist-images`. For each artist it does what `GET /artists/{id}/image` does, by
|
||||
the artist's MusicBrainz id and never by name: MusicBrainz, Wikidata, then a free picture on
|
||||
Wikimedia Commons. It needs an admin to have switched both the `commons` and the `musicbrainz`
|
||||
providers on; with either off the job ends at once and says so in `last_error`. Artists with
|
||||
no id are not touched. A picture is good for 30 days and "no picture" for a week; send
|
||||
`refresh=true` to search every artist again. A refresh never replaces a stored picture by a
|
||||
failure.
|
||||
|
||||
**This goes through the whole library.** It sends the MusicBrainz id
|
||||
of every artist, one after the other and spaced out, to MusicBrainz, Wikidata and Commons. It
|
||||
gives up, and says why in `last_error`, when the providers fail for five artists in a row.
|
||||
`DELETE /library/artist-images` stops it.
|
||||
security: [{bearerAuth: []}]
|
||||
parameters:
|
||||
- name: refresh
|
||||
in: query
|
||||
description: "`true` searches every artist again, also those with a good answer. `false` when absent."
|
||||
schema: {type: boolean, default: false}
|
||||
responses:
|
||||
'202':
|
||||
description: The job was started in the background.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required: [status]
|
||||
properties:
|
||||
status: {type: string, enum: [started]}
|
||||
'400': {$ref: '#/components/responses/InvalidInput'}
|
||||
'401': {$ref: '#/components/responses/Unauthorized'}
|
||||
'403': {$ref: '#/components/responses/AdminRequired'}
|
||||
'409':
|
||||
description: The job is already running (`artist_images_job_running`).
|
||||
content:
|
||||
application/json:
|
||||
schema: {$ref: '#/components/schemas/Error'}
|
||||
'500':
|
||||
description: The artists could not be counted (`internal`).
|
||||
content:
|
||||
application/json:
|
||||
schema: {$ref: '#/components/schemas/Error'}
|
||||
delete:
|
||||
tags: [Scan]
|
||||
operationId: stopArtistImagesJob
|
||||
summary: Stop the job that looks for the artist pictures
|
||||
description: |
|
||||
Admin only. Asks a running job to end after the artist it is on (its requests to the providers may still finish); the pictures already found
|
||||
stay. It answers the same when no job is running.
|
||||
security: [{bearerAuth: []}]
|
||||
responses:
|
||||
'204': {description: "Done, also when no job was running."}
|
||||
'401': {$ref: '#/components/responses/Unauthorized'}
|
||||
'403': {$ref: '#/components/responses/AdminRequired'}
|
||||
|
||||
/library/descriptions:
|
||||
get:
|
||||
tags: [Scan]
|
||||
operationId: getDescriptionsJob
|
||||
summary: Progress of the job that looks for the artist descriptions
|
||||
description: |
|
||||
Whether the job is running, how far it is (`progress`, only while it runs), and how the last one
|
||||
ended. The job is described at `POST /library/descriptions`.
|
||||
security: [{bearerAuth: []}]
|
||||
responses:
|
||||
'200':
|
||||
description: The job.
|
||||
content:
|
||||
application/json:
|
||||
schema: {$ref: '#/components/schemas/ArtistJobStatus'}
|
||||
'401': {$ref: '#/components/responses/Unauthorized'}
|
||||
post:
|
||||
tags: [Scan]
|
||||
operationId: startDescriptionsJob
|
||||
summary: Look for all the artist descriptions in the background
|
||||
description: |
|
||||
Admin only. Starts, in the background and one at a time like a scan, a job that looks for the
|
||||
description of every artist of the library that has a MusicBrainz id and no answer about its
|
||||
description that is still good; follow it with `GET /library/descriptions`. For each artist it
|
||||
does what `GET /artists/{id}/description` does, by the artist's MusicBrainz id and never by
|
||||
name: MusicBrainz, Wikidata, then Wikipedia. It needs an admin to have switched both the
|
||||
`wikipedia` and the `musicbrainz` providers on; with either off the job ends at once and says
|
||||
so in `last_error`. Artists with no id are not touched. A description is good for 30 days and
|
||||
"no description" for a day; send `refresh=true` to read every artist again. A refresh never
|
||||
replaces a stored description by a failure.
|
||||
|
||||
**This goes through the whole library.** It sends the MusicBrainz id
|
||||
of every artist, one after the other and spaced out, to MusicBrainz, Wikidata and Wikipedia.
|
||||
It gives up, and says why in `last_error`, when the providers fail for five artists in a row.
|
||||
`DELETE /library/descriptions` stops it.
|
||||
security: [{bearerAuth: []}]
|
||||
parameters:
|
||||
- name: refresh
|
||||
in: query
|
||||
description: "`true` reads every artist again, also those with a good answer. `false` when absent."
|
||||
schema: {type: boolean, default: false}
|
||||
responses:
|
||||
'202':
|
||||
description: The job was started in the background.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required: [status]
|
||||
properties:
|
||||
status: {type: string, enum: [started]}
|
||||
'400': {$ref: '#/components/responses/InvalidInput'}
|
||||
'401': {$ref: '#/components/responses/Unauthorized'}
|
||||
'403': {$ref: '#/components/responses/AdminRequired'}
|
||||
'409':
|
||||
description: The job is already running (`descriptions_job_running`).
|
||||
content:
|
||||
application/json:
|
||||
schema: {$ref: '#/components/schemas/Error'}
|
||||
'500':
|
||||
description: The artists could not be counted (`internal`).
|
||||
content:
|
||||
application/json:
|
||||
schema: {$ref: '#/components/schemas/Error'}
|
||||
delete:
|
||||
tags: [Scan]
|
||||
operationId: stopDescriptionsJob
|
||||
summary: Stop the job that looks for the artist descriptions
|
||||
description: |
|
||||
Admin only. Asks a running job to end after the artist it is on (its requests to the providers may still finish); the descriptions already
|
||||
found stay. It answers the same when no job is running.
|
||||
security: [{bearerAuth: []}]
|
||||
responses:
|
||||
'204': {description: "Done, also when no job was running."}
|
||||
'401': {$ref: '#/components/responses/Unauthorized'}
|
||||
'403': {$ref: '#/components/responses/AdminRequired'}
|
||||
|
||||
/library/scan:
|
||||
get:
|
||||
tags: [Scan]
|
||||
|
|
@ -2398,6 +2555,36 @@ components:
|
|||
stopped: {type: boolean, description: "It was stopped, or the application was."}
|
||||
last_error: {type: string, description: "Why the last job ended before the end, for example that the provider kept failing."}
|
||||
|
||||
ArtistJobStatus:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required: [running]
|
||||
properties:
|
||||
running: {type: boolean}
|
||||
started_at: {type: string, format: date-time, description: "When the last job started."}
|
||||
last_finished_at: {type: string, format: date-time}
|
||||
progress:
|
||||
type: object
|
||||
description: Only while the job runs.
|
||||
additionalProperties: false
|
||||
required: [done, total]
|
||||
properties:
|
||||
done: {type: integer}
|
||||
total: {type: integer, description: "Artists the job set out to do."}
|
||||
last:
|
||||
type: object
|
||||
description: How the last finished job ended.
|
||||
additionalProperties: false
|
||||
required: [checked, found, none, skipped, failed, stopped]
|
||||
properties:
|
||||
checked: {type: integer, description: "Artists looked at."}
|
||||
found: {type: integer, description: "Artists for which a picture, or a description, was found."}
|
||||
none: {type: integer, description: "Artists for which the provider has nothing."}
|
||||
skipped: {type: integer, description: "Artists that vanished, or lost their id, while the job ran."}
|
||||
failed: {type: integer, description: "Artists that could not be looked up just then."}
|
||||
stopped: {type: boolean, description: "It was stopped, or the application was."}
|
||||
last_error: {type: string, description: "Why the last job ended before the end, for example that a provider is off or kept failing."}
|
||||
|
||||
UserSettings:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
|
|
|
|||
|
|
@ -39,6 +39,25 @@ func (s *Service) ArtistImage(ctx context.Context, artistID int64, size Size) (*
|
|||
}
|
||||
}
|
||||
|
||||
built, err := s.buildArtistOnce(ctx, artistID)
|
||||
if err != nil {
|
||||
// An older picture is better than none when the provider cannot be asked now.
|
||||
if (errors.Is(err, ErrProviderDisabled) || errors.Is(err, ErrSourceUnavailable)) && ok && rec.Status == StatusOK {
|
||||
if c, err := s.open(kindArtist, artistID, size, rec); err == nil {
|
||||
return c, nil
|
||||
}
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if built.Status == StatusNone {
|
||||
return nil, ErrNoImage
|
||||
}
|
||||
return s.open(kindArtist, artistID, size, built)
|
||||
}
|
||||
|
||||
// buildArtistOnce builds the picture of an artist: one build per artist, however many ask at
|
||||
// once, on its own context. The caller may leave without stopping it.
|
||||
func (s *Service) buildArtistOnce(ctx context.Context, artistID int64) (Record, error) {
|
||||
done := s.flight.DoChan("artist/"+strconv.FormatInt(artistID, 10), func() (v any, err error) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
|
|
@ -47,26 +66,49 @@ func (s *Service) ArtistImage(ctx context.Context, artistID int64, size Size) (*
|
|||
}()
|
||||
return s.buildArtist(artistID)
|
||||
})
|
||||
var built Record
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
return Record{}, ctx.Err()
|
||||
case r := <-done:
|
||||
if r.Err != nil {
|
||||
// An older picture is better than none when the provider cannot be asked now.
|
||||
if (errors.Is(r.Err, ErrProviderDisabled) || errors.Is(r.Err, ErrSourceUnavailable)) && ok && rec.Status == StatusOK {
|
||||
if c, err := s.open(kindArtist, artistID, size, rec); err == nil {
|
||||
return c, nil
|
||||
}
|
||||
}
|
||||
return nil, r.Err
|
||||
return Record{}, r.Err
|
||||
}
|
||||
built = r.Val.(Record)
|
||||
return r.Val.(Record), nil
|
||||
}
|
||||
if built.Status == StatusNone {
|
||||
return nil, ErrNoImage
|
||||
}
|
||||
|
||||
// EnsureArtistImage makes sure the picture of an artist has been looked for, and says
|
||||
// whether there is one. A picture or an answer of "none" that is still good is left alone,
|
||||
// unless refresh asks to search again. Whatever is not an answer about the picture (the
|
||||
// provider is off, the artist is gone, the network failed) is returned and changes nothing:
|
||||
// a refresh never replaces a stored picture by a failure.
|
||||
func (s *Service) EnsureArtistImage(ctx context.Context, artistID int64, refresh bool) (bool, error) {
|
||||
if artistID <= 0 {
|
||||
return false, ErrArtistNotFound
|
||||
}
|
||||
return s.open(kindArtist, artistID, size, built)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !refresh {
|
||||
rec, ok, err := s.store.Get(ctx, kindArtist, artistID)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("read picture record: %w", err)
|
||||
}
|
||||
if ok && rec.Status == StatusNone && s.now().Sub(rec.CheckedAt) < NoImageTTL {
|
||||
return false, nil
|
||||
}
|
||||
if ok && rec.Status == StatusOK && s.now().Sub(rec.CheckedAt) < ArtistTTL {
|
||||
if c, err := s.open(kindArtist, artistID, Large, rec); err == nil {
|
||||
_ = c.File.Close()
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
built, err := s.buildArtistOnce(ctx, artistID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return built.Status == StatusOK, nil
|
||||
}
|
||||
|
||||
// MaxArtistPixels is the largest copy of an artist's picture that is decoded. The copy
|
||||
|
|
@ -100,13 +142,13 @@ func (s *Service) buildArtist(artistID int64) (Record, error) {
|
|||
var rec Record
|
||||
purged := false
|
||||
for attempt := 0; attempt < 3; attempt++ {
|
||||
cur, _, err := s.store.Get(ctx, kindArtist, artistID)
|
||||
cur, had, err := s.store.Get(ctx, kindArtist, artistID)
|
||||
if err != nil {
|
||||
return Record{}, fmt.Errorf("read picture record: %w", err)
|
||||
}
|
||||
epoch := s.purgeCount()
|
||||
var pic *Processed
|
||||
rec, pic, err = s.searchArtist(ctx, artistID)
|
||||
rec, pic, err = s.searchArtist(ctx, artistID, had && cur.Status == StatusOK)
|
||||
if errors.Is(err, errPurged) {
|
||||
continue
|
||||
}
|
||||
|
|
@ -143,7 +185,10 @@ var errPurged = errors.New("the pictures were taken back while this was being bu
|
|||
// searchArtist asks the source and, for a picture, prepares it (nil when there is none).
|
||||
// The files are written by saveArtist, together with the record, so that a purge cannot
|
||||
// fall between them.
|
||||
func (s *Service) searchArtist(ctx context.Context, artistID int64) (Record, *Processed, error) {
|
||||
//
|
||||
// keepGood says a good picture is stored: a download that cannot be used is then a failure,
|
||||
// not "no picture", so that the picture is not thrown away for a bad answer.
|
||||
func (s *Service) searchArtist(ctx context.Context, artistID int64, keepGood bool) (Record, *Processed, error) {
|
||||
select {
|
||||
case s.artistSl <- struct{}{}:
|
||||
defer func() { <-s.artistSl }()
|
||||
|
|
@ -152,7 +197,10 @@ func (s *Service) searchArtist(ctx context.Context, artistID int64) (Record, *Pr
|
|||
}
|
||||
none := Record{Status: StatusNone, CheckedAt: s.now()}
|
||||
found, err := s.artists.ArtistImage(ctx, artistID)
|
||||
bad := fmt.Errorf("%w: what the provider sent cannot be used", ErrSourceUnavailable)
|
||||
switch {
|
||||
case errors.Is(err, ErrNoImage) && errors.Is(err, ErrBadPicture) && keepGood:
|
||||
return Record{}, nil, bad
|
||||
case errors.Is(err, ErrNoImage):
|
||||
return none, nil, nil
|
||||
case err != nil:
|
||||
|
|
@ -160,6 +208,9 @@ func (s *Service) searchArtist(ctx context.Context, artistID int64) (Record, *Pr
|
|||
}
|
||||
p, err := ProcessLimited(found.Data, MaxArtistPixels)
|
||||
if err != nil {
|
||||
if keepGood {
|
||||
return Record{}, nil, bad
|
||||
}
|
||||
return none, nil, nil // what the provider sent is not a usable picture: there is none to show
|
||||
}
|
||||
return s.record(p, SourceCommons, found.Credit, none.CheckedAt), &p, nil
|
||||
|
|
|
|||
48
internal/images/artist_job.go
Normal file
48
internal/images/artist_job.go
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
package images
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"codeberg.org/squidd/sonda/internal/jobs"
|
||||
)
|
||||
|
||||
// OutcomeFound is what the artist pictures job counts for an artist whose picture was found.
|
||||
const OutcomeFound = jobs.Found
|
||||
|
||||
// ArtistLister lists the artists that need a picture looked for, so the job does not read them all at once.
|
||||
type ArtistLister interface {
|
||||
// ArtistPicturesToBuild returns up to limit ids greater than after, in order, of the artists
|
||||
// of the library that have a MusicBrainz id and no answer about their picture that is still
|
||||
// good (or all of them with refresh). Artists with no id are never listed: nothing is looked
|
||||
// up by name.
|
||||
ArtistPicturesToBuild(ctx context.Context, after int64, limit int, refresh bool, now time.Time) ([]int64, error)
|
||||
// CountArtistPicturesToBuild says how many there are.
|
||||
CountArtistPicturesToBuild(ctx context.Context, refresh bool, now time.Time) (int, error)
|
||||
}
|
||||
|
||||
// NewArtistImagesJob builds the job that looks for the picture of every artist in the
|
||||
// background. It does for each artist what opening the picture does, and leaves alone those
|
||||
// whose answer is still good. It ends at once when the provider is off.
|
||||
func NewArtistImagesJob(svc *Service, artists ArtistLister) *jobs.Job {
|
||||
return jobs.New(jobs.Task{
|
||||
Noun: "artists",
|
||||
Count: artists.CountArtistPicturesToBuild,
|
||||
List: artists.ArtistPicturesToBuild,
|
||||
Do: func(ctx context.Context, id int64, refresh bool) (string, error) {
|
||||
found, err := svc.EnsureArtistImage(ctx, id, refresh)
|
||||
switch {
|
||||
case errors.Is(err, ErrArtistNotFound), errors.Is(err, ErrNoID):
|
||||
return "", jobs.ErrSkip
|
||||
case errors.Is(err, ErrProviderDisabled):
|
||||
return "", jobs.Off("the commons or the musicbrainz provider is off")
|
||||
case err != nil:
|
||||
return "", err
|
||||
case found:
|
||||
return OutcomeFound, nil
|
||||
}
|
||||
return OutcomeNone, nil
|
||||
},
|
||||
}, svc.now)
|
||||
}
|
||||
251
internal/images/artist_job_test.go
Normal file
251
internal/images/artist_job_test.go
Normal file
|
|
@ -0,0 +1,251 @@
|
|||
package images
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image/color"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"codeberg.org/squidd/sonda/internal/jobs"
|
||||
)
|
||||
|
||||
func TestEnsuringAnArtistPictureBuildsAndStoresItWithoutOpeningIt(t *testing.T) {
|
||||
r := newArtistRig(t)
|
||||
found, err := r.svc.EnsureArtistImage(context.Background(), 7, false)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("found = %v, err = %v", found, err)
|
||||
}
|
||||
if rec := r.store.recs["artist/7"]; rec.Status != StatusOK || rec.Credit != credit {
|
||||
t.Errorf("record = %+v", rec)
|
||||
}
|
||||
r.get(t, 7, Large)
|
||||
if n := atomic.LoadInt32(&r.src.calls); n != 1 {
|
||||
t.Errorf("the provider was asked %d times, want 1: the picture is stored", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsuringAnArtistWithNoFreePictureSaysSoAndRemembersIt(t *testing.T) {
|
||||
r := newArtistRig(t)
|
||||
r.src.set(Found{}, ErrNoImage)
|
||||
found, err := r.svc.EnsureArtistImage(context.Background(), 7, false)
|
||||
if err != nil || found {
|
||||
t.Fatalf("found = %v, err = %v, want no picture and no error", found, err)
|
||||
}
|
||||
if rec := r.store.recs["artist/7"]; rec.Status != StatusNone {
|
||||
t.Errorf("record = %+v", rec)
|
||||
}
|
||||
if found, err := r.svc.EnsureArtistImage(context.Background(), 7, false); err != nil || found {
|
||||
t.Errorf("second time: found = %v, err = %v", found, err)
|
||||
}
|
||||
if n := atomic.LoadInt32(&r.src.calls); n != 1 {
|
||||
t.Errorf("the provider was asked %d times, want 1", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsuringAGoodPictureAsksNobodyUnlessARefreshIsWanted(t *testing.T) {
|
||||
r := newArtistRig(t)
|
||||
for i := 0; i < 3; i++ {
|
||||
if found, err := r.svc.EnsureArtistImage(context.Background(), 7, false); err != nil || !found {
|
||||
t.Fatalf("found = %v, err = %v", found, err)
|
||||
}
|
||||
}
|
||||
if n := atomic.LoadInt32(&r.src.calls); n != 1 {
|
||||
t.Fatalf("the provider was asked %d times, want 1", n)
|
||||
}
|
||||
if found, err := r.svc.EnsureArtistImage(context.Background(), 7, true); err != nil || !found {
|
||||
t.Fatalf("refresh: found = %v, err = %v", found, err)
|
||||
}
|
||||
if n := atomic.LoadInt32(&r.src.calls); n != 2 {
|
||||
t.Errorf("a refresh asked the provider %d times in all, want 2", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAGoodPictureWhoseFileIsGoneIsBuiltAgain(t *testing.T) {
|
||||
r := newArtistRig(t)
|
||||
_, _ = r.svc.EnsureArtistImage(context.Background(), 7, false)
|
||||
if _, err := r.svc.RemoveArtistFiles(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if found, err := r.svc.EnsureArtistImage(context.Background(), 7, false); err != nil || !found {
|
||||
t.Fatalf("found = %v, err = %v", found, err)
|
||||
}
|
||||
r.get(t, 7, Large)
|
||||
if n := atomic.LoadInt32(&r.src.calls); n != 2 {
|
||||
t.Errorf("the provider was asked %d times, want 2", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARefreshThatCannotAskKeepsTheStoredPicture(t *testing.T) {
|
||||
for name, cause := range map[string]error{"unreachable": ErrSourceUnavailable, "off": ErrProviderDisabled} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
r := newArtistRig(t)
|
||||
_, _ = r.svc.EnsureArtistImage(context.Background(), 7, false)
|
||||
before := r.store.recs["artist/7"]
|
||||
r.src.set(Found{}, cause)
|
||||
if _, err := r.svc.EnsureArtistImage(context.Background(), 7, true); !errors.Is(err, cause) {
|
||||
t.Fatalf("err = %v, want %v", err, cause)
|
||||
}
|
||||
if after := r.store.recs["artist/7"]; after != before {
|
||||
t.Errorf("record = %+v, want it untouched: %+v", after, before)
|
||||
}
|
||||
r.get(t, 7, Large) // still served
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsuringWithoutASourceIsAProviderThatIsOff(t *testing.T) {
|
||||
r := newRig(t)
|
||||
if _, err := r.svc.EnsureArtistImage(context.Background(), 7, false); !errors.Is(err, ErrProviderDisabled) {
|
||||
t.Errorf("err = %v, want ErrProviderDisabled", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsuringAnArtistThatIsNotThereIsNotFound(t *testing.T) {
|
||||
r := newArtistRig(t)
|
||||
r.src.set(Found{}, ErrArtistNotFound)
|
||||
if _, err := r.svc.EnsureArtistImage(context.Background(), 7, false); !errors.Is(err, ErrArtistNotFound) {
|
||||
t.Errorf("err = %v", err)
|
||||
}
|
||||
if _, err := r.svc.EnsureArtistImage(context.Background(), 0, false); !errors.Is(err, ErrArtistNotFound) {
|
||||
t.Errorf("id 0: err = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- the job ----
|
||||
|
||||
type fakeArtistLister struct {
|
||||
mu sync.Mutex
|
||||
ids []int64
|
||||
}
|
||||
|
||||
func (f *fakeArtistLister) ArtistPicturesToBuild(_ context.Context, after int64, limit int, _ bool, _ time.Time) ([]int64, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
var out []int64
|
||||
for _, id := range f.ids {
|
||||
if id > after && len(out) < limit {
|
||||
out = append(out, id)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (f *fakeArtistLister) CountArtistPicturesToBuild(context.Context, bool, time.Time) (int, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return len(f.ids), nil
|
||||
}
|
||||
|
||||
func newPicturesJob(t *testing.T) (*artistRig, *jobs.Job) {
|
||||
t.Helper()
|
||||
r := newArtistRig(t)
|
||||
return r, NewArtistImagesJob(r.svc, &fakeArtistLister{ids: []int64{1, 2, 3, 4, 5, 6}})
|
||||
}
|
||||
|
||||
func TestTheArtistPicturesJobFindsThePicturesAndCountsWhatHasNone(t *testing.T) {
|
||||
r, job := newPicturesJob(t)
|
||||
var seen int32
|
||||
r.svc.SetArtistSource(artistFunc(func(context.Context, int64) (Found, error) {
|
||||
if atomic.AddInt32(&seen, 1)%3 == 0 {
|
||||
return Found{}, ErrNoImage
|
||||
}
|
||||
return Found{Data: jpegBytes(t, 800, 600, color.White), Credit: credit}, nil
|
||||
}))
|
||||
if err := job.Start(context.Background(), jobs.Options{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st := waitDone(t, job)
|
||||
if st.Last == nil || st.Last.Checked != 6 || st.Last.Outcomes[OutcomeFound] != 4 || st.Last.Outcomes[OutcomeNone] != 2 || st.Last.Failed != 0 {
|
||||
t.Errorf("status = %+v", st)
|
||||
}
|
||||
}
|
||||
|
||||
type artistFunc func(ctx context.Context, id int64) (Found, error)
|
||||
|
||||
func (f artistFunc) ArtistImage(ctx context.Context, id int64) (Found, error) { return f(ctx, id) }
|
||||
|
||||
func TestTheArtistPicturesJobEndsAtOnceWhenTheProviderIsOff(t *testing.T) {
|
||||
r, job := newPicturesJob(t)
|
||||
r.src.set(Found{}, ErrProviderDisabled)
|
||||
if err := job.Start(context.Background(), jobs.Options{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st := waitDone(t, job)
|
||||
if st.LastError == "" || st.Last.Checked != 0 || st.Last.Failed != 0 {
|
||||
t.Errorf("status = %+v, want it to end saying the provider is off", st)
|
||||
}
|
||||
if n := atomic.LoadInt32(&r.src.calls); n != 1 {
|
||||
t.Errorf("the provider was asked %d times", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheArtistPicturesJobGivesUpWhenTheProviderKeepsFailing(t *testing.T) {
|
||||
r, job := newPicturesJob(t)
|
||||
r.src.set(Found{}, ErrSourceUnavailable)
|
||||
_ = job.Start(context.Background(), jobs.Options{})
|
||||
st := waitDone(t, job)
|
||||
if st.Last.Failed != jobs.MaxConsecutiveFailures || st.LastError == "" {
|
||||
t.Errorf("status = %+v", st)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnArtistThatWentIsSkippedByThePicturesJob(t *testing.T) {
|
||||
r, job := newPicturesJob(t)
|
||||
r.src.set(Found{}, ErrArtistNotFound)
|
||||
_ = job.Start(context.Background(), jobs.Options{})
|
||||
st := waitDone(t, job)
|
||||
if st.Last.Skipped != 6 || st.Last.Failed != 0 || st.LastError != "" {
|
||||
t.Errorf("status = %+v", st)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- a bad download never replaces a good picture ----
|
||||
|
||||
func TestAPictureThatCannotBeUsedNowDoesNotReplaceAGoodOne(t *testing.T) {
|
||||
bad := map[string]func(r *artistRig){
|
||||
"the download is refused": func(r *artistRig) { r.src.set(Found{}, fmt.Errorf("%w: %w", ErrNoImage, ErrBadPicture)) },
|
||||
"the bytes are not a picture": func(r *artistRig) {
|
||||
r.src.set(Found{Data: []byte("<html>502 Bad Gateway</html>"), Credit: credit}, nil)
|
||||
},
|
||||
}
|
||||
for name, breakIt := range bad {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
r := newArtistRig(t)
|
||||
_, _ = r.svc.EnsureArtistImage(context.Background(), 7, false)
|
||||
before := r.store.recs["artist/7"]
|
||||
breakIt(r)
|
||||
if _, err := r.svc.EnsureArtistImage(context.Background(), 7, true); !errors.Is(err, ErrSourceUnavailable) {
|
||||
t.Fatalf("err = %v, want ErrSourceUnavailable: the answer is not about the picture", err)
|
||||
}
|
||||
if after := r.store.recs["artist/7"]; after != before {
|
||||
t.Errorf("record = %+v, want it untouched: %+v", after, before)
|
||||
}
|
||||
r.get(t, 7, Large) // still served, files included
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPictureThatCannotBeUsedIsNoPictureWhenThereWasNoneBefore(t *testing.T) {
|
||||
r := newArtistRig(t)
|
||||
r.src.set(Found{}, fmt.Errorf("%w: %w", ErrNoImage, ErrBadPicture))
|
||||
if found, err := r.svc.EnsureArtistImage(context.Background(), 7, false); err != nil || found {
|
||||
t.Errorf("found = %v, err = %v, want no picture and no error", found, err)
|
||||
}
|
||||
if rec := r.store.recs["artist/7"]; rec.Status != StatusNone {
|
||||
t.Errorf("record = %+v", rec)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnArtistThatLostItsIdIsSkippedByThePicturesJob(t *testing.T) {
|
||||
r, job := newPicturesJob(t)
|
||||
r.src.set(Found{}, ErrNoID)
|
||||
_ = job.Start(context.Background(), jobs.Options{})
|
||||
st := waitDone(t, job)
|
||||
if st.Last.Skipped != 6 || st.Last.Failed != 0 || st.LastError != "" {
|
||||
t.Errorf("status = %+v", st)
|
||||
}
|
||||
}
|
||||
|
|
@ -123,7 +123,7 @@ func (c *CommonsSource) ArtistImage(ctx context.Context, artistID int64) (Found,
|
|||
data, err := c.Commons.Download(ctx, file.ThumbURL)
|
||||
switch {
|
||||
case errors.Is(err, commons.ErrBadRequest), errors.Is(err, commons.ErrNotFound), errors.Is(err, commons.ErrUnusable):
|
||||
return Found{}, ErrNoImage
|
||||
return Found{}, fmt.Errorf("%w: %w", ErrNoImage, ErrBadPicture)
|
||||
case err != nil:
|
||||
return Found{}, failed(ctx, err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -226,7 +226,7 @@ func TestARequestThatTheCallerCancelledIsReturnedAsCancelled(t *testing.T) {
|
|||
func TestAPictureThatCannotBeUsedFromTheHostIsNoPicture(t *testing.T) {
|
||||
c := newChain()
|
||||
c.cm.loadErr = commons.ErrUnusable // a redirect or a picture over the limit
|
||||
if _, err := c.src.ArtistImage(context.Background(), 1); !errors.Is(err, ErrNoImage) {
|
||||
t.Errorf("err = %v, want ErrNoImage", err)
|
||||
if _, err := c.src.ArtistImage(context.Background(), 1); !errors.Is(err, ErrNoImage) || !errors.Is(err, ErrBadPicture) {
|
||||
t.Errorf("err = %v, want ErrNoImage that says the picture itself was bad", err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue