From 48cb903115831ad7dcd33b45bb446c2cd62ac52d Mon Sep 17 00:00:00 2001 From: tgiachi Date: Wed, 11 Nov 2026 21:36:27 +0100 Subject: [PATCH] feat(about): artist descriptions from Wikipedia - GET /artists/{id}/description shows what Wikipedia says about an artist: a short text with its page, a link and the licence (CC BY-SA 4.0). - New provider "wikipedia" (off by default, no key, contact in the User-Agent, which Wikimedia requires). The way to the text is a chain of ids, never a name: the artist's MusicBrainz id, the Wikidata item MusicBrainz links to it, the Wikipedia page Wikidata names in the first configured language that has one. A page is used only if it is an article about that same item. - [wikipedia] languages in the config (one to five codes of two or three lower case letters, checked before they are sent: an unknown site makes Wikidata answer with every language). - Stored 30 days (a day when there was nothing to find), served stale if Wikipedia fails, removed with the provider's data (the purge now reports descriptions). - MusicBrainz client: WikidataID. Wikipedia client for Wikidata and the summary, which also reads Wikidata's errors that come with HTTP 200. - artist_descriptions table (migration 0014, SQLite and PostgreSQL). - Spec, contract test, README and example config updated. Closes #39 --- README.md | 8 +- cmd/sondad/main.go | 25 +- internal/about/about.go | 271 ++++++++++++ internal/about/about_test.go | 404 ++++++++++++++++++ internal/about/source.go | 51 +++ internal/about/source_test.go | 77 ++++ internal/api/http/api_test.go | 15 + internal/api/http/contract_test.go | 13 +- internal/api/http/description_test.go | 200 +++++++++ internal/api/http/handlers_about.go | 65 +++ internal/api/http/handlers_providers.go | 2 +- internal/api/http/providers_test.go | 4 +- internal/api/http/router.go | 8 + internal/api/openapi/openapi.yaml | 85 +++- internal/config/config.go | 23 + internal/config/config_test.go | 16 + internal/providers/musicbrainz/musicbrainz.go | 40 ++ .../providers/musicbrainz/musicbrainz_test.go | 59 +++ internal/providers/settings.go | 11 + internal/providers/settings_test.go | 45 +- internal/providers/wikipedia/wikipedia.go | 249 +++++++++++ .../providers/wikipedia/wikipedia_test.go | 249 +++++++++++ internal/storage/about_test.go | 115 +++++ internal/storage/discovery.go | 61 +++ internal/storage/storage_test.go | 6 +- .../postgres/0014_artist_descriptions.sql | 19 + .../sqlite/0014_artist_descriptions.sql | 19 + sonda.example.toml | 3 + 28 files changed, 2124 insertions(+), 19 deletions(-) create mode 100644 internal/about/about.go create mode 100644 internal/about/about_test.go create mode 100644 internal/about/source.go create mode 100644 internal/about/source_test.go create mode 100644 internal/api/http/description_test.go create mode 100644 internal/api/http/handlers_about.go create mode 100644 internal/providers/wikipedia/wikipedia.go create mode 100644 internal/providers/wikipedia/wikipedia_test.go create mode 100644 internal/storage/about_test.go create mode 100644 migrations/postgres/0014_artist_descriptions.sql create mode 100644 migrations/sqlite/0014_artist_descriptions.sql diff --git a/README.md b/README.md index 0d5594f..8a775c8 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ It is a single executable, one database (SQLite by default, PostgreSQL if you pr - **Listening history.** Clients report explicit `play`, `complete` and `skip` events. An event counts as a listen when it is `complete`, or when it was played for at least 20 seconds (`listening.min_listen_seconds`); the rule is applied when the history is read, so changing it re-reads the past. You can read your own history, with a `counted` flag on every event. It stays in your database and is never sent anywhere. - **Your taste.** `GET /me/taste` shows what Sonda has learned from your own listening: the artists you like and the kinds of music (tags) they have in common. A complete listen counts for more than a short one, a skip counts against the artist, and older listens weigh less (they halve every 90 days, `taste.half_life_days`; only the last ten half-lives are read). It is worked out on every request from data already in your database; nothing is sent to any provider. - **Discover.** `GET /discover` suggests artists you may like, in your library or not, each with a score from 0 to 1 and the parts it is made of: how close it is to the artists you like (affinity), how close its tags are to yours (genre), what your listening says about it (preference) and a bonus for what you have not played (discovery). The weights are in `[scoring]` in the config. A part with no data is left out and the rest rescaled, and every answer says which seeds led to each suggestion. It starts from the artists you name with `?seed=`, or, only if you turned on `discover_from_listening`, from the artists you play most. It needs Last.fm to be switched on to ask for anything new: when Last.fm is off and a seed's similar artists are not stored yet, the answer says `disabled` (what was stored earlier is still used). See [External providers](#external-providers). +- **Artist descriptions.** `GET /artists/{id}/description` shows what Wikipedia says about an artist: a short text, with the page it comes from, a link to it and the licence (CC BY-SA 4.0), which you must show next to it. With the `wikipedia` provider switched on, Sonda follows ids and never a name: the artist's MusicBrainz id, then the Wikidata item that MusicBrainz links to it, then the Wikipedia page that Wikidata names in the first of your languages (`[wikipedia] languages`, for example `["it", "en"]`) that has one. An artist without a MusicBrainz id has no description. It is kept for 30 days and removed with the provider's data. See [External providers](#external-providers). - **Album covers.** `GET /albums/{id}/cover` serves the cover of an album from your own files: the picture embedded in them, or else a `cover`, `folder`, `front`, `album` or `albumart` image in the album's folder. It is built the first time it is asked for (a few at a time), scaled (`?size=small` is 300 pixels, `large` 1000) and kept in the data folder (`storage.data_dir`, you can delete it: covers are built again). A picture you replace in a folder is noticed within a week, or at once after a full rescan. For an album split in `CD1`/`Disc 2` folders, the picture in the folder above is used. Nothing is sent anywhere. It also takes `?access_token=`, so an `` element can use it. - **Wishlist and "not interested".** `PUT /wishlist/{artist}` puts an artist on your wishlist (`GET /wishlist` lists it, `DELETE` takes it off), and `POST /discover/{artist}/dismiss` hides an artist from your Discover for good (`DELETE` brings it back, `GET /discover/dismissed` lists them). An artist is wished or dismissed, never both, and both lists are yours alone. Discover marks the wished artists it shows (`wishlisted`) and never suggests a dismissed one. Nothing is downloaded or sent anywhere: the wishlist is a list for you, and for a download client you may set up later. An artist that only Last.fm named is kept while you have it marked (your lists still show it, but other users cannot see that it exists), and goes with the rest of Last.fm's data if an admin removes Last.fm's data. Removing MusicBrainz's data does not touch it. - **Your own settings.** `GET` and `PATCH /me/settings` hold choices that belong to you, not to the server. Everyone starts private: `discover_from_listening` is off, so this setting never sends your listening to a provider. When you turn it on, Discover may start from the artists you listen to most, and their names are sent to Last.fm (only if an admin switched it on). What Last.fm answers is kept in the shared database like any other lookup, so another local user can see, from `retrieved_at` of `GET /artists/{id}/similar`, that someone looked up an artist, but not who. @@ -66,6 +67,7 @@ Precedence is: built-in defaults, then the file, then environment variables. An | `auth.refresh_ttl_hours` | | `720` | Lifetime of a refresh token (30 days). | | `listening.min_listen_seconds` | | `20` | A play counts as a listen after this long (1 to 3600), or when it is `complete`. | | `taste.half_life_days` | | `90` | After this many days a listen weighs half as much when learning your taste (1 to 3650). | +| `wikipedia.languages` | | `["en"]` | Languages for artist descriptions, best first: one to five codes of two or three lower case letters. | | `storage.data_dir` | `SONDA_DATA_DIR` | `data` next to the SQLite database (`data` in the working directory with PostgreSQL) | Where Sonda keeps the files it makes (covers). They are built from your music, so they need no backup. | | `scoring.weight_affinity`, `weight_genre`, `weight_preference`, `weight_discovery` | | `0.40`, `0.25`, `0.25`, `0.10` | The weights of the parts of a Discover score (0 to 10 each, not all zero). Only their proportions matter. | @@ -185,7 +187,7 @@ curl -s -X POST $BASE/users -H "$AUTH" -d '{"username":"bob","password":"another ## External providers -Sonda can ask Last.fm and MusicBrainz about artists. This is **off by default**: until an admin turns a provider on, nothing leaves your server. Today Last.fm is used for similar artists and tags, and MusicBrainz for tags. +Sonda can ask Last.fm, MusicBrainz and Wikipedia about artists. This is **off by default**: until an admin turns a provider on, nothing leaves your server. Today Last.fm is used for similar artists and tags, MusicBrainz for tags and for the link to Wikidata, and Wikipedia (with Wikidata) for descriptions of artists. - **Last.fm** needs an API key. Create one for your own use at `last.fm/api` (it needs a Last.fm account). No user name or password is involved and the "shared secret" is not needed. Last.fm data is for non-commercial use only, and you should credit Last.fm when you show it. - **MusicBrainz** needs no key, but it asks every client to identify itself with a contact (a URL or an e-mail address) in the `User-Agent`. By default Sonda uses the address of the project, so no personal address is sent unless you choose to. @@ -202,7 +204,7 @@ curl -s -X PUT $BASE/providers/musicbrainz -H "$AUTH" -d '{"enabled":true,"conta The key is stored in the database, in clear text, and is never returned by the API or written to the log: the list only says `has_api_key`. Because the database holds it, treat database backups like secrets. Removing the key turns the provider off, and each provider describes in the list exactly what it sends. -**What leaves your server, when a provider is on:** to Last.fm, the artist name (or MusicBrainz id) being looked up, your API key, your IP address and the `User-Agent`; to MusicBrainz, only the MusicBrainz id of the artist, your IP address and a `User-Agent` that carries your contact. Never your users, your listens, track titles or file paths. Requests only happen when someone asks for them, never as a bulk pass over your library, and they are spaced out, retried with backoff, and stopped by a circuit breaker if a provider is down. **Discover** asks Last.fm for the similar artists of each seed (the seed's name or MusicBrainz id) and, for the best suggestions that have no tags yet, for the tags of that suggestion (its name); never MusicBrainz. Seeds you name are sent when you ask. Seeds taken from your listening, which are the names of the artists you play most, are sent only if you turned on `discover_from_listening` yourself; otherwise Discover refuses to run without a `seed`. The one exception to "only when someone asks" is `POST /providers/{name}/test`, which an admin sends to check a key; it sends nothing but the key. +**What leaves your server, when a provider is on:** to Last.fm, the artist name (or MusicBrainz id) being looked up, your API key, your IP address and the `User-Agent`; to MusicBrainz, only the MusicBrainz id of the artist, your IP address and a `User-Agent` that carries your contact. Never your users, your listens, track titles or file paths. Requests only happen when someone asks for them, never as a bulk pass over your library, and they are spaced out, retried with backoff, and stopped by a circuit breaker if a provider is down. **Descriptions** (the Wikipedia provider) send the MusicBrainz id to MusicBrainz, the Wikidata item to Wikidata and the title of the page to Wikipedia, with your IP address and a `User-Agent` that carries your contact (Wikimedia refuses requests without one), and never a name. **Discover** asks Last.fm for the similar artists of each seed (the seed's name or MusicBrainz id) and, for the best suggestions that have no tags yet, for the tags of that suggestion (its name); never MusicBrainz. Seeds you name are sent when you ask. Seeds taken from your listening, which are the names of the artists you play most, are sent only if you turned on `discover_from_listening` yourself; otherwise Discover refuses to run without a `seed`. The one exception to "only when someone asks" is `POST /providers/{name}/test`, which an admin sends to check a key; it sends nothing but the key. **Similar artists.** `GET /api/v1/artists/{id}/similar` asks Last.fm for the artists similar to an artist (one of yours, or one Last.fm named earlier) and keeps the answer in your database: 7 days, or 1 day when Last.fm does not know the artist. Until then a second look costs nothing and sends nothing, and if Last.fm is down you still get the last answer, marked `stale`. If the provider is off the answer says `disabled` instead of failing. Each similar artist says whether you have it (`in_library`); the ones you do not have are stored as artists without files. They can be opened with `GET /artists/{id}` but are never part of your artist lists, and if you later add their music, the scan finds them there. Last.fm's `score` is relative to each answer (the best one is usually 1), so compare artists only inside one answer. @@ -230,6 +232,7 @@ Everything lives under `/api/v1`. Bodies are JSON. Errors always look like `{"er | signed in | `GET /auth/me`, `POST /auth/password` | Your account; change your password (signs you out everywhere). | | signed in | `GET /artists`, `/albums`, `/tracks` and `/…/{id}` | Browse and search. Query: `q`, `limit` (default 50, max 200), `offset`, `library_id`, `artist_id`, `album_id`. | | signed in | `GET /artists/{id}/tags` | What Last.fm and MusicBrainz say about the artist's kind of music, from each provider that is on. | +| signed in | `GET /artists/{id}/description` | What Wikipedia says about the artist, with its page and licence, from the Wikipedia provider when an admin switched it on. | | signed in | `GET /artists/{id}/similar` | Artists similar to one, from Last.fm when an admin switched it on. Query: `limit` (default 20, max 50). | | signed in | `GET`, `HEAD /tracks/{id}/stream` | Stream the original file. Also accepts `?access_token=`. | | signed in | `POST /listens` | Record a `play`, `complete` or `skip`; the answer says whether it counts as a listen. | @@ -290,6 +293,7 @@ internal/api/openapi/ the OpenAPI spec, embedded in the binary internal/auth/ users, roles, passwords, access and refresh tokens internal/catalog/ browsing and searching artists, albums, tracks internal/library/ libraries, the scanner, the event hub, the tag reader +internal/about/ descriptions of artists, from Wikipedia through MusicBrainz and Wikidata internal/discovery/ similar artists: what the providers said, kept and refreshed internal/providers/ external providers: settings, the outbound HTTP client, the Last.fm and MusicBrainz clients internal/listening/ listen events diff --git a/cmd/sondad/main.go b/cmd/sondad/main.go index cdf6af1..24d6cfd 100644 --- a/cmd/sondad/main.go +++ b/cmd/sondad/main.go @@ -14,6 +14,7 @@ import ( "github.com/gin-gonic/gin" "go.uber.org/zap" + "codeberg.org/squidd/sonda/internal/about" httpapi "codeberg.org/squidd/sonda/internal/api/http" "codeberg.org/squidd/sonda/internal/auth" "codeberg.org/squidd/sonda/internal/catalog" @@ -28,6 +29,7 @@ import ( "codeberg.org/squidd/sonda/internal/providers/lastfm" "codeberg.org/squidd/sonda/internal/providers/musicbrainz" "codeberg.org/squidd/sonda/internal/providers/outbound" + "codeberg.org/squidd/sonda/internal/providers/wikipedia" "codeberg.org/squidd/sonda/internal/storage" "codeberg.org/squidd/sonda/internal/streaming" "codeberg.org/squidd/sonda/internal/taste" @@ -46,6 +48,13 @@ const lastfmMinInterval = 250 * time.Millisecond // musicbrainzMinInterval keeps a little above MusicBrainz's rule of one request a second. const musicbrainzMinInterval = 1100 * time.Millisecond +// wikipediaMinInterval is polite: Wikimedia asks only for serial requests, and a +// description costs two of them. wikipediaMaxBody is far above any answer used here. +const ( + wikipediaMinInterval = 250 * time.Millisecond + wikipediaMaxBody = 1 << 20 +) + func main() { if err := run(); err != nil { fmt.Fprintln(os.Stderr, "sondad:", err) @@ -132,6 +141,16 @@ func run() error { providerSettings.SetChecker(providers.MusicBrainz, func(ctx context.Context, _ string) error { return musicbrainzClient.Probe(ctx) }) + // Wikipedia and Wikidata: asked only for the description of an artist, through ids. + // Wikimedia answers 403 to a request without a User-Agent, and wants a contact in it. + wikipediaClient := wikipedia.New(outbound.New(outbound.Options{ + UserAgentFunc: providerSettings.UserAgentFor(providers.Wikipedia, version), + MinInterval: wikipediaMinInterval, + MaxBody: wikipediaMaxBody, + }), wikipedia.WikidataURL, wikipedia.SummaryURL) + providerSettings.SetChecker(providers.Wikipedia, func(ctx context.Context, _ string) error { + return wikipediaClient.Probe(ctx) + }) discoveryRepo := storage.NewDiscoveryRepo(db) providerSettings.SetPurger(func(ctx context.Context, name string) (providers.Purged, error) { return discoveryRepo.Purge(ctx, name) @@ -170,8 +189,10 @@ func run() error { Discover: discover.NewService(tasteService, discoveries, tags, tasteRepo, userSettings, feedbacks, weights), Feedback: feedbacks, Images: images.NewService(cfg.Storage.DataDir, imageRepo, imageRepo), - Settings: userSettings, - Streams: streaming.NewService(libraryRepo), + About: about.NewService(discoveryRepo, about.WikipediaSource{MusicBrainz: musicbrainzClient, Wikipedia: wikipediaClient}, + providerSettings, cfg.Wikipedia.Languages, logger), + Settings: userSettings, + Streams: streaming.NewService(libraryRepo), }) // No WriteTimeout: audio streaming (later) needs long responses. diff --git a/internal/about/about.go b/internal/about/about.go new file mode 100644 index 0000000..55f81ca --- /dev/null +++ b/internal/about/about.go @@ -0,0 +1,271 @@ +// Package about tells what an artist is, with a short text from Wikipedia. +// +// The way to the text is an identifier chain, never a search by name: the artist's +// MusicBrainz id, then the Wikidata item that MusicBrainz links to it, then the +// Wikipedia page that Wikidata names in the user's languages. Only the id is +// trusted: the page is used only if it says it is about that same item. +// +// Like the other provider data, it is asked for only when the admin switched the +// provider on and someone asks, kept in the database (30 days; a day when there was +// nothing to find), served stale when Wikipedia fails, and removed with the provider's +// data. The text is CC BY-SA 4.0, so the page, its link and the licence are kept and +// returned with it. +package about + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + "unicode/utf8" + + "go.uber.org/zap" + "golang.org/x/sync/singleflight" + + "codeberg.org/squidd/sonda/internal/discovery" + "codeberg.org/squidd/sonda/internal/providers" +) + +// Provider is the provider the descriptions come from. +const Provider = providers.Wikipedia + +// What has to be shown with the text. +const ( + Attribution = "Text from Wikipedia (https://www.wikipedia.org), available under the CC BY-SA 4.0 licence." + LicenseName = "CC BY-SA 4.0" + LicenseURL = "https://creativecommons.org/licenses/by-sa/4.0/" +) + +// How long what was read is believed. A description changes slowly; "nothing found" +// is remembered shortly, in case a link was added since. +const ( + PositiveTTL = 30 * 24 * time.Hour + NegativeTTL = 24 * time.Hour +) + +// MaxTextLen is the longest text kept, in characters. A summary is a paragraph. +const MaxTextLen = 4000 + +// fetchTimeout bounds a shared read, which does not depend on whoever started it +// still waiting. It covers up to three requests, the first of them to MusicBrainz at +// one a second. +const fetchTimeout = 45 * time.Second + +// Errors a Source returns. +var ( + // ErrSourceNotFound means the provider does not know what was asked: an unknown + // artist, item or page. It is an answer, not a failure. + ErrSourceNotFound = errors.New("the provider does not know this") + // ErrSourceUnavailable stands for any failure to get an answer. + ErrSourceUnavailable = errors.New("the provider could not be reached") +) + +// Description is what Wikipedia says about an artist, with where it comes from. +type Description struct { + Text string // the first paragraph of the article, plain text + Short string // one line, like "English rock band"; may be empty + Language string // of the page: "it", "en" + Title string // of the page + URL string // of the page, on Wikipedia +} + +// Snapshot is what is stored for one artist. Fetched is false when the provider was +// never asked; None when it was and there was nothing to find. +type Snapshot struct { + Fetched bool + None bool + RetrievedAt time.Time + Description Description +} + +// Result is the answer to Description. RetrievedAt is zero when nothing is known; Stale +// means the description is older than it should be because Wikipedia could not be asked. +type Result struct { + Status string // discovery.StatusOK, StatusDisabled, StatusError or StatusNoID + Stale bool + RetrievedAt time.Time + Description *Description // nil when there is none +} + +// Store keeps the descriptions. Artist returns discovery.ErrNotFound for an id the +// catalog does not know, and so does SaveDescription for an artist removed meanwhile. +type Store interface { + Artist(ctx context.Context, id int64) (discovery.Artist, error) + Description(ctx context.Context, id int64) (Snapshot, error) + // SaveDescription replaces what is stored; a nil description records that there is + // none. wikidataID is the item MusicBrainz linked, or "". + SaveDescription(ctx context.Context, id int64, at time.Time, wikidataID string, d *Description) error +} + +// Summary is what a Wikipedia page says about itself. +type Summary struct { + Type string + Title string + Lang string + Extract string + Description string + URL string + WikibaseItem string +} + +// Source follows the chain of identifiers. Every method returns ErrSourceNotFound for +// what the provider does not know, and any other error for a failure. +type Source interface { + // WikidataID returns the Wikidata item MusicBrainz links to the artist, or "". + WikidataID(ctx context.Context, musicbrainzID string) (string, error) + // Pages returns the title of the page of the item in each of the languages that has one. + Pages(ctx context.Context, item string, languages []string) (map[string]string, error) + Summary(ctx context.Context, language, title string) (Summary, error) +} + +// Service answers "what is this artist". +type Service struct { + store Store + source Source + settings discovery.Settings + languages []string + log *zap.Logger + now func() time.Time + flight singleflight.Group +} + +// NewService builds a Service. languages are the Wikipedia languages to try, in order. +func NewService(store Store, source Source, settings discovery.Settings, languages []string, log *zap.Logger) *Service { + if log == nil { + log = zap.NewNop() + } + return &Service{store: store, source: source, settings: settings, languages: languages, log: log, now: time.Now} +} + +// Description returns the description of an artist. A provider that is off, cannot be +// asked or fails is not an error: the answer says so and carries what was stored. +func (s *Service) Description(ctx context.Context, artistID int64) (Result, error) { + if artistID <= 0 { + return Result{}, discovery.ErrNotFound + } + artist, err := s.store.Artist(ctx, artistID) + if err != nil { + return Result{}, err + } + snap, err := s.store.Description(ctx, artistID) + if err != nil { + return Result{}, err + } + if snap.Fetched && s.fresh(snap) { + return result(snap, discovery.StatusOK, false), nil + } + if _, active, err := s.settings.Active(ctx, Provider); err != nil { + return Result{}, err + } else if !active { + return result(snap, discovery.StatusDisabled, snap.Fetched), nil + } + if artist.MBID == "" { // Sonda never looks an id up by name + return result(snap, discovery.StatusNoID, snap.Fetched), nil + } + + // One read per artist, however many ask at once, on its own context. + done := s.flight.DoChan(fmt.Sprint(artistID), func() (_ any, err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("reading the description panicked: %v", r) + } + }() + return nil, s.refresh(artist) + }) + select { + case <-ctx.Done(): + return Result{}, ctx.Err() + case r := <-done: + if errors.Is(r.Err, discovery.ErrNotFound) { + return Result{}, discovery.ErrNotFound // removed while it was being read + } + if r.Err != nil { + s.log.Warn("the description could not be read", zap.String("provider", Provider), zap.Error(r.Err)) + return result(snap, discovery.StatusError, snap.Fetched), nil + } + } + snap, err = s.store.Description(ctx, artistID) + if err != nil { + return Result{}, err + } + return result(snap, discovery.StatusOK, false), nil +} + +func (s *Service) fresh(snap Snapshot) bool { + ttl := PositiveTTL + if snap.None { + ttl = NegativeTTL + } + return s.now().Sub(snap.RetrievedAt) < ttl +} + +func result(snap Snapshot, status string, stale bool) Result { + r := Result{Status: status, Stale: stale && snap.Fetched} + if snap.Fetched { + r.RetrievedAt = snap.RetrievedAt + if !snap.None { + d := snap.Description + r.Description = &d + } + } + return r +} + +// refresh follows the chain and stores what it finds. A failure is returned and not +// stored: the next request tries again, and the language that failed is not replaced +// by the next one. +func (s *Service) refresh(a discovery.Artist) error { + ctx, cancel := context.WithTimeout(context.Background(), fetchTimeout) + defer cancel() + + item, err := s.source.WikidataID(ctx, a.MBID) + if err != nil && !errors.Is(err, ErrSourceNotFound) { + return err + } + if item == "" || err != nil { + return s.store.SaveDescription(ctx, a.ID, s.now(), "", nil) + } + pages, err := s.source.Pages(ctx, item, s.languages) + if errors.Is(err, ErrSourceNotFound) { + return s.store.SaveDescription(ctx, a.ID, s.now(), item, nil) + } + if err != nil { + return err + } + for _, lang := range s.languages { + title := pages[lang] + if title == "" { + continue + } + sum, err := s.source.Summary(ctx, lang, title) + if errors.Is(err, ErrSourceNotFound) { + continue + } + if err != nil { + return err + } + text := strings.TrimSpace(sum.Extract) + // An article about this very item, with something in it. Anything else (a page + // of disambiguation, a redirect to another subject) is not a description. + if sum.Type != "standard" || sum.WikibaseItem != item || text == "" { + continue + } + title = sum.Title + if title == "" { + title = pages[lang] + } + return s.store.SaveDescription(ctx, a.ID, s.now(), item, &Description{ + Text: cut(text), Short: sum.Description, Language: lang, Title: title, URL: sum.URL, + }) + } + return s.store.SaveDescription(ctx, a.ID, s.now(), item, nil) +} + +// cut keeps at most MaxTextLen characters and marks a text that was shortened. +func cut(s string) string { + if utf8.RuneCountInString(s) <= MaxTextLen { + return s + } + return string([]rune(s)[:MaxTextLen]) + "…" +} diff --git a/internal/about/about_test.go b/internal/about/about_test.go new file mode 100644 index 0000000..1ea78d6 --- /dev/null +++ b/internal/about/about_test.go @@ -0,0 +1,404 @@ +package about + +import ( + "context" + "errors" + "strings" + "sync" + "testing" + "time" + + "go.uber.org/zap" + + "codeberg.org/squidd/sonda/internal/discovery" + "codeberg.org/squidd/sonda/internal/providers" +) + +const ( + mbid = "11111111-2222-4333-8444-555555555555" + key = "unused" +) + +var t0 = time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC) + +type saved struct { + id int64 + at time.Time + qid string + desc *Description +} + +type fakeStore struct { + mu sync.Mutex + artists map[int64]discovery.Artist + snaps map[int64]Snapshot + saves []saved + saveErr error +} + +func newStore() *fakeStore { + return &fakeStore{ + artists: map[int64]discovery.Artist{ + 1: {ID: 1, Name: "With Id", MBID: mbid}, + 2: {ID: 2, Name: "No Id"}, + }, + snaps: map[int64]Snapshot{}, + } +} + +func (f *fakeStore) Artist(_ context.Context, id int64) (discovery.Artist, error) { + f.mu.Lock() + defer f.mu.Unlock() + a, ok := f.artists[id] + if !ok { + return discovery.Artist{}, discovery.ErrNotFound + } + return a, nil +} + +func (f *fakeStore) Description(_ context.Context, id int64) (Snapshot, error) { + f.mu.Lock() + defer f.mu.Unlock() + return f.snaps[id], nil +} + +func (f *fakeStore) SaveDescription(_ context.Context, id int64, at time.Time, qid string, d *Description) error { + f.mu.Lock() + defer f.mu.Unlock() + if f.saveErr != nil { + return f.saveErr + } + f.saves = append(f.saves, saved{id, at, qid, d}) + snap := Snapshot{Fetched: true, None: d == nil, RetrievedAt: at} + if d != nil { + snap.Description = *d + } + f.snaps[id] = snap + return nil +} + +type fakeSource struct { + mu sync.Mutex + qid string + qidErr error + pages map[string]string + pagesErr error + summaries map[string]Summary // by language + sumErrs map[string]error + calls struct{ qid, pages, summary int } + asked []string // languages asked for a summary, in order + langs []string // what Pages was asked for + mbid string + release chan struct{} + boom any +} + +func (s *fakeSource) WikidataID(_ context.Context, id string) (string, error) { + s.mu.Lock() + s.calls.qid++ + s.mbid = id + release, boom := s.release, s.boom + s.mu.Unlock() + if release != nil { + <-release + } + if boom != nil { + panic(boom) + } + return s.qid, s.qidErr +} + +func (s *fakeSource) Pages(_ context.Context, qid string, langs []string) (map[string]string, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.calls.pages++ + s.langs = append([]string(nil), langs...) + return s.pages, s.pagesErr +} + +func (s *fakeSource) Summary(_ context.Context, lang, title string) (Summary, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.calls.summary++ + s.asked = append(s.asked, lang) + if err := s.sumErrs[lang]; err != nil { + return Summary{}, err + } + return s.summaries[lang], nil +} + +func (s *fakeSource) total() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.calls.qid + s.calls.pages + s.calls.summary +} + +type on map[string]bool + +func (p on) Active(_ context.Context, name string) (providers.Settings, bool, error) { + return providers.Settings{Provider: name, Enabled: p[name]}, p[name], nil +} + +type rig struct { + svc *Service + store *fakeStore + src *fakeSource + now time.Time +} + +func standard(lang, qid, text string) Summary { + return Summary{Type: "standard", Title: "Radiohead", Lang: lang, Extract: text, Description: "una band", + URL: "https://" + lang + ".wikipedia.org/wiki/Radiohead", WikibaseItem: qid} +} + +func newRig(enabled bool) *rig { + r := &rig{store: newStore(), now: t0} + r.src = &fakeSource{ + qid: "Q44190", + pages: map[string]string{"it": "Radiohead", "en": "Radiohead"}, + summaries: map[string]Summary{ + "it": standard("it", "Q44190", "I Radiohead sono un gruppo musicale."), + "en": standard("en", "Q44190", "Radiohead are an English rock band."), + }, + sumErrs: map[string]error{}, + } + r.svc = NewService(r.store, r.src, on{providers.Wikipedia: enabled}, []string{"it", "en"}, zap.NewNop()) + r.svc.now = func() time.Time { return r.now } + return r +} + +func (r *rig) get(t *testing.T, id int64) Result { + t.Helper() + res, err := r.svc.Description(context.Background(), id) + if err != nil { + t.Fatalf("Description: %v", err) + } + return res +} + +func TestTheDescriptionComesInTheFirstLanguageThatHasAPage(t *testing.T) { + r := newRig(true) + res := r.get(t, 1) + if res.Status != discovery.StatusOK || res.Description == nil { + t.Fatalf("result = %+v", res) + } + d := res.Description + if d.Language != "it" || d.Text != "I Radiohead sono un gruppo musicale." || d.Short != "una band" || d.Title != "Radiohead" || + d.URL != "https://it.wikipedia.org/wiki/Radiohead" { + t.Errorf("description = %+v", d) + } + if !res.RetrievedAt.Equal(t0) || res.Stale { + t.Errorf("retrieved %v stale %v", res.RetrievedAt, res.Stale) + } + if r.src.mbid != mbid { + t.Errorf("asked about %q, want the MusicBrainz id", r.src.mbid) + } + if got := strings.Join(r.src.langs, ","); got != "it,en" { + t.Errorf("languages = %s, want it,en in the order of the settings", got) + } + if len(r.src.asked) != 1 { + t.Errorf("summaries read: %v, want only the first language that works", r.src.asked) + } + if len(r.store.saves) != 1 || r.store.saves[0].qid != "Q44190" { + t.Errorf("saves = %+v", r.store.saves) + } +} + +func TestTheNextLanguageIsUsedWhenThereIsNoPageOrNoUsableOne(t *testing.T) { + for name, tt := range map[string]func(*rig){ + "no page in Italian": func(r *rig) { r.src.pages = map[string]string{"en": "Radiohead"} }, + "the page is gone": func(r *rig) { r.src.sumErrs["it"] = ErrSourceNotFound }, + "a disambiguation": func(r *rig) { + s := r.src.summaries["it"] + s.Type = "disambiguation" + r.src.summaries["it"] = s + }, + "another item": func(r *rig) { + s := r.src.summaries["it"] + s.WikibaseItem = "Q1" + r.src.summaries["it"] = s + }, + "no text": func(r *rig) { + s := r.src.summaries["it"] + s.Extract = " " + r.src.summaries["it"] = s + }, + } { + r := newRig(true) + tt(r) + res := r.get(t, 1) + if res.Description == nil || res.Description.Language != "en" { + t.Errorf("%s: description = %+v, want the English one", name, res.Description) + } + } +} + +func TestNothingFoundIsRememberedAsSuchForADay(t *testing.T) { + for name, tt := range map[string]func(*rig){ + "no wikidata link": func(r *rig) { r.src.qid = "" }, + "no unknown item": func(r *rig) { r.src.pagesErr = ErrSourceNotFound }, + "no page at all": func(r *rig) { r.src.pages = map[string]string{} }, + "musicbrainz does not know the artist": func(r *rig) { r.src.qidErr = ErrSourceNotFound }, + } { + r := newRig(true) + tt(r) + res := r.get(t, 1) + if res.Status != discovery.StatusOK || res.Description != nil || res.RetrievedAt.IsZero() { + t.Errorf("%s: result = %+v, want ok with no description", name, res) + continue + } + if len(r.store.saves) != 1 || r.store.saves[0].desc != nil { + t.Errorf("%s: saves = %+v, want one 'nothing' record", name, r.store.saves) + } + calls := r.src.total() + r.now = t0.Add(NegativeTTL - time.Minute) + r.get(t, 1) + if r.src.total() != calls { + t.Errorf("%s: asked again before the day was over", name) + } + r.now = t0.Add(NegativeTTL + time.Minute) + r.get(t, 1) + if r.src.total() == calls { + t.Errorf("%s: never asked again", name) + } + } +} + +func TestADescriptionIsKeptThirtyDaysAndStaleWhenWikipediaFails(t *testing.T) { + r := newRig(true) + r.get(t, 1) + calls := r.src.total() + r.now = t0.Add(PositiveTTL - time.Hour) + if res := r.get(t, 1); res.Stale || r.src.total() != calls { + t.Fatalf("within the time: stale %v, calls %d -> %d", res.Stale, calls, r.src.total()) + } + r.now = t0.Add(PositiveTTL + time.Hour) + r.src.pagesErr = ErrSourceUnavailable + res := r.get(t, 1) + if res.Status != discovery.StatusError || !res.Stale || res.Description == nil || res.Description.Language != "it" { + t.Errorf("when Wikipedia fails: %+v, want the old description, stale, with status error", res) + } + r.src.pagesErr = nil + if res = r.get(t, 1); res.Status != discovery.StatusOK || res.Stale || !res.RetrievedAt.Equal(r.now) { + t.Errorf("after it came back: %+v", res) + } +} + +func TestAFailureIsNotRememberedAndASlowLanguageDoesNotFallThrough(t *testing.T) { + r := newRig(true) + r.src.sumErrs["it"] = ErrSourceUnavailable // Italian fails: English must not be saved in its place + res := r.get(t, 1) + if res.Status != discovery.StatusError || res.Description != nil || len(r.store.saves) != 0 { + t.Errorf("result = %+v saves = %+v: a failure must not be saved, nor replaced by the next language", res, r.store.saves) + } + if len(r.src.asked) != 1 { + t.Errorf("summaries read: %v", r.src.asked) + } +} + +func TestWithTheProviderOffNobodyIsAskedAndTheOldAnswerIsStillGiven(t *testing.T) { + r := newRig(false) + res := r.get(t, 1) + if res.Status != discovery.StatusDisabled || res.Description != nil || !res.RetrievedAt.IsZero() || res.Stale || r.src.total() != 0 { + t.Errorf("result = %+v calls %d", res, r.src.total()) + } + r.store.snaps[1] = Snapshot{Fetched: true, RetrievedAt: t0.Add(-100 * 24 * time.Hour), Description: Description{Text: "old", Language: "en", Title: "T", URL: "https://en.wikipedia.org/wiki/T"}} + res = r.get(t, 1) + if res.Status != discovery.StatusDisabled || res.Description == nil || res.Description.Text != "old" || !res.Stale || r.src.total() != 0 { + t.Errorf("with an old answer: %+v", res) + } +} + +func TestAnArtistWithoutAnIdIsNeverLookedUpByName(t *testing.T) { + r := newRig(true) + res := r.get(t, 2) + if res.Status != discovery.StatusNoID || res.Description != nil || r.src.total() != 0 || len(r.store.saves) != 0 { + t.Errorf("result = %+v calls %d saves %d", res, r.src.total(), len(r.store.saves)) + } +} + +func TestAnUnknownArtistIsNotFound(t *testing.T) { + r := newRig(true) + for _, id := range []int64{0, -1, 99} { + if _, err := r.svc.Description(context.Background(), id); !errors.Is(err, discovery.ErrNotFound) { + t.Errorf("id %d: err = %v", id, err) + } + } + if r.src.total() != 0 { + t.Error("Wikipedia was asked about an artist that does not exist") + } +} + +func TestTheTextIsCutAtTheLimit(t *testing.T) { + r := newRig(true) + long := standard("it", "Q44190", strings.Repeat("è", MaxTextLen+500)) + r.src.summaries["it"] = long + d := r.get(t, 1).Description + if n := len([]rune(d.Text)); n > MaxTextLen+1 || !strings.HasSuffix(d.Text, "…") { + t.Errorf("text has %d characters, want at most %d and a mark that it was cut", n, MaxTextLen) + } +} + +func TestManyAsksAtOnceAskWikipediaOnce(t *testing.T) { + r := newRig(true) + r.src.release = make(chan struct{}) + var wg sync.WaitGroup + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if _, err := r.svc.Description(context.Background(), 1); err != nil { + t.Errorf("err = %v", err) + } + }() + } + time.Sleep(50 * time.Millisecond) + close(r.src.release) + wg.Wait() + if r.src.calls.qid != 1 { + t.Errorf("MusicBrainz asked %d times, want 1", r.src.calls.qid) + } +} + +func TestAListenerWhoLeavesDoesNotStopTheRead(t *testing.T) { + r := newRig(true) + r.src.release = make(chan struct{}) + ctx, cancel := context.WithCancel(context.Background()) + errc := make(chan error, 1) + go func() { _, err := r.svc.Description(ctx, 1); errc <- err }() + time.Sleep(30 * time.Millisecond) + cancel() + if err := <-errc; !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v", err) + } + close(r.src.release) + deadline := time.Now().Add(3 * time.Second) + for { + r.store.mu.Lock() + n := len(r.store.saves) + r.store.mu.Unlock() + if n == 1 { + break + } + if time.Now().After(deadline) { + t.Fatal("the read was stopped when the listener left") + } + time.Sleep(10 * time.Millisecond) + } +} + +func TestAPanicInTheSourceIsAFailureNotACrash(t *testing.T) { + r := newRig(true) + r.src.boom = "boom" + if res := r.get(t, 1); res.Status != discovery.StatusError { + t.Errorf("status = %q", res.Status) + } +} + +func TestAnArtistRemovedWhileItWasReadIsNotFound(t *testing.T) { + r := newRig(true) + r.store.saveErr = discovery.ErrNotFound + if _, err := r.svc.Description(context.Background(), 1); !errors.Is(err, discovery.ErrNotFound) { + t.Errorf("err = %v", err) + } +} diff --git a/internal/about/source.go b/internal/about/source.go new file mode 100644 index 0000000..9e4c6a5 --- /dev/null +++ b/internal/about/source.go @@ -0,0 +1,51 @@ +package about + +import ( + "context" + "errors" + + "codeberg.org/squidd/sonda/internal/providers/musicbrainz" + "codeberg.org/squidd/sonda/internal/providers/wikipedia" +) + +// WikipediaSource follows the chain with the real clients: MusicBrainz for the link to +// Wikidata, Wikidata for the pages, Wikipedia for the summary. +type WikipediaSource struct { + MusicBrainz *musicbrainz.Client + Wikipedia *wikipedia.Client +} + +// notFound turns what means "there is nothing to find" into ErrSourceNotFound. An +// input the clients refuse (an id that is not one) is nothing to find either: asking +// again would not change that. +func notFound(err error) error { + if errors.Is(err, musicbrainz.ErrNotFound) || errors.Is(err, musicbrainz.ErrBadRequest) || + errors.Is(err, wikipedia.ErrNotFound) || errors.Is(err, wikipedia.ErrBadRequest) { + return ErrSourceNotFound + } + return err +} + +// WikidataID implements Source. +func (w WikipediaSource) WikidataID(ctx context.Context, musicbrainzID string) (string, error) { + id, err := w.MusicBrainz.WikidataID(ctx, musicbrainzID) + return id, notFound(err) +} + +// Pages implements Source. +func (w WikipediaSource) Pages(ctx context.Context, item string, languages []string) (map[string]string, error) { + pages, err := w.Wikipedia.Pages(ctx, item, languages) + return pages, notFound(err) +} + +// Summary implements Source. +func (w WikipediaSource) Summary(ctx context.Context, language, title string) (Summary, error) { + s, err := w.Wikipedia.Summary(ctx, language, title) + if err != nil { + return Summary{}, notFound(err) + } + return Summary{ + Type: s.Type, Title: s.Title, Lang: s.Lang, Extract: s.Extract, Description: s.Description, + URL: s.URL, WikibaseItem: s.WikibaseItem, + }, nil +} diff --git a/internal/about/source_test.go b/internal/about/source_test.go new file mode 100644 index 0000000..b31dac5 --- /dev/null +++ b/internal/about/source_test.go @@ -0,0 +1,77 @@ +package about + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "codeberg.org/squidd/sonda/internal/providers/musicbrainz" + "codeberg.org/squidd/sonda/internal/providers/outbound" + "codeberg.org/squidd/sonda/internal/providers/wikipedia" +) + +// fakeWeb answers for MusicBrainz, Wikidata and Wikipedia on one server. +func fakeWeb(t *testing.T, wikidataLink string) WikipediaSource { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasPrefix(r.URL.Path, "/ws/2/artist/"+mbid): + rels := `[]` + if wikidataLink != "" { + rels = `[{"type":"wikidata","url":{"resource":"` + wikidataLink + `"}}]` + } + _, _ = w.Write([]byte(`{"id":"` + mbid + `","relations":` + rels + `}`)) + case strings.HasPrefix(r.URL.Path, "/ws/2/artist/"): + http.NotFound(w, r) + case r.URL.Path == "/w/api.php": + _, _ = w.Write([]byte(`{"entities":{"Q44190":{"sitelinks":{"itwiki":{"title":"Radiohead"},"enwiki":{"title":"Radiohead"}}}}}`)) + case r.URL.Path == "/it/summary/Radiohead": + _, _ = w.Write([]byte(`{"type":"standard","title":"Radiohead","lang":"it","extract":"Testo.","description":"gruppo","wikibase_item":"Q44190","content_urls":{"desktop":{"page":"https://it.wikipedia.org/wiki/Radiohead"}}}`)) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + client := outbound.New(outbound.Options{UserAgent: "Sonda/test ( https://example.org )", Retries: outbound.NoRetries}) + return WikipediaSource{ + MusicBrainz: musicbrainz.New(client, srv.URL+"/ws/2/"), + Wikipedia: wikipedia.New(client, srv.URL+"/w/api.php", srv.URL+"/%s/summary/"), + } +} + +func TestTheRealClientsFollowTheChain(t *testing.T) { + src := fakeWeb(t, "https://www.wikidata.org/wiki/Q44190") + ctx := context.Background() + item, err := src.WikidataID(ctx, mbid) + if err != nil || item != "Q44190" { + t.Fatalf("item = %q err = %v", item, err) + } + pages, err := src.Pages(ctx, item, []string{"it", "en"}) + if err != nil || pages["it"] != "Radiohead" || pages["en"] != "Radiohead" { + t.Fatalf("pages = %v err = %v", pages, err) + } + sum, err := src.Summary(ctx, "it", "Radiohead") + if err != nil || sum.WikibaseItem != "Q44190" || sum.Extract != "Testo." || sum.Type != "standard" || sum.Description != "gruppo" { + t.Fatalf("summary = %+v err = %v", sum, err) + } +} + +func TestWhatThereIsNothingToFindIsNotAFailure(t *testing.T) { + ctx := context.Background() + if item, err := fakeWeb(t, "").WikidataID(ctx, mbid); err != nil || item != "" { + t.Errorf("no link: %q %v", item, err) + } + src := fakeWeb(t, "https://www.wikidata.org/wiki/Q44190") + if _, err := src.WikidataID(ctx, "22222222-2222-4333-8444-555555555555"); !errors.Is(err, ErrSourceNotFound) { + t.Errorf("unknown artist: %v", err) + } + if _, err := src.WikidataID(ctx, "not an id"); !errors.Is(err, ErrSourceNotFound) { + t.Errorf("a bad id: %v", err) + } + if _, err := src.Summary(ctx, "it", "No such page"); !errors.Is(err, ErrSourceNotFound) { + t.Errorf("no page: %v", err) + } +} diff --git a/internal/api/http/api_test.go b/internal/api/http/api_test.go index 8a45935..7e5bf23 100644 --- a/internal/api/http/api_test.go +++ b/internal/api/http/api_test.go @@ -15,6 +15,7 @@ import ( "github.com/uptrace/bun" "go.uber.org/zap" + "codeberg.org/squidd/sonda/internal/about" "codeberg.org/squidd/sonda/internal/auth" "codeberg.org/squidd/sonda/internal/discover" "codeberg.org/squidd/sonda/internal/discovery" @@ -100,6 +101,9 @@ func newTestEnvDB(t *testing.T, mutate func(*Deps, *bun.DB)) (http.Handler, *bun imageRepo := storage.NewImageRepo(db) deps.Images = images.NewService(filepath.Join(t.TempDir(), "data"), imageRepo, imageRepo) mutate(&deps, db) + if deps.About == nil { + deps.About = about.NewService(storage.NewDiscoveryRepo(db), noWikipedia{}, prov, []string{"en"}, zap.NewNop()) + } // Built from the final services, so a test that swaps one gets a Discover that uses it. if deps.Discover == nil { deps.Discover = discover.NewService(deps.Taste.(discover.Taster), deps.Discovery.(discover.Similar), @@ -109,6 +113,17 @@ func newTestEnvDB(t *testing.T, mutate func(*Deps, *bun.DB)) (http.Handler, *bun return NewRouter(deps), db } +// noWikipedia is a Wikipedia that knows nothing, for tests that do not care about it. +type noWikipedia struct{} + +func (noWikipedia) WikidataID(context.Context, string) (string, error) { return "", nil } +func (noWikipedia) Pages(context.Context, string, []string) (map[string]string, error) { + return nil, about.ErrSourceNotFound +} +func (noWikipedia) Summary(context.Context, string, string) (about.Summary, error) { + return about.Summary{}, about.ErrSourceNotFound +} + // newProviders builds the provider service the way main does, with the real way // of removing provider data. func newProviders(db *bun.DB) *providers.Service { diff --git a/internal/api/http/contract_test.go b/internal/api/http/contract_test.go index aaaa74c..ae8abb1 100644 --- a/internal/api/http/contract_test.go +++ b/internal/api/http/contract_test.go @@ -419,7 +419,7 @@ func TestResponsesMatchTheSpec(t *testing.T) { // ---- providers ---- listed := c.expect(c.call("GET", "/providers", token, nil), 200, "list providers")["providers"].([]any) - if len(listed) != 2 || listed[0].(map[string]any)["enabled"] != false || listed[1].(map[string]any)["enabled"] != false { + if len(listed) != 3 || listed[0].(map[string]any)["enabled"] != false || listed[1].(map[string]any)["enabled"] != false || listed[2].(map[string]any)["enabled"] != false { t.Fatalf("providers = %v, want two, both off", listed) } c.expect(c.call("PUT", "/providers/lastfm", token, map[string]any{"enabled": true}), 400, "enable without a key", "invalid_input") @@ -536,6 +536,17 @@ func TestResponsesMatchTheSpec(t *testing.T) { c.expect(c.call("GET", "/me/taste?limit=-1", token, nil), 400, "taste with a bad limit", "invalid_input") c.expect(c.call("GET", "/me/taste?limit=abc", token, nil), 400, "taste with a limit that is not a number", "invalid_input") + // ---- description ---- + desc := c.expect(c.call("GET", "/artists/"+artistID+"/description", token, nil), 200, "description with the provider off") + if desc["provider_status"] != "disabled" || desc["description"] != nil { + t.Fatalf("description = %v", desc) + } + c.expect(c.call("PUT", "/providers/wikipedia", token, map[string]any{"enabled": true, "contact": "https://example.org/me"}), 200, "switch Wikipedia on") + c.expect(c.call("GET", "/artists/"+artistID+"/description", token, nil), 200, "description of an artist without an id") + c.expect(c.call("GET", "/artists/999/description", token, nil), 404, "description of an unknown artist", "artist_not_found") + c.expect(c.call("GET", "/artists/abc/description", token, nil), 400, "description of a bad id", "invalid_input") + c.expect(c.call("GET", "/artists/"+artistID+"/description", "", nil), 401, "description without a token", "unauthorized") + // ---- covers ---- cover := "/albums/" + albumID + "/cover" big := c.call("GET", cover, token, nil) diff --git a/internal/api/http/description_test.go b/internal/api/http/description_test.go new file mode 100644 index 0000000..1586a9a --- /dev/null +++ b/internal/api/http/description_test.go @@ -0,0 +1,200 @@ +package httpapi + +import ( + "context" + "net/http" + "strconv" + "sync" + "testing" + "time" + + "github.com/uptrace/bun" + "go.uber.org/zap" + + "codeberg.org/squidd/sonda/internal/about" + "codeberg.org/squidd/sonda/internal/providers" + "codeberg.org/squidd/sonda/internal/storage" +) + +// fakeWikipedia is the whole chain, in one place: it counts what was asked. +type fakeWikipedia struct { + mu sync.Mutex + calls int + ids []string + item string + pages map[string]string + sums map[string]about.Summary +} + +func (f *fakeWikipedia) WikidataID(_ context.Context, id string) (string, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls++ + f.ids = append(f.ids, id) + return f.item, nil +} + +func (f *fakeWikipedia) Pages(context.Context, string, []string) (map[string]string, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls++ + return f.pages, nil +} + +func (f *fakeWikipedia) Summary(_ context.Context, lang, _ string) (about.Summary, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls++ + return f.sums[lang], nil +} + +func (f *fakeWikipedia) count() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.calls +} + +type descEnv struct { + *similarEnv + wp *fakeWikipedia +} + +func newDescEnv(t *testing.T) *descEnv { + t.Helper() + e := &descEnv{wp: &fakeWikipedia{ + item: "Q44190", + pages: map[string]string{"it": "Alpha"}, + sums: map[string]about.Summary{"it": {Type: "standard", Title: "Alpha", Lang: "it", Extract: "Alpha è un gruppo.", + Description: "gruppo musicale", URL: "https://it.wikipedia.org/wiki/Alpha", WikibaseItem: "Q44190"}}, + }} + e.similarEnv = newSimilarEnvWith(t, func(d *Deps, db *bun.DB, prov *providers.Service) { + d.About = about.NewService(storage.NewDiscoveryRepo(db), e.wp, prov, []string{"it", "en"}, zap.NewNop()) + }) + return e +} + +func (e *descEnv) path(id int64) string { + return "/api/v1/artists/" + strconv.FormatInt(id, 10) + "/description" +} + +func (e *descEnv) enable(t *testing.T) { + t.Helper() + if r := do(t, e.h, "PUT", "/api/v1/providers/wikipedia", e.admin, map[string]any{"enabled": true}); r.Code != http.StatusOK { + t.Fatalf("enable wikipedia: %d %v", r.Code, r.Body) + } +} + +func (e *descEnv) giveAlphaAnID(t *testing.T) { + t.Helper() + if _, err := e.db.Exec("UPDATE artists SET mbid = ? WHERE id = ?", alphaMBID, e.alpha); err != nil { + t.Fatal(err) + } +} + +func TestDescriptionNeedsASignedInUser(t *testing.T) { + e := newDescEnv(t) + if r := do(t, e.h, "GET", e.path(e.alpha), "", nil); r.Code != http.StatusUnauthorized { + t.Errorf("no token: %d", r.Code) + } +} + +func TestDescriptionRefusesWhatIsNotAnArtist(t *testing.T) { + e := newDescEnv(t) + e.enable(t) + for name, tt := range map[string]struct { + path string + code int + err string + }{ + "unknown artist": {e.path(99999), 404, "artist_not_found"}, + "not a number": {"/api/v1/artists/abc/description", 400, "invalid_input"}, + "zero": {"/api/v1/artists/0/description", 400, "invalid_input"}, + } { + r := do(t, e.h, "GET", tt.path, e.admin, nil) + if r.Code != tt.code || errCode(r) != tt.err { + t.Errorf("%s: %d %v, want %d %s", name, r.Code, r.Body, tt.code, tt.err) + } + } + if e.wp.count() != 0 { + t.Error("Wikipedia was asked about something that is not an artist") + } +} + +func TestDescriptionWithTheProviderOffAsksNobody(t *testing.T) { + e := newDescEnv(t) + e.giveAlphaAnID(t) + r := do(t, e.h, "GET", e.path(e.alpha), e.admin, nil) + if r.Code != http.StatusOK { + t.Fatalf("%d %v", r.Code, r.Body) + } + sameKeys(t, "description", r.Body, "provider", "provider_status", "stale", "retrieved_at", "attribution", "description") + if r.Body["provider"] != "wikipedia" || r.Body["provider_status"] != "disabled" || r.Body["stale"] != false || r.Body["attribution"] == "" { + t.Errorf("answer = %v", r.Body) + } + if v, present := r.Body["description"]; !present || v != nil { + t.Errorf("description = %#v, want an explicit null", v) + } + if v, present := r.Body["retrieved_at"]; !present || v != nil { + t.Errorf("retrieved_at = %#v, want an explicit null", v) + } + if e.wp.count() != 0 { + t.Error("a provider that is off was contacted") + } +} + +func TestDescriptionOfAnArtistWithoutAnIdSaysSo(t *testing.T) { + e := newDescEnv(t) + e.enable(t) + r := do(t, e.h, "GET", e.path(e.alpha), e.admin, nil) + if r.Code != http.StatusOK || r.Body["provider_status"] != "no_id" || r.Body["description"] != nil || e.wp.count() != 0 { + t.Errorf("%d %v, calls %d: Wikipedia must never be asked about an artist by name", r.Code, r.Body, e.wp.count()) + } +} + +func TestDescriptionIsReadStoredAndShownWithItsLicence(t *testing.T) { + e := newDescEnv(t) + e.enable(t) + e.giveAlphaAnID(t) + r := do(t, e.h, "GET", e.path(e.alpha), e.admin, nil) + if r.Code != http.StatusOK || r.Body["provider_status"] != "ok" { + t.Fatalf("%d %v", r.Code, r.Body) + } + d, _ := r.Body["description"].(map[string]any) + sameKeys(t, "description body", d, "text", "short", "language", "title", "url", "license", "license_url") + if d["text"] != "Alpha è un gruppo." || d["short"] != "gruppo musicale" || d["language"] != "it" || d["title"] != "Alpha" || + d["url"] != "https://it.wikipedia.org/wiki/Alpha" || d["license"] != "CC BY-SA 4.0" || d["license_url"] != "https://creativecommons.org/licenses/by-sa/4.0/" { + t.Errorf("description = %v", d) + } + if _, err := time.Parse(time.RFC3339, r.Body["retrieved_at"].(string)); err != nil { + t.Errorf("retrieved_at = %v", r.Body["retrieved_at"]) + } + if len(e.wp.ids) != 1 || e.wp.ids[0] != alphaMBID { + t.Errorf("asked about %v, want only the MusicBrainz id", e.wp.ids) + } + // Stored: a second look asks nobody. + calls := e.wp.count() + if r = do(t, e.h, "GET", e.path(e.alpha), e.admin, nil); r.Code != http.StatusOK || e.wp.count() != calls { + t.Errorf("second look: %d, calls %d -> %d", r.Code, calls, e.wp.count()) + } +} + +func TestPurgingWikipediaTakesTheDescriptionsBack(t *testing.T) { + e := newDescEnv(t) + e.enable(t) + e.giveAlphaAnID(t) + do(t, e.h, "GET", e.path(e.alpha), e.admin, nil) + p := do(t, e.h, "DELETE", "/api/v1/providers/wikipedia/data", e.admin, nil) + if p.Code != http.StatusOK { + t.Fatalf("purge: %d %v", p.Code, p.Body) + } + sameKeys(t, "purge", p.Body, "similar_artists", "tags", "descriptions", "artists") + if p.Body["descriptions"] != float64(1) { + t.Errorf("purge = %v, want one description", p.Body) + } + // Switched off, nothing is read again, and nothing is left to show. + do(t, e.h, "PUT", "/api/v1/providers/wikipedia", e.admin, map[string]any{"enabled": false}) + r := do(t, e.h, "GET", e.path(e.alpha), e.admin, nil) + if r.Body["description"] != nil || r.Body["provider_status"] != "disabled" { + t.Errorf("after the purge: %v", r.Body) + } +} diff --git a/internal/api/http/handlers_about.go b/internal/api/http/handlers_about.go new file mode 100644 index 0000000..a664409 --- /dev/null +++ b/internal/api/http/handlers_about.go @@ -0,0 +1,65 @@ +package httpapi + +import ( + "context" + "errors" + "net/http" + + "github.com/gin-gonic/gin" + + "codeberg.org/squidd/sonda/internal/about" + "codeberg.org/squidd/sonda/internal/discovery" +) + +type aboutHandlers struct{ svc aboutService } + +type descriptionJSON struct { + Text string `json:"text"` + Short string `json:"short"` + Language string `json:"language"` + Title string `json:"title"` + URL string `json:"url"` + License string `json:"license"` + LicenseURL string `json:"license_url"` +} + +type aboutJSON struct { + Provider string `json:"provider"` + ProviderStatus string `json:"provider_status"` + Stale bool `json:"stale"` + RetrievedAt *string `json:"retrieved_at"` + Attribution string `json:"attribution"` + Description *descriptionJSON `json:"description"` +} + +// description shows what Wikipedia says about an artist. +func (h *aboutHandlers) description(c *gin.Context) { + id, ok := pathID(c) + if !ok { + return + } + res, err := h.svc.Description(c.Request.Context(), id) + switch { + case errors.Is(err, discovery.ErrNotFound): + writeError(c, http.StatusNotFound, "artist_not_found", "not found") + return + case errors.Is(err, context.Canceled): + c.AbortWithStatus(statusClientClosed) // the client left; nobody is waiting + return + case err != nil: + _ = c.Error(err) + writeError(c, http.StatusInternalServerError, "internal", "internal error") + return + } + out := aboutJSON{Provider: about.Provider, ProviderStatus: res.Status, Stale: res.Stale, Attribution: about.Attribution} + if !res.RetrievedAt.IsZero() { + out.RetrievedAt = rfc3339(res.RetrievedAt) + } + if d := res.Description; d != nil { + out.Description = &descriptionJSON{ + Text: d.Text, Short: d.Short, Language: d.Language, Title: d.Title, URL: d.URL, + License: about.LicenseName, LicenseURL: about.LicenseURL, + } + } + c.JSON(http.StatusOK, out) +} diff --git a/internal/api/http/handlers_providers.go b/internal/api/http/handlers_providers.go index 45ba325..50fc14a 100644 --- a/internal/api/http/handlers_providers.go +++ b/internal/api/http/handlers_providers.go @@ -97,5 +97,5 @@ func (h *providersHandlers) purge(c *gin.Context) { writeProviderError(c, err) return } - c.JSON(http.StatusOK, gin.H{"similar_artists": res.SimilarArtists, "tags": res.Tags, "artists": res.Artists}) + c.JSON(http.StatusOK, gin.H{"similar_artists": res.SimilarArtists, "tags": res.Tags, "descriptions": res.Descriptions, "artists": res.Artists}) } diff --git a/internal/api/http/providers_test.go b/internal/api/http/providers_test.go index 24ae0f5..5e3b958 100644 --- a/internal/api/http/providers_test.go +++ b/internal/api/http/providers_test.go @@ -46,7 +46,7 @@ func TestProvidersStartOffAndSayWhatTheySend(t *testing.T) { t.Fatalf("list: %d %v", r.Code, r.Body) } list := r.Body["providers"].([]any) - if len(list) != 2 { + if len(list) != 3 { t.Fatalf("providers = %v", list) } for _, it := range list { @@ -263,7 +263,7 @@ func TestPurgeRemovesWhatTheProviderGaveAndNothingOfMine(t *testing.T) { if p.Code != http.StatusOK { t.Fatalf("%d %v", p.Code, p.Body) } - sameKeys(t, "purge", p.Body, "similar_artists", "tags", "artists") + sameKeys(t, "purge", p.Body, "similar_artists", "tags", "descriptions", "artists") if p.Body["similar_artists"] != float64(2) || p.Body["tags"] != float64(0) || p.Body["artists"] != float64(1) { t.Errorf("purge = %v, want 2 similar artists, no tags and the one artist Gamma", p.Body) } diff --git a/internal/api/http/router.go b/internal/api/http/router.go index ac0b47f..94084f2 100644 --- a/internal/api/http/router.go +++ b/internal/api/http/router.go @@ -10,6 +10,7 @@ import ( "github.com/gin-gonic/gin" "go.uber.org/zap" + "codeberg.org/squidd/sonda/internal/about" "codeberg.org/squidd/sonda/internal/auth" "codeberg.org/squidd/sonda/internal/catalog" "codeberg.org/squidd/sonda/internal/discover" @@ -85,6 +86,10 @@ type feedbackService interface { List(ctx context.Context, userID int64, kind feedback.Kind, p feedback.Page) (feedback.Listing, error) } +type aboutService interface { + Description(ctx context.Context, artistID int64) (about.Result, error) +} + type coverService interface { AlbumCover(ctx context.Context, albumID int64, size images.Size) (*images.Cover, error) } @@ -130,6 +135,7 @@ type Deps struct { Discover discoverService Feedback feedbackService Images coverService + About aboutService Settings userSettingsService Streams streamService } @@ -184,6 +190,8 @@ func NewRouter(d Deps) *gin.Engine { protected.GET("/artists/:id/similar", disc.similar) tg := &tagsHandlers{svc: d.Tags} protected.GET("/artists/:id/tags", tg.tags) + ab := &aboutHandlers{svc: d.About} + protected.GET("/artists/:id/description", ab.description) protected.GET("/albums", cat.listAlbums) protected.GET("/albums/:id", cat.getAlbum) protected.GET("/tracks", cat.listTracks) diff --git a/internal/api/openapi/openapi.yaml b/internal/api/openapi/openapi.yaml index 18c2bef..b87dcf5 100644 --- a/internal/api/openapi/openapi.yaml +++ b/internal/api/openapi/openapi.yaml @@ -928,6 +928,46 @@ paths: '400': {$ref: '#/components/responses/InvalidInput'} '401': {$ref: '#/components/responses/Unauthorized'} + /artists/{id}/description: + parameters: + - {$ref: '#/components/parameters/Id'} + get: + tags: [Discovery] + operationId: getArtistDescription + summary: What Wikipedia says about an artist + description: | + A short text about the artist from Wikipedia, with the page it comes from. The way to it + is a chain of identifiers, never a search by name: the artist's MusicBrainz id, then the + Wikidata item that MusicBrainz links to it, then the Wikipedia page that Wikidata names in + the first of the server's languages (`[wikipedia] languages`) that has one. A page is used + only if it is an article about that same item (not a page of disambiguation). An artist + with no MusicBrainz id answers `no_id`: Sonda does not look an id up by name. + + It needs the `wikipedia` provider, which an admin switches on; nothing is sent while it is + off (`disabled`). What is sent: the MusicBrainz id to MusicBrainz, the Wikidata item to + Wikidata, the title of the page to Wikipedia, and your server's IP address and a + `User-Agent` with the contact of the admin to all three. The answer is stored: 30 days, or a + day when there was nothing to find, and an old one is returned as `stale` if Wikipedia + fails. When there is nothing to show, `description` is `null` and `provider_status` is + `ok`. + + The text is CC BY-SA 4.0. Where you show it, name the page (`title`), link to it (`url`) and + give the licence (`license`, `license_url`), and say if you changed it. + security: [{bearerAuth: []}] + responses: + '200': + description: The description, or `null` with the status that says why. + content: + application/json: + schema: {$ref: '#/components/schemas/ArtistDescription'} + '400': {$ref: '#/components/responses/InvalidInput'} + '401': {$ref: '#/components/responses/Unauthorized'} + '404': + description: No such artist (`artist_not_found`). + content: + application/json: + schema: {$ref: '#/components/schemas/Error'} + /albums: get: tags: [Catalog] @@ -1265,7 +1305,7 @@ paths: - name: name in: path required: true - schema: {type: string, enum: [lastfm, musicbrainz]} + schema: {type: string, enum: [lastfm, musicbrainz, wikipedia]} put: tags: [Providers] operationId: updateProvider @@ -1303,7 +1343,7 @@ paths: - name: name in: path required: true - schema: {type: string, enum: [lastfm, musicbrainz]} + schema: {type: string, enum: [lastfm, musicbrainz, wikipedia]} post: tags: [Providers] operationId: testProvider @@ -1338,7 +1378,7 @@ paths: - name: name in: path required: true - schema: {type: string, enum: [lastfm, musicbrainz]} + schema: {type: string, enum: [lastfm, musicbrainz, wikipedia]} delete: tags: [Providers] operationId: purgeProviderData @@ -1799,7 +1839,7 @@ components: additionalProperties: false required: [name, enabled, needs_api_key, has_api_key, contact, sends, notes, updated_at] properties: - name: {type: string, enum: [lastfm, musicbrainz]} + name: {type: string, enum: [lastfm, musicbrainz, wikipedia]} enabled: {type: boolean, description: Whether Sonda may contact this provider.} needs_api_key: {type: boolean} has_api_key: {type: boolean, description: Whether a key is stored. The key itself is never returned.} @@ -1829,12 +1869,47 @@ components: maxLength: 200 description: A URL or e-mail address, without parentheses. Empty restores the default. + ArtistDescription: + type: object + additionalProperties: false + required: [provider, provider_status, stale, retrieved_at, attribution, description] + properties: + provider: {type: string, enum: [wikipedia]} + provider_status: + type: string + enum: [ok, disabled, error, no_id] + description: | + `ok` when the stored answer is as fresh as it needs to be (which can be "nothing found"); + `disabled` when the provider is off; `error` when Wikipedia could not be asked; `no_id` + when the artist has no MusicBrainz id. + stale: {type: boolean, description: True when the description is older than it should be because Wikipedia could not be asked.} + retrieved_at: {type: string, format: date-time, nullable: true, description: When it was read; null when nothing is known.} + attribution: {type: string, description: What to show where the text is shown.} + description: + nullable: true + allOf: + - {$ref: '#/components/schemas/Description'} + + Description: + type: object + additionalProperties: false + required: [text, short, language, title, url, license, license_url] + properties: + text: {type: string, description: "The first paragraph of the article, plain text, at most 4000 characters."} + short: {type: string, description: "One line, like \"gruppo musicale inglese\". May be empty."} + language: {type: string, description: "The language of the page, like `it`."} + title: {type: string, description: "The title of the page."} + url: {type: string, format: uri, description: "The page, on Wikipedia."} + license: {type: string, description: "The licence of the text."} + license_url: {type: string, format: uri} + ProviderPurge: type: object additionalProperties: false - required: [similar_artists, tags, artists] + required: [similar_artists, tags, descriptions, artists] properties: similar_artists: {type: integer, description: Similar-artist links removed.} + descriptions: {type: integer, description: Artist descriptions removed (only the data of Wikipedia has any).} tags: {type: integer, description: Tags removed.} artists: {type: integer, description: "Artists removed because only the removed data named them and nobody has files for them."} diff --git a/internal/config/config.go b/internal/config/config.go index 44c66b2..0d61d69 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -10,6 +10,7 @@ import ( "math" "os" "path/filepath" + "regexp" "time" "github.com/pelletier/go-toml/v2" @@ -28,6 +29,7 @@ type Config struct { Taste Taste `toml:"taste"` Scoring Scoring `toml:"scoring"` Storage Storage `toml:"storage"` + Wikipedia Wikipedia `toml:"wikipedia"` } // Server holds HTTP listener settings. @@ -105,6 +107,14 @@ type Scoring struct { WeightDiscovery float64 `toml:"weight_discovery"` // bonus for what I have not played } +// Wikipedia says in which languages to look for the description of an artist, in +// order: the first language that has a page for it wins. +type Wikipedia struct { + Languages []string `toml:"languages"` // two or three lower case letters each, at most five +} + +var wikipediaLanguageRe = regexp.MustCompile(`^[a-z]{2,3}$`) + // defaultDataDir puts the data folder next to a SQLite database, so that a service // started from another working directory finds the same place; with PostgreSQL it is // "data" in the working directory. @@ -133,6 +143,7 @@ func Default() Config { Auth: Auth{AccessTTLMinutes: 15, RefreshTTLHours: 720}, // 15 minutes, 30 days Listening: Listening{MinListenSeconds: 20}, Taste: Taste{HalfLifeDays: 90}, + Wikipedia: Wikipedia{Languages: []string{"en"}}, Scoring: Scoring{WeightAffinity: 0.40, WeightGenre: 0.25, WeightPreference: 0.25, WeightDiscovery: 0.10}, } } @@ -246,6 +257,18 @@ func (c Config) Validate() error { if sum == 0 { errs = append(errs, errors.New("scoring.weight_*: at least one weight must be above 0")) } + if n := len(c.Wikipedia.Languages); n < 1 || n > 5 { + errs = append(errs, errors.New("wikipedia.languages must hold one to five languages")) + } else { + seen := map[string]bool{} + for _, l := range c.Wikipedia.Languages { + if !wikipediaLanguageRe.MatchString(l) || seen[l] { + errs = append(errs, fmt.Errorf("wikipedia.languages: %q is not a language code of two or three lower case letters, or is repeated", l)) + break + } + seen[l] = true + } + } if c.Storage.DataDir == "" { errs = append(errs, errors.New("storage.data_dir must not be empty")) } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 92b6af5..db8ed49 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -138,6 +138,12 @@ func TestValidate(t *testing.T) { {"huge weight", func(c *Config) { c.Scoring.WeightAffinity = 11 }, "scoring.weight_affinity"}, {"all weights zero", func(c *Config) { c.Scoring = Scoring{} }, "scoring.weight"}, {"one weight is enough", func(c *Config) { c.Scoring = Scoring{WeightPreference: 1} }, ""}, + {"no wikipedia language", func(c *Config) { c.Wikipedia.Languages = nil }, "wikipedia.languages"}, + {"a bad wikipedia language", func(c *Config) { c.Wikipedia.Languages = []string{"it", "Italian"} }, "wikipedia.languages"}, + {"a wikipedia language with a hyphen", func(c *Config) { c.Wikipedia.Languages = []string{"zh-yue"} }, "wikipedia.languages"}, + {"a repeated wikipedia language", func(c *Config) { c.Wikipedia.Languages = []string{"it", "it"} }, "wikipedia.languages"}, + {"too many wikipedia languages", func(c *Config) { c.Wikipedia.Languages = []string{"aa", "bb", "cc", "dd", "ee", "ff"} }, "wikipedia.languages"}, + {"two wikipedia languages", func(c *Config) { c.Wikipedia.Languages = []string{"it", "en"} }, ""}, {"empty data dir", func(c *Config) { c.Storage.DataDir = "" }, "storage.data_dir"}, {"bad workers", func(c *Config) { c.Library.ScanWorkers = 0 }, "library.scan_workers"}, {"bad access ttl", func(c *Config) { c.Auth.AccessTTLMinutes = 0 }, "auth.access_ttl_minutes"}, @@ -221,6 +227,16 @@ func TestScoringDefaultsAndFile(t *testing.T) { } } +func TestWikipediaLanguages(t *testing.T) { + if got := Default().Wikipedia.Languages; len(got) != 1 || got[0] != "en" { + t.Errorf("default languages = %v, want [en]", got) + } + cfg, err := Load(writeFile(t, t.TempDir(), "[wikipedia]\nlanguages = [\"it\", \"en\"]\n")) + if err != nil || len(cfg.Wikipedia.Languages) != 2 || cfg.Wikipedia.Languages[0] != "it" { + t.Errorf("languages = %v err = %v", cfg.Wikipedia.Languages, err) + } +} + func TestStorageDataDirDefaultsNextToTheDatabase(t *testing.T) { for name, tt := range map[string]struct{ toml, want string }{ "default database": {"", "data"}, diff --git a/internal/providers/musicbrainz/musicbrainz.go b/internal/providers/musicbrainz/musicbrainz.go index b17afad..c99fbab 100644 --- a/internal/providers/musicbrainz/musicbrainz.go +++ b/internal/providers/musicbrainz/musicbrainz.go @@ -128,6 +128,46 @@ func (c *Client) ArtistTags(ctx context.Context, id string) ([]Tag, error) { return out, nil } +// wikidataURLRe is the address MusicBrainz stores for the Wikidata item of an artist. +// The number is bounded: it only has to be an item id, not an arbitrary string. +var wikidataURLRe = regexp.MustCompile(`^https://www\.wikidata\.org/wiki/(Q[1-9][0-9]{0,15})$`) + +// WikidataID returns the Wikidata item id (like "Q44190") that MusicBrainz links to +// the artist, or "" when there is no usable link: that is not an error. Only the +// first relation of type wikidata that has not ended and points at wikidata.org is +// used; the id comes from MusicBrainz, never from a search by name. +func (c *Client) WikidataID(ctx context.Context, id string) (string, error) { + if !idRe.MatchString(id) { + return "", ErrBadRequest + } + body, err := c.get(ctx, c.base+"artist/"+strings.ToLower(id)+"?inc=url-rels&fmt=json") + if err != nil { + return "", err + } + var doc struct { + ID string `json:"id"` + Relations []struct { + Type string `json:"type"` + Ended bool `json:"ended"` + URL struct { + Resource string `json:"resource"` + } `json:"url"` + } `json:"relations"` + } + if err := json.Unmarshal(body, &doc); err != nil || doc.ID == "" { + return "", ErrBadResponse + } + for _, r := range doc.Relations { + if r.Type != "wikidata" || r.Ended { + continue + } + if m := wikidataURLRe.FindStringSubmatch(r.URL.Resource); m != nil { + return m[1], nil + } + } + return "", nil +} + // Probe asks for one public artist, to find out whether the service answers. It // is a providers.Checker, and answers with the errors the settings understand. func (c *Client) Probe(ctx context.Context) error { diff --git a/internal/providers/musicbrainz/musicbrainz_test.go b/internal/providers/musicbrainz/musicbrainz_test.go index 3c6a592..4e4d707 100644 --- a/internal/providers/musicbrainz/musicbrainz_test.go +++ b/internal/providers/musicbrainz/musicbrainz_test.go @@ -159,3 +159,62 @@ func TestProbeSpeaksTheLanguageOfTheSettings(t *testing.T) { t.Errorf("rate limited: %v", err) } } + +const relsBody = `{"id":"11111111-2222-4333-8444-555555555555","name":"Synthetic Band", + "relations":[ + {"type":"official homepage","target-type":"url","url":{"resource":"https://example.org"}}, + {"type":"wikidata","target-type":"url","ended":false,"url":{"resource":"https://www.wikidata.org/wiki/Q44190"}}, + {"type":"discogs","target-type":"url","url":{"resource":"https://www.discogs.com/artist/1"}}]}` + +func TestWikidataIDReadsTheWikidataRelation(t *testing.T) { + s := newServer(t, 200, relsBody) + got, err := s.client().WikidataID(context.Background(), mbid) + if err != nil || got != "Q44190" { + t.Fatalf("id = %q err = %v, want Q44190", got, err) + } + reqs := s.requests() + if len(reqs) != 1 || reqs[0] != "/ws/2/artist/"+mbid+"?inc=url-rels&fmt=json" { + t.Errorf("requests = %v: only the id and the literal inc leave", reqs) + } +} + +func TestWikidataIDIsEmptyWhenThereIsNoRelationOrItIsNotUsable(t *testing.T) { + for name, body := range map[string]string{ + "no relations": `{"id":"11111111-2222-4333-8444-555555555555","relations":[]}`, + "no relations key": `{"id":"11111111-2222-4333-8444-555555555555"}`, + "another host": `{"id":"x","relations":[{"type":"wikidata","url":{"resource":"https://evil.example/wiki/Q5"}}]}`, + "not an item id": `{"id":"x","relations":[{"type":"wikidata","url":{"resource":"https://www.wikidata.org/wiki/Property:P18"}}]}`, + "a huge number": `{"id":"x","relations":[{"type":"wikidata","url":{"resource":"https://www.wikidata.org/wiki/Q99999999999999999999"}}]}`, + "a wrong type only": `{"id":"x","relations":[{"type":"wikipedia","url":{"resource":"https://en.wikipedia.org/wiki/X"}}]}`, + "an ended one": `{"id":"x","relations":[{"type":"wikidata","ended":true,"url":{"resource":"https://www.wikidata.org/wiki/Q5"}}]}`, + } { + got, err := newServer(t, 200, body).client().WikidataID(context.Background(), mbid) + if err != nil || got != "" { + t.Errorf("%s: id = %q err = %v, want nothing and no error", name, got, err) + } + } +} + +func TestWikidataIDTakesTheFirstUsableRelation(t *testing.T) { + body := `{"id":"x","relations":[ + {"type":"wikidata","ended":true,"url":{"resource":"https://www.wikidata.org/wiki/Q1"}}, + {"type":"wikidata","url":{"resource":"https://www.wikidata.org/wiki/Q2"}}, + {"type":"wikidata","url":{"resource":"https://www.wikidata.org/wiki/Q3"}}]}` + if got, _ := newServer(t, 200, body).client().WikidataID(context.Background(), mbid); got != "Q2" { + t.Errorf("id = %q, want Q2", got) + } +} + +func TestWikidataIDErrors(t *testing.T) { + if _, err := newServer(t, 200, relsBody).client().WikidataID(context.Background(), "not an id"); !errors.Is(err, ErrBadRequest) { + t.Errorf("a bad id: err = %v", err) + } + for status, want := range map[int]error{404: ErrNotFound, 429: ErrRateLimited, 503: ErrUnavailable} { + if _, err := newServer(t, status, `{"error":"x"}`).client().WikidataID(context.Background(), mbid); !errors.Is(err, want) { + t.Errorf("status %d: err = %v, want %v", status, err, want) + } + } + if _, err := newServer(t, 200, `not json`).client().WikidataID(context.Background(), mbid); !errors.Is(err, ErrBadResponse) { + t.Errorf("not json: err = %v", err) + } +} diff --git a/internal/providers/settings.go b/internal/providers/settings.go index 6233ce6..8c15530 100644 --- a/internal/providers/settings.go +++ b/internal/providers/settings.go @@ -21,6 +21,7 @@ import ( const ( LastFM = "lastfm" MusicBrainz = "musicbrainz" + Wikipedia = "wikipedia" ) // Errors returned by the service. @@ -105,6 +106,15 @@ var known = []meta{ note: "Core data (artist names and ids) is public domain. Tags and genres are CC BY-NC-SA 3.0: " + "non-commercial, with attribution to MusicBrainz.", }, + { + name: Wikipedia, + sends: "The MusicBrainz id of the artist you look up goes to MusicBrainz (never a name: Sonda does not search); " + + "the Wikidata item that MusicBrainz links to it goes to Wikidata, and the title of the Wikipedia page that " + + "Wikidata names goes to Wikipedia. Your IP address and the User-Agent, which carries the contact below, go " + + "to all three. Never your users, your listens, track titles or file paths.", + note: "Wikipedia text is CC BY-SA 4.0: show the name of the page, a link to it and the licence next to " + + "the text, and say if you changed it.", + }, } func lookup(name string) (meta, bool) { @@ -360,6 +370,7 @@ func (s *Service) UserAgentFor(name, version string) func() string { type Purged struct { SimilarArtists int // lists of similar artists Tags int // tags + Descriptions int // descriptions Artists int // artists that only the removed data named } diff --git a/internal/providers/settings_test.go b/internal/providers/settings_test.go index d58194e..11ed623 100644 --- a/internal/providers/settings_test.go +++ b/internal/providers/settings_test.go @@ -53,8 +53,8 @@ func TestEveryProviderIsOffByDefault(t *testing.T) { if err != nil { t.Fatal(err) } - if len(infos) != 2 || infos[0].Name != LastFM || infos[1].Name != MusicBrainz { - t.Fatalf("providers = %+v, want lastfm then musicbrainz", infos) + if len(infos) != 3 || infos[0].Name != LastFM || infos[1].Name != MusicBrainz || infos[2].Name != Wikipedia { + t.Fatalf("providers = %+v, want lastfm, musicbrainz, then wikipedia", infos) } for _, in := range infos { if in.Enabled || in.HasAPIKey { @@ -222,7 +222,7 @@ func TestListMergesStoredRowsInAFixedOrder(t *testing.T) { st.rows[MusicBrainz] = Settings{Provider: MusicBrainz, Enabled: true, Contact: "c", UpdatedAt: time.Unix(1, 0)} st.rows["stale-unknown"] = Settings{Provider: "stale-unknown", Enabled: true} // a row of a provider that no longer exists infos, err := s.List(context.Background()) - if err != nil || len(infos) != 2 { + if err != nil || len(infos) != 3 { t.Fatalf("infos = %+v err=%v, want exactly the known providers", infos, err) } if infos[0].Name != LastFM || infos[1].Name != MusicBrainz || !infos[1].Enabled || infos[1].Contact != "c" { @@ -408,3 +408,42 @@ func TestPurgeErrors(t *testing.T) { t.Errorf("a storage error: %v", err) } } + +func TestWikipediaIsAProviderThatNeedsNoKey(t *testing.T) { + svc, _ := newSvc() + infos, err := svc.List(context.Background()) + if err != nil { + t.Fatal(err) + } + var wp *Info + for i := range infos { + if infos[i].Name == Wikipedia { + wp = &infos[i] + } + } + if wp == nil { + t.Fatalf("providers = %+v, want wikipedia among them", infos) + } + if wp.Enabled || wp.NeedsAPIKey || wp.Sends == "" || wp.Notes == "" { + t.Errorf("wikipedia = %+v: off by default, no key, and it says what is sent and under which licence", wp) + } + // Off until an admin switches it on; no key needed to do so. + if _, active, _ := svc.Active(context.Background(), Wikipedia); active { + t.Error("active before anyone switched it on") + } + on := true + if _, err := svc.Update(context.Background(), Wikipedia, Patch{Enabled: &on}); err != nil { + t.Fatalf("enable: %v", err) + } + if _, active, _ := svc.Active(context.Background(), Wikipedia); !active { + t.Error("not active after being switched on") + } + key := "0123456789abcdef0123456789abcdef" + if _, err := svc.Update(context.Background(), Wikipedia, Patch{APIKey: &key}); !errors.Is(err, ErrInvalidSettings) { + t.Errorf("a key for a provider that has none: err = %v", err) + } + contact := "https://example.org/me" + if info, err := svc.Update(context.Background(), Wikipedia, Patch{Contact: &contact}); err != nil || info.Contact != contact { + t.Errorf("contact: %+v err=%v", info, err) + } +} diff --git a/internal/providers/wikipedia/wikipedia.go b/internal/providers/wikipedia/wikipedia.go new file mode 100644 index 0000000..3e621c1 --- /dev/null +++ b/internal/providers/wikipedia/wikipedia.go @@ -0,0 +1,249 @@ +// Package wikipedia is a small read-only client for Wikidata and Wikipedia. +// +// Sonda reaches a Wikipedia page only through an identifier it already has: the +// Wikidata item that MusicBrainz links to an artist. It never searches by name, +// because a search returns look-alikes and picking one would be a guess. What leaves +// the server is that item id, the title of the page and the contact in the +// User-Agent, which Wikimedia requires (it answers 403 without one). +// +// Two things Wikidata does that a client must expect: it answers HTTP 200 to a +// request that failed (the failure is in the body), and an unknown site in the +// filter makes it answer with every language instead of none, so the languages are +// checked before they are sent. Errors never repeat the provider's own text. +package wikipedia + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + "regexp" + "strings" + "unicode" + "unicode/utf8" + + "codeberg.org/squidd/sonda/internal/providers" + "codeberg.org/squidd/sonda/internal/providers/outbound" +) + +// Addresses in production. SummaryURL has one %s, the language. +const ( + WikidataURL = "https://www.wikidata.org/w/api.php" + SummaryURL = "https://%s.wikipedia.org/api/rest_v1/page/summary/" +) + +// Limits. +const ( + // MaxLanguages is how many languages a request may ask for. + MaxLanguages = 5 + maxTitleLen = 300 // a Wikipedia title is at most 255 bytes + maxExtractLen = 20000 + maxShortLen = 300 + // probeItem is a public item (Radiohead) used to check that the service answers. + probeItem = "Q44190" +) + +// Errors a call can return, to be matched with errors.Is. +var ( + ErrBadRequest = errors.New("wikipedia: not a usable item, language or title") + ErrNotFound = errors.New("wikipedia: no such item or page") + ErrRateLimited = errors.New("wikipedia: too many requests") + ErrUnavailable = errors.New("wikipedia: service unavailable") + ErrBadResponse = errors.New("wikipedia: unreadable answer") +) + +var ( + itemRe = regexp.MustCompile(`^Q[1-9][0-9]{0,15}$`) + langRe = regexp.MustCompile(`^[a-z]{2,3}$`) +) + +// Getter performs a GET. *outbound.Client is the one used in production. +type Getter interface { + Get(ctx context.Context, rawURL string) (*outbound.Response, error) +} + +// Client talks to Wikidata and Wikipedia. It is safe for concurrent use. +type Client struct { + http Getter + wikidata string + summary string +} + +// New builds a Client that sends its requests through g, to wikidata (WikidataURL in +// production) and to summary, a format with one %s for the language (SummaryURL). +func New(g Getter, wikidata, summary string) *Client { + return &Client{http: g, wikidata: wikidata, summary: summary} +} + +// Summary is what Wikipedia says about a page. +type Summary struct { + Type string // "standard" for an article; "disambiguation" and others are not one + Title string + Lang string + Extract string // plain text + Description string // one line, like "gruppo musicale inglese"; may be empty + URL string // the page, on that language's Wikipedia + WikibaseItem string // the Wikidata item of the page +} + +// Pages returns, for each of the languages that has one, the title of the Wikipedia +// page of a Wikidata item. A language with no page is left out; an item with no page +// at all is an empty map, not an error. +func (c *Client) Pages(ctx context.Context, item string, langs []string) (map[string]string, error) { + if !itemRe.MatchString(item) || len(langs) == 0 || len(langs) > MaxLanguages { + return nil, ErrBadRequest + } + sites := make([]string, len(langs)) + asked := map[string]bool{} + for i, l := range langs { + if !langRe.MatchString(l) || asked[l] { + return nil, ErrBadRequest + } + asked[l] = true + sites[i] = l + "wiki" + } + q := url.Values{} + q.Set("action", "wbgetentities") + q.Set("ids", item) + q.Set("props", "sitelinks") + q.Set("sitefilter", strings.Join(sites, "|")) + q.Set("maxlag", "5") // Wikimedia asks automated clients to back off when its databases lag + q.Set("format", "json") + body, err := c.get(ctx, c.wikidata+"?"+q.Encode()) + if err != nil { + return nil, err + } + var doc struct { + Error *struct { + Code string `json:"code"` + } `json:"error"` + Entities map[string]struct { + Sitelinks map[string]struct { + Title string `json:"title"` + } `json:"sitelinks"` + } `json:"entities"` + } + if err := json.Unmarshal(body, &doc); err != nil { + return nil, ErrBadResponse + } + if doc.Error != nil { // Wikidata answers 200 to a failed request + switch doc.Error.Code { + case "no-such-entity": + return nil, ErrNotFound + case "maxlag": + return nil, fmt.Errorf("%w: lagged", ErrUnavailable) + } + return nil, ErrBadResponse + } + if len(doc.Entities) != 1 { // a merged item comes under the new id: take the only one + return nil, ErrBadResponse + } + out := map[string]string{} + for _, entity := range doc.Entities { + for _, l := range langs { // only what was asked for, whatever the answer holds + if sl, ok := entity.Sitelinks[l+"wiki"]; ok { + if t := strings.TrimSpace(sl.Title); t != "" && validTitle(t) { + out[l] = t + } + } + } + } + return out, nil +} + +// Summary reads the summary of a page. ErrNotFound means the page does not exist. +func (c *Client) Summary(ctx context.Context, lang, title string) (Summary, error) { + if !langRe.MatchString(lang) || !validTitle(title) { + return Summary{}, ErrBadRequest + } + body, err := c.get(ctx, fmt.Sprintf(c.summary, lang)+url.PathEscape(strings.ReplaceAll(title, " ", "_"))) + if err != nil { + return Summary{}, err + } + var doc struct { + Type string `json:"type"` + Title string `json:"title"` + Lang string `json:"lang"` + Extract string `json:"extract"` + Description string `json:"description"` + WikibaseItem string `json:"wikibase_item"` + ContentURLs struct { + Desktop struct { + Page string `json:"page"` + } `json:"desktop"` + } `json:"content_urls"` + } + if err := json.Unmarshal(body, &doc); err != nil || strings.TrimSpace(doc.Extract) == "" { + return Summary{}, ErrBadResponse + } + // The address is shown to people as the source: it must be on that Wikipedia. + if !strings.HasPrefix(doc.ContentURLs.Desktop.Page, "https://"+lang+".wikipedia.org/wiki/") { + return Summary{}, ErrBadResponse + } + return Summary{ + Type: doc.Type, Title: doc.Title, Lang: doc.Lang, WikibaseItem: doc.WikibaseItem, + Extract: truncate(strings.TrimSpace(doc.Extract), maxExtractLen), + Description: truncate(strings.TrimSpace(doc.Description), maxShortLen), + URL: doc.ContentURLs.Desktop.Page, + }, nil +} + +// Probe asks about one public item, to find out whether the service answers. It is a +// providers.Checker, and answers with the errors the settings understand. +func (c *Client) Probe(ctx context.Context) error { + _, err := c.Pages(ctx, probeItem, []string{"en"}) + switch { + case errors.Is(err, ErrRateLimited): + return providers.ErrRateLimited + case errors.Is(err, ErrUnavailable): + return providers.ErrUnreachable + } + return err +} + +func validTitle(t string) bool { + if t == "" || utf8.RuneCountInString(t) > maxTitleLen || !utf8.ValidString(t) { + return false + } + for _, r := range t { + if unicode.IsControl(r) { + return false + } + } + return true +} + +func truncate(s string, max int) string { + if utf8.RuneCountInString(s) <= max { + return s + } + r := []rune(s) + return string(r[:max]) +} + +// get sends one request and returns the body of a successful answer. +func (c *Client) get(ctx context.Context, rawURL string) ([]byte, error) { + resp, err := c.http.Get(ctx, rawURL) + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + var limited *outbound.RateLimitedError + if errors.As(err, &limited) { + return nil, ErrRateLimited + } + return nil, fmt.Errorf("%w: %w", ErrUnavailable, err) + } + switch { + case resp.Status == 200: + return resp.Body, nil + case resp.Status == 404: + return nil, ErrNotFound + case resp.Status == 429: + return nil, ErrRateLimited + case resp.Status >= 500: + return nil, fmt.Errorf("%w: status %d", ErrUnavailable, resp.Status) + } + return nil, fmt.Errorf("%w: status %d", ErrBadResponse, resp.Status) +} diff --git a/internal/providers/wikipedia/wikipedia_test.go b/internal/providers/wikipedia/wikipedia_test.go new file mode 100644 index 0000000..9786bde --- /dev/null +++ b/internal/providers/wikipedia/wikipedia_test.go @@ -0,0 +1,249 @@ +package wikipedia + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "codeberg.org/squidd/sonda/internal/providers" + "codeberg.org/squidd/sonda/internal/providers/outbound" +) + +type server struct { + *httptest.Server + mu sync.Mutex + paths []string + status int + body string +} + +func newServer(t *testing.T, status int, body string) *server { + t.Helper() + s := &server{status: status, body: body} + s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + s.paths = append(s.paths, r.URL.RequestURI()) + status, body := s.status, s.body + s.mu.Unlock() + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.WriteHeader(status) + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(s.Close) + return s +} + +func (s *server) client() *Client { + return New(outbound.New(outbound.Options{UserAgent: "Sonda/test ( https://example.org )", Retries: outbound.NoRetries}), + s.URL+"/w/api.php", s.URL+"/%s/summary/") +} + +func (s *server) requests() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.paths...) +} + +const entityBody = `{"entities":{"Q44190":{"type":"item","id":"Q44190","sitelinks":{ + "enwiki":{"site":"enwiki","title":"Radiohead","badges":["Q17437796"]}, + "itwiki":{"site":"itwiki","title":"Radiohead","badges":[]}, + "dewiki":{"site":"dewiki","title":"Radiohead (Band)","badges":[]}}}},"success":1}` + +func TestPagesReturnsTheTitleForEachLanguageAsked(t *testing.T) { + s := newServer(t, 200, entityBody) + got, err := s.client().Pages(context.Background(), "Q44190", []string{"it", "en"}) + if err != nil { + t.Fatal(err) + } + // Only the languages asked for, even if the answer holds more. + if len(got) != 2 || got["it"] != "Radiohead" || got["en"] != "Radiohead" { + t.Errorf("pages = %v", got) + } + reqs := s.requests() + if len(reqs) != 1 || !strings.Contains(reqs[0], "ids=Q44190") || !strings.Contains(reqs[0], "sitefilter=itwiki%7Cenwiki") || + !strings.Contains(reqs[0], "props=sitelinks") || !strings.Contains(reqs[0], "maxlag=5") { + t.Errorf("request = %v", reqs) + } +} + +func TestPagesAcceptsAnEntityUnderAnotherKeyAndLeavesOutWhatIsMissing(t *testing.T) { + // A merged item comes back under the id that was asked for or the new one: take the only entity. + body := `{"entities":{"Q2":{"type":"item","id":"Q2","sitelinks":{"itwiki":{"site":"itwiki","title":"Terra"}}}},"success":1}` + got, err := newServer(t, 200, body).client().Pages(context.Background(), "Q1", []string{"it", "en"}) + if err != nil || len(got) != 1 || got["it"] != "Terra" { + t.Errorf("pages = %v err = %v", got, err) + } + // No page in any of the languages is an empty answer, not an error. + got, err = newServer(t, 200, `{"entities":{"Q1":{"type":"item","id":"Q1","sitelinks":{}}},"success":1}`).client().Pages(context.Background(), "Q1", []string{"it"}) + if err != nil || len(got) != 0 { + t.Errorf("no pages: %v err = %v", got, err) + } +} + +func TestPagesRefusesWhatCouldMakeWikidataSendEverything(t *testing.T) { + // An unknown site makes Wikidata ignore the filter and answer with every language. + s := newServer(t, 200, entityBody) + c := s.client() + for name, tt := range map[string]struct { + qid string + langs []string + }{ + "no language": {"Q44190", nil}, + "a bad language": {"Q44190", []string{"it", "zz wiki"}}, + "a capital": {"Q44190", []string{"IT"}}, + "a long code": {"Q44190", []string{"italian"}}, + "a hyphen": {"Q44190", []string{"zh-yue"}}, + "a pipe": {"Q44190", []string{"it|fr"}}, + "too many": {"Q44190", []string{"aa", "bb", "cc", "dd", "ee", "ff"}}, + "repeated": {"Q44190", []string{"it", "it"}}, + "not an item": {"P18", []string{"it"}}, + "a lower case item": {"q44190", []string{"it"}}, + "an injection": {"Q44190&action=x", []string{"it"}}, + "empty": {"", []string{"it"}}, + } { + if _, err := c.Pages(context.Background(), tt.qid, tt.langs); !errors.Is(err, ErrBadRequest) { + t.Errorf("%s: err = %v, want ErrBadRequest", name, err) + } + } + if len(s.requests()) != 0 { + t.Errorf("requests were sent for invalid input: %v", s.requests()) + } +} + +func TestPagesErrors(t *testing.T) { + langs := []string{"it"} + for name, tt := range map[string]struct { + status int + body string + want error + }{ + "missing entity (HTTP 200!)": {200, `{"error":{"code":"no-such-entity","info":"Could not find an entity with the ID \"Q9\"."}}`, ErrNotFound}, + "lagged": {200, `{"error":{"code":"maxlag","info":"Waiting for a database server: 6 seconds lagged."}}`, ErrUnavailable}, + "another error": {200, `{"error":{"code":"badtoken","info":"x"}}`, ErrBadResponse}, + "rate limited": {429, `{}`, ErrRateLimited}, + "forbidden": {403, `denied`, ErrBadResponse}, + "server error": {503, ``, ErrUnavailable}, + "not json": {200, ``, ErrBadResponse}, + "no entities": {200, `{"success":1}`, ErrBadResponse}, + } { + if _, err := newServer(t, tt.status, tt.body).client().Pages(context.Background(), "Q9", langs); !errors.Is(err, tt.want) { + t.Errorf("%s: err = %v, want %v", name, err, tt.want) + } + } +} + +const summaryBody = `{"type":"standard","title":"Radiohead","lang":"it","extract":"I Radiohead sono un gruppo musicale rock alternativo britannico formatosi a Oxford nel 1992.", + "description":"gruppo musicale inglese","wikibase_item":"Q44190", + "content_urls":{"desktop":{"page":"https://it.wikipedia.org/wiki/Radiohead"},"mobile":{"page":"https://it.m.wikipedia.org/wiki/Radiohead"}}}` + +func TestSummaryReadsTheFieldsThatAreUsed(t *testing.T) { + s := newServer(t, 200, summaryBody) + got, err := s.client().Summary(context.Background(), "it", "Radiohead") + if err != nil { + t.Fatal(err) + } + want := Summary{ + Type: "standard", Title: "Radiohead", Lang: "it", WikibaseItem: "Q44190", Description: "gruppo musicale inglese", + Extract: "I Radiohead sono un gruppo musicale rock alternativo britannico formatosi a Oxford nel 1992.", + URL: "https://it.wikipedia.org/wiki/Radiohead", + } + if got != want { + t.Errorf("summary = %+v, want %+v", got, want) + } + if reqs := s.requests(); len(reqs) != 1 || reqs[0] != "/it/summary/Radiohead" { + t.Errorf("requests = %v", reqs) + } +} + +func TestSummaryEncodesTheTitle(t *testing.T) { + for title, want := range map[string]string{ + "Guns N' Roses": "/it/summary/Guns_N%27_Roses", + "AC/DC": "/it/summary/AC%2FDC", + "Sigur Rós": "/it/summary/Sigur_R%C3%B3s", + "What? #1 100%": "/it/summary/What%3F_%231_100%25", + "Thom Yorke": "/it/summary/Thom_Yorke", + "../../api/rest_v1/other": "/it/summary/..%2F..%2Fapi%2Frest_v1%2Fother", + } { + s := newServer(t, 200, summaryBody) + if _, err := s.client().Summary(context.Background(), "it", title); err != nil { + t.Fatalf("%q: %v", title, err) + } + if reqs := s.requests(); len(reqs) != 1 || reqs[0] != want { + t.Errorf("%q: requested %v, want %s", title, reqs, want) + } + } +} + +func TestSummaryRefusesBadInput(t *testing.T) { + s := newServer(t, 200, summaryBody) + for name, tt := range map[string]struct{ lang, title string }{ + "a bad language": {"it.evil.example/", "X"}, "a capital": {"IT", "X"}, "no title": {"it", ""}, + "a long title": {"it", strings.Repeat("a", 400)}, "a control character": {"it", "A\nB"}, + } { + if _, err := s.client().Summary(context.Background(), tt.lang, tt.title); !errors.Is(err, ErrBadRequest) { + t.Errorf("%s: err = %v", name, err) + } + } + if len(s.requests()) != 0 { + t.Errorf("requests were sent for invalid input: %v", s.requests()) + } +} + +func TestSummaryKeepsTheLinkOnWikipediaOnly(t *testing.T) { + // A page address must be on that language's Wikipedia: it is shown to people as the source. + body := strings.Replace(summaryBody, "https://it.wikipedia.org/wiki/Radiohead", "https://evil.example/wiki/Radiohead", 1) + if _, err := newServer(t, 200, body).client().Summary(context.Background(), "it", "Radiohead"); !errors.Is(err, ErrBadResponse) { + t.Errorf("err = %v, want ErrBadResponse", err) + } + body = strings.Replace(summaryBody, "https://it.wikipedia.org/wiki/Radiohead", "javascript:alert(1)", 1) + if _, err := newServer(t, 200, body).client().Summary(context.Background(), "it", "Radiohead"); !errors.Is(err, ErrBadResponse) { + t.Errorf("a script address: err = %v", err) + } +} + +func TestSummaryErrors(t *testing.T) { + for name, tt := range map[string]struct { + status int + body string + want error + }{ + "no such page": {404, `{"status":404,"type":"Internal error"}`, ErrNotFound}, + "rate limited": {429, ``, ErrRateLimited}, + "forbidden": {403, `blocked`, ErrBadResponse}, + "server error": {500, ``, ErrUnavailable}, + "not json": {200, ``, ErrBadResponse}, + "no text": {200, `{"type":"standard","title":"X"}`, ErrBadResponse}, + } { + if _, err := newServer(t, tt.status, tt.body).client().Summary(context.Background(), "it", "X"); !errors.Is(err, tt.want) { + t.Errorf("%s: err = %v, want %v", name, err, tt.want) + } + } +} + +func TestErrorsDoNotCopyTheProvidersText(t *testing.T) { + secret := "SECRET-FROM-THE-PROVIDER" + _, err := newServer(t, 200, `{"error":{"code":"badtoken","info":"`+secret+`"}}`).client().Pages(context.Background(), "Q9", []string{"it"}) + if err == nil || strings.Contains(err.Error(), secret) { + t.Errorf("err = %v", err) + } + _, err = newServer(t, 500, secret).client().Summary(context.Background(), "it", "X") + if err == nil || strings.Contains(err.Error(), secret) { + t.Errorf("err = %v", err) + } +} + +func TestProbeSpeaksTheLanguageOfTheSettings(t *testing.T) { + if err := newServer(t, 200, entityBody).client().Probe(context.Background()); err != nil { + t.Errorf("ok: %v", err) + } + if err := newServer(t, 429, ``).client().Probe(context.Background()); !errors.Is(err, providers.ErrRateLimited) { + t.Errorf("429: %v", err) + } + if err := newServer(t, 503, ``).client().Probe(context.Background()); !errors.Is(err, providers.ErrUnreachable) { + t.Errorf("503: %v", err) + } +} diff --git a/internal/storage/about_test.go b/internal/storage/about_test.go new file mode 100644 index 0000000..aa36972 --- /dev/null +++ b/internal/storage/about_test.go @@ -0,0 +1,115 @@ +package storage + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/uptrace/bun" + + "codeberg.org/squidd/sonda/internal/about" + "codeberg.org/squidd/sonda/internal/discovery" + "codeberg.org/squidd/sonda/internal/providers/lastfm" +) + +func TestDescriptionSaveAndRead(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + seedCatalog(t, db) + ctx := context.Background() + repo := NewDiscoveryRepo(db) + radiohead := artistID(t, db, "Radiohead") + + snap, err := repo.Description(ctx, radiohead) + if err != nil || snap.Fetched { + t.Fatalf("never asked: %+v err=%v", snap, err) + } + d := &about.Description{Text: "Un gruppo. Con è e ö.", Short: "gruppo musicale", Language: "it", Title: "Radiohead", URL: "https://it.wikipedia.org/wiki/Radiohead"} + if err := repo.SaveDescription(ctx, radiohead, at, "Q44190", d); err != nil { + t.Fatal(err) + } + snap, err = repo.Description(ctx, radiohead) + if err != nil || !snap.Fetched || snap.None || !snap.RetrievedAt.Equal(at) || snap.Description != *d { + t.Fatalf("saved: %+v err=%v, want %+v", snap, err, *d) + } + // A new answer replaces it, including "nothing found". + if err := repo.SaveDescription(ctx, radiohead, at.Add(time.Hour), "Q44190", nil); err != nil { + t.Fatal(err) + } + snap, _ = repo.Description(ctx, radiohead) + if !snap.Fetched || !snap.None || !snap.RetrievedAt.Equal(at.Add(time.Hour)) || snap.Description != (about.Description{}) { + t.Errorf("after 'nothing': %+v", snap) + } + var qid string + if err := db.QueryRow("SELECT wikidata_id FROM artist_descriptions WHERE artist_id = ?", radiohead).Scan(&qid); err != nil || qid != "Q44190" { + t.Errorf("wikidata id = %q err=%v", qid, err) + } + // Another artist is separate. + if snap, _ := repo.Description(ctx, artistID(t, db, "Björk")); snap.Fetched { + t.Error("another artist has a description") + } + }) +} + +func TestDescriptionOfAnArtistThatWentIsNotFoundAndLeavesNothing(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + seedCatalog(t, db) + ctx := context.Background() + repo := NewDiscoveryRepo(db) + if err := repo.SaveDescription(ctx, 99999, at, "", nil); !errors.Is(err, discovery.ErrNotFound) { + t.Errorf("err = %v, want not found", err) + } + if n := count(t, db, "SELECT COUNT(*) FROM artist_descriptions"); n != 0 { + t.Errorf("%d rows for an artist that does not exist", n) + } + // Deleting an artist deletes its description (one without albums: Staind, named by a list). + if err := repo.SaveSimilar(ctx, artistID(t, db, "Radiohead"), "lastfm", at, true, []lastfm.Similar{{Name: "Staind", Match: 1}}); err != nil { + t.Fatal(err) + } + staind := artistID(t, db, "Staind") + if err := repo.SaveDescription(ctx, staind, at, "Q1", &about.Description{Text: "x", Language: "en", Title: "T", URL: "https://en.wikipedia.org/wiki/T"}); err != nil { + t.Fatal(err) + } + if _, err := db.Exec("DELETE FROM artists WHERE id = ?", staind); err != nil { + t.Fatal(err) + } + if n := count(t, db, "SELECT COUNT(*) FROM artist_descriptions"); n != 0 { + t.Errorf("%d descriptions left for a deleted artist", n) + } + }) +} + +func TestPurgingWikipediaRemovesDescriptionsAndNothingElse(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + seedCatalog(t, db) + ctx := context.Background() + repo := NewDiscoveryRepo(db) + radiohead := artistID(t, db, "Radiohead") + if err := repo.SaveSimilar(ctx, radiohead, "lastfm", at, true, []lastfm.Similar{{Name: "Staind", Match: 1}}); err != nil { + t.Fatal(err) + } + _ = repo.SaveDescription(ctx, radiohead, at, "Q44190", &about.Description{Text: "x", Language: "en", Title: "T", URL: "https://en.wikipedia.org/wiki/T"}) + _ = repo.SaveDescription(ctx, artistID(t, db, "Björk"), at, "", nil) + + // Purging another provider leaves the descriptions. + if _, err := repo.Purge(ctx, "musicbrainz"); err != nil { + t.Fatal(err) + } + if n := count(t, db, "SELECT COUNT(*) FROM artist_descriptions"); n != 2 { + t.Fatalf("a purge of MusicBrainz left %d descriptions, want 2", n) + } + out, err := repo.Purge(ctx, "wikipedia") + if err != nil { + t.Fatal(err) + } + if out.Descriptions != 2 || out.SimilarArtists != 0 || out.Artists != 0 { + t.Errorf("purged = %+v, want 2 descriptions and nothing else", out) + } + if n := count(t, db, "SELECT COUNT(*) FROM artist_descriptions"); n != 0 { + t.Errorf("%d descriptions left", n) + } + if n := count(t, db, "SELECT COUNT(*) FROM artist_similar"); n != 1 { + t.Errorf("Last.fm's list was touched: %d rows", n) + } + }) +} diff --git a/internal/storage/discovery.go b/internal/storage/discovery.go index 17183d1..07d7fa1 100644 --- a/internal/storage/discovery.go +++ b/internal/storage/discovery.go @@ -11,6 +11,7 @@ import ( "github.com/uptrace/bun" "github.com/uptrace/bun/dialect" + "codeberg.org/squidd/sonda/internal/about" "codeberg.org/squidd/sonda/internal/discovery" "codeberg.org/squidd/sonda/internal/providers" "codeberg.org/squidd/sonda/internal/providers/lastfm" @@ -419,6 +420,58 @@ func (r *DiscoveryRepo) SaveTags(ctx context.Context, id int64, provider string, // MaxFileGenres bounds the genres returned for one artist. const MaxFileGenres = 50 +// Description returns what is stored about the description of an artist. +func (r *DiscoveryRepo) Description(ctx context.Context, id int64) (about.Snapshot, error) { + var ( + snap about.Snapshot + status string + when int64 + ) + err := r.db.QueryRowContext(ctx, + "SELECT status, language, title, short, body, url, retrieved_at FROM artist_descriptions WHERE artist_id = ?", id). + Scan(&status, &snap.Description.Language, &snap.Description.Title, &snap.Description.Short, + &snap.Description.Text, &snap.Description.URL, &when) + switch { + case errors.Is(err, sql.ErrNoRows): + return about.Snapshot{}, nil + case err != nil: + return about.Snapshot{}, fmt.Errorf("read description: %w", err) + } + snap.Fetched, snap.None, snap.RetrievedAt = true, status == "none", time.Unix(when, 0) + if snap.None { + snap.Description = about.Description{} + } + return snap, nil +} + +// SaveDescription replaces the description of an artist; nil records that there is +// none. An artist removed meanwhile is discovery.ErrNotFound. +func (r *DiscoveryRepo) SaveDescription(ctx context.Context, id int64, at time.Time, wikidataID string, d *about.Description) error { + return r.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + // Artists can be removed by a purge or a cleanup: take the same turn they do. + if err := lockArtistWrites(ctx, tx); err != nil { + return err + } + if err := requireArtist(ctx, tx, id); err != nil { + return err + } + status, desc := "none", about.Description{} + if d != nil { + status, desc = "ok", *d + } + if _, err := tx.ExecContext(ctx, + `INSERT INTO artist_descriptions (artist_id, status, wikidata_id, language, title, short, body, url, retrieved_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (artist_id) DO UPDATE SET status = excluded.status, wikidata_id = excluded.wikidata_id, + language = excluded.language, title = excluded.title, short = excluded.short, body = excluded.body, + url = excluded.url, retrieved_at = excluded.retrieved_at`, + id, status, wikidataID, desc.Language, desc.Title, desc.Short, desc.Text, desc.URL, at.Unix()); err != nil { + return fmt.Errorf("save description: %w", err) + } + return nil + }) +} + // FileGenres returns the genres of the artist's own playable tracks, each // weighted by how many tracks have it, most first. It comes from the tags of the // user's files and asks nobody. An artist without files has none. @@ -487,6 +540,14 @@ func (r *DiscoveryRepo) Purge(ctx context.Context, provider string) (providers.P n, _ := res.RowsAffected() *step.into = int(n) } + if provider == providers.Wikipedia { // the descriptions are all Wikipedia's + res, err := tx.ExecContext(ctx, "DELETE FROM artist_descriptions") + if err != nil { + return fmt.Errorf("purge descriptions: %w", err) + } + n, _ := res.RowsAffected() + out.Descriptions = int(n) + } // In chunks: a provider can have named more artists than a query takes. const chunk = 500 for start := 0; start < len(named); start += chunk { diff --git a/internal/storage/storage_test.go b/internal/storage/storage_test.go index 5ff10e3..3bdb452 100644 --- a/internal/storage/storage_test.go +++ b/internal/storage/storage_test.go @@ -147,7 +147,7 @@ func TestMigrateCreatesSchemaAndIsIdempotent(t *testing.T) { if err := Migrate(context.Background(), db, fsys); err != nil { t.Fatalf("second Migrate: %v", err) } - for _, table := range []string{"artists", "albums", "tracks", "track_artists", "audio_files", "users", "listening_events", "refresh_tokens", "libraries", "provider_settings", "artist_similar", "provider_fetches", "artist_tags", "track_genres", "user_settings", "artist_feedback", "images"} { + for _, table := range []string{"artists", "albums", "tracks", "track_artists", "audio_files", "users", "listening_events", "refresh_tokens", "libraries", "provider_settings", "artist_similar", "provider_fetches", "artist_tags", "track_genres", "user_settings", "artist_feedback", "images", "artist_descriptions"} { if !tableExists(t, db, driver, table) { t.Errorf("table %s missing", table) } @@ -156,8 +156,8 @@ func TestMigrateCreatesSchemaAndIsIdempotent(t *testing.T) { if err := db.QueryRow("SELECT COUNT(*) FROM schema_migrations").Scan(&applied); err != nil { t.Fatal(err) } - if applied != 13 { - t.Errorf("applied migrations = %d, want 13", applied) + if applied != 14 { + t.Errorf("applied migrations = %d, want 14", applied) } }) } diff --git a/migrations/postgres/0014_artist_descriptions.sql b/migrations/postgres/0014_artist_descriptions.sql new file mode 100644 index 0000000..ff660c2 --- /dev/null +++ b/migrations/postgres/0014_artist_descriptions.sql @@ -0,0 +1,19 @@ +-- What Wikipedia said about an artist, kept as it said it, with the page it came from. +-- The text is CC BY-SA 4.0: the page, its link and the language are what a reader needs +-- to give credit, so they are stored with it. +-- +-- One row per artist. status 'none' records that the chain was followed and there was +-- nothing to show (no Wikidata link, or no usable page), so it is not followed again at +-- once. wikidata_id is the item MusicBrainz linked to the artist, or ''. + +CREATE TABLE artist_descriptions ( + artist_id BIGINT PRIMARY KEY REFERENCES artists(id) ON DELETE CASCADE, + status TEXT NOT NULL CHECK (status IN ('ok', 'none')), + wikidata_id TEXT NOT NULL DEFAULT '', + language TEXT NOT NULL DEFAULT '', + title TEXT NOT NULL DEFAULT '', + short TEXT NOT NULL DEFAULT '', -- one line, like "gruppo musicale inglese" + body TEXT NOT NULL DEFAULT '', -- the text + url TEXT NOT NULL DEFAULT '', + retrieved_at BIGINT NOT NULL -- unix seconds +); diff --git a/migrations/sqlite/0014_artist_descriptions.sql b/migrations/sqlite/0014_artist_descriptions.sql new file mode 100644 index 0000000..59af076 --- /dev/null +++ b/migrations/sqlite/0014_artist_descriptions.sql @@ -0,0 +1,19 @@ +-- What Wikipedia said about an artist, kept as it said it, with the page it came from. +-- The text is CC BY-SA 4.0: the page, its link and the language are what a reader needs +-- to give credit, so they are stored with it. +-- +-- One row per artist. status 'none' records that the chain was followed and there was +-- nothing to show (no Wikidata link, or no usable page), so it is not followed again at +-- once. wikidata_id is the item MusicBrainz linked to the artist, or ''. + +CREATE TABLE artist_descriptions ( + artist_id INTEGER PRIMARY KEY REFERENCES artists(id) ON DELETE CASCADE, + status TEXT NOT NULL CHECK (status IN ('ok', 'none')), + wikidata_id TEXT NOT NULL DEFAULT '', + language TEXT NOT NULL DEFAULT '', + title TEXT NOT NULL DEFAULT '', + short TEXT NOT NULL DEFAULT '', -- one line, like "gruppo musicale inglese" + body TEXT NOT NULL DEFAULT '', -- the text + url TEXT NOT NULL DEFAULT '', + retrieved_at INTEGER NOT NULL -- unix seconds +); diff --git a/sonda.example.toml b/sonda.example.toml index 8240866..823ba48 100644 --- a/sonda.example.toml +++ b/sonda.example.toml @@ -33,6 +33,9 @@ half_life_days = 90 # an old listen weighs half as much after this many # data_dir = "data" # files Sonda makes (covers); built from your music, no backup needed. # Default: "data" next to the SQLite database. +[wikipedia] +languages = ["it", "en"] # descriptions of artists: the first language that has a page wins + [scoring] # weights of the parts of a Discover score (only their proportions matter) weight_affinity = 0.40 # similar to the artists I like weight_genre = 0.25 # tags close to the tags I like