Merge pull request 'feat(about): artist descriptions from Wikipedia' (#40) from feat/artist-description into main
This commit is contained in:
commit
d0f98b0dcc
28 changed files with 2394 additions and 20 deletions
|
|
@ -17,6 +17,7 @@ It is a single executable, one database (SQLite by default, PostgreSQL if you pr
|
|||
- **Listening history.** Clients report explicit `play`, `complete` and `skip` events. An event counts as a listen when it is `complete`, or when it was played for at least 20 seconds (`listening.min_listen_seconds`); the rule is applied when the history is read, so changing it re-reads the past. You can read your own history, with a `counted` flag on every event. It stays in your database and is never sent anywhere.
|
||||
- **Your taste.** `GET /me/taste` shows what Sonda has learned from your own listening: the artists you like and the kinds of music (tags) they have in common. A complete listen counts for more than a short one, a skip counts against the artist, and older listens weigh less (they halve every 90 days, `taste.half_life_days`; only the last ten half-lives are read). It is worked out on every request from data already in your database; nothing is sent to any provider.
|
||||
- **Discover.** `GET /discover` suggests artists you may like, in your library or not, each with a score from 0 to 1 and the parts it is made of: how close it is to the artists you like (affinity), how close its tags are to yours (genre), what your listening says about it (preference) and a bonus for what you have not played (discovery). The weights are in `[scoring]` in the config. A part with no data is left out and the rest rescaled, and every answer says which seeds led to each suggestion. It starts from the artists you name with `?seed=`, or, only if you turned on `discover_from_listening`, from the artists you play most. It needs Last.fm to be switched on to ask for anything new: when Last.fm is off and a seed's similar artists are not stored yet, the answer says `disabled` (what was stored earlier is still used). See [External providers](#external-providers).
|
||||
- **Artist descriptions.** `GET /artists/{id}/description` shows what Wikipedia says about an artist: a short text, with the page it comes from, a link to it and the licence (CC BY-SA 4.0), which you must show next to it. With the `wikipedia` provider and the `musicbrainz` one switched on (the link to Wikidata comes from MusicBrainz), Sonda follows ids and never a name: the artist's MusicBrainz id, then the Wikidata item that MusicBrainz links to it, then the Wikipedia page that Wikidata names in the first of your languages (`[wikipedia] languages`, the code of that Wikipedia, for example `["it", "en"]`) that has one. An artist without a MusicBrainz id has no description. It is kept for 30 days and removed with the provider's data. See [External providers](#external-providers).
|
||||
- **Album covers.** `GET /albums/{id}/cover` serves the cover of an album from your own files: the picture embedded in them, or else a `cover`, `folder`, `front`, `album` or `albumart` image in the album's folder. It is built the first time it is asked for (a few at a time), scaled (`?size=small` is 300 pixels, `large` 1000) and kept in the data folder (`storage.data_dir`, you can delete it: covers are built again). A picture you replace in a folder is noticed within a week, or at once after a full rescan. For an album split in `CD1`/`Disc 2` folders, the picture in the folder above is used. Nothing is sent anywhere. It also takes `?access_token=`, so an `<img>` element can use it.
|
||||
- **Wishlist and "not interested".** `PUT /wishlist/{artist}` puts an artist on your wishlist (`GET /wishlist` lists it, `DELETE` takes it off), and `POST /discover/{artist}/dismiss` hides an artist from your Discover for good (`DELETE` brings it back, `GET /discover/dismissed` lists them). An artist is wished or dismissed, never both, and both lists are yours alone. Discover marks the wished artists it shows (`wishlisted`) and never suggests a dismissed one. Nothing is downloaded or sent anywhere: the wishlist is a list for you, and for a download client you may set up later. An artist that only Last.fm named is kept while you have it marked (your lists still show it, but other users cannot see that it exists), and goes with the rest of Last.fm's data if an admin removes Last.fm's data. Removing MusicBrainz's data does not touch it.
|
||||
- **Your own settings.** `GET` and `PATCH /me/settings` hold choices that belong to you, not to the server. Everyone starts private: `discover_from_listening` is off, so this setting never sends your listening to a provider. When you turn it on, Discover may start from the artists you listen to most, and their names are sent to Last.fm (only if an admin switched it on). What Last.fm answers is kept in the shared database like any other lookup, so another local user can see, from `retrieved_at` of `GET /artists/{id}/similar`, that someone looked up an artist, but not who.
|
||||
|
|
@ -66,6 +67,7 @@ Precedence is: built-in defaults, then the file, then environment variables. An
|
|||
| `auth.refresh_ttl_hours` | | `720` | Lifetime of a refresh token (30 days). |
|
||||
| `listening.min_listen_seconds` | | `20` | A play counts as a listen after this long (1 to 3600), or when it is `complete`. |
|
||||
| `taste.half_life_days` | | `90` | After this many days a listen weighs half as much when learning your taste (1 to 3650). |
|
||||
| `wikipedia.languages` | | `["en"]` | Languages for artist descriptions, best first: one to five codes of two or three lower case letters. |
|
||||
| `storage.data_dir` | `SONDA_DATA_DIR` | `data` next to the SQLite database (`data` in the working directory with PostgreSQL) | Where Sonda keeps the files it makes (covers). They are built from your music, so they need no backup. |
|
||||
| `scoring.weight_affinity`, `weight_genre`, `weight_preference`, `weight_discovery` | | `0.40`, `0.25`, `0.25`, `0.10` | The weights of the parts of a Discover score (0 to 10 each, not all zero). Only their proportions matter. |
|
||||
|
||||
|
|
@ -185,7 +187,7 @@ curl -s -X POST $BASE/users -H "$AUTH" -d '{"username":"bob","password":"another
|
|||
|
||||
## External providers
|
||||
|
||||
Sonda can ask Last.fm and MusicBrainz about artists. This is **off by default**: until an admin turns a provider on, nothing leaves your server. Today Last.fm is used for similar artists and tags, and MusicBrainz for tags.
|
||||
Sonda can ask Last.fm, MusicBrainz and Wikipedia about artists. This is **off by default**: until an admin turns a provider on, nothing leaves your server. Today Last.fm is used for similar artists and tags, MusicBrainz for tags and for the link to Wikidata, and Wikipedia (with Wikidata) for descriptions of artists.
|
||||
|
||||
- **Last.fm** needs an API key. Create one for your own use at `last.fm/api` (it needs a Last.fm account). No user name or password is involved and the "shared secret" is not needed. Last.fm data is for non-commercial use only, and you should credit Last.fm when you show it.
|
||||
- **MusicBrainz** needs no key, but it asks every client to identify itself with a contact (a URL or an e-mail address) in the `User-Agent`. By default Sonda uses the address of the project, so no personal address is sent unless you choose to.
|
||||
|
|
@ -202,7 +204,7 @@ curl -s -X PUT $BASE/providers/musicbrainz -H "$AUTH" -d '{"enabled":true,"conta
|
|||
|
||||
The key is stored in the database, in clear text, and is never returned by the API or written to the log: the list only says `has_api_key`. Because the database holds it, treat database backups like secrets. Removing the key turns the provider off, and each provider describes in the list exactly what it sends.
|
||||
|
||||
**What leaves your server, when a provider is on:** to Last.fm, the artist name (or MusicBrainz id) being looked up, your API key, your IP address and the `User-Agent`; to MusicBrainz, only the MusicBrainz id of the artist, your IP address and a `User-Agent` that carries your contact. Never your users, your listens, track titles or file paths. Requests only happen when someone asks for them, never as a bulk pass over your library, and they are spaced out, retried with backoff, and stopped by a circuit breaker if a provider is down. **Discover** asks Last.fm for the similar artists of each seed (the seed's name or MusicBrainz id) and, for the best suggestions that have no tags yet, for the tags of that suggestion (its name); never MusicBrainz. Seeds you name are sent when you ask. Seeds taken from your listening, which are the names of the artists you play most, are sent only if you turned on `discover_from_listening` yourself; otherwise Discover refuses to run without a `seed`. The one exception to "only when someone asks" is `POST /providers/{name}/test`, which an admin sends to check a key; it sends nothing but the key.
|
||||
**What leaves your server, when a provider is on:** to Last.fm, the artist name (or MusicBrainz id) being looked up, your API key, your IP address and the `User-Agent`; to MusicBrainz, only the MusicBrainz id of the artist, your IP address and a `User-Agent` that carries your contact. Never your users, your listens, track titles or file paths. Requests only happen when someone asks for them, never as a bulk pass over your library, and they are spaced out, retried with backoff, and stopped by a circuit breaker if a provider is down. **Descriptions** (the Wikipedia provider, which also needs MusicBrainz on) send the MusicBrainz id to MusicBrainz, the Wikidata item to Wikidata and the title of the page to Wikipedia, with your IP address and a `User-Agent` that carries a contact (the one set for each provider; Wikimedia refuses requests without one), and never a name. **Discover** asks Last.fm for the similar artists of each seed (the seed's name or MusicBrainz id) and, for the best suggestions that have no tags yet, for the tags of that suggestion (its name); never MusicBrainz. Seeds you name are sent when you ask. Seeds taken from your listening, which are the names of the artists you play most, are sent only if you turned on `discover_from_listening` yourself; otherwise Discover refuses to run without a `seed`. The one exception to "only when someone asks" is `POST /providers/{name}/test`, which an admin sends to check a key; it sends nothing but the key.
|
||||
|
||||
**Similar artists.** `GET /api/v1/artists/{id}/similar` asks Last.fm for the artists similar to an artist (one of yours, or one Last.fm named earlier) and keeps the answer in your database: 7 days, or 1 day when Last.fm does not know the artist. Until then a second look costs nothing and sends nothing, and if Last.fm is down you still get the last answer, marked `stale`. If the provider is off the answer says `disabled` instead of failing. Each similar artist says whether you have it (`in_library`); the ones you do not have are stored as artists without files. They can be opened with `GET /artists/{id}` but are never part of your artist lists, and if you later add their music, the scan finds them there. Last.fm's `score` is relative to each answer (the best one is usually 1), so compare artists only inside one answer.
|
||||
|
||||
|
|
@ -230,6 +232,7 @@ Everything lives under `/api/v1`. Bodies are JSON. Errors always look like `{"er
|
|||
| signed in | `GET /auth/me`, `POST /auth/password` | Your account; change your password (signs you out everywhere). |
|
||||
| signed in | `GET /artists`, `/albums`, `/tracks` and `/…/{id}` | Browse and search. Query: `q`, `limit` (default 50, max 200), `offset`, `library_id`, `artist_id`, `album_id`. |
|
||||
| signed in | `GET /artists/{id}/tags` | What Last.fm and MusicBrainz say about the artist's kind of music, from each provider that is on. |
|
||||
| signed in | `GET /artists/{id}/description` | What Wikipedia says about the artist, with its page and licence, from the Wikipedia provider when an admin switched it on. |
|
||||
| signed in | `GET /artists/{id}/similar` | Artists similar to one, from Last.fm when an admin switched it on. Query: `limit` (default 20, max 50). |
|
||||
| signed in | `GET`, `HEAD /tracks/{id}/stream` | Stream the original file. Also accepts `?access_token=`. |
|
||||
| signed in | `POST /listens` | Record a `play`, `complete` or `skip`; the answer says whether it counts as a listen. |
|
||||
|
|
@ -290,6 +293,7 @@ internal/api/openapi/ the OpenAPI spec, embedded in the binary
|
|||
internal/auth/ users, roles, passwords, access and refresh tokens
|
||||
internal/catalog/ browsing and searching artists, albums, tracks
|
||||
internal/library/ libraries, the scanner, the event hub, the tag reader
|
||||
internal/about/ descriptions of artists, from Wikipedia through MusicBrainz and Wikidata
|
||||
internal/discovery/ similar artists: what the providers said, kept and refreshed
|
||||
internal/providers/ external providers: settings, the outbound HTTP client, the Last.fm and MusicBrainz clients
|
||||
internal/listening/ listen events
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import (
|
|||
"github.com/gin-gonic/gin"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"codeberg.org/squidd/sonda/internal/about"
|
||||
httpapi "codeberg.org/squidd/sonda/internal/api/http"
|
||||
"codeberg.org/squidd/sonda/internal/auth"
|
||||
"codeberg.org/squidd/sonda/internal/catalog"
|
||||
|
|
@ -28,6 +29,7 @@ import (
|
|||
"codeberg.org/squidd/sonda/internal/providers/lastfm"
|
||||
"codeberg.org/squidd/sonda/internal/providers/musicbrainz"
|
||||
"codeberg.org/squidd/sonda/internal/providers/outbound"
|
||||
"codeberg.org/squidd/sonda/internal/providers/wikipedia"
|
||||
"codeberg.org/squidd/sonda/internal/storage"
|
||||
"codeberg.org/squidd/sonda/internal/streaming"
|
||||
"codeberg.org/squidd/sonda/internal/taste"
|
||||
|
|
@ -46,6 +48,13 @@ const lastfmMinInterval = 250 * time.Millisecond
|
|||
// musicbrainzMinInterval keeps a little above MusicBrainz's rule of one request a second.
|
||||
const musicbrainzMinInterval = 1100 * time.Millisecond
|
||||
|
||||
// wikipediaMinInterval is polite: Wikimedia asks only for serial requests, and a
|
||||
// description costs two of them. wikipediaMaxBody is far above any answer used here.
|
||||
const (
|
||||
wikipediaMinInterval = 250 * time.Millisecond
|
||||
wikipediaMaxBody = 1 << 20
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := run(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "sondad:", err)
|
||||
|
|
@ -132,6 +141,16 @@ func run() error {
|
|||
providerSettings.SetChecker(providers.MusicBrainz, func(ctx context.Context, _ string) error {
|
||||
return musicbrainzClient.Probe(ctx)
|
||||
})
|
||||
// Wikipedia and Wikidata: asked only for the description of an artist, through ids.
|
||||
// Wikimedia answers 403 to a request without a User-Agent, and wants a contact in it.
|
||||
wikipediaClient := wikipedia.New(outbound.New(outbound.Options{
|
||||
UserAgentFunc: providerSettings.UserAgentFor(providers.Wikipedia, version),
|
||||
MinInterval: wikipediaMinInterval,
|
||||
MaxBody: wikipediaMaxBody,
|
||||
}), wikipedia.WikidataURL, wikipedia.SummaryURL)
|
||||
providerSettings.SetChecker(providers.Wikipedia, func(ctx context.Context, _ string) error {
|
||||
return wikipediaClient.Probe(ctx)
|
||||
})
|
||||
discoveryRepo := storage.NewDiscoveryRepo(db)
|
||||
providerSettings.SetPurger(func(ctx context.Context, name string) (providers.Purged, error) {
|
||||
return discoveryRepo.Purge(ctx, name)
|
||||
|
|
@ -170,8 +189,10 @@ func run() error {
|
|||
Discover: discover.NewService(tasteService, discoveries, tags, tasteRepo, userSettings, feedbacks, weights),
|
||||
Feedback: feedbacks,
|
||||
Images: images.NewService(cfg.Storage.DataDir, imageRepo, imageRepo),
|
||||
Settings: userSettings,
|
||||
Streams: streaming.NewService(libraryRepo),
|
||||
About: about.NewService(discoveryRepo, about.WikipediaSource{MusicBrainz: musicbrainzClient, Wikipedia: wikipediaClient},
|
||||
providerSettings, cfg.Wikipedia.Languages, logger),
|
||||
Settings: userSettings,
|
||||
Streams: streaming.NewService(libraryRepo),
|
||||
})
|
||||
|
||||
// No WriteTimeout: audio streaming (later) needs long responses.
|
||||
|
|
|
|||
306
internal/about/about.go
Normal file
306
internal/about/about.go
Normal file
|
|
@ -0,0 +1,306 @@
|
|||
// Package about tells what an artist is, with a short text from Wikipedia.
|
||||
//
|
||||
// The way to the text is an identifier chain, never a search by name: the artist's
|
||||
// MusicBrainz id, then the Wikidata item that MusicBrainz links to it, then the
|
||||
// Wikipedia page that Wikidata names in the user's languages. Only the id is
|
||||
// trusted: the page is used only if it says it is about that same item.
|
||||
//
|
||||
// Like the other provider data, it is asked for only when the admin switched the
|
||||
// provider on and someone asks, kept in the database (30 days; a day when there was
|
||||
// nothing to find), served stale when Wikipedia fails, and removed with the provider's
|
||||
// data. The text is CC BY-SA 4.0, so the page, its link and the licence are kept and
|
||||
// returned with it.
|
||||
package about
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"go.uber.org/zap"
|
||||
"golang.org/x/sync/singleflight"
|
||||
|
||||
"codeberg.org/squidd/sonda/internal/discovery"
|
||||
"codeberg.org/squidd/sonda/internal/providers"
|
||||
)
|
||||
|
||||
// Provider is the provider the descriptions come from.
|
||||
const Provider = providers.Wikipedia
|
||||
|
||||
// What has to be shown with the text.
|
||||
const (
|
||||
Attribution = "Text from Wikipedia (https://www.wikipedia.org), available under the CC BY-SA 4.0 licence."
|
||||
LicenseName = "CC BY-SA 4.0"
|
||||
LicenseURL = "https://creativecommons.org/licenses/by-sa/4.0/"
|
||||
)
|
||||
|
||||
// How long what was read is believed. A description changes slowly; "nothing found"
|
||||
// is remembered shortly, in case a link was added since.
|
||||
const (
|
||||
PositiveTTL = 30 * 24 * time.Hour
|
||||
NegativeTTL = 24 * time.Hour
|
||||
)
|
||||
|
||||
// MaxTextLen is the longest text kept, in characters. A summary is a paragraph.
|
||||
const MaxTextLen = 4000
|
||||
|
||||
// fetchTimeout bounds a shared read, which does not depend on whoever started it
|
||||
// still waiting. It covers up to three requests, the first of them to MusicBrainz at
|
||||
// one a second.
|
||||
const fetchTimeout = 45 * time.Second
|
||||
|
||||
// Errors a Source returns.
|
||||
var (
|
||||
// ErrSourceNotFound means the provider does not know what was asked: an unknown
|
||||
// artist, item or page. It is an answer, not a failure.
|
||||
ErrSourceNotFound = errors.New("the provider does not know this")
|
||||
// ErrSourceUnavailable stands for any failure to get an answer.
|
||||
ErrSourceUnavailable = errors.New("the provider could not be reached")
|
||||
)
|
||||
|
||||
// Description is what Wikipedia says about an artist, with where it comes from.
|
||||
type Description struct {
|
||||
Text string // the first paragraph of the article, plain text
|
||||
Short string // one line, like "English rock band"; may be empty
|
||||
Language string // of the page: "it", "en"
|
||||
Title string // of the page
|
||||
URL string // of the page, on Wikipedia
|
||||
}
|
||||
|
||||
// Snapshot is what is stored for one artist. Fetched is false when the provider was
|
||||
// never asked; None when it was and there was nothing to find.
|
||||
type Snapshot struct {
|
||||
Fetched bool
|
||||
None bool
|
||||
RetrievedAt time.Time
|
||||
Description Description
|
||||
}
|
||||
|
||||
// Result is the answer to Description. RetrievedAt is zero when nothing is known; Stale
|
||||
// means the description is older than it should be because Wikipedia could not be asked.
|
||||
type Result struct {
|
||||
Status string // discovery.StatusOK, StatusDisabled, StatusError or StatusNoID
|
||||
Stale bool
|
||||
RetrievedAt time.Time
|
||||
Description *Description // nil when there is none
|
||||
}
|
||||
|
||||
// Store keeps the descriptions. Artist returns discovery.ErrNotFound for an id the
|
||||
// catalog does not know, and so does SaveDescription for an artist removed meanwhile.
|
||||
type Store interface {
|
||||
Artist(ctx context.Context, id int64) (discovery.Artist, error)
|
||||
Description(ctx context.Context, id int64) (Snapshot, error)
|
||||
// SaveDescription replaces what is stored; a nil description records that there is
|
||||
// none. wikidataID is the item MusicBrainz linked, or "".
|
||||
SaveDescription(ctx context.Context, id int64, at time.Time, wikidataID string, d *Description) error
|
||||
}
|
||||
|
||||
// Summary is what a Wikipedia page says about itself.
|
||||
type Summary struct {
|
||||
Type string
|
||||
Title string
|
||||
Lang string
|
||||
Extract string
|
||||
Description string
|
||||
URL string
|
||||
WikibaseItem string
|
||||
}
|
||||
|
||||
// Source follows the chain of identifiers. Every method returns ErrSourceNotFound for
|
||||
// what the provider does not know, and any other error for a failure.
|
||||
type Source interface {
|
||||
// WikidataID returns the Wikidata item MusicBrainz links to the artist, or "".
|
||||
WikidataID(ctx context.Context, musicbrainzID string) (string, error)
|
||||
// Pages returns the title of the page of the item in each of the languages that has
|
||||
// one, and the item they belong to: the same, or the one it was merged into.
|
||||
Pages(ctx context.Context, item string, languages []string) (string, map[string]string, error)
|
||||
Summary(ctx context.Context, language, title string) (Summary, error)
|
||||
}
|
||||
|
||||
// Service answers "what is this artist".
|
||||
type Service struct {
|
||||
store Store
|
||||
source Source
|
||||
settings discovery.Settings
|
||||
languages []string
|
||||
log *zap.Logger
|
||||
now func() time.Time
|
||||
flight singleflight.Group
|
||||
}
|
||||
|
||||
// NewService builds a Service. languages are the Wikipedia languages to try, in order.
|
||||
func NewService(store Store, source Source, settings discovery.Settings, languages []string, log *zap.Logger) *Service {
|
||||
if log == nil {
|
||||
log = zap.NewNop()
|
||||
}
|
||||
return &Service{store: store, source: source, settings: settings, languages: languages, log: log, now: time.Now}
|
||||
}
|
||||
|
||||
// Description returns the description of an artist. A provider that is off, cannot be
|
||||
// asked or fails is not an error: the answer says so and carries what was stored.
|
||||
func (s *Service) Description(ctx context.Context, artistID int64) (Result, error) {
|
||||
if artistID <= 0 {
|
||||
return Result{}, discovery.ErrNotFound
|
||||
}
|
||||
artist, err := s.store.Artist(ctx, artistID)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
snap, err := s.store.Description(ctx, artistID)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
fresh := snap.Fetched && s.fresh(snap)
|
||||
active, err := s.active(ctx)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
switch {
|
||||
case !active:
|
||||
return result(snap, discovery.StatusDisabled, snap.Fetched && !fresh), nil
|
||||
case fresh:
|
||||
return result(snap, discovery.StatusOK, false), nil
|
||||
}
|
||||
if artist.MBID == "" { // Sonda never looks an id up by name
|
||||
return result(snap, discovery.StatusNoID, snap.Fetched), nil
|
||||
}
|
||||
|
||||
// One read per artist, however many ask at once, on its own context.
|
||||
done := s.flight.DoChan(fmt.Sprint(artistID), func() (_ any, err error) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
err = fmt.Errorf("reading the description panicked: %v", r)
|
||||
}
|
||||
}()
|
||||
return nil, s.refresh(artist)
|
||||
})
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return Result{}, ctx.Err()
|
||||
case r := <-done:
|
||||
if errors.Is(r.Err, discovery.ErrNotFound) {
|
||||
return Result{}, discovery.ErrNotFound // removed while it was being read
|
||||
}
|
||||
if errors.Is(r.Err, errSwitchedOff) {
|
||||
return result(snap, discovery.StatusDisabled, snap.Fetched), nil
|
||||
}
|
||||
if r.Err != nil {
|
||||
s.log.Warn("the description could not be read", zap.String("provider", Provider), zap.Error(r.Err))
|
||||
return result(snap, discovery.StatusError, snap.Fetched), nil
|
||||
}
|
||||
}
|
||||
snap, err = s.store.Description(ctx, artistID)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
return result(snap, discovery.StatusOK, false), nil
|
||||
}
|
||||
|
||||
// errSwitchedOff is what a read returns when the providers were switched off while it ran.
|
||||
var errSwitchedOff = errors.New("the provider was switched off")
|
||||
|
||||
// active says whether the chain may be followed: Wikipedia is on, and so is MusicBrainz,
|
||||
// which gives the link to Wikidata. Without that second switch nothing would stop the
|
||||
// artist's id going to a provider the admin never allowed.
|
||||
func (s *Service) active(ctx context.Context) (bool, error) {
|
||||
for _, name := range []string{Provider, providers.MusicBrainz} {
|
||||
_, on, err := s.settings.Active(ctx, name)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !on {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (s *Service) fresh(snap Snapshot) bool {
|
||||
ttl := PositiveTTL
|
||||
if snap.None {
|
||||
ttl = NegativeTTL
|
||||
}
|
||||
return s.now().Sub(snap.RetrievedAt) < ttl
|
||||
}
|
||||
|
||||
func result(snap Snapshot, status string, stale bool) Result {
|
||||
r := Result{Status: status, Stale: stale && snap.Fetched}
|
||||
if snap.Fetched {
|
||||
r.RetrievedAt = snap.RetrievedAt
|
||||
if !snap.None {
|
||||
d := snap.Description
|
||||
r.Description = &d
|
||||
}
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// refresh follows the chain and stores what it finds. A failure is returned and not
|
||||
// stored: the next request tries again, and the language that failed is not replaced
|
||||
// by the next one.
|
||||
func (s *Service) refresh(a discovery.Artist) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), fetchTimeout)
|
||||
defer cancel()
|
||||
|
||||
item, err := s.source.WikidataID(ctx, a.MBID)
|
||||
if err != nil && !errors.Is(err, ErrSourceNotFound) {
|
||||
return err
|
||||
}
|
||||
if item == "" || err != nil {
|
||||
return s.save(ctx, a.ID, "", nil)
|
||||
}
|
||||
resolved, pages, err := s.source.Pages(ctx, item, s.languages)
|
||||
if errors.Is(err, ErrSourceNotFound) {
|
||||
return s.save(ctx, a.ID, item, nil)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, lang := range s.languages {
|
||||
title := pages[lang]
|
||||
if title == "" {
|
||||
continue
|
||||
}
|
||||
sum, err := s.source.Summary(ctx, lang, title)
|
||||
if errors.Is(err, ErrSourceNotFound) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
text := strings.TrimSpace(sum.Extract)
|
||||
// An article about this very item (the one the pages belong to, which is the
|
||||
// target when the item was merged), with some text, and an address to credit.
|
||||
// Anything else (a page of disambiguation, a redirect to another subject, a page
|
||||
// with no text) is not a description.
|
||||
if sum.Type != "standard" || sum.WikibaseItem != resolved || text == "" || sum.URL == "" {
|
||||
continue
|
||||
}
|
||||
return s.save(ctx, a.ID, resolved, &Description{
|
||||
Text: cut(text), Short: sum.Description, Language: lang, Title: title, URL: sum.URL,
|
||||
})
|
||||
}
|
||||
return s.save(ctx, a.ID, resolved, nil)
|
||||
}
|
||||
|
||||
// save stores the outcome, unless the providers were switched off since the read began:
|
||||
// an admin who took the data back and switched it off must not see it come back.
|
||||
func (s *Service) save(ctx context.Context, id int64, item string, d *Description) error {
|
||||
if on, err := s.active(ctx); err != nil {
|
||||
return err
|
||||
} else if !on {
|
||||
return errSwitchedOff
|
||||
}
|
||||
return s.store.SaveDescription(ctx, id, s.now(), item, d)
|
||||
}
|
||||
|
||||
// cut keeps at most MaxTextLen characters and marks a text that was shortened.
|
||||
func cut(s string) string {
|
||||
if utf8.RuneCountInString(s) <= MaxTextLen {
|
||||
return s
|
||||
}
|
||||
return string([]rune(s)[:MaxTextLen-1]) + "…" // the mark counts: never more than MaxTextLen
|
||||
}
|
||||
513
internal/about/about_test.go
Normal file
513
internal/about/about_test.go
Normal file
|
|
@ -0,0 +1,513 @@
|
|||
package about
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"codeberg.org/squidd/sonda/internal/discovery"
|
||||
"codeberg.org/squidd/sonda/internal/providers"
|
||||
)
|
||||
|
||||
const (
|
||||
mbid = "11111111-2222-4333-8444-555555555555"
|
||||
key = "unused"
|
||||
)
|
||||
|
||||
var t0 = time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
type saved struct {
|
||||
id int64
|
||||
at time.Time
|
||||
qid string
|
||||
desc *Description
|
||||
}
|
||||
|
||||
type fakeStore struct {
|
||||
mu sync.Mutex
|
||||
artists map[int64]discovery.Artist
|
||||
snaps map[int64]Snapshot
|
||||
saves []saved
|
||||
saveErr error
|
||||
}
|
||||
|
||||
func newStore() *fakeStore {
|
||||
return &fakeStore{
|
||||
artists: map[int64]discovery.Artist{
|
||||
1: {ID: 1, Name: "With Id", MBID: mbid},
|
||||
2: {ID: 2, Name: "No Id"},
|
||||
},
|
||||
snaps: map[int64]Snapshot{},
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeStore) Artist(_ context.Context, id int64) (discovery.Artist, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
a, ok := f.artists[id]
|
||||
if !ok {
|
||||
return discovery.Artist{}, discovery.ErrNotFound
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) Description(_ context.Context, id int64) (Snapshot, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.snaps[id], nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) SaveDescription(_ context.Context, id int64, at time.Time, qid string, d *Description) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.saveErr != nil {
|
||||
return f.saveErr
|
||||
}
|
||||
f.saves = append(f.saves, saved{id, at, qid, d})
|
||||
snap := Snapshot{Fetched: true, None: d == nil, RetrievedAt: at}
|
||||
if d != nil {
|
||||
snap.Description = *d
|
||||
}
|
||||
f.snaps[id] = snap
|
||||
return nil
|
||||
}
|
||||
|
||||
type fakeSource struct {
|
||||
mu sync.Mutex
|
||||
qid string
|
||||
resolved string // the item the pages belong to; empty means the same as qid
|
||||
qidErr error
|
||||
pages map[string]string
|
||||
pagesErr error
|
||||
summaries map[string]Summary // by language
|
||||
sumErrs map[string]error
|
||||
calls struct{ qid, pages, summary int }
|
||||
asked []string // languages asked for a summary, in order
|
||||
langs []string // what Pages was asked for
|
||||
mbid string
|
||||
release chan struct{}
|
||||
boom any
|
||||
}
|
||||
|
||||
func (s *fakeSource) WikidataID(_ context.Context, id string) (string, error) {
|
||||
s.mu.Lock()
|
||||
s.calls.qid++
|
||||
s.mbid = id
|
||||
release, boom := s.release, s.boom
|
||||
s.mu.Unlock()
|
||||
if release != nil {
|
||||
<-release
|
||||
}
|
||||
if boom != nil {
|
||||
panic(boom)
|
||||
}
|
||||
return s.qid, s.qidErr
|
||||
}
|
||||
|
||||
func (s *fakeSource) Pages(_ context.Context, qid string, langs []string) (string, map[string]string, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.calls.pages++
|
||||
s.langs = append([]string(nil), langs...)
|
||||
resolved := s.resolved
|
||||
if resolved == "" {
|
||||
resolved = qid
|
||||
}
|
||||
return resolved, s.pages, s.pagesErr
|
||||
}
|
||||
|
||||
func (s *fakeSource) Summary(_ context.Context, lang, title string) (Summary, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.calls.summary++
|
||||
s.asked = append(s.asked, lang)
|
||||
if err := s.sumErrs[lang]; err != nil {
|
||||
return Summary{}, err
|
||||
}
|
||||
return s.summaries[lang], nil
|
||||
}
|
||||
|
||||
func (s *fakeSource) total() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.calls.qid + s.calls.pages + s.calls.summary
|
||||
}
|
||||
|
||||
// switches says which providers are on, and can be changed while a read is running.
|
||||
type switches struct {
|
||||
mu sync.Mutex
|
||||
m map[string]bool
|
||||
}
|
||||
|
||||
func (p *switches) Active(_ context.Context, name string) (providers.Settings, bool, error) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return providers.Settings{Provider: name, Enabled: p.m[name]}, p.m[name], nil
|
||||
}
|
||||
|
||||
func (p *switches) set(name string, on bool) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.m[name] = on
|
||||
}
|
||||
|
||||
type rig struct {
|
||||
sw *switches
|
||||
svc *Service
|
||||
store *fakeStore
|
||||
src *fakeSource
|
||||
now time.Time
|
||||
}
|
||||
|
||||
func standard(lang, qid, text string) Summary {
|
||||
return Summary{Type: "standard", Title: "Radiohead", Lang: lang, Extract: text, Description: "una band",
|
||||
URL: "https://" + lang + ".wikipedia.org/wiki/Radiohead", WikibaseItem: qid}
|
||||
}
|
||||
|
||||
func newRig(enabled bool) *rig {
|
||||
r := &rig{store: newStore(), now: t0}
|
||||
r.src = &fakeSource{
|
||||
qid: "Q44190",
|
||||
pages: map[string]string{"it": "Radiohead", "en": "Radiohead"},
|
||||
summaries: map[string]Summary{
|
||||
"it": standard("it", "Q44190", "I Radiohead sono un gruppo musicale."),
|
||||
"en": standard("en", "Q44190", "Radiohead are an English rock band."),
|
||||
},
|
||||
sumErrs: map[string]error{},
|
||||
}
|
||||
r.sw = &switches{m: map[string]bool{providers.Wikipedia: enabled, providers.MusicBrainz: enabled}}
|
||||
r.svc = NewService(r.store, r.src, r.sw, []string{"it", "en"}, zap.NewNop())
|
||||
r.svc.now = func() time.Time { return r.now }
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *rig) get(t *testing.T, id int64) Result {
|
||||
t.Helper()
|
||||
res, err := r.svc.Description(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatalf("Description: %v", err)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func TestTheDescriptionComesInTheFirstLanguageThatHasAPage(t *testing.T) {
|
||||
r := newRig(true)
|
||||
res := r.get(t, 1)
|
||||
if res.Status != discovery.StatusOK || res.Description == nil {
|
||||
t.Fatalf("result = %+v", res)
|
||||
}
|
||||
d := res.Description
|
||||
if d.Language != "it" || d.Text != "I Radiohead sono un gruppo musicale." || d.Short != "una band" || d.Title != "Radiohead" ||
|
||||
d.URL != "https://it.wikipedia.org/wiki/Radiohead" {
|
||||
t.Errorf("description = %+v", d)
|
||||
}
|
||||
if !res.RetrievedAt.Equal(t0) || res.Stale {
|
||||
t.Errorf("retrieved %v stale %v", res.RetrievedAt, res.Stale)
|
||||
}
|
||||
if r.src.mbid != mbid {
|
||||
t.Errorf("asked about %q, want the MusicBrainz id", r.src.mbid)
|
||||
}
|
||||
if got := strings.Join(r.src.langs, ","); got != "it,en" {
|
||||
t.Errorf("languages = %s, want it,en in the order of the settings", got)
|
||||
}
|
||||
if len(r.src.asked) != 1 {
|
||||
t.Errorf("summaries read: %v, want only the first language that works", r.src.asked)
|
||||
}
|
||||
if len(r.store.saves) != 1 || r.store.saves[0].qid != "Q44190" {
|
||||
t.Errorf("saves = %+v", r.store.saves)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheNextLanguageIsUsedWhenThereIsNoPageOrNoUsableOne(t *testing.T) {
|
||||
for name, tt := range map[string]func(*rig){
|
||||
"no page in Italian": func(r *rig) { r.src.pages = map[string]string{"en": "Radiohead"} },
|
||||
"the page is gone": func(r *rig) { r.src.sumErrs["it"] = ErrSourceNotFound },
|
||||
"a disambiguation": func(r *rig) {
|
||||
s := r.src.summaries["it"]
|
||||
s.Type = "disambiguation"
|
||||
r.src.summaries["it"] = s
|
||||
},
|
||||
"another item": func(r *rig) {
|
||||
s := r.src.summaries["it"]
|
||||
s.WikibaseItem = "Q1"
|
||||
r.src.summaries["it"] = s
|
||||
},
|
||||
"no text": func(r *rig) {
|
||||
s := r.src.summaries["it"]
|
||||
s.Extract = " "
|
||||
r.src.summaries["it"] = s
|
||||
},
|
||||
} {
|
||||
r := newRig(true)
|
||||
tt(r)
|
||||
res := r.get(t, 1)
|
||||
if res.Description == nil || res.Description.Language != "en" {
|
||||
t.Errorf("%s: description = %+v, want the English one", name, res.Description)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNothingFoundIsRememberedAsSuchForADay(t *testing.T) {
|
||||
for name, tt := range map[string]func(*rig){
|
||||
"no wikidata link": func(r *rig) { r.src.qid = "" },
|
||||
"no unknown item": func(r *rig) { r.src.pagesErr = ErrSourceNotFound },
|
||||
"no page at all": func(r *rig) { r.src.pages = map[string]string{} },
|
||||
"musicbrainz does not know the artist": func(r *rig) { r.src.qidErr = ErrSourceNotFound },
|
||||
} {
|
||||
r := newRig(true)
|
||||
tt(r)
|
||||
res := r.get(t, 1)
|
||||
if res.Status != discovery.StatusOK || res.Description != nil || res.RetrievedAt.IsZero() {
|
||||
t.Errorf("%s: result = %+v, want ok with no description", name, res)
|
||||
continue
|
||||
}
|
||||
if len(r.store.saves) != 1 || r.store.saves[0].desc != nil {
|
||||
t.Errorf("%s: saves = %+v, want one 'nothing' record", name, r.store.saves)
|
||||
}
|
||||
calls := r.src.total()
|
||||
r.now = t0.Add(NegativeTTL - time.Minute)
|
||||
r.get(t, 1)
|
||||
if r.src.total() != calls {
|
||||
t.Errorf("%s: asked again before the day was over", name)
|
||||
}
|
||||
r.now = t0.Add(NegativeTTL + time.Minute)
|
||||
r.get(t, 1)
|
||||
if r.src.total() == calls {
|
||||
t.Errorf("%s: never asked again", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestADescriptionIsKeptThirtyDaysAndStaleWhenWikipediaFails(t *testing.T) {
|
||||
r := newRig(true)
|
||||
r.get(t, 1)
|
||||
calls := r.src.total()
|
||||
r.now = t0.Add(PositiveTTL - time.Hour)
|
||||
if res := r.get(t, 1); res.Stale || r.src.total() != calls {
|
||||
t.Fatalf("within the time: stale %v, calls %d -> %d", res.Stale, calls, r.src.total())
|
||||
}
|
||||
r.now = t0.Add(PositiveTTL + time.Hour)
|
||||
r.src.pagesErr = ErrSourceUnavailable
|
||||
res := r.get(t, 1)
|
||||
if res.Status != discovery.StatusError || !res.Stale || res.Description == nil || res.Description.Language != "it" {
|
||||
t.Errorf("when Wikipedia fails: %+v, want the old description, stale, with status error", res)
|
||||
}
|
||||
r.src.pagesErr = nil
|
||||
if res = r.get(t, 1); res.Status != discovery.StatusOK || res.Stale || !res.RetrievedAt.Equal(r.now) {
|
||||
t.Errorf("after it came back: %+v", res)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFailureIsNotRememberedAndASlowLanguageDoesNotFallThrough(t *testing.T) {
|
||||
r := newRig(true)
|
||||
r.src.sumErrs["it"] = ErrSourceUnavailable // Italian fails: English must not be saved in its place
|
||||
res := r.get(t, 1)
|
||||
if res.Status != discovery.StatusError || res.Description != nil || len(r.store.saves) != 0 {
|
||||
t.Errorf("result = %+v saves = %+v: a failure must not be saved, nor replaced by the next language", res, r.store.saves)
|
||||
}
|
||||
if len(r.src.asked) != 1 {
|
||||
t.Errorf("summaries read: %v", r.src.asked)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithTheProviderOffNobodyIsAskedAndTheOldAnswerIsStillGiven(t *testing.T) {
|
||||
r := newRig(false)
|
||||
res := r.get(t, 1)
|
||||
if res.Status != discovery.StatusDisabled || res.Description != nil || !res.RetrievedAt.IsZero() || res.Stale || r.src.total() != 0 {
|
||||
t.Errorf("result = %+v calls %d", res, r.src.total())
|
||||
}
|
||||
r.store.snaps[1] = Snapshot{Fetched: true, RetrievedAt: t0.Add(-100 * 24 * time.Hour), Description: Description{Text: "old", Language: "en", Title: "T", URL: "https://en.wikipedia.org/wiki/T"}}
|
||||
res = r.get(t, 1)
|
||||
if res.Status != discovery.StatusDisabled || res.Description == nil || res.Description.Text != "old" || !res.Stale || r.src.total() != 0 {
|
||||
t.Errorf("with an old answer: %+v", res)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnArtistWithoutAnIdIsNeverLookedUpByName(t *testing.T) {
|
||||
r := newRig(true)
|
||||
res := r.get(t, 2)
|
||||
if res.Status != discovery.StatusNoID || res.Description != nil || r.src.total() != 0 || len(r.store.saves) != 0 {
|
||||
t.Errorf("result = %+v calls %d saves %d", res, r.src.total(), len(r.store.saves))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnknownArtistIsNotFound(t *testing.T) {
|
||||
r := newRig(true)
|
||||
for _, id := range []int64{0, -1, 99} {
|
||||
if _, err := r.svc.Description(context.Background(), id); !errors.Is(err, discovery.ErrNotFound) {
|
||||
t.Errorf("id %d: err = %v", id, err)
|
||||
}
|
||||
}
|
||||
if r.src.total() != 0 {
|
||||
t.Error("Wikipedia was asked about an artist that does not exist")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheTextIsCutAtTheLimit(t *testing.T) {
|
||||
r := newRig(true)
|
||||
long := standard("it", "Q44190", strings.Repeat("è", MaxTextLen+500))
|
||||
r.src.summaries["it"] = long
|
||||
d := r.get(t, 1).Description
|
||||
if n := len([]rune(d.Text)); n > MaxTextLen+1 || !strings.HasSuffix(d.Text, "…") {
|
||||
t.Errorf("text has %d characters, want at most %d and a mark that it was cut", n, MaxTextLen)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManyAsksAtOnceAskWikipediaOnce(t *testing.T) {
|
||||
r := newRig(true)
|
||||
r.src.release = make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 10; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if _, err := r.svc.Description(context.Background(), 1); err != nil {
|
||||
t.Errorf("err = %v", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
close(r.src.release)
|
||||
wg.Wait()
|
||||
if r.src.calls.qid != 1 {
|
||||
t.Errorf("MusicBrainz asked %d times, want 1", r.src.calls.qid)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAListenerWhoLeavesDoesNotStopTheRead(t *testing.T) {
|
||||
r := newRig(true)
|
||||
r.src.release = make(chan struct{})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
errc := make(chan error, 1)
|
||||
go func() { _, err := r.svc.Description(ctx, 1); errc <- err }()
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
cancel()
|
||||
if err := <-errc; !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("err = %v", err)
|
||||
}
|
||||
close(r.src.release)
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for {
|
||||
r.store.mu.Lock()
|
||||
n := len(r.store.saves)
|
||||
r.store.mu.Unlock()
|
||||
if n == 1 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("the read was stopped when the listener left")
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPanicInTheSourceIsAFailureNotACrash(t *testing.T) {
|
||||
r := newRig(true)
|
||||
r.src.boom = "boom"
|
||||
if res := r.get(t, 1); res.Status != discovery.StatusError {
|
||||
t.Errorf("status = %q", res.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnArtistRemovedWhileItWasReadIsNotFound(t *testing.T) {
|
||||
r := newRig(true)
|
||||
r.store.saveErr = discovery.ErrNotFound
|
||||
if _, err := r.svc.Description(context.Background(), 1); !errors.Is(err, discovery.ErrNotFound) {
|
||||
t.Errorf("err = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestItNeedsMusicBrainzToBeOnToo(t *testing.T) {
|
||||
// The link to Wikidata comes from MusicBrainz: with MusicBrainz switched off nothing is sent to it.
|
||||
r := newRig(true)
|
||||
r.sw.set(providers.MusicBrainz, false)
|
||||
res := r.get(t, 1)
|
||||
if res.Status != discovery.StatusDisabled || res.Description != nil || r.src.total() != 0 {
|
||||
t.Errorf("result = %+v, calls %d: nobody may be asked while MusicBrainz is off", res, r.src.total())
|
||||
}
|
||||
r.sw.set(providers.MusicBrainz, true)
|
||||
r.sw.set(providers.Wikipedia, false)
|
||||
if res = r.get(t, 1); res.Status != discovery.StatusDisabled || r.src.total() != 0 {
|
||||
t.Errorf("with Wikipedia off: %+v calls %d", res, r.src.total())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMergedItemIsUnderstoodThroughTheItemItWasMergedInto(t *testing.T) {
|
||||
r := newRig(true)
|
||||
r.src.qid, r.src.resolved = "Q1", "Q2" // MusicBrainz knows the old id; the pages say Q2
|
||||
for lang, sum := range r.src.summaries {
|
||||
sum.WikibaseItem = "Q2"
|
||||
r.src.summaries[lang] = sum
|
||||
}
|
||||
res := r.get(t, 1)
|
||||
if res.Description == nil || res.Description.Language != "it" {
|
||||
t.Fatalf("result = %+v, want the description of the item it was merged into", res)
|
||||
}
|
||||
if len(r.store.saves) != 1 || r.store.saves[0].qid != "Q2" {
|
||||
t.Errorf("saves = %+v, want the resolved item stored", r.store.saves)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheTitleShownIsTheOneWikidataNames(t *testing.T) {
|
||||
r := newRig(true)
|
||||
r.src.pages = map[string]string{"it": "Guns N' Roses"}
|
||||
s := r.src.summaries["it"]
|
||||
s.Title = "Guns_N'_Roses" // the REST title is the database key, with underscores
|
||||
r.src.summaries["it"] = s
|
||||
if d := r.get(t, 1).Description; d == nil || d.Title != "Guns N' Roses" {
|
||||
t.Errorf("description = %+v", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPageWithNoAddressOnWikipediaIsNotUsed(t *testing.T) {
|
||||
r := newRig(true)
|
||||
s := r.src.summaries["it"]
|
||||
s.URL = "" // the client drops an address that is not on that Wikipedia
|
||||
r.src.summaries["it"] = s
|
||||
if d := r.get(t, 1).Description; d == nil || d.Language != "en" {
|
||||
t.Errorf("description = %+v, want the next language: a text without its source cannot be shown", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheTextNeverExceedsTheLimit(t *testing.T) {
|
||||
r := newRig(true)
|
||||
r.src.summaries["it"] = standard("it", "Q44190", strings.Repeat("è", MaxTextLen+500))
|
||||
if n := len([]rune(r.get(t, 1).Description.Text)); n != MaxTextLen {
|
||||
t.Errorf("text has %d characters, want exactly %d with the mark that it was cut", n, MaxTextLen)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStoredDescriptionIsStillReportedAsDisabledWhenTheProviderIsOff(t *testing.T) {
|
||||
r := newRig(true)
|
||||
r.get(t, 1)
|
||||
r.sw.set(providers.Wikipedia, false)
|
||||
res := r.get(t, 1) // fresh, but the switch is off
|
||||
if res.Status != discovery.StatusDisabled || res.Stale || res.Description == nil {
|
||||
t.Errorf("result = %+v, want the description, not stale, with status disabled", res)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSwitchedOffWhileItWasBeingReadNothingIsSaved(t *testing.T) {
|
||||
r := newRig(true)
|
||||
r.src.release = make(chan struct{})
|
||||
done := make(chan Result, 1)
|
||||
go func() {
|
||||
res, _ := r.svc.Description(context.Background(), 1)
|
||||
done <- res
|
||||
}()
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
r.sw.set(providers.Wikipedia, false) // the admin took the data back and switched it off
|
||||
close(r.src.release)
|
||||
res := <-done
|
||||
if len(r.store.saves) != 0 {
|
||||
t.Errorf("saves = %+v: what a read that started before the switch found must not be kept", r.store.saves)
|
||||
}
|
||||
if res.Status != discovery.StatusDisabled || res.Description != nil {
|
||||
t.Errorf("result = %+v", res)
|
||||
}
|
||||
}
|
||||
51
internal/about/source.go
Normal file
51
internal/about/source.go
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
package about
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"codeberg.org/squidd/sonda/internal/providers/musicbrainz"
|
||||
"codeberg.org/squidd/sonda/internal/providers/wikipedia"
|
||||
)
|
||||
|
||||
// WikipediaSource follows the chain with the real clients: MusicBrainz for the link to
|
||||
// Wikidata, Wikidata for the pages, Wikipedia for the summary.
|
||||
type WikipediaSource struct {
|
||||
MusicBrainz *musicbrainz.Client
|
||||
Wikipedia *wikipedia.Client
|
||||
}
|
||||
|
||||
// notFound turns what means "there is nothing to find" into ErrSourceNotFound. An
|
||||
// input the clients refuse (an id that is not one) is nothing to find either: asking
|
||||
// again would not change that.
|
||||
func notFound(err error) error {
|
||||
if errors.Is(err, musicbrainz.ErrNotFound) || errors.Is(err, musicbrainz.ErrBadRequest) ||
|
||||
errors.Is(err, wikipedia.ErrNotFound) || errors.Is(err, wikipedia.ErrBadRequest) {
|
||||
return ErrSourceNotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// WikidataID implements Source.
|
||||
func (w WikipediaSource) WikidataID(ctx context.Context, musicbrainzID string) (string, error) {
|
||||
id, err := w.MusicBrainz.WikidataID(ctx, musicbrainzID)
|
||||
return id, notFound(err)
|
||||
}
|
||||
|
||||
// Pages implements Source.
|
||||
func (w WikipediaSource) Pages(ctx context.Context, item string, languages []string) (string, map[string]string, error) {
|
||||
resolved, pages, err := w.Wikipedia.Pages(ctx, item, languages)
|
||||
return resolved, pages, notFound(err)
|
||||
}
|
||||
|
||||
// Summary implements Source.
|
||||
func (w WikipediaSource) Summary(ctx context.Context, language, title string) (Summary, error) {
|
||||
s, err := w.Wikipedia.Summary(ctx, language, title)
|
||||
if err != nil {
|
||||
return Summary{}, notFound(err)
|
||||
}
|
||||
return Summary{
|
||||
Type: s.Type, Title: s.Title, Lang: s.Lang, Extract: s.Extract, Description: s.Description,
|
||||
URL: s.URL, WikibaseItem: s.WikibaseItem,
|
||||
}, nil
|
||||
}
|
||||
111
internal/about/source_test.go
Normal file
111
internal/about/source_test.go
Normal file
|
|
@ -0,0 +1,111 @@
|
|||
package about
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"codeberg.org/squidd/sonda/internal/discovery"
|
||||
"codeberg.org/squidd/sonda/internal/providers"
|
||||
"codeberg.org/squidd/sonda/internal/providers/musicbrainz"
|
||||
"codeberg.org/squidd/sonda/internal/providers/outbound"
|
||||
"codeberg.org/squidd/sonda/internal/providers/wikipedia"
|
||||
)
|
||||
|
||||
// fakeWeb answers for MusicBrainz, Wikidata and Wikipedia on one server.
|
||||
func fakeWeb(t *testing.T, wikidataLink string) WikipediaSource {
|
||||
t.Helper()
|
||||
return fakeWebWith(t, wikidataLink, "ok")
|
||||
}
|
||||
|
||||
// fakeWebWith can make the Italian page a redirect ("redirect") or a page with no text ("noextract").
|
||||
func fakeWebWith(t *testing.T, wikidataLink, italian string) WikipediaSource {
|
||||
t.Helper()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case strings.HasPrefix(r.URL.Path, "/ws/2/artist/"+mbid):
|
||||
rels := `[]`
|
||||
if wikidataLink != "" {
|
||||
rels = `[{"type":"wikidata","url":{"resource":"` + wikidataLink + `"}}]`
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"id":"` + mbid + `","relations":` + rels + `}`))
|
||||
case strings.HasPrefix(r.URL.Path, "/ws/2/artist/"):
|
||||
http.NotFound(w, r)
|
||||
case r.URL.Path == "/w/api.php":
|
||||
_, _ = w.Write([]byte(`{"entities":{"Q44190":{"id":"Q44190","sitelinks":{"itwiki":{"title":"Radiohead"},"enwiki":{"title":"Radiohead"}}}}}`))
|
||||
case r.URL.Path == "/it/summary/Radiohead" && italian == "redirect":
|
||||
http.Redirect(w, r, "https://evil.example/", http.StatusFound)
|
||||
case r.URL.Path == "/it/summary/Radiohead" && italian == "noextract":
|
||||
_, _ = w.Write([]byte(`{"type":"no-extract","title":"Radiohead","lang":"it","extract":"","wikibase_item":"Q44190","content_urls":{"desktop":{"page":"https://it.wikipedia.org/wiki/Radiohead"}}}`))
|
||||
case r.URL.Path == "/en/summary/Radiohead":
|
||||
_, _ = w.Write([]byte(`{"type":"standard","title":"Radiohead","lang":"en","extract":"Text.","description":"band","wikibase_item":"Q44190","content_urls":{"desktop":{"page":"https://en.wikipedia.org/wiki/Radiohead"}}}`))
|
||||
case r.URL.Path == "/it/summary/Radiohead":
|
||||
_, _ = w.Write([]byte(`{"type":"standard","title":"Radiohead","lang":"it","extract":"Testo.","description":"gruppo","wikibase_item":"Q44190","content_urls":{"desktop":{"page":"https://it.wikipedia.org/wiki/Radiohead"}}}`))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
client := outbound.New(outbound.Options{UserAgent: "Sonda/test ( https://example.org )", Retries: outbound.NoRetries})
|
||||
return WikipediaSource{
|
||||
MusicBrainz: musicbrainz.New(client, srv.URL+"/ws/2/"),
|
||||
Wikipedia: wikipedia.New(client, srv.URL+"/w/api.php", srv.URL+"/%s/summary/"),
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheRealClientsFollowTheChain(t *testing.T) {
|
||||
src := fakeWeb(t, "https://www.wikidata.org/wiki/Q44190")
|
||||
ctx := context.Background()
|
||||
item, err := src.WikidataID(ctx, mbid)
|
||||
if err != nil || item != "Q44190" {
|
||||
t.Fatalf("item = %q err = %v", item, err)
|
||||
}
|
||||
resolved, pages, err := src.Pages(ctx, item, []string{"it", "en"})
|
||||
if err != nil || resolved != "Q44190" || pages["it"] != "Radiohead" || pages["en"] != "Radiohead" {
|
||||
t.Fatalf("item %q pages = %v err = %v", resolved, pages, err)
|
||||
}
|
||||
sum, err := src.Summary(ctx, "it", "Radiohead")
|
||||
if err != nil || sum.WikibaseItem != "Q44190" || sum.Extract != "Testo." || sum.Type != "standard" || sum.Description != "gruppo" {
|
||||
t.Fatalf("summary = %+v err = %v", sum, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWhatThereIsNothingToFindIsNotAFailure(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
if item, err := fakeWeb(t, "").WikidataID(ctx, mbid); err != nil || item != "" {
|
||||
t.Errorf("no link: %q %v", item, err)
|
||||
}
|
||||
src := fakeWeb(t, "https://www.wikidata.org/wiki/Q44190")
|
||||
if _, err := src.WikidataID(ctx, "22222222-2222-4333-8444-555555555555"); !errors.Is(err, ErrSourceNotFound) {
|
||||
t.Errorf("unknown artist: %v", err)
|
||||
}
|
||||
if _, err := src.WikidataID(ctx, "not an id"); !errors.Is(err, ErrSourceNotFound) {
|
||||
t.Errorf("a bad id: %v", err)
|
||||
}
|
||||
if _, err := src.Summary(ctx, "it", "No such page"); !errors.Is(err, ErrSourceNotFound) {
|
||||
t.Errorf("no page: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// With the real clients, a page that cannot be used (a redirect that is never followed, a
|
||||
// page with no text) is an answer: the chain goes on to the next language, and the outcome
|
||||
// is stored, so the same page is not asked about again at every view.
|
||||
func TestAPageThatCannotBeUsedFallsThroughToTheNextLanguage(t *testing.T) {
|
||||
for _, italian := range []string{"redirect", "noextract"} {
|
||||
store := newStore()
|
||||
svc := NewService(store, fakeWebWith(t, "https://www.wikidata.org/wiki/Q44190", italian),
|
||||
&switches{m: map[string]bool{providers.Wikipedia: true, providers.MusicBrainz: true}}, []string{"it", "en"}, zap.NewNop())
|
||||
res, err := svc.Description(context.Background(), 1)
|
||||
if err != nil || res.Status != discovery.StatusOK || res.Description == nil || res.Description.Language != "en" {
|
||||
t.Errorf("Italian page %s: result = %+v err = %v, want the English description", italian, res, err)
|
||||
}
|
||||
if len(store.saves) != 1 {
|
||||
t.Errorf("Italian page %s: %d saves, want the outcome stored once", italian, len(store.saves))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -15,6 +15,7 @@ import (
|
|||
"github.com/uptrace/bun"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"codeberg.org/squidd/sonda/internal/about"
|
||||
"codeberg.org/squidd/sonda/internal/auth"
|
||||
"codeberg.org/squidd/sonda/internal/discover"
|
||||
"codeberg.org/squidd/sonda/internal/discovery"
|
||||
|
|
@ -100,6 +101,9 @@ func newTestEnvDB(t *testing.T, mutate func(*Deps, *bun.DB)) (http.Handler, *bun
|
|||
imageRepo := storage.NewImageRepo(db)
|
||||
deps.Images = images.NewService(filepath.Join(t.TempDir(), "data"), imageRepo, imageRepo)
|
||||
mutate(&deps, db)
|
||||
if deps.About == nil {
|
||||
deps.About = about.NewService(storage.NewDiscoveryRepo(db), noWikipedia{}, prov, []string{"en"}, zap.NewNop())
|
||||
}
|
||||
// Built from the final services, so a test that swaps one gets a Discover that uses it.
|
||||
if deps.Discover == nil {
|
||||
deps.Discover = discover.NewService(deps.Taste.(discover.Taster), deps.Discovery.(discover.Similar),
|
||||
|
|
@ -109,6 +113,17 @@ func newTestEnvDB(t *testing.T, mutate func(*Deps, *bun.DB)) (http.Handler, *bun
|
|||
return NewRouter(deps), db
|
||||
}
|
||||
|
||||
// noWikipedia is a Wikipedia that knows nothing, for tests that do not care about it.
|
||||
type noWikipedia struct{}
|
||||
|
||||
func (noWikipedia) WikidataID(context.Context, string) (string, error) { return "", nil }
|
||||
func (noWikipedia) Pages(context.Context, string, []string) (string, map[string]string, error) {
|
||||
return "", nil, about.ErrSourceNotFound
|
||||
}
|
||||
func (noWikipedia) Summary(context.Context, string, string) (about.Summary, error) {
|
||||
return about.Summary{}, about.ErrSourceNotFound
|
||||
}
|
||||
|
||||
// newProviders builds the provider service the way main does, with the real way
|
||||
// of removing provider data.
|
||||
func newProviders(db *bun.DB) *providers.Service {
|
||||
|
|
|
|||
|
|
@ -419,8 +419,8 @@ func TestResponsesMatchTheSpec(t *testing.T) {
|
|||
|
||||
// ---- providers ----
|
||||
listed := c.expect(c.call("GET", "/providers", token, nil), 200, "list providers")["providers"].([]any)
|
||||
if len(listed) != 2 || listed[0].(map[string]any)["enabled"] != false || listed[1].(map[string]any)["enabled"] != false {
|
||||
t.Fatalf("providers = %v, want two, both off", listed)
|
||||
if len(listed) != 3 || listed[0].(map[string]any)["enabled"] != false || listed[1].(map[string]any)["enabled"] != false || listed[2].(map[string]any)["enabled"] != false {
|
||||
t.Fatalf("providers = %v, want three, all off", listed)
|
||||
}
|
||||
c.expect(c.call("PUT", "/providers/lastfm", token, map[string]any{"enabled": true}), 400, "enable without a key", "invalid_input")
|
||||
c.expect(c.call("PUT", "/providers/lastfm", token, map[string]any{"api_key": "short"}), 400, "a bad key", "invalid_input")
|
||||
|
|
@ -536,6 +536,19 @@ func TestResponsesMatchTheSpec(t *testing.T) {
|
|||
c.expect(c.call("GET", "/me/taste?limit=-1", token, nil), 400, "taste with a bad limit", "invalid_input")
|
||||
c.expect(c.call("GET", "/me/taste?limit=abc", token, nil), 400, "taste with a limit that is not a number", "invalid_input")
|
||||
|
||||
// ---- description ----
|
||||
desc := c.expect(c.call("GET", "/artists/"+artistID+"/description", token, nil), 200, "description with the provider off")
|
||||
if desc["provider_status"] != "disabled" || desc["description"] != nil {
|
||||
t.Fatalf("description = %v", desc)
|
||||
}
|
||||
c.expect(c.call("PUT", "/providers/wikipedia", token, map[string]any{"enabled": true, "contact": "https://example.org/me"}), 200, "switch Wikipedia on")
|
||||
if noID := c.expect(c.call("GET", "/artists/"+artistID+"/description", token, nil), 200, "description of an artist without an id"); noID["provider_status"] != "no_id" {
|
||||
t.Fatalf("description = %v, want no_id", noID)
|
||||
}
|
||||
c.expect(c.call("GET", "/artists/999/description", token, nil), 404, "description of an unknown artist", "artist_not_found")
|
||||
c.expect(c.call("GET", "/artists/abc/description", token, nil), 400, "description of a bad id", "invalid_input")
|
||||
c.expect(c.call("GET", "/artists/"+artistID+"/description", "", nil), 401, "description without a token", "unauthorized")
|
||||
|
||||
// ---- covers ----
|
||||
cover := "/albums/" + albumID + "/cover"
|
||||
big := c.call("GET", cover, token, nil)
|
||||
|
|
|
|||
215
internal/api/http/description_test.go
Normal file
215
internal/api/http/description_test.go
Normal file
|
|
@ -0,0 +1,215 @@
|
|||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/uptrace/bun"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"codeberg.org/squidd/sonda/internal/about"
|
||||
"codeberg.org/squidd/sonda/internal/providers"
|
||||
"codeberg.org/squidd/sonda/internal/storage"
|
||||
)
|
||||
|
||||
// fakeWikipedia is the whole chain, in one place: it counts what was asked.
|
||||
type fakeWikipedia struct {
|
||||
mu sync.Mutex
|
||||
calls int
|
||||
ids []string
|
||||
item string
|
||||
pages map[string]string
|
||||
sums map[string]about.Summary
|
||||
}
|
||||
|
||||
func (f *fakeWikipedia) WikidataID(_ context.Context, id string) (string, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.calls++
|
||||
f.ids = append(f.ids, id)
|
||||
return f.item, nil
|
||||
}
|
||||
|
||||
func (f *fakeWikipedia) Pages(_ context.Context, item string, _ []string) (string, map[string]string, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.calls++
|
||||
return item, f.pages, nil
|
||||
}
|
||||
|
||||
func (f *fakeWikipedia) Summary(_ context.Context, lang, _ string) (about.Summary, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.calls++
|
||||
return f.sums[lang], nil
|
||||
}
|
||||
|
||||
func (f *fakeWikipedia) count() int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.calls
|
||||
}
|
||||
|
||||
type descEnv struct {
|
||||
*similarEnv
|
||||
wp *fakeWikipedia
|
||||
}
|
||||
|
||||
func newDescEnv(t *testing.T) *descEnv {
|
||||
t.Helper()
|
||||
e := &descEnv{wp: &fakeWikipedia{
|
||||
item: "Q44190",
|
||||
pages: map[string]string{"it": "Alpha"},
|
||||
sums: map[string]about.Summary{"it": {Type: "standard", Title: "Alpha", Lang: "it", Extract: "Alpha è un gruppo.",
|
||||
Description: "gruppo musicale", URL: "https://it.wikipedia.org/wiki/Alpha", WikibaseItem: "Q44190"}},
|
||||
}}
|
||||
e.similarEnv = newSimilarEnvWith(t, func(d *Deps, db *bun.DB, prov *providers.Service) {
|
||||
d.About = about.NewService(storage.NewDiscoveryRepo(db), e.wp, prov, []string{"it", "en"}, zap.NewNop())
|
||||
})
|
||||
return e
|
||||
}
|
||||
|
||||
func (e *descEnv) path(id int64) string {
|
||||
return "/api/v1/artists/" + strconv.FormatInt(id, 10) + "/description"
|
||||
}
|
||||
|
||||
// enable switches on Wikipedia and MusicBrainz: the link to Wikidata comes from the second.
|
||||
func (e *descEnv) enable(t *testing.T) {
|
||||
t.Helper()
|
||||
for _, name := range []string{"wikipedia", "musicbrainz"} {
|
||||
if r := do(t, e.h, "PUT", "/api/v1/providers/"+name, e.admin, map[string]any{"enabled": true}); r.Code != http.StatusOK {
|
||||
t.Fatalf("enable %s: %d %v", name, r.Code, r.Body)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (e *descEnv) giveAlphaAnID(t *testing.T) {
|
||||
t.Helper()
|
||||
if _, err := e.db.Exec("UPDATE artists SET mbid = ? WHERE id = ?", alphaMBID, e.alpha); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDescriptionNeedsASignedInUser(t *testing.T) {
|
||||
e := newDescEnv(t)
|
||||
if r := do(t, e.h, "GET", e.path(e.alpha), "", nil); r.Code != http.StatusUnauthorized {
|
||||
t.Errorf("no token: %d", r.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDescriptionRefusesWhatIsNotAnArtist(t *testing.T) {
|
||||
e := newDescEnv(t)
|
||||
e.enable(t)
|
||||
for name, tt := range map[string]struct {
|
||||
path string
|
||||
code int
|
||||
err string
|
||||
}{
|
||||
"unknown artist": {e.path(99999), 404, "artist_not_found"},
|
||||
"not a number": {"/api/v1/artists/abc/description", 400, "invalid_input"},
|
||||
"zero": {"/api/v1/artists/0/description", 400, "invalid_input"},
|
||||
} {
|
||||
r := do(t, e.h, "GET", tt.path, e.admin, nil)
|
||||
if r.Code != tt.code || errCode(r) != tt.err {
|
||||
t.Errorf("%s: %d %v, want %d %s", name, r.Code, r.Body, tt.code, tt.err)
|
||||
}
|
||||
}
|
||||
if e.wp.count() != 0 {
|
||||
t.Error("Wikipedia was asked about something that is not an artist")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDescriptionWithTheProviderOffAsksNobody(t *testing.T) {
|
||||
e := newDescEnv(t)
|
||||
e.giveAlphaAnID(t)
|
||||
r := do(t, e.h, "GET", e.path(e.alpha), e.admin, nil)
|
||||
if r.Code != http.StatusOK {
|
||||
t.Fatalf("%d %v", r.Code, r.Body)
|
||||
}
|
||||
sameKeys(t, "description", r.Body, "provider", "provider_status", "stale", "retrieved_at", "attribution", "description")
|
||||
if r.Body["provider"] != "wikipedia" || r.Body["provider_status"] != "disabled" || r.Body["stale"] != false || r.Body["attribution"] == "" {
|
||||
t.Errorf("answer = %v", r.Body)
|
||||
}
|
||||
if v, present := r.Body["description"]; !present || v != nil {
|
||||
t.Errorf("description = %#v, want an explicit null", v)
|
||||
}
|
||||
if v, present := r.Body["retrieved_at"]; !present || v != nil {
|
||||
t.Errorf("retrieved_at = %#v, want an explicit null", v)
|
||||
}
|
||||
if e.wp.count() != 0 {
|
||||
t.Error("a provider that is off was contacted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDescriptionOfAnArtistWithoutAnIdSaysSo(t *testing.T) {
|
||||
e := newDescEnv(t)
|
||||
e.enable(t)
|
||||
r := do(t, e.h, "GET", e.path(e.alpha), e.admin, nil)
|
||||
if r.Code != http.StatusOK || r.Body["provider_status"] != "no_id" || r.Body["description"] != nil || e.wp.count() != 0 {
|
||||
t.Errorf("%d %v, calls %d: Wikipedia must never be asked about an artist by name", r.Code, r.Body, e.wp.count())
|
||||
}
|
||||
}
|
||||
|
||||
func TestDescriptionIsReadStoredAndShownWithItsLicence(t *testing.T) {
|
||||
e := newDescEnv(t)
|
||||
e.enable(t)
|
||||
e.giveAlphaAnID(t)
|
||||
r := do(t, e.h, "GET", e.path(e.alpha), e.admin, nil)
|
||||
if r.Code != http.StatusOK || r.Body["provider_status"] != "ok" {
|
||||
t.Fatalf("%d %v", r.Code, r.Body)
|
||||
}
|
||||
d, _ := r.Body["description"].(map[string]any)
|
||||
sameKeys(t, "description body", d, "text", "short", "language", "title", "url", "license", "license_url")
|
||||
if d["text"] != "Alpha è un gruppo." || d["short"] != "gruppo musicale" || d["language"] != "it" || d["title"] != "Alpha" ||
|
||||
d["url"] != "https://it.wikipedia.org/wiki/Alpha" || d["license"] != "CC BY-SA 4.0" || d["license_url"] != "https://creativecommons.org/licenses/by-sa/4.0/" {
|
||||
t.Errorf("description = %v", d)
|
||||
}
|
||||
if _, err := time.Parse(time.RFC3339, r.Body["retrieved_at"].(string)); err != nil {
|
||||
t.Errorf("retrieved_at = %v", r.Body["retrieved_at"])
|
||||
}
|
||||
if len(e.wp.ids) != 1 || e.wp.ids[0] != alphaMBID {
|
||||
t.Errorf("asked about %v, want only the MusicBrainz id", e.wp.ids)
|
||||
}
|
||||
// Stored: a second look asks nobody.
|
||||
calls := e.wp.count()
|
||||
if r = do(t, e.h, "GET", e.path(e.alpha), e.admin, nil); r.Code != http.StatusOK || e.wp.count() != calls {
|
||||
t.Errorf("second look: %d, calls %d -> %d", r.Code, calls, e.wp.count())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPurgingWikipediaTakesTheDescriptionsBack(t *testing.T) {
|
||||
e := newDescEnv(t)
|
||||
e.enable(t)
|
||||
e.giveAlphaAnID(t)
|
||||
do(t, e.h, "GET", e.path(e.alpha), e.admin, nil)
|
||||
p := do(t, e.h, "DELETE", "/api/v1/providers/wikipedia/data", e.admin, nil)
|
||||
if p.Code != http.StatusOK {
|
||||
t.Fatalf("purge: %d %v", p.Code, p.Body)
|
||||
}
|
||||
sameKeys(t, "purge", p.Body, "similar_artists", "tags", "descriptions", "artists")
|
||||
if p.Body["descriptions"] != float64(1) {
|
||||
t.Errorf("purge = %v, want one description", p.Body)
|
||||
}
|
||||
// Switched off, nothing is read again, and nothing is left to show.
|
||||
do(t, e.h, "PUT", "/api/v1/providers/wikipedia", e.admin, map[string]any{"enabled": false})
|
||||
r := do(t, e.h, "GET", e.path(e.alpha), e.admin, nil)
|
||||
if r.Body["description"] != nil || r.Body["provider_status"] != "disabled" {
|
||||
t.Errorf("after the purge: %v", r.Body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDescriptionNeedsMusicBrainzOnToo(t *testing.T) {
|
||||
e := newDescEnv(t)
|
||||
e.giveAlphaAnID(t)
|
||||
if r := do(t, e.h, "PUT", "/api/v1/providers/wikipedia", e.admin, map[string]any{"enabled": true}); r.Code != http.StatusOK {
|
||||
t.Fatalf("enable wikipedia: %d", r.Code)
|
||||
}
|
||||
r := do(t, e.h, "GET", e.path(e.alpha), e.admin, nil)
|
||||
if r.Code != http.StatusOK || r.Body["provider_status"] != "disabled" || r.Body["description"] != nil || e.wp.count() != 0 {
|
||||
t.Errorf("%d %v calls %d: with MusicBrainz off nothing may be sent to it", r.Code, r.Body, e.wp.count())
|
||||
}
|
||||
}
|
||||
65
internal/api/http/handlers_about.go
Normal file
65
internal/api/http/handlers_about.go
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"codeberg.org/squidd/sonda/internal/about"
|
||||
"codeberg.org/squidd/sonda/internal/discovery"
|
||||
)
|
||||
|
||||
type aboutHandlers struct{ svc aboutService }
|
||||
|
||||
type descriptionJSON struct {
|
||||
Text string `json:"text"`
|
||||
Short string `json:"short"`
|
||||
Language string `json:"language"`
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
License string `json:"license"`
|
||||
LicenseURL string `json:"license_url"`
|
||||
}
|
||||
|
||||
type aboutJSON struct {
|
||||
Provider string `json:"provider"`
|
||||
ProviderStatus string `json:"provider_status"`
|
||||
Stale bool `json:"stale"`
|
||||
RetrievedAt *string `json:"retrieved_at"`
|
||||
Attribution string `json:"attribution"`
|
||||
Description *descriptionJSON `json:"description"`
|
||||
}
|
||||
|
||||
// description shows what Wikipedia says about an artist.
|
||||
func (h *aboutHandlers) description(c *gin.Context) {
|
||||
id, ok := pathID(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
res, err := h.svc.Description(c.Request.Context(), id)
|
||||
switch {
|
||||
case errors.Is(err, discovery.ErrNotFound):
|
||||
writeError(c, http.StatusNotFound, "artist_not_found", "not found")
|
||||
return
|
||||
case errors.Is(err, context.Canceled):
|
||||
c.AbortWithStatus(statusClientClosed) // the client left; nobody is waiting
|
||||
return
|
||||
case err != nil:
|
||||
_ = c.Error(err)
|
||||
writeError(c, http.StatusInternalServerError, "internal", "internal error")
|
||||
return
|
||||
}
|
||||
out := aboutJSON{Provider: about.Provider, ProviderStatus: res.Status, Stale: res.Stale, Attribution: about.Attribution}
|
||||
if !res.RetrievedAt.IsZero() {
|
||||
out.RetrievedAt = rfc3339(res.RetrievedAt)
|
||||
}
|
||||
if d := res.Description; d != nil {
|
||||
out.Description = &descriptionJSON{
|
||||
Text: d.Text, Short: d.Short, Language: d.Language, Title: d.Title, URL: d.URL,
|
||||
License: about.LicenseName, LicenseURL: about.LicenseURL,
|
||||
}
|
||||
}
|
||||
c.JSON(http.StatusOK, out)
|
||||
}
|
||||
|
|
@ -97,5 +97,5 @@ func (h *providersHandlers) purge(c *gin.Context) {
|
|||
writeProviderError(c, err)
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"similar_artists": res.SimilarArtists, "tags": res.Tags, "artists": res.Artists})
|
||||
c.JSON(http.StatusOK, gin.H{"similar_artists": res.SimilarArtists, "tags": res.Tags, "descriptions": res.Descriptions, "artists": res.Artists})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ func TestProvidersStartOffAndSayWhatTheySend(t *testing.T) {
|
|||
t.Fatalf("list: %d %v", r.Code, r.Body)
|
||||
}
|
||||
list := r.Body["providers"].([]any)
|
||||
if len(list) != 2 {
|
||||
if len(list) != 3 {
|
||||
t.Fatalf("providers = %v", list)
|
||||
}
|
||||
for _, it := range list {
|
||||
|
|
@ -263,7 +263,7 @@ func TestPurgeRemovesWhatTheProviderGaveAndNothingOfMine(t *testing.T) {
|
|||
if p.Code != http.StatusOK {
|
||||
t.Fatalf("%d %v", p.Code, p.Body)
|
||||
}
|
||||
sameKeys(t, "purge", p.Body, "similar_artists", "tags", "artists")
|
||||
sameKeys(t, "purge", p.Body, "similar_artists", "tags", "descriptions", "artists")
|
||||
if p.Body["similar_artists"] != float64(2) || p.Body["tags"] != float64(0) || p.Body["artists"] != float64(1) {
|
||||
t.Errorf("purge = %v, want 2 similar artists, no tags and the one artist Gamma", p.Body)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import (
|
|||
"github.com/gin-gonic/gin"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"codeberg.org/squidd/sonda/internal/about"
|
||||
"codeberg.org/squidd/sonda/internal/auth"
|
||||
"codeberg.org/squidd/sonda/internal/catalog"
|
||||
"codeberg.org/squidd/sonda/internal/discover"
|
||||
|
|
@ -85,6 +86,10 @@ type feedbackService interface {
|
|||
List(ctx context.Context, userID int64, kind feedback.Kind, p feedback.Page) (feedback.Listing, error)
|
||||
}
|
||||
|
||||
type aboutService interface {
|
||||
Description(ctx context.Context, artistID int64) (about.Result, error)
|
||||
}
|
||||
|
||||
type coverService interface {
|
||||
AlbumCover(ctx context.Context, albumID int64, size images.Size) (*images.Cover, error)
|
||||
}
|
||||
|
|
@ -130,6 +135,7 @@ type Deps struct {
|
|||
Discover discoverService
|
||||
Feedback feedbackService
|
||||
Images coverService
|
||||
About aboutService
|
||||
Settings userSettingsService
|
||||
Streams streamService
|
||||
}
|
||||
|
|
@ -184,6 +190,8 @@ func NewRouter(d Deps) *gin.Engine {
|
|||
protected.GET("/artists/:id/similar", disc.similar)
|
||||
tg := &tagsHandlers{svc: d.Tags}
|
||||
protected.GET("/artists/:id/tags", tg.tags)
|
||||
ab := &aboutHandlers{svc: d.About}
|
||||
protected.GET("/artists/:id/description", ab.description)
|
||||
protected.GET("/albums", cat.listAlbums)
|
||||
protected.GET("/albums/:id", cat.getAlbum)
|
||||
protected.GET("/tracks", cat.listTracks)
|
||||
|
|
|
|||
|
|
@ -928,6 +928,49 @@ paths:
|
|||
'400': {$ref: '#/components/responses/InvalidInput'}
|
||||
'401': {$ref: '#/components/responses/Unauthorized'}
|
||||
|
||||
/artists/{id}/description:
|
||||
parameters:
|
||||
- {$ref: '#/components/parameters/Id'}
|
||||
get:
|
||||
tags: [Discovery]
|
||||
operationId: getArtistDescription
|
||||
summary: What Wikipedia says about an artist
|
||||
description: |
|
||||
A short text about the artist from Wikipedia, with the page it comes from. The way to it
|
||||
is a chain of identifiers, never a search by name: the artist's MusicBrainz id, then the
|
||||
Wikidata item that MusicBrainz links to it, then the Wikipedia page that Wikidata names in
|
||||
the first of the server's languages (`[wikipedia] languages`, the code of the Wikipedia:
|
||||
`it`, `en`, `no` for Norwegian; a language with no Wikipedia finds nothing) that has one. A page is used
|
||||
only if it is an article about that same item (not a page of disambiguation). An artist
|
||||
with no MusicBrainz id answers `no_id`: Sonda does not look an id up by name.
|
||||
|
||||
It needs the `wikipedia` provider and also the `musicbrainz` one, because the link to
|
||||
Wikidata comes from MusicBrainz; an admin switches them on, and nothing is sent while
|
||||
either is off (`disabled`, also when an answer is stored). What is sent: the MusicBrainz id
|
||||
to MusicBrainz (with the contact set for MusicBrainz), the Wikidata item to Wikidata and the
|
||||
title of the page to Wikipedia (with the contact set for Wikipedia), and your server's IP
|
||||
address with them. The answer is stored: 30 days, or a
|
||||
day when there was nothing to find, and an old one is returned as `stale` if Wikipedia
|
||||
fails. When there is nothing to show, `description` is `null` and `provider_status` is
|
||||
`ok`.
|
||||
|
||||
The text is CC BY-SA 4.0. Where you show it, name the page (`title`), link to it (`url`) and
|
||||
give the licence (`license`, `license_url`), and say if you changed it.
|
||||
security: [{bearerAuth: []}]
|
||||
responses:
|
||||
'200':
|
||||
description: The description, or `null` with the status that says why.
|
||||
content:
|
||||
application/json:
|
||||
schema: {$ref: '#/components/schemas/ArtistDescription'}
|
||||
'400': {$ref: '#/components/responses/InvalidInput'}
|
||||
'401': {$ref: '#/components/responses/Unauthorized'}
|
||||
'404':
|
||||
description: No such artist (`artist_not_found`).
|
||||
content:
|
||||
application/json:
|
||||
schema: {$ref: '#/components/schemas/Error'}
|
||||
|
||||
/albums:
|
||||
get:
|
||||
tags: [Catalog]
|
||||
|
|
@ -1265,7 +1308,7 @@ paths:
|
|||
- name: name
|
||||
in: path
|
||||
required: true
|
||||
schema: {type: string, enum: [lastfm, musicbrainz]}
|
||||
schema: {type: string, enum: [lastfm, musicbrainz, wikipedia]}
|
||||
put:
|
||||
tags: [Providers]
|
||||
operationId: updateProvider
|
||||
|
|
@ -1303,7 +1346,7 @@ paths:
|
|||
- name: name
|
||||
in: path
|
||||
required: true
|
||||
schema: {type: string, enum: [lastfm, musicbrainz]}
|
||||
schema: {type: string, enum: [lastfm, musicbrainz, wikipedia]}
|
||||
post:
|
||||
tags: [Providers]
|
||||
operationId: testProvider
|
||||
|
|
@ -1338,7 +1381,7 @@ paths:
|
|||
- name: name
|
||||
in: path
|
||||
required: true
|
||||
schema: {type: string, enum: [lastfm, musicbrainz]}
|
||||
schema: {type: string, enum: [lastfm, musicbrainz, wikipedia]}
|
||||
delete:
|
||||
tags: [Providers]
|
||||
operationId: purgeProviderData
|
||||
|
|
@ -1799,7 +1842,7 @@ components:
|
|||
additionalProperties: false
|
||||
required: [name, enabled, needs_api_key, has_api_key, contact, sends, notes, updated_at]
|
||||
properties:
|
||||
name: {type: string, enum: [lastfm, musicbrainz]}
|
||||
name: {type: string, enum: [lastfm, musicbrainz, wikipedia]}
|
||||
enabled: {type: boolean, description: Whether Sonda may contact this provider.}
|
||||
needs_api_key: {type: boolean}
|
||||
has_api_key: {type: boolean, description: Whether a key is stored. The key itself is never returned.}
|
||||
|
|
@ -1829,12 +1872,48 @@ components:
|
|||
maxLength: 200
|
||||
description: A URL or e-mail address, without parentheses. Empty restores the default.
|
||||
|
||||
ArtistDescription:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required: [provider, provider_status, stale, retrieved_at, attribution, description]
|
||||
properties:
|
||||
provider: {type: string, enum: [wikipedia]}
|
||||
provider_status:
|
||||
type: string
|
||||
enum: [ok, disabled, error, no_id]
|
||||
description: |
|
||||
`ok` when the stored answer is as fresh as it needs to be (which can be "nothing found");
|
||||
`disabled` when the provider is off; `error` when Wikipedia could not be asked; `no_id`
|
||||
when the artist has no MusicBrainz id.
|
||||
stale: {type: boolean, description: True when the description is older than it should be because Wikipedia could not be asked.}
|
||||
retrieved_at: {type: string, format: date-time, nullable: true, description: When it was read; null when nothing is known.}
|
||||
attribution: {type: string, description: What to show where the text is shown.}
|
||||
description:
|
||||
type: object
|
||||
nullable: true
|
||||
allOf:
|
||||
- {$ref: '#/components/schemas/Description'}
|
||||
|
||||
Description:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required: [text, short, language, title, url, license, license_url]
|
||||
properties:
|
||||
text: {type: string, description: "The first paragraph of the article, plain text, at most 4000 characters."}
|
||||
short: {type: string, description: "One line, like \"gruppo musicale inglese\". May be empty."}
|
||||
language: {type: string, description: "The language of the page, like `it`."}
|
||||
title: {type: string, description: "The title of the page."}
|
||||
url: {type: string, format: uri, description: "The page, on Wikipedia."}
|
||||
license: {type: string, description: "The licence of the text."}
|
||||
license_url: {type: string, format: uri}
|
||||
|
||||
ProviderPurge:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required: [similar_artists, tags, artists]
|
||||
required: [similar_artists, tags, descriptions, artists]
|
||||
properties:
|
||||
similar_artists: {type: integer, description: Similar-artist links removed.}
|
||||
descriptions: {type: integer, description: Artist descriptions removed (only the data of Wikipedia has any).}
|
||||
tags: {type: integer, description: Tags removed.}
|
||||
artists: {type: integer, description: "Artists removed because only the removed data named them and nobody has files for them."}
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import (
|
|||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
"github.com/pelletier/go-toml/v2"
|
||||
|
|
@ -28,6 +29,7 @@ type Config struct {
|
|||
Taste Taste `toml:"taste"`
|
||||
Scoring Scoring `toml:"scoring"`
|
||||
Storage Storage `toml:"storage"`
|
||||
Wikipedia Wikipedia `toml:"wikipedia"`
|
||||
}
|
||||
|
||||
// Server holds HTTP listener settings.
|
||||
|
|
@ -105,6 +107,14 @@ type Scoring struct {
|
|||
WeightDiscovery float64 `toml:"weight_discovery"` // bonus for what I have not played
|
||||
}
|
||||
|
||||
// Wikipedia says in which languages to look for the description of an artist, in
|
||||
// order: the first language that has a page for it wins.
|
||||
type Wikipedia struct {
|
||||
Languages []string `toml:"languages"` // two or three lower case letters each, at most five
|
||||
}
|
||||
|
||||
var wikipediaLanguageRe = regexp.MustCompile(`^[a-z]{2,3}$`)
|
||||
|
||||
// defaultDataDir puts the data folder next to a SQLite database, so that a service
|
||||
// started from another working directory finds the same place; with PostgreSQL it is
|
||||
// "data" in the working directory.
|
||||
|
|
@ -133,6 +143,7 @@ func Default() Config {
|
|||
Auth: Auth{AccessTTLMinutes: 15, RefreshTTLHours: 720}, // 15 minutes, 30 days
|
||||
Listening: Listening{MinListenSeconds: 20},
|
||||
Taste: Taste{HalfLifeDays: 90},
|
||||
Wikipedia: Wikipedia{Languages: []string{"en"}},
|
||||
Scoring: Scoring{WeightAffinity: 0.40, WeightGenre: 0.25, WeightPreference: 0.25, WeightDiscovery: 0.10},
|
||||
}
|
||||
}
|
||||
|
|
@ -246,6 +257,18 @@ func (c Config) Validate() error {
|
|||
if sum == 0 {
|
||||
errs = append(errs, errors.New("scoring.weight_*: at least one weight must be above 0"))
|
||||
}
|
||||
if n := len(c.Wikipedia.Languages); n < 1 || n > 5 {
|
||||
errs = append(errs, errors.New("wikipedia.languages must hold one to five languages"))
|
||||
} else {
|
||||
seen := map[string]bool{}
|
||||
for _, l := range c.Wikipedia.Languages {
|
||||
if !wikipediaLanguageRe.MatchString(l) || seen[l] {
|
||||
errs = append(errs, fmt.Errorf("wikipedia.languages: %q is not a language code of two or three lower case letters, or is repeated", l))
|
||||
break
|
||||
}
|
||||
seen[l] = true
|
||||
}
|
||||
}
|
||||
if c.Storage.DataDir == "" {
|
||||
errs = append(errs, errors.New("storage.data_dir must not be empty"))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -138,6 +138,12 @@ func TestValidate(t *testing.T) {
|
|||
{"huge weight", func(c *Config) { c.Scoring.WeightAffinity = 11 }, "scoring.weight_affinity"},
|
||||
{"all weights zero", func(c *Config) { c.Scoring = Scoring{} }, "scoring.weight"},
|
||||
{"one weight is enough", func(c *Config) { c.Scoring = Scoring{WeightPreference: 1} }, ""},
|
||||
{"no wikipedia language", func(c *Config) { c.Wikipedia.Languages = nil }, "wikipedia.languages"},
|
||||
{"a bad wikipedia language", func(c *Config) { c.Wikipedia.Languages = []string{"it", "Italian"} }, "wikipedia.languages"},
|
||||
{"a wikipedia language with a hyphen", func(c *Config) { c.Wikipedia.Languages = []string{"zh-yue"} }, "wikipedia.languages"},
|
||||
{"a repeated wikipedia language", func(c *Config) { c.Wikipedia.Languages = []string{"it", "it"} }, "wikipedia.languages"},
|
||||
{"too many wikipedia languages", func(c *Config) { c.Wikipedia.Languages = []string{"aa", "bb", "cc", "dd", "ee", "ff"} }, "wikipedia.languages"},
|
||||
{"two wikipedia languages", func(c *Config) { c.Wikipedia.Languages = []string{"it", "en"} }, ""},
|
||||
{"empty data dir", func(c *Config) { c.Storage.DataDir = "" }, "storage.data_dir"},
|
||||
{"bad workers", func(c *Config) { c.Library.ScanWorkers = 0 }, "library.scan_workers"},
|
||||
{"bad access ttl", func(c *Config) { c.Auth.AccessTTLMinutes = 0 }, "auth.access_ttl_minutes"},
|
||||
|
|
@ -221,6 +227,16 @@ func TestScoringDefaultsAndFile(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestWikipediaLanguages(t *testing.T) {
|
||||
if got := Default().Wikipedia.Languages; len(got) != 1 || got[0] != "en" {
|
||||
t.Errorf("default languages = %v, want [en]", got)
|
||||
}
|
||||
cfg, err := Load(writeFile(t, t.TempDir(), "[wikipedia]\nlanguages = [\"it\", \"en\"]\n"))
|
||||
if err != nil || len(cfg.Wikipedia.Languages) != 2 || cfg.Wikipedia.Languages[0] != "it" {
|
||||
t.Errorf("languages = %v err = %v", cfg.Wikipedia.Languages, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorageDataDirDefaultsNextToTheDatabase(t *testing.T) {
|
||||
for name, tt := range map[string]struct{ toml, want string }{
|
||||
"default database": {"", "data"},
|
||||
|
|
|
|||
|
|
@ -128,6 +128,46 @@ func (c *Client) ArtistTags(ctx context.Context, id string) ([]Tag, error) {
|
|||
return out, nil
|
||||
}
|
||||
|
||||
// wikidataURLRe is the address MusicBrainz stores for the Wikidata item of an artist.
|
||||
// The number is bounded: it only has to be an item id, not an arbitrary string.
|
||||
var wikidataURLRe = regexp.MustCompile(`^https://www\.wikidata\.org/wiki/(Q[1-9][0-9]{0,15})$`)
|
||||
|
||||
// WikidataID returns the Wikidata item id (like "Q44190") that MusicBrainz links to
|
||||
// the artist, or "" when there is no usable link: that is not an error. Only the
|
||||
// first relation of type wikidata that has not ended and points at wikidata.org is
|
||||
// used; the id comes from MusicBrainz, never from a search by name.
|
||||
func (c *Client) WikidataID(ctx context.Context, id string) (string, error) {
|
||||
if !idRe.MatchString(id) {
|
||||
return "", ErrBadRequest
|
||||
}
|
||||
body, err := c.get(ctx, c.base+"artist/"+strings.ToLower(id)+"?inc=url-rels&fmt=json")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var doc struct {
|
||||
ID string `json:"id"`
|
||||
Relations []struct {
|
||||
Type string `json:"type"`
|
||||
Ended bool `json:"ended"`
|
||||
URL struct {
|
||||
Resource string `json:"resource"`
|
||||
} `json:"url"`
|
||||
} `json:"relations"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &doc); err != nil || doc.ID == "" {
|
||||
return "", ErrBadResponse
|
||||
}
|
||||
for _, r := range doc.Relations {
|
||||
if r.Type != "wikidata" || r.Ended {
|
||||
continue
|
||||
}
|
||||
if m := wikidataURLRe.FindStringSubmatch(r.URL.Resource); m != nil {
|
||||
return m[1], nil
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// Probe asks for one public artist, to find out whether the service answers. It
|
||||
// is a providers.Checker, and answers with the errors the settings understand.
|
||||
func (c *Client) Probe(ctx context.Context) error {
|
||||
|
|
|
|||
|
|
@ -159,3 +159,62 @@ func TestProbeSpeaksTheLanguageOfTheSettings(t *testing.T) {
|
|||
t.Errorf("rate limited: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
const relsBody = `{"id":"11111111-2222-4333-8444-555555555555","name":"Synthetic Band",
|
||||
"relations":[
|
||||
{"type":"official homepage","target-type":"url","url":{"resource":"https://example.org"}},
|
||||
{"type":"wikidata","target-type":"url","ended":false,"url":{"resource":"https://www.wikidata.org/wiki/Q44190"}},
|
||||
{"type":"discogs","target-type":"url","url":{"resource":"https://www.discogs.com/artist/1"}}]}`
|
||||
|
||||
func TestWikidataIDReadsTheWikidataRelation(t *testing.T) {
|
||||
s := newServer(t, 200, relsBody)
|
||||
got, err := s.client().WikidataID(context.Background(), mbid)
|
||||
if err != nil || got != "Q44190" {
|
||||
t.Fatalf("id = %q err = %v, want Q44190", got, err)
|
||||
}
|
||||
reqs := s.requests()
|
||||
if len(reqs) != 1 || reqs[0] != "/ws/2/artist/"+mbid+"?inc=url-rels&fmt=json" {
|
||||
t.Errorf("requests = %v: only the id and the literal inc leave", reqs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWikidataIDIsEmptyWhenThereIsNoRelationOrItIsNotUsable(t *testing.T) {
|
||||
for name, body := range map[string]string{
|
||||
"no relations": `{"id":"11111111-2222-4333-8444-555555555555","relations":[]}`,
|
||||
"no relations key": `{"id":"11111111-2222-4333-8444-555555555555"}`,
|
||||
"another host": `{"id":"x","relations":[{"type":"wikidata","url":{"resource":"https://evil.example/wiki/Q5"}}]}`,
|
||||
"not an item id": `{"id":"x","relations":[{"type":"wikidata","url":{"resource":"https://www.wikidata.org/wiki/Property:P18"}}]}`,
|
||||
"a huge number": `{"id":"x","relations":[{"type":"wikidata","url":{"resource":"https://www.wikidata.org/wiki/Q99999999999999999999"}}]}`,
|
||||
"a wrong type only": `{"id":"x","relations":[{"type":"wikipedia","url":{"resource":"https://en.wikipedia.org/wiki/X"}}]}`,
|
||||
"an ended one": `{"id":"x","relations":[{"type":"wikidata","ended":true,"url":{"resource":"https://www.wikidata.org/wiki/Q5"}}]}`,
|
||||
} {
|
||||
got, err := newServer(t, 200, body).client().WikidataID(context.Background(), mbid)
|
||||
if err != nil || got != "" {
|
||||
t.Errorf("%s: id = %q err = %v, want nothing and no error", name, got, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWikidataIDTakesTheFirstUsableRelation(t *testing.T) {
|
||||
body := `{"id":"x","relations":[
|
||||
{"type":"wikidata","ended":true,"url":{"resource":"https://www.wikidata.org/wiki/Q1"}},
|
||||
{"type":"wikidata","url":{"resource":"https://www.wikidata.org/wiki/Q2"}},
|
||||
{"type":"wikidata","url":{"resource":"https://www.wikidata.org/wiki/Q3"}}]}`
|
||||
if got, _ := newServer(t, 200, body).client().WikidataID(context.Background(), mbid); got != "Q2" {
|
||||
t.Errorf("id = %q, want Q2", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWikidataIDErrors(t *testing.T) {
|
||||
if _, err := newServer(t, 200, relsBody).client().WikidataID(context.Background(), "not an id"); !errors.Is(err, ErrBadRequest) {
|
||||
t.Errorf("a bad id: err = %v", err)
|
||||
}
|
||||
for status, want := range map[int]error{404: ErrNotFound, 429: ErrRateLimited, 503: ErrUnavailable} {
|
||||
if _, err := newServer(t, status, `{"error":"x"}`).client().WikidataID(context.Background(), mbid); !errors.Is(err, want) {
|
||||
t.Errorf("status %d: err = %v, want %v", status, err, want)
|
||||
}
|
||||
}
|
||||
if _, err := newServer(t, 200, `not json`).client().WikidataID(context.Background(), mbid); !errors.Is(err, ErrBadResponse) {
|
||||
t.Errorf("not json: err = %v", err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ import (
|
|||
const (
|
||||
LastFM = "lastfm"
|
||||
MusicBrainz = "musicbrainz"
|
||||
Wikipedia = "wikipedia"
|
||||
)
|
||||
|
||||
// Errors returned by the service.
|
||||
|
|
@ -105,6 +106,16 @@ var known = []meta{
|
|||
note: "Core data (artist names and ids) is public domain. Tags and genres are CC BY-NC-SA 3.0: " +
|
||||
"non-commercial, with attribution to MusicBrainz.",
|
||||
},
|
||||
{
|
||||
name: Wikipedia,
|
||||
sends: "It needs the MusicBrainz provider on too. The MusicBrainz id of the artist you look up goes to MusicBrainz " +
|
||||
"(never a name: Sonda does not search), with the contact set for MusicBrainz; the Wikidata item that " +
|
||||
"MusicBrainz links to it goes to Wikidata, and the title of the Wikipedia page that Wikidata names goes to " +
|
||||
"Wikipedia, with the contact below in the User-Agent. Your IP address goes to all three. Never your users, " +
|
||||
"your listens, track titles or file paths.",
|
||||
note: "Wikipedia text is CC BY-SA 4.0: show the name of the page, a link to it and the licence next to " +
|
||||
"the text, and say if you changed it.",
|
||||
},
|
||||
}
|
||||
|
||||
func lookup(name string) (meta, bool) {
|
||||
|
|
@ -360,6 +371,7 @@ func (s *Service) UserAgentFor(name, version string) func() string {
|
|||
type Purged struct {
|
||||
SimilarArtists int // lists of similar artists
|
||||
Tags int // tags
|
||||
Descriptions int // descriptions
|
||||
Artists int // artists that only the removed data named
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -53,8 +53,8 @@ func TestEveryProviderIsOffByDefault(t *testing.T) {
|
|||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(infos) != 2 || infos[0].Name != LastFM || infos[1].Name != MusicBrainz {
|
||||
t.Fatalf("providers = %+v, want lastfm then musicbrainz", infos)
|
||||
if len(infos) != 3 || infos[0].Name != LastFM || infos[1].Name != MusicBrainz || infos[2].Name != Wikipedia {
|
||||
t.Fatalf("providers = %+v, want lastfm, musicbrainz, then wikipedia", infos)
|
||||
}
|
||||
for _, in := range infos {
|
||||
if in.Enabled || in.HasAPIKey {
|
||||
|
|
@ -222,7 +222,7 @@ func TestListMergesStoredRowsInAFixedOrder(t *testing.T) {
|
|||
st.rows[MusicBrainz] = Settings{Provider: MusicBrainz, Enabled: true, Contact: "c", UpdatedAt: time.Unix(1, 0)}
|
||||
st.rows["stale-unknown"] = Settings{Provider: "stale-unknown", Enabled: true} // a row of a provider that no longer exists
|
||||
infos, err := s.List(context.Background())
|
||||
if err != nil || len(infos) != 2 {
|
||||
if err != nil || len(infos) != 3 {
|
||||
t.Fatalf("infos = %+v err=%v, want exactly the known providers", infos, err)
|
||||
}
|
||||
if infos[0].Name != LastFM || infos[1].Name != MusicBrainz || !infos[1].Enabled || infos[1].Contact != "c" {
|
||||
|
|
@ -408,3 +408,42 @@ func TestPurgeErrors(t *testing.T) {
|
|||
t.Errorf("a storage error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWikipediaIsAProviderThatNeedsNoKey(t *testing.T) {
|
||||
svc, _ := newSvc()
|
||||
infos, err := svc.List(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var wp *Info
|
||||
for i := range infos {
|
||||
if infos[i].Name == Wikipedia {
|
||||
wp = &infos[i]
|
||||
}
|
||||
}
|
||||
if wp == nil {
|
||||
t.Fatalf("providers = %+v, want wikipedia among them", infos)
|
||||
}
|
||||
if wp.Enabled || wp.NeedsAPIKey || wp.Sends == "" || wp.Notes == "" {
|
||||
t.Errorf("wikipedia = %+v: off by default, no key, and it says what is sent and under which licence", wp)
|
||||
}
|
||||
// Off until an admin switches it on; no key needed to do so.
|
||||
if _, active, _ := svc.Active(context.Background(), Wikipedia); active {
|
||||
t.Error("active before anyone switched it on")
|
||||
}
|
||||
on := true
|
||||
if _, err := svc.Update(context.Background(), Wikipedia, Patch{Enabled: &on}); err != nil {
|
||||
t.Fatalf("enable: %v", err)
|
||||
}
|
||||
if _, active, _ := svc.Active(context.Background(), Wikipedia); !active {
|
||||
t.Error("not active after being switched on")
|
||||
}
|
||||
key := "0123456789abcdef0123456789abcdef"
|
||||
if _, err := svc.Update(context.Background(), Wikipedia, Patch{APIKey: &key}); !errors.Is(err, ErrInvalidSettings) {
|
||||
t.Errorf("a key for a provider that has none: err = %v", err)
|
||||
}
|
||||
contact := "https://example.org/me"
|
||||
if info, err := svc.Update(context.Background(), Wikipedia, Patch{Contact: &contact}); err != nil || info.Contact != contact {
|
||||
t.Errorf("contact: %+v err=%v", info, err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue