feat(about): artist descriptions from Wikipedia
- GET /artists/{id}/description shows what Wikipedia says about an artist: a short
text with its page, a link and the licence (CC BY-SA 4.0).
- New provider "wikipedia" (off by default, no key, contact in the User-Agent, which
Wikimedia requires). The way to the text is a chain of ids, never a name: the
artist's MusicBrainz id, the Wikidata item MusicBrainz links to it, the Wikipedia
page Wikidata names in the first configured language that has one. A page is used
only if it is an article about that same item.
- [wikipedia] languages in the config (one to five codes of two or three lower case
letters, checked before they are sent: an unknown site makes Wikidata answer with
every language).
- Stored 30 days (a day when there was nothing to find), served stale if Wikipedia
fails, removed with the provider's data (the purge now reports descriptions).
- MusicBrainz client: WikidataID. Wikipedia client for Wikidata and the summary,
which also reads Wikidata's errors that come with HTTP 200.
- artist_descriptions table (migration 0014, SQLite and PostgreSQL).
- Spec, contract test, README and example config updated.
Closes #39
This commit is contained in:
parent
c91b88a83e
commit
48cb903115
1 changed files with 2124 additions and 19 deletions
|
|
@ -14,6 +14,7 @@ import (
|
|||
"github.com/gin-gonic/gin"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"codeberg.org/squidd/sonda/internal/about"
|
||||
httpapi "codeberg.org/squidd/sonda/internal/api/http"
|
||||
"codeberg.org/squidd/sonda/internal/auth"
|
||||
"codeberg.org/squidd/sonda/internal/catalog"
|
||||
|
|
@ -28,6 +29,7 @@ import (
|
|||
"codeberg.org/squidd/sonda/internal/providers/lastfm"
|
||||
"codeberg.org/squidd/sonda/internal/providers/musicbrainz"
|
||||
"codeberg.org/squidd/sonda/internal/providers/outbound"
|
||||
"codeberg.org/squidd/sonda/internal/providers/wikipedia"
|
||||
"codeberg.org/squidd/sonda/internal/storage"
|
||||
"codeberg.org/squidd/sonda/internal/streaming"
|
||||
"codeberg.org/squidd/sonda/internal/taste"
|
||||
|
|
@ -46,6 +48,13 @@ const lastfmMinInterval = 250 * time.Millisecond
|
|||
// musicbrainzMinInterval keeps a little above MusicBrainz's rule of one request a second.
|
||||
const musicbrainzMinInterval = 1100 * time.Millisecond
|
||||
|
||||
// wikipediaMinInterval is polite: Wikimedia asks only for serial requests, and a
|
||||
// description costs two of them. wikipediaMaxBody is far above any answer used here.
|
||||
const (
|
||||
wikipediaMinInterval = 250 * time.Millisecond
|
||||
wikipediaMaxBody = 1 << 20
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := run(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "sondad:", err)
|
||||
|
|
@ -132,6 +141,16 @@ func run() error {
|
|||
providerSettings.SetChecker(providers.MusicBrainz, func(ctx context.Context, _ string) error {
|
||||
return musicbrainzClient.Probe(ctx)
|
||||
})
|
||||
// Wikipedia and Wikidata: asked only for the description of an artist, through ids.
|
||||
// Wikimedia answers 403 to a request without a User-Agent, and wants a contact in it.
|
||||
wikipediaClient := wikipedia.New(outbound.New(outbound.Options{
|
||||
UserAgentFunc: providerSettings.UserAgentFor(providers.Wikipedia, version),
|
||||
MinInterval: wikipediaMinInterval,
|
||||
MaxBody: wikipediaMaxBody,
|
||||
}), wikipedia.WikidataURL, wikipedia.SummaryURL)
|
||||
providerSettings.SetChecker(providers.Wikipedia, func(ctx context.Context, _ string) error {
|
||||
return wikipediaClient.Probe(ctx)
|
||||
})
|
||||
discoveryRepo := storage.NewDiscoveryRepo(db)
|
||||
providerSettings.SetPurger(func(ctx context.Context, name string) (providers.Purged, error) {
|
||||
return discoveryRepo.Purge(ctx, name)
|
||||
|
|
@ -170,8 +189,10 @@ func run() error {
|
|||
Discover: discover.NewService(tasteService, discoveries, tags, tasteRepo, userSettings, feedbacks, weights),
|
||||
Feedback: feedbacks,
|
||||
Images: images.NewService(cfg.Storage.DataDir, imageRepo, imageRepo),
|
||||
Settings: userSettings,
|
||||
Streams: streaming.NewService(libraryRepo),
|
||||
About: about.NewService(discoveryRepo, about.WikipediaSource{MusicBrainz: musicbrainzClient, Wikipedia: wikipediaClient},
|
||||
providerSettings, cfg.Wikipedia.Languages, logger),
|
||||
Settings: userSettings,
|
||||
Streams: streaming.NewService(libraryRepo),
|
||||
})
|
||||
|
||||
// No WriteTimeout: audio streaming (later) needs long responses.
|
||||
|
|
|
|||
Loading…
Reference in a new issue