Migrating hosting from Codeberg to git.kmalloc.org. Renames the module path in go.mod, every internal import, and the OCI image source label in the Dockerfile. Build and full test suite verified green after the rename.
318 lines
11 KiB
Go
318 lines
11 KiB
Go
// Package about tells what an artist is, with a short text from Wikipedia.
|
|
//
|
|
// The way to the text is an identifier chain, never a search by name: the artist's
|
|
// MusicBrainz id, then the Wikidata item that MusicBrainz links to it, then the
|
|
// Wikipedia page that Wikidata names in the user's languages. Only the id is
|
|
// trusted: the page is used only if it says it is about that same item.
|
|
//
|
|
// Like the other provider data, it is asked for only when the admin switched the
|
|
// provider on and someone asks, kept in the database (30 days; a day when there was
|
|
// nothing to find), served stale when Wikipedia fails, and removed with the provider's
|
|
// data. The text is CC BY-SA 4.0, so the page, its link and the licence are kept and
|
|
// returned with it.
|
|
package about
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
"unicode/utf8"
|
|
|
|
"go.uber.org/zap"
|
|
"golang.org/x/sync/singleflight"
|
|
|
|
"git.kmalloc.org/squid/sonda/internal/discovery"
|
|
"git.kmalloc.org/squid/sonda/internal/providers"
|
|
)
|
|
|
|
// Provider is the provider the descriptions come from.
|
|
const Provider = providers.Wikipedia
|
|
|
|
// What has to be shown with the text.
|
|
const (
|
|
Attribution = "Text from Wikipedia (https://www.wikipedia.org), available under the CC BY-SA 4.0 licence."
|
|
LicenseName = "CC BY-SA 4.0"
|
|
LicenseURL = "https://creativecommons.org/licenses/by-sa/4.0/"
|
|
)
|
|
|
|
// How long what was read is believed. A description changes slowly; "nothing found"
|
|
// is remembered shortly, in case a link was added since.
|
|
const (
|
|
PositiveTTL = 30 * 24 * time.Hour
|
|
NegativeTTL = 24 * time.Hour
|
|
)
|
|
|
|
// MaxTextLen is the longest text kept, in characters. A summary is a paragraph.
|
|
const MaxTextLen = 4000
|
|
|
|
// fetchTimeout bounds a shared read, which does not depend on whoever started it
|
|
// still waiting. It covers up to three requests, the first of them to MusicBrainz at
|
|
// one a second.
|
|
const fetchTimeout = 45 * time.Second
|
|
|
|
// Errors a Source returns.
|
|
var (
|
|
// ErrSourceNotFound means the provider does not know what was asked: an unknown
|
|
// artist, item or page. It is an answer, not a failure.
|
|
ErrSourceNotFound = errors.New("the provider does not know this")
|
|
// ErrSourceUnavailable stands for any failure to get an answer.
|
|
ErrSourceUnavailable = errors.New("the provider could not be reached")
|
|
)
|
|
|
|
// Description is what Wikipedia says about an artist, with where it comes from.
|
|
type Description struct {
|
|
Text string // the first paragraph of the article, plain text
|
|
Short string // one line, like "English rock band"; may be empty
|
|
Language string // of the page: "it", "en"
|
|
Title string // of the page
|
|
URL string // of the page, on Wikipedia
|
|
}
|
|
|
|
// Snapshot is what is stored for one artist. Fetched is false when the provider was
|
|
// never asked; None when it was and there was nothing to find.
|
|
type Snapshot struct {
|
|
Fetched bool
|
|
None bool
|
|
RetrievedAt time.Time
|
|
Description Description
|
|
}
|
|
|
|
// Result is the answer to Description. RetrievedAt is zero when nothing is known; Stale
|
|
// means the description is older than it should be because Wikipedia could not be asked.
|
|
type Result struct {
|
|
Status string // discovery.StatusOK, StatusDisabled, StatusError or StatusNoID
|
|
Stale bool
|
|
RetrievedAt time.Time
|
|
Description *Description // nil when there is none
|
|
}
|
|
|
|
// Store keeps the descriptions. Artist returns discovery.ErrNotFound for an id the
|
|
// catalog does not know, and so does SaveDescription for an artist removed meanwhile.
|
|
type Store interface {
|
|
Artist(ctx context.Context, id int64) (discovery.Artist, error)
|
|
Description(ctx context.Context, id int64) (Snapshot, error)
|
|
// SaveDescription replaces what is stored; a nil description records that there is
|
|
// none. wikidataID is the item MusicBrainz linked, or "".
|
|
SaveDescription(ctx context.Context, id int64, at time.Time, wikidataID string, d *Description) error
|
|
}
|
|
|
|
// Summary is what a Wikipedia page says about itself.
|
|
type Summary struct {
|
|
Type string
|
|
Title string
|
|
Lang string
|
|
Extract string
|
|
Description string
|
|
URL string
|
|
WikibaseItem string
|
|
}
|
|
|
|
// Source follows the chain of identifiers. Every method returns ErrSourceNotFound for
|
|
// what the provider does not know, and any other error for a failure.
|
|
type Source interface {
|
|
// WikidataID returns the Wikidata item MusicBrainz links to the artist, or "".
|
|
WikidataID(ctx context.Context, musicbrainzID string) (string, error)
|
|
// Pages returns the title of the page of the item in each of the languages that has
|
|
// one, and the item they belong to: the same, or the one it was merged into.
|
|
Pages(ctx context.Context, item string, languages []string) (string, map[string]string, error)
|
|
Summary(ctx context.Context, language, title string) (Summary, error)
|
|
}
|
|
|
|
// Service answers "what is this artist".
|
|
type Service struct {
|
|
store Store
|
|
source Source
|
|
settings discovery.Settings
|
|
languages []string
|
|
log *zap.Logger
|
|
now func() time.Time
|
|
flight singleflight.Group
|
|
}
|
|
|
|
// NewService builds a Service. languages are the Wikipedia languages to try, in order.
|
|
func NewService(store Store, source Source, settings discovery.Settings, languages []string, log *zap.Logger) *Service {
|
|
if log == nil {
|
|
log = zap.NewNop()
|
|
}
|
|
return &Service{store: store, source: source, settings: settings, languages: languages, log: log, now: time.Now}
|
|
}
|
|
|
|
// Description returns the description of an artist. A provider that is off, cannot be
|
|
// asked or fails is not an error: the answer says so and carries what was stored.
|
|
func (s *Service) Description(ctx context.Context, artistID int64) (Result, error) {
|
|
return s.describe(ctx, artistID, false)
|
|
}
|
|
|
|
// Ensure is Description for the job that goes through the library: with refresh it reads
|
|
// again even a description that is still good. Whatever goes wrong is in the status of the
|
|
// answer, which then still carries what was stored: a refresh never replaces a description
|
|
// by a failure.
|
|
func (s *Service) Ensure(ctx context.Context, artistID int64, refresh bool) (Result, error) {
|
|
return s.describe(ctx, artistID, refresh)
|
|
}
|
|
|
|
func (s *Service) describe(ctx context.Context, artistID int64, force bool) (Result, error) {
|
|
if artistID <= 0 {
|
|
return Result{}, discovery.ErrNotFound
|
|
}
|
|
artist, err := s.store.Artist(ctx, artistID)
|
|
if err != nil {
|
|
return Result{}, err
|
|
}
|
|
snap, err := s.store.Description(ctx, artistID)
|
|
if err != nil {
|
|
return Result{}, err
|
|
}
|
|
fresh := !force && snap.Fetched && s.fresh(snap)
|
|
active, err := s.active(ctx)
|
|
if err != nil {
|
|
return Result{}, err
|
|
}
|
|
switch {
|
|
case !active:
|
|
return result(snap, discovery.StatusDisabled, snap.Fetched && !fresh), nil
|
|
case fresh:
|
|
return result(snap, discovery.StatusOK, false), nil
|
|
}
|
|
if artist.MBID == "" { // Sonda never looks an id up by name
|
|
return result(snap, discovery.StatusNoID, snap.Fetched), nil
|
|
}
|
|
|
|
// One read per artist, however many ask at once, on its own context.
|
|
done := s.flight.DoChan(fmt.Sprint(artistID), func() (_ any, err error) {
|
|
defer func() {
|
|
if r := recover(); r != nil {
|
|
err = fmt.Errorf("reading the description panicked: %v", r)
|
|
}
|
|
}()
|
|
return nil, s.refresh(artist)
|
|
})
|
|
select {
|
|
case <-ctx.Done():
|
|
return Result{}, ctx.Err()
|
|
case r := <-done:
|
|
if errors.Is(r.Err, discovery.ErrNotFound) {
|
|
return Result{}, discovery.ErrNotFound // removed while it was being read
|
|
}
|
|
if errors.Is(r.Err, errSwitchedOff) {
|
|
return result(snap, discovery.StatusDisabled, snap.Fetched), nil
|
|
}
|
|
if r.Err != nil {
|
|
s.log.Warn("the description could not be read", zap.String("provider", Provider), zap.Error(r.Err))
|
|
return result(snap, discovery.StatusError, snap.Fetched), nil
|
|
}
|
|
}
|
|
snap, err = s.store.Description(ctx, artistID)
|
|
if err != nil {
|
|
return Result{}, err
|
|
}
|
|
return result(snap, discovery.StatusOK, false), nil
|
|
}
|
|
|
|
// errSwitchedOff is what a read returns when the providers were switched off while it ran.
|
|
var errSwitchedOff = errors.New("the provider was switched off")
|
|
|
|
// active says whether the chain may be followed: Wikipedia is on, and so is MusicBrainz,
|
|
// which gives the link to Wikidata. Without that second switch nothing would stop the
|
|
// artist's id going to a provider the admin never allowed.
|
|
func (s *Service) active(ctx context.Context) (bool, error) {
|
|
for _, name := range []string{Provider, providers.MusicBrainz} {
|
|
_, on, err := s.settings.Active(ctx, name)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if !on {
|
|
return false, nil
|
|
}
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
func (s *Service) fresh(snap Snapshot) bool {
|
|
ttl := PositiveTTL
|
|
if snap.None {
|
|
ttl = NegativeTTL
|
|
}
|
|
return s.now().Sub(snap.RetrievedAt) < ttl
|
|
}
|
|
|
|
func result(snap Snapshot, status string, stale bool) Result {
|
|
r := Result{Status: status, Stale: stale && snap.Fetched}
|
|
if snap.Fetched {
|
|
r.RetrievedAt = snap.RetrievedAt
|
|
if !snap.None {
|
|
d := snap.Description
|
|
r.Description = &d
|
|
}
|
|
}
|
|
return r
|
|
}
|
|
|
|
// refresh follows the chain and stores what it finds. A failure is returned and not
|
|
// stored: the next request tries again, and the language that failed is not replaced
|
|
// by the next one.
|
|
func (s *Service) refresh(a discovery.Artist) error {
|
|
ctx, cancel := context.WithTimeout(context.Background(), fetchTimeout)
|
|
defer cancel()
|
|
|
|
item, err := s.source.WikidataID(ctx, a.MBID)
|
|
if err != nil && !errors.Is(err, ErrSourceNotFound) {
|
|
return err
|
|
}
|
|
if item == "" || err != nil {
|
|
return s.save(ctx, a.ID, "", nil)
|
|
}
|
|
resolved, pages, err := s.source.Pages(ctx, item, s.languages)
|
|
if errors.Is(err, ErrSourceNotFound) {
|
|
return s.save(ctx, a.ID, item, nil)
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, lang := range s.languages {
|
|
title := pages[lang]
|
|
if title == "" {
|
|
continue
|
|
}
|
|
sum, err := s.source.Summary(ctx, lang, title)
|
|
if errors.Is(err, ErrSourceNotFound) {
|
|
continue
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
text := strings.TrimSpace(sum.Extract)
|
|
// An article about this very item (the one the pages belong to, which is the
|
|
// target when the item was merged), with some text, and an address to credit.
|
|
// Anything else (a page of disambiguation, a redirect to another subject, a page
|
|
// with no text) is not a description.
|
|
if sum.Type != "standard" || sum.WikibaseItem != resolved || text == "" || sum.URL == "" {
|
|
continue
|
|
}
|
|
return s.save(ctx, a.ID, resolved, &Description{
|
|
Text: cut(text), Short: sum.Description, Language: lang, Title: title, URL: sum.URL,
|
|
})
|
|
}
|
|
return s.save(ctx, a.ID, resolved, nil)
|
|
}
|
|
|
|
// save stores the outcome, unless the providers were switched off since the read began:
|
|
// an admin who took the data back and switched it off must not see it come back.
|
|
func (s *Service) save(ctx context.Context, id int64, item string, d *Description) error {
|
|
if on, err := s.active(ctx); err != nil {
|
|
return err
|
|
} else if !on {
|
|
return errSwitchedOff
|
|
}
|
|
return s.store.SaveDescription(ctx, id, s.now(), item, d)
|
|
}
|
|
|
|
// cut keeps at most MaxTextLen characters and marks a text that was shortened.
|
|
func cut(s string) string {
|
|
if utf8.RuneCountInString(s) <= MaxTextLen {
|
|
return s
|
|
}
|
|
return string([]rune(s)[:MaxTextLen-1]) + "…" // the mark counts: never more than MaxTextLen
|
|
}
|