Apps can play and browse: stream and download send the original file (Range, HEAD), getCoverArt serves the cover of an album, an artist or the album of a song (300 pixels or less is the small one), getAlbumList2 knows every kind of list, getRandomSongs and getSongsByGenre list songs, search3 finds artists, albums and songs, and the starred calls answer with nothing so apps do not break. - catalog.Filter has IDs to read albums and songs by id - storage.SubsonicRepo answers the album and song lists (random, newest, frequent, recent, by name, by artist, by year, by genre)
391 lines
17 KiB
Go
391 lines
17 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"net/http"
|
|
"os"
|
|
"os/signal"
|
|
"syscall"
|
|
"time"
|
|
_ "time/tzdata" // the time zones that browsers send, also on a machine that has none
|
|
|
|
"github.com/gin-gonic/gin"
|
|
"go.uber.org/zap"
|
|
|
|
"git.kmalloc.org/squid/sonda/internal/about"
|
|
"git.kmalloc.org/squid/sonda/internal/affinity"
|
|
httpapi "git.kmalloc.org/squid/sonda/internal/api/http"
|
|
"git.kmalloc.org/squid/sonda/internal/auth"
|
|
"git.kmalloc.org/squid/sonda/internal/catalog"
|
|
"git.kmalloc.org/squid/sonda/internal/config"
|
|
"git.kmalloc.org/squid/sonda/internal/devices"
|
|
"git.kmalloc.org/squid/sonda/internal/discover"
|
|
"git.kmalloc.org/squid/sonda/internal/discovery"
|
|
"git.kmalloc.org/squid/sonda/internal/duplicates"
|
|
"git.kmalloc.org/squid/sonda/internal/federation"
|
|
"git.kmalloc.org/squid/sonda/internal/feedback"
|
|
"git.kmalloc.org/squid/sonda/internal/images"
|
|
"git.kmalloc.org/squid/sonda/internal/jobs"
|
|
"git.kmalloc.org/squid/sonda/internal/library"
|
|
"git.kmalloc.org/squid/sonda/internal/listening"
|
|
"git.kmalloc.org/squid/sonda/internal/mastodon"
|
|
"git.kmalloc.org/squid/sonda/internal/mixes"
|
|
"git.kmalloc.org/squid/sonda/internal/naming"
|
|
"git.kmalloc.org/squid/sonda/internal/opensubsonic"
|
|
"git.kmalloc.org/squid/sonda/internal/playlists"
|
|
"git.kmalloc.org/squid/sonda/internal/providers"
|
|
"git.kmalloc.org/squid/sonda/internal/providers/anthropic"
|
|
"git.kmalloc.org/squid/sonda/internal/providers/commons"
|
|
"git.kmalloc.org/squid/sonda/internal/providers/coverart"
|
|
"git.kmalloc.org/squid/sonda/internal/providers/lastfm"
|
|
"git.kmalloc.org/squid/sonda/internal/providers/lidarr"
|
|
"git.kmalloc.org/squid/sonda/internal/providers/musicbrainz"
|
|
"git.kmalloc.org/squid/sonda/internal/providers/outbound"
|
|
"git.kmalloc.org/squid/sonda/internal/providers/wikipedia"
|
|
"git.kmalloc.org/squid/sonda/internal/smart"
|
|
"git.kmalloc.org/squid/sonda/internal/spotifyimport"
|
|
"git.kmalloc.org/squid/sonda/internal/stats"
|
|
"git.kmalloc.org/squid/sonda/internal/storage"
|
|
"git.kmalloc.org/squid/sonda/internal/streaming"
|
|
"git.kmalloc.org/squid/sonda/internal/taste"
|
|
"git.kmalloc.org/squid/sonda/internal/usersettings"
|
|
"git.kmalloc.org/squid/sonda/internal/waveform"
|
|
"git.kmalloc.org/squid/sonda/internal/webui"
|
|
"git.kmalloc.org/squid/sonda/migrations"
|
|
)
|
|
|
|
const defaultConfigFile = "sonda.toml"
|
|
|
|
// version goes into the User-Agent sent to providers.
|
|
const version = "0.1.0"
|
|
|
|
// lastfmMinInterval spaces the requests to Last.fm well below its limit.
|
|
const lastfmMinInterval = 250 * time.Millisecond
|
|
|
|
// musicbrainzMinInterval keeps a little above MusicBrainz's rule of one request a second.
|
|
const musicbrainzMinInterval = 1100 * time.Millisecond
|
|
|
|
// wikipediaMinInterval is polite: Wikimedia asks only for serial requests, and a
|
|
// description costs two of them. wikipediaMaxBody is far above any answer used here.
|
|
const (
|
|
wikipediaMinInterval = 250 * time.Millisecond
|
|
wikipediaMaxBody = 1 << 20
|
|
)
|
|
|
|
// commonsMaxBody leaves room for a 960 pixel copy of a picture, which is some hundred kilobytes.
|
|
const (
|
|
coverartMinInterval = 500 * time.Millisecond
|
|
commonsMinInterval = 300 * time.Millisecond
|
|
commonsAPIMaxBody = 1 << 20
|
|
commonsMaxBody = 8 << 20
|
|
)
|
|
|
|
func main() {
|
|
if err := run(); err != nil {
|
|
fmt.Fprintln(os.Stderr, "sondad:", err)
|
|
os.Exit(1)
|
|
}
|
|
}
|
|
|
|
func run() error {
|
|
configPath := flag.String("config", "", "path to the TOML config file (default ./"+defaultConfigFile+" if present)")
|
|
flag.Parse()
|
|
|
|
path := *configPath
|
|
if path == "" {
|
|
if _, err := os.Stat(defaultConfigFile); err == nil {
|
|
path = defaultConfigFile
|
|
}
|
|
}
|
|
|
|
cfg, err := config.Load(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := cfg.Validate(); err != nil {
|
|
return fmt.Errorf("invalid config: %w", err)
|
|
}
|
|
|
|
logger, err := newLogger(cfg.Log.Env)
|
|
if err != nil {
|
|
return fmt.Errorf("create logger: %w", err)
|
|
}
|
|
defer func() { _ = logger.Sync() }()
|
|
|
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
|
defer stop()
|
|
|
|
target := cfg.Database.Path
|
|
if cfg.Database.Driver == storage.DriverPostgres {
|
|
target = cfg.Database.DSN
|
|
}
|
|
db, err := storage.Open(cfg.Database.Driver, target)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer func() { _ = db.Close() }()
|
|
|
|
dbMigrations, err := migrations.For(cfg.Database.Driver)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := storage.Migrate(ctx, db, dbMigrations); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Composition root: build services and hand them to the HTTP layer.
|
|
tokens := auth.NewTokenIssuer(cfg.Auth.JWTSecret, cfg.Auth.AccessTTL())
|
|
if cfg.Log.Env == "production" {
|
|
gin.SetMode(gin.ReleaseMode)
|
|
}
|
|
libraryRepo := storage.NewLibraryRepo(db)
|
|
// On the first start after an upgrade that adds a key this can take a while on a big library.
|
|
logger.Info("refreshing the match keys of the library")
|
|
keysStarted := time.Now()
|
|
if err := libraryRepo.RefreshMatchKeys(ctx); err != nil {
|
|
return err
|
|
}
|
|
logger.Info("match keys are up to date", zap.Duration("took", time.Since(keysStarted)))
|
|
scanner := library.NewScanner(libraryRepo, library.TaglibReader{}, library.NewHub(), cfg.Library.ScanWorkers)
|
|
scans := library.NewManager(scanner, libraryRepo)
|
|
logScanResults(logger, scans)
|
|
|
|
libraries := library.NewLibraries(libraryRepo)
|
|
libraries.SetBusy(func() bool { return scans.Status().Running })
|
|
warnAboutLibraries(ctx, logger, libraries)
|
|
|
|
// External providers: nothing is contacted unless an admin switched one on.
|
|
providerSettings := providers.NewService(storage.NewProviderRepo(db))
|
|
lastfmClient := lastfm.New(outbound.New(outbound.Options{
|
|
UserAgent: providers.UserAgent(version, ""),
|
|
MinInterval: lastfmMinInterval,
|
|
}), lastfm.BaseURL)
|
|
providerSettings.SetChecker(providers.LastFM, lastfmClient.Probe)
|
|
// MusicBrainz wants a contact in the User-Agent, which an admin can change: it
|
|
// is read again for every request.
|
|
musicbrainzClient := musicbrainz.New(outbound.New(outbound.Options{
|
|
UserAgentFunc: providerSettings.UserAgentFor(providers.MusicBrainz, version),
|
|
MinInterval: musicbrainzMinInterval,
|
|
FollowRedirect: musicbrainz.RedirectAllowed, // an id that was merged leads to the new one, and to nothing else
|
|
}), musicbrainz.BaseURL)
|
|
providerSettings.SetChecker(providers.MusicBrainz, func(ctx context.Context, _ string) error {
|
|
return musicbrainzClient.Probe(ctx)
|
|
})
|
|
// Wikipedia and Wikidata: asked only for the description of an artist, through ids.
|
|
// Wikimedia answers 403 to a request without a User-Agent, and wants a contact in it.
|
|
wikipediaClient := wikipedia.New(outbound.New(outbound.Options{
|
|
UserAgentFunc: providerSettings.UserAgentFor(providers.Wikipedia, version),
|
|
MinInterval: wikipediaMinInterval,
|
|
MaxBody: wikipediaMaxBody,
|
|
}), wikipedia.WikidataURL, wikipedia.SummaryURL)
|
|
providerSettings.SetChecker(providers.Wikipedia, func(ctx context.Context, _ string) error {
|
|
return wikipediaClient.Probe(ctx)
|
|
})
|
|
// Pictures of artists from Wikimedia Commons, through Wikidata, under free licences only.
|
|
// The pictures are downloaded through a client of their own: they are far bigger than an
|
|
// answer, and a picture host that misbehaves must not open the breaker of the API.
|
|
commonsClient := commons.New(outbound.New(outbound.Options{
|
|
UserAgentFunc: providerSettings.UserAgentFor(providers.Commons, version),
|
|
MinInterval: commonsMinInterval,
|
|
MaxBody: commonsAPIMaxBody,
|
|
}), commons.WikidataURL, commons.CommonsURL).WithDownloader(outbound.New(outbound.Options{
|
|
UserAgentFunc: providerSettings.UserAgentFor(providers.Commons, version),
|
|
MinInterval: commonsMinInterval,
|
|
MaxBody: commonsMaxBody,
|
|
}))
|
|
providerSettings.SetChecker(providers.Commons, func(ctx context.Context, _ string) error {
|
|
return commonsClient.Probe(ctx)
|
|
})
|
|
if err := os.MkdirAll(cfg.Storage.DataDir, 0o700); err != nil {
|
|
return fmt.Errorf("create the data folder: %w", err)
|
|
}
|
|
imageRepo := storage.NewImageRepo(db)
|
|
pictures := images.NewService(cfg.Storage.DataDir, imageRepo, imageRepo)
|
|
discoveryRepo := storage.NewDiscoveryRepo(db)
|
|
pictures.SetArtistSource(&images.CommonsSource{
|
|
Artists: discoveryRepo, Settings: providerSettings, MusicBrainz: musicbrainzClient, Commons: commonsClient,
|
|
})
|
|
// Covers from the Cover Art Archive when the user's own files have none. The archive
|
|
// answers with redirects (to archive.org and its data nodes): they are followed, to nobody else.
|
|
coverartClient := coverart.New(outbound.New(outbound.Options{
|
|
UserAgentFunc: providerSettings.UserAgentFor(providers.CoverArtArchive, version),
|
|
MinInterval: coverartMinInterval,
|
|
MaxBody: commonsMaxBody,
|
|
FollowRedirect: coverart.RedirectAllowed,
|
|
}), coverart.BaseURL)
|
|
providerSettings.SetChecker(providers.CoverArtArchive, func(ctx context.Context, _ string) error {
|
|
return coverartClient.Probe(ctx)
|
|
})
|
|
pictures.SetAlbumRemote(&images.CoverArtSource{Albums: imageRepo, Settings: providerSettings, Client: coverartClient, Finder: musicbrainzClient})
|
|
// Lidarr is the admin's own server: it is read from the stored settings for every request, on or off,
|
|
// since setting it up (a test, what it offers) happens before it is switched on.
|
|
lidarrClient := lidarr.New(func(ctx context.Context) (lidarr.Config, error) {
|
|
s, err := providerSettings.Stored(ctx, providers.Lidarr)
|
|
if err != nil {
|
|
return lidarr.Config{}, err
|
|
}
|
|
if s.URL == "" || s.APIKey == "" {
|
|
return lidarr.Config{}, lidarr.ErrNotConfigured
|
|
}
|
|
return lidarr.Config{URL: s.URL, APIKey: s.APIKey}, nil
|
|
}, providerSettings.UserAgentFor(providers.Lidarr, version), nil)
|
|
providerSettings.SetChecker(providers.Lidarr, func(ctx context.Context, _ string) error { return lidarrClient.Probe(ctx) })
|
|
providerSettings.SetPurger(func(ctx context.Context, name string) (providers.Purged, error) {
|
|
switch name {
|
|
case providers.Commons:
|
|
pictures.BeginPurge() // a picture being read right now must not be kept
|
|
out, err := discoveryRepo.Purge(ctx, name)
|
|
if err != nil {
|
|
return out, err
|
|
}
|
|
// The records are gone; the pictures themselves are files in the data folder.
|
|
if _, err := pictures.RemoveArtistFiles(); err != nil {
|
|
return out, fmt.Errorf("remove the pictures: %w", err)
|
|
}
|
|
return out, nil
|
|
case providers.CoverArtArchive:
|
|
pictures.BeginPurge()
|
|
out, err := discoveryRepo.Purge(ctx, name)
|
|
if err != nil {
|
|
return out, err
|
|
}
|
|
// The covers that came from the archive have a folder of their own: all of it goes.
|
|
if _, err := pictures.RemoveRemoteAlbumFiles(); err != nil {
|
|
return out, fmt.Errorf("remove the covers: %w", err)
|
|
}
|
|
return out, nil
|
|
}
|
|
return discoveryRepo.Purge(ctx, name)
|
|
})
|
|
// Claude names playlists when an admin turned the provider on with a key. Redirects are never followed there.
|
|
claudeClient := anthropic.New()
|
|
namingRepo := storage.NewNamingRepo(db)
|
|
namingService := naming.NewService(providerSettings, claudeClient, namingRepo, logger)
|
|
namingService.SetLedger(namingRepo) // every question to Claude is counted, to show what it costs
|
|
providerSettings.SetChecker(providers.Anthropic, claudeClient.Probe)
|
|
descriptions := about.NewService(discoveryRepo, about.WikipediaSource{MusicBrainz: musicbrainzClient, Wikipedia: wikipediaClient},
|
|
providerSettings, cfg.Wikipedia.Languages, logger)
|
|
discoveries := discovery.NewService(discoveryRepo, lastfmClient, providerSettings, logger)
|
|
tags := discovery.NewTagService(discoveryRepo, map[string]discovery.TagSource{
|
|
providers.LastFM: discovery.LastfmTags{Client: lastfmClient},
|
|
providers.MusicBrainz: discovery.MusicBrainzTags{Client: musicbrainzClient},
|
|
}, providerSettings, logger)
|
|
|
|
tasteRepo := storage.NewTasteRepo(db)
|
|
tasteService := taste.NewService(tasteRepo, taste.Options{MinListen: cfg.Listening.MinListen(), HalfLife: cfg.Taste.HalfLife()})
|
|
userSettings := usersettings.NewService(storage.NewUserSettingsRepo(db))
|
|
feedbacks := feedback.NewService(storage.NewFeedbackRepo(db))
|
|
affinityRepo := storage.NewAffinityRepo(db)
|
|
weights := discover.Weights{
|
|
Affinity: cfg.Scoring.WeightAffinity, Genre: cfg.Scoring.WeightGenre,
|
|
Preference: cfg.Scoring.WeightPreference, Discovery: cfg.Scoring.WeightDiscovery,
|
|
}
|
|
|
|
// Deleting the extra copy of a song is the one thing that makes Sonda write in the music folders:
|
|
// off unless the operator turned it on, and said in the log when it is on.
|
|
duplicateSvc := duplicates.NewService(storage.NewDuplicatesRepo(db), duplicates.Options{
|
|
AllowDelete: cfg.Library.AllowDelete,
|
|
ScanRunning: func() bool { return scans.Status().Running },
|
|
Logger: logger,
|
|
})
|
|
if cfg.Library.AllowDelete {
|
|
logger.Warn("library.allow_delete is on: an admin can delete duplicate files from the music folders")
|
|
}
|
|
|
|
// What a person listens to is shared on their Mastodon account only if they connected one.
|
|
playlistSvc := playlists.NewService(storage.NewPlaylistRepo(db))
|
|
mastodonSvc := mastodon.NewService(storage.NewMastodonRepo(db), mastodon.NewRemote(providers.UserAgent(version, "")), logger)
|
|
go mastodonSvc.Run(ctx)
|
|
|
|
// Other apps (OpenSubsonic) log in with the password of a device, never with the account password.
|
|
deviceSvc := devices.NewService(storage.NewDevicesRepo(db), []byte(cfg.Auth.JWTSecret))
|
|
catalogSvc := catalog.NewService(storage.NewCatalogRepo(db))
|
|
streamSvc := streaming.NewService(libraryRepo)
|
|
subsonicRepo := storage.NewSubsonicRepo(db, cfg.Listening.MinListen())
|
|
|
|
router := httpapi.NewRouter(httpapi.Deps{
|
|
Logger: logger,
|
|
Build: build(),
|
|
Stats: stats.NewService(storage.NewStatsRepo(db), cfg.Listening.MinListen()),
|
|
Auth: auth.NewService(storage.NewUserRepo(db), storage.NewRefreshRepo(db), tokens, cfg.Auth.RefreshTTL()),
|
|
Tokens: tokens,
|
|
Listening: listening.NewService(storage.NewListeningRepo(db), cfg.Listening.MinListen()),
|
|
Mastodon: mastodonSvc,
|
|
Devices: deviceSvc,
|
|
OpenSubsonic: &opensubsonic.Deps{
|
|
Login: deviceSvc, Catalog: catalogSvc, Libraries: libraries,
|
|
Extras: subsonicRepo, Lists: subsonicRepo, Streams: streamSvc, Covers: pictures, ArtistImages: pictures,
|
|
ServerVersion: version,
|
|
},
|
|
Scan: scanControl{ctx: ctx, m: scans},
|
|
Libraries: libraries,
|
|
Catalog: catalogSvc,
|
|
Providers: providerSettings,
|
|
Lidarr: lidarrClient,
|
|
LidarrArtists: lidarr.NewService(discoveryRepo, providerSettings, lidarrClient),
|
|
LidarrRequests: lidarr.NewRequests(storage.NewLidarrRequestRepo(db)),
|
|
Discovery: discoveries,
|
|
Tags: tags,
|
|
Taste: tasteService,
|
|
Discover: discover.NewService(tasteService, discoveries, tags, tasteRepo, userSettings, feedbacks, weights),
|
|
Feedback: feedbacks,
|
|
Playlists: playlistSvc,
|
|
Suggestions: affinity.NewService(affinityRepo, affinityRepo, feedbacks, affinityRepo, userSettings, cfg.Listening.MinListen()),
|
|
SpotifyImport: spotifyimport.NewService(storage.NewCatalogRepo(db), storage.NewListeningRepo(db), playlistSvc, storage.NewSpotifyImportRepo(db), storage.NewOutsidePlayRepo(db), storage.SearchKey, cfg.Listening.MinListen()),
|
|
Mixes: mixes.NewService(storage.NewMixRepo(db), tasteService, cfg.Listening.MinListen()),
|
|
Smart: smart.NewService(storage.NewSmartRepo(db), tasteService, cfg.Listening.MinListen()),
|
|
Naming: namingService,
|
|
Images: pictures,
|
|
CoversJob: jobs.Runner{Ctx: ctx, Job: images.NewCoversJob(pictures, imageRepo)},
|
|
ArtistImagesJob: jobs.Runner{Ctx: ctx, Job: images.NewArtistImagesJob(pictures, imageRepo)},
|
|
DescriptionsJob: jobs.Runner{Ctx: ctx, Job: about.NewJob(descriptions, discoveryRepo)},
|
|
Duplicates: duplicateSvc,
|
|
DuplicatesJob: jobs.Runner{Ctx: ctx, Job: duplicates.NewHashJob(duplicateSvc)},
|
|
ArtistImages: pictures,
|
|
About: descriptions,
|
|
Settings: userSettings,
|
|
Streams: streamSvc,
|
|
Federation: federation.NewService(storage.NewFederationRepo(db), federation.NewClient()),
|
|
PeerCatalog: catalog.NewService(storage.NewSharedCatalogRepo(db)),
|
|
PeerStreams: streaming.NewService(libraryRepo.Shared()),
|
|
PeerWaveform: waveform.NewService(storage.NewWaveformRepo(db), streaming.NewService(libraryRepo.Shared())),
|
|
PeerCovers: images.StoredOnly{Service: pictures},
|
|
Waveform: waveform.NewService(storage.NewWaveformRepo(db), streaming.NewService(libraryRepo)),
|
|
Web: webui.FS(),
|
|
})
|
|
|
|
// No WriteTimeout: audio streaming (later) needs long responses.
|
|
srv := &http.Server{
|
|
Addr: cfg.Server.Listen,
|
|
Handler: router,
|
|
ReadHeaderTimeout: 10 * time.Second,
|
|
IdleTimeout: 2 * time.Minute,
|
|
}
|
|
|
|
logger.Info("sondad starting",
|
|
zap.String("config", path),
|
|
zap.String("env", cfg.Log.Env),
|
|
zap.String("listen", cfg.Server.Listen),
|
|
zap.String("database_driver", cfg.Database.Driver), // never log the DSN: it holds the password
|
|
)
|
|
|
|
serveErr := make(chan error, 1)
|
|
go func() { serveErr <- srv.ListenAndServe() }()
|
|
|
|
select {
|
|
case err := <-serveErr:
|
|
return fmt.Errorf("http server: %w", err)
|
|
case <-ctx.Done():
|
|
logger.Info("shutting down")
|
|
}
|
|
|
|
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
if err := srv.Shutdown(shutdownCtx); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
|
return fmt.Errorf("shutdown: %w", err)
|
|
}
|
|
return nil
|
|
}
|