sonda/cmd/sondad/main.go
squid 3490849837
All checks were successful
release / release (push) Successful in 38s
release / image (push) Successful in 3m44s
Merge pull request 'Release by itself at every merge' (#38) from ci/semantic-release into main
2026-11-16 12:09:16 +00:00

410 lines
18 KiB
Go

package main
import (
"context"
"errors"
"flag"
"fmt"
"net/http"
"os"
"os/signal"
"syscall"
"time"
_ "time/tzdata" // the time zones that browsers send, also on a machine that has none
"github.com/gin-gonic/gin"
"go.uber.org/zap"
"git.kmalloc.org/squid/sonda/internal/about"
"git.kmalloc.org/squid/sonda/internal/affinity"
httpapi "git.kmalloc.org/squid/sonda/internal/api/http"
"git.kmalloc.org/squid/sonda/internal/auth"
"git.kmalloc.org/squid/sonda/internal/catalog"
"git.kmalloc.org/squid/sonda/internal/config"
"git.kmalloc.org/squid/sonda/internal/devices"
"git.kmalloc.org/squid/sonda/internal/discover"
"git.kmalloc.org/squid/sonda/internal/discovery"
"git.kmalloc.org/squid/sonda/internal/duplicates"
"git.kmalloc.org/squid/sonda/internal/favorites"
"git.kmalloc.org/squid/sonda/internal/federation"
"git.kmalloc.org/squid/sonda/internal/feedback"
"git.kmalloc.org/squid/sonda/internal/images"
"git.kmalloc.org/squid/sonda/internal/jobs"
"git.kmalloc.org/squid/sonda/internal/library"
"git.kmalloc.org/squid/sonda/internal/listening"
"git.kmalloc.org/squid/sonda/internal/mastodon"
"git.kmalloc.org/squid/sonda/internal/mixes"
"git.kmalloc.org/squid/sonda/internal/naming"
"git.kmalloc.org/squid/sonda/internal/opensubsonic"
"git.kmalloc.org/squid/sonda/internal/playlists"
"git.kmalloc.org/squid/sonda/internal/providers"
"git.kmalloc.org/squid/sonda/internal/providers/anthropic"
"git.kmalloc.org/squid/sonda/internal/providers/commons"
"git.kmalloc.org/squid/sonda/internal/providers/coverart"
"git.kmalloc.org/squid/sonda/internal/providers/lastfm"
"git.kmalloc.org/squid/sonda/internal/providers/lidarr"
"git.kmalloc.org/squid/sonda/internal/providers/musicbrainz"
"git.kmalloc.org/squid/sonda/internal/providers/outbound"
"git.kmalloc.org/squid/sonda/internal/providers/wikipedia"
"git.kmalloc.org/squid/sonda/internal/smart"
"git.kmalloc.org/squid/sonda/internal/spotifyimport"
"git.kmalloc.org/squid/sonda/internal/stats"
"git.kmalloc.org/squid/sonda/internal/storage"
"git.kmalloc.org/squid/sonda/internal/streaming"
"git.kmalloc.org/squid/sonda/internal/taste"
"git.kmalloc.org/squid/sonda/internal/usersettings"
"git.kmalloc.org/squid/sonda/internal/waveform"
"git.kmalloc.org/squid/sonda/internal/webui"
"git.kmalloc.org/squid/sonda/migrations"
)
const defaultConfigFile = "sonda.toml"
// version goes into the User-Agent sent to providers, and is what GET /version says. The release sets it
// (-ldflags "-X main.version=1.2.3": see the Dockerfile and the Makefile); a build that was not made by a release says
// "dev".
var version = "dev"
// lastfmMinInterval spaces the requests to Last.fm well below its limit.
const lastfmMinInterval = 250 * time.Millisecond
// musicbrainzMinInterval keeps a little above MusicBrainz's rule of one request a second.
const musicbrainzMinInterval = 1100 * time.Millisecond
// wikipediaMinInterval is polite: Wikimedia asks only for serial requests, and a
// description costs two of them. wikipediaMaxBody is far above any answer used here.
const (
wikipediaMinInterval = 250 * time.Millisecond
wikipediaMaxBody = 1 << 20
)
// commonsMaxBody leaves room for a 960 pixel copy of a picture, which is some hundred kilobytes.
const (
coverartMinInterval = 500 * time.Millisecond
commonsMinInterval = 300 * time.Millisecond
commonsAPIMaxBody = 1 << 20
commonsMaxBody = 8 << 20
)
func main() {
if err := run(); err != nil {
fmt.Fprintln(os.Stderr, "sondad:", err)
os.Exit(1)
}
}
func run() error {
configPath := flag.String("config", "", "path to the TOML config file (default ./"+defaultConfigFile+" if present)")
flag.Parse()
path := *configPath
if path == "" {
if _, err := os.Stat(defaultConfigFile); err == nil {
path = defaultConfigFile
}
}
cfg, err := config.Load(path)
if err != nil {
return err
}
if err := cfg.Validate(); err != nil {
return fmt.Errorf("invalid config: %w", err)
}
logger, err := newLogger(cfg.Log.Env)
if err != nil {
return fmt.Errorf("create logger: %w", err)
}
defer func() { _ = logger.Sync() }()
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
target := cfg.Database.Path
if cfg.Database.Driver == storage.DriverPostgres {
target = cfg.Database.DSN
}
db, err := storage.Open(cfg.Database.Driver, target)
if err != nil {
return err
}
defer func() { _ = db.Close() }()
dbMigrations, err := migrations.For(cfg.Database.Driver)
if err != nil {
return err
}
if err := storage.Migrate(ctx, db, dbMigrations); err != nil {
return err
}
// Composition root: build services and hand them to the HTTP layer.
tokens := auth.NewTokenIssuer(cfg.Auth.JWTSecret, cfg.Auth.AccessTTL())
if cfg.Log.Env == "production" {
gin.SetMode(gin.ReleaseMode)
}
libraryRepo := storage.NewLibraryRepo(db)
// On the first start after an upgrade that adds a key this can take a while on a big library.
logger.Info("refreshing the match keys of the library")
keysStarted := time.Now()
if err := libraryRepo.RefreshMatchKeys(ctx); err != nil {
return err
}
logger.Info("match keys are up to date", zap.Duration("took", time.Since(keysStarted)))
scanner := library.NewScanner(libraryRepo, library.TaglibReader{}, library.NewHub(), cfg.Library.ScanWorkers)
scans := library.NewManager(scanner, libraryRepo)
logScanResults(logger, scans)
// The libraries are scanned by themselves when their music changed: looking costs only the names of the files.
if cfg.Library.AutoScanMinutes > 0 {
auto := library.NewAutoScanner(scans, libraryRepo, time.Duration(cfg.Library.AutoScanMinutes)*time.Minute)
auto.OnScan = func(lib library.Library, signatureErr, _ error) {
if signatureErr != nil { // a NAS that is not mounted says this all the time: not news
logger.Debug("auto scan: the folder of a library cannot be read", zap.Int64("library", lib.ID), zap.Error(signatureErr))
}
}
go auto.Run(ctx)
logger.Info("libraries are scanned by themselves when their music changed", zap.Int("every_minutes", cfg.Library.AutoScanMinutes))
}
libraries := library.NewLibraries(libraryRepo)
libraries.SetBusy(func() bool { return scans.Status().Running })
warnAboutLibraries(ctx, logger, libraries)
// External providers: nothing is contacted unless an admin switched one on.
providerSettings := providers.NewService(storage.NewProviderRepo(db))
lastfmClient := lastfm.New(outbound.New(outbound.Options{
UserAgent: providers.UserAgent(version, ""),
MinInterval: lastfmMinInterval,
}), lastfm.BaseURL)
providerSettings.SetChecker(providers.LastFM, lastfmClient.Probe)
// MusicBrainz wants a contact in the User-Agent, which an admin can change: it
// is read again for every request.
musicbrainzClient := musicbrainz.New(outbound.New(outbound.Options{
UserAgentFunc: providerSettings.UserAgentFor(providers.MusicBrainz, version),
MinInterval: musicbrainzMinInterval,
FollowRedirect: musicbrainz.RedirectAllowed, // an id that was merged leads to the new one, and to nothing else
}), musicbrainz.BaseURL)
providerSettings.SetChecker(providers.MusicBrainz, func(ctx context.Context, _ string) error {
return musicbrainzClient.Probe(ctx)
})
// Wikipedia and Wikidata: asked only for the description of an artist, through ids.
// Wikimedia answers 403 to a request without a User-Agent, and wants a contact in it.
wikipediaClient := wikipedia.New(outbound.New(outbound.Options{
UserAgentFunc: providerSettings.UserAgentFor(providers.Wikipedia, version),
MinInterval: wikipediaMinInterval,
MaxBody: wikipediaMaxBody,
}), wikipedia.WikidataURL, wikipedia.SummaryURL)
providerSettings.SetChecker(providers.Wikipedia, func(ctx context.Context, _ string) error {
return wikipediaClient.Probe(ctx)
})
// Pictures of artists from Wikimedia Commons, through Wikidata, under free licences only.
// The pictures are downloaded through a client of their own: they are far bigger than an
// answer, and a picture host that misbehaves must not open the breaker of the API.
commonsClient := commons.New(outbound.New(outbound.Options{
UserAgentFunc: providerSettings.UserAgentFor(providers.Commons, version),
MinInterval: commonsMinInterval,
MaxBody: commonsAPIMaxBody,
}), commons.WikidataURL, commons.CommonsURL).WithDownloader(outbound.New(outbound.Options{
UserAgentFunc: providerSettings.UserAgentFor(providers.Commons, version),
MinInterval: commonsMinInterval,
MaxBody: commonsMaxBody,
}))
providerSettings.SetChecker(providers.Commons, func(ctx context.Context, _ string) error {
return commonsClient.Probe(ctx)
})
if err := os.MkdirAll(cfg.Storage.DataDir, 0o700); err != nil {
return fmt.Errorf("create the data folder: %w", err)
}
imageRepo := storage.NewImageRepo(db)
pictures := images.NewService(cfg.Storage.DataDir, imageRepo, imageRepo)
discoveryRepo := storage.NewDiscoveryRepo(db)
pictures.SetArtistSource(&images.CommonsSource{
Artists: discoveryRepo, Settings: providerSettings, MusicBrainz: musicbrainzClient, Commons: commonsClient,
})
// Covers from the Cover Art Archive when the user's own files have none. The archive
// answers with redirects (to archive.org and its data nodes): they are followed, to nobody else.
coverartClient := coverart.New(outbound.New(outbound.Options{
UserAgentFunc: providerSettings.UserAgentFor(providers.CoverArtArchive, version),
MinInterval: coverartMinInterval,
MaxBody: commonsMaxBody,
FollowRedirect: coverart.RedirectAllowed,
}), coverart.BaseURL)
providerSettings.SetChecker(providers.CoverArtArchive, func(ctx context.Context, _ string) error {
return coverartClient.Probe(ctx)
})
pictures.SetAlbumRemote(&images.CoverArtSource{Albums: imageRepo, Settings: providerSettings, Client: coverartClient, Finder: musicbrainzClient})
// Lidarr is the admin's own server: it is read from the stored settings for every request, on or off,
// since setting it up (a test, what it offers) happens before it is switched on.
lidarrClient := lidarr.New(func(ctx context.Context) (lidarr.Config, error) {
s, err := providerSettings.Stored(ctx, providers.Lidarr)
if err != nil {
return lidarr.Config{}, err
}
if s.URL == "" || s.APIKey == "" {
return lidarr.Config{}, lidarr.ErrNotConfigured
}
return lidarr.Config{URL: s.URL, APIKey: s.APIKey}, nil
}, providerSettings.UserAgentFor(providers.Lidarr, version), nil)
providerSettings.SetChecker(providers.Lidarr, func(ctx context.Context, _ string) error { return lidarrClient.Probe(ctx) })
providerSettings.SetPurger(func(ctx context.Context, name string) (providers.Purged, error) {
switch name {
case providers.Commons:
pictures.BeginPurge() // a picture being read right now must not be kept
out, err := discoveryRepo.Purge(ctx, name)
if err != nil {
return out, err
}
// The records are gone; the pictures themselves are files in the data folder.
if _, err := pictures.RemoveArtistFiles(); err != nil {
return out, fmt.Errorf("remove the pictures: %w", err)
}
return out, nil
case providers.CoverArtArchive:
pictures.BeginPurge()
out, err := discoveryRepo.Purge(ctx, name)
if err != nil {
return out, err
}
// The covers that came from the archive have a folder of their own: all of it goes.
if _, err := pictures.RemoveRemoteAlbumFiles(); err != nil {
return out, fmt.Errorf("remove the covers: %w", err)
}
return out, nil
}
return discoveryRepo.Purge(ctx, name)
})
// Claude names playlists when an admin turned the provider on with a key. Redirects are never followed there.
claudeClient := anthropic.New()
namingRepo := storage.NewNamingRepo(db)
namingService := naming.NewService(providerSettings, claudeClient, namingRepo, logger)
namingService.SetLedger(namingRepo) // every question to Claude is counted, to show what it costs
providerSettings.SetChecker(providers.Anthropic, claudeClient.Probe)
descriptions := about.NewService(discoveryRepo, about.WikipediaSource{MusicBrainz: musicbrainzClient, Wikipedia: wikipediaClient},
providerSettings, cfg.Wikipedia.Languages, logger)
discoveries := discovery.NewService(discoveryRepo, lastfmClient, providerSettings, logger)
tags := discovery.NewTagService(discoveryRepo, map[string]discovery.TagSource{
providers.LastFM: discovery.LastfmTags{Client: lastfmClient},
providers.MusicBrainz: discovery.MusicBrainzTags{Client: musicbrainzClient},
}, providerSettings, logger)
tasteRepo := storage.NewTasteRepo(db)
tasteService := taste.NewService(tasteRepo, taste.Options{MinListen: cfg.Listening.MinListen(), HalfLife: cfg.Taste.HalfLife()})
userSettings := usersettings.NewService(storage.NewUserSettingsRepo(db))
feedbacks := feedback.NewService(storage.NewFeedbackRepo(db))
affinityRepo := storage.NewAffinityRepo(db)
weights := discover.Weights{
Affinity: cfg.Scoring.WeightAffinity, Genre: cfg.Scoring.WeightGenre,
Preference: cfg.Scoring.WeightPreference, Discovery: cfg.Scoring.WeightDiscovery,
}
// Deleting the extra copy of a song is the one thing that makes Sonda write in the music folders:
// off unless the operator turned it on, and said in the log when it is on.
duplicateSvc := duplicates.NewService(storage.NewDuplicatesRepo(db), duplicates.Options{
AllowDelete: cfg.Library.AllowDelete,
ScanRunning: func() bool { return scans.Status().Running },
Logger: logger,
})
if cfg.Library.AllowDelete {
logger.Warn("library.allow_delete is on: an admin can delete duplicate files from the music folders")
}
// What a person listens to is shared on their Mastodon account only if they connected one.
playlistSvc := playlists.NewService(storage.NewPlaylistRepo(db))
mastodonSvc := mastodon.NewService(storage.NewMastodonRepo(db), mastodon.NewRemote(providers.UserAgent(version, "")), logger)
go mastodonSvc.Run(ctx)
// Other apps (OpenSubsonic) log in with the password of a device, never with the account password.
deviceSvc := devices.NewService(storage.NewDevicesRepo(db), []byte(cfg.Auth.JWTSecret))
catalogSvc := catalog.NewService(storage.NewCatalogRepo(db))
streamSvc := streaming.NewService(libraryRepo)
mixSvc := mixes.NewService(storage.NewMixRepo(db), tasteService, cfg.Listening.MinListen())
favoriteSvc := favorites.NewService(storage.NewFavoritesRepo(db), catalogSvc)
listeningSvc := listening.NewService(storage.NewListeningRepo(db), cfg.Listening.MinListen())
subsonicRepo := storage.NewSubsonicRepo(db, cfg.Listening.MinListen())
router := httpapi.NewRouter(httpapi.Deps{
Logger: logger,
Build: build(),
Stats: stats.NewService(storage.NewStatsRepo(db), cfg.Listening.MinListen()),
Auth: auth.NewService(storage.NewUserRepo(db), storage.NewRefreshRepo(db), tokens, cfg.Auth.RefreshTTL()),
Tokens: tokens,
Listening: listeningSvc,
Mastodon: mastodonSvc,
Devices: deviceSvc,
Favorites: favoriteSvc,
OpenSubsonic: &opensubsonic.Deps{
Login: deviceSvc, Catalog: catalogSvc, Libraries: libraries,
Extras: subsonicRepo, Lists: subsonicRepo, Streams: streamSvc, Covers: pictures, ArtistImages: pictures,
Playlists: playlistSvc, Listens: listeningSvc, Announcer: mastodonSvc, Favorites: favoriteSvc, Radio: mixSvc, Settings: userSettings,
ServerVersion: version,
},
Scan: scanControl{ctx: ctx, m: scans},
Libraries: libraries,
Catalog: catalogSvc,
Providers: providerSettings,
Lidarr: lidarrClient,
LidarrArtists: lidarr.NewService(discoveryRepo, providerSettings, lidarrClient),
LidarrRequests: lidarr.NewRequests(storage.NewLidarrRequestRepo(db)),
Discovery: discoveries,
Tags: tags,
Taste: tasteService,
Discover: discover.NewService(tasteService, discoveries, tags, tasteRepo, userSettings, feedbacks, weights),
Feedback: feedbacks,
Playlists: playlistSvc,
Suggestions: affinity.NewService(affinityRepo, affinityRepo, feedbacks, affinityRepo, userSettings, cfg.Listening.MinListen()),
SpotifyImport: spotifyimport.NewService(storage.NewCatalogRepo(db), storage.NewListeningRepo(db), playlistSvc, storage.NewSpotifyImportRepo(db), storage.NewOutsidePlayRepo(db), storage.SearchKey, cfg.Listening.MinListen()),
Mixes: mixSvc,
Smart: smart.NewService(storage.NewSmartRepo(db), tasteService, cfg.Listening.MinListen()),
Naming: namingService,
Images: pictures,
CoversJob: jobs.Runner{Ctx: ctx, Job: images.NewCoversJob(pictures, imageRepo)},
ArtistImagesJob: jobs.Runner{Ctx: ctx, Job: images.NewArtistImagesJob(pictures, imageRepo)},
DescriptionsJob: jobs.Runner{Ctx: ctx, Job: about.NewJob(descriptions, discoveryRepo)},
Duplicates: duplicateSvc,
DuplicatesJob: jobs.Runner{Ctx: ctx, Job: duplicates.NewHashJob(duplicateSvc)},
ArtistImages: pictures,
About: descriptions,
Settings: userSettings,
Streams: streamSvc,
Federation: federation.NewService(storage.NewFederationRepo(db), federation.NewClient()),
PeerCatalog: catalog.NewService(storage.NewSharedCatalogRepo(db)),
PeerStreams: streaming.NewService(libraryRepo.Shared()),
PeerWaveform: waveform.NewService(storage.NewWaveformRepo(db), streaming.NewService(libraryRepo.Shared())),
PeerCovers: images.StoredOnly{Service: pictures},
Waveform: waveform.NewService(storage.NewWaveformRepo(db), streaming.NewService(libraryRepo)),
Web: webui.FS(),
})
// No WriteTimeout: audio streaming (later) needs long responses.
srv := &http.Server{
Addr: cfg.Server.Listen,
Handler: router,
ReadHeaderTimeout: 10 * time.Second,
IdleTimeout: 2 * time.Minute,
}
logger.Info("sondad starting",
zap.String("config", path),
zap.String("env", cfg.Log.Env),
zap.String("listen", cfg.Server.Listen),
zap.String("database_driver", cfg.Database.Driver), // never log the DSN: it holds the password
)
serveErr := make(chan error, 1)
go func() { serveErr <- srv.ListenAndServe() }()
select {
case err := <-serveErr:
return fmt.Errorf("http server: %w", err)
case <-ctx.Done():
logger.Info("shutting down")
}
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := srv.Shutdown(shutdownCtx); err != nil && !errors.Is(err, http.ErrServerClosed) {
return fmt.Errorf("shutdown: %w", err)
}
return nil
}