package main import ( "context" "errors" "flag" "fmt" "net/http" "os" "os/signal" "syscall" "time" _ "time/tzdata" // the time zones that browsers send, also on a machine that has none "github.com/gin-gonic/gin" "go.uber.org/zap" "git.kmalloc.org/squid/sonda/internal/about" "git.kmalloc.org/squid/sonda/internal/affinity" httpapi "git.kmalloc.org/squid/sonda/internal/api/http" "git.kmalloc.org/squid/sonda/internal/auth" "git.kmalloc.org/squid/sonda/internal/catalog" "git.kmalloc.org/squid/sonda/internal/config" "git.kmalloc.org/squid/sonda/internal/devices" "git.kmalloc.org/squid/sonda/internal/discover" "git.kmalloc.org/squid/sonda/internal/discovery" "git.kmalloc.org/squid/sonda/internal/duplicates" "git.kmalloc.org/squid/sonda/internal/favorites" "git.kmalloc.org/squid/sonda/internal/federation" "git.kmalloc.org/squid/sonda/internal/feedback" "git.kmalloc.org/squid/sonda/internal/images" "git.kmalloc.org/squid/sonda/internal/jobs" "git.kmalloc.org/squid/sonda/internal/library" "git.kmalloc.org/squid/sonda/internal/listening" "git.kmalloc.org/squid/sonda/internal/mastodon" "git.kmalloc.org/squid/sonda/internal/mixes" "git.kmalloc.org/squid/sonda/internal/naming" "git.kmalloc.org/squid/sonda/internal/opensubsonic" "git.kmalloc.org/squid/sonda/internal/playlists" "git.kmalloc.org/squid/sonda/internal/providers" "git.kmalloc.org/squid/sonda/internal/providers/anthropic" "git.kmalloc.org/squid/sonda/internal/providers/commons" "git.kmalloc.org/squid/sonda/internal/providers/coverart" "git.kmalloc.org/squid/sonda/internal/providers/lastfm" "git.kmalloc.org/squid/sonda/internal/providers/lidarr" "git.kmalloc.org/squid/sonda/internal/providers/musicbrainz" "git.kmalloc.org/squid/sonda/internal/providers/outbound" "git.kmalloc.org/squid/sonda/internal/providers/wikipedia" "git.kmalloc.org/squid/sonda/internal/smart" "git.kmalloc.org/squid/sonda/internal/spotifyimport" "git.kmalloc.org/squid/sonda/internal/stats" "git.kmalloc.org/squid/sonda/internal/storage" "git.kmalloc.org/squid/sonda/internal/streaming" "git.kmalloc.org/squid/sonda/internal/taste" "git.kmalloc.org/squid/sonda/internal/usersettings" "git.kmalloc.org/squid/sonda/internal/waveform" "git.kmalloc.org/squid/sonda/internal/webui" "git.kmalloc.org/squid/sonda/migrations" ) const defaultConfigFile = "sonda.toml" // version goes into the User-Agent sent to providers. const version = "0.1.0" // lastfmMinInterval spaces the requests to Last.fm well below its limit. const lastfmMinInterval = 250 * time.Millisecond // musicbrainzMinInterval keeps a little above MusicBrainz's rule of one request a second. const musicbrainzMinInterval = 1100 * time.Millisecond // wikipediaMinInterval is polite: Wikimedia asks only for serial requests, and a // description costs two of them. wikipediaMaxBody is far above any answer used here. const ( wikipediaMinInterval = 250 * time.Millisecond wikipediaMaxBody = 1 << 20 ) // commonsMaxBody leaves room for a 960 pixel copy of a picture, which is some hundred kilobytes. const ( coverartMinInterval = 500 * time.Millisecond commonsMinInterval = 300 * time.Millisecond commonsAPIMaxBody = 1 << 20 commonsMaxBody = 8 << 20 ) func main() { if err := run(); err != nil { fmt.Fprintln(os.Stderr, "sondad:", err) os.Exit(1) } } func run() error { configPath := flag.String("config", "", "path to the TOML config file (default ./"+defaultConfigFile+" if present)") flag.Parse() path := *configPath if path == "" { if _, err := os.Stat(defaultConfigFile); err == nil { path = defaultConfigFile } } cfg, err := config.Load(path) if err != nil { return err } if err := cfg.Validate(); err != nil { return fmt.Errorf("invalid config: %w", err) } logger, err := newLogger(cfg.Log.Env) if err != nil { return fmt.Errorf("create logger: %w", err) } defer func() { _ = logger.Sync() }() ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() target := cfg.Database.Path if cfg.Database.Driver == storage.DriverPostgres { target = cfg.Database.DSN } db, err := storage.Open(cfg.Database.Driver, target) if err != nil { return err } defer func() { _ = db.Close() }() dbMigrations, err := migrations.For(cfg.Database.Driver) if err != nil { return err } if err := storage.Migrate(ctx, db, dbMigrations); err != nil { return err } // Composition root: build services and hand them to the HTTP layer. tokens := auth.NewTokenIssuer(cfg.Auth.JWTSecret, cfg.Auth.AccessTTL()) if cfg.Log.Env == "production" { gin.SetMode(gin.ReleaseMode) } libraryRepo := storage.NewLibraryRepo(db) // On the first start after an upgrade that adds a key this can take a while on a big library. logger.Info("refreshing the match keys of the library") keysStarted := time.Now() if err := libraryRepo.RefreshMatchKeys(ctx); err != nil { return err } logger.Info("match keys are up to date", zap.Duration("took", time.Since(keysStarted))) scanner := library.NewScanner(libraryRepo, library.TaglibReader{}, library.NewHub(), cfg.Library.ScanWorkers) scans := library.NewManager(scanner, libraryRepo) logScanResults(logger, scans) // The libraries are scanned by themselves when their music changed: looking costs only the names of the files. if cfg.Library.AutoScanMinutes > 0 { auto := library.NewAutoScanner(scans, libraryRepo, time.Duration(cfg.Library.AutoScanMinutes)*time.Minute) auto.OnScan = func(lib library.Library, signatureErr, _ error) { if signatureErr != nil { // a NAS that is not mounted says this all the time: not news logger.Debug("auto scan: the folder of a library cannot be read", zap.Int64("library", lib.ID), zap.Error(signatureErr)) } } go auto.Run(ctx) logger.Info("libraries are scanned by themselves when their music changed", zap.Int("every_minutes", cfg.Library.AutoScanMinutes)) } libraries := library.NewLibraries(libraryRepo) libraries.SetBusy(func() bool { return scans.Status().Running }) warnAboutLibraries(ctx, logger, libraries) // External providers: nothing is contacted unless an admin switched one on. providerSettings := providers.NewService(storage.NewProviderRepo(db)) lastfmClient := lastfm.New(outbound.New(outbound.Options{ UserAgent: providers.UserAgent(version, ""), MinInterval: lastfmMinInterval, }), lastfm.BaseURL) providerSettings.SetChecker(providers.LastFM, lastfmClient.Probe) // MusicBrainz wants a contact in the User-Agent, which an admin can change: it // is read again for every request. musicbrainzClient := musicbrainz.New(outbound.New(outbound.Options{ UserAgentFunc: providerSettings.UserAgentFor(providers.MusicBrainz, version), MinInterval: musicbrainzMinInterval, FollowRedirect: musicbrainz.RedirectAllowed, // an id that was merged leads to the new one, and to nothing else }), musicbrainz.BaseURL) providerSettings.SetChecker(providers.MusicBrainz, func(ctx context.Context, _ string) error { return musicbrainzClient.Probe(ctx) }) // Wikipedia and Wikidata: asked only for the description of an artist, through ids. // Wikimedia answers 403 to a request without a User-Agent, and wants a contact in it. wikipediaClient := wikipedia.New(outbound.New(outbound.Options{ UserAgentFunc: providerSettings.UserAgentFor(providers.Wikipedia, version), MinInterval: wikipediaMinInterval, MaxBody: wikipediaMaxBody, }), wikipedia.WikidataURL, wikipedia.SummaryURL) providerSettings.SetChecker(providers.Wikipedia, func(ctx context.Context, _ string) error { return wikipediaClient.Probe(ctx) }) // Pictures of artists from Wikimedia Commons, through Wikidata, under free licences only. // The pictures are downloaded through a client of their own: they are far bigger than an // answer, and a picture host that misbehaves must not open the breaker of the API. commonsClient := commons.New(outbound.New(outbound.Options{ UserAgentFunc: providerSettings.UserAgentFor(providers.Commons, version), MinInterval: commonsMinInterval, MaxBody: commonsAPIMaxBody, }), commons.WikidataURL, commons.CommonsURL).WithDownloader(outbound.New(outbound.Options{ UserAgentFunc: providerSettings.UserAgentFor(providers.Commons, version), MinInterval: commonsMinInterval, MaxBody: commonsMaxBody, })) providerSettings.SetChecker(providers.Commons, func(ctx context.Context, _ string) error { return commonsClient.Probe(ctx) }) if err := os.MkdirAll(cfg.Storage.DataDir, 0o700); err != nil { return fmt.Errorf("create the data folder: %w", err) } imageRepo := storage.NewImageRepo(db) pictures := images.NewService(cfg.Storage.DataDir, imageRepo, imageRepo) discoveryRepo := storage.NewDiscoveryRepo(db) pictures.SetArtistSource(&images.CommonsSource{ Artists: discoveryRepo, Settings: providerSettings, MusicBrainz: musicbrainzClient, Commons: commonsClient, }) // Covers from the Cover Art Archive when the user's own files have none. The archive // answers with redirects (to archive.org and its data nodes): they are followed, to nobody else. coverartClient := coverart.New(outbound.New(outbound.Options{ UserAgentFunc: providerSettings.UserAgentFor(providers.CoverArtArchive, version), MinInterval: coverartMinInterval, MaxBody: commonsMaxBody, FollowRedirect: coverart.RedirectAllowed, }), coverart.BaseURL) providerSettings.SetChecker(providers.CoverArtArchive, func(ctx context.Context, _ string) error { return coverartClient.Probe(ctx) }) pictures.SetAlbumRemote(&images.CoverArtSource{Albums: imageRepo, Settings: providerSettings, Client: coverartClient, Finder: musicbrainzClient}) // Lidarr is the admin's own server: it is read from the stored settings for every request, on or off, // since setting it up (a test, what it offers) happens before it is switched on. lidarrClient := lidarr.New(func(ctx context.Context) (lidarr.Config, error) { s, err := providerSettings.Stored(ctx, providers.Lidarr) if err != nil { return lidarr.Config{}, err } if s.URL == "" || s.APIKey == "" { return lidarr.Config{}, lidarr.ErrNotConfigured } return lidarr.Config{URL: s.URL, APIKey: s.APIKey}, nil }, providerSettings.UserAgentFor(providers.Lidarr, version), nil) providerSettings.SetChecker(providers.Lidarr, func(ctx context.Context, _ string) error { return lidarrClient.Probe(ctx) }) providerSettings.SetPurger(func(ctx context.Context, name string) (providers.Purged, error) { switch name { case providers.Commons: pictures.BeginPurge() // a picture being read right now must not be kept out, err := discoveryRepo.Purge(ctx, name) if err != nil { return out, err } // The records are gone; the pictures themselves are files in the data folder. if _, err := pictures.RemoveArtistFiles(); err != nil { return out, fmt.Errorf("remove the pictures: %w", err) } return out, nil case providers.CoverArtArchive: pictures.BeginPurge() out, err := discoveryRepo.Purge(ctx, name) if err != nil { return out, err } // The covers that came from the archive have a folder of their own: all of it goes. if _, err := pictures.RemoveRemoteAlbumFiles(); err != nil { return out, fmt.Errorf("remove the covers: %w", err) } return out, nil } return discoveryRepo.Purge(ctx, name) }) // Claude names playlists when an admin turned the provider on with a key. Redirects are never followed there. claudeClient := anthropic.New() namingRepo := storage.NewNamingRepo(db) namingService := naming.NewService(providerSettings, claudeClient, namingRepo, logger) namingService.SetLedger(namingRepo) // every question to Claude is counted, to show what it costs providerSettings.SetChecker(providers.Anthropic, claudeClient.Probe) descriptions := about.NewService(discoveryRepo, about.WikipediaSource{MusicBrainz: musicbrainzClient, Wikipedia: wikipediaClient}, providerSettings, cfg.Wikipedia.Languages, logger) discoveries := discovery.NewService(discoveryRepo, lastfmClient, providerSettings, logger) tags := discovery.NewTagService(discoveryRepo, map[string]discovery.TagSource{ providers.LastFM: discovery.LastfmTags{Client: lastfmClient}, providers.MusicBrainz: discovery.MusicBrainzTags{Client: musicbrainzClient}, }, providerSettings, logger) tasteRepo := storage.NewTasteRepo(db) tasteService := taste.NewService(tasteRepo, taste.Options{MinListen: cfg.Listening.MinListen(), HalfLife: cfg.Taste.HalfLife()}) userSettings := usersettings.NewService(storage.NewUserSettingsRepo(db)) feedbacks := feedback.NewService(storage.NewFeedbackRepo(db)) affinityRepo := storage.NewAffinityRepo(db) weights := discover.Weights{ Affinity: cfg.Scoring.WeightAffinity, Genre: cfg.Scoring.WeightGenre, Preference: cfg.Scoring.WeightPreference, Discovery: cfg.Scoring.WeightDiscovery, } // Deleting the extra copy of a song is the one thing that makes Sonda write in the music folders: // off unless the operator turned it on, and said in the log when it is on. duplicateSvc := duplicates.NewService(storage.NewDuplicatesRepo(db), duplicates.Options{ AllowDelete: cfg.Library.AllowDelete, ScanRunning: func() bool { return scans.Status().Running }, Logger: logger, }) if cfg.Library.AllowDelete { logger.Warn("library.allow_delete is on: an admin can delete duplicate files from the music folders") } // What a person listens to is shared on their Mastodon account only if they connected one. playlistSvc := playlists.NewService(storage.NewPlaylistRepo(db)) mastodonSvc := mastodon.NewService(storage.NewMastodonRepo(db), mastodon.NewRemote(providers.UserAgent(version, "")), logger) go mastodonSvc.Run(ctx) // Other apps (OpenSubsonic) log in with the password of a device, never with the account password. deviceSvc := devices.NewService(storage.NewDevicesRepo(db), []byte(cfg.Auth.JWTSecret)) catalogSvc := catalog.NewService(storage.NewCatalogRepo(db)) streamSvc := streaming.NewService(libraryRepo) mixSvc := mixes.NewService(storage.NewMixRepo(db), tasteService, cfg.Listening.MinListen()) favoriteSvc := favorites.NewService(storage.NewFavoritesRepo(db), catalogSvc) listeningSvc := listening.NewService(storage.NewListeningRepo(db), cfg.Listening.MinListen()) subsonicRepo := storage.NewSubsonicRepo(db, cfg.Listening.MinListen()) router := httpapi.NewRouter(httpapi.Deps{ Logger: logger, Build: build(), Stats: stats.NewService(storage.NewStatsRepo(db), cfg.Listening.MinListen()), Auth: auth.NewService(storage.NewUserRepo(db), storage.NewRefreshRepo(db), tokens, cfg.Auth.RefreshTTL()), Tokens: tokens, Listening: listeningSvc, Mastodon: mastodonSvc, Devices: deviceSvc, Favorites: favoriteSvc, OpenSubsonic: &opensubsonic.Deps{ Login: deviceSvc, Catalog: catalogSvc, Libraries: libraries, Extras: subsonicRepo, Lists: subsonicRepo, Streams: streamSvc, Covers: pictures, ArtistImages: pictures, Playlists: playlistSvc, Listens: listeningSvc, Announcer: mastodonSvc, Favorites: favoriteSvc, Radio: mixSvc, Settings: userSettings, ServerVersion: version, }, Scan: scanControl{ctx: ctx, m: scans}, Libraries: libraries, Catalog: catalogSvc, Providers: providerSettings, Lidarr: lidarrClient, LidarrArtists: lidarr.NewService(discoveryRepo, providerSettings, lidarrClient), LidarrRequests: lidarr.NewRequests(storage.NewLidarrRequestRepo(db)), Discovery: discoveries, Tags: tags, Taste: tasteService, Discover: discover.NewService(tasteService, discoveries, tags, tasteRepo, userSettings, feedbacks, weights), Feedback: feedbacks, Playlists: playlistSvc, Suggestions: affinity.NewService(affinityRepo, affinityRepo, feedbacks, affinityRepo, userSettings, cfg.Listening.MinListen()), SpotifyImport: spotifyimport.NewService(storage.NewCatalogRepo(db), storage.NewListeningRepo(db), playlistSvc, storage.NewSpotifyImportRepo(db), storage.NewOutsidePlayRepo(db), storage.SearchKey, cfg.Listening.MinListen()), Mixes: mixSvc, Smart: smart.NewService(storage.NewSmartRepo(db), tasteService, cfg.Listening.MinListen()), Naming: namingService, Images: pictures, CoversJob: jobs.Runner{Ctx: ctx, Job: images.NewCoversJob(pictures, imageRepo)}, ArtistImagesJob: jobs.Runner{Ctx: ctx, Job: images.NewArtistImagesJob(pictures, imageRepo)}, DescriptionsJob: jobs.Runner{Ctx: ctx, Job: about.NewJob(descriptions, discoveryRepo)}, Duplicates: duplicateSvc, DuplicatesJob: jobs.Runner{Ctx: ctx, Job: duplicates.NewHashJob(duplicateSvc)}, ArtistImages: pictures, About: descriptions, Settings: userSettings, Streams: streamSvc, Federation: federation.NewService(storage.NewFederationRepo(db), federation.NewClient()), PeerCatalog: catalog.NewService(storage.NewSharedCatalogRepo(db)), PeerStreams: streaming.NewService(libraryRepo.Shared()), PeerWaveform: waveform.NewService(storage.NewWaveformRepo(db), streaming.NewService(libraryRepo.Shared())), PeerCovers: images.StoredOnly{Service: pictures}, Waveform: waveform.NewService(storage.NewWaveformRepo(db), streaming.NewService(libraryRepo)), Web: webui.FS(), }) // No WriteTimeout: audio streaming (later) needs long responses. srv := &http.Server{ Addr: cfg.Server.Listen, Handler: router, ReadHeaderTimeout: 10 * time.Second, IdleTimeout: 2 * time.Minute, } logger.Info("sondad starting", zap.String("config", path), zap.String("env", cfg.Log.Env), zap.String("listen", cfg.Server.Listen), zap.String("database_driver", cfg.Database.Driver), // never log the DSN: it holds the password ) serveErr := make(chan error, 1) go func() { serveErr <- srv.ListenAndServe() }() select { case err := <-serveErr: return fmt.Errorf("http server: %w", err) case <-ctx.Done(): logger.Info("shutting down") } shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() if err := srv.Shutdown(shutdownCtx); err != nil && !errors.Is(err, http.ErrServerClosed) { return fmt.Errorf("shutdown: %w", err) } return nil }