From c08a234159bcbc2fabcda02ef5a0975f33636c5d Mon Sep 17 00:00:00 2001 From: tgiachi Date: Sat, 14 Nov 2026 16:25:58 +0100 Subject: [PATCH] feat(import): bring my Spotify history and playlists in I want my listening history and my playlists from Spotify in Sonda, so that Mixes, Discover and the smart playlists know what I listened to before. - POST /me/import/spotify takes the zip of a Spotify data export, or the JSON files in it (extended streaming history, the older StreamingHistory files, the playlists), as a multipart form. A request may be up to 512 MB; the zip is limited in how much it may inflate and how many files it may hold, and only the history and playlist files are opened. - Everything is read on the server and nothing is sent anywhere. The files are not kept, and of each play only the song, artist, album, time and how it ended are read: never the IP addresses, the country or the platform. - A play is matched to a song of the library by artist and title (no case, accents or punctuation; a "Remastered" note on either side; any credited artist; a compilation by its album). A match is recorded as a listen at the time of the play: complete, skip or play. Nothing is posted to Mastodon. - Importing the same files again records nothing twice, and the same play in the two shapes of history file is counted once. One import per person at a time. - A playlist becomes a playlist of mine with the songs that were found; one with a name I already have is left alone, and one with none of its songs found is not made. - The report counts what was added, what was already there and what is not in the library, and lists the missing artists, most played first. - Settings gets an Import from Spotify section; the API client can send files. - English and Italian. --- README.md | 1 + cmd/sondad/main.go | 5 +- internal/api/http/api_test.go | 2 + internal/api/http/contract_test.go | 40 +- internal/api/http/handlers_spotify_import.go | 118 ++++++ internal/api/http/middleware.go | 8 +- internal/api/http/router.go | 8 + internal/api/http/spotify_import_test.go | 191 +++++++++ internal/api/openapi/openapi.yaml | 111 +++++ internal/spotifyimport/match.go | 111 +++++ internal/spotifyimport/match_test.go | 106 +++++ internal/spotifyimport/read.go | 307 ++++++++++++++ internal/spotifyimport/read_test.go | 183 +++++++++ internal/spotifyimport/service.go | 379 ++++++++++++++++++ internal/spotifyimport/service_test.go | 363 +++++++++++++++++ internal/storage/catalog.go | 35 ++ internal/storage/listening.go | 36 ++ internal/storage/search_key.go | 4 + internal/storage/spotify_import_test.go | 116 ++++++ web/src/api/client.test.ts | 30 ++ web/src/api/client.ts | 8 +- web/src/api/importSpotify.ts | 22 + web/src/api/schema.d.ts | 134 +++++++ .../components/SpotifyImportBlock.test.tsx | 109 +++++ web/src/components/SpotifyImportBlock.tsx | 112 ++++++ web/src/i18n/locales/en.json | 34 ++ web/src/i18n/locales/it.json | 34 ++ web/src/pages/SettingsPage.tsx | 5 + 28 files changed, 2605 insertions(+), 7 deletions(-) create mode 100644 internal/api/http/handlers_spotify_import.go create mode 100644 internal/api/http/spotify_import_test.go create mode 100644 internal/spotifyimport/match.go create mode 100644 internal/spotifyimport/match_test.go create mode 100644 internal/spotifyimport/read.go create mode 100644 internal/spotifyimport/read_test.go create mode 100644 internal/spotifyimport/service.go create mode 100644 internal/spotifyimport/service_test.go create mode 100644 internal/storage/spotify_import_test.go create mode 100644 web/src/api/importSpotify.ts create mode 100644 web/src/components/SpotifyImportBlock.test.tsx create mode 100644 web/src/components/SpotifyImportBlock.tsx diff --git a/README.md b/README.md index 63d737d..cfe0dce 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ It is a single executable, one database (SQLite by default, PostgreSQL if you pr - **Playlists.** `POST /playlists` makes a playlist (a name and an optional description) and `GET /playlists` lists yours, the one changed last first. `POST /playlists/{id}/tracks` adds 1 to 500 tracks at the end or at a `position`, `GET /playlists/{id}/tracks` lists them in order, `PATCH /playlists/{id}/tracks/{item}` moves one and `DELETE` takes one out; `PATCH /playlists/{id}` renames and `DELETE /playlists/{id}` deletes (the tracks stay in the library). A track can be in a playlist more than once, which is why each place has its own item id. A playlist is yours alone: someone else's answers `playlist_not_found`, as one that does not exist. A track whose files were deleted or moved away stays in the playlist and shows `available: false`; it plays again when the files come back and a scan finds them. Each playlist says which albums stand for it (`album_ids`: up to 4 albums that have a cover, in the order they first appear in it, each once), for a picture made of their covers; nothing is fetched for it. A user can have 1000 playlists of up to 10000 tracks. Nothing is shared and nothing leaves your server. - **Wishlist and "not interested".** `PUT /wishlist/{artist}` puts an artist on your wishlist (`GET /wishlist` lists it, `DELETE` takes it off), and `POST /discover/{artist}/dismiss` hides an artist from your Discover for good (`DELETE` brings it back, `GET /discover/dismissed` lists them). An artist is wished or dismissed, never both, and both lists are yours alone. Discover marks the wished artists it shows (`wishlisted`) and never suggests a dismissed one. The wishlist is a list for you and sends nothing anywhere; Sonda downloads nothing, but an admin can send an artist that Discover showed to their own Lidarr (see [External providers](#external-providers)), which works for an artist that has a MusicBrainz id and that Sonda knows of from a list or from your files. An artist that only Last.fm named is kept while you have it marked (your lists still show it, but other users cannot see that it exists), and goes with the rest of Last.fm's data if an admin removes Last.fm's data. Removing MusicBrainz's data does not touch it. - **Your own settings.** `GET` and `PATCH /me/settings` hold choices that belong to you, not to the server. Everyone starts private: `discover_from_listening` is off, so this setting never sends your listening to a provider. When you turn it on, Discover may start from the artists you listen to most, and their names are sent to Last.fm (only if an admin switched it on). What Last.fm answers is kept in the shared database like any other lookup, so another local user can see, from `retrieved_at` of `GET /artists/{id}/similar`, that someone looked up an artist, but not who. +- **Importing from Spotify.** `POST /me/import/spotify` brings your listening history and your playlists from a Spotify data export into your own account. Ask Spotify for your data (Account, Privacy, "Extended streaming history" and "Account data") and send what it gives, as a multipart form with a `files` field for each file: the zip itself, or the JSON files in it (`Streaming_History_Audio_*.json`, the older `StreamingHistory_music_*.json` and `Playlist*.json`); a request may be up to 512 MB, and the rest of the export (the account, the searches, the library) is never opened. It all happens on your server and nothing is sent anywhere; the files are read from the request and not kept, and of each play only the song, the artist, the album, the time and how it ended are read, never the IP addresses, the country or the platform in the file. A play is matched to a song of your library by artist and title (case, accents and punctuation do not matter, a `Remastered` note on either side does not, any credited artist will do, and a play of a compilation is matched by its album when Spotify says `Various Artists`). A match is recorded as a listen at the time of the play: `complete` when it ran to its end (or nine tenths were played), `skip` when Spotify says it was skipped or it lasted less than a listen, `play` otherwise; nothing is posted to Mastodon. Importing the same files again records nothing twice, and the same play in the extended and in the older history is one. A playlist becomes a playlist of yours with the songs that were found, in order; one whose name you already have is left alone, and one with none of its songs in your library is not made. The report counts what was added, what was already there and what is not in your library (and lists those artists, most played first), so you can see what is missing. One import of yours runs at a time. Settings has an Import from Spotify section. - **Sharing on Mastodon.** `PUT /me/mastodon` connects your own Mastodon account (the address of your instance and an access token that can only write posts); `GET` shows which account is connected and `DELETE` forgets it and the token. Nothing is connected by default. Once it is, each track of this server that counts as a listen is posted to your account as an unlisted status with the title, artists, album and `#NowPlaying`, and no link: at most one post a minute, never music of a friend. The token stays on the server and is never sent back. If the instance refuses the token on a post (it was revoked or expired), `GET /me/mastodon` says `problem: token_refused`, Settings shows it, and nothing is posted until you connect again with a new token. The address of the instance must be a domain name on https, and the server never connects to a private, loopback or link-local address for it, never follows a redirect and ignores proxy settings. - **Tags.** `GET /artists/{id}/tags` shows what Last.fm and MusicBrainz say kind of music an artist plays, each with its own weights, and the genres of the artist's own files, read offline. See [External providers](#external-providers). - **Similar artists.** With Last.fm switched on, `GET /artists/{id}/similar` lists artists similar to an artist, and says which of them you already have. See [External providers](#external-providers). Without it, Sonda makes no outbound network requests at all. diff --git a/cmd/sondad/main.go b/cmd/sondad/main.go index 17d6429..2cec801 100644 --- a/cmd/sondad/main.go +++ b/cmd/sondad/main.go @@ -42,6 +42,7 @@ import ( "codeberg.org/squidd/sonda/internal/providers/outbound" "codeberg.org/squidd/sonda/internal/providers/wikipedia" "codeberg.org/squidd/sonda/internal/smart" + "codeberg.org/squidd/sonda/internal/spotifyimport" "codeberg.org/squidd/sonda/internal/storage" "codeberg.org/squidd/sonda/internal/streaming" "codeberg.org/squidd/sonda/internal/taste" @@ -289,6 +290,7 @@ func run() error { } // What a person listens to is shared on their Mastodon account only if they connected one. + playlistSvc := playlists.NewService(storage.NewPlaylistRepo(db)) mastodonSvc := mastodon.NewService(storage.NewMastodonRepo(db), mastodon.NewRemote(providers.UserAgent(version, "")), logger) go mastodonSvc.Run(ctx) @@ -309,7 +311,8 @@ func run() error { Taste: tasteService, Discover: discover.NewService(tasteService, discoveries, tags, tasteRepo, userSettings, feedbacks, weights), Feedback: feedbacks, - Playlists: playlists.NewService(storage.NewPlaylistRepo(db)), + Playlists: playlistSvc, + SpotifyImport: spotifyimport.NewService(storage.NewCatalogRepo(db), storage.NewListeningRepo(db), playlistSvc, storage.SearchKey, cfg.Listening.MinListen()), Mixes: mixes.NewService(storage.NewMixRepo(db), tasteService, cfg.Listening.MinListen()), Smart: smart.NewService(storage.NewSmartRepo(db), tasteService, cfg.Listening.MinListen()), Naming: namingService, diff --git a/internal/api/http/api_test.go b/internal/api/http/api_test.go index 1296116..af463e8 100644 --- a/internal/api/http/api_test.go +++ b/internal/api/http/api_test.go @@ -34,6 +34,7 @@ import ( "codeberg.org/squidd/sonda/internal/providers" "codeberg.org/squidd/sonda/internal/providers/anthropic" "codeberg.org/squidd/sonda/internal/smart" + "codeberg.org/squidd/sonda/internal/spotifyimport" "codeberg.org/squidd/sonda/internal/storage" "codeberg.org/squidd/sonda/internal/streaming" "codeberg.org/squidd/sonda/internal/taste" @@ -120,6 +121,7 @@ func newTestEnvDB(t *testing.T, mutate func(*Deps, *bun.DB)) (http.Handler, *bun feedbackSvc := feedback.NewService(storage.NewFeedbackRepo(db)) deps.Feedback = feedbackSvc deps.Playlists = playlists.NewService(storage.NewPlaylistRepo(db)) + deps.SpotifyImport = spotifyimport.NewService(storage.NewCatalogRepo(db), storage.NewListeningRepo(db), deps.Playlists, storage.SearchKey, 20*time.Second) deps.Mixes = mixes.NewService(storage.NewMixRepo(db), deps.Taste.(mixes.Taster), 20*time.Second) deps.Smart = smart.NewService(storage.NewSmartRepo(db), deps.Taste.(smart.Taster), 20*time.Second) namingSvc := naming.NewService(prov, noClaude{}, storage.NewNamingRepo(db), nil) diff --git a/internal/api/http/contract_test.go b/internal/api/http/contract_test.go index e348148..5462873 100644 --- a/internal/api/http/contract_test.go +++ b/internal/api/http/contract_test.go @@ -7,11 +7,13 @@ import ( "crypto/rand" "encoding/base64" "encoding/json" + "fmt" "image" "image/color" "image/png" "io" "mime" + "mime/multipart" "net/http" "net/http/httptest" "os" @@ -191,15 +193,38 @@ func newContract(t *testing.T) *contract { func (c *contract) call(method, path, token string, body any, headers ...string) *httptest.ResponseRecorder { c.t.Helper() var payload []byte + contentType := "" if body != nil { var err error if payload, err = json.Marshal(body); err != nil { c.t.Fatal(err) } + contentType = "application/json" } + return c.callRaw(method, path, token, contentType, payload, headers...) +} + +// upload sends files as a multipart form, in the field `files`. +func (c *contract) upload(path, token string, files ...string) *httptest.ResponseRecorder { + c.t.Helper() + var buf bytes.Buffer + w := multipart.NewWriter(&buf) + for i, content := range files { + part, err := w.CreateFormFile("files", fmt.Sprintf("file%d.json", i)) + if err != nil { + c.t.Fatal(err) + } + _, _ = part.Write([]byte(content)) + } + _ = w.Close() + return c.callRaw("POST", path, token, w.FormDataContentType(), buf.Bytes()) +} + +func (c *contract) callRaw(method, path, token, contentType string, payload []byte, headers ...string) *httptest.ResponseRecorder { + c.t.Helper() req := httptest.NewRequest(method, "/api/v1"+path, bytes.NewReader(payload)) - if body != nil { - req.Header.Set("Content-Type", "application/json") + if contentType != "" { + req.Header.Set("Content-Type", contentType) } if token != "" { req.Header.Set("Authorization", "Bearer "+token) @@ -839,6 +864,17 @@ func TestResponsesMatchTheSpec(t *testing.T) { c.expect(c.call("DELETE", "/me/mastodon", token, nil), 204, "disconnect Mastodon") c.expect(c.call("GET", "/me/mastodon", "", nil), 401, "Mastodon without a token", "unauthorized") + // Bringing a Spotify export in: what it did, and what is refused. + export := `[{"ts":"2023-05-01T10:00:00Z","ms_played":200000,"master_metadata_track_name":"Nowhere","master_metadata_album_artist_name":"Nobody"}]` + imported := c.expect(c.upload("/me/import/spotify", token, export, `{"playlists":[{"name":"P","items":[{"track":{"trackName":"Nowhere","artistName":"Nobody"}}]}]}`), 200, "import a Spotify export") + if h, _ := imported["history"].(map[string]any); h["entries"] != 1.0 || h["not_found"] != 1.0 { + c.t.Errorf("history = %v", h) + } + c.expect(c.upload("/me/import/spotify", token), 400, "an import with no files", "invalid_input") + c.expect(c.upload("/me/import/spotify", token, `{"hello":"world"}`), 400, "files that are not Spotify's", "no_spotify_data") + c.expect(c.upload("/me/import/spotify", token, export[:len(export)-20]), 400, "a file cut short", "unreadable_file") + c.expect(c.upload("/me/import/spotify", ""), 401, "an import without a token", "unauthorized") + // ---- duplicate songs ---- c.duplicates(t, token, libID) diff --git a/internal/api/http/handlers_spotify_import.go b/internal/api/http/handlers_spotify_import.go new file mode 100644 index 0000000..49e72b0 --- /dev/null +++ b/internal/api/http/handlers_spotify_import.go @@ -0,0 +1,118 @@ +package httpapi + +import ( + "errors" + "net/http" + + "github.com/gin-gonic/gin" + + "codeberg.org/squidd/sonda/internal/spotifyimport" +) + +// spotifyImportPath is the route of the import of a Spotify export. +const spotifyImportPath = "/api/v1/me/import/spotify" + +// importMaxBytes bounds the upload of an export (the zip, or its JSON files), which is far larger than any other +// request: ten years of listening are a few hundred megabytes. It is a variable so that a test can lower it. +var importMaxBytes int64 = 512 << 20 + +// importFormMemory is how much of an upload is kept in memory while it is read; the rest goes to a temporary file. +const importFormMemory = 32 << 20 + +// spotifyImportHandlers import the Spotify export of the signed-in person into their own listens and playlists. +type spotifyImportHandlers struct{ svc spotifyImportService } + +type spotifyHistoryJSON struct { + Entries int `json:"entries"` + Matched int `json:"matched"` + Imported int `json:"imported"` + AlreadyThere int `json:"already_there"` + NotFound int `json:"not_found"` + Ignored int `json:"ignored"` +} + +type spotifyPlaylistJSON struct { + Name string `json:"name"` + Tracks int `json:"tracks"` + Matched int `json:"matched"` + Status string `json:"status"` +} + +type spotifyMissingJSON struct { + Name string `json:"name"` + Count int `json:"count"` +} + +type spotifyReportJSON struct { + History spotifyHistoryJSON `json:"history"` + Playlists []spotifyPlaylistJSON `json:"playlists"` + Missing struct { + Total int `json:"total"` + Artists []spotifyMissingJSON `json:"artists"` + } `json:"missing"` +} + +func toSpotifyReport(r spotifyimport.Report) spotifyReportJSON { + out := spotifyReportJSON{ + History: spotifyHistoryJSON{ + Entries: r.History.Entries, Matched: r.History.Matched, Imported: r.History.Imported, + AlreadyThere: r.History.AlreadyThere, NotFound: r.History.NotFound, Ignored: r.History.Ignored, + }, + Playlists: make([]spotifyPlaylistJSON, 0, len(r.Playlists)), + } + for _, p := range r.Playlists { + out.Playlists = append(out.Playlists, spotifyPlaylistJSON{Name: p.Name, Tracks: p.Tracks, Matched: p.Matched, Status: p.Status}) + } + out.Missing.Total = r.MissingTotal + out.Missing.Artists = make([]spotifyMissingJSON, 0, len(r.Missing)) + for _, m := range r.Missing { + out.Missing.Artists = append(out.Missing.Artists, spotifyMissingJSON{Name: m.Artist, Count: m.Count}) + } + return out +} + +func (h *spotifyImportHandlers) importExport(c *gin.Context) { + if err := c.Request.ParseMultipartForm(importFormMemory); err != nil { + var tooBig *http.MaxBytesError + if errors.As(err, &tooBig) { + writeError(c, http.StatusRequestEntityTooLarge, "too_large", "these files are too large to import") + return + } + writeError(c, http.StatusBadRequest, "invalid_input", "send the files of the export as a multipart form, in the field files") + return + } + defer func() { _ = c.Request.MultipartForm.RemoveAll() }() + headers := c.Request.MultipartForm.File["files"] + if len(headers) == 0 { + writeError(c, http.StatusBadRequest, "invalid_input", "send the files of the export as a multipart form, in the field files") + return + } + files := make([]spotifyimport.File, 0, len(headers)) + for _, fh := range headers { + f, err := fh.Open() + if err != nil { + _ = c.Error(err) + writeError(c, http.StatusInternalServerError, "internal", "internal error") + return + } + defer f.Close() + files = append(files, spotifyimport.File{R: f, Size: fh.Size}) + } + + report, err := h.svc.Import(c.Request.Context(), currentUserID(c), files) + switch { + case err == nil: + c.JSON(http.StatusOK, toSpotifyReport(report)) + case errors.Is(err, spotifyimport.ErrNothingFound): + writeError(c, http.StatusBadRequest, "no_spotify_data", "these files hold no streaming history and no playlists of Spotify") + case errors.Is(err, spotifyimport.ErrUnreadable): + writeError(c, http.StatusBadRequest, "unreadable_file", "a file is damaged or cut short") + case errors.Is(err, spotifyimport.ErrTooBig): + writeError(c, http.StatusRequestEntityTooLarge, "too_large", "these files hold more than can be imported") + case errors.Is(err, spotifyimport.ErrBusy): + writeError(c, http.StatusConflict, "import_running", "an import of yours is still running") + default: + _ = c.Error(err) + writeError(c, http.StatusInternalServerError, "internal", "internal error") + } +} diff --git a/internal/api/http/middleware.go b/internal/api/http/middleware.go index a728384..7ca31a2 100644 --- a/internal/api/http/middleware.go +++ b/internal/api/http/middleware.go @@ -19,10 +19,14 @@ const ( ctxAccount = "account" // the auth.User that requireAuth already looked up ) -// limitBody caps the request body size. +// limitBody caps the request body size. The import of a Spotify export is the one request that may be far larger. func limitBody(max int64) gin.HandlerFunc { return func(c *gin.Context) { - c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, max) + limit := max + if c.FullPath() == spotifyImportPath { + limit = importMaxBytes + } + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, limit) c.Next() } } diff --git a/internal/api/http/router.go b/internal/api/http/router.go index e091608..2a128f0 100644 --- a/internal/api/http/router.go +++ b/internal/api/http/router.go @@ -30,6 +30,7 @@ import ( "codeberg.org/squidd/sonda/internal/playlists" "codeberg.org/squidd/sonda/internal/providers" "codeberg.org/squidd/sonda/internal/smart" + "codeberg.org/squidd/sonda/internal/spotifyimport" "codeberg.org/squidd/sonda/internal/streaming" "codeberg.org/squidd/sonda/internal/taste" "codeberg.org/squidd/sonda/internal/usersettings" @@ -103,6 +104,11 @@ type playlistService interface { MoveItem(ctx context.Context, userID, id, itemID int64, position int) error } +// spotifyImportService is what the handlers need from spotifyimport.Service. +type spotifyImportService interface { + Import(ctx context.Context, userID int64, files []spotifyimport.File) (spotifyimport.Report, error) +} + // mixService is what the handlers need from mixes.Service. type mixService interface { List(ctx context.Context, userID int64) ([]mixes.Mix, error) @@ -219,6 +225,7 @@ type Deps struct { Discover discoverService Feedback feedbackService Playlists playlistService + SpotifyImport spotifyImportService Mixes mixService Smart smartService Naming namingService @@ -354,6 +361,7 @@ func NewRouter(d Deps) *gin.Engine { protected.GET("/me/settings", us.get) protected.PATCH("/me/settings", us.update) ma := &mastodonHandlers{svc: d.Mastodon} + protected.POST("/me/import/spotify", (&spotifyImportHandlers{svc: d.SpotifyImport}).importExport) protected.GET("/me/mastodon", ma.get) protected.PUT("/me/mastodon", ma.connect) protected.DELETE("/me/mastodon", ma.disconnect) diff --git a/internal/api/http/spotify_import_test.go b/internal/api/http/spotify_import_test.go new file mode 100644 index 0000000..84bf008 --- /dev/null +++ b/internal/api/http/spotify_import_test.go @@ -0,0 +1,191 @@ +package httpapi + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "mime/multipart" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/uptrace/bun" + + "codeberg.org/squidd/sonda/internal/library" + "codeberg.org/squidd/sonda/internal/storage" +) + +const importPath = "/api/v1/me/import/spotify" + +// upload posts the files as a multipart form, each as `files`. +func upload(t *testing.T, h http.Handler, token string, files ...string) resp { + t.Helper() + var body bytes.Buffer + w := multipart.NewWriter(&body) + for i, content := range files { + part, err := w.CreateFormFile("files", fmt.Sprintf("file%d.json", i)) + if err != nil { + t.Fatal(err) + } + _, _ = part.Write([]byte(content)) + } + _ = w.Close() + req := httptest.NewRequest("POST", importPath, &body) + req.Header.Set("Content-Type", w.FormDataContentType()) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + out := resp{Code: rec.Code, Body: map[string]any{}} + _ = json.Unmarshal(rec.Body.Bytes(), &out.Body) + return out +} + +func seedSong(t *testing.T, db *bun.DB, rel, title, album, artist string) { + t.Helper() + repo := storage.NewLibraryRepo(db) + lib, err := repo.CreateLibrary(context.Background(), "music-"+rel, "/lib/"+rel) + if err != nil { + t.Fatal(err) + } + f := library.ParsedFile{ + FileInfo: library.FileInfo{RelPath: rel, Size: 100, MTimeUnix: 1}, + Metadata: library.Metadata{ + Tags: library.Tags{Title: title, Album: album, AlbumArtist: artist, Artists: []string{artist}}, + Properties: library.Properties{Format: "flac", DurationMs: 260_000, SampleRate: 44100, BitRate: 900}, + }, + } + if _, err := repo.SaveFile(context.Background(), lib.ID, f, 0); err != nil { + t.Fatal(err) + } +} + +const spotifyHistory = `[ + {"ts":"2023-05-01T10:00:00Z","ms_played":260000,"ip_addr":"203.0.113.9","master_metadata_track_name":"Karma Police", + "master_metadata_album_artist_name":"Radiohead","master_metadata_album_album_name":"OK Computer","reason_end":"trackdone"}, + {"ts":"2023-05-01T10:10:00Z","ms_played":100000,"master_metadata_track_name":"Creep","master_metadata_album_artist_name":"Radiohead"} +]` + +func TestImportingAHistoryRecordsTheListensAndTheReportSaysWhatHappened(t *testing.T) { + h, db := newTestEnv(t, func(*Deps) {}) + seedSong(t, db, "a.flac", "Karma Police", "OK Computer", "Radiohead") + tok := registerAndLogin(t, h) + + r := upload(t, h, tok, spotifyHistory) + if r.Code != http.StatusOK { + t.Fatalf("import: %d %v", r.Code, r.Body) + } + hist, _ := r.Body["history"].(map[string]any) + sameKeys(t, "history", hist, "entries", "matched", "imported", "already_there", "not_found", "ignored") + if hist["entries"] != 2.0 || hist["matched"] != 1.0 || hist["imported"] != 1.0 || hist["not_found"] != 1.0 { + t.Errorf("history = %v", hist) + } + missing, _ := r.Body["missing"].(map[string]any) + sameKeys(t, "missing", missing, "total", "artists") + artists, _ := missing["artists"].([]any) + if len(artists) != 1 || artists[0].(map[string]any)["name"] != "Radiohead" || artists[0].(map[string]any)["count"] != 1.0 { + t.Errorf("missing = %v", missing) + } + if pl, ok := r.Body["playlists"].([]any); !ok || len(pl) != 0 { + t.Errorf("playlists = %#v, want an empty list", r.Body["playlists"]) + } + + // The listen is in the history, with the time of the play, and counts. + list := do(t, h, "GET", "/api/v1/listens", tok, nil) + items, _ := list.Body["items"].([]any) + if len(items) != 1 { + t.Fatalf("history has %d events: %v", len(items), list.Body) + } + if ev := items[0].(map[string]any); ev["counted"] != true || ev["occurred_at"] != "2023-05-01T10:00:00Z" { + t.Errorf("event = %v", ev) + } + + // The same file again adds nothing. + r = upload(t, h, tok, spotifyHistory) + hist, _ = r.Body["history"].(map[string]any) + if r.Code != http.StatusOK || hist["imported"] != 0.0 || hist["already_there"] != 1.0 { + t.Errorf("second import: %d %v", r.Code, hist) + } +} + +func TestImportingPlaylistsMakesPlaylistsOfTheSignedInPerson(t *testing.T) { + h, db := newTestEnv(t, func(*Deps) {}) + seedSong(t, db, "a.flac", "Karma Police", "OK Computer", "Radiohead") + tok := registerAndLogin(t, h) + r := upload(t, h, tok, `{"playlists":[{"name":"Road","items":[ + {"track":{"trackName":"Karma Police","artistName":"Radiohead","albumName":"OK Computer"}}, + {"track":{"trackName":"Creep","artistName":"Radiohead","albumName":"Pablo Honey"}}]}]}`) + if r.Code != http.StatusOK { + t.Fatalf("import: %d %v", r.Code, r.Body) + } + pl := r.Body["playlists"].([]any)[0].(map[string]any) + sameKeys(t, "playlist", pl, "name", "tracks", "matched", "status") + if pl["name"] != "Road" || pl["tracks"] != 2.0 || pl["matched"] != 1.0 || pl["status"] != "created" { + t.Errorf("playlist = %v", pl) + } + list := do(t, h, "GET", "/api/v1/playlists", tok, nil) + items, _ := list.Body["items"].([]any) + if len(items) != 1 || items[0].(map[string]any)["name"] != "Road" || items[0].(map[string]any)["track_count"] != 1.0 { + t.Errorf("playlists = %v", list.Body) + } +} + +func TestTheImportNeedsASignedInPerson(t *testing.T) { + h := newTestAPI(t) + if r := upload(t, h, "", spotifyHistory); r.Code != http.StatusUnauthorized { + t.Errorf("without a token: %d, want 401", r.Code) + } +} + +func TestImportRefusesWhatIsNotAnExport(t *testing.T) { + h := newTestAPI(t) + tok := registerAndLogin(t, h) + + if r := do(t, h, "POST", importPath, tok, "{}"); r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" { + t.Errorf("a JSON body: %d %v, want 400 invalid_input", r.Code, r.Body) + } + if r := upload(t, h, tok); r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" { + t.Errorf("no files: %d %v, want 400 invalid_input", r.Code, r.Body) + } + if r := upload(t, h, tok, `{"hello":"world"}`, "words"); r.Code != http.StatusBadRequest || errCode(r) != "no_spotify_data" { + t.Errorf("not Spotify's: %d %v, want 400 no_spotify_data", r.Code, r.Body) + } + if r := upload(t, h, tok, spotifyHistory[:len(spotifyHistory)-30]); r.Code != http.StatusBadRequest || errCode(r) != "unreadable_file" { + t.Errorf("a cut file: %d %v, want 400 unreadable_file", r.Code, r.Body) + } +} + +func TestAnUploadBeyondTheLimitIsRefused(t *testing.T) { + old := importMaxBytes + importMaxBytes = 2000 + defer func() { importMaxBytes = old }() + h := newTestAPI(t) + tok := registerAndLogin(t, h) + big := "[" + strings.Repeat(`{"ts":"2023-05-01T10:00:00Z"},`, 200) + `{"ts":"2023-05-01T10:00:00Z"}]` + if r := upload(t, h, tok, big); r.Code != http.StatusRequestEntityTooLarge || errCode(r) != "too_large" { + t.Errorf("%d %v, want 413 too_large", r.Code, r.Body) + } +} + +func TestAnUploadLargerThanOtherRequestsIsAccepted(t *testing.T) { + h, db := newTestEnv(t, func(*Deps) {}) + seedSong(t, db, "a.flac", "Karma Police", "OK Computer", "Radiohead") + tok := registerAndLogin(t, h) + // A history of more than the 64 KB that any other request may be. + var plays []string + for i := 0; i < 1500; i++ { + plays = append(plays, fmt.Sprintf(`{"ts":"2023-05-%02dT%02d:%02d:00Z","ms_played":260000,"master_metadata_track_name":"Karma Police","master_metadata_album_artist_name":"Radiohead"}`, 1+i/1440, i/60%24, i%60)) + } + data := "[" + strings.Join(plays, ",") + "]" + if len(data) < 2*maxBodyBytes { + t.Fatalf("test file is %d bytes, too small to prove anything", len(data)) + } + r := upload(t, h, tok, data) + hist, _ := r.Body["history"].(map[string]any) + if r.Code != http.StatusOK || hist["imported"] != 1500.0 { + t.Errorf("%d %v", r.Code, r.Body) + } +} diff --git a/internal/api/openapi/openapi.yaml b/internal/api/openapi/openapi.yaml index ed320fd..71532b4 100644 --- a/internal/api/openapi/openapi.yaml +++ b/internal/api/openapi/openapi.yaml @@ -2162,6 +2162,70 @@ paths: '204': {description: Disconnected.} '401': {$ref: '#/components/responses/Unauthorized'} + /me/import/spotify: + post: + tags: [Taste] + operationId: importMySpotify + summary: Import your Spotify history and playlists + description: | + Brings your listening history and your playlists from a Spotify data export into your own account, so that + Mixes, Discover and the smart playlists know what you listened to before. Ask Spotify for your data (Account, + Privacy, "Extended streaming history" and "Account data"), then send what it gives: the zip itself, or the + JSON files in it (`Streaming_History_Audio_*.json`, the older `StreamingHistory_music_*.json`, and + `Playlist*.json`), as a multipart form with one `files` field for each file. A request may be up to 512 MB. + + Everything happens on this server and nothing is sent to anyone. The files are read from the request and not + kept, and of each play only the song, the artist, the album, the time and how it ended are read: the IP + addresses, the country and the platform in the file are never looked at. + + A play is matched to a song of your library by artist and title (case, accents and punctuation do not matter, + a "Remastered" note on either side does not either, and any of the credited artists will do; a play of a + compilation is matched by its album when Spotify names "Various Artists"). A match is recorded as a listen at + the time of the play: `complete` when the play ran to its end (or nine tenths of the song were played), + `skip` when Spotify says it was skipped or it was shorter than a listen, `play` otherwise. Importing does not + post anything to Mastodon. Importing the same files again records nothing twice, and the same play in the + extended history and in the older one is counted once. + + A playlist of the export becomes a playlist of yours with the songs that were found, in order. One whose + name you already have is left alone (`exists`), and one with none of its songs in your library is not made + (`empty`). The report lists what was not found, by artist, most played first, so you can see what is missing + from your library. One import of yours runs at a time. + security: [{bearerAuth: []}] + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + required: [files] + properties: + files: + type: array + items: {type: string, format: binary} + description: The zip of the export, or its JSON files. + responses: + '200': + description: What the import did. + content: + application/json: + schema: {$ref: '#/components/schemas/SpotifyImportReport'} + '400': + description: No files were sent (`invalid_input`), none of them holds a history or playlists of Spotify (`no_spotify_data`) or one is damaged or cut short (`unreadable_file`). + content: + application/json: + schema: {$ref: '#/components/schemas/Error'} + '401': {$ref: '#/components/responses/Unauthorized'} + '409': + description: An import of yours is still running (`import_running`). + content: + application/json: + schema: {$ref: '#/components/schemas/Error'} + '413': + description: The files are larger than 512 MB, or hold more than can be imported once inflated (`too_large`). + content: + application/json: + schema: {$ref: '#/components/schemas/Error'} + /artists/{id}/lidarr: parameters: - {$ref: '#/components/parameters/Id'} @@ -5402,6 +5466,53 @@ components: shared: another user can see from `retrieved_at` in `GET /artists/{id}/similar` that somebody looked an artist up, but not who. + SpotifyImportReport: + type: object + additionalProperties: false + required: [history, playlists, missing] + properties: + history: + type: object + additionalProperties: false + required: [entries, matched, imported, already_there, not_found, ignored] + properties: + entries: {type: integer, description: 'Plays of songs that were read.'} + matched: {type: integer, description: 'Of those, the plays of a song of your library.'} + imported: {type: integer, description: 'Recorded now.'} + already_there: {type: integer, description: 'Matched and already recorded by an earlier import, or the same play in two files.'} + not_found: {type: integer, description: 'Plays of songs that are not in your library.'} + ignored: {type: integer, description: 'Podcast episodes, and plays with no time that can be read.'} + playlists: + type: array + items: + type: object + additionalProperties: false + required: [name, tracks, matched, status] + properties: + name: {type: string} + tracks: {type: integer, description: 'Everything that was in the playlist, songs and not.'} + matched: {type: integer, description: 'The songs of your library.'} + status: + type: string + enum: [created, exists, empty, limit] + description: '`created`; `exists` (you already have one with that name, left alone); `empty` (none of its songs is in your library); `limit` (you have as many playlists as are allowed).' + missing: + type: object + additionalProperties: false + required: [total, artists] + properties: + total: {type: integer, description: 'How many artists have songs that were not found.'} + artists: + type: array + description: The artists with the most songs that were not found, at most 50, most first. + items: + type: object + additionalProperties: false + required: [name, count] + properties: + name: {type: string} + count: {type: integer, description: 'Plays and playlist songs of this artist that were not found.'} + MastodonAccount: type: object additionalProperties: false diff --git a/internal/spotifyimport/match.go b/internal/spotifyimport/match.go new file mode 100644 index 0000000..ff30d57 --- /dev/null +++ b/internal/spotifyimport/match.go @@ -0,0 +1,111 @@ +package spotifyimport + +import ( + "regexp" + "strings" +) + +// Candidate is a song of the library that can be played: what a Spotify entry is compared with. +type Candidate struct { + TrackID int64 + Title string + Album string // empty when it has none + Artists []string + DurationMs int64 +} + +// Index finds the song of the library that a Spotify entry is. It is built once for an import, from every playable +// song, and answers from memory. +type Index struct { + key func(string) string + byArtist map[string][]entry // artist key + title key + byBase map[string][]entry // artist key + title key without a remaster note + byAlbum map[string][]entry // album key + title key, for a compilation whose artist is "Various Artists" + lengths map[int64]int64 +} + +type entry struct { + id int64 + albumKey string +} + +// NewIndex builds the index. key makes the comparable form of a name (the same one the library search uses: no case, +// no accents, no punctuation). Candidates come in order of id, and the first of equal ones wins. +func NewIndex(key func(string) string, songs []Candidate) *Index { + ix := &Index{ + key: key, + byArtist: map[string][]entry{}, + byBase: map[string][]entry{}, + byAlbum: map[string][]entry{}, + lengths: map[int64]int64{}, + } + for _, s := range songs { + ix.lengths[s.TrackID] = s.DurationMs + title, base, album := key(s.Title), key(baseTitle(s.Title)), key(s.Album) + e := entry{id: s.TrackID, albumKey: album} + for _, artist := range s.Artists { + a := key(artist) + ix.byArtist[a+"\x00"+title] = append(ix.byArtist[a+"\x00"+title], e) + ix.byBase[a+"\x00"+base] = append(ix.byBase[a+"\x00"+base], e) + } + if album != "" { + ix.byAlbum[album+"\x00"+title] = append(ix.byAlbum[album+"\x00"+title], e) + } + } + return ix +} + +// Length is how long a song is, in milliseconds (0 when the library does not know). +func (ix *Index) Length(trackID int64) int64 { return ix.lengths[trackID] } + +// Find gives the song of the library that an artist, a title and (when known) an album name, or false. The exact +// title comes first, then the title without a remaster note on either side; when several songs are equal the one of +// the album that was asked wins, else the first. A play of a compilation is credited to "Various Artists" on Spotify, +// which no song of the library is by: for that the album and the title are enough, when that names exactly one song. +func (ix *Index) Find(artist, title, album string) (int64, bool) { + a, t, b := ix.key(artist), ix.key(title), ix.key(album) + if t == "" { + return 0, false + } + if a != "" { + if id, ok := pick(ix.byArtist[a+"\x00"+t], b); ok { + return id, true + } + if id, ok := pick(ix.byBase[a+"\x00"+ix.key(baseTitle(title))], b); ok { + return id, true + } + } + if b != "" { + if matches := ix.byAlbum[b+"\x00"+t]; len(matches) == 1 { + return matches[0].id, true + } + } + return 0, false +} + +func pick(matches []entry, albumKey string) (int64, bool) { + if len(matches) == 0 { + return 0, false + } + if albumKey != "" { + for _, m := range matches { + if m.albumKey == albumKey { + return m.id, true + } + } + } + return matches[0].id, true +} + +// remaster is a note that says a song was remastered, in brackets or after a dash: "(Remastered 2011)", +// "- 2017 Remaster", "[Remaster]". Only the note at the end of the title goes. +var remaster = regexp.MustCompile(`(?i)\s*(?:[-–—]\s*[^()\[\]]*remaster(?:ed)?[^()\[\]]*|\([^()]*remaster(?:ed)?[^()]*\)|\[[^\[\]]*remaster(?:ed)?[^\[\]]*\])\s*$`) + +// baseTitle is a title without its remaster note, which is the same song as far as a listener is concerned. +func baseTitle(s string) string { + out := strings.TrimSpace(remaster.ReplaceAllString(s, "")) + if out == "" { + return strings.TrimSpace(s) + } + return out +} diff --git a/internal/spotifyimport/match_test.go b/internal/spotifyimport/match_test.go new file mode 100644 index 0000000..42e25f5 --- /dev/null +++ b/internal/spotifyimport/match_test.go @@ -0,0 +1,106 @@ +package spotifyimport + +import ( + "strings" + "testing" +) + +// key is a stand-in for the key the server uses to compare names: lower case, no accents, no punctuation. +func key(s string) string { + r := strings.NewReplacer("ö", "o", "é", "e", ".", "", ",", "", "'", "", "’", "", "!", "", "-", " ") + return strings.Join(strings.Fields(r.Replace(strings.ToLower(s))), " ") +} + +func library() *Index { + return NewIndex(key, []Candidate{ + {TrackID: 1, Title: "Karma Police", Album: "OK Computer", Artists: []string{"Radiohead"}}, + {TrackID: 2, Title: "Karma Police", Album: "OK Computer OKNOTOK", Artists: []string{"Radiohead"}}, + {TrackID: 3, Title: "Hyperballad", Album: "Post", Artists: []string{"Björk"}}, + {TrackID: 4, Title: "Heroes (2017 Remaster)", Album: "Heroes", Artists: []string{"David Bowie"}}, + {TrackID: 5, Title: "Love Will Tear Us Apart", Album: "Substance", Artists: []string{"Joy Division"}}, + {TrackID: 6, Title: "Under Pressure", Album: "Hot Space", Artists: []string{"Queen", "David Bowie"}}, + {TrackID: 7, Title: "Paranoid Android", Album: "Now That's What I Call Music", Artists: []string{"Radiohead"}}, + {TrackID: 8, Title: "Wonderwall", Album: "Greatest Hits", Artists: []string{"Oasis"}}, + }) +} + +func TestAnExactArtistAndTitleMatches(t *testing.T) { + ix := library() + for _, tc := range []struct { + artist, title, album string + want int64 + }{ + {"Radiohead", "Karma Police", "OK Computer", 1}, + {"radiohead", "KARMA POLICE", "", 1}, // case, and no album to tell the two apart: the first + {"Bjork", "Hyperballad", "Post", 3}, // accents do not matter + {"Joy Division", "Love Will Tear Us Apart!", "", 5}, + } { + if id, ok := ix.Find(tc.artist, tc.title, tc.album); !ok || id != tc.want { + t.Errorf("Find(%q, %q, %q) = %d, %v, want %d", tc.artist, tc.title, tc.album, id, ok, tc.want) + } + } +} + +func TestTheAlbumTellsTwoVersionsApart(t *testing.T) { + if id, ok := library().Find("Radiohead", "Karma Police", "OK Computer OKNOTOK"); !ok || id != 2 { + t.Errorf("= %d, %v, want the song of the album that was asked", id, ok) + } +} + +func TestARemasterSuffixDoesNotMatter(t *testing.T) { + ix := library() + for _, title := range []string{"Heroes - 2017 Remaster", "Heroes (Remastered 2017)", "Heroes - Remastered", "Heroes"} { + if id, ok := ix.Find("David Bowie", title, ""); !ok || id != 4 { + t.Errorf("Find(%q) = %d, %v, want 4", title, id, ok) + } + } + // The other way round: the library has the plain title and Spotify the remaster. + if id, ok := ix.Find("Queen", "Under Pressure - Remastered 2011", ""); !ok || id != 6 { + t.Errorf("a remastered title for a plain one = %d, %v, want 6", id, ok) + } +} + +func TestAnyOfTheCreditedArtistsMatches(t *testing.T) { + ix := library() + for _, artist := range []string{"Queen", "David Bowie"} { + if id, ok := ix.Find(artist, "Under Pressure", ""); !ok || id != 6 { + t.Errorf("Find(%q) = %d, %v, want 6", artist, id, ok) + } + } +} + +func TestACompilationIsFoundByItsAlbumWhenTheArtistIsVariousArtists(t *testing.T) { + if id, ok := library().Find("Various Artists", "Paranoid Android", "Now That's What I Call Music"); !ok || id != 7 { + t.Errorf("= %d, %v, want 7", id, ok) + } + // ...but not by the album alone when it does not hold the song. + if _, ok := library().Find("Various Artists", "Something Else", "Now That's What I Call Music"); ok { + t.Error("a song that the album does not hold was found") + } +} + +func TestWhatIsNotInTheLibraryIsNotFound(t *testing.T) { + ix := library() + for _, tc := range [][2]string{{"Radiohead", "Creep"}, {"Blur", "Karma Police"}, {"", "Karma Police"}, {"Radiohead", ""}} { + if id, ok := ix.Find(tc[0], tc[1], ""); ok { + t.Errorf("Find(%q, %q) = %d, want nothing", tc[0], tc[1], id) + } + } +} + +func TestTheBaseOfATitle(t *testing.T) { + for in, want := range map[string]string{ + "Heroes - 2017 Remaster": "Heroes", + "Heroes - Remastered 2017": "Heroes", + "Heroes (Remastered)": "Heroes", + "Heroes [2017 Remaster]": "Heroes", + "Heroes - Mono": "Heroes - Mono", // only a remaster is taken off + "Heroes (Live)": "Heroes (Live)", + "The Remaster of Remasters": "The Remaster of Remasters", + "Song (feat. X) - 2011 Remaster": "Song (feat. X)", + } { + if got := baseTitle(in); got != want { + t.Errorf("baseTitle(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/internal/spotifyimport/read.go b/internal/spotifyimport/read.go new file mode 100644 index 0000000..c7e889a --- /dev/null +++ b/internal/spotifyimport/read.go @@ -0,0 +1,307 @@ +// Package spotifyimport brings the listening history and the playlists of a Spotify data export into Sonda. +// +// Spotify gives a person a zip of JSON files when they ask for their data. This package reads those files (the zip +// itself, or the JSON files in it), matches each song to the library by artist and title, records the matches as +// listens and makes the playlists. It runs on the server and contacts nobody: the files are read from the request, +// never written anywhere, and the parts that a file holds besides songs (the IP addresses of the person, the country, +// the platform, the user name) are never even looked at. +package spotifyimport + +import ( + "archive/zip" + "bufio" + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "path" + "strings" + "time" +) + +// Limits on what one upload may hold, so that a zip that inflates into something enormous cannot fill the memory +// or the disk of the server. +var ( + maxInflated = int64(4 << 30) // all the JSON of one upload, once it is inflated + maxZipFiles = 2000 // files in one zip +) + +// ErrTooBig is returned for an upload that holds more than the limits allow. +var ErrTooBig = errors.New("the files hold more than can be imported") + +// ErrUnreadable is returned for a file that looks like one of Spotify's and cannot be read to its end (cut short, +// damaged). +var ErrUnreadable = errors.New("a file is damaged or cut short") + +// Entry is one play of a song from a streaming history file. +type Entry struct { + PlayedAt time.Time // when the play ended + Artist string + Title string + Album string // empty in the older files, which do not say + PlayedMs int64 + Skipped bool // the file says the listener skipped it + ReasonEnd string // why the play ended ("trackdone" when it ran to the end), empty in the older files +} + +// PlaylistItem is one song of a playlist. +type PlaylistItem struct{ Artist, Title, Album string } + +// Playlist is a playlist of the export. Items are its songs, in order; Other counts what is in it that is not a song +// (a podcast episode, a file the person had on their own computer). +type Playlist struct { + Name string + Items []PlaylistItem + Other int +} + +// Reader reads files of an export and calls back for what is in them. A callback that is nil is skipped. +type Reader struct { + OnEntry func(Entry) + OnPlaylist func(Playlist) + // OnIgnored is called for a play that is not a song (a podcast episode) or that has no time that can be read. + OnIgnored func() +} + +// ReadFile reads one upload: the zip of the export, or one of the JSON files in it. It reports whether it held +// something of Spotify's (a history or playlists); a file that holds none is not an error, it is just not found. +func (r Reader) ReadFile(ra io.ReaderAt, size int64) (bool, error) { + found, err := r.readFile(ra, size) + if err != nil && !errors.Is(err, ErrTooBig) { + err = fmt.Errorf("%w: %v", ErrUnreadable, err) + } + return found, err +} + +func (r Reader) readFile(ra io.ReaderAt, size int64) (bool, error) { + head := make([]byte, 4) + if n, _ := ra.ReadAt(head, 0); n == 4 && bytes.Equal(head, []byte("PK\x03\x04")) { + return r.readZip(ra, size) + } + budget := maxInflated + return r.readJSON(&budgeted{r: io.NewSectionReader(ra, 0, size), left: &budget}) +} + +func (r Reader) readZip(ra io.ReaderAt, size int64) (bool, error) { + zr, err := zip.NewReader(ra, size) + if err != nil { + return false, nil // looks like a zip and is not one: nothing of Spotify's + } + if len(zr.File) > maxZipFiles { + return false, ErrTooBig + } + budget := maxInflated + found := false + for _, f := range zr.File { + if !wanted(f.Name) { + continue + } + rc, err := f.Open() + if err != nil { + return found, fmt.Errorf("open %s: %w", path.Base(f.Name), err) + } + ok, err := r.readJSON(&budgeted{r: rc, left: &budget}) + _ = rc.Close() + if err != nil { + return found, fmt.Errorf("read %s: %w", path.Base(f.Name), err) + } + found = found || ok + } + return found, nil +} + +// wanted says whether a file of the zip can hold what is imported: the audio history (not the video one) and the +// playlists. The rest of an export (the account, the searches, the library) is never opened. +func wanted(name string) bool { + base := strings.ToLower(path.Base(name)) + if !strings.HasSuffix(base, ".json") || strings.Contains(base, "video") { + return false + } + return strings.HasPrefix(base, "streaming_history") || strings.HasPrefix(base, "streaminghistory") || strings.HasPrefix(base, "playlist") +} + +// budgeted stops a reader that gives more than the budget left for the whole upload. +type budgeted struct { + r io.Reader + left *int64 +} + +func (b *budgeted) Read(p []byte) (int, error) { + if *b.left <= 0 { + return 0, ErrTooBig + } + if int64(len(p)) > *b.left { + p = p[:*b.left] + } + n, err := b.r.Read(p) + *b.left -= int64(n) + return n, err +} + +// rawEntry has the fields of a play that are used, in both shapes of file: the extended history (ts, ms_played, +// master_metadata_...) and the older one (endTime, msPlayed, trackName). Nothing else is decoded. +type rawEntry struct { + TS string `json:"ts"` + MsPlayed int64 `json:"ms_played"` + Track *string `json:"master_metadata_track_name"` + Artist *string `json:"master_metadata_album_artist_name"` + Album *string `json:"master_metadata_album_album_name"` + ReasonEnd string `json:"reason_end"` + Skipped *bool `json:"skipped"` + + EndTime string `json:"endTime"` + OldArtist string `json:"artistName"` + OldTrack string `json:"trackName"` + OldMsPlayed int64 `json:"msPlayed"` +} + +type rawPlaylists struct { + Playlists *[]struct { + Name string `json:"name"` + Items []struct { + Track *struct { + Name string `json:"trackName"` + Artist string `json:"artistName"` + Album string `json:"albumName"` + } `json:"track"` + } `json:"items"` + } `json:"playlists"` +} + +func (r Reader) readJSON(in io.Reader) (bool, error) { + br := bufio.NewReaderSize(in, 64<<10) + first, err := firstByte(br) + if err != nil { + if errors.Is(err, ErrTooBig) { + return false, err + } + return false, nil // empty, or not readable: nothing of Spotify's + } + dec := json.NewDecoder(br) + switch first { + case '[': + return r.readHistory(dec) + case '{': + return r.readPlaylists(dec) + } + return false, nil +} + +// firstByte looks at the first byte that is not a space, without taking it. +func firstByte(br *bufio.Reader) (byte, error) { + for { + b, err := br.ReadByte() + if err != nil { + return 0, err + } + if b == ' ' || b == '\n' || b == '\r' || b == '\t' || b == 0xEF || b == 0xBB || b == 0xBF { // spaces and a byte order mark + continue + } + return b, br.UnreadByte() + } +} + +func (r Reader) readHistory(dec *json.Decoder) (bool, error) { + if _, err := dec.Token(); err != nil { // the [ + return false, nil + } + found := false + for dec.More() { + var raw rawEntry + if err := dec.Decode(&raw); err != nil { + var typeErr *json.UnmarshalTypeError + if errors.As(err, &typeErr) { + continue // an element that is not an object: not a play + } + return found, err + } + stamp := raw.TS + if stamp == "" { + stamp = raw.EndTime + } + if stamp == "" { + continue // not a play at all: a file that only looks like a history + } + found = true + e, ok := raw.entry(stamp) + if !ok { + if r.OnIgnored != nil { + r.OnIgnored() + } + continue + } + if r.OnEntry != nil { + r.OnEntry(e) + } + } + return found, nil +} + +// entry is the play, or false for something that is not a song or has no time that can be read. +func (raw rawEntry) entry(stamp string) (Entry, bool) { + at, ok := parseTime(stamp) + if !ok { + return Entry{}, false + } + e := Entry{PlayedAt: at, PlayedMs: raw.MsPlayed, ReasonEnd: raw.ReasonEnd} + if raw.Skipped != nil { + e.Skipped = *raw.Skipped + } + if raw.Track != nil { + e.Title = *raw.Track + } + if raw.Artist != nil { + e.Artist = *raw.Artist + } + if raw.Album != nil { + e.Album = *raw.Album + } + if e.Title == "" && raw.OldTrack != "" { // the older shape + e.Title, e.Artist, e.PlayedMs = raw.OldTrack, raw.OldArtist, raw.OldMsPlayed + } + e.Title, e.Artist, e.Album = strings.TrimSpace(e.Title), strings.TrimSpace(e.Artist), strings.TrimSpace(e.Album) + if e.Title == "" || e.Artist == "" || e.PlayedMs < 0 { + return Entry{}, false // an episode of a podcast has no song name + } + return e, true +} + +// parseTime reads the time of a play: an RFC 3339 time in the extended history, "2022-01-02 08:30" (UTC) in the older one. +func parseTime(s string) (time.Time, bool) { + for _, layout := range []string{time.RFC3339, "2006-01-02 15:04", "2006-01-02 15:04:05"} { + if t, err := time.ParseInLocation(layout, strings.TrimSpace(s), time.UTC); err == nil { + return t.UTC(), true + } + } + return time.Time{}, false +} + +func (r Reader) readPlaylists(dec *json.Decoder) (bool, error) { + var raw rawPlaylists + if err := dec.Decode(&raw); err != nil { + if errors.Is(err, ErrTooBig) { + return false, err + } + return false, nil + } + if raw.Playlists == nil { + return false, nil + } + for _, p := range *raw.Playlists { + out := Playlist{Name: strings.TrimSpace(p.Name)} + for _, it := range p.Items { + if it.Track == nil || strings.TrimSpace(it.Track.Name) == "" || strings.TrimSpace(it.Track.Artist) == "" { + out.Other++ + continue + } + out.Items = append(out.Items, PlaylistItem{ + Artist: strings.TrimSpace(it.Track.Artist), Title: strings.TrimSpace(it.Track.Name), Album: strings.TrimSpace(it.Track.Album), + }) + } + if r.OnPlaylist != nil { + r.OnPlaylist(out) + } + } + return true, nil +} diff --git a/internal/spotifyimport/read_test.go b/internal/spotifyimport/read_test.go new file mode 100644 index 0000000..e7579fb --- /dev/null +++ b/internal/spotifyimport/read_test.go @@ -0,0 +1,183 @@ +package spotifyimport + +import ( + "archive/zip" + "bytes" + "errors" + "strings" + "testing" + "time" +) + +const extended = `[ + {"ts":"2023-05-01T10:00:00Z","username":"someone","platform":"linux","ms_played":215000,"conn_country":"IT","ip_addr":"203.0.113.9", + "master_metadata_track_name":"Karma Police","master_metadata_album_artist_name":"Radiohead","master_metadata_album_album_name":"OK Computer", + "spotify_track_uri":"spotify:track:1","reason_start":"trackdone","reason_end":"trackdone","shuffle":false,"skipped":null}, + {"ts":"2023-05-01T10:04:00Z","ms_played":4000,"master_metadata_track_name":"Airbag","master_metadata_album_artist_name":"Radiohead", + "master_metadata_album_album_name":"OK Computer","reason_end":"fwdbtn","skipped":true}, + {"ts":"2023-05-01T11:00:00Z","ms_played":900000,"master_metadata_track_name":null,"master_metadata_album_artist_name":null, + "episode_name":"Episode 12","episode_show_name":"A podcast","spotify_episode_uri":"spotify:episode:9"} +]` + +const legacy = `[ + {"endTime":"2022-01-02 08:30","artistName":"Björk","trackName":"Hyperballad","msPlayed":321000}, + {"endTime":"not a time","artistName":"X","trackName":"Y","msPlayed":1} +]` + +const playlistsFile = `{"playlists":[ + {"name":"Road","lastModifiedDate":"2023-01-01","items":[ + {"track":{"trackName":"Karma Police","artistName":"Radiohead","albumName":"OK Computer","trackUri":"spotify:track:1"},"episode":null,"localTrack":null,"addedDate":"2020-01-01"}, + {"track":null,"episode":{"name":"An episode"},"localTrack":null}, + {"track":null,"episode":null,"localTrack":{"uri":"spotify:local:::a:1"}} + ]}, + {"name":"Empty","items":[]} +]}` + +type got struct { + entries []Entry + playlists []Playlist + ignored int +} + +func read(t *testing.T, data []byte) (got, bool) { + t.Helper() + var g got + r := Reader{ + OnEntry: func(e Entry) { g.entries = append(g.entries, e) }, + OnPlaylist: func(p Playlist) { g.playlists = append(g.playlists, p) }, + OnIgnored: func() { g.ignored++ }, + } + found, err := r.ReadFile(bytes.NewReader(data), int64(len(data))) + if err != nil { + t.Fatal(err) + } + return g, found +} + +func TestReadsTheExtendedStreamingHistory(t *testing.T) { + g, found := read(t, []byte(extended)) + if !found || len(g.entries) != 2 || g.ignored != 1 { + t.Fatalf("found=%v entries=%d ignored=%d, want true, 2 and 1 (the podcast)", found, len(g.entries), g.ignored) + } + e := g.entries[0] + if e.Title != "Karma Police" || e.Artist != "Radiohead" || e.Album != "OK Computer" || e.PlayedMs != 215000 || e.ReasonEnd != "trackdone" || e.Skipped { + t.Errorf("first entry = %+v", e) + } + if !e.PlayedAt.Equal(time.Date(2023, 5, 1, 10, 0, 0, 0, time.UTC)) { + t.Errorf("played at %v", e.PlayedAt) + } + if s := g.entries[1]; !s.Skipped || s.ReasonEnd != "fwdbtn" { + t.Errorf("second entry = %+v, want a skip", s) + } +} + +func TestReadsTheOlderStreamingHistoryAndDropsWhatHasNoTime(t *testing.T) { + g, found := read(t, []byte(legacy)) + if !found || len(g.entries) != 1 { + t.Fatalf("found=%v entries=%d, want true and 1", found, len(g.entries)) + } + e := g.entries[0] + if e.Artist != "Björk" || e.Title != "Hyperballad" || e.Album != "" || e.PlayedMs != 321000 { + t.Errorf("entry = %+v", e) + } + if !e.PlayedAt.Equal(time.Date(2022, 1, 2, 8, 30, 0, 0, time.UTC)) { + t.Errorf("played at %v", e.PlayedAt) + } + if g.ignored != 1 { + t.Errorf("ignored = %d, want the entry with no readable time counted", g.ignored) + } +} + +func TestReadsThePlaylists(t *testing.T) { + g, found := read(t, []byte(playlistsFile)) + if !found || len(g.playlists) != 2 { + t.Fatalf("found=%v playlists=%d", found, len(g.playlists)) + } + p := g.playlists[0] + if p.Name != "Road" || len(p.Items) != 1 || p.Other != 2 { + t.Fatalf("playlist = %+v, want one song and two that are not", p) + } + if it := p.Items[0]; it.Title != "Karma Police" || it.Artist != "Radiohead" || it.Album != "OK Computer" { + t.Errorf("item = %+v", it) + } +} + +func TestSomethingThatIsNotSpotifyDataIsNotFound(t *testing.T) { + for name, data := range map[string]string{ + "other json": `{"hello":"world"}`, + "other array": `[{"query":"x","time":"y"}]`, + "not json": `just words`, + "empty": ``, + "a number": `42`, + "a null playlist": `{"playlists":null}`, + } { + g, found := read(t, []byte(data)) + if found || len(g.entries) != 0 || len(g.playlists) != 0 { + t.Errorf("%s: found=%v %+v, want nothing", name, found, g) + } + } +} + +func makeZip(t *testing.T, files map[string]string) []byte { + t.Helper() + var buf bytes.Buffer + w := zip.NewWriter(&buf) + for name, body := range files { + f, err := w.Create(name) + if err != nil { + t.Fatal(err) + } + if _, err := f.Write([]byte(body)); err != nil { + t.Fatal(err) + } + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func TestReadsTheZipOfTheExport(t *testing.T) { + data := makeZip(t, map[string]string{ + "Spotify Extended Streaming History/Streaming_History_Audio_2019-2023_0.json": extended, + "Spotify Extended Streaming History/Streaming_History_Video_2019-2023.json": extended, // videos are not music + "Spotify Account Data/StreamingHistory_music_0.json": legacy, + "Spotify Account Data/Playlist1.json": playlistsFile, + "Spotify Account Data/Userdata.json": `{"username":"someone","email":"someone@example.org"}`, + "Spotify Account Data/SearchQueries.json": `[{"platform":"x","searchTime":"y","searchQuery":"z"}]`, + "README.pdf": "ignored", + }) + g, found := read(t, data) + if !found { + t.Fatal("the zip of an export was not recognised") + } + if len(g.entries) != 3 || len(g.playlists) != 2 { + t.Errorf("entries=%d playlists=%d, want 3 (two extended, one older) and 2", len(g.entries), len(g.playlists)) + } +} + +func TestAZipWithNothingOfSpotifyIsNotFound(t *testing.T) { + _, found := read(t, makeZip(t, map[string]string{"notes.txt": "hello", "Userdata.json": `{"a":1}`})) + if found { + t.Error("a zip with no history and no playlists was taken for an export") + } +} + +func TestAZipThatInflatesTooMuchIsRefused(t *testing.T) { + old := maxInflated + maxInflated = 100 + defer func() { maxInflated = old }() + data := makeZip(t, map[string]string{"Streaming_History_Audio_0.json": "[" + strings.Repeat(" ", 1000) + "]"}) + r := Reader{} + if _, err := r.ReadFile(bytes.NewReader(data), int64(len(data))); !errors.Is(err, ErrTooBig) { + t.Errorf("err = %v, want ErrTooBig for a file that inflates past the limit", err) + } +} + +func TestAFileThatIsCutShortIsUnreadable(t *testing.T) { + data := []byte(extended[:len(extended)-40]) + _, err := Reader{}.ReadFile(bytes.NewReader(data), int64(len(data))) + if !errors.Is(err, ErrUnreadable) { + t.Errorf("err = %v, want ErrUnreadable", err) + } +} diff --git a/internal/spotifyimport/service.go b/internal/spotifyimport/service.go new file mode 100644 index 0000000..0b92055 --- /dev/null +++ b/internal/spotifyimport/service.go @@ -0,0 +1,379 @@ +package spotifyimport + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "sort" + "strings" + "sync" + "time" + "unicode/utf8" + + "codeberg.org/squidd/sonda/internal/listening" + "codeberg.org/squidd/sonda/internal/playlists" +) + +// Errors returned by Import. +var ( + // ErrNothingFound means no file held a streaming history or playlists of Spotify. + ErrNothingFound = errors.New("no Spotify history or playlists in these files") + // ErrBusy means this person's import is still running. + ErrBusy = errors.New("an import is already running") +) + +const ( + // MaxMissing is how many missing artists a report lists (the ones played most); MissingTotal says how many there are. + MaxMissing = 50 + + // listenBatch is how many listens are recorded in one go. + listenBatch = 2000 + + // completeShare says a play counts as heard to the end when this much of the song was played (nine tenths). + completeShareNum, completeShareDen = 9, 10 + + // A play longer than the song by more than this is not believed: it is brought to the length of the song, as the + // rules of listening would otherwise refuse it. + maxOverrunFactor = 2 + maxOverrunSlack = 10_000 // ms + + unnamedPlaylist = "Spotify playlist" +) + +// Catalog gives the songs of the library that can be played. +type Catalog interface { + Candidates(ctx context.Context) ([]Candidate, error) +} + +// Listens records many listens at once. A listen whose idempotency key is already there is not recorded again, and +// is not counted in what comes back. +type Listens interface { + RecordMany(ctx context.Context, userID int64, events []listening.Event) (created int, err error) +} + +// Playlists is what the import needs from the playlists of a person. +type Playlists interface { + List(ctx context.Context, userID int64, p playlists.Page) (playlists.Listing, error) + Create(ctx context.Context, userID int64, name, description string) (playlists.Playlist, error) + AddTracks(ctx context.Context, userID, id int64, trackIDs []int64, position *int) (playlists.Playlist, error) +} + +// File is one upload: the zip of the export or one of its JSON files. +type File struct { + R io.ReaderAt + Size int64 +} + +// History says what became of the plays of the history files. +type History struct { + Entries int // plays of songs that were read + Matched int // of those, the ones that are a song of the library + Imported int // recorded now + AlreadyThere int // matched, and already recorded by an earlier import (or the same play in two files) + NotFound int // not in the library + Ignored int // podcast episodes, and plays with no time that can be read +} + +// PlaylistResult says what became of one playlist. Status is `created`, `exists` (the person already has one with +// that name, which is left alone), `empty` (none of its songs is in the library) or `limit` (the person has as many +// playlists as are allowed). +type PlaylistResult struct { + Name string + Tracks int // everything that is in it + Matched int // the songs of the library + Status string +} + +// Missing is an artist that has plays or playlist songs and none of them in the library. +type Missing struct { + Artist string + Count int +} + +// Report is what an import did. It is made for the person who asked and kept nowhere. +type Report struct { + History History + Playlists []PlaylistResult + Missing []Missing // the artists with the most songs that were not found, at most MaxMissing + MissingTotal int // how many artists have songs that were not found +} + +// Service imports the data of a Spotify export for one person at a time. +type Service struct { + catalog Catalog + listens Listens + lists Playlists + key func(string) string + minListen time.Duration + running sync.Map // user id -> struct{} +} + +// NewService builds a Service. key is how names are made comparable (the key of the library search); minListen is how +// long a play must last to count as a listen, so that a play shorter than that is recorded as a skip. +func NewService(catalog Catalog, listens Listens, lists Playlists, key func(string) string, minListen time.Duration) *Service { + return &Service{catalog: catalog, listens: listens, lists: lists, key: key, minListen: minListen} +} + +// Import reads the files, records the listens that match the library and makes the playlists. A person can run one +// import at a time (ErrBusy). It is safe to import the same files again: nothing is recorded twice. What was +// recorded before an error stays recorded. +func (s *Service) Import(ctx context.Context, userID int64, files []File) (Report, error) { + if _, busy := s.running.LoadOrStore(userID, struct{}{}); busy { + return Report{}, ErrBusy + } + defer s.running.Delete(userID) + + songs, err := s.catalog.Candidates(ctx) + if err != nil { + return Report{}, fmt.Errorf("read the library: %w", err) + } + run := &run{svc: s, ctx: ctx, userID: userID, index: NewIndex(s.key, songs), memo: map[string]int64{}, missing: map[string]*Missing{}} + + reader := Reader{OnEntry: run.entry, OnPlaylist: run.playlist, OnIgnored: func() { run.report.History.Ignored++ }} + found := false + for _, f := range files { + ok, err := reader.ReadFile(f.R, f.Size) + if err != nil { + return Report{}, err + } + found = found || ok + if run.err != nil { + return Report{}, run.err + } + } + if !found { + return Report{}, ErrNothingFound + } + if err := run.flush(); err != nil { + return Report{}, err + } + if err := run.makePlaylists(); err != nil { + return Report{}, err + } + return run.finish(), nil +} + +// run is the state of one import. +type run struct { + svc *Service + ctx context.Context + userID int64 + index *Index + // memo remembers which song an artist, title and album are, so that a song played a thousand times is looked up once (0: not found). + memo map[string]int64 + batch []listening.Event + matched int + created int + missing map[string]*Missing // by the comparable name of the artist + playlists []Playlist + report Report + err error +} + +func (r *run) find(artist, title, album string) (int64, bool) { + k := artist + "\x00" + title + "\x00" + album + id, seen := r.memo[k] + if !seen { + id, _ = r.index.Find(artist, title, album) + r.memo[k] = id + } + return id, id != 0 +} + +func (r *run) miss(artist string) { + k := r.svc.key(artist) + m := r.missing[k] + if m == nil { + m = &Missing{Artist: artist} + r.missing[k] = m + } + m.Count++ +} + +func (r *run) entry(e Entry) { + if r.err != nil { + return + } + r.report.History.Entries++ + id, ok := r.find(e.Artist, e.Title, e.Album) + if !ok { + r.report.History.NotFound++ + r.miss(e.Artist) + return + } + r.matched++ + length := r.index.Length(id) + played := e.PlayedMs + if length > 0 && played > maxOverrunFactor*length+maxOverrunSlack { + played = length + } + r.batch = append(r.batch, listening.Event{ + UserID: r.userID, TrackID: id, Type: r.svc.kind(e, played, length), OccurredAt: e.PlayedAt, DurationPlayedMs: played, + IdempotencyKey: idempotencyKey(e), + }) + if len(r.batch) >= listenBatch { + r.err = r.flush() + } +} + +func (r *run) flush() error { + if len(r.batch) == 0 { + return nil + } + n, err := r.svc.listens.RecordMany(r.ctx, r.userID, r.batch) + if err != nil { + return fmt.Errorf("record the listens: %w", err) + } + r.created += n + r.batch = r.batch[:0] + return nil +} + +func (r *run) playlist(p Playlist) { r.playlists = append(r.playlists, p) } + +// kind says what a play was. The file says it when it can: a skip, or a play that ran to its end. Otherwise it is a +// complete listen when nine tenths of the song were played, a skip when it did not last as long as it takes to count +// as a listen, and a play for the rest. +func (s *Service) kind(e Entry, played, length int64) listening.EventType { + switch { + case e.Skipped: + return listening.Skip + case e.ReasonEnd == "trackdone": + return listening.Complete + case length > 0 && played*completeShareDen >= length*completeShareNum: + return listening.Complete + case played < s.minListen.Milliseconds(): + return listening.Skip + } + return listening.Play +} + +// idempotencyKey names a play so that the same play is never recorded twice: by the minute it ended (the older files +// only say the minute), the artist and the title. The same play in the extended history and in the older one is one. +func idempotencyKey(e Entry) string { + sum := sha256.Sum256([]byte(fmt.Sprintf("%d|%s|%s", e.PlayedAt.Unix()/60, strings.ToLower(e.Artist), strings.ToLower(e.Title)))) + return "spotify:" + hex.EncodeToString(sum[:])[:40] +} + +func (r *run) makePlaylists() error { + if len(r.playlists) == 0 { + return nil + } + have, err := r.playlistNames() + if err != nil { + return err + } + limit := false + for _, p := range r.playlists { + name := cleanName(p.Name) + res := PlaylistResult{Name: name, Tracks: len(p.Items) + p.Other} + var ids []int64 + for _, it := range p.Items { + if id, ok := r.find(it.Artist, it.Title, it.Album); ok { + ids = append(ids, id) + } else { + r.miss(it.Artist) + } + } + res.Matched = len(ids) + switch { + case len(ids) == 0: + res.Status = "empty" + case have[strings.ToLower(name)]: + res.Status = "exists" + case limit: + res.Status = "limit" + default: + created, err := r.fill(name, ids) + if err != nil { + return err + } + if created { + res.Status = "created" + have[strings.ToLower(name)] = true + } else { + res.Status, limit = "limit", true + } + } + r.report.Playlists = append(r.report.Playlists, res) + } + return nil +} + +// fill makes a playlist and puts the songs in it, a piece at a time. It reports false when the person already has as many +// playlists as are allowed. +func (r *run) fill(name string, ids []int64) (bool, error) { + pl, err := r.svc.lists.Create(r.ctx, r.userID, name, "") + if errors.Is(err, playlists.ErrTooMany) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("make the playlist %q: %w", name, err) + } + if len(ids) > playlists.MaxItems { + ids = ids[:playlists.MaxItems] + } + for len(ids) > 0 { + n := min(len(ids), playlists.MaxAddAtOnce) + if _, err := r.svc.lists.AddTracks(r.ctx, r.userID, pl.ID, ids[:n], nil); err != nil { + return false, fmt.Errorf("fill the playlist %q: %w", name, err) + } + ids = ids[n:] + } + return true, nil +} + +// playlistNames are the names the person already has, in lower case. +func (r *run) playlistNames() (map[string]bool, error) { + have := map[string]bool{} + for offset := 0; ; { + page, err := r.svc.lists.List(r.ctx, r.userID, playlists.Page{Limit: playlists.MaxLimit, Offset: offset}) + if err != nil { + return nil, fmt.Errorf("read the playlists: %w", err) + } + for _, p := range page.Items { + have[strings.ToLower(strings.TrimSpace(p.Name))] = true + } + offset += len(page.Items) + if len(page.Items) == 0 || offset >= page.Total { + return have, nil + } + } +} + +func cleanName(name string) string { + name = strings.TrimSpace(name) + if name == "" { + return unnamedPlaylist + } + if utf8.RuneCountInString(name) > playlists.MaxNameLen { + name = string([]rune(name)[:playlists.MaxNameLen]) + } + return name +} + +func (r *run) finish() Report { + h := &r.report.History + h.Matched = r.matched + h.Imported = r.created + h.AlreadyThere = r.matched - r.created + all := make([]Missing, 0, len(r.missing)) + for _, m := range r.missing { + all = append(all, *m) + } + sort.Slice(all, func(i, j int) bool { + if all[i].Count != all[j].Count { + return all[i].Count > all[j].Count + } + return strings.ToLower(all[i].Artist) < strings.ToLower(all[j].Artist) + }) + r.report.MissingTotal = len(all) + if len(all) > MaxMissing { + all = all[:MaxMissing] + } + r.report.Missing = all + return r.report +} diff --git a/internal/spotifyimport/service_test.go b/internal/spotifyimport/service_test.go new file mode 100644 index 0000000..51aa8c5 --- /dev/null +++ b/internal/spotifyimport/service_test.go @@ -0,0 +1,363 @@ +package spotifyimport + +import ( + "bytes" + "context" + "errors" + "fmt" + "strings" + "sync" + "testing" + "time" + + "codeberg.org/squidd/sonda/internal/listening" + "codeberg.org/squidd/sonda/internal/playlists" +) + +type fakeCatalog struct { + songs []Candidate + once sync.Once + entered chan struct{} // closed when Candidates is first called, when set + release chan struct{} // Candidates waits for it, when set +} + +func (f *fakeCatalog) Candidates(context.Context) ([]Candidate, error) { + if f.entered != nil { + f.once.Do(func() { close(f.entered) }) + } + if f.release != nil { + <-f.release + } + return f.songs, nil +} + +type fakeListens struct { + seen map[string]bool + events []listening.Event + calls int +} + +func (f *fakeListens) RecordMany(_ context.Context, userID int64, events []listening.Event) (int, error) { + f.calls++ + if f.seen == nil { + f.seen = map[string]bool{} + } + created := 0 + for _, e := range events { + k := fmt.Sprint(userID, e.IdempotencyKey) + if !f.seen[k] { + f.seen[k] = true + f.events = append(f.events, e) + created++ + } + } + return created, nil +} + +type fakePlaylists struct { + existing []string + made map[string][]int64 + order []string + max int // playlists a user may have; 0 is no limit + nextID int64 + names map[int64]string +} + +func (f *fakePlaylists) List(_ context.Context, _ int64, p playlists.Page) (playlists.Listing, error) { + var all []playlists.Playlist + for _, n := range f.existing { + all = append(all, playlists.Playlist{Name: n}) + } + end := min(p.Offset+p.Limit, len(all)) + start := min(p.Offset, end) + return playlists.Listing{Items: all[start:end], Total: len(all), Limit: p.Limit, Offset: p.Offset}, nil +} + +func (f *fakePlaylists) Create(_ context.Context, _ int64, name, _ string) (playlists.Playlist, error) { + if f.max > 0 && len(f.existing)+len(f.made) >= f.max { + return playlists.Playlist{}, playlists.ErrTooMany + } + if f.made == nil { + f.made, f.names = map[string][]int64{}, map[int64]string{} + } + f.nextID++ + f.names[f.nextID] = name + f.made[name] = nil + f.order = append(f.order, name) + return playlists.Playlist{ID: f.nextID, Name: name}, nil +} + +func (f *fakePlaylists) AddTracks(_ context.Context, _ int64, id int64, ids []int64, _ *int) (playlists.Playlist, error) { + if len(ids) > playlists.MaxAddAtOnce { + return playlists.Playlist{}, errors.New("too many at once") + } + f.made[f.names[id]] = append(f.made[f.names[id]], ids...) + return playlists.Playlist{ID: id}, nil +} + +var songs = []Candidate{ + {TrackID: 1, Title: "Karma Police", Album: "OK Computer", Artists: []string{"Radiohead"}, DurationMs: 260_000}, + {TrackID: 2, Title: "Airbag", Album: "OK Computer", Artists: []string{"Radiohead"}, DurationMs: 284_000}, + {TrackID: 3, Title: "Hyperballad", Album: "Post", Artists: []string{"Björk"}, DurationMs: 321_000}, +} + +type rig struct { + svc *Service + li *fakeListens + pl *fakePlaylists +} + +func newRig(cat *fakeCatalog) rig { + li, pl := &fakeListens{}, &fakePlaylists{} + return rig{NewService(cat, li, pl, key, 20*time.Second), li, pl} +} + +func upload(data string) File { return File{R: bytes.NewReader([]byte(data)), Size: int64(len(data))} } + +func play(ts, artist, title, album string, ms int, extra string) string { + return fmt.Sprintf(`{"ts":%q,"ms_played":%d,"master_metadata_track_name":%q,"master_metadata_album_artist_name":%q,"master_metadata_album_album_name":%q%s}`, + ts, ms, title, artist, album, extra) +} + +func history(plays ...string) string { return "[" + strings.Join(plays, ",") + "]" } + +func TestImportRecordsTheListensThatMatchTheLibrary(t *testing.T) { + r := newRig(&fakeCatalog{songs: songs}) + rep, err := r.svc.Import(context.Background(), 7, []File{upload(history( + play("2023-05-01T10:00:00Z", "Radiohead", "Karma Police", "OK Computer", 260_000, `,"reason_end":"trackdone"`), + play("2023-05-01T10:05:00Z", "Radiohead", "Airbag", "OK Computer", 4_000, `,"reason_end":"fwdbtn","skipped":true`), + play("2023-05-01T10:06:00Z", "Radiohead", "Creep", "Pablo Honey", 200_000, ``), + play("2023-05-01T10:10:00Z", "Björk", "Hyperballad", "Post", 321_000, ``), + `{"ts":"2023-05-01T11:00:00Z","ms_played":999,"master_metadata_track_name":null,"episode_name":"A show"}`, + ))}) + if err != nil { + t.Fatal(err) + } + h := rep.History + if h.Entries != 4 || h.Matched != 3 || h.Imported != 3 || h.AlreadyThere != 0 || h.NotFound != 1 || h.Ignored != 1 { + t.Errorf("history = %+v", h) + } + if len(r.li.events) != 3 { + t.Fatalf("%d events recorded, want 3", len(r.li.events)) + } + first := r.li.events[0] + if first.UserID != 7 || first.TrackID != 1 || first.Type != listening.Complete || first.DurationPlayedMs != 260_000 || + !first.OccurredAt.Equal(time.Date(2023, 5, 1, 10, 0, 0, 0, time.UTC)) || !strings.HasPrefix(first.IdempotencyKey, "spotify:") { + t.Errorf("first event = %+v", first) + } + if r.li.events[1].Type != listening.Skip { + t.Errorf("a skip was recorded as %q", r.li.events[1].Type) + } + if len(rep.Missing) != 1 || rep.Missing[0] != (Missing{Artist: "Radiohead", Count: 1}) { + t.Errorf("missing = %+v", rep.Missing) + } +} + +func TestTheKindOfAPlayComesFromHowItEndedOrHowMuchWasHeard(t *testing.T) { + r := newRig(&fakeCatalog{songs: songs}) + _, err := r.svc.Import(context.Background(), 1, []File{upload(history( + play("2023-01-01T10:00:00Z", "Radiohead", "Karma Police", "", 260_000, `,"reason_end":"trackdone"`), // ran to the end + play("2023-01-01T11:00:00Z", "Radiohead", "Karma Police", "", 250_000, `,"reason_end":"endplay"`), // 96 per cent, then stopped + play("2023-01-01T12:00:00Z", "Radiohead", "Karma Police", "", 90_000, `,"reason_end":"endplay"`), // a good part, then stopped + play("2023-01-01T13:00:00Z", "Radiohead", "Karma Police", "", 5_000, ``), // a few seconds, nothing said + play("2023-01-01T14:00:00Z", "Radiohead", "Karma Police", "", 90_000, `,"skipped":true`), // the file says skipped + play("2023-01-01T15:00:00Z", "Radiohead", "Karma Police", "", 9_999_999, `,"reason_end":"trackdone"`), // more than the song is long + ))}) + if err != nil { + t.Fatal(err) + } + want := []listening.EventType{listening.Complete, listening.Complete, listening.Play, listening.Skip, listening.Skip, listening.Complete} + for i, e := range r.li.events { + if e.Type != want[i] { + t.Errorf("play %d is %q, want %q", i, e.Type, want[i]) + } + } + // A play longer than the song would be refused by the listening rules: it is brought to the length of the song. + if last := r.li.events[5]; last.DurationPlayedMs != 260_000 { + t.Errorf("duration = %d, want the length of the song", last.DurationPlayedMs) + } +} + +func TestImportingTheSameFilesAgainAddsNothing(t *testing.T) { + r := newRig(&fakeCatalog{songs: songs}) + data := history( + play("2023-05-01T10:00:00Z", "Radiohead", "Karma Police", "OK Computer", 260_000, ``), + play("2023-05-01T10:10:00Z", "Björk", "Hyperballad", "Post", 321_000, ``), + ) + if _, err := r.svc.Import(context.Background(), 1, []File{upload(data)}); err != nil { + t.Fatal(err) + } + rep, err := r.svc.Import(context.Background(), 1, []File{upload(data)}) + if err != nil { + t.Fatal(err) + } + if rep.History.Imported != 0 || rep.History.AlreadyThere != 2 || len(r.li.events) != 2 { + t.Errorf("second import: %+v, %d events", rep.History, len(r.li.events)) + } +} + +func TestTheSamePlayInTheTwoShapesOfFileIsCountedOnce(t *testing.T) { + r := newRig(&fakeCatalog{songs: songs}) + extended := history(play("2022-01-02T08:30:41Z", "Björk", "Hyperballad", "Post", 321_000, `,"reason_end":"trackdone"`)) + older := `[{"endTime":"2022-01-02 08:30","artistName":"Björk","trackName":"Hyperballad","msPlayed":321000}]` + rep, err := r.svc.Import(context.Background(), 1, []File{upload(extended), upload(older)}) + if err != nil { + t.Fatal(err) + } + if len(r.li.events) != 1 || rep.History.Imported != 1 || rep.History.AlreadyThere != 1 { + t.Errorf("%d events, %+v: want the play once", len(r.li.events), rep.History) + } +} + +func TestPlaylistsBecomePlaylistsWithTheSongsThatWereFound(t *testing.T) { + r := newRig(&fakeCatalog{songs: songs}) + r.pl.existing = []string{"Mine already"} + rep, err := r.svc.Import(context.Background(), 1, []File{upload(`{"playlists":[ + {"name":"Road","items":[ + {"track":{"trackName":"Hyperballad","artistName":"Björk","albumName":"Post"}}, + {"track":{"trackName":"Creep","artistName":"Radiohead","albumName":"Pablo Honey"}}, + {"track":{"trackName":"Karma Police","artistName":"Radiohead","albumName":"OK Computer"}}, + {"track":null,"episode":{"name":"A show"}}]}, + {"name":"mine ALREADY","items":[{"track":{"trackName":"Airbag","artistName":"Radiohead"}}]}, + {"name":"Nothing here","items":[{"track":{"trackName":"Creep","artistName":"Radiohead"}}]}, + {"name":"","items":[{"track":{"trackName":"Airbag","artistName":"Radiohead"}}]} + ]}`)}) + if err != nil { + t.Fatal(err) + } + if got := r.pl.made["Road"]; len(got) != 2 || got[0] != 3 || got[1] != 1 { + t.Errorf("Road has %v, want the found songs in the order of the playlist: 3, 1", got) + } + if _, made := r.pl.made["mine ALREADY"]; made { + t.Error("a playlist whose name the person already has was made again") + } + if _, made := r.pl.made["Nothing here"]; made { + t.Error("a playlist with none of its songs found was made") + } + if got := r.pl.made["Spotify playlist"]; len(got) != 1 { + t.Errorf("a playlist with no name got %v under %v", got, r.pl.order) + } + want := []PlaylistResult{ + {Name: "Road", Tracks: 4, Matched: 2, Status: "created"}, + {Name: "mine ALREADY", Tracks: 1, Matched: 1, Status: "exists"}, + {Name: "Nothing here", Tracks: 1, Matched: 0, Status: "empty"}, + {Name: "Spotify playlist", Tracks: 1, Matched: 1, Status: "created"}, + } + if fmt.Sprint(rep.Playlists) != fmt.Sprint(want) { + t.Errorf("playlists = %+v, want %+v", rep.Playlists, want) + } + if len(rep.Missing) != 1 || rep.Missing[0].Artist != "Radiohead" || rep.Missing[0].Count != 2 { + t.Errorf("missing = %+v, want Radiohead twice (Creep in two playlists)", rep.Missing) + } +} + +func TestALongPlaylistIsAddedInPieces(t *testing.T) { + r := newRig(&fakeCatalog{songs: songs}) + var items []string + for i := 0; i < 1234; i++ { + items = append(items, `{"track":{"trackName":"Airbag","artistName":"Radiohead"}}`) + } + rep, err := r.svc.Import(context.Background(), 1, []File{upload(`{"playlists":[{"name":"Long","items":[` + strings.Join(items, ",") + `]}]}`)}) + if err != nil { + t.Fatal(err) + } + if len(r.pl.made["Long"]) != 1234 || rep.Playlists[0].Status != "created" { + t.Errorf("%d songs, %+v", len(r.pl.made["Long"]), rep.Playlists) + } +} + +func TestWhenThePersonHasAsManyPlaylistsAsAllowedTheRestAreSaidSo(t *testing.T) { + r := newRig(&fakeCatalog{songs: songs}) + r.pl.max = 1 + item := `{"track":{"trackName":"Airbag","artistName":"Radiohead"}}` + rep, err := r.svc.Import(context.Background(), 1, []File{upload(`{"playlists":[{"name":"A","items":[` + item + `]},{"name":"B","items":[` + item + `]},{"name":"C","items":[` + item + `]}]}`)}) + if err != nil { + t.Fatal(err) + } + got := []string{rep.Playlists[0].Status, rep.Playlists[1].Status, rep.Playlists[2].Status} + if fmt.Sprint(got) != "[created limit limit]" { + t.Errorf("statuses = %v", got) + } +} + +func TestTheArtistsThatAreMissingAreListedMostPlayedFirst(t *testing.T) { + r := newRig(&fakeCatalog{songs: songs}) + var plays []string + for i := 0; i < 5; i++ { + plays = append(plays, play(fmt.Sprintf("2023-01-01T10:0%d:00Z", i), "Blur", "Song 2", "", 120_000, ``)) + } + for i := 0; i < 3; i++ { + plays = append(plays, play(fmt.Sprintf("2023-01-02T10:0%d:00Z", i), "oasis", "Wonderwall", "", 120_000, ``)) + plays = append(plays, play(fmt.Sprintf("2023-01-03T10:0%d:00Z", i), "Oasis", "Supersonic", "", 120_000, ``)) + } + rep, err := r.svc.Import(context.Background(), 1, []File{upload(history(plays...))}) + if err != nil { + t.Fatal(err) + } + if len(rep.Missing) != 2 || rep.Missing[0] != (Missing{"Oasis", 6}) && rep.Missing[0] != (Missing{"oasis", 6}) || rep.Missing[1] != (Missing{"Blur", 5}) { + t.Errorf("missing = %+v, want Oasis 6 (the spellings together) then Blur 5", rep.Missing) + } +} + +func TestOnlyTheMostPlayedMissingArtistsAreListed(t *testing.T) { + r := newRig(&fakeCatalog{songs: songs}) + var plays []string + for i := 0; i < MaxMissing+30; i++ { + plays = append(plays, play("2023-01-01T10:00:00Z", fmt.Sprintf("Artist %03d", i), "Song", "", 120_000, ``)) + } + rep, err := r.svc.Import(context.Background(), 1, []File{upload(history(plays...))}) + if err != nil { + t.Fatal(err) + } + if len(rep.Missing) != MaxMissing || rep.MissingTotal != MaxMissing+30 { + t.Errorf("%d listed of %d, want %d of %d", len(rep.Missing), rep.MissingTotal, MaxMissing, MaxMissing+30) + } +} + +func TestFilesWithNothingOfSpotifyAreRefused(t *testing.T) { + r := newRig(&fakeCatalog{songs: songs}) + _, err := r.svc.Import(context.Background(), 1, []File{upload(`{"hello":"world"}`), upload("words")}) + if !errors.Is(err, ErrNothingFound) { + t.Errorf("err = %v, want ErrNothingFound", err) + } + if len(r.li.events) != 0 { + t.Error("something was recorded") + } +} + +func TestAnImportThatIsRunningBlocksAnotherOfTheSamePerson(t *testing.T) { + cat := &fakeCatalog{songs: songs, entered: make(chan struct{}), release: make(chan struct{})} + r := newRig(cat) + done := make(chan error, 1) + go func() { + _, err := r.svc.Import(context.Background(), 5, []File{upload(history(play("2023-01-01T10:00:00Z", "Radiohead", "Airbag", "", 1000, ``)))}) + done <- err + }() + <-cat.entered + if _, err := r.svc.Import(context.Background(), 5, []File{upload(`[]`)}); !errors.Is(err, ErrBusy) { + t.Errorf("a second import of the same person: %v, want ErrBusy", err) + } + close(cat.release) + if err := <-done; err != nil { + t.Fatal(err) + } + // Once it is over, the person can import again, and somebody else could all along. + if _, err := r.svc.Import(context.Background(), 5, []File{upload(`[]`)}); errors.Is(err, ErrBusy) { + t.Error("still busy after the import ended") + } +} + +func TestManyListensAreRecordedInBatches(t *testing.T) { + r := newRig(&fakeCatalog{songs: songs}) + var plays []string + for i := 0; i < 2500; i++ { + plays = append(plays, play(time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC).Add(time.Duration(i)*time.Minute).Format(time.RFC3339), "Radiohead", "Airbag", "", 100_000, ``)) + } + rep, err := r.svc.Import(context.Background(), 1, []File{upload(history(plays...))}) + if err != nil { + t.Fatal(err) + } + if rep.History.Imported != 2500 || r.li.calls < 2 { + t.Errorf("imported %d in %d calls, want 2500 in several", rep.History.Imported, r.li.calls) + } +} diff --git a/internal/storage/catalog.go b/internal/storage/catalog.go index 8ab122e..b2c2618 100644 --- a/internal/storage/catalog.go +++ b/internal/storage/catalog.go @@ -11,6 +11,7 @@ import ( "github.com/uptrace/bun/dialect" "codeberg.org/squidd/sonda/internal/catalog" + "codeberg.org/squidd/sonda/internal/spotifyimport" ) // CatalogRepo reads the catalog. It implements catalog.Store. @@ -569,3 +570,37 @@ func normalizeFormat(f string) string { } return f } + +// Candidates lists every song that has a file to play, with its album, its credited artists and its length, in order of +// id: what the import of a Spotify export compares its plays with. +func (r *CatalogRepo) Candidates(ctx context.Context) ([]spotifyimport.Candidate, error) { + rows, err := r.db.QueryContext(ctx, + `SELECT t.id, t.title, COALESCE(al.title, ''), COALESCE(t.duration_ms, 0), a.name + FROM tracks t + JOIN track_artists ta ON ta.track_id = t.id + JOIN artists a ON a.id = ta.artist_id + LEFT JOIN albums al ON al.id = t.album_id + WHERE EXISTS (SELECT 1 FROM audio_files f WHERE f.track_id = t.id AND f.missing_at IS NULL) + ORDER BY t.id, a.id`) + if err != nil { + return nil, fmt.Errorf("list the songs: %w", err) + } + defer rows.Close() + var out []spotifyimport.Candidate + for rows.Next() { + var ( + id, duration int64 + title, album string + artist string + ) + if err := rows.Scan(&id, &title, &album, &duration, &artist); err != nil { + return nil, err + } + if n := len(out); n > 0 && out[n-1].TrackID == id { + out[n-1].Artists = append(out[n-1].Artists, artist) + continue + } + out = append(out, spotifyimport.Candidate{TrackID: id, Title: title, Album: album, DurationMs: duration, Artists: []string{artist}}) + } + return out, rows.Err() +} diff --git a/internal/storage/listening.go b/internal/storage/listening.go index bc1946a..3e2bf7f 100644 --- a/internal/storage/listening.go +++ b/internal/storage/listening.go @@ -78,6 +78,42 @@ func (r *ListeningRepo) Record(ctx context.Context, e listening.Event) (bool, er return n > 0, nil } +// recordChunk is how many events RecordMany inserts with one statement: seven columns each, well below the limit of +// variables of either database. +const recordChunk = 500 + +// RecordMany inserts events of one user in one transaction, and says how many were new: an event whose idempotency key +// is already there (also one that is twice in the batch) is skipped. The tracks must exist. +func (r *ListeningRepo) RecordMany(ctx context.Context, userID int64, events []listening.Event) (int, error) { + created := 0 + err := r.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + for start := 0; start < len(events); start += recordChunk { + end := min(start+recordChunk, len(events)) + rows := make([]listeningRow, 0, end-start) + for _, e := range events[start:end] { + rows = append(rows, listeningRow{ + UserID: userID, TrackID: e.TrackID, EventType: string(e.Type), OccurredAt: e.OccurredAt.Unix(), + DurationPlayedMs: e.DurationPlayedMs, IdempotencyKey: e.IdempotencyKey, + }) + } + res, err := tx.NewInsert().Model(&rows).On("CONFLICT (user_id, idempotency_key) DO NOTHING").Exec(ctx) + if err != nil { + return fmt.Errorf("insert listening events: %w", err) + } + n, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("rows affected: %w", err) + } + created += int(n) + } + return nil + }) + if err != nil { + return 0, err + } + return created, nil +} + // TrackDurationMs returns the length of a track in milliseconds, 0 when it is // not known, or listening.ErrTrackNotFound. func (r *ListeningRepo) TrackDurationMs(ctx context.Context, trackID int64) (int64, error) { diff --git a/internal/storage/search_key.go b/internal/storage/search_key.go index ecdbf07..b05c276 100644 --- a/internal/storage/search_key.go +++ b/internal/storage/search_key.go @@ -34,3 +34,7 @@ func searchKey(s string) string { } return searchLetters.Replace(norm.NFC.String(b.String())) } + +// SearchKey is searchKey for the packages that compare names with the ones in the library (the import of a Spotify +// export): the same key, so that "Björk" and "Bjork" are one name there too. +func SearchKey(s string) string { return searchKey(s) } diff --git a/internal/storage/spotify_import_test.go b/internal/storage/spotify_import_test.go new file mode 100644 index 0000000..4e89a91 --- /dev/null +++ b/internal/storage/spotify_import_test.go @@ -0,0 +1,116 @@ +package storage + +import ( + "context" + "fmt" + "sort" + "testing" + "time" + + "github.com/uptrace/bun" + + "codeberg.org/squidd/sonda/internal/library" + "codeberg.org/squidd/sonda/internal/listening" +) + +func TestRecordManyStoresTheListensOnceAndCountsWhatIsNew(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + ctx := context.Background() + u, err := NewUserRepo(db).CreateUser(ctx, "alice", "h", false) + if err != nil { + t.Fatal(err) + } + var track int64 + if err := db.QueryRow("INSERT INTO tracks (title) VALUES ('t') RETURNING id").Scan(&track); err != nil { + t.Fatal(err) + } + repo := NewListeningRepo(db) + ev := func(key string, at int64) listening.Event { + return listening.Event{UserID: u.ID, TrackID: track, Type: listening.Complete, OccurredAt: time.Unix(at, 0), DurationPlayedMs: 1000, IdempotencyKey: key} + } + batch := []listening.Event{ev("a", 100), ev("b", 200), ev("a", 100)} // the same key twice in one batch + n, err := repo.RecordMany(ctx, u.ID, batch) + if err != nil || n != 2 { + t.Fatalf("RecordMany = %d, %v, want 2 new", n, err) + } + n, err = repo.RecordMany(ctx, u.ID, []listening.Event{ev("a", 100), ev("c", 300)}) + if err != nil || n != 1 { + t.Fatalf("second RecordMany = %d, %v, want only the new one", n, err) + } + if got := count(t, db, "SELECT COUNT(*) FROM listening_events WHERE user_id = ?", u.ID); got != 3 { + t.Errorf("%d events stored, want 3", got) + } + if n, err := repo.RecordMany(ctx, u.ID, nil); err != nil || n != 0 { + t.Errorf("an empty batch = %d, %v", n, err) + } + }) +} + +func TestRecordManyHandlesMoreThanOneStatementHolds(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + ctx := context.Background() + u, err := NewUserRepo(db).CreateUser(ctx, "alice", "h", false) + if err != nil { + t.Fatal(err) + } + var track int64 + if err := db.QueryRow("INSERT INTO tracks (title) VALUES ('t') RETURNING id").Scan(&track); err != nil { + t.Fatal(err) + } + var events []listening.Event + for i := 0; i < 5000; i++ { + events = append(events, listening.Event{UserID: u.ID, TrackID: track, Type: listening.Play, OccurredAt: time.Unix(int64(1000+i), 0), IdempotencyKey: fmt.Sprintf("k%d", i)}) + } + n, err := NewListeningRepo(db).RecordMany(ctx, u.ID, events) + if err != nil || n != 5000 { + t.Fatalf("RecordMany = %d, %v, want 5000", n, err) + } + }) +} + +func TestCandidatesAreTheSongsThatCanBePlayed(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + ctx := context.Background() + repo := NewLibraryRepo(db) + lib := mkLib(t, repo, "music") + a := parsed("a/1.flac", "Under Pressure", "Hot Space", "Queen", "David Bowie") + a.DurationMs = 248000 + b := parsed("a/2.flac", "Gone", "Album", "Radiohead") + c := parsed("a/3.flac", "Loose", "", "Björk") + c.Album, c.AlbumArtist = "", "" + for _, f := range []library.ParsedFile{a, b, c} { + if _, err := repo.SaveFile(ctx, lib, f, 0); err != nil { + t.Fatal(err) + } + } + if _, err := db.Exec("UPDATE audio_files SET missing_at = 5 WHERE rel_path = 'a/2.flac'"); err != nil { + t.Fatal(err) + } + got, err := NewCatalogRepo(db).Candidates(ctx) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 { + t.Fatalf("%d candidates, want 2 (the one with no file left is not there): %+v", len(got), got) + } + by := map[string]int{} + for i, g := range got { + by[g.Title] = i + } + up := got[by["Under Pressure"]] + artists := append([]string(nil), up.Artists...) + sort.Strings(artists) + if up.Album != "Hot Space" || up.DurationMs != 248000 || len(artists) != 2 || artists[0] != "David Bowie" || artists[1] != "Queen" || up.TrackID == 0 { + t.Errorf("Under Pressure = %+v", up) + } + if l := got[by["Loose"]]; l.Album != "" || len(l.Artists) != 1 || l.Artists[0] != "Björk" { + t.Errorf("a song with no album = %+v", l) + } + }) +} + +func TestSearchKeyIsExported(t *testing.T) { + if SearchKey("Björk") != "bjork" { + t.Errorf("SearchKey(Björk) = %q", SearchKey("Björk")) + } +} diff --git a/web/src/api/client.test.ts b/web/src/api/client.test.ts index 17f13d5..cf50ab6 100644 --- a/web/src/api/client.test.ts +++ b/web/src/api/client.test.ts @@ -85,6 +85,36 @@ test('a JSON body is sent with its content type', async () => { expect(JSON.parse(calls[0].init.body as string)).toEqual({ track_id: 3 }) }) +test('files are sent as a form, which sets its own content type, and with the token', async () => { + const { calls } = mockFetch({ 'POST /api/v1/me/import/spotify': () => ({ ok: true }) }) + const form = new FormData() + form.append('files', new File(['[]'], 'a.json')) + await api.upload('/me/import/spotify', form) + expect(calls[0].init.body).toBe(form) + const headers = calls[0].init.headers as Record + expect(headers['Content-Type']).toBeUndefined() // the browser adds it, with the boundary + expect(headers.Authorization).toBe('Bearer t1') +}) + +test('files that were refused with a 401 are sent again with the renewed token', async () => { + let first = true + const { calls } = mockFetch({ + 'POST /api/v1/me/import/spotify': () => { + if (first) { + first = false + return apiError(401, 'unauthorized', 'expired') + } + return { ok: true } + }, + }) + const form = new FormData() + form.append('files', new File(['[]'], 'a.json')) + await api.upload('/me/import/spotify', form) + expect(calls).toHaveLength(2) + expect(calls[1].init.body).toBe(form) + expect(authorization(calls[1])).toBe('Bearer t2') +}) + test('keepalive lets a request outlive the page', async () => { const { calls } = mockFetch({ 'POST /api/v1/listens': () => ({ status: 'recorded' }) }) await api.post('/listens', { track_id: 3 }, { keepalive: true }) diff --git a/web/src/api/client.ts b/web/src/api/client.ts index 426b2cf..f9348f1 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -83,13 +83,15 @@ async function send( signal: AbortSignal | undefined, ): Promise { const headers: Record = { Accept: 'application/json' } - if (body !== undefined) headers['Content-Type'] = 'application/json' + // A form (files) is sent as it is: the browser writes its content type, which carries the boundary. + const form = body instanceof FormData + if (body !== undefined && !form) headers['Content-Type'] = 'application/json' if (token) headers.Authorization = `Bearer ${token}` try { return await fetch(url, { method, headers, - body: body === undefined ? undefined : JSON.stringify(body), + body: body === undefined ? undefined : form ? body : JSON.stringify(body), keepalive: keepalive || undefined, signal, }) @@ -119,6 +121,8 @@ async function request(method: string, path: string, params: Query | undefine export const api = { get: (path: string, params?: Query, opts: CallOptions = {}) => request('GET', path, params, undefined, opts), post: (path: string, body?: unknown, opts: CallOptions = {}) => request('POST', path, undefined, body ?? {}, opts), + /** Sends files, as a form. */ + upload: (path: string, form: FormData, opts: CallOptions = {}) => request('POST', path, undefined, form, opts), patch: (path: string, body?: unknown, opts: CallOptions = {}) => request('PATCH', path, undefined, body ?? {}, opts), put: (path: string, body?: unknown, opts: CallOptions = {}) => request('PUT', path, undefined, body ?? {}, opts), del: (path: string, opts: CallOptions = {}) => request('DELETE', path, undefined, undefined, opts), diff --git a/web/src/api/importSpotify.ts b/web/src/api/importSpotify.ts new file mode 100644 index 0000000..1957612 --- /dev/null +++ b/web/src/api/importSpotify.ts @@ -0,0 +1,22 @@ +import { useMutation, useQueryClient } from '@tanstack/react-query' +import { api } from '@/api/client' +import type { components } from '@/api/schema' + +export type SpotifyImportReport = components['schemas']['SpotifyImportReport'] + +/** + * Sends the files of a Spotify export (the zip, or the JSON files in it) to this server, which matches them to the + * library and records the listens and the playlists. When it is done everything that is made of listens or playlists is + * read again. + */ +export function useImportSpotify() { + const client = useQueryClient() + return useMutation({ + mutationFn: (files: File[]) => { + const form = new FormData() + for (const file of files) form.append('files', file) + return api.upload('/me/import/spotify', form) + }, + onSuccess: () => client.invalidateQueries(), + }) +} diff --git a/web/src/api/schema.d.ts b/web/src/api/schema.d.ts index 380b861..6cad3f7 100644 --- a/web/src/api/schema.d.ts +++ b/web/src/api/schema.d.ts @@ -1546,6 +1546,47 @@ export interface paths { patch?: never; trace?: never; }; + "/me/import/spotify": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Import your Spotify history and playlists + * @description Brings your listening history and your playlists from a Spotify data export into your own account, so that + * Mixes, Discover and the smart playlists know what you listened to before. Ask Spotify for your data (Account, + * Privacy, "Extended streaming history" and "Account data"), then send what it gives: the zip itself, or the + * JSON files in it (`Streaming_History_Audio_*.json`, the older `StreamingHistory_music_*.json`, and + * `Playlist*.json`), as a multipart form with one `files` field for each file. A request may be up to 512 MB. + * + * Everything happens on this server and nothing is sent to anyone. The files are read from the request and not + * kept, and of each play only the song, the artist, the album, the time and how it ended are read: the IP + * addresses, the country and the platform in the file are never looked at. + * + * A play is matched to a song of your library by artist and title (case, accents and punctuation do not matter, + * a "Remastered" note on either side does not either, and any of the credited artists will do; a play of a + * compilation is matched by its album when Spotify names "Various Artists"). A match is recorded as a listen at + * the time of the play: `complete` when the play ran to its end (or nine tenths of the song were played), + * `skip` when Spotify says it was skipped or it was shorter than a listen, `play` otherwise. Importing does not + * post anything to Mastodon. Importing the same files again records nothing twice, and the same play in the + * extended history and in the older one is counted once. + * + * A playlist of the export becomes a playlist of yours with the songs that were found, in order. One whose + * name you already have is left alone (`exists`), and one with none of its songs in your library is not made + * (`empty`). The report lists what was not found, by artist, most played first, so you can see what is missing + * from your library. One import of yours runs at a time. + */ + post: operations["importMySpotify"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/artists/{id}/lidarr": { parameters: { query?: never; @@ -3760,6 +3801,44 @@ export interface components { */ discover_from_listening: boolean; }; + SpotifyImportReport: { + history: { + /** @description Plays of songs that were read. */ + entries: number; + /** @description Of those, the plays of a song of your library. */ + matched: number; + /** @description Recorded now. */ + imported: number; + /** @description Matched and already recorded by an earlier import, or the same play in two files. */ + already_there: number; + /** @description Plays of songs that are not in your library. */ + not_found: number; + /** @description Podcast episodes, and plays with no time that can be read. */ + ignored: number; + }; + playlists: { + name: string; + /** @description Everything that was in the playlist, songs and not. */ + tracks: number; + /** @description The songs of your library. */ + matched: number; + /** + * @description `created`; `exists` (you already have one with that name, left alone); `empty` (none of its songs is in your library); `limit` (you have as many playlists as are allowed). + * @enum {string} + */ + status: "created" | "exists" | "empty" | "limit"; + }[]; + missing: { + /** @description How many artists have songs that were not found. */ + total: number; + /** @description The artists with the most songs that were not found, at most 50, most first. */ + artists: { + name: string; + /** @description Plays and playlist songs of this artist that were not found. */ + count: number; + }[]; + }; + }; MastodonAccount: { connected: boolean; /** @description The instance, like `https://mastodon.social`. Only when connected. */ @@ -6567,6 +6646,61 @@ export interface operations { 401: components["responses"]["Unauthorized"]; }; }; + importMySpotify: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "multipart/form-data": { + /** @description The zip of the export, or its JSON files. */ + files: string[]; + }; + }; + }; + responses: { + /** @description What the import did. */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SpotifyImportReport"]; + }; + }; + /** @description No files were sent (`invalid_input`), none of them holds a history or playlists of Spotify (`no_spotify_data`) or one is damaged or cut short (`unreadable_file`). */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Error"]; + }; + }; + 401: components["responses"]["Unauthorized"]; + /** @description An import of yours is still running (`import_running`). */ + 409: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Error"]; + }; + }; + /** @description The files are larger than 512 MB, or hold more than can be imported once inflated (`too_large`). */ + 413: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Error"]; + }; + }; + }; + }; getArtistInLidarr: { parameters: { query?: never; diff --git a/web/src/components/SpotifyImportBlock.test.tsx b/web/src/components/SpotifyImportBlock.test.tsx new file mode 100644 index 0000000..5203f85 --- /dev/null +++ b/web/src/components/SpotifyImportBlock.test.tsx @@ -0,0 +1,109 @@ +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { render, screen, within } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { beforeEach, expect, test, vi } from 'vitest' +import { initI18n } from '@/i18n' +import { apiError, mockFetch, type MockRoutes } from '@/test/mockFetch' +import SpotifyImportBlock from './SpotifyImportBlock' + +beforeEach(async () => { + vi.spyOn(navigator, 'language', 'get').mockReturnValue('en-US') + await initI18n() +}) + +const report = { + history: { entries: 120, matched: 100, imported: 80, already_there: 20, not_found: 20, ignored: 3 }, + playlists: [ + { name: 'Road', tracks: 20, matched: 15, status: 'created' }, + { name: 'Mine', tracks: 5, matched: 5, status: 'exists' }, + { name: 'Nothing', tracks: 4, matched: 0, status: 'empty' }, + ], + missing: { total: 3, artists: [{ name: 'Blur', count: 9 }, { name: 'Oasis', count: 4 }] }, +} + +function setup(over: MockRoutes = {}) { + const mocked = mockFetch({ 'POST /api/v1/me/import/spotify': () => report, ...over }) + const client = new QueryClient({ defaultOptions: { queries: { retry: false }, mutations: { retry: false } } }) + render( + + + , + ) + return mocked +} + +const zip = new File(['PK'], 'my_spotify_data.zip', { type: 'application/zip' }) +const json = new File(['[]'], 'Streaming_History_Audio_2020_0.json', { type: 'application/json' }) + +test('it says what is imported and that nothing leaves the server', () => { + setup() + expect(screen.getByText(/Extended streaming history/)).toBeInTheDocument() + expect(screen.getByText(/Nothing is sent anywhere and the files are not kept/)).toBeInTheDocument() + expect(screen.getByLabelText('Files from Spotify')).toHaveAttribute('multiple') + expect(screen.getByRole('button', { name: 'Import' })).toBeDisabled() +}) + +test('the files are sent together as a form, and the report says what was done', async () => { + const { calls } = setup() + await userEvent.upload(screen.getByLabelText('Files from Spotify'), [zip, json]) + expect(screen.getByText('2 files chosen')).toBeInTheDocument() + await userEvent.click(screen.getByRole('button', { name: 'Import' })) + + const done = await screen.findByRole('region', { name: 'Import done' }) + const sent = calls.filter((c) => c.key === 'POST /api/v1/me/import/spotify') + expect(sent).toHaveLength(1) + const form = sent[0].init.body as FormData + expect(form.getAll('files').map((f) => (f as File).name)).toEqual(['my_spotify_data.zip', 'Streaming_History_Audio_2020_0.json']) + + expect(within(done).getByText('80 listens added to your history.')).toBeInTheDocument() + expect(within(done).getByText('20 were already there.')).toBeInTheDocument() + expect(within(done).getByText('20 plays are of songs that are not in your library.')).toBeInTheDocument() + expect(within(done).getByText(/3 plays skipped/)).toBeInTheDocument() + expect(within(done).getByText('Road: 15 of 20 songs found')).toBeInTheDocument() + expect(within(done).getByText('Made')).toBeInTheDocument() + expect(within(done).getByText('You already have one with this name')).toBeInTheDocument() + expect(within(done).getByText('None of its songs is in your library')).toBeInTheDocument() + expect(within(done).getByText('Blur')).toBeInTheDocument() + expect(within(done).getByText('9')).toBeInTheDocument() + expect(within(done).getByText('and 1 more')).toBeInTheDocument() // 3 artists have songs not found, 2 are shown +}) + +test('while it works the button is off and says so', async () => { + let finish: (r: Response) => void = () => {} + setup({ 'POST /api/v1/me/import/spotify': () => new Promise((resolve) => (finish = resolve)) }) + await userEvent.upload(screen.getByLabelText('Files from Spotify'), zip) + await userEvent.click(screen.getByRole('button', { name: 'Import' })) + expect(await screen.findByRole('status')).toHaveTextContent(/Importing/) + expect(screen.getByRole('button', { name: 'Import' })).toBeDisabled() + finish(new Response(JSON.stringify(report), { status: 200, headers: { 'Content-Type': 'application/json' } })) + expect(await screen.findByRole('region', { name: 'Import done' })).toBeInTheDocument() +}) + +test('files that are not Spotify’s are explained', async () => { + setup({ 'POST /api/v1/me/import/spotify': () => apiError(400, 'no_spotify_data', 'nothing') }) + await userEvent.upload(screen.getByLabelText('Files from Spotify'), json) + await userEvent.click(screen.getByRole('button', { name: 'Import' })) + expect(await screen.findByRole('alert')).toHaveTextContent(/do not hold a Spotify streaming history or playlists/) + expect(screen.queryByRole('region', { name: 'Import done' })).not.toBeInTheDocument() +}) + +test('an import that is too large, damaged or already running has its own message', async () => { + for (const [code, text] of [ + ['too_large', /too large to import/], + ['unreadable_file', /damaged or cut short/], + ['import_running', /still running/], + ] as const) { + document.body.innerHTML = '' + setup({ 'POST /api/v1/me/import/spotify': () => apiError(code === 'too_large' ? 413 : code === 'import_running' ? 409 : 400, code, 'x') }) + await userEvent.upload(screen.getByLabelText('Files from Spotify'), zip) + await userEvent.click(screen.getByRole('button', { name: 'Import' })) + expect(await screen.findByRole('alert')).toHaveTextContent(text) + } +}) + +test('an import with no plays in it says so', async () => { + setup({ 'POST /api/v1/me/import/spotify': () => ({ ...report, history: { entries: 0, matched: 0, imported: 0, already_there: 0, not_found: 0, ignored: 0 }, playlists: [], missing: { total: 0, artists: [] } }) }) + await userEvent.upload(screen.getByLabelText('Files from Spotify'), json) + await userEvent.click(screen.getByRole('button', { name: 'Import' })) + expect(await screen.findByText('No plays in these files.')).toBeInTheDocument() +}) diff --git a/web/src/components/SpotifyImportBlock.tsx b/web/src/components/SpotifyImportBlock.tsx new file mode 100644 index 0000000..091039d --- /dev/null +++ b/web/src/components/SpotifyImportBlock.tsx @@ -0,0 +1,112 @@ +import { useState, type ChangeEvent } from 'react' +import { useTranslation } from 'react-i18next' +import { ApiError } from '@/api/client' +import { useImportSpotify, type SpotifyImportReport } from '@/api/importSpotify' +import { Button } from '@/components/ui/button' +import { errorMessageKey } from '@/i18n' + +/** + * Bringing the history and the playlists of a Spotify export in. The files are sent to this server, which reads them + * and keeps nothing of them but the listens and playlists that match the library; the report says what was done and + * which artists are missing from the library. + */ +export default function SpotifyImportBlock() { + const { t } = useTranslation() + const run = useImportSpotify() + const [files, setFiles] = useState([]) + + function choose(event: ChangeEvent) { + setFiles(Array.from(event.target.files ?? [])) + } + + return ( +
+

{t('spotify.intro')}

+

{t('spotify.privacy')}

+ +
+ + + {files.length > 0 &&

{t('spotify.chosen', { count: files.length })}

} +
+ + + + {run.isPending && ( +

+ {t('spotify.running')} +

+ )} + {run.isError && ( +

+ {t(errorMessageKey(run.error instanceof ApiError ? run.error.code : 'unknown'))} +

+ )} + {run.data && !run.isPending && } +
+ ) +} + +function Report({ report }: { report: SpotifyImportReport }) { + const { t } = useTranslation() + const h = report.history + const more = report.missing.total - report.missing.artists.length + return ( +
+

{t('spotify.done')}

+ + {h.entries === 0 && report.playlists.length === 0 ? ( +

{t('spotify.noPlays')}

+ ) : ( + h.entries > 0 && ( +
    +
  • {t('spotify.imported', { count: h.imported })}
  • + {h.already_there > 0 &&
  • {t('spotify.already', { count: h.already_there })}
  • } + {h.not_found > 0 &&
  • {t('spotify.notFound', { count: h.not_found })}
  • } + {h.ignored > 0 &&
  • {t('spotify.ignored', { count: h.ignored })}
  • } +
+ ) + )} + + {report.playlists.length > 0 && ( +
+

{t('spotify.playlists')}

+
    + {report.playlists.map((p, i) => ( +
  • + {t('spotify.playlistLine', { name: p.name, matched: p.matched, tracks: p.tracks })} + {t(`spotify.status.${p.status}`)} +
  • + ))} +
+
+ )} + + {report.missing.artists.length > 0 && ( +
+

{t('spotify.missing')}

+
    + {report.missing.artists.map((a) => ( +
  • + {a.name} + {a.count} +
  • + ))} +
+ {more > 0 &&

{t('spotify.missingMore', { count: more })}

} +
+ )} +
+ ) +} diff --git a/web/src/i18n/locales/en.json b/web/src/i18n/locales/en.json index 15ea7fe..3653e38 100644 --- a/web/src/i18n/locales/en.json +++ b/web/src/i18n/locales/en.json @@ -99,6 +99,7 @@ "replayGainTrack": "Song by song", "replayGainAlbum": "Album by album", "replayGainOff": "Off", + "spotify": "Import from Spotify", "password": "Password", "currentPassword": "Current password", "newPassword": "New password", @@ -768,6 +769,10 @@ "invalid_token": "The token is empty or too long.", "mastodon_refused": "The instance refused that token. Check that it has the write:statuses permission.", "mastodon_unreachable": "That instance could not be reached. Check the address and try again.", + "no_spotify_data": "These files do not hold a Spotify streaming history or playlists. Choose the zip that Spotify sent you, or the files named Streaming_History_Audio… and Playlist….", + "unreadable_file": "One of the files is damaged or cut short. Download your export from Spotify again.", + "too_large": "These files are too large to import (the limit is 512 MB). Choose only the files named Streaming_History_Audio… and Playlist….", + "import_running": "An import of yours is still running. Wait for it to end.", "unknown": "Something went wrong. Try again.", "playlist_not_found": "This playlist does not exist.", "limit_reached": "Limit reached: a user can have 1000 playlists, and a playlist up to 10000 songs.", @@ -779,6 +784,35 @@ "suggested_not_found": "This suggested smart playlist does not exist.", "smart_playlist_empty": "These rules give no songs to keep." }, + "spotify": { + "intro": "Bring in your listening history and your playlists from Spotify, so that your mixes, Discover and smart playlists know what you listened to before. Ask Spotify for your data (Account, Privacy, “Extended streaming history” and “Account data”), then choose the zip it sends you, or the JSON files inside it.", + "privacy": "Everything is read on your server. Nothing is sent anywhere and the files are not kept. Songs are matched to your library by artist and title; the ones you do not have are only counted.", + "files": "Files from Spotify", + "chosen_one": "{{count}} file chosen", + "chosen_other": "{{count}} files chosen", + "start": "Import", + "running": "Importing. With a long history this can take a minute…", + "done": "Import done", + "noPlays": "No plays in these files.", + "imported_one": "{{count}} listen added to your history.", + "imported_other": "{{count}} listens added to your history.", + "already_one": "{{count}} was already there.", + "already_other": "{{count}} were already there.", + "notFound_one": "{{count}} play is of a song that is not in your library.", + "notFound_other": "{{count}} plays are of songs that are not in your library.", + "ignored_one": "{{count}} play skipped (a podcast, or with no readable time).", + "ignored_other": "{{count}} plays skipped (podcasts, or with no readable time).", + "playlists": "Playlists", + "playlistLine": "{{name}}: {{matched}} of {{tracks}} songs found", + "status": { + "created": "Made", + "exists": "You already have one with this name", + "empty": "None of its songs is in your library", + "limit": "You have reached the limit of playlists" + }, + "missing": "Artists you play on Spotify and have no music of here", + "missingMore": "and {{count}} more" + }, "playlists": { "empty": "You have no playlists yet. Make one below, or put a song in a new one with the playlist button next to it.", "newLabel": "Name of the new playlist", diff --git a/web/src/i18n/locales/it.json b/web/src/i18n/locales/it.json index b7162c7..3f6198c 100644 --- a/web/src/i18n/locales/it.json +++ b/web/src/i18n/locales/it.json @@ -99,6 +99,7 @@ "replayGainTrack": "Brano per brano", "replayGainAlbum": "Album per album", "replayGainOff": "Disattivato", + "spotify": "Importa da Spotify", "password": "Password", "currentPassword": "Password attuale", "newPassword": "Nuova password", @@ -768,6 +769,10 @@ "invalid_token": "Il token è vuoto o troppo lungo.", "mastodon_refused": "L'istanza ha rifiutato il token. Controlla che abbia il permesso write:statuses.", "mastodon_unreachable": "Non riesco a raggiungere quell'istanza. Controlla l'indirizzo e riprova.", + "no_spotify_data": "Questi file non contengono una cronologia di streaming né playlist di Spotify. Scegli lo zip che Spotify ti ha mandato, oppure i file che si chiamano Streaming_History_Audio… e Playlist….", + "unreadable_file": "Uno dei file è danneggiato o tagliato. Scarica di nuovo i tuoi dati da Spotify.", + "too_large": "Questi file sono troppo grandi da importare (il limite è 512 MB). Scegli solo i file che si chiamano Streaming_History_Audio… e Playlist….", + "import_running": "Un tuo import è ancora in corso. Aspetta che finisca.", "unknown": "Qualcosa è andato storto. Riprova.", "playlist_not_found": "Questa playlist non esiste.", "limit_reached": "Limite raggiunto: un utente può avere 1000 playlist e una playlist fino a 10000 brani.", @@ -779,6 +784,35 @@ "suggested_not_found": "Questa playlist smart suggerita non esiste.", "smart_playlist_empty": "Con queste regole non ci sono brani da salvare." }, + "spotify": { + "intro": "Porta qui la cronologia di ascolto e le playlist di Spotify, così i tuoi mix, Discover e le smart playlist sanno cosa ascoltavi prima. Chiedi a Spotify i tuoi dati (Account, Privacy, «Cronologia di streaming estesa» e «Dati dell'account»), poi scegli lo zip che ti manda, oppure i file JSON che contiene.", + "privacy": "Tutto viene letto sul tuo server. Non viene inviato niente a nessuno e i file non vengono conservati. I brani si abbinano alla tua libreria per artista e titolo; quelli che non hai vengono solo contati.", + "files": "File di Spotify", + "chosen_one": "{{count}} file scelto", + "chosen_other": "{{count}} file scelti", + "start": "Importa", + "running": "Importazione in corso. Con una cronologia lunga può volerci un minuto…", + "done": "Importazione finita", + "noPlays": "Nessun ascolto in questi file.", + "imported_one": "{{count}} ascolto aggiunto alla cronologia.", + "imported_other": "{{count}} ascolti aggiunti alla cronologia.", + "already_one": "{{count}} c'era già.", + "already_other": "{{count}} c'erano già.", + "notFound_one": "{{count}} ascolto è di un brano che non è nella tua libreria.", + "notFound_other": "{{count}} ascolti sono di brani che non sono nella tua libreria.", + "ignored_one": "{{count}} ascolto saltato (un podcast, o senza un orario leggibile).", + "ignored_other": "{{count}} ascolti saltati (podcast, o senza un orario leggibile).", + "playlists": "Playlist", + "playlistLine": "{{name}}: trovati {{matched}} brani su {{tracks}}", + "status": { + "created": "Creata", + "exists": "Ne hai già una con questo nome", + "empty": "Nessuno dei suoi brani è nella tua libreria", + "limit": "Hai raggiunto il limite di playlist" + }, + "missing": "Artisti che ascolti su Spotify e di cui qui non hai musica", + "missingMore": "e altri {{count}}" + }, "playlists": { "empty": "Non hai ancora nessuna playlist. Creane una qui sotto, oppure metti un brano in una nuova con il pulsante accanto al brano.", "newLabel": "Nome della nuova playlist", diff --git a/web/src/pages/SettingsPage.tsx b/web/src/pages/SettingsPage.tsx index a6211bb..8809145 100644 --- a/web/src/pages/SettingsPage.tsx +++ b/web/src/pages/SettingsPage.tsx @@ -5,6 +5,7 @@ import { logout, useSession } from '@/auth/session' import ChoiceGroup from '@/components/ChoiceGroup' import ListeningSwitch from '@/components/ListeningSwitch' import MastodonBlock from '@/components/MastodonBlock' +import SpotifyImportBlock from '@/components/SpotifyImportBlock' import Section from '@/components/Section' import { useInstall } from '@/pwa/useInstall' import { Button } from '@/components/ui/button' @@ -183,6 +184,10 @@ export default function SettingsPage() { +
+ +
+