From aa4883b9553bdefa8814b87965dac415b0f74883 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Sun, 15 Nov 2026 10:34:12 +0100 Subject: [PATCH] feat(import): an Import page, with the numbers of every import kept The report of an import used to be shown once in Settings and lost. Each import is now stored (migration 0030, spotify_imports) as the report in JSON, and GET /me/import/spotify lists the past ones, newest first. POST /me/import/spotify answers with the same shape plus id and created_at. The web app gets an Import page (menu, More tab) with the form and one card for each past import: plays read, added, already there, not in the library, the playlists it made and the artists that are missing. Settings only links to it. An import that finds nothing is not kept, and a report that cannot be stored is still returned to the person who asked. --- README.md | 2 +- cmd/sondad/main.go | 2 +- internal/api/http/api_test.go | 2 +- internal/api/http/contract_test.go | 6 + internal/api/http/handlers_spotify_import.go | 45 +++++- internal/api/http/router.go | 7 +- internal/api/http/spotify_import_test.go | 59 ++++++++ internal/api/openapi/openapi.yaml | 41 +++++- internal/spotifyimport/service.go | 112 +++++++++++---- internal/spotifyimport/service_test.go | 133 +++++++++++++++--- internal/storage/spotify_import_test.go | 63 +++++++++ internal/storage/spotify_imports.go | 59 ++++++++ internal/storage/storage_test.go | 4 +- migrations/postgres/0030_spotify_imports.sql | 10 ++ migrations/sqlite/0030_spotify_imports.sql | 11 ++ web/src/api/importSpotify.ts | 13 +- web/src/api/schema.d.ts | 54 ++++++- web/src/components/ImportReportCard.test.tsx | 59 ++++++++ web/src/components/ImportReportCard.tsx | 64 +++++++++ .../components/SpotifyImportBlock.test.tsx | 32 ++--- web/src/components/SpotifyImportBlock.tsx | 65 ++------- web/src/components/layout/Sidebar.test.tsx | 2 +- web/src/components/layout/Sidebar.tsx | 1 + web/src/i18n/locales/en.json | 17 ++- web/src/i18n/locales/it.json | 17 ++- web/src/pages/ImportPage.test.tsx | 81 +++++++++++ web/src/pages/ImportPage.tsx | 38 +++++ web/src/pages/LibraryPage.test.tsx | 3 +- web/src/pages/MorePage.tsx | 5 +- web/src/pages/SettingsPage.test.tsx | 7 + web/src/pages/SettingsPage.tsx | 7 +- web/src/router.tsx | 1 + 32 files changed, 868 insertions(+), 154 deletions(-) create mode 100644 internal/storage/spotify_imports.go create mode 100644 migrations/postgres/0030_spotify_imports.sql create mode 100644 migrations/sqlite/0030_spotify_imports.sql create mode 100644 web/src/components/ImportReportCard.test.tsx create mode 100644 web/src/components/ImportReportCard.tsx create mode 100644 web/src/pages/ImportPage.test.tsx create mode 100644 web/src/pages/ImportPage.tsx diff --git a/README.md b/README.md index cfe0dce..d9020bc 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ It is a single executable, one database (SQLite by default, PostgreSQL if you pr - **Playlists.** `POST /playlists` makes a playlist (a name and an optional description) and `GET /playlists` lists yours, the one changed last first. `POST /playlists/{id}/tracks` adds 1 to 500 tracks at the end or at a `position`, `GET /playlists/{id}/tracks` lists them in order, `PATCH /playlists/{id}/tracks/{item}` moves one and `DELETE` takes one out; `PATCH /playlists/{id}` renames and `DELETE /playlists/{id}` deletes (the tracks stay in the library). A track can be in a playlist more than once, which is why each place has its own item id. A playlist is yours alone: someone else's answers `playlist_not_found`, as one that does not exist. A track whose files were deleted or moved away stays in the playlist and shows `available: false`; it plays again when the files come back and a scan finds them. Each playlist says which albums stand for it (`album_ids`: up to 4 albums that have a cover, in the order they first appear in it, each once), for a picture made of their covers; nothing is fetched for it. A user can have 1000 playlists of up to 10000 tracks. Nothing is shared and nothing leaves your server. - **Wishlist and "not interested".** `PUT /wishlist/{artist}` puts an artist on your wishlist (`GET /wishlist` lists it, `DELETE` takes it off), and `POST /discover/{artist}/dismiss` hides an artist from your Discover for good (`DELETE` brings it back, `GET /discover/dismissed` lists them). An artist is wished or dismissed, never both, and both lists are yours alone. Discover marks the wished artists it shows (`wishlisted`) and never suggests a dismissed one. The wishlist is a list for you and sends nothing anywhere; Sonda downloads nothing, but an admin can send an artist that Discover showed to their own Lidarr (see [External providers](#external-providers)), which works for an artist that has a MusicBrainz id and that Sonda knows of from a list or from your files. An artist that only Last.fm named is kept while you have it marked (your lists still show it, but other users cannot see that it exists), and goes with the rest of Last.fm's data if an admin removes Last.fm's data. Removing MusicBrainz's data does not touch it. - **Your own settings.** `GET` and `PATCH /me/settings` hold choices that belong to you, not to the server. Everyone starts private: `discover_from_listening` is off, so this setting never sends your listening to a provider. When you turn it on, Discover may start from the artists you listen to most, and their names are sent to Last.fm (only if an admin switched it on). What Last.fm answers is kept in the shared database like any other lookup, so another local user can see, from `retrieved_at` of `GET /artists/{id}/similar`, that someone looked up an artist, but not who. -- **Importing from Spotify.** `POST /me/import/spotify` brings your listening history and your playlists from a Spotify data export into your own account. Ask Spotify for your data (Account, Privacy, "Extended streaming history" and "Account data") and send what it gives, as a multipart form with a `files` field for each file: the zip itself, or the JSON files in it (`Streaming_History_Audio_*.json`, the older `StreamingHistory_music_*.json` and `Playlist*.json`); a request may be up to 512 MB, and the rest of the export (the account, the searches, the library) is never opened. It all happens on your server and nothing is sent anywhere; the files are read from the request and not kept, and of each play only the song, the artist, the album, the time and how it ended are read, never the IP addresses, the country or the platform in the file. A play is matched to a song of your library by artist and title (case, accents and punctuation do not matter, a `Remastered` note on either side does not, any credited artist will do, and a play of a compilation is matched by its album when Spotify says `Various Artists`). A match is recorded as a listen at the time of the play: `complete` when it ran to its end (or nine tenths were played), `skip` when Spotify says it was skipped or it lasted less than a listen, `play` otherwise; nothing is posted to Mastodon. Importing the same files again records nothing twice, and the same play in the extended and in the older history is one. A playlist becomes a playlist of yours with the songs that were found, in order; one whose name you already have is left alone, and one with none of its songs in your library is not made. The report counts what was added, what was already there and what is not in your library (and lists those artists, most played first), so you can see what is missing. One import of yours runs at a time. Settings has an Import from Spotify section. +- **Importing from Spotify.** `POST /me/import/spotify` brings your listening history and your playlists from a Spotify data export into your own account. Ask Spotify for your data (Account, Privacy, "Extended streaming history" and "Account data") and send what it gives, as a multipart form with a `files` field for each file: the zip itself, or the JSON files in it (`Streaming_History_Audio_*.json`, the older `StreamingHistory_music_*.json` and `Playlist*.json`); a request may be up to 512 MB, and the rest of the export (the account, the searches, the library) is never opened. It all happens on your server and nothing is sent anywhere; the files are read from the request and not kept, and of each play only the song, the artist, the album, the time and how it ended are read, never the IP addresses, the country or the platform in the file. A play is matched to a song of your library by artist and title (case, accents and punctuation do not matter, a `Remastered` note on either side does not, any credited artist will do, and a play of a compilation is matched by its album when Spotify says `Various Artists`). A match is recorded as a listen at the time of the play: `complete` when it ran to its end (or nine tenths were played), `skip` when Spotify says it was skipped or it lasted less than a listen, `play` otherwise; nothing is posted to Mastodon. Importing the same files again records nothing twice, and the same play in the extended and in the older history is one. A playlist becomes a playlist of yours with the songs that were found, in order; one whose name you already have is left alone, and one with none of its songs in your library is not made. The report counts what was added, what was already there and what is not in your library (and lists those artists, most played first), so you can see what is missing. One import of yours runs at a time. The report is kept: `GET /me/import/spotify` lists your past imports, newest first, with the numbers of each (never the files). The web app has an Import page for the form and for those numbers. - **Sharing on Mastodon.** `PUT /me/mastodon` connects your own Mastodon account (the address of your instance and an access token that can only write posts); `GET` shows which account is connected and `DELETE` forgets it and the token. Nothing is connected by default. Once it is, each track of this server that counts as a listen is posted to your account as an unlisted status with the title, artists, album and `#NowPlaying`, and no link: at most one post a minute, never music of a friend. The token stays on the server and is never sent back. If the instance refuses the token on a post (it was revoked or expired), `GET /me/mastodon` says `problem: token_refused`, Settings shows it, and nothing is posted until you connect again with a new token. The address of the instance must be a domain name on https, and the server never connects to a private, loopback or link-local address for it, never follows a redirect and ignores proxy settings. - **Tags.** `GET /artists/{id}/tags` shows what Last.fm and MusicBrainz say kind of music an artist plays, each with its own weights, and the genres of the artist's own files, read offline. See [External providers](#external-providers). - **Similar artists.** With Last.fm switched on, `GET /artists/{id}/similar` lists artists similar to an artist, and says which of them you already have. See [External providers](#external-providers). Without it, Sonda makes no outbound network requests at all. diff --git a/cmd/sondad/main.go b/cmd/sondad/main.go index 2cec801..3ba32c8 100644 --- a/cmd/sondad/main.go +++ b/cmd/sondad/main.go @@ -312,7 +312,7 @@ func run() error { Discover: discover.NewService(tasteService, discoveries, tags, tasteRepo, userSettings, feedbacks, weights), Feedback: feedbacks, Playlists: playlistSvc, - SpotifyImport: spotifyimport.NewService(storage.NewCatalogRepo(db), storage.NewListeningRepo(db), playlistSvc, storage.SearchKey, cfg.Listening.MinListen()), + SpotifyImport: spotifyimport.NewService(storage.NewCatalogRepo(db), storage.NewListeningRepo(db), playlistSvc, storage.NewSpotifyImportRepo(db), storage.SearchKey, cfg.Listening.MinListen()), Mixes: mixes.NewService(storage.NewMixRepo(db), tasteService, cfg.Listening.MinListen()), Smart: smart.NewService(storage.NewSmartRepo(db), tasteService, cfg.Listening.MinListen()), Naming: namingService, diff --git a/internal/api/http/api_test.go b/internal/api/http/api_test.go index af463e8..4109aff 100644 --- a/internal/api/http/api_test.go +++ b/internal/api/http/api_test.go @@ -121,7 +121,7 @@ func newTestEnvDB(t *testing.T, mutate func(*Deps, *bun.DB)) (http.Handler, *bun feedbackSvc := feedback.NewService(storage.NewFeedbackRepo(db)) deps.Feedback = feedbackSvc deps.Playlists = playlists.NewService(storage.NewPlaylistRepo(db)) - deps.SpotifyImport = spotifyimport.NewService(storage.NewCatalogRepo(db), storage.NewListeningRepo(db), deps.Playlists, storage.SearchKey, 20*time.Second) + deps.SpotifyImport = spotifyimport.NewService(storage.NewCatalogRepo(db), storage.NewListeningRepo(db), deps.Playlists, storage.NewSpotifyImportRepo(db), storage.SearchKey, 20*time.Second) deps.Mixes = mixes.NewService(storage.NewMixRepo(db), deps.Taste.(mixes.Taster), 20*time.Second) deps.Smart = smart.NewService(storage.NewSmartRepo(db), deps.Taste.(smart.Taster), 20*time.Second) namingSvc := naming.NewService(prov, noClaude{}, storage.NewNamingRepo(db), nil) diff --git a/internal/api/http/contract_test.go b/internal/api/http/contract_test.go index 5462873..1b464e7 100644 --- a/internal/api/http/contract_test.go +++ b/internal/api/http/contract_test.go @@ -870,6 +870,12 @@ func TestResponsesMatchTheSpec(t *testing.T) { if h, _ := imported["history"].(map[string]any); h["entries"] != 1.0 || h["not_found"] != 1.0 { c.t.Errorf("history = %v", h) } + past := c.expect(c.call("GET", "/me/import/spotify", token, nil), 200, "the past imports") + if items, _ := past["items"].([]any); past["total"] != 1.0 || len(items) != 1 || items[0].(map[string]any)["id"] != imported["id"] { + c.t.Errorf("past imports = %v, want the one just made", past) + } + c.expect(c.call("GET", "/me/import/spotify?limit=-1", token, nil), 400, "a negative limit", "invalid_input") + c.expect(c.call("GET", "/me/import/spotify", "", nil), 401, "past imports without a token", "unauthorized") c.expect(c.upload("/me/import/spotify", token), 400, "an import with no files", "invalid_input") c.expect(c.upload("/me/import/spotify", token, `{"hello":"world"}`), 400, "files that are not Spotify's", "no_spotify_data") c.expect(c.upload("/me/import/spotify", token, export[:len(export)-20]), 400, "a file cut short", "unreadable_file") diff --git a/internal/api/http/handlers_spotify_import.go b/internal/api/http/handlers_spotify_import.go index 49e72b0..30059cc 100644 --- a/internal/api/http/handlers_spotify_import.go +++ b/internal/api/http/handlers_spotify_import.go @@ -3,6 +3,7 @@ package httpapi import ( "errors" "net/http" + "time" "github.com/gin-gonic/gin" @@ -44,6 +45,8 @@ type spotifyMissingJSON struct { } type spotifyReportJSON struct { + ID int64 `json:"id"` + CreatedAt time.Time `json:"created_at"` History spotifyHistoryJSON `json:"history"` Playlists []spotifyPlaylistJSON `json:"playlists"` Missing struct { @@ -52,8 +55,10 @@ type spotifyReportJSON struct { } `json:"missing"` } -func toSpotifyReport(r spotifyimport.Report) spotifyReportJSON { +func toSpotifyReport(run spotifyimport.Run) spotifyReportJSON { + r := run.Report out := spotifyReportJSON{ + ID: run.ID, CreatedAt: run.At.UTC(), History: spotifyHistoryJSON{ Entries: r.History.Entries, Matched: r.History.Matched, Imported: r.History.Imported, AlreadyThere: r.History.AlreadyThere, NotFound: r.History.NotFound, Ignored: r.History.Ignored, @@ -99,10 +104,14 @@ func (h *spotifyImportHandlers) importExport(c *gin.Context) { files = append(files, spotifyimport.File{R: f, Size: fh.Size}) } - report, err := h.svc.Import(c.Request.Context(), currentUserID(c), files) + run, err := h.svc.Import(c.Request.Context(), currentUserID(c), files) switch { case err == nil: - c.JSON(http.StatusOK, toSpotifyReport(report)) + c.JSON(http.StatusOK, toSpotifyReport(run)) + case errors.Is(err, spotifyimport.ErrNotKept): + // The listens are recorded, so say so; the report just will not be on the page of past imports. + _ = c.Error(err) + c.JSON(http.StatusOK, toSpotifyReport(run)) case errors.Is(err, spotifyimport.ErrNothingFound): writeError(c, http.StatusBadRequest, "no_spotify_data", "these files hold no streaming history and no playlists of Spotify") case errors.Is(err, spotifyimport.ErrUnreadable): @@ -116,3 +125,33 @@ func (h *spotifyImportHandlers) importExport(c *gin.Context) { writeError(c, http.StatusInternalServerError, "internal", "internal error") } } + +type spotifyImportPageJSON struct { + Items []spotifyReportJSON `json:"items"` + Total int `json:"total"` + Limit int `json:"limit"` + Offset int `json:"offset"` +} + +// list gives the imports of the signed-in person that were kept, newest first. +func (h *spotifyImportHandlers) list(c *gin.Context) { + page, ok := pageOf(c) + if !ok { + return + } + got, err := h.svc.History(c.Request.Context(), currentUserID(c), page.Limit, page.Offset) + switch { + case errors.Is(err, spotifyimport.ErrInvalidPage): + writeError(c, http.StatusBadRequest, "invalid_input", err.Error()) + return + case err != nil: + _ = c.Error(err) + writeError(c, http.StatusInternalServerError, "internal", "internal error") + return + } + out := spotifyImportPageJSON{Items: make([]spotifyReportJSON, 0, len(got.Items)), Total: got.Total, Limit: got.Limit, Offset: got.Offset} + for _, run := range got.Items { + out.Items = append(out.Items, toSpotifyReport(run)) + } + c.JSON(http.StatusOK, out) +} diff --git a/internal/api/http/router.go b/internal/api/http/router.go index 2a128f0..e460e14 100644 --- a/internal/api/http/router.go +++ b/internal/api/http/router.go @@ -106,7 +106,8 @@ type playlistService interface { // spotifyImportService is what the handlers need from spotifyimport.Service. type spotifyImportService interface { - Import(ctx context.Context, userID int64, files []spotifyimport.File) (spotifyimport.Report, error) + Import(ctx context.Context, userID int64, files []spotifyimport.File) (spotifyimport.Run, error) + History(ctx context.Context, userID int64, limit, offset int) (spotifyimport.RunPage, error) } // mixService is what the handlers need from mixes.Service. @@ -361,7 +362,9 @@ func NewRouter(d Deps) *gin.Engine { protected.GET("/me/settings", us.get) protected.PATCH("/me/settings", us.update) ma := &mastodonHandlers{svc: d.Mastodon} - protected.POST("/me/import/spotify", (&spotifyImportHandlers{svc: d.SpotifyImport}).importExport) + spotify := &spotifyImportHandlers{svc: d.SpotifyImport} + protected.POST("/me/import/spotify", spotify.importExport) + protected.GET("/me/import/spotify", spotify.list) protected.GET("/me/mastodon", ma.get) protected.PUT("/me/mastodon", ma.connect) protected.DELETE("/me/mastodon", ma.disconnect) diff --git a/internal/api/http/spotify_import_test.go b/internal/api/http/spotify_import_test.go index 84bf008..5cd4d49 100644 --- a/internal/api/http/spotify_import_test.go +++ b/internal/api/http/spotify_import_test.go @@ -78,6 +78,9 @@ func TestImportingAHistoryRecordsTheListensAndTheReportSaysWhatHappened(t *testi if r.Code != http.StatusOK { t.Fatalf("import: %d %v", r.Code, r.Body) } + if r.Body["id"] != 1.0 || r.Body["created_at"] == nil { + t.Errorf("id and time = %v %v, want the import kept as 1", r.Body["id"], r.Body["created_at"]) + } hist, _ := r.Body["history"].(map[string]any) sameKeys(t, "history", hist, "entries", "matched", "imported", "already_there", "not_found", "ignored") if hist["entries"] != 2.0 || hist["matched"] != 1.0 || hist["imported"] != 1.0 || hist["not_found"] != 1.0 { @@ -189,3 +192,59 @@ func TestAnUploadLargerThanOtherRequestsIsAccepted(t *testing.T) { t.Errorf("%d %v", r.Code, r.Body) } } + +func TestPastImportsAreListedNewestFirstAndAreOnlyMine(t *testing.T) { + h, db := newTestEnv(t, func(*Deps) {}) + seedSong(t, db, "a.flac", "Karma Police", "OK Computer", "Radiohead") + tok := registerAndLogin(t, h) + upload(t, h, tok, spotifyHistory) + upload(t, h, tok, spotifyHistory) + + r := do(t, h, "GET", importPath, tok, nil) + if r.Code != http.StatusOK || r.Body["total"] != 2.0 || r.Body["limit"] != 20.0 || r.Body["offset"] != 0.0 { + t.Fatalf("list: %d %v", r.Code, r.Body) + } + items, _ := r.Body["items"].([]any) + if len(items) != 2 { + t.Fatalf("%d imports, want 2", len(items)) + } + newest, oldest := items[0].(map[string]any), items[1].(map[string]any) + sameKeys(t, "import", newest, "id", "created_at", "history", "playlists", "missing") + if newest["id"] != 2.0 || oldest["id"] != 1.0 { + t.Errorf("order = %v, %v, want 2 then 1", newest["id"], oldest["id"]) + } + if h, _ := oldest["history"].(map[string]any); h["imported"] != 1.0 { + t.Errorf("the first import = %v, want one listen recorded", h) + } + if h, _ := newest["history"].(map[string]any); h["already_there"] != 1.0 || h["imported"] != 0.0 { + t.Errorf("the second import = %v, want the play already there", h) + } + if m, _ := newest["missing"].(map[string]any); m["total"] != 1.0 { + t.Errorf("missing = %v, want the one artist", m) + } + + if page := do(t, h, "GET", importPath+"?limit=1&offset=1", tok, nil); page.Code != http.StatusOK || len(page.Body["items"].([]any)) != 1 || page.Body["items"].([]any)[0].(map[string]any)["id"] != 1.0 { + t.Errorf("second page: %d %v", page.Code, page.Body) + } + if r := do(t, h, "GET", importPath+"?offset=-3", tok, nil); r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" { + t.Errorf("a negative offset: %d %v", r.Code, r.Body) + } + + // Somebody else sees none of it. + _, other := addUser(t, h, tok, "other", false) + if r := do(t, h, "GET", importPath, other, nil); r.Code != http.StatusOK || r.Body["total"] != 0.0 || len(r.Body["items"].([]any)) != 0 { + t.Errorf("another person's list: %d %v, want empty", r.Code, r.Body) + } +} + +func TestAnImportThatFindsNothingIsNotListed(t *testing.T) { + h := newTestAPI(t) + tok := registerAndLogin(t, h) + upload(t, h, tok, `{"hello":"world"}`) + if r := do(t, h, "GET", importPath, tok, nil); r.Body["total"] != 0.0 { + t.Errorf("list = %v, want nothing", r.Body) + } + if r := do(t, h, "GET", importPath, "", nil); r.Code != http.StatusUnauthorized { + t.Errorf("without a token: %d, want 401", r.Code) + } +} diff --git a/internal/api/openapi/openapi.yaml b/internal/api/openapi/openapi.yaml index 71532b4..b1c64b8 100644 --- a/internal/api/openapi/openapi.yaml +++ b/internal/api/openapi/openapi.yaml @@ -2163,6 +2163,27 @@ paths: '401': {$ref: '#/components/responses/Unauthorized'} /me/import/spotify: + get: + tags: [Taste] + operationId: listMySpotifyImports + summary: Your past imports of Spotify data + description: | + What each import of yours did, newest first: how many plays were recorded, how many were already there, + how many are not in your library, the playlists it made and the artists you are missing. The files + themselves are never kept, only these numbers. Only yours. An import is listed once it is done; one that + found nothing is not. + security: [{bearerAuth: []}] + parameters: + - {$ref: '#/components/parameters/Limit'} + - {$ref: '#/components/parameters/Offset'} + responses: + '200': + description: A page of imports. + content: + application/json: + schema: {$ref: '#/components/schemas/SpotifyImportPage'} + '400': {$ref: '#/components/responses/InvalidInput'} + '401': {$ref: '#/components/responses/Unauthorized'} post: tags: [Taste] operationId: importMySpotify @@ -2205,7 +2226,9 @@ paths: description: The zip of the export, or its JSON files. responses: '200': - description: What the import did. + description: | + What the import did. It is also kept (`GET /me/import/spotify`); if keeping it failed the report is still + sent, with `id` 0. content: application/json: schema: {$ref: '#/components/schemas/SpotifyImportReport'} @@ -5466,11 +5489,25 @@ components: shared: another user can see from `retrieved_at` in `GET /artists/{id}/similar` that somebody looked an artist up, but not who. + SpotifyImportPage: + type: object + additionalProperties: false + required: [items, total, limit, offset] + properties: + items: + type: array + items: {$ref: '#/components/schemas/SpotifyImportReport'} + total: {type: integer, description: "Size of the whole result, not of this page."} + limit: {type: integer} + offset: {type: integer} + SpotifyImportReport: type: object additionalProperties: false - required: [history, playlists, missing] + required: [id, created_at, history, playlists, missing] properties: + id: {type: integer, description: 'The import. 0 when its report could not be kept.'} + created_at: {type: string, format: date-time, description: When the import was done.} history: type: object additionalProperties: false diff --git a/internal/spotifyimport/service.go b/internal/spotifyimport/service.go index 0b92055..1af91d1 100644 --- a/internal/spotifyimport/service.go +++ b/internal/spotifyimport/service.go @@ -23,12 +23,20 @@ var ( ErrNothingFound = errors.New("no Spotify history or playlists in these files") // ErrBusy means this person's import is still running. ErrBusy = errors.New("an import is already running") + // ErrNotKept means the import was done but its report could not be stored; the Run that comes with it is complete. + ErrNotKept = errors.New("the report of the import could not be kept") + // ErrInvalidPage means a negative limit or offset was asked for. + ErrInvalidPage = errors.New("limit and offset cannot be negative") ) const ( // MaxMissing is how many missing artists a report lists (the ones played most); MissingTotal says how many there are. MaxMissing = 50 + // DefaultPage and MaxPage are how many past imports a page holds when nothing is asked, and at the most. + DefaultPage = 20 + MaxPage = 100 + // listenBatch is how many listens are recorded in one go. listenBatch = 2000 @@ -69,36 +77,57 @@ type File struct { // History says what became of the plays of the history files. type History struct { - Entries int // plays of songs that were read - Matched int // of those, the ones that are a song of the library - Imported int // recorded now - AlreadyThere int // matched, and already recorded by an earlier import (or the same play in two files) - NotFound int // not in the library - Ignored int // podcast episodes, and plays with no time that can be read + Entries int `json:"entries"` // plays of songs that were read + Matched int `json:"matched"` // of those, the ones that are a song of the library + Imported int `json:"imported"` // recorded now + AlreadyThere int `json:"already_there"` // matched, and already recorded by an earlier import (or the same play in two files) + NotFound int `json:"not_found"` // not in the library + Ignored int `json:"ignored"` // podcast episodes, and plays with no time that can be read } // PlaylistResult says what became of one playlist. Status is `created`, `exists` (the person already has one with // that name, which is left alone), `empty` (none of its songs is in the library) or `limit` (the person has as many // playlists as are allowed). type PlaylistResult struct { - Name string - Tracks int // everything that is in it - Matched int // the songs of the library - Status string + Name string `json:"name"` + Tracks int `json:"tracks"` // everything that is in it + Matched int `json:"matched"` // the songs of the library + Status string `json:"status"` } // Missing is an artist that has plays or playlist songs and none of them in the library. type Missing struct { - Artist string - Count int + Artist string `json:"artist"` + Count int `json:"count"` } -// Report is what an import did. It is made for the person who asked and kept nowhere. +// Report is what an import did. It is kept as JSON (the tags are the stored shape), so a name here is not changed +// lightly. type Report struct { - History History - Playlists []PlaylistResult - Missing []Missing // the artists with the most songs that were not found, at most MaxMissing - MissingTotal int // how many artists have songs that were not found + History History `json:"history"` + Playlists []PlaylistResult `json:"playlists"` + Missing []Missing `json:"missing"` // the artists with the most songs that were not found, at most MaxMissing + MissingTotal int `json:"missing_total"` // how many artists have songs that were not found +} + +// Run is a report that was kept: the import with its number and the time it was done. +type Run struct { + ID int64 // 0 when the report could not be kept + At time.Time + Report Report +} + +// RunPage is a page of the imports of a person, newest first. +type RunPage struct { + Items []Run + Total int + Limit, Offset int +} + +// Runs keeps the reports of the imports. +type Runs interface { + Save(ctx context.Context, userID int64, at time.Time, r Report) (id int64, err error) + List(ctx context.Context, userID int64, limit, offset int) ([]Run, int, error) } // Service imports the data of a Spotify export for one person at a time. @@ -106,6 +135,7 @@ type Service struct { catalog Catalog listens Listens lists Playlists + runs Runs key func(string) string minListen time.Duration running sync.Map // user id -> struct{} @@ -113,22 +143,40 @@ type Service struct { // NewService builds a Service. key is how names are made comparable (the key of the library search); minListen is how // long a play must last to count as a listen, so that a play shorter than that is recorded as a skip. -func NewService(catalog Catalog, listens Listens, lists Playlists, key func(string) string, minListen time.Duration) *Service { - return &Service{catalog: catalog, listens: listens, lists: lists, key: key, minListen: minListen} +func NewService(catalog Catalog, listens Listens, lists Playlists, runs Runs, key func(string) string, minListen time.Duration) *Service { + return &Service{catalog: catalog, listens: listens, lists: lists, runs: runs, key: key, minListen: minListen} +} + +// History lists the imports of a person that were kept, newest first. A limit of 0 is DefaultPage, and one above +// MaxPage is lowered to it. +func (s *Service) History(ctx context.Context, userID int64, limit, offset int) (RunPage, error) { + if limit < 0 || offset < 0 { + return RunPage{}, ErrInvalidPage + } + if limit == 0 { + limit = DefaultPage + } + limit = min(limit, MaxPage) + items, total, err := s.runs.List(ctx, userID, limit, offset) + if err != nil { + return RunPage{}, fmt.Errorf("list the imports: %w", err) + } + return RunPage{Items: items, Total: total, Limit: limit, Offset: offset}, nil } // Import reads the files, records the listens that match the library and makes the playlists. A person can run one // import at a time (ErrBusy). It is safe to import the same files again: nothing is recorded twice. What was -// recorded before an error stays recorded. -func (s *Service) Import(ctx context.Context, userID int64, files []File) (Report, error) { +// recorded before an error stays recorded. The report is kept for the person to read again; when that fails the +// Run is still returned, with ID 0, together with ErrNotKept. +func (s *Service) Import(ctx context.Context, userID int64, files []File) (Run, error) { if _, busy := s.running.LoadOrStore(userID, struct{}{}); busy { - return Report{}, ErrBusy + return Run{}, ErrBusy } defer s.running.Delete(userID) songs, err := s.catalog.Candidates(ctx) if err != nil { - return Report{}, fmt.Errorf("read the library: %w", err) + return Run{}, fmt.Errorf("read the library: %w", err) } run := &run{svc: s, ctx: ctx, userID: userID, index: NewIndex(s.key, songs), memo: map[string]int64{}, missing: map[string]*Missing{}} @@ -137,23 +185,29 @@ func (s *Service) Import(ctx context.Context, userID int64, files []File) (Repor for _, f := range files { ok, err := reader.ReadFile(f.R, f.Size) if err != nil { - return Report{}, err + return Run{}, err } found = found || ok if run.err != nil { - return Report{}, run.err + return Run{}, run.err } } if !found { - return Report{}, ErrNothingFound + return Run{}, ErrNothingFound } if err := run.flush(); err != nil { - return Report{}, err + return Run{}, err } if err := run.makePlaylists(); err != nil { - return Report{}, err + return Run{}, err } - return run.finish(), nil + rep := run.finish() + at := time.Now() + id, err := s.runs.Save(ctx, userID, at, rep) + if err != nil { + return Run{At: at, Report: rep}, fmt.Errorf("%w: %v", ErrNotKept, err) + } + return Run{ID: id, At: at, Report: rep}, nil } // run is the state of one import. diff --git a/internal/spotifyimport/service_test.go b/internal/spotifyimport/service_test.go index 51aa8c5..51c1222 100644 --- a/internal/spotifyimport/service_test.go +++ b/internal/spotifyimport/service_test.go @@ -101,15 +101,47 @@ var songs = []Candidate{ {TrackID: 3, Title: "Hyperballad", Album: "Post", Artists: []string{"Björk"}, DurationMs: 321_000}, } +// fakeRuns keeps the reports in memory, newest first when listed. +type fakeRuns struct { + saved []Run + fail error +} + +func (f *fakeRuns) Save(_ context.Context, _ int64, at time.Time, r Report) (int64, error) { + if f.fail != nil { + return 0, f.fail + } + id := int64(len(f.saved) + 1) + f.saved = append(f.saved, Run{ID: id, At: at, Report: r}) + return id, nil +} + +func (f *fakeRuns) List(_ context.Context, _ int64, limit, offset int) ([]Run, int, error) { + var all []Run + for i := len(f.saved) - 1; i >= 0; i-- { + all = append(all, f.saved[i]) + } + end := min(offset+limit, len(all)) + start := min(offset, end) + return all[start:end], len(all), nil +} + type rig struct { - svc *Service - li *fakeListens - pl *fakePlaylists + svc *Service + li *fakeListens + pl *fakePlaylists + runs *fakeRuns } func newRig(cat *fakeCatalog) rig { - li, pl := &fakeListens{}, &fakePlaylists{} - return rig{NewService(cat, li, pl, key, 20*time.Second), li, pl} + li, pl, runs := &fakeListens{}, &fakePlaylists{}, &fakeRuns{} + return rig{NewService(cat, li, pl, runs, key, 20*time.Second), li, pl, runs} +} + +// report runs an import and gives the report of it. +func (r rig) report(userID int64, files []File) (Report, error) { + run, err := r.svc.Import(context.Background(), userID, files) + return run.Report, err } func upload(data string) File { return File{R: bytes.NewReader([]byte(data)), Size: int64(len(data))} } @@ -123,7 +155,7 @@ func history(plays ...string) string { return "[" + strings.Join(plays, ",") + " func TestImportRecordsTheListensThatMatchTheLibrary(t *testing.T) { r := newRig(&fakeCatalog{songs: songs}) - rep, err := r.svc.Import(context.Background(), 7, []File{upload(history( + rep, err := r.report(7, []File{upload(history( play("2023-05-01T10:00:00Z", "Radiohead", "Karma Police", "OK Computer", 260_000, `,"reason_end":"trackdone"`), play("2023-05-01T10:05:00Z", "Radiohead", "Airbag", "OK Computer", 4_000, `,"reason_end":"fwdbtn","skipped":true`), play("2023-05-01T10:06:00Z", "Radiohead", "Creep", "Pablo Honey", 200_000, ``), @@ -155,7 +187,7 @@ func TestImportRecordsTheListensThatMatchTheLibrary(t *testing.T) { func TestTheKindOfAPlayComesFromHowItEndedOrHowMuchWasHeard(t *testing.T) { r := newRig(&fakeCatalog{songs: songs}) - _, err := r.svc.Import(context.Background(), 1, []File{upload(history( + _, err := r.report(1, []File{upload(history( play("2023-01-01T10:00:00Z", "Radiohead", "Karma Police", "", 260_000, `,"reason_end":"trackdone"`), // ran to the end play("2023-01-01T11:00:00Z", "Radiohead", "Karma Police", "", 250_000, `,"reason_end":"endplay"`), // 96 per cent, then stopped play("2023-01-01T12:00:00Z", "Radiohead", "Karma Police", "", 90_000, `,"reason_end":"endplay"`), // a good part, then stopped @@ -184,10 +216,10 @@ func TestImportingTheSameFilesAgainAddsNothing(t *testing.T) { play("2023-05-01T10:00:00Z", "Radiohead", "Karma Police", "OK Computer", 260_000, ``), play("2023-05-01T10:10:00Z", "Björk", "Hyperballad", "Post", 321_000, ``), ) - if _, err := r.svc.Import(context.Background(), 1, []File{upload(data)}); err != nil { + if _, err := r.report(1, []File{upload(data)}); err != nil { t.Fatal(err) } - rep, err := r.svc.Import(context.Background(), 1, []File{upload(data)}) + rep, err := r.report(1, []File{upload(data)}) if err != nil { t.Fatal(err) } @@ -200,7 +232,7 @@ func TestTheSamePlayInTheTwoShapesOfFileIsCountedOnce(t *testing.T) { r := newRig(&fakeCatalog{songs: songs}) extended := history(play("2022-01-02T08:30:41Z", "Björk", "Hyperballad", "Post", 321_000, `,"reason_end":"trackdone"`)) older := `[{"endTime":"2022-01-02 08:30","artistName":"Björk","trackName":"Hyperballad","msPlayed":321000}]` - rep, err := r.svc.Import(context.Background(), 1, []File{upload(extended), upload(older)}) + rep, err := r.report(1, []File{upload(extended), upload(older)}) if err != nil { t.Fatal(err) } @@ -212,7 +244,7 @@ func TestTheSamePlayInTheTwoShapesOfFileIsCountedOnce(t *testing.T) { func TestPlaylistsBecomePlaylistsWithTheSongsThatWereFound(t *testing.T) { r := newRig(&fakeCatalog{songs: songs}) r.pl.existing = []string{"Mine already"} - rep, err := r.svc.Import(context.Background(), 1, []File{upload(`{"playlists":[ + rep, err := r.report(1, []File{upload(`{"playlists":[ {"name":"Road","items":[ {"track":{"trackName":"Hyperballad","artistName":"Björk","albumName":"Post"}}, {"track":{"trackName":"Creep","artistName":"Radiohead","albumName":"Pablo Honey"}}, @@ -257,7 +289,7 @@ func TestALongPlaylistIsAddedInPieces(t *testing.T) { for i := 0; i < 1234; i++ { items = append(items, `{"track":{"trackName":"Airbag","artistName":"Radiohead"}}`) } - rep, err := r.svc.Import(context.Background(), 1, []File{upload(`{"playlists":[{"name":"Long","items":[` + strings.Join(items, ",") + `]}]}`)}) + rep, err := r.report(1, []File{upload(`{"playlists":[{"name":"Long","items":[` + strings.Join(items, ",") + `]}]}`)}) if err != nil { t.Fatal(err) } @@ -270,7 +302,7 @@ func TestWhenThePersonHasAsManyPlaylistsAsAllowedTheRestAreSaidSo(t *testing.T) r := newRig(&fakeCatalog{songs: songs}) r.pl.max = 1 item := `{"track":{"trackName":"Airbag","artistName":"Radiohead"}}` - rep, err := r.svc.Import(context.Background(), 1, []File{upload(`{"playlists":[{"name":"A","items":[` + item + `]},{"name":"B","items":[` + item + `]},{"name":"C","items":[` + item + `]}]}`)}) + rep, err := r.report(1, []File{upload(`{"playlists":[{"name":"A","items":[` + item + `]},{"name":"B","items":[` + item + `]},{"name":"C","items":[` + item + `]}]}`)}) if err != nil { t.Fatal(err) } @@ -290,7 +322,7 @@ func TestTheArtistsThatAreMissingAreListedMostPlayedFirst(t *testing.T) { plays = append(plays, play(fmt.Sprintf("2023-01-02T10:0%d:00Z", i), "oasis", "Wonderwall", "", 120_000, ``)) plays = append(plays, play(fmt.Sprintf("2023-01-03T10:0%d:00Z", i), "Oasis", "Supersonic", "", 120_000, ``)) } - rep, err := r.svc.Import(context.Background(), 1, []File{upload(history(plays...))}) + rep, err := r.report(1, []File{upload(history(plays...))}) if err != nil { t.Fatal(err) } @@ -305,7 +337,7 @@ func TestOnlyTheMostPlayedMissingArtistsAreListed(t *testing.T) { for i := 0; i < MaxMissing+30; i++ { plays = append(plays, play("2023-01-01T10:00:00Z", fmt.Sprintf("Artist %03d", i), "Song", "", 120_000, ``)) } - rep, err := r.svc.Import(context.Background(), 1, []File{upload(history(plays...))}) + rep, err := r.report(1, []File{upload(history(plays...))}) if err != nil { t.Fatal(err) } @@ -316,7 +348,7 @@ func TestOnlyTheMostPlayedMissingArtistsAreListed(t *testing.T) { func TestFilesWithNothingOfSpotifyAreRefused(t *testing.T) { r := newRig(&fakeCatalog{songs: songs}) - _, err := r.svc.Import(context.Background(), 1, []File{upload(`{"hello":"world"}`), upload("words")}) + _, err := r.report(1, []File{upload(`{"hello":"world"}`), upload("words")}) if !errors.Is(err, ErrNothingFound) { t.Errorf("err = %v, want ErrNothingFound", err) } @@ -330,11 +362,11 @@ func TestAnImportThatIsRunningBlocksAnotherOfTheSamePerson(t *testing.T) { r := newRig(cat) done := make(chan error, 1) go func() { - _, err := r.svc.Import(context.Background(), 5, []File{upload(history(play("2023-01-01T10:00:00Z", "Radiohead", "Airbag", "", 1000, ``)))}) + _, err := r.report(5, []File{upload(history(play("2023-01-01T10:00:00Z", "Radiohead", "Airbag", "", 1000, ``)))}) done <- err }() <-cat.entered - if _, err := r.svc.Import(context.Background(), 5, []File{upload(`[]`)}); !errors.Is(err, ErrBusy) { + if _, err := r.report(5, []File{upload(`[]`)}); !errors.Is(err, ErrBusy) { t.Errorf("a second import of the same person: %v, want ErrBusy", err) } close(cat.release) @@ -342,7 +374,7 @@ func TestAnImportThatIsRunningBlocksAnotherOfTheSamePerson(t *testing.T) { t.Fatal(err) } // Once it is over, the person can import again, and somebody else could all along. - if _, err := r.svc.Import(context.Background(), 5, []File{upload(`[]`)}); errors.Is(err, ErrBusy) { + if _, err := r.report(5, []File{upload(`[]`)}); errors.Is(err, ErrBusy) { t.Error("still busy after the import ended") } } @@ -353,7 +385,7 @@ func TestManyListensAreRecordedInBatches(t *testing.T) { for i := 0; i < 2500; i++ { plays = append(plays, play(time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC).Add(time.Duration(i)*time.Minute).Format(time.RFC3339), "Radiohead", "Airbag", "", 100_000, ``)) } - rep, err := r.svc.Import(context.Background(), 1, []File{upload(history(plays...))}) + rep, err := r.report(1, []File{upload(history(plays...))}) if err != nil { t.Fatal(err) } @@ -361,3 +393,64 @@ func TestManyListensAreRecordedInBatches(t *testing.T) { t.Errorf("imported %d in %d calls, want 2500 in several", rep.History.Imported, r.li.calls) } } + +func TestAnImportIsKeptAndListedNewestFirst(t *testing.T) { + r := newRig(&fakeCatalog{songs: songs}) + first, err := r.svc.Import(context.Background(), 1, []File{upload(history(play("2023-01-01T10:00:00Z", "Radiohead", "Airbag", "", 280_000, `,"reason_end":"trackdone"`)))}) + if err != nil || first.ID != 1 || first.At.IsZero() || first.Report.History.Imported != 1 { + t.Fatalf("first import = %+v, %v, want it kept as 1 with one listen", first, err) + } + second, err := r.svc.Import(context.Background(), 1, []File{upload(history(play("2023-01-01T10:00:00Z", "Radiohead", "Airbag", "", 280_000, `,"reason_end":"trackdone"`)))}) + if err != nil || second.ID != 2 || second.Report.History.AlreadyThere != 1 { + t.Fatalf("second import = %+v, %v, want 2 with the play already there", second, err) + } + page, err := r.svc.History(context.Background(), 1, 10, 0) + if err != nil || page.Total != 2 || len(page.Items) != 2 || page.Items[0].ID != 2 || page.Items[1].ID != 1 { + t.Fatalf("History = %+v, %v, want the two imports, newest first", page, err) + } + if page.Limit != 10 || page.Offset != 0 { + t.Errorf("page = %d/%d, want 10/0", page.Limit, page.Offset) + } +} + +func TestAnImportThatFindsNothingIsNotKept(t *testing.T) { + r := newRig(&fakeCatalog{songs: songs}) + if _, err := r.report(1, []File{upload(`{"hello":"world"}`)}); !errors.Is(err, ErrNothingFound) { + t.Fatalf("err = %v, want ErrNothingFound", err) + } + if len(r.runs.saved) != 0 { + t.Errorf("%d reports kept, want none", len(r.runs.saved)) + } +} + +func TestAReportThatCannotBeKeptStillComesBack(t *testing.T) { + r := newRig(&fakeCatalog{songs: songs}) + r.runs.fail = errors.New("disk full") + run, err := r.svc.Import(context.Background(), 1, []File{upload(history(play("2023-01-01T10:00:00Z", "Radiohead", "Airbag", "", 280_000, `,"reason_end":"trackdone"`)))}) + if !errors.Is(err, ErrNotKept) { + t.Fatalf("err = %v, want ErrNotKept", err) + } + if run.ID != 0 || run.Report.History.Imported != 1 || run.At.IsZero() { + t.Errorf("run = %+v, want the report with ID 0", run) + } +} + +func TestHistoryPagesAreBounded(t *testing.T) { + r := newRig(&fakeCatalog{songs: songs}) + for i := 0; i < 3; i++ { + r.runs.saved = append(r.runs.saved, Run{ID: int64(i + 1)}) + } + page, err := r.svc.History(context.Background(), 1, 0, 0) + if err != nil || page.Limit != DefaultPage || len(page.Items) != 3 { + t.Errorf("no limit = %+v, %v, want the default page", page, err) + } + if page, _ := r.svc.History(context.Background(), 1, 100000, 0); page.Limit != MaxPage { + t.Errorf("a huge limit = %d, want %d", page.Limit, MaxPage) + } + if _, err := r.svc.History(context.Background(), 1, -1, 0); !errors.Is(err, ErrInvalidPage) { + t.Errorf("a negative limit: %v, want ErrInvalidPage", err) + } + if _, err := r.svc.History(context.Background(), 1, 10, -1); !errors.Is(err, ErrInvalidPage) { + t.Errorf("a negative offset: %v, want ErrInvalidPage", err) + } +} diff --git a/internal/storage/spotify_import_test.go b/internal/storage/spotify_import_test.go index 4e89a91..490376c 100644 --- a/internal/storage/spotify_import_test.go +++ b/internal/storage/spotify_import_test.go @@ -3,6 +3,7 @@ package storage import ( "context" "fmt" + "reflect" "sort" "testing" "time" @@ -11,6 +12,7 @@ import ( "codeberg.org/squidd/sonda/internal/library" "codeberg.org/squidd/sonda/internal/listening" + "codeberg.org/squidd/sonda/internal/spotifyimport" ) func TestRecordManyStoresTheListensOnceAndCountsWhatIsNew(t *testing.T) { @@ -114,3 +116,64 @@ func TestSearchKeyIsExported(t *testing.T) { t.Errorf("SearchKey(Björk) = %q", SearchKey("Björk")) } } + +func TestSpotifyImportsAreKeptPerUserNewestFirst(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + ctx := context.Background() + users := NewUserRepo(db) + alice, err := users.CreateUser(ctx, "alice", "h", false) + if err != nil { + t.Fatal(err) + } + bob, err := users.CreateUser(ctx, "bob", "h", false) + if err != nil { + t.Fatal(err) + } + repo := NewSpotifyImportRepo(db) + rep := spotifyimport.Report{ + History: spotifyimport.History{Entries: 10, Matched: 4, Imported: 3, AlreadyThere: 1, NotFound: 6, Ignored: 2}, + Playlists: []spotifyimport.PlaylistResult{{Name: "Road trip", Tracks: 3, Matched: 1, Status: "created"}}, + Missing: []spotifyimport.Missing{{Artist: "Blur", Count: 4}}, + MissingTotal: 7, + } + first, err := repo.Save(ctx, alice.ID, time.Unix(1000, 0), rep) + if err != nil || first == 0 { + t.Fatalf("Save = %d, %v", first, err) + } + later := rep + later.History.Imported = 9 + second, err := repo.Save(ctx, alice.ID, time.Unix(2000, 0), later) + if err != nil || second == first { + t.Fatalf("second Save = %d, %v", second, err) + } + if _, err := repo.Save(ctx, bob.ID, time.Unix(3000, 0), rep); err != nil { + t.Fatal(err) + } + + runs, total, err := repo.List(ctx, alice.ID, 10, 0) + if err != nil || total != 2 || len(runs) != 2 { + t.Fatalf("List = %d runs, total %d, %v, want her two", len(runs), total, err) + } + if runs[0].ID != second || runs[0].Report.History.Imported != 9 || runs[0].At.Unix() != 2000 { + t.Errorf("newest = %+v, want the second import", runs[0]) + } + if !reflect.DeepEqual(runs[1].Report, rep) || runs[1].At.Unix() != 1000 { + t.Errorf("oldest = %+v, want the report as it was saved", runs[1]) + } + page, total, err := repo.List(ctx, alice.ID, 1, 1) + if err != nil || total != 2 || len(page) != 1 || page[0].ID != first { + t.Errorf("second page = %+v, total %d, %v, want the first import", page, total, err) + } + none, total, err := repo.List(ctx, alice.ID, 10, 5) + if err != nil || total != 2 || len(none) != 0 { + t.Errorf("past the end = %d runs, total %d, %v", len(none), total, err) + } + + if _, err := db.ExecContext(ctx, "DELETE FROM users WHERE id = ?", alice.ID); err != nil { + t.Fatal(err) + } + if runs, total, _ := repo.List(ctx, alice.ID, 10, 0); total != 0 || len(runs) != 0 { + t.Errorf("after the account is gone: %d runs, total %d, want none", len(runs), total) + } + }) +} diff --git a/internal/storage/spotify_imports.go b/internal/storage/spotify_imports.go new file mode 100644 index 0000000..95bbe1d --- /dev/null +++ b/internal/storage/spotify_imports.go @@ -0,0 +1,59 @@ +package storage + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/uptrace/bun" + + "codeberg.org/squidd/sonda/internal/spotifyimport" +) + +type spotifyImportRow struct { + bun.BaseModel `bun:"table:spotify_imports"` + + ID int64 `bun:"id,pk,autoincrement"` + UserID int64 `bun:"user_id,notnull"` + CreatedAt int64 `bun:"created_at,notnull"` + Report string `bun:"report,notnull"` +} + +// SpotifyImportRepo keeps the reports of the imports of Spotify exports. It implements spotifyimport.Runs. +type SpotifyImportRepo struct{ db *bun.DB } + +// NewSpotifyImportRepo builds a SpotifyImportRepo. +func NewSpotifyImportRepo(db *bun.DB) *SpotifyImportRepo { return &SpotifyImportRepo{db: db} } + +// Save keeps a report and gives its id. +func (r *SpotifyImportRepo) Save(ctx context.Context, userID int64, at time.Time, rep spotifyimport.Report) (int64, error) { + body, err := json.Marshal(rep) + if err != nil { + return 0, fmt.Errorf("encode the report: %w", err) + } + row := &spotifyImportRow{UserID: userID, CreatedAt: at.Unix(), Report: string(body)} + if _, err := r.db.NewInsert().Model(row).Exec(ctx); err != nil { + return 0, fmt.Errorf("insert spotify import: %w", err) + } + return row.ID, nil +} + +// List gives one page of the reports of a person, newest first, and how many there are. +func (r *SpotifyImportRepo) List(ctx context.Context, userID int64, limit, offset int) ([]spotifyimport.Run, int, error) { + var rows []spotifyImportRow + total, err := r.db.NewSelect().Model(&rows).Where("user_id = ?", userID). + OrderExpr("created_at DESC, id DESC").Limit(int64(limit)).Offset(int64(offset)).ScanAndCount(ctx) + if err != nil { + return nil, 0, fmt.Errorf("list spotify imports: %w", err) + } + runs := make([]spotifyimport.Run, 0, len(rows)) + for _, row := range rows { + run := spotifyimport.Run{ID: row.ID, At: time.Unix(row.CreatedAt, 0)} + if err := json.Unmarshal([]byte(row.Report), &run.Report); err != nil { + return nil, 0, fmt.Errorf("decode spotify import %d: %w", row.ID, err) + } + runs = append(runs, run) + } + return runs, int(total), nil +} diff --git a/internal/storage/storage_test.go b/internal/storage/storage_test.go index 5aa4c9f..5820e03 100644 --- a/internal/storage/storage_test.go +++ b/internal/storage/storage_test.go @@ -156,8 +156,8 @@ func TestMigrateCreatesSchemaAndIsIdempotent(t *testing.T) { if err := db.QueryRow("SELECT COUNT(*) FROM schema_migrations").Scan(&applied); err != nil { t.Fatal(err) } - if applied != 29 { - t.Errorf("applied migrations = %d, want 29", applied) + if applied != 30 { + t.Errorf("applied migrations = %d, want 30", applied) } }) } diff --git a/migrations/postgres/0030_spotify_imports.sql b/migrations/postgres/0030_spotify_imports.sql new file mode 100644 index 0000000..23301cf --- /dev/null +++ b/migrations/postgres/0030_spotify_imports.sql @@ -0,0 +1,10 @@ +-- PostgreSQL flavour of sqlite/0030_spotify_imports.sql. + +CREATE TABLE spotify_imports ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + user_id BIGINT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + created_at BIGINT NOT NULL, + report TEXT NOT NULL +); + +CREATE INDEX idx_spotify_imports_user ON spotify_imports(user_id, created_at DESC); diff --git a/migrations/sqlite/0030_spotify_imports.sql b/migrations/sqlite/0030_spotify_imports.sql new file mode 100644 index 0000000..55d5fff --- /dev/null +++ b/migrations/sqlite/0030_spotify_imports.sql @@ -0,0 +1,11 @@ +-- What each import of a Spotify export did, kept so the numbers can be read again later: the report as JSON (see +-- internal/spotifyimport). The files themselves are never kept. + +CREATE TABLE spotify_imports ( + id INTEGER PRIMARY KEY, + user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, + created_at INTEGER NOT NULL, -- unix seconds + report TEXT NOT NULL +); + +CREATE INDEX idx_spotify_imports_user ON spotify_imports(user_id, created_at DESC); diff --git a/web/src/api/importSpotify.ts b/web/src/api/importSpotify.ts index 1957612..bb0d3e7 100644 --- a/web/src/api/importSpotify.ts +++ b/web/src/api/importSpotify.ts @@ -1,4 +1,4 @@ -import { useMutation, useQueryClient } from '@tanstack/react-query' +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' import { api } from '@/api/client' import type { components } from '@/api/schema' @@ -20,3 +20,14 @@ export function useImportSpotify() { onSuccess: () => client.invalidateQueries(), }) } + +/** How many past imports the page reads: they are rare, so the page shows them all. */ +const PAST_LIMIT = 100 + +/** What each import did, newest first. */ +export function useSpotifyImports() { + return useQuery({ + queryKey: ['imports', 'spotify'], + queryFn: () => api.get(`/me/import/spotify?limit=${PAST_LIMIT}`), + }) +} diff --git a/web/src/api/schema.d.ts b/web/src/api/schema.d.ts index 6cad3f7..e1ae665 100644 --- a/web/src/api/schema.d.ts +++ b/web/src/api/schema.d.ts @@ -1553,7 +1553,14 @@ export interface paths { path?: never; cookie?: never; }; - get?: never; + /** + * Your past imports of Spotify data + * @description What each import of yours did, newest first: how many plays were recorded, how many were already there, + * how many are not in your library, the playlists it made and the artists you are missing. The files + * themselves are never kept, only these numbers. Only yours. An import is listed once it is done; one that + * found nothing is not. + */ + get: operations["listMySpotifyImports"]; put?: never; /** * Import your Spotify history and playlists @@ -3801,7 +3808,21 @@ export interface components { */ discover_from_listening: boolean; }; + SpotifyImportPage: { + items: components["schemas"]["SpotifyImportReport"][]; + /** @description Size of the whole result, not of this page. */ + total: number; + limit: number; + offset: number; + }; SpotifyImportReport: { + /** @description The import. 0 when its report could not be kept. */ + id: number; + /** + * Format: date-time + * @description When the import was done. + */ + created_at: string; history: { /** @description Plays of songs that were read. */ entries: number; @@ -6646,6 +6667,32 @@ export interface operations { 401: components["responses"]["Unauthorized"]; }; }; + listMySpotifyImports: { + parameters: { + query?: { + /** @description Page size. 50 when absent or 0; values above 200 are lowered to 200. */ + limit?: components["parameters"]["Limit"]; + offset?: components["parameters"]["Offset"]; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description A page of imports. */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SpotifyImportPage"]; + }; + }; + 400: components["responses"]["InvalidInput"]; + 401: components["responses"]["Unauthorized"]; + }; + }; importMySpotify: { parameters: { query?: never; @@ -6662,7 +6709,10 @@ export interface operations { }; }; responses: { - /** @description What the import did. */ + /** + * @description What the import did. It is also kept (`GET /me/import/spotify`); if keeping it failed the report is still + * sent, with `id` 0. + */ 200: { headers: { [name: string]: unknown; diff --git a/web/src/components/ImportReportCard.test.tsx b/web/src/components/ImportReportCard.test.tsx new file mode 100644 index 0000000..47c776d --- /dev/null +++ b/web/src/components/ImportReportCard.test.tsx @@ -0,0 +1,59 @@ +import { render, screen, within } from '@testing-library/react' +import { beforeEach, expect, test, vi } from 'vitest' +import type { SpotifyImportReport } from '@/api/importSpotify' +import { initI18n } from '@/i18n' +import ImportReportCard from './ImportReportCard' + +beforeEach(async () => { + vi.spyOn(navigator, 'language', 'get').mockReturnValue('en-US') + await initI18n() +}) + +const report: SpotifyImportReport = { + id: 3, + created_at: '2026-11-15T10:00:00Z', + history: { entries: 120, matched: 100, imported: 80, already_there: 20, not_found: 20, ignored: 3 }, + playlists: [ + { name: 'Road', tracks: 20, matched: 15, status: 'created' }, + { name: 'Mine', tracks: 5, matched: 5, status: 'exists' }, + { name: 'Nothing', tracks: 4, matched: 0, status: 'empty' }, + ], + missing: { total: 3, artists: [{ name: 'Blur', count: 9 }, { name: 'Oasis', count: 4 }] }, +} + +test('a card says when the import was done and what it did', () => { + render() + const card = screen.getByRole('article', { name: /^Import of November 15, 2026/ }) + expect(within(card).getByText('120 plays read from the files.')).toBeInTheDocument() + expect(within(card).getByText('80 listens added to your history.')).toBeInTheDocument() + expect(within(card).getByText('20 were already there.')).toBeInTheDocument() + expect(within(card).getByText('20 plays are of songs that are not in your library.')).toBeInTheDocument() + expect(within(card).getByText(/3 plays skipped/)).toBeInTheDocument() +}) + +test('the playlists say what became of each', () => { + render() + expect(screen.getByText('Road: 15 of 20 songs found')).toBeInTheDocument() + expect(screen.getByText('Made')).toBeInTheDocument() + expect(screen.getByText('You already have one with this name')).toBeInTheDocument() + expect(screen.getByText('None of its songs is in your library')).toBeInTheDocument() +}) + +test('the missing artists are behind a summary that counts them, most played first', () => { + render() + expect(screen.getByText('3 artists you are missing')).toBeInTheDocument() + expect(screen.getByText('Blur')).toBeInTheDocument() + expect(screen.getByText('9')).toBeInTheDocument() + expect(screen.getByText('and 1 more')).toBeInTheDocument() // 3 artists miss songs, 2 are listed +}) + +test('an import with nothing to show says so, and has no empty lists', () => { + render( + , + ) + expect(screen.getByText('No plays in these files.')).toBeInTheDocument() + expect(screen.queryByText('Playlists')).not.toBeInTheDocument() + expect(screen.queryByText(/you are missing/)).not.toBeInTheDocument() +}) diff --git a/web/src/components/ImportReportCard.tsx b/web/src/components/ImportReportCard.tsx new file mode 100644 index 0000000..738152b --- /dev/null +++ b/web/src/components/ImportReportCard.tsx @@ -0,0 +1,64 @@ +import { useTranslation } from 'react-i18next' +import type { SpotifyImportReport } from '@/api/importSpotify' + +/** + * What one import of Spotify data did, with the day it was done: the plays added, the ones that were already there, + * the ones of songs that are not in the library, the playlists it made and the artists that are missing. + */ +export default function ImportReportCard({ report }: { report: SpotifyImportReport }) { + const { t, i18n } = useTranslation() + const h = report.history + const more = report.missing.total - report.missing.artists.length + const when = new Intl.DateTimeFormat(i18n.language, { dateStyle: 'long', timeStyle: 'short' }).format(new Date(report.created_at)) + const title = t('spotify.importOf', { when }) + + return ( +
+

{title}

+ + {h.entries === 0 && report.playlists.length === 0 ? ( +

{t('spotify.noPlays')}

+ ) : ( + h.entries > 0 && ( +
    +
  • {t('spotify.read', { count: h.entries })}
  • +
  • {t('spotify.imported', { count: h.imported })}
  • + {h.already_there > 0 &&
  • {t('spotify.already', { count: h.already_there })}
  • } + {h.not_found > 0 &&
  • {t('spotify.notFound', { count: h.not_found })}
  • } + {h.ignored > 0 &&
  • {t('spotify.ignored', { count: h.ignored })}
  • } +
+ ) + )} + + {report.playlists.length > 0 && ( +
+

{t('spotify.playlists')}

+
    + {report.playlists.map((p, i) => ( +
  • + {t('spotify.playlistLine', { name: p.name, matched: p.matched, tracks: p.tracks })} + {t(`spotify.status.${p.status}`)} +
  • + ))} +
+
+ )} + + {report.missing.artists.length > 0 && ( +
+ {t('spotify.missingCount', { count: report.missing.total })} +

{t('spotify.missing')}

+
    + {report.missing.artists.map((a) => ( +
  • + {a.name} + {a.count} +
  • + ))} +
+ {more > 0 &&

{t('spotify.missingMore', { count: more })}

} +
+ )} +
+ ) +} diff --git a/web/src/components/SpotifyImportBlock.test.tsx b/web/src/components/SpotifyImportBlock.test.tsx index 5203f85..70f44df 100644 --- a/web/src/components/SpotifyImportBlock.test.tsx +++ b/web/src/components/SpotifyImportBlock.test.tsx @@ -1,5 +1,5 @@ import { QueryClient, QueryClientProvider } from '@tanstack/react-query' -import { render, screen, within } from '@testing-library/react' +import { render, screen } from '@testing-library/react' import userEvent from '@testing-library/user-event' import { beforeEach, expect, test, vi } from 'vitest' import { initI18n } from '@/i18n' @@ -12,6 +12,8 @@ beforeEach(async () => { }) const report = { + id: 3, + created_at: '2026-11-15T10:00:00Z', history: { entries: 120, matched: 100, imported: 80, already_there: 20, not_found: 20, ignored: 3 }, playlists: [ { name: 'Road', tracks: 20, matched: 15, status: 'created' }, @@ -43,31 +45,20 @@ test('it says what is imported and that nothing leaves the server', () => { expect(screen.getByRole('button', { name: 'Import' })).toBeDisabled() }) -test('the files are sent together as a form, and the report says what was done', async () => { +test('the files are sent together as a form, and the import says it is done', async () => { const { calls } = setup() await userEvent.upload(screen.getByLabelText('Files from Spotify'), [zip, json]) expect(screen.getByText('2 files chosen')).toBeInTheDocument() await userEvent.click(screen.getByRole('button', { name: 'Import' })) - const done = await screen.findByRole('region', { name: 'Import done' }) + expect(await screen.findByRole('status')).toHaveTextContent(/Import done/) const sent = calls.filter((c) => c.key === 'POST /api/v1/me/import/spotify') expect(sent).toHaveLength(1) const form = sent[0].init.body as FormData expect(form.getAll('files').map((f) => (f as File).name)).toEqual(['my_spotify_data.zip', 'Streaming_History_Audio_2020_0.json']) - - expect(within(done).getByText('80 listens added to your history.')).toBeInTheDocument() - expect(within(done).getByText('20 were already there.')).toBeInTheDocument() - expect(within(done).getByText('20 plays are of songs that are not in your library.')).toBeInTheDocument() - expect(within(done).getByText(/3 plays skipped/)).toBeInTheDocument() - expect(within(done).getByText('Road: 15 of 20 songs found')).toBeInTheDocument() - expect(within(done).getByText('Made')).toBeInTheDocument() - expect(within(done).getByText('You already have one with this name')).toBeInTheDocument() - expect(within(done).getByText('None of its songs is in your library')).toBeInTheDocument() - expect(within(done).getByText('Blur')).toBeInTheDocument() - expect(within(done).getByText('9')).toBeInTheDocument() - expect(within(done).getByText('and 1 more')).toBeInTheDocument() // 3 artists have songs not found, 2 are shown }) + test('while it works the button is off and says so', async () => { let finish: (r: Response) => void = () => {} setup({ 'POST /api/v1/me/import/spotify': () => new Promise((resolve) => (finish = resolve)) }) @@ -76,7 +67,7 @@ test('while it works the button is off and says so', async () => { expect(await screen.findByRole('status')).toHaveTextContent(/Importing/) expect(screen.getByRole('button', { name: 'Import' })).toBeDisabled() finish(new Response(JSON.stringify(report), { status: 200, headers: { 'Content-Type': 'application/json' } })) - expect(await screen.findByRole('region', { name: 'Import done' })).toBeInTheDocument() + expect(await screen.findByText(/Import done/)).toBeInTheDocument() }) test('files that are not Spotify’s are explained', async () => { @@ -84,7 +75,7 @@ test('files that are not Spotify’s are explained', async () => { await userEvent.upload(screen.getByLabelText('Files from Spotify'), json) await userEvent.click(screen.getByRole('button', { name: 'Import' })) expect(await screen.findByRole('alert')).toHaveTextContent(/do not hold a Spotify streaming history or playlists/) - expect(screen.queryByRole('region', { name: 'Import done' })).not.toBeInTheDocument() + expect(screen.queryByText(/Import done/)).not.toBeInTheDocument() }) test('an import that is too large, damaged or already running has its own message', async () => { @@ -100,10 +91,3 @@ test('an import that is too large, damaged or already running has its own messag expect(await screen.findByRole('alert')).toHaveTextContent(text) } }) - -test('an import with no plays in it says so', async () => { - setup({ 'POST /api/v1/me/import/spotify': () => ({ ...report, history: { entries: 0, matched: 0, imported: 0, already_there: 0, not_found: 0, ignored: 0 }, playlists: [], missing: { total: 0, artists: [] } }) }) - await userEvent.upload(screen.getByLabelText('Files from Spotify'), json) - await userEvent.click(screen.getByRole('button', { name: 'Import' })) - expect(await screen.findByText('No plays in these files.')).toBeInTheDocument() -}) diff --git a/web/src/components/SpotifyImportBlock.tsx b/web/src/components/SpotifyImportBlock.tsx index 091039d..786a3e2 100644 --- a/web/src/components/SpotifyImportBlock.tsx +++ b/web/src/components/SpotifyImportBlock.tsx @@ -1,14 +1,14 @@ import { useState, type ChangeEvent } from 'react' import { useTranslation } from 'react-i18next' import { ApiError } from '@/api/client' -import { useImportSpotify, type SpotifyImportReport } from '@/api/importSpotify' +import { useImportSpotify } from '@/api/importSpotify' import { Button } from '@/components/ui/button' import { errorMessageKey } from '@/i18n' /** * Bringing the history and the playlists of a Spotify export in. The files are sent to this server, which reads them - * and keeps nothing of them but the listens and playlists that match the library; the report says what was done and - * which artists are missing from the library. + * and keeps nothing of them but the listens and playlists that match the library. What was done is kept as a report, + * which the list of past imports shows (ImportReportCard). */ export default function SpotifyImportBlock() { const { t } = useTranslation() @@ -53,60 +53,11 @@ export default function SpotifyImportBlock() { {t(errorMessageKey(run.error instanceof ApiError ? run.error.code : 'unknown'))}

)} - {run.data && !run.isPending && } + {run.isSuccess && ( +

+ {t('spotify.finished')} +

+ )} ) } - -function Report({ report }: { report: SpotifyImportReport }) { - const { t } = useTranslation() - const h = report.history - const more = report.missing.total - report.missing.artists.length - return ( -
-

{t('spotify.done')}

- - {h.entries === 0 && report.playlists.length === 0 ? ( -

{t('spotify.noPlays')}

- ) : ( - h.entries > 0 && ( -
    -
  • {t('spotify.imported', { count: h.imported })}
  • - {h.already_there > 0 &&
  • {t('spotify.already', { count: h.already_there })}
  • } - {h.not_found > 0 &&
  • {t('spotify.notFound', { count: h.not_found })}
  • } - {h.ignored > 0 &&
  • {t('spotify.ignored', { count: h.ignored })}
  • } -
- ) - )} - - {report.playlists.length > 0 && ( -
-

{t('spotify.playlists')}

-
    - {report.playlists.map((p, i) => ( -
  • - {t('spotify.playlistLine', { name: p.name, matched: p.matched, tracks: p.tracks })} - {t(`spotify.status.${p.status}`)} -
  • - ))} -
-
- )} - - {report.missing.artists.length > 0 && ( -
-

{t('spotify.missing')}

-
    - {report.missing.artists.map((a) => ( -
  • - {a.name} - {a.count} -
  • - ))} -
- {more > 0 &&

{t('spotify.missingMore', { count: more })}

} -
- )} -
- ) -} diff --git a/web/src/components/layout/Sidebar.test.tsx b/web/src/components/layout/Sidebar.test.tsx index 18ab387..5b4a85e 100644 --- a/web/src/components/layout/Sidebar.test.tsx +++ b/web/src/components/layout/Sidebar.test.tsx @@ -27,7 +27,7 @@ function renderSidebar(friends: { id: number; name: string }[] = [], path = '/al test('the menu is words, one for each page, and the page that is open is the current one', () => { const nav = renderSidebar() - for (const name of ['Home', 'Search', 'Discover', 'Artists', 'Albums', 'Songs', 'Mixes', 'Playlists', 'Smart playlists', 'Admin', 'Settings']) { + for (const name of ['Home', 'Search', 'Discover', 'Artists', 'Albums', 'Songs', 'Mixes', 'Playlists', 'Smart playlists', 'Import', 'Admin', 'Settings']) { expect(within(nav).getByRole('link', { name })).toBeInTheDocument() } expect(within(nav).getByRole('link', { name: 'Albums' })).toHaveAttribute('aria-current', 'page') diff --git a/web/src/components/layout/Sidebar.tsx b/web/src/components/layout/Sidebar.tsx index d76fe1e..1c895e3 100644 --- a/web/src/components/layout/Sidebar.tsx +++ b/web/src/components/layout/Sidebar.tsx @@ -15,6 +15,7 @@ const LINKS: { to: string; key: string; end: boolean; adminOnly: boolean; friend { to: '/mixes', key: 'nav.mixes', end: false, adminOnly: false }, { to: '/playlists', key: 'nav.playlists', end: false, adminOnly: false }, { to: '/smart-playlists', key: 'nav.smartPlaylists', end: false, adminOnly: false }, + { to: '/import', key: 'nav.import', end: false, adminOnly: false }, { to: '/friends', key: 'nav.friends', end: false, adminOnly: false, friends: true }, { to: '/admin', key: 'nav.admin', end: false, adminOnly: true }, { to: '/settings', key: 'nav.settings', end: false, adminOnly: false }, diff --git a/web/src/i18n/locales/en.json b/web/src/i18n/locales/en.json index 3653e38..7e9bf03 100644 --- a/web/src/i18n/locales/en.json +++ b/web/src/i18n/locales/en.json @@ -19,6 +19,7 @@ "friends": "Friends", "admin": "Admin", "settings": "Settings", + "import": "Import", "more": "More", "tabs": "Tabs" }, @@ -99,7 +100,9 @@ "replayGainTrack": "Song by song", "replayGainAlbum": "Album by album", "replayGainOff": "Off", - "spotify": "Import from Spotify", + "spotify": "Import", + "importHint": "Bring in your Spotify history and playlists, and see what each import did, on the Import page.", + "importLink": "Open the Import page", "password": "Password", "currentPassword": "Current password", "newPassword": "New password", @@ -792,7 +795,12 @@ "chosen_other": "{{count}} files chosen", "start": "Import", "running": "Importing. With a long history this can take a minute…", - "done": "Import done", + "finished": "Import done. What it did is the first one in the list below.", + "importOf": "Import of {{when}}", + "read_one": "{{count}} play read from the files.", + "read_other": "{{count}} plays read from the files.", + "missingCount_one": "{{count}} artist you are missing", + "missingCount_other": "{{count}} artists you are missing", "noPlays": "No plays in these files.", "imported_one": "{{count}} listen added to your history.", "imported_other": "{{count}} listens added to your history.", @@ -813,6 +821,11 @@ "missing": "Artists you play on Spotify and have no music of here", "missingMore": "and {{count}} more" }, + "import": { + "spotify": "Spotify", + "past": "Past imports", + "none": "No import yet. When you bring in a Spotify export, what it did is kept here." + }, "playlists": { "empty": "You have no playlists yet. Make one below, or put a song in a new one with the playlist button next to it.", "newLabel": "Name of the new playlist", diff --git a/web/src/i18n/locales/it.json b/web/src/i18n/locales/it.json index 3f6198c..0492f23 100644 --- a/web/src/i18n/locales/it.json +++ b/web/src/i18n/locales/it.json @@ -19,6 +19,7 @@ "friends": "Amici", "admin": "Admin", "settings": "Impostazioni", + "import": "Importa", "more": "Altro", "tabs": "Schede" }, @@ -99,7 +100,9 @@ "replayGainTrack": "Brano per brano", "replayGainAlbum": "Album per album", "replayGainOff": "Disattivato", - "spotify": "Importa da Spotify", + "spotify": "Importa", + "importHint": "Porta qui la cronologia e le playlist di Spotify, e guarda cosa ha fatto ogni importazione, nella pagina Importa.", + "importLink": "Apri la pagina Importa", "password": "Password", "currentPassword": "Password attuale", "newPassword": "Nuova password", @@ -792,7 +795,12 @@ "chosen_other": "{{count}} file scelti", "start": "Importa", "running": "Importazione in corso. Con una cronologia lunga può volerci un minuto…", - "done": "Importazione finita", + "finished": "Importazione finita. Cosa ha fatto è la prima della lista qui sotto.", + "importOf": "Importazione del {{when}}", + "read_one": "{{count}} ascolto letto dai file.", + "read_other": "{{count}} ascolti letti dai file.", + "missingCount_one": "{{count}} artista che ti manca", + "missingCount_other": "{{count}} artisti che ti mancano", "noPlays": "Nessun ascolto in questi file.", "imported_one": "{{count}} ascolto aggiunto alla cronologia.", "imported_other": "{{count}} ascolti aggiunti alla cronologia.", @@ -813,6 +821,11 @@ "missing": "Artisti che ascolti su Spotify e di cui qui non hai musica", "missingMore": "e altri {{count}}" }, + "import": { + "spotify": "Spotify", + "past": "Importazioni passate", + "none": "Nessuna importazione, per ora. Quando porti qui un export di Spotify, cosa ha fatto resta salvato qui." + }, "playlists": { "empty": "Non hai ancora nessuna playlist. Creane una qui sotto, oppure metti un brano in una nuova con il pulsante accanto al brano.", "newLabel": "Nome della nuova playlist", diff --git a/web/src/pages/ImportPage.test.tsx b/web/src/pages/ImportPage.test.tsx new file mode 100644 index 0000000..cbedf22 --- /dev/null +++ b/web/src/pages/ImportPage.test.tsx @@ -0,0 +1,81 @@ +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { render, screen, within } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { MemoryRouter } from 'react-router-dom' +import { beforeEach, expect, test, vi } from 'vitest' +import { initI18n } from '@/i18n' +import { apiError, mockFetch, type MockRoutes } from '@/test/mockFetch' +import ImportPage from './ImportPage' + +beforeEach(async () => { + vi.spyOn(navigator, 'language', 'get').mockReturnValue('en-US') + await initI18n() +}) + +const run = (id: number, imported: number, at: string) => ({ + id, + created_at: at, + history: { entries: imported + 5, matched: imported, imported, already_there: 0, not_found: 5, ignored: 0 }, + playlists: [], + missing: { total: 1, artists: [{ name: 'Blur', count: 5 }] }, +}) + +function setup(routes: MockRoutes) { + const mocked = mockFetch(routes) + const client = new QueryClient({ defaultOptions: { queries: { retry: false }, mutations: { retry: false } } }) + render( + + + + + , + ) + return mocked +} + +test('the page has the form to import and the imports of the past, newest first', async () => { + setup({ + 'GET /api/v1/me/import/spotify': () => ({ items: [run(2, 40, '2026-11-15T10:00:00Z'), run(1, 7, '2026-11-01T09:00:00Z')], total: 2, limit: 100, offset: 0 }), + }) + expect(screen.getByRole('heading', { level: 1, name: 'Import' })).toBeInTheDocument() + expect(screen.getByLabelText('Files from Spotify')).toBeInTheDocument() + + const past = await screen.findByRole('region', { name: 'Past imports' }) + const cards = await within(past).findAllByRole('article') + expect(cards).toHaveLength(2) + expect(within(cards[0]).getByText('40 listens added to your history.')).toBeInTheDocument() + expect(within(cards[1]).getByText('7 listens added to your history.')).toBeInTheDocument() +}) + +test('with no import yet it says where they will be kept', async () => { + setup({ 'GET /api/v1/me/import/spotify': () => ({ items: [], total: 0, limit: 100, offset: 0 }) }) + expect(await screen.findByText(/No import yet/)).toBeInTheDocument() +}) + +test('a list that cannot be read has a way to try again', async () => { + let fail = true + const { calls } = setup({ + 'GET /api/v1/me/import/spotify': () => (fail ? apiError(500, 'internal', 'x') : { items: [run(1, 3, '2026-11-01T09:00:00Z')], total: 1, limit: 100, offset: 0 }), + }) + const retry = await screen.findByRole('button', { name: 'Try again' }) + fail = false + await userEvent.click(retry) + expect(await screen.findByText('3 listens added to your history.')).toBeInTheDocument() + expect(calls.filter((c) => c.key === 'GET /api/v1/me/import/spotify').length).toBeGreaterThan(1) +}) + +test('an import that is done shows up in the list', async () => { + let done = false + setup({ + 'GET /api/v1/me/import/spotify': () => ({ items: done ? [run(1, 12, '2026-11-15T10:00:00Z')] : [], total: done ? 1 : 0, limit: 100, offset: 0 }), + 'POST /api/v1/me/import/spotify': () => { + done = true + return run(1, 12, '2026-11-15T10:00:00Z') + }, + }) + await screen.findByText(/No import yet/) + await userEvent.upload(screen.getByLabelText('Files from Spotify'), new File(['PK'], 'my_spotify_data.zip')) + await userEvent.click(screen.getByRole('button', { name: 'Import' })) + expect(await screen.findByText('12 listens added to your history.')).toBeInTheDocument() + expect(screen.queryByText(/No import yet/)).not.toBeInTheDocument() +}) diff --git a/web/src/pages/ImportPage.tsx b/web/src/pages/ImportPage.tsx new file mode 100644 index 0000000..5debb08 --- /dev/null +++ b/web/src/pages/ImportPage.tsx @@ -0,0 +1,38 @@ +import { useTranslation } from 'react-i18next' +import { useSpotifyImports } from '@/api/importSpotify' +import ImportReportCard from '@/components/ImportReportCard' +import PageTitle from '@/components/PageTitle' +import Section from '@/components/Section' +import SpotifyImportBlock from '@/components/SpotifyImportBlock' +import { EmptyMessage, ErrorMessage, Loading } from '@/components/Status' + +/** Bringing music data in from elsewhere (for now Spotify), and what each import did, kept so it can be read again. */ +export default function ImportPage() { + const { t } = useTranslation() + const past = useSpotifyImports() + return ( +
+ {t('nav.import')} + +
+ +
+ +
+ {past.isError ? ( + void past.refetch()} /> + ) : past.isPending ? ( + + ) : past.data.items.length === 0 ? ( + {t('import.none')} + ) : ( +
+ {past.data.items.map((report) => ( + + ))} +
+ )} +
+
+ ) +} diff --git a/web/src/pages/LibraryPage.test.tsx b/web/src/pages/LibraryPage.test.tsx index c9c41d5..0cf4ce5 100644 --- a/web/src/pages/LibraryPage.test.tsx +++ b/web/src/pages/LibraryPage.test.tsx @@ -31,7 +31,7 @@ test('Library is a list of the places of the music, each a row that leads there' ]) }) -test('More has Settings, and Admin only for an admin', () => { +test('More has Import and Settings, and Admin only for an admin', () => { sessionStore.setState({ status: 'authenticated', user: { id: 1, email: 'ann@example.org', is_admin: false } }) const { unmount } = render( @@ -39,6 +39,7 @@ test('More has Settings, and Admin only for an admin', () => { , ) expect(screen.getByRole('heading', { level: 1, name: 'More' })).toBeInTheDocument() + expect(screen.getByRole('link', { name: 'Import' })).toHaveAttribute('href', '/import') expect(screen.getByRole('link', { name: 'Settings' })).toHaveAttribute('href', '/settings') expect(screen.queryByRole('link', { name: 'Admin' })).not.toBeInTheDocument() unmount() diff --git a/web/src/pages/MorePage.tsx b/web/src/pages/MorePage.tsx index 308e437..24efb15 100644 --- a/web/src/pages/MorePage.tsx +++ b/web/src/pages/MorePage.tsx @@ -7,7 +7,10 @@ import PageTitle from '@/components/PageTitle' export default function MorePage() { const { t } = useTranslation() const { user } = useSession() - const links = [{ to: '/settings', label: t('nav.settings') }] + const links = [ + { to: '/import', label: t('nav.import') }, + { to: '/settings', label: t('nav.settings') }, + ] if (user?.is_admin) links.push({ to: '/admin', label: t('nav.admin') }) return (
diff --git a/web/src/pages/SettingsPage.test.tsx b/web/src/pages/SettingsPage.test.tsx index db72a26..37699ae 100644 --- a/web/src/pages/SettingsPage.test.tsx +++ b/web/src/pages/SettingsPage.test.tsx @@ -104,6 +104,13 @@ test('choosing Italian changes the words on the page', async () => { expect(localStorage.getItem('sonda.lang')).toBe('it') }) +test('Settings points to the Import page instead of holding the form', () => { + renderSettings() + const group = screen.getByRole('region', { name: 'Import' }) + expect(within(group).getByRole('link', { name: 'Open the Import page' })).toHaveAttribute('href', '/import') + expect(screen.queryByLabelText('Files from Spotify')).not.toBeInTheDocument() +}) + test('Discover has its own group with the switch for using your listening, off until you turn it on', async () => { renderSettings() expect(screen.getByRole('region', { name: 'Discover' })).toBeInTheDocument() diff --git a/web/src/pages/SettingsPage.tsx b/web/src/pages/SettingsPage.tsx index 8809145..dc8097f 100644 --- a/web/src/pages/SettingsPage.tsx +++ b/web/src/pages/SettingsPage.tsx @@ -1,11 +1,11 @@ import { useState, type FormEvent } from 'react' import { useTranslation } from 'react-i18next' +import { Link } from 'react-router-dom' import { api, ApiError } from '@/api/client' import { logout, useSession } from '@/auth/session' import ChoiceGroup from '@/components/ChoiceGroup' import ListeningSwitch from '@/components/ListeningSwitch' import MastodonBlock from '@/components/MastodonBlock' -import SpotifyImportBlock from '@/components/SpotifyImportBlock' import Section from '@/components/Section' import { useInstall } from '@/pwa/useInstall' import { Button } from '@/components/ui/button' @@ -185,7 +185,10 @@ export default function SettingsPage() {
- +

{t('settings.importHint')}

+ + {t('settings.importLink')} +
diff --git a/web/src/router.tsx b/web/src/router.tsx index e299956..9ffb893 100644 --- a/web/src/router.tsx +++ b/web/src/router.tsx @@ -34,6 +34,7 @@ export const routes: RouteObject[] = [ { path: '/smart-playlists/suggested/:key', lazy: page(() => import('@/pages/SuggestedSmartPage')) }, { path: '/smart-playlists/:id', lazy: page(() => import('@/pages/SmartPlaylistPage')) }, { path: '/smart-playlists/:id/edit', lazy: page(() => import('@/pages/SmartEditorPage')) }, + { path: '/import', lazy: page(() => import('@/pages/ImportPage')) }, { path: '/settings', lazy: page(() => import('@/pages/SettingsPage')) }, { path: '/admin',