feat(import): an Import page, with the numbers of every import kept

The report of an import used to be shown once in Settings and lost. Each
import is now stored (migration 0030, spotify_imports) as the report in
JSON, and GET /me/import/spotify lists the past ones, newest first.
POST /me/import/spotify answers with the same shape plus id and created_at.

The web app gets an Import page (menu, More tab) with the form and one card
for each past import: plays read, added, already there, not in the library,
the playlists it made and the artists that are missing. Settings only links
to it. An import that finds nothing is not kept, and a report that cannot be
stored is still returned to the person who asked.
This commit is contained in:
tgiachi 2026-11-15 10:34:12 +01:00
commit aa4883b955
11 changed files with 868 additions and 154 deletions

View file

@ -121,7 +121,7 @@ func newTestEnvDB(t *testing.T, mutate func(*Deps, *bun.DB)) (http.Handler, *bun
feedbackSvc := feedback.NewService(storage.NewFeedbackRepo(db))
deps.Feedback = feedbackSvc
deps.Playlists = playlists.NewService(storage.NewPlaylistRepo(db))
deps.SpotifyImport = spotifyimport.NewService(storage.NewCatalogRepo(db), storage.NewListeningRepo(db), deps.Playlists, storage.SearchKey, 20*time.Second)
deps.SpotifyImport = spotifyimport.NewService(storage.NewCatalogRepo(db), storage.NewListeningRepo(db), deps.Playlists, storage.NewSpotifyImportRepo(db), storage.SearchKey, 20*time.Second)
deps.Mixes = mixes.NewService(storage.NewMixRepo(db), deps.Taste.(mixes.Taster), 20*time.Second)
deps.Smart = smart.NewService(storage.NewSmartRepo(db), deps.Taste.(smart.Taster), 20*time.Second)
namingSvc := naming.NewService(prov, noClaude{}, storage.NewNamingRepo(db), nil)

View file

@ -870,6 +870,12 @@ func TestResponsesMatchTheSpec(t *testing.T) {
if h, _ := imported["history"].(map[string]any); h["entries"] != 1.0 || h["not_found"] != 1.0 {
c.t.Errorf("history = %v", h)
}
past := c.expect(c.call("GET", "/me/import/spotify", token, nil), 200, "the past imports")
if items, _ := past["items"].([]any); past["total"] != 1.0 || len(items) != 1 || items[0].(map[string]any)["id"] != imported["id"] {
c.t.Errorf("past imports = %v, want the one just made", past)
}
c.expect(c.call("GET", "/me/import/spotify?limit=-1", token, nil), 400, "a negative limit", "invalid_input")
c.expect(c.call("GET", "/me/import/spotify", "", nil), 401, "past imports without a token", "unauthorized")
c.expect(c.upload("/me/import/spotify", token), 400, "an import with no files", "invalid_input")
c.expect(c.upload("/me/import/spotify", token, `{"hello":"world"}`), 400, "files that are not Spotify's", "no_spotify_data")
c.expect(c.upload("/me/import/spotify", token, export[:len(export)-20]), 400, "a file cut short", "unreadable_file")

View file

@ -3,6 +3,7 @@ package httpapi
import (
"errors"
"net/http"
"time"
"github.com/gin-gonic/gin"
@ -44,6 +45,8 @@ type spotifyMissingJSON struct {
}
type spotifyReportJSON struct {
ID int64 `json:"id"`
CreatedAt time.Time `json:"created_at"`
History spotifyHistoryJSON `json:"history"`
Playlists []spotifyPlaylistJSON `json:"playlists"`
Missing struct {
@ -52,8 +55,10 @@ type spotifyReportJSON struct {
} `json:"missing"`
}
func toSpotifyReport(r spotifyimport.Report) spotifyReportJSON {
func toSpotifyReport(run spotifyimport.Run) spotifyReportJSON {
r := run.Report
out := spotifyReportJSON{
ID: run.ID, CreatedAt: run.At.UTC(),
History: spotifyHistoryJSON{
Entries: r.History.Entries, Matched: r.History.Matched, Imported: r.History.Imported,
AlreadyThere: r.History.AlreadyThere, NotFound: r.History.NotFound, Ignored: r.History.Ignored,
@ -99,10 +104,14 @@ func (h *spotifyImportHandlers) importExport(c *gin.Context) {
files = append(files, spotifyimport.File{R: f, Size: fh.Size})
}
report, err := h.svc.Import(c.Request.Context(), currentUserID(c), files)
run, err := h.svc.Import(c.Request.Context(), currentUserID(c), files)
switch {
case err == nil:
c.JSON(http.StatusOK, toSpotifyReport(report))
c.JSON(http.StatusOK, toSpotifyReport(run))
case errors.Is(err, spotifyimport.ErrNotKept):
// The listens are recorded, so say so; the report just will not be on the page of past imports.
_ = c.Error(err)
c.JSON(http.StatusOK, toSpotifyReport(run))
case errors.Is(err, spotifyimport.ErrNothingFound):
writeError(c, http.StatusBadRequest, "no_spotify_data", "these files hold no streaming history and no playlists of Spotify")
case errors.Is(err, spotifyimport.ErrUnreadable):
@ -116,3 +125,33 @@ func (h *spotifyImportHandlers) importExport(c *gin.Context) {
writeError(c, http.StatusInternalServerError, "internal", "internal error")
}
}
type spotifyImportPageJSON struct {
Items []spotifyReportJSON `json:"items"`
Total int `json:"total"`
Limit int `json:"limit"`
Offset int `json:"offset"`
}
// list gives the imports of the signed-in person that were kept, newest first.
func (h *spotifyImportHandlers) list(c *gin.Context) {
page, ok := pageOf(c)
if !ok {
return
}
got, err := h.svc.History(c.Request.Context(), currentUserID(c), page.Limit, page.Offset)
switch {
case errors.Is(err, spotifyimport.ErrInvalidPage):
writeError(c, http.StatusBadRequest, "invalid_input", err.Error())
return
case err != nil:
_ = c.Error(err)
writeError(c, http.StatusInternalServerError, "internal", "internal error")
return
}
out := spotifyImportPageJSON{Items: make([]spotifyReportJSON, 0, len(got.Items)), Total: got.Total, Limit: got.Limit, Offset: got.Offset}
for _, run := range got.Items {
out.Items = append(out.Items, toSpotifyReport(run))
}
c.JSON(http.StatusOK, out)
}

View file

@ -106,7 +106,8 @@ type playlistService interface {
// spotifyImportService is what the handlers need from spotifyimport.Service.
type spotifyImportService interface {
Import(ctx context.Context, userID int64, files []spotifyimport.File) (spotifyimport.Report, error)
Import(ctx context.Context, userID int64, files []spotifyimport.File) (spotifyimport.Run, error)
History(ctx context.Context, userID int64, limit, offset int) (spotifyimport.RunPage, error)
}
// mixService is what the handlers need from mixes.Service.
@ -361,7 +362,9 @@ func NewRouter(d Deps) *gin.Engine {
protected.GET("/me/settings", us.get)
protected.PATCH("/me/settings", us.update)
ma := &mastodonHandlers{svc: d.Mastodon}
protected.POST("/me/import/spotify", (&spotifyImportHandlers{svc: d.SpotifyImport}).importExport)
spotify := &spotifyImportHandlers{svc: d.SpotifyImport}
protected.POST("/me/import/spotify", spotify.importExport)
protected.GET("/me/import/spotify", spotify.list)
protected.GET("/me/mastodon", ma.get)
protected.PUT("/me/mastodon", ma.connect)
protected.DELETE("/me/mastodon", ma.disconnect)

View file

@ -78,6 +78,9 @@ func TestImportingAHistoryRecordsTheListensAndTheReportSaysWhatHappened(t *testi
if r.Code != http.StatusOK {
t.Fatalf("import: %d %v", r.Code, r.Body)
}
if r.Body["id"] != 1.0 || r.Body["created_at"] == nil {
t.Errorf("id and time = %v %v, want the import kept as 1", r.Body["id"], r.Body["created_at"])
}
hist, _ := r.Body["history"].(map[string]any)
sameKeys(t, "history", hist, "entries", "matched", "imported", "already_there", "not_found", "ignored")
if hist["entries"] != 2.0 || hist["matched"] != 1.0 || hist["imported"] != 1.0 || hist["not_found"] != 1.0 {
@ -189,3 +192,59 @@ func TestAnUploadLargerThanOtherRequestsIsAccepted(t *testing.T) {
t.Errorf("%d %v", r.Code, r.Body)
}
}
func TestPastImportsAreListedNewestFirstAndAreOnlyMine(t *testing.T) {
h, db := newTestEnv(t, func(*Deps) {})
seedSong(t, db, "a.flac", "Karma Police", "OK Computer", "Radiohead")
tok := registerAndLogin(t, h)
upload(t, h, tok, spotifyHistory)
upload(t, h, tok, spotifyHistory)
r := do(t, h, "GET", importPath, tok, nil)
if r.Code != http.StatusOK || r.Body["total"] != 2.0 || r.Body["limit"] != 20.0 || r.Body["offset"] != 0.0 {
t.Fatalf("list: %d %v", r.Code, r.Body)
}
items, _ := r.Body["items"].([]any)
if len(items) != 2 {
t.Fatalf("%d imports, want 2", len(items))
}
newest, oldest := items[0].(map[string]any), items[1].(map[string]any)
sameKeys(t, "import", newest, "id", "created_at", "history", "playlists", "missing")
if newest["id"] != 2.0 || oldest["id"] != 1.0 {
t.Errorf("order = %v, %v, want 2 then 1", newest["id"], oldest["id"])
}
if h, _ := oldest["history"].(map[string]any); h["imported"] != 1.0 {
t.Errorf("the first import = %v, want one listen recorded", h)
}
if h, _ := newest["history"].(map[string]any); h["already_there"] != 1.0 || h["imported"] != 0.0 {
t.Errorf("the second import = %v, want the play already there", h)
}
if m, _ := newest["missing"].(map[string]any); m["total"] != 1.0 {
t.Errorf("missing = %v, want the one artist", m)
}
if page := do(t, h, "GET", importPath+"?limit=1&offset=1", tok, nil); page.Code != http.StatusOK || len(page.Body["items"].([]any)) != 1 || page.Body["items"].([]any)[0].(map[string]any)["id"] != 1.0 {
t.Errorf("second page: %d %v", page.Code, page.Body)
}
if r := do(t, h, "GET", importPath+"?offset=-3", tok, nil); r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" {
t.Errorf("a negative offset: %d %v", r.Code, r.Body)
}
// Somebody else sees none of it.
_, other := addUser(t, h, tok, "other", false)
if r := do(t, h, "GET", importPath, other, nil); r.Code != http.StatusOK || r.Body["total"] != 0.0 || len(r.Body["items"].([]any)) != 0 {
t.Errorf("another person's list: %d %v, want empty", r.Code, r.Body)
}
}
func TestAnImportThatFindsNothingIsNotListed(t *testing.T) {
h := newTestAPI(t)
tok := registerAndLogin(t, h)
upload(t, h, tok, `{"hello":"world"}`)
if r := do(t, h, "GET", importPath, tok, nil); r.Body["total"] != 0.0 {
t.Errorf("list = %v, want nothing", r.Body)
}
if r := do(t, h, "GET", importPath, "", nil); r.Code != http.StatusUnauthorized {
t.Errorf("without a token: %d, want 401", r.Code)
}
}

View file

@ -2163,6 +2163,27 @@ paths:
'401': {$ref: '#/components/responses/Unauthorized'}
/me/import/spotify:
get:
tags: [Taste]
operationId: listMySpotifyImports
summary: Your past imports of Spotify data
description: |
What each import of yours did, newest first: how many plays were recorded, how many were already there,
how many are not in your library, the playlists it made and the artists you are missing. The files
themselves are never kept, only these numbers. Only yours. An import is listed once it is done; one that
found nothing is not.
security: [{bearerAuth: []}]
parameters:
- {$ref: '#/components/parameters/Limit'}
- {$ref: '#/components/parameters/Offset'}
responses:
'200':
description: A page of imports.
content:
application/json:
schema: {$ref: '#/components/schemas/SpotifyImportPage'}
'400': {$ref: '#/components/responses/InvalidInput'}
'401': {$ref: '#/components/responses/Unauthorized'}
post:
tags: [Taste]
operationId: importMySpotify
@ -2205,7 +2226,9 @@ paths:
description: The zip of the export, or its JSON files.
responses:
'200':
description: What the import did.
description: |
What the import did. It is also kept (`GET /me/import/spotify`); if keeping it failed the report is still
sent, with `id` 0.
content:
application/json:
schema: {$ref: '#/components/schemas/SpotifyImportReport'}
@ -5466,11 +5489,25 @@ components:
shared: another user can see from `retrieved_at` in `GET /artists/{id}/similar` that
somebody looked an artist up, but not who.
SpotifyImportPage:
type: object
additionalProperties: false
required: [items, total, limit, offset]
properties:
items:
type: array
items: {$ref: '#/components/schemas/SpotifyImportReport'}
total: {type: integer, description: "Size of the whole result, not of this page."}
limit: {type: integer}
offset: {type: integer}
SpotifyImportReport:
type: object
additionalProperties: false
required: [history, playlists, missing]
required: [id, created_at, history, playlists, missing]
properties:
id: {type: integer, description: 'The import. 0 when its report could not be kept.'}
created_at: {type: string, format: date-time, description: When the import was done.}
history:
type: object
additionalProperties: false

View file

@ -23,12 +23,20 @@ var (
ErrNothingFound = errors.New("no Spotify history or playlists in these files")
// ErrBusy means this person's import is still running.
ErrBusy = errors.New("an import is already running")
// ErrNotKept means the import was done but its report could not be stored; the Run that comes with it is complete.
ErrNotKept = errors.New("the report of the import could not be kept")
// ErrInvalidPage means a negative limit or offset was asked for.
ErrInvalidPage = errors.New("limit and offset cannot be negative")
)
const (
// MaxMissing is how many missing artists a report lists (the ones played most); MissingTotal says how many there are.
MaxMissing = 50
// DefaultPage and MaxPage are how many past imports a page holds when nothing is asked, and at the most.
DefaultPage = 20
MaxPage = 100
// listenBatch is how many listens are recorded in one go.
listenBatch = 2000
@ -69,36 +77,57 @@ type File struct {
// History says what became of the plays of the history files.
type History struct {
Entries int // plays of songs that were read
Matched int // of those, the ones that are a song of the library
Imported int // recorded now
AlreadyThere int // matched, and already recorded by an earlier import (or the same play in two files)
NotFound int // not in the library
Ignored int // podcast episodes, and plays with no time that can be read
Entries int `json:"entries"` // plays of songs that were read
Matched int `json:"matched"` // of those, the ones that are a song of the library
Imported int `json:"imported"` // recorded now
AlreadyThere int `json:"already_there"` // matched, and already recorded by an earlier import (or the same play in two files)
NotFound int `json:"not_found"` // not in the library
Ignored int `json:"ignored"` // podcast episodes, and plays with no time that can be read
}
// PlaylistResult says what became of one playlist. Status is `created`, `exists` (the person already has one with
// that name, which is left alone), `empty` (none of its songs is in the library) or `limit` (the person has as many
// playlists as are allowed).
type PlaylistResult struct {
Name string
Tracks int // everything that is in it
Matched int // the songs of the library
Status string
Name string `json:"name"`
Tracks int `json:"tracks"` // everything that is in it
Matched int `json:"matched"` // the songs of the library
Status string `json:"status"`
}
// Missing is an artist that has plays or playlist songs and none of them in the library.
type Missing struct {
Artist string
Count int
Artist string `json:"artist"`
Count int `json:"count"`
}
// Report is what an import did. It is made for the person who asked and kept nowhere.
// Report is what an import did. It is kept as JSON (the tags are the stored shape), so a name here is not changed
// lightly.
type Report struct {
History History
Playlists []PlaylistResult
Missing []Missing // the artists with the most songs that were not found, at most MaxMissing
MissingTotal int // how many artists have songs that were not found
History History `json:"history"`
Playlists []PlaylistResult `json:"playlists"`
Missing []Missing `json:"missing"` // the artists with the most songs that were not found, at most MaxMissing
MissingTotal int `json:"missing_total"` // how many artists have songs that were not found
}
// Run is a report that was kept: the import with its number and the time it was done.
type Run struct {
ID int64 // 0 when the report could not be kept
At time.Time
Report Report
}
// RunPage is a page of the imports of a person, newest first.
type RunPage struct {
Items []Run
Total int
Limit, Offset int
}
// Runs keeps the reports of the imports.
type Runs interface {
Save(ctx context.Context, userID int64, at time.Time, r Report) (id int64, err error)
List(ctx context.Context, userID int64, limit, offset int) ([]Run, int, error)
}
// Service imports the data of a Spotify export for one person at a time.
@ -106,6 +135,7 @@ type Service struct {
catalog Catalog
listens Listens
lists Playlists
runs Runs
key func(string) string
minListen time.Duration
running sync.Map // user id -> struct{}
@ -113,22 +143,40 @@ type Service struct {
// NewService builds a Service. key is how names are made comparable (the key of the library search); minListen is how
// long a play must last to count as a listen, so that a play shorter than that is recorded as a skip.
func NewService(catalog Catalog, listens Listens, lists Playlists, key func(string) string, minListen time.Duration) *Service {
return &Service{catalog: catalog, listens: listens, lists: lists, key: key, minListen: minListen}
func NewService(catalog Catalog, listens Listens, lists Playlists, runs Runs, key func(string) string, minListen time.Duration) *Service {
return &Service{catalog: catalog, listens: listens, lists: lists, runs: runs, key: key, minListen: minListen}
}
// History lists the imports of a person that were kept, newest first. A limit of 0 is DefaultPage, and one above
// MaxPage is lowered to it.
func (s *Service) History(ctx context.Context, userID int64, limit, offset int) (RunPage, error) {
if limit < 0 || offset < 0 {
return RunPage{}, ErrInvalidPage
}
if limit == 0 {
limit = DefaultPage
}
limit = min(limit, MaxPage)
items, total, err := s.runs.List(ctx, userID, limit, offset)
if err != nil {
return RunPage{}, fmt.Errorf("list the imports: %w", err)
}
return RunPage{Items: items, Total: total, Limit: limit, Offset: offset}, nil
}
// Import reads the files, records the listens that match the library and makes the playlists. A person can run one
// import at a time (ErrBusy). It is safe to import the same files again: nothing is recorded twice. What was
// recorded before an error stays recorded.
func (s *Service) Import(ctx context.Context, userID int64, files []File) (Report, error) {
// recorded before an error stays recorded. The report is kept for the person to read again; when that fails the
// Run is still returned, with ID 0, together with ErrNotKept.
func (s *Service) Import(ctx context.Context, userID int64, files []File) (Run, error) {
if _, busy := s.running.LoadOrStore(userID, struct{}{}); busy {
return Report{}, ErrBusy
return Run{}, ErrBusy
}
defer s.running.Delete(userID)
songs, err := s.catalog.Candidates(ctx)
if err != nil {
return Report{}, fmt.Errorf("read the library: %w", err)
return Run{}, fmt.Errorf("read the library: %w", err)
}
run := &run{svc: s, ctx: ctx, userID: userID, index: NewIndex(s.key, songs), memo: map[string]int64{}, missing: map[string]*Missing{}}
@ -137,23 +185,29 @@ func (s *Service) Import(ctx context.Context, userID int64, files []File) (Repor
for _, f := range files {
ok, err := reader.ReadFile(f.R, f.Size)
if err != nil {
return Report{}, err
return Run{}, err
}
found = found || ok
if run.err != nil {
return Report{}, run.err
return Run{}, run.err
}
}
if !found {
return Report{}, ErrNothingFound
return Run{}, ErrNothingFound
}
if err := run.flush(); err != nil {
return Report{}, err
return Run{}, err
}
if err := run.makePlaylists(); err != nil {
return Report{}, err
return Run{}, err
}
return run.finish(), nil
rep := run.finish()
at := time.Now()
id, err := s.runs.Save(ctx, userID, at, rep)
if err != nil {
return Run{At: at, Report: rep}, fmt.Errorf("%w: %v", ErrNotKept, err)
}
return Run{ID: id, At: at, Report: rep}, nil
}
// run is the state of one import.

View file

@ -101,15 +101,47 @@ var songs = []Candidate{
{TrackID: 3, Title: "Hyperballad", Album: "Post", Artists: []string{"Björk"}, DurationMs: 321_000},
}
// fakeRuns keeps the reports in memory, newest first when listed.
type fakeRuns struct {
saved []Run
fail error
}
func (f *fakeRuns) Save(_ context.Context, _ int64, at time.Time, r Report) (int64, error) {
if f.fail != nil {
return 0, f.fail
}
id := int64(len(f.saved) + 1)
f.saved = append(f.saved, Run{ID: id, At: at, Report: r})
return id, nil
}
func (f *fakeRuns) List(_ context.Context, _ int64, limit, offset int) ([]Run, int, error) {
var all []Run
for i := len(f.saved) - 1; i >= 0; i-- {
all = append(all, f.saved[i])
}
end := min(offset+limit, len(all))
start := min(offset, end)
return all[start:end], len(all), nil
}
type rig struct {
svc *Service
li *fakeListens
pl *fakePlaylists
svc *Service
li *fakeListens
pl *fakePlaylists
runs *fakeRuns
}
func newRig(cat *fakeCatalog) rig {
li, pl := &fakeListens{}, &fakePlaylists{}
return rig{NewService(cat, li, pl, key, 20*time.Second), li, pl}
li, pl, runs := &fakeListens{}, &fakePlaylists{}, &fakeRuns{}
return rig{NewService(cat, li, pl, runs, key, 20*time.Second), li, pl, runs}
}
// report runs an import and gives the report of it.
func (r rig) report(userID int64, files []File) (Report, error) {
run, err := r.svc.Import(context.Background(), userID, files)
return run.Report, err
}
func upload(data string) File { return File{R: bytes.NewReader([]byte(data)), Size: int64(len(data))} }
@ -123,7 +155,7 @@ func history(plays ...string) string { return "[" + strings.Join(plays, ",") + "
func TestImportRecordsTheListensThatMatchTheLibrary(t *testing.T) {
r := newRig(&fakeCatalog{songs: songs})
rep, err := r.svc.Import(context.Background(), 7, []File{upload(history(
rep, err := r.report(7, []File{upload(history(
play("2023-05-01T10:00:00Z", "Radiohead", "Karma Police", "OK Computer", 260_000, `,"reason_end":"trackdone"`),
play("2023-05-01T10:05:00Z", "Radiohead", "Airbag", "OK Computer", 4_000, `,"reason_end":"fwdbtn","skipped":true`),
play("2023-05-01T10:06:00Z", "Radiohead", "Creep", "Pablo Honey", 200_000, ``),
@ -155,7 +187,7 @@ func TestImportRecordsTheListensThatMatchTheLibrary(t *testing.T) {
func TestTheKindOfAPlayComesFromHowItEndedOrHowMuchWasHeard(t *testing.T) {
r := newRig(&fakeCatalog{songs: songs})
_, err := r.svc.Import(context.Background(), 1, []File{upload(history(
_, err := r.report(1, []File{upload(history(
play("2023-01-01T10:00:00Z", "Radiohead", "Karma Police", "", 260_000, `,"reason_end":"trackdone"`), // ran to the end
play("2023-01-01T11:00:00Z", "Radiohead", "Karma Police", "", 250_000, `,"reason_end":"endplay"`), // 96 per cent, then stopped
play("2023-01-01T12:00:00Z", "Radiohead", "Karma Police", "", 90_000, `,"reason_end":"endplay"`), // a good part, then stopped
@ -184,10 +216,10 @@ func TestImportingTheSameFilesAgainAddsNothing(t *testing.T) {
play("2023-05-01T10:00:00Z", "Radiohead", "Karma Police", "OK Computer", 260_000, ``),
play("2023-05-01T10:10:00Z", "Björk", "Hyperballad", "Post", 321_000, ``),
)
if _, err := r.svc.Import(context.Background(), 1, []File{upload(data)}); err != nil {
if _, err := r.report(1, []File{upload(data)}); err != nil {
t.Fatal(err)
}
rep, err := r.svc.Import(context.Background(), 1, []File{upload(data)})
rep, err := r.report(1, []File{upload(data)})
if err != nil {
t.Fatal(err)
}
@ -200,7 +232,7 @@ func TestTheSamePlayInTheTwoShapesOfFileIsCountedOnce(t *testing.T) {
r := newRig(&fakeCatalog{songs: songs})
extended := history(play("2022-01-02T08:30:41Z", "Björk", "Hyperballad", "Post", 321_000, `,"reason_end":"trackdone"`))
older := `[{"endTime":"2022-01-02 08:30","artistName":"Björk","trackName":"Hyperballad","msPlayed":321000}]`
rep, err := r.svc.Import(context.Background(), 1, []File{upload(extended), upload(older)})
rep, err := r.report(1, []File{upload(extended), upload(older)})
if err != nil {
t.Fatal(err)
}
@ -212,7 +244,7 @@ func TestTheSamePlayInTheTwoShapesOfFileIsCountedOnce(t *testing.T) {
func TestPlaylistsBecomePlaylistsWithTheSongsThatWereFound(t *testing.T) {
r := newRig(&fakeCatalog{songs: songs})
r.pl.existing = []string{"Mine already"}
rep, err := r.svc.Import(context.Background(), 1, []File{upload(`{"playlists":[
rep, err := r.report(1, []File{upload(`{"playlists":[
{"name":"Road","items":[
{"track":{"trackName":"Hyperballad","artistName":"Björk","albumName":"Post"}},
{"track":{"trackName":"Creep","artistName":"Radiohead","albumName":"Pablo Honey"}},
@ -257,7 +289,7 @@ func TestALongPlaylistIsAddedInPieces(t *testing.T) {
for i := 0; i < 1234; i++ {
items = append(items, `{"track":{"trackName":"Airbag","artistName":"Radiohead"}}`)
}
rep, err := r.svc.Import(context.Background(), 1, []File{upload(`{"playlists":[{"name":"Long","items":[` + strings.Join(items, ",") + `]}]}`)})
rep, err := r.report(1, []File{upload(`{"playlists":[{"name":"Long","items":[` + strings.Join(items, ",") + `]}]}`)})
if err != nil {
t.Fatal(err)
}
@ -270,7 +302,7 @@ func TestWhenThePersonHasAsManyPlaylistsAsAllowedTheRestAreSaidSo(t *testing.T)
r := newRig(&fakeCatalog{songs: songs})
r.pl.max = 1
item := `{"track":{"trackName":"Airbag","artistName":"Radiohead"}}`
rep, err := r.svc.Import(context.Background(), 1, []File{upload(`{"playlists":[{"name":"A","items":[` + item + `]},{"name":"B","items":[` + item + `]},{"name":"C","items":[` + item + `]}]}`)})
rep, err := r.report(1, []File{upload(`{"playlists":[{"name":"A","items":[` + item + `]},{"name":"B","items":[` + item + `]},{"name":"C","items":[` + item + `]}]}`)})
if err != nil {
t.Fatal(err)
}
@ -290,7 +322,7 @@ func TestTheArtistsThatAreMissingAreListedMostPlayedFirst(t *testing.T) {
plays = append(plays, play(fmt.Sprintf("2023-01-02T10:0%d:00Z", i), "oasis", "Wonderwall", "", 120_000, ``))
plays = append(plays, play(fmt.Sprintf("2023-01-03T10:0%d:00Z", i), "Oasis", "Supersonic", "", 120_000, ``))
}
rep, err := r.svc.Import(context.Background(), 1, []File{upload(history(plays...))})
rep, err := r.report(1, []File{upload(history(plays...))})
if err != nil {
t.Fatal(err)
}
@ -305,7 +337,7 @@ func TestOnlyTheMostPlayedMissingArtistsAreListed(t *testing.T) {
for i := 0; i < MaxMissing+30; i++ {
plays = append(plays, play("2023-01-01T10:00:00Z", fmt.Sprintf("Artist %03d", i), "Song", "", 120_000, ``))
}
rep, err := r.svc.Import(context.Background(), 1, []File{upload(history(plays...))})
rep, err := r.report(1, []File{upload(history(plays...))})
if err != nil {
t.Fatal(err)
}
@ -316,7 +348,7 @@ func TestOnlyTheMostPlayedMissingArtistsAreListed(t *testing.T) {
func TestFilesWithNothingOfSpotifyAreRefused(t *testing.T) {
r := newRig(&fakeCatalog{songs: songs})
_, err := r.svc.Import(context.Background(), 1, []File{upload(`{"hello":"world"}`), upload("words")})
_, err := r.report(1, []File{upload(`{"hello":"world"}`), upload("words")})
if !errors.Is(err, ErrNothingFound) {
t.Errorf("err = %v, want ErrNothingFound", err)
}
@ -330,11 +362,11 @@ func TestAnImportThatIsRunningBlocksAnotherOfTheSamePerson(t *testing.T) {
r := newRig(cat)
done := make(chan error, 1)
go func() {
_, err := r.svc.Import(context.Background(), 5, []File{upload(history(play("2023-01-01T10:00:00Z", "Radiohead", "Airbag", "", 1000, ``)))})
_, err := r.report(5, []File{upload(history(play("2023-01-01T10:00:00Z", "Radiohead", "Airbag", "", 1000, ``)))})
done <- err
}()
<-cat.entered
if _, err := r.svc.Import(context.Background(), 5, []File{upload(`[]`)}); !errors.Is(err, ErrBusy) {
if _, err := r.report(5, []File{upload(`[]`)}); !errors.Is(err, ErrBusy) {
t.Errorf("a second import of the same person: %v, want ErrBusy", err)
}
close(cat.release)
@ -342,7 +374,7 @@ func TestAnImportThatIsRunningBlocksAnotherOfTheSamePerson(t *testing.T) {
t.Fatal(err)
}
// Once it is over, the person can import again, and somebody else could all along.
if _, err := r.svc.Import(context.Background(), 5, []File{upload(`[]`)}); errors.Is(err, ErrBusy) {
if _, err := r.report(5, []File{upload(`[]`)}); errors.Is(err, ErrBusy) {
t.Error("still busy after the import ended")
}
}
@ -353,7 +385,7 @@ func TestManyListensAreRecordedInBatches(t *testing.T) {
for i := 0; i < 2500; i++ {
plays = append(plays, play(time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC).Add(time.Duration(i)*time.Minute).Format(time.RFC3339), "Radiohead", "Airbag", "", 100_000, ``))
}
rep, err := r.svc.Import(context.Background(), 1, []File{upload(history(plays...))})
rep, err := r.report(1, []File{upload(history(plays...))})
if err != nil {
t.Fatal(err)
}
@ -361,3 +393,64 @@ func TestManyListensAreRecordedInBatches(t *testing.T) {
t.Errorf("imported %d in %d calls, want 2500 in several", rep.History.Imported, r.li.calls)
}
}
func TestAnImportIsKeptAndListedNewestFirst(t *testing.T) {
r := newRig(&fakeCatalog{songs: songs})
first, err := r.svc.Import(context.Background(), 1, []File{upload(history(play("2023-01-01T10:00:00Z", "Radiohead", "Airbag", "", 280_000, `,"reason_end":"trackdone"`)))})
if err != nil || first.ID != 1 || first.At.IsZero() || first.Report.History.Imported != 1 {
t.Fatalf("first import = %+v, %v, want it kept as 1 with one listen", first, err)
}
second, err := r.svc.Import(context.Background(), 1, []File{upload(history(play("2023-01-01T10:00:00Z", "Radiohead", "Airbag", "", 280_000, `,"reason_end":"trackdone"`)))})
if err != nil || second.ID != 2 || second.Report.History.AlreadyThere != 1 {
t.Fatalf("second import = %+v, %v, want 2 with the play already there", second, err)
}
page, err := r.svc.History(context.Background(), 1, 10, 0)
if err != nil || page.Total != 2 || len(page.Items) != 2 || page.Items[0].ID != 2 || page.Items[1].ID != 1 {
t.Fatalf("History = %+v, %v, want the two imports, newest first", page, err)
}
if page.Limit != 10 || page.Offset != 0 {
t.Errorf("page = %d/%d, want 10/0", page.Limit, page.Offset)
}
}
func TestAnImportThatFindsNothingIsNotKept(t *testing.T) {
r := newRig(&fakeCatalog{songs: songs})
if _, err := r.report(1, []File{upload(`{"hello":"world"}`)}); !errors.Is(err, ErrNothingFound) {
t.Fatalf("err = %v, want ErrNothingFound", err)
}
if len(r.runs.saved) != 0 {
t.Errorf("%d reports kept, want none", len(r.runs.saved))
}
}
func TestAReportThatCannotBeKeptStillComesBack(t *testing.T) {
r := newRig(&fakeCatalog{songs: songs})
r.runs.fail = errors.New("disk full")
run, err := r.svc.Import(context.Background(), 1, []File{upload(history(play("2023-01-01T10:00:00Z", "Radiohead", "Airbag", "", 280_000, `,"reason_end":"trackdone"`)))})
if !errors.Is(err, ErrNotKept) {
t.Fatalf("err = %v, want ErrNotKept", err)
}
if run.ID != 0 || run.Report.History.Imported != 1 || run.At.IsZero() {
t.Errorf("run = %+v, want the report with ID 0", run)
}
}
func TestHistoryPagesAreBounded(t *testing.T) {
r := newRig(&fakeCatalog{songs: songs})
for i := 0; i < 3; i++ {
r.runs.saved = append(r.runs.saved, Run{ID: int64(i + 1)})
}
page, err := r.svc.History(context.Background(), 1, 0, 0)
if err != nil || page.Limit != DefaultPage || len(page.Items) != 3 {
t.Errorf("no limit = %+v, %v, want the default page", page, err)
}
if page, _ := r.svc.History(context.Background(), 1, 100000, 0); page.Limit != MaxPage {
t.Errorf("a huge limit = %d, want %d", page.Limit, MaxPage)
}
if _, err := r.svc.History(context.Background(), 1, -1, 0); !errors.Is(err, ErrInvalidPage) {
t.Errorf("a negative limit: %v, want ErrInvalidPage", err)
}
if _, err := r.svc.History(context.Background(), 1, 10, -1); !errors.Is(err, ErrInvalidPage) {
t.Errorf("a negative offset: %v, want ErrInvalidPage", err)
}
}

View file

@ -3,6 +3,7 @@ package storage
import (
"context"
"fmt"
"reflect"
"sort"
"testing"
"time"
@ -11,6 +12,7 @@ import (
"codeberg.org/squidd/sonda/internal/library"
"codeberg.org/squidd/sonda/internal/listening"
"codeberg.org/squidd/sonda/internal/spotifyimport"
)
func TestRecordManyStoresTheListensOnceAndCountsWhatIsNew(t *testing.T) {
@ -114,3 +116,64 @@ func TestSearchKeyIsExported(t *testing.T) {
t.Errorf("SearchKey(Björk) = %q", SearchKey("Björk"))
}
}
func TestSpotifyImportsAreKeptPerUserNewestFirst(t *testing.T) {
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
ctx := context.Background()
users := NewUserRepo(db)
alice, err := users.CreateUser(ctx, "alice", "h", false)
if err != nil {
t.Fatal(err)
}
bob, err := users.CreateUser(ctx, "bob", "h", false)
if err != nil {
t.Fatal(err)
}
repo := NewSpotifyImportRepo(db)
rep := spotifyimport.Report{
History: spotifyimport.History{Entries: 10, Matched: 4, Imported: 3, AlreadyThere: 1, NotFound: 6, Ignored: 2},
Playlists: []spotifyimport.PlaylistResult{{Name: "Road trip", Tracks: 3, Matched: 1, Status: "created"}},
Missing: []spotifyimport.Missing{{Artist: "Blur", Count: 4}},
MissingTotal: 7,
}
first, err := repo.Save(ctx, alice.ID, time.Unix(1000, 0), rep)
if err != nil || first == 0 {
t.Fatalf("Save = %d, %v", first, err)
}
later := rep
later.History.Imported = 9
second, err := repo.Save(ctx, alice.ID, time.Unix(2000, 0), later)
if err != nil || second == first {
t.Fatalf("second Save = %d, %v", second, err)
}
if _, err := repo.Save(ctx, bob.ID, time.Unix(3000, 0), rep); err != nil {
t.Fatal(err)
}
runs, total, err := repo.List(ctx, alice.ID, 10, 0)
if err != nil || total != 2 || len(runs) != 2 {
t.Fatalf("List = %d runs, total %d, %v, want her two", len(runs), total, err)
}
if runs[0].ID != second || runs[0].Report.History.Imported != 9 || runs[0].At.Unix() != 2000 {
t.Errorf("newest = %+v, want the second import", runs[0])
}
if !reflect.DeepEqual(runs[1].Report, rep) || runs[1].At.Unix() != 1000 {
t.Errorf("oldest = %+v, want the report as it was saved", runs[1])
}
page, total, err := repo.List(ctx, alice.ID, 1, 1)
if err != nil || total != 2 || len(page) != 1 || page[0].ID != first {
t.Errorf("second page = %+v, total %d, %v, want the first import", page, total, err)
}
none, total, err := repo.List(ctx, alice.ID, 10, 5)
if err != nil || total != 2 || len(none) != 0 {
t.Errorf("past the end = %d runs, total %d, %v", len(none), total, err)
}
if _, err := db.ExecContext(ctx, "DELETE FROM users WHERE id = ?", alice.ID); err != nil {
t.Fatal(err)
}
if runs, total, _ := repo.List(ctx, alice.ID, 10, 0); total != 0 || len(runs) != 0 {
t.Errorf("after the account is gone: %d runs, total %d, want none", len(runs), total)
}
})
}

View file

@ -0,0 +1,59 @@
package storage
import (
"context"
"encoding/json"
"fmt"
"time"
"github.com/uptrace/bun"
"codeberg.org/squidd/sonda/internal/spotifyimport"
)
type spotifyImportRow struct {
bun.BaseModel `bun:"table:spotify_imports"`
ID int64 `bun:"id,pk,autoincrement"`
UserID int64 `bun:"user_id,notnull"`
CreatedAt int64 `bun:"created_at,notnull"`
Report string `bun:"report,notnull"`
}
// SpotifyImportRepo keeps the reports of the imports of Spotify exports. It implements spotifyimport.Runs.
type SpotifyImportRepo struct{ db *bun.DB }
// NewSpotifyImportRepo builds a SpotifyImportRepo.
func NewSpotifyImportRepo(db *bun.DB) *SpotifyImportRepo { return &SpotifyImportRepo{db: db} }
// Save keeps a report and gives its id.
func (r *SpotifyImportRepo) Save(ctx context.Context, userID int64, at time.Time, rep spotifyimport.Report) (int64, error) {
body, err := json.Marshal(rep)
if err != nil {
return 0, fmt.Errorf("encode the report: %w", err)
}
row := &spotifyImportRow{UserID: userID, CreatedAt: at.Unix(), Report: string(body)}
if _, err := r.db.NewInsert().Model(row).Exec(ctx); err != nil {
return 0, fmt.Errorf("insert spotify import: %w", err)
}
return row.ID, nil
}
// List gives one page of the reports of a person, newest first, and how many there are.
func (r *SpotifyImportRepo) List(ctx context.Context, userID int64, limit, offset int) ([]spotifyimport.Run, int, error) {
var rows []spotifyImportRow
total, err := r.db.NewSelect().Model(&rows).Where("user_id = ?", userID).
OrderExpr("created_at DESC, id DESC").Limit(int64(limit)).Offset(int64(offset)).ScanAndCount(ctx)
if err != nil {
return nil, 0, fmt.Errorf("list spotify imports: %w", err)
}
runs := make([]spotifyimport.Run, 0, len(rows))
for _, row := range rows {
run := spotifyimport.Run{ID: row.ID, At: time.Unix(row.CreatedAt, 0)}
if err := json.Unmarshal([]byte(row.Report), &run.Report); err != nil {
return nil, 0, fmt.Errorf("decode spotify import %d: %w", row.ID, err)
}
runs = append(runs, run)
}
return runs, int(total), nil
}

View file

@ -156,8 +156,8 @@ func TestMigrateCreatesSchemaAndIsIdempotent(t *testing.T) {
if err := db.QueryRow("SELECT COUNT(*) FROM schema_migrations").Scan(&applied); err != nil {
t.Fatal(err)
}
if applied != 29 {
t.Errorf("applied migrations = %d, want 29", applied)
if applied != 30 {
t.Errorf("applied migrations = %d, want 30", applied)
}
})
}