feat(import): bring my Spotify history and playlists in

I want my listening history and my playlists from Spotify in Sonda, so that Mixes, Discover and the smart playlists know what I listened to before.

- POST /me/import/spotify takes the zip of a Spotify data export, or the JSON files in it (extended streaming history, the older StreamingHistory files, the playlists), as a multipart form. A request may be up to 512 MB; the zip is limited in how much it may inflate and how many files it may hold, and only the history and playlist files are opened.
- Everything is read on the server and nothing is sent anywhere. The files are not kept, and of each play only the song, artist, album, time and how it ended are read: never the IP addresses, the country or the platform.
- A play is matched to a song of the library by artist and title (no case, accents or punctuation; a "Remastered" note on either side; any credited artist; a compilation by its album). A match is recorded as a listen at the time of the play: complete, skip or play. Nothing is posted to Mastodon.
- Importing the same files again records nothing twice, and the same play in the two shapes of history file is counted once. One import per person at a time.
- A playlist becomes a playlist of mine with the songs that were found; one with a name I already have is left alone, and one with none of its songs found is not made.
- The report counts what was added, what was already there and what is not in the library, and lists the missing artists, most played first.
- Settings gets an Import from Spotify section; the API client can send files.
- English and Italian.
This commit is contained in:
tgiachi 2026-11-14 16:25:58 +01:00
commit c08a234159
17 changed files with 2605 additions and 7 deletions

View file

@ -34,6 +34,7 @@ import (
"codeberg.org/squidd/sonda/internal/providers"
"codeberg.org/squidd/sonda/internal/providers/anthropic"
"codeberg.org/squidd/sonda/internal/smart"
"codeberg.org/squidd/sonda/internal/spotifyimport"
"codeberg.org/squidd/sonda/internal/storage"
"codeberg.org/squidd/sonda/internal/streaming"
"codeberg.org/squidd/sonda/internal/taste"
@ -120,6 +121,7 @@ func newTestEnvDB(t *testing.T, mutate func(*Deps, *bun.DB)) (http.Handler, *bun
feedbackSvc := feedback.NewService(storage.NewFeedbackRepo(db))
deps.Feedback = feedbackSvc
deps.Playlists = playlists.NewService(storage.NewPlaylistRepo(db))
deps.SpotifyImport = spotifyimport.NewService(storage.NewCatalogRepo(db), storage.NewListeningRepo(db), deps.Playlists, storage.SearchKey, 20*time.Second)
deps.Mixes = mixes.NewService(storage.NewMixRepo(db), deps.Taste.(mixes.Taster), 20*time.Second)
deps.Smart = smart.NewService(storage.NewSmartRepo(db), deps.Taste.(smart.Taster), 20*time.Second)
namingSvc := naming.NewService(prov, noClaude{}, storage.NewNamingRepo(db), nil)

View file

@ -7,11 +7,13 @@ import (
"crypto/rand"
"encoding/base64"
"encoding/json"
"fmt"
"image"
"image/color"
"image/png"
"io"
"mime"
"mime/multipart"
"net/http"
"net/http/httptest"
"os"
@ -191,15 +193,38 @@ func newContract(t *testing.T) *contract {
func (c *contract) call(method, path, token string, body any, headers ...string) *httptest.ResponseRecorder {
c.t.Helper()
var payload []byte
contentType := ""
if body != nil {
var err error
if payload, err = json.Marshal(body); err != nil {
c.t.Fatal(err)
}
contentType = "application/json"
}
return c.callRaw(method, path, token, contentType, payload, headers...)
}
// upload sends files as a multipart form, in the field `files`.
func (c *contract) upload(path, token string, files ...string) *httptest.ResponseRecorder {
c.t.Helper()
var buf bytes.Buffer
w := multipart.NewWriter(&buf)
for i, content := range files {
part, err := w.CreateFormFile("files", fmt.Sprintf("file%d.json", i))
if err != nil {
c.t.Fatal(err)
}
_, _ = part.Write([]byte(content))
}
_ = w.Close()
return c.callRaw("POST", path, token, w.FormDataContentType(), buf.Bytes())
}
func (c *contract) callRaw(method, path, token, contentType string, payload []byte, headers ...string) *httptest.ResponseRecorder {
c.t.Helper()
req := httptest.NewRequest(method, "/api/v1"+path, bytes.NewReader(payload))
if body != nil {
req.Header.Set("Content-Type", "application/json")
if contentType != "" {
req.Header.Set("Content-Type", contentType)
}
if token != "" {
req.Header.Set("Authorization", "Bearer "+token)
@ -839,6 +864,17 @@ func TestResponsesMatchTheSpec(t *testing.T) {
c.expect(c.call("DELETE", "/me/mastodon", token, nil), 204, "disconnect Mastodon")
c.expect(c.call("GET", "/me/mastodon", "", nil), 401, "Mastodon without a token", "unauthorized")
// Bringing a Spotify export in: what it did, and what is refused.
export := `[{"ts":"2023-05-01T10:00:00Z","ms_played":200000,"master_metadata_track_name":"Nowhere","master_metadata_album_artist_name":"Nobody"}]`
imported := c.expect(c.upload("/me/import/spotify", token, export, `{"playlists":[{"name":"P","items":[{"track":{"trackName":"Nowhere","artistName":"Nobody"}}]}]}`), 200, "import a Spotify export")
if h, _ := imported["history"].(map[string]any); h["entries"] != 1.0 || h["not_found"] != 1.0 {
c.t.Errorf("history = %v", h)
}
c.expect(c.upload("/me/import/spotify", token), 400, "an import with no files", "invalid_input")
c.expect(c.upload("/me/import/spotify", token, `{"hello":"world"}`), 400, "files that are not Spotify's", "no_spotify_data")
c.expect(c.upload("/me/import/spotify", token, export[:len(export)-20]), 400, "a file cut short", "unreadable_file")
c.expect(c.upload("/me/import/spotify", ""), 401, "an import without a token", "unauthorized")
// ---- duplicate songs ----
c.duplicates(t, token, libID)

View file

@ -0,0 +1,118 @@
package httpapi
import (
"errors"
"net/http"
"github.com/gin-gonic/gin"
"codeberg.org/squidd/sonda/internal/spotifyimport"
)
// spotifyImportPath is the route of the import of a Spotify export.
const spotifyImportPath = "/api/v1/me/import/spotify"
// importMaxBytes bounds the upload of an export (the zip, or its JSON files), which is far larger than any other
// request: ten years of listening are a few hundred megabytes. It is a variable so that a test can lower it.
var importMaxBytes int64 = 512 << 20
// importFormMemory is how much of an upload is kept in memory while it is read; the rest goes to a temporary file.
const importFormMemory = 32 << 20
// spotifyImportHandlers import the Spotify export of the signed-in person into their own listens and playlists.
type spotifyImportHandlers struct{ svc spotifyImportService }
type spotifyHistoryJSON struct {
Entries int `json:"entries"`
Matched int `json:"matched"`
Imported int `json:"imported"`
AlreadyThere int `json:"already_there"`
NotFound int `json:"not_found"`
Ignored int `json:"ignored"`
}
type spotifyPlaylistJSON struct {
Name string `json:"name"`
Tracks int `json:"tracks"`
Matched int `json:"matched"`
Status string `json:"status"`
}
type spotifyMissingJSON struct {
Name string `json:"name"`
Count int `json:"count"`
}
type spotifyReportJSON struct {
History spotifyHistoryJSON `json:"history"`
Playlists []spotifyPlaylistJSON `json:"playlists"`
Missing struct {
Total int `json:"total"`
Artists []spotifyMissingJSON `json:"artists"`
} `json:"missing"`
}
func toSpotifyReport(r spotifyimport.Report) spotifyReportJSON {
out := spotifyReportJSON{
History: spotifyHistoryJSON{
Entries: r.History.Entries, Matched: r.History.Matched, Imported: r.History.Imported,
AlreadyThere: r.History.AlreadyThere, NotFound: r.History.NotFound, Ignored: r.History.Ignored,
},
Playlists: make([]spotifyPlaylistJSON, 0, len(r.Playlists)),
}
for _, p := range r.Playlists {
out.Playlists = append(out.Playlists, spotifyPlaylistJSON{Name: p.Name, Tracks: p.Tracks, Matched: p.Matched, Status: p.Status})
}
out.Missing.Total = r.MissingTotal
out.Missing.Artists = make([]spotifyMissingJSON, 0, len(r.Missing))
for _, m := range r.Missing {
out.Missing.Artists = append(out.Missing.Artists, spotifyMissingJSON{Name: m.Artist, Count: m.Count})
}
return out
}
func (h *spotifyImportHandlers) importExport(c *gin.Context) {
if err := c.Request.ParseMultipartForm(importFormMemory); err != nil {
var tooBig *http.MaxBytesError
if errors.As(err, &tooBig) {
writeError(c, http.StatusRequestEntityTooLarge, "too_large", "these files are too large to import")
return
}
writeError(c, http.StatusBadRequest, "invalid_input", "send the files of the export as a multipart form, in the field files")
return
}
defer func() { _ = c.Request.MultipartForm.RemoveAll() }()
headers := c.Request.MultipartForm.File["files"]
if len(headers) == 0 {
writeError(c, http.StatusBadRequest, "invalid_input", "send the files of the export as a multipart form, in the field files")
return
}
files := make([]spotifyimport.File, 0, len(headers))
for _, fh := range headers {
f, err := fh.Open()
if err != nil {
_ = c.Error(err)
writeError(c, http.StatusInternalServerError, "internal", "internal error")
return
}
defer f.Close()
files = append(files, spotifyimport.File{R: f, Size: fh.Size})
}
report, err := h.svc.Import(c.Request.Context(), currentUserID(c), files)
switch {
case err == nil:
c.JSON(http.StatusOK, toSpotifyReport(report))
case errors.Is(err, spotifyimport.ErrNothingFound):
writeError(c, http.StatusBadRequest, "no_spotify_data", "these files hold no streaming history and no playlists of Spotify")
case errors.Is(err, spotifyimport.ErrUnreadable):
writeError(c, http.StatusBadRequest, "unreadable_file", "a file is damaged or cut short")
case errors.Is(err, spotifyimport.ErrTooBig):
writeError(c, http.StatusRequestEntityTooLarge, "too_large", "these files hold more than can be imported")
case errors.Is(err, spotifyimport.ErrBusy):
writeError(c, http.StatusConflict, "import_running", "an import of yours is still running")
default:
_ = c.Error(err)
writeError(c, http.StatusInternalServerError, "internal", "internal error")
}
}

View file

@ -19,10 +19,14 @@ const (
ctxAccount = "account" // the auth.User that requireAuth already looked up
)
// limitBody caps the request body size.
// limitBody caps the request body size. The import of a Spotify export is the one request that may be far larger.
func limitBody(max int64) gin.HandlerFunc {
return func(c *gin.Context) {
c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, max)
limit := max
if c.FullPath() == spotifyImportPath {
limit = importMaxBytes
}
c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, limit)
c.Next()
}
}

View file

@ -30,6 +30,7 @@ import (
"codeberg.org/squidd/sonda/internal/playlists"
"codeberg.org/squidd/sonda/internal/providers"
"codeberg.org/squidd/sonda/internal/smart"
"codeberg.org/squidd/sonda/internal/spotifyimport"
"codeberg.org/squidd/sonda/internal/streaming"
"codeberg.org/squidd/sonda/internal/taste"
"codeberg.org/squidd/sonda/internal/usersettings"
@ -103,6 +104,11 @@ type playlistService interface {
MoveItem(ctx context.Context, userID, id, itemID int64, position int) error
}
// spotifyImportService is what the handlers need from spotifyimport.Service.
type spotifyImportService interface {
Import(ctx context.Context, userID int64, files []spotifyimport.File) (spotifyimport.Report, error)
}
// mixService is what the handlers need from mixes.Service.
type mixService interface {
List(ctx context.Context, userID int64) ([]mixes.Mix, error)
@ -219,6 +225,7 @@ type Deps struct {
Discover discoverService
Feedback feedbackService
Playlists playlistService
SpotifyImport spotifyImportService
Mixes mixService
Smart smartService
Naming namingService
@ -354,6 +361,7 @@ func NewRouter(d Deps) *gin.Engine {
protected.GET("/me/settings", us.get)
protected.PATCH("/me/settings", us.update)
ma := &mastodonHandlers{svc: d.Mastodon}
protected.POST("/me/import/spotify", (&spotifyImportHandlers{svc: d.SpotifyImport}).importExport)
protected.GET("/me/mastodon", ma.get)
protected.PUT("/me/mastodon", ma.connect)
protected.DELETE("/me/mastodon", ma.disconnect)

View file

@ -0,0 +1,191 @@
package httpapi
import (
"bytes"
"context"
"encoding/json"
"fmt"
"mime/multipart"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/uptrace/bun"
"codeberg.org/squidd/sonda/internal/library"
"codeberg.org/squidd/sonda/internal/storage"
)
const importPath = "/api/v1/me/import/spotify"
// upload posts the files as a multipart form, each as `files`.
func upload(t *testing.T, h http.Handler, token string, files ...string) resp {
t.Helper()
var body bytes.Buffer
w := multipart.NewWriter(&body)
for i, content := range files {
part, err := w.CreateFormFile("files", fmt.Sprintf("file%d.json", i))
if err != nil {
t.Fatal(err)
}
_, _ = part.Write([]byte(content))
}
_ = w.Close()
req := httptest.NewRequest("POST", importPath, &body)
req.Header.Set("Content-Type", w.FormDataContentType())
if token != "" {
req.Header.Set("Authorization", "Bearer "+token)
}
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
out := resp{Code: rec.Code, Body: map[string]any{}}
_ = json.Unmarshal(rec.Body.Bytes(), &out.Body)
return out
}
func seedSong(t *testing.T, db *bun.DB, rel, title, album, artist string) {
t.Helper()
repo := storage.NewLibraryRepo(db)
lib, err := repo.CreateLibrary(context.Background(), "music-"+rel, "/lib/"+rel)
if err != nil {
t.Fatal(err)
}
f := library.ParsedFile{
FileInfo: library.FileInfo{RelPath: rel, Size: 100, MTimeUnix: 1},
Metadata: library.Metadata{
Tags: library.Tags{Title: title, Album: album, AlbumArtist: artist, Artists: []string{artist}},
Properties: library.Properties{Format: "flac", DurationMs: 260_000, SampleRate: 44100, BitRate: 900},
},
}
if _, err := repo.SaveFile(context.Background(), lib.ID, f, 0); err != nil {
t.Fatal(err)
}
}
const spotifyHistory = `[
{"ts":"2023-05-01T10:00:00Z","ms_played":260000,"ip_addr":"203.0.113.9","master_metadata_track_name":"Karma Police",
"master_metadata_album_artist_name":"Radiohead","master_metadata_album_album_name":"OK Computer","reason_end":"trackdone"},
{"ts":"2023-05-01T10:10:00Z","ms_played":100000,"master_metadata_track_name":"Creep","master_metadata_album_artist_name":"Radiohead"}
]`
func TestImportingAHistoryRecordsTheListensAndTheReportSaysWhatHappened(t *testing.T) {
h, db := newTestEnv(t, func(*Deps) {})
seedSong(t, db, "a.flac", "Karma Police", "OK Computer", "Radiohead")
tok := registerAndLogin(t, h)
r := upload(t, h, tok, spotifyHistory)
if r.Code != http.StatusOK {
t.Fatalf("import: %d %v", r.Code, r.Body)
}
hist, _ := r.Body["history"].(map[string]any)
sameKeys(t, "history", hist, "entries", "matched", "imported", "already_there", "not_found", "ignored")
if hist["entries"] != 2.0 || hist["matched"] != 1.0 || hist["imported"] != 1.0 || hist["not_found"] != 1.0 {
t.Errorf("history = %v", hist)
}
missing, _ := r.Body["missing"].(map[string]any)
sameKeys(t, "missing", missing, "total", "artists")
artists, _ := missing["artists"].([]any)
if len(artists) != 1 || artists[0].(map[string]any)["name"] != "Radiohead" || artists[0].(map[string]any)["count"] != 1.0 {
t.Errorf("missing = %v", missing)
}
if pl, ok := r.Body["playlists"].([]any); !ok || len(pl) != 0 {
t.Errorf("playlists = %#v, want an empty list", r.Body["playlists"])
}
// The listen is in the history, with the time of the play, and counts.
list := do(t, h, "GET", "/api/v1/listens", tok, nil)
items, _ := list.Body["items"].([]any)
if len(items) != 1 {
t.Fatalf("history has %d events: %v", len(items), list.Body)
}
if ev := items[0].(map[string]any); ev["counted"] != true || ev["occurred_at"] != "2023-05-01T10:00:00Z" {
t.Errorf("event = %v", ev)
}
// The same file again adds nothing.
r = upload(t, h, tok, spotifyHistory)
hist, _ = r.Body["history"].(map[string]any)
if r.Code != http.StatusOK || hist["imported"] != 0.0 || hist["already_there"] != 1.0 {
t.Errorf("second import: %d %v", r.Code, hist)
}
}
func TestImportingPlaylistsMakesPlaylistsOfTheSignedInPerson(t *testing.T) {
h, db := newTestEnv(t, func(*Deps) {})
seedSong(t, db, "a.flac", "Karma Police", "OK Computer", "Radiohead")
tok := registerAndLogin(t, h)
r := upload(t, h, tok, `{"playlists":[{"name":"Road","items":[
{"track":{"trackName":"Karma Police","artistName":"Radiohead","albumName":"OK Computer"}},
{"track":{"trackName":"Creep","artistName":"Radiohead","albumName":"Pablo Honey"}}]}]}`)
if r.Code != http.StatusOK {
t.Fatalf("import: %d %v", r.Code, r.Body)
}
pl := r.Body["playlists"].([]any)[0].(map[string]any)
sameKeys(t, "playlist", pl, "name", "tracks", "matched", "status")
if pl["name"] != "Road" || pl["tracks"] != 2.0 || pl["matched"] != 1.0 || pl["status"] != "created" {
t.Errorf("playlist = %v", pl)
}
list := do(t, h, "GET", "/api/v1/playlists", tok, nil)
items, _ := list.Body["items"].([]any)
if len(items) != 1 || items[0].(map[string]any)["name"] != "Road" || items[0].(map[string]any)["track_count"] != 1.0 {
t.Errorf("playlists = %v", list.Body)
}
}
func TestTheImportNeedsASignedInPerson(t *testing.T) {
h := newTestAPI(t)
if r := upload(t, h, "", spotifyHistory); r.Code != http.StatusUnauthorized {
t.Errorf("without a token: %d, want 401", r.Code)
}
}
func TestImportRefusesWhatIsNotAnExport(t *testing.T) {
h := newTestAPI(t)
tok := registerAndLogin(t, h)
if r := do(t, h, "POST", importPath, tok, "{}"); r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" {
t.Errorf("a JSON body: %d %v, want 400 invalid_input", r.Code, r.Body)
}
if r := upload(t, h, tok); r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" {
t.Errorf("no files: %d %v, want 400 invalid_input", r.Code, r.Body)
}
if r := upload(t, h, tok, `{"hello":"world"}`, "words"); r.Code != http.StatusBadRequest || errCode(r) != "no_spotify_data" {
t.Errorf("not Spotify's: %d %v, want 400 no_spotify_data", r.Code, r.Body)
}
if r := upload(t, h, tok, spotifyHistory[:len(spotifyHistory)-30]); r.Code != http.StatusBadRequest || errCode(r) != "unreadable_file" {
t.Errorf("a cut file: %d %v, want 400 unreadable_file", r.Code, r.Body)
}
}
func TestAnUploadBeyondTheLimitIsRefused(t *testing.T) {
old := importMaxBytes
importMaxBytes = 2000
defer func() { importMaxBytes = old }()
h := newTestAPI(t)
tok := registerAndLogin(t, h)
big := "[" + strings.Repeat(`{"ts":"2023-05-01T10:00:00Z"},`, 200) + `{"ts":"2023-05-01T10:00:00Z"}]`
if r := upload(t, h, tok, big); r.Code != http.StatusRequestEntityTooLarge || errCode(r) != "too_large" {
t.Errorf("%d %v, want 413 too_large", r.Code, r.Body)
}
}
func TestAnUploadLargerThanOtherRequestsIsAccepted(t *testing.T) {
h, db := newTestEnv(t, func(*Deps) {})
seedSong(t, db, "a.flac", "Karma Police", "OK Computer", "Radiohead")
tok := registerAndLogin(t, h)
// A history of more than the 64 KB that any other request may be.
var plays []string
for i := 0; i < 1500; i++ {
plays = append(plays, fmt.Sprintf(`{"ts":"2023-05-%02dT%02d:%02d:00Z","ms_played":260000,"master_metadata_track_name":"Karma Police","master_metadata_album_artist_name":"Radiohead"}`, 1+i/1440, i/60%24, i%60))
}
data := "[" + strings.Join(plays, ",") + "]"
if len(data) < 2*maxBodyBytes {
t.Fatalf("test file is %d bytes, too small to prove anything", len(data))
}
r := upload(t, h, tok, data)
hist, _ := r.Body["history"].(map[string]any)
if r.Code != http.StatusOK || hist["imported"] != 1500.0 {
t.Errorf("%d %v", r.Code, r.Body)
}
}

View file

@ -2162,6 +2162,70 @@ paths:
'204': {description: Disconnected.}
'401': {$ref: '#/components/responses/Unauthorized'}
/me/import/spotify:
post:
tags: [Taste]
operationId: importMySpotify
summary: Import your Spotify history and playlists
description: |
Brings your listening history and your playlists from a Spotify data export into your own account, so that
Mixes, Discover and the smart playlists know what you listened to before. Ask Spotify for your data (Account,
Privacy, "Extended streaming history" and "Account data"), then send what it gives: the zip itself, or the
JSON files in it (`Streaming_History_Audio_*.json`, the older `StreamingHistory_music_*.json`, and
`Playlist*.json`), as a multipart form with one `files` field for each file. A request may be up to 512 MB.
Everything happens on this server and nothing is sent to anyone. The files are read from the request and not
kept, and of each play only the song, the artist, the album, the time and how it ended are read: the IP
addresses, the country and the platform in the file are never looked at.
A play is matched to a song of your library by artist and title (case, accents and punctuation do not matter,
a "Remastered" note on either side does not either, and any of the credited artists will do; a play of a
compilation is matched by its album when Spotify names "Various Artists"). A match is recorded as a listen at
the time of the play: `complete` when the play ran to its end (or nine tenths of the song were played),
`skip` when Spotify says it was skipped or it was shorter than a listen, `play` otherwise. Importing does not
post anything to Mastodon. Importing the same files again records nothing twice, and the same play in the
extended history and in the older one is counted once.
A playlist of the export becomes a playlist of yours with the songs that were found, in order. One whose
name you already have is left alone (`exists`), and one with none of its songs in your library is not made
(`empty`). The report lists what was not found, by artist, most played first, so you can see what is missing
from your library. One import of yours runs at a time.
security: [{bearerAuth: []}]
requestBody:
required: true
content:
multipart/form-data:
schema:
type: object
required: [files]
properties:
files:
type: array
items: {type: string, format: binary}
description: The zip of the export, or its JSON files.
responses:
'200':
description: What the import did.
content:
application/json:
schema: {$ref: '#/components/schemas/SpotifyImportReport'}
'400':
description: No files were sent (`invalid_input`), none of them holds a history or playlists of Spotify (`no_spotify_data`) or one is damaged or cut short (`unreadable_file`).
content:
application/json:
schema: {$ref: '#/components/schemas/Error'}
'401': {$ref: '#/components/responses/Unauthorized'}
'409':
description: An import of yours is still running (`import_running`).
content:
application/json:
schema: {$ref: '#/components/schemas/Error'}
'413':
description: The files are larger than 512 MB, or hold more than can be imported once inflated (`too_large`).
content:
application/json:
schema: {$ref: '#/components/schemas/Error'}
/artists/{id}/lidarr:
parameters:
- {$ref: '#/components/parameters/Id'}
@ -5402,6 +5466,53 @@ components:
shared: another user can see from `retrieved_at` in `GET /artists/{id}/similar` that
somebody looked an artist up, but not who.
SpotifyImportReport:
type: object
additionalProperties: false
required: [history, playlists, missing]
properties:
history:
type: object
additionalProperties: false
required: [entries, matched, imported, already_there, not_found, ignored]
properties:
entries: {type: integer, description: 'Plays of songs that were read.'}
matched: {type: integer, description: 'Of those, the plays of a song of your library.'}
imported: {type: integer, description: 'Recorded now.'}
already_there: {type: integer, description: 'Matched and already recorded by an earlier import, or the same play in two files.'}
not_found: {type: integer, description: 'Plays of songs that are not in your library.'}
ignored: {type: integer, description: 'Podcast episodes, and plays with no time that can be read.'}
playlists:
type: array
items:
type: object
additionalProperties: false
required: [name, tracks, matched, status]
properties:
name: {type: string}
tracks: {type: integer, description: 'Everything that was in the playlist, songs and not.'}
matched: {type: integer, description: 'The songs of your library.'}
status:
type: string
enum: [created, exists, empty, limit]
description: '`created`; `exists` (you already have one with that name, left alone); `empty` (none of its songs is in your library); `limit` (you have as many playlists as are allowed).'
missing:
type: object
additionalProperties: false
required: [total, artists]
properties:
total: {type: integer, description: 'How many artists have songs that were not found.'}
artists:
type: array
description: The artists with the most songs that were not found, at most 50, most first.
items:
type: object
additionalProperties: false
required: [name, count]
properties:
name: {type: string}
count: {type: integer, description: 'Plays and playlist songs of this artist that were not found.'}
MastodonAccount:
type: object
additionalProperties: false

View file

@ -0,0 +1,111 @@
package spotifyimport
import (
"regexp"
"strings"
)
// Candidate is a song of the library that can be played: what a Spotify entry is compared with.
type Candidate struct {
TrackID int64
Title string
Album string // empty when it has none
Artists []string
DurationMs int64
}
// Index finds the song of the library that a Spotify entry is. It is built once for an import, from every playable
// song, and answers from memory.
type Index struct {
key func(string) string
byArtist map[string][]entry // artist key + title key
byBase map[string][]entry // artist key + title key without a remaster note
byAlbum map[string][]entry // album key + title key, for a compilation whose artist is "Various Artists"
lengths map[int64]int64
}
type entry struct {
id int64
albumKey string
}
// NewIndex builds the index. key makes the comparable form of a name (the same one the library search uses: no case,
// no accents, no punctuation). Candidates come in order of id, and the first of equal ones wins.
func NewIndex(key func(string) string, songs []Candidate) *Index {
ix := &Index{
key: key,
byArtist: map[string][]entry{},
byBase: map[string][]entry{},
byAlbum: map[string][]entry{},
lengths: map[int64]int64{},
}
for _, s := range songs {
ix.lengths[s.TrackID] = s.DurationMs
title, base, album := key(s.Title), key(baseTitle(s.Title)), key(s.Album)
e := entry{id: s.TrackID, albumKey: album}
for _, artist := range s.Artists {
a := key(artist)
ix.byArtist[a+"\x00"+title] = append(ix.byArtist[a+"\x00"+title], e)
ix.byBase[a+"\x00"+base] = append(ix.byBase[a+"\x00"+base], e)
}
if album != "" {
ix.byAlbum[album+"\x00"+title] = append(ix.byAlbum[album+"\x00"+title], e)
}
}
return ix
}
// Length is how long a song is, in milliseconds (0 when the library does not know).
func (ix *Index) Length(trackID int64) int64 { return ix.lengths[trackID] }
// Find gives the song of the library that an artist, a title and (when known) an album name, or false. The exact
// title comes first, then the title without a remaster note on either side; when several songs are equal the one of
// the album that was asked wins, else the first. A play of a compilation is credited to "Various Artists" on Spotify,
// which no song of the library is by: for that the album and the title are enough, when that names exactly one song.
func (ix *Index) Find(artist, title, album string) (int64, bool) {
a, t, b := ix.key(artist), ix.key(title), ix.key(album)
if t == "" {
return 0, false
}
if a != "" {
if id, ok := pick(ix.byArtist[a+"\x00"+t], b); ok {
return id, true
}
if id, ok := pick(ix.byBase[a+"\x00"+ix.key(baseTitle(title))], b); ok {
return id, true
}
}
if b != "" {
if matches := ix.byAlbum[b+"\x00"+t]; len(matches) == 1 {
return matches[0].id, true
}
}
return 0, false
}
func pick(matches []entry, albumKey string) (int64, bool) {
if len(matches) == 0 {
return 0, false
}
if albumKey != "" {
for _, m := range matches {
if m.albumKey == albumKey {
return m.id, true
}
}
}
return matches[0].id, true
}
// remaster is a note that says a song was remastered, in brackets or after a dash: "(Remastered 2011)",
// "- 2017 Remaster", "[Remaster]". Only the note at the end of the title goes.
var remaster = regexp.MustCompile(`(?i)\s*(?:[-–—]\s*[^()\[\]]*remaster(?:ed)?[^()\[\]]*|\([^()]*remaster(?:ed)?[^()]*\)|\[[^\[\]]*remaster(?:ed)?[^\[\]]*\])\s*$`)
// baseTitle is a title without its remaster note, which is the same song as far as a listener is concerned.
func baseTitle(s string) string {
out := strings.TrimSpace(remaster.ReplaceAllString(s, ""))
if out == "" {
return strings.TrimSpace(s)
}
return out
}

View file

@ -0,0 +1,106 @@
package spotifyimport
import (
"strings"
"testing"
)
// key is a stand-in for the key the server uses to compare names: lower case, no accents, no punctuation.
func key(s string) string {
r := strings.NewReplacer("ö", "o", "é", "e", ".", "", ",", "", "'", "", "’", "", "!", "", "-", " ")
return strings.Join(strings.Fields(r.Replace(strings.ToLower(s))), " ")
}
func library() *Index {
return NewIndex(key, []Candidate{
{TrackID: 1, Title: "Karma Police", Album: "OK Computer", Artists: []string{"Radiohead"}},
{TrackID: 2, Title: "Karma Police", Album: "OK Computer OKNOTOK", Artists: []string{"Radiohead"}},
{TrackID: 3, Title: "Hyperballad", Album: "Post", Artists: []string{"Björk"}},
{TrackID: 4, Title: "Heroes (2017 Remaster)", Album: "Heroes", Artists: []string{"David Bowie"}},
{TrackID: 5, Title: "Love Will Tear Us Apart", Album: "Substance", Artists: []string{"Joy Division"}},
{TrackID: 6, Title: "Under Pressure", Album: "Hot Space", Artists: []string{"Queen", "David Bowie"}},
{TrackID: 7, Title: "Paranoid Android", Album: "Now That's What I Call Music", Artists: []string{"Radiohead"}},
{TrackID: 8, Title: "Wonderwall", Album: "Greatest Hits", Artists: []string{"Oasis"}},
})
}
func TestAnExactArtistAndTitleMatches(t *testing.T) {
ix := library()
for _, tc := range []struct {
artist, title, album string
want int64
}{
{"Radiohead", "Karma Police", "OK Computer", 1},
{"radiohead", "KARMA POLICE", "", 1}, // case, and no album to tell the two apart: the first
{"Bjork", "Hyperballad", "Post", 3}, // accents do not matter
{"Joy Division", "Love Will Tear Us Apart!", "", 5},
} {
if id, ok := ix.Find(tc.artist, tc.title, tc.album); !ok || id != tc.want {
t.Errorf("Find(%q, %q, %q) = %d, %v, want %d", tc.artist, tc.title, tc.album, id, ok, tc.want)
}
}
}
func TestTheAlbumTellsTwoVersionsApart(t *testing.T) {
if id, ok := library().Find("Radiohead", "Karma Police", "OK Computer OKNOTOK"); !ok || id != 2 {
t.Errorf("= %d, %v, want the song of the album that was asked", id, ok)
}
}
func TestARemasterSuffixDoesNotMatter(t *testing.T) {
ix := library()
for _, title := range []string{"Heroes - 2017 Remaster", "Heroes (Remastered 2017)", "Heroes - Remastered", "Heroes"} {
if id, ok := ix.Find("David Bowie", title, ""); !ok || id != 4 {
t.Errorf("Find(%q) = %d, %v, want 4", title, id, ok)
}
}
// The other way round: the library has the plain title and Spotify the remaster.
if id, ok := ix.Find("Queen", "Under Pressure - Remastered 2011", ""); !ok || id != 6 {
t.Errorf("a remastered title for a plain one = %d, %v, want 6", id, ok)
}
}
func TestAnyOfTheCreditedArtistsMatches(t *testing.T) {
ix := library()
for _, artist := range []string{"Queen", "David Bowie"} {
if id, ok := ix.Find(artist, "Under Pressure", ""); !ok || id != 6 {
t.Errorf("Find(%q) = %d, %v, want 6", artist, id, ok)
}
}
}
func TestACompilationIsFoundByItsAlbumWhenTheArtistIsVariousArtists(t *testing.T) {
if id, ok := library().Find("Various Artists", "Paranoid Android", "Now That's What I Call Music"); !ok || id != 7 {
t.Errorf("= %d, %v, want 7", id, ok)
}
// ...but not by the album alone when it does not hold the song.
if _, ok := library().Find("Various Artists", "Something Else", "Now That's What I Call Music"); ok {
t.Error("a song that the album does not hold was found")
}
}
func TestWhatIsNotInTheLibraryIsNotFound(t *testing.T) {
ix := library()
for _, tc := range [][2]string{{"Radiohead", "Creep"}, {"Blur", "Karma Police"}, {"", "Karma Police"}, {"Radiohead", ""}} {
if id, ok := ix.Find(tc[0], tc[1], ""); ok {
t.Errorf("Find(%q, %q) = %d, want nothing", tc[0], tc[1], id)
}
}
}
func TestTheBaseOfATitle(t *testing.T) {
for in, want := range map[string]string{
"Heroes - 2017 Remaster": "Heroes",
"Heroes - Remastered 2017": "Heroes",
"Heroes (Remastered)": "Heroes",
"Heroes [2017 Remaster]": "Heroes",
"Heroes - Mono": "Heroes - Mono", // only a remaster is taken off
"Heroes (Live)": "Heroes (Live)",
"The Remaster of Remasters": "The Remaster of Remasters",
"Song (feat. X) - 2011 Remaster": "Song (feat. X)",
} {
if got := baseTitle(in); got != want {
t.Errorf("baseTitle(%q) = %q, want %q", in, got, want)
}
}
}

View file

@ -0,0 +1,307 @@
// Package spotifyimport brings the listening history and the playlists of a Spotify data export into Sonda.
//
// Spotify gives a person a zip of JSON files when they ask for their data. This package reads those files (the zip
// itself, or the JSON files in it), matches each song to the library by artist and title, records the matches as
// listens and makes the playlists. It runs on the server and contacts nobody: the files are read from the request,
// never written anywhere, and the parts that a file holds besides songs (the IP addresses of the person, the country,
// the platform, the user name) are never even looked at.
package spotifyimport
import (
"archive/zip"
"bufio"
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"path"
"strings"
"time"
)
// Limits on what one upload may hold, so that a zip that inflates into something enormous cannot fill the memory
// or the disk of the server.
var (
maxInflated = int64(4 << 30) // all the JSON of one upload, once it is inflated
maxZipFiles = 2000 // files in one zip
)
// ErrTooBig is returned for an upload that holds more than the limits allow.
var ErrTooBig = errors.New("the files hold more than can be imported")
// ErrUnreadable is returned for a file that looks like one of Spotify's and cannot be read to its end (cut short,
// damaged).
var ErrUnreadable = errors.New("a file is damaged or cut short")
// Entry is one play of a song from a streaming history file.
type Entry struct {
PlayedAt time.Time // when the play ended
Artist string
Title string
Album string // empty in the older files, which do not say
PlayedMs int64
Skipped bool // the file says the listener skipped it
ReasonEnd string // why the play ended ("trackdone" when it ran to the end), empty in the older files
}
// PlaylistItem is one song of a playlist.
type PlaylistItem struct{ Artist, Title, Album string }
// Playlist is a playlist of the export. Items are its songs, in order; Other counts what is in it that is not a song
// (a podcast episode, a file the person had on their own computer).
type Playlist struct {
Name string
Items []PlaylistItem
Other int
}
// Reader reads files of an export and calls back for what is in them. A callback that is nil is skipped.
type Reader struct {
OnEntry func(Entry)
OnPlaylist func(Playlist)
// OnIgnored is called for a play that is not a song (a podcast episode) or that has no time that can be read.
OnIgnored func()
}
// ReadFile reads one upload: the zip of the export, or one of the JSON files in it. It reports whether it held
// something of Spotify's (a history or playlists); a file that holds none is not an error, it is just not found.
func (r Reader) ReadFile(ra io.ReaderAt, size int64) (bool, error) {
found, err := r.readFile(ra, size)
if err != nil && !errors.Is(err, ErrTooBig) {
err = fmt.Errorf("%w: %v", ErrUnreadable, err)
}
return found, err
}
func (r Reader) readFile(ra io.ReaderAt, size int64) (bool, error) {
head := make([]byte, 4)
if n, _ := ra.ReadAt(head, 0); n == 4 && bytes.Equal(head, []byte("PK\x03\x04")) {
return r.readZip(ra, size)
}
budget := maxInflated
return r.readJSON(&budgeted{r: io.NewSectionReader(ra, 0, size), left: &budget})
}
func (r Reader) readZip(ra io.ReaderAt, size int64) (bool, error) {
zr, err := zip.NewReader(ra, size)
if err != nil {
return false, nil // looks like a zip and is not one: nothing of Spotify's
}
if len(zr.File) > maxZipFiles {
return false, ErrTooBig
}
budget := maxInflated
found := false
for _, f := range zr.File {
if !wanted(f.Name) {
continue
}
rc, err := f.Open()
if err != nil {
return found, fmt.Errorf("open %s: %w", path.Base(f.Name), err)
}
ok, err := r.readJSON(&budgeted{r: rc, left: &budget})
_ = rc.Close()
if err != nil {
return found, fmt.Errorf("read %s: %w", path.Base(f.Name), err)
}
found = found || ok
}
return found, nil
}
// wanted says whether a file of the zip can hold what is imported: the audio history (not the video one) and the
// playlists. The rest of an export (the account, the searches, the library) is never opened.
func wanted(name string) bool {
base := strings.ToLower(path.Base(name))
if !strings.HasSuffix(base, ".json") || strings.Contains(base, "video") {
return false
}
return strings.HasPrefix(base, "streaming_history") || strings.HasPrefix(base, "streaminghistory") || strings.HasPrefix(base, "playlist")
}
// budgeted stops a reader that gives more than the budget left for the whole upload.
type budgeted struct {
r io.Reader
left *int64
}
func (b *budgeted) Read(p []byte) (int, error) {
if *b.left <= 0 {
return 0, ErrTooBig
}
if int64(len(p)) > *b.left {
p = p[:*b.left]
}
n, err := b.r.Read(p)
*b.left -= int64(n)
return n, err
}
// rawEntry has the fields of a play that are used, in both shapes of file: the extended history (ts, ms_played,
// master_metadata_...) and the older one (endTime, msPlayed, trackName). Nothing else is decoded.
type rawEntry struct {
TS string `json:"ts"`
MsPlayed int64 `json:"ms_played"`
Track *string `json:"master_metadata_track_name"`
Artist *string `json:"master_metadata_album_artist_name"`
Album *string `json:"master_metadata_album_album_name"`
ReasonEnd string `json:"reason_end"`
Skipped *bool `json:"skipped"`
EndTime string `json:"endTime"`
OldArtist string `json:"artistName"`
OldTrack string `json:"trackName"`
OldMsPlayed int64 `json:"msPlayed"`
}
type rawPlaylists struct {
Playlists *[]struct {
Name string `json:"name"`
Items []struct {
Track *struct {
Name string `json:"trackName"`
Artist string `json:"artistName"`
Album string `json:"albumName"`
} `json:"track"`
} `json:"items"`
} `json:"playlists"`
}
func (r Reader) readJSON(in io.Reader) (bool, error) {
br := bufio.NewReaderSize(in, 64<<10)
first, err := firstByte(br)
if err != nil {
if errors.Is(err, ErrTooBig) {
return false, err
}
return false, nil // empty, or not readable: nothing of Spotify's
}
dec := json.NewDecoder(br)
switch first {
case '[':
return r.readHistory(dec)
case '{':
return r.readPlaylists(dec)
}
return false, nil
}
// firstByte looks at the first byte that is not a space, without taking it.
func firstByte(br *bufio.Reader) (byte, error) {
for {
b, err := br.ReadByte()
if err != nil {
return 0, err
}
if b == ' ' || b == '\n' || b == '\r' || b == '\t' || b == 0xEF || b == 0xBB || b == 0xBF { // spaces and a byte order mark
continue
}
return b, br.UnreadByte()
}
}
func (r Reader) readHistory(dec *json.Decoder) (bool, error) {
if _, err := dec.Token(); err != nil { // the [
return false, nil
}
found := false
for dec.More() {
var raw rawEntry
if err := dec.Decode(&raw); err != nil {
var typeErr *json.UnmarshalTypeError
if errors.As(err, &typeErr) {
continue // an element that is not an object: not a play
}
return found, err
}
stamp := raw.TS
if stamp == "" {
stamp = raw.EndTime
}
if stamp == "" {
continue // not a play at all: a file that only looks like a history
}
found = true
e, ok := raw.entry(stamp)
if !ok {
if r.OnIgnored != nil {
r.OnIgnored()
}
continue
}
if r.OnEntry != nil {
r.OnEntry(e)
}
}
return found, nil
}
// entry is the play, or false for something that is not a song or has no time that can be read.
func (raw rawEntry) entry(stamp string) (Entry, bool) {
at, ok := parseTime(stamp)
if !ok {
return Entry{}, false
}
e := Entry{PlayedAt: at, PlayedMs: raw.MsPlayed, ReasonEnd: raw.ReasonEnd}
if raw.Skipped != nil {
e.Skipped = *raw.Skipped
}
if raw.Track != nil {
e.Title = *raw.Track
}
if raw.Artist != nil {
e.Artist = *raw.Artist
}
if raw.Album != nil {
e.Album = *raw.Album
}
if e.Title == "" && raw.OldTrack != "" { // the older shape
e.Title, e.Artist, e.PlayedMs = raw.OldTrack, raw.OldArtist, raw.OldMsPlayed
}
e.Title, e.Artist, e.Album = strings.TrimSpace(e.Title), strings.TrimSpace(e.Artist), strings.TrimSpace(e.Album)
if e.Title == "" || e.Artist == "" || e.PlayedMs < 0 {
return Entry{}, false // an episode of a podcast has no song name
}
return e, true
}
// parseTime reads the time of a play: an RFC 3339 time in the extended history, "2022-01-02 08:30" (UTC) in the older one.
func parseTime(s string) (time.Time, bool) {
for _, layout := range []string{time.RFC3339, "2006-01-02 15:04", "2006-01-02 15:04:05"} {
if t, err := time.ParseInLocation(layout, strings.TrimSpace(s), time.UTC); err == nil {
return t.UTC(), true
}
}
return time.Time{}, false
}
func (r Reader) readPlaylists(dec *json.Decoder) (bool, error) {
var raw rawPlaylists
if err := dec.Decode(&raw); err != nil {
if errors.Is(err, ErrTooBig) {
return false, err
}
return false, nil
}
if raw.Playlists == nil {
return false, nil
}
for _, p := range *raw.Playlists {
out := Playlist{Name: strings.TrimSpace(p.Name)}
for _, it := range p.Items {
if it.Track == nil || strings.TrimSpace(it.Track.Name) == "" || strings.TrimSpace(it.Track.Artist) == "" {
out.Other++
continue
}
out.Items = append(out.Items, PlaylistItem{
Artist: strings.TrimSpace(it.Track.Artist), Title: strings.TrimSpace(it.Track.Name), Album: strings.TrimSpace(it.Track.Album),
})
}
if r.OnPlaylist != nil {
r.OnPlaylist(out)
}
}
return true, nil
}

View file

@ -0,0 +1,183 @@
package spotifyimport
import (
"archive/zip"
"bytes"
"errors"
"strings"
"testing"
"time"
)
const extended = `[
{"ts":"2023-05-01T10:00:00Z","username":"someone","platform":"linux","ms_played":215000,"conn_country":"IT","ip_addr":"203.0.113.9",
"master_metadata_track_name":"Karma Police","master_metadata_album_artist_name":"Radiohead","master_metadata_album_album_name":"OK Computer",
"spotify_track_uri":"spotify:track:1","reason_start":"trackdone","reason_end":"trackdone","shuffle":false,"skipped":null},
{"ts":"2023-05-01T10:04:00Z","ms_played":4000,"master_metadata_track_name":"Airbag","master_metadata_album_artist_name":"Radiohead",
"master_metadata_album_album_name":"OK Computer","reason_end":"fwdbtn","skipped":true},
{"ts":"2023-05-01T11:00:00Z","ms_played":900000,"master_metadata_track_name":null,"master_metadata_album_artist_name":null,
"episode_name":"Episode 12","episode_show_name":"A podcast","spotify_episode_uri":"spotify:episode:9"}
]`
const legacy = `[
{"endTime":"2022-01-02 08:30","artistName":"Björk","trackName":"Hyperballad","msPlayed":321000},
{"endTime":"not a time","artistName":"X","trackName":"Y","msPlayed":1}
]`
const playlistsFile = `{"playlists":[
{"name":"Road","lastModifiedDate":"2023-01-01","items":[
{"track":{"trackName":"Karma Police","artistName":"Radiohead","albumName":"OK Computer","trackUri":"spotify:track:1"},"episode":null,"localTrack":null,"addedDate":"2020-01-01"},
{"track":null,"episode":{"name":"An episode"},"localTrack":null},
{"track":null,"episode":null,"localTrack":{"uri":"spotify:local:::a:1"}}
]},
{"name":"Empty","items":[]}
]}`
type got struct {
entries []Entry
playlists []Playlist
ignored int
}
func read(t *testing.T, data []byte) (got, bool) {
t.Helper()
var g got
r := Reader{
OnEntry: func(e Entry) { g.entries = append(g.entries, e) },
OnPlaylist: func(p Playlist) { g.playlists = append(g.playlists, p) },
OnIgnored: func() { g.ignored++ },
}
found, err := r.ReadFile(bytes.NewReader(data), int64(len(data)))
if err != nil {
t.Fatal(err)
}
return g, found
}
func TestReadsTheExtendedStreamingHistory(t *testing.T) {
g, found := read(t, []byte(extended))
if !found || len(g.entries) != 2 || g.ignored != 1 {
t.Fatalf("found=%v entries=%d ignored=%d, want true, 2 and 1 (the podcast)", found, len(g.entries), g.ignored)
}
e := g.entries[0]
if e.Title != "Karma Police" || e.Artist != "Radiohead" || e.Album != "OK Computer" || e.PlayedMs != 215000 || e.ReasonEnd != "trackdone" || e.Skipped {
t.Errorf("first entry = %+v", e)
}
if !e.PlayedAt.Equal(time.Date(2023, 5, 1, 10, 0, 0, 0, time.UTC)) {
t.Errorf("played at %v", e.PlayedAt)
}
if s := g.entries[1]; !s.Skipped || s.ReasonEnd != "fwdbtn" {
t.Errorf("second entry = %+v, want a skip", s)
}
}
func TestReadsTheOlderStreamingHistoryAndDropsWhatHasNoTime(t *testing.T) {
g, found := read(t, []byte(legacy))
if !found || len(g.entries) != 1 {
t.Fatalf("found=%v entries=%d, want true and 1", found, len(g.entries))
}
e := g.entries[0]
if e.Artist != "Björk" || e.Title != "Hyperballad" || e.Album != "" || e.PlayedMs != 321000 {
t.Errorf("entry = %+v", e)
}
if !e.PlayedAt.Equal(time.Date(2022, 1, 2, 8, 30, 0, 0, time.UTC)) {
t.Errorf("played at %v", e.PlayedAt)
}
if g.ignored != 1 {
t.Errorf("ignored = %d, want the entry with no readable time counted", g.ignored)
}
}
func TestReadsThePlaylists(t *testing.T) {
g, found := read(t, []byte(playlistsFile))
if !found || len(g.playlists) != 2 {
t.Fatalf("found=%v playlists=%d", found, len(g.playlists))
}
p := g.playlists[0]
if p.Name != "Road" || len(p.Items) != 1 || p.Other != 2 {
t.Fatalf("playlist = %+v, want one song and two that are not", p)
}
if it := p.Items[0]; it.Title != "Karma Police" || it.Artist != "Radiohead" || it.Album != "OK Computer" {
t.Errorf("item = %+v", it)
}
}
func TestSomethingThatIsNotSpotifyDataIsNotFound(t *testing.T) {
for name, data := range map[string]string{
"other json": `{"hello":"world"}`,
"other array": `[{"query":"x","time":"y"}]`,
"not json": `just words`,
"empty": ``,
"a number": `42`,
"a null playlist": `{"playlists":null}`,
} {
g, found := read(t, []byte(data))
if found || len(g.entries) != 0 || len(g.playlists) != 0 {
t.Errorf("%s: found=%v %+v, want nothing", name, found, g)
}
}
}
func makeZip(t *testing.T, files map[string]string) []byte {
t.Helper()
var buf bytes.Buffer
w := zip.NewWriter(&buf)
for name, body := range files {
f, err := w.Create(name)
if err != nil {
t.Fatal(err)
}
if _, err := f.Write([]byte(body)); err != nil {
t.Fatal(err)
}
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
return buf.Bytes()
}
func TestReadsTheZipOfTheExport(t *testing.T) {
data := makeZip(t, map[string]string{
"Spotify Extended Streaming History/Streaming_History_Audio_2019-2023_0.json": extended,
"Spotify Extended Streaming History/Streaming_History_Video_2019-2023.json": extended, // videos are not music
"Spotify Account Data/StreamingHistory_music_0.json": legacy,
"Spotify Account Data/Playlist1.json": playlistsFile,
"Spotify Account Data/Userdata.json": `{"username":"someone","email":"someone@example.org"}`,
"Spotify Account Data/SearchQueries.json": `[{"platform":"x","searchTime":"y","searchQuery":"z"}]`,
"README.pdf": "ignored",
})
g, found := read(t, data)
if !found {
t.Fatal("the zip of an export was not recognised")
}
if len(g.entries) != 3 || len(g.playlists) != 2 {
t.Errorf("entries=%d playlists=%d, want 3 (two extended, one older) and 2", len(g.entries), len(g.playlists))
}
}
func TestAZipWithNothingOfSpotifyIsNotFound(t *testing.T) {
_, found := read(t, makeZip(t, map[string]string{"notes.txt": "hello", "Userdata.json": `{"a":1}`}))
if found {
t.Error("a zip with no history and no playlists was taken for an export")
}
}
func TestAZipThatInflatesTooMuchIsRefused(t *testing.T) {
old := maxInflated
maxInflated = 100
defer func() { maxInflated = old }()
data := makeZip(t, map[string]string{"Streaming_History_Audio_0.json": "[" + strings.Repeat(" ", 1000) + "]"})
r := Reader{}
if _, err := r.ReadFile(bytes.NewReader(data), int64(len(data))); !errors.Is(err, ErrTooBig) {
t.Errorf("err = %v, want ErrTooBig for a file that inflates past the limit", err)
}
}
func TestAFileThatIsCutShortIsUnreadable(t *testing.T) {
data := []byte(extended[:len(extended)-40])
_, err := Reader{}.ReadFile(bytes.NewReader(data), int64(len(data)))
if !errors.Is(err, ErrUnreadable) {
t.Errorf("err = %v, want ErrUnreadable", err)
}
}

View file

@ -0,0 +1,379 @@
package spotifyimport
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"sort"
"strings"
"sync"
"time"
"unicode/utf8"
"codeberg.org/squidd/sonda/internal/listening"
"codeberg.org/squidd/sonda/internal/playlists"
)
// Errors returned by Import.
var (
// ErrNothingFound means no file held a streaming history or playlists of Spotify.
ErrNothingFound = errors.New("no Spotify history or playlists in these files")
// ErrBusy means this person's import is still running.
ErrBusy = errors.New("an import is already running")
)
const (
// MaxMissing is how many missing artists a report lists (the ones played most); MissingTotal says how many there are.
MaxMissing = 50
// listenBatch is how many listens are recorded in one go.
listenBatch = 2000
// completeShare says a play counts as heard to the end when this much of the song was played (nine tenths).
completeShareNum, completeShareDen = 9, 10
// A play longer than the song by more than this is not believed: it is brought to the length of the song, as the
// rules of listening would otherwise refuse it.
maxOverrunFactor = 2
maxOverrunSlack = 10_000 // ms
unnamedPlaylist = "Spotify playlist"
)
// Catalog gives the songs of the library that can be played.
type Catalog interface {
Candidates(ctx context.Context) ([]Candidate, error)
}
// Listens records many listens at once. A listen whose idempotency key is already there is not recorded again, and
// is not counted in what comes back.
type Listens interface {
RecordMany(ctx context.Context, userID int64, events []listening.Event) (created int, err error)
}
// Playlists is what the import needs from the playlists of a person.
type Playlists interface {
List(ctx context.Context, userID int64, p playlists.Page) (playlists.Listing, error)
Create(ctx context.Context, userID int64, name, description string) (playlists.Playlist, error)
AddTracks(ctx context.Context, userID, id int64, trackIDs []int64, position *int) (playlists.Playlist, error)
}
// File is one upload: the zip of the export or one of its JSON files.
type File struct {
R io.ReaderAt
Size int64
}
// History says what became of the plays of the history files.
type History struct {
Entries int // plays of songs that were read
Matched int // of those, the ones that are a song of the library
Imported int // recorded now
AlreadyThere int // matched, and already recorded by an earlier import (or the same play in two files)
NotFound int // not in the library
Ignored int // podcast episodes, and plays with no time that can be read
}
// PlaylistResult says what became of one playlist. Status is `created`, `exists` (the person already has one with
// that name, which is left alone), `empty` (none of its songs is in the library) or `limit` (the person has as many
// playlists as are allowed).
type PlaylistResult struct {
Name string
Tracks int // everything that is in it
Matched int // the songs of the library
Status string
}
// Missing is an artist that has plays or playlist songs and none of them in the library.
type Missing struct {
Artist string
Count int
}
// Report is what an import did. It is made for the person who asked and kept nowhere.
type Report struct {
History History
Playlists []PlaylistResult
Missing []Missing // the artists with the most songs that were not found, at most MaxMissing
MissingTotal int // how many artists have songs that were not found
}
// Service imports the data of a Spotify export for one person at a time.
type Service struct {
catalog Catalog
listens Listens
lists Playlists
key func(string) string
minListen time.Duration
running sync.Map // user id -> struct{}
}
// NewService builds a Service. key is how names are made comparable (the key of the library search); minListen is how
// long a play must last to count as a listen, so that a play shorter than that is recorded as a skip.
func NewService(catalog Catalog, listens Listens, lists Playlists, key func(string) string, minListen time.Duration) *Service {
return &Service{catalog: catalog, listens: listens, lists: lists, key: key, minListen: minListen}
}
// Import reads the files, records the listens that match the library and makes the playlists. A person can run one
// import at a time (ErrBusy). It is safe to import the same files again: nothing is recorded twice. What was
// recorded before an error stays recorded.
func (s *Service) Import(ctx context.Context, userID int64, files []File) (Report, error) {
if _, busy := s.running.LoadOrStore(userID, struct{}{}); busy {
return Report{}, ErrBusy
}
defer s.running.Delete(userID)
songs, err := s.catalog.Candidates(ctx)
if err != nil {
return Report{}, fmt.Errorf("read the library: %w", err)
}
run := &run{svc: s, ctx: ctx, userID: userID, index: NewIndex(s.key, songs), memo: map[string]int64{}, missing: map[string]*Missing{}}
reader := Reader{OnEntry: run.entry, OnPlaylist: run.playlist, OnIgnored: func() { run.report.History.Ignored++ }}
found := false
for _, f := range files {
ok, err := reader.ReadFile(f.R, f.Size)
if err != nil {
return Report{}, err
}
found = found || ok
if run.err != nil {
return Report{}, run.err
}
}
if !found {
return Report{}, ErrNothingFound
}
if err := run.flush(); err != nil {
return Report{}, err
}
if err := run.makePlaylists(); err != nil {
return Report{}, err
}
return run.finish(), nil
}
// run is the state of one import.
type run struct {
svc *Service
ctx context.Context
userID int64
index *Index
// memo remembers which song an artist, title and album are, so that a song played a thousand times is looked up once (0: not found).
memo map[string]int64
batch []listening.Event
matched int
created int
missing map[string]*Missing // by the comparable name of the artist
playlists []Playlist
report Report
err error
}
func (r *run) find(artist, title, album string) (int64, bool) {
k := artist + "\x00" + title + "\x00" + album
id, seen := r.memo[k]
if !seen {
id, _ = r.index.Find(artist, title, album)
r.memo[k] = id
}
return id, id != 0
}
func (r *run) miss(artist string) {
k := r.svc.key(artist)
m := r.missing[k]
if m == nil {
m = &Missing{Artist: artist}
r.missing[k] = m
}
m.Count++
}
func (r *run) entry(e Entry) {
if r.err != nil {
return
}
r.report.History.Entries++
id, ok := r.find(e.Artist, e.Title, e.Album)
if !ok {
r.report.History.NotFound++
r.miss(e.Artist)
return
}
r.matched++
length := r.index.Length(id)
played := e.PlayedMs
if length > 0 && played > maxOverrunFactor*length+maxOverrunSlack {
played = length
}
r.batch = append(r.batch, listening.Event{
UserID: r.userID, TrackID: id, Type: r.svc.kind(e, played, length), OccurredAt: e.PlayedAt, DurationPlayedMs: played,
IdempotencyKey: idempotencyKey(e),
})
if len(r.batch) >= listenBatch {
r.err = r.flush()
}
}
func (r *run) flush() error {
if len(r.batch) == 0 {
return nil
}
n, err := r.svc.listens.RecordMany(r.ctx, r.userID, r.batch)
if err != nil {
return fmt.Errorf("record the listens: %w", err)
}
r.created += n
r.batch = r.batch[:0]
return nil
}
func (r *run) playlist(p Playlist) { r.playlists = append(r.playlists, p) }
// kind says what a play was. The file says it when it can: a skip, or a play that ran to its end. Otherwise it is a
// complete listen when nine tenths of the song were played, a skip when it did not last as long as it takes to count
// as a listen, and a play for the rest.
func (s *Service) kind(e Entry, played, length int64) listening.EventType {
switch {
case e.Skipped:
return listening.Skip
case e.ReasonEnd == "trackdone":
return listening.Complete
case length > 0 && played*completeShareDen >= length*completeShareNum:
return listening.Complete
case played < s.minListen.Milliseconds():
return listening.Skip
}
return listening.Play
}
// idempotencyKey names a play so that the same play is never recorded twice: by the minute it ended (the older files
// only say the minute), the artist and the title. The same play in the extended history and in the older one is one.
func idempotencyKey(e Entry) string {
sum := sha256.Sum256([]byte(fmt.Sprintf("%d|%s|%s", e.PlayedAt.Unix()/60, strings.ToLower(e.Artist), strings.ToLower(e.Title))))
return "spotify:" + hex.EncodeToString(sum[:])[:40]
}
func (r *run) makePlaylists() error {
if len(r.playlists) == 0 {
return nil
}
have, err := r.playlistNames()
if err != nil {
return err
}
limit := false
for _, p := range r.playlists {
name := cleanName(p.Name)
res := PlaylistResult{Name: name, Tracks: len(p.Items) + p.Other}
var ids []int64
for _, it := range p.Items {
if id, ok := r.find(it.Artist, it.Title, it.Album); ok {
ids = append(ids, id)
} else {
r.miss(it.Artist)
}
}
res.Matched = len(ids)
switch {
case len(ids) == 0:
res.Status = "empty"
case have[strings.ToLower(name)]:
res.Status = "exists"
case limit:
res.Status = "limit"
default:
created, err := r.fill(name, ids)
if err != nil {
return err
}
if created {
res.Status = "created"
have[strings.ToLower(name)] = true
} else {
res.Status, limit = "limit", true
}
}
r.report.Playlists = append(r.report.Playlists, res)
}
return nil
}
// fill makes a playlist and puts the songs in it, a piece at a time. It reports false when the person already has as many
// playlists as are allowed.
func (r *run) fill(name string, ids []int64) (bool, error) {
pl, err := r.svc.lists.Create(r.ctx, r.userID, name, "")
if errors.Is(err, playlists.ErrTooMany) {
return false, nil
}
if err != nil {
return false, fmt.Errorf("make the playlist %q: %w", name, err)
}
if len(ids) > playlists.MaxItems {
ids = ids[:playlists.MaxItems]
}
for len(ids) > 0 {
n := min(len(ids), playlists.MaxAddAtOnce)
if _, err := r.svc.lists.AddTracks(r.ctx, r.userID, pl.ID, ids[:n], nil); err != nil {
return false, fmt.Errorf("fill the playlist %q: %w", name, err)
}
ids = ids[n:]
}
return true, nil
}
// playlistNames are the names the person already has, in lower case.
func (r *run) playlistNames() (map[string]bool, error) {
have := map[string]bool{}
for offset := 0; ; {
page, err := r.svc.lists.List(r.ctx, r.userID, playlists.Page{Limit: playlists.MaxLimit, Offset: offset})
if err != nil {
return nil, fmt.Errorf("read the playlists: %w", err)
}
for _, p := range page.Items {
have[strings.ToLower(strings.TrimSpace(p.Name))] = true
}
offset += len(page.Items)
if len(page.Items) == 0 || offset >= page.Total {
return have, nil
}
}
}
func cleanName(name string) string {
name = strings.TrimSpace(name)
if name == "" {
return unnamedPlaylist
}
if utf8.RuneCountInString(name) > playlists.MaxNameLen {
name = string([]rune(name)[:playlists.MaxNameLen])
}
return name
}
func (r *run) finish() Report {
h := &r.report.History
h.Matched = r.matched
h.Imported = r.created
h.AlreadyThere = r.matched - r.created
all := make([]Missing, 0, len(r.missing))
for _, m := range r.missing {
all = append(all, *m)
}
sort.Slice(all, func(i, j int) bool {
if all[i].Count != all[j].Count {
return all[i].Count > all[j].Count
}
return strings.ToLower(all[i].Artist) < strings.ToLower(all[j].Artist)
})
r.report.MissingTotal = len(all)
if len(all) > MaxMissing {
all = all[:MaxMissing]
}
r.report.Missing = all
return r.report
}

View file

@ -0,0 +1,363 @@
package spotifyimport
import (
"bytes"
"context"
"errors"
"fmt"
"strings"
"sync"
"testing"
"time"
"codeberg.org/squidd/sonda/internal/listening"
"codeberg.org/squidd/sonda/internal/playlists"
)
type fakeCatalog struct {
songs []Candidate
once sync.Once
entered chan struct{} // closed when Candidates is first called, when set
release chan struct{} // Candidates waits for it, when set
}
func (f *fakeCatalog) Candidates(context.Context) ([]Candidate, error) {
if f.entered != nil {
f.once.Do(func() { close(f.entered) })
}
if f.release != nil {
<-f.release
}
return f.songs, nil
}
type fakeListens struct {
seen map[string]bool
events []listening.Event
calls int
}
func (f *fakeListens) RecordMany(_ context.Context, userID int64, events []listening.Event) (int, error) {
f.calls++
if f.seen == nil {
f.seen = map[string]bool{}
}
created := 0
for _, e := range events {
k := fmt.Sprint(userID, e.IdempotencyKey)
if !f.seen[k] {
f.seen[k] = true
f.events = append(f.events, e)
created++
}
}
return created, nil
}
type fakePlaylists struct {
existing []string
made map[string][]int64
order []string
max int // playlists a user may have; 0 is no limit
nextID int64
names map[int64]string
}
func (f *fakePlaylists) List(_ context.Context, _ int64, p playlists.Page) (playlists.Listing, error) {
var all []playlists.Playlist
for _, n := range f.existing {
all = append(all, playlists.Playlist{Name: n})
}
end := min(p.Offset+p.Limit, len(all))
start := min(p.Offset, end)
return playlists.Listing{Items: all[start:end], Total: len(all), Limit: p.Limit, Offset: p.Offset}, nil
}
func (f *fakePlaylists) Create(_ context.Context, _ int64, name, _ string) (playlists.Playlist, error) {
if f.max > 0 && len(f.existing)+len(f.made) >= f.max {
return playlists.Playlist{}, playlists.ErrTooMany
}
if f.made == nil {
f.made, f.names = map[string][]int64{}, map[int64]string{}
}
f.nextID++
f.names[f.nextID] = name
f.made[name] = nil
f.order = append(f.order, name)
return playlists.Playlist{ID: f.nextID, Name: name}, nil
}
func (f *fakePlaylists) AddTracks(_ context.Context, _ int64, id int64, ids []int64, _ *int) (playlists.Playlist, error) {
if len(ids) > playlists.MaxAddAtOnce {
return playlists.Playlist{}, errors.New("too many at once")
}
f.made[f.names[id]] = append(f.made[f.names[id]], ids...)
return playlists.Playlist{ID: id}, nil
}
var songs = []Candidate{
{TrackID: 1, Title: "Karma Police", Album: "OK Computer", Artists: []string{"Radiohead"}, DurationMs: 260_000},
{TrackID: 2, Title: "Airbag", Album: "OK Computer", Artists: []string{"Radiohead"}, DurationMs: 284_000},
{TrackID: 3, Title: "Hyperballad", Album: "Post", Artists: []string{"Björk"}, DurationMs: 321_000},
}
type rig struct {
svc *Service
li *fakeListens
pl *fakePlaylists
}
func newRig(cat *fakeCatalog) rig {
li, pl := &fakeListens{}, &fakePlaylists{}
return rig{NewService(cat, li, pl, key, 20*time.Second), li, pl}
}
func upload(data string) File { return File{R: bytes.NewReader([]byte(data)), Size: int64(len(data))} }
func play(ts, artist, title, album string, ms int, extra string) string {
return fmt.Sprintf(`{"ts":%q,"ms_played":%d,"master_metadata_track_name":%q,"master_metadata_album_artist_name":%q,"master_metadata_album_album_name":%q%s}`,
ts, ms, title, artist, album, extra)
}
func history(plays ...string) string { return "[" + strings.Join(plays, ",") + "]" }
func TestImportRecordsTheListensThatMatchTheLibrary(t *testing.T) {
r := newRig(&fakeCatalog{songs: songs})
rep, err := r.svc.Import(context.Background(), 7, []File{upload(history(
play("2023-05-01T10:00:00Z", "Radiohead", "Karma Police", "OK Computer", 260_000, `,"reason_end":"trackdone"`),
play("2023-05-01T10:05:00Z", "Radiohead", "Airbag", "OK Computer", 4_000, `,"reason_end":"fwdbtn","skipped":true`),
play("2023-05-01T10:06:00Z", "Radiohead", "Creep", "Pablo Honey", 200_000, ``),
play("2023-05-01T10:10:00Z", "Björk", "Hyperballad", "Post", 321_000, ``),
`{"ts":"2023-05-01T11:00:00Z","ms_played":999,"master_metadata_track_name":null,"episode_name":"A show"}`,
))})
if err != nil {
t.Fatal(err)
}
h := rep.History
if h.Entries != 4 || h.Matched != 3 || h.Imported != 3 || h.AlreadyThere != 0 || h.NotFound != 1 || h.Ignored != 1 {
t.Errorf("history = %+v", h)
}
if len(r.li.events) != 3 {
t.Fatalf("%d events recorded, want 3", len(r.li.events))
}
first := r.li.events[0]
if first.UserID != 7 || first.TrackID != 1 || first.Type != listening.Complete || first.DurationPlayedMs != 260_000 ||
!first.OccurredAt.Equal(time.Date(2023, 5, 1, 10, 0, 0, 0, time.UTC)) || !strings.HasPrefix(first.IdempotencyKey, "spotify:") {
t.Errorf("first event = %+v", first)
}
if r.li.events[1].Type != listening.Skip {
t.Errorf("a skip was recorded as %q", r.li.events[1].Type)
}
if len(rep.Missing) != 1 || rep.Missing[0] != (Missing{Artist: "Radiohead", Count: 1}) {
t.Errorf("missing = %+v", rep.Missing)
}
}
func TestTheKindOfAPlayComesFromHowItEndedOrHowMuchWasHeard(t *testing.T) {
r := newRig(&fakeCatalog{songs: songs})
_, err := r.svc.Import(context.Background(), 1, []File{upload(history(
play("2023-01-01T10:00:00Z", "Radiohead", "Karma Police", "", 260_000, `,"reason_end":"trackdone"`), // ran to the end
play("2023-01-01T11:00:00Z", "Radiohead", "Karma Police", "", 250_000, `,"reason_end":"endplay"`), // 96 per cent, then stopped
play("2023-01-01T12:00:00Z", "Radiohead", "Karma Police", "", 90_000, `,"reason_end":"endplay"`), // a good part, then stopped
play("2023-01-01T13:00:00Z", "Radiohead", "Karma Police", "", 5_000, ``), // a few seconds, nothing said
play("2023-01-01T14:00:00Z", "Radiohead", "Karma Police", "", 90_000, `,"skipped":true`), // the file says skipped
play("2023-01-01T15:00:00Z", "Radiohead", "Karma Police", "", 9_999_999, `,"reason_end":"trackdone"`), // more than the song is long
))})
if err != nil {
t.Fatal(err)
}
want := []listening.EventType{listening.Complete, listening.Complete, listening.Play, listening.Skip, listening.Skip, listening.Complete}
for i, e := range r.li.events {
if e.Type != want[i] {
t.Errorf("play %d is %q, want %q", i, e.Type, want[i])
}
}
// A play longer than the song would be refused by the listening rules: it is brought to the length of the song.
if last := r.li.events[5]; last.DurationPlayedMs != 260_000 {
t.Errorf("duration = %d, want the length of the song", last.DurationPlayedMs)
}
}
func TestImportingTheSameFilesAgainAddsNothing(t *testing.T) {
r := newRig(&fakeCatalog{songs: songs})
data := history(
play("2023-05-01T10:00:00Z", "Radiohead", "Karma Police", "OK Computer", 260_000, ``),
play("2023-05-01T10:10:00Z", "Björk", "Hyperballad", "Post", 321_000, ``),
)
if _, err := r.svc.Import(context.Background(), 1, []File{upload(data)}); err != nil {
t.Fatal(err)
}
rep, err := r.svc.Import(context.Background(), 1, []File{upload(data)})
if err != nil {
t.Fatal(err)
}
if rep.History.Imported != 0 || rep.History.AlreadyThere != 2 || len(r.li.events) != 2 {
t.Errorf("second import: %+v, %d events", rep.History, len(r.li.events))
}
}
func TestTheSamePlayInTheTwoShapesOfFileIsCountedOnce(t *testing.T) {
r := newRig(&fakeCatalog{songs: songs})
extended := history(play("2022-01-02T08:30:41Z", "Björk", "Hyperballad", "Post", 321_000, `,"reason_end":"trackdone"`))
older := `[{"endTime":"2022-01-02 08:30","artistName":"Björk","trackName":"Hyperballad","msPlayed":321000}]`
rep, err := r.svc.Import(context.Background(), 1, []File{upload(extended), upload(older)})
if err != nil {
t.Fatal(err)
}
if len(r.li.events) != 1 || rep.History.Imported != 1 || rep.History.AlreadyThere != 1 {
t.Errorf("%d events, %+v: want the play once", len(r.li.events), rep.History)
}
}
func TestPlaylistsBecomePlaylistsWithTheSongsThatWereFound(t *testing.T) {
r := newRig(&fakeCatalog{songs: songs})
r.pl.existing = []string{"Mine already"}
rep, err := r.svc.Import(context.Background(), 1, []File{upload(`{"playlists":[
{"name":"Road","items":[
{"track":{"trackName":"Hyperballad","artistName":"Björk","albumName":"Post"}},
{"track":{"trackName":"Creep","artistName":"Radiohead","albumName":"Pablo Honey"}},
{"track":{"trackName":"Karma Police","artistName":"Radiohead","albumName":"OK Computer"}},
{"track":null,"episode":{"name":"A show"}}]},
{"name":"mine ALREADY","items":[{"track":{"trackName":"Airbag","artistName":"Radiohead"}}]},
{"name":"Nothing here","items":[{"track":{"trackName":"Creep","artistName":"Radiohead"}}]},
{"name":"","items":[{"track":{"trackName":"Airbag","artistName":"Radiohead"}}]}
]}`)})
if err != nil {
t.Fatal(err)
}
if got := r.pl.made["Road"]; len(got) != 2 || got[0] != 3 || got[1] != 1 {
t.Errorf("Road has %v, want the found songs in the order of the playlist: 3, 1", got)
}
if _, made := r.pl.made["mine ALREADY"]; made {
t.Error("a playlist whose name the person already has was made again")
}
if _, made := r.pl.made["Nothing here"]; made {
t.Error("a playlist with none of its songs found was made")
}
if got := r.pl.made["Spotify playlist"]; len(got) != 1 {
t.Errorf("a playlist with no name got %v under %v", got, r.pl.order)
}
want := []PlaylistResult{
{Name: "Road", Tracks: 4, Matched: 2, Status: "created"},
{Name: "mine ALREADY", Tracks: 1, Matched: 1, Status: "exists"},
{Name: "Nothing here", Tracks: 1, Matched: 0, Status: "empty"},
{Name: "Spotify playlist", Tracks: 1, Matched: 1, Status: "created"},
}
if fmt.Sprint(rep.Playlists) != fmt.Sprint(want) {
t.Errorf("playlists = %+v, want %+v", rep.Playlists, want)
}
if len(rep.Missing) != 1 || rep.Missing[0].Artist != "Radiohead" || rep.Missing[0].Count != 2 {
t.Errorf("missing = %+v, want Radiohead twice (Creep in two playlists)", rep.Missing)
}
}
func TestALongPlaylistIsAddedInPieces(t *testing.T) {
r := newRig(&fakeCatalog{songs: songs})
var items []string
for i := 0; i < 1234; i++ {
items = append(items, `{"track":{"trackName":"Airbag","artistName":"Radiohead"}}`)
}
rep, err := r.svc.Import(context.Background(), 1, []File{upload(`{"playlists":[{"name":"Long","items":[` + strings.Join(items, ",") + `]}]}`)})
if err != nil {
t.Fatal(err)
}
if len(r.pl.made["Long"]) != 1234 || rep.Playlists[0].Status != "created" {
t.Errorf("%d songs, %+v", len(r.pl.made["Long"]), rep.Playlists)
}
}
func TestWhenThePersonHasAsManyPlaylistsAsAllowedTheRestAreSaidSo(t *testing.T) {
r := newRig(&fakeCatalog{songs: songs})
r.pl.max = 1
item := `{"track":{"trackName":"Airbag","artistName":"Radiohead"}}`
rep, err := r.svc.Import(context.Background(), 1, []File{upload(`{"playlists":[{"name":"A","items":[` + item + `]},{"name":"B","items":[` + item + `]},{"name":"C","items":[` + item + `]}]}`)})
if err != nil {
t.Fatal(err)
}
got := []string{rep.Playlists[0].Status, rep.Playlists[1].Status, rep.Playlists[2].Status}
if fmt.Sprint(got) != "[created limit limit]" {
t.Errorf("statuses = %v", got)
}
}
func TestTheArtistsThatAreMissingAreListedMostPlayedFirst(t *testing.T) {
r := newRig(&fakeCatalog{songs: songs})
var plays []string
for i := 0; i < 5; i++ {
plays = append(plays, play(fmt.Sprintf("2023-01-01T10:0%d:00Z", i), "Blur", "Song 2", "", 120_000, ``))
}
for i := 0; i < 3; i++ {
plays = append(plays, play(fmt.Sprintf("2023-01-02T10:0%d:00Z", i), "oasis", "Wonderwall", "", 120_000, ``))
plays = append(plays, play(fmt.Sprintf("2023-01-03T10:0%d:00Z", i), "Oasis", "Supersonic", "", 120_000, ``))
}
rep, err := r.svc.Import(context.Background(), 1, []File{upload(history(plays...))})
if err != nil {
t.Fatal(err)
}
if len(rep.Missing) != 2 || rep.Missing[0] != (Missing{"Oasis", 6}) && rep.Missing[0] != (Missing{"oasis", 6}) || rep.Missing[1] != (Missing{"Blur", 5}) {
t.Errorf("missing = %+v, want Oasis 6 (the spellings together) then Blur 5", rep.Missing)
}
}
func TestOnlyTheMostPlayedMissingArtistsAreListed(t *testing.T) {
r := newRig(&fakeCatalog{songs: songs})
var plays []string
for i := 0; i < MaxMissing+30; i++ {
plays = append(plays, play("2023-01-01T10:00:00Z", fmt.Sprintf("Artist %03d", i), "Song", "", 120_000, ``))
}
rep, err := r.svc.Import(context.Background(), 1, []File{upload(history(plays...))})
if err != nil {
t.Fatal(err)
}
if len(rep.Missing) != MaxMissing || rep.MissingTotal != MaxMissing+30 {
t.Errorf("%d listed of %d, want %d of %d", len(rep.Missing), rep.MissingTotal, MaxMissing, MaxMissing+30)
}
}
func TestFilesWithNothingOfSpotifyAreRefused(t *testing.T) {
r := newRig(&fakeCatalog{songs: songs})
_, err := r.svc.Import(context.Background(), 1, []File{upload(`{"hello":"world"}`), upload("words")})
if !errors.Is(err, ErrNothingFound) {
t.Errorf("err = %v, want ErrNothingFound", err)
}
if len(r.li.events) != 0 {
t.Error("something was recorded")
}
}
func TestAnImportThatIsRunningBlocksAnotherOfTheSamePerson(t *testing.T) {
cat := &fakeCatalog{songs: songs, entered: make(chan struct{}), release: make(chan struct{})}
r := newRig(cat)
done := make(chan error, 1)
go func() {
_, err := r.svc.Import(context.Background(), 5, []File{upload(history(play("2023-01-01T10:00:00Z", "Radiohead", "Airbag", "", 1000, ``)))})
done <- err
}()
<-cat.entered
if _, err := r.svc.Import(context.Background(), 5, []File{upload(`[]`)}); !errors.Is(err, ErrBusy) {
t.Errorf("a second import of the same person: %v, want ErrBusy", err)
}
close(cat.release)
if err := <-done; err != nil {
t.Fatal(err)
}
// Once it is over, the person can import again, and somebody else could all along.
if _, err := r.svc.Import(context.Background(), 5, []File{upload(`[]`)}); errors.Is(err, ErrBusy) {
t.Error("still busy after the import ended")
}
}
func TestManyListensAreRecordedInBatches(t *testing.T) {
r := newRig(&fakeCatalog{songs: songs})
var plays []string
for i := 0; i < 2500; i++ {
plays = append(plays, play(time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC).Add(time.Duration(i)*time.Minute).Format(time.RFC3339), "Radiohead", "Airbag", "", 100_000, ``))
}
rep, err := r.svc.Import(context.Background(), 1, []File{upload(history(plays...))})
if err != nil {
t.Fatal(err)
}
if rep.History.Imported != 2500 || r.li.calls < 2 {
t.Errorf("imported %d in %d calls, want 2500 in several", rep.History.Imported, r.li.calls)
}
}

View file

@ -11,6 +11,7 @@ import (
"github.com/uptrace/bun/dialect"
"codeberg.org/squidd/sonda/internal/catalog"
"codeberg.org/squidd/sonda/internal/spotifyimport"
)
// CatalogRepo reads the catalog. It implements catalog.Store.
@ -569,3 +570,37 @@ func normalizeFormat(f string) string {
}
return f
}
// Candidates lists every song that has a file to play, with its album, its credited artists and its length, in order of
// id: what the import of a Spotify export compares its plays with.
func (r *CatalogRepo) Candidates(ctx context.Context) ([]spotifyimport.Candidate, error) {
rows, err := r.db.QueryContext(ctx,
`SELECT t.id, t.title, COALESCE(al.title, ''), COALESCE(t.duration_ms, 0), a.name
FROM tracks t
JOIN track_artists ta ON ta.track_id = t.id
JOIN artists a ON a.id = ta.artist_id
LEFT JOIN albums al ON al.id = t.album_id
WHERE EXISTS (SELECT 1 FROM audio_files f WHERE f.track_id = t.id AND f.missing_at IS NULL)
ORDER BY t.id, a.id`)
if err != nil {
return nil, fmt.Errorf("list the songs: %w", err)
}
defer rows.Close()
var out []spotifyimport.Candidate
for rows.Next() {
var (
id, duration int64
title, album string
artist string
)
if err := rows.Scan(&id, &title, &album, &duration, &artist); err != nil {
return nil, err
}
if n := len(out); n > 0 && out[n-1].TrackID == id {
out[n-1].Artists = append(out[n-1].Artists, artist)
continue
}
out = append(out, spotifyimport.Candidate{TrackID: id, Title: title, Album: album, DurationMs: duration, Artists: []string{artist}})
}
return out, rows.Err()
}

View file

@ -78,6 +78,42 @@ func (r *ListeningRepo) Record(ctx context.Context, e listening.Event) (bool, er
return n > 0, nil
}
// recordChunk is how many events RecordMany inserts with one statement: seven columns each, well below the limit of
// variables of either database.
const recordChunk = 500
// RecordMany inserts events of one user in one transaction, and says how many were new: an event whose idempotency key
// is already there (also one that is twice in the batch) is skipped. The tracks must exist.
func (r *ListeningRepo) RecordMany(ctx context.Context, userID int64, events []listening.Event) (int, error) {
created := 0
err := r.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
for start := 0; start < len(events); start += recordChunk {
end := min(start+recordChunk, len(events))
rows := make([]listeningRow, 0, end-start)
for _, e := range events[start:end] {
rows = append(rows, listeningRow{
UserID: userID, TrackID: e.TrackID, EventType: string(e.Type), OccurredAt: e.OccurredAt.Unix(),
DurationPlayedMs: e.DurationPlayedMs, IdempotencyKey: e.IdempotencyKey,
})
}
res, err := tx.NewInsert().Model(&rows).On("CONFLICT (user_id, idempotency_key) DO NOTHING").Exec(ctx)
if err != nil {
return fmt.Errorf("insert listening events: %w", err)
}
n, err := res.RowsAffected()
if err != nil {
return fmt.Errorf("rows affected: %w", err)
}
created += int(n)
}
return nil
})
if err != nil {
return 0, err
}
return created, nil
}
// TrackDurationMs returns the length of a track in milliseconds, 0 when it is
// not known, or listening.ErrTrackNotFound.
func (r *ListeningRepo) TrackDurationMs(ctx context.Context, trackID int64) (int64, error) {

View file

@ -34,3 +34,7 @@ func searchKey(s string) string {
}
return searchLetters.Replace(norm.NFC.String(b.String()))
}
// SearchKey is searchKey for the packages that compare names with the ones in the library (the import of a Spotify
// export): the same key, so that "Björk" and "Bjork" are one name there too.
func SearchKey(s string) string { return searchKey(s) }

View file

@ -0,0 +1,116 @@
package storage
import (
"context"
"fmt"
"sort"
"testing"
"time"
"github.com/uptrace/bun"
"codeberg.org/squidd/sonda/internal/library"
"codeberg.org/squidd/sonda/internal/listening"
)
func TestRecordManyStoresTheListensOnceAndCountsWhatIsNew(t *testing.T) {
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
ctx := context.Background()
u, err := NewUserRepo(db).CreateUser(ctx, "alice", "h", false)
if err != nil {
t.Fatal(err)
}
var track int64
if err := db.QueryRow("INSERT INTO tracks (title) VALUES ('t') RETURNING id").Scan(&track); err != nil {
t.Fatal(err)
}
repo := NewListeningRepo(db)
ev := func(key string, at int64) listening.Event {
return listening.Event{UserID: u.ID, TrackID: track, Type: listening.Complete, OccurredAt: time.Unix(at, 0), DurationPlayedMs: 1000, IdempotencyKey: key}
}
batch := []listening.Event{ev("a", 100), ev("b", 200), ev("a", 100)} // the same key twice in one batch
n, err := repo.RecordMany(ctx, u.ID, batch)
if err != nil || n != 2 {
t.Fatalf("RecordMany = %d, %v, want 2 new", n, err)
}
n, err = repo.RecordMany(ctx, u.ID, []listening.Event{ev("a", 100), ev("c", 300)})
if err != nil || n != 1 {
t.Fatalf("second RecordMany = %d, %v, want only the new one", n, err)
}
if got := count(t, db, "SELECT COUNT(*) FROM listening_events WHERE user_id = ?", u.ID); got != 3 {
t.Errorf("%d events stored, want 3", got)
}
if n, err := repo.RecordMany(ctx, u.ID, nil); err != nil || n != 0 {
t.Errorf("an empty batch = %d, %v", n, err)
}
})
}
func TestRecordManyHandlesMoreThanOneStatementHolds(t *testing.T) {
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
ctx := context.Background()
u, err := NewUserRepo(db).CreateUser(ctx, "alice", "h", false)
if err != nil {
t.Fatal(err)
}
var track int64
if err := db.QueryRow("INSERT INTO tracks (title) VALUES ('t') RETURNING id").Scan(&track); err != nil {
t.Fatal(err)
}
var events []listening.Event
for i := 0; i < 5000; i++ {
events = append(events, listening.Event{UserID: u.ID, TrackID: track, Type: listening.Play, OccurredAt: time.Unix(int64(1000+i), 0), IdempotencyKey: fmt.Sprintf("k%d", i)})
}
n, err := NewListeningRepo(db).RecordMany(ctx, u.ID, events)
if err != nil || n != 5000 {
t.Fatalf("RecordMany = %d, %v, want 5000", n, err)
}
})
}
func TestCandidatesAreTheSongsThatCanBePlayed(t *testing.T) {
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
ctx := context.Background()
repo := NewLibraryRepo(db)
lib := mkLib(t, repo, "music")
a := parsed("a/1.flac", "Under Pressure", "Hot Space", "Queen", "David Bowie")
a.DurationMs = 248000
b := parsed("a/2.flac", "Gone", "Album", "Radiohead")
c := parsed("a/3.flac", "Loose", "", "Björk")
c.Album, c.AlbumArtist = "", ""
for _, f := range []library.ParsedFile{a, b, c} {
if _, err := repo.SaveFile(ctx, lib, f, 0); err != nil {
t.Fatal(err)
}
}
if _, err := db.Exec("UPDATE audio_files SET missing_at = 5 WHERE rel_path = 'a/2.flac'"); err != nil {
t.Fatal(err)
}
got, err := NewCatalogRepo(db).Candidates(ctx)
if err != nil {
t.Fatal(err)
}
if len(got) != 2 {
t.Fatalf("%d candidates, want 2 (the one with no file left is not there): %+v", len(got), got)
}
by := map[string]int{}
for i, g := range got {
by[g.Title] = i
}
up := got[by["Under Pressure"]]
artists := append([]string(nil), up.Artists...)
sort.Strings(artists)
if up.Album != "Hot Space" || up.DurationMs != 248000 || len(artists) != 2 || artists[0] != "David Bowie" || artists[1] != "Queen" || up.TrackID == 0 {
t.Errorf("Under Pressure = %+v", up)
}
if l := got[by["Loose"]]; l.Album != "" || len(l.Artists) != 1 || l.Artists[0] != "Björk" {
t.Errorf("a song with no album = %+v", l)
}
})
}
func TestSearchKeyIsExported(t *testing.T) {
if SearchKey("Björk") != "bjork" {
t.Errorf("SearchKey(Björk) = %q", SearchKey("Björk"))
}
}