I want my listening history and my playlists from Spotify in Sonda, so that Mixes, Discover and the smart playlists know what I listened to before. - POST /me/import/spotify takes the zip of a Spotify data export, or the JSON files in it (extended streaming history, the older StreamingHistory files, the playlists), as a multipart form. A request may be up to 512 MB; the zip is limited in how much it may inflate and how many files it may hold, and only the history and playlist files are opened. - Everything is read on the server and nothing is sent anywhere. The files are not kept, and of each play only the song, artist, album, time and how it ended are read: never the IP addresses, the country or the platform. - A play is matched to a song of the library by artist and title (no case, accents or punctuation; a "Remastered" note on either side; any credited artist; a compilation by its album). A match is recorded as a listen at the time of the play: complete, skip or play. Nothing is posted to Mastodon. - Importing the same files again records nothing twice, and the same play in the two shapes of history file is counted once. One import per person at a time. - A playlist becomes a playlist of mine with the songs that were found; one with a name I already have is left alone, and one with none of its songs found is not made. - The report counts what was added, what was already there and what is not in the library, and lists the missing artists, most played first. - Settings gets an Import from Spotify section; the API client can send files. - English and Italian.
183 lines
6.6 KiB
Go
183 lines
6.6 KiB
Go
package spotifyimport
|
|
|
|
import (
|
|
"archive/zip"
|
|
"bytes"
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
const extended = `[
|
|
{"ts":"2023-05-01T10:00:00Z","username":"someone","platform":"linux","ms_played":215000,"conn_country":"IT","ip_addr":"203.0.113.9",
|
|
"master_metadata_track_name":"Karma Police","master_metadata_album_artist_name":"Radiohead","master_metadata_album_album_name":"OK Computer",
|
|
"spotify_track_uri":"spotify:track:1","reason_start":"trackdone","reason_end":"trackdone","shuffle":false,"skipped":null},
|
|
{"ts":"2023-05-01T10:04:00Z","ms_played":4000,"master_metadata_track_name":"Airbag","master_metadata_album_artist_name":"Radiohead",
|
|
"master_metadata_album_album_name":"OK Computer","reason_end":"fwdbtn","skipped":true},
|
|
{"ts":"2023-05-01T11:00:00Z","ms_played":900000,"master_metadata_track_name":null,"master_metadata_album_artist_name":null,
|
|
"episode_name":"Episode 12","episode_show_name":"A podcast","spotify_episode_uri":"spotify:episode:9"}
|
|
]`
|
|
|
|
const legacy = `[
|
|
{"endTime":"2022-01-02 08:30","artistName":"Björk","trackName":"Hyperballad","msPlayed":321000},
|
|
{"endTime":"not a time","artistName":"X","trackName":"Y","msPlayed":1}
|
|
]`
|
|
|
|
const playlistsFile = `{"playlists":[
|
|
{"name":"Road","lastModifiedDate":"2023-01-01","items":[
|
|
{"track":{"trackName":"Karma Police","artistName":"Radiohead","albumName":"OK Computer","trackUri":"spotify:track:1"},"episode":null,"localTrack":null,"addedDate":"2020-01-01"},
|
|
{"track":null,"episode":{"name":"An episode"},"localTrack":null},
|
|
{"track":null,"episode":null,"localTrack":{"uri":"spotify:local:::a:1"}}
|
|
]},
|
|
{"name":"Empty","items":[]}
|
|
]}`
|
|
|
|
type got struct {
|
|
entries []Entry
|
|
playlists []Playlist
|
|
ignored int
|
|
}
|
|
|
|
func read(t *testing.T, data []byte) (got, bool) {
|
|
t.Helper()
|
|
var g got
|
|
r := Reader{
|
|
OnEntry: func(e Entry) { g.entries = append(g.entries, e) },
|
|
OnPlaylist: func(p Playlist) { g.playlists = append(g.playlists, p) },
|
|
OnIgnored: func() { g.ignored++ },
|
|
}
|
|
found, err := r.ReadFile(bytes.NewReader(data), int64(len(data)))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return g, found
|
|
}
|
|
|
|
func TestReadsTheExtendedStreamingHistory(t *testing.T) {
|
|
g, found := read(t, []byte(extended))
|
|
if !found || len(g.entries) != 2 || g.ignored != 1 {
|
|
t.Fatalf("found=%v entries=%d ignored=%d, want true, 2 and 1 (the podcast)", found, len(g.entries), g.ignored)
|
|
}
|
|
e := g.entries[0]
|
|
if e.Title != "Karma Police" || e.Artist != "Radiohead" || e.Album != "OK Computer" || e.PlayedMs != 215000 || e.ReasonEnd != "trackdone" || e.Skipped {
|
|
t.Errorf("first entry = %+v", e)
|
|
}
|
|
if !e.PlayedAt.Equal(time.Date(2023, 5, 1, 10, 0, 0, 0, time.UTC)) {
|
|
t.Errorf("played at %v", e.PlayedAt)
|
|
}
|
|
if s := g.entries[1]; !s.Skipped || s.ReasonEnd != "fwdbtn" {
|
|
t.Errorf("second entry = %+v, want a skip", s)
|
|
}
|
|
}
|
|
|
|
func TestReadsTheOlderStreamingHistoryAndDropsWhatHasNoTime(t *testing.T) {
|
|
g, found := read(t, []byte(legacy))
|
|
if !found || len(g.entries) != 1 {
|
|
t.Fatalf("found=%v entries=%d, want true and 1", found, len(g.entries))
|
|
}
|
|
e := g.entries[0]
|
|
if e.Artist != "Björk" || e.Title != "Hyperballad" || e.Album != "" || e.PlayedMs != 321000 {
|
|
t.Errorf("entry = %+v", e)
|
|
}
|
|
if !e.PlayedAt.Equal(time.Date(2022, 1, 2, 8, 30, 0, 0, time.UTC)) {
|
|
t.Errorf("played at %v", e.PlayedAt)
|
|
}
|
|
if g.ignored != 1 {
|
|
t.Errorf("ignored = %d, want the entry with no readable time counted", g.ignored)
|
|
}
|
|
}
|
|
|
|
func TestReadsThePlaylists(t *testing.T) {
|
|
g, found := read(t, []byte(playlistsFile))
|
|
if !found || len(g.playlists) != 2 {
|
|
t.Fatalf("found=%v playlists=%d", found, len(g.playlists))
|
|
}
|
|
p := g.playlists[0]
|
|
if p.Name != "Road" || len(p.Items) != 1 || p.Other != 2 {
|
|
t.Fatalf("playlist = %+v, want one song and two that are not", p)
|
|
}
|
|
if it := p.Items[0]; it.Title != "Karma Police" || it.Artist != "Radiohead" || it.Album != "OK Computer" {
|
|
t.Errorf("item = %+v", it)
|
|
}
|
|
}
|
|
|
|
func TestSomethingThatIsNotSpotifyDataIsNotFound(t *testing.T) {
|
|
for name, data := range map[string]string{
|
|
"other json": `{"hello":"world"}`,
|
|
"other array": `[{"query":"x","time":"y"}]`,
|
|
"not json": `just words`,
|
|
"empty": ``,
|
|
"a number": `42`,
|
|
"a null playlist": `{"playlists":null}`,
|
|
} {
|
|
g, found := read(t, []byte(data))
|
|
if found || len(g.entries) != 0 || len(g.playlists) != 0 {
|
|
t.Errorf("%s: found=%v %+v, want nothing", name, found, g)
|
|
}
|
|
}
|
|
}
|
|
|
|
func makeZip(t *testing.T, files map[string]string) []byte {
|
|
t.Helper()
|
|
var buf bytes.Buffer
|
|
w := zip.NewWriter(&buf)
|
|
for name, body := range files {
|
|
f, err := w.Create(name)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := f.Write([]byte(body)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := w.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return buf.Bytes()
|
|
}
|
|
|
|
func TestReadsTheZipOfTheExport(t *testing.T) {
|
|
data := makeZip(t, map[string]string{
|
|
"Spotify Extended Streaming History/Streaming_History_Audio_2019-2023_0.json": extended,
|
|
"Spotify Extended Streaming History/Streaming_History_Video_2019-2023.json": extended, // videos are not music
|
|
"Spotify Account Data/StreamingHistory_music_0.json": legacy,
|
|
"Spotify Account Data/Playlist1.json": playlistsFile,
|
|
"Spotify Account Data/Userdata.json": `{"username":"someone","email":"someone@example.org"}`,
|
|
"Spotify Account Data/SearchQueries.json": `[{"platform":"x","searchTime":"y","searchQuery":"z"}]`,
|
|
"README.pdf": "ignored",
|
|
})
|
|
g, found := read(t, data)
|
|
if !found {
|
|
t.Fatal("the zip of an export was not recognised")
|
|
}
|
|
if len(g.entries) != 3 || len(g.playlists) != 2 {
|
|
t.Errorf("entries=%d playlists=%d, want 3 (two extended, one older) and 2", len(g.entries), len(g.playlists))
|
|
}
|
|
}
|
|
|
|
func TestAZipWithNothingOfSpotifyIsNotFound(t *testing.T) {
|
|
_, found := read(t, makeZip(t, map[string]string{"notes.txt": "hello", "Userdata.json": `{"a":1}`}))
|
|
if found {
|
|
t.Error("a zip with no history and no playlists was taken for an export")
|
|
}
|
|
}
|
|
|
|
func TestAZipThatInflatesTooMuchIsRefused(t *testing.T) {
|
|
old := maxInflated
|
|
maxInflated = 100
|
|
defer func() { maxInflated = old }()
|
|
data := makeZip(t, map[string]string{"Streaming_History_Audio_0.json": "[" + strings.Repeat(" ", 1000) + "]"})
|
|
r := Reader{}
|
|
if _, err := r.ReadFile(bytes.NewReader(data), int64(len(data))); !errors.Is(err, ErrTooBig) {
|
|
t.Errorf("err = %v, want ErrTooBig for a file that inflates past the limit", err)
|
|
}
|
|
}
|
|
|
|
func TestAFileThatIsCutShortIsUnreadable(t *testing.T) {
|
|
data := []byte(extended[:len(extended)-40])
|
|
_, err := Reader{}.ReadFile(bytes.NewReader(data), int64(len(data)))
|
|
if !errors.Is(err, ErrUnreadable) {
|
|
t.Errorf("err = %v, want ErrUnreadable", err)
|
|
}
|
|
}
|