// Package spotifyimport brings the listening history and the playlists of a Spotify data export into Sonda. // // Spotify gives a person a zip of JSON files when they ask for their data. This package reads those files (the zip // itself, or the JSON files in it), matches each song to the library by artist and title, records the matches as // listens and makes the playlists. It runs on the server and contacts nobody: the files are read from the request, // never written anywhere, and the parts that a file holds besides songs (the IP addresses of the person, the country, // the platform, the user name) are never even looked at. package spotifyimport import ( "archive/zip" "bufio" "bytes" "encoding/json" "errors" "fmt" "io" "path" "strings" "time" ) // Limits on what one upload may hold, so that a zip that inflates into something enormous cannot fill the memory // or the disk of the server. var ( maxInflated = int64(4 << 30) // all the JSON of one upload, once it is inflated maxZipFiles = 2000 // files in one zip ) // ErrTooBig is returned for an upload that holds more than the limits allow. var ErrTooBig = errors.New("the files hold more than can be imported") // ErrUnreadable is returned for a file that looks like one of Spotify's and cannot be read to its end (cut short, // damaged). var ErrUnreadable = errors.New("a file is damaged or cut short") // Entry is one play of a song from a streaming history file. type Entry struct { PlayedAt time.Time // when the play ended Artist string Title string Album string // empty in the older files, which do not say PlayedMs int64 Skipped bool // the file says the listener skipped it ReasonEnd string // why the play ended ("trackdone" when it ran to the end), empty in the older files } // PlaylistItem is one song of a playlist. type PlaylistItem struct{ Artist, Title, Album string } // Playlist is a playlist of the export. Items are its songs, in order; Other counts what is in it that is not a song // (a podcast episode, a file the person had on their own computer). type Playlist struct { Name string Items []PlaylistItem Other int } // Reader reads files of an export and calls back for what is in them. A callback that is nil is skipped. type Reader struct { OnEntry func(Entry) OnPlaylist func(Playlist) // OnIgnored is called for a play that is not a song (a podcast episode) or that has no time that can be read. OnIgnored func() } // ReadFile reads one upload: the zip of the export, or one of the JSON files in it. It reports whether it held // something of Spotify's (a history or playlists); a file that holds none is not an error, it is just not found. func (r Reader) ReadFile(ra io.ReaderAt, size int64) (bool, error) { found, err := r.readFile(ra, size) if err != nil && !errors.Is(err, ErrTooBig) { err = fmt.Errorf("%w: %v", ErrUnreadable, err) } return found, err } func (r Reader) readFile(ra io.ReaderAt, size int64) (bool, error) { head := make([]byte, 4) if n, _ := ra.ReadAt(head, 0); n == 4 && bytes.Equal(head, []byte("PK\x03\x04")) { return r.readZip(ra, size) } budget := maxInflated return r.readJSON(&budgeted{r: io.NewSectionReader(ra, 0, size), left: &budget}) } func (r Reader) readZip(ra io.ReaderAt, size int64) (bool, error) { zr, err := zip.NewReader(ra, size) if err != nil { return false, nil // looks like a zip and is not one: nothing of Spotify's } if len(zr.File) > maxZipFiles { return false, ErrTooBig } budget := maxInflated found := false for _, f := range zr.File { if !wanted(f.Name) { continue } rc, err := f.Open() if err != nil { return found, fmt.Errorf("open %s: %w", path.Base(f.Name), err) } ok, err := r.readJSON(&budgeted{r: rc, left: &budget}) _ = rc.Close() if err != nil { return found, fmt.Errorf("read %s: %w", path.Base(f.Name), err) } found = found || ok } return found, nil } // wanted says whether a file of the zip can hold what is imported: the audio history (not the video one) and the // playlists. The rest of an export (the account, the searches, the library) is never opened. func wanted(name string) bool { base := strings.ToLower(path.Base(name)) if !strings.HasSuffix(base, ".json") || strings.Contains(base, "video") { return false } return strings.HasPrefix(base, "streaming_history") || strings.HasPrefix(base, "streaminghistory") || strings.HasPrefix(base, "playlist") } // budgeted stops a reader that gives more than the budget left for the whole upload. type budgeted struct { r io.Reader left *int64 } func (b *budgeted) Read(p []byte) (int, error) { if *b.left <= 0 { return 0, ErrTooBig } if int64(len(p)) > *b.left { p = p[:*b.left] } n, err := b.r.Read(p) *b.left -= int64(n) return n, err } // rawEntry has the fields of a play that are used, in both shapes of file: the extended history (ts, ms_played, // master_metadata_...) and the older one (endTime, msPlayed, trackName). Nothing else is decoded. type rawEntry struct { TS string `json:"ts"` MsPlayed int64 `json:"ms_played"` Track *string `json:"master_metadata_track_name"` Artist *string `json:"master_metadata_album_artist_name"` Album *string `json:"master_metadata_album_album_name"` ReasonEnd string `json:"reason_end"` Skipped *bool `json:"skipped"` EndTime string `json:"endTime"` OldArtist string `json:"artistName"` OldTrack string `json:"trackName"` OldMsPlayed int64 `json:"msPlayed"` } type rawPlaylists struct { Playlists *[]struct { Name string `json:"name"` Items []struct { Track *struct { Name string `json:"trackName"` Artist string `json:"artistName"` Album string `json:"albumName"` } `json:"track"` } `json:"items"` } `json:"playlists"` } func (r Reader) readJSON(in io.Reader) (bool, error) { br := bufio.NewReaderSize(in, 64<<10) first, err := firstByte(br) if err != nil { if errors.Is(err, ErrTooBig) { return false, err } return false, nil // empty, or not readable: nothing of Spotify's } dec := json.NewDecoder(br) switch first { case '[': return r.readHistory(dec) case '{': return r.readPlaylists(dec) } return false, nil } // firstByte looks at the first byte that is not a space, without taking it. func firstByte(br *bufio.Reader) (byte, error) { for { b, err := br.ReadByte() if err != nil { return 0, err } if b == ' ' || b == '\n' || b == '\r' || b == '\t' || b == 0xEF || b == 0xBB || b == 0xBF { // spaces and a byte order mark continue } return b, br.UnreadByte() } } func (r Reader) readHistory(dec *json.Decoder) (bool, error) { if _, err := dec.Token(); err != nil { // the [ return false, nil } found := false for dec.More() { var raw rawEntry if err := dec.Decode(&raw); err != nil { var typeErr *json.UnmarshalTypeError if errors.As(err, &typeErr) { continue // an element that is not an object: not a play } return found, err } stamp := raw.TS if stamp == "" { stamp = raw.EndTime } if stamp == "" { continue // not a play at all: a file that only looks like a history } found = true e, ok := raw.entry(stamp) if !ok { if r.OnIgnored != nil { r.OnIgnored() } continue } if r.OnEntry != nil { r.OnEntry(e) } } return found, nil } // entry is the play, or false for something that is not a song or has no time that can be read. func (raw rawEntry) entry(stamp string) (Entry, bool) { at, ok := parseTime(stamp) if !ok { return Entry{}, false } e := Entry{PlayedAt: at, PlayedMs: raw.MsPlayed, ReasonEnd: raw.ReasonEnd} if raw.Skipped != nil { e.Skipped = *raw.Skipped } if raw.Track != nil { e.Title = *raw.Track } if raw.Artist != nil { e.Artist = *raw.Artist } if raw.Album != nil { e.Album = *raw.Album } if e.Title == "" && raw.OldTrack != "" { // the older shape e.Title, e.Artist, e.PlayedMs = raw.OldTrack, raw.OldArtist, raw.OldMsPlayed } e.Title, e.Artist, e.Album = strings.TrimSpace(e.Title), strings.TrimSpace(e.Artist), strings.TrimSpace(e.Album) if e.Title == "" || e.Artist == "" || e.PlayedMs < 0 { return Entry{}, false // an episode of a podcast has no song name } return e, true } // parseTime reads the time of a play: an RFC 3339 time in the extended history, "2022-01-02 08:30" (UTC) in the older one. func parseTime(s string) (time.Time, bool) { for _, layout := range []string{time.RFC3339, "2006-01-02 15:04", "2006-01-02 15:04:05"} { if t, err := time.ParseInLocation(layout, strings.TrimSpace(s), time.UTC); err == nil { return t.UTC(), true } } return time.Time{}, false } func (r Reader) readPlaylists(dec *json.Decoder) (bool, error) { var raw rawPlaylists if err := dec.Decode(&raw); err != nil { if errors.Is(err, ErrTooBig) { return false, err } return false, nil } if raw.Playlists == nil { return false, nil } for _, p := range *raw.Playlists { out := Playlist{Name: strings.TrimSpace(p.Name)} for _, it := range p.Items { if it.Track == nil || strings.TrimSpace(it.Track.Name) == "" || strings.TrimSpace(it.Track.Artist) == "" { out.Other++ continue } out.Items = append(out.Items, PlaylistItem{ Artist: strings.TrimSpace(it.Track.Artist), Title: strings.TrimSpace(it.Track.Name), Album: strings.TrimSpace(it.Track.Album), }) } if r.OnPlaylist != nil { r.OnPlaylist(out) } } return true, nil }