I want my listening history and my playlists from Spotify in Sonda, so that Mixes, Discover and the smart playlists know what I listened to before. - POST /me/import/spotify takes the zip of a Spotify data export, or the JSON files in it (extended streaming history, the older StreamingHistory files, the playlists), as a multipart form. A request may be up to 512 MB; the zip is limited in how much it may inflate and how many files it may hold, and only the history and playlist files are opened. - Everything is read on the server and nothing is sent anywhere. The files are not kept, and of each play only the song, artist, album, time and how it ended are read: never the IP addresses, the country or the platform. - A play is matched to a song of the library by artist and title (no case, accents or punctuation; a "Remastered" note on either side; any credited artist; a compilation by its album). A match is recorded as a listen at the time of the play: complete, skip or play. Nothing is posted to Mastodon. - Importing the same files again records nothing twice, and the same play in the two shapes of history file is counted once. One import per person at a time. - A playlist becomes a playlist of mine with the songs that were found; one with a name I already have is left alone, and one with none of its songs found is not made. - The report counts what was added, what was already there and what is not in the library, and lists the missing artists, most played first. - Settings gets an Import from Spotify section; the API client can send files. - English and Italian.
307 lines
9.2 KiB
Go
307 lines
9.2 KiB
Go
// Package spotifyimport brings the listening history and the playlists of a Spotify data export into Sonda.
|
|
//
|
|
// Spotify gives a person a zip of JSON files when they ask for their data. This package reads those files (the zip
|
|
// itself, or the JSON files in it), matches each song to the library by artist and title, records the matches as
|
|
// listens and makes the playlists. It runs on the server and contacts nobody: the files are read from the request,
|
|
// never written anywhere, and the parts that a file holds besides songs (the IP addresses of the person, the country,
|
|
// the platform, the user name) are never even looked at.
|
|
package spotifyimport
|
|
|
|
import (
|
|
"archive/zip"
|
|
"bufio"
|
|
"bytes"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"path"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Limits on what one upload may hold, so that a zip that inflates into something enormous cannot fill the memory
|
|
// or the disk of the server.
|
|
var (
|
|
maxInflated = int64(4 << 30) // all the JSON of one upload, once it is inflated
|
|
maxZipFiles = 2000 // files in one zip
|
|
)
|
|
|
|
// ErrTooBig is returned for an upload that holds more than the limits allow.
|
|
var ErrTooBig = errors.New("the files hold more than can be imported")
|
|
|
|
// ErrUnreadable is returned for a file that looks like one of Spotify's and cannot be read to its end (cut short,
|
|
// damaged).
|
|
var ErrUnreadable = errors.New("a file is damaged or cut short")
|
|
|
|
// Entry is one play of a song from a streaming history file.
|
|
type Entry struct {
|
|
PlayedAt time.Time // when the play ended
|
|
Artist string
|
|
Title string
|
|
Album string // empty in the older files, which do not say
|
|
PlayedMs int64
|
|
Skipped bool // the file says the listener skipped it
|
|
ReasonEnd string // why the play ended ("trackdone" when it ran to the end), empty in the older files
|
|
}
|
|
|
|
// PlaylistItem is one song of a playlist.
|
|
type PlaylistItem struct{ Artist, Title, Album string }
|
|
|
|
// Playlist is a playlist of the export. Items are its songs, in order; Other counts what is in it that is not a song
|
|
// (a podcast episode, a file the person had on their own computer).
|
|
type Playlist struct {
|
|
Name string
|
|
Items []PlaylistItem
|
|
Other int
|
|
}
|
|
|
|
// Reader reads files of an export and calls back for what is in them. A callback that is nil is skipped.
|
|
type Reader struct {
|
|
OnEntry func(Entry)
|
|
OnPlaylist func(Playlist)
|
|
// OnIgnored is called for a play that is not a song (a podcast episode) or that has no time that can be read.
|
|
OnIgnored func()
|
|
}
|
|
|
|
// ReadFile reads one upload: the zip of the export, or one of the JSON files in it. It reports whether it held
|
|
// something of Spotify's (a history or playlists); a file that holds none is not an error, it is just not found.
|
|
func (r Reader) ReadFile(ra io.ReaderAt, size int64) (bool, error) {
|
|
found, err := r.readFile(ra, size)
|
|
if err != nil && !errors.Is(err, ErrTooBig) {
|
|
err = fmt.Errorf("%w: %v", ErrUnreadable, err)
|
|
}
|
|
return found, err
|
|
}
|
|
|
|
func (r Reader) readFile(ra io.ReaderAt, size int64) (bool, error) {
|
|
head := make([]byte, 4)
|
|
if n, _ := ra.ReadAt(head, 0); n == 4 && bytes.Equal(head, []byte("PK\x03\x04")) {
|
|
return r.readZip(ra, size)
|
|
}
|
|
budget := maxInflated
|
|
return r.readJSON(&budgeted{r: io.NewSectionReader(ra, 0, size), left: &budget})
|
|
}
|
|
|
|
func (r Reader) readZip(ra io.ReaderAt, size int64) (bool, error) {
|
|
zr, err := zip.NewReader(ra, size)
|
|
if err != nil {
|
|
return false, nil // looks like a zip and is not one: nothing of Spotify's
|
|
}
|
|
if len(zr.File) > maxZipFiles {
|
|
return false, ErrTooBig
|
|
}
|
|
budget := maxInflated
|
|
found := false
|
|
for _, f := range zr.File {
|
|
if !wanted(f.Name) {
|
|
continue
|
|
}
|
|
rc, err := f.Open()
|
|
if err != nil {
|
|
return found, fmt.Errorf("open %s: %w", path.Base(f.Name), err)
|
|
}
|
|
ok, err := r.readJSON(&budgeted{r: rc, left: &budget})
|
|
_ = rc.Close()
|
|
if err != nil {
|
|
return found, fmt.Errorf("read %s: %w", path.Base(f.Name), err)
|
|
}
|
|
found = found || ok
|
|
}
|
|
return found, nil
|
|
}
|
|
|
|
// wanted says whether a file of the zip can hold what is imported: the audio history (not the video one) and the
|
|
// playlists. The rest of an export (the account, the searches, the library) is never opened.
|
|
func wanted(name string) bool {
|
|
base := strings.ToLower(path.Base(name))
|
|
if !strings.HasSuffix(base, ".json") || strings.Contains(base, "video") {
|
|
return false
|
|
}
|
|
return strings.HasPrefix(base, "streaming_history") || strings.HasPrefix(base, "streaminghistory") || strings.HasPrefix(base, "playlist")
|
|
}
|
|
|
|
// budgeted stops a reader that gives more than the budget left for the whole upload.
|
|
type budgeted struct {
|
|
r io.Reader
|
|
left *int64
|
|
}
|
|
|
|
func (b *budgeted) Read(p []byte) (int, error) {
|
|
if *b.left <= 0 {
|
|
return 0, ErrTooBig
|
|
}
|
|
if int64(len(p)) > *b.left {
|
|
p = p[:*b.left]
|
|
}
|
|
n, err := b.r.Read(p)
|
|
*b.left -= int64(n)
|
|
return n, err
|
|
}
|
|
|
|
// rawEntry has the fields of a play that are used, in both shapes of file: the extended history (ts, ms_played,
|
|
// master_metadata_...) and the older one (endTime, msPlayed, trackName). Nothing else is decoded.
|
|
type rawEntry struct {
|
|
TS string `json:"ts"`
|
|
MsPlayed int64 `json:"ms_played"`
|
|
Track *string `json:"master_metadata_track_name"`
|
|
Artist *string `json:"master_metadata_album_artist_name"`
|
|
Album *string `json:"master_metadata_album_album_name"`
|
|
ReasonEnd string `json:"reason_end"`
|
|
Skipped *bool `json:"skipped"`
|
|
|
|
EndTime string `json:"endTime"`
|
|
OldArtist string `json:"artistName"`
|
|
OldTrack string `json:"trackName"`
|
|
OldMsPlayed int64 `json:"msPlayed"`
|
|
}
|
|
|
|
type rawPlaylists struct {
|
|
Playlists *[]struct {
|
|
Name string `json:"name"`
|
|
Items []struct {
|
|
Track *struct {
|
|
Name string `json:"trackName"`
|
|
Artist string `json:"artistName"`
|
|
Album string `json:"albumName"`
|
|
} `json:"track"`
|
|
} `json:"items"`
|
|
} `json:"playlists"`
|
|
}
|
|
|
|
func (r Reader) readJSON(in io.Reader) (bool, error) {
|
|
br := bufio.NewReaderSize(in, 64<<10)
|
|
first, err := firstByte(br)
|
|
if err != nil {
|
|
if errors.Is(err, ErrTooBig) {
|
|
return false, err
|
|
}
|
|
return false, nil // empty, or not readable: nothing of Spotify's
|
|
}
|
|
dec := json.NewDecoder(br)
|
|
switch first {
|
|
case '[':
|
|
return r.readHistory(dec)
|
|
case '{':
|
|
return r.readPlaylists(dec)
|
|
}
|
|
return false, nil
|
|
}
|
|
|
|
// firstByte looks at the first byte that is not a space, without taking it.
|
|
func firstByte(br *bufio.Reader) (byte, error) {
|
|
for {
|
|
b, err := br.ReadByte()
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
if b == ' ' || b == '\n' || b == '\r' || b == '\t' || b == 0xEF || b == 0xBB || b == 0xBF { // spaces and a byte order mark
|
|
continue
|
|
}
|
|
return b, br.UnreadByte()
|
|
}
|
|
}
|
|
|
|
func (r Reader) readHistory(dec *json.Decoder) (bool, error) {
|
|
if _, err := dec.Token(); err != nil { // the [
|
|
return false, nil
|
|
}
|
|
found := false
|
|
for dec.More() {
|
|
var raw rawEntry
|
|
if err := dec.Decode(&raw); err != nil {
|
|
var typeErr *json.UnmarshalTypeError
|
|
if errors.As(err, &typeErr) {
|
|
continue // an element that is not an object: not a play
|
|
}
|
|
return found, err
|
|
}
|
|
stamp := raw.TS
|
|
if stamp == "" {
|
|
stamp = raw.EndTime
|
|
}
|
|
if stamp == "" {
|
|
continue // not a play at all: a file that only looks like a history
|
|
}
|
|
found = true
|
|
e, ok := raw.entry(stamp)
|
|
if !ok {
|
|
if r.OnIgnored != nil {
|
|
r.OnIgnored()
|
|
}
|
|
continue
|
|
}
|
|
if r.OnEntry != nil {
|
|
r.OnEntry(e)
|
|
}
|
|
}
|
|
return found, nil
|
|
}
|
|
|
|
// entry is the play, or false for something that is not a song or has no time that can be read.
|
|
func (raw rawEntry) entry(stamp string) (Entry, bool) {
|
|
at, ok := parseTime(stamp)
|
|
if !ok {
|
|
return Entry{}, false
|
|
}
|
|
e := Entry{PlayedAt: at, PlayedMs: raw.MsPlayed, ReasonEnd: raw.ReasonEnd}
|
|
if raw.Skipped != nil {
|
|
e.Skipped = *raw.Skipped
|
|
}
|
|
if raw.Track != nil {
|
|
e.Title = *raw.Track
|
|
}
|
|
if raw.Artist != nil {
|
|
e.Artist = *raw.Artist
|
|
}
|
|
if raw.Album != nil {
|
|
e.Album = *raw.Album
|
|
}
|
|
if e.Title == "" && raw.OldTrack != "" { // the older shape
|
|
e.Title, e.Artist, e.PlayedMs = raw.OldTrack, raw.OldArtist, raw.OldMsPlayed
|
|
}
|
|
e.Title, e.Artist, e.Album = strings.TrimSpace(e.Title), strings.TrimSpace(e.Artist), strings.TrimSpace(e.Album)
|
|
if e.Title == "" || e.Artist == "" || e.PlayedMs < 0 {
|
|
return Entry{}, false // an episode of a podcast has no song name
|
|
}
|
|
return e, true
|
|
}
|
|
|
|
// parseTime reads the time of a play: an RFC 3339 time in the extended history, "2022-01-02 08:30" (UTC) in the older one.
|
|
func parseTime(s string) (time.Time, bool) {
|
|
for _, layout := range []string{time.RFC3339, "2006-01-02 15:04", "2006-01-02 15:04:05"} {
|
|
if t, err := time.ParseInLocation(layout, strings.TrimSpace(s), time.UTC); err == nil {
|
|
return t.UTC(), true
|
|
}
|
|
}
|
|
return time.Time{}, false
|
|
}
|
|
|
|
func (r Reader) readPlaylists(dec *json.Decoder) (bool, error) {
|
|
var raw rawPlaylists
|
|
if err := dec.Decode(&raw); err != nil {
|
|
if errors.Is(err, ErrTooBig) {
|
|
return false, err
|
|
}
|
|
return false, nil
|
|
}
|
|
if raw.Playlists == nil {
|
|
return false, nil
|
|
}
|
|
for _, p := range *raw.Playlists {
|
|
out := Playlist{Name: strings.TrimSpace(p.Name)}
|
|
for _, it := range p.Items {
|
|
if it.Track == nil || strings.TrimSpace(it.Track.Name) == "" || strings.TrimSpace(it.Track.Artist) == "" {
|
|
out.Other++
|
|
continue
|
|
}
|
|
out.Items = append(out.Items, PlaylistItem{
|
|
Artist: strings.TrimSpace(it.Track.Artist), Title: strings.TrimSpace(it.Track.Name), Album: strings.TrimSpace(it.Track.Album),
|
|
})
|
|
}
|
|
if r.OnPlaylist != nil {
|
|
r.OnPlaylist(out)
|
|
}
|
|
}
|
|
return true, nil
|
|
}
|