Migrating hosting from Codeberg to git.kmalloc.org. Renames the module path in go.mod, every internal import, and the OCI image source label in the Dockerfile. Build and full test suite verified green after the rename.
464 lines
15 KiB
Go
464 lines
15 KiB
Go
package spotifyimport
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
"unicode/utf8"
|
|
|
|
"git.kmalloc.org/squid/sonda/internal/listening"
|
|
"git.kmalloc.org/squid/sonda/internal/playlists"
|
|
)
|
|
|
|
// Errors returned by Import.
|
|
var (
|
|
// ErrNothingFound means no file held a streaming history or playlists of Spotify.
|
|
ErrNothingFound = errors.New("no Spotify history or playlists in these files")
|
|
// ErrBusy means this person's import is still running.
|
|
ErrBusy = errors.New("an import is already running")
|
|
// ErrNotKept means the import was done but its report could not be stored; the Run that comes with it is complete.
|
|
ErrNotKept = errors.New("the report of the import could not be kept")
|
|
// ErrInvalidPage means a negative limit or offset was asked for.
|
|
ErrInvalidPage = errors.New("limit and offset cannot be negative")
|
|
)
|
|
|
|
const (
|
|
// MaxMissing is how many missing artists a report lists (the ones played most); MissingTotal says how many there are.
|
|
MaxMissing = 50
|
|
|
|
// DefaultPage and MaxPage are how many past imports a page holds when nothing is asked, and at the most.
|
|
DefaultPage = 20
|
|
MaxPage = 100
|
|
|
|
// listenBatch is how many listens are recorded in one go.
|
|
listenBatch = 2000
|
|
|
|
// completeShare says a play counts as heard to the end when this much of the song was played (nine tenths).
|
|
completeShareNum, completeShareDen = 9, 10
|
|
|
|
// A play longer than the song by more than this is not believed: it is brought to the length of the song, as the
|
|
// rules of listening would otherwise refuse it.
|
|
maxOverrunFactor = 2
|
|
maxOverrunSlack = 10_000 // ms
|
|
|
|
unnamedPlaylist = "Spotify playlist"
|
|
)
|
|
|
|
// Catalog gives the songs of the library that can be played.
|
|
type Catalog interface {
|
|
Candidates(ctx context.Context) ([]Candidate, error)
|
|
}
|
|
|
|
// Listens records many listens at once. A listen whose idempotency key is already there is not recorded again, and
|
|
// is not counted in what comes back.
|
|
type Listens interface {
|
|
RecordMany(ctx context.Context, userID int64, events []listening.Event) (created int, err error)
|
|
}
|
|
|
|
// OutsidePlay is a play of an artist the person has no music of: the name and the time, nothing else.
|
|
type OutsidePlay struct {
|
|
Key string // the idempotency key of the play, the same as a listening event of it would have
|
|
Artist string
|
|
PlayedAt time.Time
|
|
}
|
|
|
|
// Outside keeps the plays of artists that are not in the library, so that what the person listens to can be learned
|
|
// from. A play whose key is already kept is not kept again, and is not counted in what comes back.
|
|
type Outside interface {
|
|
RecordOutside(ctx context.Context, userID int64, plays []OutsidePlay) (created int, err error)
|
|
}
|
|
|
|
// Playlists is what the import needs from the playlists of a person.
|
|
type Playlists interface {
|
|
List(ctx context.Context, userID int64, p playlists.Page) (playlists.Listing, error)
|
|
Create(ctx context.Context, userID int64, name, description string) (playlists.Playlist, error)
|
|
AddTracks(ctx context.Context, userID, id int64, trackIDs []int64, position *int) (playlists.Playlist, error)
|
|
}
|
|
|
|
// File is one upload: the zip of the export or one of its JSON files.
|
|
type File struct {
|
|
R io.ReaderAt
|
|
Size int64
|
|
}
|
|
|
|
// History says what became of the plays of the history files.
|
|
type History struct {
|
|
Entries int `json:"entries"` // plays of songs that were read
|
|
Matched int `json:"matched"` // of those, the ones that are a song of the library
|
|
Imported int `json:"imported"` // recorded now
|
|
AlreadyThere int `json:"already_there"` // matched, and already recorded by an earlier import (or the same play in two files)
|
|
NotFound int `json:"not_found"` // not in the library
|
|
Ignored int `json:"ignored"` // podcast episodes, and plays with no time that can be read
|
|
KeptOutside int `json:"kept_outside"` // of the not found, the plays that count as listened: the artist and the time are kept
|
|
}
|
|
|
|
// PlaylistResult says what became of one playlist. Status is `created`, `exists` (the person already has one with
|
|
// that name, which is left alone), `empty` (none of its songs is in the library) or `limit` (the person has as many
|
|
// playlists as are allowed).
|
|
type PlaylistResult struct {
|
|
Name string `json:"name"`
|
|
Tracks int `json:"tracks"` // everything that is in it
|
|
Matched int `json:"matched"` // the songs of the library
|
|
Status string `json:"status"`
|
|
}
|
|
|
|
// Missing is an artist that has plays or playlist songs and none of them in the library.
|
|
type Missing struct {
|
|
Artist string `json:"artist"`
|
|
Count int `json:"count"`
|
|
}
|
|
|
|
// Report is what an import did. It is kept as JSON (the tags are the stored shape), so a name here is not changed
|
|
// lightly.
|
|
type Report struct {
|
|
History History `json:"history"`
|
|
Playlists []PlaylistResult `json:"playlists"`
|
|
Missing []Missing `json:"missing"` // the artists with the most songs that were not found, at most MaxMissing
|
|
MissingTotal int `json:"missing_total"` // how many artists have songs that were not found
|
|
}
|
|
|
|
// Run is a report that was kept: the import with its number and the time it was done.
|
|
type Run struct {
|
|
ID int64 // 0 when the report could not be kept
|
|
At time.Time
|
|
Report Report
|
|
}
|
|
|
|
// RunPage is a page of the imports of a person, newest first.
|
|
type RunPage struct {
|
|
Items []Run
|
|
Total int
|
|
Limit, Offset int
|
|
}
|
|
|
|
// Runs keeps the reports of the imports.
|
|
type Runs interface {
|
|
Save(ctx context.Context, userID int64, at time.Time, r Report) (id int64, err error)
|
|
List(ctx context.Context, userID int64, limit, offset int) ([]Run, int, error)
|
|
}
|
|
|
|
// Service imports the data of a Spotify export for one person at a time.
|
|
type Service struct {
|
|
catalog Catalog
|
|
listens Listens
|
|
lists Playlists
|
|
runs Runs
|
|
outside Outside
|
|
key func(string) string
|
|
minListen time.Duration
|
|
running sync.Map // user id -> struct{}
|
|
}
|
|
|
|
// NewService builds a Service. key is how names are made comparable (the key of the library search); minListen is how
|
|
// long a play must last to count as a listen, so that a play shorter than that is recorded as a skip.
|
|
func NewService(catalog Catalog, listens Listens, lists Playlists, runs Runs, outside Outside, key func(string) string, minListen time.Duration) *Service {
|
|
return &Service{catalog: catalog, listens: listens, lists: lists, runs: runs, outside: outside, key: key, minListen: minListen}
|
|
}
|
|
|
|
// History lists the imports of a person that were kept, newest first. A limit of 0 is DefaultPage, and one above
|
|
// MaxPage is lowered to it.
|
|
func (s *Service) History(ctx context.Context, userID int64, limit, offset int) (RunPage, error) {
|
|
if limit < 0 || offset < 0 {
|
|
return RunPage{}, ErrInvalidPage
|
|
}
|
|
if limit == 0 {
|
|
limit = DefaultPage
|
|
}
|
|
limit = min(limit, MaxPage)
|
|
items, total, err := s.runs.List(ctx, userID, limit, offset)
|
|
if err != nil {
|
|
return RunPage{}, fmt.Errorf("list the imports: %w", err)
|
|
}
|
|
return RunPage{Items: items, Total: total, Limit: limit, Offset: offset}, nil
|
|
}
|
|
|
|
// Import reads the files, records the listens that match the library and makes the playlists. A person can run one
|
|
// import at a time (ErrBusy). It is safe to import the same files again: nothing is recorded twice. What was
|
|
// recorded before an error stays recorded. The report is kept for the person to read again; when that fails the
|
|
// Run is still returned, with ID 0, together with ErrNotKept.
|
|
func (s *Service) Import(ctx context.Context, userID int64, files []File) (Run, error) {
|
|
if _, busy := s.running.LoadOrStore(userID, struct{}{}); busy {
|
|
return Run{}, ErrBusy
|
|
}
|
|
defer s.running.Delete(userID)
|
|
|
|
songs, err := s.catalog.Candidates(ctx)
|
|
if err != nil {
|
|
return Run{}, fmt.Errorf("read the library: %w", err)
|
|
}
|
|
run := &run{svc: s, ctx: ctx, userID: userID, index: NewIndex(s.key, songs), memo: map[string]int64{}, missing: map[string]*Missing{}}
|
|
|
|
reader := Reader{OnEntry: run.entry, OnPlaylist: run.playlist, OnIgnored: func() { run.report.History.Ignored++ }}
|
|
found := false
|
|
for _, f := range files {
|
|
ok, err := reader.ReadFile(f.R, f.Size)
|
|
if err != nil {
|
|
return Run{}, err
|
|
}
|
|
found = found || ok
|
|
if run.err != nil {
|
|
return Run{}, run.err
|
|
}
|
|
}
|
|
if !found {
|
|
return Run{}, ErrNothingFound
|
|
}
|
|
if err := run.flush(); err != nil {
|
|
return Run{}, err
|
|
}
|
|
if err := run.makePlaylists(); err != nil {
|
|
return Run{}, err
|
|
}
|
|
rep := run.finish()
|
|
at := time.Now()
|
|
id, err := s.runs.Save(ctx, userID, at, rep)
|
|
if err != nil {
|
|
return Run{At: at, Report: rep}, fmt.Errorf("%w: %v", ErrNotKept, err)
|
|
}
|
|
return Run{ID: id, At: at, Report: rep}, nil
|
|
}
|
|
|
|
// run is the state of one import.
|
|
type run struct {
|
|
svc *Service
|
|
ctx context.Context
|
|
userID int64
|
|
index *Index
|
|
// memo remembers which song an artist, title and album are, so that a song played a thousand times is looked up once (0: not found).
|
|
memo map[string]int64
|
|
batch []listening.Event
|
|
away []OutsidePlay // plays of artists that are not in the library, waiting to be kept
|
|
kept int
|
|
matched int
|
|
created int
|
|
missing map[string]*Missing // by the comparable name of the artist
|
|
playlists []Playlist
|
|
report Report
|
|
err error
|
|
}
|
|
|
|
func (r *run) find(artist, title, album string) (int64, bool) {
|
|
k := artist + "\x00" + title + "\x00" + album
|
|
id, seen := r.memo[k]
|
|
if !seen {
|
|
id, _ = r.index.Find(artist, title, album)
|
|
r.memo[k] = id
|
|
}
|
|
return id, id != 0
|
|
}
|
|
|
|
func (r *run) miss(artist string) {
|
|
k := r.svc.key(artist)
|
|
m := r.missing[k]
|
|
if m == nil {
|
|
m = &Missing{Artist: artist}
|
|
r.missing[k] = m
|
|
}
|
|
m.Count++
|
|
}
|
|
|
|
func (r *run) entry(e Entry) {
|
|
if r.err != nil {
|
|
return
|
|
}
|
|
r.report.History.Entries++
|
|
id, ok := r.find(e.Artist, e.Title, e.Album)
|
|
if !ok {
|
|
r.report.History.NotFound++
|
|
r.miss(e.Artist)
|
|
// A skip says nothing about what goes together; a play that counts keeps its artist and its time.
|
|
if r.svc.kind(e, e.PlayedMs, 0) != listening.Skip {
|
|
r.away = append(r.away, OutsidePlay{Key: idempotencyKey(e), Artist: e.Artist, PlayedAt: e.PlayedAt})
|
|
if len(r.away) >= listenBatch {
|
|
r.err = r.flush()
|
|
}
|
|
}
|
|
return
|
|
}
|
|
r.matched++
|
|
length := r.index.Length(id)
|
|
played := e.PlayedMs
|
|
if length > 0 && played > maxOverrunFactor*length+maxOverrunSlack {
|
|
played = length
|
|
}
|
|
r.batch = append(r.batch, listening.Event{
|
|
UserID: r.userID, TrackID: id, Type: r.svc.kind(e, played, length), OccurredAt: e.PlayedAt, DurationPlayedMs: played,
|
|
IdempotencyKey: idempotencyKey(e),
|
|
})
|
|
if len(r.batch) >= listenBatch {
|
|
r.err = r.flush()
|
|
}
|
|
}
|
|
|
|
func (r *run) flush() error {
|
|
if len(r.batch) > 0 {
|
|
n, err := r.svc.listens.RecordMany(r.ctx, r.userID, r.batch)
|
|
if err != nil {
|
|
return fmt.Errorf("record the listens: %w", err)
|
|
}
|
|
r.created += n
|
|
r.batch = r.batch[:0]
|
|
}
|
|
if len(r.away) > 0 {
|
|
if _, err := r.svc.outside.RecordOutside(r.ctx, r.userID, r.away); err != nil {
|
|
return fmt.Errorf("keep the plays of artists that are not in the library: %w", err)
|
|
}
|
|
r.kept += len(r.away)
|
|
r.away = r.away[:0]
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *run) playlist(p Playlist) { r.playlists = append(r.playlists, p) }
|
|
|
|
// kind says what a play was. The file says it when it can: a skip, or a play that ran to its end. Otherwise it is a
|
|
// complete listen when nine tenths of the song were played, a skip when it did not last as long as it takes to count
|
|
// as a listen, and a play for the rest.
|
|
func (s *Service) kind(e Entry, played, length int64) listening.EventType {
|
|
switch {
|
|
case e.Skipped:
|
|
return listening.Skip
|
|
case e.ReasonEnd == "trackdone":
|
|
return listening.Complete
|
|
case length > 0 && played*completeShareDen >= length*completeShareNum:
|
|
return listening.Complete
|
|
case played < s.minListen.Milliseconds():
|
|
return listening.Skip
|
|
}
|
|
return listening.Play
|
|
}
|
|
|
|
// idempotencyKey names a play so that the same play is never recorded twice: by the minute it ended (the older files
|
|
// only say the minute), the artist and the title. The same play in the extended history and in the older one is one.
|
|
func idempotencyKey(e Entry) string {
|
|
sum := sha256.Sum256([]byte(fmt.Sprintf("%d|%s|%s", e.PlayedAt.Unix()/60, strings.ToLower(e.Artist), strings.ToLower(e.Title))))
|
|
return "spotify:" + hex.EncodeToString(sum[:])[:40]
|
|
}
|
|
|
|
func (r *run) makePlaylists() error {
|
|
if len(r.playlists) == 0 {
|
|
return nil
|
|
}
|
|
have, err := r.playlistNames()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
limit := false
|
|
for _, p := range r.playlists {
|
|
name := cleanName(p.Name)
|
|
res := PlaylistResult{Name: name, Tracks: len(p.Items) + p.Other}
|
|
var ids []int64
|
|
for _, it := range p.Items {
|
|
if id, ok := r.find(it.Artist, it.Title, it.Album); ok {
|
|
ids = append(ids, id)
|
|
} else {
|
|
r.miss(it.Artist)
|
|
}
|
|
}
|
|
res.Matched = len(ids)
|
|
switch {
|
|
case len(ids) == 0:
|
|
res.Status = "empty"
|
|
case have[strings.ToLower(name)]:
|
|
res.Status = "exists"
|
|
case limit:
|
|
res.Status = "limit"
|
|
default:
|
|
created, err := r.fill(name, ids)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if created {
|
|
res.Status = "created"
|
|
have[strings.ToLower(name)] = true
|
|
} else {
|
|
res.Status, limit = "limit", true
|
|
}
|
|
}
|
|
r.report.Playlists = append(r.report.Playlists, res)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// fill makes a playlist and puts the songs in it, a piece at a time. It reports false when the person already has as many
|
|
// playlists as are allowed.
|
|
func (r *run) fill(name string, ids []int64) (bool, error) {
|
|
pl, err := r.svc.lists.Create(r.ctx, r.userID, name, "")
|
|
if errors.Is(err, playlists.ErrTooMany) {
|
|
return false, nil
|
|
}
|
|
if err != nil {
|
|
return false, fmt.Errorf("make the playlist %q: %w", name, err)
|
|
}
|
|
if len(ids) > playlists.MaxItems {
|
|
ids = ids[:playlists.MaxItems]
|
|
}
|
|
for len(ids) > 0 {
|
|
n := min(len(ids), playlists.MaxAddAtOnce)
|
|
if _, err := r.svc.lists.AddTracks(r.ctx, r.userID, pl.ID, ids[:n], nil); err != nil {
|
|
return false, fmt.Errorf("fill the playlist %q: %w", name, err)
|
|
}
|
|
ids = ids[n:]
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
// playlistNames are the names the person already has, in lower case.
|
|
func (r *run) playlistNames() (map[string]bool, error) {
|
|
have := map[string]bool{}
|
|
for offset := 0; ; {
|
|
page, err := r.svc.lists.List(r.ctx, r.userID, playlists.Page{Limit: playlists.MaxLimit, Offset: offset})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read the playlists: %w", err)
|
|
}
|
|
for _, p := range page.Items {
|
|
have[strings.ToLower(strings.TrimSpace(p.Name))] = true
|
|
}
|
|
offset += len(page.Items)
|
|
if len(page.Items) == 0 || offset >= page.Total {
|
|
return have, nil
|
|
}
|
|
}
|
|
}
|
|
|
|
func cleanName(name string) string {
|
|
name = strings.TrimSpace(name)
|
|
if name == "" {
|
|
return unnamedPlaylist
|
|
}
|
|
if utf8.RuneCountInString(name) > playlists.MaxNameLen {
|
|
name = string([]rune(name)[:playlists.MaxNameLen])
|
|
}
|
|
return name
|
|
}
|
|
|
|
func (r *run) finish() Report {
|
|
h := &r.report.History
|
|
h.Matched = r.matched
|
|
h.Imported = r.created
|
|
h.AlreadyThere = r.matched - r.created
|
|
h.KeptOutside = r.kept
|
|
all := make([]Missing, 0, len(r.missing))
|
|
for _, m := range r.missing {
|
|
all = append(all, *m)
|
|
}
|
|
sort.Slice(all, func(i, j int) bool {
|
|
if all[i].Count != all[j].Count {
|
|
return all[i].Count > all[j].Count
|
|
}
|
|
return strings.ToLower(all[i].Artist) < strings.ToLower(all[j].Artist)
|
|
})
|
|
r.report.MissingTotal = len(all)
|
|
if len(all) > MaxMissing {
|
|
all = all[:MaxMissing]
|
|
}
|
|
r.report.Missing = all
|
|
return r.report
|
|
}
|