sonda/internal/spotifyimport/service.go
tgiachi a880b87a9e chore: move the module path to git.kmalloc.org/squid/sonda
Migrating hosting from Codeberg to git.kmalloc.org. Renames the
module path in go.mod, every internal import, and the OCI image
source label in the Dockerfile. Build and full test suite verified
green after the rename.
2026-11-15 17:30:29 +01:00

464 lines
15 KiB
Go

package spotifyimport
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"sort"
"strings"
"sync"
"time"
"unicode/utf8"
"git.kmalloc.org/squid/sonda/internal/listening"
"git.kmalloc.org/squid/sonda/internal/playlists"
)
// Errors returned by Import.
var (
// ErrNothingFound means no file held a streaming history or playlists of Spotify.
ErrNothingFound = errors.New("no Spotify history or playlists in these files")
// ErrBusy means this person's import is still running.
ErrBusy = errors.New("an import is already running")
// ErrNotKept means the import was done but its report could not be stored; the Run that comes with it is complete.
ErrNotKept = errors.New("the report of the import could not be kept")
// ErrInvalidPage means a negative limit or offset was asked for.
ErrInvalidPage = errors.New("limit and offset cannot be negative")
)
const (
// MaxMissing is how many missing artists a report lists (the ones played most); MissingTotal says how many there are.
MaxMissing = 50
// DefaultPage and MaxPage are how many past imports a page holds when nothing is asked, and at the most.
DefaultPage = 20
MaxPage = 100
// listenBatch is how many listens are recorded in one go.
listenBatch = 2000
// completeShare says a play counts as heard to the end when this much of the song was played (nine tenths).
completeShareNum, completeShareDen = 9, 10
// A play longer than the song by more than this is not believed: it is brought to the length of the song, as the
// rules of listening would otherwise refuse it.
maxOverrunFactor = 2
maxOverrunSlack = 10_000 // ms
unnamedPlaylist = "Spotify playlist"
)
// Catalog gives the songs of the library that can be played.
type Catalog interface {
Candidates(ctx context.Context) ([]Candidate, error)
}
// Listens records many listens at once. A listen whose idempotency key is already there is not recorded again, and
// is not counted in what comes back.
type Listens interface {
RecordMany(ctx context.Context, userID int64, events []listening.Event) (created int, err error)
}
// OutsidePlay is a play of an artist the person has no music of: the name and the time, nothing else.
type OutsidePlay struct {
Key string // the idempotency key of the play, the same as a listening event of it would have
Artist string
PlayedAt time.Time
}
// Outside keeps the plays of artists that are not in the library, so that what the person listens to can be learned
// from. A play whose key is already kept is not kept again, and is not counted in what comes back.
type Outside interface {
RecordOutside(ctx context.Context, userID int64, plays []OutsidePlay) (created int, err error)
}
// Playlists is what the import needs from the playlists of a person.
type Playlists interface {
List(ctx context.Context, userID int64, p playlists.Page) (playlists.Listing, error)
Create(ctx context.Context, userID int64, name, description string) (playlists.Playlist, error)
AddTracks(ctx context.Context, userID, id int64, trackIDs []int64, position *int) (playlists.Playlist, error)
}
// File is one upload: the zip of the export or one of its JSON files.
type File struct {
R io.ReaderAt
Size int64
}
// History says what became of the plays of the history files.
type History struct {
Entries int `json:"entries"` // plays of songs that were read
Matched int `json:"matched"` // of those, the ones that are a song of the library
Imported int `json:"imported"` // recorded now
AlreadyThere int `json:"already_there"` // matched, and already recorded by an earlier import (or the same play in two files)
NotFound int `json:"not_found"` // not in the library
Ignored int `json:"ignored"` // podcast episodes, and plays with no time that can be read
KeptOutside int `json:"kept_outside"` // of the not found, the plays that count as listened: the artist and the time are kept
}
// PlaylistResult says what became of one playlist. Status is `created`, `exists` (the person already has one with
// that name, which is left alone), `empty` (none of its songs is in the library) or `limit` (the person has as many
// playlists as are allowed).
type PlaylistResult struct {
Name string `json:"name"`
Tracks int `json:"tracks"` // everything that is in it
Matched int `json:"matched"` // the songs of the library
Status string `json:"status"`
}
// Missing is an artist that has plays or playlist songs and none of them in the library.
type Missing struct {
Artist string `json:"artist"`
Count int `json:"count"`
}
// Report is what an import did. It is kept as JSON (the tags are the stored shape), so a name here is not changed
// lightly.
type Report struct {
History History `json:"history"`
Playlists []PlaylistResult `json:"playlists"`
Missing []Missing `json:"missing"` // the artists with the most songs that were not found, at most MaxMissing
MissingTotal int `json:"missing_total"` // how many artists have songs that were not found
}
// Run is a report that was kept: the import with its number and the time it was done.
type Run struct {
ID int64 // 0 when the report could not be kept
At time.Time
Report Report
}
// RunPage is a page of the imports of a person, newest first.
type RunPage struct {
Items []Run
Total int
Limit, Offset int
}
// Runs keeps the reports of the imports.
type Runs interface {
Save(ctx context.Context, userID int64, at time.Time, r Report) (id int64, err error)
List(ctx context.Context, userID int64, limit, offset int) ([]Run, int, error)
}
// Service imports the data of a Spotify export for one person at a time.
type Service struct {
catalog Catalog
listens Listens
lists Playlists
runs Runs
outside Outside
key func(string) string
minListen time.Duration
running sync.Map // user id -> struct{}
}
// NewService builds a Service. key is how names are made comparable (the key of the library search); minListen is how
// long a play must last to count as a listen, so that a play shorter than that is recorded as a skip.
func NewService(catalog Catalog, listens Listens, lists Playlists, runs Runs, outside Outside, key func(string) string, minListen time.Duration) *Service {
return &Service{catalog: catalog, listens: listens, lists: lists, runs: runs, outside: outside, key: key, minListen: minListen}
}
// History lists the imports of a person that were kept, newest first. A limit of 0 is DefaultPage, and one above
// MaxPage is lowered to it.
func (s *Service) History(ctx context.Context, userID int64, limit, offset int) (RunPage, error) {
if limit < 0 || offset < 0 {
return RunPage{}, ErrInvalidPage
}
if limit == 0 {
limit = DefaultPage
}
limit = min(limit, MaxPage)
items, total, err := s.runs.List(ctx, userID, limit, offset)
if err != nil {
return RunPage{}, fmt.Errorf("list the imports: %w", err)
}
return RunPage{Items: items, Total: total, Limit: limit, Offset: offset}, nil
}
// Import reads the files, records the listens that match the library and makes the playlists. A person can run one
// import at a time (ErrBusy). It is safe to import the same files again: nothing is recorded twice. What was
// recorded before an error stays recorded. The report is kept for the person to read again; when that fails the
// Run is still returned, with ID 0, together with ErrNotKept.
func (s *Service) Import(ctx context.Context, userID int64, files []File) (Run, error) {
if _, busy := s.running.LoadOrStore(userID, struct{}{}); busy {
return Run{}, ErrBusy
}
defer s.running.Delete(userID)
songs, err := s.catalog.Candidates(ctx)
if err != nil {
return Run{}, fmt.Errorf("read the library: %w", err)
}
run := &run{svc: s, ctx: ctx, userID: userID, index: NewIndex(s.key, songs), memo: map[string]int64{}, missing: map[string]*Missing{}}
reader := Reader{OnEntry: run.entry, OnPlaylist: run.playlist, OnIgnored: func() { run.report.History.Ignored++ }}
found := false
for _, f := range files {
ok, err := reader.ReadFile(f.R, f.Size)
if err != nil {
return Run{}, err
}
found = found || ok
if run.err != nil {
return Run{}, run.err
}
}
if !found {
return Run{}, ErrNothingFound
}
if err := run.flush(); err != nil {
return Run{}, err
}
if err := run.makePlaylists(); err != nil {
return Run{}, err
}
rep := run.finish()
at := time.Now()
id, err := s.runs.Save(ctx, userID, at, rep)
if err != nil {
return Run{At: at, Report: rep}, fmt.Errorf("%w: %v", ErrNotKept, err)
}
return Run{ID: id, At: at, Report: rep}, nil
}
// run is the state of one import.
type run struct {
svc *Service
ctx context.Context
userID int64
index *Index
// memo remembers which song an artist, title and album are, so that a song played a thousand times is looked up once (0: not found).
memo map[string]int64
batch []listening.Event
away []OutsidePlay // plays of artists that are not in the library, waiting to be kept
kept int
matched int
created int
missing map[string]*Missing // by the comparable name of the artist
playlists []Playlist
report Report
err error
}
func (r *run) find(artist, title, album string) (int64, bool) {
k := artist + "\x00" + title + "\x00" + album
id, seen := r.memo[k]
if !seen {
id, _ = r.index.Find(artist, title, album)
r.memo[k] = id
}
return id, id != 0
}
func (r *run) miss(artist string) {
k := r.svc.key(artist)
m := r.missing[k]
if m == nil {
m = &Missing{Artist: artist}
r.missing[k] = m
}
m.Count++
}
func (r *run) entry(e Entry) {
if r.err != nil {
return
}
r.report.History.Entries++
id, ok := r.find(e.Artist, e.Title, e.Album)
if !ok {
r.report.History.NotFound++
r.miss(e.Artist)
// A skip says nothing about what goes together; a play that counts keeps its artist and its time.
if r.svc.kind(e, e.PlayedMs, 0) != listening.Skip {
r.away = append(r.away, OutsidePlay{Key: idempotencyKey(e), Artist: e.Artist, PlayedAt: e.PlayedAt})
if len(r.away) >= listenBatch {
r.err = r.flush()
}
}
return
}
r.matched++
length := r.index.Length(id)
played := e.PlayedMs
if length > 0 && played > maxOverrunFactor*length+maxOverrunSlack {
played = length
}
r.batch = append(r.batch, listening.Event{
UserID: r.userID, TrackID: id, Type: r.svc.kind(e, played, length), OccurredAt: e.PlayedAt, DurationPlayedMs: played,
IdempotencyKey: idempotencyKey(e),
})
if len(r.batch) >= listenBatch {
r.err = r.flush()
}
}
func (r *run) flush() error {
if len(r.batch) > 0 {
n, err := r.svc.listens.RecordMany(r.ctx, r.userID, r.batch)
if err != nil {
return fmt.Errorf("record the listens: %w", err)
}
r.created += n
r.batch = r.batch[:0]
}
if len(r.away) > 0 {
if _, err := r.svc.outside.RecordOutside(r.ctx, r.userID, r.away); err != nil {
return fmt.Errorf("keep the plays of artists that are not in the library: %w", err)
}
r.kept += len(r.away)
r.away = r.away[:0]
}
return nil
}
func (r *run) playlist(p Playlist) { r.playlists = append(r.playlists, p) }
// kind says what a play was. The file says it when it can: a skip, or a play that ran to its end. Otherwise it is a
// complete listen when nine tenths of the song were played, a skip when it did not last as long as it takes to count
// as a listen, and a play for the rest.
func (s *Service) kind(e Entry, played, length int64) listening.EventType {
switch {
case e.Skipped:
return listening.Skip
case e.ReasonEnd == "trackdone":
return listening.Complete
case length > 0 && played*completeShareDen >= length*completeShareNum:
return listening.Complete
case played < s.minListen.Milliseconds():
return listening.Skip
}
return listening.Play
}
// idempotencyKey names a play so that the same play is never recorded twice: by the minute it ended (the older files
// only say the minute), the artist and the title. The same play in the extended history and in the older one is one.
func idempotencyKey(e Entry) string {
sum := sha256.Sum256([]byte(fmt.Sprintf("%d|%s|%s", e.PlayedAt.Unix()/60, strings.ToLower(e.Artist), strings.ToLower(e.Title))))
return "spotify:" + hex.EncodeToString(sum[:])[:40]
}
func (r *run) makePlaylists() error {
if len(r.playlists) == 0 {
return nil
}
have, err := r.playlistNames()
if err != nil {
return err
}
limit := false
for _, p := range r.playlists {
name := cleanName(p.Name)
res := PlaylistResult{Name: name, Tracks: len(p.Items) + p.Other}
var ids []int64
for _, it := range p.Items {
if id, ok := r.find(it.Artist, it.Title, it.Album); ok {
ids = append(ids, id)
} else {
r.miss(it.Artist)
}
}
res.Matched = len(ids)
switch {
case len(ids) == 0:
res.Status = "empty"
case have[strings.ToLower(name)]:
res.Status = "exists"
case limit:
res.Status = "limit"
default:
created, err := r.fill(name, ids)
if err != nil {
return err
}
if created {
res.Status = "created"
have[strings.ToLower(name)] = true
} else {
res.Status, limit = "limit", true
}
}
r.report.Playlists = append(r.report.Playlists, res)
}
return nil
}
// fill makes a playlist and puts the songs in it, a piece at a time. It reports false when the person already has as many
// playlists as are allowed.
func (r *run) fill(name string, ids []int64) (bool, error) {
pl, err := r.svc.lists.Create(r.ctx, r.userID, name, "")
if errors.Is(err, playlists.ErrTooMany) {
return false, nil
}
if err != nil {
return false, fmt.Errorf("make the playlist %q: %w", name, err)
}
if len(ids) > playlists.MaxItems {
ids = ids[:playlists.MaxItems]
}
for len(ids) > 0 {
n := min(len(ids), playlists.MaxAddAtOnce)
if _, err := r.svc.lists.AddTracks(r.ctx, r.userID, pl.ID, ids[:n], nil); err != nil {
return false, fmt.Errorf("fill the playlist %q: %w", name, err)
}
ids = ids[n:]
}
return true, nil
}
// playlistNames are the names the person already has, in lower case.
func (r *run) playlistNames() (map[string]bool, error) {
have := map[string]bool{}
for offset := 0; ; {
page, err := r.svc.lists.List(r.ctx, r.userID, playlists.Page{Limit: playlists.MaxLimit, Offset: offset})
if err != nil {
return nil, fmt.Errorf("read the playlists: %w", err)
}
for _, p := range page.Items {
have[strings.ToLower(strings.TrimSpace(p.Name))] = true
}
offset += len(page.Items)
if len(page.Items) == 0 || offset >= page.Total {
return have, nil
}
}
}
func cleanName(name string) string {
name = strings.TrimSpace(name)
if name == "" {
return unnamedPlaylist
}
if utf8.RuneCountInString(name) > playlists.MaxNameLen {
name = string([]rune(name)[:playlists.MaxNameLen])
}
return name
}
func (r *run) finish() Report {
h := &r.report.History
h.Matched = r.matched
h.Imported = r.created
h.AlreadyThere = r.matched - r.created
h.KeptOutside = r.kept
all := make([]Missing, 0, len(r.missing))
for _, m := range r.missing {
all = append(all, *m)
}
sort.Slice(all, func(i, j int) bool {
if all[i].Count != all[j].Count {
return all[i].Count > all[j].Count
}
return strings.ToLower(all[i].Artist) < strings.ToLower(all[j].Artist)
})
r.report.MissingTotal = len(all)
if len(all) > MaxMissing {
all = all[:MaxMissing]
}
r.report.Missing = all
return r.report
}