// Package stats works out the numbers of the statistics page: what the library holds, and what a listener listened to. // // The store gives raw counts; the shares, the decades, the years with no listens and the hours and weekdays (which // depend on the time zone of the listener) are worked out here, so that SQLite and PostgreSQL give the same answer. package stats import ( "context" "errors" "math" "sort" "time" ) // ErrInvalidRange is returned when a period ends before it starts. var ErrInvalidRange = errors.New("the period ends before it starts") const ( // topGenres is how many genres are named; the rest are one "other" share. topGenres = 10 // topRanked is how many artists and songs the lists have. topRanked = 10 // A year outside these is not a year a song came out: a bad tag. minYear, maxYear = 1000, 2999 // timelineGenres is how many genres the chart over time names; the rest are one "other" part. timelineGenres = 5 ) // Range is a period: From is included and Until is excluded. A zero time leaves that side open. type Range struct{ From, Until time.Time } // Totals is what the library holds that can be played. type Totals struct { Artists, Albums, Songs int DurationMs, SizeBytes int64 } // FormatCount is how many files of a format there are, and the space they take. type FormatCount struct { Format string Files int Bytes int64 } // GenreWeight is one genre: how many items (songs, or listens) have it, and their weight. An item that has several // genres is split between them, so that the weights of all the genres add up to the items that have any. type GenreWeight struct { Genre string Songs int Weight float64 } // GenreListen is a listen with one of the genres of its song, and the part of the listen that genre has: 1 for a song // with one genre, 1/n for a song with n. A listen of a song with no genre is one row with an empty Genre and weight 1. type GenreListen struct { At int64 // unix seconds Genre string Weight float64 } // YearCount is how many songs came out in a year. type YearCount struct{ Year, Songs int } // ListenTotals sums up the listens in a period. type ListenTotals struct { Listens, Songs, Artists int PlayedMs int64 First int64 // unix seconds of the first listen, 0 when there is none } // Ranked is an artist or a song with how often it was listened to. type Ranked struct { ID int64 Name string Artists []string // the artists of a song Listens int } // Store reads the raw counts. A listen is a complete event, or a play of at least minMs. type Store interface { Totals(ctx context.Context) (Totals, error) Formats(ctx context.Context) ([]FormatCount, error) // Genres gives the weight of each genre in the playable songs, and how many songs have at least one. Genres(ctx context.Context) ([]GenreWeight, int, error) Years(ctx context.Context) ([]YearCount, error) ListenTotals(ctx context.Context, userID, minMs int64, r Range) (ListenTotals, error) // ListenTimes gives the moment of every listen, in unix seconds. ListenTimes(ctx context.Context, userID, minMs int64, r Range) ([]int64, error) TopArtists(ctx context.Context, userID, minMs int64, r Range, limit int) ([]Ranked, error) TopSongs(ctx context.Context, userID, minMs int64, r Range, limit int) ([]Ranked, error) // ListenGenres gives the weight of each genre in the listens, and how many listens have at least one. ListenGenres(ctx context.Context, userID, minMs int64, r Range) ([]GenreWeight, int, error) // ListenGenreTimes gives every listen with the genres of its song, for the chart of the genres over time. ListenGenreTimes(ctx context.Context, userID, minMs int64, r Range) ([]GenreListen, error) } // Share is a genre and how much of the whole it is, from 0 to 1. type Share struct { Name string Count int Share float64 } // Genres is the share of each genre. Top are the biggest, Other is what the smaller ones add up to, and Untagged is the // part that has no genre at all. Top, Other and Untagged add up to 1. type Genres struct { Top []Share Other float64 Untagged int UntaggedShare float64 } // Format is the files of one format. type Format struct { Format string Files int Bytes int64 } // Decade is how many songs came out in a decade: 1990 is the nineties. type Decade struct{ Decade, Songs int } // Library is what the server holds. type Library struct { Totals Formats []Format Genres Genres Decades []Decade NoYear int // songs whose file has no year } // TimelineBucket is a year or a month of listens, and what share of them each genre of the timeline has. Shares go with // Timeline.Genres; Other is the part of the smaller genres and Untagged the part with no genre. A bucket with listens // adds up to 1; one with none is all zero. type TimelineBucket struct { Year, Month int // Month is 0 when the timeline is by year Listens int Shares []float64 Other float64 Untagged float64 } // Timeline is how the genres you listen to changed over time: by year, or by month when the listens are all in one // year. The years and months with no listen between the first and the last are there, empty. type Timeline struct { Unit string // "year" or "month" Genres []string Buckets []TimelineBucket } // Bucket is a count under a key: a year. type Bucket struct{ Key, Count int } // Listening is what a person listened to in a period. type Listening struct { Listens, Songs, Artists int PlayedMs int64 First time.Time // the zero time when there is no listen Years []Bucket Hours [24]int // from midnight, in the time zone of the listener Weekdays [7]int // from Monday TopArtists, TopSongs []Ranked Genres Genres GenreTimeline Timeline } // Service works out the statistics. type Service struct { store Store minListen time.Duration } // NewService returns a service. minListen is the least a play must last to count as a listen. func NewService(store Store, minListen time.Duration) *Service { return &Service{store: store, minListen: minListen} } // Library describes what the library holds. func (s *Service) Library(ctx context.Context) (Library, error) { totals, err := s.store.Totals(ctx) if err != nil { return Library{}, err } formats, err := s.store.Formats(ctx) if err != nil { return Library{}, err } weights, tagged, err := s.store.Genres(ctx) if err != nil { return Library{}, err } years, err := s.store.Years(ctx) if err != nil { return Library{}, err } out := Library{Totals: totals, Formats: []Format{}, Genres: shapeGenres(weights, tagged, totals.Songs)} for _, f := range formats { out.Formats = append(out.Formats, Format(f)) } sort.SliceStable(out.Formats, func(i, j int) bool { return out.Formats[i].Files > out.Formats[j].Files }) out.Decades, out.NoYear = decades(years, totals.Songs) return out, nil } // Listening describes what a person listened to in a period. Hours, weekdays and years are those of loc. func (s *Service) Listening(ctx context.Context, userID int64, r Range, loc *time.Location) (Listening, error) { if !r.From.IsZero() && !r.Until.IsZero() && r.Until.Before(r.From) { return Listening{}, ErrInvalidRange } if loc == nil { loc = time.UTC } minMs := s.minListen.Milliseconds() totals, err := s.store.ListenTotals(ctx, userID, minMs, r) if err != nil { return Listening{}, err } times, err := s.store.ListenTimes(ctx, userID, minMs, r) if err != nil { return Listening{}, err } artists, err := s.store.TopArtists(ctx, userID, minMs, r, topRanked) if err != nil { return Listening{}, err } songs, err := s.store.TopSongs(ctx, userID, minMs, r, topRanked) if err != nil { return Listening{}, err } weights, tagged, err := s.store.ListenGenres(ctx, userID, minMs, r) if err != nil { return Listening{}, err } genreTimes, err := s.store.ListenGenreTimes(ctx, userID, minMs, r) if err != nil { return Listening{}, err } out := Listening{ GenreTimeline: shapeTimeline(genreTimes, loc), Listens: totals.Listens, Songs: totals.Songs, Artists: totals.Artists, PlayedMs: totals.PlayedMs, Years: []Bucket{}, TopArtists: nonNil(artists), TopSongs: nonNil(songs), Genres: shapeGenres(weights, tagged, totals.Listens), } if totals.First > 0 { out.First = time.Unix(totals.First, 0) } perYear := map[int]int{} for _, at := range times { t := time.Unix(at, 0).In(loc) perYear[t.Year()]++ out.Hours[t.Hour()]++ out.Weekdays[(int(t.Weekday())+6)%7]++ } out.Years = yearBuckets(perYear) return out, nil } // shapeTimeline groups the genres of the listens by year, or by month when they all fall in one year, in the time zone // loc, and works out the share of each of the biggest genres in each. func shapeTimeline(rows []GenreListen, loc *time.Location) Timeline { out := Timeline{Unit: "year", Genres: []string{}, Buckets: []TimelineBucket{}} if len(rows) == 0 { return out } years := map[int]bool{} weight := map[string]float64{} for _, r := range rows { years[time.Unix(r.At, 0).In(loc).Year()] = true if r.Genre != "" { weight[r.Genre] += r.Weight } } if len(years) < 2 { out.Unit = "month" } names := make([]string, 0, len(weight)) for g := range weight { names = append(names, g) } sort.Slice(names, func(i, j int) bool { if weight[names[i]] != weight[names[j]] { return weight[names[i]] > weight[names[j]] } return names[i] < names[j] }) if len(names) > timelineGenres { names = names[:timelineGenres] } out.Genres = names column := map[string]int{} for i, g := range names { column[g] = i } type sums struct { total, other, untagged float64 genre []float64 } key := func(t time.Time) int { // a year, or a year and a month: 202603 if out.Unit == "year" { return t.Year() * 100 } return t.Year()*100 + int(t.Month()) } by := map[int]*sums{} first, last := math.MaxInt, 0 for _, r := range rows { k := key(time.Unix(r.At, 0).In(loc)) first, last = min(first, k), max(last, k) b := by[k] if b == nil { b = &sums{genre: make([]float64, len(names))} by[k] = b } b.total += r.Weight switch i, named := column[r.Genre]; { case r.Genre == "": b.untagged += r.Weight case named: b.genre[i] += r.Weight default: b.other += r.Weight } } for k := first; k <= last; { year, month := k/100, k%100 bucket := TimelineBucket{Year: year, Month: month, Shares: make([]float64, len(names))} if b := by[k]; b != nil && b.total > 0 { bucket.Listens = int(math.Round(b.total)) for i, w := range b.genre { bucket.Shares[i] = w / b.total } bucket.Other, bucket.Untagged = b.other/b.total, b.untagged/b.total } out.Buckets = append(out.Buckets, bucket) if out.Unit == "year" { k += 100 } else if month == 12 { k = (year+1)*100 + 1 } else { k++ } } return out } func nonNil(r []Ranked) []Ranked { if r == nil { return []Ranked{} } return r } // yearBuckets lists every year from the first to the last that has a listen, the ones with none as zero. func yearBuckets(perYear map[int]int) []Bucket { out := []Bucket{} if len(perYear) == 0 { return out } first, last := 1<<31-1, 0 for y := range perYear { first, last = min(first, y), max(last, y) } for y := first; y <= last; y++ { out = append(out, Bucket{Key: y, Count: perYear[y]}) } return out } // shapeGenres turns the weights into shares of total, the items counted, tagged of which have a genre. func shapeGenres(weights []GenreWeight, tagged, total int) Genres { out := Genres{Top: []Share{}} if total <= 0 { return out } sorted := append([]GenreWeight(nil), weights...) sort.Slice(sorted, func(i, j int) bool { if sorted[i].Weight != sorted[j].Weight { return sorted[i].Weight > sorted[j].Weight } return sorted[i].Genre < sorted[j].Genre }) whole := float64(total) for i, w := range sorted { if i < topGenres { out.Top = append(out.Top, Share{Name: w.Genre, Count: w.Songs, Share: w.Weight / whole}) } else { out.Other += w.Weight / whole } } if out.Untagged = total - tagged; out.Untagged < 0 { out.Untagged = 0 } out.UntaggedShare = float64(out.Untagged) / whole return out } // decades groups the years of the songs; the songs left over, with no year or a bad one, are returned apart. func decades(years []YearCount, songs int) ([]Decade, int) { by := map[int]int{} dated := 0 for _, y := range years { if y.Year < minYear || y.Year > maxYear { continue } by[y.Year/10*10] += y.Songs dated += y.Songs } out := make([]Decade, 0, len(by)) for d, n := range by { out = append(out, Decade{Decade: d, Songs: n}) } sort.Slice(out, func(i, j int) bool { return out[i].Decade < out[j].Decade }) return out, max(songs-dated, 0) }