sonda/internal/stats/stats.go
tgiachi 08c83c948f feat: how the genres I listen to changed over time
- GET /stats/listening has genre_timeline: the share of the five biggest genres, of the smaller ones and of the songs with no genre, for each year, or for each month when all the listens are in one year, in the listener's time zone; years and months with no listen are empty
- web: a stacked column chart on the Statistics page, coloured in tints of the colour of this server so it follows every theme, with a legend; left out when there is only one column
- README: the new field
2026-11-15 18:17:33 +01:00

420 lines
13 KiB
Go

// Package stats works out the numbers of the statistics page: what the library holds, and what a listener listened to.
//
// The store gives raw counts; the shares, the decades, the years with no listens and the hours and weekdays (which
// depend on the time zone of the listener) are worked out here, so that SQLite and PostgreSQL give the same answer.
package stats
import (
"context"
"errors"
"math"
"sort"
"time"
)
// ErrInvalidRange is returned when a period ends before it starts.
var ErrInvalidRange = errors.New("the period ends before it starts")
const (
// topGenres is how many genres are named; the rest are one "other" share.
topGenres = 10
// topRanked is how many artists and songs the lists have.
topRanked = 10
// A year outside these is not a year a song came out: a bad tag.
minYear, maxYear = 1000, 2999
// timelineGenres is how many genres the chart over time names; the rest are one "other" part.
timelineGenres = 5
)
// Range is a period: From is included and Until is excluded. A zero time leaves that side open.
type Range struct{ From, Until time.Time }
// Totals is what the library holds that can be played.
type Totals struct {
Artists, Albums, Songs int
DurationMs, SizeBytes int64
}
// FormatCount is how many files of a format there are, and the space they take.
type FormatCount struct {
Format string
Files int
Bytes int64
}
// GenreWeight is one genre: how many items (songs, or listens) have it, and their weight. An item that has several
// genres is split between them, so that the weights of all the genres add up to the items that have any.
type GenreWeight struct {
Genre string
Songs int
Weight float64
}
// GenreListen is a listen with one of the genres of its song, and the part of the listen that genre has: 1 for a song
// with one genre, 1/n for a song with n. A listen of a song with no genre is one row with an empty Genre and weight 1.
type GenreListen struct {
At int64 // unix seconds
Genre string
Weight float64
}
// YearCount is how many songs came out in a year.
type YearCount struct{ Year, Songs int }
// ListenTotals sums up the listens in a period.
type ListenTotals struct {
Listens, Songs, Artists int
PlayedMs int64
First int64 // unix seconds of the first listen, 0 when there is none
}
// Ranked is an artist or a song with how often it was listened to.
type Ranked struct {
ID int64
Name string
Artists []string // the artists of a song
Listens int
}
// Store reads the raw counts. A listen is a complete event, or a play of at least minMs.
type Store interface {
Totals(ctx context.Context) (Totals, error)
Formats(ctx context.Context) ([]FormatCount, error)
// Genres gives the weight of each genre in the playable songs, and how many songs have at least one.
Genres(ctx context.Context) ([]GenreWeight, int, error)
Years(ctx context.Context) ([]YearCount, error)
ListenTotals(ctx context.Context, userID, minMs int64, r Range) (ListenTotals, error)
// ListenTimes gives the moment of every listen, in unix seconds.
ListenTimes(ctx context.Context, userID, minMs int64, r Range) ([]int64, error)
TopArtists(ctx context.Context, userID, minMs int64, r Range, limit int) ([]Ranked, error)
TopSongs(ctx context.Context, userID, minMs int64, r Range, limit int) ([]Ranked, error)
// ListenGenres gives the weight of each genre in the listens, and how many listens have at least one.
ListenGenres(ctx context.Context, userID, minMs int64, r Range) ([]GenreWeight, int, error)
// ListenGenreTimes gives every listen with the genres of its song, for the chart of the genres over time.
ListenGenreTimes(ctx context.Context, userID, minMs int64, r Range) ([]GenreListen, error)
}
// Share is a genre and how much of the whole it is, from 0 to 1.
type Share struct {
Name string
Count int
Share float64
}
// Genres is the share of each genre. Top are the biggest, Other is what the smaller ones add up to, and Untagged is the
// part that has no genre at all. Top, Other and Untagged add up to 1.
type Genres struct {
Top []Share
Other float64
Untagged int
UntaggedShare float64
}
// Format is the files of one format.
type Format struct {
Format string
Files int
Bytes int64
}
// Decade is how many songs came out in a decade: 1990 is the nineties.
type Decade struct{ Decade, Songs int }
// Library is what the server holds.
type Library struct {
Totals
Formats []Format
Genres Genres
Decades []Decade
NoYear int // songs whose file has no year
}
// TimelineBucket is a year or a month of listens, and what share of them each genre of the timeline has. Shares go with
// Timeline.Genres; Other is the part of the smaller genres and Untagged the part with no genre. A bucket with listens
// adds up to 1; one with none is all zero.
type TimelineBucket struct {
Year, Month int // Month is 0 when the timeline is by year
Listens int
Shares []float64
Other float64
Untagged float64
}
// Timeline is how the genres you listen to changed over time: by year, or by month when the listens are all in one
// year. The years and months with no listen between the first and the last are there, empty.
type Timeline struct {
Unit string // "year" or "month"
Genres []string
Buckets []TimelineBucket
}
// Bucket is a count under a key: a year.
type Bucket struct{ Key, Count int }
// Listening is what a person listened to in a period.
type Listening struct {
Listens, Songs, Artists int
PlayedMs int64
First time.Time // the zero time when there is no listen
Years []Bucket
Hours [24]int // from midnight, in the time zone of the listener
Weekdays [7]int // from Monday
TopArtists, TopSongs []Ranked
Genres Genres
GenreTimeline Timeline
}
// Service works out the statistics.
type Service struct {
store Store
minListen time.Duration
}
// NewService returns a service. minListen is the least a play must last to count as a listen.
func NewService(store Store, minListen time.Duration) *Service {
return &Service{store: store, minListen: minListen}
}
// Library describes what the library holds.
func (s *Service) Library(ctx context.Context) (Library, error) {
totals, err := s.store.Totals(ctx)
if err != nil {
return Library{}, err
}
formats, err := s.store.Formats(ctx)
if err != nil {
return Library{}, err
}
weights, tagged, err := s.store.Genres(ctx)
if err != nil {
return Library{}, err
}
years, err := s.store.Years(ctx)
if err != nil {
return Library{}, err
}
out := Library{Totals: totals, Formats: []Format{}, Genres: shapeGenres(weights, tagged, totals.Songs)}
for _, f := range formats {
out.Formats = append(out.Formats, Format(f))
}
sort.SliceStable(out.Formats, func(i, j int) bool { return out.Formats[i].Files > out.Formats[j].Files })
out.Decades, out.NoYear = decades(years, totals.Songs)
return out, nil
}
// Listening describes what a person listened to in a period. Hours, weekdays and years are those of loc.
func (s *Service) Listening(ctx context.Context, userID int64, r Range, loc *time.Location) (Listening, error) {
if !r.From.IsZero() && !r.Until.IsZero() && r.Until.Before(r.From) {
return Listening{}, ErrInvalidRange
}
if loc == nil {
loc = time.UTC
}
minMs := s.minListen.Milliseconds()
totals, err := s.store.ListenTotals(ctx, userID, minMs, r)
if err != nil {
return Listening{}, err
}
times, err := s.store.ListenTimes(ctx, userID, minMs, r)
if err != nil {
return Listening{}, err
}
artists, err := s.store.TopArtists(ctx, userID, minMs, r, topRanked)
if err != nil {
return Listening{}, err
}
songs, err := s.store.TopSongs(ctx, userID, minMs, r, topRanked)
if err != nil {
return Listening{}, err
}
weights, tagged, err := s.store.ListenGenres(ctx, userID, minMs, r)
if err != nil {
return Listening{}, err
}
genreTimes, err := s.store.ListenGenreTimes(ctx, userID, minMs, r)
if err != nil {
return Listening{}, err
}
out := Listening{
GenreTimeline: shapeTimeline(genreTimes, loc),
Listens: totals.Listens, Songs: totals.Songs, Artists: totals.Artists, PlayedMs: totals.PlayedMs,
Years: []Bucket{},
TopArtists: nonNil(artists), TopSongs: nonNil(songs),
Genres: shapeGenres(weights, tagged, totals.Listens),
}
if totals.First > 0 {
out.First = time.Unix(totals.First, 0)
}
perYear := map[int]int{}
for _, at := range times {
t := time.Unix(at, 0).In(loc)
perYear[t.Year()]++
out.Hours[t.Hour()]++
out.Weekdays[(int(t.Weekday())+6)%7]++
}
out.Years = yearBuckets(perYear)
return out, nil
}
// shapeTimeline groups the genres of the listens by year, or by month when they all fall in one year, in the time zone
// loc, and works out the share of each of the biggest genres in each.
func shapeTimeline(rows []GenreListen, loc *time.Location) Timeline {
out := Timeline{Unit: "year", Genres: []string{}, Buckets: []TimelineBucket{}}
if len(rows) == 0 {
return out
}
years := map[int]bool{}
weight := map[string]float64{}
for _, r := range rows {
years[time.Unix(r.At, 0).In(loc).Year()] = true
if r.Genre != "" {
weight[r.Genre] += r.Weight
}
}
if len(years) < 2 {
out.Unit = "month"
}
names := make([]string, 0, len(weight))
for g := range weight {
names = append(names, g)
}
sort.Slice(names, func(i, j int) bool {
if weight[names[i]] != weight[names[j]] {
return weight[names[i]] > weight[names[j]]
}
return names[i] < names[j]
})
if len(names) > timelineGenres {
names = names[:timelineGenres]
}
out.Genres = names
column := map[string]int{}
for i, g := range names {
column[g] = i
}
type sums struct {
total, other, untagged float64
genre []float64
}
key := func(t time.Time) int { // a year, or a year and a month: 202603
if out.Unit == "year" {
return t.Year() * 100
}
return t.Year()*100 + int(t.Month())
}
by := map[int]*sums{}
first, last := math.MaxInt, 0
for _, r := range rows {
k := key(time.Unix(r.At, 0).In(loc))
first, last = min(first, k), max(last, k)
b := by[k]
if b == nil {
b = &sums{genre: make([]float64, len(names))}
by[k] = b
}
b.total += r.Weight
switch i, named := column[r.Genre]; {
case r.Genre == "":
b.untagged += r.Weight
case named:
b.genre[i] += r.Weight
default:
b.other += r.Weight
}
}
for k := first; k <= last; {
year, month := k/100, k%100
bucket := TimelineBucket{Year: year, Month: month, Shares: make([]float64, len(names))}
if b := by[k]; b != nil && b.total > 0 {
bucket.Listens = int(math.Round(b.total))
for i, w := range b.genre {
bucket.Shares[i] = w / b.total
}
bucket.Other, bucket.Untagged = b.other/b.total, b.untagged/b.total
}
out.Buckets = append(out.Buckets, bucket)
if out.Unit == "year" {
k += 100
} else if month == 12 {
k = (year+1)*100 + 1
} else {
k++
}
}
return out
}
func nonNil(r []Ranked) []Ranked {
if r == nil {
return []Ranked{}
}
return r
}
// yearBuckets lists every year from the first to the last that has a listen, the ones with none as zero.
func yearBuckets(perYear map[int]int) []Bucket {
out := []Bucket{}
if len(perYear) == 0 {
return out
}
first, last := 1<<31-1, 0
for y := range perYear {
first, last = min(first, y), max(last, y)
}
for y := first; y <= last; y++ {
out = append(out, Bucket{Key: y, Count: perYear[y]})
}
return out
}
// shapeGenres turns the weights into shares of total, the items counted, tagged of which have a genre.
func shapeGenres(weights []GenreWeight, tagged, total int) Genres {
out := Genres{Top: []Share{}}
if total <= 0 {
return out
}
sorted := append([]GenreWeight(nil), weights...)
sort.Slice(sorted, func(i, j int) bool {
if sorted[i].Weight != sorted[j].Weight {
return sorted[i].Weight > sorted[j].Weight
}
return sorted[i].Genre < sorted[j].Genre
})
whole := float64(total)
for i, w := range sorted {
if i < topGenres {
out.Top = append(out.Top, Share{Name: w.Genre, Count: w.Songs, Share: w.Weight / whole})
} else {
out.Other += w.Weight / whole
}
}
if out.Untagged = total - tagged; out.Untagged < 0 {
out.Untagged = 0
}
out.UntaggedShare = float64(out.Untagged) / whole
return out
}
// decades groups the years of the songs; the songs left over, with no year or a bad one, are returned apart.
func decades(years []YearCount, songs int) ([]Decade, int) {
by := map[int]int{}
dated := 0
for _, y := range years {
if y.Year < minYear || y.Year > maxYear {
continue
}
by[y.Year/10*10] += y.Songs
dated += y.Songs
}
out := make([]Decade, 0, len(by))
for d, n := range by {
out = append(out, Decade{Decade: d, Songs: n})
}
sort.Slice(out, func(i, j int) bool { return out[i].Decade < out[j].Decade })
return out, max(songs-dated, 0)
}