- GET /stats/listening has genre_timeline: the share of the five biggest genres, of the smaller ones and of the songs with no genre, for each year, or for each month when all the listens are in one year, in the listener's time zone; years and months with no listen are empty - web: a stacked column chart on the Statistics page, coloured in tints of the colour of this server so it follows every theme, with a legend; left out when there is only one column - README: the new field
420 lines
13 KiB
Go
420 lines
13 KiB
Go
// Package stats works out the numbers of the statistics page: what the library holds, and what a listener listened to.
|
|
//
|
|
// The store gives raw counts; the shares, the decades, the years with no listens and the hours and weekdays (which
|
|
// depend on the time zone of the listener) are worked out here, so that SQLite and PostgreSQL give the same answer.
|
|
package stats
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"math"
|
|
"sort"
|
|
"time"
|
|
)
|
|
|
|
// ErrInvalidRange is returned when a period ends before it starts.
|
|
var ErrInvalidRange = errors.New("the period ends before it starts")
|
|
|
|
const (
|
|
// topGenres is how many genres are named; the rest are one "other" share.
|
|
topGenres = 10
|
|
// topRanked is how many artists and songs the lists have.
|
|
topRanked = 10
|
|
// A year outside these is not a year a song came out: a bad tag.
|
|
minYear, maxYear = 1000, 2999
|
|
// timelineGenres is how many genres the chart over time names; the rest are one "other" part.
|
|
timelineGenres = 5
|
|
)
|
|
|
|
// Range is a period: From is included and Until is excluded. A zero time leaves that side open.
|
|
type Range struct{ From, Until time.Time }
|
|
|
|
// Totals is what the library holds that can be played.
|
|
type Totals struct {
|
|
Artists, Albums, Songs int
|
|
DurationMs, SizeBytes int64
|
|
}
|
|
|
|
// FormatCount is how many files of a format there are, and the space they take.
|
|
type FormatCount struct {
|
|
Format string
|
|
Files int
|
|
Bytes int64
|
|
}
|
|
|
|
// GenreWeight is one genre: how many items (songs, or listens) have it, and their weight. An item that has several
|
|
// genres is split between them, so that the weights of all the genres add up to the items that have any.
|
|
type GenreWeight struct {
|
|
Genre string
|
|
Songs int
|
|
Weight float64
|
|
}
|
|
|
|
// GenreListen is a listen with one of the genres of its song, and the part of the listen that genre has: 1 for a song
|
|
// with one genre, 1/n for a song with n. A listen of a song with no genre is one row with an empty Genre and weight 1.
|
|
type GenreListen struct {
|
|
At int64 // unix seconds
|
|
Genre string
|
|
Weight float64
|
|
}
|
|
|
|
// YearCount is how many songs came out in a year.
|
|
type YearCount struct{ Year, Songs int }
|
|
|
|
// ListenTotals sums up the listens in a period.
|
|
type ListenTotals struct {
|
|
Listens, Songs, Artists int
|
|
PlayedMs int64
|
|
First int64 // unix seconds of the first listen, 0 when there is none
|
|
}
|
|
|
|
// Ranked is an artist or a song with how often it was listened to.
|
|
type Ranked struct {
|
|
ID int64
|
|
Name string
|
|
Artists []string // the artists of a song
|
|
Listens int
|
|
}
|
|
|
|
// Store reads the raw counts. A listen is a complete event, or a play of at least minMs.
|
|
type Store interface {
|
|
Totals(ctx context.Context) (Totals, error)
|
|
Formats(ctx context.Context) ([]FormatCount, error)
|
|
// Genres gives the weight of each genre in the playable songs, and how many songs have at least one.
|
|
Genres(ctx context.Context) ([]GenreWeight, int, error)
|
|
Years(ctx context.Context) ([]YearCount, error)
|
|
|
|
ListenTotals(ctx context.Context, userID, minMs int64, r Range) (ListenTotals, error)
|
|
// ListenTimes gives the moment of every listen, in unix seconds.
|
|
ListenTimes(ctx context.Context, userID, minMs int64, r Range) ([]int64, error)
|
|
TopArtists(ctx context.Context, userID, minMs int64, r Range, limit int) ([]Ranked, error)
|
|
TopSongs(ctx context.Context, userID, minMs int64, r Range, limit int) ([]Ranked, error)
|
|
// ListenGenres gives the weight of each genre in the listens, and how many listens have at least one.
|
|
ListenGenres(ctx context.Context, userID, minMs int64, r Range) ([]GenreWeight, int, error)
|
|
// ListenGenreTimes gives every listen with the genres of its song, for the chart of the genres over time.
|
|
ListenGenreTimes(ctx context.Context, userID, minMs int64, r Range) ([]GenreListen, error)
|
|
}
|
|
|
|
// Share is a genre and how much of the whole it is, from 0 to 1.
|
|
type Share struct {
|
|
Name string
|
|
Count int
|
|
Share float64
|
|
}
|
|
|
|
// Genres is the share of each genre. Top are the biggest, Other is what the smaller ones add up to, and Untagged is the
|
|
// part that has no genre at all. Top, Other and Untagged add up to 1.
|
|
type Genres struct {
|
|
Top []Share
|
|
Other float64
|
|
Untagged int
|
|
UntaggedShare float64
|
|
}
|
|
|
|
// Format is the files of one format.
|
|
type Format struct {
|
|
Format string
|
|
Files int
|
|
Bytes int64
|
|
}
|
|
|
|
// Decade is how many songs came out in a decade: 1990 is the nineties.
|
|
type Decade struct{ Decade, Songs int }
|
|
|
|
// Library is what the server holds.
|
|
type Library struct {
|
|
Totals
|
|
Formats []Format
|
|
Genres Genres
|
|
Decades []Decade
|
|
NoYear int // songs whose file has no year
|
|
}
|
|
|
|
// TimelineBucket is a year or a month of listens, and what share of them each genre of the timeline has. Shares go with
|
|
// Timeline.Genres; Other is the part of the smaller genres and Untagged the part with no genre. A bucket with listens
|
|
// adds up to 1; one with none is all zero.
|
|
type TimelineBucket struct {
|
|
Year, Month int // Month is 0 when the timeline is by year
|
|
Listens int
|
|
Shares []float64
|
|
Other float64
|
|
Untagged float64
|
|
}
|
|
|
|
// Timeline is how the genres you listen to changed over time: by year, or by month when the listens are all in one
|
|
// year. The years and months with no listen between the first and the last are there, empty.
|
|
type Timeline struct {
|
|
Unit string // "year" or "month"
|
|
Genres []string
|
|
Buckets []TimelineBucket
|
|
}
|
|
|
|
// Bucket is a count under a key: a year.
|
|
type Bucket struct{ Key, Count int }
|
|
|
|
// Listening is what a person listened to in a period.
|
|
type Listening struct {
|
|
Listens, Songs, Artists int
|
|
PlayedMs int64
|
|
First time.Time // the zero time when there is no listen
|
|
Years []Bucket
|
|
Hours [24]int // from midnight, in the time zone of the listener
|
|
Weekdays [7]int // from Monday
|
|
TopArtists, TopSongs []Ranked
|
|
Genres Genres
|
|
GenreTimeline Timeline
|
|
}
|
|
|
|
// Service works out the statistics.
|
|
type Service struct {
|
|
store Store
|
|
minListen time.Duration
|
|
}
|
|
|
|
// NewService returns a service. minListen is the least a play must last to count as a listen.
|
|
func NewService(store Store, minListen time.Duration) *Service {
|
|
return &Service{store: store, minListen: minListen}
|
|
}
|
|
|
|
// Library describes what the library holds.
|
|
func (s *Service) Library(ctx context.Context) (Library, error) {
|
|
totals, err := s.store.Totals(ctx)
|
|
if err != nil {
|
|
return Library{}, err
|
|
}
|
|
formats, err := s.store.Formats(ctx)
|
|
if err != nil {
|
|
return Library{}, err
|
|
}
|
|
weights, tagged, err := s.store.Genres(ctx)
|
|
if err != nil {
|
|
return Library{}, err
|
|
}
|
|
years, err := s.store.Years(ctx)
|
|
if err != nil {
|
|
return Library{}, err
|
|
}
|
|
|
|
out := Library{Totals: totals, Formats: []Format{}, Genres: shapeGenres(weights, tagged, totals.Songs)}
|
|
for _, f := range formats {
|
|
out.Formats = append(out.Formats, Format(f))
|
|
}
|
|
sort.SliceStable(out.Formats, func(i, j int) bool { return out.Formats[i].Files > out.Formats[j].Files })
|
|
out.Decades, out.NoYear = decades(years, totals.Songs)
|
|
return out, nil
|
|
}
|
|
|
|
// Listening describes what a person listened to in a period. Hours, weekdays and years are those of loc.
|
|
func (s *Service) Listening(ctx context.Context, userID int64, r Range, loc *time.Location) (Listening, error) {
|
|
if !r.From.IsZero() && !r.Until.IsZero() && r.Until.Before(r.From) {
|
|
return Listening{}, ErrInvalidRange
|
|
}
|
|
if loc == nil {
|
|
loc = time.UTC
|
|
}
|
|
minMs := s.minListen.Milliseconds()
|
|
totals, err := s.store.ListenTotals(ctx, userID, minMs, r)
|
|
if err != nil {
|
|
return Listening{}, err
|
|
}
|
|
times, err := s.store.ListenTimes(ctx, userID, minMs, r)
|
|
if err != nil {
|
|
return Listening{}, err
|
|
}
|
|
artists, err := s.store.TopArtists(ctx, userID, minMs, r, topRanked)
|
|
if err != nil {
|
|
return Listening{}, err
|
|
}
|
|
songs, err := s.store.TopSongs(ctx, userID, minMs, r, topRanked)
|
|
if err != nil {
|
|
return Listening{}, err
|
|
}
|
|
weights, tagged, err := s.store.ListenGenres(ctx, userID, minMs, r)
|
|
if err != nil {
|
|
return Listening{}, err
|
|
}
|
|
|
|
genreTimes, err := s.store.ListenGenreTimes(ctx, userID, minMs, r)
|
|
if err != nil {
|
|
return Listening{}, err
|
|
}
|
|
|
|
out := Listening{
|
|
GenreTimeline: shapeTimeline(genreTimes, loc),
|
|
Listens: totals.Listens, Songs: totals.Songs, Artists: totals.Artists, PlayedMs: totals.PlayedMs,
|
|
Years: []Bucket{},
|
|
TopArtists: nonNil(artists), TopSongs: nonNil(songs),
|
|
Genres: shapeGenres(weights, tagged, totals.Listens),
|
|
}
|
|
if totals.First > 0 {
|
|
out.First = time.Unix(totals.First, 0)
|
|
}
|
|
perYear := map[int]int{}
|
|
for _, at := range times {
|
|
t := time.Unix(at, 0).In(loc)
|
|
perYear[t.Year()]++
|
|
out.Hours[t.Hour()]++
|
|
out.Weekdays[(int(t.Weekday())+6)%7]++
|
|
}
|
|
out.Years = yearBuckets(perYear)
|
|
return out, nil
|
|
}
|
|
|
|
// shapeTimeline groups the genres of the listens by year, or by month when they all fall in one year, in the time zone
|
|
// loc, and works out the share of each of the biggest genres in each.
|
|
func shapeTimeline(rows []GenreListen, loc *time.Location) Timeline {
|
|
out := Timeline{Unit: "year", Genres: []string{}, Buckets: []TimelineBucket{}}
|
|
if len(rows) == 0 {
|
|
return out
|
|
}
|
|
years := map[int]bool{}
|
|
weight := map[string]float64{}
|
|
for _, r := range rows {
|
|
years[time.Unix(r.At, 0).In(loc).Year()] = true
|
|
if r.Genre != "" {
|
|
weight[r.Genre] += r.Weight
|
|
}
|
|
}
|
|
if len(years) < 2 {
|
|
out.Unit = "month"
|
|
}
|
|
names := make([]string, 0, len(weight))
|
|
for g := range weight {
|
|
names = append(names, g)
|
|
}
|
|
sort.Slice(names, func(i, j int) bool {
|
|
if weight[names[i]] != weight[names[j]] {
|
|
return weight[names[i]] > weight[names[j]]
|
|
}
|
|
return names[i] < names[j]
|
|
})
|
|
if len(names) > timelineGenres {
|
|
names = names[:timelineGenres]
|
|
}
|
|
out.Genres = names
|
|
column := map[string]int{}
|
|
for i, g := range names {
|
|
column[g] = i
|
|
}
|
|
|
|
type sums struct {
|
|
total, other, untagged float64
|
|
genre []float64
|
|
}
|
|
key := func(t time.Time) int { // a year, or a year and a month: 202603
|
|
if out.Unit == "year" {
|
|
return t.Year() * 100
|
|
}
|
|
return t.Year()*100 + int(t.Month())
|
|
}
|
|
by := map[int]*sums{}
|
|
first, last := math.MaxInt, 0
|
|
for _, r := range rows {
|
|
k := key(time.Unix(r.At, 0).In(loc))
|
|
first, last = min(first, k), max(last, k)
|
|
b := by[k]
|
|
if b == nil {
|
|
b = &sums{genre: make([]float64, len(names))}
|
|
by[k] = b
|
|
}
|
|
b.total += r.Weight
|
|
switch i, named := column[r.Genre]; {
|
|
case r.Genre == "":
|
|
b.untagged += r.Weight
|
|
case named:
|
|
b.genre[i] += r.Weight
|
|
default:
|
|
b.other += r.Weight
|
|
}
|
|
}
|
|
for k := first; k <= last; {
|
|
year, month := k/100, k%100
|
|
bucket := TimelineBucket{Year: year, Month: month, Shares: make([]float64, len(names))}
|
|
if b := by[k]; b != nil && b.total > 0 {
|
|
bucket.Listens = int(math.Round(b.total))
|
|
for i, w := range b.genre {
|
|
bucket.Shares[i] = w / b.total
|
|
}
|
|
bucket.Other, bucket.Untagged = b.other/b.total, b.untagged/b.total
|
|
}
|
|
out.Buckets = append(out.Buckets, bucket)
|
|
if out.Unit == "year" {
|
|
k += 100
|
|
} else if month == 12 {
|
|
k = (year+1)*100 + 1
|
|
} else {
|
|
k++
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func nonNil(r []Ranked) []Ranked {
|
|
if r == nil {
|
|
return []Ranked{}
|
|
}
|
|
return r
|
|
}
|
|
|
|
// yearBuckets lists every year from the first to the last that has a listen, the ones with none as zero.
|
|
func yearBuckets(perYear map[int]int) []Bucket {
|
|
out := []Bucket{}
|
|
if len(perYear) == 0 {
|
|
return out
|
|
}
|
|
first, last := 1<<31-1, 0
|
|
for y := range perYear {
|
|
first, last = min(first, y), max(last, y)
|
|
}
|
|
for y := first; y <= last; y++ {
|
|
out = append(out, Bucket{Key: y, Count: perYear[y]})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// shapeGenres turns the weights into shares of total, the items counted, tagged of which have a genre.
|
|
func shapeGenres(weights []GenreWeight, tagged, total int) Genres {
|
|
out := Genres{Top: []Share{}}
|
|
if total <= 0 {
|
|
return out
|
|
}
|
|
sorted := append([]GenreWeight(nil), weights...)
|
|
sort.Slice(sorted, func(i, j int) bool {
|
|
if sorted[i].Weight != sorted[j].Weight {
|
|
return sorted[i].Weight > sorted[j].Weight
|
|
}
|
|
return sorted[i].Genre < sorted[j].Genre
|
|
})
|
|
whole := float64(total)
|
|
for i, w := range sorted {
|
|
if i < topGenres {
|
|
out.Top = append(out.Top, Share{Name: w.Genre, Count: w.Songs, Share: w.Weight / whole})
|
|
} else {
|
|
out.Other += w.Weight / whole
|
|
}
|
|
}
|
|
if out.Untagged = total - tagged; out.Untagged < 0 {
|
|
out.Untagged = 0
|
|
}
|
|
out.UntaggedShare = float64(out.Untagged) / whole
|
|
return out
|
|
}
|
|
|
|
// decades groups the years of the songs; the songs left over, with no year or a bad one, are returned apart.
|
|
func decades(years []YearCount, songs int) ([]Decade, int) {
|
|
by := map[int]int{}
|
|
dated := 0
|
|
for _, y := range years {
|
|
if y.Year < minYear || y.Year > maxYear {
|
|
continue
|
|
}
|
|
by[y.Year/10*10] += y.Songs
|
|
dated += y.Songs
|
|
}
|
|
out := make([]Decade, 0, len(by))
|
|
for d, n := range by {
|
|
out = append(out, Decade{Decade: d, Songs: n})
|
|
}
|
|
sort.Slice(out, func(i, j int) bool { return out[i].Decade < out[j].Decade })
|
|
return out, max(songs-dated, 0)
|
|
}
|