feat(storage): candidates for duplicates, file fingerprints and removing a copy from the index
Candidates are the playable files whose title another file shares. A fingerprint is kept only for the file that was read (same size and date), and a scan that finds the file changed forgets it. Removing a copy moves its listens to the track that stays, because they would be deleted with the track, then removes the track with its file, credits and genres. Tested on SQLite and PostgreSQL.
This commit is contained in:
parent
7e1e63bdd2
commit
65edfc4918
4 changed files with 526 additions and 11 deletions
32
internal/duplicates/types.go
Normal file
32
internal/duplicates/types.go
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
package duplicates
|
||||
|
||||
import "errors"
|
||||
|
||||
// Why a removal is refused. The API turns each into its own answer.
|
||||
var (
|
||||
// ErrTrackNotFound: a track that was named does not exist (any more).
|
||||
ErrTrackNotFound = errors.New("no such track")
|
||||
// ErrDisabled: the operator has not allowed Sonda to delete files (library.allow_delete).
|
||||
ErrDisabled = errors.New("deleting files is not allowed: turn on library.allow_delete")
|
||||
// ErrScanRunning: a scan is running, and what it reads and what is removed must not meet.
|
||||
ErrScanRunning = errors.New("a scan is running")
|
||||
// ErrInvalid: the tracks named are not copies of one song, or the request makes no sense.
|
||||
ErrInvalid = errors.New("these tracks are not copies of the same song")
|
||||
// ErrChanged: a file is not what the index says (another size or date, a link, gone), or the
|
||||
// copy to keep is not there any more. Nothing is removed; a new scan brings the index up to date.
|
||||
ErrChanged = errors.New("a file changed since it was scanned")
|
||||
// ErrReadOnly: the folder cannot be written to.
|
||||
ErrReadOnly = errors.New("the music folder is read-only")
|
||||
)
|
||||
|
||||
// FileRecord is a file as the index knows it: where it is and what it looked like when it was scanned.
|
||||
type FileRecord struct {
|
||||
FileID int64
|
||||
TrackID int64
|
||||
Root string // the path of its library
|
||||
RelPath string // relative to the library
|
||||
SizeBytes int64
|
||||
MTimeUnix int64
|
||||
Hash string // SHA-256, empty until read
|
||||
Missing bool // flagged as not found by a scan
|
||||
}
|
||||
208
internal/storage/duplicates.go
Normal file
208
internal/storage/duplicates.go
Normal file
|
|
@ -0,0 +1,208 @@
|
|||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/uptrace/bun"
|
||||
|
||||
"codeberg.org/squidd/sonda/internal/duplicates"
|
||||
)
|
||||
|
||||
// DuplicatesRepo reads the files that may be copies of one song, keeps their fingerprints, and
|
||||
// removes a copy from the index. It implements duplicates.Store.
|
||||
type DuplicatesRepo struct{ db *bun.DB }
|
||||
|
||||
// NewDuplicatesRepo builds a DuplicatesRepo.
|
||||
func NewDuplicatesRepo(db *bun.DB) *DuplicatesRepo { return &DuplicatesRepo{db: db} }
|
||||
|
||||
type candidateRow struct {
|
||||
TrackID int64 `bun:"track_id"`
|
||||
FileID int64 `bun:"file_id"`
|
||||
Title string `bun:"title"`
|
||||
TitleKey string `bun:"title_key"`
|
||||
DurationMs int64 `bun:"duration_ms"`
|
||||
AlbumID int64 `bun:"album_id"`
|
||||
Album string `bun:"album"`
|
||||
Format string `bun:"format"`
|
||||
BitRate int `bun:"bitrate"`
|
||||
SizeBytes int64 `bun:"size_bytes"`
|
||||
Hash string `bun:"hash"`
|
||||
}
|
||||
|
||||
type creditRow struct {
|
||||
TrackID int64 `bun:"track_id"`
|
||||
ArtistID int64 `bun:"artist_id"`
|
||||
Name string `bun:"name"`
|
||||
}
|
||||
|
||||
// idsPerQuery keeps a list of ids well under what a database accepts in one statement.
|
||||
const idsPerQuery = 500
|
||||
|
||||
// Candidates returns the playable files whose title is the title of another playable file: the only
|
||||
// ones that can have a copy. A track that has no file that is there, or more than one, is left out.
|
||||
// What makes two of them the same song (the artists, the durations) is decided by duplicates.Build.
|
||||
func (r *DuplicatesRepo) Candidates(ctx context.Context) ([]duplicates.Candidate, error) {
|
||||
var rows []candidateRow
|
||||
err := r.db.NewRaw(`
|
||||
SELECT t.id AS track_id, f.id AS file_id, t.title AS title, t.title_key AS title_key,
|
||||
COALESCE(t.duration_ms, 0) AS duration_ms, COALESCE(t.album_id, 0) AS album_id,
|
||||
COALESCE(al.title, '') AS album, f.format AS format, COALESCE(f.bitrate, 0) AS bitrate,
|
||||
f.size_bytes AS size_bytes, COALESCE(f.content_hash, '') AS hash
|
||||
FROM tracks t
|
||||
JOIN audio_files f ON f.track_id = t.id AND f.missing_at IS NULL
|
||||
LEFT JOIN albums al ON al.id = t.album_id
|
||||
WHERE t.id IN (SELECT track_id FROM audio_files WHERE missing_at IS NULL GROUP BY track_id HAVING COUNT(*) = 1)
|
||||
AND t.title_key IN (
|
||||
SELECT t2.title_key FROM tracks t2
|
||||
JOIN audio_files f2 ON f2.track_id = t2.id AND f2.missing_at IS NULL
|
||||
GROUP BY t2.title_key HAVING COUNT(*) > 1)
|
||||
ORDER BY t.id`).Scan(ctx, &rows)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("select the candidates: %w", err)
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
ids := make([]int64, len(rows))
|
||||
for i, row := range rows {
|
||||
ids[i] = row.TrackID
|
||||
}
|
||||
artists, err := r.mainArtists(ctx, ids)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out := make([]duplicates.Candidate, len(rows))
|
||||
for i, row := range rows {
|
||||
out[i] = duplicates.Candidate{
|
||||
TrackID: row.TrackID, FileID: row.FileID, Artists: artists[row.TrackID], Title: row.Title, TitleKey: row.TitleKey,
|
||||
DurationMs: row.DurationMs, AlbumID: row.AlbumID, Album: row.Album, Format: row.Format,
|
||||
BitRate: row.BitRate, SizeBytes: row.SizeBytes, Hash: row.Hash,
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// mainArtists reads the main artists of some tracks.
|
||||
func (r *DuplicatesRepo) mainArtists(ctx context.Context, trackIDs []int64) (map[int64][]duplicates.ArtistRef, error) {
|
||||
out := map[int64][]duplicates.ArtistRef{}
|
||||
for start := 0; start < len(trackIDs); start += idsPerQuery {
|
||||
chunk := trackIDs[start:min(start+idsPerQuery, len(trackIDs))]
|
||||
var rows []creditRow
|
||||
err := r.db.NewRaw(`
|
||||
SELECT ta.track_id AS track_id, a.id AS artist_id, a.name AS name
|
||||
FROM track_artists ta JOIN artists a ON a.id = ta.artist_id
|
||||
WHERE ta.role = 'main' AND ta.track_id IN (?)
|
||||
ORDER BY ta.track_id, a.id`, bun.In(chunk)).Scan(ctx, &rows)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("select the artists of the candidates: %w", err)
|
||||
}
|
||||
for _, row := range rows {
|
||||
out[row.TrackID] = append(out[row.TrackID], duplicates.ArtistRef{ID: row.ArtistID, Name: row.Name})
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
type fileRow struct {
|
||||
FileID int64 `bun:"file_id"`
|
||||
TrackID int64 `bun:"track_id"`
|
||||
Root string `bun:"root"`
|
||||
RelPath string `bun:"rel_path"`
|
||||
SizeBytes int64 `bun:"size_bytes"`
|
||||
MTimeUnix int64 `bun:"mtime_unix"`
|
||||
Hash string `bun:"hash"`
|
||||
Missing bool `bun:"missing"`
|
||||
}
|
||||
|
||||
func (row fileRow) record() duplicates.FileRecord {
|
||||
return duplicates.FileRecord{
|
||||
FileID: row.FileID, TrackID: row.TrackID, Root: row.Root, RelPath: row.RelPath,
|
||||
SizeBytes: row.SizeBytes, MTimeUnix: row.MTimeUnix, Hash: row.Hash, Missing: row.Missing,
|
||||
}
|
||||
}
|
||||
|
||||
const fileSelect = `
|
||||
SELECT f.id AS file_id, f.track_id AS track_id, l.path AS root, f.rel_path AS rel_path,
|
||||
f.size_bytes AS size_bytes, f.mtime_unix AS mtime_unix, COALESCE(f.content_hash, '') AS hash,
|
||||
(f.missing_at IS NOT NULL) AS missing
|
||||
FROM audio_files f JOIN libraries l ON l.id = f.library_id`
|
||||
|
||||
// FileOf returns the file of a track: the oldest one that is there, else the oldest one. It returns
|
||||
// duplicates.ErrTrackNotFound when the track has no file.
|
||||
func (r *DuplicatesRepo) FileOf(ctx context.Context, trackID int64) (duplicates.FileRecord, error) {
|
||||
var row fileRow
|
||||
err := r.db.NewRaw(fileSelect+` WHERE f.track_id = ? ORDER BY (f.missing_at IS NOT NULL), f.id LIMIT 1`, trackID).Scan(ctx, &row)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return duplicates.FileRecord{}, duplicates.ErrTrackNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return duplicates.FileRecord{}, fmt.Errorf("select the file of a track: %w", err)
|
||||
}
|
||||
return row.record(), nil
|
||||
}
|
||||
|
||||
// FileByID returns one file, or duplicates.ErrTrackNotFound.
|
||||
func (r *DuplicatesRepo) FileByID(ctx context.Context, fileID int64) (duplicates.FileRecord, error) {
|
||||
var row fileRow
|
||||
err := r.db.NewRaw(fileSelect+` WHERE f.id = ?`, fileID).Scan(ctx, &row)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return duplicates.FileRecord{}, duplicates.ErrTrackNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return duplicates.FileRecord{}, fmt.Errorf("select a file: %w", err)
|
||||
}
|
||||
return row.record(), nil
|
||||
}
|
||||
|
||||
// SaveHash keeps the fingerprint of a file, but only if the file is still the one that was read: the
|
||||
// size and date are what they were. It says whether it was kept.
|
||||
func (r *DuplicatesRepo) SaveHash(ctx context.Context, fileID, sizeBytes, mtimeUnix int64, hash string) (bool, error) {
|
||||
res, err := r.db.NewUpdate().Model((*audioFileModel)(nil)).
|
||||
Set("content_hash = ?", hash).
|
||||
Where("id = ?", fileID).Where("size_bytes = ?", sizeBytes).Where("mtime_unix = ?", mtimeUnix).
|
||||
Exec(ctx)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("save the hash: %w", err)
|
||||
}
|
||||
n, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("save the hash: %w", err)
|
||||
}
|
||||
return n > 0, nil
|
||||
}
|
||||
|
||||
// RemoveCopy takes a track out of the index in favour of another that stays: the listens of the
|
||||
// removed track become listens of the one that stays (they would be deleted with it otherwise), and
|
||||
// the track goes, with its file, credits and genres. The covers of the albums involved are built
|
||||
// again. It is one transaction.
|
||||
func (r *DuplicatesRepo) RemoveCopy(ctx context.Context, keepTrackID, removeTrackID int64) error {
|
||||
if keepTrackID == removeTrackID {
|
||||
return duplicates.ErrInvalid
|
||||
}
|
||||
return r.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
|
||||
albums := map[int64]int64{}
|
||||
for _, id := range []int64{keepTrackID, removeTrackID} {
|
||||
var album sql.NullInt64
|
||||
err := tx.QueryRowContext(ctx, "SELECT album_id FROM tracks WHERE id = ?", id).Scan(&album)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return duplicates.ErrTrackNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("read a track: %w", err)
|
||||
}
|
||||
albums[id] = album.Int64
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, "UPDATE listening_events SET track_id = ? WHERE track_id = ?", keepTrackID, removeTrackID); err != nil {
|
||||
return fmt.Errorf("move the listens: %w", err)
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, "DELETE FROM tracks WHERE id = ?", removeTrackID); err != nil {
|
||||
return fmt.Errorf("remove the track: %w", err)
|
||||
}
|
||||
return clearAlbumCovers(ctx, tx, albums[keepTrackID], albums[removeTrackID])
|
||||
})
|
||||
}
|
||||
269
internal/storage/duplicates_test.go
Normal file
269
internal/storage/duplicates_test.go
Normal file
|
|
@ -0,0 +1,269 @@
|
|||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/uptrace/bun"
|
||||
|
||||
"codeberg.org/squidd/sonda/internal/duplicates"
|
||||
"codeberg.org/squidd/sonda/internal/library"
|
||||
"codeberg.org/squidd/sonda/internal/listening"
|
||||
)
|
||||
|
||||
// song is a scanned file: a title on an album by artists, with the facts that tell copies apart.
|
||||
func song(rel, title, album string, durationMs, size int64, format string, bitrate int, artists ...string) library.ParsedFile {
|
||||
p := parsed(rel, title, album, artists...)
|
||||
p.DurationMs, p.Size, p.Format, p.BitRate = durationMs, size, format, bitrate
|
||||
return p
|
||||
}
|
||||
|
||||
func saved(t *testing.T, repo *LibraryRepo, lib int64, f library.ParsedFile) {
|
||||
t.Helper()
|
||||
if _, err := repo.SaveFile(context.Background(), lib, f, 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func trackOf(t *testing.T, db *bun.DB, rel string) int64 {
|
||||
t.Helper()
|
||||
var id int64
|
||||
if err := db.QueryRow("SELECT track_id FROM audio_files WHERE rel_path = ?", rel).Scan(&id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func candidateByRel(t *testing.T, db *bun.DB, cs []duplicates.Candidate, rel string) (duplicates.Candidate, bool) {
|
||||
t.Helper()
|
||||
id := trackOf(t, db, rel)
|
||||
for _, c := range cs {
|
||||
if c.TrackID == id {
|
||||
return c, true
|
||||
}
|
||||
}
|
||||
return duplicates.Candidate{}, false
|
||||
}
|
||||
|
||||
func TestDuplicateCandidatesAreTheFilesWhoseTitleIsSharedWithAnotherFile(t *testing.T) {
|
||||
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
|
||||
ctx := context.Background()
|
||||
lib := NewLibraryRepo(db)
|
||||
l := mkLib(t, lib, "music")
|
||||
saved(t, lib, l, song("a/1.flac", "Song", "Album X", 200_000, 30_000_000, "flac", 900, "Ann"))
|
||||
saved(t, lib, l, song("b/2.mp3", "Song", "Album Y", 201_000, 5_000_000, "mp3", 192, "Ann"))
|
||||
saved(t, lib, l, song("c/3.flac", "Song", "Album Z", 200_000, 30_000_000, "flac", 900, "Bob"))
|
||||
saved(t, lib, l, song("d/4.flac", "Song", "Album W", 200_000, 30_000_000, "flac", 900, "Ann", "Bob"))
|
||||
saved(t, lib, l, song("e/5.flac", "Other", "Album X", 200_000, 30_000_000, "flac", 900, "Ann"))
|
||||
|
||||
cs, err := NewDuplicatesRepo(db).Candidates(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := candidateByRel(t, db, cs, "e/5.flac"); ok {
|
||||
t.Error("a title nobody shares is a candidate")
|
||||
}
|
||||
if len(cs) != 4 {
|
||||
t.Fatalf("%d candidates, want the 4 files called Song", len(cs))
|
||||
}
|
||||
|
||||
c, ok := candidateByRel(t, db, cs, "b/2.mp3")
|
||||
if !ok {
|
||||
t.Fatal("the mp3 is not a candidate")
|
||||
}
|
||||
if c.Title != "Song" || c.TitleKey != "song" || c.Album != "Album Y" || c.Format != "mp3" || c.BitRate != 192 ||
|
||||
c.SizeBytes != 5_000_000 || c.DurationMs != 201_000 || c.FileID == 0 || c.Hash != "" {
|
||||
t.Errorf("candidate = %+v", c)
|
||||
}
|
||||
if len(c.Artists) != 1 || c.Artists[0].Name != "Ann" {
|
||||
t.Errorf("artists = %+v, want Ann", c.Artists)
|
||||
}
|
||||
duo, _ := candidateByRel(t, db, cs, "d/4.flac")
|
||||
if len(duo.Artists) != 2 {
|
||||
t.Errorf("a duet has %d artists, want 2", len(duo.Artists))
|
||||
}
|
||||
|
||||
// Seen through the grouping, the two by Ann alone are the same song, the others are not.
|
||||
groups := duplicates.Build(cs, duplicates.DurationTolerance)
|
||||
if len(groups) != 1 || len(groups[0].Files) != 2 {
|
||||
t.Errorf("groups = %+v, want one of the two copies by Ann", groups)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestAFileFlaggedMissingIsNotACandidate(t *testing.T) {
|
||||
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
|
||||
ctx := context.Background()
|
||||
lib := NewLibraryRepo(db)
|
||||
l := mkLib(t, lib, "music")
|
||||
saved(t, lib, l, song("a/1.flac", "Song", "A", 200_000, 1, "flac", 900, "Ann"))
|
||||
saved(t, lib, l, song("b/2.flac", "Song", "B", 200_000, 1, "flac", 900, "Ann"))
|
||||
repo := NewDuplicatesRepo(db)
|
||||
if cs, _ := repo.Candidates(ctx); len(cs) != 2 {
|
||||
t.Fatalf("%d candidates before, want 2", len(cs))
|
||||
}
|
||||
var id int64
|
||||
if err := db.QueryRow("SELECT id FROM audio_files WHERE rel_path = 'b/2.flac'").Scan(&id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := lib.MarkMissing(ctx, []int64{id}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cs, _ := repo.Candidates(ctx); len(cs) != 0 {
|
||||
t.Errorf("%d candidates after one went missing, want 0", len(cs))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestFileOfSaysWhereTheFileIsAndWhatItLookedLike(t *testing.T) {
|
||||
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
|
||||
ctx := context.Background()
|
||||
lib := NewLibraryRepo(db)
|
||||
l := mkLib(t, lib, "music")
|
||||
saved(t, lib, l, song("a/1.flac", "Song", "A", 200_000, 1234, "flac", 900, "Ann"))
|
||||
repo := NewDuplicatesRepo(db)
|
||||
rec, err := repo.FileOf(ctx, trackOf(t, db, "a/1.flac"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rec.Root != "/lib/music" || rec.RelPath != "a/1.flac" || rec.SizeBytes != 1234 || rec.MTimeUnix != 1_700_000_000 || rec.Hash != "" || rec.Missing {
|
||||
t.Errorf("record = %+v", rec)
|
||||
}
|
||||
if _, err := repo.FileOf(ctx, 9999); !errors.Is(err, duplicates.ErrTrackNotFound) {
|
||||
t.Errorf("an unknown track gave %v, want ErrTrackNotFound", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestAHashIsStoredOnlyForTheFileThatWasRead(t *testing.T) {
|
||||
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
|
||||
ctx := context.Background()
|
||||
lib := NewLibraryRepo(db)
|
||||
l := mkLib(t, lib, "music")
|
||||
saved(t, lib, l, song("a/1.flac", "Song", "A", 200_000, 1234, "flac", 900, "Ann"))
|
||||
saved(t, lib, l, song("b/2.flac", "Song", "B", 200_000, 1234, "flac", 900, "Ann"))
|
||||
repo := NewDuplicatesRepo(db)
|
||||
rec, _ := repo.FileOf(ctx, trackOf(t, db, "a/1.flac"))
|
||||
|
||||
// The file changed after it was read: its hash is no hash of it.
|
||||
if ok, err := repo.SaveHash(ctx, rec.FileID, rec.SizeBytes+1, rec.MTimeUnix, "aa"); err != nil || ok {
|
||||
t.Errorf("a hash of a file that changed in size was stored: ok=%v err=%v", ok, err)
|
||||
}
|
||||
if ok, err := repo.SaveHash(ctx, rec.FileID, rec.SizeBytes, rec.MTimeUnix+1, "aa"); err != nil || ok {
|
||||
t.Errorf("a hash of a file that changed in date was stored: ok=%v err=%v", ok, err)
|
||||
}
|
||||
if ok, err := repo.SaveHash(ctx, rec.FileID, rec.SizeBytes, rec.MTimeUnix, "aa"); err != nil || !ok {
|
||||
t.Fatalf("the hash was not stored: ok=%v err=%v", ok, err)
|
||||
}
|
||||
cs, _ := repo.Candidates(ctx)
|
||||
c, _ := candidateByRel(t, db, cs, "a/1.flac")
|
||||
other, _ := candidateByRel(t, db, cs, "b/2.flac")
|
||||
if c.Hash != "aa" || other.Hash != "" {
|
||||
t.Errorf("hashes = %q and %q, want aa and nothing", c.Hash, other.Hash)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// A hash says "this is what the file held when it was read". A scan that finds the file changed forgets it.
|
||||
func TestAScanThatFindsAFileChangedForgetsItsHash(t *testing.T) {
|
||||
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
|
||||
ctx := context.Background()
|
||||
lib := NewLibraryRepo(db)
|
||||
l := mkLib(t, lib, "music")
|
||||
saved(t, lib, l, song("a/1.flac", "Song", "A", 200_000, 1234, "flac", 900, "Ann"))
|
||||
repo := NewDuplicatesRepo(db)
|
||||
rec, _ := repo.FileOf(ctx, trackOf(t, db, "a/1.flac"))
|
||||
if _, err := repo.SaveHash(ctx, rec.FileID, rec.SizeBytes, rec.MTimeUnix, "aa"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The same file read again, unchanged (a full rescan): the hash is still true.
|
||||
if _, err := lib.SaveFile(ctx, l, song("a/1.flac", "Song", "A", 200_000, 1234, "flac", 900, "Ann"), rec.FileID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, _ := repo.FileOf(ctx, rec.TrackID); got.Hash != "aa" {
|
||||
t.Errorf("an unchanged file lost its hash: %q", got.Hash)
|
||||
}
|
||||
|
||||
// The file was edited: another size.
|
||||
if _, err := lib.SaveFile(ctx, l, song("a/1.flac", "Song", "A", 200_000, 4321, "flac", 900, "Ann"), rec.FileID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, _ := repo.FileOf(ctx, rec.TrackID); got.Hash != "" || got.SizeBytes != 4321 {
|
||||
t.Errorf("after a change: hash %q size %d, want no hash and the new size", got.Hash, got.SizeBytes)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRemovingACopyMovesItsListensToTheOneThatStays(t *testing.T) {
|
||||
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
|
||||
ctx := context.Background()
|
||||
lib := NewLibraryRepo(db)
|
||||
l := mkLib(t, lib, "music")
|
||||
saved(t, lib, l, song("a/1.flac", "Song", "A", 200_000, 1, "flac", 900, "Ann"))
|
||||
saved(t, lib, l, song("b/2.mp3", "Song", "B", 200_000, 1, "mp3", 192, "Ann"))
|
||||
keep, gone := trackOf(t, db, "a/1.flac"), trackOf(t, db, "b/2.mp3")
|
||||
|
||||
user, err := NewUserRepo(db).CreateUser(ctx, "alice", "h", false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plays := NewListeningRepo(db)
|
||||
for _, e := range []struct {
|
||||
track int64
|
||||
key string
|
||||
}{{keep, "k1"}, {gone, "g1"}, {gone, "g2"}} {
|
||||
if _, err := plays.Record(ctx, listening.Event{UserID: user.ID, TrackID: e.track, Type: listening.Complete, DurationPlayedMs: 200_000, IdempotencyKey: e.key}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := NewDuplicatesRepo(db).RemoveCopy(ctx, keep, gone); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if n := count(t, db, "SELECT COUNT(*) FROM listening_events WHERE track_id = ?", keep); n != 3 {
|
||||
t.Errorf("the track that stays has %d listens, want its own and the 2 moved (3)", n)
|
||||
}
|
||||
if n := count(t, db, "SELECT COUNT(*) FROM listening_events"); n != 3 {
|
||||
t.Errorf("%d listens in all, want 3: none lost", n)
|
||||
}
|
||||
if n := count(t, db, "SELECT COUNT(*) FROM tracks WHERE id = ?", gone); n != 0 {
|
||||
t.Error("the removed track is still there")
|
||||
}
|
||||
if n := count(t, db, "SELECT COUNT(*) FROM audio_files WHERE track_id = ?", gone); n != 0 {
|
||||
t.Error("the file row of the removed track is still there")
|
||||
}
|
||||
if n := count(t, db, "SELECT COUNT(*) FROM track_artists WHERE track_id = ?", gone); n != 0 {
|
||||
t.Error("the credits of the removed track are still there")
|
||||
}
|
||||
if n := count(t, db, "SELECT COUNT(*) FROM tracks WHERE id = ?", keep); n != 1 {
|
||||
t.Error("the track that stays is gone")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRemovingACopyNeedsTwoTracksThatExistAndAreDifferent(t *testing.T) {
|
||||
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
|
||||
ctx := context.Background()
|
||||
lib := NewLibraryRepo(db)
|
||||
l := mkLib(t, lib, "music")
|
||||
saved(t, lib, l, song("a/1.flac", "Song", "A", 200_000, 1, "flac", 900, "Ann"))
|
||||
keep := trackOf(t, db, "a/1.flac")
|
||||
repo := NewDuplicatesRepo(db)
|
||||
if err := repo.RemoveCopy(ctx, keep, 9999); !errors.Is(err, duplicates.ErrTrackNotFound) {
|
||||
t.Errorf("a track that does not exist: %v, want ErrTrackNotFound", err)
|
||||
}
|
||||
if err := repo.RemoveCopy(ctx, 9999, keep); !errors.Is(err, duplicates.ErrTrackNotFound) {
|
||||
t.Errorf("a track to keep that does not exist: %v, want ErrTrackNotFound", err)
|
||||
}
|
||||
if err := repo.RemoveCopy(ctx, keep, keep); !errors.Is(err, duplicates.ErrInvalid) {
|
||||
t.Errorf("a track removed in favour of itself: %v, want ErrInvalid", err)
|
||||
}
|
||||
if n := count(t, db, "SELECT COUNT(*) FROM tracks"); n != 1 {
|
||||
t.Errorf("%d tracks left, want the one that was there", n)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
@ -60,16 +60,18 @@ type trackArtistModel struct {
|
|||
type audioFileModel struct {
|
||||
bun.BaseModel `bun:"table:audio_files"`
|
||||
|
||||
ID int64 `bun:"id,pk,autoincrement"`
|
||||
TrackID int64 `bun:"track_id,notnull"`
|
||||
LibraryID int64 `bun:"library_id,notnull"`
|
||||
RelPath string `bun:"rel_path,notnull"`
|
||||
Format string `bun:"format,notnull"`
|
||||
SizeBytes int64 `bun:"size_bytes,notnull"`
|
||||
MTimeUnix int64 `bun:"mtime_unix,notnull"`
|
||||
SampleRate int `bun:"sample_rate,nullzero"`
|
||||
BitRate int `bun:"bitrate,nullzero"`
|
||||
MissingAt sql.NullInt64 `bun:"missing_at"`
|
||||
ID int64 `bun:"id,pk,autoincrement"`
|
||||
TrackID int64 `bun:"track_id,notnull"`
|
||||
LibraryID int64 `bun:"library_id,notnull"`
|
||||
RelPath string `bun:"rel_path,notnull"`
|
||||
Format string `bun:"format,notnull"`
|
||||
SizeBytes int64 `bun:"size_bytes,notnull"`
|
||||
MTimeUnix int64 `bun:"mtime_unix,notnull"`
|
||||
SampleRate int `bun:"sample_rate,nullzero"`
|
||||
BitRate int `bun:"bitrate,nullzero"`
|
||||
// ContentHash is the SHA-256 of the file as it was when it was read for duplicates; empty until then.
|
||||
ContentHash sql.NullString `bun:"content_hash"`
|
||||
MissingAt sql.NullInt64 `bun:"missing_at"`
|
||||
}
|
||||
|
||||
type libraryModel struct {
|
||||
|
|
@ -198,11 +200,15 @@ func (r *LibraryRepo) SaveFile(ctx context.Context, libraryID int64, f library.P
|
|||
return err
|
||||
}
|
||||
}
|
||||
// A hash says what the file held when it was read: a file that changed has no hash any more.
|
||||
if file.SizeBytes != f.Size || file.MTimeUnix != f.MTimeUnix {
|
||||
file.ContentHash = sql.NullString{}
|
||||
}
|
||||
file.Format, file.SizeBytes, file.MTimeUnix = f.Format, f.Size, f.MTimeUnix
|
||||
file.SampleRate, file.BitRate = f.SampleRate, f.BitRate
|
||||
file.MissingAt = sql.NullInt64{}
|
||||
if _, err := tx.NewUpdate().Model(&file).
|
||||
Column("format", "size_bytes", "mtime_unix", "sample_rate", "bitrate", "missing_at").
|
||||
Column("format", "size_bytes", "mtime_unix", "sample_rate", "bitrate", "content_hash", "missing_at").
|
||||
WherePK().Exec(ctx); err != nil {
|
||||
return fmt.Errorf("update audio file: %w", err)
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue