Merge pull request 'feat: find duplicate songs and delete the extra copies (server)' (#90) from feat/duplicates-api into main

This commit is contained in:
squidd 2026-11-12 17:02:30 +01:00
commit eebeb2b10c
22 changed files with 2964 additions and 15 deletions

View file

@ -9,9 +9,10 @@ It is a single executable, one database (SQLite by default, PostgreSQL if you pr
## What works today
- **Accounts.** An account is an email address and a password. The first user to register becomes the admin and registration then closes; admins create the other users. The address only identifies the account: Sonda sends no mail, does not check that the address exists, and keeps it on your server. It is trimmed and lowercased, and unique. Accounts made before addresses were asked for keep the plain name they had as their login. Short-lived access tokens (JWT) plus rotating refresh tokens, so a stolen refresh token is detected.
- **Libraries.** A library is a name and a folder, managed through the API. Sonda only ever reads your music folders: it never writes, renames or deletes a file.
- **Libraries.** A library is a name and a folder, managed through the API. Sonda only reads your music folders: it never writes or renames a file, and it deletes one only when an admin asks to remove the extra copy of a song and you turned on `library.allow_delete` (see **Duplicates**), which is off by default.
- **Scanner.** Reads MP3 and FLAC tags and audio properties with several workers in parallel. Rescans are incremental, renamed files keep their identity, vanished files are flagged (never deleted), and a library whose folder looks empty or unreadable (an unmounted NAS) is left alone instead of being wiped. Audio in other formats (m4a, ogg, opus, wav, wma and similar) is not read yet, but it is counted as `unsupported` in the scan result instead of disappearing silently. NAS housekeeping folders (`@Recycle`, `#recycle`, `@eaDir`, `.@__thumb`, `$RECYCLE.BIN`) are skipped.
- **Catalog.** Browse and search artists, albums and tracks, with paging, by library, artist or album. Only tracks you can actually play are listed.
- **Duplicates.** `GET /library/duplicates` (admin) lists the songs that are in more than one file: the same set of main artists, the same title and durations within 3 seconds, possibly on different albums. It is worked out from the index, touches no file, and returns no file path. Each group proposes the copy to keep (lossless first, then the higher bitrate, then the bigger file, then the oldest). A background job (`POST /library/duplicates/check`, like the other bulk jobs) reads the files of the groups and keeps their SHA-256, so a group can be called `identical` (byte for byte). `POST /library/duplicates/clean` **deletes the extra copies of one song from the disk**, and it is the one thing Sonda does that writes in your music folders, so it is off until you set `library.allow_delete` (or `SONDA_ALLOW_DELETE=true`) and mount the folders writable. It deletes only when an admin asks, never by itself, and the server checks again at that moment that no scan is running, that the tracks are copies of one song, and that every file is still the one the last scan saw (same size and date, a regular file, not a link, inside its library): so the last copy of a song is never deleted, a file that changed is not deleted, and every file is checked before the first is deleted. The listens and the playlist items of a deleted copy move to the one that stays, so your taste and your playlists are not lost. Folders are never removed, and nothing is sent anywhere.
- **Streaming.** `GET` and `HEAD` with `Range` support, so players can seek. Opening or prefetching a stream does not record a listen.
- **API documentation.** The full API is described in OpenAPI and browsable with Swagger UI, served by Sonda itself. The UI is embedded in the binary, so opening it contacts no other server.
- **Listening history.** Clients report explicit `play`, `complete` and `skip` events. An event counts as a listen when it is `complete`, or when it was played for at least 20 seconds (`listening.min_listen_seconds`); the rule is applied when the history is read, so changing it re-reads the past. You can read your own history, with a `counted` flag on every event. It stays in your database and is never sent anywhere.
@ -61,7 +62,7 @@ docker run -d --name sonda --restart unless-stopped --stop-timeout 15 \
sonda
```
Mount your music read-only wherever you like, then add the library through the API with the path *inside* the container, `/music` above (not the folder of your host): the [Demo](#demo) works the same way with `BASE=http://127.0.0.1:8080/api/v1`, except that step 4 creates the library with `"path":"/music"`. Keep `sonda.env` and pass it every time you run a new container, for example after an upgrade: a new secret signs out everybody. The container's health check asks the port in `SONDA_LISTEN`, so it works for `0.0.0.0`, `[::]` and a port alone, but not if you bind the server to one interface address. Published like this the server answers only on this machine; to serve your network, publish the port without `127.0.0.1:` and put a reverse proxy with https in front. The other settings are the environment variables of the [table below](#configure) (`SONDA_DB_DRIVER` and `SONDA_DB_DSN` for PostgreSQL); the image sets `SONDA_LISTEN`, `SONDA_DB_PATH` and `SONDA_DATA_DIR` for the volume.
Mount your music read-only wherever you like (only if you want Sonda to delete duplicate files, mount it writable and set `SONDA_ALLOW_DELETE=true`: see **Duplicates**), then add the library through the API with the path *inside* the container, `/music` above (not the folder of your host): the [Demo](#demo) works the same way with `BASE=http://127.0.0.1:8080/api/v1`, except that step 4 creates the library with `"path":"/music"`. Keep `sonda.env` and pass it every time you run a new container, for example after an upgrade: a new secret signs out everybody. The container's health check asks the port in `SONDA_LISTEN`, so it works for `0.0.0.0`, `[::]` and a port alone, but not if you bind the server to one interface address. Published like this the server answers only on this machine; to serve your network, publish the port without `127.0.0.1:` and put a reverse proxy with https in front. The other settings are the environment variables of the [table below](#configure) (`SONDA_DB_DRIVER` and `SONDA_DB_DSN` for PostgreSQL); the image sets `SONDA_LISTEN`, `SONDA_DB_PATH` and `SONDA_DATA_DIR` for the volume.
To keep the data in a folder of the host instead of a volume, run the container as the owner of that folder, or it cannot write there: add `--user "$(id -u):$(id -g)"` and `-v /path/to/sonda-data:/data`. Back up `sonda.db` (and treat the backup like a secret: it holds the API keys of the providers); the rest of `/data` is built from your music.
@ -87,6 +88,7 @@ Precedence is: built-in defaults, then the file, then environment variables. An
|---|---|---|---|
| `server.listen` | `SONDA_LISTEN` | `127.0.0.1:8080` | Address to listen on. Localhost by default. |
| `library.scan_workers` | | `4` | Files whose tags are read in parallel during a scan. |
| `library.allow_delete` | `SONDA_ALLOW_DELETE` | `false` | Lets an admin delete the extra copy of a song from the music folders (see **Duplicates**). It is the one thing that makes Sonda write there, so it needs the folders mounted writable. A value that is not `true` or `false` is an error. |
| `database.driver` | `SONDA_DB_DRIVER` | `sqlite` | `sqlite` or `postgres`. |
| `database.path` | `SONDA_DB_PATH` | `sonda.db` | SQLite file (driver `sqlite`). |
| `database.dsn` | `SONDA_DB_DSN` | | Connection string (driver `postgres`). |
@ -301,6 +303,7 @@ Everything lives under `/api/v1`. Bodies are JSON. Errors always look like `{"er
| admin | `GET`, `POST /libraries`; `PATCH`, `DELETE /libraries/{id}` | Manage libraries. Deleting one removes its index entries, never your files. |
| admin | `POST`, `DELETE /library/covers`; `POST`, `DELETE /library/artist-images`; `POST`, `DELETE /library/descriptions` | Start, in the background, a job that builds the cover of every album, looks for the picture of every artist or for the description of every artist (`?refresh=true` for all), or stop it. One of each at a time. |
| admin | `POST /library/scan`, `POST /libraries/{id}/scan` | Scan everything, or one library. `?full=true` reads every file again. |
| admin | `GET /library/duplicates`; `POST`, `GET`, `DELETE /library/duplicates/check`; `POST /library/duplicates/clean` | Find the songs that are in more than one file, read their files to tell the identical ones, and delete the extra copies. See **Duplicates**. |
| admin | `GET /providers`, `PUT /providers/{name}`, `POST /providers/{name}/test`, `DELETE /providers/{name}/data` | See, set up and check the providers (Last.fm, MusicBrainz, Wikipedia, Commons, the Cover Art Archive, Lidarr), and remove what they gave. The API key is write-only. |
| admin | `GET /providers/lidarr/options` | The profiles and root folders that Lidarr offers, to choose the ones artists are sent with. |
| admin | `POST /artists/{id}/lidarr` | Send an artist to your own Lidarr, by its MusicBrainz id, which then looks for the music. Never by name, never by itself. |
@ -313,7 +316,7 @@ A library path must be an absolute, existing folder, never the filesystem root,
- Sonda listens on localhost only, unless you change `server.listen`. To use it from other machines, put it behind a reverse proxy that terminates HTTPS.
- Passwords are hashed with Argon2id. Refresh tokens are stored only as hashes. An access token is checked against the database on every request, so it stops working as soon as its account is deleted, even if the same user id is given to a new account.
- Music folders are read-only to Sonda. Streaming opens files through `os.Root`, so a path can never leave its library, even if a symlink is swapped in later.
- Music folders are read-only to Sonda, except for one thing that is off until you turn it on: deleting the extra copy of a song (`library.allow_delete`). Streaming and that deletion open files through `os.Root`, so a path can never leave its library, even if a symlink is swapped in later.
- File paths are never returned by the catalog, and audio file paths are not logged.
- Provider API keys are write-only through the API and never logged. They sit in the database, so protect your backups.

View file

@ -21,6 +21,7 @@ import (
"codeberg.org/squidd/sonda/internal/config"
"codeberg.org/squidd/sonda/internal/discover"
"codeberg.org/squidd/sonda/internal/discovery"
"codeberg.org/squidd/sonda/internal/duplicates"
"codeberg.org/squidd/sonda/internal/feedback"
"codeberg.org/squidd/sonda/internal/images"
"codeberg.org/squidd/sonda/internal/jobs"
@ -262,6 +263,17 @@ func run() error {
Preference: cfg.Scoring.WeightPreference, Discovery: cfg.Scoring.WeightDiscovery,
}
// Deleting the extra copy of a song is the one thing that makes Sonda write in the music folders:
// off unless the operator turned it on, and said in the log when it is on.
duplicateSvc := duplicates.NewService(storage.NewDuplicatesRepo(db), duplicates.Options{
AllowDelete: cfg.Library.AllowDelete,
ScanRunning: func() bool { return scans.Status().Running },
Logger: logger,
})
if cfg.Library.AllowDelete {
logger.Warn("library.allow_delete is on: an admin can delete duplicate files from the music folders")
}
router := httpapi.NewRouter(httpapi.Deps{
Logger: logger,
Auth: auth.NewService(storage.NewUserRepo(db), storage.NewRefreshRepo(db), tokens, cfg.Auth.RefreshTTL()),
@ -283,6 +295,8 @@ func run() error {
CoversJob: jobs.Runner{Ctx: ctx, Job: images.NewCoversJob(pictures, imageRepo)},
ArtistImagesJob: jobs.Runner{Ctx: ctx, Job: images.NewArtistImagesJob(pictures, imageRepo)},
DescriptionsJob: jobs.Runner{Ctx: ctx, Job: about.NewJob(descriptions, discoveryRepo)},
Duplicates: duplicateSvc,
DuplicatesJob: jobs.Runner{Ctx: ctx, Job: duplicates.NewHashJob(duplicateSvc)},
ArtistImages: pictures,
About: descriptions,
Settings: userSettings,

View file

@ -21,7 +21,7 @@ services:
# TZ: Europe/Rome
volumes:
- sonda-data:/data # the database and the files Sonda makes: back up the database
- ${MUSIC_DIR:?set MUSIC_DIR in .env to the folder of your music}:/music:ro # your music, read-only: a library is added with the path /music
- ${MUSIC_DIR:?set MUSIC_DIR in .env to the folder of your music}:/music:ro # your music, read-only: a library is added with the path /music. To let Sonda delete duplicate files, drop ":ro" and set SONDA_ALLOW_DELETE: "true" above
volumes:
sonda-data:

View file

@ -20,6 +20,7 @@ import (
"codeberg.org/squidd/sonda/internal/catalog"
"codeberg.org/squidd/sonda/internal/discover"
"codeberg.org/squidd/sonda/internal/discovery"
"codeberg.org/squidd/sonda/internal/duplicates"
"codeberg.org/squidd/sonda/internal/feedback"
"codeberg.org/squidd/sonda/internal/images"
"codeberg.org/squidd/sonda/internal/jobs"
@ -117,6 +118,14 @@ func newTestEnvDB(t *testing.T, mutate func(*Deps, *bun.DB)) (http.Handler, *bun
if deps.DescriptionsJob == nil {
deps.DescriptionsJob = jobs.Runner{Ctx: context.Background(), Job: about.NewJob(about.NewService(storage.NewDiscoveryRepo(db), noWikipedia{}, prov, []string{"en"}, zap.NewNop()), storage.NewDiscoveryRepo(db))}
}
// Deleting is off unless a test turns it on.
if deps.Duplicates == nil {
svc := duplicates.NewService(storage.NewDuplicatesRepo(db), duplicates.Options{Logger: zap.NewNop()})
deps.Duplicates = svc
if deps.DuplicatesJob == nil {
deps.DuplicatesJob = jobs.Runner{Ctx: context.Background(), Job: duplicates.NewHashJob(svc)}
}
}
// Built from the final services, so a test that swaps one gets a Discover that uses it.
if deps.Discover == nil {
deps.Discover = discover.NewService(deps.Taste.(discover.Taster), deps.Discovery.(discover.Similar),

View file

@ -26,7 +26,9 @@ import (
"codeberg.org/squidd/sonda/internal/catalog"
"codeberg.org/squidd/sonda/internal/discovery"
"codeberg.org/squidd/sonda/internal/duplicates"
"codeberg.org/squidd/sonda/internal/images"
"codeberg.org/squidd/sonda/internal/jobs"
"codeberg.org/squidd/sonda/internal/library"
"codeberg.org/squidd/sonda/internal/providers"
"codeberg.org/squidd/sonda/internal/providers/lastfm"
@ -140,6 +142,11 @@ func newContract(t *testing.T) *contract {
libs := library.NewLibraries(repo)
libs.SetBusy(func() bool { return mgr.Status().Running })
d.Scan = scanAdapter{mgr}
dupSvc := duplicates.NewService(storage.NewDuplicatesRepo(db), duplicates.Options{
AllowDelete: true, ScanRunning: func() bool { return mgr.Status().Running }, Logger: zap.NewNop(),
})
d.Duplicates = dupSvc
d.DuplicatesJob = jobs.Runner{Ctx: context.Background(), Job: duplicates.NewHashJob(dupSvc)}
d.Libraries = libs
d.Catalog = catalog.NewService(storage.NewCatalogRepo(db))
d.Streams = streaming.NewService(repo)
@ -812,6 +819,9 @@ func TestResponsesMatchTheSpec(t *testing.T) {
c.expect(c.call("PATCH", "/me/settings", token, map[string]any{"nope": true}), 400, "a setting that does not exist", "invalid_input")
c.expect(c.call("GET", "/me/settings", "", nil), 401, "settings without a token", "unauthorized")
// ---- duplicate songs ----
c.duplicates(t, token, libID)
// ---- every operation in the spec was exercised ----
var missing []string
for _, op := range specOperations(loadSpec(t)) {
@ -824,6 +834,132 @@ func TestResponsesMatchTheSpec(t *testing.T) {
}
}
// duplicates exercises the duplicate songs: the same song as a.mp3 is added to the library, as real
// files, so that what is checked against the spec is what the real services answer.
func (c *contract) duplicates(t *testing.T, token, libID string) {
t.Helper()
repo := storage.NewLibraryRepo(c.db)
libInt, err := strconv.ParseInt(libID, 10, 64)
if err != nil {
t.Fatal(err)
}
// addCopy puts a second file with the tags of Song A in the library, as the scanner would.
addCopy := func(name string) {
t.Helper()
path := filepath.Join(c.music, name)
if err := os.WriteFile(path, bytes.Repeat([]byte{0xAB}, 400), 0o644); err != nil {
t.Fatal(err)
}
info, err := os.Stat(path)
if err != nil {
t.Fatal(err)
}
f := library.ParsedFile{
FileInfo: library.FileInfo{RelPath: name, Size: info.Size(), MTimeUnix: info.ModTime().Unix()},
Metadata: library.Metadata{
Tags: library.Tags{Title: "Song A", Album: "Compilation", AlbumArtist: "Artist X", Artists: []string{"Artist X"}, TrackNo: 1, DiscNo: 1},
Properties: library.Properties{Format: "mpeg", DurationMs: 180000, SampleRate: 44100},
},
}
if _, err := repo.SaveFile(context.Background(), libInt, f, 0); err != nil {
t.Fatal(err)
}
}
trackOf := func(name string) int64 {
t.Helper()
var id int64
if err := c.db.QueryRow("SELECT track_id FROM audio_files WHERE rel_path = ?", name).Scan(&id); err != nil {
t.Fatal(err)
}
return id
}
exists := func(name string) bool { _, err := os.Lstat(filepath.Join(c.music, name)); return err == nil }
body := func(keep int64, remove ...int64) map[string]any {
return map[string]any{"keep_track_id": keep, "remove_track_ids": remove}
}
none := c.expect(c.call("GET", "/library/duplicates", token, nil), 200, "duplicates when there are none")
if none["total"] != float64(0) || none["groups"] == nil || none["deletion_allowed"] != true {
t.Fatalf("duplicates = %v, want no group, an empty list and deleting allowed", none)
}
addCopy("dup.mp3")
keep, copyID := trackOf("a.mp3"), trackOf("dup.mp3")
list := c.expect(c.call("GET", "/library/duplicates", token, nil), 200, "duplicates")
groups, _ := list["groups"].([]any)
if list["total"] != float64(1) || len(groups) != 1 {
t.Fatalf("duplicates = %v, want the one group of Song A", list)
}
if g := groups[0].(map[string]any); g["title"] != "Song A" || g["suggested_keep_track_id"] != float64(keep) || g["hashed"] != false {
t.Fatalf("group = %v, want Song A with a.mp3 to keep and nothing read yet", g)
}
c.expect(c.call("GET", "/library/duplicates?limit=-1", token, nil), 400, "duplicates with a bad limit", "invalid_input")
c.expect(c.call("GET", "/library/duplicates", "", nil), 401, "duplicates without a token", "unauthorized")
// Not an admin: nothing of this is theirs.
c.expect(c.call("POST", "/users", token, map[string]any{"email": "carol@example.org", "password": "carols long pass"}), 201, "create a plain user")
carol := c.expect(c.call("POST", "/auth/login", "", map[string]string{"email": "carol@example.org", "password": "carols long pass"}), 200, "carol logs in")["access_token"].(string)
c.expect(c.call("GET", "/library/duplicates", carol, nil), 403, "duplicates as a plain user", "admin_required")
c.expect(c.call("POST", "/library/duplicates/clean", carol, body(keep, copyID)), 403, "cleaning as a plain user", "admin_required")
// The check reads the files, and the group is then identical.
if idle := c.expect(c.call("GET", "/library/duplicates/check", token, nil), 200, "the check before it ran"); idle["running"] != false {
t.Fatalf("check = %v", idle)
}
c.expect(c.call("POST", "/library/duplicates/check?refresh=maybe", token, nil), 400, "a check with a bad refresh", "invalid_input")
c.expect(c.call("POST", "/library/duplicates/check", "", nil), 401, "a check without a token", "unauthorized")
c.expect(c.call("POST", "/library/duplicates/check", token, nil), 202, "start the check")
for i := 0; ; i++ {
st := c.expect(c.call("GET", "/library/duplicates/check", token, nil), 200, "the check running")
if st["running"] == false {
if last, _ := st["last"].(map[string]any); last == nil || last["checked"] != float64(2) || last["hashed"] != float64(2) {
t.Fatalf("check = %v, want 2 files checked and hashed", st)
}
break
}
if i > 400 {
t.Fatal("the check did not finish")
}
time.Sleep(25 * time.Millisecond)
}
c.expect(c.call("DELETE", "/library/duplicates/check", token, nil), 204, "stop the check when none runs")
after := c.expect(c.call("GET", "/library/duplicates", token, nil), 200, "duplicates after the check")
if g := after["groups"].([]any)[0].(map[string]any); g["hashed"] != true || g["identical"] != true {
t.Fatalf("group = %v, want hashed and identical", g)
}
// Refusals first: nothing is removed by any of them.
c.expect(c.call("POST", "/library/duplicates/clean", token, body(keep, keep)), 400, "cleaning the copy to keep", "invalid_input")
c.expect(c.call("POST", "/library/duplicates/clean", token, map[string]any{"keep_track_id": keep}), 400, "cleaning with nothing to remove", "invalid_input")
c.expect(c.call("POST", "/library/duplicates/clean", token, body(keep, 99999)), 404, "cleaning a track that does not exist", "track_not_found")
c.expect(c.call("POST", "/library/duplicates/clean", "", body(keep, copyID)), 401, "cleaning without a token", "unauthorized")
other := trackOf("b.flac")
c.expect(c.call("POST", "/library/duplicates/clean", token, body(keep, other)), 400, "cleaning something that is not a copy", "invalid_input")
if !exists("dup.mp3") || !exists("b.flac") {
t.Fatal("a refused request removed a file")
}
// A copy that changed since the scan is not removed.
addCopy("dup2.mp3")
dup2 := trackOf("dup2.mp3")
if err := os.WriteFile(filepath.Join(c.music, "dup2.mp3"), bytes.Repeat([]byte{0xCD}, 500), 0o644); err != nil {
t.Fatal(err)
}
c.expect(c.call("POST", "/library/duplicates/clean", token, body(keep, dup2)), 409, "cleaning a copy that changed", "duplicate_changed")
if !exists("dup2.mp3") {
t.Fatal("a copy that changed was removed")
}
// The real thing: the extra copy leaves the disk and the index, the other stays.
done := c.expect(c.call("POST", "/library/duplicates/clean", token, body(keep, copyID)), 200, "clean the duplicate")
if done["removed"] != float64(1) || done["freed_bytes"] != float64(400) {
t.Fatalf("clean = %v, want 1 file and 400 bytes", done)
}
if exists("dup.mp3") || !exists("a.mp3") {
t.Fatalf("after the clean dup.mp3 exists=%v a.mp3 exists=%v", exists("dup.mp3"), exists("a.mp3"))
}
}
type orphanIDs struct{ album, track string }
// addOrphan inserts an album with no album artist and a track in it whose file

View file

@ -0,0 +1,375 @@
package httpapi
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
"github.com/uptrace/bun"
"go.uber.org/zap"
"codeberg.org/squidd/sonda/internal/duplicates"
"codeberg.org/squidd/sonda/internal/jobs"
"codeberg.org/squidd/sonda/internal/library"
"codeberg.org/squidd/sonda/internal/storage"
)
// dupAPI is the API over a real library on a real folder.
type dupAPI struct {
t *testing.T
h http.Handler
db *bun.DB
dir string
token string
scanning bool
}
func newDupAPI(t *testing.T, allow bool) *dupAPI {
t.Helper()
a := &dupAPI{t: t}
a.h, a.db = newTestEnvDB(t, func(d *Deps, db *bun.DB) {
svc := duplicates.NewService(storage.NewDuplicatesRepo(db), duplicates.Options{
AllowDelete: allow, ScanRunning: func() bool { return a.scanning }, Logger: zap.NewNop(),
})
d.Duplicates = svc
d.DuplicatesJob = jobs.Runner{Ctx: context.Background(), Job: duplicates.NewHashJob(svc)}
})
dir, err := filepath.EvalSymlinks(t.TempDir())
if err != nil {
t.Fatal(err)
}
a.dir = dir
a.token = registerAndLogin(t, a.h)
if _, err := storage.NewLibraryRepo(a.db).CreateLibrary(context.Background(), "music", dir); err != nil {
t.Fatal(err)
}
return a
}
// seed writes a file and indexes it as it is, and returns its track.
func (a *dupAPI) seed(rel, content, title, album string, durationMs int64, format string, bitrate int, artists ...string) int64 {
a.t.Helper()
path := filepath.Join(a.dir, filepath.FromSlash(rel))
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
a.t.Fatal(err)
}
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
a.t.Fatal(err)
}
info, err := os.Stat(path)
if err != nil {
a.t.Fatal(err)
}
f := library.ParsedFile{
FileInfo: library.FileInfo{RelPath: rel, Size: info.Size(), MTimeUnix: info.ModTime().Unix()},
Metadata: library.Metadata{
Tags: library.Tags{Title: title, Album: album, AlbumArtist: artists[0], Artists: artists, TrackNo: 1, DiscNo: 1},
Properties: library.Properties{Format: format, DurationMs: durationMs, SampleRate: 44100, BitRate: bitrate},
},
}
if _, err := storage.NewLibraryRepo(a.db).SaveFile(context.Background(), 1, f, 0); err != nil {
a.t.Fatal(err)
}
var id int64
if err := a.db.QueryRow("SELECT track_id FROM audio_files WHERE rel_path = ?", rel).Scan(&id); err != nil {
a.t.Fatal(err)
}
return id
}
func (a *dupAPI) twoCopies() (flac, mp3 int64) {
flac = a.seed("a/song.flac", "FLAC-CONTENT-OF-THE-SONG", "Song", "Album", 200_000, "flac", 900, "Ann")
mp3 = a.seed("b/song.mp3", "mp3 content", "Song", "Greatest hits", 200_500, "mp3", 192, "Ann")
return flac, mp3
}
func (a *dupAPI) exists(rel string) bool {
_, err := os.Lstat(filepath.Join(a.dir, filepath.FromSlash(rel)))
return err == nil
}
func (a *dupAPI) get(path string) resp { return do(a.t, a.h, "GET", path, a.token, nil) }
func (a *dupAPI) clean(keep int64, remove ...int64) resp {
return do(a.t, a.h, "POST", "/api/v1/library/duplicates/clean", a.token, map[string]any{"keep_track_id": keep, "remove_track_ids": remove})
}
func TestDuplicateRoutesAreAdminOnly(t *testing.T) {
a := newDupAPI(t, true)
_, bob := addUser(t, a.h, a.token, "bob", false)
calls := []struct{ method, path string }{
{"GET", "/api/v1/library/duplicates"},
{"POST", "/api/v1/library/duplicates/clean"},
{"GET", "/api/v1/library/duplicates/check"},
{"POST", "/api/v1/library/duplicates/check"},
{"DELETE", "/api/v1/library/duplicates/check"},
}
for _, c := range calls {
if r := do(t, a.h, c.method, c.path, "", nil); r.Code != http.StatusUnauthorized {
t.Errorf("%s %s without a token: %d", c.method, c.path, r.Code)
}
r := do(t, a.h, c.method, c.path, bob, map[string]any{"keep_track_id": 1, "remove_track_ids": []int{2}})
if r.Code != http.StatusForbidden || errCode(r) != "admin_required" {
t.Errorf("%s %s as a plain user: %d %v", c.method, c.path, r.Code, r.Body)
}
}
}
func TestTheListShowsTheGroupsAndNeverAPath(t *testing.T) {
a := newDupAPI(t, true)
flac, mp3 := a.twoCopies()
a.seed("c/other.flac", "else", "Other", "Album", 100_000, "flac", 900, "Ann")
rec := httptest.NewRecorder()
req := httptest.NewRequest("GET", "/api/v1/library/duplicates", nil)
req.Header.Set("Authorization", "Bearer "+a.token)
a.h.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status %d: %s", rec.Code, rec.Body)
}
raw := rec.Body.String()
for _, secret := range []string{a.dir, "song.flac", "song.mp3", "rel_path", "path"} {
if strings.Contains(raw, secret) {
t.Errorf("the answer contains %q: file paths are private", secret)
}
}
var out struct {
DeletionAllowed bool `json:"deletion_allowed"`
Total, Limit int
Offset int
Groups []struct {
Title string `json:"title"`
Artists []struct {
ID int64 `json:"id"`
Name string `json:"name"`
} `json:"artists"`
Files []struct {
TrackID int64 `json:"track_id"`
Album *struct {
ID int64 `json:"id"`
Title string `json:"title"`
} `json:"album"`
Format string `json:"format"`
Bitrate int `json:"bitrate"`
SizeBytes int64 `json:"size_bytes"`
DurationMs int64 `json:"duration_ms"`
Hashed bool `json:"hashed"`
} `json:"files"`
Hashed bool `json:"hashed"`
Identical bool `json:"identical"`
Keep int64 `json:"suggested_keep_track_id"`
FreedBytes int64 `json:"freed_bytes"`
} `json:"groups"`
}
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
t.Fatal(err)
}
if !out.DeletionAllowed || out.Total != 1 || out.Limit != 50 || out.Offset != 0 || len(out.Groups) != 1 {
t.Fatalf("answer = %+v", out)
}
g := out.Groups[0]
if g.Title != "Song" || len(g.Artists) != 1 || g.Artists[0].Name != "Ann" || g.Keep != flac || g.FreedBytes != int64(len("mp3 content")) || g.Hashed || g.Identical {
t.Errorf("group = %+v", g)
}
if len(g.Files) != 2 || g.Files[1].TrackID != mp3 || g.Files[1].Album == nil || g.Files[1].Album.Title != "Greatest hits" ||
g.Files[1].Format != "mp3" || g.Files[1].Bitrate != 192 || g.Files[1].DurationMs != 200_500 || g.Files[1].Hashed {
t.Errorf("files = %+v", g.Files)
}
}
func TestTheListSaysWhenDeletingIsOff(t *testing.T) {
a := newDupAPI(t, false)
a.twoCopies()
r := a.get("/api/v1/library/duplicates")
if r.Code != 200 || r.Body["deletion_allowed"] != false || r.Body["total"] != float64(1) {
t.Errorf("answer = %d %v", r.Code, r.Body)
}
}
func TestTheListIsPagedAndCheckedForGoodParameters(t *testing.T) {
a := newDupAPI(t, true)
for _, title := range []string{"A song", "B song"} {
a.seed("x/"+title+".mp3", title+"1", title, "Album", 100_000, "mp3", 192, "Ann")
a.seed("y/"+title+".mp3", title+"22", title, "Album", 100_000, "mp3", 192, "Ann")
}
first := a.get("/api/v1/library/duplicates?limit=1")
second := a.get("/api/v1/library/duplicates?limit=1&offset=1")
if first.Body["total"] != float64(2) || len(first.Body["groups"].([]any)) != 1 || len(second.Body["groups"].([]any)) != 1 {
t.Fatalf("pages = %v and %v", first.Body, second.Body)
}
if first.Body["limit"] != float64(1) || second.Body["offset"] != float64(1) {
t.Errorf("limit and offset are not echoed: %v %v", first.Body, second.Body)
}
if past := a.get("/api/v1/library/duplicates?offset=10"); len(past.Body["groups"].([]any)) != 0 || past.Body["groups"] == nil {
t.Errorf("past the end: %v, want an empty list and not null", past.Body)
}
if big := a.get("/api/v1/library/duplicates?limit=100000"); big.Body["limit"] != float64(200) {
t.Errorf("a huge limit became %v, want 200", big.Body["limit"])
}
for _, q := range []string{"limit=-1", "offset=-1", "limit=x", "offset=1.5"} {
if r := a.get("/api/v1/library/duplicates?" + q); r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" {
t.Errorf("?%s: %d %v, want 400 invalid_input", q, r.Code, r.Body)
}
}
}
func TestCleaningRemovesTheCopyAndSaysHowMuchWasFreed(t *testing.T) {
a := newDupAPI(t, true)
flac, mp3 := a.twoCopies()
r := a.clean(flac, mp3)
if r.Code != http.StatusOK || r.Body["removed"] != float64(1) || r.Body["freed_bytes"] != float64(len("mp3 content")) {
t.Fatalf("answer = %d %v", r.Code, r.Body)
}
if a.exists("b/song.mp3") || !a.exists("a/song.flac") {
t.Error("the wrong file was removed")
}
if list := a.get("/api/v1/library/duplicates"); list.Body["total"] != float64(0) {
t.Errorf("the list still shows %v groups", list.Body["total"])
}
}
func TestEveryRefusalHasItsOwnStatusAndCode(t *testing.T) {
t.Run("deleting is off", func(t *testing.T) {
a := newDupAPI(t, false)
flac, mp3 := a.twoCopies()
if r := a.clean(flac, mp3); r.Code != http.StatusConflict || errCode(r) != "deletion_disabled" || !a.exists("b/song.mp3") {
t.Errorf("%d %v", r.Code, r.Body)
}
})
t.Run("a scan is running", func(t *testing.T) {
a := newDupAPI(t, true)
flac, mp3 := a.twoCopies()
a.scanning = true
if r := a.clean(flac, mp3); r.Code != http.StatusConflict || errCode(r) != "scan_running" || !a.exists("b/song.mp3") {
t.Errorf("%d %v", r.Code, r.Body)
}
})
t.Run("not copies of one song", func(t *testing.T) {
a := newDupAPI(t, true)
flac, _ := a.twoCopies()
other := a.seed("c/other.flac", "else", "Other", "Album", 100_000, "flac", 900, "Ann")
if r := a.clean(flac, other); r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" || !a.exists("c/other.flac") {
t.Errorf("%d %v", r.Code, r.Body)
}
})
t.Run("a track that does not exist", func(t *testing.T) {
a := newDupAPI(t, true)
flac, _ := a.twoCopies()
if r := a.clean(flac, 9999); r.Code != http.StatusNotFound || errCode(r) != "track_not_found" {
t.Errorf("%d %v", r.Code, r.Body)
}
})
t.Run("a file that changed since the scan", func(t *testing.T) {
a := newDupAPI(t, true)
flac, mp3 := a.twoCopies()
if err := os.WriteFile(filepath.Join(a.dir, "b/song.mp3"), []byte("edited, and longer than it was"), 0o644); err != nil {
t.Fatal(err)
}
if r := a.clean(flac, mp3); r.Code != http.StatusConflict || errCode(r) != "duplicate_changed" || !a.exists("b/song.mp3") {
t.Errorf("%d %v", r.Code, r.Body)
}
})
t.Run("a read-only folder", func(t *testing.T) {
if runtime.GOOS == "windows" || os.Geteuid() == 0 {
t.Skip("permissions are not enforced here")
}
a := newDupAPI(t, true)
flac, mp3 := a.twoCopies()
dir := filepath.Join(a.dir, "b")
if err := os.Chmod(dir, 0o555); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = os.Chmod(dir, 0o755) })
if r := a.clean(flac, mp3); r.Code != http.StatusConflict || errCode(r) != "library_read_only" || !a.exists("b/song.mp3") {
t.Errorf("%d %v", r.Code, r.Body)
}
})
}
func TestACleanRequestMustBeWellFormed(t *testing.T) {
a := newDupAPI(t, true)
flac, mp3 := a.twoCopies()
bodies := map[string]any{
"malformed JSON": `{"keep_track_id":`,
"no copy to keep": map[string]any{"remove_track_ids": []int64{mp3}},
"nothing to remove": map[string]any{"keep_track_id": flac, "remove_track_ids": []int64{}},
"the one to keep too": map[string]any{"keep_track_id": flac, "remove_track_ids": []int64{flac, mp3}},
"a negative id": map[string]any{"keep_track_id": flac, "remove_track_ids": []int64{-1}},
"ids that are strings": map[string]any{"keep_track_id": "1", "remove_track_ids": []string{"2"}},
}
for name, body := range bodies {
r := do(t, a.h, "POST", "/api/v1/library/duplicates/clean", a.token, body)
if r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" {
t.Errorf("%s: %d %v, want 400 invalid_input", name, r.Code, r.Body)
}
}
if !a.exists("b/song.mp3") || !a.exists("a/song.flac") {
t.Error("a file was removed by a bad request")
}
}
// --- the check job ---------------------------------------------------------------------------------
func TestTheCheckJobReadsTheFilesAndReportsWhatItDid(t *testing.T) {
a := newDupAPI(t, true)
a.seed("a/1.mp3", "the same bytes", "Same", "A", 100_000, "mp3", 192, "Ann")
a.seed("b/2.mp3", "the same bytes", "Same", "B", 100_000, "mp3", 192, "Ann")
if idle := a.get("/api/v1/library/duplicates/check"); idle.Code != 200 || idle.Body["running"] != false {
t.Fatalf("before it ran: %d %v", idle.Code, idle.Body)
}
if r := do(t, a.h, "POST", "/api/v1/library/duplicates/check?refresh=maybe", a.token, nil); r.Code != 400 || errCode(r) != "invalid_input" {
t.Errorf("a bad refresh: %d %v", r.Code, r.Body)
}
if r := do(t, a.h, "POST", "/api/v1/library/duplicates/check", a.token, nil); r.Code != http.StatusAccepted || r.Body["status"] != "started" {
t.Fatalf("start: %d %v", r.Code, r.Body)
}
var st resp
for i := 0; ; i++ {
st = a.get("/api/v1/library/duplicates/check")
if st.Body["running"] == false {
break
}
if i > 400 {
t.Fatal("the job did not finish")
}
time.Sleep(25 * time.Millisecond)
}
last, _ := st.Body["last"].(map[string]any)
if last["checked"] != float64(2) || last["hashed"] != float64(2) || last["skipped"] != float64(0) || last["failed"] != float64(0) || last["stopped"] != false {
t.Errorf("last = %v, want 2 checked and 2 hashed", last)
}
if list := a.get("/api/v1/library/duplicates"); list.Body["groups"].([]any)[0].(map[string]any)["identical"] != true {
t.Errorf("the group is not called identical after the check: %v", list.Body)
}
if r := do(t, a.h, "DELETE", "/api/v1/library/duplicates/check", a.token, nil); r.Code != http.StatusNoContent {
t.Errorf("stop: %d", r.Code)
}
}
type alreadyRunning struct{}
func (alreadyRunning) Start(bool) error { return jobs.ErrRunning }
func (alreadyRunning) Stop() bool { return false }
func (alreadyRunning) Status() jobs.Status { return jobs.Status{} }
func TestStartingTheCheckWhileItRunsIsExplained(t *testing.T) {
a := &dupAPI{t: t}
a.h, a.db = newTestEnvDB(t, func(d *Deps, db *bun.DB) {
d.Duplicates = duplicates.NewService(storage.NewDuplicatesRepo(db), duplicates.Options{})
d.DuplicatesJob = alreadyRunning{}
})
a.token = registerAndLogin(t, a.h)
r := do(t, a.h, "POST", "/api/v1/library/duplicates/check", a.token, nil)
if r.Code != http.StatusConflict || errCode(r) != "duplicates_check_running" {
t.Errorf("%d %v, want 409 duplicates_check_running", r.Code, r.Body)
}
}

View file

@ -0,0 +1,169 @@
package httpapi
import (
"errors"
"net/http"
"github.com/gin-gonic/gin"
"codeberg.org/squidd/sonda/internal/duplicates"
"codeberg.org/squidd/sonda/internal/jobs"
)
// duplicateHandlers serve the admin's view of the songs that are in more than one file, and the
// removal of the extra copies. File paths never appear in any answer.
type duplicateHandlers struct{ svc duplicateService }
const (
defaultDuplicatesLimit = 50
maxDuplicatesLimit = 200
)
type duplicateFileJSON struct {
TrackID int64 `json:"track_id"`
Album *albumRefJSON `json:"album"`
Format string `json:"format"`
Bitrate int `json:"bitrate"`
SizeBytes int64 `json:"size_bytes"`
DurationMs int64 `json:"duration_ms"`
Hashed bool `json:"hashed"`
}
type duplicateGroupJSON struct {
Title string `json:"title"`
Artists []artistRefJSON `json:"artists"`
Files []duplicateFileJSON `json:"files"`
Hashed bool `json:"hashed"`
Identical bool `json:"identical"`
SuggestedID int64 `json:"suggested_keep_track_id"`
FreedBytes int64 `json:"freed_bytes"`
}
type duplicatePageJSON struct {
DeletionAllowed bool `json:"deletion_allowed"`
Total int `json:"total"`
Limit int `json:"limit"`
Offset int `json:"offset"`
Groups []duplicateGroupJSON `json:"groups"`
}
// list gives the groups of copies, a page at a time.
func (h *duplicateHandlers) list(c *gin.Context) {
limit, ok := queryInt(c, "limit")
if !ok {
return
}
offset, ok := queryInt(c, "offset")
if !ok {
return
}
if limit < 0 || offset < 0 {
writeError(c, http.StatusBadRequest, "invalid_input", "limit and offset cannot be negative")
return
}
size := clampInt(limit)
if size == 0 {
size = defaultDuplicatesLimit
}
size = min(size, maxDuplicatesLimit)
page, err := h.svc.Groups(c.Request.Context(), size, clampInt(offset))
if err != nil {
_ = c.Error(err)
writeError(c, http.StatusInternalServerError, "internal", "internal error")
return
}
out := duplicatePageJSON{
DeletionAllowed: page.DeletionAllowed, Total: page.Total, Limit: size, Offset: clampInt(offset),
Groups: make([]duplicateGroupJSON, len(page.Groups)),
}
for i, g := range page.Groups {
out.Groups[i] = toDuplicateGroupJSON(g)
}
c.JSON(http.StatusOK, out)
}
func toDuplicateGroupJSON(g duplicates.Group) duplicateGroupJSON {
out := duplicateGroupJSON{
Title: g.Title, Artists: make([]artistRefJSON, len(g.Artists)), Files: make([]duplicateFileJSON, len(g.Files)),
Hashed: g.Hashed, Identical: g.Identical, SuggestedID: g.KeepTrackID, FreedBytes: g.FreedBytes,
}
for i, a := range g.Artists {
out.Artists[i] = artistRefJSON{ID: a.ID, Name: a.Name}
}
for i, f := range g.Files {
file := duplicateFileJSON{
TrackID: f.TrackID, Format: f.Format, Bitrate: f.BitRate, SizeBytes: f.SizeBytes, DurationMs: f.DurationMs, Hashed: f.Hash != "",
}
if f.AlbumID != 0 {
file.Album = &albumRefJSON{ID: f.AlbumID, Title: f.Album}
}
out.Files[i] = file
}
return out
}
type cleanRequest struct {
KeepTrackID int64 `json:"keep_track_id"`
RemoveTrackIDs []int64 `json:"remove_track_ids"`
}
// clean removes the copies named of one song, keeping the one named.
func (h *duplicateHandlers) clean(c *gin.Context) {
var req cleanRequest
if !bindJSON(c, &req) {
return
}
if req.KeepTrackID <= 0 || len(req.RemoveTrackIDs) == 0 {
writeError(c, http.StatusBadRequest, "invalid_input", "name the copy to keep and at least one copy to remove")
return
}
for _, id := range req.RemoveTrackIDs {
if id <= 0 {
writeError(c, http.StatusBadRequest, "invalid_input", "track ids must be positive numbers")
return
}
}
res, err := h.svc.Clean(c.Request.Context(), req.KeepTrackID, req.RemoveTrackIDs)
if err != nil {
writeDuplicatesError(c, err)
return
}
c.JSON(http.StatusOK, gin.H{"removed": res.Removed, "freed_bytes": res.FreedBytes})
}
// writeDuplicatesError gives each way a removal is refused its own status and code, so a client can tell them apart.
func writeDuplicatesError(c *gin.Context, err error) {
switch {
case errors.Is(err, duplicates.ErrDisabled):
writeError(c, http.StatusConflict, "deletion_disabled", "deleting files is not turned on: set library.allow_delete")
case errors.Is(err, duplicates.ErrScanRunning):
writeError(c, http.StatusConflict, "scan_running", "wait for the running scan to finish")
case errors.Is(err, duplicates.ErrInvalid):
writeError(c, http.StatusBadRequest, "invalid_input", err.Error())
case errors.Is(err, duplicates.ErrTrackNotFound):
writeError(c, http.StatusNotFound, "track_not_found", "no such track")
case errors.Is(err, duplicates.ErrChanged):
writeError(c, http.StatusConflict, "duplicate_changed", "a file is not what the last scan saw: scan again")
case errors.Is(err, duplicates.ErrReadOnly):
writeError(c, http.StatusConflict, "library_read_only", "the music folder cannot be written to")
default:
_ = c.Error(err)
writeError(c, http.StatusInternalServerError, "internal", "internal error")
}
}
type duplicatesCheckSummaryJSON struct {
Checked int `json:"checked"`
Hashed int `json:"hashed"`
Skipped int `json:"skipped"`
Failed int `json:"failed"`
Stopped bool `json:"stopped"`
}
// duplicatesCheckSummary is the summary of the job that fingerprints the files of the groups.
func duplicatesCheckSummary(s *jobs.Summary) any {
return duplicatesCheckSummaryJSON{
Checked: s.Checked, Hashed: s.Outcomes[duplicates.OutcomeHashed], Skipped: s.Skipped, Failed: s.Failed, Stopped: s.Stopped,
}
}

View file

@ -18,6 +18,7 @@ import (
"codeberg.org/squidd/sonda/internal/catalog"
"codeberg.org/squidd/sonda/internal/discover"
"codeberg.org/squidd/sonda/internal/discovery"
"codeberg.org/squidd/sonda/internal/duplicates"
"codeberg.org/squidd/sonda/internal/feedback"
"codeberg.org/squidd/sonda/internal/images"
"codeberg.org/squidd/sonda/internal/jobs"
@ -119,6 +120,11 @@ type jobService interface {
Status() jobs.Status
}
type duplicateService interface {
Groups(ctx context.Context, limit, offset int) (duplicates.Page, error)
Clean(ctx context.Context, keep int64, remove []int64) (duplicates.Result, error)
}
type coverService interface {
AlbumCover(ctx context.Context, albumID int64, size images.Size) (*images.Cover, error)
}
@ -170,6 +176,8 @@ type Deps struct {
CoversJob jobService
ArtistImagesJob jobService
DescriptionsJob jobService
Duplicates duplicateService
DuplicatesJob jobService
About aboutService
ArtistImages artistImageService
Settings userSettingsService
@ -299,6 +307,14 @@ func NewRouter(d Deps) *gin.Engine {
admin.DELETE("/library/artist-images", aj.stop)
admin.POST("/library/descriptions", dj.start)
admin.DELETE("/library/descriptions", dj.stop)
dup := &duplicateHandlers{svc: d.Duplicates}
dupJob := &jobHandlers{job: d.DuplicatesJob, runningCode: "duplicates_check_running", summary: duplicatesCheckSummary}
admin.GET("/library/duplicates", dup.list)
admin.POST("/library/duplicates/clean", dup.clean)
admin.GET("/library/duplicates/check", dupJob.status)
admin.POST("/library/duplicates/check", dupJob.start)
admin.DELETE("/library/duplicates/check", dupJob.stop)
libs := &librariesHandlers{svc: d.Libraries}
admin.GET("/libraries", libs.list)
admin.POST("/libraries", libs.create)

View file

@ -717,6 +717,162 @@ paths:
'401': {$ref: '#/components/responses/Unauthorized'}
'403': {$ref: '#/components/responses/AdminRequired'}
/library/duplicates:
get:
tags: [Scan]
operationId: listDuplicates
summary: The songs that are in more than one file
description: |
Admin only. Groups of files that are the same song by the same artists, worked out from the index and
never from the disk, so it is quick and touches no file. Two files are the same song when they have the
same set of main artists, the same title (the key Sonda matches titles by) and durations no more than
3 seconds apart, every pair in a group within that. A file with no duration or no artist never matches,
a file flagged as missing does not count, and a copy may be on a different album: `files` says which.
Each group says which copy Sonda would keep (`suggested_keep_track_id`): the one without loss (FLAC)
before one with loss, then the higher bitrate, then the bigger file, then the oldest. `hashed` is true when
every file of the group has been read by the check (`POST /library/duplicates/check`) and `identical` when,
besides, they are byte for byte the same. No file path is ever returned.
`deletion_allowed` says whether `POST /library/duplicates/clean` can work: with `library.allow_delete`
off (the default) the groups can still be seen.
security: [{bearerAuth: []}]
parameters:
- {$ref: '#/components/parameters/Limit'}
- {$ref: '#/components/parameters/Offset'}
responses:
'200':
description: A page of groups, in a fixed order (by title).
content:
application/json:
schema: {$ref: '#/components/schemas/DuplicatesPage'}
'400': {$ref: '#/components/responses/InvalidInput'}
'401': {$ref: '#/components/responses/Unauthorized'}
'403': {$ref: '#/components/responses/AdminRequired'}
/library/duplicates/check:
get:
tags: [Scan]
operationId: getDuplicatesCheck
summary: Progress of the check that reads the files of the duplicates
description: |
Whether the check is running, how far it is (`progress`, only while it runs), and how the last one ended.
The check is described at `POST /library/duplicates/check`.
security: [{bearerAuth: []}]
responses:
'200':
description: The job.
content:
application/json:
schema: {$ref: '#/components/schemas/DuplicatesCheckStatus'}
'401': {$ref: '#/components/responses/Unauthorized'}
'403': {$ref: '#/components/responses/AdminRequired'}
post:
tags: [Scan]
operationId: startDuplicatesCheck
summary: Read the files of the duplicates, to tell which are identical
description: |
Admin only. Starts, in the background and one at a time like a scan, a job that reads every file that is in
a group of copies and keeps its SHA-256, so that a group of files that are byte for byte the same can be
called `identical`. It only reads: nothing is removed or sent anywhere. **It reads whole files**, which on a
network drive takes a while, so it is started on purpose. A file that has a fingerprint is skipped, and the
fingerprint of a file is forgotten when a scan finds the file changed; send `refresh=true` to read every file
again. A file that changed since the scan is skipped. It gives up, and says why in `last_error`, when files
cannot be read five times in a row (a disk that is not mounted). `DELETE /library/duplicates/check` stops it.
security: [{bearerAuth: []}]
parameters:
- name: refresh
in: query
description: "`true` reads every file again, also those that have a fingerprint. `false` when absent."
schema: {type: boolean, default: false}
responses:
'202':
description: The check was started in the background.
content:
application/json:
schema:
type: object
additionalProperties: false
required: [status]
properties:
status: {type: string, enum: [started]}
'400': {$ref: '#/components/responses/InvalidInput'}
'401': {$ref: '#/components/responses/Unauthorized'}
'403': {$ref: '#/components/responses/AdminRequired'}
'409':
description: The check is already running (`duplicates_check_running`).
content:
application/json:
schema: {$ref: '#/components/schemas/Error'}
'500':
description: The groups could not be worked out (`internal`).
content:
application/json:
schema: {$ref: '#/components/schemas/Error'}
delete:
tags: [Scan]
operationId: stopDuplicatesCheck
summary: Stop the check of the duplicates
description: |
Admin only. Asks a running check to end after the file it is on; the fingerprints already kept stay. It
answers the same when no check is running.
security: [{bearerAuth: []}]
responses:
'204': {description: "Done, also when no check was running."}
'401': {$ref: '#/components/responses/Unauthorized'}
'403': {$ref: '#/components/responses/AdminRequired'}
/library/duplicates/clean:
post:
tags: [Scan]
operationId: cleanDuplicates
summary: Delete the extra copies of one song
description: |
Admin only. **Deletes files from the music folder.** It is the one thing Sonda does that writes there, so it
is off until the operator turns on `library.allow_delete` (answer `deletion_disabled` otherwise), the folder
has to be writable (`library_read_only`), and it is never done by itself: one request removes the copies
`remove_track_ids` of one song and keeps `keep_track_id`.
The server decides again, whatever the client saw, and removes nothing unless every check passes: no scan is
running (`scan_running`); the tracks exist (`track_not_found`); they are copies of one song by the rule of
`GET /library/duplicates` (`invalid_input` otherwise, and also when the copy to keep is among those to
remove, a copy is named twice or there are none or more than 100); and the copy to keep and every copy to
remove are still the file the last scan saw, a regular file with the same size and date (`duplicate_changed`
for a file that changed, is gone, became a link or is outside the library: scan again). So the last copy of
a song is never removed. Every file is checked before the first is removed.
Each copy is removed from the disk through the library folder (a path cannot leave it, not even through a
link) and its track leaves the index; **the listens of the removed track become listens of the track that
stays**, so the taste is not lost. Folders are never removed, not even empty ones. If it fails part way (for
example the folder turns out to be read-only for the second file), what was removed stays removed.
security: [{bearerAuth: []}]
requestBody:
required: true
content:
application/json:
schema: {$ref: '#/components/schemas/CleanDuplicates'}
responses:
'200':
description: The extra copies were removed.
content:
application/json:
schema: {$ref: '#/components/schemas/DuplicatesCleaned'}
'400': {$ref: '#/components/responses/InvalidInput'}
'401': {$ref: '#/components/responses/Unauthorized'}
'403': {$ref: '#/components/responses/AdminRequired'}
'404':
description: A track that was named does not exist (`track_not_found`).
content:
application/json:
schema: {$ref: '#/components/schemas/Error'}
'409':
description: |
Deleting is not turned on (`deletion_disabled`), a scan is running (`scan_running`), a file is not what the last
scan saw (`duplicate_changed`), or the music folder cannot be written to (`library_read_only`).
content:
application/json:
schema: {$ref: '#/components/schemas/Error'}
/library/scan:
get:
tags: [Scan]
@ -3100,6 +3256,108 @@ components:
stopped: {type: boolean, description: "It was stopped, or the application was."}
last_error: {type: string, description: "Why the last job ended before the end, for example that a provider is off or kept failing."}
DuplicatesPage:
type: object
additionalProperties: false
required: [deletion_allowed, total, limit, offset, groups]
properties:
deletion_allowed: {type: boolean, description: "Whether `POST /library/duplicates/clean` can work (`library.allow_delete`)."}
total: {type: integer, description: "Groups in all, not in this page."}
limit: {type: integer}
offset: {type: integer}
groups:
type: array
items: {$ref: '#/components/schemas/DuplicateGroup'}
DuplicateGroup:
type: object
additionalProperties: false
required: [title, artists, files, hashed, identical, suggested_keep_track_id, freed_bytes]
properties:
title: {type: string}
artists:
type: array
description: The main artists the copies share.
items: {$ref: '#/components/schemas/ArtistRef'}
files:
type: array
description: The copies, by track id; at least two.
items: {$ref: '#/components/schemas/DuplicateFile'}
hashed: {type: boolean, description: "Every file of the group has been read by the check."}
identical: {type: boolean, description: "Hashed, and byte for byte the same."}
suggested_keep_track_id: {type: integer, format: int64, description: "The copy Sonda would keep: without loss first, then the higher bitrate, then the bigger file, then the oldest."}
freed_bytes: {type: integer, format: int64, description: "The size of every copy but the one to keep."}
DuplicateFile:
type: object
additionalProperties: false
required: [track_id, album, format, bitrate, size_bytes, duration_ms, hashed]
properties:
track_id: {type: integer, format: int64}
album:
type: object
nullable: true
additionalProperties: false
required: [id, title]
properties:
id: {type: integer, format: int64}
title: {type: string}
format: {type: string, description: "`mp3`, `flac`..."}
bitrate: {type: integer, description: "0 when unknown."}
size_bytes: {type: integer, format: int64}
duration_ms: {type: integer, format: int64}
hashed: {type: boolean, description: "This file has been read by the check."}
CleanDuplicates:
type: object
additionalProperties: false
required: [keep_track_id, remove_track_ids]
properties:
keep_track_id: {type: integer, format: int64, minimum: 1, description: "The copy that stays."}
remove_track_ids:
type: array
minItems: 1
maxItems: 100
description: "The copies to delete, of the same song as the one that stays, none twice."
items: {type: integer, format: int64, minimum: 1}
DuplicatesCleaned:
type: object
additionalProperties: false
required: [removed, freed_bytes]
properties:
removed: {type: integer, description: "Files deleted."}
freed_bytes: {type: integer, format: int64}
DuplicatesCheckStatus:
type: object
additionalProperties: false
required: [running]
properties:
running: {type: boolean}
started_at: {type: string, format: date-time, description: "When the last check started."}
last_finished_at: {type: string, format: date-time}
progress:
type: object
description: Only while the check runs.
additionalProperties: false
required: [done, total]
properties:
done: {type: integer}
total: {type: integer, description: "Files the check set out to read."}
last:
type: object
description: How the last finished check ended.
additionalProperties: false
required: [checked, hashed, skipped, failed, stopped]
properties:
checked: {type: integer, description: "Files looked at."}
hashed: {type: integer, description: "Files read and fingerprinted."}
skipped: {type: integer, description: "Files that changed since the scan, or are gone."}
failed: {type: integer, description: "Files that could not be read just then."}
stopped: {type: boolean, description: "It was stopped, or the application was."}
last_error: {type: string, description: "Why the last check ended before the end, for example that the disk is not mounted."}
UserSettings:
type: object
additionalProperties: false

View file

@ -11,6 +11,7 @@ import (
"os"
"path/filepath"
"regexp"
"strconv"
"time"
"github.com/pelletier/go-toml/v2"
@ -41,6 +42,10 @@ type Server struct {
// in the database and are managed through the API.
type Library struct {
ScanWorkers int `toml:"scan_workers"` // parallel tag readers per scan
// AllowDelete lets an admin delete the extra copies of a song from the music folders. It is the
// one thing that makes Sonda write there, so it is off until the operator turns it on (and it
// needs the folders mounted writable).
AllowDelete bool `toml:"allow_delete"`
}
// Database selects and configures the storage backend.
@ -172,6 +177,14 @@ func Load(path string) (Config, error) {
}
applyEnv(&cfg)
// A typo here must not be ignored in silence: it decides whether Sonda may delete files.
if v := os.Getenv("SONDA_ALLOW_DELETE"); v != "" {
allow, err := strconv.ParseBool(v)
if err != nil {
return Config{}, fmt.Errorf("SONDA_ALLOW_DELETE must be true or false, got %q", v)
}
cfg.Library.AllowDelete = allow
}
if cfg.Storage.DataDir == "" {
cfg.Storage.DataDir = defaultDataDir(cfg.Database)
}

View file

@ -292,3 +292,52 @@ func TestLoadRefusesTheOldLastFMEnvironmentVariable(t *testing.T) {
t.Errorf("the error repeats the key: %v", err)
}
}
// Deleting files from the music folders is the one thing Sonda does not do unless it is told to.
func TestDeletingIsOffByDefault(t *testing.T) {
cfg, err := Load("")
if err != nil {
t.Fatal(err)
}
if cfg.Library.AllowDelete {
t.Error("library.allow_delete is on by default")
}
}
func TestAllowDeleteIsReadFromTheFile(t *testing.T) {
cfg, err := Load(writeFile(t, t.TempDir(), "[library]\nallow_delete = true\n"))
if err != nil {
t.Fatal(err)
}
if !cfg.Library.AllowDelete {
t.Error("allow_delete = true in the file was not read")
}
}
func TestAllowDeleteEnvironmentVariableWinsOverTheFile(t *testing.T) {
p := writeFile(t, t.TempDir(), "[library]\nallow_delete = true\n")
t.Setenv("SONDA_ALLOW_DELETE", "false")
cfg, err := Load(p)
if err != nil {
t.Fatal(err)
}
if cfg.Library.AllowDelete {
t.Error("SONDA_ALLOW_DELETE=false did not turn it off")
}
t.Setenv("SONDA_ALLOW_DELETE", "true")
cfg, err = Load("")
if err != nil {
t.Fatal(err)
}
if !cfg.Library.AllowDelete {
t.Error("SONDA_ALLOW_DELETE=true did not turn it on")
}
}
// A typo in a setting that lets Sonda delete files must not be ignored in silence.
func TestAllowDeleteEnvironmentVariableMustBeABoolean(t *testing.T) {
t.Setenv("SONDA_ALLOW_DELETE", "yes please")
if _, err := Load(""); err == nil || !strings.Contains(err.Error(), "SONDA_ALLOW_DELETE") {
t.Errorf("got %v, want an error naming SONDA_ALLOW_DELETE", err)
}
}

View file

@ -0,0 +1,80 @@
package duplicates
import (
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
"syscall"
)
// openRoot opens the folder of a library so that nothing done through it can leave it: a ../ or a
// link that points outside is refused by the system at the moment of the call, with no gap between
// checking a path and using it. It returns ErrChanged when the folder is not there (a disk that is
// not mounted).
func openRoot(root string) (*os.Root, error) {
clean := filepath.Clean(root)
if !filepath.IsAbs(clean) {
return nil, fmt.Errorf("%w: the library folder is not an absolute path", ErrChanged)
}
r, err := os.OpenRoot(clean)
if errors.Is(err, fs.ErrNotExist) {
return nil, fmt.Errorf("%w: the library folder is not there", ErrChanged)
}
if err != nil {
return nil, fmt.Errorf("open the library folder: %w", err)
}
return r, nil
}
// relPath is the path of a file inside its library, in the system's own form. An empty or absolute
// path is not one.
func relPath(rec FileRecord) (string, error) {
rel := filepath.FromSlash(rec.RelPath)
if rel == "" || filepath.IsAbs(rel) {
return "", fmt.Errorf("%w: the path of the file is not inside the library", ErrChanged)
}
return rel, nil
}
// isEscape says whether an error from an os.Root is the refusal to leave the root (it has no exported value).
func isEscape(err error) bool {
return err != nil && strings.Contains(err.Error(), "path escapes from parent")
}
// sameAsScanned checks, through the root, that the path is a regular file (not a link, not a
// folder) with the size and date the index has for it. Anything else is ErrChanged.
func sameAsScanned(root *os.Root, rel string, rec FileRecord) error {
info, err := root.Lstat(rel)
switch {
case errors.Is(err, fs.ErrNotExist), isEscape(err):
return fmt.Errorf("%w: the file is not there", ErrChanged)
case err != nil:
return fmt.Errorf("look at the file: %w", err)
}
if !info.Mode().IsRegular() {
return fmt.Errorf("%w: it is not a regular file any more", ErrChanged)
}
if info.Size() != rec.SizeBytes || info.ModTime().Unix() != rec.MTimeUnix {
return fmt.Errorf("%w: its size or date is not what the scan saw", ErrChanged)
}
return nil
}
// removeFile deletes one file inside the root, and turns the ways it can fail into the answers the API gives.
func removeFile(root *os.Root, rel string) error {
err := root.Remove(rel)
switch {
case err == nil:
return nil
case errors.Is(err, fs.ErrNotExist):
return fmt.Errorf("%w: the file is gone", ErrChanged)
case isEscape(err):
return fmt.Errorf("%w: the path leaves the library", ErrChanged)
case errors.Is(err, fs.ErrPermission), errors.Is(err, syscall.EROFS):
return fmt.Errorf("%w: %v", ErrReadOnly, err)
}
return fmt.Errorf("remove the file: %w", err)
}

View file

@ -0,0 +1,175 @@
// Package duplicates finds the same song by the same artist in more than one file, and removes
// the extra copies when an admin says so. Finding only reads the index. Removing is the one thing
// Sonda does to the music folders, so it is off until the operator turns it on, and every
// removal is checked again at the moment it is made.
package duplicates
import (
"slices"
"strconv"
"strings"
"time"
)
// DurationTolerance is how far apart two durations may be for two files to be the same song. It
// keeps a live version that is minutes longer apart from the studio one.
const DurationTolerance = 3 * time.Second
// ArtistRef names an artist.
type ArtistRef struct {
ID int64
Name string
}
// Candidate is one playable file as the grouping sees it: its track, its main artists and the
// facts that tell the copies apart.
type Candidate struct {
TrackID int64
FileID int64
Artists []ArtistRef // the main artists of the track
Title string
TitleKey string // the key Sonda matches titles by
DurationMs int64
AlbumID int64
Album string
Format string
BitRate int
SizeBytes int64
Hash string // SHA-256 of the file, empty until it has been read
}
// Group is a song that is in more than one file.
type Group struct {
Artists []ArtistRef
Title string
Files []Candidate // by track id
// Hashed is true when every file has been read; Identical when, besides, they are byte for byte the same.
Hashed, Identical bool
// KeepTrackID is the copy to keep, by Suggest; FreedBytes is the size of all the others.
KeepTrackID int64
FreedBytes int64
}
// artistsKey identifies a set of artists, whatever the order they are credited in.
func artistsKey(artists []ArtistRef) string {
ids := make([]int64, len(artists))
for i, a := range artists {
ids[i] = a.ID
}
slices.Sort(ids)
parts := make([]string, len(ids))
for i, id := range ids {
parts[i] = strconv.FormatInt(id, 10)
}
return strings.Join(parts, ",")
}
// Build finds the groups. Two files are the same song when they have the same set of main
// artists, the same title and durations no more than tolerance apart (every pair in a group, not
// a chain of near ones). A file with no known duration or no artist never matches. The groups
// and the files in them come in a fixed order, whatever the order of cands.
func Build(cands []Candidate, tolerance time.Duration) []Group {
buckets := map[string][]Candidate{}
for _, c := range cands {
if c.DurationMs <= 0 || len(c.Artists) == 0 {
continue
}
key := artistsKey(c.Artists) + "\x00" + c.TitleKey
buckets[key] = append(buckets[key], c)
}
tolMs := tolerance.Milliseconds()
var groups []Group
for _, bucket := range buckets {
if len(bucket) < 2 {
continue
}
slices.SortFunc(bucket, func(a, b Candidate) int {
if a.DurationMs != b.DurationMs {
return int(a.DurationMs - b.DurationMs)
}
return int(a.TrackID - b.TrackID)
})
for i := 0; i < len(bucket); {
j := i + 1
for j < len(bucket) && bucket[j].DurationMs-bucket[i].DurationMs <= tolMs {
j++
}
if j-i >= 2 {
groups = append(groups, newGroup(bucket[i:j]))
}
i = j
}
}
slices.SortFunc(groups, func(a, b Group) int {
if c := strings.Compare(a.Files[0].TitleKey, b.Files[0].TitleKey); c != 0 {
return c
}
if c := strings.Compare(artistsKey(a.Artists), artistsKey(b.Artists)); c != 0 {
return c
}
return int(a.Files[0].TrackID - b.Files[0].TrackID)
})
return groups
}
func newGroup(files []Candidate) Group {
files = slices.Clone(files)
slices.SortFunc(files, func(a, b Candidate) int { return int(a.TrackID - b.TrackID) })
artists := slices.Clone(files[0].Artists)
slices.SortFunc(artists, func(a, b ArtistRef) int { return int(a.ID - b.ID) })
g := Group{Artists: artists, Title: files[0].Title, Files: files, KeepTrackID: Suggest(files)}
g.Hashed = true
for _, f := range files {
if f.Hash == "" {
g.Hashed = false
}
if f.TrackID != g.KeepTrackID {
g.FreedBytes += f.SizeBytes
}
}
g.Identical = g.Hashed
for _, f := range files[1:] {
if f.Hash != files[0].Hash {
g.Identical = false
}
}
return g
}
// lossless says whether a format keeps everything the recording had.
func lossless(format string) bool {
switch strings.ToLower(format) {
case "flac", "wav", "alac", "ape", "wv", "aiff":
return true
}
return false
}
// Suggest picks the copy to keep: one without loss before one with loss, then the higher bitrate,
// then the bigger file, and at a tie the oldest track. The answer does not depend on the order.
func Suggest(files []Candidate) int64 {
best := files[0]
for _, f := range files[1:] {
if better(f, best) {
best = f
}
}
return best.TrackID
}
// better says whether a should be kept over b.
func better(a, b Candidate) bool {
if la, lb := lossless(a.Format), lossless(b.Format); la != lb {
return la
}
if a.BitRate != b.BitRate {
return a.BitRate > b.BitRate
}
if a.SizeBytes != b.SizeBytes {
return a.SizeBytes > b.SizeBytes
}
return a.TrackID < b.TrackID
}

View file

@ -0,0 +1,198 @@
package duplicates
import (
"reflect"
"testing"
"time"
)
const tol = 3 * time.Second
func cand(track int64, title string, durationMs int64, artists ...int64) Candidate {
refs := make([]ArtistRef, len(artists))
for i, id := range artists {
refs[i] = ArtistRef{ID: id, Name: "Artist"}
}
return Candidate{
TrackID: track, FileID: track * 10, Artists: refs, Title: title, TitleKey: title,
DurationMs: durationMs, AlbumID: 1, Album: "Album", Format: "mp3", BitRate: 192, SizeBytes: 5_000_000,
}
}
func trackIDs(g Group) []int64 {
var out []int64
for _, f := range g.Files {
out = append(out, f.TrackID)
}
return out
}
func TestTheSameSongBySameArtistsIsAGroup(t *testing.T) {
groups := Build([]Candidate{
cand(1, "song", 200_000, 7),
cand(2, "song", 201_000, 7),
}, tol)
if len(groups) != 1 || !reflect.DeepEqual(trackIDs(groups[0]), []int64{1, 2}) {
t.Fatalf("groups = %+v, want one group of tracks 1 and 2", groups)
}
}
func TestASongThatIsAloneIsNotAGroup(t *testing.T) {
if got := Build([]Candidate{cand(1, "song", 200_000, 7)}, tol); len(got) != 0 {
t.Errorf("a single copy made a group: %+v", got)
}
}
func TestSameTitleByAnotherArtistIsAnotherSong(t *testing.T) {
if got := Build([]Candidate{cand(1, "song", 200_000, 7), cand(2, "song", 200_000, 8)}, tol); len(got) != 0 {
t.Errorf("two artists made a group: %+v", got)
}
}
func TestTheSetOfArtistsMustBeTheSame(t *testing.T) {
// A duet is not the song of one of the two.
if got := Build([]Candidate{cand(1, "song", 200_000, 7), cand(2, "song", 200_000, 7, 8)}, tol); len(got) != 0 {
t.Errorf("a solo and a duet made a group: %+v", got)
}
// The order in which the artists are credited does not matter.
got := Build([]Candidate{cand(1, "song", 200_000, 7, 8), cand(2, "song", 200_000, 8, 7)}, tol)
if len(got) != 1 {
t.Errorf("the same artists in another order did not make a group: %+v", got)
}
}
func TestADifferentTitleIsAnotherSong(t *testing.T) {
if got := Build([]Candidate{cand(1, "song", 200_000, 7), cand(2, "song (live)", 200_000, 7)}, tol); len(got) != 0 {
t.Errorf("two titles made a group: %+v", got)
}
}
func TestDurationsMustBeWithinTheTolerance(t *testing.T) {
// 3 seconds apart is the same song, 3.001 is not.
if got := Build([]Candidate{cand(1, "song", 200_000, 7), cand(2, "song", 203_000, 7)}, tol); len(got) != 1 {
t.Errorf("3 s apart: %+v, want a group", got)
}
if got := Build([]Candidate{cand(1, "song", 200_000, 7), cand(2, "song", 203_001, 7)}, tol); len(got) != 0 {
t.Errorf("3.001 s apart: %+v, want none", got)
}
}
func TestALongLiveVersionIsNotMixedWithTheStudioOnes(t *testing.T) {
groups := Build([]Candidate{
cand(1, "song", 220_000, 7),
cand(2, "song", 221_000, 7),
cand(3, "song", 370_000, 7), // the same title, but 2.5 minutes longer
}, tol)
if len(groups) != 1 || !reflect.DeepEqual(trackIDs(groups[0]), []int64{1, 2}) {
t.Fatalf("groups = %+v, want only tracks 1 and 2 together", groups)
}
}
// Every pair in a group is within the tolerance, so a chain of songs each a little longer than the last
// does not become one group.
func TestAChainOfSlightlyLongerCopiesDoesNotGrowOneGroup(t *testing.T) {
groups := Build([]Candidate{
cand(1, "song", 200_000, 7),
cand(2, "song", 202_000, 7),
cand(3, "song", 204_000, 7),
cand(4, "song", 206_000, 7),
}, tol)
for _, g := range groups {
ids := trackIDs(g)
var lo, hi int64 = 1 << 60, 0
for _, f := range g.Files {
lo, hi = min(lo, f.DurationMs), max(hi, f.DurationMs)
}
if hi-lo > 3000 {
t.Errorf("group %v spans %d ms", ids, hi-lo)
}
}
}
func TestATrackWithNoDurationOrNoArtistNeverMatches(t *testing.T) {
noDuration := cand(2, "song", 0, 7)
noArtist := cand(3, "song", 200_000)
if got := Build([]Candidate{cand(1, "song", 200_000, 7), noDuration, noArtist}, tol); len(got) != 0 {
t.Errorf("groups = %+v, want none", got)
}
}
func TestTheSameSongOnDifferentAlbumsIsAGroup(t *testing.T) {
other := cand(2, "song", 200_000, 7)
other.AlbumID, other.Album = 2, "Greatest hits"
groups := Build([]Candidate{cand(1, "song", 200_000, 7), other}, tol)
if len(groups) != 1 || groups[0].Files[0].Album == groups[0].Files[1].Album {
t.Fatalf("groups = %+v, want one group with two albums", groups)
}
}
func TestSeveralGroupsComeInAStableOrder(t *testing.T) {
a := []Candidate{cand(1, "b song", 200_000, 7), cand(2, "b song", 200_000, 7), cand(3, "a song", 100_000, 7), cand(4, "a song", 100_000, 7)}
b := []Candidate{a[3], a[2], a[1], a[0]}
first, second := Build(a, tol), Build(b, tol)
if !reflect.DeepEqual(first, second) {
t.Errorf("the order of the input changes the answer:\n%+v\n%+v", first, second)
}
if first[0].Title != "a song" {
t.Errorf("first group is %q, want the titles in order", first[0].Title)
}
}
func TestAGroupIsIdenticalOnlyWhenEveryFileHasTheSameHash(t *testing.T) {
x, y := cand(1, "song", 200_000, 7), cand(2, "song", 200_000, 7)
if g := Build([]Candidate{x, y}, tol)[0]; g.Hashed || g.Identical {
t.Errorf("no hashes: hashed=%v identical=%v, want both false", g.Hashed, g.Identical)
}
x.Hash = "aa"
if g := Build([]Candidate{x, y}, tol)[0]; g.Hashed || g.Identical {
t.Errorf("one hash of two: hashed=%v identical=%v, want both false", g.Hashed, g.Identical)
}
y.Hash = "aa"
if g := Build([]Candidate{x, y}, tol)[0]; !g.Hashed || !g.Identical {
t.Errorf("same hashes: hashed=%v identical=%v, want both true", g.Hashed, g.Identical)
}
y.Hash = "bb"
if g := Build([]Candidate{x, y}, tol)[0]; !g.Hashed || g.Identical {
t.Errorf("different hashes: hashed=%v identical=%v, want hashed and not identical", g.Hashed, g.Identical)
}
}
func TestFreedBytesAreTheSizeOfEverythingButTheCopyToKeep(t *testing.T) {
big, small := cand(1, "song", 200_000, 7), cand(2, "song", 200_000, 7)
big.Format, big.SizeBytes = "flac", 30_000_000
small.SizeBytes = 5_000_000
g := Build([]Candidate{big, small}, tol)[0]
if g.KeepTrackID != 1 || g.FreedBytes != 5_000_000 {
t.Errorf("keep %d, freed %d; want 1 and 5000000", g.KeepTrackID, g.FreedBytes)
}
}
// --- which copy to keep ---------------------------------------------------------------------------
func TestSuggestPrefersLosslessThenBitrateThenSizeThenTheOldest(t *testing.T) {
flac := Candidate{TrackID: 5, Format: "flac", BitRate: 800, SizeBytes: 20_000_000}
hiMP3 := Candidate{TrackID: 2, Format: "mp3", BitRate: 320, SizeBytes: 9_000_000}
loMP3 := Candidate{TrackID: 1, Format: "mp3", BitRate: 128, SizeBytes: 4_000_000}
if got := Suggest([]Candidate{loMP3, hiMP3, flac}); got != 5 {
t.Errorf("flac among mp3: kept %d, want 5", got)
}
if got := Suggest([]Candidate{loMP3, hiMP3}); got != 2 {
t.Errorf("320 against 128: kept %d, want 2", got)
}
bigger := Candidate{TrackID: 7, Format: "mp3", BitRate: 320, SizeBytes: 9_500_000}
if got := Suggest([]Candidate{hiMP3, bigger}); got != 7 {
t.Errorf("same bitrate, bigger file: kept %d, want 7", got)
}
twin := Candidate{TrackID: 9, Format: "mp3", BitRate: 320, SizeBytes: 9_000_000}
if got := Suggest([]Candidate{twin, hiMP3}); got != 2 {
t.Errorf("all equal: kept %d, want the oldest (2)", got)
}
}
func TestSuggestIsTheSameWhateverTheOrder(t *testing.T) {
a := Candidate{TrackID: 1, Format: "mp3", BitRate: 320, SizeBytes: 9}
b := Candidate{TrackID: 2, Format: "mp3", BitRate: 320, SizeBytes: 9}
if Suggest([]Candidate{a, b}) != Suggest([]Candidate{b, a}) {
t.Error("the answer depends on the order")
}
}

View file

@ -0,0 +1,120 @@
package duplicates
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"os"
"slices"
"sort"
"sync"
"time"
"codeberg.org/squidd/sonda/internal/jobs"
)
// OutcomeHashed is what the check counts for a file it read and fingerprinted.
const OutcomeHashed = "hashed"
// NewHashJob builds the job that reads the files of the groups of copies and keeps their SHA-256, so
// that a group of files that are byte for byte the same can be said to be identical. It is started by
// hand like the other bulk jobs: it reads whole files, which on a NAS takes a while.
func NewHashJob(s *Service) *jobs.Job {
t := &hashTask{s: s}
return jobs.New(jobs.Task{Noun: "files", Count: t.count, List: t.list, Do: t.do}, s.now)
}
// hashTask keeps, between Count and the Lists that follow, the files to read: those of the groups
// that have no fingerprint yet (all of them to do everything again).
type hashTask struct {
s *Service
mu sync.Mutex
ids []int64
}
func (t *hashTask) count(ctx context.Context, refresh bool, _ time.Time) (int, error) {
groups, err := t.s.groups(ctx)
if err != nil {
return 0, err
}
var ids []int64
for _, g := range groups {
for _, f := range g.Files {
if refresh || f.Hash == "" {
ids = append(ids, f.FileID)
}
}
}
slices.Sort(ids)
t.mu.Lock()
t.ids = ids
t.mu.Unlock()
return len(ids), nil
}
func (t *hashTask) list(_ context.Context, after int64, limit int, _ bool, _ time.Time) ([]int64, error) {
t.mu.Lock()
defer t.mu.Unlock()
start := sort.Search(len(t.ids), func(i int) bool { return t.ids[i] > after })
return slices.Clone(t.ids[start:min(start+limit, len(t.ids))]), nil
}
// do reads one file and keeps its fingerprint, if the file is still the one the scan saw.
func (t *hashTask) do(ctx context.Context, fileID int64, _ bool) (string, error) {
rec, err := t.s.store.FileByID(ctx, fileID)
if errors.Is(err, ErrTrackNotFound) || err == nil && rec.Missing {
return "", jobs.ErrSkip
}
if err != nil {
return "", err
}
root, err := openRoot(rec.Root)
if err != nil {
return "", err // a disk that is not mounted: every file fails the same way, and the job gives up
}
defer root.Close()
rel, err := relPath(rec)
if err != nil {
return "", jobs.ErrSkip
}
if err := sameAsScanned(root, rel, rec); errors.Is(err, ErrChanged) {
return "", jobs.ErrSkip
} else if err != nil {
return "", err
}
f, err := root.Open(rel)
if err != nil {
return "", fmt.Errorf("open the file: %w", err)
}
defer f.Close()
h := sha256.New()
if _, err := io.Copy(h, &contextReader{ctx: ctx, r: f}); err != nil {
return "", fmt.Errorf("read the file: %w", err)
}
saved, err := t.s.store.SaveHash(ctx, fileID, rec.SizeBytes, rec.MTimeUnix, hex.EncodeToString(h.Sum(nil)))
if err != nil {
return "", err
}
if !saved {
return "", jobs.ErrSkip // it changed while it was read
}
return OutcomeHashed, nil
}
// contextReader stops reading when the job is stopped, instead of finishing a long file first.
type contextReader struct {
ctx context.Context
r *os.File
}
func (c *contextReader) Read(p []byte) (int, error) {
if err := c.ctx.Err(); err != nil {
return 0, err
}
return c.r.Read(p)
}

View file

@ -0,0 +1,237 @@
package duplicates
import (
"context"
"fmt"
"os"
"slices"
"time"
"go.uber.org/zap"
)
// Store is what the service reads and writes in the index. internal/storage implements it.
type Store interface {
Candidates(ctx context.Context) ([]Candidate, error)
FileOf(ctx context.Context, trackID int64) (FileRecord, error)
FileByID(ctx context.Context, fileID int64) (FileRecord, error)
SaveHash(ctx context.Context, fileID, sizeBytes, mtimeUnix int64, hash string) (bool, error)
RemoveCopy(ctx context.Context, keepTrackID, removeTrackID int64) error
}
// Options say what the service may do.
type Options struct {
// AllowDelete is library.allow_delete. Without it nothing is ever removed.
AllowDelete bool
// ScanRunning says whether a scan is running; a removal waits for it to end. Nil means never.
ScanRunning func() bool
// Logger records each removal, by track and without the path: audio paths are private.
Logger *zap.Logger
}
// Service finds duplicate songs and removes the extra copies.
type Service struct {
store Store
allow bool
scanning func() bool
log *zap.Logger
now func() time.Time
}
// NewService builds a Service.
func NewService(store Store, o Options) *Service {
log := o.Logger
if log == nil {
log = zap.NewNop()
}
return &Service{store: store, allow: o.AllowDelete, scanning: o.ScanRunning, log: log, now: time.Now}
}
// Page is some of the groups, with how many there are in all.
type Page struct {
Groups []Group
Total int
// DeletionAllowed says whether Clean can work: with it off, the groups can still be seen.
DeletionAllowed bool
}
// Groups returns the groups of copies in the library, worked out from the index and never from the disk.
func (s *Service) Groups(ctx context.Context, limit, offset int) (Page, error) {
all, err := s.groups(ctx)
if err != nil {
return Page{}, err
}
page := Page{Total: len(all), DeletionAllowed: s.allow}
if offset < len(all) {
page.Groups = all[offset:min(offset+limit, len(all))]
}
return page, nil
}
func (s *Service) groups(ctx context.Context) ([]Group, error) {
candidates, err := s.store.Candidates(ctx)
if err != nil {
return nil, err
}
return Build(candidates, DurationTolerance), nil
}
// maxRemovals is the most copies of one song that one request may remove.
const maxRemovals = 100
// Result says what a removal did.
type Result struct {
Removed int
FreedBytes int64
}
// Clean removes the copies `remove` of a song from the disk and the index, keeping `keep`. Nothing
// is removed unless every check passes, and the checks are made here, again, whatever the client
// saw: deleting is allowed, no scan is running, the tracks are copies of one song by the rule of
// Build, the copy to keep is still the file that was scanned (so it is never the last copy that is
// removed), and so is every copy to remove (a file that changed, a link, or a path that leaves the
// library is refused). The files are checked all together before the first is removed. Each removed
// file takes its track out of the index, and its listens and playlist items go to the track that stays.
//
// On an error part way (the folder turned out to be read-only on the second file), the Result says
// what was already removed.
func (s *Service) Clean(ctx context.Context, keep int64, remove []int64) (Result, error) {
var res Result
if !s.allow {
return res, ErrDisabled
}
if s.scanning != nil && s.scanning() {
return res, ErrScanRunning
}
if err := checkRequest(keep, remove); err != nil {
return res, err
}
keepFile, err := s.store.FileOf(ctx, keep)
if err != nil {
return res, err
}
files := make(map[int64]FileRecord, len(remove))
for _, id := range remove {
rec, err := s.store.FileOf(ctx, id)
if err != nil {
return res, err
}
files[id] = rec
}
if keepFile.Missing {
return res, fmt.Errorf("%w: the copy to keep is flagged as missing", ErrChanged)
}
for _, rec := range files {
if rec.Missing {
return res, fmt.Errorf("%w: a copy to remove is flagged as missing", ErrChanged)
}
}
if err := s.checkSameSong(ctx, keep, remove); err != nil {
return res, err
}
// Look at every file before touching any.
roots := map[string]*os.Root{}
defer func() {
for _, r := range roots {
_ = r.Close()
}
}()
rootOf := func(path string) (*os.Root, error) {
if r, ok := roots[path]; ok {
return r, nil
}
r, err := openRoot(path)
if err != nil {
return nil, err
}
roots[path] = r
return r, nil
}
inspect := func(rec FileRecord) (*os.Root, string, error) {
root, err := rootOf(rec.Root)
if err != nil {
return nil, "", err
}
rel, err := relPath(rec)
if err != nil {
return nil, "", err
}
return root, rel, sameAsScanned(root, rel, rec)
}
if _, _, err := inspect(keepFile); err != nil {
return res, err
}
type target struct {
id int64
root *os.Root
rel string
}
targets := make([]target, 0, len(remove))
for _, id := range remove {
root, rel, err := inspect(files[id])
if err != nil {
return res, err
}
targets = append(targets, target{id, root, rel})
}
for _, t := range targets {
if err := removeFile(t.root, t.rel); err != nil {
return res, err
}
// The file is gone, so the index must follow whether or not the client is still there: a request
// that was cancelled now would leave a track whose file does not exist. If the index still cannot
// follow, the next scan flags the path as missing, and nothing is left broken; the error says so.
if err := s.store.RemoveCopy(context.WithoutCancel(ctx), keep, t.id); err != nil {
s.log.Error("a duplicate file was removed but its track is still in the index; a scan will flag it as missing",
zap.Int64("track_id", t.id), zap.Int64("kept_track_id", keep), zap.Error(err))
return res, fmt.Errorf("the file was removed but the index could not be updated: %w", err)
}
res.Removed++
res.FreedBytes += files[t.id].SizeBytes
s.log.Info("a duplicate file was removed", zap.Int64("track_id", t.id), zap.Int64("kept_track_id", keep))
}
return res, nil
}
// checkRequest refuses a request that makes no sense, before anything is read: nothing to remove,
// too much, the copy to keep among those to remove, or a copy named twice.
func checkRequest(keep int64, remove []int64) error {
if len(remove) == 0 || len(remove) > maxRemovals {
return fmt.Errorf("%w: name between 1 and %d copies to remove", ErrInvalid, maxRemovals)
}
seen := map[int64]bool{}
for _, id := range remove {
if id == keep {
return fmt.Errorf("%w: the copy to keep is among those to remove", ErrInvalid)
}
if seen[id] {
return fmt.Errorf("%w: a copy is named twice", ErrInvalid)
}
seen[id] = true
}
return nil
}
// checkSameSong works out the groups again and checks that every copy to remove is in the group of
// the copy to keep.
func (s *Service) checkSameSong(ctx context.Context, keep int64, remove []int64) error {
groups, err := s.groups(ctx)
if err != nil {
return err
}
for _, g := range groups {
if !slices.ContainsFunc(g.Files, func(f Candidate) bool { return f.TrackID == keep }) {
continue
}
for _, id := range remove {
if !slices.ContainsFunc(g.Files, func(f Candidate) bool { return f.TrackID == id }) {
return ErrInvalid
}
}
return nil
}
return ErrInvalid
}

View file

@ -0,0 +1,531 @@
package duplicates_test
import (
"context"
"errors"
"os"
"path/filepath"
"runtime"
"testing"
"time"
"github.com/uptrace/bun"
"go.uber.org/zap"
"codeberg.org/squidd/sonda/internal/duplicates"
"codeberg.org/squidd/sonda/internal/jobs"
"codeberg.org/squidd/sonda/internal/library"
"codeberg.org/squidd/sonda/internal/listening"
"codeberg.org/squidd/sonda/internal/storage"
"codeberg.org/squidd/sonda/migrations"
)
// env is a real library on a real folder, indexed in a real database.
type env struct {
t *testing.T
ctx context.Context
db *bun.DB
dir string
libID int64
lib *storage.LibraryRepo
store *storage.DuplicatesRepo
scanning bool
}
func newEnv(t *testing.T) *env {
t.Helper()
db, err := storage.Open(storage.DriverSQLite, filepath.Join(t.TempDir(), "d.db"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = db.Close() })
fsys, err := migrations.For(storage.DriverSQLite)
if err != nil {
t.Fatal(err)
}
if err := storage.Migrate(context.Background(), db, fsys); err != nil {
t.Fatal(err)
}
dir, err := filepath.EvalSymlinks(t.TempDir())
if err != nil {
t.Fatal(err)
}
e := &env{t: t, ctx: context.Background(), db: db, dir: dir, lib: storage.NewLibraryRepo(db), store: storage.NewDuplicatesRepo(db)}
l, err := e.lib.CreateLibrary(e.ctx, "music", dir)
if err != nil {
t.Fatal(err)
}
e.libID = l.ID
return e
}
func (e *env) service(allow bool) *duplicates.Service {
return duplicates.NewService(e.store, duplicates.Options{AllowDelete: allow, ScanRunning: func() bool { return e.scanning }, Logger: zap.NewNop()})
}
// add writes a file and indexes it exactly as it is on the disk, and returns its track.
func (e *env) add(rel, content, title, album string, durationMs int64, format string, bitrate int, artists ...string) int64 {
e.t.Helper()
path := filepath.Join(e.dir, filepath.FromSlash(rel))
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
e.t.Fatal(err)
}
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
e.t.Fatal(err)
}
return e.index(rel, path, title, album, durationMs, format, bitrate, artists...)
}
// index puts a file that is on the disk into the index with the size and date it has.
func (e *env) index(rel, path, title, album string, durationMs int64, format string, bitrate int, artists ...string) int64 {
e.t.Helper()
info, err := os.Stat(path)
if err != nil {
e.t.Fatal(err)
}
f := library.ParsedFile{
FileInfo: library.FileInfo{RelPath: rel, Size: info.Size(), MTimeUnix: info.ModTime().Unix()},
Metadata: library.Metadata{
Tags: library.Tags{Title: title, Album: album, AlbumArtist: artists[0], Artists: artists, TrackNo: 1, DiscNo: 1},
Properties: library.Properties{Format: format, DurationMs: durationMs, SampleRate: 44100, BitRate: bitrate},
},
}
if _, err := e.lib.SaveFile(e.ctx, e.libID, f, 0); err != nil {
e.t.Fatal(err)
}
var id int64
if err := e.db.QueryRow("SELECT track_id FROM audio_files WHERE rel_path = ?", rel).Scan(&id); err != nil {
e.t.Fatal(err)
}
return id
}
func (e *env) exists(rel string) bool {
_, err := os.Lstat(filepath.Join(e.dir, filepath.FromSlash(rel)))
return err == nil
}
func (e *env) tracks() int {
var n int
if err := e.db.QueryRow("SELECT COUNT(*) FROM tracks").Scan(&n); err != nil {
e.t.Fatal(err)
}
return n
}
// twoCopies is the usual case: the same song twice, an mp3 and a flac, on different albums.
func twoCopies(e *env) (flac, mp3 int64) {
flac = e.add("a/song.flac", "FLAC-CONTENT-OF-THE-SONG", "Song", "Album", 200_000, "flac", 900, "Ann")
mp3 = e.add("b/song.mp3", "mp3 content", "Song", "Greatest hits", 200_500, "mp3", 192, "Ann")
return flac, mp3
}
// --- finding ---------------------------------------------------------------------------------------
func TestGroupsListsTheCopiesWithWhatTheAdminNeedsToChoose(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
e.add("c/other.flac", "something else", "Other", "Album", 100_000, "flac", 900, "Ann")
page, err := e.service(true).Groups(e.ctx, 10, 0)
if err != nil {
t.Fatal(err)
}
if page.Total != 1 || len(page.Groups) != 1 || !page.DeletionAllowed {
t.Fatalf("page = %+v, want one group and deleting allowed", page)
}
g := page.Groups[0]
if g.Title != "Song" || len(g.Files) != 2 || g.KeepTrackID != flac || g.FreedBytes != int64(len("mp3 content")) {
t.Errorf("group = %+v, want the flac kept and the size of the mp3 freed", g)
}
if g.Files[1].TrackID != mp3 || g.Files[1].Album != "Greatest hits" || g.Files[1].Format != "mp3" {
t.Errorf("files = %+v", g.Files)
}
}
func TestGroupsReportsThatDeletingIsOff(t *testing.T) {
e := newEnv(t)
twoCopies(e)
page, err := e.service(false).Groups(e.ctx, 10, 0)
if err != nil {
t.Fatal(err)
}
if page.DeletionAllowed || len(page.Groups) != 1 {
t.Errorf("page = %+v: the groups are visible and deleting is not allowed", page)
}
}
func TestGroupsArePaged(t *testing.T) {
e := newEnv(t)
for _, title := range []string{"A song", "B song", "C song"} {
e.add("x/"+title+"-1.mp3", title+"1", title, "Album", 100_000, "mp3", 192, "Ann")
e.add("y/"+title+"-2.mp3", title+"22", title, "Album", 100_000, "mp3", 192, "Ann")
}
svc := e.service(true)
first, _ := svc.Groups(e.ctx, 2, 0)
second, _ := svc.Groups(e.ctx, 2, 2)
if first.Total != 3 || len(first.Groups) != 2 || len(second.Groups) != 1 {
t.Fatalf("pages of %d and %d of %d, want 2, 1 of 3", len(first.Groups), len(second.Groups), first.Total)
}
if first.Groups[0].Title != "A song" || second.Groups[0].Title != "C song" {
t.Errorf("titles %q then %q, want them in order", first.Groups[0].Title, second.Groups[0].Title)
}
if beyond, _ := svc.Groups(e.ctx, 2, 10); len(beyond.Groups) != 0 || beyond.Total != 3 {
t.Errorf("past the end: %+v", beyond)
}
}
// --- fingerprints ------------------------------------------------------------------------------------
func runJob(t *testing.T, j *jobs.Job, refresh bool) jobs.Status {
t.Helper()
if err := j.Start(context.Background(), jobs.Options{Refresh: refresh}); err != nil {
t.Fatal(err)
}
deadline := time.Now().Add(10 * time.Second)
for time.Now().Before(deadline) {
if st := j.Status(); !st.Running {
return st
}
time.Sleep(10 * time.Millisecond)
}
t.Fatal("the job did not finish")
return jobs.Status{}
}
func TestTheCheckReadsTheFilesOfTheGroupsAndSaysWhichAreIdentical(t *testing.T) {
e := newEnv(t)
e.add("a/1.mp3", "the very same bytes", "Same", "A", 100_000, "mp3", 192, "Ann")
e.add("b/2.mp3", "the very same bytes", "Same", "B", 100_000, "mp3", 192, "Ann")
e.add("a/3.mp3", "one version", "Alike", "A", 100_000, "mp3", 192, "Ann")
e.add("b/4.mp3", "another version!", "Alike", "B", 100_000, "mp3", 192, "Ann")
e.add("c/5.mp3", "alone", "Alone", "A", 100_000, "mp3", 192, "Ann")
svc := e.service(true)
st := runJob(t, duplicates.NewHashJob(svc), false)
if st.Last == nil || st.Last.Checked != 4 || st.Last.Outcomes[duplicates.OutcomeHashed] != 4 || st.LastError != "" {
t.Fatalf("status = %+v, want the 4 files of the groups hashed and the one alone left", st)
}
page, _ := svc.Groups(e.ctx, 10, 0)
byTitle := map[string]duplicates.Group{}
for _, g := range page.Groups {
byTitle[g.Title] = g
}
if g := byTitle["Same"]; !g.Hashed || !g.Identical {
t.Errorf("the same bytes: hashed=%v identical=%v, want both", g.Hashed, g.Identical)
}
if g := byTitle["Alike"]; !g.Hashed || g.Identical {
t.Errorf("different bytes: hashed=%v identical=%v, want hashed and not identical", g.Hashed, g.Identical)
}
}
func TestTheCheckSkipsWhatIsHashedUnlessAskedToDoItAgain(t *testing.T) {
e := newEnv(t)
twoCopies(e)
job := duplicates.NewHashJob(e.service(true))
if st := runJob(t, job, false); st.Last.Checked != 2 {
t.Fatalf("first run checked %d, want 2", st.Last.Checked)
}
if st := runJob(t, job, false); st.Last.Checked != 0 {
t.Errorf("second run checked %d, want 0: everything is hashed", st.Last.Checked)
}
if st := runJob(t, job, true); st.Last.Checked != 2 {
t.Errorf("a run that does everything again checked %d, want 2", st.Last.Checked)
}
}
func TestTheCheckSkipsAFileThatChangedAfterTheScan(t *testing.T) {
e := newEnv(t)
twoCopies(e)
if err := os.WriteFile(filepath.Join(e.dir, "b/song.mp3"), []byte("the file was edited, so it is longer now"), 0o644); err != nil {
t.Fatal(err)
}
st := runJob(t, duplicates.NewHashJob(e.service(true)), false)
if st.Last.Outcomes[duplicates.OutcomeHashed] != 1 || st.Last.Skipped != 1 {
t.Errorf("summary = %+v, want the file that is as scanned hashed and the edited one skipped", st.Last)
}
page, _ := e.service(true).Groups(e.ctx, 10, 0)
if page.Groups[0].Hashed {
t.Error("the group claims every file was read, but one was not")
}
}
func TestTheCheckFailsOnAFileItCannotRead(t *testing.T) {
e := newEnv(t)
twoCopies(e)
if err := os.Remove(filepath.Join(e.dir, "b/song.mp3")); err != nil {
t.Fatal(err)
}
st := runJob(t, duplicates.NewHashJob(e.service(true)), false)
if st.Last.Skipped+st.Last.Failed != 1 || st.Last.Outcomes[duplicates.OutcomeHashed] != 1 {
t.Errorf("summary = %+v, want one file hashed and the missing one not counted as read", st.Last)
}
}
// --- removing -----------------------------------------------------------------------------------------
func TestWithDeletingOffNothingIsTouched(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
_, err := e.service(false).Clean(e.ctx, flac, []int64{mp3})
if !errors.Is(err, duplicates.ErrDisabled) {
t.Fatalf("got %v, want ErrDisabled", err)
}
if !e.exists("b/song.mp3") || e.tracks() != 2 {
t.Error("something was removed")
}
}
func TestNothingIsRemovedWhileAScanRuns(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
e.scanning = true
if _, err := e.service(true).Clean(e.ctx, flac, []int64{mp3}); !errors.Is(err, duplicates.ErrScanRunning) {
t.Fatalf("got %v, want ErrScanRunning", err)
}
if !e.exists("b/song.mp3") {
t.Error("a file was removed during a scan")
}
}
func TestCleaningRemovesTheCopyFromTheDiskAndTheIndexAndKeepsTheOther(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
res, err := e.service(true).Clean(e.ctx, flac, []int64{mp3})
if err != nil {
t.Fatal(err)
}
if res.Removed != 1 || res.FreedBytes != int64(len("mp3 content")) {
t.Errorf("result = %+v, want 1 removed and the size of the mp3 freed", res)
}
if e.exists("b/song.mp3") {
t.Error("the copy is still on the disk")
}
if !e.exists("a/song.flac") {
t.Error("the copy to keep is gone")
}
if e.tracks() != 1 {
t.Errorf("%d tracks, want 1", e.tracks())
}
if page, _ := e.service(true).Groups(e.ctx, 10, 0); page.Total != 0 {
t.Errorf("%d groups are left, want none", page.Total)
}
if _, err := e.store.FileOf(e.ctx, mp3); !errors.Is(err, duplicates.ErrTrackNotFound) {
t.Errorf("the removed track is still known: %v", err)
}
}
func TestSeveralCopiesAreRemovedAtOnce(t *testing.T) {
e := newEnv(t)
keep := e.add("a/1.flac", "best", "Song", "A", 200_000, "flac", 900, "Ann")
b := e.add("b/2.mp3", "two", "Song", "B", 200_000, "mp3", 192, "Ann")
c := e.add("c/3.mp3", "three!", "Song", "C", 200_000, "mp3", 128, "Ann")
res, err := e.service(true).Clean(e.ctx, keep, []int64{b, c})
if err != nil {
t.Fatal(err)
}
if res.Removed != 2 || e.exists("b/2.mp3") || e.exists("c/3.mp3") || !e.exists("a/1.flac") {
t.Errorf("result %+v; b exists=%v c exists=%v keep exists=%v", res, e.exists("b/2.mp3"), e.exists("c/3.mp3"), e.exists("a/1.flac"))
}
}
func TestTheCopyToKeepCannotBeInTheListToRemove(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
for name, remove := range map[string][]int64{"both": {flac, mp3}, "only the one to keep": {flac}, "nothing": {}, "the same twice": {mp3, mp3}} {
if _, err := e.service(true).Clean(e.ctx, flac, remove); !errors.Is(err, duplicates.ErrInvalid) {
t.Errorf("%s: got %v, want ErrInvalid", name, err)
}
}
if !e.exists("a/song.flac") || !e.exists("b/song.mp3") || e.tracks() != 2 {
t.Error("something was removed")
}
}
// What the client sends is not trusted: the server works out again that these are copies of one song.
func TestOnlyCopiesOfTheSameSongCanBeRemovedInFavourOfEachOther(t *testing.T) {
e := newEnv(t)
flac, _ := twoCopies(e)
other := e.add("c/other.flac", "something else", "Other", "Album", 100_000, "flac", 900, "Ann")
otherArtist := e.add("d/song.mp3", "by someone else", "Song", "Album", 200_000, "mp3", 192, "Bob")
live := e.add("e/song-live.mp3", "a long live version", "Song", "Live", 420_000, "mp3", 192, "Ann")
for name, id := range map[string]int64{"another song": other, "another artist": otherArtist, "a much longer version": live} {
if _, err := e.service(true).Clean(e.ctx, flac, []int64{id}); !errors.Is(err, duplicates.ErrInvalid) {
t.Errorf("%s: got %v, want ErrInvalid", name, err)
}
}
for _, rel := range []string{"c/other.flac", "d/song.mp3", "e/song-live.mp3", "a/song.flac", "b/song.mp3"} {
if !e.exists(rel) {
t.Errorf("%s was removed", rel)
}
}
}
func TestATrackThatDoesNotExistIsRefused(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
if _, err := e.service(true).Clean(e.ctx, flac, []int64{9999}); !errors.Is(err, duplicates.ErrTrackNotFound) {
t.Errorf("an unknown copy: %v, want ErrTrackNotFound", err)
}
if _, err := e.service(true).Clean(e.ctx, 9999, []int64{mp3}); !errors.Is(err, duplicates.ErrTrackNotFound) {
t.Errorf("an unknown copy to keep: %v, want ErrTrackNotFound", err)
}
if !e.exists("b/song.mp3") {
t.Error("a file was removed")
}
}
// If the copy to keep is not there any more, the other one is the last, and the last is never removed.
func TestTheLastCopyIsNeverRemoved(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
if err := os.Remove(filepath.Join(e.dir, "a/song.flac")); err != nil {
t.Fatal(err)
}
if _, err := e.service(true).Clean(e.ctx, flac, []int64{mp3}); !errors.Is(err, duplicates.ErrChanged) {
t.Fatalf("got %v, want ErrChanged", err)
}
if !e.exists("b/song.mp3") || e.tracks() != 2 {
t.Error("the only copy left was removed")
}
}
func TestACopyThatChangedAfterTheScanIsNotRemoved(t *testing.T) {
e := newEnv(t)
keep := e.add("a/1.flac", "best", "Song", "A", 200_000, "flac", 900, "Ann")
b := e.add("b/2.mp3", "two", "Song", "B", 200_000, "mp3", 192, "Ann")
c := e.add("c/3.mp3", "three!", "Song", "C", 200_000, "mp3", 128, "Ann")
// c is edited after the scan: it is not the file the admin saw.
if err := os.WriteFile(filepath.Join(e.dir, "c/3.mp3"), []byte("edited and longer than before"), 0o644); err != nil {
t.Fatal(err)
}
if _, err := e.service(true).Clean(e.ctx, keep, []int64{b, c}); !errors.Is(err, duplicates.ErrChanged) {
t.Fatalf("got %v, want ErrChanged", err)
}
// Nothing at all was removed, not even b, which was fine: every file is checked before any is removed.
if !e.exists("b/2.mp3") || !e.exists("c/3.mp3") || e.tracks() != 3 {
t.Error("something was removed although a file in the request had changed")
}
}
func TestACopyThatBecameALinkIsNotFollowedOrRemoved(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
target := filepath.Join(e.dir, "precious.txt")
if err := os.WriteFile(target, []byte("do not touch"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.Remove(filepath.Join(e.dir, "b/song.mp3")); err != nil {
t.Fatal(err)
}
if err := os.Symlink(target, filepath.Join(e.dir, "b/song.mp3")); err != nil {
t.Skipf("no symlinks here: %v", err)
}
if _, err := e.service(true).Clean(e.ctx, flac, []int64{mp3}); !errors.Is(err, duplicates.ErrChanged) {
t.Fatalf("got %v, want ErrChanged", err)
}
if !e.exists("precious.txt") || !e.exists("b/song.mp3") {
t.Error("the link or what it points at was removed")
}
}
// A path that was recorded as leaving the library is never followed out of it.
func TestAPathThatLeavesTheLibraryIsNeverRemoved(t *testing.T) {
e := newEnv(t)
flac, _ := twoCopies(e)
outside := filepath.Join(filepath.Dir(e.dir), "outside-"+filepath.Base(e.dir))
if err := os.MkdirAll(outside, 0o755); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = os.RemoveAll(outside) })
victim := filepath.Join(outside, "song.mp3")
if err := os.WriteFile(victim, []byte("not in the library"), 0o644); err != nil {
t.Fatal(err)
}
rel := "../" + filepath.Base(outside) + "/song.mp3"
escaped := e.index(rel, victim, "Song", "Escaped", 200_000, "mp3", 192, "Ann")
if _, err := e.service(true).Clean(e.ctx, flac, []int64{escaped}); err == nil {
t.Fatal("a file outside the library was removed without an error")
}
if _, err := os.Stat(victim); err != nil {
t.Errorf("the file outside the library was removed: %v", err)
}
}
func TestAReadOnlyFolderGivesAClearErrorAndRemovesNothing(t *testing.T) {
if runtime.GOOS == "windows" || os.Geteuid() == 0 {
t.Skip("permissions are not enforced here")
}
e := newEnv(t)
flac, mp3 := twoCopies(e)
dir := filepath.Join(e.dir, "b")
if err := os.Chmod(dir, 0o555); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = os.Chmod(dir, 0o755) })
if _, err := e.service(true).Clean(e.ctx, flac, []int64{mp3}); !errors.Is(err, duplicates.ErrReadOnly) {
t.Fatalf("got %v, want ErrReadOnly", err)
}
if !e.exists("b/song.mp3") || e.tracks() != 2 {
t.Error("something was removed")
}
}
// Listening history must survive: it moves to the copy that stays.
func TestTheListensOfARemovedCopyMoveToTheOneThatStays(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
user, err := storage.NewUserRepo(e.db).CreateUser(e.ctx, "a@example.org", "h", false)
if err != nil {
t.Fatal(err)
}
plays := storage.NewListeningRepo(e.db)
for i, id := range []int64{mp3, mp3, flac} {
ev := listening.Event{UserID: user.ID, TrackID: id, Type: listening.Complete, DurationPlayedMs: 200_000, IdempotencyKey: string(rune('a' + i))}
if _, err := plays.Record(e.ctx, ev); err != nil {
t.Fatal(err)
}
}
if _, err := e.service(true).Clean(e.ctx, flac, []int64{mp3}); err != nil {
t.Fatal(err)
}
var n int
if err := e.db.QueryRow("SELECT COUNT(*) FROM listening_events WHERE track_id = ?", flac).Scan(&n); err != nil || n != 3 {
t.Errorf("the copy that stays has %d listens, want 3 (err=%v)", n, err)
}
}
// cancelsBeforeUpdating ends the request at the worst moment: after the file was deleted, when the
// index is about to follow. The context the service hands to the index is the one that is cancelled,
// unless the service has made sure that what it began is finished.
type cancelsBeforeUpdating struct {
duplicates.Store
cancel context.CancelFunc
}
func (s cancelsBeforeUpdating) RemoveCopy(ctx context.Context, keep, remove int64) error {
s.cancel()
return s.Store.RemoveCopy(ctx, keep, remove)
}
// A client that disconnects once the file is deleted must not leave the index behind: the file is gone,
// so the track has to go too, whether or not anyone is still waiting for the answer.
func TestTheIndexIsUpdatedEvenIfTheClientLeavesAfterTheFileIsDeleted(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
ctx, cancel := context.WithCancel(e.ctx)
defer cancel()
svc := duplicates.NewService(cancelsBeforeUpdating{Store: e.store, cancel: cancel},
duplicates.Options{AllowDelete: true, Logger: zap.NewNop()})
if _, err := svc.Clean(ctx, flac, []int64{mp3}); err != nil {
t.Fatalf("got %v: the request was cancelled, but the work had begun and has to be finished", err)
}
if e.exists("b/song.mp3") {
t.Error("the copy is still on the disk")
}
if e.tracks() != 1 {
t.Errorf("%d tracks, want 1: the file is gone but its track is still in the index", e.tracks())
}
}

View file

@ -0,0 +1,32 @@
package duplicates
import "errors"
// Why a removal is refused. The API turns each into its own answer.
var (
// ErrTrackNotFound: a track that was named does not exist (any more).
ErrTrackNotFound = errors.New("no such track")
// ErrDisabled: the operator has not allowed Sonda to delete files (library.allow_delete).
ErrDisabled = errors.New("deleting files is not allowed: turn on library.allow_delete")
// ErrScanRunning: a scan is running, and what it reads and what is removed must not meet.
ErrScanRunning = errors.New("a scan is running")
// ErrInvalid: the tracks named are not copies of one song, or the request makes no sense.
ErrInvalid = errors.New("these tracks are not copies of the same song")
// ErrChanged: a file is not what the index says (another size or date, a link, gone), or the
// copy to keep is not there any more. Nothing is removed; a new scan brings the index up to date.
ErrChanged = errors.New("a file changed since it was scanned")
// ErrReadOnly: the folder cannot be written to.
ErrReadOnly = errors.New("the music folder is read-only")
)
// FileRecord is a file as the index knows it: where it is and what it looked like when it was scanned.
type FileRecord struct {
FileID int64
TrackID int64
Root string // the path of its library
RelPath string // relative to the library
SizeBytes int64
MTimeUnix int64
Hash string // SHA-256, empty until read
Missing bool // flagged as not found by a scan
}

View file

@ -0,0 +1,213 @@
package storage
import (
"context"
"database/sql"
"errors"
"fmt"
"github.com/uptrace/bun"
"codeberg.org/squidd/sonda/internal/duplicates"
)
// DuplicatesRepo reads the files that may be copies of one song, keeps their fingerprints, and
// removes a copy from the index. It implements duplicates.Store.
type DuplicatesRepo struct{ db *bun.DB }
// NewDuplicatesRepo builds a DuplicatesRepo.
func NewDuplicatesRepo(db *bun.DB) *DuplicatesRepo { return &DuplicatesRepo{db: db} }
type candidateRow struct {
TrackID int64 `bun:"track_id"`
FileID int64 `bun:"file_id"`
Title string `bun:"title"`
TitleKey string `bun:"title_key"`
DurationMs int64 `bun:"duration_ms"`
AlbumID int64 `bun:"album_id"`
Album string `bun:"album"`
Format string `bun:"format"`
BitRate int `bun:"bitrate"`
SizeBytes int64 `bun:"size_bytes"`
Hash string `bun:"hash"`
}
type creditRow struct {
TrackID int64 `bun:"track_id"`
ArtistID int64 `bun:"artist_id"`
Name string `bun:"name"`
}
// idsPerQuery keeps a list of ids well under what a database accepts in one statement.
const idsPerQuery = 500
// Candidates returns the playable files whose title is the title of another playable file: the only
// ones that can have a copy. A track that has no file that is there, or more than one, is left out.
// What makes two of them the same song (the artists, the durations) is decided by duplicates.Build.
func (r *DuplicatesRepo) Candidates(ctx context.Context) ([]duplicates.Candidate, error) {
var rows []candidateRow
err := r.db.NewRaw(`
SELECT t.id AS track_id, f.id AS file_id, t.title AS title, t.title_key AS title_key,
COALESCE(t.duration_ms, 0) AS duration_ms, COALESCE(t.album_id, 0) AS album_id,
COALESCE(al.title, '') AS album, f.format AS format, COALESCE(f.bitrate, 0) AS bitrate,
f.size_bytes AS size_bytes, COALESCE(f.content_hash, '') AS hash
FROM tracks t
JOIN audio_files f ON f.track_id = t.id AND f.missing_at IS NULL
LEFT JOIN albums al ON al.id = t.album_id
WHERE t.id IN (SELECT track_id FROM audio_files WHERE missing_at IS NULL GROUP BY track_id HAVING COUNT(*) = 1)
AND t.title_key IN (
SELECT t2.title_key FROM tracks t2
JOIN audio_files f2 ON f2.track_id = t2.id AND f2.missing_at IS NULL
GROUP BY t2.title_key HAVING COUNT(*) > 1)
ORDER BY t.id`).Scan(ctx, &rows)
if err != nil {
return nil, fmt.Errorf("select the candidates: %w", err)
}
if len(rows) == 0 {
return nil, nil
}
ids := make([]int64, len(rows))
for i, row := range rows {
ids[i] = row.TrackID
}
artists, err := r.mainArtists(ctx, ids)
if err != nil {
return nil, err
}
out := make([]duplicates.Candidate, len(rows))
for i, row := range rows {
out[i] = duplicates.Candidate{
TrackID: row.TrackID, FileID: row.FileID, Artists: artists[row.TrackID], Title: row.Title, TitleKey: row.TitleKey,
DurationMs: row.DurationMs, AlbumID: row.AlbumID, Album: row.Album, Format: row.Format,
BitRate: row.BitRate, SizeBytes: row.SizeBytes, Hash: row.Hash,
}
}
return out, nil
}
// mainArtists reads the main artists of some tracks.
func (r *DuplicatesRepo) mainArtists(ctx context.Context, trackIDs []int64) (map[int64][]duplicates.ArtistRef, error) {
out := map[int64][]duplicates.ArtistRef{}
for start := 0; start < len(trackIDs); start += idsPerQuery {
chunk := trackIDs[start:min(start+idsPerQuery, len(trackIDs))]
var rows []creditRow
err := r.db.NewRaw(`
SELECT ta.track_id AS track_id, a.id AS artist_id, a.name AS name
FROM track_artists ta JOIN artists a ON a.id = ta.artist_id
WHERE ta.role = 'main' AND ta.track_id IN (?)
ORDER BY ta.track_id, a.id`, bun.In(chunk)).Scan(ctx, &rows)
if err != nil {
return nil, fmt.Errorf("select the artists of the candidates: %w", err)
}
for _, row := range rows {
out[row.TrackID] = append(out[row.TrackID], duplicates.ArtistRef{ID: row.ArtistID, Name: row.Name})
}
}
return out, nil
}
type fileRow struct {
FileID int64 `bun:"file_id"`
TrackID int64 `bun:"track_id"`
Root string `bun:"root"`
RelPath string `bun:"rel_path"`
SizeBytes int64 `bun:"size_bytes"`
MTimeUnix int64 `bun:"mtime_unix"`
Hash string `bun:"hash"`
Missing bool `bun:"missing"`
}
func (row fileRow) record() duplicates.FileRecord {
return duplicates.FileRecord{
FileID: row.FileID, TrackID: row.TrackID, Root: row.Root, RelPath: row.RelPath,
SizeBytes: row.SizeBytes, MTimeUnix: row.MTimeUnix, Hash: row.Hash, Missing: row.Missing,
}
}
const fileSelect = `
SELECT f.id AS file_id, f.track_id AS track_id, l.path AS root, f.rel_path AS rel_path,
f.size_bytes AS size_bytes, f.mtime_unix AS mtime_unix, COALESCE(f.content_hash, '') AS hash,
(f.missing_at IS NOT NULL) AS missing
FROM audio_files f JOIN libraries l ON l.id = f.library_id`
// FileOf returns the file of a track: the oldest one that is there, else the oldest one. It returns
// duplicates.ErrTrackNotFound when the track has no file.
func (r *DuplicatesRepo) FileOf(ctx context.Context, trackID int64) (duplicates.FileRecord, error) {
var row fileRow
err := r.db.NewRaw(fileSelect+` WHERE f.track_id = ? ORDER BY (f.missing_at IS NOT NULL), f.id LIMIT 1`, trackID).Scan(ctx, &row)
if errors.Is(err, sql.ErrNoRows) {
return duplicates.FileRecord{}, duplicates.ErrTrackNotFound
}
if err != nil {
return duplicates.FileRecord{}, fmt.Errorf("select the file of a track: %w", err)
}
return row.record(), nil
}
// FileByID returns one file, or duplicates.ErrTrackNotFound.
func (r *DuplicatesRepo) FileByID(ctx context.Context, fileID int64) (duplicates.FileRecord, error) {
var row fileRow
err := r.db.NewRaw(fileSelect+` WHERE f.id = ?`, fileID).Scan(ctx, &row)
if errors.Is(err, sql.ErrNoRows) {
return duplicates.FileRecord{}, duplicates.ErrTrackNotFound
}
if err != nil {
return duplicates.FileRecord{}, fmt.Errorf("select a file: %w", err)
}
return row.record(), nil
}
// SaveHash keeps the fingerprint of a file, but only if the file is still the one that was read: the
// size and date are what they were. It says whether it was kept.
func (r *DuplicatesRepo) SaveHash(ctx context.Context, fileID, sizeBytes, mtimeUnix int64, hash string) (bool, error) {
res, err := r.db.NewUpdate().Model((*audioFileModel)(nil)).
Set("content_hash = ?", hash).
Where("id = ?", fileID).Where("size_bytes = ?", sizeBytes).Where("mtime_unix = ?", mtimeUnix).
Exec(ctx)
if err != nil {
return false, fmt.Errorf("save the hash: %w", err)
}
n, err := res.RowsAffected()
if err != nil {
return false, fmt.Errorf("save the hash: %w", err)
}
return n > 0, nil
}
// RemoveCopy takes a track out of the index in favour of another that stays: the listens and the
// playlist items of the removed track move to the one that stays (they would be deleted with it
// otherwise), and the track goes, with its file, credits and genres. The covers of the albums
// involved are built again. It is one transaction.
func (r *DuplicatesRepo) RemoveCopy(ctx context.Context, keepTrackID, removeTrackID int64) error {
if keepTrackID == removeTrackID {
return duplicates.ErrInvalid
}
return r.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
albums := map[int64]int64{}
for _, id := range []int64{keepTrackID, removeTrackID} {
var album sql.NullInt64
err := tx.QueryRowContext(ctx, "SELECT album_id FROM tracks WHERE id = ?", id).Scan(&album)
if errors.Is(err, sql.ErrNoRows) {
return duplicates.ErrTrackNotFound
}
if err != nil {
return fmt.Errorf("read a track: %w", err)
}
albums[id] = album.Int64
}
if _, err := tx.ExecContext(ctx, "UPDATE listening_events SET track_id = ? WHERE track_id = ?", keepTrackID, removeTrackID); err != nil {
return fmt.Errorf("move the listens: %w", err)
}
// Playlist items go the same way: an item has its own id and position, so the song may appear
// twice in a playlist and nothing shifts.
if _, err := tx.ExecContext(ctx, "UPDATE playlist_items SET track_id = ? WHERE track_id = ?", keepTrackID, removeTrackID); err != nil {
return fmt.Errorf("move the playlist items: %w", err)
}
if _, err := tx.ExecContext(ctx, "DELETE FROM tracks WHERE id = ?", removeTrackID); err != nil {
return fmt.Errorf("remove the track: %w", err)
}
return clearAlbumCovers(ctx, tx, albums[keepTrackID], albums[removeTrackID])
})
}

View file

@ -0,0 +1,312 @@
package storage
import (
"context"
"errors"
"slices"
"testing"
"time"
"github.com/uptrace/bun"
"codeberg.org/squidd/sonda/internal/duplicates"
"codeberg.org/squidd/sonda/internal/library"
"codeberg.org/squidd/sonda/internal/listening"
)
// song is a scanned file: a title on an album by artists, with the facts that tell copies apart.
func song(rel, title, album string, durationMs, size int64, format string, bitrate int, artists ...string) library.ParsedFile {
p := parsed(rel, title, album, artists...)
p.DurationMs, p.Size, p.Format, p.BitRate = durationMs, size, format, bitrate
return p
}
func saved(t *testing.T, repo *LibraryRepo, lib int64, f library.ParsedFile) {
t.Helper()
if _, err := repo.SaveFile(context.Background(), lib, f, 0); err != nil {
t.Fatal(err)
}
}
func trackOf(t *testing.T, db *bun.DB, rel string) int64 {
t.Helper()
var id int64
if err := db.QueryRow("SELECT track_id FROM audio_files WHERE rel_path = ?", rel).Scan(&id); err != nil {
t.Fatal(err)
}
return id
}
func candidateByRel(t *testing.T, db *bun.DB, cs []duplicates.Candidate, rel string) (duplicates.Candidate, bool) {
t.Helper()
id := trackOf(t, db, rel)
for _, c := range cs {
if c.TrackID == id {
return c, true
}
}
return duplicates.Candidate{}, false
}
func TestDuplicateCandidatesAreTheFilesWhoseTitleIsSharedWithAnotherFile(t *testing.T) {
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
ctx := context.Background()
lib := NewLibraryRepo(db)
l := mkLib(t, lib, "music")
saved(t, lib, l, song("a/1.flac", "Song", "Album X", 200_000, 30_000_000, "flac", 900, "Ann"))
saved(t, lib, l, song("b/2.mp3", "Song", "Album Y", 201_000, 5_000_000, "mp3", 192, "Ann"))
saved(t, lib, l, song("c/3.flac", "Song", "Album Z", 200_000, 30_000_000, "flac", 900, "Bob"))
saved(t, lib, l, song("d/4.flac", "Song", "Album W", 200_000, 30_000_000, "flac", 900, "Ann", "Bob"))
saved(t, lib, l, song("e/5.flac", "Other", "Album X", 200_000, 30_000_000, "flac", 900, "Ann"))
cs, err := NewDuplicatesRepo(db).Candidates(ctx)
if err != nil {
t.Fatal(err)
}
if _, ok := candidateByRel(t, db, cs, "e/5.flac"); ok {
t.Error("a title nobody shares is a candidate")
}
if len(cs) != 4 {
t.Fatalf("%d candidates, want the 4 files called Song", len(cs))
}
c, ok := candidateByRel(t, db, cs, "b/2.mp3")
if !ok {
t.Fatal("the mp3 is not a candidate")
}
if c.Title != "Song" || c.TitleKey != "song" || c.Album != "Album Y" || c.Format != "mp3" || c.BitRate != 192 ||
c.SizeBytes != 5_000_000 || c.DurationMs != 201_000 || c.FileID == 0 || c.Hash != "" {
t.Errorf("candidate = %+v", c)
}
if len(c.Artists) != 1 || c.Artists[0].Name != "Ann" {
t.Errorf("artists = %+v, want Ann", c.Artists)
}
duo, _ := candidateByRel(t, db, cs, "d/4.flac")
if len(duo.Artists) != 2 {
t.Errorf("a duet has %d artists, want 2", len(duo.Artists))
}
// Seen through the grouping, the two by Ann alone are the same song, the others are not.
groups := duplicates.Build(cs, duplicates.DurationTolerance)
if len(groups) != 1 || len(groups[0].Files) != 2 {
t.Errorf("groups = %+v, want one of the two copies by Ann", groups)
}
})
}
func TestAFileFlaggedMissingIsNotACandidate(t *testing.T) {
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
ctx := context.Background()
lib := NewLibraryRepo(db)
l := mkLib(t, lib, "music")
saved(t, lib, l, song("a/1.flac", "Song", "A", 200_000, 1, "flac", 900, "Ann"))
saved(t, lib, l, song("b/2.flac", "Song", "B", 200_000, 1, "flac", 900, "Ann"))
repo := NewDuplicatesRepo(db)
if cs, _ := repo.Candidates(ctx); len(cs) != 2 {
t.Fatalf("%d candidates before, want 2", len(cs))
}
var id int64
if err := db.QueryRow("SELECT id FROM audio_files WHERE rel_path = 'b/2.flac'").Scan(&id); err != nil {
t.Fatal(err)
}
if err := lib.MarkMissing(ctx, []int64{id}, time.Now()); err != nil {
t.Fatal(err)
}
if cs, _ := repo.Candidates(ctx); len(cs) != 0 {
t.Errorf("%d candidates after one went missing, want 0", len(cs))
}
})
}
func TestFileOfSaysWhereTheFileIsAndWhatItLookedLike(t *testing.T) {
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
ctx := context.Background()
lib := NewLibraryRepo(db)
l := mkLib(t, lib, "music")
saved(t, lib, l, song("a/1.flac", "Song", "A", 200_000, 1234, "flac", 900, "Ann"))
repo := NewDuplicatesRepo(db)
rec, err := repo.FileOf(ctx, trackOf(t, db, "a/1.flac"))
if err != nil {
t.Fatal(err)
}
if rec.Root != "/lib/music" || rec.RelPath != "a/1.flac" || rec.SizeBytes != 1234 || rec.MTimeUnix != 1_700_000_000 || rec.Hash != "" || rec.Missing {
t.Errorf("record = %+v", rec)
}
if _, err := repo.FileOf(ctx, 9999); !errors.Is(err, duplicates.ErrTrackNotFound) {
t.Errorf("an unknown track gave %v, want ErrTrackNotFound", err)
}
})
}
func TestAHashIsStoredOnlyForTheFileThatWasRead(t *testing.T) {
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
ctx := context.Background()
lib := NewLibraryRepo(db)
l := mkLib(t, lib, "music")
saved(t, lib, l, song("a/1.flac", "Song", "A", 200_000, 1234, "flac", 900, "Ann"))
saved(t, lib, l, song("b/2.flac", "Song", "B", 200_000, 1234, "flac", 900, "Ann"))
repo := NewDuplicatesRepo(db)
rec, _ := repo.FileOf(ctx, trackOf(t, db, "a/1.flac"))
// The file changed after it was read: its hash is no hash of it.
if ok, err := repo.SaveHash(ctx, rec.FileID, rec.SizeBytes+1, rec.MTimeUnix, "aa"); err != nil || ok {
t.Errorf("a hash of a file that changed in size was stored: ok=%v err=%v", ok, err)
}
if ok, err := repo.SaveHash(ctx, rec.FileID, rec.SizeBytes, rec.MTimeUnix+1, "aa"); err != nil || ok {
t.Errorf("a hash of a file that changed in date was stored: ok=%v err=%v", ok, err)
}
if ok, err := repo.SaveHash(ctx, rec.FileID, rec.SizeBytes, rec.MTimeUnix, "aa"); err != nil || !ok {
t.Fatalf("the hash was not stored: ok=%v err=%v", ok, err)
}
cs, _ := repo.Candidates(ctx)
c, _ := candidateByRel(t, db, cs, "a/1.flac")
other, _ := candidateByRel(t, db, cs, "b/2.flac")
if c.Hash != "aa" || other.Hash != "" {
t.Errorf("hashes = %q and %q, want aa and nothing", c.Hash, other.Hash)
}
})
}
// A hash says "this is what the file held when it was read". A scan that finds the file changed forgets it.
func TestAScanThatFindsAFileChangedForgetsItsHash(t *testing.T) {
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
ctx := context.Background()
lib := NewLibraryRepo(db)
l := mkLib(t, lib, "music")
saved(t, lib, l, song("a/1.flac", "Song", "A", 200_000, 1234, "flac", 900, "Ann"))
repo := NewDuplicatesRepo(db)
rec, _ := repo.FileOf(ctx, trackOf(t, db, "a/1.flac"))
if _, err := repo.SaveHash(ctx, rec.FileID, rec.SizeBytes, rec.MTimeUnix, "aa"); err != nil {
t.Fatal(err)
}
// The same file read again, unchanged (a full rescan): the hash is still true.
if _, err := lib.SaveFile(ctx, l, song("a/1.flac", "Song", "A", 200_000, 1234, "flac", 900, "Ann"), rec.FileID); err != nil {
t.Fatal(err)
}
if got, _ := repo.FileOf(ctx, rec.TrackID); got.Hash != "aa" {
t.Errorf("an unchanged file lost its hash: %q", got.Hash)
}
// The file was edited: another size.
if _, err := lib.SaveFile(ctx, l, song("a/1.flac", "Song", "A", 200_000, 4321, "flac", 900, "Ann"), rec.FileID); err != nil {
t.Fatal(err)
}
if got, _ := repo.FileOf(ctx, rec.TrackID); got.Hash != "" || got.SizeBytes != 4321 {
t.Errorf("after a change: hash %q size %d, want no hash and the new size", got.Hash, got.SizeBytes)
}
})
}
func TestRemovingACopyMovesItsListensToTheOneThatStays(t *testing.T) {
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
ctx := context.Background()
lib := NewLibraryRepo(db)
l := mkLib(t, lib, "music")
saved(t, lib, l, song("a/1.flac", "Song", "A", 200_000, 1, "flac", 900, "Ann"))
saved(t, lib, l, song("b/2.mp3", "Song", "B", 200_000, 1, "mp3", 192, "Ann"))
keep, gone := trackOf(t, db, "a/1.flac"), trackOf(t, db, "b/2.mp3")
user, err := NewUserRepo(db).CreateUser(ctx, "alice", "h", false)
if err != nil {
t.Fatal(err)
}
plays := NewListeningRepo(db)
for _, e := range []struct {
track int64
key string
}{{keep, "k1"}, {gone, "g1"}, {gone, "g2"}} {
if _, err := plays.Record(ctx, listening.Event{UserID: user.ID, TrackID: e.track, Type: listening.Complete, DurationPlayedMs: 200_000, IdempotencyKey: e.key}); err != nil {
t.Fatal(err)
}
}
if err := NewDuplicatesRepo(db).RemoveCopy(ctx, keep, gone); err != nil {
t.Fatal(err)
}
if n := count(t, db, "SELECT COUNT(*) FROM listening_events WHERE track_id = ?", keep); n != 3 {
t.Errorf("the track that stays has %d listens, want its own and the 2 moved (3)", n)
}
if n := count(t, db, "SELECT COUNT(*) FROM listening_events"); n != 3 {
t.Errorf("%d listens in all, want 3: none lost", n)
}
if n := count(t, db, "SELECT COUNT(*) FROM tracks WHERE id = ?", gone); n != 0 {
t.Error("the removed track is still there")
}
if n := count(t, db, "SELECT COUNT(*) FROM audio_files WHERE track_id = ?", gone); n != 0 {
t.Error("the file row of the removed track is still there")
}
if n := count(t, db, "SELECT COUNT(*) FROM track_artists WHERE track_id = ?", gone); n != 0 {
t.Error("the credits of the removed track are still there")
}
if n := count(t, db, "SELECT COUNT(*) FROM tracks WHERE id = ?", keep); n != 1 {
t.Error("the track that stays is gone")
}
})
}
func TestRemovingACopyKeepsItsPlaceInThePlaylists(t *testing.T) {
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
ctx := context.Background()
lib := NewLibraryRepo(db)
l := mkLib(t, lib, "music")
saved(t, lib, l, song("a/1.flac", "Song", "A", 200_000, 1, "flac", 900, "Ann"))
saved(t, lib, l, song("b/2.mp3", "Song", "B", 200_000, 1, "mp3", 192, "Ann"))
saved(t, lib, l, song("c/3.mp3", "Other", "C", 100_000, 1, "mp3", 192, "Ann"))
keep, gone, other := trackOf(t, db, "a/1.flac"), trackOf(t, db, "b/2.mp3"), trackOf(t, db, "c/3.mp3")
user, err := NewUserRepo(db).CreateUser(ctx, "alice@example.org", "h", false)
if err != nil {
t.Fatal(err)
}
pl := NewPlaylistRepo(db)
now := time.Unix(1_700_000_000, 0)
p, err := pl.Create(ctx, user.ID, "Mix", "", now)
if err != nil {
t.Fatal(err)
}
// The song twice in a row (the removed copy between two others), then another song.
if _, err := pl.AddTracks(ctx, user.ID, p.ID, []int64{keep, gone, other}, nil, now); err != nil {
t.Fatal(err)
}
if err := NewDuplicatesRepo(db).RemoveCopy(ctx, keep, gone); err != nil {
t.Fatal(err)
}
items, total, err := pl.Items(ctx, user.ID, p.ID, 50, 0)
if err != nil {
t.Fatal(err)
}
if total != 3 || len(items) != 3 {
t.Fatalf("the playlist has %d items, want 3: the removed copy's place must stay", total)
}
if got := []int64{items[0].Track.ID, items[1].Track.ID, items[2].Track.ID}; !slices.Equal(got, []int64{keep, keep, other}) {
t.Errorf("tracks in the playlist = %v, want %v", got, []int64{keep, keep, other})
}
})
}
func TestRemovingACopyNeedsTwoTracksThatExistAndAreDifferent(t *testing.T) {
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
ctx := context.Background()
lib := NewLibraryRepo(db)
l := mkLib(t, lib, "music")
saved(t, lib, l, song("a/1.flac", "Song", "A", 200_000, 1, "flac", 900, "Ann"))
keep := trackOf(t, db, "a/1.flac")
repo := NewDuplicatesRepo(db)
if err := repo.RemoveCopy(ctx, keep, 9999); !errors.Is(err, duplicates.ErrTrackNotFound) {
t.Errorf("a track that does not exist: %v, want ErrTrackNotFound", err)
}
if err := repo.RemoveCopy(ctx, 9999, keep); !errors.Is(err, duplicates.ErrTrackNotFound) {
t.Errorf("a track to keep that does not exist: %v, want ErrTrackNotFound", err)
}
if err := repo.RemoveCopy(ctx, keep, keep); !errors.Is(err, duplicates.ErrInvalid) {
t.Errorf("a track removed in favour of itself: %v, want ErrInvalid", err)
}
if n := count(t, db, "SELECT COUNT(*) FROM tracks"); n != 1 {
t.Errorf("%d tracks left, want the one that was there", n)
}
})
}

Some files were not shown because too many files have changed in this diff Show more