feat(api): list the duplicate songs, check their files and delete the extra copies
Admin only. GET /library/duplicates lists the groups with what is needed to choose and never a path; POST|GET|DELETE /library/duplicates/check is the job that fingerprints the files; POST /library/duplicates/clean deletes the chosen copies of one song, each refusal with its own status and code (deletion_disabled, scan_running, invalid_input, track_not_found, duplicate_changed, library_read_only). In the spec with the contract test, and wired in main, where an info line says when library.allow_delete is on.
This commit is contained in:
parent
d3e3bde2e4
commit
30f45bb383
7 changed files with 977 additions and 0 deletions
|
|
@ -21,6 +21,7 @@ import (
|
|||
"codeberg.org/squidd/sonda/internal/config"
|
||||
"codeberg.org/squidd/sonda/internal/discover"
|
||||
"codeberg.org/squidd/sonda/internal/discovery"
|
||||
"codeberg.org/squidd/sonda/internal/duplicates"
|
||||
"codeberg.org/squidd/sonda/internal/feedback"
|
||||
"codeberg.org/squidd/sonda/internal/images"
|
||||
"codeberg.org/squidd/sonda/internal/jobs"
|
||||
|
|
@ -261,6 +262,17 @@ func run() error {
|
|||
Preference: cfg.Scoring.WeightPreference, Discovery: cfg.Scoring.WeightDiscovery,
|
||||
}
|
||||
|
||||
// Deleting the extra copy of a song is the one thing that makes Sonda write in the music folders:
|
||||
// off unless the operator turned it on, and said in the log when it is on.
|
||||
duplicateSvc := duplicates.NewService(storage.NewDuplicatesRepo(db), duplicates.Options{
|
||||
AllowDelete: cfg.Library.AllowDelete,
|
||||
ScanRunning: func() bool { return scans.Status().Running },
|
||||
Logger: logger,
|
||||
})
|
||||
if cfg.Library.AllowDelete {
|
||||
logger.Warn("library.allow_delete is on: an admin can delete duplicate files from the music folders")
|
||||
}
|
||||
|
||||
router := httpapi.NewRouter(httpapi.Deps{
|
||||
Logger: logger,
|
||||
Auth: auth.NewService(storage.NewUserRepo(db), storage.NewRefreshRepo(db), tokens, cfg.Auth.RefreshTTL()),
|
||||
|
|
@ -281,6 +293,8 @@ func run() error {
|
|||
CoversJob: jobs.Runner{Ctx: ctx, Job: images.NewCoversJob(pictures, imageRepo)},
|
||||
ArtistImagesJob: jobs.Runner{Ctx: ctx, Job: images.NewArtistImagesJob(pictures, imageRepo)},
|
||||
DescriptionsJob: jobs.Runner{Ctx: ctx, Job: about.NewJob(descriptions, discoveryRepo)},
|
||||
Duplicates: duplicateSvc,
|
||||
DuplicatesJob: jobs.Runner{Ctx: ctx, Job: duplicates.NewHashJob(duplicateSvc)},
|
||||
ArtistImages: pictures,
|
||||
About: descriptions,
|
||||
Settings: userSettings,
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ import (
|
|||
"codeberg.org/squidd/sonda/internal/catalog"
|
||||
"codeberg.org/squidd/sonda/internal/discover"
|
||||
"codeberg.org/squidd/sonda/internal/discovery"
|
||||
"codeberg.org/squidd/sonda/internal/duplicates"
|
||||
"codeberg.org/squidd/sonda/internal/feedback"
|
||||
"codeberg.org/squidd/sonda/internal/images"
|
||||
"codeberg.org/squidd/sonda/internal/jobs"
|
||||
|
|
@ -115,6 +116,14 @@ func newTestEnvDB(t *testing.T, mutate func(*Deps, *bun.DB)) (http.Handler, *bun
|
|||
if deps.DescriptionsJob == nil {
|
||||
deps.DescriptionsJob = jobs.Runner{Ctx: context.Background(), Job: about.NewJob(about.NewService(storage.NewDiscoveryRepo(db), noWikipedia{}, prov, []string{"en"}, zap.NewNop()), storage.NewDiscoveryRepo(db))}
|
||||
}
|
||||
// Deleting is off unless a test turns it on.
|
||||
if deps.Duplicates == nil {
|
||||
svc := duplicates.NewService(storage.NewDuplicatesRepo(db), duplicates.Options{Logger: zap.NewNop()})
|
||||
deps.Duplicates = svc
|
||||
if deps.DuplicatesJob == nil {
|
||||
deps.DuplicatesJob = jobs.Runner{Ctx: context.Background(), Job: duplicates.NewHashJob(svc)}
|
||||
}
|
||||
}
|
||||
// Built from the final services, so a test that swaps one gets a Discover that uses it.
|
||||
if deps.Discover == nil {
|
||||
deps.Discover = discover.NewService(deps.Taste.(discover.Taster), deps.Discovery.(discover.Similar),
|
||||
|
|
|
|||
|
|
@ -26,7 +26,9 @@ import (
|
|||
|
||||
"codeberg.org/squidd/sonda/internal/catalog"
|
||||
"codeberg.org/squidd/sonda/internal/discovery"
|
||||
"codeberg.org/squidd/sonda/internal/duplicates"
|
||||
"codeberg.org/squidd/sonda/internal/images"
|
||||
"codeberg.org/squidd/sonda/internal/jobs"
|
||||
"codeberg.org/squidd/sonda/internal/library"
|
||||
"codeberg.org/squidd/sonda/internal/providers"
|
||||
"codeberg.org/squidd/sonda/internal/providers/lastfm"
|
||||
|
|
@ -140,6 +142,11 @@ func newContract(t *testing.T) *contract {
|
|||
libs := library.NewLibraries(repo)
|
||||
libs.SetBusy(func() bool { return mgr.Status().Running })
|
||||
d.Scan = scanAdapter{mgr}
|
||||
dupSvc := duplicates.NewService(storage.NewDuplicatesRepo(db), duplicates.Options{
|
||||
AllowDelete: true, ScanRunning: func() bool { return mgr.Status().Running }, Logger: zap.NewNop(),
|
||||
})
|
||||
d.Duplicates = dupSvc
|
||||
d.DuplicatesJob = jobs.Runner{Ctx: context.Background(), Job: duplicates.NewHashJob(dupSvc)}
|
||||
d.Libraries = libs
|
||||
d.Catalog = catalog.NewService(storage.NewCatalogRepo(db))
|
||||
d.Streams = streaming.NewService(repo)
|
||||
|
|
@ -767,6 +774,9 @@ func TestResponsesMatchTheSpec(t *testing.T) {
|
|||
c.expect(c.call("PATCH", "/me/settings", token, map[string]any{"nope": true}), 400, "a setting that does not exist", "invalid_input")
|
||||
c.expect(c.call("GET", "/me/settings", "", nil), 401, "settings without a token", "unauthorized")
|
||||
|
||||
// ---- duplicate songs ----
|
||||
c.duplicates(t, token, libID)
|
||||
|
||||
// ---- every operation in the spec was exercised ----
|
||||
var missing []string
|
||||
for _, op := range specOperations(loadSpec(t)) {
|
||||
|
|
@ -779,6 +789,132 @@ func TestResponsesMatchTheSpec(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// duplicates exercises the duplicate songs: the same song as a.mp3 is added to the library, as real
|
||||
// files, so that what is checked against the spec is what the real services answer.
|
||||
func (c *contract) duplicates(t *testing.T, token, libID string) {
|
||||
t.Helper()
|
||||
repo := storage.NewLibraryRepo(c.db)
|
||||
libInt, err := strconv.ParseInt(libID, 10, 64)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// addCopy puts a second file with the tags of Song A in the library, as the scanner would.
|
||||
addCopy := func(name string) {
|
||||
t.Helper()
|
||||
path := filepath.Join(c.music, name)
|
||||
if err := os.WriteFile(path, bytes.Repeat([]byte{0xAB}, 400), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := library.ParsedFile{
|
||||
FileInfo: library.FileInfo{RelPath: name, Size: info.Size(), MTimeUnix: info.ModTime().Unix()},
|
||||
Metadata: library.Metadata{
|
||||
Tags: library.Tags{Title: "Song A", Album: "Compilation", AlbumArtist: "Artist X", Artists: []string{"Artist X"}, TrackNo: 1, DiscNo: 1},
|
||||
Properties: library.Properties{Format: "mpeg", DurationMs: 180000, SampleRate: 44100},
|
||||
},
|
||||
}
|
||||
if _, err := repo.SaveFile(context.Background(), libInt, f, 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
trackOf := func(name string) int64 {
|
||||
t.Helper()
|
||||
var id int64
|
||||
if err := c.db.QueryRow("SELECT track_id FROM audio_files WHERE rel_path = ?", name).Scan(&id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
exists := func(name string) bool { _, err := os.Lstat(filepath.Join(c.music, name)); return err == nil }
|
||||
body := func(keep int64, remove ...int64) map[string]any {
|
||||
return map[string]any{"keep_track_id": keep, "remove_track_ids": remove}
|
||||
}
|
||||
|
||||
none := c.expect(c.call("GET", "/library/duplicates", token, nil), 200, "duplicates when there are none")
|
||||
if none["total"] != float64(0) || none["groups"] == nil || none["deletion_allowed"] != true {
|
||||
t.Fatalf("duplicates = %v, want no group, an empty list and deleting allowed", none)
|
||||
}
|
||||
|
||||
addCopy("dup.mp3")
|
||||
keep, copyID := trackOf("a.mp3"), trackOf("dup.mp3")
|
||||
list := c.expect(c.call("GET", "/library/duplicates", token, nil), 200, "duplicates")
|
||||
groups, _ := list["groups"].([]any)
|
||||
if list["total"] != float64(1) || len(groups) != 1 {
|
||||
t.Fatalf("duplicates = %v, want the one group of Song A", list)
|
||||
}
|
||||
if g := groups[0].(map[string]any); g["title"] != "Song A" || g["suggested_keep_track_id"] != float64(keep) || g["hashed"] != false {
|
||||
t.Fatalf("group = %v, want Song A with a.mp3 to keep and nothing read yet", g)
|
||||
}
|
||||
c.expect(c.call("GET", "/library/duplicates?limit=-1", token, nil), 400, "duplicates with a bad limit", "invalid_input")
|
||||
c.expect(c.call("GET", "/library/duplicates", "", nil), 401, "duplicates without a token", "unauthorized")
|
||||
|
||||
// Not an admin: nothing of this is theirs.
|
||||
c.expect(c.call("POST", "/users", token, map[string]any{"email": "carol@example.org", "password": "carols long pass"}), 201, "create a plain user")
|
||||
carol := c.expect(c.call("POST", "/auth/login", "", map[string]string{"email": "carol@example.org", "password": "carols long pass"}), 200, "carol logs in")["access_token"].(string)
|
||||
c.expect(c.call("GET", "/library/duplicates", carol, nil), 403, "duplicates as a plain user", "admin_required")
|
||||
c.expect(c.call("POST", "/library/duplicates/clean", carol, body(keep, copyID)), 403, "cleaning as a plain user", "admin_required")
|
||||
|
||||
// The check reads the files, and the group is then identical.
|
||||
if idle := c.expect(c.call("GET", "/library/duplicates/check", token, nil), 200, "the check before it ran"); idle["running"] != false {
|
||||
t.Fatalf("check = %v", idle)
|
||||
}
|
||||
c.expect(c.call("POST", "/library/duplicates/check?refresh=maybe", token, nil), 400, "a check with a bad refresh", "invalid_input")
|
||||
c.expect(c.call("POST", "/library/duplicates/check", "", nil), 401, "a check without a token", "unauthorized")
|
||||
c.expect(c.call("POST", "/library/duplicates/check", token, nil), 202, "start the check")
|
||||
for i := 0; ; i++ {
|
||||
st := c.expect(c.call("GET", "/library/duplicates/check", token, nil), 200, "the check running")
|
||||
if st["running"] == false {
|
||||
if last, _ := st["last"].(map[string]any); last == nil || last["checked"] != float64(2) || last["hashed"] != float64(2) {
|
||||
t.Fatalf("check = %v, want 2 files checked and hashed", st)
|
||||
}
|
||||
break
|
||||
}
|
||||
if i > 400 {
|
||||
t.Fatal("the check did not finish")
|
||||
}
|
||||
time.Sleep(25 * time.Millisecond)
|
||||
}
|
||||
c.expect(c.call("DELETE", "/library/duplicates/check", token, nil), 204, "stop the check when none runs")
|
||||
after := c.expect(c.call("GET", "/library/duplicates", token, nil), 200, "duplicates after the check")
|
||||
if g := after["groups"].([]any)[0].(map[string]any); g["hashed"] != true || g["identical"] != true {
|
||||
t.Fatalf("group = %v, want hashed and identical", g)
|
||||
}
|
||||
|
||||
// Refusals first: nothing is removed by any of them.
|
||||
c.expect(c.call("POST", "/library/duplicates/clean", token, body(keep, keep)), 400, "cleaning the copy to keep", "invalid_input")
|
||||
c.expect(c.call("POST", "/library/duplicates/clean", token, map[string]any{"keep_track_id": keep}), 400, "cleaning with nothing to remove", "invalid_input")
|
||||
c.expect(c.call("POST", "/library/duplicates/clean", token, body(keep, 99999)), 404, "cleaning a track that does not exist", "track_not_found")
|
||||
c.expect(c.call("POST", "/library/duplicates/clean", "", body(keep, copyID)), 401, "cleaning without a token", "unauthorized")
|
||||
other := trackOf("b.flac")
|
||||
c.expect(c.call("POST", "/library/duplicates/clean", token, body(keep, other)), 400, "cleaning something that is not a copy", "invalid_input")
|
||||
if !exists("dup.mp3") || !exists("b.flac") {
|
||||
t.Fatal("a refused request removed a file")
|
||||
}
|
||||
|
||||
// A copy that changed since the scan is not removed.
|
||||
addCopy("dup2.mp3")
|
||||
dup2 := trackOf("dup2.mp3")
|
||||
if err := os.WriteFile(filepath.Join(c.music, "dup2.mp3"), bytes.Repeat([]byte{0xCD}, 500), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.expect(c.call("POST", "/library/duplicates/clean", token, body(keep, dup2)), 409, "cleaning a copy that changed", "duplicate_changed")
|
||||
if !exists("dup2.mp3") {
|
||||
t.Fatal("a copy that changed was removed")
|
||||
}
|
||||
|
||||
// The real thing: the extra copy leaves the disk and the index, the other stays.
|
||||
done := c.expect(c.call("POST", "/library/duplicates/clean", token, body(keep, copyID)), 200, "clean the duplicate")
|
||||
if done["removed"] != float64(1) || done["freed_bytes"] != float64(400) {
|
||||
t.Fatalf("clean = %v, want 1 file and 400 bytes", done)
|
||||
}
|
||||
if exists("dup.mp3") || !exists("a.mp3") {
|
||||
t.Fatalf("after the clean dup.mp3 exists=%v a.mp3 exists=%v", exists("dup.mp3"), exists("a.mp3"))
|
||||
}
|
||||
}
|
||||
|
||||
type orphanIDs struct{ album, track string }
|
||||
|
||||
// addOrphan inserts an album with no album artist and a track in it whose file
|
||||
|
|
|
|||
375
internal/api/http/duplicates_test.go
Normal file
375
internal/api/http/duplicates_test.go
Normal file
|
|
@ -0,0 +1,375 @@
|
|||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/uptrace/bun"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"codeberg.org/squidd/sonda/internal/duplicates"
|
||||
"codeberg.org/squidd/sonda/internal/jobs"
|
||||
"codeberg.org/squidd/sonda/internal/library"
|
||||
"codeberg.org/squidd/sonda/internal/storage"
|
||||
)
|
||||
|
||||
// dupAPI is the API over a real library on a real folder.
|
||||
type dupAPI struct {
|
||||
t *testing.T
|
||||
h http.Handler
|
||||
db *bun.DB
|
||||
dir string
|
||||
token string
|
||||
scanning bool
|
||||
}
|
||||
|
||||
func newDupAPI(t *testing.T, allow bool) *dupAPI {
|
||||
t.Helper()
|
||||
a := &dupAPI{t: t}
|
||||
a.h, a.db = newTestEnvDB(t, func(d *Deps, db *bun.DB) {
|
||||
svc := duplicates.NewService(storage.NewDuplicatesRepo(db), duplicates.Options{
|
||||
AllowDelete: allow, ScanRunning: func() bool { return a.scanning }, Logger: zap.NewNop(),
|
||||
})
|
||||
d.Duplicates = svc
|
||||
d.DuplicatesJob = jobs.Runner{Ctx: context.Background(), Job: duplicates.NewHashJob(svc)}
|
||||
})
|
||||
dir, err := filepath.EvalSymlinks(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.dir = dir
|
||||
a.token = registerAndLogin(t, a.h)
|
||||
if _, err := storage.NewLibraryRepo(a.db).CreateLibrary(context.Background(), "music", dir); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
// seed writes a file and indexes it as it is, and returns its track.
|
||||
func (a *dupAPI) seed(rel, content, title, album string, durationMs int64, format string, bitrate int, artists ...string) int64 {
|
||||
a.t.Helper()
|
||||
path := filepath.Join(a.dir, filepath.FromSlash(rel))
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
a.t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
a.t.Fatal(err)
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
a.t.Fatal(err)
|
||||
}
|
||||
f := library.ParsedFile{
|
||||
FileInfo: library.FileInfo{RelPath: rel, Size: info.Size(), MTimeUnix: info.ModTime().Unix()},
|
||||
Metadata: library.Metadata{
|
||||
Tags: library.Tags{Title: title, Album: album, AlbumArtist: artists[0], Artists: artists, TrackNo: 1, DiscNo: 1},
|
||||
Properties: library.Properties{Format: format, DurationMs: durationMs, SampleRate: 44100, BitRate: bitrate},
|
||||
},
|
||||
}
|
||||
if _, err := storage.NewLibraryRepo(a.db).SaveFile(context.Background(), 1, f, 0); err != nil {
|
||||
a.t.Fatal(err)
|
||||
}
|
||||
var id int64
|
||||
if err := a.db.QueryRow("SELECT track_id FROM audio_files WHERE rel_path = ?", rel).Scan(&id); err != nil {
|
||||
a.t.Fatal(err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func (a *dupAPI) twoCopies() (flac, mp3 int64) {
|
||||
flac = a.seed("a/song.flac", "FLAC-CONTENT-OF-THE-SONG", "Song", "Album", 200_000, "flac", 900, "Ann")
|
||||
mp3 = a.seed("b/song.mp3", "mp3 content", "Song", "Greatest hits", 200_500, "mp3", 192, "Ann")
|
||||
return flac, mp3
|
||||
}
|
||||
|
||||
func (a *dupAPI) exists(rel string) bool {
|
||||
_, err := os.Lstat(filepath.Join(a.dir, filepath.FromSlash(rel)))
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func (a *dupAPI) get(path string) resp { return do(a.t, a.h, "GET", path, a.token, nil) }
|
||||
|
||||
func (a *dupAPI) clean(keep int64, remove ...int64) resp {
|
||||
return do(a.t, a.h, "POST", "/api/v1/library/duplicates/clean", a.token, map[string]any{"keep_track_id": keep, "remove_track_ids": remove})
|
||||
}
|
||||
|
||||
func TestDuplicateRoutesAreAdminOnly(t *testing.T) {
|
||||
a := newDupAPI(t, true)
|
||||
_, bob := addUser(t, a.h, a.token, "bob", false)
|
||||
calls := []struct{ method, path string }{
|
||||
{"GET", "/api/v1/library/duplicates"},
|
||||
{"POST", "/api/v1/library/duplicates/clean"},
|
||||
{"GET", "/api/v1/library/duplicates/check"},
|
||||
{"POST", "/api/v1/library/duplicates/check"},
|
||||
{"DELETE", "/api/v1/library/duplicates/check"},
|
||||
}
|
||||
for _, c := range calls {
|
||||
if r := do(t, a.h, c.method, c.path, "", nil); r.Code != http.StatusUnauthorized {
|
||||
t.Errorf("%s %s without a token: %d", c.method, c.path, r.Code)
|
||||
}
|
||||
r := do(t, a.h, c.method, c.path, bob, map[string]any{"keep_track_id": 1, "remove_track_ids": []int{2}})
|
||||
if r.Code != http.StatusForbidden || errCode(r) != "admin_required" {
|
||||
t.Errorf("%s %s as a plain user: %d %v", c.method, c.path, r.Code, r.Body)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheListShowsTheGroupsAndNeverAPath(t *testing.T) {
|
||||
a := newDupAPI(t, true)
|
||||
flac, mp3 := a.twoCopies()
|
||||
a.seed("c/other.flac", "else", "Other", "Album", 100_000, "flac", 900, "Ann")
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest("GET", "/api/v1/library/duplicates", nil)
|
||||
req.Header.Set("Authorization", "Bearer "+a.token)
|
||||
a.h.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status %d: %s", rec.Code, rec.Body)
|
||||
}
|
||||
raw := rec.Body.String()
|
||||
for _, secret := range []string{a.dir, "song.flac", "song.mp3", "rel_path", "path"} {
|
||||
if strings.Contains(raw, secret) {
|
||||
t.Errorf("the answer contains %q: file paths are private", secret)
|
||||
}
|
||||
}
|
||||
|
||||
var out struct {
|
||||
DeletionAllowed bool `json:"deletion_allowed"`
|
||||
Total, Limit int
|
||||
Offset int
|
||||
Groups []struct {
|
||||
Title string `json:"title"`
|
||||
Artists []struct {
|
||||
ID int64 `json:"id"`
|
||||
Name string `json:"name"`
|
||||
} `json:"artists"`
|
||||
Files []struct {
|
||||
TrackID int64 `json:"track_id"`
|
||||
Album *struct {
|
||||
ID int64 `json:"id"`
|
||||
Title string `json:"title"`
|
||||
} `json:"album"`
|
||||
Format string `json:"format"`
|
||||
Bitrate int `json:"bitrate"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
DurationMs int64 `json:"duration_ms"`
|
||||
Hashed bool `json:"hashed"`
|
||||
} `json:"files"`
|
||||
Hashed bool `json:"hashed"`
|
||||
Identical bool `json:"identical"`
|
||||
Keep int64 `json:"suggested_keep_track_id"`
|
||||
FreedBytes int64 `json:"freed_bytes"`
|
||||
} `json:"groups"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !out.DeletionAllowed || out.Total != 1 || out.Limit != 50 || out.Offset != 0 || len(out.Groups) != 1 {
|
||||
t.Fatalf("answer = %+v", out)
|
||||
}
|
||||
g := out.Groups[0]
|
||||
if g.Title != "Song" || len(g.Artists) != 1 || g.Artists[0].Name != "Ann" || g.Keep != flac || g.FreedBytes != int64(len("mp3 content")) || g.Hashed || g.Identical {
|
||||
t.Errorf("group = %+v", g)
|
||||
}
|
||||
if len(g.Files) != 2 || g.Files[1].TrackID != mp3 || g.Files[1].Album == nil || g.Files[1].Album.Title != "Greatest hits" ||
|
||||
g.Files[1].Format != "mp3" || g.Files[1].Bitrate != 192 || g.Files[1].DurationMs != 200_500 || g.Files[1].Hashed {
|
||||
t.Errorf("files = %+v", g.Files)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheListSaysWhenDeletingIsOff(t *testing.T) {
|
||||
a := newDupAPI(t, false)
|
||||
a.twoCopies()
|
||||
r := a.get("/api/v1/library/duplicates")
|
||||
if r.Code != 200 || r.Body["deletion_allowed"] != false || r.Body["total"] != float64(1) {
|
||||
t.Errorf("answer = %d %v", r.Code, r.Body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheListIsPagedAndCheckedForGoodParameters(t *testing.T) {
|
||||
a := newDupAPI(t, true)
|
||||
for _, title := range []string{"A song", "B song"} {
|
||||
a.seed("x/"+title+".mp3", title+"1", title, "Album", 100_000, "mp3", 192, "Ann")
|
||||
a.seed("y/"+title+".mp3", title+"22", title, "Album", 100_000, "mp3", 192, "Ann")
|
||||
}
|
||||
first := a.get("/api/v1/library/duplicates?limit=1")
|
||||
second := a.get("/api/v1/library/duplicates?limit=1&offset=1")
|
||||
if first.Body["total"] != float64(2) || len(first.Body["groups"].([]any)) != 1 || len(second.Body["groups"].([]any)) != 1 {
|
||||
t.Fatalf("pages = %v and %v", first.Body, second.Body)
|
||||
}
|
||||
if first.Body["limit"] != float64(1) || second.Body["offset"] != float64(1) {
|
||||
t.Errorf("limit and offset are not echoed: %v %v", first.Body, second.Body)
|
||||
}
|
||||
if past := a.get("/api/v1/library/duplicates?offset=10"); len(past.Body["groups"].([]any)) != 0 || past.Body["groups"] == nil {
|
||||
t.Errorf("past the end: %v, want an empty list and not null", past.Body)
|
||||
}
|
||||
if big := a.get("/api/v1/library/duplicates?limit=100000"); big.Body["limit"] != float64(200) {
|
||||
t.Errorf("a huge limit became %v, want 200", big.Body["limit"])
|
||||
}
|
||||
for _, q := range []string{"limit=-1", "offset=-1", "limit=x", "offset=1.5"} {
|
||||
if r := a.get("/api/v1/library/duplicates?" + q); r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" {
|
||||
t.Errorf("?%s: %d %v, want 400 invalid_input", q, r.Code, r.Body)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleaningRemovesTheCopyAndSaysHowMuchWasFreed(t *testing.T) {
|
||||
a := newDupAPI(t, true)
|
||||
flac, mp3 := a.twoCopies()
|
||||
r := a.clean(flac, mp3)
|
||||
if r.Code != http.StatusOK || r.Body["removed"] != float64(1) || r.Body["freed_bytes"] != float64(len("mp3 content")) {
|
||||
t.Fatalf("answer = %d %v", r.Code, r.Body)
|
||||
}
|
||||
if a.exists("b/song.mp3") || !a.exists("a/song.flac") {
|
||||
t.Error("the wrong file was removed")
|
||||
}
|
||||
if list := a.get("/api/v1/library/duplicates"); list.Body["total"] != float64(0) {
|
||||
t.Errorf("the list still shows %v groups", list.Body["total"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestEveryRefusalHasItsOwnStatusAndCode(t *testing.T) {
|
||||
t.Run("deleting is off", func(t *testing.T) {
|
||||
a := newDupAPI(t, false)
|
||||
flac, mp3 := a.twoCopies()
|
||||
if r := a.clean(flac, mp3); r.Code != http.StatusConflict || errCode(r) != "deletion_disabled" || !a.exists("b/song.mp3") {
|
||||
t.Errorf("%d %v", r.Code, r.Body)
|
||||
}
|
||||
})
|
||||
t.Run("a scan is running", func(t *testing.T) {
|
||||
a := newDupAPI(t, true)
|
||||
flac, mp3 := a.twoCopies()
|
||||
a.scanning = true
|
||||
if r := a.clean(flac, mp3); r.Code != http.StatusConflict || errCode(r) != "scan_running" || !a.exists("b/song.mp3") {
|
||||
t.Errorf("%d %v", r.Code, r.Body)
|
||||
}
|
||||
})
|
||||
t.Run("not copies of one song", func(t *testing.T) {
|
||||
a := newDupAPI(t, true)
|
||||
flac, _ := a.twoCopies()
|
||||
other := a.seed("c/other.flac", "else", "Other", "Album", 100_000, "flac", 900, "Ann")
|
||||
if r := a.clean(flac, other); r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" || !a.exists("c/other.flac") {
|
||||
t.Errorf("%d %v", r.Code, r.Body)
|
||||
}
|
||||
})
|
||||
t.Run("a track that does not exist", func(t *testing.T) {
|
||||
a := newDupAPI(t, true)
|
||||
flac, _ := a.twoCopies()
|
||||
if r := a.clean(flac, 9999); r.Code != http.StatusNotFound || errCode(r) != "track_not_found" {
|
||||
t.Errorf("%d %v", r.Code, r.Body)
|
||||
}
|
||||
})
|
||||
t.Run("a file that changed since the scan", func(t *testing.T) {
|
||||
a := newDupAPI(t, true)
|
||||
flac, mp3 := a.twoCopies()
|
||||
if err := os.WriteFile(filepath.Join(a.dir, "b/song.mp3"), []byte("edited, and longer than it was"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r := a.clean(flac, mp3); r.Code != http.StatusConflict || errCode(r) != "duplicate_changed" || !a.exists("b/song.mp3") {
|
||||
t.Errorf("%d %v", r.Code, r.Body)
|
||||
}
|
||||
})
|
||||
t.Run("a read-only folder", func(t *testing.T) {
|
||||
if runtime.GOOS == "windows" || os.Geteuid() == 0 {
|
||||
t.Skip("permissions are not enforced here")
|
||||
}
|
||||
a := newDupAPI(t, true)
|
||||
flac, mp3 := a.twoCopies()
|
||||
dir := filepath.Join(a.dir, "b")
|
||||
if err := os.Chmod(dir, 0o555); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = os.Chmod(dir, 0o755) })
|
||||
if r := a.clean(flac, mp3); r.Code != http.StatusConflict || errCode(r) != "library_read_only" || !a.exists("b/song.mp3") {
|
||||
t.Errorf("%d %v", r.Code, r.Body)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestACleanRequestMustBeWellFormed(t *testing.T) {
|
||||
a := newDupAPI(t, true)
|
||||
flac, mp3 := a.twoCopies()
|
||||
bodies := map[string]any{
|
||||
"malformed JSON": `{"keep_track_id":`,
|
||||
"no copy to keep": map[string]any{"remove_track_ids": []int64{mp3}},
|
||||
"nothing to remove": map[string]any{"keep_track_id": flac, "remove_track_ids": []int64{}},
|
||||
"the one to keep too": map[string]any{"keep_track_id": flac, "remove_track_ids": []int64{flac, mp3}},
|
||||
"a negative id": map[string]any{"keep_track_id": flac, "remove_track_ids": []int64{-1}},
|
||||
"ids that are strings": map[string]any{"keep_track_id": "1", "remove_track_ids": []string{"2"}},
|
||||
}
|
||||
for name, body := range bodies {
|
||||
r := do(t, a.h, "POST", "/api/v1/library/duplicates/clean", a.token, body)
|
||||
if r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" {
|
||||
t.Errorf("%s: %d %v, want 400 invalid_input", name, r.Code, r.Body)
|
||||
}
|
||||
}
|
||||
if !a.exists("b/song.mp3") || !a.exists("a/song.flac") {
|
||||
t.Error("a file was removed by a bad request")
|
||||
}
|
||||
}
|
||||
|
||||
// --- the check job ---------------------------------------------------------------------------------
|
||||
|
||||
func TestTheCheckJobReadsTheFilesAndReportsWhatItDid(t *testing.T) {
|
||||
a := newDupAPI(t, true)
|
||||
a.seed("a/1.mp3", "the same bytes", "Same", "A", 100_000, "mp3", 192, "Ann")
|
||||
a.seed("b/2.mp3", "the same bytes", "Same", "B", 100_000, "mp3", 192, "Ann")
|
||||
|
||||
if idle := a.get("/api/v1/library/duplicates/check"); idle.Code != 200 || idle.Body["running"] != false {
|
||||
t.Fatalf("before it ran: %d %v", idle.Code, idle.Body)
|
||||
}
|
||||
if r := do(t, a.h, "POST", "/api/v1/library/duplicates/check?refresh=maybe", a.token, nil); r.Code != 400 || errCode(r) != "invalid_input" {
|
||||
t.Errorf("a bad refresh: %d %v", r.Code, r.Body)
|
||||
}
|
||||
if r := do(t, a.h, "POST", "/api/v1/library/duplicates/check", a.token, nil); r.Code != http.StatusAccepted || r.Body["status"] != "started" {
|
||||
t.Fatalf("start: %d %v", r.Code, r.Body)
|
||||
}
|
||||
var st resp
|
||||
for i := 0; ; i++ {
|
||||
st = a.get("/api/v1/library/duplicates/check")
|
||||
if st.Body["running"] == false {
|
||||
break
|
||||
}
|
||||
if i > 400 {
|
||||
t.Fatal("the job did not finish")
|
||||
}
|
||||
time.Sleep(25 * time.Millisecond)
|
||||
}
|
||||
last, _ := st.Body["last"].(map[string]any)
|
||||
if last["checked"] != float64(2) || last["hashed"] != float64(2) || last["skipped"] != float64(0) || last["failed"] != float64(0) || last["stopped"] != false {
|
||||
t.Errorf("last = %v, want 2 checked and 2 hashed", last)
|
||||
}
|
||||
if list := a.get("/api/v1/library/duplicates"); list.Body["groups"].([]any)[0].(map[string]any)["identical"] != true {
|
||||
t.Errorf("the group is not called identical after the check: %v", list.Body)
|
||||
}
|
||||
if r := do(t, a.h, "DELETE", "/api/v1/library/duplicates/check", a.token, nil); r.Code != http.StatusNoContent {
|
||||
t.Errorf("stop: %d", r.Code)
|
||||
}
|
||||
}
|
||||
|
||||
type alreadyRunning struct{}
|
||||
|
||||
func (alreadyRunning) Start(bool) error { return jobs.ErrRunning }
|
||||
func (alreadyRunning) Stop() bool { return false }
|
||||
func (alreadyRunning) Status() jobs.Status { return jobs.Status{} }
|
||||
|
||||
func TestStartingTheCheckWhileItRunsIsExplained(t *testing.T) {
|
||||
a := &dupAPI{t: t}
|
||||
a.h, a.db = newTestEnvDB(t, func(d *Deps, db *bun.DB) {
|
||||
d.Duplicates = duplicates.NewService(storage.NewDuplicatesRepo(db), duplicates.Options{})
|
||||
d.DuplicatesJob = alreadyRunning{}
|
||||
})
|
||||
a.token = registerAndLogin(t, a.h)
|
||||
r := do(t, a.h, "POST", "/api/v1/library/duplicates/check", a.token, nil)
|
||||
if r.Code != http.StatusConflict || errCode(r) != "duplicates_check_running" {
|
||||
t.Errorf("%d %v, want 409 duplicates_check_running", r.Code, r.Body)
|
||||
}
|
||||
}
|
||||
169
internal/api/http/handlers_duplicates.go
Normal file
169
internal/api/http/handlers_duplicates.go
Normal file
|
|
@ -0,0 +1,169 @@
|
|||
package httpapi
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"codeberg.org/squidd/sonda/internal/duplicates"
|
||||
"codeberg.org/squidd/sonda/internal/jobs"
|
||||
)
|
||||
|
||||
// duplicateHandlers serve the admin's view of the songs that are in more than one file, and the
|
||||
// removal of the extra copies. File paths never appear in any answer.
|
||||
type duplicateHandlers struct{ svc duplicateService }
|
||||
|
||||
const (
|
||||
defaultDuplicatesLimit = 50
|
||||
maxDuplicatesLimit = 200
|
||||
)
|
||||
|
||||
type duplicateFileJSON struct {
|
||||
TrackID int64 `json:"track_id"`
|
||||
Album *albumRefJSON `json:"album"`
|
||||
Format string `json:"format"`
|
||||
Bitrate int `json:"bitrate"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
DurationMs int64 `json:"duration_ms"`
|
||||
Hashed bool `json:"hashed"`
|
||||
}
|
||||
|
||||
type duplicateGroupJSON struct {
|
||||
Title string `json:"title"`
|
||||
Artists []artistRefJSON `json:"artists"`
|
||||
Files []duplicateFileJSON `json:"files"`
|
||||
Hashed bool `json:"hashed"`
|
||||
Identical bool `json:"identical"`
|
||||
SuggestedID int64 `json:"suggested_keep_track_id"`
|
||||
FreedBytes int64 `json:"freed_bytes"`
|
||||
}
|
||||
|
||||
type duplicatePageJSON struct {
|
||||
DeletionAllowed bool `json:"deletion_allowed"`
|
||||
Total int `json:"total"`
|
||||
Limit int `json:"limit"`
|
||||
Offset int `json:"offset"`
|
||||
Groups []duplicateGroupJSON `json:"groups"`
|
||||
}
|
||||
|
||||
// list gives the groups of copies, a page at a time.
|
||||
func (h *duplicateHandlers) list(c *gin.Context) {
|
||||
limit, ok := queryInt(c, "limit")
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
offset, ok := queryInt(c, "offset")
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if limit < 0 || offset < 0 {
|
||||
writeError(c, http.StatusBadRequest, "invalid_input", "limit and offset cannot be negative")
|
||||
return
|
||||
}
|
||||
size := clampInt(limit)
|
||||
if size == 0 {
|
||||
size = defaultDuplicatesLimit
|
||||
}
|
||||
size = min(size, maxDuplicatesLimit)
|
||||
|
||||
page, err := h.svc.Groups(c.Request.Context(), size, clampInt(offset))
|
||||
if err != nil {
|
||||
_ = c.Error(err)
|
||||
writeError(c, http.StatusInternalServerError, "internal", "internal error")
|
||||
return
|
||||
}
|
||||
out := duplicatePageJSON{
|
||||
DeletionAllowed: page.DeletionAllowed, Total: page.Total, Limit: size, Offset: clampInt(offset),
|
||||
Groups: make([]duplicateGroupJSON, len(page.Groups)),
|
||||
}
|
||||
for i, g := range page.Groups {
|
||||
out.Groups[i] = toDuplicateGroupJSON(g)
|
||||
}
|
||||
c.JSON(http.StatusOK, out)
|
||||
}
|
||||
|
||||
func toDuplicateGroupJSON(g duplicates.Group) duplicateGroupJSON {
|
||||
out := duplicateGroupJSON{
|
||||
Title: g.Title, Artists: make([]artistRefJSON, len(g.Artists)), Files: make([]duplicateFileJSON, len(g.Files)),
|
||||
Hashed: g.Hashed, Identical: g.Identical, SuggestedID: g.KeepTrackID, FreedBytes: g.FreedBytes,
|
||||
}
|
||||
for i, a := range g.Artists {
|
||||
out.Artists[i] = artistRefJSON{ID: a.ID, Name: a.Name}
|
||||
}
|
||||
for i, f := range g.Files {
|
||||
file := duplicateFileJSON{
|
||||
TrackID: f.TrackID, Format: f.Format, Bitrate: f.BitRate, SizeBytes: f.SizeBytes, DurationMs: f.DurationMs, Hashed: f.Hash != "",
|
||||
}
|
||||
if f.AlbumID != 0 {
|
||||
file.Album = &albumRefJSON{ID: f.AlbumID, Title: f.Album}
|
||||
}
|
||||
out.Files[i] = file
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type cleanRequest struct {
|
||||
KeepTrackID int64 `json:"keep_track_id"`
|
||||
RemoveTrackIDs []int64 `json:"remove_track_ids"`
|
||||
}
|
||||
|
||||
// clean removes the copies named of one song, keeping the one named.
|
||||
func (h *duplicateHandlers) clean(c *gin.Context) {
|
||||
var req cleanRequest
|
||||
if !bindJSON(c, &req) {
|
||||
return
|
||||
}
|
||||
if req.KeepTrackID <= 0 || len(req.RemoveTrackIDs) == 0 {
|
||||
writeError(c, http.StatusBadRequest, "invalid_input", "name the copy to keep and at least one copy to remove")
|
||||
return
|
||||
}
|
||||
for _, id := range req.RemoveTrackIDs {
|
||||
if id <= 0 {
|
||||
writeError(c, http.StatusBadRequest, "invalid_input", "track ids must be positive numbers")
|
||||
return
|
||||
}
|
||||
}
|
||||
res, err := h.svc.Clean(c.Request.Context(), req.KeepTrackID, req.RemoveTrackIDs)
|
||||
if err != nil {
|
||||
writeDuplicatesError(c, err)
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"removed": res.Removed, "freed_bytes": res.FreedBytes})
|
||||
}
|
||||
|
||||
// writeDuplicatesError gives each way a removal is refused its own status and code, so a client can tell them apart.
|
||||
func writeDuplicatesError(c *gin.Context, err error) {
|
||||
switch {
|
||||
case errors.Is(err, duplicates.ErrDisabled):
|
||||
writeError(c, http.StatusConflict, "deletion_disabled", "deleting files is not turned on: set library.allow_delete")
|
||||
case errors.Is(err, duplicates.ErrScanRunning):
|
||||
writeError(c, http.StatusConflict, "scan_running", "wait for the running scan to finish")
|
||||
case errors.Is(err, duplicates.ErrInvalid):
|
||||
writeError(c, http.StatusBadRequest, "invalid_input", err.Error())
|
||||
case errors.Is(err, duplicates.ErrTrackNotFound):
|
||||
writeError(c, http.StatusNotFound, "track_not_found", "no such track")
|
||||
case errors.Is(err, duplicates.ErrChanged):
|
||||
writeError(c, http.StatusConflict, "duplicate_changed", "a file is not what the last scan saw: scan again")
|
||||
case errors.Is(err, duplicates.ErrReadOnly):
|
||||
writeError(c, http.StatusConflict, "library_read_only", "the music folder cannot be written to")
|
||||
default:
|
||||
_ = c.Error(err)
|
||||
writeError(c, http.StatusInternalServerError, "internal", "internal error")
|
||||
}
|
||||
}
|
||||
|
||||
type duplicatesCheckSummaryJSON struct {
|
||||
Checked int `json:"checked"`
|
||||
Hashed int `json:"hashed"`
|
||||
Skipped int `json:"skipped"`
|
||||
Failed int `json:"failed"`
|
||||
Stopped bool `json:"stopped"`
|
||||
}
|
||||
|
||||
// duplicatesCheckSummary is the summary of the job that fingerprints the files of the groups.
|
||||
func duplicatesCheckSummary(s *jobs.Summary) any {
|
||||
return duplicatesCheckSummaryJSON{
|
||||
Checked: s.Checked, Hashed: s.Outcomes[duplicates.OutcomeHashed], Skipped: s.Skipped, Failed: s.Failed, Stopped: s.Stopped,
|
||||
}
|
||||
}
|
||||
|
|
@ -18,6 +18,7 @@ import (
|
|||
"codeberg.org/squidd/sonda/internal/catalog"
|
||||
"codeberg.org/squidd/sonda/internal/discover"
|
||||
"codeberg.org/squidd/sonda/internal/discovery"
|
||||
"codeberg.org/squidd/sonda/internal/duplicates"
|
||||
"codeberg.org/squidd/sonda/internal/feedback"
|
||||
"codeberg.org/squidd/sonda/internal/images"
|
||||
"codeberg.org/squidd/sonda/internal/jobs"
|
||||
|
|
@ -105,6 +106,11 @@ type jobService interface {
|
|||
Status() jobs.Status
|
||||
}
|
||||
|
||||
type duplicateService interface {
|
||||
Groups(ctx context.Context, limit, offset int) (duplicates.Page, error)
|
||||
Clean(ctx context.Context, keep int64, remove []int64) (duplicates.Result, error)
|
||||
}
|
||||
|
||||
type coverService interface {
|
||||
AlbumCover(ctx context.Context, albumID int64, size images.Size) (*images.Cover, error)
|
||||
}
|
||||
|
|
@ -155,6 +161,8 @@ type Deps struct {
|
|||
CoversJob jobService
|
||||
ArtistImagesJob jobService
|
||||
DescriptionsJob jobService
|
||||
Duplicates duplicateService
|
||||
DuplicatesJob jobService
|
||||
About aboutService
|
||||
ArtistImages artistImageService
|
||||
Settings userSettingsService
|
||||
|
|
@ -274,6 +282,14 @@ func NewRouter(d Deps) *gin.Engine {
|
|||
admin.DELETE("/library/artist-images", aj.stop)
|
||||
admin.POST("/library/descriptions", dj.start)
|
||||
admin.DELETE("/library/descriptions", dj.stop)
|
||||
|
||||
dup := &duplicateHandlers{svc: d.Duplicates}
|
||||
dupJob := &jobHandlers{job: d.DuplicatesJob, runningCode: "duplicates_check_running", summary: duplicatesCheckSummary}
|
||||
admin.GET("/library/duplicates", dup.list)
|
||||
admin.POST("/library/duplicates/clean", dup.clean)
|
||||
admin.GET("/library/duplicates/check", dupJob.status)
|
||||
admin.POST("/library/duplicates/check", dupJob.start)
|
||||
admin.DELETE("/library/duplicates/check", dupJob.stop)
|
||||
libs := &librariesHandlers{svc: d.Libraries}
|
||||
admin.GET("/libraries", libs.list)
|
||||
admin.POST("/libraries", libs.create)
|
||||
|
|
|
|||
|
|
@ -715,6 +715,162 @@ paths:
|
|||
'401': {$ref: '#/components/responses/Unauthorized'}
|
||||
'403': {$ref: '#/components/responses/AdminRequired'}
|
||||
|
||||
/library/duplicates:
|
||||
get:
|
||||
tags: [Scan]
|
||||
operationId: listDuplicates
|
||||
summary: The songs that are in more than one file
|
||||
description: |
|
||||
Admin only. Groups of files that are the same song by the same artists, worked out from the index and
|
||||
never from the disk, so it is quick and touches no file. Two files are the same song when they have the
|
||||
same set of main artists, the same title (the key Sonda matches titles by) and durations no more than
|
||||
3 seconds apart, every pair in a group within that. A file with no duration or no artist never matches,
|
||||
a file flagged as missing does not count, and a copy may be on a different album: `files` says which.
|
||||
|
||||
Each group says which copy Sonda would keep (`suggested_keep_track_id`): the one without loss (FLAC)
|
||||
before one with loss, then the higher bitrate, then the bigger file, then the oldest. `hashed` is true when
|
||||
every file of the group has been read by the check (`POST /library/duplicates/check`) and `identical` when,
|
||||
besides, they are byte for byte the same. No file path is ever returned.
|
||||
|
||||
`deletion_allowed` says whether `POST /library/duplicates/clean` can work: with `library.allow_delete`
|
||||
off (the default) the groups can still be seen.
|
||||
security: [{bearerAuth: []}]
|
||||
parameters:
|
||||
- {$ref: '#/components/parameters/Limit'}
|
||||
- {$ref: '#/components/parameters/Offset'}
|
||||
responses:
|
||||
'200':
|
||||
description: A page of groups, in a fixed order (by title).
|
||||
content:
|
||||
application/json:
|
||||
schema: {$ref: '#/components/schemas/DuplicatesPage'}
|
||||
'400': {$ref: '#/components/responses/InvalidInput'}
|
||||
'401': {$ref: '#/components/responses/Unauthorized'}
|
||||
'403': {$ref: '#/components/responses/AdminRequired'}
|
||||
|
||||
/library/duplicates/check:
|
||||
get:
|
||||
tags: [Scan]
|
||||
operationId: getDuplicatesCheck
|
||||
summary: Progress of the check that reads the files of the duplicates
|
||||
description: |
|
||||
Whether the check is running, how far it is (`progress`, only while it runs), and how the last one ended.
|
||||
The check is described at `POST /library/duplicates/check`.
|
||||
security: [{bearerAuth: []}]
|
||||
responses:
|
||||
'200':
|
||||
description: The job.
|
||||
content:
|
||||
application/json:
|
||||
schema: {$ref: '#/components/schemas/DuplicatesCheckStatus'}
|
||||
'401': {$ref: '#/components/responses/Unauthorized'}
|
||||
'403': {$ref: '#/components/responses/AdminRequired'}
|
||||
post:
|
||||
tags: [Scan]
|
||||
operationId: startDuplicatesCheck
|
||||
summary: Read the files of the duplicates, to tell which are identical
|
||||
description: |
|
||||
Admin only. Starts, in the background and one at a time like a scan, a job that reads every file that is in
|
||||
a group of copies and keeps its SHA-256, so that a group of files that are byte for byte the same can be
|
||||
called `identical`. It only reads: nothing is removed or sent anywhere. **It reads whole files**, which on a
|
||||
network drive takes a while, so it is started on purpose. A file that has a fingerprint is skipped, and the
|
||||
fingerprint of a file is forgotten when a scan finds the file changed; send `refresh=true` to read every file
|
||||
again. A file that changed since the scan is skipped. It gives up, and says why in `last_error`, when files
|
||||
cannot be read five times in a row (a disk that is not mounted). `DELETE /library/duplicates/check` stops it.
|
||||
security: [{bearerAuth: []}]
|
||||
parameters:
|
||||
- name: refresh
|
||||
in: query
|
||||
description: "`true` reads every file again, also those that have a fingerprint. `false` when absent."
|
||||
schema: {type: boolean, default: false}
|
||||
responses:
|
||||
'202':
|
||||
description: The check was started in the background.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required: [status]
|
||||
properties:
|
||||
status: {type: string, enum: [started]}
|
||||
'400': {$ref: '#/components/responses/InvalidInput'}
|
||||
'401': {$ref: '#/components/responses/Unauthorized'}
|
||||
'403': {$ref: '#/components/responses/AdminRequired'}
|
||||
'409':
|
||||
description: The check is already running (`duplicates_check_running`).
|
||||
content:
|
||||
application/json:
|
||||
schema: {$ref: '#/components/schemas/Error'}
|
||||
'500':
|
||||
description: The groups could not be worked out (`internal`).
|
||||
content:
|
||||
application/json:
|
||||
schema: {$ref: '#/components/schemas/Error'}
|
||||
delete:
|
||||
tags: [Scan]
|
||||
operationId: stopDuplicatesCheck
|
||||
summary: Stop the check of the duplicates
|
||||
description: |
|
||||
Admin only. Asks a running check to end after the file it is on; the fingerprints already kept stay. It
|
||||
answers the same when no check is running.
|
||||
security: [{bearerAuth: []}]
|
||||
responses:
|
||||
'204': {description: "Done, also when no check was running."}
|
||||
'401': {$ref: '#/components/responses/Unauthorized'}
|
||||
'403': {$ref: '#/components/responses/AdminRequired'}
|
||||
|
||||
/library/duplicates/clean:
|
||||
post:
|
||||
tags: [Scan]
|
||||
operationId: cleanDuplicates
|
||||
summary: Delete the extra copies of one song
|
||||
description: |
|
||||
Admin only. **Deletes files from the music folder.** It is the one thing Sonda does that writes there, so it
|
||||
is off until the operator turns on `library.allow_delete` (answer `deletion_disabled` otherwise), the folder
|
||||
has to be writable (`library_read_only`), and it is never done by itself: one request removes the copies
|
||||
`remove_track_ids` of one song and keeps `keep_track_id`.
|
||||
|
||||
The server decides again, whatever the client saw, and removes nothing unless every check passes: no scan is
|
||||
running (`scan_running`); the tracks exist (`track_not_found`); they are copies of one song by the rule of
|
||||
`GET /library/duplicates` (`invalid_input` otherwise, and also when the copy to keep is among those to
|
||||
remove, a copy is named twice or there are none or more than 100); and the copy to keep and every copy to
|
||||
remove are still the file the last scan saw, a regular file with the same size and date (`duplicate_changed`
|
||||
for a file that changed, is gone, became a link or is outside the library: scan again). So the last copy of
|
||||
a song is never removed. Every file is checked before the first is removed.
|
||||
|
||||
Each copy is removed from the disk through the library folder (a path cannot leave it, not even through a
|
||||
link) and its track leaves the index; **the listens of the removed track become listens of the track that
|
||||
stays**, so the taste is not lost. Folders are never removed, not even empty ones. If it fails part way (for
|
||||
example the folder turns out to be read-only for the second file), what was removed stays removed.
|
||||
security: [{bearerAuth: []}]
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema: {$ref: '#/components/schemas/CleanDuplicates'}
|
||||
responses:
|
||||
'200':
|
||||
description: The extra copies were removed.
|
||||
content:
|
||||
application/json:
|
||||
schema: {$ref: '#/components/schemas/DuplicatesCleaned'}
|
||||
'400': {$ref: '#/components/responses/InvalidInput'}
|
||||
'401': {$ref: '#/components/responses/Unauthorized'}
|
||||
'403': {$ref: '#/components/responses/AdminRequired'}
|
||||
'404':
|
||||
description: A track that was named does not exist (`track_not_found`).
|
||||
content:
|
||||
application/json:
|
||||
schema: {$ref: '#/components/schemas/Error'}
|
||||
'409':
|
||||
description: |
|
||||
Deleting is not turned on (`deletion_disabled`), a scan is running (`scan_running`), a file is not what the last
|
||||
scan saw (`duplicate_changed`), or the music folder cannot be written to (`library_read_only`).
|
||||
content:
|
||||
application/json:
|
||||
schema: {$ref: '#/components/schemas/Error'}
|
||||
|
||||
/library/scan:
|
||||
get:
|
||||
tags: [Scan]
|
||||
|
|
@ -2800,6 +2956,108 @@ components:
|
|||
stopped: {type: boolean, description: "It was stopped, or the application was."}
|
||||
last_error: {type: string, description: "Why the last job ended before the end, for example that a provider is off or kept failing."}
|
||||
|
||||
DuplicatesPage:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required: [deletion_allowed, total, limit, offset, groups]
|
||||
properties:
|
||||
deletion_allowed: {type: boolean, description: "Whether `POST /library/duplicates/clean` can work (`library.allow_delete`)."}
|
||||
total: {type: integer, description: "Groups in all, not in this page."}
|
||||
limit: {type: integer}
|
||||
offset: {type: integer}
|
||||
groups:
|
||||
type: array
|
||||
items: {$ref: '#/components/schemas/DuplicateGroup'}
|
||||
|
||||
DuplicateGroup:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required: [title, artists, files, hashed, identical, suggested_keep_track_id, freed_bytes]
|
||||
properties:
|
||||
title: {type: string}
|
||||
artists:
|
||||
type: array
|
||||
description: The main artists the copies share.
|
||||
items: {$ref: '#/components/schemas/ArtistRef'}
|
||||
files:
|
||||
type: array
|
||||
description: The copies, by track id; at least two.
|
||||
items: {$ref: '#/components/schemas/DuplicateFile'}
|
||||
hashed: {type: boolean, description: "Every file of the group has been read by the check."}
|
||||
identical: {type: boolean, description: "Hashed, and byte for byte the same."}
|
||||
suggested_keep_track_id: {type: integer, format: int64, description: "The copy Sonda would keep: without loss first, then the higher bitrate, then the bigger file, then the oldest."}
|
||||
freed_bytes: {type: integer, format: int64, description: "The size of every copy but the one to keep."}
|
||||
|
||||
DuplicateFile:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required: [track_id, album, format, bitrate, size_bytes, duration_ms, hashed]
|
||||
properties:
|
||||
track_id: {type: integer, format: int64}
|
||||
album:
|
||||
type: object
|
||||
nullable: true
|
||||
additionalProperties: false
|
||||
required: [id, title]
|
||||
properties:
|
||||
id: {type: integer, format: int64}
|
||||
title: {type: string}
|
||||
format: {type: string, description: "`mp3`, `flac`..."}
|
||||
bitrate: {type: integer, description: "0 when unknown."}
|
||||
size_bytes: {type: integer, format: int64}
|
||||
duration_ms: {type: integer, format: int64}
|
||||
hashed: {type: boolean, description: "This file has been read by the check."}
|
||||
|
||||
CleanDuplicates:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required: [keep_track_id, remove_track_ids]
|
||||
properties:
|
||||
keep_track_id: {type: integer, format: int64, minimum: 1, description: "The copy that stays."}
|
||||
remove_track_ids:
|
||||
type: array
|
||||
minItems: 1
|
||||
maxItems: 100
|
||||
description: "The copies to delete, of the same song as the one that stays, none twice."
|
||||
items: {type: integer, format: int64, minimum: 1}
|
||||
|
||||
DuplicatesCleaned:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required: [removed, freed_bytes]
|
||||
properties:
|
||||
removed: {type: integer, description: "Files deleted."}
|
||||
freed_bytes: {type: integer, format: int64}
|
||||
|
||||
DuplicatesCheckStatus:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required: [running]
|
||||
properties:
|
||||
running: {type: boolean}
|
||||
started_at: {type: string, format: date-time, description: "When the last check started."}
|
||||
last_finished_at: {type: string, format: date-time}
|
||||
progress:
|
||||
type: object
|
||||
description: Only while the check runs.
|
||||
additionalProperties: false
|
||||
required: [done, total]
|
||||
properties:
|
||||
done: {type: integer}
|
||||
total: {type: integer, description: "Files the check set out to read."}
|
||||
last:
|
||||
type: object
|
||||
description: How the last finished check ended.
|
||||
additionalProperties: false
|
||||
required: [checked, hashed, skipped, failed, stopped]
|
||||
properties:
|
||||
checked: {type: integer, description: "Files looked at."}
|
||||
hashed: {type: integer, description: "Files read and fingerprinted."}
|
||||
skipped: {type: integer, description: "Files that changed since the scan, or are gone."}
|
||||
failed: {type: integer, description: "Files that could not be read just then."}
|
||||
stopped: {type: boolean, description: "It was stopped, or the application was."}
|
||||
last_error: {type: string, description: "Why the last check ended before the end, for example that the disk is not mounted."}
|
||||
|
||||
UserSettings:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
|
|
|
|||
Loading…
Reference in a new issue