From 4ede6d4118b35fe37d2aa414267cdf09c536a5ca Mon Sep 17 00:00:00 2001 From: tgiachi Date: Sat, 14 Nov 2026 09:52:57 +0100 Subject: [PATCH] feat(web): a Duplicates section in the admin area I can now see the songs that are in more than one file and delete the extra copies from Admin. - A table of the copies of each song (album, format, bitrate, size, length) with the copy to keep already chosen and changeable. - The check that reads the files, with a question first, progress, Stop, and the result of the last run; the groups are read again when it ends. - Delete the others for one song and Clean all suggestions for all of them, each after a question that lists the files and the space it frees. It stops at the first refusal and says what was already deleted. - With deleting off everything is visible and the delete buttons are disabled, with the reason. - English and Italian. --- README.md | 2 +- web/src/api/duplicates.ts | 61 +++++ web/src/i18n/locales/en.json | 60 +++++ web/src/i18n/locales/it.json | 60 +++++ web/src/lib/bytes.test.ts | 14 ++ web/src/lib/bytes.ts | 13 ++ web/src/pages/admin/AdminLayout.tsx | 1 + web/src/pages/admin/Confirm.tsx | 5 + web/src/pages/admin/DuplicateGroupCard.tsx | 90 ++++++++ web/src/pages/admin/DuplicatesCheck.tsx | 132 +++++++++++ web/src/pages/admin/DuplicatesPage.test.tsx | 242 ++++++++++++++++++++ web/src/pages/admin/DuplicatesPage.tsx | 158 +++++++++++++ web/src/pages/admin/duplicateFiles.ts | 14 ++ web/src/router.tsx | 1 + 14 files changed, 852 insertions(+), 1 deletion(-) create mode 100644 web/src/api/duplicates.ts create mode 100644 web/src/lib/bytes.test.ts create mode 100644 web/src/lib/bytes.ts create mode 100644 web/src/pages/admin/DuplicateGroupCard.tsx create mode 100644 web/src/pages/admin/DuplicatesCheck.tsx create mode 100644 web/src/pages/admin/DuplicatesPage.test.tsx create mode 100644 web/src/pages/admin/DuplicatesPage.tsx create mode 100644 web/src/pages/admin/duplicateFiles.ts diff --git a/README.md b/README.md index a58a698..c350993 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ It is a single executable, one database (SQLite by default, PostgreSQL if you pr - **Libraries.** A library is a name and a folder, managed through the API. Sonda only reads your music folders: it never writes or renames a file, and it deletes one only when an admin asks to remove the extra copy of a song and you turned on `library.allow_delete` (see **Duplicates**), which is off by default. - **Scanner.** Reads MP3 and FLAC tags and audio properties with several workers in parallel. Rescans are incremental, renamed files keep their identity, vanished files are flagged (never deleted), and a library whose folder looks empty or unreadable (an unmounted NAS) is left alone instead of being wiped. Audio in other formats (m4a, ogg, opus, wav, wma and similar) is not read yet, but it is counted as `unsupported` in the scan result instead of disappearing silently. NAS housekeeping folders (`@Recycle`, `#recycle`, `@eaDir`, `.@__thumb`, `$RECYCLE.BIN`) are skipped. - **Catalog.** Browse and search artists, albums and tracks, with paging, by library, artist or album. Only tracks you can actually play are listed. -- **Duplicates.** `GET /library/duplicates` (admin) lists the songs that are in more than one file: the same set of main artists, the same title and durations within 3 seconds, possibly on different albums. It is worked out from the index, touches no file, and returns no file path. Each group proposes the copy to keep (lossless first, then the higher bitrate, then the bigger file, then the oldest). A background job (`POST /library/duplicates/check`, like the other bulk jobs) reads the files of the groups and keeps their SHA-256, so a group can be called `identical` (byte for byte). `POST /library/duplicates/clean` **deletes the extra copies of one song from the disk**, and it is the one thing Sonda does that writes in your music folders, so it is off until you set `library.allow_delete` (or `SONDA_ALLOW_DELETE=true`) and mount the folders writable. It deletes only when an admin asks, never by itself, and the server checks again at that moment that no scan is running, that the tracks are copies of one song, and that every file is still the one the last scan saw (same size and date, a regular file, not a link, inside its library): so the last copy of a song is never deleted, a file that changed is not deleted, and every file is checked before the first is deleted. The listens and the playlist items of a deleted copy move to the one that stays, so your taste and your playlists are not lost. Folders are never removed, and nothing is sent anywhere. +- **Duplicates.** `GET /library/duplicates` (admin) lists the songs that are in more than one file: the same set of main artists, the same title and durations within 3 seconds, possibly on different albums. It is worked out from the index, touches no file, and returns no file path. Each group proposes the copy to keep (lossless first, then the higher bitrate, then the bigger file, then the oldest). A background job (`POST /library/duplicates/check`, like the other bulk jobs) reads the files of the groups and keeps their SHA-256, so a group can be called `identical` (byte for byte). `POST /library/duplicates/clean` **deletes the extra copies of one song from the disk**, and it is the one thing Sonda does that writes in your music folders, so it is off until you set `library.allow_delete` (or `SONDA_ALLOW_DELETE=true`) and mount the folders writable. It deletes only when an admin asks, never by itself, and the server checks again at that moment that no scan is running, that the tracks are copies of one song, and that every file is still the one the last scan saw (same size and date, a regular file, not a link, inside its library): so the last copy of a song is never deleted, a file that changed is not deleted, and every file is checked before the first is deleted. The listens and the playlist items of a deleted copy move to the one that stays, so your taste and your playlists are not lost. Folders are never removed, and nothing is sent anywhere. The web app has a Duplicates section in Admin: a table of the copies of each song with the copy to keep already chosen (you can pick another), the check with its progress and a Stop button, **Delete the others** for one song and **Clean all suggestions** for all of them, each after a question that lists the files and the space it frees; with deleting off everything is visible and the delete buttons are disabled. - **Streaming.** `GET` and `HEAD` with `Range` support, so players can seek. Opening or prefetching a stream does not record a listen. `GET /tracks/{id}/peaks` gives the shape of a track for the seek bar: 120 levels from 0 to 255, how loud it is along its length. The server finds it by decoding the file (MP3 and FLAC, in pure Go: nothing else to install), which takes a few seconds, so the first request starts the work in the background and answers `202` with `Retry-After`; it is done once for each file, worked out again if the size or the date of the file change, and one track at a time, so a NAS is not kept busy. A file that cannot be decoded answers `waveform_unavailable`, and that is remembered too. A friend's track has no shape yet. - **API documentation.** The full API is described in OpenAPI and browsable with Swagger UI, served by Sonda itself. The UI is embedded in the binary, so opening it contacts no other server. - **Listening history.** Clients report explicit `play`, `complete` and `skip` events. An event counts as a listen when it is `complete`, or when it was played for at least 20 seconds (`listening.min_listen_seconds`); the rule is applied when the history is read, so changing it re-reads the past. You can read your own history, with a `counted` flag on every event. It stays in your database and is never sent anywhere. diff --git a/web/src/api/duplicates.ts b/web/src/api/duplicates.ts new file mode 100644 index 0000000..023af14 --- /dev/null +++ b/web/src/api/duplicates.ts @@ -0,0 +1,61 @@ +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import { api } from './client' +import type { components } from './schema' + +export type DuplicatesPage = components['schemas']['DuplicatesPage'] +export type DuplicateGroup = components['schemas']['DuplicateGroup'] +export type DuplicateFile = components['schemas']['DuplicateFile'] +export type DuplicatesCheckStatus = components['schemas']['DuplicatesCheckStatus'] +export type CleanDuplicates = components['schemas']['CleanDuplicates'] +export type DuplicatesCleaned = components['schemas']['DuplicatesCleaned'] + +/** Groups read at once. The server gives at most 200 in a page. */ +export const GROUPS_LIMIT = 200 + +const listKey = ['library', 'duplicates'] +const checkKey = ['library', 'duplicates-check'] + +/** The groups of copies (a page of them), read from the index: quick, touches no file. */ +export function useDuplicates() { + return useQuery({ + queryKey: listKey, + queryFn: () => api.get(`/library/duplicates?limit=${GROUPS_LIMIT}`), + }) +} + +/** Whether the check runs, how far it is, and how the last one ended. While it runs it is read again every `intervalMs`. */ +export function useDuplicatesCheck({ intervalMs = 2000 }: { intervalMs?: number } = {}) { + return useQuery({ + queryKey: checkKey, + queryFn: () => api.get('/library/duplicates/check'), + refetchInterval: (query) => (query.state.data?.running ? intervalMs : false), + }) +} + +/** Starts the check in the background; the groups are read again when it ends (see `useRefreshWhenDone`). */ +export function useStartDuplicatesCheck() { + const client = useQueryClient() + return useMutation({ + mutationFn: ({ refresh }: { refresh: boolean }) => + api.post<{ status: 'started' }>(refresh ? '/library/duplicates/check?refresh=true' : '/library/duplicates/check'), + onSettled: () => client.invalidateQueries({ queryKey: checkKey }), + }) +} + +/** Asks the check to end after the file it is on. */ +export function useStopDuplicatesCheck() { + const client = useQueryClient() + return useMutation({ + mutationFn: () => api.del('/library/duplicates/check'), + onSettled: () => client.invalidateQueries({ queryKey: checkKey }), + }) +} + +/** Deletes the extra copies of one song. Whatever the answer, the groups are read again. */ +export function useCleanDuplicates() { + const client = useQueryClient() + return useMutation({ + mutationFn: (body: CleanDuplicates) => api.post('/library/duplicates/clean', body), + onSettled: () => client.invalidateQueries({ queryKey: listKey }), + }) +} diff --git a/web/src/i18n/locales/en.json b/web/src/i18n/locales/en.json index ce0e689..5fc052f 100644 --- a/web/src/i18n/locales/en.json +++ b/web/src/i18n/locales/en.json @@ -109,6 +109,7 @@ "sectionFederation": "Federation", "sectionCosts": "Costs", "sectionJobs": "Jobs", + "sectionDuplicates": "Duplicates", "costs": { "intro": "Every question that Sonda asks of Claude is counted here: the names it made, the ones taken from the names it kept (which cost nothing), the ones that failed, and the tokens that were billed. Nothing of what was sent is kept.", "period": "Period", @@ -186,6 +187,65 @@ } } }, + "duplicates": { + "intro": "These are songs that are in more than one file: the same artists, the same title and almost the same length. Looking is free and reads only the index. Deleting is a different thing: it removes files from your music folder.", + "checkTitle": "Check the files", + "checkWhat": "Reads every file that has a copy and keeps its fingerprint, so a group of files that are byte for byte the same is called identical. It only reads: nothing is deleted or sent anywhere.", + "checkAgain": "Read every file again", + "checkStart": "Check the files", + "checkStop": "Stop the check", + "checkNever": "The check has not run yet.", + "checkConfirmTitle": "Check the files?", + "checkConfirmBody": "This reads whole files, which takes a while on a network drive. Nothing is deleted or sent anywhere.", + "checkConfirmAgain": "It reads again also the files that already have a fingerprint.", + "checkConfirm": "Start the check", + "counts": { + "checked": "{{count}} looked at", + "hashed": "{{count}} read", + "skipped": "{{count}} skipped", + "failed": "{{count}} could not be read" + }, + "groupsTitle": "Copies", + "deletionOff": "Deleting is turned off on this server, so you can look but not delete. To allow it, the person who runs the server sets library.allow_delete.", + "none": "No song is in more than one file.", + "summary_one": "{{count}} song is in more than one file. Keeping the chosen copies frees {{size}}.", + "summary_other": "{{count}} songs are in more than one file. Keeping the chosen copies frees {{size}}.", + "partial": "Showing {{shown}} of {{total}} groups.", + "cleanAll": "Clean all suggestions", + "keep": "Keep", + "album": "Album", + "format": "Format", + "bitrate": "Bitrate", + "size": "Size", + "duration": "Length", + "noAlbum": "No album", + "bitrateUnknown": "unknown bitrate", + "keepCopy": "Keep this copy: {{copy}}", + "deleteOthers": "Delete the others", + "deleteOthersOne": "Delete the other copies of {{title}}", + "frees": "Frees {{size}}", + "identical": "Byte for byte the same", + "different": "Read by the check: the files differ", + "notCompared": "Not compared yet", + "confirmOneTitle": "Delete the other copies?", + "confirmAllTitle_one": "Clean {{count}} song?", + "confirmAllTitle_other": "Clean {{count}} songs?", + "confirmBody": "These files are deleted from your music folder and cannot be brought back. It frees {{size}}. The plays of a deleted copy move to the copy that stays.", + "confirmList": "Files", + "willKeep": "Keep:", + "willDelete": "Delete:", + "confirm": "Delete files", + "cleaned_one": "{{count}} file deleted, {{size}} freed.", + "cleaned_other": "{{count}} files deleted, {{size}} freed.", + "errors": { + "deletion_disabled": "Deleting is turned off on this server.", + "scan_running": "A scan is running. Try again when it ends.", + "duplicate_changed": "A file is not what the last scan saw. Scan the library again, then try again.", + "library_read_only": "The music folder cannot be written to, so nothing was deleted.", + "track_not_found": "One of the songs is gone. The list is read again.", + "duplicates_check_running": "The check is already running." + } + }, "cancel": "Cancel", "save": "Save", "errors": { diff --git a/web/src/i18n/locales/it.json b/web/src/i18n/locales/it.json index 4f408c3..4d3fa6d 100644 --- a/web/src/i18n/locales/it.json +++ b/web/src/i18n/locales/it.json @@ -109,6 +109,7 @@ "sectionFederation": "Federazione", "sectionCosts": "Costi", "sectionJobs": "Lavori", + "sectionDuplicates": "Duplicati", "costs": { "intro": "Qui si contano tutte le domande che Sonda fa a Claude: i nomi fatti, quelli presi dai nomi già salvati (che non costano niente), quelli falliti e i token fatturati. Di quello che è stato inviato non si conserva niente.", "period": "Periodo", @@ -186,6 +187,65 @@ } } }, + "duplicates": { + "intro": "Sono canzoni che stanno in più di un file: stessi artisti, stesso titolo e durata quasi uguale. Guardare non costa niente e legge solo l'indice. Cancellare è un'altra cosa: toglie file dalla tua cartella musicale.", + "checkTitle": "Controlla i file", + "checkWhat": "Legge ogni file che ha una copia e ne tiene l'impronta, così un gruppo di file identici byte per byte si dice identico. Legge soltanto: non cancella e non invia niente.", + "checkAgain": "Rileggi tutti i file", + "checkStart": "Controlla i file", + "checkStop": "Ferma il controllo", + "checkNever": "Il controllo non è ancora partito.", + "checkConfirmTitle": "Controllare i file?", + "checkConfirmBody": "Legge file interi, e su un disco di rete ci vuole un po'. Non cancella e non invia niente.", + "checkConfirmAgain": "Rilegge anche i file che hanno già un'impronta.", + "checkConfirm": "Avvia il controllo", + "counts": { + "checked": "{{count}} guardati", + "hashed": "{{count}} letti", + "skipped": "{{count}} saltati", + "failed": "{{count}} non leggibili" + }, + "groupsTitle": "Copie", + "deletionOff": "La cancellazione è spenta su questo server: puoi guardare ma non cancellare. Per permetterla, chi gestisce il server imposta library.allow_delete.", + "none": "Nessuna canzone sta in più di un file.", + "summary_one": "{{count}} canzone sta in più di un file. Tenendo le copie scelte si liberano {{size}}.", + "summary_other": "{{count}} canzoni stanno in più di un file. Tenendo le copie scelte si liberano {{size}}.", + "partial": "Mostro {{shown}} gruppi su {{total}}.", + "cleanAll": "Pulisci tutti i suggerimenti", + "keep": "Tieni", + "album": "Album", + "format": "Formato", + "bitrate": "Bitrate", + "size": "Dimensione", + "duration": "Durata", + "noAlbum": "Nessun album", + "bitrateUnknown": "bitrate sconosciuto", + "keepCopy": "Tieni questa copia: {{copy}}", + "deleteOthers": "Cancella le altre", + "deleteOthersOne": "Cancella le altre copie di {{title}}", + "frees": "Libera {{size}}", + "identical": "Identici byte per byte", + "different": "Letti dal controllo: i file sono diversi", + "notCompared": "Non ancora confrontati", + "confirmOneTitle": "Cancellare le altre copie?", + "confirmAllTitle_one": "Pulire {{count}} canzone?", + "confirmAllTitle_other": "Pulire {{count}} canzoni?", + "confirmBody": "Questi file vengono cancellati dalla tua cartella musicale e non si recuperano. Liberi {{size}}. Gli ascolti di una copia cancellata passano alla copia che resta.", + "confirmList": "File", + "willKeep": "Tieni:", + "willDelete": "Cancella:", + "confirm": "Cancella i file", + "cleaned_one": "{{count}} file cancellato, {{size}} liberati.", + "cleaned_other": "{{count}} file cancellati, {{size}} liberati.", + "errors": { + "deletion_disabled": "La cancellazione è spenta su questo server.", + "scan_running": "Una scansione è in corso. Riprova quando finisce.", + "duplicate_changed": "Un file non è più quello visto dall'ultima scansione. Scansiona di nuovo la libreria, poi riprova.", + "library_read_only": "La cartella musicale non è scrivibile, quindi non è stato cancellato niente.", + "track_not_found": "Una delle canzoni non c'è più. L'elenco viene riletto.", + "duplicates_check_running": "Il controllo è già in corso." + } + }, "cancel": "Annulla", "save": "Salva", "errors": { diff --git a/web/src/lib/bytes.test.ts b/web/src/lib/bytes.test.ts new file mode 100644 index 0000000..9a58fba --- /dev/null +++ b/web/src/lib/bytes.test.ts @@ -0,0 +1,14 @@ +import { expect, test } from 'vitest' +import { formatBytes } from './bytes' + +test('a size is written with the unit that keeps it short', () => { + expect(formatBytes(0)).toBe('0 B') + expect(formatBytes(1023)).toBe('1023 B') + expect(formatBytes(2048)).toBe('2 KB') + expect(formatBytes(4_404_019)).toBe('4.2 MB') + expect(formatBytes(1_395_864_371)).toBe('1.3 GB') +}) + +test('a size below zero is shown as zero', () => { + expect(formatBytes(-5)).toBe('0 B') +}) diff --git a/web/src/lib/bytes.ts b/web/src/lib/bytes.ts new file mode 100644 index 0000000..9e9afd0 --- /dev/null +++ b/web/src/lib/bytes.ts @@ -0,0 +1,13 @@ +const UNITS = ['B', 'KB', 'MB', 'GB', 'TB'] + +/** A size in bytes as a short text: `0 B`, `812 KB`, `4.2 MB`, `1.3 GB` (steps of 1024, one decimal from MB up). */ +export function formatBytes(bytes: number): string { + let value = Math.max(0, bytes) + let unit = 0 + while (value >= 1024 && unit < UNITS.length - 1) { + value /= 1024 + unit++ + } + const text = unit < 2 ? String(Math.round(value)) : value.toFixed(1) + return `${text} ${UNITS[unit]}` +} diff --git a/web/src/pages/admin/AdminLayout.tsx b/web/src/pages/admin/AdminLayout.tsx index 075ab6c..35e984f 100644 --- a/web/src/pages/admin/AdminLayout.tsx +++ b/web/src/pages/admin/AdminLayout.tsx @@ -12,6 +12,7 @@ const SECTIONS = [ { to: '/admin/providers', key: 'admin.sectionProviders' }, { to: '/admin/federation', key: 'admin.sectionFederation' }, { to: '/admin/jobs', key: 'admin.sectionJobs' }, + { to: '/admin/duplicates', key: 'admin.sectionDuplicates' }, { to: '/admin/costs', key: 'admin.sectionCosts' }, ] diff --git a/web/src/pages/admin/Confirm.tsx b/web/src/pages/admin/Confirm.tsx index 8737c4c..e7967c7 100644 --- a/web/src/pages/admin/Confirm.tsx +++ b/web/src/pages/admin/Confirm.tsx @@ -1,3 +1,4 @@ +import type { ReactNode } from 'react' import { useTranslation } from 'react-i18next' import { Button } from '@/components/ui/button' import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' @@ -7,12 +8,15 @@ export default function Confirm({ title, body, confirm, + children, onConfirm, onClose, }: { title: string body: string confirm: string + /** More to read before answering, such as the list of what goes away. */ + children?: ReactNode onConfirm: () => void onClose: () => void }) { @@ -24,6 +28,7 @@ export default function Confirm({ {title} {body} + {children} + {t('admin.duplicates.frees', { size: formatBytes(freed) })} + + + + ) +} diff --git a/web/src/pages/admin/DuplicatesCheck.tsx b/web/src/pages/admin/DuplicatesCheck.tsx new file mode 100644 index 0000000..2175f32 --- /dev/null +++ b/web/src/pages/admin/DuplicatesCheck.tsx @@ -0,0 +1,132 @@ +import { useEffect, useRef, useState } from 'react' +import { useTranslation } from 'react-i18next' +import { useQueryClient } from '@tanstack/react-query' +import { useDuplicatesCheck, useStartDuplicatesCheck, useStopDuplicatesCheck } from '@/api/duplicates' +import { ErrorMessage, Loading } from '@/components/Status' +import { Button } from '@/components/ui/button' +import Confirm from './Confirm' +import { errorText } from './errorText' + +const SCOPES = ['admin.duplicates.errors', 'admin.errors'] + +/** + * The check that reads every file of the copies, to tell which are byte for byte the same: what it does, + * start (after a question, because it reads whole files), progress, stop and how the last one ended. When a + * run ends the groups are read again, so `identical` shows up without a reload. + */ +export default function DuplicatesCheck() { + const { t, i18n } = useTranslation() + const client = useQueryClient() + const status = useDuplicatesCheck() + const start = useStartDuplicatesCheck() + const stop = useStopDuplicatesCheck() + const [asking, setAsking] = useState(false) + const [refresh, setRefresh] = useState(false) + const running = status.data?.running === true + const progress = status.data?.progress + const last = status.data?.last + const failure = start.error ?? stop.error + + // A run that ended: the groups know more now. + const wasRunning = useRef(false) + useEffect(() => { + if (wasRunning.current && !running) void client.invalidateQueries({ queryKey: ['library', 'duplicates'] }) + wasRunning.current = running + }, [running, client]) + + const rows = last + ? (['checked', 'hashed', 'skipped', 'failed'] as const).filter((key) => key === 'checked' || last[key] > 0) + : [] + + return ( +
+

{t('admin.duplicates.checkTitle')}

+

{t('admin.duplicates.checkWhat')}

+ + + +
+ + {running && ( + + )} +
+ + {failure ? ( +

+ {errorText(t, failure, SCOPES)} +

+ ) : null} + + {status.isError ? ( + void status.refetch()} /> + ) : status.isPending ? ( + + ) : running ? ( + progress && ( +
+
+
0 ? Math.min(100, (progress.done / progress.total) * 100) : 0}%` }} /> +
+

+ {t('admin.jobs.progress', { done: progress.done, total: progress.total })} +

+
+ ) + ) : ( + <> + {last && ( +
+
    + {rows.map((key) => ( +
  • + {t(`admin.duplicates.counts.${key}`, { count: last[key] })} +
  • + ))} +
+ {last.stopped &&

{t('admin.jobs.stopped')}

} + {status.data.last_finished_at && ( +

+ {t('admin.jobs.finished', { when: new Date(status.data.last_finished_at).toLocaleString(i18n.language) })} +

+ )} +
+ )} + {status.data.last_error && ( +

+ {status.data.last_error} +

+ )} + {!last && !status.data.last_error &&

{t('admin.duplicates.checkNever')}

} + + )} + + {asking && ( + setAsking(false)} + onConfirm={() => { + start.mutate({ refresh }) + setAsking(false) + }} + /> + )} +
+ ) +} diff --git a/web/src/pages/admin/DuplicatesPage.test.tsx b/web/src/pages/admin/DuplicatesPage.test.tsx new file mode 100644 index 0000000..80db7fa --- /dev/null +++ b/web/src/pages/admin/DuplicatesPage.test.tsx @@ -0,0 +1,242 @@ +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { render, screen, waitFor, within } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { beforeEach, expect, test, vi } from 'vitest' +import { sessionStore } from '@/auth/session' +import { initI18n } from '@/i18n' +import { apiError, mockFetch, type MockRoutes } from '@/test/mockFetch' +import DuplicatesPage from './DuplicatesPage' + +const file = (track_id: number, format: string, bitrate: number, size_bytes: number, album = 'OK Computer') => ({ + track_id, + album: { id: 1, title: album }, + format, + bitrate, + size_bytes, + duration_ms: 284_000, + hashed: false, +}) + +const airbag = { + title: 'Airbag', + artists: [{ id: 1, name: 'Radiohead' }], + files: [file(1, 'mp3', 192, 5_000_000), file(2, 'flac', 0, 30_000_000)], + hashed: false, + identical: false, + suggested_keep_track_id: 2, + freed_bytes: 5_000_000, +} +const karma = { + title: 'Karma Police', + artists: [{ id: 1, name: 'Radiohead' }], + files: [file(3, 'mp3', 320, 9_000_000), file(4, 'mp3', 320, 9_000_000, 'Best Of')], + hashed: true, + identical: true, + suggested_keep_track_id: 3, + freed_bytes: 9_000_000, +} + +const idle = { running: false } + +function routes(over: MockRoutes = {}, groups = [airbag, karma], allowed = true): MockRoutes { + return { + 'GET /api/v1/library/duplicates': () => ({ deletion_allowed: allowed, total: groups.length, limit: 200, offset: 0, groups }), + 'GET /api/v1/library/duplicates/check': () => idle, + ...over, + } +} + +function renderPage() { + const client = new QueryClient({ defaultOptions: { queries: { retry: false }, mutations: { retry: false } } }) + return render( + + + , + ) +} + +const noContent = () => new Response(null, { status: 204 }) + +beforeEach(async () => { + vi.spyOn(navigator, 'language', 'get').mockReturnValue('en-US') + await initI18n() + sessionStore.setState({ status: 'authenticated', user: { id: 1, email: 'tom@example.org', is_admin: true } }) +}) + +test('each song is a group with a table of its copies, and the copy Sonda suggests is the one kept', async () => { + mockFetch(routes()) + renderPage() + const group = await screen.findByRole('region', { name: 'Airbag' }) + expect(within(group).getByText('Radiohead')).toBeInTheDocument() + const rows = within(group).getAllByRole('row').slice(1) + expect(rows).toHaveLength(2) + expect(within(rows[0]).getByText('192 kbps')).toBeInTheDocument() + expect(within(rows[1]).getByText('flac')).toBeInTheDocument() + expect(within(rows[1]).getByText('28.6 MB')).toBeInTheDocument() + expect(within(rows[1]).getByRole('radio')).toBeChecked() + expect(within(rows[0]).getByRole('radio')).not.toBeChecked() + expect(within(group).getByText('Not compared yet')).toBeInTheDocument() + expect(within(await screen.findByRole('region', { name: 'Karma Police' })).getByText('Byte for byte the same')).toBeInTheDocument() +}) + +test('with no duplicates the page says so', async () => { + mockFetch(routes({}, [])) + renderPage() + expect(await screen.findByText('No song is in more than one file.')).toBeInTheDocument() + expect(screen.queryByRole('button', { name: 'Clean all suggestions' })).not.toBeInTheDocument() +}) + +test('with deleting off everything is visible and the delete buttons are disabled, with the reason', async () => { + mockFetch(routes({}, [airbag, karma], false)) + renderPage() + const group = await screen.findByRole('region', { name: 'Airbag' }) + expect(screen.getByText(/Deleting is turned off on this server/)).toBeInTheDocument() + expect(within(group).getByRole('button', { name: 'Delete the other copies of Airbag' })).toBeDisabled() + expect(screen.getByRole('button', { name: 'Clean all suggestions' })).toBeDisabled() +}) + +test('the copy to keep can be changed, and what is deleted follows', async () => { + const { calls } = mockFetch( + routes({ 'POST /api/v1/library/duplicates/clean': () => ({ removed: 1, freed_bytes: 30_000_000 }) }), + ) + renderPage() + const group = await screen.findByRole('region', { name: 'Airbag' }) + await userEvent.click(within(group).getByRole('radio', { name: /mp3, 192 kbps/ })) + expect(within(group).getByText('Frees 28.6 MB')).toBeInTheDocument() + await userEvent.click(within(group).getByRole('button', { name: 'Delete the other copies of Airbag' })) + const dialog = await screen.findByRole('dialog') + expect(within(dialog).getByText(/Keep:.*mp3, 192 kbps/)).toBeInTheDocument() + expect(within(dialog).getByText(/Delete:.*flac/)).toBeInTheDocument() + await userEvent.click(within(dialog).getByRole('button', { name: 'Delete files' })) + await waitFor(() => expect(calls.some((c) => c.key === 'POST /api/v1/library/duplicates/clean')).toBe(true)) + const sent = JSON.parse(calls.find((c) => c.key === 'POST /api/v1/library/duplicates/clean')!.init.body as string) + expect(sent).toEqual({ keep_track_id: 1, remove_track_ids: [2] }) + expect(await screen.findByText('1 file deleted, 28.6 MB freed.')).toBeInTheDocument() +}) + +test('deleting asks first, and Cancel sends nothing', async () => { + const { calls } = mockFetch(routes()) + renderPage() + const group = await screen.findByRole('region', { name: 'Karma Police' }) + await userEvent.click(within(group).getByRole('button', { name: 'Delete the other copies of Karma Police' })) + await userEvent.click(within(await screen.findByRole('dialog')).getByRole('button', { name: 'Cancel' })) + expect(calls.filter((c) => c.key === 'POST /api/v1/library/duplicates/clean')).toHaveLength(0) +}) + +test('Clean all suggestions lists everything and the space freed, then removes group after group', async () => { + const { calls } = mockFetch( + routes({ 'POST /api/v1/library/duplicates/clean': () => ({ removed: 1, freed_bytes: 1_000_000 }) }), + ) + renderPage() + await screen.findByRole('region', { name: 'Airbag' }) + expect(screen.getByText(/2 songs are in more than one file. Keeping the chosen copies frees 13.4 MB./)).toBeInTheDocument() + await userEvent.click(screen.getByRole('button', { name: 'Clean all suggestions' })) + const dialog = await screen.findByRole('dialog') + expect(within(dialog).getByText('Clean 2 songs?')).toBeInTheDocument() + expect(within(dialog).getByText(/It frees 13.4 MB/)).toBeInTheDocument() + const list = within(dialog).getByRole('list', { name: 'Files' }) + expect(within(list).getByText('Airbag')).toBeInTheDocument() + expect(within(list).getByText('Karma Police')).toBeInTheDocument() + await userEvent.click(within(dialog).getByRole('button', { name: 'Delete files' })) + await screen.findByText('2 files deleted, 1.9 MB freed.') + const sent = calls.filter((c) => c.key === 'POST /api/v1/library/duplicates/clean').map((c) => JSON.parse(c.init.body as string)) + expect(sent).toEqual([ + { keep_track_id: 2, remove_track_ids: [1] }, + { keep_track_id: 3, remove_track_ids: [4] }, + ]) +}) + +test('when a file is not what the last scan saw, it says so, and keeps what was done', async () => { + let n = 0 + mockFetch( + routes({ + 'POST /api/v1/library/duplicates/clean': () => (++n === 1 ? { removed: 1, freed_bytes: 5_000_000 } : apiError(409, 'duplicate_changed')), + }), + ) + renderPage() + await screen.findByRole('region', { name: 'Airbag' }) + await userEvent.click(screen.getByRole('button', { name: 'Clean all suggestions' })) + await userEvent.click(within(await screen.findByRole('dialog')).getByRole('button', { name: 'Delete files' })) + expect(await screen.findByRole('alert')).toHaveTextContent('A file is not what the last scan saw. Scan the library again, then try again.') + expect(screen.getByText('1 file deleted, 4.8 MB freed.')).toBeInTheDocument() +}) + +test('when there are more groups than one page holds, it says how many are shown', async () => { + mockFetch({ + 'GET /api/v1/library/duplicates': () => ({ deletion_allowed: true, total: 350, limit: 200, offset: 0, groups: [airbag] }), + 'GET /api/v1/library/duplicates/check': () => idle, + }) + renderPage() + expect(await screen.findByText('Showing 1 of 350 groups.')).toBeInTheDocument() +}) + +// --- the check ------------------------------------------------------------------------------------------------- + +test('the check is started after a question, and says it reads whole files', async () => { + const { calls } = mockFetch(routes({ 'POST /api/v1/library/duplicates/check': () => new Response(JSON.stringify({ status: 'started' }), { status: 202 }) })) + renderPage() + await screen.findByRole('region', { name: 'Airbag' }) + expect(screen.getByText('The check has not run yet.')).toBeInTheDocument() + await userEvent.click(screen.getByRole('button', { name: 'Check the files' })) + const dialog = await screen.findByRole('dialog') + expect(dialog).toHaveTextContent('This reads whole files') + await userEvent.click(within(dialog).getByRole('button', { name: 'Start the check' })) + await waitFor(() => expect(calls.some((c) => c.key === 'POST /api/v1/library/duplicates/check')).toBe(true)) + expect(calls.find((c) => c.key === 'POST /api/v1/library/duplicates/check')!.url.search).toBe('') +}) + +test('"Read every file again" starts the check with refresh', async () => { + const { calls } = mockFetch(routes({ 'POST /api/v1/library/duplicates/check': () => new Response(JSON.stringify({ status: 'started' }), { status: 202 }) })) + renderPage() + await screen.findByRole('region', { name: 'Airbag' }) + await userEvent.click(screen.getByRole('checkbox', { name: 'Read every file again' })) + await userEvent.click(screen.getByRole('button', { name: 'Check the files' })) + await userEvent.click(within(await screen.findByRole('dialog')).getByRole('button', { name: 'Start the check' })) + await waitFor(() => expect(calls.some((c) => c.key === 'POST /api/v1/library/duplicates/check')).toBe(true)) + expect(calls.find((c) => c.key === 'POST /api/v1/library/duplicates/check')!.url.search).toBe('?refresh=true') +}) + +test('while the check runs it shows how far it is and can be stopped', async () => { + const { calls } = mockFetch( + routes({ + 'GET /api/v1/library/duplicates/check': () => ({ running: true, progress: { done: 3, total: 10 } }), + 'DELETE /api/v1/library/duplicates/check': noContent, + }), + ) + renderPage() + const bar = await screen.findByRole('progressbar') + expect(bar).toHaveAttribute('aria-valuenow', '3') + expect(bar).toHaveAttribute('aria-valuemax', '10') + expect(screen.getByRole('button', { name: 'Check the files' })).toBeDisabled() + await userEvent.click(screen.getByRole('button', { name: 'Stop the check' })) + await waitFor(() => expect(calls.some((c) => c.key === 'DELETE /api/v1/library/duplicates/check')).toBe(true)) +}) + +test('the last check is summed up, and a reason it gave up is shown', async () => { + mockFetch( + routes({ + 'GET /api/v1/library/duplicates/check': () => ({ + running: false, + last: { checked: 12, hashed: 10, skipped: 1, failed: 0, stopped: false }, + last_finished_at: '2026-11-14T10:00:00Z', + last_error: 'the disk is not mounted', + }), + }), + ) + renderPage() + const list = await screen.findByRole('list', { name: 'Last run' }) + expect(within(list).getByText('12 looked at')).toBeInTheDocument() + expect(within(list).getByText('10 read')).toBeInTheDocument() + expect(within(list).getByText('1 skipped')).toBeInTheDocument() + expect(within(list).queryByText(/could not be read/)).not.toBeInTheDocument() + expect(screen.getByText('the disk is not mounted')).toBeInTheDocument() +}) + +test('a check already running is explained when the server says so', async () => { + mockFetch(routes({ 'POST /api/v1/library/duplicates/check': () => apiError(409, 'duplicates_check_running') })) + renderPage() + await screen.findByRole('region', { name: 'Airbag' }) + await userEvent.click(screen.getByRole('button', { name: 'Check the files' })) + await userEvent.click(within(await screen.findByRole('dialog')).getByRole('button', { name: 'Start the check' })) + expect(await screen.findByRole('alert')).toHaveTextContent('The check is already running.') +}) diff --git a/web/src/pages/admin/DuplicatesPage.tsx b/web/src/pages/admin/DuplicatesPage.tsx new file mode 100644 index 0000000..c2f8ea5 --- /dev/null +++ b/web/src/pages/admin/DuplicatesPage.tsx @@ -0,0 +1,158 @@ +import { useState } from 'react' +import { useTranslation } from 'react-i18next' +import { useCleanDuplicates, useDuplicates, type DuplicateGroup } from '@/api/duplicates' +import Section from '@/components/Section' +import { EmptyMessage, ErrorMessage, Loading } from '@/components/Status' +import { Button } from '@/components/ui/button' +import { formatBytes } from '@/lib/bytes' +import Confirm from './Confirm' +import DuplicateGroupCard from './DuplicateGroupCard' +import { describeFile, removals } from './duplicateFiles' +import DuplicatesCheck from './DuplicatesCheck' +import { errorText } from './errorText' + +const SCOPES = ['admin.duplicates.errors', 'admin.errors'] + +/** What one request asks the server for, and what it would free. */ +interface Plan { + group: DuplicateGroup + keep: number +} + +/** The songs that are in more than one file, the check that tells which copies are identical, and the cleaning. */ +export default function DuplicatesPage() { + const { t } = useTranslation() + const list = useDuplicates() + const clean = useCleanDuplicates() + // The copy to keep, by group (a group is known by the copy Sonda suggests); a group not in here keeps the suggestion. + const [chosen, setChosen] = useState>({}) + const [asking, setAsking] = useState(null) + const [working, setWorking] = useState(false) + const [outcome, setOutcome] = useState<{ removed: number; freed: number } | null>(null) + const [failure, setFailure] = useState(null) + + const page = list.data + const groups = page?.groups ?? [] + const allowed = page?.deletion_allowed === true + const keepOf = (group: DuplicateGroup) => chosen[group.suggested_keep_track_id] ?? group.suggested_keep_track_id + const plans: Plan[] = groups.map((group) => ({ group, keep: keepOf(group) })) + const freedBy = (list: Plan[]) => list.reduce((sum, p) => sum + removals(p.group, p.keep).reduce((s, f) => s + f.size_bytes, 0), 0) + + /** One request for each group, in order. It stops at the first refusal; what was removed stays removed. */ + async function run(list: Plan[]) { + setAsking(null) + setWorking(true) + setOutcome(null) + setFailure(null) + let removed = 0 + let freed = 0 + try { + for (const { group, keep } of list) { + const done = await clean.mutateAsync({ + keep_track_id: keep, + remove_track_ids: removals(group, keep).map((f) => f.track_id), + }) + removed += done.removed + freed += done.freed_bytes + } + } catch (e) { + setFailure(e) + } finally { + setOutcome({ removed, freed }) + setWorking(false) + } + } + + return ( +
+

{t('admin.duplicates.intro')}

+ + +
+

{t('admin.duplicates.groupsTitle')}

+ + {list.isError ? ( + void list.refetch()} /> + ) : list.isPending ? ( + + ) : ( + <> + {!allowed && ( +

+ {t('admin.duplicates.deletionOff')} +

+ )} + + {outcome && ( +

+ {t('admin.duplicates.cleaned', { count: outcome.removed, size: formatBytes(outcome.freed) })} +

+ )} + {failure ? ( +

+ {errorText(t, failure, SCOPES)} +

+ ) : null} + + {groups.length === 0 ? ( + {t('admin.duplicates.none')} + ) : ( + <> +
+

+ {t('admin.duplicates.summary', { count: page!.total, size: formatBytes(freedBy(plans)) })} +

+ +
+ {page!.total > groups.length && ( +

{t('admin.duplicates.partial', { shown: groups.length, total: page!.total })}

+ )} +
+ {groups.map((group) => ( + setChosen((c) => ({ ...c, [group.suggested_keep_track_id]: id }))} + canDelete={allowed} + busy={working} + onDelete={() => setAsking([{ group, keep: keepOf(group) }])} + /> + ))} +
+ + )} + + )} +
+ + {asking && ( + setAsking(null)} + onConfirm={() => void run(asking)} + > +
    + {asking.map(({ group, keep }) => ( +
  • + {group.title} + + {t('admin.duplicates.willKeep')} {describeFile(t, group.files.find((f) => f.track_id === keep)!)} + + {removals(group, keep).map((file) => ( + + {t('admin.duplicates.willDelete')} {describeFile(t, file)} + + ))} +
  • + ))} +
+
+ )} +
+ ) +} diff --git a/web/src/pages/admin/duplicateFiles.ts b/web/src/pages/admin/duplicateFiles.ts new file mode 100644 index 0000000..5dea9ee --- /dev/null +++ b/web/src/pages/admin/duplicateFiles.ts @@ -0,0 +1,14 @@ +import type { useTranslation } from 'react-i18next' +import type { DuplicateFile, DuplicateGroup } from '@/api/duplicates' +import { formatBytes } from '@/lib/bytes' + +/** The copies a group would lose when `keep` stays. */ +export function removals(group: DuplicateGroup, keep: number): DuplicateFile[] { + return group.files.filter((file) => file.track_id !== keep) +} + +/** A copy in words, for a list that asks before deleting: `Album, flac, 320 kbps, 4.2 MB`. */ +export function describeFile(t: ReturnType['t'], file: DuplicateFile): string { + const bitrate = file.bitrate > 0 ? `${file.bitrate} kbps` : t('admin.duplicates.bitrateUnknown') + return [file.album?.title ?? t('admin.duplicates.noAlbum'), file.format, bitrate, formatBytes(file.size_bytes)].join(', ') +} diff --git a/web/src/router.tsx b/web/src/router.tsx index 3a89d15..e299956 100644 --- a/web/src/router.tsx +++ b/web/src/router.tsx @@ -45,6 +45,7 @@ export const routes: RouteObject[] = [ { path: 'providers', lazy: page(() => import('@/pages/admin/ProvidersPage')) }, { path: 'federation', lazy: page(() => import('@/pages/admin/FederationPage')) }, { path: 'jobs', lazy: page(() => import('@/pages/admin/JobsPage')) }, + { path: 'duplicates', lazy: page(() => import('@/pages/admin/DuplicatesPage')) }, { path: 'costs', lazy: page(() => import('@/pages/admin/CostsPage')) }, ], },