diff --git a/web/src/api/duplicates.test.tsx b/web/src/api/duplicates.test.tsx
new file mode 100644
index 0000000..f970850
--- /dev/null
+++ b/web/src/api/duplicates.test.tsx
@@ -0,0 +1,189 @@
+import { QueryClient, QueryClientProvider } from '@tanstack/react-query'
+import { act, renderHook, waitFor } from '@testing-library/react'
+import type { ReactNode } from 'react'
+import { expect, test } from 'vitest'
+import { apiError, mockFetch } from '@/test/mockFetch'
+import { useCleanDuplicates, useDuplicates, useDuplicatesCheck } from './duplicates'
+import { useJobStatus, useStartJob, useStopJob } from './jobs'
+
+function setup() {
+ const client = new QueryClient({ defaultOptions: { queries: { retry: false }, mutations: { retry: false } } })
+ const wrapper = ({ children }: { children: ReactNode }) => {children}
+ return { client, wrapper }
+}
+
+const file = (trackId: number) => ({ track_id: trackId, album: { id: 1, title: 'Album' }, format: 'mp3', bitrate: 192, size_bytes: 1000, duration_ms: 200_000, hashed: false })
+const group = (a: number, b: number) => ({
+ title: `Song ${a}`,
+ artists: [{ id: 1, name: 'Ann' }],
+ files: [file(a), file(b)],
+ hashed: false,
+ identical: false,
+ suggested_keep_track_id: a,
+ freed_bytes: 1000,
+})
+
+test('the groups are read page by page, with the total and whether deleting is allowed', async () => {
+ const { calls } = mockFetch({
+ 'GET /api/v1/library/duplicates': ({ url }) => {
+ const offset = Number(url.searchParams.get('offset'))
+ return offset === 0
+ ? { deletion_allowed: true, total: 3, limit: 2, offset: 0, groups: [group(1, 2), group(3, 4)] }
+ : { deletion_allowed: true, total: 3, limit: 2, offset: 2, groups: [group(5, 6)] }
+ },
+ })
+ const { result } = renderHook(() => useDuplicates({ pageSize: 2 }), { wrapper: setup().wrapper })
+ await waitFor(() => expect(result.current.data?.groups).toHaveLength(2))
+ expect(result.current.data).toMatchObject({ total: 3, deletionAllowed: true })
+ expect(result.current.hasNextPage).toBe(true)
+
+ await act(async () => {
+ await result.current.fetchNextPage()
+ })
+ await waitFor(() => expect(result.current.data?.groups).toHaveLength(3))
+ expect(result.current.hasNextPage).toBe(false)
+ expect(calls.map((c) => [c.url.searchParams.get('limit'), c.url.searchParams.get('offset')])).toEqual([
+ ['2', '0'],
+ ['2', '2'],
+ ])
+})
+
+test('with deleting turned off the groups are still there and the flag says so', async () => {
+ mockFetch({ 'GET /api/v1/library/duplicates': () => ({ deletion_allowed: false, total: 1, limit: 50, offset: 0, groups: [group(1, 2)] }) })
+ const { result } = renderHook(() => useDuplicates(), { wrapper: setup().wrapper })
+ await waitFor(() => expect(result.current.data?.groups).toHaveLength(1))
+ expect(result.current.data?.deletionAllowed).toBe(false)
+})
+
+test('cleaning sends the copy to keep and the copies to delete, then reads the groups again', async () => {
+ let reads = 0
+ const { calls } = mockFetch({
+ 'GET /api/v1/library/duplicates': () => {
+ reads++
+ return { deletion_allowed: true, total: 0, limit: 50, offset: 0, groups: [] }
+ },
+ 'POST /api/v1/library/duplicates/clean': () => ({ removed: 2, freed_bytes: 2000 }),
+ })
+ const { wrapper } = setup()
+ const list = renderHook(() => useDuplicates(), { wrapper })
+ const clean = renderHook(() => useCleanDuplicates(), { wrapper })
+ await waitFor(() => expect(list.result.current.data).toBeDefined())
+
+ let answer: unknown
+ await act(async () => {
+ answer = await clean.result.current.mutateAsync([{ keep: 7, remove: [8, 9] }])
+ })
+
+ expect(answer).toEqual({ removed: 2, freedBytes: 2000, groupsDone: 1 })
+ const post = calls.find((c) => c.key === 'POST /api/v1/library/duplicates/clean')
+ expect(JSON.parse(String(post?.init.body))).toEqual({ keep_track_id: 7, remove_track_ids: [8, 9] })
+ await waitFor(() => expect(reads).toBe(2))
+})
+
+test('several songs are cleaned one after the other and the groups are read again once, at the end', async () => {
+ let reads = 0
+ const bodies: unknown[] = []
+ mockFetch({
+ 'GET /api/v1/library/duplicates': () => {
+ reads++
+ return { deletion_allowed: true, total: 0, limit: 50, offset: 0, groups: [] }
+ },
+ 'POST /api/v1/library/duplicates/clean': ({ init }) => {
+ bodies.push(JSON.parse(String(init.body)))
+ return { removed: 1, freed_bytes: 500 }
+ },
+ })
+ const { wrapper } = setup()
+ const list = renderHook(() => useDuplicates(), { wrapper })
+ const clean = renderHook(() => useCleanDuplicates(), { wrapper })
+ await waitFor(() => expect(list.result.current.data).toBeDefined())
+ let answer: unknown
+ await act(async () => {
+ answer = await clean.result.current.mutateAsync([
+ { keep: 1, remove: [2] },
+ { keep: 3, remove: [4] },
+ { keep: 5, remove: [6] },
+ ])
+ })
+ expect(answer).toEqual({ removed: 3, freedBytes: 1500, groupsDone: 3 })
+ expect(bodies).toEqual([
+ { keep_track_id: 1, remove_track_ids: [2] },
+ { keep_track_id: 3, remove_track_ids: [4] },
+ { keep_track_id: 5, remove_track_ids: [6] },
+ ])
+ await waitFor(() => expect(reads).toBe(2))
+ await new Promise((r) => setTimeout(r, 80))
+ expect(reads).toBe(2)
+})
+
+test('at the first refusal it stops, says what was already deleted and why it stopped, and the groups are read again', async () => {
+ let reads = 0
+ const keeps: number[] = []
+ mockFetch({
+ 'GET /api/v1/library/duplicates': () => {
+ reads++
+ return { deletion_allowed: true, total: 0, limit: 50, offset: 0, groups: [] }
+ },
+ 'POST /api/v1/library/duplicates/clean': ({ init }) => {
+ const keep = JSON.parse(String(init.body)).keep_track_id as number
+ keeps.push(keep)
+ return keep === 3 ? apiError(409, 'duplicate_changed') : { removed: 1, freed_bytes: 500 }
+ },
+ })
+ const { wrapper } = setup()
+ const list = renderHook(() => useDuplicates(), { wrapper })
+ const clean = renderHook(() => useCleanDuplicates(), { wrapper })
+ await waitFor(() => expect(list.result.current.data).toBeDefined())
+ let answer: { removed: number; freedBytes: number; groupsDone: number; error?: unknown } | undefined
+ await act(async () => {
+ answer = await clean.result.current.mutateAsync([
+ { keep: 1, remove: [2] },
+ { keep: 3, remove: [4] },
+ { keep: 5, remove: [6] },
+ ])
+ })
+ expect(keeps).toEqual([1, 3])
+ expect(answer).toMatchObject({ removed: 1, freedBytes: 500, groupsDone: 1, error: { code: 'duplicate_changed' } })
+ await waitFor(() => expect(reads).toBe(2))
+})
+
+test('the check of the candidates is a job like the others, at its own address', async () => {
+ const none = () => new Response(null, { status: 204 })
+ const { calls } = mockFetch({
+ 'GET /api/v1/library/duplicates/check': () => ({ running: false }),
+ 'POST /api/v1/library/duplicates/check': () => new Response(JSON.stringify({ status: 'started' }), { status: 202, headers: { 'Content-Type': 'application/json' } }),
+ 'DELETE /api/v1/library/duplicates/check': none,
+ })
+ const { wrapper } = setup()
+ const status = renderHook(() => useJobStatus('duplicates/check'), { wrapper })
+ const start = renderHook(() => useStartJob('duplicates/check'), { wrapper })
+ const stop = renderHook(() => useStopJob('duplicates/check'), { wrapper })
+ await waitFor(() => expect(status.result.current.data).toEqual({ running: false }))
+ await act(async () => {
+ await start.result.current.mutateAsync({ refresh: false })
+ await stop.result.current.mutateAsync()
+ })
+ expect(calls.map((c) => c.key)).toEqual(
+ expect.arrayContaining(['GET /api/v1/library/duplicates/check', 'POST /api/v1/library/duplicates/check', 'DELETE /api/v1/library/duplicates/check']),
+ )
+})
+
+test('when a check ends the groups are read again, because their files have been read', async () => {
+ let reads = 0
+ let running = true
+ mockFetch({
+ 'GET /api/v1/library/duplicates': () => {
+ reads++
+ return { deletion_allowed: true, total: 0, limit: 50, offset: 0, groups: [] }
+ },
+ 'GET /api/v1/library/duplicates/check': () => ({ running }),
+ })
+ const { wrapper } = setup()
+ const list = renderHook(() => useDuplicates(), { wrapper })
+ const status = renderHook(() => useDuplicatesCheck({ intervalMs: 20 }), { wrapper })
+ await waitFor(() => expect(list.result.current.data).toBeDefined())
+ await waitFor(() => expect(status.result.current.data?.running).toBe(true))
+ running = false
+ await waitFor(() => expect(status.result.current.data?.running).toBe(false), { timeout: 2000 })
+ await waitFor(() => expect(reads).toBe(2), { timeout: 2000 })
+})
diff --git a/web/src/api/duplicates.ts b/web/src/api/duplicates.ts
new file mode 100644
index 0000000..6e4a680
--- /dev/null
+++ b/web/src/api/duplicates.ts
@@ -0,0 +1,92 @@
+import { useInfiniteQuery, useMutation, useQueryClient } from '@tanstack/react-query'
+import { useEffect, useRef } from 'react'
+import { api } from './client'
+import { useJobStatus } from './jobs'
+import type { components } from './schema'
+
+export type DuplicatesPage = components['schemas']['DuplicatesPage']
+export type DuplicateGroup = components['schemas']['DuplicateGroup']
+export type DuplicateFile = components['schemas']['DuplicateFile']
+export type DuplicatesCleaned = components['schemas']['DuplicatesCleaned']
+
+const GROUPS = ['library', 'duplicates']
+
+/**
+ * The groups of copies, read a page after the other. They come from the server's index, which touches no file.
+ * `deletionAllowed` is whether the server lets copies be deleted at all (`library.allow_delete`).
+ */
+export function useDuplicates({ pageSize = 50 }: { pageSize?: number } = {}) {
+ return useInfiniteQuery({
+ queryKey: [...GROUPS, pageSize],
+ initialPageParam: 0,
+ queryFn: ({ pageParam }) => api.get('/library/duplicates', { limit: pageSize, offset: pageParam }),
+ getNextPageParam: (last) => {
+ const next = last.offset + last.groups.length
+ return last.groups.length > 0 && next < last.total ? next : undefined
+ },
+ select: (data) => ({
+ groups: data.pages.flatMap((p) => p.groups),
+ total: data.pages[0]?.total ?? 0,
+ deletionAllowed: data.pages[0]?.deletion_allowed ?? false,
+ }),
+ })
+}
+
+export interface CleanItem {
+ /** The track that stays. */
+ keep: number
+ /** The tracks whose files are deleted. */
+ remove: number[]
+}
+
+/** What a cleaning did: `error` is why it stopped before the end, when it did. */
+export interface CleanResult {
+ removed: number
+ freedBytes: number
+ /** Songs cleaned in full. */
+ groupsDone: number
+ error?: unknown
+}
+
+/**
+ * Deletes, for each song, the copies `remove` from the disk and keeps `keep`. Songs go one after the other
+ * and it stops at the first refusal: the result says what was already deleted and why it stopped, and a
+ * refusal is not an error of the mutation, so the page can show both. The groups are read again at the end,
+ * whatever happened: a refusal often means the library changed since they were read.
+ */
+export function useCleanDuplicates() {
+ const client = useQueryClient()
+ return useMutation({
+ mutationFn: async (items: CleanItem[]): Promise => {
+ const result: CleanResult = { removed: 0, freedBytes: 0, groupsDone: 0 }
+ for (const { keep, remove } of items) {
+ try {
+ const done = await api.post('/library/duplicates/clean', { keep_track_id: keep, remove_track_ids: remove })
+ result.removed += done.removed
+ result.freedBytes += done.freed_bytes
+ result.groupsDone++
+ } catch (error) {
+ return { ...result, error }
+ }
+ }
+ return result
+ },
+ onSettled: () => client.invalidateQueries({ queryKey: GROUPS }),
+ })
+}
+
+/**
+ * The check that reads the files of the groups (see `useJobStatus`). When a run ends the groups are read
+ * again, because they now say which copies are byte for byte the same.
+ */
+export function useDuplicatesCheck(opts: { intervalMs?: number } = {}) {
+ const client = useQueryClient()
+ const status = useJobStatus('duplicates/check', opts)
+ const wasRunning = useRef(false)
+ const running = status.data?.running === true
+ useEffect(() => {
+ if (wasRunning.current && !running) void client.invalidateQueries({ queryKey: GROUPS })
+ wasRunning.current = running
+ }, [running, client])
+ return status
+}
diff --git a/web/src/api/jobs.ts b/web/src/api/jobs.ts
index e9fac9f..96415b1 100644
--- a/web/src/api/jobs.ts
+++ b/web/src/api/jobs.ts
@@ -4,15 +4,18 @@ import type { components } from './schema'
export type CoversJobStatus = components['schemas']['CoversJobStatus']
export type ArtistJobStatus = components['schemas']['ArtistJobStatus']
+export type DuplicatesCheckStatus = components['schemas']['DuplicatesCheckStatus']
-/** The bulk jobs that go through the whole library, each at its own address under `/library`. */
+/** The bulk jobs that go through the whole library with the providers, each at its own address under `/library`. */
export type JobKind = 'covers' | 'artist-images' | 'descriptions'
-export type JobStatus = K extends 'covers' ? CoversJobStatus : ArtistJobStatus
+/** Every job that is started, followed and stopped the same way: the bulk ones, and the check of the duplicates. */
+export type JobPath = JobKind | 'duplicates/check'
+export type JobStatus = K extends 'covers' ? CoversJobStatus : K extends 'duplicates/check' ? DuplicatesCheckStatus : ArtistJobStatus
-const statusKey = (kind: JobKind) => ['library', kind]
+const statusKey = (kind: JobPath) => ['library', kind]
/** Whether the job runs, how far it is, and how the last one ended. While it runs it is read again every `intervalMs`. */
-export function useJobStatus(kind: K, { intervalMs = 2000 }: { intervalMs?: number } = {}) {
+export function useJobStatus(kind: K, { intervalMs = 2000 }: { intervalMs?: number } = {}) {
return useQuery({
queryKey: statusKey(kind),
queryFn: () => api.get>(`/library/${kind}`),
@@ -24,7 +27,7 @@ export function useJobStatus(kind: K, { intervalMs = 2000 }:
* Starts the job in the background. `refresh` goes through every item again, also those with a good answer.
* Whatever the answer (also "already running"), the status is read at once so the page follows the job.
*/
-export function useStartJob(kind: JobKind) {
+export function useStartJob(kind: JobPath) {
const client = useQueryClient()
return useMutation({
mutationFn: ({ refresh }: { refresh: boolean }) => api.post<{ status: 'started' }>(refresh ? `/library/${kind}?refresh=true` : `/library/${kind}`),
@@ -33,7 +36,7 @@ export function useStartJob(kind: JobKind) {
}
/** Asks the job to end after the item it is on. What it already did stays. */
-export function useStopJob(kind: JobKind) {
+export function useStopJob(kind: JobPath) {
const client = useQueryClient()
return useMutation({
mutationFn: () => api.del(`/library/${kind}`),
diff --git a/web/src/api/schema.d.ts b/web/src/api/schema.d.ts
index 05ef681..1aa7261 100644
--- a/web/src/api/schema.d.ts
+++ b/web/src/api/schema.d.ts
@@ -443,6 +443,110 @@ export interface paths {
patch?: never;
trace?: never;
};
+ "/library/duplicates": {
+ parameters: {
+ query?: never;
+ header?: never;
+ path?: never;
+ cookie?: never;
+ };
+ /**
+ * The songs that are in more than one file
+ * @description Admin only. Groups of files that are the same song by the same artists, worked out from the index and
+ * never from the disk, so it is quick and touches no file. Two files are the same song when they have the
+ * same set of main artists, the same title (the key Sonda matches titles by) and durations no more than
+ * 3 seconds apart, every pair in a group within that. A file with no duration or no artist never matches,
+ * a file flagged as missing does not count, and a copy may be on a different album: `files` says which.
+ *
+ * Each group says which copy Sonda would keep (`suggested_keep_track_id`): the one without loss (FLAC)
+ * before one with loss, then the higher bitrate, then the bigger file, then the oldest. `hashed` is true when
+ * every file of the group has been read by the check (`POST /library/duplicates/check`) and `identical` when,
+ * besides, they are byte for byte the same. No file path is ever returned.
+ *
+ * `deletion_allowed` says whether `POST /library/duplicates/clean` can work: with `library.allow_delete`
+ * off (the default) the groups can still be seen.
+ */
+ get: operations["listDuplicates"];
+ put?: never;
+ post?: never;
+ delete?: never;
+ options?: never;
+ head?: never;
+ patch?: never;
+ trace?: never;
+ };
+ "/library/duplicates/check": {
+ parameters: {
+ query?: never;
+ header?: never;
+ path?: never;
+ cookie?: never;
+ };
+ /**
+ * Progress of the check that reads the files of the duplicates
+ * @description Whether the check is running, how far it is (`progress`, only while it runs), and how the last one ended.
+ * The check is described at `POST /library/duplicates/check`.
+ */
+ get: operations["getDuplicatesCheck"];
+ put?: never;
+ /**
+ * Read the files of the duplicates, to tell which are identical
+ * @description Admin only. Starts, in the background and one at a time like a scan, a job that reads every file that is in
+ * a group of copies and keeps its SHA-256, so that a group of files that are byte for byte the same can be
+ * called `identical`. It only reads: nothing is removed or sent anywhere. **It reads whole files**, which on a
+ * network drive takes a while, so it is started on purpose. A file that has a fingerprint is skipped, and the
+ * fingerprint of a file is forgotten when a scan finds the file changed; send `refresh=true` to read every file
+ * again. A file that changed since the scan is skipped. It gives up, and says why in `last_error`, when files
+ * cannot be read five times in a row (a disk that is not mounted). `DELETE /library/duplicates/check` stops it.
+ */
+ post: operations["startDuplicatesCheck"];
+ /**
+ * Stop the check of the duplicates
+ * @description Admin only. Asks a running check to end after the file it is on; the fingerprints already kept stay. It
+ * answers the same when no check is running.
+ */
+ delete: operations["stopDuplicatesCheck"];
+ options?: never;
+ head?: never;
+ patch?: never;
+ trace?: never;
+ };
+ "/library/duplicates/clean": {
+ parameters: {
+ query?: never;
+ header?: never;
+ path?: never;
+ cookie?: never;
+ };
+ get?: never;
+ put?: never;
+ /**
+ * Delete the extra copies of one song
+ * @description Admin only. **Deletes files from the music folder.** It is the one thing Sonda does that writes there, so it
+ * is off until the operator turns on `library.allow_delete` (answer `deletion_disabled` otherwise), the folder
+ * has to be writable (`library_read_only`), and it is never done by itself: one request removes the copies
+ * `remove_track_ids` of one song and keeps `keep_track_id`.
+ *
+ * The server decides again, whatever the client saw, and removes nothing unless every check passes: no scan is
+ * running (`scan_running`); the tracks exist (`track_not_found`); they are copies of one song by the rule of
+ * `GET /library/duplicates` (`invalid_input` otherwise, and also when the copy to keep is among those to
+ * remove, a copy is named twice or there are none or more than 100); and the copy to keep and every copy to
+ * remove are still the file the last scan saw, a regular file with the same size and date (`duplicate_changed`
+ * for a file that changed, is gone, became a link or is outside the library: scan again). So the last copy of
+ * a song is never removed. Every file is checked before the first is removed.
+ *
+ * Each copy is removed from the disk through the library folder (a path cannot leave it, not even through a
+ * link) and its track leaves the index; **the listens of the removed track become listens of the track that
+ * stays**, so the taste is not lost. Folders are never removed, not even empty ones. If it fails part way (for
+ * example the folder turns out to be read-only for the second file), what was removed stays removed.
+ */
+ post: operations["cleanDuplicates"];
+ delete?: never;
+ options?: never;
+ head?: never;
+ patch?: never;
+ trace?: never;
+ };
"/library/scan": {
parameters: {
query?: never;
@@ -2072,6 +2176,101 @@ export interface components {
/** @description Why the last job ended before the end, for example that a provider is off or kept failing. */
last_error?: string;
};
+ DuplicatesPage: {
+ /** @description Whether `POST /library/duplicates/clean` can work (`library.allow_delete`). */
+ deletion_allowed: boolean;
+ /** @description Groups in all, not in this page. */
+ total: number;
+ limit: number;
+ offset: number;
+ groups: components["schemas"]["DuplicateGroup"][];
+ };
+ DuplicateGroup: {
+ title: string;
+ /** @description The main artists the copies share. */
+ artists: components["schemas"]["ArtistRef"][];
+ /** @description The copies, by track id; at least two. */
+ files: components["schemas"]["DuplicateFile"][];
+ /** @description Every file of the group has been read by the check. */
+ hashed: boolean;
+ /** @description Hashed, and byte for byte the same. */
+ identical: boolean;
+ /**
+ * Format: int64
+ * @description The copy Sonda would keep: without loss first, then the higher bitrate, then the bigger file, then the oldest.
+ */
+ suggested_keep_track_id: number;
+ /**
+ * Format: int64
+ * @description The size of every copy but the one to keep.
+ */
+ freed_bytes: number;
+ };
+ DuplicateFile: {
+ /** Format: int64 */
+ track_id: number;
+ album: {
+ /** Format: int64 */
+ id: number;
+ title: string;
+ } | null;
+ /** @description `mp3`, `flac`... */
+ format: string;
+ /** @description 0 when unknown. */
+ bitrate: number;
+ /** Format: int64 */
+ size_bytes: number;
+ /** Format: int64 */
+ duration_ms: number;
+ /** @description This file has been read by the check. */
+ hashed: boolean;
+ };
+ CleanDuplicates: {
+ /**
+ * Format: int64
+ * @description The copy that stays.
+ */
+ keep_track_id: number;
+ /** @description The copies to delete, of the same song as the one that stays, none twice. */
+ remove_track_ids: number[];
+ };
+ DuplicatesCleaned: {
+ /** @description Files deleted. */
+ removed: number;
+ /** Format: int64 */
+ freed_bytes: number;
+ };
+ DuplicatesCheckStatus: {
+ running: boolean;
+ /**
+ * Format: date-time
+ * @description When the last check started.
+ */
+ started_at?: string;
+ /** Format: date-time */
+ last_finished_at?: string;
+ /** @description Only while the check runs. */
+ progress?: {
+ done: number;
+ /** @description Files the check set out to read. */
+ total: number;
+ };
+ /** @description How the last finished check ended. */
+ last?: {
+ /** @description Files looked at. */
+ checked: number;
+ /** @description Files read and fingerprinted. */
+ hashed: number;
+ /** @description Files that changed since the scan, or are gone. */
+ skipped: number;
+ /** @description Files that could not be read just then. */
+ failed: number;
+ /** @description It was stopped, or the application was. */
+ stopped: boolean;
+ };
+ /** @description Why the last check ended before the end, for example that the disk is not mounted. */
+ last_error?: string;
+ };
UserSettings: {
/**
* @description When true, Discover may start from the artists you listen to most. That sends their
@@ -3063,6 +3262,170 @@ export interface operations {
403: components["responses"]["AdminRequired"];
};
};
+ listDuplicates: {
+ parameters: {
+ query?: {
+ /** @description Page size. 50 when absent or 0; values above 200 are lowered to 200. */
+ limit?: components["parameters"]["Limit"];
+ offset?: components["parameters"]["Offset"];
+ };
+ header?: never;
+ path?: never;
+ cookie?: never;
+ };
+ requestBody?: never;
+ responses: {
+ /** @description A page of groups, in a fixed order (by title). */
+ 200: {
+ headers: {
+ [name: string]: unknown;
+ };
+ content: {
+ "application/json": components["schemas"]["DuplicatesPage"];
+ };
+ };
+ 400: components["responses"]["InvalidInput"];
+ 401: components["responses"]["Unauthorized"];
+ 403: components["responses"]["AdminRequired"];
+ };
+ };
+ getDuplicatesCheck: {
+ parameters: {
+ query?: never;
+ header?: never;
+ path?: never;
+ cookie?: never;
+ };
+ requestBody?: never;
+ responses: {
+ /** @description The job. */
+ 200: {
+ headers: {
+ [name: string]: unknown;
+ };
+ content: {
+ "application/json": components["schemas"]["DuplicatesCheckStatus"];
+ };
+ };
+ 401: components["responses"]["Unauthorized"];
+ 403: components["responses"]["AdminRequired"];
+ };
+ };
+ startDuplicatesCheck: {
+ parameters: {
+ query?: {
+ /** @description `true` reads every file again, also those that have a fingerprint. `false` when absent. */
+ refresh?: boolean;
+ };
+ header?: never;
+ path?: never;
+ cookie?: never;
+ };
+ requestBody?: never;
+ responses: {
+ /** @description The check was started in the background. */
+ 202: {
+ headers: {
+ [name: string]: unknown;
+ };
+ content: {
+ "application/json": {
+ /** @enum {string} */
+ status: "started";
+ };
+ };
+ };
+ 400: components["responses"]["InvalidInput"];
+ 401: components["responses"]["Unauthorized"];
+ 403: components["responses"]["AdminRequired"];
+ /** @description The check is already running (`duplicates_check_running`). */
+ 409: {
+ headers: {
+ [name: string]: unknown;
+ };
+ content: {
+ "application/json": components["schemas"]["Error"];
+ };
+ };
+ /** @description The groups could not be worked out (`internal`). */
+ 500: {
+ headers: {
+ [name: string]: unknown;
+ };
+ content: {
+ "application/json": components["schemas"]["Error"];
+ };
+ };
+ };
+ };
+ stopDuplicatesCheck: {
+ parameters: {
+ query?: never;
+ header?: never;
+ path?: never;
+ cookie?: never;
+ };
+ requestBody?: never;
+ responses: {
+ /** @description Done, also when no check was running. */
+ 204: {
+ headers: {
+ [name: string]: unknown;
+ };
+ content?: never;
+ };
+ 401: components["responses"]["Unauthorized"];
+ 403: components["responses"]["AdminRequired"];
+ };
+ };
+ cleanDuplicates: {
+ parameters: {
+ query?: never;
+ header?: never;
+ path?: never;
+ cookie?: never;
+ };
+ requestBody: {
+ content: {
+ "application/json": components["schemas"]["CleanDuplicates"];
+ };
+ };
+ responses: {
+ /** @description The extra copies were removed. */
+ 200: {
+ headers: {
+ [name: string]: unknown;
+ };
+ content: {
+ "application/json": components["schemas"]["DuplicatesCleaned"];
+ };
+ };
+ 400: components["responses"]["InvalidInput"];
+ 401: components["responses"]["Unauthorized"];
+ 403: components["responses"]["AdminRequired"];
+ /** @description A track that was named does not exist (`track_not_found`). */
+ 404: {
+ headers: {
+ [name: string]: unknown;
+ };
+ content: {
+ "application/json": components["schemas"]["Error"];
+ };
+ };
+ /**
+ * @description Deleting is not turned on (`deletion_disabled`), a scan is running (`scan_running`), a file is not what the last
+ * scan saw (`duplicate_changed`), or the music folder cannot be written to (`library_read_only`).
+ */
+ 409: {
+ headers: {
+ [name: string]: unknown;
+ };
+ content: {
+ "application/json": components["schemas"]["Error"];
+ };
+ };
+ };
+ };
getScanStatus: {
parameters: {
query?: never;
diff --git a/web/src/i18n/locales/en.json b/web/src/i18n/locales/en.json
index d7b2010..2e183db 100644
--- a/web/src/i18n/locales/en.json
+++ b/web/src/i18n/locales/en.json
@@ -71,6 +71,7 @@
"sectionUsers": "Users",
"sectionProviders": "Providers",
"sectionJobs": "Jobs",
+ "sectionDuplicates": "Duplicates",
"jobs": {
"intro": "These jobs go through the whole library in the background, one item after the other. Each card says what it sends and to whom.",
"needs": "Needs {{names}} turned on: use the Providers section.",
@@ -129,6 +130,69 @@
}
}
},
+ "duplicates": {
+ "intro": "Songs that are in more than one file: the same artists, the same title and almost the same length, even when they are on different albums. All of this is worked out on this server; nothing leaves it.",
+ "off": "Deleting is turned off. To delete copies, set library.allow_delete (or SONDA_ALLOW_DELETE=true), mount the music folders so that Sonda can write to them, and restart the server.",
+ "check": {
+ "title": "Check the candidates",
+ "what": "Reads the files of the songs below and keeps a fingerprint (SHA-256) of each one, to tell which copies are byte for byte the same file. It takes a while on a big library.",
+ "sends": "It only reads your files. Nothing leaves this server.",
+ "start": "Start the check",
+ "stop": "Stop the check",
+ "progress": "{{done}} of {{total}} files",
+ "never": "The check has not run yet.",
+ "counts": {
+ "checked": "{{count}} looked at",
+ "hashed": "{{count}} read",
+ "skipped": "{{count}} skipped (changed or gone)",
+ "failed": "{{count}} could not be read"
+ }
+ },
+ "none": "No duplicates found.",
+ "count_one": "{{count}} song is in more than one file",
+ "count_other": "{{count}} songs are in more than one file",
+ "cleanAll": "Clean all suggestions",
+ "cleanAllShown": "Only the songs shown below are cleaned. Show more to include the rest.",
+ "keepColumn": "Keep",
+ "album": "Album",
+ "format": "Format",
+ "bitrate": "Bitrate",
+ "size": "Size",
+ "noAlbum": "No album",
+ "kbps": "{{count}} kbps",
+ "suggested": "Suggested",
+ "keepCopy": "Keep {{what}}",
+ "status": {
+ "identical": "The files are byte for byte the same.",
+ "different": "The files are not the same, even if the song is.",
+ "unchecked": "Not checked yet: run the check to see if they are the same file."
+ },
+ "deleteOthers": "Delete the others",
+ "deleteOthersOf": "Delete the other copies of {{title}}",
+ "confirmOneTitle_one": "Delete {{count}} copy of {{title}}?",
+ "confirmOneTitle_other": "Delete {{count}} copies of {{title}}?",
+ "confirmOneBody": "The files below are deleted from the disk for good, {{size}} freed. The listens and playlists that had them move to the copy that stays.",
+ "confirmAllTitle_one": "Clean {{count}} song?",
+ "confirmAllTitle_other": "Clean {{count}} songs?",
+ "confirmAllBody_one": "In each song the copy chosen stays and the others are deleted from the disk for good: {{count}} file, {{size}} freed. The listens and playlists move to the copy that stays.",
+ "confirmAllBody_other": "In each song the copy chosen stays and the others are deleted from the disk for good: {{count}} files, {{size}} freed. The listens and playlists move to the copy that stays.",
+ "confirmDelete_one": "Delete {{count}} file",
+ "confirmDelete_other": "Delete {{count}} files",
+ "stays": "Stays: {{what}}",
+ "goes": "Delete: {{what}}",
+ "deleting": "Deleting files…",
+ "deleted_one": "Deleted {{count}} file, {{size}} freed.",
+ "deleted_other": "Deleted {{count}} files, {{size}} freed.",
+ "errors": {
+ "duplicates_check_running": "The check is already running.",
+ "scan_running": "A scan is running. Try again when it ends.",
+ "deletion_disabled": "Deleting is turned off on the server.",
+ "duplicate_changed": "A file changed since the last scan, so nothing was deleted for that song. Run a scan and look again.",
+ "library_read_only": "The music folder is read-only, so nothing was deleted. Mount it so that Sonda can write to it.",
+ "invalid_input": "Those files are no longer copies of the same song, so nothing was deleted. Look at the list again.",
+ "track_not_found": "A song is already gone, so nothing was deleted for it. Look at the list again."
+ }
+ },
"cancel": "Cancel",
"save": "Save",
"errors": {
diff --git a/web/src/i18n/locales/it.json b/web/src/i18n/locales/it.json
index d7c7fd7..e56c972 100644
--- a/web/src/i18n/locales/it.json
+++ b/web/src/i18n/locales/it.json
@@ -71,6 +71,7 @@
"sectionUsers": "Utenti",
"sectionProviders": "Provider",
"sectionJobs": "Lavori",
+ "sectionDuplicates": "Duplicati",
"jobs": {
"intro": "Questi lavori passano in rassegna tutta la libreria in background, un elemento alla volta. Ogni scheda dice cosa manda e a chi.",
"needs": "Serve che {{names}} sia acceso: vai nella sezione Provider.",
@@ -129,6 +130,69 @@
}
}
},
+ "duplicates": {
+ "intro": "Canzoni che stanno in più di un file: stessi artisti, stesso titolo e durata quasi uguale, anche se sono su album diversi. Tutto viene calcolato su questo server; niente ne esce.",
+ "off": "La cancellazione è spenta. Per cancellare le copie imposta library.allow_delete (o SONDA_ALLOW_DELETE=true), monta le cartelle della musica in modo che Sonda possa scriverci e riavvia il server.",
+ "check": {
+ "title": "Controlla i candidati",
+ "what": "Legge i file delle canzoni qui sotto e ne tiene un'impronta (SHA-256), per capire quali copie sono lo stesso file byte per byte. Su una libreria grande ci vuole un po'.",
+ "sends": "Legge soltanto i tuoi file. Dal server non esce niente.",
+ "start": "Avvia il controllo",
+ "stop": "Ferma il controllo",
+ "progress": "{{done}} file su {{total}}",
+ "never": "Il controllo non è ancora stato eseguito.",
+ "counts": {
+ "checked": "{{count}} esaminati",
+ "hashed": "{{count}} letti",
+ "skipped": "{{count}} saltati (cambiati o spariti)",
+ "failed": "{{count}} non leggibili"
+ }
+ },
+ "none": "Nessun duplicato trovato.",
+ "count_one": "{{count}} canzone è in più di un file",
+ "count_other": "{{count}} canzoni sono in più di un file",
+ "cleanAll": "Pulisci tutti i suggerimenti",
+ "cleanAllShown": "Vengono pulite solo le canzoni mostrate qui sotto. Mostra altri per includere il resto.",
+ "keepColumn": "Tieni",
+ "album": "Album",
+ "format": "Formato",
+ "bitrate": "Bitrate",
+ "size": "Dimensione",
+ "noAlbum": "Nessun album",
+ "kbps": "{{count}} kbps",
+ "suggested": "Suggerita",
+ "keepCopy": "Tieni {{what}}",
+ "status": {
+ "identical": "I file sono uguali byte per byte.",
+ "different": "I file non sono uguali, anche se la canzone sì.",
+ "unchecked": "Non ancora controllato: avvia il controllo per vedere se sono lo stesso file."
+ },
+ "deleteOthers": "Cancella le altre",
+ "deleteOthersOf": "Cancella le altre copie di {{title}}",
+ "confirmOneTitle_one": "Cancellare {{count}} copia di {{title}}?",
+ "confirmOneTitle_other": "Cancellare {{count}} copie di {{title}}?",
+ "confirmOneBody": "I file qui sotto vengono cancellati dal disco per sempre, {{size}} liberati. Gli ascolti e le playlist che li avevano passano alla copia che resta.",
+ "confirmAllTitle_one": "Pulire {{count}} canzone?",
+ "confirmAllTitle_other": "Pulire {{count}} canzoni?",
+ "confirmAllBody_one": "Per ogni canzone resta la copia scelta e le altre vengono cancellate dal disco per sempre: {{count}} file, {{size}} liberati. Gli ascolti e le playlist passano alla copia che resta.",
+ "confirmAllBody_other": "Per ogni canzone resta la copia scelta e le altre vengono cancellate dal disco per sempre: {{count}} file, {{size}} liberati. Gli ascolti e le playlist passano alla copia che resta.",
+ "confirmDelete_one": "Cancella {{count}} file",
+ "confirmDelete_other": "Cancella {{count}} file",
+ "stays": "Resta: {{what}}",
+ "goes": "Da cancellare: {{what}}",
+ "deleting": "Cancello i file…",
+ "deleted_one": "Cancellato {{count}} file, {{size}} liberati.",
+ "deleted_other": "Cancellati {{count}} file, {{size}} liberati.",
+ "errors": {
+ "duplicates_check_running": "Il controllo è già in corso.",
+ "scan_running": "È in corso una scansione. Riprova quando finisce.",
+ "deletion_disabled": "La cancellazione è spenta sul server.",
+ "duplicate_changed": "Un file è cambiato dall'ultima scansione, quindi per quella canzone non è stato cancellato niente. Avvia una scansione e guarda di nuovo.",
+ "library_read_only": "La cartella della musica è in sola lettura, quindi non è stato cancellato niente. Montala in modo che Sonda possa scriverci.",
+ "invalid_input": "Quei file non sono più copie della stessa canzone, quindi non è stato cancellato niente. Guarda di nuovo l'elenco.",
+ "track_not_found": "Una canzone è già sparita, quindi per lei non è stato cancellato niente. Guarda di nuovo l'elenco."
+ }
+ },
"cancel": "Annulla",
"save": "Salva",
"errors": {
diff --git a/web/src/lib/size.test.ts b/web/src/lib/size.test.ts
new file mode 100644
index 0000000..82494e4
--- /dev/null
+++ b/web/src/lib/size.test.ts
@@ -0,0 +1,20 @@
+import { expect, test } from 'vitest'
+import { formatBytes } from './size'
+
+test.each([
+ [0, 'en', '0 B'],
+ [999, 'en', '999 B'],
+ [1000, 'en', '1 kB'],
+ [1_500, 'en', '1.5 kB'],
+ [42_000_000, 'en', '42 MB'],
+ [4_320_000_000, 'en', '4.3 GB'],
+ [4_320_000_000, 'it', '4,3 GB'],
+ [2_000_000_000_000, 'en', '2 TB'],
+])('%i bytes are written %s in %s', (bytes, lang, text) => {
+ expect(formatBytes(bytes, lang)).toBe(text)
+})
+
+test('anything that is not a size is 0 B', () => {
+ expect(formatBytes(-5, 'en')).toBe('0 B')
+ expect(formatBytes(Number.NaN, 'en')).toBe('0 B')
+})
diff --git a/web/src/lib/size.ts b/web/src/lib/size.ts
new file mode 100644
index 0000000..8c183a5
--- /dev/null
+++ b/web/src/lib/size.ts
@@ -0,0 +1,10 @@
+const UNITS = ['B', 'kB', 'MB', 'GB', 'TB']
+
+/** A size in bytes as people read it ("4.3 GB"): units of 1000, one decimal at most, the number in the reader's language. */
+export function formatBytes(bytes: number, language?: string): string {
+ if (!Number.isFinite(bytes) || bytes <= 0) return '0 B'
+ const unit = Math.min(Math.floor(Math.log10(bytes) / 3), UNITS.length - 1)
+ const value = bytes / 1000 ** unit
+ const digits = unit === 0 ? 0 : 1
+ return `${new Intl.NumberFormat(language, { maximumFractionDigits: digits }).format(value)} ${UNITS[unit]}`
+}
diff --git a/web/src/pages/admin/AdminLayout.test.tsx b/web/src/pages/admin/AdminLayout.test.tsx
index 6c190f8..4009e7d 100644
--- a/web/src/pages/admin/AdminLayout.test.tsx
+++ b/web/src/pages/admin/AdminLayout.test.tsx
@@ -32,6 +32,12 @@ test('an admin gets the admin page with its sections and the page of the section
expect(screen.getByText('the libraries page')).toBeInTheDocument()
})
+test('the sections include the duplicates, at their own address', () => {
+ sessionStore.setState({ status: 'authenticated', user: { id: 1, email: 'tom@example.org', is_admin: true } })
+ renderAdmin()
+ expect(screen.getByRole('link', { name: 'Duplicates' })).toHaveAttribute('href', '/admin/duplicates')
+})
+
test('anyone else sees a notice, not the section, and nothing is asked of the server', async () => {
sessionStore.setState({ status: 'authenticated', user: { id: 2, email: 'ann@example.org', is_admin: false } })
const { calls } = mockFetch({})
diff --git a/web/src/pages/admin/AdminLayout.tsx b/web/src/pages/admin/AdminLayout.tsx
index 0b87de9..b423ddb 100644
--- a/web/src/pages/admin/AdminLayout.tsx
+++ b/web/src/pages/admin/AdminLayout.tsx
@@ -10,6 +10,7 @@ const SECTIONS = [
{ to: '/admin/users', key: 'admin.sectionUsers' },
{ to: '/admin/providers', key: 'admin.sectionProviders' },
{ to: '/admin/jobs', key: 'admin.sectionJobs' },
+ { to: '/admin/duplicates', key: 'admin.sectionDuplicates' },
]
/**
diff --git a/web/src/pages/admin/Confirm.tsx b/web/src/pages/admin/Confirm.tsx
index 8737c4c..a31146d 100644
--- a/web/src/pages/admin/Confirm.tsx
+++ b/web/src/pages/admin/Confirm.tsx
@@ -1,3 +1,4 @@
+import type { ReactNode } from 'react'
import { useTranslation } from 'react-i18next'
import { Button } from '@/components/ui/button'
import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog'
@@ -7,12 +8,15 @@ export default function Confirm({
title,
body,
confirm,
+ children,
onConfirm,
onClose,
}: {
title: string
body: string
confirm: string
+ /** More to read before answering (a list of what goes), under the body. */
+ children?: ReactNode
onConfirm: () => void
onClose: () => void
}) {
@@ -24,6 +28,7 @@ export default function Confirm({
{title}{body}
+ {children}