diff --git a/web/src/api/duplicates.test.tsx b/web/src/api/duplicates.test.tsx new file mode 100644 index 0000000..f970850 --- /dev/null +++ b/web/src/api/duplicates.test.tsx @@ -0,0 +1,189 @@ +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { act, renderHook, waitFor } from '@testing-library/react' +import type { ReactNode } from 'react' +import { expect, test } from 'vitest' +import { apiError, mockFetch } from '@/test/mockFetch' +import { useCleanDuplicates, useDuplicates, useDuplicatesCheck } from './duplicates' +import { useJobStatus, useStartJob, useStopJob } from './jobs' + +function setup() { + const client = new QueryClient({ defaultOptions: { queries: { retry: false }, mutations: { retry: false } } }) + const wrapper = ({ children }: { children: ReactNode }) => {children} + return { client, wrapper } +} + +const file = (trackId: number) => ({ track_id: trackId, album: { id: 1, title: 'Album' }, format: 'mp3', bitrate: 192, size_bytes: 1000, duration_ms: 200_000, hashed: false }) +const group = (a: number, b: number) => ({ + title: `Song ${a}`, + artists: [{ id: 1, name: 'Ann' }], + files: [file(a), file(b)], + hashed: false, + identical: false, + suggested_keep_track_id: a, + freed_bytes: 1000, +}) + +test('the groups are read page by page, with the total and whether deleting is allowed', async () => { + const { calls } = mockFetch({ + 'GET /api/v1/library/duplicates': ({ url }) => { + const offset = Number(url.searchParams.get('offset')) + return offset === 0 + ? { deletion_allowed: true, total: 3, limit: 2, offset: 0, groups: [group(1, 2), group(3, 4)] } + : { deletion_allowed: true, total: 3, limit: 2, offset: 2, groups: [group(5, 6)] } + }, + }) + const { result } = renderHook(() => useDuplicates({ pageSize: 2 }), { wrapper: setup().wrapper }) + await waitFor(() => expect(result.current.data?.groups).toHaveLength(2)) + expect(result.current.data).toMatchObject({ total: 3, deletionAllowed: true }) + expect(result.current.hasNextPage).toBe(true) + + await act(async () => { + await result.current.fetchNextPage() + }) + await waitFor(() => expect(result.current.data?.groups).toHaveLength(3)) + expect(result.current.hasNextPage).toBe(false) + expect(calls.map((c) => [c.url.searchParams.get('limit'), c.url.searchParams.get('offset')])).toEqual([ + ['2', '0'], + ['2', '2'], + ]) +}) + +test('with deleting turned off the groups are still there and the flag says so', async () => { + mockFetch({ 'GET /api/v1/library/duplicates': () => ({ deletion_allowed: false, total: 1, limit: 50, offset: 0, groups: [group(1, 2)] }) }) + const { result } = renderHook(() => useDuplicates(), { wrapper: setup().wrapper }) + await waitFor(() => expect(result.current.data?.groups).toHaveLength(1)) + expect(result.current.data?.deletionAllowed).toBe(false) +}) + +test('cleaning sends the copy to keep and the copies to delete, then reads the groups again', async () => { + let reads = 0 + const { calls } = mockFetch({ + 'GET /api/v1/library/duplicates': () => { + reads++ + return { deletion_allowed: true, total: 0, limit: 50, offset: 0, groups: [] } + }, + 'POST /api/v1/library/duplicates/clean': () => ({ removed: 2, freed_bytes: 2000 }), + }) + const { wrapper } = setup() + const list = renderHook(() => useDuplicates(), { wrapper }) + const clean = renderHook(() => useCleanDuplicates(), { wrapper }) + await waitFor(() => expect(list.result.current.data).toBeDefined()) + + let answer: unknown + await act(async () => { + answer = await clean.result.current.mutateAsync([{ keep: 7, remove: [8, 9] }]) + }) + + expect(answer).toEqual({ removed: 2, freedBytes: 2000, groupsDone: 1 }) + const post = calls.find((c) => c.key === 'POST /api/v1/library/duplicates/clean') + expect(JSON.parse(String(post?.init.body))).toEqual({ keep_track_id: 7, remove_track_ids: [8, 9] }) + await waitFor(() => expect(reads).toBe(2)) +}) + +test('several songs are cleaned one after the other and the groups are read again once, at the end', async () => { + let reads = 0 + const bodies: unknown[] = [] + mockFetch({ + 'GET /api/v1/library/duplicates': () => { + reads++ + return { deletion_allowed: true, total: 0, limit: 50, offset: 0, groups: [] } + }, + 'POST /api/v1/library/duplicates/clean': ({ init }) => { + bodies.push(JSON.parse(String(init.body))) + return { removed: 1, freed_bytes: 500 } + }, + }) + const { wrapper } = setup() + const list = renderHook(() => useDuplicates(), { wrapper }) + const clean = renderHook(() => useCleanDuplicates(), { wrapper }) + await waitFor(() => expect(list.result.current.data).toBeDefined()) + let answer: unknown + await act(async () => { + answer = await clean.result.current.mutateAsync([ + { keep: 1, remove: [2] }, + { keep: 3, remove: [4] }, + { keep: 5, remove: [6] }, + ]) + }) + expect(answer).toEqual({ removed: 3, freedBytes: 1500, groupsDone: 3 }) + expect(bodies).toEqual([ + { keep_track_id: 1, remove_track_ids: [2] }, + { keep_track_id: 3, remove_track_ids: [4] }, + { keep_track_id: 5, remove_track_ids: [6] }, + ]) + await waitFor(() => expect(reads).toBe(2)) + await new Promise((r) => setTimeout(r, 80)) + expect(reads).toBe(2) +}) + +test('at the first refusal it stops, says what was already deleted and why it stopped, and the groups are read again', async () => { + let reads = 0 + const keeps: number[] = [] + mockFetch({ + 'GET /api/v1/library/duplicates': () => { + reads++ + return { deletion_allowed: true, total: 0, limit: 50, offset: 0, groups: [] } + }, + 'POST /api/v1/library/duplicates/clean': ({ init }) => { + const keep = JSON.parse(String(init.body)).keep_track_id as number + keeps.push(keep) + return keep === 3 ? apiError(409, 'duplicate_changed') : { removed: 1, freed_bytes: 500 } + }, + }) + const { wrapper } = setup() + const list = renderHook(() => useDuplicates(), { wrapper }) + const clean = renderHook(() => useCleanDuplicates(), { wrapper }) + await waitFor(() => expect(list.result.current.data).toBeDefined()) + let answer: { removed: number; freedBytes: number; groupsDone: number; error?: unknown } | undefined + await act(async () => { + answer = await clean.result.current.mutateAsync([ + { keep: 1, remove: [2] }, + { keep: 3, remove: [4] }, + { keep: 5, remove: [6] }, + ]) + }) + expect(keeps).toEqual([1, 3]) + expect(answer).toMatchObject({ removed: 1, freedBytes: 500, groupsDone: 1, error: { code: 'duplicate_changed' } }) + await waitFor(() => expect(reads).toBe(2)) +}) + +test('the check of the candidates is a job like the others, at its own address', async () => { + const none = () => new Response(null, { status: 204 }) + const { calls } = mockFetch({ + 'GET /api/v1/library/duplicates/check': () => ({ running: false }), + 'POST /api/v1/library/duplicates/check': () => new Response(JSON.stringify({ status: 'started' }), { status: 202, headers: { 'Content-Type': 'application/json' } }), + 'DELETE /api/v1/library/duplicates/check': none, + }) + const { wrapper } = setup() + const status = renderHook(() => useJobStatus('duplicates/check'), { wrapper }) + const start = renderHook(() => useStartJob('duplicates/check'), { wrapper }) + const stop = renderHook(() => useStopJob('duplicates/check'), { wrapper }) + await waitFor(() => expect(status.result.current.data).toEqual({ running: false })) + await act(async () => { + await start.result.current.mutateAsync({ refresh: false }) + await stop.result.current.mutateAsync() + }) + expect(calls.map((c) => c.key)).toEqual( + expect.arrayContaining(['GET /api/v1/library/duplicates/check', 'POST /api/v1/library/duplicates/check', 'DELETE /api/v1/library/duplicates/check']), + ) +}) + +test('when a check ends the groups are read again, because their files have been read', async () => { + let reads = 0 + let running = true + mockFetch({ + 'GET /api/v1/library/duplicates': () => { + reads++ + return { deletion_allowed: true, total: 0, limit: 50, offset: 0, groups: [] } + }, + 'GET /api/v1/library/duplicates/check': () => ({ running }), + }) + const { wrapper } = setup() + const list = renderHook(() => useDuplicates(), { wrapper }) + const status = renderHook(() => useDuplicatesCheck({ intervalMs: 20 }), { wrapper }) + await waitFor(() => expect(list.result.current.data).toBeDefined()) + await waitFor(() => expect(status.result.current.data?.running).toBe(true)) + running = false + await waitFor(() => expect(status.result.current.data?.running).toBe(false), { timeout: 2000 }) + await waitFor(() => expect(reads).toBe(2), { timeout: 2000 }) +}) diff --git a/web/src/api/duplicates.ts b/web/src/api/duplicates.ts new file mode 100644 index 0000000..6e4a680 --- /dev/null +++ b/web/src/api/duplicates.ts @@ -0,0 +1,92 @@ +import { useInfiniteQuery, useMutation, useQueryClient } from '@tanstack/react-query' +import { useEffect, useRef } from 'react' +import { api } from './client' +import { useJobStatus } from './jobs' +import type { components } from './schema' + +export type DuplicatesPage = components['schemas']['DuplicatesPage'] +export type DuplicateGroup = components['schemas']['DuplicateGroup'] +export type DuplicateFile = components['schemas']['DuplicateFile'] +export type DuplicatesCleaned = components['schemas']['DuplicatesCleaned'] + +const GROUPS = ['library', 'duplicates'] + +/** + * The groups of copies, read a page after the other. They come from the server's index, which touches no file. + * `deletionAllowed` is whether the server lets copies be deleted at all (`library.allow_delete`). + */ +export function useDuplicates({ pageSize = 50 }: { pageSize?: number } = {}) { + return useInfiniteQuery({ + queryKey: [...GROUPS, pageSize], + initialPageParam: 0, + queryFn: ({ pageParam }) => api.get('/library/duplicates', { limit: pageSize, offset: pageParam }), + getNextPageParam: (last) => { + const next = last.offset + last.groups.length + return last.groups.length > 0 && next < last.total ? next : undefined + }, + select: (data) => ({ + groups: data.pages.flatMap((p) => p.groups), + total: data.pages[0]?.total ?? 0, + deletionAllowed: data.pages[0]?.deletion_allowed ?? false, + }), + }) +} + +export interface CleanItem { + /** The track that stays. */ + keep: number + /** The tracks whose files are deleted. */ + remove: number[] +} + +/** What a cleaning did: `error` is why it stopped before the end, when it did. */ +export interface CleanResult { + removed: number + freedBytes: number + /** Songs cleaned in full. */ + groupsDone: number + error?: unknown +} + +/** + * Deletes, for each song, the copies `remove` from the disk and keeps `keep`. Songs go one after the other + * and it stops at the first refusal: the result says what was already deleted and why it stopped, and a + * refusal is not an error of the mutation, so the page can show both. The groups are read again at the end, + * whatever happened: a refusal often means the library changed since they were read. + */ +export function useCleanDuplicates() { + const client = useQueryClient() + return useMutation({ + mutationFn: async (items: CleanItem[]): Promise => { + const result: CleanResult = { removed: 0, freedBytes: 0, groupsDone: 0 } + for (const { keep, remove } of items) { + try { + const done = await api.post('/library/duplicates/clean', { keep_track_id: keep, remove_track_ids: remove }) + result.removed += done.removed + result.freedBytes += done.freed_bytes + result.groupsDone++ + } catch (error) { + return { ...result, error } + } + } + return result + }, + onSettled: () => client.invalidateQueries({ queryKey: GROUPS }), + }) +} + +/** + * The check that reads the files of the groups (see `useJobStatus`). When a run ends the groups are read + * again, because they now say which copies are byte for byte the same. + */ +export function useDuplicatesCheck(opts: { intervalMs?: number } = {}) { + const client = useQueryClient() + const status = useJobStatus('duplicates/check', opts) + const wasRunning = useRef(false) + const running = status.data?.running === true + useEffect(() => { + if (wasRunning.current && !running) void client.invalidateQueries({ queryKey: GROUPS }) + wasRunning.current = running + }, [running, client]) + return status +} diff --git a/web/src/api/jobs.ts b/web/src/api/jobs.ts index e9fac9f..96415b1 100644 --- a/web/src/api/jobs.ts +++ b/web/src/api/jobs.ts @@ -4,15 +4,18 @@ import type { components } from './schema' export type CoversJobStatus = components['schemas']['CoversJobStatus'] export type ArtistJobStatus = components['schemas']['ArtistJobStatus'] +export type DuplicatesCheckStatus = components['schemas']['DuplicatesCheckStatus'] -/** The bulk jobs that go through the whole library, each at its own address under `/library`. */ +/** The bulk jobs that go through the whole library with the providers, each at its own address under `/library`. */ export type JobKind = 'covers' | 'artist-images' | 'descriptions' -export type JobStatus = K extends 'covers' ? CoversJobStatus : ArtistJobStatus +/** Every job that is started, followed and stopped the same way: the bulk ones, and the check of the duplicates. */ +export type JobPath = JobKind | 'duplicates/check' +export type JobStatus = K extends 'covers' ? CoversJobStatus : K extends 'duplicates/check' ? DuplicatesCheckStatus : ArtistJobStatus -const statusKey = (kind: JobKind) => ['library', kind] +const statusKey = (kind: JobPath) => ['library', kind] /** Whether the job runs, how far it is, and how the last one ended. While it runs it is read again every `intervalMs`. */ -export function useJobStatus(kind: K, { intervalMs = 2000 }: { intervalMs?: number } = {}) { +export function useJobStatus(kind: K, { intervalMs = 2000 }: { intervalMs?: number } = {}) { return useQuery({ queryKey: statusKey(kind), queryFn: () => api.get>(`/library/${kind}`), @@ -24,7 +27,7 @@ export function useJobStatus(kind: K, { intervalMs = 2000 }: * Starts the job in the background. `refresh` goes through every item again, also those with a good answer. * Whatever the answer (also "already running"), the status is read at once so the page follows the job. */ -export function useStartJob(kind: JobKind) { +export function useStartJob(kind: JobPath) { const client = useQueryClient() return useMutation({ mutationFn: ({ refresh }: { refresh: boolean }) => api.post<{ status: 'started' }>(refresh ? `/library/${kind}?refresh=true` : `/library/${kind}`), @@ -33,7 +36,7 @@ export function useStartJob(kind: JobKind) { } /** Asks the job to end after the item it is on. What it already did stays. */ -export function useStopJob(kind: JobKind) { +export function useStopJob(kind: JobPath) { const client = useQueryClient() return useMutation({ mutationFn: () => api.del(`/library/${kind}`), diff --git a/web/src/api/schema.d.ts b/web/src/api/schema.d.ts index 05ef681..1aa7261 100644 --- a/web/src/api/schema.d.ts +++ b/web/src/api/schema.d.ts @@ -443,6 +443,110 @@ export interface paths { patch?: never; trace?: never; }; + "/library/duplicates": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * The songs that are in more than one file + * @description Admin only. Groups of files that are the same song by the same artists, worked out from the index and + * never from the disk, so it is quick and touches no file. Two files are the same song when they have the + * same set of main artists, the same title (the key Sonda matches titles by) and durations no more than + * 3 seconds apart, every pair in a group within that. A file with no duration or no artist never matches, + * a file flagged as missing does not count, and a copy may be on a different album: `files` says which. + * + * Each group says which copy Sonda would keep (`suggested_keep_track_id`): the one without loss (FLAC) + * before one with loss, then the higher bitrate, then the bigger file, then the oldest. `hashed` is true when + * every file of the group has been read by the check (`POST /library/duplicates/check`) and `identical` when, + * besides, they are byte for byte the same. No file path is ever returned. + * + * `deletion_allowed` says whether `POST /library/duplicates/clean` can work: with `library.allow_delete` + * off (the default) the groups can still be seen. + */ + get: operations["listDuplicates"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/library/duplicates/check": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Progress of the check that reads the files of the duplicates + * @description Whether the check is running, how far it is (`progress`, only while it runs), and how the last one ended. + * The check is described at `POST /library/duplicates/check`. + */ + get: operations["getDuplicatesCheck"]; + put?: never; + /** + * Read the files of the duplicates, to tell which are identical + * @description Admin only. Starts, in the background and one at a time like a scan, a job that reads every file that is in + * a group of copies and keeps its SHA-256, so that a group of files that are byte for byte the same can be + * called `identical`. It only reads: nothing is removed or sent anywhere. **It reads whole files**, which on a + * network drive takes a while, so it is started on purpose. A file that has a fingerprint is skipped, and the + * fingerprint of a file is forgotten when a scan finds the file changed; send `refresh=true` to read every file + * again. A file that changed since the scan is skipped. It gives up, and says why in `last_error`, when files + * cannot be read five times in a row (a disk that is not mounted). `DELETE /library/duplicates/check` stops it. + */ + post: operations["startDuplicatesCheck"]; + /** + * Stop the check of the duplicates + * @description Admin only. Asks a running check to end after the file it is on; the fingerprints already kept stay. It + * answers the same when no check is running. + */ + delete: operations["stopDuplicatesCheck"]; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/library/duplicates/clean": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Delete the extra copies of one song + * @description Admin only. **Deletes files from the music folder.** It is the one thing Sonda does that writes there, so it + * is off until the operator turns on `library.allow_delete` (answer `deletion_disabled` otherwise), the folder + * has to be writable (`library_read_only`), and it is never done by itself: one request removes the copies + * `remove_track_ids` of one song and keeps `keep_track_id`. + * + * The server decides again, whatever the client saw, and removes nothing unless every check passes: no scan is + * running (`scan_running`); the tracks exist (`track_not_found`); they are copies of one song by the rule of + * `GET /library/duplicates` (`invalid_input` otherwise, and also when the copy to keep is among those to + * remove, a copy is named twice or there are none or more than 100); and the copy to keep and every copy to + * remove are still the file the last scan saw, a regular file with the same size and date (`duplicate_changed` + * for a file that changed, is gone, became a link or is outside the library: scan again). So the last copy of + * a song is never removed. Every file is checked before the first is removed. + * + * Each copy is removed from the disk through the library folder (a path cannot leave it, not even through a + * link) and its track leaves the index; **the listens of the removed track become listens of the track that + * stays**, so the taste is not lost. Folders are never removed, not even empty ones. If it fails part way (for + * example the folder turns out to be read-only for the second file), what was removed stays removed. + */ + post: operations["cleanDuplicates"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/library/scan": { parameters: { query?: never; @@ -2072,6 +2176,101 @@ export interface components { /** @description Why the last job ended before the end, for example that a provider is off or kept failing. */ last_error?: string; }; + DuplicatesPage: { + /** @description Whether `POST /library/duplicates/clean` can work (`library.allow_delete`). */ + deletion_allowed: boolean; + /** @description Groups in all, not in this page. */ + total: number; + limit: number; + offset: number; + groups: components["schemas"]["DuplicateGroup"][]; + }; + DuplicateGroup: { + title: string; + /** @description The main artists the copies share. */ + artists: components["schemas"]["ArtistRef"][]; + /** @description The copies, by track id; at least two. */ + files: components["schemas"]["DuplicateFile"][]; + /** @description Every file of the group has been read by the check. */ + hashed: boolean; + /** @description Hashed, and byte for byte the same. */ + identical: boolean; + /** + * Format: int64 + * @description The copy Sonda would keep: without loss first, then the higher bitrate, then the bigger file, then the oldest. + */ + suggested_keep_track_id: number; + /** + * Format: int64 + * @description The size of every copy but the one to keep. + */ + freed_bytes: number; + }; + DuplicateFile: { + /** Format: int64 */ + track_id: number; + album: { + /** Format: int64 */ + id: number; + title: string; + } | null; + /** @description `mp3`, `flac`... */ + format: string; + /** @description 0 when unknown. */ + bitrate: number; + /** Format: int64 */ + size_bytes: number; + /** Format: int64 */ + duration_ms: number; + /** @description This file has been read by the check. */ + hashed: boolean; + }; + CleanDuplicates: { + /** + * Format: int64 + * @description The copy that stays. + */ + keep_track_id: number; + /** @description The copies to delete, of the same song as the one that stays, none twice. */ + remove_track_ids: number[]; + }; + DuplicatesCleaned: { + /** @description Files deleted. */ + removed: number; + /** Format: int64 */ + freed_bytes: number; + }; + DuplicatesCheckStatus: { + running: boolean; + /** + * Format: date-time + * @description When the last check started. + */ + started_at?: string; + /** Format: date-time */ + last_finished_at?: string; + /** @description Only while the check runs. */ + progress?: { + done: number; + /** @description Files the check set out to read. */ + total: number; + }; + /** @description How the last finished check ended. */ + last?: { + /** @description Files looked at. */ + checked: number; + /** @description Files read and fingerprinted. */ + hashed: number; + /** @description Files that changed since the scan, or are gone. */ + skipped: number; + /** @description Files that could not be read just then. */ + failed: number; + /** @description It was stopped, or the application was. */ + stopped: boolean; + }; + /** @description Why the last check ended before the end, for example that the disk is not mounted. */ + last_error?: string; + }; UserSettings: { /** * @description When true, Discover may start from the artists you listen to most. That sends their @@ -3063,6 +3262,170 @@ export interface operations { 403: components["responses"]["AdminRequired"]; }; }; + listDuplicates: { + parameters: { + query?: { + /** @description Page size. 50 when absent or 0; values above 200 are lowered to 200. */ + limit?: components["parameters"]["Limit"]; + offset?: components["parameters"]["Offset"]; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description A page of groups, in a fixed order (by title). */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DuplicatesPage"]; + }; + }; + 400: components["responses"]["InvalidInput"]; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["AdminRequired"]; + }; + }; + getDuplicatesCheck: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description The job. */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DuplicatesCheckStatus"]; + }; + }; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["AdminRequired"]; + }; + }; + startDuplicatesCheck: { + parameters: { + query?: { + /** @description `true` reads every file again, also those that have a fingerprint. `false` when absent. */ + refresh?: boolean; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description The check was started in the background. */ + 202: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + /** @enum {string} */ + status: "started"; + }; + }; + }; + 400: components["responses"]["InvalidInput"]; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["AdminRequired"]; + /** @description The check is already running (`duplicates_check_running`). */ + 409: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Error"]; + }; + }; + /** @description The groups could not be worked out (`internal`). */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Error"]; + }; + }; + }; + }; + stopDuplicatesCheck: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Done, also when no check was running. */ + 204: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["AdminRequired"]; + }; + }; + cleanDuplicates: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["CleanDuplicates"]; + }; + }; + responses: { + /** @description The extra copies were removed. */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DuplicatesCleaned"]; + }; + }; + 400: components["responses"]["InvalidInput"]; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["AdminRequired"]; + /** @description A track that was named does not exist (`track_not_found`). */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Error"]; + }; + }; + /** + * @description Deleting is not turned on (`deletion_disabled`), a scan is running (`scan_running`), a file is not what the last + * scan saw (`duplicate_changed`), or the music folder cannot be written to (`library_read_only`). + */ + 409: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Error"]; + }; + }; + }; + }; getScanStatus: { parameters: { query?: never; diff --git a/web/src/i18n/locales/en.json b/web/src/i18n/locales/en.json index d7b2010..2e183db 100644 --- a/web/src/i18n/locales/en.json +++ b/web/src/i18n/locales/en.json @@ -71,6 +71,7 @@ "sectionUsers": "Users", "sectionProviders": "Providers", "sectionJobs": "Jobs", + "sectionDuplicates": "Duplicates", "jobs": { "intro": "These jobs go through the whole library in the background, one item after the other. Each card says what it sends and to whom.", "needs": "Needs {{names}} turned on: use the Providers section.", @@ -129,6 +130,69 @@ } } }, + "duplicates": { + "intro": "Songs that are in more than one file: the same artists, the same title and almost the same length, even when they are on different albums. All of this is worked out on this server; nothing leaves it.", + "off": "Deleting is turned off. To delete copies, set library.allow_delete (or SONDA_ALLOW_DELETE=true), mount the music folders so that Sonda can write to them, and restart the server.", + "check": { + "title": "Check the candidates", + "what": "Reads the files of the songs below and keeps a fingerprint (SHA-256) of each one, to tell which copies are byte for byte the same file. It takes a while on a big library.", + "sends": "It only reads your files. Nothing leaves this server.", + "start": "Start the check", + "stop": "Stop the check", + "progress": "{{done}} of {{total}} files", + "never": "The check has not run yet.", + "counts": { + "checked": "{{count}} looked at", + "hashed": "{{count}} read", + "skipped": "{{count}} skipped (changed or gone)", + "failed": "{{count}} could not be read" + } + }, + "none": "No duplicates found.", + "count_one": "{{count}} song is in more than one file", + "count_other": "{{count}} songs are in more than one file", + "cleanAll": "Clean all suggestions", + "cleanAllShown": "Only the songs shown below are cleaned. Show more to include the rest.", + "keepColumn": "Keep", + "album": "Album", + "format": "Format", + "bitrate": "Bitrate", + "size": "Size", + "noAlbum": "No album", + "kbps": "{{count}} kbps", + "suggested": "Suggested", + "keepCopy": "Keep {{what}}", + "status": { + "identical": "The files are byte for byte the same.", + "different": "The files are not the same, even if the song is.", + "unchecked": "Not checked yet: run the check to see if they are the same file." + }, + "deleteOthers": "Delete the others", + "deleteOthersOf": "Delete the other copies of {{title}}", + "confirmOneTitle_one": "Delete {{count}} copy of {{title}}?", + "confirmOneTitle_other": "Delete {{count}} copies of {{title}}?", + "confirmOneBody": "The files below are deleted from the disk for good, {{size}} freed. The listens and playlists that had them move to the copy that stays.", + "confirmAllTitle_one": "Clean {{count}} song?", + "confirmAllTitle_other": "Clean {{count}} songs?", + "confirmAllBody_one": "In each song the copy chosen stays and the others are deleted from the disk for good: {{count}} file, {{size}} freed. The listens and playlists move to the copy that stays.", + "confirmAllBody_other": "In each song the copy chosen stays and the others are deleted from the disk for good: {{count}} files, {{size}} freed. The listens and playlists move to the copy that stays.", + "confirmDelete_one": "Delete {{count}} file", + "confirmDelete_other": "Delete {{count}} files", + "stays": "Stays: {{what}}", + "goes": "Delete: {{what}}", + "deleting": "Deleting files…", + "deleted_one": "Deleted {{count}} file, {{size}} freed.", + "deleted_other": "Deleted {{count}} files, {{size}} freed.", + "errors": { + "duplicates_check_running": "The check is already running.", + "scan_running": "A scan is running. Try again when it ends.", + "deletion_disabled": "Deleting is turned off on the server.", + "duplicate_changed": "A file changed since the last scan, so nothing was deleted for that song. Run a scan and look again.", + "library_read_only": "The music folder is read-only, so nothing was deleted. Mount it so that Sonda can write to it.", + "invalid_input": "Those files are no longer copies of the same song, so nothing was deleted. Look at the list again.", + "track_not_found": "A song is already gone, so nothing was deleted for it. Look at the list again." + } + }, "cancel": "Cancel", "save": "Save", "errors": { diff --git a/web/src/i18n/locales/it.json b/web/src/i18n/locales/it.json index d7c7fd7..e56c972 100644 --- a/web/src/i18n/locales/it.json +++ b/web/src/i18n/locales/it.json @@ -71,6 +71,7 @@ "sectionUsers": "Utenti", "sectionProviders": "Provider", "sectionJobs": "Lavori", + "sectionDuplicates": "Duplicati", "jobs": { "intro": "Questi lavori passano in rassegna tutta la libreria in background, un elemento alla volta. Ogni scheda dice cosa manda e a chi.", "needs": "Serve che {{names}} sia acceso: vai nella sezione Provider.", @@ -129,6 +130,69 @@ } } }, + "duplicates": { + "intro": "Canzoni che stanno in più di un file: stessi artisti, stesso titolo e durata quasi uguale, anche se sono su album diversi. Tutto viene calcolato su questo server; niente ne esce.", + "off": "La cancellazione è spenta. Per cancellare le copie imposta library.allow_delete (o SONDA_ALLOW_DELETE=true), monta le cartelle della musica in modo che Sonda possa scriverci e riavvia il server.", + "check": { + "title": "Controlla i candidati", + "what": "Legge i file delle canzoni qui sotto e ne tiene un'impronta (SHA-256), per capire quali copie sono lo stesso file byte per byte. Su una libreria grande ci vuole un po'.", + "sends": "Legge soltanto i tuoi file. Dal server non esce niente.", + "start": "Avvia il controllo", + "stop": "Ferma il controllo", + "progress": "{{done}} file su {{total}}", + "never": "Il controllo non è ancora stato eseguito.", + "counts": { + "checked": "{{count}} esaminati", + "hashed": "{{count}} letti", + "skipped": "{{count}} saltati (cambiati o spariti)", + "failed": "{{count}} non leggibili" + } + }, + "none": "Nessun duplicato trovato.", + "count_one": "{{count}} canzone è in più di un file", + "count_other": "{{count}} canzoni sono in più di un file", + "cleanAll": "Pulisci tutti i suggerimenti", + "cleanAllShown": "Vengono pulite solo le canzoni mostrate qui sotto. Mostra altri per includere il resto.", + "keepColumn": "Tieni", + "album": "Album", + "format": "Formato", + "bitrate": "Bitrate", + "size": "Dimensione", + "noAlbum": "Nessun album", + "kbps": "{{count}} kbps", + "suggested": "Suggerita", + "keepCopy": "Tieni {{what}}", + "status": { + "identical": "I file sono uguali byte per byte.", + "different": "I file non sono uguali, anche se la canzone sì.", + "unchecked": "Non ancora controllato: avvia il controllo per vedere se sono lo stesso file." + }, + "deleteOthers": "Cancella le altre", + "deleteOthersOf": "Cancella le altre copie di {{title}}", + "confirmOneTitle_one": "Cancellare {{count}} copia di {{title}}?", + "confirmOneTitle_other": "Cancellare {{count}} copie di {{title}}?", + "confirmOneBody": "I file qui sotto vengono cancellati dal disco per sempre, {{size}} liberati. Gli ascolti e le playlist che li avevano passano alla copia che resta.", + "confirmAllTitle_one": "Pulire {{count}} canzone?", + "confirmAllTitle_other": "Pulire {{count}} canzoni?", + "confirmAllBody_one": "Per ogni canzone resta la copia scelta e le altre vengono cancellate dal disco per sempre: {{count}} file, {{size}} liberati. Gli ascolti e le playlist passano alla copia che resta.", + "confirmAllBody_other": "Per ogni canzone resta la copia scelta e le altre vengono cancellate dal disco per sempre: {{count}} file, {{size}} liberati. Gli ascolti e le playlist passano alla copia che resta.", + "confirmDelete_one": "Cancella {{count}} file", + "confirmDelete_other": "Cancella {{count}} file", + "stays": "Resta: {{what}}", + "goes": "Da cancellare: {{what}}", + "deleting": "Cancello i file…", + "deleted_one": "Cancellato {{count}} file, {{size}} liberati.", + "deleted_other": "Cancellati {{count}} file, {{size}} liberati.", + "errors": { + "duplicates_check_running": "Il controllo è già in corso.", + "scan_running": "È in corso una scansione. Riprova quando finisce.", + "deletion_disabled": "La cancellazione è spenta sul server.", + "duplicate_changed": "Un file è cambiato dall'ultima scansione, quindi per quella canzone non è stato cancellato niente. Avvia una scansione e guarda di nuovo.", + "library_read_only": "La cartella della musica è in sola lettura, quindi non è stato cancellato niente. Montala in modo che Sonda possa scriverci.", + "invalid_input": "Quei file non sono più copie della stessa canzone, quindi non è stato cancellato niente. Guarda di nuovo l'elenco.", + "track_not_found": "Una canzone è già sparita, quindi per lei non è stato cancellato niente. Guarda di nuovo l'elenco." + } + }, "cancel": "Annulla", "save": "Salva", "errors": { diff --git a/web/src/lib/size.test.ts b/web/src/lib/size.test.ts new file mode 100644 index 0000000..82494e4 --- /dev/null +++ b/web/src/lib/size.test.ts @@ -0,0 +1,20 @@ +import { expect, test } from 'vitest' +import { formatBytes } from './size' + +test.each([ + [0, 'en', '0 B'], + [999, 'en', '999 B'], + [1000, 'en', '1 kB'], + [1_500, 'en', '1.5 kB'], + [42_000_000, 'en', '42 MB'], + [4_320_000_000, 'en', '4.3 GB'], + [4_320_000_000, 'it', '4,3 GB'], + [2_000_000_000_000, 'en', '2 TB'], +])('%i bytes are written %s in %s', (bytes, lang, text) => { + expect(formatBytes(bytes, lang)).toBe(text) +}) + +test('anything that is not a size is 0 B', () => { + expect(formatBytes(-5, 'en')).toBe('0 B') + expect(formatBytes(Number.NaN, 'en')).toBe('0 B') +}) diff --git a/web/src/lib/size.ts b/web/src/lib/size.ts new file mode 100644 index 0000000..8c183a5 --- /dev/null +++ b/web/src/lib/size.ts @@ -0,0 +1,10 @@ +const UNITS = ['B', 'kB', 'MB', 'GB', 'TB'] + +/** A size in bytes as people read it ("4.3 GB"): units of 1000, one decimal at most, the number in the reader's language. */ +export function formatBytes(bytes: number, language?: string): string { + if (!Number.isFinite(bytes) || bytes <= 0) return '0 B' + const unit = Math.min(Math.floor(Math.log10(bytes) / 3), UNITS.length - 1) + const value = bytes / 1000 ** unit + const digits = unit === 0 ? 0 : 1 + return `${new Intl.NumberFormat(language, { maximumFractionDigits: digits }).format(value)} ${UNITS[unit]}` +} diff --git a/web/src/pages/admin/AdminLayout.test.tsx b/web/src/pages/admin/AdminLayout.test.tsx index 6c190f8..4009e7d 100644 --- a/web/src/pages/admin/AdminLayout.test.tsx +++ b/web/src/pages/admin/AdminLayout.test.tsx @@ -32,6 +32,12 @@ test('an admin gets the admin page with its sections and the page of the section expect(screen.getByText('the libraries page')).toBeInTheDocument() }) +test('the sections include the duplicates, at their own address', () => { + sessionStore.setState({ status: 'authenticated', user: { id: 1, email: 'tom@example.org', is_admin: true } }) + renderAdmin() + expect(screen.getByRole('link', { name: 'Duplicates' })).toHaveAttribute('href', '/admin/duplicates') +}) + test('anyone else sees a notice, not the section, and nothing is asked of the server', async () => { sessionStore.setState({ status: 'authenticated', user: { id: 2, email: 'ann@example.org', is_admin: false } }) const { calls } = mockFetch({}) diff --git a/web/src/pages/admin/AdminLayout.tsx b/web/src/pages/admin/AdminLayout.tsx index 0b87de9..b423ddb 100644 --- a/web/src/pages/admin/AdminLayout.tsx +++ b/web/src/pages/admin/AdminLayout.tsx @@ -10,6 +10,7 @@ const SECTIONS = [ { to: '/admin/users', key: 'admin.sectionUsers' }, { to: '/admin/providers', key: 'admin.sectionProviders' }, { to: '/admin/jobs', key: 'admin.sectionJobs' }, + { to: '/admin/duplicates', key: 'admin.sectionDuplicates' }, ] /** diff --git a/web/src/pages/admin/Confirm.tsx b/web/src/pages/admin/Confirm.tsx index 8737c4c..a31146d 100644 --- a/web/src/pages/admin/Confirm.tsx +++ b/web/src/pages/admin/Confirm.tsx @@ -1,3 +1,4 @@ +import type { ReactNode } from 'react' import { useTranslation } from 'react-i18next' import { Button } from '@/components/ui/button' import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' @@ -7,12 +8,15 @@ export default function Confirm({ title, body, confirm, + children, onConfirm, onClose, }: { title: string body: string confirm: string + /** More to read before answering (a list of what goes), under the body. */ + children?: ReactNode onConfirm: () => void onClose: () => void }) { @@ -24,6 +28,7 @@ export default function Confirm({ {title} {body} + {children} + + + ) +} diff --git a/web/src/pages/admin/DuplicatesCheck.test.tsx b/web/src/pages/admin/DuplicatesCheck.test.tsx new file mode 100644 index 0000000..f7df26c --- /dev/null +++ b/web/src/pages/admin/DuplicatesCheck.test.tsx @@ -0,0 +1,94 @@ +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { render, screen, waitFor, within } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { beforeEach, expect, test, vi } from 'vitest' +import { sessionStore } from '@/auth/session' +import { initI18n } from '@/i18n' +import { apiError, mockFetch, type MockRoutes } from '@/test/mockFetch' +import DuplicatesCheck from './DuplicatesCheck' + +const PATH = '/api/v1/library/duplicates/check' +const started = () => new Response(JSON.stringify({ status: 'started' }), { status: 202, headers: { 'Content-Type': 'application/json' } }) +const none = () => new Response(null, { status: 204 }) + +function renderCard() { + const client = new QueryClient({ defaultOptions: { queries: { retry: false }, mutations: { retry: false } } }) + return render( + + + , + ) +} + +const routes = (status: object, over: MockRoutes = {}): MockRoutes => ({ [`GET ${PATH}`]: () => status, ...over }) +const card = () => screen.findByRole('region', { name: 'Check the candidates' }) + +beforeEach(async () => { + vi.spyOn(navigator, 'language', 'get').mockReturnValue('en-US') + await initI18n() + sessionStore.setState({ status: 'authenticated', user: { id: 1, email: 'tom@example.org', is_admin: true } }) +}) + +test('it says what the check does and that nothing leaves the server', async () => { + mockFetch(routes({ running: false })) + renderCard() + const c = await card() + expect(await within(c).findByText('The check has not run yet.')).toBeInTheDocument() + expect(c).toHaveTextContent('byte for byte') + expect(c).toHaveTextContent('Nothing leaves this server.') +}) + +test('starting the check is one click, with no question, because it only reads', async () => { + const { calls } = mockFetch(routes({ running: false }, { [`POST ${PATH}`]: started })) + renderCard() + await userEvent.click(within(await card()).getByRole('button', { name: 'Start the check' })) + await waitFor(() => expect(calls.filter((c) => c.key === `POST ${PATH}`)).toHaveLength(1)) + expect(screen.queryByRole('dialog')).not.toBeInTheDocument() +}) + +test('while it runs it shows how far it is, offers to stop it and cannot be started again', async () => { + const { calls } = mockFetch(routes({ running: true, progress: { done: 3, total: 12 } }, { [`DELETE ${PATH}`]: none })) + renderCard() + const c = await card() + expect(await within(c).findByRole('progressbar')).toHaveAttribute('aria-valuenow', '3') + expect(within(c).getByRole('progressbar')).toHaveAttribute('aria-valuemax', '12') + expect(within(c).getByText('3 of 12 files')).toBeInTheDocument() + expect(within(c).getByRole('button', { name: 'Start the check' })).toBeDisabled() + await userEvent.click(within(c).getByRole('button', { name: 'Stop the check' })) + await waitFor(() => expect(calls.filter((x) => x.key === `DELETE ${PATH}`)).toHaveLength(1)) +}) + +test('when it is not running there is no stop button', async () => { + mockFetch(routes({ running: false })) + renderCard() + await card() + expect(screen.queryByRole('button', { name: 'Stop the check' })).not.toBeInTheDocument() +}) + +test('the last run is summed up, leaving out what is zero except what was looked at', async () => { + mockFetch(routes({ running: false, last_finished_at: '2026-11-12T10:00:00Z', last: { checked: 10, hashed: 8, skipped: 2, failed: 0, stopped: false } })) + renderCard() + const list = await within(await card()).findByRole('list', { name: 'Last run' }) + expect(within(list).getAllByRole('listitem').map((li) => li.textContent)).toEqual(['10 looked at', '8 read', '2 skipped (changed or gone)']) +}) + +test('a check that was stopped says so, and one that ended with an error shows why', async () => { + mockFetch(routes({ running: false, last: { checked: 3, hashed: 3, skipped: 0, failed: 0, stopped: true }, last_error: 'the disk is not mounted' })) + renderCard() + const c = await card() + expect(await within(c).findByText('It was stopped before the end.')).toBeInTheDocument() + expect(within(c).getByRole('alert')).toHaveTextContent('the disk is not mounted') +}) + +test('a check that is already running is said in words', async () => { + mockFetch(routes({ running: false }, { [`POST ${PATH}`]: () => apiError(409, 'duplicates_check_running') })) + renderCard() + await userEvent.click(within(await card()).getByRole('button', { name: 'Start the check' })) + expect(await screen.findByRole('alert')).toHaveTextContent('The check is already running.') +}) + +test('when the status cannot be read it says so and offers to try again', async () => { + mockFetch(routes({}, { [`GET ${PATH}`]: () => apiError(500, 'internal') })) + renderCard() + expect(await within(await card()).findByRole('button', { name: 'Try again' })).toBeInTheDocument() +}) diff --git a/web/src/pages/admin/DuplicatesCheck.tsx b/web/src/pages/admin/DuplicatesCheck.tsx new file mode 100644 index 0000000..8db4002 --- /dev/null +++ b/web/src/pages/admin/DuplicatesCheck.tsx @@ -0,0 +1,98 @@ +import { useTranslation } from 'react-i18next' +import { useDuplicatesCheck } from '@/api/duplicates' +import { useStartJob, useStopJob } from '@/api/jobs' +import { ErrorMessage, Loading } from '@/components/Status' +import { Button } from '@/components/ui/button' +import { errorText } from './errorText' + +const COUNTS = ['checked', 'hashed', 'skipped', 'failed'] as const + +/** + * The check that reads the files of the groups and keeps a fingerprint of each, so a group can say whether its + * copies are byte for byte the same. It only reads, so it starts without a question. The status follows the + * job by itself while it runs, and the groups are read again when it ends. + */ +export default function DuplicatesCheck() { + const { t, i18n } = useTranslation() + const status = useDuplicatesCheck() + const start = useStartJob('duplicates/check') + const stop = useStopJob('duplicates/check') + const title = t('admin.duplicates.check.title') + const running = status.data?.running === true + const progress = status.data?.progress + const last = status.data?.last + const failure = start.error ?? stop.error + + return ( +
+

{title}

+

{t('admin.duplicates.check.what')}

+

{t('admin.duplicates.check.sends')}

+ +
+ + {running && ( + + )} +
+ + {failure ? ( +

+ {errorText(t, failure, ['admin.duplicates.errors', 'admin.errors'])} +

+ ) : null} + + {status.isError ? ( + void status.refetch()} /> + ) : status.isPending ? ( + + ) : running ? ( + progress && ( +
+
+
0 ? Math.min(100, (progress.done / progress.total) * 100) : 0}%` }} /> +
+

+ {t('admin.duplicates.check.progress', { done: progress.done, total: progress.total })} +

+
+ ) + ) : ( + <> + {last && ( +
+
    + {COUNTS.filter((key) => key === 'checked' || last[key] > 0).map((key) => ( +
  • + {t(`admin.duplicates.check.counts.${key}`, { count: last[key] })} +
  • + ))} +
+ {last.stopped &&

{t('admin.jobs.stopped')}

} + {status.data.last_finished_at && ( +

{t('admin.jobs.finished', { when: new Date(status.data.last_finished_at).toLocaleString(i18n.language) })}

+ )} +
+ )} + {status.data.last_error && ( +

+ {status.data.last_error} +

+ )} + {!last && !status.data.last_error &&

{t('admin.duplicates.check.never')}

} + + )} +
+ ) +} diff --git a/web/src/pages/admin/DuplicatesPage.test.tsx b/web/src/pages/admin/DuplicatesPage.test.tsx new file mode 100644 index 0000000..0e46384 --- /dev/null +++ b/web/src/pages/admin/DuplicatesPage.test.tsx @@ -0,0 +1,323 @@ +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { render, screen, waitFor, within } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { beforeEach, expect, test, vi } from 'vitest' +import { sessionStore } from '@/auth/session' +import { initI18n } from '@/i18n' +import { apiError, mockFetch, type MockRoutes, type RecordedCall } from '@/test/mockFetch' +import DuplicatesPage from './DuplicatesPage' + +const file = (trackId: number, album: string | null, format: string, bitrate: number, size: number, hashed = true) => ({ + track_id: trackId, + album: album === null ? null : { id: trackId, title: album }, + format, + bitrate, + size_bytes: size, + duration_ms: 200_000, + hashed, +}) + +// Song A: a flac and an mp3 that are not the same file. Song B: two copies that are byte for byte the same. +// Song C: copies the check has not read yet. +const groupA = { + title: 'Song A', + artists: [{ id: 1, name: 'Ann' }], + files: [file(1, 'Album One', 'flac', 900, 30_000_000), file(2, 'Greatest Hits', 'mp3', 192, 8_000_000)], + hashed: true, + identical: false, + suggested_keep_track_id: 1, + freed_bytes: 8_000_000, +} +const groupB = { + title: 'Song B', + artists: [{ id: 1, name: 'Ann' }, { id: 2, name: 'Bob' }], + files: [file(3, 'Album X', 'mp3', 192, 5_000_000), file(4, 'Album Y', 'mp3', 192, 5_000_000)], + hashed: true, + identical: true, + suggested_keep_track_id: 3, + freed_bytes: 5_000_000, +} +const groupC = { + title: 'Song C', + artists: [{ id: 1, name: 'Ann' }], + files: [file(5, null, 'mp3', 0, 4_000_000, false), file(6, 'Single', 'mp3', 128, 3_000_000, false)], + hashed: false, + identical: false, + suggested_keep_track_id: 5, + freed_bytes: 3_000_000, +} + +const page = (groups: object[], over: object = {}) => ({ deletion_allowed: true, total: groups.length, limit: 50, offset: 0, groups, ...over }) + +function routes(groups: object[], over: MockRoutes = {}, pageOver: object = {}): MockRoutes { + return { + 'GET /api/v1/library/duplicates': () => page(groups, pageOver), + 'GET /api/v1/library/duplicates/check': () => ({ running: false }), + ...over, + } +} + +function renderPage() { + const client = new QueryClient({ defaultOptions: { queries: { retry: false }, mutations: { retry: false } } }) + return render( + + + , + ) +} + +const group = (name: string) => screen.findByRole('region', { name }) +const posts = (calls: RecordedCall[]) => calls.filter((c) => c.key === 'POST /api/v1/library/duplicates/clean').map((c) => JSON.parse(String(c.init.body))) + +beforeEach(async () => { + vi.spyOn(navigator, 'language', 'get').mockReturnValue('en-US') + await initI18n() + sessionStore.setState({ status: 'authenticated', user: { id: 1, email: 'tom@example.org', is_admin: true } }) +}) + +// --- what is shown ----------------------------------------------------------------------------- + +test('with no duplicates it says so', async () => { + mockFetch(routes([])) + renderPage() + expect(await screen.findByText('No duplicates found.')).toBeInTheDocument() + expect(screen.queryByRole('button', { name: 'Clean all suggestions' })).not.toBeInTheDocument() +}) + +test('a group shows the song, the artists and a row for each copy with its album, format, bitrate and size', async () => { + mockFetch(routes([groupA, groupB])) + renderPage() + const a = await group('Song A') + expect(a).toHaveTextContent('Ann') + const rows = within(within(a).getByRole('table')).getAllByRole('row').slice(1) + expect(rows).toHaveLength(2) + expect(rows[0]).toHaveTextContent('Album One') + expect(rows[0]).toHaveTextContent('flac') + expect(rows[0]).toHaveTextContent('900 kbps') + expect(rows[0]).toHaveTextContent('30 MB') + expect(rows[1]).toHaveTextContent('Greatest Hits') + expect(rows[1]).toHaveTextContent('192 kbps') + expect(rows[1]).toHaveTextContent('8 MB') + expect(await group('Song B')).toHaveTextContent('Ann, Bob') +}) + +test('a copy with no album or no bitrate says so, and the page says how many songs and how much space', async () => { + mockFetch(routes([groupC], {}, { total: 7 })) + renderPage() + const c = await group('Song C') + const rows = within(c).getAllByRole('row').slice(1) + expect(rows[0]).toHaveTextContent('No album') + expect(rows[0]).not.toHaveTextContent('kbps') + expect(screen.getByText('7 songs are in more than one file')).toBeInTheDocument() +}) + +test('each group says whether its copies are the same file, were not the same, or have not been checked', async () => { + mockFetch(routes([groupA, groupB, groupC])) + renderPage() + expect(await group('Song B')).toHaveTextContent('The files are byte for byte the same.') + expect(screen.getByRole('region', { name: 'Song A' })).toHaveTextContent('The files are not the same, even if the song is.') + expect(screen.getByRole('region', { name: 'Song C' })).toHaveTextContent('Not checked yet: run the check to see if they are the same file.') +}) + +test('the copy the server suggests is chosen and marked, and another can be chosen', async () => { + mockFetch(routes([groupA])) + renderPage() + const a = await group('Song A') + const flac = within(a).getByRole('radio', { name: /Keep Album One/ }) + const mp3 = within(a).getByRole('radio', { name: /Keep Greatest Hits/ }) + expect(flac).toBeChecked() + expect(a).toHaveTextContent('Suggested') + await userEvent.click(mp3) + expect(mp3).toBeChecked() + expect(flac).not.toBeChecked() +}) + +test('more groups are loaded on request', async () => { + mockFetch({ + ...routes([]), + 'GET /api/v1/library/duplicates': ({ url }) => + url.searchParams.get('offset') === '0' ? page([groupA], { total: 2 }) : page([groupB], { total: 2, offset: 1 }), + }) + renderPage() + await group('Song A') + expect(screen.queryByRole('region', { name: 'Song B' })).not.toBeInTheDocument() + await userEvent.click(screen.getByRole('button', { name: 'Show more' })) + expect(await group('Song B')).toBeInTheDocument() +}) + +// --- deleting turned off ----------------------------------------------------------------------- + +test('with deleting turned off everything is shown, the delete buttons are disabled and the reason is on the page', async () => { + const { calls } = mockFetch(routes([groupA, groupB], {}, { deletion_allowed: false })) + renderPage() + const a = await group('Song A') + expect(screen.getByText(/Deleting is turned off/)).toHaveTextContent('library.allow_delete') + const del = within(a).getByRole('button', { name: 'Delete the other copies of Song A' }) + expect(del).toBeDisabled() + expect(del).toHaveAccessibleDescription(/Deleting is turned off/) + expect(screen.getByRole('button', { name: 'Clean all suggestions' })).toBeDisabled() + await userEvent.click(del) + expect(posts(calls)).toEqual([]) +}) + +test('with deleting turned on there is no such notice', async () => { + mockFetch(routes([groupA])) + renderPage() + await group('Song A') + expect(screen.queryByText(/Deleting is turned off/)).not.toBeInTheDocument() +}) + +// --- deleting the others in one group ---------------------------------------------------------- + +test('deleting the others asks first, lists what stays and what goes, and sends nothing until it is confirmed', async () => { + const { calls } = mockFetch(routes([groupA], { 'POST /api/v1/library/duplicates/clean': () => ({ removed: 1, freed_bytes: 8_000_000 }) })) + renderPage() + const a = await group('Song A') + await userEvent.click(within(a).getByRole('button', { name: 'Delete the other copies of Song A' })) + + const dialog = await screen.findByRole('dialog', { name: 'Delete 1 copy of Song A?' }) + expect(dialog).toHaveTextContent('for good') + expect(within(dialog).getByText('Stays: Album One, flac, 900 kbps, 30 MB')).toBeInTheDocument() + expect(within(dialog).getByText('Delete: Greatest Hits, mp3, 192 kbps, 8 MB')).toBeInTheDocument() + expect(dialog).toHaveTextContent('8 MB freed') + expect(posts(calls)).toEqual([]) + + await userEvent.click(within(dialog).getByRole('button', { name: 'Cancel' })) + expect(posts(calls)).toEqual([]) + expect(screen.queryByRole('dialog')).not.toBeInTheDocument() +}) + +test('confirming deletes the others and keeps the copy chosen, then says what was deleted', async () => { + const { calls } = mockFetch(routes([groupA], { 'POST /api/v1/library/duplicates/clean': () => ({ removed: 1, freed_bytes: 30_000_000 }) })) + renderPage() + const a = await group('Song A') + await userEvent.click(within(a).getByRole('radio', { name: /Keep Greatest Hits/ })) + await userEvent.click(within(a).getByRole('button', { name: 'Delete the other copies of Song A' })) + const dialog = await screen.findByRole('dialog') + expect(within(dialog).getByText('Stays: Greatest Hits, mp3, 192 kbps, 8 MB')).toBeInTheDocument() + await userEvent.click(within(dialog).getByRole('button', { name: 'Delete 1 file' })) + + expect(await screen.findByText('Deleted 1 file, 30 MB freed.')).toBeInTheDocument() + expect(posts(calls)).toEqual([{ keep_track_id: 2, remove_track_ids: [1] }]) +}) + +test('a song with three copies sends the two that go', async () => { + const three = { ...groupA, files: [...groupA.files, file(9, 'Live', 'mp3', 128, 3_000_000)] } + const { calls } = mockFetch(routes([three], { 'POST /api/v1/library/duplicates/clean': () => ({ removed: 2, freed_bytes: 11_000_000 }) })) + renderPage() + const a = await group('Song A') + await userEvent.click(within(a).getByRole('button', { name: 'Delete the other copies of Song A' })) + const dialog = await screen.findByRole('dialog', { name: 'Delete 2 copies of Song A?' }) + await userEvent.click(within(dialog).getByRole('button', { name: 'Delete 2 files' })) + expect(await screen.findByText('Deleted 2 files, 11 MB freed.')).toBeInTheDocument() + expect(posts(calls)).toEqual([{ keep_track_id: 1, remove_track_ids: [2, 9] }]) +}) + +test('the groups are read again after a deletion', async () => { + let reads = 0 + mockFetch( + routes([groupA], { + 'GET /api/v1/library/duplicates': () => { + reads++ + return reads === 1 ? page([groupA]) : page([]) + }, + 'POST /api/v1/library/duplicates/clean': () => ({ removed: 1, freed_bytes: 8_000_000 }), + }), + ) + renderPage() + const a = await group('Song A') + await userEvent.click(within(a).getByRole('button', { name: 'Delete the other copies of Song A' })) + await userEvent.click(within(await screen.findByRole('dialog')).getByRole('button', { name: 'Delete 1 file' })) + expect(await screen.findByText('No duplicates found.')).toBeInTheDocument() + expect(screen.getByText('Deleted 1 file, 8 MB freed.')).toBeInTheDocument() +}) + +// --- cleaning every group ---------------------------------------------------------------------- + +test('cleaning all suggestions lists every song and the space freed in one question, then deletes song by song', async () => { + const { calls } = mockFetch( + routes([groupA, groupB], { + 'POST /api/v1/library/duplicates/clean': ({ init }) => ({ removed: 1, freed_bytes: JSON.parse(String(init.body)).keep_track_id === 1 ? 8_000_000 : 5_000_000 }), + }), + ) + renderPage() + await group('Song A') + await userEvent.click(screen.getByRole('button', { name: 'Clean all suggestions' })) + + const dialog = await screen.findByRole('dialog', { name: 'Clean 2 songs?' }) + expect(dialog).toHaveTextContent('2 files') + expect(dialog).toHaveTextContent('13 MB freed') + expect(within(dialog).getByText('Song A')).toBeInTheDocument() + expect(within(dialog).getByText('Song B')).toBeInTheDocument() + expect(within(dialog).getByText('Stays: Album One, flac, 900 kbps, 30 MB')).toBeInTheDocument() + expect(within(dialog).getByText('Delete: Album Y, mp3, 192 kbps, 5 MB')).toBeInTheDocument() + expect(posts(calls)).toEqual([]) + + await userEvent.click(within(dialog).getByRole('button', { name: 'Delete 2 files' })) + expect(await screen.findByText('Deleted 2 files, 13 MB freed.')).toBeInTheDocument() + expect(posts(calls)).toEqual([ + { keep_track_id: 1, remove_track_ids: [2] }, + { keep_track_id: 3, remove_track_ids: [4] }, + ]) +}) + +test('cleaning all keeps the copy chosen in each group, also when it is not the one suggested', async () => { + const { calls } = mockFetch(routes([groupA, groupB], { 'POST /api/v1/library/duplicates/clean': () => ({ removed: 1, freed_bytes: 1 }) })) + renderPage() + const a = await group('Song A') + await userEvent.click(within(a).getByRole('radio', { name: /Keep Greatest Hits/ })) + await userEvent.click(screen.getByRole('button', { name: 'Clean all suggestions' })) + await userEvent.click(within(await screen.findByRole('dialog')).getByRole('button', { name: 'Delete 2 files' })) + await screen.findByText(/Deleted 2 files/) + expect(posts(calls)).toEqual([ + { keep_track_id: 2, remove_track_ids: [1] }, + { keep_track_id: 3, remove_track_ids: [4] }, + ]) +}) + +// --- what the server refuses ------------------------------------------------------------------- + +test('a refusal is said in words, with what was already deleted, and the groups are shown again', async () => { + let reads = 0 + mockFetch( + routes([groupA, groupB], { + 'GET /api/v1/library/duplicates': () => { + reads++ + return page([groupA, groupB]) + }, + 'POST /api/v1/library/duplicates/clean': ({ init }) => + JSON.parse(String(init.body)).keep_track_id === 1 ? { removed: 1, freed_bytes: 8_000_000 } : apiError(409, 'duplicate_changed'), + }), + ) + renderPage() + await group('Song A') + await userEvent.click(screen.getByRole('button', { name: 'Clean all suggestions' })) + await userEvent.click(within(await screen.findByRole('dialog')).getByRole('button', { name: 'Delete 2 files' })) + + const alert = await screen.findByRole('alert') + expect(alert).toHaveTextContent('A file changed since the last scan') + expect(alert).toHaveTextContent('Deleted 1 file, 8 MB freed.') + await waitFor(() => expect(reads).toBe(2)) +}) + +test.each([ + ['library_read_only', 'The music folder is read-only'], + ['scan_running', 'A scan is running'], + ['deletion_disabled', 'Deleting is turned off on the server'], + ['invalid_input', 'no longer copies of the same song'], + ['track_not_found', 'already gone'], +])('the refusal %s is explained', async (code, text) => { + mockFetch(routes([groupA], { 'POST /api/v1/library/duplicates/clean': () => apiError(409, code) })) + renderPage() + const a = await group('Song A') + await userEvent.click(within(a).getByRole('button', { name: 'Delete the other copies of Song A' })) + await userEvent.click(within(await screen.findByRole('dialog')).getByRole('button', { name: 'Delete 1 file' })) + expect(await screen.findByRole('alert')).toHaveTextContent(text) + expect(screen.queryByText(/^Deleted \d+ file/)).not.toBeInTheDocument() +}) + +test('when the groups cannot be read it says so and offers to try again', async () => { + mockFetch(routes([], { 'GET /api/v1/library/duplicates': () => apiError(500, 'internal') })) + renderPage() + expect(await screen.findByRole('button', { name: 'Try again' })).toBeInTheDocument() +}) diff --git a/web/src/pages/admin/DuplicatesPage.tsx b/web/src/pages/admin/DuplicatesPage.tsx new file mode 100644 index 0000000..cda7c7c --- /dev/null +++ b/web/src/pages/admin/DuplicatesPage.tsx @@ -0,0 +1,130 @@ +import { useState } from 'react' +import { useTranslation } from 'react-i18next' +import { useCleanDuplicates, useDuplicates, type CleanResult } from '@/api/duplicates' +import LoadMore from '@/components/LoadMore' +import Section from '@/components/Section' +import { EmptyMessage, ErrorMessage, Loading } from '@/components/Status' +import { Button } from '@/components/ui/button' +import { formatBytes } from '@/lib/size' +import Confirm from './Confirm' +import DuplicateGroupCard, { OFF_NOTICE } from './DuplicateGroupCard' +import DuplicatesCheck from './DuplicatesCheck' +import { artistsOf, describeCopy, groupKey, planFor, type Plan } from './duplicatesText' +import { errorText } from './errorText' + +const SCOPES = ['admin.duplicates.errors', 'admin.errors'] + +/** + * The songs that are in more than one file, and the way to delete the extra copies. Which copy stays is + * chosen in each group (the server's suggestion to begin with); deleting asks first and lists every file + * that goes. With deleting turned off on the server everything can be seen and nothing can be deleted. + */ +export default function DuplicatesPage() { + const { t, i18n } = useTranslation() + const list = useDuplicates() + const clean = useCleanDuplicates() + const [chosen, setChosen] = useState>({}) + const [asking, setAsking] = useState<{ plans: Plan[]; all: boolean } | null>(null) + const [outcome, setOutcome] = useState(null) + + const groups = list.data?.groups ?? [] + const canDelete = list.data?.deletionAllowed === true + const plans = groups.map((group) => planFor(group, chosen[groupKey(group)])) + const size = (bytes: number) => formatBytes(bytes, i18n.language) + + const run = (toClean: Plan[]) => { + setAsking(null) + setOutcome(null) + clean.mutate( + toClean.map((p) => ({ keep: p.keep.track_id, remove: p.remove.map((f) => f.track_id) })), + { onSuccess: setOutcome }, + ) + } + + const files = (toClean: Plan[]) => toClean.reduce((n, p) => n + p.remove.length, 0) + const freed = (toClean: Plan[]) => toClean.reduce((n, p) => n + p.remove.reduce((m, f) => m + f.size_bytes, 0), 0) + + return ( +
+

{t('admin.duplicates.intro')}

+ {list.data && !canDelete && ( +

+ {t('admin.duplicates.off')} +

+ )} + + + + {outcome && ( +
+ {outcome.error ?

{errorText(t, outcome.error, SCOPES)}

: null} + {outcome.removed > 0 &&

{t('admin.duplicates.deleted', { count: outcome.removed, size: size(outcome.freedBytes) })}

} +
+ )} + {clean.isPending &&

{t('admin.duplicates.deleting')}

} + + {list.isError ? ( + void list.refetch()} /> + ) : list.isPending ? ( + + ) : groups.length === 0 ? ( + {t('admin.duplicates.none')} + ) : ( + <> +
+

{t('admin.duplicates.count', { count: list.data.total })}

+ +
+ {list.hasNextPage &&

{t('admin.duplicates.cleanAllShown')}

} +
+ {plans.map((plan) => ( + setChosen((c) => ({ ...c, [groupKey(plan.group)]: id }))} + canDelete={canDelete} + busy={clean.isPending} + onDelete={() => setAsking({ plans: [plan], all: false })} + /> + ))} +
+ + + )} + + {asking && ( + setAsking(null)} + onConfirm={() => run(asking.plans)} + > +
    + {asking.plans.map((plan) => ( +
  • + {asking.all && ( + <> +

    {plan.group.title}

    +

    {artistsOf(plan.group)}

    + + )} +

    {t('admin.duplicates.stays', { what: describeCopy(t, plan.keep, i18n.language) })}

    + {plan.remove.map((file) => ( +

    {t('admin.duplicates.goes', { what: describeCopy(t, file, i18n.language) })}

    + ))} +
  • + ))} +
+
+ )} +
+ ) +} diff --git a/web/src/pages/admin/duplicatesText.ts b/web/src/pages/admin/duplicatesText.ts new file mode 100644 index 0000000..df3dad7 --- /dev/null +++ b/web/src/pages/admin/duplicatesText.ts @@ -0,0 +1,33 @@ +import type { TFunction } from 'i18next' +import type { DuplicateFile, DuplicateGroup } from '@/api/duplicates' +import { formatBytes } from '@/lib/size' + +/** A copy in a line: "Album One, flac, 900 kbps, 30 MB". The bitrate is left out when the server does not know it. */ +export function describeCopy(t: TFunction, file: DuplicateFile, language: string): string { + return [file.album?.title ?? t('admin.duplicates.noAlbum'), file.format, file.bitrate > 0 ? t('admin.duplicates.kbps', { count: file.bitrate }) : null, formatBytes(file.size_bytes, language)] + .filter(Boolean) + .join(', ') +} + +/** The artists of a group in a line. */ +export const artistsOf = (group: DuplicateGroup) => group.artists.map((a) => a.name).join(', ') + +/** The same set of copies always has the same key, whatever the order, so a choice made for a group stays with it. */ +export const groupKey = (group: DuplicateGroup) => + group.files + .map((f) => f.track_id) + .sort((a, b) => a - b) + .join('-') + +/** What cleaning a group would do: the copy that stays and the copies that go. */ +export interface Plan { + group: DuplicateGroup + keep: DuplicateFile + remove: DuplicateFile[] +} + +/** The plan for a group, keeping the copy `keepId` (the one the server suggests when it is not one of the copies). */ +export function planFor(group: DuplicateGroup, keepId: number | undefined): Plan { + const keep = group.files.find((f) => f.track_id === keepId) ?? group.files.find((f) => f.track_id === group.suggested_keep_track_id) ?? group.files[0] + return { group, keep, remove: group.files.filter((f) => f !== keep) } +} diff --git a/web/src/router.tsx b/web/src/router.tsx index d5b7b8c..8e0da3e 100644 --- a/web/src/router.tsx +++ b/web/src/router.tsx @@ -7,6 +7,7 @@ import ArtistsPage from '@/pages/ArtistsPage' import DiscoverPage from '@/pages/DiscoverPage' import HomePage from '@/pages/HomePage' import AdminLayout from '@/pages/admin/AdminLayout' +import DuplicatesPage from '@/pages/admin/DuplicatesPage' import JobsPage from '@/pages/admin/JobsPage' import LibrariesPage from '@/pages/admin/LibrariesPage' import ProvidersPage from '@/pages/admin/ProvidersPage' @@ -38,6 +39,7 @@ export const routes: RouteObject[] = [ { path: 'users', element: }, { path: 'providers', element: }, { path: 'jobs', element: }, + { path: 'duplicates', element: }, ], }, { path: '*', element: },