diff --git a/README.md b/README.md index 31552dc..43cdaf2 100644 --- a/README.md +++ b/README.md @@ -253,7 +253,7 @@ Everything lives under `/api/v1`. Bodies are JSON. Errors always look like `{"er | admin | `GET /providers`, `PUT /providers/{name}`, `POST /providers/{name}/test`, `DELETE /providers/{name}/data` | See, set up and check Last.fm and MusicBrainz, and remove what they gave. The API key is write-only. | | admin | `GET`, `POST /users`; `PATCH`, `DELETE /users/{id}`; `PUT /users/{id}/password` | Manage users and the admin role. The last admin cannot be demoted or deleted. | -A library path must be an absolute, existing folder, never the filesystem root, and never inside or around another library. Search is a case-insensitive substring match: `BJÖRK` finds `Björk`, but accents are kept, so `bjork` does not. +A library path must be an absolute, existing folder, never the filesystem root, and never inside or around another library. Search is a case- and accent-insensitive substring match: `bjork`, `BJORK` and `BJÖRK` all find `Björk`, and `strasse` finds `Straße`. Accents still tell artists apart: `Beyoncé` and `Beyonce` stay two artists, and a search for either finds both. ## Security notes diff --git a/cmd/sondad/main.go b/cmd/sondad/main.go index 68323b4..d0940fb 100644 --- a/cmd/sondad/main.go +++ b/cmd/sondad/main.go @@ -125,9 +125,13 @@ func run() error { gin.SetMode(gin.ReleaseMode) } libraryRepo := storage.NewLibraryRepo(db) + // On the first start after an upgrade that adds a key this can take a while on a big library. + logger.Info("refreshing the match keys of the library") + keysStarted := time.Now() if err := libraryRepo.RefreshMatchKeys(ctx); err != nil { return err } + logger.Info("match keys are up to date", zap.Duration("took", time.Since(keysStarted))) scanner := library.NewScanner(libraryRepo, library.TaglibReader{}, library.NewHub(), cfg.Library.ScanWorkers) scans := library.NewManager(scanner, libraryRepo) logScanResults(logger, scans) diff --git a/go.mod b/go.mod index d7be0c8..a8c971e 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( go.uber.org/zap v1.28.0 golang.org/x/crypto v0.58.0 golang.org/x/sync v0.24.0 + golang.org/x/text v0.43.0 modernc.org/sqlite v1.62.0 ) @@ -64,7 +65,6 @@ require ( golang.org/x/arch v0.22.0 // indirect golang.org/x/net v0.60.0 // indirect golang.org/x/sys v0.49.0 // indirect - golang.org/x/text v0.43.0 // indirect google.golang.org/protobuf v1.36.10 // indirect modernc.org/libc v1.80.1 // indirect modernc.org/mathutil v1.7.1 // indirect diff --git a/internal/api/openapi/openapi.yaml b/internal/api/openapi/openapi.yaml index 58e733b..1fc67ed 100644 --- a/internal/api/openapi/openapi.yaml +++ b/internal/api/openapi/openapi.yaml @@ -1900,8 +1900,8 @@ components: name: q in: query description: | - Case-insensitive substring of the name or title. Accented letters are matched ignoring - case (`BJÖRK` finds `Björk`), but accents are kept (`bjork` does not). At most 200 characters. + Case- and accent-insensitive substring of the name or title: `bjork`, `BJORK` and `BJÖRK` all + find `Björk`, and `strasse` finds `Straße`. At most 200 characters. schema: {type: string, maxLength: 200} Full: name: full diff --git a/internal/storage/catalog.go b/internal/storage/catalog.go index 6e333c0..e13d701 100644 --- a/internal/storage/catalog.go +++ b/internal/storage/catalog.go @@ -56,14 +56,25 @@ func visible(alias string, libraryID int64) (string, []any) { return q + ")", args } -// likePattern turns a search text into a LIKE pattern for the match keys. The +// likePattern turns a search text into a LIKE pattern for the search keys (no accents). The // wildcard characters of the text are escaped with '!', so "100%" only matches // a literal percent sign. func likePattern(q string) string { - esc := strings.NewReplacer("!", "!!", "%", "!%", "_", "!_").Replace(matchKey(q)) + esc := strings.NewReplacer("!", "!!", "%", "!%", "_", "!_").Replace(searchKey(q)) return "%" + esc + "%" } +// addSearch adds the condition for a search text on a column of search keys. A text that has +// nothing left once folded (only accents) matches nothing, where a pattern made of it would match +// everything; a text of blanks only is no search at all and matches everything. +func addSearch(w *where, column, q string) { + if strings.TrimSpace(q) != "" && searchKey(q) == "" { + w.add("1 = 0") + return + } + w.add(column+" LIKE ? ESCAPE '!'", likePattern(q)) +} + // binary orders by plain byte value. PostgreSQL would otherwise follow the // database locale while SQLite compares bytes, and listings would differ. func (r *CatalogRepo) binary(expr string) string { @@ -143,7 +154,7 @@ func (r *CatalogRepo) listArtists(ctx context.Context, f catalog.Filter, id int6 w.add("a.id = ?", id) } if f.Query != "" { - w.add("a.name_key LIKE ? ESCAPE '!'", likePattern(f.Query)) + addSearch(&w, "a.search_key", f.Query) } total, err := r.count(ctx, "SELECT COUNT(*) FROM artists a WHERE "+w.sql(), w.args...) @@ -241,7 +252,7 @@ func (r *CatalogRepo) listAlbums(ctx context.Context, f catalog.Filter, id int64 append([]any{f.ArtistID, f.ArtistID}, visArgs...)...) } if f.Query != "" { - w.add("al.title_key LIKE ? ESCAPE '!'", likePattern(f.Query)) + addSearch(&w, "al.search_key", f.Query) } total, err := r.count(ctx, "SELECT COUNT(*) FROM albums al WHERE "+w.sql(), w.args...) @@ -351,7 +362,7 @@ func (r *CatalogRepo) listTracks(ctx context.Context, f catalog.Filter, id int64 w.add("EXISTS (SELECT 1 FROM track_artists ta WHERE ta.track_id = t.id AND ta.artist_id = ?)", f.ArtistID) } if f.Query != "" { - w.add("t.title_key LIKE ? ESCAPE '!'", likePattern(f.Query)) + addSearch(&w, "t.search_key", f.Query) } total, err := r.count(ctx, "SELECT COUNT(*) FROM tracks t WHERE "+w.sql(), w.args...) diff --git a/internal/storage/catalog_test.go b/internal/storage/catalog_test.go index 5a0c54c..4ac770a 100644 --- a/internal/storage/catalog_test.go +++ b/internal/storage/catalog_test.go @@ -211,7 +211,6 @@ func TestCatalogSearch(t *testing.T) { search := func(q string) catalog.Filter { return catalog.Filter{Query: q, Limit: 50} } // Case folding, accented letters included, is the same on both databases. - // Accents themselves are not folded: "bjork" does not find "Björk". for _, q := range []string{"björk", "BJÖRK", "jö"} { if got, _ := s.repo.ListArtists(ctx, search(q)); !equal(artistNames(got), []string{"Björk"}) { t.Errorf("artists %q = %v", q, artistNames(got)) @@ -226,9 +225,6 @@ func TestCatalogSearch(t *testing.T) { if got, _ := s.repo.ListAlbums(ctx, search("kind OF")); !equal(albumTitles(got), []string{"Kind of Blue"}) { t.Errorf("albums kind OF = %v", albumTitles(got)) } - if got, _ := s.repo.ListArtists(ctx, search("bjork")); got.Total != 0 { - t.Errorf(`"bjork" = %v: accents are not folded`, artistNames(got)) - } if got, _ := s.repo.ListArtists(ctx, search("zzz")); got.Total != 0 || len(got.Items) != 0 { t.Errorf("no match: %+v", got) } diff --git a/internal/storage/discovery.go b/internal/storage/discovery.go index ab5252d..38ca705 100644 --- a/internal/storage/discovery.go +++ b/internal/storage/discovery.go @@ -304,7 +304,7 @@ func insertDiscovered(ctx context.Context, tx bun.Tx, name, mbid string) (int64, } var id int64 err := tx.QueryRowContext(ctx, - "INSERT INTO artists (mbid, name, name_key, discovered) VALUES (?, ?, ?, ?) RETURNING id", m, name, matchKey(name), true).Scan(&id) + "INSERT INTO artists (mbid, name, name_key, search_key, discovered) VALUES (?, ?, ?, ?, ?) RETURNING id", m, name, matchKey(name), searchKey(name), true).Scan(&id) if err != nil { return 0, fmt.Errorf("insert artist: %w", err) } diff --git a/internal/storage/library.go b/internal/storage/library.go index 0ce6c1e..a3ea41d 100644 --- a/internal/storage/library.go +++ b/internal/storage/library.go @@ -20,17 +20,19 @@ type artistModel struct { ID int64 `bun:"id,pk,autoincrement"` Name string `bun:"name,notnull"` NameKey string `bun:"name_key,notnull"` + SearchKey string `bun:"search_key,notnull"` Discovered bool `bun:"discovered,notnull"` // named by a provider, never seen in a file } type albumModel struct { bun.BaseModel `bun:"table:albums"` - ID int64 `bun:"id,pk,autoincrement"` - Title string `bun:"title,notnull"` - TitleKey string `bun:"title_key,notnull"` - ArtistID int64 `bun:"artist_id,nullzero"` - MBID string `bun:"mbid,nullzero"` // the MusicBrainz release group id, from the files' tags + ID int64 `bun:"id,pk,autoincrement"` + Title string `bun:"title,notnull"` + TitleKey string `bun:"title_key,notnull"` + SearchKey string `bun:"search_key,notnull"` + ArtistID int64 `bun:"artist_id,nullzero"` + MBID string `bun:"mbid,nullzero"` // the MusicBrainz release group id, from the files' tags } type trackModel struct { @@ -39,6 +41,7 @@ type trackModel struct { ID int64 `bun:"id,pk,autoincrement"` Title string `bun:"title,notnull"` TitleKey string `bun:"title_key,notnull"` + SearchKey string `bun:"search_key,notnull"` DurationMs int64 `bun:"duration_ms,nullzero"` TrackNo int `bun:"track_no,nullzero"` DiscNo int `bun:"disc_no,nullzero"` @@ -145,7 +148,7 @@ func (r *LibraryRepo) SaveFile(ctx context.Context, libraryID int64, f library.P } track := &trackModel{ - Title: f.Title, TitleKey: matchKey(f.Title), DurationMs: f.DurationMs, TrackNo: f.TrackNo, DiscNo: f.DiscNo, AlbumID: albumID, + Title: f.Title, TitleKey: matchKey(f.Title), SearchKey: searchKey(f.Title), DurationMs: f.DurationMs, TrackNo: f.TrackNo, DiscNo: f.DiscNo, AlbumID: albumID, } if existingID == 0 { @@ -180,7 +183,7 @@ func (r *LibraryRepo) SaveFile(ctx context.Context, libraryID int64, f library.P return err } if _, err := tx.NewUpdate().Model(track). - Column("title", "title_key", "duration_ms", "track_no", "disc_no", "album_id"). + Column("title", "title_key", "search_key", "duration_ms", "track_no", "disc_no", "album_id"). WherePK().Exec(ctx); err != nil { return fmt.Errorf("update track: %w", err) } @@ -356,60 +359,62 @@ func (r *LibraryRepo) DeleteLibrary(ctx context.Context, id int64) error { return nil } -// RefreshMatchKeys recomputes the match keys of every artist, album and track -// whose stored key differs from the current rule. It is idempotent and cheap -// when nothing changed, so it can run at every start: rows created before -// migration 0003 only have an approximate key from SQL lower(). +// RefreshMatchKeys recomputes the match keys and the search keys of every artist, album and +// track whose stored key differs from the current rule. It is idempotent and cheap when nothing +// changed, so it can run at every start: rows created before migration 0003 only have an +// approximate key from SQL lower(), and those from before migration 0016 have no search key. func (r *LibraryRepo) RefreshMatchKeys(ctx context.Context) error { return r.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { - var artists []artistModel - if err := tx.NewSelect().Model(&artists).Column("id", "name", "name_key").Scan(ctx); err != nil { - return fmt.Errorf("list artists: %w", err) - } - for _, a := range artists { - key := matchKey(a.Name) - if key == a.NameKey { - continue - } - if _, err := tx.NewUpdate().Model((*artistModel)(nil)). - Set("name_key = ?", key).Where("id = ?", a.ID).Exec(ctx); err != nil { - return fmt.Errorf("update artist key: %w", err) - } - } - - var tracks []trackModel - if err := tx.NewSelect().Model(&tracks).Column("id", "title", "title_key").Scan(ctx); err != nil { - return fmt.Errorf("list tracks: %w", err) - } - for _, t := range tracks { - key := matchKey(t.Title) - if key == t.TitleKey { - continue - } - if _, err := tx.NewUpdate().Model((*trackModel)(nil)). - Set("title_key = ?", key).Where("id = ?", t.ID).Exec(ctx); err != nil { - return fmt.Errorf("update track key: %w", err) - } - } - - var albums []albumModel - if err := tx.NewSelect().Model(&albums).Column("id", "title", "title_key").Scan(ctx); err != nil { - return fmt.Errorf("list albums: %w", err) - } - for _, al := range albums { - key := matchKey(al.Title) - if key == al.TitleKey { - continue - } - if _, err := tx.NewUpdate().Model((*albumModel)(nil)). - Set("title_key = ?", key).Where("id = ?", al.ID).Exec(ctx); err != nil { - return fmt.Errorf("update album key: %w", err) - } - } - return nil + return refreshKeys(ctx, tx, []keyedTable{ + {"artists", "name", "name_key"}, + {"tracks", "title", "title_key"}, + {"albums", "title", "title_key"}, + }) }) } +// keyedTable is a table with a text column and the keys made from it. +type keyedTable struct{ table, text, matchKey string } + +func refreshKeys(ctx context.Context, tx bun.Tx, tables []keyedTable) error { + for _, t := range tables { + rows, err := tx.QueryContext(ctx, "SELECT id, "+t.text+", "+t.matchKey+", search_key FROM "+t.table) + if err != nil { + return fmt.Errorf("list %s: %w", t.table, err) + } + type fix struct { + id int64 + match, find string + } + var fixes []fix + for rows.Next() { + var ( + id int64 + text, match, current string + ) + if err := rows.Scan(&id, &text, &match, ¤t); err != nil { + _ = rows.Close() + return err + } + m, s := matchKey(text), searchKey(text) + if m != match || s != current { + fixes = append(fixes, fix{id: id, match: m, find: s}) + } + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return err + } + _ = rows.Close() + for _, f := range fixes { + if _, err := tx.ExecContext(ctx, "UPDATE "+t.table+" SET "+t.matchKey+" = ?, search_key = ? WHERE id = ?", f.match, f.find, f.id); err != nil { + return fmt.Errorf("update %s key: %w", t.table, err) + } + } + } + return nil +} + // ---- helpers (run inside a transaction) ---- // findOrCreateArtist returns the artist a file belongs to, creating it if needed. @@ -481,15 +486,15 @@ func findOrCreateArtist(ctx context.Context, tx bun.Tx, name, mbid string) (int6 return byName.ID, exec("set artist id", "UPDATE artists SET mbid = ? WHERE id = ?", mbid, byName.ID) case haveID: return byID.ID, exec("confirm discovered artist", - "UPDATE artists SET name = ?, name_key = ?, discovered = ? WHERE id = ?", name, key, false, byID.ID) + "UPDATE artists SET name = ?, name_key = ?, search_key = ?, discovered = ? WHERE id = ?", name, key, searchKey(name), false, byID.ID) case haveName: return byName.ID, exec("adopt discovered artist", - "UPDATE artists SET name = ?, name_key = ?, discovered = ?, mbid = ? WHERE id = ?", name, key, false, mbid, byName.ID) + "UPDATE artists SET name = ?, name_key = ?, search_key = ?, discovered = ?, mbid = ? WHERE id = ?", name, key, searchKey(name), false, mbid, byName.ID) } var id int64 if err := tx.QueryRowContext(ctx, - "INSERT INTO artists (mbid, name, name_key, discovered) VALUES (?, ?, ?, ?) RETURNING id", - mbid, name, key, false).Scan(&id); err != nil { + "INSERT INTO artists (mbid, name, name_key, search_key, discovered) VALUES (?, ?, ?, ?, ?) RETURNING id", + mbid, name, key, searchKey(name), false).Scan(&id); err != nil { return 0, fmt.Errorf("insert artist: %w", err) } return id, nil @@ -500,7 +505,7 @@ func findOrCreateArtist(ctx context.Context, tx bun.Tx, name, mbid string) (int6 return 0, fmt.Errorf("find artist: %w", err) } if !ok { - a := artistModel{Name: name, NameKey: key} + a := artistModel{Name: name, NameKey: key, SearchKey: searchKey(name)} if _, err := tx.NewInsert().Model(&a).Exec(ctx); err != nil { return 0, fmt.Errorf("insert artist: %w", err) } @@ -508,7 +513,7 @@ func findOrCreateArtist(ctx context.Context, tx bun.Tx, name, mbid string) (int6 } if found.Discovered { if err := exec("adopt discovered artist", - "UPDATE artists SET name = ?, name_key = ?, discovered = ?, mbid = NULL WHERE id = ?", name, key, false, found.ID); err != nil { + "UPDATE artists SET name = ?, name_key = ?, search_key = ?, discovered = ?, mbid = NULL WHERE id = ?", name, key, searchKey(name), false, found.ID); err != nil { return 0, err } if found.MBID != "" { @@ -587,7 +592,7 @@ func findOrCreateAlbum(ctx context.Context, tx bun.Tx, title string, artistID in if !errors.Is(err, sql.ErrNoRows) { return 0, fmt.Errorf("find album: %w", err) } - a = albumModel{Title: title, TitleKey: matchKey(title), ArtistID: artistID} + a = albumModel{Title: title, TitleKey: matchKey(title), SearchKey: searchKey(title), ArtistID: artistID} if mbid != "" { var taken bool if err := tx.QueryRowContext(ctx, "SELECT EXISTS (SELECT 1 FROM albums WHERE mbid = ?)", mbid).Scan(&taken); err != nil { diff --git a/internal/storage/search_key.go b/internal/storage/search_key.go new file mode 100644 index 0000000..ecdbf07 --- /dev/null +++ b/internal/storage/search_key.go @@ -0,0 +1,36 @@ +package storage + +import ( + "strings" + "unicode" + + "golang.org/x/text/unicode/norm" +) + +// letters that have no accent to take off but are written with the plain letters in a search: +// the key of "Straße" is "strasse" and the one of "Ørsted" is "orsted". +var searchLetters = strings.NewReplacer( + "ς", "σ", "ß", "ss", "æ", "ae", "œ", "oe", "ø", "o", "đ", "d", "ð", "d", "ł", "l", "ı", "i", "ħ", "h", +) + +// searchKey is the key a search looks in: matchKey with the accents taken off, so "bjork" finds +// "Björk". It is not the key that identifies artists and albums, which keeps accents: otherwise +// "Beyoncé" and "Beyonce" would be one artist. +// +// Compatibility forms are folded too (the ligature "fi" is "fi", a full-width "A" is "a"). +// +// Only the combining marks of the Latin, Greek and Cyrillic range (U+0300 to U+036F) go, so the +// marks that are part of a letter in other scripts (the voicing mark of a kana, a Hangul syllable) +// stay. It is computed here, not in SQL, so SQLite and PostgreSQL agree. +func searchKey(s string) string { + d := norm.NFKD.String(matchKey(s)) + var b strings.Builder + b.Grow(len(d)) + for _, r := range d { + if r >= 0x0300 && r <= 0x036F && unicode.Is(unicode.Mn, r) { + continue + } + b.WriteRune(r) + } + return searchLetters.Replace(norm.NFC.String(b.String())) +} diff --git a/internal/storage/search_keys_test.go b/internal/storage/search_keys_test.go new file mode 100644 index 0000000..4eb9a9a --- /dev/null +++ b/internal/storage/search_keys_test.go @@ -0,0 +1,226 @@ +package storage + +import ( + "context" + "testing" + + "github.com/uptrace/bun" + + "codeberg.org/squidd/sonda/internal/catalog" +) + +func TestSearchKeyIgnoresCaseAndAccents(t *testing.T) { + for in, want := range map[string]string{ + "Björk": "bjork", + "BJÖRK": "bjork", + "Beyoncé": "beyonce", + " Café ": "cafe", + "Mötley Crüe": "motley crue", + "Straße": "strasse", + "Ørsted": "orsted", + "Æon": "aeon", + "Łódź": "lodz", + "Sigur Rós": "sigur ros", + "Ελλάδα": "ελλαδα", + "ñandú": "nandu", + "AC/DC": "ac/dc", + "plain": "plain", + "宇多田ヒカル": "宇多田ヒカル", + "ガガ": "ガガ", // the marks of kana are part of the letter + "한글": "한글", + "Dvořák": "dvorak", + "Beyoncé Knowles": "beyonce knowles", + "école": "ecole", // an e and a separate accent + "İstanbul": "istanbul", + "PHỞ": "pho", + "ÉCOLE": "ecole", // a capital E and a separate accent + "ΣΟΦΟΣ": "σοφοσ", + "σοφός": "σοφοσ", // the final sigma is a sigma + "fire": "fire", + "ABC": "abc", + "́": "", // a lone accent says nothing + } { + if got := searchKey(in); got != want { + t.Errorf("searchKey(%q) = %q, want %q", in, got, want) + } + } +} + +func TestSearchFindsNamesWithoutTheAccentsAndTheOtherWayRound(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + s := seedCatalog(t, db) + ctx := context.Background() + search := func(q string) catalog.Filter { return catalog.Filter{Query: q, Limit: 50} } + + for _, q := range []string{"bjork", "BJORK", "björk", "BJÖRK", "bjö", "jork"} { + if got, _ := s.repo.ListArtists(ctx, search(q)); !equal(artistNames(got), []string{"Björk"}) { + t.Errorf("artists %q = %v", q, artistNames(got)) + } + } + if got, _ := s.repo.ListArtists(ctx, search("unicode")); !equal(artistNames(got), []string{"Ünïcode Band"}) { + t.Errorf("artists unicode = %v", artistNames(got)) + } + if got, _ := s.repo.ListTracks(ctx, search("unicode trck")); !equal(trackTitles(got), []string{"Ünïcode Trck"}) { + t.Errorf("tracks = %v", trackTitles(got)) + } + if got, _ := s.repo.ListAlbums(ctx, search("post")); !equal(albumTitles(got), []string{"Post"}) { + t.Errorf("albums = %v", albumTitles(got)) + } + // The wildcards of the text are still literal. + if got, _ := s.repo.ListTracks(ctx, search("100%")); !equal(trackTitles(got), []string{"100% Pure"}) { + t.Errorf("tracks 100%% = %v", trackTitles(got)) + } + }) +} + +// "Beyoncé" and "Beyonce" are two names as far as identity goes: the scanner must not merge +// them; only the search treats them alike. +func TestAccentsDoNotMergeArtistsButTheSearchFindsBoth(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + ctx := context.Background() + repo := NewLibraryRepo(db) + lib := mkLib(t, repo, "m") + if _, err := repo.SaveFile(ctx, lib, parsed("1.flac", "Halo", "I Am", "Beyoncé"), 0); err != nil { + t.Fatal(err) + } + if _, err := repo.SaveFile(ctx, lib, parsed("2.flac", "Other", "Other Album", "Beyonce"), 0); err != nil { + t.Fatal(err) + } + if n := count(t, db, "SELECT COUNT(*) FROM artists"); n != 2 { + t.Fatalf("artists = %d, want 2: accents identify an artist", n) + } + got, err := NewCatalogRepo(db).ListArtists(ctx, catalog.Filter{Query: "beyonce", Limit: 10}) + if err != nil || got.Total != 2 { + t.Errorf("search = %v err = %v, want both", artistNames(got), err) + } + }) +} + +func TestRefreshMatchKeysFillsTheSearchKeyOfOldRows(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + ctx := context.Background() + repo := NewLibraryRepo(db) + lib := mkLib(t, repo, "m") + if _, err := repo.SaveFile(ctx, lib, parsed("1.flac", "Army of Me", "Post", "Björk"), 0); err != nil { + t.Fatal(err) + } + // Rows from before the column existed have none. + for _, q := range []string{"UPDATE artists SET search_key = ''", "UPDATE albums SET search_key = ''", "UPDATE tracks SET search_key = ''"} { + if _, err := db.Exec(q); err != nil { + t.Fatal(err) + } + } + if got, _ := NewCatalogRepo(db).ListArtists(ctx, catalog.Filter{Query: "bjork", Limit: 10}); got.Total != 0 { + t.Fatal("found an artist that has no search key") + } + if err := repo.RefreshMatchKeys(ctx); err != nil { + t.Fatal(err) + } + if err := repo.RefreshMatchKeys(ctx); err != nil { // idempotent + t.Fatal(err) + } + cat := NewCatalogRepo(db) + if got, _ := cat.ListArtists(ctx, catalog.Filter{Query: "bjork", Limit: 10}); got.Total != 1 { + t.Errorf("artists = %d, want 1", got.Total) + } + if got, _ := cat.ListAlbums(ctx, catalog.Filter{Query: "post", Limit: 10}); got.Total != 1 { + t.Errorf("albums = %d, want 1", got.Total) + } + if got, _ := cat.ListTracks(ctx, catalog.Filter{Query: "army", Limit: 10}); got.Total != 1 { + t.Errorf("tracks = %d, want 1", got.Total) + } + }) +} + +func TestADiscoveredArtistGetsASearchKey(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + ctx := context.Background() + if err := db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + _, err := insertDiscovered(ctx, tx, "Sigur Rós", "") + return err + }); err != nil { + t.Fatal(err) + } + var key string + if err := db.QueryRow("SELECT search_key FROM artists WHERE name = 'Sigur Rós'").Scan(&key); err != nil || key != "sigur ros" { + t.Errorf("search key = %q err = %v", key, err) + } + }) +} + +func TestAQueryThatSaysNothingOnceFoldedMatchesNothing(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + s := seedCatalog(t, db) + ctx := context.Background() + for _, q := range []string{"́", "́́"} { + if got, _ := s.repo.ListArtists(ctx, catalog.Filter{Query: q, Limit: 50}); got.Total != 0 { + t.Errorf("artists %q = %v, want none", q, artistNames(got)) + } + if got, _ := s.repo.ListTracks(ctx, catalog.Filter{Query: q, Limit: 50}); got.Total != 0 { + t.Errorf("tracks %q = %v, want none", q, trackTitles(got)) + } + if got, _ := s.repo.ListAlbums(ctx, catalog.Filter{Query: q, Limit: 50}); got.Total != 0 { + t.Errorf("albums %q = %v, want none", q, albumTitles(got)) + } + } + }) +} + +func TestARetitledTrackIsFoundByItsNewTitle(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + ctx := context.Background() + repo := NewLibraryRepo(db) + lib := mkLib(t, repo, "m") + if _, err := repo.SaveFile(ctx, lib, parsed("1.flac", "Old Name", "Album", "Band"), 0); err != nil { + t.Fatal(err) + } + existing, err := repo.ExistingFiles(ctx, lib) + if err != nil || len(existing) != 1 { + t.Fatalf("existing = %v err = %v", existing, err) + } + if _, err := repo.SaveFile(ctx, lib, parsed("1.flac", "Café Nuovo", "Album", "Band"), existing[0].ID); err != nil { + t.Fatal(err) + } + cat := NewCatalogRepo(db) + if got, _ := cat.ListTracks(ctx, catalog.Filter{Query: "cafe", Limit: 10}); !equal(trackTitles(got), []string{"Café Nuovo"}) { + t.Errorf("by the new title = %v", trackTitles(got)) + } + if got, _ := cat.ListTracks(ctx, catalog.Filter{Query: "old name", Limit: 10}); got.Total != 0 { + t.Errorf("by the old title = %v", trackTitles(got)) + } + }) +} + +// A refresh fixes both keys of a row together, and never merges what the accents keep apart. +func TestRefreshKeepsAccentedAndPlainArtistsApart(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + ctx := context.Background() + repo := NewLibraryRepo(db) + lib := mkLib(t, repo, "m") + for i, name := range []string{"Beyoncé", "Beyonce"} { + if _, err := repo.SaveFile(ctx, lib, parsed(string(rune('a'+i))+".flac", "Song", "Album "+name, name), 0); err != nil { + t.Fatal(err) + } + } + if _, err := db.Exec("UPDATE artists SET name_key = 'wrong', search_key = ''"); err != nil { + t.Fatal(err) + } + if err := repo.RefreshMatchKeys(ctx); err != nil { + t.Fatal(err) + } + var keys []string + rows, err := db.QueryContext(ctx, "SELECT name_key || '/' || search_key FROM artists ORDER BY id") + if err != nil { + t.Fatal(err) + } + defer rows.Close() + for rows.Next() { + var k string + _ = rows.Scan(&k) + keys = append(keys, k) + } + if !equal(keys, []string{"beyoncé/beyonce", "beyonce/beyonce"}) { + t.Errorf("keys = %v, want the identity key with the accent and the search key without", keys) + } + }) +} diff --git a/internal/storage/storage_test.go b/internal/storage/storage_test.go index 9299325..9de55c6 100644 --- a/internal/storage/storage_test.go +++ b/internal/storage/storage_test.go @@ -156,8 +156,8 @@ func TestMigrateCreatesSchemaAndIsIdempotent(t *testing.T) { if err := db.QueryRow("SELECT COUNT(*) FROM schema_migrations").Scan(&applied); err != nil { t.Fatal(err) } - if applied != 15 { - t.Errorf("applied migrations = %d, want 15", applied) + if applied != 16 { + t.Errorf("applied migrations = %d, want 16", applied) } }) } diff --git a/migrations/postgres/0016_search_keys.sql b/migrations/postgres/0016_search_keys.sql new file mode 100644 index 0000000..cafe60d --- /dev/null +++ b/migrations/postgres/0016_search_keys.sql @@ -0,0 +1,8 @@ +-- A key for searching, apart from the keys that identify artists, albums and tracks: it has +-- the accents taken off, so "bjork" finds "Björk", while "Beyoncé" and "Beyonce" stay two +-- artists. It is computed in Go (Unicode-aware) and filled in for existing rows at start, +-- by RefreshMatchKeys. + +ALTER TABLE artists ADD COLUMN search_key TEXT NOT NULL DEFAULT ''; +ALTER TABLE albums ADD COLUMN search_key TEXT NOT NULL DEFAULT ''; +ALTER TABLE tracks ADD COLUMN search_key TEXT NOT NULL DEFAULT ''; diff --git a/migrations/sqlite/0016_search_keys.sql b/migrations/sqlite/0016_search_keys.sql new file mode 100644 index 0000000..cafe60d --- /dev/null +++ b/migrations/sqlite/0016_search_keys.sql @@ -0,0 +1,8 @@ +-- A key for searching, apart from the keys that identify artists, albums and tracks: it has +-- the accents taken off, so "bjork" finds "Björk", while "Beyoncé" and "Beyonce" stay two +-- artists. It is computed in Go (Unicode-aware) and filled in for existing rows at start, +-- by RefreshMatchKeys. + +ALTER TABLE artists ADD COLUMN search_key TEXT NOT NULL DEFAULT ''; +ALTER TABLE albums ADD COLUMN search_key TEXT NOT NULL DEFAULT ''; +ALTER TABLE tracks ADD COLUMN search_key TEXT NOT NULL DEFAULT '';