diff --git a/cmd/sondad/main.go b/cmd/sondad/main.go index 68323b4..d0940fb 100644 --- a/cmd/sondad/main.go +++ b/cmd/sondad/main.go @@ -125,9 +125,13 @@ func run() error { gin.SetMode(gin.ReleaseMode) } libraryRepo := storage.NewLibraryRepo(db) + // On the first start after an upgrade that adds a key this can take a while on a big library. + logger.Info("refreshing the match keys of the library") + keysStarted := time.Now() if err := libraryRepo.RefreshMatchKeys(ctx); err != nil { return err } + logger.Info("match keys are up to date", zap.Duration("took", time.Since(keysStarted))) scanner := library.NewScanner(libraryRepo, library.TaglibReader{}, library.NewHub(), cfg.Library.ScanWorkers) scans := library.NewManager(scanner, libraryRepo) logScanResults(logger, scans) diff --git a/internal/storage/catalog.go b/internal/storage/catalog.go index f30303d..e13d701 100644 --- a/internal/storage/catalog.go +++ b/internal/storage/catalog.go @@ -64,6 +64,17 @@ func likePattern(q string) string { return "%" + esc + "%" } +// addSearch adds the condition for a search text on a column of search keys. A text that has +// nothing left once folded (only accents) matches nothing, where a pattern made of it would match +// everything; a text of blanks only is no search at all and matches everything. +func addSearch(w *where, column, q string) { + if strings.TrimSpace(q) != "" && searchKey(q) == "" { + w.add("1 = 0") + return + } + w.add(column+" LIKE ? ESCAPE '!'", likePattern(q)) +} + // binary orders by plain byte value. PostgreSQL would otherwise follow the // database locale while SQLite compares bytes, and listings would differ. func (r *CatalogRepo) binary(expr string) string { @@ -143,7 +154,7 @@ func (r *CatalogRepo) listArtists(ctx context.Context, f catalog.Filter, id int6 w.add("a.id = ?", id) } if f.Query != "" { - w.add("a.search_key LIKE ? ESCAPE '!'", likePattern(f.Query)) + addSearch(&w, "a.search_key", f.Query) } total, err := r.count(ctx, "SELECT COUNT(*) FROM artists a WHERE "+w.sql(), w.args...) @@ -241,7 +252,7 @@ func (r *CatalogRepo) listAlbums(ctx context.Context, f catalog.Filter, id int64 append([]any{f.ArtistID, f.ArtistID}, visArgs...)...) } if f.Query != "" { - w.add("al.search_key LIKE ? ESCAPE '!'", likePattern(f.Query)) + addSearch(&w, "al.search_key", f.Query) } total, err := r.count(ctx, "SELECT COUNT(*) FROM albums al WHERE "+w.sql(), w.args...) @@ -351,7 +362,7 @@ func (r *CatalogRepo) listTracks(ctx context.Context, f catalog.Filter, id int64 w.add("EXISTS (SELECT 1 FROM track_artists ta WHERE ta.track_id = t.id AND ta.artist_id = ?)", f.ArtistID) } if f.Query != "" { - w.add("t.search_key LIKE ? ESCAPE '!'", likePattern(f.Query)) + addSearch(&w, "t.search_key", f.Query) } total, err := r.count(ctx, "SELECT COUNT(*) FROM tracks t WHERE "+w.sql(), w.args...) diff --git a/internal/storage/search_key.go b/internal/storage/search_key.go index 23acc74..ecdbf07 100644 --- a/internal/storage/search_key.go +++ b/internal/storage/search_key.go @@ -10,18 +10,20 @@ import ( // letters that have no accent to take off but are written with the plain letters in a search: // the key of "Straße" is "strasse" and the one of "Ørsted" is "orsted". var searchLetters = strings.NewReplacer( - "ß", "ss", "æ", "ae", "œ", "oe", "ø", "o", "đ", "d", "ð", "d", "ł", "l", "ı", "i", "ħ", "h", + "ς", "σ", "ß", "ss", "æ", "ae", "œ", "oe", "ø", "o", "đ", "d", "ð", "d", "ł", "l", "ı", "i", "ħ", "h", ) // searchKey is the key a search looks in: matchKey with the accents taken off, so "bjork" finds // "Björk". It is not the key that identifies artists and albums, which keeps accents: otherwise // "Beyoncé" and "Beyonce" would be one artist. // +// Compatibility forms are folded too (the ligature "fi" is "fi", a full-width "A" is "a"). +// // Only the combining marks of the Latin, Greek and Cyrillic range (U+0300 to U+036F) go, so the // marks that are part of a letter in other scripts (the voicing mark of a kana, a Hangul syllable) // stay. It is computed here, not in SQL, so SQLite and PostgreSQL agree. func searchKey(s string) string { - d := norm.NFD.String(matchKey(s)) + d := norm.NFKD.String(matchKey(s)) var b strings.Builder b.Grow(len(d)) for _, r := range d { diff --git a/internal/storage/search_keys_test.go b/internal/storage/search_keys_test.go index 3a7effd..4eb9a9a 100644 --- a/internal/storage/search_keys_test.go +++ b/internal/storage/search_keys_test.go @@ -31,6 +31,14 @@ func TestSearchKeyIgnoresCaseAndAccents(t *testing.T) { "Dvořák": "dvorak", "Beyoncé Knowles": "beyonce knowles", "école": "ecole", // an e and a separate accent + "İstanbul": "istanbul", + "PHỞ": "pho", + "ÉCOLE": "ecole", // a capital E and a separate accent + "ΣΟΦΟΣ": "σοφοσ", + "σοφός": "σοφοσ", // the final sigma is a sigma + "fire": "fire", + "ABC": "abc", + "́": "", // a lone accent says nothing } { if got := searchKey(in); got != want { t.Errorf("searchKey(%q) = %q, want %q", in, got, want) @@ -139,3 +147,80 @@ func TestADiscoveredArtistGetsASearchKey(t *testing.T) { } }) } + +func TestAQueryThatSaysNothingOnceFoldedMatchesNothing(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + s := seedCatalog(t, db) + ctx := context.Background() + for _, q := range []string{"́", "́́"} { + if got, _ := s.repo.ListArtists(ctx, catalog.Filter{Query: q, Limit: 50}); got.Total != 0 { + t.Errorf("artists %q = %v, want none", q, artistNames(got)) + } + if got, _ := s.repo.ListTracks(ctx, catalog.Filter{Query: q, Limit: 50}); got.Total != 0 { + t.Errorf("tracks %q = %v, want none", q, trackTitles(got)) + } + if got, _ := s.repo.ListAlbums(ctx, catalog.Filter{Query: q, Limit: 50}); got.Total != 0 { + t.Errorf("albums %q = %v, want none", q, albumTitles(got)) + } + } + }) +} + +func TestARetitledTrackIsFoundByItsNewTitle(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + ctx := context.Background() + repo := NewLibraryRepo(db) + lib := mkLib(t, repo, "m") + if _, err := repo.SaveFile(ctx, lib, parsed("1.flac", "Old Name", "Album", "Band"), 0); err != nil { + t.Fatal(err) + } + existing, err := repo.ExistingFiles(ctx, lib) + if err != nil || len(existing) != 1 { + t.Fatalf("existing = %v err = %v", existing, err) + } + if _, err := repo.SaveFile(ctx, lib, parsed("1.flac", "Café Nuovo", "Album", "Band"), existing[0].ID); err != nil { + t.Fatal(err) + } + cat := NewCatalogRepo(db) + if got, _ := cat.ListTracks(ctx, catalog.Filter{Query: "cafe", Limit: 10}); !equal(trackTitles(got), []string{"Café Nuovo"}) { + t.Errorf("by the new title = %v", trackTitles(got)) + } + if got, _ := cat.ListTracks(ctx, catalog.Filter{Query: "old name", Limit: 10}); got.Total != 0 { + t.Errorf("by the old title = %v", trackTitles(got)) + } + }) +} + +// A refresh fixes both keys of a row together, and never merges what the accents keep apart. +func TestRefreshKeepsAccentedAndPlainArtistsApart(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + ctx := context.Background() + repo := NewLibraryRepo(db) + lib := mkLib(t, repo, "m") + for i, name := range []string{"Beyoncé", "Beyonce"} { + if _, err := repo.SaveFile(ctx, lib, parsed(string(rune('a'+i))+".flac", "Song", "Album "+name, name), 0); err != nil { + t.Fatal(err) + } + } + if _, err := db.Exec("UPDATE artists SET name_key = 'wrong', search_key = ''"); err != nil { + t.Fatal(err) + } + if err := repo.RefreshMatchKeys(ctx); err != nil { + t.Fatal(err) + } + var keys []string + rows, err := db.QueryContext(ctx, "SELECT name_key || '/' || search_key FROM artists ORDER BY id") + if err != nil { + t.Fatal(err) + } + defer rows.Close() + for rows.Next() { + var k string + _ = rows.Scan(&k) + keys = append(keys, k) + } + if !equal(keys, []string{"beyoncé/beyonce", "beyonce/beyonce"}) { + t.Errorf("keys = %v, want the identity key with the accent and the search key without", keys) + } + }) +}