fix(search): empty folded queries, compatibility forms, final sigma, start-up log

- A search text that has nothing left once its accents are taken off matches
  nothing instead of everything.
- The search key folds compatibility forms (the ligature "fi", full-width
  letters) and the Greek final sigma.
- The start-up refresh of the keys logs when it starts and how long it took, since
  it can take a while on a big library the first time.
- More tests: a retitled track is found by its new title, a refresh fixes both
  keys and keeps "Beyoncé" and "Beyonce" apart, more cases for the key.
This commit is contained in:
tgiachi 2026-11-12 10:24:44 +01:00
commit 4ec3ae3bb0
4 changed files with 107 additions and 5 deletions

View file

@ -125,9 +125,13 @@ func run() error {
gin.SetMode(gin.ReleaseMode)
}
libraryRepo := storage.NewLibraryRepo(db)
// On the first start after an upgrade that adds a key this can take a while on a big library.
logger.Info("refreshing the match keys of the library")
keysStarted := time.Now()
if err := libraryRepo.RefreshMatchKeys(ctx); err != nil {
return err
}
logger.Info("match keys are up to date", zap.Duration("took", time.Since(keysStarted)))
scanner := library.NewScanner(libraryRepo, library.TaglibReader{}, library.NewHub(), cfg.Library.ScanWorkers)
scans := library.NewManager(scanner, libraryRepo)
logScanResults(logger, scans)

View file

@ -64,6 +64,17 @@ func likePattern(q string) string {
return "%" + esc + "%"
}
// addSearch adds the condition for a search text on a column of search keys. A text that has
// nothing left once folded (only accents) matches nothing, where a pattern made of it would match
// everything; a text of blanks only is no search at all and matches everything.
func addSearch(w *where, column, q string) {
if strings.TrimSpace(q) != "" && searchKey(q) == "" {
w.add("1 = 0")
return
}
w.add(column+" LIKE ? ESCAPE '!'", likePattern(q))
}
// binary orders by plain byte value. PostgreSQL would otherwise follow the
// database locale while SQLite compares bytes, and listings would differ.
func (r *CatalogRepo) binary(expr string) string {
@ -143,7 +154,7 @@ func (r *CatalogRepo) listArtists(ctx context.Context, f catalog.Filter, id int6
w.add("a.id = ?", id)
}
if f.Query != "" {
w.add("a.search_key LIKE ? ESCAPE '!'", likePattern(f.Query))
addSearch(&w, "a.search_key", f.Query)
}
total, err := r.count(ctx, "SELECT COUNT(*) FROM artists a WHERE "+w.sql(), w.args...)
@ -241,7 +252,7 @@ func (r *CatalogRepo) listAlbums(ctx context.Context, f catalog.Filter, id int64
append([]any{f.ArtistID, f.ArtistID}, visArgs...)...)
}
if f.Query != "" {
w.add("al.search_key LIKE ? ESCAPE '!'", likePattern(f.Query))
addSearch(&w, "al.search_key", f.Query)
}
total, err := r.count(ctx, "SELECT COUNT(*) FROM albums al WHERE "+w.sql(), w.args...)
@ -351,7 +362,7 @@ func (r *CatalogRepo) listTracks(ctx context.Context, f catalog.Filter, id int64
w.add("EXISTS (SELECT 1 FROM track_artists ta WHERE ta.track_id = t.id AND ta.artist_id = ?)", f.ArtistID)
}
if f.Query != "" {
w.add("t.search_key LIKE ? ESCAPE '!'", likePattern(f.Query))
addSearch(&w, "t.search_key", f.Query)
}
total, err := r.count(ctx, "SELECT COUNT(*) FROM tracks t WHERE "+w.sql(), w.args...)

View file

@ -10,18 +10,20 @@ import (
// letters that have no accent to take off but are written with the plain letters in a search:
// the key of "Straße" is "strasse" and the one of "Ørsted" is "orsted".
var searchLetters = strings.NewReplacer(
"ß", "ss", "æ", "ae", "œ", "oe", "ø", "o", "đ", "d", "ð", "d", "ł", "l", "ı", "i", "ħ", "h",
"ς", "σ", "ß", "ss", "æ", "ae", "œ", "oe", "ø", "o", "đ", "d", "ð", "d", "ł", "l", "ı", "i", "ħ", "h",
)
// searchKey is the key a search looks in: matchKey with the accents taken off, so "bjork" finds
// "Björk". It is not the key that identifies artists and albums, which keeps accents: otherwise
// "Beyoncé" and "Beyonce" would be one artist.
//
// Compatibility forms are folded too (the ligature "fi" is "fi", a full-width "A" is "a").
//
// Only the combining marks of the Latin, Greek and Cyrillic range (U+0300 to U+036F) go, so the
// marks that are part of a letter in other scripts (the voicing mark of a kana, a Hangul syllable)
// stay. It is computed here, not in SQL, so SQLite and PostgreSQL agree.
func searchKey(s string) string {
d := norm.NFD.String(matchKey(s))
d := norm.NFKD.String(matchKey(s))
var b strings.Builder
b.Grow(len(d))
for _, r := range d {

View file

@ -31,6 +31,14 @@ func TestSearchKeyIgnoresCaseAndAccents(t *testing.T) {
"Dvořák": "dvorak",
"Beyoncé Knowles": "beyonce knowles",
"école": "ecole", // an e and a separate accent
"İstanbul": "istanbul",
"PHỞ": "pho",
"ÉCOLE": "ecole", // a capital E and a separate accent
"ΣΟΦΟΣ": "σοφοσ",
"σοφός": "σοφοσ", // the final sigma is a sigma
"fire": "fire",
"ABC": "abc",
"́": "", // a lone accent says nothing
} {
if got := searchKey(in); got != want {
t.Errorf("searchKey(%q) = %q, want %q", in, got, want)
@ -139,3 +147,80 @@ func TestADiscoveredArtistGetsASearchKey(t *testing.T) {
}
})
}
func TestAQueryThatSaysNothingOnceFoldedMatchesNothing(t *testing.T) {
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
s := seedCatalog(t, db)
ctx := context.Background()
for _, q := range []string{"́", "́́"} {
if got, _ := s.repo.ListArtists(ctx, catalog.Filter{Query: q, Limit: 50}); got.Total != 0 {
t.Errorf("artists %q = %v, want none", q, artistNames(got))
}
if got, _ := s.repo.ListTracks(ctx, catalog.Filter{Query: q, Limit: 50}); got.Total != 0 {
t.Errorf("tracks %q = %v, want none", q, trackTitles(got))
}
if got, _ := s.repo.ListAlbums(ctx, catalog.Filter{Query: q, Limit: 50}); got.Total != 0 {
t.Errorf("albums %q = %v, want none", q, albumTitles(got))
}
}
})
}
func TestARetitledTrackIsFoundByItsNewTitle(t *testing.T) {
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
ctx := context.Background()
repo := NewLibraryRepo(db)
lib := mkLib(t, repo, "m")
if _, err := repo.SaveFile(ctx, lib, parsed("1.flac", "Old Name", "Album", "Band"), 0); err != nil {
t.Fatal(err)
}
existing, err := repo.ExistingFiles(ctx, lib)
if err != nil || len(existing) != 1 {
t.Fatalf("existing = %v err = %v", existing, err)
}
if _, err := repo.SaveFile(ctx, lib, parsed("1.flac", "Café Nuovo", "Album", "Band"), existing[0].ID); err != nil {
t.Fatal(err)
}
cat := NewCatalogRepo(db)
if got, _ := cat.ListTracks(ctx, catalog.Filter{Query: "cafe", Limit: 10}); !equal(trackTitles(got), []string{"Café Nuovo"}) {
t.Errorf("by the new title = %v", trackTitles(got))
}
if got, _ := cat.ListTracks(ctx, catalog.Filter{Query: "old name", Limit: 10}); got.Total != 0 {
t.Errorf("by the old title = %v", trackTitles(got))
}
})
}
// A refresh fixes both keys of a row together, and never merges what the accents keep apart.
func TestRefreshKeepsAccentedAndPlainArtistsApart(t *testing.T) {
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
ctx := context.Background()
repo := NewLibraryRepo(db)
lib := mkLib(t, repo, "m")
for i, name := range []string{"Beyoncé", "Beyonce"} {
if _, err := repo.SaveFile(ctx, lib, parsed(string(rune('a'+i))+".flac", "Song", "Album "+name, name), 0); err != nil {
t.Fatal(err)
}
}
if _, err := db.Exec("UPDATE artists SET name_key = 'wrong', search_key = ''"); err != nil {
t.Fatal(err)
}
if err := repo.RefreshMatchKeys(ctx); err != nil {
t.Fatal(err)
}
var keys []string
rows, err := db.QueryContext(ctx, "SELECT name_key || '/' || search_key FROM artists ORDER BY id")
if err != nil {
t.Fatal(err)
}
defer rows.Close()
for rows.Next() {
var k string
_ = rows.Scan(&k)
keys = append(keys, k)
}
if !equal(keys, []string{"beyoncé/beyonce", "beyonce/beyonce"}) {
t.Errorf("keys = %v, want the identity key with the accent and the search key without", keys)
}
})
}