fix(search): empty folded queries, compatibility forms, final sigma, start-up log
- A search text that has nothing left once its accents are taken off matches nothing instead of everything. - The search key folds compatibility forms (the ligature "fi", full-width letters) and the Greek final sigma. - The start-up refresh of the keys logs when it starts and how long it took, since it can take a while on a big library the first time. - More tests: a retitled track is found by its new title, a refresh fixes both keys and keeps "Beyoncé" and "Beyonce" apart, more cases for the key.
This commit is contained in:
parent
271c88743c
commit
4ec3ae3bb0
4 changed files with 107 additions and 5 deletions
|
|
@ -125,9 +125,13 @@ func run() error {
|
|||
gin.SetMode(gin.ReleaseMode)
|
||||
}
|
||||
libraryRepo := storage.NewLibraryRepo(db)
|
||||
// On the first start after an upgrade that adds a key this can take a while on a big library.
|
||||
logger.Info("refreshing the match keys of the library")
|
||||
keysStarted := time.Now()
|
||||
if err := libraryRepo.RefreshMatchKeys(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
logger.Info("match keys are up to date", zap.Duration("took", time.Since(keysStarted)))
|
||||
scanner := library.NewScanner(libraryRepo, library.TaglibReader{}, library.NewHub(), cfg.Library.ScanWorkers)
|
||||
scans := library.NewManager(scanner, libraryRepo)
|
||||
logScanResults(logger, scans)
|
||||
|
|
|
|||
|
|
@ -64,6 +64,17 @@ func likePattern(q string) string {
|
|||
return "%" + esc + "%"
|
||||
}
|
||||
|
||||
// addSearch adds the condition for a search text on a column of search keys. A text that has
|
||||
// nothing left once folded (only accents) matches nothing, where a pattern made of it would match
|
||||
// everything; a text of blanks only is no search at all and matches everything.
|
||||
func addSearch(w *where, column, q string) {
|
||||
if strings.TrimSpace(q) != "" && searchKey(q) == "" {
|
||||
w.add("1 = 0")
|
||||
return
|
||||
}
|
||||
w.add(column+" LIKE ? ESCAPE '!'", likePattern(q))
|
||||
}
|
||||
|
||||
// binary orders by plain byte value. PostgreSQL would otherwise follow the
|
||||
// database locale while SQLite compares bytes, and listings would differ.
|
||||
func (r *CatalogRepo) binary(expr string) string {
|
||||
|
|
@ -143,7 +154,7 @@ func (r *CatalogRepo) listArtists(ctx context.Context, f catalog.Filter, id int6
|
|||
w.add("a.id = ?", id)
|
||||
}
|
||||
if f.Query != "" {
|
||||
w.add("a.search_key LIKE ? ESCAPE '!'", likePattern(f.Query))
|
||||
addSearch(&w, "a.search_key", f.Query)
|
||||
}
|
||||
|
||||
total, err := r.count(ctx, "SELECT COUNT(*) FROM artists a WHERE "+w.sql(), w.args...)
|
||||
|
|
@ -241,7 +252,7 @@ func (r *CatalogRepo) listAlbums(ctx context.Context, f catalog.Filter, id int64
|
|||
append([]any{f.ArtistID, f.ArtistID}, visArgs...)...)
|
||||
}
|
||||
if f.Query != "" {
|
||||
w.add("al.search_key LIKE ? ESCAPE '!'", likePattern(f.Query))
|
||||
addSearch(&w, "al.search_key", f.Query)
|
||||
}
|
||||
|
||||
total, err := r.count(ctx, "SELECT COUNT(*) FROM albums al WHERE "+w.sql(), w.args...)
|
||||
|
|
@ -351,7 +362,7 @@ func (r *CatalogRepo) listTracks(ctx context.Context, f catalog.Filter, id int64
|
|||
w.add("EXISTS (SELECT 1 FROM track_artists ta WHERE ta.track_id = t.id AND ta.artist_id = ?)", f.ArtistID)
|
||||
}
|
||||
if f.Query != "" {
|
||||
w.add("t.search_key LIKE ? ESCAPE '!'", likePattern(f.Query))
|
||||
addSearch(&w, "t.search_key", f.Query)
|
||||
}
|
||||
|
||||
total, err := r.count(ctx, "SELECT COUNT(*) FROM tracks t WHERE "+w.sql(), w.args...)
|
||||
|
|
|
|||
|
|
@ -10,18 +10,20 @@ import (
|
|||
// letters that have no accent to take off but are written with the plain letters in a search:
|
||||
// the key of "Straße" is "strasse" and the one of "Ørsted" is "orsted".
|
||||
var searchLetters = strings.NewReplacer(
|
||||
"ß", "ss", "æ", "ae", "œ", "oe", "ø", "o", "đ", "d", "ð", "d", "ł", "l", "ı", "i", "ħ", "h",
|
||||
"ς", "σ", "ß", "ss", "æ", "ae", "œ", "oe", "ø", "o", "đ", "d", "ð", "d", "ł", "l", "ı", "i", "ħ", "h",
|
||||
)
|
||||
|
||||
// searchKey is the key a search looks in: matchKey with the accents taken off, so "bjork" finds
|
||||
// "Björk". It is not the key that identifies artists and albums, which keeps accents: otherwise
|
||||
// "Beyoncé" and "Beyonce" would be one artist.
|
||||
//
|
||||
// Compatibility forms are folded too (the ligature "fi" is "fi", a full-width "A" is "a").
|
||||
//
|
||||
// Only the combining marks of the Latin, Greek and Cyrillic range (U+0300 to U+036F) go, so the
|
||||
// marks that are part of a letter in other scripts (the voicing mark of a kana, a Hangul syllable)
|
||||
// stay. It is computed here, not in SQL, so SQLite and PostgreSQL agree.
|
||||
func searchKey(s string) string {
|
||||
d := norm.NFD.String(matchKey(s))
|
||||
d := norm.NFKD.String(matchKey(s))
|
||||
var b strings.Builder
|
||||
b.Grow(len(d))
|
||||
for _, r := range d {
|
||||
|
|
|
|||
|
|
@ -31,6 +31,14 @@ func TestSearchKeyIgnoresCaseAndAccents(t *testing.T) {
|
|||
"Dvořák": "dvorak",
|
||||
"Beyoncé Knowles": "beyonce knowles",
|
||||
"école": "ecole", // an e and a separate accent
|
||||
"İstanbul": "istanbul",
|
||||
"PHỞ": "pho",
|
||||
"ÉCOLE": "ecole", // a capital E and a separate accent
|
||||
"ΣΟΦΟΣ": "σοφοσ",
|
||||
"σοφός": "σοφοσ", // the final sigma is a sigma
|
||||
"fire": "fire",
|
||||
"ABC": "abc",
|
||||
"́": "", // a lone accent says nothing
|
||||
} {
|
||||
if got := searchKey(in); got != want {
|
||||
t.Errorf("searchKey(%q) = %q, want %q", in, got, want)
|
||||
|
|
@ -139,3 +147,80 @@ func TestADiscoveredArtistGetsASearchKey(t *testing.T) {
|
|||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestAQueryThatSaysNothingOnceFoldedMatchesNothing(t *testing.T) {
|
||||
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
|
||||
s := seedCatalog(t, db)
|
||||
ctx := context.Background()
|
||||
for _, q := range []string{"́", "́́"} {
|
||||
if got, _ := s.repo.ListArtists(ctx, catalog.Filter{Query: q, Limit: 50}); got.Total != 0 {
|
||||
t.Errorf("artists %q = %v, want none", q, artistNames(got))
|
||||
}
|
||||
if got, _ := s.repo.ListTracks(ctx, catalog.Filter{Query: q, Limit: 50}); got.Total != 0 {
|
||||
t.Errorf("tracks %q = %v, want none", q, trackTitles(got))
|
||||
}
|
||||
if got, _ := s.repo.ListAlbums(ctx, catalog.Filter{Query: q, Limit: 50}); got.Total != 0 {
|
||||
t.Errorf("albums %q = %v, want none", q, albumTitles(got))
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestARetitledTrackIsFoundByItsNewTitle(t *testing.T) {
|
||||
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
|
||||
ctx := context.Background()
|
||||
repo := NewLibraryRepo(db)
|
||||
lib := mkLib(t, repo, "m")
|
||||
if _, err := repo.SaveFile(ctx, lib, parsed("1.flac", "Old Name", "Album", "Band"), 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
existing, err := repo.ExistingFiles(ctx, lib)
|
||||
if err != nil || len(existing) != 1 {
|
||||
t.Fatalf("existing = %v err = %v", existing, err)
|
||||
}
|
||||
if _, err := repo.SaveFile(ctx, lib, parsed("1.flac", "Café Nuovo", "Album", "Band"), existing[0].ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cat := NewCatalogRepo(db)
|
||||
if got, _ := cat.ListTracks(ctx, catalog.Filter{Query: "cafe", Limit: 10}); !equal(trackTitles(got), []string{"Café Nuovo"}) {
|
||||
t.Errorf("by the new title = %v", trackTitles(got))
|
||||
}
|
||||
if got, _ := cat.ListTracks(ctx, catalog.Filter{Query: "old name", Limit: 10}); got.Total != 0 {
|
||||
t.Errorf("by the old title = %v", trackTitles(got))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// A refresh fixes both keys of a row together, and never merges what the accents keep apart.
|
||||
func TestRefreshKeepsAccentedAndPlainArtistsApart(t *testing.T) {
|
||||
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
|
||||
ctx := context.Background()
|
||||
repo := NewLibraryRepo(db)
|
||||
lib := mkLib(t, repo, "m")
|
||||
for i, name := range []string{"Beyoncé", "Beyonce"} {
|
||||
if _, err := repo.SaveFile(ctx, lib, parsed(string(rune('a'+i))+".flac", "Song", "Album "+name, name), 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := db.Exec("UPDATE artists SET name_key = 'wrong', search_key = ''"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := repo.RefreshMatchKeys(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var keys []string
|
||||
rows, err := db.QueryContext(ctx, "SELECT name_key || '/' || search_key FROM artists ORDER BY id")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var k string
|
||||
_ = rows.Scan(&k)
|
||||
keys = append(keys, k)
|
||||
}
|
||||
if !equal(keys, []string{"beyoncé/beyonce", "beyonce/beyonce"}) {
|
||||
t.Errorf("keys = %v, want the identity key with the accent and the search key without", keys)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue