Merge pull request 'feat(search): search ignores accents' (#50) from feat/search-accents into main
This commit is contained in:
commit
227e267d73
13 changed files with 367 additions and 73 deletions
|
|
@ -253,7 +253,7 @@ Everything lives under `/api/v1`. Bodies are JSON. Errors always look like `{"er
|
|||
| admin | `GET /providers`, `PUT /providers/{name}`, `POST /providers/{name}/test`, `DELETE /providers/{name}/data` | See, set up and check Last.fm and MusicBrainz, and remove what they gave. The API key is write-only. |
|
||||
| admin | `GET`, `POST /users`; `PATCH`, `DELETE /users/{id}`; `PUT /users/{id}/password` | Manage users and the admin role. The last admin cannot be demoted or deleted. |
|
||||
|
||||
A library path must be an absolute, existing folder, never the filesystem root, and never inside or around another library. Search is a case-insensitive substring match: `BJÖRK` finds `Björk`, but accents are kept, so `bjork` does not.
|
||||
A library path must be an absolute, existing folder, never the filesystem root, and never inside or around another library. Search is a case- and accent-insensitive substring match: `bjork`, `BJORK` and `BJÖRK` all find `Björk`, and `strasse` finds `Straße`. Accents still tell artists apart: `Beyoncé` and `Beyonce` stay two artists, and a search for either finds both.
|
||||
|
||||
## Security notes
|
||||
|
||||
|
|
|
|||
|
|
@ -125,9 +125,13 @@ func run() error {
|
|||
gin.SetMode(gin.ReleaseMode)
|
||||
}
|
||||
libraryRepo := storage.NewLibraryRepo(db)
|
||||
// On the first start after an upgrade that adds a key this can take a while on a big library.
|
||||
logger.Info("refreshing the match keys of the library")
|
||||
keysStarted := time.Now()
|
||||
if err := libraryRepo.RefreshMatchKeys(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
logger.Info("match keys are up to date", zap.Duration("took", time.Since(keysStarted)))
|
||||
scanner := library.NewScanner(libraryRepo, library.TaglibReader{}, library.NewHub(), cfg.Library.ScanWorkers)
|
||||
scans := library.NewManager(scanner, libraryRepo)
|
||||
logScanResults(logger, scans)
|
||||
|
|
|
|||
2
go.mod
2
go.mod
|
|
@ -16,6 +16,7 @@ require (
|
|||
go.uber.org/zap v1.28.0
|
||||
golang.org/x/crypto v0.58.0
|
||||
golang.org/x/sync v0.24.0
|
||||
golang.org/x/text v0.43.0
|
||||
modernc.org/sqlite v1.62.0
|
||||
)
|
||||
|
||||
|
|
@ -64,7 +65,6 @@ require (
|
|||
golang.org/x/arch v0.22.0 // indirect
|
||||
golang.org/x/net v0.60.0 // indirect
|
||||
golang.org/x/sys v0.49.0 // indirect
|
||||
golang.org/x/text v0.43.0 // indirect
|
||||
google.golang.org/protobuf v1.36.10 // indirect
|
||||
modernc.org/libc v1.80.1 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
|
|
|
|||
|
|
@ -1900,8 +1900,8 @@ components:
|
|||
name: q
|
||||
in: query
|
||||
description: |
|
||||
Case-insensitive substring of the name or title. Accented letters are matched ignoring
|
||||
case (`BJÖRK` finds `Björk`), but accents are kept (`bjork` does not). At most 200 characters.
|
||||
Case- and accent-insensitive substring of the name or title: `bjork`, `BJORK` and `BJÖRK` all
|
||||
find `Björk`, and `strasse` finds `Straße`. At most 200 characters.
|
||||
schema: {type: string, maxLength: 200}
|
||||
Full:
|
||||
name: full
|
||||
|
|
|
|||
|
|
@ -56,14 +56,25 @@ func visible(alias string, libraryID int64) (string, []any) {
|
|||
return q + ")", args
|
||||
}
|
||||
|
||||
// likePattern turns a search text into a LIKE pattern for the match keys. The
|
||||
// likePattern turns a search text into a LIKE pattern for the search keys (no accents). The
|
||||
// wildcard characters of the text are escaped with '!', so "100%" only matches
|
||||
// a literal percent sign.
|
||||
func likePattern(q string) string {
|
||||
esc := strings.NewReplacer("!", "!!", "%", "!%", "_", "!_").Replace(matchKey(q))
|
||||
esc := strings.NewReplacer("!", "!!", "%", "!%", "_", "!_").Replace(searchKey(q))
|
||||
return "%" + esc + "%"
|
||||
}
|
||||
|
||||
// addSearch adds the condition for a search text on a column of search keys. A text that has
|
||||
// nothing left once folded (only accents) matches nothing, where a pattern made of it would match
|
||||
// everything; a text of blanks only is no search at all and matches everything.
|
||||
func addSearch(w *where, column, q string) {
|
||||
if strings.TrimSpace(q) != "" && searchKey(q) == "" {
|
||||
w.add("1 = 0")
|
||||
return
|
||||
}
|
||||
w.add(column+" LIKE ? ESCAPE '!'", likePattern(q))
|
||||
}
|
||||
|
||||
// binary orders by plain byte value. PostgreSQL would otherwise follow the
|
||||
// database locale while SQLite compares bytes, and listings would differ.
|
||||
func (r *CatalogRepo) binary(expr string) string {
|
||||
|
|
@ -143,7 +154,7 @@ func (r *CatalogRepo) listArtists(ctx context.Context, f catalog.Filter, id int6
|
|||
w.add("a.id = ?", id)
|
||||
}
|
||||
if f.Query != "" {
|
||||
w.add("a.name_key LIKE ? ESCAPE '!'", likePattern(f.Query))
|
||||
addSearch(&w, "a.search_key", f.Query)
|
||||
}
|
||||
|
||||
total, err := r.count(ctx, "SELECT COUNT(*) FROM artists a WHERE "+w.sql(), w.args...)
|
||||
|
|
@ -241,7 +252,7 @@ func (r *CatalogRepo) listAlbums(ctx context.Context, f catalog.Filter, id int64
|
|||
append([]any{f.ArtistID, f.ArtistID}, visArgs...)...)
|
||||
}
|
||||
if f.Query != "" {
|
||||
w.add("al.title_key LIKE ? ESCAPE '!'", likePattern(f.Query))
|
||||
addSearch(&w, "al.search_key", f.Query)
|
||||
}
|
||||
|
||||
total, err := r.count(ctx, "SELECT COUNT(*) FROM albums al WHERE "+w.sql(), w.args...)
|
||||
|
|
@ -351,7 +362,7 @@ func (r *CatalogRepo) listTracks(ctx context.Context, f catalog.Filter, id int64
|
|||
w.add("EXISTS (SELECT 1 FROM track_artists ta WHERE ta.track_id = t.id AND ta.artist_id = ?)", f.ArtistID)
|
||||
}
|
||||
if f.Query != "" {
|
||||
w.add("t.title_key LIKE ? ESCAPE '!'", likePattern(f.Query))
|
||||
addSearch(&w, "t.search_key", f.Query)
|
||||
}
|
||||
|
||||
total, err := r.count(ctx, "SELECT COUNT(*) FROM tracks t WHERE "+w.sql(), w.args...)
|
||||
|
|
|
|||
|
|
@ -211,7 +211,6 @@ func TestCatalogSearch(t *testing.T) {
|
|||
search := func(q string) catalog.Filter { return catalog.Filter{Query: q, Limit: 50} }
|
||||
|
||||
// Case folding, accented letters included, is the same on both databases.
|
||||
// Accents themselves are not folded: "bjork" does not find "Björk".
|
||||
for _, q := range []string{"björk", "BJÖRK", "jö"} {
|
||||
if got, _ := s.repo.ListArtists(ctx, search(q)); !equal(artistNames(got), []string{"Björk"}) {
|
||||
t.Errorf("artists %q = %v", q, artistNames(got))
|
||||
|
|
@ -226,9 +225,6 @@ func TestCatalogSearch(t *testing.T) {
|
|||
if got, _ := s.repo.ListAlbums(ctx, search("kind OF")); !equal(albumTitles(got), []string{"Kind of Blue"}) {
|
||||
t.Errorf("albums kind OF = %v", albumTitles(got))
|
||||
}
|
||||
if got, _ := s.repo.ListArtists(ctx, search("bjork")); got.Total != 0 {
|
||||
t.Errorf(`"bjork" = %v: accents are not folded`, artistNames(got))
|
||||
}
|
||||
if got, _ := s.repo.ListArtists(ctx, search("zzz")); got.Total != 0 || len(got.Items) != 0 {
|
||||
t.Errorf("no match: %+v", got)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -304,7 +304,7 @@ func insertDiscovered(ctx context.Context, tx bun.Tx, name, mbid string) (int64,
|
|||
}
|
||||
var id int64
|
||||
err := tx.QueryRowContext(ctx,
|
||||
"INSERT INTO artists (mbid, name, name_key, discovered) VALUES (?, ?, ?, ?) RETURNING id", m, name, matchKey(name), true).Scan(&id)
|
||||
"INSERT INTO artists (mbid, name, name_key, search_key, discovered) VALUES (?, ?, ?, ?, ?) RETURNING id", m, name, matchKey(name), searchKey(name), true).Scan(&id)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("insert artist: %w", err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,17 +20,19 @@ type artistModel struct {
|
|||
ID int64 `bun:"id,pk,autoincrement"`
|
||||
Name string `bun:"name,notnull"`
|
||||
NameKey string `bun:"name_key,notnull"`
|
||||
SearchKey string `bun:"search_key,notnull"`
|
||||
Discovered bool `bun:"discovered,notnull"` // named by a provider, never seen in a file
|
||||
}
|
||||
|
||||
type albumModel struct {
|
||||
bun.BaseModel `bun:"table:albums"`
|
||||
|
||||
ID int64 `bun:"id,pk,autoincrement"`
|
||||
Title string `bun:"title,notnull"`
|
||||
TitleKey string `bun:"title_key,notnull"`
|
||||
ArtistID int64 `bun:"artist_id,nullzero"`
|
||||
MBID string `bun:"mbid,nullzero"` // the MusicBrainz release group id, from the files' tags
|
||||
ID int64 `bun:"id,pk,autoincrement"`
|
||||
Title string `bun:"title,notnull"`
|
||||
TitleKey string `bun:"title_key,notnull"`
|
||||
SearchKey string `bun:"search_key,notnull"`
|
||||
ArtistID int64 `bun:"artist_id,nullzero"`
|
||||
MBID string `bun:"mbid,nullzero"` // the MusicBrainz release group id, from the files' tags
|
||||
}
|
||||
|
||||
type trackModel struct {
|
||||
|
|
@ -39,6 +41,7 @@ type trackModel struct {
|
|||
ID int64 `bun:"id,pk,autoincrement"`
|
||||
Title string `bun:"title,notnull"`
|
||||
TitleKey string `bun:"title_key,notnull"`
|
||||
SearchKey string `bun:"search_key,notnull"`
|
||||
DurationMs int64 `bun:"duration_ms,nullzero"`
|
||||
TrackNo int `bun:"track_no,nullzero"`
|
||||
DiscNo int `bun:"disc_no,nullzero"`
|
||||
|
|
@ -145,7 +148,7 @@ func (r *LibraryRepo) SaveFile(ctx context.Context, libraryID int64, f library.P
|
|||
}
|
||||
|
||||
track := &trackModel{
|
||||
Title: f.Title, TitleKey: matchKey(f.Title), DurationMs: f.DurationMs, TrackNo: f.TrackNo, DiscNo: f.DiscNo, AlbumID: albumID,
|
||||
Title: f.Title, TitleKey: matchKey(f.Title), SearchKey: searchKey(f.Title), DurationMs: f.DurationMs, TrackNo: f.TrackNo, DiscNo: f.DiscNo, AlbumID: albumID,
|
||||
}
|
||||
|
||||
if existingID == 0 {
|
||||
|
|
@ -180,7 +183,7 @@ func (r *LibraryRepo) SaveFile(ctx context.Context, libraryID int64, f library.P
|
|||
return err
|
||||
}
|
||||
if _, err := tx.NewUpdate().Model(track).
|
||||
Column("title", "title_key", "duration_ms", "track_no", "disc_no", "album_id").
|
||||
Column("title", "title_key", "search_key", "duration_ms", "track_no", "disc_no", "album_id").
|
||||
WherePK().Exec(ctx); err != nil {
|
||||
return fmt.Errorf("update track: %w", err)
|
||||
}
|
||||
|
|
@ -356,60 +359,62 @@ func (r *LibraryRepo) DeleteLibrary(ctx context.Context, id int64) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// RefreshMatchKeys recomputes the match keys of every artist, album and track
|
||||
// whose stored key differs from the current rule. It is idempotent and cheap
|
||||
// when nothing changed, so it can run at every start: rows created before
|
||||
// migration 0003 only have an approximate key from SQL lower().
|
||||
// RefreshMatchKeys recomputes the match keys and the search keys of every artist, album and
|
||||
// track whose stored key differs from the current rule. It is idempotent and cheap when nothing
|
||||
// changed, so it can run at every start: rows created before migration 0003 only have an
|
||||
// approximate key from SQL lower(), and those from before migration 0016 have no search key.
|
||||
func (r *LibraryRepo) RefreshMatchKeys(ctx context.Context) error {
|
||||
return r.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
|
||||
var artists []artistModel
|
||||
if err := tx.NewSelect().Model(&artists).Column("id", "name", "name_key").Scan(ctx); err != nil {
|
||||
return fmt.Errorf("list artists: %w", err)
|
||||
}
|
||||
for _, a := range artists {
|
||||
key := matchKey(a.Name)
|
||||
if key == a.NameKey {
|
||||
continue
|
||||
}
|
||||
if _, err := tx.NewUpdate().Model((*artistModel)(nil)).
|
||||
Set("name_key = ?", key).Where("id = ?", a.ID).Exec(ctx); err != nil {
|
||||
return fmt.Errorf("update artist key: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
var tracks []trackModel
|
||||
if err := tx.NewSelect().Model(&tracks).Column("id", "title", "title_key").Scan(ctx); err != nil {
|
||||
return fmt.Errorf("list tracks: %w", err)
|
||||
}
|
||||
for _, t := range tracks {
|
||||
key := matchKey(t.Title)
|
||||
if key == t.TitleKey {
|
||||
continue
|
||||
}
|
||||
if _, err := tx.NewUpdate().Model((*trackModel)(nil)).
|
||||
Set("title_key = ?", key).Where("id = ?", t.ID).Exec(ctx); err != nil {
|
||||
return fmt.Errorf("update track key: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
var albums []albumModel
|
||||
if err := tx.NewSelect().Model(&albums).Column("id", "title", "title_key").Scan(ctx); err != nil {
|
||||
return fmt.Errorf("list albums: %w", err)
|
||||
}
|
||||
for _, al := range albums {
|
||||
key := matchKey(al.Title)
|
||||
if key == al.TitleKey {
|
||||
continue
|
||||
}
|
||||
if _, err := tx.NewUpdate().Model((*albumModel)(nil)).
|
||||
Set("title_key = ?", key).Where("id = ?", al.ID).Exec(ctx); err != nil {
|
||||
return fmt.Errorf("update album key: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return refreshKeys(ctx, tx, []keyedTable{
|
||||
{"artists", "name", "name_key"},
|
||||
{"tracks", "title", "title_key"},
|
||||
{"albums", "title", "title_key"},
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// keyedTable is a table with a text column and the keys made from it.
|
||||
type keyedTable struct{ table, text, matchKey string }
|
||||
|
||||
func refreshKeys(ctx context.Context, tx bun.Tx, tables []keyedTable) error {
|
||||
for _, t := range tables {
|
||||
rows, err := tx.QueryContext(ctx, "SELECT id, "+t.text+", "+t.matchKey+", search_key FROM "+t.table)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list %s: %w", t.table, err)
|
||||
}
|
||||
type fix struct {
|
||||
id int64
|
||||
match, find string
|
||||
}
|
||||
var fixes []fix
|
||||
for rows.Next() {
|
||||
var (
|
||||
id int64
|
||||
text, match, current string
|
||||
)
|
||||
if err := rows.Scan(&id, &text, &match, ¤t); err != nil {
|
||||
_ = rows.Close()
|
||||
return err
|
||||
}
|
||||
m, s := matchKey(text), searchKey(text)
|
||||
if m != match || s != current {
|
||||
fixes = append(fixes, fix{id: id, match: m, find: s})
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
_ = rows.Close()
|
||||
return err
|
||||
}
|
||||
_ = rows.Close()
|
||||
for _, f := range fixes {
|
||||
if _, err := tx.ExecContext(ctx, "UPDATE "+t.table+" SET "+t.matchKey+" = ?, search_key = ? WHERE id = ?", f.match, f.find, f.id); err != nil {
|
||||
return fmt.Errorf("update %s key: %w", t.table, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---- helpers (run inside a transaction) ----
|
||||
|
||||
// findOrCreateArtist returns the artist a file belongs to, creating it if needed.
|
||||
|
|
@ -481,15 +486,15 @@ func findOrCreateArtist(ctx context.Context, tx bun.Tx, name, mbid string) (int6
|
|||
return byName.ID, exec("set artist id", "UPDATE artists SET mbid = ? WHERE id = ?", mbid, byName.ID)
|
||||
case haveID:
|
||||
return byID.ID, exec("confirm discovered artist",
|
||||
"UPDATE artists SET name = ?, name_key = ?, discovered = ? WHERE id = ?", name, key, false, byID.ID)
|
||||
"UPDATE artists SET name = ?, name_key = ?, search_key = ?, discovered = ? WHERE id = ?", name, key, searchKey(name), false, byID.ID)
|
||||
case haveName:
|
||||
return byName.ID, exec("adopt discovered artist",
|
||||
"UPDATE artists SET name = ?, name_key = ?, discovered = ?, mbid = ? WHERE id = ?", name, key, false, mbid, byName.ID)
|
||||
"UPDATE artists SET name = ?, name_key = ?, search_key = ?, discovered = ?, mbid = ? WHERE id = ?", name, key, searchKey(name), false, mbid, byName.ID)
|
||||
}
|
||||
var id int64
|
||||
if err := tx.QueryRowContext(ctx,
|
||||
"INSERT INTO artists (mbid, name, name_key, discovered) VALUES (?, ?, ?, ?) RETURNING id",
|
||||
mbid, name, key, false).Scan(&id); err != nil {
|
||||
"INSERT INTO artists (mbid, name, name_key, search_key, discovered) VALUES (?, ?, ?, ?, ?) RETURNING id",
|
||||
mbid, name, key, searchKey(name), false).Scan(&id); err != nil {
|
||||
return 0, fmt.Errorf("insert artist: %w", err)
|
||||
}
|
||||
return id, nil
|
||||
|
|
@ -500,7 +505,7 @@ func findOrCreateArtist(ctx context.Context, tx bun.Tx, name, mbid string) (int6
|
|||
return 0, fmt.Errorf("find artist: %w", err)
|
||||
}
|
||||
if !ok {
|
||||
a := artistModel{Name: name, NameKey: key}
|
||||
a := artistModel{Name: name, NameKey: key, SearchKey: searchKey(name)}
|
||||
if _, err := tx.NewInsert().Model(&a).Exec(ctx); err != nil {
|
||||
return 0, fmt.Errorf("insert artist: %w", err)
|
||||
}
|
||||
|
|
@ -508,7 +513,7 @@ func findOrCreateArtist(ctx context.Context, tx bun.Tx, name, mbid string) (int6
|
|||
}
|
||||
if found.Discovered {
|
||||
if err := exec("adopt discovered artist",
|
||||
"UPDATE artists SET name = ?, name_key = ?, discovered = ?, mbid = NULL WHERE id = ?", name, key, false, found.ID); err != nil {
|
||||
"UPDATE artists SET name = ?, name_key = ?, search_key = ?, discovered = ?, mbid = NULL WHERE id = ?", name, key, searchKey(name), false, found.ID); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if found.MBID != "" {
|
||||
|
|
@ -587,7 +592,7 @@ func findOrCreateAlbum(ctx context.Context, tx bun.Tx, title string, artistID in
|
|||
if !errors.Is(err, sql.ErrNoRows) {
|
||||
return 0, fmt.Errorf("find album: %w", err)
|
||||
}
|
||||
a = albumModel{Title: title, TitleKey: matchKey(title), ArtistID: artistID}
|
||||
a = albumModel{Title: title, TitleKey: matchKey(title), SearchKey: searchKey(title), ArtistID: artistID}
|
||||
if mbid != "" {
|
||||
var taken bool
|
||||
if err := tx.QueryRowContext(ctx, "SELECT EXISTS (SELECT 1 FROM albums WHERE mbid = ?)", mbid).Scan(&taken); err != nil {
|
||||
|
|
|
|||
36
internal/storage/search_key.go
Normal file
36
internal/storage/search_key.go
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
package storage
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"golang.org/x/text/unicode/norm"
|
||||
)
|
||||
|
||||
// letters that have no accent to take off but are written with the plain letters in a search:
|
||||
// the key of "Straße" is "strasse" and the one of "Ørsted" is "orsted".
|
||||
var searchLetters = strings.NewReplacer(
|
||||
"ς", "σ", "ß", "ss", "æ", "ae", "œ", "oe", "ø", "o", "đ", "d", "ð", "d", "ł", "l", "ı", "i", "ħ", "h",
|
||||
)
|
||||
|
||||
// searchKey is the key a search looks in: matchKey with the accents taken off, so "bjork" finds
|
||||
// "Björk". It is not the key that identifies artists and albums, which keeps accents: otherwise
|
||||
// "Beyoncé" and "Beyonce" would be one artist.
|
||||
//
|
||||
// Compatibility forms are folded too (the ligature "fi" is "fi", a full-width "A" is "a").
|
||||
//
|
||||
// Only the combining marks of the Latin, Greek and Cyrillic range (U+0300 to U+036F) go, so the
|
||||
// marks that are part of a letter in other scripts (the voicing mark of a kana, a Hangul syllable)
|
||||
// stay. It is computed here, not in SQL, so SQLite and PostgreSQL agree.
|
||||
func searchKey(s string) string {
|
||||
d := norm.NFKD.String(matchKey(s))
|
||||
var b strings.Builder
|
||||
b.Grow(len(d))
|
||||
for _, r := range d {
|
||||
if r >= 0x0300 && r <= 0x036F && unicode.Is(unicode.Mn, r) {
|
||||
continue
|
||||
}
|
||||
b.WriteRune(r)
|
||||
}
|
||||
return searchLetters.Replace(norm.NFC.String(b.String()))
|
||||
}
|
||||
226
internal/storage/search_keys_test.go
Normal file
226
internal/storage/search_keys_test.go
Normal file
|
|
@ -0,0 +1,226 @@
|
|||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/uptrace/bun"
|
||||
|
||||
"codeberg.org/squidd/sonda/internal/catalog"
|
||||
)
|
||||
|
||||
func TestSearchKeyIgnoresCaseAndAccents(t *testing.T) {
|
||||
for in, want := range map[string]string{
|
||||
"Björk": "bjork",
|
||||
"BJÖRK": "bjork",
|
||||
"Beyoncé": "beyonce",
|
||||
" Café ": "cafe",
|
||||
"Mötley Crüe": "motley crue",
|
||||
"Straße": "strasse",
|
||||
"Ørsted": "orsted",
|
||||
"Æon": "aeon",
|
||||
"Łódź": "lodz",
|
||||
"Sigur Rós": "sigur ros",
|
||||
"Ελλάδα": "ελλαδα",
|
||||
"ñandú": "nandu",
|
||||
"AC/DC": "ac/dc",
|
||||
"plain": "plain",
|
||||
"宇多田ヒカル": "宇多田ヒカル",
|
||||
"ガガ": "ガガ", // the marks of kana are part of the letter
|
||||
"한글": "한글",
|
||||
"Dvořák": "dvorak",
|
||||
"Beyoncé Knowles": "beyonce knowles",
|
||||
"école": "ecole", // an e and a separate accent
|
||||
"İstanbul": "istanbul",
|
||||
"PHỞ": "pho",
|
||||
"ÉCOLE": "ecole", // a capital E and a separate accent
|
||||
"ΣΟΦΟΣ": "σοφοσ",
|
||||
"σοφός": "σοφοσ", // the final sigma is a sigma
|
||||
"fire": "fire",
|
||||
"ABC": "abc",
|
||||
"́": "", // a lone accent says nothing
|
||||
} {
|
||||
if got := searchKey(in); got != want {
|
||||
t.Errorf("searchKey(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchFindsNamesWithoutTheAccentsAndTheOtherWayRound(t *testing.T) {
|
||||
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
|
||||
s := seedCatalog(t, db)
|
||||
ctx := context.Background()
|
||||
search := func(q string) catalog.Filter { return catalog.Filter{Query: q, Limit: 50} }
|
||||
|
||||
for _, q := range []string{"bjork", "BJORK", "björk", "BJÖRK", "bjö", "jork"} {
|
||||
if got, _ := s.repo.ListArtists(ctx, search(q)); !equal(artistNames(got), []string{"Björk"}) {
|
||||
t.Errorf("artists %q = %v", q, artistNames(got))
|
||||
}
|
||||
}
|
||||
if got, _ := s.repo.ListArtists(ctx, search("unicode")); !equal(artistNames(got), []string{"Ünïcode Band"}) {
|
||||
t.Errorf("artists unicode = %v", artistNames(got))
|
||||
}
|
||||
if got, _ := s.repo.ListTracks(ctx, search("unicode trck")); !equal(trackTitles(got), []string{"Ünïcode Trck"}) {
|
||||
t.Errorf("tracks = %v", trackTitles(got))
|
||||
}
|
||||
if got, _ := s.repo.ListAlbums(ctx, search("post")); !equal(albumTitles(got), []string{"Post"}) {
|
||||
t.Errorf("albums = %v", albumTitles(got))
|
||||
}
|
||||
// The wildcards of the text are still literal.
|
||||
if got, _ := s.repo.ListTracks(ctx, search("100%")); !equal(trackTitles(got), []string{"100% Pure"}) {
|
||||
t.Errorf("tracks 100%% = %v", trackTitles(got))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// "Beyoncé" and "Beyonce" are two names as far as identity goes: the scanner must not merge
|
||||
// them; only the search treats them alike.
|
||||
func TestAccentsDoNotMergeArtistsButTheSearchFindsBoth(t *testing.T) {
|
||||
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
|
||||
ctx := context.Background()
|
||||
repo := NewLibraryRepo(db)
|
||||
lib := mkLib(t, repo, "m")
|
||||
if _, err := repo.SaveFile(ctx, lib, parsed("1.flac", "Halo", "I Am", "Beyoncé"), 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := repo.SaveFile(ctx, lib, parsed("2.flac", "Other", "Other Album", "Beyonce"), 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := count(t, db, "SELECT COUNT(*) FROM artists"); n != 2 {
|
||||
t.Fatalf("artists = %d, want 2: accents identify an artist", n)
|
||||
}
|
||||
got, err := NewCatalogRepo(db).ListArtists(ctx, catalog.Filter{Query: "beyonce", Limit: 10})
|
||||
if err != nil || got.Total != 2 {
|
||||
t.Errorf("search = %v err = %v, want both", artistNames(got), err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRefreshMatchKeysFillsTheSearchKeyOfOldRows(t *testing.T) {
|
||||
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
|
||||
ctx := context.Background()
|
||||
repo := NewLibraryRepo(db)
|
||||
lib := mkLib(t, repo, "m")
|
||||
if _, err := repo.SaveFile(ctx, lib, parsed("1.flac", "Army of Me", "Post", "Björk"), 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Rows from before the column existed have none.
|
||||
for _, q := range []string{"UPDATE artists SET search_key = ''", "UPDATE albums SET search_key = ''", "UPDATE tracks SET search_key = ''"} {
|
||||
if _, err := db.Exec(q); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if got, _ := NewCatalogRepo(db).ListArtists(ctx, catalog.Filter{Query: "bjork", Limit: 10}); got.Total != 0 {
|
||||
t.Fatal("found an artist that has no search key")
|
||||
}
|
||||
if err := repo.RefreshMatchKeys(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := repo.RefreshMatchKeys(ctx); err != nil { // idempotent
|
||||
t.Fatal(err)
|
||||
}
|
||||
cat := NewCatalogRepo(db)
|
||||
if got, _ := cat.ListArtists(ctx, catalog.Filter{Query: "bjork", Limit: 10}); got.Total != 1 {
|
||||
t.Errorf("artists = %d, want 1", got.Total)
|
||||
}
|
||||
if got, _ := cat.ListAlbums(ctx, catalog.Filter{Query: "post", Limit: 10}); got.Total != 1 {
|
||||
t.Errorf("albums = %d, want 1", got.Total)
|
||||
}
|
||||
if got, _ := cat.ListTracks(ctx, catalog.Filter{Query: "army", Limit: 10}); got.Total != 1 {
|
||||
t.Errorf("tracks = %d, want 1", got.Total)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestADiscoveredArtistGetsASearchKey(t *testing.T) {
|
||||
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
|
||||
ctx := context.Background()
|
||||
if err := db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
|
||||
_, err := insertDiscovered(ctx, tx, "Sigur Rós", "")
|
||||
return err
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var key string
|
||||
if err := db.QueryRow("SELECT search_key FROM artists WHERE name = 'Sigur Rós'").Scan(&key); err != nil || key != "sigur ros" {
|
||||
t.Errorf("search key = %q err = %v", key, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestAQueryThatSaysNothingOnceFoldedMatchesNothing(t *testing.T) {
|
||||
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
|
||||
s := seedCatalog(t, db)
|
||||
ctx := context.Background()
|
||||
for _, q := range []string{"́", "́́"} {
|
||||
if got, _ := s.repo.ListArtists(ctx, catalog.Filter{Query: q, Limit: 50}); got.Total != 0 {
|
||||
t.Errorf("artists %q = %v, want none", q, artistNames(got))
|
||||
}
|
||||
if got, _ := s.repo.ListTracks(ctx, catalog.Filter{Query: q, Limit: 50}); got.Total != 0 {
|
||||
t.Errorf("tracks %q = %v, want none", q, trackTitles(got))
|
||||
}
|
||||
if got, _ := s.repo.ListAlbums(ctx, catalog.Filter{Query: q, Limit: 50}); got.Total != 0 {
|
||||
t.Errorf("albums %q = %v, want none", q, albumTitles(got))
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestARetitledTrackIsFoundByItsNewTitle(t *testing.T) {
|
||||
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
|
||||
ctx := context.Background()
|
||||
repo := NewLibraryRepo(db)
|
||||
lib := mkLib(t, repo, "m")
|
||||
if _, err := repo.SaveFile(ctx, lib, parsed("1.flac", "Old Name", "Album", "Band"), 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
existing, err := repo.ExistingFiles(ctx, lib)
|
||||
if err != nil || len(existing) != 1 {
|
||||
t.Fatalf("existing = %v err = %v", existing, err)
|
||||
}
|
||||
if _, err := repo.SaveFile(ctx, lib, parsed("1.flac", "Café Nuovo", "Album", "Band"), existing[0].ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cat := NewCatalogRepo(db)
|
||||
if got, _ := cat.ListTracks(ctx, catalog.Filter{Query: "cafe", Limit: 10}); !equal(trackTitles(got), []string{"Café Nuovo"}) {
|
||||
t.Errorf("by the new title = %v", trackTitles(got))
|
||||
}
|
||||
if got, _ := cat.ListTracks(ctx, catalog.Filter{Query: "old name", Limit: 10}); got.Total != 0 {
|
||||
t.Errorf("by the old title = %v", trackTitles(got))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// A refresh fixes both keys of a row together, and never merges what the accents keep apart.
|
||||
func TestRefreshKeepsAccentedAndPlainArtistsApart(t *testing.T) {
|
||||
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
|
||||
ctx := context.Background()
|
||||
repo := NewLibraryRepo(db)
|
||||
lib := mkLib(t, repo, "m")
|
||||
for i, name := range []string{"Beyoncé", "Beyonce"} {
|
||||
if _, err := repo.SaveFile(ctx, lib, parsed(string(rune('a'+i))+".flac", "Song", "Album "+name, name), 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := db.Exec("UPDATE artists SET name_key = 'wrong', search_key = ''"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := repo.RefreshMatchKeys(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var keys []string
|
||||
rows, err := db.QueryContext(ctx, "SELECT name_key || '/' || search_key FROM artists ORDER BY id")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var k string
|
||||
_ = rows.Scan(&k)
|
||||
keys = append(keys, k)
|
||||
}
|
||||
if !equal(keys, []string{"beyoncé/beyonce", "beyonce/beyonce"}) {
|
||||
t.Errorf("keys = %v, want the identity key with the accent and the search key without", keys)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
@ -156,8 +156,8 @@ func TestMigrateCreatesSchemaAndIsIdempotent(t *testing.T) {
|
|||
if err := db.QueryRow("SELECT COUNT(*) FROM schema_migrations").Scan(&applied); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if applied != 15 {
|
||||
t.Errorf("applied migrations = %d, want 15", applied)
|
||||
if applied != 16 {
|
||||
t.Errorf("applied migrations = %d, want 16", applied)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
|||
8
migrations/postgres/0016_search_keys.sql
Normal file
8
migrations/postgres/0016_search_keys.sql
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
-- A key for searching, apart from the keys that identify artists, albums and tracks: it has
|
||||
-- the accents taken off, so "bjork" finds "Björk", while "Beyoncé" and "Beyonce" stay two
|
||||
-- artists. It is computed in Go (Unicode-aware) and filled in for existing rows at start,
|
||||
-- by RefreshMatchKeys.
|
||||
|
||||
ALTER TABLE artists ADD COLUMN search_key TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE albums ADD COLUMN search_key TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE tracks ADD COLUMN search_key TEXT NOT NULL DEFAULT '';
|
||||
8
migrations/sqlite/0016_search_keys.sql
Normal file
8
migrations/sqlite/0016_search_keys.sql
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
-- A key for searching, apart from the keys that identify artists, albums and tracks: it has
|
||||
-- the accents taken off, so "bjork" finds "Björk", while "Beyoncé" and "Beyonce" stay two
|
||||
-- artists. It is computed in Go (Unicode-aware) and filled in for existing rows at start,
|
||||
-- by RefreshMatchKeys.
|
||||
|
||||
ALTER TABLE artists ADD COLUMN search_key TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE albums ADD COLUMN search_key TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE tracks ADD COLUMN search_key TEXT NOT NULL DEFAULT '';
|
||||
Loading…
Reference in a new issue