diff --git a/internal/storage/library.go b/internal/storage/library.go index a3ea41d..c4a24e6 100644 --- a/internal/storage/library.go +++ b/internal/storage/library.go @@ -9,6 +9,7 @@ import ( "time" "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect" "codeberg.org/squidd/sonda/internal/library" "codeberg.org/squidd/sonda/internal/streaming" @@ -382,11 +383,7 @@ func refreshKeys(ctx context.Context, tx bun.Tx, tables []keyedTable) error { if err != nil { return fmt.Errorf("list %s: %w", t.table, err) } - type fix struct { - id int64 - match, find string - } - var fixes []fix + var fixes []keyFix for rows.Next() { var ( id int64 @@ -398,7 +395,7 @@ func refreshKeys(ctx context.Context, tx bun.Tx, tables []keyedTable) error { } m, s := matchKey(text), searchKey(text) if m != match || s != current { - fixes = append(fixes, fix{id: id, match: m, find: s}) + fixes = append(fixes, keyFix{id: id, match: m, find: s}) } } if err := rows.Err(); err != nil { @@ -406,15 +403,55 @@ func refreshKeys(ctx context.Context, tx bun.Tx, tables []keyedTable) error { return err } _ = rows.Close() - for _, f := range fixes { - if _, err := tx.ExecContext(ctx, "UPDATE "+t.table+" SET "+t.matchKey+" = ?, search_key = ? WHERE id = ?", f.match, f.find, f.id); err != nil { - return fmt.Errorf("update %s key: %w", t.table, err) + for start := 0; start < len(fixes); start += keyBatch { + batch := fixes[start:min(start+keyBatch, len(fixes))] + if err := updateKeys(ctx, tx, t, batch); err != nil { + return err } } } return nil } +// keyBatch is how many rows one UPDATE fixes: 500 rows are 1500 parameters, far below the limit +// of both databases, and one round trip instead of 500 on PostgreSQL. +const keyBatch = 500 + +// updateKeys fixes a batch of rows. On PostgreSQL it is one statement, and the rows come as a VALUES list, whose +// columns are called column1, column2 and column3. +func updateKeys(ctx context.Context, tx bun.Tx, t keyedTable, batch []keyFix) error { + if tx.Dialect().Name() == dialect.SQLite { + // The file is local: there is no round trip to save, and one UPDATE per row is faster + // than the join with a VALUES list. + for _, f := range batch { + if _, err := tx.ExecContext(ctx, "UPDATE "+t.table+" SET "+t.matchKey+" = ?, search_key = ? WHERE id = ?", f.match, f.find, f.id); err != nil { + return fmt.Errorf("update %s key: %w", t.table, err) + } + } + return nil + } + var b strings.Builder + args := make([]any, 0, len(batch)*3) + for i, f := range batch { + if i > 0 { + b.WriteString(", ") + } + b.WriteString("(CAST(? AS BIGINT), CAST(? AS TEXT), CAST(? AS TEXT))") + args = append(args, f.id, f.match, f.find) + } + q := "UPDATE " + t.table + " SET " + t.matchKey + " = v.column2, search_key = v.column3 FROM (VALUES " + b.String() + ") AS v WHERE " + t.table + ".id = v.column1" + if _, err := tx.ExecContext(ctx, q, args...); err != nil { + return fmt.Errorf("update %s keys: %w", t.table, err) + } + return nil +} + +// keyFix is the new keys of one row. +type keyFix struct { + id int64 + match, find string +} + // ---- helpers (run inside a transaction) ---- // findOrCreateArtist returns the artist a file belongs to, creating it if needed. diff --git a/internal/storage/search_keys_test.go b/internal/storage/search_keys_test.go index 4eb9a9a..5482a06 100644 --- a/internal/storage/search_keys_test.go +++ b/internal/storage/search_keys_test.go @@ -224,3 +224,38 @@ func TestRefreshKeepsAccentedAndPlainArtistsApart(t *testing.T) { } }) } + +// The rows to fix go in batches: a library with thousands of stale rows ends up with every key +// right, the rows that were already right keep their value, and a second run changes nothing. +func TestRefreshMatchKeysFixesEveryRowAcrossBatches(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, driver string) { + ctx := context.Background() + const n = 1250 // more than two batches + var insert string + if driver == DriverPostgres { + insert = "INSERT INTO artists (name, name_key, search_key) SELECT 'Sigur Rós ' || g, CASE WHEN g % 5 = 0 THEN 'sigur rós ' || g ELSE 'stale' END, CASE WHEN g % 5 = 0 THEN 'sigur ros ' || g ELSE '' END FROM generate_series(1, ?) g" + } else { + insert = "WITH RECURSIVE c(g) AS (SELECT 1 UNION ALL SELECT g+1 FROM c WHERE g < ?) INSERT INTO artists (name, name_key, search_key) SELECT 'Sigur Rós ' || g, CASE WHEN g % 5 = 0 THEN 'sigur rós ' || g ELSE 'stale' END, CASE WHEN g % 5 = 0 THEN 'sigur ros ' || g ELSE '' END FROM c" + } + if _, err := db.ExecContext(ctx, insert, n); err != nil { + t.Fatal(err) + } + repo := NewLibraryRepo(db) + for i := 0; i < 2; i++ { + if err := repo.RefreshMatchKeys(ctx); err != nil { + t.Fatal(err) + } + } + var wrong int + if err := db.QueryRow("SELECT count(*) FROM artists WHERE name_key <> lower(name) OR search_key <> replace(lower(name), 'ó', 'o')").Scan(&wrong); err != nil { + t.Fatal(err) + } + if wrong != 0 { + t.Errorf("%d artists have a wrong key", wrong) + } + var total int + if err := db.QueryRow("SELECT count(*) FROM artists").Scan(&total); err != nil || total != n { + t.Errorf("artists = %d err = %v, want %d", total, err, n) + } + }) +}