From 77be31ea4ac1616ef64cbe7d968f84c6889cb13b Mon Sep 17 00:00:00 2001 From: tgiachi Date: Thu, 12 Nov 2026 14:08:40 +0100 Subject: [PATCH] perf(storage): fix stale match and search keys in batches on PostgreSQL The refresh at start fixed each stale row with its own UPDATE. On PostgreSQL that is one round trip per row: 100,000 stale rows took 5.3 s on localhost, and more over a network. The rows now go 500 at a time in one UPDATE ... FROM (VALUES ...): 1.9 s for the same rows. SQLite keeps one UPDATE per row, which is faster there (no round trip to save). A run with nothing to fix is unchanged (about 0.1 s per 100,000 rows). A test with 1,250 stale rows covers more than two batches on both databases. --- internal/storage/library.go | 55 +++++++++++++++++++++++----- internal/storage/search_keys_test.go | 35 ++++++++++++++++++ 2 files changed, 81 insertions(+), 9 deletions(-) diff --git a/internal/storage/library.go b/internal/storage/library.go index a3ea41d..c4a24e6 100644 --- a/internal/storage/library.go +++ b/internal/storage/library.go @@ -9,6 +9,7 @@ import ( "time" "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect" "codeberg.org/squidd/sonda/internal/library" "codeberg.org/squidd/sonda/internal/streaming" @@ -382,11 +383,7 @@ func refreshKeys(ctx context.Context, tx bun.Tx, tables []keyedTable) error { if err != nil { return fmt.Errorf("list %s: %w", t.table, err) } - type fix struct { - id int64 - match, find string - } - var fixes []fix + var fixes []keyFix for rows.Next() { var ( id int64 @@ -398,7 +395,7 @@ func refreshKeys(ctx context.Context, tx bun.Tx, tables []keyedTable) error { } m, s := matchKey(text), searchKey(text) if m != match || s != current { - fixes = append(fixes, fix{id: id, match: m, find: s}) + fixes = append(fixes, keyFix{id: id, match: m, find: s}) } } if err := rows.Err(); err != nil { @@ -406,15 +403,55 @@ func refreshKeys(ctx context.Context, tx bun.Tx, tables []keyedTable) error { return err } _ = rows.Close() - for _, f := range fixes { - if _, err := tx.ExecContext(ctx, "UPDATE "+t.table+" SET "+t.matchKey+" = ?, search_key = ? WHERE id = ?", f.match, f.find, f.id); err != nil { - return fmt.Errorf("update %s key: %w", t.table, err) + for start := 0; start < len(fixes); start += keyBatch { + batch := fixes[start:min(start+keyBatch, len(fixes))] + if err := updateKeys(ctx, tx, t, batch); err != nil { + return err } } } return nil } +// keyBatch is how many rows one UPDATE fixes: 500 rows are 1500 parameters, far below the limit +// of both databases, and one round trip instead of 500 on PostgreSQL. +const keyBatch = 500 + +// updateKeys fixes a batch of rows. On PostgreSQL it is one statement, and the rows come as a VALUES list, whose +// columns are called column1, column2 and column3. +func updateKeys(ctx context.Context, tx bun.Tx, t keyedTable, batch []keyFix) error { + if tx.Dialect().Name() == dialect.SQLite { + // The file is local: there is no round trip to save, and one UPDATE per row is faster + // than the join with a VALUES list. + for _, f := range batch { + if _, err := tx.ExecContext(ctx, "UPDATE "+t.table+" SET "+t.matchKey+" = ?, search_key = ? WHERE id = ?", f.match, f.find, f.id); err != nil { + return fmt.Errorf("update %s key: %w", t.table, err) + } + } + return nil + } + var b strings.Builder + args := make([]any, 0, len(batch)*3) + for i, f := range batch { + if i > 0 { + b.WriteString(", ") + } + b.WriteString("(CAST(? AS BIGINT), CAST(? AS TEXT), CAST(? AS TEXT))") + args = append(args, f.id, f.match, f.find) + } + q := "UPDATE " + t.table + " SET " + t.matchKey + " = v.column2, search_key = v.column3 FROM (VALUES " + b.String() + ") AS v WHERE " + t.table + ".id = v.column1" + if _, err := tx.ExecContext(ctx, q, args...); err != nil { + return fmt.Errorf("update %s keys: %w", t.table, err) + } + return nil +} + +// keyFix is the new keys of one row. +type keyFix struct { + id int64 + match, find string +} + // ---- helpers (run inside a transaction) ---- // findOrCreateArtist returns the artist a file belongs to, creating it if needed. diff --git a/internal/storage/search_keys_test.go b/internal/storage/search_keys_test.go index 4eb9a9a..5482a06 100644 --- a/internal/storage/search_keys_test.go +++ b/internal/storage/search_keys_test.go @@ -224,3 +224,38 @@ func TestRefreshKeepsAccentedAndPlainArtistsApart(t *testing.T) { } }) } + +// The rows to fix go in batches: a library with thousands of stale rows ends up with every key +// right, the rows that were already right keep their value, and a second run changes nothing. +func TestRefreshMatchKeysFixesEveryRowAcrossBatches(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, driver string) { + ctx := context.Background() + const n = 1250 // more than two batches + var insert string + if driver == DriverPostgres { + insert = "INSERT INTO artists (name, name_key, search_key) SELECT 'Sigur Rós ' || g, CASE WHEN g % 5 = 0 THEN 'sigur rós ' || g ELSE 'stale' END, CASE WHEN g % 5 = 0 THEN 'sigur ros ' || g ELSE '' END FROM generate_series(1, ?) g" + } else { + insert = "WITH RECURSIVE c(g) AS (SELECT 1 UNION ALL SELECT g+1 FROM c WHERE g < ?) INSERT INTO artists (name, name_key, search_key) SELECT 'Sigur Rós ' || g, CASE WHEN g % 5 = 0 THEN 'sigur rós ' || g ELSE 'stale' END, CASE WHEN g % 5 = 0 THEN 'sigur ros ' || g ELSE '' END FROM c" + } + if _, err := db.ExecContext(ctx, insert, n); err != nil { + t.Fatal(err) + } + repo := NewLibraryRepo(db) + for i := 0; i < 2; i++ { + if err := repo.RefreshMatchKeys(ctx); err != nil { + t.Fatal(err) + } + } + var wrong int + if err := db.QueryRow("SELECT count(*) FROM artists WHERE name_key <> lower(name) OR search_key <> replace(lower(name), 'ó', 'o')").Scan(&wrong); err != nil { + t.Fatal(err) + } + if wrong != 0 { + t.Errorf("%d artists have a wrong key", wrong) + } + var total int + if err := db.QueryRow("SELECT count(*) FROM artists").Scan(&total); err != nil || total != n { + t.Errorf("artists = %d err = %v, want %d", total, err, n) + } + }) +}