Merge pull request 'perf(storage): fix stale search keys in batches on PostgreSQL' (#63) from perf/refresh-keys-batches into main

This commit is contained in:
squidd 2026-11-12 14:10:18 +01:00
commit 850ae11f3d
2 changed files with 79 additions and 7 deletions

View file

@ -9,6 +9,7 @@ import (
"time"
"github.com/uptrace/bun"
"github.com/uptrace/bun/dialect"
"codeberg.org/squidd/sonda/internal/library"
"codeberg.org/squidd/sonda/internal/streaming"
@ -382,11 +383,7 @@ func refreshKeys(ctx context.Context, tx bun.Tx, tables []keyedTable) error {
if err != nil {
return fmt.Errorf("list %s: %w", t.table, err)
}
type fix struct {
id int64
match, find string
}
var fixes []fix
var fixes []keyFix
for rows.Next() {
var (
id int64
@ -398,7 +395,7 @@ func refreshKeys(ctx context.Context, tx bun.Tx, tables []keyedTable) error {
}
m, s := matchKey(text), searchKey(text)
if m != match || s != current {
fixes = append(fixes, fix{id: id, match: m, find: s})
fixes = append(fixes, keyFix{id: id, match: m, find: s})
}
}
if err := rows.Err(); err != nil {
@ -406,15 +403,55 @@ func refreshKeys(ctx context.Context, tx bun.Tx, tables []keyedTable) error {
return err
}
_ = rows.Close()
for _, f := range fixes {
if _, err := tx.ExecContext(ctx, "UPDATE "+t.table+" SET "+t.matchKey+" = ?, search_key = ? WHERE id = ?", f.match, f.find, f.id); err != nil {
return fmt.Errorf("update %s key: %w", t.table, err)
for start := 0; start < len(fixes); start += keyBatch {
batch := fixes[start:min(start+keyBatch, len(fixes))]
if err := updateKeys(ctx, tx, t, batch); err != nil {
return err
}
}
}
return nil
}
// keyBatch is how many rows one UPDATE fixes: 500 rows are 1500 parameters, far below the limit
// of both databases, and one round trip instead of 500 on PostgreSQL.
const keyBatch = 500
// updateKeys fixes a batch of rows. On PostgreSQL it is one statement, and the rows come as a VALUES list, whose
// columns are called column1, column2 and column3.
func updateKeys(ctx context.Context, tx bun.Tx, t keyedTable, batch []keyFix) error {
if tx.Dialect().Name() == dialect.SQLite {
// The file is local: there is no round trip to save, and one UPDATE per row is faster
// than the join with a VALUES list.
for _, f := range batch {
if _, err := tx.ExecContext(ctx, "UPDATE "+t.table+" SET "+t.matchKey+" = ?, search_key = ? WHERE id = ?", f.match, f.find, f.id); err != nil {
return fmt.Errorf("update %s key: %w", t.table, err)
}
}
return nil
}
var b strings.Builder
args := make([]any, 0, len(batch)*3)
for i, f := range batch {
if i > 0 {
b.WriteString(", ")
}
b.WriteString("(CAST(? AS BIGINT), CAST(? AS TEXT), CAST(? AS TEXT))")
args = append(args, f.id, f.match, f.find)
}
q := "UPDATE " + t.table + " SET " + t.matchKey + " = v.column2, search_key = v.column3 FROM (VALUES " + b.String() + ") AS v WHERE " + t.table + ".id = v.column1"
if _, err := tx.ExecContext(ctx, q, args...); err != nil {
return fmt.Errorf("update %s keys: %w", t.table, err)
}
return nil
}
// keyFix is the new keys of one row.
type keyFix struct {
id int64
match, find string
}
// ---- helpers (run inside a transaction) ----
// findOrCreateArtist returns the artist a file belongs to, creating it if needed.

View file

@ -224,3 +224,38 @@ func TestRefreshKeepsAccentedAndPlainArtistsApart(t *testing.T) {
}
})
}
// The rows to fix go in batches: a library with thousands of stale rows ends up with every key
// right, the rows that were already right keep their value, and a second run changes nothing.
func TestRefreshMatchKeysFixesEveryRowAcrossBatches(t *testing.T) {
forEachBackend(t, func(t *testing.T, db *bun.DB, driver string) {
ctx := context.Background()
const n = 1250 // more than two batches
var insert string
if driver == DriverPostgres {
insert = "INSERT INTO artists (name, name_key, search_key) SELECT 'Sigur Rós ' || g, CASE WHEN g % 5 = 0 THEN 'sigur rós ' || g ELSE 'stale' END, CASE WHEN g % 5 = 0 THEN 'sigur ros ' || g ELSE '' END FROM generate_series(1, ?) g"
} else {
insert = "WITH RECURSIVE c(g) AS (SELECT 1 UNION ALL SELECT g+1 FROM c WHERE g < ?) INSERT INTO artists (name, name_key, search_key) SELECT 'Sigur Rós ' || g, CASE WHEN g % 5 = 0 THEN 'sigur rós ' || g ELSE 'stale' END, CASE WHEN g % 5 = 0 THEN 'sigur ros ' || g ELSE '' END FROM c"
}
if _, err := db.ExecContext(ctx, insert, n); err != nil {
t.Fatal(err)
}
repo := NewLibraryRepo(db)
for i := 0; i < 2; i++ {
if err := repo.RefreshMatchKeys(ctx); err != nil {
t.Fatal(err)
}
}
var wrong int
if err := db.QueryRow("SELECT count(*) FROM artists WHERE name_key <> lower(name) OR search_key <> replace(lower(name), 'ó', 'o')").Scan(&wrong); err != nil {
t.Fatal(err)
}
if wrong != 0 {
t.Errorf("%d artists have a wrong key", wrong)
}
var total int
if err := db.QueryRow("SELECT count(*) FROM artists").Scan(&total); err != nil || total != n {
t.Errorf("artists = %d err = %v, want %d", total, err, n)
}
})
}