Merge pull request 'perf(storage): fix stale search keys in batches on PostgreSQL' (#63) from perf/refresh-keys-batches into main
This commit is contained in:
commit
850ae11f3d
2 changed files with 79 additions and 7 deletions
|
|
@ -9,6 +9,7 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/uptrace/bun"
|
||||
"github.com/uptrace/bun/dialect"
|
||||
|
||||
"codeberg.org/squidd/sonda/internal/library"
|
||||
"codeberg.org/squidd/sonda/internal/streaming"
|
||||
|
|
@ -382,11 +383,7 @@ func refreshKeys(ctx context.Context, tx bun.Tx, tables []keyedTable) error {
|
|||
if err != nil {
|
||||
return fmt.Errorf("list %s: %w", t.table, err)
|
||||
}
|
||||
type fix struct {
|
||||
id int64
|
||||
match, find string
|
||||
}
|
||||
var fixes []fix
|
||||
var fixes []keyFix
|
||||
for rows.Next() {
|
||||
var (
|
||||
id int64
|
||||
|
|
@ -398,7 +395,7 @@ func refreshKeys(ctx context.Context, tx bun.Tx, tables []keyedTable) error {
|
|||
}
|
||||
m, s := matchKey(text), searchKey(text)
|
||||
if m != match || s != current {
|
||||
fixes = append(fixes, fix{id: id, match: m, find: s})
|
||||
fixes = append(fixes, keyFix{id: id, match: m, find: s})
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
|
|
@ -406,15 +403,55 @@ func refreshKeys(ctx context.Context, tx bun.Tx, tables []keyedTable) error {
|
|||
return err
|
||||
}
|
||||
_ = rows.Close()
|
||||
for _, f := range fixes {
|
||||
if _, err := tx.ExecContext(ctx, "UPDATE "+t.table+" SET "+t.matchKey+" = ?, search_key = ? WHERE id = ?", f.match, f.find, f.id); err != nil {
|
||||
return fmt.Errorf("update %s key: %w", t.table, err)
|
||||
for start := 0; start < len(fixes); start += keyBatch {
|
||||
batch := fixes[start:min(start+keyBatch, len(fixes))]
|
||||
if err := updateKeys(ctx, tx, t, batch); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// keyBatch is how many rows one UPDATE fixes: 500 rows are 1500 parameters, far below the limit
|
||||
// of both databases, and one round trip instead of 500 on PostgreSQL.
|
||||
const keyBatch = 500
|
||||
|
||||
// updateKeys fixes a batch of rows. On PostgreSQL it is one statement, and the rows come as a VALUES list, whose
|
||||
// columns are called column1, column2 and column3.
|
||||
func updateKeys(ctx context.Context, tx bun.Tx, t keyedTable, batch []keyFix) error {
|
||||
if tx.Dialect().Name() == dialect.SQLite {
|
||||
// The file is local: there is no round trip to save, and one UPDATE per row is faster
|
||||
// than the join with a VALUES list.
|
||||
for _, f := range batch {
|
||||
if _, err := tx.ExecContext(ctx, "UPDATE "+t.table+" SET "+t.matchKey+" = ?, search_key = ? WHERE id = ?", f.match, f.find, f.id); err != nil {
|
||||
return fmt.Errorf("update %s key: %w", t.table, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
var b strings.Builder
|
||||
args := make([]any, 0, len(batch)*3)
|
||||
for i, f := range batch {
|
||||
if i > 0 {
|
||||
b.WriteString(", ")
|
||||
}
|
||||
b.WriteString("(CAST(? AS BIGINT), CAST(? AS TEXT), CAST(? AS TEXT))")
|
||||
args = append(args, f.id, f.match, f.find)
|
||||
}
|
||||
q := "UPDATE " + t.table + " SET " + t.matchKey + " = v.column2, search_key = v.column3 FROM (VALUES " + b.String() + ") AS v WHERE " + t.table + ".id = v.column1"
|
||||
if _, err := tx.ExecContext(ctx, q, args...); err != nil {
|
||||
return fmt.Errorf("update %s keys: %w", t.table, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// keyFix is the new keys of one row.
|
||||
type keyFix struct {
|
||||
id int64
|
||||
match, find string
|
||||
}
|
||||
|
||||
// ---- helpers (run inside a transaction) ----
|
||||
|
||||
// findOrCreateArtist returns the artist a file belongs to, creating it if needed.
|
||||
|
|
|
|||
|
|
@ -224,3 +224,38 @@ func TestRefreshKeepsAccentedAndPlainArtistsApart(t *testing.T) {
|
|||
}
|
||||
})
|
||||
}
|
||||
|
||||
// The rows to fix go in batches: a library with thousands of stale rows ends up with every key
|
||||
// right, the rows that were already right keep their value, and a second run changes nothing.
|
||||
func TestRefreshMatchKeysFixesEveryRowAcrossBatches(t *testing.T) {
|
||||
forEachBackend(t, func(t *testing.T, db *bun.DB, driver string) {
|
||||
ctx := context.Background()
|
||||
const n = 1250 // more than two batches
|
||||
var insert string
|
||||
if driver == DriverPostgres {
|
||||
insert = "INSERT INTO artists (name, name_key, search_key) SELECT 'Sigur Rós ' || g, CASE WHEN g % 5 = 0 THEN 'sigur rós ' || g ELSE 'stale' END, CASE WHEN g % 5 = 0 THEN 'sigur ros ' || g ELSE '' END FROM generate_series(1, ?) g"
|
||||
} else {
|
||||
insert = "WITH RECURSIVE c(g) AS (SELECT 1 UNION ALL SELECT g+1 FROM c WHERE g < ?) INSERT INTO artists (name, name_key, search_key) SELECT 'Sigur Rós ' || g, CASE WHEN g % 5 = 0 THEN 'sigur rós ' || g ELSE 'stale' END, CASE WHEN g % 5 = 0 THEN 'sigur ros ' || g ELSE '' END FROM c"
|
||||
}
|
||||
if _, err := db.ExecContext(ctx, insert, n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repo := NewLibraryRepo(db)
|
||||
for i := 0; i < 2; i++ {
|
||||
if err := repo.RefreshMatchKeys(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var wrong int
|
||||
if err := db.QueryRow("SELECT count(*) FROM artists WHERE name_key <> lower(name) OR search_key <> replace(lower(name), 'ó', 'o')").Scan(&wrong); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if wrong != 0 {
|
||||
t.Errorf("%d artists have a wrong key", wrong)
|
||||
}
|
||||
var total int
|
||||
if err := db.QueryRow("SELECT count(*) FROM artists").Scan(&total); err != nil || total != n {
|
||||
t.Errorf("artists = %d err = %v, want %d", total, err, n)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue