perf(storage): fix stale match and search keys in batches on PostgreSQL
The refresh at start fixed each stale row with its own UPDATE. On PostgreSQL that is one round trip per row: 100,000 stale rows took 5.3 s on localhost, and more over a network. The rows now go 500 at a time in one UPDATE ... FROM (VALUES ...): 1.9 s for the same rows. SQLite keeps one UPDATE per row, which is faster there (no round trip to save). A run with nothing to fix is unchanged (about 0.1 s per 100,000 rows). A test with 1,250 stale rows covers more than two batches on both databases.
This commit is contained in:
parent
64c14a51c5
commit
77be31ea4a
2 changed files with 79 additions and 7 deletions
|
|
@ -9,6 +9,7 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/uptrace/bun"
|
||||
"github.com/uptrace/bun/dialect"
|
||||
|
||||
"codeberg.org/squidd/sonda/internal/library"
|
||||
"codeberg.org/squidd/sonda/internal/streaming"
|
||||
|
|
@ -382,11 +383,7 @@ func refreshKeys(ctx context.Context, tx bun.Tx, tables []keyedTable) error {
|
|||
if err != nil {
|
||||
return fmt.Errorf("list %s: %w", t.table, err)
|
||||
}
|
||||
type fix struct {
|
||||
id int64
|
||||
match, find string
|
||||
}
|
||||
var fixes []fix
|
||||
var fixes []keyFix
|
||||
for rows.Next() {
|
||||
var (
|
||||
id int64
|
||||
|
|
@ -398,7 +395,7 @@ func refreshKeys(ctx context.Context, tx bun.Tx, tables []keyedTable) error {
|
|||
}
|
||||
m, s := matchKey(text), searchKey(text)
|
||||
if m != match || s != current {
|
||||
fixes = append(fixes, fix{id: id, match: m, find: s})
|
||||
fixes = append(fixes, keyFix{id: id, match: m, find: s})
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
|
|
@ -406,15 +403,55 @@ func refreshKeys(ctx context.Context, tx bun.Tx, tables []keyedTable) error {
|
|||
return err
|
||||
}
|
||||
_ = rows.Close()
|
||||
for _, f := range fixes {
|
||||
if _, err := tx.ExecContext(ctx, "UPDATE "+t.table+" SET "+t.matchKey+" = ?, search_key = ? WHERE id = ?", f.match, f.find, f.id); err != nil {
|
||||
return fmt.Errorf("update %s key: %w", t.table, err)
|
||||
for start := 0; start < len(fixes); start += keyBatch {
|
||||
batch := fixes[start:min(start+keyBatch, len(fixes))]
|
||||
if err := updateKeys(ctx, tx, t, batch); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// keyBatch is how many rows one UPDATE fixes: 500 rows are 1500 parameters, far below the limit
|
||||
// of both databases, and one round trip instead of 500 on PostgreSQL.
|
||||
const keyBatch = 500
|
||||
|
||||
// updateKeys fixes a batch of rows. On PostgreSQL it is one statement, and the rows come as a VALUES list, whose
|
||||
// columns are called column1, column2 and column3.
|
||||
func updateKeys(ctx context.Context, tx bun.Tx, t keyedTable, batch []keyFix) error {
|
||||
if tx.Dialect().Name() == dialect.SQLite {
|
||||
// The file is local: there is no round trip to save, and one UPDATE per row is faster
|
||||
// than the join with a VALUES list.
|
||||
for _, f := range batch {
|
||||
if _, err := tx.ExecContext(ctx, "UPDATE "+t.table+" SET "+t.matchKey+" = ?, search_key = ? WHERE id = ?", f.match, f.find, f.id); err != nil {
|
||||
return fmt.Errorf("update %s key: %w", t.table, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
var b strings.Builder
|
||||
args := make([]any, 0, len(batch)*3)
|
||||
for i, f := range batch {
|
||||
if i > 0 {
|
||||
b.WriteString(", ")
|
||||
}
|
||||
b.WriteString("(CAST(? AS BIGINT), CAST(? AS TEXT), CAST(? AS TEXT))")
|
||||
args = append(args, f.id, f.match, f.find)
|
||||
}
|
||||
q := "UPDATE " + t.table + " SET " + t.matchKey + " = v.column2, search_key = v.column3 FROM (VALUES " + b.String() + ") AS v WHERE " + t.table + ".id = v.column1"
|
||||
if _, err := tx.ExecContext(ctx, q, args...); err != nil {
|
||||
return fmt.Errorf("update %s keys: %w", t.table, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// keyFix is the new keys of one row.
|
||||
type keyFix struct {
|
||||
id int64
|
||||
match, find string
|
||||
}
|
||||
|
||||
// ---- helpers (run inside a transaction) ----
|
||||
|
||||
// findOrCreateArtist returns the artist a file belongs to, creating it if needed.
|
||||
|
|
|
|||
|
|
@ -224,3 +224,38 @@ func TestRefreshKeepsAccentedAndPlainArtistsApart(t *testing.T) {
|
|||
}
|
||||
})
|
||||
}
|
||||
|
||||
// The rows to fix go in batches: a library with thousands of stale rows ends up with every key
|
||||
// right, the rows that were already right keep their value, and a second run changes nothing.
|
||||
func TestRefreshMatchKeysFixesEveryRowAcrossBatches(t *testing.T) {
|
||||
forEachBackend(t, func(t *testing.T, db *bun.DB, driver string) {
|
||||
ctx := context.Background()
|
||||
const n = 1250 // more than two batches
|
||||
var insert string
|
||||
if driver == DriverPostgres {
|
||||
insert = "INSERT INTO artists (name, name_key, search_key) SELECT 'Sigur Rós ' || g, CASE WHEN g % 5 = 0 THEN 'sigur rós ' || g ELSE 'stale' END, CASE WHEN g % 5 = 0 THEN 'sigur ros ' || g ELSE '' END FROM generate_series(1, ?) g"
|
||||
} else {
|
||||
insert = "WITH RECURSIVE c(g) AS (SELECT 1 UNION ALL SELECT g+1 FROM c WHERE g < ?) INSERT INTO artists (name, name_key, search_key) SELECT 'Sigur Rós ' || g, CASE WHEN g % 5 = 0 THEN 'sigur rós ' || g ELSE 'stale' END, CASE WHEN g % 5 = 0 THEN 'sigur ros ' || g ELSE '' END FROM c"
|
||||
}
|
||||
if _, err := db.ExecContext(ctx, insert, n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repo := NewLibraryRepo(db)
|
||||
for i := 0; i < 2; i++ {
|
||||
if err := repo.RefreshMatchKeys(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var wrong int
|
||||
if err := db.QueryRow("SELECT count(*) FROM artists WHERE name_key <> lower(name) OR search_key <> replace(lower(name), 'ó', 'o')").Scan(&wrong); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if wrong != 0 {
|
||||
t.Errorf("%d artists have a wrong key", wrong)
|
||||
}
|
||||
var total int
|
||||
if err := db.QueryRow("SELECT count(*) FROM artists").Scan(&total); err != nil || total != n {
|
||||
t.Errorf("artists = %d err = %v, want %d", total, err, n)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue