- fix: on SQLite a transaction that read and then wrote failed at once with "database is locked" when another writer committed in between, because the busy timeout does not apply there. Transactions now start with BEGIN IMMEDIATE. Before this change a scan running while someone opened similar artists made files fail, and parallel requests stored nothing - fix: on PostgreSQL two requests naming the same new artist could fail on the unique MusicBrainz id or leave two rows with one name. Creating artists, in the discovery store and in the scanner, now takes turns through an advisory lock and looks again once it has the turn - fix: music arriving for an artist that a provider only named used to keep the provider's spelling and its unchecked MusicBrainz id, which later lookups would then send instead of the name. Such artists are now marked (artists.discovered); the first scan that finds them renames them to the spelling of the file tags, drops the id and clears the mark - fix: a panic in the shared provider call is turned into a failure; the singleflight group would have re-raised it where nothing can recover - fix: a client that leaves while Last.fm is being asked no longer gets a 500 and an error in the log - fix: no page can ask for more than is stored (limit tops out at 50), and the docs no longer say the best score is always 1 - test: concurrent saves plus a scan on both databases, checked by removing each fix; adoption keeps neither the name nor the id; the panic; the client that leaves
148 lines
4.2 KiB
Go
148 lines
4.2 KiB
Go
// Package storage opens the database (SQLite or PostgreSQL) and applies schema
|
|
// migrations. Repositories live here too.
|
|
//
|
|
// This is the only place that should import the ORM (bun) and the database
|
|
// drivers. Services depend on small interfaces, never on this package directly.
|
|
package storage
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"fmt"
|
|
"io/fs"
|
|
"strings"
|
|
"time"
|
|
|
|
_ "github.com/jackc/pgx/v5/stdlib" // PostgreSQL driver ("pgx")
|
|
"github.com/uptrace/bun"
|
|
"github.com/uptrace/bun/dialect/pgdialect"
|
|
"github.com/uptrace/bun/dialect/sqlitedialect"
|
|
_ "modernc.org/sqlite" // pure-Go SQLite driver ("sqlite")
|
|
)
|
|
|
|
// Supported database drivers.
|
|
const (
|
|
DriverSQLite = "sqlite"
|
|
DriverPostgres = "postgres"
|
|
)
|
|
|
|
// pathEscaper protects characters that are special in a file: URI.
|
|
var pathEscaper = strings.NewReplacer("%", "%25", "?", "%3F", "#", "%23")
|
|
|
|
// Open connects to the database. For "sqlite" target is a file path (created if
|
|
// needed); for "postgres" it is a connection string (DSN or URL).
|
|
func Open(driver, target string) (*bun.DB, error) {
|
|
switch driver {
|
|
case DriverSQLite:
|
|
return openSQLite(target)
|
|
case DriverPostgres:
|
|
return openPostgres(target)
|
|
default:
|
|
return nil, fmt.Errorf("unknown database driver %q", driver)
|
|
}
|
|
}
|
|
|
|
// openSQLite enables WAL, foreign keys and a busy timeout on every connection.
|
|
// Transactions start with BEGIN IMMEDIATE: one that reads and then writes would
|
|
// otherwise fail at once with "database is locked" when another writer commits in
|
|
// between, and the busy timeout does not apply to that case.
|
|
func openSQLite(path string) (*bun.DB, error) {
|
|
dsn := "file:" + pathEscaper.Replace(path) +
|
|
"?_pragma=foreign_keys(1)&_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)&_txlock=immediate"
|
|
|
|
sqldb, err := sql.Open("sqlite", dsn)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("open sqlite: %w", err)
|
|
}
|
|
if err := sqldb.Ping(); err != nil {
|
|
_ = sqldb.Close()
|
|
return nil, fmt.Errorf("ping sqlite: %w", err)
|
|
}
|
|
return bun.NewDB(sqldb, sqlitedialect.New()), nil
|
|
}
|
|
|
|
func openPostgres(dsn string) (*bun.DB, error) {
|
|
sqldb, err := sql.Open("pgx", dsn)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("open postgres: %w", err)
|
|
}
|
|
if err := sqldb.Ping(); err != nil {
|
|
_ = sqldb.Close()
|
|
return nil, fmt.Errorf("ping postgres: %w", err)
|
|
}
|
|
return bun.NewDB(sqldb, pgdialect.New()), nil
|
|
}
|
|
|
|
// Migrate applies, in lexical order, every *.sql file in fsys that has not
|
|
// been applied yet. Each file runs in its own transaction.
|
|
func Migrate(ctx context.Context, db *bun.DB, fsys fs.FS) error {
|
|
_, err := db.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS schema_migrations (
|
|
version TEXT PRIMARY KEY,
|
|
applied_at BIGINT NOT NULL
|
|
)`)
|
|
if err != nil {
|
|
return fmt.Errorf("create schema_migrations: %w", err)
|
|
}
|
|
|
|
applied, err := appliedVersions(ctx, db)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
entries, err := fs.ReadDir(fsys, ".")
|
|
if err != nil {
|
|
return fmt.Errorf("read migrations: %w", err)
|
|
}
|
|
for _, e := range entries {
|
|
name := e.Name()
|
|
if e.IsDir() || !strings.HasSuffix(name, ".sql") || applied[name] {
|
|
continue
|
|
}
|
|
if err := applyOne(ctx, db, fsys, name); err != nil {
|
|
return fmt.Errorf("migration %s: %w", name, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func appliedVersions(ctx context.Context, db *bun.DB) (map[string]bool, error) {
|
|
rows, err := db.QueryContext(ctx, "SELECT version FROM schema_migrations")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("list applied migrations: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
|
|
applied := map[string]bool{}
|
|
for rows.Next() {
|
|
var v string
|
|
if err := rows.Scan(&v); err != nil {
|
|
return nil, err
|
|
}
|
|
applied[v] = true
|
|
}
|
|
return applied, rows.Err()
|
|
}
|
|
|
|
func applyOne(ctx context.Context, db *bun.DB, fsys fs.FS, name string) error {
|
|
body, err := fs.ReadFile(fsys, name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
tx, err := db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer func() { _ = tx.Rollback() }() // no-op after a successful Commit
|
|
|
|
// The file body goes straight to the driver: it may contain several
|
|
// statements and must not go through placeholder formatting.
|
|
if _, err := tx.Tx.ExecContext(ctx, string(body)); err != nil {
|
|
return err
|
|
}
|
|
_, err = tx.ExecContext(ctx,
|
|
"INSERT INTO schema_migrations (version, applied_at) VALUES (?, ?)", name, time.Now().Unix())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return tx.Commit()
|
|
}
|