sqldb/v2: Use `MigrationStream` for migrations
What changed, and why it matters
This commit refactors how LND's new sqldb/v2 package runs database migrations. It replaces a hand-rolled migration loop with a new MigrationStream abstraction, adds automatic SQLite backups before migrations, prevents accidental database downgrades, and rejects running migrations when the database is in a 'dirty' (partially failed) state. The changes are mostly defensive hardening and code cleanup rather than a fix for a known active vulnerability.
Treat this as a routine refactor with defensive hardening. Reviewers should verify that the new ProgrammaticMigrEntry callbacks are registered at the correct versions, that downgrade/dirty-state errors are handled gracefully by callers, and that the SQLite backup path does not leak sensitive database files to unexpected locations. No urgent security patch is indicated.
Security signals we found
New dirty-state guard aborts migrations if a previous migration failed
New downgrade guard prevents running older code against a newer schema
SQLite migrations now create a VACUUM INTO backup before upgrading
Migration tracking table name is now configurable per MigrationSet
Custom migration functions moved from MigrationConfig to golang-migrate programmatic migration entries
Removed direct context.Context and sqlc.Queries migration orchestration in favor of MigrationExecutor interface
Evidence from the diff
The patch rewrites the sqldb/v2 migration path. Key changes: (1) MigrationConfig loses its inline MigrationFn; custom migrations are now supplied as golang-migrate ProgrammaticMigrEntry callbacks via MigrationSet. (2) ApplyMigrations is replaced by ApplyAllMigrations over []MigrationSet. (3) applyMigrations now checks the dirty flag and returns an error if set, and refuses to run if the DB version is newer than the latest known migration (downgrade protection). (4) SQLite gains backupSqliteDatabase using VACUUM INTO, controlled by a new SkipMigrationDbBackup config flag, and only creates the backup when an upgrade is actually pending. (5) Postgres and SQLite ExecuteMigrations now use per-set tracking tables and latest-version metadata. (6) The global sqlSchemas embed.FS is removed; each MigrationSet carries its own SQLFiles and directory path. No CVE, advisory, or researcher attribution is present in the commit or supplied references.
Changed components
sqldb/v2sqldb/v2/sqlite.gosqldb/v2/postgres.gosqldb/v2/migrations.gosqldb/v2/config.gosqldb/v2/interfaces.gosqldb/v2/no_sqlite.gosqldb/v2/postgres_fixture.gosqldb/v2/postgres_test.gosqldb/v2/sqlite_test.gosqldb/v2/schemas.go (deleted)sqldb/v2/go.modsqldb/v2/go.sumInspect captured patch +287 / −240
diff --git a/sqldb/v2/config.go b/sqldb/v2/config.go
index 86ae791..61abbb6 100644
--- a/sqldb/v2/config.go
+++ b/sqldb/v2/config.go
@@ -28,7 +28,12 @@ type SqliteConfig struct {
MaxConnections int `long:"maxconnections" description:"The maximum number of open connections to the database. Set to zero for unlimited."`
PragmaOptions []string `long:"pragmaoptions" description:"A list of pragma options to set on a database connection. For example, 'auto_vacuum=incremental'. Note that the flag must be specified multiple times if multiple options are to be set."`
SkipMigrations bool `long:"skipmigrations" description:"Skip applying migrations on startup."`
- QueryConfig `group:"query" namespace:"query"`
+
+ // SkipMigrationDbBackup if true, then a backup of the database will not
+ // be created before applying migrations.
+ SkipMigrationDbBackup bool `long:"skipmigrationdbbackup" description:"Skip creating a backup of the database before applying migrations."`
+
+ QueryConfig `group:"query" namespace:"query"`
}
const (
diff --git a/sqldb/v2/go.mod b/sqldb/v2/go.mod
index 7b75907..5e62626 100644
--- a/sqldb/v2/go.mod
+++ b/sqldb/v2/go.mod
@@ -7,6 +7,7 @@ require (
github.com/jackc/pgconn v1.14.3
github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438
github.com/jackc/pgx/v5 v5.5.4
+ github.com/lightningnetwork/lnd/fn/v2 v2.0.8
github.com/lightningnetwork/lnd/sqldb v1.0.10
github.com/ory/dockertest/v3 v3.10.0
github.com/pmezard/go-difflib v1.0.0
diff --git a/sqldb/v2/go.sum b/sqldb/v2/go.sum
index 35f7518..69570b5 100644
--- a/sqldb/v2/go.sum
+++ b/sqldb/v2/go.sum
@@ -91,6 +91,8 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lightninglabs/migrate/v4 v4.18.2-9023d66a-fork-pr-2.0.20251211093704-71c1eef09789 h1:7kX7vUgHUazAHcCJ6uzBDa4/2MEGEbMEfa01GtfqmTQ=
github.com/lightninglabs/migrate/v4 v4.18.2-9023d66a-fork-pr-2.0.20251211093704-71c1eef09789/go.mod h1:99BKpIi6ruaaXRM1A77eqZ+FWPQ3cfRa+ZVy5bmWMaY=
+github.com/lightningnetwork/lnd/fn/v2 v2.0.8 h1:r2SLz7gZYQPVc3IZhU82M66guz3Zk2oY+Rlj9QN5S3g=
+github.com/lightningnetwork/lnd/fn/v2 v2.0.8/go.mod h1:TOzwrhjB/Azw1V7aa8t21ufcQmdsQOQMDtxVOQWNl8s=
github.com/lightningnetwork/lnd/sqldb v1.0.10 h1:ZLV7TGwjnKupVfCd+DJ43MAc9BKVSFCnvhpSPGKdN3M=
github.com/lightningnetwork/lnd/sqldb v1.0.10/go.mod h1:c/vWoQfcxu6FAfHzGajkIQi7CEIeIZFhhH4DYh1BJpc=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
diff --git a/sqldb/v2/interfaces.go b/sqldb/v2/interfaces.go
index e801123..b9cf552 100644
--- a/sqldb/v2/interfaces.go
+++ b/sqldb/v2/interfaces.go
@@ -388,13 +388,10 @@ func (t *TransactionExecutor[Q]) ExecTx(ctx context.Context,
// DB is an interface that represents a generic SQL database. It provides
// methods to apply migrations and access the underlying database connection.
type DB interface {
+ MigrationExecutor
+
// GetBaseDB returns the underlying BaseDB instance.
GetBaseDB() *BaseDB
-
- // ApplyAllMigrations applies all migrations to the database including
- // both sqlc and custom in-code migrations.
- ApplyAllMigrations(ctx context.Context,
- customMigrations []MigrationConfig) error
}
// BaseDB is the base database struct that each implementation can embed to
diff --git a/sqldb/v2/migrations.go b/sqldb/v2/migrations.go
index 78fa5c0..1406e29 100644
--- a/sqldb/v2/migrations.go
+++ b/sqldb/v2/migrations.go
@@ -2,8 +2,6 @@ package sqldb
import (
"bytes"
- "context"
- "database/sql"
"embed"
"errors"
"fmt"
@@ -12,14 +10,13 @@ import (
"net/http"
"reflect"
"strings"
- "time"
"github.com/btcsuite/btclog/v2"
"github.com/davecgh/go-spew/spew"
"github.com/golang-migrate/migrate/v4"
"github.com/golang-migrate/migrate/v4/database"
"github.com/golang-migrate/migrate/v4/source/httpfs"
- "github.com/lightningnetwork/lnd/sqldb/sqlc"
+ "github.com/lightningnetwork/lnd/fn/v2"
"github.com/pmezard/go-difflib/difflib"
)
@@ -48,12 +45,6 @@ type MigrationConfig struct {
// SchemaVersion represents the schema version tracked by golang-migrate
// at which the migration is applied.
SchemaVersion int
-
- // MigrationFn is the function executed for custom migrations at the
- // specified version. It is used to handle migrations that cannot be
- // performed through SQL alone. If set to nil, no custom migration is
- // applied.
- MigrationFn func(tx *sqlc.Queries) error
}
// MigrationSet encapsulates all necessary information to manage and apply
@@ -95,8 +86,12 @@ type MigrationSet struct {
}
// MigrationTarget is a functional option that can be passed to applyMigrations
-// to specify a target version to migrate to.
-type MigrationTarget func(mig *migrate.Migrate) error
+// to specify a target version to migrate to. `currentDbVersion` is the current
+// (migration) version of the database, or None if unknown.
+// `maxMigrationVersion` is the maximum migration version known to the driver,
+// or None if unknown.
+type MigrationTarget func(mig *migrate.Migrate,
+ currentDbVersion int, maxMigrationVersion uint) error
// MigrationExecutor is an interface that abstracts the migration functionality.
type MigrationExecutor interface {
@@ -106,7 +101,7 @@ type MigrationExecutor interface {
// Developers must ensure that migrations are defined in the correct
// order. Migration details are stored in the global variable
// migrationConfig.
- ExecuteMigrations(target MigrationTarget) error
+ ExecuteMigrations(target MigrationTarget, set MigrationSet) error
// GetSchemaVersion returns the current schema version of the database.
GetSchemaVersion() (int, bool, error)
@@ -116,24 +111,74 @@ type MigrationExecutor interface {
// NOTE: This alters the internal database schema tracker. USE WITH
// CAUTION!!!
SetSchemaVersion(version int, dirty bool) error
+
+ // DefaultTarget returns the default migration target.
+ DefaultTarget() MigrationTarget
+
+ // SkipMigrations indicates if the SQL and corresponding code migrations
+ // will be skipped.
+ SkipMigrations() bool
}
var (
// TargetLatest is a MigrationTarget that migrates to the latest
// version available.
- TargetLatest = func(mig *migrate.Migrate) error {
+ TargetLatest = func(mig *migrate.Migrate, _ int, _ uint) error {
return mig.Up()
}
// TargetVersion is a MigrationTarget that migrates to the given
// version.
TargetVersion = func(version uint) MigrationTarget {
- return func(mig *migrate.Migrate) error {
+ return func(mig *migrate.Migrate, _ int, _ uint) error {
return mig.Migrate(version)
}
}
+
+ // ErrMigrationDowngrade is returned when a database downgrade is
+ // detected.
+ ErrMigrationDowngrade = errors.New("database downgrade detected")
)
+// migrationOption is a functional option that can be passed to migrate related
+// methods to modify their behavior.
+type migrateOptions struct {
+ latestVersion fn.Option[uint]
+ programmaticMigrs map[uint]migrate.ProgrammaticMigrEntry
+}
+
+// defaultMigrateOptions returns a new migrateOptions instance with default
+// settings.
+func defaultMigrateOptions() *migrateOptions {
+ return &migrateOptions{
+ programmaticMigrs: make(map[uint]migrate.ProgrammaticMigrEntry),
+ }
+}
+
+// MigrateOpt is a functional option that can be passed to migrate related
+// methods to modify behavior.
+type MigrateOpt func(*migrateOptions)
+
+// WithLatestVersion allows callers to override the default latest version
+// setting.
+func WithLatestVersion(version uint) MigrateOpt {
+ return func(o *migrateOptions) {
+ o.latestVersion = fn.Some(version)
+ }
+}
+
+// WithProgrammaticMigrations is an option that can be used to set a map of
+// ProgrammaticMigrEntry functions that can be used to execute a Golang based
+// migration step. The key is the migration version and the value is the
+// Golang migration function entry that should be run for the migration version.
+func WithProgrammaticMigrations(
+ programmaticMigrs map[uint]migrate.ProgrammaticMigrEntry) MigrateOpt {
+
+ return func(o *migrateOptions) {
+ o.programmaticMigrs = programmaticMigrs
+ }
+}
+
// migrationLogger is a logger that wraps the passed btclog.Logger so it can be
// used to log migrations.
type migrationLogger struct {
@@ -172,7 +217,8 @@ func (m *migrationLogger) Verbose() bool {
// system under the given path, using the passed database driver and database
// name.
func applyMigrations(fs fs.FS, driver database.Driver, path,
- dbName string, targetVersion MigrationTarget) error {
+ dbName string, targetVersion MigrationTarget,
+ opts *migrateOptions) error {
// With the migrate instance open, we'll create a new migration source
// using the embedded file system stored in sqlSchemas. The library
@@ -188,30 +234,66 @@ func applyMigrations(fs fs.FS, driver database.Driver, path,
// above.
sqlMigrate, err := migrate.NewWithInstance(
"migrations", migrateFileServer, dbName, driver,
+ migrate.WithProgrammaticMigrations(opts.programmaticMigrs),
)
if err != nil {
return err
}
- migrationVersion, _, err := sqlMigrate.Version()
+ migrationVersion, dirty, err := sqlMigrate.Version()
if err != nil && !errors.Is(err, migrate.ErrNilVersion) {
- log.Errorf("Unable to determine current migration version: %v",
- err)
+ return fmt.Errorf("unable to determine current migration "+
+ "version: %w", err)
+ }
- return err
+ // If the migration version is dirty, we should not proceed with further
+ // migrations, as this indicates that a previous migration did not
+ // complete successfully and requires manual intervention.
+ if dirty {
+ return fmt.Errorf("database is in a dirty state at version "+
+ "%v, manual intervention required", migrationVersion)
}
- log.Infof("Applying migrations from version=%v", migrationVersion)
+ // As the down migrations may end up *dropping* data, we want to
+ // prevent that without explicit accounting.
+ latestVersion, err := opts.latestVersion.UnwrapOrErr(
+ fmt.Errorf("latest version not set"),
+ )
+ if err != nil {
+ return fmt.Errorf("unable to get latest version: %w", err)
+ }
+ if migrationVersion > latestVersion {
+ return fmt.Errorf("%w: database version is newer than the "+
+ "latest migration version, preventing downgrade: "+
+ "db_version=%v, latest_migration_version=%v",
+ ErrMigrationDowngrade, migrationVersion, latestVersion)
+ }
+
+ // Report the current version of the database before the migration.
+ currentDbVersion, _, err := driver.Version()
+ if err != nil {
+ return fmt.Errorf("unable to get current db version: %w", err)
+ }
+ log.Infof("Attempting to apply migration(s) "+
+ "(current_db_version=%v, latest_migration_version=%v)",
+ currentDbVersion, latestVersion)
// Apply our local logger to the migration instance.
sqlMigrate.Log = &migrationLogger{log}
// Execute the migration based on the target given.
- err = targetVersion(sqlMigrate)
+ err = targetVersion(sqlMigrate, currentDbVersion, latestVersion)
if err != nil && !errors.Is(err, migrate.ErrNoChange) {
return err
}
+ // Report the current version of the database after the migration.
+ currentDbVersion, _, err = driver.Version()
+ if err != nil {
+ return fmt.Errorf("unable to get current db version: %w", err)
+ }
+ log.Infof("Database version after migration: %v", currentDbVersion)
+
return nil
}
@@ -317,143 +399,18 @@ func (t *replacerFile) Close() error {
return nil
}
-// ApplyMigrations applies the provided migrations to the database in sequence.
-// It ensures migrations are executed in the correct order, applying both custom
-// migration functions and SQL migrations as needed.
-func ApplyMigrations(ctx context.Context, db *BaseDB,
- migrator MigrationExecutor, migrations []MigrationConfig) error {
-
- // Ensure that the migrations are sorted by version.
- for i := 0; i < len(migrations); i++ {
- if migrations[i].Version != i+1 {
- return fmt.Errorf("migration version %d is out of "+
- "order. Expected %d", migrations[i].Version,
- i+1)
- }
- }
- // Construct a transaction executor to apply custom migrations.
- executor := NewTransactionExecutor(db, func(tx *sql.Tx) *sqlc.Queries {
- return db.WithTx(tx)
- })
-
- currentVersion := 0
- version, err := db.GetDatabaseVersion(ctx)
- if !errors.Is(err, sql.ErrNoRows) {
- if err != nil {
- return fmt.Errorf("error getting current database "+
- "version: %w", err)
- }
-
- currentVersion = int(version)
- } else {
- // Since we don't have a version tracked by our own table yet,
- // we'll use the schema version reported by sqlc to determine
- // the current version.
- //
- // NOTE: This is safe because the first in-code migration was
- // introduced in version 7. This is only possible if the user
- // has a schema version <= 4.
- var dirty bool
- currentVersion, dirty, err = migrator.GetSchemaVersion()
- if err != nil {
- return err
- }
-
- log.Infof("No database version found, using schema version %d "+
- "(dirty=%v) as base version", currentVersion, dirty)
- }
-
- // Due to an a migration issue in v0.19.0-rc1 we may be at version 2 and
- // have a dirty schema due to failing migration 3. If this is indeed the
- // case, we need to reset the dirty flag to be able to apply the fixed
- // migration.
- // NOTE: this could be removed as soon as we drop v0.19.0-beta.
- if version == 2 {
- schemaVersion, dirty, err := migrator.GetSchemaVersion()
- if err != nil {
- return err
- }
-
- if schemaVersion == 3 && dirty {
- log.Warnf("Schema version %d is dirty. This is "+
- "likely a consequence of a failed migration "+
- "in v0.19.0-rc1. Attempting to recover by "+
- "resetting the dirty flag", schemaVersion)
-
- err = migrator.SetSchemaVersion(4, false)
- if err != nil {
- return err
- }
- }
+// ApplyAllMigrations applies both the SQLC and custom in-code migrations to the
+// SQLite database.
+func ApplyAllMigrations(executor MigrationExecutor, sets []MigrationSet) error {
+ // Execute migrations unless configured to skip them.
+ if executor.SkipMigrations() {
+ return nil
}
- for _, migration := range migrations {
- if migration.Version <= currentVersion {
- log.Infof("Skipping migration '%s' (version %d) as it "+
- "has already been applied", migration.Name,
- migration.Version)
-
- continue
- }
-
- log.Infof("Migrating SQL schema to version %d",
- migration.SchemaVersion)
-
- // Execute SQL schema migrations up to the target version.
- err = migrator.ExecuteMigrations(
- TargetVersion(uint(migration.SchemaVersion)),
- )
- if err != nil {
- return fmt.Errorf("error executing schema migrations "+
- "to target version %d: %w",
- migration.SchemaVersion, err)
- }
-
- opts := WriteTxOpt()
-
- // Run the custom migration as a transaction to ensure
- // atomicity. If successful, mark the migration as complete in
- // the migration tracker table.
- err = executor.ExecTx(ctx, opts, func(tx *sqlc.Queries) error {
- // Apply the migration function if one is provided.
- if migration.MigrationFn != nil {
- log.Infof("Applying custom migration '%v' "+
- "(version %d) to schema version %d",
- migration.Name, migration.Version,
- migration.SchemaVersion)
-
- err = migration.MigrationFn(tx)
- if err != nil {
- return fmt.Errorf("error applying "+
- "migration '%v' (version %d) "+
- "to schema version %d: %w",
- migration.Name,
- migration.Version,
- migration.SchemaVersion, err)
- }
-
- log.Infof("Migration '%v' (version %d) "+
- "applied ", migration.Name,
- migration.Version)
- }
-
- // Mark the migration as complete by adding the version
- // to the migration tracker table along with the current
- // timestamp.
- err = tx.SetMigration(ctx, sqlc.SetMigrationParams{
- Version: int32(migration.Version),
- MigrationTime: time.Now(),
- })
- if err != nil {
- return fmt.Errorf("error setting migration "+
- "version %d: %w", migration.Version,
- err)
- }
-
- return nil
- }, func() {})
+ for _, set := range sets {
+ err := executor.ExecuteMigrations(executor.DefaultTarget(), set)
if err != nil {
- return err
+ return fmt.Errorf("error applying migrations: %w", err)
}
}
diff --git a/sqldb/v2/no_sqlite.go b/sqldb/v2/no_sqlite.go
index ad0cae6..ee5a272 100644
--- a/sqldb/v2/no_sqlite.go
+++ b/sqldb/v2/no_sqlite.go
@@ -34,7 +34,7 @@ func (s *SqliteStore) GetBaseDB() *BaseDB {
// ApplyAllMigrations applies both the SQLC and custom in-code migrations to
// the SQLite database.
func (s *SqliteStore) ApplyAllMigrations(context.Context,
- []MigrationConfig) error {
+ []MigrationSet) error {
return fmt.Errorf("SQLite backend not supported in WebAssembly")
}
diff --git a/sqldb/v2/postgres.go b/sqldb/v2/postgres.go
index 4f50761..6db0e40 100644
--- a/sqldb/v2/postgres.go
+++ b/sqldb/v2/postgres.go
@@ -1,7 +1,6 @@
package sqldb
import (
- "context"
"database/sql"
"fmt"
"net/url"
@@ -12,6 +11,7 @@ import (
pgx_migrate "github.com/golang-migrate/migrate/v4/database/pgx/v5"
_ "github.com/golang-migrate/migrate/v4/source/file" // Read migrations from files. // nolint:ll
_ "github.com/jackc/pgx/v5"
+ "github.com/lightningnetwork/lnd/fn/v2"
"github.com/lightningnetwork/lnd/sqldb/sqlc"
)
@@ -148,41 +148,44 @@ func (s *PostgresStore) GetBaseDB() *BaseDB {
return s.BaseDB
}
-// ApplyAllMigrations applies both the SQLC and custom in-code migrations to the
-// Postgres database.
-func (s *PostgresStore) ApplyAllMigrations(ctx context.Context,
- migrations []MigrationConfig) error {
-
- // Execute migrations unless configured to skip them.
- if s.cfg.SkipMigrations {
- return nil
- }
-
- return ApplyMigrations(ctx, s.BaseDB, s, migrations)
-}
-
func errPostgresMigration(err error) error {
return fmt.Errorf("error creating postgres migration: %w", err)
}
// ExecuteMigrations runs migrations for the Postgres database, depending on the
// target given, either all migrations or up to a given version.
-func (s *PostgresStore) ExecuteMigrations(target MigrationTarget) error {
+func (s *PostgresStore) ExecuteMigrations(target MigrationTarget,
+ set MigrationSet) error {
+
dbName, err := getDatabaseNameFromDSN(s.cfg.Dsn)
if err != nil {
return err
}
- driver, err := pgx_migrate.WithInstance(s.DB, &pgx_migrate.Config{})
+ driver, err := pgx_migrate.WithInstance(s.DB, &pgx_migrate.Config{
+ MigrationsTable: set.TrackingTableName,
+ })
if err != nil {
return errPostgresMigration(err)
}
+ opts := &migrateOptions{
+ latestVersion: fn.Some(set.LatestMigrationVersion),
+ }
+
+ if set.MakeProgrammaticMigrations != nil {
+ postMigSteps, err := set.MakeProgrammaticMigrations(s.BaseDB)
+ if err != nil {
+ return errPostgresMigration(err)
+ }
+ opts.programmaticMigrs = postMigSteps
+ }
+
// Populate the database with our set of schemas based on our embedded
// in-memory file system.
- postgresFS := newReplacerFS(sqlSchemas, postgresSchemaReplacements)
+ postgresFS := newReplacerFS(set.SQLFiles, postgresSchemaReplacements)
return applyMigrations(
- postgresFS, driver, "../sqlc/migrations", dbName, target,
+ postgresFS, driver, set.SQLFileDirectory, dbName, target, opts,
)
}
@@ -213,3 +216,11 @@ func (s *PostgresStore) SetSchemaVersion(version int, dirty bool) error {
return driver.SetVersion(version, dirty)
}
+
+func (s *PostgresStore) DefaultTarget() MigrationTarget {
+ return TargetLatest
+}
+
+func (s *PostgresStore) SkipMigrations() bool {
+ return s.cfg.SkipMigrations
+}
diff --git a/sqldb/v2/postgres_fixture.go b/sqldb/v2/postgres_fixture.go
index cb4e1bf..7a4a86f 100644
--- a/sqldb/v2/postgres_fixture.go
+++ b/sqldb/v2/postgres_fixture.go
@@ -124,8 +124,8 @@ func (f *TestPgFixture) TearDown(t testing.TB) {
require.NoError(t, err, "Could not purge resource")
}
-// randomDBName generates a random database name.
-func randomDBName(t testing.TB) string {
+// RandomDBName generates a random database name.
+func RandomDBName(t testing.TB) string {
randBytes := make([]byte, 8)
_, err := rand.Read(randBytes)
require.NoError(t, err)
@@ -135,16 +135,12 @@ func randomDBName(t testing.TB) string {
// NewTestPostgresDB is a helper function that creates a Postgres database for
// testing using the given fixture.
-//
-// NOTE: This function differs from the one in sqldb/postgres_fixture.go as that
-// function does not expect any migrations to be passed in, and instead always
-// applies the lnd specific migrations.
func NewTestPostgresDB(t testing.TB, fixture *TestPgFixture,
- migrations []MigrationConfig) *PostgresStore {
+ sets []MigrationSet) *PostgresStore {
t.Helper()
- dbName := randomDBName(t)
+ dbName := RandomDBName(t)
t.Logf("Creating new Postgres DB '%s' for testing", dbName)
@@ -157,9 +153,7 @@ func NewTestPostgresDB(t testing.TB, fixture *TestPgFixture,
store, err := NewPostgresStore(cfg)
require.NoError(t, err)
- require.NoError(t, store.ApplyAllMigrations(
- context.Background(), migrations),
- )
+ require.NoError(t, ApplyAllMigrations(store, sets))
t.Cleanup(func() {
require.NoError(t, store.DB.Close())
@@ -170,15 +164,15 @@ func NewTestPostgresDB(t testing.TB, fixture *TestPgFixture,
// NewTestPostgresDBWithVersion is a helper function that creates a Postgres
// database for testing and migrates it to the given version.
-func NewTestPostgresDBWithVersion(t *testing.T, fixture *TestPgFixture,
- version uint) *PostgresStore {
+func NewTestPostgresDBWithVersion(t testing.TB, fixture *TestPgFixture,
+ sets MigrationSet, version uint) *PostgresStore {
t.Helper()
t.Logf("Creating new Postgres DB for testing, migrating to version %d",
version)
- dbName := randomDBName(t)
+ dbName := RandomDBName(t)
_, err := fixture.db.ExecContext(
context.Background(), "CREATE DATABASE "+dbName,
)
@@ -189,7 +183,7 @@ func NewTestPostgresDBWithVersion(t *testing.T, fixture *TestPgFixture,
store, err := NewPostgresStore(storeCfg)
require.NoError(t, err)
- err = store.ExecuteMigrations(TargetVersion(version))
+ err = store.ExecuteMigrations(TargetVersion(version), sets)
require.NoError(t, err)
t.Cleanup(func() {
diff --git a/sqldb/v2/postgres_test.go b/sqldb/v2/postgres_test.go
index cabf8cb..de6bac1 100644
--- a/sqldb/v2/postgres_test.go
+++ b/sqldb/v2/postgres_test.go
@@ -16,22 +16,24 @@ import (
const isSQLite = false
// NewTestDB is a helper function that creates a Postgres database for testing.
-func NewTestDB(t *testing.T, migrations []MigrationConfig) *PostgresStore {
+func NewTestDB(t *testing.T, sets []MigrationSet) *PostgresStore {
pgFixture := NewTestPgFixture(t, DefaultPostgresFixtureLifetime)
t.Cleanup(func() {
pgFixture.TearDown(t)
})
- return NewTestPostgresDB(t, pgFixture, migrations)
+ return NewTestPostgresDB(t, pgFixture, sets)
}
// NewTestDBWithVersion is a helper function that creates a Postgres database
// for testing and migrates it to the given version.
-func NewTestDBWithVersion(t *testing.T, version uint) *PostgresStore {
+func NewTestDBWithVersion(t *testing.T, version uint,
+ set MigrationSet) *PostgresStore {
+
pgFixture := NewTestPgFixture(t, DefaultPostgresFixtureLifetime)
t.Cleanup(func() {
pgFixture.TearDown(t)
})
- return NewTestPostgresDBWithVersion(t, pgFixture, version)
+ return NewTestPostgresDBWithVersion(t, pgFixture, set, version)
}
diff --git a/sqldb/v2/schemas.go b/sqldb/v2/schemas.go
deleted file mode 100644
index ec81fa8..0000000
--- a/sqldb/v2/schemas.go
+++ /dev/null
@@ -1,8 +0,0 @@
-package sqldb
-
-import (
- "embed"
-)
-
-//go:embed ../sqlc/migrations/*.up.sql
-var sqlSchemas embed.FS
diff --git a/sqldb/v2/sqlite.go b/sqldb/v2/sqlite.go
index ac97ff2..017eb06 100644
--- a/sqldb/v2/sqlite.go
+++ b/sqldb/v2/sqlite.go
@@ -3,12 +3,14 @@
package sqldb
import (
- "context"
"database/sql"
"fmt"
+ "github.com/golang-migrate/migrate/v4"
+ "github.com/lightningnetwork/lnd/fn/v2"
"net/url"
"path/filepath"
"testing"
+ "time"
sqlite_migrate "github.com/golang-migrate/migrate/v4/database/sqlite"
"github.com/lightningnetwork/lnd/sqldb/sqlc"
@@ -48,7 +50,9 @@ type pragmaOption struct {
// SqliteStore is a database store implementation that uses a sqlite backend.
type SqliteStore struct {
- cfg *SqliteConfig
+ DbPath string
+
+ Config *SqliteConfig
*BaseDB
}
@@ -142,7 +146,8 @@ func NewSqliteStore(cfg *SqliteConfig, dbPath string) (*SqliteStore, error) {
queries := sqlc.New(db)
s := &SqliteStore{
- cfg: cfg,
+ Config: cfg,
+ DbPath: dbPath,
BaseDB: &BaseDB{
DB: db,
Queries: queries,
@@ -158,41 +163,125 @@ func (s *SqliteStore) GetBaseDB() *BaseDB {
return s.BaseDB
}
-// ApplyAllMigrations applies both the SQLC and custom in-code migrations to the
-// SQLite database.
-func (s *SqliteStore) ApplyAllMigrations(ctx context.Context,
- migrations []MigrationConfig) error {
+func errSqliteMigration(err error) error {
+ return fmt.Errorf("error creating sqlite migration: %w", err)
+}
- // Execute migrations unless configured to skip them.
- if s.cfg.SkipMigrations {
- return nil
+// backupSqliteDatabase creates a backup of the given SQLite database.
+func backupSqliteDatabase(srcDB *sql.DB, dbFullFilePath string) error {
+ if srcDB == nil {
+ return fmt.Errorf("backup source database is nil")
+ }
+
+ // Create a database backup file full path from the given source
+ // database full file path.
+ //
+ // Get the current time and format it as a Unix timestamp in
+ // nanoseconds.
+ timestamp := time.Now().UnixNano()
+
+ // Add the timestamp to the backup name.
+ backupFullFilePath := fmt.Sprintf(
+ "%s.%d.backup", dbFullFilePath, timestamp,
+ )
+
+ log.Infof("Creating backup of database file: %v -> %v",
+ dbFullFilePath, backupFullFilePath)
+
+ // Create the database backup.
+ vacuumIntoQuery := "VACUUM INTO ?;"
+ stmt, err := srcDB.Prepare(vacuumIntoQuery)
+ if err != nil {
+ return err
}
+ defer stmt.Close()
- return ApplyMigrations(ctx, s.BaseDB, s, migrations)
+ _, err = stmt.Exec(backupFullFilePath)
+ if err != nil {
+ return err
+ }
+
+ return nil
}
-func errSqliteMigration(err error) error {
- return fmt.Errorf("error creating sqlite migration: %w", err)
+// backupAndMigrate is a helper function that creates a database backup before
+// initiating the migration, and then migrates the database to the latest
+// version.
+func (s *SqliteStore) backupAndMigrate(mig *migrate.Migrate,
+ currentDbVersion int, maxMigrationVersion uint) error {
+
+ // Determine if a database migration is necessary given the current
+ // database version and the maximum migration version.
+ versionUpgradePending := currentDbVersion < int(maxMigrationVersion)
+ if !versionUpgradePending {
+ log.Infof("Current database version is up-to-date, skipping "+
+ "migration attempt and backup creation "+
+ "(current_db_version=%v, max_migration_version=%v)",
+ currentDbVersion, maxMigrationVersion)
+ return nil
+ }
+
+ // At this point, we know that a database migration is necessary.
+ // Create a backup of the database before starting the migration.
+ if !s.Config.SkipMigrationDbBackup {
+ log.Infof("Creating database backup (before applying " +
+ "migration(s))")
+
+ err := backupSqliteDatabase(s.DB, s.DbPath)
+ if err != nil {
+ return err
+ }
+ } else {
+ log.Infof("Skipping database backup creation before applying " +
+ "migration(s)")
+ }
+
+ log.Infof("Applying migrations to database")
+ return mig.Up()
}
// ExecuteMigrations runs migrations for the sqlite database, depending on the
// target given, either all migrations or up to a given version.
-func (s *SqliteStore) ExecuteMigrations(target MigrationTarget) error {
+func (s *SqliteStore) ExecuteMigrations(target MigrationTarget,
+ set MigrationSet) error {
+
driver, err := sqlite_migrate.WithInstance(
- s.DB, &sqlite_migrate.Config{},
+ s.DB, &sqlite_migrate.Config{
+ MigrationsTable: set.TrackingTableName,
+ },
)
if err != nil {
return errSqliteMigration(err)
}
+ opts := &migrateOptions{
+ latestVersion: fn.Some(set.LatestMigrationVersion),
+ }
+
+ if set.MakeProgrammaticMigrations != nil {
+ postMigSteps, err := set.MakeProgrammaticMigrations(s.BaseDB)
+ if err != nil {
+ return errPostgresMigration(err)
+ }
+ opts.programmaticMigrs = postMigSteps
+ }
+
// Populate the database with our set of schemas based on our embedded
// in-memory file system.
- sqliteFS := newReplacerFS(sqlSchemas, sqliteSchemaReplacements)
+ sqliteFS := newReplacerFS(set.SQLFiles, sqliteSchemaReplacements)
return applyMigrations(
- sqliteFS, driver, "../sqlc/migrations", "sqlite", target,
+ sqliteFS, driver, set.SQLFileDirectory, "sqlite", target, opts,
)
}
+func (s *SqliteStore) DefaultTarget() MigrationTarget {
+ return s.backupAndMigrate
+}
+
+func (s *SqliteStore) SkipMigrations() bool {
+ return s.Config.SkipMigrations
+}
+
// GetSchemaVersion returns the current schema version of the SQLite database.
func (s *SqliteStore) GetSchemaVersion() (int, bool, error) {
driver, err := sqlite_migrate.WithInstance(
@@ -226,11 +315,7 @@ func (s *SqliteStore) SetSchemaVersion(version int, dirty bool) error {
// NewTestSqliteDB is a helper function that creates an SQLite database for
// testing.
-//
-// NOTE: This function differs from the one in sqldb/sqlite.go as that
-// function does not expect any migrations to be passed in, and instead always
-// applies the lnd specific migrations.
-func NewTestSqliteDB(t testing.TB, migrations []MigrationConfig) *SqliteStore {
+func NewTestSqliteDB(t testing.TB, sets []MigrationSet) *SqliteStore {
t.Helper()
t.Logf("Creating new SQLite DB for testing")
@@ -243,9 +328,7 @@ func NewTestSqliteDB(t testing.TB, migrations []MigrationConfig) *SqliteStore {
}, dbFileName)
require.NoError(t, err)
- require.NoError(t, sqlDB.ApplyAllMigrations(
- context.Background(), migrations),
- )
+ require.NoError(t, ApplyAllMigrations(sqlDB, sets))
t.Cleanup(func() {
require.NoError(t, sqlDB.DB.Close())
@@ -256,7 +339,9 @@ func NewTestSqliteDB(t testing.TB, migrations []MigrationConfig) *SqliteStore {
// NewTestSqliteDBWithVersion is a helper function that creates an SQLite
// database for testing and migrates it to the given version.
-func NewTestSqliteDBWithVersion(t *testing.T, version uint) *SqliteStore {
+func NewTestSqliteDBWithVersion(t *testing.T, set MigrationSet,
+ version uint) *SqliteStore {
+
t.Helper()
t.Logf("Creating new SQLite DB for testing, migrating to version %d",
@@ -270,7 +355,7 @@ func NewTestSqliteDBWithVersion(t *testing.T, version uint) *SqliteStore {
}, dbFileName)
require.NoError(t, err)
- err = sqlDB.ExecuteMigrations(TargetVersion(version))
+ err = sqlDB.ExecuteMigrations(TargetVersion(version), set)
require.NoError(t, err)
t.Cleanup(func() {
diff --git a/sqldb/v2/sqlite_test.go b/sqldb/v2/sqlite_test.go
index 4901dd0..89d2a09 100644
--- a/sqldb/v2/sqlite_test.go
+++ b/sqldb/v2/sqlite_test.go
@@ -1,5 +1,4 @@
//go:build !test_db_postgres
-// +build !test_db_postgres
package sqldb
@@ -16,12 +15,14 @@ import (
const isSQLite = true
// NewTestDB is a helper function that creates an SQLite database for testing.
-func NewTestDB(t *testing.T, migrations []MigrationConfig) *SqliteStore {
- return NewTestSqliteDB(t, migrations)
+func NewTestDB(t *testing.T, sets []MigrationSet) *SqliteStore {
+ return NewTestSqliteDB(t, sets)
}
// NewTestDBWithVersion is a helper function that creates an SQLite database
// for testing and migrates it to the given version.
-func NewTestDBWithVersion(t *testing.T, version uint) *SqliteStore {
- return NewTestSqliteDBWithVersion(t, version)
+func NewTestDBWithVersion(t *testing.T, set MigrationSet,
+ version uint) *SqliteStore {
+
+ return NewTestSqliteDBWithVersion(t, set, version)
}
Why this scored 32/100
Community notes
Notes can correct, qualify, or add evidence to the AI analysis. Every note shown here has been validated by a human moderator.
The AI analysis stands alone for now. Submit a note if you can add evidence or important context.