multi: make sure previous inconsitent states are fixed
What changed, and why it matters
This commit adds a startup repair routine for the LND Lightning node database. It fixes cases where a previous bug could delete 'link node' records even though the node still had open payment channels. On startup, LND now scans for missing link nodes and recreates them so the database stays consistent. The commit itself is a repair/cleanup patch, not an active vulnerability fix, but it addresses the after-effects of a prior race-condition bug.
Treat this as a data-integrity hardening patch. Operators should upgrade to ensure prior link-node pruning race conditions are repaired automatically on startup. Monitor logs for 'Repaired %d missing link nodes on startup' messages. No immediate active-exploit mitigation is required from this commit alone, but the underlying race condition that caused the inconsistency should be reviewed separately.
Security signals we found
Database consistency repair for previously corrupted link-node state
Startup-time remediation invoked before chain arbitrator and channel loading
Explicit reference to prior race condition in link node pruning
Idempotent repair helper functions added with unit tests
No input validation changes or network-exposed code paths in the diff
Evidence from the diff
The patch introduces RepairLinkNodes in channeldb/db.go, plus FindMissingLinkNodes and CreateLinkNodes helpers in channeldb/nodes.go. On server startup, before subsystems load channels, s.chanStateDB.RepairLinkNodes is called to enumerate peers with open channels, detect missing link-node entries, and recreate them. The commit message explicitly states this repairs ‘previous inconsistent states’ caused by a ‘race condition in link node pruning’. Tests verify idempotency and network-parameter handling. No remote exploit vector is present in this diff; it is a data-consistency remediation.
Changed components
channeldb/db.gochanneldb/nodes.gochanneldb/nodes_test.goserver.go startup sequenceInspect captured patch +438 / −3
diff --git a/channeldb/db.go b/channeldb/db.go
index 6064975..91f1886 100644
--- a/channeldb/db.go
+++ b/channeldb/db.go
@@ -1516,6 +1516,93 @@ func (c *ChannelStateDB) PruneLinkNodes() error {
return nil
}
+// RepairLinkNodes scans all channels in the database and ensures that a
+// link node exists for each remote peer. This should be called on startup to
+// ensure that our database is consistent.
+//
+// NOTE: This function is designed to repair database inconsistencies that may
+// have occurred due to the race condition in link node pruning (where link
+// nodes could be incorrectly deleted while channels still existed). This can
+// be removed once we move to native sql.
+func (c *ChannelStateDB) RepairLinkNodes(network wire.BitcoinNet) error {
+ // In a single read transaction, build a list of all peers with open
+ // channels and check which ones are missing link nodes.
+ var missingPeers []*btcec.PublicKey
+
+ err := kvdb.View(c.backend, func(tx kvdb.RTx) error {
+ openChanBucket := tx.ReadBucket(openChannelBucket)
+ if openChanBucket == nil {
+ return ErrNoActiveChannels
+ }
+
+ var peersWithChannels []*btcec.PublicKey
+
+ err := openChanBucket.ForEach(func(nodePubBytes,
+ _ []byte) error {
+
+ nodePub, err := btcec.ParsePubKey(nodePubBytes)
+ if err != nil {
+ return err
+ }
+
+ channels, err := c.fetchOpenChannels(tx, nodePub)
+ if err != nil {
+ return err
+ }
+
+ if len(channels) > 0 {
+ peersWithChannels = append(
+ peersWithChannels, nodePub,
+ )
+ }
+
+ return nil
+ })
+ if err != nil {
+ return err
+ }
+
+ // Now check which peers are missing link nodes within the
+ // same transaction.
+ missingPeers, err = c.linkNodeDB.FindMissingLinkNodes(
+ tx, peersWithChannels,
+ )
+
+ return err
+ }, func() {
+ missingPeers = nil
+ })
+ if err != nil && !errors.Is(err, ErrNoActiveChannels) {
+ return fmt.Errorf("unable to fetch channels: %w", err)
+ }
+
+ // Early exit if no repairs needed.
+ if len(missingPeers) == 0 {
+ return nil
+ }
+
+ // Create all missing link nodes in a single write transaction
+ // using the LinkNodeDB abstraction.
+ linkNodesToCreate := make([]*LinkNode, 0, len(missingPeers))
+ for _, remotePub := range missingPeers {
+ linkNode := NewLinkNode(c.linkNodeDB, network, remotePub)
+ linkNodesToCreate = append(linkNodesToCreate, linkNode)
+
+ log.Infof("Repairing missing link node for peer %x",
+ remotePub.SerializeCompressed())
+ }
+
+ err = c.linkNodeDB.CreateLinkNodes(nil, linkNodesToCreate)
+ if err != nil {
+ return err
+ }
+
+ log.Infof("Repaired %d missing link nodes on startup",
+ len(missingPeers))
+
+ return nil
+}
+
// ChannelShell is a shell of a channel that is meant to be used for channel
// recovery purposes. It contains a minimal OpenChannel instance along with
// addresses for that target node.
diff --git a/channeldb/nodes.go b/channeldb/nodes.go
index b17d5c3..70f6fad 100644
--- a/channeldb/nodes.go
+++ b/channeldb/nodes.go
@@ -2,6 +2,8 @@ package channeldb
import (
"bytes"
+ "errors"
+ "fmt"
"io"
"net"
"time"
@@ -134,6 +136,95 @@ type LinkNodeDB struct {
backend kvdb.Backend
}
+// FindMissingLinkNodes checks which of the provided public keys do not have
+// corresponding link nodes in the database. If tx is nil, a new read
+// transaction will be created. Otherwise, the provided transaction is used,
+// allowing this to be part of a larger batch operation.
+func (l *LinkNodeDB) FindMissingLinkNodes(tx kvdb.RTx,
+ pubKeys []*btcec.PublicKey) ([]*btcec.PublicKey, error) {
+
+ var missing []*btcec.PublicKey
+
+ findMissing := func(readTx kvdb.RTx) error {
+ nodeMetaBucket := readTx.ReadBucket(nodeInfoBucket)
+ if nodeMetaBucket == nil {
+ // If the bucket doesn't exist, all peers are missing.
+ missing = pubKeys
+ return nil
+ }
+
+ for _, pubKey := range pubKeys {
+ _, err := fetchLinkNode(readTx, pubKey)
+ if err == nil {
+ // Link node exists.
+ continue
+ }
+
+ if !errors.Is(err, ErrNodeNotFound) {
+ return fmt.Errorf("unable to check link node "+
+ "for peer %x: %w",
+ pubKey.SerializeCompressed(), err)
+ }
+
+ // Link node doesn't exist.
+ missing = append(missing, pubKey)
+ }
+
+ return nil
+ }
+
+ // If no transaction provided, create our own.
+ if tx == nil {
+ err := kvdb.View(l.backend, findMissing, func() {
+ missing = nil
+ })
+
+ return missing, err
+ }
+
+ // Use the provided transaction.
+ err := findMissing(tx)
+
+ return missing, err
+}
+
+// CreateLinkNodes creates multiple link nodes. If tx is nil, a new write
+// transaction will be created. Otherwise, the provided transaction is used,
+// allowing this to be part of a larger batch operation.
+func (l *LinkNodeDB) CreateLinkNodes(tx kvdb.RwTx,
+ linkNodes []*LinkNode) error {
+
+ createNodes := func(writeTx kvdb.RwTx) error {
+ nodeMetaBucket, err := writeTx.CreateTopLevelBucket(
+ nodeInfoBucket,
+ )
+ if err != nil {
+ return err
+ }
+
+ for _, linkNode := range linkNodes {
+ err := putLinkNode(nodeMetaBucket, linkNode)
+ if err != nil {
+ pubKey := linkNode.IdentityPub.
+ SerializeCompressed()
+
+ return fmt.Errorf("unable to create link "+
+ "node for peer %x: %w", pubKey, err)
+ }
+ }
+
+ return nil
+ }
+
+ // If no transaction provided, create our own.
+ if tx == nil {
+ return kvdb.Update(l.backend, createNodes, func() {})
+ }
+
+ // Use the provided transaction.
+ return createNodes(tx)
+}
+
// DeleteLinkNode removes the link node with the given identity from the
// database.
func (l *LinkNodeDB) DeleteLinkNode(identity *btcec.PublicKey) error {
diff --git a/channeldb/nodes_test.go b/channeldb/nodes_test.go
index b54cf00..a88e452 100644
--- a/channeldb/nodes_test.go
+++ b/channeldb/nodes_test.go
@@ -8,6 +8,7 @@ import (
"github.com/btcsuite/btcd/btcec/v2"
"github.com/btcsuite/btcd/wire"
+ "github.com/lightningnetwork/lnd/kvdb"
"github.com/stretchr/testify/require"
)
@@ -129,3 +130,245 @@ func TestDeleteLinkNode(t *testing.T) {
t.Fatal("should not have found link node in db, but did")
}
}
+
+// TestRepairLinkNodes tests that the RepairLinkNodes function correctly
+// identifies and repairs missing link nodes for channels that exist in the
+// database.
+func TestRepairLinkNodes(t *testing.T) {
+ t.Parallel()
+
+ fullDB, err := MakeTestDB(t)
+ require.NoError(t, err, "unable to make test database")
+
+ cdb := fullDB.ChannelStateDB()
+
+ // Create a test channel and save it to the database.
+ channel1 := createTestChannel(t, cdb)
+
+ // Manually create a link node for the channel.
+ linkNode1 := NewLinkNode(
+ cdb.linkNodeDB, wire.MainNet, channel1.IdentityPub,
+ )
+ err = linkNode1.Sync()
+ require.NoError(t, err, "unable to sync link node")
+
+ // Verify that link node was created.
+ fetchedLinkNode, err := cdb.linkNodeDB.FetchLinkNode(
+ channel1.IdentityPub,
+ )
+ require.NoError(t, err, "link node should exist")
+ require.NotNil(t, fetchedLinkNode, "link node should not be nil")
+
+ // Now, manually delete one of the link nodes to simulate the race
+ // condition scenario where a link node was incorrectly pruned.
+ err = cdb.linkNodeDB.DeleteLinkNode(channel1.IdentityPub)
+ require.NoError(t, err, "unable to delete link node")
+
+ // Verify the link node is gone.
+ _, err = cdb.linkNodeDB.FetchLinkNode(channel1.IdentityPub)
+ require.ErrorIs(
+ t, err, ErrNodeNotFound,
+ "link node should be deleted",
+ )
+
+ // Now run the repair function with the correct network.
+ err = cdb.RepairLinkNodes(wire.MainNet)
+ require.NoError(t, err, "repair should succeed")
+
+ // Verify that the link node has been restored.
+ repairedLinkNode, err := cdb.linkNodeDB.FetchLinkNode(
+ channel1.IdentityPub,
+ )
+ require.NoError(t, err, "repaired link node should exist")
+ require.NotNil(
+ t, repairedLinkNode, "repaired link node should not be nil",
+ )
+ require.Equal(
+ t, wire.MainNet, repairedLinkNode.Network,
+ "repaired link node should have correct network",
+ )
+
+ // Run repair again - it should be idempotent and not fail.
+ err = cdb.RepairLinkNodes(wire.MainNet)
+ require.NoError(t, err, "second repair should succeed")
+
+ // Test with different network to ensure network parameter is used.
+ err = cdb.linkNodeDB.DeleteLinkNode(channel1.IdentityPub)
+ require.NoError(t, err, "unable to delete link node")
+
+ err = cdb.RepairLinkNodes(wire.TestNet3)
+ require.NoError(t, err, "repair with testnet should succeed")
+
+ repairedLinkNode, err = cdb.linkNodeDB.FetchLinkNode(
+ channel1.IdentityPub,
+ )
+ require.NoError(t, err, "repaired link node should exist")
+ require.Equal(
+ t, wire.TestNet3, repairedLinkNode.Network,
+ "repaired link node should use provided network",
+ )
+}
+
+// TestFindMissingLinkNodes tests the FindMissingLinkNodes method with various
+// scenarios.
+func TestFindMissingLinkNodes(t *testing.T) {
+ t.Parallel()
+
+ fullDB, err := MakeTestDB(t)
+ require.NoError(t, err, "unable to make test database")
+
+ cdb := fullDB.ChannelStateDB()
+
+ // Create three test public keys.
+ _, pub1 := btcec.PrivKeyFromBytes(key[:])
+ _, pub2 := btcec.PrivKeyFromBytes(rev[:])
+ testKey := [32]byte{0x03}
+ _, pub3 := btcec.PrivKeyFromBytes(testKey[:])
+
+ // Test 1: All nodes missing (empty database).
+ allPubs := []*btcec.PublicKey{pub1, pub2, pub3}
+ missing, err := cdb.linkNodeDB.FindMissingLinkNodes(nil, allPubs)
+ require.NoError(t, err, "FindMissingLinkNodes should succeed")
+ require.Len(t, missing, 3, "all nodes should be missing")
+
+ // Test 2: Create one link node, verify only 2 are missing.
+ node1 := NewLinkNode(cdb.linkNodeDB, wire.MainNet, pub1)
+ err = node1.Sync()
+ require.NoError(t, err, "unable to sync link node")
+
+ missing, err = cdb.linkNodeDB.FindMissingLinkNodes(nil, allPubs)
+ require.NoError(t, err, "FindMissingLinkNodes should succeed")
+ require.Len(t, missing, 2, "two nodes should be missing")
+ require.Contains(t, missing, pub2, "pub2 should be missing")
+ require.Contains(t, missing, pub3, "pub3 should be missing")
+ require.NotContains(t, missing, pub1, "pub1 should exist")
+
+ // Test 3: Create remaining nodes, verify none are missing.
+ node2 := NewLinkNode(cdb.linkNodeDB, wire.MainNet, pub2)
+ err = node2.Sync()
+ require.NoError(t, err, "unable to sync link node")
+
+ node3 := NewLinkNode(cdb.linkNodeDB, wire.MainNet, pub3)
+ err = node3.Sync()
+ require.NoError(t, err, "unable to sync link node")
+
+ missing, err = cdb.linkNodeDB.FindMissingLinkNodes(nil, allPubs)
+ require.NoError(t, err, "FindMissingLinkNodes should succeed")
+ require.Len(t, missing, 0, "no nodes should be missing")
+
+ // Test 4: Use with a provided transaction.
+ err = cdb.linkNodeDB.DeleteLinkNode(pub2)
+ require.NoError(t, err, "unable to delete link node")
+
+ backend := fullDB.ChannelStateDB().backend
+ err = kvdb.View(backend, func(tx kvdb.RTx) error {
+ missing, err := cdb.linkNodeDB.FindMissingLinkNodes(
+ tx, allPubs,
+ )
+ require.NoError(t, err, "FindMissingLinkNodes should succeed")
+ require.Len(t, missing, 1, "one node should be missing")
+ require.Contains(t, missing, pub2, "pub2 should be missing")
+
+ return nil
+ }, func() {})
+ require.NoError(t, err, "transaction should succeed")
+
+ // Test 5: Empty input list.
+ missing, err = cdb.linkNodeDB.FindMissingLinkNodes(nil, nil)
+ require.NoError(t, err, "FindMissingLinkNodes should succeed")
+ require.Len(t, missing, 0, "no nodes should be missing for empty input")
+}
+
+// TestCreateLinkNodes tests the CreateLinkNodes method with various scenarios.
+func TestCreateLinkNodes(t *testing.T) {
+ t.Parallel()
+
+ fullDB, err := MakeTestDB(t)
+ require.NoError(t, err, "unable to make test database")
+
+ cdb := fullDB.ChannelStateDB()
+
+ // Create three test public keys and link nodes.
+ _, pub1 := btcec.PrivKeyFromBytes(key[:])
+ _, pub2 := btcec.PrivKeyFromBytes(rev[:])
+ testKey := [32]byte{0x03}
+ _, pub3 := btcec.PrivKeyFromBytes(testKey[:])
+
+ node1 := NewLinkNode(cdb.linkNodeDB, wire.MainNet, pub1)
+ node2 := NewLinkNode(cdb.linkNodeDB, wire.TestNet3, pub2)
+ node3 := NewLinkNode(cdb.linkNodeDB, wire.SimNet, pub3)
+
+ // Test 1: Create multiple link nodes at once with nil transaction.
+ nodesToCreate := []*LinkNode{node1, node2, node3}
+ err = cdb.linkNodeDB.CreateLinkNodes(nil, nodesToCreate)
+ require.NoError(t, err, "CreateLinkNodes should succeed")
+
+ // Verify all nodes were created correctly.
+ fetchedNode1, err := cdb.linkNodeDB.FetchLinkNode(pub1)
+ require.NoError(t, err, "node1 should exist")
+ require.Equal(t, wire.MainNet, fetchedNode1.Network,
+ "node1 should have correct network")
+
+ fetchedNode2, err := cdb.linkNodeDB.FetchLinkNode(pub2)
+ require.NoError(t, err, "node2 should exist")
+ require.Equal(t, wire.TestNet3, fetchedNode2.Network,
+ "node2 should have correct network")
+
+ fetchedNode3, err := cdb.linkNodeDB.FetchLinkNode(pub3)
+ require.NoError(t, err, "node3 should exist")
+ require.Equal(t, wire.SimNet, fetchedNode3.Network,
+ "node3 should have correct network")
+
+ // Test 2: Create nodes within a provided transaction.
+ err = cdb.linkNodeDB.DeleteLinkNode(pub2)
+ require.NoError(t, err, "unable to delete link node")
+
+ // Verify node2 is deleted.
+ _, err = cdb.linkNodeDB.FetchLinkNode(pub2)
+ require.ErrorIs(t, err, ErrNodeNotFound, "node2 should be deleted")
+
+ // Recreate node2 using a provided transaction.
+ backend := fullDB.ChannelStateDB().backend
+ err = kvdb.Update(backend, func(tx kvdb.RwTx) error {
+ return cdb.linkNodeDB.CreateLinkNodes(tx, []*LinkNode{node2})
+ }, func() {})
+ require.NoError(t, err, "transaction should succeed")
+
+ // Verify node2 was recreated.
+ fetchedNode2, err = cdb.linkNodeDB.FetchLinkNode(pub2)
+ require.NoError(t, err, "node2 should exist after recreation")
+ require.Equal(t, wire.TestNet3, fetchedNode2.Network,
+ "node2 should have correct network")
+
+ // Test 3: Creating nodes that already exist should succeed
+ // (idempotent behavior).
+ err = cdb.linkNodeDB.CreateLinkNodes(nil, nodesToCreate)
+ require.NoError(t, err, "recreating existing nodes should succeed")
+
+ // Verify nodes still exist with correct data.
+ fetchedNode1, err = cdb.linkNodeDB.FetchLinkNode(pub1)
+ require.NoError(t, err, "node1 should still exist")
+ require.Equal(t, wire.MainNet, fetchedNode1.Network,
+ "node1 should still have correct network")
+
+ // Test 4: Empty input list.
+ err = cdb.linkNodeDB.CreateLinkNodes(nil, nil)
+ require.NoError(
+ t, err, "CreateLinkNodes with empty list should succeed",
+ )
+
+ // Test 5: Create single node.
+ testKey4 := [32]byte{0x04}
+ _, pub4 := btcec.PrivKeyFromBytes(testKey4[:])
+ node4 := NewLinkNode(cdb.linkNodeDB, wire.MainNet, pub4)
+
+ err = cdb.linkNodeDB.CreateLinkNodes(nil, []*LinkNode{node4})
+ require.NoError(
+ t, err, "CreateLinkNodes with single node should succeed",
+ )
+
+ fetchedNode4, err := cdb.linkNodeDB.FetchLinkNode(pub4)
+ require.NoError(t, err, "node4 should exist")
+ require.Equal(t, wire.MainNet, fetchedNode4.Network,
+ "node4 should have correct network")
+}
diff --git a/server.go b/server.go
index c3b724e..f7b4351 100644
--- a/server.go
+++ b/server.go
@@ -2144,6 +2144,21 @@ func (s *server) Start(ctx context.Context) error {
cleanup := cleaner{}
s.start.Do(func() {
+ // Before starting any subsystems, repair any link nodes that
+ // may have been incorrectly pruned due to the race condition
+ // that was fixed in the link node pruning logic. This must
+ // happen before the chain arbitrator and other subsystems load
+ // channels, to ensure the invariant "link node exists iff
+ // channels exist" is maintained.
+ err := s.chanStateDB.RepairLinkNodes(s.cfg.ActiveNetParams.Net)
+ if err != nil {
+ srvrLog.Errorf("Failed to repair link nodes: %v", err)
+
+ startErr = err
+
+ return
+ }
+
cleanup = cleanup.add(s.customMessageServer.Stop)
if err := s.customMessageServer.Start(); err != nil {
startErr = err
@@ -2473,9 +2488,8 @@ func (s *server) Start(ctx context.Context) error {
// With all the relevant sub-systems started, we'll now attempt
// to establish persistent connections to our direct channel
// collaborators within the network. Before doing so however,
- // we'll prune our set of link nodes found within the database
- // to ensure we don't reconnect to any nodes we no longer have
- // open channels with.
+ // we'll prune our set of link nodes to ensure we don't
+ // reconnect to any nodes we no longer have open channels with.
if err := s.chanStateDB.PruneLinkNodes(); err != nil {
srvrLog.Errorf("Failed to prune link nodes: %v", err)
Why this scored 46/100
Community notes
Notes can correct, qualify, or add evidence to the AI analysis. Every note shown here has been validated by a human moderator.
The AI analysis stands alone for now. Submit a note if you can add evidence or important context.