server: serialize peer lifecycle via single goroutine
What changed, and why it matters
This commit is a follow-up fix for a race condition in how btcd adds and removes peers. The previous fix tried to ensure that when a peer connects and quickly disconnects, the 'peer added' and 'peer done' notifications are always seen in the right order by the sync manager. This commit makes that guarantee stronger by having a single goroutine handle both events, instead of two separate code paths. It also adds tests that demonstrate the bug could corrupt sync manager state and prevent the node from syncing.
Treat this as a security-hardening fix for a denial-of-sync condition. Reviewers should verify that peerLifecycleHandler cannot deadlock if OnVerAck never fires, confirm that all peerAdd/peerDone sends are now routed through this goroutine, and run the new integration tests under race detection. Consider backporting to maintained release branches.
Security signals we found
Race condition in peer add/done event ordering
Sync manager can be left with a dead peer as its sync peer
Node may stop syncing from new legitimate peers after attack traffic
Single-goroutine serialization of peer lifecycle events
New select-based disconnect detection via peer.Peer.Done()
Integration test demonstrates real-world corruption scenario
Evidence from the diff
The patch reworks peer lifecycle event delivery in btcd’s server. Previously, OnVerAck called server.AddPeer to enqueue a peerAdd event, while a separate peerDoneHandler goroutine enqueued peerDone after disconnect. This left a race window (notably around negotiateTimeout) where peerDone could be processed before peerAdd, leaving the sync manager with a dead sync peer and blocking future sync. The fix introduces peerLifecycleHandler, a per-peer goroutine that selects on verAckCh (closed by OnVerAck) and Peer.Done(), sends peerAdd only if verack occurred, then waits for disconnect and sends peerDone. A new peer.Peer.Done() method exposes the quit channel for select. The integration test TestSyncManagerRaceCorruption reproduces the corruption by stressing a node with handshake-then-disconnect peers and verifying it fails to sync; TestPreVerackDisconnect covers the no-verack disconnect path.
Changed components
server.go peer lifecycle handlingpeer/peer.go disconnect signalingintegration/sync_race_test.go race regression testsInspect captured patch +89 / −52
diff --git a/integration/sync_race_test.go b/integration/sync_race_test.go
index 910e637..b313599 100644
--- a/integration/sync_race_test.go
+++ b/integration/sync_race_test.go
@@ -23,10 +23,12 @@ const (
syncRaceProofWait = 8 * time.Second
)
-// fakePeerConn connects to the node at nodeAddr, performs the minimum version/verack
-// handshake so the node registers a peer (NewPeer) and then disconnects so the node
-// runs DonePeer. This simulates attacker traffic: many connections that complete
-// handshake then drop, stressing the sync manager's ordering of NewPeer/DonePeer.
+// fakePeerConn connects to the node at nodeAddr, performs the
+// minimum version/verack handshake so the node registers a peer
+// (NewPeer) and then disconnects so the node runs DonePeer. This
+// simulates attacker traffic: many connections that complete the
+// handshake then drop, stressing the sync manager's ordering of
+// NewPeer/DonePeer.
func fakePeerConn(nodeAddr string) error {
conn, err := net.DialTimeout("tcp", nodeAddr, 5*time.Second)
if err != nil {
@@ -54,7 +56,10 @@ func fakePeerConn(nodeAddr string) error {
msgVersion := wire.NewMsgVersion(me, you, nonce, 0)
msgVersion.Services = wire.SFNodeNetwork | wire.SFNodeWitness
- if err := wire.WriteMessage(conn, msgVersion, wire.ProtocolVersion, wire.SimNet); err != nil {
+ err = wire.WriteMessage(
+ conn, msgVersion, wire.ProtocolVersion, wire.SimNet,
+ )
+ if err != nil {
return err
}
@@ -66,7 +71,11 @@ func fakePeerConn(nodeAddr string) error {
switch msg.(type) {
case *wire.MsgVersion:
// Node's version; send verack.
- if err := wire.WriteMessage(conn, wire.NewMsgVerAck(), wire.ProtocolVersion, wire.SimNet); err != nil {
+ err := wire.WriteMessage(
+ conn, wire.NewMsgVerAck(),
+ wire.ProtocolVersion, wire.SimNet,
+ )
+ if err != nil {
return err
}
@@ -83,12 +92,12 @@ func fakePeerConn(nodeAddr string) error {
}
}
-// TestSyncManagerRaceCorruption stresses a single simnet node with many inbound
-// connections that complete the version/verack handshake then disconnect. It then
-// proves corruption without a dedicated RPC: connect a fresh node that generates
-// blocks; if the stressed node does not sync, it was stuck with a dead sync peer
-// (getpeerinfo returns 0 peers when all disconnected; in the corrupted state the
-// sync manager still has a dead peer as sync peer, so it ignores the new live one).
+// TestSyncManagerRaceCorruption stresses a single simnet node
+// with many inbound connections that complete the version/verack
+// handshake then disconnect. It then proves corruption: connect a
+// fresh node that generates blocks; if the stressed node does not
+// sync, it was stuck with a dead sync peer (the sync manager still
+// has a dead peer as sync peer, so it ignores the new live one).
func TestSyncManagerRaceCorruption(t *testing.T) {
stressedHarness, err := rpctest.New(&chaincfg.SimNetParams, nil, nil, "")
require.NoError(t, err)
@@ -117,8 +126,9 @@ func TestSyncManagerRaceCorruption(t *testing.T) {
iter += syncRaceConcurrency
}
- // Prove corruption: connect a live node and generate blocks.
- // If the stressed node was corrupted (dead sync peer, 0 connected peers per getpeerinfo), it will not sync from the new one.
+ // Prove corruption: connect a live node and generate blocks. If
+ // the stressed node was corrupted (dead sync peer, 0 connected
+ // peers), it will not sync from the new one.
newHarness, err := rpctest.New(&chaincfg.SimNetParams, nil, nil, "")
require.NoError(t, err)
require.NoError(t, newHarness.SetUp(true, 0))
@@ -138,18 +148,23 @@ func TestSyncManagerRaceCorruption(t *testing.T) {
_, heightAfter, err := stressedHarness.Client.GetBestBlock()
require.NoError(t, err)
- if heightAfter < heightBefore+int32(syncRaceProofBlocks) {
- t.Fatalf("proved sync manager corruption after %d fake peer cycles: stressed node did not sync from new peer (height %d -> %d); it was stuck with a dead sync peer instead of the new live one",
- done, heightBefore, heightAfter)
- }
+ expected := heightBefore + int32(syncRaceProofBlocks)
+ require.GreaterOrEqualf(t, heightAfter, expected,
+ "sync manager corruption after %d fake "+
+ "peer cycles: node stuck with dead "+
+ "sync peer (height %d -> %d)",
+ done, heightBefore, heightAfter)
- t.Logf("completed %d fake peer cycles; stressed node synced from new peer (height %d -> %d), no corruption observed", done, heightBefore, heightAfter)
+ t.Logf("completed %d fake peer cycles; "+
+ "node synced (height %d -> %d)",
+ done, heightBefore, heightAfter)
}
-// TestPreVerackDisconnect verifies that a peer disconnecting before completing
-// the version/verack handshake does not corrupt the sync manager state. In this
-// case only a peerDone event is produced (no peerAdd), since AddPeer is only
-// called from OnVerAck. The node must remain healthy and able to sync afterward.
+// TestPreVerackDisconnect verifies that a peer disconnecting
+// before completing the version/verack handshake does not corrupt
+// the sync manager state. In this case only a peerDone event is
+// produced (no peerAdd), since peerLifecycleHandler only sends
+// peerAdd when verAckCh is closed. The node must remain healthy.
func TestPreVerackDisconnect(t *testing.T) {
harness, err := rpctest.New(&chaincfg.SimNetParams, nil, nil, "")
require.NoError(t, err)
diff --git a/peer/peer.go b/peer/peer.go
index ee6f317..0cd5706 100644
--- a/peer/peer.go
+++ b/peer/peer.go
@@ -2477,6 +2477,13 @@ func (p *Peer) WaitForDisconnect() {
<-p.quit
}
+// Done returns a channel that is closed when the peer has been
+// disconnected. This allows callers to select on peer disconnect
+// alongside other channels.
+func (p *Peer) Done() <-chan struct{} {
+ return p.quit
+}
+
// ShouldDowngradeToV1 is called when we try to connect to a peer via v2 BIP324
// transport and they hang up. In this case, we should reconnect with the
// legacy transport.
diff --git a/server.go b/server.go
index f5b6c0b..a8a5fe0 100644
--- a/server.go
+++ b/server.go
@@ -155,14 +155,14 @@ type updatePeerHeightsMsg struct {
type peerLifecycleAction uint8
const (
- peerAdd peerLifecycleAction = iota
+ peerAdd peerLifecycleAction = iota
peerDone
)
-// peerLifecycleEvent represents a peer connection or disconnection event.
-// Using a single channel for both event types guarantees FIFO ordering:
-// the add event from OnVerAck is always enqueued before the done event
-// from peerDoneHandler, so the receiver always sees add before done.
+// peerLifecycleEvent represents a peer connection or disconnection
+// event. Both event types for a given peer are sent by a single
+// goroutine (peerLifecycleHandler), guaranteeing that peerAdd is
+// always enqueued before peerDone.
type peerLifecycleEvent struct {
action peerLifecycleAction
sp *serverPeer
@@ -295,6 +295,7 @@ type serverPeer struct {
knownAddresses lru.Cache
banScore connmgr.DynamicBanScore
quit chan struct{}
+ verAckCh chan struct{} // closed when OnVerAck fires
// The following chans are used to sync blockmanager and server.
txProcessed chan struct{}
blockProcessed chan struct{}
@@ -309,6 +310,7 @@ func newServerPeer(s *server, isPersistent bool) *serverPeer {
filter: bloom.LoadFilter(nil),
knownAddresses: lru.NewCache(5000),
quit: make(chan struct{}),
+ verAckCh: make(chan struct{}),
txProcessed: make(chan struct{}, 1),
blockProcessed: make(chan struct{}, 1),
}
@@ -551,10 +553,11 @@ func (sp *serverPeer) OnVersion(_ *peer.Peer, msg *wire.MsgVersion) *wire.MsgRej
return nil
}
-// OnVerAck is invoked when a peer receives a verack bitcoin message and is used
-// to kick start communication with them.
+// OnVerAck is invoked when a peer receives a verack bitcoin message.
+// It signals the peer's lifecycle handler that the handshake is
+// complete so it can register the peer with the server.
func (sp *serverPeer) OnVerAck(_ *peer.Peer, _ *wire.MsgVerAck) {
- sp.server.AddPeer(sp)
+ close(sp.verAckCh)
}
// OnMemPool is invoked when a peer receives a mempool bitcoin message.
@@ -1925,11 +1928,8 @@ func (s *server) handleDonePeerMsg(state *peerState, sp *serverPeer) {
srvrLog.Debugf("Removed peer %s", sp)
}
- // Notify the sync manager the peer is gone and evict any remaining
- // orphans that were sent by the peer. This is done here rather than in
- // peerDoneHandler so that the notification is serialized with NewPeer
- // calls through the peerHandler goroutine, guaranteeing that the sync
- // manager always sees NewPeer before DonePeer for a given peer.
+ // Notify the sync manager the peer is gone and evict any
+ // remaining orphans that were sent by the peer.
if sp.VerAckReceived() {
s.syncManager.DonePeer(sp.Peer)
@@ -2262,7 +2262,7 @@ func (s *server) inboundPeerConnected(conn net.Conn) {
sp.isWhitelisted = isWhitelisted(conn.RemoteAddr())
sp.Peer = peer.NewInboundPeer(newPeerConfig(sp))
sp.AssociateConnection(conn)
- go s.peerDoneHandler(sp)
+ go s.peerLifecycleHandler(sp)
}
// outboundPeerConnected is invoked by the connection manager when a new
@@ -2304,25 +2304,45 @@ func (s *server) outboundPeerConnected(c *connmgr.ConnReq, conn net.Conn) {
sp.connReq = c
sp.isWhitelisted = isWhitelisted(conn.RemoteAddr())
sp.AssociateConnection(conn)
- go s.peerDoneHandler(sp)
+ go s.peerLifecycleHandler(sp)
}
-// peerDoneHandler handles peer disconnects by notifying the server that it's
-// done along with other performing other desirable cleanup.
-func (s *server) peerDoneHandler(sp *serverPeer) {
+// peerLifecycleHandler is the sole sender of lifecycle events for a
+// given peer. It waits for either verack (handshake complete) or
+// disconnect (handshake failed/timed out), sends peerAdd if verack
+// was received, then waits for disconnect and sends peerDone.
+// Because both sends originate from this single goroutine,
+// peerAdd is always enqueued before peerDone.
+func (s *server) peerLifecycleHandler(sp *serverPeer) {
+ // Wait for the handshake to complete or the peer to
+ // disconnect, whichever comes first.
+ select {
+ case <-sp.verAckCh:
+ s.peerLifecycle <- peerLifecycleEvent{
+ action: peerAdd, sp: sp,
+ }
+
+ case <-sp.Peer.Done():
+ // Disconnected before verack; no peerAdd needed.
+ }
+
+ // Wait for full disconnect (may already be done).
sp.WaitForDisconnect()
- // If this is an outbound peer and the shouldDowngradeToV1 bool is set
- // on the underlying Peer, trigger a reconnect using the OG v1
- // connection scheme.
+ // If this is an outbound peer and the shouldDowngradeToV1
+ // bool is set on the underlying Peer, trigger a reconnect
+ // using the OG v1 connection scheme.
if !sp.Inbound() && sp.Peer.ShouldDowngradeToV1() {
- srvrLog.Infof("Peer %s indicated v2->v1 downgrade. "+
- "Marking for next attempt as v1.", sp.Addr())
+ srvrLog.Infof("Peer %s indicated v2->v1 downgrade."+
+ " Marking for next attempt as v1.",
+ sp.Addr())
s.p2pDowngrader.MarkForDowngrade(sp.Addr())
}
- s.peerLifecycle <- peerLifecycleEvent{action: peerDone, sp: sp}
+ s.peerLifecycle <- peerLifecycleEvent{
+ action: peerDone, sp: sp,
+ }
close(sp.quit)
}
@@ -2427,11 +2447,6 @@ cleanup:
srvrLog.Tracef("Peer handler done")
}
-// AddPeer adds a new peer that has already been connected to the server.
-func (s *server) AddPeer(sp *serverPeer) {
- s.peerLifecycle <- peerLifecycleEvent{action: peerAdd, sp: sp}
-}
-
// BanPeer bans a peer that has already been connected to the server by ip.
func (s *server) BanPeer(sp *serverPeer) {
s.banPeers <- sp
Why this scored 69/100
Community notes
Notes can correct, qualify, or add evidence to the AI analysis. Every note shown here has been validated by a human moderator.
The AI analysis stands alone for now. Submit a note if you can add evidence or important context.