Merge PR 'Resume channels stuck awaiting a completed monitor persistence on reload' (#5610)
What changed, and why it matters
This commit fixes a bug where a Lightning node could get stuck after restarting. If the node crashed or restarted while it was still saving the initial channel monitor to disk, and that save later finished in another process, the restarted node would not realize the save was done. As a result, it would never broadcast the funding transaction or resume the channel, leaving funds temporarily inaccessible and the channel unusable. The fix makes the node check on reload whether the monitor persistence actually completed and, if so, resume the channel normally.
Treat as a reliability/availability fix with minor security relevance. Users running LDK nodes should upgrade to avoid channels getting stuck after restarts involving asynchronous ChannelMonitor persistence. No immediate emergency response is indicated, but operators should monitor for channels stuck in a pending state after reload on older versions.
Security signals we found
Fixes a state-machine stall on node reload that could leave channels non-functional
Removes an overstrict assertion that would panic on a legitimate post-reload state
Adds explicit detection of monitor persistence completed in a prior process
Includes regression tests for async monitor persistence + restart scenarios
Touches funding-transaction broadcast and channel_ready generation paths
Evidence from the diff
The patch addresses a state-recovery issue in LDK’s ChannelManager reload path. When an outbound channel’s initial ChannelMonitor persistence was pending at the time the ChannelManager was serialized, and that persistence completed (and the funding transaction was even broadcast/confirmed) before the current process started, the loaded ChannelManager would remain in an awaiting-monitor-update state with no in-flight or blocked updates. The new code detects this condition by checking funded_chan.is_awaiting_monitor_update(), blocked_monitor_updates_pending() == 0, and monitor.get_latest_update_id() == funded_chan.get_latest_unblocked_monitor_update_id(). It then emits a MonitorUpdatesComplete background event to resume the channel. A related assertion in channel.rs that incorrectly assumed monitor_pending_channel_ready could not occur for non-0conf outbound channels after reload is removed. New regression tests cover both the completed-persistence reload case and the case where funding confirms during downtime.
Changed components
lightning/src/ln/channelmanager.rslightning/src/ln/channel.rslightning/src/ln/chanmon_update_fail_tests.rslightning/src/ln/monitor_tests.rsInspect captured patch +334 / −7
### lightning/src/ln/chanmon_update_fail_tests.rs
@@ -3298,6 +3298,300 @@ fn test_outbound_reload_without_init_mon() {
do_test_outbound_reload_without_init_mon(false);
}
+fn do_test_outbound_reload_with_completed_init_mon(use_0conf: bool) {
+ // Test that if the monitor update generated in funding_signed is stored async and we restart
+ // with a ChannelManager serialized while it was pending, but the ChannelMonitor persistence
+ // did complete before the restart, we resume the channel and broadcast the funding
+ // transaction.
+ let chanmon_cfgs = create_chanmon_cfgs(2);
+ let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
+
+ let persister;
+ let new_chain_monitor;
+
+ let mut chan_config = test_default_channel_config();
+ chan_config.channel_handshake_limits.trust_own_funding_0conf = true;
+
+ let node_chanmgrs =
+ create_node_chanmgrs(2, &node_cfgs, &[Some(chan_config.clone()), Some(chan_config)]);
+ let node_a_reload;
+
+ let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
+
+ let node_a_id = nodes[0].node.get_our_node_id();
+ let node_b_id = nodes[1].node.get_our_node_id();
+
+ nodes[0].node.create_channel(node_b_id, 100000, 10001, 43, None, None).unwrap();
+ nodes[1].node.handle_open_channel(
+ node_a_id,
+ &get_event_msg!(nodes[0], MessageSendEvent::SendOpenChannel, node_b_id),
+ );
+
+ let events = nodes[1].node.get_and_clear_pending_events();
+ assert_eq!(events.len(), 1);
+ match events[0] {
+ Event::OpenChannelRequest { temporary_channel_id: chan_id, .. } => {
+ if use_0conf {
+ nodes[1]
+ .node
+ .accept_inbound_channel_from_trusted_peer(
+ &chan_id,
+ &node_a_id,
+ 0,
+ TrustedChannelFeatures::ZeroConf,
+ None,
+ )
+ .unwrap();
+ } else {
+ nodes[1].node.accept_inbound_channel(&chan_id, &node_a_id, 0, None).unwrap();
+ }
+ },
+ _ => panic!("Unexpected event"),
+ };
+
+ nodes[0].node.handle_accept_channel(
+ node_b_id,
+ &get_event_msg!(nodes[1], MessageSendEvent::SendAcceptChannel, node_a_id),
+ );
+
+ let (temporary_channel_id, funding_tx, ..) =
+ create_funding_transaction(&nodes[0], &node_b_id, 100000, 43);
+
+ nodes[0]
+ .node
+ .funding_transaction_generated(temporary_channel_id, node_b_id, funding_tx.clone())
+ .unwrap();
+ check_added_monitors(&nodes[0], 0);
+
+ let funding_created_msg =
+ get_event_msg!(nodes[0], MessageSendEvent::SendFundingCreated, node_b_id);
+ nodes[1].node.handle_funding_created(node_a_id, &funding_created_msg);
+ check_added_monitors(&nodes[1], 1);
+ expect_channel_pending_event(&nodes[1], &node_a_id);
+
+ let bs_signed_locked = nodes[1].node.get_and_clear_pending_msg_events();
+ assert_eq!(bs_signed_locked.len(), if use_0conf { 2 } else { 1 });
+ let chan_id = match &bs_signed_locked[0] {
+ MessageSendEvent::SendFundingSigned { msg, .. } => {
+ chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::InProgress);
+
+ nodes[0].node.handle_funding_signed(node_b_id, &msg);
+ check_added_monitors(&nodes[0], 1);
+ msg.channel_id
+ },
+ _ => panic!("Unexpected event"),
+ };
+ if use_0conf {
+ match &bs_signed_locked[1] {
+ MessageSendEvent::SendChannelReady { msg, .. } => {
+ nodes[0].node.handle_channel_ready(node_b_id, &msg);
+ },
+ _ => panic!("Unexpected event"),
+ }
+ }
+
+ assert!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().is_empty());
+ assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
+ assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
+
+ // nodes[0] is now waiting on the first ChannelMonitor persistence to complete in order to
+ // broadcast the funding transaction. If nodes[0] restarts at this point with the
+ // ChannelMonitor persisted, it should resume the channel.
+ let node_a_ser = nodes[0].node.encode();
+ let mon_ser = get_monitor!(nodes[0], chan_id).encode();
+ reload_node!(nodes[0], &node_a_ser, &[&mon_ser], persister, new_chain_monitor, node_a_reload);
+
+ nodes[0].node.test_process_background_events();
+ assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0), [funding_tx]);
+ let events = nodes[0].node.get_and_clear_pending_events();
+ assert_eq!(events.len(), if use_0conf { 2 } else { 1 });
+ assert!(matches!(events[0], Event::ChannelPending { .. }));
+ if use_0conf {
+ // We already have the peer's channel_ready, so the channel is ready immediately.
+ assert!(matches!(events[1], Event::ChannelReady { .. }));
+ }
+
+ nodes[1].node.peer_disconnected(node_a_id);
+ let mut reconnect_args = ReconnectArgs::new(&nodes[0], &nodes[1]);
+ reconnect_args.send_channel_ready = (use_0conf, use_0conf);
+ reconnect_nodes(reconnect_args);
+ if use_0conf {
+ expect_channel_ready_event(&nodes[1], &node_a_id);
+ assert_eq!(nodes[0].node.list_usable_channels().len(), 1);
+ assert_eq!(nodes[1].node.list_usable_channels().len(), 1);
+ }
+}
+
+#[test]
+fn test_outbound_reload_with_completed_init_mon() {
+ do_test_outbound_reload_with_completed_init_mon(true);
+ do_test_outbound_reload_with_completed_init_mon(false);
+}
+
+#[test]
+fn test_outbound_reload_funding_confirmed_during_downtime() {
+ // Test that an outbound channel whose funding transaction was broadcast before the node went
+ // down and confirmed to the required depth while it was down is resumed without hitting an
+ // assertion, even if the chain is synced before the pending background events are processed.
+ // Chain sync then holds channel_ready back on the still-paused channel, which the resume has
+ // to release.
+ let chanmon_cfgs = create_chanmon_cfgs(2);
+ let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
+
+ let persister;
+ let new_chain_monitor;
+
+ let mut chan_config = test_default_channel_config();
+ chan_config.channel_handshake_limits.trust_own_funding_0conf = true;
+
+ let node_chanmgrs =
+ create_node_chanmgrs(2, &node_cfgs, &[Some(chan_config.clone()), Some(chan_config)]);
+ let node_a_reload;
+
+ let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
+
+ let node_a_id = nodes[0].node.get_our_node_id();
+ let node_b_id = nodes[1].node.get_our_node_id();
+
+ nodes[0].node.create_channel(node_b_id, 100000, 10001, 43, None, None).unwrap();
+ nodes[1].node.handle_open_channel(
+ node_a_id,
+ &get_event_msg!(nodes[0], MessageSendEvent::SendOpenChannel, node_b_id),
+ );
+
+ let events = nodes[1].node.get_and_clear_pending_events();
+ assert_eq!(events.len(), 1);
+ match events[0] {
+ Event::OpenChannelRequest { temporary_channel_id: chan_id, .. } => {
+ nodes[1].node.accept_inbound_channel(&chan_id, &node_a_id, 0, None).unwrap();
+ },
+ _ => panic!("Unexpected event"),
+ };
+
+ nodes[0].node.handle_accept_channel(
+ node_b_id,
+ &get_event_msg!(nodes[1], MessageSendEvent::SendAcceptChannel, node_a_id),
+ );
+
+ let (temporary_channel_id, funding_tx, ..) =
+ create_funding_transaction(&nodes[0], &node_b_id, 100000, 43);
+
+ nodes[0]
+ .node
+ .funding_transaction_generated(temporary_channel_id, node_b_id, funding_tx.clone())
+ .unwrap();
+ check_added_monitors(&nodes[0], 0);
+
+ let funding_created_msg =
+ get_event_msg!(nodes[0], MessageSendEvent::SendFundingCreated, node_b_id);
+ nodes[1].node.handle_funding_created(node_a_id, &funding_created_msg);
+ check_added_monitors(&nodes[1], 1);
+ expect_channel_pending_event(&nodes[1], &node_a_id);
+
+ let chan_id = match &nodes[1].node.get_and_clear_pending_msg_events()[0] {
+ MessageSendEvent::SendFundingSigned { msg, .. } => {
+ chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::InProgress);
+
+ nodes[0].node.handle_funding_signed(node_b_id, &msg);
+ check_added_monitors(&nodes[0], 1);
+ msg.channel_id
+ },
+ _ => panic!("Unexpected event"),
+ };
+
+ assert!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().is_empty());
+ assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
+ assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
+
+ // nodes[0] is now waiting on the first ChannelMonitor persistence to complete in order to
+ // broadcast the funding transaction. Serializing the ChannelManager now is what a crash
+ // leaves on disk.
+ let node_a_ser = nodes[0].node.encode();
+
+ // The persistence then completes and the funding transaction is broadcast before the
+ // ChannelManager is ever written out.
+ chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
+ let (latest_update, _) = nodes[0].chain_monitor.get_latest_mon_update_id(chan_id);
+ nodes[0].chain_monitor.chain_monitor.force_channel_monitor_updated(chan_id, latest_update);
+ check_added_monitors(&nodes[0], 0);
+ expect_channel_pending_event(&nodes[0], &node_b_id);
+ assert_eq!(
+ nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0),
+ [funding_tx.clone()]
+ );
+
+ let mon_ser = get_monitor!(nodes[0], chan_id).encode();
+ reload_node!(nodes[0], &node_a_ser, &[&mon_ser], persister, new_chain_monitor, node_a_reload);
+
+ // The funding transaction confirms to the required depth while the node is down, and the
+ // chain is synced before the pending background events are processed. Note that we must not
+ // process the pending background events in between, so we connect the blocks by hand.
+ mine_transaction_without_consistency_checks(&nodes[0], &funding_tx);
+ let mut prev_blockhash = nodes[0].best_block_hash();
+ let mut height = nodes[0].best_block_info().1;
+ for _ in 0..CHAN_CONFIRM_DEPTH - 1 {
+ height += 1;
+ let header = create_dummy_header(prev_blockhash, height);
+ nodes[0].chain_monitor.chain_monitor.best_block_updated(&header, height);
+ nodes[0].chain_monitor.chain_monitor.transactions_confirmed(&header, &[], height);
+ nodes[0].node.best_block_updated(&header, height);
+ nodes[0].node.transactions_confirmed(&header, &[], height);
+ prev_blockhash = header.block_hash();
+ }
+
+ nodes[0].node.test_process_background_events();
+
+ // The channel is resumed, but as the peer is still disconnected the channel_ready it
+ // generated is held until the peers reconnect.
+ expect_channel_pending_event(&nodes[0], &node_b_id);
+ assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
+ assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
+
+ nodes[1].node.peer_disconnected(node_a_id);
+ let init_msg = msgs::Init {
+ features: nodes[1].node.init_features(),
+ networks: None,
+ remote_network_address: None,
+ };
+ nodes[0].node.peer_connected(node_b_id, &init_msg, true).unwrap();
+ let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
+ assert_eq!(reestablish_1.len(), 1);
+ nodes[1].node.peer_connected(node_a_id, &init_msg, false).unwrap();
+ let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
+ assert_eq!(reestablish_2.len(), 1);
+ nodes[1].node.handle_channel_reestablish(node_a_id, &reestablish_1[0]);
+ let resp_1 = handle_chan_reestablish_msgs!(nodes[1], nodes[0]);
+ assert!(resp_1.0.is_none() && resp_1.1.is_none() && resp_1.2.is_none());
+ nodes[0].node.handle_channel_reestablish(node_b_id, &reestablish_2[0]);
+ let resp_0 = handle_chan_reestablish_msgs!(nodes[0], nodes[1]);
+
+ // nodes[0] retransmits the channel_ready it generated when it resumed the channel.
+ let channel_ready = resp_0.0.expect("expected a retransmitted channel_ready");
+ nodes[1].node.handle_channel_ready(node_a_id, &channel_ready);
+ assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
+ assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
+
+ confirm_transaction(&nodes[1], &funding_tx);
+ let channel_ready_b = nodes[1]
+ .node
+ .get_and_clear_pending_msg_events()
+ .iter()
+ .find_map(|ev| match ev {
+ MessageSendEvent::SendChannelReady { node_id, msg } if node_id == &node_a_id => {
+ Some(msg.clone())
+ },
+ _ => None,
+ })
+ .unwrap();
+ nodes[0].node.handle_channel_ready(node_b_id, &channel_ready_b);
+ expect_channel_ready_event(&nodes[1], &node_a_id);
+ expect_channel_ready_event(&nodes[0], &node_b_id);
+ nodes[0].node.get_and_clear_pending_msg_events();
+ nodes[1].node.get_and_clear_pending_msg_events();
+ assert_eq!(nodes[0].node.list_usable_channels().len(), 1);
+ assert_eq!(nodes[1].node.list_usable_channels().len(), 1);
+}
+
fn do_test_inbound_reload_without_init_mon(use_0conf: bool, lock_commitment: bool) {
// Test that if the monitor update generated by funding_transaction_generated is stored async
// and we restart with the latest ChannelManager but the ChannelMonitor persistence never
### lightning/src/ln/channel.rs
@@ -10379,13 +10379,15 @@ where
// We will never broadcast the funding transaction when we're in MonitorUpdateInProgress
// (and we assume the user never directly broadcasts the funding transaction and waits for
- // us to do it). Thus, we can only ever hit monitor_pending_channel_ready when we're
+ // us to do it). Thus, we can hit monitor_pending_channel_ready when we're
// * an inbound channel that failed to persist the monitor on funding_created and we got
- // the funding transaction confirmed before the monitor was persisted, or
- // * a 0-conf channel and intended to send the channel_ready before any broadcast at all.
+ // the funding transaction confirmed before the monitor was persisted,
+ // * a 0-conf channel and intended to send the channel_ready before any broadcast at all, or
+ // * an outbound channel resumed from a `ChannelManager` that was serialized while the
+ // initial `ChannelMonitor` persistence was pending - a prior process broadcast the
+ // funding transaction after that serialization, and chain sync can have seen it confirm
+ // before the pending completion is processed here.
let channel_ready = if self.context.monitor_pending_channel_ready {
- assert!(!self.funding.is_outbound() || self.context.minimum_depth == Some(0),
- "Funding transaction broadcast by the local client before it should have - LDK didn't do it!");
self.context.monitor_pending_channel_ready = false;
let channel_ready = self.get_channel_ready(logger);
requires_channel_manager_persistence |= channel_ready.is_some();
### lightning/src/ln/channelmanager.rs
@@ -20736,6 +20736,31 @@ impl<
""
),
);
+ } else if funded_chan.is_awaiting_monitor_update()
+ && funded_chan.blocked_monitor_updates_pending() == 0
+ && monitor.get_latest_update_id()
+ == funded_chan.get_latest_unblocked_monitor_update_id()
+ {
+ // Neither the initial `ChannelMonitor` persistence nor blocked updates
+ // (which we dropped above if the `ChannelMonitor` already applied them)
+ // are tracked as in-flight updates. If the channel is still awaiting a
+ // monitor update, but has none in-flight or blocked, the persistence it
+ // was waiting on (whose result we have a `ChannelMonitor` for) completed
+ // after we were last serialized. Resume the channel the same way we do
+ // when all of its in-flight updates completed, otherwise it would never
+ // make progress (e.g. an outbound channel would never broadcast its
+ // funding transaction).
+ //
+ // The update id equality above implies the `ChannelMonitor` has applied
+ // every update the channel generated: a channel lagging its
+ // `ChannelMonitor` was force-closed above, and one ahead of it is
+ // rejected by the stale-`ChannelManager` check below.
+ log_debug!(logger, "A ChannelMonitor persistence completed since the ChannelManager was last serialized");
+ pending_background_events.push(BackgroundEvent::MonitorUpdatesComplete {
+ counterparty_node_id: *counterparty_id,
+ channel_id: *chan_id,
+ highest_update_id_completed: monitor.get_latest_update_id(),
+ });
}
if funded_chan.get_latest_unblocked_monitor_update_id()
> max_in_flight_update_id
### lightning/src/ln/monitor_tests.rs
@@ -3466,8 +3466,14 @@ fn test_event_replay_causing_monitor_replay() {
let serialized_monitor = get_monitor!(nodes[0], chan.2).encode();
reload_node!(nodes[0], &serialized_channel_manager, &[&serialized_monitor], persister, new_chain_monitor, node_deserialized);
- // Expect the `PaymentSent` to get replayed, this time without the duplicate monitor update
- expect_payment_sent(&nodes[0], payment_preimage, None, false, false /* expected post-event monitor update*/);
+ // Expect the `PaymentSent` to get replayed, this time without the duplicate monitor update.
+ // As the RAA `ChannelMonitorUpdate` already completed, the channel is resumed on startup,
+ // finalizing the claim.
+ expect_payment_sent(&nodes[0], payment_preimage, None, true, false /* expected post-event monitor update*/);
+
+ nodes[1].node.peer_disconnected(nodes[0].node.get_our_node_id());
+ reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
+ send_payment(&nodes[0], &[&nodes[1]], 100_000);
}
#[test]Why this scored 43/100
Community notes
Notes can correct, qualify, or add evidence to the AI analysis. Every note shown here has been validated by a human moderator.
The AI analysis stands alone for now. Submit a note if you can add evidence or important context.