Merge rust-bitcoin/rust-bitcoin#6941: consensus_encoding: do not drop encoders with an empty first chunk
What changed, and why it matters
This commit fixes a bug in the way the rust-bitcoin library chains together small data encoders. Previously, if an encoder's first output chunk was empty but it later produced real bytes, the library would silently throw the encoder away and lose those bytes. The fix makes the library advance each encoder past any initial empty chunks before deciding whether to keep it. This could change the bytes produced when encoding certain Bitcoin network messages or other composite data structures.
Review any code paths that build `IterEncoder` or composite encoders with components that may initially yield empty chunks, especially in consensus-critical serialization. Upgrade to the patched version and run the new regression tests. Consider whether the changed output could affect compatibility with peers or stored data.
Security signals we found
Data-loss bug in serialization path: encoders with empty first chunk were dropped, causing incorrect encoded output
Violation of documented `Encoder` contract: `current_chunk` may return empty slice, but `IterEncoder::new` assumed it meant the encoder was empty
Potential for consensus-relevant encoding mismatch if composite encoders are used for transaction or network message serialization
No explicit security advisory or CVE referenced in commit materials
Evidence from the diff
The patch corrects IterEncoder::new and the composite EncoderN macros in bitcoin-consensus-encoding. The old code dropped encoders whose current_chunk() was empty on construction, violating the documented contract that current_chunk may return an empty slice. The new code advances encoders until they either yield a non-empty chunk or report Finished, sharing the same skip_empty helper between construction and advance(). Tests are added for Encoder2/Encoder3/Encoder4/Encoder6 and IterEncoder with composite encoders that have empty leading components. A p2p round-trip test for an unknown empty addrv2 address is also added.
Changed components
consensus_encoding/src/encode/iter.rsconsensus_encoding/src/encode/encoders.rsconsensus_encoding/src/encode/mod.rsconsensus_encoding/tests/encode.rsconsensus_encoding/tests/iter_encoder.rsp2p/src/address.rsInspect captured patch +123 / −36
### consensus_encoding/src/encode/encoders.rs
@@ -224,33 +224,58 @@ macro_rules! define_encoder_n {
}
}
+ impl<$($enc_ty: Encoder,)*> $name<$($enc_ty,)*> {
+ #[inline]
+ fn skip_empty(&mut self) {
+ $(
+ if self.cur_idx == $enc_idx {
+ if self.$enc_field.current_chunk().is_empty() {
+ self.cur_idx += 1;
+ } else {
+ return;
+ }
+ }
+ )*
+ }
+ }
+
impl<$($enc_ty: Encoder,)*> Encoder for $name<$($enc_ty,)*> {
#[inline]
fn current_chunk(&self) -> &[u8] {
match self.cur_idx {
$($enc_idx => self.$enc_field.current_chunk(),)*
- _ => unreachable!("index never reaches this value"),
+ _ => &[],
}
}
#[inline]
fn advance(&mut self) -> EncoderStatus {
- match self.cur_idx {
- $(
- $enc_idx => {
- // For the last encoder, just pass through
- if $enc_idx == $idx_limit - 1 {
- return self.$enc_field.advance()
- }
- // For all others, return EncoderStatus::HasMore, or increment to next encoder
- if self.$enc_field.advance().has_finished() {
- self.cur_idx += 1;
- }
- EncoderStatus::HasMore
+ $(
+ if self.cur_idx == $enc_idx {
+ if self.$enc_field.current_chunk().is_empty() {
+ self.cur_idx += 1;
+ self.skip_empty();
+ return if self.cur_idx == $idx_limit {
+ EncoderStatus::Finished
+ } else {
+ EncoderStatus::HasMore
+ };
}
- )*
- _ => EncoderStatus::Finished,
- }
+ if self.$enc_field.advance().has_more()
+ && !self.$enc_field.current_chunk().is_empty()
+ {
+ return EncoderStatus::HasMore;
+ }
+ self.cur_idx += 1;
+ self.skip_empty();
+ return if self.cur_idx == $idx_limit {
+ EncoderStatus::Finished
+ } else {
+ EncoderStatus::HasMore
+ };
+ }
+ )*
+ EncoderStatus::Finished
}
}
### consensus_encoding/src/encode/iter.rs
@@ -65,18 +65,33 @@ where
pub fn new(iter: impl IntoIterator<IntoIter = I>) -> Self {
// Protect against poorly implemented iterators.
let mut iter = iter.into_iter().fuse();
- // Advance past any leading empty encoders so that the first call to
- // `current_chunk` satisfies the `Encoder` contract that it must return
- // non-empty bytes or the encoder must be `Done`.
- let state = loop {
- match iter.next() {
- Some(enc) if !enc.current_chunk().is_empty() =>
- break EncoderState::Encoding { current: enc, remaining: iter },
- Some(_) => {}
- None => break EncoderState::Done,
- }
+ let state = match iter.next() {
+ Some(enc) => EncoderState::Encoding { current: enc, remaining: iter },
+ None => EncoderState::Done,
};
- Self { state }
+ let mut this = Self { state };
+ this.skip_empty();
+ this
+ }
+
+ fn skip_empty(&mut self) {
+ let EncoderState::Encoding { current, remaining } = &mut self.state else {
+ return;
+ };
+
+ loop {
+ if !current.current_chunk().is_empty() {
+ return;
+ }
+ if current.advance().has_finished() {
+ if let Some(next) = remaining.next() {
+ *current = next;
+ } else {
+ self.state = EncoderState::Done;
+ return;
+ }
+ }
+ }
}
}
### consensus_encoding/src/encode/mod.rs
@@ -79,8 +79,9 @@ pub trait Encoder {
/// Yields the current encoded byte slice.
///
/// Will always return the same value until [`Self::advance`] is called.
- /// May return an empty slice, however implementors should avoid returning empty slices unless
- /// the encoded type is truly empty.
+ /// May return an empty slice, however implementors MUST NOT return an empty slice unless the
+ /// encoded type is truly empty. Once an empty slice is returned, callers may assume that no
+ /// more data will be yielded even after further calls to [`Self::advance`].
fn current_chunk(&self) -> &[u8];
/// Moves the encoder to its next state.
### consensus_encoding/tests/encode.rs
@@ -193,14 +193,37 @@ fn encode_slice_encoder_mixed_empty_and_data() {
}
#[test]
-fn encode_encoder2_with_first_empty_encoder() {
- // Test Encoder2 when first encoder produces no data.
- let enc1 = ArrayEncoder::<0>::without_length_prefix([]);
- let enc2 = ArrayEncoder::<3>::without_length_prefix([1, 2, 3]);
+fn composite_encoders_skip_empty_components() {
+ let mut encoder2 = Encoder2::new(
+ ArrayEncoder::<0>::without_length_prefix([]),
+ ArrayEncoder::without_length_prefix([1]),
+ );
+ check_encoder(&mut encoder2, &[1]);
- let mut encoder = Encoder2::new(enc1, enc2);
+ let mut encoder3 = Encoder3::new(
+ ArrayEncoder::<0>::without_length_prefix([]),
+ ArrayEncoder::without_length_prefix([2]),
+ ArrayEncoder::<0>::without_length_prefix([]),
+ );
+ check_encoder(&mut encoder3, &[2]);
- check_encoder(&mut encoder, &[1, 2, 3]);
+ let mut encoder4 = Encoder4::new(
+ ArrayEncoder::<0>::without_length_prefix([]),
+ ArrayEncoder::without_length_prefix([3]),
+ ArrayEncoder::<0>::without_length_prefix([]),
+ ArrayEncoder::without_length_prefix([4]),
+ );
+ check_encoder(&mut encoder4, &[3, 4]);
+
+ let mut encoder6 = Encoder6::new(
+ ArrayEncoder::<0>::without_length_prefix([]),
+ ArrayEncoder::without_length_prefix([5]),
+ ArrayEncoder::<0>::without_length_prefix([]),
+ ArrayEncoder::without_length_prefix([6]),
+ ArrayEncoder::<0>::without_length_prefix([]),
+ ArrayEncoder::without_length_prefix([7]),
+ );
+ check_encoder(&mut encoder6, &[5, 6, 7]);
}
#[test]
### consensus_encoding/tests/iter_encoder.rs
@@ -4,7 +4,7 @@
#![cfg(feature = "alloc")]
-use bitcoin_consensus_encoding::{drain_to_vec, BytesEncoder, IterEncoder};
+use bitcoin_consensus_encoding::{drain_to_vec, BytesEncoder, Encoder, Encoder2, IterEncoder};
/// Constructs a [`BytesEncoder`] from a static byte slice for use in tests.
fn enc(b: &'static [u8]) -> BytesEncoder<'static> { BytesEncoder::without_length_prefix(b) }
@@ -45,6 +45,12 @@ fn leading_empty_encoders_skipped() {
assert_eq!(encode([enc(&[]), enc(&[]), enc(&[1, 2, 3])]), [1, 2, 3]);
}
+#[test]
+fn leading_empty_encoders_skipped_on_construction() {
+ let encoder = IterEncoder::new([enc(&[]), enc(&[1, 2, 3])]);
+ assert_eq!(encoder.current_chunk(), [1, 2, 3]);
+}
+
#[test]
fn trailing_empty_encoders_skipped() {
assert_eq!(encode([enc(&[1, 2, 3]), enc(&[]), enc(&[])]), [1, 2, 3]);
@@ -54,3 +60,13 @@ fn trailing_empty_encoders_skipped() {
fn interleaved_empty_encoders_skipped() {
assert_eq!(encode([enc(&[1]), enc(&[]), enc(&[2]), enc(&[]), enc(&[3])]), [1, 2, 3]);
}
+
+#[test]
+fn composite_encoder_with_empty_leading_component_not_dropped() {
+ let encoders = [
+ Encoder2::new(enc(&[]), enc(&[1])),
+ Encoder2::new(enc(&[2]), enc(&[])),
+ Encoder2::new(enc(&[]), enc(&[3])),
+ ];
+ assert_eq!(encode(encoders), [1, 2, 3]);
+}
### p2p/src/address.rs
@@ -1358,6 +1358,13 @@ mod test {
assert_eq!(encoding::encode_to_vec(&addresses), raw);
}
+ #[test]
+ fn addrv2message_unknown_empty_address_roundtrip() {
+ let raw = hex!("010a0000000000000000");
+ let addresses: AddrV2Payload = encoding::decode_from_slice(&raw).unwrap();
+ assert_eq!(encoding::encode_to_vec(&addresses), raw);
+ }
+
#[test]
fn addrv2_to_ipaddr_ipv4() {
let addr = AddrV2::Ipv4(Ipv4Addr::new(192, 168, 1, 1));Why this scored 64/100
Community notes
Notes can correct, qualify, or add evidence to the AI analysis. Every note shown here has been validated by a human moderator.
The AI analysis stands alone for now. Submit a note if you can add evidence or important context.