test: move IPC helpers to ipc_util.py
What changed, and why it matters
This commit is a simple code cleanup in Bitcoin Core's test suite. It moves helper functions used by the IPC (inter-process communication) tests from one file into a new shared utility file and renames some functions for clarity. No production code, consensus rules, or wallet/security logic is changed.
No security action needed; this is a benign test-only refactor.
Security signals we found
No strong security signals were identified.
Evidence from the diff
The diff refactors test/functional/interface_ipc.py by extracting IPC test helpers into a new module test/functional/test_framework/ipc_util.py. The helpers (destroying, wait_and_do, load_capnp_modules, make_capnp_init_ctx, create_block_template, wait_next_template, parse_and_deserialize_block, get_coinbase_raw_tx, parse_and_deserialize_coinbase) are moved verbatim with only renaming for consistency (mining_* prefix). Imports are adjusted and a dataclass (CoinbaseTxData) is relocated. Logic and behavior remain identical.
Changed components
test/functional/interface_ipc.pytest/functional/test_framework/ipc_util.pyInspect captured patch +182 / −146
diff --git a/test/functional/interface_ipc.py b/test/functional/interface_ipc.py
index 531a6789..b3426360 100755
--- a/test/functional/interface_ipc.py
+++ b/test/functional/interface_ipc.py
@@ -4,15 +4,10 @@
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test the IPC (multiprocess) interface."""
import asyncio
-import inspect
-from contextlib import asynccontextmanager, AsyncExitStack
-from dataclasses import dataclass
+from contextlib import AsyncExitStack
from io import BytesIO
-from pathlib import Path
-import shutil
from test_framework.blocktools import NULL_OUTPOINT
from test_framework.messages import (
- CBlock,
CTransaction,
CTxIn,
CTxOut,
@@ -30,18 +25,17 @@ from test_framework.util import (
assert_not_equal,
)
from test_framework.wallet import MiniWallet
-from typing import Optional
-
-# Stores the result of getCoinbaseTx()
-@dataclass
-class CoinbaseTxData:
- version: int
- sequence: int
- scriptSigPrefix: bytes
- witness: Optional[bytes]
- blockRewardRemaining: int
- requiredOutputs: list[bytes]
- lockTime: int
+from test_framework.ipc_util import (
+ destroying,
+ mining_create_block_template,
+ load_capnp_modules,
+ make_capnp_init_ctx,
+ mining_get_block,
+ mining_get_coinbase_tx,
+ mining_get_coinbase_raw_tx,
+ mining_wait_next_template,
+ wait_and_do,
+)
# Test may be skipped and not have capnp installed
try:
@@ -49,44 +43,6 @@ try:
except ModuleNotFoundError:
pass
-@asynccontextmanager
-async def destroying(obj, ctx):
- """Call obj.destroy(ctx) at end of with: block. Similar to contextlib.closing."""
- try:
- yield obj
- finally:
- await obj.destroy(ctx)
-
-async def create_block_template(mining, stack, ctx, opts):
- """Call mining.createNewBlock() and return template, then call template.destroy() when stack exits."""
- return await stack.enter_async_context(destroying((await mining.createNewBlock(opts)).result, ctx))
-
-async def wait_next_template(template, stack, ctx, opts):
- """Call template.waitNext() and return template, then call template.destroy() when stack exits."""
- return await stack.enter_async_context(destroying((await template.waitNext(ctx, opts)).result, ctx))
-
-async def wait_and_do(wait_fn, do_fn):
- """Call wait_fn, then sleep, then call do_fn in a parallel task. Wait for
- both tasks to complete."""
- wait_started = asyncio.Event()
- result = None
-
- async def wait():
- nonlocal result
- wait_started.set()
- result = await wait_fn
-
- async def do():
- await wait_started.wait()
- await asyncio.sleep(0.1)
- # Let do_fn be either a callable or an awaitable object
- if inspect.isawaitable(do_fn):
- await do_fn
- else:
- do_fn()
-
- await asyncio.gather(wait(), do())
- return result
class IPCInterfaceTest(BitcoinTestFramework):
@@ -94,30 +50,6 @@ class IPCInterfaceTest(BitcoinTestFramework):
self.skip_if_no_ipc()
self.skip_if_no_py_capnp()
- def load_capnp_modules(self):
- if capnp_bin := shutil.which("capnp"):
- # Add the system cap'nproto path so include/capnp/c++.capnp can be found.
- capnp_dir = Path(capnp_bin).resolve().parent.parent / "include"
- else:
- # If there is no system cap'nproto, the pycapnp module should have its own "bundled"
- # includes at this location. If pycapnp was installed with bundled capnp,
- # capnp/c++.capnp can be found here.
- capnp_dir = Path(capnp.__path__[0]).parent
- src_dir = Path(self.config['environment']['SRCDIR']) / "src"
- mp_dir = src_dir / "ipc" / "libmultiprocess" / "include"
- # List of import directories. Note: it is important for mp_dir to be
- # listed first, in case there are other libmultiprocess installations on
- # the system, to ensure that `import "/mp/proxy.capnp"` lines load the
- # same file as capnp.load() loads directly below, and there are not
- # "failed: Duplicate ID @0xcc316e3f71a040fb" errors.
- imports = [str(mp_dir), str(capnp_dir), str(src_dir)]
- return {
- "proxy": capnp.load(str(mp_dir / "mp" / "proxy.capnp"), imports=imports),
- "init": capnp.load(str(src_dir / "ipc" / "capnp" / "init.capnp"), imports=imports),
- "echo": capnp.load(str(src_dir / "ipc" / "capnp" / "echo.capnp"), imports=imports),
- "mining": capnp.load(str(src_dir / "ipc" / "capnp" / "mining.capnp"), imports=imports),
- }
-
def set_test_params(self):
self.num_nodes = 2
@@ -126,59 +58,12 @@ class IPCInterfaceTest(BitcoinTestFramework):
super().setup_nodes()
# Use this function to also load the capnp modules (we cannot use set_test_params for this,
# as it is being called before knowing whether capnp is available).
- self.capnp_modules = self.load_capnp_modules()
-
- async def make_capnp_init_ctx(self):
- node = self.nodes[0]
- # Establish a connection, and create Init proxy object.
- connection = await capnp.AsyncIoStream.create_unix_connection(node.ipc_socket_path)
- client = capnp.TwoPartyClient(connection)
- init = client.bootstrap().cast_as(self.capnp_modules['init'].Init)
- # Create a remote thread on the server for the IPC calls to be executed in.
- threadmap = init.construct().threadMap
- thread = threadmap.makeThread("pythread").result
- ctx = self.capnp_modules['proxy'].Context()
- ctx.thread = thread
- # Return both.
- return ctx, init
-
- async def parse_and_deserialize_block(self, block_template, ctx):
- block_data = BytesIO((await block_template.getBlock(ctx)).result)
- block = CBlock()
- block.deserialize(block_data)
- return block
-
- async def get_coinbase_raw_tx(self, block_template, ctx):
- assert block_template is not None
- coinbase_data = BytesIO((await block_template.getCoinbaseRawTx(ctx)).result)
- tx = CTransaction()
- tx.deserialize(coinbase_data)
- return tx
-
- async def parse_and_deserialize_coinbase(self, block_template, ctx) -> CoinbaseTxData:
- assert block_template is not None
- # Note: the template_capnp struct will be garbage-collected when this
- # method returns, so it is important to copy any Data fields from it
- # which need to be accessed later using the bytes() cast. Starting with
- # pycapnp v2.2.0, Data fields have type `memoryview` and are ephemeral.
- template_capnp = (await block_template.getCoinbaseTx(ctx)).result
- witness: Optional[bytes] = None
- if template_capnp._has("witness"):
- witness = bytes(template_capnp.witness)
- return CoinbaseTxData(
- version=int(template_capnp.version),
- sequence=int(template_capnp.sequence),
- scriptSigPrefix=bytes(template_capnp.scriptSigPrefix),
- witness=witness,
- blockRewardRemaining=int(template_capnp.blockRewardRemaining),
- requiredOutputs=[bytes(output) for output in template_capnp.requiredOutputs],
- lockTime=int(template_capnp.lockTime),
- )
+ self.capnp_modules = load_capnp_modules(self.config)
def run_echo_test(self):
self.log.info("Running echo test")
async def async_routine():
- ctx, init = await self.make_capnp_init_ctx()
+ ctx, init = await make_capnp_init_ctx(self)
self.log.debug("Create Echo proxy object")
echo = init.makeEcho(ctx).result
self.log.debug("Test a few invocations of echo")
@@ -192,7 +77,7 @@ class IPCInterfaceTest(BitcoinTestFramework):
async def build_coinbase_test(self, template, ctx, miniwallet):
self.log.debug("Build coinbase transaction using getCoinbaseTx()")
assert template is not None
- coinbase_res = await self.parse_and_deserialize_coinbase(template, ctx)
+ coinbase_res = await mining_get_coinbase_tx(template, ctx)
coinbase_tx = CTransaction()
coinbase_tx.version = coinbase_res.version
coinbase_tx.vin = [CTxIn()]
@@ -233,10 +118,9 @@ class IPCInterfaceTest(BitcoinTestFramework):
assert_equal(has_witness, found_witness_op_return)
coinbase_tx.nLockTime = coinbase_res.lockTime
-
# Compare to dummy coinbase transaction provided by the deprecated
# getCoinbaseRawTx()
- coinbase_legacy = await self.get_coinbase_raw_tx(template, ctx)
+ coinbase_legacy = await mining_get_coinbase_raw_tx(template, ctx)
assert_equal(coinbase_legacy.vout[0].nValue, coinbase_res.blockRewardRemaining)
# Swap dummy output for our own
coinbase_legacy.vout[0].scriptPubKey = coinbase_tx.vout[0].scriptPubKey
@@ -252,7 +136,7 @@ class IPCInterfaceTest(BitcoinTestFramework):
miniwallet = MiniWallet(self.nodes[0])
async def async_routine():
- ctx, init = await self.make_capnp_init_ctx()
+ ctx, init = await make_capnp_init_ctx(self)
self.log.debug("Create Mining proxy object")
mining = init.makeMining(ctx).result
self.log.debug("Test simple inspectors")
@@ -281,12 +165,12 @@ class IPCInterfaceTest(BitcoinTestFramework):
opts.useMempool = True
opts.blockReservedWeight = 4000
opts.coinbaseOutputMaxAdditionalSigops = 0
- template = await create_block_template(mining, stack, ctx, opts)
+ template = await mining_create_block_template(mining, stack, ctx, opts)
self.log.debug("Test some inspectors of Template")
header = (await template.getBlockHeader(ctx)).result
assert_equal(len(header), block_header_size)
- block = await self.parse_and_deserialize_block(template, ctx)
+ block = await mining_get_block(template, ctx)
assert_equal(ser_uint256(block.hashPrevBlock), newblockref.hash)
assert len(block.vtx) >= 1
txfees = await template.getTxFees(ctx)
@@ -299,9 +183,9 @@ class IPCInterfaceTest(BitcoinTestFramework):
waitoptions.timeout = timeout
waitoptions.feeThreshold = 1
template2 = await wait_and_do(
- wait_next_template(template, stack, ctx, waitoptions),
+ mining_wait_next_template(template, stack, ctx, waitoptions),
lambda: self.generate(self.nodes[0], 1))
- block2 = await self.parse_and_deserialize_block(template2, ctx)
+ block2 = await mining_get_block(template2, ctx)
assert_equal(len(block2.vtx), 1)
self.log.debug("Wait for another, but time out")
@@ -310,23 +194,23 @@ class IPCInterfaceTest(BitcoinTestFramework):
self.log.debug("Wait for another, get one after increase in fees in the mempool")
template4 = await wait_and_do(
- wait_next_template(template2, stack, ctx, waitoptions),
+ mining_wait_next_template(template2, stack, ctx, waitoptions),
lambda: miniwallet.send_self_transfer(fee_rate=10, from_node=self.nodes[0]))
- block3 = await self.parse_and_deserialize_block(template4, ctx)
+ block3 = await mining_get_block(template4, ctx)
assert_equal(len(block3.vtx), 2)
self.log.debug("Wait again, this should return the same template, since the fee threshold is zero")
waitoptions.feeThreshold = 0
- template5 = await wait_next_template(template4, stack, ctx, waitoptions)
- block4 = await self.parse_and_deserialize_block(template5, ctx)
+ template5 = await mining_wait_next_template(template4, stack, ctx, waitoptions)
+ block4 = await mining_get_block(template5, ctx)
assert_equal(len(block4.vtx), 2)
waitoptions.feeThreshold = 1
self.log.debug("Wait for another, get one after increase in fees in the mempool")
template6 = await wait_and_do(
- wait_next_template(template5, stack, ctx, waitoptions),
+ mining_wait_next_template(template5, stack, ctx, waitoptions),
lambda: miniwallet.send_self_transfer(fee_rate=10, from_node=self.nodes[0]))
- block4 = await self.parse_and_deserialize_block(template6, ctx)
+ block4 = await mining_get_block(template6, ctx)
assert_equal(len(block4.vtx), 3)
self.log.debug("Wait for another, but time out, since the fee threshold is set now")
@@ -345,7 +229,7 @@ class IPCInterfaceTest(BitcoinTestFramework):
current_block_height = self.nodes[0].getchaintips()[0]["height"]
check_opts = self.capnp_modules['mining'].BlockCheckOptions()
async with destroying((await mining.createNewBlock(opts)).result, ctx) as template:
- block = await self.parse_and_deserialize_block(template, ctx)
+ block = await mining_get_block(template, ctx)
balance = miniwallet.get_balance()
coinbase = await self.build_coinbase_test(template, ctx, miniwallet)
# Reduce payout for balance comparison simplicity
@@ -370,7 +254,7 @@ class IPCInterfaceTest(BitcoinTestFramework):
assert_equal(block_valid, True)
# The remote template block will be mutated, capture the original:
- remote_block_before = await self.parse_and_deserialize_block(template, ctx)
+ remote_block_before = await mining_get_block(template, ctx)
self.log.debug("Submitted coinbase must include witness")
assert_not_equal(coinbase.serialize_without_witness().hex(), coinbase.serialize().hex())
@@ -380,7 +264,7 @@ class IPCInterfaceTest(BitcoinTestFramework):
self.log.debug("Even a rejected submitBlock() mutates the template's block")
# Can be used by clients to download and inspect the (rejected)
# reconstructed block.
- remote_block_after = await self.parse_and_deserialize_block(template, ctx)
+ remote_block_after = await mining_get_block(template, ctx)
assert_not_equal(remote_block_before.serialize().hex(), remote_block_after.serialize().hex())
self.log.debug("Submit again, with the witness")
diff --git a/test/functional/test_framework/ipc_util.py b/test/functional/test_framework/ipc_util.py
new file mode 100644
index 00000000..fc451faa
--- /dev/null
+++ b/test/functional/test_framework/ipc_util.py
@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+# Copyright (c) The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Shared utilities for IPC (multiprocess) interface tests."""
+import asyncio
+import inspect
+from contextlib import asynccontextmanager
+from dataclasses import dataclass
+from io import BytesIO
+from pathlib import Path
+import shutil
+from typing import Optional
+
+from test_framework.messages import CBlock, CTransaction
+
+# Test may be skipped and not have capnp installed
+try:
+ import capnp # type: ignore[import] # noqa: F401
+except ModuleNotFoundError:
+ pass
+
+
+# Stores the result of getCoinbaseTx()
+@dataclass
+class CoinbaseTxData:
+ version: int
+ sequence: int
+ scriptSigPrefix: bytes
+ witness: Optional[bytes]
+ blockRewardRemaining: int
+ requiredOutputs: list[bytes]
+ lockTime: int
+
+
+@asynccontextmanager
+async def destroying(obj, ctx):
+ """Call obj.destroy(ctx) at end of with: block. Similar to contextlib.closing."""
+ try:
+ yield obj
+ finally:
+ await obj.destroy(ctx)
+
+
+async def wait_and_do(wait_fn, do_fn):
+ """Call wait_fn, then sleep, then call do_fn in a parallel task. Wait for
+ both tasks to complete."""
+ wait_started = asyncio.Event()
+ result = None
+
+ async def wait():
+ nonlocal result
+ wait_started.set()
+ result = await wait_fn
+
+ async def do():
+ await wait_started.wait()
+ await asyncio.sleep(0.1)
+ # Let do_fn be either a callable or an awaitable object
+ if inspect.isawaitable(do_fn):
+ await do_fn
+ else:
+ do_fn()
+
+ await asyncio.gather(wait(), do())
+ return result
+
+
+def load_capnp_modules(config):
+ if capnp_bin := shutil.which("capnp"):
+ # Add the system cap'nproto path so include/capnp/c++.capnp can be found.
+ capnp_dir = Path(capnp_bin).resolve().parent.parent / "include"
+ else:
+ # If there is no system cap'nproto, the pycapnp module should have its own "bundled"
+ # includes at this location. If pycapnp was installed with bundled capnp,
+ # capnp/c++.capnp can be found here.
+ capnp_dir = Path(capnp.__path__[0]).parent
+ src_dir = Path(config['environment']['SRCDIR']) / "src"
+ mp_dir = src_dir / "ipc" / "libmultiprocess" / "include"
+ # List of import directories. Note: it is important for mp_dir to be
+ # listed first, in case there are other libmultiprocess installations on
+ # the system, to ensure that `import "/mp/proxy.capnp"` lines load the
+ # same file as capnp.load() loads directly below, and there are not
+ # "failed: Duplicate ID @0xcc316e3f71a040fb" errors.
+ imports = [str(mp_dir), str(capnp_dir), str(src_dir)]
+ return {
+ "proxy": capnp.load(str(mp_dir / "mp" / "proxy.capnp"), imports=imports),
+ "init": capnp.load(str(src_dir / "ipc" / "capnp" / "init.capnp"), imports=imports),
+ "echo": capnp.load(str(src_dir / "ipc" / "capnp" / "echo.capnp"), imports=imports),
+ "mining": capnp.load(str(src_dir / "ipc" / "capnp" / "mining.capnp"), imports=imports),
+ }
+
+
+async def make_capnp_init_ctx(self):
+ node = self.nodes[0]
+ # Establish a connection, and create Init proxy object.
+ connection = await capnp.AsyncIoStream.create_unix_connection(node.ipc_socket_path)
+ client = capnp.TwoPartyClient(connection)
+ init = client.bootstrap().cast_as(self.capnp_modules['init'].Init)
+ # Create a remote thread on the server for the IPC calls to be executed in.
+ threadmap = init.construct().threadMap
+ thread = threadmap.makeThread("pythread").result
+ ctx = self.capnp_modules['proxy'].Context()
+ ctx.thread = thread
+ # Return both.
+ return ctx, init
+
+
+async def mining_create_block_template(mining, stack, ctx, opts):
+ """Call mining.createNewBlock() and return template, then call template.destroy() when stack exits."""
+ return await stack.enter_async_context(destroying((await mining.createNewBlock(opts)).result, ctx))
+
+
+async def mining_wait_next_template(template, stack, ctx, opts):
+ """Call template.waitNext() and return template, then call template.destroy() when stack exits."""
+ return await stack.enter_async_context(destroying((await template.waitNext(ctx, opts)).result, ctx))
+
+
+async def mining_get_block(block_template, ctx):
+ block_data = BytesIO((await block_template.getBlock(ctx)).result)
+ block = CBlock()
+ block.deserialize(block_data)
+ return block
+
+
+async def mining_get_coinbase_raw_tx(block_template, ctx):
+ assert block_template is not None
+ coinbase_data = BytesIO((await block_template.getCoinbaseRawTx(ctx)).result)
+ tx = CTransaction()
+ tx.deserialize(coinbase_data)
+ return tx
+
+
+async def mining_get_coinbase_tx(block_template, ctx) -> CoinbaseTxData:
+ assert block_template is not None
+ # Note: the template_capnp struct will be garbage-collected when this
+ # method returns, so it is important to copy any Data fields from it
+ # which need to be accessed later using the bytes() cast. Starting with
+ # pycapnp v2.2.0, Data fields have type `memoryview` and are ephemeral.
+ template_capnp = (await block_template.getCoinbaseTx(ctx)).result
+ witness: Optional[bytes] = None
+ if template_capnp._has("witness"):
+ witness = bytes(template_capnp.witness)
+ return CoinbaseTxData(
+ version=int(template_capnp.version),
+ sequence=int(template_capnp.sequence),
+ scriptSigPrefix=bytes(template_capnp.scriptSigPrefix),
+ witness=witness,
+ blockRewardRemaining=int(template_capnp.blockRewardRemaining),
+ requiredOutputs=[bytes(output) for output in template_capnp.requiredOutputs],
+ lockTime=int(template_capnp.lockTime),
+ )
Why this scored 15/100
Community notes
Notes can correct, qualify, or add evidence to the AI analysis. Every note shown here has been validated by a human moderator.
The AI analysis stands alone for now. Submit a note if you can add evidence or important context.