testing: stabilize simulator multiprocess runs
What changed, and why it matters
This commit only changes test scripts and test infrastructure for the COLDCARD firmware simulator. It adjusts timeouts, adds helper functions for waiting on screen text, fixes test flakiness, and tweaks how tests are split across parallel jobs. There are no changes to the actual firmware code that runs on the device, so this cannot affect real users or introduce a security vulnerability in the product itself.
No security action required. Treat as routine test-maintenance commit.
Security signals we found
No strong security signals were identified.
Evidence from the diff
The diff is confined to the testing/ directory. Changes include: adding a BITCOIND_RPC_TIMEOUT constant and applying it to AuthServiceProxy calls; increasing the default need_keypress timeout; adding a wait_for_story pytest fixture; reordering/prioritizing simulator test queue entries; setting CKCC_DEFAULT_TIMEOUT=10000 in the subprocess environment; adjusting screen-scroll counts and timing sleeps in UI tests; updating backup restore test code to account for simulator-specific PA_ZERO_SECRET behavior; and changing a PSBT unknown key from b'\xfcsize-check' to b'\xfdsize-check' in a test. None of these alter firmware runtime behavior or attack surface.
Changed components
testing/api.pytesting/conftest.pytesting/run_sim_tests.pytesting/test_bip39pw.pytesting/test_multisig.pytesting/test_se2.pytesting/test_sign.pytesting/test_unit.pytesting/test_ux.pytesting/test_vdisk.pyInspect captured patch +84 / −33
### testing/api.py
@@ -7,6 +7,8 @@
from helpers import xfp2str
from ckcc.protocol import CCProtocolPacker
+BITCOIND_RPC_TIMEOUT = 60
+
def find_bitcoind():
# search for the binary we need
@@ -84,7 +86,7 @@ def get_free_port():
with open(cookie_path) as f:
self.userpass = f.readline().lstrip().rstrip()
self.rpc_url = f"http://{self.userpass}@127.0.0.1:{self.rpc_port}"
- self.rpc = AuthServiceProxy(self.rpc_url)
+ self.rpc = AuthServiceProxy(self.rpc_url, timeout=BITCOIND_RPC_TIMEOUT)
# Wait for bitcoind to be ready
ready = False
@@ -113,7 +115,7 @@ def get_free_port():
def get_wallet_rpc(self, wallet):
url = self.rpc_url + f"/wallet/{wallet}"
- return AuthServiceProxy(url)
+ return AuthServiceProxy(url, timeout=BITCOIND_RPC_TIMEOUT)
def create_wallet(self, wallet_name: str, disable_private_keys: bool = False, blank: bool = False,
passphrase: str = None, avoid_reuse: bool = False, descriptors: bool = True,
### testing/conftest.py
@@ -141,7 +141,7 @@ def X(is_q1):
@pytest.fixture
def need_keypress(dev, request):
- def doit(k, timeout=1000):
+ def doit(k, timeout=3000):
if request.config.getoption("--manual"):
# need actual user interaction
print("==> NOW, on the Coldcard, press key: %r (then enter here)" % k, file=sys.stderr)
@@ -428,6 +428,19 @@ def cap_story(dev):
return f
+@pytest.fixture
+def wait_for_story(cap_story):
+ def doit(expected, check_title=False):
+ for _ in range(50):
+ title, story = cap_story()
+ if expected in (title if check_title else story):
+ return title, story
+ time.sleep(0.2)
+ pytest.fail(f'Timed out waiting for: {expected!r}')
+
+ return doit
+
+
@pytest.fixture
def cap_image(request, sim_exec, is_q1, is_headless, sim_root_dir):
### testing/run_sim_tests.py
@@ -401,20 +401,21 @@ def add_to_queue(module_name, simulator_args, queue):
if module_name == "test_multisig.py":
# split takes too much time
queue.append((0, [module_name, simulator_args, "not tutorial and not airgapped and not ms_address and not descriptor_export", ""]))
- queue.append((0, [module_name, simulator_args, "airgapped", "-sep1"]))
+ queue.append((2, [module_name, simulator_args, "airgapped", "-sep1"]))
queue.append((0, [module_name, simulator_args, "tutorial", "-sep2"]))
- queue.append((0, [module_name, simulator_args, "ms_address", "-sep3"]))
- queue.append((0, [module_name, simulator_args, "descriptor_export", "-sep4"]))
+ queue.append((1, [module_name, simulator_args, "ms_address", "-sep3"]))
+ queue.append((2, [module_name, simulator_args, "descriptor_export", "-sep4"]))
elif module_name == "test_seed_xor.py":
# split takes too much time
- queue.append((0, [module_name, simulator_args, "test_import_xor", "-sep1"]))
- queue.append((0, [module_name, simulator_args, "not test_import_xor", ""]))
+ queue.append((1, [module_name, simulator_args, "test_import_xor", "-sep1"]))
+ queue.append((2, [module_name, simulator_args, "not test_import_xor", ""]))
elif module_name in ["test_export.py", "test_ephemeral.py", "test_sign.py", "test_msg.py",
- "test_backup.py"]:
+ "test_backup.py", "test_bsms.py"]:
# higher priority
- queue.append((1, [module_name, simulator_args, None, ""]))
+ queue.append((1 if module_name == "test_export.py" else 0,
+ [module_name, simulator_args, None, ""]))
else:
# standard priority
@@ -501,7 +502,9 @@ def add_to_queue(module_name, simulator_args, queue):
cmd_list.append("--psbt2")
if k:
cmd_list.extend(["-k", k])
- p = subprocess.Popen(cmd_list, preexec_fn=os.setsid, stdout=out_fd, stderr=out_fd)
+ env = dict(os.environ, CKCC_DEFAULT_TIMEOUT="10000")
+ p = subprocess.Popen(cmd_list, preexec_fn=os.setsid, stdout=out_fd,
+ stderr=out_fd, env=env)
if "q1" in log_dir:
mark = "Q"
elif "mk5" in log_dir:
### testing/test_bip39pw.py
@@ -324,20 +324,28 @@ def test_bip39_complex(target, pick_menu_item, cap_story, goto_home,
expect = BIP32Node.from_master_secret(seed, netcode="XTN")
enter_complex(target, apply=True)
- scroll_down = press_down if is_q1 else press_right
for _ in range(3):
screen = cap_screen()
- if 'Scroll down to' in screen:
+ if 'Above is the' in screen:
break
time.sleep(.01)
else:
pytest.fail('passphrase scroll notice not shown')
assert 'Passphrase:' not in screen
+ if is_q1:
+ # bigger display
+ assert "Scroll down to view" in screen
+ n = 1
+ else:
+ # more scrolling is needed for Mk
+ n = 14
+
+ for _ in range(n):
+ press_down()
+ time.sleep(.1)
- scroll_down()
- time.sleep(.1)
assert 'Passphrase:' in cap_screen()
press_select()
### testing/test_multisig.py
@@ -1908,7 +1908,7 @@ def test_make_airgapped(addr_fmt, acct_num, M_N, goto_home, cap_story, pick_menu
def test_reject_oversized_airgapped_xpub_qr(goto_home, pick_menu_item, need_keypress,
press_select, scan_a_qr, cap_screen,
- clear_ms, is_q1):
+ clear_ms, is_q1, wait_for_story):
if not is_q1:
pytest.skip("needs scanner")
@@ -1917,8 +1917,13 @@ def test_reject_oversized_airgapped_xpub_qr(goto_home, pick_menu_item, need_keyp
pick_menu_item('Settings')
pick_menu_item('Multisig Wallets')
pick_menu_item('Create Airgapped')
+
+ title, story = wait_for_story('QR or SD Card?', check_title=True)
+ assert 'XPUBs from QR codes' in story
need_keypress(KEY_QR)
- time.sleep(.1)
+
+ title, story = wait_for_story('Address Format', check_title=True)
+ assert 'default address format' in story
press_select()
oversized = json.dumps({'junk': 'x' * 1100})
### testing/test_se2.py
@@ -789,7 +789,7 @@ def test_ux_changing_pins(true_pin, repl, force_main_pin, goto_trick_menu,
clear_all_tricks()
def test_se2_trick_backups(goto_trick_menu, clear_all_tricks, repl, unit_test,
- new_trick_pin, new_pin_confirmed, pick_menu_item, press_select):
+ new_trick_pin, new_pin_confirmed, pick_menu_item, press_select, clear_ms):
def decode_backup(txt):
import json
vals = dict()
@@ -810,6 +810,7 @@ def decode_backup(txt):
return vals, trimmed
+ clear_ms()
clear_all_tricks()
# - make wallets of all duress types (x2 each)
@@ -847,9 +848,15 @@ def decode_backup(txt):
assert 'duress_1002_words' in trimmed
assert 'duress_1003_words' in trimmed
+ clear_all_tricks()
unit_test('devtest/clear_seed.py')
-
- repl.exec(f'import backups; backups.restore_from_dict_ll({vals!r})')
+
+ # Real bootrom clears PA_ZERO_SECRET when pa.change() installs the restored
+ # secret, but sim_secel does not update that response flag.
+ repl.exec(f'import backups; from pincodes import PA_ZERO_SECRET; '
+ f'pa.state_flags &= ~PA_ZERO_SECRET; d={vals!r}; '
+ 'raw,_=backups.extract_raw_secret(d); '
+ 'backups.restore_from_dict_ll(d, raw)')
# recover from recovery
repl.exec(f'import backups; pa.setup(pa.pin); pa.login(); from actions import goto_top_menu; goto_top_menu()')
@@ -866,6 +873,10 @@ def decode_backup(txt):
if 'setting.vidsk' in vals and vals['setting.vidsk']:
vals['setting.vidsk'] = 0 # restoring from backup always set VDisk to default OFF
+ tp1 = {pin: spec[1:] for pin, spec in vals.pop('setting.tp').items()}
+ tp2 = {pin: spec[1:] for pin, spec in vals2.pop('setting.tp').items()}
+ assert tp1 == tp2
+
assert vals == vals2
assert trimmed == tr2
### testing/test_sign.py
@@ -3653,8 +3653,8 @@ def test_txout_explorer_qr_too_big_single_item(fake_txn, start_sign, cap_story,
scr = cap_screen()
assert "QR too big" in scr
- press_cancel()
- press_cancel()
+ for _ in range(4):
+ press_cancel()
def test_low_R_grinding(dev, goto_home, microsd_path, press_select, offer_ms_import,
### testing/test_unit.py
@@ -137,9 +137,11 @@ def test_addr_decode(unit_test):
# - runs som known examples thru CTxIn and check it categories, and extracts pubkey/pkh right
unit_test('devtest/unit_addrs.py')
-def test_clear_seed(unit_test):
- # just testing the test?
- unit_test('devtest/clear_seed.py')
+def test_clear_seed(unit_test, reset_seed_words):
+ try:
+ unit_test('devtest/clear_seed.py')
+ finally:
+ reset_seed_words()
def test_slip132(unit_test):
# slip132 ?pub stuff
### testing/test_ux.py
@@ -49,8 +49,9 @@ def test_get_secrets(get_secrets, master_xpub):
assert v['xpub'] == master_xpub
def test_home_menu(cap_menu, cap_story, cap_screen, need_keypress, reset_seed_words,
- press_select, press_cancel, press_down, is_q1):
+ press_select, press_cancel, press_down, is_q1, microsd_wipe):
reset_seed_words()
+ microsd_wipe()
# get to top, force a redraw
press_cancel()
press_cancel()
@@ -90,11 +91,12 @@ def test_home_menu(cap_menu, cap_story, cap_screen, need_keypress, reset_seed_wo
need_keypress('0')
press_select()
- time.sleep(.01) # required
+ time.sleep(.1) # required
title, body = cap_story()
assert title == 'NO-TITLE'
- assert 'transactions' in body or 'Choose PSBT' in body, body
+ assert ('transactions' in body or 'Choose PSBT' in body
+ or 'filename must end in psbt' in body), body
press_cancel()
@@ -474,7 +476,7 @@ def finish_entropy():
@pytest.mark.parametrize('nwords', [12, 24])
def test_view_trng_words_verifies_dice_mix(nwords, pick_menu_item, cap_menu, cap_story, unit_test,
press_select, need_keypress, seed_story_to_words, is_q1,
- cap_screen, press_down):
+ cap_screen, press_down, wait_for_story):
unit_test('devtest/clear_seed.py')
pick_menu_item('New Seed Words')
pick_menu_item(f'{nwords} Words')
@@ -505,15 +507,17 @@ def test_view_trng_words_verifies_dice_mix(nwords, pick_menu_item, cap_menu, cap
pick_menu_item('Dice Rolls')
press_select()
+ time.sleep(.5)
rolls = ('123456' * 8) + '12'
for ch in rolls:
need_keypress(ch)
+ time.sleep(0.01)
time.sleep(0.1)
done_key = KEY_ENTER if is_q1 else 'y'
need_keypress(done_key)
time.sleep(0.1)
- _, body = cap_story()
+ _, body = wait_for_story(f'Record these {nwords} secret words!', check_title=is_q1)
words = seed_story_to_words(body) if is_q1 else \
[w[3:].strip() for w in body.split('\n') if w and w[2] == ':']
@@ -531,7 +535,8 @@ def test_view_trng_words_verifies_dice_mix(nwords, pick_menu_item, cap_menu, cap
@pytest.mark.parametrize('nwords', [12, 24])
def test_view_trng_words_verifies_coin_mix(nwords, pick_menu_item, cap_menu, cap_story, unit_test,
- press_select, need_keypress, seed_story_to_words, is_q1):
+ press_select, need_keypress, seed_story_to_words, is_q1,
+ wait_for_story):
unit_test('devtest/clear_seed.py')
pick_menu_item('New Seed Words')
pick_menu_item(f'{nwords} Words')
@@ -550,15 +555,17 @@ def test_view_trng_words_verifies_coin_mix(nwords, pick_menu_item, cap_menu, cap
pick_menu_item('Coin Flips')
press_select()
+ time.sleep(.5)
flips = '01' * 64
for ch in flips:
need_keypress(ch)
+ time.sleep(0.01)
time.sleep(0.1)
done_key = KEY_ENTER if is_q1 else 'y'
need_keypress(done_key)
time.sleep(0.1)
- _, body = cap_story()
+ _, body = wait_for_story(f'Record these {nwords} secret words!', check_title=is_q1)
words = seed_story_to_words(body) if is_q1 else \
[w[3:].strip() for w in body.split('\n') if w and w[2] == ':']
### testing/test_vdisk.py
@@ -290,7 +290,7 @@ def test_virtdisk_wrapped_base64_uses_decoded_size(fake_txn, virtdisk_path, cap_
psbt = BasicPSBT().parse(fake_txn(1, 1, segwit_in=True))
padding = 0
for _ in range(3):
- psbt.unknown[b'\xfcsize-check'] = bytes(padding)
+ psbt.unknown[b'\xfdsize-check'] = bytes(padding)
raw = psbt.as_bytes()
if len(raw) == target_len:
breakWhy this scored 15/100
Community notes
Notes can correct, qualify, or add evidence to the AI analysis. Every note shown here has been validated by a human moderator.
The AI analysis stands alone for now. Submit a note if you can add evidence or important context.