tests: update authproxy.py to the latest version from Bitcoin Core
What changed, and why it matters
This commit updates a test helper file used to talk to a Bitcoin node during automated testing. The old helper could accidentally send the same command twice if the network connection hiccuped, which could cause a test-only action like sending coins to happen twice. The new version removes that automatic retry and instead opens a fresh connection for each test command. This is a test-infrastructure hardening change, not a fix in the Ledger app that users install on their device.
No immediate action required for end users. For developers running the test suite, ensure the updated authproxy.py and conftest.py are used so that tests do not accidentally double-spend test coins or duplicate other state-changing RPCs. Review other test helpers for similar unsafe retry patterns.
Security signals we found
Removed transparent retry of non-idempotent RPC requests after connection resets
Test helper could execute state-changing RPCs twice before the fix
New connection-per-call configuration mitigates idle-timeout disconnects without retry risk
Change is in test utilities, not the device firmware
Evidence from the diff
The commit replaces test_utils/authproxy.py with a newer copy from Bitcoin Core and modifies tests/conftest.py to disable HTTP connection reuse. The old authproxy.py retried POST requests on BrokenPipeError/ConnectionResetError/OSError(41), which is unsafe for non-idempotent RPCs such as sendtoaddress because the server may have already processed the first request. The new authproxy.py drops the retry logic. To compensate for bitcoind closing idle connections after -rpcservertimeout while tests interact with the device, conftest.py now sets reuse_http_connections=False on AuthServiceProxy instances, forcing a new connection per call. The change is confined to the test harness and does not alter the on-device Bitcoin app firmware.
Changed components
test_utils/authproxy.pytests/conftest.pyLedger Bitcoin app test harnessInspect captured patch +91 / −52
### test_utils/authproxy.py
@@ -20,16 +20,18 @@
# along with this software; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
-# Taken from https://raw.githubusercontent.com/bitcoin/bitcoin/fad02274ba28189f28f5af536b8ea0ecddf9df55/test/functional/test_framework/authproxy.py
-
+# Taken from https://raw.githubusercontent.com/bitcoin/bitcoin/f42226d526ebb3eb9217e738108e4f8148a1b069/test/functional/test_framework/authproxy.py
+# Local changes:
+# - JSONRPCException and assert_equal are defined in this file, instead of being imported from
+# test_framework/util.py.
"""HTTP proxy for opening RPC connection to bitcoind.
AuthServiceProxy has the following improvements over python-jsonrpc's
ServiceProxy class:
- HTTP connections persist for the life of the AuthServiceProxy object
(if server supports HTTP/1.1)
-- sends protocol 'version', per JSON-RPC 1.1
+- sends "jsonrpc":"2.0", per JSON-RPC 2.0
- sends proper, incrementing 'id'
- sends Basic HTTP authentication headers
- parses all JSON numbers that look like floats as Decimal
@@ -42,16 +44,18 @@
import http.client
import json
import logging
-import os
+import pathlib
import socket
import time
import urllib.parse
+
HTTP_TIMEOUT = 30
USER_AGENT = "AuthServiceProxy/0.1"
log = logging.getLogger("BitcoinRPC")
+
class JSONRPCException(Exception):
def __init__(self, rpc_error, http_status=None):
try:
@@ -63,9 +67,16 @@ def __init__(self, rpc_error, http_status=None):
self.http_status = http_status
-def EncodeDecimal(o):
+def assert_equal(thing1, thing2):
+ if thing1 != thing2:
+ raise AssertionError("not(%s == %s)" % (str(thing1), str(thing2)))
+
+
+def serialization_fallback(o):
if isinstance(o, decimal.Decimal):
return str(o)
+ if isinstance(o, pathlib.Path):
+ return str(o)
raise TypeError(repr(o) + " is not JSON serializable")
class AuthServiceProxy():
@@ -76,12 +87,16 @@ def __init__(self, service_url, service_name=None, timeout=HTTP_TIMEOUT, connect
self.__service_url = service_url
self._service_name = service_name
self.ensure_ascii = ensure_ascii # can be toggled on the fly by tests
+ self.reuse_http_connections = True
self.__url = urllib.parse.urlparse(service_url)
user = None if self.__url.username is None else self.__url.username.encode('utf8')
passwd = None if self.__url.password is None else self.__url.password.encode('utf8')
authpair = user + b':' + passwd
self.__auth_header = b'Basic ' + base64.b64encode(authpair)
- self.timeout = timeout
+ # clamp the socket timeout, since larger values can cause an
+ # "Invalid argument" exception in Python's HTTP(S) client
+ # library on some operating systems (e.g. OpenBSD, FreeBSD)
+ self.timeout = min(timeout, 2147483)
self._set_conn(connection)
def __getattr__(self, name):
@@ -90,77 +105,79 @@ def __getattr__(self, name):
raise AttributeError
if self._service_name is not None:
name = "%s.%s" % (self._service_name, name)
+ if not self.reuse_http_connections:
+ self._set_conn()
return AuthServiceProxy(self.__service_url, name, connection=self.__conn)
def _request(self, method, path, postdata):
'''
- Do a HTTP request, with retry if we get disconnected (e.g. due to a timeout).
- This is a workaround for https://bugs.python.org/issue3566 which is fixed in Python 3.5.
+ Do a HTTP request.
'''
headers = {'Host': self.__url.hostname,
'User-Agent': USER_AGENT,
'Authorization': self.__auth_header,
'Content-type': 'application/json'}
- if os.name == 'nt':
- # Windows somehow does not like to re-use connections
- # TODO: Find out why the connection would disconnect occasionally and make it reusable on Windows
- # Avoid "ConnectionAbortedError: [WinError 10053] An established connection was aborted by the software in your host machine"
+ if not self.reuse_http_connections:
self._set_conn()
- try:
- self.__conn.request(method, path, postdata, headers)
- return self._get_response()
- except (BrokenPipeError, ConnectionResetError):
- # Python 3.5+ raises BrokenPipeError when the connection was reset
- # ConnectionResetError happens on FreeBSD
- self.__conn.close()
- self.__conn.request(method, path, postdata, headers)
- return self._get_response()
- except OSError as e:
- # Workaround for a bug on macOS. See https://bugs.python.org/issue33450
- retry = '[Errno 41] Protocol wrong type for socket' in str(e)
- if retry:
- self.__conn.close()
- self.__conn.request(method, path, postdata, headers)
- return self._get_response()
- else:
- raise
+ self.__conn.request(method, path, postdata, headers)
+ return self._get_response()
+
+ def _json_dumps(self, obj):
+ return json.dumps(obj, default=serialization_fallback, ensure_ascii=self.ensure_ascii)
def get_request(self, *args, **argsn):
AuthServiceProxy.__id_count += 1
- log.debug("-{}-> {} {}".format(
+ log.debug("-{}-> {} {} {}".format(
AuthServiceProxy.__id_count,
self._service_name,
- json.dumps(args or argsn, default=EncodeDecimal, ensure_ascii=self.ensure_ascii),
+ self._json_dumps(args),
+ self._json_dumps(argsn),
))
+
if args and argsn:
- raise ValueError('Cannot handle both named and positional arguments')
- return {'version': '1.1',
+ params = dict(args=args, **argsn)
+ else:
+ params = args or argsn
+ return {'jsonrpc': '2.0',
'method': self._service_name,
- 'params': args or argsn,
+ 'params': params,
'id': AuthServiceProxy.__id_count}
def __call__(self, *args, **argsn):
- postdata = json.dumps(self.get_request(*args, **argsn), default=EncodeDecimal, ensure_ascii=self.ensure_ascii)
+ postdata = self._json_dumps(self.get_request(*args, **argsn))
response, status = self._request('POST', self.__url.path, postdata.encode('utf-8'))
- if response['error'] is not None:
- raise JSONRPCException(response['error'], status)
- elif 'result' not in response:
- raise JSONRPCException({
- 'code': -343, 'message': 'missing JSON-RPC result'}, status)
- elif status != HTTPStatus.OK:
- raise JSONRPCException({
- 'code': -342, 'message': 'non-200 HTTP status code but no JSON-RPC error'}, status)
+ # For backwards compatibility tests, accept JSON RPC 1.1 responses
+ if 'jsonrpc' not in response:
+ if response['error'] is not None:
+ raise JSONRPCException(response['error'], status)
+ elif 'result' not in response:
+ raise JSONRPCException({
+ 'code': -343, 'message': 'missing JSON-RPC result'}, status)
+ elif status != HTTPStatus.OK:
+ raise JSONRPCException({
+ 'code': -342, 'message': 'non-200 HTTP status code but no JSON-RPC error'}, status)
+ else:
+ return response['result']
else:
+ assert_equal(response['jsonrpc'], '2.0')
+ if status != HTTPStatus.OK:
+ raise JSONRPCException({
+ 'code': -342, 'message': 'non-200 HTTP status code'}, status)
+ if 'error' in response:
+ raise JSONRPCException(response['error'], status)
+ elif 'result' not in response:
+ raise JSONRPCException({
+ 'code': -343, 'message': 'missing JSON-RPC 2.0 result and error'}, status)
return response['result']
def batch(self, rpc_call_list):
- postdata = json.dumps(list(rpc_call_list), default=EncodeDecimal, ensure_ascii=self.ensure_ascii)
+ postdata = self._json_dumps(list(rpc_call_list))
log.debug("--> " + postdata)
response, status = self._request('POST', self.__url.path, postdata.encode('utf-8'))
if status != HTTPStatus.OK:
raise JSONRPCException({
- 'code': -342, 'message': 'non-200 HTTP status code but no JSON-RPC error'}, status)
+ 'code': -342, 'message': 'non-200 HTTP status code'}, status)
return response
def _get_response(self):
@@ -173,22 +190,36 @@ def _get_response(self):
'message': '%r RPC took longer than %f seconds. Consider '
'using larger timeout for calls that take '
'longer to return.' % (self._service_name,
- self.__conn.timeout)})
+ self.__conn.timeout)}) from None
if http_response is None:
raise JSONRPCException({
'code': -342, 'message': 'missing HTTP response from server'})
+ # Check for no-content HTTP status code, which can be returned when an
+ # RPC client requests a JSON-RPC 2.0 "notification" with no response.
+ # Currently this is only possible if clients call the _request() method
+ # directly to send a raw request.
+ if http_response.status == HTTPStatus.NO_CONTENT:
+ if len(http_response.read()) != 0:
+ raise JSONRPCException({'code': -342, 'message': 'Content received with NO CONTENT status code'})
+ return None, http_response.status
+
content_type = http_response.getheader('Content-Type')
if content_type != 'application/json':
raise JSONRPCException(
- {'code': -342, 'message': 'non-JSON HTTP response with \'%i %s\' from server' % (http_response.status, http_response.reason)},
+ {'code': -342, 'message': f"non-JSON HTTP response with \'{http_response.status} {http_response.reason}\' from server: {http_response.read().decode()}"},
http_response.status)
- responsedata = http_response.read().decode('utf8')
+ data = http_response.read()
+ try:
+ responsedata = data.decode('utf8')
+ except UnicodeDecodeError as e:
+ raise JSONRPCException({
+ 'code': -342, 'message': f'Cannot decode response in utf8 format, content: {data}, exception: {e}'})
response = json.loads(responsedata, parse_float=decimal.Decimal)
elapsed = time.time() - req_start_time
if "error" in response and response["error"] is None:
- log.debug("<-%s- [%.6f] %s" % (response["id"], elapsed, json.dumps(response["result"], default=EncodeDecimal, ensure_ascii=self.ensure_ascii)))
+ log.debug("<-%s- [%.6f] %s" % (response["id"], elapsed, self._json_dumps(response["result"])))
else:
log.debug("<-- [%.6f] %s" % (elapsed, responsedata))
return response, http_response.status
### tests/conftest.py
@@ -75,12 +75,20 @@ def additional_speculos_arguments(request) -> List[str]:
btc_addr = ""
+def _new_rpc(url: str) -> AuthServiceProxy:
+ rpc = AuthServiceProxy(url)
+ # Use a new connection for each call, as bitcoind closes idle connections after
+ # -rpcservertimeout, and tests can be idle for long while interacting with the device.
+ rpc.reuse_http_connections = False
+ return rpc
+
+
def get_rpc() -> AuthServiceProxy:
- return AuthServiceProxy(rpc_url)
+ return _new_rpc(rpc_url)
def get_wallet_rpc(wallet_name: str) -> AuthServiceProxy:
- return AuthServiceProxy(f"{rpc_url}/wallet/{wallet_name}")
+ return _new_rpc(f"{rpc_url}/wallet/{wallet_name}")
def setup_node():Why this scored 24/100
Community notes
Notes can correct, qualify, or add evidence to the AI analysis. Every note shown here has been validated by a human moderator.
The AI analysis stands alone for now. Submit a note if you can add evidence or important context.