contrib: Use text=True in subprocess over manual encoding handling
What changed, and why it matters
This commit is a routine code cleanup in Bitcoin Core's helper and test scripts. It replaces manual UTF-8 decoding of subprocess output with Python's built-in text=True option. There is no security-relevant change: the behavior is functionally equivalent, only the code is simpler and more consistent.
No security action required. This is a benign refactoring commit.
Security signals we found
No strong security signals were identified.
Evidence from the diff
The commit refactors 12 Python scripts under contrib/ and test/lint/ to use subprocess’s text=True parameter instead of explicitly calling .decode(‘utf-8’) or .decode(‘utf8’) on bytes output. In one case (lint-submodule.py), it removes a redundant encoding=’utf8’ argument where text=True was already present. The changes are purely stylistic and do not alter encoding, error handling, command arguments, or program logic. No input validation, privilege boundaries, or cryptographic operations are modified.
Changed components
contrib/devtools/copyright_header.pycontrib/verify-commits/verify-commits.pytest/functional/feature_framework_startup_failures.pytest/functional/tool_signet_miner.pytest/lint/check-doc.pytest/lint/lint-files.pytest/lint/lint-include-guards.pytest/lint/lint-python-dead-code.pytest/lint/lint-python.pytest/lint/lint-shell-locale.pytest/lint/lint-spelling.pytest/lint/lint-submodule.pyInspect captured patch +43 / −47
diff --git a/contrib/devtools/copyright_header.py b/contrib/devtools/copyright_header.py
index 0d9f3543..aa77de8a 100755
--- a/contrib/devtools/copyright_header.py
+++ b/contrib/devtools/copyright_header.py
@@ -54,12 +54,12 @@ GIT_LS_CMD = 'git ls-files --full-name'.split(' ')
GIT_TOPLEVEL_CMD = 'git rev-parse --show-toplevel'.split(' ')
def call_git_ls(base_directory):
- out = subprocess.check_output([*GIT_LS_CMD, base_directory])
- return [f for f in out.decode("utf-8").split('\n') if f != '']
+ out = subprocess.check_output([*GIT_LS_CMD, base_directory], text=True)
+ return [f for f in out.split('\n') if f != '']
def call_git_toplevel():
"Returns the absolute path to the project root"
- return subprocess.check_output(GIT_TOPLEVEL_CMD).strip().decode("utf-8")
+ return subprocess.check_output(GIT_TOPLEVEL_CMD, text=True).strip()
def get_filenames_to_examine(base_directory):
"Returns an array of absolute paths to any project files in the base_directory that pass the include/exclude filters"
@@ -298,8 +298,8 @@ def report_cmd(argv):
GIT_LOG_CMD = "git log --pretty=format:%%ai %s"
def call_git_log(filename):
- out = subprocess.check_output((GIT_LOG_CMD % filename).split(' '))
- return out.decode("utf-8").split('\n')
+ out = subprocess.check_output((GIT_LOG_CMD % filename).split(' '), text=True)
+ return out.split('\n')
def get_git_change_years(filename):
git_log_lines = call_git_log(filename)
diff --git a/contrib/verify-commits/verify-commits.py b/contrib/verify-commits/verify-commits.py
index b5c028cf..1af6b031 100755
--- a/contrib/verify-commits/verify-commits.py
+++ b/contrib/verify-commits/verify-commits.py
@@ -140,8 +140,8 @@ def main():
# Check that the commit (and parents) was signed with a trusted key
valid_sig = False
- verify_res = subprocess.run([GIT, '-c', 'gpg.program={}/gpg.sh'.format(dirname), 'verify-commit', "--raw", current_commit], capture_output=True)
- for line in verify_res.stderr.decode().splitlines():
+ verify_res = subprocess.run([GIT, '-c', 'gpg.program={}/gpg.sh'.format(dirname), 'verify-commit', "--raw", current_commit], text=True, capture_output=True)
+ for line in verify_res.stderr.splitlines():
if line.startswith("[GNUPG:] VALIDSIG "):
key = line.split(" ")[-1]
valid_sig = key in trusted_keys
@@ -152,7 +152,7 @@ def main():
if prev_commit != "":
print("No parent of {} was signed with a trusted key!".format(prev_commit), file=sys.stderr)
print("Parents are:", file=sys.stderr)
- parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', prev_commit]).decode('utf8').splitlines()[0].split(' ')
+ parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', prev_commit], text=True).splitlines()[0].split(' ')
for parent in parents:
subprocess.call([GIT, 'show', '-s', parent], stdout=sys.stderr)
else:
@@ -162,29 +162,29 @@ def main():
# Check the Tree-SHA512
if (verify_tree or prev_commit == "") and current_commit not in incorrect_sha512_allowed:
tree_hash = tree_sha512sum(current_commit)
- if ("Tree-SHA512: {}".format(tree_hash)) not in subprocess.check_output([GIT, 'show', '-s', '--format=format:%B', current_commit]).decode('utf8').splitlines():
+ if ("Tree-SHA512: {}".format(tree_hash)) not in subprocess.check_output([GIT, 'show', '-s', '--format=format:%B', current_commit], text=True).splitlines():
print("Tree-SHA512 did not match for commit " + current_commit, file=sys.stderr)
sys.exit(1)
# Merge commits should only have two parents
- parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', current_commit]).decode('utf8').splitlines()[0].split(' ')
+ parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', current_commit], text=True).splitlines()[0].split(' ')
if len(parents) > 2:
print("Commit {} is an octopus merge".format(current_commit), file=sys.stderr)
sys.exit(1)
# Check that the merge commit is clean
- commit_time = int(subprocess.check_output([GIT, 'show', '-s', '--format=format:%ct', current_commit]).decode('utf8').splitlines()[0])
+ commit_time = int(subprocess.check_output([GIT, 'show', '-s', '--format=format:%ct', current_commit], text=True).splitlines()[0])
check_merge = commit_time > time.time() - args.clean_merge * 24 * 60 * 60 # Only check commits in clean_merge days
allow_unclean = current_commit in unclean_merge_allowed
if len(parents) == 2 and check_merge and not allow_unclean:
- current_tree = subprocess.check_output([GIT, 'show', '--format=%T', current_commit]).decode('utf8').splitlines()[0]
+ current_tree = subprocess.check_output([GIT, 'show', '--format=%T', current_commit], text=True).splitlines()[0]
# This merge-tree functionality requires git >= 2.38. The
# --write-tree option was added in order to opt-in to the new
# behavior. Older versions of git will not recognize the option and
# will instead exit with code 128.
try:
- recreated_tree = subprocess.check_output([GIT, "merge-tree", "--write-tree", parents[0], parents[1]]).decode('utf8').splitlines()[0]
+ recreated_tree = subprocess.check_output([GIT, "merge-tree", "--write-tree", parents[0], parents[1]], text=True).splitlines()[0]
except subprocess.CalledProcessError as e:
if e.returncode == 128:
print("git v2.38+ is required for this functionality.", file=sys.stderr)
diff --git a/test/functional/feature_framework_startup_failures.py b/test/functional/feature_framework_startup_failures.py
index 84a7d9d8..43dbeae7 100755
--- a/test/functional/feature_framework_startup_failures.py
+++ b/test/functional/feature_framework_startup_failures.py
@@ -43,15 +43,13 @@ class FeatureFrameworkStartupFailures(BitcoinTestFramework):
args = [sys.executable] + sys.argv + [f"--tmpdir={self.options.tmpdir}/{subdir}", f"--internal_test={test.__name__}"] + internal_args
try:
- # The subprocess encoding argument gives different results for e.output
- # on Linux/Windows, so handle decoding by ourselves for consistency.
output = subprocess.run(args, timeout=60 * self.options.timeout_factor,
- stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.decode("utf-8")
+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True).stdout
except subprocess.TimeoutExpired as e:
print("Unexpected child process timeout!\n"
"WARNING: Timeouts like this halt execution of TestNode logic, "
"meaning dangling bitcoind processes are to be expected.\n"
- f"<CHILD OUTPUT BEGIN>\n{e.output.decode('utf-8')}\n<CHILD OUTPUT END>",
+ f"<CHILD OUTPUT BEGIN>\n{e.output}\n<CHILD OUTPUT END>",
file=sys.stderr)
raise
diff --git a/test/functional/tool_signet_miner.py b/test/functional/tool_signet_miner.py
index dbf64b64..2be47332 100755
--- a/test/functional/tool_signet_miner.py
+++ b/test/functional/tool_signet_miner.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
-# Copyright (c) 2022 The Bitcoin Core developers
+# Copyright (c) 2022-present The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test signet miner tool"""
@@ -102,8 +102,8 @@ class SignetMinerTest(BitcoinTestFramework):
'genpsbt',
f'--address={node.getnewaddress()}',
'--poolnum=98',
- ], check=True, input=json.dumps(template).encode('utf8'), capture_output=True)
- psbt = genpsbt.stdout.decode('utf8').strip()
+ ], check=True, text=True, input=json.dumps(template), capture_output=True)
+ psbt = genpsbt.stdout.strip()
if sign:
self.log.debug("Sign the PSBT")
res = node.walletprocesspsbt(psbt=psbt, sign=True, sighashtype='ALL')
@@ -112,8 +112,8 @@ class SignetMinerTest(BitcoinTestFramework):
solvepsbt = subprocess.run(base_cmd + [
'solvepsbt',
f'--grind-cmd={shlex.join(util_argv)}',
- ], check=True, input=psbt.encode('utf8'), capture_output=True)
- node.submitblock(solvepsbt.stdout.decode('utf8').strip())
+ ], check=True, text=True, input=psbt, capture_output=True)
+ node.submitblock(solvepsbt.stdout.strip())
assert_equal(node.getblockcount(), n_blocks + 1)
def run_test(self):
diff --git a/test/lint/check-doc.py b/test/lint/check-doc.py
index e47ff30f..4dc605b1 100755
--- a/test/lint/check-doc.py
+++ b/test/lint/check-doc.py
@@ -1,13 +1,11 @@
#!/usr/bin/env python3
-# Copyright (c) 2015-2022 The Bitcoin Core developers
+# Copyright (c) 2015-present The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
'''
This checks if all command line args are documented.
Return value is 0 to indicate no error.
-
-Author: @MarcoFalke
'''
from subprocess import check_output
@@ -27,8 +25,8 @@ SET_DOC_OPTIONAL = set(['-h', '-?', '-dbcrashratio', '-forcecompactdb', '-ipccon
def lint_missing_argument_documentation():
- used = check_output(CMD_GREP_ARGS, shell=True).decode('utf8').strip()
- docd = check_output(CMD_GREP_DOCS, shell=True).decode('utf8').strip()
+ used = check_output(CMD_GREP_ARGS, shell=True, text=True).strip()
+ docd = check_output(CMD_GREP_DOCS, shell=True, text=True).strip()
args_used = set(re.findall(re.compile(REGEX_ARG), used))
args_docd = set(re.findall(re.compile(REGEX_DOC), docd)).union(SET_DOC_OPTIONAL)
@@ -46,8 +44,8 @@ def lint_missing_argument_documentation():
def lint_missing_hidden_wallet_args():
- wallet_args = check_output(CMD_GREP_WALLET_ARGS, shell=True).decode('utf8').strip()
- wallet_hidden_args = check_output(CMD_GREP_WALLET_HIDDEN_ARGS, shell=True).decode('utf8').strip()
+ wallet_args = check_output(CMD_GREP_WALLET_ARGS, shell=True, text=True).strip()
+ wallet_hidden_args = check_output(CMD_GREP_WALLET_HIDDEN_ARGS, shell=True, text=True).strip()
wallet_args = set(re.findall(re.compile(REGEX_DOC), wallet_args))
wallet_hidden_args = set(re.findall(re.compile(r' "([^"=]+)'), wallet_hidden_args))
diff --git a/test/lint/lint-files.py b/test/lint/lint-files.py
index b510830d..6d9d7d27 100755
--- a/test/lint/lint-files.py
+++ b/test/lint/lint-files.py
@@ -76,7 +76,7 @@ def get_git_file_metadata() -> dict[str, FileMeta]:
'''
Return a dictionary mapping the name of all files in the repository to git tree metadata.
'''
- files_raw = check_output(CMD_ALL_FILES).decode("utf8").rstrip("\0").split("\0")
+ files_raw = check_output(CMD_ALL_FILES, text=True).rstrip("\0").split("\0")
files = {}
for file_spec in files_raw:
meta = FileMeta(file_spec)
@@ -169,7 +169,7 @@ def check_shebang_file_permissions(files_meta) -> int:
"""
Checks every file that contains a shebang line to ensure it has an executable permission
"""
- filenames = check_output(CMD_SHEBANG_FILES).decode("utf8").strip().split("\n")
+ filenames = check_output(CMD_SHEBANG_FILES, text=True).strip().split("\n")
# The git grep command we use returns files which contain a shebang on any line within the file
# so we need to filter the list to only files with the shebang on the first line
@@ -198,7 +198,7 @@ def check_shebang_file_permissions(files_meta) -> int:
def main() -> NoReturn:
- root_dir = check_output(CMD_TOP_LEVEL).decode("utf8").strip()
+ root_dir = check_output(CMD_TOP_LEVEL, text=True).strip()
os.chdir(root_dir)
files = get_git_file_metadata()
diff --git a/test/lint/lint-include-guards.py b/test/lint/lint-include-guards.py
index c56eeab3..dbbf0325 100755
--- a/test/lint/lint-include-guards.py
+++ b/test/lint/lint-include-guards.py
@@ -31,7 +31,7 @@ def _get_header_file_lst() -> list[str]:
"""
git_cmd_lst = ['git', 'ls-files', '--', '*.h']
header_file_lst = check_output(
- git_cmd_lst).decode('utf-8').splitlines()
+ git_cmd_lst, text=True).splitlines()
header_file_lst = [hf for hf in header_file_lst
if not any(ef in hf for ef
diff --git a/test/lint/lint-python-dead-code.py b/test/lint/lint-python-dead-code.py
index b3f93947..cd6471e2 100755
--- a/test/lint/lint-python-dead-code.py
+++ b/test/lint/lint-python-dead-code.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python3
#
-# Copyright (c) 2022 The Bitcoin Core developers
+# Copyright (c) 2022-present The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
@@ -24,15 +24,15 @@ def check_vulture_install():
def main():
check_vulture_install()
- files = check_output(FILES_ARGS).decode("utf-8").splitlines()
+ files = check_output(FILES_ARGS, text=True).splitlines()
# --min-confidence 100 will only report code that is guaranteed to be unused within the analyzed files.
# Any value below 100 introduces the risk of false positives, which would create an unacceptable maintenance burden.
vulture_args = ['vulture', '--min-confidence=100'] + files
try:
- check_output(vulture_args, stderr=STDOUT)
+ check_output(vulture_args, stderr=STDOUT, text=True)
except CalledProcessError as e:
- print(e.output.decode("utf-8"), end="")
+ print(e.output, end="")
print("Python dead code detection found some issues")
exit(1)
diff --git a/test/lint/lint-python.py b/test/lint/lint-python.py
index e2dbe25b..fd1a8495 100755
--- a/test/lint/lint-python.py
+++ b/test/lint/lint-python.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python3
#
-# Copyright (c) 2022 The Bitcoin Core developers
+# Copyright (c) 2022-present The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
@@ -37,7 +37,7 @@ def check_dependencies():
def main():
check_dependencies()
- mypy_files = subprocess.check_output(MYPY_FILES_ARGS).decode("utf-8").splitlines()
+ mypy_files = subprocess.check_output(MYPY_FILES_ARGS, text=True).splitlines()
mypy_args = ['mypy', '--show-error-codes'] + mypy_files
try:
diff --git a/test/lint/lint-shell-locale.py b/test/lint/lint-shell-locale.py
index 3602289d..80b1d4d0 100755
--- a/test/lint/lint-shell-locale.py
+++ b/test/lint/lint-shell-locale.py
@@ -27,10 +27,10 @@ def get_shell_files_list():
'*.sh',
]
try:
- return subprocess.check_output(command, stderr = subprocess.STDOUT).decode('utf-8').splitlines()
+ return subprocess.check_output(command, stderr = subprocess.STDOUT, text=True).splitlines()
except subprocess.CalledProcessError as e:
if e.returncode > 1: # return code is 1 when match is empty
- print(e.output.decode('utf-8'), end='')
+ print(e.output, end='')
sys.exit(1)
return []
diff --git a/test/lint/lint-spelling.py b/test/lint/lint-spelling.py
index 945288a3..cd34162c 100755
--- a/test/lint/lint-spelling.py
+++ b/test/lint/lint-spelling.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python3
#
-# Copyright (c) 2022 The Bitcoin Core developers
+# Copyright (c) 2022-present The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
@@ -29,13 +29,13 @@ def check_codespell_install():
def main():
check_codespell_install()
- files = check_output(FILES_ARGS).decode("utf-8").splitlines()
+ files = check_output(FILES_ARGS, text=True).splitlines()
codespell_args = ['codespell', '--check-filenames', '--disable-colors', '--quiet-level=7', '--ignore-words={}'.format(IGNORE_WORDS_FILE)] + files
try:
- check_output(codespell_args, stderr=STDOUT)
+ check_output(codespell_args, stderr=STDOUT, text=True)
except CalledProcessError as e:
- print(e.output.decode("utf-8"), end="")
+ print(e.output, end="")
print('^ Warning: codespell identified likely spelling errors. Any false positives? Add them to the list of ignored words in {}'.format(IGNORE_WORDS_FILE))
diff --git a/test/lint/lint-submodule.py b/test/lint/lint-submodule.py
index 4d2fbf08..d5700d33 100755
--- a/test/lint/lint-submodule.py
+++ b/test/lint/lint-submodule.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python3
#
-# Copyright (c) 2022 The Bitcoin Core developers
+# Copyright (c) 2022-present The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
@@ -13,7 +13,7 @@ import sys
def main():
submodules_list = subprocess.check_output(['git', 'submodule', 'status', '--recursive'],
- text = True, encoding = 'utf8').rstrip('\n')
+ text = True).rstrip('\n')
if submodules_list:
print("These submodules were found, delete them:\n", submodules_list)
sys.exit(1)
Why this scored 15/100
Community notes
Notes can correct, qualify, or add evidence to the AI analysis. Every note shown here has been validated by a human moderator.
The AI analysis stands alone for now. Submit a note if you can add evidence or important context.