scripts: make OTS output deterministic for identical inputs
What changed, and why it matters
This commit changes a release-maintenance helper script so that running it twice with the same inputs produces byte-for-byte identical output files. It removes timestamps, local backend details, and non-deterministic ordering from the generated artifacts, and fixes environment settings that could make the OpenTimestamps client behave differently on different machines. These are build/reproducibility hardening changes, not a fix for an exploitable vulnerability in the Lightning node itself.
No urgent action required. Treat as a normal maintenance/reproducibility improvement. Reviewers verifying release artifacts should confirm the script now produces identical output across runs and that the upload.sh no longer contains local node details.
Security signals we found
Reproducible build artifact generation reduces supply-chain tampering opportunities
Removal of local backend URL and credentials from generated upload.sh reduces information leakage in published release artifacts
Deterministic ordering and environment hardening reduce non-malicious variance that could hide malicious changes during review
Evidence from the diff
The patch modifies scripts/upgrade-ots.py to make OpenTimestamp proof upgrade artifacts reproducible: it sets PYTHONHASHSEED=0, LC_ALL=C, TZ=UTC for the ots subprocess; sorts proofs, results, and upload.sh replacement entries; removes generation timestamps and local bitcoind URLs from generated files; writes files with explicit UTF-8 encoding and 0o644/0o755 permissions; and moves relative pending-age diagnostics to the terminal instead of summary files. The script still exits with an error if proofs are pending too long, but no longer embeds that age in comparable output. No changes affect lnd runtime, networking, wallet, or consensus code.
Changed components
scripts/upgrade-ots.pyInspect captured patch +48 / −43
### scripts/upgrade-ots.py
@@ -41,6 +41,11 @@
Requires Python 3.9 or later, and the ots client (opentimestamps-client) in
PATH.
+
+Successful runs with identical release assets, calendar responses, options and
+tool versions produce identical output contents. The output directory name
+still defaults to the current UTC time. Calendar evidence can change between
+runs; differing evidence can produce differing upgraded proofs.
"""
import argparse
@@ -109,7 +114,7 @@
UPLOAD_SH = r"""#!/bin/bash
#
-# Generated by upgrade-ots.py at {time} UTC.
+# Generated by upgrade-ots.py.
#
{intro}
#
@@ -241,8 +246,10 @@ def run(self, command, *args):
"""Returns the exit status and the output of an ots command."""
argv = ["ots", *self.options, command, *map(str, args)]
- # ots prints the time of an attestation in the local time zone.
- env = dict(os.environ, TZ="UTC")
+ # ots iterates sets of attestations while upgrading. Fix their order,
+ # as well as verification dates and messages, across client processes.
+ # --no-cache above prevents local cached evidence affecting the result.
+ env = dict(os.environ, TZ="UTC", LC_ALL="C", PYTHONHASHSEED="0")
try:
done = subprocess.run(argv, capture_output=True, text=True,
env=env, timeout=OTS_TIMEOUT)
@@ -371,11 +378,6 @@ def plural(n, noun):
return f"{n} {noun}" + ("" if n == 1 else "s")
-def without_credentials(url):
- parts = urllib.parse.urlsplit(url)
- return parts._replace(netloc=parts.netloc.rpartition("@")[2]).geturl()
-
-
def fetch(url, headers=None, timeout=HTTP_TIMEOUT):
"""Returns the body of the response to a GET request. A network error or
a server error is retried once."""
@@ -444,8 +446,7 @@ def verify(release, proof, ots):
return f"Bitcoin block {match[1]}, {match[2]}" if match else ""
-def process(release, asset, work_dir, ots, max_pending_days,
- verify_complete):
+def process(release, asset, work_dir, ots, verify_complete):
"""Upgrades and verifies one proof. Leaves the upgraded proof, the
original (.bak) and the file it timestamps in work_dir/<tag>.
@@ -483,20 +484,17 @@ def process(release, asset, work_dir, ots, max_pending_days,
if not ended or not PENDING_ANSWER.search(output):
raise ProofError(f"ots upgrade failed (exit status {status}):\n"
+ indent(tidy(output)))
- uploaded = datetime.datetime.strptime(
- asset["created_at"], "%Y-%m-%dT%H:%M:%S%z")
- age = (datetime.datetime.now(datetime.timezone.utc) - uploaded).days
- if age > max_pending_days:
- raise ProofError(f"still pending after {age} days")
- return PENDING, f"uploaded {age} days ago"
+ # Relative ages belong in terminal diagnostics, not comparable files.
+ return PENDING, f"uploaded {asset['created_at']}"
backup = directory / (name + ".bak")
if not backup.is_file() or sha256_file(backup) != original:
raise ProofError("ots upgrade didn't keep the original as .bak")
return UPGRADED, verify(release, proof, ots)
-def write_upload_sh(out, repo, upgraded, started, backend, esplora):
+def write_upload_sh(out, repo, upgraded, backend, esplora):
+ upgraded = sorted(upgraded, key=lambda r: (r.tag, r.name))
replacements = []
for r in upgraded:
sha256 = sha256_file(out / r.tag / r.name)
@@ -513,19 +511,18 @@ def write_upload_sh(out, repo, upgraded, started, backend, esplora):
if esplora:
warning = "\necho {} \\\n {} >&2\n".format(
shlex.quote("WARNING: these proofs were verified against block "
- "headers from"),
- shlex.quote(f"{esplora}, not from a Bitcoin node."))
+ "headers from Esplora,"),
+ shlex.quote("not from a Bitcoin node."))
path = out / "upload.sh"
- path.write_text(UPLOAD_SH.format(
- time=f"{started:%Y-%m-%d %H:%M}",
+ path.write_bytes(UPLOAD_SH.format(
intro=comment(intro),
repo_sh=shlex.quote(repo),
example=f"{upgraded[0].tag}/{upgraded[0].name}",
backend=comment(backend),
warning=warning,
replacements="\n".join(replacements),
- ))
+ ).encode("utf-8"))
path.chmod(0o755)
@@ -542,12 +539,13 @@ def summarize(results, heading):
lines = heading + ["", count(results)]
for status, title in ((UPGRADED, "Upgraded"), (PENDING, "Pending"),
(ERROR, "Errors")):
- selected = [r for r in results if r.status == status]
+ selected = sorted((r for r in results if r.status == status),
+ key=lambda r: (r.tag, r.name))
if selected:
lines += ["", f"{title}:"]
for r in selected:
- lines.append(f" {r.tag}/{r.name}: "
- + r.detail.replace("\n", "\n "))
+ detail = "see terminal for diagnostics" if status == ERROR else r.detail
+ lines.append(f" {r.tag}/{r.name}: " + detail.replace("\n", "\n "))
return "\n".join(lines) + "\n"
@@ -606,18 +604,13 @@ def main():
bitcoin_node = args.bitcoin_node
if args.esplora:
bitcoin_node = EsploraProxy(args.esplora).url
- backend = (f"WARNING: block headers came from {args.esplora}, not "
- "from a Bitcoin node, so the proofs were only verified "
- "as far as that server can be trusted.")
+ backend = "WARNING: block headers came from trusted Esplora, not a Bitcoin node."
print(f"WARNING: verifying proofs against block headers from "
f"{args.esplora}. This trusts that server to serve the real "
f"Bitcoin chain. Use your own node (the default) for proofs "
f"you publish.", file=sys.stderr)
- elif bitcoin_node:
- backend = ("Block headers came from bitcoind at "
- f"{without_credentials(bitcoin_node)}.")
else:
- backend = "Block headers came from the local bitcoind."
+ backend = "Block headers came from a Bitcoin node."
ots = Ots(bitcoin_node)
try:
@@ -633,19 +626,20 @@ def main():
f"{', '.join(unknown)}")
releases = [r for r in releases if r["tag_name"] in args.tags]
- proofs = [(r, a) for r in releases for a in r["assets"]
- if a["name"].endswith(".ots")]
+ proofs = sorted(((r, a) for r in releases for a in r["assets"]
+ if a["name"].endswith(".ots")),
+ key=lambda pair: (pair[0]["tag_name"], pair[1]["name"]))
print(f"Checking {plural(len(proofs), 'proof')} of "
f"{plural(len(releases), 'release')} of {args.repo}.", flush=True)
results = []
stopped = None
+ old_pending = False
with tempfile.TemporaryDirectory(prefix="upgrade-ots-") as work:
work_dir = Path(work)
for release, asset in proofs:
try:
status, detail = process(release, asset, work_dir, ots,
- args.max_pending_days,
args.verify_complete)
except ProofError as e:
status, detail = ERROR, str(e)
@@ -657,33 +651,44 @@ def main():
r = Result(release["tag_name"], asset["name"], status, detail,
asset["id"])
results.append(r)
+ if status == PENDING:
+ uploaded = datetime.datetime.strptime(
+ asset["created_at"], "%Y-%m-%dT%H:%M:%S%z")
+ age = (started - uploaded).days
+ if age > args.max_pending_days:
+ print(f"ERROR: {r.tag}/{r.name}: still pending after {age} days",
+ file=sys.stderr)
+ old_pending = True
line = f"{r.tag}: {r.name}: {r.status}"
print(f"{line}: {r.detail}" if r.detail else line, flush=True)
upgraded = [r for r in results if r.status == UPGRADED]
if upgraded:
out.mkdir(parents=True)
+ out.chmod(0o755)
for r in upgraded:
(out / r.tag).mkdir(exist_ok=True)
+ (out / r.tag).chmod(0o755)
for name in (r.name, r.name + ".bak", r.name[:-len(".ots")]):
shutil.move(str(work_dir / r.tag / name),
str(out / r.tag / name))
+ (out / r.tag / name).chmod(0o644)
releases_line = "all published releases"
if args.tags:
- releases_line = ", ".join(args.tags)
+ releases_line = ", ".join(sorted(set(args.tags)))
heading = [
- f"Checked the proofs of {args.repo} ({releases_line}) at "
- f"{started:%Y-%m-%d %H:%M} UTC.",
+ f"Checked the proofs of {args.repo} ({releases_line}).",
backend,
"Proofs that were already complete were verified again."
if args.verify_complete else
"Proofs that were already complete were left alone, without "
"verifying them again.",
- ] + ([stopped] if stopped else [])
- (out / "summary.txt").write_text(summarize(results, heading))
- write_upload_sh(out, args.repo, upgraded, started, backend,
- args.esplora)
+ ] + (["Run stopped; see terminal for diagnostics."] if stopped else [])
+ summary = out / "summary.txt"
+ summary.write_bytes(summarize(results, heading).encode("utf-8"))
+ summary.chmod(0o644)
+ write_upload_sh(out, args.repo, upgraded, backend, args.esplora)
print()
print(count(results))
@@ -695,7 +700,7 @@ def main():
if stopped:
sys.exit("ERROR: " + stopped)
- if any(r.status == ERROR for r in results):
+ if old_pending or any(r.status == ERROR for r in results):
sys.exit(1)
Why this scored 18/100
Community notes
Notes can correct, qualify, or add evidence to the AI analysis. Every note shown here has been validated by a human moderator.
The AI analysis stands alone for now. Submit a note if you can add evidence or important context.