feat(ci, clear_signing): add clear signing device tests as a nightly run.
What changed, and why it matters
This commit adds a new nightly automated testing workflow for Ethereum 'clear signing' on Trezor hardware wallets. It does not change any wallet firmware code that users run; it only adds test scripts, CI configuration, documentation, and reporting improvements. There is no indication this commit fixes or introduces a security vulnerability.
No security action required. Treat as a normal test-infrastructure change. If reviewing for operational security, verify that the new workflow's AWS role and S3 bucket permissions follow the principle of least privilege, and that the Slack webhook secret is scoped to this workflow.
Security signals we found
No strong security signals were identified.
Evidence from the diff
The commit introduces a complete CI pipeline (.github/workflows/core-clear-signing.yml) that runs ERC-7730 registry test cases against the emulator using unsigned dev definitions. It adds supporting Python modules (nightly.py, registry.py, test_registry.py), Makefile targets, documentation, UI test fixture handling changes to allow alternate fixture files via TREZOR_UI_FIXTURES, and reporting enhancements for ‘new’ and ‘removed’ tests. No firmware signing logic, cryptography, or privileged code paths are modified.
Changed components
.github/workflows/core-clear-signing.yml.github/actions/ui-report/action.ymlci/prepare_ui_artifacts.pycore/Makefiletests/clear_signing_tests/nightly.pytests/clear_signing_tests/registry.pytests/clear_signing_tests/test_registry.pytests/ui_tests/common.pytests/ui_tests/__init__.pytests/ui_tests/reporting/testreport.pyInspect captured patch +2986 / −21
### .github/actions/ui-report/action.yml
@@ -11,6 +11,10 @@ inputs:
description: 'Either "success" or "failure"'
required: false
default: "failure"
+ all-screens:
+ description: 'Upload the screens of every test, not only those that match the fixtures (for tests that compare with their previous run)'
+ required: false
+ default: "false"
runs:
using: composite
steps:
@@ -20,10 +24,13 @@ runs:
role-to-assume: arn:aws:iam::538326561891:role/gh_actions_deploy_dev_firmware_data
aws-region: eu-central-1
- run: |
- MODELJOB=${{ inputs.model }}-${{ inputs.lang }}-${{ github.job }}
- OUTDIR=${{ github.run_id }}
+ OUTDIR=$GITHUB_RUN_ID
mkdir -p $OUTDIR
- $UV_RUN ci/prepare_ui_artifacts.py || true
+ if [ "$ALL_SCREENS" = "true" ]; then
+ $UV_RUN ci/prepare_ui_artifacts.py --all || true
+ else
+ $UV_RUN ci/prepare_ui_artifacts.py || true
+ fi
mv tests/ui_tests/reports/test/* $OUTDIR
mv tests/ui_tests/fixtures.*.json $OUTDIR
@@ -39,12 +46,16 @@ runs:
done
cd ..
- if [ "${{ inputs.status }}" = "success" ]; then
+ if [ "${INPUTS_STATUS}" = "success" ]; then
cp .github/actions/ui-report/success.png $OUTDIR/$MODELJOB-status.png
else
cp .github/actions/ui-report/failure.png $OUTDIR/$MODELJOB-status.png
fi
shell: sh
+ env:
+ MODELJOB: "${{ inputs.model }}-${{ inputs.lang }}-${{ github.job }}"
+ INPUTS_STATUS: ${{ inputs.status }}
+ ALL_SCREENS: ${{ inputs.all-screens }}
- name: Upload test results
run: |
# Cache s5cmd package (to be reused by the next calls)
@@ -53,8 +64,8 @@ runs:
CMD="s5cmd --log error cp --concurrency 100"
# Upload report
- du -sh ${{ github.run_id }}
- $NIX_RUN bash -c "$CMD ${{ github.run_id }}/ s3://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/" &
+ du -sh $GITHUB_RUN_ID
+ $NIX_RUN bash -c "$CMD $GITHUB_RUN_ID/ s3://data.trezor.io/dev/firmware/ui_report/$GITHUB_RUN_ID/" &
PID1=$!
# Upload test screen recording
### .github/workflows/core-clear-signing.yml
@@ -0,0 +1,227 @@
+name: Clear signing tests
+run-name: Clear signing tests on ${{ inputs.models_json != '' && join(fromJSON(inputs.models_json), ', ') || 'all models' }}
+
+on:
+ schedule:
+ - cron: '45 23 * * *' # every day @ 23:45
+ workflow_dispatch:
+ inputs:
+ models_json:
+ description: models as a JSON list
+ type: string
+ required: true
+ default: "[\"T2T1\",\"T3B1\",\"T3T1\",\"T3W1\"]"
+ definitions_sha:
+ description: trezor/definitions commit (default is the one of the last unsigned dev deploy)
+ type: string
+ required: false
+ default: ""
+
+# Queue, do not cancel: each run compares with the screens of the previous one, so it
+# must start after that one has uploaded them.
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: false
+
+permissions:
+ contents: read # for actions/checkout
+
+# [Clear-signing tests](../tests/clear-signing-tests.md): the ERC-7730 registry test
+# cases (mainnet) on the emulator, with the unsigned dev definitions of trezor/definitions.
+# Both come from the trezor/definitions commit of its last unsigned dev deploy. The screens
+# are compared with those of the last nightly run (`nightly.py fetch-baseline`), not with
+# the device tests' fixtures.
+#
+# Runs nightly on `main`, and can be started manually.
+jobs:
+ prepare:
+ name: Resolve inputs
+ runs-on: ubuntu-latest
+ timeout-minutes: 30
+ # No scheduled runs on forks.
+ if: github.repository == 'trezor/trezor-firmware' || github.event_name != 'schedule'
+ permissions:
+ actions: read # for listing the earlier nightly runs (fetch-baseline)
+ contents: read # for actions/checkout
+ id-token: write # for fetching the OIDC token for AWS (dev definitions)
+ outputs:
+ models: ${{ steps.models.outputs.models }}
+ steps:
+ - id: models
+ env:
+ MODELS: ${{ github.event_name == 'workflow_dispatch' && inputs.models_json || '["T2T1","T3B1","T3T1","T3W1"]' }}
+ run: echo "models=$(echo "$MODELS" | jq -cM .)" >> $GITHUB_OUTPUT
+ - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # actions/checkout@v6.0.2
+ with:
+ submodules: recursive
+ persist-credentials: false
+ - uses: $/.github/actions/environment
+ - name: Resolve the definitions and registry commits
+ id: inputs
+ env:
+ GH_TOKEN: ${{ github.token }}
+ DEFINITIONS_SHA: ${{ inputs.definitions_sha }}
+ run: $UV_RUN tests/clear_signing_tests/nightly.py resolve-inputs ${DEFINITIONS_SHA:+--definitions-sha "$DEFINITIONS_SHA"} -o clear-signing-inputs/inputs.json
+ - name: Checkout the registry tests
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # actions/checkout@v6.0.2
+ with:
+ repository: ethereum/clear-signing-erc7730-registry
+ ref: ${{ steps.inputs.outputs.registry_sha }}
+ path: clear-signing-inputs/registry
+ sparse-checkout: /registry/*/testsv2/
+ sparse-checkout-cone-mode: false
+ persist-credentials: false
+ - name: Configure aws credentials
+ uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # aws-actions/configure-aws-credentials@v6.0.0
+ with:
+ role-to-assume: arn:aws:iam::538326561891:role/gh_actions_deploy_dev_firmware_data
+ aws-region: eu-central-1
+ - name: Download the unsigned dev definitions (mainnet)
+ run: |
+ aws s3 sync --only-show-errors \
+ s3://dev.firmware.sldev.cz/unsigned/v2/definitions/eth/chain-id/1/ \
+ clear-signing-inputs/definitions/eth/chain-id/1/
+ - name: Check the dev definitions
+ run: $UV_RUN tests/clear_signing_tests/nightly.py check-definitions clear-signing-inputs/definitions
+ - name: Fetch the screens of the last nightly run
+ env:
+ GH_TOKEN: ${{ github.token }}
+ MODELS: ${{ steps.models.outputs.models }}
+ run: |
+ $UV_RUN tests/clear_signing_tests/nightly.py fetch-baseline \
+ --models "$MODELS" \
+ -o clear-signing-inputs/fixtures.json \
+ --sources clear-signing-inputs/baseline.json
+ - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # actions/upload-artifact@v7.0.0
+ with:
+ name: clear-signing-inputs
+ path: clear-signing-inputs
+ retention-days: 7
+
+ core_clear_signing_test:
+ # The job id is part of the UI report and fixtures results names on S3,
+ # see JOB in tests/clear_signing_tests/nightly.py; it must not contain "-".
+ name: Clear signing tests (${{ matrix.model }})
+ runs-on: ubuntu-latest
+ needs: prepare
+ timeout-minutes: 120
+ permissions:
+ contents: read # for actions/checkout
+ id-token: write # for fetching the OIDC token for AWS (UI report)
+ strategy:
+ fail-fast: false
+ matrix:
+ model: ${{ fromJSON(needs.prepare.outputs.models) }}
+ env:
+ TREZOR_MODEL: ${{ matrix.model }}
+ BITCOIN_ONLY: 0
+ PYOPT: 0
+ TESTOPTS: "--durations 10 --session-timeout 5400" # pytest global timeout
+ TREZOR_CLEAR_SIGNING_REGISTRY: ${{ github.workspace }}/clear-signing-inputs/registry
+ TREZOR_CLEAR_SIGNING_DEFINITIONS: ${{ github.workspace }}/clear-signing-inputs/definitions
+ CLEAR_SIGNING_FIXTURES: ${{ github.workspace }}/clear-signing-inputs/fixtures.json
+ steps:
+ - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # actions/checkout@v6.0.2
+ with:
+ submodules: recursive
+ persist-credentials: false
+ - uses: $/.github/actions/environment
+ - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # actions/download-artifact@v8.0.0
+ with:
+ name: clear-signing-inputs
+ path: clear-signing-inputs
+ - run: $UV_RUN make -C core build_unix_frozen
+ - name: Run clear signing tests
+ run: $UV_RUN make -C core test_emu_clear_signing_ui
+ - run: tail -v -n50 tests/trezor*.log || true
+ if: failure()
+ - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # actions/upload-artifact@v7.0.0
+ with:
+ name: clear-signing-results-${{ matrix.model }}
+ path: tests/clear_signing_tests/results
+ retention-days: 7
+ if: always()
+ - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # actions/upload-artifact@v7.0.0
+ with:
+ name: clear-signing-logs-${{ matrix.model }}
+ path: |
+ tests/trezor*.log
+ tests/junit.xml
+ retention-days: 7
+ if: always()
+ - uses: $/.github/actions/ui-report
+ with:
+ model: ${{ matrix.model }}
+ lang: en
+ status: ${{ job.status }}
+ # the next run compares with all of these screens
+ all-screens: true
+ if: always()
+ continue-on-error: true
+
+ summary:
+ name: Summary
+ runs-on: ubuntu-latest
+ timeout-minutes: 30
+ needs: [prepare, core_clear_signing_test]
+ if: always() && needs.prepare.result == 'success'
+ steps:
+ - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # actions/checkout@v6.0.2
+ with:
+ submodules: recursive
+ persist-credentials: false
+ - uses: $/.github/actions/environment
+ - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # actions/download-artifact@v8.0.0
+ with:
+ name: clear-signing-inputs
+ path: clear-signing-inputs
+ - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # actions/download-artifact@v8.0.0
+ with:
+ pattern: clear-signing-results-*
+ path: clear-signing-results
+ merge-multiple: true
+ - name: Write the summary
+ env:
+ MODELS: ${{ needs.prepare.outputs.models }}
+ run: |
+ $UV_RUN tests/clear_signing_tests/nightly.py summarize \
+ --inputs clear-signing-inputs/inputs.json \
+ --registry clear-signing-inputs/registry \
+ --definitions clear-signing-inputs/definitions \
+ --results clear-signing-results \
+ --baseline clear-signing-inputs/baseline.json \
+ --run-id "$GITHUB_RUN_ID" \
+ --models "$MODELS" >> "$GITHUB_STEP_SUMMARY"
+
+ notify_scheduled_failure_to_slack:
+ name: Notify Slack on nightly failure
+ runs-on: ubuntu-latest
+ timeout-minutes: 5
+ # Only nightly runs; manual runs are watched by whoever started them.
+ if: failure() && github.event_name == 'schedule'
+ needs: [prepare, core_clear_signing_test, summary]
+ permissions:
+ actions: read # to list the jobs of this run
+ steps:
+ - name: Notify scheduled job failure to Slack
+ env:
+ GH_TOKEN: ${{ github.token }}
+ GITHUB_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+ COMMIT_URL: ${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}
+ COMMIT_SHA: ${{ github.sha }}
+ SLACK_WEBHOOK_URL: ${{ secrets.SLACK_CLEAR_SIGNING_WEBHOOK }} # zizmor: ignore[secrets-outside-env] a repository secret, like SLACK_HW_TESTS_WEBHOOK in core-hw.yml
+ run: |
+ if [ -z "$SLACK_WEBHOOK_URL" ]; then
+ echo "::warning::SLACK_CLEAR_SIGNING_WEBHOOK is not set, not notifying Slack"
+ exit 0
+ fi
+ JOBS=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/attempts/$GITHUB_RUN_ATTEMPT/jobs?per_page=100")
+ # cancelled also covers jobs that hit timeout-minutes
+ FAILED=$(jq -r '.jobs[] | select(.conclusion == "failure" or .conclusion == "cancelled") | "• <\(.html_url)|\(.name)>"' <<< "$JOBS")
+ # count only test jobs; must match the names of the `prepare` and `summary` jobs
+ COUNTS=$(jq -r '[.jobs[] | select(.name != "Resolve inputs" and .name != "Summary" and .conclusion != null and .conclusion != "skipped")]
+ | "\(map(select(.conclusion != "success")) | length) of \(length) test jobs failed"' <<< "$JOBS")
+ HEADER="Nightly clear signing tests failed on \`$GITHUB_REF_NAME\` (<$COMMIT_URL|${COMMIT_SHA::10}>), $COUNTS: <$GITHUB_RUN_URL|GitHub run>"
+ jq -n --arg header "$HEADER" --arg failed "$FAILED" \
+ '{text: (if $failed == "" then $header else $header + "\n" + $failed end)}' \
+ | curl -sSf -X POST -H 'Content-type: application/json' --data @- "$SLACK_WEBHOOK_URL"
### ci/prepare_ui_artifacts.py
@@ -1,8 +1,11 @@
import shutil
import sys
+from collections.abc import Iterable
from multiprocessing import Pool
from pathlib import Path
+import click
+
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))
from tests.ui_tests.common import TestResult, get_current_fixtures # isort:skip
@@ -31,14 +34,29 @@ def create_zip(item: tuple[str, TestResult]) -> None:
shutil.make_archive(archive_path, "zip", result.test.actual_dir)
-def main():
+@click.command()
+@click.option(
+ "--all",
+ "all_screens",
+ is_flag=True,
+ help="Archive the screens of every test, also the failed and the new ones. For "
+ "tests whose next run compares with this one (tests/clear_signing_tests), so "
+ "that its report can download these screens to show the differences.",
+)
+def main(all_screens: bool) -> None:
with Pool() as pool:
all_results = list(TestResult.recent_results())
print(f"Hashing {len(all_results)} results")
- # deduplicate results by `actual_hash`, and skip failed tests
+ checked: Iterable[TestResult | None]
+ if all_screens:
+ checked = all_results
+ else:
+ # skip failed tests
+ checked = pool.imap_unordered(compute_hash, all_results)
+ # deduplicate results by `actual_hash`
results_map = {
str(ROOT / "ci/ui_test_records" / result.actual_hash): result
- for result in pool.imap_unordered(compute_hash, all_results)
+ for result in checked
if result is not None
}
print(f"Creating {len(results_map)} ZIP files")
### core/Makefile
@@ -8,6 +8,7 @@
test_emu_click test_emu_click_ui test_emu_click_ui_multicore \
test_emu_persistence test_emu_persistence_ui test_emu_upgrade \
test_emu_ui test_emu_ui_multicore test_emu_ui_record test_emu_ui_record_multicore \
+ test_emu_clear_signing_ui test_emu_clear_signing_ui_record \
test_emu_prodtest \
pylint mypy typecheck pyright clippy \
audit_rust vet_rust \
@@ -199,6 +200,22 @@ test_emu_ui_record_multicore: ## quickly record all screens
make test_emu_ui_multicore || echo "All errors are recorded in fixtures.json"
../tests/update_fixtures.py local --remove-missing
+# Nightly clear-signing tests, with their own UI fixtures; see docs/tests/clear-signing-tests.md.
+# They need TREZOR_CLEAR_SIGNING_REGISTRY and TREZOR_CLEAR_SIGNING_DEFINITIONS. CI sets
+# CLEAR_SIGNING_FIXTURES to the screens of the last nightly run (nightly.py fetch-baseline).
+CLEAR_SIGNING_FIXTURES ?= $(TESTPATH)/clear_signing_tests/fixtures.json
+CLEAR_SIGNING_UI_FIXTURES = TREZOR_UI_FIXTURES=$(CLEAR_SIGNING_FIXTURES)
+
+test_emu_clear_signing_ui: ## run the clear-signing registry tests with UI testing using multiple cores
+ rm -rf $(TESTPATH)/clear_signing_tests/results
+ $(CLEAR_SIGNING_UI_FIXTURES) $(PYTEST) -n $(MULTICORE) $(TESTPATH)/clear_signing_tests $(TESTOPTS) \
+ --timeout $(PYTEST_TIMEOUT) --ui=test --ui-check-missing --lang=en \
+ --control-emulators --model=core --random-order-seed=$(RANDOM)
+
+test_emu_clear_signing_ui_record: ## record the screens of all clear-signing registry tests
+ make test_emu_clear_signing_ui || echo "All errors are recorded in the fixtures"
+ $(CLEAR_SIGNING_UI_FIXTURES) ../tests/update_fixtures.py local --remove-missing
+
test_emu_prodtest: ## run prodtest device tests
$(PRODTEST_EMU) --headless -c $(PYTEST) $(TESTPATH)/prodtest_tests $(TESTOPTS)
### docs/SUMMARY.md
@@ -53,6 +53,7 @@
- [Upgrade Tests](tests/upgrade-tests.md)
- [UI Tests](tests/ui-tests.md)
- [Click Tests](tests/click-tests.md)
+ - [Clear-signing Tests](tests/clear-signing-tests.md)
- [Miscellaneous](misc/index.md)
- [Affected third-parties](misc/third-parties.md)
- [Changelog](misc/changelog.md)
### docs/tests/clear-signing-tests.md
@@ -0,0 +1,133 @@
+# Clear-signing tests
+
+Nightly UI tests of Ethereum clear signing (ERC-7730). They sign the test transactions of
+the [clear-signing registry](https://github.com/ethereum/clear-signing-erc7730-registry)
+on the emulator, with the display formats of the unsigned dev definitions of
+[trezor/definitions](https://github.com/trezor/definitions). Only mainnet, only English,
+on all core models.
+
+They are separate from the [device tests](device-tests.md): they live in
+`tests/clear_signing_tests/`, run in their own workflow
+(`.github/workflows/core-clear-signing.yml`), and do not use the device tests' UI fixtures.
+Each run compares its screens with those of the **last nightly run**, so it shows what
+changed since then, and its own screens are what the next run compares with.
+
+## Inputs
+
+The test cases and the definitions must match, so both come from one trezor/definitions
+commit: the one built by the last run of its "[Deploy] unsigned dev firmware
+definitions" workflow (from any branch).
+
+- **Definitions:** that deploy, `s3://dev.firmware.sldev.cz/unsigned/v2/definitions/`
+ (from the internal network also `https://dev.firmware.sldev.cz/unsigned/v2/definitions/`).
+- **Test cases:** the `registry/*/testsv2/calldata-*.tests.json` files of the registry
+ commit that this trezor/definitions commit pins as a submodule.
+
+So the tests change exactly when someone deploys the unsigned dev definitions.
+`tests/clear_signing_tests/nightly.py resolve-inputs` finds the commits. It fails when the
+last deploy did not succeed, since a failed deploy may have left the bucket half-synced.
+`nightly.py check-definitions` fails when the deployed definitions are older than the
+minimum that the firmware accepts
+(`common/defs/ethereum/released-definitions-timestamp.json`). In both cases, re-run the
+deploy.
+
+## What fails
+
+Each registry case with a chain ID 1 transaction is one test. A case is skipped when the
+dev definitions have no display format for its contract and function (the definitions
+converter dropped it, or it is not deployed there), or when its calldata is longer than
+the firmware clear-signs (6144 bytes).
+
+A test fails when:
+
+- signing fails,
+- its screens differ from those of the last nightly run, or the test did not run then
+ (`UI_FAILED`, `UI_MISSING`),
+- a test of the last nightly run no longer runs, e.g. because a display format disappeared.
+
+Blind signing (the firmware shows the `confirm_data` screen although the dev
+definitions have a display format) is not a failure. It is recorded like clear signing,
+and listed in the summary. A case that changes from one to the other fails on its
+screens.
+
+Since the next run compares with this one, a UI change fails only the one run that
+brings it. In the UI report, a new test shows *missing* in place of the previous screens,
+and a test that no longer runs (under "Removed tests") shows its previous screens next
+to *missing* ones.
+
+The registry's `expected` texts (intent, labels and values) are also looked up on the
+screens, but a text that is not found is only reported: the firmware formats some values
+differently.
+
+## Baseline
+
+`tests/clear_signing_tests/nightly.py fetch-baseline` writes the fixtures that a run
+compares with. For each model, they are the screens of the last finished scheduled run
+on `main` that has them: the `fixtures.results.json` of its UI report on
+`data.trezor.io`. The UI report of these tests uploads the screens of every test, not
+only of the passing ones, so that the next report can show the differences.
+
+A model that none of the last 10 nightly runs has screens for (e.g. before the first
+run) falls back to the base fixtures in `tests/clear_signing_tests/fixtures.json`, recorded
+locally (see below). Re-record them only when that fallback matters, e.g. after a long
+outage of the nightly runs.
+
+## Results
+
+The summary of the workflow run has, per model, the counts, links to the
+[UI report](ui-tests.md) and to the run it was compared with, and lists the skipped
+cases, the failures, and the expected texts that were not found. The UI report is the
+same as that of the device tests: `index.html` lists the failed, new and passed tests,
+each failed test shows its screens next to those of the last run, with a diff, and
+`differing_screens.html` has all the changed screens on one page. The per-case results
+(screens' texts, ButtonRequests) are in the `clear-signing-results-<model>` artifacts.
+
+## Running locally
+
+From the internal network, the dev definitions can be used by their URL. Check out the
+registry at the commit from `resolve_inputs.py`:
+
+```sh
+tests/clear_signing_tests/nightly.py resolve-inputs # prints registry_sha
+git clone --filter=blob:none --sparse https://github.com/ethereum/clear-signing-erc7730-registry /tmp/registry
+git -C /tmp/registry sparse-checkout set --no-cone '/registry/*/testsv2/'
+git -C /tmp/registry checkout <registry_sha>
+
+export TREZOR_CLEAR_SIGNING_REGISTRY=/tmp/registry
+export TREZOR_CLEAR_SIGNING_DEFINITIONS=https://dev.firmware.sldev.cz/unsigned/v2/definitions/
+```
+
+Then, with an emulator built with debuglink (e.g. `xtask build firmware --emulator -m T3W1 -p test`):
+
+```sh
+# a few cases against a temporary emulator
+TREZOR_UI_FIXTURES=tests/clear_signing_tests/fixtures.json \
+ ./core/emu.py -h -t -q -c pytest tests/clear_signing_tests -k 1inch --ui=test --lang=en
+# all cases, like CI
+make -C core test_emu_clear_signing_ui
+# then open the report, with the diffs
+TREZOR_UI_FIXTURES=tests/clear_signing_tests/fixtures.json ./tests/show_results.py
+```
+
+Always set `TREZOR_UI_FIXTURES` when running these tests with `--ui`, and when opening
+their report with `show_results.py`, otherwise their screens go to
+`tests/ui_tests/fixtures.json`. The make targets do it, and the end of the pytest output
+prints the `show_results.py` command with it.
+
+To compare with the last nightly run instead of the base fixtures:
+
+```sh
+tests/clear_signing_tests/nightly.py fetch-baseline -o /tmp/clear-signing-fixtures.json
+make -C core test_emu_clear_signing_ui CLEAR_SIGNING_FIXTURES=/tmp/clear-signing-fixtures.json
+TREZOR_UI_FIXTURES=/tmp/clear-signing-fixtures.json ./tests/show_results.py
+```
+
+To re-record the base fixtures, record each model in turn (each run replaces only the
+screens of its own model):
+
+```sh
+for MODEL in T2T1 T3B1 T3T1 T3W1; do
+ xtask build firmware --emulator -m $MODEL -p test
+ make -C core test_emu_clear_signing_ui_record
+done
+```
### docs/tests/index.md
@@ -25,6 +25,12 @@ See [ui-tests.md](ui-tests.md) for more info.
Click tests are a next-generation of the Device tests. The tests are quite similar, but they are capable of imitating user's interaction with the screen.
+### Clear-signing tests
+
+Nightly UI tests that sign the test transactions of the ERC-7730 clear-signing registry with the unsigned dev definitions. They are separate from the device tests, and compare their screens with those of the previous nightly run.
+
+See [clear-signing-tests.md](clear-signing-tests.md) for more info.
+
## Fido tests
Implement U2F/FIDO2 tests.
### tests/clear_signing_tests/.gitignore
@@ -0,0 +1,2 @@
+# Per-case results of the last local run, see registry.py
+results/
### tests/clear_signing_tests/__init__.py
[binary or diff unavailable]
### tests/clear_signing_tests/fixtures.json
[binary or diff unavailable]
### tests/clear_signing_tests/nightly.py
@@ -0,0 +1,443 @@
+#!/usr/bin/env python3
+
+# This file is part of the Trezor project.
+#
+# Copyright (C) SatoshiLabs and contributors
+#
+# This library is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3
+# as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the License along with this library.
+# If not, see <https://www.gnu.org/licenses/lgpl-3.0.html>.
+
+"""The steps of the nightly clear-signing workflow around the tests themselves.
+
+See docs/tests/clear-signing-tests.md.
+
+- `resolve-inputs`: find the trezor/definitions and registry commits to test.
+- `check-definitions`: check that a copy of the dev definitions is usable.
+- `fetch-baseline`: get the UI fixtures to compare with, the screens of the last nightly run.
+- `summarize`: write the results as Markdown (summary.md.mako), for $GITHUB_STEP_SUMMARY.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from collections import Counter, defaultdict
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+
+import click
+import requests
+from mako.template import Template
+from registry import definitions_source, load_cases, read_results
+
+from trezorlib import definitions
+
+ROOT = Path(__file__).resolve().parents[2]
+
+
+@click.group()
+def cli() -> None:
+ pass
+
+
+# --- resolve-inputs --------------------------------------------------------------
+
+DEFINITIONS_REPO = "trezor/definitions"
+DEPLOY_WORKFLOW = "deploy-dev-definitions.yaml"
+REGISTRY_SUBMODULE = "ethereum/clear-signing-erc7730-registry"
+REGISTRY_REPO = "ethereum/clear-signing-erc7730-registry"
+
+
+def _github_get(path: str) -> dict:
+ headers = {"Accept": "application/vnd.github+json"}
+ token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN")
+ if token:
+ headers["Authorization"] = f"Bearer {token}"
+ response = requests.get(
+ f"https://api.github.com/{path}", headers=headers, timeout=30
+ )
+ response.raise_for_status()
+ return response.json()
+
+
+def _last_deploy() -> dict:
+ runs = _github_get(
+ f"repos/{DEFINITIONS_REPO}/actions/workflows/{DEPLOY_WORKFLOW}/runs?per_page=1"
+ )["workflow_runs"]
+ if not runs:
+ raise click.ClickException(f"{DEFINITIONS_REPO} has no {DEPLOY_WORKFLOW} runs")
+ run = runs[0]
+ url = run["html_url"]
+ # A run that did not finish may have left the bucket half-synced (`aws s3 sync --delete`).
+ if run["status"] != "completed":
+ raise click.ClickException(
+ f"The last unsigned dev deploy is still {run['status']}: {url}"
+ )
+ if run["conclusion"] != "success":
+ raise click.ClickException(
+ f"The last unsigned dev deploy ended with '{run['conclusion']}', so the dev "
+ f"definitions may be incomplete: {url}\nRe-run the deploy."
+ )
+ return {
+ "definitions_sha": run["head_sha"],
+ "definitions_ref": run["head_branch"],
+ "deploy_run_url": url,
+ "deployed_at": run["updated_at"],
+ }
+
+
+@cli.command()
+@click.option(
+ "--definitions-sha",
+ help="Use this trezor/definitions commit instead of the last unsigned dev deploy.",
+)
+@click.option(
+ "-o",
+ "--output",
+ type=click.Path(dir_okay=False, path_type=Path),
+ help="Write the inputs as JSON to this file.",
+)
+def resolve_inputs(definitions_sha: str | None, output: Path | None) -> None:
+ """Resolve the inputs of the nightly clear-signing tests.
+
+ The tests run the ERC-7730 registry cases against the unsigned dev definitions. Both
+ must come from the same trezor/definitions commit: the one that the last run of its
+ "[Deploy] unsigned dev firmware definitions" workflow built (from any branch). The
+ registry is the submodule that this commit pins.
+
+ Writes the result as JSON, and as step outputs when $GITHUB_OUTPUT is set.
+ """
+ if definitions_sha:
+ inputs = {
+ "definitions_sha": definitions_sha,
+ "definitions_ref": None,
+ "deploy_run_url": None,
+ "deployed_at": None,
+ }
+ else:
+ inputs = _last_deploy()
+
+ submodule = _github_get(
+ f"repos/{DEFINITIONS_REPO}/contents/{REGISTRY_SUBMODULE}"
+ f"?ref={inputs['definitions_sha']}"
+ )
+ if submodule.get("type") != "submodule":
+ raise click.ClickException(
+ f"{REGISTRY_SUBMODULE} is not a submodule at {inputs['definitions_sha']}"
+ )
+ inputs["registry_repo"] = REGISTRY_REPO
+ inputs["registry_sha"] = submodule["sha"]
+
+ text = json.dumps(inputs, indent=2)
+ click.echo(text)
+ if output:
+ output.parent.mkdir(parents=True, exist_ok=True)
+ output.write_text(text + "\n")
+ if github_output := os.environ.get("GITHUB_OUTPUT"):
+ with open(github_output, "a") as f:
+ for key in ("definitions_sha", "registry_sha"):
+ f.write(f"{key}={inputs[key]}\n")
+
+
+# --- check-definitions -----------------------------------------------------------
+
+TIMESTAMPS_FILE = (
+ ROOT / "common" / "defs" / "ethereum" / "released-definitions-timestamp.json"
+)
+FORMAT_VERSION = "2"
+
+
+def _utc(timestamp: int) -> str:
+ return datetime.fromtimestamp(timestamp, timezone.utc).isoformat()
+
+
+@cli.command()
+@click.argument(
+ "definitions_dir",
+ type=click.Path(exists=True, file_okay=False, path_type=Path),
+)
+@click.option("--chain-id", type=int, default=1, show_default=True)
+def check_definitions(definitions_dir: Path, chain_id: int) -> None:
+ """Check DEFINITIONS_DIR, a copy of `<deploy>/v2/definitions/`.
+
+ The firmware rejects definitions older than its minimum data version, which comes
+ from common/defs/ethereum/released-definitions-timestamp.json. Since the dev
+ definitions are only deployed on demand, they go stale whenever that timestamp
+ moves past them.
+ """
+ chain_dir = definitions_dir / "eth" / "chain-id" / str(chain_id)
+ files = sorted(chain_dir.rglob("*.dat"))
+ if not files:
+ raise click.ClickException(
+ f"No definitions in {chain_dir}. Did the download from S3 fail?"
+ )
+ if not (chain_dir / "network.dat").exists():
+ raise click.ClickException(f"No network definition in {chain_dir}.")
+
+ min_timestamp = int(
+ datetime.fromisoformat(
+ json.loads(TIMESTAMPS_FILE.read_text())[FORMAT_VERSION]
+ ).timestamp()
+ )
+ kinds: Counter[str] = Counter()
+ stale: list[Path] = []
+ timestamps: list[int] = []
+ for path in files:
+ payload = definitions.Definition.parse(path.read_bytes()).payload
+ if payload.version.decode() != FORMAT_VERSION:
+ raise click.ClickException(
+ f"{path}: format version {payload.version!r}, expected {FORMAT_VERSION}"
+ )
+ kinds[payload.data_type.name] += 1
+ timestamps.append(payload.timestamp)
+ if payload.timestamp < min_timestamp:
+ stale.append(path)
+
+ for kind, count in sorted(kinds.items()):
+ click.echo(f"{kind}: {count}")
+ click.echo(f"Timestamps: {_utc(min(timestamps))} .. {_utc(max(timestamps))}")
+ click.echo(f"Firmware minimum: {_utc(min_timestamp)}")
+
+ if stale:
+ raise click.ClickException(
+ f"{len(stale)} of {len(files)} dev definitions are older than the minimum "
+ f"that the firmware accepts ({_utc(min_timestamp)}), e.g. {stale[0].name}. "
+ "Re-run '[Deploy] unsigned dev firmware definitions' in trezor/definitions."
+ )
+
+
+# --- fetch-baseline --------------------------------------------------------------
+
+FIRMWARE_REPO = "trezor/trezor-firmware"
+WORKFLOW = "core-clear-signing.yml"
+JOB = "core_clear_signing_test"
+LANG = "en"
+MODELS = ["T2T1", "T3B1", "T3T1", "T3W1"]
+REPORT_URL = (
+ "https://data.trezor.io/dev/firmware/ui_report/{run}/{model}-{lang}-{job}-{page}"
+)
+BASE_FIXTURES = Path(__file__).resolve().parent / "fixtures.json"
+
+
+def _run_fixtures(run_id: int, model: str) -> dict | None:
+ """The screens that a run recorded for `model`, from its UI report."""
+ url = REPORT_URL.format(
+ run=run_id, model=model, lang=LANG, job=JOB, page="fixtures.results.json"
+ )
+ response = requests.get(url, timeout=30)
+ # S3 answers 403 for objects that do not exist.
+ if response.status_code in (403, 404):
+ return None
+ response.raise_for_status()
+ return response.json().get(model, {})
+
+
+@cli.command()
+@click.option(
+ "-o",
+ "--output",
+ type=click.Path(dir_okay=False, path_type=Path),
+ required=True,
+ help="Write the fixtures to this file.",
+)
+@click.option(
+ "--models",
+ "models_json",
+ default=json.dumps(MODELS),
+ show_default=True,
+ help="Models as a JSON list.",
+)
+@click.option("--branch", default="main", show_default=True)
+@click.option(
+ "--max-runs",
+ type=int,
+ default=10,
+ show_default=True,
+ help="How many earlier runs to look through.",
+)
+@click.option(
+ "--sources",
+ type=click.Path(dir_okay=False, path_type=Path),
+ help="Write which run the fixtures of each model come from as JSON.",
+)
+def fetch_baseline(
+ output: Path,
+ models_json: str,
+ branch: str,
+ max_runs: int,
+ sources: Path | None,
+) -> None:
+ """Get the UI fixtures to compare a run with: the screens of the last nightly run.
+
+ For each model, these are the screens of the last finished scheduled run on BRANCH
+ that has them (its `fixtures.results.json` in the UI report). So every nightly run
+ shows what changed since the one before, and its own screens are the baseline of
+ the next one. Models that no recent run has screens for keep the base fixtures,
+ tests/clear_signing_tests/fixtures.json.
+ """
+ models: list[str] = json.loads(models_json)
+ try:
+ runs = _github_get(
+ f"repos/{FIRMWARE_REPO}/actions/workflows/{WORKFLOW}/runs"
+ f"?branch={branch}&event=schedule&status=completed&per_page={max_runs}"
+ )["workflow_runs"]
+ except requests.HTTPError as e:
+ # The workflow is not on the default branch yet.
+ if e.response is None or e.response.status_code != 404:
+ raise
+ runs = []
+
+ fixtures = json.loads(BASE_FIXTURES.read_text())
+ used: dict[str, dict | None] = {}
+ for model in models:
+ for run in runs:
+ run_fixtures = _run_fixtures(run["id"], model)
+ if run_fixtures is None:
+ continue
+ fixtures[model] = run_fixtures
+ used[model] = {
+ "run_id": run["id"],
+ "run_number": run["run_number"],
+ "run_url": run["html_url"],
+ "created_at": run["created_at"],
+ }
+ count = len(run_fixtures.get("clear_signing_tests", {}))
+ click.echo(f"{model}: {count} tests from run {run['html_url']}")
+ break
+ else:
+ used[model] = None
+ click.echo(
+ f"{model}: no run in the last {len(runs)}, using {BASE_FIXTURES.name}"
+ )
+
+ output.parent.mkdir(parents=True, exist_ok=True)
+ output.write_text(json.dumps(fixtures, indent=0, sort_keys=True) + "\n")
+ if sources:
+ sources.parent.mkdir(parents=True, exist_ok=True)
+ sources.write_text(json.dumps(used, indent=2) + "\n")
+
+
+# --- summarize -------------------------------------------------------------------
+
+SUMMARY_TEMPLATE = Path(__file__).resolve().parent / "summary.md.mako"
+# Longer lists are cut, the step summary has a size limit.
+MAX_LINES = 300
+
+
+@dataclass
+class _ModelRow:
+ model: str
+ counts: Counter[str]
+ report_url: str
+ differing_url: str
+ # the run whose screens this one was compared with, None for the base fixtures
+ baseline: dict | None
+
+
+@cli.command()
+@click.option(
+ "--inputs",
+ "inputs_file",
+ type=click.Path(exists=True, dir_okay=False, path_type=Path),
+ required=True,
+)
+@click.option(
+ "--registry",
+ type=click.Path(exists=True, file_okay=False, path_type=Path),
+ required=True,
+)
+@click.option(
+ "--definitions", required=True, help="Copy or URL of the dev definitions."
+)
+@click.option(
+ "--results",
+ "results_dir",
+ type=click.Path(file_okay=False, path_type=Path),
+ required=True,
+)
+@click.option(
+ "--baseline",
+ "baseline_file",
+ type=click.Path(exists=True, dir_okay=False, path_type=Path),
+ help="The `fetch-baseline --sources` file.",
+)
+@click.option("--run-id", required=True)
+@click.option(
+ "--models", "models_json", required=True, help="Tested models as a JSON list."
+)
+def summarize(
+ inputs_file: Path,
+ registry: Path,
+ definitions: str,
+ results_dir: Path,
+ baseline_file: Path | None,
+ run_id: str,
+ models_json: str,
+) -> None:
+ """Summarize the test results as Markdown, for $GITHUB_STEP_SUMMARY."""
+ inputs = json.loads(inputs_file.read_text())
+ baseline = json.loads(baseline_file.read_text()) if baseline_file else {}
+ models: list[str] = json.loads(models_json)
+ cases = load_cases(registry, definitions_source(definitions))
+ to_run = [case for case in cases if not case.skip_reason]
+ results = read_results(results_dir) if results_dir.exists() else {}
+
+ rows: list[_ModelRow] = []
+ # test_id -> ["T3W1: error DataError: ...", ...]
+ problems: dict[str, list[str]] = defaultdict(list)
+ # test_id -> models
+ blind: dict[str, list[str]] = defaultdict(list)
+ # test_id -> (models, missing texts)
+ not_found: dict[str, tuple[list[str], list[str]]] = {}
+ for model in models:
+ model_results = results.get(model, {})
+ counts: Counter[str] = Counter()
+ for case in to_run:
+ result = model_results.get(case.test_id)
+ status = result["status"] if result else "not run"
+ counts[status] += 1
+ if status == "clear_signed":
+ if result and (missing := result.get("missing")):
+ counts["text not found"] += 1
+ entry = not_found.setdefault(case.test_id, ([], missing))
+ entry[0].append(model)
+ elif status == "blind":
+ blind[case.test_id].append(model)
+ else:
+ detail = (result or {}).get("message", "")
+ problems[case.test_id].append(f"{model}: {status} {detail}".strip())
+ report_url, differing_url = (
+ REPORT_URL.format(run=run_id, model=model, lang=LANG, job=JOB, page=page)
+ for page in ("index.html", "differing_screens.html")
+ )
+ rows.append(
+ _ModelRow(model, counts, report_url, differing_url, baseline.get(model))
+ )
+
+ skipped = Counter(case.skip_reason for case in cases if case.skip_reason)
+ summary = Template(filename=str(SUMMARY_TEMPLATE), strict_undefined=True).render(
+ inputs=inputs,
+ total=len(cases),
+ run=len(to_run),
+ rows=rows,
+ skipped=skipped.most_common(),
+ problems=sorted(problems.items()),
+ blind=sorted(blind.items()),
+ not_found=sorted(not_found.items()),
+ max_lines=MAX_LINES,
+ )
+ click.echo(summary, nl=False)
+
+
+if __name__ == "__main__":
+ cli()
### tests/clear_signing_tests/registry.py
@@ -0,0 +1,439 @@
+# This file is part of the Trezor project.
+#
+# Copyright (C) SatoshiLabs and contributors
+#
+# This library is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3
+# as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the License along with this library.
+# If not, see <https://www.gnu.org/licenses/lgpl-3.0.html>.
+
+"""Test cases of the ERC-7730 clear-signing registry, for the nightly clear-signing tests.
+
+The cases come from the `registry/<entity>/testsv2/calldata-*.tests.json` files of
+https://github.com/ethereum/clear-signing-erc7730-registry, at the commit that the last
+unsigned dev deploy of trezor/definitions used. The network, display formats and tokens
+come from that deploy: a local copy of its `eth/chain-id/<id>/...` tree (CI), or its URL
+(only reachable from the internal network).
+
+This module does not depend on pytest, so that `nightly.py summarize` can load
+the same cases.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import typing as t
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import requests
+
+from trezorlib.definitions import FilesystemSource, Source, UrlSource
+
+if t.TYPE_CHECKING:
+ from trezorlib.debuglink import LayoutContent
+ from trezorlib.testing.common import BRGeneratorType
+
+REGISTRY_ENV = "TREZOR_CLEAR_SIGNING_REGISTRY"
+DEFINITIONS_ENV = "TREZOR_CLEAR_SIGNING_DEFINITIONS"
+DEV_DEFINITIONS_URL = "https://dev.firmware.sldev.cz/unsigned/v2/definitions/"
+
+RESULTS_DIR = Path(__file__).resolve().parent / "results"
+
+# Only mainnet for now.
+CHAIN_IDS = (1,)
+
+# Calldata longer than this is never clear-signed: see `_MAX_DATA_STORED` in
+# core/src/apps/ethereum/sign_tx.py.
+MAX_CLEAR_SIGNED_DATA = 6144
+
+# The blind-signing fallback always starts with this ButtonRequest, on every layout:
+# see `DataChunkConfirmer` in core/src/apps/ethereum/helpers.py.
+BLIND_SIGNING_BR_NAME = "confirm_data"
+
+TEST_FILES_GLOB = "registry/*/testsv2/calldata-*.tests.json"
+
+
+# --- RLP -------------------------------------------------------------------------
+
+RlpItem = bytes | list["RlpItem"]
+
+
+def rlp_decode(data: bytes) -> RlpItem:
+ item, end = _rlp_item(data, 0)
+ if end != len(data):
+ raise ValueError("trailing data after the RLP item")
+ return item
+
+
+def _rlp_payload(data: bytes, pos: int, offset: int) -> tuple[int, int]:
+ """Return the start and the length of the payload of the item at `pos`."""
+ prefix = data[pos]
+ if prefix <= offset + 55:
+ start, length = pos + 1, prefix - offset
+ else:
+ length_of_length = prefix - offset - 55
+ start = pos + 1 + length_of_length
+ if start > len(data):
+ raise ValueError("truncated RLP")
+ length = int.from_bytes(data[pos + 1 : start], "big")
+ if start + length > len(data):
+ raise ValueError("truncated RLP")
+ return start, length
+
+
+def _rlp_item(data: bytes, pos: int) -> tuple[RlpItem, int]:
+ if pos >= len(data):
+ raise ValueError("truncated RLP")
+ prefix = data[pos]
+ if prefix < 0x80:
+ return data[pos : pos + 1], pos + 1
+ if prefix < 0xC0:
+ start, length = _rlp_payload(data, pos, 0x80)
+ return data[start : start + length], start + length
+ start, length = _rlp_payload(data, pos, 0xC0)
+ end = start + length
+ items: list[RlpItem] = []
+ pos = start
+ while pos < end:
+ item, pos = _rlp_item(data, pos)
+ items.append(item)
+ if pos != end:
+ raise ValueError("RLP list length mismatch")
+ return items, end
+
+
+def _rlp_bytes(item: RlpItem) -> bytes:
+ if not isinstance(item, bytes):
+ raise ValueError("expected an RLP string, got a list")
+ return item
+
+
+def _rlp_list(item: RlpItem) -> list[RlpItem]:
+ if not isinstance(item, list):
+ raise ValueError("expected an RLP list, got a string")
+ return item
+
+
+def _rlp_int(item: RlpItem) -> int:
+ return int.from_bytes(_rlp_bytes(item), "big")
+
+
+# --- Transactions ----------------------------------------------------------------
+
+
+class UnsupportedTransaction(ValueError):
+ def __init__(self, reason: str, chain_id: int | None = None) -> None:
+ super().__init__(reason)
+ self.chain_id = chain_id
+
+
+@dataclass(frozen=True)
+class Transaction:
+ tx_type: int # 0 (legacy, EIP-155) or 2 (EIP-1559)
+ chain_id: int
+ nonce: int
+ gas_limit: int
+ to: str # 0x-prefixed, lowercase
+ value: int
+ data: bytes
+ gas_price: int = 0 # legacy only
+ max_gas_fee: int = 0 # EIP-1559 only
+ max_priority_fee: int = 0 # EIP-1559 only
+ access_list: tuple[tuple[str, tuple[bytes, ...]], ...] = ()
+
+
+def _address(item: RlpItem, chain_id: int) -> str:
+ address = _rlp_bytes(item)
+ if not address:
+ raise UnsupportedTransaction("contract creation", chain_id)
+ if len(address) != 20:
+ raise ValueError(f"invalid address length {len(address)}")
+ return "0x" + address.hex()
+
+
+def decode_raw_tx(raw_tx: str) -> Transaction:
+ """Decode an unsigned or signed legacy (EIP-155) or EIP-1559 transaction.
+
+ Raises `UnsupportedTransaction` (with the chain ID, if known) for transactions
+ that cannot be signed by this test, and `ValueError` for malformed ones.
+ """
+ raw = bytes.fromhex(raw_tx.removeprefix("0x"))
+ if not raw:
+ raise ValueError("empty rawTx")
+
+ if raw[0] >= 0xC0:
+ fields = _rlp_list(rlp_decode(raw))
+ if len(fields) == 6:
+ raise UnsupportedTransaction("legacy transaction without chain ID")
+ if len(fields) != 9:
+ raise ValueError(f"legacy transaction with {len(fields)} fields")
+ v, r, s = (_rlp_int(f) for f in fields[6:9])
+ if r == 0 and s == 0:
+ chain_id = v # unsigned EIP-155
+ elif v >= 35:
+ chain_id = (v - 35) // 2
+ else:
+ raise UnsupportedTransaction("legacy transaction without chain ID")
+ nonce, gas_price, gas_limit, to, value, data = fields[:6]
+ return Transaction(
+ tx_type=0,
+ chain_id=chain_id,
+ nonce=_rlp_int(nonce),
+ gas_price=_rlp_int(gas_price),
+ gas_limit=_rlp_int(gas_limit),
+ to=_address(to, chain_id),
+ value=_rlp_int(value),
+ data=_rlp_bytes(data),
+ )
+
+ tx_type = raw[0]
+ if tx_type > 0x7F:
+ raise ValueError(f"invalid transaction type {tx_type:#x}")
+ fields = _rlp_list(rlp_decode(raw[1:]))
+ if not fields:
+ raise ValueError("empty typed transaction")
+ chain_id = _rlp_int(fields[0])
+ if tx_type != 2:
+ raise UnsupportedTransaction(f"transaction type {tx_type}", chain_id)
+ if len(fields) not in (9, 12):
+ raise ValueError(f"EIP-1559 transaction with {len(fields)} fields")
+ _, nonce, max_priority_fee, max_gas_fee, gas_limit, to, value, data = fields[:8]
+ access_list = []
+ for entry in _rlp_list(fields[8]):
+ address, storage_keys = _rlp_list(entry)
+ access_list.append(
+ (
+ _address(address, chain_id),
+ tuple(_rlp_bytes(key) for key in _rlp_list(storage_keys)),
+ )
+ )
+ return Transaction(
+ tx_type=2,
+ chain_id=chain_id,
+ nonce=_rlp_int(nonce),
+ max_priority_fee=_rlp_int(max_priority_fee),
+ max_gas_fee=_rlp_int(max_gas_fee),
+ gas_limit=_rlp_int(gas_limit),
+ to=_address(to, chain_id),
+ value=_rlp_int(value),
+ data=_rlp_bytes(data),
+ access_list=tuple(access_list),
+ )
+
+
+# --- Definitions -----------------------------------------------------------------
+
+
+class _DevUrlSource(UrlSource):
+ """The dev bucket answers 403, not 404, for objects that do not exist."""
+
+ def fetch_path(self, *components: str) -> bytes | None:
+ try:
+ return super().fetch_path(*components)
+ except requests.HTTPError as e:
+ if e.response is not None and e.response.status_code in (401, 403):
+ return None
+ raise
+
+
+def definitions_source(location: str) -> Source:
+ """A local copy of a definitions deploy, or the URL of one."""
+ if location.startswith(("https://", "http://")):
+ return _DevUrlSource(location.rstrip("/") + "/")
+ return FilesystemSource(Path(location))
+
+
+# --- Registry cases --------------------------------------------------------------
+
+
+@dataclass(frozen=True)
+class RegistryCase:
+ test_id: str
+ entity: str
+ descriptor: str # e.g. "calldata-AggregationRouterV6"
+ test_file: str # relative to the registry root
+ description: str
+ sender: str # the `from` of the case
+ expected: dict[str, t.Any]
+ tx: Transaction | None
+ skip_reason: str | None
+
+
+def _slug(text: str) -> str:
+ return re.sub(r"[^A-Za-z0-9_.-]+", "_", text).strip("_")
+
+
+def _skip_reason(tx: Transaction, source: Source) -> str | None:
+ if len(tx.data) < 4:
+ return "calldata has no function selector"
+ if len(tx.data) > MAX_CLEAR_SIGNED_DATA:
+ return (
+ f"calldata longer than {MAX_CLEAR_SIGNED_DATA} bytes is never clear-signed"
+ )
+ if source.get_eth_display_format(tx.chain_id, tx.to, tx.data[:4]) is None:
+ return "no display format in the dev definitions"
+ return None
+
+
+def load_cases(registry_root: Path, source: Source) -> list[RegistryCase]:
+ """Load the mainnet calldata test cases, in a stable order.
+
+ A case that cannot run gets a `skip_reason`. Cases on other chains are left out.
+ """
+ cases: list[RegistryCase] = []
+ seen_ids: set[str] = set()
+
+ def add(case_id: str, **kwargs: t.Any) -> None:
+ test_id = case_id
+ n = 2
+ while test_id in seen_ids:
+ test_id = f"{case_id}_{n}"
+ n += 1
+ seen_ids.add(test_id)
+ cases.append(RegistryCase(test_id=test_id, **kwargs))
+
+ for test_file in sorted(registry_root.glob(TEST_FILES_GLOB)):
+ entity = test_file.parent.parent.name
+ descriptor = test_file.name.removesuffix(".tests.json")
+ file_id = f"{_slug(entity)}__{_slug(descriptor.removeprefix('calldata-'))}"
+ relative_path = test_file.relative_to(registry_root).as_posix()
+ common = dict(entity=entity, descriptor=descriptor, test_file=relative_path)
+
+ try:
+ tests = json.loads(test_file.read_text())["tests"]
+ except (ValueError, KeyError) as e:
+ add(
+ f"{file_id}__invalid",
+ description="",
+ sender="",
+ expected={},
+ tx=None,
+ skip_reason=f"cannot read the test file: {e}",
+ **common,
+ )
+ continue
+
+ for test in tests:
+ description = test.get("description", "")
+ case = dict(
+ description=description,
+ sender=test.get("from", ""),
+ expected=test.get("expected", {}),
+ **common,
+ )
+ case_id = f"{file_id}__{_slug(description)}"
+ try:
+ tx = decode_raw_tx(test["rawTx"])
+ except UnsupportedTransaction as e:
+ if e.chain_id is not None and e.chain_id not in CHAIN_IDS:
+ continue
+ add(case_id, tx=None, skip_reason=f"unsupported: {e}", **case)
+ continue
+ except (ValueError, KeyError) as e:
+ add(case_id, tx=None, skip_reason=f"cannot decode rawTx: {e}", **case)
+ continue
+ if tx.chain_id not in CHAIN_IDS:
+ continue
+ add(case_id, tx=tx, skip_reason=_skip_reason(tx, source), **case)
+
+ return cases
+
+
+# --- Running a case --------------------------------------------------------------
+
+
+@dataclass
+class FlowRecorder:
+ """Record the ButtonRequest names and the text of every page of a signing flow."""
+
+ br_names: list[str] = field(default_factory=list)
+ pages: list[str] = field(default_factory=list)
+
+ def on_page(self, layout: LayoutContent) -> None:
+ self.pages.append(layout.text_content())
+
+ def wrap(
+ self, input_flow: Callable[[], BRGeneratorType]
+ ) -> Callable[[], BRGeneratorType]:
+ def wrapped() -> BRGeneratorType:
+ inner = input_flow()
+ next(inner)
+ while True:
+ br = yield
+ self.br_names.append(br.name or "")
+ try:
+ inner.send(br)
+ except StopIteration:
+ return
+
+ return wrapped
+
+ @property
+ def clear_signed(self) -> bool:
+ return BLIND_SIGNING_BR_NAME not in self.br_names
+
+
+def _normalize(text: str) -> str:
+ return re.sub(r"\s+", "", text).lower()
+
+
+def find_missing_texts(
+ expected: dict[str, t.Any], sender: str, device_address: str, pages: list[str]
+) -> list[str]:
+ """Return the expected texts (intent, owner, labels, values) that no page shows.
+
+ This is informational: the firmware formats some values differently than the
+ registry expects. Whitespace and case are ignored, and a value equal to the `from`
+ of the case also matches the address of the device, which signs instead.
+ """
+ shown = _normalize("\n".join(pages))
+ missing: list[str] = []
+
+ def check(kind: str, *alternatives: str) -> None:
+ if not any(_normalize(text) in shown for text in alternatives):
+ missing.append(f"{kind}: {alternatives[0]}")
+
+ def walk(block: dict[str, t.Any]) -> None:
+ for kind in ("intent", "owner"):
+ if isinstance(block.get(kind), str):
+ check(kind, block[kind])
+ for entry in block.get("fields", []):
+ label, value = entry.get("label"), entry.get("value")
+ if isinstance(label, str):
+ check("label", label)
+ if isinstance(value, dict):
+ walk(value)
+ elif isinstance(value, str):
+ if sender and value.lower() == sender.lower():
+ check("value", value, device_address)
+ else:
+ check("value", value)
+
+ walk(expected)
+ return missing
+
+
+def write_result(model: str, case: RegistryCase, result: dict[str, t.Any]) -> None:
+ path = RESULTS_DIR / model / f"{case.test_id}.json"
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(json.dumps(result, indent=2) + "\n")
+
+
+def read_results(results_dir: Path) -> dict[str, dict[str, dict[str, t.Any]]]:
+ """Read the results written by `write_result`: {model: {test_id: result}}."""
+ results: dict[str, dict[str, dict[str, t.Any]]] = {}
+ for path in sorted(results_dir.glob("*/*.json")):
+ results.setdefault(path.parent.name, {})[path.stem] = json.loads(
+ path.read_text()
+ )
+ return results
### tests/clear_signing_tests/summary.md.mako
@@ -0,0 +1,73 @@
+<%doc>
+ The summary of the nightly clear-signing tests, for $GITHUB_STEP_SUMMARY.
+ Rendered by `nightly.py summarize`. Lines starting with "##" are Mako comments,
+ so Markdown headings are written as ${"##"}.
+</%doc>\
+<%!
+ def short(sha):
+ return f"`{sha[:10]}`" if sha else "?"
+
+ def baseline_link(run):
+ if not run:
+ return "`fixtures.json`"
+ return f"[#{run['run_number']}]({run['run_url']}) ({run['created_at'][:10]})"
+%>\
+<%def name="capped(lines)">\
+% for line in lines[:max_lines]:
+${line}
+% endfor
+% if len(lines) > max_lines:
+
+… and ${len(lines) - max_lines} more, see the results artifacts.
+% endif
+</%def>\
+${"##"} Clear-signing registry tests
+
+Definitions: \
+% if inputs.get("definitions_ref"):
+`${inputs["definitions_ref"]}` @ \
+% endif
+[${short(inputs.get("definitions_sha"))}](https://github.com/trezor/definitions/commit/${inputs.get("definitions_sha")})\
+% if inputs.get("deploy_run_url"):
+, [deployed ${inputs["deployed_at"]}](${inputs["deploy_run_url"]})\
+% endif
+ · Registry: [${short(inputs.get("registry_sha"))}](https://github.com/${inputs.get("registry_repo")}/tree/${inputs.get("registry_sha")})
+
+${total} mainnet cases: ${run} run on each model, ${total - run} skipped.
+
+| Model | Clear-signed | Blind-signed | Error | Not run | Text not found | UI report | Compared with |
+|---|---:|---:|---:|---:|---:|---|---|
+% for row in rows:
+| ${row.model} | ${row.counts["clear_signed"]} | ${row.counts["blind"]} | ${row.counts["error"]} | ${row.counts["not run"]} | ${row.counts["text not found"]} | [report](${row.report_url}) · [differing screens](${row.differing_url}) | ${baseline_link(row.baseline)} |
+% endfor
+
+The UI report compares the screens of each model with those of the last nightly run (`fixtures.json` when there is none). There is nothing to accept: the screens of this run are what the next one compares with.
+
+<details><summary>Skipped cases (${sum(count for _, count in skipped)})</summary>
+
+| Reason | Cases |
+|---|---:|
+% for reason, count in skipped:
+| ${reason} | ${count} |
+% endfor
+
+</details>
+
+<details><summary>Failed or not run (${len(problems)})</summary>
+
+${capped([f"- `{test_id}`: {'; '.join(lines)}" for test_id, lines in problems])}
+</details>
+
+<details><summary>Blind-signed (${len(blind)})</summary>
+
+The firmware fell back to blind signing although the dev definitions have a display format. Not a failure: the UI report shows when a case changes between clear and blind signing.
+
+${capped([f"- `{test_id}` ({', '.join(models)})" for test_id, models in blind])}
+</details>
+
+<details><summary>Clear-signed, but expected text not found on screen (${len(not_found)})</summary>
+
+The registry's `expected` texts that no page showed. The firmware formats some values differently, so this is for review, not a failure.
+
+${capped([f"- `{test_id}` ({', '.join(models)}): {'; '.join(missing)}" for test_id, (models, missing) in not_found])}
+</details>
### tests/clear_signing_tests/test_registry.py
@@ -0,0 +1,163 @@
+# This file is part of the Trezor project.
+#
+# Copyright (C) SatoshiLabs and contributors
+#
+# This library is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3
+# as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the License along with this library.
+# If not, see <https://www.gnu.org/licenses/lgpl-3.0.html>.
+
+"""Nightly clear-signing tests: ERC-7730 registry cases against the dev definitions.
+
+See docs/tests/clear-signing-tests.md. Run with `make -C core test_emu_clear_signing_ui`.
+"""
+
+from __future__ import annotations
+
+import os
+import typing as t
+from pathlib import Path
+
+import pytest
+
+from trezorlib import ethereum, messages
+from trezorlib.debuglink import DebugSession as Session
+from trezorlib.tools import parse_path
+
+from ..input_flows import InputFlowConfirmAllWarnings
+from .registry import (
+ DEFINITIONS_ENV,
+ REGISTRY_ENV,
+ FlowRecorder,
+ RegistryCase,
+ definitions_source,
+ find_missing_texts,
+ load_cases,
+ write_result,
+)
+
+pytestmark = [
+ pytest.mark.altcoin,
+ pytest.mark.ethereum,
+ pytest.mark.models("core"),
+]
+
+_REGISTRY = os.environ.get(REGISTRY_ENV)
+_DEFINITIONS = os.environ.get(DEFINITIONS_ENV)
+if not _REGISTRY or not _DEFINITIONS:
+ pytest.skip(
+ f"set {REGISTRY_ENV} and {DEFINITIONS_ENV} to run the clear-signing tests",
+ allow_module_level=True,
+ )
+
+SOURCE = definitions_source(_DEFINITIONS)
+CASES = load_cases(Path(_REGISTRY), SOURCE)
+PATH = parse_path("m/44h/60h/0h/0/0")
+
+
+@pytest.mark.parametrize(
+ "case",
+ [
+ pytest.param(
+ case,
+ id=case.test_id,
+ marks=[pytest.mark.skip(reason=case.skip_reason)]
+ if case.skip_reason
+ else [],
+ )
+ for case in CASES
+ ],
+)
+def test_registry_case(session: Session, case: RegistryCase) -> None:
+ tx = case.tx
+ assert tx is not None
+
+ model = session.test_ctx.model
+ assert model is not None
+ result: dict[str, t.Any] = {
+ "test_id": case.test_id,
+ "entity": case.entity,
+ "descriptor": case.descriptor,
+ "test_file": case.test_file,
+ "description": case.description,
+ "to": tx.to,
+ "selector": tx.data[:4].hex(),
+ }
+ recorder = FlowRecorder()
+ definitions = messages.EthereumDefinitions(
+ # Mainnet is not built into the tested models.
+ encoded_network=SOURCE.get_eth_network(tx.chain_id)
+ )
+
+ try:
+ device_address = ethereum.get_address(
+ session, PATH, encoded_network=definitions.encoded_network
+ )
+ with session.test_ctx as client:
+ client.set_input_flow(
+ recorder.wrap(
+ InputFlowConfirmAllWarnings(session, on_page=recorder.on_page).get()
+ )
+ )
+ if tx.tx_type == 2:
+ ethereum.sign_tx_eip1559(
+ session,
+ PATH,
+ nonce=tx.nonce,
+ gas_limit=tx.gas_limit,
+ to=tx.to,
+ value=tx.value,
+ data=tx.data,
+ chain_id=tx.chain_id,
+ max_gas_fee=tx.max_gas_fee,
+ max_priority_fee=tx.max_priority_fee,
+ access_list=[
+ messages.EthereumAccessList(
+ address=address, storage_keys=list(keys)
+ )
+ for address, keys in tx.access_list
+ ],
+ definitions=definitions,
+ supports_definition_request=True,
+ definition_source=SOURCE,
+ )
+ else:
+ ethereum.sign_tx(
+ session,
+ PATH,
+ nonce=tx.nonce,
+ gas_price=tx.gas_price,
+ gas_limit=tx.gas_limit,
+ to=tx.to,
+ value=tx.value,
+ data=tx.data,
+ chain_id=tx.chain_id,
+ definitions=definitions,
+ supports_definition_request=True,
+ definition_source=SOURCE,
+ )
+ except Exception as e:
+ result["status"] = "error"
+ result["message"] = f"{type(e).__name__}: {e}"
+ raise
+ else:
+ result["status"] = "clear_signed" if recorder.clear_signed else "blind"
+ result["missing"] = find_missing_texts(
+ case.expected, case.sender, device_address, recorder.pages
+ )
+ finally:
+ result["br_names"] = recorder.br_names
+ result["pages"] = recorder.pages
+ write_result(model.internal_name, case, result)
+
+ # Only errors fail the test. Blind signing is a result like clear signing: its
+ # screens are compared with those of the last run, so a case that changes from one
+ # to the other fails on its screens. Differences from the registry's `expected`
+ # are only reported in the summary.
### tests/ui_tests/__init__.py
@@ -107,12 +107,16 @@ def setup(main_runner: bool) -> None:
testreport.setup(main_runner)
-def list_missing() -> set[str]:
+def _missing_tests() -> set[TestCase]:
# Only listing the ones for the current model
_, missing = common.prepare_fixtures(
TestResult.recent_results(), remove_missing=True
)
- return {test.id for test in missing}
+ return missing
+
+
+def list_missing() -> set[str]:
+ return {test.id for test in _missing_tests()}
def update_fixtures(remove_missing: bool = False) -> int:
@@ -172,11 +176,15 @@ def terminal_summary(
println(f"UI_MISSING: {result.test.id} ({result.actual_hash})")
else:
println(f"UI_FAILED: {result.test.id} ({result.actual_hash})")
- println("Run ./tests/show_results.py to open test summary")
+ # tests with their own fixtures file need it for these commands too
+ env = ""
+ if common.FIXTURES_FILE != UI_TESTS_DIR / "fixtures.json":
+ env = f"TREZOR_UI_FIXTURES={common.FIXTURES_FILE} "
+ println(f"Run {env}./tests/show_results.py to open test summary")
println("")
println("-------- Accepting all recent UI changes: --------")
- println("Run ./tests/update_fixtures.py to apply all changes")
+ println(f"Run {env}./tests/update_fixtures.py to apply all changes")
println("")
@@ -189,6 +197,11 @@ def sessionfinish(
if not _should_write_ui_report(exitstatus):
return exitstatus
+ if test_ui == "test" and check_missing:
+ fixtures = common.get_current_fixtures()
+ for test in _missing_tests():
+ expected_hash = fixtures[test.model][test.group][test.fixtures_name]
+ testreport.removed(test, expected_hash)
testreport.generate_reports(do_master_diff)
recents = list(TestResult.recent_results())
### tests/ui_tests/common.py
@@ -18,6 +18,7 @@
import hashlib
import json
+import os
import re
import shutil
import typing as t
@@ -38,7 +39,11 @@
UI_TESTS_DIR = Path(__file__).resolve().parent
SCREENS_DIR = UI_TESTS_DIR / "screens"
IMAGES_DIR = SCREENS_DIR / "all_images"
-FIXTURES_FILE = UI_TESTS_DIR / "fixtures.json"
+# Test classes that keep their own fixtures (tests/clear_signing_tests) point
+# TREZOR_UI_FIXTURES at their file, so tests/ui_tests/fixtures.json is untouched.
+FIXTURES_FILE = Path(
+ os.environ.get("TREZOR_UI_FIXTURES") or UI_TESTS_DIR / "fixtures.json"
+)
# fixtures.json are structured as follows:
# {
@@ -90,7 +95,7 @@ def prepare_fixtures(
new_tests = set(new_content.keys())
old_tests = set(current_content.keys())
missing_tests |= {
- TestCase(model, group, test, language) for test in old_tests - new_tests
+ TestCase.from_fixtures(test, group) for test in old_tests - new_tests
}
current_content.clear()
current_content.update(diff_languages)
@@ -157,7 +162,7 @@ def _get_bytes_from_png(png_file: Path) -> bytes:
return Image.open(str(png_file)).tobytes()
-def _hash_files(path: Path) -> str:
+def hash_files(path: Path) -> str:
files = path.iterdir()
hasher = hashlib.sha256()
for file in sorted(files):
@@ -309,7 +314,7 @@ def build_result(self, request: pytest.FixtureRequest) -> TestResult:
result = TestResult(
test=self,
passed=get_last_call_test_result(request),
- actual_hash=_hash_files(self.actual_dir),
+ actual_hash=hash_files(self.actual_dir),
images=self.actual_screens[1],
)
result.save_metadata()
### tests/ui_tests/reporting/testreport.py
@@ -17,6 +17,7 @@
from __future__ import annotations
import shutil
+import tempfile
from collections import defaultdict
from datetime import datetime
from pathlib import Path
@@ -40,7 +41,14 @@
)
from dominate.util import text
-from ..common import FixturesType, TestCase, TestResult
+from ..common import (
+ FixturesType,
+ TestCase,
+ TestResult,
+ hash_files,
+ screens_and_hashes,
+ screens_diff,
+)
from . import download, html
from .common import (
GIF_SCRIPT,
@@ -98,6 +106,7 @@ def setup(main_runner: bool) -> None:
(TESTREPORT_PATH / "failed").mkdir()
(TESTREPORT_PATH / "passed").mkdir()
(TESTREPORT_PATH / "new").mkdir()
+ (TESTREPORT_PATH / "removed").mkdir()
(TESTREPORT_PATH / "diff").mkdir()
IMAGES_PATH.mkdir(parents=True)
@@ -109,6 +118,8 @@ def index() -> Path:
passed_tests = list((TESTREPORT_PATH / "passed").iterdir())
failed_tests = list((TESTREPORT_PATH / "failed").iterdir())
new_tests = list((TESTREPORT_PATH / "new").iterdir())
+ removed_dir = TESTREPORT_PATH / "removed"
+ removed_tests = list(removed_dir.iterdir()) if removed_dir.exists() else []
actual_hashes = {
result.test.id: result.actual_hash for result in TestResult.recent_results()
@@ -121,7 +132,7 @@ def index() -> Path:
with doc:
h1("UI Test report")
- if not failed_tests:
+ if not failed_tests and not removed_tests:
p("All tests succeeded!", style="color: green; font-weight: bold;")
else:
p("Some tests failed!", style="color: red; font-weight: bold;")
@@ -151,6 +162,11 @@ def index() -> Path:
h2("New tests", style="color: blue;")
html.report_links(new_tests, TESTREPORT_PATH)
+ if removed_tests:
+ h2("Removed tests", style="color: darkred;")
+ i("Expected by the fixtures, but did not run:")
+ html.report_links(removed_tests, TESTREPORT_PATH)
+
h2("Passed", style="color: green;")
html.report_links(passed_tests, TESTREPORT_PATH)
@@ -395,9 +411,77 @@ def passed(result: TestResult) -> Path:
return recorded(result, header="Passed")
+def _screens_table(
+ diff: list[tuple[str | None, str | None]], dir: str, header: str = "Expected"
+) -> None:
+ with table(border=1, width=600):
+ with tr():
+ th(header)
+ th("Actual")
+ th("Diff")
+
+ html.diff_table(diff, TESTREPORT_PATH / dir)
+
+
def missing(result: TestResult) -> Path:
- """Generate an HTML file for a newly seen test-case."""
- return recorded(result, header="New testcase", dir="new")
+ """Generate an HTML file for a newly seen test-case.
+
+ Shows its screens next to missing expected ones.
+ """
+ _copy_deduplicated(result.test)
+
+ doc = document(title=result.test.id, model=result.test.model)
+ with doc.head:
+ script(type="text/javascript", src="../testreport.js")
+ with doc:
+ _header(result, "new")
+ _screens_table(screens_diff([], result.images), "new")
+ return html.write(TESTREPORT_PATH / "new", doc, result.test.id + ".html")
+
+
+def removed(test: TestCase, expected_hash: str) -> Path:
+ """Generate an HTML file for a test-case of the fixtures that did not run.
+
+ Shows its expected screens next to missing actual ones.
+ """
+ # Download the expected screens, or keep the local ones if they are the same.
+ download_failed = False
+ with tempfile.TemporaryDirectory() as tmp:
+ try:
+ download.fetch_recorded(expected_hash, Path(tmp))
+ except Exception:
+ download_failed = True
+ else:
+ shutil.rmtree(test.recorded_dir, ignore_errors=True)
+ shutil.copytree(tmp, test.recorded_dir)
+ if download_failed and (
+ not test.recorded_dir.exists() or hash_files(test.recorded_dir) != expected_hash
+ ):
+ expected: list[str] = []
+ else:
+ screens, expected = screens_and_hashes(test.recorded_dir)
+ html.store_images(screens, expected)
+
+ doc = document(title=test.id, model=test.model)
+ with doc.head:
+ script(type="text/javascript", src="../testreport.js")
+ with doc:
+ h1(test.id)
+ with div():
+ p(
+ "This test is in the fixtures, but did not run.",
+ style="color: darkred; font-weight: bold;",
+ )
+ p("Expected: ", expected_hash)
+ hr()
+
+ if not expected:
+ with p():
+ strong("WARNING:")
+ text(" failed to download the expected screens.")
+
+ _screens_table(screens_diff(expected, []), "removed")
+ return html.write(TESTREPORT_PATH / "removed", doc, test.id + ".html")
def recorded(result: TestResult, header: str = "Recorded", dir: str = "passed") -> Path:Why this scored 15/100
Community notes
Notes can correct, qualify, or add evidence to the AI analysis. Every note shown here has been validated by a human moderator.
The AI analysis stands alone for now. Submit a note if you can add evidence or important context.