Merge rust-bitcoin/rust-bitcoin#6811: ci: add corpus fuzzing crash store

### .github/workflows/corpus-fuzzing.yml
@@ -105,13 +105,13 @@ jobs:
       - name: Report crashes
         if: failure()
         working-directory: rust-bitcoin
-        run: ./fuzz/report-crashes.sh "crash-$TARGET"
+        run: ./fuzz/report-crashes.sh "crash-$TARGET" fuzz
       - name: Upload crash artifacts
         if: failure()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: crash-${{ matrix.fuzz_target }}
-          path: rust-bitcoin/fuzz/artifacts/${{ matrix.fuzz_target }}
+          path: rust-bitcoin/fuzz/artifacts/
           if-no-files-found: ignore
 
   # Merges the corpus of each target back into the qa-assets repository.
@@ -136,16 +136,70 @@ jobs:
         with:
           pattern: corpus-*
           path: incoming
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        continue-on-error: true
+        with:
+          pattern: crash-*
+          merge-multiple: true
+          path: incoming-crashes
       - name: Refresh corpora
         env:
           TARGETS: ${{ needs.setup.outputs.targets }}
         run: |
           echo "$TARGETS" | jq -r '.[]' > "$RUNNER_TEMP/targets.txt"
-          rust-bitcoin/fuzz/corpora.sh refresh incoming "$RUNNER_TEMP/targets.txt"
+          rust-bitcoin/fuzz/corpora.sh refresh incoming incoming-crashes "$RUNNER_TEMP/targets.txt"
       - name: Commit and push
         env:
           QA_ASSETS_PUSH_TOKEN: ${{ secrets.QA_ASSETS_PUSH_TOKEN }}
           SOURCE: ${{ github.repository }}@${{ needs.setup.outputs.source_sha }}
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
           BRANCH: master
         run: rust-bitcoin/fuzz/corpora.sh push
+
+  # Replay the crash store to fail job if a bad fuzz input is in codebase.
+  # Separate job so that fuzzing can keep running despite other crashes.
+  replay-crash-store:
+    needs: [setup, merge]
+    if: ${{ always() && needs.merge.result == 'success' }}
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          repository: ${{ github.repository_owner }}/qa-assets
+          persist-credentials: false
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          ref: ${{ needs.setup.outputs.source_sha }}
+          path: rust-bitcoin
+          persist-credentials: false
+      - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: |
+            ~/.cargo/bin
+            ~/.cargo/.crates.toml
+            ~/.cargo/.crates2.json
+            rust-bitcoin/fuzz/target
+            rust-bitcoin/target
+          key: fuzz-${{ hashFiles('rust-bitcoin/**/Cargo.toml', 'rust-bitcoin/rbmt-version') }}
+          restore-keys: fuzz-
+      - name: Install cargo-rbmt
+        working-directory: rust-bitcoin
+        run: |
+          cargo install \
+            --git https://git.rust-bitcoin.org/rust-bitcoin/rust-bitcoin-maintainer-tools \
+            --rev "$(cat rbmt-version)" \
+            cargo-rbmt \
+            --locked
+      - name: Install nightly toolchain
+        working-directory: rust-bitcoin
+        run: rustup toolchain install "$(cargo rbmt toolchains --nightly)" --profile minimal --no-self-update
+      - name: Install cargo-fuzz
+        run: cargo install --locked --version "$CARGO_FUZZ_VERSION" cargo-fuzz
+      - name: Replay crash store
+        run: rust-bitcoin/fuzz/corpora.sh replay .
+      - name: Report crashes
+        if: failure()
+        working-directory: rust-bitcoin
+        run: ./fuzz/report-crashes.sh "qa-assets fuzz_crashes store" crash-store

### fuzz/corpora.sh
@@ -2,19 +2,30 @@
 
 # Manage the shared fuzz corpora stored in the qa-assets repository.
 #
-# Usage: corpora.sh {seed QA_DIR TARGET | refresh INCOMING_DIR TARGETS_FILE | push}
+# Usage: corpora.sh {seed QA_DIR TARGET | refresh INCOMING_CORPORA INCOMING_CRASHES TARGETS_FILE | replay QA_DIR | push}
 #
 # Commands:
 #   seed     Copy a target's stored corpus into fuzz/corpus.
-#   refresh  Replace each stored corpus with the one under INCOMING_DIR and drop
-#            targets not listed in TARGETS_FILE.
+#   refresh  Replace each stored corpus with the one under INCOMING_CORPORA, add
+#            crash inputs under INCOMING_CRASHES to the fuzz_crashes store and
+#            drop targets not listed in TARGETS_FILE. The crash store is
+#            then trimmed to MAX_STORED_CRASHES.
+#   replay   Run every input in the fuzz_crashes store under QA_DIR against
+#            its target and fail if any of them still crashes.
 #   push     CI only, not meant to be run locally. Commit and push to
 #            qa-assets. Reads QA_ASSETS_PUSH_TOKEN, SOURCE, RUN_URL and
 #            BRANCH from the environment.
 
 set -euo pipefail
 
-usage="Usage: $0 {seed QA_DIR TARGET | refresh INCOMING_DIR TARGETS_FILE | push}"
+# shellcheck source=fuzz/rustflags.sh
+. "$(dirname "$0")/rustflags.sh"
+
+usage="Usage: $0 {seed QA_DIR TARGET | refresh INCOMING_CORPORA INCOMING_CRASHES TARGETS_FILE | replay QA_DIR | push}"
+
+# Upper bound on inputs kept in the crash store, oldest dropped first.
+# 40MB is the limit, because libFuzzer default max input size is 4KB.
+readonly MAX_STORED_CRASHES=10000
 
 seed() {
   local qa="${1:?$usage}" target="${2:?$usage}"
@@ -25,28 +36,89 @@ seed() {
     find "$qa/fuzz_corpora/$target" -maxdepth 1 -type f -exec cp -t "$corpus/" {} +
   fi
 }
+# Trim the crash store to MAX_STORED_CRASHES, oldest first.
+maybe_prune_old_crashes() {
+  local total excess file oldest_first
+
+  total=$(find fuzz_crashes -type f | wc -l)
+  excess=$((total - MAX_STORED_CRASHES))
+  [ "$excess" -gt 0 ] || return 0
+  echo "Crash store holds $total inputs, dropping $excess oldest"
+
+  # Get files by the age of first presence in commit, ascending.
+  oldest_first=$(git log --reverse --diff-filter=A --format='' --name-only \
+                     -- fuzz_crashes | awk 'NF && !seen[$0]++')
+
+  while IFS= read -r file; do
+    [ "$excess" -gt 0 ] || break
+    [ -f "$file" ] || continue
+    rm -f -- "$file"
+    excess=$((excess - 1))
+  done <<< "$oldest_first"
+  find fuzz_crashes -mindepth 1 -type d -empty -delete
+}
 
 refresh() {
-  local incoming="${1:?$usage}" targets_file="${2:?$usage}"
+  local corpora="${1:?$usage}" crashes="${2:?$usage}" targets_file="${3:?$usage}"
   local dir name
-  mkdir -p fuzz_corpora "$incoming"
-  for dir in "$incoming"/corpus-*/; do
+  [ -s "$targets_file" ] || { echo "empty target list, refusing to prune" >&2; exit 1; }
+  mkdir -p fuzz_corpora fuzz_crashes "$corpora" "$crashes"
+
+  # Copy and replace the corpora into the fuzz_corpora store.
+  for dir in "$corpora"/corpus-*/; do
     [ -d "$dir" ] || continue # skip on empty corpora, string expansion yields literal pattern
     name=$(basename "$dir")
     name=${name#corpus-}
     rm -rf "fuzz_corpora/$name"
     mkdir -p "fuzz_corpora/$name"
     find "$dir" -maxdepth 1 -type f -exec cp -t "fuzz_corpora/$name/" {} +
   done
-  for dir in fuzz_corpora/*/; do
+
+  # Copy without replacing the crashes into the fuzz_crashes store.
+  for dir in "$crashes"/*/; do
+    [ -d "$dir" ] || continue
+    name=$(basename "$dir")
+    mkdir -p "fuzz_crashes/$name"
+    find "$dir" -maxdepth 1 -type f -exec cp -n -t "fuzz_crashes/$name/" {} +
+  done
+
+  # Drop targets that no longer exist upstream
+  for dir in fuzz_corpora/*/ fuzz_crashes/*/; do
     [ -d "$dir" ] || continue
     name=$(basename "$dir")
-    grep -qx "$name" "$targets_file" || rm -rf "$dir" # drop targets that no longer exist upstream
+    grep -qx "$name" "$targets_file" || rm -rf "$dir"
+  done
+  maybe_prune_old_crashes
+}
+
+replay() {
+  local qa="${1:?$usage}"
+  local fuzz dir input target rustflags nightly failed=0
+  qa="$(cd "$qa" && pwd)"
+  fuzz="$(cd "$(dirname "$0")" && pwd)"
+  cd "$fuzz"
+  nightly="$(cargo rbmt toolchains --nightly)"
+
+  for dir in "$qa"/fuzz_crashes/*/; do
+    [ -d "$dir" ] || continue
+    target=$(basename "$dir")
+    rustflags="$(fuzz_rustflags "$target")"
+
+    # Inputs as files so libFuzzer runs each one, not as a corpus to fuzz.
+    while IFS= read -r -d '' input; do
+      echo "Replaying $target/$(basename "$input")"
+      if ! RUSTFLAGS="${RUSTFLAGS:-} $rustflags" cargo +"$nightly" fuzz run "$target" "$input"; then
+        mkdir -p "artifacts/$target"
+        cp -n "$input" "artifacts/$target/"
+        failed=1
+      fi
+    done < <(find "$dir" -maxdepth 1 -type f -print0)
   done
+  [ "$failed" = 0 ] || exit 1
 }
 
 push() {
-  git add -A fuzz_corpora
+  git add -A fuzz_corpora fuzz_crashes
   if git diff --cached --quiet; then
     echo "No corpus changes"
     exit 0
@@ -72,7 +144,7 @@ export LC_ALL=C
 cmd="${1:?$usage}"
 shift
 case "$cmd" in
-  seed | refresh | push) "$cmd" "$@" ;;
+  seed | refresh | replay | push) "$cmd" "$@" ;;
   *)
     echo "$usage" >&2
     exit 2

### fuzz/fuzz.sh
@@ -12,6 +12,9 @@
 
 set -euo pipefail
 
+# shellcheck source=fuzz/rustflags.sh
+. "$(dirname "$0")/rustflags.sh"
+
 target=
 max_total_time=
 cycle_mode=false
@@ -80,12 +83,7 @@ fi
 while :; do
   for targetName in $targets; do
     echo "Fuzzing target $targetName for $max_total_time seconds"
-    # Enable fuzz stubs in the hashes and cryptography libraries by default,
-    # unless we are fuzzing the hashes targets themselves.
-    fuzz_rustflags=''
-    if [[ ! "$targetName" =~ ^hashes_ ]]; then
-      fuzz_rustflags='--cfg=hashes_fuzz --cfg=secp256k1_fuzz'
-    fi
+    fuzz_rustflags="$(fuzz_rustflags "$targetName")"
     # cargo-fuzz will check for the corpus at fuzz/corpus/<target>
     # Use chrt to run at SCHED_IDLE priority (lowest) to avoid blocking other work.
     chrt_cmd=''

### fuzz/report-crashes.sh
@@ -1,9 +1,20 @@
 #!/usr/bin/env bash
 # Report every crash input under fuzz/artifacts.
+#
+# SOURCE: either "fuzz" for inputs found by this
+# fuzz run, or "crash-store" for stored inputs replayed.
 
 set -euo pipefail
 
-artifact_name="${1:?Usage: $0 ARTIFACT_NAME}"
+# shellcheck source=fuzz/rustflags.sh
+. "$(dirname "$0")/rustflags.sh"
+
+artifact_name="${1:?Usage: $0 ARTIFACT_NAME SOURCE}"
+source="${2:?Usage: $0 ARTIFACT_NAME SOURCE}"
+case "$source" in
+  fuzz | crash-store) ;;
+  *) echo "SOURCE must be 'fuzz' or 'crash-store', got '$source'" >&2; exit 2 ;;
+esac
 summary="${GITHUB_STEP_SUMMARY:-/dev/stdout}"
 
 cd "$(git rev-parse --show-toplevel)"
@@ -20,6 +31,8 @@ for tdir in fuzz/artifacts/*/; do
     {
       echo "## Fuzz crash: $target"
       echo ""
+      echo "Source: \`$source\`"
+      echo ""
       echo "\`$name\` ($size bytes)"
       echo ""
       if [ "$size" -gt 1024 ]; then
@@ -43,16 +56,17 @@ for tdir in fuzz/artifacts/*/; do
         echo ""
       fi
       echo '```sh'
-      if [[ "$target" != hashes_* ]]; then
-        echo 'export RUSTFLAGS="--cfg=hashes_fuzz --cfg=secp256k1_fuzz"'
+      flags="$(fuzz_rustflags "$target")"
+      if [ -n "$flags" ]; then
+        echo "export RUSTFLAGS=\"$flags\""
       fi
       echo "base64 -d > crash <<'EOF' # paste the base64 line above in terminal"
       echo "EOF"
       echo "cargo +nightly fuzz run $target crash"
       echo '```'
       echo ""
     } >> "$summary"
-    echo "::error title=Fuzz crash in $target::$name ($size bytes), read the job summary github page for reproduction instructions"
+    echo "::error title=Fuzz crash in $target [$source]::$name ($size bytes), read the job summary github page for reproduction instructions"
   done
 done
 

### fuzz/rustflags.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+# Shared RUSTFLAGS selection for the fuzz targets, sourced by fuzz scripts.
+
+# Use stubs as hashes and cryptography libraries by default, unless targets
+# are fuzzing the hashes themselves.
+fuzz_rustflags() {
+  local target="${1:?fuzz_rustflags TARGET}"
+  if [[ "$target" =~ ^hashes_ ]]; then
+    echo ''
+  else
+    echo '--cfg=hashes_fuzz --cfg=secp256k1_fuzz'
+  fi
+}