request bitcoin core wallet spends in batches from v28 when starting cormorant and wait for the initial load before polling
What changed, and why it matters
This commit improves how the Sparrow Wallet desktop app talks to a self-hosted Bitcoin Core node when using the built-in 'Cormorant' connection. It batches transaction lookups on newer Bitcoin Core versions, waits for the initial wallet history load before starting background polling, and tweaks error messages so they don't wrongly blame firewall settings when the user is connecting through Tor. The changes are mostly reliability and performance fixes, not a patch for a remote attack.
No urgent security action is required. Users running Bitcoin Core v28+ through Sparrow's Cormorant connection will benefit from faster wallet sync and fewer redundant RPC calls. Operators should still ensure Bitcoin Core rpcallowip and authentication settings are correct, and should upgrade Sparrow when this release is available as a general maintenance update.
Security signals we found
Avoids redundant full-wallet history requests during initialization, reducing load and potential race conditions
Adds batch JSON-RPC response parsing, fixing possible parsing failures on batched gettransaction calls
Improves error-message accuracy for Tor/proxy connections to prevent misdiagnosis of connection failures
Partial-failure handling caches successfully fetched spent outputs before a batch/request failure
Evidence from the diff
The patch modifies BitcoindClient to request wallet spends in JSON-RPC batches (max 100) for Bitcoin Core v28+ (version 280000), falling back to individual gettransaction calls for older nodes. It adds a storeLoaded guard so the polling loop returns early until initialize() has finished loading the wallet history, avoiding duplicate full-history fetches. BitcoindTransport now accepts array-shaped JSON-RPC batch responses and adjusts the ‘closed connection’ error message when a proxy is in use. New unit tests verify batching, fallback behavior, partial-failure caching, and proxy-specific diagnostics.
Changed components
src/main/java/com/sparrowwallet/sparrow/net/cormorant/bitcoind/BitcoindClient.javasrc/main/java/com/sparrowwallet/sparrow/net/cormorant/bitcoind/BitcoindTransport.javaInspect captured patch +354 / −12
### src/main/java/com/sparrowwallet/sparrow/net/cormorant/bitcoind/BitcoindClient.java
@@ -1,8 +1,11 @@
package com.sparrowwallet.sparrow.net.cormorant.bitcoind;
import com.github.arteam.simplejsonrpc.client.JsonRpcClient;
+import com.github.arteam.simplejsonrpc.client.builder.BatchRequestBuilder;
+import com.github.arteam.simplejsonrpc.client.exception.JsonRpcBatchException;
import com.github.arteam.simplejsonrpc.client.exception.JsonRpcException;
import com.google.common.base.Suppliers;
+import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.sparrowwallet.drongo.KeyPurpose;
import com.sparrowwallet.drongo.OutputDescriptor;
@@ -41,6 +44,7 @@
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.BiConsumer;
import java.util.function.Supplier;
import java.util.stream.Collectors;
@@ -49,6 +53,8 @@ public class BitcoindClient {
public static final String CORE_WALLET_NAME = "cormorant";
private static final int DEFAULT_GAP_LIMIT = 1000;
+ private static final int TRANSACTION_BATCH_SIZE = 100;
+ private static final int JSON_RPC_2_BATCH_VERSION = 280000;
private static final int POSTMIX_GAP_LIMIT = 4000;
private static final long PRUNED_RESCAN_TIMEGAP_MILLIS = 7200*1000;
@@ -64,7 +70,8 @@ public class BitcoindClient {
private final BitcoindTransport bitcoindTransport;
private final Store store = new Store();
- private NetworkInfo networkInfo;
+ //Package visible so tests can set the node version
+ NetworkInfo networkInfo;
private String lastBlock;
private ElectrumBlockHeader tip;
@@ -75,6 +82,7 @@ public class BitcoindClient {
private final Map<String, Integer> descriptorUsedIndexes = new ConcurrentHashMap<>();
private boolean initialized;
+ private volatile boolean storeLoaded;
private boolean stopped;
private Exception lastPollException;
@@ -182,6 +190,7 @@ public void initialize() throws CormorantBitcoindException {
ListSinceBlock listSinceBlock = getListSinceBlock(null);
updateStore(listSinceBlock);
+ storeLoaded = true;
}
private ListSinceBlock getListSinceBlock(String blockHash) {
@@ -512,12 +521,12 @@ private synchronized void updateStore(ListSinceBlock listSinceBlock) {
}
}
+ Map<String, Set<HashIndex>> storedSpentOutputs = store.getSpentOutputs();
+ List<String> uncachedTxids = sentTransactions.stream().map(ListTransaction::txid).distinct().filter(txid -> !storedSpentOutputs.containsKey(txid)).toList();
+ fetchSpentOutputs(uncachedTxids, storedSpentOutputs);
+
for(ListTransaction sentTransaction : sentTransactions) {
- Set<HashIndex> spentOutputs = store.getSpentOutputs().computeIfAbsent(sentTransaction.txid(), txid -> {
- String txhex = getTransaction(txid);
- Transaction tx = new Transaction(Utils.hexToBytes(txhex));
- return tx.getInputs().stream().map(txInput -> new HashIndex(txInput.getOutpoint().getHash(), txInput.getOutpoint().getIndex())).collect(Collectors.toSet());
- });
+ Set<HashIndex> spentOutputs = storedSpentOutputs.get(sentTransaction.txid());
boolean foundFundingAddress = false;
for(HashIndex spentOutput : spentOutputs) {
@@ -546,6 +555,44 @@ private synchronized void updateStore(ListSinceBlock listSinceBlock) {
}
}
+ //Requested in batches from Bitcoin Core v28 and later, as a wallet may hold hundreds of spends.
+ //Earlier versions answer a batch in the JSON-RPC 1.0 form, which the client rejects, so each transaction is requested on its own.
+ //Each transaction is added to spentOutputs as it is received, so that after a failure the next poll requests only those that remain.
+ void fetchSpentOutputs(List<String> txids, Map<String, Set<HashIndex>> spentOutputs) {
+ if(networkInfo.version() >= JSON_RPC_2_BATCH_VERSION) {
+ for(List<String> batch : Lists.partition(txids, TRANSACTION_BATCH_SIZE)) {
+ fetchTransactions(batch, (txid, txhex) -> spentOutputs.put(txid, getInputOutpoints(txhex)));
+ }
+ } else {
+ for(String txid : txids) {
+ spentOutputs.put(txid, getInputOutpoints(getTransaction(txid)));
+ }
+ }
+ }
+
+ private static Set<HashIndex> getInputOutpoints(String txhex) {
+ Transaction tx = new Transaction(Utils.hexToBytes(txhex));
+ return tx.getInputs().stream().map(txInput -> new HashIndex(txInput.getOutpoint().getHash(), txInput.getOutpoint().getIndex())).collect(Collectors.toSet());
+ }
+
+ @SuppressWarnings("rawtypes")
+ private void fetchTransactions(List<String> txids, BiConsumer<String, String> transactionReceived) {
+ BatchRequestBuilder<String, Map> batchRequest = jsonRpcClient.createBatchRequest().keysType(String.class).returnType(Map.class);
+ for(String txid : txids) {
+ batchRequest.add(txid, "gettransaction", txid, true, false);
+ }
+
+ try {
+ batchRequest.execute().forEach((txid, result) -> transactionReceived.accept(txid, result.get("hex").toString()));
+ } catch(JsonRpcBatchException e) {
+ e.getSuccesses().forEach((txid, result) -> transactionReceived.accept((String)txid, ((Map<?, ?>)result).get("hex").toString()));
+ //A transaction the wallet no longer holds is requested from the node's transaction index, as getTransaction() does once gettransaction fails
+ for(Object txid : e.getErrors().keySet()) {
+ transactionReceived.accept((String)txid, getBitcoindService().getRawTransaction((String)txid, false).toString());
+ }
+ }
+ }
+
private String getTransaction(String txid) {
try {
return getBitcoindService().getTransaction(txid, true, false).get("hex").toString();
@@ -732,6 +779,11 @@ public void run() {
}
}
+ //The store is first loaded with the wallet's full history by initialize(), which this would otherwise request again in full alongside it
+ if(!storeLoaded) {
+ return;
+ }
+
if(lastBlock != null && tip != null) {
try {
String blockhash = getBitcoindService().getBlockHash(tip.height());
### src/main/java/com/sparrowwallet/sparrow/net/cormorant/bitcoind/BitcoindTransport.java
@@ -93,8 +93,9 @@ public String pass(String request) throws IOException {
log.debug("> " + request);
- //From v32.1 Bitcoin Core closes the connection from a client outside rpcallowip without answering, where earlier versions answered 403 below
- String closedMessage = "Bitcoin Core at " + bitcoindUri.getAuthority() + " closed the connection without responding, check its rpcallowip settings";
+ //From v32.1 Bitcoin Core closes the connection from a client outside rpcallowip without answering, where earlier versions answered 403 below.
+ //Tor delivers connections to an onion node locally, which rpcallowip admits by default, so through a proxy a closed connection more likely failed on the way.
+ String closedMessage = "Bitcoin Core at " + bitcoindUri.getAuthority() + " closed the connection without responding" + (proxy == null ? ", check its rpcallowip settings" : "");
HttpEndpointClient.Response httpResponse;
try {
@@ -139,8 +140,9 @@ public String pass(String request) throws IOException {
log.debug("< " + response);
//A response carrying neither a result nor an error leaves the client constructing a JsonRpcException from a null error message, which throws a NullPointerException naming nothing.
- //Bitcoin Core always answers with a JSON-RPC object, so an empty or HTML body here comes from something else on the network path - typically a TLS terminating proxy sent a plain HTTP request.
- if(!response.startsWith("{")) {
+ //Bitcoin Core always answers with a JSON-RPC object, or an array of them for a batch, so an empty or HTML body here comes from something else on the network path - typically
+ //a TLS terminating proxy sent a plain HTTP request.
+ if(!response.startsWith("{") && !response.startsWith("[")) {
throw new IOException("Bitcoin Core at " + bitcoindUri.getAuthority() + " did not return a JSON-RPC response to the " + bitcoindUri.getScheme() + " request (HTTP " + statusCode + ")");
}
### src/test/java/com/sparrowwallet/sparrow/net/cormorant/bitcoind/BitcoindClientSpentOutputsTest.java
@@ -0,0 +1,279 @@
+package com.sparrowwallet.sparrow.net.cormorant.bitcoind;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.sparrowwallet.drongo.Utils;
+import com.sparrowwallet.drongo.protocol.HashIndex;
+import com.sparrowwallet.drongo.protocol.Script;
+import com.sparrowwallet.drongo.protocol.Sha256Hash;
+import com.sparrowwallet.drongo.protocol.Transaction;
+import com.sparrowwallet.sparrow.SparrowWallet;
+import com.sparrowwallet.sparrow.io.Config;
+import com.sparrowwallet.sparrow.io.Server;
+import com.sparrowwallet.sparrow.net.CoreAuthType;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.*;
+import java.net.InetAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+public class BitcoindClientSpentOutputsTest {
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ @TempDir
+ private static Path tempHome;
+
+ @BeforeAll
+ public static void setUp() {
+ //Config.get() caches its instance for the life of the JVM, so keep these tests from loading the developer's real config and leaving it cached for those that follow
+ System.setProperty(SparrowWallet.APP_HOME_PROPERTY, tempHome.toString());
+ }
+
+ @AfterAll
+ public static void tearDown() {
+ System.clearProperty(SparrowWallet.APP_HOME_PROPERTY);
+ }
+
+ @Test
+ public void spentOutputsAreRequestedInBatches() throws Exception {
+ //Two batches of 100 and 51, then getrawtransaction for the transaction the wallet does not hold
+ assertSpentOutputs(310100, List.of(100, 51), 3);
+ }
+
+ @Test
+ public void spentOutputsAreRequestedSinglyFromNodesBeforeV28() throws Exception {
+ //One gettransaction for each, then getrawtransaction for the transaction the wallet does not hold
+ assertSpentOutputs(270200, List.of(), 152);
+ }
+
+ @Test
+ public void spentOutputsFetchedBeforeAFailedBatchAreKept() throws Exception {
+ //The first batch, before the batch holding the failing transaction
+ assertSpentOutputsBeforeFailureAreKept(310100, 100);
+ }
+
+ @Test
+ public void spentOutputsFetchedBeforeAFailedRequestAreKeptFromNodesBeforeV28() throws Exception {
+ //Every transaction requested before the failing one
+ assertSpentOutputsBeforeFailureAreKept(270200, 120);
+ }
+
+ private void assertSpentOutputsBeforeFailureAreKept(int nodeVersion, int expectedKept) throws Exception {
+ Map<String, String> walletTransactions = new LinkedHashMap<>();
+ Map<String, Set<HashIndex>> expected = new HashMap<>();
+ for(int i = 0; i < 150; i++) {
+ addTransaction(i, walletTransactions, expected);
+ }
+ List<String> txids = new ArrayList<>(walletTransactions.keySet());
+
+ try(FakeNode node = new FakeNode(walletTransactions, Map.of(), nodeVersion < 280000)) {
+ node.failingTxid = txids.get(120);
+ BitcoindClient client = createClient(node, nodeVersion);
+ try {
+ Map<String, Set<HashIndex>> spentOutputs = new HashMap<>();
+ assertTimeoutPreemptively(Duration.ofSeconds(20), () -> assertThrows(RuntimeException.class, () -> client.fetchSpentOutputs(txids, spentOutputs)));
+ assertEquals(expectedKept, spentOutputs.size());
+ for(String txid : txids.subList(0, expectedKept)) {
+ assertEquals(expected.get(txid), spentOutputs.get(txid));
+ }
+ } finally {
+ client.stop();
+ }
+ }
+ }
+
+ private static BitcoindClient createClient(FakeNode node, int nodeVersion) {
+ Config.get().setCoreServer(new Server("http://127.0.0.1:" + node.getPort()));
+ Config.get().setCoreAuthType(CoreAuthType.USERPASS);
+ Config.get().setCoreAuth("user:pass");
+ BitcoindClient client = new BitcoindClient(false);
+ client.networkInfo = new NetworkInfo(nodeVersion, "", true);
+ return client;
+ }
+
+ private void assertSpentOutputs(int nodeVersion, List<Integer> expectedBatchSizes, int expectedRequests) throws Exception {
+ Map<String, String> walletTransactions = new LinkedHashMap<>();
+ Map<String, Set<HashIndex>> expected = new HashMap<>();
+ for(int i = 0; i < 150; i++) {
+ addTransaction(i, walletTransactions, expected);
+ }
+ //Held only in the node's transaction index, so that gettransaction fails and getrawtransaction answers
+ Map<String, String> indexedTransactions = new HashMap<>();
+ addTransaction(1000, indexedTransactions, expected);
+
+ List<String> txids = new ArrayList<>(walletTransactions.keySet());
+ txids.add(50, indexedTransactions.keySet().iterator().next());
+
+ try(FakeNode node = new FakeNode(walletTransactions, indexedTransactions, nodeVersion < 280000)) {
+ BitcoindClient client = createClient(node, nodeVersion);
+ try {
+ Map<String, Set<HashIndex>> spentOutputs = new HashMap<>();
+ assertTimeoutPreemptively(Duration.ofSeconds(20), () -> client.fetchSpentOutputs(txids, spentOutputs));
+ assertEquals(expected, spentOutputs);
+ assertEquals(expectedBatchSizes, node.batchSizes);
+ assertEquals(expectedRequests, node.requests.get());
+ } finally {
+ client.stop();
+ }
+ }
+ }
+
+ private static void addTransaction(int index, Map<String, String> transactions, Map<String, Set<HashIndex>> expected) {
+ Transaction transaction = new Transaction();
+ transaction.setVersion(2);
+ Sha256Hash spentHash = Sha256Hash.of(("spent" + index).getBytes(StandardCharsets.UTF_8));
+ transaction.addInput(spentHash, index % 4, new Script(new byte[0]));
+ transaction.addOutput(1000L + index, new Script(new byte[] {0x51}));
+ String txid = transaction.getTxId().toString();
+ transactions.put(txid, Utils.bytesToHex(transaction.bitcoinSerialize()));
+ expected.put(txid, Set.of(new HashIndex(spentHash, index % 4)));
+ }
+
+ //Answers gettransaction and getrawtransaction requests, singly or in a batch, on kept-alive connections. Before v28 Bitcoin Core answers in the JSON-RPC 1.0 form,
+ //without a version and with both result and error, and with HTTP 500 for a failed request. From v28 it answers a JSON-RPC 2.0 request with HTTP 200 and one of the two.
+ private static class FakeNode implements Closeable {
+ private final ServerSocket serverSocket = new ServerSocket(0, 50, InetAddress.getLoopbackAddress());
+ private final Map<String, String> walletTransactions;
+ private final Map<String, String> indexedTransactions;
+ private final boolean jsonRpc1Replies;
+ //Any request naming this transaction is answered with an error page instead of JSON-RPC, as by a failing proxy in front of the node
+ private volatile String failingTxid;
+ private final AtomicInteger requests = new AtomicInteger();
+ private final List<Integer> batchSizes = Collections.synchronizedList(new ArrayList<>());
+
+ FakeNode(Map<String, String> walletTransactions, Map<String, String> indexedTransactions, boolean jsonRpc1Replies) throws IOException {
+ this.walletTransactions = walletTransactions;
+ this.indexedTransactions = indexedTransactions;
+ this.jsonRpc1Replies = jsonRpc1Replies;
+ Thread acceptor = new Thread(() -> {
+ try {
+ while(true) {
+ Socket socket = serverSocket.accept();
+ Thread handler = new Thread(() -> serve(socket));
+ handler.setDaemon(true);
+ handler.start();
+ }
+ } catch(IOException e) {
+ //Closed at the end of the test
+ }
+ });
+ acceptor.setDaemon(true);
+ acceptor.start();
+ }
+
+ int getPort() {
+ return serverSocket.getLocalPort();
+ }
+
+ private void serve(Socket socket) {
+ try(socket) {
+ InputStream in = new BufferedInputStream(socket.getInputStream());
+ while(true) {
+ int contentLength = -1;
+ for(String line = readLine(in); ; line = readLine(in)) {
+ if(line == null) {
+ return;
+ }
+ if(line.isEmpty()) {
+ break;
+ }
+ if(line.toLowerCase(Locale.ROOT).startsWith("content-length:")) {
+ contentLength = Integer.parseInt(line.substring("content-length:".length()).trim());
+ }
+ }
+ byte[] requestBytes = in.readNBytes(contentLength);
+ JsonNode request = MAPPER.readTree(requestBytes);
+ requests.incrementAndGet();
+
+ if(failingTxid != null && new String(requestBytes, StandardCharsets.UTF_8).contains(failingTxid)) {
+ String page = "Internal Server Error";
+ socket.getOutputStream().write(("HTTP/1.1 500 Internal Server Error\r\nContent-Type: text/plain\r\nContent-Length: " + page.length() + "\r\n\r\n" + page)
+ .getBytes(StandardCharsets.US_ASCII));
+ socket.getOutputStream().flush();
+ continue;
+ }
+
+ int status = 200;
+ String body;
+ if(request.isArray()) {
+ batchSizes.add(request.size());
+ ArrayNode responses = MAPPER.createArrayNode();
+ request.forEach(element -> responses.add(answer(element)));
+ body = MAPPER.writeValueAsString(responses);
+ } else {
+ ObjectNode response = answer(request);
+ status = jsonRpc1Replies && !response.get("error").isNull() ? 500 : 200;
+ body = MAPPER.writeValueAsString(response);
+ }
+
+ byte[] bytes = body.getBytes(StandardCharsets.UTF_8);
+ socket.getOutputStream().write(("HTTP/1.1 " + status + " Status\r\nContent-Type: application/json\r\nContent-Length: " + bytes.length + "\r\n\r\n")
+ .getBytes(StandardCharsets.US_ASCII));
+ socket.getOutputStream().write(bytes);
+ socket.getOutputStream().flush();
+ }
+ } catch(IOException e) {
+ //Closed by the client
+ }
+ }
+
+ private ObjectNode answer(JsonNode request) {
+ ObjectNode response = MAPPER.createObjectNode();
+ if(!jsonRpc1Replies) {
+ response.put("jsonrpc", "2.0");
+ }
+ String method = request.get("method").asText();
+ String txid = request.get("params").get(0).asText();
+ if(method.equals("gettransaction") && walletTransactions.containsKey(txid)) {
+ response.putObject("result").put("hex", walletTransactions.get(txid));
+ if(jsonRpc1Replies) {
+ response.putNull("error");
+ }
+ } else if(method.equals("getrawtransaction") && indexedTransactions.containsKey(txid)) {
+ response.put("result", indexedTransactions.get(txid));
+ if(jsonRpc1Replies) {
+ response.putNull("error");
+ }
+ } else {
+ if(jsonRpc1Replies) {
+ response.putNull("result");
+ }
+ ObjectNode error = response.putObject("error");
+ error.put("code", -5);
+ error.put("message", "Invalid or non-wallet transaction id");
+ }
+ response.set("id", request.get("id"));
+
+ return response;
+ }
+
+ private static String readLine(InputStream in) throws IOException {
+ StringBuilder line = new StringBuilder();
+ int b;
+ while((b = in.read()) != -1 && b != '\n') {
+ if(b != '\r') {
+ line.append((char)b);
+ }
+ }
+ return b == -1 && line.isEmpty() ? null : line.toString();
+ }
+
+ @Override
+ public void close() throws IOException {
+ serverSocket.close();
+ }
+ }
+}
### src/test/java/com/sparrowwallet/sparrow/net/cormorant/bitcoind/BitcoindTransportProxyTest.java
@@ -115,11 +115,20 @@ public void authenticationFailureIsReported() {
}
@Test
- public void closedConnectionPointsToRpcAllowIp() {
+ public void closedConnectionThroughProxyDoesNotPointToRpcAllowIp() {
proxy.response = "";
BitcoindTransport transport = new BitcoindTransport(new Server("http://" + onionHost('e') + ":8332"), "cormorant", RPC_AUTH);
IOException thrown = assertPassThrows(transport);
- assertTrue(thrown.getMessage().contains("rpcallowip"), thrown.toString());
+ assertTrue(thrown.getMessage().contains("closed the connection without responding"), thrown.toString());
+ assertFalse(thrown.getMessage().contains("rpcallowip"), thrown.toString());
+ }
+
+ @Test
+ public void batchResponseIsReturned() {
+ String batchResponse = "[" + RPC_RESPONSE + "," + RPC_RESPONSE.replace("\"id\":1", "\"id\":2") + "]";
+ proxy.response = "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: " + batchResponse.length() + "\r\n\r\n" + batchResponse;
+ BitcoindTransport transport = new BitcoindTransport(new Server("http://" + onionHost('k') + ":8332"), "cormorant", RPC_AUTH);
+ assertEquals(batchResponse, pass(transport));
}
@TestWhy this scored 29/100
Community notes
Notes can correct, qualify, or add evidence to the AI analysis. Every note shown here has been validated by a human moderator.
The AI analysis stands alone for now. Submit a note if you can add evidence or important context.