Merge pull request #10945 from accumulator/stale_utxo_data_fixes

### electrum/address_synchronizer.py
@@ -275,9 +275,7 @@ def get_transaction(self, txid: str) -> Optional[Transaction]:
         if tx:
             tx.deserialize()
             for txin in tx._inputs:
-                tx_mined_info = self.get_tx_height(txin.prevout.txid.hex())
-                txin.block_height = tx_mined_info.height()
-                txin.block_txpos = tx_mined_info.txpos
+                txin.set_mined_info(self.get_tx_height(txin.prevout.txid.hex()))
         return tx
 
     def add_transaction(self, tx: Transaction, *, allow_unrelated=False, is_new=True) -> bool:
@@ -457,6 +455,7 @@ def receive_history_callback(self, addr: str, hist, tx_fees: Dict[str, int]):
                 self.unverified_tx.pop(tx_hash, None)
                 self.unconfirmed_tx.pop(tx_hash, None)
                 self.db.remove_verified_tx(tx_hash)
+                self.invalidate_cache()
                 if self.verifier:
                     self.verifier.remove_spv_proof_for_tx(tx_hash)
         self.db.set_addr_history(addr, hist)
@@ -635,19 +634,24 @@ def add_unverified_or_unconfirmed_tx(self, tx_hash: str, tx_height: int) -> None
                 # tx was previously SPV-verified but now in mempool (probably reorg)
                 self.db.remove_verified_tx(tx_hash)
                 self.unconfirmed_tx[tx_hash] = tx_height
+                self.invalidate_cache()
                 if self.verifier:
                     self.verifier.remove_spv_proof_for_tx(tx_hash)
         else:
             if tx_height > 0:
+                self.unconfirmed_tx.pop(tx_hash, None)
                 self.unverified_tx[tx_hash] = tx_height
             else:
+                self.unverified_tx.pop(tx_hash, None)
                 self.unconfirmed_tx[tx_hash] = tx_height
+            self.invalidate_cache()
 
     @with_lock
     def remove_unverified_tx(self, tx_hash: str, tx_height: int) -> None:
         new_height = self.unverified_tx.get(tx_hash)
         if new_height == tx_height:
             self.unverified_tx.pop(tx_hash, None)
+            self.invalidate_cache()
 
     def add_verified_tx(self, tx_hash: str, info: TxMinedInfo):
         # Remove from the unverified map and add to the verified map
@@ -684,6 +688,8 @@ def undo_verifications(self, blockchain: Blockchain, above_height: int) -> Set[s
                         # a status update, that will overwrite it.
                         self.unverified_tx[tx_hash] = tx_height
                         txs.add(tx_hash)
+            if txs:
+                self.invalidate_cache()
 
         for tx_hash in txs:
             util.trigger_callback('adb_removed_verified_tx', self, tx_hash)

### electrum/gui/qml/qeaddresslistmodel.py
@@ -177,10 +177,8 @@ def addr_to_model(self, addrtype: str, addridx: int, address: str):
 
     def coin_to_model(self, addrtype: str, coin: 'PartialTxInput'):
         txid = coin.prevout.txid.hex()
-        short_id = ''
-        # check below duplicated from TxInput as we cannot get short_id unambiguously
-        if coin.block_txpos is not None and coin.block_txpos >= 0:
-            short_id = str(coin.short_id)
+        # short_id falls back to the outpoint if the coin is not mined, hence the check
+        short_id = str(coin.short_id) if coin.has_short_id() else ''
         item = {
             'type': addrtype,
             'amount': QEAmount(amount_sat=coin.value_sats()),

### electrum/gui/qt/transaction_dialog.py
@@ -47,7 +47,7 @@
 from electrum.plugin import run_hook
 from electrum.transaction import SerializationError, Transaction, PartialTransaction, TxOutpoint, TxinDataFetchProgress
 from electrum.logging import get_logger
-from electrum.util import (ShortID, get_asyncio_loop, UI_UNIT_NAME_TXSIZE_VBYTES, delta_time_str,
+from electrum.util import (get_asyncio_loop, UI_UNIT_NAME_TXSIZE_VBYTES, delta_time_str,
                            UserCancelled)
 from electrum.network import Network
 from electrum.wallet import TxSighashRiskLevel, TxSighashDanger
@@ -278,16 +278,12 @@ def insert_tx_io(
         o_text.clear()
         o_text.setFont(QFont(MONOSPACE_FONT))
         o_text.setReadOnly(True)
-        tx_height, tx_pos = None, None
         tx_hash = self.tx.txid()
-        if tx_hash:
-            tx_mined_info = self.wallet.adb.get_tx_height(tx_hash)
-            tx_height = tx_mined_info.height()
-            tx_pos = tx_mined_info.txpos
+        tx_mined_info = self.wallet.adb.get_tx_height(tx_hash) if tx_hash else None
         cursor = o_text.textCursor()
         for txout_idx, o in enumerate(self.tx.outputs()):
-            if tx_height is not None and tx_pos is not None and tx_pos >= 0:
-                short_id = ShortID.from_components(tx_height, tx_pos, txout_idx)
+            if tx_mined_info and tx_mined_info.short_id():
+                short_id = f"{tx_mined_info.short_id()}x{txout_idx}"
             elif tx_hash:
                 short_id = TxOutpoint(bytes.fromhex(tx_hash), txout_idx).short_name()
             else:

### electrum/gui/qt/utxo_dialog.py
@@ -108,9 +108,7 @@ def print_ascii_tree(_txid, prefix, is_last, is_uncle):
             if _txid not in parents:
                 return
             tx_mined_info = self.wallet.adb.get_tx_height(_txid)
-            tx_height = tx_mined_info.height()
-            tx_pos = tx_mined_info.txpos
-            key = "%dx%d"%(tx_height, tx_pos) if tx_pos is not None else _txid[0:8]
+            key = tx_mined_info.short_id() or _txid[0:8]
             label = self.wallet.get_label_for_txid(_txid) or ""
             if _txid not in parents_copy:
                 label = '[duplicate]'

### electrum/transaction.py
@@ -51,7 +51,7 @@
 )
 from .crypto import sha256d, sha256
 from .logging import get_logger
-from .util import ShortID, OldTaskGroup
+from .util import ShortID, OldTaskGroup, TxMinedInfo
 from .descriptor import Descriptor, MissingSolutionPiece, create_dummy_descriptor_from_address, DUMMY_DER_SIG
 
 if TYPE_CHECKING:
@@ -363,9 +363,17 @@ def get_block_based_relative_locktime(self) -> Optional[int]:
             return self.nsequence & 0xffff
         return None
 
+    def set_mined_info(self, info: TxMinedInfo) -> None:
+        self.block_height = info.height()
+        self.block_txpos = info.txpos
+
+    def has_short_id(self) -> bool:
+        return (self.block_height is not None and self.block_height > 0
+                and self.block_txpos is not None and self.block_txpos >= 0)
+
     @property
     def short_id(self):
-        if self.block_txpos is not None and self.block_txpos >= 0:
+        if self.has_short_id():
             return ShortID.from_components(self.block_height, self.block_txpos, self.prevout.out_idx)
         else:
             return self.prevout.short_name()

### electrum/util.py
@@ -1283,8 +1283,8 @@ def height(self) -> int:
             return h
 
     def short_id(self) -> Optional[str]:
-        if self.txpos is not None and self.txpos >= 0:
-            assert self.height() > 0
+        """'<height>x<txpos>' if mined and SPV-verified, else None."""
+        if self.height() > 0 and self.txpos is not None and self.txpos >= 0:
             return f"{self.height()}x{self.txpos}"
         return None
 

### electrum/wallet.py
@@ -2737,7 +2737,7 @@ def add_input_info(
             txin.script_descriptor = desc
         txin.is_mine = True
         self._add_txinout_derivation_info(txin, address, only_der_suffix=only_der_suffix)
-        txin.block_height = self.adb.get_tx_height(txin.prevout.txid.hex()).height()
+        txin.set_mined_info(self.adb.get_tx_height(txin.prevout.txid.hex()))
 
     def has_support_for_slip_19_ownership_proofs(self) -> bool:
         return False

### tests/test_transaction.py
@@ -159,6 +159,28 @@ def test_estimated_output_size(self):
         self.assertEqual(estimated_output_size('bc1q3g5tmkmlvxryhh843v4dz026avatc0zzr6h3af'), 31)
         self.assertEqual(estimated_output_size('bc1qnvks7gfdu72de8qv6q6rhkkzu70fqz4wpjzuxjf6aydsx7wxfwcqnlxuv3'), 43)
 
+    def test_txin_short_id(self):
+        prevout = TxOutpoint.from_str(
+            'db949963c3787c90a40fb689ffdc3146c27a9874a970d1fd20921afbe79a7aa9:0')
+        outpoint_str = 'db949963c3:0'
+
+        def short_id_for(*, block_height, block_txpos):
+            txin = PartialTxInput(prevout=prevout)
+            txin.block_height = block_height
+            txin.block_txpos = block_txpos
+            return str(txin.short_id)
+
+        # mined and SPV-ed:
+        self.assertEqual('600000x7x0', short_id_for(block_height=600000, block_txpos=7))
+        self.assertEqual('600000x0x0', short_id_for(block_height=600000, block_txpos=0))
+        # position in block unknown:
+        self.assertEqual(outpoint_str, short_id_for(block_height=600000, block_txpos=None))
+        self.assertEqual(outpoint_str, short_id_for(block_height=600000, block_txpos=-1))
+        # not mined (or not known to be mined).
+        for height in (0, -1, -2, -3, None):
+            self.assertEqual(outpoint_str, short_id_for(block_height=height, block_txpos=7))
+            self.assertEqual(outpoint_str, short_id_for(block_height=height, block_txpos=None))
+
     # TODO other tests for segwit tx
     def test_tx_signed_segwit(self):
         tx = transaction.Transaction(signed_segwit_blob)

### tests/test_wallet_vertical.py
@@ -4734,6 +4734,87 @@ async def test_get_tx_status_feerate_for_local_2of3_multisig_signed_tx(self):
                          wallet1.get_tx_status(tx.txid(), TxMinedInfo(_height=TX_HEIGHT_LOCAL, conf=0)))
 
 
+class TestWalletHistory_CacheInvalidation(ElectrumTestCase):
+    TESTNET = True
+    # funds tb1qwllx6238azrqcfudf5kdjzadw94mj5s653v5e7 (of the seed below) with 1_999_890 sat:
+    FUNDING_TX = "0200000000010132515e6aade1b79ec7dd3bac0896d8b32c56195d23d07d48e21659cef24301560100000000fdffffff0112841e000000000016001477fe6d2a27e8860c278d4d2cd90bad716bb9521a02473044022041ed68ef7ef122813ac6a5e996b8284f645c53fbe6823b8e430604a8915a867802203233f5f4d347a687eb19b2aa570829ab12aeeb29a24cc6d6d20b8b3d79e971ae012102bee0ee043817e50ac1bb31132770f7c41e35946ccdcb771750fb9696bdd1b307ad951d00"
+    FUNDING_TXID = "db949963c3787c90a40fb689ffdc3146c27a9874a970d1fd20921afbe79a7aa9"
+
+    def setUp(self):
+        super().setUp()
+        self.config = SimpleConfig({'electrum_path': self.electrum_path})
+
+    def create_wallet(self, *, tx_height: int) -> Abstract_Wallet:
+        w = restore_wallet_from_text__for_unittest(
+            "cross end slow expose giraffe fuel track awake turtle capital ranch pulp",
+            path=None, gap_limit=5, config=self.config)['wallet']
+        w.db.put('stored_height', 1010)
+        w.adb.receive_tx_callback(Transaction(self.FUNDING_TX), tx_height=tx_height)
+        return w
+
+    def create_wallet_with_mined_funding_tx(self) -> Abstract_Wallet:
+        w = self.create_wallet(tx_height=TX_HEIGHT_UNCONFIRMED)
+        w.adb.add_verified_tx(
+            self.FUNDING_TXID,
+            TxMinedInfo(_height=1001, timestamp=1700000001, txpos=7, header_hash="01"*32))
+        return w
+
+    async def test_caches_are_invalidated_when_mined_tx_goes_back_to_mempool(self):
+        w = self.create_wallet_with_mined_funding_tx()
+        self.assertEqual((1999890, 0, 0), w.get_balance())
+        self.assertEqual('1001x7x0', str(w.get_utxos()[0].short_id))
+        # the server tells us the tx is in the mempool again (e.g. after a reorg)
+        w.adb.add_unverified_or_unconfirmed_tx(self.FUNDING_TXID, TX_HEIGHT_UNCONF_PARENT)
+        self.assertEqual((0, 1999890, 0), w.get_balance())
+        utxo = w.get_utxos()[0]
+        self.assertEqual(TX_HEIGHT_UNCONF_PARENT, utxo.block_height)
+        self.assertFalse(utxo.has_short_id())
+
+    async def test_caches_are_invalidated_by_undo_verifications(self):
+        w = self.create_wallet_with_mined_funding_tx()
+        self.assertEqual((1999890, 0, 0), w.get_balance())
+        # reorg: the block that mined the tx is gone
+        blockchain = mock.Mock()
+        blockchain.read_header.return_value = None
+        self.assertEqual({self.FUNDING_TXID}, w.adb.undo_verifications(blockchain, above_height=1000))
+        self.assertEqual((0, 1999890, 0), w.get_balance())
+        utxo = w.get_utxos()[0]
+        self.assertEqual(0, utxo.block_height)  # unverified, so treated as unconfirmed
+        self.assertFalse(utxo.has_short_id())
+
+    async def test_caches_are_invalidated_when_unverified_tx_becomes_local(self):
+        w = self.create_wallet(tx_height=1001)  # mined, but not SPV-ed yet
+        self.assertEqual((0, 1999890, 0), w.get_balance())
+        self.assertEqual(0, w.get_utxos()[0].block_height)
+        w.adb.remove_unverified_tx(self.FUNDING_TXID, 1001)
+        self.assertEqual(TX_HEIGHT_LOCAL, w.get_utxos()[0].block_height)
+
+    async def test_caches_are_invalidated_when_mined_tx_disappears_from_server_history(self):
+        w = self.create_wallet_with_mined_funding_tx()
+        self.assertEqual((1999890, 0, 0), w.get_balance())
+        utxo = w.get_utxos()[0]
+        self.assertEqual(1001, utxo.block_height)
+        # the server no longer knows the tx at all (e.g. reorged out and evicted from the mempool)
+        w.adb.receive_history_callback(utxo.address, [], {})
+        self.assertEqual(TX_HEIGHT_LOCAL, w.adb.get_tx_height(self.FUNDING_TXID).height())
+        self.assertEqual((0, 1999890, 0), w.get_balance())
+        self.assertEqual(TX_HEIGHT_LOCAL, w.get_utxos()[0].block_height)
+
+    async def test_short_id_of_txin_of_tx_that_went_back_to_mempool(self):
+        # the coin is cached while its funding tx is mined, but by the time the
+        # tx is built, the funding tx is back in the mempool.
+        w = self.create_wallet_with_mined_funding_tx()
+        coins = w.get_spendable_coins(None)  # fills the utxo cache
+        w.adb.add_unverified_or_unconfirmed_tx(self.FUNDING_TXID, TX_HEIGHT_UNCONF_PARENT)
+        outputs = [PartialTxOutput.from_address_and_value("tb1qgh5c088he4d559wl0hw27hrdeg8p2z96pefn4q", 100_000)]
+        tx = w.make_unsigned_transaction(outputs=outputs, coins=coins, fee_policy=FixedFeePolicy(5000))
+        txin = tx.inputs()[0]
+        self.assertEqual(TX_HEIGHT_UNCONF_PARENT, txin.block_height)
+        self.assertIsNone(txin.block_txpos)
+        self.assertFalse(txin.has_short_id())
+        self.assertEqual("db949963c3:0", str(txin.short_id))
+
+
 class TestImportedWallet(ElectrumTestCase):
     TESTNET = True
     transactions = {