Merge bitcoin/bitcoin#36725: refactor: Use u64 consistently for flatfile sizes
What changed, and why it matters
This is a code cleanup (refactor) that switches file-size variables from 32-bit or platform-dependent types to a consistent 64-bit unsigned type. It also adds unit tests for variable-integer overflow and simplifies a Windows file-size API call. The changes reduce the risk of future bugs related to very large files or integer overflow, but they do not by themselves fix a known exploitable vulnerability.
Treat as routine defensive hardening. Reviewers should verify that no implicit narrowing conversions remain at call sites and that the new tests pass. No urgent security response is indicated by the commit content alone.
Security signals we found
Integer type widening for file sizes (defensive hardening)
Added unit tests for VarInt deserialization overflow
Windows file-size API changed to use QuadPart (no functional change)
No explicit vulnerability fix or advisory language in commit
Evidence from the diff
The commit changes flatfile/blockstorage/filter-index size and chunk-size variables from unsigned int/size_t to uint64_t, updates AllocateFileRange to take uint64_t length, uses LARGE_INTEGER.QuadPart on Windows, and adds serialize_tests coverage for VarInt overflow paths. The VarInt overflow tests exercise existing throw conditions in ReadVarInt with improved error messages. No new runtime behavior that would prevent an active attack is introduced; it is primarily a type-consistency and test-coverage improvement.
Changed components
src/flatfile.cpp / src/flatfile.hsrc/index/blockfilterindex.cpp / src/index/blockfilterindex.hsrc/node/blockstorage.cpp / src/node/blockstorage.hsrc/serialize.hsrc/test/serialize_tests.cppsrc/util/fs_helpers.cpp / src/util/fs_helpers.hInspect captured patch +64 / −37
### src/flatfile.cpp
@@ -12,8 +12,8 @@
#include <stdexcept>
-FlatFileSeq::FlatFileSeq(fs::path dir, const char* prefix, size_t chunk_size) :
- m_dir(std::move(dir)),
+FlatFileSeq::FlatFileSeq(fs::path dir, const char* prefix, uint64_t chunk_size)
+ : m_dir(std::move(dir)),
m_prefix(prefix),
m_chunk_size(chunk_size)
{
@@ -56,16 +56,16 @@ FILE* FlatFileSeq::Open(const FlatFilePos& pos, bool read_only) const
return file;
}
-size_t FlatFileSeq::Allocate(const FlatFilePos& pos, size_t add_size, bool& out_of_space) const
+uint64_t FlatFileSeq::Allocate(const FlatFilePos& pos, uint64_t add_size, bool& out_of_space) const
{
out_of_space = false;
- uint64_t n_old_chunks = CeilDiv(pos.nPos, uint64_t{m_chunk_size});
- uint64_t n_new_chunks = CeilDiv(pos.nPos + add_size, uint64_t{m_chunk_size});
+ uint64_t n_old_chunks = CeilDiv(pos.nPos, m_chunk_size);
+ uint64_t n_new_chunks = CeilDiv(pos.nPos + add_size, m_chunk_size);
if (n_new_chunks > n_old_chunks) {
uint64_t old_size = pos.nPos;
uint64_t new_size = n_new_chunks * m_chunk_size;
- size_t inc_size = new_size - old_size;
+ uint64_t inc_size{new_size - old_size};
if (CheckDiskSpace(m_dir, inc_size)) {
FILE *file = Open(pos);
### src/flatfile.h
@@ -43,7 +43,7 @@ class FlatFileSeq
private:
const fs::path m_dir;
const char* const m_prefix;
- const size_t m_chunk_size;
+ const uint64_t m_chunk_size;
public:
/**
@@ -53,7 +53,7 @@ class FlatFileSeq
* @param prefix A short prefix given to all file names.
* @param chunk_size Disk space is pre-allocated in multiples of this amount.
*/
- FlatFileSeq(fs::path dir, const char* prefix, size_t chunk_size);
+ FlatFileSeq(fs::path dir, const char* prefix, uint64_t chunk_size);
/** Get the name of the file at the given position. */
fs::path FileName(const FlatFilePos& pos) const;
@@ -70,7 +70,7 @@ class FlatFileSeq
* @param[out] out_of_space Whether the allocation failed due to insufficient disk space.
* @return The number of bytes successfully allocated.
*/
- size_t Allocate(const FlatFilePos& pos, size_t add_size, bool& out_of_space) const;
+ uint64_t Allocate(const FlatFilePos& pos, uint64_t add_size, bool& out_of_space) const;
/**
* Commit a file to disk, and optionally truncate off extra pre-allocated bytes if final.
### src/index/blockfilterindex.cpp
@@ -50,9 +50,9 @@
*/
constexpr uint8_t DB_FILTER_POS{'P'};
-constexpr unsigned int MAX_FLTR_FILE_SIZE{16_MiB};
+constexpr uint64_t MAX_FLTR_FILE_SIZE{16_MiB};
/** The pre-allocation chunk size for fltr?????.dat files */
-constexpr unsigned int FLTR_FILE_CHUNK_SIZE{1_MiB};
+constexpr uint64_t FLTR_FILE_CHUNK_SIZE{1_MiB};
/** Maximum size of the cfheaders cache
* We have a limit to prevent a bug in filling this cache
* potentially turning into an OOM. At 2000 entries, this cache
@@ -184,7 +184,7 @@ bool BlockFilterIndex::ReadFilterFromDisk(const FlatFilePos& pos, const uint256&
return true;
}
-size_t BlockFilterIndex::WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& filter)
+uint64_t BlockFilterIndex::WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& filter)
{
assert(filter.GetFilterType() == GetFilterType());
@@ -268,7 +268,7 @@ bool BlockFilterIndex::CustomAppend(const interfaces::BlockInfo& block)
bool BlockFilterIndex::Write(const BlockFilter& filter, uint32_t block_height, const uint256& filter_header)
{
- size_t bytes_written = WriteFilterToDisk(m_next_filter_pos, filter);
+ uint64_t bytes_written{WriteFilterToDisk(m_next_filter_pos, filter)};
if (bytes_written == 0) return false;
std::pair<uint256, DBVal> value;
### src/index/blockfilterindex.h
@@ -47,7 +47,7 @@ class BlockFilterIndex final : public BaseIndex
std::unique_ptr<FlatFileSeq> m_filter_fileseq;
bool ReadFilterFromDisk(const FlatFilePos& pos, const uint256& hash, BlockFilter& filter) const;
- size_t WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& filter);
+ uint64_t WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& filter);
Mutex m_cs_headers_cache;
/** cache of block hash to filter header, to avoid disk access when responding to getcfcheckpt. */
### src/node/blockstorage.cpp
@@ -847,7 +847,7 @@ fs::path BlockManager::GetBlockPosFilename(const FlatFilePos& pos) const
return m_block_file_seq.FileName(pos);
}
-FlatFilePos BlockManager::FindNextBlockPos(unsigned int nAddSize, unsigned int nHeight, uint64_t nTime)
+FlatFilePos BlockManager::FindNextBlockPos(uint64_t nAddSize, unsigned int nHeight, uint64_t nTime)
{
AssertLockHeld(::cs_main);
const BlockfileType chain_type = BlockfileTypeForHeight(nHeight);
@@ -867,7 +867,7 @@ FlatFilePos BlockManager::FindNextBlockPos(unsigned int nAddSize, unsigned int n
}
bool finalize_undo = false;
- unsigned int max_blockfile_size{MAX_BLOCKFILE_SIZE};
+ uint64_t max_blockfile_size{MAX_BLOCKFILE_SIZE};
// Use smaller blockfiles in test-only -fastprune mode - but avoid
// the possibility of having a block not fit into the block file.
if (m_opts.fast_prune) {
@@ -923,7 +923,7 @@ FlatFilePos BlockManager::FindNextBlockPos(unsigned int nAddSize, unsigned int n
m_blockfile_info[nFile].nSize += nAddSize;
bool out_of_space;
- size_t bytes_allocated = m_block_file_seq.Allocate(pos, nAddSize, out_of_space);
+ uint64_t bytes_allocated{m_block_file_seq.Allocate(pos, nAddSize, out_of_space)};
if (out_of_space) {
m_opts.notifications.fatalError(_("Disk space is too low!"));
return {};
@@ -947,7 +947,7 @@ void BlockManager::UpdateBlockInfo(const CBlock& block, unsigned int nHeight, co
}
// Update the file information with the current block.
- const unsigned int added_size = ::GetSerializeSize(TX_WITH_WITNESS(block));
+ const uint64_t added_size{::GetSerializeSize(TX_WITH_WITNESS(block))};
const int nFile = pos.nFile;
if (static_cast<int>(m_blockfile_info.size()) <= nFile) {
m_blockfile_info.resize(nFile + 1);
@@ -957,7 +957,7 @@ void BlockManager::UpdateBlockInfo(const CBlock& block, unsigned int nHeight, co
m_dirty_fileinfo.insert(nFile);
}
-bool BlockManager::FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize)
+bool BlockManager::FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, uint64_t nAddSize)
{
AssertLockHeld(::cs_main);
pos.nFile = nFile;
@@ -967,7 +967,7 @@ bool BlockManager::FindUndoPos(BlockValidationState& state, int nFile, FlatFileP
m_dirty_fileinfo.insert(nFile);
bool out_of_space;
- size_t bytes_allocated = m_undo_file_seq.Allocate(pos, nAddSize, out_of_space);
+ uint64_t bytes_allocated{m_undo_file_seq.Allocate(pos, nAddSize, out_of_space)};
if (out_of_space) {
return FatalError(m_opts.notifications, state, _("Disk space is too low!"));
}
### src/node/blockstorage.h
@@ -117,17 +117,17 @@ using kernel::CBlockFileInfo;
using kernel::BlockTreeDB;
/** The pre-allocation chunk size for blk?????.dat files (since 0.8) */
-inline constexpr unsigned int BLOCKFILE_CHUNK_SIZE{16_MiB};
+inline constexpr uint64_t BLOCKFILE_CHUNK_SIZE{16_MiB};
/** The pre-allocation chunk size for rev?????.dat files (since 0.8) */
-inline constexpr unsigned int UNDOFILE_CHUNK_SIZE{1_MiB};
+inline constexpr uint64_t UNDOFILE_CHUNK_SIZE{1_MiB};
/** The maximum size of a blk?????.dat file (since 0.8) */
-inline constexpr unsigned int MAX_BLOCKFILE_SIZE{128_MiB};
+inline constexpr uint64_t MAX_BLOCKFILE_SIZE{128_MiB};
/** Size of header written by WriteBlock before a serialized CBlock (8 bytes) */
-inline constexpr uint32_t STORAGE_HEADER_BYTES{std::tuple_size_v<MessageStartChars> + sizeof(unsigned int)};
+inline constexpr uint64_t STORAGE_HEADER_BYTES{std::tuple_size_v<MessageStartChars> + sizeof(uint32_t)};
/** Total overhead when writing undo data: header (8 bytes) plus checksum (32 bytes) */
-inline constexpr uint32_t UNDO_DATA_DISK_OVERHEAD{STORAGE_HEADER_BYTES + uint256::size()};
+inline constexpr uint64_t UNDO_DATA_DISK_OVERHEAD{STORAGE_HEADER_BYTES + uint256::size()};
// Because validation code takes pointers to the map's CBlockIndex objects, if
// we ever switch to another associative container, we need to either use a
@@ -221,9 +221,9 @@ class BlockManager
* The nAddSize argument passed to this function should include not just the size of the serialized CBlock, but also the size of
* separator fields (STORAGE_HEADER_BYTES).
*/
- [[nodiscard]] FlatFilePos FindNextBlockPos(unsigned int nAddSize, unsigned int nHeight, uint64_t nTime) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
+ [[nodiscard]] FlatFilePos FindNextBlockPos(uint64_t nAddSize, unsigned int nHeight, uint64_t nTime) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
[[nodiscard]] bool FlushChainstateBlockFile(int tip_height) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
- [[nodiscard]] bool FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
+ [[nodiscard]] bool FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, uint64_t nAddSize) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
AutoFile OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false) const;
### src/serialize.h
@@ -450,12 +450,12 @@ I ReadVarInt(Stream& is)
while(true) {
unsigned char chData = ser_readdata8(is);
if (n > (std::numeric_limits<I>::max() >> 7)) {
- throw std::ios_base::failure("ReadVarInt(): size too large");
+ throw std::ios_base::failure("VarInt exceeds destination range");
}
n = (n << 7) | (chData & 0x7F);
if (chData & 0x80) {
if (n == std::numeric_limits<I>::max()) {
- throw std::ios_base::failure("ReadVarInt(): size too large");
+ throw std::ios_base::failure("VarInt increment exceeds destination range");
}
n++;
} else {
### src/test/serialize_tests.cpp
@@ -9,12 +9,13 @@
#include <test/util/setup_common.h>
#include <util/strencodings.h>
+#include <boost/test/unit_test.hpp>
+
#include <algorithm>
#include <cstdint>
+#include <limits>
#include <string>
-#include <boost/test/unit_test.hpp>
-
BOOST_FIXTURE_TEST_SUITE(serialize_tests, BasicTestingSetup)
// For testing move-semantics, declare a version of datastream that can be moved
@@ -135,6 +136,35 @@ BOOST_AUTO_TEST_CASE(varints)
}
}
+BOOST_AUTO_TEST_CASE(varints_overflow)
+{
+ const auto read_varint = [](uint64_t serialized, auto value) {
+ DataStream ss{};
+ ss << VARINT(serialized);
+ ss >> VARINT(value);
+ return value;
+ };
+
+ for (const uint32_t small_value : {
+ uint32_t{0},
+ uint32_t{1},
+ std::numeric_limits<uint32_t>::max(),
+ }) {
+ BOOST_CHECK_EQUAL(read_varint(small_value, uint32_t{}), small_value);
+ }
+
+ for (const uint64_t large_value : {
+ uint64_t{std::numeric_limits<uint32_t>::max()} + 1,
+ std::numeric_limits<uint64_t>::max(),
+ }) {
+ BOOST_CHECK_EQUAL(read_varint(large_value, uint64_t{}), large_value);
+
+ BOOST_CHECK_EXCEPTION(read_varint(large_value, uint32_t{}), std::ios_base::failure, HasReason{"VarInt exceeds destination range"});
+ }
+
+ BOOST_CHECK_EXCEPTION(read_varint(uint64_t{1} << 39, uint32_t{}), std::ios_base::failure, HasReason{"VarInt increment exceeds destination range"});
+}
+
BOOST_AUTO_TEST_CASE(varints_bitpatterns)
{
DataStream ss{};
### src/util/fs_helpers.cpp
@@ -197,15 +197,12 @@ int RaiseFileDescriptorLimit(int min_fd)
* this function tries to make a particular range of a file allocated (corresponding to disk space)
* it is advisory, and the range specified in the arguments will never contain live data
*/
-void AllocateFileRange(FILE* file, uint64_t offset, unsigned int length)
+void AllocateFileRange(FILE* file, uint64_t offset, uint64_t length)
{
#if defined(WIN32)
// Windows-specific version
HANDLE hFile = (HANDLE)_get_osfhandle(_fileno(file));
- LARGE_INTEGER nFileSize;
- int64_t nEndPos = (int64_t)offset + length;
- nFileSize.u.LowPart = nEndPos & 0xFFFFFFFF;
- nFileSize.u.HighPart = nEndPos >> 32;
+ LARGE_INTEGER nFileSize{.QuadPart = int64_t(offset + length)};
SetFilePointerEx(hFile, nFileSize, 0, FILE_BEGIN);
SetEndOfFile(hFile);
#elif defined(__APPLE__)
### src/util/fs_helpers.h
@@ -87,7 +87,7 @@ inline int64_t FileTell64(FILE* file)
*/
int RaiseFileDescriptorLimit(int min_fd);
-void AllocateFileRange(FILE* file, uint64_t offset, unsigned int length);
+void AllocateFileRange(FILE* file, uint64_t offset, uint64_t length);
/**
* Rename src to dest.Why this scored 27/100
Community notes
Notes can correct, qualify, or add evidence to the AI analysis. Every note shown here has been validated by a human moderator.
The AI analysis stands alone for now. Submit a note if you can add evidence or important context.