common: implement tal_arr_append and tal_arr_appendn, and use them.
What changed, and why it matters
This commit is a routine code cleanup: it introduces two helper functions, tal_arr_append and tal_arr_appendn, to replace repeated open-coded patterns of resizing a memory array and copying data into it. The change touches many files but does not fix any reported security bug. It adds extra memory-checking under Valgrind and a new unit test. There is no indication this patch is a security fix or that it addresses a known vulnerability.
No security action required. Treat as normal code-quality/maintenance commit. Reviewers may optionally verify that the new helper's TAKES semantics and memcheck behavior match all converted call sites, and that the new unit test covers empty, non-empty, partial, zero-length, struct, and take() cases.
Security signals we found
Refactoring only: replaces explicit tal_resize/memcpy pairs with a common helper
Adds memcheck() on appended buffers for extra Valgrind coverage
Adds unit tests for the new helper functions
No change to input validation, parsing, or cryptographic operations
No vendor disclosure or CVE reference present
Evidence from the diff
The commit adds tal_arr_append_() and tal_arr_appendn_() in common/utils.c, with type-safe macros in common/utils.h. These helpers wrap tal_resize() + memcpy() and honor the project’s TAKES/take() memory-ownership convention. Call sites in bitcoin/psbt.c, bitcoin/script.c, common/bolt11.c, connectd/multiplex.c, lightningd/plugin.c, plugins/commando.c, tools/lightning-downgrade.c, and two test files are converted to use the helpers. The implementation also memcheck()s the source buffer and adds a new unit test (common/test/run-utils-tal_arr_append.c). The diff is purely refactor-plus-test; no bounds-checking logic, input validation, or protocol behavior changes are introduced.
Changed components
common/utils.ccommon/utils.hbitcoin/psbt.cbitcoin/script.ccommon/bolt11.cconnectd/multiplex.clightningd/plugin.cplugins/commando.ctools/lightning-downgrade.ccommon/test/run-utils-tal_arr_append.cInspect captured patch +287 / −37
diff --git a/bitcoin/psbt.c b/bitcoin/psbt.c
index 68564595..10dada97 100644
--- a/bitcoin/psbt.c
+++ b/bitcoin/psbt.c
@@ -673,9 +673,7 @@ size_t psbt_output_get_weight(const struct wally_psbt *psbt,
static void add(u8 **key, const void *mem, size_t len)
{
- size_t oldlen = tal_count(*key);
- tal_resize(key, oldlen + len);
- memcpy(*key + oldlen, memcheck(mem, len), len);
+ tal_arr_appendn(key, mem, len);
}
static void add_type(u8 **key, const u8 num)
diff --git a/bitcoin/script.c b/bitcoin/script.c
index 3b38eae9..ede201e9 100644
--- a/bitcoin/script.c
+++ b/bitcoin/script.c
@@ -37,9 +37,7 @@ static void hash160(struct ripemd160 *redeemhash, const void *mem, size_t len)
static void add(u8 **scriptp, const void *mem, size_t len)
{
- size_t oldlen = tal_count(*scriptp);
- tal_resize(scriptp, oldlen + len);
- memcpy(*scriptp + oldlen, mem, len);
+ tal_arr_appendn(scriptp, mem, len);
}
static void add_op(u8 **scriptp, u8 op)
diff --git a/common/bolt11.c b/common/bolt11.c
index 27e5bd24..bc851ed4 100644
--- a/common/bolt11.c
+++ b/common/bolt11.c
@@ -1251,8 +1251,6 @@ static void maybe_encode_9(u5 **data, const u8 *features,
static bool encode_extra(u5 **data, const struct bolt11_field *extra)
{
- size_t len;
-
/* Can't encode an invalid tag. */
if (bech32_charset_rev[(unsigned char)extra->tag] == -1)
return false;
@@ -1261,9 +1259,7 @@ static bool encode_extra(u5 **data, const struct bolt11_field *extra)
push_varlen_uint(data, tal_count(extra->data), 10);
/* extra->data is already u5s, so do this raw. */
- len = tal_count(*data);
- tal_resize(data, len + tal_count(extra->data));
- memcpy(*data + len, extra->data, tal_count(extra->data));
+ tal_arr_append(data, extra->data);
return true;
}
diff --git a/common/test/run-bolt12_merkle.c b/common/test/run-bolt12_merkle.c
index 902268de..59ab359f 100644
--- a/common/test/run-bolt12_merkle.c
+++ b/common/test/run-bolt12_merkle.c
@@ -48,8 +48,7 @@ static LAST_ARG_NULL void *concat_(const void *p, ...)
va_start(ap, p);
do {
- tal_resize(&ret, len + tal_bytelen(p));
- memcpy(ret + len, p, tal_bytelen(p));
+ tal_arr_append(&ret, p);
len += tal_bytelen(p);
} while ((p = va_arg(ap, const void *)) != NULL);
va_end(ap);
diff --git a/common/test/run-utils-tal_arr_append.c b/common/test/run-utils-tal_arr_append.c
new file mode 100644
index 00000000..483d8b28
--- /dev/null
+++ b/common/test/run-utils-tal_arr_append.c
@@ -0,0 +1,238 @@
+/* ChatGPT-derived */
+#include "config.h"
+#include <assert.h>
+#include <ccan/array_size/array_size.h>
+#include <ccan/tal/tal.h>
+#include <common/amount.h>
+#include <common/pseudorand.h>
+#include <common/setup.h>
+#include <common/utils.h>
+#include <wire/wire.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* AUTOGENERATED MOCKS START */
+/* Generated stub for amount_asset_is_main */
+bool amount_asset_is_main(struct amount_asset *asset UNNEEDED)
+{ fprintf(stderr, "amount_asset_is_main called!\n"); abort(); }
+/* Generated stub for amount_asset_to_sat */
+struct amount_sat amount_asset_to_sat(struct amount_asset *asset UNNEEDED)
+{ fprintf(stderr, "amount_asset_to_sat called!\n"); abort(); }
+/* Generated stub for amount_feerate */
+ bool amount_feerate(u32 *feerate UNNEEDED, struct amount_sat fee UNNEEDED, size_t weight UNNEEDED)
+{ fprintf(stderr, "amount_feerate called!\n"); abort(); }
+/* Generated stub for amount_sat */
+struct amount_sat amount_sat(u64 satoshis UNNEEDED)
+{ fprintf(stderr, "amount_sat called!\n"); abort(); }
+/* Generated stub for amount_sat_add */
+ bool amount_sat_add(struct amount_sat *val UNNEEDED,
+ struct amount_sat a UNNEEDED,
+ struct amount_sat b UNNEEDED)
+{ fprintf(stderr, "amount_sat_add called!\n"); abort(); }
+/* Generated stub for amount_sat_eq */
+bool amount_sat_eq(struct amount_sat a UNNEEDED, struct amount_sat b UNNEEDED)
+{ fprintf(stderr, "amount_sat_eq called!\n"); abort(); }
+/* Generated stub for amount_sat_greater_eq */
+bool amount_sat_greater_eq(struct amount_sat a UNNEEDED, struct amount_sat b UNNEEDED)
+{ fprintf(stderr, "amount_sat_greater_eq called!\n"); abort(); }
+/* Generated stub for amount_sat_sub */
+ bool amount_sat_sub(struct amount_sat *val UNNEEDED,
+ struct amount_sat a UNNEEDED,
+ struct amount_sat b UNNEEDED)
+{ fprintf(stderr, "amount_sat_sub called!\n"); abort(); }
+/* Generated stub for amount_sat_to_asset */
+struct amount_asset amount_sat_to_asset(struct amount_sat *sat UNNEEDED, const u8 *asset UNNEEDED)
+{ fprintf(stderr, "amount_sat_to_asset called!\n"); abort(); }
+/* Generated stub for amount_tx_fee */
+struct amount_sat amount_tx_fee(u32 fee_per_kw UNNEEDED, size_t weight UNNEEDED)
+{ fprintf(stderr, "amount_tx_fee called!\n"); abort(); }
+/* Generated stub for fromwire */
+const u8 *fromwire(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, void *copy UNNEEDED, size_t n UNNEEDED)
+{ fprintf(stderr, "fromwire called!\n"); abort(); }
+/* Generated stub for fromwire_bool */
+bool fromwire_bool(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
+{ fprintf(stderr, "fromwire_bool called!\n"); abort(); }
+/* Generated stub for fromwire_fail */
+void *fromwire_fail(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
+{ fprintf(stderr, "fromwire_fail called!\n"); abort(); }
+/* Generated stub for fromwire_secp256k1_ecdsa_signature */
+void fromwire_secp256k1_ecdsa_signature(const u8 **cursor UNNEEDED, size_t *max UNNEEDED,
+ secp256k1_ecdsa_signature *signature UNNEEDED)
+{ fprintf(stderr, "fromwire_secp256k1_ecdsa_signature called!\n"); abort(); }
+/* Generated stub for fromwire_sha256 */
+void fromwire_sha256(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct sha256 *sha256 UNNEEDED)
+{ fprintf(stderr, "fromwire_sha256 called!\n"); abort(); }
+/* Generated stub for fromwire_tal_arrn */
+u8 *fromwire_tal_arrn(const tal_t *ctx UNNEEDED,
+ const u8 **cursor UNNEEDED, size_t *max UNNEEDED, size_t num UNNEEDED)
+{ fprintf(stderr, "fromwire_tal_arrn called!\n"); abort(); }
+/* Generated stub for fromwire_u32 */
+u32 fromwire_u32(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
+{ fprintf(stderr, "fromwire_u32 called!\n"); abort(); }
+/* Generated stub for fromwire_u64 */
+u64 fromwire_u64(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
+{ fprintf(stderr, "fromwire_u64 called!\n"); abort(); }
+/* Generated stub for fromwire_u8 */
+u8 fromwire_u8(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
+{ fprintf(stderr, "fromwire_u8 called!\n"); abort(); }
+/* Generated stub for fromwire_u8_array */
+void fromwire_u8_array(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, u8 *arr UNNEEDED, size_t num UNNEEDED)
+{ fprintf(stderr, "fromwire_u8_array called!\n"); abort(); }
+/* Generated stub for siphash_seed */
+const struct siphash_seed *siphash_seed(void)
+{ fprintf(stderr, "siphash_seed called!\n"); abort(); }
+/* Generated stub for towire */
+void towire(u8 **pptr UNNEEDED, const void *data UNNEEDED, size_t len UNNEEDED)
+{ fprintf(stderr, "towire called!\n"); abort(); }
+/* Generated stub for towire_bool */
+void towire_bool(u8 **pptr UNNEEDED, bool v UNNEEDED)
+{ fprintf(stderr, "towire_bool called!\n"); abort(); }
+/* Generated stub for towire_secp256k1_ecdsa_signature */
+void towire_secp256k1_ecdsa_signature(u8 **pptr UNNEEDED,
+ const secp256k1_ecdsa_signature *signature UNNEEDED)
+{ fprintf(stderr, "towire_secp256k1_ecdsa_signature called!\n"); abort(); }
+/* Generated stub for towire_sha256 */
+void towire_sha256(u8 **pptr UNNEEDED, const struct sha256 *sha256 UNNEEDED)
+{ fprintf(stderr, "towire_sha256 called!\n"); abort(); }
+/* Generated stub for towire_u32 */
+void towire_u32(u8 **pptr UNNEEDED, u32 v UNNEEDED)
+{ fprintf(stderr, "towire_u32 called!\n"); abort(); }
+/* Generated stub for towire_u64 */
+void towire_u64(u8 **pptr UNNEEDED, u64 v UNNEEDED)
+{ fprintf(stderr, "towire_u64 called!\n"); abort(); }
+/* Generated stub for towire_u8 */
+void towire_u8(u8 **pptr UNNEEDED, u8 v UNNEEDED)
+{ fprintf(stderr, "towire_u8 called!\n"); abort(); }
+/* Generated stub for towire_u8_array */
+void towire_u8_array(u8 **pptr UNNEEDED, const u8 *arr UNNEEDED, size_t num UNNEEDED)
+{ fprintf(stderr, "towire_u8_array called!\n"); abort(); }
+/* AUTOGENERATED MOCKS END */
+
+static void test_append_empty(void)
+{
+ int *arr = tal_arr(NULL, int, 0);
+ int one[] = { 1, 2, 3 };
+
+ tal_arr_append(&arr, tal_dup_arr(arr, int, one, ARRAY_SIZE(one), 0));
+
+ assert(tal_count(arr) == 3);
+ assert(arr[0] == 1);
+ assert(arr[1] == 2);
+ assert(arr[2] == 3);
+
+ tal_free(arr);
+}
+
+static void test_append_nonempty(void)
+{
+ int *arr = tal_arr(NULL, int, 2);
+ arr[0] = 10;
+ arr[1] = 20;
+
+ int more[] = { 30, 40 };
+
+ tal_arr_append(&arr, tal_dup_arr(arr, int, more, ARRAY_SIZE(more), 0));
+
+ assert(tal_count(arr) == 4);
+ assert(arr[0] == 10);
+ assert(arr[1] == 20);
+ assert(arr[2] == 30);
+ assert(arr[3] == 40);
+
+ tal_free(arr);
+}
+
+static void test_appendn_partial(void)
+{
+ int *arr = tal_arr(NULL, int, 1);
+ arr[0] = 5;
+
+ int more[] = { 6, 7, 8, 9 };
+
+ tal_arr_appendn(&arr, more, 2);
+
+ assert(tal_count(arr) == 3);
+ assert(arr[0] == 5);
+ assert(arr[1] == 6);
+ assert(arr[2] == 7);
+
+ tal_free(arr);
+}
+
+static void test_appendn_zero(void)
+{
+ int *arr = tal_arr(NULL, int, 3);
+ arr[0] = 1;
+ arr[1] = 2;
+ arr[2] = 3;
+
+ int more[] = { 4, 5 };
+
+ tal_arr_appendn(&arr, more, 0);
+
+ assert(tal_count(arr) == 3);
+ assert(arr[0] == 1);
+ assert(arr[1] == 2);
+ assert(arr[2] == 3);
+
+ tal_free(arr);
+}
+
+static void test_append_structs(void)
+{
+ struct foo {
+ int a;
+ int b;
+ };
+
+ struct foo *arr = tal_arr(NULL, struct foo, 1);
+ arr[0].a = 1;
+ arr[0].b = 2;
+
+ struct foo more[] = {
+ { 3, 4 },
+ { 5, 6 },
+ };
+
+ tal_arr_append(&arr, tal_dup_arr(arr, struct foo, more, ARRAY_SIZE(more), 0));
+
+ assert(tal_count(arr) == 3);
+ assert(arr[0].a == 1 && arr[0].b == 2);
+ assert(arr[1].a == 3 && arr[1].b == 4);
+ assert(arr[2].a == 5 && arr[2].b == 6);
+
+ tal_free(arr);
+}
+
+static void test_append_take(void)
+{
+ int *arr = tal_arr(NULL, int, 0);
+ int more[] = { 4, 5 };
+ const int *moreptr = tal_dup_arr(NULL, int, more, ARRAY_SIZE(more), 0);
+
+ tal_arr_appendn(&arr, take(moreptr), 1);
+ moreptr = tal_dup_arr(NULL, int, more, ARRAY_SIZE(more), 0);
+ tal_arr_append(&arr, take(moreptr));
+
+ assert(tal_count(arr) == 3);
+ assert(arr[0] == 4);
+ assert(arr[1] == 4);
+ assert(arr[2] == 5);
+
+ tal_free(arr);
+}
+
+int main(int argc, char *argv[])
+{
+ common_setup(argv[0]);
+
+ test_append_empty();
+ test_append_nonempty();
+ test_appendn_partial();
+ test_appendn_zero();
+ test_append_structs();
+ test_append_take();
+
+ common_shutdown();
+}
diff --git a/common/utils.c b/common/utils.c
index 102eed7a..860cd446 100644
--- a/common/utils.c
+++ b/common/utils.c
@@ -147,6 +147,28 @@ void tal_arr_remove_(void *p, size_t elemsize, size_t n)
tal_resize((char **)p, len - elemsize);
}
+static void tal_arr_append_bytes(void *p, const void *append, size_t bytes)
+{
+ void **pptr = p;
+ size_t oldsize = tal_bytelen(*pptr);
+ tal_resize(pptr, oldsize + bytes);
+ /* Blah blah blah memcpy NULL blah blah */
+ if (append || bytes)
+ memcpy(*pptr + oldsize, memcheck(append, bytes), bytes);
+ if (taken(append))
+ tal_free(append);
+}
+
+void tal_arr_append_(void *p, const void *append TAKES)
+{
+ tal_arr_append_bytes(p, append, tal_bytelen(append));
+}
+
+void tal_arr_appendn_(void *p, const void *append TAKES, size_t bytes)
+{
+ tal_arr_append_bytes(p, append, bytes);
+}
+
/* Check for valid UTF-8 */
bool utf8_check(const void *vbuf, size_t buflen)
{
diff --git a/common/utils.h b/common/utils.h
index cc378e62..7a5c98aa 100644
--- a/common/utils.h
+++ b/common/utils.h
@@ -104,6 +104,21 @@ static inline void tal_free_if_taken(const tal_t *p)
tal_free(p);
}
+/**
+ * Append one tal array to another (TAKES)
+ */
+#define tal_arr_append(pptr, append) \
+ tal_arr_append_((pptr) + 0*sizeof(*(pptr) == (append)), (append))
+
+/**
+ * Append an array of appendnum elements to a tal arr (TAKES)
+ */
+#define tal_arr_appendn(pptr, append, appendnum) \
+ tal_arr_appendn_((pptr), (append), (appendnum) * sizeof(*append) + 0*sizeof(*(pptr) == (append)))
+
+void tal_arr_append_(void *p, const void *append TAKES);
+void tal_arr_appendn_(void *p, const void *append TAKES, size_t bytes);
+
/* Check for valid UTF-8 */
bool utf8_check(const void *buf, size_t buflen);
diff --git a/connectd/multiplex.c b/connectd/multiplex.c
index 46e3aeb0..59e483d9 100644
--- a/connectd/multiplex.c
+++ b/connectd/multiplex.c
@@ -376,7 +376,6 @@ static bool is_urgent(enum peer_wire type)
static u8 *process_batch_elements(const tal_t *ctx, struct peer *peer, const u8 *msg TAKES)
{
u8 *ret = tal_arr(ctx, u8, 0);
- size_t ret_size = 0;
const u8 *cursor = msg;
size_t plen = tal_count(msg);
@@ -422,9 +421,7 @@ static u8 *process_batch_elements(const tal_t *ctx, struct peer *peer, const u8
enc_msg = cryptomsg_encrypt_msg(tmpctx, &peer->cs,
take(element_bytes));
- tal_resize(&ret, ret_size + tal_bytelen(enc_msg));
- memcpy(&ret[ret_size], enc_msg, tal_bytelen(enc_msg));
- ret_size += tal_bytelen(enc_msg);
+ tal_arr_append(&ret, enc_msg);
} while(plen);
diff --git a/connectd/test/run-websocket.c b/connectd/test/run-websocket.c
index d31e889b..2d133f06 100644
--- a/connectd/test/run-websocket.c
+++ b/connectd/test/run-websocket.c
@@ -2,6 +2,7 @@
#include <assert.h>
#include <common/amount.h>
#include <common/memleak.h>
+#include <common/utils.h>
#include <ccan/io/io.h>
#include <ccan/read_write_all/read_write_all.h>
#include <wire/wire.h>
@@ -31,9 +32,7 @@ static bool my_read_all(int fd, void *buf, size_t count)
static ssize_t my_write(int fd, const void *buf, size_t count)
{
- size_t buflen = tal_bytelen(my_wbuf);
- tal_resize(&my_wbuf, buflen + count);
- memcpy(my_wbuf + buflen, buf, count);
+ tal_arr_appendn(&my_wbuf, buf, count);
return count;
}
static bool my_write_all(int fd, const void *buf, size_t count)
diff --git a/lightningd/plugin.c b/lightningd/plugin.c
index a9098ad0..ca4a2f64 100644
--- a/lightningd/plugin.c
+++ b/lightningd/plugin.c
@@ -190,23 +190,15 @@ struct command_result *plugin_register_all_complete(struct lightningd *ld,
static void tell_connectd_custommsgs(struct plugins *plugins)
{
struct plugin *p;
- size_t n = 0;
- u16 *all_msgs = tal_arr(tmpctx, u16, n);
+ u16 *all_msgs = tal_arr(tmpctx, u16, 0);
/* Not when shutting down */
if (!plugins->ld->connectd)
return;
/* Gather from all plugins. */
- list_for_each(&plugins->plugins, p, list) {
- size_t num = tal_count(p->custom_msgs);
- /* Blah blah blah memcpy NULL blah blah */
- if (num == 0)
- continue;
- tal_resize(&all_msgs, n + num);
- memcpy(all_msgs + n, p->custom_msgs, num * sizeof(*p->custom_msgs));
- n += num;
- }
+ list_for_each(&plugins->plugins, p, list)
+ tal_arr_append(&all_msgs, p->custom_msgs);
/* Don't bother sorting or uniquifying. If plugins are dumb, they deserve it. */
subd_send_msg(plugins->ld->connectd,
diff --git a/plugins/commando.c b/plugins/commando.c
index d0d2e91c..a256f0b0 100644
--- a/plugins/commando.c
+++ b/plugins/commando.c
@@ -102,8 +102,7 @@ static void append_contents(struct commando *commando, const u8 *msg, size_t msg
return;
}
- tal_resize(&commando->contents, len + msglen);
- memcpy(commando->contents + len, msg, msglen);
+ tal_arr_appendn(&commando->contents, msg, msglen);
}
struct reply {
diff --git a/tools/lightning-downgrade.c b/tools/lightning-downgrade.c
index 70a8f220..a1fc9beb 100644
--- a/tools/lightning-downgrade.c
+++ b/tools/lightning-downgrade.c
@@ -41,10 +41,7 @@ struct layer {
static void copy_data(u8 **out, const u8 *in, size_t len)
{
- size_t oldlen = tal_bytelen(*out);
-
- tal_resize(out, oldlen + len);
- memcpy(*out + oldlen, in, len);
+ tal_arr_appendn(out, in, len);
}
/* askrene added DSTORE_CHANNEL_BIAS_V2 (convertable) and
Why this scored 19/100
Community notes
Notes can correct, qualify, or add evidence to the AI analysis. Every note shown here has been validated by a human moderator.
The AI analysis stands alone for now. Submit a note if you can add evidence or important context.