common: reject banned Unicode categories in protocol text fields

channeld/test/Makefile

@@ -29,6 +29,7 @@ channeld/test/run-full_channel:			\
 	common/pseudorand.o			\
 	common/randbytes.o			\
 	common/setup.o				\
+	common/unicode_category.o		\
 	common/utils.o
 
 $(CHANNELD_TEST_OBJS): $(CHANNELD_HEADERS) $(CHANNELD_SRC) channeld/test/Makefile

common/Makefile

@@ -105,6 +105,7 @@ COMMON_SRC_NOGEN :=				\
 	common/timeout.c			\
 	common/trace.c				\
 	common/tx_roles.c			\
+	common/unicode_category.c		\
 	common/utils.c				\
 	common/utxo.c				\
 	common/version.c			\

common/bolt11.c

@@ -212,7 +212,7 @@ static const char *decode_d(struct bolt11 *b11,
 		return err;
 
 	*have_d = true;
-	b11->description = utf8_str(b11, take(desc), tal_bytelen(desc));
+	b11->description = utf8_str_text(b11, take(desc), tal_bytelen(desc));
 	if (b11->description)
 		return NULL;
 

common/bolt12_proof.c

@@ -157,7 +157,7 @@ struct tlv_payer_proof *make_unsigned_proof_(const tal_t *ctx,
 	if (note) {
 		/* Not nul-terminated! */
 		pptlv->proof_note = tal_dup_arr(pptlv, utf8, note, strlen(note), 0);
-		assert(utf8_check(pptlv->proof_note, tal_bytelen(pptlv->proof_note)));
+		assert(utf8_check_text(pptlv->proof_note, tal_bytelen(pptlv->proof_note)));
 	}
 
 	/* Make sure pptlv->fields correctly reflects values */

common/json_param.c

@@ -453,6 +453,22 @@ struct command_result *param_escaped_string(struct command *cmd,
 				     "should be a string (without \\u)");
 }
 
+struct command_result *param_escaped_utf8_string(struct command *cmd,
+						  const char *name,
+						  const char *buffer,
+						  const jsmntok_t *tok,
+						  const char **str)
+{
+	struct command_result *ret = param_escaped_string(cmd, name, buffer, tok, str);
+	if (ret)
+		return ret;
+
+	if (!utf8_check_text(*str, strlen(*str)))
+		return command_fail_badparam(cmd, name, buffer, tok,
+					     "should not contain control, format, private-use or unassigned Unicode characters");
+	return NULL;
+}
+
 struct command_result *param_string(struct command *cmd, const char *name,
 				    const char * buffer, const jsmntok_t *tok,
 				    const char **str)
@@ -462,6 +478,20 @@ struct command_result *param_string(struct command *cmd, const char *name,
 	return NULL;
 }
 
+struct command_result *param_utf8_string(struct command *cmd, const char *name,
+					 const char * buffer, const jsmntok_t *tok,
+					 const char **str)
+{
+	struct command_result *ret = param_string(cmd, name, buffer, tok, str);
+	if (ret)
+		return ret;
+
+	if (!utf8_check_text(*str, strlen(*str)))
+		return command_fail_badparam(cmd, name, buffer, tok,
+					     "should not contain control, format, private-use or unassigned Unicode characters");
+	return NULL;
+}
+
 /* Extract a string or a json array */
 struct command_result *param_string_or_array(struct command *cmd, const char *name,
 					     const char * buffer, const jsmntok_t *tok,

common/json_param.h

@@ -193,11 +193,26 @@ struct command_result *param_escaped_string(struct command *cmd,
 					    const jsmntok_t *tok,
 					    const char **str);
 
+/* Extract an escaped string (and unescape it), and reject it if it
+ * contains characters banned from protocol text fields - see
+ * param_utf8_string() below. */
+struct command_result *param_escaped_utf8_string(struct command *cmd,
+						  const char *name,
+						  const char *buffer,
+						  const jsmntok_t *tok,
+						  const char **str);
+
 /* Extract a string */
 struct command_result *param_string(struct command *cmd, const char *name,
 				    const char * buffer, const jsmntok_t *tok,
 				    const char **str);
 
+/* Extract a string, and reject it if it contains characters banned
+ * from protocol text fields - see param_escaped_utf8_string() above!! */
+struct command_result *param_utf8_string(struct command *cmd, const char *name,
+					 const char * buffer, const jsmntok_t *tok,
+					 const char **str);
+
 struct str_or_arr
 {
 	const char *str;

common/test/Makefile

@@ -8,6 +8,7 @@ COMMON_TEST_COMMON_OBJS :=				\
 	common/randbytes.o				\
 	common/clock_time.o				\
 	common/setup.o					\
+	common/unicode_category.o			\
 	common/utils.o
 
 $(COMMON_TEST_PROGRAMS): $(COMMON_TEST_COMMON_OBJS) $(BITCOIN_OBJS)

common/test/run-param.c

@@ -607,6 +607,34 @@ static void advanced_fail(void)
 	}
 }
 
+static void utf8_param_fail(void)
+{
+	{
+		struct json *j = json_parse(cmd, "[ '\xee\x80\x80' ]");
+		const char *v;
+		assert(!param(cmd, j->buffer, j->toks,
+			      p_req("description", param_escaped_utf8_string, &v),
+			      NULL));
+		assert(check_fail());
+	}
+	{
+		struct json *j = json_parse(cmd, "[ '\xee\x80\x80' ]");
+		const char *v;
+		assert(!param(cmd, j->buffer, j->toks,
+			      p_req("payer_note", param_utf8_string, &v),
+			      NULL));
+		assert(check_fail());
+	}
+	{
+		struct json *j = json_parse(cmd, "[ 'hello world' ]");
+		const char *v;
+		assert(param(cmd, j->buffer, j->toks,
+			     p_req("description", param_escaped_utf8_string, &v),
+			     NULL));
+		assert(streq(v, "hello world"));
+	}
+}
+
 #define test_cb(cb, T, json_, value, pass) \
 { \
 	struct json *j = json_parse(cmd, json_); \
@@ -696,6 +724,7 @@ int main(int argc, char *argv[])
 	sendpay_nulltok();
 	advanced();
 	advanced_fail();
+	utf8_param_fail();
 	param_tests();
 	usage();
 	invalid_bech32m();

common/test/run-utils-utf8_check.c

@@ -0,0 +1,173 @@
+#include "config.h"
+#include <assert.h>
+#include <ccan/array_size/array_size.h>
+#include <common/amount.h>
+#include <common/pseudorand.h>
+#include <common/setup.h>
+#include <common/utils.h>
+#include <stdio.h>
+#include <string.h>
+#include <wire/wire.h>
+
+
+bool amount_asset_is_main(struct amount_asset *asset UNNEEDED)
+{ fprintf(stderr, "amount_asset_is_main called!\n"); abort(); }
+
+struct amount_sat amount_asset_to_sat(struct amount_asset *asset UNNEEDED)
+{ fprintf(stderr, "amount_asset_to_sat called!\n"); abort(); }
+
+ bool amount_feerate(u32 *feerate UNNEEDED, struct amount_sat fee UNNEEDED, size_t weight UNNEEDED)
+{ fprintf(stderr, "amount_feerate called!\n"); abort(); }
+
+struct amount_sat amount_sat(u64 satoshis UNNEEDED)
+{ fprintf(stderr, "amount_sat called!\n"); abort(); }
+
+ bool amount_sat_add(struct amount_sat *val UNNEEDED,
+				       struct amount_sat a UNNEEDED,
+				       struct amount_sat b UNNEEDED)
+{ fprintf(stderr, "amount_sat_add called!\n"); abort(); }
+
+bool amount_sat_eq(struct amount_sat a UNNEEDED, struct amount_sat b UNNEEDED)
+{ fprintf(stderr, "amount_sat_eq called!\n"); abort(); }
+
+bool amount_sat_greater_eq(struct amount_sat a UNNEEDED, struct amount_sat b UNNEEDED)
+{ fprintf(stderr, "amount_sat_greater_eq called!\n"); abort(); }
+
+ bool amount_sat_sub(struct amount_sat *val UNNEEDED,
+				       struct amount_sat a UNNEEDED,
+				       struct amount_sat b UNNEEDED)
+{ fprintf(stderr, "amount_sat_sub called!\n"); abort(); }
+
+struct amount_asset amount_sat_to_asset(struct amount_sat *sat UNNEEDED, const u8 *asset UNNEEDED)
+{ fprintf(stderr, "amount_sat_to_asset called!\n"); abort(); }
+
+struct amount_sat amount_tx_fee(u32 fee_per_kw UNNEEDED, size_t weight UNNEEDED)
+{ fprintf(stderr, "amount_tx_fee called!\n"); abort(); }
+
+const u8 *fromwire(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, void *copy UNNEEDED, size_t n UNNEEDED)
+{ fprintf(stderr, "fromwire called!\n"); abort(); }
+
+bool fromwire_bool(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
+{ fprintf(stderr, "fromwire_bool called!\n"); abort(); }
+
+void *fromwire_fail(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
+{ fprintf(stderr, "fromwire_fail called!\n"); abort(); }
+
+void fromwire_secp256k1_ecdsa_signature(const u8 **cursor UNNEEDED, size_t *max UNNEEDED,
+					secp256k1_ecdsa_signature *signature UNNEEDED)
+{ fprintf(stderr, "fromwire_secp256k1_ecdsa_signature called!\n"); abort(); }
+
+void fromwire_sha256(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct sha256 *sha256 UNNEEDED)
+{ fprintf(stderr, "fromwire_sha256 called!\n"); abort(); }
+
+u8 *fromwire_tal_arrn(const tal_t *ctx UNNEEDED,
+		       const u8 **cursor UNNEEDED, size_t *max UNNEEDED, size_t num UNNEEDED)
+{ fprintf(stderr, "fromwire_tal_arrn called!\n"); abort(); }
+
+u32 fromwire_u32(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
+{ fprintf(stderr, "fromwire_u32 called!\n"); abort(); }
+
+u64 fromwire_u64(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
+{ fprintf(stderr, "fromwire_u64 called!\n"); abort(); }
+
+u8 fromwire_u8(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
+{ fprintf(stderr, "fromwire_u8 called!\n"); abort(); }
+
+void fromwire_u8_array(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, u8 *arr UNNEEDED, size_t num UNNEEDED)
+{ fprintf(stderr, "fromwire_u8_array called!\n"); abort(); }
+
+const struct siphash_seed *siphash_seed(void)
+{ fprintf(stderr, "siphash_seed called!\n"); abort(); }
+
+void towire(u8 **pptr UNNEEDED, const void *data UNNEEDED, size_t len UNNEEDED)
+{ fprintf(stderr, "towire called!\n"); abort(); }
+
+void towire_bool(u8 **pptr UNNEEDED, bool v UNNEEDED)
+{ fprintf(stderr, "towire_bool called!\n"); abort(); }
+
+void towire_secp256k1_ecdsa_signature(u8 **pptr UNNEEDED,
+			      const secp256k1_ecdsa_signature *signature UNNEEDED)
+{ fprintf(stderr, "towire_secp256k1_ecdsa_signature called!\n"); abort(); }
+
+void towire_sha256(u8 **pptr UNNEEDED, const struct sha256 *sha256 UNNEEDED)
+{ fprintf(stderr, "towire_sha256 called!\n"); abort(); }
+
+void towire_u32(u8 **pptr UNNEEDED, u32 v UNNEEDED)
+{ fprintf(stderr, "towire_u32 called!\n"); abort(); }
+
+void towire_u64(u8 **pptr UNNEEDED, u64 v UNNEEDED)
+{ fprintf(stderr, "towire_u64 called!\n"); abort(); }
+
+void towire_u8(u8 **pptr UNNEEDED, u8 v UNNEEDED)
+{ fprintf(stderr, "towire_u8 called!\n"); abort(); }
+
+void towire_u8_array(u8 **pptr UNNEEDED, const u8 *arr UNNEEDED, size_t num UNNEEDED)
+{ fprintf(stderr, "towire_u8_array called!\n"); abort(); }
+
+static void test_valid(void)
+{
+	assert(utf8_check("hello world", strlen("hello world")));
+	assert(utf8_check_text("hello world", strlen("hello world")));
+
+	{
+		static const u8 nansensu[] = {
+			0xe3, 0x83, 0x8a, 0xe3, 0x83, 0xb3, 0xe3, 0x82,
+			0xbb, 0xe3, 0x83, 0xb3, 0xe3, 0x82, 0xb9
+		};
+		assert(utf8_check(nansensu, sizeof(nansensu)));
+		assert(utf8_check_text(nansensu, sizeof(nansensu)));
+	}
+
+	assert(utf8_check("", 0));
+	assert(utf8_check_text("", 0));
+}
+
+/* NUL, surrogates and overlong encodings are rejected by ccan/utf8
+ * itself (utf8_decode()), regardless of category filtering - even
+ * plain utf8_check() rejects those */
+static void test_encoding_banned(void)
+{
+	static const u8 embedded_nul[] = { 'a', 0x00, 'b' };
+	assert(!utf8_check(embedded_nul, sizeof(embedded_nul)));
+	assert(!utf8_check_text(embedded_nul, sizeof(embedded_nul)));
+}
+
+/* Codepoints in Unicode categories Cc/Cf/Co/Cn are valid UTF-8
+ * encoding (plain utf8_check() accepts them - it's used for general
+ * JSON-RPC input, datastore, which shouldn't be restricted this
+ * way), but utf8_check_text() must reject them for protocol text
+ * fields */
+static void test_text_banned(void)
+{
+	static const u8 tab[] = { 'a', '\t', 'b' };
+	static const u8 del[] = { 'a', 0x7f, 'b' };
+	static const u8 c1_control[] = { 'a', 0xc2, 0x85, 'b' };
+	static const u8 rtl_override[] = { 'a', 0xe2, 0x80, 0xae, 'b' };
+	static const u8 private_use[] = { 'a', 0xee, 0x80, 0x80, 'b' };
+	static const u8 unassigned[] = { 'a', 0xcd, 0xb8, 'b' };
+	static const u8 *cases[] = {
+		tab, del, c1_control, rtl_override, private_use, unassigned
+	};
+	static const size_t lens[] = {
+		sizeof(tab), sizeof(del), sizeof(c1_control),
+		sizeof(rtl_override), sizeof(private_use), sizeof(unassigned)
+	};
+
+	for (size_t i = 0; i < ARRAY_SIZE(cases); i++) {
+		/* Plain utf8_check() stays permissive - valid encoding */
+		assert(utf8_check(cases[i], lens[i]));
+		/* utf8_check_text() rejects the banned category */
+		assert(!utf8_check_text(cases[i], lens[i]));
+	}
+}
+
+int main(int argc, char *argv[])
+{
+	common_setup(argv[0]);
+
+	test_valid();
+	test_encoding_banned();
+	test_text_banned();
+
+	common_shutdown();
+}