diff --git a/channeld/test/Makefile b/channeld/test/Makefile index f683ec31b26f..aa9ef66cf941 100644 --- a/channeld/test/Makefile +++ b/channeld/test/Makefile @@ -29,6 +29,7 @@ channeld/test/run-full_channel: \ common/pseudorand.o \ common/randbytes.o \ common/setup.o \ + common/unicode_category.o \ common/utils.o $(CHANNELD_TEST_OBJS): $(CHANNELD_HEADERS) $(CHANNELD_SRC) channeld/test/Makefile diff --git a/common/Makefile b/common/Makefile index d60606420807..d70abd6e50fc 100644 --- a/common/Makefile +++ b/common/Makefile @@ -105,6 +105,7 @@ COMMON_SRC_NOGEN := \ common/timeout.c \ common/trace.c \ common/tx_roles.c \ + common/unicode_category.c \ common/utils.c \ common/utxo.c \ common/version.c \ diff --git a/common/bolt11.c b/common/bolt11.c index bc851ed49114..b59d126271d9 100644 --- a/common/bolt11.c +++ b/common/bolt11.c @@ -212,7 +212,7 @@ static const char *decode_d(struct bolt11 *b11, return err; *have_d = true; - b11->description = utf8_str(b11, take(desc), tal_bytelen(desc)); + b11->description = utf8_str_text(b11, take(desc), tal_bytelen(desc)); if (b11->description) return NULL; diff --git a/common/bolt12_proof.c b/common/bolt12_proof.c index aec5fef8102e..986059ecad1d 100644 --- a/common/bolt12_proof.c +++ b/common/bolt12_proof.c @@ -157,7 +157,7 @@ struct tlv_payer_proof *make_unsigned_proof_(const tal_t *ctx, if (note) { /* Not nul-terminated! */ pptlv->proof_note = tal_dup_arr(pptlv, utf8, note, strlen(note), 0); - assert(utf8_check(pptlv->proof_note, tal_bytelen(pptlv->proof_note))); + assert(utf8_check_text(pptlv->proof_note, tal_bytelen(pptlv->proof_note))); } /* Make sure pptlv->fields correctly reflects values */ diff --git a/common/json_param.c b/common/json_param.c index 18bd284c0f22..96313da3671a 100644 --- a/common/json_param.c +++ b/common/json_param.c @@ -453,6 +453,22 @@ struct command_result *param_escaped_string(struct command *cmd, "should be a string (without \\u)"); } +struct command_result *param_escaped_utf8_string(struct command *cmd, + const char *name, + const char *buffer, + const jsmntok_t *tok, + const char **str) +{ + struct command_result *ret = param_escaped_string(cmd, name, buffer, tok, str); + if (ret) + return ret; + + if (!utf8_check_text(*str, strlen(*str))) + return command_fail_badparam(cmd, name, buffer, tok, + "should not contain control, format, private-use or unassigned Unicode characters"); + return NULL; +} + struct command_result *param_string(struct command *cmd, const char *name, const char * buffer, const jsmntok_t *tok, const char **str) @@ -462,6 +478,20 @@ struct command_result *param_string(struct command *cmd, const char *name, return NULL; } +struct command_result *param_utf8_string(struct command *cmd, const char *name, + const char * buffer, const jsmntok_t *tok, + const char **str) +{ + struct command_result *ret = param_string(cmd, name, buffer, tok, str); + if (ret) + return ret; + + if (!utf8_check_text(*str, strlen(*str))) + return command_fail_badparam(cmd, name, buffer, tok, + "should not contain control, format, private-use or unassigned Unicode characters"); + return NULL; +} + /* Extract a string or a json array */ struct command_result *param_string_or_array(struct command *cmd, const char *name, const char * buffer, const jsmntok_t *tok, diff --git a/common/json_param.h b/common/json_param.h index 3f211f2d03c9..10b594ee0823 100644 --- a/common/json_param.h +++ b/common/json_param.h @@ -193,11 +193,26 @@ struct command_result *param_escaped_string(struct command *cmd, const jsmntok_t *tok, const char **str); +/* Extract an escaped string (and unescape it), and reject it if it + * contains characters banned from protocol text fields - see + * param_utf8_string() below. */ +struct command_result *param_escaped_utf8_string(struct command *cmd, + const char *name, + const char *buffer, + const jsmntok_t *tok, + const char **str); + /* Extract a string */ struct command_result *param_string(struct command *cmd, const char *name, const char * buffer, const jsmntok_t *tok, const char **str); +/* Extract a string, and reject it if it contains characters banned + * from protocol text fields - see param_escaped_utf8_string() above!! */ +struct command_result *param_utf8_string(struct command *cmd, const char *name, + const char * buffer, const jsmntok_t *tok, + const char **str); + struct str_or_arr { const char *str; diff --git a/common/test/Makefile b/common/test/Makefile index 71f9dc17d981..5957c953652d 100644 --- a/common/test/Makefile +++ b/common/test/Makefile @@ -8,6 +8,7 @@ COMMON_TEST_COMMON_OBJS := \ common/randbytes.o \ common/clock_time.o \ common/setup.o \ + common/unicode_category.o \ common/utils.o $(COMMON_TEST_PROGRAMS): $(COMMON_TEST_COMMON_OBJS) $(BITCOIN_OBJS) diff --git a/common/test/run-param.c b/common/test/run-param.c index 5b7d2952b33d..d6a64b30f15a 100644 --- a/common/test/run-param.c +++ b/common/test/run-param.c @@ -607,6 +607,34 @@ static void advanced_fail(void) } } +static void utf8_param_fail(void) +{ + { + struct json *j = json_parse(cmd, "[ '\xee\x80\x80' ]"); + const char *v; + assert(!param(cmd, j->buffer, j->toks, + p_req("description", param_escaped_utf8_string, &v), + NULL)); + assert(check_fail()); + } + { + struct json *j = json_parse(cmd, "[ '\xee\x80\x80' ]"); + const char *v; + assert(!param(cmd, j->buffer, j->toks, + p_req("payer_note", param_utf8_string, &v), + NULL)); + assert(check_fail()); + } + { + struct json *j = json_parse(cmd, "[ 'hello world' ]"); + const char *v; + assert(param(cmd, j->buffer, j->toks, + p_req("description", param_escaped_utf8_string, &v), + NULL)); + assert(streq(v, "hello world")); + } +} + #define test_cb(cb, T, json_, value, pass) \ { \ struct json *j = json_parse(cmd, json_); \ @@ -696,6 +724,7 @@ int main(int argc, char *argv[]) sendpay_nulltok(); advanced(); advanced_fail(); + utf8_param_fail(); param_tests(); usage(); invalid_bech32m(); diff --git a/common/test/run-utils-utf8_check.c b/common/test/run-utils-utf8_check.c new file mode 100644 index 000000000000..0f5be0d2078e --- /dev/null +++ b/common/test/run-utils-utf8_check.c @@ -0,0 +1,173 @@ +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +bool amount_asset_is_main(struct amount_asset *asset UNNEEDED) +{ fprintf(stderr, "amount_asset_is_main called!\n"); abort(); } + +struct amount_sat amount_asset_to_sat(struct amount_asset *asset UNNEEDED) +{ fprintf(stderr, "amount_asset_to_sat called!\n"); abort(); } + + bool amount_feerate(u32 *feerate UNNEEDED, struct amount_sat fee UNNEEDED, size_t weight UNNEEDED) +{ fprintf(stderr, "amount_feerate called!\n"); abort(); } + +struct amount_sat amount_sat(u64 satoshis UNNEEDED) +{ fprintf(stderr, "amount_sat called!\n"); abort(); } + + bool amount_sat_add(struct amount_sat *val UNNEEDED, + struct amount_sat a UNNEEDED, + struct amount_sat b UNNEEDED) +{ fprintf(stderr, "amount_sat_add called!\n"); abort(); } + +bool amount_sat_eq(struct amount_sat a UNNEEDED, struct amount_sat b UNNEEDED) +{ fprintf(stderr, "amount_sat_eq called!\n"); abort(); } + +bool amount_sat_greater_eq(struct amount_sat a UNNEEDED, struct amount_sat b UNNEEDED) +{ fprintf(stderr, "amount_sat_greater_eq called!\n"); abort(); } + + bool amount_sat_sub(struct amount_sat *val UNNEEDED, + struct amount_sat a UNNEEDED, + struct amount_sat b UNNEEDED) +{ fprintf(stderr, "amount_sat_sub called!\n"); abort(); } + +struct amount_asset amount_sat_to_asset(struct amount_sat *sat UNNEEDED, const u8 *asset UNNEEDED) +{ fprintf(stderr, "amount_sat_to_asset called!\n"); abort(); } + +struct amount_sat amount_tx_fee(u32 fee_per_kw UNNEEDED, size_t weight UNNEEDED) +{ fprintf(stderr, "amount_tx_fee called!\n"); abort(); } + +const u8 *fromwire(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, void *copy UNNEEDED, size_t n UNNEEDED) +{ fprintf(stderr, "fromwire called!\n"); abort(); } + +bool fromwire_bool(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) +{ fprintf(stderr, "fromwire_bool called!\n"); abort(); } + +void *fromwire_fail(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) +{ fprintf(stderr, "fromwire_fail called!\n"); abort(); } + +void fromwire_secp256k1_ecdsa_signature(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, + secp256k1_ecdsa_signature *signature UNNEEDED) +{ fprintf(stderr, "fromwire_secp256k1_ecdsa_signature called!\n"); abort(); } + +void fromwire_sha256(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct sha256 *sha256 UNNEEDED) +{ fprintf(stderr, "fromwire_sha256 called!\n"); abort(); } + +u8 *fromwire_tal_arrn(const tal_t *ctx UNNEEDED, + const u8 **cursor UNNEEDED, size_t *max UNNEEDED, size_t num UNNEEDED) +{ fprintf(stderr, "fromwire_tal_arrn called!\n"); abort(); } + +u32 fromwire_u32(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) +{ fprintf(stderr, "fromwire_u32 called!\n"); abort(); } + +u64 fromwire_u64(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) +{ fprintf(stderr, "fromwire_u64 called!\n"); abort(); } + +u8 fromwire_u8(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) +{ fprintf(stderr, "fromwire_u8 called!\n"); abort(); } + +void fromwire_u8_array(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, u8 *arr UNNEEDED, size_t num UNNEEDED) +{ fprintf(stderr, "fromwire_u8_array called!\n"); abort(); } + +const struct siphash_seed *siphash_seed(void) +{ fprintf(stderr, "siphash_seed called!\n"); abort(); } + +void towire(u8 **pptr UNNEEDED, const void *data UNNEEDED, size_t len UNNEEDED) +{ fprintf(stderr, "towire called!\n"); abort(); } + +void towire_bool(u8 **pptr UNNEEDED, bool v UNNEEDED) +{ fprintf(stderr, "towire_bool called!\n"); abort(); } + +void towire_secp256k1_ecdsa_signature(u8 **pptr UNNEEDED, + const secp256k1_ecdsa_signature *signature UNNEEDED) +{ fprintf(stderr, "towire_secp256k1_ecdsa_signature called!\n"); abort(); } + +void towire_sha256(u8 **pptr UNNEEDED, const struct sha256 *sha256 UNNEEDED) +{ fprintf(stderr, "towire_sha256 called!\n"); abort(); } + +void towire_u32(u8 **pptr UNNEEDED, u32 v UNNEEDED) +{ fprintf(stderr, "towire_u32 called!\n"); abort(); } + +void towire_u64(u8 **pptr UNNEEDED, u64 v UNNEEDED) +{ fprintf(stderr, "towire_u64 called!\n"); abort(); } + +void towire_u8(u8 **pptr UNNEEDED, u8 v UNNEEDED) +{ fprintf(stderr, "towire_u8 called!\n"); abort(); } + +void towire_u8_array(u8 **pptr UNNEEDED, const u8 *arr UNNEEDED, size_t num UNNEEDED) +{ fprintf(stderr, "towire_u8_array called!\n"); abort(); } + +static void test_valid(void) +{ + assert(utf8_check("hello world", strlen("hello world"))); + assert(utf8_check_text("hello world", strlen("hello world"))); + + { + static const u8 nansensu[] = { + 0xe3, 0x83, 0x8a, 0xe3, 0x83, 0xb3, 0xe3, 0x82, + 0xbb, 0xe3, 0x83, 0xb3, 0xe3, 0x82, 0xb9 + }; + assert(utf8_check(nansensu, sizeof(nansensu))); + assert(utf8_check_text(nansensu, sizeof(nansensu))); + } + + assert(utf8_check("", 0)); + assert(utf8_check_text("", 0)); +} + +/* NUL, surrogates and overlong encodings are rejected by ccan/utf8 + * itself (utf8_decode()), regardless of category filtering - even + * plain utf8_check() rejects those */ +static void test_encoding_banned(void) +{ + static const u8 embedded_nul[] = { 'a', 0x00, 'b' }; + assert(!utf8_check(embedded_nul, sizeof(embedded_nul))); + assert(!utf8_check_text(embedded_nul, sizeof(embedded_nul))); +} + +/* Codepoints in Unicode categories Cc/Cf/Co/Cn are valid UTF-8 + * encoding (plain utf8_check() accepts them - it's used for general + * JSON-RPC input, datastore, which shouldn't be restricted this + * way), but utf8_check_text() must reject them for protocol text + * fields */ +static void test_text_banned(void) +{ + static const u8 tab[] = { 'a', '\t', 'b' }; + static const u8 del[] = { 'a', 0x7f, 'b' }; + static const u8 c1_control[] = { 'a', 0xc2, 0x85, 'b' }; + static const u8 rtl_override[] = { 'a', 0xe2, 0x80, 0xae, 'b' }; + static const u8 private_use[] = { 'a', 0xee, 0x80, 0x80, 'b' }; + static const u8 unassigned[] = { 'a', 0xcd, 0xb8, 'b' }; + static const u8 *cases[] = { + tab, del, c1_control, rtl_override, private_use, unassigned + }; + static const size_t lens[] = { + sizeof(tab), sizeof(del), sizeof(c1_control), + sizeof(rtl_override), sizeof(private_use), sizeof(unassigned) + }; + + for (size_t i = 0; i < ARRAY_SIZE(cases); i++) { + /* Plain utf8_check() stays permissive - valid encoding */ + assert(utf8_check(cases[i], lens[i])); + /* utf8_check_text() rejects the banned category */ + assert(!utf8_check_text(cases[i], lens[i])); + } +} + +int main(int argc, char *argv[]) +{ + common_setup(argv[0]); + + test_valid(); + test_encoding_banned(); + test_text_banned(); + + common_shutdown(); +} diff --git a/common/unicode_category.c b/common/unicode_category.c new file mode 100644 index 000000000000..0bd7a64e9870 --- /dev/null +++ b/common/unicode_category.c @@ -0,0 +1,778 @@ + +/* This file generated by devtools/gen-unicode-category.py */ +/* Do not modify this file!!! Modify the script, or regenerate from a new + * UnicodeData.txt, instead */ +/* Generated from Unicode 17.0.0 UnicodeData.txt */ + +#include "config.h" +#include +#include "common/unicode_category.h" +#include + +struct unicode_range { + u32 start, end; +}; + +/* Sorted, non-overlapping, coalesced ranges banned as Cc/Cf/Co/Cn. */ +static const struct unicode_range banned_ranges[] = { + { 0x000000, 0x00001F }, + { 0x00007F, 0x00009F }, + { 0x0000AD, 0x0000AD }, + { 0x000378, 0x000379 }, + { 0x000380, 0x000383 }, + { 0x00038B, 0x00038B }, + { 0x00038D, 0x00038D }, + { 0x0003A2, 0x0003A2 }, + { 0x000530, 0x000530 }, + { 0x000557, 0x000558 }, + { 0x00058B, 0x00058C }, + { 0x000590, 0x000590 }, + { 0x0005C8, 0x0005CF }, + { 0x0005EB, 0x0005EE }, + { 0x0005F5, 0x000605 }, + { 0x00061C, 0x00061C }, + { 0x0006DD, 0x0006DD }, + { 0x00070E, 0x00070F }, + { 0x00074B, 0x00074C }, + { 0x0007B2, 0x0007BF }, + { 0x0007FB, 0x0007FC }, + { 0x00082E, 0x00082F }, + { 0x00083F, 0x00083F }, + { 0x00085C, 0x00085D }, + { 0x00085F, 0x00085F }, + { 0x00086B, 0x00086F }, + { 0x000890, 0x000896 }, + { 0x0008E2, 0x0008E2 }, + { 0x000984, 0x000984 }, + { 0x00098D, 0x00098E }, + { 0x000991, 0x000992 }, + { 0x0009A9, 0x0009A9 }, + { 0x0009B1, 0x0009B1 }, + { 0x0009B3, 0x0009B5 }, + { 0x0009BA, 0x0009BB }, + { 0x0009C5, 0x0009C6 }, + { 0x0009C9, 0x0009CA }, + { 0x0009CF, 0x0009D6 }, + { 0x0009D8, 0x0009DB }, + { 0x0009DE, 0x0009DE }, + { 0x0009E4, 0x0009E5 }, + { 0x0009FF, 0x000A00 }, + { 0x000A04, 0x000A04 }, + { 0x000A0B, 0x000A0E }, + { 0x000A11, 0x000A12 }, + { 0x000A29, 0x000A29 }, + { 0x000A31, 0x000A31 }, + { 0x000A34, 0x000A34 }, + { 0x000A37, 0x000A37 }, + { 0x000A3A, 0x000A3B }, + { 0x000A3D, 0x000A3D }, + { 0x000A43, 0x000A46 }, + { 0x000A49, 0x000A4A }, + { 0x000A4E, 0x000A50 }, + { 0x000A52, 0x000A58 }, + { 0x000A5D, 0x000A5D }, + { 0x000A5F, 0x000A65 }, + { 0x000A77, 0x000A80 }, + { 0x000A84, 0x000A84 }, + { 0x000A8E, 0x000A8E }, + { 0x000A92, 0x000A92 }, + { 0x000AA9, 0x000AA9 }, + { 0x000AB1, 0x000AB1 }, + { 0x000AB4, 0x000AB4 }, + { 0x000ABA, 0x000ABB }, + { 0x000AC6, 0x000AC6 }, + { 0x000ACA, 0x000ACA }, + { 0x000ACE, 0x000ACF }, + { 0x000AD1, 0x000ADF }, + { 0x000AE4, 0x000AE5 }, + { 0x000AF2, 0x000AF8 }, + { 0x000B00, 0x000B00 }, + { 0x000B04, 0x000B04 }, + { 0x000B0D, 0x000B0E }, + { 0x000B11, 0x000B12 }, + { 0x000B29, 0x000B29 }, + { 0x000B31, 0x000B31 }, + { 0x000B34, 0x000B34 }, + { 0x000B3A, 0x000B3B }, + { 0x000B45, 0x000B46 }, + { 0x000B49, 0x000B4A }, + { 0x000B4E, 0x000B54 }, + { 0x000B58, 0x000B5B }, + { 0x000B5E, 0x000B5E }, + { 0x000B64, 0x000B65 }, + { 0x000B78, 0x000B81 }, + { 0x000B84, 0x000B84 }, + { 0x000B8B, 0x000B8D }, + { 0x000B91, 0x000B91 }, + { 0x000B96, 0x000B98 }, + { 0x000B9B, 0x000B9B }, + { 0x000B9D, 0x000B9D }, + { 0x000BA0, 0x000BA2 }, + { 0x000BA5, 0x000BA7 }, + { 0x000BAB, 0x000BAD }, + { 0x000BBA, 0x000BBD }, + { 0x000BC3, 0x000BC5 }, + { 0x000BC9, 0x000BC9 }, + { 0x000BCE, 0x000BCF }, + { 0x000BD1, 0x000BD6 }, + { 0x000BD8, 0x000BE5 }, + { 0x000BFB, 0x000BFF }, + { 0x000C0D, 0x000C0D }, + { 0x000C11, 0x000C11 }, + { 0x000C29, 0x000C29 }, + { 0x000C3A, 0x000C3B }, + { 0x000C45, 0x000C45 }, + { 0x000C49, 0x000C49 }, + { 0x000C4E, 0x000C54 }, + { 0x000C57, 0x000C57 }, + { 0x000C5B, 0x000C5B }, + { 0x000C5E, 0x000C5F }, + { 0x000C64, 0x000C65 }, + { 0x000C70, 0x000C76 }, + { 0x000C8D, 0x000C8D }, + { 0x000C91, 0x000C91 }, + { 0x000CA9, 0x000CA9 }, + { 0x000CB4, 0x000CB4 }, + { 0x000CBA, 0x000CBB }, + { 0x000CC5, 0x000CC5 }, + { 0x000CC9, 0x000CC9 }, + { 0x000CCE, 0x000CD4 }, + { 0x000CD7, 0x000CDB }, + { 0x000CDF, 0x000CDF }, + { 0x000CE4, 0x000CE5 }, + { 0x000CF0, 0x000CF0 }, + { 0x000CF4, 0x000CFF }, + { 0x000D0D, 0x000D0D }, + { 0x000D11, 0x000D11 }, + { 0x000D45, 0x000D45 }, + { 0x000D49, 0x000D49 }, + { 0x000D50, 0x000D53 }, + { 0x000D64, 0x000D65 }, + { 0x000D80, 0x000D80 }, + { 0x000D84, 0x000D84 }, + { 0x000D97, 0x000D99 }, + { 0x000DB2, 0x000DB2 }, + { 0x000DBC, 0x000DBC }, + { 0x000DBE, 0x000DBF }, + { 0x000DC7, 0x000DC9 }, + { 0x000DCB, 0x000DCE }, + { 0x000DD5, 0x000DD5 }, + { 0x000DD7, 0x000DD7 }, + { 0x000DE0, 0x000DE5 }, + { 0x000DF0, 0x000DF1 }, + { 0x000DF5, 0x000E00 }, + { 0x000E3B, 0x000E3E }, + { 0x000E5C, 0x000E80 }, + { 0x000E83, 0x000E83 }, + { 0x000E85, 0x000E85 }, + { 0x000E8B, 0x000E8B }, + { 0x000EA4, 0x000EA4 }, + { 0x000EA6, 0x000EA6 }, + { 0x000EBE, 0x000EBF }, + { 0x000EC5, 0x000EC5 }, + { 0x000EC7, 0x000EC7 }, + { 0x000ECF, 0x000ECF }, + { 0x000EDA, 0x000EDB }, + { 0x000EE0, 0x000EFF }, + { 0x000F48, 0x000F48 }, + { 0x000F6D, 0x000F70 }, + { 0x000F98, 0x000F98 }, + { 0x000FBD, 0x000FBD }, + { 0x000FCD, 0x000FCD }, + { 0x000FDB, 0x000FFF }, + { 0x0010C6, 0x0010C6 }, + { 0x0010C8, 0x0010CC }, + { 0x0010CE, 0x0010CF }, + { 0x001249, 0x001249 }, + { 0x00124E, 0x00124F }, + { 0x001257, 0x001257 }, + { 0x001259, 0x001259 }, + { 0x00125E, 0x00125F }, + { 0x001289, 0x001289 }, + { 0x00128E, 0x00128F }, + { 0x0012B1, 0x0012B1 }, + { 0x0012B6, 0x0012B7 }, + { 0x0012BF, 0x0012BF }, + { 0x0012C1, 0x0012C1 }, + { 0x0012C6, 0x0012C7 }, + { 0x0012D7, 0x0012D7 }, + { 0x001311, 0x001311 }, + { 0x001316, 0x001317 }, + { 0x00135B, 0x00135C }, + { 0x00137D, 0x00137F }, + { 0x00139A, 0x00139F }, + { 0x0013F6, 0x0013F7 }, + { 0x0013FE, 0x0013FF }, + { 0x00169D, 0x00169F }, + { 0x0016F9, 0x0016FF }, + { 0x001716, 0x00171E }, + { 0x001737, 0x00173F }, + { 0x001754, 0x00175F }, + { 0x00176D, 0x00176D }, + { 0x001771, 0x001771 }, + { 0x001774, 0x00177F }, + { 0x0017DE, 0x0017DF }, + { 0x0017EA, 0x0017EF }, + { 0x0017FA, 0x0017FF }, + { 0x00180E, 0x00180E }, + { 0x00181A, 0x00181F }, + { 0x001879, 0x00187F }, + { 0x0018AB, 0x0018AF }, + { 0x0018F6, 0x0018FF }, + { 0x00191F, 0x00191F }, + { 0x00192C, 0x00192F }, + { 0x00193C, 0x00193F }, + { 0x001941, 0x001943 }, + { 0x00196E, 0x00196F }, + { 0x001975, 0x00197F }, + { 0x0019AC, 0x0019AF }, + { 0x0019CA, 0x0019CF }, + { 0x0019DB, 0x0019DD }, + { 0x001A1C, 0x001A1D }, + { 0x001A5F, 0x001A5F }, + { 0x001A7D, 0x001A7E }, + { 0x001A8A, 0x001A8F }, + { 0x001A9A, 0x001A9F }, + { 0x001AAE, 0x001AAF }, + { 0x001ADE, 0x001ADF }, + { 0x001AEC, 0x001AFF }, + { 0x001B4D, 0x001B4D }, + { 0x001BF4, 0x001BFB }, + { 0x001C38, 0x001C3A }, + { 0x001C4A, 0x001C4C }, + { 0x001C8B, 0x001C8F }, + { 0x001CBB, 0x001CBC }, + { 0x001CC8, 0x001CCF }, + { 0x001CFB, 0x001CFF }, + { 0x001F16, 0x001F17 }, + { 0x001F1E, 0x001F1F }, + { 0x001F46, 0x001F47 }, + { 0x001F4E, 0x001F4F }, + { 0x001F58, 0x001F58 }, + { 0x001F5A, 0x001F5A }, + { 0x001F5C, 0x001F5C }, + { 0x001F5E, 0x001F5E }, + { 0x001F7E, 0x001F7F }, + { 0x001FB5, 0x001FB5 }, + { 0x001FC5, 0x001FC5 }, + { 0x001FD4, 0x001FD5 }, + { 0x001FDC, 0x001FDC }, + { 0x001FF0, 0x001FF1 }, + { 0x001FF5, 0x001FF5 }, + { 0x001FFF, 0x001FFF }, + { 0x00200B, 0x00200F }, + { 0x00202A, 0x00202E }, + { 0x002060, 0x00206F }, + { 0x002072, 0x002073 }, + { 0x00208F, 0x00208F }, + { 0x00209D, 0x00209F }, + { 0x0020C2, 0x0020CF }, + { 0x0020F1, 0x0020FF }, + { 0x00218C, 0x00218F }, + { 0x00242A, 0x00243F }, + { 0x00244B, 0x00245F }, + { 0x002B74, 0x002B75 }, + { 0x002CF4, 0x002CF8 }, + { 0x002D26, 0x002D26 }, + { 0x002D28, 0x002D2C }, + { 0x002D2E, 0x002D2F }, + { 0x002D68, 0x002D6E }, + { 0x002D71, 0x002D7E }, + { 0x002D97, 0x002D9F }, + { 0x002DA7, 0x002DA7 }, + { 0x002DAF, 0x002DAF }, + { 0x002DB7, 0x002DB7 }, + { 0x002DBF, 0x002DBF }, + { 0x002DC7, 0x002DC7 }, + { 0x002DCF, 0x002DCF }, + { 0x002DD7, 0x002DD7 }, + { 0x002DDF, 0x002DDF }, + { 0x002E5E, 0x002E7F }, + { 0x002E9A, 0x002E9A }, + { 0x002EF4, 0x002EFF }, + { 0x002FD6, 0x002FEF }, + { 0x003040, 0x003040 }, + { 0x003097, 0x003098 }, + { 0x003100, 0x003104 }, + { 0x003130, 0x003130 }, + { 0x00318F, 0x00318F }, + { 0x0031E6, 0x0031EE }, + { 0x00321F, 0x00321F }, + { 0x00A48D, 0x00A48F }, + { 0x00A4C7, 0x00A4CF }, + { 0x00A62C, 0x00A63F }, + { 0x00A6F8, 0x00A6FF }, + { 0x00A7DD, 0x00A7F0 }, + { 0x00A82D, 0x00A82F }, + { 0x00A83A, 0x00A83F }, + { 0x00A878, 0x00A87F }, + { 0x00A8C6, 0x00A8CD }, + { 0x00A8DA, 0x00A8DF }, + { 0x00A954, 0x00A95E }, + { 0x00A97D, 0x00A97F }, + { 0x00A9CE, 0x00A9CE }, + { 0x00A9DA, 0x00A9DD }, + { 0x00A9FF, 0x00A9FF }, + { 0x00AA37, 0x00AA3F }, + { 0x00AA4E, 0x00AA4F }, + { 0x00AA5A, 0x00AA5B }, + { 0x00AAC3, 0x00AADA }, + { 0x00AAF7, 0x00AB00 }, + { 0x00AB07, 0x00AB08 }, + { 0x00AB0F, 0x00AB10 }, + { 0x00AB17, 0x00AB1F }, + { 0x00AB27, 0x00AB27 }, + { 0x00AB2F, 0x00AB2F }, + { 0x00AB6C, 0x00AB6F }, + { 0x00ABEE, 0x00ABEF }, + { 0x00ABFA, 0x00ABFF }, + { 0x00D7A4, 0x00D7AF }, + { 0x00D7C7, 0x00D7CA }, + { 0x00D7FC, 0x00D7FF }, + { 0x00E000, 0x00F8FF }, + { 0x00FA6E, 0x00FA6F }, + { 0x00FADA, 0x00FAFF }, + { 0x00FB07, 0x00FB12 }, + { 0x00FB18, 0x00FB1C }, + { 0x00FB37, 0x00FB37 }, + { 0x00FB3D, 0x00FB3D }, + { 0x00FB3F, 0x00FB3F }, + { 0x00FB42, 0x00FB42 }, + { 0x00FB45, 0x00FB45 }, + { 0x00FDD0, 0x00FDEF }, + { 0x00FE1A, 0x00FE1F }, + { 0x00FE53, 0x00FE53 }, + { 0x00FE67, 0x00FE67 }, + { 0x00FE6C, 0x00FE6F }, + { 0x00FE75, 0x00FE75 }, + { 0x00FEFD, 0x00FF00 }, + { 0x00FFBF, 0x00FFC1 }, + { 0x00FFC8, 0x00FFC9 }, + { 0x00FFD0, 0x00FFD1 }, + { 0x00FFD8, 0x00FFD9 }, + { 0x00FFDD, 0x00FFDF }, + { 0x00FFE7, 0x00FFE7 }, + { 0x00FFEF, 0x00FFFB }, + { 0x00FFFE, 0x00FFFF }, + { 0x01000C, 0x01000C }, + { 0x010027, 0x010027 }, + { 0x01003B, 0x01003B }, + { 0x01003E, 0x01003E }, + { 0x01004E, 0x01004F }, + { 0x01005E, 0x01007F }, + { 0x0100FB, 0x0100FF }, + { 0x010103, 0x010106 }, + { 0x010134, 0x010136 }, + { 0x01018F, 0x01018F }, + { 0x01019D, 0x01019F }, + { 0x0101A1, 0x0101CF }, + { 0x0101FE, 0x01027F }, + { 0x01029D, 0x01029F }, + { 0x0102D1, 0x0102DF }, + { 0x0102FC, 0x0102FF }, + { 0x010324, 0x01032C }, + { 0x01034B, 0x01034F }, + { 0x01037B, 0x01037F }, + { 0x01039E, 0x01039E }, + { 0x0103C4, 0x0103C7 }, + { 0x0103D6, 0x0103FF }, + { 0x01049E, 0x01049F }, + { 0x0104AA, 0x0104AF }, + { 0x0104D4, 0x0104D7 }, + { 0x0104FC, 0x0104FF }, + { 0x010528, 0x01052F }, + { 0x010564, 0x01056E }, + { 0x01057B, 0x01057B }, + { 0x01058B, 0x01058B }, + { 0x010593, 0x010593 }, + { 0x010596, 0x010596 }, + { 0x0105A2, 0x0105A2 }, + { 0x0105B2, 0x0105B2 }, + { 0x0105BA, 0x0105BA }, + { 0x0105BD, 0x0105BF }, + { 0x0105F4, 0x0105FF }, + { 0x010737, 0x01073F }, + { 0x010756, 0x01075F }, + { 0x010768, 0x01077F }, + { 0x010786, 0x010786 }, + { 0x0107B1, 0x0107B1 }, + { 0x0107BB, 0x0107FF }, + { 0x010806, 0x010807 }, + { 0x010809, 0x010809 }, + { 0x010836, 0x010836 }, + { 0x010839, 0x01083B }, + { 0x01083D, 0x01083E }, + { 0x010856, 0x010856 }, + { 0x01089F, 0x0108A6 }, + { 0x0108B0, 0x0108DF }, + { 0x0108F3, 0x0108F3 }, + { 0x0108F6, 0x0108FA }, + { 0x01091C, 0x01091E }, + { 0x01093A, 0x01093E }, + { 0x01095A, 0x01097F }, + { 0x0109B8, 0x0109BB }, + { 0x0109D0, 0x0109D1 }, + { 0x010A04, 0x010A04 }, + { 0x010A07, 0x010A0B }, + { 0x010A14, 0x010A14 }, + { 0x010A18, 0x010A18 }, + { 0x010A36, 0x010A37 }, + { 0x010A3B, 0x010A3E }, + { 0x010A49, 0x010A4F }, + { 0x010A59, 0x010A5F }, + { 0x010AA0, 0x010ABF }, + { 0x010AE7, 0x010AEA }, + { 0x010AF7, 0x010AFF }, + { 0x010B36, 0x010B38 }, + { 0x010B56, 0x010B57 }, + { 0x010B73, 0x010B77 }, + { 0x010B92, 0x010B98 }, + { 0x010B9D, 0x010BA8 }, + { 0x010BB0, 0x010BFF }, + { 0x010C49, 0x010C7F }, + { 0x010CB3, 0x010CBF }, + { 0x010CF3, 0x010CF9 }, + { 0x010D28, 0x010D2F }, + { 0x010D3A, 0x010D3F }, + { 0x010D66, 0x010D68 }, + { 0x010D86, 0x010D8D }, + { 0x010D90, 0x010E5F }, + { 0x010E7F, 0x010E7F }, + { 0x010EAA, 0x010EAA }, + { 0x010EAE, 0x010EAF }, + { 0x010EB2, 0x010EC1 }, + { 0x010EC8, 0x010ECF }, + { 0x010ED9, 0x010EF9 }, + { 0x010F28, 0x010F2F }, + { 0x010F5A, 0x010F6F }, + { 0x010F8A, 0x010FAF }, + { 0x010FCC, 0x010FDF }, + { 0x010FF7, 0x010FFF }, + { 0x01104E, 0x011051 }, + { 0x011076, 0x01107E }, + { 0x0110BD, 0x0110BD }, + { 0x0110C3, 0x0110CF }, + { 0x0110E9, 0x0110EF }, + { 0x0110FA, 0x0110FF }, + { 0x011135, 0x011135 }, + { 0x011148, 0x01114F }, + { 0x011177, 0x01117F }, + { 0x0111E0, 0x0111E0 }, + { 0x0111F5, 0x0111FF }, + { 0x011212, 0x011212 }, + { 0x011242, 0x01127F }, + { 0x011287, 0x011287 }, + { 0x011289, 0x011289 }, + { 0x01128E, 0x01128E }, + { 0x01129E, 0x01129E }, + { 0x0112AA, 0x0112AF }, + { 0x0112EB, 0x0112EF }, + { 0x0112FA, 0x0112FF }, + { 0x011304, 0x011304 }, + { 0x01130D, 0x01130E }, + { 0x011311, 0x011312 }, + { 0x011329, 0x011329 }, + { 0x011331, 0x011331 }, + { 0x011334, 0x011334 }, + { 0x01133A, 0x01133A }, + { 0x011345, 0x011346 }, + { 0x011349, 0x01134A }, + { 0x01134E, 0x01134F }, + { 0x011351, 0x011356 }, + { 0x011358, 0x01135C }, + { 0x011364, 0x011365 }, + { 0x01136D, 0x01136F }, + { 0x011375, 0x01137F }, + { 0x01138A, 0x01138A }, + { 0x01138C, 0x01138D }, + { 0x01138F, 0x01138F }, + { 0x0113B6, 0x0113B6 }, + { 0x0113C1, 0x0113C1 }, + { 0x0113C3, 0x0113C4 }, + { 0x0113C6, 0x0113C6 }, + { 0x0113CB, 0x0113CB }, + { 0x0113D6, 0x0113D6 }, + { 0x0113D9, 0x0113E0 }, + { 0x0113E3, 0x0113FF }, + { 0x01145C, 0x01145C }, + { 0x011462, 0x01147F }, + { 0x0114C8, 0x0114CF }, + { 0x0114DA, 0x01157F }, + { 0x0115B6, 0x0115B7 }, + { 0x0115DE, 0x0115FF }, + { 0x011645, 0x01164F }, + { 0x01165A, 0x01165F }, + { 0x01166D, 0x01167F }, + { 0x0116BA, 0x0116BF }, + { 0x0116CA, 0x0116CF }, + { 0x0116E4, 0x0116FF }, + { 0x01171B, 0x01171C }, + { 0x01172C, 0x01172F }, + { 0x011747, 0x0117FF }, + { 0x01183C, 0x01189F }, + { 0x0118F3, 0x0118FE }, + { 0x011907, 0x011908 }, + { 0x01190A, 0x01190B }, + { 0x011914, 0x011914 }, + { 0x011917, 0x011917 }, + { 0x011936, 0x011936 }, + { 0x011939, 0x01193A }, + { 0x011947, 0x01194F }, + { 0x01195A, 0x01199F }, + { 0x0119A8, 0x0119A9 }, + { 0x0119D8, 0x0119D9 }, + { 0x0119E5, 0x0119FF }, + { 0x011A48, 0x011A4F }, + { 0x011AA3, 0x011AAF }, + { 0x011AF9, 0x011AFF }, + { 0x011B0A, 0x011B5F }, + { 0x011B68, 0x011BBF }, + { 0x011BE2, 0x011BEF }, + { 0x011BFA, 0x011BFF }, + { 0x011C09, 0x011C09 }, + { 0x011C37, 0x011C37 }, + { 0x011C46, 0x011C4F }, + { 0x011C6D, 0x011C6F }, + { 0x011C90, 0x011C91 }, + { 0x011CA8, 0x011CA8 }, + { 0x011CB7, 0x011CFF }, + { 0x011D07, 0x011D07 }, + { 0x011D0A, 0x011D0A }, + { 0x011D37, 0x011D39 }, + { 0x011D3B, 0x011D3B }, + { 0x011D3E, 0x011D3E }, + { 0x011D48, 0x011D4F }, + { 0x011D5A, 0x011D5F }, + { 0x011D66, 0x011D66 }, + { 0x011D69, 0x011D69 }, + { 0x011D8F, 0x011D8F }, + { 0x011D92, 0x011D92 }, + { 0x011D99, 0x011D9F }, + { 0x011DAA, 0x011DAF }, + { 0x011DDC, 0x011DDF }, + { 0x011DEA, 0x011EDF }, + { 0x011EF9, 0x011EFF }, + { 0x011F11, 0x011F11 }, + { 0x011F3B, 0x011F3D }, + { 0x011F5B, 0x011FAF }, + { 0x011FB1, 0x011FBF }, + { 0x011FF2, 0x011FFE }, + { 0x01239A, 0x0123FF }, + { 0x01246F, 0x01246F }, + { 0x012475, 0x01247F }, + { 0x012544, 0x012F8F }, + { 0x012FF3, 0x012FFF }, + { 0x013430, 0x01343F }, + { 0x013456, 0x01345F }, + { 0x0143FB, 0x0143FF }, + { 0x014647, 0x0160FF }, + { 0x01613A, 0x0167FF }, + { 0x016A39, 0x016A3F }, + { 0x016A5F, 0x016A5F }, + { 0x016A6A, 0x016A6D }, + { 0x016ABF, 0x016ABF }, + { 0x016ACA, 0x016ACF }, + { 0x016AEE, 0x016AEF }, + { 0x016AF6, 0x016AFF }, + { 0x016B46, 0x016B4F }, + { 0x016B5A, 0x016B5A }, + { 0x016B62, 0x016B62 }, + { 0x016B78, 0x016B7C }, + { 0x016B90, 0x016D3F }, + { 0x016D7A, 0x016E3F }, + { 0x016E9B, 0x016E9F }, + { 0x016EB9, 0x016EBA }, + { 0x016ED4, 0x016EFF }, + { 0x016F4B, 0x016F4E }, + { 0x016F88, 0x016F8E }, + { 0x016FA0, 0x016FDF }, + { 0x016FE5, 0x016FEF }, + { 0x016FF7, 0x016FFF }, + { 0x018CD6, 0x018CFE }, + { 0x018D1F, 0x018D7F }, + { 0x018DF3, 0x01AFEF }, + { 0x01AFF4, 0x01AFF4 }, + { 0x01AFFC, 0x01AFFC }, + { 0x01AFFF, 0x01AFFF }, + { 0x01B123, 0x01B131 }, + { 0x01B133, 0x01B14F }, + { 0x01B153, 0x01B154 }, + { 0x01B156, 0x01B163 }, + { 0x01B168, 0x01B16F }, + { 0x01B2FC, 0x01BBFF }, + { 0x01BC6B, 0x01BC6F }, + { 0x01BC7D, 0x01BC7F }, + { 0x01BC89, 0x01BC8F }, + { 0x01BC9A, 0x01BC9B }, + { 0x01BCA0, 0x01CBFF }, + { 0x01CCFD, 0x01CCFF }, + { 0x01CEB4, 0x01CEB9 }, + { 0x01CED1, 0x01CEDF }, + { 0x01CEF1, 0x01CEFF }, + { 0x01CF2E, 0x01CF2F }, + { 0x01CF47, 0x01CF4F }, + { 0x01CFC4, 0x01CFFF }, + { 0x01D0F6, 0x01D0FF }, + { 0x01D127, 0x01D128 }, + { 0x01D173, 0x01D17A }, + { 0x01D1EB, 0x01D1FF }, + { 0x01D246, 0x01D2BF }, + { 0x01D2D4, 0x01D2DF }, + { 0x01D2F4, 0x01D2FF }, + { 0x01D357, 0x01D35F }, + { 0x01D379, 0x01D3FF }, + { 0x01D455, 0x01D455 }, + { 0x01D49D, 0x01D49D }, + { 0x01D4A0, 0x01D4A1 }, + { 0x01D4A3, 0x01D4A4 }, + { 0x01D4A7, 0x01D4A8 }, + { 0x01D4AD, 0x01D4AD }, + { 0x01D4BA, 0x01D4BA }, + { 0x01D4BC, 0x01D4BC }, + { 0x01D4C4, 0x01D4C4 }, + { 0x01D506, 0x01D506 }, + { 0x01D50B, 0x01D50C }, + { 0x01D515, 0x01D515 }, + { 0x01D51D, 0x01D51D }, + { 0x01D53A, 0x01D53A }, + { 0x01D53F, 0x01D53F }, + { 0x01D545, 0x01D545 }, + { 0x01D547, 0x01D549 }, + { 0x01D551, 0x01D551 }, + { 0x01D6A6, 0x01D6A7 }, + { 0x01D7CC, 0x01D7CD }, + { 0x01DA8C, 0x01DA9A }, + { 0x01DAA0, 0x01DAA0 }, + { 0x01DAB0, 0x01DEFF }, + { 0x01DF1F, 0x01DF24 }, + { 0x01DF2B, 0x01DFFF }, + { 0x01E007, 0x01E007 }, + { 0x01E019, 0x01E01A }, + { 0x01E022, 0x01E022 }, + { 0x01E025, 0x01E025 }, + { 0x01E02B, 0x01E02F }, + { 0x01E06E, 0x01E08E }, + { 0x01E090, 0x01E0FF }, + { 0x01E12D, 0x01E12F }, + { 0x01E13E, 0x01E13F }, + { 0x01E14A, 0x01E14D }, + { 0x01E150, 0x01E28F }, + { 0x01E2AF, 0x01E2BF }, + { 0x01E2FA, 0x01E2FE }, + { 0x01E300, 0x01E4CF }, + { 0x01E4FA, 0x01E5CF }, + { 0x01E5FB, 0x01E5FE }, + { 0x01E600, 0x01E6BF }, + { 0x01E6DF, 0x01E6DF }, + { 0x01E6F6, 0x01E6FD }, + { 0x01E700, 0x01E7DF }, + { 0x01E7E7, 0x01E7E7 }, + { 0x01E7EC, 0x01E7EC }, + { 0x01E7EF, 0x01E7EF }, + { 0x01E7FF, 0x01E7FF }, + { 0x01E8C5, 0x01E8C6 }, + { 0x01E8D7, 0x01E8FF }, + { 0x01E94C, 0x01E94F }, + { 0x01E95A, 0x01E95D }, + { 0x01E960, 0x01EC70 }, + { 0x01ECB5, 0x01ED00 }, + { 0x01ED3E, 0x01EDFF }, + { 0x01EE04, 0x01EE04 }, + { 0x01EE20, 0x01EE20 }, + { 0x01EE23, 0x01EE23 }, + { 0x01EE25, 0x01EE26 }, + { 0x01EE28, 0x01EE28 }, + { 0x01EE33, 0x01EE33 }, + { 0x01EE38, 0x01EE38 }, + { 0x01EE3A, 0x01EE3A }, + { 0x01EE3C, 0x01EE41 }, + { 0x01EE43, 0x01EE46 }, + { 0x01EE48, 0x01EE48 }, + { 0x01EE4A, 0x01EE4A }, + { 0x01EE4C, 0x01EE4C }, + { 0x01EE50, 0x01EE50 }, + { 0x01EE53, 0x01EE53 }, + { 0x01EE55, 0x01EE56 }, + { 0x01EE58, 0x01EE58 }, + { 0x01EE5A, 0x01EE5A }, + { 0x01EE5C, 0x01EE5C }, + { 0x01EE5E, 0x01EE5E }, + { 0x01EE60, 0x01EE60 }, + { 0x01EE63, 0x01EE63 }, + { 0x01EE65, 0x01EE66 }, + { 0x01EE6B, 0x01EE6B }, + { 0x01EE73, 0x01EE73 }, + { 0x01EE78, 0x01EE78 }, + { 0x01EE7D, 0x01EE7D }, + { 0x01EE7F, 0x01EE7F }, + { 0x01EE8A, 0x01EE8A }, + { 0x01EE9C, 0x01EEA0 }, + { 0x01EEA4, 0x01EEA4 }, + { 0x01EEAA, 0x01EEAA }, + { 0x01EEBC, 0x01EEEF }, + { 0x01EEF2, 0x01EFFF }, + { 0x01F02C, 0x01F02F }, + { 0x01F094, 0x01F09F }, + { 0x01F0AF, 0x01F0B0 }, + { 0x01F0C0, 0x01F0C0 }, + { 0x01F0D0, 0x01F0D0 }, + { 0x01F0F6, 0x01F0FF }, + { 0x01F1AE, 0x01F1E5 }, + { 0x01F203, 0x01F20F }, + { 0x01F23C, 0x01F23F }, + { 0x01F249, 0x01F24F }, + { 0x01F252, 0x01F25F }, + { 0x01F266, 0x01F2FF }, + { 0x01F6D9, 0x01F6DB }, + { 0x01F6ED, 0x01F6EF }, + { 0x01F6FD, 0x01F6FF }, + { 0x01F7DA, 0x01F7DF }, + { 0x01F7EC, 0x01F7EF }, + { 0x01F7F1, 0x01F7FF }, + { 0x01F80C, 0x01F80F }, + { 0x01F848, 0x01F84F }, + { 0x01F85A, 0x01F85F }, + { 0x01F888, 0x01F88F }, + { 0x01F8AE, 0x01F8AF }, + { 0x01F8BC, 0x01F8BF }, + { 0x01F8C2, 0x01F8CF }, + { 0x01F8D9, 0x01F8FF }, + { 0x01FA58, 0x01FA5F }, + { 0x01FA6E, 0x01FA6F }, + { 0x01FA7D, 0x01FA7F }, + { 0x01FA8B, 0x01FA8D }, + { 0x01FAC7, 0x01FAC7 }, + { 0x01FAC9, 0x01FACC }, + { 0x01FADD, 0x01FADE }, + { 0x01FAEB, 0x01FAEE }, + { 0x01FAF9, 0x01FAFF }, + { 0x01FB93, 0x01FB93 }, + { 0x01FBFB, 0x01FFFF }, + { 0x02A6E0, 0x02A6FF }, + { 0x02B81E, 0x02B81F }, + { 0x02CEAE, 0x02CEAF }, + { 0x02EBE1, 0x02EBEF }, + { 0x02EE5E, 0x02F7FF }, + { 0x02FA1E, 0x02FFFF }, + { 0x03134B, 0x03134F }, + { 0x03347A, 0x0E00FF }, + { 0x0E01F0, 0x10FFFF }, +}; + +bool unicode_is_banned_codepoint(u32 cp) +{ + size_t lo = 0, hi = ARRAY_SIZE(banned_ranges); + + while (lo < hi) { + size_t mid = lo + (hi - lo) / 2; + const struct unicode_range *r = &banned_ranges[mid]; + + if (cp < r->start) + hi = mid; + else if (cp > r->end) + lo = mid + 1; + else + return true; + } + return false; +} diff --git a/common/unicode_category.h b/common/unicode_category.h new file mode 100644 index 000000000000..9a064af0f0c2 --- /dev/null +++ b/common/unicode_category.h @@ -0,0 +1,14 @@ + +/* This file generated by devtools/gen-unicode-category.py */ +/* Do not modify this file!!! Modify the script, or regenerate from a new + * UnicodeData.txt, instead */ + +#ifndef LIGHTNING_COMMON_UNICODE_CATEGORY_H +#define LIGHTNING_COMMON_UNICODE_CATEGORY_H +#include "config.h" +#include +#include + +bool unicode_is_banned_codepoint(u32 cp); + +#endif /* LIGHTNING_COMMON_UNICODE_CATEGORY_H */ diff --git a/common/utils.c b/common/utils.c index 0bd7a72cb341..5cf6c537e466 100644 --- a/common/utils.c +++ b/common/utils.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -180,8 +181,7 @@ void tal_arr_appendn_(void *p, const void *append TAKES, size_t bytes) tal_arr_append_bytes(p, append, bytes); } -/* Check for valid UTF-8 */ -bool utf8_check(const void *vbuf, size_t buflen) +static bool utf8_check_(const void *vbuf, size_t buflen, bool reject_banned) { const u8 *buf = vbuf; struct utf8_state utf8_state = UTF8_STATE_INIT; @@ -195,15 +195,29 @@ bool utf8_check(const void *vbuf, size_t buflen) need_more = false; if (errno != 0) return false; + if (reject_banned && unicode_is_banned_codepoint(utf8_state.c)) + return false; } return !need_more; } -char *utf8_str(const tal_t *ctx, const u8 *buf TAKES, size_t buflen) +/* Check for valid UTF-8 */ +bool utf8_check(const void *vbuf, size_t buflen) +{ + return utf8_check_(vbuf, buflen, false); +} + +bool utf8_check_text(const void *vbuf, size_t buflen) +{ + return utf8_check_(vbuf, buflen, true); +} + +static char *utf8_str_(const tal_t *ctx, const u8 *buf TAKES, size_t buflen, + bool reject_banned) { char *ret; - if (!utf8_check(buf, buflen)) { + if (!utf8_check_(buf, buflen, reject_banned)) { tal_free_if_taken(buf); return NULL; } @@ -214,6 +228,16 @@ char *utf8_str(const tal_t *ctx, const u8 *buf TAKES, size_t buflen) return ret; } +char *utf8_str(const tal_t *ctx, const u8 *buf TAKES, size_t buflen) +{ + return utf8_str_(ctx, buf, buflen, false); +} + +char *utf8_str_text(const tal_t *ctx, const u8 *buf TAKES, size_t buflen) +{ + return utf8_str_(ctx, buf, buflen, true); +} + char *tal_strdup_or_null(const tal_t *ctx, const char *str) { if (!str) { diff --git a/common/utils.h b/common/utils.h index ac719d0cc5d9..030f0d8f8a51 100644 --- a/common/utils.h +++ b/common/utils.h @@ -139,6 +139,13 @@ bool utf8_check(const void *buf, size_t buflen); /* Check it's UTF-8, return copy (or same if TAKES), or NULL if not valid. */ char *utf8_str(const tal_t *ctx, const u8 *buf TAKES, size_t buflen); +/* Like utf8_check(), but also reject codepoints in Unicode general + * categories Cc/Cf/Co/Cn (control, format, private-use, unassigned) */ +bool utf8_check_text(const void *buf, size_t buflen); + +/* Like utf8_str(), but using utf8_check_text() instead of utf8_check(). */ +char *utf8_str_text(const tal_t *ctx, const u8 *buf TAKES, size_t buflen); + /* Strdup, or pass through NULL */ char *tal_strdup_or_null(const tal_t *ctx, const char *str); diff --git a/devtools/gen-unicode-category.py b/devtools/gen-unicode-category.py new file mode 100644 index 000000000000..850d09f617aa --- /dev/null +++ b/devtools/gen-unicode-category.py @@ -0,0 +1,161 @@ +#! /usr/bin/env python3 + +from argparse import ArgumentParser + +BANNED_CATEGORIES = ("Cc", "Cf", "Co") +MAX_CODEPOINT = 0x10FFFF + +HEADER_BANNER = """ +/* This file generated by devtools/gen-unicode-category.py */ +/* Do not modify this file!!! Modify the script, or regenerate from a new + * UnicodeData.txt, instead */ +""" + + +def parse_unicode_data(path): + assigned = [] + pending_range_start = None + pending_range_category = None + + with open(path, encoding="utf-8") as f: + for line in f: + fields = line.rstrip("\n").split(";") + codepoint = int(fields[0], 16) + name = fields[1] + category = fields[2] + + if name.endswith(", First>"): + pending_range_start = codepoint + pending_range_category = category + continue + + if name.endswith(", Last>"): + assert pending_range_start is not None + assert pending_range_category == category + assigned.append((pending_range_start, codepoint, category)) + pending_range_start = None + pending_range_category = None + continue + + assigned.append((codepoint, codepoint, category)) + + assigned.sort(key=lambda r: r[0]) + return assigned + + +def compute_banned_ranges(assigned): + banned = [] + + for start, end, category in assigned: + if category in BANNED_CATEGORIES: + banned.append((start, end)) + + next_free = 0 + for start, end, _category in assigned: + if start > next_free: + banned.append((next_free, start - 1)) + next_free = max(next_free, end + 1) + if next_free <= MAX_CODEPOINT: + banned.append((next_free, MAX_CODEPOINT)) + + banned.sort(key=lambda r: r[0]) + + merged = [] + for start, end in banned: + if merged and start <= merged[-1][1] + 1: + merged[-1] = (merged[-1][0], max(merged[-1][1], end)) + else: + merged.append((start, end)) + return merged + + +def write_header(path, guard): + with open(path, "w", encoding="utf-8") as f: + f.write(HEADER_BANNER) + f.write( + f""" +#ifndef {guard} +#define {guard} +#include "config.h" +#include +#include + +bool unicode_is_banned_codepoint(u32 cp); + +#endif /* {guard} */ +""" + ) + + +def write_source(path, header_path, ranges, unicode_version): + with open(path, "w", encoding="utf-8") as f: + f.write(HEADER_BANNER) + f.write(f"/* Generated from Unicode {unicode_version} UnicodeData.txt */\n") + f.write( + f'\n' + f'#include "config.h"\n' + f'#include \n' + f'#include "{header_path}"\n' + f'#include \n' + f'\n' + f'struct unicode_range {{\n' + f'\tu32 start, end;\n' + f'}};\n' + f'\n' + f'/* Sorted, non-overlapping, coalesced ranges banned as Cc/Cf/Co/Cn. */\n' + f'static const struct unicode_range banned_ranges[] = {{\n' + ) + for start, end in ranges: + f.write(f"\t{{ 0x{start:06X}, 0x{end:06X} }},\n") + f.write( + '};\n' + '\n' + 'bool unicode_is_banned_codepoint(u32 cp)\n' + '{\n' + '\tsize_t lo = 0, hi = ARRAY_SIZE(banned_ranges);\n' + '\n' + '\twhile (lo < hi) {\n' + '\t\tsize_t mid = lo + (hi - lo) / 2;\n' + '\t\tconst struct unicode_range *r = &banned_ranges[mid];\n' + '\n' + '\t\tif (cp < r->start)\n' + '\t\t\thi = mid;\n' + '\t\telse if (cp > r->end)\n' + '\t\t\tlo = mid + 1;\n' + '\t\telse\n' + '\t\t\treturn true;\n' + '\t}\n' + '\treturn false;\n' + '}\n' + ) + + +def main(): + parser = ArgumentParser(description=__doc__) + parser.add_argument("unicodedata", help="Path to UnicodeData.txt") + parser.add_argument( + "outbase", + help="Output basename, common/unicode_category " + "(writes .c and .h)", + ) + parser.add_argument( + "--unicode-version", + default="unknown", + help="Unicode version string to record in a comment", + ) + args = parser.parse_args() + + assigned = parse_unicode_data(args.unicodedata) + ranges = compute_banned_ranges(assigned) + + header_path = args.outbase + ".h" + source_path = args.outbase + ".c" + guard = "LIGHTNING_" + args.outbase.upper().replace("/", "_") + "_H" + write_header(header_path, guard) + write_source(source_path, header_path, ranges, args.unicode_version) + + print(f"Wrote {header_path} and {source_path}: {len(ranges)} banned ranges") + + +if __name__ == "__main__": + main() diff --git a/lightningd/invoice.c b/lightningd/invoice.c index 28173383d23d..cd92596d88fb 100644 --- a/lightningd/invoice.c +++ b/lightningd/invoice.c @@ -1134,7 +1134,7 @@ static struct command_result *json_invoice(struct command *cmd, if (!param_check(cmd, buffer, params, p_req("amount_msat", param_positive_msat_or_any, &msatoshi_val), p_req("label", param_label, &info->label), - p_req("description", param_escaped_string, &desc_val), + p_req("description", param_escaped_utf8_string, &desc_val), p_opt_def("expiry", param_u64, &expiry, 3600*24*7), p_opt("fallbacks", param_array, &fallbacks), p_opt("preimage", param_preimage, &preimage), diff --git a/onchaind/test/Makefile b/onchaind/test/Makefile index b776ea1fd1a7..09aafa39ac5e 100644 --- a/onchaind/test/Makefile +++ b/onchaind/test/Makefile @@ -22,6 +22,7 @@ onchaind/test/run-grind_feerate-bug: \ common/psbt_keypath.o \ common/pseudorand.o \ common/setup.o \ + common/unicode_category.o \ common/utils.o # Test objects depend on ../ src and headers. diff --git a/plugins/fetchinvoice.c b/plugins/fetchinvoice.c index 48a52a051f39..0937528509bf 100644 --- a/plugins/fetchinvoice.c +++ b/plugins/fetchinvoice.c @@ -874,7 +874,7 @@ struct command_result *json_fetchinvoice(struct command *cmd, p_opt("recurrence_start", param_number, &recurrence_start), p_opt("recurrence_label", param_label, &rec_label), p_opt_def("timeout", param_number, &timeout, 60), - p_opt("payer_note", param_string, &payer_note), + p_opt("payer_note", param_utf8_string, &payer_note), p_opt("payer_metadata", param_bin_from_hex, &payer_metadata), p_opt("bip353", param_bip353, &bip353), p_opt("dev_path_use_scidd", param_dev_scidd, &sent->dev_path_use_scidd), @@ -1120,7 +1120,7 @@ struct command_result *json_cancelrecurringinvoice(struct command *cmd, p_req("recurrence_counter", param_number, &recurrence_counter), p_req("recurrence_label", param_label, &rec_label), p_opt("recurrence_start", param_number, &recurrence_start), - p_opt("payer_note", param_string, &payer_note), + p_opt("payer_note", param_utf8_string, &payer_note), p_opt("bip353", param_bip353, &bip353), NULL)) return command_param_failed(); diff --git a/plugins/offers_offer.c b/plugins/offers_offer.c index 0bd87ad726ba..8ca767541914 100644 --- a/plugins/offers_offer.c +++ b/plugins/offers_offer.c @@ -427,8 +427,8 @@ struct command_result *json_offer(struct command *cmd, if (!param_check(cmd, buffer, params, p_req("amount", param_amount, offer), - p_opt("description", param_escaped_string, &desc), - p_opt("issuer", param_escaped_string, &issuer), + p_opt("description", param_escaped_utf8_string, &desc), + p_opt("issuer", param_escaped_utf8_string, &issuer), p_opt("label", param_escaped_string, &offinfo->label), p_opt("quantity_max", param_u64, &offer->offer_quantity_max), p_opt("absolute_expiry", param_u64, &offer->offer_absolute_expiry), @@ -690,8 +690,8 @@ struct command_result *json_invoicerequest(struct command *cmd, if (!param(cmd, buffer, params, p_req("amount", param_msat, &msat), - p_opt("description", param_escaped_string, &desc), - p_opt("issuer", param_escaped_string, &issuer), + p_opt("description", param_escaped_utf8_string, &desc), + p_opt("issuer", param_escaped_utf8_string, &issuer), p_opt("label", param_escaped_string, &label), p_opt("absolute_expiry", param_u64, &invreq->offer_absolute_expiry), diff --git a/wire/fromwire.c b/wire/fromwire.c index 9fded589e2d0..9a397fe91ccd 100644 --- a/wire/fromwire.c +++ b/wire/fromwire.c @@ -232,7 +232,7 @@ void fromwire_u8_array(const u8 **cursor, size_t *max, u8 *arr, size_t num) void fromwire_utf8_array(const u8 **cursor, size_t *max, char *arr, size_t num) { fromwire(cursor, max, arr, num); - if (!utf8_check(arr, num)) + if (!utf8_check_text(arr, num)) fromwire_fail(cursor, max); } diff --git a/wire/test/Makefile b/wire/test/Makefile index ad3720ae3891..0bd08988bc16 100644 --- a/wire/test/Makefile +++ b/wire/test/Makefile @@ -14,6 +14,7 @@ WIRE_TEST_COMMON_OBJS := \ common/base32.o \ common/pseudorand.o \ common/setup.o \ + common/unicode_category.o \ common/utils.o \ common/wireaddr.o diff --git a/wire/towire.c b/wire/towire.c index b90c7e3d1793..e84899665cef 100644 --- a/wire/towire.c +++ b/wire/towire.c @@ -142,7 +142,7 @@ void towire_u8_array(u8 **pptr, const u8 *arr, size_t num) void towire_utf8_array(u8 **pptr, const char *arr, size_t num) { - assert(utf8_check(arr, num)); + assert(utf8_check_text(arr, num)); towire(pptr, arr, num); }