diff --git a/include/proxy/hdrs/HdrToken.h b/include/proxy/hdrs/HdrToken.h index 9d862bb4eae..c8dfcec8011 100644 --- a/include/proxy/hdrs/HdrToken.h +++ b/include/proxy/hdrs/HdrToken.h @@ -24,6 +24,8 @@ #pragma once #include +#include +#include #include #include "tscore/ink_assert.h" #include "tscore/ink_atomic.h" @@ -31,7 +33,6 @@ #include "tscore/ink_memory.h" #include "tscore/ink_string.h" #include "tscore/Allocator.h" -#include "tsutil/Regex.h" #include "tscore/ink_apidefs.h" //////////////////////////////////////////////////////////////////////////// @@ -57,25 +58,33 @@ enum class HdrTokenInfoFlags : uint32_t { PROXYAUTH = 1 << 3, }; -inline HdrTokenInfoFlags +constexpr HdrTokenInfoFlags operator|(HdrTokenInfoFlags a, HdrTokenInfoFlags b) { return static_cast(static_cast(a) | static_cast(b)); } -inline HdrTokenInfoFlags +constexpr HdrTokenInfoFlags operator&(HdrTokenInfoFlags a, HdrTokenInfoFlags b) { return static_cast(static_cast(a) & static_cast(b)); } -struct HdrTokenFieldInfo { +// One row of the field initializer table. The name binds the row to a well-known string and is not +// needed once the row is resolved, so it is absent from HdrTokenFieldInfo below. +struct HdrTokenFieldInit { const char *name; int32_t slotid; uint64_t mask; HdrTokenInfoFlags flags; }; +struct HdrTokenFieldInfo { + int32_t slotid; + uint64_t mask; + HdrTokenInfoFlags flags; +}; + struct HdrTokenTypeSpecific { union { struct { @@ -92,8 +101,20 @@ struct HdrTokenHeapPrefix { HdrTokenTypeSpecific wks_type_specific; }; -extern DFA *hdrtoken_strs_dfa; -extern int hdrtoken_num_wks; +// Storage reserved per well-known string, including the NUL terminator. The size is fixed here +// rather than computed from the longest string because hdrtoken_wks_to_prefix() needs +// sizeof(HdrTokenWksEntry) to recover an entry index from a string pointer; HdrToken.cc statically +// asserts that every string fits. +constexpr size_t HDRTOKEN_WKS_STORAGE = 32; + +struct HdrTokenWksEntry { + HdrTokenHeapPrefix prefix; + char str[HDRTOKEN_WKS_STORAGE]; +}; + +extern const HdrTokenWksEntry *const hdrtoken_wks_entries; + +extern int hdrtoken_num_wks; extern const char *hdrtoken_strs[]; extern int hdrtoken_str_lengths[]; @@ -109,7 +130,6 @@ extern HdrTokenInfoFlags hdrtoken_str_flags[]; //////////////////////////////////////////////////////////////////////////// extern void hdrtoken_init(); -extern int hdrtoken_tokenize_dfa(const char *string, int string_len, const char **wks_string_out = nullptr); extern int hdrtoken_tokenize(const char *string, int string_len, const char **wks_string_out = nullptr); extern int hdrtoken_method_tokenize(const char *string, int string_len); extern const char *hdrtoken_string_to_wks(const char *string); @@ -141,12 +161,20 @@ hdrtoken_is_valid_wks_idx(int wks_idx) /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ -// ToDo: This, and dependencies / users should probably be const HdrTokenHeapPrefix * IMO. -inline HdrTokenHeapPrefix * +// The well-known strings live in a table that is built at compile time and is read-only, so a +// prefix is only ever read through. +inline const HdrTokenHeapPrefix * hdrtoken_wks_to_prefix(const char *wks) { ink_assert(hdrtoken_is_wks(wks)); - return reinterpret_cast(const_cast(wks) - sizeof(HdrTokenHeapPrefix)); + + // Recover the entry index through integer arithmetic. Subtracting sizeof(HdrTokenHeapPrefix) + // from wks directly would form a pointer outside the str array member, which is undefined + // behavior even though the prefix is physically adjacent. + uintptr_t const offset = reinterpret_cast(wks) - reinterpret_cast(hdrtoken_wks_entries); + + ink_assert(offset % sizeof(HdrTokenWksEntry) == offsetof(HdrTokenWksEntry, str)); + return &hdrtoken_wks_entries[offset / sizeof(HdrTokenWksEntry)].prefix; } /*------------------------------------------------------------------------- @@ -194,7 +222,7 @@ hdrtoken_index_to_flags(int wks_idx) return hdrtoken_str_flags[wks_idx]; } -inline HdrTokenHeapPrefix * +inline const HdrTokenHeapPrefix * hdrtoken_index_to_prefix(int wks_idx) { ink_assert(hdrtoken_is_valid_wks_idx(wks_idx)); @@ -236,7 +264,7 @@ inline uint64_t hdrtoken_wks_to_mask(const char *wks) { ink_assert(hdrtoken_is_wks(wks)); - HdrTokenHeapPrefix *prefix = hdrtoken_wks_to_prefix(wks); + const HdrTokenHeapPrefix *prefix = hdrtoken_wks_to_prefix(wks); return prefix->wks_info.mask; } diff --git a/include/proxy/hdrs/MIME.h b/include/proxy/hdrs/MIME.h index 932217a5e5c..210b77489ad 100644 --- a/include/proxy/hdrs/MIME.h +++ b/include/proxy/hdrs/MIME.h @@ -720,7 +720,6 @@ void mime_hdr_presence_unset(MIMEHdrImpl *h, int well_known_str_index); void mime_hdr_sanity_check(MIMEHdrImpl *mh); void mime_init(); -void mime_init_cache_control_cooking_masks(); void mime_init_date_format_table(); MIMEHdrImpl *mime_hdr_create(HdrHeap *heap); diff --git a/src/proxy/hdrs/HdrToken.cc b/src/proxy/hdrs/HdrToken.cc index 6d2beabfec9..24dbbd43ce1 100644 --- a/src/proxy/hdrs/HdrToken.cc +++ b/src/proxy/hdrs/HdrToken.cc @@ -22,16 +22,16 @@ */ #include "tscore/ink_platform.h" -#include "tscore/HashFNV.h" #include "tscore/Diags.h" #include "tscore/ink_memory.h" -#include -#include "tscore/Allocator.h" -#include "proxy/hdrs/HTTP.h" #include "proxy/hdrs/HdrToken.h" #include "proxy/hdrs/MIME.h" -#include "tsutil/Regex.h" -#include "proxy/hdrs/URL.h" + +#include +#include +#include +#include +#include namespace { @@ -40,20 +40,19 @@ DbgCtl dbg_ctl_hdr_token{"hdr_token"}; /* WARNING: Indexes into this array are stored on disk for cached objects. New strings must be added at the end of the array to avoid changing the indexes of pre-existing entries, unless the cache format version number is increased. - - You want a regexp like 'Accept' after "greedier" choices so it doesn't match 'Accept-Ranges' earlier than - it should. The regexp are anchored (^Accept), but I dont see a way with the current system to - match the word ONLY without making _hdrtoken_strs a real PCRE2, but then that breaks the hashing - hdrtoken_hash("^Accept$") != hdrtoken_hash("Accept") - - So, the current hack is to have "Accept" follow "Accept-.*", lame, I know - - /ericb - - */ +struct HdrTokenFrozen { + size_t count; + uint32_t fingerprint; +}; + +// When you append strings to _hdrtoken_strs, also append an entry to _hdrtoken_strs_frozen. +// This ledger ensures that WKS strings are append-only. +constexpr HdrTokenFrozen _hdrtoken_strs_frozen[] = { + {135, 0x9ea577a9u}, +}; -const char *const _hdrtoken_strs[] = { +constexpr std::string_view _hdrtoken_strs[] = { // MIME Field names "Accept-Charset", "Accept-Encoding", "Accept-Language", "Accept-Ranges", "Accept", "Age", "Allow", "Approved", // NNTP @@ -130,7 +129,7 @@ const char *const _hdrtoken_strs[] = { // RFC-9213 Targeted Cache Control "CDN-Cache-Control"}; -HdrTokenTypeBinding _hdrtoken_strs_type_initializers[] = { +constexpr HdrTokenTypeBinding _hdrtoken_strs_type_initializers[] = { {"file", HdrTokenType::SCHEME }, {"ftp", HdrTokenType::SCHEME }, {"gopher", HdrTokenType::SCHEME }, @@ -180,7 +179,7 @@ HdrTokenTypeBinding _hdrtoken_strs_type_initializers[] = { {(char *)nullptr, static_cast(0)}, }; -HdrTokenFieldInfo _hdrtoken_strs_field_initializers[] = { +constexpr HdrTokenFieldInit _hdrtoken_strs_field_initializers[] = { {"Accept", MIME_SLOTID_ACCEPT, MIME_PRESENCE_ACCEPT, (HdrTokenInfoFlags::COMMAS | HdrTokenInfoFlags::MULTVALS) }, {"Accept-Charset", MIME_SLOTID_ACCEPT_CHARSET, MIME_PRESENCE_ACCEPT_CHARSET, (HdrTokenInfoFlags::COMMAS | HdrTokenInfoFlags::MULTVALS) }, @@ -274,239 +273,403 @@ HdrTokenFieldInfo _hdrtoken_strs_field_initializers[] = { {nullptr, 0, 0, HdrTokenInfoFlags::NONE }, }; -} // end anonymous namespace - -const char *_hdrtoken_strs_heap_f = nullptr; // storage first byte -const char *_hdrtoken_strs_heap_l = nullptr; // storage last byte - -int hdrtoken_num_wks = SIZEOF(_hdrtoken_strs); // # of well-known strings - -const char *hdrtoken_strs[SIZEOF(_hdrtoken_strs)]; // wks_idx -> heap ptr -int hdrtoken_str_lengths[SIZEOF(_hdrtoken_strs)]; // wks_idx -> length -HdrTokenType hdrtoken_str_token_types[SIZEOF(_hdrtoken_strs)]; // wks_idx -> token type -int32_t hdrtoken_str_slotids[SIZEOF(_hdrtoken_strs)]; // wks_idx -> slot id -uint64_t hdrtoken_str_masks[SIZEOF(_hdrtoken_strs)]; // wks_idx -> presence mask -HdrTokenInfoFlags hdrtoken_str_flags[SIZEOF(_hdrtoken_strs)]; // wks_idx -> flags +struct HdrTokenCacheControlBinding { + const char *name; + uint32_t mask; +}; -DFA *hdrtoken_strs_dfa = nullptr; +// The cooked mask for each Cache-Control directive. This is baked into the well-known-string +// table, so it belongs beside the other initializers rather than in MIME.cc. +constexpr HdrTokenCacheControlBinding _hdrtoken_strs_cc_initializers[] = { + {"max-age", MIME_COOKED_MASK_CC_MAX_AGE }, + {"no-cache", MIME_COOKED_MASK_CC_NO_CACHE }, + {"no-store", MIME_COOKED_MASK_CC_NO_STORE }, + {"no-transform", MIME_COOKED_MASK_CC_NO_TRANSFORM }, + {"max-stale", MIME_COOKED_MASK_CC_MAX_STALE }, + {"min-fresh", MIME_COOKED_MASK_CC_MIN_FRESH }, + {"only-if-cached", MIME_COOKED_MASK_CC_ONLY_IF_CACHED }, + {"public", MIME_COOKED_MASK_CC_PUBLIC }, + {"private", MIME_COOKED_MASK_CC_PRIVATE }, + {"must-revalidate", MIME_COOKED_MASK_CC_MUST_REVALIDATE }, + {"proxy-revalidate", MIME_COOKED_MASK_CC_PROXY_REVALIDATE }, + {"s-maxage", MIME_COOKED_MASK_CC_S_MAXAGE }, + {"need-revalidate-once", MIME_COOKED_MASK_CC_NEED_REVALIDATE_ONCE}, + {nullptr, 0 }, +}; /*********************************************************************** * * - * H A S H T A B L E * + * C O M P I L E - T I M E W K S T A B L E * * * ***********************************************************************/ -static constexpr size_t HDRTOKEN_HASH_TABLE_SIZE = 65536; +// hash_to_slot() folds a hash down to this many bits, so the table needs exactly one bucket per +// value those bits can take. +constexpr uint32_t HDRTOKEN_HASH_SLOT_BITS = 15; +constexpr uint32_t HDRTOKEN_HASH_SLOT_MASK = (1 << HDRTOKEN_HASH_SLOT_BITS) - 1; +constexpr size_t HDRTOKEN_HASH_TABLE_SIZE = static_cast(HDRTOKEN_HASH_SLOT_MASK) + 1; -struct HdrTokenHashBucket { - const char *wks; - uint32_t hash; -}; +constexpr uint32_t +hash_to_slot(uint32_t hash) +{ + return ((hash >> HDRTOKEN_HASH_SLOT_BITS) ^ hash) & HDRTOKEN_HASH_SLOT_MASK; +} -HdrTokenHashBucket hdrtoken_hash_table[HDRTOKEN_HASH_TABLE_SIZE]; +constexpr unsigned char +hdrtoken_ascii_toupper(unsigned char c) +{ + return (c >= 'a' && c <= 'z') ? static_cast(c - ('a' - 'A')) : c; +} -/** - basic FNV hash -**/ -#define TINY_MASK(x) (((uint32_t)1 << (x)) - 1) +constexpr uint32_t HDRTOKEN_HASH_SEED = 0x811c9dc5u; // FNV-1a 32-bit offset basis -inline uint32_t -hash_to_slot(uint32_t hash) +// One raw FNV-1a step. hdrtoken_hash() folds case on top of it; the frozen-ledger fingerprint +// deliberately does not. +constexpr uint32_t +hdrtoken_hash_step(uint32_t hval, unsigned char c) { - return ((hash >> 15) ^ hash) & TINY_MASK(15); + return (hval ^ c) * 0x01000193u; } -inline uint32_t -hdrtoken_hash(const unsigned char *string, unsigned int length) +// The one hash function, shared by compile-time table construction and hdrtoken_tokenize(), so the +// two can never disagree. +constexpr uint32_t +hdrtoken_hash(std::string_view s) { - ATSHash32FNV1a fnv; - fnv.update(string, length, ATSHash::nocase()); - fnv.final(); - return fnv.get(); -} + uint32_t hval = HDRTOKEN_HASH_SEED; -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ + for (char const c : s) { + hval = hdrtoken_hash_step(hval, hdrtoken_ascii_toupper(static_cast(c))); + } + return hval; +} -void -hdrtoken_hash_init() +constexpr uint32_t +hdrtoken_frozen_fingerprint(size_t count) { - uint32_t i; - int num_collisions; - - memset(hdrtoken_hash_table, 0, sizeof(hdrtoken_hash_table)); - num_collisions = 0; - - for (i = 0; i < static_cast SIZEOF(_hdrtoken_strs); i++) { - // convert the common string to the well-known token - unsigned const char *wks; - int wks_idx = - hdrtoken_tokenize_dfa(_hdrtoken_strs[i], static_cast(strlen(_hdrtoken_strs[i])), reinterpret_cast(&wks)); - ink_release_assert(wks_idx >= 0); - - uint32_t hash = hdrtoken_hash(wks, hdrtoken_str_lengths[wks_idx]); - uint32_t slot = hash_to_slot(hash); - - if (hdrtoken_hash_table[slot].wks) { - printf("ERROR: hdrtoken_hash_table[%u] collision: '%s' replacing '%s'\n", slot, reinterpret_cast(wks), - hdrtoken_hash_table[slot].wks); - ++num_collisions; + // Hashes the raw bytes, without case folding, because case is significant for frozen entries: + // hdrtoken_method_tokenize() matches methods case-sensitively against the stored bytes. + uint32_t hval = HDRTOKEN_HASH_SEED; + + for (size_t i = 0; i < count; ++i) { + for (char const c : _hdrtoken_strs[i]) { + hval = hdrtoken_hash_step(hval, static_cast(c)); } - hdrtoken_hash_table[slot].wks = reinterpret_cast(wks); - hdrtoken_hash_table[slot].hash = hash; + hval = hdrtoken_hash_step(hval, '\0'); // fold in a terminator so entry boundaries matter } + return hval; +} + +constexpr bool +hdrtoken_frozen_rows_valid() +{ + size_t prev_count = 0; - if (num_collisions > 0) { - abort(); + for (auto const &f : _hdrtoken_strs_frozen) { + if (f.count <= prev_count || f.count > std::size(_hdrtoken_strs)) { + return false; + } + if (hdrtoken_frozen_fingerprint(f.count) != f.fingerprint) { + return false; + } + prev_count = f.count; } + return true; } -/*********************************************************************** - * * - * M A I N H D R T O K E N C O D E * - * * - ***********************************************************************/ +static_assert(hdrtoken_frozen_rows_valid(), + "A frozen well-known string changed. Indexes are stored in cached objects, so entries may only be appended, " + "never inserted, reordered, removed, or edited"); +static_assert(_hdrtoken_strs_frozen[std::size(_hdrtoken_strs_frozen) - 1].count == std::size(_hdrtoken_strs), + "The well-known string table grew without being re-frozen; append a {count, fingerprint} row to " + "_hdrtoken_strs_frozen"); -/** - @return returns 0 for n=0, unit*n for n <= unit -*/ +constexpr size_t +hdrtoken_max_literal_length() +{ + const auto longest = std::max_element(std::cbegin(_hdrtoken_strs), std::cend(_hdrtoken_strs), + [](std::string_view a, std::string_view b) { return a.length() < b.length(); }); + return longest->length(); +} + +static_assert(hdrtoken_max_literal_length() + 1 <= HDRTOKEN_WKS_STORAGE, + "a well-known string does not fit its entry; raise HDRTOKEN_WKS_STORAGE"); -static inline unsigned int -snap_up_to_multiple(unsigned int n, unsigned int unit) +constexpr bool +hdrtoken_literals_equal_nocase(std::string_view a, std::string_view b) { - return ((n + (unit - 1)) / unit) * unit; + if (a.size() != b.size()) { + return false; + } + for (size_t i = 0; i < a.size(); ++i) { + if (hdrtoken_ascii_toupper(static_cast(a[i])) != hdrtoken_ascii_toupper(static_cast(b[i]))) { + return false; + } + } + return true; } -/** - */ -void -hdrtoken_init() +constexpr int +hdrtoken_index_of_literal(std::string_view name) { - static int inited = 0; + for (size_t i = 0; i < std::size(_hdrtoken_strs); ++i) { + if (hdrtoken_literals_equal_nocase(_hdrtoken_strs[i], name)) { + return static_cast(i); + } + } + return -1; +} - int i; +// Each initializer name must be an entry in _hdrtoken_strs, and no two rows of one table may name +// the same entry. +template +constexpr bool +hdrtoken_names_resolve_uniquely(Table const &table) +{ + std::array seen{}; - if (!inited) { - inited = 1; + for (auto const &row : table) { + if (row.name == nullptr) { + continue; + } + int const idx = hdrtoken_index_of_literal(row.name); - hdrtoken_strs_dfa = new DFA; - hdrtoken_strs_dfa->compile(_hdrtoken_strs, SIZEOF(_hdrtoken_strs), (RE_CASE_INSENSITIVE)); - - // all the tokenized hdrtoken strings are placed in a special heap, - // and each string is prepended with a HdrTokenHeapPrefix --- - // this makes it easy to tell that a string is a tokenized - // string (because its address is within the heap), and - // makes it easy to find the length, index, flags, mask, and - // other info from the prefix. - - int heap_size = 0; - for (i = 0; i < static_cast SIZEOF(_hdrtoken_strs); i++) { - hdrtoken_str_lengths[i] = static_cast(strlen(_hdrtoken_strs[i])); - int sstr_len = snap_up_to_multiple(hdrtoken_str_lengths[i] + 1, sizeof(HdrTokenHeapPrefix)); - int packed_prefix_str_len = sizeof(HdrTokenHeapPrefix) + sstr_len; - heap_size += packed_prefix_str_len; + if (idx < 0 || seen[idx]) { + return false; } + seen[idx] = true; + } + return true; +} - _hdrtoken_strs_heap_f = static_cast(ats_calloc(1, heap_size)); - _hdrtoken_strs_heap_l = _hdrtoken_strs_heap_f + heap_size - 1; +static_assert(hdrtoken_names_resolve_uniquely(_hdrtoken_strs_type_initializers), + "a token-type initializer names a string that is missing from _hdrtoken_strs or already claimed"); +static_assert(hdrtoken_names_resolve_uniquely(_hdrtoken_strs_field_initializers), + "a field initializer names a string that is missing from _hdrtoken_strs or already claimed"); +static_assert(hdrtoken_names_resolve_uniquely(_hdrtoken_strs_cc_initializers), + "a Cache-Control initializer names a string that is missing from _hdrtoken_strs or already claimed"); + +// Resolution folds ASCII case and matches whole strings only. Exact-length matching is what allows +// a string to be appended to the table even when an existing entry is its prefix. +static_assert(hdrtoken_index_of_literal("cache-control") == hdrtoken_index_of_literal("Cache-Control"), + "resolution must be ASCII case-insensitive"); +static_assert(hdrtoken_index_of_literal("Content-Len") == -1, "a prefix of a well-known string must not resolve"); +static_assert(hdrtoken_index_of_literal("Accept") >= 0 && + hdrtoken_index_of_literal("Accept") != hdrtoken_index_of_literal("Accept-Encoding"), + "a well-known string that is a prefix of another must resolve to its own entry"); + +// The field rows feed the fast MIME slot and presence-bit machinery, so each non-NONE slot id must +// be a valid, unclaimed slot and each nonzero presence mask must be a distinct single bit. +constexpr bool +hdrtoken_field_init_semantics_ok() +{ + std::array slot_seen{}; // MIME_SLOTID_* values are 0..31 + uint64_t mask_seen = 0; - char *heap_ptr = const_cast(_hdrtoken_strs_heap_f); + for (auto const &f : _hdrtoken_strs_field_initializers) { + if (f.name == nullptr) { + continue; + } + if (f.slotid != MIME_SLOTID_NONE) { + if (f.slotid < 0 || f.slotid >= static_cast(slot_seen.size()) || slot_seen[f.slotid]) { + return false; + } + slot_seen[f.slotid] = true; + } + if (f.mask != 0) { + if ((f.mask & (f.mask - 1)) != 0 || (mask_seen & f.mask) != 0) { + return false; + } + mask_seen |= f.mask; + } + } + return true; +} - for (i = 0; i < static_cast SIZEOF(_hdrtoken_strs); i++) { - HdrTokenHeapPrefix prefix; +static_assert(hdrtoken_field_init_semantics_ok(), + "a field initializer has an out-of-range or duplicate slot id, or a multi-bit or duplicate presence mask"); - memset(&prefix, 0, sizeof(HdrTokenHeapPrefix)); +// Every Cache-Control row must carry a distinct single-bit cooked mask, and its string must be +// typed CACHE_CONTROL to satisfy HTTPHdr::is_cache_control_set(), which asserts that type for any +// directive whose mask it consults. +constexpr bool +hdrtoken_cc_init_semantics_ok() +{ + uint32_t mask_seen = 0; - prefix.wks_idx = i; - prefix.wks_length = hdrtoken_str_lengths[i]; - prefix.wks_token_type = HdrTokenType::OTHER; // default, can override later - prefix.wks_info.name = nullptr; // default, can override later - prefix.wks_info.slotid = MIME_SLOTID_NONE; // default, can override later - prefix.wks_info.mask = TOK_64_CONST(0); // default, can override later - prefix.wks_info.flags = HdrTokenInfoFlags::MULTVALS; // default, can override later + for (auto const &c : _hdrtoken_strs_cc_initializers) { + if (c.name == nullptr) { + continue; + } + if (c.mask == 0 || (c.mask & (c.mask - 1)) != 0 || (mask_seen & c.mask) != 0) { + return false; + } + mask_seen |= c.mask; - int sstr_len = snap_up_to_multiple(hdrtoken_str_lengths[i] + 1, sizeof(HdrTokenHeapPrefix)); + bool cc_typed = false; - *reinterpret_cast(heap_ptr) = prefix; // set string prefix - heap_ptr += sizeof(HdrTokenHeapPrefix); // advance heap ptr past index - hdrtoken_strs[i] = heap_ptr; // record string pointer - // coverity[secure_coding] - ink_strlcpy(const_cast(hdrtoken_strs[i]), _hdrtoken_strs[i], - heap_size - sizeof(HdrTokenHeapPrefix)); // copy string into heap - heap_ptr += sstr_len; // advance heap ptr past string - heap_size -= sstr_len; + for (auto const &b : _hdrtoken_strs_type_initializers) { + if (b.name != nullptr && hdrtoken_literals_equal_nocase(b.name, c.name)) { + cc_typed = (b.type == HdrTokenType::CACHE_CONTROL); + break; + } + } + if (!cc_typed) { + return false; } + } + return true; +} - // Set the token types for certain tokens - for (i = 0; _hdrtoken_strs_type_initializers[i].name != nullptr; i++) { - int wks_idx; - HdrTokenHeapPrefix *prefix; +static_assert(hdrtoken_cc_init_semantics_ok(), + "a Cache-Control initializer has a zero, multi-bit, or duplicate mask, or its entry is not typed CACHE_CONTROL"); - wks_idx = hdrtoken_tokenize_dfa(_hdrtoken_strs_type_initializers[i].name, - static_cast(strlen(_hdrtoken_strs_type_initializers[i].name))); +constexpr bool +hdrtoken_wks_slots_unique() +{ + // std::sort and std::adjacent_find are only constexpr in libstdc++ 12 and later + std::array seen{}; + + for (std::string_view const s : _hdrtoken_strs) { + uint32_t const slot = hash_to_slot(hdrtoken_hash(s)); - ink_assert((wks_idx >= 0) && (wks_idx < (int)SIZEOF(hdrtoken_strs))); - // coverity[negative_returns] - prefix = hdrtoken_index_to_prefix(wks_idx); - prefix->wks_token_type = _hdrtoken_strs_type_initializers[i].type; + if (seen[slot]) { + return false; } + seen[slot] = true; + } + return true; +} - // Set special data for field names - for (i = 0; _hdrtoken_strs_field_initializers[i].name != nullptr; i++) { - int wks_idx; - HdrTokenHeapPrefix *prefix; +static_assert(hdrtoken_wks_slots_unique(), "Two well-known strings hash to the same slot. Change the table or the hash!"); - wks_idx = hdrtoken_tokenize_dfa(_hdrtoken_strs_field_initializers[i].name, - static_cast(strlen(_hdrtoken_strs_field_initializers[i].name))); +constexpr std::array +hdrtoken_build_wks_table() +{ + std::array table{}; + + for (size_t i = 0; i < std::size(_hdrtoken_strs); ++i) { + HdrTokenWksEntry &e = table[i]; + std::string_view const name = _hdrtoken_strs[i]; + + for (size_t k = 0; k < name.size(); ++k) { + e.str[k] = name[k]; + } + e.prefix.wks_idx = static_cast(i); + e.prefix.wks_length = static_cast(name.size()); + e.prefix.wks_token_type = HdrTokenType::OTHER; + e.prefix.wks_info.slotid = MIME_SLOTID_NONE; + e.prefix.wks_info.mask = TOK_64_CONST(0); + e.prefix.wks_info.flags = HdrTokenInfoFlags::MULTVALS; + } - ink_assert((wks_idx >= 0) && (wks_idx < (int)SIZEOF(hdrtoken_strs))); - prefix = hdrtoken_index_to_prefix(wks_idx); - prefix->wks_info.slotid = _hdrtoken_strs_field_initializers[i].slotid; - prefix->wks_info.flags = _hdrtoken_strs_field_initializers[i].flags; - prefix->wks_info.mask = _hdrtoken_strs_field_initializers[i].mask; + for (auto const &b : _hdrtoken_strs_type_initializers) { + if (b.name != nullptr) { + table[hdrtoken_index_of_literal(b.name)].prefix.wks_token_type = b.type; } + } + + for (auto const &f : _hdrtoken_strs_field_initializers) { + if (f.name != nullptr) { + HdrTokenFieldInfo &info = table[hdrtoken_index_of_literal(f.name)].prefix.wks_info; - for (i = 0; i < static_cast SIZEOF(_hdrtoken_strs); i++) { - HdrTokenHeapPrefix *prefix = hdrtoken_index_to_prefix(i); - prefix->wks_info.name = hdrtoken_strs[i]; - hdrtoken_str_token_types[i] = prefix->wks_token_type; // parallel array for speed - hdrtoken_str_slotids[i] = prefix->wks_info.slotid; // parallel array for speed - hdrtoken_str_masks[i] = prefix->wks_info.mask; // parallel array for speed - hdrtoken_str_flags[i] = prefix->wks_info.flags; // parallel array for speed + info.slotid = f.slotid; + info.mask = f.mask; + info.flags = f.flags; } + } - hdrtoken_hash_init(); + for (auto const &c : _hdrtoken_strs_cc_initializers) { + if (c.name != nullptr) { + table[hdrtoken_index_of_literal(c.name)].prefix.wks_type_specific.u.cache_control.cc_mask = c.mask; + } } + + return table; } -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ +constexpr std::array hdrtoken_wks_table = hdrtoken_build_wks_table(); -int -hdrtoken_tokenize_dfa(const char *string, int string_len, const char **wks_string_out) +/*********************************************************************** + * * + * H A S H T A B L E * + * * + ***********************************************************************/ + +struct HdrTokenHashBucket { + uint32_t wks_idx_plus_one; // biased by one so that a value-initialized bucket reads as empty + uint32_t hash; +}; + +constexpr std::array +hdrtoken_build_hash_table() { - int wks_idx; + std::array table{}; - wks_idx = hdrtoken_strs_dfa->match({string, static_cast(string_len)}); + // static_assert(hdrtoken_wks_slots_unique()) proves no two strings share a slot, so no bucket is + // assigned twice here. + for (size_t i = 0; i < std::size(_hdrtoken_strs); i++) { + uint32_t const hash = hdrtoken_hash(_hdrtoken_strs[i]); - if (wks_idx < 0) { - wks_idx = -1; + table[hash_to_slot(hash)] = {static_cast(i) + 1, hash}; } - if (wks_string_out) { - if (wks_idx >= 0) { - *wks_string_out = hdrtoken_index_to_wks(wks_idx); - } else { - *wks_string_out = nullptr; + return table; +} + +constexpr std::array hdrtoken_hash_table = hdrtoken_build_hash_table(); + +} // end anonymous namespace + +// hdrtoken_wks_to_prefix() maps a string pointer back to its entry through this table. +const HdrTokenWksEntry *const hdrtoken_wks_entries = hdrtoken_wks_table.data(); + +// Header string pointers in this range are well-known. +const char *_hdrtoken_strs_heap_f = &hdrtoken_wks_table[0].str[0]; // storage first byte +const char *_hdrtoken_strs_heap_l = &hdrtoken_wks_table[std::size(_hdrtoken_strs) - 1].str[HDRTOKEN_WKS_STORAGE - 1]; + +int hdrtoken_num_wks = std::size(_hdrtoken_strs); // # of well-known strings + +const char *hdrtoken_strs[std::size(_hdrtoken_strs)]; // wks_idx -> string +int hdrtoken_str_lengths[std::size(_hdrtoken_strs)]; // wks_idx -> length +HdrTokenType hdrtoken_str_token_types[std::size(_hdrtoken_strs)]; // wks_idx -> token type +int32_t hdrtoken_str_slotids[std::size(_hdrtoken_strs)]; // wks_idx -> slot id +uint64_t hdrtoken_str_masks[std::size(_hdrtoken_strs)]; // wks_idx -> presence mask +HdrTokenInfoFlags hdrtoken_str_flags[std::size(_hdrtoken_strs)]; // wks_idx -> flags + +/*********************************************************************** + * * + * M A I N H D R T O K E N C O D E * + * * + ***********************************************************************/ + +/** + */ +void +hdrtoken_init() +{ + static int inited = 0; + + if (!inited) { + inited = 1; + + // hdrtoken_wks_table already holds every string with its prefix, resolved at compile time. + // Copy the hot fields out into the parallel arrays. + for (int i = 0; i < static_cast(std::size(_hdrtoken_strs)); i++) { + HdrTokenHeapPrefix const &prefix = hdrtoken_wks_table[i].prefix; + + hdrtoken_strs[i] = hdrtoken_wks_table[i].str; + hdrtoken_str_lengths[i] = prefix.wks_length; + hdrtoken_str_token_types[i] = prefix.wks_token_type; + hdrtoken_str_slotids[i] = prefix.wks_info.slotid; + hdrtoken_str_masks[i] = prefix.wks_info.mask; + hdrtoken_str_flags[i] = prefix.wks_info.flags; } } - // printf("hdrtoken_tokenize_dfa(%d,*s) - return %d\n",string_len,string,wks_idx); - - return wks_idx; } /*------------------------------------------------------------------------- - Have to work around that methods are case insensitive while the DFA is - case insensitive. + Have to work around that methods are case sensitive while hdrtoken_tokenize() + is case insensitive. -------------------------------------------------------------------------*/ int @@ -534,29 +697,31 @@ hdrtoken_method_tokenize(const char *string, int string_len) int hdrtoken_tokenize(const char *string, int string_len, const char **wks_string_out) { - int wks_idx; - HdrTokenHashBucket *bucket; - ink_assert(string != nullptr); if (hdrtoken_is_wks(string)) { - wks_idx = hdrtoken_wks_to_index(string); + int const wks_idx = hdrtoken_wks_to_index(string); + if (wks_string_out) { *wks_string_out = string; } return wks_idx; } - uint32_t hash = hdrtoken_hash(reinterpret_cast(string), static_cast(string_len)); - uint32_t slot = hash_to_slot(hash); + uint32_t const hash = hdrtoken_hash(std::string_view{string, static_cast(string_len)}); - bucket = &(hdrtoken_hash_table[slot]); - if ((bucket->wks != nullptr) && (bucket->hash == hash) && (hdrtoken_wks_to_length(bucket->wks) == string_len)) { - wks_idx = hdrtoken_wks_to_index(bucket->wks); - if (wks_string_out) { - *wks_string_out = bucket->wks; + HdrTokenHashBucket const &bucket = hdrtoken_hash_table[hash_to_slot(hash)]; + + if ((bucket.wks_idx_plus_one != 0) && (bucket.hash == hash)) { + int const wks_idx = static_cast(bucket.wks_idx_plus_one - 1); + HdrTokenWksEntry const &entry = hdrtoken_wks_table[wks_idx]; + + if (entry.prefix.wks_length == string_len) { + if (wks_string_out) { + *wks_string_out = entry.str; + } + return wks_idx; } - return wks_idx; } Dbg(dbg_ctl_hdr_token, "Did not find a WKS for '%.*s'", string_len, string); diff --git a/src/proxy/hdrs/MIME.cc b/src/proxy/hdrs/MIME.cc index 9430385d42c..a05284d8868 100644 --- a/src/proxy/hdrs/MIME.cc +++ b/src/proxy/hdrs/MIME.cc @@ -833,37 +833,6 @@ mime_init() MIME_VALUE_H2C = hdrtoken_string_to_wks_sv(MIME_UPGRADE_H2C_TOKEN); mime_init_date_format_table(); - mime_init_cache_control_cooking_masks(); - } -} - -void -mime_init_cache_control_cooking_masks() -{ - static struct { - const char *name; - uint32_t mask; - } cc_mask_table[] = { - {"max-age", MIME_COOKED_MASK_CC_MAX_AGE }, - {"no-cache", MIME_COOKED_MASK_CC_NO_CACHE }, - {"no-store", MIME_COOKED_MASK_CC_NO_STORE }, - {"no-transform", MIME_COOKED_MASK_CC_NO_TRANSFORM }, - {"max-stale", MIME_COOKED_MASK_CC_MAX_STALE }, - {"min-fresh", MIME_COOKED_MASK_CC_MIN_FRESH }, - {"only-if-cached", MIME_COOKED_MASK_CC_ONLY_IF_CACHED }, - {"public", MIME_COOKED_MASK_CC_PUBLIC }, - {"private", MIME_COOKED_MASK_CC_PRIVATE }, - {"must-revalidate", MIME_COOKED_MASK_CC_MUST_REVALIDATE }, - {"proxy-revalidate", MIME_COOKED_MASK_CC_PROXY_REVALIDATE }, - {"s-maxage", MIME_COOKED_MASK_CC_S_MAXAGE }, - {"need-revalidate-once", MIME_COOKED_MASK_CC_NEED_REVALIDATE_ONCE}, - {nullptr, 0 } - }; - - for (int i = 0; cc_mask_table[i].name != nullptr; i++) { - const char *wks = hdrtoken_string_to_wks(cc_mask_table[i].name); - HdrTokenHeapPrefix *p = hdrtoken_wks_to_prefix(wks); - p->wks_type_specific.u.cache_control.cc_mask = cc_mask_table[i].mask; } } @@ -1209,8 +1178,8 @@ _mime_hdr_field_list_search_by_slotnum(MIMEHdrImpl *mh, int slotnum) MIMEField * mime_hdr_field_find(MIMEHdrImpl *mh, std::string_view field_name) { - HdrTokenHeapPrefix *token_info; - const bool is_wks = hdrtoken_is_wks(field_name.data()); + const HdrTokenHeapPrefix *token_info; + const bool is_wks = hdrtoken_is_wks(field_name.data()); ink_assert(!field_name.empty()); @@ -3760,7 +3729,7 @@ MIMEHdrImpl::recompute_cooked_stuff(MIMEField *changing_field_or_null, const std Dbg(dbg_ctl_http, "recompute_cooked_stuff: got field '%s'", token_wks); #endif - HdrTokenHeapPrefix *p = hdrtoken_wks_to_prefix(token_wks); + const HdrTokenHeapPrefix *p = hdrtoken_wks_to_prefix(token_wks); mask = p->wks_type_specific.u.cache_control.cc_mask; m_cooked_stuff.m_cache_control.m_mask |= mask; csv_value_mask |= mask;