From d3def1917d07300529947f42c58741b0565ffeef Mon Sep 17 00:00:00 2001 From: Phong Nguyen Date: Wed, 19 Aug 2026 18:19:50 +0000 Subject: [PATCH] Use __atomic builtins as the freelist 128-bit CAS fallback Platforms without an inline 128-bit CAS, such as riscv64, fail the build with "unsupported processor". Neither GCC nor LLVM emit an inline 128-bit CAS on riscv64, even with the Zacas extension, so a hand-written pointer-packing branch would be the only alternative and would depend on the kernel's virtual address width. Fall back to the __atomic builtins instead. They lower to libatomic calls, which may take internal locks; that is correct because every access to a shared head_p goes through INK_QUEUE_LD and ink_atomic_cas. Also revive the orphaned atomic list stress test as Catch2 tests and remove the dead INK_QUEUE_NT code. Fixes: #13555 Co-Authored-By: Claude Fable 5 --- cmake/Check128BitCas.cmake | 59 +++++- include/tscore/ink_atomic.h | 12 ++ include/tscore/ink_config.h.cmake.in | 1 + include/tscore/ink_queue.h | 15 +- src/traffic_layout/info.cc | 1 + src/tscore/CMakeLists.txt | 5 + src/tscore/test_atomic.cc | 218 ------------------- src/tscore/unit_tests/test_InkAtomicList.cc | 221 ++++++++++++++++++++ 8 files changed, 305 insertions(+), 227 deletions(-) delete mode 100644 src/tscore/test_atomic.cc create mode 100644 src/tscore/unit_tests/test_InkAtomicList.cc diff --git a/cmake/Check128BitCas.cmake b/cmake/Check128BitCas.cmake index 850bc398fe3..5c075eeff62 100644 --- a/cmake/Check128BitCas.cmake +++ b/cmake/Check128BitCas.cmake @@ -21,6 +21,16 @@ # # TS_HAS_128BIT_CAS # TS_NEEDS_MCX16_FOR_CAS +# TS_HAS_128BIT_CAS_LIBATOMIC +# TS_NEEDS_LIBATOMIC_FOR_CAS +# +# TS_HAS_128BIT_CAS means the 16-byte __sync builtins compile and link, which the +# compiler only allows when it can emit an inline lock-free sequence. +# +# TS_HAS_128BIT_CAS_LIBATOMIC is the fallback for targets with no inline 128-bit +# CAS (e.g. riscv64): the __atomic builtins lower to libatomic calls, which may +# be lock-based there. The __sync builtins never lower to libatomic calls, so +# the fallback has to use __atomic. The two are mutually exclusive. # set(CHECK_PROGRAM @@ -33,6 +43,22 @@ set(CHECK_PROGRAM " ) +set(CHECK_PROGRAM_ATOMIC + " + int main(void) + { + __int128_t x = 0; + __int128_t y = 0; + __atomic_load(&x, &y, __ATOMIC_SEQ_CST); + return !__atomic_compare_exchange_n(&x, &y, 10, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); + } + " +) + +set(NEED_MCX16 FALSE) +set(USE_LIBATOMIC_CAS FALSE) +set(NEED_LIBATOMIC FALSE) + include(CheckCSourceCompiles) check_c_source_compiles("${CHECK_PROGRAM}" TS_HAS_128BIT_CAS) @@ -44,12 +70,41 @@ if(NOT TS_HAS_128BIT_CAS) unset(CMAKE_REQUIRED_FLAGS) endif() +if(NOT TS_HAS_128BIT_CAS) + check_c_source_compiles("${CHECK_PROGRAM_ATOMIC}" TS_HAS_128BIT_CAS_BUILTIN_ATOMIC) + if(TS_HAS_128BIT_CAS_BUILTIN_ATOMIC) + set(USE_LIBATOMIC_CAS TRUE) + else() + unset(TS_HAS_128BIT_CAS_BUILTIN_ATOMIC CACHE) + set(CMAKE_REQUIRED_LIBRARIES atomic) + check_c_source_compiles("${CHECK_PROGRAM_ATOMIC}" TS_HAS_128BIT_CAS_BUILTIN_ATOMIC) + unset(CMAKE_REQUIRED_LIBRARIES) + if(TS_HAS_128BIT_CAS_BUILTIN_ATOMIC) + set(USE_LIBATOMIC_CAS TRUE) + set(NEED_LIBATOMIC TRUE) + endif() + endif() +endif() + set(TS_NEEDS_MCX16_FOR_CAS ${NEED_MCX16} CACHE BOOL "Whether -mcx16 is needed to compile CAS" ) +set(TS_HAS_128BIT_CAS_LIBATOMIC + ${USE_LIBATOMIC_CAS} + CACHE BOOL "Whether 128-bit CAS uses the __atomic builtins as a fallback" +) + +set(TS_NEEDS_LIBATOMIC_FOR_CAS + ${NEED_LIBATOMIC} + CACHE BOOL "Whether libatomic is needed to link CAS" +) + unset(CHECK_PROGRAM) -unset(NEEDS_MCX16) +unset(CHECK_PROGRAM_ATOMIC) +unset(NEED_MCX16) +unset(USE_LIBATOMIC_CAS) +unset(NEED_LIBATOMIC) -mark_as_advanced(TS_HAS_128BIT_CAS TS_NEEDS_MCX16_FOR_CAS) +mark_as_advanced(TS_HAS_128BIT_CAS TS_NEEDS_MCX16_FOR_CAS TS_HAS_128BIT_CAS_LIBATOMIC TS_NEEDS_LIBATOMIC_FOR_CAS) diff --git a/include/tscore/ink_atomic.h b/include/tscore/ink_atomic.h index 5b4d4a01f22..e5a5fdaf1d3 100644 --- a/include/tscore/ink_atomic.h +++ b/include/tscore/ink_atomic.h @@ -69,6 +69,18 @@ ink_atomic_cas(T *mem, T prev, T next) return __sync_bool_compare_and_swap(mem, prev, next); } +#if TS_HAS_128BIT_CAS_LIBATOMIC && !TS_HAS_128BIT_CAS +// The 16-byte __sync builtins never lower to libatomic calls, so targets with +// no inline 128-bit CAS (e.g. riscv64) must use the __atomic builtins instead. +// libatomic may implement them with internal locks; see INK_QUEUE_LD. +template <> +inline bool +ink_atomic_cas<__int128_t>(__int128_t *mem, __int128_t prev, __int128_t next) +{ + return __atomic_compare_exchange_n(mem, &prev, next, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); +} +#endif + // ink_atomic_increment(ptr, count) // Increment @ptr by @count, returning the previous value. template diff --git a/include/tscore/ink_config.h.cmake.in b/include/tscore/ink_config.h.cmake.in index fe19d38e484..dc77e7ef0ba 100644 --- a/include/tscore/ink_config.h.cmake.in +++ b/include/tscore/ink_config.h.cmake.in @@ -139,6 +139,7 @@ const int DEFAULT_STACKSIZE = @DEFAULT_STACK_SIZE@; /* Feature Flags */ #cmakedefine01 TS_HAS_128BIT_CAS +#cmakedefine01 TS_HAS_128BIT_CAS_LIBATOMIC #cmakedefine01 TS_HAS_BACKTRACE #cmakedefine01 TS_HAS_IN6_IS_ADDR_UNSPECIFIED #cmakedefine01 TS_HAS_IP_TOS diff --git a/include/tscore/ink_queue.h b/include/tscore/ink_queue.h index a0492213df3..9cbe6c757d1 100644 --- a/include/tscore/ink_queue.h +++ b/include/tscore/ink_queue.h @@ -64,6 +64,12 @@ void ink_queue_load_64(void *dst, void *src); const volatile __int128_t iqld0 = 0; \ *(__int128_t *)&(dst) = __sync_val_compare_and_swap((__int128_t *)&(src), 0, iqld0); \ } while (0) +#elif TS_HAS_128BIT_CAS_LIBATOMIC +// On targets with no inline 128-bit CAS (e.g. riscv64) libatomic may implement +// the 16-byte __atomic builtins with internal locks. That is only correct +// because every access to a shared head_p goes through INK_QUEUE_LD and +// ink_atomic_cas, so all of them serialize on the same libatomic lock. +#define INK_QUEUE_LD(dst, src) __atomic_load((__int128_t *)&(src), (__int128_t *)&(dst), __ATOMIC_SEQ_CST) #else #define INK_QUEUE_LD(dst, src) INK_QUEUE_LD64(dst, src) #endif @@ -79,7 +85,7 @@ union head_p { #if (defined(__i386__) || defined(__arm__) || defined(__mips__)) && (SIZEOF_VOIDP == 4) typedef int32_t version_type; typedef int64_t data_type; -#elif TS_HAS_128BIT_CAS +#elif TS_HAS_128BIT_CAS || TS_HAS_128BIT_CAS_LIBATOMIC typedef int64_t version_type; typedef __int128_t data_type; #else @@ -124,7 +130,7 @@ union head_p { #define SET_FREELIST_POINTER_VERSION(_x, _p, _v) \ (_x).s.pointer = _p; \ (_x).s.version = _v -#elif TS_HAS_128BIT_CAS +#elif TS_HAS_128BIT_CAS || TS_HAS_128BIT_CAS_LIBATOMIC #define FREELIST_POINTER(_x) (_x).s.pointer #define FREELIST_VERSION(_x) (_x).s.version #define SET_FREELIST_POINTER_VERSION(_x, _p, _v) \ @@ -218,12 +224,7 @@ struct InkAtomicList { uint32_t offset = 0; }; -#if !defined(INK_QUEUE_NT) #define INK_ATOMICLIST_EMPTY(_x) (!(TO_PTR(FREELIST_POINTER((_x.head))))) -#else -/* ink_queue_nt.c doesn't do the FROM/TO pointer swizzling */ -#define INK_ATOMICLIST_EMPTY(_x) (!((FREELIST_POINTER((_x.head))))) -#endif // WARNING: the "name" string is not copied, it has to be a statically-stored constant string. // diff --git a/src/traffic_layout/info.cc b/src/traffic_layout/info.cc index 91b0677e042..1a84405139b 100644 --- a/src/traffic_layout/info.cc +++ b/src/traffic_layout/info.cc @@ -169,6 +169,7 @@ produce_features(bool json) print_feature("SIZEOF_VOIDP", SIZEOF_VOIDP, json); print_feature("TS_IP_TRANSPARENT", TS_IP_TRANSPARENT, json); print_feature("TS_HAS_128BIT_CAS", TS_HAS_128BIT_CAS, json); + print_feature("TS_HAS_128BIT_CAS_LIBATOMIC", TS_HAS_128BIT_CAS_LIBATOMIC, json); print_feature("TS_HAS_TESTS", TS_HAS_TESTS, json); print_feature("TS_MAX_THREADS_IN_EACH_THREAD_TYPE", TS_MAX_THREADS_IN_EACH_THREAD_TYPE, json); print_feature("TS_MAX_NUMBER_EVENT_THREADS", TS_MAX_NUMBER_EVENT_THREADS, json); diff --git a/src/tscore/CMakeLists.txt b/src/tscore/CMakeLists.txt index f951dec2bb4..4e3dbee89ab 100644 --- a/src/tscore/CMakeLists.txt +++ b/src/tscore/CMakeLists.txt @@ -127,6 +127,10 @@ if(TS_HAS_128BIT_CAS AND TS_NEEDS_MCX16_FOR_CAS) target_compile_options(tscore PUBLIC "-mcx16") endif() +if(TS_NEEDS_LIBATOMIC_FOR_CAS) + target_link_libraries(tscore PUBLIC atomic) +endif() + if(BUILD_SHARED_LIBS) install( TARGETS tscore @@ -146,6 +150,7 @@ if(BUILD_TESTING) unit_tests/test_HKDF.cc unit_tests/test_Histogram.cc unit_tests/test_History.cc + unit_tests/test_InkAtomicList.cc unit_tests/test_IntrusivePtr.cc unit_tests/test_List.cc unit_tests/test_MMH.cc diff --git a/src/tscore/test_atomic.cc b/src/tscore/test_atomic.cc deleted file mode 100644 index ce02c7b1d13..00000000000 --- a/src/tscore/test_atomic.cc +++ /dev/null @@ -1,218 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "tscore/ink_atomic.h" -#include "tscore/ink_queue.h" -#include "tscore/ink_thread.h" - -#ifndef LONG_ATOMICLIST_TEST - -#define MAX_ALIST_TEST 10 -#define MAX_ALIST_ARRAY 100000 -InkAtomicList al[MAX_ALIST_TEST]; -void *al_test[MAX_ALIST_TEST][MAX_ALIST_ARRAY]; -int al_done = 0; - -void * -testalist(void *ame) -{ - int me = static_cast((uintptr_t)ame); - int j, k; - for (k = 0; k < MAX_ALIST_ARRAY; k++) { - ink_atomiclist_push(&al[k % MAX_ALIST_TEST], &al_test[me][k]); - } - void *x; - for (j = 0; j < 1000000; j++) { - if ((x = ink_atomiclist_pop(&al[me]))) { - ink_atomiclist_push(&al[rand() % MAX_ALIST_TEST], x); - } - } - ink_atomic_increment(&al_done, 1); - return nullptr; -} -#endif // !LONG_ATOMICLIST_TEST - -#ifdef LONG_ATOMICLIST_TEST -/************************************************************************/ -#define MAX_ATOMIC_LISTS (4 * 1024) -#define MAX_ITEMS_PER_LIST (1 * 1024) -#define MAX_TEST_THREADS 64 -static InkAtomicList alists[MAX_ATOMIC_LISTS]; -struct listItem *items[MAX_ATOMIC_LISTS * MAX_ITEMS_PER_LIST]; - -struct listItem { - int data1; - int data2; - void *link; - int data3; - int data4; - int check; -}; - -void -init_data() -{ - int j; - int ali; - struct listItem l; - struct listItem *plistItem; - - for (ali = 0; ali < MAX_ATOMIC_LISTS; ali++) - ink_atomiclist_init(&alists[ali], "alist", ((char *)&l.link - (char *)&l)); - - for (ali = 0; ali < MAX_ATOMIC_LISTS; ali++) { - for (j = 0; j < MAX_ITEMS_PER_LIST; j++) { - plistItem = (struct listItem *)malloc(sizeof(struct listItem)); - items[ali + j] = plistItem; - plistItem->data1 = ali + j; - plistItem->data2 = ali + rand(); - plistItem->link = 0; - plistItem->data3 = j + rand(); - plistItem->data4 = ali + j + rand(); - plistItem->check = (plistItem->data1 ^ plistItem->data2 ^ plistItem->data3 ^ plistItem->data4); - ink_atomiclist_push(&alists[ali], plistItem); - } - } -} - -void -cycle_data(void *d) -{ - InkAtomicList *l; - struct listItem *pli; - struct listItem *pli_next; - int iterations; - int me; - - me = (int)d; - iterations = 0; - - while (1) { - l = &alists[(me + rand()) % MAX_ATOMIC_LISTS]; - - pli = (struct listItem *)ink_atomiclist_popall(l); - if (!pli) - continue; - - // Place listItems into random queues - while (pli) { - ink_assert((pli->data1 ^ pli->data2 ^ pli->data3 ^ pli->data4) == pli->check); - pli_next = (struct listItem *)pli->link; - pli->link = 0; - ink_atomiclist_push(&alists[(me + rand()) % MAX_ATOMIC_LISTS], (void *)pli); - pli = pli_next; - } - iterations++; - poll(0, 0, 10); // 10 msec delay - if ((iterations % 100) == 0) - printf("%d ", me); - } -} - -/************************************************************************/ -#endif // LONG_ATOMICLIST_TEST - -int -main(int /* argc ATS_UNUSED */, const char * /* argv ATS_UNUSED */[]) -{ -#ifndef LONG_ATOMICLIST_TEST - int32_t m = 1, n = 100; - // int64 lm = 1LL, ln = 100LL; - const char *m2 = "hello"; - char *n2; - - printf("sizeof(int32_t)==%d sizeof(void *)==%d\n", static_cast(sizeof(int32_t)), static_cast(sizeof(void *))); - - printf("CAS: %d == 1 then 2\n", m); - n = ink_atomic_cas(&m, 1, 2); - printf("changed to: %d, result=%s\n", m, n ? "true" : "false"); - - printf("CAS: %d == 1 then 3\n", m); - n = ink_atomic_cas(&m, 1, 3); - printf("changed to: %d, result=%s\n", m, n ? "true" : "false"); - - printf("CAS pointer: '%s' == 'hello' then 'new'\n", m2); - n = ink_atomic_cas(&m2, "hello", "new"); - printf("changed to: %s, result=%s\n", m2, n ? (char *)"true" : (char *)"false"); - - printf("CAS pointer: '%s' == 'hello' then 'new2'\n", m2); - n = ink_atomic_cas(&m2, m2, "new2"); - printf("changed to: %s, result=%s\n", m2, n ? "true" : "false"); - - n = 100; - printf("Atomic Inc of %d\n", n); - m = ink_atomic_increment(static_cast(&n), 1); - printf("changed to: %d, result=%d\n", n, m); - - printf("Atomic Fetch-and-Add 2 to pointer to '%s'\n", m2); - n2 = static_cast(ink_atomic_increment((void **)&m2, (void *)2)); - printf("changed to: %s, result=%s\n", m2, n2); - - printf("Testing atomic lists\n"); - { - int ali; - srand(time(nullptr)); - printf("sizeof(al_test) = %d\n", static_cast(sizeof(al_test))); - memset(&al_test[0][0], 0, sizeof(al_test)); - for (ali = 0; ali < MAX_ALIST_TEST; ali++) { - ink_atomiclist_init(&al[ali], "foo", 0); - } - for (ali = 0; ali < MAX_ALIST_TEST; ali++) { - ink_thread tid; - pthread_attr_t attr; - - pthread_attr_init(&attr); -#if !defined(freebsd) - pthread_attr_setstacksize(&attr, 1024 * 1024); -#endif - ink_assert(pthread_create(&tid, &attr, testalist, (void *)((intptr_t)ali)) == 0); - } - while (al_done != MAX_ALIST_TEST) { - sleep(1); - } - } -#endif // !LONG_ATOMICLIST_TEST - -#ifdef LONG_ATOMICLIST_TEST - printf("Testing atomic lists (long version)\n"); - { - int id; - - init_data(); - for (id = 0; id < MAX_TEST_THREADS; id++) { - ink_assert(thr_create(NULL, 0, cycle_data, (void *)id, THR_NEW_LWP, NULL) == 0); - } - } - while (1) { - poll(0, 0, 10); // 10 msec delay - } -#endif // LONG_ATOMICLIST_TEST - - return 0; -} diff --git a/src/tscore/unit_tests/test_InkAtomicList.cc b/src/tscore/unit_tests/test_InkAtomicList.cc new file mode 100644 index 00000000000..b7ba97fd74e --- /dev/null +++ b/src/tscore/unit_tests/test_InkAtomicList.cc @@ -0,0 +1,221 @@ +/** @file + + Concurrency stress tests for InkAtomicList and InkFreeList. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include + +#include +#include +#include +#include +#include + +#include "tscore/ink_queue.h" + +namespace +{ + +// Deterministic per-thread PRNG so failures are reproducible. +struct XorShift { + uint64_t state; + + explicit XorShift(uint64_t seed) : state(seed | 1) {} + + uint32_t + next() + { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + return static_cast(state >> 32); + } +}; + +struct Item { + Item *next = nullptr; + std::atomic in_hand{0}; + uint32_t owner = 0; + uint32_t serial = 0; + uint32_t check = 0; +}; + +constexpr int NUM_LISTS = 8; +constexpr int NUM_THREADS = 6; +constexpr int ITEMS_PER_THREAD = 2000; +constexpr int OPS_PER_THREAD = 100000; + +// Claim exclusive ownership of a popped item. A second concurrent claim means +// the same item was reachable from two places, i.e. the list is corrupt. +bool +claim(Item *item) +{ + return item->in_hand.exchange(1, std::memory_order_acq_rel) == 0; +} + +void +release(Item *item) +{ + item->in_hand.store(0, std::memory_order_release); +} + +} // end anonymous namespace + +TEST_CASE("InkAtomicList: concurrent push/pop/popall conserves items", "[libts][InkAtomicList]") +{ + InkAtomicList lists[NUM_LISTS]; + std::vector items(static_cast(NUM_THREADS) * ITEMS_PER_THREAD); + + for (int i = 0; i < NUM_LISTS; i++) { + ink_atomiclist_init(&lists[i], "test_InkAtomicList", offsetof(Item, next)); + } + + std::atomic claim_failures{0}; + std::atomic check_failures{0}; + + auto worker = [&](int me) { + XorShift rng(0x9e3779b97f4a7c15ull * (me + 1)); + + for (int k = 0; k < ITEMS_PER_THREAD; k++) { + Item *item = &items[static_cast(me) * ITEMS_PER_THREAD + k]; + item->owner = me; + item->serial = k; + item->check = item->owner ^ item->serial ^ 0xdeadbeef; + ink_atomiclist_push(&lists[k % NUM_LISTS], item); + } + + for (int op = 0; op < OPS_PER_THREAD; op++) { + InkAtomicList *l = &lists[rng.next() % NUM_LISTS]; + + if ((op & 1023) == 0) { + // Drain a whole list and scatter it back. + Item *chain = static_cast(ink_atomiclist_popall(l)); + while (chain != nullptr) { + Item *next_item = chain->next; + if (!claim(chain)) { + claim_failures++; + } + if (chain->check != (chain->owner ^ chain->serial ^ 0xdeadbeef)) { + check_failures++; + } + release(chain); + ink_atomiclist_push(&lists[rng.next() % NUM_LISTS], chain); + chain = next_item; + } + } else { + Item *item = static_cast(ink_atomiclist_pop(l)); + if (item == nullptr) { + continue; + } + if (!claim(item)) { + claim_failures++; + } + if (item->check != (item->owner ^ item->serial ^ 0xdeadbeef)) { + check_failures++; + } + release(item); + ink_atomiclist_push(&lists[rng.next() % NUM_LISTS], item); + } + } + }; + + std::vector threads; + for (int t = 0; t < NUM_THREADS; t++) { + threads.emplace_back(worker, t); + } + for (auto &t : threads) { + t.join(); + } + + REQUIRE(claim_failures == 0); + REQUIRE(check_failures == 0); + + // Every item must be reachable exactly once across all lists. + size_t drained = 0; + for (int i = 0; i < NUM_LISTS; i++) { + Item *chain = static_cast(ink_atomiclist_popall(&lists[i])); + while (chain != nullptr) { + REQUIRE(claim(chain)); + REQUIRE(chain->check == (chain->owner ^ chain->serial ^ 0xdeadbeef)); + drained++; + chain = chain->next; + } + } + REQUIRE(drained == items.size()); +} + +TEST_CASE("InkAtomicList: remove", "[libts][InkAtomicList]") +{ + InkAtomicList l; + Item items[3]; + + ink_atomiclist_init(&l, "test_InkAtomicList_remove", offsetof(Item, next)); + for (auto &item : items) { + ink_atomiclist_push(&l, &item); + } + + // Remove from the middle, the head, then a missing item. + REQUIRE(ink_atomiclist_remove(&l, &items[1]) == &items[1]); + REQUIRE(ink_atomiclist_remove(&l, &items[2]) == &items[2]); + REQUIRE(ink_atomiclist_remove(&l, &items[1]) == nullptr); + REQUIRE(ink_atomiclist_pop(&l) == &items[0]); + REQUIRE(INK_ATOMICLIST_EMPTY(l)); +} + +TEST_CASE("InkFreeList: concurrent new/free", "[libts][InkFreeList]") +{ + constexpr int SLOTS = 32; + constexpr int FL_OPS = 50000; + constexpr uint32_t OBJ_SIZE = 128; + + InkFreeList *f = ink_freelist_create("test_InkFreeList", OBJ_SIZE, 64, 8); + + auto worker = [&](int me) { + XorShift rng(0xc2b2ae3d27d4eb4full * (me + 1)); + void *slots[SLOTS] = {nullptr}; + + for (int op = 0; op < FL_OPS; op++) { + int i = rng.next() % SLOTS; + if (slots[i] != nullptr) { + ink_freelist_free(f, slots[i]); + slots[i] = nullptr; + } else { + slots[i] = ink_freelist_new(f); + memset(slots[i], me, OBJ_SIZE); + } + } + for (auto &slot : slots) { + if (slot != nullptr) { + ink_freelist_free(f, slot); + } + } + }; + + std::vector threads; + for (int t = 0; t < NUM_THREADS; t++) { + threads.emplace_back(worker, t); + } + for (auto &t : threads) { + t.join(); + } + + REQUIRE(f->used == 0); +}