Use __atomic builtins as the freelist 128-bit CAS fallback

cmake/Check128BitCas.cmake

@@ -21,6 +21,16 @@
 #
 #     TS_HAS_128BIT_CAS
 #     TS_NEEDS_MCX16_FOR_CAS
+#     TS_HAS_128BIT_CAS_LIBATOMIC
+#     TS_NEEDS_LIBATOMIC_FOR_CAS
+#
+# TS_HAS_128BIT_CAS means the 16-byte __sync builtins compile and link, which the
+# compiler only allows when it can emit an inline lock-free sequence.
+#
+# TS_HAS_128BIT_CAS_LIBATOMIC is the fallback for targets with no inline 128-bit
+# CAS (e.g. riscv64): the __atomic builtins lower to libatomic calls, which may
+# be lock-based there.  The __sync builtins never lower to libatomic calls, so
+# the fallback has to use __atomic.  The two are mutually exclusive.
 #
 
 set(CHECK_PROGRAM
@@ -33,6 +43,22 @@ set(CHECK_PROGRAM
     "
 )
 
+set(CHECK_PROGRAM_ATOMIC
+    "
+    int main(void)
+    {
+        __int128_t x = 0;
+        __int128_t y = 0;
+        __atomic_load(&x, &y, __ATOMIC_SEQ_CST);
+        return !__atomic_compare_exchange_n(&x, &y, 10, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
+    }
+    "
+)
+
+set(NEED_MCX16 FALSE)
+set(USE_LIBATOMIC_CAS FALSE)
+set(NEED_LIBATOMIC FALSE)
+
 include(CheckCSourceCompiles)
 check_c_source_compiles("${CHECK_PROGRAM}" TS_HAS_128BIT_CAS)
 
@@ -44,12 +70,41 @@ if(NOT TS_HAS_128BIT_CAS)
   unset(CMAKE_REQUIRED_FLAGS)
 endif()
 
+if(NOT TS_HAS_128BIT_CAS)
+  check_c_source_compiles("${CHECK_PROGRAM_ATOMIC}" TS_HAS_128BIT_CAS_BUILTIN_ATOMIC)
+  if(TS_HAS_128BIT_CAS_BUILTIN_ATOMIC)
+    set(USE_LIBATOMIC_CAS TRUE)
+  else()
+    unset(TS_HAS_128BIT_CAS_BUILTIN_ATOMIC CACHE)
+    set(CMAKE_REQUIRED_LIBRARIES atomic)
+    check_c_source_compiles("${CHECK_PROGRAM_ATOMIC}" TS_HAS_128BIT_CAS_BUILTIN_ATOMIC)
+    unset(CMAKE_REQUIRED_LIBRARIES)
+    if(TS_HAS_128BIT_CAS_BUILTIN_ATOMIC)
+      set(USE_LIBATOMIC_CAS TRUE)
+      set(NEED_LIBATOMIC TRUE)
+    endif()
+  endif()
+endif()
+
 set(TS_NEEDS_MCX16_FOR_CAS
     ${NEED_MCX16}
     CACHE BOOL "Whether -mcx16 is needed to compile CAS"
 )
 
+set(TS_HAS_128BIT_CAS_LIBATOMIC
+    ${USE_LIBATOMIC_CAS}
+    CACHE BOOL "Whether 128-bit CAS uses the __atomic builtins as a fallback"
+)
+
+set(TS_NEEDS_LIBATOMIC_FOR_CAS
+    ${NEED_LIBATOMIC}
+    CACHE BOOL "Whether libatomic is needed to link CAS"
+)
+
 unset(CHECK_PROGRAM)
-unset(NEEDS_MCX16)
+unset(CHECK_PROGRAM_ATOMIC)
+unset(NEED_MCX16)
+unset(USE_LIBATOMIC_CAS)
+unset(NEED_LIBATOMIC)
 
-mark_as_advanced(TS_HAS_128BIT_CAS TS_NEEDS_MCX16_FOR_CAS)
+mark_as_advanced(TS_HAS_128BIT_CAS TS_NEEDS_MCX16_FOR_CAS TS_HAS_128BIT_CAS_LIBATOMIC TS_NEEDS_LIBATOMIC_FOR_CAS)

include/tscore/ink_atomic.h

@@ -69,6 +69,18 @@ ink_atomic_cas(T *mem, T prev, T next)
   return __sync_bool_compare_and_swap(mem, prev, next);
 }
 
+#if TS_HAS_128BIT_CAS_LIBATOMIC && !TS_HAS_128BIT_CAS
+// The 16-byte __sync builtins never lower to libatomic calls, so targets with
+// no inline 128-bit CAS (e.g. riscv64) must use the __atomic builtins instead.
+// libatomic may implement them with internal locks; see INK_QUEUE_LD.
+template <>
+inline bool
+ink_atomic_cas<__int128_t>(__int128_t *mem, __int128_t prev, __int128_t next)
+{
+  return __atomic_compare_exchange_n(mem, &prev, next, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
+}
+#endif
+
 // ink_atomic_increment(ptr, count)
 // Increment @ptr by @count, returning the previous value.
 template <typename Type, typename Amount>

include/tscore/ink_config.h.cmake.in

@@ -139,6 +139,7 @@ const int DEFAULT_STACKSIZE = @DEFAULT_STACK_SIZE@;
 
 /* Feature Flags */
 #cmakedefine01 TS_HAS_128BIT_CAS
+#cmakedefine01 TS_HAS_128BIT_CAS_LIBATOMIC
 #cmakedefine01 TS_HAS_BACKTRACE
 #cmakedefine01 TS_HAS_IN6_IS_ADDR_UNSPECIFIED
 #cmakedefine01 TS_HAS_IP_TOS

include/tscore/ink_queue.h

@@ -64,6 +64,12 @@ void ink_queue_load_64(void *dst, void *src);
     const volatile __int128_t iqld0 = 0;                                                           \
     *(__int128_t *)&(dst)           = __sync_val_compare_and_swap((__int128_t *)&(src), 0, iqld0); \
   } while (0)
+#elif TS_HAS_128BIT_CAS_LIBATOMIC
+// On targets with no inline 128-bit CAS (e.g. riscv64) libatomic may implement
+// the 16-byte __atomic builtins with internal locks. That is only correct
+// because every access to a shared head_p goes through INK_QUEUE_LD and
+// ink_atomic_cas, so all of them serialize on the same libatomic lock.
+#define INK_QUEUE_LD(dst, src) __atomic_load((__int128_t *)&(src), (__int128_t *)&(dst), __ATOMIC_SEQ_CST)
 #else
 #define INK_QUEUE_LD(dst, src) INK_QUEUE_LD64(dst, src)
 #endif
@@ -79,7 +85,7 @@ union head_p {
 #if (defined(__i386__) || defined(__arm__) || defined(__mips__)) && (SIZEOF_VOIDP == 4)
   typedef int32_t version_type;
   typedef int64_t data_type;
-#elif TS_HAS_128BIT_CAS
+#elif TS_HAS_128BIT_CAS || TS_HAS_128BIT_CAS_LIBATOMIC
   typedef int64_t    version_type;
   typedef __int128_t data_type;
 #else
@@ -124,7 +130,7 @@ union head_p {
 #define SET_FREELIST_POINTER_VERSION(_x, _p, _v) \
   (_x).s.pointer = _p;                           \
   (_x).s.version = _v
-#elif TS_HAS_128BIT_CAS
+#elif TS_HAS_128BIT_CAS || TS_HAS_128BIT_CAS_LIBATOMIC
 #define FREELIST_POINTER(_x) (_x).s.pointer
 #define FREELIST_VERSION(_x) (_x).s.version
 #define SET_FREELIST_POINTER_VERSION(_x, _p, _v) \
@@ -218,12 +224,7 @@ struct InkAtomicList {
   uint32_t    offset = 0;
 };
 
-#if !defined(INK_QUEUE_NT)
 #define INK_ATOMICLIST_EMPTY(_x) (!(TO_PTR(FREELIST_POINTER((_x.head)))))
-#else
-/* ink_queue_nt.c doesn't do the FROM/TO pointer swizzling */
-#define INK_ATOMICLIST_EMPTY(_x) (!((FREELIST_POINTER((_x.head)))))
-#endif
 
 // WARNING: the "name" string is not copied, it has to be a statically-stored constant string.
 //

src/traffic_layout/info.cc

@@ -169,6 +169,7 @@ produce_features(bool json)
   print_feature("SIZEOF_VOIDP", SIZEOF_VOIDP, json);
   print_feature("TS_IP_TRANSPARENT", TS_IP_TRANSPARENT, json);
   print_feature("TS_HAS_128BIT_CAS", TS_HAS_128BIT_CAS, json);
+  print_feature("TS_HAS_128BIT_CAS_LIBATOMIC", TS_HAS_128BIT_CAS_LIBATOMIC, json);
   print_feature("TS_HAS_TESTS", TS_HAS_TESTS, json);
   print_feature("TS_MAX_THREADS_IN_EACH_THREAD_TYPE", TS_MAX_THREADS_IN_EACH_THREAD_TYPE, json);
   print_feature("TS_MAX_NUMBER_EVENT_THREADS", TS_MAX_NUMBER_EVENT_THREADS, json);

src/tscore/CMakeLists.txt

@@ -127,6 +127,10 @@ if(TS_HAS_128BIT_CAS AND TS_NEEDS_MCX16_FOR_CAS)
   target_compile_options(tscore PUBLIC "-mcx16")
 endif()
 
+if(TS_NEEDS_LIBATOMIC_FOR_CAS)
+  target_link_libraries(tscore PUBLIC atomic)
+endif()
+
 if(BUILD_SHARED_LIBS)
   install(
     TARGETS tscore
@@ -146,6 +150,7 @@ if(BUILD_TESTING)
     unit_tests/test_HKDF.cc
     unit_tests/test_Histogram.cc
     unit_tests/test_History.cc
+    unit_tests/test_InkAtomicList.cc
     unit_tests/test_IntrusivePtr.cc
     unit_tests/test_List.cc
     unit_tests/test_MMH.cc