From af14d1d34a170daa721f7058d9e05026808b6f28 Mon Sep 17 00:00:00 2001 From: Zheming Jin Date: Sat, 18 Jul 2026 03:47:18 -0700 Subject: [PATCH] [libclc][amdgpu] Implement __spirv_GroupNonUniform*Shuffle* builtins The amdgcn libspirv device library declares the SPIR-V group non-uniform shuffle builtins (__spirv_GroupNonUniformShuffle{,Up,Down,Xor}) via libspirv/group/group_shuffle*.h but never defines them for the amdgpu target. The SYCL headers (sycl/detail/spirv.hpp) emit these directly for sub-group shuffle/permute/scan, and oneDPL work-group algorithms (inclusive_scan, sort, ...) rely on them. Device linking for amdgcn-amd-amdhsa therefore fails with e.g. undefined symbol: __spirv_GroupNonUniformShuffleUp Unlike SPIR-V targets there is no runtime translation of these instructions, so the device library must provide them. Add group/group_shuffle.cl for the amdgpu target, mapping the four non-uniform shuffle operations onto the existing, validated __spirv_SubgroupShuffle*INTEL primitives (misc/sub_group_shuffle.cl), which lower to the hardware ds_bpermute wavefront shuffle. These primitives derive the wavefront size from __spirv_BuiltInSubgroupMaxSize() rather than hardcoding it and use only ds_bpermute, so they are correct across all wave64 CDNA generations (CDNA1 gfx908, CDNA2 gfx90a, CDNA3 gfx942, CDNA4 gfx950) with no arch-specific code. Only scalar overloads are defined: the SYCL headers scalarize all vector/marray shuffles and lower bitcast/generic shuffles onto integer scalars before reaching the intrinsic, matching the scalar-only __spirv_GroupBroadcast definitions in group/collectives.cl. Co-authored-by: Cursor --- libclc/libspirv/lib/amdgpu/CMakeLists.txt | 1 + .../lib/amdgpu/group/group_shuffle.cl | 70 +++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 libclc/libspirv/lib/amdgpu/group/group_shuffle.cl diff --git a/libclc/libspirv/lib/amdgpu/CMakeLists.txt b/libclc/libspirv/lib/amdgpu/CMakeLists.txt index e032a204ddf2f..f02e8c4fae535 100644 --- a/libclc/libspirv/lib/amdgpu/CMakeLists.txt +++ b/libclc/libspirv/lib/amdgpu/CMakeLists.txt @@ -7,6 +7,7 @@ libclc_add_sources(${LIBCLC_LIBSPIRV_TARGET} FILES group/group_ballot.cl group/collectives.cl group/collectives_helpers.cl + group/group_shuffle.cl conversion/GenericCastToPtrExplicit.cl synchronization/barrier.cl images/image_common.cl diff --git a/libclc/libspirv/lib/amdgpu/group/group_shuffle.cl b/libclc/libspirv/lib/amdgpu/group/group_shuffle.cl new file mode 100644 index 0000000000000..3228a23569030 --- /dev/null +++ b/libclc/libspirv/lib/amdgpu/group/group_shuffle.cl @@ -0,0 +1,70 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include + +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_fp64 : enable + +// The __spirv_GroupNonUniform*Shuffle* builtins are emitted directly by the +// SYCL headers (see sycl/detail/spirv.hpp) for sub-group shuffle, permute and +// scan operations, and are relied upon by oneDPL work-group algorithms. Unlike +// SPIR-V targets, the amdgcn target has no runtime translation for these +// instructions, so they must be provided by the device library. +// +// The SYCL headers scalarize every shuffle before reaching the intrinsic +// (vectors and marrays are handled element-wise, and bitcast/generic shuffles +// are lowered onto integer scalars), so only scalar overloads are required +// here - mirroring the scalar-only __spirv_GroupBroadcast definitions in +// group/collectives.cl. +// +// Each operation maps directly onto the corresponding, already validated, +// __spirv_SubgroupShuffle*INTEL primitive (see misc/sub_group_shuffle.cl), +// which lowers to the hardware ds_bpermute wavefront shuffle. The Shuffle Up +// and Down primitives take a pair of (previous/current) or (current/next) +// operands; for the single-operand SPIR-V form we pass the same value for both, +// which is correct within a single sub-group and preserves the SPIR-V contract +// that out-of-range indices produce an undefined result. +// The scope operand is unused: the delegated __spirv_SubgroupShuffle*INTEL +// primitives operate at sub-group (wavefront) granularity, matching the +// Subgroup path of __spirv_GroupBroadcast in group/collectives.cl. +#define __CLC_GROUP_NON_UNIFORM_SHUFFLE(TYPE) \ + _CLC_DEF _CLC_OVERLOAD _CLC_CONVERGENT TYPE __spirv_GroupNonUniformShuffle( \ + int scope, TYPE value, uint id) { \ + (void)scope; \ + return __spirv_SubgroupShuffleINTEL(value, id); \ + } \ + _CLC_DEF _CLC_OVERLOAD _CLC_CONVERGENT TYPE \ + __spirv_GroupNonUniformShuffleXor(int scope, TYPE value, uint mask) { \ + (void)scope; \ + return __spirv_SubgroupShuffleXorINTEL(value, mask); \ + } \ + _CLC_DEF _CLC_OVERLOAD _CLC_CONVERGENT TYPE \ + __spirv_GroupNonUniformShuffleUp(int scope, TYPE value, uint delta) { \ + (void)scope; \ + return __spirv_SubgroupShuffleUpINTEL(value, value, delta); \ + } \ + _CLC_DEF _CLC_OVERLOAD _CLC_CONVERGENT TYPE \ + __spirv_GroupNonUniformShuffleDown(int scope, TYPE value, uint delta) { \ + (void)scope; \ + return __spirv_SubgroupShuffleDownINTEL(value, value, delta); \ + } + +__CLC_GROUP_NON_UNIFORM_SHUFFLE(char) +__CLC_GROUP_NON_UNIFORM_SHUFFLE(uchar) +__CLC_GROUP_NON_UNIFORM_SHUFFLE(short) +__CLC_GROUP_NON_UNIFORM_SHUFFLE(ushort) +__CLC_GROUP_NON_UNIFORM_SHUFFLE(int) +__CLC_GROUP_NON_UNIFORM_SHUFFLE(uint) +__CLC_GROUP_NON_UNIFORM_SHUFFLE(long) +__CLC_GROUP_NON_UNIFORM_SHUFFLE(ulong) +__CLC_GROUP_NON_UNIFORM_SHUFFLE(half) +__CLC_GROUP_NON_UNIFORM_SHUFFLE(float) +__CLC_GROUP_NON_UNIFORM_SHUFFLE(double) + +#undef __CLC_GROUP_NON_UNIFORM_SHUFFLE