diff --git a/diskann-wide/src/arch/x86_64/v3/conversion.rs b/diskann-wide/src/arch/x86_64/v3/conversion.rs index ff991d4ce2..a03ac92d66 100644 --- a/diskann-wide/src/arch/x86_64/v3/conversion.rs +++ b/diskann-wide/src/arch/x86_64/v3/conversion.rs @@ -6,7 +6,9 @@ // x86 intrinsics use std::arch::x86_64::*; -use super::{f16x8, f16x16, f32x8, f32x16, i8x16, i16x8, i16x16, i32x8, u8x16, u32x8}; +use super::{ + f16x8, f16x16, f32x8, f32x16, i8x16, i8x32, i16x8, i16x16, i32x8, u8x16, u8x32, u32x8, +}; use crate::{LoHi, SIMDCast, SIMDReinterpret, SplitJoin, helpers}; ///////////////// @@ -100,6 +102,18 @@ impl SIMDReinterpret for u32x8 { } } +impl SIMDReinterpret for u32x8 { + fn reinterpret_simd(self) -> u8x32 { + u8x32(self.0) + } +} + +impl SIMDReinterpret for u32x8 { + fn reinterpret_simd(self) -> i8x32 { + i8x32(self.0) + } +} + impl SIMDReinterpret for i16x8 { fn reinterpret_simd(self) -> u8x16 { u8x16(self.0) @@ -121,6 +135,22 @@ mod test_x86_conversions { use super::*; use crate::{arch::x86_64::V3, test_utils}; + test_utils::ops::test_reinterpret!( + u32x8 => i16x16, 0x763582a914ec0bfd, V3::new_checked_uncached() + ); + test_utils::ops::test_reinterpret!( + u32x8 => u8x32, 0x69b42dafe0917358, V3::new_checked_uncached() + ); + test_utils::ops::test_reinterpret!( + u32x8 => i8x32, 0x69b42dafe0917358, V3::new_checked_uncached() + ); + test_utils::ops::test_reinterpret!( + i16x8 => u8x16, 0x4b96d02e85f713ca, V3::new_checked_uncached() + ); + test_utils::ops::test_reinterpret!( + u8x16 => i16x8, 0xd5302e6b194a87fc, V3::new_checked_uncached() + ); + // Lossless Conversions #[cfg(not(miri))] test_utils::ops::test_lossless_convert!( diff --git a/diskann-wide/src/arch/x86_64/v4/conversion.rs b/diskann-wide/src/arch/x86_64/v4/conversion.rs index 896dac76b5..16f679db69 100644 --- a/diskann-wide/src/arch/x86_64/v4/conversion.rs +++ b/diskann-wide/src/arch/x86_64/v4/conversion.rs @@ -9,7 +9,7 @@ use std::arch::x86_64::*; use super::{ f16x8_::f16x8, f16x16_::f16x16, f32x8_::f32x8, f32x16_::f32x16, i8x16_::i8x16, i8x32_::i8x32, i8x64_::i8x64, i16x8_::i16x8, i16x16_::i16x16, i16x32_::i16x32, i32x8_::i32x8, u8x16_::u8x16, - u8x32_::u8x32, u8x64_::u8x64, u32x8_::u32x8, u32x16_::u32x16, + u8x32_::u8x32, u8x64_::u8x64, u32x8_::u32x8, u32x16_::u32x16, u64x8_::u64x8, }; use crate::{SIMDCast, SIMDReinterpret, SIMDVector, arch::x86_64::v3, helpers}; @@ -112,6 +112,18 @@ impl SIMDReinterpret for u32x16 { } } +impl SIMDReinterpret for u64x8 { + fn reinterpret_simd(self) -> u8x64 { + u8x64(self.0) + } +} + +impl SIMDReinterpret for u64x8 { + fn reinterpret_simd(self) -> i8x64 { + i8x64(self.0) + } +} + impl SIMDReinterpret for u8x64 { fn reinterpret_simd(self) -> u32x16 { u32x16(self.0) @@ -145,6 +157,37 @@ mod test_x86_conversions { use super::*; use crate::{arch::x86_64::V4, test_utils}; + test_utils::ops::test_reinterpret!( + u32x8 => i16x16, 0x763582a914ec0bfd, V4::new_checked_uncached() + ); + test_utils::ops::test_reinterpret!( + u32x16 => i16x32, 0xe96b52d0743af18c, V4::new_checked_uncached() + ); + test_utils::ops::test_reinterpret!( + u32x16 => u8x64, 0x372ab891f06e5cd4, V4::new_checked_uncached() + ); + test_utils::ops::test_reinterpret!( + u32x16 => i8x64, 0x951d80aefc476b23, V4::new_checked_uncached() + ); + test_utils::ops::test_reinterpret!( + u64x8 => u8x64, 0x891d3c75ef2064ba, V4::new_checked_uncached() + ); + test_utils::ops::test_reinterpret!( + u64x8 => i8x64, 0x891d3c75ef2064ba, V4::new_checked_uncached() + ); + test_utils::ops::test_reinterpret!( + u8x64 => u32x16, 0x28fc7d6140a953be, V4::new_checked_uncached() + ); + test_utils::ops::test_reinterpret!( + i8x64 => u32x16, 0xbf6308d2e471a59c, V4::new_checked_uncached() + ); + test_utils::ops::test_reinterpret!( + i16x8 => u8x16, 0x4b96d02e85f713ca, V4::new_checked_uncached() + ); + test_utils::ops::test_reinterpret!( + u8x16 => i16x8, 0xd5302e6b194a87fc, V4::new_checked_uncached() + ); + // Lossless Conversions #[cfg(not(miri))] test_utils::ops::test_lossless_convert!( diff --git a/diskann-wide/src/emulated.rs b/diskann-wide/src/emulated.rs index df08821d6b..be12697949 100644 --- a/diskann-wide/src/emulated.rs +++ b/diskann-wide/src/emulated.rs @@ -631,6 +631,11 @@ impl_little_endian_transmute_cast!( => ); impl_little_endian_transmute_cast!( => ); impl_little_endian_transmute_cast!( => ); +impl_little_endian_transmute_cast!( => ); +impl_little_endian_transmute_cast!( => ); +impl_little_endian_transmute_cast!( => ); +impl_little_endian_transmute_cast!( => ); + impl_little_endian_transmute_cast!( => ); impl_little_endian_transmute_cast!( => ); @@ -1110,6 +1115,55 @@ mod test_emulated { Emulated => Emulated, 0x75929494c5d333d0, SC ); + #[cfg(target_endian = "little")] + test_utils::ops::test_reinterpret!( + Emulated => Emulated, 0x763582a914ec0bfd, SC + ); + #[cfg(target_endian = "little")] + test_utils::ops::test_reinterpret!( + Emulated => Emulated, 0xe96b52d0743af18c, SC + ); + #[cfg(target_endian = "little")] + test_utils::ops::test_reinterpret!( + Emulated => Emulated, 0x4b96d02e85f713ca, SC + ); + #[cfg(target_endian = "little")] + test_utils::ops::test_reinterpret!( + Emulated => Emulated, 0xd5302e6b194a87fc, SC + ); + #[cfg(target_endian = "little")] + test_utils::ops::test_reinterpret!( + Emulated => Emulated, 0x372ab891f06e5cd4, SC + ); + #[cfg(target_endian = "little")] + test_utils::ops::test_reinterpret!( + Emulated => Emulated, 0x951d80aefc476b23, SC + ); + #[cfg(target_endian = "little")] + test_utils::ops::test_reinterpret!( + Emulated => Emulated, 0x69b42dafe0917358, SC + ); + #[cfg(target_endian = "little")] + test_utils::ops::test_reinterpret!( + Emulated => Emulated, 0x69b42dafe0917358, SC + ); + #[cfg(target_endian = "little")] + test_utils::ops::test_reinterpret!( + Emulated => Emulated, 0x891d3c75ef2064ba, SC + ); + #[cfg(target_endian = "little")] + test_utils::ops::test_reinterpret!( + Emulated => Emulated, 0x891d3c75ef2064ba, SC + ); + #[cfg(target_endian = "little")] + test_utils::ops::test_reinterpret!( + Emulated => Emulated, 0x28fc7d6140a953be, SC + ); + #[cfg(target_endian = "little")] + test_utils::ops::test_reinterpret!( + Emulated => Emulated, 0xbf6308d2e471a59c, SC + ); + /////////// // Casts // /////////// diff --git a/diskann-wide/src/test_utils/ops.rs b/diskann-wide/src/test_utils/ops.rs index fc15661f1e..f742dd6e3f 100644 --- a/diskann-wide/src/test_utils/ops.rs +++ b/diskann-wide/src/test_utils/ops.rs @@ -532,6 +532,48 @@ macro_rules! test_cast { } } +macro_rules! test_reinterpret { + ( + $from:ident $(< $($fs:tt),+ >)? => $to:ident $(< $($ts:tt),+ >)?, + $seed:literal, + $arch:expr + ) => { + paste::paste! { + #[test] + fn []() { + use $crate::{SIMDReinterpret, SIMDVector}; + + type From = $from $(< $($fs),+>)?; + type To = $to $(< $($ts),+>)?; + + if let Some(arch) = $arch { + let f = move |input: &[::Scalar]| { + let got: To = From::from_array(arch, input.try_into().unwrap()) + .reinterpret_simd(); + assert_eq!( + bytemuck::must_cast_slice::<_, u8>(&got.to_array()), + bytemuck::must_cast_slice::<_, u8>(input), + "input: {input:?}", + ); + }; + + // Distinct byte positions detect lane/byte reordering. + let mut input = [::Scalar::default(); From::LANES]; + for start in [0u8, 0x7f, 0x80, 0xff] { + for (i, byte) in bytemuck::must_cast_slice_mut::<_, u8>(&mut input) + .iter_mut().enumerate() + { + *byte = start.wrapping_add(i as u8); + } + f(&input); + } + $crate::test_utils::driver::drive_unary(&f, From::LANES, $seed); + } + } + } + }; +} + macro_rules! test_abs { ($wide:ident $(< $($ps:tt),+ >)?, $seed:literal, $arch:expr) => { paste::paste! { @@ -1146,6 +1188,7 @@ pub(crate) use test_lossless_convert; pub(crate) use test_minmax; pub(crate) use test_mul; pub(crate) use test_popcount; +pub(crate) use test_reinterpret; pub(crate) use test_select; pub(crate) use test_splitjoin; pub(crate) use test_sub;