Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 100 additions & 5 deletions tools/clang/unittests/HLSLExec/LinAlgTests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2118,6 +2118,7 @@ class DxilConf_SM610_LinAlg {
TEST_METHOD(LoadStoreDescriptor_Wave_16x16_F16);
TEST_METHOD(LoadStoreDescriptor_Wave_4x8_F16_RowMajorOffsetPadded);
TEST_METHOD(LoadStoreDescriptor_Wave_4x8_F32_RowMajorToColumnMajor);
TEST_METHOD(LoadStoreDescriptor_Wave_4x8_F16_RowMajorToColumnMajor);
TEST_METHOD(LoadDescriptorOOB_Wave_16x16_F16_PartialView);
TEST_METHOD(LoadDescriptorOOB_Wave_4x8_F16_OffsetPaddedPartialView);
TEST_METHOD(SplatStore_Wave_16x16_F16);
Expand All @@ -2134,6 +2135,8 @@ class DxilConf_SM610_LinAlg {
TEST_METHOD(ElementSet_Wave_16x16_F16);
TEST_METHOD(ElementGetOOB_Wave_4x8_F32);
TEST_METHOD(ElementSetOOB_Wave_4x8_F32);
TEST_METHOD(ElementGetOOB_Wave_16x16_F16);
TEST_METHOD(ElementSetOOB_Wave_16x16_F16);

// Cast/Convert
TEST_METHOD(CopyConvert_Wave_16x16_F16);
Expand Down Expand Up @@ -2562,6 +2565,49 @@ void DxilConf_SM610_LinAlg::
VerboseLogging, SelectedWaveSize);
}

// The same cross-layout axis on F16, because no tier is required to support
// Fp32 matrices and the F32 case above can skip in its entirety. The shape
// must stay non-square: swapping the two layouts transposes on load and back
// on store, and for a square matrix those cancel byte for byte whatever
// strides are used.
void DxilConf_SM610_LinAlg::
LoadStoreDescriptor_Wave_4x8_F16_RowMajorToColumnMajor() {
MatrixParams Params = {};
Params.CompType = ComponentType::F16;
Params.M = 4;
Params.N = 8;
Params.Use = MatrixUse::A;
Params.Scope = MatrixScope::Wave;
Params.Layout = MatrixLayout::RowMajor;
Params.NumThreads = 128;
Params.Enable16Bit = true;

UINT SelectedWaveSize = 0;
if (!matrixConstructionApplicable(
D3DDevice, Params, {Params.Use},
L"LoadStoreDescriptor_Wave_4x8_F16_RowMajorToColumnMajor",
SelectedWaveSize))
return;

// Source rows of 8 F16 values are 16 bytes packed, padded here to 48.
const cpu_oracle::MatrixBufferLayout LoadLayout = {
MatrixLayout::RowMajor,
/*OffsetBytes=*/DescriptorAlignedOffset,
/*StrideBytes=*/48,
};

// Destination columns of 4 F16 values are 8 bytes, padded here to 16 so the
// column-major side carries a gap of its own rather than sitting packed.
const cpu_oracle::MatrixBufferLayout StoreLayout = {
MatrixLayout::ColumnMajor,
/*OffsetBytes=*/DescriptorAlignedOffset,
/*StrideBytes=*/16,
};

runLoadStoreDescriptor(D3DDevice, DxcSupport, Params, LoadLayout, StoreLayout,
VerboseLogging, SelectedWaveSize);
}

// Half the source matrix lies outside the view the descriptor carries. The
// boundary is deliberately placed mid-row rather than on a row boundary, so an
// implementation that bounds checks a row at a time cannot pass it.
Expand Down Expand Up @@ -3051,7 +3097,7 @@ void DxilConf_SM610_LinAlg::ElementSet_Wave_16x16_F16() {
// only at the wave total and one that wraps a large index back into range.
static constexpr UINT FarOOBOffset = 64;

// Per-lane record: {uint Length, uint Executed, ELEM_TYPE Just, ELEM_TYPE Far}.
// Per-lane record: {uint Length, uint Executed, float Just, float Far}.
static constexpr UINT OOBRecordSize = 16;

// Seeds every output byte so a lane that never writes cannot be mistaken for a
Expand Down Expand Up @@ -3100,8 +3146,11 @@ static const char ElementGetOOBShader[] = R"(
uint Base = threadID * OOB_RECORD_SIZE;
Output.Store<uint>(Base + 0, Len);
Output.Store<uint>(Base + 4, 1);
Output.Store<ELEM_TYPE>(Base + 8, Just);
Output.Store<ELEM_TYPE>(Base + 12, Far);
// Widened to float so the runner can read a fixed-width record whatever
// the element type: a half store would leave the record's upper two bytes
// holding the sentinel. Half to float is lossless, so no value is masked.
Output.Store<float>(Base + 8, (float)Just);
Output.Store<float>(Base + 12, (float)Far);
}
)";

Expand Down Expand Up @@ -3141,8 +3190,9 @@ static void runElementGetOOB(ID3D12Device *Device,
dxc::SpecificDllLoader &DxcSupport,
const MatrixParams &Params, bool Verbose,
UINT ForcedWaveSize) {
VERIFY_IS_TRUE(Params.CompType == ComponentType::F32,
"Out-of-bounds Get records assume a 4-byte element");
VERIFY_IS_TRUE(Params.CompType == ComponentType::F32 ||
Params.CompType == ComponentType::F16,
"Out-of-bounds Get records widen the element to float");
const size_t NumElements = Params.totalElements();
const size_t NumThreads = Params.NumThreads;
const size_t MatrixSize = Params.totalBytes();
Expand Down Expand Up @@ -3345,6 +3395,51 @@ void DxilConf_SM610_LinAlg::ElementSetOOB_Wave_4x8_F32() {
SelectedWaveSize);
}

// Out-of-bounds element access on F16. Both cases above pin the boundary
// behaviour to F32, which no tier is required to support, so a conforming
// F16-only device would exercise neither.
Comment on lines +3398 to +3400

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit:

Suggested change
// Out-of-bounds element access on F16. Both cases above pin the boundary
// behaviour to F32, which no tier is required to support, so a conforming
// F16-only device would exercise neither.
// Add F16 variants so devices that support 16x16 F16 accumulator matrices,
// but not the existing 4x8 F32 shape, can exercise OOB element access.

Also, how reasonable is it to assume 16x16 F16 accumulator matrix support? I'm guessing that's expected to be supported on most/all hardware?

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We are going to have to exercise the entire 'matrix' of support at the end of the day anyway. Later on I plan to do a sweep back over to make sure we haven't missed a format or scope combination

void DxilConf_SM610_LinAlg::ElementGetOOB_Wave_16x16_F16() {
MatrixParams Params = {};
Params.CompType = ComponentType::F16;
Params.M = 16;
Params.N = 16;
Params.Use = MatrixUse::Accumulator;
Params.Scope = MatrixScope::Wave;
Params.Layout = MatrixLayout::RowMajor;
Params.NumThreads = 128;
Params.Enable16Bit = true;

UINT SelectedWaveSize = 0;
if (!matrixConstructionApplicable(D3DDevice, Params, {Params.Use},
L"ElementGetOOB_Wave_16x16_F16",
SelectedWaveSize))
return;

runElementGetOOB(D3DDevice, DxcSupport, Params, VerboseLogging,
SelectedWaveSize);
}

void DxilConf_SM610_LinAlg::ElementSetOOB_Wave_16x16_F16() {
MatrixParams Params = {};
Params.CompType = ComponentType::F16;
Params.M = 16;
Params.N = 16;
Params.Use = MatrixUse::Accumulator;
Params.Scope = MatrixScope::Wave;
Params.Layout = MatrixLayout::RowMajor;
Params.NumThreads = 128;
Params.Enable16Bit = true;

UINT SelectedWaveSize = 0;
if (!matrixConstructionApplicable(D3DDevice, Params, {Params.Use},
L"ElementSetOOB_Wave_16x16_F16",
SelectedWaveSize))
return;

runElementSetOOB(D3DDevice, DxcSupport, Params, VerboseLogging,
SelectedWaveSize);
}

static const char CopyConvertShader[] = R"(
RWByteAddressBuffer Input : register(u0);
RWByteAddressBuffer Output : register(u1);
Expand Down
Loading