From e6daaca3607093d4fdf248973792c79b5cc14e9e Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Sun, 4 Oct 2026 09:11:55 +0000 Subject: [PATCH 1/3] [tmva][sofie] Fix the SOFIE tests on Windows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SOFIE tests are currently disabled on Windows CI (test_tmva_sofie=OFF in windows10.txt); enabling them surfaces two Windows-only failures: * RModelParser_ONNX::Parse split the model path at backslashes only, so a forward-slash path like dir/model.onnx (which the Windows file APIs happily accept) yielded an empty model directory, and the external weight data of such a model was looked up in the working directory instead of next to the model (TestSofieParser. ExternalDataLocationRelativeToModelDirectory). On Windows, split at the last of either separator; on POSIX a backslash stays a filename character. * Catching an exception thrown by cling-JIT code does not work on Windows: the runtime finds no handler and terminates the process with 0xE06D7363, so the safetensors tests whose whole point is checking that the generated session constructors throw can only run elsewhere (TestSafetensorsWeights dies in MissingFileThrows, and all the other throw-probing tests would follow). Skip them on MSVC; the success-path construction (ModerateNestingAccepted) stays enabled. 🤖 Done with the help of AI --- tmva/sofie/test/TestSafetensorsWeights.cxx | 16 ++++++++++++++++ tmva/sofie_parsers/src/RModelParser_ONNX.cxx | 9 ++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/tmva/sofie/test/TestSafetensorsWeights.cxx b/tmva/sofie/test/TestSafetensorsWeights.cxx index 8161136131a84..de9df6075c0b8 100644 --- a/tmva/sofie/test/TestSafetensorsWeights.cxx +++ b/tmva/sofie/test/TestSafetensorsWeights.cxx @@ -303,6 +303,15 @@ std::vector ExpectedOutput() #define SOFIE_SKIP_ON_BIG_ENDIAN GTEST_SKIP() << "safetensors weights are little-endian only" #endif +// On Windows, an exception thrown by cling-compiled (JIT) code is not caught: +// the runtime finds no handler and terminates the process (0xE06D7363). The +// tests expecting the session constructors to throw can only run elsewhere. +#ifdef _MSC_VER +#define SOFIE_SKIP_JIT_THROW_TESTS GTEST_SKIP() << "C++ exceptions from cling-JIT code cannot be caught on Windows" +#else +#define SOFIE_SKIP_JIT_THROW_TESTS +#endif + // The safetensors writer emits a well-formed file: 8-byte little-endian JSON // header size, the JSON header describing each tensor, and the raw payloads TEST(SOFIESafetensors, FileLayoutAndPayload) @@ -391,6 +400,7 @@ TEST(SOFIESafetensors, DefaultWeightFileName) TEST(SOFIESafetensors, MissingFileThrows) { SOFIE_SKIP_ON_BIG_ENDIAN; + SOFIE_SKIP_JIT_THROW_TESTS; EXPECT_TRUE(SessionConstructorThrows("safetensors_mul", "safetensors_mul_bogus.safetensors")); } @@ -399,6 +409,7 @@ TEST(SOFIESafetensors, MissingFileThrows) TEST(SOFIESafetensors, WrongDtypeThrows) { SOFIE_SKIP_ON_BIG_ENDIAN; + SOFIE_SKIP_JIT_THROW_TESTS; std::ifstream in("safetensors_mul.safetensors", std::ios::binary); std::string content{std::istreambuf_iterator(in), std::istreambuf_iterator()}; in.close(); @@ -453,6 +464,7 @@ TEST(SOFIESafetensors, ThirdPartyFileLayout) TEST(SOFIESafetensors, MalformedJsonThrows) { SOFIE_SKIP_ON_BIG_ENDIAN; + SOFIE_SKIP_JIT_THROW_TESTS; std::string headerStr = R"({"tensor_W": {"dtype": "F32", "data_offsets": [0, 24},)"; // note the broken bracket std::ofstream out("safetensors_mul_malformed.safetensors", std::ios::binary); for (int i = 0; i < 8; ++i) @@ -489,6 +501,7 @@ TEST(SOFIESafetensors, InMemoryWriterMatchesFile) TEST(SOFIESafetensors, TruncatedBlobThrows) { SOFIE_SKIP_ON_BIG_ENDIAN; + SOFIE_SKIP_JIT_THROW_TESTS; const std::string full = ReadWholeFile("safetensors_mul.safetensors"); ASSERT_GT(full.size(), 10u); EXPECT_TRUE(BlobSessionConstructorThrows("safetensors_mul", full.substr(0, full.size() - 10))); @@ -516,6 +529,7 @@ TEST(SOFIESafetensors, WeightlessModelFromBlob) TEST(SOFIESafetensors, DeepNestingThrows) { SOFIE_SKIP_ON_BIG_ENDIAN; + SOFIE_SKIP_JIT_THROW_TESTS; std::string header = R"({"tensor_W":{"dtype":"F32","shape":[2,3],"data_offsets":[0,24]},"__metadata__":)"; header += std::string(10000, '[') + std::string(10000, ']') + "}"; EXPECT_TRUE(BlobSessionConstructorThrows("safetensors_mul", MakeSafetensorsPayload(header, std::string(24, '\0')))); @@ -535,6 +549,7 @@ TEST(SOFIESafetensors, ModerateNestingAccepted) TEST(SOFIESafetensors, WrappingOffsetsThrow) { SOFIE_SKIP_ON_BIG_ENDIAN; + SOFIE_SKIP_JIT_THROW_TESTS; // payload base is 8 + header size; this end offset overflows when added const std::string header = R"({"tensor_W":{"dtype":"F32","shape":[2,3],"data_offsets":[0,18446744073709551600]}})"; @@ -545,6 +560,7 @@ TEST(SOFIESafetensors, WrappingOffsetsThrow) TEST(SOFIESafetensors, NonIntegerOffsetsThrow) { SOFIE_SKIP_ON_BIG_ENDIAN; + SOFIE_SKIP_JIT_THROW_TESTS; const std::string header = R"({"tensor_W":{"dtype":"F32","shape":[2,3],"data_offsets":[0,24.0]}})"; EXPECT_TRUE(BlobSessionConstructorThrows("safetensors_mul", MakeSafetensorsPayload(header, std::string(24, '\0')))); } diff --git a/tmva/sofie_parsers/src/RModelParser_ONNX.cxx b/tmva/sofie_parsers/src/RModelParser_ONNX.cxx index a733441ae52d1..7391f7405d72a 100644 --- a/tmva/sofie_parsers/src/RModelParser_ONNX.cxx +++ b/tmva/sofie_parsers/src/RModelParser_ONNX.cxx @@ -523,11 +523,14 @@ RModel RModelParser_ONNX::Parse(std::string const &filename, bool verbose) std::string parsetime(std::asctime(gmt_time)); // get name of model (filename without directory name) - char sep = '/'; #ifdef _WIN32 - sep = '\\'; + // The Windows file APIs accept both separators, and forward-slash paths + // do get passed (e.g. from Python or CMake) + size_t isep = filename.find_last_of("/\\"); +#else + // On POSIX, a backslash is a valid filename character, not a separator + size_t isep = filename.rfind('/', filename.length()); #endif - size_t isep = filename.rfind(sep, filename.length()); std::string filename_nodir = filename; if (isep != std::string::npos) { filename_nodir = (filename.substr(isep + 1, filename.length() - isep)); From a3ac8a92b66319b81046ac153e9055244e6874e2 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Fri, 2 Oct 2026 19:16:06 +0000 Subject: [PATCH 2/3] [tmva][sofie] Drop the BLAS dependency of the generated code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The inference code SOFIE emits no longer calls any BLAS routines. Matrix multiplications are now done by Gemm, a self-contained reference implementation of the sgemm contract emitted into the generated header next to the other inference helpers, and bias accumulations by the corresponding Axpy (saxpy). Both keep the pointer-based BLAS calling convention, so Gemm_Call, which the Gemm and Conv operators use, and the emission sites in the ConvTranspose, Einsum, GRU, LSTM and RNN operators only change the name of the function they call. For a non-transposed A, Gemm walks C column by column with unit stride so the compiler can auto-vectorize the innermost loop; this is close to BLAS performance for the per-event (gemv-shaped) inference SOFIE is used for in RDataFrame loops and RooFit, while removing - the link-time BLAS dependency of every generated model, - symbol collisions with whatever sgemm_ happens to be loaded in the process (e.g. a conda OpenBLAS in PyROOT sessions), - the implicit, environment-dependent multithreading of the BLAS implementation picked up at runtime, which interferes with RDataFrame implicit MT. Consequently deleted: the BLAS namespace emission in RModel::GenerateHeaderInfo, fNeededBlasRoutines / AddBlasRoutines / GetBlasRoutines, the extern "C" sgemm_/sgemv_/saxpy_ declarations (dead Gemv), the commented-out BLAS call sites in Conv/ConvTranspose, the hand-rolled CMake BLAS handling of the SOFIE tests (BLAS_FOUND guards, BLAS::BLAS link libraries, the hard configure error in SearchInstalledSoftware), and the veto of the SOFIE tutorials if no BLAS is found. The hand-written Clad pullbacks/pushforwards for Gemm_Call are kept: they are pure C++ built on Gemm_Call itself, and much cheaper than the tape Clad would generate for the loops in Gemm. Also remove the test_tmva_sofie configuration option: with the BLAS requirement gone the tests need nothing beyond what any other ROOT test needs, so they now follow the global 'testing' option like all other subsystems. Verified: all SOFIE tests (TestSofieModels, TestCustomModelsFromONNX, TestCladAutodiff, TestGemmDerivative, TestSofieParser, ...), testRooONNXFunc and the SOFIE tutorials pass; a generated LSTM and ConvTranspose model header compiles and runs with plain g++ -std=c++17 with no ROOT and no BLAS. Performance (Zen5-class 12-core machine, emitted code compiled -O2, vs OpenBLAS 0.3.33 single-threaded -- ROOT's BLAS dependency here): Per-event inference (batch=1 GEMV shapes, the RDataFrame / RooFit SBI pattern) is at parity up to layer size ~512, both at kernel level (gemv 512x1x512: 3.5 vs 3.6 GFLOP/s) and end-to-end through Session::infer of a 4-layer MLP (D32: 2.1 vs 2.1 us, D128: 31 vs 27 us, D512: 464 vs 440 us per event). A regression sets in around layer size 1024 (weights ~4 MB): 3-5x kernel, 2.9x end-to-end (1899 vs 649 us). The transposed-A case (which is what per-event inference of row-major weights emits) needs the dot-product loop nest in Gemm: with the plain accumulation nest the strided A reads make it 27x slower than BLAS at D=1024 instead of 2.9x. Large *batched* evaluation regresses 20-100x end-to-end from D=128/batch=1000 up (D1024 B1000: 1.8 s vs 18 ms), matching the kernel gap for square GEMMs (~48x at 512^3, ~64-116x at 2048^3). Threaded OpenBLAS (12 threads) does not change the picture for typical SOFIE shapes: it is equal or slower than its single-threaded self below ~2048^3, and the shapes where it wins are exactly the ones SOFIE-emitted code does not target. 🤖 Done with the help of AI --- .../root-ci-config/buildconfig/global.txt | 1 - .../root-ci-config/buildconfig/windows10.txt | 1 - README/ReleaseNotes/v644/index.md | 7 + cmake/modules/RootBuildOptions.cmake | 1 - cmake/modules/SearchInstalledSoftware.cmake | 12 -- tmva/sofie/CMakeLists.txt | 5 +- tmva/sofie/README.md | 6 +- tmva/sofie/inc/TMVA/RModel.hxx | 12 -- tmva/sofie/inc/TMVA/ROperator.hxx | 3 - .../inc/TMVA/ROperator_BatchNormalization.hxx | 1 - tmva/sofie/inc/TMVA/ROperator_Conv.hxx | 29 ---- .../inc/TMVA/ROperator_ConvTranspose.hxx | 32 +---- tmva/sofie/inc/TMVA/ROperator_Einsum.hxx | 7 +- tmva/sofie/inc/TMVA/ROperator_GRU.hxx | 68 ++++----- tmva/sofie/inc/TMVA/ROperator_Gemm.hxx | 2 - tmva/sofie/inc/TMVA/ROperator_LSTM.hxx | 75 +++++----- .../inc/TMVA/ROperator_LayerNormalization.hxx | 2 - tmva/sofie/inc/TMVA/ROperator_RNN.hxx | 19 +-- tmva/sofie/inc/TMVA/SOFIE_common.hxx | 18 +-- tmva/sofie/src/RModel.cxx | 30 +--- tmva/sofie/src/SOFIE_common_helpers.cxx | 130 +++++++++++++++--- tmva/sofie/test/CMakeLists.txt | 49 +++---- tmva/sofie/test/TestCladAutodiff.cxx | 4 +- tutorials/CMakeLists.txt | 5 - 24 files changed, 239 insertions(+), 280 deletions(-) diff --git a/.github/workflows/root-ci-config/buildconfig/global.txt b/.github/workflows/root-ci-config/buildconfig/global.txt index 42c3409226dce..993841815faaf 100644 --- a/.github/workflows/root-ci-config/buildconfig/global.txt +++ b/.github/workflows/root-ci-config/buildconfig/global.txt @@ -86,7 +86,6 @@ ssl=ON test_distrdf_dask=ON test_distrdf_pyspark=ON test_roofit_hs3testsuite=ON -test_tmva_sofie=ON testing=ON tmva-cpu=ON tmva-gpu=OFF diff --git a/.github/workflows/root-ci-config/buildconfig/windows10.txt b/.github/workflows/root-ci-config/buildconfig/windows10.txt index eb33f67625d12..4a5a0fb4b3344 100644 --- a/.github/workflows/root-ci-config/buildconfig/windows10.txt +++ b/.github/workflows/root-ci-config/buildconfig/windows10.txt @@ -30,7 +30,6 @@ runtime_cxxmodules=OFF ssl=OFF test_distrdf_dask=OFF test_distrdf_pyspark=OFF -test_tmva_sofie=OFF use_gsl_cblas=ON vdt=OFF x11=OFF diff --git a/README/ReleaseNotes/v644/index.md b/README/ReleaseNotes/v644/index.md index dde65790e5f56..13c581110269d 100644 --- a/README/ReleaseNotes/v644/index.md +++ b/README/ReleaseNotes/v644/index.md @@ -28,6 +28,7 @@ The following people have contributed to this new version: * The TMVA deep learning methods `MethodDL` (`TMVA::Types::kDL`) and `MethodDNN` (`TMVA::Types::kDNN`), together with all classes in the `TMVA::DNN` namespace and their helper classes `TMVA::Monitoring` and `Pattern`, are deprecated and will be removed in ROOT 6.46. Neural networks are nowadays trained with dedicated machine learning libraries, so maintaining a separate implementation with its own CPU, CUDA and cuDNN backends is no longer justified. Users are encouraged to train their models directly with these libraries, which integrate well with ROOT via the `ROOT.Experimental.ML.RDataLoader`, and to use SOFIE for inference in C++. The `TMVA_CNN_Classification` and `TMVA_RNN_Classification` tutorials are removed already, and the other TMVA tutorials do not use these methods anymore. We are aware that models trained with these methods may be stored in TMVA XML weight files and still be evaluated via `TMVA::Reader` in production workflows. If you need to keep running such models after ROOT 6.46, please reach out to the ROOT team by opening a [GitHub issue](https://github.com/root-project/root/issues) or posting on the [ROOT forum](https://root-forum.cern.ch/), so that a migration path to SOFIE can be provided. * The build options `tmva-cpu`, `tmva-gpu`, `tmva-cudnn` and `use_gsl_cblas` only affect the deprecated TMVA deep learning methods, so they are deprecated as well and will be removed in ROOT 6.46. The `tmva-cpu` option is now `OFF` by default, also with `-Dall=ON`, so `MethodDL` runs on a single thread without BLAS and `MethodDNN` cannot be trained, unless ROOT is configured with `-Dtmva-cpu=ON`. Evaluating existing weight files with `TMVA::Reader` works in both cases. +* The `test_tmva_sofie` build option was removed. The TMVA SOFIE tests don't need BLAS anymore, so they are now enabled by the `testing` option like all other tests. ## Build system @@ -42,6 +43,12 @@ RenderCore, the WebGL engine of the REve client, is fetched at build time as the ## Python Interface +## Machine Learning + +### SOFIE inference code without BLAS dependency + +The C++ code that SOFIE generates for the inference of ONNX models doesn't call BLAS anymore: the matrix multiplications are done by functions that are emitted into the generated header itself. The generated code is therefore self-contained and doesn't need to be linked against a BLAS library. For the per-event inference that SOFIE is mostly used for, for example in RDataFrame event loops, this is as fast as BLAS, but large batches are evaluated considerably slower. + ## I/O ## Core diff --git a/cmake/modules/RootBuildOptions.cmake b/cmake/modules/RootBuildOptions.cmake index 637ecd6c5e392..d89df590639db 100644 --- a/cmake/modules/RootBuildOptions.cmake +++ b/cmake/modules/RootBuildOptions.cmake @@ -196,7 +196,6 @@ option(minimal "Enable only required options by default" OFF) option(rootbench "Build rootbench if rootbench exists in root or if it is a sibling directory (implies testing=ON)" OFF) option(roottest "Build roottest (implies testing=ON)" OFF) option(test_roofit_hs3testsuite "Setup and use the HS3 conformance test suite (requires network)" OFF) -option(test_tmva_sofie "Enable SOFIE tests (requires BLAS library that can be found with CMake's FindBLAS)" ON) option(testing "Enable testing with CTest" OFF) option(asan "Build ROOT with address sanitizer instrumentation (see core/sanitizer for details)" OFF) option(_wheel_build "ROOT is being packaged as a wheel, do not install .dist-info metadata" OFF) diff --git a/cmake/modules/SearchInstalledSoftware.cmake b/cmake/modules/SearchInstalledSoftware.cmake index 5a52f594aa54a..0b24dce676ee7 100644 --- a/cmake/modules/SearchInstalledSoftware.cmake +++ b/cmake/modules/SearchInstalledSoftware.cmake @@ -937,18 +937,6 @@ if(experimental_adaptivecpp) endif() endif() -#---Check for optional TMVA-SOFIE testing dependency (BLAS)------------------------------- -# SOFIE itself has no external dependencies: ONNX models are read with a small -# self-contained protobuf wire-format decoder (tmva/sofie_parsers/src/onnx.hxx). - -if(tmva AND testing AND test_tmva_sofie) - message(STATUS "Looking for BLAS as an optional testing dependency of TMVA-SOFIE") - find_package(BLAS) - if(NOT BLAS_FOUND) - message(SEND_ERROR "BLAS not found, but it's required for TMVA-SOFIE testing. Please install BLAS or configure with test_tmva_sofie=OFF") - endif() -endif() - #---Figure out if TMVA CPU should be built and which BLAS we will use ------------------ if(tmva-cpu) if (NOT tmva) diff --git a/tmva/sofie/CMakeLists.txt b/tmva/sofie/CMakeLists.txt index 0d5be051cd468..803be4a67cd0c 100644 --- a/tmva/sofie/CMakeLists.txt +++ b/tmva/sofie/CMakeLists.txt @@ -103,7 +103,4 @@ PRIVATE inc/LinkDef.h ) -# Tests are not enabled unconditionally because they require BLAS -if(test_tmva_sofie) - ROOT_ADD_TEST_SUBDIRECTORY(test) -endif() +ROOT_ADD_TEST_SUBDIRECTORY(test) diff --git a/tmva/sofie/README.md b/tmva/sofie/README.md index 7216d72867227..70769ec0b5a2e 100644 --- a/tmva/sofie/README.md +++ b/tmva/sofie/README.md @@ -7,7 +7,11 @@ This is a new development in TMVA and is currently in early experimental stage. ## Prerequisite -- BLAS or Eigen (for execution of the generated code for inference) + +None: the generated inference code is self-contained C++ (the matrix +multiplications it contains are reference implementations emitted into the +generated header itself, so no BLAS or other external library has to be +linked to run it). ## Installation diff --git a/tmva/sofie/inc/TMVA/RModel.hxx b/tmva/sofie/inc/TMVA/RModel.hxx index 6bf6dc52597b3..cf4deac971ea1 100644 --- a/tmva/sofie/inc/TMVA/RModel.hxx +++ b/tmva/sofie/inc/TMVA/RModel.hxx @@ -55,12 +55,6 @@ private: WeightFileType fWeightFile = WeightFileType::Text; - std::unordered_set fNeededBlasRoutines; - // Set to true once GenerateHeaderInfo has emitted the extern "C" declaration - // of the BLAS sgemm_ routine (from fNeededBlasRoutines). It lets the - // standalone Gemm_Call helper skip emitting a second, duplicate declaration. - bool fBlasSgemmDeclared = false; - std::unordered_set fNeededStdLib = {"vector"}; std::unordered_set fCustomOpHeaders; @@ -313,12 +307,6 @@ public: std::string GetFilename() { return fName; } const std::string &GetName() const { return fName; } - void AddBlasRoutines(std::vector routines) - { - for (auto &routine : routines) { - fNeededBlasRoutines.insert(routine); - } - } void AddNeededStdLib(std::string libname) { // if the library is already in the set, insert does nothing, so we don't need to check before inserting diff --git a/tmva/sofie/inc/TMVA/ROperator.hxx b/tmva/sofie/inc/TMVA/ROperator.hxx index 13c501b0905d6..3e971816fece6 100644 --- a/tmva/sofie/inc/TMVA/ROperator.hxx +++ b/tmva/sofie/inc/TMVA/ROperator.hxx @@ -16,7 +16,6 @@ class RModel; class ROperator { public: - virtual std::vector GetBlasRoutines() { return {}; } virtual std::vector GetStdLibs() { return {}; } virtual void Initialize(RModel&) = 0; virtual std::string Generate(std::string OpName) = 0; //expect unique opName for each operator within the same RModel @@ -31,8 +30,6 @@ public: /// check if the output of the operator is Constant and is evaluated at initialization time bool IsOutputConstant() const { return fIsOutputConstant; } - //virtual void Forward_reference() = 0; - //virtual void Forward_blas() = 0; virtual ~ROperator(){} protected: diff --git a/tmva/sofie/inc/TMVA/ROperator_BatchNormalization.hxx b/tmva/sofie/inc/TMVA/ROperator_BatchNormalization.hxx index f96a075d2b2c7..b8fb90d5e6979 100644 --- a/tmva/sofie/inc/TMVA/ROperator_BatchNormalization.hxx +++ b/tmva/sofie/inc/TMVA/ROperator_BatchNormalization.hxx @@ -162,7 +162,6 @@ public: return out.str(); } - std::vector GetBlasRoutines() override { return {}; } }; }//SOFIE diff --git a/tmva/sofie/inc/TMVA/ROperator_Conv.hxx b/tmva/sofie/inc/TMVA/ROperator_Conv.hxx index 1f7ae0d65c9f4..09bf233df3ea4 100644 --- a/tmva/sofie/inc/TMVA/ROperator_Conv.hxx +++ b/tmva/sofie/inc/TMVA/ROperator_Conv.hxx @@ -495,7 +495,6 @@ public: << "," << fAttrDilations[1] << "," << fAttrDilations[2] << "," << "tensor_" << imcol << ");\n\n "; } - // BLAS out << SP << "Gemm_Call(" << "tensor_" << fNY << " + out_offset, false, false, " << OpName << "_m, " << OpName << "_n, " << OpName << "_k, " << OpName << "_alpha, " << "tensor_" << imcol << ", tensor_" << convK << ", " << OpName @@ -506,11 +505,6 @@ public: out << "nullptr"; out << ");\n"; - // out << SP << SP << "BLAS::sgemm_(&" << OpName << "_transA, &" << OpName << "_transB, &" << OpName << "_m, &" - // << OpName << "_n, &" << OpName << "_k, &" << OpName << "_alpha, " << "tensor_" << imcol << ", &" << - // OpName << "_m,\n"; // use m if op_xcol is not transpose , otherwise k - // out << SP << SP << SP << "tensor_" << convK << ", &" << OpName << "_k, &" << OpName << "_beta, tensor_" << - // fNY << " + out_offset, &" << OpName << "_m);\n"; } else { // case of group convolution // Unroll (IM2COL) the input tensor- make loop on groups and repeat operations (IM2COL + GEMM for each @@ -552,7 +546,6 @@ public: << "," << fAttrDilations[1] << "," << fAttrDilations[2] << ",tensor_" << imcol << ");\n\n "; } - // BLAS // n must be divided by the number of groups out << SP << SP << SP << OpName << "_n = " << fShapeW[0] / fAttrGroup << ";\n"; // offset g must be g * k * n @@ -570,35 +563,13 @@ public: out << "nullptr"; out << ");\n"; - // out << SP << SP << "BLAS::sgemm_(&" << OpName << "_transA, &" << OpName << "_transB, &" << OpName << "_m, &" - // << OpName << "_n, &" << OpName << "_k, &" << OpName << "_alpha, tensor_" << imcol << ", &" << OpName - // << "_m,\n"; // use m if op_xcol is not transpose , otherwise k - // out << SP << SP << SP << "tensor_" << convK << " + offset_f, &" << OpName << "_k, &" << OpName << "_beta, - // tensor_" << fNY - // << " + out_offset" - // << ", &" << OpName << "_m);\n"; - out << SP << SP << "}\n"; // end of group loop } - // if (fNB != "") { - // out << SP << "int " << OpName << "_size = " << outputBatchStride << ";\n"; - // out << SP << "float " << OpName << "_gamma = 1.0;\n"; - // out << SP << "int " << OpName << "_incx = 1;\n"; - // out << SP << "int " << OpName << "_incy = 1;\n"; - - // out << SP << "BLAS::saxpy_(&" << OpName << "_size, &" << OpName << "_gamma, tensor_" << fNB << ", &" - // << OpName << "_incx, tensor_" << fNY << " + out_offset, &" << OpName << "_incy);\n"; - - // } out << SP << "}\n"; // end of batch size loop return out.str(); } - - /*! \brief Returns the blas routines needed to compile the generated code - */ - std::vector GetBlasRoutines() override { return { std::string("Gemm"), std::string("Axpy") }; } }; } // namespace SOFIE diff --git a/tmva/sofie/inc/TMVA/ROperator_ConvTranspose.hxx b/tmva/sofie/inc/TMVA/ROperator_ConvTranspose.hxx index c21c97c35b21e..a14b41a23ef5d 100644 --- a/tmva/sofie/inc/TMVA/ROperator_ConvTranspose.hxx +++ b/tmva/sofie/inc/TMVA/ROperator_ConvTranspose.hxx @@ -117,10 +117,6 @@ public: * \param opName name of the operator */ std::string Generate(std::string opName) override; - - /*! \brief Returns the blas routines needed to compile the generated code - */ - std::vector GetBlasRoutines() override { return {std::string("Gemm"), std::string("Axpy")}; } }; template @@ -293,6 +289,8 @@ void ROperator_ConvTranspose::Initialize(RModel &model) // register the inference helper functions used by the generated code // (only the <3D case is supported, which uses col2im) model.AddNeededHelperFunction("col2im"); + model.AddNeededHelperFunction("Gemm"); + model.AddNeededHelperFunction("Axpy"); if (!fNB.empty()) model.AddNeededHelperFunction("BroadcastConvBias"); } @@ -427,9 +425,7 @@ std::string ROperator_ConvTranspose::Generate(std::string OpName) out << SP << SP << "size_t x_offset = n * " << fShapeX[1] * iDepth * iHeight * iWidth << ";\n"; out << SP << SP << "size_t out_offset = n * " << fShapeY[1] * oDepth * oHeight * oWidth << ";\n"; - // DO BLAS before: - // BLAS - out << SP << SP << "BLAS::sgemm_(&" << OpName << "_transA, &" << OpName << "_transB, &" << OpName << "_m, &" + out << SP << SP << "Gemm(&" << OpName << "_transA, &" << OpName << "_transB, &" << OpName << "_m, &" << OpName << "_n, &" << OpName << "_k, &" << OpName << "_alpha, " << "tensor_" << fNX << " + x_offset, &" << OpName << "_m,\n"; // use m if op_xcol is not transpose , otherwise k @@ -467,12 +463,6 @@ std::string ROperator_ConvTranspose::Generate(std::string OpName) << fAttrStrides[0] << "," << fAttrStrides[1] << "," << fAttrStrides[2] << "," << fAttrDilations[0] << "," << fAttrDilations[1] << "," << fAttrDilations[2] << ",tensor_" << fImcol << ");\n\n "; } - // // BLAS - // out << SP << SP << "BLAS::sgemm_(&" << OpName << "_transA, &" << OpName << "_transB, &" << OpName << "_m, &" - // << OpName << "_n, &" << OpName << "_k, &" << OpName << "_alpha, tensor_" << fImcol << ", &" << OpName - // << "_m,\n"; // use m if op_xcol is not transpose , otherwise k - // out << SP << SP << SP <<"tensor_" << fConvK << ", &" << OpName << "_k, &" << OpName << "_beta, tensor_" << fNY - // << " + out_offset, &" << OpName << "_m);\n"; } else { // case of group transposed convolution // Unroll (IM2COL) the input tensor- make loop on groups and repeat operations (IM2COL + GEMM for each @@ -487,7 +477,7 @@ std::string ROperator_ConvTranspose::Generate(std::string OpName) out << SP << SP << SP << "size_t offset_f = g * " << (fShapeW[0] / fAttrGroup) * icstrideDil * fShapeW[1] << ";\n"; - out << SP << SP << "BLAS::sgemm_(&" << OpName << "_transA, &" << OpName << "_transB, &" << OpName << "_m, &" + out << SP << SP << "Gemm(&" << OpName << "_transA, &" << OpName << "_transB, &" << OpName << "_m, &" << OpName << "_n, &" << OpName << "_k, &" << OpName << "_alpha, " << "tensor_" << fNX << " + x_offset, &" << OpName << "_m,\n"; // use m if op_xcol is not transpose , otherwise k @@ -524,18 +514,6 @@ std::string ROperator_ConvTranspose::Generate(std::string OpName) << fAttrDilations[1] << "," << fAttrDilations[2] << "," << "tensor_" << fImcol << ");\n\n "; } - // // BLAS - // // offset g must be g * k * n - // out << SP << SP << SP << "size_t offset_f = g * " << fShapeW[0] * fShapeW[1] * icstrideDil / fAttrGroup << - // ";\n"; out << SP << SP << "BLAS::sgemm_(&" << OpName << "_transA, &" << OpName << "_transB, &" << OpName << - // "_m, &" - // << OpName << "_n, &" << OpName << "_k, &" << OpName << "_alpha, tensor_" << fImcol << ", &" << OpName - // << "_m,\n"; // use m if op_xcol is not transpose , otherwise k - // out << SP << SP << SP << "tensor_" << fConvK << " + offset_f, &" << OpName << "_k, &" << OpName << "_beta, - // tensor_" << fNY - // << " + out_offset" - // << ", &" << OpName << "_m);\n"; - out << SP << SP << "}\n"; // end of group loop } @@ -547,7 +525,7 @@ std::string ROperator_ConvTranspose::Generate(std::string OpName) out << SP << "int " << OpName << "_incx = 1;\n"; out << SP << "int " << OpName << "_incy = 1;\n"; - out << SP << "BLAS::saxpy_(&" << OpName << "_size, &" << OpName << "_gamma, tensor_" << fNBroadcastedB << ", &" + out << SP << "Axpy(&" << OpName << "_size, &" << OpName << "_gamma, tensor_" << fNBroadcastedB << ", &" << OpName << "_incx, tensor_" << fNY << ", &" << OpName << "_incy);\n"; } diff --git a/tmva/sofie/inc/TMVA/ROperator_Einsum.hxx b/tmva/sofie/inc/TMVA/ROperator_Einsum.hxx index ea35dbc87b84c..8ef11319d3024 100644 --- a/tmva/sofie/inc/TMVA/ROperator_Einsum.hxx +++ b/tmva/sofie/inc/TMVA/ROperator_Einsum.hxx @@ -112,6 +112,8 @@ public: } void Initialize(RModel& model) override { + // the matmul reduction emitted in Generate() uses Gemm + model.AddNeededHelperFunction("Gemm"); // input must be a graph input, or already initialized intermediate tensor size_t i = 0; std::map labelsMap; @@ -305,7 +307,7 @@ public: return strst.str(); }; // only float type supported - out << SP << "BLAS::sgemm_(&" << opName << "_transB, &" << opName << "_transA, &" << opName + out << SP << "Gemm(&" << opName << "_transB, &" << opName << "_transA, &" << opName << "_n, &" << opName << "_m, &" << opName << "_k, &" << opName << "_alpha, " << "&tensor_" << fNInputs[1] << "[" << tensorOffset(inputStrideB, fInputLabels[1]) << "], &" << opName << "_ldb, " @@ -324,9 +326,6 @@ public: return out.str(); } - std::vector GetBlasRoutines() override { - return { std::string("Gemm") }; - } }; }//SOFIE diff --git a/tmva/sofie/inc/TMVA/ROperator_GRU.hxx b/tmva/sofie/inc/TMVA/ROperator_GRU.hxx index 1164c28ccedc5..913e4e9672954 100644 --- a/tmva/sofie/inc/TMVA/ROperator_GRU.hxx +++ b/tmva/sofie/inc/TMVA/ROperator_GRU.hxx @@ -136,9 +136,6 @@ template class ROperator_GRU final : public ROperator { */ std::string Generate(std::string /*OpName*/) override; - /*! \brief Returns the blas routines needed to compile the generated code - */ - std::vector GetBlasRoutines() override { return { std::string("Gemm"), std::string("Axpy") }; } }; template @@ -164,6 +161,9 @@ auto ROperator_GRU::ShapeInference(std::vector> input) -> template void ROperator_GRU::Initialize(RModel &model) { + // the gate updates emitted in Generate() use Gemm and Axpy + model.AddNeededHelperFunction("Gemm"); + model.AddNeededHelperFunction("Axpy"); // Check the input and output tensors if (!model.CheckIfTensorAlreadyExist(fNX)) { @@ -414,19 +414,19 @@ auto ROperator_GRU::Generate(std::string OpName) -> std::string if (direction == 0) { if (fType == "float") { // f_update_gate = input * weight_z^T - out << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &" + out << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &" << OpName << "_m, &" << OpName << "_k, &" << OpName << "_alpha, tensor_" << fNW << ", &" << OpName << "_k, " << OpName << "_input, &" << OpName << "_k, &" << OpName << "_beta, " << OpName << "_f_update_gate, &" << OpName << "_n);\n"; // f_reset_gate = input * weight_r^T size_t wr_offset = fAttrHiddenSize * input_size; - out << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &" + out << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &" << OpName << "_m, &" << OpName << "_k, &" << OpName << "_alpha, tensor_" << fNW << " + " << wr_offset << ", &" << OpName << "_k, " << OpName << "_input, &" << OpName << "_k, &" << OpName << "_beta, " << OpName << "_f_reset_gate, &" << OpName << "_n);\n"; // f_hidden_gate = input * weight_h^T size_t wh_offset = 2 * fAttrHiddenSize * input_size; - out << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &" + out << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &" << OpName << "_m, &" << OpName << "_k, &" << OpName << "_alpha, tensor_" << fNW << " + " << wh_offset << ", &" << OpName << "_k, " << OpName << "_input, &" << OpName << "_k, &" << OpName << "_beta, " << OpName << "_f_hidden_gate, &" << OpName << "_n);\n"; @@ -435,19 +435,19 @@ auto ROperator_GRU::Generate(std::string OpName) -> std::string if (fType == "float") { // f_update_gate = input * weight_z^T size_t wz_offset = 3 * fAttrHiddenSize * input_size; - out << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &" + out << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &" << OpName << "_m, &" << OpName << "_k, &" << OpName << "_alpha, tensor_" << fNW << " + " << wz_offset << ", &" << OpName << "_k, " << OpName << "_input, &" << OpName << "_k, &" << OpName << "_beta, " << OpName << "_f_update_gate, &" << OpName << "_n);\n"; // f_reset_gate = input * weight_r^T size_t wr_offset = 3 * fAttrHiddenSize * input_size + fAttrHiddenSize * input_size; - out << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &" + out << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &" << OpName << "_m, &" << OpName << "_k, &" << OpName << "_alpha, tensor_" << fNW << " + " << wr_offset << ", &" << OpName << "_k, " << OpName << "_input, &" << OpName << "_k, &" << OpName << "_beta, " << OpName << "_f_reset_gate, &" << OpName << "_n);\n"; // f_hidden_gate = input * weight_h^T size_t wh_offset = 3 * fAttrHiddenSize * input_size + 2 * fAttrHiddenSize * input_size; - out << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &" + out << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &" << OpName << "_m, &" << OpName << "_k, &" << OpName << "_alpha, tensor_" << fNW << " + " << wh_offset << ", &" << OpName << "_k, " << OpName << "_input, &" << OpName << "_k, &" << OpName << "_beta, " << OpName << "_f_hidden_gate, &" << OpName << "_n);\n"; @@ -458,33 +458,33 @@ auto ROperator_GRU::Generate(std::string OpName) -> std::string if (direction == 0) { if (fType == "float") { // Add the bias of the weight to f_update_gate - out << SP << "BLAS::saxpy_(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << ", &" + out << SP << "Axpy(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << ", &" << OpName << "_incx, " << OpName << "_f_update_gate, &" << OpName << "_incy);\n"; // Add the bias of the recurrence to f_update_gate size_t rbz_offset = 3 * batch_size * seq_length * fAttrHiddenSize; - out << SP << "BLAS::saxpy_(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + " + out << SP << "Axpy(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + " << rbz_offset << ", &" << OpName << "_incx, " << OpName << "_f_update_gate, &" << OpName << "_incy);\n"; // Add the bias of the weight to f_reset_gate size_t wbr_offset = batch_size * seq_length * fAttrHiddenSize; - out << SP << "BLAS::saxpy_(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + " + out << SP << "Axpy(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + " << wbr_offset << ", &" << OpName << "_incx, " << OpName << "_f_reset_gate, &" << OpName << "_incy);\n"; // Add the bias of the recurrence to f_reset_gate // size_t rbr_offset = fAttrHiddenSize * fAttrHiddenSize + 3 * batch_size * fAttrHiddenSize; size_t rbr_offset = 4 * batch_size * seq_length * fAttrHiddenSize; - out << SP << "BLAS::saxpy_(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + " + out << SP << "Axpy(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + " << rbr_offset << ", &" << OpName << "_incx, " << OpName << "_f_reset_gate, &" << OpName << "_incy);\n"; // Add the bias of the weight to f_hidden_gate size_t wbh_offset = 2 * batch_size * seq_length * fAttrHiddenSize; - out << SP << "BLAS::saxpy_(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + " + out << SP << "Axpy(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + " << wbh_offset << ", &" << OpName << "_incx, " << OpName << "_f_hidden_gate, &" << OpName << "_incy);\n"; if (fAttrLinearBeforeReset == 0) { // Add the bias of the recurrence to f_hidden_gate size_t rbh_offset = 5 * batch_size * seq_length * fAttrHiddenSize; - out << SP << "BLAS::saxpy_(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB + out << SP << "Axpy(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + " << rbh_offset << ", &" << OpName << "_incx, " << OpName << "_f_hidden_gate, &" << OpName << "_incy);\n"; } @@ -493,34 +493,34 @@ auto ROperator_GRU::Generate(std::string OpName) -> std::string if (fType == "float") { // Add the bias of the weight to f_update_gate size_t wbz_offset = 6 * batch_size * seq_length * fAttrHiddenSize; - out << SP << "BLAS::saxpy_(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + " + out << SP << "Axpy(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + " << wbz_offset << ", &" << OpName << "_incx, " << OpName << "_f_update_gate, &" << OpName << "_incy);\n"; // Add the bias of the recurrence to f_update_gate // size_t rbz_offset = 3 * fAttrHiddenSize * fAttrHiddenSize + 3 * batch_size * fAttrHiddenSize; size_t rbz_offset = 9 * batch_size * seq_length * fAttrHiddenSize; - out << SP << "BLAS::saxpy_(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + " + out << SP << "Axpy(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + " << rbz_offset << ", &" << OpName << "_incx, " << OpName << "_f_update_gate, &" << OpName << "_incy);\n"; // Add the bias of the weight to f_reset_gate size_t wbr_offset = 7 * batch_size * seq_length * fAttrHiddenSize; - out << SP << "BLAS::saxpy_(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + " + out << SP << "Axpy(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + " << wbr_offset << ", &" << OpName << "_incx, " << OpName << "_f_reset_gate, &" << OpName << "_incy);\n"; // Add the bias of the recurrence to f_reset_gate size_t rbr_offset = 10 * batch_size * seq_length * fAttrHiddenSize; - out << SP << "BLAS::saxpy_(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + " + out << SP << "Axpy(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + " << rbr_offset << ", &" << OpName << "_incx, " << OpName << "_f_reset_gate, &" << OpName << "_incy);\n"; // Add the bias of the weight to f_hidden_gate size_t wbh_offset = 8 * batch_size * seq_length * fAttrHiddenSize; - out << SP << "BLAS::saxpy_(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + " + out << SP << "Axpy(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + " << wbh_offset << ", &" << OpName << "_incx, " << OpName << "_f_hidden_gate, &" << OpName << "_incy);\n"; if (fAttrLinearBeforeReset == 0) { // Add the bias of the recurrence to f_hidden_gate size_t rbh_offset = 11 * batch_size * seq_length * fAttrHiddenSize; - out << SP << "BLAS::saxpy_(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB + out << SP << "Axpy(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + " << rbh_offset << ", &" << OpName << "_incx, " << OpName << "_f_hidden_gate, &" << OpName << "_incy);\n"; } @@ -565,12 +565,12 @@ auto ROperator_GRU::Generate(std::string OpName) -> std::string if (!fNInitial_h.empty()) { if (direction == 0) { if (fType == "float") { - out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName + out << SP << SP << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << ", &" << OpName << "_n, " << OpName << "_initial_hidden_state, &" << OpName << "_n, &" << OpName << "_alpha, " << OpName << "_update_gate + offset, &" << OpName << "_n);\n"; size_t rr_offset = fAttrHiddenSize * fAttrHiddenSize; - out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName + out << SP << SP << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << rr_offset << ", &" << OpName << "_n, " << OpName << "_initial_hidden_state, &" << OpName << "_n, &" << OpName << "_alpha, " << OpName << "_reset_gate + offset, &" << OpName << "_n);\n"; @@ -578,12 +578,12 @@ auto ROperator_GRU::Generate(std::string OpName) -> std::string } else { // direction=1 if (fType == "float") { size_t rz_offset = 3 * fAttrHiddenSize * fAttrHiddenSize; - out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName + out << SP << SP << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << rz_offset << ", &" << OpName << "_n, " << OpName << "_initial_hidden_state, &" << OpName << "_n, &" << OpName << "_alpha, " << OpName << "_update_gate + offset, &" << OpName << "_n);\n"; size_t rr_offset = 4 * fAttrHiddenSize * fAttrHiddenSize; - out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName + out << SP << SP << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << rr_offset << ", &" << OpName << "_n, " << OpName << "_initial_hidden_state, &" << OpName << "_n, &" << OpName << "_alpha, " << OpName << "_reset_gate + offset, &" << OpName << "_n);\n"; @@ -601,12 +601,12 @@ auto ROperator_GRU::Generate(std::string OpName) -> std::string << num_directions * batch_size * fAttrHiddenSize << ";\n"; } if (fType == "float") { - out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName + out << SP << SP << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << ", &" << OpName << "_n, " << OpName << "_hidden_state + previous_offset, &" << OpName << "_n, &" << OpName << "_alpha, " << OpName << "_update_gate + offset, &" << OpName << "_n);\n"; size_t rr_offset = fAttrHiddenSize * fAttrHiddenSize; - out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName + out << SP << SP << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << rr_offset << ", &" << OpName << "_n, " << OpName << "_hidden_state + previous_offset, &" << OpName << "_n, &" << OpName << "_alpha, " << OpName << "_reset_gate + offset, &" << OpName << "_n);\n"; @@ -616,12 +616,12 @@ auto ROperator_GRU::Generate(std::string OpName) -> std::string << num_directions * batch_size * fAttrHiddenSize << " + " << batch_size * fAttrHiddenSize << ";\n"; if (fType == "float") { size_t rz_offset = 3 * fAttrHiddenSize * fAttrHiddenSize; - out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName + out << SP << SP << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << rz_offset << ", &" << OpName << "_n, " << OpName << "_hidden_state + previous_offset, &" << OpName << "_n, &" << OpName << "_alpha, " << OpName << "_update_gate + offset, &" << OpName << "_n);\n"; size_t rr_offset = 4 * fAttrHiddenSize * fAttrHiddenSize; - out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName + out << SP << SP << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << rr_offset << ", &" << OpName << "_n, " << OpName << "_hidden_state + previous_offset, &" << OpName << "_n, &" << OpName << "_alpha, " << OpName << "_reset_gate + offset, &" << OpName << "_n);\n"; @@ -781,7 +781,7 @@ auto ROperator_GRU::Generate(std::string OpName) -> std::string size_t rh_offset = (direction == 0) ? 2 * fAttrHiddenSize * fAttrHiddenSize : 3 * fAttrHiddenSize * fAttrHiddenSize + 2 * fAttrHiddenSize * fAttrHiddenSize; - out << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &" + out << SP << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &" << OpName << "_m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << rh_offset << ", &" << OpName << "_n, " << OpName << "_feedback, &" << OpName << "_n, &" << OpName << "_beta, " << OpName << "_feedback, &" << OpName << "_n);\n"; @@ -794,7 +794,7 @@ auto ROperator_GRU::Generate(std::string OpName) -> std::string out << SP << SP << "if (seq == 0) {\n"; if (!fNInitial_h.empty()) { // feedback = W * initial_hidden_state + bias - out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName + out << SP << SP << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &" << OpName << "_m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << rh_offset << ", &" << OpName << "_n, " << OpName << "_initial_hidden_state, &" << OpName << "_n, &" << OpName << "_beta, " << OpName << "_feedback, &" << OpName << "_n);\n"; @@ -813,7 +813,7 @@ auto ROperator_GRU::Generate(std::string OpName) -> std::string out << SP << SP << SP << "size_t previous_offset = (index + 1) * " << num_directions * batch_size * fAttrHiddenSize << " + " << batch_size * fAttrHiddenSize << ";\n"; } - out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName + out << SP << SP << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &" << OpName << "_m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << rh_offset << ", &" << OpName << "_n, " << OpName << "_hidden_state + previous_offset, &" << OpName << "_n, &" << OpName << "_beta, " << OpName << "_feedback, &" << OpName << "_n);\n"; @@ -823,7 +823,7 @@ auto ROperator_GRU::Generate(std::string OpName) -> std::string if (!fNB.empty()) { size_t rbh_offset = (direction == 0) ? 5 * batch_size * seq_length * fAttrHiddenSize : 11 * batch_size * seq_length * fAttrHiddenSize; - out << SP << SP << "BLAS::saxpy_(&" << OpName << "_feedback_size, &" << OpName << "_alpha, tensor_" << fNB + out << SP << SP << "Axpy(&" << OpName << "_feedback_size, &" << OpName << "_alpha, tensor_" << fNB << " + " << rbh_offset << ", &" << OpName << "_incx, " << OpName << "_feedback, &" << OpName << "_incy);\n"; } @@ -834,7 +834,7 @@ auto ROperator_GRU::Generate(std::string OpName) -> std::string } // hidden_gate = hidden_gate + feedback - out << SP << SP << "BLAS::saxpy_(&" << OpName << "_feedback_size, &" << OpName << "_alpha, " << OpName + out << SP << SP << "Axpy(&" << OpName << "_feedback_size, &" << OpName << "_alpha, " << OpName << "_feedback, &" << OpName << "_incx, " << OpName << "_hidden_gate + offset, &" << OpName << "_incy);\n"; // Clip the elements of the hidden gate into the range [-fClip, fClip] diff --git a/tmva/sofie/inc/TMVA/ROperator_Gemm.hxx b/tmva/sofie/inc/TMVA/ROperator_Gemm.hxx index e177ef97605ef..3c505afafc7c3 100644 --- a/tmva/sofie/inc/TMVA/ROperator_Gemm.hxx +++ b/tmva/sofie/inc/TMVA/ROperator_Gemm.hxx @@ -604,8 +604,6 @@ namespace SOFIE{ return out.str(); } - std::vector GetBlasRoutines() override { return {"Gemm", "Gemv"}; } - }; diff --git a/tmva/sofie/inc/TMVA/ROperator_LSTM.hxx b/tmva/sofie/inc/TMVA/ROperator_LSTM.hxx index 457fe22e2bdbf..8a302f1ef5e3f 100644 --- a/tmva/sofie/inc/TMVA/ROperator_LSTM.hxx +++ b/tmva/sofie/inc/TMVA/ROperator_LSTM.hxx @@ -162,9 +162,6 @@ template class ROperator_LSTM final : public ROperator { */ std::string GenerateSessionMembersCode(std::string opName) override; - /*! \brief Returns the blas routines needed to compile the generated code - */ - std::vector GetBlasRoutines() override { return { std::string("Gemm"), std::string("Axpy") }; } }; template @@ -192,6 +189,10 @@ auto ROperator_LSTM::ShapeInference(std::vector> input) - template auto ROperator_LSTM::Initialize(RModel &model) -> void { + // the gate updates emitted in Generate() use Gemm and Axpy + model.AddNeededHelperFunction("Gemm"); + model.AddNeededHelperFunction("Axpy"); + // Check the input and output tensors if (!model.CheckIfTensorAlreadyExist(fNX)) { throw std::runtime_error("TMVA SOFIE LSTM Op input tensor " + fNX + " is not found in model."); @@ -547,9 +548,9 @@ auto ROperator_LSTM::Generate(std::string OpName) -> std::string out << SP << "int " << OpName << "_incy = 1;\n"; } - auto emit_sgemm = [&](const std::string &out_name, size_t offset) -> std::string { + auto emit_gemm = [&](const std::string &out_name, size_t offset) -> std::string { std::stringstream ss; - ss << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &" << OpName + ss << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &" << OpName << "_m, &" << OpName << "_k, &" << OpName << "_alpha, tensor_" << fNW; if (offset != 0) @@ -564,24 +565,24 @@ auto ROperator_LSTM::Generate(std::string OpName) -> std::string if (direction == 0) { if (fType == "float") { // input_gate = input * weight_i^T - out << SP << emit_sgemm("ff_input_gate", 0); + out << SP << emit_gemm("ff_input_gate", 0); // output_gate = input * weight_o^T size_t wo_offset = fAttrHiddenSize * input_size; - out << SP << emit_sgemm("ff_output_gate", wo_offset); + out << SP << emit_gemm("ff_output_gate", wo_offset); // cell_gate = input * weight_c^T size_t wc_offset = 3 * fAttrHiddenSize * input_size; - out << SP << emit_sgemm("ff_cell_gate", wc_offset); + out << SP << emit_gemm("ff_cell_gate", wc_offset); } } else { if (fType == "float") { // input_gate = input * weight_i^T - out << SP << emit_sgemm("ff_input_gate", 4 * fAttrHiddenSize * input_size); + out << SP << emit_gemm("ff_input_gate", 4 * fAttrHiddenSize * input_size); // output_gate = input * weight_o^T size_t wo_offset = 4 * fAttrHiddenSize * input_size + 1 * fAttrHiddenSize * input_size; - out << SP << emit_sgemm("ff_output_gate", wo_offset); + out << SP << emit_gemm("ff_output_gate", wo_offset); // cell_gate = input * weight_c^T size_t wc_offset = 4 * fAttrHiddenSize * input_size + 3 * fAttrHiddenSize * input_size; - out << SP << emit_sgemm("ff_cell_gate", wc_offset); + out << SP << emit_gemm("ff_cell_gate", wc_offset); } } if (fAttrInputForget == 0) { @@ -589,12 +590,12 @@ auto ROperator_LSTM::Generate(std::string OpName) -> std::string if (direction == 0) { if (fType == "float") { size_t wf_offset = 2 * fAttrHiddenSize * input_size; - out << SP << emit_sgemm("ff_forget_gate", wf_offset); + out << SP << emit_gemm("ff_forget_gate", wf_offset); } } else { if (fType == "float") { size_t wf_offset = 4 * fAttrHiddenSize * input_size + 2 * fAttrHiddenSize * input_size; - out << SP << emit_sgemm("ff_forget_gate", wf_offset); + out << SP << emit_gemm("ff_forget_gate", wf_offset); } } } @@ -604,16 +605,16 @@ auto ROperator_LSTM::Generate(std::string OpName) -> std::string if (direction == 0) { if (fType == "float") { // ff_input_gate += bias_i - out << SP << "BLAS::saxpy_(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << ", &" + out << SP << "Axpy(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << ", &" << OpName << "_incx, " << OpName << "_ff_input_gate, &" << OpName << "_incy);\n"; // ff_output_gate += bias_o size_t bo_offset = seq_length * batch_size * fAttrHiddenSize; - out << SP << "BLAS::saxpy_(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + " + out << SP << "Axpy(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + " << bo_offset << ", &" << OpName << "_incx, " << OpName << "_ff_output_gate, &" << OpName << "_incy);\n"; // ff_cell_gate += bias_c size_t bc_offset = 3 * seq_length * batch_size * fAttrHiddenSize; - out << SP << "BLAS::saxpy_(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + " + out << SP << "Axpy(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + " << bc_offset << ", &" << OpName << "_incx, " << OpName << "_ff_cell_gate, &" << OpName << "_incy);\n"; } @@ -621,19 +622,19 @@ auto ROperator_LSTM::Generate(std::string OpName) -> std::string if (fType == "float") { // ff_input_gate += bias_i size_t bi_offset = 4 * seq_length * batch_size * fAttrHiddenSize; - out << SP << "BLAS::saxpy_(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + " + out << SP << "Axpy(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + " << bi_offset << ", &" << OpName << "_incx, " << OpName << "_ff_input_gate, &" << OpName << "_incy);\n"; // ff_output_gate += bias_o size_t bo_offset = 4 * seq_length * batch_size * fAttrHiddenSize + seq_length * batch_size * fAttrHiddenSize; - out << SP << "BLAS::saxpy_(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + " + out << SP << "Axpy(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + " << bo_offset << ", &" << OpName << "_incx, " << OpName << "_ff_output_gate, &" << OpName << "_incy);\n"; // ff_cell_gate += bias_c size_t bc_offset = 4 * num_directions * seq_length * batch_size * fAttrHiddenSize + 3 * seq_length * batch_size * fAttrHiddenSize; - out << SP << "BLAS::saxpy_(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + " + out << SP << "Axpy(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + " << bc_offset << ", &" << OpName << "_incx, " << OpName << "_ff_cell_gate, &" << OpName << "_incy);\n"; } @@ -643,7 +644,7 @@ auto ROperator_LSTM::Generate(std::string OpName) -> std::string if (direction == 0) { if (fType == "float") { size_t bo_offset = 2 * seq_length * batch_size * fAttrHiddenSize; - out << SP << "BLAS::saxpy_(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB + out << SP << "Axpy(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + " << bo_offset << ", &" << OpName << "_incx, " << OpName << "_ff_forget_gate, &" << OpName << "_incy);\n"; } @@ -651,7 +652,7 @@ auto ROperator_LSTM::Generate(std::string OpName) -> std::string if (fType == "float") { size_t bo_offset = 4 * seq_length * batch_size * fAttrHiddenSize + 2 * seq_length * batch_size * fAttrHiddenSize; - out << SP << "BLAS::saxpy_(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB + out << SP << "Axpy(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + " << bo_offset << ", &" << OpName << "_incx, " << OpName << "_ff_forget_gate, &" << OpName << "_incy);\n"; } @@ -701,23 +702,23 @@ auto ROperator_LSTM::Generate(std::string OpName) -> std::string if (!fNInitial_h.empty()) { if (direction == 0) { if (fType == "float") { - out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName + out << SP << SP << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << ", &" << OpName << "_n, " << OpName << "_initial_hidden_state, &" << OpName << "_n, &" << OpName << "_alpha, " << OpName << "_input_gate + offset, &" << OpName << "_n);\n"; size_t ro_offset = fAttrHiddenSize * fAttrHiddenSize; - out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName + out << SP << SP << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << ro_offset << ", &" << OpName << "_n, " << OpName << "_initial_hidden_state, &" << OpName << "_n, &" << OpName << "_alpha, " << OpName << "_output_gate + offset, &" << OpName << "_n);\n"; size_t rc_offset = 3 * fAttrHiddenSize * fAttrHiddenSize; - out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName + out << SP << SP << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << rc_offset << ", &" << OpName << "_n, " << OpName << "_initial_hidden_state, &" << OpName << "_n, &" << OpName << "_alpha, " << OpName << "_cell_gate + offset, &" << OpName << "_n);\n"; if (fAttrInputForget == 0) { size_t rf_offset = 2 * fAttrHiddenSize * fAttrHiddenSize; - out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" + out << SP << SP << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << rf_offset << ", &" << OpName << "_n, " << OpName << "_initial_hidden_state, &" << OpName << "_n, &" << OpName << "_alpha, " << OpName << "_forget_gate + offset, &" << OpName << "_n);\n"; @@ -726,23 +727,23 @@ auto ROperator_LSTM::Generate(std::string OpName) -> std::string } else { // direction=1 if (fType == "float") { size_t ri_offset = 4 * fAttrHiddenSize * fAttrHiddenSize; - out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName + out << SP << SP << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << ri_offset << ", &" << OpName << "_n, " << OpName << "_initial_hidden_state, &" << OpName << "_n, &" << OpName << "_alpha, " << OpName << "_input_gate + offset, &" << OpName << "_n);\n"; size_t ro_offset = 4 * fAttrHiddenSize * fAttrHiddenSize + 1 * fAttrHiddenSize * fAttrHiddenSize; - out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName + out << SP << SP << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << ro_offset << ", &" << OpName << "_n, " << OpName << "_initial_hidden_state, &" << OpName << "_n, &" << OpName << "_alpha, " << OpName << "_output_gate + offset, &" << OpName << "_n);\n"; size_t rc_offset = 4 * fAttrHiddenSize * fAttrHiddenSize + 3 * fAttrHiddenSize * fAttrHiddenSize; - out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName + out << SP << SP << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << rc_offset << ", &" << OpName << "_n, " << OpName << "_initial_hidden_state, &" << OpName << "_n, &" << OpName << "_alpha, " << OpName << "_cell_gate + offset, &" << OpName << "_n);\n"; if (fAttrInputForget == 0) { size_t rf_offset = 4 * fAttrHiddenSize * fAttrHiddenSize + 2 * fAttrHiddenSize * fAttrHiddenSize; - out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" + out << SP << SP << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << rf_offset << ", &" << OpName << "_n, " << OpName << "_initial_hidden_state, &" << OpName << "_n, &" << OpName << "_alpha, " << OpName << "_forget_gate + offset, &" << OpName << "_n);\n"; @@ -761,23 +762,23 @@ auto ROperator_LSTM::Generate(std::string OpName) -> std::string << num_directions * batch_size * fAttrHiddenSize << ";\n"; } if (fType == "float") { - out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName + out << SP << SP << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << ", &" << OpName << "_n, " << OpName << "_hidden_state + previous_offset, &" << OpName << "_n, &" << OpName << "_alpha, " << OpName << "_input_gate + offset, &" << OpName << "_n);\n"; size_t ro_offset = 1 * fAttrHiddenSize * fAttrHiddenSize; - out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName + out << SP << SP << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << ro_offset << ", &" << OpName << "_n, " << OpName << "_hidden_state + previous_offset, &" << OpName << "_n, &" << OpName << "_alpha, " << OpName << "_output_gate + offset, &" << OpName << "_n);\n"; size_t rc_offset = 3 * fAttrHiddenSize * fAttrHiddenSize; - out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName + out << SP << SP << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << rc_offset << ", &" << OpName << "_n, " << OpName << "_hidden_state + previous_offset, &" << OpName << "_n, &" << OpName << "_alpha, " << OpName << "_cell_gate + offset, &" << OpName << "_n);\n"; if (fAttrInputForget == 0) { size_t rf_offset = 2 * fAttrHiddenSize * fAttrHiddenSize; - out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName + out << SP << SP << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << rf_offset << ", &" << OpName << "_n, " << OpName << "_hidden_state + previous_offset, &" << OpName << "_n, &" << OpName << "_alpha, " << OpName << "_forget_gate + offset, &" << OpName << "_n);\n"; @@ -788,23 +789,23 @@ auto ROperator_LSTM::Generate(std::string OpName) -> std::string << num_directions * batch_size * fAttrHiddenSize << " + " << batch_size * fAttrHiddenSize << ";\n"; if (fType == "float") { size_t ri_offset = 4 * fAttrHiddenSize * fAttrHiddenSize; - out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName + out << SP << SP << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << ri_offset << ", &" << OpName << "_n, " << OpName << "_hidden_state + previous_offset, &" << OpName << "_n, &" << OpName << "_alpha, " << OpName << "_input_gate + offset, &" << OpName << "_n);\n"; size_t ro_offset = 4 * fAttrHiddenSize * fAttrHiddenSize + fAttrHiddenSize * fAttrHiddenSize; - out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName + out << SP << SP << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << ro_offset << ", &" << OpName << "_n, " << OpName << "_hidden_state + previous_offset, &" << OpName << "_n, &" << OpName << "_alpha, " << OpName << "_output_gate + offset, &" << OpName << "_n);\n"; size_t rc_offset = 4 * fAttrHiddenSize * fAttrHiddenSize + 3 * fAttrHiddenSize * fAttrHiddenSize; - out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName + out << SP << SP << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << rc_offset << ", &" << OpName << "_n, " << OpName << "_hidden_state + previous_offset, &" << OpName << "_n, &" << OpName << "_alpha, " << OpName << "_cell_gate + offset, &" << OpName << "_n);\n"; if (fAttrInputForget == 0) { size_t rf_offset = 4 * fAttrHiddenSize * fAttrHiddenSize + 2 * fAttrHiddenSize * fAttrHiddenSize; - out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName + out << SP << SP << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << rf_offset << ", &" << OpName << "_n, " << OpName << "_hidden_state + previous_offset, &" << OpName << "_n, &" << OpName << "_alpha, " << OpName << "_forget_gate + offset, &" << OpName << "_n);\n"; diff --git a/tmva/sofie/inc/TMVA/ROperator_LayerNormalization.hxx b/tmva/sofie/inc/TMVA/ROperator_LayerNormalization.hxx index 4e5a2203c4914..df1ec62784a97 100644 --- a/tmva/sofie/inc/TMVA/ROperator_LayerNormalization.hxx +++ b/tmva/sofie/inc/TMVA/ROperator_LayerNormalization.hxx @@ -329,8 +329,6 @@ public: return out.str(); } - std::vector GetBlasRoutines() override { return { std::string("Axpy") }; } - std::vector GetStdLibs() override { return { std::string("cmath") }; } }; diff --git a/tmva/sofie/inc/TMVA/ROperator_RNN.hxx b/tmva/sofie/inc/TMVA/ROperator_RNN.hxx index f8fd826605745..640ce1a2aeeb8 100644 --- a/tmva/sofie/inc/TMVA/ROperator_RNN.hxx +++ b/tmva/sofie/inc/TMVA/ROperator_RNN.hxx @@ -135,9 +135,6 @@ template class ROperator_RNN final : public ROperator { // generate code for Session data members (e.g. internal vectors) std::string GenerateSessionMembersCode(std::string opName) override; - /*! \brief Returns the blas routines needed to compile the generated code - */ - std::vector GetBlasRoutines() override { return { std::string("Gemm"), std::string("Axpy") }; } }; template @@ -163,6 +160,10 @@ auto ROperator_RNN::ShapeInference(std::vector> input) -> template auto ROperator_RNN::Initialize(RModel &model) -> void { + // the gate updates emitted in Generate() use Gemm and Axpy + model.AddNeededHelperFunction("Gemm"); + model.AddNeededHelperFunction("Axpy"); + // Check the input and output tensors if (!model.CheckIfTensorAlreadyExist(fNX)) { throw std::runtime_error("TMVA SOFIE RNN Op input tensor " + fNX + " is not found in model."); @@ -411,13 +412,13 @@ auto ROperator_RNN::Generate(std::string OpName) -> std::string // feedforward = input * W^T + bias if (fType == "float") { if (direction == 0) { - out << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &" + out << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &" << OpName << "_m, &" << OpName << "_k, &" << OpName << "_alpha, tensor_" << fNW << ", &" << OpName << "_k, " << OpName << "_input, &" << OpName << "_k, &" << OpName << "_beta, " << OpName << "_feedforward, &" << OpName << "_n);\n"; } else { out << SP << "size_t " << OpName << "_w_offset = " << fAttrHiddenSize * input_size << ";\n"; - out << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &" + out << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &" << OpName << "_m, &" << OpName << "_k, &" << OpName << "_alpha, tensor_" << fNW << " + " << OpName << "_w_offset, &" << OpName << "_k, " << OpName << "_input, &" << OpName << "_k, &" << OpName << "_beta, " << OpName << "_feedforward, &" << OpName << "_n);\n"; @@ -427,12 +428,12 @@ auto ROperator_RNN::Generate(std::string OpName) -> std::string if (!fNB.empty()) { if (fType == "float") { if (direction == 0) { - out << SP << "BLAS::saxpy_(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << ", &" + out << SP << "Axpy(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << ", &" << OpName << "_incx, " << OpName << "_feedforward, &" << OpName << "_incy);\n"; } else { out << SP << "size_t " << OpName << "_bias_offset = " << seq_length * batch_size * fAttrHiddenSize << ";\n"; - out << SP << "BLAS::saxpy_(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + " + out << SP << "Axpy(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + " << OpName << "_bias_offset, &" << OpName << "_incx, " << OpName << "_feedforward, &" << OpName << "_incy);\n"; } @@ -467,7 +468,7 @@ auto ROperator_RNN::Generate(std::string OpName) -> std::string out << SP << SP << SP << "size_t initial_hidden_state_offset = " << direction * batch_size * fAttrHiddenSize << ";\n"; if (fType == "float") { - out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName + out << SP << SP << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + r_offset, &" << OpName << "_n, " << OpName << "_initial_hidden_state + initial_hidden_state_offset, &" << OpName << "_n, &" << OpName << "_alpha, " << OpName << "_hidden_state + offset, &" << OpName << "_n);\n"; @@ -486,7 +487,7 @@ auto ROperator_RNN::Generate(std::string OpName) -> std::string << ";\n"; } if (fType == "float") { - out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName + out << SP << SP << SP << "Gemm(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + r_offset, &" << OpName << "_n, " << OpName << "_hidden_state + previous_offset, &" << OpName << "_n, &" << OpName << "_alpha, " << OpName << "_hidden_state + offset, &" << OpName << "_n);\n"; diff --git a/tmva/sofie/inc/TMVA/SOFIE_common.hxx b/tmva/sofie/inc/TMVA/SOFIE_common.hxx index d2f346e5f5c5e..88d97eae1d90f 100644 --- a/tmva/sofie/inc/TMVA/SOFIE_common.hxx +++ b/tmva/sofie/inc/TMVA/SOFIE_common.hxx @@ -513,13 +513,6 @@ std::vector ComputeStrideFromShape(const std::vector & shape); } // end namespace UTILITY -namespace BLAS{ -extern "C" void sgemm_(const char * transa, const char * transb, const int * m, const int * n, const int * k, - const float * alpha, const float * A, const int * lda, const float * B, const int * ldb, - const float * beta, float * C, const int * ldc); -}//BLAS - - //Utility functions to generate code void EmitNestedLoops(std::stringstream &out, size_t loopRank, const std::vector shape); void CloseNestedLoops(std::stringstream &out, size_t loopRank); @@ -539,19 +532,14 @@ struct HelperFunctionsCode { /// in `neededHelpers` (see RModel::AddNeededHelperFunction), resolving /// their inter-dependencies. Recognised keys are: "Im2col", "Im2col_3d", /// "col2im", "UnidirectionalBroadcast", "BroadcastConvBias", "Gemm_Call", -/// "Relu", "Fill", "Copy", "ReadTensorFromStream", "SafetensorsBlob", -/// "SafetensorsReader", "InputTensorDims", "DynamicMemory". +/// "Gemm", "Axpy", "Relu", "Fill", "Copy", "ReadTensorFromStream", +/// "SafetensorsBlob", "SafetensorsReader", "InputTensorDims", "DynamicMemory". /// /// `modelNamespace` (e.g. "TMVA_SOFIE_MyModel") is the generated model namespace; /// the Clad pullbacks are emitted into clad::custom_derivatives:: /// so the model stays differentiable without SOFIE_common.hxx / CladDerivator.h. -/// -/// `sgemmAlreadyDeclared`: set true if the caller already emitted the `extern "C"` -/// sgemm_ declaration (fNeededBlasRoutines block), so Gemm_Call skips its own and -/// avoids a duplicate. Default false emits it, keeping the returned code self-contained. HelperFunctionsCode GenerateHelperFunctionsCode(const std::set & neededHelpers, - const std::string & modelNamespace, - bool sgemmAlreadyDeclared = false); + const std::string & modelNamespace); } // namespace TMVA::Experimental::SOFIE diff --git a/tmva/sofie/src/RModel.cxx b/tmva/sofie/src/RModel.cxx index 52b40c676ca9f..c4c6f3b179692 100644 --- a/tmva/sofie/src/RModel.cxx +++ b/tmva/sofie/src/RModel.cxx @@ -235,7 +235,6 @@ void RModel::AddInputTensorName(std::string input_name) { void RModel::AddOperator(std::unique_ptr op, int order_execution) { - AddBlasRoutines(op->GetBlasRoutines()); auto libs = op->GetStdLibs(); auto op_input_tensors = op->GetOpInputTensors(); for (auto &stdlib : libs) { @@ -834,11 +833,7 @@ void RModel::InitializeSubGraph(std::shared_ptr graph) { // set the same options as parent model graph->fWeightFile = fWeightFile; graph->fUseWeightFile = fUseWeightFile; - // add needed blas routines and libs - std::vector blasRoutines; - for (auto & e : graph->fNeededBlasRoutines) - blasRoutines.push_back(e); - AddBlasRoutines(blasRoutines); + // add needed std libs for (auto e : graph->fNeededStdLib) AddNeededStdLib(e); // helper functions used by the subgraph must be emitted in the top-level @@ -2186,27 +2181,6 @@ void RModel::GenerateHeaderInfo(std::string& hgname) { fGC += "#include \n"; fGC += "\nnamespace TMVA_SOFIE_" + fName + "{\n"; - if (!fNeededBlasRoutines.empty()) { - fGC += ("namespace BLAS{\n"); - for (auto &routine : fNeededBlasRoutines) { - if (routine == "Gemm") { - fGC += ("\textern \"C\" void sgemm_(const char * transa, const char * transb, const int * m, const int * n, const int * k,\n" - "\t const float * alpha, const float * A, const int * lda, const float * B, const int * ldb,\n" - "\t const float * beta, float * C, const int * ldc);\n"); - // sgemm_ now declared; the standalone Gemm_Call helper will skip its copy. - fBlasSgemmDeclared = true; - } else if (routine == "Gemv") { - fGC += ("\textern \"C\" void sgemv_(const char * trans, const int * m, const int * n, const float * alpha, const float * A,\n" - "\t const int * lda, const float * X, const int * incx, const float * beta, const float * Y, const int * incy);\n"); - } else if (routine == "Axpy") { - fGC += ("\textern \"C\" void saxpy_(const int * n, const float * alpha, const float * x,\n" - "\t const int * incx, float * y, const int * incy);\n"); - } else if (routine == "Copy") { - fGC += ("\textern \"C\" void scopy_(const int *n, const float* x, const int *incx, float* y, const int* incy);\n"); - } - } - fGC += ("}//BLAS\n"); - } // Placeholder for the standalone definitions of the inference helper // functions used by this model (filled in by EmitHelperFunctionsCode). It // sits inside the generated model namespace, right before the session code. @@ -2216,7 +2190,7 @@ void RModel::GenerateHeaderInfo(std::string& hgname) { void RModel::EmitHelperFunctionsCode() { HelperFunctionsCode code = - GenerateHelperFunctionsCode(fNeededHelperFunctions, "TMVA_SOFIE_" + fName, fBlasSgemmDeclared); + GenerateHelperFunctionsCode(fNeededHelperFunctions, "TMVA_SOFIE_" + fName); auto replaceMarker = [this](const std::string &marker, const std::string &replacement) { auto pos = fGC.find(marker); diff --git a/tmva/sofie/src/SOFIE_common_helpers.cxx b/tmva/sofie/src/SOFIE_common_helpers.cxx index 0e804c91fe329..8ed4a8717659f 100644 --- a/tmva/sofie/src/SOFIE_common_helpers.cxx +++ b/tmva/sofie/src/SOFIE_common_helpers.cxx @@ -15,17 +15,97 @@ namespace { // Standalone helper snippets, each emitted verbatim inside the generated model // namespace. They depend only on the C++ stdlib (headers collected in -// GenerateHelperFunctionsCode) and, for Gemm, on the local BLAS::sgemm_ below. - -// extern "C" declaration of the BLAS routine Gemm_Call needs, in a nested BLAS -// namespace. Skipped when the caller already declared sgemm_ (see the -// sgemmAlreadyDeclared parameter of GenerateHelperFunctionsCode). -constexpr const char *kBlasSgemm = R"SOFIE( -namespace BLAS { -extern "C" void sgemm_(const char *transa, const char *transb, const int *m, const int *n, const int *k, - const float *alpha, const float *A, const int *lda, const float *B, const int *ldb, - const float *beta, float *C, const int *ldc); -} // namespace BLAS +// GenerateHelperFunctionsCode); the matrix routines below are self-contained +// reference implementations, so the generated code has no BLAS dependency. + +// Reference implementation of the BLAS sgemm contract: +// C := alpha * op(A) * op(B) + beta * C, op(X) = X or X^T +// with A, B, C in BLAS column-major storage and the given leading dimensions. +// It is a drop-in replacement for the Fortran sgemm_ symbol: all SOFIE +// operators call it with the same arguments they used to pass to BLAS (SOFIE +// tensors are row-major, so the call sites hand over the operands swapped — +// that stays unchanged here). Unlike BLAS, it has no quick return for +// alpha == 0, so an Inf or NaN in A or B still propagates to C then. For a +// non-transposed A, the j-l-i loop nest walks C column by column with unit +// stride so the compiler can auto-vectorize the innermost loop; a transposed A +// uses a dot-product nest instead (see below). +constexpr const char *kGemmRef = R"SOFIE( +inline void Gemm(const char *transa, const char *transb, const int *m, const int *n, const int *k, + const float *alpha, const float *A, const int *lda, const float *B, const int *ldb, + const float *beta, float *C, const int *ldc) +{ + const bool ta = *transa == 't' || *transa == 'T' || *transa == 'c' || *transa == 'C'; + const bool tb = *transb == 't' || *transb == 'T' || *transb == 'c' || *transb == 'C'; + const int M = *m, N = *n, K = *k; + const int LDA = *lda, LDB = *ldb, LDC = *ldc; + const float alpha_ = *alpha, beta_ = *beta; + for (int j = 0; j < N; ++j) { + float *Cj = C + j * LDC; + // beta == 0 must overwrite C without reading it (BLAS semantics), so + // that a C filled with NaNs or left uninitialized is not propagated. + if (beta_ == 0.0f) { + for (int i = 0; i < M; ++i) + Cj[i] = 0.0f; + } else if (beta_ != 1.0f) { + for (int i = 0; i < M; ++i) + Cj[i] *= beta_; + } + if (!ta) { + // accumulate over l into a contiguous C column: unit stride on A, C + for (int l = 0; l < K; ++l) { + const float ab = alpha_ * (tb ? B[j + l * LDB] : B[l + j * LDB]); + const float *Al = A + l * LDA; + for (int i = 0; i < M; ++i) + Cj[i] += ab * Al[i]; + } + } else { + // transposed A: switch to a dot-product nest so the innermost loop + // reads both a row of A and a column of B with unit stride (the + // B column stays in cache across the i loop). The 8 independent + // accumulators vectorize and keep the FMA latency hidden: a single + // accumulator would be latency-bound at ~1 add/cycle. + for (int i = 0; i < M; ++i) { + const float *Ai = A + i * LDA; + const float *Bj = tb ? B + j : B + j * LDB; + const int strideB = tb ? LDB : 1; + float sum; + if (strideB == 1) { + float acc[8] = {}; + int l = 0; + for (; l + 8 <= K; l += 8) + for (int u = 0; u < 8; ++u) + acc[u] += Ai[l + u] * Bj[l + u]; + sum = ((acc[0] + acc[1]) + (acc[2] + acc[3])) + ((acc[4] + acc[5]) + (acc[6] + acc[7])); + for (; l < K; ++l) + sum += Ai[l] * Bj[l]; + } else { + sum = 0.0f; + for (int l = 0; l < K; ++l) + sum += Ai[l] * Bj[l * strideB]; + } + Cj[i] += alpha_ * sum; + } + } + } +} +)SOFIE"; + +// Reference implementation of the BLAS saxpy contract: +// y += alpha * x (elementwise, with BLAS vector strides) +// Unlike BLAS, it only supports positive strides, which is all SOFIE uses. +constexpr const char *kAxpyRef = R"SOFIE( +inline void Axpy(const int *n, const float *alpha, const float *x, const int *incx, float *y, const int *incy) +{ + const int N = *n; + const float alpha_ = *alpha; + if (*incx == 1 && *incy == 1) { + for (int i = 0; i < N; ++i) + y[i] += alpha_ * x[i]; + } else { + for (int i = 0, px = 0, py = 0; i < N; ++i, px += *incx, py += *incy) + y[py] += alpha_ * x[px]; + } +} )SOFIE"; constexpr const char *kConvertShapeToLength = R"SOFIE( @@ -62,7 +142,7 @@ inline bool is_a_ge_zero_and_a_lt_b(int a, int b) } )SOFIE"; -// im2col: re-arrange convolution input into a matrix usable directly by BLAS. +// im2col: re-arrange convolution input into a matrix usable directly by Gemm. // It loops over each element of the filtered region first, following the input // layout, so reads/writes stay consecutive in memory; the result is already // transposed -- a (channels*kernel_h*kernel_w , output_h*output_w) matrix. @@ -357,14 +437,16 @@ inline void Gemm_Call(float *output, bool transa, bool transb, int m, int n, int if (C != nullptr) { std::copy(C, C + m * n, output); } - BLAS::sgemm_(transa ? &ct : &cn, transb ? &ct : &cn, &m, &n, &k, &alpha, A, lda, B, ldb, &beta, output, ldc); + Gemm(transa ? &ct : &cn, transb ? &ct : &cn, &m, &n, &k, &alpha, A, lda, B, ldb, &beta, output, ldc); } )SOFIE"; // Custom Clad reverse-mode pullbacks for the helpers (they used to live in -// Math/CladDerivator.h). Gemm_Call and Copy need a hand-written pullback because -// their bodies call bodyless routines (sgemm_, std::copy -> memmove) that Clad -// cannot differentiate; Fill and Relu are included for completeness so a +// Math/CladDerivator.h). Gemm_Call gets a hand-written pullback in terms of +// Gemm_Call itself, which is much cheaper than the (correct but tape-heavy) +// code Clad would generate for the loops in Gemm. Copy needs one because its +// body calls a bodyless routine (std::copy -> memmove) that Clad cannot +// differentiate; Fill and Relu are included for completeness so a // differentiated model needs neither SOFIE_common.hxx nor CladDerivator.h. How // they are placed and found is explained at the emission site // (GenerateHelperFunctionsCode). @@ -1158,7 +1240,7 @@ inline MemoryResult OrganizeMemory(const std::vector &tensorsInf } // anonymous namespace HelperFunctionsCode GenerateHelperFunctionsCode(const std::set &neededHelpers, - const std::string &modelNamespace, bool sgemmAlreadyDeclared) + const std::string &modelNamespace) { auto need = [&](const char *key) { return neededHelpers.count(key) > 0; }; @@ -1168,6 +1250,9 @@ HelperFunctionsCode GenerateHelperFunctionsCode(const std::set &nee const bool uniBroadcast = need("UnidirectionalBroadcast"); const bool convBias = need("BroadcastConvBias"); const bool gemm = need("Gemm_Call"); + // Gemm_Call forwards to Gemm, so requesting Gemm_Call pulls it in too. + const bool needGemm = gemm || need("Gemm"); + const bool needAxpy = need("Axpy"); const bool relu = need("Relu"); const bool fill = need("Fill"); const bool copy = need("Copy"); @@ -1214,9 +1299,10 @@ HelperFunctionsCode GenerateHelperFunctionsCode(const std::set &nee std::string defs; defs += "\n// --- Standalone SOFIE inference helper functions ---\n"; - // sgemm_ declaration for Gemm_Call, unless the caller already emitted one. - if (gemm && !sgemmAlreadyDeclared) - defs += kBlasSgemm; + if (needAxpy) + defs += kAxpyRef; + if (needGemm) + defs += kGemmRef; if (needConvertLength) defs += kConvertShapeToLength; @@ -1262,8 +1348,8 @@ HelperFunctionsCode GenerateHelperFunctionsCode(const std::set &nee defs += "// --- End of SOFIE inference helper functions ---\n\n"; // ---- Clad custom derivatives (pullbacks and pushforwards) -------------- - // Some helpers cannot be differentiated automatically by Clad (Gemm_Call - // calls the bodyless BLAS routine sgemm_). For those we emit a hand-written + // Some helpers cannot be differentiated efficiently or at all by Clad + // (see the comment at kGemmCallPullback). For those we emit a hand-written // pullback (reverse mode) and pushforward (forward mode). Clad looks up // custom derivatives in // clad::custom_derivatives::, so they are placed diff --git a/tmva/sofie/test/CMakeLists.txt b/tmva/sofie/test/CMakeLists.txt index 2826b6b823c25..e2198f3631a1e 100644 --- a/tmva/sofie/test/CMakeLists.txt +++ b/tmva/sofie/test/CMakeLists.txt @@ -113,30 +113,26 @@ if (ONNX_MODELS_DIR) FIXTURES_REQUIRED ${alias_test_fixtures}) # Creating a Google Test - if (BLAS_FOUND) # we need BLAS for compiling the models - ROOT_EXECUTABLE(TestCustomModelsFromONNX TestCustomModelsFromONNX.cxx + ROOT_EXECUTABLE(TestCustomModelsFromONNX TestCustomModelsFromONNX.cxx + LIBRARIES Core GTest::gtest GTest::gtest_main + ) + ROOTTEST_ADD_TEST(TestCustomModelsFromONNX + EXEC ./TestCustomModelsFromONNX + FIXTURES_REQUIRED sofie-compile-models-onnx) + + if (clad) + ROOT_EXECUTABLE(TestCladAutodiff TestCladAutodiff.cxx LIBRARIES Core GTest::gtest GTest::gtest_main ) - ROOTTEST_ADD_TEST(TestCustomModelsFromONNX - EXEC ./TestCustomModelsFromONNX + ROOTTEST_ADD_TEST(TestCladAutodiff + EXEC ./TestCladAutodiff FIXTURES_REQUIRED sofie-compile-models-onnx) - - if (clad) - ROOT_EXECUTABLE(TestCladAutodiff TestCladAutodiff.cxx - LIBRARIES Core GTest::gtest GTest::gtest_main - ) - ROOTTEST_ADD_TEST(TestCladAutodiff - EXEC ./TestCladAutodiff - FIXTURES_REQUIRED sofie-compile-models-onnx) - endif() endif() endif() -if (BLAS_FOUND) - if (clad) - # Creating a Google Test for the automatic differentiation of Gemm_Call - ROOT_ADD_GTEST(TestGemmDerivative TestGemmDerivative.cxx LIBRARIES Core BLAS::BLAS ROOTTMVASofie) - endif() +if (clad) + # Creating a Google Test for the automatic differentiation of Gemm_Call + ROOT_ADD_GTEST(TestGemmDerivative TestGemmDerivative.cxx LIBRARIES Core ROOTTMVASofie) endif() # Unit tests for the ONNX parser layer: the tests write the model files @@ -157,15 +153,12 @@ if (ROOT_TORCH_FOUND AND ROOT_ONNX_FOUND AND NOT broken_onnx) configure_file(${generator}ModelGenerator.py ${generator}ModelGenerator.py COPYONLY) endforeach() - if (BLAS_FOUND) - ROOT_ADD_GTEST(TestSofieModels TestSofieModels.cxx - LIBRARIES - ROOTTMVASofie - ROOTTMVASofieParser - BLAS::BLAS - INCLUDE_DIRS - ${CMAKE_CURRENT_BINARY_DIR} - ) - endif() + ROOT_ADD_GTEST(TestSofieModels TestSofieModels.cxx + LIBRARIES + ROOTTMVASofie + ROOTTMVASofieParser + INCLUDE_DIRS + ${CMAKE_CURRENT_BINARY_DIR} + ) endif() diff --git a/tmva/sofie/test/TestCladAutodiff.cxx b/tmva/sofie/test/TestCladAutodiff.cxx index 5cc18f2d3a349..b7d6301715c1a 100644 --- a/tmva/sofie/test/TestCladAutodiff.cxx +++ b/tmva/sofie/test/TestCladAutodiff.cxx @@ -174,8 +174,8 @@ float sig_outer(TMVA_SOFIE_LinearWithSigmoid::Session const &session, float cons // H * dinput (in the input adjoint) and the gradient (in the dinput adjoint). // Both pointer arguments must be requested as active: for a non-varied // argument clad would look up the custom Gemm_Call pullback with a -// reduced signature, not find it, and silently fall back to differentiating -// the BLAS call, which yields zero. +// reduced signature, not find it, and fall back to differentiating +// Gemm_Call and the loops in Gemm instead of using the custom pullback. float sig_dir(TMVA_SOFIE_LinearWithSigmoid::Session const &session, TMVA_SOFIE_LinearWithSigmoid::Session const &zeroSession, float const *input, float const *dinput) { diff --git a/tutorials/CMakeLists.txt b/tutorials/CMakeLists.txt index dfafbb86291ef..361382d37821f 100644 --- a/tutorials/CMakeLists.txt +++ b/tutorials/CMakeLists.txt @@ -362,11 +362,6 @@ else() ROOT_FIND_PYTHON_MODULE(keras) ROOT_FIND_PYTHON_MODULE(onnx) - - if (NOT BLAS_FOUND) - list(APPEND tmva_veto machine_learning/TMVA_SOFIE_RDataFrame.C) - list(APPEND tmva_veto machine_learning/TMVA_SOFIE_RSofieReader.C) - endif() # These SOFIE tutorials take the ONNX HiggsModel trained with PyTorch as input if (NOT ROOT_TORCH_FOUND OR NOT ROOT_ONNX_FOUND) list(APPEND tmva_veto machine_learning/TMVA_SOFIE_PyTorch_HiggsModel.py) From 07a9e82e26168fd907f4ef4285e1bcd091c70b1b Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Fri, 20 Nov 2026 22:05:36 +0100 Subject: [PATCH 3/3] [tmva][sofie] Make it possible to use BLAS in the generated code again The built-in Gemm and Axpy are as fast as BLAS for per-event inference, but much slower for large batches, so some users prefer the BLAS version that SOFIE used before. It can now be requested with RModel::SetUseBLAS(true) before Generate(), like the other code generation options (e.g. SetUseVDT and SetOptimizationLevel). The default is still the built-in implementation without BLAS dependency. With BLAS, Gemm and Axpy are emitted as forwarders to the BLAS routines sgemm_ and saxpy_ instead of as reference implementations. The two have the same calling convention, so the operators don't need to know which variant they call, and Gemm_Call and with it all operators follow the choice. Like before, the generated code then has to be linked against a BLAS library with the usual 32-bit integer interface, or one has to be loaded in the ROOT session. Clad treats the BLAS routines as constants and only warns about it, so gradients through them are silently zero. With BLAS, only the operators that use Gemm_Call (Gemm and Conv), which has hand-written derivatives, give correct gradients, and only if Clad uses these derivatives. This is documented at RModel::SetUseBLAS. Testing: * TestSofieParser checks that a Gemm model calls BLAS if and only if it is requested. * The new TestSofieBLAS compiles the emitted Gemm and Axpy with cling and compares them with a naive implementation of the BLAS contract: for all transpositions, in upper case and in the lower case that Gemm_Call uses, for a range of shapes, leading dimensions and alpha/beta values, and with a C that must not be read because beta == 0. If BLAS is found, the test is also linked against it and covers the variant that calls BLAS, and BLAS itself as a check of the naive implementation. For that, CMake looks for BLAS again as an optional testing dependency of SOFIE, without the configuration error if it is not found. Assisted-by: claude-code:claude-opus-5-5 --- README/ReleaseNotes/v644/index.md | 2 +- cmake/modules/SearchInstalledSoftware.cmake | 9 + tmva/sofie/README.md | 22 +- tmva/sofie/inc/TMVA/RModel.hxx | 13 ++ tmva/sofie/inc/TMVA/SOFIE_common.hxx | 5 +- tmva/sofie/src/RModel.cxx | 2 +- tmva/sofie/src/SOFIE_common_helpers.cxx | 41 +++- tmva/sofie/test/CMakeLists.txt | 8 + tmva/sofie/test/TestSofieBLAS.cxx | 226 ++++++++++++++++++++ tmva/sofie/test/TestSofieParser.cxx | 21 +- 10 files changed, 339 insertions(+), 10 deletions(-) create mode 100644 tmva/sofie/test/TestSofieBLAS.cxx diff --git a/README/ReleaseNotes/v644/index.md b/README/ReleaseNotes/v644/index.md index 13c581110269d..96e700bf485c8 100644 --- a/README/ReleaseNotes/v644/index.md +++ b/README/ReleaseNotes/v644/index.md @@ -47,7 +47,7 @@ RenderCore, the WebGL engine of the REve client, is fetched at build time as the ### SOFIE inference code without BLAS dependency -The C++ code that SOFIE generates for the inference of ONNX models doesn't call BLAS anymore: the matrix multiplications are done by functions that are emitted into the generated header itself. The generated code is therefore self-contained and doesn't need to be linked against a BLAS library. For the per-event inference that SOFIE is mostly used for, for example in RDataFrame event loops, this is as fast as BLAS, but large batches are evaluated considerably slower. +The C++ code that SOFIE generates for the inference of ONNX models doesn't call BLAS anymore: the matrix multiplications are done by functions that are emitted into the generated header itself. The generated code is therefore self-contained and doesn't need to be linked against a BLAS library. For the per-event inference that SOFIE is mostly used for, for example in RDataFrame event loops, this is as fast as BLAS, but large batches are evaluated considerably slower. If you prefer BLAS, call `RModel::SetUseBLAS(true)` before `RModel::Generate()`: then the generated code calls BLAS like before, and it has to be linked against a BLAS library again. ## I/O diff --git a/cmake/modules/SearchInstalledSoftware.cmake b/cmake/modules/SearchInstalledSoftware.cmake index 0b24dce676ee7..b0484bedc6466 100644 --- a/cmake/modules/SearchInstalledSoftware.cmake +++ b/cmake/modules/SearchInstalledSoftware.cmake @@ -937,6 +937,15 @@ if(experimental_adaptivecpp) endif() endif() +#---Check for optional TMVA-SOFIE testing dependency (BLAS)------------------------------- +# The code generated by SOFIE only calls BLAS if asked to (RModel::SetUseBLAS). +# If a BLAS library is found, the SOFIE tests also cover that mode. + +if(tmva AND testing) + message(STATUS "Looking for BLAS as an optional testing dependency of TMVA-SOFIE") + find_package(BLAS) +endif() + #---Figure out if TMVA CPU should be built and which BLAS we will use ------------------ if(tmva-cpu) if (NOT tmva) diff --git a/tmva/sofie/README.md b/tmva/sofie/README.md index 70769ec0b5a2e..557eb5722a98d 100644 --- a/tmva/sofie/README.md +++ b/tmva/sofie/README.md @@ -11,7 +11,8 @@ This is a new development in TMVA and is currently in early experimental stage. None: the generated inference code is self-contained C++ (the matrix multiplications it contains are reference implementations emitted into the generated header itself, so no BLAS or other external library has to be -linked to run it). +linked to run it). Optionally, the matrix multiplications can be done with BLAS +instead, see below. ## Installation @@ -80,6 +81,25 @@ and can be inspected with the standard Python and Rust safetensors tooling: model.Generate(Options::kSafetensorsWeightFile); ``` +The matrix multiplications in the generated code are done by built-in reference +implementations, which are as fast as BLAS for the per-event inference SOFIE is +mostly used for, but much slower for large batches. If you prefer BLAS, request +it before generating the code: + +```c++ +model.SetUseBLAS(true); +model.Generate(); +``` + +The generated code then calls the BLAS routines `sgemm_` and `saxpy_`, so it has +to be linked against a BLAS library such as OpenBLAS, or one has to be loaded in +the ROOT session (e.g. with `gSystem->Load("libopenblas")`) before the generated +code is used. The generated code declares these routines itself, with 32-bit +integer arguments, so the BLAS library must have this usual interface (not the +64-bit integer one of ILP64 builds), and the generated code can't be compiled +together with headers that declare these routines differently, like OpenBLAS's +`f77blas.h`. + SOFIE also supports generating inference code with RDataFrame as inputs, refer to the tutorials below for examples. ## Supported ONNX operators diff --git a/tmva/sofie/inc/TMVA/RModel.hxx b/tmva/sofie/inc/TMVA/RModel.hxx index cf4deac971ea1..c2ed50041312a 100644 --- a/tmva/sofie/inc/TMVA/RModel.hxx +++ b/tmva/sofie/inc/TMVA/RModel.hxx @@ -77,6 +77,7 @@ private: bool fIsInitialized = false; bool fIsSubGraph = false; bool fUseVDT = false; + bool fUseBLAS = false; int fVerbose = 0; int fBatchSize = -1; size_t fConstantTensorSize = 0; // size (in Bytes) of the allocated constant tensors @@ -342,6 +343,18 @@ public: } bool UseVDT() const { return fUseVDT;} + // flag to do the matrix multiplications in the generated code with the BLAS + // routines sgemm_ and saxpy_ instead of the built-in implementations (the + // default), which is faster for large batches. The generated code then has to + // be linked against a BLAS library, or one has to be loaded in the ROOT + // session. Clad treats the BLAS routines as constants and only warns about + // it, so gradients through them are silently zero: then only the operators + // that use Gemm_Call (Gemm and Conv), which has hand-written derivatives, give + // correct gradients, and only if Clad uses these derivatives. Has to be set + // before Generate(). + void SetUseBLAS(bool on) { fUseBLAS = on; } + bool UseBLAS() const { return fUseBLAS; } + // RModel is an internal representation that doesn't support ROOT IO (if you need model IO, use ONNX directly). ClassDefNV(RModel, 0); }; diff --git a/tmva/sofie/inc/TMVA/SOFIE_common.hxx b/tmva/sofie/inc/TMVA/SOFIE_common.hxx index 88d97eae1d90f..3c91ee109d0f6 100644 --- a/tmva/sofie/inc/TMVA/SOFIE_common.hxx +++ b/tmva/sofie/inc/TMVA/SOFIE_common.hxx @@ -538,8 +538,11 @@ struct HelperFunctionsCode { /// `modelNamespace` (e.g. "TMVA_SOFIE_MyModel") is the generated model namespace; /// the Clad pullbacks are emitted into clad::custom_derivatives:: /// so the model stays differentiable without SOFIE_common.hxx / CladDerivator.h. +/// +/// `useBLAS`: make "Gemm" and "Axpy" call the BLAS routines sgemm_ and +/// saxpy_ instead of being reference implementations (see RModel::SetUseBLAS). HelperFunctionsCode GenerateHelperFunctionsCode(const std::set & neededHelpers, - const std::string & modelNamespace); + const std::string & modelNamespace, bool useBLAS = false); } // namespace TMVA::Experimental::SOFIE diff --git a/tmva/sofie/src/RModel.cxx b/tmva/sofie/src/RModel.cxx index c4c6f3b179692..c2f42edac1841 100644 --- a/tmva/sofie/src/RModel.cxx +++ b/tmva/sofie/src/RModel.cxx @@ -2190,7 +2190,7 @@ void RModel::GenerateHeaderInfo(std::string& hgname) { void RModel::EmitHelperFunctionsCode() { HelperFunctionsCode code = - GenerateHelperFunctionsCode(fNeededHelperFunctions, "TMVA_SOFIE_" + fName); + GenerateHelperFunctionsCode(fNeededHelperFunctions, "TMVA_SOFIE_" + fName, fUseBLAS); auto replaceMarker = [this](const std::string &marker, const std::string &replacement) { auto pos = fGC.find(marker); diff --git a/tmva/sofie/src/SOFIE_common_helpers.cxx b/tmva/sofie/src/SOFIE_common_helpers.cxx index 8ed4a8717659f..02a2dcb154f50 100644 --- a/tmva/sofie/src/SOFIE_common_helpers.cxx +++ b/tmva/sofie/src/SOFIE_common_helpers.cxx @@ -16,7 +16,8 @@ namespace { // Standalone helper snippets, each emitted verbatim inside the generated model // namespace. They depend only on the C++ stdlib (headers collected in // GenerateHelperFunctionsCode); the matrix routines below are self-contained -// reference implementations, so the generated code has no BLAS dependency. +// reference implementations, so the generated code has no BLAS dependency, +// unless the model asks for BLAS (RModel::SetUseBLAS, see kGemmBlas). // Reference implementation of the BLAS sgemm contract: // C := alpha * op(A) * op(B) + beta * C, op(X) = X or X^T @@ -108,6 +109,38 @@ inline void Axpy(const int *n, const float *alpha, const float *x, const int *in } )SOFIE"; +// Gemm and Axpy for models generated with RModel::SetUseBLAS(true): +// instead of the reference implementations above, they forward to the BLAS +// routines sgemm_ and saxpy_, which have the same calling convention. The +// generated code then has to be linked against a BLAS library. +constexpr const char *kGemmBlas = R"SOFIE( +namespace BLAS { +extern "C" void sgemm_(const char *transa, const char *transb, const int *m, const int *n, const int *k, + const float *alpha, const float *A, const int *lda, const float *B, const int *ldb, + const float *beta, float *C, const int *ldc); +} // namespace BLAS + +// Calls BLAS, because the model was generated with RModel::SetUseBLAS(true). +inline void Gemm(const char *transa, const char *transb, const int *m, const int *n, const int *k, + const float *alpha, const float *A, const int *lda, const float *B, const int *ldb, + const float *beta, float *C, const int *ldc) +{ + BLAS::sgemm_(transa, transb, m, n, k, alpha, A, lda, B, ldb, beta, C, ldc); +} +)SOFIE"; + +constexpr const char *kAxpyBlas = R"SOFIE( +namespace BLAS { +extern "C" void saxpy_(const int *n, const float *alpha, const float *x, const int *incx, float *y, const int *incy); +} // namespace BLAS + +// Calls BLAS, because the model was generated with RModel::SetUseBLAS(true). +inline void Axpy(const int *n, const float *alpha, const float *x, const int *incx, float *y, const int *incy) +{ + BLAS::saxpy_(n, alpha, x, incx, y, incy); +} +)SOFIE"; + constexpr const char *kConvertShapeToLength = R"SOFIE( inline std::size_t ConvertShapeToLength(const std::vector &shape) { @@ -1240,7 +1273,7 @@ inline MemoryResult OrganizeMemory(const std::vector &tensorsInf } // anonymous namespace HelperFunctionsCode GenerateHelperFunctionsCode(const std::set &neededHelpers, - const std::string &modelNamespace) + const std::string &modelNamespace, bool useBLAS) { auto need = [&](const char *key) { return neededHelpers.count(key) > 0; }; @@ -1300,9 +1333,9 @@ HelperFunctionsCode GenerateHelperFunctionsCode(const std::set &nee defs += "\n// --- Standalone SOFIE inference helper functions ---\n"; if (needAxpy) - defs += kAxpyRef; + defs += useBLAS ? kAxpyBlas : kAxpyRef; if (needGemm) - defs += kGemmRef; + defs += useBLAS ? kGemmBlas : kGemmRef; if (needConvertLength) defs += kConvertShapeToLength; diff --git a/tmva/sofie/test/CMakeLists.txt b/tmva/sofie/test/CMakeLists.txt index e2198f3631a1e..f7b25f03848c6 100644 --- a/tmva/sofie/test/CMakeLists.txt +++ b/tmva/sofie/test/CMakeLists.txt @@ -135,6 +135,14 @@ if (clad) ROOT_ADD_GTEST(TestGemmDerivative TestGemmDerivative.cxx LIBRARIES Core ROOTTMVASofie) endif() +# The matrix routines of the generated code against a naive implementation. If +# BLAS is found, that includes the ones that call BLAS (RModel::SetUseBLAS(true)). +ROOT_ADD_GTEST(TestSofieBLAS TestSofieBLAS.cxx LIBRARIES Core ROOTTMVASofie) +if (BLAS_FOUND) + target_link_libraries(TestSofieBLAS PRIVATE BLAS::BLAS) + target_compile_definitions(TestSofieBLAS PRIVATE SOFIE_TEST_WITH_BLAS) +endif() + # Unit tests for the ONNX parser layer: the tests write the model files # themselves as hand-crafted protobuf wire format, so they need no Python ROOT_ADD_GTEST(TestSofieParser TestSofieParser.cxx diff --git a/tmva/sofie/test/TestSofieBLAS.cxx b/tmva/sofie/test/TestSofieBLAS.cxx new file mode 100644 index 0000000000000..8af8485845fa5 --- /dev/null +++ b/tmva/sofie/test/TestSofieBLAS.cxx @@ -0,0 +1,226 @@ +// Tests for the matrix routines in the code SOFIE generates. By default, Gemm +// and Axpy are built-in reference implementations of the BLAS routines sgemm_ +// and saxpy_; with RModel::SetUseBLAS(true), they call these BLAS routines +// instead. The emitted helper code is compiled with cling here and compared +// with a naive implementation of the BLAS contract: the built-in variant +// always, and the BLAS variant if the test is linked to BLAS +// (SOFIE_TEST_WITH_BLAS). + +#include "TMVA/SOFIE_common.hxx" + +#include + +#include "gtest/gtest.h" + +#include +#include +#include +#include +#include +#include +#include + +#ifdef SOFIE_TEST_WITH_BLAS +// The BLAS routines. Calling them directly checks the naive implementations +// below, and it makes sure that the BLAS library stays linked to this test, so +// that cling finds them when it compiles the code that uses BLAS. +extern "C" void sgemm_(const char *transa, const char *transb, const int *m, const int *n, const int *k, + const float *alpha, const float *A, const int *lda, const float *B, const int *ldb, + const float *beta, float *C, const int *ldc); +extern "C" void saxpy_(const int *n, const float *alpha, const float *x, const int *incx, float *y, const int *incy); +#endif + +namespace { + +using GemmFunc = void (*)(const char *, const char *, const int *, const int *, const int *, const float *, + const float *, const int *, const float *, const int *, const float *, float *, const int *); +using AxpyFunc = void (*)(const int *, const float *, const float *, const int *, float *, const int *); + +constexpr const char *kGemmType = "void (*)(const char *, const char *, const int *, const int *, const int *, " + "const float *, const float *, const int *, const float *, const int *, " + "const float *, float *, const int *)"; +constexpr const char *kAxpyType = "void (*)(const int *, const float *, const float *, const int *, float *, " + "const int *)"; + +bool IsTransposed(char trans) +{ + return trans == 'T' || trans == 't'; +} + +// Naive implementation of the BLAS sgemm contract, with A, B and C in +// column-major storage: C := alpha * op(A) * op(B) + beta * C, where C is not +// read if beta == 0. +void NaiveGemm(const char *transa, const char *transb, const int *m, const int *n, const int *k, const float *alpha, + const float *A, const int *lda, const float *B, const int *ldb, const float *beta, float *C, + const int *ldc) +{ + for (int j = 0; j < *n; ++j) { + for (int i = 0; i < *m; ++i) { + double sum = 0.; + for (int l = 0; l < *k; ++l) { + const float a = IsTransposed(*transa) ? A[l + i * *lda] : A[i + l * *lda]; + const float b = IsTransposed(*transb) ? B[j + l * *ldb] : B[l + j * *ldb]; + sum += static_cast(a) * b; + } + float &c = C[i + j * *ldc]; + c = static_cast(*alpha * sum + (*beta == 0.f ? 0. : static_cast(*beta) * c)); + } + } +} + +// Naive implementation of the BLAS saxpy contract, for positive strides. +void NaiveAxpy(const int *n, const float *alpha, const float *x, const int *incx, float *y, const int *incy) +{ + for (int i = 0; i < *n; ++i) + y[i * *incy] += *alpha * x[i * *incx]; +} + +struct HelperFunctions { + GemmFunc gemm = nullptr; + AxpyFunc axpy = nullptr; +}; + +// Compile the helper code that SOFIE emits into generated models, with or +// without BLAS, and return its Gemm and Axpy. +HelperFunctions DeclareHelpers(bool useBLAS) +{ + using TMVA::Experimental::SOFIE::GenerateHelperFunctionsCode; + const std::string ns = useBLAS ? "TMVA_SOFIE_BLASTest" : "TMVA_SOFIE_BuiltinTest"; + const auto code = GenerateHelperFunctionsCode({"Gemm", "Axpy"}, ns, useBLAS); + const std::string src = code.includes + "namespace " + ns + " {\n" + code.definitions + "}\n"; + if (!gInterpreter->Declare(src.c_str())) + throw std::runtime_error("TestSofieBLAS: failed to declare the emitted SOFIE helper code"); + + auto address = [&](const char *type, const char *func) { + const std::string expr = std::string("static_cast<") + type + ">(&" + ns + "::" + func + ");"; + return gInterpreter->ProcessLine(expr.c_str()); + }; + HelperFunctions helpers; + helpers.gemm = reinterpret_cast(address(kGemmType, "Gemm")); + helpers.axpy = reinterpret_cast(address(kAxpyType, "Axpy")); + if (!helpers.gemm || !helpers.axpy) + throw std::runtime_error("TestSofieBLAS: failed to get the emitted SOFIE helper functions"); + return helpers; +} + +const HelperFunctions &BuiltinHelpers() +{ + static const HelperFunctions helpers = DeclareHelpers(false); + return helpers; +} + +#ifdef SOFIE_TEST_WITH_BLAS +const HelperFunctions &BLASHelpers() +{ + static const HelperFunctions helpers = DeclareHelpers(true); + return helpers; +} +#endif + +std::vector RandomVector(std::size_t size, std::mt19937 &rng) +{ + std::uniform_real_distribution dist(-1.f, 1.f); + std::vector v(size); + for (auto &x : v) + x = dist(rng); + return v; +} + +void ExpectClose(const std::vector &actual, const std::vector &expected) +{ + ASSERT_EQ(actual.size(), expected.size()); + for (std::size_t i = 0; i < actual.size(); ++i) + EXPECT_NEAR(actual[i], expected[i], 1e-4f * (1.f + std::abs(expected[i]))) << "at index " << i; +} + +} // namespace + +// transA, transB, {m, n, k}, padding of the leading dimensions, alpha, beta +using GemmParams = std::tuple, int, float, float>; + +class SofieGemm : public testing::TestWithParam {}; + +TEST_P(SofieGemm, AgreesWithNaiveImplementation) +{ + // No structured bindings: lambdas can only capture them since C++20. + const char transA = std::get<0>(GetParam()); + const char transB = std::get<1>(GetParam()); + const int m = std::get<0>(std::get<2>(GetParam())); + const int n = std::get<1>(std::get<2>(GetParam())); + const int k = std::get<2>(std::get<2>(GetParam())); + const int pad = std::get<3>(GetParam()); + const float alpha = std::get<4>(GetParam()); + const float beta = std::get<5>(GetParam()); + + // Column-major storage: op(A) is m x k and op(B) is k x n. + const int lda = (IsTransposed(transA) ? k : m) + pad; + const int ldb = (IsTransposed(transB) ? n : k) + pad; + const int ldc = m + pad; + + std::mt19937 rng(m * 10000 + n * 100 + k); + const auto a = RandomVector(lda * (IsTransposed(transA) ? m : k), rng); + const auto b = RandomVector(ldb * (IsTransposed(transB) ? k : n), rng); + auto c = RandomVector(ldc * n, rng); + // With beta == 0, C has to be overwritten without being read, so whatever + // it contains must not matter. + if (beta == 0.f) { + for (int j = 0; j < n; ++j) + for (int i = 0; i < m; ++i) + c[i + j * ldc] = std::numeric_limits::quiet_NaN(); + } + + auto run = [&](GemmFunc gemm) { + std::vector out = c; + gemm(&transA, &transB, &m, &n, &k, &alpha, a.data(), &lda, b.data(), &ldb, &beta, out.data(), &ldc); + return out; + }; + + const auto expected = run(&NaiveGemm); + ExpectClose(run(BuiltinHelpers().gemm), expected); +#ifdef SOFIE_TEST_WITH_BLAS + ExpectClose(run(&sgemm_), expected); + ExpectClose(run(BLASHelpers().gemm), expected); +#endif +} + +// Gemm_Call passes the transposition flags in lower case. +INSTANTIATE_TEST_SUITE_P(SofieBLAS, SofieGemm, + testing::Combine(testing::Values('N', 'T', 'n', 't'), testing::Values('N', 'T', 'n', 't'), + testing::Values(std::make_tuple(1, 1, 1), std::make_tuple(3, 5, 7), + std::make_tuple(17, 4, 33), std::make_tuple(64, 1, 128)), + testing::Values(0, 3), testing::Values(1.f, 0.5f), + testing::Values(0.f, 1.f, -0.75f))); + +// n, incx, incy, alpha +using AxpyParams = std::tuple; + +class SofieAxpy : public testing::TestWithParam {}; + +TEST_P(SofieAxpy, AgreesWithNaiveImplementation) +{ + const int n = std::get<0>(GetParam()); + const int incx = std::get<1>(GetParam()); + const int incy = std::get<2>(GetParam()); + const float alpha = std::get<3>(GetParam()); + + std::mt19937 rng(n * 100 + incx * 10 + incy); + const auto x = RandomVector(n * incx, rng); + const auto y = RandomVector(n * incy, rng); + + auto run = [&](AxpyFunc axpy) { + std::vector out = y; + axpy(&n, &alpha, x.data(), &incx, out.data(), &incy); + return out; + }; + + const auto expected = run(&NaiveAxpy); + ExpectClose(run(BuiltinHelpers().axpy), expected); +#ifdef SOFIE_TEST_WITH_BLAS + ExpectClose(run(&saxpy_), expected); + ExpectClose(run(BLASHelpers().axpy), expected); +#endif +} + +INSTANTIATE_TEST_SUITE_P(SofieBLAS, SofieAxpy, + testing::Combine(testing::Values(1, 7, 33), testing::Values(1, 2), testing::Values(1, 3), + testing::Values(1.f, -0.5f))); diff --git a/tmva/sofie/test/TestSofieParser.cxx b/tmva/sofie/test/TestSofieParser.cxx index f5a700e62ab0e..ebe95c7a7108a 100644 --- a/tmva/sofie/test/TestSofieParser.cxx +++ b/tmva/sofie/test/TestSofieParser.cxx @@ -152,8 +152,8 @@ std::string FloatValueInfo(const TestTensor &tensor, int elemType = 1) // Write a model holding a single Gemm node and return the code SOFIE generates // for it. Every operand is a graph input, which keeps the model file tiny. -std::string -GenerateGemmCode(const std::string &fileName, const std::vector &inputs, const TestTensor &output) +std::string GenerateGemmCode(const std::string &fileName, const std::vector &inputs, + const TestTensor &output, bool useBLAS = false) { std::string node; for (const TestTensor &input : inputs) @@ -181,6 +181,7 @@ GenerateGemmCode(const std::string &fileName, const std::vector &inp file.close(); RModel rmodel = RModelParser_ONNX{}.Parse(fileName); + rmodel.SetUseBLAS(useBLAS); rmodel.Generate(Options::kNoWeightFile); std::ostringstream code; rmodel.PrintGenerated(code); @@ -318,6 +319,22 @@ TEST(SOFIEParser, GemmDoesNotBroadcastABiasThatHasTheOutputShape) EXPECT_NE(code.find(",tensor_c);"), std::string::npos); } +// The matrix multiplication calls BLAS only if requested with +// RModel::SetUseBLAS(true); by default, the generated code has no BLAS +// dependency. +TEST(SOFIEParser, GemmCallsBLASOnlyIfRequested) +{ + const std::vector inputs{{"x", {"1", "10"}}, {"w", {"10", "32"}}, {"b", {"32"}}}; + const TestTensor output{"y", {"1", "32"}}; + + const std::string builtinCode = GenerateGemmCode("gemm_builtin.onnx", inputs, output); + EXPECT_EQ(builtinCode.find("sgemm_"), std::string::npos); + + const std::string blasCode = GenerateGemmCode("gemm_blas.onnx", inputs, output, /*useBLAS=*/true); + EXPECT_NE(blasCode.find("extern \"C\" void sgemm_("), std::string::npos); + EXPECT_NE(blasCode.find("BLAS::sgemm_("), std::string::npos); +} + // An input of a type that SOFIE does not support (FLOAT16) must make // Generate() throw, instead of producing code that does not compile. TEST(SOFIEParser, InputOfUnsupportedTypeThrows)