Cuda13 mods

include/occa/core/device.hpp

@@ -297,6 +297,21 @@ namespace occa {
      */
     const std::string& arch() const;
 
+    /**
+     * @startDoc{name}
+     *
+     * Description:
+     *   Returns a human-readable name for the physical device.
+     *   For GPU backends this is the product name (e.g. "NVIDIA RTX A4000").
+     *   For host-based backends this returns "CPU".
+     *
+     * Returns:
+     *   The device name string.
+     *
+     * @endDoc
+     */
+    const std::string& name() const;
+
     const occa::json& kernelProperties() const;
     occa::json kernelProperties(const occa::json &additionalProps) const;
 
@@ -334,6 +349,22 @@ namespace occa {
      */
     udim_t memorySize() const;
 
+    /**
+     * @startDoc{memoryFree}
+     *
+     * Description:
+     *   Finds how much device memory is currently free, accounting for
+     *   allocations made outside of OCCA. For host-based backends (Serial,
+     *   OpenMP) this reports system-wide free memory, not a per-process
+     *   budget.
+     *
+     * Description:
+     *   Returns the free memory in bytes.
+     *
+     * @endDoc
+     */
+    udim_t memoryFree() const;
+
     /**
      * @startDoc{memoryAllocated}
      *

include/occa/defines/occa.hpp

@@ -17,4 +17,7 @@
 
 #define OCCA_DEFAULT_MEM_BYTE_ALIGN 32
 
+// Defined in this fork to signal availability of device.name() and device.memoryFree()
+#define OCCA_FORK_EXTENSIONS 1
+
 #endif

scripts/spack/packages/occa/package.py

@@ -0,0 +1,58 @@
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Occa(CMakePackage):
+    """OCCA: Portable approach for parallel architectures."""
+
+    homepage = "https://libocca.org"
+    git      = "https://github.com/dylanjude/occa.git"
+    url      = "https://github.com/dylanjude/occa/archive/refs/tags/v2.0.0.tar.gz"
+
+    maintainers("dylanjude")
+    license("MIT")
+
+    version("main", branch="main")
+    version("2.0.0", sha256="f2521901fed5d199193d54c7db4186479a974bdae92ac97779c47fa2bb68badd")
+    version("1.6.0", sha256="b863a24171000097121aff5c43dadf22416c143824598a51c653689fd917794f")
+    version("1.5.0", sha256="b939f826f3e970b45aa77089568995399355fcdad3bc787b60cf73eb4962b0b7")
+    version("1.4.0", sha256="5995288615f45dd2cf2f3e13b9e04c3b89edde1e014d36c6da014c11b9adb4a7")
+
+    variant("cuda",   default=True,  description="Activates support for CUDA")
+    variant("hip",    default=False, description="Activates support for HIP")
+    variant("openmp", default=True,  description="Activates support for OpenMP")
+    variant("opencl", default=False, description="Activates support for OpenCL")
+    variant("dpcpp",  default=False, description="Activates support for DPCPP")
+
+    depends_on("c", type="build")
+    depends_on("cxx", type="build")
+
+    depends_on("cuda", when="+cuda")
+    depends_on("hip", when="+hip")
+
+    depends_on("cmake", type="build")
+
+    def cmake_args(self):
+        args = [
+            self.define("CMAKE_BUILD_TYPE", "Release"),
+            self.define_from_variant("OCCA_ENABLE_CUDA", "cuda"),
+            self.define_from_variant("OCCA_ENABLE_HIP", "hip"),
+            self.define_from_variant("OCCA_ENABLE_OPENMP", "openmp"),
+            self.define_from_variant("OCCA_ENABLE_OPENCL", "opencl"),
+            self.define_from_variant("OCCA_ENABLE_DPCPP", "dpcpp"),
+        ]
+        return args
+
+    def setup_dependent_build_environment(self, env, dependent_spec):
+        env.set("OCCA_INSTALL_DIR", self.prefix)
+        env.set("OCCA_DIR", self.prefix)
+        env.set("OCCA_HOME", self.prefix)
+
+    def setup_run_environment(self, env):
+        env.set("OCCA_INSTALL_DIR", self.prefix)
+        env.set("OCCA_DIR", self.prefix)
+        env.set("OCCA_HOME", self.prefix)

scripts/spack/repo.yaml

@@ -0,0 +1,2 @@
+repo:
+  namespace: occa

src/core/device.cpp

@@ -183,6 +183,13 @@ namespace occa {
             : noArch);
   }
 
+  const std::string& device::name() const {
+    static const std::string noName = "No Name";
+    return (modeDevice
+            ? modeDevice->name
+            : noName);
+  }
+
   const occa::json& device::kernelProperties() const {
     assertInitialized();
     return (const occa::json&) modeDevice->properties["kernel"];
@@ -233,6 +240,13 @@ namespace occa {
     return 0;
   }
 
+  udim_t device::memoryFree() const {
+    if (modeDevice) {
+      return modeDevice->memoryFree();
+    }
+    return 0;
+  }
+
   udim_t device::memoryAllocated() const {
     if (modeDevice) {
       return modeDevice->bytesAllocated;

src/occa/internal/core/device.hpp

@@ -11,6 +11,7 @@ namespace occa {
    public:
     std::string mode;
     std::string arch;
+    std::string name;
     occa::json properties;
     bool needsLauncherKernel;
 
@@ -121,6 +122,8 @@ namespace occa {
     virtual modeMemoryPool_t* createMemoryPool(const occa::json &props)=0;
 
     virtual udim_t memorySize() const = 0;
+
+    virtual udim_t memoryFree() const { return 0; }
     //  |===============================
 
     virtual void* unwrap() = 0;

src/occa/internal/modes/cuda/device.cpp

@@ -61,6 +61,11 @@ namespace occa {
 
       getDeviceArchVersion(cuDevice, archMajorVersion, archMinorVersion);
       arch = getDeviceArch(cuDevice);
+
+      char deviceName[1024];
+      OCCA_CUDA_ERROR("Device: Getting Device Name",
+                      cuDeviceGetName(deviceName, 1024, cuDevice));
+      name = std::string(deviceName);
     }
 
     device::~device() {
@@ -82,7 +87,13 @@ namespace occa {
         std::stringstream ss;
         ss << "major: " << archMajorVersion << ' '
            << "minor: " << archMinorVersion;
-        hash_ = occa::hash(ss.str());
+
+        CUuuid uuid;
+        OCCA_CUDA_ERROR("Getting device UUID",
+                        cuDeviceGetUuid(&uuid, cuDevice));
+
+        hash_ = (occa::hash(ss.str())
+                 ^ occa::hash(uuid.bytes, sizeof(uuid.bytes)));
       }
       return hash_;
     }
@@ -446,6 +457,12 @@ namespace occa {
     udim_t device::memorySize() const {
       return cuda::getDeviceMemorySize(cuDevice);
     }
+
+    udim_t device::memoryFree() const {
+      OCCA_CUDA_ERROR("Device: Setting Context",
+                      cuCtxSetCurrent(cuContext));
+      return cuda::getDeviceMemoryFree();
+    }
     //==================================
 
     void* device::unwrap() {

src/occa/internal/modes/cuda/device.hpp

@@ -100,6 +100,7 @@ namespace occa {
       modeMemoryPool_t* createMemoryPool(const occa::json &props) override;
 
       udim_t memorySize() const override;
+      udim_t memoryFree() const override;
       //================================
 
       void* unwrap() override;

src/occa/internal/modes/cuda/polyfill.hpp

@@ -16,6 +16,7 @@
 namespace occa {
   //---[ Types ]------------------------
   struct _CUdeviceptr {};
+  struct CUuuid { char bytes[16]; };
   typedef struct _CUcontext*            CUcontext;
   typedef int                           CUdevice;
   typedef struct _CUdeviceptr*          CUdeviceptr;
@@ -160,6 +161,10 @@ namespace occa {
     return OCCA_CUDA_IS_NOT_ENABLED;
   }
 
+  inline CUresult cuDeviceGetUuid(CUuuid *uuid, CUdevice dev) {
+    return OCCA_CUDA_IS_NOT_ENABLED;
+  }
+
   //   ---[ Event ]---------------------
   inline CUresult cuEventCreate(CUevent *phEvent, unsigned int Flags) {
     return OCCA_CUDA_IS_NOT_ENABLED;
@@ -187,6 +192,10 @@ namespace occa {
     return OCCA_CUDA_IS_NOT_ENABLED;
   }
 
+  inline CUresult cuMemGetInfo(size_t *free, size_t *total) {
+    return OCCA_CUDA_IS_NOT_ENABLED;
+  }
+
   inline CUresult cuMemAllocHost(void **pp, size_t bytesize) {
     return OCCA_CUDA_IS_NOT_ENABLED;
   }

src/occa/internal/modes/cuda/utils.cpp

@@ -44,6 +44,14 @@ namespace occa {
       return bytes;
     }
 
+    udim_t getDeviceMemoryFree() {
+      size_t freeBytes = 0;
+      size_t totalBytes = 0;
+      OCCA_CUDA_ERROR("Finding free memory on device",
+                      cuMemGetInfo(&freeBytes, &totalBytes));
+      return freeBytes;
+    }
+
     std::string getVersion() {
       std::stringstream ss;
       ss << ((int) (CUDA_VERSION / 1000))
@@ -185,11 +193,26 @@ namespace occa {
                            ? ((cuda::device*) device.getModeDevice())->cuDevice
                            : CU_DEVICE_CPU);
 
+#if CUDA_VERSION >= 11020
+      // CUDA 11.2+ replaced the CUdevice argument with a CUmemLocation
+      CUmemLocation location;
+      location.type = (device.mode() == "CUDA")
+        ? CU_MEM_LOCATION_TYPE_DEVICE
+        : CU_MEM_LOCATION_TYPE_HOST;
+      location.id   = cuDevice;
+
+      OCCA_CUDA_ERROR("Advising about unified memory",
+                      cuMemAdvise(((cuda::memory*) mem.getModeMemory())->cuPtr,
+                                  (size_t) bytes_,
+                                  advice,
+                                  location));
+#else
       OCCA_CUDA_ERROR("Advising about unified memory",
                       cuMemAdvise(((cuda::memory*) mem.getModeMemory())->cuPtr,
                                   (size_t) bytes_,
                                   advice,
                                   cuDevice));
+#endif
 #else
       OCCA_FORCE_ERROR("CUDA version ["
                        << cuda::getVersion()
@@ -215,11 +238,27 @@ namespace occa {
                            ? ((cuda::device*) device.getModeDevice())->cuDevice
                            : CU_DEVICE_CPU);
       occa::stream stream = device.getStream();
+#if CUDA_VERSION >= 11020
+      // CUDA 11.2+ replaced the CUdevice argument with a CUmemLocation plus flags
+      CUmemLocation location;
+      location.type = (device.mode() == "CUDA")
+        ? CU_MEM_LOCATION_TYPE_DEVICE
+        : CU_MEM_LOCATION_TYPE_HOST;
+      location.id   = cuDevice;
+
+      OCCA_CUDA_ERROR("Prefetching unified memory",
+                      cuMemPrefetchAsync(((cuda::memory*) mem.getModeMemory())->cuPtr,
+                                         (size_t) bytes_,
+                                         location,
+                                         0,
+                                         *((CUstream*) stream.getModeStream())) );
+#else
       OCCA_CUDA_ERROR("Prefetching unified memory",
                       cuMemPrefetchAsync(((cuda::memory*) mem.getModeMemory())->cuPtr,
                                          (size_t) bytes_,
                                          cuDevice,
                                          *((CUstream*) stream.getModeStream())) );
+#endif
 #else
       OCCA_FORCE_ERROR("CUDA version ["
                        << cuda::getVersion()

src/occa/internal/modes/cuda/utils.hpp

@@ -17,6 +17,7 @@ namespace occa {
     int getDeviceCount();
     CUdevice getDevice(const int id);
     udim_t getDeviceMemorySize(CUdevice device);
+    udim_t getDeviceMemoryFree();
 
     std::string getVersion();
 

src/occa/internal/modes/hip/device.cpp

@@ -61,6 +61,11 @@ namespace occa {
       kernelProps["compiler_flags"] = compilerFlags;
 
       arch = getDeviceArch(deviceID);
+
+      hipDeviceProp_t hipProps;
+      OCCA_HIP_ERROR("Device: Getting Device Properties",
+                     hipGetDeviceProperties(&hipProps, deviceID));
+      name = std::string(hipProps.name);
     }
 
     device::~device() { }
@@ -71,7 +76,21 @@ namespace occa {
 
     hash_t device::hash() const {
       if (!hash_.initialized) {
-        hash_ = occa::hash(arch);
+#if !defined(HIP_VERSION) || (HIP_VERSION >= 50200000)
+        hipUUID uuid;
+        OCCA_HIP_ERROR("Getting device UUID",
+                       hipDeviceGetUuid(&uuid, hipDevice));
+
+        hash_ = (occa::hash(arch)
+                 ^ occa::hash(uuid.bytes, sizeof(uuid.bytes)));
+#else
+        char pciId[64] = {0};
+        OCCA_HIP_ERROR("Getting device PCI bus ID",
+                       hipDeviceGetPCIBusId(pciId, sizeof(pciId), hipDevice));
+
+        hash_ = (occa::hash(arch)
+                 ^ occa::hash(std::string(pciId)));
+#endif
       }
       return hash_;
     }
@@ -419,6 +438,12 @@ namespace occa {
     udim_t device::memorySize() const {
       return hip::getDeviceMemorySize(hipDevice);
     }
+
+    udim_t device::memoryFree() const {
+      OCCA_HIP_ERROR("Device: Setting Device",
+                     hipSetDevice(deviceID));
+      return hip::getDeviceMemoryFree();
+    }
     //==================================
 
     void* device::unwrap() {

src/occa/internal/modes/hip/device.hpp

@@ -90,6 +90,7 @@ namespace occa {
       modeMemoryPool_t* createMemoryPool(const occa::json &props) override;
 
       udim_t memorySize() const override;
+      udim_t memoryFree() const override;
       //================================
 
       void* unwrap() override;

src/occa/internal/modes/hip/polyfill.hpp

@@ -17,6 +17,7 @@
 // are indistinguisable inside the occa namespace
 namespace occa {
   //---[ Types ]------------------------
+  struct hipUUID { char bytes[16]; };
   typedef struct _hipCtx_t*               hipCtx_t;
   typedef int                             hipDevice_t;
   typedef void*                           hipDeviceptr_t;
@@ -203,6 +204,14 @@ namespace occa {
     return OCCA_HIP_IS_NOT_ENABLED;
   }
 
+  inline hipError_t hipMemGetInfo(size_t *free, size_t *total) {
+    return OCCA_HIP_IS_NOT_ENABLED;
+  }
+
+  inline hipError_t hipDeviceGetUuid(hipUUID *uuid, hipDevice_t dev) {
+    return OCCA_HIP_IS_NOT_ENABLED;
+  }
+
   inline hipError_t hipFree(hipDeviceptr_t dptr) {
     return OCCA_HIP_IS_NOT_ENABLED;
   }