Refactor: detail/compute_capability -> detail/gpu/cuda/compute_capability which is cuda-only.

Usage is guarded out, and previously macro'd out tests are now enabled (but heterogenous AMD systems may encounter test failures)

Author: ptheywood

…e/flamegpu/detail/compute_capability.cuh …u/detail/gpu/cuda/compute_capability.cuhinclude/flamegpu/detail/compute_capability.cuh renamed to include/flamegpu/detail/gpu/cuda/compute_capability.cuh include/flamegpu/detail/compute_capability.cuh renamed to include/flamegpu/detail/gpu/cuda/compute_capability.cuh

@@ -1,8 +1,7 @@
-#ifndef INCLUDE_FLAMEGPU_DETAIL_COMPUTE_CAPABILITY_CUH_
-#define INCLUDE_FLAMEGPU_DETAIL_COMPUTE_CAPABILITY_CUH_
+#ifndef INCLUDE_FLAMEGPU_DETAIL_GPU_CUDA_COMPUTE_CAPABILITY_CUH_
+#define INCLUDE_FLAMEGPU_DETAIL_GPU_CUDA_COMPUTE_CAPABILITY_CUH_
 
-// Todo: device on the AMD equivalent compute capability stuff
-#ifdef FLAMEGPU_USE_CUDA
+#if defined(FLAMEGPU_USE_CUDA)
 
 #include <vector>
 #include <string>
@@ -12,10 +11,17 @@
 
 namespace flamegpu {
 namespace detail {
+namespace gpu {
+/**
+ * CUDA-specifc internal code
+ */
+namespace cuda {
 namespace compute_capability {
 
 /**
  * get the compute capability for a device
+ * 
+ * @note CUDA only
  * @param deviceIndex the index of the device to be queried
  * @return integer value representing the compute capability, i.e 70 for SM_70
  */
@@ -27,6 +33,8 @@ int getComputeCapability(int deviceIndex);
  * Extracts the first element from `__CUDA_ARCH_LIST__` which is an ordered list of integer architectures passed to nvcc, with the 0th value being the lowest.
  *
  * __CUDA_ARCH_LIST__ values are 3 or 4 digits long, with SM_80 becoming 800, or SM_103 becoming 1030, so they are scaled back down to match SM_ notation.
+ * 
+ * @note CUDA only
  *
  * @return the minimum compute capability from __CUDA_ARCH_LIST__ as a 2+ digit integer, i.e. 80 for SM_80, or 0 if __CUDA_ARCH_LIST__ was undefined
  */
@@ -44,6 +52,9 @@ std::string compiledCompiledComputeCapabilitiesString();
  * This assumes JIT support is enabled for future (major) architectures.
  * If __CUDA_ARCH_LIST__ could not be used to extract a minimum compiled architecture, no decision can be made so it is assumed to be successful.
  * If a or f post-fixed architectures are used, this check may also be insufficient for forwards compatibilty, but this requires an SM >= 100 device to test this on.
+ * 
+ * @note CUDA only
+ * 
  * @param deviceIndex the index of the device to be checked.
  * @return boolean indicating if the executable can run on the specified device.
  */
@@ -52,6 +63,9 @@ bool checkComputeCapability(int deviceIndex);
 /**
  * Get the comptue capabilities supported by the linked NVRTC, irrespective of whether FLAMEGPU was configured for that architecture.
  * CUDA 11.2 or greater provides methods to make this dynamic. Older versions of CUDA are hardcoded (11.1, 11.0 and 10.x only).
+ * 
+ * @note CUDA only
+ * 
  * @return vector of compute capability integers ((major * 10) + minor) in ascending order 
  */
 std::vector<int> getNVRTCSupportedComputeCapabilties();
@@ -62,16 +76,20 @@ std::vector<int> getNVRTCSupportedComputeCapabilties();
  *
  * This method has been separated from JitifyCache::preprocessKernel so that it can be tested generically, without having to write tests which are relative to the linked nvrtc and/or the current device.
  * 
+ * @note CUDA only
+ * 
  * @param target compute capability to find the best match for
  * @param archictectures a vector of architectures in ascending order
  * @return the best compute capability to use (the largest value LE target), or 0 if none are appropriate.
  */
 int selectAppropraiteComputeCapability(const int target, const std::vector<int>& architectures);
 
 }  // namespace compute_capability
+}  // namespace cuda
+}  // namespace gpu
 }  // namespace detail
 }  // namespace flamegpu
 
-#endif  // FLAMEGPU_USE_CUDA
+#endif  // defined(FLAMEGPU_USE_CUDA)
 
-#endif  // INCLUDE_FLAMEGPU_DETAIL_COMPUTE_CAPABILITY_CUH_
+#endif  // INCLUDE_FLAMEGPU_DETAIL_GPU_CUDA_COMPUTE_CAPABILITY_CUH_

src/CMakeLists.txt

@@ -287,10 +287,10 @@ SET(SRC_INCLUDE
     ${FLAMEGPU_ROOT}/include/flamegpu/detail/gpu/types.hpp
     ${FLAMEGPU_ROOT}/include/flamegpu/detail/gpu/gpu_api_error_checking.cuh
     ${FLAMEGPU_ROOT}/include/flamegpu/detail/gpu/device_name.hpp
+    ${FLAMEGPU_ROOT}/include/flamegpu/detail/gpu/cuda/compute_capability.cuh
     ${FLAMEGPU_ROOT}/include/flamegpu/detail/Any.h
     ${FLAMEGPU_ROOT}/include/flamegpu/detail/demangle.h
     ${FLAMEGPU_ROOT}/include/flamegpu/detail/type_decode.h
-    ${FLAMEGPU_ROOT}/include/flamegpu/detail/compute_capability.cuh
     ${FLAMEGPU_ROOT}/include/flamegpu/detail/curand.cuh
     ${FLAMEGPU_ROOT}/include/flamegpu/detail/wddm.cuh
     ${FLAMEGPU_ROOT}/include/flamegpu/detail/CUDAEventTimer.cuh
@@ -386,8 +386,8 @@ SET(SRC_FLAMEGPU
     ${FLAMEGPU_ROOT}/src/flamegpu/io/Telemetry.cpp
     ${FLAMEGPU_ROOT}/src/flamegpu/util/cleanup.cu
     ${FLAMEGPU_ROOT}/src/flamegpu/detail/gpu/device_name.cu
+    ${FLAMEGPU_ROOT}/src/flamegpu/detail/gpu/cuda/compute_capability.cu
     ${FLAMEGPU_ROOT}/src/flamegpu/detail/demangle.cpp
-    ${FLAMEGPU_ROOT}/src/flamegpu/detail/compute_capability.cu
     ${FLAMEGPU_ROOT}/src/flamegpu/detail/wddm.cu
     ${FLAMEGPU_ROOT}/src/flamegpu/detail/JitifyCache.cu
     ${FLAMEGPU_ROOT}/src/flamegpu/detail/TestSuiteTelemetry.cpp

src/flamegpu/detail/JitifyCache.cu

@@ -17,7 +17,7 @@
 
 #include "flamegpu/version.h"
 #include "flamegpu/exception/FLAMEGPUException.h"
-#include "flamegpu/detail/compute_capability.cuh"
+#include "flamegpu/detail/gpu/cuda/compute_capability.cuh"
 #include "flamegpu/util/nvtx.h"
 #include "flamegpu/detail/gpu/macros.hpp"
 
@@ -396,7 +396,7 @@ std::unique_ptr<jitify2::LinkedProgramData> JitifyCache::buildProgram(
 #endif
 
     // Set the cuda compuate capability architecture to optimize / generate for, based on the values supported by the current dynamiclaly linked nvrtc and the device in question.
-    std::vector<int> nvrtcArchitectures = detail::compute_capability::getNVRTCSupportedComputeCapabilties();
+    std::vector<int> nvrtcArchitectures = detail::gpu::cuda::compute_capability::getNVRTCSupportedComputeCapabilties();
     if (nvrtcArchitectures.size()) {
         int currentDeviceIdx = 0;
         if (FLAMEGPU_GPU_RUNTIME_SYMBOL(Success) == cudaGetDevice(&currentDeviceIdx)) {

src/flamegpu/detail/compute_capability.cu …pu/detail/gpu/cuda/compute_capability.cusrc/flamegpu/detail/compute_capability.cu renamed to src/flamegpu/detail/gpu/cuda/compute_capability.cu src/flamegpu/detail/compute_capability.cu renamed to src/flamegpu/detail/gpu/cuda/compute_capability.cu

@@ -1,20 +1,20 @@
-#ifdef FLAMEGPU_USE_CUDA
+// only implemented for CUDA
+#if defined(FLAMEGPU_USE_CUDA)
 #include <nvrtc.h>
-#endif  // FLAMEGPU_USE_CUDA
 
 #include <cassert>
 #include <array>
 #include <vector>
 #include <string>
 
-#include "flamegpu/detail/compute_capability.cuh"
+#include "flamegpu/detail/gpu/cuda/compute_capability.cuh"
 #include "flamegpu/detail/gpu/gpu_api_error_checking.cuh"
 #include "flamegpu/detail/gpu/macros.hpp"
 
-#ifdef FLAMEGPU_USE_CUDA
-
 namespace flamegpu {
 namespace detail {
+namespace gpu {
+namespace cuda {
 
 namespace {
     /**
@@ -137,7 +137,9 @@ int compute_capability::selectAppropraiteComputeCapability(const int target, con
     return maxArch;
 }
 
+}  // namespace cuda
+}  // namespace gpu
 }  // namespace detail
 }  // namespace flamegpu
 
-#endif  // FLAMEGPU_USE_CUDA
+#endif  // defined(FLAMEGPU_USE_CUDA)

src/flamegpu/detail/gpu/device_name.cu

@@ -1,7 +1,7 @@
 #include <set>
 #include <string>
 
-#include "flamegpu/detail/compute_capability.cuh"
+#include "flamegpu/detail/gpu/cuda/compute_capability.cuh"
 #include "flamegpu/detail/gpu/macros.hpp"
 #include "flamegpu/detail/gpu/gpu_api_error_checking.cuh"
 

src/flamegpu/simulation/CUDAEnsemble.cu

@@ -22,7 +22,7 @@
 #include "flamegpu/version.h"
 #include "flamegpu/model/ModelDescription.h"
 #include "flamegpu/simulation/RunPlanVector.h"
-#include "flamegpu/detail/compute_capability.cuh"
+#include "flamegpu/detail/gpu/cuda/compute_capability.cuh"
 #include "flamegpu/detail/SteadyClockTimer.h"
 #include "flamegpu/simulation/CUDASimulation.h"
 #include "flamegpu/io/StateWriterFactory.h"
@@ -174,21 +174,22 @@ unsigned int CUDAEnsemble::simulate(const RunPlanVector& plans) {
     devices = mpi->devicesForThisRank(devices);
 #endif  // ifdef FLAMEGPU_ENABLE_MPI
 
-    // Check that each device is capable, and init cuda context
+    // Check that each device is capable (CUDA only), and init cuda context
     for (auto d = devices.begin(); d != devices.end(); ++d) {
-        // todo: HIP equiavalent
-#ifdef FLAMEGPU_USE_CUDA
-        if (!detail::compute_capability::checkComputeCapability(*d)) {
-#else  // FLAMEGPU_USE_CUDA
-        if (false) {
-#endif  // FLAMEGPU_USE_CUDA
+        bool compatible_gpu_arch = true;
+        // if CUDA, we can check the compute capability and emit a useful error if not compatible
+#if defined(FLAMEGPU_USE_CUDA)
+        compatible_gpu_arch = detail::gpu::cuda::compute_capability::checkComputeCapability(*d)
+#endif  // defined(FLAMEGPU_USE_CUDA)
+        if (!compatible_gpu_arch) {
             // Emit a warning unless quiet verbosity was specified.
             if (config.verbosity >= Verbosity::Default) {
                 fprintf(stderr, "FLAMEGPU2 has not been built with an appropriate compute capability for device %d, this device will not be used.\n", *d);
             }
             d = devices.erase(d);
             --d;
         } else {
+            // Initialise the context on the device
             flamegpu::detail::gpuCheck(FLAMEGPU_GPU_RUNTIME_SYMBOL(SetDevice)(*d));
             flamegpu::detail::gpuCheck(flamegpu::detail::cuda::cudaFree(nullptr));
         }

src/flamegpu/simulation/CUDASimulation.cu

@@ -25,7 +25,7 @@
 #include "flamegpu/simulation/detail/CUDAEnvironmentDirectedGraphBuffers.cuh"
 #include "flamegpu/simulation/detail/CUDAScanCompaction.h"
 #include "flamegpu/util/nvtx.h"
-#include "flamegpu/detail/compute_capability.cuh"
+#include "flamegpu/detail/gpu/cuda/compute_capability.cuh"
 #include "flamegpu/detail/SignalHandlers.h"
 #include "flamegpu/detail/wddm.cuh"
 #include "flamegpu/detail/SteadyClockTimer.h"
@@ -1570,16 +1570,15 @@ void CUDASimulation::applyConfig_derived() {
         THROW exception::InvalidCUDAdevice("Unable to set CUDA device to '%d' after the CUDASimulation has already initialised on device '%d'.", config.device_id, deviceInitialised);
     }
 
-// Todo: HIP equivalent
-#ifdef FLAMEGPU_USE_CUDA
-    // Check the compute capability of the device, throw an exception if not valid for the executable.
-    if (!detail::compute_capability::checkComputeCapability(static_cast<int>(config.device_id))) {
-        int min_cc = detail::compute_capability::minimumCompiledComputeCapability();
-        std::string compiled_ccs = detail::compute_capability::compiledCompiledComputeCapabilitiesString();
-        int cc = detail::compute_capability::getComputeCapability(static_cast<int>(config.device_id));
+#if defined(FLAMEGPU_USE_CUDA)
+    // On CUDA, check the compute capability of the device, throw an exception if not valid for the executable.
+    if (!detail::gpu::cuda::compute_capability::checkComputeCapability(static_cast<int>(config.device_id))) {
+        int min_cc = detail::gpu::cuda::compute_capability::minimumCompiledComputeCapability();
+        std::string compiled_ccs = detail::gpu::cuda::compute_capability::compiledCompiledComputeCapabilitiesString();
+        int cc = detail::gpu::cuda::compute_capability::getComputeCapability(static_cast<int>(config.device_id));
         THROW exception::InvalidCUDAComputeCapability("Error application compiled for CUDA Compute Capabilities \"%s\". Rebuild including compute capability <= %d for device %u.", compiled_ccs.c_str(), cc, config.device_id);
     }
-#endif  // FLAMEGPU_USE_CUDA
+#endif  // defined(FLAMEGPU_USE_CUDA)
 
     cudaStatus = FLAMEGPU_GPU_RUNTIME_SYMBOL(SetDevice)(static_cast<int>(config.device_id));
     if (cudaStatus != FLAMEGPU_GPU_RUNTIME_SYMBOL(Success)) {
@@ -1622,17 +1621,15 @@ void CUDASimulation::reseed(const uint64_t seed) {
 void CUDASimulation::initialiseSingletons() {
     // Only do this once.
     if (!singletonsInitialised) {
-        // If the device has not been specified, also check the compute capability is OK
-        // Check the compute capability of the device, throw an exception if not valid for the executable.
-        // todo: HIP equivalent
-#ifdef FLAMEGPU_USE_CUDA
-        if (!detail::compute_capability::checkComputeCapability(static_cast<int>(config.device_id))) {
-            int min_cc = detail::compute_capability::minimumCompiledComputeCapability();
-            std::string compiled_ccs = detail::compute_capability::compiledCompiledComputeCapabilitiesString();
-            int cc = detail::compute_capability::getComputeCapability(static_cast<int>(config.device_id));
+#if defined(FLAMEGPU_USE_CUDA)
+        // On CUDA, Check the compute capability of the device, throw an exception if not valid for the executable.
+        if (!detail::gpu::cuda::compute_capability::checkComputeCapability(static_cast<int>(config.device_id))) {
+            int min_cc = detail::gpu::cuda::compute_capability::minimumCompiledComputeCapability();
+            std::string compiled_ccs = detail::gpu::cuda::compute_capability::compiledCompiledComputeCapabilitiesString();
+            int cc = detail::gpu::cuda::compute_capability::getComputeCapability(static_cast<int>(config.device_id));
             THROW exception::InvalidCUDAComputeCapability("Error application compiled for CUDA Compute Capabilities \"%s\". Rebuild including compute capability <= %d for device %u.", compiled_ccs.c_str(), cc, config.device_id);
         }
-#endif  // FLAMEGPU_USE_CUDA
+#endif  // defined(FLAMEGPU_USE_CUDA)
         flamegpu::detail::gpuCheck(FLAMEGPU_GPU_RUNTIME_SYMBOL(GetDevice)(&deviceInitialised));
         // Get references to all required singleton and store in the instance.
         singletons = new Singletons((!submodel)?

src/flamegpu/simulation/detail/CUDAAgent.cu

@@ -57,7 +57,7 @@ namespace cub = hipcub;
 #include "flamegpu/runtime/detail/curve/HostCurve.cuh"
 #include "flamegpu/runtime/detail/curve/curve_rtc.cuh"
 #include "flamegpu/simulation/detail/CUDAScatter.cuh"
-#include "flamegpu/detail/compute_capability.cuh"
+#include "flamegpu/detail/gpu/cuda/compute_capability.cuh"
 #include "flamegpu/util/nvtx.h"
 #include "flamegpu/runtime/agent/DeviceAgentVector_impl.h"
 #include "flamegpu/simulation/detail/CUDAEnvironmentDirectedGraphBuffers.cuh"

tests/CMakeLists.txt

@@ -14,7 +14,7 @@ endif()
 # Prepare source files for the tests target
 SET(TESTS_SRC
     ${CMAKE_CURRENT_SOURCE_DIR}/test_cases/detail/gpu/test_device_name.cu
-    ${CMAKE_CURRENT_SOURCE_DIR}/test_cases/detail/test_compute_capability.cu
+    ${CMAKE_CURRENT_SOURCE_DIR}/test_cases/detail/gpu/cuda/test_compute_capability.cu
     ${CMAKE_CURRENT_SOURCE_DIR}/test_cases/detail/test_cuda.cu
     ${CMAKE_CURRENT_SOURCE_DIR}/test_cases/detail/test_wddm.cu
     ${CMAKE_CURRENT_SOURCE_DIR}/test_cases/detail/test_dependency_versions.cu