Support for runtime CPU/CUDA selection

dlib/cuda/cuda_dlib.cu

@@ -5,10 +5,42 @@
 #include "cuda_dlib.h"
 #include "cudnn_dlibapi.h"
 #include <math_constants.h>
+#include <cstdlib>
+#include <cstring>
 
 
 namespace dlib 
 { 
+    namespace
+    {
+        bool cuda_device_available (
+        )
+        {
+            int num_devices;
+            return cudaGetDeviceCount(&num_devices) == cudaSuccess && num_devices > 0;
+        }
+
+        bool cuda_disabled_by_environment (
+        )
+        {
+            const char* var = std::getenv("DLIB_DISABLE_CUDA_USE");
+            return var != nullptr &&
+                std::strcmp(var, "") != 0 &&
+                std::strcmp(var, "0") != 0 &&
+                std::strcmp(var, "false") != 0 &&
+                std::strcmp(var, "False") != 0 &&
+                std::strcmp(var, "FALSE") != 0;
+        }
+
+        bool use_cuda_impl (
+        )
+        {
+            static const bool var = !cuda_disabled_by_environment() && cuda_device_available();
+            return var;
+        }
+
+    }
+
     namespace cuda 
     {
 
@@ -18,21 +50,26 @@ namespace dlib
             int dev
         )
         {
-            CHECK_CUDA(cudaSetDevice(dev));
+            if (use_cuda())
+                CHECK_CUDA(cudaSetDevice(dev));
         }
 
         int get_device (
         )
         {
-            int dev = 0;
-            CHECK_CUDA(cudaGetDevice(&dev));
+            int dev = -1;
+            if (use_cuda())
+                CHECK_CUDA(cudaGetDevice(&dev));
             return dev;
         }
 
         std::string get_device_name (
             int device
         )
         {
+            if (!use_cuda())
+                return "CUDA_DISABLED";
+
             cudaDeviceProp props;
             CHECK_CUDA(cudaGetDeviceProperties(&props, device));
             return props.name;
@@ -41,19 +78,38 @@ namespace dlib
         void set_current_device_blocking_sync(
         )
         {
-            CHECK_CUDA(cudaSetDeviceFlags(cudaDeviceScheduleBlockingSync));
+            if (use_cuda())
+                CHECK_CUDA(cudaSetDeviceFlags(cudaDeviceScheduleBlockingSync));
+        }
+
+        bool is_available(
+        ) 
+        {
+            return use_cuda();
+        }
+
+        bool use_cuda(
+        )
+        {
+            return use_cuda_impl();
         }
 
         int get_num_devices (
         )
         {
+            if (!use_cuda())
+                return 0;
+
             int num_devices;
             CHECK_CUDA(cudaGetDeviceCount(&num_devices));
             return num_devices;
         }
 
         bool can_access_peer (int device_id, int peer_device_id)
         {
+            if (!use_cuda())
+                return false;
+
             int can_access;
             CHECK_CUDA(cudaDeviceCanAccessPeer(&can_access, device_id, peer_device_id));
             return can_access != 0;
@@ -65,6 +121,9 @@ namespace dlib
 
         void device_synchronize (int dev) 
         { 
+            if (!use_cuda())
+                return;
+
             raii_set_device set_dev(dev);
             CHECK_CUDA(cudaDeviceSynchronize());
         }
@@ -3220,4 +3279,3 @@ namespace dlib
 
     }
 }
-

dlib/cuda/cuda_dlib.h

@@ -25,6 +25,12 @@ namespace dlib
         int get_num_devices (
         );
 
+        bool is_available (
+        );
+
+        bool use_cuda(
+        );
+
         std::string get_device_name (
             int device
         );
@@ -942,6 +948,12 @@ namespace dlib
         inline int get_num_devices (
         ) { return 1; }
 
+        inline bool is_available (
+        ) { return false; }
+
+        inline bool use_cuda(
+        ) { return false; }
+
         inline std::string get_device_name (
             int device
         ) 
@@ -979,4 +991,3 @@ namespace dlib
 
 
 #endif // DLIB_DNN_CuDA_H_
-

dlib/cuda/curand_dlibapi.cpp

@@ -6,6 +6,7 @@
 #ifdef DLIB_USE_CUDA
 
 #include "curand_dlibapi.h"
+#include "cuda_dlib.h"
 #include <curand.h>
 #include "../string.h"
 
@@ -47,11 +48,14 @@ namespace dlib
             unsigned long long seed
         ) : handle(nullptr)
         {
-            curandGenerator_t gen;
-            CHECK_CURAND(curandCreateGenerator(&gen, CURAND_RNG_PSEUDO_DEFAULT));
-            handle = gen;
+            if (use_cuda())
+            {
+                curandGenerator_t gen;
+                CHECK_CURAND(curandCreateGenerator(&gen, CURAND_RNG_PSEUDO_DEFAULT));
+                handle = gen;
 
-            CHECK_CURAND(curandSetPseudoRandomGeneratorSeed(gen, seed));
+                CHECK_CURAND(curandSetPseudoRandomGeneratorSeed(gen, seed));
+            }
         }
 
         curand_generator::
@@ -110,4 +114,3 @@ namespace dlib
 #endif // DLIB_USE_CUDA
 
 #endif // DLIB_DNN_CuRAND_CPP_
-

dlib/cuda/gpu_data.cpp

@@ -54,13 +54,19 @@ namespace dlib
         }
         else
         {
+            if (!cuda::use_cuda())
+            {
+                std::memcpy(dest.host()+dest_offset, src.host()+src_offset, num*sizeof(float));
+                return;
+            }
+
             // if we write to the entire thing then we can use device_write_only()
             if (dest_offset == 0 && num == dest.size())
             {
                 // copy the memory efficiently based on which copy is current in each object.
                 if (src.device_ready())
                     CHECK_CUDA(cudaMemcpy(dest.device_write_only(), src.device()+src_offset,  num*sizeof(float), cudaMemcpyDeviceToDevice));
-                else 
+                else
                     CHECK_CUDA(cudaMemcpy(dest.device_write_only(), src.host()+src_offset,    num*sizeof(float), cudaMemcpyHostToDevice));
             }
             else
@@ -72,7 +78,7 @@ namespace dlib
                     CHECK_CUDA(cudaMemcpy(dest.host()+dest_offset, src.device()+src_offset,   num*sizeof(float), cudaMemcpyDeviceToHost));
                 else if (dest.device_ready() && !src.device_ready())
                     CHECK_CUDA(cudaMemcpy(dest.device()+dest_offset, src.host()+src_offset,   num*sizeof(float), cudaMemcpyHostToDevice));
-                else 
+                else
                     CHECK_CUDA(cudaMemcpy(dest.host()+dest_offset, src.host()+src_offset,     num*sizeof(float), cudaMemcpyHostToHost));
             }
         }
@@ -199,6 +205,13 @@ namespace dlib
             device_current = true;
             device_in_use = false;
 
+            if (!cuda::use_cuda())
+            {
+                data_host.reset(new float[new_size], std::default_delete<float[]>());
+                the_device_id = -1;
+                return;
+            }
+
             try
             {
                 CHECK_CUDA(cudaGetDevice(&the_device_id));
@@ -251,4 +264,3 @@ namespace dlib
 #endif // DLIB_USE_CUDA
 
 #endif // DLIB_GPU_DaTA_CPP_
-

dlib/cuda/gpu_data.h

@@ -12,6 +12,14 @@
 namespace dlib
 {
 
+// ----------------------------------------------------------------------------------------
+
+    namespace cuda
+    {
+        bool use_cuda(
+        );
+    }
+
 // ----------------------------------------------------------------------------------------
 
     class gpu_data 
@@ -93,14 +101,16 @@ namespace dlib
         float* host() 
         {
             copy_to_host();
-            device_current = false;
+            if (device_id() >= 0)
+                device_current = false;
             return data_host.get(); 
         }
 
         float* host_write_only() 
         {
             host_current = true;
-            device_current = false;
+            if (device_id() >= 0)
+                device_current = false;
             return data_host.get(); 
         }
 
@@ -109,6 +119,7 @@ namespace dlib
 #ifndef DLIB_USE_CUDA
             DLIB_CASSERT(false, "CUDA NOT ENABLED");
 #endif
+            DLIB_CASSERT(device_id() >= 0, "This data is host only");
             copy_to_device();
             device_in_use = true;
             return data_device.get(); 
@@ -119,6 +130,7 @@ namespace dlib
 #ifndef DLIB_USE_CUDA
             DLIB_CASSERT(false, "CUDA NOT ENABLED");
 #endif
+            DLIB_CASSERT(device_id() >= 0, "This data is host only");
             copy_to_device();
             host_current = false;
             device_in_use = true;
@@ -130,6 +142,7 @@ namespace dlib
 #ifndef DLIB_USE_CUDA
             DLIB_CASSERT(false, "CUDA NOT ENABLED");
 #endif
+            DLIB_CASSERT(device_id() >= 0, "This data is host only");
             wait_for_transfer_to_finish();
             host_current = false;
             device_current = true;
@@ -141,7 +154,7 @@ namespace dlib
         ) const { return host_current; }
 
         bool device_ready (
-        ) const { return device_current && !have_active_transfer; }
+        ) const { return device_current && !have_active_transfer && device_id() >= 0; }
 
         size_t size() const { return data_size; }
 

dlib/cuda/tensor.h

@@ -12,6 +12,18 @@
 #include <memory>
 #include "../any.h"
 
+#ifdef DLIB_USE_CUDA
+#define IF_DLIB_USE_CUDA(...) if (cuda::use_cuda()) { __VA_ARGS__ }
+#else
+#define IF_DLIB_USE_CUDA(...)
+#endif
+
+#ifdef DLIB_USE_CUDA
+#define IF_DLIB_NOT_USE_CUDA(...) if (!cuda::use_cuda()) { __VA_ARGS__ }
+#else
+#define IF_DLIB_NOT_USE_CUDA(...) __VA_ARGS__
+#endif
+
 namespace dlib
 {
 
@@ -77,17 +89,18 @@ namespace dlib
 
         tensor& operator= (float val)
         {
-#ifdef DLIB_USE_CUDA
-            // If you are using CUDA then presumably you will be mostly using tensors on
-            // the GPU.  So unless you seem to be actively working with the host side's
-            // data then we do this initialization on the device side since this avoids a
-            // host to device transfer that would likely immediately follow.
-            if (data().device_ready())
-            {
-                cuda::set_tensor(*this, val);
-                return *this;
-            }
-#endif
+            IF_DLIB_USE_CUDA(
+                // If you are using CUDA then presumably you will be mostly using tensors on
+                // the GPU.  So unless you seem to be actively working with the host side's
+                // data then we do this initialization on the device side since this avoids a
+                // host to device transfer that would likely immediately follow.
+                if (data().device_ready())
+                {
+                    cuda::set_tensor(*this, val);
+                    return *this;
+                }
+            )
+
             auto d = host_write_only();
             for (size_t i = 0; i < size(); ++i)
                 d[i] = val;
@@ -97,15 +110,16 @@ namespace dlib
 
         tensor& operator*= (float val)
         {
-#ifdef DLIB_USE_CUDA
-            cuda::scale_tensor(*this, val);
-            return *this;
-#else
-            for (auto& d : *this)
-                d *= val;
+            IF_DLIB_USE_CUDA(
+                cuda::scale_tensor(*this, val);
+            )
+
+            IF_DLIB_NOT_USE_CUDA(
+                for (auto& d : *this)
+                    d *= val;
+            )
 
             return *this;
-#endif
         }
 
         tensor& operator/= (float val)