built latest HEAD om Linux successfully with Cuda 12.9 and cuDSS 0.6, but getting this memory allocation error (2.2GB) - RTX 4070 with 12GB, 11GB free! #215

Activity

1. hradec commented on Sep 23, 2025

   

   hradec

   on Sep 23, 2025

   Last edited by hradec

   Author

   More actions

   btw, to get the amount of memory cudaHostAlloc was trying to allocate, since the original message just states it failed, I have re-built ceres with this patch applied:

   diff --git a/internal/ceres/block_sparse_matrix.cc b/internal/ceres/block_sparse_matrix.cc
   index e79d62a..4bbf95d 100644
   --- a/internal/ceres/block_sparse_matrix.cc
   +++ b/internal/ceres/block_sparse_matrix.cc
   @@ -37,9 +37,11 @@
    #include <random>
    #include <vector>
    
   +#include "absl/container/flat_hash_set.h"
    #include "absl/log/check.h"
    #include "absl/log/log.h"
    #include "absl/strings/str_format.h"
   +#include "absl/synchronization/mutex.h"
    #include "ceres/block_structure.h"
    #include "ceres/crs_matrix.h"
    #include "ceres/internal/eigen.h"
   @@ -55,6 +57,16 @@
    namespace ceres::internal {
    
    namespace {
   +absl::Mutex& FallbackAllocationMutex() {
   +  static auto* mutex = new absl::Mutex();
   +  return *mutex;
   +}
   +
   +absl::flat_hash_set<double*>& FallbackAllocations() {
   +  static auto* allocations = new absl::flat_hash_set<double*>();
   +  return *allocations;
   +}
   +
    void ComputeCumulativeNumberOfNonZeros(std::vector<CompressedList>& rows) {
      if (rows.empty()) {
        return;
   @@ -815,8 +827,23 @@ double* BlockSparseMatrix::AllocateValues(int size) {
    #ifndef CERES_NO_CUDA
    
      double* values = nullptr;
   -  CHECK_EQ(cudaSuccess,
   -           cudaHostAlloc(&values, sizeof(double) * size, cudaHostAllocDefault));
   +  const size_t bytes = sizeof(double) * static_cast<size_t>(size);
   +  const double gib =
   +      static_cast<double>(bytes) / (1024.0 * 1024.0 * 1024.0);
   +  const cudaError_t status =
   +      cudaHostAlloc(&values, bytes, cudaHostAllocDefault);
   +  if (status == cudaSuccess) {
   +    return values;
   +  }
   +
   +  LOG(WARNING) << "cudaHostAlloc failed with status " << status
   +               << " while allocating " << bytes << " bytes (~" << gib
   +               << " GiB); falling back to pageable host memory.";
   +  values = new double[size];
   +  {
   +    absl::MutexLock lock(&FallbackAllocationMutex());
   +    FallbackAllocations().insert(values);
   +  }
      return values;
    #else
      LOG(FATAL) << "Page locked memory requested when CUDA is not available. "
   @@ -833,6 +860,17 @@ void BlockSparseMatrix::FreeValues(double*& values) {
      }
    
    #ifndef CERES_NO_CUDA
   +  {
   +    absl::MutexLock lock(&FallbackAllocationMutex());
   +    auto& allocations = FallbackAllocations();
   +    auto it = allocations.find(values);
   +    if (it != allocations.end()) {
   +      allocations.erase(it);
   +      delete[] values;
   +      values = nullptr;
   +      return;
   +    }
   +  }
      CHECK_EQ(cudaSuccess, cudaFreeHost(values));
      values = nullptr;
    #else
   

   As cudaHostAlloc tries to allocate "locked memory", I changed my linux locked memory limit do unlimited , as you can see in the ulimit -a output:

   ulimit -a
   real-time non-blocking time  (microseconds, -R) unlimited
   core file size              (blocks, -c) 0
   data seg size               (kbytes, -d) unlimited
   scheduling priority                 (-e) 0
   file size                   (blocks, -f) unlimited
   pending signals                     (-i) 257149
   max locked memory           (kbytes, -l) unlimited
   max memory size             (kbytes, -m) unlimited
   open files                          (-n) 1024
   pipe size                (512 bytes, -p) 8
   POSIX message queues         (bytes, -q) 819200
   real-time priority                  (-r) 0
   stack size                  (kbytes, -s) 8192
   cpu time                   (seconds, -t) unlimited
   max user processes                  (-u) 257149
   virtual memory              (kbytes, -v) unlimited
   file locks                          (-x) unlimited
   

   And I do have 64GB of RAM, with 48GB free RAM and 11GB free of VRAM when the errror ocurred.

   As an attempt to fix the locked memory problem, you can see the patch also adds a fallback to pageable memory if it fails to allocated locked memory, which I think is the reason cuDSS fails in the posted error message.

   This is running in WSL2 Debian 13, btw, But the build environment is controlled by pixi, so all libraries, including gcc, cuda, cuDSS, Boost, etc; are download at build time from conda. I only build Ceres from source to make sure it is built with Cuda and cuDSS support.

   Have anyone seem this cudaHostAlloc failing to allocate locked memory in this fashion before? Any insights would be greatly appreciate.
2. hradec commented on Sep 24, 2025

   

   hradec

   on Sep 24, 2025

   Author

   More actions

   this is the error without the patch:

   -------------------------------------
   Running global positioning ...
   -------------------------------------
   I20250923 17:06:11.221094 261218 image_undistorter.cc:20] Undistorting images..
   I20250923 17:06:11.221278 261218 image_undistorter.cc:41] Image undistortion done
   I20250923 17:06:12.989602 261218 global_positioning.cc:46] Setting up the global positioner problem
   I20250923 17:06:44.560084 261218 global_positioning.cc:71] Solving the global positioner problem
   WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
   F0000 00:00:1758672440.709023  261218 block_sparse_matrix.cc:818] Check failed: cudaSuccess == cudaHostAlloc(&values, sizeof(double) * size, cudaHostAllocDefault) (0 vs. 2) 
   *** Check failure stack trace: ***
   *** Aborted at 1758672440 (unix time) try "date -d @1758672440" if you are using GNU date ***
   PC: @                0x0 (unknown)
   *** SIGABRT (@0x2580003fc62) received by PID 261218 (TID 0x76dac7185000) from PID 261218; stack trace: ***
       @     0x76db3d6f7197 google::(anonymous namespace)::FailureSignalHandler()
       @     0x76db3c834df0 (unknown)
       @     0x76db3c88995c (unknown)
       @     0x76db3c834cc2 gsignal
       @     0x76db3c81d4ac abort
       @     0x76db3d6c5465 absl::lts_20250127::log_internal::LogMessage::FailWithoutStackTrace()
       @     0x76db3d6c6136 absl::lts_20250127::log_internal::LogMessage::Die()
       @     0x76db3d6c714a absl::lts_20250127::log_internal::LogMessageFatal::~LogMessageFatal()
       @     0x76db3d0ec6e2 ceres::internal::BlockSparseMatrix::AllocateValues()
       @     0x76db3d0ec8d2 ceres::internal::BlockSparseMatrix::BlockSparseMatrix()
       @     0x76db3d0d9f0c ceres::internal::BlockJacobianWriter::CreateJacobian()
       @     0x76db3d1c4802 ceres::internal::ProgramEvaluator<>::CreateJacobian()
       @     0x76db3d24d4fb ceres::internal::TrustRegionPreprocessor::Preprocess()
       @     0x76db3d278688 ceres::Solver::Solve()
       @     0x76db3d279c69 ceres::Solve()
       @     0x5825f93f636f glomap::GlobalPositioner::Solve()
       @     0x5825f939be62 glomap::GlobalMapper::Solve()
       @     0x5825f9394460 glomap::RunMapper()
       @     0x5825f938d6bc main
       @     0x76db3c81eca8 (unknown)
       @     0x76db3c81ed65 __libc_start_main
       @     0x5825f938f459 (unknown)
       @                0x0 (unknown)