mcap/modules/onnxruntime/cuda.jai

// Whether onnxruntime's CUDA provider can run here, and if not, what is missing and how to fix it.
// Shared by the modules that run models (OCR, Labels); what names the feature in the explanations.
// The provider needs the NVIDIA driver, parts of the CUDA 13 toolkit and cuDNN 9 for CUDA 13; it
// fails late and obscurely without them, so each library is looked for first, by the name the
// provider loads it by (from onnxruntime_providers_cuda.dll's imports and the names it loads).
//
// cuDNN's Windows installer does not put its DLLs on PATH. When they are not found, the newest
// cuDNN 9 for CUDA 13 under Program Files\NVIDIA\CUDNN is added to this process's PATH, so the
// provider finds it without anyone editing PATH.

Cuda_Library :: struct {
  name: string;
  part: Cuda_Part;
}

Cuda_Part :: enum { DRIVER; TOOLKIT; CUDNN; }

#if OS == .WINDOWS {
  CUDA_LIBRARIES :: Cuda_Library.[
    .{"nvcuda.dll",                             .DRIVER},
    .{"cudart64_13.dll",                        .TOOLKIT},
    .{"cublas64_13.dll",                        .TOOLKIT},
    .{"cublasLt64_13.dll",                      .TOOLKIT},
    .{"cufft64_12.dll",                         .TOOLKIT},
    .{"cudnn64_9.dll",                          .CUDNN},
    .{"cudnn_graph64_9.dll",                    .CUDNN},
    .{"cudnn_ops64_9.dll",                      .CUDNN},
    .{"cudnn_cnn64_9.dll",                      .CUDNN},
    .{"cudnn_heuristic64_9.dll",                .CUDNN},
    .{"cudnn_engines_precompiled64_9.dll",      .CUDNN},
    .{"cudnn_engines_runtime_compiled64_9.dll", .CUDNN},
  ];
  CUDA_PROVIDER_LIBRARY :: "onnxruntime_providers_cuda.dll";
} else #if OS == .LINUX {
  CUDA_LIBRARIES :: Cuda_Library.[
    .{"libcuda.so.1",        .DRIVER},
    .{"libcudart.so.13",     .TOOLKIT},
    .{"libcublas.so.13",     .TOOLKIT},
    .{"libcublasLt.so.13",   .TOOLKIT},
    .{"libcufft.so.12",      .TOOLKIT},
    .{"libcudnn.so.9",       .CUDNN},
  ];
  CUDA_PROVIDER_LIBRARY :: "libonnxruntime_providers_cuda.so";
} else {
  CUDA_LIBRARIES :: Cuda_Library.[];
  CUDA_PROVIDER_LIBRARY :: "";
}

// Whether the CUDA provider should be tried, and when not, an explanation for people: what is
// missing, and the steps that fix it, starting "<what> runs on the CPU". Temp.
ort_cuda_check :: (what: string) -> available: bool, explanation: string {
  #if OS == .MACOS {
    return false, tprint("% runs on the CPU: onnxruntime has no CUDA on macOS.", what);
  }

  provider_path := tprint("%/%", path_strip_filename(ort.library_path), CUDA_PROVIDER_LIBRARY);
  if !file_exists(provider_path) {
    return false, tprint("% runs on the CPU: the onnxruntime beside the program is the CPU-only build (% is not there).\nTo use an NVIDIA GPU, run `jai fetch.jai` in modules/onnxruntime (without `cpu`) and rebuild.", what, CUDA_PROVIDER_LIBRARY);
  }

  #if OS == .WINDOWS  ort_add_cudnn_to_path();

  missing_driver, missing_toolkit, missing_cudnn: [..] string;
  missing_driver.allocator, missing_toolkit.allocator, missing_cudnn.allocator = temp, temp, temp;
  for CUDA_LIBRARIES {
    if ort_library_loads(it.name)  continue;
    if it.part == {
      case .DRIVER;   array_add(*missing_driver,  it.name);
      case .TOOLKIT;  array_add(*missing_toolkit, it.name);
      case .CUDNN;    array_add(*missing_cudnn,   it.name);
    }
  }
  if !missing_driver.count && !missing_toolkit.count && !missing_cudnn.count  return true, "";

  builder: String_Builder;
  builder.allocator = temp;
  if missing_driver.count {
    print_to_builder(*builder, "% runs on the CPU: no NVIDIA driver was found (", what);
    append(*builder, join(..missing_driver, ", ",, temp));
    append(*builder, "), so there is no NVIDIA GPU to use.");
    return false, builder_to_string(*builder,, temp);
  }

  print_to_builder(*builder, "% runs on the CPU. The NVIDIA GPU can run it once these are installed:\n", what);
  step := 1;
  if missing_toolkit.count {
    print_to_builder(*builder, "\n%. The CUDA Toolkit 13.x, which has % that are missing.\n", step, join(..missing_toolkit, ", ",, temp));
    append(*builder, "   https://developer.nvidia.com/cuda-downloads\n");
    #if OS == .WINDOWS  append(*builder, "   Its installer puts its bin folder on PATH.\n");
    else                append(*builder, "   Then make sure its lib64 folder is on LD_LIBRARY_PATH.\n");
    step += 1;
  }
  if missing_cudnn.count {
    print_to_builder(*builder, "\n%. cuDNN 9.x for CUDA 13, which has % that are missing.\n", step, join(..missing_cudnn, ", ",, temp));
    append(*builder, "   https://developer.nvidia.com/cudnn-downloads\n");
    #if OS == .WINDOWS {
      append(*builder, "   Its installer does not change PATH. Installed in its usual place, C:\\Program Files\\NVIDIA\\CUDNN,\n");
      append(*builder, "   it is found there; elsewhere, add the folder with these DLLs to PATH yourself.\n");
    } else {
      append(*builder, "   Then make sure its lib folder is on LD_LIBRARY_PATH.\n");
    }
    step += 1;
  }
  append(*builder, "\nCopying the DLLs beside the program also works. Restart the program afterwards; ");
  append(*builder, "PATH is read when it starts.");
  return false, builder_to_string(*builder,, temp);
}

#scope_module

// Whether the library loads; one that does stays loaded, for the life of the process. Freeing it
// again let a check on one thread unload cuBLASLt or cuDNN while onnxruntime on another thread was
// loading or using it, which crashed there (a DEP access violation in the unloaded library's memory).
// They are about to be used anyway.
ort_library_loads :: (name: string) -> bool {
  #if OS == .WINDOWS {
    // The search order the CUDA provider's own loads use: beside the program, the system folders, PATH.
    return LoadLibraryW(utf8_to_wide(name,, temp)) != null;
  } else {
    return dlopen(temp_c_string(name), RTLD_LAZY) != null;
  }
}

#if OS == .WINDOWS {
  // When cuDNN does not load, puts the newest Program Files\NVIDIA\CUDNN\v9.*\bin\13.*\x64 (or
  // bin\13.*, as older releases lay it out) holding cudnn64_9.dll at the front of this process's PATH.
  ort_add_cudnn_to_path :: () {
    if ort_library_loads("cudnn64_9.dll")  return;
    program_files := ort_environment_variable("ProgramW6432");
    if !program_files  program_files = ort_environment_variable("ProgramFiles");
    if !program_files  return;

    versions := ort_subfolders_newest_first(tprint("%\\NVIDIA\\CUDNN", program_files), "v9.");
    for version: versions {
      for cuda: ort_subfolders_newest_first(tprint("%\\bin", version), "13.") {
        for folder: string.[tprint("%\\x64", cuda), cuda] {
          if !file_exists(tprint("%\\cudnn64_9.dll", folder))  continue;
          found := replace(folder, "/", "\\",, temp);
          path := ort_environment_variable("PATH");
          OrtSetEnvironmentVariableW(utf8_to_wide("PATH",, temp), utf8_to_wide(tprint("%;%", found, path),, temp));
          log("[onnxruntime] cuDNN is not on PATH; using the one in %.", found);
          return;
        }
      }
    }
  }

  // The folders in parent whose names start with prefix, the highest version first ("v9.26" above
  // "v9.3"). Temp.
  ort_subfolders_newest_first :: (parent: string, prefix: string) -> [..] string {
    Found :: struct { names: [..] string; prefix: string; }
    found: Found;
    found.names.allocator = temp;
    found.prefix = prefix;
    visit_files(parent, recursive = false, *found, (info: *File_Visit_Info, found: *Found) {
      if begins_with(info.short_name, found.prefix)  array_add(*found.names, copy_string(info.full_name,, temp));
    }, visit_files = false, visit_directories = true);
    quick_sort(found.names, (a: string, b: string) -> s64 { return ort_compare_versions(path_filename(b), path_filename(a)); });
    return found.names;
  }

  // Compares the numbers in two names one by one: "v9.26" > "v9.3".
  ort_compare_versions :: (a: string, b: string) -> s64 {
    next_number :: (s: *string) -> s64, found: bool {
      while s.count && (s.*[0] < #char "0" || s.*[0] > #char "9")  advance(s);
      if !s.count  return 0, false;
      n: s64;
      while s.count && s.*[0] >= #char "0" && s.*[0] <= #char "9" { n = n * 10 + s.*[0] - #char "0"; advance(s); }
      return n, true;
    }
    x, y := a, b;
    while true {
      p, p_found := next_number(*x);
      q, q_found := next_number(*y);
      if !p_found || !q_found  return ifx p_found then 1 else ifx q_found then -1 else 0;
      if p != q  return ifx p > q then 1 else -1;
    }
    return 0;
  }

  // Temp; "" when it is not set.
  ort_environment_variable :: (name: string) -> string {
    wide_name := utf8_to_wide(name,, temp);
    size := OrtGetEnvironmentVariableW(wide_name, null, 0);
    if !size  return "";
    buffer := NewArray(size, u16,, temp);
    written := OrtGetEnvironmentVariableW(wide_name, buffer.data, size);
    if !written || written >= size  return "";
    return wide_to_utf8(buffer.data, cast(s32) written,, temp);
  }

  OrtGetEnvironmentVariableW :: (name: *u16, buffer: *u16, size: u32) -> u32 #foreign kernel32 "GetEnvironmentVariableW";
  OrtSetEnvironmentVariableW :: (name: *u16, value: *u16) -> s32 #foreign kernel32 "SetEnvironmentVariableW";
}