[0.1.26] gpu-bench: a memory trace per step, -ocr-workers and -ort-log

Each result now has the GPU's memory over the baseline after loading, the
first image, each warm-up pass and each timed pass (with the pass's peak), in
the output and as trace= in the results file, so growth across passes (a leak)
shows apart from a working set. -ocr-workers sets OCR's GPU worker threads, and
-ort-log info prints onnxruntime's arena extensions: what found the cause of
0.1.24's memory fix.

.gitattributes

@@ -0,0 +1 @@
+tools/gpu_bench/corpus/** binary

README.md

@@ -101,16 +101,15 @@ grid (in its order, so through a search's matches).
 - **Model:** PP-OCRv5 mobile (light, about 85 ms a capture on the GPU) or server (more accurate,
   about 0.3 s when it runs well; see below). The choice is kept in the settings file (`model` under
   `[ocr]`).
-- **Known issue, the server model on the GPU:** its speed is bimodal. The same program on the same
-  20 captures (RTX 5080 laptop, 16 GB) runs either at about 0.27 s a capture with a peak of about
-  2.4 GB of process memory, or at about 1.5 s with about 6.6 GB, with no change between runs and
-  15 GB of the GPU's memory free beforehand. Longer runs fare worse: 50 captures peaked at 16 GB, and
-  100 averaged about 4.6 s a capture. The extra memory looks like GPU allocations spilling into shared
-  system memory, which Windows then pages: what the GPU memory defaults (onnxruntime's
-  kNextPowerOfTwo arena, no cuDNN max workspace, 4 workers) were meant to stop, and which they
-  only partly did. It is the same with Jai's allocator and with rpmalloc. Until it is fixed, scan a
-  large library with the mobile model. Measured on 2026-10-13 with
-  `ocr-bench -models ppocr-v5-server -gpu-only -limit 20`.
+- **GPU memory** (fixed in 0.1.24): OCR used to fill the GPU, 9 GB with the mobile model on
+  gpu-bench's 30 images and past 16 GB with the server model, which Windows then paged into shared
+  system memory, running it at 1.5 to 11 s a capture. onnxruntime's arena keeps every block it takes
+  from CUDA, and OCR's inputs change size every run, so each larger image or wider batch of lines left
+  another block behind. OCR now runs the largest images and widest batches first, has the detector's
+  and recognizer's arenas give back what they hold after each stage of a call, and keeps its 4 worker
+  threads for the engine's life. The mobile model now peaks at 5.7 GB and drops to 2 GB between
+  calls, at 142 ms a capture against 126 before (gpu-bench, 2026-10-17). The server model has not been
+  measured again yet.
 - **Running on:** the GPU or the CPU, and if the CPU, what to install for the GPU.
 - **Scan the library** reads every capture the model has not read yet and, with **Labels too**,
   labels every capture not yet labelled with the current list. While a search is active the button

check_targets.jai

@@ -47,7 +47,7 @@ APP_NAME        :: "mcap";
 APP_FOLDER_NAME :: "mcap";
 GIT_BRANCH    :: "check";
 GIT_HASH      :: "-";
-VERSION       :: "0.1.23";
+VERSION       :: "0.1.26";
 BUILD_DATE    :: "-";
 DEBUG         :: true;
 COMPILE_TESTS :: true;

first.jai

@@ -8,7 +8,7 @@ APP_NAME              :: "mcap";
 APP_FOLDER_NAME       :: "mcap";
 APP_BUNDLE_IDENTIFIER :: "com.example.mcap";
 
-VERSION :: "0.1.23";
+VERSION :: "0.1.26";
 // Where everything goes: bin/, or bin-headless/ for builds made with no_crash_dialog, so automated
 // builds never write over the ones you run by hand.
 build_directory := "bin";
@@ -70,6 +70,10 @@ build_project :: () {
     // jai first.jai - record_probe: also bin/record-probe, a screen recorded with modules/Screen_Record (video-capture-plan.html, phase 1).
     if array_find(args, "record_probe")  array_add(*pending, start_tool_build("record-probe", "tools/record_probe.jai", optimized = build_release));
 
+    // The gpu-bench branch: bin/gpu-bench, every GPU model timed over a fixed corpus (tools/gpu_bench),
+    // always built, as the GPU Benchmark window runs it.
+    array_add(*pending, start_tool_build("gpu-bench", "tools/gpu_bench/gpu_bench.jai", optimized = build_release));
+
     wait_for_builds(pending);
 
     for pending  if !it.failed {
@@ -312,6 +316,11 @@ copy_assets :: () {
     } else {
         log("Segmentation: the model has not been made; see modules/Segment/README.md.");
     }
+
+    // gpu-bench's corpus, in bin/gpu-bench-corpus: in git, so every machine has the same images.
+    make_directory_if_it_does_not_exist(tprint("%/gpu-bench-corpus", build_directory));
+    copy_if_changed("tools/gpu_bench/corpus.txt", tprint("%/gpu-bench-corpus/corpus.txt", build_directory));
+    for file_list("tools/gpu_bench/corpus")  copy_if_changed(it, tprint("%/gpu-bench-corpus/%", build_directory, path_filename(it)));
 }
 
 copy_if_changed :: (from: string, to: string) {

modules/Labels/module.jai

@@ -102,7 +102,7 @@ labels_create :: (models_folder: string, model_name: string, prefer_gpu := true)
   engine := New(Labels_Engine);
   engine.model = model.*;
   api := ort.api;
-  ok, error := ort_check(api.CreateEnv(.ORT_LOGGING_LEVEL_WARNING, "labels", *engine.env));
+  ok, error := ort_check(api.CreateEnv(ort.log_level, "labels", *engine.env));
   if !ok { labels_destroy(engine); return null, false, error; }
   ort_check(api.CreateCpuMemoryInfo(.OrtArenaAllocator, .OrtMemTypeDefault, *engine.memory_info));
 

modules/OCR/models.jai

@@ -89,9 +89,11 @@ ocr_create_sessions :: (engine: *Ocr_Engine, models_folder: string, gpu: bool, i
   model := *engine.model;
   ok: bool;
   error: string;
-  engine.det, ok, error = ocr_create_session(engine.env, tprint("%/%", models_folder, model.detector), gpu, intra_op_threads, engine.options);
+  // The detector's and recognizer's inputs change size with nearly every run: their arenas give back
+  // what no run is using once per call (ocr_shrink_arena), or they keep a block of every size.
+  engine.det, ok, error = ocr_create_session(engine.env, tprint("%/%", models_folder, model.detector), gpu, intra_op_threads, engine.options, shrink_arena = true);
   if !ok  return false, error;
-  engine.rec, ok, error = ocr_create_session(engine.env, tprint("%/%", models_folder, model.recognizer), gpu, intra_op_threads, engine.options);
+  engine.rec, ok, error = ocr_create_session(engine.env, tprint("%/%", models_folder, model.recognizer), gpu, intra_op_threads, engine.options, shrink_arena = true);
   if !ok  return false, error;
   if model.textline_ori {
     engine.orientation, ok, error = ocr_create_session(engine.env, tprint("%/%", models_folder, model.textline_ori), gpu, intra_op_threads, engine.options);
@@ -115,6 +117,19 @@ ocr_release_sessions :: (engine: *Ocr_Engine) {
 // engine destroyed without ever having run left the next engine to crash in cublasLtCtxInit, with a
 // DEP access violation, on its first run; an engine that has run once leaves nothing broken. It also
 // moves the first run's setup into creation. On failure, the reason.
+// On CUDA, the session's arena (the detector's or the recognizer's) gives back every block no run is
+// using: called by ocr_process once a stage's runs are all done, as a run on a small input, since
+// onnxruntime shrinks an arena only at the end of a run. Once per run instead cost a third to a half
+// of OCR's speed.
+ocr_shrink_arena :: (engine: *Ocr_Engine, session: *Ocr_Session) {
+  if engine.provider != .CUDA || !session.session  return;
+  side := ifx session == *engine.rec then cast(s64) engine.model.rec_height else 32;
+  shape := s64.[1, 3, side, side];
+  input := NewArray(3 * side * side, float, initialized = true,, temp);
+  out, ok := ocr_run(session, engine.memory_info, input, shape, shrink = true);
+  if ok  ocr_release_output(*out);
+}
+
 ocr_warm_up :: (engine: *Ocr_Engine) -> ok: bool, error: string {
   Warm_Up :: struct { session: *Ocr_Session; height, width: s64; }
   runs := Warm_Up.[
@@ -135,8 +150,8 @@ ocr_warm_up :: (engine: *Ocr_Engine) -> ok: bool, error: string {
 }
 
 // A session for the model file, set up as Ocr_Options says (modules/onnxruntime/session.jai).
-ocr_create_session :: (env: *OrtEnv, path: string, gpu: bool, intra_op_threads: s32, ocr_options: Ocr_Options) -> Ocr_Session, ok: bool, error: string {
-  settings := Ort_Gpu_Settings.{arena_extend_strategy = ocr_options.cuda_arena_extend_strategy, cudnn_max_workspace = ocr_options.cudnn_max_workspace};
+ocr_create_session :: (env: *OrtEnv, path: string, gpu: bool, intra_op_threads: s32, ocr_options: Ocr_Options, shrink_arena := false) -> Ocr_Session, ok: bool, error: string {
+  settings := Ort_Gpu_Settings.{arena_extend_strategy = ocr_options.cuda_arena_extend_strategy, cudnn_max_workspace = ocr_options.cudnn_max_workspace, shrink_arena = shrink_arena};
   session, ok, error := ort_create_session(env, path, gpu, intra_op_threads, settings, missing_hint = "run `jai fetch_models.jai` in modules/OCR and rebuild");
   return session, ok, error;
 }

modules/OCR/module.jai

@@ -47,6 +47,7 @@ Ocr_Engine :: struct {
   rec_batch: s32;             // text lines per recognizer run: the model's rec_batch_gpu or rec_batch_cpu
   options: Ocr_Options;
   stats: Ocr_Stats;           // since creation, or the last ocr_reset_stats
+  pool: Ocr_Pool;             // the worker threads, for the engine's life (run_on_workers)
 
   // When set, and true, ocr_process stops starting work: the batch of lines or image each worker is
   // on is finished, and then every image fails as "cancelled" (none is part read). Set by the caller.
@@ -72,13 +73,13 @@ Ocr_Stats :: struct {
 ocr_stats :: (engine: *Ocr_Engine) -> Ocr_Stats { return engine.stats; }
 ocr_reset_stats :: (engine: *Ocr_Engine) { engine.stats = .{}; }
 
-// How an engine runs on the GPU, beyond its model's models.config entry. The defaults are what ran
-// fastest on the 20 sample captures (RTX 5080 laptop, onnxruntime 1.30.0): nearly every image and
-// batch is a different size, and with onnxruntime's exact-size arena and cuDNN's largest workspaces
-// GPU memory filled up and the server model ran several times slower.
+// How an engine runs on the GPU, beyond its model's models.config entry. Nearly every image and
+// batch is a different size. onnxruntime's arena keeps each block it takes from CUDA and never joins
+// or frees them, so inputs that grow one run after another leave a block of every size behind: the
+// runs go largest first (ocr_process) so later ones fit in what the first took.
 Ocr_Options :: struct {
-  gpu_workers: s32 = 4;           // threads preparing images and decoding results while the GPU runs the networks; more let the server model fill GPU memory
-  cuda_arena_extend_strategy := "kNextPowerOfTwo";   // reuses freed blocks for other sizes; "kSameAsRequested" grows by exactly what is asked
+  gpu_workers: s32 = 4;           // threads preparing images and decoding results while the GPU runs the networks
+  cuda_arena_extend_strategy := "kSameAsRequested";  // how much the arena takes when it runs short: what is asked; "kNextPowerOfTwo" takes twice as much each time, up to twice what the runs need
   cudnn_max_workspace := false;   // let cuDNN take as much working memory as it finds best (onnxruntime's own default)
   rec_width_step: s32 = 64;       // a recognizer batch's width is rounded up to a multiple of this, so shapes recur
 }
@@ -109,7 +110,7 @@ ocr_create :: (models_folder: string, model_name: string, prefer_gpu := true, op
   engine.dictionary = entries;
 
   api := ort.api;
-  ok, error := ort_check(api.CreateEnv(.ORT_LOGGING_LEVEL_WARNING, "ocr", *engine.env));
+  ok, error := ort_check(api.CreateEnv(ort.log_level, "ocr", *engine.env));
   if !ok { ocr_destroy(engine); return null, false, error; }
   ort_check(api.CreateCpuMemoryInfo(.OrtArenaAllocator, .OrtMemTypeDefault, *engine.memory_info));
 
@@ -149,6 +150,7 @@ ocr_create :: (models_folder: string, model_name: string, prefer_gpu := true, op
     engine.provider = .CPU;
     engine.rec_batch = max(engine.model.rec_batch_cpu, 1);
   }
+  ocr_pool_start(engine);
   return engine, true, "";
 }
 
@@ -163,6 +165,7 @@ ocr_gpu_status :: (engine: *Ocr_Engine) -> in_use: bool, explanation: string {
 
 ocr_destroy :: (engine: *Ocr_Engine) {
   if !engine  return;
+  ocr_pool_stop(engine);
   if engine.provider == .CUDA  ort_gpu_lock();
   ocr_release_sessions(engine);
   if engine.provider == .CUDA  ort_gpu_unlock();
@@ -185,13 +188,22 @@ ocr_process :: (engine: *Ocr_Engine, images: [] Ocr_Image, allocator := context.
     array_free(per_image,, context.default_allocator);
   }
   for * per_image  it.image = *images[it_index];
-  run_on_workers(engine, per_image.count, (engine: *Ocr_Engine, index: s64, data: *void) {
-    work := cast(*Image_Work) data + index;
+  // The largest first: the detector's arena then holds a block that fits every later image
+  // (Ocr_Options). Smallest first, it took a new one for each larger image and kept them all.
+  detect_order := NewArray(per_image.count, *Image_Work,, context.default_allocator);
+  defer array_free(detect_order,, context.default_allocator);
+  for * per_image  detect_order[it_index] = it;
+  quick_sort(detect_order, (a: *Image_Work, b: *Image_Work) -> s64 {
+    return cast(s64) b.image.width * b.image.height - cast(s64) a.image.width * a.image.height;
+  });
+  run_on_workers(engine, detect_order.count, (engine: *Ocr_Engine, index: s64, data: *void) {
+    work := (cast(**Image_Work) data)[index];
     start := current_time_monotonic();
     work.boxes, work.ok, work.error = ocr_detect(engine, work.image.*);
     atomic_add(*engine.stats.detect_us, to_microseconds(current_time_monotonic() - start));
     if work.error  work.error = copy_string(work.error,, context.default_allocator);
-  }, per_image.data);
+  }, detect_order.data);
+  ocr_shrink_arena(engine, *engine.det);   // before recognition, so the two stages' blocks are not held at once
   if ocr_cancelled(engine)  return ocr_all_cancelled(results, allocator);
 
   // Recognition: every line of every image, batched by width so little of each batch is padding.
@@ -247,7 +259,10 @@ ocr_process :: (engine: *Ocr_Engine, images: [] Ocr_Image, allocator := context.
   order := NewArray(jobs.count, *Line_Job,, context.default_allocator);
   defer array_free(order,, context.default_allocator);
   for * jobs  order[it_index] = it;
-  quick_sort(order, (a: *Line_Job, b: *Line_Job) -> s64 { return a.width - b.width; });
+  // Widest first, so each batch fits in the arena block the first one took (Ocr_Options). Narrowest
+  // first, every batch was wider than the last and took a new block: 5.7 GB of them for the mobile
+  // model on gpu-bench's 30 images, where the widest batch needs 0.9 GB.
+  quick_sort(order, (a: *Line_Job, b: *Line_Job) -> s64 { return b.width - a.width; });
 
   batch_size := max(engine.rec_batch, 1);
   batch_count := (order.count + batch_size - 1) / batch_size;
@@ -281,6 +296,8 @@ ocr_process :: (engine: *Ocr_Engine, images: [] Ocr_Image, allocator := context.
     }
   }
 
+  ocr_shrink_arena(engine, *engine.rec);   // models.jai: what this call's runs took, given back
+
   // Each image's lines, in reading order.
   for work, image_index: per_image {
     result := *results[image_index];
@@ -337,48 +354,93 @@ Image_Work :: struct {
 Work_Proc :: #type (engine: *Ocr_Engine, index: s64, data: *void);
 
 // Calls proc for each index below count, on engine.workers threads at once.
-run_on_workers :: (engine: *Ocr_Engine, count: s64, proc: Work_Proc, data: *void) {
-  if count <= 0  return;
-  Shared :: struct { engine: *Ocr_Engine; count: s64; next: s64; proc: Work_Proc; data: *void; }
-  shared := Shared.{engine = engine, count = count, proc = proc, data = data};
+// The engine's worker threads: started with it and kept until ocr_destroy, never one set per call.
+// onnxruntime (1.26 to 1.30 at least) sets up a CUDA stream with cuBLAS and cuDNN handles for every
+// OS thread that runs a session, about 10 MB each, kept after the thread ends (microsoft/onnxruntime
+// #33037). Workers made afresh for every stage of every call, as they were, gave the OCR sessions
+// thousands of threads in a scan, and starting them cost time: 6% on gpu-bench's mobile OCR.
+Ocr_Pool :: struct {
+  threads: [] Thread;
+  started: s64;
+  mutex: Mutex;              // one run_on_workers at a time
+  wake: Semaphore;           // signalled once per worker for each job, and for quitting
+  done: Semaphore;           // each worker signals it when it has finished its part of a job
+  quit: bool;
+  // The job: items 0..count-1, taken in turn.
+  count, next: s64;
+  proc: Work_Proc;
+  data: *void;
+}
 
-  worker_proc :: (thread: *Thread) -> s64 {
-    // The context comes from the calling thread, allocator included, and that may be its temporary
-    // allocator, which belongs to that thread alone. Workers allocate from the heap, and use their
-    // own temporary storage.
-    context.allocator = context.default_allocator;
-    shared := cast(*Shared) thread.data;
+ocr_pool_start :: (engine: *Ocr_Engine) {
+  pool := *engine.pool;
+  init(*pool.mutex);
+  init(*pool.wake);
+  init(*pool.done);
+  // thread_init allocates the thread's temporary storage from the context allocator.
+  push_allocator(context.default_allocator);
+  pool.threads = NewArray(max(engine.workers, 1), Thread);
+  for * pool.threads {
+    if !thread_init(it, ocr_pool_worker, temporary_storage_size = 1024 * 1024)  break;
+    thread_start(it, engine);   // sets thread.data
+    pool.started += 1;
+  }
+}
+
+ocr_pool_stop :: (engine: *Ocr_Engine) {
+  pool := *engine.pool;
+  if !pool.threads.count  return;
+  pool.quit = true;
+  for 1..pool.started  signal(*pool.wake);
+  for 0..pool.started - 1 {
+    thread_is_done(*pool.threads[it], -1);
+    thread_deinit(*pool.threads[it]);
+  }
+  array_free(pool.threads,, context.default_allocator);
+  destroy(*pool.wake);
+  destroy(*pool.done);
+  destroy(*pool.mutex);
+  pool.* = .{};
+}
+
+ocr_pool_worker :: (thread: *Thread) -> s64 {
+  // Workers allocate from the heap, and use their own temporary storage.
+  context.allocator = context.default_allocator;
+  engine := cast(*Ocr_Engine) thread.data;
+  pool := *engine.pool;
+  while true {
+    Default_Allocator.thread_idle();   // parked until the next job: give back what this thread's heap holds free
+    wait_for(*pool.wake);
+    if pool.quit  break;
     while true {
-      if ocr_cancelled(shared.engine)  break;   // what is left fails as cancelled (ocr_process)
-      index := atomic_add(*shared.next, 1);
-      if index >= shared.count  break;
-      shared.proc(shared.engine, index, shared.data);
+      if ocr_cancelled(engine)  break;   // what is left fails as cancelled (ocr_process)
+      index := atomic_add(*pool.next, 1);
+      if index >= pool.count  break;
+      pool.proc(engine, index, pool.data);
       reset_temporary_storage();
     }
-    return 0;
+    signal(*pool.done);
   }
+  return 0;
+}
 
-  // thread_init allocates the thread's temporary storage from the context allocator and tells it
-  // the thread is starting; the caller's may be a temporary allocator, which refuses that.
-  push_allocator(context.default_allocator);
-  thread_count := min(cast(s64) engine.workers, count);
-  threads := NewArray(thread_count, Thread,, context.default_allocator);
-  defer array_free(threads,, context.default_allocator);
-  started := 0;
-  for * threads {
-    if !thread_init(it, worker_proc, temporary_storage_size = 1024 * 1024)  break;
-    thread_start(it, *shared);   // sets thread.data
-    started += 1;
-  }
-  if !started {   // no threads: do it here
-    here: Thread;
-    here.data = *shared;
-    worker_proc(*here);
-  }
-  for 0..started - 1 {
-    thread_is_done(*threads[it], -1);
-    thread_deinit(*threads[it]);
+run_on_workers :: (engine: *Ocr_Engine, count: s64, proc: Work_Proc, data: *void) {
+  if count <= 0  return;
+  pool := *engine.pool;
+  if !pool.started {   // no threads: do it here
+    for 0..count - 1 {
+      if ocr_cancelled(engine)  break;
+      proc(engine, it, data);
+    }
+    return;
   }
+  lock(*pool.mutex);
+  defer unlock(*pool.mutex);
+  pool.count, pool.next, pool.proc, pool.data = count, 0, proc, data;
+  // Every worker is woken and reports back, even with fewer items than workers: a worker that finds
+  // nothing left reports at once, so no wake is left over for the next job.
+  for 1..pool.started  signal(*pool.wake);
+  for 1..pool.started  wait_for(*pool.done);
 }
 
 #load "models.jai";
@@ -395,6 +457,7 @@ run_on_workers :: (engine: *Ocr_Engine, count: s64, proc: Work_Proc, data: *void
 #import "Thread";
 #import "Atomics";
 #import "onnxruntime";
+Default_Allocator :: #import "Default_Allocator";   // rpmalloc (modules/Default_Allocator), for thread_idle where the workers wait
 #if OS == .WINDOWS {
   #import "Windows";
   #import "Windows_Utf8";

modules/Segment/module.jai

@@ -148,7 +148,7 @@ segment_create :: (models_folder: string, model_name: string, prefer_gpu := true
   if !ok { segment_destroy(engine); return null, false, error; }
 
   api := ort.api;
-  ok, error = ort_check(api.CreateEnv(.ORT_LOGGING_LEVEL_WARNING, "segment", *engine.env));
+  ok, error = ort_check(api.CreateEnv(ort.log_level, "segment", *engine.env));
   if !ok { segment_destroy(engine); return null, false, error; }
   ort_check(api.CreateCpuMemoryInfo(.OrtArenaAllocator, .OrtMemTypeDefault, *engine.memory_info));
 

modules/onnxruntime/module.jai

@@ -9,6 +9,9 @@ ort: struct {
   api: *OrtApi;
   library_path: string;   // the library that was loaded
   version: string;        // onnxruntime's own version string, such as 1.30.0
+  // What the engines' environments log, to the console: set before the first engine is made, as
+  // onnxruntime keeps one environment per process. INFO shows each GPU arena extension and its size.
+  log_level := OrtLoggingLevel.ORT_LOGGING_LEVEL_WARNING;
 }
 
 // Loads onnxruntime from the first folder that has it: beside the running executable, then each

modules/onnxruntime/session.jai

@@ -20,17 +20,21 @@ ort_gpu_held: s32;
 
 Ort_Session :: struct {
   session: *OrtSession;
+  shrink_options: *OrtRunOptions; // for a run that shrinks the arena afterwards (ort_run's shrink); null when the settings did not ask for it
   input_name, output_name: *u8;   // heap C strings
   input_shape: [4] s64;           // as the model declares it: -1 for a size it leaves open
   input_rank: s64;
 }
 
-// How a CUDA session uses GPU memory. The defaults came from measuring OCR on the sample captures
-// (RTX 5080 laptop): with nearly every input a new size, onnxruntime's exact-size arena filled GPU
-// memory, and cuDNN's largest workspaces made it worse.
+// How a CUDA session uses GPU memory. Its arena takes blocks from CUDA as runs need them, and by
+// default never joins or frees them: a model whose input size changes (OCR's) leaves a block of
+// every larger size behind, gigabytes of them, unless the arena is shrunk now and then. Shrinking
+// after every run costs a third to a half of OCR's speed (CUDA frees and allocates again, waiting for
+// the GPU), so OCR shrinks once per call (ocr_process).
 Ort_Gpu_Settings :: struct {
-  arena_extend_strategy := "kNextPowerOfTwo";   // reuses freed blocks for other sizes; "kSameAsRequested" grows by exactly what is asked
+  arena_extend_strategy := "kNextPowerOfTwo";   // how much the arena takes when it runs short: twice as much each time; "kSameAsRequested" takes what is asked
   cudnn_max_workspace := false;                 // let cuDNN take as much working memory as it finds best (onnxruntime's own default)
+  shrink_arena := false;                        // a run with ort_run's shrink set then gives back the arena's blocks no run is using
 }
 
 // A session for the model file at path; with CUDA when gpu is set. output names the output that
@@ -65,6 +69,12 @@ ort_create_session :: (env: *OrtEnv, path: string, gpu: bool, intra_op_threads:
   if !file_exists(path)  return result, false, tprint("% is not there%.", path, ifx missing_hint then tprint("; %", missing_hint) else "");
   ok, error = ort_check(api.CreateSession(env, ort_path(path), options, *result.session));
   if !ok  return result, false, error;
+  if gpu && settings.shrink_arena {
+    // Device 0's arena, as device_id above; the CPU's is left alone.
+    ok, error = ort_check(api.CreateRunOptions(*result.shrink_options));
+    if ok  ok, error = ort_check(api.AddRunConfigEntry(result.shrink_options, "memory.enable_memory_arena_shrinkage", "gpu:0"));
+    if !ok { ort_release_session(*result); return result, false, error; }
+  }
 
   allocator: *OrtAllocator;
   ort_check(api.GetAllocatorWithDefaultOptions(*allocator));
@@ -107,6 +117,7 @@ ort_create_session :: (env: *OrtEnv, path: string, gpu: bool, intra_op_threads:
 
 ort_release_session :: (s: *Ort_Session) {
   if s.session  ort.api.ReleaseSession(s.session);
+  if s.shrink_options  ort.api.ReleaseRunOptions(s.shrink_options);
   free(s.input_name,, context.default_allocator);
   free(s.output_name,, context.default_allocator);
   s.* = .{};
@@ -121,8 +132,9 @@ Ort_Output :: struct {
 }
 
 // Runs the session on one input tensor of shape, bytes long at data, with element type element.
-// A session may be run from several threads at once.
-ort_run :: (s: *Ort_Session, memory_info: *OrtMemoryInfo, data: *void, bytes: s64, element: ONNXTensorElementDataType, shape: [] s64) -> Ort_Output, ok: bool, error: string {
+// A session may be run from several threads at once. With shrink, and a session made with
+// shrink_arena, the arena gives back the blocks no run is using once this run is done.
+ort_run :: (s: *Ort_Session, memory_info: *OrtMemoryInfo, data: *void, bytes: s64, element: ONNXTensorElementDataType, shape: [] s64, shrink := false) -> Ort_Output, ok: bool, error: string {
   out: Ort_Output;
   api := ort.api;
   input: *OrtValue;
@@ -130,7 +142,7 @@ ort_run :: (s: *Ort_Session, memory_info: *OrtMemoryInfo, data: *void, bytes: s6
   if !ok  return out, false, error;
   defer api.ReleaseValue(input);
 
-  ok, error = ort_check(api.Run(s.session, null, *s.input_name, *input, 1, *s.output_name, 1, *out.value));
+  ok, error = ort_check(api.Run(s.session, ifx shrink then s.shrink_options else null, *s.input_name, *input, 1, *s.output_name, 1, *out.value));
   if !ok  return out, false, error;
 
   raw: *void;
@@ -146,8 +158,8 @@ ort_run :: (s: *Ort_Session, memory_info: *OrtMemoryInfo, data: *void, bytes: s6
   return out, true, "";
 }
 
-ort_run_float :: (s: *Ort_Session, memory_info: *OrtMemoryInfo, data: [] float, shape: [] s64) -> Ort_Output, ok: bool, error: string {
-  out, ok, error := ort_run(s, memory_info, data.data, data.count * size_of(float), .ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, shape);
+ort_run_float :: (s: *Ort_Session, memory_info: *OrtMemoryInfo, data: [] float, shape: [] s64, shrink := false) -> Ort_Output, ok: bool, error: string {
+  out, ok, error := ort_run(s, memory_info, data.data, data.count * size_of(float), .ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, shape, shrink);
   return out, ok, error;
 }
 

src/App.jai

@@ -26,6 +26,8 @@ USING_WINDOWS :: OS == .WINDOWS;
 #load "app/import.jai";
 #load "app/open_file.jai";
 #load "app/segment_ui.jai";
+#load "app/gpu_bench_ui.jai";          // the gpu-bench branch: the GPU Benchmark window
+#load "../tools/gpu_bench/results.jai";  // its results files
 #load "app/library_folders.jai";
 #load "app/exit_shutdown.jai";
 #load "app/screen_capture.jai";

src/app/app_ui.jai

@@ -39,6 +39,7 @@ app_ui_shutdown :: () {
     open_file_shutdown();
     return;
   }
+  gpu_bench_ui_shutdown();   // ends a benchmark run still going
   app_shutdown_step(.CAPTURE);
   screen_capture_shutdown();
   app_shutdown_step(.IMPORT);
@@ -121,6 +122,7 @@ app_main_window_ui :: () {
   labels_ui_frame();
   segment_ui_frame();
   stats_ui_frame();
+  gpu_bench_ui_frame();   // app/gpu_bench_ui.jai
   import_frame();
   open_file_frame();   // after import_frame, which hands the grid what an import of it wrote
   screen_record_leftovers_popup();

src/app/gpu_bench_ui.jai

@@ -0,0 +1,525 @@
+// The GPU Benchmark window (the gpu-bench branch): runs gpu-bench (tools/gpu_bench) in a process of
+// its own, so the app's engines and memory do not touch its numbers, shows how far it has got, and
+// lays the results files in gpu-bench-results side by side. To compare with another machine, copy
+// its results file into that folder (Open folder).
+
+GPU_BENCH_WINDOW :: "GPU Benchmark";
+GPU_BENCH_LOG_LINES :: 2000;
+
+gpu_bench_ui: struct {
+  window_open: bool;
+  run_ocr := true;
+  run_labels := true;
+  run_segment := true;
+  cpu: bool;
+  repeats: s32 = 3;
+  note: [128] u8;
+
+  // A run: the reader thread owns the child process and appends what it prints to output.
+  running: bool;
+  mutex_ready: bool;
+  mutex: Mutex;
+  thread: Thread;
+  command: [..] string;   // heap; for the thread
+  output: [..] u8;        // under mutex; heap
+  finished: bool;         // under mutex: the child has exited, or could not start
+  exit_code: s32;         // under mutex
+  start_error: string;    // under mutex; heap
+  cancelled: bool;
+  #if OS == .WINDOWS  process: HANDLE;   // under mutex
+  else                process: Process;
+  started_at: Apollo_Time;
+  parsed: s64;            // bytes of output made into lines
+  log: [..] string;       // heap
+  scroll_log: bool;
+  status: string;         // heap: where the run is, or how it ended
+
+  reports: [..] Bench_Report;   // newest first
+  shown: [..] bool;             // per report: a column in the comparison
+  baseline: string;             // heap: the path of the report the others are compared with
+  loaded: bool;
+};
+
+gpu_bench_results_folder :: () -> string {
+  return tprint("%gpu-bench-results", path_strip_filename(get_path_of_running_executable()));
+}
+
+// The release build, else the debug one; "" when neither is beside the app.
+gpu_bench_executable :: () -> path: string, debug: bool {
+  folder := path_strip_filename(get_path_of_running_executable());
+  extension := ifx OS == .WINDOWS then ".exe" else "";
+  release := tprint("%gpu-bench%", folder, extension);
+  if file_exists(release)  return release, false;
+  debug := tprint("%gpu-bench-debug%", folder, extension);
+  if file_exists(debug)  return debug, true;
+  return "", false;
+}
+
+gpu_bench_ui_frame :: () { using ImGui;
+  if gpu_bench_ui.running  gpu_bench_poll();
+  if !gpu_bench_ui.window_open  return;
+  if !gpu_bench_ui.loaded  gpu_bench_load_reports();
+  SetNextWindowSize(.{980, 680}, .FirstUseEver);
+  if Begin(GPU_BENCH_WINDOW, *gpu_bench_ui.window_open)  gpu_bench_window_contents();
+  End();
+}
+
+gpu_bench_ui_shutdown :: () {
+  if gpu_bench_ui.running {
+    gpu_bench_cancel();
+    thread_is_done(*gpu_bench_ui.thread, -1);
+    thread_deinit(*gpu_bench_ui.thread);
+    gpu_bench_ui.running = false;
+  }
+  push_allocator(context.default_allocator);
+  for * gpu_bench_ui.reports  bench_report_free(it);
+  array_free(gpu_bench_ui.reports);
+  array_free(gpu_bench_ui.shown);
+  for gpu_bench_ui.log  free(it);
+  array_free(gpu_bench_ui.log);
+  array_free(gpu_bench_ui.output);
+  for gpu_bench_ui.command  free(it);
+  array_free(gpu_bench_ui.command);
+  free(gpu_bench_ui.status);
+  free(gpu_bench_ui.baseline);
+  free(gpu_bench_ui.start_error);
+}
+
+#scope_file
+
+gpu_bench_window_contents :: () { using ImGui;
+  TextWrapped("Times each model mcap runs on the GPU (OCR, labels, segmentation) over a fixed corpus of 30 images that every machine has, each checked against its SHA-256 before anything runs. Each model's files and settings are fingerprinted too, so a comparison shows whether it is like for like.");
+  exe, debug := gpu_bench_executable();
+
+  if gpu_bench_ui.running {
+    Text("%", gpu_bench_ui.status);
+    SameLine();
+    TextColored(app_theme.palette.dim, "(% s)", formatFloat(to_float64_seconds(current_time_monotonic() - gpu_bench_ui.started_at), trailing_width = 0));
+    SameLine();
+    if Button("Cancel")  gpu_bench_cancel();
+  } else {
+    Checkbox("OCR", *gpu_bench_ui.run_ocr);
+    SameLine(); Checkbox("Labels", *gpu_bench_ui.run_labels);
+    SameLine(); Checkbox("Segmentation", *gpu_bench_ui.run_segment);
+    SameLine(); Checkbox("CPU too", *gpu_bench_ui.cpu);
+    if IsItemHovered()  SetTooltip("Each model on the CPU as well, after the GPU, and the label model's text encoder (a like: search). Slow: minutes for segmentation.");
+    SameLine();
+    SetNextItemWidth(120);
+    SliderInt("Timed passes", *gpu_bench_ui.repeats, 1, 10);
+    if IsItemHovered()  SetTooltip("Passes over the corpus that are timed, after one untimed warm-up pass; the median is reported");
+    SetNextItemWidth(-FLOAT32_MIN);
+    InputTextWithHint("##note", "A note kept with the results, such as: plugged in, best performance mode", gpu_bench_ui.note.data, gpu_bench_ui.note.count);
+
+    reason := "";
+    if !exe                              reason = "gpu-bench is not beside the app: build with jai first.jai - release.";
+    else if ocr_ui.thread_busy           reason = "A scan or a reading is running on the GPU: let it finish, or pause or cancel it, first.";
+    else if !gpu_bench_ui.run_ocr && !gpu_bench_ui.run_labels && !gpu_bench_ui.run_segment  reason = "Choose a kind of model to run.";
+    if reason  BeginDisabled();
+    if Button("Run the benchmark")  gpu_bench_start(exe);
+    if reason  EndDisabled();
+    if reason { SameLine(); TextColored(app_theme.palette.warning, "%", reason); }
+    else if debug { SameLine(); TextColored(app_theme.palette.warning, "Only the debug gpu-bench is built: its CPU work is slow, so build with - release to compare."); }
+    else if gpu_bench_ui.status { SameLine(); TextColored(app_theme.palette.dim, "%", gpu_bench_ui.status); }
+    TextColored(app_theme.palette.dim, "mcap's own models stay loaded while it runs. For the cleanest numbers, close mcap and run gpu-bench in a terminal; its results land in the same folder.");
+  }
+
+  if CollapsingHeader(temp_c_string(tprint("Output (% lines)###output", gpu_bench_ui.log.count))) {
+    if BeginChild("##log", .{0, 220}, .Borders, .HorizontalScrollbar) {
+      for gpu_bench_ui.log  TextUnformatted(it);
+      if gpu_bench_ui.scroll_log { SetScrollHereY(1); gpu_bench_ui.scroll_log = false; }
+    }
+    EndChild();
+  }
+
+  Separator();
+  folder := gpu_bench_results_folder();
+  Text("Results (% files)", gpu_bench_ui.reports.count);
+  SameLine();
+  if SmallButton("Open folder") {
+    make_directory_if_it_does_not_exist(folder, recursive = true);
+    platform_open_file_in_default_program(folder, folder);
+  }
+  if IsItemHovered()  SetTooltip("% : copy another machine's results files here to compare", folder);
+  SameLine();
+  if SmallButton("Reload")  gpu_bench_load_reports();
+  if !gpu_bench_ui.reports.count {
+    TextColored(app_theme.palette.dim, "None yet: run the benchmark.");
+    return;
+  }
+  gpu_bench_reports_table();
+  Spacing();
+  gpu_bench_comparison_table();
+}
+
+// The results files: which are shown, and which is the baseline.
+gpu_bench_reports_table :: () { using ImGui;
+  height := min(cast(float) gpu_bench_ui.reports.count + 1, 7) * GetFrameHeightWithSpacing();
+  if !BeginTable("##reports", 7, .Borders | .RowBg | .SizingFixedFit | .ScrollY, .{0, height})  return;
+  TableSetupScrollFreeze(0, 1);
+  TableSetupColumn("Show");
+  TableSetupColumn("Baseline");
+  TableSetupColumn("Date");
+  TableSetupColumn("Machine");
+  TableSetupColumn("GPU");
+  TableSetupColumn("Power");
+  TableSetupColumn("Note", .WidthStretch);
+  TableHeadersRow();
+  for * report, n: gpu_bench_ui.reports {
+    PushID(cast(s32) n);
+    TableNextRow();
+    TableNextColumn(); Checkbox("##show", *gpu_bench_ui.shown[n]);
+    TableNextColumn();
+    if RadioButton("##baseline", report.path == gpu_bench_ui.baseline) {
+      free(gpu_bench_ui.baseline,, context.default_allocator);
+      gpu_bench_ui.baseline = copy_string(report.path,, context.default_allocator);
+      gpu_bench_ui.shown[n] = true;
+    }
+    TableNextColumn(); TextUnformatted(bench_header(report, "date"));
+    if IsItemHovered()  SetTooltip("%", report.path);
+    TableNextColumn(); TextUnformatted(bench_header(report, "machine"));
+    TableNextColumn(); TextUnformatted(bench_header(report, "gpu"));
+    TableNextColumn();
+    limit := bench_header(report, "power_limit_w");
+    power := bench_header(report, "power");
+    TextUnformatted(tprint("%%%", ifx limit then tprint("% W", limit) else "", ifx limit && power then ", " else "", power));
+    TableNextColumn(); TextUnformatted(bench_header(report, "note"));
+    PopID();
+  }
+  EndTable();
+}
+
+// The shown reports side by side: what each machine is, then each model's time, against the baseline's.
+gpu_bench_comparison_table :: () { using ImGui;
+  columns: [..] *Bench_Report;
+  columns.allocator = temp;
+  baseline: *Bench_Report;
+  for * gpu_bench_ui.reports {
+    if gpu_bench_ui.shown[it_index]  array_add(*columns, it);
+    if it.path == gpu_bench_ui.baseline  baseline = it;
+  }
+  if !columns.count { TextColored(app_theme.palette.dim, "Tick Show for the results to compare."); return; }
+  if !baseline  baseline = columns[0];
+
+  // Every task and provider any of them ran, in the order they ran.
+  Bench_Key :: struct { task, provider: string; }
+  keys: [..] Bench_Key;
+  keys.allocator = temp;
+  for report: columns  for report.rows {
+    found := false;
+    for key: keys  if key.task == it.task && key.provider == it.provider  found = true;
+    if !found  array_add(*keys, .{it.task, it.provider});
+  }
+
+  if !BeginTable("##compare", cast(s32) columns.count + 1, .Borders | .RowBg | .SizingFixedFit | .ScrollX | .ScrollY, .{0, -FLOAT32_MIN})  return;
+  TableSetupScrollFreeze(1, 1);
+  TableSetupColumn("");
+  for columns  TableSetupColumn(temp_c_string(tprint("%##%", bench_header(it, "machine"), it_index)));
+  TableHeadersRow();
+
+  header_row :: (label: string, key: string, tooltip := "") #expand {
+    TableNextRow();
+    TableNextColumn(); TextColored(app_theme.palette.dim, "%", label);
+    if tooltip && IsItemHovered()  SetTooltip("%", tooltip);
+    for columns {
+      TableNextColumn();
+      value := bench_header(it, key);
+      if it != baseline && value != bench_header(baseline, key)  TextColored(app_theme.palette.accent, "%", value);
+      else  TextUnformatted(value);
+    }
+  }
+  header_row("GPU", "gpu");
+  header_row("Memory", "vram_gb", "GB of GPU memory");
+  header_row("Power limit", "power_limit_w", "The board's enforced limit in watts: a laptop's GPU is often far below the desktop part's");
+  header_row("Power", "power");
+  header_row("Driver", "driver");
+  header_row("cuDNN", "cudnn");
+  header_row("onnxruntime", "onnxruntime");
+  header_row("CPU", "cpu");
+  header_row("Date", "date");
+  header_row("Note", "note");
+
+  // The corpus: different images make every number not comparable.
+  TableNextRow();
+  TableNextColumn(); TextColored(app_theme.palette.dim, "Corpus");
+  for columns {
+    TableNextColumn();
+    corpus := bench_header(it, "corpus");
+    if corpus == bench_header(baseline, "corpus")  TextColored(app_theme.palette.success, "same (%)", slice(corpus, 0, min(corpus.count, 12)));
+    else  TextColored(app_theme.palette.error, "DIFFERENT: not comparable");
+  }
+
+  for key: keys {
+    TableNextRow();
+    TableNextColumn();
+    TextUnformatted(tprint("% %", key.task, ifx key.provider == "CUDA" then "(GPU)" else "(CPU)"));
+    base_row := bench_row(baseline, key.task, key.provider);
+    base_ms := ifx base_row then string_to_float64(bench_field(base_row, "ms")) else 0;
+    for report: columns {
+      TableNextColumn();
+      row := bench_row(report, key.task, key.provider);
+      if !row {
+        error := bench_header(report, tprint("error % %", key.task, key.provider));
+        if !error  error = bench_header(report, tprint("error % CPU", key.task));   // a GPU run that fell back
+        if error { TextColored(app_theme.palette.error, "failed"); if IsItemHovered()  SetTooltip("%", error); }
+        else  TextColored(app_theme.palette.dim, "-");
+        continue;
+      }
+      ms := string_to_float64(bench_field(row, "ms"));
+      unit := bench_field(row, "unit");
+      same_model := bench_header(report, tprint("model %", key.task)) == bench_header(baseline, tprint("model %", key.task));
+      text := tprint("% ms/%", formatFloat(ms, trailing_width = ifx ms < 10 then 2 else 1), unit);
+      if report != baseline && base_ms > 0 && ms > 0 {
+        ratio := base_ms / ms;   // above 1: faster than the baseline
+        colour := ifx ratio > 1.05 then app_theme.palette.success else ifx ratio < 0.95 then app_theme.palette.warning else app_theme.palette.neutral;
+        TextColored(colour, "%  (%x)%", text, formatFloat(ratio, trailing_width = 2), ifx same_model then "" else " *");
+      } else {
+        TextUnformatted(tprint("%%", text, ifx same_model then "" else " *"));
+      }
+      if IsItemHovered() {
+        builder: String_Builder;
+        builder.allocator = temp;
+        if !same_model  append(*builder, "* Different model files or models.config settings from the baseline's: not like for like.\n\n");
+        print_to_builder(*builder, "% on %: % %s per second", key.task, report.path, bench_field(row, "per_s"), unit);
+        mp := bench_field(row, "mp_per_s");
+        if mp  print_to_builder(*builder, ", % megapixels per second", mp);
+        print_to_builder(*builder, "\nmedian of the timed passes; best % ms, worst % ms\n", bench_field(row, "best_ms"), bench_field(row, "worst_ms"));
+        print_to_builder(*builder, "loading % s, the first % % s, batch %\n", bench_field(row, "load_s"), unit, bench_field(row, "first_s"), bench_field(row, "batch"));
+        vram := bench_field(row, "vram_mb");
+        if vram  print_to_builder(*builder, "GPU: % MB more memory at the peak, % percent busy, % W, % MHz, up to % C\n", vram,
+                                  bench_field(row, "gpu_util"), bench_field(row, "power_w"), bench_field(row, "sm_mhz"), bench_field(row, "temp_c"));
+        stages := bench_field(row, "stages");
+        if stages  print_to_builder(*builder, "ms per %, summed over threads: %\n", unit, stages);
+        print_to_builder(*builder, "found %", bench_field(row, "output"));
+        if report != baseline && base_row {
+          same_output := bench_field(row, "digest") == bench_field(base_row, "digest");
+          print_to_builder(*builder, ", % the baseline's", ifx same_output then "the same as" else "not quite the same as (GPUs round differently)");
+        }
+        SetTooltip("%", builder_to_string(*builder,, temp));
+      }
+    }
+  }
+  EndTable();
+}
+
+gpu_bench_load_reports :: () {
+  push_allocator(context.default_allocator);
+  for * gpu_bench_ui.reports  bench_report_free(it);
+  array_reset(*gpu_bench_ui.reports);
+  array_reset(*gpu_bench_ui.shown);
+  gpu_bench_ui.loaded = true;
+  folder := gpu_bench_results_folder();
+  if !file_exists(folder)  return;
+  for file_list(folder,, temp) {
+    if !ends_with_nocase(it, ".txt")  continue;
+    text, ok := read_entire_file(it, log_errors = false,, temp);
+    if !ok  continue;
+    report, parsed, error := bench_parse(text, it);
+    if !parsed { log_error("[gpu-bench] %", error); bench_report_free(*report); continue; }
+    array_add(*gpu_bench_ui.reports, report);
+  }
+  quick_sort(gpu_bench_ui.reports, (a: Bench_Report, b: Bench_Report) -> s64 { return compare(bench_header(*b, "date"), bench_header(*a, "date")); });
+  array_resize(*gpu_bench_ui.shown, gpu_bench_ui.reports.count);
+  for * gpu_bench_ui.shown  it.* = true;
+  found := false;
+  for gpu_bench_ui.reports  if it.path == gpu_bench_ui.baseline  found = true;
+  if !found && gpu_bench_ui.reports.count {
+    free(gpu_bench_ui.baseline);
+    gpu_bench_ui.baseline = copy_string(gpu_bench_ui.reports[gpu_bench_ui.reports.count - 1].path);   // the oldest: what later runs are measured against
+  }
+}
+
+gpu_bench_start :: (exe: string) {
+  push_allocator(context.default_allocator);
+  if !gpu_bench_ui.mutex_ready { init(*gpu_bench_ui.mutex); gpu_bench_ui.mutex_ready = true; }
+  for gpu_bench_ui.command  free(it);
+  array_reset(*gpu_bench_ui.command);
+  add :: (s: string) #expand { array_add(*gpu_bench_ui.command, copy_string(s)); }
+  add(exe);
+  add("-progress");
+  add("-repeats"); add(tprint("%", gpu_bench_ui.repeats));
+  add("-out"); add(gpu_bench_results_folder());
+  only: [..] string;
+  only.allocator = temp;
+  if gpu_bench_ui.run_ocr      array_add(*only, "ocr");
+  if gpu_bench_ui.run_labels   array_add(*only, "labels");
+  if gpu_bench_ui.run_segment  array_add(*only, "segment");
+  add("-only"); add(join(..only, ",",, temp));
+  if gpu_bench_ui.cpu  add("-cpu");
+  note := trim(to_string(gpu_bench_ui.note.data));
+  if note { add("-note"); add(note); }
+
+  for gpu_bench_ui.log  free(it);
+  array_reset(*gpu_bench_ui.log);
+  array_reset(*gpu_bench_ui.output);
+  free(gpu_bench_ui.start_error);
+  gpu_bench_ui.start_error = "";
+  gpu_bench_ui.finished = false;
+  gpu_bench_ui.cancelled = false;
+  gpu_bench_ui.exit_code = 0;
+  gpu_bench_ui.parsed = 0;
+  gpu_bench_set_status("Starting...");
+  gpu_bench_ui.started_at = current_time_monotonic();
+  thread_init(*gpu_bench_ui.thread, gpu_bench_thread_proc, name = "gpu-bench");
+  thread_start(*gpu_bench_ui.thread);
+  gpu_bench_ui.running = true;
+  log("[gpu-bench] Running %", join(..gpu_bench_ui.command, " ",, temp));
+}
+
+gpu_bench_cancel :: () {
+  lock(*gpu_bench_ui.mutex);
+  gpu_bench_ui.cancelled = true;
+  #if OS == .WINDOWS { if gpu_bench_ui.process  TerminateProcess(gpu_bench_ui.process, 1); }
+  else               kill_process(*gpu_bench_ui.process);
+  unlock(*gpu_bench_ui.mutex);
+}
+
+// What the child printed since the last frame, made into lines; and the end of the run.
+gpu_bench_poll :: () {
+  push_allocator(context.default_allocator);
+  lock(*gpu_bench_ui.mutex);
+  finished := gpu_bench_ui.finished;
+  exit_code := gpu_bench_ui.exit_code;
+  new_text := copy_string(cast(string) array_view(gpu_bench_ui.output, gpu_bench_ui.parsed, gpu_bench_ui.output.count - gpu_bench_ui.parsed),, temp);
+  start_error := copy_string(gpu_bench_ui.start_error,, temp);
+  unlock(*gpu_bench_ui.mutex);
+
+  // Whole lines only, until the end.
+  end := find_index_from_right(new_text, #char "\n");
+  if finished  end = new_text.count;
+  if end > 0 {
+    gpu_bench_ui.parsed += end;
+    for raw: split(slice(new_text, 0, end), "\n",, temp) {
+      line := trim_right(raw, "\r");
+      if !line.count  continue;
+      if line[0] == 0x1b  continue;   // onnxruntime's coloured warnings
+      if begins_with(line, "@progress ") { gpu_bench_set_status(gpu_bench_progress_words(advance(line, 10))); continue; }
+      if begins_with(line, "@results ")  continue;
+      array_add(*gpu_bench_ui.log, copy_string(line));
+      gpu_bench_ui.scroll_log = true;
+    }
+    while gpu_bench_ui.log.count > GPU_BENCH_LOG_LINES {
+      free(gpu_bench_ui.log[0]);
+      array_ordered_remove_by_index(*gpu_bench_ui.log, 0);
+    }
+  }
+  if !finished  return;
+
+  thread_is_done(*gpu_bench_ui.thread, -1);
+  thread_deinit(*gpu_bench_ui.thread);
+  gpu_bench_ui.running = false;
+  seconds := formatFloat(to_float64_seconds(current_time_monotonic() - gpu_bench_ui.started_at), trailing_width = 0);
+  if start_error                    gpu_bench_set_status(start_error);
+  else if gpu_bench_ui.cancelled    gpu_bench_set_status("Cancelled.");
+  else if exit_code == 0            gpu_bench_set_status(tprint("Done in % s.", seconds));
+  else                              gpu_bench_set_status(tprint("gpu-bench failed (exit code %) after % s: Output says why.", exit_code, seconds));
+  log("[gpu-bench] %", gpu_bench_ui.status);
+  gpu_bench_load_reports();
+}
+
+// "@progress ocr/ppocr-v5-mobile CUDA timing 2 3" in words.
+gpu_bench_progress_words :: (text: string) -> string {
+  parts := split(text, " ",, temp);
+  if parts.count < 5  return text;
+  task, provider, phase, done, total := parts[0], parts[1], parts[2], parts[3], parts[4];
+  on := ifx provider == "CUDA" then "on the GPU" else ifx provider == "CPU" then "on the CPU" else "";
+  if task == "corpus"   return tprint("The corpus: % %...", phase, ifx done != "0" then tprint("image %", done) else "");
+  if task == "hashing"  return tprint("Fingerprinting %: % (% of %)", provider, phase, done, total);
+  if phase == {
+    case "loading";  return tprint("% %: loading the model...", task, on);
+    case "first";    return tprint("% %: the first image...", task, on);
+    case "warm-up";  return tprint("% %: warm-up pass % of %...", task, on, done, total);
+    case "timing";   return tprint("% %: timed pass % of %...", task, on, done, total);
+  }
+  return text;
+}
+
+gpu_bench_set_status :: (text: string) {
+  free(gpu_bench_ui.status,, context.default_allocator);
+  gpu_bench_ui.status = copy_string(text,, context.default_allocator);
+}
+
+// The reader thread: starts gpu-bench, takes what it prints until it exits.
+gpu_bench_thread_proc :: (thread: *Thread) -> s64 {
+  append_output :: (data: *u8, count: s64) {
+    lock(*gpu_bench_ui.mutex);
+    old := gpu_bench_ui.output.count;
+    array_resize(*gpu_bench_ui.output, old + count, initialize = false,, context.default_allocator);
+    memcpy(gpu_bench_ui.output.data + old, data, count);
+    unlock(*gpu_bench_ui.mutex);
+  }
+  finish :: (exit_code: s32, error := "") {
+    lock(*gpu_bench_ui.mutex);
+    gpu_bench_ui.exit_code = exit_code;
+    if error  gpu_bench_ui.start_error = copy_string(error,, context.default_allocator);
+    gpu_bench_ui.finished = true;
+    unlock(*gpu_bench_ui.mutex);
+  }
+  buffer: [4096] u8;
+
+  #if OS == .WINDOWS {
+    // Not the Process module: from a GUI program, it would open a console window for gpu-bench.
+    attributes: SECURITY_ATTRIBUTES;
+    attributes.nLength = size_of(SECURITY_ATTRIBUTES);
+    attributes.bInheritHandle = 1;
+    read_end, write_end: HANDLE;
+    if !CreatePipe(*read_end, *write_end, *attributes, 0) { finish(-1, tprint("Could not start gpu-bench: no pipe (Windows error %).", GetLastError())); return 0; }
+    SetHandleInformation(read_end, HANDLE_FLAG_INHERIT, 0);
+    startup: STARTUPINFOW;
+    startup.cb = size_of(STARTUPINFOW);
+    startup.dwFlags = STARTF_USESTDHANDLES;
+    startup.hStdOutput = write_end;
+    startup.hStdError = write_end;
+    builder: String_Builder;
+    builder.allocator = temp;
+    for gpu_bench_ui.command { if it_index  append(*builder, " "); append(*builder, windows_quote_argument(it)); }
+    command_line := builder_to_string(*builder,, temp);
+    info: PROCESS_INFORMATION;
+    working := path_strip_filename(gpu_bench_ui.command[0]);
+    started := CreateProcessW(null, utf8_to_wide(command_line,, temp), null, null, 1, CREATE_NO_WINDOW, null, utf8_to_wide(working,, temp), *startup, *info);
+    CloseHandle(write_end);   // the child's copy is the only one left: reading ends when it exits
+    if !started {
+      error := GetLastError();
+      CloseHandle(read_end);
+      finish(-1, tprint("Could not start % (Windows error %).", gpu_bench_ui.command[0], error));
+      return 0;
+    }
+    CloseHandle(info.hThread);
+    lock(*gpu_bench_ui.mutex);
+    gpu_bench_ui.process = info.hProcess;
+    cancelled := gpu_bench_ui.cancelled;
+    unlock(*gpu_bench_ui.mutex);
+    if cancelled  TerminateProcess(info.hProcess, 1);
+    while true {
+      read: u32;
+      if !ReadFile(read_end, buffer.data, buffer.count, *read) || !read  break;
+      append_output(buffer.data, read);
+    }
+    CloseHandle(read_end);
+    WaitForSingleObject(info.hProcess, INFINITE);
+    exit_code: u32;
+    GetExitCodeProcess(info.hProcess, *exit_code);
+    lock(*gpu_bench_ui.mutex);
+    gpu_bench_ui.process = null;
+    unlock(*gpu_bench_ui.mutex);
+    CloseHandle(info.hProcess);
+    finish(cast,no_check(s32) exit_code);
+  } else {
+    errors: [4096] u8;
+    if !create_process(*gpu_bench_ui.process, ..gpu_bench_ui.command, capture_and_return_output = true) {
+      finish(-1, tprint("Could not start %.", gpu_bench_ui.command[0]));
+      return 0;
+    }
+    while true {
+      ok, output_bytes, error_bytes := read_from_process(*gpu_bench_ui.process, buffer, errors, -1);
+      if output_bytes  append_output(buffer.data, output_bytes);
+      if error_bytes   append_output(errors.data, error_bytes);
+      if !ok || (gpu_bench_ui.process.output.eof && gpu_bench_ui.process.error.eof)  break;
+    }
+    _, result := get_process_result(*gpu_bench_ui.process, -1);
+    deinit(*gpu_bench_ui.process);
+    finish(cast(s32) result.exit_code);
+  }
+  return 0;
+}
+
+#import "Process";

src/app/viewer.jai

@@ -1115,6 +1115,9 @@ viewer_ui :: () { using ImGui;
   if Button("Segments")  segment_ui.window_open = !segment_ui.window_open;   // app/segment_ui.jai
   if IsItemHovered()  SetTooltip("Segmentation: what is where in the images of the library folders you choose");
   SameLine();
+  if Button("GPU Bench")  gpu_bench_ui.window_open = !gpu_bench_ui.window_open;   // app/gpu_bench_ui.jai
+  if IsItemHovered()  SetTooltip("Time every GPU model over a fixed corpus, and compare machines");
+  SameLine();
   if Button("Import")  importer.window_open = !importer.window_open;   // app/import.jai
   if IsItemHovered()  SetTooltip("Bring in a folder of screenshots and photos");
   SameLine();

tools/gpu_bench/README.md

@@ -0,0 +1,50 @@
+# gpu-bench (the `gpu-bench` branch, not for master)
+
+Times every model mcap runs on the GPU over a fixed corpus, so machines can be compared like for
+like:
+
+| Task | What runs |
+| --- | --- |
+| `ocr/ppocr-v5-mobile`, `ocr/ppocr-v5-server` | detection, orientation and recognition, 8 images per call, as a scan does |
+| `labels/siglip2-base-patch16-384` | the label model's image encoder |
+| `segment/mask2former-swin-base-coco-panoptic` | panoptic segmentation |
+| `labels-text/...` (with `-cpu`) | the text encoder, one phrase at a time as a `like:` search runs it (always on the CPU) |
+
+## Keeping it consistent
+
+- **The corpus** (`corpus/`, 30 images: 16 COCO val2017 photos, 2 of them scaled to 12 MP, and 12
+  synthetic screenshots from 720p to 4K made by `make_corpus.py`) is in git. The build copies it to
+  `bin/gpu-bench-corpus`. Before anything runs, every file is checked against its SHA-256 in
+  `corpus.txt`, and a run stops if one differs. The corpus fingerprint, a hash of those name and hash
+  pairs, goes in the results. `.gitattributes` marks the images binary, so git never changes them.
+- **The models:** each task's fingerprint covers the hash of each of its files and its
+  `models.config` entry (comments and spacing left out). It also says whether every file is the one
+  the fetch scripts pin. The segmentation `.onnx` is exported on each machine, so it has no pin:
+  copy it from one machine to the others.
+- **The machine:** each result records the GPU, its memory, enforced power limit and clocks, the
+  driver, the CUDA runtime and cuDNN actually loaded (with their paths), onnxruntime, the CPU, RAM,
+  and AC or battery.
+- **The run:** each model gets one untimed run on the first image, one untimed warm-up pass over
+  everything, then N timed passes (default 3). The median is reported, with the best and worst
+  passes. While the timed passes run, NVML samples the GPU every 100 ms: peak memory over the
+  device's use before the model loaded, utilisation, power, SM clock and temperature.
+- **The output:** what the models found (OCR lines and characters, segments, best phrases) and a
+  digest of it. Different GPUs can round differently, so a different digest is a hint, not an error.
+
+## Running
+
+`jai first.jai - release` builds `bin/gpu-bench.exe` with mcap. Then either:
+
+- **In mcap:** click **GPU Bench** above the grid, choose what to run, and click **Run the benchmark**.
+  It runs gpu-bench in a separate process with no console and shows its progress. Close mcap and use
+  the terminal for the cleanest numbers, as mcap's own models stay loaded while it runs.
+- **In a terminal:** `bin\gpu-bench.exe [-only ocr,labels,segment] [-models a,b] [-repeats N] [-warmup N] [-cpu | -cpu-only] [-note TEXT]`
+
+Results go to `bin/gpu-bench-results/<machine>_<date>.txt` (plain text, `results.jai`). To compare
+machines, copy their files into that folder: the window puts the results side by side. Each result
+shows its speed against the baseline (the oldest file, or the one you pick). Its corpus row turns red
+when the images differ, and a `*` marks a model whose files or settings differ. Hover a time for its
+details.
+
+The server OCR model can take a lot of GPU and system memory and a long time (see the main README's
+known issue). Leave it out with `-models ppocr-v5-mobile,siglip2-base-patch16-384,mask2former-swin-base-coco-panoptic`.

tools/gpu_bench/corpus.txt

@@ -0,0 +1,35 @@
+# The GPU benchmark's corpus: every image in corpus/, with its SHA-256. gpu-bench checks each file
+# against this before it runs, and its corpus fingerprint is the SHA-256 of these name and hash
+# pairs, so two results are comparable only when the fingerprints match. coco-*: COCO val2017
+# (images.cocodataset.org/val2017, CC licences); screen-* and photo-*: make_corpus.py.
+
+ffe0f0cec3b2e27aab1967229cdf0a0d7751dcdd5800322f0b8ac0dffb3b8a8d  coco-000000000139.jpg
+f3a2974ce3686332609124c70e3e6a2e3aca43fccf1cd1bd7c5c03820977f57d  coco-000000000285.jpg
+a4cd7f45ac1ce27eaafb254b23af7c0b18a064be08870ceaaf03b2147f2ce550  coco-000000000632.jpg
+5c0e559c75d3969c8e3e297b61f61063f78045c9d4802b526ba616361f3823fd  coco-000000000724.jpg
+1dd31e9059c491992be2f562624eb4093e17aee08b4f7baf5ff9ea24543b0a33  coco-000000000776.jpg
+83981537a7baeafbeb9c8cb67b3484dc26433f574b3685d021fa537e277e4726  coco-000000000785.jpg
+d5b79e7fa716f85ca86f46e5f518da9b6c5e26414925624d76e6476861aec495  coco-000000000802.jpg
+c2aa138ee3a59b057a7ba6fc5a6a18e62af531aa7dab78a7bfd33c1cd7e55eb6  coco-000000000872.jpg
+1c67b783f78b18ddb5dd20aaba4d4aacaf98e5277051688a96da096afa9cdf1f  coco-000000000885.jpg
+24bb77a31928404e45a0454b06f6a0bd54a8db103590c9d7917288a1e0269f05  coco-000000001000.jpg
+8e85a2e71ee512fe748a3a3be9bb40b8591e8c501c0f3896efcb5be5999ec236  coco-000000001268.jpg
+34a98da7c11bc8811e6eec445145bb16f3d5f4338945ba116edcc1fe554d181b  coco-000000001296.jpg
+1cccef217a1e5eec2acc74d7c08a105da3ef6fdb8514e221cdea9b0b373b60ea  coco-000000001353.jpg
+a044804a0b0186a1267acae34975932f14ede96c4a2b78fcee74e6c2fbd8414d  coco-000000001490.jpg
+974f2af6d81f65627a680ea9f42bdee1067a0ddcdb9f8e7a945bacd21d4dc2ce  coco-000000001532.jpg
+77131e2f74aa7e374f951a69157cf7f51e00201f81bacfa8a9ab7309b245553f  coco-000000001584.jpg
+3c34557e033b8022567bf41d1b9f6da6aa3547305b77a922f1f351c197040111  photo-12mp-livingroom.jpg
+769a74c85bb0b03322864cffc252738a7858c6947b4a45bf643708a766ea0f73  photo-12mp-tennis.jpg
+c989015b2ecf47630acfa1c5afe828e406accc5f0345814e4d5a5cf53cdb4bc7  screen-chat-phone.png
+41e445c52f991a659b65e435a5e08173817a53d150da3f7b5380b0a697e50ecc  screen-dialog-1080p.png
+76224a64b1460169bb0684c3a8eefc977b1cbbd556a170fe7559928b246012ad  screen-document-1080p.png
+96f00112b373d990162a13e85d524f5acf6a3b4bb671883742127879f3b39246  screen-document-4k.png
+2112042a1f22349805af2136a2b9aee346d47387d3d31ca698d0d570e3215c00  screen-document-strip.png
+973ba96240163e51c9e92e0bf7d927ca07dab78e2236b1ecf4e902ef4e170187  screen-editor-dark-1080p.png
+51f62f081a73d05bbb2dac313ca5e716628961dcf2760c073c6f365090959258  screen-editor-dark-1440p.png
+9d763d163ada6d85e3e4d0415295ccffac1d9b5eaa60b6a6209c7af6c6b06d43  screen-editor-light-1080p.png
+35bdf92f31eee19599f2f20a17c8cedb536289e3605b9f91f32c1e7dc0a5169a  screen-spreadsheet-1080p.png
+84b5074a25e1ea13ca17be3476dff40609fe11c0f70ecdcbf6920c59c99272b1  screen-spreadsheet-1440p.png
+54f30b7745bdba5890c25c9ae90e417263fa6126ffe41925f98e981920986448  screen-terminal-1080p.png
+0f64a278adb72e482f9526af9037ce4fa8ae6826a1fa9b94c8d0d4102bb6e22e  screen-terminal-720p.png

tools/gpu_bench/gpu_bench.jai

@@ -0,0 +1,710 @@
+// gpu-bench: times every model mcap runs on the GPU (each OCR model, the label model's image encoder,
+// segmentation) over a fixed corpus, so machines can be compared. The corpus (gpu-bench-corpus beside
+// this program, from tools/gpu_bench/corpus) is checked file by file against corpus.txt's SHA-256s
+// before anything runs, and each model's files and models.config entry are fingerprinted, so a
+// comparison of two results is known to be like for like. mcap's GPU Benchmark window runs this and
+// lays the results side by side.
+//
+//   gpu-bench [-only ocr,labels,segment] [-models a,b] [-repeats N] [-warmup N] [-cpu | -cpu-only]
+//             [-corpus FOLDER] [-out FOLDER] [-note TEXT] [-progress]
+//
+//   -only      the kinds of model to run (default all)
+//   -models    the models to run, by their models.config names (default every one)
+//   -repeats   timed passes over the corpus; the median is reported (default 3)
+//   -warmup    untimed passes first, so every image size has been seen (default 1)
+//   -cpu       the CPU too, after the GPU (slow: minutes for segmentation); also times the label
+//              model's text encoder, which always runs on the CPU, as a like: search uses it
+//   -cpu-only  the CPU only
+//   -corpus    the corpus folder (default gpu-bench-corpus beside this program)
+//   -out       where the results file goes (default gpu-bench-results beside this program)
+//   -note      a few words kept with the results, such as "plugged in, max performance"
+//   -progress  also print @progress lines, for the GPU Benchmark window
+//   -ocr-workers N  OCR's worker threads on the GPU, in place of Ocr_Options' (for investigating memory)
+//   -ort-log LEVEL  onnxruntime's console log: verbose, info (each GPU arena extension), warning (default)
+//
+// Build: jai first.jai - release (it is built with mcap on the gpu-bench branch).
+
+USAGE :: "Usage: gpu-bench [-only ocr,labels,segment] [-models a,b] [-repeats N] [-warmup N] [-cpu | -cpu-only] [-corpus FOLDER] [-out FOLDER] [-note TEXT] [-progress] [-ocr-workers N] [-ort-log LEVEL]\n";
+
+options: struct {
+  only: [..] string;
+  models: [..] string;
+  repeats := 3;
+  warmup := 1;
+  gpu := true;
+  cpu := false;
+  progress := false;
+  ocr_workers: s32;   // 0: Ocr_Options' default
+  corpus_folder, out_folder, note: string;
+};
+
+main :: () {
+  runtime_support_headless = true;   // a command-line tool: a failure is reported on the console and exits with 1, no message box
+  args := get_command_line_arguments();
+  exe_folder := path_strip_filename(get_path_of_running_executable());
+  options.corpus_folder = sprint("%gpu-bench-corpus", exe_folder);
+  options.out_folder    = sprint("%gpu-bench-results", exe_folder);
+  i := 1;
+  while i < args.count {
+    arg := args[i];
+    value := ifx i + 1 < args.count then args[i + 1] else "";
+    if arg == {
+      case "-only";     for split(value, ",")  array_add(*options.only, trim(it)); i += 2;
+      case "-models";   for split(value, ",")  array_add(*options.models, trim(it)); i += 2;
+      case "-repeats";  options.repeats = max(1, string_to_int(value)); i += 2;
+      case "-warmup";   options.warmup = max(0, string_to_int(value)); i += 2;
+      case "-cpu";      options.cpu = true; i += 1;
+      case "-cpu-only"; options.cpu = true; options.gpu = false; i += 1;
+      case "-corpus";   options.corpus_folder = value; i += 2;
+      case "-out";      options.out_folder = value; i += 2;
+      case "-note";     options.note = value; i += 2;
+      case "-progress"; options.progress = true; i += 1;
+      case "-ocr-workers"; options.ocr_workers = cast(s32) max(1, string_to_int(value)); i += 2;
+      case "-ort-log";
+        if value == {
+          case "verbose"; ort.log_level = .ORT_LOGGING_LEVEL_VERBOSE;
+          case "info";    ort.log_level = .ORT_LOGGING_LEVEL_INFO;
+          case "warning"; ort.log_level = .ORT_LOGGING_LEVEL_WARNING;
+          case;           print("-ort-log takes verbose, info or warning.\n"); exit(2);
+        }
+        i += 2;
+      case "-help";     print(USAGE); exit(0);
+      case;             print("Unknown option %.\n%", arg, USAGE); exit(2);
+    }
+  }
+  make_directory_if_it_does_not_exist(options.out_folder, recursive = true);
+  started_at := current_time_consensus();
+
+  // The machine.
+  info := system_info();
+  ort_ok, ort_error := ort_load();
+  print("gpu-bench, mcap % (%, %)\n", VERSION, GIT_HASH, GIT_BRANCH);
+  print("machine:     %, %\n", info.machine, info.os);
+  print("cpu:         % (% threads), % GB of memory\n", info.cpu, info.logical_cores, FF(info.ram_gb, 1));
+  if info.gpu  print("gpu:         %, % GB, driver % (CUDA %), power limit % W, max clocks % / % MHz\n", info.gpu, FF(info.vram_gb, 1), info.driver, info.cuda_driver, FF(info.power_limit_w, 0), info.max_sm_clock_mhz, info.max_mem_clock_mhz);
+  else         print("gpu:         none (%)\n", info.gpu_error);
+  if info.power_source  print("power:       %\n", info.power_source);
+  if !ort_ok { print("%\n", ort_error); exit(1); }
+  print("onnxruntime: % (%)\n", ort.version, ort.library_path);
+
+  // The corpus: every file as corpus.txt pins it, or nothing runs.
+  progress("corpus", "-", "checking", 0, 0);
+  corpus, corpus_error := corpus_load(options.corpus_folder);
+  if corpus_error { print("\nThe corpus cannot be used: %\n", corpus_error); exit(1); }
+  print("corpus:      % images, % megapixels, fingerprint % (decoded in % s)\n\n", corpus.images.count, FF(corpus.megapixels, 1), short(corpus.fingerprint), FF(corpus.decode_s, 2));
+
+  // The models, and what each is made of.
+  tasks := list_tasks(exe_folder);
+  if !tasks.count { print("No models to run: none matched -only and -models, or none are beside this program (run jai first.jai).\n"); exit(1); }
+  hash_cache_load();
+  for * tasks  fingerprint_task(it);
+  hash_cache_save();
+
+  runs: [..] Run;
+  for * task: tasks {
+    providers: [..] string;
+    providers.allocator = temp;
+    if task.kind == .LABELS_TEXT { if options.cpu  array_add(*providers, "CPU"); }
+    else {
+      if options.gpu  array_add(*providers, "CUDA");
+      if options.cpu  array_add(*providers, "CPU");
+    }
+    for provider: providers {
+      run := run_task(task, provider == "CUDA", *corpus);
+      array_add(*runs, run);
+      if !run.ok { print("== % on %: %\n\n", task.name, provider, run.error); continue; }
+      print_run(task, run);
+    }
+  }
+
+  cudart, cudnn := cuda_library_versions();
+  path := write_results(info, started_at, *corpus, tasks, runs, cudart, cudnn);
+  print("Results: %\n", path);
+  if options.progress  print("@results %\n", path);
+}
+
+Task_Kind :: enum { OCR; LABELS; LABELS_TEXT; SEGMENT; }
+
+Task :: struct {
+  kind: Task_Kind;
+  model: string;           // as models.config names it
+  name: string;            // kind/model: ocr/ppocr-v5-mobile
+  models_folder: string;
+  files: [..] string;      // the model's files, in models_folder
+  config: string;          // its models.config entry, as compared
+  fingerprint: string;     // of files and config
+  pinned: string;          // "yes" when every file is one a fetch script pins, "no" when one differs, "-" when none are pinned
+}
+
+Run :: struct {
+  task: *Task;
+  ok: bool;
+  error: string;
+  provider: string;        // what it really ran on: a GPU run can fall back to the CPU
+  items: s64;              // per pass
+  unit: string;            // image or phrase
+  megapixels: float64;     // per pass
+  load_s, first_s: float64;
+  pass_s: [..] float64;
+  output: string;          // what the models found, in words without spaces: lines:123,chars:4567
+  digest: string;          // of what they found, to see whether two machines' models agree
+  stages: string;          // where the time went, per item
+  batch: s64;
+  gpu: Gpu_Stats;
+  trace: [..] string;      // the GPU's memory above the baseline, MB, after each step: load:812 first:1020 warm1:... pass1:.../peak...
+  explanation: string;     // why a GPU run is on the CPU
+}
+
+Gpu_Stats :: struct {
+  sampled: bool;
+  baseline_mb, peak_mb: float64;   // device memory before the engine, and the most during the timed passes
+  pass_peak_mb: float64;           // the most during the current timed pass
+  util_sum, power_sum, clock_sum: float64;
+  temperature_max: float64;
+  samples: s64;
+}
+
+// The corpus, decoded.
+Corpus :: struct {
+  fingerprint: string;
+  images: [..] Corpus_Image;
+  megapixels: float64;
+  decode_s: float64;
+}
+
+Corpus_Image :: struct { name: string; pixels: *u8; width, height: s32; }
+
+// Every file corpus.txt names, checked against its hash; the error says what is wrong.
+corpus_load :: (folder: string) -> Corpus, error: string {
+  corpus: Corpus;
+  manifest_path := tprint("%/corpus.txt", folder);
+  manifest, ok := read_entire_file(manifest_path, log_errors = false);
+  if !ok  return corpus, tprint("% cannot be read. The build copies it, with the images, from tools/gpu_bench.", manifest_path);
+  fingerprint: Sha256;
+  bad: [..] string;
+  bad.allocator = temp;
+  start := current_time_monotonic();
+  for raw: split(manifest, "\n") {
+    line := trim(raw);
+    if !line.count || line[0] == #char "#"  continue;
+    space := find_index_from_left(line, #char " ");
+    if space < 0  continue;
+    expected := slice(line, 0, space);
+    name := trim(advance(line, space));
+    path := tprint("%/%", folder, name);
+    data, read_ok := read_entire_file(path, log_errors = false);
+    if !read_ok { array_add(*bad, tprint("% is missing", name)); continue; }
+    hash := sha256_of_string(data,, temp);
+    if hash != expected { array_add(*bad, tprint("% is not the file corpus.txt pins (SHA-256 %)", name, hash)); free(data); continue; }
+    sha256_update(*fingerprint, tprint("% %\n", name, hash));
+    progress("corpus", "-", "decoding", corpus.images.count + 1, 0);
+    image, decoded, decode_error := image_decode(cast([] u8) data);
+    free(data);
+    if !decoded { array_add(*bad, tprint("% cannot be decoded (%)", name, decode_error)); continue; }
+    array_add(*corpus.images, .{copy_string(name), image.pixels, image.width, image.height});
+    corpus.megapixels += cast(float64) image.width * image.height / 1_000_000;
+    reset_temporary_storage();
+  }
+  corpus.decode_s = seconds_since(start);
+  if bad.count  return corpus, join(..bad, "; ");
+  if !corpus.images.count  return corpus, tprint("% names no images.", manifest_path);
+  corpus.fingerprint = sha256_finish(*fingerprint);
+  return corpus, "";
+}
+
+// The models beside this program, as -only and -models choose.
+list_tasks :: (exe_folder: string) -> [..] Task {
+  tasks: [..] Task;
+  add :: (tasks: *[..] Task, kind: Task_Kind, model: string, folder: string, files: .. string) {
+    prefix := ifx kind == .OCR then "ocr" else ifx kind == .LABELS then "labels" else ifx kind == .LABELS_TEXT then "labels-text" else "segment";
+    if options.only.count && !array_find(options.only, ifx kind == .LABELS_TEXT then "labels" else prefix)  return;
+    if options.models.count && !array_find(options.models, model)  return;
+    task := array_add(tasks);
+    task.* = .{kind = kind, model = model, name = sprint("%/%", prefix, model), models_folder = folder};
+    for files  if it  array_add(*task.files, it);
+    task.config = config_entry(tprint("%/models.config", folder), model);
+  }
+  ocr_folder := sprint("%ocr-models", exe_folder);
+  ocr_models, ocr_error := ocr_list_models(ocr_folder);
+  if ocr_error  print("OCR: %\n", ocr_error);
+  for ocr_models  add(*tasks, .OCR, it.name, ocr_folder, it.detector, it.recognizer, it.dictionary, it.textline_ori);
+  labels_folder := sprint("%label-models", exe_folder);
+  labels_models, labels_error := labels_list_models(labels_folder);
+  if labels_error  print("Labels: %\n", labels_error);
+  for labels_models  add(*tasks, .LABELS, it.name, labels_folder, it.vision, it.text, it.tokenizer);
+  for labels_models  add(*tasks, .LABELS_TEXT, it.name, labels_folder, it.vision, it.text, it.tokenizer);
+  segment_folder := sprint("%segment-models", exe_folder);
+  segment_models, segment_error := segment_list_models(segment_folder);
+  if segment_error  print("Segmentation: %\n", segment_error);
+  for segment_models  add(*tasks, .SEGMENT, it.name, segment_folder, it.model, it.classes);
+  return tasks;
+}
+
+// A model's [model NAME] entry, comments and spacing taken out, so only a change that matters
+// changes the fingerprint.
+config_entry :: (path: string, model: string) -> string {
+  text, ok := read_entire_file(path, log_errors = false,, temp);
+  if !ok  return "";
+  builder: String_Builder;
+  inside := false;
+  for raw: split(text, "\n",, temp) {
+    line := raw;
+    hash := find_index_from_left(line, #char "#");
+    if hash >= 0  line = slice(line, 0, hash);
+    line = trim(line);
+    if !line.count  continue;
+    if begins_with(line, "[model ") { inside = line == tprint("[model %]", model); continue; }
+    if !inside  continue;
+    colon := find_index_from_left(line, #char ":");
+    if colon >= 0  line = tprint("%: %", trim(slice(line, 0, colon)), trim(advance(line, colon + 1)));
+    print_to_builder(*builder, "%\n", line);
+  }
+  return builder_to_string(*builder);
+}
+
+// The files the fetch scripts pin (modules/OCR/fetch_models.jai, modules/Labels/fetch_models.jai).
+// Segmentation's model is made on each machine by tools/export_mask2former.py, so it has no pin:
+// copy the .onnx from one machine to the others to be sure they run the same one.
+Pin :: struct { name: string; sha256: string; }
+PINNED :: Pin.[
+  .{"ch_PP-OCRv5_det_mobile.onnx", "4d97c44a20d30a81aad087d6a396b08f786c4635742afc391f6621f5c6ae78ae"},
+  .{"ch_PP-OCRv5_rec_mobile.onnx", "5825fc7ebf84ae7a412be049820b4d86d77620f204a041697b0494669b1742c5"},
+  .{"ppocrv5_dict.txt",            "d1979e9f794c464c0d2e0b70a7fe14dd978e9dc644c0e71f14158cdf8342af1b"},
+  .{"ch_PP-LCNet_x0_25_textline_ori_cls_mobile.onnx", "54379ae5174d026780215fc748a7f31910dee36818e63d49e17dc598ecc82df7"},
+  .{"ch_PP-OCRv5_det_server.onnx", "0f8846b1d4bba223a2a2f9d9b44022fbc22cc019051a602b41a7fda9667e4cad"},
+  .{"ch_PP-OCRv5_rec_server.onnx", "e09385400eaaaef34ceff54aeb7c4f0f1fe014c27fa8b9905d4709b65746562a"},
+  .{"siglip2-base-patch16-384-vision.onnx",    "49ae4958b1098ca995e929d646f7be05a69c65e6344beae07d58c6598ffc5210"},
+  .{"siglip2-base-patch16-384-text-int8.onnx", "30a94b43b9ddc39c04ab67f80cba4b808262682b2fa1a2a8b924bf1e7bd84c61"},
+  .{"siglip2-tokenizer.json",                  "cb9140fae3ac5122c972d37adf83e1248471a38147ad76f8215c8872c6fd8322"},
+];
+
+fingerprint_task :: (task: *Task) {
+  h: Sha256;
+  pinned, differs := 0, 0;
+  for task.files {
+    progress("hashing", task.name, it, it_index + 1, task.files.count);
+    hash := hash_file_cached(tprint("%/%", task.models_folder, it));
+    if !hash  hash = "missing";
+    sha256_update(*h, tprint("% %\n", it, hash));
+    for pin: PINNED  if pin.name == it { if pin.sha256 == hash  pinned += 1; else differs += 1; }
+  }
+  sha256_update(*h, task.config);
+  task.fingerprint = sha256_finish(*h);
+  task.pinned = ifx differs then "no" else ifx pinned then "yes" else "-";
+}
+
+run_task :: (task: *Task, prefer_gpu: bool, corpus: *Corpus) -> Run {
+  run: Run;
+  run.task = task;
+  run.provider = ifx prefer_gpu then "CUDA" else "CPU";
+  run.unit = ifx task.kind == .LABELS_TEXT then "phrase" else "image";
+  images := corpus.images;
+  run.items = ifx task.kind == .LABELS_TEXT then PHRASES.count else images.count;
+  if task.kind != .LABELS_TEXT  run.megapixels = corpus.megapixels;
+
+  sampler_start(*run.gpu);   // the baseline: the device's memory before the engine
+  sampler_pause();
+
+  // The engine.
+  progress(task.name, run.provider, "loading", 0, 0);
+  load_start := current_time_monotonic();
+  engine: struct { ocr: *Ocr_Engine; labels: *Labels_Engine; segment: *Segment_Engine; };
+  ok: bool;
+  error: string;
+  phrase_vectors: [] float;
+  if task.kind == {
+    case .OCR;
+      ocr_options: Ocr_Options;
+      if options.ocr_workers  ocr_options.gpu_workers = options.ocr_workers;
+      engine.ocr, ok, error = ocr_create(task.models_folder, task.model, prefer_gpu, ocr_options);
+      if ok {
+        ocr_gpu, ocr_why := ocr_gpu_status(engine.ocr);
+        if prefer_gpu && !ocr_gpu  run.explanation = copy_string(ocr_why);
+        run.provider = sprint("%", ocr_provider(engine.ocr));
+        run.batch = engine.ocr.rec_batch;
+      }
+    case .LABELS; #through;
+    case .LABELS_TEXT;
+      engine.labels, ok, error = labels_create(task.models_folder, task.model, prefer_gpu && task.kind == .LABELS);
+      if ok {
+        labels_gpu, labels_why := labels_gpu_status(engine.labels);
+        if prefer_gpu && !labels_gpu  run.explanation = copy_string(labels_why);
+        run.provider = ifx task.kind == .LABELS_TEXT then "CPU" else sprint("%", engine.labels.provider);
+        run.batch = ifx task.kind == .LABELS_TEXT then 1 else engine.labels.model.image_batch;
+        phrases := PHRASES;
+        phrase_vectors = labels_embed_texts(engine.labels, phrases);
+      }
+    case .SEGMENT;
+      engine.segment, ok, error = segment_create(task.models_folder, task.model, prefer_gpu);
+      if ok {
+        if prefer_gpu && engine.segment.provider != .CUDA  run.explanation = copy_string(engine.segment.gpu_explanation);
+        run.provider = sprint("%", engine.segment.provider);
+        run.batch = ifx engine.segment.provider == .CUDA then engine.segment.model.batch_gpu else engine.segment.model.batch_cpu;
+      }
+  }
+  run.load_s = seconds_since(load_start);
+  trace(*run, "load");
+  if !ok { sampler_stop(); run.error = copy_string(error); return run; }
+  defer {
+    ocr_destroy(engine.ocr);
+    labels_destroy(engine.labels);
+    segment_destroy(engine.segment);
+    array_free(phrase_vectors);
+  }
+  if prefer_gpu && run.provider != "CUDA" {
+    // Said, and not timed: the CPU run, when asked for, is the same thing.
+    sampler_stop();
+    run.error = tprint("not on the GPU: %", run.explanation,, context.allocator);
+    return run;
+  }
+
+  // One pass over the corpus, or over its first image alone; what it found, and its digest.
+  pass :: (first_only: bool) -> output: string, digest: string #expand {
+    h: Sha256;
+    builder: String_Builder;
+    builder.allocator = temp;
+    count := ifx first_only then 1 else `run.items;
+    if `task.kind == {
+      case .OCR;
+        lines, characters: s64;
+        done := 0;
+        while done < count {
+          n := min(OCR_BATCH, count - done);
+          batch := NewArray(n, Ocr_Image,, temp);
+          for 0..n - 1 { image := `images[done + it]; batch[it] = .{image.pixels, image.width, image.height, image.width * 4}; }
+          results := ocr_process(`engine.ocr, batch,, temp);
+          for results {
+            if it.failed { sha256_update(*h, "failed\n"); continue; }
+            lines += it.lines.count;
+            for line: it.lines { characters += line.text.count; sha256_update(*h, line.text); sha256_update(*h, "\n"); }
+          }
+          done += n;
+        }
+        print_to_builder(*builder, "lines:%,chars:%", lines, characters);
+      case .LABELS;
+        input := NewArray(count, Labels_Image,, temp);
+        for 0..count - 1 { image := `images[it]; input[it] = .{image.pixels, image.width, image.height, image.width * 4}; }
+        vectors, error := labels_embed_images(`engine.labels, input,, temp);
+        if error  sha256_update(*h, "failed\n");
+        // Each image's best phrase: what the label list would make of it.
+        dims := `engine.labels.dimensions;
+        best_count: [PHRASES.count] s64;
+        for image: 0..count - 1 {
+          best, best_score := -1, -1.0;
+          for phrase: 0..PHRASES.count - 1 {
+            score := labels_score(`engine.labels, vectors.data + image * dims, `phrase_vectors.data + phrase * dims);
+            if score > best_score { best, best_score = phrase, score; }
+          }
+          sha256_update(*h, tprint("%\n", best));
+          best_count[best] += 1;
+        }
+        distinct := 0;
+        for best_count  if it  distinct += 1;
+        print_to_builder(*builder, "best_phrases:%", distinct);
+      case .LABELS_TEXT;
+        for 0..count - 1 {
+          one: [1] string;
+          one[0] = PHRASES[it];
+          vectors, error := labels_embed_texts(`engine.labels, one,, temp);   // one at a time, as a search does
+          sha256_update(*h, ifx error then "failed\n" else "ok\n");
+        }
+        print_to_builder(*builder, "phrases:%", count);
+      case .SEGMENT;
+        input := NewArray(count, Segment_Image,, temp);
+        for 0..count - 1 { image := `images[it]; input[it] = .{image.pixels, image.width, image.height, image.width * 4}; }
+        results := segment_images(`engine.segment, input,, temp);
+        segments: s64;
+        for results {
+          if it.failed { sha256_update(*h, "failed\n"); continue; }
+          segments += it.segments.count;
+          for segment: it.segments  sha256_update(*h, tprint("% ", segment.class));
+          sha256_update(*h, "\n");
+        }
+        print_to_builder(*builder, "segments:%", segments);
+    }
+    return builder_to_string(*builder,, temp), sha256_finish(*h,, temp);
+  }
+
+  // The first image alone: the first run pays for choosing kernels and setting up memory.
+  progress(task.name, run.provider, "first", 0, 0);
+  first_start := current_time_monotonic();
+  pass(true);
+  run.first_s = seconds_since(first_start);
+  trace(*run, "first");
+  reset_temporary_storage();
+  for 1..options.warmup {
+    progress(task.name, run.provider, "warm-up", it, options.warmup);
+    pass(false);
+    trace(*run, tprint("warm%", it));
+    reset_temporary_storage();
+  }
+  if task.kind == .OCR      ocr_reset_stats(engine.ocr);
+  if task.kind == .SEGMENT  engine.segment.stats = .{};
+
+  sampler_resume();
+  for 1..options.repeats {
+    progress(task.name, run.provider, "timing", it, options.repeats);
+    start := current_time_monotonic();
+    sampler_pass_start();
+    output, digest := pass(false);
+    array_add(*run.pass_s, seconds_since(start));
+    trace(*run, tprint("pass%", it), sampler_pass_peak());
+    if it == 1 { run.output = copy_string(output); run.digest = copy_string(short(digest)); }
+    reset_temporary_storage();
+  }
+  sampler_stop();
+
+  per := cast(float64) max(run.items * options.repeats, 1) * 1000;
+  if task.kind == .OCR {
+    s := ocr_stats(engine.ocr);
+    run.stages = sprint("detect:%,orient:%,recognize:%", FF(s.detect_us / per, 2), FF(s.orientation_us / per, 2), FF(s.recognize_us / per, 2));
+  } else if task.kind == .SEGMENT {
+    s := engine.segment.stats;
+    run.stages = sprint("prepare:%,model:%,post:%", FF(s.prepare_us / per, 2), FF(s.model_us / per, 2), FF(s.post_us / per, 2));
+  }
+  run.ok = true;
+  return run;
+}
+
+OCR_BATCH :: 8;   // images per ocr_process call, as the app's scan gives them
+
+// Phrases for the label model: its image vectors are scored against them (the output's digest), and
+// the text encoder is timed on them, one at a time, as a like: search runs it.
+PHRASES :: string.[
+  "a screenshot of a code editor showing source code", "a screenshot of a terminal window",
+  "a spreadsheet with rows and columns of numbers", "a document with paragraphs of text",
+  "a chat conversation on a phone", "a dialog box asking to save changes",
+  "a photo of a living room", "a photo of people playing tennis",
+  "a photo of food on a plate", "a photo of a street with cars",
+  "a photo of an animal", "a photo of a person", "an electronic circuit schematic diagram",
+  "a printed circuit board", "a graph with several colored lines", "a web page in a browser",
+];
+
+Median :: struct { median, best, worst: float64; }
+
+median_of :: (values: [] float64) -> Median {
+  sorted := array_copy(values,, temp);
+  quick_sort(sorted, (a: float64, b: float64) -> s64 { return ifx a < b then -1 else ifx a > b then 1 else 0; });
+  m: Median;
+  if !sorted.count  return m;
+  m.median = ifx sorted.count % 2 then sorted[sorted.count / 2] else (sorted[sorted.count / 2 - 1] + sorted[sorted.count / 2]) / 2;
+  m.best, m.worst = sorted[0], sorted[sorted.count - 1];
+  return m;
+}
+
+print_run :: (task: *Task, run: Run) {
+  m := median_of(run.pass_s);
+  print("== % on % (batch %)\n", task.name, run.provider, run.batch);
+  print("   load % s, first % % s\n", FF(run.load_s, 2), run.unit, FF(run.first_s, 3));
+  print("   % ms per % (median of %; best %, worst %), % %s/s", FF(1000 * m.median / run.items, 2), run.unit, run.pass_s.count,
+        FF(1000 * m.best / run.items, 2), FF(1000 * m.worst / run.items, 2), FF(run.items / m.median, 1), run.unit);
+  if run.megapixels  print(", % megapixels/s", FF(run.megapixels / m.median, 1));
+  print("\n");
+  if run.stages  print("   per %, summed over threads (ms): %\n", run.unit, run.stages);
+  if run.gpu.samples  print("   GPU: % MB more memory at the peak, % percent busy, % W, % MHz, up to % C\n",
+                            FF(run.gpu.peak_mb - run.gpu.baseline_mb, 0), FF(run.gpu.util_sum / run.gpu.samples, 0),
+                            FF(run.gpu.power_sum / run.gpu.samples, 0), FF(run.gpu.clock_sum / run.gpu.samples, 0), FF(run.gpu.temperature_max, 0));
+  if run.trace.count  print("   GPU memory over the baseline (MB), after each step [and its peak]: %\n", join(..run.trace, " ",, temp));
+  print("   found % (digest %)\n\n", run.output, run.digest);
+}
+
+write_results :: (info: System_Info, started_at: Apollo_Time, corpus: *Corpus, tasks: [] Task, runs: [] Run, cudart: string, cudnn: string) -> path: string {
+  b: String_Builder;
+  print_to_builder(*b, "%\n", BENCH_RESULTS_FORMAT);
+  date := to_calendar(started_at, .LOCAL);
+  print_to_builder(*b, "date: %-%-% %:%:%\n", date.year, pad2(date.month_starting_at_0 + 1), pad2(date.day_of_month_starting_at_0 + 1), pad2(date.hour), pad2(date.minute), pad2(date.second));
+  print_to_builder(*b, "machine: %\n", one_line(info.machine));
+  if options.note  print_to_builder(*b, "note: %\n", one_line(options.note));
+  print_to_builder(*b, "os: %\n", info.os);
+  print_to_builder(*b, "cpu: % (% threads)\n", one_line(info.cpu), info.logical_cores);
+  print_to_builder(*b, "ram_gb: %\n", FF(info.ram_gb, 1));
+  print_to_builder(*b, "gpu: %\n", ifx info.gpu then one_line(info.gpu) else tprint("none (%)", info.gpu_error));
+  if info.gpu {
+    print_to_builder(*b, "vram_gb: %\n", FF(info.vram_gb, 1));
+    print_to_builder(*b, "driver: % (CUDA %)\n", info.driver, info.cuda_driver);
+    print_to_builder(*b, "power_limit_w: %\n", FF(info.power_limit_w, 0));
+    print_to_builder(*b, "max_clocks_mhz: % / %\n", info.max_sm_clock_mhz, info.max_mem_clock_mhz);
+  }
+  if info.power_source  print_to_builder(*b, "power: %\n", info.power_source);
+  print_to_builder(*b, "onnxruntime: %\n", ort.version);
+  if cudart  print_to_builder(*b, "cudart: %\n", cudart);
+  if cudnn   print_to_builder(*b, "cudnn: %\n", cudnn);
+  print_to_builder(*b, "mcap: % (%, %)\n", VERSION, GIT_HASH, GIT_BRANCH);
+  print_to_builder(*b, "passes: % timed, % warm-up\n", options.repeats, options.warmup);
+  print_to_builder(*b, "corpus: %\n", corpus.fingerprint);
+  print_to_builder(*b, "corpus_size: % images, % megapixels\n", corpus.images.count, FF(corpus.megapixels, 1));
+  for tasks {
+    print_to_builder(*b, "model: % %\n", it.name, it.fingerprint);
+    print_to_builder(*b, "pinned %: %\n", it.name, it.pinned);
+  }
+  for run: runs {
+    task := run.task;
+    if !run.ok { print_to_builder(*b, "error % %: %\n", task.name, run.provider, one_line(run.error)); continue; }
+    m := median_of(run.pass_s);
+    print_to_builder(*b, "result: % % unit=% items=% batch=% ms=% best_ms=% worst_ms=% per_s=% load_s=% first_s=%",
+                     task.name, run.provider, run.unit, run.items, run.batch, FF(1000 * m.median / run.items, 3), FF(1000 * m.best / run.items, 3),
+                     FF(1000 * m.worst / run.items, 3), FF(run.items / m.median, 2), FF(run.load_s, 2), FF(run.first_s, 3));
+    if run.megapixels  print_to_builder(*b, " mp_per_s=%", FF(run.megapixels / m.median, 2));
+    if run.gpu.samples {
+      g := run.gpu;
+      print_to_builder(*b, " vram_mb=% gpu_util=% power_w=% sm_mhz=% temp_c=%", FF(g.peak_mb - g.baseline_mb, 0), FF(g.util_sum / g.samples, 0),
+                       FF(g.power_sum / g.samples, 0), FF(g.clock_sum / g.samples, 0), FF(g.temperature_max, 0));
+    }
+    if run.stages  print_to_builder(*b, " stages=%", run.stages);
+    if run.trace.count  print_to_builder(*b, " trace=%", join(..run.trace, ",",, temp));
+    print_to_builder(*b, " output=% digest=%\n", run.output, run.digest);
+  }
+  name := tprint("%_%-%-%_%-%-%.txt", file_safe(info.machine), date.year, pad2(date.month_starting_at_0 + 1), pad2(date.day_of_month_starting_at_0 + 1),
+                 pad2(date.hour), pad2(date.minute), pad2(date.second));
+  path := tprint("%/%", options.out_folder, name);
+  if !write_entire_file(path, *b)  print("Could not write %.\n", path);
+  return path;
+}
+
+// For the GPU Benchmark window: where the run is. total 0: no count.
+progress :: (task: string, provider: string, phase: string, done: s64, total: s64) {
+  if !options.progress  return;
+  print("@progress % % % % %\n", task, provider, phase, done, total);
+}
+
+// The GPU, sampled every SAMPLE_MS on a thread of its own while the timed passes run.
+SAMPLE_MS :: 100;
+
+sampler: struct {
+  thread: Thread;
+  stats: *Gpu_Stats;
+  running: bool;   // the thread is up
+  paused: bool;    // it is up, and not sampling
+  mutex: Mutex;
+};
+
+sampler_start :: (stats: *Gpu_Stats) {
+  s, ok := gpu_sample();
+  if !ok  return;
+  stats.sampled = true;
+  stats.baseline_mb = s.memory_used_mb;
+  sampler.stats = stats;
+  init(*sampler.mutex);
+  sampler.running = true;
+  sampler.paused = false;
+  thread_init(*sampler.thread, (thread: *Thread) -> s64 {
+    while sampler.running {
+      sleep_milliseconds(SAMPLE_MS);
+      s, ok := gpu_sample();
+      lock(*sampler.mutex);
+      if ok && !sampler.paused && sampler.running {
+        g := sampler.stats;
+        g.peak_mb = max(g.peak_mb, s.memory_used_mb);
+        g.pass_peak_mb = max(g.pass_peak_mb, s.memory_used_mb);
+        g.util_sum += s.util_percent;
+        g.power_sum += s.power_w;
+        g.clock_sum += s.sm_clock_mhz;
+        g.temperature_max = max(g.temperature_max, s.temperature_c);
+        g.samples += 1;
+      }
+      unlock(*sampler.mutex);
+    }
+    return 0;
+  });
+  thread_start(*sampler.thread);
+}
+
+// The GPU's memory now, over the run's baseline, as a step of its trace; with the step's peak when known.
+trace :: (run: *Run, step: string, peak: float64 = -1) {
+  if !run.gpu.sampled  return;
+  s, ok := gpu_sample();
+  if !ok  return;
+  now := s.memory_used_mb - run.gpu.baseline_mb;
+  if peak >= 0  array_add(*run.trace, sprint("%:%[%]", step, FF(now, 0), FF(peak - run.gpu.baseline_mb, 0)));
+  else          array_add(*run.trace, sprint("%:%", step, FF(now, 0)));
+}
+
+// The peak of one timed pass: reset as it starts, read as it ends.
+sampler_pass_start :: () { if !sampler.running  return; lock(*sampler.mutex); sampler.stats.pass_peak_mb = 0; unlock(*sampler.mutex); }
+sampler_pass_peak :: () -> float64 {
+  if !sampler.running  return -1;
+  lock(*sampler.mutex);
+  peak := sampler.stats.pass_peak_mb;
+  unlock(*sampler.mutex);
+  return peak;
+}
+
+sampler_pause  :: () { if !sampler.running  return; lock(*sampler.mutex); sampler.paused = true;  unlock(*sampler.mutex); }
+sampler_resume :: () { if !sampler.running  return; lock(*sampler.mutex); sampler.paused = false; unlock(*sampler.mutex); }
+
+sampler_stop :: () {
+  if !sampler.running  return;
+  lock(*sampler.mutex);
+  sampler.running = false;
+  unlock(*sampler.mutex);
+  thread_is_done(*sampler.thread, -1);
+  thread_deinit(*sampler.thread);
+  destroy(*sampler.mutex);
+}
+
+// Model files are hundreds of MB, so their hashes are kept in the results folder, by path, size and
+// time: hashed again only when one changes.
+hash_cache: [..] struct { key: string; hash: string; };
+
+hash_cache_load :: () {
+  text, ok := read_entire_file(tprint("%/.hash-cache", options.out_folder), log_errors = false);
+  if !ok  return;
+  for split(text, "\n") {
+    tab := find_index_from_left(it, #char "\t");
+    if tab > 0  array_add(*hash_cache, .{trim(advance(it, tab + 1)), slice(it, 0, tab)});
+  }
+}
+
+hash_cache_save :: () {
+  b: String_Builder;
+  for hash_cache  print_to_builder(*b, "%\t%\n", it.hash, it.key);
+  write_entire_file(tprint("%/.hash-cache", options.out_folder), *b);
+}
+
+hash_file_cached :: (path: string) -> string {
+  modtime, size, ok := file_modtime_and_size(path);
+  if !ok  return "";
+  key := sprint("% % % %", size, modtime.low, modtime.high, path);
+  for hash_cache  if it.key == key  return it.hash;
+  hash := sha256_of_file(path);
+  if hash  array_add(*hash_cache, .{key, hash});
+  return hash;
+}
+
+seconds_since :: (start: Apollo_Time) -> float64 {
+  return to_float64_seconds(current_time_monotonic() - start);
+}
+
+short :: (hash: string) -> string { return slice(hash, 0, min(hash.count, 12)); }
+
+pad2 :: (n: s64) -> string { return tprint("%", formatInt(n, minimum_digits = 2)); }
+
+one_line :: (text: string) -> string {
+  return replace(replace(trim(text), "\r", " ",, temp), "\n", " ",, temp);
+}
+
+file_safe :: (text: string) -> string {
+  result := copy_string(text,, temp);
+  for 0..result.count - 1 {
+    c := result[it];
+    if !(is_alnum(c) || c == #char "-")  result[it] = #char "_";
+  }
+  return result;
+}
+
+#load "sha256.jai";
+#load "system_info.jai";
+#load "results.jai";
+
+#import "Basic";
+#import "String";
+#import "File";
+#import "File_Utilities";
+#import "Sort";
+#import "System";
+#import "Thread";
+#import "Image_Decode";
+#import "onnxruntime";
+#import "OCR";
+#import "Labels";
+#import "Segment";

tools/gpu_bench/make_corpus.py

@@ -0,0 +1,192 @@
+# Makes the synthetic part of the GPU benchmark's corpus (tools/gpu_bench/corpus): screenshot-like
+# images full of text, for OCR, and two phone-sized photos scaled up from COCO ones. The images are
+# committed, and corpus.txt pins each one's SHA-256, so every machine benchmarks the same pixels; this
+# script is kept to show how they were made, not to be rerun (another Pillow or FreeType could draw
+# them a pixel differently, and then they would no longer match corpus.txt).
+#
+#   python tools/gpu_bench/make_corpus.py
+
+import os, random
+from PIL import Image, ImageDraw, ImageFont
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+OUT = os.path.join(HERE, "corpus")
+FONTS = os.path.join(HERE, "..", "..", "assets", "fonts")
+
+WORDS = ("the of and to in is that for it as with was on be by this are from or at have an not but "
+         "which they you were all one their there been has more when will would can if about what "
+         "into some time only other new could these two may first then its such like over also back "
+         "after use our work way well even because any most people where much through before right "
+         "capture library image model search folder window screen label segment vector memory driver "
+         "thread allocator buffer request response latency throughput benchmark kernel tensor batch "
+         "session provider runtime device budget scheduler compiler release version settings export "
+         "import config circuit board resistor capacitor footprint schematic voltage current signal "
+         "amplifier channel sample filter frequency spectrum electrode impedance firmware bootloader").split()
+
+def font(name, size):
+    return ImageFont.truetype(os.path.join(FONTS, name), size)
+
+def sentence(rng, n):
+    words = [rng.choice(WORDS) for _ in range(n)]
+    words[0] = words[0].capitalize()
+    return " ".join(words) + rng.choice([".", ".", ",", ":", "?"])
+
+def code_line(rng):
+    kind = rng.randrange(6)
+    a, b, c = rng.choice(WORDS), rng.choice(WORDS), rng.choice(WORDS)
+    if kind == 0: return f"{a}_{b} := {c}_{rng.choice(WORDS)}({rng.randrange(1000)}, {a});"
+    if kind == 1: return f"if {a}.{b} > {rng.randrange(64)} {{ {c}({a}); return false; }}"
+    if kind == 2: return f"// {sentence(rng, rng.randrange(4, 12))}"
+    if kind == 3: return f"for {a}: {b}_{c}  array_add(*{a}s, it.{c});"
+    if kind == 4: return f"{a}_{b} :: (x: s64, {c}: *{a.capitalize()}) -> bool {{"
+    return f"print(\"% {a} {b}\\n\", {c}.count);"
+
+def editor(rng, w, h, dark, mono, size):
+    bg, fg, dim, side = ((30, 30, 30), (212, 212, 212), (110, 110, 110), (37, 37, 38)) if dark else \
+                        ((255, 255, 255), (30, 30, 30), (150, 150, 150), (243, 243, 243))
+    im = Image.new("RGB", (w, h), bg)
+    d = ImageDraw.Draw(im)
+    ui, code = font("OpenSans-Regular.ttf", 14), font(mono, size)
+    d.rectangle([0, 0, w, 32], fill=side)
+    for i, x in enumerate(range(10, w - 200, 180)):
+        d.text((x, 8), f"{rng.choice(WORDS)}_{rng.choice(WORDS)}.jai", font=ui, fill=fg if i == 0 else dim)
+    d.rectangle([0, 32, 260, h], fill=side)
+    y = 44
+    while y < h - 30:
+        d.text((16 + 12 * rng.randrange(3), y), f"{rng.choice(WORDS)}_{rng.choice(WORDS)}", font=ui, fill=fg)
+        y += 24
+    colours = [fg, (86, 156, 214), (206, 145, 120), (106, 153, 85), (197, 134, 192)] if dark else \
+              [fg, (0, 0, 255), (163, 21, 21), (0, 128, 0), (175, 0, 219)]
+    line, y = rng.randrange(1, 900), 44
+    while y < h - 30:
+        d.text((272, y), str(line).rjust(4), font=code, fill=dim)
+        if rng.random() > 0.12:
+            d.text((330 + 4 * size // 2 * rng.randrange(4), y), code_line(rng), font=code, fill=rng.choice(colours))
+        y += int(size * 1.5); line += 1
+    d.rectangle([0, h - 24, w, h], fill=(0, 122, 204))
+    d.text((10, h - 21), f"Ln {line}, Col {rng.randrange(80)}   UTF-8   CRLF   Jai   {rng.choice(WORDS)}", font=ui, fill=(255, 255, 255))
+    return im
+
+def document(rng, w, h, size):
+    im = Image.new("RGB", (w, h), (250, 250, 248))
+    d = ImageDraw.Draw(im)
+    title, body = font("OpenSans-Bold.ttf", size * 2), font("OpenSans-Regular.ttf", size)
+    margin = w // 8
+    d.text((margin, 40), sentence(rng, 5)[:-1], font=title, fill=(20, 20, 20))
+    y = 60 + size * 3
+    while y < h - size * 2:
+        if rng.random() < 0.15:
+            d.text((margin, y), sentence(rng, 4)[:-1], font=font("OpenSans-SemiBold.ttf", int(size * 1.3)), fill=(40, 40, 90))
+            y += int(size * 2.2); continue
+        for _ in range(rng.randrange(2, 7)):
+            text = sentence(rng, 40)
+            words, current = text.split(), ""
+            for word in words:
+                trial = (current + " " + word).strip()
+                if d.textlength(trial, font=body) > w - 2 * margin:
+                    d.text((margin, y), current, font=body, fill=(30, 30, 30)); y += int(size * 1.6); current = word
+                else:
+                    current = trial
+            d.text((margin, y), current, font=body, fill=(30, 30, 30)); y += int(size * 1.6)
+            if y > h - size * 2: break
+        y += size
+    return im
+
+def spreadsheet(rng, w, h):
+    im = Image.new("RGB", (w, h), (255, 255, 255))
+    d = ImageDraw.Draw(im)
+    f, bold = font("OpenSans-Regular.ttf", 13), font("OpenSans-SemiBold.ttf", 13)
+    cols = [40] + [rng.choice([90, 110, 140, 180]) for _ in range(30)]
+    d.rectangle([0, 0, w, 60], fill=(33, 115, 70))
+    d.text((12, 18), f"{rng.choice(WORDS)}_{rng.choice(WORDS)}.xlsx", font=font("OpenSans-SemiBold.ttf", 18), fill=(255, 255, 255))
+    y, row = 60, 0
+    while y < h:
+        x = 0
+        for c, cw in enumerate(cols):
+            if x > w: break
+            d.rectangle([x, y, x + cw, y + 22], outline=(218, 220, 224), fill=(243, 243, 243) if row == 0 or c == 0 else None)
+            if row == 0 and c > 0: text, ft = chr(64 + c) if c <= 26 else "A" + chr(38 + c), bold
+            elif c == 0: text, ft = str(row) if row else "", bold
+            elif row == 1: text, ft = rng.choice(WORDS).capitalize(), bold
+            else:
+                r = rng.random()
+                text = f"{rng.uniform(0, 99999):,.2f}" if r < 0.5 else (str(rng.randrange(10000)) if r < 0.7 else rng.choice(WORDS))
+                ft = f
+            d.text((x + 4, y + 3), text, font=ft, fill=(20, 20, 20))
+            x += cw
+        y += 22; row += 1
+    return im
+
+def terminal(rng, w, h, size):
+    im = Image.new("RGB", (w, h), (12, 12, 12))
+    d = ImageDraw.Draw(im)
+    f = font("FiraCode-Retina.ttf", size)
+    y = 8
+    while y < h - size:
+        r = rng.random()
+        if r < 0.2:   text, colour = f"PS D:\\{rng.choice(WORDS)}\\{rng.choice(WORDS)}> jai first.jai - {rng.choice(WORDS)}", (240, 240, 240)
+        elif r < 0.3: text, colour = f"error: {sentence(rng, 8)}", (240, 80, 80)
+        elif r < 0.4: text, colour = f"[{rng.randrange(24):02}:{rng.randrange(60):02}:{rng.randrange(60):02}] {rng.choice(['INFO', 'WARN', 'DEBUG'])} {sentence(rng, 9)}", (180, 220, 120)
+        else:         text, colour = " ".join(f"{rng.randrange(1 << 16):04x}" for _ in range(rng.randrange(4, 18))), (200, 200, 200)
+        d.text((8, y), text, font=f, fill=colour)
+        y += int(size * 1.35)
+    return im
+
+def chat(rng, w, h):
+    im = Image.new("RGB", (w, h), (245, 245, 250))
+    d = ImageDraw.Draw(im)
+    f, name = font("OpenSans-Regular.ttf", 30), font("OpenSans-Bold.ttf", 26)
+    d.rectangle([0, 0, w, 140], fill=(60, 60, 140))
+    d.text((40, 60), rng.choice(WORDS).capitalize() + " " + rng.choice(WORDS).capitalize(), font=font("OpenSans-SemiBold.ttf", 40), fill=(255, 255, 255))
+    y = 170
+    while y < h - 200:
+        mine = rng.random() < 0.5
+        lines = [sentence(rng, rng.randrange(2, 6)) for _ in range(rng.randrange(1, 4))]
+        width = max(d.textlength(l, font=f) for l in lines) + 40
+        height = 44 * len(lines) + 56
+        x = w - width - 40 if mine else 40
+        d.rounded_rectangle([x, y, x + width, y + height], 24, fill=(90, 120, 230) if mine else (255, 255, 255))
+        d.text((x + 20, y + 10), f"{rng.randrange(1, 13)}:{rng.randrange(60):02} {'PM' if rng.random() < 0.5 else 'AM'}", font=name, fill=(230, 230, 255) if mine else (120, 120, 120))
+        for i, l in enumerate(lines):
+            d.text((x + 20, y + 48 + 44 * i), l, font=f, fill=(255, 255, 255) if mine else (20, 20, 20))
+        y += height + 24
+    return im
+
+def dialog(rng, w, h):
+    im = editor(rng, w, h, True, "JetBrainsMono-Regular.ttf", 14).point(lambda v: v // 2)
+    d = ImageDraw.Draw(im)
+    x0, y0, x1, y1 = w // 2 - 320, h // 2 - 160, w // 2 + 320, h // 2 + 160
+    d.rectangle([x0, y0, x1, y1], fill=(250, 250, 250), outline=(120, 120, 120))
+    d.text((x0 + 24, y0 + 20), sentence(rng, 4)[:-1], font=font("OpenSans-SemiBold.ttf", 22), fill=(20, 20, 20))
+    for i in range(4):
+        d.text((x0 + 24, y0 + 70 + 30 * i), sentence(rng, 9), font=font("OpenSans-Regular.ttf", 17), fill=(40, 40, 40))
+    for i, label in enumerate(["Cancel", "Don't save", "Save"]):
+        bx = x1 - 130 * (3 - i) - 10
+        d.rectangle([bx, y1 - 60, bx + 115, y1 - 22], fill=(0, 103, 192) if label == "Save" else (225, 225, 225))
+        d.text((bx + 20, y1 - 52), label, font=font("OpenSans-Regular.ttf", 17), fill=(255, 255, 255) if label == "Save" else (20, 20, 20))
+    return im
+
+def main():
+    rng = random.Random(20261017)
+    made = [
+        ("screen-editor-dark-1080p.png",   editor(rng, 1920, 1080, True,  "JetBrainsMono-Regular.ttf", 15)),
+        ("screen-editor-light-1080p.png",  editor(rng, 1920, 1080, False, "RobotoMono-Regular.ttf", 14)),
+        ("screen-editor-dark-1440p.png",   editor(rng, 2560, 1440, True,  "FiraCode-Retina.ttf", 16)),
+        ("screen-document-1080p.png",      document(rng, 1920, 1080, 18)),
+        ("screen-document-4k.png",         document(rng, 3840, 2160, 28)),
+        ("screen-spreadsheet-1080p.png",   spreadsheet(rng, 1920, 1080)),
+        ("screen-spreadsheet-1440p.png",   spreadsheet(rng, 2560, 1440)),
+        ("screen-terminal-1080p.png",      terminal(rng, 1920, 1080, 15)),
+        ("screen-terminal-720p.png",       terminal(rng, 1280, 720, 13)),
+        ("screen-chat-phone.png",          chat(rng, 1080, 2400)),
+        ("screen-dialog-1080p.png",        dialog(rng, 1920, 1080)),
+        ("screen-document-strip.png",      document(rng, 1600, 300, 16)),
+    ]
+    for name, im in made:
+        im.save(os.path.join(OUT, name), optimize=False, compress_level=6)
+    # Phone-sized photos: the decoders and resizers see a 12-megapixel image, as from a phone.
+    for source, name in [("coco-000000000139.jpg", "photo-12mp-livingroom.jpg"), ("coco-000000001000.jpg", "photo-12mp-tennis.jpg")]:
+        im = Image.open(os.path.join(OUT, source)).convert("RGB").resize((4032, 3024), Image.LANCZOS)
+        im.save(os.path.join(OUT, name), quality=92)
+
+main()

tools/gpu_bench/results.jai

@@ -0,0 +1,91 @@
+// gpu-bench's results file: what gpu-bench writes, and what mcap's GPU Benchmark window reads to put
+// several machines side by side. Plain text, one fact a line:
+//
+//   gpu-bench 1
+//   machine: NAME                      a header line: key, colon, value
+//   ...
+//   model: ocr/ppocr-v5-mobile 1a2b... the fingerprint of the files a task's model is made of
+//   result: ocr/ppocr-v5-mobile CUDA ms_per_image=12.3 images_per_s=81.2 ...
+//
+// Two results are comparable only when their corpus line (the images) and each task's model line
+// match; the window marks them where they do not.
+
+BENCH_RESULTS_FORMAT :: "gpu-bench 1";
+
+Bench_Pair :: struct { key, value: string; }
+
+Bench_Row :: struct {
+  task: string;          // ocr/ppocr-v5-mobile, labels/siglip2-base-patch16-384, segment/...
+  provider: string;      // CUDA or CPU
+  fields: [..] Bench_Pair;
+}
+
+Bench_Report :: struct {
+  path: string;
+  header: [..] Bench_Pair;   // in the file's order; model lines too, as "model ocr/x" -> hash
+  rows: [..] Bench_Row;
+}
+
+bench_header :: (report: *Bench_Report, key: string) -> string {
+  for report.header  if it.key == key  return it.value;
+  return "";
+}
+
+bench_field :: (row: *Bench_Row, key: string) -> string {
+  for row.fields  if it.key == key  return it.value;
+  return "";
+}
+
+bench_row :: (report: *Bench_Report, task: string, provider: string) -> *Bench_Row {
+  for * report.rows  if it.task == task && it.provider == provider  return it;
+  return null;
+}
+
+// A results file, or ok false and why; from the context allocator.
+bench_parse :: (text: string, path := "") -> Bench_Report, ok: bool, error: string {
+  report: Bench_Report;
+  report.path = copy_string(path);
+  lines := split(text, "\n",, temp);
+  if !lines.count || trim(lines[0]) != BENCH_RESULTS_FORMAT  return report, false, tprint("% is not a gpu-bench results file (no \"%\" on its first line).", path, BENCH_RESULTS_FORMAT);
+  for raw: lines {
+    if it_index == 0  continue;
+    line := trim(raw);
+    if !line.count || line[0] == #char "#"  continue;
+    colon := find_index_from_left(line, #char ":");
+    if colon < 0  continue;
+    key   := trim(slice(line, 0, colon));
+    value := trim(advance(line, colon + 1));
+    if key == "result" {
+      parts := split(value, " ",, temp);
+      if parts.count < 2  continue;
+      row := array_add(*report.rows);
+      row.* = .{task = copy_string(parts[0]), provider = copy_string(parts[1])};
+      for 2..parts.count - 1 {
+        part := parts[it];
+        equals := find_index_from_left(part, #char "=");
+        if equals < 0  continue;
+        array_add(*row.fields, .{copy_string(slice(part, 0, equals)), copy_string(advance(part, equals + 1))});
+      }
+    } else if key == "model" {
+      space := find_index_from_left(value, #char " ");
+      if space < 0  continue;
+      array_add(*report.header, .{tprint("model %", slice(value, 0, space),, context.allocator), copy_string(trim(advance(value, space + 1)))});
+    } else {
+      array_add(*report.header, .{copy_string(key), copy_string(value)});
+    }
+  }
+  return report, true, "";
+}
+
+bench_report_free :: (report: *Bench_Report) {
+  free(report.path);
+  for report.header { free(it.key); free(it.value); }
+  array_free(report.header);
+  for * report.rows {
+    free(it.task); free(it.provider);
+    for it.fields { free(it.key); free(it.value); }
+    array_free(it.fields);
+  }
+  array_free(report.rows);
+  report.* = .{};
+}

tools/gpu_bench/sha256.jai

@@ -0,0 +1,118 @@
+// SHA-256 (FIPS 180-4), for gpu-bench's corpus and model fingerprints: the same hashes as the
+// fetch scripts check (certutil, sha256sum), so they can be read against each other.
+
+Sha256 :: struct {
+  state: [8] u32 = .[0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19];
+  block: [64] u8;
+  used: s64;     // bytes in block
+  total: u64;    // bytes hashed
+}
+
+sha256_update :: (h: *Sha256, data: [] u8) {
+  bytes := data;
+  h.total += cast(u64) bytes.count;
+  if h.used {
+    take := min(64 - h.used, bytes.count);
+    memcpy(h.block.data + h.used, bytes.data, take);
+    h.used += take;
+    bytes = array_view(bytes, take, bytes.count - take);
+    if h.used < 64  return;
+    sha256_blocks(h, h.block.data, 1);
+    h.used = 0;
+  }
+  whole := bytes.count / 64;
+  if whole  sha256_blocks(h, bytes.data, whole);
+  rest := bytes.count - whole * 64;
+  if rest { memcpy(h.block.data, bytes.data + whole * 64, rest); h.used = rest; }
+}
+
+sha256_update :: (h: *Sha256, text: string) { sha256_update(h, cast([] u8) text); }
+
+// Lowercase hex, 64 characters; from allocator.
+sha256_finish :: (h: *Sha256, allocator := context.allocator) -> string {
+  bits := h.total * 8;
+  pad: [72] u8;
+  pad[0] = 0x80;
+  pad_count := ifx h.used < 56 then 56 - h.used else 120 - h.used;
+  for 0..7  pad[pad_count + it] = cast,trunc(u8) (bits >> cast(u64) (56 - 8 * it));
+  total := h.total;
+  sha256_update(h, array_view(pad, 0, pad_count + 8));
+  h.total = total;
+  HEX :: "0123456789abcdef";
+  result := alloc_string(64,, allocator);
+  for word, w: h.state  for b: 0..3 {
+    byte := (word >> cast(u32) (24 - 8 * b)) & 0xff;
+    result[w * 8 + b * 2]     = HEX[byte >> 4];
+    result[w * 8 + b * 2 + 1] = HEX[byte & 0xf];
+  }
+  return result;
+}
+
+sha256_of_string :: (text: string, allocator := context.allocator) -> string {
+  h: Sha256;
+  sha256_update(*h, text);
+  return sha256_finish(*h, allocator);
+}
+
+// The file's hash, read a few MB at a time; "" when it cannot be read.
+sha256_of_file :: (path: string, allocator := context.allocator) -> string {
+  file, ok := file_open(path, log_errors = false);
+  if !ok  return "";
+  defer file_close(*file);
+  CHUNK :: 4 * 1024 * 1024;
+  buffer := NewArray(CHUNK, u8,, context.default_allocator);
+  defer array_free(buffer,, context.default_allocator);
+  h: Sha256;
+  size := file_length(file);
+  done: s64;
+  while done < size {
+    count := min(CHUNK, size - done);
+    if !file_read(file, buffer.data, count)  return "";
+    sha256_update(*h, array_view(buffer, 0, count));
+    done += count;
+  }
+  return sha256_finish(*h, allocator);
+}
+
+#scope_file
+
+K :: u32.[
+  0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
+  0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
+  0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
+  0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
+  0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
+  0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
+  0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
+  0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2,
+];
+
+rotr :: inline (x: u32, n: u32) -> u32 #no_aoc { return (x >> n) | (x << (32 - n)); }
+
+sha256_blocks :: (h: *Sha256, data: *u8, count: s64) #no_aoc {
+  w: [64] u32 = ---;
+  for block: 0..count - 1 {
+    p := data + block * 64;
+    for i: 0..15 {
+      q := p + i * 4;
+      w[i] = ((cast(u32) q[0]) << 24) | ((cast(u32) q[1]) << 16) | ((cast(u32) q[2]) << 8) | (cast(u32) q[3]);
+    }
+    for i: 16..63 {
+      s0 := rotr(w[i - 15], 7) ^ rotr(w[i - 15], 18) ^ (w[i - 15] >> 3);
+      s1 := rotr(w[i - 2], 17) ^ rotr(w[i - 2], 19) ^ (w[i - 2] >> 10);
+      w[i] = w[i - 16] + s0 + w[i - 7] + s1;
+    }
+    a, b, c, d, e, f, g, hh := h.state[0], h.state[1], h.state[2], h.state[3], h.state[4], h.state[5], h.state[6], h.state[7];
+    for i: 0..63 {
+      S1 := rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25);
+      ch := (e & f) ^ (~e & g);
+      t1 := hh + S1 + ch + K[i] + w[i];
+      S0 := rotr(a, 2) ^ rotr(a, 13) ^ rotr(a, 22);
+      maj := (a & b) ^ (a & c) ^ (b & c);
+      t2 := S0 + maj;
+      hh = g; g = f; f = e; e = d + t1; d = c; c = b; b = a; a = t1 + t2;
+    }
+    h.state[0] += a; h.state[1] += b; h.state[2] += c; h.state[3] += d;
+    h.state[4] += e; h.state[5] += f; h.state[6] += g; h.state[7] += hh;
+  }
+}

tools/gpu_bench/system_info.jai

@@ -0,0 +1,214 @@
+// What gpu-bench says about the machine: the computer, CPU and memory, and the NVIDIA GPU through
+// NVML (the driver's management library, nvml.dll / libnvidia-ml.so.1), which also gives the GPU's
+// memory, load, temperature, power and clock while the models run. Without NVML (no NVIDIA driver)
+// the GPU lines say so and the samples are empty.
+
+System_Info :: struct {
+  machine: string;
+  os: string;
+  cpu: string;
+  logical_cores: s64;
+  ram_gb: float64;
+  gpu: string;              // "" without NVML
+  vram_gb: float64;
+  driver: string;
+  cuda_driver: string;      // the CUDA version the driver supports, such as 13.0
+  power_limit_w: float64;   // the board's enforced limit: a laptop's GPU is often far below the desktop part's
+  max_sm_clock_mhz: s64;
+  max_mem_clock_mhz: s64;
+  gpu_error: string;        // why there is no GPU information
+  power_source: string;     // a laptop's: "AC" or "battery", which can change its GPU's speed several times over
+}
+
+// One sample of the GPU while a model runs.
+Gpu_Sample :: struct {
+  memory_used_mb: float64;   // the whole device's, every process's
+  util_percent: float64;
+  temperature_c: float64;
+  power_w: float64;
+  sm_clock_mhz: float64;
+}
+
+system_info :: () -> System_Info {
+  info: System_Info;
+  info.logical_cores = get_number_of_processors(.ALL_LOGICAL);
+  #if OS == .WINDOWS {
+    name: [256] u16;
+    size: u32 = name.count;
+    if GetComputerNameW(name.data, *size)  info.machine = wide_to_utf8(name.data, cast(s32) size);
+    v: OSVERSIONINFOW;
+    v.dwOSVersionInfoSize = size_of(OSVERSIONINFOW);
+    RtlGetVersion(*v);
+    info.os = sprint("Windows %.%.%", v.dwMajorVersion, v.dwMinorVersion, v.dwBuildNumber);
+    buffer: [256] u8;
+    bytes: u32 = buffer.count;
+    if RegGetValueA(HKEY_LOCAL_MACHINE, "HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0", "ProcessorNameString", RRF_RT_REG_SZ, null, buffer.data, *bytes) == 0
+      info.cpu = copy_string(trim(to_string(buffer.data)));
+    memory: MEMORYSTATUSEX;
+    memory.dwLength = size_of(MEMORYSTATUSEX);
+    if GlobalMemoryStatusEx(*memory)  info.ram_gb = memory.ullTotalPhys / (1024.0 * 1024 * 1024);
+    power: SYSTEM_POWER_STATUS;
+    if GetSystemPowerStatus(*power) && power.BatteryFlag != 128 {   // 128: no battery
+      if power.ACLineStatus == 1       info.power_source = "AC";
+      else if power.ACLineStatus == 0  info.power_source = sprint("battery (% percent)", power.BatteryLifePercent);
+    }
+  } else #if OS == .LINUX {
+    host, host_ok := read_entire_file("/etc/hostname", log_errors = false);
+    if host_ok  info.machine = trim(host);
+    release, release_ok := read_entire_file("/proc/sys/kernel/osrelease", log_errors = false);
+    info.os = ifx release_ok then sprint("Linux %", trim(release)) else "Linux";
+    cpuinfo, cpu_ok := read_entire_file("/proc/cpuinfo", log_errors = false);
+    if cpu_ok  for split(cpuinfo, "\n",, temp)  if begins_with(it, "model name") { info.cpu = trim(advance(it, find_index_from_left(it, #char ":") + 1)); break; }
+    meminfo, mem_ok := read_entire_file("/proc/meminfo", log_errors = false);
+    if mem_ok  for split(meminfo, "\n",, temp)  if begins_with(it, "MemTotal:") { info.ram_gb = string_to_int(trim(trim(advance(it, 9)), " kB")) / (1024.0 * 1024); break; }
+  } else {
+    info.os = tprint("%", OS);
+  }
+  if !info.machine  info.machine = "unknown";
+
+  if !nvml_start() { info.gpu_error = nvml.error; return info; }
+  text: [96] u8;
+  if nvml.DeviceGetName(nvml.device, text.data, text.count) == 0  info.gpu = copy_string(to_string(text.data));
+  if nvml.SystemGetDriverVersion(text.data, text.count) == 0  info.driver = copy_string(to_string(text.data));
+  cuda: s32;
+  if nvml.SystemGetCudaDriverVersion(*cuda) == 0  info.cuda_driver = sprint("%.%", cuda / 1000, (cuda % 1000) / 10);
+  vram: Nvml_Memory;
+  if nvml.DeviceGetMemoryInfo(nvml.device, *vram) == 0  info.vram_gb = vram.total / (1024.0 * 1024 * 1024);
+  limit: u32;
+  if nvml.DeviceGetEnforcedPowerLimit(nvml.device, *limit) == 0  info.power_limit_w = limit / 1000.0;
+  clock: u32;
+  if nvml.DeviceGetMaxClockInfo(nvml.device, NVML_CLOCK_SM, *clock) == 0   info.max_sm_clock_mhz = clock;
+  if nvml.DeviceGetMaxClockInfo(nvml.device, NVML_CLOCK_MEM, *clock) == 0  info.max_mem_clock_mhz = clock;
+  return info;
+}
+
+// The GPU now; false without NVML.
+gpu_sample :: () -> Gpu_Sample, ok: bool {
+  s: Gpu_Sample;
+  if !nvml.started  return s, false;
+  memory: Nvml_Memory;
+  if nvml.DeviceGetMemoryInfo(nvml.device, *memory) == 0  s.memory_used_mb = memory.used / (1024.0 * 1024);
+  util: Nvml_Utilization;
+  if nvml.DeviceGetUtilizationRates(nvml.device, *util) == 0  s.util_percent = cast(float64) util.gpu;
+  value: u32;
+  if nvml.DeviceGetTemperature(nvml.device, NVML_TEMPERATURE_GPU, *value) == 0  s.temperature_c = cast(float64) value;
+  if nvml.DeviceGetPowerUsage(nvml.device, *value) == 0  s.power_w = value / 1000.0;
+  if nvml.DeviceGetClockInfo(nvml.device, NVML_CLOCK_SM, *value) == 0  s.sm_clock_mhz = cast(float64) value;
+  return s, true;
+}
+
+// The CUDA runtime's and cuDNN's versions, as loaded into this process by onnxruntime's CUDA provider,
+// with where each came from; "" for one not loaded (before a GPU engine is made).
+cuda_library_versions :: () -> cudart: string, cudnn: string {
+  cudart, cudnn: string;
+  #if OS == .WINDOWS {
+    module := GetModuleHandleA("cudart64_13.dll");
+    if module {
+      get := cast(#type (version: *s32) -> s32 #c_call) GetProcAddress(module, "cudaRuntimeGetVersion");
+      version: s32;
+      if get && get(*version) == 0  cudart = sprint("%.% (%)", version / 1000, (version % 1000) / 10, module_path(module));
+    }
+    module = GetModuleHandleA("cudnn64_9.dll");
+    if module {
+      get := cast(#type () -> u64 #c_call) GetProcAddress(module, "cudnnGetVersion");
+      if get {
+        version := get();   // 9.x: major * 10000 + minor * 100 + patch
+        cudnn = sprint("%.%.% (%)", version / 10000, (version % 10000) / 100, version % 100, module_path(module));
+      }
+    }
+  }
+  return cudart, cudnn;
+}
+
+#scope_file
+
+#if OS == .WINDOWS {
+  SYSTEM_POWER_STATUS :: struct {
+    ACLineStatus, BatteryFlag, BatteryLifePercent, SystemStatusFlag: u8;
+    BatteryLifeTime, BatteryFullLifeTime: u32;
+  }
+  GetSystemPowerStatus :: (status: *SYSTEM_POWER_STATUS) -> BOOL #foreign kernel32;
+  kernel32 :: #system_library "kernel32";
+
+  module_path :: (module: HANDLE) -> string {
+    name: [1024] u16;
+    count := GetModuleFileNameW(module, name.data, name.count);
+    return wide_to_utf8(name.data, count,, temp);
+  }
+}
+
+Nvml_Device :: *void;
+Nvml_Memory :: struct { total, free, used: u64; }
+Nvml_Utilization :: struct { gpu, memory: u32; }
+NVML_CLOCK_SM  :: 1;
+NVML_CLOCK_MEM :: 2;
+NVML_TEMPERATURE_GPU :: 0;
+
+nvml: struct {
+  started: bool;
+  error: string;
+  device: Nvml_Device;
+
+  Init: #type () -> s32 #c_call;
+  DeviceGetHandleByIndex: #type (index: u32, device: *Nvml_Device) -> s32 #c_call;
+  DeviceGetName: #type (device: Nvml_Device, name: *u8, length: u32) -> s32 #c_call;
+  SystemGetDriverVersion: #type (version: *u8, length: u32) -> s32 #c_call;
+  SystemGetCudaDriverVersion: #type (version: *s32) -> s32 #c_call;
+  DeviceGetMemoryInfo: #type (device: Nvml_Device, memory: *Nvml_Memory) -> s32 #c_call;
+  DeviceGetUtilizationRates: #type (device: Nvml_Device, util: *Nvml_Utilization) -> s32 #c_call;
+  DeviceGetTemperature: #type (device: Nvml_Device, sensor: s32, value: *u32) -> s32 #c_call;
+  DeviceGetPowerUsage: #type (device: Nvml_Device, milliwatts: *u32) -> s32 #c_call;
+  DeviceGetEnforcedPowerLimit: #type (device: Nvml_Device, milliwatts: *u32) -> s32 #c_call;
+  DeviceGetClockInfo: #type (device: Nvml_Device, type: s32, mhz: *u32) -> s32 #c_call;
+  DeviceGetMaxClockInfo: #type (device: Nvml_Device, type: s32, mhz: *u32) -> s32 #c_call;
+}
+
+// NVML, and its first GPU: the one onnxruntime's CUDA provider uses (device 0) on a machine with one.
+nvml_start :: () -> bool {
+  if nvml.started  return true;
+  #if OS == .WINDOWS {
+    library := LoadLibraryA("nvml.dll");
+    if !library  library = LoadLibraryA("C:\\Program Files\\NVIDIA Corporation\\NVSMI\\nvml.dll");
+    if !library { nvml.error = "nvml.dll was not found: no NVIDIA driver."; return false; }
+    find :: (name: *u8) -> *void #expand { return GetProcAddress(library, name); }
+  } else #if OS == .LINUX {
+    library := dlopen("libnvidia-ml.so.1", RTLD_NOW);
+    if !library { nvml.error = "libnvidia-ml.so.1 was not found: no NVIDIA driver."; return false; }
+    find :: (name: *u8) -> *void #expand { return dlsym(library, name); }
+  } else {
+    nvml.error = "No NVML on this OS.";
+    return false;
+  }
+  nvml.Init                        = xx find("nvmlInit_v2");
+  nvml.DeviceGetHandleByIndex      = xx find("nvmlDeviceGetHandleByIndex_v2");
+  nvml.DeviceGetName               = xx find("nvmlDeviceGetName");
+  nvml.SystemGetDriverVersion      = xx find("nvmlSystemGetDriverVersion");
+  nvml.SystemGetCudaDriverVersion  = xx find("nvmlSystemGetCudaDriverVersion");
+  nvml.DeviceGetMemoryInfo         = xx find("nvmlDeviceGetMemoryInfo");
+  nvml.DeviceGetUtilizationRates   = xx find("nvmlDeviceGetUtilizationRates");
+  nvml.DeviceGetTemperature        = xx find("nvmlDeviceGetTemperature");
+  nvml.DeviceGetPowerUsage         = xx find("nvmlDeviceGetPowerUsage");
+  nvml.DeviceGetEnforcedPowerLimit = xx find("nvmlDeviceGetEnforcedPowerLimit");
+  nvml.DeviceGetClockInfo          = xx find("nvmlDeviceGetClockInfo");
+  nvml.DeviceGetMaxClockInfo       = xx find("nvmlDeviceGetMaxClockInfo");
+  if !nvml.Init || !nvml.DeviceGetHandleByIndex || !nvml.DeviceGetName || !nvml.SystemGetDriverVersion || !nvml.SystemGetCudaDriverVersion
+     || !nvml.DeviceGetMemoryInfo || !nvml.DeviceGetUtilizationRates || !nvml.DeviceGetTemperature || !nvml.DeviceGetPowerUsage
+     || !nvml.DeviceGetEnforcedPowerLimit || !nvml.DeviceGetClockInfo || !nvml.DeviceGetMaxClockInfo {
+    nvml.error = "NVML lacks a function gpu-bench needs: an old driver?";
+    return false;
+  }
+  status := nvml.Init();
+  if status != 0 { nvml.error = tprint("NVML did not start (error %).", status); return false; }
+  status = nvml.DeviceGetHandleByIndex(0, *nvml.device);
+  if status != 0 { nvml.error = tprint("NVML found no GPU (error %).", status); return false; }
+  nvml.started = true;
+  return true;
+}
+
+#if OS == .WINDOWS {
+  #import "Windows";
+  #import "Windows_Utf8";
+  #import "Windows_Registry";
+} else #if OS == .LINUX {
+  #import "POSIX";
+}