MusaMahmood/mcap
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

[0.1.36] GPU benchmark (gpu-bench branch, not for master)

Browse Source 
bin/gpu-bench times every GPU model (OCR mobile and server, the label
image encoder, segmentation; optionally the CPU and the label text
encoder) over a fixed 30-image corpus in tools/gpu_bench/corpus. Every
corpus file is checked against corpus.txt's SHA-256 before anything
runs; each model's files and models.config entry are fingerprinted;
the GPU, driver, power limit, CUDA runtime, cuDNN and onnxruntime are
recorded, and NVML samples memory, load, power and clocks during the
timed passes. Results go to bin/gpu-bench-results as text files.

The GPU Bench button opens a window that runs gpu-bench in its own
process (no console) and lays results files from several machines side
by side against a baseline, marking a different corpus or model.
This commit is contained in:
Musa Mahmood 2026-10-17 12:38:25 -04:00
parent 5dd62ac201
commit 32af1f6496
44 changed files with 1910 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitattributes vendored Normal file
View File

@ -0,0 +1 @@
tools/gpu_bench/corpus/** binary

2
check_targets.jai
View File

@ -52,7 +52,7 @@ APP_NAME :: "mcap";
APP_FOLDER_NAME :: "mcap"; APP_FOLDER_NAME :: "mcap";
GIT_BRANCH :: "check"; GIT_BRANCH :: "check";
GIT_HASH :: "-"; GIT_HASH :: "-";
VERSION :: "0.1.35"; VERSION :: "0.1.36";
BUILD_DATE :: "-"; BUILD_DATE :: "-";
DEBUG :: true; DEBUG :: true;
COMPILE_TESTS :: true; COMPILE_TESTS :: true;

11
first.jai
View File

@ -8,7 +8,7 @@ APP_NAME :: "mcap";
APP_FOLDER_NAME :: "mcap"; APP_FOLDER_NAME :: "mcap";
APP_BUNDLE_IDENTIFIER :: "com.example.mcap"; APP_BUNDLE_IDENTIFIER :: "com.example.mcap";
VERSION :: "0.1.35"; VERSION :: "0.1.36";
// Where everything goes: bin/, or bin-headless/ for builds made with no_crash_dialog, so automated // Where everything goes: bin/, or bin-headless/ for builds made with no_crash_dialog, so automated
// builds never write over the ones you run by hand. // builds never write over the ones you run by hand.
build_directory := "bin"; build_directory := "bin";
@ -71,6 +71,10 @@ build_project :: () {
// jai first.jai - record_probe: also bin/record-probe, a screen recorded with modules/Screen_Record (video-capture-plan.html, phase 1). // jai first.jai - record_probe: also bin/record-probe, a screen recorded with modules/Screen_Record (video-capture-plan.html, phase 1).
if array_find(args, "record_probe") array_add(*pending, start_tool_build("record-probe", "tools/record_probe.jai", optimized = build_release)); if array_find(args, "record_probe") array_add(*pending, start_tool_build("record-probe", "tools/record_probe.jai", optimized = build_release));
// The gpu-bench branch: bin/gpu-bench, every GPU model timed over a fixed corpus (tools/gpu_bench),
// always built, as the GPU Benchmark window runs it.
array_add(*pending, start_tool_build("gpu-bench", "tools/gpu_bench/gpu_bench.jai", optimized = build_release));
wait_for_builds(pending); wait_for_builds(pending);
for pending if !it.failed { for pending if !it.failed {
@ -434,6 +438,11 @@ copy_assets :: () {
} else { } else {
log("Segmentation: the model has not been made; see modules/Segment/README.md."); log("Segmentation: the model has not been made; see modules/Segment/README.md.");
} }
// gpu-bench's corpus, in bin/gpu-bench-corpus: in git, so every machine has the same images.
make_directory_if_it_does_not_exist(tprint("%/gpu-bench-corpus", build_directory));
copy_if_changed("tools/gpu_bench/corpus.txt", tprint("%/gpu-bench-corpus/corpus.txt", build_directory));
for file_list("tools/gpu_bench/corpus") copy_if_changed(it, tprint("%/gpu-bench-corpus/%", build_directory, path_filename(it)));
} }
copy_if_changed :: (from: string, to: string) { copy_if_changed :: (from: string, to: string) {

2
src/App.jai
View File

@ -26,6 +26,8 @@ USING_WINDOWS :: OS == .WINDOWS;
#load "app/import.jai"; #load "app/import.jai";
#load "app/open_file.jai"; #load "app/open_file.jai";
#load "app/segment_ui.jai"; #load "app/segment_ui.jai";
#load "app/gpu_bench_ui.jai"; // the gpu-bench branch: the GPU Benchmark window
#load "../tools/gpu_bench/results.jai"; // its results files
#load "app/library_folders.jai"; #load "app/library_folders.jai";
#load "app/exit_shutdown.jai"; #load "app/exit_shutdown.jai";
#load "app/screen_capture.jai"; #load "app/screen_capture.jai";

2
src/app/app_ui.jai
View File

@ -39,6 +39,7 @@ app_ui_shutdown :: () {
open_file_shutdown(); open_file_shutdown();
return; return;
} }
gpu_bench_ui_shutdown(); // ends a benchmark run still going
app_shutdown_step(.CAPTURE); app_shutdown_step(.CAPTURE);
screen_capture_shutdown(); screen_capture_shutdown();
app_shutdown_step(.IMPORT); app_shutdown_step(.IMPORT);
@ -121,6 +122,7 @@ app_main_window_ui :: () {
labels_ui_frame(); labels_ui_frame();
segment_ui_frame(); segment_ui_frame();
stats_ui_frame(); stats_ui_frame();
gpu_bench_ui_frame(); // app/gpu_bench_ui.jai
import_frame(); import_frame();
open_file_frame(); // after import_frame, which hands the grid what an import of it wrote open_file_frame(); // after import_frame, which hands the grid what an import of it wrote
screen_record_leftovers_popup(); screen_record_leftovers_popup();

525
src/app/gpu_bench_ui.jai Normal file
View File

@ -0,0 +1,525 @@
// The GPU Benchmark window (the gpu-bench branch): runs gpu-bench (tools/gpu_bench) in a process of
// its own, so the app's engines and memory do not touch its numbers, shows how far it has got, and
// lays the results files in gpu-bench-results side by side. To compare with another machine, copy
// its results file into that folder (Open folder).
GPU_BENCH_WINDOW :: "GPU Benchmark";
GPU_BENCH_LOG_LINES :: 2000;
gpu_bench_ui: struct {
window_open: bool;
run_ocr := true;
run_labels := true;
run_segment := true;
cpu: bool;
repeats: s32 = 3;
note: [128] u8;
// A run: the reader thread owns the child process and appends what it prints to output.
running: bool;
mutex_ready: bool;
mutex: Mutex;
thread: Thread;
command: [..] string; // heap; for the thread
output: [..] u8; // under mutex; heap
finished: bool; // under mutex: the child has exited, or could not start
exit_code: s32; // under mutex
start_error: string; // under mutex; heap
cancelled: bool;
#if OS == .WINDOWS process: HANDLE; // under mutex
else process: Process;
started_at: Apollo_Time;
parsed: s64; // bytes of output made into lines
log: [..] string; // heap
scroll_log: bool;
status: string; // heap: where the run is, or how it ended
reports: [..] Bench_Report; // newest first
shown: [..] bool; // per report: a column in the comparison
baseline: string; // heap: the path of the report the others are compared with
loaded: bool;
};
gpu_bench_results_folder :: () -> string {
return tprint("%gpu-bench-results", path_strip_filename(get_path_of_running_executable()));
}
// The release build, else the debug one; "" when neither is beside the app.
gpu_bench_executable :: () -> path: string, debug: bool {
folder := path_strip_filename(get_path_of_running_executable());
extension := ifx OS == .WINDOWS then ".exe" else "";
release := tprint("%gpu-bench%", folder, extension);
if file_exists(release) return release, false;
debug := tprint("%gpu-bench-debug%", folder, extension);
if file_exists(debug) return debug, true;
return "", false;
}
gpu_bench_ui_frame :: () { using ImGui;
if gpu_bench_ui.running gpu_bench_poll();
if !gpu_bench_ui.window_open return;
if !gpu_bench_ui.loaded gpu_bench_load_reports();
SetNextWindowSize(.{980, 680}, .FirstUseEver);
if Begin(GPU_BENCH_WINDOW, *gpu_bench_ui.window_open) gpu_bench_window_contents();
End();
}
gpu_bench_ui_shutdown :: () {
if gpu_bench_ui.running {
gpu_bench_cancel();
thread_is_done(*gpu_bench_ui.thread, -1);
thread_deinit(*gpu_bench_ui.thread);
gpu_bench_ui.running = false;
}
push_allocator(context.default_allocator);
for * gpu_bench_ui.reports bench_report_free(it);
array_free(gpu_bench_ui.reports);
array_free(gpu_bench_ui.shown);
for gpu_bench_ui.log free(it);
array_free(gpu_bench_ui.log);
array_free(gpu_bench_ui.output);
for gpu_bench_ui.command free(it);
array_free(gpu_bench_ui.command);
free(gpu_bench_ui.status);
free(gpu_bench_ui.baseline);
free(gpu_bench_ui.start_error);
}
#scope_file
gpu_bench_window_contents :: () { using ImGui;
TextWrapped("Times each model mcap runs on the GPU (OCR, labels, segmentation) over a fixed corpus of 30 images that every machine has, each checked against its SHA-256 before anything runs. Each model's files and settings are fingerprinted too, so a comparison shows whether it is like for like.");
exe, debug := gpu_bench_executable();
if gpu_bench_ui.running {
Text("%", gpu_bench_ui.status);
SameLine();
TextColored(app_theme.palette.dim, "(% s)", formatFloat(to_float64_seconds(current_time_monotonic() - gpu_bench_ui.started_at), trailing_width = 0));
SameLine();
if Button("Cancel") gpu_bench_cancel();
} else {
Checkbox("OCR", *gpu_bench_ui.run_ocr);
SameLine(); Checkbox("Labels", *gpu_bench_ui.run_labels);
SameLine(); Checkbox("Segmentation", *gpu_bench_ui.run_segment);
SameLine(); Checkbox("CPU too", *gpu_bench_ui.cpu);
if IsItemHovered() SetTooltip("Each model on the CPU as well, after the GPU, and the label model's text encoder (a like: search). Slow: minutes for segmentation.");
SameLine();
SetNextItemWidth(120);
SliderInt("Timed passes", *gpu_bench_ui.repeats, 1, 10);
if IsItemHovered() SetTooltip("Passes over the corpus that are timed, after one untimed warm-up pass; the median is reported");
SetNextItemWidth(-FLOAT32_MIN);
InputTextWithHint("##note", "A note kept with the results, such as: plugged in, best performance mode", gpu_bench_ui.note.data, gpu_bench_ui.note.count);
reason := "";
if !exe reason = "gpu-bench is not beside the app: build with jai first.jai - release.";
else if ocr_ui.thread_busy reason = "A scan or a reading is running on the GPU: let it finish, or pause or cancel it, first.";
else if !gpu_bench_ui.run_ocr && !gpu_bench_ui.run_labels && !gpu_bench_ui.run_segment reason = "Choose a kind of model to run.";
if reason BeginDisabled();
if Button("Run the benchmark") gpu_bench_start(exe);
if reason EndDisabled();
if reason { SameLine(); TextColored(app_theme.palette.warning, "%", reason); }
else if debug { SameLine(); TextColored(app_theme.palette.warning, "Only the debug gpu-bench is built: its CPU work is slow, so build with - release to compare."); }
else if gpu_bench_ui.status { SameLine(); TextColored(app_theme.palette.dim, "%", gpu_bench_ui.status); }
TextColored(app_theme.palette.dim, "mcap's own models stay loaded while it runs. For the cleanest numbers, close mcap and run gpu-bench in a terminal; its results land in the same folder.");
}
if CollapsingHeader(temp_c_string(tprint("Output (% lines)###output", gpu_bench_ui.log.count))) {
if BeginChild("##log", .{0, 220}, .Borders, .HorizontalScrollbar) {
for gpu_bench_ui.log TextUnformatted(it);
if gpu_bench_ui.scroll_log { SetScrollHereY(1); gpu_bench_ui.scroll_log = false; }
}
EndChild();
}
Separator();
folder := gpu_bench_results_folder();
Text("Results (% files)", gpu_bench_ui.reports.count);
SameLine();
if SmallButton("Open folder") {
make_directory_if_it_does_not_exist(folder, recursive = true);
platform_open_file_in_default_program(folder, folder);
}
if IsItemHovered() SetTooltip("% : copy another machine's results files here to compare", folder);
SameLine();
if SmallButton("Reload") gpu_bench_load_reports();
if !gpu_bench_ui.reports.count {
TextColored(app_theme.palette.dim, "None yet: run the benchmark.");
return;
}
gpu_bench_reports_table();
Spacing();
gpu_bench_comparison_table();
}
// The results files: which are shown, and which is the baseline.
gpu_bench_reports_table :: () { using ImGui;
height := min(cast(float) gpu_bench_ui.reports.count + 1, 7) * GetFrameHeightWithSpacing();
if !BeginTable("##reports", 7, .Borders | .RowBg | .SizingFixedFit | .ScrollY, .{0, height}) return;
TableSetupScrollFreeze(0, 1);
TableSetupColumn("Show");
TableSetupColumn("Baseline");
TableSetupColumn("Date");
TableSetupColumn("Machine");
TableSetupColumn("GPU");
TableSetupColumn("Power");
TableSetupColumn("Note", .WidthStretch);
TableHeadersRow();
for * report, n: gpu_bench_ui.reports {
PushID(cast(s32) n);
TableNextRow();
TableNextColumn(); Checkbox("##show", *gpu_bench_ui.shown[n]);
TableNextColumn();
if RadioButton("##baseline", report.path == gpu_bench_ui.baseline) {
free(gpu_bench_ui.baseline,, context.default_allocator);
gpu_bench_ui.baseline = copy_string(report.path,, context.default_allocator);
gpu_bench_ui.shown[n] = true;
}
TableNextColumn(); TextUnformatted(bench_header(report, "date"));
if IsItemHovered() SetTooltip("%", report.path);
TableNextColumn(); TextUnformatted(bench_header(report, "machine"));
TableNextColumn(); TextUnformatted(bench_header(report, "gpu"));
TableNextColumn();
limit := bench_header(report, "power_limit_w");
power := bench_header(report, "power");
TextUnformatted(tprint("%%%", ifx limit then tprint("% W", limit) else "", ifx limit && power then ", " else "", power));
TableNextColumn(); TextUnformatted(bench_header(report, "note"));
PopID();
}
EndTable();
}
// The shown reports side by side: what each machine is, then each model's time, against the baseline's.
gpu_bench_comparison_table :: () { using ImGui;
columns: [..] *Bench_Report;
columns.allocator = temp;
baseline: *Bench_Report;
for * gpu_bench_ui.reports {
if gpu_bench_ui.shown[it_index] array_add(*columns, it);
if it.path == gpu_bench_ui.baseline baseline = it;
}
if !columns.count { TextColored(app_theme.palette.dim, "Tick Show for the results to compare."); return; }
if !baseline baseline = columns[0];
// Every task and provider any of them ran, in the order they ran.
Bench_Key :: struct { task, provider: string; }
keys: [..] Bench_Key;
keys.allocator = temp;
for report: columns for report.rows {
found := false;
for key: keys if key.task == it.task && key.provider == it.provider found = true;
if !found array_add(*keys, .{it.task, it.provider});
}
if !BeginTable("##compare", cast(s32) columns.count + 1, .Borders | .RowBg | .SizingFixedFit | .ScrollX | .ScrollY, .{0, -FLOAT32_MIN}) return;
TableSetupScrollFreeze(1, 1);
TableSetupColumn("");
for columns TableSetupColumn(temp_c_string(tprint("%##%", bench_header(it, "machine"), it_index)));
TableHeadersRow();
header_row :: (label: string, key: string, tooltip := "") #expand {
TableNextRow();
TableNextColumn(); TextColored(app_theme.palette.dim, "%", label);
if tooltip && IsItemHovered() SetTooltip("%", tooltip);
for columns {
TableNextColumn();
value := bench_header(it, key);
if it != baseline && value != bench_header(baseline, key) TextColored(app_theme.palette.accent, "%", value);
else TextUnformatted(value);
}
}
header_row("GPU", "gpu");
header_row("Memory", "vram_gb", "GB of GPU memory");
header_row("Power limit", "power_limit_w", "The board's enforced limit in watts: a laptop's GPU is often far below the desktop part's");
header_row("Power", "power");
header_row("Driver", "driver");
header_row("cuDNN", "cudnn");
header_row("onnxruntime", "onnxruntime");
header_row("CPU", "cpu");
header_row("Date", "date");
header_row("Note", "note");
// The corpus: different images make every number not comparable.
TableNextRow();
TableNextColumn(); TextColored(app_theme.palette.dim, "Corpus");
for columns {
TableNextColumn();
corpus := bench_header(it, "corpus");
if corpus == bench_header(baseline, "corpus") TextColored(app_theme.palette.success, "same (%)", slice(corpus, 0, min(corpus.count, 12)));
else TextColored(app_theme.palette.error, "DIFFERENT: not comparable");
}
for key: keys {
TableNextRow();
TableNextColumn();
TextUnformatted(tprint("% %", key.task, ifx key.provider == "CUDA" then "(GPU)" else "(CPU)"));
base_row := bench_row(baseline, key.task, key.provider);
base_ms := ifx base_row then string_to_float64(bench_field(base_row, "ms")) else 0;
for report: columns {
TableNextColumn();
row := bench_row(report, key.task, key.provider);
if !row {
error := bench_header(report, tprint("error % %", key.task, key.provider));
if !error error = bench_header(report, tprint("error % CPU", key.task)); // a GPU run that fell back
if error { TextColored(app_theme.palette.error, "failed"); if IsItemHovered() SetTooltip("%", error); }
else TextColored(app_theme.palette.dim, "-");
continue;
}
ms := string_to_float64(bench_field(row, "ms"));
unit := bench_field(row, "unit");
same_model := bench_header(report, tprint("model %", key.task)) == bench_header(baseline, tprint("model %", key.task));
text := tprint("% ms/%", formatFloat(ms, trailing_width = ifx ms < 10 then 2 else 1), unit);
if report != baseline && base_ms > 0 && ms > 0 {
ratio := base_ms / ms; // above 1: faster than the baseline
colour := ifx ratio > 1.05 then app_theme.palette.success else ifx ratio < 0.95 then app_theme.palette.warning else app_theme.palette.neutral;
TextColored(colour, "% (%x)%", text, formatFloat(ratio, trailing_width = 2), ifx same_model then "" else " *");
} else {
TextUnformatted(tprint("%%", text, ifx same_model then "" else " *"));
}
if IsItemHovered() {
builder: String_Builder;
builder.allocator = temp;
if !same_model append(*builder, "* Different model files or models.config settings from the baseline's: not like for like.\n\n");
print_to_builder(*builder, "% on %: % %s per second", key.task, report.path, bench_field(row, "per_s"), unit);
mp := bench_field(row, "mp_per_s");
if mp print_to_builder(*builder, ", % megapixels per second", mp);
print_to_builder(*builder, "\nmedian of the timed passes; best % ms, worst % ms\n", bench_field(row, "best_ms"), bench_field(row, "worst_ms"));
print_to_builder(*builder, "loading % s, the first % % s, batch %\n", bench_field(row, "load_s"), unit, bench_field(row, "first_s"), bench_field(row, "batch"));
vram := bench_field(row, "vram_mb");
if vram print_to_builder(*builder, "GPU: % MB more memory at the peak, % percent busy, % W, % MHz, up to % C\n", vram,
bench_field(row, "gpu_util"), bench_field(row, "power_w"), bench_field(row, "sm_mhz"), bench_field(row, "temp_c"));
stages := bench_field(row, "stages");
if stages print_to_builder(*builder, "ms per %, summed over threads: %\n", unit, stages);
print_to_builder(*builder, "found %", bench_field(row, "output"));
if report != baseline && base_row {
same_output := bench_field(row, "digest") == bench_field(base_row, "digest");
print_to_builder(*builder, ", % the baseline's", ifx same_output then "the same as" else "not quite the same as (GPUs round differently)");
}
SetTooltip("%", builder_to_string(*builder,, temp));
}
}
}
EndTable();
}
gpu_bench_load_reports :: () {
push_allocator(context.default_allocator);
for * gpu_bench_ui.reports bench_report_free(it);
array_reset(*gpu_bench_ui.reports);
array_reset(*gpu_bench_ui.shown);
gpu_bench_ui.loaded = true;
folder := gpu_bench_results_folder();
if !file_exists(folder) return;
for file_list(folder,, temp) {
if !ends_with_nocase(it, ".txt") continue;
text, ok := read_entire_file(it, log_errors = false,, temp);
if !ok continue;
report, parsed, error := bench_parse(text, it);
if !parsed { log_error("[gpu-bench] %", error); bench_report_free(*report); continue; }
array_add(*gpu_bench_ui.reports, report);
}
quick_sort(gpu_bench_ui.reports, (a: Bench_Report, b: Bench_Report) -> s64 { return compare(bench_header(*b, "date"), bench_header(*a, "date")); });
array_resize(*gpu_bench_ui.shown, gpu_bench_ui.reports.count);
for * gpu_bench_ui.shown it.* = true;
found := false;
for gpu_bench_ui.reports if it.path == gpu_bench_ui.baseline found = true;
if !found && gpu_bench_ui.reports.count {
free(gpu_bench_ui.baseline);
gpu_bench_ui.baseline = copy_string(gpu_bench_ui.reports[gpu_bench_ui.reports.count - 1].path); // the oldest: what later runs are measured against
}
}
gpu_bench_start :: (exe: string) {
push_allocator(context.default_allocator);
if !gpu_bench_ui.mutex_ready { init(*gpu_bench_ui.mutex); gpu_bench_ui.mutex_ready = true; }
for gpu_bench_ui.command free(it);
array_reset(*gpu_bench_ui.command);
add :: (s: string) #expand { array_add(*gpu_bench_ui.command, copy_string(s)); }
add(exe);
add("-progress");
add("-repeats"); add(tprint("%", gpu_bench_ui.repeats));
add("-out"); add(gpu_bench_results_folder());
only: [..] string;
only.allocator = temp;
if gpu_bench_ui.run_ocr array_add(*only, "ocr");
if gpu_bench_ui.run_labels array_add(*only, "labels");
if gpu_bench_ui.run_segment array_add(*only, "segment");
add("-only"); add(join(..only, ",",, temp));
if gpu_bench_ui.cpu add("-cpu");
note := trim(to_string(gpu_bench_ui.note.data));
if note { add("-note"); add(note); }
for gpu_bench_ui.log free(it);
array_reset(*gpu_bench_ui.log);
array_reset(*gpu_bench_ui.output);
free(gpu_bench_ui.start_error);
gpu_bench_ui.start_error = "";
gpu_bench_ui.finished = false;
gpu_bench_ui.cancelled = false;
gpu_bench_ui.exit_code = 0;
gpu_bench_ui.parsed = 0;
gpu_bench_set_status("Starting...");
gpu_bench_ui.started_at = current_time_monotonic();
thread_init(*gpu_bench_ui.thread, gpu_bench_thread_proc, name = "gpu-bench");
thread_start(*gpu_bench_ui.thread);
gpu_bench_ui.running = true;
log("[gpu-bench] Running %", join(..gpu_bench_ui.command, " ",, temp));
}
gpu_bench_cancel :: () {
lock(*gpu_bench_ui.mutex);
gpu_bench_ui.cancelled = true;
#if OS == .WINDOWS { if gpu_bench_ui.process TerminateProcess(gpu_bench_ui.process, 1); }
else kill_process(*gpu_bench_ui.process);
unlock(*gpu_bench_ui.mutex);
}
// What the child printed since the last frame, made into lines; and the end of the run.
gpu_bench_poll :: () {
push_allocator(context.default_allocator);
lock(*gpu_bench_ui.mutex);
finished := gpu_bench_ui.finished;
exit_code := gpu_bench_ui.exit_code;
new_text := copy_string(cast(string) array_view(gpu_bench_ui.output, gpu_bench_ui.parsed, gpu_bench_ui.output.count - gpu_bench_ui.parsed),, temp);
start_error := copy_string(gpu_bench_ui.start_error,, temp);
unlock(*gpu_bench_ui.mutex);
// Whole lines only, until the end.
end := find_index_from_right(new_text, #char "\n");
if finished end = new_text.count;
if end > 0 {
gpu_bench_ui.parsed += end;
for raw: split(slice(new_text, 0, end), "\n",, temp) {
line := trim_right(raw, "\r");
if !line.count continue;
if line[0] == 0x1b continue; // onnxruntime's coloured warnings
if begins_with(line, "@progress ") { gpu_bench_set_status(gpu_bench_progress_words(advance(line, 10))); continue; }
if begins_with(line, "@results ") continue;
array_add(*gpu_bench_ui.log, copy_string(line));
gpu_bench_ui.scroll_log = true;
}
while gpu_bench_ui.log.count > GPU_BENCH_LOG_LINES {
free(gpu_bench_ui.log[0]);
array_ordered_remove_by_index(*gpu_bench_ui.log, 0);
}
}
if !finished return;
thread_is_done(*gpu_bench_ui.thread, -1);
thread_deinit(*gpu_bench_ui.thread);
gpu_bench_ui.running = false;
seconds := formatFloat(to_float64_seconds(current_time_monotonic() - gpu_bench_ui.started_at), trailing_width = 0);
if start_error gpu_bench_set_status(start_error);
else if gpu_bench_ui.cancelled gpu_bench_set_status("Cancelled.");
else if exit_code == 0 gpu_bench_set_status(tprint("Done in % s.", seconds));
else gpu_bench_set_status(tprint("gpu-bench failed (exit code %) after % s: Output says why.", exit_code, seconds));
log("[gpu-bench] %", gpu_bench_ui.status);
gpu_bench_load_reports();
}
// "@progress ocr/ppocr-v5-mobile CUDA timing 2 3" in words.
gpu_bench_progress_words :: (text: string) -> string {
parts := split(text, " ",, temp);
if parts.count < 5 return text;
task, provider, phase, done, total := parts[0], parts[1], parts[2], parts[3], parts[4];
on := ifx provider == "CUDA" then "on the GPU" else ifx provider == "CPU" then "on the CPU" else "";
if task == "corpus" return tprint("The corpus: % %...", phase, ifx done != "0" then tprint("image %", done) else "");
if task == "hashing" return tprint("Fingerprinting %: % (% of %)", provider, phase, done, total);
if phase == {
case "loading"; return tprint("% %: loading the model...", task, on);
case "first"; return tprint("% %: the first image...", task, on);
case "warm-up"; return tprint("% %: warm-up pass % of %...", task, on, done, total);
case "timing"; return tprint("% %: timed pass % of %...", task, on, done, total);
}
return text;
}
gpu_bench_set_status :: (text: string) {
free(gpu_bench_ui.status,, context.default_allocator);
gpu_bench_ui.status = copy_string(text,, context.default_allocator);
}
// The reader thread: starts gpu-bench, takes what it prints until it exits.
gpu_bench_thread_proc :: (thread: *Thread) -> s64 {
append_output :: (data: *u8, count: s64) {
lock(*gpu_bench_ui.mutex);
old := gpu_bench_ui.output.count;
array_resize(*gpu_bench_ui.output, old + count, initialize = false,, context.default_allocator);
memcpy(gpu_bench_ui.output.data + old, data, count);
unlock(*gpu_bench_ui.mutex);
}
finish :: (exit_code: s32, error := "") {
lock(*gpu_bench_ui.mutex);
gpu_bench_ui.exit_code = exit_code;
if error gpu_bench_ui.start_error = copy_string(error,, context.default_allocator);
gpu_bench_ui.finished = true;
unlock(*gpu_bench_ui.mutex);
}
buffer: [4096] u8;
#if OS == .WINDOWS {
// Not the Process module: from a GUI program, it would open a console window for gpu-bench.
attributes: SECURITY_ATTRIBUTES;
attributes.nLength = size_of(SECURITY_ATTRIBUTES);
attributes.bInheritHandle = 1;
read_end, write_end: HANDLE;
if !CreatePipe(*read_end, *write_end, *attributes, 0) { finish(-1, tprint("Could not start gpu-bench: no pipe (Windows error %).", GetLastError())); return 0; }
SetHandleInformation(read_end, HANDLE_FLAG_INHERIT, 0);
startup: STARTUPINFOW;
startup.cb = size_of(STARTUPINFOW);
startup.dwFlags = STARTF_USESTDHANDLES;
startup.hStdOutput = write_end;
startup.hStdError = write_end;
builder: String_Builder;
builder.allocator = temp;
for gpu_bench_ui.command { if it_index append(*builder, " "); append(*builder, windows_quote_argument(it)); }
command_line := builder_to_string(*builder,, temp);
info: PROCESS_INFORMATION;
working := path_strip_filename(gpu_bench_ui.command[0]);
started := CreateProcessW(null, utf8_to_wide(command_line,, temp), null, null, 1, CREATE_NO_WINDOW, null, utf8_to_wide(working,, temp), *startup, *info);
CloseHandle(write_end); // the child's copy is the only one left: reading ends when it exits
if !started {
error := GetLastError();
CloseHandle(read_end);
finish(-1, tprint("Could not start % (Windows error %).", gpu_bench_ui.command[0], error));
return 0;
}
CloseHandle(info.hThread);
lock(*gpu_bench_ui.mutex);
gpu_bench_ui.process = info.hProcess;
cancelled := gpu_bench_ui.cancelled;
unlock(*gpu_bench_ui.mutex);
if cancelled TerminateProcess(info.hProcess, 1);
while true {
read: u32;
if !ReadFile(read_end, buffer.data, buffer.count, *read) || !read break;
append_output(buffer.data, read);
}
CloseHandle(read_end);
WaitForSingleObject(info.hProcess, INFINITE);
exit_code: u32;
GetExitCodeProcess(info.hProcess, *exit_code);
lock(*gpu_bench_ui.mutex);
gpu_bench_ui.process = null;
unlock(*gpu_bench_ui.mutex);
CloseHandle(info.hProcess);
finish(cast,no_check(s32) exit_code);
} else {
errors: [4096] u8;
if !create_process(*gpu_bench_ui.process, ..gpu_bench_ui.command, capture_and_return_output = true) {
finish(-1, tprint("Could not start %.", gpu_bench_ui.command[0]));
return 0;
}
while true {
ok, output_bytes, error_bytes := read_from_process(*gpu_bench_ui.process, buffer, errors, -1);
if output_bytes append_output(buffer.data, output_bytes);
if error_bytes append_output(errors.data, error_bytes);
if !ok || (gpu_bench_ui.process.output.eof && gpu_bench_ui.process.error.eof) break;
}
_, result := get_process_result(*gpu_bench_ui.process, -1);
deinit(*gpu_bench_ui.process);
finish(cast(s32) result.exit_code);
}
return 0;
}
#import "Process";

3
src/app/viewer.jai
View File

@ -1152,6 +1152,9 @@ viewer_ui :: () { using ImGui;
if Button("Segments") segment_ui.window_open = !segment_ui.window_open; // app/segment_ui.jai if Button("Segments") segment_ui.window_open = !segment_ui.window_open; // app/segment_ui.jai
if IsItemHovered() SetTooltip("Segmentation: what is where in the images of the library folders you choose"); if IsItemHovered() SetTooltip("Segmentation: what is where in the images of the library folders you choose");
SameLine(); SameLine();
if Button("GPU Bench") gpu_bench_ui.window_open = !gpu_bench_ui.window_open; // app/gpu_bench_ui.jai
if IsItemHovered() SetTooltip("Time every GPU model over a fixed corpus, and compare machines");
SameLine();
if Button("Import") importer.window_open = !importer.window_open; // app/import.jai if Button("Import") importer.window_open = !importer.window_open; // app/import.jai
if IsItemHovered() SetTooltip("Bring in a folder of screenshots and photos"); if IsItemHovered() SetTooltip("Bring in a folder of screenshots and photos");
SameLine(); SameLine();

50
tools/gpu_bench/README.md Normal file
View File

@ -0,0 +1,50 @@
# gpu-bench (the `gpu-bench` branch, not for master)
Times every model mcap runs on the GPU over a fixed corpus, so machines can be compared like for
like:
| Task | What runs |
| --- | --- |
| `ocr/ppocr-v5-mobile`, `ocr/ppocr-v5-server` | detection, orientation and recognition, 8 images per call, as a scan does |
| `labels/siglip2-base-patch16-384` | the label model's image encoder |
| `segment/mask2former-swin-base-coco-panoptic` | panoptic segmentation |
| `labels-text/...` (with `-cpu`) | the text encoder, one phrase at a time as a `like:` search runs it (always on the CPU) |
## Keeping it consistent
- **The corpus** (`corpus/`, 30 images: 16 COCO val2017 photos, 2 of them scaled to 12 MP, and 12
synthetic screenshots from 720p to 4K made by `make_corpus.py`) is in git. The build copies it to
`bin/gpu-bench-corpus`. Before anything runs, every file is checked against its SHA-256 in
`corpus.txt`, and a run stops if one differs. The corpus fingerprint, a hash of those name and hash
pairs, goes in the results. `.gitattributes` marks the images binary, so git never changes them.
- **The models:** each task's fingerprint covers the hash of each of its files and its
`models.config` entry (comments and spacing left out). It also says whether every file is the one
the fetch scripts pin. The segmentation `.onnx` is exported on each machine, so it has no pin:
copy it from one machine to the others.
- **The machine:** each result records the GPU, its memory, enforced power limit and clocks, the
driver, the CUDA runtime and cuDNN actually loaded (with their paths), onnxruntime, the CPU, RAM,
and AC or battery.
- **The run:** each model gets one untimed run on the first image, one untimed warm-up pass over
everything, then N timed passes (default 3). The median is reported, with the best and worst
passes. While the timed passes run, NVML samples the GPU every 100 ms: peak memory over the
device's use before the model loaded, utilisation, power, SM clock and temperature.
- **The output:** what the models found (OCR lines and characters, segments, best phrases) and a
digest of it. Different GPUs can round differently, so a different digest is a hint, not an error.
## Running
`jai first.jai - release` builds `bin/gpu-bench.exe` with mcap. Then either:
- **In mcap:** click **GPU Bench** above the grid, choose what to run, and click **Run the benchmark**.
It runs gpu-bench in a separate process with no console and shows its progress. Close mcap and use
the terminal for the cleanest numbers, as mcap's own models stay loaded while it runs.
- **In a terminal:** `bin\gpu-bench.exe [-only ocr,labels,segment] [-models a,b] [-repeats N] [-warmup N] [-cpu | -cpu-only] [-note TEXT]`
Results go to `bin/gpu-bench-results/<machine>_<date>.txt` (plain text, `results.jai`). To compare
machines, copy their files into that folder: the window puts the results side by side. Each result
shows its speed against the baseline (the oldest file, or the one you pick). Its corpus row turns red
when the images differ, and a `*` marks a model whose files or settings differ. Hover a time for its
details.
The server OCR model can take a lot of GPU and system memory and a long time (see the main README's
known issue). Leave it out with `-models ppocr-v5-mobile,siglip2-base-patch16-384,mask2former-swin-base-coco-panoptic`.

35
tools/gpu_bench/corpus.txt Normal file
View File

@ -0,0 +1,35 @@
# The GPU benchmark's corpus: every image in corpus/, with its SHA-256. gpu-bench checks each file
# against this before it runs, and its corpus fingerprint is the SHA-256 of these name and hash
# pairs, so two results are comparable only when the fingerprints match. coco-*: COCO val2017
# (images.cocodataset.org/val2017, CC licences); screen-* and photo-*: make_corpus.py.
ffe0f0cec3b2e27aab1967229cdf0a0d7751dcdd5800322f0b8ac0dffb3b8a8d coco-000000000139.jpg
f3a2974ce3686332609124c70e3e6a2e3aca43fccf1cd1bd7c5c03820977f57d coco-000000000285.jpg
a4cd7f45ac1ce27eaafb254b23af7c0b18a064be08870ceaaf03b2147f2ce550 coco-000000000632.jpg
5c0e559c75d3969c8e3e297b61f61063f78045c9d4802b526ba616361f3823fd coco-000000000724.jpg
1dd31e9059c491992be2f562624eb4093e17aee08b4f7baf5ff9ea24543b0a33 coco-000000000776.jpg
83981537a7baeafbeb9c8cb67b3484dc26433f574b3685d021fa537e277e4726 coco-000000000785.jpg
d5b79e7fa716f85ca86f46e5f518da9b6c5e26414925624d76e6476861aec495 coco-000000000802.jpg
c2aa138ee3a59b057a7ba6fc5a6a18e62af531aa7dab78a7bfd33c1cd7e55eb6 coco-000000000872.jpg
1c67b783f78b18ddb5dd20aaba4d4aacaf98e5277051688a96da096afa9cdf1f coco-000000000885.jpg
24bb77a31928404e45a0454b06f6a0bd54a8db103590c9d7917288a1e0269f05 coco-000000001000.jpg
8e85a2e71ee512fe748a3a3be9bb40b8591e8c501c0f3896efcb5be5999ec236 coco-000000001268.jpg
34a98da7c11bc8811e6eec445145bb16f3d5f4338945ba116edcc1fe554d181b coco-000000001296.jpg
1cccef217a1e5eec2acc74d7c08a105da3ef6fdb8514e221cdea9b0b373b60ea coco-000000001353.jpg
a044804a0b0186a1267acae34975932f14ede96c4a2b78fcee74e6c2fbd8414d coco-000000001490.jpg
974f2af6d81f65627a680ea9f42bdee1067a0ddcdb9f8e7a945bacd21d4dc2ce coco-000000001532.jpg
77131e2f74aa7e374f951a69157cf7f51e00201f81bacfa8a9ab7309b245553f coco-000000001584.jpg
3c34557e033b8022567bf41d1b9f6da6aa3547305b77a922f1f351c197040111 photo-12mp-livingroom.jpg
769a74c85bb0b03322864cffc252738a7858c6947b4a45bf643708a766ea0f73 photo-12mp-tennis.jpg
c989015b2ecf47630acfa1c5afe828e406accc5f0345814e4d5a5cf53cdb4bc7 screen-chat-phone.png
41e445c52f991a659b65e435a5e08173817a53d150da3f7b5380b0a697e50ecc screen-dialog-1080p.png
76224a64b1460169bb0684c3a8eefc977b1cbbd556a170fe7559928b246012ad screen-document-1080p.png
96f00112b373d990162a13e85d524f5acf6a3b4bb671883742127879f3b39246 screen-document-4k.png
2112042a1f22349805af2136a2b9aee346d47387d3d31ca698d0d570e3215c00 screen-document-strip.png
973ba96240163e51c9e92e0bf7d927ca07dab78e2236b1ecf4e902ef4e170187 screen-editor-dark-1080p.png
51f62f081a73d05bbb2dac313ca5e716628961dcf2760c073c6f365090959258 screen-editor-dark-1440p.png
9d763d163ada6d85e3e4d0415295ccffac1d9b5eaa60b6a6209c7af6c6b06d43 screen-editor-light-1080p.png
35bdf92f31eee19599f2f20a17c8cedb536289e3605b9f91f32c1e7dc0a5169a screen-spreadsheet-1080p.png
84b5074a25e1ea13ca17be3476dff40609fe11c0f70ecdcbf6920c59c99272b1 screen-spreadsheet-1440p.png
54f30b7745bdba5890c25c9ae90e417263fa6126ffe41925f98e981920986448 screen-terminal-1080p.png
0f64a278adb72e482f9526af9037ce4fa8ae6826a1fa9b94c8d0d4102bb6e22e screen-terminal-720p.png

BIN
tools/gpu_bench/corpus/coco-000000000139.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 158 KiB

BIN
tools/gpu_bench/corpus/coco-000000000285.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 328 KiB

BIN
tools/gpu_bench/corpus/coco-000000000632.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 152 KiB

BIN
tools/gpu_bench/corpus/coco-000000000724.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 127 KiB

BIN
tools/gpu_bench/corpus/coco-000000000776.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 172 KiB

BIN
tools/gpu_bench/corpus/coco-000000000785.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 130 KiB

BIN
tools/gpu_bench/corpus/coco-000000000802.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 61 KiB

BIN
tools/gpu_bench/corpus/coco-000000000872.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 310 KiB

BIN
tools/gpu_bench/corpus/coco-000000000885.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 109 KiB

BIN
tools/gpu_bench/corpus/coco-000000001000.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 314 KiB

BIN
tools/gpu_bench/corpus/coco-000000001268.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 176 KiB

BIN
tools/gpu_bench/corpus/coco-000000001296.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 200 KiB

BIN
tools/gpu_bench/corpus/coco-000000001353.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 81 KiB

BIN
tools/gpu_bench/corpus/coco-000000001490.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 99 KiB

BIN
tools/gpu_bench/corpus/coco-000000001532.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 170 KiB

BIN
tools/gpu_bench/corpus/coco-000000001584.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 135 KiB

BIN
tools/gpu_bench/corpus/photo-12mp-livingroom.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.1 MiB

BIN
tools/gpu_bench/corpus/photo-12mp-tennis.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.0 MiB

BIN
tools/gpu_bench/corpus/screen-chat-phone.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 135 KiB

BIN
tools/gpu_bench/corpus/screen-dialog-1080p.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 201 KiB

BIN
tools/gpu_bench/corpus/screen-document-1080p.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 179 KiB

BIN
tools/gpu_bench/corpus/screen-document-4k.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 409 KiB

BIN
tools/gpu_bench/corpus/screen-document-strip.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 43 KiB

BIN
tools/gpu_bench/corpus/screen-editor-dark-1080p.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 234 KiB

BIN
tools/gpu_bench/corpus/screen-editor-dark-1440p.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 311 KiB

BIN
tools/gpu_bench/corpus/screen-editor-light-1080p.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 202 KiB

BIN
tools/gpu_bench/corpus/screen-spreadsheet-1080p.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 218 KiB

BIN
tools/gpu_bench/corpus/screen-spreadsheet-1440p.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 341 KiB

BIN
tools/gpu_bench/corpus/screen-terminal-1080p.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 190 KiB

BIN
tools/gpu_bench/corpus/screen-terminal-720p.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 144 KiB

666
tools/gpu_bench/gpu_bench.jai Normal file
View File

@ -0,0 +1,666 @@
// gpu-bench: times every model mcap runs on the GPU (each OCR model, the label model's image encoder,
// segmentation) over a fixed corpus, so machines can be compared. The corpus (gpu-bench-corpus beside
// this program, from tools/gpu_bench/corpus) is checked file by file against corpus.txt's SHA-256s
// before anything runs, and each model's files and models.config entry are fingerprinted, so a
// comparison of two results is known to be like for like. mcap's GPU Benchmark window runs this and
// lays the results side by side.
//
// gpu-bench [-only ocr,labels,segment] [-models a,b] [-repeats N] [-warmup N] [-cpu | -cpu-only]
// [-corpus FOLDER] [-out FOLDER] [-note TEXT] [-progress]
//
// -only the kinds of model to run (default all)
// -models the models to run, by their models.config names (default every one)
// -repeats timed passes over the corpus; the median is reported (default 3)
// -warmup untimed passes first, so every image size has been seen (default 1)
// -cpu the CPU too, after the GPU (slow: minutes for segmentation); also times the label
// model's text encoder, which always runs on the CPU, as a like: search uses it
// -cpu-only the CPU only
// -corpus the corpus folder (default gpu-bench-corpus beside this program)
// -out where the results file goes (default gpu-bench-results beside this program)
// -note a few words kept with the results, such as "plugged in, max performance"
// -progress also print @progress lines, for the GPU Benchmark window
//
// Build: jai first.jai - release (it is built with mcap on the gpu-bench branch).
USAGE :: "Usage: gpu-bench [-only ocr,labels,segment] [-models a,b] [-repeats N] [-warmup N] [-cpu | -cpu-only] [-corpus FOLDER] [-out FOLDER] [-note TEXT] [-progress]\n";
options: struct {
only: [..] string;
models: [..] string;
repeats := 3;
warmup := 1;
gpu := true;
cpu := false;
progress := false;
corpus_folder, out_folder, note: string;
};
main :: () {
runtime_support_headless = true; // a command-line tool: a failure is reported on the console and exits with 1, no message box
args := get_command_line_arguments();
exe_folder := path_strip_filename(get_path_of_running_executable());
options.corpus_folder = sprint("%gpu-bench-corpus", exe_folder);
options.out_folder = sprint("%gpu-bench-results", exe_folder);
i := 1;
while i < args.count {
arg := args[i];
value := ifx i + 1 < args.count then args[i + 1] else "";
if arg == {
case "-only"; for split(value, ",") array_add(*options.only, trim(it)); i += 2;
case "-models"; for split(value, ",") array_add(*options.models, trim(it)); i += 2;
case "-repeats"; options.repeats = max(1, string_to_int(value)); i += 2;
case "-warmup"; options.warmup = max(0, string_to_int(value)); i += 2;
case "-cpu"; options.cpu = true; i += 1;
case "-cpu-only"; options.cpu = true; options.gpu = false; i += 1;
case "-corpus"; options.corpus_folder = value; i += 2;
case "-out"; options.out_folder = value; i += 2;
case "-note"; options.note = value; i += 2;
case "-progress"; options.progress = true; i += 1;
case "-help"; print(USAGE); exit(0);
case; print("Unknown option %.\n%", arg, USAGE); exit(2);
}
}
make_directory_if_it_does_not_exist(options.out_folder, recursive = true);
started_at := current_time_consensus();
// The machine.
info := system_info();
ort_ok, ort_error := ort_load();
print("gpu-bench, mcap % (%, %)\n", VERSION, GIT_HASH, GIT_BRANCH);
print("machine: %, %\n", info.machine, info.os);
print("cpu: % (% threads), % GB of memory\n", info.cpu, info.logical_cores, FF(info.ram_gb, 1));
if info.gpu print("gpu: %, % GB, driver % (CUDA %), power limit % W, max clocks % / % MHz\n", info.gpu, FF(info.vram_gb, 1), info.driver, info.cuda_driver, FF(info.power_limit_w, 0), info.max_sm_clock_mhz, info.max_mem_clock_mhz);
else print("gpu: none (%)\n", info.gpu_error);
if info.power_source print("power: %\n", info.power_source);
if !ort_ok { print("%\n", ort_error); exit(1); }
print("onnxruntime: % (%)\n", ort.version, ort.library_path);
// The corpus: every file as corpus.txt pins it, or nothing runs.
progress("corpus", "-", "checking", 0, 0);
corpus, corpus_error := corpus_load(options.corpus_folder);
if corpus_error { print("\nThe corpus cannot be used: %\n", corpus_error); exit(1); }
print("corpus: % images, % megapixels, fingerprint % (decoded in % s)\n\n", corpus.images.count, FF(corpus.megapixels, 1), short(corpus.fingerprint), FF(corpus.decode_s, 2));
// The models, and what each is made of.
tasks := list_tasks(exe_folder);
if !tasks.count { print("No models to run: none matched -only and -models, or none are beside this program (run jai first.jai).\n"); exit(1); }
hash_cache_load();
for * tasks fingerprint_task(it);
hash_cache_save();
runs: [..] Run;
for * task: tasks {
providers: [..] string;
providers.allocator = temp;
if task.kind == .LABELS_TEXT { if options.cpu array_add(*providers, "CPU"); }
else {
if options.gpu array_add(*providers, "CUDA");
if options.cpu array_add(*providers, "CPU");
}
for provider: providers {
run := run_task(task, provider == "CUDA", *corpus);
array_add(*runs, run);
if !run.ok { print("== % on %: %\n\n", task.name, provider, run.error); continue; }
print_run(task, run);
}
}
cudart, cudnn := cuda_library_versions();
path := write_results(info, started_at, *corpus, tasks, runs, cudart, cudnn);
print("Results: %\n", path);
if options.progress print("@results %\n", path);
}
Task_Kind :: enum { OCR; LABELS; LABELS_TEXT; SEGMENT; }
Task :: struct {
kind: Task_Kind;
model: string; // as models.config names it
name: string; // kind/model: ocr/ppocr-v5-mobile
models_folder: string;
files: [..] string; // the model's files, in models_folder
config: string; // its models.config entry, as compared
fingerprint: string; // of files and config
pinned: string; // "yes" when every file is one a fetch script pins, "no" when one differs, "-" when none are pinned
}
Run :: struct {
task: *Task;
ok: bool;
error: string;
provider: string; // what it really ran on: a GPU run can fall back to the CPU
items: s64; // per pass
unit: string; // image or phrase
megapixels: float64; // per pass
load_s, first_s: float64;
pass_s: [..] float64;
output: string; // what the models found, in words without spaces: lines:123,chars:4567
digest: string; // of what they found, to see whether two machines' models agree
stages: string; // where the time went, per item
batch: s64;
gpu: Gpu_Stats;
explanation: string; // why a GPU run is on the CPU
}
Gpu_Stats :: struct {
sampled: bool;
baseline_mb, peak_mb: float64; // device memory before the engine, and the most during the timed passes
util_sum, power_sum, clock_sum: float64;
temperature_max: float64;
samples: s64;
}
// The corpus, decoded.
Corpus :: struct {
fingerprint: string;
images: [..] Corpus_Image;
megapixels: float64;
decode_s: float64;
}
Corpus_Image :: struct { name: string; pixels: *u8; width, height: s32; }
// Every file corpus.txt names, checked against its hash; the error says what is wrong.
corpus_load :: (folder: string) -> Corpus, error: string {
corpus: Corpus;
manifest_path := tprint("%/corpus.txt", folder);
manifest, ok := read_entire_file(manifest_path, log_errors = false);
if !ok return corpus, tprint("% cannot be read. The build copies it, with the images, from tools/gpu_bench.", manifest_path);
fingerprint: Sha256;
bad: [..] string;
bad.allocator = temp;
start := current_time_monotonic();
for raw: split(manifest, "\n") {
line := trim(raw);
if !line.count || line[0] == #char "#" continue;
space := find_index_from_left(line, #char " ");
if space < 0 continue;
expected := slice(line, 0, space);
name := trim(advance(line, space));
path := tprint("%/%", folder, name);
data, read_ok := read_entire_file(path, log_errors = false);
if !read_ok { array_add(*bad, tprint("% is missing", name)); continue; }
hash := sha256_of_string(data,, temp);
if hash != expected { array_add(*bad, tprint("% is not the file corpus.txt pins (SHA-256 %)", name, hash)); free(data); continue; }
sha256_update(*fingerprint, tprint("% %\n", name, hash));
progress("corpus", "-", "decoding", corpus.images.count + 1, 0);
image, decoded, decode_error := image_decode(cast([] u8) data);
free(data);
if !decoded { array_add(*bad, tprint("% cannot be decoded (%)", name, decode_error)); continue; }
array_add(*corpus.images, .{copy_string(name), image.pixels, image.width, image.height});
corpus.megapixels += cast(float64) image.width * image.height / 1_000_000;
reset_temporary_storage();
}
corpus.decode_s = seconds_since(start);
if bad.count return corpus, join(..bad, "; ");
if !corpus.images.count return corpus, tprint("% names no images.", manifest_path);
corpus.fingerprint = sha256_finish(*fingerprint);
return corpus, "";
}
// The models beside this program, as -only and -models choose.
list_tasks :: (exe_folder: string) -> [..] Task {
tasks: [..] Task;
add :: (tasks: *[..] Task, kind: Task_Kind, model: string, folder: string, files: .. string) {
prefix := ifx kind == .OCR then "ocr" else ifx kind == .LABELS then "labels" else ifx kind == .LABELS_TEXT then "labels-text" else "segment";
if options.only.count && !array_find(options.only, ifx kind == .LABELS_TEXT then "labels" else prefix) return;
if options.models.count && !array_find(options.models, model) return;
task := array_add(tasks);
task.* = .{kind = kind, model = model, name = sprint("%/%", prefix, model), models_folder = folder};
for files if it array_add(*task.files, it);
task.config = config_entry(tprint("%/models.config", folder), model);
}
ocr_folder := sprint("%ocr-models", exe_folder);
ocr_models, ocr_error := ocr_list_models(ocr_folder);
if ocr_error print("OCR: %\n", ocr_error);
for ocr_models add(*tasks, .OCR, it.name, ocr_folder, it.detector, it.recognizer, it.dictionary, it.textline_ori);
labels_folder := sprint("%label-models", exe_folder);
labels_models, labels_error := labels_list_models(labels_folder);
if labels_error print("Labels: %\n", labels_error);
for labels_models add(*tasks, .LABELS, it.name, labels_folder, it.vision, it.text, it.tokenizer);
for labels_models add(*tasks, .LABELS_TEXT, it.name, labels_folder, it.vision, it.text, it.tokenizer);
segment_folder := sprint("%segment-models", exe_folder);
segment_models, segment_error := segment_list_models(segment_folder);
if segment_error print("Segmentation: %\n", segment_error);
for segment_models add(*tasks, .SEGMENT, it.name, segment_folder, it.model, it.classes);
return tasks;
}
// A model's [model NAME] entry, comments and spacing taken out, so only a change that matters
// changes the fingerprint.
config_entry :: (path: string, model: string) -> string {
text, ok := read_entire_file(path, log_errors = false,, temp);
if !ok return "";
builder: String_Builder;
inside := false;
for raw: split(text, "\n",, temp) {
line := raw;
hash := find_index_from_left(line, #char "#");
if hash >= 0 line = slice(line, 0, hash);
line = trim(line);
if !line.count continue;
if begins_with(line, "[model ") { inside = line == tprint("[model %]", model); continue; }
if !inside continue;
colon := find_index_from_left(line, #char ":");
if colon >= 0 line = tprint("%: %", trim(slice(line, 0, colon)), trim(advance(line, colon + 1)));
print_to_builder(*builder, "%\n", line);
}
return builder_to_string(*builder);
}
// The files the fetch scripts pin (modules/OCR/fetch_models.jai, modules/Labels/fetch_models.jai).
// Segmentation's model is made on each machine by tools/export_mask2former.py, so it has no pin:
// copy the .onnx from one machine to the others to be sure they run the same one.
Pin :: struct { name: string; sha256: string; }
PINNED :: Pin.[
.{"ch_PP-OCRv5_det_mobile.onnx", "4d97c44a20d30a81aad087d6a396b08f786c4635742afc391f6621f5c6ae78ae"},
.{"ch_PP-OCRv5_rec_mobile.onnx", "5825fc7ebf84ae7a412be049820b4d86d77620f204a041697b0494669b1742c5"},
.{"ppocrv5_dict.txt", "d1979e9f794c464c0d2e0b70a7fe14dd978e9dc644c0e71f14158cdf8342af1b"},
.{"ch_PP-LCNet_x0_25_textline_ori_cls_mobile.onnx", "54379ae5174d026780215fc748a7f31910dee36818e63d49e17dc598ecc82df7"},
.{"ch_PP-OCRv5_det_server.onnx", "0f8846b1d4bba223a2a2f9d9b44022fbc22cc019051a602b41a7fda9667e4cad"},
.{"ch_PP-OCRv5_rec_server.onnx", "e09385400eaaaef34ceff54aeb7c4f0f1fe014c27fa8b9905d4709b65746562a"},
.{"siglip2-base-patch16-384-vision.onnx", "49ae4958b1098ca995e929d646f7be05a69c65e6344beae07d58c6598ffc5210"},
.{"siglip2-base-patch16-384-text-int8.onnx", "30a94b43b9ddc39c04ab67f80cba4b808262682b2fa1a2a8b924bf1e7bd84c61"},
.{"siglip2-tokenizer.json", "cb9140fae3ac5122c972d37adf83e1248471a38147ad76f8215c8872c6fd8322"},
];
fingerprint_task :: (task: *Task) {
h: Sha256;
pinned, differs := 0, 0;
for task.files {
progress("hashing", task.name, it, it_index + 1, task.files.count);
hash := hash_file_cached(tprint("%/%", task.models_folder, it));
if !hash hash = "missing";
sha256_update(*h, tprint("% %\n", it, hash));
for pin: PINNED if pin.name == it { if pin.sha256 == hash pinned += 1; else differs += 1; }
}
sha256_update(*h, task.config);
task.fingerprint = sha256_finish(*h);
task.pinned = ifx differs then "no" else ifx pinned then "yes" else "-";
}
run_task :: (task: *Task, prefer_gpu: bool, corpus: *Corpus) -> Run {
run: Run;
run.task = task;
run.provider = ifx prefer_gpu then "CUDA" else "CPU";
run.unit = ifx task.kind == .LABELS_TEXT then "phrase" else "image";
images := corpus.images;
run.items = ifx task.kind == .LABELS_TEXT then PHRASES.count else images.count;
if task.kind != .LABELS_TEXT run.megapixels = corpus.megapixels;
sampler_start(*run.gpu); // the baseline: the device's memory before the engine
sampler_pause();
// The engine.
progress(task.name, run.provider, "loading", 0, 0);
load_start := current_time_monotonic();
engine: struct { ocr: *Ocr_Engine; labels: *Labels_Engine; segment: *Segment_Engine; };
ok: bool;
error: string;
phrase_vectors: [] float;
if task.kind == {
case .OCR;
engine.ocr, ok, error = ocr_create(task.models_folder, task.model, prefer_gpu);
if ok {
ocr_gpu, ocr_why := ocr_gpu_status(engine.ocr);
if prefer_gpu && !ocr_gpu run.explanation = copy_string(ocr_why);
run.provider = sprint("%", ocr_provider(engine.ocr));
run.batch = engine.ocr.rec_batch;
}
case .LABELS; #through;
case .LABELS_TEXT;
engine.labels, ok, error = labels_create(task.models_folder, task.model, prefer_gpu && task.kind == .LABELS);
if ok {
labels_gpu, labels_why := labels_gpu_status(engine.labels);
if prefer_gpu && !labels_gpu run.explanation = copy_string(labels_why);
run.provider = ifx task.kind == .LABELS_TEXT then "CPU" else sprint("%", engine.labels.provider);
run.batch = ifx task.kind == .LABELS_TEXT then 1 else engine.labels.model.image_batch;
phrases := PHRASES;
phrase_vectors = labels_embed_texts(engine.labels, phrases);
}
case .SEGMENT;
engine.segment, ok, error = segment_create(task.models_folder, task.model, prefer_gpu);
if ok {
if prefer_gpu && engine.segment.provider != .CUDA run.explanation = copy_string(engine.segment.gpu_explanation);
run.provider = sprint("%", engine.segment.provider);
run.batch = ifx engine.segment.provider == .CUDA then engine.segment.model.batch_gpu else engine.segment.model.batch_cpu;
}
}
run.load_s = seconds_since(load_start);
if !ok { sampler_stop(); run.error = copy_string(error); return run; }
defer {
ocr_destroy(engine.ocr);
labels_destroy(engine.labels);
segment_destroy(engine.segment);
array_free(phrase_vectors);
}
if prefer_gpu && run.provider != "CUDA" {
// Said, and not timed: the CPU run, when asked for, is the same thing.
sampler_stop();
run.error = tprint("not on the GPU: %", run.explanation,, context.allocator);
return run;
}
// One pass over the corpus, or over its first image alone; what it found, and its digest.
pass :: (first_only: bool) -> output: string, digest: string #expand {
h: Sha256;
builder: String_Builder;
builder.allocator = temp;
count := ifx first_only then 1 else `run.items;
if `task.kind == {
case .OCR;
lines, characters: s64;
done := 0;
while done < count {
n := min(OCR_BATCH, count - done);
batch := NewArray(n, Ocr_Image,, temp);
for 0..n - 1 { image := `images[done + it]; batch[it] = .{image.pixels, image.width, image.height, image.width * 4}; }
results := ocr_process(`engine.ocr, batch,, temp);
for results {
if it.failed { sha256_update(*h, "failed\n"); continue; }
lines += it.lines.count;
for line: it.lines { characters += line.text.count; sha256_update(*h, line.text); sha256_update(*h, "\n"); }
}
done += n;
}
print_to_builder(*builder, "lines:%,chars:%", lines, characters);
case .LABELS;
input := NewArray(count, Labels_Image,, temp);
for 0..count - 1 { image := `images[it]; input[it] = .{image.pixels, image.width, image.height, image.width * 4}; }
vectors, error := labels_embed_images(`engine.labels, input,, temp);
if error sha256_update(*h, "failed\n");
// Each image's best phrase: what the label list would make of it.
dims := `engine.labels.dimensions;
best_count: [PHRASES.count] s64;
for image: 0..count - 1 {
best, best_score := -1, -1.0;
for phrase: 0..PHRASES.count - 1 {
score := labels_score(`engine.labels, vectors.data + image * dims, `phrase_vectors.data + phrase * dims);
if score > best_score { best, best_score = phrase, score; }
}
sha256_update(*h, tprint("%\n", best));
best_count[best] += 1;
}
distinct := 0;
for best_count if it distinct += 1;
print_to_builder(*builder, "best_phrases:%", distinct);
case .LABELS_TEXT;
for 0..count - 1 {
one: [1] string;
one[0] = PHRASES[it];
vectors, error := labels_embed_texts(`engine.labels, one,, temp); // one at a time, as a search does
sha256_update(*h, ifx error then "failed\n" else "ok\n");
}
print_to_builder(*builder, "phrases:%", count);
case .SEGMENT;
input := NewArray(count, Segment_Image,, temp);
for 0..count - 1 { image := `images[it]; input[it] = .{image.pixels, image.width, image.height, image.width * 4}; }
results := segment_images(`engine.segment, input,, temp);
segments: s64;
for results {
if it.failed { sha256_update(*h, "failed\n"); continue; }
segments += it.segments.count;
for segment: it.segments sha256_update(*h, tprint("% ", segment.class));
sha256_update(*h, "\n");
}
print_to_builder(*builder, "segments:%", segments);
}
return builder_to_string(*builder,, temp), sha256_finish(*h,, temp);
}
// The first image alone: the first run pays for choosing kernels and setting up memory.
progress(task.name, run.provider, "first", 0, 0);
first_start := current_time_monotonic();
pass(true);
run.first_s = seconds_since(first_start);
reset_temporary_storage();
for 1..options.warmup {
progress(task.name, run.provider, "warm-up", it, options.warmup);
pass(false);
reset_temporary_storage();
}
if task.kind == .OCR ocr_reset_stats(engine.ocr);
if task.kind == .SEGMENT engine.segment.stats = .{};
sampler_resume();
for 1..options.repeats {
progress(task.name, run.provider, "timing", it, options.repeats);
start := current_time_monotonic();
output, digest := pass(false);
array_add(*run.pass_s, seconds_since(start));
if it == 1 { run.output = copy_string(output); run.digest = copy_string(short(digest)); }
reset_temporary_storage();
}
sampler_stop();
per := cast(float64) max(run.items * options.repeats, 1) * 1000;
if task.kind == .OCR {
s := ocr_stats(engine.ocr);
run.stages = sprint("detect:%,orient:%,recognize:%", FF(s.detect_us / per, 2), FF(s.orientation_us / per, 2), FF(s.recognize_us / per, 2));
} else if task.kind == .SEGMENT {
s := engine.segment.stats;
run.stages = sprint("prepare:%,model:%,post:%", FF(s.prepare_us / per, 2), FF(s.model_us / per, 2), FF(s.post_us / per, 2));
}
run.ok = true;
return run;
}
OCR_BATCH :: 8; // images per ocr_process call, as the app's scan gives them
// Phrases for the label model: its image vectors are scored against them (the output's digest), and
// the text encoder is timed on them, one at a time, as a like: search runs it.
PHRASES :: string.[
"a screenshot of a code editor showing source code", "a screenshot of a terminal window",
"a spreadsheet with rows and columns of numbers", "a document with paragraphs of text",
"a chat conversation on a phone", "a dialog box asking to save changes",
"a photo of a living room", "a photo of people playing tennis",
"a photo of food on a plate", "a photo of a street with cars",
"a photo of an animal", "a photo of a person", "an electronic circuit schematic diagram",
"a printed circuit board", "a graph with several colored lines", "a web page in a browser",
];
Median :: struct { median, best, worst: float64; }
median_of :: (values: [] float64) -> Median {
sorted := array_copy(values,, temp);
quick_sort(sorted, (a: float64, b: float64) -> s64 { return ifx a < b then -1 else ifx a > b then 1 else 0; });
m: Median;
if !sorted.count return m;
m.median = ifx sorted.count % 2 then sorted[sorted.count / 2] else (sorted[sorted.count / 2 - 1] + sorted[sorted.count / 2]) / 2;
m.best, m.worst = sorted[0], sorted[sorted.count - 1];
return m;
}
print_run :: (task: *Task, run: Run) {
m := median_of(run.pass_s);
print("== % on % (batch %)\n", task.name, run.provider, run.batch);
print(" load % s, first % % s\n", FF(run.load_s, 2), run.unit, FF(run.first_s, 3));
print(" % ms per % (median of %; best %, worst %), % %s/s", FF(1000 * m.median / run.items, 2), run.unit, run.pass_s.count,
FF(1000 * m.best / run.items, 2), FF(1000 * m.worst / run.items, 2), FF(run.items / m.median, 1), run.unit);
if run.megapixels print(", % megapixels/s", FF(run.megapixels / m.median, 1));
print("\n");
if run.stages print(" per %, summed over threads (ms): %\n", run.unit, run.stages);
if run.gpu.samples print(" GPU: % MB more memory at the peak, % percent busy, % W, % MHz, up to % C\n",
FF(run.gpu.peak_mb - run.gpu.baseline_mb, 0), FF(run.gpu.util_sum / run.gpu.samples, 0),
FF(run.gpu.power_sum / run.gpu.samples, 0), FF(run.gpu.clock_sum / run.gpu.samples, 0), FF(run.gpu.temperature_max, 0));
print(" found % (digest %)\n\n", run.output, run.digest);
}
write_results :: (info: System_Info, started_at: Apollo_Time, corpus: *Corpus, tasks: [] Task, runs: [] Run, cudart: string, cudnn: string) -> path: string {
b: String_Builder;
print_to_builder(*b, "%\n", BENCH_RESULTS_FORMAT);
date := to_calendar(started_at, .LOCAL);
print_to_builder(*b, "date: %-%-% %:%:%\n", date.year, pad2(date.month_starting_at_0 + 1), pad2(date.day_of_month_starting_at_0 + 1), pad2(date.hour), pad2(date.minute), pad2(date.second));
print_to_builder(*b, "machine: %\n", one_line(info.machine));
if options.note print_to_builder(*b, "note: %\n", one_line(options.note));
print_to_builder(*b, "os: %\n", info.os);
print_to_builder(*b, "cpu: % (% threads)\n", one_line(info.cpu), info.logical_cores);
print_to_builder(*b, "ram_gb: %\n", FF(info.ram_gb, 1));
print_to_builder(*b, "gpu: %\n", ifx info.gpu then one_line(info.gpu) else tprint("none (%)", info.gpu_error));
if info.gpu {
print_to_builder(*b, "vram_gb: %\n", FF(info.vram_gb, 1));
print_to_builder(*b, "driver: % (CUDA %)\n", info.driver, info.cuda_driver);
print_to_builder(*b, "power_limit_w: %\n", FF(info.power_limit_w, 0));
print_to_builder(*b, "max_clocks_mhz: % / %\n", info.max_sm_clock_mhz, info.max_mem_clock_mhz);
}
if info.power_source print_to_builder(*b, "power: %\n", info.power_source);
print_to_builder(*b, "onnxruntime: %\n", ort.version);
if cudart print_to_builder(*b, "cudart: %\n", cudart);
if cudnn print_to_builder(*b, "cudnn: %\n", cudnn);
print_to_builder(*b, "mcap: % (%, %)\n", VERSION, GIT_HASH, GIT_BRANCH);
print_to_builder(*b, "passes: % timed, % warm-up\n", options.repeats, options.warmup);
print_to_builder(*b, "corpus: %\n", corpus.fingerprint);
print_to_builder(*b, "corpus_size: % images, % megapixels\n", corpus.images.count, FF(corpus.megapixels, 1));
for tasks {
print_to_builder(*b, "model: % %\n", it.name, it.fingerprint);
print_to_builder(*b, "pinned %: %\n", it.name, it.pinned);
}
for run: runs {
task := run.task;
if !run.ok { print_to_builder(*b, "error % %: %\n", task.name, run.provider, one_line(run.error)); continue; }
m := median_of(run.pass_s);
print_to_builder(*b, "result: % % unit=% items=% batch=% ms=% best_ms=% worst_ms=% per_s=% load_s=% first_s=%",
task.name, run.provider, run.unit, run.items, run.batch, FF(1000 * m.median / run.items, 3), FF(1000 * m.best / run.items, 3),
FF(1000 * m.worst / run.items, 3), FF(run.items / m.median, 2), FF(run.load_s, 2), FF(run.first_s, 3));
if run.megapixels print_to_builder(*b, " mp_per_s=%", FF(run.megapixels / m.median, 2));
if run.gpu.samples {
g := run.gpu;
print_to_builder(*b, " vram_mb=% gpu_util=% power_w=% sm_mhz=% temp_c=%", FF(g.peak_mb - g.baseline_mb, 0), FF(g.util_sum / g.samples, 0),
FF(g.power_sum / g.samples, 0), FF(g.clock_sum / g.samples, 0), FF(g.temperature_max, 0));
}
if run.stages print_to_builder(*b, " stages=%", run.stages);
print_to_builder(*b, " output=% digest=%\n", run.output, run.digest);
}
name := tprint("%_%-%-%_%-%-%.txt", file_safe(info.machine), date.year, pad2(date.month_starting_at_0 + 1), pad2(date.day_of_month_starting_at_0 + 1),
pad2(date.hour), pad2(date.minute), pad2(date.second));
path := tprint("%/%", options.out_folder, name);
if !write_entire_file(path, *b) print("Could not write %.\n", path);
return path;
}
// For the GPU Benchmark window: where the run is. total 0: no count.
progress :: (task: string, provider: string, phase: string, done: s64, total: s64) {
if !options.progress return;
print("@progress % % % % %\n", task, provider, phase, done, total);
}
// The GPU, sampled every SAMPLE_MS on a thread of its own while the timed passes run.
SAMPLE_MS :: 100;
sampler: struct {
thread: Thread;
stats: *Gpu_Stats;
running: bool; // the thread is up
paused: bool; // it is up, and not sampling
mutex: Mutex;
};
sampler_start :: (stats: *Gpu_Stats) {
s, ok := gpu_sample();
if !ok return;
stats.sampled = true;
stats.baseline_mb = s.memory_used_mb;
sampler.stats = stats;
init(*sampler.mutex);
sampler.running = true;
sampler.paused = false;
thread_init(*sampler.thread, (thread: *Thread) -> s64 {
while sampler.running {
sleep_milliseconds(SAMPLE_MS);
s, ok := gpu_sample();
lock(*sampler.mutex);
if ok && !sampler.paused && sampler.running {
g := sampler.stats;
g.peak_mb = max(g.peak_mb, s.memory_used_mb);
g.util_sum += s.util_percent;
g.power_sum += s.power_w;
g.clock_sum += s.sm_clock_mhz;
g.temperature_max = max(g.temperature_max, s.temperature_c);
g.samples += 1;
}
unlock(*sampler.mutex);
}
return 0;
});
thread_start(*sampler.thread);
}
sampler_pause :: () { if !sampler.running return; lock(*sampler.mutex); sampler.paused = true; unlock(*sampler.mutex); }
sampler_resume :: () { if !sampler.running return; lock(*sampler.mutex); sampler.paused = false; unlock(*sampler.mutex); }
sampler_stop :: () {
if !sampler.running return;
lock(*sampler.mutex);
sampler.running = false;
unlock(*sampler.mutex);
thread_is_done(*sampler.thread, -1);
thread_deinit(*sampler.thread);
destroy(*sampler.mutex);
}
// Model files are hundreds of MB, so their hashes are kept in the results folder, by path, size and
// time: hashed again only when one changes.
hash_cache: [..] struct { key: string; hash: string; };
hash_cache_load :: () {
text, ok := read_entire_file(tprint("%/.hash-cache", options.out_folder), log_errors = false);
if !ok return;
for split(text, "\n") {
tab := find_index_from_left(it, #char "\t");
if tab > 0 array_add(*hash_cache, .{trim(advance(it, tab + 1)), slice(it, 0, tab)});
}
}
hash_cache_save :: () {
b: String_Builder;
for hash_cache print_to_builder(*b, "%\t%\n", it.hash, it.key);
write_entire_file(tprint("%/.hash-cache", options.out_folder), *b);
}
hash_file_cached :: (path: string) -> string {
modtime, size, ok := file_modtime_and_size(path);
if !ok return "";
key := sprint("% % % %", size, modtime.low, modtime.high, path);
for hash_cache if it.key == key return it.hash;
hash := sha256_of_file(path);
if hash array_add(*hash_cache, .{key, hash});
return hash;
}
seconds_since :: (start: Apollo_Time) -> float64 {
return to_float64_seconds(current_time_monotonic() - start);
}
short :: (hash: string) -> string { return slice(hash, 0, min(hash.count, 12)); }
pad2 :: (n: s64) -> string { return tprint("%", formatInt(n, minimum_digits = 2)); }
one_line :: (text: string) -> string {
return replace(replace(trim(text), "\r", " ",, temp), "\n", " ",, temp);
}
file_safe :: (text: string) -> string {
result := copy_string(text,, temp);
for 0..result.count - 1 {
c := result[it];
if !(is_alnum(c) || c == #char "-") result[it] = #char "_";
}
return result;
}
#load "sha256.jai";
#load "system_info.jai";
#load "results.jai";
#import "Basic";
#import "String";
#import "File";
#import "File_Utilities";
#import "Sort";
#import "System";
#import "Thread";
#import "Image_Decode";
#import "onnxruntime";
#import "OCR";
#import "Labels";
#import "Segment";

192
tools/gpu_bench/make_corpus.py Normal file
View File

@ -0,0 +1,192 @@
# Makes the synthetic part of the GPU benchmark's corpus (tools/gpu_bench/corpus): screenshot-like
# images full of text, for OCR, and two phone-sized photos scaled up from COCO ones. The images are
# committed, and corpus.txt pins each one's SHA-256, so every machine benchmarks the same pixels; this
# script is kept to show how they were made, not to be rerun (another Pillow or FreeType could draw
# them a pixel differently, and then they would no longer match corpus.txt).
#
# python tools/gpu_bench/make_corpus.py
import os, random
from PIL import Image, ImageDraw, ImageFont
HERE = os.path.dirname(os.path.abspath(__file__))
OUT = os.path.join(HERE, "corpus")
FONTS = os.path.join(HERE, "..", "..", "assets", "fonts")
WORDS = ("the of and to in is that for it as with was on be by this are from or at have an not but "
"which they you were all one their there been has more when will would can if about what "
"into some time only other new could these two may first then its such like over also back "
"after use our work way well even because any most people where much through before right "
"capture library image model search folder window screen label segment vector memory driver "
"thread allocator buffer request response latency throughput benchmark kernel tensor batch "
"session provider runtime device budget scheduler compiler release version settings export "
"import config circuit board resistor capacitor footprint schematic voltage current signal "
"amplifier channel sample filter frequency spectrum electrode impedance firmware bootloader").split()
def font(name, size):
return ImageFont.truetype(os.path.join(FONTS, name), size)
def sentence(rng, n):
words = [rng.choice(WORDS) for _ in range(n)]
words[0] = words[0].capitalize()
return " ".join(words) + rng.choice([".", ".", ",", ":", "?"])
def code_line(rng):
kind = rng.randrange(6)
a, b, c = rng.choice(WORDS), rng.choice(WORDS), rng.choice(WORDS)
if kind == 0: return f"{a}_{b} := {c}_{rng.choice(WORDS)}({rng.randrange(1000)}, {a});"
if kind == 1: return f"if {a}.{b} > {rng.randrange(64)} {{ {c}({a}); return false; }}"
if kind == 2: return f"// {sentence(rng, rng.randrange(4, 12))}"
if kind == 3: return f"for {a}: {b}_{c} array_add(*{a}s, it.{c});"
if kind == 4: return f"{a}_{b} :: (x: s64, {c}: *{a.capitalize()}) -> bool {{"
return f"print(\"% {a} {b}\\n\", {c}.count);"
def editor(rng, w, h, dark, mono, size):
bg, fg, dim, side = ((30, 30, 30), (212, 212, 212), (110, 110, 110), (37, 37, 38)) if dark else \
((255, 255, 255), (30, 30, 30), (150, 150, 150), (243, 243, 243))
im = Image.new("RGB", (w, h), bg)
d = ImageDraw.Draw(im)
ui, code = font("OpenSans-Regular.ttf", 14), font(mono, size)
d.rectangle([0, 0, w, 32], fill=side)
for i, x in enumerate(range(10, w - 200, 180)):
d.text((x, 8), f"{rng.choice(WORDS)}_{rng.choice(WORDS)}.jai", font=ui, fill=fg if i == 0 else dim)
d.rectangle([0, 32, 260, h], fill=side)
y = 44
while y < h - 30:
d.text((16 + 12 * rng.randrange(3), y), f"{rng.choice(WORDS)}_{rng.choice(WORDS)}", font=ui, fill=fg)
y += 24
colours = [fg, (86, 156, 214), (206, 145, 120), (106, 153, 85), (197, 134, 192)] if dark else \
[fg, (0, 0, 255), (163, 21, 21), (0, 128, 0), (175, 0, 219)]
line, y = rng.randrange(1, 900), 44
while y < h - 30:
d.text((272, y), str(line).rjust(4), font=code, fill=dim)
if rng.random() > 0.12:
d.text((330 + 4 * size // 2 * rng.randrange(4), y), code_line(rng), font=code, fill=rng.choice(colours))
y += int(size * 1.5); line += 1
d.rectangle([0, h - 24, w, h], fill=(0, 122, 204))
d.text((10, h - 21), f"Ln {line}, Col {rng.randrange(80)} UTF-8 CRLF Jai {rng.choice(WORDS)}", font=ui, fill=(255, 255, 255))
return im
def document(rng, w, h, size):
im = Image.new("RGB", (w, h), (250, 250, 248))
d = ImageDraw.Draw(im)
title, body = font("OpenSans-Bold.ttf", size * 2), font("OpenSans-Regular.ttf", size)
margin = w // 8
d.text((margin, 40), sentence(rng, 5)[:-1], font=title, fill=(20, 20, 20))
y = 60 + size * 3
while y < h - size * 2:
if rng.random() < 0.15:
d.text((margin, y), sentence(rng, 4)[:-1], font=font("OpenSans-SemiBold.ttf", int(size * 1.3)), fill=(40, 40, 90))
y += int(size * 2.2); continue
for _ in range(rng.randrange(2, 7)):
text = sentence(rng, 40)
words, current = text.split(), ""
for word in words:
trial = (current + " " + word).strip()
if d.textlength(trial, font=body) > w - 2 * margin:
d.text((margin, y), current, font=body, fill=(30, 30, 30)); y += int(size * 1.6); current = word
else:
current = trial
d.text((margin, y), current, font=body, fill=(30, 30, 30)); y += int(size * 1.6)
if y > h - size * 2: break
y += size
return im
def spreadsheet(rng, w, h):
im = Image.new("RGB", (w, h), (255, 255, 255))
d = ImageDraw.Draw(im)
f, bold = font("OpenSans-Regular.ttf", 13), font("OpenSans-SemiBold.ttf", 13)
cols = [40] + [rng.choice([90, 110, 140, 180]) for _ in range(30)]
d.rectangle([0, 0, w, 60], fill=(33, 115, 70))
d.text((12, 18), f"{rng.choice(WORDS)}_{rng.choice(WORDS)}.xlsx", font=font("OpenSans-SemiBold.ttf", 18), fill=(255, 255, 255))
y, row = 60, 0
while y < h:
x = 0
for c, cw in enumerate(cols):
if x > w: break
d.rectangle([x, y, x + cw, y + 22], outline=(218, 220, 224), fill=(243, 243, 243) if row == 0 or c == 0 else None)
if row == 0 and c > 0: text, ft = chr(64 + c) if c <= 26 else "A" + chr(38 + c), bold
elif c == 0: text, ft = str(row) if row else "", bold
elif row == 1: text, ft = rng.choice(WORDS).capitalize(), bold
else:
r = rng.random()
text = f"{rng.uniform(0, 99999):,.2f}" if r < 0.5 else (str(rng.randrange(10000)) if r < 0.7 else rng.choice(WORDS))
ft = f
d.text((x + 4, y + 3), text, font=ft, fill=(20, 20, 20))
x += cw
y += 22; row += 1
return im
def terminal(rng, w, h, size):
im = Image.new("RGB", (w, h), (12, 12, 12))
d = ImageDraw.Draw(im)
f = font("FiraCode-Retina.ttf", size)
y = 8
while y < h - size:
r = rng.random()
if r < 0.2: text, colour = f"PS D:\\{rng.choice(WORDS)}\\{rng.choice(WORDS)}> jai first.jai - {rng.choice(WORDS)}", (240, 240, 240)
elif r < 0.3: text, colour = f"error: {sentence(rng, 8)}", (240, 80, 80)
elif r < 0.4: text, colour = f"[{rng.randrange(24):02}:{rng.randrange(60):02}:{rng.randrange(60):02}] {rng.choice(['INFO', 'WARN', 'DEBUG'])} {sentence(rng, 9)}", (180, 220, 120)
else: text, colour = " ".join(f"{rng.randrange(1 << 16):04x}" for _ in range(rng.randrange(4, 18))), (200, 200, 200)
d.text((8, y), text, font=f, fill=colour)
y += int(size * 1.35)
return im
def chat(rng, w, h):
im = Image.new("RGB", (w, h), (245, 245, 250))
d = ImageDraw.Draw(im)
f, name = font("OpenSans-Regular.ttf", 30), font("OpenSans-Bold.ttf", 26)
d.rectangle([0, 0, w, 140], fill=(60, 60, 140))
d.text((40, 60), rng.choice(WORDS).capitalize() + " " + rng.choice(WORDS).capitalize(), font=font("OpenSans-SemiBold.ttf", 40), fill=(255, 255, 255))
y = 170
while y < h - 200:
mine = rng.random() < 0.5
lines = [sentence(rng, rng.randrange(2, 6)) for _ in range(rng.randrange(1, 4))]
width = max(d.textlength(l, font=f) for l in lines) + 40
height = 44 * len(lines) + 56
x = w - width - 40 if mine else 40
d.rounded_rectangle([x, y, x + width, y + height], 24, fill=(90, 120, 230) if mine else (255, 255, 255))
d.text((x + 20, y + 10), f"{rng.randrange(1, 13)}:{rng.randrange(60):02} {'PM' if rng.random() < 0.5 else 'AM'}", font=name, fill=(230, 230, 255) if mine else (120, 120, 120))
for i, l in enumerate(lines):
d.text((x + 20, y + 48 + 44 * i), l, font=f, fill=(255, 255, 255) if mine else (20, 20, 20))
y += height + 24
return im
def dialog(rng, w, h):
im = editor(rng, w, h, True, "JetBrainsMono-Regular.ttf", 14).point(lambda v: v // 2)
d = ImageDraw.Draw(im)
x0, y0, x1, y1 = w // 2 - 320, h // 2 - 160, w // 2 + 320, h // 2 + 160
d.rectangle([x0, y0, x1, y1], fill=(250, 250, 250), outline=(120, 120, 120))
d.text((x0 + 24, y0 + 20), sentence(rng, 4)[:-1], font=font("OpenSans-SemiBold.ttf", 22), fill=(20, 20, 20))
for i in range(4):
d.text((x0 + 24, y0 + 70 + 30 * i), sentence(rng, 9), font=font("OpenSans-Regular.ttf", 17), fill=(40, 40, 40))
for i, label in enumerate(["Cancel", "Don't save", "Save"]):
bx = x1 - 130 * (3 - i) - 10
d.rectangle([bx, y1 - 60, bx + 115, y1 - 22], fill=(0, 103, 192) if label == "Save" else (225, 225, 225))
d.text((bx + 20, y1 - 52), label, font=font("OpenSans-Regular.ttf", 17), fill=(255, 255, 255) if label == "Save" else (20, 20, 20))
return im
def main():
rng = random.Random(20261017)
made = [
("screen-editor-dark-1080p.png", editor(rng, 1920, 1080, True, "JetBrainsMono-Regular.ttf", 15)),
("screen-editor-light-1080p.png", editor(rng, 1920, 1080, False, "RobotoMono-Regular.ttf", 14)),
("screen-editor-dark-1440p.png", editor(rng, 2560, 1440, True, "FiraCode-Retina.ttf", 16)),
("screen-document-1080p.png", document(rng, 1920, 1080, 18)),
("screen-document-4k.png", document(rng, 3840, 2160, 28)),
("screen-spreadsheet-1080p.png", spreadsheet(rng, 1920, 1080)),
("screen-spreadsheet-1440p.png", spreadsheet(rng, 2560, 1440)),
("screen-terminal-1080p.png", terminal(rng, 1920, 1080, 15)),
("screen-terminal-720p.png", terminal(rng, 1280, 720, 13)),
("screen-chat-phone.png", chat(rng, 1080, 2400)),
("screen-dialog-1080p.png", dialog(rng, 1920, 1080)),
("screen-document-strip.png", document(rng, 1600, 300, 16)),
]
for name, im in made:
im.save(os.path.join(OUT, name), optimize=False, compress_level=6)
# Phone-sized photos: the decoders and resizers see a 12-megapixel image, as from a phone.
for source, name in [("coco-000000000139.jpg", "photo-12mp-livingroom.jpg"), ("coco-000000001000.jpg", "photo-12mp-tennis.jpg")]:
im = Image.open(os.path.join(OUT, source)).convert("RGB").resize((4032, 3024), Image.LANCZOS)
im.save(os.path.join(OUT, name), quality=92)
main()

91
tools/gpu_bench/results.jai Normal file
View File

@ -0,0 +1,91 @@
// gpu-bench's results file: what gpu-bench writes, and what mcap's GPU Benchmark window reads to put
// several machines side by side. Plain text, one fact a line:
//
// gpu-bench 1
// machine: NAME a header line: key, colon, value
// ...
// model: ocr/ppocr-v5-mobile 1a2b... the fingerprint of the files a task's model is made of
// result: ocr/ppocr-v5-mobile CUDA ms_per_image=12.3 images_per_s=81.2 ...
//
// Two results are comparable only when their corpus line (the images) and each task's model line
// match; the window marks them where they do not.
BENCH_RESULTS_FORMAT :: "gpu-bench 1";
Bench_Pair :: struct { key, value: string; }
Bench_Row :: struct {
task: string; // ocr/ppocr-v5-mobile, labels/siglip2-base-patch16-384, segment/...
provider: string; // CUDA or CPU
fields: [..] Bench_Pair;
}
Bench_Report :: struct {
path: string;
header: [..] Bench_Pair; // in the file's order; model lines too, as "model ocr/x" -> hash
rows: [..] Bench_Row;
}
bench_header :: (report: *Bench_Report, key: string) -> string {
for report.header if it.key == key return it.value;
return "";
}
bench_field :: (row: *Bench_Row, key: string) -> string {
for row.fields if it.key == key return it.value;
return "";
}
bench_row :: (report: *Bench_Report, task: string, provider: string) -> *Bench_Row {
for * report.rows if it.task == task && it.provider == provider return it;
return null;
}
// A results file, or ok false and why; from the context allocator.
bench_parse :: (text: string, path := "") -> Bench_Report, ok: bool, error: string {
report: Bench_Report;
report.path = copy_string(path);
lines := split(text, "\n",, temp);
if !lines.count || trim(lines[0]) != BENCH_RESULTS_FORMAT return report, false, tprint("% is not a gpu-bench results file (no \"%\" on its first line).", path, BENCH_RESULTS_FORMAT);
for raw: lines {
if it_index == 0 continue;
line := trim(raw);
if !line.count || line[0] == #char "#" continue;
colon := find_index_from_left(line, #char ":");
if colon < 0 continue;
key := trim(slice(line, 0, colon));
value := trim(advance(line, colon + 1));
if key == "result" {
parts := split(value, " ",, temp);
if parts.count < 2 continue;
row := array_add(*report.rows);
row.* = .{task = copy_string(parts[0]), provider = copy_string(parts[1])};
for 2..parts.count - 1 {
part := parts[it];
equals := find_index_from_left(part, #char "=");
if equals < 0 continue;
array_add(*row.fields, .{copy_string(slice(part, 0, equals)), copy_string(advance(part, equals + 1))});
}
} else if key == "model" {
space := find_index_from_left(value, #char " ");
if space < 0 continue;
array_add(*report.header, .{tprint("model %", slice(value, 0, space),, context.allocator), copy_string(trim(advance(value, space + 1)))});
} else {
array_add(*report.header, .{copy_string(key), copy_string(value)});
}
}
return report, true, "";
}
bench_report_free :: (report: *Bench_Report) {
free(report.path);
for report.header { free(it.key); free(it.value); }
array_free(report.header);
for * report.rows {
free(it.task); free(it.provider);
for it.fields { free(it.key); free(it.value); }
array_free(it.fields);
}
array_free(report.rows);
report.* = .{};
}

118
tools/gpu_bench/sha256.jai Normal file
View File

@ -0,0 +1,118 @@
// SHA-256 (FIPS 180-4), for gpu-bench's corpus and model fingerprints: the same hashes as the
// fetch scripts check (certutil, sha256sum), so they can be read against each other.
Sha256 :: struct {
state: [8] u32 = .[0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19];
block: [64] u8;
used: s64; // bytes in block
total: u64; // bytes hashed
}
sha256_update :: (h: *Sha256, data: [] u8) {
bytes := data;
h.total += cast(u64) bytes.count;
if h.used {
take := min(64 - h.used, bytes.count);
memcpy(h.block.data + h.used, bytes.data, take);
h.used += take;
bytes = array_view(bytes, take, bytes.count - take);
if h.used < 64 return;
sha256_blocks(h, h.block.data, 1);
h.used = 0;
}
whole := bytes.count / 64;
if whole sha256_blocks(h, bytes.data, whole);
rest := bytes.count - whole * 64;
if rest { memcpy(h.block.data, bytes.data + whole * 64, rest); h.used = rest; }
}
sha256_update :: (h: *Sha256, text: string) { sha256_update(h, cast([] u8) text); }
// Lowercase hex, 64 characters; from allocator.
sha256_finish :: (h: *Sha256, allocator := context.allocator) -> string {
bits := h.total * 8;
pad: [72] u8;
pad[0] = 0x80;
pad_count := ifx h.used < 56 then 56 - h.used else 120 - h.used;
for 0..7 pad[pad_count + it] = cast,trunc(u8) (bits >> cast(u64) (56 - 8 * it));
total := h.total;
sha256_update(h, array_view(pad, 0, pad_count + 8));
h.total = total;
HEX :: "0123456789abcdef";
result := alloc_string(64,, allocator);
for word, w: h.state for b: 0..3 {
byte := (word >> cast(u32) (24 - 8 * b)) & 0xff;
result[w * 8 + b * 2] = HEX[byte >> 4];
result[w * 8 + b * 2 + 1] = HEX[byte & 0xf];
}
return result;
}
sha256_of_string :: (text: string, allocator := context.allocator) -> string {
h: Sha256;
sha256_update(*h, text);
return sha256_finish(*h, allocator);
}
// The file's hash, read a few MB at a time; "" when it cannot be read.
sha256_of_file :: (path: string, allocator := context.allocator) -> string {
file, ok := file_open(path, log_errors = false);
if !ok return "";
defer file_close(*file);
CHUNK :: 4 * 1024 * 1024;
buffer := NewArray(CHUNK, u8,, context.default_allocator);
defer array_free(buffer,, context.default_allocator);
h: Sha256;
size := file_length(file);
done: s64;
while done < size {
count := min(CHUNK, size - done);
if !file_read(file, buffer.data, count) return "";
sha256_update(*h, array_view(buffer, 0, count));
done += count;
}
return sha256_finish(*h, allocator);
}
#scope_file
K :: u32.[
0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2,
];
rotr :: inline (x: u32, n: u32) -> u32 #no_aoc { return (x >> n) | (x << (32 - n)); }
sha256_blocks :: (h: *Sha256, data: *u8, count: s64) #no_aoc {
w: [64] u32 = ---;
for block: 0..count - 1 {
p := data + block * 64;
for i: 0..15 {
q := p + i * 4;
w[i] = ((cast(u32) q[0]) << 24) | ((cast(u32) q[1]) << 16) | ((cast(u32) q[2]) << 8) | (cast(u32) q[3]);
}
for i: 16..63 {
s0 := rotr(w[i - 15], 7) ^ rotr(w[i - 15], 18) ^ (w[i - 15] >> 3);
s1 := rotr(w[i - 2], 17) ^ rotr(w[i - 2], 19) ^ (w[i - 2] >> 10);
w[i] = w[i - 16] + s0 + w[i - 7] + s1;
}
a, b, c, d, e, f, g, hh := h.state[0], h.state[1], h.state[2], h.state[3], h.state[4], h.state[5], h.state[6], h.state[7];
for i: 0..63 {
S1 := rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25);
ch := (e & f) ^ (~e & g);
t1 := hh + S1 + ch + K[i] + w[i];
S0 := rotr(a, 2) ^ rotr(a, 13) ^ rotr(a, 22);
maj := (a & b) ^ (a & c) ^ (b & c);
t2 := S0 + maj;
hh = g; g = f; f = e; e = d + t1; d = c; c = b; b = a; a = t1 + t2;
}
h.state[0] += a; h.state[1] += b; h.state[2] += c; h.state[3] += d;
h.state[4] += e; h.state[5] += f; h.state[6] += g; h.state[7] += hh;
}
}

214
tools/gpu_bench/system_info.jai Normal file
View File

@ -0,0 +1,214 @@
// What gpu-bench says about the machine: the computer, CPU and memory, and the NVIDIA GPU through
// NVML (the driver's management library, nvml.dll / libnvidia-ml.so.1), which also gives the GPU's
// memory, load, temperature, power and clock while the models run. Without NVML (no NVIDIA driver)
// the GPU lines say so and the samples are empty.
System_Info :: struct {
machine: string;
os: string;
cpu: string;
logical_cores: s64;
ram_gb: float64;
gpu: string; // "" without NVML
vram_gb: float64;
driver: string;
cuda_driver: string; // the CUDA version the driver supports, such as 13.0
power_limit_w: float64; // the board's enforced limit: a laptop's GPU is often far below the desktop part's
max_sm_clock_mhz: s64;
max_mem_clock_mhz: s64;
gpu_error: string; // why there is no GPU information
power_source: string; // a laptop's: "AC" or "battery", which can change its GPU's speed several times over
}
// One sample of the GPU while a model runs.
Gpu_Sample :: struct {
memory_used_mb: float64; // the whole device's, every process's
util_percent: float64;
temperature_c: float64;
power_w: float64;
sm_clock_mhz: float64;
}
system_info :: () -> System_Info {
info: System_Info;
info.logical_cores = get_number_of_processors(.ALL_LOGICAL);
#if OS == .WINDOWS {
name: [256] u16;
size: u32 = name.count;
if GetComputerNameW(name.data, *size) info.machine = wide_to_utf8(name.data, cast(s32) size);
v: OSVERSIONINFOW;
v.dwOSVersionInfoSize = size_of(OSVERSIONINFOW);
RtlGetVersion(*v);
info.os = sprint("Windows %.%.%", v.dwMajorVersion, v.dwMinorVersion, v.dwBuildNumber);
buffer: [256] u8;
bytes: u32 = buffer.count;
if RegGetValueA(HKEY_LOCAL_MACHINE, "HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0", "ProcessorNameString", RRF_RT_REG_SZ, null, buffer.data, *bytes) == 0
info.cpu = copy_string(trim(to_string(buffer.data)));
memory: MEMORYSTATUSEX;
memory.dwLength = size_of(MEMORYSTATUSEX);
if GlobalMemoryStatusEx(*memory) info.ram_gb = memory.ullTotalPhys / (1024.0 * 1024 * 1024);
power: SYSTEM_POWER_STATUS;
if GetSystemPowerStatus(*power) && power.BatteryFlag != 128 { // 128: no battery
if power.ACLineStatus == 1 info.power_source = "AC";
else if power.ACLineStatus == 0 info.power_source = sprint("battery (% percent)", power.BatteryLifePercent);
}
} else #if OS == .LINUX {
host, host_ok := read_entire_file("/etc/hostname", log_errors = false);
if host_ok info.machine = trim(host);
release, release_ok := read_entire_file("/proc/sys/kernel/osrelease", log_errors = false);
info.os = ifx release_ok then sprint("Linux %", trim(release)) else "Linux";
cpuinfo, cpu_ok := read_entire_file("/proc/cpuinfo", log_errors = false);
if cpu_ok for split(cpuinfo, "\n",, temp) if begins_with(it, "model name") { info.cpu = trim(advance(it, find_index_from_left(it, #char ":") + 1)); break; }
meminfo, mem_ok := read_entire_file("/proc/meminfo", log_errors = false);
if mem_ok for split(meminfo, "\n",, temp) if begins_with(it, "MemTotal:") { info.ram_gb = string_to_int(trim(trim(advance(it, 9)), " kB")) / (1024.0 * 1024); break; }
} else {
info.os = tprint("%", OS);
}
if !info.machine info.machine = "unknown";
if !nvml_start() { info.gpu_error = nvml.error; return info; }
text: [96] u8;
if nvml.DeviceGetName(nvml.device, text.data, text.count) == 0 info.gpu = copy_string(to_string(text.data));
if nvml.SystemGetDriverVersion(text.data, text.count) == 0 info.driver = copy_string(to_string(text.data));
cuda: s32;
if nvml.SystemGetCudaDriverVersion(*cuda) == 0 info.cuda_driver = sprint("%.%", cuda / 1000, (cuda % 1000) / 10);
vram: Nvml_Memory;
if nvml.DeviceGetMemoryInfo(nvml.device, *vram) == 0 info.vram_gb = vram.total / (1024.0 * 1024 * 1024);
limit: u32;
if nvml.DeviceGetEnforcedPowerLimit(nvml.device, *limit) == 0 info.power_limit_w = limit / 1000.0;
clock: u32;
if nvml.DeviceGetMaxClockInfo(nvml.device, NVML_CLOCK_SM, *clock) == 0 info.max_sm_clock_mhz = clock;
if nvml.DeviceGetMaxClockInfo(nvml.device, NVML_CLOCK_MEM, *clock) == 0 info.max_mem_clock_mhz = clock;
return info;
}
// The GPU now; false without NVML.
gpu_sample :: () -> Gpu_Sample, ok: bool {
s: Gpu_Sample;
if !nvml.started return s, false;
memory: Nvml_Memory;
if nvml.DeviceGetMemoryInfo(nvml.device, *memory) == 0 s.memory_used_mb = memory.used / (1024.0 * 1024);
util: Nvml_Utilization;
if nvml.DeviceGetUtilizationRates(nvml.device, *util) == 0 s.util_percent = cast(float64) util.gpu;
value: u32;
if nvml.DeviceGetTemperature(nvml.device, NVML_TEMPERATURE_GPU, *value) == 0 s.temperature_c = cast(float64) value;
if nvml.DeviceGetPowerUsage(nvml.device, *value) == 0 s.power_w = value / 1000.0;
if nvml.DeviceGetClockInfo(nvml.device, NVML_CLOCK_SM, *value) == 0 s.sm_clock_mhz = cast(float64) value;
return s, true;
}
// The CUDA runtime's and cuDNN's versions, as loaded into this process by onnxruntime's CUDA provider,
// with where each came from; "" for one not loaded (before a GPU engine is made).
cuda_library_versions :: () -> cudart: string, cudnn: string {
cudart, cudnn: string;
#if OS == .WINDOWS {
module := GetModuleHandleA("cudart64_13.dll");
if module {
get := cast(#type (version: *s32) -> s32 #c_call) GetProcAddress(module, "cudaRuntimeGetVersion");
version: s32;
if get && get(*version) == 0 cudart = sprint("%.% (%)", version / 1000, (version % 1000) / 10, module_path(module));
}
module = GetModuleHandleA("cudnn64_9.dll");
if module {
get := cast(#type () -> u64 #c_call) GetProcAddress(module, "cudnnGetVersion");
if get {
version := get(); // 9.x: major * 10000 + minor * 100 + patch
cudnn = sprint("%.%.% (%)", version / 10000, (version % 10000) / 100, version % 100, module_path(module));
}
}
}
return cudart, cudnn;
}
#scope_file
#if OS == .WINDOWS {
SYSTEM_POWER_STATUS :: struct {
ACLineStatus, BatteryFlag, BatteryLifePercent, SystemStatusFlag: u8;
BatteryLifeTime, BatteryFullLifeTime: u32;
}
GetSystemPowerStatus :: (status: *SYSTEM_POWER_STATUS) -> BOOL #foreign kernel32;
kernel32 :: #system_library "kernel32";
module_path :: (module: HANDLE) -> string {
name: [1024] u16;
count := GetModuleFileNameW(module, name.data, name.count);
return wide_to_utf8(name.data, count,, temp);
}
}
Nvml_Device :: *void;
Nvml_Memory :: struct { total, free, used: u64; }
Nvml_Utilization :: struct { gpu, memory: u32; }
NVML_CLOCK_SM :: 1;
NVML_CLOCK_MEM :: 2;
NVML_TEMPERATURE_GPU :: 0;
nvml: struct {
started: bool;
error: string;
device: Nvml_Device;
Init: #type () -> s32 #c_call;
DeviceGetHandleByIndex: #type (index: u32, device: *Nvml_Device) -> s32 #c_call;
DeviceGetName: #type (device: Nvml_Device, name: *u8, length: u32) -> s32 #c_call;
SystemGetDriverVersion: #type (version: *u8, length: u32) -> s32 #c_call;
SystemGetCudaDriverVersion: #type (version: *s32) -> s32 #c_call;
DeviceGetMemoryInfo: #type (device: Nvml_Device, memory: *Nvml_Memory) -> s32 #c_call;
DeviceGetUtilizationRates: #type (device: Nvml_Device, util: *Nvml_Utilization) -> s32 #c_call;
DeviceGetTemperature: #type (device: Nvml_Device, sensor: s32, value: *u32) -> s32 #c_call;
DeviceGetPowerUsage: #type (device: Nvml_Device, milliwatts: *u32) -> s32 #c_call;
DeviceGetEnforcedPowerLimit: #type (device: Nvml_Device, milliwatts: *u32) -> s32 #c_call;
DeviceGetClockInfo: #type (device: Nvml_Device, type: s32, mhz: *u32) -> s32 #c_call;
DeviceGetMaxClockInfo: #type (device: Nvml_Device, type: s32, mhz: *u32) -> s32 #c_call;
}
// NVML, and its first GPU: the one onnxruntime's CUDA provider uses (device 0) on a machine with one.
nvml_start :: () -> bool {
if nvml.started return true;
#if OS == .WINDOWS {
library := LoadLibraryA("nvml.dll");
if !library library = LoadLibraryA("C:\\Program Files\\NVIDIA Corporation\\NVSMI\\nvml.dll");
if !library { nvml.error = "nvml.dll was not found: no NVIDIA driver."; return false; }
find :: (name: *u8) -> *void #expand { return GetProcAddress(library, name); }
} else #if OS == .LINUX {
library := dlopen("libnvidia-ml.so.1", RTLD_NOW);
if !library { nvml.error = "libnvidia-ml.so.1 was not found: no NVIDIA driver."; return false; }
find :: (name: *u8) -> *void #expand { return dlsym(library, name); }
} else {
nvml.error = "No NVML on this OS.";
return false;
}
nvml.Init = xx find("nvmlInit_v2");
nvml.DeviceGetHandleByIndex = xx find("nvmlDeviceGetHandleByIndex_v2");
nvml.DeviceGetName = xx find("nvmlDeviceGetName");
nvml.SystemGetDriverVersion = xx find("nvmlSystemGetDriverVersion");
nvml.SystemGetCudaDriverVersion = xx find("nvmlSystemGetCudaDriverVersion");
nvml.DeviceGetMemoryInfo = xx find("nvmlDeviceGetMemoryInfo");
nvml.DeviceGetUtilizationRates = xx find("nvmlDeviceGetUtilizationRates");
nvml.DeviceGetTemperature = xx find("nvmlDeviceGetTemperature");
nvml.DeviceGetPowerUsage = xx find("nvmlDeviceGetPowerUsage");
nvml.DeviceGetEnforcedPowerLimit = xx find("nvmlDeviceGetEnforcedPowerLimit");
nvml.DeviceGetClockInfo = xx find("nvmlDeviceGetClockInfo");
nvml.DeviceGetMaxClockInfo = xx find("nvmlDeviceGetMaxClockInfo");
if !nvml.Init || !nvml.DeviceGetHandleByIndex || !nvml.DeviceGetName || !nvml.SystemGetDriverVersion || !nvml.SystemGetCudaDriverVersion
|| !nvml.DeviceGetMemoryInfo || !nvml.DeviceGetUtilizationRates || !nvml.DeviceGetTemperature || !nvml.DeviceGetPowerUsage
|| !nvml.DeviceGetEnforcedPowerLimit || !nvml.DeviceGetClockInfo || !nvml.DeviceGetMaxClockInfo {
nvml.error = "NVML lacks a function gpu-bench needs: an old driver?";
return false;
}
status := nvml.Init();
if status != 0 { nvml.error = tprint("NVML did not start (error %).", status); return false; }
status = nvml.DeviceGetHandleByIndex(0, *nvml.device);
if status != 0 { nvml.error = tprint("NVML found no GPU (error %).", status); return false; }
nvml.started = true;
return true;
}
#if OS == .WINDOWS {
#import "Windows";
#import "Windows_Utf8";
#import "Windows_Registry";
} else #if OS == .LINUX {
#import "POSIX";
}