// The GPU Benchmark window (the gpu-bench branch): runs gpu-bench (tools/gpu_bench) in a process of // its own, so the app's engines and memory do not touch its numbers, shows how far it has got, and // lays the results files in gpu-bench-results side by side. To compare with another machine, copy // its results file into that folder (Open folder). GPU_BENCH_WINDOW :: "GPU Benchmark"; GPU_BENCH_LOG_LINES :: 2000; gpu_bench_ui: struct { window_open: bool; run_ocr := true; run_labels := true; run_segment := true; cpu: bool; repeats: s32 = 3; note: [128] u8; // A run: the reader thread owns the child process and appends what it prints to output. running: bool; mutex_ready: bool; mutex: Mutex; thread: Thread; command: [..] string; // heap; for the thread output: [..] u8; // under mutex; heap finished: bool; // under mutex: the child has exited, or could not start exit_code: s32; // under mutex start_error: string; // under mutex; heap cancelled: bool; #if OS == .WINDOWS process: HANDLE; // under mutex else process: Process; started_at: Apollo_Time; parsed: s64; // bytes of output made into lines log: [..] string; // heap scroll_log: bool; status: string; // heap: where the run is, or how it ended reports: [..] Bench_Report; // newest first shown: [..] bool; // per report: a column in the comparison baseline: string; // heap: the path of the report the others are compared with loaded: bool; }; gpu_bench_results_folder :: () -> string { return tprint("%gpu-bench-results", path_strip_filename(get_path_of_running_executable())); } // The release build, else the debug one; "" when neither is beside the app. gpu_bench_executable :: () -> path: string, debug: bool { folder := path_strip_filename(get_path_of_running_executable()); extension := ifx OS == .WINDOWS then ".exe" else ""; release := tprint("%gpu-bench%", folder, extension); if file_exists(release) return release, false; debug := tprint("%gpu-bench-debug%", folder, extension); if file_exists(debug) return debug, true; return "", false; } gpu_bench_ui_frame :: () { using ImGui; if gpu_bench_ui.running gpu_bench_poll(); if !gpu_bench_ui.window_open return; if !gpu_bench_ui.loaded gpu_bench_load_reports(); SetNextWindowSize(.{980, 680}, .FirstUseEver); if Begin(GPU_BENCH_WINDOW, *gpu_bench_ui.window_open) gpu_bench_window_contents(); End(); } gpu_bench_ui_shutdown :: () { if gpu_bench_ui.running { gpu_bench_cancel(); thread_is_done(*gpu_bench_ui.thread, -1); thread_deinit(*gpu_bench_ui.thread); gpu_bench_ui.running = false; } push_allocator(context.default_allocator); for * gpu_bench_ui.reports bench_report_free(it); array_free(gpu_bench_ui.reports); array_free(gpu_bench_ui.shown); for gpu_bench_ui.log free(it); array_free(gpu_bench_ui.log); array_free(gpu_bench_ui.output); for gpu_bench_ui.command free(it); array_free(gpu_bench_ui.command); free(gpu_bench_ui.status); free(gpu_bench_ui.baseline); free(gpu_bench_ui.start_error); } #scope_file gpu_bench_window_contents :: () { using ImGui; TextWrapped("Times each model mcap runs on the GPU (OCR, labels, segmentation) over a fixed corpus of 30 images that every machine has, each checked against its SHA-256 before anything runs. Each model's files and settings are fingerprinted too, so a comparison shows whether it is like for like."); exe, debug := gpu_bench_executable(); if gpu_bench_ui.running { Text("%", gpu_bench_ui.status); SameLine(); TextColored(app_theme.palette.dim, "(% s)", formatFloat(to_float64_seconds(current_time_monotonic() - gpu_bench_ui.started_at), trailing_width = 0)); SameLine(); if Button("Cancel") gpu_bench_cancel(); } else { Checkbox("OCR", *gpu_bench_ui.run_ocr); SameLine(); Checkbox("Labels", *gpu_bench_ui.run_labels); SameLine(); Checkbox("Segmentation", *gpu_bench_ui.run_segment); SameLine(); Checkbox("CPU too", *gpu_bench_ui.cpu); if IsItemHovered() SetTooltip("Each model on the CPU as well, after the GPU, and the label model's text encoder (a like: search). Slow: minutes for segmentation."); SameLine(); SetNextItemWidth(120); SliderInt("Timed passes", *gpu_bench_ui.repeats, 1, 10); if IsItemHovered() SetTooltip("Passes over the corpus that are timed, after one untimed warm-up pass; the median is reported"); SetNextItemWidth(-FLOAT32_MIN); InputTextWithHint("##note", "A note kept with the results, such as: plugged in, best performance mode", gpu_bench_ui.note.data, gpu_bench_ui.note.count); reason := ""; if !exe reason = "gpu-bench is not beside the app: build with jai first.jai - release."; else if ocr_ui.thread_busy reason = "A scan or a reading is running on the GPU: let it finish, or pause or cancel it, first."; else if !gpu_bench_ui.run_ocr && !gpu_bench_ui.run_labels && !gpu_bench_ui.run_segment reason = "Choose a kind of model to run."; if reason BeginDisabled(); if Button("Run the benchmark") gpu_bench_start(exe); if reason EndDisabled(); if reason { SameLine(); TextColored(app_theme.palette.warning, "%", reason); } else if debug { SameLine(); TextColored(app_theme.palette.warning, "Only the debug gpu-bench is built: its CPU work is slow, so build with - release to compare."); } else if gpu_bench_ui.status { SameLine(); TextColored(app_theme.palette.dim, "%", gpu_bench_ui.status); } TextColored(app_theme.palette.dim, "mcap's own models stay loaded while it runs. For the cleanest numbers, close mcap and run gpu-bench in a terminal; its results land in the same folder."); } if CollapsingHeader(temp_c_string(tprint("Output (% lines)###output", gpu_bench_ui.log.count))) { if BeginChild("##log", .{0, 220}, .Borders, .HorizontalScrollbar) { for gpu_bench_ui.log TextUnformatted(it); if gpu_bench_ui.scroll_log { SetScrollHereY(1); gpu_bench_ui.scroll_log = false; } } EndChild(); } Separator(); folder := gpu_bench_results_folder(); Text("Results (% files)", gpu_bench_ui.reports.count); SameLine(); if SmallButton("Open folder") { make_directory_if_it_does_not_exist(folder, recursive = true); platform_open_file_in_default_program(folder, folder); } if IsItemHovered() SetTooltip("% : copy another machine's results files here to compare", folder); SameLine(); if SmallButton("Reload") gpu_bench_load_reports(); if !gpu_bench_ui.reports.count { TextColored(app_theme.palette.dim, "None yet: run the benchmark."); return; } gpu_bench_reports_table(); Spacing(); gpu_bench_comparison_table(); } // The results files: which are shown, and which is the baseline. gpu_bench_reports_table :: () { using ImGui; height := min(cast(float) gpu_bench_ui.reports.count + 1, 7) * GetFrameHeightWithSpacing(); if !BeginTable("##reports", 7, .Borders | .RowBg | .SizingFixedFit | .ScrollY, .{0, height}) return; TableSetupScrollFreeze(0, 1); TableSetupColumn("Show"); TableSetupColumn("Baseline"); TableSetupColumn("Date"); TableSetupColumn("Machine"); TableSetupColumn("GPU"); TableSetupColumn("Power"); TableSetupColumn("Note", .WidthStretch); TableHeadersRow(); for * report, n: gpu_bench_ui.reports { PushID(cast(s32) n); TableNextRow(); TableNextColumn(); Checkbox("##show", *gpu_bench_ui.shown[n]); TableNextColumn(); if RadioButton("##baseline", report.path == gpu_bench_ui.baseline) { free(gpu_bench_ui.baseline,, context.default_allocator); gpu_bench_ui.baseline = copy_string(report.path,, context.default_allocator); gpu_bench_ui.shown[n] = true; } TableNextColumn(); TextUnformatted(bench_header(report, "date")); if IsItemHovered() SetTooltip("%", report.path); TableNextColumn(); TextUnformatted(bench_header(report, "machine")); TableNextColumn(); TextUnformatted(bench_header(report, "gpu")); TableNextColumn(); limit := bench_header(report, "power_limit_w"); power := bench_header(report, "power"); TextUnformatted(tprint("%%%", ifx limit then tprint("% W", limit) else "", ifx limit && power then ", " else "", power)); TableNextColumn(); TextUnformatted(bench_header(report, "note")); PopID(); } EndTable(); } // The shown reports side by side: what each machine is, then each model's time, against the baseline's. gpu_bench_comparison_table :: () { using ImGui; columns: [..] *Bench_Report; columns.allocator = temp; baseline: *Bench_Report; for * gpu_bench_ui.reports { if gpu_bench_ui.shown[it_index] array_add(*columns, it); if it.path == gpu_bench_ui.baseline baseline = it; } if !columns.count { TextColored(app_theme.palette.dim, "Tick Show for the results to compare."); return; } if !baseline baseline = columns[0]; // Every task and provider any of them ran, in the order they ran. Bench_Key :: struct { task, provider: string; } keys: [..] Bench_Key; keys.allocator = temp; for report: columns for report.rows { found := false; for key: keys if key.task == it.task && key.provider == it.provider found = true; if !found array_add(*keys, .{it.task, it.provider}); } if !BeginTable("##compare", cast(s32) columns.count + 1, .Borders | .RowBg | .SizingFixedFit | .ScrollX | .ScrollY, .{0, -FLOAT32_MIN}) return; TableSetupScrollFreeze(1, 1); TableSetupColumn(""); for columns TableSetupColumn(temp_c_string(tprint("%##%", bench_header(it, "machine"), it_index))); TableHeadersRow(); header_row :: (label: string, key: string, tooltip := "") #expand { TableNextRow(); TableNextColumn(); TextColored(app_theme.palette.dim, "%", label); if tooltip && IsItemHovered() SetTooltip("%", tooltip); for columns { TableNextColumn(); value := bench_header(it, key); if it != baseline && value != bench_header(baseline, key) TextColored(app_theme.palette.accent, "%", value); else TextUnformatted(value); } } header_row("GPU", "gpu"); header_row("Memory", "vram_gb", "GB of GPU memory"); header_row("Power limit", "power_limit_w", "The board's enforced limit in watts: a laptop's GPU is often far below the desktop part's"); header_row("Power", "power"); header_row("Driver", "driver"); header_row("cuDNN", "cudnn"); header_row("onnxruntime", "onnxruntime"); header_row("CPU", "cpu"); header_row("Date", "date"); header_row("Note", "note"); // The corpus: different images make every number not comparable. TableNextRow(); TableNextColumn(); TextColored(app_theme.palette.dim, "Corpus"); for columns { TableNextColumn(); corpus := bench_header(it, "corpus"); if corpus == bench_header(baseline, "corpus") TextColored(app_theme.palette.success, "same (%)", slice(corpus, 0, min(corpus.count, 12))); else TextColored(app_theme.palette.error, "DIFFERENT: not comparable"); } for key: keys { TableNextRow(); TableNextColumn(); TextUnformatted(tprint("% %", key.task, ifx key.provider == "CUDA" then "(GPU)" else "(CPU)")); base_row := bench_row(baseline, key.task, key.provider); base_ms := ifx base_row then string_to_float64(bench_field(base_row, "ms")) else 0; for report: columns { TableNextColumn(); row := bench_row(report, key.task, key.provider); if !row { error := bench_header(report, tprint("error % %", key.task, key.provider)); if !error error = bench_header(report, tprint("error % CPU", key.task)); // a GPU run that fell back if error { TextColored(app_theme.palette.error, "failed"); if IsItemHovered() SetTooltip("%", error); } else TextColored(app_theme.palette.dim, "-"); continue; } ms := string_to_float64(bench_field(row, "ms")); unit := bench_field(row, "unit"); same_model := bench_header(report, tprint("model %", key.task)) == bench_header(baseline, tprint("model %", key.task)); text := tprint("% ms/%", formatFloat(ms, trailing_width = ifx ms < 10 then 2 else 1), unit); if report != baseline && base_ms > 0 && ms > 0 { ratio := base_ms / ms; // above 1: faster than the baseline colour := ifx ratio > 1.05 then app_theme.palette.success else ifx ratio < 0.95 then app_theme.palette.warning else app_theme.palette.neutral; TextColored(colour, "% (%x)%", text, formatFloat(ratio, trailing_width = 2), ifx same_model then "" else " *"); } else { TextUnformatted(tprint("%%", text, ifx same_model then "" else " *")); } if IsItemHovered() { builder: String_Builder; builder.allocator = temp; if !same_model append(*builder, "* Different model files or models.config settings from the baseline's: not like for like.\n\n"); print_to_builder(*builder, "% on %: % %s per second", key.task, report.path, bench_field(row, "per_s"), unit); mp := bench_field(row, "mp_per_s"); if mp print_to_builder(*builder, ", % megapixels per second", mp); print_to_builder(*builder, "\nmedian of the timed passes; best % ms, worst % ms\n", bench_field(row, "best_ms"), bench_field(row, "worst_ms")); print_to_builder(*builder, "loading % s, the first % % s, batch %\n", bench_field(row, "load_s"), unit, bench_field(row, "first_s"), bench_field(row, "batch")); vram := bench_field(row, "vram_mb"); if vram print_to_builder(*builder, "GPU: % MB more memory at the peak, % percent busy, % W, % MHz, up to % C\n", vram, bench_field(row, "gpu_util"), bench_field(row, "power_w"), bench_field(row, "sm_mhz"), bench_field(row, "temp_c")); stages := bench_field(row, "stages"); if stages print_to_builder(*builder, "ms per %, summed over threads: %\n", unit, stages); print_to_builder(*builder, "found %", bench_field(row, "output")); if report != baseline && base_row { same_output := bench_field(row, "digest") == bench_field(base_row, "digest"); print_to_builder(*builder, ", % the baseline's", ifx same_output then "the same as" else "not quite the same as (GPUs round differently)"); } SetTooltip("%", builder_to_string(*builder,, temp)); } } } EndTable(); } gpu_bench_load_reports :: () { push_allocator(context.default_allocator); for * gpu_bench_ui.reports bench_report_free(it); array_reset(*gpu_bench_ui.reports); array_reset(*gpu_bench_ui.shown); gpu_bench_ui.loaded = true; folder := gpu_bench_results_folder(); if !file_exists(folder) return; for file_list(folder,, temp) { if !ends_with_nocase(it, ".txt") continue; text, ok := read_entire_file(it, log_errors = false,, temp); if !ok continue; report, parsed, error := bench_parse(text, it); if !parsed { log_error("[gpu-bench] %", error); bench_report_free(*report); continue; } array_add(*gpu_bench_ui.reports, report); } quick_sort(gpu_bench_ui.reports, (a: Bench_Report, b: Bench_Report) -> s64 { return compare(bench_header(*b, "date"), bench_header(*a, "date")); }); array_resize(*gpu_bench_ui.shown, gpu_bench_ui.reports.count); for * gpu_bench_ui.shown it.* = true; found := false; for gpu_bench_ui.reports if it.path == gpu_bench_ui.baseline found = true; if !found && gpu_bench_ui.reports.count { free(gpu_bench_ui.baseline); gpu_bench_ui.baseline = copy_string(gpu_bench_ui.reports[gpu_bench_ui.reports.count - 1].path); // the oldest: what later runs are measured against } } gpu_bench_start :: (exe: string) { push_allocator(context.default_allocator); if !gpu_bench_ui.mutex_ready { init(*gpu_bench_ui.mutex); gpu_bench_ui.mutex_ready = true; } for gpu_bench_ui.command free(it); array_reset(*gpu_bench_ui.command); add :: (s: string) #expand { array_add(*gpu_bench_ui.command, copy_string(s)); } add(exe); add("-progress"); add("-repeats"); add(tprint("%", gpu_bench_ui.repeats)); add("-out"); add(gpu_bench_results_folder()); only: [..] string; only.allocator = temp; if gpu_bench_ui.run_ocr array_add(*only, "ocr"); if gpu_bench_ui.run_labels array_add(*only, "labels"); if gpu_bench_ui.run_segment array_add(*only, "segment"); add("-only"); add(join(..only, ",",, temp)); if gpu_bench_ui.cpu add("-cpu"); note := trim(to_string(gpu_bench_ui.note.data)); if note { add("-note"); add(note); } for gpu_bench_ui.log free(it); array_reset(*gpu_bench_ui.log); array_reset(*gpu_bench_ui.output); free(gpu_bench_ui.start_error); gpu_bench_ui.start_error = ""; gpu_bench_ui.finished = false; gpu_bench_ui.cancelled = false; gpu_bench_ui.exit_code = 0; gpu_bench_ui.parsed = 0; gpu_bench_set_status("Starting..."); gpu_bench_ui.started_at = current_time_monotonic(); thread_init(*gpu_bench_ui.thread, gpu_bench_thread_proc, name = "gpu-bench"); thread_start(*gpu_bench_ui.thread); gpu_bench_ui.running = true; log("[gpu-bench] Running %", join(..gpu_bench_ui.command, " ",, temp)); } gpu_bench_cancel :: () { lock(*gpu_bench_ui.mutex); gpu_bench_ui.cancelled = true; #if OS == .WINDOWS { if gpu_bench_ui.process TerminateProcess(gpu_bench_ui.process, 1); } else kill_process(*gpu_bench_ui.process); unlock(*gpu_bench_ui.mutex); } // What the child printed since the last frame, made into lines; and the end of the run. gpu_bench_poll :: () { push_allocator(context.default_allocator); lock(*gpu_bench_ui.mutex); finished := gpu_bench_ui.finished; exit_code := gpu_bench_ui.exit_code; new_text := copy_string(cast(string) array_view(gpu_bench_ui.output, gpu_bench_ui.parsed, gpu_bench_ui.output.count - gpu_bench_ui.parsed),, temp); start_error := copy_string(gpu_bench_ui.start_error,, temp); unlock(*gpu_bench_ui.mutex); // Whole lines only, until the end. end := find_index_from_right(new_text, #char "\n"); if finished end = new_text.count; if end > 0 { gpu_bench_ui.parsed += end; for raw: split(slice(new_text, 0, end), "\n",, temp) { line := trim_right(raw, "\r"); if !line.count continue; if line[0] == 0x1b continue; // onnxruntime's coloured warnings if begins_with(line, "@progress ") { gpu_bench_set_status(gpu_bench_progress_words(advance(line, 10))); continue; } if begins_with(line, "@results ") continue; array_add(*gpu_bench_ui.log, copy_string(line)); gpu_bench_ui.scroll_log = true; } while gpu_bench_ui.log.count > GPU_BENCH_LOG_LINES { free(gpu_bench_ui.log[0]); array_ordered_remove_by_index(*gpu_bench_ui.log, 0); } } if !finished return; thread_is_done(*gpu_bench_ui.thread, -1); thread_deinit(*gpu_bench_ui.thread); gpu_bench_ui.running = false; seconds := formatFloat(to_float64_seconds(current_time_monotonic() - gpu_bench_ui.started_at), trailing_width = 0); if start_error gpu_bench_set_status(start_error); else if gpu_bench_ui.cancelled gpu_bench_set_status("Cancelled."); else if exit_code == 0 gpu_bench_set_status(tprint("Done in % s.", seconds)); else gpu_bench_set_status(tprint("gpu-bench failed (exit code %) after % s: Output says why.", exit_code, seconds)); log("[gpu-bench] %", gpu_bench_ui.status); gpu_bench_load_reports(); } // "@progress ocr/ppocr-v5-mobile CUDA timing 2 3" in words. gpu_bench_progress_words :: (text: string) -> string { parts := split(text, " ",, temp); if parts.count < 5 return text; task, provider, phase, done, total := parts[0], parts[1], parts[2], parts[3], parts[4]; on := ifx provider == "CUDA" then "on the GPU" else ifx provider == "CPU" then "on the CPU" else ""; if task == "corpus" return tprint("The corpus: % %...", phase, ifx done != "0" then tprint("image %", done) else ""); if task == "hashing" return tprint("Fingerprinting %: % (% of %)", provider, phase, done, total); if phase == { case "loading"; return tprint("% %: loading the model...", task, on); case "first"; return tprint("% %: the first image...", task, on); case "warm-up"; return tprint("% %: warm-up pass % of %...", task, on, done, total); case "timing"; return tprint("% %: timed pass % of %...", task, on, done, total); } return text; } gpu_bench_set_status :: (text: string) { free(gpu_bench_ui.status,, context.default_allocator); gpu_bench_ui.status = copy_string(text,, context.default_allocator); } // The reader thread: starts gpu-bench, takes what it prints until it exits. gpu_bench_thread_proc :: (thread: *Thread) -> s64 { append_output :: (data: *u8, count: s64) { lock(*gpu_bench_ui.mutex); old := gpu_bench_ui.output.count; array_resize(*gpu_bench_ui.output, old + count, initialize = false,, context.default_allocator); memcpy(gpu_bench_ui.output.data + old, data, count); unlock(*gpu_bench_ui.mutex); } finish :: (exit_code: s32, error := "") { lock(*gpu_bench_ui.mutex); gpu_bench_ui.exit_code = exit_code; if error gpu_bench_ui.start_error = copy_string(error,, context.default_allocator); gpu_bench_ui.finished = true; unlock(*gpu_bench_ui.mutex); } buffer: [4096] u8; #if OS == .WINDOWS { // Not the Process module: from a GUI program, it would open a console window for gpu-bench. attributes: SECURITY_ATTRIBUTES; attributes.nLength = size_of(SECURITY_ATTRIBUTES); attributes.bInheritHandle = 1; read_end, write_end: HANDLE; if !CreatePipe(*read_end, *write_end, *attributes, 0) { finish(-1, tprint("Could not start gpu-bench: no pipe (Windows error %).", GetLastError())); return 0; } SetHandleInformation(read_end, HANDLE_FLAG_INHERIT, 0); startup: STARTUPINFOW; startup.cb = size_of(STARTUPINFOW); startup.dwFlags = STARTF_USESTDHANDLES; startup.hStdOutput = write_end; startup.hStdError = write_end; builder: String_Builder; builder.allocator = temp; for gpu_bench_ui.command { if it_index append(*builder, " "); append(*builder, windows_quote_argument(it)); } command_line := builder_to_string(*builder,, temp); info: PROCESS_INFORMATION; working := path_strip_filename(gpu_bench_ui.command[0]); started := CreateProcessW(null, utf8_to_wide(command_line,, temp), null, null, 1, CREATE_NO_WINDOW, null, utf8_to_wide(working,, temp), *startup, *info); CloseHandle(write_end); // the child's copy is the only one left: reading ends when it exits if !started { error := GetLastError(); CloseHandle(read_end); finish(-1, tprint("Could not start % (Windows error %).", gpu_bench_ui.command[0], error)); return 0; } CloseHandle(info.hThread); lock(*gpu_bench_ui.mutex); gpu_bench_ui.process = info.hProcess; cancelled := gpu_bench_ui.cancelled; unlock(*gpu_bench_ui.mutex); if cancelled TerminateProcess(info.hProcess, 1); while true { read: u32; if !ReadFile(read_end, buffer.data, buffer.count, *read) || !read break; append_output(buffer.data, read); } CloseHandle(read_end); WaitForSingleObject(info.hProcess, INFINITE); exit_code: u32; GetExitCodeProcess(info.hProcess, *exit_code); lock(*gpu_bench_ui.mutex); gpu_bench_ui.process = null; unlock(*gpu_bench_ui.mutex); CloseHandle(info.hProcess); finish(cast,no_check(s32) exit_code); } else { errors: [4096] u8; if !create_process(*gpu_bench_ui.process, ..gpu_bench_ui.command, capture_and_return_output = true) { finish(-1, tprint("Could not start %.", gpu_bench_ui.command[0])); return 0; } while true { ok, output_bytes, error_bytes := read_from_process(*gpu_bench_ui.process, buffer, errors, -1); if output_bytes append_output(buffer.data, output_bytes); if error_bytes append_output(errors.data, error_bytes); if !ok || (gpu_bench_ui.process.output.eof && gpu_bench_ui.process.error.eof) break; } _, result := get_process_result(*gpu_bench_ui.process, -1); deinit(*gpu_bench_ui.process); finish(cast(s32) result.exit_code); } return 0; } #import "Process";