diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..4aa1a41 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +tools/gpu_bench/corpus/** binary diff --git a/check_targets.jai b/check_targets.jai index 8da168c..c5769cb 100644 --- a/check_targets.jai +++ b/check_targets.jai @@ -47,7 +47,7 @@ APP_NAME :: "mcap"; APP_FOLDER_NAME :: "mcap"; GIT_BRANCH :: "check"; GIT_HASH :: "-"; -VERSION :: "0.1.24"; +VERSION :: "0.1.25"; BUILD_DATE :: "-"; DEBUG :: true; COMPILE_TESTS :: true; diff --git a/first.jai b/first.jai index b1b0600..24778c6 100644 --- a/first.jai +++ b/first.jai @@ -8,7 +8,7 @@ APP_NAME :: "mcap"; APP_FOLDER_NAME :: "mcap"; APP_BUNDLE_IDENTIFIER :: "com.example.mcap"; -VERSION :: "0.1.24"; +VERSION :: "0.1.25"; // Where everything goes: bin/, or bin-headless/ for builds made with no_crash_dialog, so automated // builds never write over the ones you run by hand. build_directory := "bin"; @@ -70,6 +70,10 @@ build_project :: () { // jai first.jai - record_probe: also bin/record-probe, a screen recorded with modules/Screen_Record (video-capture-plan.html, phase 1). if array_find(args, "record_probe") array_add(*pending, start_tool_build("record-probe", "tools/record_probe.jai", optimized = build_release)); + // The gpu-bench branch: bin/gpu-bench, every GPU model timed over a fixed corpus (tools/gpu_bench), + // always built, as the GPU Benchmark window runs it. + array_add(*pending, start_tool_build("gpu-bench", "tools/gpu_bench/gpu_bench.jai", optimized = build_release)); + wait_for_builds(pending); for pending if !it.failed { @@ -312,6 +316,11 @@ copy_assets :: () { } else { log("Segmentation: the model has not been made; see modules/Segment/README.md."); } + + // gpu-bench's corpus, in bin/gpu-bench-corpus: in git, so every machine has the same images. + make_directory_if_it_does_not_exist(tprint("%/gpu-bench-corpus", build_directory)); + copy_if_changed("tools/gpu_bench/corpus.txt", tprint("%/gpu-bench-corpus/corpus.txt", build_directory)); + for file_list("tools/gpu_bench/corpus") copy_if_changed(it, tprint("%/gpu-bench-corpus/%", build_directory, path_filename(it))); } copy_if_changed :: (from: string, to: string) { diff --git a/src/App.jai b/src/App.jai index cc07ce4..b388a04 100644 --- a/src/App.jai +++ b/src/App.jai @@ -26,6 +26,8 @@ USING_WINDOWS :: OS == .WINDOWS; #load "app/import.jai"; #load "app/open_file.jai"; #load "app/segment_ui.jai"; +#load "app/gpu_bench_ui.jai"; // the gpu-bench branch: the GPU Benchmark window +#load "../tools/gpu_bench/results.jai"; // its results files #load "app/library_folders.jai"; #load "app/exit_shutdown.jai"; #load "app/screen_capture.jai"; diff --git a/src/app/app_ui.jai b/src/app/app_ui.jai index e9c596e..5fc4a62 100644 --- a/src/app/app_ui.jai +++ b/src/app/app_ui.jai @@ -39,6 +39,7 @@ app_ui_shutdown :: () { open_file_shutdown(); return; } + gpu_bench_ui_shutdown(); // ends a benchmark run still going app_shutdown_step(.CAPTURE); screen_capture_shutdown(); app_shutdown_step(.IMPORT); @@ -121,6 +122,7 @@ app_main_window_ui :: () { labels_ui_frame(); segment_ui_frame(); stats_ui_frame(); + gpu_bench_ui_frame(); // app/gpu_bench_ui.jai import_frame(); open_file_frame(); // after import_frame, which hands the grid what an import of it wrote screen_record_leftovers_popup(); diff --git a/src/app/gpu_bench_ui.jai b/src/app/gpu_bench_ui.jai new file mode 100644 index 0000000..0e2ff30 --- /dev/null +++ b/src/app/gpu_bench_ui.jai @@ -0,0 +1,525 @@ +// The GPU Benchmark window (the gpu-bench branch): runs gpu-bench (tools/gpu_bench) in a process of +// its own, so the app's engines and memory do not touch its numbers, shows how far it has got, and +// lays the results files in gpu-bench-results side by side. To compare with another machine, copy +// its results file into that folder (Open folder). + +GPU_BENCH_WINDOW :: "GPU Benchmark"; +GPU_BENCH_LOG_LINES :: 2000; + +gpu_bench_ui: struct { + window_open: bool; + run_ocr := true; + run_labels := true; + run_segment := true; + cpu: bool; + repeats: s32 = 3; + note: [128] u8; + + // A run: the reader thread owns the child process and appends what it prints to output. + running: bool; + mutex_ready: bool; + mutex: Mutex; + thread: Thread; + command: [..] string; // heap; for the thread + output: [..] u8; // under mutex; heap + finished: bool; // under mutex: the child has exited, or could not start + exit_code: s32; // under mutex + start_error: string; // under mutex; heap + cancelled: bool; + #if OS == .WINDOWS process: HANDLE; // under mutex + else process: Process; + started_at: Apollo_Time; + parsed: s64; // bytes of output made into lines + log: [..] string; // heap + scroll_log: bool; + status: string; // heap: where the run is, or how it ended + + reports: [..] Bench_Report; // newest first + shown: [..] bool; // per report: a column in the comparison + baseline: string; // heap: the path of the report the others are compared with + loaded: bool; +}; + +gpu_bench_results_folder :: () -> string { + return tprint("%gpu-bench-results", path_strip_filename(get_path_of_running_executable())); +} + +// The release build, else the debug one; "" when neither is beside the app. +gpu_bench_executable :: () -> path: string, debug: bool { + folder := path_strip_filename(get_path_of_running_executable()); + extension := ifx OS == .WINDOWS then ".exe" else ""; + release := tprint("%gpu-bench%", folder, extension); + if file_exists(release) return release, false; + debug := tprint("%gpu-bench-debug%", folder, extension); + if file_exists(debug) return debug, true; + return "", false; +} + +gpu_bench_ui_frame :: () { using ImGui; + if gpu_bench_ui.running gpu_bench_poll(); + if !gpu_bench_ui.window_open return; + if !gpu_bench_ui.loaded gpu_bench_load_reports(); + SetNextWindowSize(.{980, 680}, .FirstUseEver); + if Begin(GPU_BENCH_WINDOW, *gpu_bench_ui.window_open) gpu_bench_window_contents(); + End(); +} + +gpu_bench_ui_shutdown :: () { + if gpu_bench_ui.running { + gpu_bench_cancel(); + thread_is_done(*gpu_bench_ui.thread, -1); + thread_deinit(*gpu_bench_ui.thread); + gpu_bench_ui.running = false; + } + push_allocator(context.default_allocator); + for * gpu_bench_ui.reports bench_report_free(it); + array_free(gpu_bench_ui.reports); + array_free(gpu_bench_ui.shown); + for gpu_bench_ui.log free(it); + array_free(gpu_bench_ui.log); + array_free(gpu_bench_ui.output); + for gpu_bench_ui.command free(it); + array_free(gpu_bench_ui.command); + free(gpu_bench_ui.status); + free(gpu_bench_ui.baseline); + free(gpu_bench_ui.start_error); +} + +#scope_file + +gpu_bench_window_contents :: () { using ImGui; + TextWrapped("Times each model mcap runs on the GPU (OCR, labels, segmentation) over a fixed corpus of 30 images that every machine has, each checked against its SHA-256 before anything runs. Each model's files and settings are fingerprinted too, so a comparison shows whether it is like for like."); + exe, debug := gpu_bench_executable(); + + if gpu_bench_ui.running { + Text("%", gpu_bench_ui.status); + SameLine(); + TextColored(app_theme.palette.dim, "(% s)", formatFloat(to_float64_seconds(current_time_monotonic() - gpu_bench_ui.started_at), trailing_width = 0)); + SameLine(); + if Button("Cancel") gpu_bench_cancel(); + } else { + Checkbox("OCR", *gpu_bench_ui.run_ocr); + SameLine(); Checkbox("Labels", *gpu_bench_ui.run_labels); + SameLine(); Checkbox("Segmentation", *gpu_bench_ui.run_segment); + SameLine(); Checkbox("CPU too", *gpu_bench_ui.cpu); + if IsItemHovered() SetTooltip("Each model on the CPU as well, after the GPU, and the label model's text encoder (a like: search). Slow: minutes for segmentation."); + SameLine(); + SetNextItemWidth(120); + SliderInt("Timed passes", *gpu_bench_ui.repeats, 1, 10); + if IsItemHovered() SetTooltip("Passes over the corpus that are timed, after one untimed warm-up pass; the median is reported"); + SetNextItemWidth(-FLOAT32_MIN); + InputTextWithHint("##note", "A note kept with the results, such as: plugged in, best performance mode", gpu_bench_ui.note.data, gpu_bench_ui.note.count); + + reason := ""; + if !exe reason = "gpu-bench is not beside the app: build with jai first.jai - release."; + else if ocr_ui.thread_busy reason = "A scan or a reading is running on the GPU: let it finish, or pause or cancel it, first."; + else if !gpu_bench_ui.run_ocr && !gpu_bench_ui.run_labels && !gpu_bench_ui.run_segment reason = "Choose a kind of model to run."; + if reason BeginDisabled(); + if Button("Run the benchmark") gpu_bench_start(exe); + if reason EndDisabled(); + if reason { SameLine(); TextColored(app_theme.palette.warning, "%", reason); } + else if debug { SameLine(); TextColored(app_theme.palette.warning, "Only the debug gpu-bench is built: its CPU work is slow, so build with - release to compare."); } + else if gpu_bench_ui.status { SameLine(); TextColored(app_theme.palette.dim, "%", gpu_bench_ui.status); } + TextColored(app_theme.palette.dim, "mcap's own models stay loaded while it runs. For the cleanest numbers, close mcap and run gpu-bench in a terminal; its results land in the same folder."); + } + + if CollapsingHeader(temp_c_string(tprint("Output (% lines)###output", gpu_bench_ui.log.count))) { + if BeginChild("##log", .{0, 220}, .Borders, .HorizontalScrollbar) { + for gpu_bench_ui.log TextUnformatted(it); + if gpu_bench_ui.scroll_log { SetScrollHereY(1); gpu_bench_ui.scroll_log = false; } + } + EndChild(); + } + + Separator(); + folder := gpu_bench_results_folder(); + Text("Results (% files)", gpu_bench_ui.reports.count); + SameLine(); + if SmallButton("Open folder") { + make_directory_if_it_does_not_exist(folder, recursive = true); + platform_open_file_in_default_program(folder, folder); + } + if IsItemHovered() SetTooltip("% : copy another machine's results files here to compare", folder); + SameLine(); + if SmallButton("Reload") gpu_bench_load_reports(); + if !gpu_bench_ui.reports.count { + TextColored(app_theme.palette.dim, "None yet: run the benchmark."); + return; + } + gpu_bench_reports_table(); + Spacing(); + gpu_bench_comparison_table(); +} + +// The results files: which are shown, and which is the baseline. +gpu_bench_reports_table :: () { using ImGui; + height := min(cast(float) gpu_bench_ui.reports.count + 1, 7) * GetFrameHeightWithSpacing(); + if !BeginTable("##reports", 7, .Borders | .RowBg | .SizingFixedFit | .ScrollY, .{0, height}) return; + TableSetupScrollFreeze(0, 1); + TableSetupColumn("Show"); + TableSetupColumn("Baseline"); + TableSetupColumn("Date"); + TableSetupColumn("Machine"); + TableSetupColumn("GPU"); + TableSetupColumn("Power"); + TableSetupColumn("Note", .WidthStretch); + TableHeadersRow(); + for * report, n: gpu_bench_ui.reports { + PushID(cast(s32) n); + TableNextRow(); + TableNextColumn(); Checkbox("##show", *gpu_bench_ui.shown[n]); + TableNextColumn(); + if RadioButton("##baseline", report.path == gpu_bench_ui.baseline) { + free(gpu_bench_ui.baseline,, context.default_allocator); + gpu_bench_ui.baseline = copy_string(report.path,, context.default_allocator); + gpu_bench_ui.shown[n] = true; + } + TableNextColumn(); TextUnformatted(bench_header(report, "date")); + if IsItemHovered() SetTooltip("%", report.path); + TableNextColumn(); TextUnformatted(bench_header(report, "machine")); + TableNextColumn(); TextUnformatted(bench_header(report, "gpu")); + TableNextColumn(); + limit := bench_header(report, "power_limit_w"); + power := bench_header(report, "power"); + TextUnformatted(tprint("%%%", ifx limit then tprint("% W", limit) else "", ifx limit && power then ", " else "", power)); + TableNextColumn(); TextUnformatted(bench_header(report, "note")); + PopID(); + } + EndTable(); +} + +// The shown reports side by side: what each machine is, then each model's time, against the baseline's. +gpu_bench_comparison_table :: () { using ImGui; + columns: [..] *Bench_Report; + columns.allocator = temp; + baseline: *Bench_Report; + for * gpu_bench_ui.reports { + if gpu_bench_ui.shown[it_index] array_add(*columns, it); + if it.path == gpu_bench_ui.baseline baseline = it; + } + if !columns.count { TextColored(app_theme.palette.dim, "Tick Show for the results to compare."); return; } + if !baseline baseline = columns[0]; + + // Every task and provider any of them ran, in the order they ran. + Bench_Key :: struct { task, provider: string; } + keys: [..] Bench_Key; + keys.allocator = temp; + for report: columns for report.rows { + found := false; + for key: keys if key.task == it.task && key.provider == it.provider found = true; + if !found array_add(*keys, .{it.task, it.provider}); + } + + if !BeginTable("##compare", cast(s32) columns.count + 1, .Borders | .RowBg | .SizingFixedFit | .ScrollX | .ScrollY, .{0, -FLOAT32_MIN}) return; + TableSetupScrollFreeze(1, 1); + TableSetupColumn(""); + for columns TableSetupColumn(temp_c_string(tprint("%##%", bench_header(it, "machine"), it_index))); + TableHeadersRow(); + + header_row :: (label: string, key: string, tooltip := "") #expand { + TableNextRow(); + TableNextColumn(); TextColored(app_theme.palette.dim, "%", label); + if tooltip && IsItemHovered() SetTooltip("%", tooltip); + for columns { + TableNextColumn(); + value := bench_header(it, key); + if it != baseline && value != bench_header(baseline, key) TextColored(app_theme.palette.accent, "%", value); + else TextUnformatted(value); + } + } + header_row("GPU", "gpu"); + header_row("Memory", "vram_gb", "GB of GPU memory"); + header_row("Power limit", "power_limit_w", "The board's enforced limit in watts: a laptop's GPU is often far below the desktop part's"); + header_row("Power", "power"); + header_row("Driver", "driver"); + header_row("cuDNN", "cudnn"); + header_row("onnxruntime", "onnxruntime"); + header_row("CPU", "cpu"); + header_row("Date", "date"); + header_row("Note", "note"); + + // The corpus: different images make every number not comparable. + TableNextRow(); + TableNextColumn(); TextColored(app_theme.palette.dim, "Corpus"); + for columns { + TableNextColumn(); + corpus := bench_header(it, "corpus"); + if corpus == bench_header(baseline, "corpus") TextColored(app_theme.palette.success, "same (%)", slice(corpus, 0, min(corpus.count, 12))); + else TextColored(app_theme.palette.error, "DIFFERENT: not comparable"); + } + + for key: keys { + TableNextRow(); + TableNextColumn(); + TextUnformatted(tprint("% %", key.task, ifx key.provider == "CUDA" then "(GPU)" else "(CPU)")); + base_row := bench_row(baseline, key.task, key.provider); + base_ms := ifx base_row then string_to_float64(bench_field(base_row, "ms")) else 0; + for report: columns { + TableNextColumn(); + row := bench_row(report, key.task, key.provider); + if !row { + error := bench_header(report, tprint("error % %", key.task, key.provider)); + if !error error = bench_header(report, tprint("error % CPU", key.task)); // a GPU run that fell back + if error { TextColored(app_theme.palette.error, "failed"); if IsItemHovered() SetTooltip("%", error); } + else TextColored(app_theme.palette.dim, "-"); + continue; + } + ms := string_to_float64(bench_field(row, "ms")); + unit := bench_field(row, "unit"); + same_model := bench_header(report, tprint("model %", key.task)) == bench_header(baseline, tprint("model %", key.task)); + text := tprint("% ms/%", formatFloat(ms, trailing_width = ifx ms < 10 then 2 else 1), unit); + if report != baseline && base_ms > 0 && ms > 0 { + ratio := base_ms / ms; // above 1: faster than the baseline + colour := ifx ratio > 1.05 then app_theme.palette.success else ifx ratio < 0.95 then app_theme.palette.warning else app_theme.palette.neutral; + TextColored(colour, "% (%x)%", text, formatFloat(ratio, trailing_width = 2), ifx same_model then "" else " *"); + } else { + TextUnformatted(tprint("%%", text, ifx same_model then "" else " *")); + } + if IsItemHovered() { + builder: String_Builder; + builder.allocator = temp; + if !same_model append(*builder, "* Different model files or models.config settings from the baseline's: not like for like.\n\n"); + print_to_builder(*builder, "% on %: % %s per second", key.task, report.path, bench_field(row, "per_s"), unit); + mp := bench_field(row, "mp_per_s"); + if mp print_to_builder(*builder, ", % megapixels per second", mp); + print_to_builder(*builder, "\nmedian of the timed passes; best % ms, worst % ms\n", bench_field(row, "best_ms"), bench_field(row, "worst_ms")); + print_to_builder(*builder, "loading % s, the first % % s, batch %\n", bench_field(row, "load_s"), unit, bench_field(row, "first_s"), bench_field(row, "batch")); + vram := bench_field(row, "vram_mb"); + if vram print_to_builder(*builder, "GPU: % MB more memory at the peak, % percent busy, % W, % MHz, up to % C\n", vram, + bench_field(row, "gpu_util"), bench_field(row, "power_w"), bench_field(row, "sm_mhz"), bench_field(row, "temp_c")); + stages := bench_field(row, "stages"); + if stages print_to_builder(*builder, "ms per %, summed over threads: %\n", unit, stages); + print_to_builder(*builder, "found %", bench_field(row, "output")); + if report != baseline && base_row { + same_output := bench_field(row, "digest") == bench_field(base_row, "digest"); + print_to_builder(*builder, ", % the baseline's", ifx same_output then "the same as" else "not quite the same as (GPUs round differently)"); + } + SetTooltip("%", builder_to_string(*builder,, temp)); + } + } + } + EndTable(); +} + +gpu_bench_load_reports :: () { + push_allocator(context.default_allocator); + for * gpu_bench_ui.reports bench_report_free(it); + array_reset(*gpu_bench_ui.reports); + array_reset(*gpu_bench_ui.shown); + gpu_bench_ui.loaded = true; + folder := gpu_bench_results_folder(); + if !file_exists(folder) return; + for file_list(folder,, temp) { + if !ends_with_nocase(it, ".txt") continue; + text, ok := read_entire_file(it, log_errors = false,, temp); + if !ok continue; + report, parsed, error := bench_parse(text, it); + if !parsed { log_error("[gpu-bench] %", error); bench_report_free(*report); continue; } + array_add(*gpu_bench_ui.reports, report); + } + quick_sort(gpu_bench_ui.reports, (a: Bench_Report, b: Bench_Report) -> s64 { return compare(bench_header(*b, "date"), bench_header(*a, "date")); }); + array_resize(*gpu_bench_ui.shown, gpu_bench_ui.reports.count); + for * gpu_bench_ui.shown it.* = true; + found := false; + for gpu_bench_ui.reports if it.path == gpu_bench_ui.baseline found = true; + if !found && gpu_bench_ui.reports.count { + free(gpu_bench_ui.baseline); + gpu_bench_ui.baseline = copy_string(gpu_bench_ui.reports[gpu_bench_ui.reports.count - 1].path); // the oldest: what later runs are measured against + } +} + +gpu_bench_start :: (exe: string) { + push_allocator(context.default_allocator); + if !gpu_bench_ui.mutex_ready { init(*gpu_bench_ui.mutex); gpu_bench_ui.mutex_ready = true; } + for gpu_bench_ui.command free(it); + array_reset(*gpu_bench_ui.command); + add :: (s: string) #expand { array_add(*gpu_bench_ui.command, copy_string(s)); } + add(exe); + add("-progress"); + add("-repeats"); add(tprint("%", gpu_bench_ui.repeats)); + add("-out"); add(gpu_bench_results_folder()); + only: [..] string; + only.allocator = temp; + if gpu_bench_ui.run_ocr array_add(*only, "ocr"); + if gpu_bench_ui.run_labels array_add(*only, "labels"); + if gpu_bench_ui.run_segment array_add(*only, "segment"); + add("-only"); add(join(..only, ",",, temp)); + if gpu_bench_ui.cpu add("-cpu"); + note := trim(to_string(gpu_bench_ui.note.data)); + if note { add("-note"); add(note); } + + for gpu_bench_ui.log free(it); + array_reset(*gpu_bench_ui.log); + array_reset(*gpu_bench_ui.output); + free(gpu_bench_ui.start_error); + gpu_bench_ui.start_error = ""; + gpu_bench_ui.finished = false; + gpu_bench_ui.cancelled = false; + gpu_bench_ui.exit_code = 0; + gpu_bench_ui.parsed = 0; + gpu_bench_set_status("Starting..."); + gpu_bench_ui.started_at = current_time_monotonic(); + thread_init(*gpu_bench_ui.thread, gpu_bench_thread_proc, name = "gpu-bench"); + thread_start(*gpu_bench_ui.thread); + gpu_bench_ui.running = true; + log("[gpu-bench] Running %", join(..gpu_bench_ui.command, " ",, temp)); +} + +gpu_bench_cancel :: () { + lock(*gpu_bench_ui.mutex); + gpu_bench_ui.cancelled = true; + #if OS == .WINDOWS { if gpu_bench_ui.process TerminateProcess(gpu_bench_ui.process, 1); } + else kill_process(*gpu_bench_ui.process); + unlock(*gpu_bench_ui.mutex); +} + +// What the child printed since the last frame, made into lines; and the end of the run. +gpu_bench_poll :: () { + push_allocator(context.default_allocator); + lock(*gpu_bench_ui.mutex); + finished := gpu_bench_ui.finished; + exit_code := gpu_bench_ui.exit_code; + new_text := copy_string(cast(string) array_view(gpu_bench_ui.output, gpu_bench_ui.parsed, gpu_bench_ui.output.count - gpu_bench_ui.parsed),, temp); + start_error := copy_string(gpu_bench_ui.start_error,, temp); + unlock(*gpu_bench_ui.mutex); + + // Whole lines only, until the end. + end := find_index_from_right(new_text, #char "\n"); + if finished end = new_text.count; + if end > 0 { + gpu_bench_ui.parsed += end; + for raw: split(slice(new_text, 0, end), "\n",, temp) { + line := trim_right(raw, "\r"); + if !line.count continue; + if line[0] == 0x1b continue; // onnxruntime's coloured warnings + if begins_with(line, "@progress ") { gpu_bench_set_status(gpu_bench_progress_words(advance(line, 10))); continue; } + if begins_with(line, "@results ") continue; + array_add(*gpu_bench_ui.log, copy_string(line)); + gpu_bench_ui.scroll_log = true; + } + while gpu_bench_ui.log.count > GPU_BENCH_LOG_LINES { + free(gpu_bench_ui.log[0]); + array_ordered_remove_by_index(*gpu_bench_ui.log, 0); + } + } + if !finished return; + + thread_is_done(*gpu_bench_ui.thread, -1); + thread_deinit(*gpu_bench_ui.thread); + gpu_bench_ui.running = false; + seconds := formatFloat(to_float64_seconds(current_time_monotonic() - gpu_bench_ui.started_at), trailing_width = 0); + if start_error gpu_bench_set_status(start_error); + else if gpu_bench_ui.cancelled gpu_bench_set_status("Cancelled."); + else if exit_code == 0 gpu_bench_set_status(tprint("Done in % s.", seconds)); + else gpu_bench_set_status(tprint("gpu-bench failed (exit code %) after % s: Output says why.", exit_code, seconds)); + log("[gpu-bench] %", gpu_bench_ui.status); + gpu_bench_load_reports(); +} + +// "@progress ocr/ppocr-v5-mobile CUDA timing 2 3" in words. +gpu_bench_progress_words :: (text: string) -> string { + parts := split(text, " ",, temp); + if parts.count < 5 return text; + task, provider, phase, done, total := parts[0], parts[1], parts[2], parts[3], parts[4]; + on := ifx provider == "CUDA" then "on the GPU" else ifx provider == "CPU" then "on the CPU" else ""; + if task == "corpus" return tprint("The corpus: % %...", phase, ifx done != "0" then tprint("image %", done) else ""); + if task == "hashing" return tprint("Fingerprinting %: % (% of %)", provider, phase, done, total); + if phase == { + case "loading"; return tprint("% %: loading the model...", task, on); + case "first"; return tprint("% %: the first image...", task, on); + case "warm-up"; return tprint("% %: warm-up pass % of %...", task, on, done, total); + case "timing"; return tprint("% %: timed pass % of %...", task, on, done, total); + } + return text; +} + +gpu_bench_set_status :: (text: string) { + free(gpu_bench_ui.status,, context.default_allocator); + gpu_bench_ui.status = copy_string(text,, context.default_allocator); +} + +// The reader thread: starts gpu-bench, takes what it prints until it exits. +gpu_bench_thread_proc :: (thread: *Thread) -> s64 { + append_output :: (data: *u8, count: s64) { + lock(*gpu_bench_ui.mutex); + old := gpu_bench_ui.output.count; + array_resize(*gpu_bench_ui.output, old + count, initialize = false,, context.default_allocator); + memcpy(gpu_bench_ui.output.data + old, data, count); + unlock(*gpu_bench_ui.mutex); + } + finish :: (exit_code: s32, error := "") { + lock(*gpu_bench_ui.mutex); + gpu_bench_ui.exit_code = exit_code; + if error gpu_bench_ui.start_error = copy_string(error,, context.default_allocator); + gpu_bench_ui.finished = true; + unlock(*gpu_bench_ui.mutex); + } + buffer: [4096] u8; + + #if OS == .WINDOWS { + // Not the Process module: from a GUI program, it would open a console window for gpu-bench. + attributes: SECURITY_ATTRIBUTES; + attributes.nLength = size_of(SECURITY_ATTRIBUTES); + attributes.bInheritHandle = 1; + read_end, write_end: HANDLE; + if !CreatePipe(*read_end, *write_end, *attributes, 0) { finish(-1, tprint("Could not start gpu-bench: no pipe (Windows error %).", GetLastError())); return 0; } + SetHandleInformation(read_end, HANDLE_FLAG_INHERIT, 0); + startup: STARTUPINFOW; + startup.cb = size_of(STARTUPINFOW); + startup.dwFlags = STARTF_USESTDHANDLES; + startup.hStdOutput = write_end; + startup.hStdError = write_end; + builder: String_Builder; + builder.allocator = temp; + for gpu_bench_ui.command { if it_index append(*builder, " "); append(*builder, windows_quote_argument(it)); } + command_line := builder_to_string(*builder,, temp); + info: PROCESS_INFORMATION; + working := path_strip_filename(gpu_bench_ui.command[0]); + started := CreateProcessW(null, utf8_to_wide(command_line,, temp), null, null, 1, CREATE_NO_WINDOW, null, utf8_to_wide(working,, temp), *startup, *info); + CloseHandle(write_end); // the child's copy is the only one left: reading ends when it exits + if !started { + error := GetLastError(); + CloseHandle(read_end); + finish(-1, tprint("Could not start % (Windows error %).", gpu_bench_ui.command[0], error)); + return 0; + } + CloseHandle(info.hThread); + lock(*gpu_bench_ui.mutex); + gpu_bench_ui.process = info.hProcess; + cancelled := gpu_bench_ui.cancelled; + unlock(*gpu_bench_ui.mutex); + if cancelled TerminateProcess(info.hProcess, 1); + while true { + read: u32; + if !ReadFile(read_end, buffer.data, buffer.count, *read) || !read break; + append_output(buffer.data, read); + } + CloseHandle(read_end); + WaitForSingleObject(info.hProcess, INFINITE); + exit_code: u32; + GetExitCodeProcess(info.hProcess, *exit_code); + lock(*gpu_bench_ui.mutex); + gpu_bench_ui.process = null; + unlock(*gpu_bench_ui.mutex); + CloseHandle(info.hProcess); + finish(cast,no_check(s32) exit_code); + } else { + errors: [4096] u8; + if !create_process(*gpu_bench_ui.process, ..gpu_bench_ui.command, capture_and_return_output = true) { + finish(-1, tprint("Could not start %.", gpu_bench_ui.command[0])); + return 0; + } + while true { + ok, output_bytes, error_bytes := read_from_process(*gpu_bench_ui.process, buffer, errors, -1); + if output_bytes append_output(buffer.data, output_bytes); + if error_bytes append_output(errors.data, error_bytes); + if !ok || (gpu_bench_ui.process.output.eof && gpu_bench_ui.process.error.eof) break; + } + _, result := get_process_result(*gpu_bench_ui.process, -1); + deinit(*gpu_bench_ui.process); + finish(cast(s32) result.exit_code); + } + return 0; +} + +#import "Process"; diff --git a/src/app/viewer.jai b/src/app/viewer.jai index 228dbeb..81fa858 100644 --- a/src/app/viewer.jai +++ b/src/app/viewer.jai @@ -1115,6 +1115,9 @@ viewer_ui :: () { using ImGui; if Button("Segments") segment_ui.window_open = !segment_ui.window_open; // app/segment_ui.jai if IsItemHovered() SetTooltip("Segmentation: what is where in the images of the library folders you choose"); SameLine(); + if Button("GPU Bench") gpu_bench_ui.window_open = !gpu_bench_ui.window_open; // app/gpu_bench_ui.jai + if IsItemHovered() SetTooltip("Time every GPU model over a fixed corpus, and compare machines"); + SameLine(); if Button("Import") importer.window_open = !importer.window_open; // app/import.jai if IsItemHovered() SetTooltip("Bring in a folder of screenshots and photos"); SameLine(); diff --git a/tools/gpu_bench/README.md b/tools/gpu_bench/README.md new file mode 100644 index 0000000..7a19441 --- /dev/null +++ b/tools/gpu_bench/README.md @@ -0,0 +1,50 @@ +# gpu-bench (the `gpu-bench` branch, not for master) + +Times every model mcap runs on the GPU over a fixed corpus, so machines can be compared like for +like: + +| Task | What runs | +| --- | --- | +| `ocr/ppocr-v5-mobile`, `ocr/ppocr-v5-server` | detection, orientation and recognition, 8 images per call, as a scan does | +| `labels/siglip2-base-patch16-384` | the label model's image encoder | +| `segment/mask2former-swin-base-coco-panoptic` | panoptic segmentation | +| `labels-text/...` (with `-cpu`) | the text encoder, one phrase at a time as a `like:` search runs it (always on the CPU) | + +## Keeping it consistent + +- **The corpus** (`corpus/`, 30 images: 16 COCO val2017 photos, 2 of them scaled to 12 MP, and 12 + synthetic screenshots from 720p to 4K made by `make_corpus.py`) is in git. The build copies it to + `bin/gpu-bench-corpus`. Before anything runs, every file is checked against its SHA-256 in + `corpus.txt`, and a run stops if one differs. The corpus fingerprint, a hash of those name and hash + pairs, goes in the results. `.gitattributes` marks the images binary, so git never changes them. +- **The models:** each task's fingerprint covers the hash of each of its files and its + `models.config` entry (comments and spacing left out). It also says whether every file is the one + the fetch scripts pin. The segmentation `.onnx` is exported on each machine, so it has no pin: + copy it from one machine to the others. +- **The machine:** each result records the GPU, its memory, enforced power limit and clocks, the + driver, the CUDA runtime and cuDNN actually loaded (with their paths), onnxruntime, the CPU, RAM, + and AC or battery. +- **The run:** each model gets one untimed run on the first image, one untimed warm-up pass over + everything, then N timed passes (default 3). The median is reported, with the best and worst + passes. While the timed passes run, NVML samples the GPU every 100 ms: peak memory over the + device's use before the model loaded, utilisation, power, SM clock and temperature. +- **The output:** what the models found (OCR lines and characters, segments, best phrases) and a + digest of it. Different GPUs can round differently, so a different digest is a hint, not an error. + +## Running + +`jai first.jai - release` builds `bin/gpu-bench.exe` with mcap. Then either: + +- **In mcap:** click **GPU Bench** above the grid, choose what to run, and click **Run the benchmark**. + It runs gpu-bench in a separate process with no console and shows its progress. Close mcap and use + the terminal for the cleanest numbers, as mcap's own models stay loaded while it runs. +- **In a terminal:** `bin\gpu-bench.exe [-only ocr,labels,segment] [-models a,b] [-repeats N] [-warmup N] [-cpu | -cpu-only] [-note TEXT]` + +Results go to `bin/gpu-bench-results/_.txt` (plain text, `results.jai`). To compare +machines, copy their files into that folder: the window puts the results side by side. Each result +shows its speed against the baseline (the oldest file, or the one you pick). Its corpus row turns red +when the images differ, and a `*` marks a model whose files or settings differ. Hover a time for its +details. + +The server OCR model can take a lot of GPU and system memory and a long time (see the main README's +known issue). Leave it out with `-models ppocr-v5-mobile,siglip2-base-patch16-384,mask2former-swin-base-coco-panoptic`. diff --git a/tools/gpu_bench/corpus.txt b/tools/gpu_bench/corpus.txt new file mode 100644 index 0000000..960f14a --- /dev/null +++ b/tools/gpu_bench/corpus.txt @@ -0,0 +1,35 @@ +# The GPU benchmark's corpus: every image in corpus/, with its SHA-256. gpu-bench checks each file +# against this before it runs, and its corpus fingerprint is the SHA-256 of these name and hash +# pairs, so two results are comparable only when the fingerprints match. coco-*: COCO val2017 +# (images.cocodataset.org/val2017, CC licences); screen-* and photo-*: make_corpus.py. + +ffe0f0cec3b2e27aab1967229cdf0a0d7751dcdd5800322f0b8ac0dffb3b8a8d coco-000000000139.jpg +f3a2974ce3686332609124c70e3e6a2e3aca43fccf1cd1bd7c5c03820977f57d coco-000000000285.jpg +a4cd7f45ac1ce27eaafb254b23af7c0b18a064be08870ceaaf03b2147f2ce550 coco-000000000632.jpg +5c0e559c75d3969c8e3e297b61f61063f78045c9d4802b526ba616361f3823fd coco-000000000724.jpg +1dd31e9059c491992be2f562624eb4093e17aee08b4f7baf5ff9ea24543b0a33 coco-000000000776.jpg +83981537a7baeafbeb9c8cb67b3484dc26433f574b3685d021fa537e277e4726 coco-000000000785.jpg +d5b79e7fa716f85ca86f46e5f518da9b6c5e26414925624d76e6476861aec495 coco-000000000802.jpg +c2aa138ee3a59b057a7ba6fc5a6a18e62af531aa7dab78a7bfd33c1cd7e55eb6 coco-000000000872.jpg +1c67b783f78b18ddb5dd20aaba4d4aacaf98e5277051688a96da096afa9cdf1f coco-000000000885.jpg +24bb77a31928404e45a0454b06f6a0bd54a8db103590c9d7917288a1e0269f05 coco-000000001000.jpg +8e85a2e71ee512fe748a3a3be9bb40b8591e8c501c0f3896efcb5be5999ec236 coco-000000001268.jpg +34a98da7c11bc8811e6eec445145bb16f3d5f4338945ba116edcc1fe554d181b coco-000000001296.jpg +1cccef217a1e5eec2acc74d7c08a105da3ef6fdb8514e221cdea9b0b373b60ea coco-000000001353.jpg +a044804a0b0186a1267acae34975932f14ede96c4a2b78fcee74e6c2fbd8414d coco-000000001490.jpg +974f2af6d81f65627a680ea9f42bdee1067a0ddcdb9f8e7a945bacd21d4dc2ce coco-000000001532.jpg +77131e2f74aa7e374f951a69157cf7f51e00201f81bacfa8a9ab7309b245553f coco-000000001584.jpg +3c34557e033b8022567bf41d1b9f6da6aa3547305b77a922f1f351c197040111 photo-12mp-livingroom.jpg +769a74c85bb0b03322864cffc252738a7858c6947b4a45bf643708a766ea0f73 photo-12mp-tennis.jpg +c989015b2ecf47630acfa1c5afe828e406accc5f0345814e4d5a5cf53cdb4bc7 screen-chat-phone.png +41e445c52f991a659b65e435a5e08173817a53d150da3f7b5380b0a697e50ecc screen-dialog-1080p.png +76224a64b1460169bb0684c3a8eefc977b1cbbd556a170fe7559928b246012ad screen-document-1080p.png +96f00112b373d990162a13e85d524f5acf6a3b4bb671883742127879f3b39246 screen-document-4k.png +2112042a1f22349805af2136a2b9aee346d47387d3d31ca698d0d570e3215c00 screen-document-strip.png +973ba96240163e51c9e92e0bf7d927ca07dab78e2236b1ecf4e902ef4e170187 screen-editor-dark-1080p.png +51f62f081a73d05bbb2dac313ca5e716628961dcf2760c073c6f365090959258 screen-editor-dark-1440p.png +9d763d163ada6d85e3e4d0415295ccffac1d9b5eaa60b6a6209c7af6c6b06d43 screen-editor-light-1080p.png +35bdf92f31eee19599f2f20a17c8cedb536289e3605b9f91f32c1e7dc0a5169a screen-spreadsheet-1080p.png +84b5074a25e1ea13ca17be3476dff40609fe11c0f70ecdcbf6920c59c99272b1 screen-spreadsheet-1440p.png +54f30b7745bdba5890c25c9ae90e417263fa6126ffe41925f98e981920986448 screen-terminal-1080p.png +0f64a278adb72e482f9526af9037ce4fa8ae6826a1fa9b94c8d0d4102bb6e22e screen-terminal-720p.png diff --git a/tools/gpu_bench/corpus/coco-000000000139.jpg b/tools/gpu_bench/corpus/coco-000000000139.jpg new file mode 100644 index 0000000..19023f7 Binary files /dev/null and b/tools/gpu_bench/corpus/coco-000000000139.jpg differ diff --git a/tools/gpu_bench/corpus/coco-000000000285.jpg b/tools/gpu_bench/corpus/coco-000000000285.jpg new file mode 100644 index 0000000..7d76080 Binary files /dev/null and b/tools/gpu_bench/corpus/coco-000000000285.jpg differ diff --git a/tools/gpu_bench/corpus/coco-000000000632.jpg b/tools/gpu_bench/corpus/coco-000000000632.jpg new file mode 100644 index 0000000..e04fea5 Binary files /dev/null and b/tools/gpu_bench/corpus/coco-000000000632.jpg differ diff --git a/tools/gpu_bench/corpus/coco-000000000724.jpg b/tools/gpu_bench/corpus/coco-000000000724.jpg new file mode 100644 index 0000000..2a17e0c Binary files /dev/null and b/tools/gpu_bench/corpus/coco-000000000724.jpg differ diff --git a/tools/gpu_bench/corpus/coco-000000000776.jpg b/tools/gpu_bench/corpus/coco-000000000776.jpg new file mode 100644 index 0000000..26360e2 Binary files /dev/null and b/tools/gpu_bench/corpus/coco-000000000776.jpg differ diff --git a/tools/gpu_bench/corpus/coco-000000000785.jpg b/tools/gpu_bench/corpus/coco-000000000785.jpg new file mode 100644 index 0000000..78718f5 Binary files /dev/null and b/tools/gpu_bench/corpus/coco-000000000785.jpg differ diff --git a/tools/gpu_bench/corpus/coco-000000000802.jpg b/tools/gpu_bench/corpus/coco-000000000802.jpg new file mode 100644 index 0000000..d8d1184 Binary files /dev/null and b/tools/gpu_bench/corpus/coco-000000000802.jpg differ diff --git a/tools/gpu_bench/corpus/coco-000000000872.jpg b/tools/gpu_bench/corpus/coco-000000000872.jpg new file mode 100644 index 0000000..2eac5c2 Binary files /dev/null and b/tools/gpu_bench/corpus/coco-000000000872.jpg differ diff --git a/tools/gpu_bench/corpus/coco-000000000885.jpg b/tools/gpu_bench/corpus/coco-000000000885.jpg new file mode 100644 index 0000000..677a010 Binary files /dev/null and b/tools/gpu_bench/corpus/coco-000000000885.jpg differ diff --git a/tools/gpu_bench/corpus/coco-000000001000.jpg b/tools/gpu_bench/corpus/coco-000000001000.jpg new file mode 100644 index 0000000..ba333ca Binary files /dev/null and b/tools/gpu_bench/corpus/coco-000000001000.jpg differ diff --git a/tools/gpu_bench/corpus/coco-000000001268.jpg b/tools/gpu_bench/corpus/coco-000000001268.jpg new file mode 100644 index 0000000..986b5b2 Binary files /dev/null and b/tools/gpu_bench/corpus/coco-000000001268.jpg differ diff --git a/tools/gpu_bench/corpus/coco-000000001296.jpg b/tools/gpu_bench/corpus/coco-000000001296.jpg new file mode 100644 index 0000000..ac68a88 Binary files /dev/null and b/tools/gpu_bench/corpus/coco-000000001296.jpg differ diff --git a/tools/gpu_bench/corpus/coco-000000001353.jpg b/tools/gpu_bench/corpus/coco-000000001353.jpg new file mode 100644 index 0000000..c4cfd59 Binary files /dev/null and b/tools/gpu_bench/corpus/coco-000000001353.jpg differ diff --git a/tools/gpu_bench/corpus/coco-000000001490.jpg b/tools/gpu_bench/corpus/coco-000000001490.jpg new file mode 100644 index 0000000..1335553 Binary files /dev/null and b/tools/gpu_bench/corpus/coco-000000001490.jpg differ diff --git a/tools/gpu_bench/corpus/coco-000000001532.jpg b/tools/gpu_bench/corpus/coco-000000001532.jpg new file mode 100644 index 0000000..23a3c72 Binary files /dev/null and b/tools/gpu_bench/corpus/coco-000000001532.jpg differ diff --git a/tools/gpu_bench/corpus/coco-000000001584.jpg b/tools/gpu_bench/corpus/coco-000000001584.jpg new file mode 100644 index 0000000..a01cd65 Binary files /dev/null and b/tools/gpu_bench/corpus/coco-000000001584.jpg differ diff --git a/tools/gpu_bench/corpus/photo-12mp-livingroom.jpg b/tools/gpu_bench/corpus/photo-12mp-livingroom.jpg new file mode 100644 index 0000000..842ace5 Binary files /dev/null and b/tools/gpu_bench/corpus/photo-12mp-livingroom.jpg differ diff --git a/tools/gpu_bench/corpus/photo-12mp-tennis.jpg b/tools/gpu_bench/corpus/photo-12mp-tennis.jpg new file mode 100644 index 0000000..a0f00cc Binary files /dev/null and b/tools/gpu_bench/corpus/photo-12mp-tennis.jpg differ diff --git a/tools/gpu_bench/corpus/screen-chat-phone.png b/tools/gpu_bench/corpus/screen-chat-phone.png new file mode 100644 index 0000000..065a440 Binary files /dev/null and b/tools/gpu_bench/corpus/screen-chat-phone.png differ diff --git a/tools/gpu_bench/corpus/screen-dialog-1080p.png b/tools/gpu_bench/corpus/screen-dialog-1080p.png new file mode 100644 index 0000000..a439935 Binary files /dev/null and b/tools/gpu_bench/corpus/screen-dialog-1080p.png differ diff --git a/tools/gpu_bench/corpus/screen-document-1080p.png b/tools/gpu_bench/corpus/screen-document-1080p.png new file mode 100644 index 0000000..acd0aff Binary files /dev/null and b/tools/gpu_bench/corpus/screen-document-1080p.png differ diff --git a/tools/gpu_bench/corpus/screen-document-4k.png b/tools/gpu_bench/corpus/screen-document-4k.png new file mode 100644 index 0000000..ed52c3c Binary files /dev/null and b/tools/gpu_bench/corpus/screen-document-4k.png differ diff --git a/tools/gpu_bench/corpus/screen-document-strip.png b/tools/gpu_bench/corpus/screen-document-strip.png new file mode 100644 index 0000000..8ad7abb Binary files /dev/null and b/tools/gpu_bench/corpus/screen-document-strip.png differ diff --git a/tools/gpu_bench/corpus/screen-editor-dark-1080p.png b/tools/gpu_bench/corpus/screen-editor-dark-1080p.png new file mode 100644 index 0000000..d17a909 Binary files /dev/null and b/tools/gpu_bench/corpus/screen-editor-dark-1080p.png differ diff --git a/tools/gpu_bench/corpus/screen-editor-dark-1440p.png b/tools/gpu_bench/corpus/screen-editor-dark-1440p.png new file mode 100644 index 0000000..efc1442 Binary files /dev/null and b/tools/gpu_bench/corpus/screen-editor-dark-1440p.png differ diff --git a/tools/gpu_bench/corpus/screen-editor-light-1080p.png b/tools/gpu_bench/corpus/screen-editor-light-1080p.png new file mode 100644 index 0000000..ed65462 Binary files /dev/null and b/tools/gpu_bench/corpus/screen-editor-light-1080p.png differ diff --git a/tools/gpu_bench/corpus/screen-spreadsheet-1080p.png b/tools/gpu_bench/corpus/screen-spreadsheet-1080p.png new file mode 100644 index 0000000..524b4dc Binary files /dev/null and b/tools/gpu_bench/corpus/screen-spreadsheet-1080p.png differ diff --git a/tools/gpu_bench/corpus/screen-spreadsheet-1440p.png b/tools/gpu_bench/corpus/screen-spreadsheet-1440p.png new file mode 100644 index 0000000..a793914 Binary files /dev/null and b/tools/gpu_bench/corpus/screen-spreadsheet-1440p.png differ diff --git a/tools/gpu_bench/corpus/screen-terminal-1080p.png b/tools/gpu_bench/corpus/screen-terminal-1080p.png new file mode 100644 index 0000000..0592f07 Binary files /dev/null and b/tools/gpu_bench/corpus/screen-terminal-1080p.png differ diff --git a/tools/gpu_bench/corpus/screen-terminal-720p.png b/tools/gpu_bench/corpus/screen-terminal-720p.png new file mode 100644 index 0000000..8378b5f Binary files /dev/null and b/tools/gpu_bench/corpus/screen-terminal-720p.png differ diff --git a/tools/gpu_bench/gpu_bench.jai b/tools/gpu_bench/gpu_bench.jai new file mode 100644 index 0000000..12e0e8c --- /dev/null +++ b/tools/gpu_bench/gpu_bench.jai @@ -0,0 +1,666 @@ +// gpu-bench: times every model mcap runs on the GPU (each OCR model, the label model's image encoder, +// segmentation) over a fixed corpus, so machines can be compared. The corpus (gpu-bench-corpus beside +// this program, from tools/gpu_bench/corpus) is checked file by file against corpus.txt's SHA-256s +// before anything runs, and each model's files and models.config entry are fingerprinted, so a +// comparison of two results is known to be like for like. mcap's GPU Benchmark window runs this and +// lays the results side by side. +// +// gpu-bench [-only ocr,labels,segment] [-models a,b] [-repeats N] [-warmup N] [-cpu | -cpu-only] +// [-corpus FOLDER] [-out FOLDER] [-note TEXT] [-progress] +// +// -only the kinds of model to run (default all) +// -models the models to run, by their models.config names (default every one) +// -repeats timed passes over the corpus; the median is reported (default 3) +// -warmup untimed passes first, so every image size has been seen (default 1) +// -cpu the CPU too, after the GPU (slow: minutes for segmentation); also times the label +// model's text encoder, which always runs on the CPU, as a like: search uses it +// -cpu-only the CPU only +// -corpus the corpus folder (default gpu-bench-corpus beside this program) +// -out where the results file goes (default gpu-bench-results beside this program) +// -note a few words kept with the results, such as "plugged in, max performance" +// -progress also print @progress lines, for the GPU Benchmark window +// +// Build: jai first.jai - release (it is built with mcap on the gpu-bench branch). + +USAGE :: "Usage: gpu-bench [-only ocr,labels,segment] [-models a,b] [-repeats N] [-warmup N] [-cpu | -cpu-only] [-corpus FOLDER] [-out FOLDER] [-note TEXT] [-progress]\n"; + +options: struct { + only: [..] string; + models: [..] string; + repeats := 3; + warmup := 1; + gpu := true; + cpu := false; + progress := false; + corpus_folder, out_folder, note: string; +}; + +main :: () { + runtime_support_headless = true; // a command-line tool: a failure is reported on the console and exits with 1, no message box + args := get_command_line_arguments(); + exe_folder := path_strip_filename(get_path_of_running_executable()); + options.corpus_folder = sprint("%gpu-bench-corpus", exe_folder); + options.out_folder = sprint("%gpu-bench-results", exe_folder); + i := 1; + while i < args.count { + arg := args[i]; + value := ifx i + 1 < args.count then args[i + 1] else ""; + if arg == { + case "-only"; for split(value, ",") array_add(*options.only, trim(it)); i += 2; + case "-models"; for split(value, ",") array_add(*options.models, trim(it)); i += 2; + case "-repeats"; options.repeats = max(1, string_to_int(value)); i += 2; + case "-warmup"; options.warmup = max(0, string_to_int(value)); i += 2; + case "-cpu"; options.cpu = true; i += 1; + case "-cpu-only"; options.cpu = true; options.gpu = false; i += 1; + case "-corpus"; options.corpus_folder = value; i += 2; + case "-out"; options.out_folder = value; i += 2; + case "-note"; options.note = value; i += 2; + case "-progress"; options.progress = true; i += 1; + case "-help"; print(USAGE); exit(0); + case; print("Unknown option %.\n%", arg, USAGE); exit(2); + } + } + make_directory_if_it_does_not_exist(options.out_folder, recursive = true); + started_at := current_time_consensus(); + + // The machine. + info := system_info(); + ort_ok, ort_error := ort_load(); + print("gpu-bench, mcap % (%, %)\n", VERSION, GIT_HASH, GIT_BRANCH); + print("machine: %, %\n", info.machine, info.os); + print("cpu: % (% threads), % GB of memory\n", info.cpu, info.logical_cores, FF(info.ram_gb, 1)); + if info.gpu print("gpu: %, % GB, driver % (CUDA %), power limit % W, max clocks % / % MHz\n", info.gpu, FF(info.vram_gb, 1), info.driver, info.cuda_driver, FF(info.power_limit_w, 0), info.max_sm_clock_mhz, info.max_mem_clock_mhz); + else print("gpu: none (%)\n", info.gpu_error); + if info.power_source print("power: %\n", info.power_source); + if !ort_ok { print("%\n", ort_error); exit(1); } + print("onnxruntime: % (%)\n", ort.version, ort.library_path); + + // The corpus: every file as corpus.txt pins it, or nothing runs. + progress("corpus", "-", "checking", 0, 0); + corpus, corpus_error := corpus_load(options.corpus_folder); + if corpus_error { print("\nThe corpus cannot be used: %\n", corpus_error); exit(1); } + print("corpus: % images, % megapixels, fingerprint % (decoded in % s)\n\n", corpus.images.count, FF(corpus.megapixels, 1), short(corpus.fingerprint), FF(corpus.decode_s, 2)); + + // The models, and what each is made of. + tasks := list_tasks(exe_folder); + if !tasks.count { print("No models to run: none matched -only and -models, or none are beside this program (run jai first.jai).\n"); exit(1); } + hash_cache_load(); + for * tasks fingerprint_task(it); + hash_cache_save(); + + runs: [..] Run; + for * task: tasks { + providers: [..] string; + providers.allocator = temp; + if task.kind == .LABELS_TEXT { if options.cpu array_add(*providers, "CPU"); } + else { + if options.gpu array_add(*providers, "CUDA"); + if options.cpu array_add(*providers, "CPU"); + } + for provider: providers { + run := run_task(task, provider == "CUDA", *corpus); + array_add(*runs, run); + if !run.ok { print("== % on %: %\n\n", task.name, provider, run.error); continue; } + print_run(task, run); + } + } + + cudart, cudnn := cuda_library_versions(); + path := write_results(info, started_at, *corpus, tasks, runs, cudart, cudnn); + print("Results: %\n", path); + if options.progress print("@results %\n", path); +} + +Task_Kind :: enum { OCR; LABELS; LABELS_TEXT; SEGMENT; } + +Task :: struct { + kind: Task_Kind; + model: string; // as models.config names it + name: string; // kind/model: ocr/ppocr-v5-mobile + models_folder: string; + files: [..] string; // the model's files, in models_folder + config: string; // its models.config entry, as compared + fingerprint: string; // of files and config + pinned: string; // "yes" when every file is one a fetch script pins, "no" when one differs, "-" when none are pinned +} + +Run :: struct { + task: *Task; + ok: bool; + error: string; + provider: string; // what it really ran on: a GPU run can fall back to the CPU + items: s64; // per pass + unit: string; // image or phrase + megapixels: float64; // per pass + load_s, first_s: float64; + pass_s: [..] float64; + output: string; // what the models found, in words without spaces: lines:123,chars:4567 + digest: string; // of what they found, to see whether two machines' models agree + stages: string; // where the time went, per item + batch: s64; + gpu: Gpu_Stats; + explanation: string; // why a GPU run is on the CPU +} + +Gpu_Stats :: struct { + sampled: bool; + baseline_mb, peak_mb: float64; // device memory before the engine, and the most during the timed passes + util_sum, power_sum, clock_sum: float64; + temperature_max: float64; + samples: s64; +} + +// The corpus, decoded. +Corpus :: struct { + fingerprint: string; + images: [..] Corpus_Image; + megapixels: float64; + decode_s: float64; +} + +Corpus_Image :: struct { name: string; pixels: *u8; width, height: s32; } + +// Every file corpus.txt names, checked against its hash; the error says what is wrong. +corpus_load :: (folder: string) -> Corpus, error: string { + corpus: Corpus; + manifest_path := tprint("%/corpus.txt", folder); + manifest, ok := read_entire_file(manifest_path, log_errors = false); + if !ok return corpus, tprint("% cannot be read. The build copies it, with the images, from tools/gpu_bench.", manifest_path); + fingerprint: Sha256; + bad: [..] string; + bad.allocator = temp; + start := current_time_monotonic(); + for raw: split(manifest, "\n") { + line := trim(raw); + if !line.count || line[0] == #char "#" continue; + space := find_index_from_left(line, #char " "); + if space < 0 continue; + expected := slice(line, 0, space); + name := trim(advance(line, space)); + path := tprint("%/%", folder, name); + data, read_ok := read_entire_file(path, log_errors = false); + if !read_ok { array_add(*bad, tprint("% is missing", name)); continue; } + hash := sha256_of_string(data,, temp); + if hash != expected { array_add(*bad, tprint("% is not the file corpus.txt pins (SHA-256 %)", name, hash)); free(data); continue; } + sha256_update(*fingerprint, tprint("% %\n", name, hash)); + progress("corpus", "-", "decoding", corpus.images.count + 1, 0); + image, decoded, decode_error := image_decode(cast([] u8) data); + free(data); + if !decoded { array_add(*bad, tprint("% cannot be decoded (%)", name, decode_error)); continue; } + array_add(*corpus.images, .{copy_string(name), image.pixels, image.width, image.height}); + corpus.megapixels += cast(float64) image.width * image.height / 1_000_000; + reset_temporary_storage(); + } + corpus.decode_s = seconds_since(start); + if bad.count return corpus, join(..bad, "; "); + if !corpus.images.count return corpus, tprint("% names no images.", manifest_path); + corpus.fingerprint = sha256_finish(*fingerprint); + return corpus, ""; +} + +// The models beside this program, as -only and -models choose. +list_tasks :: (exe_folder: string) -> [..] Task { + tasks: [..] Task; + add :: (tasks: *[..] Task, kind: Task_Kind, model: string, folder: string, files: .. string) { + prefix := ifx kind == .OCR then "ocr" else ifx kind == .LABELS then "labels" else ifx kind == .LABELS_TEXT then "labels-text" else "segment"; + if options.only.count && !array_find(options.only, ifx kind == .LABELS_TEXT then "labels" else prefix) return; + if options.models.count && !array_find(options.models, model) return; + task := array_add(tasks); + task.* = .{kind = kind, model = model, name = sprint("%/%", prefix, model), models_folder = folder}; + for files if it array_add(*task.files, it); + task.config = config_entry(tprint("%/models.config", folder), model); + } + ocr_folder := sprint("%ocr-models", exe_folder); + ocr_models, ocr_error := ocr_list_models(ocr_folder); + if ocr_error print("OCR: %\n", ocr_error); + for ocr_models add(*tasks, .OCR, it.name, ocr_folder, it.detector, it.recognizer, it.dictionary, it.textline_ori); + labels_folder := sprint("%label-models", exe_folder); + labels_models, labels_error := labels_list_models(labels_folder); + if labels_error print("Labels: %\n", labels_error); + for labels_models add(*tasks, .LABELS, it.name, labels_folder, it.vision, it.text, it.tokenizer); + for labels_models add(*tasks, .LABELS_TEXT, it.name, labels_folder, it.vision, it.text, it.tokenizer); + segment_folder := sprint("%segment-models", exe_folder); + segment_models, segment_error := segment_list_models(segment_folder); + if segment_error print("Segmentation: %\n", segment_error); + for segment_models add(*tasks, .SEGMENT, it.name, segment_folder, it.model, it.classes); + return tasks; +} + +// A model's [model NAME] entry, comments and spacing taken out, so only a change that matters +// changes the fingerprint. +config_entry :: (path: string, model: string) -> string { + text, ok := read_entire_file(path, log_errors = false,, temp); + if !ok return ""; + builder: String_Builder; + inside := false; + for raw: split(text, "\n",, temp) { + line := raw; + hash := find_index_from_left(line, #char "#"); + if hash >= 0 line = slice(line, 0, hash); + line = trim(line); + if !line.count continue; + if begins_with(line, "[model ") { inside = line == tprint("[model %]", model); continue; } + if !inside continue; + colon := find_index_from_left(line, #char ":"); + if colon >= 0 line = tprint("%: %", trim(slice(line, 0, colon)), trim(advance(line, colon + 1))); + print_to_builder(*builder, "%\n", line); + } + return builder_to_string(*builder); +} + +// The files the fetch scripts pin (modules/OCR/fetch_models.jai, modules/Labels/fetch_models.jai). +// Segmentation's model is made on each machine by tools/export_mask2former.py, so it has no pin: +// copy the .onnx from one machine to the others to be sure they run the same one. +Pin :: struct { name: string; sha256: string; } +PINNED :: Pin.[ + .{"ch_PP-OCRv5_det_mobile.onnx", "4d97c44a20d30a81aad087d6a396b08f786c4635742afc391f6621f5c6ae78ae"}, + .{"ch_PP-OCRv5_rec_mobile.onnx", "5825fc7ebf84ae7a412be049820b4d86d77620f204a041697b0494669b1742c5"}, + .{"ppocrv5_dict.txt", "d1979e9f794c464c0d2e0b70a7fe14dd978e9dc644c0e71f14158cdf8342af1b"}, + .{"ch_PP-LCNet_x0_25_textline_ori_cls_mobile.onnx", "54379ae5174d026780215fc748a7f31910dee36818e63d49e17dc598ecc82df7"}, + .{"ch_PP-OCRv5_det_server.onnx", "0f8846b1d4bba223a2a2f9d9b44022fbc22cc019051a602b41a7fda9667e4cad"}, + .{"ch_PP-OCRv5_rec_server.onnx", "e09385400eaaaef34ceff54aeb7c4f0f1fe014c27fa8b9905d4709b65746562a"}, + .{"siglip2-base-patch16-384-vision.onnx", "49ae4958b1098ca995e929d646f7be05a69c65e6344beae07d58c6598ffc5210"}, + .{"siglip2-base-patch16-384-text-int8.onnx", "30a94b43b9ddc39c04ab67f80cba4b808262682b2fa1a2a8b924bf1e7bd84c61"}, + .{"siglip2-tokenizer.json", "cb9140fae3ac5122c972d37adf83e1248471a38147ad76f8215c8872c6fd8322"}, +]; + +fingerprint_task :: (task: *Task) { + h: Sha256; + pinned, differs := 0, 0; + for task.files { + progress("hashing", task.name, it, it_index + 1, task.files.count); + hash := hash_file_cached(tprint("%/%", task.models_folder, it)); + if !hash hash = "missing"; + sha256_update(*h, tprint("% %\n", it, hash)); + for pin: PINNED if pin.name == it { if pin.sha256 == hash pinned += 1; else differs += 1; } + } + sha256_update(*h, task.config); + task.fingerprint = sha256_finish(*h); + task.pinned = ifx differs then "no" else ifx pinned then "yes" else "-"; +} + +run_task :: (task: *Task, prefer_gpu: bool, corpus: *Corpus) -> Run { + run: Run; + run.task = task; + run.provider = ifx prefer_gpu then "CUDA" else "CPU"; + run.unit = ifx task.kind == .LABELS_TEXT then "phrase" else "image"; + images := corpus.images; + run.items = ifx task.kind == .LABELS_TEXT then PHRASES.count else images.count; + if task.kind != .LABELS_TEXT run.megapixels = corpus.megapixels; + + sampler_start(*run.gpu); // the baseline: the device's memory before the engine + sampler_pause(); + + // The engine. + progress(task.name, run.provider, "loading", 0, 0); + load_start := current_time_monotonic(); + engine: struct { ocr: *Ocr_Engine; labels: *Labels_Engine; segment: *Segment_Engine; }; + ok: bool; + error: string; + phrase_vectors: [] float; + if task.kind == { + case .OCR; + engine.ocr, ok, error = ocr_create(task.models_folder, task.model, prefer_gpu); + if ok { + ocr_gpu, ocr_why := ocr_gpu_status(engine.ocr); + if prefer_gpu && !ocr_gpu run.explanation = copy_string(ocr_why); + run.provider = sprint("%", ocr_provider(engine.ocr)); + run.batch = engine.ocr.rec_batch; + } + case .LABELS; #through; + case .LABELS_TEXT; + engine.labels, ok, error = labels_create(task.models_folder, task.model, prefer_gpu && task.kind == .LABELS); + if ok { + labels_gpu, labels_why := labels_gpu_status(engine.labels); + if prefer_gpu && !labels_gpu run.explanation = copy_string(labels_why); + run.provider = ifx task.kind == .LABELS_TEXT then "CPU" else sprint("%", engine.labels.provider); + run.batch = ifx task.kind == .LABELS_TEXT then 1 else engine.labels.model.image_batch; + phrases := PHRASES; + phrase_vectors = labels_embed_texts(engine.labels, phrases); + } + case .SEGMENT; + engine.segment, ok, error = segment_create(task.models_folder, task.model, prefer_gpu); + if ok { + if prefer_gpu && engine.segment.provider != .CUDA run.explanation = copy_string(engine.segment.gpu_explanation); + run.provider = sprint("%", engine.segment.provider); + run.batch = ifx engine.segment.provider == .CUDA then engine.segment.model.batch_gpu else engine.segment.model.batch_cpu; + } + } + run.load_s = seconds_since(load_start); + if !ok { sampler_stop(); run.error = copy_string(error); return run; } + defer { + ocr_destroy(engine.ocr); + labels_destroy(engine.labels); + segment_destroy(engine.segment); + array_free(phrase_vectors); + } + if prefer_gpu && run.provider != "CUDA" { + // Said, and not timed: the CPU run, when asked for, is the same thing. + sampler_stop(); + run.error = tprint("not on the GPU: %", run.explanation,, context.allocator); + return run; + } + + // One pass over the corpus, or over its first image alone; what it found, and its digest. + pass :: (first_only: bool) -> output: string, digest: string #expand { + h: Sha256; + builder: String_Builder; + builder.allocator = temp; + count := ifx first_only then 1 else `run.items; + if `task.kind == { + case .OCR; + lines, characters: s64; + done := 0; + while done < count { + n := min(OCR_BATCH, count - done); + batch := NewArray(n, Ocr_Image,, temp); + for 0..n - 1 { image := `images[done + it]; batch[it] = .{image.pixels, image.width, image.height, image.width * 4}; } + results := ocr_process(`engine.ocr, batch,, temp); + for results { + if it.failed { sha256_update(*h, "failed\n"); continue; } + lines += it.lines.count; + for line: it.lines { characters += line.text.count; sha256_update(*h, line.text); sha256_update(*h, "\n"); } + } + done += n; + } + print_to_builder(*builder, "lines:%,chars:%", lines, characters); + case .LABELS; + input := NewArray(count, Labels_Image,, temp); + for 0..count - 1 { image := `images[it]; input[it] = .{image.pixels, image.width, image.height, image.width * 4}; } + vectors, error := labels_embed_images(`engine.labels, input,, temp); + if error sha256_update(*h, "failed\n"); + // Each image's best phrase: what the label list would make of it. + dims := `engine.labels.dimensions; + best_count: [PHRASES.count] s64; + for image: 0..count - 1 { + best, best_score := -1, -1.0; + for phrase: 0..PHRASES.count - 1 { + score := labels_score(`engine.labels, vectors.data + image * dims, `phrase_vectors.data + phrase * dims); + if score > best_score { best, best_score = phrase, score; } + } + sha256_update(*h, tprint("%\n", best)); + best_count[best] += 1; + } + distinct := 0; + for best_count if it distinct += 1; + print_to_builder(*builder, "best_phrases:%", distinct); + case .LABELS_TEXT; + for 0..count - 1 { + one: [1] string; + one[0] = PHRASES[it]; + vectors, error := labels_embed_texts(`engine.labels, one,, temp); // one at a time, as a search does + sha256_update(*h, ifx error then "failed\n" else "ok\n"); + } + print_to_builder(*builder, "phrases:%", count); + case .SEGMENT; + input := NewArray(count, Segment_Image,, temp); + for 0..count - 1 { image := `images[it]; input[it] = .{image.pixels, image.width, image.height, image.width * 4}; } + results := segment_images(`engine.segment, input,, temp); + segments: s64; + for results { + if it.failed { sha256_update(*h, "failed\n"); continue; } + segments += it.segments.count; + for segment: it.segments sha256_update(*h, tprint("% ", segment.class)); + sha256_update(*h, "\n"); + } + print_to_builder(*builder, "segments:%", segments); + } + return builder_to_string(*builder,, temp), sha256_finish(*h,, temp); + } + + // The first image alone: the first run pays for choosing kernels and setting up memory. + progress(task.name, run.provider, "first", 0, 0); + first_start := current_time_monotonic(); + pass(true); + run.first_s = seconds_since(first_start); + reset_temporary_storage(); + for 1..options.warmup { + progress(task.name, run.provider, "warm-up", it, options.warmup); + pass(false); + reset_temporary_storage(); + } + if task.kind == .OCR ocr_reset_stats(engine.ocr); + if task.kind == .SEGMENT engine.segment.stats = .{}; + + sampler_resume(); + for 1..options.repeats { + progress(task.name, run.provider, "timing", it, options.repeats); + start := current_time_monotonic(); + output, digest := pass(false); + array_add(*run.pass_s, seconds_since(start)); + if it == 1 { run.output = copy_string(output); run.digest = copy_string(short(digest)); } + reset_temporary_storage(); + } + sampler_stop(); + + per := cast(float64) max(run.items * options.repeats, 1) * 1000; + if task.kind == .OCR { + s := ocr_stats(engine.ocr); + run.stages = sprint("detect:%,orient:%,recognize:%", FF(s.detect_us / per, 2), FF(s.orientation_us / per, 2), FF(s.recognize_us / per, 2)); + } else if task.kind == .SEGMENT { + s := engine.segment.stats; + run.stages = sprint("prepare:%,model:%,post:%", FF(s.prepare_us / per, 2), FF(s.model_us / per, 2), FF(s.post_us / per, 2)); + } + run.ok = true; + return run; +} + +OCR_BATCH :: 8; // images per ocr_process call, as the app's scan gives them + +// Phrases for the label model: its image vectors are scored against them (the output's digest), and +// the text encoder is timed on them, one at a time, as a like: search runs it. +PHRASES :: string.[ + "a screenshot of a code editor showing source code", "a screenshot of a terminal window", + "a spreadsheet with rows and columns of numbers", "a document with paragraphs of text", + "a chat conversation on a phone", "a dialog box asking to save changes", + "a photo of a living room", "a photo of people playing tennis", + "a photo of food on a plate", "a photo of a street with cars", + "a photo of an animal", "a photo of a person", "an electronic circuit schematic diagram", + "a printed circuit board", "a graph with several colored lines", "a web page in a browser", +]; + +Median :: struct { median, best, worst: float64; } + +median_of :: (values: [] float64) -> Median { + sorted := array_copy(values,, temp); + quick_sort(sorted, (a: float64, b: float64) -> s64 { return ifx a < b then -1 else ifx a > b then 1 else 0; }); + m: Median; + if !sorted.count return m; + m.median = ifx sorted.count % 2 then sorted[sorted.count / 2] else (sorted[sorted.count / 2 - 1] + sorted[sorted.count / 2]) / 2; + m.best, m.worst = sorted[0], sorted[sorted.count - 1]; + return m; +} + +print_run :: (task: *Task, run: Run) { + m := median_of(run.pass_s); + print("== % on % (batch %)\n", task.name, run.provider, run.batch); + print(" load % s, first % % s\n", FF(run.load_s, 2), run.unit, FF(run.first_s, 3)); + print(" % ms per % (median of %; best %, worst %), % %s/s", FF(1000 * m.median / run.items, 2), run.unit, run.pass_s.count, + FF(1000 * m.best / run.items, 2), FF(1000 * m.worst / run.items, 2), FF(run.items / m.median, 1), run.unit); + if run.megapixels print(", % megapixels/s", FF(run.megapixels / m.median, 1)); + print("\n"); + if run.stages print(" per %, summed over threads (ms): %\n", run.unit, run.stages); + if run.gpu.samples print(" GPU: % MB more memory at the peak, % percent busy, % W, % MHz, up to % C\n", + FF(run.gpu.peak_mb - run.gpu.baseline_mb, 0), FF(run.gpu.util_sum / run.gpu.samples, 0), + FF(run.gpu.power_sum / run.gpu.samples, 0), FF(run.gpu.clock_sum / run.gpu.samples, 0), FF(run.gpu.temperature_max, 0)); + print(" found % (digest %)\n\n", run.output, run.digest); +} + +write_results :: (info: System_Info, started_at: Apollo_Time, corpus: *Corpus, tasks: [] Task, runs: [] Run, cudart: string, cudnn: string) -> path: string { + b: String_Builder; + print_to_builder(*b, "%\n", BENCH_RESULTS_FORMAT); + date := to_calendar(started_at, .LOCAL); + print_to_builder(*b, "date: %-%-% %:%:%\n", date.year, pad2(date.month_starting_at_0 + 1), pad2(date.day_of_month_starting_at_0 + 1), pad2(date.hour), pad2(date.minute), pad2(date.second)); + print_to_builder(*b, "machine: %\n", one_line(info.machine)); + if options.note print_to_builder(*b, "note: %\n", one_line(options.note)); + print_to_builder(*b, "os: %\n", info.os); + print_to_builder(*b, "cpu: % (% threads)\n", one_line(info.cpu), info.logical_cores); + print_to_builder(*b, "ram_gb: %\n", FF(info.ram_gb, 1)); + print_to_builder(*b, "gpu: %\n", ifx info.gpu then one_line(info.gpu) else tprint("none (%)", info.gpu_error)); + if info.gpu { + print_to_builder(*b, "vram_gb: %\n", FF(info.vram_gb, 1)); + print_to_builder(*b, "driver: % (CUDA %)\n", info.driver, info.cuda_driver); + print_to_builder(*b, "power_limit_w: %\n", FF(info.power_limit_w, 0)); + print_to_builder(*b, "max_clocks_mhz: % / %\n", info.max_sm_clock_mhz, info.max_mem_clock_mhz); + } + if info.power_source print_to_builder(*b, "power: %\n", info.power_source); + print_to_builder(*b, "onnxruntime: %\n", ort.version); + if cudart print_to_builder(*b, "cudart: %\n", cudart); + if cudnn print_to_builder(*b, "cudnn: %\n", cudnn); + print_to_builder(*b, "mcap: % (%, %)\n", VERSION, GIT_HASH, GIT_BRANCH); + print_to_builder(*b, "passes: % timed, % warm-up\n", options.repeats, options.warmup); + print_to_builder(*b, "corpus: %\n", corpus.fingerprint); + print_to_builder(*b, "corpus_size: % images, % megapixels\n", corpus.images.count, FF(corpus.megapixels, 1)); + for tasks { + print_to_builder(*b, "model: % %\n", it.name, it.fingerprint); + print_to_builder(*b, "pinned %: %\n", it.name, it.pinned); + } + for run: runs { + task := run.task; + if !run.ok { print_to_builder(*b, "error % %: %\n", task.name, run.provider, one_line(run.error)); continue; } + m := median_of(run.pass_s); + print_to_builder(*b, "result: % % unit=% items=% batch=% ms=% best_ms=% worst_ms=% per_s=% load_s=% first_s=%", + task.name, run.provider, run.unit, run.items, run.batch, FF(1000 * m.median / run.items, 3), FF(1000 * m.best / run.items, 3), + FF(1000 * m.worst / run.items, 3), FF(run.items / m.median, 2), FF(run.load_s, 2), FF(run.first_s, 3)); + if run.megapixels print_to_builder(*b, " mp_per_s=%", FF(run.megapixels / m.median, 2)); + if run.gpu.samples { + g := run.gpu; + print_to_builder(*b, " vram_mb=% gpu_util=% power_w=% sm_mhz=% temp_c=%", FF(g.peak_mb - g.baseline_mb, 0), FF(g.util_sum / g.samples, 0), + FF(g.power_sum / g.samples, 0), FF(g.clock_sum / g.samples, 0), FF(g.temperature_max, 0)); + } + if run.stages print_to_builder(*b, " stages=%", run.stages); + print_to_builder(*b, " output=% digest=%\n", run.output, run.digest); + } + name := tprint("%_%-%-%_%-%-%.txt", file_safe(info.machine), date.year, pad2(date.month_starting_at_0 + 1), pad2(date.day_of_month_starting_at_0 + 1), + pad2(date.hour), pad2(date.minute), pad2(date.second)); + path := tprint("%/%", options.out_folder, name); + if !write_entire_file(path, *b) print("Could not write %.\n", path); + return path; +} + +// For the GPU Benchmark window: where the run is. total 0: no count. +progress :: (task: string, provider: string, phase: string, done: s64, total: s64) { + if !options.progress return; + print("@progress % % % % %\n", task, provider, phase, done, total); +} + +// The GPU, sampled every SAMPLE_MS on a thread of its own while the timed passes run. +SAMPLE_MS :: 100; + +sampler: struct { + thread: Thread; + stats: *Gpu_Stats; + running: bool; // the thread is up + paused: bool; // it is up, and not sampling + mutex: Mutex; +}; + +sampler_start :: (stats: *Gpu_Stats) { + s, ok := gpu_sample(); + if !ok return; + stats.sampled = true; + stats.baseline_mb = s.memory_used_mb; + sampler.stats = stats; + init(*sampler.mutex); + sampler.running = true; + sampler.paused = false; + thread_init(*sampler.thread, (thread: *Thread) -> s64 { + while sampler.running { + sleep_milliseconds(SAMPLE_MS); + s, ok := gpu_sample(); + lock(*sampler.mutex); + if ok && !sampler.paused && sampler.running { + g := sampler.stats; + g.peak_mb = max(g.peak_mb, s.memory_used_mb); + g.util_sum += s.util_percent; + g.power_sum += s.power_w; + g.clock_sum += s.sm_clock_mhz; + g.temperature_max = max(g.temperature_max, s.temperature_c); + g.samples += 1; + } + unlock(*sampler.mutex); + } + return 0; + }); + thread_start(*sampler.thread); +} + +sampler_pause :: () { if !sampler.running return; lock(*sampler.mutex); sampler.paused = true; unlock(*sampler.mutex); } +sampler_resume :: () { if !sampler.running return; lock(*sampler.mutex); sampler.paused = false; unlock(*sampler.mutex); } + +sampler_stop :: () { + if !sampler.running return; + lock(*sampler.mutex); + sampler.running = false; + unlock(*sampler.mutex); + thread_is_done(*sampler.thread, -1); + thread_deinit(*sampler.thread); + destroy(*sampler.mutex); +} + +// Model files are hundreds of MB, so their hashes are kept in the results folder, by path, size and +// time: hashed again only when one changes. +hash_cache: [..] struct { key: string; hash: string; }; + +hash_cache_load :: () { + text, ok := read_entire_file(tprint("%/.hash-cache", options.out_folder), log_errors = false); + if !ok return; + for split(text, "\n") { + tab := find_index_from_left(it, #char "\t"); + if tab > 0 array_add(*hash_cache, .{trim(advance(it, tab + 1)), slice(it, 0, tab)}); + } +} + +hash_cache_save :: () { + b: String_Builder; + for hash_cache print_to_builder(*b, "%\t%\n", it.hash, it.key); + write_entire_file(tprint("%/.hash-cache", options.out_folder), *b); +} + +hash_file_cached :: (path: string) -> string { + modtime, size, ok := file_modtime_and_size(path); + if !ok return ""; + key := sprint("% % % %", size, modtime.low, modtime.high, path); + for hash_cache if it.key == key return it.hash; + hash := sha256_of_file(path); + if hash array_add(*hash_cache, .{key, hash}); + return hash; +} + +seconds_since :: (start: Apollo_Time) -> float64 { + return to_float64_seconds(current_time_monotonic() - start); +} + +short :: (hash: string) -> string { return slice(hash, 0, min(hash.count, 12)); } + +pad2 :: (n: s64) -> string { return tprint("%", formatInt(n, minimum_digits = 2)); } + +one_line :: (text: string) -> string { + return replace(replace(trim(text), "\r", " ",, temp), "\n", " ",, temp); +} + +file_safe :: (text: string) -> string { + result := copy_string(text,, temp); + for 0..result.count - 1 { + c := result[it]; + if !(is_alnum(c) || c == #char "-") result[it] = #char "_"; + } + return result; +} + +#load "sha256.jai"; +#load "system_info.jai"; +#load "results.jai"; + +#import "Basic"; +#import "String"; +#import "File"; +#import "File_Utilities"; +#import "Sort"; +#import "System"; +#import "Thread"; +#import "Image_Decode"; +#import "onnxruntime"; +#import "OCR"; +#import "Labels"; +#import "Segment"; diff --git a/tools/gpu_bench/make_corpus.py b/tools/gpu_bench/make_corpus.py new file mode 100644 index 0000000..e85e79c --- /dev/null +++ b/tools/gpu_bench/make_corpus.py @@ -0,0 +1,192 @@ +# Makes the synthetic part of the GPU benchmark's corpus (tools/gpu_bench/corpus): screenshot-like +# images full of text, for OCR, and two phone-sized photos scaled up from COCO ones. The images are +# committed, and corpus.txt pins each one's SHA-256, so every machine benchmarks the same pixels; this +# script is kept to show how they were made, not to be rerun (another Pillow or FreeType could draw +# them a pixel differently, and then they would no longer match corpus.txt). +# +# python tools/gpu_bench/make_corpus.py + +import os, random +from PIL import Image, ImageDraw, ImageFont + +HERE = os.path.dirname(os.path.abspath(__file__)) +OUT = os.path.join(HERE, "corpus") +FONTS = os.path.join(HERE, "..", "..", "assets", "fonts") + +WORDS = ("the of and to in is that for it as with was on be by this are from or at have an not but " + "which they you were all one their there been has more when will would can if about what " + "into some time only other new could these two may first then its such like over also back " + "after use our work way well even because any most people where much through before right " + "capture library image model search folder window screen label segment vector memory driver " + "thread allocator buffer request response latency throughput benchmark kernel tensor batch " + "session provider runtime device budget scheduler compiler release version settings export " + "import config circuit board resistor capacitor footprint schematic voltage current signal " + "amplifier channel sample filter frequency spectrum electrode impedance firmware bootloader").split() + +def font(name, size): + return ImageFont.truetype(os.path.join(FONTS, name), size) + +def sentence(rng, n): + words = [rng.choice(WORDS) for _ in range(n)] + words[0] = words[0].capitalize() + return " ".join(words) + rng.choice([".", ".", ",", ":", "?"]) + +def code_line(rng): + kind = rng.randrange(6) + a, b, c = rng.choice(WORDS), rng.choice(WORDS), rng.choice(WORDS) + if kind == 0: return f"{a}_{b} := {c}_{rng.choice(WORDS)}({rng.randrange(1000)}, {a});" + if kind == 1: return f"if {a}.{b} > {rng.randrange(64)} {{ {c}({a}); return false; }}" + if kind == 2: return f"// {sentence(rng, rng.randrange(4, 12))}" + if kind == 3: return f"for {a}: {b}_{c} array_add(*{a}s, it.{c});" + if kind == 4: return f"{a}_{b} :: (x: s64, {c}: *{a.capitalize()}) -> bool {{" + return f"print(\"% {a} {b}\\n\", {c}.count);" + +def editor(rng, w, h, dark, mono, size): + bg, fg, dim, side = ((30, 30, 30), (212, 212, 212), (110, 110, 110), (37, 37, 38)) if dark else \ + ((255, 255, 255), (30, 30, 30), (150, 150, 150), (243, 243, 243)) + im = Image.new("RGB", (w, h), bg) + d = ImageDraw.Draw(im) + ui, code = font("OpenSans-Regular.ttf", 14), font(mono, size) + d.rectangle([0, 0, w, 32], fill=side) + for i, x in enumerate(range(10, w - 200, 180)): + d.text((x, 8), f"{rng.choice(WORDS)}_{rng.choice(WORDS)}.jai", font=ui, fill=fg if i == 0 else dim) + d.rectangle([0, 32, 260, h], fill=side) + y = 44 + while y < h - 30: + d.text((16 + 12 * rng.randrange(3), y), f"{rng.choice(WORDS)}_{rng.choice(WORDS)}", font=ui, fill=fg) + y += 24 + colours = [fg, (86, 156, 214), (206, 145, 120), (106, 153, 85), (197, 134, 192)] if dark else \ + [fg, (0, 0, 255), (163, 21, 21), (0, 128, 0), (175, 0, 219)] + line, y = rng.randrange(1, 900), 44 + while y < h - 30: + d.text((272, y), str(line).rjust(4), font=code, fill=dim) + if rng.random() > 0.12: + d.text((330 + 4 * size // 2 * rng.randrange(4), y), code_line(rng), font=code, fill=rng.choice(colours)) + y += int(size * 1.5); line += 1 + d.rectangle([0, h - 24, w, h], fill=(0, 122, 204)) + d.text((10, h - 21), f"Ln {line}, Col {rng.randrange(80)} UTF-8 CRLF Jai {rng.choice(WORDS)}", font=ui, fill=(255, 255, 255)) + return im + +def document(rng, w, h, size): + im = Image.new("RGB", (w, h), (250, 250, 248)) + d = ImageDraw.Draw(im) + title, body = font("OpenSans-Bold.ttf", size * 2), font("OpenSans-Regular.ttf", size) + margin = w // 8 + d.text((margin, 40), sentence(rng, 5)[:-1], font=title, fill=(20, 20, 20)) + y = 60 + size * 3 + while y < h - size * 2: + if rng.random() < 0.15: + d.text((margin, y), sentence(rng, 4)[:-1], font=font("OpenSans-SemiBold.ttf", int(size * 1.3)), fill=(40, 40, 90)) + y += int(size * 2.2); continue + for _ in range(rng.randrange(2, 7)): + text = sentence(rng, 40) + words, current = text.split(), "" + for word in words: + trial = (current + " " + word).strip() + if d.textlength(trial, font=body) > w - 2 * margin: + d.text((margin, y), current, font=body, fill=(30, 30, 30)); y += int(size * 1.6); current = word + else: + current = trial + d.text((margin, y), current, font=body, fill=(30, 30, 30)); y += int(size * 1.6) + if y > h - size * 2: break + y += size + return im + +def spreadsheet(rng, w, h): + im = Image.new("RGB", (w, h), (255, 255, 255)) + d = ImageDraw.Draw(im) + f, bold = font("OpenSans-Regular.ttf", 13), font("OpenSans-SemiBold.ttf", 13) + cols = [40] + [rng.choice([90, 110, 140, 180]) for _ in range(30)] + d.rectangle([0, 0, w, 60], fill=(33, 115, 70)) + d.text((12, 18), f"{rng.choice(WORDS)}_{rng.choice(WORDS)}.xlsx", font=font("OpenSans-SemiBold.ttf", 18), fill=(255, 255, 255)) + y, row = 60, 0 + while y < h: + x = 0 + for c, cw in enumerate(cols): + if x > w: break + d.rectangle([x, y, x + cw, y + 22], outline=(218, 220, 224), fill=(243, 243, 243) if row == 0 or c == 0 else None) + if row == 0 and c > 0: text, ft = chr(64 + c) if c <= 26 else "A" + chr(38 + c), bold + elif c == 0: text, ft = str(row) if row else "", bold + elif row == 1: text, ft = rng.choice(WORDS).capitalize(), bold + else: + r = rng.random() + text = f"{rng.uniform(0, 99999):,.2f}" if r < 0.5 else (str(rng.randrange(10000)) if r < 0.7 else rng.choice(WORDS)) + ft = f + d.text((x + 4, y + 3), text, font=ft, fill=(20, 20, 20)) + x += cw + y += 22; row += 1 + return im + +def terminal(rng, w, h, size): + im = Image.new("RGB", (w, h), (12, 12, 12)) + d = ImageDraw.Draw(im) + f = font("FiraCode-Retina.ttf", size) + y = 8 + while y < h - size: + r = rng.random() + if r < 0.2: text, colour = f"PS D:\\{rng.choice(WORDS)}\\{rng.choice(WORDS)}> jai first.jai - {rng.choice(WORDS)}", (240, 240, 240) + elif r < 0.3: text, colour = f"error: {sentence(rng, 8)}", (240, 80, 80) + elif r < 0.4: text, colour = f"[{rng.randrange(24):02}:{rng.randrange(60):02}:{rng.randrange(60):02}] {rng.choice(['INFO', 'WARN', 'DEBUG'])} {sentence(rng, 9)}", (180, 220, 120) + else: text, colour = " ".join(f"{rng.randrange(1 << 16):04x}" for _ in range(rng.randrange(4, 18))), (200, 200, 200) + d.text((8, y), text, font=f, fill=colour) + y += int(size * 1.35) + return im + +def chat(rng, w, h): + im = Image.new("RGB", (w, h), (245, 245, 250)) + d = ImageDraw.Draw(im) + f, name = font("OpenSans-Regular.ttf", 30), font("OpenSans-Bold.ttf", 26) + d.rectangle([0, 0, w, 140], fill=(60, 60, 140)) + d.text((40, 60), rng.choice(WORDS).capitalize() + " " + rng.choice(WORDS).capitalize(), font=font("OpenSans-SemiBold.ttf", 40), fill=(255, 255, 255)) + y = 170 + while y < h - 200: + mine = rng.random() < 0.5 + lines = [sentence(rng, rng.randrange(2, 6)) for _ in range(rng.randrange(1, 4))] + width = max(d.textlength(l, font=f) for l in lines) + 40 + height = 44 * len(lines) + 56 + x = w - width - 40 if mine else 40 + d.rounded_rectangle([x, y, x + width, y + height], 24, fill=(90, 120, 230) if mine else (255, 255, 255)) + d.text((x + 20, y + 10), f"{rng.randrange(1, 13)}:{rng.randrange(60):02} {'PM' if rng.random() < 0.5 else 'AM'}", font=name, fill=(230, 230, 255) if mine else (120, 120, 120)) + for i, l in enumerate(lines): + d.text((x + 20, y + 48 + 44 * i), l, font=f, fill=(255, 255, 255) if mine else (20, 20, 20)) + y += height + 24 + return im + +def dialog(rng, w, h): + im = editor(rng, w, h, True, "JetBrainsMono-Regular.ttf", 14).point(lambda v: v // 2) + d = ImageDraw.Draw(im) + x0, y0, x1, y1 = w // 2 - 320, h // 2 - 160, w // 2 + 320, h // 2 + 160 + d.rectangle([x0, y0, x1, y1], fill=(250, 250, 250), outline=(120, 120, 120)) + d.text((x0 + 24, y0 + 20), sentence(rng, 4)[:-1], font=font("OpenSans-SemiBold.ttf", 22), fill=(20, 20, 20)) + for i in range(4): + d.text((x0 + 24, y0 + 70 + 30 * i), sentence(rng, 9), font=font("OpenSans-Regular.ttf", 17), fill=(40, 40, 40)) + for i, label in enumerate(["Cancel", "Don't save", "Save"]): + bx = x1 - 130 * (3 - i) - 10 + d.rectangle([bx, y1 - 60, bx + 115, y1 - 22], fill=(0, 103, 192) if label == "Save" else (225, 225, 225)) + d.text((bx + 20, y1 - 52), label, font=font("OpenSans-Regular.ttf", 17), fill=(255, 255, 255) if label == "Save" else (20, 20, 20)) + return im + +def main(): + rng = random.Random(20261017) + made = [ + ("screen-editor-dark-1080p.png", editor(rng, 1920, 1080, True, "JetBrainsMono-Regular.ttf", 15)), + ("screen-editor-light-1080p.png", editor(rng, 1920, 1080, False, "RobotoMono-Regular.ttf", 14)), + ("screen-editor-dark-1440p.png", editor(rng, 2560, 1440, True, "FiraCode-Retina.ttf", 16)), + ("screen-document-1080p.png", document(rng, 1920, 1080, 18)), + ("screen-document-4k.png", document(rng, 3840, 2160, 28)), + ("screen-spreadsheet-1080p.png", spreadsheet(rng, 1920, 1080)), + ("screen-spreadsheet-1440p.png", spreadsheet(rng, 2560, 1440)), + ("screen-terminal-1080p.png", terminal(rng, 1920, 1080, 15)), + ("screen-terminal-720p.png", terminal(rng, 1280, 720, 13)), + ("screen-chat-phone.png", chat(rng, 1080, 2400)), + ("screen-dialog-1080p.png", dialog(rng, 1920, 1080)), + ("screen-document-strip.png", document(rng, 1600, 300, 16)), + ] + for name, im in made: + im.save(os.path.join(OUT, name), optimize=False, compress_level=6) + # Phone-sized photos: the decoders and resizers see a 12-megapixel image, as from a phone. + for source, name in [("coco-000000000139.jpg", "photo-12mp-livingroom.jpg"), ("coco-000000001000.jpg", "photo-12mp-tennis.jpg")]: + im = Image.open(os.path.join(OUT, source)).convert("RGB").resize((4032, 3024), Image.LANCZOS) + im.save(os.path.join(OUT, name), quality=92) + +main() diff --git a/tools/gpu_bench/results.jai b/tools/gpu_bench/results.jai new file mode 100644 index 0000000..f1501c7 --- /dev/null +++ b/tools/gpu_bench/results.jai @@ -0,0 +1,91 @@ +// gpu-bench's results file: what gpu-bench writes, and what mcap's GPU Benchmark window reads to put +// several machines side by side. Plain text, one fact a line: +// +// gpu-bench 1 +// machine: NAME a header line: key, colon, value +// ... +// model: ocr/ppocr-v5-mobile 1a2b... the fingerprint of the files a task's model is made of +// result: ocr/ppocr-v5-mobile CUDA ms_per_image=12.3 images_per_s=81.2 ... +// +// Two results are comparable only when their corpus line (the images) and each task's model line +// match; the window marks them where they do not. + +BENCH_RESULTS_FORMAT :: "gpu-bench 1"; + +Bench_Pair :: struct { key, value: string; } + +Bench_Row :: struct { + task: string; // ocr/ppocr-v5-mobile, labels/siglip2-base-patch16-384, segment/... + provider: string; // CUDA or CPU + fields: [..] Bench_Pair; +} + +Bench_Report :: struct { + path: string; + header: [..] Bench_Pair; // in the file's order; model lines too, as "model ocr/x" -> hash + rows: [..] Bench_Row; +} + +bench_header :: (report: *Bench_Report, key: string) -> string { + for report.header if it.key == key return it.value; + return ""; +} + +bench_field :: (row: *Bench_Row, key: string) -> string { + for row.fields if it.key == key return it.value; + return ""; +} + +bench_row :: (report: *Bench_Report, task: string, provider: string) -> *Bench_Row { + for * report.rows if it.task == task && it.provider == provider return it; + return null; +} + +// A results file, or ok false and why; from the context allocator. +bench_parse :: (text: string, path := "") -> Bench_Report, ok: bool, error: string { + report: Bench_Report; + report.path = copy_string(path); + lines := split(text, "\n",, temp); + if !lines.count || trim(lines[0]) != BENCH_RESULTS_FORMAT return report, false, tprint("% is not a gpu-bench results file (no \"%\" on its first line).", path, BENCH_RESULTS_FORMAT); + for raw: lines { + if it_index == 0 continue; + line := trim(raw); + if !line.count || line[0] == #char "#" continue; + colon := find_index_from_left(line, #char ":"); + if colon < 0 continue; + key := trim(slice(line, 0, colon)); + value := trim(advance(line, colon + 1)); + if key == "result" { + parts := split(value, " ",, temp); + if parts.count < 2 continue; + row := array_add(*report.rows); + row.* = .{task = copy_string(parts[0]), provider = copy_string(parts[1])}; + for 2..parts.count - 1 { + part := parts[it]; + equals := find_index_from_left(part, #char "="); + if equals < 0 continue; + array_add(*row.fields, .{copy_string(slice(part, 0, equals)), copy_string(advance(part, equals + 1))}); + } + } else if key == "model" { + space := find_index_from_left(value, #char " "); + if space < 0 continue; + array_add(*report.header, .{tprint("model %", slice(value, 0, space),, context.allocator), copy_string(trim(advance(value, space + 1)))}); + } else { + array_add(*report.header, .{copy_string(key), copy_string(value)}); + } + } + return report, true, ""; +} + +bench_report_free :: (report: *Bench_Report) { + free(report.path); + for report.header { free(it.key); free(it.value); } + array_free(report.header); + for * report.rows { + free(it.task); free(it.provider); + for it.fields { free(it.key); free(it.value); } + array_free(it.fields); + } + array_free(report.rows); + report.* = .{}; +} diff --git a/tools/gpu_bench/sha256.jai b/tools/gpu_bench/sha256.jai new file mode 100644 index 0000000..2867a51 --- /dev/null +++ b/tools/gpu_bench/sha256.jai @@ -0,0 +1,118 @@ +// SHA-256 (FIPS 180-4), for gpu-bench's corpus and model fingerprints: the same hashes as the +// fetch scripts check (certutil, sha256sum), so they can be read against each other. + +Sha256 :: struct { + state: [8] u32 = .[0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19]; + block: [64] u8; + used: s64; // bytes in block + total: u64; // bytes hashed +} + +sha256_update :: (h: *Sha256, data: [] u8) { + bytes := data; + h.total += cast(u64) bytes.count; + if h.used { + take := min(64 - h.used, bytes.count); + memcpy(h.block.data + h.used, bytes.data, take); + h.used += take; + bytes = array_view(bytes, take, bytes.count - take); + if h.used < 64 return; + sha256_blocks(h, h.block.data, 1); + h.used = 0; + } + whole := bytes.count / 64; + if whole sha256_blocks(h, bytes.data, whole); + rest := bytes.count - whole * 64; + if rest { memcpy(h.block.data, bytes.data + whole * 64, rest); h.used = rest; } +} + +sha256_update :: (h: *Sha256, text: string) { sha256_update(h, cast([] u8) text); } + +// Lowercase hex, 64 characters; from allocator. +sha256_finish :: (h: *Sha256, allocator := context.allocator) -> string { + bits := h.total * 8; + pad: [72] u8; + pad[0] = 0x80; + pad_count := ifx h.used < 56 then 56 - h.used else 120 - h.used; + for 0..7 pad[pad_count + it] = cast,trunc(u8) (bits >> cast(u64) (56 - 8 * it)); + total := h.total; + sha256_update(h, array_view(pad, 0, pad_count + 8)); + h.total = total; + HEX :: "0123456789abcdef"; + result := alloc_string(64,, allocator); + for word, w: h.state for b: 0..3 { + byte := (word >> cast(u32) (24 - 8 * b)) & 0xff; + result[w * 8 + b * 2] = HEX[byte >> 4]; + result[w * 8 + b * 2 + 1] = HEX[byte & 0xf]; + } + return result; +} + +sha256_of_string :: (text: string, allocator := context.allocator) -> string { + h: Sha256; + sha256_update(*h, text); + return sha256_finish(*h, allocator); +} + +// The file's hash, read a few MB at a time; "" when it cannot be read. +sha256_of_file :: (path: string, allocator := context.allocator) -> string { + file, ok := file_open(path, log_errors = false); + if !ok return ""; + defer file_close(*file); + CHUNK :: 4 * 1024 * 1024; + buffer := NewArray(CHUNK, u8,, context.default_allocator); + defer array_free(buffer,, context.default_allocator); + h: Sha256; + size := file_length(file); + done: s64; + while done < size { + count := min(CHUNK, size - done); + if !file_read(file, buffer.data, count) return ""; + sha256_update(*h, array_view(buffer, 0, count)); + done += count; + } + return sha256_finish(*h, allocator); +} + +#scope_file + +K :: u32.[ + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, +]; + +rotr :: inline (x: u32, n: u32) -> u32 #no_aoc { return (x >> n) | (x << (32 - n)); } + +sha256_blocks :: (h: *Sha256, data: *u8, count: s64) #no_aoc { + w: [64] u32 = ---; + for block: 0..count - 1 { + p := data + block * 64; + for i: 0..15 { + q := p + i * 4; + w[i] = ((cast(u32) q[0]) << 24) | ((cast(u32) q[1]) << 16) | ((cast(u32) q[2]) << 8) | (cast(u32) q[3]); + } + for i: 16..63 { + s0 := rotr(w[i - 15], 7) ^ rotr(w[i - 15], 18) ^ (w[i - 15] >> 3); + s1 := rotr(w[i - 2], 17) ^ rotr(w[i - 2], 19) ^ (w[i - 2] >> 10); + w[i] = w[i - 16] + s0 + w[i - 7] + s1; + } + a, b, c, d, e, f, g, hh := h.state[0], h.state[1], h.state[2], h.state[3], h.state[4], h.state[5], h.state[6], h.state[7]; + for i: 0..63 { + S1 := rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25); + ch := (e & f) ^ (~e & g); + t1 := hh + S1 + ch + K[i] + w[i]; + S0 := rotr(a, 2) ^ rotr(a, 13) ^ rotr(a, 22); + maj := (a & b) ^ (a & c) ^ (b & c); + t2 := S0 + maj; + hh = g; g = f; f = e; e = d + t1; d = c; c = b; b = a; a = t1 + t2; + } + h.state[0] += a; h.state[1] += b; h.state[2] += c; h.state[3] += d; + h.state[4] += e; h.state[5] += f; h.state[6] += g; h.state[7] += hh; + } +} diff --git a/tools/gpu_bench/system_info.jai b/tools/gpu_bench/system_info.jai new file mode 100644 index 0000000..a22b738 --- /dev/null +++ b/tools/gpu_bench/system_info.jai @@ -0,0 +1,214 @@ +// What gpu-bench says about the machine: the computer, CPU and memory, and the NVIDIA GPU through +// NVML (the driver's management library, nvml.dll / libnvidia-ml.so.1), which also gives the GPU's +// memory, load, temperature, power and clock while the models run. Without NVML (no NVIDIA driver) +// the GPU lines say so and the samples are empty. + +System_Info :: struct { + machine: string; + os: string; + cpu: string; + logical_cores: s64; + ram_gb: float64; + gpu: string; // "" without NVML + vram_gb: float64; + driver: string; + cuda_driver: string; // the CUDA version the driver supports, such as 13.0 + power_limit_w: float64; // the board's enforced limit: a laptop's GPU is often far below the desktop part's + max_sm_clock_mhz: s64; + max_mem_clock_mhz: s64; + gpu_error: string; // why there is no GPU information + power_source: string; // a laptop's: "AC" or "battery", which can change its GPU's speed several times over +} + +// One sample of the GPU while a model runs. +Gpu_Sample :: struct { + memory_used_mb: float64; // the whole device's, every process's + util_percent: float64; + temperature_c: float64; + power_w: float64; + sm_clock_mhz: float64; +} + +system_info :: () -> System_Info { + info: System_Info; + info.logical_cores = get_number_of_processors(.ALL_LOGICAL); + #if OS == .WINDOWS { + name: [256] u16; + size: u32 = name.count; + if GetComputerNameW(name.data, *size) info.machine = wide_to_utf8(name.data, cast(s32) size); + v: OSVERSIONINFOW; + v.dwOSVersionInfoSize = size_of(OSVERSIONINFOW); + RtlGetVersion(*v); + info.os = sprint("Windows %.%.%", v.dwMajorVersion, v.dwMinorVersion, v.dwBuildNumber); + buffer: [256] u8; + bytes: u32 = buffer.count; + if RegGetValueA(HKEY_LOCAL_MACHINE, "HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0", "ProcessorNameString", RRF_RT_REG_SZ, null, buffer.data, *bytes) == 0 + info.cpu = copy_string(trim(to_string(buffer.data))); + memory: MEMORYSTATUSEX; + memory.dwLength = size_of(MEMORYSTATUSEX); + if GlobalMemoryStatusEx(*memory) info.ram_gb = memory.ullTotalPhys / (1024.0 * 1024 * 1024); + power: SYSTEM_POWER_STATUS; + if GetSystemPowerStatus(*power) && power.BatteryFlag != 128 { // 128: no battery + if power.ACLineStatus == 1 info.power_source = "AC"; + else if power.ACLineStatus == 0 info.power_source = sprint("battery (% percent)", power.BatteryLifePercent); + } + } else #if OS == .LINUX { + host, host_ok := read_entire_file("/etc/hostname", log_errors = false); + if host_ok info.machine = trim(host); + release, release_ok := read_entire_file("/proc/sys/kernel/osrelease", log_errors = false); + info.os = ifx release_ok then sprint("Linux %", trim(release)) else "Linux"; + cpuinfo, cpu_ok := read_entire_file("/proc/cpuinfo", log_errors = false); + if cpu_ok for split(cpuinfo, "\n",, temp) if begins_with(it, "model name") { info.cpu = trim(advance(it, find_index_from_left(it, #char ":") + 1)); break; } + meminfo, mem_ok := read_entire_file("/proc/meminfo", log_errors = false); + if mem_ok for split(meminfo, "\n",, temp) if begins_with(it, "MemTotal:") { info.ram_gb = string_to_int(trim(trim(advance(it, 9)), " kB")) / (1024.0 * 1024); break; } + } else { + info.os = tprint("%", OS); + } + if !info.machine info.machine = "unknown"; + + if !nvml_start() { info.gpu_error = nvml.error; return info; } + text: [96] u8; + if nvml.DeviceGetName(nvml.device, text.data, text.count) == 0 info.gpu = copy_string(to_string(text.data)); + if nvml.SystemGetDriverVersion(text.data, text.count) == 0 info.driver = copy_string(to_string(text.data)); + cuda: s32; + if nvml.SystemGetCudaDriverVersion(*cuda) == 0 info.cuda_driver = sprint("%.%", cuda / 1000, (cuda % 1000) / 10); + vram: Nvml_Memory; + if nvml.DeviceGetMemoryInfo(nvml.device, *vram) == 0 info.vram_gb = vram.total / (1024.0 * 1024 * 1024); + limit: u32; + if nvml.DeviceGetEnforcedPowerLimit(nvml.device, *limit) == 0 info.power_limit_w = limit / 1000.0; + clock: u32; + if nvml.DeviceGetMaxClockInfo(nvml.device, NVML_CLOCK_SM, *clock) == 0 info.max_sm_clock_mhz = clock; + if nvml.DeviceGetMaxClockInfo(nvml.device, NVML_CLOCK_MEM, *clock) == 0 info.max_mem_clock_mhz = clock; + return info; +} + +// The GPU now; false without NVML. +gpu_sample :: () -> Gpu_Sample, ok: bool { + s: Gpu_Sample; + if !nvml.started return s, false; + memory: Nvml_Memory; + if nvml.DeviceGetMemoryInfo(nvml.device, *memory) == 0 s.memory_used_mb = memory.used / (1024.0 * 1024); + util: Nvml_Utilization; + if nvml.DeviceGetUtilizationRates(nvml.device, *util) == 0 s.util_percent = cast(float64) util.gpu; + value: u32; + if nvml.DeviceGetTemperature(nvml.device, NVML_TEMPERATURE_GPU, *value) == 0 s.temperature_c = cast(float64) value; + if nvml.DeviceGetPowerUsage(nvml.device, *value) == 0 s.power_w = value / 1000.0; + if nvml.DeviceGetClockInfo(nvml.device, NVML_CLOCK_SM, *value) == 0 s.sm_clock_mhz = cast(float64) value; + return s, true; +} + +// The CUDA runtime's and cuDNN's versions, as loaded into this process by onnxruntime's CUDA provider, +// with where each came from; "" for one not loaded (before a GPU engine is made). +cuda_library_versions :: () -> cudart: string, cudnn: string { + cudart, cudnn: string; + #if OS == .WINDOWS { + module := GetModuleHandleA("cudart64_13.dll"); + if module { + get := cast(#type (version: *s32) -> s32 #c_call) GetProcAddress(module, "cudaRuntimeGetVersion"); + version: s32; + if get && get(*version) == 0 cudart = sprint("%.% (%)", version / 1000, (version % 1000) / 10, module_path(module)); + } + module = GetModuleHandleA("cudnn64_9.dll"); + if module { + get := cast(#type () -> u64 #c_call) GetProcAddress(module, "cudnnGetVersion"); + if get { + version := get(); // 9.x: major * 10000 + minor * 100 + patch + cudnn = sprint("%.%.% (%)", version / 10000, (version % 10000) / 100, version % 100, module_path(module)); + } + } + } + return cudart, cudnn; +} + +#scope_file + +#if OS == .WINDOWS { + SYSTEM_POWER_STATUS :: struct { + ACLineStatus, BatteryFlag, BatteryLifePercent, SystemStatusFlag: u8; + BatteryLifeTime, BatteryFullLifeTime: u32; + } + GetSystemPowerStatus :: (status: *SYSTEM_POWER_STATUS) -> BOOL #foreign kernel32; + kernel32 :: #system_library "kernel32"; + + module_path :: (module: HANDLE) -> string { + name: [1024] u16; + count := GetModuleFileNameW(module, name.data, name.count); + return wide_to_utf8(name.data, count,, temp); + } +} + +Nvml_Device :: *void; +Nvml_Memory :: struct { total, free, used: u64; } +Nvml_Utilization :: struct { gpu, memory: u32; } +NVML_CLOCK_SM :: 1; +NVML_CLOCK_MEM :: 2; +NVML_TEMPERATURE_GPU :: 0; + +nvml: struct { + started: bool; + error: string; + device: Nvml_Device; + + Init: #type () -> s32 #c_call; + DeviceGetHandleByIndex: #type (index: u32, device: *Nvml_Device) -> s32 #c_call; + DeviceGetName: #type (device: Nvml_Device, name: *u8, length: u32) -> s32 #c_call; + SystemGetDriverVersion: #type (version: *u8, length: u32) -> s32 #c_call; + SystemGetCudaDriverVersion: #type (version: *s32) -> s32 #c_call; + DeviceGetMemoryInfo: #type (device: Nvml_Device, memory: *Nvml_Memory) -> s32 #c_call; + DeviceGetUtilizationRates: #type (device: Nvml_Device, util: *Nvml_Utilization) -> s32 #c_call; + DeviceGetTemperature: #type (device: Nvml_Device, sensor: s32, value: *u32) -> s32 #c_call; + DeviceGetPowerUsage: #type (device: Nvml_Device, milliwatts: *u32) -> s32 #c_call; + DeviceGetEnforcedPowerLimit: #type (device: Nvml_Device, milliwatts: *u32) -> s32 #c_call; + DeviceGetClockInfo: #type (device: Nvml_Device, type: s32, mhz: *u32) -> s32 #c_call; + DeviceGetMaxClockInfo: #type (device: Nvml_Device, type: s32, mhz: *u32) -> s32 #c_call; +} + +// NVML, and its first GPU: the one onnxruntime's CUDA provider uses (device 0) on a machine with one. +nvml_start :: () -> bool { + if nvml.started return true; + #if OS == .WINDOWS { + library := LoadLibraryA("nvml.dll"); + if !library library = LoadLibraryA("C:\\Program Files\\NVIDIA Corporation\\NVSMI\\nvml.dll"); + if !library { nvml.error = "nvml.dll was not found: no NVIDIA driver."; return false; } + find :: (name: *u8) -> *void #expand { return GetProcAddress(library, name); } + } else #if OS == .LINUX { + library := dlopen("libnvidia-ml.so.1", RTLD_NOW); + if !library { nvml.error = "libnvidia-ml.so.1 was not found: no NVIDIA driver."; return false; } + find :: (name: *u8) -> *void #expand { return dlsym(library, name); } + } else { + nvml.error = "No NVML on this OS."; + return false; + } + nvml.Init = xx find("nvmlInit_v2"); + nvml.DeviceGetHandleByIndex = xx find("nvmlDeviceGetHandleByIndex_v2"); + nvml.DeviceGetName = xx find("nvmlDeviceGetName"); + nvml.SystemGetDriverVersion = xx find("nvmlSystemGetDriverVersion"); + nvml.SystemGetCudaDriverVersion = xx find("nvmlSystemGetCudaDriverVersion"); + nvml.DeviceGetMemoryInfo = xx find("nvmlDeviceGetMemoryInfo"); + nvml.DeviceGetUtilizationRates = xx find("nvmlDeviceGetUtilizationRates"); + nvml.DeviceGetTemperature = xx find("nvmlDeviceGetTemperature"); + nvml.DeviceGetPowerUsage = xx find("nvmlDeviceGetPowerUsage"); + nvml.DeviceGetEnforcedPowerLimit = xx find("nvmlDeviceGetEnforcedPowerLimit"); + nvml.DeviceGetClockInfo = xx find("nvmlDeviceGetClockInfo"); + nvml.DeviceGetMaxClockInfo = xx find("nvmlDeviceGetMaxClockInfo"); + if !nvml.Init || !nvml.DeviceGetHandleByIndex || !nvml.DeviceGetName || !nvml.SystemGetDriverVersion || !nvml.SystemGetCudaDriverVersion + || !nvml.DeviceGetMemoryInfo || !nvml.DeviceGetUtilizationRates || !nvml.DeviceGetTemperature || !nvml.DeviceGetPowerUsage + || !nvml.DeviceGetEnforcedPowerLimit || !nvml.DeviceGetClockInfo || !nvml.DeviceGetMaxClockInfo { + nvml.error = "NVML lacks a function gpu-bench needs: an old driver?"; + return false; + } + status := nvml.Init(); + if status != 0 { nvml.error = tprint("NVML did not start (error %).", status); return false; } + status = nvml.DeviceGetHandleByIndex(0, *nvml.device); + if status != 0 { nvml.error = tprint("NVML found no GPU (error %).", status); return false; } + nvml.started = true; + return true; +} + +#if OS == .WINDOWS { + #import "Windows"; + #import "Windows_Utf8"; + #import "Windows_Registry"; +} else #if OS == .LINUX { + #import "POSIX"; +}