Musa Mahmood
8c8b2c6e38
OCR filled the GPU: 9 GB for the mobile model on gpu-bench's 30 images, past 16 GB for the server model, which Windows then paged into shared memory (1.5 to 11 s a capture). onnxruntime's logged arena extensions showed why: the arena keeps every block it takes from CUDA and never joins or frees them, and OCR's inputs grow from run to run (recognizer batches went narrowest first; images in any order across calls), so each larger run left another block behind: 5.7 GB of recognizer blocks where the widest batch needs 0.9 GB. - Detection runs the largest images first, recognition the widest batches. - The detector's and recognizer's arenas shrink once each stage of an ocr_process call is done, by a run on a small input with memory.enable_memory_arena_shrinkage. Shrinking after every run cost 33 to 59% of the speed. - The worker threads are made with the engine and kept, not made for every stage of every call (onnxruntime keeps GPU state for each thread that runs a session, microsoft/onnxruntime#33037); 6% faster. - OCR's arenas grow by what is asked (kSameAsRequested). - ort.log_level sets what the engines' onnxruntime environments log; INFO shows every arena extension. Mobile OCR on gpu-bench (RTX 5080 laptop, 4 workers): peak 5.7 GB over the baseline, 2 GB between calls, 142 ms a capture, against 9.0 GB and 126 ms. The text read differs by a line in 2,986 (batches are grouped differently). The server model is not yet measured again.
72 lines
2.6 KiB
Plaintext
72 lines
2.6 KiB
Plaintext
// Type-checks src/App.jai for another OS without producing or linking a binary, to catch errors
|
|
// in code your machine does not build:
|
|
//
|
|
// jai check_targets.jai - linux (from any OS)
|
|
// jai check_targets.jai - macos (from a Mac only: the Objective_C module runs code at compile time)
|
|
//
|
|
// The constants below stand in for the ones first.jai generates; keep their names in step.
|
|
#run,stallable {
|
|
set_build_options_dc(.{do_output = false});
|
|
args := get_build_options().compile_time_command_line;
|
|
target := Operating_System_Tag.LINUX;
|
|
cpu := CPU_Tag.X64;
|
|
if array_find(args, "macos") { target = .MACOS; cpu = .ARM64; }
|
|
|
|
options := get_build_options();
|
|
options.os_target = target;
|
|
options.cpu_target = cpu;
|
|
options.output_type = .NO_OUTPUT;
|
|
options.backend = .LLVM;
|
|
options.output_executable_name = "check";
|
|
options.output_path = ".build";
|
|
|
|
// rpmalloc (modules/Default_Allocator) also runs at compile time, with the target's OS layer,
|
|
// which crashes on another OS. A check for another OS uses Jai's own allocator instead, through a
|
|
// module of the same name ahead of modules/ (with the thread_idle the app calls, doing nothing).
|
|
if target != OS {
|
|
jai_modules := trim_right(options.import_path[options.import_path.count - 1], "/");
|
|
shim := tprint("%/.build/check_modules", get_working_directory());
|
|
make_directory_if_it_does_not_exist(tprint("%/Default_Allocator", shim), recursive = true);
|
|
write_entire_file(tprint("%/Default_Allocator/module.jai", shim), tprint(#string SHIM
|
|
// Written by check_targets.jai: Jai's own allocator, for type-checking another OS.
|
|
#load "%/Default_Allocator/module.jai";
|
|
thread_idle :: () {}
|
|
SHIM, jai_modules));
|
|
paths: [..] string;
|
|
array_add(*paths, shim);
|
|
array_add(*paths, ..options.import_path);
|
|
options.import_path = paths;
|
|
}
|
|
|
|
w := compiler_create_workspace("check");
|
|
set_build_options(options, w);
|
|
compiler_begin_intercept(w);
|
|
add_build_file("src/App.jai", w);
|
|
add_build_string(#string S
|
|
APP_NAME :: "mcap";
|
|
APP_FOLDER_NAME :: "mcap";
|
|
GIT_BRANCH :: "check";
|
|
GIT_HASH :: "-";
|
|
VERSION :: "0.1.24";
|
|
BUILD_DATE :: "-";
|
|
DEBUG :: true;
|
|
COMPILE_TESTS :: true;
|
|
COMPILE_ARGS :: "";
|
|
NO_CRASH_DIALOG :: false;
|
|
S, w);
|
|
while true {
|
|
message := compiler_wait_for_message();
|
|
if message.kind == .COMPLETE {
|
|
m := cast(*Message_Complete) message;
|
|
log("% check: %", target, m.error_code);
|
|
break;
|
|
}
|
|
}
|
|
compiler_end_intercept(w);
|
|
}
|
|
|
|
#import "Basic";
|
|
#import "Compiler";
|
|
#import "File";
|
|
#import "String";
|