Musa Mahmood
4c2a39ed55
What the engine measured its runs to need (detector MB fixed + per megapixel, recognizer per 1000 line-pixels), the limit at the end, and budget_waits, the runs that waited for GPU memory, in the stages.
433 lines
19 KiB
Plaintext
433 lines
19 KiB
Plaintext
MIN_COMPILER_VERSION :: Version_Info.{ major = 0, minor = 2, micro = 30 };
|
|
MAX_COMPILER_VERSION :: Version_Info.{ major = 0, minor = 2, micro = 99 };
|
|
|
|
// Rename the app here. APP_NAME is what people see: the window title, the tray tooltip and the
|
|
// macOS bundle. APP_FOLDER_NAME names the executable and the per-user data folder, so it should be
|
|
// one lowercase word. APP_BUNDLE_IDENTIFIER is macOS's reverse-DNS id for the bundle.
|
|
APP_NAME :: "mcap";
|
|
APP_FOLDER_NAME :: "mcap";
|
|
APP_BUNDLE_IDENTIFIER :: "com.example.mcap";
|
|
|
|
VERSION :: "0.1.28";
|
|
// Where everything goes: bin/, or bin-headless/ for builds made with no_crash_dialog, so automated
|
|
// builds never write over the ones you run by hand.
|
|
build_directory := "bin";
|
|
|
|
base_options: Build_Options;
|
|
|
|
// A workspace that has been started but whose COMPLETE message has not yet been seen.
|
|
Pending_Build :: struct {
|
|
workspace: Workspace;
|
|
executable_name: string;
|
|
optimized: bool;
|
|
failed: bool;
|
|
tool: bool; // a command-line tool, not the app: no icon, manifest or bundle
|
|
}
|
|
|
|
#run,stallable build_project();
|
|
|
|
build_project :: () {
|
|
check_compiler_version();
|
|
#if OS == .MACOS && CPU != .ARM64 {
|
|
compiler_report("This app builds for macOS on Apple silicon (arm64) only; run a native jai, not one under Rosetta (arch -x86_64).");
|
|
}
|
|
|
|
base_options = get_build_options();
|
|
args := base_options.compile_time_command_line;
|
|
|
|
full_release := array_find(args, "full_release");
|
|
build_release := full_release || array_find(args, "release");
|
|
build_debug := full_release || !build_release;
|
|
|
|
if full_release log("Building debug and release");
|
|
else if build_release log("Building in release mode");
|
|
else log("Building in debug mode");
|
|
if build_release && array_find(args, "compile_tests") log("Compiling tests into the release build");
|
|
// jai first.jai - no_crash_dialog [...]: for automated runs. NO_CRASH_DIALOG makes the app
|
|
// headless whatever its arguments (no message boxes; see runtime_support_headless), and the
|
|
// build goes to bin-headless/ instead of bin/.
|
|
if array_find(args, "no_crash_dialog") {
|
|
build_directory = "bin-headless";
|
|
log("Building with NO_CRASH_DIALOG into %/", build_directory);
|
|
}
|
|
set_build_options_dc(.{do_output = false});
|
|
|
|
set_working_directory(#filepath); // so that we load fonts correctly.
|
|
make_directory_if_it_does_not_exist(build_directory);
|
|
|
|
// Workspaces compile concurrently once source is added, so every build is
|
|
// started before any is waited on.
|
|
pending: [..] Pending_Build;
|
|
|
|
if build_debug array_add(*pending, start_main_build(optimized = false));
|
|
if build_release array_add(*pending, start_main_build(optimized = true));
|
|
// jai first.jai - ocr_bench: also bin/ocr-bench, OCR timing and output over a folder of .mcap files.
|
|
if array_find(args, "ocr_bench") array_add(*pending, start_tool_build("ocr-bench", "tools/ocr_bench.jai", optimized = build_release));
|
|
// jai first.jai - image_probe: also bin/image-probe, what modules/Image_Decode reads from image files.
|
|
if array_find(args, "image_probe") array_add(*pending, start_tool_build("image-probe", "tools/image_probe.jai", optimized = build_release));
|
|
// jai first.jai - segment_probe: also bin/segment-probe, what modules/Segment makes of image files.
|
|
if array_find(args, "segment_probe") array_add(*pending, start_tool_build("segment-probe", "tools/segment_probe.jai", optimized = build_release));
|
|
// jai first.jai - record_probe: also bin/record-probe, a screen recorded with modules/Screen_Record (video-capture-plan.html, phase 1).
|
|
if array_find(args, "record_probe") array_add(*pending, start_tool_build("record-probe", "tools/record_probe.jai", optimized = build_release));
|
|
|
|
// The gpu-bench branch: bin/gpu-bench, every GPU model timed over a fixed corpus (tools/gpu_bench),
|
|
// always built, as the GPU Benchmark window runs it.
|
|
array_add(*pending, start_tool_build("gpu-bench", "tools/gpu_bench/gpu_bench.jai", optimized = build_release));
|
|
|
|
wait_for_builds(pending);
|
|
|
|
for pending if !it.failed {
|
|
if it.tool log("Built %/%", build_directory, it.executable_name);
|
|
else finish_main_build(it);
|
|
}
|
|
|
|
// The assets are the same for every configuration, so they are copied once.
|
|
for pending if !it.failed { copy_assets(); break; }
|
|
}
|
|
|
|
wait_for_builds :: (pending: [] Pending_Build) {
|
|
remaining := pending.count;
|
|
|
|
while remaining > 0 {
|
|
message := compiler_wait_for_message();
|
|
if message.kind != .COMPLETE continue;
|
|
|
|
for * pending {
|
|
if it.workspace != message.workspace continue;
|
|
|
|
m := cast(*Message_Complete) message;
|
|
it.failed = m.error_code == .COMPILATION_FAILED;
|
|
compiler_end_intercept(it.workspace);
|
|
remaining -= 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
start_main_build :: (optimized: bool) -> Pending_Build {
|
|
log("Building main executable (%)...", ifx optimized then "release" else "debug");
|
|
|
|
options := base_options;
|
|
|
|
// Builds for the compiler's own CPU; on macOS that must be arm64 (see build_project).
|
|
#if OS == .MACOS
|
|
output_executable_name_temp := tprint("%-macos", APP_FOLDER_NAME);
|
|
#if OS == .LINUX
|
|
output_executable_name_temp := tprint("%-linux", APP_FOLDER_NAME);
|
|
#if OS == .WINDOWS
|
|
output_executable_name_temp := APP_FOLDER_NAME;
|
|
if (optimized) {
|
|
set_optimization(*options, .VERY_OPTIMIZED, preserve_debug_info=true);
|
|
options.backend = .LLVM;
|
|
options.llvm_options.enable_split_modules = false;
|
|
options.output_executable_name = output_executable_name_temp;
|
|
options.array_bounds_check = .ALWAYS;
|
|
options.null_pointer_check = .ON;
|
|
options.arithmetic_overflow_check = .FATAL;
|
|
options.cast_bounds_check = .FATAL;
|
|
options.dead_code_elimination = .ALL;
|
|
} else {
|
|
#if CPU == .X64 options.backend = .X64; // the fastest to compile; LLVM elsewhere
|
|
options.output_executable_name = tprint("%-debug", output_executable_name_temp);
|
|
}
|
|
|
|
#if OS == .WINDOWS {
|
|
file_delete(tprint("%/%.pdb", build_directory, options.output_executable_name));
|
|
file_delete(tprint("%/%.rdi", build_directory, options.output_executable_name));
|
|
}
|
|
|
|
options.temporary_storage_size = 64 * 4096;
|
|
options.output_path = build_directory;
|
|
|
|
w := compiler_create_workspace(options.output_executable_name);
|
|
set_build_options(options, w);
|
|
|
|
#if OS == .WINDOWS {
|
|
// Disable runtime console
|
|
if optimized {
|
|
set_build_options_dc(.{append_linker_arguments=.["/SUBSYSTEM:windows", "/ENTRY:mainCRTStartup", "/IGNORE:4099"]}, w);
|
|
} else {
|
|
set_build_options_dc(.{append_linker_arguments=.["msvcprtd.lib", "/IGNORE:4099"]}, w);
|
|
}
|
|
}
|
|
|
|
compiler_begin_intercept(w);
|
|
|
|
add_build_file("src/App.jai", w);
|
|
add_build_constants(w, optimized);
|
|
|
|
return .{ workspace = w, executable_name = options.output_executable_name, optimized = optimized };
|
|
}
|
|
|
|
start_tool_build :: (name: string, file: string, optimized: bool) -> Pending_Build {
|
|
log("Building % (%)...", name, ifx optimized then "release" else "debug");
|
|
options := base_options;
|
|
if optimized {
|
|
set_optimization(*options, .VERY_OPTIMIZED, preserve_debug_info=true);
|
|
options.backend = .LLVM;
|
|
options.output_executable_name = name;
|
|
} else {
|
|
#if CPU == .X64 options.backend = .X64;
|
|
options.output_executable_name = tprint("%-debug", name);
|
|
}
|
|
options.output_path = build_directory;
|
|
|
|
w := compiler_create_workspace(options.output_executable_name);
|
|
set_build_options(options, w);
|
|
#if OS == .WINDOWS set_build_options_dc(.{append_linker_arguments=.["/IGNORE:4099"]}, w);
|
|
compiler_begin_intercept(w);
|
|
add_build_file(file, w);
|
|
add_build_constants(w, optimized);
|
|
return .{ workspace = w, executable_name = options.output_executable_name, optimized = optimized, tool = true };
|
|
}
|
|
|
|
finish_main_build :: (build: Pending_Build) {
|
|
#if OS == .WINDOWS {
|
|
exe_path := tprint("%/%.exe", build_directory, build.executable_name);
|
|
|
|
if build.optimized {
|
|
ico_data := create_ico_file_from_bitmap_filename("assets/images/icon.png");
|
|
success := set_icon_by_data(exe_path, ico_data);
|
|
if !success {
|
|
log_error("ERROR: Couldn't set icon for '%'\n", exe_path);
|
|
}
|
|
|
|
manifest_options: Manifest_Options;
|
|
success = add_manifest_to_executable(exe_path, manifest_options);
|
|
if !success {
|
|
log_error("ERROR: Couldn't add manifest to executable '%'\n", exe_path);
|
|
}
|
|
}
|
|
} else #if OS == .MACOS {
|
|
if build.optimized create_macos_app_bundle(build.executable_name);
|
|
}
|
|
|
|
log("Built %/%", build_directory, build.executable_name);
|
|
}
|
|
|
|
#if OS == .MACOS {
|
|
MACOS_APP_BUNDLE :: #run tprint("%.app", APP_NAME);
|
|
MACOS_BUNDLE_DYLIBS :: string.["libSDL3.0.dylib"];
|
|
|
|
// The .app holds only the executable, the dylibs it loads from beside itself, the icon and
|
|
// Info.plist. The app finds data/, logs/ and images/ in the folder holding the bundle; see
|
|
// app_install_directory.
|
|
create_macos_app_bundle :: (executable_name: string) {
|
|
app := tprint("%/%", build_directory, MACOS_APP_BUNDLE);
|
|
run_command("rm", "-rf", app);
|
|
executable_folder := tprint("%/Contents/MacOS", app);
|
|
resources_folder := tprint("%/Contents/Resources", app);
|
|
make_directory_if_it_does_not_exist(executable_folder, recursive = true);
|
|
make_directory_if_it_does_not_exist(resources_folder, recursive = true);
|
|
|
|
copied := copy_file(tprint("%/%", build_directory, executable_name), tprint("%/%", executable_folder, executable_name));
|
|
for MACOS_BUNDLE_DYLIBS copied &&= copy_file(tprint("assets/%", it), tprint("%/%", executable_folder, it));
|
|
if !copied {
|
|
log_error("ERROR: Couldn't copy the executable and its dylibs into %\n", app);
|
|
return;
|
|
}
|
|
run_command("chmod", "+x", tprint("%/%", executable_folder, executable_name));
|
|
|
|
// sips makes an .icns only from a size the format holds, and icon.png may be any square size.
|
|
resized := tprint("%/icon_1024.png", build_directory);
|
|
result := run_command("sips", "-z", "1024", "1024", "assets/images/icon.png", "--out", resized);
|
|
if result.exit_code == 0 result = run_command("sips", "-s", "format", "icns", resized, "--out", tprint("%/%.icns", resources_folder, APP_FOLDER_NAME));
|
|
file_delete(resized);
|
|
if result.exit_code != 0 log_error("ERROR: Couldn't make the icon for %; the Dock will show the generic one.\n", app);
|
|
|
|
if !write_entire_file(tprint("%/Contents/Info.plist", app), tprint(MACOS_INFO_PLIST, APP_NAME, APP_NAME, executable_name, APP_BUNDLE_IDENTIFIER, APP_FOLDER_NAME, VERSION, VERSION)) {
|
|
log_error("ERROR: Couldn't write %/Contents/Info.plist\n", app);
|
|
return;
|
|
}
|
|
|
|
// Ad hoc: a signature over the whole bundle, which macOS expects, not a Developer ID.
|
|
result = run_command("codesign", "--force", "--deep", "--sign", "-", app);
|
|
if result.exit_code != 0 log_error("WARNING: Couldn't sign %; it may still run.\n", app);
|
|
|
|
log("Built %", app);
|
|
}
|
|
|
|
MACOS_INFO_PLIST :: #string PLIST
|
|
<?xml version="1.0" encoding="UTF-8"?>
|
|
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
|
<plist version="1.0">
|
|
<dict>
|
|
<key>CFBundleDevelopmentRegion</key> <string>en</string>
|
|
<key>CFBundleDisplayName</key> <string>%</string>
|
|
<key>CFBundleName</key> <string>%</string>
|
|
<key>CFBundleExecutable</key> <string>%</string>
|
|
<key>CFBundleIdentifier</key> <string>%</string>
|
|
<key>CFBundleIconFile</key> <string>%.icns</string>
|
|
<key>CFBundleInfoDictionaryVersion</key><string>6.0</string>
|
|
<key>CFBundlePackageType</key> <string>APPL</string>
|
|
<key>CFBundleShortVersionString</key> <string>%</string>
|
|
<key>CFBundleVersion</key> <string>%</string>
|
|
<key>NSHighResolutionCapable</key> <true/>
|
|
</dict>
|
|
</plist>
|
|
PLIST
|
|
}
|
|
|
|
copy_assets :: () {
|
|
copy_directory("assets", build_directory); // This puts the CONTENTS of assets into the build directory. Not sure if that's what we want, but I'll roll with it.
|
|
|
|
#if OS == .MACOS copy_file("assets/libSDL3.0.dylib", tprint("%/libSDL3.0.dylib", build_directory));
|
|
|
|
// OCR: onnxruntime beside the executable, where modules/onnxruntime looks for it first, and the
|
|
// models in bin/ocr-models. Both are fetched, not in git (see their fetch scripts), and large,
|
|
// so a file is only copied when it is missing or differs in size or time.
|
|
#if OS == .WINDOWS ORT_FOLDER :: "modules/onnxruntime/windows";
|
|
else #if OS == .LINUX ORT_FOLDER :: "modules/onnxruntime/linux";
|
|
else ORT_FOLDER :: "modules/onnxruntime/macos";
|
|
if file_exists(ORT_FOLDER) {
|
|
for file_list(ORT_FOLDER) {
|
|
name := path_filename(it);
|
|
if contains(name, ".dll") || contains(name, ".so") || contains(name, ".dylib") copy_if_changed(it, tprint("%/%", build_directory, name));
|
|
}
|
|
} else {
|
|
log("OCR: onnxruntime has not been fetched; run jai fetch.jai in modules/onnxruntime for OCR.");
|
|
}
|
|
make_directory_if_it_does_not_exist(tprint("%/ocr-models", build_directory));
|
|
copy_if_changed("modules/OCR/models.config", tprint("%/ocr-models/models.config", build_directory));
|
|
if file_exists("modules/OCR/models") {
|
|
for file_list("modules/OCR/models") copy_if_changed(it, tprint("%/ocr-models/%", build_directory, path_filename(it)));
|
|
} else {
|
|
log("OCR: the models have not been fetched; run jai fetch_models.jai in modules/OCR for OCR.");
|
|
}
|
|
|
|
// Videos: FFmpeg's DLLs beside the executable (modules/ffmpeg/windows, in git), the same way.
|
|
#if OS == .WINDOWS {
|
|
for file_list("modules/ffmpeg/windows") if ends_with(it, ".dll") copy_if_changed(it, tprint("%/%", build_directory, path_filename(it)));
|
|
}
|
|
|
|
// Labels: its models in bin/label-models, the same way.
|
|
make_directory_if_it_does_not_exist(tprint("%/label-models", build_directory));
|
|
copy_if_changed("modules/Labels/models.config", tprint("%/label-models/models.config", build_directory));
|
|
if file_exists("modules/Labels/models") {
|
|
for file_list("modules/Labels/models") copy_if_changed(it, tprint("%/label-models/%", build_directory, path_filename(it)));
|
|
} else {
|
|
log("Labels: the models have not been fetched; run jai fetch_models.jai in modules/Labels for labels.");
|
|
}
|
|
|
|
// Segmentation: its model in bin/segment-models, the same way (made by tools/export_mask2former.py).
|
|
make_directory_if_it_does_not_exist(tprint("%/segment-models", build_directory));
|
|
copy_if_changed("modules/Segment/models.config", tprint("%/segment-models/models.config", build_directory));
|
|
if file_exists("modules/Segment/models") {
|
|
for file_list("modules/Segment/models") copy_if_changed(it, tprint("%/segment-models/%", build_directory, path_filename(it)));
|
|
} else {
|
|
log("Segmentation: the model has not been made; see modules/Segment/README.md.");
|
|
}
|
|
|
|
// gpu-bench's corpus, in bin/gpu-bench-corpus: in git, so every machine has the same images.
|
|
make_directory_if_it_does_not_exist(tprint("%/gpu-bench-corpus", build_directory));
|
|
copy_if_changed("tools/gpu_bench/corpus.txt", tprint("%/gpu-bench-corpus/corpus.txt", build_directory));
|
|
for file_list("tools/gpu_bench/corpus") copy_if_changed(it, tprint("%/gpu-bench-corpus/%", build_directory, path_filename(it)));
|
|
}
|
|
|
|
copy_if_changed :: (from: string, to: string) {
|
|
from_time, from_size, from_ok := file_modtime_and_size(from);
|
|
to_time, to_size, to_ok := file_modtime_and_size(to);
|
|
if to_ok && from_ok && to_size == from_size && to_time >= from_time return;
|
|
if !copy_file(from, to) log_error("Could not copy % to %.", from, to);
|
|
}
|
|
|
|
add_build_constants :: (w: Workspace, optimized: bool) {
|
|
BUILD_DATE :: #run -> string {
|
|
current_time := to_calendar(current_time_consensus());
|
|
return calendar_to_string(current_time);
|
|
}
|
|
|
|
GIT_HASH :: #run -> string {
|
|
result, revision := run_command("git", "describe", "--always", "--dirty", "--exclude=*", capture_and_return_output = true);
|
|
if revision return trim(revision);
|
|
return "-";
|
|
}
|
|
|
|
GIT_BRANCH :: #run -> string {
|
|
result, branch_name := run_command("git", "branch", "--show-current", capture_and_return_output = true);
|
|
return trim(branch_name);
|
|
}
|
|
|
|
build_constants := tprint(#string STRING
|
|
APP_NAME :: "%";
|
|
APP_FOLDER_NAME :: "%";
|
|
GIT_BRANCH :: "%";
|
|
GIT_HASH :: "%";
|
|
VERSION :: "%";
|
|
BUILD_DATE :: "%";
|
|
DEBUG :: %;
|
|
COMPILE_TESTS :: %;
|
|
COMPILE_ARGS :: "%";
|
|
NO_CRASH_DIALOG :: %;
|
|
STRING,
|
|
escape_for_string_literal(APP_NAME),
|
|
escape_for_string_literal(APP_FOLDER_NAME),
|
|
escape_for_string_literal(GIT_BRANCH),
|
|
escape_for_string_literal(GIT_HASH),
|
|
VERSION,
|
|
BUILD_DATE,
|
|
ifx optimized then "false" else "true",
|
|
|
|
ifx !optimized || array_find(base_options.compile_time_command_line, "compile_tests") then "true" else "false",
|
|
escape_for_string_literal(join(..base_options.compile_time_command_line, " ")),
|
|
ifx array_find(base_options.compile_time_command_line, "no_crash_dialog") then "true" else "false"
|
|
);
|
|
|
|
add_build_string(build_constants, w);
|
|
}
|
|
|
|
check_compiler_version :: () {
|
|
version_info: Version_Info;
|
|
version_str := compiler_get_version_info(*version_info);
|
|
|
|
min_version_str := tprint("%.%.%", MIN_COMPILER_VERSION.major, MIN_COMPILER_VERSION.minor, MIN_COMPILER_VERSION.micro);
|
|
max_version_str := tprint("%.%.%", MAX_COMPILER_VERSION.major, MAX_COMPILER_VERSION.minor, MAX_COMPILER_VERSION.micro);
|
|
|
|
if compare_versions(version_info, MIN_COMPILER_VERSION) < 0 {
|
|
error := tprint("Compiler version '%' is below the minimum required version '%'", version_str, min_version_str);
|
|
compiler_report(error);
|
|
}
|
|
|
|
if compare_versions(version_info, MAX_COMPILER_VERSION) > 0 {
|
|
error := tprint("Compiler version '%' is above the maximum supported version '%'", version_str, max_version_str);
|
|
compiler_report(error);
|
|
}
|
|
}
|
|
|
|
// Lexicographic: a later component only matters when every earlier one is equal.
|
|
compare_versions :: (a: Version_Info, b: Version_Info) -> int {
|
|
if a.major != b.major return ifx a.major < b.major then -1 else 1;
|
|
if a.minor != b.minor return ifx a.minor < b.minor then -1 else 1;
|
|
if a.micro != b.micro return ifx a.micro < b.micro then -1 else 1;
|
|
return 0;
|
|
}
|
|
|
|
// For text spliced between double quotes in generated source.
|
|
escape_for_string_literal :: (s: string) -> string {
|
|
builder: String_Builder;
|
|
builder.allocator = temp;
|
|
for 0..s.count-1 {
|
|
c := s[it];
|
|
if c == {
|
|
case #char "\\"; append(*builder, "\\\\");
|
|
case #char "\""; append(*builder, "\\\"");
|
|
case #char "\n"; append(*builder, "\\n");
|
|
case #char "\r"; append(*builder, "\\r");
|
|
case; append(*builder, c);
|
|
}
|
|
}
|
|
return builder_to_string(*builder,, temp);
|
|
}
|
|
|
|
#import "Basic";
|
|
#import "Compiler";
|
|
#import "String";
|
|
#import "File";
|
|
#import "Process";
|
|
#import "File_Utilities";
|
|
|
|
#if OS == .WINDOWS {
|
|
#import "Ico_File";
|
|
#import "Windows_Resources";
|
|
#import "Windows_Utf8";
|
|
}
|