68 lines
2.8 KiB
Plaintext
68 lines
2.8 KiB
Plaintext
// Downloads the files models.config names into models/, checking each one's SHA-256. The models
|
|
// are not in git; run this after cloning. Files already there with the right hash are skipped.
|
|
//
|
|
// jai fetch_models.jai (about 690 MB)
|
|
//
|
|
// SigLIP 2 (Google, Apache 2.0), as converted to ONNX by onnx-community:
|
|
// https://huggingface.co/onnx-community/siglip2-base-patch16-384-ONNX
|
|
|
|
SOURCE :: "https://huggingface.co/onnx-community/siglip2-base-patch16-384-ONNX/resolve/main";
|
|
|
|
Model_File :: struct { name: string; source_path: string; sha256: string; }
|
|
|
|
FILES :: Model_File.[
|
|
.{"siglip2-base-patch16-384-vision.onnx", "onnx/vision_model.onnx", "49ae4958b1098ca995e929d646f7be05a69c65e6344beae07d58c6598ffc5210"},
|
|
.{"siglip2-base-patch16-384-text-int8.onnx", "onnx/text_model_int8.onnx", "30a94b43b9ddc39c04ab67f80cba4b808262682b2fa1a2a8b924bf1e7bd84c61"},
|
|
.{"siglip2-tokenizer.json", "tokenizer.json", "cb9140fae3ac5122c972d37adf83e1248471a38147ad76f8215c8872c6fd8322"},
|
|
];
|
|
|
|
#run,stallable { if !fetch() exit(1); }
|
|
|
|
fetch :: () -> bool {
|
|
set_build_options_dc(.{do_output = false});
|
|
here := trim_right(path_strip_filename(#file), "/\\");
|
|
folder := tprint("%/models", here);
|
|
make_directory_if_it_does_not_exist(folder);
|
|
|
|
for FILES {
|
|
path := tprint("%/%", folder, it.name);
|
|
if file_exists(path) && sha256_of(path) == it.sha256 { log("% is already here.", it.name); continue; }
|
|
|
|
url := tprint("%/%", SOURCE, it.source_path);
|
|
log("Downloading %...", url);
|
|
result := run_command("curl", "-L", "--fail", "--progress-bar", "-o", path, url);
|
|
if result.exit_code != 0 { log_error("curl failed for % (exit code %).", it.name, result.exit_code); return false; }
|
|
hash := sha256_of(path);
|
|
if hash != it.sha256 {
|
|
log_error("% has SHA-256 %, expected %; it was deleted.", it.name, hash, it.sha256);
|
|
file_delete(path);
|
|
return false;
|
|
}
|
|
}
|
|
log("The models are in %.", folder);
|
|
return true;
|
|
}
|
|
|
|
// Lowercase hex, or "" when it could not be worked out.
|
|
sha256_of :: (path: string) -> string {
|
|
#if OS == .WINDOWS {
|
|
result, output := run_command("certutil", "-hashfile", path, "SHA256", capture_and_return_output = true);
|
|
if result.exit_code != 0 return "";
|
|
lines := split(output, "\n");
|
|
if lines.count < 2 return "";
|
|
return to_lower_copy(replace(trim(lines[1]), " ", "")); // line 2 is the hash
|
|
} else {
|
|
command := ifx OS == .MACOS then string.["shasum", "-a", "256", path] else string.["sha256sum", path];
|
|
result, output := run_command(..command, capture_and_return_output = true);
|
|
if result.exit_code != 0 return "";
|
|
return to_lower_copy(trim(split(output, " ")[0]));
|
|
}
|
|
}
|
|
|
|
#import "Basic";
|
|
#import "Compiler";
|
|
#import "File";
|
|
#import "File_Utilities";
|
|
#import "Process";
|
|
#import "String";
|