112 lines
4.8 KiB
Plaintext
112 lines
4.8 KiB
Plaintext
// The chunks a label model adds to a capture (.mcap version 3), each keyed by the model's name:
|
|
//
|
|
// "labl", in front: its labels, as text in the style of the Ex4 text formats
|
|
//
|
|
// @version 1
|
|
// engine: "onnxruntime 1.30.0, CUDA"
|
|
// date: 2026-10-08T21:04:11Z
|
|
// list: 9c4f0b2d81e3a675
|
|
// label: 0.0647 "Terminal"
|
|
// label: 0.0234 "Schematic"
|
|
//
|
|
// list: says which label list they were scored against (a hash of its names and prompts), so labels from an
|
|
// older list can be told apart. label: a score (0 to 1, the model's sigmoid) and a name, quoted
|
|
// with "" for a quote; best first. A line that cannot be read is skipped, as are unknown keys.
|
|
//
|
|
// "embd", in front: the image's vector, binary: u32 length, u8 format (1: 16-bit floats), then the
|
|
// values, little-endian. A unit vector, so a phrase's vector scores against it with a dot product.
|
|
|
|
MCAP_LABELS_TAG :: "labl";
|
|
MCAP_EMBEDDING_TAG :: "embd";
|
|
MCAP_LABELS_TEXT_VERSION :: 1;
|
|
MCAP_EMBEDDING_HALF :: 1;
|
|
|
|
Mcap_Label :: struct {
|
|
score: float;
|
|
name: string;
|
|
}
|
|
|
|
Mcap_Labels :: struct {
|
|
engine: string; // what ran it, such as "onnxruntime 1.30.0, CUDA"
|
|
date: string; // when, UTC: 2026-10-08T21:04:11Z
|
|
list: string; // the label list's hash
|
|
labels: [..] Mcap_Label; // best first
|
|
}
|
|
|
|
// A labl chunk's text. Temp.
|
|
mcap_labels_text :: (labels: Mcap_Labels) -> string {
|
|
builder: String_Builder;
|
|
builder.allocator = temp;
|
|
print_to_builder(*builder, "@version %\n", MCAP_LABELS_TEXT_VERSION);
|
|
print_to_builder(*builder, "engine: %\n", mcap_quote(labels.engine));
|
|
print_to_builder(*builder, "date: %\n", labels.date);
|
|
print_to_builder(*builder, "list: %\n", labels.list);
|
|
for labels.labels print_to_builder(*builder, "label: % %\n", formatFloat(it.score, trailing_width = 4, zero_removal = .NO), mcap_quote(it.name));
|
|
return builder_to_string(*builder,, temp);
|
|
}
|
|
|
|
// A labl chunk's labels; strings and arrays from the context allocator.
|
|
mcap_labels_parse :: (text: string) -> Mcap_Labels, ok: bool, error: string {
|
|
result: Mcap_Labels;
|
|
version_seen := false;
|
|
for raw: split(text, "\n",, temp) {
|
|
line := trim(raw);
|
|
if !line.count || line[0] == #char "#" continue;
|
|
if begins_with(line, "@version") {
|
|
rest := trim(advance(line, 8));
|
|
version, ok := string_to_int(rest);
|
|
if !ok || version < 1 || version > MCAP_LABELS_TEXT_VERSION return result, false, tprint("its labels are version %, but this build reads up to %", rest, MCAP_LABELS_TEXT_VERSION);
|
|
version_seen = true;
|
|
continue;
|
|
}
|
|
if !version_seen return result, false, "its labels do not start with @version";
|
|
colon := find_index_from_left(line, #char ":");
|
|
if colon < 0 continue;
|
|
key := trim(slice(line, 0, colon));
|
|
value := trim(advance(line, colon + 1));
|
|
if key == {
|
|
case "engine";
|
|
quoted, _, ok := mcap_read_quoted(value);
|
|
if ok result.engine = copy_string(quoted);
|
|
case "date"; result.date = copy_string(value);
|
|
case "list"; result.list = copy_string(value);
|
|
case "label";
|
|
space := find_index_from_left(value, #char " ");
|
|
if space < 0 continue;
|
|
score, score_ok := string_to_float(slice(value, 0, space));
|
|
name, after, name_ok := mcap_read_quoted(trim_left(advance(value, space)));
|
|
if score_ok && name_ok && !trim(after).count array_add(*result.labels, .{score, copy_string(name)});
|
|
}
|
|
}
|
|
return result, true, "";
|
|
}
|
|
|
|
// An embd chunk's data for a vector stored as 16-bit floats. Temp.
|
|
mcap_embedding_bytes :: (half: [] u16) -> [] u8 {
|
|
bytes := NewArray(5 + half.count * 2, u8,, temp);
|
|
(cast(*u32) bytes.data).* = cast(u32) half.count;
|
|
bytes[4] = MCAP_EMBEDDING_HALF;
|
|
memcpy(bytes.data + 5, half.data, half.count * 2);
|
|
return bytes;
|
|
}
|
|
|
|
// An embd chunk's vector, as 16-bit floats; a copy from the context allocator.
|
|
mcap_embedding_parse :: (data: [] u8) -> [] u16, ok: bool, error: string {
|
|
if data.count < 5 return .[], false, "its embedding is cut short";
|
|
count := cast(s64) (cast(*u32) data.data).*;
|
|
if data[4] != MCAP_EMBEDDING_HALF return .[], false, tprint("its embedding is in format %, which this build does not read", data[4]);
|
|
if data.count != 5 + count * 2 return .[], false, "its embedding's length does not match its size";
|
|
half := NewArray(count, u16);
|
|
memcpy(half.data, data.data + 5, count * 2); // the data need not be aligned
|
|
return half, true, "";
|
|
}
|
|
|
|
// The chunk for labels by model, ready to store with mcap_store_chunk. Its data is temp.
|
|
mcap_labels_chunk :: (model: string, labels: Mcap_Labels) -> Mcap_Chunk {
|
|
return .{tag = mcap_tag(MCAP_LABELS_TAG), key = model, data = cast([] u8) mcap_labels_text(labels)};
|
|
}
|
|
|
|
mcap_embedding_chunk :: (model: string, half: [] u16) -> Mcap_Chunk {
|
|
return .{tag = mcap_tag(MCAP_EMBEDDING_TAG), key = model, data = mcap_embedding_bytes(half)};
|
|
}
|