[0.0.6] Erase OCR readings, or labels and vectors, from the whole library (for testing)
This commit is contained in:
parent
cf60dfa535
commit
bedeaebe41
@ -47,7 +47,10 @@ and everything around them, ready for an app to be written into `src/app/app_ui.
|
||||
- **OCR** (`modules/OCR`): the OCR window shows whether it runs on the GPU and, if not, what to install,
|
||||
and switches model (kept as `model` under `[ocr]` in the settings file). **Scan the library** reads
|
||||
every capture shown that the model has not read yet, storing each reading in its file as it goes,
|
||||
so it can pause, cancel and pick up again; with **Labels too** it labels them in the same pass. The
|
||||
so it can pause, cancel and pick up again; with **Labels too** it labels them in the same pass.
|
||||
For testing, the OCR window's **Erase OCR readings** and the Labels window's **Erase labels and
|
||||
vectors** take them out of every file in the library again (after asking); `-erase ocr|labels|all`
|
||||
does the same without the UI and exits. The
|
||||
Capture window's **OCR** button reads one capture; a capture read by several models keeps each
|
||||
reading, and the window can show any of them. `-library FOLDER` shows another folder of `.mcap` files;
|
||||
`-scan` scans it with the chosen model once it has loaded and exits (0, or 1 if any capture
|
||||
|
||||
@ -29,7 +29,7 @@ APP_NAME :: "mcap";
|
||||
APP_FOLDER_NAME :: "mcap";
|
||||
GIT_BRANCH :: "check";
|
||||
GIT_HASH :: "-";
|
||||
VERSION :: "0.0.5";
|
||||
VERSION :: "0.0.6";
|
||||
BUILD_DATE :: "-";
|
||||
DEBUG :: true;
|
||||
COMPILE_TESTS :: true;
|
||||
|
||||
@ -8,7 +8,7 @@ APP_NAME :: "mcap";
|
||||
APP_FOLDER_NAME :: "mcap";
|
||||
APP_BUNDLE_IDENTIFIER :: "com.example.mcap";
|
||||
|
||||
VERSION :: "0.0.5";
|
||||
VERSION :: "0.0.6";
|
||||
// Where everything goes: bin/, or bin-headless/ for builds made with no_crash_dialog, so automated
|
||||
// builds never write over the ones you run by hand.
|
||||
build_directory := "bin";
|
||||
|
||||
@ -321,6 +321,40 @@ mcap_store :: (path: string, ocr: *Mcap_Ocr_Section = null, front: [] Mcap_Chunk
|
||||
return written, write_error;
|
||||
}
|
||||
|
||||
// Takes out of the .mcap at path: with ocr, every OCR section; and every chunk, front or back, whose
|
||||
// tag is one of tags (whatever its key). Everything else stays as it is. The file is rewritten (and
|
||||
// upgraded to the current version) only when there was something to take out; changed says whether.
|
||||
mcap_erase :: (path: string, ocr := false, tags: [] string = .[]) -> ok: bool, error: string, changed: bool {
|
||||
data, read_ok := read_entire_file(path, log_errors = false);
|
||||
if !read_ok return false, "cannot be read", false;
|
||||
defer free(data);
|
||||
m, parsed, parse_error := mcap_parse(cast([] u8) data);
|
||||
if !parsed return false, parse_error, false;
|
||||
|
||||
push_allocator(temp);
|
||||
changed := false;
|
||||
if ocr && m.ocr.count { m.ocr = ""; changed = true; }
|
||||
without :: (chunks: [] Mcap_Chunk, tags: [] string, changed: *bool) -> [] Mcap_Chunk {
|
||||
kept: [..] Mcap_Chunk;
|
||||
for chunks {
|
||||
erase := false;
|
||||
for tag: tags if mcap_same_tag(it.tag, mcap_tag(tag)) erase = true;
|
||||
if erase changed.* = true;
|
||||
else array_add(*kept, it);
|
||||
}
|
||||
return kept;
|
||||
}
|
||||
m.front_chunks = without(m.front_chunks, tags, *changed);
|
||||
m.back_chunks = without(m.back_chunks, tags, *changed);
|
||||
if !changed return true, "", false;
|
||||
|
||||
m.version = MCAP_VERSION;
|
||||
serializer, serialized, serialize_error := mcap_serialize(m);
|
||||
if !serialized return false, serialize_error, false;
|
||||
written, write_error := mcap_write(path, serializer);
|
||||
return written, write_error, written;
|
||||
}
|
||||
|
||||
// Stores chunk alone: in front, or with back after the images. See mcap_store.
|
||||
mcap_store_chunk :: (path: string, chunk: Mcap_Chunk, back := false) -> ok: bool, error: string {
|
||||
if !back {
|
||||
|
||||
@ -184,6 +184,7 @@ App :: struct {
|
||||
scan_and_exit: bool; // -scan: scan the library with the chosen OCR model once it has loaded, then exit (0, or 1 if any capture failed)
|
||||
stats_and_exit: bool; // -stats: log the Statistics window's numbers once the catalog has loaded, then exit
|
||||
start_search: string; // -search QUERY: the search box's text at startup
|
||||
erase_and_exit: string; // -erase ocr|labels|all: erase that from the whole library once it has loaded, then exit
|
||||
last_window_restore_time: Apollo_Time;
|
||||
|
||||
imgui_context: *ImGui.ImGuiContext;
|
||||
|
||||
@ -290,6 +290,14 @@ labels_window_contents :: () { using ImGui;
|
||||
}
|
||||
SeparatorText("The library");
|
||||
Text("% of % captures labelled; % with this list.", labels_ui.labelled_count, viewer_capture_count(), labels_ui.current_count);
|
||||
if ocr_ui.scan.running {
|
||||
ocr_scan_progress_ui();
|
||||
} else {
|
||||
if busy BeginDisabled();
|
||||
if SmallButton("Erase labels and vectors from the library...") ocr_ui.erase_asked = .LABELS;
|
||||
if busy EndDisabled();
|
||||
if IsItemHovered(.AllowWhenDisabled) SetTooltip("Takes every capture's labels and image vector, of every label model, out of its file (asks first). For testing: a scan with Labels too makes them again.");
|
||||
}
|
||||
TextColored(app_theme.palette.dim, "The OCR window's scan labels the rest, and labels again from their vectors those scored with an earlier list.");
|
||||
if SmallButton("OCR window") ocr_ui.window_open = true;
|
||||
|
||||
|
||||
@ -21,6 +21,7 @@ main :: () {
|
||||
app.scan_and_exit = array_find(app.args, "-scan");
|
||||
app.stats_and_exit = array_find(app.args, "-stats");
|
||||
for app.args if it == "-search" && it_index + 1 < app.args.count app.start_search = app.args[it_index + 1];
|
||||
for app.args if it == "-erase" && it_index + 1 < app.args.count app.erase_and_exit = app.args[it_index + 1];
|
||||
|
||||
defer report_memory_leaks();
|
||||
|
||||
|
||||
@ -36,6 +36,7 @@ ocr_ui: struct {
|
||||
|
||||
scan: Ocr_Scan;
|
||||
scan_requested: bool; // -scan has started its scan
|
||||
erase_asked: Erase_What; // the erase waiting for its confirmation
|
||||
scan_logged_step: s64; // -scan's last progress line, in twentieths of the scan
|
||||
|
||||
// The Capture window.
|
||||
@ -48,7 +49,7 @@ ocr_ui: struct {
|
||||
toast_shown := false;
|
||||
}
|
||||
|
||||
Ocr_Task_Kind :: enum { NONE; CREATE; READ; SCAN; LABELS_CREATE; LABEL_SET; }
|
||||
Ocr_Task_Kind :: enum { NONE; CREATE; READ; SCAN; LABELS_CREATE; LABEL_SET; ERASE; }
|
||||
|
||||
Ocr_Task :: struct {
|
||||
kind: Ocr_Task_Kind;
|
||||
@ -75,9 +76,12 @@ Ocr_Scan :: struct {
|
||||
pause_start: Apollo_Time;
|
||||
elapsed_seconds: float64; // once it has ended: the time spent scanning, pauses left out
|
||||
label_set: *Label_Set; // labels scored against this; null when the scan does not label
|
||||
erase_ocr, erase_labels: bool; // an erase, not a scan: what it takes out of each file
|
||||
}
|
||||
|
||||
// A capture to read, and what it needs.
|
||||
Erase_What :: enum u8 { NONE; OCR; LABELS; ALL; }
|
||||
|
||||
Scan_Item :: struct {
|
||||
index: s32;
|
||||
path: string;
|
||||
@ -211,6 +215,45 @@ ocr_ui_thread_proc :: (thread: *Thread) -> s64 {
|
||||
log("[OCR scan] % of % captures read with %% in % (% failed)%.", scan.done, scan.total, scan.model, ifx scan.label_set then tprint(" and labelled with %", LABELS_MODEL) else "",
|
||||
ocr_duration_text(scan.elapsed_seconds), scan.failed, ifx scan.cancel then ", cancelled" else "");
|
||||
atomic_swap(*scan.running, false);
|
||||
|
||||
case .ERASE;
|
||||
scan := *ocr_ui.scan;
|
||||
// Not in temporary storage, which is reset after each capture.
|
||||
label_tags := string.[MCAP_LABELS_TAG, MCAP_EMBEDDING_TAG];
|
||||
tags: [] string;
|
||||
if scan.erase_labels tags = label_tags;
|
||||
erased := 0;
|
||||
for item: scan.items {
|
||||
while scan.paused && !scan.cancel sleep_milliseconds(100);
|
||||
if scan.cancel break;
|
||||
ok, changed: bool;
|
||||
error: string;
|
||||
for 1..OCR_STORE_ATTEMPTS {
|
||||
ok, error, changed = mcap_erase(item.path, scan.erase_ocr, tags);
|
||||
if ok break;
|
||||
sleep_milliseconds(250);
|
||||
}
|
||||
if !ok {
|
||||
atomic_add(*scan.failed, 1);
|
||||
log_error("[Erase] % %", item.path, error);
|
||||
} else if changed {
|
||||
erased += 1;
|
||||
entry, load_error := catalog_load(item.path);
|
||||
if entry {
|
||||
lock(*viewer.mutex);
|
||||
array_add(*viewer.catalog_updates, .{item.index, entry});
|
||||
unlock(*viewer.mutex);
|
||||
} else {
|
||||
log_error("[Erase] % was erased, but could not be read back: %", item.path, load_error);
|
||||
}
|
||||
}
|
||||
atomic_add(*scan.done, 1);
|
||||
reset_temporary_storage();
|
||||
}
|
||||
scan.elapsed_seconds = to_float64_seconds(current_time_monotonic() - scan.start) - scan.paused_seconds;
|
||||
log("[Erase] % erased from % of % captures in % (% failed)%.", erase_what_text(scan), erased, scan.total,
|
||||
ocr_duration_text(scan.elapsed_seconds), scan.failed, ifx scan.cancel then ", cancelled" else "");
|
||||
atomic_swap(*scan.running, false);
|
||||
}
|
||||
free(task.model);
|
||||
free(task.path);
|
||||
@ -349,6 +392,91 @@ ocr_read_and_store :: (items: [] Scan_Item, set: *Label_Set) -> [] string {
|
||||
|
||||
// ---------------------------------------------------------------- the scan
|
||||
|
||||
// -erase: starts the erase once the catalog is loaded, and ends the app when it is done.
|
||||
ocr_ui_erase_and_exit :: (busy: bool) {
|
||||
if !ocr_ui.scan_requested {
|
||||
if busy || viewer.catalog_loaded < viewer.captures.count return;
|
||||
what: Erase_What;
|
||||
if app.erase_and_exit == {
|
||||
case "ocr"; what = .OCR;
|
||||
case "labels"; what = .LABELS;
|
||||
case "all"; what = .ALL;
|
||||
case;
|
||||
log_error("[Erase] -erase takes ocr, labels or all, not %.", app.erase_and_exit);
|
||||
app.exit_code = 2; app.should_exit = true; app.erase_and_exit = ""; return;
|
||||
}
|
||||
ocr_ui.scan_requested = true;
|
||||
ocr_ui.window_open = true; // its progress bar
|
||||
ocr_erase_start(what);
|
||||
return;
|
||||
}
|
||||
if busy || ocr_ui.scan.running return;
|
||||
log("[Erase] -erase is done: % in %, % failed.", ocr_scan_progress_text(ocr_ui.scan.done, ocr_ui.scan.total), ocr_duration_text(ocr_ui.scan.elapsed_seconds), ocr_ui.scan.failed);
|
||||
app.exit_code = ifx ocr_ui.scan.failed then cast(s32) 1 else 0;
|
||||
app.should_exit = true;
|
||||
app.erase_and_exit = "";
|
||||
}
|
||||
|
||||
// What an erase takes out, for people. Temp.
|
||||
erase_what_text :: (scan: *Ocr_Scan) -> string {
|
||||
if scan.erase_ocr && scan.erase_labels return "OCR, labels and vectors";
|
||||
if scan.erase_ocr return "OCR readings";
|
||||
return "labels and vectors";
|
||||
}
|
||||
|
||||
// Takes OCR readings, labels and vectors, or both, out of every capture in the library (not only
|
||||
// those shown) that has any, on the OCR thread, as a scan runs: with its progress, pause and cancel.
|
||||
ocr_erase_start :: (what: Erase_What) {
|
||||
if ocr_ui.thread_busy || what == .NONE return;
|
||||
scan := *ocr_ui.scan;
|
||||
for scan.items free(it.path);
|
||||
array_reset(*scan.items);
|
||||
free(scan.model);
|
||||
scan.* = .{};
|
||||
scan.items.allocator = context.default_allocator;
|
||||
scan.erase_ocr = what == .OCR || what == .ALL;
|
||||
scan.erase_labels = what == .LABELS || what == .ALL;
|
||||
for * viewer.captures {
|
||||
if it.removed continue;
|
||||
entry := it.catalog;
|
||||
has := !entry // not known: looked at anyway
|
||||
|| (scan.erase_ocr && entry.ocr.count > 0)
|
||||
|| (scan.erase_labels && (entry.labels.count > 0 || entry.embedding.count > 0 || entry.label_list.count > 0));
|
||||
if !has continue;
|
||||
array_add(*scan.items, .{index = cast(s32) it_index, path = copy_string(it.path,, context.default_allocator)});
|
||||
}
|
||||
scan.total = scan.items.count;
|
||||
if !scan.total { toast_show(.Success, "No capture has % to erase", erase_what_text(scan)); return; }
|
||||
scan.running = true;
|
||||
scan.start = current_time_monotonic();
|
||||
ocr_ui_start(.{kind = .ERASE});
|
||||
}
|
||||
|
||||
// The confirmation for ocr_ui.erase_asked, as a modal popup; at the top level, each frame.
|
||||
ocr_erase_popup :: () { using ImGui;
|
||||
POPUP :: "Erase from the library?";
|
||||
if ocr_ui.erase_asked != .NONE && !IsPopupOpen(POPUP) OpenPopup(POPUP);
|
||||
if !BeginPopupModal(POPUP, null, .AlwaysAutoResize) return;
|
||||
what := ifx ocr_ui.erase_asked == .OCR then "every OCR reading (all models)" else ifx ocr_ui.erase_asked == .LABELS then "every capture's labels and image vectors (all label models)" else "every OCR reading, label and image vector";
|
||||
Text("Erase % from all % captures in %?", what, viewer_capture_count(), viewer_folder());
|
||||
TextColored(app_theme.palette.dim, "Each file holding any is rewritten without them; its images, text and preview stay as they are.");
|
||||
TextColored(app_theme.palette.warning, "This cannot be undone: only scanning the library again brings them back.");
|
||||
Spacing();
|
||||
if Button("Erase") {
|
||||
what_asked := ocr_ui.erase_asked;
|
||||
ocr_ui.erase_asked = .NONE;
|
||||
CloseCurrentPopup();
|
||||
ocr_ui.window_open = true; // its progress bar
|
||||
ocr_erase_start(what_asked);
|
||||
}
|
||||
SameLine();
|
||||
if Button("Cancel") || IsKeyPressed(.Escape) {
|
||||
ocr_ui.erase_asked = .NONE;
|
||||
CloseCurrentPopup();
|
||||
}
|
||||
EndPopup();
|
||||
}
|
||||
|
||||
// -scan: starts the scan once the engine and the catalog are ready, and ends the app when it is done.
|
||||
ocr_ui_scan_and_exit :: (busy: bool) {
|
||||
finish :: (code: s32, format: string, args: ..Any) {
|
||||
@ -432,14 +560,27 @@ ocr_scan_ui :: () { using ImGui;
|
||||
if !catalog_ready TextColored(app_theme.palette.dim, "(once the catalog has loaded)");
|
||||
else TextColored(app_theme.palette.dim, "Reads every capture this model has not read, and labels those not yet labelled with this list. Each is stored as it is read, so a scan can stop and start again.");
|
||||
if scan.total && scan.done {
|
||||
TextColored(app_theme.palette.dim, "Last scan: % of % read with % in %, % failed.", scan.done, scan.total, scan.model, ocr_duration_text(scan.elapsed_seconds), scan.failed);
|
||||
if scan.erase_ocr || scan.erase_labels TextColored(app_theme.palette.dim, "Last erase: % from % of % captures in %, % failed.", erase_what_text(scan), scan.done, scan.total, ocr_duration_text(scan.elapsed_seconds), scan.failed);
|
||||
else TextColored(app_theme.palette.dim, "Last scan: % of % read with % in %, % failed.", scan.done, scan.total, scan.model, ocr_duration_text(scan.elapsed_seconds), scan.failed);
|
||||
}
|
||||
if busy BeginDisabled();
|
||||
if SmallButton("Erase OCR readings from the library...") ocr_ui.erase_asked = .OCR;
|
||||
if busy EndDisabled();
|
||||
if IsItemHovered(.AllowWhenDisabled) SetTooltip("Takes every OCR reading, of every model, out of every capture's file (asks first). For testing: a scan reads them again.");
|
||||
return;
|
||||
}
|
||||
ocr_scan_progress_ui();
|
||||
}
|
||||
|
||||
// The running scan's or erase's progress bar and buttons (the OCR and Labels windows show it).
|
||||
ocr_scan_progress_ui :: () { using ImGui;
|
||||
scan := *ocr_ui.scan;
|
||||
if scan.erase_ocr || scan.erase_labels Text("Erasing %:", erase_what_text(scan));
|
||||
|
||||
done, total, failed, rate, left, elapsed := ocr_scan_progress();
|
||||
ProgressBar(cast(float) done / max(total, 1), .{-FLOAT32_MIN, 0}, temp_c_string(ocr_scan_progress_text(done, total)));
|
||||
Text("% elapsed, % left, % captures/s, % failed, on %", ocr_duration_text(elapsed), ocr_duration_text(left), FF(rate, 1), failed, ifx ocr_ui.engine && ocr_ui.engine.provider == .CUDA then "the GPU" else "the CPU");
|
||||
if scan.erase_ocr || scan.erase_labels Text("% elapsed, % left, % captures/s, % failed", ocr_duration_text(elapsed), ocr_duration_text(left), FF(rate, 1), failed);
|
||||
else Text("% elapsed, % left, % captures/s, % failed, on %", ocr_duration_text(elapsed), ocr_duration_text(left), FF(rate, 1), failed, ifx ocr_ui.engine && ocr_ui.engine.provider == .CUDA then "the GPU" else "the CPU");
|
||||
if scan.paused {
|
||||
if Button("Resume") { scan.paused_seconds += to_float64_seconds(current_time_monotonic() - scan.pause_start); atomic_swap(*scan.paused, false); }
|
||||
} else {
|
||||
@ -483,6 +624,8 @@ ocr_duration_text :: (seconds: float64) -> string {
|
||||
ocr_ui_frame :: () { using ImGui;
|
||||
busy := ocr_ui.thread_busy;
|
||||
if app.scan_and_exit ocr_ui_scan_and_exit(busy);
|
||||
if app.erase_and_exit ocr_ui_erase_and_exit(busy);
|
||||
ocr_erase_popup();
|
||||
if !busy && ocr_ui.engine && !ocr_ui.toast_shown {
|
||||
ocr_ui.toast_shown = true;
|
||||
in_use, explanation := ocr_gpu_status(ocr_ui.engine);
|
||||
|
||||
@ -790,7 +790,9 @@ viewer_ui :: () { using ImGui;
|
||||
if ocr_ui.scan.running {
|
||||
done, total, failed, rate, left, elapsed := ocr_scan_progress();
|
||||
ProgressBar(cast(float) done / max(total, 1), .{-FLOAT32_MIN, max(2.0, GetFontSize() * 0.15)}, "");
|
||||
if IsItemHovered() SetTooltip("OCR scan with %: %, % elapsed, % left, % failed%", ocr_ui.scan.model, ocr_scan_progress_text(done, total), ocr_duration_text(elapsed), ocr_duration_text(left), failed, ifx ocr_ui.scan.paused then " (paused)" else "");
|
||||
scan := *ocr_ui.scan;
|
||||
doing := ifx scan.erase_ocr || scan.erase_labels then tprint("Erasing %", erase_what_text(scan)) else tprint("OCR scan with %", scan.model);
|
||||
if IsItemHovered() SetTooltip("%: %, % elapsed, % left, % failed%", doing, ocr_scan_progress_text(done, total), ocr_duration_text(elapsed), ocr_duration_text(left), failed, ifx scan.paused then " (paused)" else "");
|
||||
}
|
||||
|
||||
BeginChild("grid");
|
||||
|
||||
Loading…
Reference in New Issue
Block a user