From bedeaebe41263625896126ecbd9a8512c2a58c4f Mon Sep 17 00:00:00 2001 From: Musa Mahmood Date: Thu, 8 Oct 2026 11:47:30 +0100 Subject: [PATCH] [0.0.6] Erase OCR readings, or labels and vectors, from the whole library (for testing) --- README.md | 5 +- check_targets.jai | 2 +- first.jai | 2 +- modules/Capture_File/module.jai | 34 ++++++++ src/App.jai | 1 + src/app/labels_ui.jai | 8 ++ src/app/main.jai | 1 + src/app/ocr_ui.jai | 149 +++++++++++++++++++++++++++++++- src/app/viewer.jai | 4 +- 9 files changed, 199 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index b520be7..1265f1d 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,10 @@ and everything around them, ready for an app to be written into `src/app/app_ui. - **OCR** (`modules/OCR`): the OCR window shows whether it runs on the GPU and, if not, what to install, and switches model (kept as `model` under `[ocr]` in the settings file). **Scan the library** reads every capture shown that the model has not read yet, storing each reading in its file as it goes, - so it can pause, cancel and pick up again; with **Labels too** it labels them in the same pass. The + so it can pause, cancel and pick up again; with **Labels too** it labels them in the same pass. + For testing, the OCR window's **Erase OCR readings** and the Labels window's **Erase labels and + vectors** take them out of every file in the library again (after asking); `-erase ocr|labels|all` + does the same without the UI and exits. The Capture window's **OCR** button reads one capture; a capture read by several models keeps each reading, and the window can show any of them. `-library FOLDER` shows another folder of `.mcap` files; `-scan` scans it with the chosen model once it has loaded and exits (0, or 1 if any capture diff --git a/check_targets.jai b/check_targets.jai index 3891cb4..8ec9445 100644 --- a/check_targets.jai +++ b/check_targets.jai @@ -29,7 +29,7 @@ APP_NAME :: "mcap"; APP_FOLDER_NAME :: "mcap"; GIT_BRANCH :: "check"; GIT_HASH :: "-"; -VERSION :: "0.0.5"; +VERSION :: "0.0.6"; BUILD_DATE :: "-"; DEBUG :: true; COMPILE_TESTS :: true; diff --git a/first.jai b/first.jai index 5421613..703ab45 100644 --- a/first.jai +++ b/first.jai @@ -8,7 +8,7 @@ APP_NAME :: "mcap"; APP_FOLDER_NAME :: "mcap"; APP_BUNDLE_IDENTIFIER :: "com.example.mcap"; -VERSION :: "0.0.5"; +VERSION :: "0.0.6"; // Where everything goes: bin/, or bin-headless/ for builds made with no_crash_dialog, so automated // builds never write over the ones you run by hand. build_directory := "bin"; diff --git a/modules/Capture_File/module.jai b/modules/Capture_File/module.jai index 5292924..9e686e4 100644 --- a/modules/Capture_File/module.jai +++ b/modules/Capture_File/module.jai @@ -321,6 +321,40 @@ mcap_store :: (path: string, ocr: *Mcap_Ocr_Section = null, front: [] Mcap_Chunk return written, write_error; } +// Takes out of the .mcap at path: with ocr, every OCR section; and every chunk, front or back, whose +// tag is one of tags (whatever its key). Everything else stays as it is. The file is rewritten (and +// upgraded to the current version) only when there was something to take out; changed says whether. +mcap_erase :: (path: string, ocr := false, tags: [] string = .[]) -> ok: bool, error: string, changed: bool { + data, read_ok := read_entire_file(path, log_errors = false); + if !read_ok return false, "cannot be read", false; + defer free(data); + m, parsed, parse_error := mcap_parse(cast([] u8) data); + if !parsed return false, parse_error, false; + + push_allocator(temp); + changed := false; + if ocr && m.ocr.count { m.ocr = ""; changed = true; } + without :: (chunks: [] Mcap_Chunk, tags: [] string, changed: *bool) -> [] Mcap_Chunk { + kept: [..] Mcap_Chunk; + for chunks { + erase := false; + for tag: tags if mcap_same_tag(it.tag, mcap_tag(tag)) erase = true; + if erase changed.* = true; + else array_add(*kept, it); + } + return kept; + } + m.front_chunks = without(m.front_chunks, tags, *changed); + m.back_chunks = without(m.back_chunks, tags, *changed); + if !changed return true, "", false; + + m.version = MCAP_VERSION; + serializer, serialized, serialize_error := mcap_serialize(m); + if !serialized return false, serialize_error, false; + written, write_error := mcap_write(path, serializer); + return written, write_error, written; +} + // Stores chunk alone: in front, or with back after the images. See mcap_store. mcap_store_chunk :: (path: string, chunk: Mcap_Chunk, back := false) -> ok: bool, error: string { if !back { diff --git a/src/App.jai b/src/App.jai index 29fd8b4..00eee3e 100644 --- a/src/App.jai +++ b/src/App.jai @@ -184,6 +184,7 @@ App :: struct { scan_and_exit: bool; // -scan: scan the library with the chosen OCR model once it has loaded, then exit (0, or 1 if any capture failed) stats_and_exit: bool; // -stats: log the Statistics window's numbers once the catalog has loaded, then exit start_search: string; // -search QUERY: the search box's text at startup + erase_and_exit: string; // -erase ocr|labels|all: erase that from the whole library once it has loaded, then exit last_window_restore_time: Apollo_Time; imgui_context: *ImGui.ImGuiContext; diff --git a/src/app/labels_ui.jai b/src/app/labels_ui.jai index 94f24dc..4345c47 100644 --- a/src/app/labels_ui.jai +++ b/src/app/labels_ui.jai @@ -290,6 +290,14 @@ labels_window_contents :: () { using ImGui; } SeparatorText("The library"); Text("% of % captures labelled; % with this list.", labels_ui.labelled_count, viewer_capture_count(), labels_ui.current_count); + if ocr_ui.scan.running { + ocr_scan_progress_ui(); + } else { + if busy BeginDisabled(); + if SmallButton("Erase labels and vectors from the library...") ocr_ui.erase_asked = .LABELS; + if busy EndDisabled(); + if IsItemHovered(.AllowWhenDisabled) SetTooltip("Takes every capture's labels and image vector, of every label model, out of its file (asks first). For testing: a scan with Labels too makes them again."); + } TextColored(app_theme.palette.dim, "The OCR window's scan labels the rest, and labels again from their vectors those scored with an earlier list."); if SmallButton("OCR window") ocr_ui.window_open = true; diff --git a/src/app/main.jai b/src/app/main.jai index 1ddd689..712eb29 100644 --- a/src/app/main.jai +++ b/src/app/main.jai @@ -21,6 +21,7 @@ main :: () { app.scan_and_exit = array_find(app.args, "-scan"); app.stats_and_exit = array_find(app.args, "-stats"); for app.args if it == "-search" && it_index + 1 < app.args.count app.start_search = app.args[it_index + 1]; + for app.args if it == "-erase" && it_index + 1 < app.args.count app.erase_and_exit = app.args[it_index + 1]; defer report_memory_leaks(); diff --git a/src/app/ocr_ui.jai b/src/app/ocr_ui.jai index b5357da..a6f24a2 100644 --- a/src/app/ocr_ui.jai +++ b/src/app/ocr_ui.jai @@ -36,6 +36,7 @@ ocr_ui: struct { scan: Ocr_Scan; scan_requested: bool; // -scan has started its scan + erase_asked: Erase_What; // the erase waiting for its confirmation scan_logged_step: s64; // -scan's last progress line, in twentieths of the scan // The Capture window. @@ -48,7 +49,7 @@ ocr_ui: struct { toast_shown := false; } -Ocr_Task_Kind :: enum { NONE; CREATE; READ; SCAN; LABELS_CREATE; LABEL_SET; } +Ocr_Task_Kind :: enum { NONE; CREATE; READ; SCAN; LABELS_CREATE; LABEL_SET; ERASE; } Ocr_Task :: struct { kind: Ocr_Task_Kind; @@ -75,9 +76,12 @@ Ocr_Scan :: struct { pause_start: Apollo_Time; elapsed_seconds: float64; // once it has ended: the time spent scanning, pauses left out label_set: *Label_Set; // labels scored against this; null when the scan does not label + erase_ocr, erase_labels: bool; // an erase, not a scan: what it takes out of each file } // A capture to read, and what it needs. +Erase_What :: enum u8 { NONE; OCR; LABELS; ALL; } + Scan_Item :: struct { index: s32; path: string; @@ -211,6 +215,45 @@ ocr_ui_thread_proc :: (thread: *Thread) -> s64 { log("[OCR scan] % of % captures read with %% in % (% failed)%.", scan.done, scan.total, scan.model, ifx scan.label_set then tprint(" and labelled with %", LABELS_MODEL) else "", ocr_duration_text(scan.elapsed_seconds), scan.failed, ifx scan.cancel then ", cancelled" else ""); atomic_swap(*scan.running, false); + + case .ERASE; + scan := *ocr_ui.scan; + // Not in temporary storage, which is reset after each capture. + label_tags := string.[MCAP_LABELS_TAG, MCAP_EMBEDDING_TAG]; + tags: [] string; + if scan.erase_labels tags = label_tags; + erased := 0; + for item: scan.items { + while scan.paused && !scan.cancel sleep_milliseconds(100); + if scan.cancel break; + ok, changed: bool; + error: string; + for 1..OCR_STORE_ATTEMPTS { + ok, error, changed = mcap_erase(item.path, scan.erase_ocr, tags); + if ok break; + sleep_milliseconds(250); + } + if !ok { + atomic_add(*scan.failed, 1); + log_error("[Erase] % %", item.path, error); + } else if changed { + erased += 1; + entry, load_error := catalog_load(item.path); + if entry { + lock(*viewer.mutex); + array_add(*viewer.catalog_updates, .{item.index, entry}); + unlock(*viewer.mutex); + } else { + log_error("[Erase] % was erased, but could not be read back: %", item.path, load_error); + } + } + atomic_add(*scan.done, 1); + reset_temporary_storage(); + } + scan.elapsed_seconds = to_float64_seconds(current_time_monotonic() - scan.start) - scan.paused_seconds; + log("[Erase] % erased from % of % captures in % (% failed)%.", erase_what_text(scan), erased, scan.total, + ocr_duration_text(scan.elapsed_seconds), scan.failed, ifx scan.cancel then ", cancelled" else ""); + atomic_swap(*scan.running, false); } free(task.model); free(task.path); @@ -349,6 +392,91 @@ ocr_read_and_store :: (items: [] Scan_Item, set: *Label_Set) -> [] string { // ---------------------------------------------------------------- the scan +// -erase: starts the erase once the catalog is loaded, and ends the app when it is done. +ocr_ui_erase_and_exit :: (busy: bool) { + if !ocr_ui.scan_requested { + if busy || viewer.catalog_loaded < viewer.captures.count return; + what: Erase_What; + if app.erase_and_exit == { + case "ocr"; what = .OCR; + case "labels"; what = .LABELS; + case "all"; what = .ALL; + case; + log_error("[Erase] -erase takes ocr, labels or all, not %.", app.erase_and_exit); + app.exit_code = 2; app.should_exit = true; app.erase_and_exit = ""; return; + } + ocr_ui.scan_requested = true; + ocr_ui.window_open = true; // its progress bar + ocr_erase_start(what); + return; + } + if busy || ocr_ui.scan.running return; + log("[Erase] -erase is done: % in %, % failed.", ocr_scan_progress_text(ocr_ui.scan.done, ocr_ui.scan.total), ocr_duration_text(ocr_ui.scan.elapsed_seconds), ocr_ui.scan.failed); + app.exit_code = ifx ocr_ui.scan.failed then cast(s32) 1 else 0; + app.should_exit = true; + app.erase_and_exit = ""; +} + +// What an erase takes out, for people. Temp. +erase_what_text :: (scan: *Ocr_Scan) -> string { + if scan.erase_ocr && scan.erase_labels return "OCR, labels and vectors"; + if scan.erase_ocr return "OCR readings"; + return "labels and vectors"; +} + +// Takes OCR readings, labels and vectors, or both, out of every capture in the library (not only +// those shown) that has any, on the OCR thread, as a scan runs: with its progress, pause and cancel. +ocr_erase_start :: (what: Erase_What) { + if ocr_ui.thread_busy || what == .NONE return; + scan := *ocr_ui.scan; + for scan.items free(it.path); + array_reset(*scan.items); + free(scan.model); + scan.* = .{}; + scan.items.allocator = context.default_allocator; + scan.erase_ocr = what == .OCR || what == .ALL; + scan.erase_labels = what == .LABELS || what == .ALL; + for * viewer.captures { + if it.removed continue; + entry := it.catalog; + has := !entry // not known: looked at anyway + || (scan.erase_ocr && entry.ocr.count > 0) + || (scan.erase_labels && (entry.labels.count > 0 || entry.embedding.count > 0 || entry.label_list.count > 0)); + if !has continue; + array_add(*scan.items, .{index = cast(s32) it_index, path = copy_string(it.path,, context.default_allocator)}); + } + scan.total = scan.items.count; + if !scan.total { toast_show(.Success, "No capture has % to erase", erase_what_text(scan)); return; } + scan.running = true; + scan.start = current_time_monotonic(); + ocr_ui_start(.{kind = .ERASE}); +} + +// The confirmation for ocr_ui.erase_asked, as a modal popup; at the top level, each frame. +ocr_erase_popup :: () { using ImGui; + POPUP :: "Erase from the library?"; + if ocr_ui.erase_asked != .NONE && !IsPopupOpen(POPUP) OpenPopup(POPUP); + if !BeginPopupModal(POPUP, null, .AlwaysAutoResize) return; + what := ifx ocr_ui.erase_asked == .OCR then "every OCR reading (all models)" else ifx ocr_ui.erase_asked == .LABELS then "every capture's labels and image vectors (all label models)" else "every OCR reading, label and image vector"; + Text("Erase % from all % captures in %?", what, viewer_capture_count(), viewer_folder()); + TextColored(app_theme.palette.dim, "Each file holding any is rewritten without them; its images, text and preview stay as they are."); + TextColored(app_theme.palette.warning, "This cannot be undone: only scanning the library again brings them back."); + Spacing(); + if Button("Erase") { + what_asked := ocr_ui.erase_asked; + ocr_ui.erase_asked = .NONE; + CloseCurrentPopup(); + ocr_ui.window_open = true; // its progress bar + ocr_erase_start(what_asked); + } + SameLine(); + if Button("Cancel") || IsKeyPressed(.Escape) { + ocr_ui.erase_asked = .NONE; + CloseCurrentPopup(); + } + EndPopup(); +} + // -scan: starts the scan once the engine and the catalog are ready, and ends the app when it is done. ocr_ui_scan_and_exit :: (busy: bool) { finish :: (code: s32, format: string, args: ..Any) { @@ -432,14 +560,27 @@ ocr_scan_ui :: () { using ImGui; if !catalog_ready TextColored(app_theme.palette.dim, "(once the catalog has loaded)"); else TextColored(app_theme.palette.dim, "Reads every capture this model has not read, and labels those not yet labelled with this list. Each is stored as it is read, so a scan can stop and start again."); if scan.total && scan.done { - TextColored(app_theme.palette.dim, "Last scan: % of % read with % in %, % failed.", scan.done, scan.total, scan.model, ocr_duration_text(scan.elapsed_seconds), scan.failed); + if scan.erase_ocr || scan.erase_labels TextColored(app_theme.palette.dim, "Last erase: % from % of % captures in %, % failed.", erase_what_text(scan), scan.done, scan.total, ocr_duration_text(scan.elapsed_seconds), scan.failed); + else TextColored(app_theme.palette.dim, "Last scan: % of % read with % in %, % failed.", scan.done, scan.total, scan.model, ocr_duration_text(scan.elapsed_seconds), scan.failed); } + if busy BeginDisabled(); + if SmallButton("Erase OCR readings from the library...") ocr_ui.erase_asked = .OCR; + if busy EndDisabled(); + if IsItemHovered(.AllowWhenDisabled) SetTooltip("Takes every OCR reading, of every model, out of every capture's file (asks first). For testing: a scan reads them again."); return; } + ocr_scan_progress_ui(); +} + +// The running scan's or erase's progress bar and buttons (the OCR and Labels windows show it). +ocr_scan_progress_ui :: () { using ImGui; + scan := *ocr_ui.scan; + if scan.erase_ocr || scan.erase_labels Text("Erasing %:", erase_what_text(scan)); done, total, failed, rate, left, elapsed := ocr_scan_progress(); ProgressBar(cast(float) done / max(total, 1), .{-FLOAT32_MIN, 0}, temp_c_string(ocr_scan_progress_text(done, total))); - Text("% elapsed, % left, % captures/s, % failed, on %", ocr_duration_text(elapsed), ocr_duration_text(left), FF(rate, 1), failed, ifx ocr_ui.engine && ocr_ui.engine.provider == .CUDA then "the GPU" else "the CPU"); + if scan.erase_ocr || scan.erase_labels Text("% elapsed, % left, % captures/s, % failed", ocr_duration_text(elapsed), ocr_duration_text(left), FF(rate, 1), failed); + else Text("% elapsed, % left, % captures/s, % failed, on %", ocr_duration_text(elapsed), ocr_duration_text(left), FF(rate, 1), failed, ifx ocr_ui.engine && ocr_ui.engine.provider == .CUDA then "the GPU" else "the CPU"); if scan.paused { if Button("Resume") { scan.paused_seconds += to_float64_seconds(current_time_monotonic() - scan.pause_start); atomic_swap(*scan.paused, false); } } else { @@ -483,6 +624,8 @@ ocr_duration_text :: (seconds: float64) -> string { ocr_ui_frame :: () { using ImGui; busy := ocr_ui.thread_busy; if app.scan_and_exit ocr_ui_scan_and_exit(busy); + if app.erase_and_exit ocr_ui_erase_and_exit(busy); + ocr_erase_popup(); if !busy && ocr_ui.engine && !ocr_ui.toast_shown { ocr_ui.toast_shown = true; in_use, explanation := ocr_gpu_status(ocr_ui.engine); diff --git a/src/app/viewer.jai b/src/app/viewer.jai index aa8851c..e87d515 100644 --- a/src/app/viewer.jai +++ b/src/app/viewer.jai @@ -790,7 +790,9 @@ viewer_ui :: () { using ImGui; if ocr_ui.scan.running { done, total, failed, rate, left, elapsed := ocr_scan_progress(); ProgressBar(cast(float) done / max(total, 1), .{-FLOAT32_MIN, max(2.0, GetFontSize() * 0.15)}, ""); - if IsItemHovered() SetTooltip("OCR scan with %: %, % elapsed, % left, % failed%", ocr_ui.scan.model, ocr_scan_progress_text(done, total), ocr_duration_text(elapsed), ocr_duration_text(left), failed, ifx ocr_ui.scan.paused then " (paused)" else ""); + scan := *ocr_ui.scan; + doing := ifx scan.erase_ocr || scan.erase_labels then tprint("Erasing %", erase_what_text(scan)) else tprint("OCR scan with %", scan.model); + if IsItemHovered() SetTooltip("%: %, % elapsed, % left, % failed%", doing, ocr_scan_progress_text(done, total), ocr_duration_text(elapsed), ocr_duration_text(left), failed, ifx scan.paused then " (paused)" else ""); } BeginChild("grid");