[0.0.6] Erase OCR readings, or labels and vectors, from the whole library (for testing)

README.md

@@ -47,7 +47,10 @@ and everything around them, ready for an app to be written into `src/app/app_ui.
 - **OCR** (`modules/OCR`): the OCR window shows whether it runs on the GPU and, if not, what to install,
   and switches model (kept as `model` under `[ocr]` in the settings file). **Scan the library** reads
   every capture shown that the model has not read yet, storing each reading in its file as it goes,
-  so it can pause, cancel and pick up again; with **Labels too** it labels them in the same pass. The
+  so it can pause, cancel and pick up again; with **Labels too** it labels them in the same pass.
+  For testing, the OCR window's **Erase OCR readings** and the Labels window's **Erase labels and
+  vectors** take them out of every file in the library again (after asking); `-erase ocr|labels|all`
+  does the same without the UI and exits. The
   Capture window's **OCR** button reads one capture; a capture read by several models keeps each
   reading, and the window can show any of them. `-library FOLDER` shows another folder of `.mcap` files;
   `-scan` scans it with the chosen model once it has loaded and exits (0, or 1 if any capture

check_targets.jai

@@ -29,7 +29,7 @@ APP_NAME        :: "mcap";
 APP_FOLDER_NAME :: "mcap";
 GIT_BRANCH    :: "check";
 GIT_HASH      :: "-";
-VERSION       :: "0.0.5";
+VERSION       :: "0.0.6";
 BUILD_DATE    :: "-";
 DEBUG         :: true;
 COMPILE_TESTS :: true;

first.jai

@@ -8,7 +8,7 @@ APP_NAME              :: "mcap";
 APP_FOLDER_NAME       :: "mcap";
 APP_BUNDLE_IDENTIFIER :: "com.example.mcap";
 
-VERSION :: "0.0.5";
+VERSION :: "0.0.6";
 // Where everything goes: bin/, or bin-headless/ for builds made with no_crash_dialog, so automated
 // builds never write over the ones you run by hand.
 build_directory := "bin";

modules/Capture_File/module.jai

@@ -321,6 +321,40 @@ mcap_store :: (path: string, ocr: *Mcap_Ocr_Section = null, front: [] Mcap_Chunk
   return written, write_error;
 }
 
+// Takes out of the .mcap at path: with ocr, every OCR section; and every chunk, front or back, whose
+// tag is one of tags (whatever its key). Everything else stays as it is. The file is rewritten (and
+// upgraded to the current version) only when there was something to take out; changed says whether.
+mcap_erase :: (path: string, ocr := false, tags: [] string = .[]) -> ok: bool, error: string, changed: bool {
+  data, read_ok := read_entire_file(path, log_errors = false);
+  if !read_ok  return false, "cannot be read", false;
+  defer free(data);
+  m, parsed, parse_error := mcap_parse(cast([] u8) data);
+  if !parsed  return false, parse_error, false;
+
+  push_allocator(temp);
+  changed := false;
+  if ocr && m.ocr.count { m.ocr = ""; changed = true; }
+  without :: (chunks: [] Mcap_Chunk, tags: [] string, changed: *bool) -> [] Mcap_Chunk {
+    kept: [..] Mcap_Chunk;
+    for chunks {
+      erase := false;
+      for tag: tags  if mcap_same_tag(it.tag, mcap_tag(tag))  erase = true;
+      if erase  changed.* = true;
+      else      array_add(*kept, it);
+    }
+    return kept;
+  }
+  m.front_chunks = without(m.front_chunks, tags, *changed);
+  m.back_chunks  = without(m.back_chunks,  tags, *changed);
+  if !changed  return true, "", false;
+
+  m.version = MCAP_VERSION;
+  serializer, serialized, serialize_error := mcap_serialize(m);
+  if !serialized  return false, serialize_error, false;
+  written, write_error := mcap_write(path, serializer);
+  return written, write_error, written;
+}
+
 // Stores chunk alone: in front, or with back after the images. See mcap_store.
 mcap_store_chunk :: (path: string, chunk: Mcap_Chunk, back := false) -> ok: bool, error: string {
   if !back {

src/App.jai

@@ -184,6 +184,7 @@ App :: struct {
   scan_and_exit: bool;      // -scan: scan the library with the chosen OCR model once it has loaded, then exit (0, or 1 if any capture failed)
   stats_and_exit: bool;     // -stats: log the Statistics window's numbers once the catalog has loaded, then exit
   start_search: string;     // -search QUERY: the search box's text at startup
+  erase_and_exit: string;   // -erase ocr|labels|all: erase that from the whole library once it has loaded, then exit
   last_window_restore_time: Apollo_Time;
 
   imgui_context: *ImGui.ImGuiContext;

src/app/labels_ui.jai

@@ -290,6 +290,14 @@ labels_window_contents :: () { using ImGui;
   }
   SeparatorText("The library");
   Text("% of % captures labelled; % with this list.", labels_ui.labelled_count, viewer_capture_count(), labels_ui.current_count);
+  if ocr_ui.scan.running {
+    ocr_scan_progress_ui();
+  } else {
+    if busy  BeginDisabled();
+    if SmallButton("Erase labels and vectors from the library...")  ocr_ui.erase_asked = .LABELS;
+    if busy  EndDisabled();
+    if IsItemHovered(.AllowWhenDisabled)  SetTooltip("Takes every capture's labels and image vector, of every label model, out of its file (asks first). For testing: a scan with Labels too makes them again.");
+  }
   TextColored(app_theme.palette.dim, "The OCR window's scan labels the rest, and labels again from their vectors those scored with an earlier list.");
   if SmallButton("OCR window")  ocr_ui.window_open = true;
 

src/app/main.jai

@@ -21,6 +21,7 @@ main :: () {
   app.scan_and_exit = array_find(app.args, "-scan");
   app.stats_and_exit = array_find(app.args, "-stats");
   for app.args  if it == "-search" && it_index + 1 < app.args.count  app.start_search = app.args[it_index + 1];
+  for app.args  if it == "-erase" && it_index + 1 < app.args.count  app.erase_and_exit = app.args[it_index + 1];
 
   defer report_memory_leaks();
 

src/app/ocr_ui.jai

@@ -36,6 +36,7 @@ ocr_ui: struct {
 
   scan: Ocr_Scan;
   scan_requested: bool;   // -scan has started its scan
+  erase_asked: Erase_What;  // the erase waiting for its confirmation
   scan_logged_step: s64;  // -scan's last progress line, in twentieths of the scan
 
   // The Capture window.
@@ -48,7 +49,7 @@ ocr_ui: struct {
   toast_shown := false;
 }
 
-Ocr_Task_Kind :: enum { NONE; CREATE; READ; SCAN; LABELS_CREATE; LABEL_SET; }
+Ocr_Task_Kind :: enum { NONE; CREATE; READ; SCAN; LABELS_CREATE; LABEL_SET; ERASE; }
 
 Ocr_Task :: struct {
   kind: Ocr_Task_Kind;
@@ -75,9 +76,12 @@ Ocr_Scan :: struct {
   pause_start: Apollo_Time;
   elapsed_seconds: float64;      // once it has ended: the time spent scanning, pauses left out
   label_set: *Label_Set;         // labels scored against this; null when the scan does not label
+  erase_ocr, erase_labels: bool; // an erase, not a scan: what it takes out of each file
 }
 
 // A capture to read, and what it needs.
+Erase_What :: enum u8 { NONE; OCR; LABELS; ALL; }
+
 Scan_Item :: struct {
   index: s32;
   path: string;
@@ -211,6 +215,45 @@ ocr_ui_thread_proc :: (thread: *Thread) -> s64 {
       log("[OCR scan] % of % captures read with %% in % (% failed)%.", scan.done, scan.total, scan.model, ifx scan.label_set then tprint(" and labelled with %", LABELS_MODEL) else "",
           ocr_duration_text(scan.elapsed_seconds), scan.failed, ifx scan.cancel then ", cancelled" else "");
       atomic_swap(*scan.running, false);
+
+    case .ERASE;
+      scan := *ocr_ui.scan;
+      // Not in temporary storage, which is reset after each capture.
+      label_tags := string.[MCAP_LABELS_TAG, MCAP_EMBEDDING_TAG];
+      tags: [] string;
+      if scan.erase_labels  tags = label_tags;
+      erased := 0;
+      for item: scan.items {
+        while scan.paused && !scan.cancel  sleep_milliseconds(100);
+        if scan.cancel  break;
+        ok, changed: bool;
+        error: string;
+        for 1..OCR_STORE_ATTEMPTS {
+          ok, error, changed = mcap_erase(item.path, scan.erase_ocr, tags);
+          if ok  break;
+          sleep_milliseconds(250);
+        }
+        if !ok {
+          atomic_add(*scan.failed, 1);
+          log_error("[Erase] % %", item.path, error);
+        } else if changed {
+          erased += 1;
+          entry, load_error := catalog_load(item.path);
+          if entry {
+            lock(*viewer.mutex);
+            array_add(*viewer.catalog_updates, .{item.index, entry});
+            unlock(*viewer.mutex);
+          } else {
+            log_error("[Erase] % was erased, but could not be read back: %", item.path, load_error);
+          }
+        }
+        atomic_add(*scan.done, 1);
+        reset_temporary_storage();
+      }
+      scan.elapsed_seconds = to_float64_seconds(current_time_monotonic() - scan.start) - scan.paused_seconds;
+      log("[Erase] % erased from % of % captures in % (% failed)%.", erase_what_text(scan), erased, scan.total,
+          ocr_duration_text(scan.elapsed_seconds), scan.failed, ifx scan.cancel then ", cancelled" else "");
+      atomic_swap(*scan.running, false);
   }
   free(task.model);
   free(task.path);
@@ -349,6 +392,91 @@ ocr_read_and_store :: (items: [] Scan_Item, set: *Label_Set) -> [] string {
 
 // ---------------------------------------------------------------- the scan
 
+// -erase: starts the erase once the catalog is loaded, and ends the app when it is done.
+ocr_ui_erase_and_exit :: (busy: bool) {
+  if !ocr_ui.scan_requested {
+    if busy || viewer.catalog_loaded < viewer.captures.count  return;
+    what: Erase_What;
+    if app.erase_and_exit == {
+      case "ocr";     what = .OCR;
+      case "labels";  what = .LABELS;
+      case "all";     what = .ALL;
+      case;
+        log_error("[Erase] -erase takes ocr, labels or all, not %.", app.erase_and_exit);
+        app.exit_code = 2; app.should_exit = true; app.erase_and_exit = ""; return;
+    }
+    ocr_ui.scan_requested = true;
+    ocr_ui.window_open = true;   // its progress bar
+    ocr_erase_start(what);
+    return;
+  }
+  if busy || ocr_ui.scan.running  return;
+  log("[Erase] -erase is done: % in %, % failed.", ocr_scan_progress_text(ocr_ui.scan.done, ocr_ui.scan.total), ocr_duration_text(ocr_ui.scan.elapsed_seconds), ocr_ui.scan.failed);
+  app.exit_code = ifx ocr_ui.scan.failed then cast(s32) 1 else 0;
+  app.should_exit = true;
+  app.erase_and_exit = "";
+}
+
+// What an erase takes out, for people. Temp.
+erase_what_text :: (scan: *Ocr_Scan) -> string {
+  if scan.erase_ocr && scan.erase_labels  return "OCR, labels and vectors";
+  if scan.erase_ocr  return "OCR readings";
+  return "labels and vectors";
+}
+
+// Takes OCR readings, labels and vectors, or both, out of every capture in the library (not only
+// those shown) that has any, on the OCR thread, as a scan runs: with its progress, pause and cancel.
+ocr_erase_start :: (what: Erase_What) {
+  if ocr_ui.thread_busy || what == .NONE  return;
+  scan := *ocr_ui.scan;
+  for scan.items  free(it.path);
+  array_reset(*scan.items);
+  free(scan.model);
+  scan.* = .{};
+  scan.items.allocator = context.default_allocator;
+  scan.erase_ocr    = what == .OCR || what == .ALL;
+  scan.erase_labels = what == .LABELS || what == .ALL;
+  for * viewer.captures {
+    if it.removed  continue;
+    entry := it.catalog;
+    has := !entry   // not known: looked at anyway
+        || (scan.erase_ocr && entry.ocr.count > 0)
+        || (scan.erase_labels && (entry.labels.count > 0 || entry.embedding.count > 0 || entry.label_list.count > 0));
+    if !has  continue;
+    array_add(*scan.items, .{index = cast(s32) it_index, path = copy_string(it.path,, context.default_allocator)});
+  }
+  scan.total = scan.items.count;
+  if !scan.total { toast_show(.Success, "No capture has % to erase", erase_what_text(scan)); return; }
+  scan.running = true;
+  scan.start = current_time_monotonic();
+  ocr_ui_start(.{kind = .ERASE});
+}
+
+// The confirmation for ocr_ui.erase_asked, as a modal popup; at the top level, each frame.
+ocr_erase_popup :: () { using ImGui;
+  POPUP :: "Erase from the library?";
+  if ocr_ui.erase_asked != .NONE && !IsPopupOpen(POPUP)  OpenPopup(POPUP);
+  if !BeginPopupModal(POPUP, null, .AlwaysAutoResize)  return;
+  what := ifx ocr_ui.erase_asked == .OCR then "every OCR reading (all models)" else ifx ocr_ui.erase_asked == .LABELS then "every capture's labels and image vectors (all label models)" else "every OCR reading, label and image vector";
+  Text("Erase % from all % captures in %?", what, viewer_capture_count(), viewer_folder());
+  TextColored(app_theme.palette.dim, "Each file holding any is rewritten without them; its images, text and preview stay as they are.");
+  TextColored(app_theme.palette.warning, "This cannot be undone: only scanning the library again brings them back.");
+  Spacing();
+  if Button("Erase") {
+    what_asked := ocr_ui.erase_asked;
+    ocr_ui.erase_asked = .NONE;
+    CloseCurrentPopup();
+    ocr_ui.window_open = true;   // its progress bar
+    ocr_erase_start(what_asked);
+  }
+  SameLine();
+  if Button("Cancel") || IsKeyPressed(.Escape) {
+    ocr_ui.erase_asked = .NONE;
+    CloseCurrentPopup();
+  }
+  EndPopup();
+}
+
 // -scan: starts the scan once the engine and the catalog are ready, and ends the app when it is done.
 ocr_ui_scan_and_exit :: (busy: bool) {
   finish :: (code: s32, format: string, args: ..Any) {
@@ -432,14 +560,27 @@ ocr_scan_ui :: () { using ImGui;
     if !catalog_ready  TextColored(app_theme.palette.dim, "(once the catalog has loaded)");
     else               TextColored(app_theme.palette.dim, "Reads every capture this model has not read, and labels those not yet labelled with this list. Each is stored as it is read, so a scan can stop and start again.");
     if scan.total && scan.done {
-      TextColored(app_theme.palette.dim, "Last scan: % of % read with % in %, % failed.", scan.done, scan.total, scan.model, ocr_duration_text(scan.elapsed_seconds), scan.failed);
+      if scan.erase_ocr || scan.erase_labels  TextColored(app_theme.palette.dim, "Last erase: % from % of % captures in %, % failed.", erase_what_text(scan), scan.done, scan.total, ocr_duration_text(scan.elapsed_seconds), scan.failed);
+      else  TextColored(app_theme.palette.dim, "Last scan: % of % read with % in %, % failed.", scan.done, scan.total, scan.model, ocr_duration_text(scan.elapsed_seconds), scan.failed);
     }
+    if busy  BeginDisabled();
+    if SmallButton("Erase OCR readings from the library...")  ocr_ui.erase_asked = .OCR;
+    if busy  EndDisabled();
+    if IsItemHovered(.AllowWhenDisabled)  SetTooltip("Takes every OCR reading, of every model, out of every capture's file (asks first). For testing: a scan reads them again.");
     return;
   }
+  ocr_scan_progress_ui();
+}
+
+// The running scan's or erase's progress bar and buttons (the OCR and Labels windows show it).
+ocr_scan_progress_ui :: () { using ImGui;
+  scan := *ocr_ui.scan;
+  if scan.erase_ocr || scan.erase_labels  Text("Erasing %:", erase_what_text(scan));
 
   done, total, failed, rate, left, elapsed := ocr_scan_progress();
   ProgressBar(cast(float) done / max(total, 1), .{-FLOAT32_MIN, 0}, temp_c_string(ocr_scan_progress_text(done, total)));
-  Text("% elapsed, % left, % captures/s, % failed, on %", ocr_duration_text(elapsed), ocr_duration_text(left), FF(rate, 1), failed, ifx ocr_ui.engine && ocr_ui.engine.provider == .CUDA then "the GPU" else "the CPU");
+  if scan.erase_ocr || scan.erase_labels  Text("% elapsed, % left, % captures/s, % failed", ocr_duration_text(elapsed), ocr_duration_text(left), FF(rate, 1), failed);
+  else  Text("% elapsed, % left, % captures/s, % failed, on %", ocr_duration_text(elapsed), ocr_duration_text(left), FF(rate, 1), failed, ifx ocr_ui.engine && ocr_ui.engine.provider == .CUDA then "the GPU" else "the CPU");
   if scan.paused {
     if Button("Resume") { scan.paused_seconds += to_float64_seconds(current_time_monotonic() - scan.pause_start); atomic_swap(*scan.paused, false); }
   } else {
@@ -483,6 +624,8 @@ ocr_duration_text :: (seconds: float64) -> string {
 ocr_ui_frame :: () { using ImGui;
   busy := ocr_ui.thread_busy;
   if app.scan_and_exit  ocr_ui_scan_and_exit(busy);
+  if app.erase_and_exit  ocr_ui_erase_and_exit(busy);
+  ocr_erase_popup();
   if !busy && ocr_ui.engine && !ocr_ui.toast_shown {
     ocr_ui.toast_shown = true;
     in_use, explanation := ocr_gpu_status(ocr_ui.engine);

src/app/viewer.jai

@@ -790,7 +790,9 @@ viewer_ui :: () { using ImGui;
   if ocr_ui.scan.running {
     done, total, failed, rate, left, elapsed := ocr_scan_progress();
     ProgressBar(cast(float) done / max(total, 1), .{-FLOAT32_MIN, max(2.0, GetFontSize() * 0.15)}, "");
-    if IsItemHovered()  SetTooltip("OCR scan with %: %, % elapsed, % left, % failed%", ocr_ui.scan.model, ocr_scan_progress_text(done, total), ocr_duration_text(elapsed), ocr_duration_text(left), failed, ifx ocr_ui.scan.paused then " (paused)" else "");
+    scan := *ocr_ui.scan;
+    doing := ifx scan.erase_ocr || scan.erase_labels then tprint("Erasing %", erase_what_text(scan)) else tprint("OCR scan with %", scan.model);
+    if IsItemHovered()  SetTooltip("%: %, % elapsed, % left, % failed%", doing, ocr_scan_progress_text(done, total), ocr_duration_text(elapsed), ocr_duration_text(left), failed, ifx scan.paused then " (paused)" else "");
   }
 
   BeginChild("grid");