[0.1.14] The library is watched: files added or deleted outside the app come and go on their own

File_Watcher brought up to Ex4's (the pending index reset keeping its memory). Each changed .mcap is
looked at once settled: added when the grid lacks it, removed when its file is gone, read again when
it failed to load. A folder added, renamed or deleted, or lost events, compare the whole library as
Refresh does, quietly. Removals wait while the OCR thread is busy. viewer_add_capture returns the
index a capture already has, so an import or screen capture the watcher saw first is still scanned.
Off Windows, the deleted folder is no longer listed as part of the library.

README.md

@@ -58,7 +58,9 @@ see [Screen capture](#screen-capture).
   move the file to the Recycle Bin or to delete it permanently, overwriting it with random bytes
   first; "Don't ask again" keeps the choice until the app restarts. Only the `.mcap` goes: the
   `.snagx` it was converted from is untouched.
-- **Refresh** looks at the folder again, after files were added or deleted outside the app.
+- **Files added or deleted outside the app** (in Explorer, by another program) appear in the grid
+  or leave it on their own, folders added, renamed or deleted too: the library is watched.
+  **Refresh** looks at the whole folder again, should the watcher have missed something.
 - **OCR**, **Stats** and **Labels** open their windows (below).
 - A **thin bar** under the search box shows a running scan; hover it for the details.
 

check_targets.jai

@@ -29,7 +29,7 @@ APP_NAME        :: "mcap";
 APP_FOLDER_NAME :: "mcap";
 GIT_BRANCH    :: "check";
 GIT_HASH      :: "-";
-VERSION       :: "0.1.13";
+VERSION       :: "0.1.14";
 BUILD_DATE    :: "-";
 DEBUG         :: true;
 COMPILE_TESTS :: true;

first.jai

@@ -8,7 +8,7 @@ APP_NAME              :: "mcap";
 APP_FOLDER_NAME       :: "mcap";
 APP_BUNDLE_IDENTIFIER :: "com.example.mcap";
 
-VERSION :: "0.1.13";
+VERSION :: "0.1.14";
 // Where everything goes: bin/, or bin-headless/ for builds made with no_crash_dialog, so automated
 // builds never write over the ones you run by hand.
 build_directory := "bin";

modules/File_Watcher/module.jai

@@ -372,8 +372,9 @@ emit_changes :: (using watcher: *$T/File_Watcher) -> changed: bool, needs_wait:
 		}
     }
 
-    // remove moved the last change into each emitted one's slot, so the indices are rebuilt.
-    table_reset(*pending_index);
+    // remove moved the last change into each emitted one's slot, so the indices are rebuilt, in the
+    // table's own slots: this runs on every poll while changes wait.
+    table_reset_keeping_memory(*pending_index);
     for changes  table_add(*pending_index, it.full_path, it_index);
 
     if changes.count == 0 {

src/app/viewer.jai

@@ -13,7 +13,8 @@
 //
 // A capture keeps its index for as long as the app runs: the workers, the OCR scan and Stats refer
 // to captures by index. Refresh (viewer_refresh) marks the files gone from the folder as removed and
-// appends the new ones.
+// appends the new ones. The library is watched (viewer_watch_frame), so files added or deleted
+// outside the app come and go on their own; Refresh is still there for a watcher that missed some.
 
 VIEWER_FOLDER           :: "D:/McapIndex/mcap";   // hardcoded for now; -library FOLDER shows another
 
@@ -55,6 +56,7 @@ Viewer_Capture :: struct {
   in_flight: bool;              // under viewer.mutex: queued for a worker, or being decoded
   catalog: *Catalog_Entry;      // main thread only; null until loaded
   removed: bool;                // main thread only: its file was gone at a refresh; shown nowhere
+  catalog_failed: bool;         // main thread only: its catalog entry could not be loaded
 }
 
 // A catalog entry made off the main thread, for the main thread to put in place.
@@ -165,6 +167,8 @@ viewer_startup :: () {
 
   viewer.folders.allocator      = context.default_allocator;
 
+  viewer_watch_start();   // before the listing, so nothing written meanwhile is missed
+
   start := current_time_monotonic();
   for viewer_list_files() {
     capture := array_add(*viewer.captures);
@@ -198,6 +202,7 @@ viewer_shutdown :: () {
   for 0..viewer.workers_started - 1  thread_is_done(*viewer.workers[it], -1);
   if viewer.open_started  thread_is_done(*viewer.open_thread, -1);
   background_shutdown();
+  viewer_watch_stop();
 }
 
 // ---------------------------------------------------------------- the workers
@@ -451,6 +456,7 @@ viewer_apply_catalog_updates :: () {
   for updates {
     capture := *viewer.captures[it.index];
     if !capture.catalog  viewer.catalog_loaded += 1;
+    if !capture.catalog  capture.catalog_failed = !it.entry;
     if !capture.catalog || !it.entry || it.entry.date != capture.catalog.date  order_changes = true;
     if it.entry || !capture.catalog {
       background_retire(capture.catalog);   // a search or Stats job may be reading it
@@ -473,7 +479,8 @@ viewer_apply_catalog_updates :: () {
 // then), and only those matching the search.
 // Looks at the folder again: captures whose file is gone are marked removed, and new files are
 // added, their previews and catalog entries loaded as the others' were. Indices stay as they are.
-viewer_refresh :: () {
+// quiet (the watcher's): no toast, and no log line when nothing changed.
+viewer_refresh :: (quiet := false) {
   start := current_time_monotonic();
   present: Table(string, bool);   // the folder's .mcap files now
   present.allocator = temp;
@@ -507,20 +514,22 @@ viewer_refresh :: () {
   viewer.catalog_generation += 1;
   viewer.visible_dirty = true;
   viewer.visible_rebuilt_at = .{};
+  if quiet && !removed && !added  return;
   log("[viewer] Refreshed %: % removed, % added, % captures, in % ms.", viewer_folder(), removed, added,
       viewer.captures.count - viewer.removed_count, FF(to_float64_seconds(current_time_monotonic() - start) * 1000.0, 1));
-  toast_show(.Success, "% removed, % added", removed, added);
+  if !quiet  toast_show(.Success, "% removed, % added", removed, added);
 }
 
 // One capture written into the library since it was looked at (an import): added as a refresh would
-// add it. Its index, or -1 when it is already there.
+// add it. Its index; the one it has when it is already there (the watcher may have seen it first).
 viewer_add_capture :: (path: string) -> s32 {
-  for viewer.captures  if !it.removed && it.path == path  return -1;
+  index := viewer_capture_find(path);
+  if index >= 0  return index;
   lock(*viewer.mutex);
   capture := array_add(*viewer.captures);
   capture.* = .{};
   viewer_capture_init(capture, path);
-  index := cast(s32) (viewer.captures.count - 1);
+  index = cast(s32) (viewer.captures.count - 1);
   unlock(*viewer.mutex);
   signal(*viewer.wake);   // its catalog entry
   if viewer.scan_error { free(viewer.scan_error); viewer.scan_error = ""; }
@@ -534,16 +543,134 @@ viewer_add_capture :: (path: string) -> s32 {
 viewer_list_files :: () -> [..] string {
   files: [..] string;
   files.allocator = temp;
-  extension := tprint(".%", MCAP_EXTENSION);
   for file_list(viewer_folder(), recursive = true,, temp) {
-    if !ends_with_nocase(it, extension)  continue;
-    relative := advance(it, viewer_folder().count + 1);
-    if begins_with(relative, ".") || contains(relative, "/.")  continue;
-    array_add(*files, it);
+    if viewer_is_library_file(it)  array_add(*files, it);
   }
   return files;
 }
 
+// Whether path, a file or folder, is where the library keeps captures: under it, and not hidden or
+// in the deleted folder viewer_delete_capture moves files to off Windows.
+viewer_is_library_path :: (path: string) -> bool {
+  folder := viewer_folder();
+  if path.count <= folder.count + 1 || !begins_with(path, folder) || path[folder.count] != #char "/"  return false;
+  relative := advance(path, folder.count + 1);
+  if begins_with(relative, ".") || contains(relative, "/.")  return false;
+  #if OS != .WINDOWS  if relative == "deleted" || begins_with(relative, "deleted/")  return false;
+  return true;
+}
+
+viewer_is_library_file :: (path: string) -> bool {
+  return ends_with_nocase(path, tprint(".%", MCAP_EXTENSION)) && viewer_is_library_path(path);
+}
+
+// The capture whose file is path, of those not removed; -1 when none.
+viewer_capture_find :: (path: string) -> s32 {
+  for viewer.captures  if !it.removed && it.path == path  return cast(s32) it_index;
+  return -1;
+}
+
+// ---------------------------------------------------------------- watching the library
+
+// The watcher reports which paths changed, not what to do about them: a capture written in place
+// (the app's own writes go to NAME.mcap.tmp and are renamed over NAME.mcap) shows as a remove and a
+// move, a copy in progress as an add and modifies. So each path it reports is looked at as it is
+// now, once it has settled: a file the grid lacks is added, a capture whose file is gone is
+// removed. A folder added, removed or renamed reports no events for what is in it, and lost events
+// say nothing of what was lost, so those compare the whole library (viewer_refresh) instead.
+//
+// Removals wait while the OCR thread is busy, as Refresh does: a scan or reading refers to captures
+// by index. Main thread only, but for viewer_watch_stop on the shutdown thread once the frames stop.
+
+VIEWER_WATCH_BUFFER_SIZE :: 64 * 1024;   // per watched folder on Windows; the most a network share takes
+
+viewer_watch: struct {
+  watcher: Watcher.File_Watcher(void);
+  started: bool;
+  paths: [..] string;   // .mcap files changed, not looked at yet (heap)
+  rescan: bool;         // compare the whole library
+}
+
+viewer_watch_start :: () {
+  push_allocator(context.default_allocator);
+  viewer_watch.paths.allocator = context.default_allocator;
+  folder := viewer_folder();
+  if !Watcher.init(*viewer_watch.watcher, viewer_watch_callback, merge_window_seconds = 0.25, event_buffer_size = VIEWER_WATCH_BUFFER_SIZE) {
+    log_error("[viewer] Could not watch %; Refresh shows what changed in it.", folder);
+    return;
+  }
+  if !Watcher.add_directories(*viewer_watch.watcher, folder) {
+    log_error("[viewer] Could not watch %; Refresh shows what changed in it.", folder);
+    Watcher.deinit(viewer_watch.watcher);
+    return;
+  }
+  viewer_watch.started = true;
+}
+
+viewer_watch_stop :: () {
+  if !viewer_watch.started  return;
+  Watcher.deinit(viewer_watch.watcher);
+  viewer_watch.started = false;
+  for viewer_watch.paths  free(it,, context.default_allocator);
+  array_free(viewer_watch.paths);
+}
+
+viewer_watch_frame :: () {
+  if !viewer_watch.started  return;
+  Watcher.process_changes(*viewer_watch.watcher);
+  if !viewer_watch.rescan && !viewer_watch.paths.count  return;
+
+  busy := ocr_ui.thread_busy;   // removals wait for it
+  if viewer_watch.rescan {
+    if busy  return;
+    viewer_watch.rescan = false;
+    for viewer_watch.paths  free(it,, context.default_allocator);
+    array_reset_keeping_memory(*viewer_watch.paths);
+    viewer_refresh(quiet = true);
+    return;
+  }
+
+  kept: s64;   // the paths that wait for the OCR thread, moved to the front
+  for path: viewer_watch.paths {
+    index := viewer_capture_find(path);
+    there := file_exists(path) && !is_directory(path);
+    wait := false;
+    if there && index < 0 {
+      viewer_add_capture(path);
+      log("[viewer] % was added to the library.", path);
+    } else if !there && index >= 0 {
+      if busy  wait = true;
+      else { viewer_mark_removed(index); log("[viewer] % was removed from the library.", path); }
+    } else if there && (viewer.captures[index].failed || viewer.captures[index].catalog_failed) {
+      // Read while it was still being written (a copy in progress): read it again, as a new capture.
+      if busy  wait = true;
+      else { viewer_mark_removed(index); viewer_add_capture(path); }
+    }
+    if wait { viewer_watch.paths[kept] = path; kept += 1; }
+    else    free(path,, context.default_allocator);
+  }
+  viewer_watch.paths.count = kept;
+}
+
+viewer_watch_callback :: (watcher: *Watcher.File_Watcher(void), change: *Watcher.File_Change, user_data: *void) {
+  Event :: Watcher.File_Change_Event;
+  if change.events & (Event.OVERFLOW | .SCAN_CHILDREN) { viewer_watch.rescan = true; return; }
+  path := change.full_path;
+  if !viewer_is_library_path(path)  return;
+  if viewer_is_library_file(path) && !(change.events & .IS_FOLDER) {
+    for viewer_watch.paths  if it == path  return;
+    array_add(*viewer_watch.paths, copy_string(path,, context.default_allocator));
+    return;
+  }
+  // A folder's entries changing makes it MODIFIED, which says nothing about the folder itself.
+  if !(change.events & (Event.ADDED | .REMOVED | .MOVED))  return;
+  if ends_with_nocase(path, ".tmp")  return;   // the app's own writes, on their way to a .mcap
+  // A folder now, or one that held captures: what is in it came or went with it.
+  if is_directory(path) { viewer_watch.rescan = true; return; }
+  prefix := tprint("%/", path);
+  for viewer.captures  if !it.removed && begins_with(it.path, prefix) { viewer_watch.rescan = true; return; }
+}
+
 viewer_capture_init :: (capture: *Viewer_Capture, path: string) {
   push_allocator(context.default_allocator);
   capture.path = copy_string(path);
@@ -844,6 +971,7 @@ viewer_request_full :: (index: s32) {
 // ---------------------------------------------------------------- drawing
 
 viewer_ui :: () { using ImGui;
+  viewer_watch_frame();   // files added or deleted outside the app
   viewer_frame_gpu_work();
 
   if viewer.scan_error {
@@ -1212,3 +1340,7 @@ viewer_full_window :: () { using ImGui;
   }
   End();
 }
+
+#scope_file
+
+Watcher :: #import "File_Watcher";