Musa Mahmood
3e6e733a9d
- .library-folders in the library (src/app/library_folders.jai): each library folder's options, starting with segment: true. Every folder starts off; unknown keys are kept when it is rewritten. - The Segmentation window (Segments, above the grid): each folder's Segment checkbox, how many of its images are segmented, and Erase per folder; a button to segment the folders set to segment. - The segmentation scan (src/app/segment_ui.jai) runs on the OCR thread as its own task, with the OCR scan's progress, pause and cancel: images only (never videos), not yet segmented, in folders set to segment. The engine loads with the first scan (2.4 GB of GPU memory). Each image's segs and mask are stored in one rewrite and its catalog entry loaded again. - Importing into a folder set to segment segments the new images; the Import window's checkbox is the folder's setting. - -segment segments and exits; -erase segments, and -erase all now includes segments. - The catalog reads each capture's segs (class list hash, segment count). The exit overlay names the segmentation scan and model. segment_mcap_chunks moved into modules/Segment, shared with segment-probe. The quoting helpers of Capture_File's text formats are public. - Tests: .library-folders read and written back (quoted names, unknown keys, versions).
1128 lines
51 KiB
Plaintext
1128 lines
51 KiB
Plaintext
// Importing a folder of screenshots, photos and videos into the library: every image under it (PNG,
|
|
// JPEG, HEIC, AVIF, WebP, GIF, BMP, TIFF, camera RAW), every video (MP4, MOV, MKV, WebM, AVI, WMV, MPEG;
|
|
// on Windows, with FFmpeg) and every Snagit .snagx becomes an .mcap in a folder of its own in the
|
|
// library, one folder for each place imported from (D:\McapIndex\mcap\SnagitLibraryWork, ...). The
|
|
// files imported from are only read, never changed.
|
|
//
|
|
// An image's .mcap holds the file's bytes exactly as they were (its original), a preview turned the
|
|
// right way up, and Capture Text with metadata.json (CaptureDate, WindowName as the file's name,
|
|
// AppName as the camera or phone) and import.json (where it came from, its EXIF). A video's is the
|
|
// same, with a frame from it, turned the right way up, as its rendered image (what OCR and labels
|
|
// read), and what its container says in import.json. A video is copied into its .mcap a few MB at a
|
|
// time, never read whole, so it can be any size. A .snagx becomes the same .mcap SnagitConverter
|
|
// makes of it (modules/Snagx).
|
|
//
|
|
// Each image is stored once in the whole library: the library index (IMPORT_INDEX_NAME, in the library
|
|
// folder) holds a hash of every .mcap's original image, made once for the files already there and kept
|
|
// up to date after that; an image whose hash is in it is reported as a duplicate, with where it is.
|
|
//
|
|
// The import runs on a thread of its own, which lists the files, brings the index up to date, and then
|
|
// imports with IMPORT_WORKERS threads. New captures go into the grid as they are written; when it ends,
|
|
// the OCR scan reads and labels them, if asked to.
|
|
|
|
IMPORT_WINDOW :: "Import";
|
|
IMPORT_INDEX_NAME :: ".import-index"; // in the library folder: path, size, time and hash of each .mcap
|
|
IMPORT_SOURCES_NAME :: ".import-sources"; // in the library folder: the folder each library folder was last imported from
|
|
IMPORT_MAX_FILE :: 1024 * 1024 * 1024; // larger images are not read (videos are streamed, any size)
|
|
IMPORT_NOTES_MAX :: 5000; // failures and duplicates listed; the rest are only counted
|
|
IMPORT_SAVE_EVERY :: 250; // captures imported between saves of the index
|
|
IMPORT_HEAD_SIZE :: 256; // the first bytes looked at, to tell what a file is
|
|
|
|
// Videos, by extension, for those whose first bytes do not say so (MPEG transport streams).
|
|
IMPORT_VIDEO_EXTENSIONS :: string.["mp4", "mov", "m4v", "avi", "mkv", "webm", "3gp", "3g2", "wmv", "mts", "m2ts", "ts", "mpg", "mpeg", "flv"];
|
|
// Not images or videos, by extension: skipped without being read. Anything else is read and looked
|
|
// at. thm and lrv are a GoPro's thumbnails and low-resolution copies of its videos.
|
|
IMPORT_OTHER_EXTENSIONS :: string.["aae", "xmp", "json", "txt", "ini", "db", "pdf", "zip", "lnk", "url", "html", "htm", "doc", "docx", "xls", "xlsx", "ppt", "pptx", "mp3", "m4a", "wav", "aac", "exe", "dll", "mcap", "tmp", "thm", "lrv", "mpo"];
|
|
|
|
Import_Phase :: enum u8 { IDLE; LISTING; INDEXING; IMPORTING; DONE; }
|
|
|
|
Import_Outcome :: enum u8 { IMPORTED; DUPLICATE; SKIPPED; FAILED; }
|
|
|
|
// A file that failed, or was a duplicate, and why or of what. Heap.
|
|
Import_Note :: struct {
|
|
path: string;
|
|
text: string;
|
|
}
|
|
|
|
Index_Record :: struct {
|
|
size: s64;
|
|
modified: s64; // nanoseconds
|
|
hash: u64;
|
|
}
|
|
|
|
importer: struct {
|
|
window_open: bool;
|
|
source_buffer: [1024] u8;
|
|
dest_buffer: [256] u8;
|
|
dest_edited: bool; // typed in, so a new source does not replace it
|
|
read_after := true; // read and label what was imported, with the OCR scan
|
|
sources: Table(string, string); // library folder name -> the folder last imported into it; main thread
|
|
|
|
// Set by the main thread before the thread starts; read-only while it runs.
|
|
source: string;
|
|
dest_name: string;
|
|
dest_folder: string;
|
|
library: string;
|
|
|
|
thread: Thread;
|
|
thread_started: bool;
|
|
running: bool; // set by the main thread, cleared by the import thread when it ends
|
|
phase: Import_Phase;
|
|
paused, cancel: bool;
|
|
start: Apollo_Time;
|
|
paused_seconds: float64;
|
|
pause_start: Apollo_Time;
|
|
elapsed_seconds: float64; // once it has ended
|
|
|
|
files: [..] string; // what LISTING found; the import thread's
|
|
listed: s64;
|
|
index_total, index_done: s64;
|
|
next: s64; // the next file a worker takes
|
|
done, imported, duplicates, skipped, failed: s64;
|
|
error: string; // why the import could not run at all
|
|
|
|
// Under mutex.
|
|
mutex: Mutex;
|
|
failures: [..] Import_Note;
|
|
duplicate_notes: [..] Import_Note;
|
|
skipped_kinds: Table(string, s64); // "mp4", "video", "unknown" ... -> count
|
|
new_paths: [..] string; // written, for the main thread to put in the grid
|
|
by_hash: Table(u64, string); // every original in the library -> its path, relative to the library
|
|
records: Table(string, Index_Record);
|
|
reserved_names: Table(string, bool); // output paths being written
|
|
since_save: s64;
|
|
|
|
// Main thread.
|
|
new_indices: [..] s32; // the captures this import added, for the scan after it
|
|
scan_pending: bool;
|
|
segment_pending: bool; // the folder imported into is set to segment: its new images, after the scan
|
|
flag_started: bool; // -import has started its import
|
|
}
|
|
|
|
// ---------------------------------------------------------------- startup and shutdown
|
|
|
|
import_startup :: () {
|
|
init(*importer.mutex, "import");
|
|
importer.files.allocator = context.default_allocator;
|
|
importer.failures.allocator = context.default_allocator;
|
|
importer.duplicate_notes.allocator = context.default_allocator;
|
|
importer.new_paths.allocator = context.default_allocator;
|
|
importer.new_indices.allocator = context.default_allocator;
|
|
import_sources_load();
|
|
}
|
|
|
|
import_shutdown :: () {
|
|
atomic_swap(*importer.cancel, true);
|
|
atomic_swap(*importer.paused, false);
|
|
if importer.thread_started thread_is_done(*importer.thread, -1);
|
|
}
|
|
|
|
// ---------------------------------------------------------------- starting
|
|
|
|
// Starts importing source into the library folder dest_name; false, with a toast, when it cannot.
|
|
import_start :: (source: string, dest_name: string) -> bool {
|
|
if importer.running return false;
|
|
source_path := trim(source);
|
|
while source_path.count > 3 && (source_path[source_path.count - 1] == #char "/" || source_path[source_path.count - 1] == #char "\\") source_path.count -= 1;
|
|
dest := trim(dest_name);
|
|
if !source_path.count || !dest.count { toast_show(.Error, "Choose a folder to import and a library folder to import it into"); return false; }
|
|
if !import_name_is_valid(dest) { toast_show(.Error, "\"%\" cannot be a folder's name", dest); return false; }
|
|
if !file_exists(source_path) { toast_show(.Error, "% does not exist", source_path); return false; }
|
|
|
|
push_allocator(context.default_allocator);
|
|
free(importer.source); free(importer.dest_name); free(importer.dest_folder); free(importer.library); free(importer.error);
|
|
importer.source = copy_string(source_path);
|
|
path_overwrite_separators(importer.source, #char "/");
|
|
importer.dest_name = copy_string(dest);
|
|
importer.library = copy_string(viewer_folder());
|
|
importer.dest_folder = sprint("%/%", importer.library, dest);
|
|
importer.error = "";
|
|
|
|
// The library is never imported into itself.
|
|
lower_source := to_lower_copy(importer.source,, temp);
|
|
lower_library := to_lower_copy(importer.library,, temp);
|
|
path_overwrite_separators(lower_library, #char "/");
|
|
if lower_source == lower_library || begins_with(lower_source, tprint("%/", lower_library)) || begins_with(lower_library, tprint("%/", lower_source)) {
|
|
toast_show(.Error, "The folder to import holds the library, or is inside it");
|
|
return false;
|
|
}
|
|
|
|
for importer.files free(it);
|
|
array_reset(*importer.files);
|
|
lock(*importer.mutex);
|
|
for importer.failures { free(it.path); free(it.text); }
|
|
for importer.duplicate_notes { free(it.path); free(it.text); }
|
|
array_reset(*importer.failures);
|
|
array_reset(*importer.duplicate_notes);
|
|
for _, key: importer.skipped_kinds free(key);
|
|
table_reset(*importer.skipped_kinds);
|
|
unlock(*importer.mutex);
|
|
array_reset(*importer.new_indices);
|
|
importer.listed = 0; importer.index_total = 0; importer.index_done = 0;
|
|
importer.next = 0; importer.done = 0; importer.imported = 0; importer.duplicates = 0; importer.skipped = 0; importer.failed = 0;
|
|
importer.paused = false; importer.cancel = false;
|
|
importer.paused_seconds = 0; importer.elapsed_seconds = 0;
|
|
importer.phase = .LISTING;
|
|
importer.start = current_time_monotonic();
|
|
importer.scan_pending = false;
|
|
|
|
table_set(*importer.sources, copy_string(dest), copy_string(importer.source));
|
|
import_sources_save();
|
|
|
|
importer.running = true;
|
|
if importer.thread_started thread_deinit(*importer.thread);
|
|
importer.thread_started = thread_init(*importer.thread, import_thread_proc, temporary_storage_size = 4 * 1024 * 1024, name = "import");
|
|
if !importer.thread_started { importer.running = false; importer.phase = .IDLE; toast_show(.Error, "Could not start the import"); return false; }
|
|
thread_start(*importer.thread);
|
|
log("[Import] Importing % into %.", importer.source, importer.dest_folder);
|
|
return true;
|
|
}
|
|
|
|
import_name_is_valid :: (name: string) -> bool {
|
|
if !name.count || name == "." || name == ".." || name[0] == #char "." return false;
|
|
for name if it < 32 || it == #char "/" || it == #char "\\" || it == #char ":" || it == #char "*" || it == #char "?" || it == #char "\"" || it == #char "<" || it == #char ">" || it == #char "|" return false;
|
|
return name[name.count - 1] != #char " " && name[name.count - 1] != #char ".";
|
|
}
|
|
|
|
// ---------------------------------------------------------------- the import thread
|
|
|
|
import_thread_proc :: (thread: *Thread) -> s64 {
|
|
context.allocator = context.default_allocator;
|
|
defer {
|
|
importer.elapsed_seconds = import_elapsed();
|
|
atomic_swap(*importer.phase, Import_Phase.DONE);
|
|
atomic_swap(*importer.running, false);
|
|
}
|
|
|
|
// The files to import, sorted so a second run goes through them in the same order.
|
|
Found :: struct { files: *[..] string; }
|
|
found := Found.{*importer.files};
|
|
visited := visit_files(importer.source, true, *found, (info: *File_Visit_Info, found: *Found) {
|
|
if info.is_directory {
|
|
name := info.short_name;
|
|
if name.count && (name[0] == #char "." || name[0] == #char "$" || name == "System Volume Information") info.descend_into_directory = false;
|
|
return;
|
|
}
|
|
array_add(found.files, copy_string(info.full_name));
|
|
atomic_add(*importer.listed, 1);
|
|
}, visit_files = true, visit_directories = true, follow_directory_symlinks = false);
|
|
if !visited || importer.cancel {
|
|
if !visited importer.error = sprint("% cannot be read", importer.source);
|
|
return 0;
|
|
}
|
|
quick_sort(importer.files, compare);
|
|
|
|
if !make_directory_if_it_does_not_exist(importer.dest_folder, recursive = true) {
|
|
importer.error = sprint("% cannot be made", importer.dest_folder);
|
|
return 0;
|
|
}
|
|
|
|
atomic_swap(*importer.phase, Import_Phase.INDEXING);
|
|
if !import_index_update() || importer.cancel return 0;
|
|
|
|
atomic_swap(*importer.phase, Import_Phase.IMPORTING);
|
|
import_run_workers(import_worker);
|
|
import_index_save();
|
|
log("[Import] % into %: % imported, % duplicates, % skipped, % failed, of % files, in %%.", importer.source, importer.dest_name,
|
|
importer.imported, importer.duplicates, importer.skipped, importer.failed, importer.files.count, ocr_duration_text(import_elapsed()),
|
|
ifx importer.cancel then " (cancelled)" else "");
|
|
return 0;
|
|
}
|
|
|
|
IMPORT_WORKERS :: 0; // 0: worked out from the processors
|
|
|
|
// Runs proc on several threads until it returns false on each, then waits for them.
|
|
import_run_workers :: (proc: (thread: *Thread) -> s64) {
|
|
count := clamp(get_number_of_processors(.ALL_LOGICAL) / 2, 2, 8);
|
|
if IMPORT_WORKERS count = IMPORT_WORKERS;
|
|
threads := NewArray(count, Thread,, context.default_allocator);
|
|
defer array_free(threads,, context.default_allocator);
|
|
started := 0;
|
|
for * threads {
|
|
// Reading an image and its .mcap takes about three times its size; temporary storage grows past this as needed.
|
|
if !thread_init(it, proc, temporary_storage_size = 16 * 1024 * 1024, name = tprint("import %", it_index)) break;
|
|
thread_start(it);
|
|
started += 1;
|
|
}
|
|
if !started proc(null); // on this thread, then
|
|
for 0..started - 1 {
|
|
thread_is_done(*threads[it], -1);
|
|
thread_deinit(*threads[it]);
|
|
}
|
|
}
|
|
|
|
import_worker :: (thread: *Thread) -> s64 {
|
|
context.allocator = context.default_allocator;
|
|
while true {
|
|
while importer.paused && !importer.cancel sleep_milliseconds(100);
|
|
if importer.cancel break;
|
|
n := atomic_add(*importer.next, 1);
|
|
if n >= importer.files.count break;
|
|
path := importer.files[n];
|
|
|
|
outcome, text, new_path := import_file(path);
|
|
lock(*importer.mutex);
|
|
if outcome == {
|
|
case .IMPORTED;
|
|
importer.imported += 1;
|
|
array_add(*importer.new_paths, copy_string(new_path));
|
|
importer.since_save += 1;
|
|
case .DUPLICATE;
|
|
importer.duplicates += 1;
|
|
if importer.duplicate_notes.count < IMPORT_NOTES_MAX array_add(*importer.duplicate_notes, .{copy_string(path), copy_string(text)});
|
|
case .SKIPPED;
|
|
importer.skipped += 1;
|
|
found, count := table_find(*importer.skipped_kinds, text);
|
|
if found table_set(*importer.skipped_kinds, text, count + 1);
|
|
else table_set(*importer.skipped_kinds, copy_string(text), 1);
|
|
case .FAILED;
|
|
importer.failed += 1;
|
|
if importer.failures.count < IMPORT_NOTES_MAX array_add(*importer.failures, .{copy_string(path), copy_string(text)});
|
|
}
|
|
save := importer.since_save >= IMPORT_SAVE_EVERY;
|
|
if save importer.since_save = 0;
|
|
unlock(*importer.mutex);
|
|
if outcome == .FAILED log_error("[Import] % %", path, text);
|
|
atomic_add(*importer.done, 1);
|
|
if save import_index_save();
|
|
reset_temporary_storage();
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
// Imports one file. The text says why it failed or was skipped (what kind of file: "aae", "empty"),
|
|
// or, for a duplicate, where it already is; new_path is where an import went. Temp.
|
|
import_file :: (path: string) -> Import_Outcome, text: string, new_path: string {
|
|
name := path_filename(path);
|
|
extension := to_lower_copy(path_extension(name),, temp);
|
|
if array_find(IMPORT_OTHER_EXTENSIONS, extension) return .SKIPPED, extension, "";
|
|
|
|
modified, size, stat_ok := file_modtime_and_size(path);
|
|
if !stat_ok return .FAILED, "cannot be read", "";
|
|
if size == 0 return .SKIPPED, "empty", "";
|
|
|
|
// A look at the first bytes, before reading what may not be an image at all.
|
|
head := import_read_head(path, IMPORT_HEAD_SIZE);
|
|
kind := image_kind_from_name(image_kind(head), name);
|
|
is_snagx := extension == "snagx" && head.count >= 4 && memcmp(head.data, "PK\x03\x04".data, 4) == 0;
|
|
if !is_snagx {
|
|
if kind == .VIDEO || (kind == .UNKNOWN && array_find(IMPORT_VIDEO_EXTENSIONS, extension)) {
|
|
outcome, text, new_path := import_video_file(path, name, head, size, modified);
|
|
return outcome, text, new_path;
|
|
}
|
|
if kind == .UNKNOWN return .SKIPPED, ifx extension then extension else "no extension", "";
|
|
}
|
|
if size > IMPORT_MAX_FILE return .FAILED, tprint("is % MB, larger than the % MB an import reads", size / (1024 * 1024), IMPORT_MAX_FILE / (1024 * 1024)), "";
|
|
|
|
data, read_ok := read_entire_file(path, log_errors = false,, temp);
|
|
if !read_ok return .FAILED, "cannot be read", "";
|
|
bytes := cast([] u8) data;
|
|
|
|
m: Mcap;
|
|
out_bytes: [] u8;
|
|
hash: u64;
|
|
out_name: string;
|
|
if is_snagx {
|
|
ok: bool;
|
|
error: string;
|
|
m, out_bytes, ok, error = snagx_to_mcap(data);
|
|
if !ok return .FAILED, error, "";
|
|
hash = import_hash(m.original);
|
|
out_name = tprint("%.%", path_strip_extension(name), MCAP_EXTENSION);
|
|
} else {
|
|
hash = import_hash(bytes);
|
|
}
|
|
|
|
// Once in the whole library: the hash is held for this file until it is written, or let go.
|
|
claimed, where := import_claim(hash);
|
|
if !claimed return .DUPLICATE, where, "";
|
|
let_go := true;
|
|
defer if let_go import_let_go(hash);
|
|
|
|
if !is_snagx {
|
|
ok: bool;
|
|
error: string;
|
|
m, out_name, ok, error = import_image_mcap(path, name, bytes, kind, hash, modified);
|
|
if !ok return .FAILED, error, "";
|
|
serializer, serialized, serialize_error := mcap_serialize(m,, temp);
|
|
if !serialized return .FAILED, serialize_error, "";
|
|
out_bytes = cast([] u8) string.{serializer.count, serializer.data};
|
|
}
|
|
|
|
out_path := import_reserve_path(out_name);
|
|
written, write_error := mcap_write(out_path, out_bytes);
|
|
new_path := import_written(out_path, hash, written);
|
|
if !written return .FAILED, write_error, "";
|
|
let_go = false;
|
|
return .IMPORTED, "", new_path;
|
|
}
|
|
|
|
// A video: hashed and copied into its .mcap a few MB at a time, so it can be larger than memory.
|
|
import_video_file :: (path: string, name: string, head: [] u8, size: s64, modified: Apollo_Time) -> Import_Outcome, text: string, new_path: string {
|
|
if OS != .WINDOWS return .SKIPPED, "video (Windows only)", "";
|
|
|
|
hash, hashed := import_hash_file(.{path = path, offset = 0, size = size});
|
|
if !hashed return .FAILED, "cannot be read", "";
|
|
claimed, where := import_claim(hash);
|
|
if !claimed return .DUPLICATE, where, "";
|
|
let_go := true;
|
|
defer if let_go import_let_go(hash);
|
|
|
|
m, out_name, ok, error := import_video_mcap(path, name, head, size, hash, modified);
|
|
if !ok return .FAILED, error, "";
|
|
out_path := import_reserve_path(out_name);
|
|
written, write_error := mcap_write_streamed(out_path, m, .{path = path, offset = 0, size = size});
|
|
new_path := import_written(out_path, hash, written);
|
|
if !written return .FAILED, write_error, "";
|
|
let_go = false;
|
|
return .IMPORTED, "", new_path;
|
|
}
|
|
|
|
// Holds hash for a file being imported, so no other worker imports the same bytes; false, with where
|
|
// they already are, when the library has them, or another file of this import does.
|
|
import_claim :: (hash: u64) -> claimed: bool, where: string {
|
|
lock(*importer.mutex);
|
|
defer unlock(*importer.mutex);
|
|
found, existing := table_find(*importer.by_hash, hash);
|
|
if found return false, ifx existing then copy_string(existing,, temp) else "(another file in this import)";
|
|
table_set(*importer.by_hash, hash, ""); // "" while it is being written
|
|
return true, "";
|
|
}
|
|
|
|
// Lets go of a hash claimed for a file that was not imported after all.
|
|
import_let_go :: (hash: u64) {
|
|
lock(*importer.mutex);
|
|
table_remove(*importer.by_hash, hash);
|
|
unlock(*importer.mutex);
|
|
}
|
|
|
|
// Where out_name goes in the library folder, held until import_written: another file there by that
|
|
// name (a different capture) gets _2, _3 ... Heap.
|
|
import_reserve_path :: (out_name: string) -> string {
|
|
lock(*importer.mutex);
|
|
defer unlock(*importer.mutex);
|
|
base := path_strip_extension(out_name);
|
|
out_path := tprint("%/%", importer.dest_folder, out_name);
|
|
number := 2;
|
|
while file_exists(out_path) || table_contains(*importer.reserved_names, out_path) {
|
|
out_path = tprint("%/%_%.%", importer.dest_folder, base, number, MCAP_EXTENSION);
|
|
number += 1;
|
|
}
|
|
out_path = copy_string(out_path);
|
|
table_set(*importer.reserved_names, out_path, true);
|
|
return out_path;
|
|
}
|
|
|
|
// After out_path (from import_reserve_path, which this frees) was written, or failed to be: the
|
|
// index learns of it. Where it is (temp), for the grid.
|
|
import_written :: (out_path: string, hash: u64, written: bool) -> string {
|
|
out_modified, out_size, _ := file_modtime_and_size(out_path);
|
|
relative := import_relative(out_path);
|
|
lock(*importer.mutex);
|
|
table_remove(*importer.reserved_names, out_path);
|
|
if written {
|
|
table_set(*importer.by_hash, hash, copy_string(relative));
|
|
import_record_set(relative, .{size = out_size, modified = to_nanoseconds(out_modified), hash = hash});
|
|
}
|
|
unlock(*importer.mutex);
|
|
new_path := tprint("%/%", importer.library, relative);
|
|
free(out_path);
|
|
return new_path;
|
|
}
|
|
|
|
// An image file as an .mcap (temp) and the name it gets: the date it was taken and its hash.
|
|
import_image_mcap :: (path: string, name: string, bytes: [] u8, kind: Image_Kind, hash: u64, modified: Apollo_Time) -> Mcap, name: string, ok: bool, error: string {
|
|
m: Mcap;
|
|
image, decoded, decode_error := image_decode(bytes, kind);
|
|
if !decoded return m, "", false, decode_error;
|
|
defer image_free(*image);
|
|
png, preview_w, preview_h, preview_ok, preview_error := image_preview_png(image, MCAP_PREVIEW_MAX_SIDE);
|
|
if !preview_ok return m, "", false, preview_error;
|
|
exif := exif_read(bytes, kind);
|
|
|
|
// When it was taken: the camera's clock, else a date in the file's name, else when the file was last changed.
|
|
date, date_from := import_exif_date(exif.date_original), "EXIF";
|
|
if !date { date = import_name_date(name); date_from = "file name"; }
|
|
if !date { date = import_calendar_text(to_calendar(modified, .LOCAL)); date_from = "modified time"; }
|
|
|
|
make := import_text(exif.make);
|
|
doc, info := import_document(path, name, import_device(make, exif.model), date, bytes.count, modified, hash, image_kind_name(kind), image.width, image.height, date_from);
|
|
if kind == .GIF {
|
|
frames, duration_ms := gif_info(bytes);
|
|
if frames > 1 { // an animation
|
|
import_add_number(doc, info, "Frames", xx frames);
|
|
import_add_number(doc, info, "Duration", duration_ms / 1000.0);
|
|
}
|
|
}
|
|
if exif.found {
|
|
import_add_number(doc, info, "Orientation", xx exif.orientation);
|
|
if make import_add(doc, info, "CameraMake", make);
|
|
if exif.model import_add(doc, info, "CameraModel", import_text(exif.model));
|
|
if exif.software import_add(doc, info, "Software", import_text(exif.software));
|
|
if exif.offset_original import_add(doc, info, "TimeZoneOffset", import_text(exif.offset_original));
|
|
if exif.has_gps {
|
|
import_add_number(doc, info, "Latitude", exif.latitude);
|
|
import_add_number(doc, info, "Longitude", exif.longitude);
|
|
if exif.altitude != 0 import_add_number(doc, info, "Altitude", exif.altitude);
|
|
}
|
|
}
|
|
text, text_ok := import_document_text(doc, path);
|
|
if !text_ok return m, "", false, "its Capture Text does not read back cleanly (see above)";
|
|
|
|
m.text = text;
|
|
m.preview = .{width = cast(u16) preview_w, height = cast(u16) preview_h, png = png};
|
|
m.original_name = name;
|
|
m.original = bytes;
|
|
m.rendered_kind = .NONE;
|
|
return m, import_out_name(date, hash), true, "";
|
|
}
|
|
|
|
// A video as an .mcap (temp), but for its original, which mcap_write_streamed copies from the file,
|
|
// and the name it gets. Its rendered image is a frame from it, which OCR and labels read.
|
|
import_video_mcap :: (path: string, name: string, head: [] u8, size: s64, hash: u64, modified: Apollo_Time) -> Mcap, name: string, ok: bool, error: string {
|
|
m: Mcap;
|
|
image, video, decoded, decode_error := video_poster(path);
|
|
if !decoded return m, "", false, decode_error;
|
|
defer image_free(*image);
|
|
png, preview_w, preview_h, preview_ok, preview_error := image_preview_png(image, MCAP_PREVIEW_MAX_SIDE);
|
|
if !preview_ok return m, "", false, preview_error;
|
|
poster, _, _, poster_ok, poster_error := image_preview_png(image, max(image.width, image.height)); // at its own size
|
|
if !poster_ok return m, "", false, poster_error;
|
|
|
|
// When it was taken: what the phone wrote (local time where it was), else a date in the file's name
|
|
// (local time too, as recorders write it), else when the container was made (UTC, so turned into
|
|
// this computer's time zone, which may not be where it was), else when the file was last changed.
|
|
date, offset := import_video_local_date(video.creation_date);
|
|
date_from := "metadata";
|
|
if !date { date = import_name_date(name); date_from = "file name"; }
|
|
if !date { date = import_video_utc_date(video.creation_time); date_from = "metadata"; }
|
|
if !date { date = import_calendar_text(to_calendar(modified, .LOCAL)); date_from = "modified time"; }
|
|
|
|
make := import_text(video.make);
|
|
doc, info := import_document(path, name, import_device(make, video.model), date, size, modified, hash, video_format_name(head, name), image.width, image.height, date_from);
|
|
import_add_number(doc, info, "Duration", import_milliseconds(video.duration));
|
|
if video.frame_rate > 0 import_add_number(doc, info, "FrameRate", video.frame_rate);
|
|
import_add(doc, info, "VideoCodec", import_text(video.codec));
|
|
import_add_number(doc, info, "Rotation", xx video.rotation);
|
|
import_add_number(doc, info, "PosterTime", import_milliseconds(video.poster_time));
|
|
capture_object_add(doc, *info.root, "Audio", capture_boolean(video.has_audio));
|
|
if make import_add(doc, info, "CameraMake", make);
|
|
if video.model import_add(doc, info, "CameraModel", import_text(video.model));
|
|
if video.software && !begins_with(video.software, "Lavf") import_add(doc, info, "Software", import_text(video.software)); // not FFmpeg's own tag
|
|
if offset import_add(doc, info, "TimeZoneOffset", offset);
|
|
latitude, longitude, altitude, has_altitude, located := iso6709_parse(video.location);
|
|
if located {
|
|
import_add_number(doc, info, "Latitude", latitude);
|
|
import_add_number(doc, info, "Longitude", longitude);
|
|
if has_altitude && altitude != 0 import_add_number(doc, info, "Altitude", altitude);
|
|
}
|
|
text, text_ok := import_document_text(doc, path);
|
|
if !text_ok return m, "", false, "its Capture Text does not read back cleanly (see above)";
|
|
|
|
m.text = text;
|
|
m.preview = .{width = cast(u16) preview_w, height = cast(u16) preview_h, png = png};
|
|
m.original_name = name;
|
|
m.rendered_kind = .OWN;
|
|
m.rendered = poster;
|
|
return m, import_out_name(date, hash), true, "";
|
|
}
|
|
|
|
// An imported file's Capture Text so far (temp): metadata.json, and import.json with the keys every
|
|
// import has; the caller adds its own to info (import_add), then import_document_text.
|
|
import_document :: (path: string, name: string, device: string, date: string, size: s64, modified: Apollo_Time, hash: u64, format: string, width: s64, height: s64, date_from: string) -> *Capture_Document, info: *Capture_File {
|
|
doc := New(Capture_Document,, temp);
|
|
capture_document_init(doc, temp);
|
|
doc.comment = tprint("Imported by % % from %.", APP_NAME, VERSION, import_text(path_filename(path)));
|
|
metadata := capture_add_file(doc, "metadata.json");
|
|
import_add(doc, metadata, "AppName", device);
|
|
import_add(doc, metadata, "CaptureDate", date);
|
|
import_add(doc, metadata, "WindowName", import_text(name));
|
|
|
|
info := capture_add_file(doc, "import.json");
|
|
import_add(doc, info, "SourcePath", import_text(path));
|
|
import_add_number(doc, info, "SourceSize", xx size);
|
|
import_add(doc, info, "SourceModified", import_calendar_text(to_calendar(modified, .LOCAL)));
|
|
import_add(doc, info, "ContentHash", import_hash_text(hash));
|
|
import_add(doc, info, "Format", format);
|
|
import_add_number(doc, info, "Width", xx width);
|
|
import_add_number(doc, info, "Height", xx height);
|
|
import_add(doc, info, "DateFrom", date_from);
|
|
import_add(doc, info, "ImportDate", import_calendar_text(to_calendar(current_time_consensus(), .LOCAL)));
|
|
return doc, info;
|
|
}
|
|
|
|
import_add :: (doc: *Capture_Document, file: *Capture_File, key: string, value: string) {
|
|
capture_object_add(doc, *file.root, key, capture_string(doc, value));
|
|
}
|
|
|
|
import_add_number :: (doc: *Capture_Document, file: *Capture_File, key: string, value: float64) {
|
|
capture_object_add(doc, *file.root, key, capture_number(value));
|
|
}
|
|
|
|
// Seconds to the millisecond: 1.667, not 1.6666666666666668.
|
|
import_milliseconds :: (seconds: float64) -> float64 {
|
|
return floor(seconds * 1000 + 0.5) / 1000;
|
|
}
|
|
|
|
// The document as Capture Text (temp), once it reads back as written.
|
|
import_document_text :: (doc: *Capture_Document, path: string) -> string, ok: bool {
|
|
text := capture_text_write(doc);
|
|
back: Capture_Document;
|
|
capture_document_init(*back, temp);
|
|
if !capture_text_parse(*back, path, text) || back.mistakes || back.files.count != 2 return "", false;
|
|
return text, true;
|
|
}
|
|
|
|
// The camera or phone, as AppName: "Apple iPhone 15 Pro", or its model alone when that starts with
|
|
// its make.
|
|
import_device :: (make: string, model: string) -> string {
|
|
device := import_text(model);
|
|
if make && !begins_with_nocase(device, make) device = tprint("% %", make, device);
|
|
return trim(device);
|
|
}
|
|
|
|
// 2024-05-12_10-15-30_1a2b3c4d.mcap: newest last, as Snagit names its captures.
|
|
import_out_name :: (date: string, hash: u64) -> string {
|
|
stamp := replace(replace(date, " ", "_",, temp), ":", "-",, temp);
|
|
return tprint("%_%.%", stamp, slice(import_hash_text(hash), 0, 8), MCAP_EXTENSION);
|
|
}
|
|
|
|
// Apple's creationdate, "2024-05-12T10:15:30+0100": the local time it was taken, as Snagit writes
|
|
// CaptureDate, and its offset from UTC, "+01:00" ("" when it has none). "" when it is not a date.
|
|
import_video_local_date :: (text: string) -> date: string, offset: string {
|
|
date := import_exif_date(text);
|
|
if !date return "", "";
|
|
rest := advance(text, 19);
|
|
while rest.count && (rest[0] == #char "." || is_digit(rest[0])) rest = advance(rest, 1); // fractions of a second
|
|
offset := "";
|
|
if rest.count >= 5 && (rest[0] == #char "+" || rest[0] == #char "-") && is_digit(rest[1]) && is_digit(rest[2]) {
|
|
minutes := ifx rest[3] == #char ":" then slice(rest, 4, 2) else slice(rest, 3, 2);
|
|
if minutes.count == 2 && is_digit(minutes[0]) && is_digit(minutes[1]) offset = tprint("%:%", slice(rest, 0, 3), minutes);
|
|
} else if rest.count && rest[0] == #char "Z" {
|
|
offset = "+00:00";
|
|
}
|
|
return date, offset;
|
|
}
|
|
|
|
// A container's creation_time, "2024-05-12T09:15:30.000000Z", which is UTC, as local time in the form
|
|
// Snagit writes CaptureDate. "" when it is not a date, or is the zero date a camera with no clock
|
|
// writes (1904 or 1970).
|
|
import_video_utc_date :: (text: string) -> string {
|
|
date := import_exif_date(text);
|
|
if !date return "";
|
|
year := string_to_int(slice(date, 0, 4));
|
|
if year <= 1970 return "";
|
|
utc: Calendar_Time;
|
|
utc.year = cast(s32) year;
|
|
utc.month_starting_at_0 = cast(s8) (string_to_int(slice(date, 5, 2)) - 1);
|
|
utc.day_of_month_starting_at_0 = cast(s8) (string_to_int(slice(date, 8, 2)) - 1);
|
|
utc.hour = cast(s8) string_to_int(slice(date, 11, 2));
|
|
utc.minute = cast(s8) string_to_int(slice(date, 14, 2));
|
|
utc.second = cast(s8) string_to_int(slice(date, 17, 2));
|
|
utc.time_zone = .UTC;
|
|
return import_calendar_text(to_calendar(calendar_to_apollo(utc), .LOCAL));
|
|
}
|
|
|
|
import_hash :: (bytes: [] u8) -> u64 {
|
|
return fnv1a_hash(bytes.data, bytes.count);
|
|
}
|
|
|
|
// import_hash of the bytes ref points at, read a few MB at a time: the same hash, as FNV-1a carries
|
|
// on from where it left off.
|
|
import_hash_file :: (ref: Mcap_Original_Ref) -> u64, ok: bool {
|
|
CHUNK :: 4 * 1024 * 1024;
|
|
file, opened := file_open(ref.path, log_errors = false);
|
|
if !opened return 0, false;
|
|
defer file_close(*file);
|
|
if !file_set_position(file, ref.offset) return 0, false;
|
|
buffer := NewArray(min(ref.size, CHUNK), u8, initialized = false,, temp);
|
|
hash := FNV_64_OFFSET_BIAS;
|
|
left := ref.size;
|
|
while left > 0 {
|
|
count := min(left, CHUNK);
|
|
ok, read := file_read(file, buffer.data, count);
|
|
if !ok || read != count return 0, false;
|
|
hash = fnv1a_hash(buffer.data, count, hash);
|
|
left -= count;
|
|
}
|
|
return hash, true;
|
|
}
|
|
|
|
import_hash_text :: (hash: u64) -> string {
|
|
return tprint("%", formatInt(hash, base = 16, minimum_digits = 16));
|
|
}
|
|
|
|
// "2024-05-12 10:15:30", as Snagit writes CaptureDate. Temp.
|
|
import_calendar_text :: (c: Calendar_Time) -> string {
|
|
two :: (n: s64) -> string { return tprint("%", formatInt(n, minimum_digits = 2)); } // a string: a FormatInt would point at n
|
|
return tprint("%-%-% %:%:%", c.year, two(c.month_starting_at_0 + 1), two(c.day_of_month_starting_at_0 + 1), two(c.hour), two(c.minute), two(c.second));
|
|
}
|
|
|
|
// EXIF's "2024:05:12 10:15:30" as "2024-05-12 10:15:30"; "" when it is not a date (cameras with no
|
|
// clock set write zeros or blanks).
|
|
import_exif_date :: (text: string) -> string {
|
|
if text.count < 19 return "";
|
|
digits := "0000:00:00 00:00:00";
|
|
for 0..18 {
|
|
if digits[it] == #char "0" { if !is_digit(text[it]) return ""; }
|
|
else if text[it] != #char ":" && text[it] != #char "-" && text[it] != #char " " && text[it] != #char "T" return "";
|
|
}
|
|
year := string_to_int(slice(text, 0, 4));
|
|
month := string_to_int(slice(text, 5, 2));
|
|
day := string_to_int(slice(text, 8, 2));
|
|
if year < 1900 || month < 1 || month > 12 || day < 1 || day > 31 return "";
|
|
return tprint("%-%-% %", slice(text, 0, 4), slice(text, 5, 2), slice(text, 8, 2), slice(text, 11, 8));
|
|
}
|
|
|
|
// A date, and a time when there is one, in a file's name, as phones and screenshot tools write them:
|
|
// IMG_20240512_101530, PXL_20240512_101530123, Screenshot_20240512-101530, IMG-20240512-WA0001,
|
|
// Screenshot 2024-05-12 at 10.15.30, Screenshot_2024-05-12-10-15-30, 2024-05-12_10-15-30. "" for none.
|
|
import_name_date :: (name: string) -> string {
|
|
digits_at :: (s: string, at: s64, count: s64) -> s64 {
|
|
if at < 0 || at + count > s.count return -1;
|
|
for at..at + count - 1 if !is_digit(s[it]) return -1;
|
|
return string_to_int(slice(s, at, count));
|
|
}
|
|
separator :: (s: string, at: s64) -> bool {
|
|
return at < s.count && (s[at] == #char "-" || s[at] == #char "_" || s[at] == #char "." || s[at] == #char " " || s[at] == #char ":");
|
|
}
|
|
for start: 0..name.count - 8 {
|
|
if start > 0 && is_digit(name[start - 1]) continue; // the year starts a run of digits
|
|
year := digits_at(name, start, 4);
|
|
if year < 1990 || year > 2099 continue;
|
|
at := start + 4;
|
|
if separator(name, at) at += 1;
|
|
month := digits_at(name, at, 2);
|
|
at += 2;
|
|
if separator(name, at) at += 1;
|
|
day := digits_at(name, at, 2);
|
|
at += 2;
|
|
if month < 1 || month > 12 || day < 1 || day > 31 continue;
|
|
if at < name.count && is_digit(name[at]) && !(at + 6 <= name.count && digits_at(name, at, 6) >= 0) continue;
|
|
|
|
// The time: after a separator, or " at ", or nothing.
|
|
hour, minute, second := 0, 0, 0;
|
|
t := at;
|
|
if begins_with(advance(name, t), " at ") t += 4;
|
|
else if t < name.count && (separator(name, t) || name[t] == #char "T") t += 1;
|
|
h := digits_at(name, t, 2);
|
|
if h >= 0 && h < 24 {
|
|
t2 := t + 2;
|
|
if separator(name, t2) t2 += 1;
|
|
m := digits_at(name, t2, 2);
|
|
t3 := t2 + 2;
|
|
if separator(name, t3) t3 += 1;
|
|
s := digits_at(name, t3, 2);
|
|
if m >= 0 && m < 60 && s >= 0 && s < 60 { hour, minute, second = h, m, s; }
|
|
}
|
|
two :: (n: s64) -> string { return tprint("%", formatInt(n, minimum_digits = 2)); } // a string: a FormatInt would point at n
|
|
return tprint("%-%-% %:%:%", year, two(month), two(day), two(hour), two(minute), two(second));
|
|
}
|
|
return "";
|
|
}
|
|
|
|
// Text from a file (a name, an EXIF field) made safe for Capture Text: control characters as spaces,
|
|
// and bytes that are not UTF-8 (EXIF written in Latin-1) as ?. Temp.
|
|
import_text :: (text: string) -> string {
|
|
if !text return "";
|
|
out := copy_string(text,, temp);
|
|
for * out if it.* < 32 || it.* == 127 it.* = #char " ";
|
|
if !is_valid_utf8(out) for * out if it.* >= 128 it.* = #char "?";
|
|
return trim(out);
|
|
}
|
|
|
|
import_read_head :: (path: string, count: s64) -> [] u8 {
|
|
file, ok := file_open(path, log_errors = false);
|
|
if !ok return .[];
|
|
defer file_close(*file);
|
|
head := NewArray(count, u8,, temp);
|
|
read_ok, read := file_read(file, head.data, count);
|
|
if !read_ok return .[];
|
|
head.count = read;
|
|
return head;
|
|
}
|
|
|
|
// path inside the library, as the index keeps it: "SnagitLibraryWork/2024-05-12_10-15-30.mcap".
|
|
import_relative :: (path: string) -> string {
|
|
library := importer.library;
|
|
if path.count > library.count + 1 && begins_with_nocase(path, library) && (path[library.count] == #char "/" || path[library.count] == #char "\\") return advance(path, library.count + 1);
|
|
return path;
|
|
}
|
|
|
|
import_elapsed :: () -> float64 {
|
|
elapsed := to_float64_seconds(current_time_monotonic() - importer.start) - importer.paused_seconds;
|
|
if importer.paused elapsed -= to_float64_seconds(current_time_monotonic() - importer.pause_start);
|
|
return elapsed;
|
|
}
|
|
|
|
// ---------------------------------------------------------------- the library index
|
|
|
|
// Brings the index up to date with the library: files it does not know, or that changed, have their
|
|
// original hashed; files gone are dropped. False, with importer.error, when it cannot.
|
|
import_index_update :: () -> bool {
|
|
lock(*importer.mutex);
|
|
for _, key: importer.records free(key);
|
|
table_reset(*importer.records);
|
|
for value: importer.by_hash free(value);
|
|
table_reset(*importer.by_hash);
|
|
unlock(*importer.mutex);
|
|
|
|
// What the index says.
|
|
index_path := tprint("%/%", importer.library, IMPORT_INDEX_NAME);
|
|
data, read_ok := read_entire_file(index_path, log_errors = false,, temp);
|
|
if read_ok {
|
|
lines := split(data, "\n",, temp);
|
|
if lines.count && trim(lines[0]) == "@version 1" {
|
|
for 1..lines.count - 1 {
|
|
parts := split(trim_right(lines[it], "\r"), "\t",, temp);
|
|
if parts.count != 4 continue;
|
|
hash, hash_ok := string_to_int(parts[0], 16, u64);
|
|
size, size_ok := string_to_int(parts[1]);
|
|
modified, modified_ok := string_to_int(parts[2]);
|
|
if hash_ok && size_ok && modified_ok import_record_set(parts[3], .{size, modified, hash});
|
|
}
|
|
}
|
|
}
|
|
|
|
// What the library holds.
|
|
stale: [..] Stale_File;
|
|
stale.allocator = temp;
|
|
present: Table(string, bool);
|
|
present.allocator = temp;
|
|
for file_list(importer.library, recursive = true,, temp) {
|
|
if !ends_with_nocase(it, tprint(".%", MCAP_EXTENSION)) continue;
|
|
relative := import_relative(it);
|
|
table_set(*present, relative, true);
|
|
modified, size, ok := file_modtime_and_size(it);
|
|
if !ok continue;
|
|
nanoseconds := to_nanoseconds(modified);
|
|
found, record := table_find(*importer.records, relative);
|
|
if found && record.size == size && record.modified == nanoseconds continue;
|
|
array_add(*stale, .{copy_string(relative,, temp), size, nanoseconds});
|
|
}
|
|
gone: [..] string;
|
|
gone.allocator = temp;
|
|
for _, key: importer.records if !table_contains(*present, key) array_add(*gone, key);
|
|
for gone {
|
|
table_remove(*importer.records, it);
|
|
free(it,, context.default_allocator);
|
|
}
|
|
|
|
// The stale ones hashed, on the workers.
|
|
index_stale = stale;
|
|
importer.index_total = stale.count;
|
|
importer.index_done = 0;
|
|
index_next = 0;
|
|
if stale.count log("[Import] Indexing % of the library's .mcap files.", stale.count);
|
|
import_run_workers(import_index_worker);
|
|
if importer.cancel return false;
|
|
|
|
lock(*importer.mutex);
|
|
for record, relative: importer.records {
|
|
if !table_contains(*importer.by_hash, record.hash) table_set(*importer.by_hash, record.hash, copy_string(relative));
|
|
}
|
|
unlock(*importer.mutex);
|
|
import_index_save();
|
|
return true;
|
|
}
|
|
|
|
import_index_worker :: (thread: *Thread) -> s64 {
|
|
context.allocator = context.default_allocator;
|
|
while true {
|
|
while importer.paused && !importer.cancel sleep_milliseconds(100);
|
|
if importer.cancel break;
|
|
n := atomic_add(*index_next, 1);
|
|
if n >= index_stale.count break;
|
|
item := index_stale[n];
|
|
path := tprint("%/%", importer.library, item.relative);
|
|
// The original is hashed where it lies, a few MB at a time: it may be a video larger than memory.
|
|
m, original, parsed, error := mcap_read_without_original(path,, temp);
|
|
hash: u64;
|
|
hashed: bool;
|
|
if parsed hash, hashed = import_hash_file(original);
|
|
if hashed {
|
|
lock(*importer.mutex);
|
|
import_record_set(item.relative, .{item.size, item.modified, hash});
|
|
unlock(*importer.mutex);
|
|
} else {
|
|
log_error("[Import] % is not a readable .mcap (%); it is left out of the index.", path, ifx parsed then "its original cannot be read" else error);
|
|
}
|
|
atomic_add(*importer.index_done, 1);
|
|
reset_temporary_storage();
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
import_record_set :: (relative: string, record: Index_Record) {
|
|
pointer := table_find_pointer(*importer.records, relative);
|
|
if pointer { pointer.* = record; return; }
|
|
table_set(*importer.records, copy_string(relative,, context.default_allocator), record);
|
|
}
|
|
|
|
// Writes the index beside itself and swaps it in.
|
|
import_index_save :: () {
|
|
builder: String_Builder;
|
|
builder.allocator = temp;
|
|
append(*builder, "@version 1\n");
|
|
lock(*importer.mutex);
|
|
for record, relative: importer.records {
|
|
print_to_builder(*builder, "%\t%\t%\t%\n", import_hash_text(record.hash), record.size, record.modified, relative);
|
|
}
|
|
unlock(*importer.mutex);
|
|
path := tprint("%/%", importer.library, IMPORT_INDEX_NAME);
|
|
temp_path := tprint("%.tmp", path);
|
|
if !write_entire_file(temp_path, *builder) || !atomic_file_replace_or_just_rename_if_destination_file_does_not_exist(temp_path, path) {
|
|
log_error("[Import] Could not write %.", path);
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------- where each library folder came from
|
|
|
|
import_sources_load :: () {
|
|
data, ok := read_entire_file(tprint("%/%", viewer_folder(), IMPORT_SOURCES_NAME), log_errors = false,, temp);
|
|
if !ok return;
|
|
for split(data, "\n",, temp) {
|
|
parts := split(trim_right(it, "\r"), "\t",, temp);
|
|
if parts.count != 2 || !parts[0].count continue;
|
|
table_set(*importer.sources, copy_string(parts[0],, context.default_allocator), copy_string(parts[1],, context.default_allocator));
|
|
}
|
|
}
|
|
|
|
import_sources_save :: () {
|
|
builder: String_Builder;
|
|
builder.allocator = temp;
|
|
for source, dest: importer.sources print_to_builder(*builder, "%\t%\n", dest, source);
|
|
write_entire_file(tprint("%/%", viewer_folder(), IMPORT_SOURCES_NAME), *builder);
|
|
}
|
|
|
|
// The library folder a source goes into: the one it went into last time, else its own folder's name.
|
|
import_dest_for :: (source: string) -> string {
|
|
normal := copy_string(trim(source),, temp);
|
|
path_overwrite_separators(normal, #char "/");
|
|
while normal.count > 3 && normal[normal.count - 1] == #char "/" normal.count -= 1;
|
|
for value, key: importer.sources if to_lower_copy(value,, temp) == to_lower_copy(normal,, temp) return key;
|
|
name := path_filename(normal);
|
|
return ifx import_name_is_valid(name) then name else "";
|
|
}
|
|
|
|
// ---------------------------------------------------------------- each frame
|
|
|
|
import_frame :: () { using ImGui;
|
|
import_take_new_paths();
|
|
if app.import_source import_flag_frame();
|
|
|
|
// Ended: the scan of what came in, when asked for.
|
|
if !importer.running && importer.thread_started && importer.phase == .DONE {
|
|
thread_is_done(*importer.thread, -1);
|
|
thread_deinit(*importer.thread);
|
|
importer.thread_started = false;
|
|
if importer.error toast_show(.Error, "Import: %", importer.error);
|
|
else toast_show(.Success, "Imported % into % (% duplicates, % skipped, % failed)", importer.imported, importer.dest_name, importer.duplicates, importer.skipped, importer.failed);
|
|
importer.scan_pending = importer.read_after && importer.new_indices.count > 0 && !app.import_source;
|
|
importer.segment_pending = library_folder_segments(importer.dest_name) && importer.new_indices.count > 0 && !app.import_source;
|
|
}
|
|
if importer.scan_pending && !ocr_ui.thread_busy && ocr_ui.engine && labels_ui_settled() {
|
|
importer.scan_pending = false;
|
|
ocr_scan_start(importer.new_indices);
|
|
}
|
|
// Then segmentation (app/segment_ui.jai), once the new captures' catalog entries say which are videos.
|
|
if importer.segment_pending && !importer.scan_pending && !ocr_ui.thread_busy && !ocr_ui.scan.running {
|
|
catalogued := true;
|
|
for importer.new_indices if !viewer.captures[it].catalog && !viewer.captures[it].removed { catalogued = false; break; }
|
|
if catalogued {
|
|
importer.segment_pending = false;
|
|
segment_scan_start(importer.new_indices, quiet = true);
|
|
}
|
|
}
|
|
|
|
if !importer.window_open return;
|
|
SetNextWindowSize(.{720, 520}, .FirstUseEver);
|
|
if Begin(IMPORT_WINDOW, *importer.window_open) {
|
|
import_window();
|
|
}
|
|
End();
|
|
}
|
|
|
|
// The captures written since the last frame, into the grid.
|
|
import_take_new_paths :: () {
|
|
lock(*importer.mutex);
|
|
paths := array_copy(importer.new_paths,, temp);
|
|
array_reset_keeping_memory(*importer.new_paths);
|
|
unlock(*importer.mutex);
|
|
for paths {
|
|
index := viewer_add_capture(it);
|
|
if index >= 0 array_add(*importer.new_indices, index);
|
|
free(it,, context.default_allocator);
|
|
}
|
|
}
|
|
|
|
// -import SOURCE DEST: imports once the viewer has started, then ends the app (1 if any file failed).
|
|
import_flag_frame :: () {
|
|
if !importer.flag_started {
|
|
importer.flag_started = true;
|
|
importer.read_after = false;
|
|
if !import_start(app.import_source, app.import_dest) {
|
|
log_error("[Import] -import % % could not start.", app.import_source, app.import_dest);
|
|
app.exit_code = 2; app.should_exit = true; app.import_source = "";
|
|
}
|
|
return;
|
|
}
|
|
if importer.running || importer.thread_started return;
|
|
log("[Import] -import is done: % imported, % duplicates, % skipped, % failed, in %.", importer.imported, importer.duplicates, importer.skipped, importer.failed, ocr_duration_text(importer.elapsed_seconds));
|
|
app.exit_code = ifx importer.failed || importer.error then cast(s32) 1 else 0;
|
|
app.should_exit = true;
|
|
app.import_source = "";
|
|
}
|
|
|
|
import_window :: () { using ImGui;
|
|
running := importer.running;
|
|
TextColored(app_theme.palette.dim, "Imports every image (PNG, JPEG, HEIC, AVIF, WebP, GIF, BMP, TIFF, camera RAW), video (MP4, MOV, MKV, WebM, AVI, WMV, MPEG) and Snagit .snagx capture under a folder into a folder of its own in %. The files there are only read. Each is stored once in the whole library.", viewer_folder());
|
|
Spacing();
|
|
|
|
if running BeginDisabled();
|
|
SetNextItemWidth(-200);
|
|
if InputTextWithHint("##source", "The folder to import, e.g. D:\\Snagit or E:\\Phone backup\\DCIM", importer.source_buffer.data, importer.source_buffer.count) {
|
|
import_source_changed();
|
|
}
|
|
SameLine();
|
|
#if OS == .WINDOWS {
|
|
if Button("Browse...") {
|
|
chosen := windows_choose_folder("Choose a folder to import");
|
|
if chosen {
|
|
count := min(chosen.count, importer.source_buffer.count - 1);
|
|
memcpy(importer.source_buffer.data, chosen.data, count);
|
|
importer.source_buffer[count] = 0;
|
|
import_source_changed();
|
|
}
|
|
}
|
|
SameLine();
|
|
}
|
|
Text("Import from");
|
|
|
|
SetNextItemWidth(-200);
|
|
if InputTextWithHint("##dest", "A folder in the library, e.g. SnagitLibraryWork", importer.dest_buffer.data, importer.dest_buffer.count) {
|
|
importer.dest_edited = importer.dest_buffer[0] != 0;
|
|
}
|
|
SameLine();
|
|
Text("Into library folder");
|
|
source := to_string(importer.source_buffer.data);
|
|
dest := trim(to_string(importer.dest_buffer.data));
|
|
if dest {
|
|
if !import_name_is_valid(dest) TextColored(app_theme.palette.warning, "\"%\" cannot be a folder's name.", dest);
|
|
else TextColored(app_theme.palette.dim, "Into %/%", viewer_folder(), dest);
|
|
}
|
|
Checkbox("Read text and labels in what is imported, once it is in", *importer.read_after);
|
|
if dest && import_name_is_valid(dest) {
|
|
// The library folder's own option (app/library_folders.jai): it stays with the folder.
|
|
segment := library_folder_segments(dest);
|
|
if Checkbox("Segment the images in this folder (what is where: people, animals, things, sky...)", *segment) library_folder_set_segment(dest, segment);
|
|
if IsItemHovered() SetTooltip("Kept for the library folder: the Segmentation window has it too. Videos are never segmented.");
|
|
}
|
|
if running EndDisabled();
|
|
Spacing();
|
|
|
|
if !running {
|
|
can_start := trim(source).count > 0 && dest.count > 0 && import_name_is_valid(dest);
|
|
if !can_start BeginDisabled();
|
|
if Button("Import") import_start(source, dest);
|
|
if !can_start EndDisabled();
|
|
} else {
|
|
if importer.paused {
|
|
if Button("Resume") { importer.paused_seconds += to_float64_seconds(current_time_monotonic() - importer.pause_start); atomic_swap(*importer.paused, false); }
|
|
} else {
|
|
if Button("Pause") { importer.pause_start = current_time_monotonic(); atomic_swap(*importer.paused, true); }
|
|
}
|
|
SameLine();
|
|
if Button("Cancel") {
|
|
if importer.paused importer.paused_seconds += to_float64_seconds(current_time_monotonic() - importer.pause_start);
|
|
atomic_swap(*importer.cancel, true);
|
|
atomic_swap(*importer.paused, false);
|
|
}
|
|
}
|
|
|
|
if importer.phase == .IDLE return;
|
|
Separator();
|
|
if importer.error TextColored(app_theme.palette.warning, "%", importer.error);
|
|
elapsed := ifx running then import_elapsed() else importer.elapsed_seconds;
|
|
if importer.phase == {
|
|
case .LISTING;
|
|
Text("Looking through %: % files so far", importer.source, importer.listed);
|
|
case .INDEXING;
|
|
Text("Indexing the library (once; this finds duplicates):");
|
|
ProgressBar(cast(float) importer.index_done / max(importer.index_total, 1), .{-FLOAT32_MIN, 0}, temp_c_string(ocr_scan_progress_text(importer.index_done, importer.index_total)));
|
|
case;
|
|
total := importer.files.count;
|
|
done := importer.done;
|
|
rate := ifx elapsed > 0 then done / elapsed else 0;
|
|
left := ifx rate > 0 then (total - done) / rate else 0;
|
|
if running {
|
|
Text("Importing % into %:", importer.source, importer.dest_name);
|
|
ProgressBar(cast(float) done / max(total, 1), .{-FLOAT32_MIN, 0}, temp_c_string(ocr_scan_progress_text(done, total)));
|
|
Text("% elapsed, % left, % files/s%", ocr_duration_text(elapsed), ocr_duration_text(left), FF(rate, 1), ifx importer.paused then " (paused)" else "");
|
|
} else {
|
|
Text("Last import, % into %: % of % files in %%.", importer.source, importer.dest_name, done, total, ocr_duration_text(elapsed), ifx importer.cancel then " (cancelled)" else "");
|
|
}
|
|
}
|
|
Text("% imported, % duplicates, % skipped, % failed", importer.imported, importer.duplicates, importer.skipped, importer.failed);
|
|
|
|
lock(*importer.mutex);
|
|
defer unlock(*importer.mutex);
|
|
if importer.failures.count && CollapsingHeader(temp_c_string(tprint("Failed (%)###failed", importer.failed))) {
|
|
if SmallButton("Copy the list##failed") clipboard_set_text(import_notes_text(importer.failures));
|
|
BeginChild("failed", .{0, GetTextLineHeightWithSpacing() * min(cast(float) importer.failures.count + 1, 12)});
|
|
import_list(importer.failures, "% %");
|
|
EndChild();
|
|
}
|
|
if importer.duplicate_notes.count && CollapsingHeader(temp_c_string(tprint("Duplicates (%), already in the library###duplicates", importer.duplicates))) {
|
|
if SmallButton("Copy the list##duplicates") clipboard_set_text(import_notes_text(importer.duplicate_notes));
|
|
BeginChild("duplicates", .{0, GetTextLineHeightWithSpacing() * min(cast(float) importer.duplicate_notes.count + 1, 12)});
|
|
import_list(importer.duplicate_notes, "% is %");
|
|
EndChild();
|
|
}
|
|
if importer.skipped && CollapsingHeader(temp_c_string(tprint("Skipped (%), by kind###skipped", importer.skipped))) {
|
|
kinds: [..] string;
|
|
kinds.allocator = temp;
|
|
for _, key: importer.skipped_kinds array_add(*kinds, key);
|
|
quick_sort(kinds, compare);
|
|
for kinds {
|
|
_, count := table_find(*importer.skipped_kinds, it);
|
|
Text("%: %", it, count);
|
|
}
|
|
}
|
|
}
|
|
|
|
// The notes, a line each, only those in view drawn.
|
|
import_list :: (notes: [] Import_Note, format: string) { using ImGui;
|
|
clipper: ListClipper;
|
|
clipper.Constructor(*clipper);
|
|
defer clipper.Destructor(*clipper);
|
|
clipper.Begin(*clipper, xx notes.count);
|
|
while clipper.Step(*clipper) {
|
|
for clipper.DisplayStart..clipper.DisplayEnd - 1 TextUnformatted(tprint(format, notes[it].path, notes[it].text));
|
|
}
|
|
clipper.End(*clipper);
|
|
}
|
|
|
|
import_notes_text :: (notes: [] Import_Note) -> string {
|
|
builder: String_Builder;
|
|
builder.allocator = temp;
|
|
for notes print_to_builder(*builder, "%\t%\n", it.path, it.text);
|
|
return builder_to_string(*builder,, temp);
|
|
}
|
|
|
|
// A new source: the library folder it goes into, unless one was typed.
|
|
import_source_changed :: () {
|
|
if importer.dest_edited && importer.dest_buffer[0] return;
|
|
dest := import_dest_for(to_string(importer.source_buffer.data));
|
|
count := min(dest.count, importer.dest_buffer.count - 1);
|
|
memcpy(importer.dest_buffer.data, dest.data, count);
|
|
importer.dest_buffer[count] = 0;
|
|
importer.dest_edited = false;
|
|
}
|
|
|
|
#scope_file
|
|
|
|
Stale_File :: struct { relative: string; size: s64; modified: s64; }
|
|
index_stale: [] Stale_File;
|
|
index_next: s64;
|