[0.0.0] Statistics window: what OCR found in the captures shown; elapsed time for scans
This commit is contained in:
parent
962f0d7515
commit
cf3a745588
@ -45,7 +45,11 @@ of previews, and one capture at full size in the Capture window.
|
||||
every capture shown that the model has not read yet, storing each reading in its file as it goes,
|
||||
so it can pause, cancel and pick up again. `-library FOLDER` shows another folder of `.mcap` files;
|
||||
`-scan` scans it with the chosen model once it has loaded and exits (0, or 1 if any capture
|
||||
failed), for a scan without the UI, such as on a copy. The Capture window's **OCR** button reads one capture;
|
||||
failed), for a scan without the UI, such as on a copy.
|
||||
- **Stats** above the grid: what OCR found in the captures shown, by the chosen model: captures with
|
||||
text, lines, words, characters, confidence, words per capture, a year table, the captures with
|
||||
the most words (click to open) and the commonest words (click to search). `-stats` logs the same
|
||||
numbers once the catalog has loaded, and exits. The Capture window's **OCR** button reads one capture;
|
||||
a capture read by several models keeps each reading, and the window can show any of them.
|
||||
|
||||
## Building
|
||||
|
||||
@ -18,6 +18,7 @@ USING_WINDOWS :: OS == .WINDOWS;
|
||||
#load "app/viewer.jai";
|
||||
#load "app/ocr_ui.jai";
|
||||
#load "app/catalog.jai";
|
||||
#load "app/stats_ui.jai";
|
||||
#load "app/exit_shutdown.jai";
|
||||
|
||||
#load "base/circle_buffer.jai";
|
||||
@ -174,6 +175,7 @@ App :: struct {
|
||||
exit_code: s32; // the process's exit code when the main loop ends
|
||||
library_folder: string; // -library FOLDER: the .mcap folder the viewer shows, in place of VIEWER_FOLDER
|
||||
scan_and_exit: bool; // -scan: scan the library with the chosen OCR model once it has loaded, then exit (0, or 1 if any capture failed)
|
||||
stats_and_exit: bool; // -stats: log the Statistics window's numbers once the catalog has loaded, then exit
|
||||
last_window_restore_time: Apollo_Time;
|
||||
|
||||
imgui_context: *ImGui.ImGuiContext;
|
||||
|
||||
@ -19,10 +19,12 @@ app_ui_frame_start :: () {
|
||||
app_ui_shutdown :: () {
|
||||
viewer_shutdown();
|
||||
ocr_ui_shutdown();
|
||||
stats_ui_shutdown();
|
||||
}
|
||||
|
||||
app_main_window_ui :: () {
|
||||
push_imgui_window(APP_MAIN_WINDOW_LABEL);
|
||||
viewer_ui();
|
||||
ocr_ui_frame();
|
||||
stats_ui_frame();
|
||||
}
|
||||
|
||||
@ -19,6 +19,7 @@ main :: () {
|
||||
app.allow_multiple_instances = array_find(app.args, "-allow-multiple-instances");
|
||||
for app.args if it == "-library" && it_index + 1 < app.args.count app.library_folder = app.args[it_index + 1];
|
||||
app.scan_and_exit = array_find(app.args, "-scan");
|
||||
app.stats_and_exit = array_find(app.args, "-stats");
|
||||
|
||||
defer report_memory_leaks();
|
||||
|
||||
|
||||
@ -67,6 +67,7 @@ Ocr_Scan :: struct {
|
||||
start: Apollo_Time;
|
||||
paused_seconds: float64; // time spent paused, left out of the rate
|
||||
pause_start: Apollo_Time;
|
||||
elapsed_seconds: float64; // once it has ended: the time spent scanning, pauses left out
|
||||
}
|
||||
|
||||
Scan_Item :: struct { index: s32; path: string; }
|
||||
@ -177,7 +178,8 @@ ocr_ui_thread_proc :: (thread: *Thread) -> s64 {
|
||||
position += count;
|
||||
reset_temporary_storage();
|
||||
}
|
||||
log("[OCR scan] % of % captures read with % (% failed)%.", scan.done, scan.total, scan.model, scan.failed, ifx scan.cancel then ", cancelled" else "");
|
||||
scan.elapsed_seconds = to_float64_seconds(current_time_monotonic() - scan.start) - scan.paused_seconds;
|
||||
log("[OCR scan] % of % captures read with % in % (% failed)%.", scan.done, scan.total, scan.model, ocr_duration_text(scan.elapsed_seconds), scan.failed, ifx scan.cancel then ", cancelled" else "");
|
||||
atomic_swap(*scan.running, false);
|
||||
}
|
||||
free(task.model);
|
||||
@ -277,16 +279,16 @@ ocr_ui_scan_and_exit :: (busy: bool) {
|
||||
return;
|
||||
}
|
||||
if ocr_ui.scan.running {
|
||||
done, total, failed, rate, left := ocr_scan_progress();
|
||||
done, total, failed, rate, left, elapsed := ocr_scan_progress();
|
||||
step := max(total / 20, 1);
|
||||
if done / step > ocr_ui.scan_logged_step {
|
||||
ocr_ui.scan_logged_step = done / step;
|
||||
log("[OCR scan] % read, % failed, % captures/s, % left.", ocr_scan_progress_text(done, total), failed, FF(rate, 1), ocr_duration_text(left));
|
||||
log("[OCR scan] % read, % failed, % elapsed, % left, % captures/s.", ocr_scan_progress_text(done, total), failed, ocr_duration_text(elapsed), ocr_duration_text(left), FF(rate, 1));
|
||||
}
|
||||
return;
|
||||
}
|
||||
if busy return;
|
||||
finish(ifx ocr_ui.scan.failed then cast(s32) 1 else 0, "[OCR scan] -scan is done: % read, % failed.", ocr_ui.scan.done, ocr_ui.scan.failed);
|
||||
finish(ifx ocr_ui.scan.failed then cast(s32) 1 else 0, "[OCR scan] -scan is done: % read in %, % failed.", ocr_ui.scan.done, ocr_duration_text(ocr_ui.scan.elapsed_seconds), ocr_ui.scan.failed);
|
||||
}
|
||||
|
||||
ocr_scan_start :: () {
|
||||
@ -326,33 +328,37 @@ ocr_scan_ui :: () { using ImGui;
|
||||
if !catalog_ready TextColored(app_theme.palette.dim, "(once the catalog has loaded)");
|
||||
else TextColored(app_theme.palette.dim, "Reads every capture without a reading by this model. Each is stored as it is read, so a scan can stop and start again.");
|
||||
if scan.total && scan.done {
|
||||
TextColored(app_theme.palette.dim, "Last scan: % of % read with %, % failed.", scan.done, scan.total, scan.model, scan.failed);
|
||||
TextColored(app_theme.palette.dim, "Last scan: % of % read with % in %, % failed.", scan.done, scan.total, scan.model, ocr_duration_text(scan.elapsed_seconds), scan.failed);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
done, total, failed, rate, left := ocr_scan_progress();
|
||||
done, total, failed, rate, left, elapsed := ocr_scan_progress();
|
||||
ProgressBar(cast(float) done / max(total, 1), .{-FLOAT32_MIN, 0}, temp_c_string(ocr_scan_progress_text(done, total)));
|
||||
Text("% captures/s, % left, % failed, on %", FF(rate, 1), ocr_duration_text(left), failed, ifx ocr_ui.engine && ocr_ui.engine.provider == .CUDA then "the GPU" else "the CPU");
|
||||
Text("% elapsed, % left, % captures/s, % failed, on %", ocr_duration_text(elapsed), ocr_duration_text(left), FF(rate, 1), failed, ifx ocr_ui.engine && ocr_ui.engine.provider == .CUDA then "the GPU" else "the CPU");
|
||||
if scan.paused {
|
||||
if Button("Resume") { scan.paused_seconds += to_float64_seconds(current_time_monotonic() - scan.pause_start); atomic_swap(*scan.paused, false); }
|
||||
} else {
|
||||
if Button("Pause") { scan.pause_start = current_time_monotonic(); atomic_swap(*scan.paused, true); }
|
||||
}
|
||||
SameLine();
|
||||
if Button("Cancel") { atomic_swap(*scan.cancel, true); atomic_swap(*scan.paused, false); }
|
||||
if Button("Cancel") {
|
||||
if scan.paused scan.paused_seconds += to_float64_seconds(current_time_monotonic() - scan.pause_start); // not counted as scanning
|
||||
atomic_swap(*scan.cancel, true);
|
||||
atomic_swap(*scan.paused, false);
|
||||
}
|
||||
}
|
||||
|
||||
// Where a running scan is: captures read (or failed) of the total, the rate over the time not paused,
|
||||
// and the time left at that rate.
|
||||
ocr_scan_progress :: () -> done: s64, total: s64, failed: s64, rate: float64, seconds_left: float64 {
|
||||
// the time left at that rate, and the time spent so far (pauses left out).
|
||||
ocr_scan_progress :: () -> done: s64, total: s64, failed: s64, rate: float64, seconds_left: float64, elapsed: float64 {
|
||||
scan := *ocr_ui.scan;
|
||||
done, total, failed := scan.done, scan.total, scan.failed;
|
||||
elapsed := to_float64_seconds(current_time_monotonic() - scan.start) - scan.paused_seconds;
|
||||
if scan.paused elapsed -= to_float64_seconds(current_time_monotonic() - scan.pause_start);
|
||||
rate := ifx elapsed > 0 then done / elapsed else 0;
|
||||
left := ifx rate > 0 then (total - done) / rate else 0;
|
||||
return done, total, failed, rate, left;
|
||||
return done, total, failed, rate, left, elapsed;
|
||||
}
|
||||
|
||||
// "1234 of 8637 (14%)". Temp.
|
||||
|
||||
334
src/app/stats_ui.jai
Normal file
334
src/app/stats_ui.jai
Normal file
@ -0,0 +1,334 @@
|
||||
// The Statistics window: what OCR found in the captures shown (the whole library, or a search's
|
||||
// matches), by the model chosen in the OCR window. Worked out from the catalog, in memory, again when
|
||||
// the catalog, the search or the model changes; at most once a second while a scan adds readings.
|
||||
|
||||
STATS_WINDOW :: "Statistics";
|
||||
STATS_TOP_CAPTURES :: 10;
|
||||
STATS_TOP_WORDS :: 40;
|
||||
STATS_LOW_CONFIDENCE :: 0.8;
|
||||
|
||||
// Words per capture, in these ranges: 0, 1-10, 11-50, 51-200, 201-1000, more.
|
||||
STATS_BUCKET_TOPS :: s64.[0, 10, 50, 200, 1000, S64_MAX];
|
||||
STATS_BUCKET_LABELS :: string.["no text", "1 to 10", "11 to 50", "51 to 200", "201 to 1,000", "over 1,000"];
|
||||
|
||||
Stats_Year :: struct { year: string; captures, with_text, words: s64; }
|
||||
Stats_Capture :: struct { index: s32; words: s64; }
|
||||
Stats_Word :: struct { word: string; count: s64; }
|
||||
|
||||
stats_ui: struct {
|
||||
window_open: bool;
|
||||
|
||||
// What the numbers were worked out from; any change makes them stale.
|
||||
model: string; // heap
|
||||
catalog_generation, visible_generation: s64;
|
||||
computed: bool;
|
||||
computed_at: Apollo_Time;
|
||||
compute_ms: float64;
|
||||
|
||||
captures, read, with_text: s64;
|
||||
lines, words, characters, cjk_characters, turned_lines, low_confidence_lines: s64;
|
||||
confidence_sum: float64;
|
||||
words_median, words_p90, words_p99, words_max: s64;
|
||||
buckets: [STATS_BUCKET_TOPS.count] s64;
|
||||
years: [..] Stats_Year; // oldest first; year strings on the heap
|
||||
most_words: [..] Stats_Capture; // most first
|
||||
common_words: [..] Stats_Word; // commonest first; words on the heap
|
||||
};
|
||||
|
||||
stats_ui_shutdown :: () {
|
||||
stats_free_results();
|
||||
free(stats_ui.model,, context.default_allocator);
|
||||
array_free(stats_ui.years);
|
||||
array_free(stats_ui.most_words);
|
||||
array_free(stats_ui.common_words);
|
||||
}
|
||||
|
||||
// The model the numbers are for: the one chosen in the OCR window.
|
||||
stats_model :: () -> string {
|
||||
if ocr_ui.models.count return ocr_ui.models[ocr_ui.model_index].name;
|
||||
return OCR_DEFAULT_MODEL;
|
||||
}
|
||||
|
||||
stats_ui_frame :: () { using ImGui;
|
||||
if app.stats_and_exit && viewer.captures.count && viewer.catalog_loaded >= viewer.captures.count && !ocr_ui.thread_busy {
|
||||
stats_compute(stats_model());
|
||||
stats_log();
|
||||
app.stats_and_exit = false;
|
||||
app.should_exit = true;
|
||||
}
|
||||
if !stats_ui.window_open return;
|
||||
model := stats_model();
|
||||
stale := !stats_ui.computed || stats_ui.model != model
|
||||
|| stats_ui.catalog_generation != viewer.catalog_generation || stats_ui.visible_generation != viewer.visible_generation;
|
||||
if stale && (!stats_ui.computed || to_float64_seconds(current_time_monotonic() - stats_ui.computed_at) >= 1) stats_compute(model);
|
||||
|
||||
SetNextWindowSize(.{560, 640}, .FirstUseEver);
|
||||
if Begin(STATS_WINDOW, *stats_ui.window_open) {
|
||||
stats_window_contents();
|
||||
}
|
||||
End();
|
||||
}
|
||||
|
||||
#scope_file
|
||||
|
||||
// The numbers, as the window shows them, to the log: for -stats.
|
||||
stats_log :: () {
|
||||
s := *stats_ui;
|
||||
log("[stats] % with %: % captures, % read, % with text (%), % lines, % words, % characters (% CJK), % at an angle, mean confidence %, % under %, worked out in % ms.",
|
||||
viewer_folder(), s.model, s.captures, s.read, s.with_text, stats_percent(s.with_text, s.read), s.lines, s.words, s.characters, s.cjk_characters,
|
||||
s.turned_lines, FF(s.confidence_sum / max(s.lines, 1), 3), s.low_confidence_lines, FF(STATS_LOW_CONFIDENCE, 1), FF(s.compute_ms, 1));
|
||||
log("[stats] words per capture: mean %, median %, 90th %, 99th %, most %; by range %.", s.words / max(s.read, 1), s.words_median, s.words_p90, s.words_p99, s.words_max, s.buckets);
|
||||
for s.years log("[stats] % : % captures, % with text, % words", it.year, it.captures, it.with_text, it.words);
|
||||
builder: String_Builder;
|
||||
builder.allocator = temp;
|
||||
for s.common_words print_to_builder(*builder, "% %, ", it.word, it.count);
|
||||
log("[stats] commonest: %", builder_to_string(*builder,, temp));
|
||||
}
|
||||
|
||||
stats_window_contents :: () { using ImGui;
|
||||
s := *stats_ui;
|
||||
shown := ifx viewer.search_terms.count then "the captures matching the search" else "the library";
|
||||
TextWrapped("OCR with % over %: % captures.", s.model, shown, stats_number(s.captures));
|
||||
if viewer.catalog_loaded < viewer.captures.count {
|
||||
TextColored(app_theme.palette.warning, "The catalog is still loading (% of %); these numbers grow until it has.", viewer.catalog_loaded, viewer.captures.count);
|
||||
}
|
||||
if s.read < s.captures {
|
||||
TextColored(app_theme.palette.dim, "% have not been read with this model; the OCR window's scan reads them.", stats_number(s.captures - s.read));
|
||||
}
|
||||
if !s.read { TextColored(app_theme.palette.dim, "No readings yet."); return; }
|
||||
|
||||
SeparatorText("Text found");
|
||||
if BeginTable("##summary", 2, .SizingFixedFit) {
|
||||
row :: (label: string, value: string) #expand {
|
||||
ImGui.TableNextRow();
|
||||
ImGui.TableNextColumn(); ImGui.TextUnformatted(label);
|
||||
ImGui.TableNextColumn(); ImGui.TextUnformatted(value);
|
||||
}
|
||||
row("Captures with text", tprint("% (%)", stats_number(s.with_text), stats_percent(s.with_text, s.read)));
|
||||
row("Captures without", stats_number(s.read - s.with_text));
|
||||
row("Text lines", stats_number(s.lines));
|
||||
row("Words", stats_number(s.words));
|
||||
row("Characters", tprint("% (% Chinese, Japanese or Korean)", stats_number(s.characters), stats_number(s.cjk_characters)));
|
||||
row("Lines at an angle", tprint("% (%)", stats_number(s.turned_lines), stats_percent(s.turned_lines, s.lines)));
|
||||
row("Mean confidence", tprint("%", FF(s.confidence_sum / max(s.lines, 1), 3)));
|
||||
row(tprint("Lines under %", FF(STATS_LOW_CONFIDENCE, 1)), tprint("% (%)", stats_number(s.low_confidence_lines), stats_percent(s.low_confidence_lines, s.lines)));
|
||||
EndTable();
|
||||
}
|
||||
|
||||
SeparatorText("Words per capture");
|
||||
Text("Mean %, median %, 90th percentile %, 99th %, most %.", stats_number(s.words / max(s.read, 1)), stats_number(s.words_median), stats_number(s.words_p90), stats_number(s.words_p99), stats_number(s.words_max));
|
||||
if BeginTable("##buckets", 3, .SizingFixedFit) {
|
||||
TableSetupColumn("words");
|
||||
TableSetupColumn("captures");
|
||||
TableSetupColumn("bar", .WidthStretch);
|
||||
for s.buckets {
|
||||
TableNextRow();
|
||||
TableNextColumn(); TextUnformatted(STATS_BUCKET_LABELS[it_index]);
|
||||
TableNextColumn(); TextUnformatted(stats_number(it));
|
||||
TableNextColumn(); ProgressBar(cast(float) it / max(s.read, 1), .{-FLOAT32_MIN, 0}, temp_c_string(stats_percent(it, s.read)));
|
||||
}
|
||||
EndTable();
|
||||
}
|
||||
|
||||
SeparatorText("By year");
|
||||
if BeginTable("##years", 5, .SizingFixedFit | .RowBg) {
|
||||
TableSetupColumn("Year");
|
||||
TableSetupColumn("Captures");
|
||||
TableSetupColumn("With text");
|
||||
TableSetupColumn("Words");
|
||||
TableSetupColumn("Words per capture");
|
||||
TableHeadersRow();
|
||||
for s.years {
|
||||
TableNextRow();
|
||||
TableNextColumn(); TextUnformatted(it.year);
|
||||
TableNextColumn(); TextUnformatted(stats_number(it.captures));
|
||||
TableNextColumn(); TextUnformatted(stats_number(it.with_text));
|
||||
TableNextColumn(); TextUnformatted(stats_number(it.words));
|
||||
TableNextColumn(); TextUnformatted(stats_number(it.words / max(it.captures, 1)));
|
||||
}
|
||||
EndTable();
|
||||
}
|
||||
|
||||
SeparatorText("Most words");
|
||||
TextColored(app_theme.palette.dim, "Click one to open it.");
|
||||
for s.most_words {
|
||||
if it.index < 0 || it.index >= viewer.captures.count continue;
|
||||
capture := *viewer.captures[it.index];
|
||||
date := ifx capture.catalog && capture.catalog.date then capture.catalog.date else "undated";
|
||||
if Selectable(temp_c_string(tprint("% words % %##most%", stats_number(it.words), date, capture.name, it_index))) viewer_request_full(it.index);
|
||||
}
|
||||
|
||||
SeparatorText("Commonest words");
|
||||
TextColored(app_theme.palette.dim, "Words of 3 letters or more, without the commonest English ones. Click one to search for it.");
|
||||
right_edge := GetCursorScreenPos().x + GetContentRegionAvail().x;
|
||||
for s.common_words {
|
||||
label := temp_c_string(tprint("% %##word%", it.word, stats_number(it.count), it_index));
|
||||
width := CalcTextSize(label, null, true).x + GetStyle().FramePadding.x * 2;
|
||||
if it_index > 0 {
|
||||
SameLine();
|
||||
if GetCursorScreenPos().x + width > right_edge NewLine();
|
||||
}
|
||||
if SmallButton(label) viewer_set_search(tprint("ocr:%", it.word));
|
||||
}
|
||||
Spacing();
|
||||
TextColored(app_theme.palette.dim, "Worked out in % ms.", FF(s.compute_ms, 1));
|
||||
}
|
||||
|
||||
// Recounts everything over viewer.visible for model.
|
||||
stats_compute :: (model: string) {
|
||||
start := current_time_monotonic();
|
||||
s := *stats_ui;
|
||||
stats_free_results();
|
||||
free(s.model,, context.default_allocator);
|
||||
s.model = copy_string(model,, context.default_allocator);
|
||||
s.catalog_generation, s.visible_generation = viewer.catalog_generation, viewer.visible_generation;
|
||||
s.years.allocator = context.default_allocator;
|
||||
s.most_words.allocator = context.default_allocator;
|
||||
s.common_words.allocator = context.default_allocator;
|
||||
|
||||
per_capture: [..] Stats_Capture;
|
||||
per_capture.allocator = temp;
|
||||
years: Table(string, Stats_Year); // keys borrow the catalog's dates
|
||||
years.allocator = temp;
|
||||
words: Table(string, s64); // keys in temp
|
||||
words.allocator = temp;
|
||||
|
||||
for index: viewer.visible {
|
||||
s.captures += 1;
|
||||
entry := viewer.captures[index].catalog;
|
||||
year_text := ifx entry && entry.date.count >= 4 then slice(entry.date, 0, 4) else "undated";
|
||||
year := find_or_add(*years, year_text);
|
||||
year.year = year_text;
|
||||
year.captures += 1;
|
||||
|
||||
section := catalog_section(entry, model);
|
||||
if !section continue;
|
||||
s.read += 1;
|
||||
capture_words: s64;
|
||||
for line: section.lines {
|
||||
s.lines += 1;
|
||||
s.confidence_sum += line.confidence;
|
||||
if line.confidence < STATS_LOW_CONFIDENCE s.low_confidence_lines += 1;
|
||||
corners := mcap_ocr_corners(line);
|
||||
x0, y0, x1, y1 := line.x, line.y, line.x + line.width, line.y + line.height;
|
||||
box := s32.[x0, y0, x1, y0, x1, y1, x0, y1];
|
||||
for corners if it != box[it_index] { s.turned_lines += 1; break; }
|
||||
capture_words += stats_count_line(line.text, *words);
|
||||
}
|
||||
s.words += capture_words;
|
||||
if section.lines.count { s.with_text += 1; year.with_text += 1; }
|
||||
year.words += capture_words;
|
||||
array_add(*per_capture, .{index, capture_words});
|
||||
}
|
||||
|
||||
// Percentiles, ranges and the captures with the most words.
|
||||
quick_sort(per_capture, (a: Stats_Capture, b: Stats_Capture) -> s64 { return b.words - a.words; });
|
||||
at :: (fraction: float64) -> s64 #expand {
|
||||
if !per_capture.count return 0;
|
||||
return per_capture[clamp(cast(s64) ((1 - fraction) * per_capture.count), 0, per_capture.count - 1)].words;
|
||||
}
|
||||
s.words_median, s.words_p90, s.words_p99 = at(0.5), at(0.9), at(0.99);
|
||||
s.words_max = ifx per_capture.count then per_capture[0].words else 0;
|
||||
for per_capture {
|
||||
for top: STATS_BUCKET_TOPS if it.words <= top { s.buckets[it_index] += 1; break; }
|
||||
if it_index < STATS_TOP_CAPTURES array_add(*s.most_words, it);
|
||||
}
|
||||
|
||||
for years array_add(*s.years, .{year = copy_string(it.year,, context.default_allocator), captures = it.captures, with_text = it.with_text, words = it.words});
|
||||
quick_sort(s.years, (a: Stats_Year, b: Stats_Year) -> s64 { return compare(a.year, b.year); });
|
||||
|
||||
ranked: [..] Stats_Word;
|
||||
ranked.allocator = temp;
|
||||
for count, word: words array_add(*ranked, .{word, count});
|
||||
quick_sort(ranked, (a: Stats_Word, b: Stats_Word) -> s64 { return ifx a.count != b.count then b.count - a.count else compare(a.word, b.word); });
|
||||
for ranked {
|
||||
if it_index >= STATS_TOP_WORDS break;
|
||||
array_add(*s.common_words, .{copy_string(it.word,, context.default_allocator), it.count});
|
||||
}
|
||||
|
||||
s.computed = true;
|
||||
s.computed_at = current_time_monotonic();
|
||||
s.compute_ms = to_float64_seconds(s.computed_at - start) * 1000;
|
||||
}
|
||||
|
||||
// Adds the line's characters to the counts, and its Latin words to words; returns its words (runs
|
||||
// between spaces, numbers and symbols included).
|
||||
stats_count_line :: (text: string, words: *Table(string, s64)) -> s64 {
|
||||
s := *stats_ui;
|
||||
count: s64;
|
||||
in_word := false;
|
||||
latin_end := 0; // the end of the last Latin word, which its hyphens and apostrophes belong to
|
||||
i := 0;
|
||||
while i < text.count {
|
||||
c := text[i];
|
||||
if (c & 0xC0) != 0x80 s.characters += 1; // not a UTF-8 continuation byte: a character starts
|
||||
if (c & 0xF0) == 0xE0 && i + 2 < text.count {
|
||||
code := ((cast(u32) c & 0x0F) << 12) | ((cast(u32) text[i + 1] & 0x3F) << 6) | (cast(u32) text[i + 2] & 0x3F);
|
||||
if (code >= 0x3040 && code <= 0x30FF) || (code >= 0x3400 && code <= 0x4DBF) || (code >= 0x4E00 && code <= 0x9FFF) || (code >= 0xAC00 && code <= 0xD7AF) s.cjk_characters += 1;
|
||||
}
|
||||
space := c == #char " " || c == #char "\t";
|
||||
if !space && !in_word count += 1;
|
||||
in_word = !space;
|
||||
|
||||
// A Latin word: a letter, then letters, apostrophes and hyphens.
|
||||
if i >= latin_end && stats_is_letter(c) && (i == 0 || !stats_is_letter(text[i - 1])) {
|
||||
end := i + 1;
|
||||
while end < text.count && (stats_is_letter(text[end]) || text[end] == #char "'" || text[end] == #char "-") end += 1;
|
||||
latin_end = end;
|
||||
if end - i >= 3 {
|
||||
word := to_lower_copy(slice(text, i, end - i),, temp);
|
||||
if !array_find(STATS_STOP_WORDS, word) {
|
||||
value, added := find_or_add(words, word);
|
||||
value.* += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
stats_is_letter :: (c: u8) -> bool {
|
||||
return (c >= #char "a" && c <= #char "z") || (c >= #char "A" && c <= #char "Z");
|
||||
}
|
||||
|
||||
STATS_STOP_WORDS :: string.[
|
||||
"the", "and", "for", "you", "that", "with", "this", "are", "not", "was", "but", "have", "from", "your",
|
||||
"can", "all", "they", "will", "one", "has", "had", "his", "her", "she", "him", "its", "our", "out", "any",
|
||||
"who", "what", "when", "where", "which", "how", "why", "their", "them", "then", "than", "there", "these",
|
||||
"those", "also", "into", "just", "more", "some", "very", "would", "could", "should", "about", "been",
|
||||
"were", "being", "over", "only", "such", "other", "each", "most", "may", "use", "used", "using", "get",
|
||||
"got", "off", "per", "via", "yes", "new",
|
||||
];
|
||||
|
||||
stats_free_results :: () {
|
||||
s := *stats_ui;
|
||||
for s.years free(it.year,, context.default_allocator);
|
||||
for s.common_words free(it.word,, context.default_allocator);
|
||||
array_reset_keeping_memory(*s.years);
|
||||
array_reset_keeping_memory(*s.most_words);
|
||||
array_reset_keeping_memory(*s.common_words);
|
||||
s.captures, s.read, s.with_text = 0, 0, 0;
|
||||
s.lines, s.words, s.characters, s.cjk_characters, s.turned_lines, s.low_confidence_lines = 0, 0, 0, 0, 0, 0;
|
||||
s.confidence_sum = 0;
|
||||
s.words_median, s.words_p90, s.words_p99, s.words_max = 0, 0, 0, 0;
|
||||
for * s.buckets it.* = 0;
|
||||
}
|
||||
|
||||
// 874477 as "874,477". Temp.
|
||||
stats_number :: (n: s64) -> string {
|
||||
digits := tprint("%", abs(n));
|
||||
builder: String_Builder;
|
||||
builder.allocator = temp;
|
||||
if n < 0 append(*builder, "-");
|
||||
for i: 0..digits.count - 1 {
|
||||
if i > 0 && (digits.count - i) % 3 == 0 append(*builder, ",");
|
||||
append(*builder, digits[i]);
|
||||
}
|
||||
return builder_to_string(*builder,, temp);
|
||||
}
|
||||
|
||||
// "98.9%". Temp.
|
||||
stats_percent :: (part: s64, whole: s64) -> string {
|
||||
return join(tprint("%", FF(100.0 * part / max(whole, 1), 1)), "%",, temp);
|
||||
}
|
||||
@ -101,6 +101,8 @@ viewer: struct {
|
||||
visible_dirty := true;
|
||||
visible_rebuilt_at: Apollo_Time;
|
||||
catalog_loaded: s64; // entries put in place, or found unreadable
|
||||
catalog_generation: s64; // counts changes to catalog entries (loading, an OCR reading stored)
|
||||
visible_generation: s64; // counts rebuilds of visible
|
||||
sorted_by_date: bool;
|
||||
search_buffer: [256] u8;
|
||||
search_terms: [..] Search_Term;
|
||||
@ -390,6 +392,7 @@ viewer_apply_catalog_updates :: () {
|
||||
capture.catalog = it.entry;
|
||||
}
|
||||
}
|
||||
if updates.count viewer.catalog_generation += 1;
|
||||
if updates.count && viewer.search_terms.count viewer.visible_dirty = true;
|
||||
if !viewer.sorted_by_date && viewer.catalog_loaded >= viewer.captures.count {
|
||||
viewer.sorted_by_date = true;
|
||||
@ -401,11 +404,30 @@ viewer_apply_catalog_updates :: () {
|
||||
|
||||
// The captures in display order: by date, newest first, once every date is known (by name until
|
||||
// then), and only those matching the search.
|
||||
// Sets the search box to query and searches, as typing it would.
|
||||
viewer_set_search :: (query: string) {
|
||||
count := min(query.count, viewer.search_buffer.count - 1);
|
||||
memcpy(viewer.search_buffer.data, query.data, count);
|
||||
viewer.search_buffer[count] = 0;
|
||||
viewer_search_changed();
|
||||
}
|
||||
|
||||
viewer_search_changed :: () {
|
||||
query := to_string(viewer.search_buffer.data);
|
||||
terms := search_parse(query);
|
||||
for viewer.search_terms free(it.text);
|
||||
array_reset_keeping_memory(*viewer.search_terms);
|
||||
for terms array_add(*viewer.search_terms, .{it.field, copy_string(it.text)});
|
||||
viewer.visible_dirty = true;
|
||||
viewer.visible_rebuilt_at = .{}; // at once, not throttled
|
||||
}
|
||||
|
||||
viewer_rebuild_visible :: () {
|
||||
// While the catalog loads under a search, matches arrive every frame: a few rebuilds a second do.
|
||||
if viewer.search_terms.count && !viewer.sorted_by_date && to_float64_seconds(current_time_monotonic() - viewer.visible_rebuilt_at) < 0.25 return;
|
||||
viewer.visible_dirty = false;
|
||||
viewer.visible_rebuilt_at = current_time_monotonic();
|
||||
viewer.visible_generation += 1;
|
||||
|
||||
array_reset_keeping_memory(*viewer.visible);
|
||||
for viewer.captures {
|
||||
@ -526,13 +548,7 @@ viewer_ui :: () { using ImGui;
|
||||
|
||||
SetNextItemWidth(min(GetContentRegionAvail().x * 0.5, 520));
|
||||
if InputTextWithHint("##search", "Search text, titles, URLs, apps, notes. \"a phrase\" ocr: title: url: app: note:", viewer.search_buffer.data, viewer.search_buffer.count) {
|
||||
query := to_string(viewer.search_buffer.data);
|
||||
terms := search_parse(query);
|
||||
for viewer.search_terms free(it.text);
|
||||
array_reset_keeping_memory(*viewer.search_terms);
|
||||
for terms array_add(*viewer.search_terms, .{it.field, copy_string(it.text)});
|
||||
viewer.visible_dirty = true;
|
||||
viewer.visible_rebuilt_at = .{}; // at once, not throttled
|
||||
viewer_search_changed();
|
||||
}
|
||||
SameLine();
|
||||
if viewer.search_terms.count Text("% of % captures", viewer.visible.count, viewer.captures.count);
|
||||
@ -547,6 +563,8 @@ viewer_ui :: () { using ImGui;
|
||||
SameLine();
|
||||
// The OCR window (app/ocr_ui.jai): the model, GPU or CPU, and the library scan.
|
||||
if Button("OCR") ocr_ui.window_open = !ocr_ui.window_open;
|
||||
SameLine();
|
||||
if Button("Stats") stats_ui.window_open = !stats_ui.window_open; // app/stats_ui.jai
|
||||
if IsItemHovered() {
|
||||
engine: *Ocr_Engine;
|
||||
if !ocr_ui.thread_busy engine = ocr_ui.engine; // the background thread owns it while busy
|
||||
@ -556,9 +574,9 @@ viewer_ui :: () { using ImGui;
|
||||
|
||||
// A running OCR scan, as a thin bar under the search, so it shows with the OCR window closed.
|
||||
if ocr_ui.scan.running {
|
||||
done, total, failed, rate, left := ocr_scan_progress();
|
||||
done, total, failed, rate, left, elapsed := ocr_scan_progress();
|
||||
ProgressBar(cast(float) done / max(total, 1), .{-FLOAT32_MIN, max(2.0, GetFontSize() * 0.15)}, "");
|
||||
if IsItemHovered() SetTooltip("OCR scan with %: %, % left, % failed%", ocr_ui.scan.model, ocr_scan_progress_text(done, total), ocr_duration_text(left), failed, ifx ocr_ui.scan.paused then " (paused)" else "");
|
||||
if IsItemHovered() SetTooltip("OCR scan with %: %, % elapsed, % left, % failed%", ocr_ui.scan.model, ocr_scan_progress_text(done, total), ocr_duration_text(elapsed), ocr_duration_text(left), failed, ifx ocr_ui.scan.paused then " (paused)" else "");
|
||||
}
|
||||
|
||||
BeginChild("grid");
|
||||
|
||||
Loading…
Reference in New Issue
Block a user