[WIP] String analysis
This commit is contained in:
parent
0fc39896f2
commit
f5ba11c2b1
@ -2,7 +2,7 @@
|
||||
// To generate the intermediate to see how many lines are being compiled, in x64 Native Tools Command Prompt for VS2022 or whatever
|
||||
// cl /P /EP exe_main.cpp
|
||||
// tokei exe_main.i
|
||||
VERSION :: "0.1a";
|
||||
VERSION :: "0.2";
|
||||
|
||||
#run,stallable build_cpp_project();
|
||||
|
||||
@ -101,7 +101,7 @@ os_target: Operating_System_Tag = .WINDOWS;
|
||||
generate_meta_file :: (debug: bool) {
|
||||
sb: String_Builder;
|
||||
append(*sb, "#pragma once\n\n");
|
||||
print_to_builder(*sb, "constexpr const char* MUSA_LIB_VERSION = \"%\";\n", VERSION);
|
||||
print_to_builder(*sb, "const char* MUSA_LIB_VERSION = \"%\";\n", VERSION);
|
||||
print_to_builder(*sb, "#define BUILD_DEBUG %\n", cast(s32)debug);
|
||||
|
||||
print_to_builder(*sb, "#define OS_WINDOWS %\n", ifx os_target == .WINDOWS then 1 else 0);
|
||||
|
||||
@ -23,6 +23,7 @@
|
||||
#include "lib/third_party/dear-imgui/imgui_impl_dx11.h"
|
||||
#include "src/ImGui_Supplementary.cpp"
|
||||
|
||||
#include "src/String_Analysis.cpp"
|
||||
#include "src/explorer_main.cpp"
|
||||
#endif
|
||||
|
||||
|
||||
@ -182,8 +182,6 @@ void array_add (Array<T>& src, T new_item) {
|
||||
src.data[src.count] = new_item;
|
||||
|
||||
src.count += 1;
|
||||
// auto dst_ptr = &src.data[src.count-1];
|
||||
// memcpy(dst_ptr, &new_item, sizeof(T));
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
@ -194,6 +192,16 @@ s64 array_find (Array<T>& src, T item) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
bool array_add_if_unique (Array<T>& src, T new_item) {
|
||||
if (array_find(src, new_item) == -1) {
|
||||
array_add(src, new_item);
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void array_ordered_remove_by_index (Array<T>& src, s64 index) {
|
||||
Assert(index >= 0); Assert(index < src.count);
|
||||
|
||||
@ -3,6 +3,9 @@
|
||||
#define LANG_CPP 1
|
||||
#define BUILD_CONSOLE_INTERFACE BUILD_DEBUG
|
||||
|
||||
#include <stdio.h> // vsnprintf
|
||||
#include <cstdarg> // va_list, ...
|
||||
|
||||
#if ARCH_CPU_X64
|
||||
#include "CPU_X64.cpp"
|
||||
#define PLATFORM_MEMORY_PAGE_SIZE 4096
|
||||
@ -13,15 +16,12 @@
|
||||
#error "CPU not supported (yet)!"
|
||||
#endif
|
||||
|
||||
#include <stdio.h> // vsnprintf
|
||||
#include <cstdarg> // va_list, ...
|
||||
|
||||
|
||||
#if OS_WINDOWS
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#include <Windows.h>
|
||||
#undef ERROR // why...
|
||||
#undef NO_ERROR // ugh...
|
||||
#include <winioctl.h>
|
||||
#else
|
||||
#error "This configuration is NOT supported. Only Windows with MSVC is currently supported."
|
||||
#endif
|
||||
|
||||
@ -63,4 +63,4 @@ void temp_reset_keeping_memory() {
|
||||
void temp_reset() { // alias: reset_temporary_storage.
|
||||
Thread_Context* context = thread_context();
|
||||
arena_reset(context->temp, true);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,11 +1,21 @@
|
||||
#if OS_WINDOWS
|
||||
#include <intrin.h>
|
||||
|
||||
int cpu_max_feature_leaf () {
|
||||
int cpuInfo[4];
|
||||
|
||||
__cpuid(cpuInfo, 0);
|
||||
int maxLeaf = cpuInfo[0];
|
||||
|
||||
return maxLeaf;
|
||||
}
|
||||
|
||||
int CPU_Base_Frequency() {
|
||||
int cpuInfo[4] = {0};
|
||||
|
||||
// Call CPUID with EAX = 0x16 (Base CPU Frequency)
|
||||
__cpuid(cpuInfo, 0x16);
|
||||
if (cpu_max_feature_leaf() >= 0x16)
|
||||
__cpuid(cpuInfo, 0x16);
|
||||
|
||||
return cpuInfo[0];
|
||||
}
|
||||
|
||||
@ -4,7 +4,8 @@ enum class ErrorClass: s32 {
|
||||
NONE = 0, // should not be used, just to avoid a default value being assigned.
|
||||
WARNING = 1,
|
||||
ERROR = 2,
|
||||
FATAL = 3
|
||||
FATAL = 3,
|
||||
TODO = 4,
|
||||
};
|
||||
|
||||
// #downcasts to string
|
||||
@ -38,6 +39,10 @@ char* error_severity (ErrorClass severity) {
|
||||
case ErrorClass::FATAL: {
|
||||
return "[FATAL ERROR]";
|
||||
} break;
|
||||
case ErrorClass::TODO: {
|
||||
return "[TODO]";
|
||||
} break;
|
||||
|
||||
}
|
||||
return "";
|
||||
}
|
||||
@ -48,6 +53,9 @@ string to_string (Error* error) {
|
||||
return { error->count, error->data };
|
||||
}
|
||||
|
||||
#define log_todo(fmt, ...) \
|
||||
Log_Error_2(__FILE__, __FUNCTION__, __LINE__, ErrorClass::TODO, fmt, ##__VA_ARGS__)
|
||||
|
||||
#define log_fatal_error(fmt, ...) \
|
||||
Log_Error_2(__FILE__, __FUNCTION__, __LINE__, ErrorClass::FATAL, fmt, ##__VA_ARGS__)
|
||||
|
||||
@ -136,6 +144,7 @@ void push_error (Thread_Context* tctx, Error* new_error) {
|
||||
tctx->current_error = new_error;
|
||||
|
||||
switch (new_error->severity) {
|
||||
case ErrorClass::TODO:
|
||||
case ErrorClass::NONE:
|
||||
case ErrorClass::WARNING: {
|
||||
print(to_string(new_error));
|
||||
|
||||
52
lib/Base/Unicode.cpp
Normal file
52
lib/Base/Unicode.cpp
Normal file
@ -0,0 +1,52 @@
|
||||
constexpr u8 trailing_bytes_for_utf8[] = {
|
||||
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
|
||||
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
|
||||
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
|
||||
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
|
||||
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
|
||||
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
|
||||
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
|
||||
2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5};
|
||||
constexpr u32 UNI_REPLACEMENT_CHAR = 0x0000FFFD;
|
||||
constexpr u8 utf8_inital_byte_mask[6] = { 0x7F, 0x1F, 0x0F, 0x07, 0x03, 0x01 };
|
||||
constexpr u32 UNI_MAX_UTF32 = 0x7FFFFFFF;
|
||||
|
||||
bool character_utf8_to_utf32 (u8* data, s64 source_length, u32* utf32, s64* source_length_out) {
|
||||
u8 first_character = data[0];
|
||||
s32 continuation_bytes = trailing_bytes_for_utf8[first_character];
|
||||
|
||||
if ((continuation_bytes + 1) > source_length) {
|
||||
(*utf32) = UNI_REPLACEMENT_CHAR;
|
||||
(*source_length_out) = source_length;
|
||||
return false;
|
||||
}
|
||||
|
||||
u32 ch = data[0] & utf8_inital_byte_mask[continuation_bytes];
|
||||
|
||||
for (s64 i = 1; i < continuation_bytes + 1; i += 1) {
|
||||
ch = ch << 6;
|
||||
//if strict ...
|
||||
ch |= data[i] & 0x3F;
|
||||
}
|
||||
|
||||
// #if strict... {}
|
||||
(*utf32) = ch;
|
||||
(*source_length_out) = continuation_bytes + 1;
|
||||
|
||||
if (ch > UNI_MAX_UTF32) {
|
||||
(*utf32) = UNI_REPLACEMENT_CHAR;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool next_utf8_to_utf32 (string& s, u32* utf32_char_out) {
|
||||
s64 codepoint_source_length;
|
||||
bool success = character_utf8_to_utf32(s.data, s.count, utf32_char_out, &codepoint_source_length);
|
||||
|
||||
s.data += codepoint_source_length;
|
||||
s.count -= codepoint_source_length;
|
||||
|
||||
Assert(s.count >= 0);
|
||||
return success;
|
||||
}
|
||||
@ -1,5 +1,5 @@
|
||||
void run_pre_setup_tests() {
|
||||
// #no_context: context will not be initialized at this point.
|
||||
// #no_context: context will not be initialized at this point, so log() doesn't work
|
||||
printf("Running pre-setup tests...\n");
|
||||
printf("\nFinished running pre-setup tests...\n");
|
||||
}
|
||||
|
||||
@ -1,8 +1,9 @@
|
||||
// #TODO: #OS_Win32
|
||||
// [ ] #Exception handling code in `Win32_Exception_Filter`
|
||||
// [~] #Thread cleanup: in `thread_deinit` is there any requirement to cleanup child threads?
|
||||
// - I think no? Threads should handle their own lifetimes, and the parent threads should ensure child threads are complete
|
||||
// - I think: no? Threads should handle their own lifetimes, and the parent threads should ensure child threads are complete
|
||||
// before terminating.
|
||||
// Or we can move child threads up to the parent?
|
||||
|
||||
constexpr s64 FILETIME_TO_UNIX = 116444736000000000i64;
|
||||
f64 GetUnixTimestamp () {
|
||||
@ -34,7 +35,7 @@ string format_time_datetime (FILETIME ft) {
|
||||
SYSTEMTIME stUTC, st;
|
||||
FileTimeToSystemTime(&ft, &stUTC);
|
||||
SystemTimeToTzSpecificLocalTime(nullptr, &stUTC, &st);
|
||||
// YYYY-MM-DD-
|
||||
|
||||
return format_string("%04u-%02u-%02u %02u:%02u:%02u.%03u",
|
||||
st.wYear,
|
||||
st.wMonth,
|
||||
@ -252,7 +253,6 @@ internal void Win32_Entry_Point (int argc, WCHAR **argv) {
|
||||
// [ ] Get Working directory (info->working_path)
|
||||
// [ ] GetEnvironmentStringsW
|
||||
temp_reset();
|
||||
printf("Hello there!\n\n");
|
||||
}
|
||||
|
||||
C_LINKAGE DWORD OS_Windows_Thread_Entry_Point (void* parameter) {
|
||||
@ -1147,6 +1147,14 @@ void init (STFE_Results* results) {
|
||||
results->modtimes = arena_array_new<u64>(1024*1024*4, Arena_Reserve::Size_2G);
|
||||
}
|
||||
|
||||
void STFE_Results_Free (STFE_Results* results) {
|
||||
arena_array_free(*results->strings);
|
||||
arena_array_free(*results->offsets);
|
||||
arena_array_free(*results->lengths);
|
||||
arena_array_free(*results->sizes);
|
||||
arena_array_free(*results->modtimes);
|
||||
}
|
||||
|
||||
struct ST_File_Enumeration { // global state
|
||||
ArrayView<OS_Drive*> drives;
|
||||
Thread* master_thread;
|
||||
@ -1163,6 +1171,20 @@ struct ST_File_Enumeration { // global state
|
||||
|
||||
global ST_File_Enumeration* stfe;
|
||||
|
||||
void free_stfe_and_reset () {
|
||||
push_allocator(GPAllocator());
|
||||
|
||||
array_free(stfe->drives);
|
||||
|
||||
internal_free(stfe->master_thread);
|
||||
|
||||
STFE_Results_Free(&stfe->dirs);
|
||||
STFE_Results_Free(&stfe->files);
|
||||
|
||||
internal_free(stfe);
|
||||
stfe = nullptr; // final step
|
||||
}
|
||||
|
||||
string add_record (ST_File_Enumeration* stfe, string full_path, bool is_directory, WIN32_FIND_DATAW* find_data) {
|
||||
// return the string copy!
|
||||
if (is_directory) {
|
||||
@ -1257,7 +1279,8 @@ s64 win32_file_enum_thread_proc (Thread* thread) {
|
||||
init(&task->dirs);
|
||||
init(&task->files);
|
||||
|
||||
// while files available?
|
||||
// Allocates to thread_context->arena, which is cleaned up
|
||||
// when the thread completes. see: thread_deinit
|
||||
Array<string> paths_to_enumerate;
|
||||
|
||||
for_each(d, task->drives) {
|
||||
@ -1441,7 +1464,6 @@ void Win32_Enable_USN_Journal_Monitoring (ArrayView<OS_Drive*> drives) {
|
||||
}
|
||||
}
|
||||
|
||||
#include <winioctl.h>
|
||||
void Query_USN_Journal (ArrayView<OS_Drive*> drives) {
|
||||
Win32_Enable_USN_Journal_Monitoring(drives);
|
||||
for_each(d, drives) {
|
||||
|
||||
334
lib/OS/OS_Win32_File_Enumeration.cpp
Normal file
334
lib/OS/OS_Win32_File_Enumeration.cpp
Normal file
@ -0,0 +1,334 @@
|
||||
struct Parent_Index {
|
||||
s32 thread_index; // group->worker_info[thread_index].thread
|
||||
s32 parent_index; // index into d_*offsets/lengths/etc.
|
||||
};
|
||||
|
||||
struct File_Enumeration_Thread_Results { // #userdata
|
||||
Arena* arena; // for strings
|
||||
// Directories
|
||||
ArenaArray<u32>* d_offsets;
|
||||
ArenaArray<s16>* d_lengths;
|
||||
ArenaArray<Parent_Index>* d_parent_indices;
|
||||
ArenaArray<u64>* d_modtime;
|
||||
// s64 dirs_enumerated = 0;
|
||||
// Files
|
||||
ArenaArray<u32>* f_offsets;
|
||||
ArenaArray<s16>* f_lengths;
|
||||
ArenaArray<Parent_Index>* f_parent_indices;
|
||||
ArenaArray<u64>* f_sizes;
|
||||
ArenaArray<u64>* f_modtime;
|
||||
};
|
||||
|
||||
void initialize (File_Enumeration_Thread_Results* fcr) { // Preallocate for 2^22 files:
|
||||
fcr->arena = next_arena(Arena_Reserve::Size_2G);
|
||||
|
||||
fcr->d_offsets = arena_array_new<u32>(4194304, Arena_Reserve::Size_2G);
|
||||
fcr->d_lengths = arena_array_new<s16>(4194304, Arena_Reserve::Size_2G);
|
||||
fcr->d_parent_indices = arena_array_new<Parent_Index>(4194304, Arena_Reserve::Size_2G);
|
||||
fcr->d_modtime = arena_array_new<u64>(4194304, Arena_Reserve::Size_2G);
|
||||
|
||||
fcr->f_offsets = arena_array_new<u32>(4194304, Arena_Reserve::Size_2G);
|
||||
fcr->f_lengths = arena_array_new<s16>(4194304, Arena_Reserve::Size_2G);
|
||||
fcr->f_parent_indices = arena_array_new<Parent_Index>(4194304, Arena_Reserve::Size_2G);
|
||||
fcr->f_sizes = arena_array_new<u64>(4194304, Arena_Reserve::Size_2G);
|
||||
fcr->f_modtime = arena_array_new<u64>(4194304, Arena_Reserve::Size_2G);
|
||||
}
|
||||
|
||||
struct Enumeration_Work {
|
||||
string first_directory;
|
||||
Parent_Index parent;
|
||||
bool is_root = false;
|
||||
Array<Enumeration_Work*> next;
|
||||
};
|
||||
|
||||
struct Drive_Enumeration { // master thread struct
|
||||
ArrayView<OS_Drive*> drives;
|
||||
Thread* master_thread;
|
||||
|
||||
s32 thread_count;
|
||||
bool thread_started;
|
||||
bool thread_completed;
|
||||
|
||||
// Files_Combined_Results paths;
|
||||
// Files_Combined_Results files;
|
||||
|
||||
s32 work_added = 0;
|
||||
s32 work_completed = 0;
|
||||
};
|
||||
|
||||
// void push_root (Drive_Enumeration* de, string label, s32 index) {
|
||||
// array_add(*de->paths.name, label);
|
||||
// array_add(*de->paths.parent_indices, index);
|
||||
// array_add(*de->paths.sizes, (u64)0);
|
||||
// array_add(*de->paths.modtime, (u64)0);
|
||||
// }
|
||||
|
||||
global Drive_Enumeration* drive_enumeration;
|
||||
|
||||
// File_Enumeration_Thread_Results* results_from_thread_index (Thread_Group* group, s32 thread_index) {
|
||||
// return ;
|
||||
// }
|
||||
|
||||
string path_from_parent_index (Thread_Group* group, Parent_Index pid, Parent_Index* next_pid) {
|
||||
if (pid.parent_index == -1) return "";
|
||||
|
||||
auto results = (File_Enumeration_Thread_Results*)group->worker_info[pid.thread_index].thread.context->userdata;
|
||||
u8* offset = (results->arena->memory_base + (*results->d_offsets)[pid.parent_index]);
|
||||
u32 length = (*results->d_lengths)[pid.parent_index];
|
||||
|
||||
(*next_pid) = (*results->d_parent_indices)[pid.parent_index];
|
||||
|
||||
return {(s64)length, offset};
|
||||
}
|
||||
|
||||
// This is much stupider and more complicated than I would like, unfortunately.
|
||||
string directory_get_full_path (Thread_Group* group, Parent_Index pid, string dir_name) {
|
||||
push_allocator(GPAllocator()); // to copy from String_Builder
|
||||
|
||||
Array<string> paths;
|
||||
paths.allocator = temp();
|
||||
|
||||
Parent_Index this_pid = pid;
|
||||
Parent_Index next_pid = {};
|
||||
string parent_dir = path_from_parent_index(group, this_pid, &next_pid);
|
||||
array_add(paths, parent_dir);
|
||||
|
||||
this_pid = next_pid;
|
||||
next_pid = {};
|
||||
|
||||
while (this_pid.parent_index != -1) {
|
||||
parent_dir = path_from_parent_index(group, this_pid, &next_pid);
|
||||
array_add(paths, parent_dir);
|
||||
|
||||
this_pid = next_pid;
|
||||
next_pid = {};
|
||||
}
|
||||
|
||||
// go in reverse order and add together string
|
||||
String_Builder* sb = new_string_builder(Arena_Reserve::Size_64K);
|
||||
for (s64 i = paths.count-1; i >= 0; i -= 1) {
|
||||
append(sb, paths[i]);
|
||||
append(sb, "\\");
|
||||
}
|
||||
append(sb, dir_name);
|
||||
|
||||
return builder_to_string(sb);
|
||||
}
|
||||
|
||||
/*void update_results (Drive_Enumeration* de, Enumeration_Work* ew) {
|
||||
// merge results and release resources!
|
||||
// unfortunately this is a LOT of copying!
|
||||
for_each(i, (*ew->d_offsets)) {
|
||||
u8* string_ptr = (ew->thread_arena->memory_base + (*ew->d_offsets)[i]);
|
||||
string name = {(*ew->d_lengths)[i], string_ptr};
|
||||
array_add(*de->paths.name, name);
|
||||
array_add(*de->paths.parent_indices, (*ew->d_parent_indices)[i]);
|
||||
array_add(*de->paths.sizes, (*ew->d_sizes)[i]);
|
||||
array_add(*de->paths.modtime, (*ew->d_modtime)[i]);
|
||||
}
|
||||
for_each(i, (*ew->offsets)) {
|
||||
u8* string_ptr = (ew->thread_arena->memory_base + (*ew->offsets)[i]);
|
||||
string name = {(*ew->lengths)[i], string_ptr};
|
||||
array_add(*de->files.name, name);
|
||||
array_add(*de->files.parent_indices, (*ew->parent_indices)[i]);
|
||||
array_add(*de->files.sizes, (*ew->sizes)[i]);
|
||||
array_add(*de->files.modtime, (*ew->modtime)[i]);
|
||||
}
|
||||
}*/
|
||||
|
||||
void add_record (File_Enumeration_Thread_Results* results,
|
||||
WIN32_FIND_DATAW* find_data,
|
||||
string name,
|
||||
Parent_Index parent_index,
|
||||
bool is_directory) {
|
||||
u32 offset = (u32)(name.data - results->arena->memory_base);
|
||||
u64 size = ((u64)find_data->nFileSizeHigh << 32) | ((u64)find_data->nFileSizeLow & 0xFFFFFFFF);
|
||||
u64 modtime = FILETIME_to_ticks(find_data->ftLastWriteTime);
|
||||
|
||||
if (is_directory) {
|
||||
array_add((*results->d_offsets), offset);
|
||||
array_add((*results->d_lengths), (s16)name.count);
|
||||
array_add((*results->d_parent_indices), parent_index); // #parent_index
|
||||
array_add((*results->d_modtime), modtime);
|
||||
} else {
|
||||
array_add((*results->f_offsets), offset);
|
||||
array_add((*results->f_lengths), (s16)name.count);
|
||||
array_add((*results->f_parent_indices), parent_index); // #parent_index
|
||||
array_add((*results->f_sizes), size);
|
||||
array_add((*results->f_modtime), modtime);
|
||||
}
|
||||
}
|
||||
|
||||
Thread_Continue_Status file_enumeration_thread_group_proc (Thread_Group* group, Thread* thread, void* work) {
|
||||
// 1. setup userdata as an Arena*:
|
||||
// #TODO: replace userdata with a struct that manages the thread-local data for this
|
||||
// particular problem. This data can be rescued before we
|
||||
File_Enumeration_Thread_Results* results;
|
||||
if (!thread->context->userdata) {
|
||||
thread->context->userdata = New<File_Enumeration_Thread_Results>(GPAllocator());
|
||||
initialize((File_Enumeration_Thread_Results*)thread->context->userdata);
|
||||
}
|
||||
results = (File_Enumeration_Thread_Results*)thread->context->userdata;
|
||||
|
||||
Enumeration_Work* enum_work = (Enumeration_Work*)work;
|
||||
|
||||
// Validate thread context?
|
||||
push_allocator(temp());
|
||||
auto_release_temp();
|
||||
|
||||
// log("file_enumeration_thread_group_proc, thread index: %d", thread->index);
|
||||
|
||||
// MAKE SURE PATH IS NULL TERMINATED!
|
||||
wstring wildcard_name = utf8_to_wide(format_string("%s\\*", enum_work->first_directory.data)); // #temp
|
||||
WIN32_FIND_DATAW find_data;
|
||||
HANDLE h = FindFirstFileExW((LPCWSTR)wildcard_name.data, FindExInfoBasic, &find_data,
|
||||
FindExSearchNameMatch, nullptr, FIND_FIRST_EX_LARGE_FETCH);
|
||||
if (h == INVALID_HANDLE_VALUE) {
|
||||
return Thread_Continue_Status::CONTINUE;
|
||||
}
|
||||
|
||||
s32 thread_index = get_thread_index(group, (s32)thread->index); // zero-indexed to thread group
|
||||
|
||||
Parent_Index pi = enum_work->parent;
|
||||
|
||||
push_arena(results->arena);
|
||||
if (enum_work->is_root) { // see add_record
|
||||
string name = copy_string(enum_work->first_directory);
|
||||
u32 offset = (u32)(name.data - results->arena->memory_base);
|
||||
s32 current_index = (s32)(*results->d_offsets).count;
|
||||
pi = {thread_index, current_index};
|
||||
array_add((*results->d_offsets), offset);
|
||||
array_add((*results->d_lengths), (s16)name.count);
|
||||
Parent_Index root_pi = {thread_index, -1};
|
||||
array_add((*results->d_parent_indices), root_pi); // #parent_index
|
||||
array_add((*results->d_modtime), (u64)0);
|
||||
// results->dirs_enumerated += 1;
|
||||
}
|
||||
|
||||
while (true) {
|
||||
string name = wide_to_utf8((u16*)find_data.cFileName);
|
||||
bool should_continue = (name.count == 0 || name == "." || name == "..");
|
||||
if (should_continue) {
|
||||
bool success = FindNextFileW(h, &find_data);
|
||||
if (!success)
|
||||
break;
|
||||
continue;
|
||||
}
|
||||
|
||||
bool is_directory = (find_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0;
|
||||
Parent_Index next_index = {thread_index, enum_work->parent.parent_index};
|
||||
add_record(results, &find_data, name, next_index, is_directory);
|
||||
|
||||
if (is_directory) {
|
||||
push_allocator(GPAllocator());
|
||||
auto new_work = New<Enumeration_Work>(false);
|
||||
new_work->first_directory = directory_get_full_path(group, pi, name);
|
||||
new_work->parent = next_index;
|
||||
new_work->is_root = false;
|
||||
new_work->next = {};
|
||||
|
||||
array_add(enum_work->next, new_work);
|
||||
}
|
||||
|
||||
bool success = FindNextFileW(h, &find_data);
|
||||
if (!success) break;
|
||||
}
|
||||
|
||||
FindClose(h);
|
||||
return Thread_Continue_Status::CONTINUE;
|
||||
}
|
||||
|
||||
s64 multithreaded_file_enumeration_master_proc (Thread* thread) {
|
||||
auto task = thread_task(Drive_Enumeration);
|
||||
|
||||
Thread_Group* file_enum_thread_group = New<Thread_Group>();
|
||||
|
||||
s32 thread_count = os_cpu_physical_core_count();
|
||||
|
||||
push_allocator(GPAllocator());
|
||||
thread_group_init(file_enum_thread_group, thread_count, file_enumeration_thread_group_proc, true);
|
||||
|
||||
|
||||
for_each(d, task->drives) {
|
||||
auto work = New<Enumeration_Work>(GPAllocator(), false); //replace with arena bootstrap?
|
||||
work->first_directory = task->drives[d]->label; // this includes the colon-slash, (e.g. `C:\`).
|
||||
work->parent = {-1, -1}; // #HACK: (s32)d
|
||||
work->is_root = true;
|
||||
work->next = {};
|
||||
|
||||
add_work(file_enum_thread_group, work);
|
||||
task->work_added += 1;
|
||||
}
|
||||
|
||||
start(file_enum_thread_group);
|
||||
// set task completed.
|
||||
|
||||
s64 path_index = task->drives.count;
|
||||
|
||||
while (true) {
|
||||
auto_release_temp();
|
||||
ArrayView<void*> cw = get_completed_work(file_enum_thread_group);
|
||||
task->work_completed += (s32)cw.count;
|
||||
for_each(i, cw) {
|
||||
auto ew = (Enumeration_Work*)cw[i];
|
||||
|
||||
for_each(w, ew->next) {
|
||||
auto new_work = ew->next[w];
|
||||
|
||||
add_work(file_enum_thread_group, new_work);
|
||||
}
|
||||
task->work_added += (s32)ew->next.count;
|
||||
|
||||
string_free(ew->first_directory);
|
||||
array_free(ew->next);
|
||||
internal_free(ew);
|
||||
}
|
||||
|
||||
log("work completed: %d/%d",task->work_completed, task->work_added);
|
||||
// if (task->work_completed >= task->work_added) break;
|
||||
Sleep(1);
|
||||
}
|
||||
|
||||
shutdown(file_enum_thread_group);
|
||||
|
||||
task->thread_completed = true;
|
||||
return 0;
|
||||
}
|
||||
|
||||
void os_run_file_enumeration_multithreaded () {
|
||||
// Need some struct to track the state of this operation.
|
||||
drive_enumeration = New<Drive_Enumeration>();
|
||||
(*drive_enumeration) = {
|
||||
os_get_available_drives(),
|
||||
New<Thread>(),
|
||||
os_cpu_physical_core_count(),
|
||||
0, 0, 0, 0
|
||||
};
|
||||
|
||||
// initialize(&drive_enumeration->paths);
|
||||
// initialize(&drive_enumeration->files);
|
||||
|
||||
// We start 1 thread to run the thread group and track the threads
|
||||
string thread_name = "Multithreaded Enumeration: Master Thread";
|
||||
bool success = thread_init(drive_enumeration->master_thread,
|
||||
multithreaded_file_enumeration_master_proc, thread_name);
|
||||
Assert(success);
|
||||
thread_start(drive_enumeration->master_thread, drive_enumeration);
|
||||
drive_enumeration->thread_started = true;
|
||||
}
|
||||
|
||||
bool file_enum_multithreading_started () {
|
||||
if (drive_enumeration == nullptr) return false;
|
||||
return drive_enumeration->thread_started;
|
||||
}
|
||||
|
||||
bool file_enum_multithreading_active () {
|
||||
if (drive_enumeration == nullptr) return false;
|
||||
if (drive_enumeration->thread_completed) {
|
||||
return false;
|
||||
}
|
||||
if (drive_enumeration->thread_started) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
@ -63,7 +63,7 @@ struct NTFS_ResidentAttributeHeader : NTFS_AttributeHeader {
|
||||
};
|
||||
|
||||
struct NTFS_FileNameAttributeHeader : NTFS_ResidentAttributeHeader {
|
||||
u64 parentRecordNumber : 48;
|
||||
u64 parentRecordNumber : 48; // low 48 bits
|
||||
u64 sequenceNumber : 16;
|
||||
u64 creationTime;
|
||||
u64 modificationTime;
|
||||
@ -97,7 +97,7 @@ struct NTFS_RunHeader {
|
||||
#pragma pack(pop)
|
||||
|
||||
struct NTFS_File {
|
||||
u32 parent_id;
|
||||
u32 parent_id; // #TODO: FRNs should be 64-bit!
|
||||
u32 record_id;
|
||||
u16* name_data;
|
||||
u64 file_modtime; // FILETIME
|
||||
@ -585,8 +585,32 @@ bool Serialize_Win32_Drives (ArrayView<Win32_Drive*> drives, string file_path) {
|
||||
return true;
|
||||
}
|
||||
|
||||
void ntfs_create_enumeration_threads (s32 thread_count) {
|
||||
if (!ex1_ntfs.initialized) { Timed_Block_Print("Thread initialization (ntfs)");
|
||||
ex1_ntfs.initialized = true;
|
||||
ex1_ntfs.threads = ArrayView<Thread>(thread_count);
|
||||
ex1_ntfs.threads_in_flight.allocator = GPAllocator();
|
||||
for_each(t, ex1_ntfs.threads) {
|
||||
string thread_name = format_string("ntfs_enumeration_thread#%d", t);
|
||||
bool success = thread_init(&ex1_ntfs.threads[t], ntfs_enumeration_thread_proc, thread_name);
|
||||
Assert(success);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
void Ex1_show_ntfs_workspace () { using namespace ImGui;
|
||||
|
||||
SliderInt("Select path index", &ex1w.path_select, 0, count_paths(stfe)-1);
|
||||
Text("%s", get_path_copy(stfe, ex1w.path_select).data);
|
||||
Text("time modified: %s", format_time_datetime(get_path_modtime(stfe, ex1w.path_select)).data);
|
||||
// #TODO: modtime (to indextime)
|
||||
SliderInt("Select file index", &ex1w.file_select, 0, count_files(stfe)-1);
|
||||
Text("%s", get_file_copy(stfe, ex1w.file_select).data);
|
||||
Text("size: %s", format_bytes(get_file_size_bytes(stfe, ex1w.file_select)).data);
|
||||
Text("time modified: %s", format_time_datetime(get_file_modtime(stfe, ex1w.file_select)).data);
|
||||
|
||||
push_allocator(temp());
|
||||
for_each(d, ntfs_workspace.drives) {
|
||||
OS_Drive* drive = ntfs_workspace.drives[d];
|
||||
@ -633,4 +657,111 @@ void Ex1_show_ntfs_workspace () { using namespace ImGui;
|
||||
}
|
||||
}
|
||||
}
|
||||
*/
|
||||
*/
|
||||
/*SeparatorText("ex1_ntfs");
|
||||
Text("Threads in flight count: %d", ex1_ntfs.threads_in_flight.count);
|
||||
for_each(i, ex1_ntfs.threads) {
|
||||
Text(" [%d] initialized: %d, has_context: %d, has_data: %d",
|
||||
i, ex1_ntfs.threads[i].proc != nullptr, ex1_ntfs.threads[i].context != nullptr, ex1_ntfs.threads[i].data != nullptr);
|
||||
}*/
|
||||
/*// #NTFS_MFT_RAW
|
||||
push_allocator(GPAllocator());
|
||||
Array<ArrayView<OS_Drive*>> drive_split;
|
||||
drive_split.allocator = temp(); // this is only needed for this frame
|
||||
|
||||
if (drives.count > os_cpu_physical_core_count()) {
|
||||
s32 thread_count = os_cpu_physical_core_count();
|
||||
array_resize(drive_split, thread_count);
|
||||
ntfs_create_enumeration_threads(thread_count);
|
||||
|
||||
s32 threads_to_create = thread_count;
|
||||
s64 drives_per_thread = (drives.count / thread_count);
|
||||
s64 remainder = drives.count % thread_count;
|
||||
s64 current_drive = 0;
|
||||
|
||||
for_each(d, drive_split) {
|
||||
if (d == drive_split.count) {
|
||||
drive_split[d] = ArrayView<OS_Drive*>(remainder);
|
||||
} else {
|
||||
drive_split[d] = ArrayView<OS_Drive*>(drives_per_thread);
|
||||
}
|
||||
|
||||
for (s64 i = 0; i < drive_split[d].count; i += 1) {
|
||||
drive_split[d][i] = drives[current_drive];
|
||||
current_drive += 1;
|
||||
}
|
||||
}
|
||||
|
||||
debug_break(); // #TODO: Check that the work has been distributed correctly.
|
||||
} else { // more threads than drives, or same amount
|
||||
s32 thread_count = (s32)drives.count;
|
||||
array_resize(drive_split, drives.count);
|
||||
ntfs_create_enumeration_threads(thread_count);
|
||||
|
||||
for_each(d, drives) {
|
||||
auto drive = drives[d];
|
||||
drive_split[d] = ArrayView<OS_Drive*>(1); // Arrays of size one are sad :pensive:
|
||||
|
||||
drive_split[d][0] = drive;
|
||||
}
|
||||
}
|
||||
|
||||
s64 active_thread_count = drive_split.count;
|
||||
|
||||
ex1_ntfs.threads_started = true;
|
||||
for (s64 t = 0; t < active_thread_count; t += 1) {
|
||||
Thread* thread = &ex1_ntfs.threads[t];
|
||||
Arena* thread_arena = next_arena(Arena_Reserve::Size_64K);
|
||||
push_arena(thread_arena);
|
||||
auto thread_data = New<NTFS_Enumeration_Task>();
|
||||
thread_data->pool = thread_arena;
|
||||
thread_data->drives = drive_split[t];
|
||||
thread_start(thread, thread_data);
|
||||
array_add(ex1_ntfs.threads_in_flight, thread);
|
||||
}*/
|
||||
|
||||
|
||||
/* #NTFS_MFT_RAW
|
||||
if (ex1_ntfs.threads_in_flight.count) {
|
||||
Text("Threads in flight: %d", ex1_ntfs.threads_in_flight.count);
|
||||
|
||||
for_each(t, ex1_ntfs.threads_in_flight) {
|
||||
if (thread_is_done(ex1_ntfs.threads_in_flight[t])) {
|
||||
push_allocator(GPAllocator());
|
||||
|
||||
Thread* thread = ex1_ntfs.threads_in_flight[t];
|
||||
auto task = thread_task(NTFS_Enumeration_Task);
|
||||
array_free(task->drives);
|
||||
|
||||
// make sure to retreive any data you need to from here!
|
||||
release_arena(task->pool);
|
||||
|
||||
thread_deinit(ex1_ntfs.threads_in_flight[t], false);
|
||||
array_unordered_remove_by_index(ex1_ntfs.threads_in_flight, t);
|
||||
t -= 1; // check this element index again!
|
||||
}
|
||||
}
|
||||
}*/
|
||||
|
||||
/* #NTFS_MFT_RAW
|
||||
if (ex1_ntfs.threads_started && !ex1_ntfs.threads_in_flight.count) {
|
||||
// All threads are complete, we're free to clean up remaining memory
|
||||
push_allocator(GPAllocator());
|
||||
array_free(ex1_ntfs.threads);
|
||||
array_free(ex1_ntfs.threads_in_flight);
|
||||
|
||||
// Instead maybe we should just memset this to zero.
|
||||
reset_struct(&ex1_ntfs);
|
||||
}
|
||||
|
||||
// How do I tell when all files are enumerated?
|
||||
// check drives[i]->data.paths.wstrings.count count?
|
||||
if (all_drives_enumerated && Button("Save drive data")) {
|
||||
string file_path = format_string_temp("%s_DriveData.bin", os_get_machine_name().data);
|
||||
bool success = Serialize_Win32_Drives(drives, file_path);
|
||||
if (!success) { log_error("Failed to save Win32_Drive data"); }
|
||||
}
|
||||
|
||||
if (all_drives_enumerated && Button("Clear all drive data")) {
|
||||
os_clear_drive_data();
|
||||
}*/
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
constexpr const char* MUSA_LIB_VERSION = "0.1a";
|
||||
const char* MUSA_LIB_VERSION = "0.2";
|
||||
#define BUILD_DEBUG 1
|
||||
#define OS_WINDOWS 1
|
||||
#define OS_LINUX 0
|
||||
|
||||
@ -36,6 +36,7 @@
|
||||
|
||||
#include "lib/Base/Arena.cpp"
|
||||
#include "lib/Base/String.cpp"
|
||||
#include "lib/Base/Unicode.cpp"
|
||||
#include "lib/Base/RadixSort.cpp"
|
||||
|
||||
#include "lib/Base/Base_Thread_Context.cpp"
|
||||
@ -53,7 +54,7 @@
|
||||
|
||||
#if OS_WINDOWS
|
||||
# include "lib/OS/OS_Win32.cpp"
|
||||
# include "lib/OS/OS_Win32_NTFS.cpp"
|
||||
// # include "lib/OS/OS_Win32_NTFS.cpp"
|
||||
#endif
|
||||
|
||||
#include "lib/Graphics.cpp"
|
||||
|
||||
@ -17,7 +17,9 @@ internal void Main_Entry_Point (int argc, WCHAR **argv);
|
||||
#endif
|
||||
|
||||
internal void Main_Entry_Point (int argc, WCHAR **argv) { // #entry_point
|
||||
set_cpu_base_frequency(4000); // REQUIRED FOR TIMING MODULE! will depend on CPU
|
||||
// #TODO: Check if base frequency is even available.
|
||||
u32 base_frequency = (u32)CPU_Base_Frequency();
|
||||
set_cpu_base_frequency(base_frequency); // REQUIRED FOR TIMING MODULE! will depend on CPU
|
||||
|
||||
#if BASE_RUN_TESTS
|
||||
run_pre_setup_tests(); // #no_context: context will not be initialized at this point.
|
||||
|
||||
235
src/Ex1.cpp
235
src/Ex1.cpp
@ -1,6 +1,6 @@
|
||||
struct ExplorerUI {
|
||||
u8 search_input[64];
|
||||
u8 secondary_input[64];
|
||||
// u8 search_input[64];
|
||||
// u8 secondary_input[64];
|
||||
};
|
||||
|
||||
struct Explorer {
|
||||
@ -22,19 +22,6 @@ global ExplorerUI explorer_ui;
|
||||
global Explorer explorer;
|
||||
global Ex1_NTFS_Enumeration ex1_ntfs;
|
||||
|
||||
void ntfs_create_enumeration_threads (s32 thread_count) {
|
||||
if (!ex1_ntfs.initialized) { Timed_Block_Print("Thread initialization (ntfs)");
|
||||
ex1_ntfs.initialized = true;
|
||||
ex1_ntfs.threads = ArrayView<Thread>(thread_count);
|
||||
ex1_ntfs.threads_in_flight.allocator = GPAllocator();
|
||||
for_each(t, ex1_ntfs.threads) {
|
||||
string thread_name = format_string("ntfs_enumeration_thread#%d", t);
|
||||
bool success = thread_init(&ex1_ntfs.threads[t], ntfs_enumeration_thread_proc, thread_name);
|
||||
Assert(success);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#define HOTKEY_ID_BRING_TO_FOREGROUND 1
|
||||
#define VK_SPACE_KEY_CODE 0x20
|
||||
// #define HOTKEY_ID_HIDE_TITLEBAR
|
||||
@ -91,8 +78,26 @@ struct Ex1_Workspace {
|
||||
ArrayView<string> files_sorted_by_modtime;
|
||||
};
|
||||
|
||||
global Ex1_Workspace ex1w;
|
||||
|
||||
void free_ex1_workspace_and_reset () {
|
||||
if (ex1w.sort_completed) {
|
||||
push_allocator(GPAllocator());
|
||||
|
||||
radix_sort_free(&ex1w.file_size_radix);
|
||||
radix_sort_free(&ex1w.file_modtime_radix);
|
||||
radix_sort_free(&ex1w.dir_modtime_radix);
|
||||
|
||||
array_free(ex1w.files_sorted_by_size);
|
||||
array_free(ex1w.files_sorted_by_modtime);
|
||||
|
||||
zero_struct(&ex1w);
|
||||
}
|
||||
}
|
||||
|
||||
// #TODO: Move all sort stuff to OS_Win32?
|
||||
// Make a general version of this that takes two ArrayView<T> and reorders.
|
||||
// There's no need to do this until we have the filtered results.
|
||||
void os_win32_reorder_files_by_radix (RadixSort* r, ArrayView<string>* files, bool reverse_order=false) {
|
||||
Timed_Block_Print("os_win32_reorder_files_by_radix");
|
||||
// Where are my source files!?
|
||||
@ -103,24 +108,10 @@ void os_win32_reorder_files_by_radix (RadixSort* r, ArrayView<string>* files, bo
|
||||
}
|
||||
}
|
||||
|
||||
global Ex1_Workspace ex1w;
|
||||
|
||||
void Ex1_show_enumeration_workspace () { using namespace ImGui;
|
||||
push_imgui_window("Enumerated Data Workspace");
|
||||
|
||||
/*
|
||||
SliderInt("Select path index", &ex1w.path_select, 0, count_paths(stfe)-1);
|
||||
Text("%s", get_path_copy(stfe, ex1w.path_select).data);
|
||||
Text("time modified: %s", format_time_datetime(get_path_modtime(stfe, ex1w.path_select)).data);
|
||||
// #TODO: modtime (to indextime)
|
||||
SliderInt("Select file index", &ex1w.file_select, 0, count_files(stfe)-1);
|
||||
Text("%s", get_file_copy(stfe, ex1w.file_select).data);
|
||||
Text("size: %s", format_bytes(get_file_size_bytes(stfe, ex1w.file_select)).data);
|
||||
Text("time modified: %s", format_time_datetime(get_file_modtime(stfe, ex1w.file_select)).data);
|
||||
*/
|
||||
// #TODO: size, modtime
|
||||
|
||||
if (!ex1w.sort_completed || Button("sort file sizes")) {
|
||||
if (!ex1w.sort_completed) {
|
||||
push_allocator(GPAllocator());
|
||||
Timed_Block_Print("radix_sort_u64: file sizes, file modtimes, directory modtimes");
|
||||
ArrayView<u64> sizes = to_view(*stfe->files.sizes);
|
||||
@ -136,24 +127,43 @@ void Ex1_show_enumeration_workspace () { using namespace ImGui;
|
||||
ex1w.sort_completed = true;
|
||||
}
|
||||
|
||||
if (ex1w.sort_completed) {
|
||||
SeparatorText("Files ordered by modtime");
|
||||
s32 file_count = (s32)ex1w.files_sorted_by_modtime.count;
|
||||
SliderInt("Select file index", &ex1w.file_select, 0, file_count-1);
|
||||
string file_name = copy_string(ex1w.files_sorted_by_modtime[ex1w.file_select]);
|
||||
Text("%s", file_name.data);
|
||||
u32 radix_index = rank(&ex1w.file_modtime_radix, ex1w.file_select);
|
||||
Text("date modified: %s", format_time_datetime(get_file_modtime(stfe, radix_index)).data);
|
||||
if (!ex1w.sort_completed) { return; }
|
||||
|
||||
SeparatorText("Files ordered by modtime");
|
||||
s32 file_count = (s32)ex1w.files_sorted_by_modtime.count;
|
||||
SliderInt("Select file index", &ex1w.file_select, 0, file_count-1);
|
||||
string file_name = copy_string(ex1w.files_sorted_by_modtime[ex1w.file_select]);
|
||||
Text("%s", file_name.data);
|
||||
u32 radix_index = rank(&ex1w.file_modtime_radix, ex1w.file_select);
|
||||
Text("date modified: %s", format_time_datetime(get_file_modtime(stfe, radix_index)).data);
|
||||
|
||||
SeparatorText("Files ordered by size");
|
||||
file_count = (s32)ex1w.files_sorted_by_size.count;
|
||||
// SliderInt("Select file index", &ex1w.file_select, 0, file_count-1);
|
||||
file_name = copy_string(ex1w.files_sorted_by_size[ex1w.file_select]);
|
||||
Text("%s", file_name.data);
|
||||
radix_index = rank(&ex1w.file_size_radix, ex1w.file_select);
|
||||
Text("size: %s", format_bytes(get_file_size_bytes(stfe, radix_index)).data);
|
||||
// Text("date modified: %s", format_time_datetime(get_file_modtime(stfe, radix_index)).data);
|
||||
|
||||
if (Button("Count unique UTF-8 characters")) {
|
||||
count_unique_utf8_chars();
|
||||
}
|
||||
if (ex1w.sort_completed) {
|
||||
SeparatorText("Files ordered by size");
|
||||
s32 file_count = (s32)ex1w.files_sorted_by_size.count;
|
||||
// SliderInt("Select file index", &ex1w.file_select, 0, file_count-1);
|
||||
string file_name = copy_string(ex1w.files_sorted_by_size[ex1w.file_select]);
|
||||
Text("%s", file_name.data);
|
||||
u32 radix_index = rank(&ex1w.file_size_radix, ex1w.file_select);
|
||||
Text("size: %s", format_bytes(get_file_size_bytes(stfe, radix_index)).data);
|
||||
// Text("date modified: %s", format_time_datetime(get_file_modtime(stfe, radix_index)).data);
|
||||
|
||||
Text("unique_codepoints_utf32.count: %", unique_codepoints_utf32.count);
|
||||
|
||||
for_each(u, unique_codepoints_utf32) {
|
||||
Text("[%d] Code point as hex: 0x%X", u, unique_codepoints_utf32[u]);
|
||||
}
|
||||
|
||||
Text("files_sorted_by_size size in bytes: %lld", ex1w.files_sorted_by_size.count * sizeof(string));
|
||||
Text("files_sorted_by_modtime size in bytes: %lld", ex1w.files_sorted_by_modtime.count * sizeof(string));
|
||||
|
||||
for (s64 i = 1; i < 128; i += 1) {
|
||||
u8 cstring[2] = {};
|
||||
cstring[0] = (u8)i;
|
||||
cstring[1] = 0;
|
||||
Text("codepoint[0x%X]: %s, count: %lld", i, cstring, count_ascii_codepoints[i]);
|
||||
}
|
||||
}
|
||||
|
||||
@ -215,10 +225,12 @@ void Ex1_Control_Panel () { using namespace ImGui;
|
||||
// bool all_drives_enumerated = !ex1_ntfs.threads_in_flight.count
|
||||
// && (drives_enumerated == drives.count);
|
||||
|
||||
string file_path = format_string_temp("%s_DriveData.bin", os_get_machine_name().data);
|
||||
// if (!all_drives_enumerated && file_exists(file_path)) {
|
||||
// Deserialize_ST_File_Enumeration(file_path);
|
||||
// }
|
||||
// string file_path = format_string_temp("%s_DriveData.bin", os_get_machine_name().data);
|
||||
string file_path = "D:/Projects/Cpp/Musa-Cpp-Lib-V2/bin/MUSA-PC3_DriveData.bin";// FIXED path.
|
||||
Text("fixed file_path: %s", file_path.data);
|
||||
if (!all_drives_enumerated && file_exists(file_path)) { // #autoload
|
||||
Deserialize_ST_File_Enumeration(file_path);
|
||||
}
|
||||
if (drives.count > 0 && !all_drives_enumerated && file_exists(file_path) && Button("Load from file (this machine)")) {
|
||||
Deserialize_ST_File_Enumeration(file_path);
|
||||
// Deserialize_Win32_Drives(file_path);
|
||||
@ -234,8 +246,6 @@ void Ex1_Control_Panel () { using namespace ImGui;
|
||||
// }
|
||||
// }
|
||||
|
||||
|
||||
|
||||
// #FileEnumerationST
|
||||
if (stfe && stfe->thread_started) {
|
||||
// print dirs enumerated, etc
|
||||
@ -259,11 +269,8 @@ void Ex1_Control_Panel () { using namespace ImGui;
|
||||
if (stfe && stfe->thread_started && !stfe->thread_completed) {
|
||||
Assert(stfe->master_thread != nullptr);
|
||||
if (thread_is_done(stfe->master_thread)) {
|
||||
push_allocator(GPAllocator());
|
||||
thread_deinit(stfe->master_thread, true);
|
||||
stfe->thread_completed = true;
|
||||
// Delete(GPAllocator(), stfe);
|
||||
// stfe = nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
@ -274,129 +281,29 @@ void Ex1_Control_Panel () { using namespace ImGui;
|
||||
}
|
||||
}
|
||||
|
||||
if (all_drives_enumerated && Button("Reset State")) {
|
||||
free_ex1_workspace_and_reset();
|
||||
free_stfe_and_reset();
|
||||
return;
|
||||
}
|
||||
|
||||
if (all_drives_enumerated) {
|
||||
Ex1_show_enumeration_workspace();
|
||||
}
|
||||
|
||||
if (drives.count > 0 && !all_drives_enumerated && Button("Enumerate all NTFS drives")) { // && ex1_ntfs.initialized
|
||||
os_run_file_enumeration_single_threaded();
|
||||
// run_multithreaded_enumeration_thread();
|
||||
// os_run_file_enumeration_multithreaded(); // #disabled for now
|
||||
return;
|
||||
/*// #NTFS_MFT_RAW
|
||||
push_allocator(GPAllocator());
|
||||
Array<ArrayView<OS_Drive*>> drive_split;
|
||||
drive_split.allocator = temp(); // this is only needed for this frame
|
||||
|
||||
if (drives.count > os_cpu_physical_core_count()) {
|
||||
s32 thread_count = os_cpu_physical_core_count();
|
||||
array_resize(drive_split, thread_count);
|
||||
ntfs_create_enumeration_threads(thread_count);
|
||||
|
||||
s32 threads_to_create = thread_count;
|
||||
s64 drives_per_thread = (drives.count / thread_count);
|
||||
s64 remainder = drives.count % thread_count;
|
||||
s64 current_drive = 0;
|
||||
|
||||
for_each(d, drive_split) {
|
||||
if (d == drive_split.count) {
|
||||
drive_split[d] = ArrayView<OS_Drive*>(remainder);
|
||||
} else {
|
||||
drive_split[d] = ArrayView<OS_Drive*>(drives_per_thread);
|
||||
}
|
||||
|
||||
for (s64 i = 0; i < drive_split[d].count; i += 1) {
|
||||
drive_split[d][i] = drives[current_drive];
|
||||
current_drive += 1;
|
||||
}
|
||||
}
|
||||
|
||||
debug_break(); // #TODO: Check that the work has been distributed correctly.
|
||||
} else { // more threads than drives, or same amount
|
||||
s32 thread_count = (s32)drives.count;
|
||||
array_resize(drive_split, drives.count);
|
||||
ntfs_create_enumeration_threads(thread_count);
|
||||
|
||||
for_each(d, drives) {
|
||||
auto drive = drives[d];
|
||||
drive_split[d] = ArrayView<OS_Drive*>(1); // Arrays of size one are sad :pensive:
|
||||
|
||||
drive_split[d][0] = drive;
|
||||
}
|
||||
}
|
||||
|
||||
s64 active_thread_count = drive_split.count;
|
||||
|
||||
ex1_ntfs.threads_started = true;
|
||||
for (s64 t = 0; t < active_thread_count; t += 1) {
|
||||
Thread* thread = &ex1_ntfs.threads[t];
|
||||
Arena* thread_arena = next_arena(Arena_Reserve::Size_64K);
|
||||
push_arena(thread_arena);
|
||||
auto thread_data = New<NTFS_Enumeration_Task>();
|
||||
thread_data->pool = thread_arena;
|
||||
thread_data->drives = drive_split[t];
|
||||
thread_start(thread, thread_data);
|
||||
array_add(ex1_ntfs.threads_in_flight, thread);
|
||||
}*/
|
||||
}
|
||||
|
||||
/* #NTFS_MFT_RAW
|
||||
if (ex1_ntfs.threads_in_flight.count) {
|
||||
Text("Threads in flight: %d", ex1_ntfs.threads_in_flight.count);
|
||||
|
||||
for_each(t, ex1_ntfs.threads_in_flight) {
|
||||
if (thread_is_done(ex1_ntfs.threads_in_flight[t])) {
|
||||
push_allocator(GPAllocator());
|
||||
|
||||
Thread* thread = ex1_ntfs.threads_in_flight[t];
|
||||
auto task = thread_task(NTFS_Enumeration_Task);
|
||||
array_free(task->drives);
|
||||
|
||||
// make sure to retreive any data you need to from here!
|
||||
release_arena(task->pool);
|
||||
|
||||
thread_deinit(ex1_ntfs.threads_in_flight[t], false);
|
||||
array_unordered_remove_by_index(ex1_ntfs.threads_in_flight, t);
|
||||
t -= 1; // check this element index again!
|
||||
}
|
||||
}
|
||||
}*/
|
||||
|
||||
/* #NTFS_MFT_RAW
|
||||
if (ex1_ntfs.threads_started && !ex1_ntfs.threads_in_flight.count) {
|
||||
// All threads are complete, we're free to clean up remaining memory
|
||||
push_allocator(GPAllocator());
|
||||
array_free(ex1_ntfs.threads);
|
||||
array_free(ex1_ntfs.threads_in_flight);
|
||||
|
||||
// Instead maybe we should just memset this to zero.
|
||||
reset_struct(&ex1_ntfs);
|
||||
}
|
||||
|
||||
// How do I tell when all files are enumerated?
|
||||
// check drives[i]->data.paths.wstrings.count count?
|
||||
if (all_drives_enumerated && Button("Save drive data")) {
|
||||
string file_path = format_string_temp("%s_DriveData.bin", os_get_machine_name().data);
|
||||
bool success = Serialize_Win32_Drives(drives, file_path);
|
||||
if (!success) { log_error("Failed to save Win32_Drive data"); }
|
||||
}
|
||||
|
||||
if (all_drives_enumerated && Button("Clear all drive data")) {
|
||||
os_clear_drive_data();
|
||||
}*/
|
||||
}
|
||||
|
||||
void ImGui_Debug_Panel () { using namespace ImGui;
|
||||
push_allocator(temp());
|
||||
|
||||
Begin("Debug Panel");
|
||||
/*SeparatorText("ex1_ntfs");
|
||||
Text("Threads in flight count: %d", ex1_ntfs.threads_in_flight.count);
|
||||
for_each(i, ex1_ntfs.threads) {
|
||||
Text(" [%d] initialized: %d, has_context: %d, has_data: %d",
|
||||
i, ex1_ntfs.threads[i].proc != nullptr, ex1_ntfs.threads[i].context != nullptr, ex1_ntfs.threads[i].data != nullptr);
|
||||
}*/
|
||||
// #cpuid
|
||||
// Text("[cpus] physical: %d, logical: %d, primary: %d, secondary: %d", os_cpu_physical_core_count(), os_cpu_logical_core_count(), os_cpu_primary_core_count(), os_cpu_secondary_core_count());
|
||||
Text("[cpus] physical: %d, logical: %d, primary: %d, secondary: %d", os_cpu_physical_core_count(), os_cpu_logical_core_count(), os_cpu_primary_core_count(), os_cpu_secondary_core_count());
|
||||
{ SeparatorText("Arena In-Use List");
|
||||
lock_guard(&arena_free_list->mutex);
|
||||
for (u8 i = 0; i < Arena_Reserve_Count; i += 1) {
|
||||
|
||||
50
src/String_Analysis.cpp
Normal file
50
src/String_Analysis.cpp
Normal file
@ -0,0 +1,50 @@
|
||||
// 1. Count unique occurrences:
|
||||
|
||||
s64 count_ascii_codepoints[128] = {};
|
||||
// s64 extended_codepoints[65536] = {};
|
||||
|
||||
Array<u32> unique_codepoints_utf32;
|
||||
|
||||
force_inline void unique_codepoints_utf32_add_unique (u32 unique) {
|
||||
array_add_if_unique(unique_codepoints_utf32, unique);
|
||||
}
|
||||
|
||||
void count_unique_chars_from_string (string s) {
|
||||
string s_copy = s;
|
||||
|
||||
while (s_copy.count > 0) {
|
||||
if (s_copy.data[0] == 0x5C) { // `\` character
|
||||
s_copy.data += 1;
|
||||
s_copy.count -= 1;
|
||||
}
|
||||
u32 utf32_codepoint; s64 codepoint_source_length;
|
||||
bool success = character_utf8_to_utf32(s_copy.data, s_copy.count, &utf32_codepoint, &codepoint_source_length);
|
||||
|
||||
if (success) {
|
||||
if (codepoint_source_length == 1) {
|
||||
Assert(utf32_codepoint <= 127);
|
||||
// Add to table
|
||||
count_ascii_codepoints[utf32_codepoint] += 1;
|
||||
} else {
|
||||
unique_codepoints_utf32_add_unique(utf32_codepoint);
|
||||
push_allocator(temp());
|
||||
string codepoint = copy_string(string_view(s_copy, 0, codepoint_source_length));
|
||||
log_todo("#TODO: keep track of unique codepoints. Codepoint: %s", codepoint);
|
||||
}
|
||||
}
|
||||
|
||||
s_copy.data += codepoint_source_length;
|
||||
s_copy.count -= codepoint_source_length;
|
||||
}
|
||||
}
|
||||
|
||||
void count_unique_utf8_chars () { Timed_Block_Print("count_unique_utf8_chars");
|
||||
unique_codepoints_utf32.allocator = GPAllocator();
|
||||
|
||||
Assert(stfe != nullptr);
|
||||
for (s64 i = 0; i < stfe->dirs.offsets->count; i += 1) {
|
||||
string sample = get_file_string_view(stfe, i);
|
||||
|
||||
count_unique_chars_from_string(sample);
|
||||
}
|
||||
}
|
||||
@ -44,7 +44,7 @@ void Explorer_ImGui_Application_Win32 () {
|
||||
|
||||
// Setup Dear ImGui context
|
||||
IMGUI_CHECKVERSION();
|
||||
printf("ImGui Version %s \n", ImGui::GetVersion());
|
||||
log("ImGui Version %s \n", ImGui::GetVersion());
|
||||
|
||||
imgui_context = ImGui::CreateContext();
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user