MusaMahmood/Musa-Cpp-Lib-V2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Replace multithreaded enumeration with single-threaded (temporarily) #2

Merged
MusaMahmood merged 4 commits from fix-multithreaded-enumeration into main 2025-12-17 15:18:20 +00:00
Conversation 0 Commits 4 Files Changed 27 +402 -334
9 changed files with 402 additions and 334 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit d1d775075d - Show all commits

11
lib/Base/Arena_Free_List.cpp
View File

@ -86,6 +86,17 @@ void release_arena (Arena* arena, bool delete_extra_pages) {
// } // }
} }
s64 bytes_in_use (ArrayView<Arena*> arenas) {
// does not include overhead from committed pages!
s64 sum = 0;
for (s64 i = 0; i < arenas.count; i += 1) {
sum += arena_usage_bytes(arenas[i]);
}
return sum;
}
s64 committed_bytes (ArrayView<Arena*> arenas) { s64 committed_bytes (ArrayView<Arena*> arenas) {
s64 sum = 0; s64 sum = 0;

2
lib/Base/Base.h
View File

@ -185,7 +185,7 @@ force_inline s64 Next_Power_Of_Two(s64 v) {
#define auto_release(x) \ #define auto_release(x) \
Auto_Release Concat(_auto_release_guard_, __LINE__)(x) Auto_Release Concat(_auto_release_guard_, __LINE__)(x)
#define auto_release_temp() \ #define auto_release_temp() \
auto_release(thread_context()->temp); auto_release(thread_context()->temp)
#define thread_context() thread_local_context #define thread_context() thread_local_context
#define temp() allocator(thread_context()->temp) #define temp() allocator(thread_context()->temp)

6
lib/Base/Expandable_Arena.cpp
View File

@ -109,6 +109,12 @@ Allocator allocator (ExpandableArena* arena_ex) {
// #TODO: currently this keeps the final arena's memory. Fix this! // #TODO: currently this keeps the final arena's memory. Fix this!
void arena_reset_to (ExpandableArena* arena_ex, Arena* last_arena, u8* starting_point) { void arena_reset_to (ExpandableArena* arena_ex, Arena* last_arena, u8* starting_point) {
// going backwards from end of arena list // going backwards from end of arena list
if (!arena_ex->next_arenas.count) {
arena_ex->current_point = starting_point;
return;
}
// for (s64 i = arena_ex->next_arenas.count-1; i >= 0; i -= 1) { // for (s64 i = arena_ex->next_arenas.count-1; i >= 0; i -= 1) {
for_each_reverse(i, arena_ex->next_arenas) { for_each_reverse(i, arena_ex->next_arenas) {
Arena* arena = arena_ex->next_arenas[i]; Arena* arena = arena_ex->next_arenas[i];

17
lib/Base/Serializer.h
View File

@ -146,4 +146,19 @@ force_inline void ReadString16 (Deserializer* ds, string& s) { // #no_alloc
ReadStringView(ds, s, (s64)str_len); ReadStringView(ds, s, (s64)str_len);
} }
// ReadString_MakeCopy... // This is specialized for filesystem storage of strings.
force_inline u32 AddString_NoCount (Serializer* serializer, u8* data, s16 count) {
u32 original_count = (u32)serializer->count;
u8* current_point = &serializer->data[original_count];
s64 final_count = serializer->allocated + (count * sizeof(u8));
if (serializer->allocated < final_count) {
array_reserve(*serializer, final_count);
}
memcpy(current_point, data, count * sizeof(u8));
serializer->count += count * sizeof(u8);
return original_count;
}

11
lib/Base/Threads.cpp
View File

@ -70,3 +70,14 @@ struct Thread_Group {
bool started = false; bool started = false;
bool should_exit = false; bool should_exit = false;
}; };
// This might be too slow.
s32 get_thread_index (Thread_Group* group, s32 thread_index) {
for_each(w, group->worker_info) {
if (group->worker_info[w].thread.index == thread_index) {
return (s32)w; // zero-indexed to thread group
}
}
return -1;
}

17
lib/OS/OS_Filesystem.cpp
View File

@ -21,23 +21,6 @@
// if we need ordered insertions and deletes. // if we need ordered insertions and deletes.
// //
// Returns offset
force_inline u32 AddString_NoCount (Serializer* serializer, u8* data, u8 count) { // #TODO: , bool null_terminate=false
u32 original_count = (u32)serializer->count;
u8* current_point = &serializer->data[original_count];
s64 final_count = serializer->allocated + (count * sizeof(u8));
if (serializer->allocated < final_count) {
array_reserve(*serializer, final_count);
}
memcpy(current_point, data, count * sizeof(u8));
serializer->count += count * sizeof(u8);
return original_count;
}
constexpr s64 DFS_Preallocation_Count = 4194304; // 2^22 constexpr s64 DFS_Preallocation_Count = 4194304; // 2^22
// template <typename Length_Type> // template <typename Length_Type>

528
lib/OS/OS_Win32.cpp
View File

@ -4,7 +4,6 @@
// - I think no? Threads should handle their own lifetimes, and the parent threads should ensure child threads are complete // - I think no? Threads should handle their own lifetimes, and the parent threads should ensure child threads are complete
// before terminating. // before terminating.
#if OS_WINDOWS
constexpr s64 FILETIME_TO_UNIX = 116444736000000000i64; constexpr s64 FILETIME_TO_UNIX = 116444736000000000i64;
f64 GetUnixTimestamp () { f64 GetUnixTimestamp () {
FILETIME fileTime; FILETIME fileTime;
@ -31,7 +30,20 @@ u64 FILETIME_to_ticks (FILETIME fileTime) {
return ticks; return ticks;
} }
#endif string format_time_datetime (FILETIME ft) {
SYSTEMTIME stUTC, st;
FileTimeToSystemTime(&ft, &stUTC);
SystemTimeToTzSpecificLocalTime(nullptr, &stUTC, &st);
// YYYY-MM-DD-
return format_string("%04u-%02u-%02u %02u:%02u:%02u.%03u",
st.wYear,
st.wMonth,
st.wDay,
st.wHour,
st.wMinute,
st.wSecond,
st.wMilliseconds);
}
struct OS_System_Info { struct OS_System_Info {
// #cpuid // #cpuid
@ -1117,323 +1129,277 @@ string os_get_machine_name () {
// [ ] get_mouse_pointer_position // [ ] get_mouse_pointer_position
// [ ] ... What APIs do I need for Keyboard // [ ] ... What APIs do I need for Keyboard
struct Enumeration_Work { // #FileEnumerationST
string first_directory;
s32 parent_index;
Arena* thread_arena; // pointer to relevant tctx->arena
// Directories
ArenaArray<u32>* d_offsets;
ArenaArray<s16>* d_lengths;
ArenaArray<s32>* d_parent_indices;
ArenaArray<u64>* d_sizes;
ArenaArray<u64>* d_modtime;
// Files
ArenaArray<u32>* offsets;
ArenaArray<s16>* lengths;
ArenaArray<s32>* parent_indices;
ArenaArray<u64>* sizes;
ArenaArray<u64>* modtime;
};
struct Files_Combined_Results { struct STFE_Results {
// ArenaArray<string> full_path; Serializer* strings; // Serializer?
ArenaArray<string>* name; ArenaArray<u32>* offsets;
ArenaArray<s32>* parent_indices; ArenaArray<s16>* lengths;
ArenaArray<u64>* sizes; ArenaArray<u64>* sizes;
ArenaArray<u64>* modtime; ArenaArray<u64>* modtimes;
}; };
struct Drive_Enumeration { // master thread struct void init (STFE_Results* results) {
Arena* arena; results->strings = (Serializer*)arena_array_new<u8> (1024*1024*4*16, Arena_Reserve::Size_2G);
results->offsets = arena_array_new<u32>(1024*1024*4, Arena_Reserve::Size_2G);
results->lengths = arena_array_new<s16>(1024*1024*4, Arena_Reserve::Size_2G);
results->sizes = arena_array_new<u64>(1024*1024*4, Arena_Reserve::Size_2G);
results->modtimes = arena_array_new<u64>(1024*1024*4, Arena_Reserve::Size_2G);
}
struct ST_File_Enumeration { // global state
ArrayView<OS_Drive*> drives; ArrayView<OS_Drive*> drives;
Thread* master_thread; Thread* master_thread;
s32 thread_count; STFE_Results dirs;
STFE_Results files;
s32 directories_enumerated; // going sequentially
bool thread_started; bool thread_started;
bool thread_completed; bool thread_completed;
Files_Combined_Results paths; f64 start_time;
Files_Combined_Results files; f64 end_time;
s32 work_added = 0;
s32 work_completed = 0;
}; };
void push_root (Drive_Enumeration* de, string label, s32 index) { global ST_File_Enumeration* stfe;
array_add(*de->paths.name, label);
array_add(*de->paths.parent_indices, index);
array_add(*de->paths.sizes, (u64)0);
array_add(*de->paths.modtime, (u64)0);
}
global Drive_Enumeration* drive_enumeration; string add_record (ST_File_Enumeration* stfe, string full_path, bool is_directory, WIN32_FIND_DATAW* find_data) {
// return the string copy!
string directory_get_full_path (Drive_Enumeration* de, s64 index) {
push_allocator(GPAllocator()); // to copy from String_Builder
Files_Combined_Results* f = &de->paths;
string dir_name = (*f->name)[index];
s32 parent_index = (*f->parent_indices)[index];
s32 next_parent = (*f->parent_indices)[parent_index];
Array<string> paths;
paths.allocator = temp();
array_add(paths, (*f->name)[parent_index]);
while (parent_index != next_parent) {
parent_index = next_parent;
next_parent = (*f->parent_indices)[parent_index];
array_add(paths, (*f->name)[parent_index]);
}
// while (parent_index > -1) { // should be while(true)
//
// s32 next_parent = (*f->parent_indices)[parent_index];
// if (parent_index == next_parent) break;
// s32 parent_index = next_parent;
// }
// go in reverse order and add together string
String_Builder* sb = new_string_builder(Arena_Reserve::Size_64K);
for (s64 i = paths.count-1; i >= 0; i -= 1) {
append(sb, paths[i]);
append(sb, "\\");
}
append(sb, dir_name);
return builder_to_string(sb);
}
void update_results (Drive_Enumeration* de, Enumeration_Work* ew) {
// merge results and release resources!
// unfortunately this is a LOT of copying!
for_each(i, (*ew->d_offsets)) {
u8* string_ptr = (ew->thread_arena->memory_base + (*ew->d_offsets)[i]);
string name = {(*ew->d_lengths)[i], string_ptr};
array_add(*de->paths.name, name);
array_add(*de->paths.parent_indices, (*ew->d_parent_indices)[i]);
array_add(*de->paths.sizes, (*ew->d_sizes)[i]);
array_add(*de->paths.modtime, (*ew->d_modtime)[i]);
}
for_each(i, (*ew->offsets)) {
u8* string_ptr = (ew->thread_arena->memory_base + (*ew->offsets)[i]);
string name = {(*ew->lengths)[i], string_ptr};
array_add(*de->files.name, name);
array_add(*de->files.parent_indices, (*ew->parent_indices)[i]);
array_add(*de->files.sizes, (*ew->sizes)[i]);
array_add(*de->files.modtime, (*ew->modtime)[i]);
}
}
void add_record (Enumeration_Work* ew, WIN32_FIND_DATAW* find_data, string name, s32 parent_index=-1) {
u32 offset = (u32)(name.data - ew->thread_arena->memory_base);
bool is_directory = (find_data->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0;
u64 size = ((u64)find_data->nFileSizeHigh << 32) | ((u64)find_data->nFileSizeLow & 0xFFFFFFFF);
if (is_directory) { if (is_directory) {
array_add((*ew->d_offsets), offset); STFE_Results* r = &stfe->dirs;
array_add((*ew->d_lengths), (s16)name.count); u32 offset = AddString_NoCount(r->strings, full_path.data, (s16)full_path.count);
array_add((*ew->d_parent_indices), parent_index); // #TODO #parent_index array_add((*r->offsets), offset);
array_add((*ew->d_sizes), size); array_add((*r->lengths), (s16)full_path.count);
array_add((*ew->d_modtime), FILETIME_to_ticks(find_data->ftLastWriteTime)); // No size for directories.
u64 modtime = FILETIME_to_ticks(find_data->ftLastWriteTime);
array_add((*r->modtimes), modtime);
string path_copy = {full_path.count, &r->strings->data[offset]};
return path_copy;
} else { } else {
array_add((*ew->offsets), offset); STFE_Results* r = &stfe->files;
array_add((*ew->lengths), (s16)name.count); u32 offset = AddString_NoCount(r->strings, full_path.data, (s16)full_path.count);
array_add((*ew->parent_indices), parent_index); // #TODO #parent_index array_add((*r->offsets), offset);
array_add((*ew->sizes), size); array_add((*r->lengths), (s16)full_path.count);
array_add((*ew->modtime), FILETIME_to_ticks(find_data->ftLastWriteTime));
u64 size = ((u64)find_data->nFileSizeHigh << 32) | ((u64)find_data->nFileSizeLow & 0xFFFFFFFF);
u64 modtime = FILETIME_to_ticks(find_data->ftLastWriteTime);
array_add((*r->sizes), size);
array_add((*r->modtimes), modtime);
string path_copy = {full_path.count, &r->strings->data[offset]};
return path_copy;
} }
Assert(false);
return {};
} }
Thread_Continue_Status file_enumeration_thread_group_proc (Thread_Group* group, Thread* thread, void* work) { s32 count_paths (ST_File_Enumeration* stfe) {
// 1. setup userdata as an Arena*: STFE_Results* r = &stfe->dirs;
Arena* result_arena; return (s32)r->offsets->count;
if (!thread->context->userdata) { }
result_arena = next_arena(Arena_Reserve::Size_64G); s32 count_files (ST_File_Enumeration* stfe) {
thread->context->userdata = result_arena; STFE_Results* r = &stfe->files;
} else { return (s32)r->offsets->count;
result_arena = (Arena*)thread->context->userdata;
}
Enumeration_Work* enum_work = (Enumeration_Work*)work;
enum_work->thread_arena = (Arena*)thread->context->userdata;
enum_work->d_offsets = arena_array_new<u32>(4096, Arena_Reserve::Size_2M);
enum_work->d_lengths = arena_array_new<s16>(4096, Arena_Reserve::Size_2M);
enum_work->d_parent_indices = arena_array_new<s32>(4096, Arena_Reserve::Size_2M);
enum_work->d_sizes = arena_array_new<u64>(4096, Arena_Reserve::Size_2M);
enum_work->d_modtime = arena_array_new<u64>(4096, Arena_Reserve::Size_2M);
enum_work->offsets = arena_array_new<u32>(4096, Arena_Reserve::Size_2M);
enum_work->lengths = arena_array_new<s16>(4096, Arena_Reserve::Size_2M);
enum_work->parent_indices = arena_array_new<s32>(4096, Arena_Reserve::Size_2M);
enum_work->sizes = arena_array_new<u64>(4096, Arena_Reserve::Size_2M);
enum_work->modtime = arena_array_new<u64>(4096, Arena_Reserve::Size_2M);
// Validate thread context?
push_allocator(temp());
auto_release_temp();
// log("file_enumeration_thread_group_proc, thread index: %d", thread->index);
// MAKE SURE PATH IS NULL TERMINATED!
wstring wildcard_name = utf8_to_wide(format_string("%s\\*", enum_work->first_directory.data)); // #temp
WIN32_FIND_DATAW find_data;
HANDLE h = FindFirstFileExW((LPCWSTR)wildcard_name.data, FindExInfoBasic, &find_data,
FindExSearchNameMatch, nullptr, FIND_FIRST_EX_LARGE_FETCH);
if (h == INVALID_HANDLE_VALUE) {
return Thread_Continue_Status::CONTINUE;
}
while (true) {
push_arena(result_arena);
string name = wide_to_utf8((u16*)find_data.cFileName); // #NOT_TEMP
bool should_continue = (name.count == 0 || name == "." || name == "..");
if (should_continue) {
bool success = FindNextFileW(h, &find_data);
if (!success)
break;
continue;
}
add_record(enum_work, &find_data, name, enum_work->parent_index);
bool success = FindNextFileW(h, &find_data);
if (!success) break;
}
FindClose(h);
return Thread_Continue_Status::CONTINUE;
} }
s64 multithreaded_file_enumeration_master_proc (Thread* thread) { // #UI #TEMP - just for visualization!
auto task = thread_task(Drive_Enumeration); string get_file_copy (ST_File_Enumeration* stfe, s64 index) {
STFE_Results* r = &stfe->files;
push_arena(task->arena); Assert(index >= 0 && index < count_files(stfe));
s64 strlength = (*r->lengths)[index];
Thread_Group* file_enum_thread_group = New<Thread_Group>(); u32 offset = (*r->offsets)[index];
u8* string_ptr = &r->strings->data[offset];
s32 thread_count = os_cpu_physical_core_count(); string file = {strlength, string_ptr};
return copy_string(file);
push_allocator(GPAllocator()); }
thread_group_init(file_enum_thread_group, thread_count, file_enumeration_thread_group_proc, true); string get_path_copy (ST_File_Enumeration* stfe, s64 index) {
STFE_Results* r = &stfe->dirs;
Assert(index >= 0 && index < count_paths(stfe));
s64 strlength = (*r->lengths)[index];
u32 offset = (*r->offsets)[index];
u8* string_ptr = &r->strings->data[offset];
string path = {strlength, string_ptr};
return copy_string(path);
}
s64 get_file_size_bytes (ST_File_Enumeration* stfe, s64 index) {
STFE_Results* r = &stfe->files;
return (s64)(*r->sizes)[index];
}
FILETIME get_file_modtime (ST_File_Enumeration* stfe, s64 index) {
STFE_Results* r = &stfe->files;
FILETIME ft;
memcpy(&ft, &(*r->modtimes)[index], sizeof(u64));
return ft;
}
FILETIME get_path_modtime (ST_File_Enumeration* stfe, s64 index) {
STFE_Results* r = &stfe->dirs;
FILETIME ft;
memcpy(&ft, &(*r->modtimes)[index], sizeof(u64));
return ft;
}
s64 win32_file_enum_thread_proc (Thread* thread) {
auto task = thread_task(ST_File_Enumeration);
init(&task->dirs);
init(&task->files);
// while files available?
Array<string> paths_to_enumerate;
for_each(d, task->drives) { for_each(d, task->drives) {
auto work = New<Enumeration_Work>(GPAllocator()); //replace with arena bootstrap? string parent_directory = task->drives[d]->label; // includes a trailing slash
work->first_directory = task->drives[d]->label; // this includes the colon-slash, (e.g. `C:\`). if (parent_directory.data[2] == (u8)'\\') {
work->parent_index = (s32)d; // #HACK? parent_directory.count -= 1; //#hack to quickly remove trailing slash.
// add label root to combined results, so we can look it up later!
push_root(task, work->first_directory, work->parent_index);
add_work(file_enum_thread_group, work);
task->work_added += 1;
}
start(file_enum_thread_group);
// set task completed.
s64 path_index = task->drives.count;
// #TODO: Get completed work!
while (task->work_completed < task->work_added) {
auto_release_temp();
ArrayView<void*> cw = get_completed_work(file_enum_thread_group);
for_each(i, cw) {
auto ew = (Enumeration_Work*)cw[i];
update_results(task, ew);
arena_array_free(*ew->d_offsets, false);
arena_array_free(*ew->d_lengths, false);
arena_array_free(*ew->d_parent_indices, false);
arena_array_free(*ew->d_sizes, false);
arena_array_free(*ew->d_modtime, false);
arena_array_free(*ew->offsets, false);
arena_array_free(*ew->lengths, false);
arena_array_free(*ew->parent_indices, false);
arena_array_free(*ew->sizes, false);
arena_array_free(*ew->modtime, false);
string_free(ew->first_directory);
internal_free(ew);
} }
task->work_completed += (s32)cw.count; array_add(paths_to_enumerate, parent_directory);
// For each new directory: while (paths_to_enumerate.count > 0) {
// s64 dirs_to_enumerate = task->paths.name->count - path_index; push_allocator(temp());
for (s64 i = path_index; i < task->paths.name->count; i += 1) { auto_release_temp();
auto work = New<Enumeration_Work>(GPAllocator()); // This needs to be null-terminated:
work->first_directory = directory_get_full_path(task, i);// need full name here! string next_directory = copy_string(pop(paths_to_enumerate)); // LIFO. maybe not the best way?
work->parent_index = (s32)i; wstring wildcard_name = utf8_to_wide(format_string("%s\\*", next_directory.data));
add_work(file_enum_thread_group, work); WIN32_FIND_DATAW find_data;
HANDLE h = FindFirstFileExW((LPCWSTR)wildcard_name.data, FindExInfoBasic, &find_data,
FindExSearchNameMatch, nullptr, FIND_FIRST_EX_LARGE_FETCH);
if (h == INVALID_HANDLE_VALUE) {
log_error("FindFirstFileExW failed for %s", wide_to_utf8(wildcard_name.data, (s32)wildcard_name.count).data);
os_log_error();
continue;
}
task->work_added += 1; while (true) { auto_release_temp();
} string name = wide_to_utf8((u16*)find_data.cFileName);
path_index = task->paths.name->count; bool should_continue = (name.count == 0 || name == "." || name == "..");
if (should_continue) {
Sleep(1); bool success = FindNextFileW(h, &find_data);
log("work completed: %d/%d",task->work_completed, task->work_added); if (!success) { break; }
} continue;
}
bool is_directory = (find_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0;
string full_path = format_string("%s\\%s", next_directory.data, name.data);
string full_path_copy = add_record(task, full_path, is_directory, &find_data);
if (is_directory) {
array_add(paths_to_enumerate, full_path_copy);
}
bool success = FindNextFileW(h, &find_data);
if (!success) break;
} // while (true) -> FindNextFileW
FindClose(h);
} // while (parent_directory)
} // for_each(d, drives)
shutdown(file_enum_thread_group); task->end_time = GetUnixTimestamp();
task->thread_completed = true;
return 0; return 0;
} }
void initialize (Files_Combined_Results* fcr) { void os_run_file_enumeration_single_threaded () {
fcr->name = arena_array_new<string>(4194304, Arena_Reserve::Size_2G); // 2GB @ 16-byte strings => 134.2M entries. 64 might be better here for really large file collections! push_allocator(GPAllocator());
fcr->parent_indices = arena_array_new<s32>(4194304, Arena_Reserve::Size_2G);
fcr->sizes = arena_array_new<u64>(4194304, Arena_Reserve::Size_2G);
fcr->modtime = arena_array_new<u64>(4194304, Arena_Reserve::Size_2G);
}
void run_multithreaded_enumeration_thread () {
// Need some struct to track the state of this operation.
Arena* arena = next_arena(Arena_Reserve::Size_64K);
push_arena(arena);
drive_enumeration = New<Drive_Enumeration>(); stfe = New<ST_File_Enumeration>();
(*drive_enumeration) = { (*stfe) = {
arena, os_get_available_drives(),
os_get_available_drives(),
New<Thread>(), New<Thread>(),
os_cpu_physical_core_count(), STFE_Results(), STFE_Results(),
0, false, false, {}, {}, 0, true, false, GetUnixTimestamp(), 0
0, 0
}; };
initialize(&drive_enumeration->paths); string thread_name = "Single Thread Enumeration - Master Thread";
initialize(&drive_enumeration->files); bool success = thread_init(stfe->master_thread, win32_file_enum_thread_proc, thread_name);
if (!success) {
// We start 1 thread to run the thread group and track the threads log_error("Failed to initialize thread (stft->master_thread)");
string thread_name = "Multithreaded Enumeration: Master Thread"; os_log_error();
bool success = thread_init(drive_enumeration->master_thread, }
multithreaded_file_enumeration_master_proc, thread_name);
Assert(success); Assert(success);
thread_start(drive_enumeration->master_thread, drive_enumeration); thread_start(stfe->master_thread, stfe);
drive_enumeration->thread_started = true;
} }
bool file_enum_multithreading_started () { constexpr u32 STFE_Magic_Number = 0x19075fee;
if (drive_enumeration == nullptr) return false;
return drive_enumeration->thread_started; bool Serialize_ST_File_Enumeration (string file_path) {
Timed_Block_Print("Serialize_ST_File_Enumeration");
File f = file_open(file_path, true, false, true);
if (!file_is_valid(f)) return false;
bool success = true;
// #TODO #Serialization Unfortunately, there's a lot of needless copying here
// it would be a lot nicer if we could just write-file in place. idk how to do that though ;_;
Serializer* s = new_serializer(Arena_Reserve::Size_64G);
Add(s, (u32)STFE_Magic_Number);
Add(s, (s32)stfe->drives.count);
// Dirs:
STFE_Results* r = &stfe->dirs;
AddArray(s, to_view(*r->strings));
AddArray(s, to_view(*r->offsets));
AddArray(s, to_view(*r->lengths));
AddArray(s, to_view(*r->modtimes));
// Files:
r = &stfe->files;
AddArray(s, to_view(*r->strings));
AddArray(s, to_view(*r->offsets));
AddArray(s, to_view(*r->lengths));
AddArray(s, to_view(*r->sizes));
AddArray(s, to_view(*r->modtimes));
success = file_write(&f, to_view(*s));
reset_serializer(s);
file_close(&f);
free_serializer(s);
return success;
} }
bool file_enum_multithreading_active () { bool Deserialize_ST_File_Enumeration (string file_path) {
if (drive_enumeration == nullptr) return false; Timed_Block_Print("Deserialize_ST_File_Enumeration");
if (drive_enumeration->thread_completed) { push_allocator(GPAllocator());
return false; if (!stfe) stfe = New<ST_File_Enumeration>();
} (*stfe) = {
if (drive_enumeration->thread_started) { {},
return true; {},
} STFE_Results(), STFE_Results(),
return false; 0, false, false, GetUnixTimestamp(), 0
} };
// if (drive_enumeration != nullptr) { push_allocator(temp());
// // Check if task is completed, clean up thread. auto_release_temp();
// discard arena and zero drive_enumeration.
// } Deserializer deserializer = read_entire_file(file_path, true);
if (deserializer.count == 0) return false;
auto d = &deserializer;
u32 magic_number; s32 drive_count;
Read(d, &magic_number);
Assert(magic_number == STFE_Magic_Number);
Read(d, &drive_count);
init(&stfe->dirs);
init(&stfe->files);
STFE_Results* r = &stfe->dirs;
ReadToArenaArray(d, r->strings);
ReadToArenaArray(d, r->offsets);
ReadToArenaArray(d, r->lengths);
ReadToArenaArray(d, r->modtimes);
r = &stfe->files;
ReadToArenaArray(d, r->strings);
ReadToArenaArray(d, r->offsets);
ReadToArenaArray(d, r->lengths);
ReadToArenaArray(d, r->sizes);
ReadToArenaArray(d, r->modtimes);
stfe->thread_started = true;
stfe->thread_completed = true;
stfe->end_time = GetUnixTimestamp();
return true;
}

2
src/Base_Entry_Point.cpp
View File

@ -17,7 +17,7 @@ internal void Main_Entry_Point (int argc, WCHAR **argv);
#endif #endif
internal void Main_Entry_Point (int argc, WCHAR **argv) { // #entry_point internal void Main_Entry_Point (int argc, WCHAR **argv) { // #entry_point
set_cpu_base_frequency(3200); // REQUIRED FOR TIMING MODULE! will depend on CPU set_cpu_base_frequency(4000); // REQUIRED FOR TIMING MODULE! will depend on CPU
#if BASE_RUN_TESTS #if BASE_RUN_TESTS
run_pre_setup_tests(); // #no_context: context will not be initialized at this point. run_pre_setup_tests(); // #no_context: context will not be initialized at this point.

142
src/Ex1.cpp
View File

@ -116,7 +116,7 @@ void Ex1_show_ntfs_workspace () { using namespace ImGui;
bool success = file_length(full_path, &v.size); // temp, obviously we don't wanna call this every frame lol bool success = file_length(full_path, &v.size); // temp, obviously we don't wanna call this every frame lol
Text(" > size: %lld B", v.size); Text(" > size: %lld B", v.size);
Text(" > size: %s", format_bytes(v.size).data); Text(" > size: %s", format_bytes(v.size).data);
// Text(" > modtime: %s", idk how to convert FILETIME to calendar time // Text(" > modtime: %s",
} else { } else {
// DFS_Array* dfsa = &ntfs_workspace.drives[d]->data->paths; // DFS_Array* dfsa = &ntfs_workspace.drives[d]->data->paths;
@ -124,7 +124,28 @@ void Ex1_show_ntfs_workspace () { using namespace ImGui;
} }
} }
struct Ex1_Workspace {
s32 path_select;
s32 file_select;
};
global Ex1_Workspace ex1w;
void Ex1_show_enumeration_workspace () { using namespace ImGui;
push_imgui_window("Enumerated Data Workspace");
// #TODO SliderInt for each
SliderInt("Select path index", &ex1w.path_select, 0, count_paths(stfe)-1);
Text("%s", get_path_copy(stfe, ex1w.path_select).data);
Text("time modified: %s", format_time_datetime(get_path_modtime(stfe, ex1w.path_select)).data);
// #TODO: modtime (to indextime)
SliderInt("Select file index", &ex1w.file_select, 0, count_files(stfe)-1);
Text("%s", get_file_copy(stfe, ex1w.file_select).data);
Text("size: %s", format_bytes(get_file_size_bytes(stfe, ex1w.file_select)).data);
Text("time modified: %s", format_time_datetime(get_file_modtime(stfe, ex1w.file_select)).data);
// #TODO: size, modtime
}
void Ex1_Control_Panel () { using namespace ImGui; void Ex1_Control_Panel () { using namespace ImGui;
f64 frame_time = GetUnixTimestamp();
Table<string, OS_Drive*>* drive_table = get_drive_table(); Table<string, OS_Drive*>* drive_table = get_drive_table();
push_imgui_window("Control Panel"); push_imgui_window("Control Panel");
@ -138,23 +159,26 @@ void Ex1_Control_Panel () { using namespace ImGui;
return; return;
} }
Text("drive_table is valid: %d", table_is_valid(drive_table)); bool all_drives_enumerated = stfe && stfe->thread_completed;
push_allocator(temp()); push_allocator(temp());
ArrayView<OS_Drive*> drives = os_get_available_drives(); // only includes drives that are ready. ArrayView<OS_Drive*> drives = os_get_available_drives(); // only includes drives that are ready.
for_each(i, drives) { if (!all_drives_enumerated) {
OS_Drive* drive = drives[i]; // Text("drive_table is valid: %d", table_is_valid(drive_table));
for_each(i, drives) {
Text(" > [%d] drive letter: %s (is_present: %d)", i + 1, drive->label.data, drive->is_present); OS_Drive* drive = drives[i];
if (drive->time_to_enumerate != 0) {
SameLine(); Text(" > [%d] drive letter: %s (is_present: %d)", i + 1, drive->label.data, drive->is_present);
Text("Enumerated in %.2f seconds", drive->time_to_enumerate); if (drive->time_to_enumerate != 0) {
SameLine();
Text("Enumerated in %.2f seconds", drive->time_to_enumerate);
}
// SameLine();
// if (Button(format_cstring("Read NTFS MFT Raw##%s", drive->label.data))) {
// push_arena(thread_context()->arena);
// Error* error = NTFS_MFT_read_raw(drive);
// }
} }
// SameLine();
// if (Button(format_cstring("Read NTFS MFT Raw##%s", drive->label.data))) {
// push_arena(thread_context()->arena);
// Error* error = NTFS_MFT_read_raw(drive);
// }
} }
s32 drives_enumerated = 0; s32 drives_enumerated = 0;
@ -167,31 +191,77 @@ void Ex1_Control_Panel () { using namespace ImGui;
drives_enumerated += 1; drives_enumerated += 1;
} }
} }
bool all_drives_enumerated = !ex1_ntfs.threads_in_flight.count // bool all_drives_enumerated = !ex1_ntfs.threads_in_flight.count
&& (drives_enumerated == drives.count); // && (drives_enumerated == drives.count);
string file_path = format_string_temp("%s_DriveData.bin", os_get_machine_name().data); string file_path = format_string_temp("%s_DriveData.bin", os_get_machine_name().data);
if (!all_drives_enumerated && file_exists(file_path)) {
Deserialize_ST_File_Enumeration(file_path);
}
if (drives.count > 0 && !all_drives_enumerated && file_exists(file_path) && Button("Load from file (this machine)")) { if (drives.count > 0 && !all_drives_enumerated && file_exists(file_path) && Button("Load from file (this machine)")) {
Deserialize_Win32_Drives(file_path); Deserialize_ST_File_Enumeration(file_path);
// Deserialize_Win32_Drives(file_path);
} }
if (file_enum_multithreading_started()) { // if (file_enum_multithreading_started()) {
if (thread_is_done(drive_enumeration->master_thread)) { // if (thread_is_done(drive_enumeration->master_thread)) {
push_allocator(GPAllocator()); // push_allocator(GPAllocator());
// Thread* thread = drive_enumeration->master_thread; // // Thread* thread = drive_enumeration->master_thread;
// auto task = thread_task(Drive_Enumeration); // // auto task = thread_task(Drive_Enumeration);
// Nothing to free? // // Nothing to free?
thread_deinit(drive_enumeration->master_thread, true); // thread_deinit(drive_enumeration->master_thread, true);
// }
// }
// #FileEnumerationST
if (stfe && stfe->thread_started) {
// print dirs enumerated, etc
if (!stfe->thread_completed) Text("Enumeration Thread Active (elapsed: %s)", format_time_seconds(frame_time-stfe->start_time).data);
if (stfe->dirs.offsets) {
s64 dirs_enumerated = stfe->dirs.offsets->count;
Text("Dirs enumerated: %lld", dirs_enumerated);
// if (!stfe->thread_completed) Text("Current dir: %s", get_last_path_copy(stfe).data);
} }
if (stfe->files.offsets) {
s64 files_enumerated = stfe->files.offsets->count;
Text("Files enumerated: %lld", files_enumerated);
// if (!stfe->thread_completed) Text("Current file: %s", get_last_file_copy(stfe).data);
}
if (stfe->thread_completed) {
Text("String bytes stored: %s", format_bytes(stfe->dirs.strings->count + stfe->files.strings->count).data);
Text("Elapsed time: %s", format_time_seconds(stfe->end_time-stfe->start_time).data);
}
}
if (stfe && stfe->thread_started && !stfe->thread_completed) {
Assert(stfe->master_thread != nullptr);
if (thread_is_done(stfe->master_thread)) {
push_allocator(GPAllocator());
thread_deinit(stfe->master_thread, true);
stfe->thread_completed = true;
// Delete(GPAllocator(), stfe);
// stfe = nullptr;
}
}
if (all_drives_enumerated && Button("Save enumerated data")) {
if (!Serialize_ST_File_Enumeration(file_path)) {
log_error("Failed to write enumerated files");
os_log_error();
}
}
if (all_drives_enumerated) {
Ex1_show_enumeration_workspace();
} }
if (drives.count > 0 && !all_drives_enumerated && Button("Enumerate all NTFS drives")) { // && ex1_ntfs.initialized if (drives.count > 0 && !all_drives_enumerated && Button("Enumerate all NTFS drives")) { // && ex1_ntfs.initialized
// if drive count exceeds the number of threads, we need to group them so each thread os_run_file_enumeration_single_threaded();
// can enumerate multiple drives. // run_multithreaded_enumeration_thread();
// We need to distribute the drives across our available threads: return;
{ run_multithreaded_enumeration_thread(); /*// #NTFS_MFT_RAW
return;
}
push_allocator(GPAllocator()); push_allocator(GPAllocator());
Array<ArrayView<OS_Drive*>> drive_split; Array<ArrayView<OS_Drive*>> drive_split;
drive_split.allocator = temp(); // this is only needed for this frame drive_split.allocator = temp(); // this is only needed for this frame
@ -245,9 +315,10 @@ void Ex1_Control_Panel () { using namespace ImGui;
thread_data->drives = drive_split[t]; thread_data->drives = drive_split[t];
thread_start(thread, thread_data); thread_start(thread, thread_data);
array_add(ex1_ntfs.threads_in_flight, thread); array_add(ex1_ntfs.threads_in_flight, thread);
} }*/
} }
/* #NTFS_MFT_RAW
if (ex1_ntfs.threads_in_flight.count) { if (ex1_ntfs.threads_in_flight.count) {
Text("Threads in flight: %d", ex1_ntfs.threads_in_flight.count); Text("Threads in flight: %d", ex1_ntfs.threads_in_flight.count);
@ -267,8 +338,9 @@ void Ex1_Control_Panel () { using namespace ImGui;
t -= 1; // check this element index again! t -= 1; // check this element index again!
} }
} }
} }*/
/* #NTFS_MFT_RAW
if (ex1_ntfs.threads_started && !ex1_ntfs.threads_in_flight.count) { if (ex1_ntfs.threads_started && !ex1_ntfs.threads_in_flight.count) {
// All threads are complete, we're free to clean up remaining memory // All threads are complete, we're free to clean up remaining memory
push_allocator(GPAllocator()); push_allocator(GPAllocator());
@ -289,7 +361,7 @@ void Ex1_Control_Panel () { using namespace ImGui;
if (all_drives_enumerated && Button("Clear all drive data")) { if (all_drives_enumerated && Button("Clear all drive data")) {
os_clear_drive_data(); os_clear_drive_data();
} }*/
} }
void ImGui_Debug_Panel () { using namespace ImGui; void ImGui_Debug_Panel () { using namespace ImGui;
@ -338,6 +410,10 @@ void ImGui_Debug_Panel () { using namespace ImGui;
SeparatorText("Child Threads"); SeparatorText("Child Threads");
SeparatorText("Errors"); SeparatorText("Errors");
ArrayView<Error*> errors = get_all_errors(thread_context()); ArrayView<Error*> errors = get_all_errors(thread_context());
if (errors.count && Button("Clear all errors")) {
clear_errors(thread_context());
errors.count = 0;
}
for_each(e, errors) { for_each(e, errors) {
auto button_label = format_cstring("Clear##%d", e); auto button_label = format_cstring("Clear##%d", e);
if (Button(button_label)) { if (Button(button_label)) {