diff --git a/lib/Base/Arena_Free_List.cpp b/lib/Base/Arena_Free_List.cpp index e3b15b0..0cf87d1 100644 --- a/lib/Base/Arena_Free_List.cpp +++ b/lib/Base/Arena_Free_List.cpp @@ -86,6 +86,17 @@ void release_arena (Arena* arena, bool delete_extra_pages) { // } } +s64 bytes_in_use (ArrayView arenas) { + // does not include overhead from committed pages! + s64 sum = 0; + + for (s64 i = 0; i < arenas.count; i += 1) { + sum += arena_usage_bytes(arenas[i]); + } + + return sum; +} + s64 committed_bytes (ArrayView arenas) { s64 sum = 0; diff --git a/lib/Base/Base.h b/lib/Base/Base.h index 9489411..4d9793b 100644 --- a/lib/Base/Base.h +++ b/lib/Base/Base.h @@ -185,7 +185,7 @@ force_inline s64 Next_Power_Of_Two(s64 v) { #define auto_release(x) \ Auto_Release Concat(_auto_release_guard_, __LINE__)(x) #define auto_release_temp() \ - auto_release(thread_context()->temp); + auto_release(thread_context()->temp) #define thread_context() thread_local_context #define temp() allocator(thread_context()->temp) diff --git a/lib/Base/Expandable_Arena.cpp b/lib/Base/Expandable_Arena.cpp index dbff1ab..b024f10 100644 --- a/lib/Base/Expandable_Arena.cpp +++ b/lib/Base/Expandable_Arena.cpp @@ -109,6 +109,12 @@ Allocator allocator (ExpandableArena* arena_ex) { // #TODO: currently this keeps the final arena's memory. Fix this! void arena_reset_to (ExpandableArena* arena_ex, Arena* last_arena, u8* starting_point) { // going backwards from end of arena list + + if (!arena_ex->next_arenas.count) { + arena_ex->current_point = starting_point; + return; + } + // for (s64 i = arena_ex->next_arenas.count-1; i >= 0; i -= 1) { for_each_reverse(i, arena_ex->next_arenas) { Arena* arena = arena_ex->next_arenas[i]; diff --git a/lib/Base/Serializer.h b/lib/Base/Serializer.h index 3ef80f5..9cfd8ed 100644 --- a/lib/Base/Serializer.h +++ b/lib/Base/Serializer.h @@ -146,4 +146,19 @@ force_inline void ReadString16 (Deserializer* ds, string& s) { // #no_alloc ReadStringView(ds, s, (s64)str_len); } -// ReadString_MakeCopy... +// This is specialized for filesystem storage of strings. +force_inline u32 AddString_NoCount (Serializer* serializer, u8* data, s16 count) { + u32 original_count = (u32)serializer->count; + u8* current_point = &serializer->data[original_count]; + + s64 final_count = serializer->allocated + (count * sizeof(u8)); + + if (serializer->allocated < final_count) { + array_reserve(*serializer, final_count); + } + + memcpy(current_point, data, count * sizeof(u8)); + serializer->count += count * sizeof(u8); + + return original_count; +} \ No newline at end of file diff --git a/lib/Base/Threads.cpp b/lib/Base/Threads.cpp index 253d9c7..df83fe5 100644 --- a/lib/Base/Threads.cpp +++ b/lib/Base/Threads.cpp @@ -70,3 +70,14 @@ struct Thread_Group { bool started = false; bool should_exit = false; }; + +// This might be too slow. +s32 get_thread_index (Thread_Group* group, s32 thread_index) { + for_each(w, group->worker_info) { + if (group->worker_info[w].thread.index == thread_index) { + return (s32)w; // zero-indexed to thread group + } + } + + return -1; +} \ No newline at end of file diff --git a/lib/OS/OS_Filesystem.cpp b/lib/OS/OS_Filesystem.cpp index 940d91a..a33f838 100644 --- a/lib/OS/OS_Filesystem.cpp +++ b/lib/OS/OS_Filesystem.cpp @@ -21,23 +21,6 @@ // if we need ordered insertions and deletes. // -// Returns offset -force_inline u32 AddString_NoCount (Serializer* serializer, u8* data, u8 count) { // #TODO: , bool null_terminate=false - u32 original_count = (u32)serializer->count; - u8* current_point = &serializer->data[original_count]; - - s64 final_count = serializer->allocated + (count * sizeof(u8)); - - if (serializer->allocated < final_count) { - array_reserve(*serializer, final_count); - } - - memcpy(current_point, data, count * sizeof(u8)); - serializer->count += count * sizeof(u8); - - return original_count; -} - constexpr s64 DFS_Preallocation_Count = 4194304; // 2^22 // template diff --git a/lib/OS/OS_Win32.cpp b/lib/OS/OS_Win32.cpp index df937e9..e02be22 100644 --- a/lib/OS/OS_Win32.cpp +++ b/lib/OS/OS_Win32.cpp @@ -4,7 +4,6 @@ // - I think no? Threads should handle their own lifetimes, and the parent threads should ensure child threads are complete // before terminating. -#if OS_WINDOWS constexpr s64 FILETIME_TO_UNIX = 116444736000000000i64; f64 GetUnixTimestamp () { FILETIME fileTime; @@ -31,7 +30,20 @@ u64 FILETIME_to_ticks (FILETIME fileTime) { return ticks; } -#endif +string format_time_datetime (FILETIME ft) { + SYSTEMTIME stUTC, st; + FileTimeToSystemTime(&ft, &stUTC); + SystemTimeToTzSpecificLocalTime(nullptr, &stUTC, &st); + // YYYY-MM-DD- + return format_string("%04u-%02u-%02u %02u:%02u:%02u.%03u", + st.wYear, + st.wMonth, + st.wDay, + st.wHour, + st.wMinute, + st.wSecond, + st.wMilliseconds); +} struct OS_System_Info { // #cpuid @@ -1117,323 +1129,277 @@ string os_get_machine_name () { // [ ] get_mouse_pointer_position // [ ] ... What APIs do I need for Keyboard -struct Enumeration_Work { - string first_directory; - s32 parent_index; - - Arena* thread_arena; // pointer to relevant tctx->arena - // Directories - ArenaArray* d_offsets; - ArenaArray* d_lengths; - ArenaArray* d_parent_indices; - ArenaArray* d_sizes; - ArenaArray* d_modtime; - // Files - ArenaArray* offsets; - ArenaArray* lengths; - ArenaArray* parent_indices; - ArenaArray* sizes; - ArenaArray* modtime; -}; +// #FileEnumerationST -struct Files_Combined_Results { - // ArenaArray full_path; - ArenaArray* name; - ArenaArray* parent_indices; +struct STFE_Results { + Serializer* strings; // Serializer? + ArenaArray* offsets; + ArenaArray* lengths; ArenaArray* sizes; - ArenaArray* modtime; + ArenaArray* modtimes; }; -struct Drive_Enumeration { // master thread struct - Arena* arena; +void init (STFE_Results* results) { + results->strings = (Serializer*)arena_array_new (1024*1024*4*16, Arena_Reserve::Size_2G); + results->offsets = arena_array_new(1024*1024*4, Arena_Reserve::Size_2G); + results->lengths = arena_array_new(1024*1024*4, Arena_Reserve::Size_2G); + results->sizes = arena_array_new(1024*1024*4, Arena_Reserve::Size_2G); + results->modtimes = arena_array_new(1024*1024*4, Arena_Reserve::Size_2G); +} + +struct ST_File_Enumeration { // global state ArrayView drives; Thread* master_thread; - s32 thread_count; + STFE_Results dirs; + STFE_Results files; + s32 directories_enumerated; // going sequentially bool thread_started; bool thread_completed; - Files_Combined_Results paths; - Files_Combined_Results files; - - s32 work_added = 0; - s32 work_completed = 0; + f64 start_time; + f64 end_time; }; -void push_root (Drive_Enumeration* de, string label, s32 index) { - array_add(*de->paths.name, label); - array_add(*de->paths.parent_indices, index); - array_add(*de->paths.sizes, (u64)0); - array_add(*de->paths.modtime, (u64)0); -} +global ST_File_Enumeration* stfe; -global Drive_Enumeration* drive_enumeration; - -string directory_get_full_path (Drive_Enumeration* de, s64 index) { - push_allocator(GPAllocator()); // to copy from String_Builder - Files_Combined_Results* f = &de->paths; - string dir_name = (*f->name)[index]; - s32 parent_index = (*f->parent_indices)[index]; - s32 next_parent = (*f->parent_indices)[parent_index]; - - Array paths; - paths.allocator = temp(); - - array_add(paths, (*f->name)[parent_index]); - - while (parent_index != next_parent) { - parent_index = next_parent; - next_parent = (*f->parent_indices)[parent_index]; - array_add(paths, (*f->name)[parent_index]); - } - - // while (parent_index > -1) { // should be while(true) - // - // s32 next_parent = (*f->parent_indices)[parent_index]; - // if (parent_index == next_parent) break; - // s32 parent_index = next_parent; - // } - - // go in reverse order and add together string - String_Builder* sb = new_string_builder(Arena_Reserve::Size_64K); - for (s64 i = paths.count-1; i >= 0; i -= 1) { - append(sb, paths[i]); - append(sb, "\\"); - } - append(sb, dir_name); - - return builder_to_string(sb); -} - -void update_results (Drive_Enumeration* de, Enumeration_Work* ew) { - // merge results and release resources! - // unfortunately this is a LOT of copying! - for_each(i, (*ew->d_offsets)) { - u8* string_ptr = (ew->thread_arena->memory_base + (*ew->d_offsets)[i]); - string name = {(*ew->d_lengths)[i], string_ptr}; - array_add(*de->paths.name, name); - array_add(*de->paths.parent_indices, (*ew->d_parent_indices)[i]); - array_add(*de->paths.sizes, (*ew->d_sizes)[i]); - array_add(*de->paths.modtime, (*ew->d_modtime)[i]); - } - for_each(i, (*ew->offsets)) { - u8* string_ptr = (ew->thread_arena->memory_base + (*ew->offsets)[i]); - string name = {(*ew->lengths)[i], string_ptr}; - array_add(*de->files.name, name); - array_add(*de->files.parent_indices, (*ew->parent_indices)[i]); - array_add(*de->files.sizes, (*ew->sizes)[i]); - array_add(*de->files.modtime, (*ew->modtime)[i]); - } -} - -void add_record (Enumeration_Work* ew, WIN32_FIND_DATAW* find_data, string name, s32 parent_index=-1) { - u32 offset = (u32)(name.data - ew->thread_arena->memory_base); - bool is_directory = (find_data->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0; - u64 size = ((u64)find_data->nFileSizeHigh << 32) | ((u64)find_data->nFileSizeLow & 0xFFFFFFFF); - +string add_record (ST_File_Enumeration* stfe, string full_path, bool is_directory, WIN32_FIND_DATAW* find_data) { + // return the string copy! if (is_directory) { - array_add((*ew->d_offsets), offset); - array_add((*ew->d_lengths), (s16)name.count); - array_add((*ew->d_parent_indices), parent_index); // #TODO #parent_index - array_add((*ew->d_sizes), size); - array_add((*ew->d_modtime), FILETIME_to_ticks(find_data->ftLastWriteTime)); + STFE_Results* r = &stfe->dirs; + u32 offset = AddString_NoCount(r->strings, full_path.data, (s16)full_path.count); + array_add((*r->offsets), offset); + array_add((*r->lengths), (s16)full_path.count); + // No size for directories. + u64 modtime = FILETIME_to_ticks(find_data->ftLastWriteTime); + array_add((*r->modtimes), modtime); + + string path_copy = {full_path.count, &r->strings->data[offset]}; + return path_copy; } else { - array_add((*ew->offsets), offset); - array_add((*ew->lengths), (s16)name.count); - array_add((*ew->parent_indices), parent_index); // #TODO #parent_index - array_add((*ew->sizes), size); - array_add((*ew->modtime), FILETIME_to_ticks(find_data->ftLastWriteTime)); + STFE_Results* r = &stfe->files; + u32 offset = AddString_NoCount(r->strings, full_path.data, (s16)full_path.count); + array_add((*r->offsets), offset); + array_add((*r->lengths), (s16)full_path.count); + + u64 size = ((u64)find_data->nFileSizeHigh << 32) | ((u64)find_data->nFileSizeLow & 0xFFFFFFFF); + u64 modtime = FILETIME_to_ticks(find_data->ftLastWriteTime); + array_add((*r->sizes), size); + array_add((*r->modtimes), modtime); + + string path_copy = {full_path.count, &r->strings->data[offset]}; + return path_copy; } + + Assert(false); + return {}; } -Thread_Continue_Status file_enumeration_thread_group_proc (Thread_Group* group, Thread* thread, void* work) { - // 1. setup userdata as an Arena*: - Arena* result_arena; - if (!thread->context->userdata) { - result_arena = next_arena(Arena_Reserve::Size_64G); - thread->context->userdata = result_arena; - } else { - result_arena = (Arena*)thread->context->userdata; - } - - Enumeration_Work* enum_work = (Enumeration_Work*)work; - enum_work->thread_arena = (Arena*)thread->context->userdata; - - enum_work->d_offsets = arena_array_new(4096, Arena_Reserve::Size_2M); - enum_work->d_lengths = arena_array_new(4096, Arena_Reserve::Size_2M); - enum_work->d_parent_indices = arena_array_new(4096, Arena_Reserve::Size_2M); - enum_work->d_sizes = arena_array_new(4096, Arena_Reserve::Size_2M); - enum_work->d_modtime = arena_array_new(4096, Arena_Reserve::Size_2M); - - enum_work->offsets = arena_array_new(4096, Arena_Reserve::Size_2M); - enum_work->lengths = arena_array_new(4096, Arena_Reserve::Size_2M); - enum_work->parent_indices = arena_array_new(4096, Arena_Reserve::Size_2M); - enum_work->sizes = arena_array_new(4096, Arena_Reserve::Size_2M); - enum_work->modtime = arena_array_new(4096, Arena_Reserve::Size_2M); - - // Validate thread context? - push_allocator(temp()); - auto_release_temp(); - - // log("file_enumeration_thread_group_proc, thread index: %d", thread->index); - - // MAKE SURE PATH IS NULL TERMINATED! - wstring wildcard_name = utf8_to_wide(format_string("%s\\*", enum_work->first_directory.data)); // #temp - WIN32_FIND_DATAW find_data; - HANDLE h = FindFirstFileExW((LPCWSTR)wildcard_name.data, FindExInfoBasic, &find_data, - FindExSearchNameMatch, nullptr, FIND_FIRST_EX_LARGE_FETCH); - if (h == INVALID_HANDLE_VALUE) { - return Thread_Continue_Status::CONTINUE; - } - - while (true) { - push_arena(result_arena); - string name = wide_to_utf8((u16*)find_data.cFileName); // #NOT_TEMP - bool should_continue = (name.count == 0 || name == "." || name == ".."); - if (should_continue) { - bool success = FindNextFileW(h, &find_data); - if (!success) - break; - continue; - } - - add_record(enum_work, &find_data, name, enum_work->parent_index); - - bool success = FindNextFileW(h, &find_data); - if (!success) break; - } - - FindClose(h); - return Thread_Continue_Status::CONTINUE; +s32 count_paths (ST_File_Enumeration* stfe) { + STFE_Results* r = &stfe->dirs; + return (s32)r->offsets->count; +} +s32 count_files (ST_File_Enumeration* stfe) { + STFE_Results* r = &stfe->files; + return (s32)r->offsets->count; } -s64 multithreaded_file_enumeration_master_proc (Thread* thread) { - auto task = thread_task(Drive_Enumeration); - - push_arena(task->arena); - - Thread_Group* file_enum_thread_group = New(); - - s32 thread_count = os_cpu_physical_core_count(); - - push_allocator(GPAllocator()); - thread_group_init(file_enum_thread_group, thread_count, file_enumeration_thread_group_proc, true); +// #UI #TEMP - just for visualization! +string get_file_copy (ST_File_Enumeration* stfe, s64 index) { + STFE_Results* r = &stfe->files; + Assert(index >= 0 && index < count_files(stfe)); + s64 strlength = (*r->lengths)[index]; + u32 offset = (*r->offsets)[index]; + u8* string_ptr = &r->strings->data[offset]; + string file = {strlength, string_ptr}; + return copy_string(file); +} +string get_path_copy (ST_File_Enumeration* stfe, s64 index) { + STFE_Results* r = &stfe->dirs; + Assert(index >= 0 && index < count_paths(stfe)); + s64 strlength = (*r->lengths)[index]; + u32 offset = (*r->offsets)[index]; + u8* string_ptr = &r->strings->data[offset]; + string path = {strlength, string_ptr}; + return copy_string(path); +} +s64 get_file_size_bytes (ST_File_Enumeration* stfe, s64 index) { + STFE_Results* r = &stfe->files; + return (s64)(*r->sizes)[index]; +} + +FILETIME get_file_modtime (ST_File_Enumeration* stfe, s64 index) { + STFE_Results* r = &stfe->files; + FILETIME ft; + memcpy(&ft, &(*r->modtimes)[index], sizeof(u64)); + return ft; +} + +FILETIME get_path_modtime (ST_File_Enumeration* stfe, s64 index) { + STFE_Results* r = &stfe->dirs; + FILETIME ft; + memcpy(&ft, &(*r->modtimes)[index], sizeof(u64)); + return ft; +} + +s64 win32_file_enum_thread_proc (Thread* thread) { + auto task = thread_task(ST_File_Enumeration); + init(&task->dirs); + init(&task->files); + // while files available? + Array paths_to_enumerate; for_each(d, task->drives) { - auto work = New(GPAllocator()); //replace with arena bootstrap? - work->first_directory = task->drives[d]->label; // this includes the colon-slash, (e.g. `C:\`). - work->parent_index = (s32)d; // #HACK? - // add label root to combined results, so we can look it up later! - push_root(task, work->first_directory, work->parent_index); - - add_work(file_enum_thread_group, work); - task->work_added += 1; - } - - start(file_enum_thread_group); - // set task completed. - - s64 path_index = task->drives.count; - - // #TODO: Get completed work! - while (task->work_completed < task->work_added) { - auto_release_temp(); - ArrayView cw = get_completed_work(file_enum_thread_group); - for_each(i, cw) { - auto ew = (Enumeration_Work*)cw[i]; - update_results(task, ew); - - arena_array_free(*ew->d_offsets, false); - arena_array_free(*ew->d_lengths, false); - arena_array_free(*ew->d_parent_indices, false); - arena_array_free(*ew->d_sizes, false); - arena_array_free(*ew->d_modtime, false); - arena_array_free(*ew->offsets, false); - arena_array_free(*ew->lengths, false); - arena_array_free(*ew->parent_indices, false); - arena_array_free(*ew->sizes, false); - arena_array_free(*ew->modtime, false); - - string_free(ew->first_directory); - internal_free(ew); + string parent_directory = task->drives[d]->label; // includes a trailing slash + if (parent_directory.data[2] == (u8)'\\') { + parent_directory.count -= 1; //#hack to quickly remove trailing slash. } - task->work_completed += (s32)cw.count; + array_add(paths_to_enumerate, parent_directory); - // For each new directory: - // s64 dirs_to_enumerate = task->paths.name->count - path_index; - for (s64 i = path_index; i < task->paths.name->count; i += 1) { - auto work = New(GPAllocator()); - work->first_directory = directory_get_full_path(task, i);// need full name here! - work->parent_index = (s32)i; + while (paths_to_enumerate.count > 0) { + push_allocator(temp()); + auto_release_temp(); + // This needs to be null-terminated: + string next_directory = copy_string(pop(paths_to_enumerate)); // LIFO. maybe not the best way? + wstring wildcard_name = utf8_to_wide(format_string("%s\\*", next_directory.data)); - add_work(file_enum_thread_group, work); + WIN32_FIND_DATAW find_data; + HANDLE h = FindFirstFileExW((LPCWSTR)wildcard_name.data, FindExInfoBasic, &find_data, + FindExSearchNameMatch, nullptr, FIND_FIRST_EX_LARGE_FETCH); + if (h == INVALID_HANDLE_VALUE) { + log_error("FindFirstFileExW failed for %s", wide_to_utf8(wildcard_name.data, (s32)wildcard_name.count).data); + os_log_error(); + continue; + } - task->work_added += 1; - } - path_index = task->paths.name->count; - - Sleep(1); - log("work completed: %d/%d",task->work_completed, task->work_added); - } + while (true) { auto_release_temp(); + string name = wide_to_utf8((u16*)find_data.cFileName); + bool should_continue = (name.count == 0 || name == "." || name == ".."); + if (should_continue) { + bool success = FindNextFileW(h, &find_data); + if (!success) { break; } + continue; + } + + bool is_directory = (find_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0; + string full_path = format_string("%s\\%s", next_directory.data, name.data); + string full_path_copy = add_record(task, full_path, is_directory, &find_data); + if (is_directory) { + array_add(paths_to_enumerate, full_path_copy); + } + + bool success = FindNextFileW(h, &find_data); + if (!success) break; + } // while (true) -> FindNextFileW + + FindClose(h); + } // while (parent_directory) + } // for_each(d, drives) - shutdown(file_enum_thread_group); + task->end_time = GetUnixTimestamp(); - task->thread_completed = true; return 0; } -void initialize (Files_Combined_Results* fcr) { - fcr->name = arena_array_new(4194304, Arena_Reserve::Size_2G); // 2GB @ 16-byte strings => 134.2M entries. 64 might be better here for really large file collections! - fcr->parent_indices = arena_array_new(4194304, Arena_Reserve::Size_2G); - fcr->sizes = arena_array_new(4194304, Arena_Reserve::Size_2G); - fcr->modtime = arena_array_new(4194304, Arena_Reserve::Size_2G); -} - -void run_multithreaded_enumeration_thread () { - // Need some struct to track the state of this operation. - Arena* arena = next_arena(Arena_Reserve::Size_64K); - push_arena(arena); +void os_run_file_enumeration_single_threaded () { + push_allocator(GPAllocator()); - drive_enumeration = New(); - (*drive_enumeration) = { - arena, - os_get_available_drives(), + stfe = New(); + (*stfe) = { + os_get_available_drives(), New(), - os_cpu_physical_core_count(), - 0, false, false, {}, {}, - 0, 0 + STFE_Results(), STFE_Results(), + 0, true, false, GetUnixTimestamp(), 0 }; - initialize(&drive_enumeration->paths); - initialize(&drive_enumeration->files); - - // We start 1 thread to run the thread group and track the threads - string thread_name = "Multithreaded Enumeration: Master Thread"; - bool success = thread_init(drive_enumeration->master_thread, - multithreaded_file_enumeration_master_proc, thread_name); + string thread_name = "Single Thread Enumeration - Master Thread"; + bool success = thread_init(stfe->master_thread, win32_file_enum_thread_proc, thread_name); + if (!success) { + log_error("Failed to initialize thread (stft->master_thread)"); + os_log_error(); + } Assert(success); - thread_start(drive_enumeration->master_thread, drive_enumeration); - drive_enumeration->thread_started = true; + thread_start(stfe->master_thread, stfe); } -bool file_enum_multithreading_started () { - if (drive_enumeration == nullptr) return false; - return drive_enumeration->thread_started; +constexpr u32 STFE_Magic_Number = 0x19075fee; + +bool Serialize_ST_File_Enumeration (string file_path) { + Timed_Block_Print("Serialize_ST_File_Enumeration"); + File f = file_open(file_path, true, false, true); + if (!file_is_valid(f)) return false; + bool success = true; + // #TODO #Serialization Unfortunately, there's a lot of needless copying here + // it would be a lot nicer if we could just write-file in place. idk how to do that though ;_; + Serializer* s = new_serializer(Arena_Reserve::Size_64G); + Add(s, (u32)STFE_Magic_Number); + Add(s, (s32)stfe->drives.count); + // Dirs: + STFE_Results* r = &stfe->dirs; + AddArray(s, to_view(*r->strings)); + AddArray(s, to_view(*r->offsets)); + AddArray(s, to_view(*r->lengths)); + AddArray(s, to_view(*r->modtimes)); + + // Files: + r = &stfe->files; + AddArray(s, to_view(*r->strings)); + AddArray(s, to_view(*r->offsets)); + AddArray(s, to_view(*r->lengths)); + AddArray(s, to_view(*r->sizes)); + AddArray(s, to_view(*r->modtimes)); + + success = file_write(&f, to_view(*s)); + reset_serializer(s); + + file_close(&f); + free_serializer(s); + + return success; } -bool file_enum_multithreading_active () { - if (drive_enumeration == nullptr) return false; - if (drive_enumeration->thread_completed) { - return false; - } - if (drive_enumeration->thread_started) { - return true; - } - return false; -} - -// if (drive_enumeration != nullptr) { -// // Check if task is completed, clean up thread. -// discard arena and zero drive_enumeration. -// } \ No newline at end of file +bool Deserialize_ST_File_Enumeration (string file_path) { + Timed_Block_Print("Deserialize_ST_File_Enumeration"); + push_allocator(GPAllocator()); + if (!stfe) stfe = New(); + (*stfe) = { + {}, + {}, + STFE_Results(), STFE_Results(), + 0, false, false, GetUnixTimestamp(), 0 + }; + + push_allocator(temp()); + auto_release_temp(); + + Deserializer deserializer = read_entire_file(file_path, true); + if (deserializer.count == 0) return false; + auto d = &deserializer; + + u32 magic_number; s32 drive_count; + Read(d, &magic_number); + Assert(magic_number == STFE_Magic_Number); + Read(d, &drive_count); + + init(&stfe->dirs); + init(&stfe->files); + + STFE_Results* r = &stfe->dirs; + ReadToArenaArray(d, r->strings); + ReadToArenaArray(d, r->offsets); + ReadToArenaArray(d, r->lengths); + ReadToArenaArray(d, r->modtimes); + r = &stfe->files; + ReadToArenaArray(d, r->strings); + ReadToArenaArray(d, r->offsets); + ReadToArenaArray(d, r->lengths); + ReadToArenaArray(d, r->sizes); + ReadToArenaArray(d, r->modtimes); + + stfe->thread_started = true; + stfe->thread_completed = true; + stfe->end_time = GetUnixTimestamp(); + + return true; +} \ No newline at end of file diff --git a/src/Base_Entry_Point.cpp b/src/Base_Entry_Point.cpp index c0d2b8e..fd98655 100644 --- a/src/Base_Entry_Point.cpp +++ b/src/Base_Entry_Point.cpp @@ -17,7 +17,7 @@ internal void Main_Entry_Point (int argc, WCHAR **argv); #endif internal void Main_Entry_Point (int argc, WCHAR **argv) { // #entry_point - set_cpu_base_frequency(3200); // REQUIRED FOR TIMING MODULE! will depend on CPU + set_cpu_base_frequency(4000); // REQUIRED FOR TIMING MODULE! will depend on CPU #if BASE_RUN_TESTS run_pre_setup_tests(); // #no_context: context will not be initialized at this point. diff --git a/src/Ex1.cpp b/src/Ex1.cpp index ee31965..69f3c51 100644 --- a/src/Ex1.cpp +++ b/src/Ex1.cpp @@ -116,7 +116,7 @@ void Ex1_show_ntfs_workspace () { using namespace ImGui; bool success = file_length(full_path, &v.size); // temp, obviously we don't wanna call this every frame lol Text(" > size: %lld B", v.size); Text(" > size: %s", format_bytes(v.size).data); - // Text(" > modtime: %s", idk how to convert FILETIME to calendar time + // Text(" > modtime: %s", } else { // DFS_Array* dfsa = &ntfs_workspace.drives[d]->data->paths; @@ -124,7 +124,28 @@ void Ex1_show_ntfs_workspace () { using namespace ImGui; } } +struct Ex1_Workspace { + s32 path_select; + s32 file_select; +}; +global Ex1_Workspace ex1w; + +void Ex1_show_enumeration_workspace () { using namespace ImGui; + push_imgui_window("Enumerated Data Workspace"); + // #TODO SliderInt for each + SliderInt("Select path index", &ex1w.path_select, 0, count_paths(stfe)-1); + Text("%s", get_path_copy(stfe, ex1w.path_select).data); + Text("time modified: %s", format_time_datetime(get_path_modtime(stfe, ex1w.path_select)).data); + // #TODO: modtime (to indextime) + SliderInt("Select file index", &ex1w.file_select, 0, count_files(stfe)-1); + Text("%s", get_file_copy(stfe, ex1w.file_select).data); + Text("size: %s", format_bytes(get_file_size_bytes(stfe, ex1w.file_select)).data); + Text("time modified: %s", format_time_datetime(get_file_modtime(stfe, ex1w.file_select)).data); + // #TODO: size, modtime +} + void Ex1_Control_Panel () { using namespace ImGui; + f64 frame_time = GetUnixTimestamp(); Table* drive_table = get_drive_table(); push_imgui_window("Control Panel"); @@ -138,23 +159,26 @@ void Ex1_Control_Panel () { using namespace ImGui; return; } - Text("drive_table is valid: %d", table_is_valid(drive_table)); + bool all_drives_enumerated = stfe && stfe->thread_completed; push_allocator(temp()); ArrayView drives = os_get_available_drives(); // only includes drives that are ready. - for_each(i, drives) { - OS_Drive* drive = drives[i]; - - Text(" > [%d] drive letter: %s (is_present: %d)", i + 1, drive->label.data, drive->is_present); - if (drive->time_to_enumerate != 0) { - SameLine(); - Text("Enumerated in %.2f seconds", drive->time_to_enumerate); + if (!all_drives_enumerated) { + // Text("drive_table is valid: %d", table_is_valid(drive_table)); + for_each(i, drives) { + OS_Drive* drive = drives[i]; + + Text(" > [%d] drive letter: %s (is_present: %d)", i + 1, drive->label.data, drive->is_present); + if (drive->time_to_enumerate != 0) { + SameLine(); + Text("Enumerated in %.2f seconds", drive->time_to_enumerate); + } + // SameLine(); + // if (Button(format_cstring("Read NTFS MFT Raw##%s", drive->label.data))) { + // push_arena(thread_context()->arena); + // Error* error = NTFS_MFT_read_raw(drive); + // } } - // SameLine(); - // if (Button(format_cstring("Read NTFS MFT Raw##%s", drive->label.data))) { - // push_arena(thread_context()->arena); - // Error* error = NTFS_MFT_read_raw(drive); - // } } s32 drives_enumerated = 0; @@ -167,31 +191,77 @@ void Ex1_Control_Panel () { using namespace ImGui; drives_enumerated += 1; } } - bool all_drives_enumerated = !ex1_ntfs.threads_in_flight.count - && (drives_enumerated == drives.count); + // bool all_drives_enumerated = !ex1_ntfs.threads_in_flight.count + // && (drives_enumerated == drives.count); string file_path = format_string_temp("%s_DriveData.bin", os_get_machine_name().data); + if (!all_drives_enumerated && file_exists(file_path)) { + Deserialize_ST_File_Enumeration(file_path); + } if (drives.count > 0 && !all_drives_enumerated && file_exists(file_path) && Button("Load from file (this machine)")) { - Deserialize_Win32_Drives(file_path); + Deserialize_ST_File_Enumeration(file_path); + // Deserialize_Win32_Drives(file_path); } - if (file_enum_multithreading_started()) { - if (thread_is_done(drive_enumeration->master_thread)) { - push_allocator(GPAllocator()); - // Thread* thread = drive_enumeration->master_thread; - // auto task = thread_task(Drive_Enumeration); - // Nothing to free? - thread_deinit(drive_enumeration->master_thread, true); + // if (file_enum_multithreading_started()) { + // if (thread_is_done(drive_enumeration->master_thread)) { + // push_allocator(GPAllocator()); + // // Thread* thread = drive_enumeration->master_thread; + // // auto task = thread_task(Drive_Enumeration); + // // Nothing to free? + // thread_deinit(drive_enumeration->master_thread, true); + // } + // } + + + + // #FileEnumerationST + if (stfe && stfe->thread_started) { + // print dirs enumerated, etc + if (!stfe->thread_completed) Text("Enumeration Thread Active (elapsed: %s)", format_time_seconds(frame_time-stfe->start_time).data); + if (stfe->dirs.offsets) { + s64 dirs_enumerated = stfe->dirs.offsets->count; + Text("Dirs enumerated: %lld", dirs_enumerated); + // if (!stfe->thread_completed) Text("Current dir: %s", get_last_path_copy(stfe).data); } + if (stfe->files.offsets) { + s64 files_enumerated = stfe->files.offsets->count; + Text("Files enumerated: %lld", files_enumerated); + // if (!stfe->thread_completed) Text("Current file: %s", get_last_file_copy(stfe).data); + } + + if (stfe->thread_completed) { + Text("String bytes stored: %s", format_bytes(stfe->dirs.strings->count + stfe->files.strings->count).data); + Text("Elapsed time: %s", format_time_seconds(stfe->end_time-stfe->start_time).data); + } + } + if (stfe && stfe->thread_started && !stfe->thread_completed) { + Assert(stfe->master_thread != nullptr); + if (thread_is_done(stfe->master_thread)) { + push_allocator(GPAllocator()); + thread_deinit(stfe->master_thread, true); + stfe->thread_completed = true; + // Delete(GPAllocator(), stfe); + // stfe = nullptr; + } + } + + if (all_drives_enumerated && Button("Save enumerated data")) { + if (!Serialize_ST_File_Enumeration(file_path)) { + log_error("Failed to write enumerated files"); + os_log_error(); + } + } + + if (all_drives_enumerated) { + Ex1_show_enumeration_workspace(); } if (drives.count > 0 && !all_drives_enumerated && Button("Enumerate all NTFS drives")) { // && ex1_ntfs.initialized - // if drive count exceeds the number of threads, we need to group them so each thread - // can enumerate multiple drives. - // We need to distribute the drives across our available threads: - { run_multithreaded_enumeration_thread(); - return; - } + os_run_file_enumeration_single_threaded(); + // run_multithreaded_enumeration_thread(); + return; + /*// #NTFS_MFT_RAW push_allocator(GPAllocator()); Array> drive_split; drive_split.allocator = temp(); // this is only needed for this frame @@ -245,9 +315,10 @@ void Ex1_Control_Panel () { using namespace ImGui; thread_data->drives = drive_split[t]; thread_start(thread, thread_data); array_add(ex1_ntfs.threads_in_flight, thread); - } + }*/ } + /* #NTFS_MFT_RAW if (ex1_ntfs.threads_in_flight.count) { Text("Threads in flight: %d", ex1_ntfs.threads_in_flight.count); @@ -267,8 +338,9 @@ void Ex1_Control_Panel () { using namespace ImGui; t -= 1; // check this element index again! } } - } + }*/ + /* #NTFS_MFT_RAW if (ex1_ntfs.threads_started && !ex1_ntfs.threads_in_flight.count) { // All threads are complete, we're free to clean up remaining memory push_allocator(GPAllocator()); @@ -289,7 +361,7 @@ void Ex1_Control_Panel () { using namespace ImGui; if (all_drives_enumerated && Button("Clear all drive data")) { os_clear_drive_data(); - } + }*/ } void ImGui_Debug_Panel () { using namespace ImGui; @@ -338,6 +410,10 @@ void ImGui_Debug_Panel () { using namespace ImGui; SeparatorText("Child Threads"); SeparatorText("Errors"); ArrayView errors = get_all_errors(thread_context()); + if (errors.count && Button("Clear all errors")) { + clear_errors(thread_context()); + errors.count = 0; + } for_each(e, errors) { auto button_label = format_cstring("Clear##%d", e); if (Button(button_label)) {