Replace multithreaded enumeration with single-threaded (temporarily) #2

Merged
MusaMahmood merged 4 commits from fix-multithreaded-enumeration into main 2025-12-17 15:18:20 +00:00
4 changed files with 128 additions and 57 deletions
Showing only changes of commit 0fc39896f2 - Show all commits

View File

@ -1414,3 +1414,54 @@ bool Deserialize_ST_File_Enumeration (string file_path) {
return true; return true;
} }
// #USNJrnl stuff:
// This should work even if our other indices are not ready yet!
bool USN_Journal_Monitoring_Ready(OS_Drive* drive) {
return (drive->jrnl.hVol != nullptr && drive->jrnl.hVol != INVALID_HANDLE_VALUE);
}
void Win32_Enable_USN_Journal_Monitoring (ArrayView<OS_Drive*> drives) {
push_allocator(temp());
// #TODO: Put any relevant data into Win32_Drive.
for_each(d, drives) {
OS_Drive* drive = drives[d];
if (drive->jrnl.no_permission) continue;
if (USN_Journal_Monitoring_Ready(drive)) continue;
string drive_letter = Win32_drive_letter(drive->label);
string create_file_target = format_string("\\\\.\\%s:", drive_letter.data);
drive->jrnl.hVol = CreateFileA((LPCSTR)create_file_target.data, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr,
OPEN_EXISTING, 0, nullptr);
if (drive->jrnl.hVol == INVALID_HANDLE_VALUE) {
log_error("CreateFileA failed on target %s", create_file_target.data);
os_log_error();
drive->jrnl.no_permission = true;
}
}
}
#include <winioctl.h>
void Query_USN_Journal (ArrayView<OS_Drive*> drives) {
Win32_Enable_USN_Journal_Monitoring(drives);
for_each(d, drives) {
OS_Drive* drive = drives[d];
if (!USN_Journal_Monitoring_Ready(drive)) continue;
USN_JOURNAL_DATA_V0 usn_jd;
DWORD bytes_returned;
BOOL ok = DeviceIoControl(drive->jrnl.hVol, FSCTL_QUERY_USN_JOURNAL,
nullptr, 0,
&usn_jd, sizeof(usn_jd),
&bytes_returned,
nullptr);
if (!ok) {
log_error("DeviceIoControl failed on target %s", drive->label.data);
os_log_error();
return;
}
log("[DeviceIoControl] target %s", drive->label.data);
log(" > Journal ID: %llu", usn_jd.UsnJournalID);
log(" > First USN: %llu", usn_jd.FirstUsn);
debug_break(); // #TODO #continue
}
}

View File

@ -176,6 +176,13 @@ File_System Win32_filesystem_from_string (string s) {
return File_System::Unknown; return File_System::Unknown;
} }
struct Dense_FS; // #hack forward declare! struct Dense_FS; // #hack forward declare!
struct NTFS_USN_Journal {
bool no_permission;
HANDLE hVol;
// ArrayView<USN_Journal_Change> changes;
};
struct Win32_Drive { struct Win32_Drive {
string label; string label;
string volume_name; string volume_name;
@ -192,6 +199,8 @@ struct Win32_Drive {
s64 file_count; s64 file_count;
f32 time_to_enumerate; f32 time_to_enumerate;
Dense_FS* data; Dense_FS* data;
NTFS_USN_Journal jrnl;
}; };
typedef Win32_Drive OS_Drive; typedef Win32_Drive OS_Drive;

View File

@ -585,3 +585,52 @@ bool Serialize_Win32_Drives (ArrayView<Win32_Drive*> drives, string file_path) {
return true; return true;
} }
/*
void Ex1_show_ntfs_workspace () { using namespace ImGui;
push_allocator(temp());
for_each(d, ntfs_workspace.drives) {
OS_Drive* drive = ntfs_workspace.drives[d];
Text("%d. %s paths: %lld, files: %lld",
d, drive->label.data,
drive->data->paths.offsets->count,
drive->data->files.offsets->count);
}
// SliderInt("Results to Show", &ntfs_workspace.results_to_show, 0, 50);
for_each(d, ntfs_workspace.drives) {
OS_Drive* drive = ntfs_workspace.drives[d];
// #TODO: Radio button for choosing between paths, files
char* rb1 = format_cstring("paths##%s", drive->label.data);
RadioButton(rb1, &ntfs_workspace.supplementary[d].radio_button, 1);
SameLine();
char* rb2 = format_cstring("files##%s", drive->label.data);
RadioButton(rb2, &ntfs_workspace.supplementary[d].radio_button, 0);
SameLine();
s32 max_count = (s32)drive->data->paths.offsets->count;
if (ntfs_workspace.supplementary[d].radio_button == 0) {
max_count = (s32)drive->data->files.offsets->count;
}
char* slider_label = format_cstring("%s index", drive->label.data);
if (SliderInt(slider_label, &ntfs_workspace.supplementary[d].index, 0, max_count)) { }
}
for_each(d, ntfs_workspace.drives) {
if (ntfs_workspace.supplementary[d].radio_button == 0) { // files
OS_Drive* drive = ntfs_workspace.drives[d];
Dense_FS* dfs = drive->data;
DFS_Array* dfsa = &drive->data->files;
s64 file_index = ntfs_workspace.supplementary[d].index;
DFS_Value v = get_value(dfs, dfsa, file_index);
// #TODO NOTE: v.full_path is NOT the full path #rename
Text("Filename: %s, parent_id: %d", copy_string(v.full_path).data, v.parent_index);
string full_path = get_full_path_from_index(drive, dfsa, file_index);
Text("Full path: %s", full_path.data);
bool success = file_length(full_path, &v.size); // temp, obviously we don't wanna call this every frame lol
Text(" > size: %lld B", v.size);
Text(" > size: %s", format_bytes(v.size).data);
// Text(" > modtime: %s",
} else {
// DFS_Array* dfsa = &ntfs_workspace.drives[d]->data->paths;
}
}
}
*/

View File

@ -76,54 +76,6 @@ bool Ex1_check_key_combinations() {
return false; return false;
} }
void Ex1_show_ntfs_workspace () { using namespace ImGui;
push_allocator(temp());
for_each(d, ntfs_workspace.drives) {
OS_Drive* drive = ntfs_workspace.drives[d];
Text("%d. %s paths: %lld, files: %lld",
d, drive->label.data,
drive->data->paths.offsets->count,
drive->data->files.offsets->count);
}
// SliderInt("Results to Show", &ntfs_workspace.results_to_show, 0, 50);
for_each(d, ntfs_workspace.drives) {
OS_Drive* drive = ntfs_workspace.drives[d];
// #TODO: Radio button for choosing between paths, files
char* rb1 = format_cstring("paths##%s", drive->label.data);
RadioButton(rb1, &ntfs_workspace.supplementary[d].radio_button, 1);
SameLine();
char* rb2 = format_cstring("files##%s", drive->label.data);
RadioButton(rb2, &ntfs_workspace.supplementary[d].radio_button, 0);
SameLine();
s32 max_count = (s32)drive->data->paths.offsets->count;
if (ntfs_workspace.supplementary[d].radio_button == 0) {
max_count = (s32)drive->data->files.offsets->count;
}
char* slider_label = format_cstring("%s index", drive->label.data);
if (SliderInt(slider_label, &ntfs_workspace.supplementary[d].index, 0, max_count)) { }
}
for_each(d, ntfs_workspace.drives) {
if (ntfs_workspace.supplementary[d].radio_button == 0) { // files
OS_Drive* drive = ntfs_workspace.drives[d];
Dense_FS* dfs = drive->data;
DFS_Array* dfsa = &drive->data->files;
s64 file_index = ntfs_workspace.supplementary[d].index;
DFS_Value v = get_value(dfs, dfsa, file_index);
// #TODO NOTE: v.full_path is NOT the full path #rename
Text("Filename: %s, parent_id: %d", copy_string(v.full_path).data, v.parent_index);
string full_path = get_full_path_from_index(drive, dfsa, file_index);
Text("Full path: %s", full_path.data);
bool success = file_length(full_path, &v.size); // temp, obviously we don't wanna call this every frame lol
Text(" > size: %lld B", v.size);
Text(" > size: %s", format_bytes(v.size).data);
// Text(" > modtime: %s",
} else {
// DFS_Array* dfsa = &ntfs_workspace.drives[d]->data->paths;
}
}
}
// #Workspaces are FOR DEVELOPMENT ONLY. // #Workspaces are FOR DEVELOPMENT ONLY.
struct Ex1_Workspace { struct Ex1_Workspace {
s32 path_select; s32 path_select;
@ -139,8 +91,10 @@ struct Ex1_Workspace {
ArrayView<string> files_sorted_by_modtime; ArrayView<string> files_sorted_by_modtime;
}; };
void reorder_files_by_radix (RadixSort* r, ArrayView<string>* files, bool reverse_order=false) { // #TODO: Move all sort stuff to OS_Win32?
Timed_Block_Print("reorder_files_by_radix"); // Make a general version of this that takes two ArrayView<T> and reorders.
void os_win32_reorder_files_by_radix (RadixSort* r, ArrayView<string>* files, bool reverse_order=false) {
Timed_Block_Print("os_win32_reorder_files_by_radix");
// Where are my source files!? // Where are my source files!?
(*files) = ArrayView<string>(r->ranks.count); (*files) = ArrayView<string>(r->ranks.count);
for_each(f, (*files)) { for_each(f, (*files)) {
@ -176,8 +130,8 @@ void Ex1_show_enumeration_workspace () { using namespace ImGui;
ArrayView<u64> dirs_modtimes = to_view(*stfe->dirs.modtimes); ArrayView<u64> dirs_modtimes = to_view(*stfe->dirs.modtimes);
radix_sort_u64(&ex1w.dir_modtime_radix, dirs_modtimes.data, (u32)dirs_modtimes.count); radix_sort_u64(&ex1w.dir_modtime_radix, dirs_modtimes.data, (u32)dirs_modtimes.count);
// Create ArrayView<string>, ArrayView<u64> sizes, and ArrayView<u64> modtimes // Create ArrayView<string>, ArrayView<u64> sizes, and ArrayView<u64> modtimes
reorder_files_by_radix(&ex1w.file_size_radix, &ex1w.files_sorted_by_size); os_win32_reorder_files_by_radix(&ex1w.file_size_radix, &ex1w.files_sorted_by_size);
reorder_files_by_radix(&ex1w.file_modtime_radix, &ex1w.files_sorted_by_modtime); os_win32_reorder_files_by_radix(&ex1w.file_modtime_radix, &ex1w.files_sorted_by_modtime);
// reordering by the rank permutations generated by RadixSort. // reordering by the rank permutations generated by RadixSort.
ex1w.sort_completed = true; ex1w.sort_completed = true;
} }
@ -213,15 +167,23 @@ void Ex1_Control_Panel () { using namespace ImGui;
if (/*Button("Discover drives") ||*/!table_is_valid(drive_table)) { Win32_Discover_Drives(); } if (/*Button("Discover drives") ||*/!table_is_valid(drive_table)) { Win32_Discover_Drives(); }
// Text("ntfs_workspace_files_loaded: %s", ntfs_workspace_files_loaded()? "true": "false"); // Text("ntfs_workspace_files_loaded: %s", ntfs_workspace_files_loaded()? "true": "false");
if (ntfs_workspace_files_loaded()) { // if (ntfs_workspace_files_loaded()) {
Ex1_show_ntfs_workspace(); // Ex1_show_ntfs_workspace();
return; // return;
} // }
bool all_drives_enumerated = stfe && stfe->thread_completed; bool all_drives_enumerated = stfe && stfe->thread_completed;
push_allocator(temp()); push_allocator(temp());
ArrayView<OS_Drive*> drives = os_get_available_drives(); // only includes drives that are ready. ArrayView<OS_Drive*> drives = os_get_available_drives(); // only includes drives that are ready.
if (!USN_Journal_Monitoring_Ready(drives[0]) && Button("Enable USN Monitoring for all drives")) {
Win32_Enable_USN_Journal_Monitoring(drives);
}
if (USN_Journal_Monitoring_Ready(drives[0]) && Button("Query USN Journal")) {
Query_USN_Journal(drives);
}
if (!all_drives_enumerated) { if (!all_drives_enumerated) {
// Text("drive_table is valid: %d", table_is_valid(drive_table)); // Text("drive_table is valid: %d", table_is_valid(drive_table));
for_each(i, drives) { for_each(i, drives) {