mcap/modules/Default_Allocator/os_windows.jai

// OS entry points for Windows (2.0.1: rpmalloc.c:878-1161, the PLATFORM_WINDOWS branches),
// plus the Windows part of rpmalloc_initialize (2.0.1: rpmalloc.c:2765-2782).
//
// #jai deliberate differences from upstream, all in the error paths:
//   - A failed map or commit returns null / non-zero without asserting (finding N1). Upstream
//     asserts when no map_fail_callback is set and ENABLE_ASSERTS is on.
//   - Every entry point works before initialize(): it initialises the OS globals on demand and
//     tolerates a null global_memory_interface (used by the test hooks).

// Fills os_map_granularity, os_page_size and os_virtualalloc2. Idempotent.
// 2.0.1: rpmalloc.c:2765-2782 (the PLATFORM_WINDOWS part of rpmalloc_initialize)
os_initialize :: () {
    if os_page_size  return;
    info: SYSTEM_INFO_W;
    GetSystemInfo_w(*info);
    os_map_granularity = info.dwAllocationGranularity;
    os_virtualalloc2 = null;
    // #jai GetModuleHandleA with an ASCII name; upstream uses the W variant. Same module.
    kernelbase := GetModuleHandleA_w("kernelbase.dll".data);
    if kernelbase  os_virtualalloc2 = cast(VirtualAlloc2_Proc) GetProcAddress_w(kernelbase, "VirtualAlloc2".data);
    os_page_size = info.dwPageSize;
}

os_set_page_name :: (address: *void, size: u64) {   // 2.0.1: rpmalloc.c:885 (Linux only; a no-op here)
}

os_mmap :: (size: u64, alignment: u64, offset: *u64, mapped_size: *u64) -> *void {   // 2.0.1: rpmalloc.c:900
    return os_map(size, alignment, offset, mapped_size, false);
}

// #jai fix E9: a mapping that is committed as well, in the same call. Committing a range that is only
// reserved costs about 15 us per 24 MiB, and makes releasing it cost about 48 us instead of 7 us
// (MEM_RESERVE | MEM_COMMIT in one VirtualAlloc: 0 and 7 us). Huge blocks are committed whole at once
// anyway, so heap_allocate_block_huge maps them this way: churn_huge missed the huge cache on a quarter
// of its steps and spent 13 of its 19.6 us per step releasing such mappings.
os_mmap_committed :: (size: u64, alignment: u64, offset: *u64, mapped_size: *u64) -> *void {
    ptr := os_map(size, alignment, offset, mapped_size, true);
    #if ENABLE_DECOMMIT  if ptr  statistics_os_commit(size);
    return ptr;
}

os_map :: (size: u64, alignment: u64, offset: *u64, mapped_size: *u64, commit: bool) -> *void {
    os_initialize();
    #if TEST_HOOKS  if test_take_fault(*test_fail_maps)  return os_map_failed(size, alignment, offset, mapped_size);

    map_size := size + alignment;   // UPSTREAM-BUG U1: wraps for sizes near U64_MAX.
    // Ok to MEM_COMMIT: according to MSDN, "actual physical pages are not allocated unless/until the
    // virtual addresses are actually accessed". But with decommit enabled it is better not to commit
    // immediately and instead commit per page, to avoid saturating the OS commit limit.
    #if ENABLE_DECOMMIT {
        do_commit: u32 = ifx commit || global_config.disable_decommit then cast(u32) MEM_COMMIT_W else 0;
    } else {
        do_commit: u32 = MEM_COMMIT_W;
    }
    base_flags: u32 = MEM_RESERVE_W | do_commit;
    large_flag: u32 = ifx os_huge_pages then cast(u32) MEM_LARGE_PAGES_W else 0;
    ptr: *void;
    if alignment && os_virtualalloc2 {
        // Map an aligned region directly, so no alignment padding is kept per mapping.
        reqs: MEM_ADDRESS_REQUIREMENTS_W;
        reqs.Alignment = alignment;
        param: MEM_EXTENDED_PARAMETER_W;
        param.Type = MemExtendedParameterAddressRequirements;
        param.Pointer = *reqs;
        map_size = size;
        ptr = os_virtualalloc2(null, null, map_size, large_flag | base_flags, PAGE_READWRITE_W, *param, 1);
        if !ptr && large_flag
            ptr = os_virtualalloc2(null, null, map_size, base_flags, PAGE_READWRITE_W, *param, 1);
    } else {
        // Pre Windows 10: over-allocate and offset for alignment below.
        ptr = VirtualAlloc_w(null, map_size, large_flag | base_flags, PAGE_READWRITE_W);
        if !ptr && large_flag
            ptr = VirtualAlloc_w(null, map_size, base_flags, PAGE_READWRITE_W);
    }
    if !ptr  return os_map_failed(size, alignment, offset, mapped_size);

    if alignment {
        padding := cast(u64) ptr & (alignment - 1);
        if padding  padding = alignment - padding;
        rpmalloc_assert(padding <= alignment, "Internal failure in padding");
        rpmalloc_assert(!(padding % 8), "Internal failure in padding");
        ptr = pointer_offset(ptr, padding);
        offset.* = padding;
    }
    mapped_size.* = map_size;
    statistics_os_map(map_size);   // 2.0.1: rpmalloc.c:1023-1033
    return ptr;
}

// The failure tail of os_mmap (2.0.1: rpmalloc.c:990-998).
// #jai returns null without asserting when there is no map_fail_callback (finding N1).
os_map_failed :: (size: u64, alignment: u64, offset: *u64, mapped_size: *u64) -> *void {
    iface := global_memory_interface;
    if iface && iface.map_fail_callback && iface.map_fail_callback(size + alignment)
        return os_mmap(size, alignment, offset, mapped_size);
    return null;
}

os_commit_on_demand :: inline () -> bool {   // 2.0.1: rpmalloc.c:1040
    #if ENABLE_DECOMMIT  return !global_config.disable_decommit;
    else                 return false;
}

os_mcommit :: (address: *void, size: u64) -> s32 {   // 2.0.1: rpmalloc.c:1049
    #if ENABLE_DECOMMIT {
        // Gated by callers on the per-span commit_on_demand flag. Spans committed in full at map
        // time (including large pages, which cannot be partially committed) never reach here.
        failed := false;
        #if TEST_HOOKS  failed = test_take_fault(*test_fail_commits);
        if !failed  failed = !VirtualAlloc_w(address, size, MEM_COMMIT_W, PAGE_READWRITE_W);
        if failed {
            iface := global_memory_interface;
            if iface && iface.map_fail_callback && iface.map_fail_callback(size)
                return os_mcommit(address, size);
            // #jai no assert (finding N1); the caller handles the failure.
            return 1;
        }
        statistics_os_commit(size);   // 2.0.1: rpmalloc.c:1067-1078
    }
    return 0;
}

os_mdecommit :: (address: *void, size: u64) -> s32 {   // 2.0.1: rpmalloc.c:1086
    #if ENABLE_DECOMMIT {
        // Unlike os_mcommit, honor the live flag (the Linux fallback can set it mid-run).
        if global_config.disable_decommit  return 1;
        if !VirtualFree_w(address, size, MEM_DECOMMIT_W) {
            rpmalloc_assert(false, "Failed to decommit virtual memory block");
            return 1;
        }
        statistics_os_decommit(size);   // 2.0.1: rpmalloc.c:1129-1133
    }
    return 0;
}

os_munmap :: (address: *void, offset: u64, mapped_size: u64) {   // 2.0.1: rpmalloc.c:1142
    address = pointer_offset(address, -cast(s64) offset);
    #if ENABLE_UNMAP {
        if !VirtualFree_w(address, 0, MEM_RELEASE_W)
            rpmalloc_assert(false, "Failed to unmap virtual memory block");
        statistics_os_unmap(mapped_size);   // 2.0.1: rpmalloc.c:1154-1158
    }
}

// VirtualAlloc2 is resolved at run time from kernelbase.dll (Windows 10 and later).
VirtualAlloc2_Proc :: #type (process: *void, base_address: *void, size: u64, allocation_type: u32, page_protection: u32,
                             parameters: *MEM_EXTENDED_PARAMETER_W, parameter_count: u32) -> *void #c_call;
os_virtualalloc2: VirtualAlloc2_Proc;   // 2.0.1: rpmalloc.c:77

MEM_ADDRESS_REQUIREMENTS_W :: struct {
    LowestStartingAddress: *void;
    HighestEndingAddress:  *void;
    Alignment:             u64;
}

// MEM_EXTENDED_PARAMETER: a 64-bit word (Type:8, Reserved:56) followed by a pointer-sized union.
MEM_EXTENDED_PARAMETER_W :: struct {
    Type:    u64;
    Pointer: *void;
}

os_thread_yield :: () {   // 2.0.1: rpmalloc.c:396 (SwitchToThread)
    SwitchToThread_w();
}

// Coarse monotonic time for the huge block cache.
os_monotonic_time_ms :: inline () -> u64 {   // 2.0.1: rpmalloc.c:1586
    return GetTickCount64_w();
}

//
// Thread-local heap storage, thread identity and the thread exit hook.
//
// #jai upstream keeps the thread heap in a C11 _Thread_local variable and uses FLS only for the
// thread exit destructor. This Jai version has no #thread_local, and keeping the heap in the
// Context would let copied contexts share a heap between threads (review H3), so the heap pointer
// lives in an OS TLS slot (TlsAlloc). An FLS slot (FlsAlloc) carries the exit destructor, as
// upstream. Possible performance follow-up: read the TEB slot directly instead of calling TlsGetValue.

os_tls_create :: (destructor: (value: *void) #c_call) -> bool {
    tls_index = TlsAlloc_w();
    if tls_index == TLS_OUT_OF_INDEXES_W  return false;
    fls_index = FlsAlloc_w(destructor);   // 2.0.1: rpmalloc.c:2952
    return true;
}

os_tls_destroy :: () {
    FlsFree_w(fls_index);   // 2.0.1: rpmalloc.c:3099
    fls_index = 0;
    TlsFree_w(tls_index);
    tls_index = TLS_OUT_OF_INDEXES_W;
}

// #jai fix E10: the calling thread's stack, [low, high).
os_thread_stack_range :: () -> low: u64, high: u64 {
    low, high: u64;
    GetCurrentThreadStackLimits_w(*low, *high);
    return low, high;
}

os_tls_get :: inline () -> *void {
    return TlsGetValue_w(tls_index);
}

os_tls_set :: (value: *void) {
    TlsSetValue_w(tls_index, value);
    FlsSetValue_w(fls_index, value);   // 2.0.1: rpmalloc.c:803
}

// #jai upstream returns the TEB address (NtCurrentTeb, rpmalloc.c:762). GetCurrentThreadId is
// equally unique among live threads, never 0, and needs no inline assembly.
// #jai fix U22: true while the process is exiting. ExitProcess stops every other thread first, so a
// lock one of them held stays held; the thread-exit destructor must not wait for it then.
os_process_exiting :: () -> bool {
    return RtlDllShutdownInProgress_w() != 0;
}

os_thread_id :: inline () -> u64 {
    return GetCurrentThreadId_w();
}

// Large page detection (2.0.1: rpmalloc.c:2798-2830).
os_detect_huge_pages :: () {
    token: *void;
    large_page_minimum := GetLargePageMinimum_w();
    if large_page_minimum  OpenProcessToken_w(GetCurrentProcess_w(), TOKEN_ADJUST_PRIVILEGES_W | TOKEN_QUERY_W, *token);
    if token {
        luid: LUID_W;
        if LookupPrivilegeValueA_w(null, "SeLockMemoryPrivilege".data, *luid) {
            privileges: TOKEN_PRIVILEGES_W;
            privileges.PrivilegeCount = 1;
            privileges.Privileges[0].Luid = luid;
            privileges.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED_W;
            if AdjustTokenPrivileges_w(token, 0, *privileges, 0, null, null) {
                if GetLastError_w() == 0  os_huge_pages = true;   // ERROR_SUCCESS
            }
        }
        CloseHandle_w(token);
    }
    // Disable huge pages if the large page size is not a power of two that fits the builtin geometry.
    if os_huge_pages && ((large_page_minimum & (large_page_minimum - 1)) || large_page_minimum > LARGE_PAGE_SIZE)
        os_huge_pages = false;
    if os_huge_pages {
        if large_page_minimum > os_page_size        os_page_size = large_page_minimum;
        if large_page_minimum > os_map_granularity  os_map_granularity = large_page_minimum;
    }
}

// Transparent huge pages are Linux only (2.0.1: rpmalloc.c:2921-2936).
os_configure_thp :: () { }

#scope_file

kernel32_rpm :: #library,system "kernel32";

SYSTEM_INFO_W :: struct {
    wProcessorArchitecture:      u16;
    wReserved:                   u16;
    dwPageSize:                  u32;
    lpMinimumApplicationAddress: *void;
    lpMaximumApplicationAddress: *void;
    dwActiveProcessorMask:       u64;
    dwNumberOfProcessors:        u32;
    dwProcessorType:             u32;
    dwAllocationGranularity:     u32;
    wProcessorLevel:             u16;
    wProcessorRevision:          u16;
}

MemExtendedParameterAddressRequirements :: 1;

MEM_COMMIT_W      :: 0x0000_1000;
MEM_RESERVE_W     :: 0x0000_2000;
MEM_DECOMMIT_W    :: 0x0000_4000;
MEM_RELEASE_W     :: 0x0000_8000;
MEM_LARGE_PAGES_W :: 0x2000_0000;
PAGE_READWRITE_W  :: 0x04;

VirtualAlloc_w     :: (address: *void, size: u64, type: u32, protect: u32) -> *void #foreign kernel32_rpm "VirtualAlloc";
VirtualFree_w      :: (address: *void, size: u64, type: u32) -> s32 #foreign kernel32_rpm "VirtualFree";
GetSystemInfo_w    :: (info: *SYSTEM_INFO_W) #foreign kernel32_rpm "GetSystemInfo";
GetModuleHandleA_w :: (name: *u8) -> *void #foreign kernel32_rpm "GetModuleHandleA";
GetProcAddress_w   :: (module: *void, name: *u8) -> *void #foreign kernel32_rpm "GetProcAddress";
SwitchToThread_w   :: () -> s32 #foreign kernel32_rpm "SwitchToThread";
GetTickCount64_w   :: () -> u64 #foreign kernel32_rpm "GetTickCount64";

// Thread-local storage and huge page detection.

TLS_OUT_OF_INDEXES_W :: 0xFFFF_FFFF;
tls_index: u32 = TLS_OUT_OF_INDEXES_W;
fls_index: u32;

// The Windows LUID is {DWORD LowPart; LONG HighPart}: 4-byte aligned, so LUID_AND_ATTRIBUTES is 12
// bytes and TOKEN_PRIVILEGES.Privileges starts at offset 4. Declaring the LUID as a u64 shifts it
// to offset 8, which made large pages silently fail in the 1.4.4 Jai port (review M5).
LUID_W :: struct {
    LowPart:  u32;
    HighPart: s32;
}
LUID_AND_ATTRIBUTES_W :: struct {
    Luid:       LUID_W;
    Attributes: u32;
}
TOKEN_PRIVILEGES_W :: struct {
    PrivilegeCount: u32;
    Privileges:     [1] LUID_AND_ATTRIBUTES_W;
}
#assert size_of(TOKEN_PRIVILEGES_W) == 16  "TOKEN_PRIVILEGES layout (review M5)";

TOKEN_ADJUST_PRIVILEGES_W :: 0x0020;
TOKEN_QUERY_W             :: 0x0008;
SE_PRIVILEGE_ENABLED_W    :: 0x0002;

kernel32_tls :: #library,system "kernel32";
GetCurrentThreadStackLimits_w :: (low: *u64, high: *u64) #foreign kernel32_tls "GetCurrentThreadStackLimits";
advapi32_tls :: #library,system "advapi32";
ntdll_tls    :: #library,system "ntdll";
RtlDllShutdownInProgress_w :: () -> u8 #foreign ntdll_tls "RtlDllShutdownInProgress";

TlsAlloc_w            :: () -> u32 #foreign kernel32_tls "TlsAlloc";
TlsFree_w             :: (index: u32) -> s32 #foreign kernel32_tls "TlsFree";
TlsGetValue_w         :: (index: u32) -> *void #foreign kernel32_tls "TlsGetValue";
TlsSetValue_w         :: (index: u32, value: *void) -> s32 #foreign kernel32_tls "TlsSetValue";
FlsAlloc_w            :: (callback: (value: *void) #c_call) -> u32 #foreign kernel32_tls "FlsAlloc";
FlsFree_w             :: (index: u32) -> s32 #foreign kernel32_tls "FlsFree";
FlsSetValue_w         :: (index: u32, value: *void) -> s32 #foreign kernel32_tls "FlsSetValue";
GetCurrentThreadId_w  :: () -> u32 #foreign kernel32_tls "GetCurrentThreadId";
GetCurrentProcess_w   :: () -> *void #foreign kernel32_tls "GetCurrentProcess";
GetLargePageMinimum_w :: () -> u64 #foreign kernel32_tls "GetLargePageMinimum";
GetLastError_w        :: () -> u32 #foreign kernel32_tls "GetLastError";
CloseHandle_w         :: (handle: *void) -> s32 #foreign kernel32_tls "CloseHandle";
OpenProcessToken_w        :: (process: *void, access: u32, token: **void) -> s32 #foreign advapi32_tls "OpenProcessToken";
LookupPrivilegeValueA_w   :: (system: *u8, name: *u8, luid: *LUID_W) -> s32 #foreign advapi32_tls "LookupPrivilegeValueA";
AdjustTokenPrivileges_w   :: (token: *void, disable_all: s32, new_state: *TOKEN_PRIVILEGES_W, buffer_length: u32, previous: *void, return_length: *u32) -> s32 #foreign advapi32_tls "AdjustTokenPrivileges";