mcap/modules/Default_Allocator/os_posix.jai

// OS entry points for macOS and Linux (2.0.1: rpmalloc.c:878-1161, the POSIX branches), plus
// the POSIX part of rpmalloc_initialize (2.0.1: rpmalloc.c:2776-2782).
//
// Everything goes through libc on every CPU. The 1.4.4 Jai port's raw syscalls caused review
// findings H4, H7 and M1; none of that code exists here.
//
// #jai deliberate differences from upstream:
//   - A failed map returns null without asserting (finding N1).
//   - macOS decommits with MADV_FREE_REUSABLE, retried on EAGAIN, and commits with
//     MADV_FREE_REUSE. Upstream prefers MADV_DONTNEED, which exists on macOS but does not reduce
//     the process footprint there, so decommit is a no-op on macOS (upstream bug U21).
//   - Every entry point works before initialize(), like os_windows.jai.

// Fills os_map_granularity and os_page_size. Idempotent.
// 2.0.1: rpmalloc.c:2776-2782 (the POSIX part of rpmalloc_initialize)
os_initialize :: () {
    if os_page_size  return;
    os_map_granularity = cast(u64) sysconf_p(_SC_PAGESIZE);
    os_page_size = os_map_granularity;
}

os_set_page_name :: (address: *void, size: u64) {   // 2.0.1: rpmalloc.c:885
    #if OS == .LINUX {
        name := ifx os_huge_pages then global_config.huge_page_name else global_config.page_name;
        if address == MAP_FAILED_P || !name  return;
        // prctl wants a C string; names are short, copy into a terminated buffer.
        buffer: [80] u8;
        n := min(name.count, buffer.count - 1);
        memcpy(buffer.data, name.data, n);
        buffer[n] = 0;
        // If the kernel lacks CONFIG_ANON_VMA_NAME, or the name is invalid, this does nothing.
        prctl_p(PR_SET_VMA, PR_SET_VMA_ANON_NAME, cast(u64) address, size, cast(u64) buffer.data);
    }
}

// #jai fix E9: a mapping that is committed as well. On POSIX a fresh anonymous mapping is usable as it
// is, so this only counts the commit (no madvise); see os_windows.jai for why huge blocks use it.
os_mmap_committed :: (size: u64, alignment: u64, offset: *u64, mapped_size: *u64) -> *void {
    ptr := os_mmap(size, alignment, offset, mapped_size);
    #if ENABLE_DECOMMIT  if ptr  statistics_os_commit(size);
    return ptr;
}

os_mmap :: (size: u64, alignment: u64, offset: *u64, mapped_size: *u64) -> *void {   // 2.0.1: rpmalloc.c:900
    os_initialize();
    #if TEST_HOOKS  if test_take_fault(*test_fail_maps)  return os_map_failed(size, alignment, offset, mapped_size);

    map_size := size + alignment;   // UPSTREAM-BUG U1: wraps for sizes near U64_MAX.
    flags: s32 = MAP_PRIVATE_P | MAP_ANONYMOUS_P | MAP_UNINITIALIZED_P;
    #if OS == .MACOS {
        fd := VM_MAKE_TAG_240;
        if os_huge_pages  fd |= VM_FLAGS_SUPERPAGE_SIZE_2MB;
        ptr := mmap_p(null, map_size, PROT_READ_WRITE_P, flags, fd, 0);
        if (ptr == MAP_FAILED_P || !ptr) && os_huge_pages {
            // Superpage allocations can fail through physical memory fragmentation; fall back
            // to regular pages.
            ptr = mmap_p(null, map_size, PROT_READ_WRITE_P, flags, VM_MAKE_TAG_240, 0);
        }
    } else {
        huge_flags: s32 = 0;
        if os_huge_pages {
            huge_flags = MAP_HUGETLB_P;
            // Explicitly request the detected huge page size; the system default huge page size
            // can be a different (unusable) size like 1 GiB.
            if os_huge_page_shift  huge_flags |= cast(s32)(os_huge_page_shift << MAP_HUGE_SHIFT_P);
        }
        ptr := mmap_p(null, map_size, PROT_READ_WRITE_P, huge_flags | flags, -1, 0);
        if (ptr == MAP_FAILED_P || !ptr) && os_huge_pages {
            // Huge page allocations can fail: fall back to normal pages and advise transparent
            // huge pages (best effort; for example EINVAL when THP is disabled system-wide).
            ptr = mmap_p(null, map_size, PROT_READ_WRITE_P, flags, -1, 0);
            if ptr && ptr != MAP_FAILED_P  madvise_p(ptr, map_size, MADV_HUGEPAGE_P);
        } else if os_thp && ptr && ptr != MAP_FAILED_P {
            // Transparent huge page mode: ask for huge pages without a preallocated pool.
            madvise_p(ptr, map_size, MADV_HUGEPAGE_P);
        }
        os_set_page_name(ptr, map_size);
    }
    if ptr == MAP_FAILED_P  ptr = null;
    if !ptr  return os_map_failed(size, alignment, offset, mapped_size);

    if alignment {
        padding := cast(u64) ptr & (alignment - 1);
        if padding  padding = alignment - padding;
        rpmalloc_assert(padding <= alignment, "Internal failure in padding");
        rpmalloc_assert(!(padding % 8), "Internal failure in padding");
        // Unmap the alignment padding head and tail, so no unusable address space (or huge page
        // reservation) is kept. Both the mmap result and the alignment are multiples of the
        // mapping page size, so head and tail stay page aligned.
        if padding  munmap_p(ptr, padding);
        ptr = pointer_offset(ptr, padding);
        if alignment - padding  munmap_p(pointer_offset(ptr, size), alignment - padding);
        offset.* = 0;
        map_size = size;
    }
    mapped_size.* = map_size;
    statistics_os_map(map_size);   // 2.0.1: rpmalloc.c:1023-1033
    return ptr;
}

// The failure tail of os_mmap (2.0.1: rpmalloc.c:990-998).
// #jai returns null without asserting when there is no map_fail_callback (finding N1).
os_map_failed :: (size: u64, alignment: u64, offset: *u64, mapped_size: *u64) -> *void {
    iface := global_memory_interface;
    if iface && iface.map_fail_callback && iface.map_fail_callback(size + alignment)
        return os_mmap(size, alignment, offset, mapped_size);
    return null;
}

os_commit_on_demand :: inline () -> bool {   // 2.0.1: rpmalloc.c:1040
    #if ENABLE_DECOMMIT  return !global_config.disable_decommit;
    else                 return false;
}

os_mcommit :: (address: *void, size: u64) -> s32 {   // 2.0.1: rpmalloc.c:1049
    #if ENABLE_DECOMMIT {
        #if TEST_HOOKS  if test_take_fault(*test_fail_commits)  return 1;
        // POSIX commits on first touch, so upstream does nothing here (its mprotect is commented
        // out). #jai on macOS, tell the kernel the pages given back with MADV_FREE_REUSABLE are
        // in use again, so the footprint accounting stays right (the pairing Apple documents).
        #if OS == .MACOS  madvise_p(address, size, MADV_FREE_REUSE_P);
        statistics_os_commit(size);   // 2.0.1: rpmalloc.c:1067-1078
    }
    return 0;
}

os_mdecommit :: (address: *void, size: u64) -> s32 {   // 2.0.1: rpmalloc.c:1086
    #if ENABLE_DECOMMIT {
        // Unlike os_mcommit, honor the live flag (the Linux fallback below can set it mid-run).
        if global_config.disable_decommit  return 1;
        #if OS == .MACOS {
            // #jai MADV_FREE_REUSABLE, not MADV_DONTNEED (upstream bug U21).
            ret := madvise_p(address, size, MADV_FREE_REUSABLE_P);
            while ret == -1 && errno_p().* == EAGAIN_P {
                errno_p().* = 0;
                ret = madvise_p(address, size, MADV_FREE_REUSABLE_P);
            }
            failed := ret == -1;
        } else {
            failed := madvise_p(address, size, MADV_DONTNEED_P) != 0;
            if failed && os_huge_pages {
                // MADV_DONTNEED on huge page mappings needs Linux 5.18 or later: keep pages
                // committed and stop further decommit attempts.
                global_config.disable_decommit = true;
                return 1;
            }
        }
        if failed {
            rpmalloc_assert(false, "Failed to decommit virtual memory block");
            return 1;
        }
        statistics_os_decommit(size);   // 2.0.1: rpmalloc.c:1129-1133
    }
    return 0;
}

os_munmap :: (address: *void, offset: u64, mapped_size: u64) {   // 2.0.1: rpmalloc.c:1142
    address = pointer_offset(address, -cast(s64) offset);
    #if ENABLE_UNMAP {
        if munmap_p(address, mapped_size)
            rpmalloc_assert(false, "Failed to unmap virtual memory block");
        statistics_os_unmap(mapped_size);   // 2.0.1: rpmalloc.c:1154-1158
    }
}

os_thread_yield :: () {   // 2.0.1: rpmalloc.c:396 (sched_yield)
    sched_yield_p();
}

// Coarse monotonic time for the huge block cache.
os_monotonic_time_ms :: () -> u64 {   // 2.0.1: rpmalloc.c:1586
    ts: timespec_p;
    clock_gettime_p(CLOCK_MONOTONIC_FOR_CACHE, *ts);
    return cast(u64) ts.tv_sec * 1000 + cast(u64) ts.tv_nsec / 1_000_000;
}

//
// Thread-local heap storage, thread identity and the thread exit hook.
//
// #jai upstream keeps the thread heap in a C11 _Thread_local variable and uses a pthread key only
// for the thread exit destructor. This Jai version has no #thread_local, and keeping the heap in
// the Context would let copied contexts share a heap between threads (review H3), so one pthread
// key does both: it stores the heap pointer and runs the destructor at thread exit. The destructor
// matters more here than upstream: Jai's Thread module never sends THREAD_STOP on POSIX (finding
// N4), so it is what returns a finished Jai thread's heap.

os_tls_create :: (destructor: (value: *void) #c_call) -> bool {
    return pthread_key_create_p(*tls_key, destructor) == 0;   // 2.0.1: rpmalloc.c:2954
}

os_tls_destroy :: () {
    pthread_key_delete_p(tls_key);   // 2.0.1: rpmalloc.c:3102
    tls_key = 0;
}

// #jai fix E10: the calling thread's stack, [low, high); 0, 0 if it cannot be found out (the Context
// cache of the thread heap is then never used).
os_thread_stack_range :: () -> low: u64, high: u64 {
    self := pthread_self_p();
    #if OS == .MACOS {
        high := cast(u64) pthread_get_stackaddr_np_p(self);   // The top: the stack grows down.
        size := pthread_get_stacksize_np_p(self);
        if !high || size > high  return 0, 0;
        return high - size, high;
    } else {
        attr: [8] u64;   // pthread_attr_t: 56 bytes on x86-64, 64 on ARM64 glibc.
        if pthread_getattr_np_p(self, attr.data) != 0  return 0, 0;
        address: *void;
        size: u64;
        ok := pthread_attr_getstack_p(attr.data, *address, *size) == 0;
        pthread_attr_destroy_p(attr.data);
        if !ok  return 0, 0;
        return cast(u64) address, cast(u64) address + size;
    }
}

os_tls_get :: inline () -> *void {
    return pthread_getspecific_p(tls_key);
}

os_tls_set :: (value: *void) {
    pthread_setspecific_p(tls_key, value);   // 2.0.1: rpmalloc.c:805
}

// #jai upstream reads the thread pointer (__builtin_thread_pointer or fs/gs/tpidr registers,
// rpmalloc.c:763-790). pthread_self is equally unique among live threads and never 0.
// See os_windows.jai (#jai fix U22). POSIX exit() does not run other threads' key destructors, and
// the exiting thread's own run only from pthread_exit, so there is nothing to skip here.
os_process_exiting :: inline () -> bool { return false; }

os_thread_id :: inline () -> u64 {
    return cast(u64) pthread_self_p();
}

// Huge page detection (2.0.1: rpmalloc.c:2831-2892, the Linux and Apple branches).
os_detect_huge_pages :: () {
    #if OS == .LINUX {
        huge_page_size: u64 = 0;
        meminfo: [4096] u8;
        n := os_read_system_file("/proc/meminfo", meminfo.data, meminfo.count);
        if n {
            text := string.{n, meminfo.data};
            at := find_substring(text, "Hugepagesize:");
            if at >= 0  huge_page_size = parse_decimal(advance_string(text, at + 13)) * 1024;
        }
        huge_page_size = usable_huge_page_size(huge_page_size);
        if huge_page_size {
            // Hugepagesize only reports the configured size, not whether any huge pages are
            // available. Probe by mapping one huge page with the flags real mappings use.
            huge_shift: u64 = 0;
            while (cast(u64) 1 << huge_shift) < huge_page_size  huge_shift += 1;
            probe_flags := MAP_PRIVATE_P | MAP_ANONYMOUS_P | MAP_HUGETLB_P | cast(s32)(huge_shift << MAP_HUGE_SHIFT_P);
            probe := mmap_p(null, huge_page_size, PROT_READ_WRITE_P, probe_flags, -1, 0);
            if probe != MAP_FAILED_P && probe {
                munmap_p(probe, huge_page_size);
                os_huge_pages = true;
                os_huge_page_shift = huge_shift;
                os_page_size = huge_page_size;
                os_map_granularity = huge_page_size;
            }
        }
    } else #if OS == .MACOS && CPU == .X64 {
        // Superpages are only supported on x86-64; Apple silicon has no huge page support.
        os_huge_pages = true;
        os_page_size = 2 * 1024 * 1024;
        os_map_granularity = os_page_size;
    }
}

// Transparent huge pages (2.0.1: rpmalloc.c:2921-2936, Linux only).
os_configure_thp :: () {
    #if OS == .LINUX {
        if global_config.enable_thp && !os_huge_pages && !global_config.disable_thp {
            // With THP disabled system-wide ("never") madvise(MADV_HUGEPAGE) does nothing, so
            // report THP as disabled rather than claim it is in use.
            state: [64] u8;
            n := os_read_system_file("/sys/kernel/mm/transparent_hugepage/enabled", state.data, state.count);
            thp_never := n && find_substring(string.{n, state.data}, "[never]") >= 0;
            if !thp_never  os_thp = true;
        }
        if global_config.disable_thp  prctl_p(PR_SET_THP_DISABLE_P, 1, 0, 0, 0);
    }
}

#if OS == .LINUX {
    // Read a small (pseudo) file such as /proc/meminfo into a caller buffer with raw open/read, so
    // nothing allocates during initialisation. Returns the bytes read (the buffer is always zero
    // terminated), or 0.
    os_read_system_file :: (path: string, buffer: *u8, capacity: s64) -> s64 {   // 2.0.1: rpmalloc.c:2684
        if capacity < 2  return 0;
        fd := open_p(path.data, O_RDONLY_P | O_CLOEXEC_P);   // path is a zero-terminated literal
        if fd < 0  return 0;
        total: s64 = 0;
        while total + 1 < capacity {
            got := read_p(fd, buffer + total, cast(u64)((capacity - 1) - total));
            if got <= 0  break;
            total += got;
        }
        close_p(fd);
        buffer[total] = 0;
        return total;
    }

    find_substring :: (text: string, pattern: string) -> s64 {
        if pattern.count > text.count  return -1;
        for i: 0..text.count - pattern.count {
            match := true;
            for j: 0..pattern.count-1  if text[i + j] != pattern[j] { match = false; break; }
            if match  return i;
        }
        return -1;
    }

    advance_string :: (text: string, n: s64) -> string {
        if n >= text.count  return "";
        return string.{text.count - n, text.data + n};
    }

    // strtol(..., 10) for a non-negative number: skips leading blanks, stops at the first non-digit.
    parse_decimal :: (text: string) -> u64 {
        i := 0;
        while i < text.count && (text[i] == #char " " || text[i] == #char "\t")  i += 1;
        value: u64 = 0;
        while i < text.count && text[i] >= #char "0" && text[i] <= #char "9" {
            value = value * 10 + (text[i] - #char "0");
            i += 1;
        }
        return value;
    }
}

#scope_file

libc_rpm :: #library,system "libc";

mmap_p    :: (address: *void, length: u64, prot: s32, flags: s32, fd: s32, offset: s64) -> *void #foreign libc_rpm "mmap";
munmap_p  :: (address: *void, length: u64) -> s32 #foreign libc_rpm "munmap";
madvise_p :: (address: *void, length: u64, advice: s32) -> s32 #foreign libc_rpm "madvise";
sysconf_p :: (name: s32) -> s64 #foreign libc_rpm "sysconf";
sched_yield_p :: () -> s32 #foreign libc_rpm "sched_yield";

timespec_p :: struct {
    tv_sec:  s64;
    tv_nsec: s64;
}
clock_gettime_p :: (clock_id: s32, ts: *timespec_p) -> s32 #foreign libc_rpm "clock_gettime";
// 6 is CLOCK_MONOTONIC_COARSE on Linux (read through the vDSO, as upstream prefers) and
// CLOCK_MONOTONIC on macOS, which has no coarse clock.
CLOCK_MONOTONIC_FOR_CACHE :: 6;

MAP_FAILED_P      :: cast(*void) -1;
PROT_READ_WRITE_P :: 0x1 | 0x2;
MAP_PRIVATE_P     :: 0x0002;

#if OS == .MACOS {
    errno_p :: () -> *s32 #foreign libc_rpm "__error";
    MAP_ANONYMOUS_P      :: 0x1000;
    MAP_UNINITIALIZED_P  :: 0;
    _SC_PAGESIZE         :: 29;
    EAGAIN_P             :: 35;
    MADV_FREE_REUSABLE_P :: 7;
    MADV_FREE_REUSE_P    :: 8;
    VM_MAKE_TAG_240      :: cast(s32)(240 << 24);   // VM_MAKE_TAG(240U)
    VM_FLAGS_SUPERPAGE_SIZE_2MB :: 2 << 16;         // VM_FLAGS_SUPERPAGE_SIZE_2MB (x86-64 macOS)
} else {
    prctl_p :: (option: s32, arg2: u64, arg3: u64, arg4: u64, arg5: u64) -> s32 #foreign libc_rpm "prctl";
    MAP_ANONYMOUS_P      :: 0x20;
    MAP_UNINITIALIZED_P  :: 0x400_0000;   // Only honoured on no-MMU kernels.
    MAP_HUGETLB_P        :: 0x4_0000;
    MAP_HUGE_SHIFT_P     :: 26;
    MADV_DONTNEED_P      :: 4;
    MADV_HUGEPAGE_P      :: 14;
    _SC_PAGESIZE         :: 30;
    PR_SET_VMA           :: 0x5356_4d41;
    PR_SET_VMA_ANON_NAME :: 0;
}

// Thread-local storage and huge page detection.

libc_tls :: #library,system "libc";

#if OS == .MACOS  pthread_key_t_p :: u64;   // unsigned long
else              pthread_key_t_p :: u32;   // unsigned int

tls_key: pthread_key_t_p;

pthread_key_create_p  :: (key: *pthread_key_t_p, destructor: (value: *void) #c_call) -> s32 #foreign libc_tls "pthread_key_create";
pthread_key_delete_p  :: (key: pthread_key_t_p) -> s32 #foreign libc_tls "pthread_key_delete";
pthread_getspecific_p :: (key: pthread_key_t_p) -> *void #foreign libc_tls "pthread_getspecific";
pthread_setspecific_p :: (key: pthread_key_t_p, value: *void) -> s32 #foreign libc_tls "pthread_setspecific";
pthread_self_p        :: () -> *void #foreign libc_tls "pthread_self";
#if OS == .MACOS {
    pthread_get_stackaddr_np_p :: (thread: *void) -> *void #foreign libc_tls "pthread_get_stackaddr_np";
    pthread_get_stacksize_np_p :: (thread: *void) -> u64 #foreign libc_tls "pthread_get_stacksize_np";
} else {
    pthread_getattr_np_p     :: (thread: *void, attr: *u64) -> s32 #foreign libc_tls "pthread_getattr_np";
    pthread_attr_getstack_p  :: (attr: *u64, address: **void, size: *u64) -> s32 #foreign libc_tls "pthread_attr_getstack";
    pthread_attr_destroy_p   :: (attr: *u64) -> s32 #foreign libc_tls "pthread_attr_destroy";
}

#if OS == .LINUX {
    open_p  :: (path: *u8, flags: s32) -> s32 #foreign libc_tls "open";
    read_p  :: (fd: s32, buffer: *u8, count: u64) -> s64 #foreign libc_tls "read";
    close_p :: (fd: s32) -> s32 #foreign libc_tls "close";
    O_RDONLY_P  :: 0;
    O_CLOEXEC_P :: 0x8_0000;
    PR_SET_THP_DISABLE_P :: 41;
}