MusaMahmood/mcap
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity
2d0fb855b1
mcap/modules/Basic/arena_platform.jai

495 lines
20 KiB
Plaintext
Raw Blame History

arena_delete :: (arena: *Arena) {
if (!is_valid(arena)) return;
#if ARENA_DEBUG {
free(arena.label,, context.default_allocator);
free(arena.file_path,, context.default_allocator);
}
bootstrapped := arena_is_bootstrapped(arena);
#if OS == .WINDOWS {
success := VirtualFree(arena.memory_base, 0, MEM_RELEASE) != .FALSE;
if !success {
value, description := System.get_error_value_and_string();
log_error("[Basic/Arena] In arena_delete, VirtualFree failed with code %: %\n", value, description);
return;
}
}
else #if OS_IS_UNIX {
success := P.munmap(arena.memory_base, cast(u64)reserve_size_bytes(arena)) == 0;
if !success {
value, description := System.get_error_value_and_string();
log_error("[Basic/Arena] In arena_delete, munmap failed with code %: %\n", value, description);
return;
}
}
if !bootstrapped arena.memory_base = null;
}
// Windows MEM_LARGE_PAGES and Darwin superpages both commit and wire the whole reservation at
// map time, and both refuse a partial decommit of it. Linux hugetlb pages can still be dropped
// with madvise, so only these two make free_pages_down_to a no-op.
LARGE_PAGES_ARE_FULLY_COMMITTED :: OS == .WINDOWS || OS == .MACOS;
#scope_module
UNQUERIED :: -1; // Sentinel for the cached large page size, which is legitimately 0 when unsupported.
// Whether this arena locks and unlocks its own pages as it commits and decommits them.
//
// Large pages are non-pageable by construction, so a .Secure_Arena that got them already holds
// the residency guarantee lock_range exists to provide. Locking them again would at best be a
// wasted syscall on the allocation path and at worst a spurious .Secure_Lock_Failed -- and the
// unlock paths have to agree, or they would unlock a range that was never locked.
arena_manages_page_locks :: (arena: *Arena) -> bool #expand {
return arena_is_secure(arena) && !arena_large_pages_active(arena);
}
// Whether this arena should try for large pages, and at what granularity. Only what the OS call
// itself requires is checked: there is deliberately no ceiling on the reserve class, so a caller
// asking for large pages on a huge reservation gets what it asked for on the platforms that
// commit the whole thing.
large_page_granularity_for :: (arena: *Arena, size: s64) -> (page_size: s64, eligible: bool) {
if !arena_is_large_pages(arena) return 0, false;
large := platform_large_page_size();
if large <= 0 || (large & (large - 1)) != 0 {
// Every real large page size is a power of two; refusing a reported non-power-of-two keeps
// the log2 that all the page arithmetic runs on well defined.
log_warning("[Basic/Arena] .Large_Pages was requested but this machine reports no usable large page size (got %). Using % byte pages.",
large, PLATFORM_MEMORY_PAGE_SIZE);
return 0, false;
}
if size < large || (size % large) != 0 {
log_warning("[Basic/Arena] .Large_Pages was requested for a % byte reservation, which is not a whole multiple of the % byte large page size. Using % byte pages.",
size, large, PLATFORM_MEMORY_PAGE_SIZE);
return 0, false;
}
return large, true;
}
log2_of_power_of_two :: (value: s64) -> u8 {
assert(value > 0 && (value & (value - 1)) == 0);
result: u8 = 0;
remaining := value;
while remaining != 1 {
remaining >>= 1;
result += 1;
}
return result;
}
ceil_log2 :: (value: s64) -> u8 {
assert(value >= 1);
result: u8 = 0;
while (cast(s64) 1 << result) < value result += 1;
return result;
}
#scope_export
#if OS == .WINDOWS {
platform_large_page_size :: () -> s64 {
// Reports what the CPU supports, independently of whether the process holds
// SeLockMemoryPrivilege. Without the privilege the VirtualAlloc below simply fails, which
// the fallback already handles, and granting it is the main program's job.
if cached_large_page_size == UNQUERIED cached_large_page_size = cast(s64) GetLargePageMinimum();
return cached_large_page_size;
}
arena_platform_init :: (using arena: *Arena, new_reserve: Arena_Reserve) -> bool {
reserve = new_reserve;
size := reserve_size_bytes(new_reserve);
page_size_log2 = PLATFORM_MEMORY_PAGE_SIZE_LOG2;
large_page_size, eligible := large_page_granularity_for(arena, size);
if eligible {
// MEM_LARGE_PAGES has no reserve-only form, so this one call commits and wires all `size`
// bytes. That is why first_uncommitted_page starts at the top: there is nothing left for
// extend_committed_pages to commit, and MEM_DECOMMIT cannot take part of it back.
p := VirtualAlloc(null, cast(u64) size, MEM_RESERVE | MEM_COMMIT | MEM_LARGE_PAGES, PAGE_READWRITE);
if p {
memory_base = p;
page_size_log2 = log2_of_power_of_two(large_page_size);
first_uncommitted_page = memory_base + size;
return true;
}
// Usually a missing SeLockMemoryPrivilege, or no free physical run of `size` contiguous
// bytes. Neither is fatal: the 4 KB reservation below is a working arena.
value, description := System.get_error_value_and_string();
log_warning("[Basic/Arena] Large page VirtualAlloc of % bytes failed with code %: %. Falling back to % byte pages.",
size, value, description, PLATFORM_MEMORY_PAGE_SIZE);
}
if arena_is_large_pages(arena) status |= .Large_Pages_Failed;
memory_base = VirtualAlloc(null, cast(u64) size, MEM_RESERVE, PAGE_READWRITE);
success := memory_base != null;
if !success {
value, description := System.get_error_value_and_string();
log_error("[Basic/Arena] In arena_platform_init, VirtualAlloc failed with code %: %\n", value, description);
return false;
}
first_uncommitted_page = memory_base;
return true;
}
// A fully committed large-page arena never reaches the commit call below: its
// first_uncommitted_page already sits at the top of the reservation, so `delta` cannot be
// positive and this returns at the guard.
extend_committed_pages :: (arena: *Arena, end: *u8) -> bool {
delta := cast(s64)(end - arena.first_uncommitted_page);
if delta <= 0 return true;
size := Align(delta, arena_page_size(arena));
p := VirtualAlloc(arena.first_uncommitted_page, cast(u64)size, MEM_COMMIT, PAGE_READWRITE);
if !p {
value, description := System.get_error_value_and_string();
log_error("[Basic/Arena] In extend_committed_pages, VirtualAlloc failed with code %: %\n", value, description);
return false;
}
if arena_manages_page_locks(arena) {
if !lock_range(arena.first_uncommitted_page, size) arena.status |= .Secure_Lock_Failed;
}
arena.first_uncommitted_page += size;
return true;
}
#scope_module
cached_large_page_size: s64 = UNQUERIED;
lock_range :: (start: *u8, byte_count: s64) -> bool {
if byte_count <= 0 return true;
if cast(bool) VirtualLock(start, cast(u64)byte_count) return true;
value, description := System.get_error_value_and_string();
log_error("[Basic/Arena] VirtualLock of % bytes failed with code %: %\n", byte_count, value, description);
return false;
}
unlock_range :: (start: *u8, byte_count: s64) -> bool {
if byte_count <= 0 return true;
if cast(bool) VirtualUnlock(start, cast(u64)byte_count) return true;
value, description := System.get_error_value_and_string();
log_error("[Basic/Arena] VirtualUnlock of % bytes failed with code %: %\n", byte_count, value, description);
return false;
}
free_pages_down_to :: (arena: *Arena, pages_to_keep: s64) {
if (arena == null) return;
if arena_is_bootstrapped(arena) assert(pages_to_keep >= 1, "Always keep at least 1 page if we're bootstrapping arena");
// A large-page allocation is released as a whole or not at all -- there is no partial
// MEM_DECOMMIT for one -- so such an arena rewinds but never drops below its reservation.
#if LARGE_PAGES_ARE_FULLY_COMMITTED {
if arena_large_pages_active(arena) return;
}
bytes_to_keep := pages_to_keep * arena_page_size(arena);
assert(bytes_to_keep <= reserve_size_bytes(arena), "Requesting to keep more bytes than there is address space!");
bytes_committed := cast(s64)(arena.first_uncommitted_page - arena.memory_base);
if (bytes_to_keep > bytes_committed) {
return; // nothing to do
}
start_address := arena.memory_base + bytes_to_keep;
bytes_to_decommit := cast(u64)(arena.first_uncommitted_page - start_address);
// `VirtualFree(start_address, 0, MEM_DECOMMIT)` decommits the entire region:
if bytes_to_decommit {
// Decommitting drops the lock on these pages anyway; unlocking first keeps the process's
// locked-page accounting honest, so the next commit is not refused against a stale total.
if arena_manages_page_locks(arena) unlock_range(start_address, cast(s64)bytes_to_decommit);
VirtualFree(start_address, bytes_to_decommit, MEM_DECOMMIT);
arena.first_uncommitted_page = start_address;
}
}
#scope_export
arena_lock_pages :: (arena: *Arena) -> bool {
if !is_valid(arena) return false;
// Large pages are already non-pageable, so the guarantee this provides is one the arena
// holds either way. Reporting success without the syscall keeps it paired with the unlock.
if arena_large_pages_active(arena) return true;
byte_count := arena_committed_bytes(arena);
if byte_count == 0 {
log_error("Attempting to lock pages in an arena with no active pages!");
return false;
}
return lock_range(arena.memory_base, byte_count);
}
arena_unlock_pages :: (arena: *Arena) -> bool {
if !is_valid(arena) return false;
if arena_large_pages_active(arena) return true; // Nothing was locked; see arena_lock_pages.
byte_count := arena_committed_bytes(arena);
if byte_count == 0 return true;
return unlock_range(arena.memory_base, byte_count);
}
} else #if OS_IS_UNIX {
P :: #import "POSIX"; // Can remove this dependency since we only use a few things.
platform_large_page_size :: () -> s64 {
if cached_large_page_size != UNQUERIED return cached_large_page_size;
#if OS == .MACOS {
// Darwin superpages are an x86_64-only facility; Apple Silicon exposes no user-facing
// equivalent, so there is nothing to report there.
#if CPU == .X64 {
cached_large_page_size = 2 * 1024 * 1024;
} else {
cached_large_page_size = 0;
}
} else {
cached_large_page_size = linux_default_huge_page_size();
}
return cached_large_page_size;
}
arena_platform_init :: (using arena: *Arena, new_reserve: Arena_Reserve) -> bool {
reserve = new_reserve;
size := reserve_size_bytes(new_reserve);
page_size_log2 = PLATFORM_MEMORY_PAGE_SIZE_LOG2;
PROT :: P.PROT_READ | P.PROT_WRITE;
FLAGS :: P.MAP_PRIVATE | P.MAP_ANONYMOUS;
large_page_size, eligible := large_page_granularity_for(arena, size);
if eligible {
mapped: *void = P.MAP_FAILED;
#if OS == .MACOS {
// There is no mmap flag for superpages; the size class travels in the fd argument.
// Wired and fully committed at map time, the way Windows large pages are.
mapped = P.mmap(null, cast(u64) size, PROT, FLAGS, VM_FLAGS_SUPERPAGE_SIZE_2MB, 0);
} else {
// MAP_HUGETLB draws from the hugetlbfs pool an administrator preallocated with
// vm.nr_hugepages, and returns a huge-page-aligned address. It is the only form that
// actually guarantees the granularity, so it is what gets tried.
mapped = P.mmap(null, cast(u64) size, PROT, FLAGS | P.MAP_HUGETLB, 0, 0);
}
if mapped != P.MAP_FAILED {
memory_base = mapped;
page_size_log2 = log2_of_power_of_two(large_page_size);
#if LARGE_PAGES_ARE_FULLY_COMMITTED {
first_uncommitted_page = memory_base + size; // Wired in full at map time; nothing left to commit.
} else {
first_uncommitted_page = memory_base;
}
return true;
}
// On Linux this is the common case: no preallocated hugetlbfs pool.
value, text := System.get_error_value_and_string();
log_warning("[Basic/Arena] Large page mmap of % bytes failed with code %: %. Falling back to % byte pages.",
size, value, text, PLATFORM_MEMORY_PAGE_SIZE);
}
if arena_is_large_pages(arena) status |= .Large_Pages_Failed;
memory_base = P.mmap(null, cast(u64) size, PROT, FLAGS, 0, 0);
success := memory_base != P.MAP_FAILED;
if !success {
value, text := System.get_error_value_and_string();
log_error("[Basic/Arena] In Arena, mmap failed with code %: %\n", value, text);
memory_base = null;
return false;
}
#if OS == .LINUX {
// Transparent huge pages: strictly advisory. The kernel may back the range with huge pages
// and may split them again, so the arena keeps 4 KB arithmetic and still reports
// arena_large_pages_active as false. It costs one syscall and may recover most of the TLB
// win the hugetlbfs pool would have given, so a caller that asked gets it anyway.
if arena_is_large_pages(arena) P.madvise(memory_base, cast(u64) size, P.MADV_HUGEPAGE);
}
first_uncommitted_page = memory_base;
return true;
}
extend_committed_pages :: (arena: *Arena, end: *u8) -> bool {
delta := end - arena.first_uncommitted_page;
if delta <= 0 return true;
size := Align(delta, arena_page_size(arena));
// mlock forces the pages resident immediately rather than on first touch.
//
// Recorded rather than propagated, for the reason given in the Windows arm: a low
// RLIMIT_MEMLOCK must not become an out-of-memory.
if arena_manages_page_locks(arena) {
if !lock_range(arena.first_uncommitted_page, size) arena.status |= .Secure_Lock_Failed;
}
arena.first_uncommitted_page += size;
return true;
}
#scope_module
cached_large_page_size: s64 = UNQUERIED;
#if OS != .MACOS {
// `Hugepagesize:` in /proc/meminfo is the size MAP_HUGETLB maps at when no explicit size is
// requested. Basic cannot import File -- File imports Basic -- so this goes through libc
// directly, in the style of the nested bindings in posix.jai.
linux_default_huge_page_size :: () -> s64 {
FALLBACK :: 2 * 1024 * 1024; // What x86_64 and 4 KB-granule arm64 both default to.
KEY :: "Hugepagesize:";
fd := P.open("/proc/meminfo".data, P.O_RDONLY);
if fd < 0 return FALLBACK;
defer P.close(fd);
// /proc/meminfo reports a size of 0, so it has to be read rather than stat'd. The key sits
// in the first few hundred bytes in practice; one page is a generous margin.
buffer: [4096] u8 = ---;
bytes_read := P.read(fd, buffer.data, cast(u64) buffer.count);
if bytes_read <= 0 return FALLBACK;
at := 0;
while at + KEY.count <= bytes_read {
matched := true;
for i: 0 .. KEY.count - 1 {
if buffer[at + i] != KEY[i] {
matched = false;
break;
}
}
if matched break;
at += 1;
}
if at + KEY.count > bytes_read return FALLBACK;
cursor := at + KEY.count;
while cursor < bytes_read && buffer[cursor] == #char " " cursor += 1;
kilobytes: s64 = 0; // The file states its own unit, and it is always kB for this key.
while cursor < bytes_read && buffer[cursor] >= #char "0" && buffer[cursor] <= #char "9" {
kilobytes = kilobytes * 10 + cast(s64)(buffer[cursor] - #char "0");
cursor += 1;
}
if kilobytes <= 0 return FALLBACK;
return kilobytes * 1024;
}
}
lock_range :: (start: *u8, byte_count: s64) -> bool {
if byte_count <= 0 return true;
if P.mlock(start, cast(u64)byte_count) == 0 return true;
value, description := System.get_error_value_and_string();
log_error("[Basic/Arena] mlock of % bytes failed with code %: %\n", byte_count, value, description);
return false;
}
unlock_range :: (start: *u8, byte_count: s64) -> bool {
if byte_count <= 0 return true;
if P.munlock(start, cast(u64)byte_count) == 0 return true;
value, description := System.get_error_value_and_string();
log_error("[Basic/Arena] munlock of % bytes failed with code %: %\n", byte_count, value, description);
return false;
}
free_pages_down_to :: (using arena: *Arena, pages_to_keep: s64) {
if (arena == null) return;
if arena_is_bootstrapped(arena) assert(pages_to_keep >= 1, "Always keep at least 1 page if we're bootstrapping arena");
// Darwin superpages are wired at map time and cannot be partially released; Linux hugetlb
// pages can, at huge-page granularity, which page_size_log2 already accounts for.
#if LARGE_PAGES_ARE_FULLY_COMMITTED {
if arena_large_pages_active(arena) return;
}
bytes_to_keep := pages_to_keep * arena_page_size(arena);
assert(bytes_to_keep <= reserve_size_bytes(arena), "Requesting to keep more bytes than there is address space!");
bytes_committed := cast(s64)(arena.first_uncommitted_page - arena.memory_base);
if (bytes_to_keep > bytes_committed) {
return; // nothing to do
}
start_address := arena.memory_base + bytes_to_keep;
bytes_to_decommit := cast(u64)(arena.first_uncommitted_page - start_address);
if bytes_to_decommit {
if arena_manages_page_locks(arena) unlock_range(start_address, cast(s64)bytes_to_decommit);
if P.madvise(start_address, bytes_to_decommit, P.MADV_DONTNEED) != 0 {
value, description := System.get_error_value_and_string();
log_error("[Basic/Arena] In free_pages_down_to, madvise failed with code %: %\n", value, description);
}
arena.first_uncommitted_page = start_address;
}
}
#scope_export
arena_lock_pages :: (arena: *Arena) -> bool {
if !is_valid(arena) return false;
// Large pages are already non-pageable, so the guarantee this provides is one the arena
// holds either way. Reporting success without the syscall keeps it paired with the unlock.
if arena_large_pages_active(arena) return true;
byte_count := arena_committed_bytes(arena);
if byte_count == 0 {
log_error("Attempting to lock pages in an arena with no active pages!");
return false;
}
return lock_range(arena.memory_base, byte_count);
}
arena_unlock_pages :: (arena: *Arena) -> bool {
if !is_valid(arena) return false;
if arena_large_pages_active(arena) return true; // Nothing was locked; see arena_lock_pages.
byte_count := arena_committed_bytes(arena);
if byte_count == 0 return true;
return unlock_range(arena.memory_base, byte_count);
}
}
Align :: inline (n: $T, $$ alignment: s64) -> T {
assert(alignment > 0 && (alignment & (alignment - 1)) == 0);
return (n + alignment-1) & ~(alignment-1);
}
System :: #import "System";
#scope_file
#if OS == .WINDOWS {
kernel32 :: #library,system "kernel32";
VirtualAlloc :: (lpAddress: *void, dwSize: u64, flAllocationType: u32, flProtect: u32) -> *void #foreign kernel32;
VirtualFree :: (lpAddress: *void, dwSize: u64, dwFreeType: u32) -> BOOL #foreign kernel32;
VirtualLock :: (lpAddress: *void, dwSize: u64) -> BOOL #foreign kernel32;
VirtualUnlock :: (lpAddress: *void, dwSize: u64) -> BOOL #foreign kernel32;
GetLargePageMinimum :: () -> u64 #foreign kernel32;
PAGE_READWRITE :: 0x04;
MEM_COMMIT :: 0x00001000;
MEM_RESERVE :: 0x00002000;
MEM_DECOMMIT :: 0x00004000;
MEM_RELEASE :: 0x00008000;
MEM_LARGE_PAGES :: 0x20000000;
BOOL :: enum s32 { FALSE :: 0; TRUE :: 1; }
}
else #if OS == .MACOS {
// From macOS/bindings/mach.jai. mmap takes these in its fd argument rather than its flags.
SUPERPAGE_SIZE_2MB :: 2;
VM_FLAGS_SUPERPAGE_SHIFT :: 16;
VM_FLAGS_SUPERPAGE_SIZE_2MB :: SUPERPAGE_SIZE_2MB << VM_FLAGS_SUPERPAGE_SHIFT;
}
Reference in New Issue View Git Blame Copy Permalink