MusaMahmood/mcap
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity
da616d2802
mcap/modules/Default_Allocator/rpmalloc.jai
Musa Mahmood 22cad98500 [0.1.15] Startup: a catalog cache, the catalog on the background threads, rpmalloc, and [startup] timings 
The catalog is kept in .catalog-cache in the library, each entry with the size and modified time of
the file it was read from; a start takes the entries whose file is unchanged from it and reads only
the rest. It is saved once the catalog is complete and has changed (at most every 30 s) and at
shutdown. After a reboot the catalog was complete 145 ms after launch, against 3.5 s reading every
file.

The catalog loads on the background Thread_Group, every worker of it, instead of the four preview
workers. rpmalloc (modules/Default_Allocator, 2.0.1-jai.16) takes the place of Jai's single locked
heap: entries from the cache 380 -> 50 ms, and reading the files now scales with threads.
thread_idle is called where workers idle. check_targets.jai checks other OSes with Jai's own
allocator, as rpmalloc runs at compile time with the target's OS layer.

-cold drops the library from the system's file cache first; -no-catalog-cache and -catalog-threads N
are for measuring. [startup] log lines time the listing, the first frame, the cache, the first
preview and the complete catalog.
2026-10-13 07:26:42 +01:00

3491 lines
167 KiB
Plaintext
Raw Blame History

// The allocator core: a port of rpmalloc 2.0.1 rpmalloc.c.
//
// Layout follows the C file section by section, in the same order. Procedures keep the C name
// (minus rpmalloc_ prefixes) and carry a `// 2.0.1: rpmalloc.c:NNNN` reference, so the port can be
// compared with upstream and later upstream fixes applied mechanically.
//
// Deliberate differences from C are marked `// #jai` with the reason. Known upstream bugs that are
// ported as-is, until their fix lands as a separate commit, are marked `// UPSTREAM-BUG <id>`; the
// ids and the fix order are in PLAN.md.
////////////
///
/// Build time configurable limits (2.0.1: rpmalloc.c:135-190)
///
//////
// Module parameters replace the C #defines (ENABLE_VALIDATE_ARGS, ENABLE_ASSERTS, ENABLE_UNMAP,
// ENABLE_DECOMMIT, ENABLE_STATISTICS, ENABLE_LEAK_DETECTION); see module.jai. ENABLE_OVERRIDE
// and ENABLE_DYNAMIC_LINK are C-only and are not ported.
////////////
///
/// Built in size configurations (2.0.1: rpmalloc.c:191-241)
///
//////
PAGE_HEADER_SIZE :: 128; // 2.0.1: rpmalloc.c:197
SPAN_HEADER_SIZE :: PAGE_HEADER_SIZE; // 2.0.1: rpmalloc.c:198
// Alignment of each heap control structure within a shared mapping, so adjacent heaps sit on
// separate cache lines (128 bytes covers Apple Silicon and x86 destructive interference).
HEAP_ALIGNMENT :: 128; // 2.0.1: rpmalloc.c:204
SMALL_GRANULARITY :: 16; // 2.0.1: rpmalloc.c:206
SMALL_BLOCK_SIZE_LIMIT :: 4 * 1024; // 2.0.1: rpmalloc.c:208
MEDIUM_SMALL_BLOCK_SIZE_LIMIT :: 32 * 1024; // 2.0.1: rpmalloc.c:209
MEDIUM_LARGE_BLOCK_SIZE_LIMIT :: 256 * 1024; // 2.0.1: rpmalloc.c:210
LARGE_BLOCK_SIZE_LIMIT :: 2 * 1024 * 1024; // 2.0.1: rpmalloc.c:211
NATURAL_ALIGNMENT_LIMIT :: 2048; // #jai upstream PR #424: largest alignment a class gets from its start offset.
SMALL_SIZE_CLASS_COUNT :: 73; // 2.0.1: rpmalloc.c:213
MEDIUM_SMALL_SIZE_CLASS_COUNT :: 12; // 2.0.1: rpmalloc.c:214
MEDIUM_LARGE_SIZE_CLASS_COUNT :: 12; // 2.0.1: rpmalloc.c:215
LARGE_SIZE_CLASS_COUNT :: 12; // 2.0.1: rpmalloc.c:216
SIZE_CLASS_COUNT :: SMALL_SIZE_CLASS_COUNT + MEDIUM_SMALL_SIZE_CLASS_COUNT + MEDIUM_LARGE_SIZE_CLASS_COUNT + LARGE_SIZE_CLASS_COUNT; // 2.0.1: rpmalloc.c:217
SMALL_PAGE_SIZE_SHIFT :: 16; // 2.0.1: rpmalloc.c:220
SMALL_PAGE_SIZE :: 1 << SMALL_PAGE_SIZE_SHIFT;
SMALL_PAGE_MASK :: ~cast(u64)(SMALL_PAGE_SIZE - 1);
MEDIUM_SMALL_PAGE_SIZE_SHIFT :: 20; // 2.0.1: rpmalloc.c:223
MEDIUM_SMALL_PAGE_SIZE :: 1 << MEDIUM_SMALL_PAGE_SIZE_SHIFT;
MEDIUM_SMALL_PAGE_MASK :: ~cast(u64)(MEDIUM_SMALL_PAGE_SIZE - 1);
MEDIUM_LARGE_PAGE_SIZE_SHIFT :: 22; // 2.0.1: rpmalloc.c:226
MEDIUM_LARGE_PAGE_SIZE :: 1 << MEDIUM_LARGE_PAGE_SIZE_SHIFT;
MEDIUM_LARGE_PAGE_MASK :: ~cast(u64)(MEDIUM_LARGE_PAGE_SIZE - 1);
LARGE_PAGE_SIZE_SHIFT :: 24; // 2.0.1: rpmalloc.c:229
LARGE_PAGE_SIZE :: 1 << LARGE_PAGE_SIZE_SHIFT;
LARGE_PAGE_MASK :: ~cast(u64)(LARGE_PAGE_SIZE - 1);
SPAN_SIZE :: 256 * 1024 * 1024; // 2.0.1: rpmalloc.c:233
SPAN_MASK :: ~cast(u64)(SPAN_SIZE - 1); // 2.0.1: rpmalloc.c:234
SPAN_SIZE_SHIFT :: 28; // #jai for the IS_THIS_YOURS span registry
#assert (1 << SPAN_SIZE_SHIFT) == SPAN_SIZE;
// Maximum allocation size to avoid integer overflow (2.0.1: rpmalloc.c:236-240; upstream only
// defines it with ENABLE_VALIDATE_ARGS).
MAX_ALLOC_SIZE :: 0xFFFF_FFFF_FFFF_FFFF - cast(u64) SPAN_SIZE;
// Geometry invariants. Upstream has none, so changing SPAN_SIZE silently breaks the allocator
// (upstream issue #364). #jai checked at compile time.
#assert (SMALL_PAGE_SIZE & (SMALL_PAGE_SIZE - 1)) == 0 && (SPAN_SIZE & (SPAN_SIZE - 1)) == 0 "page and span sizes must be powers of two";
#assert SMALL_PAGE_SIZE < MEDIUM_SMALL_PAGE_SIZE && MEDIUM_SMALL_PAGE_SIZE < MEDIUM_LARGE_PAGE_SIZE && MEDIUM_LARGE_PAGE_SIZE < LARGE_PAGE_SIZE "page sizes must increase by page type";
#assert LARGE_PAGE_SIZE <= SPAN_SIZE "a span must hold at least one large page";
#assert SPAN_SIZE <= 0xFFFF_FFFF "page offsets within a span must fit in 32 bits";
#assert RPMALLOC_MAX_ALIGNMENT < LARGE_PAGE_SIZE "huge aligned blocks must stay within the span header's reach";
#assert PAGE_HEADER_SIZE % SMALL_GRANULARITY == 0 "blocks must start 16-byte aligned";
////////////
///
/// Utility macros (2.0.1: rpmalloc.c:242-281)
///
//////
// EXPECTED / UNEXPECTED have no Jai equivalent and are dropped.
pointer_offset :: inline (ptr: *void, offset: $T) -> *void { // 2.0.1: rpmalloc.c:132
return cast(*u8) ptr + cast,no_check(s64) offset;
}
pointer_diff :: inline (first: *void, second: *void) -> s64 { // 2.0.1: rpmalloc.c:133
return cast(*u8) first - cast(*u8) second;
}
// 2.0.1: rpmalloc.c:256-268. #jai reports through the memory interface's error_callback when one
// is set (upstream declares that callback but never calls it), otherwise through
// context.assertion_failed.
#if ENABLE_ASSERTS {
rpmalloc_assert :: (truth: bool, message: string, loc := #caller_location) {
if truth return;
iface := global_memory_interface;
if iface && iface.error_callback {
iface.error_callback(message);
return;
}
context.handling_assertion_failure = true;
context.assertion_failed(loc, message);
context.handling_assertion_failure = false;
}
} else {
rpmalloc_assert :: ($truth: Code, message: string) #expand {}
}
////////////
///
/// Statistics (2.0.1: rpmalloc.c:282-360)
///
//////
#if ENABLE_STATISTICS {
Rpmalloc_Statistics :: struct { // 2.0.1: rpmalloc.c:290
page_mapped: u64;
page_mapped_peak: u64;
page_commit: u64;
page_decommit: u64;
page_active: u64;
page_active_peak: u64;
huge_alloc: u64;
huge_alloc_peak: u64;
heap_count: u64;
}
global_counters: Rpmalloc_Statistics; // 2.0.1: rpmalloc.c:302 global_statistics (renamed: global_statistics is the public procedure)
// Subtracts from a counter, clamping at zero. Counters are in units of the configured page size,
// and heaps with committed pages are kept across finalize and initialize cycles where the page
// size can change, so strict accounting is not possible.
statistics_sub_saturating :: (counter: *u64, value: u64) { // 2.0.1: rpmalloc.c:308
current := atomic_load(counter);
while true {
next := ifx current > value then current - value else 0;
swapped, actual := compare_and_swap(counter, current, next);
if swapped break;
current = actual;
}
}
// #jai the add-then-raise-the-peak loop that upstream writes out at each site.
statistics_add_with_peak :: (counter: *u64, peak: *u64, value: u64) {
current := atomic_add(counter, value) + value;
observed := atomic_load(peak);
while current > observed {
swapped, actual := compare_and_swap(peak, observed, current);
if swapped break;
observed = actual;
}
}
// Per size class allocation counters, updated only by the owning thread (cross-thread frees are
// deferred and counted when the owner processes them), so plain integers suffice.
// #jai fix U13: 64-bit; upstream's 32-bit counters wrap on long runs (undefined behaviour in C).
Heap_Size_Use :: struct { // 2.0.1: rpmalloc.c:319
alloc_current: s64;
alloc_peak: s64;
alloc_total: s64;
free_total: s64;
}
} // #if ENABLE_STATISTICS
// #jai fix U13: heap statistics are updated by the threads that free into a heap too, so every
// update is atomic (upstream used plain += and -=). Only HEAP_STATISTICS builds pay for it.
// Subtraction adds the two's complement, and a counter can be below zero for a moment (the first
// allocation already sees -16), so the sums wrap on purpose, as upstream's unsigned arithmetic does.
heap_stat_add :: inline (field: *u64, value: u64) { atomic_add_wrapping(field, value); }
heap_stat_sub :: inline (field: *u64, value: u64) { atomic_add_wrapping(field, cast,no_check(u64) -cast,no_check(s64) value); }
// #jai Jai's atomic_add wraps on X64 (lock xadd), but on other CPUs it computes old + value in Jai,
// which traps in a program built with arithmetic_overflow_check = .FATAL.
atomic_add_wrapping :: inline (field: *u64, value: u64) {
#if CPU == .X64 {
atomic_add(field, value);
} else {
old := atomic_load(field);
while true {
new := ifx old > 0xFFFF_FFFF_FFFF_FFFF - value then old - (0xFFFF_FFFF_FFFF_FFFF - value) - 1 else old + value;
swapped, actual := compare_and_swap(field, old, new);
if swapped break;
old = actual;
}
}
}
// The _rpmalloc_stat_* macros (2.0.1: rpmalloc.c:327-360); no-ops without ENABLE_STATISTICS.
stat_inc_alloc :: inline (heap: *Heap, class_idx: u32) {
#if ENABLE_STATISTICS {
su := *heap.size_use[class_idx];
su.alloc_current += 1;
if su.alloc_current > su.alloc_peak su.alloc_peak = su.alloc_current;
su.alloc_total += 1;
}
}
stat_add_free :: inline (heap: *Heap, class_idx: u32, count: u32) {
#if ENABLE_STATISTICS {
su := *heap.size_use[class_idx];
su.alloc_current -= cast(s64) count;
su.free_total += cast(s64) count;
}
}
stat_inc_span_map :: inline (heap: *Heap, page_type: Page_Type) {
#if ENABLE_STATISTICS heap.span_map_calls[cast(u32) page_type] += 1;
}
// The OS layer's counters (2.0.1: rpmalloc.c:1023-1033, 1067-1078, 1129-1133, 1154-1158).
// #jai each skips counting while the page size is still unknown: the white-box tests (W-05, W-06)
// call the OS layer before initialisation, which upstream never does.
statistics_os_map :: inline (map_size: u64) {
if !global_config.page_size return;
#if ENABLE_STATISTICS statistics_add_with_peak(*global_counters.page_mapped, *global_counters.page_mapped_peak, map_size / global_config.page_size);
}
statistics_os_commit :: inline (size: u64) {
if !global_config.page_size return;
#if ENABLE_STATISTICS {
page_count := size / global_config.page_size;
atomic_add(*global_counters.page_commit, page_count);
statistics_add_with_peak(*global_counters.page_active, *global_counters.page_active_peak, page_count);
}
}
statistics_os_decommit :: inline (size: u64) {
if !global_config.page_size return;
#if ENABLE_STATISTICS {
page_count := size / global_config.page_size;
atomic_add(*global_counters.page_decommit, page_count);
statistics_sub_saturating(*global_counters.page_active, page_count);
}
}
statistics_os_unmap :: inline (mapped_size: u64) {
if !global_config.page_size return;
#if ENABLE_STATISTICS {
page_count := mapped_size / global_config.page_size;
statistics_sub_saturating(*global_counters.page_mapped, page_count);
statistics_sub_saturating(*global_counters.page_active, page_count);
}
}
////////////
///
/// Low level abstractions (2.0.1: rpmalloc.c:361-476)
///
//////
// memset_const / memcpy_const are plain memset / memcpy.
// Yield the rest of this thread's quantum, so another runnable thread (for example a lock holder
// that was preempted) can run. A real scheduler yield, unlike the CPU hint in wait_spin.
thread_yield :: inline () { // 2.0.1: rpmalloc.c:396
os_thread_yield();
}
// Number of cheap CPU-pause spins before wait_spin escalates to an OS scheduler yield.
WAIT_SPIN_YIELD_THRESHOLD :: 100; // 2.0.1: rpmalloc.c:406
wait_spin :: inline (spin_count: *u32) { // 2.0.1: rpmalloc.c:410
spin_count.* += 1;
if spin_count.* >= WAIT_SPIN_YIELD_THRESHOLD {
thread_yield();
return;
}
#if CPU == .X64 {
#asm { pause; }
} else #if CPU == .ARM64 {
#bytes .[0x3F, 0x20, 0x03, 0xD5]; // YIELD
}
}
// #jai Basic has no memmove, and faking it with memcpy was review finding M7. Only the
// realloc-of-an-aligned-block paths move overlapping data, so a simple version is enough.
memmove_rp :: (dest: *void, source: *void, count: s64) {
if dest == source || count <= 0 return;
d := cast(*u8) dest;
s := cast(*u8) source;
if d + count <= s || s + count <= d {
memcpy(d, s, count);
} else if d < s {
for i: 0..count-1 d[i] = s[i];
} else {
for < i: count-1..0 d[i] = s[i];
}
}
// Index of the most significant set bit of a non-zero value. Replaces rpmalloc_clz
// (2.0.1: rpmalloc.c:368), which get_size_class uses as 63 - clz(x).
// #jai bsr on x64, which needs no LZCNT support (the MSVC build's __lzcnt64 silently gives wrong
// answers on CPUs without LZCNT, see upstream issue #404). Elsewhere a six-step binary search,
// because Bit_Operations' non-x64 fallback scans bit by bit.
msb_index :: inline (x: u64) -> u32 {
#if CPU == .X64 {
return cast(u32)(bit_scan_reverse(x) - 1);
} else {
n: u32 = 0;
if x >= 1 << 32 { x >>= 32; n += 32; }
if x >= 1 << 16 { x >>= 16; n += 16; }
if x >= 1 << 8 { x >>= 8; n += 8; }
if x >= 1 << 4 { x >>= 4; n += 4; }
if x >= 1 << 2 { x >>= 2; n += 2; }
if x >= 1 << 1 { n += 1; }
return n;
}
}
////////////
///
/// Data types (2.0.1: rpmalloc.c:477-637)
///
//////
Page_Type :: enum u8 { // 2.0.1: rpmalloc.c:495 page_type_t
SMALL; // 64 KiB pages
MEDIUM_SMALL; // 1 MiB pages
MEDIUM_LARGE; // 4 MiB pages
LARGE; // 16 MiB pages
HUGE;
}
Size_Class :: struct { // 2.0.1: rpmalloc.c:504 size_class_t
block_size: u32; // Size of blocks in this class.
block_count: u32; // Number of blocks in each page.
}
Block :: struct { // 2.0.1: rpmalloc.c:512 block_t
next: *Block;
}
// The packed flag word of page_t (2.0.1: rpmalloc.c:532-552). Bits 0-6 are flags; bits 8-13 hold
// commit_prefix_shift (see page_commit_prefix_shift). Packing keeps Page at 72 bytes and Span at
// exactly SPAN_HEADER_SIZE, as upstream.
Page_Flags :: enum_flags u32 {
IS_FULL; // Part of the heap full list.
IS_FREE; // Part of the heap free list.
IS_ZERO; // Blocks are zero initialised.
IS_DECOMMITTED; // Memory pages have been decommitted.
// Has aligned blocks. Note: has_aligned_block and is_full are read on the cross-thread free
// path while the owner thread writes this word. The owner is the only writer, the word is
// naturally aligned and the flags read are monotonic/tolerant, so the race is benign
// (upstream comment, rpmalloc.c:539-545).
HAS_ALIGNED_BLOCK;
COMMIT_ON_DEMAND; // The span maps reserve-only; pages commit on demand and may be decommitted.
}
COMMIT_PREFIX_SHIFT_BIT :: 8;
COMMIT_PREFIX_SHIFT_MASK :: 0x3F;
Page :: struct { // 2.0.1: rpmalloc.c:518 page_t
size_class: u32;
block_size: u32;
block_count: u32;
block_initialized: u32;
block_used: u32;
// Flags, commit_prefix_shift and (#jai PR #422) the page type, see Page_Flags. The page type moved
// into this word, as upstream's PR #422 does, so Page.heap_fast fits: Span stays 128 bytes.
flags: Page_Flags;
local_free: *Block; // Local free list.
heap: *Heap; // Owning heap.
// #jai upstream PR #422 (867ea07): the owning heap while the page takes the fast free path, null
// for huge, full or aligned-block pages, so the free fast path checks ownership and eligibility
// with one compare. (Replaces 2.0.1's generic_free flag and local_free_count, which always equals
// block_initialized - block_used.)
heap_fast: *Heap;
next: *Page;
prev: *Page;
// Multithreaded free list: block index in the low 32 bits, list count in the high 32 bits.
thread_free: u64;
}
Span :: struct { // 2.0.1: rpmalloc.c:568 span_t
page: Page; // Page header of page 0.
// Owning heap. Not deduplicated into page.heap, which is the owner of page 0 as a memory page,
// while the span owner must stay stable.
heap: *Heap;
page_address_mask: u64;
page_initialized: u32; // Number of pages initialized.
page_count: u32; // Number of pages in use (huge: number of OS pages).
page_size: u32; // Bytes per page.
page_type: Page_Type;
offset: u32; // Offset to the start of the mapped region.
mapped_size: u64;
next: *Span;
}
// Control structure for a heap: a thread heap, or a first-class heap.
Heap :: struct { // 2.0.1: rpmalloc.c:594 heap_t
// #jai develop #409 (3302251): the first HEAP_ALIGNMENT bytes hold the fields other threads
// access. owner_thread is read and thread_free is written by every cross-thread free into a full
// page; keeping them apart from the owner's hot allocation state avoids invalidating the owner's
// local free lists on every remote free, and the remote CAS line being pulled away by owner writes.
owner_thread: u64; // Owning thread id (0: first-class heap).
thread_free: [4] u64; // Multithreaded free list (atomic) per type.
// #jai fix E6: taken by frees into the heap while it is released (E4), and by the thread adopting
// it, so those frees no longer take the global heap lock. On the line other threads write anyway.
free_lock: u64;
remote_pad: [HEAP_ALIGNMENT - 2 * size_of(u64) - 4 * size_of(u64)] u8;
local_free: [SIZE_CLASS_COUNT] *Block; // Heap local free list per size class.
page_available: [SIZE_CLASS_COUNT] *Page; // Non-full pages per size class.
page_free: [4] *Page; // Free pages per page type, committed pages first.
page_free_tail: [4] *Page; // #jai PR #420: last page of each free list.
page_free_commit_count: [4] u32; // Free but still committed pages per type.
// #jai develop #416: when (monotonic ms) the free committed page count of each page type first
// went over the overflow threshold; zero while below it.
page_free_overflow_ms: [4] u64;
span_partial: [4] *Span; // Partially initialized spans per type.
span_used: [5] *Span; // Spans in full use per type.
next: *Heap;
prev: *Heap;
id: u32;
finalize: u32;
release_trim: u32; // #jai develop #419: released untrimmed; the adopting thread trims it first.
released: bool; // #jai fix U14: queued or back in the pristine list; set under the heap lock.
// #jai fix E7: the id of the thread that parked the heap with thread_idle, 0 when not parked. Set
// and cleared under the global heap lock and the heap's free_lock.
parked_thread: u64;
offset: u32; // Memory map region offset.
mapped_size: u64; // Memory map size.
#if HEAP_STATISTICS stats: Heap_Statistics; // 2.0.1: rpmalloc.c:624
#if ENABLE_STATISTICS {
size_use: [SIZE_CLASS_COUNT] Heap_Size_Use; // 2.0.1: rpmalloc.c:628
span_map_calls: [5] u32; // 2.0.1: rpmalloc.c:630
}
// #jai handed out by heap_acquire. IS_THIS_YOURS on the default allocator must not claim the
// blocks of first-class heaps; owner_thread cannot tell them apart from released thread heaps.
is_first_class: bool;
// #jai fix E10: for the Context cache of the thread heap. The stack range of the thread whose heap
// this is, and whether it is that thread's heap now (cleared when the thread parks the heap, sets
// another current, or releases it). Written by the owning thread.
stack_low: u64;
stack_high: u64;
thread_current: bool;
}
#assert size_of(Page) <= PAGE_HEADER_SIZE "Invalid page header size"; // 2.0.1: rpmalloc.c:634
#assert #run every_class_fits_after_its_start_offset() "PR #424: a size class no longer fits its page after its block start offset";
every_class_fits_after_its_start_offset :: () -> bool {
page_size_of :: (index: s64) -> u64 {
if index < SMALL_SIZE_CLASS_COUNT return SMALL_PAGE_SIZE;
if index < SMALL_SIZE_CLASS_COUNT + MEDIUM_SMALL_SIZE_CLASS_COUNT return MEDIUM_SMALL_PAGE_SIZE;
if index < SMALL_SIZE_CLASS_COUNT + MEDIUM_SMALL_SIZE_CLASS_COUNT + MEDIUM_LARGE_SIZE_CLASS_COUNT return MEDIUM_LARGE_PAGE_SIZE;
return LARGE_PAGE_SIZE;
}
for global_size_class {
if it.block_size && cast(u64) it.block_count * it.block_size + block_start_offset(it.block_size) > page_size_of(it_index) return false;
}
return true;
}
#assert size_of(Span) <= SPAN_HEADER_SIZE "Invalid span header size"; // 2.0.1: rpmalloc.c:635
// 2.0.1: rpmalloc.c:636. #jai with ENABLE_STATISTICS the 64-bit size class counters (fix U13) make the
// heap larger than 4 KiB; heap_allocate_new rounds each heap up to whole pages, so a statistics build
// simply uses more of the mapping per heap.
#assert ENABLE_STATISTICS || size_of(Heap) <= 4096 "Invalid heap size";
#assert #run heap_member_offset("local_free") == HEAP_ALIGNMENT "Remotely accessed heap fields must fill one line"; // develop #409
heap_member_offset :: (name: string) -> s64 {
for type_info(Heap).members if it.name == name return it.offset_in_bytes;
return -1;
}
page_has :: inline (page: *Page, flag: Page_Flags) -> bool {
return (page.flags & flag) != 0;
}
page_set :: inline (page: *Page, flag: Page_Flags, on: bool) {
if on page.flags |= flag;
else page.flags &= ~flag;
}
PAGE_TYPE_BIT :: 16;
PAGE_TYPE_MASK :: 0x7;
page_get_type :: inline (page: *Page) -> Page_Type {
return cast(Page_Type)(((cast(u32) page.flags) >> PAGE_TYPE_BIT) & PAGE_TYPE_MASK);
}
page_set_type :: inline (page: *Page, page_type: Page_Type) {
bits := (cast(u32) page.flags) & ~((cast(u32) PAGE_TYPE_MASK) << PAGE_TYPE_BIT);
page.flags = cast(Page_Flags)(bits | ((cast(u32) page_type) << PAGE_TYPE_BIT));
}
page_commit_prefix_shift :: inline (page: *Page) -> u32 {
return ((cast(u32) page.flags) >> COMMIT_PREFIX_SHIFT_BIT) & COMMIT_PREFIX_SHIFT_MASK;
}
page_set_commit_prefix_shift :: inline (page: *Page, shift: u32) {
bits := (cast(u32) page.flags) & ~((cast(u32) COMMIT_PREFIX_SHIFT_MASK) << COMMIT_PREFIX_SHIFT_BIT);
page.flags = cast(Page_Flags)(bits | ((shift & COMMIT_PREFIX_SHIFT_MASK) << COMMIT_PREFIX_SHIFT_BIT));
}
// #jai fix E3: commit a page as it fills. Upstream commits a whole page (64 KiB to 16 MiB) when it is
// first used, so a page holding a few long-lived blocks at its start kept all of it committed: in Ex4
// 1.45.0, 84 MB of pages for 8 MB of live blocks, left by 24 short-lived walk threads, whose pages had
// been carved only to 11 MB (RET-17). A page of a reserve-only span is committed in chunks of 1/16 of
// the page, at least 64 KiB (so a small page is one chunk, as before), as page_initialize_blocks carves
// blocks into it. The number of committed chunks is kept in the flags word (Span has no room for a
// field): 0 means only the commit prefix, the first OS page holding the header, is committed (a
// decommitted page). It survives reuse of the page until the page is decommitted.
COMMITTED_CHUNKS_BIT :: 20;
COMMITTED_CHUNKS_MASK :: 0x1F;
COMMIT_CHUNKS_PER_PAGE :: 16;
COMMIT_CHUNK_MIN :: 64 * 1024;
page_committed_chunks :: inline (page: *Page) -> u32 {
return ((cast(u32) page.flags) >> COMMITTED_CHUNKS_BIT) & COMMITTED_CHUNKS_MASK;
}
page_set_committed_chunks :: inline (page: *Page, chunks: u32) {
bits := (cast(u32) page.flags) & ~((cast(u32) COMMITTED_CHUNKS_MASK) << COMMITTED_CHUNKS_BIT);
page.flags = cast(Page_Flags)(bits | ((chunks & COMMITTED_CHUNKS_MASK) << COMMITTED_CHUNKS_BIT));
}
// The commit chunk of a page of this size: 1/16 of it, at least COMMIT_CHUNK_MIN and the commit prefix,
// at most the page.
page_commit_chunk_size :: inline (page_size: u64, commit_prefix: u64) -> u64 {
chunk := page_size / COMMIT_CHUNKS_PER_PAGE;
if chunk < COMMIT_CHUNK_MIN chunk = COMMIT_CHUNK_MIN;
if chunk < commit_prefix chunk = commit_prefix;
if chunk > page_size chunk = page_size;
return chunk;
}
// Commits the page up to `extent` bytes from its start (rounded up to a chunk), if it is not yet.
// Non-zero on failure, with nothing changed. Only for pages of reserve-only spans.
page_commit_to :: (page: *Page, extent: u64) -> s32 {
page_size := page_get_size(page);
commit_prefix := cast(u64) 1 << page_commit_prefix_shift(page);
chunk := page_commit_chunk_size(page_size, commit_prefix);
needed := cast(u32)((extent + chunk - 1) / chunk);
have := page_committed_chunks(page);
if needed <= have return 0;
from := ifx have then cast(u64) have * chunk else commit_prefix;
to := cast(u64) needed * chunk;
if to > page_size to = page_size;
if to > from {
if global_memory_interface.memory_commit(pointer_offset(page, from), to - from) != 0 return 1;
#if HEAP_STATISTICS && ENABLE_DECOMMIT {
if page.heap heap_stat_add(*page.heap.stats.committed_size, to - from);
}
}
page_set_committed_chunks(page, needed);
return 0;
}
// Bytes of the page committed now: what decommit gives back beyond the prefix.
page_committed_size :: inline (page: *Page) -> u64 {
page_size := page_get_size(page);
commit_prefix := cast(u64) 1 << page_commit_prefix_shift(page);
have := page_committed_chunks(page);
if !have return commit_prefix;
size := cast(u64) have * page_commit_chunk_size(page_size, commit_prefix);
return ifx size < page_size then size else page_size;
}
////////////
///
/// Global data (2.0.1: rpmalloc.c:638-728)
///
//////
global_heap_id: u32 = 1; // 2.0.1: rpmalloc.c:664 (atomic) Heap id counter.
// Size classes (2.0.1: rpmalloc.c:685-709). The table is the same list of block sizes in units of
// SMALL_GRANULARITY as upstream's SCLASS/MSCLASS/MLCLASS/LCLASS entries; block counts are computed
// the same way, (page size of the class's page type - PAGE_HEADER_SIZE) / block size.
SIZE_CLASS_UNITS :: u32.[
1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26,
27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50,
51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 80, 96, 112, 128, 160, 192, 224, 256, // SCLASS
320, 384, 448, 512, 640, 768, 896, 1024, 1280, 1536, 1792, 2048, // MSCLASS
2560, 3072, 3584, 4096, 5120, 6144, 7168, 8192, 10240, 12288, 14336, 16384, // MLCLASS
20480, 24576, 28672, 32768, 40960, 49152, 57344, 65536, 81920, 98304, 114688, 131072, // LCLASS
];
#assert SIZE_CLASS_UNITS.count == SIZE_CLASS_COUNT "size class table must have SIZE_CLASS_COUNT entries";
global_size_class :: #run make_size_class_table(); // 2.0.1: rpmalloc.c:693
make_size_class_table :: () -> [SIZE_CLASS_COUNT] Size_Class {
table: [SIZE_CLASS_COUNT] Size_Class;
for units, index: SIZE_CLASS_UNITS {
block_size := units * SMALL_GRANULARITY;
page_size: u32;
if index < SMALL_SIZE_CLASS_COUNT page_size = SMALL_PAGE_SIZE;
else if index < SMALL_SIZE_CLASS_COUNT + MEDIUM_SMALL_SIZE_CLASS_COUNT page_size = MEDIUM_SMALL_PAGE_SIZE;
else if index < SMALL_SIZE_CLASS_COUNT + MEDIUM_SMALL_SIZE_CLASS_COUNT + MEDIUM_LARGE_SIZE_CLASS_COUNT page_size = MEDIUM_LARGE_PAGE_SIZE;
else page_size = LARGE_PAGE_SIZE;
table[index] = .{block_size, (page_size - PAGE_HEADER_SIZE) / block_size};
}
return table;
}
// Threshold number of free pages per page type at which free pages are decommitted, and the
// number of pages to retain when that happens (2.0.1: rpmalloc.c:711-715). Not constant:
// initialisation adjusts the thresholds to the OS page size.
global_page_free_overflow: [5] u32 = .[16, 8, 4, 2, 0];
global_page_free_retain: [5] u32 = .[4, 2, 1, 1, 0];
global_memory_interface: *Memory_Interface; // 2.0.1: rpmalloc.c:676
global_memory_interface_default: Memory_Interface; // 2.0.1: rpmalloc.c:678
global_memory_interface_copy: Memory_Interface; // #jai fix U8: the caller's interface, with OS defaults filled in.
global_config: Config; // 2.0.1: rpmalloc.c:680
os_huge_pages: bool; // 2.0.1: rpmalloc.c:718 Huge (large) pages are in use.
os_thp: bool; // 2.0.1: rpmalloc.c:721 Transparent huge pages are in use (Linux).
os_huge_page_shift: u64; // 2.0.1: rpmalloc.c:723 log2 of the huge page size, for MAP_HUGE_SHIFT.
os_map_granularity: u64; // 2.0.1: rpmalloc.c:725 Granularity of OS mappings.
os_page_size: u64; // 2.0.1: rpmalloc.c:727 OS page size (0 until os_initialize).
////////////
///
/// Thread local heap and ID (2.0.1: rpmalloc.c:729-877)
///
//////
// #jai the thread heap lives in OS thread-local storage (see os_tls_* in os_windows.jai /
// os_posix.jai): this Jai version has no #thread_local, and the Context can be copied between
// threads (review H3).
// The fallback heap: every thread starts on it. It is all zero (id 0), so the allocation fast path
// finds nothing and drops into the slow path, which initialises the allocator and the thread's real
// heap (heap_get_page_generic and heap_allocate_block_huge check heap.id == 0).
global_heap_fallback: Heap; // 2.0.1: rpmalloc.c:645
global_heap_default := *global_heap_fallback; // 2.0.1: rpmalloc.c:647
global_heap_queue: *Heap; // 2.0.1: rpmalloc.c:650 Released heaps; only handed to thread heaps.
global_heap_pristine: *Heap; // 2.0.1: rpmalloc.c:653 Never-used heaps carved from a shared block.
global_heap_used: *Heap; // 2.0.1: rpmalloc.c:655 Heaps in use.
// Set while a thread maps and carves a new shared heap block, so concurrent thread starts wait for
// the carved heaps instead of each mapping a block. A user memory interface callback must not
// allocate through rpmalloc on the creating thread: that would wait on itself.
global_heap_creating: s32; // 2.0.1: rpmalloc.c:660
global_heap_lock: u64; // 2.0.1: rpmalloc.c:662 (atomic)
// Global initialisation state machine (2.0.1: rpmalloc.c:665-674). The first allocation on any
// thread may initialise, so exactly one thread does the global setup while the others wait; none
// may see a half-written global_config.
RPMALLOC_INIT_UNINIT :: 0;
RPMALLOC_INIT_RUNNING :: 1;
RPMALLOC_INIT_DONE :: 2;
global_rpmalloc_init_state: s32; // atomic
global_main_thread_id: u64; // 2.0.1: rpmalloc.c:682
// Fast thread id (2.0.1: rpmalloc.c:760). #jai see os_thread_id: GetCurrentThreadId / pthread_self
// instead of the TEB / thread pointer.
get_thread_id :: inline () -> u64 {
return os_thread_id();
}
// Set the current thread heap (2.0.1: rpmalloc.c:796).
set_thread_heap :: (heap: *Heap) {
// #jai fix E10: the heap this thread had is no longer its heap, whatever Context caches it.
if atomic_load(*global_rpmalloc_init_state) == RPMALLOC_INIT_DONE {
previous := cast(*Heap) os_tls_get();
if previous && previous != heap previous.thread_current = false;
}
if heap && heap.id != 0 {
rpmalloc_assert(heap.id != 0, "Default heap being used");
heap.owner_thread = get_thread_id();
heap.stack_low, heap.stack_high = os_thread_stack_range(); // #jai fix E10
heap.thread_current = true;
}
// #jai one OS TLS store carries both the heap pointer (upstream's _Thread_local variable) and
// the FLS / pthread key value that drives the thread exit destructor.
os_tls_set(heap);
}
get_thread_heap_allocate :: () -> *Heap { // 2.0.1: rpmalloc.c:810
heap := heap_take_parked(); // #jai fix E7: a thread waking from thread_idle gets its own heap back.
if !heap heap = heap_allocate(false);
set_thread_heap(heap);
return heap;
}
// #jai fix E7: parked heaps (thread_idle), so thread starts skip the search when there are none.
global_parked_count: s64; // atomic
// The heap the calling thread parked, unparked and its own again (its owner id set), or null. The
// owner's thread-local slot holds the fallback heap while it is parked, so its first allocator call
// after thread_idle comes here through the uninitialised-thread path.
heap_take_parked :: () -> *Heap {
if !atomic_load(*global_parked_count) return null;
me := get_thread_id();
heap_lock_acquire();
heap := global_heap_used;
while heap && heap.parked_thread != me heap = heap.next;
if heap {
heap_free_lock_acquire(heap); // Frees into it (as a released heap) take this lock (E6).
heap.parked_thread = 0;
heap.owner_thread = me;
heap_free_lock_release(heap);
atomic_add(*global_parked_count, -1);
}
heap_lock_release();
return heap;
}
// Get the current thread heap (2.0.1: rpmalloc.c:818).
// #jai until the first initialisation creates the TLS slot, and on any thread that has not stored
// a heap yet, this returns the fallback heap: upstream's _Thread_local default value.
// #jai fix E10: every allocation and free looks up the thread heap, and the OS thread-local storage
// lookup is a call (TlsGetValue, pthread_getspecific): 1.3 ns of the 2.3 ns per allocate+free pair the
// port was behind C 2.0.1 (which reads a _Thread_local variable inline). Jai has no thread-local
// variables, but each thread runs with its own Context, so the heap is cached there too. The Context
// can be copied to another thread, or the heap stop being this thread's, so the cache is used only if
// the stack in use is the stack of the heap's thread and the heap is still that thread's heap; otherwise
// the OS lookup runs and refreshes it. A fiber, whose stack is not its thread's, always takes the OS
// lookup. Benchmarks: pair_*, churn_small and mt_churn_* 4-9% faster.
get_thread_heap :: inline () -> *Heap {
#if CONTEXT_HEAP_CACHE {
cached := cast(*Heap) context_thread_heap().*;
if cached {
here := cast(u64) *cached; // An address on the stack in use.
if here >= cached.stack_low && here < cached.stack_high && cached.thread_current return cached;
}
}
return get_thread_heap_from_os();
}
get_thread_heap_from_os :: no_inline () -> *Heap {
if atomic_load(*global_rpmalloc_init_state) == RPMALLOC_INIT_DONE {
heap := cast(*Heap) os_tls_get();
if heap {
#if CONTEXT_HEAP_CACHE if heap.thread_current context_thread_heap().* = heap;
return heap;
}
}
return global_heap_default;
}
// Size class for tiny blocks: sizes up to 16 times the minimum granularity (1024 bytes).
get_size_class_tiny :: inline (size: u64) -> u32 { // 2.0.1: rpmalloc.c:824
return (cast,trunc(u32) size + (SMALL_GRANULARITY - 1)) / SMALL_GRANULARITY;
}
get_size_class :: inline (size: u64) -> u32 { // 2.0.1: rpmalloc.c:830
// #jai The block count rounded up without 2.0.1's size + 15, which wraps for sizes within 15 of
// U64_MAX (and traps with arithmetic_overflow_check). Such sizes get a class >= SIZE_CLASS_COUNT,
// and the huge path refuses them (#jai fix U1, heap_allocate_block_huge).
minblock_count := size / SMALL_GRANULARITY + cast(u64) ((size & (SMALL_GRANULARITY - 1)) != 0);
// Up to 64 times the minimum granularity (1024 bytes) the size class is the block count.
if size <= SMALL_GRANULARITY * 64 {
rpmalloc_assert(global_size_class[minblock_count].block_size >= size, "Size class misconfiguration");
return cast(u32) ifx minblock_count then minblock_count else 1;
}
minblock_count -= 1;
// minblock_count is now > 64, so its most significant bit is at position 6 or above.
most_significant_bit := msb_index(minblock_count);
// Class sizes have the bit format [..]000xxx000[..]: the most significant bit, then the two
// bits after it select the subclass.
subclass_bits := cast(u32)(minblock_count >> (most_significant_bit - 2)) & 0x03;
class_idx := (most_significant_bit << 2) + subclass_bits + 41;
rpmalloc_assert(class_idx >= SIZE_CLASS_COUNT || global_size_class[class_idx].block_size >= size, "Size class misconfiguration");
rpmalloc_assert(class_idx >= SIZE_CLASS_COUNT || global_size_class[class_idx - 1].block_size < size, "Size class misconfiguration");
return class_idx;
}
get_page_type :: inline (size_class: u32) -> Page_Type { // 2.0.1: rpmalloc.c:858
if size_class < SMALL_SIZE_CLASS_COUNT return .SMALL;
else if size_class < SMALL_SIZE_CLASS_COUNT + MEDIUM_SMALL_SIZE_CLASS_COUNT return .MEDIUM_SMALL;
else if size_class < SMALL_SIZE_CLASS_COUNT + MEDIUM_SMALL_SIZE_CLASS_COUNT + MEDIUM_LARGE_SIZE_CLASS_COUNT return .MEDIUM_LARGE;
else if size_class < SIZE_CLASS_COUNT return .LARGE;
return .HUGE;
}
get_page_aligned_size :: inline (size: u64) -> u64 { // 2.0.1: rpmalloc.c:871
unalign := size % global_config.page_size;
if unalign size += global_config.page_size - unalign; // Callers keep size below MAX_ALLOC_SIZE (#jai fix U1).
return size;
}
////////////
///
/// OS entry points (2.0.1: rpmalloc.c:878-1161)
///
//////
// os_set_page_name:885, os_mmap:900, os_commit_on_demand:1040, os_mcommit:1049,
// os_mdecommit:1086, os_munmap:1142. These live in os_windows.jai / os_posix.jai.
////////////
///
/// Page interface (2.0.1: rpmalloc.c:1162-1500)
///
//////
page_get_span :: inline (page: *Page) -> *Span { // 2.0.1: rpmalloc.c:1169
return cast(*Span)(cast(u64) page & SPAN_MASK);
}
page_get_size :: inline (page: *Page) -> u64 { // 2.0.1: rpmalloc.c:1174
if page_get_type(page) == {
case .SMALL; return SMALL_PAGE_SIZE;
case .MEDIUM_SMALL; return MEDIUM_SMALL_PAGE_SIZE;
case .MEDIUM_LARGE; return MEDIUM_LARGE_PAGE_SIZE;
case .LARGE; return LARGE_PAGE_SIZE;
}
return page_get_span(page).page_size;
}
// #jai fix #407 (as upstream develop 30bc8859): ownership by heap, not by thread id. 2.0.1 compared
// owner_thread with the caller's thread id, and a released heap keeps its owner_thread, so a thread
// freeing after its THREAD_STOP (or a new thread reusing its id) took the unsynchronised local path
// on a heap queued for or adopted by another thread.
page_is_thread_heap :: inline (page: *Page) -> bool { // 2.0.1: rpmalloc.c:1188
#if FIRST_CLASS_HEAPS return page.heap == get_thread_heap() || !page.heap.owner_thread;
else return page.heap == get_thread_heap();
}
// #jai upstream PR #424 (2759a3e): the first block of a page starts at the largest power of two
// dividing the block size, at least the page header size and at most NATURAL_ALIGNMENT_LIMIT, so every
// block of a class is aligned to that power of two. The tail slack of every page is larger than the
// extra offset, so no class loses a block (checked at compile time below). The offset is derived from
// the block size, not stored in the page.
block_start_offset :: inline (block_size: u32) -> u32 {
offset := lowest_set_bit(block_size);
if offset < PAGE_HEADER_SIZE return PAGE_HEADER_SIZE;
if offset > NATURAL_ALIGNMENT_LIMIT return NATURAL_ALIGNMENT_LIMIT;
return offset;
}
// #jai The largest power of two dividing x; 0 for 0. C's x & -x relies on unsigned wraparound, which a
// Jai program built with arithmetic_overflow_check = .FATAL aborts on: x - 1 cannot wrap once x is
// non-zero.
lowest_set_bit :: inline (x: u32) -> u32 {
if !x return 0;
return x & ~(x - 1);
}
// Sanity check of the huge page size the OS reports (Linux Hugepagesize; 2.0.1: rpmalloc.c:2850-2856).
// The builtin page geometry cannot use huge pages larger than the largest builtin page size (for
// example 1 GiB defaults). The huge page size is passed to mmap explicitly, so a non-default 2 MiB pool
// can still be used. 0 means no usable huge pages. #jai here rather than in os_posix.jai, so the tests
// reach it on every platform.
usable_huge_page_size :: (reported: u64) -> u64 {
size := reported;
if size && (size & (size - 1)) size = 0; // #jai 0 first: 0 - 1 would wrap (arithmetic_overflow_check)
if size > LARGE_PAGE_SIZE size = 2 * 1024 * 1024;
return size;
}
page_block_start_offset :: inline (page: *Page) -> u32 {
return block_start_offset(page.block_size);
}
page_block_start :: inline (page: *Page) -> *Block { // 2.0.1: rpmalloc.c:1197
return pointer_offset(page, page_block_start_offset(page));
}
page_block :: inline (page: *Page, block_index: u32) -> *Block { // 2.0.1: rpmalloc.c:1202
return pointer_offset(page, page_block_start_offset(page) + cast(u64) page.block_size * block_index);
}
// #jai develop #411 (03537c6): the thread-free token holds the head block's byte offset from the
// page start instead of its index, so a cross-thread free needs no division and a decode no
// multiplication. Pages are at most LARGE_PAGE_SIZE (16 MiB), so the offset fits in 32 bits.
page_block_offset :: inline (page: *Page, block: *Block) -> u32 { // 2.0.1: rpmalloc.c:1207 page_block_index
return cast(u32) pointer_diff(block, page);
}
// #jai fix PR #426 (upstream 0aaf830, adapted to the 2.0.1 token): set in the thread-free token of a
// full page, whose list is then empty. The list count takes the 31 bits below it.
THREAD_FREE_FULL :: cast(u64) 1 << 63;
page_block_from_thread_free_list :: inline (page: *Page, token: u64, block: **Block) -> u32 { // 2.0.1: rpmalloc.c:1213
block_offset := cast(u32)(token & 0xFFFF_FFFF);
list_count := cast(u32)((token >> 32) & 0x7FFF_FFFF);
block.* = ifx list_count then cast(*Block) pointer_offset(page, block_offset) else null;
return list_count;
}
page_block_to_thread_free_list :: inline (page: *Page, block_offset: u32, list_count: u32) -> u64 { // 2.0.1: rpmalloc.c:1221
// #jai fix P-2 (port bug): `cast(u64) list_count << 32` casts the shifted u32, so the count was
// lost and every cross-thread free into a page that was not full went missing.
return ((cast(u64) list_count) << 32) | cast(u64) block_offset;
}
// #jai PR #424: the modulo is taken on the offset from the page start, then adjusted by the block start
// offset. Aligned blocks come from classes at least the alignment larger than the request, whose block
// start offset is at most the block size.
page_block_realign :: inline (page: *Page, block: *Block) -> *Block { // 2.0.1: rpmalloc.c:1227
block_size := page.block_size;
start_offset := page_block_start_offset(page);
rpmalloc_assert(start_offset <= block_size, "Block start offset exceeds aligned block size");
page_offset := (cast(u32) pointer_diff(block, page)) % block_size;
block_offset := ifx page_offset >= start_offset then page_offset - start_offset else page_offset + block_size - start_offset;
return pointer_offset(block, -cast(s64) block_offset);
}
page_get_local_free_block :: (page: *Page) -> *Block { // 2.0.1: rpmalloc.c:1234
block := page.local_free;
page.local_free = block.next;
page.block_used += 1;
return block;
}
page_decommit_memory_pages :: inline (page: *Page) { // 2.0.1: rpmalloc.c:1243
if page_has(page, .IS_DECOMMITTED) return;
// Only reserve-only spans support decommit.
if !page_has(page, .COMMIT_ON_DEMAND) return;
// Keep the prefix (holding the page header) committed and decommit the rest. The prefix is
// the span's fixed map-time page size, so commit and decommit always use the same one.
commit_prefix := cast(u64) 1 << page_commit_prefix_shift(page);
page_size := page_get_size(page);
if page_size <= commit_prefix return;
// #jai fix E3: only the chunks committed so far.
extra_page := pointer_offset(page, commit_prefix);
extra_page_size := page_committed_size(page) - commit_prefix;
if extra_page_size && global_memory_interface.memory_decommit(extra_page, extra_page_size) != 0 return;
#if HEAP_STATISTICS && ENABLE_DECOMMIT {
if page.heap heap_stat_sub(*page.heap.stats.committed_size, extra_page_size);
}
page_set(page, .IS_DECOMMITTED, true);
page_set_committed_chunks(page, 0);
#if TEST_HOOKS atomic_add(*test_pages_decommitted[cast(u32) page_get_type(page)], 1);
}
page_commit_memory_pages :: inline (page: *Page) -> s32 { // 2.0.1: rpmalloc.c:1267
if !page_has(page, .IS_DECOMMITTED) return 0;
commit_prefix := cast(u64) 1 << page_commit_prefix_shift(page);
// #jai fix E3: only the first chunk; page_initialize_blocks commits the rest as it carves.
if page_commit_to(page, commit_prefix + 1) != 0 return 1;
page_set(page, .IS_DECOMMITTED, false);
#if TEST_HOOKS atomic_add(*test_pages_recommitted[cast(u32) page_get_type(page)], 1);
#if ENABLE_DECOMMIT {
#if OS != .MACOS {
// When a page is recommitted, the blocks from the second memory page onwards are
// zeroed by the OS: take advantage in zalloc/calloc, and zero the first page here.
first_page := pointer_offset(page, PAGE_HEADER_SIZE);
memset(first_page, 0, cast(s64)(commit_prefix - PAGE_HEADER_SIZE));
page_set(page, .IS_ZERO, true);
}
}
return 0;
}
page_available_to_free :: (page: *Page, keep_last := true) { // 2.0.1: rpmalloc.c:1294
rpmalloc_assert(!page_has(page, .IS_FULL), "Page full flag internal failure");
rpmalloc_assert(!page_has(page, .IS_DECOMMITTED), "Page decommitted flag internal failure");
heap := page.heap;
if heap.page_available[page.size_class] == page {
// #jai develop #413 (857888a): keep the last available page of a size class in place.
// Retiring it made the next allocation of the class take the generic path and reinitialise
// the page from scratch, which alternating alloc/free then repeated on every pair (W-11).
// At most one empty page per size class stays committed this way.
// keep_last is false only from heap_retire_kept_pages (#jai fix, see there).
if keep_last && !page.next return;
heap.page_available[page.size_class] = page.next;
} else {
page.prev.next = page.next;
if page.next page.next.prev = page.prev;
}
page_set(page, .IS_FREE, true);
page_set(page, .IS_ZERO, false);
page_type := cast(u32) page_get_type(page);
page.next = heap.page_free[page_type];
heap.page_free[page_type] = page;
if !page.next heap.page_free_tail[page_type] = page; // #jai PR #420
heap.page_free_commit_count[page_type] += 1;
if heap.page_free_commit_count[page_type] >= global_page_free_overflow[page_type]
heap_page_free_overflow(heap, page_type); // #jai develop #416
}
page_full_to_available :: (page: *Page) { // 2.0.1: rpmalloc.c:1314
rpmalloc_assert(page_has(page, .IS_FULL), "Page full flag internal failure");
rpmalloc_assert(!page_has(page, .IS_DECOMMITTED), "Page decommitted flag internal failure");
heap := page.heap;
page.next = heap.page_available[page.size_class];
if page.next page.next.prev = page;
heap.page_available[page.size_class] = page;
page_set(page, .IS_FULL, false);
if !page_has(page, .HAS_ALIGNED_BLOCK) page.heap_fast = heap; // #jai PR #422
// #jai fix PR #426: other threads do not change the token of a page flagged full; it holds only
// the flag.
atomic_store(*page.thread_free, 0);
}
// page_full_to_free_on_new_heap (2.0.1: rpmalloc.c:1328) is dead code upstream; not ported.
page_available_to_full :: (page: *Page) { // 2.0.1: rpmalloc.c:1343
heap := page.heap;
if heap.page_available[page.size_class] == page {
heap.page_available[page.size_class] = page.next;
} else {
page.prev.next = page.next;
if page.next page.next.prev = page.prev;
}
page_set(page, .IS_FULL, true);
page_set(page, .IS_ZERO, false);
page.heap_fast = null; // #jai PR #422
}
page_put_local_free_block :: inline (page: *Page, block: *Block) { // 2.0.1: rpmalloc.c:1358
block.next = page.local_free;
page.local_free = block;
stat_add_free(page.heap, page.size_class, 1);
page.block_used -= 1;
if page.block_used == 0 {
page_available_to_free(page);
} else if page_has(page, .IS_FULL) {
page_full_to_available(page);
}
}
page_adopt_thread_free_block_list :: no_inline (page: *Page) { // 2.0.1: rpmalloc.c:1371
if page.local_free return;
thread_free := atomic_load(*page.thread_free);
if thread_free != 0 {
// Other threads can only replace it with another valid list head; it never changes to 0
// in other threads. #jai develop #415 (686917f): so one exchange claims the whole list, with
// no retry loop or backoff on the owner's path.
thread_free = atomic_exchange(*page.thread_free, 0);
thread_free_count := page_block_from_thread_free_list(page, thread_free, *page.local_free);
rpmalloc_assert(thread_free_count <= page.block_used, "Page thread free list count internal failure");
page.block_used -= thread_free_count;
stat_add_free(page.heap, page.size_class, thread_free_count);
}
}
// Pushes a block freed into a full page to the heap-level thread-free list; otherwise the owner would
// not pick it up until a thread-local free happens on that page.
// #jai develop #412 (50fd91a): the uncontended case is a single CAS; the spin and yield retry loops
// live in separate cold procedures, so the common path stays small.
heap_put_thread_free_block :: (page: *Page, block: *Block) {
list := *page.heap.thread_free[cast(u32) page_get_type(page)];
prev_head := atomic_load(list);
block.next = cast(*Block) prev_head;
swapped, actual := compare_and_swap(list, prev_head, cast(u64) block);
if !swapped heap_put_thread_free_block_retry(list, block, actual);
}
heap_put_thread_free_block_retry :: no_inline (list: *u64, block: *Block, prev_head: u64) {
spin: u32;
while true {
block.next = cast(*Block) prev_head;
swapped, actual := compare_and_swap(list, prev_head, cast(u64) block);
if swapped return;
prev_head = actual;
wait_spin(*spin);
}
}
// Pushes a block to another thread's deferred free list.
// #jai fix PR #426: 2.0.1 chose between the page's list and the heap's by reading the owner's
// is_full flag, while the owner marked the page full with a plain store after finding the page's
// list empty. A free in between landed in the page's list of a page then marked full, and if it was
// among the page's last frees the page never became available or free again (W-09). The owner now
// marks a page full by setting THREAD_FREE_FULL in the token with a compare and swap, which fails
// if a free got in first, and frees route by that flag.
page_put_thread_free_block :: no_inline (page: *Page, block: *Block) { // 2.0.1: rpmalloc.c:1389
prev_thread_free := atomic_load(*page.thread_free);
if prev_thread_free & THREAD_FREE_FULL {
heap_put_thread_free_block(page, block);
return;
}
block_offset := page_block_offset(page, block);
rpmalloc_assert(!((block_offset - page_block_start_offset(page)) % page.block_size), "Block pointer is not aligned to start of block");
list_size := page_block_from_thread_free_list(page, prev_thread_free, *block.next) + 1;
thread_free := page_block_to_thread_free_list(page, block_offset, list_size);
swapped, actual := compare_and_swap(*page.thread_free, prev_thread_free, thread_free);
if !swapped page_put_thread_free_block_retry(page, block, block_offset, actual); // #jai develop #412
}
// The contended retry path for pushing to a page's thread-free list, or to the heap-level list once the
// owner has flagged the page full (#jai develop #412, with PR #426).
page_put_thread_free_block_retry :: no_inline (page: *Page, block: *Block, block_offset: u32, prev_thread_free: u64) {
spin: u32;
while true {
if prev_thread_free & THREAD_FREE_FULL {
heap_put_thread_free_block(page, block);
return;
}
list_size := page_block_from_thread_free_list(page, prev_thread_free, *block.next) + 1;
thread_free := page_block_to_thread_free_list(page, block_offset, list_size);
wait_spin(*spin);
swapped, actual := compare_and_swap(*page.thread_free, prev_thread_free, thread_free);
if swapped return;
prev_thread_free = actual;
}
}
page_push_local_free_to_heap :: (page: *Page) { // 2.0.1: rpmalloc.c:1420
// Push the page free list as the fast-track list of free blocks for the heap.
page.heap.local_free[page.size_class] = page.local_free;
// Every initialised block not counted as used is on the local free list, so after handing the
// whole list to the heap every initialised block is in use (#jai PR #422).
page.block_used = page.block_initialized;
page.local_free = null;
}
page_initialize_blocks :: no_inline (page: *Page) -> *Block { // 2.0.1: rpmalloc.c:1429
rpmalloc_assert(page.block_initialized < page.block_count, "Block initialization internal failure");
// #jai fix E3: commit the page up to the end of the new block first (a small page is one chunk,
// so the free list linked below stays inside it). Null when the OS refuses; nothing has changed.
if page_has(page, .COMMIT_ON_DEMAND) {
extent := cast(u64) page_block_start_offset(page) + (cast(u64) page.block_initialized + 1) * page.block_size;
if page_commit_to(page, extent) != 0 return null;
}
block := page_block(page, page.block_initialized);
page.block_initialized += 1;
page.block_used += 1;
if page_get_type(page) == .SMALL && cast(u64) page.block_size < (global_config.page_size >> 1) {
// Link up blocks until the next memory page into the free list.
memory_page_start := cast(*void)(cast(u64) block & ~(global_config.page_size - 1));
memory_page_next := pointer_offset(memory_page_start, global_config.page_size);
free_block: *Block = pointer_offset(block, page.block_size);
first_block := free_block;
last_block := free_block;
list_count: u32 = 0;
max_list_count := page.block_count - page.block_initialized;
while cast(*void) free_block < memory_page_next && list_count < max_list_count {
last_block = free_block;
free_block.next = pointer_offset(free_block, page.block_size);
free_block = free_block.next;
list_count += 1;
}
if list_count {
last_block.next = null;
page.local_free = first_block;
page.block_initialized += list_count;
}
}
return block;
}
page_allocate_block :: inline (page: *Page, zero: bool) -> *void { // 2.0.1: rpmalloc.c:1462
is_zero := false;
block := ifx page.local_free then page_get_local_free_block(page) else null;
if block == null {
if atomic_load(*page.thread_free) != 0 {
page_adopt_thread_free_block_list(page);
block = ifx page.local_free then page_get_local_free_block(page) else null;
}
if block == null {
block = page_initialize_blocks(page);
if !block return null; // #jai fix E3: the commit for the new block failed.
is_zero = page_has(page, .IS_ZERO);
}
}
rpmalloc_assert(page.block_used <= page.block_count, "Page block use counter out of sync");
if page.local_free && !page.heap.local_free[page.size_class]
page_push_local_free_to_heap(page);
// The page might be full once its free list has been pushed to the heap local free list:
// check whether there is a thread free list to adopt.
if page.block_used == page.block_count {
page_adopt_thread_free_block_list(page);
if page.block_used == page.block_count {
// The page is now fully used.
rpmalloc_assert(!page_has(page, .IS_FULL), "Page block use counter out of sync with full flag");
#if TEST_HOOKS if test_before_mark_full test_before_mark_full(page);
// #jai fix PR #426: flag the page full in the token, unless a cross-thread free just
// added a block; then the page stays available and the block is adopted on the next
// refill.
swapped := compare_and_swap(*page.thread_free, 0, THREAD_FREE_FULL);
if swapped page_available_to_full(page);
}
}
if zero {
if !is_zero memset(block, 0, page.block_size);
else (cast(*u64) block).* = 0;
}
return block;
}
////////////
///
/// Span interface (2.0.1: rpmalloc.c:1501-1771)
///
//////
// #jai fix E8: milliseconds from `since` to `now`, 0 when `since` is later. A thread reads the clock and
// then compares under a lock, so another thread may have stored a later reading in between: the huge
// cache (an entry cached after this thread read the clock) and, since E4 and E7 let other threads free
// into a heap, a heap's decommit delay. Upstream's now - since wraps there, and the entry merely counts
// as aged early; in a program built with arithmetic_overflow_check it trapped (THR-08 on macOS, W-17).
elapsed_ms :: inline (now: u64, since: u64) -> u64 {
return ifx now > since then now - since else 0;
}
// The huge block cache (2.0.1: rpmalloc.c:1561-1734). Recently freed huge mappings are kept for reuse
// instead of paying munmap + mmap + page faults per huge allocation cycle. Entries are bounded by a
// committed byte budget and unmapped after an idle epoch. A cached mapping is reused for requests down
// to 1 / HUGE_CACHE_REUSE_FACTOR of its size (#jai PR #427; 25% in 2.0.1).
HUGE_CACHE_COMMITTED_LIMIT :: cast(u64) HUGE_CACHE_LIMIT_MB * 1024 * 1024; // 2.0.1: rpmalloc.c:1570 (128 MiB; #jai fix E11: a module parameter)
#assert HUGE_CACHE_LIMIT_MB > 0 "HUGE_CACHE_LIMIT_MB must be positive; HUGE_CACHE_SLOT_COUNT = 0 disables the cache";
// The largest single mapping the cache holds, so one rare giant cannot take the whole budget from
// the smaller, more often cycled sizes.
HUGE_CACHE_ENTRY_LIMIT :: HUGE_CACHE_COMMITTED_LIMIT / 4; // 2.0.1: rpmalloc.c:1576
HUGE_CACHE_EPOCH_MS :: 1000; // 2.0.1: rpmalloc.c:1579
// #jai upstream PR #427 (eb6d48b), as a constant: a cached mapping serves requests down to 1 / this
// of its size. 2.0.1 reused a mapping only within 25% of the request (1.25), so buffers of varying
// sizes mostly mapped fresh memory and faulted every page in (RET-14). A larger factor reuses more
// and wastes up to that much of a mapping; upstream measured malloc-large 10.0 -> 6.8 s, peak RSS
// 527 -> 557 MiB with 2.
HUGE_CACHE_REUSE_FACTOR :: 2;
#if HUGE_CACHE_SLOT_COUNT {
Huge_Cache :: struct { // 2.0.1: rpmalloc.c:1600
lock: u64; // Cache lock.
committed: u64; // Committed bytes currently cached (decommitted entries do not count).
// #jai fix E5: when the oldest committed entry was cached (0: none), so the allocation slow path
// can tell without the lock whether an entry is due for decommit. Written under the lock.
oldest_committed_ms: u64;
slot: [HUGE_CACHE_SLOT_COUNT] struct {
span: *Span;
free_ms: u64; // When the mapping was cached, or (decommitted) when it was decommitted.
decommitted: bool; // #jai fix E5: only the header page is committed; the address space is kept.
};
}
global_huge_cache: Huge_Cache; // 2.0.1: rpmalloc.c:1612
huge_cache_lock_acquire :: inline () { // 2.0.1: rpmalloc.c:1615
spin: u32;
while !compare_and_swap(*global_huge_cache.lock, 0, 1) wait_spin(*spin);
}
huge_cache_lock_release :: inline () { // 2.0.1: rpmalloc.c:1626
atomic_store(*global_huge_cache.lock, 0);
}
huge_cache_entry_size :: inline (span: *Span) -> u64 {
return cast(u64) span.page_count * span.page_size;
}
// Under the lock.
huge_cache_update_oldest :: () {
oldest: u64 = 0;
for global_huge_cache.slot if it.span && !it.decommitted && (!oldest || it.free_ms < oldest) oldest = it.free_ms;
atomic_store(*global_huge_cache.oldest_committed_ms, oldest);
}
// #jai fix E5: Ex4 1.45.0's cache filled to its 128 MiB limit and stayed committed, because idle
// entries were let go only at the next huge free (huge_cache_push) or thread_collect (U23), and a
// program that stops freeing huge blocks does neither (RET-19). An entry idle for an epoch now has its
// contents decommitted rather than its mapping unmapped: reusing it skips the mapping (most of the
// cache's value; the pages fault in again either way), and the reservation costs no memory. Without
// decommit (ENABLE_DECOMMIT off, or decommit disabled) it is unmapped, as before. The OS calls are made
// outside the lock. Runs from huge_cache_push, huge allocations, thread_collect and, through
// huge_cache_purge_if_due, the allocation slow path.
huge_cache_purge_aged :: (now: u64) { // 2.0.1: rpmalloc.c:1633
aged: [HUGE_CACHE_SLOT_COUNT] *Span = ---;
aged_count := 0;
huge_cache_lock_acquire();
for * slot: global_huge_cache.slot {
span := slot.span;
if span && !slot.decommitted && elapsed_ms(now, slot.free_ms) >= HUGE_CACHE_EPOCH_MS {
global_huge_cache.committed -= huge_cache_entry_size(span);
slot.span = null;
aged[aged_count] = span;
aged_count += 1;
}
}
huge_cache_update_oldest();
huge_cache_lock_release();
for ispan: 0..aged_count - 1 {
span := aged[ispan];
page_size := cast(u64) span.page_size;
size := huge_cache_entry_size(span);
kept := false;
if os_commit_on_demand() && size > page_size &&
global_memory_interface.memory_decommit(pointer_offset(span, page_size), size - page_size) == 0 {
// Back in, decommitted, if a slot is still free; otherwise let it go.
huge_cache_lock_acquire();
for * slot: global_huge_cache.slot {
if !slot.span {
slot.span = span;
slot.free_ms = now;
slot.decommitted = true;
kept = true;
break;
}
}
huge_cache_lock_release();
}
if !kept span_unmap(span);
}
}
// The allocation slow path's check: purge only when a committed entry has been idle for an epoch. One
// load while nothing committed is cached.
huge_cache_purge_if_due :: inline () {
oldest := atomic_load(*global_huge_cache.oldest_committed_ms);
if !oldest return;
now := os_monotonic_time_ms();
if elapsed_ms(now, oldest) >= HUGE_CACHE_EPOCH_MS huge_cache_purge_aged(now);
}
// Unmaps every cached mapping. The cache must not outlive allocator finalization.
huge_cache_flush :: () { // 2.0.1: rpmalloc.c:1652
purge: [HUGE_CACHE_SLOT_COUNT] *Span = ---;
purge_count := 0;
huge_cache_lock_acquire();
for * slot: global_huge_cache.slot {
span := slot.span;
if span {
if !slot.decommitted global_huge_cache.committed -= huge_cache_entry_size(span);
slot.span = null;
purge[purge_count] = span;
purge_count += 1;
}
}
huge_cache_update_oldest();
huge_cache_lock_release();
for ispan: 0..purge_count - 1
span_unmap(purge[ispan]);
}
// Tries to cache a freed huge mapping; returns true if it was cached.
huge_cache_push :: (span: *Span) -> bool { // 2.0.1: rpmalloc.c:1671
committed_size := huge_cache_entry_size(span);
if committed_size > HUGE_CACHE_ENTRY_LIMIT return false;
now := os_monotonic_time_ms();
if !now now = 1; // 0 marks "no committed entry" in oldest_committed_ms.
huge_cache_purge_aged(now);
cached := false;
evicted: *Span;
huge_cache_lock_acquire();
if global_huge_cache.committed + committed_size <= HUGE_CACHE_COMMITTED_LIMIT {
// A free slot, or else (#jai fix E5) the slot of the longest decommitted entry.
target: s64 = -1;
for global_huge_cache.slot if !it.span { target = it_index; break; }
if target < 0 {
for global_huge_cache.slot {
if it.decommitted && (target < 0 || it.free_ms < global_huge_cache.slot[target].free_ms) target = it_index;
}
if target >= 0 evicted = global_huge_cache.slot[target].span;
}
if target >= 0 {
slot := *global_huge_cache.slot[target];
// #jai a cached mapping belongs to no heap until it is handed out again
// (heap_allocate_block_huge sets both fields), so IS_THIS_YOURS rejects it.
span.heap = null;
span.page.heap = null;
slot.span = span;
slot.free_ms = now;
slot.decommitted = false;
global_huge_cache.committed += committed_size;
cached = true;
}
}
huge_cache_update_oldest();
huge_cache_lock_release();
if evicted span_unmap(evicted);
// #jai E11 kept a block beyond the budget decommitted instead of letting the caller unmap it. On
// macOS that made churn_huge slower: 4.5 us per step before, 12.4 with it, 6.3 even with E12
// (decommit + recommit cost about 20 us there per block, more than an munmap and a fresh mapping).
// A program that frees huge blocks often does better with a larger HUGE_CACHE_LIMIT_MB.
return cached;
}
// Takes the smallest cached mapping that fits the request within the size class spacing; null on a
// miss. The cached span keeps its own offset, mapped_size and page_count. #jai fix E5: a decommitted
// entry is committed again here; if the OS refuses, it is unmapped and the caller maps afresh.
// #jai fix E12: a committed entry that fits goes before any decommitted one, not only one of the same
// size. Taking the smallest let committed entries sit idle while decommitted ones were recommitted:
// while E11 kept blocks beyond the budget decommitted, churn_huge on macOS paid a decommit and a
// recommit on 58% of steps (21% with this order; pattern simulated). Entries decommitted after idling
// (E5) are still reused this way. A committed entry costs nothing to hand out, and taking it frees
// committed budget.
huge_cache_pop :: (alloc_size: u64) -> *Span { // 2.0.1: rpmalloc.c:1697
// A racy emptiness peek is fine: a missed entry only costs a fresh mapping.
empty := true;
for global_huge_cache.slot {
if it.span {
empty = false;
break;
}
}
if empty return null;
size_limit := alloc_size * HUGE_CACHE_REUSE_FACTOR; // #jai PR #427
best: *Span;
best_size: u64;
best_slot: s64;
best_decommitted: bool;
huge_cache_lock_acquire();
for global_huge_cache.slot {
span := it.span;
if !span continue;
size := huge_cache_entry_size(span);
if size < alloc_size || size > size_limit continue;
better := !best || (best_decommitted && !it.decommitted) ||
(best_decommitted == it.decommitted && size < best_size);
if better {
best = span;
best_size = size;
best_slot = it_index;
best_decommitted = it.decommitted;
}
}
if best {
global_huge_cache.slot[best_slot].span = null;
if !best_decommitted global_huge_cache.committed -= best_size;
huge_cache_update_oldest();
}
huge_cache_lock_release();
if best && best_decommitted {
page_size := cast(u64) best.page_size;
if global_memory_interface.memory_commit(pointer_offset(best, page_size), best_size - page_size) != 0 {
span_unmap(best);
return null;
}
}
return best;
}
} // #if HUGE_CACHE_SLOT_COUNT
span_is_thread_heap :: inline (span: *Span) -> bool { // 2.0.1: rpmalloc.c:1508 (#jai fix #407, as page_is_thread_heap)
#if FIRST_CLASS_HEAPS return span.heap == get_thread_heap() || !span.heap.owner_thread;
else return span.heap == get_thread_heap();
}
span_get_page_from_block :: inline (span: *Span, block: *void) -> *Page { // 2.0.1: rpmalloc.c:1517
return cast(*Page)(cast(u64) block & span.page_address_mask);
}
// Initialize a new page from the given span.
span_allocate_page :: inline (span: *Span) -> *Page { // 2.0.1: rpmalloc.c:1523
rpmalloc_assert(span.page_initialized < span.page_count, "Page initialization internal failure");
heap := span.heap;
page: *Page = pointer_offset(span, cast(u64) span.page_size * span.page_initialized);
#if ENABLE_DECOMMIT {
// Reserve-only spans commit later pages on demand; fully committed spans are already
// committed from the map and must not be committed per page.
// #jai fix E3: the first chunk only; page_initialize_blocks commits the rest as it carves.
if span.page_initialized && page_has(*span.page, .COMMIT_ON_DEMAND) {
chunk := page_commit_chunk_size(span.page_size, cast(u64) 1 << page_commit_prefix_shift(*span.page));
if global_memory_interface.memory_commit(page, chunk) != 0 return null;
#if HEAP_STATISTICS heap_stat_add(*heap.stats.committed_size, chunk);
}
}
span.page_initialized += 1;
page_set_type(page, span.page_type);
page_set(page, .COMMIT_ON_DEMAND, page_has(*span.page, .COMMIT_ON_DEMAND));
page_set_commit_prefix_shift(page, page_commit_prefix_shift(*span.page));
page_set_committed_chunks(page, 1); // #jai fix E3 (page 0: committed by the span map)
page_set(page, .IS_ZERO, true);
page.heap = heap;
rpmalloc_assert(page_is_thread_heap(page), "Page owner thread mismatch");
if span.page_initialized == span.page_count {
// Span fully used.
page_type := cast(u32) span.page_type;
rpmalloc_assert(span == heap.span_partial[page_type], "Span partial tracking out of sync");
heap.span_partial[page_type] = null;
span.next = heap.span_used[page_type];
heap.span_used[page_type] = span;
}
return page;
}
// #jai fix U3: removes a freed huge span from its first-class heap's list. A first-class heap is used
// by one thread at a time, which is also the thread that frees its blocks, so the list needs no lock.
huge_span_unlink :: (heap: *Heap, span: *Span) {
link := *heap.span_used[Page_Type.HUGE];
while link.* {
if link.* == span {
link.* = span.next;
span.next = null;
return;
}
link = *link.*.next;
}
}
// #jai upstream PR #425 (d668610): the huge block free and the free of a block in a page holding
// aligned blocks are out of line, so the generic free path, taken by every cross-thread free, needs no
// stack frame.
span_deallocate_huge :: no_inline (span: *Span) {
#if ENABLE_STATISTICS statistics_sub_saturating(*global_counters.huge_alloc, cast(u64) span.page_count * span.page_size);
#if HEAP_STATISTICS {
if span.heap {
heap_stat_sub(*span.heap.stats.mapped_size, span.mapped_size);
#if ENABLE_DECOMMIT heap_stat_sub(*span.heap.stats.committed_size, cast(u64) span.page_count * span.page_size);
else heap_stat_sub(*span.heap.stats.committed_size, span.mapped_size);
}
}
// #jai fix U3: a first-class heap lists its huge spans in span_used[HUGE] (so heap_free_all can
// release them), but upstream never unlinks one when it is freed. The span then goes to the
// huge cache or is unmapped, and a later heap_free_all unmaps it again, possibly while another
// heap uses the cached mapping. The list is singly linked and Span has no room for a back
// pointer, so this walks it: O(live huge blocks of that heap) on a path that makes an OS call.
// Thread heaps never link huge spans, so for them the list is empty.
if span.heap huge_span_unlink(span.heap, span);
#if HUGE_CACHE_SLOT_COUNT {
if huge_cache_push(span) return;
}
span_unmap(span);
}
span_deallocate_aligned_block :: no_inline (page: *Page, block: *void) {
block = page_block_realign(page, block);
if page_is_thread_heap(page) page_put_local_free_block(page, block);
else page_put_thread_free_block(page, block);
}
span_deallocate_block :: no_inline (span: *Span, page: *Page, block: *void) { // 2.0.1: rpmalloc.c:1736
if page_get_type(page) == .HUGE {
span_deallocate_huge(span);
return;
}
if page_has(page, .HAS_ALIGNED_BLOCK) {
span_deallocate_aligned_block(page, block);
return;
}
if page_is_thread_heap(page) {
page_put_local_free_block(page, block);
} else {
// #jai fix E4: a released heap has no owner to take in a deferred block until a thread
// adopts the heap, which may be never.
if page.heap.owner_thread == HEAP_RELEASED_OWNER && page_free_into_released_heap(page, block) return;
// Multithreaded deallocation: push to the deferred deallocation list.
page_put_thread_free_block(page, block);
}
}
// #jai fix E4: a block freed into a released heap (its thread has exited, or heap_release) went on a
// deferred list that only a thread adopting the heap drains, so its page stayed in use and committed
// until then: in Ex4, buffers that walk threads grew and the service thread freed after the walkers
// had exited (RET-18). Under the heap lock, which adoption also takes (heap_allocate_new), nobody owns
// a heap that is still released, so the block is freed as its owner would free it, the page takes in
// the frees already pending in it, and a page left empty is retired and decommitted, as at thread exit
// (E1). False when the heap has been adopted meanwhile: the caller defers the block as usual.
// #jai fix E6: under the heap's own free_lock, not the global heap lock, which every thread start and
// exit takes; frees into different released heaps no longer serialise either.
page_free_into_released_heap :: no_inline (page: *Page, block: *Block) -> bool {
heap := page.heap;
heap_free_lock_acquire(heap);
if heap.owner_thread != HEAP_RELEASED_OWNER {
heap_free_lock_release(heap);
return false;
}
page_put_local_free_block(page, block);
if !page_has(page, .IS_FREE) && !page_has(page, .IS_FULL) {
page_adopt_thread_free_merging(page);
// Empty: retire it, also when it is the page #413 keeps as the last of its class.
if page.block_used == 0 page_available_to_free(page, keep_last = false);
}
page_type := cast(u32) page_get_type(page);
if heap.page_free_commit_count[page_type] {
heap.page_free_overflow_ms[page_type] = 0;
heap_page_free_decommit(heap, page_type, 0);
}
heap_free_lock_release(heap);
return true;
}
////////////
///
/// Block interface (2.0.1: rpmalloc.c:1772-1829)
///
//////
block_get_span :: inline (block: *void) -> *Span { // 2.0.1: rpmalloc.c:1779
return cast(*Span)(cast(u64) block & SPAN_MASK);
}
// #jai PR #422: frees a block, taking the fast path when the page's heap_fast is the given heap. The
// caller owns that heap: it is the thread heap, or the first-class heap the block came from.
block_deallocate_heap :: inline (block: *Block, owned_heap: *Heap) { // 2.0.1: rpmalloc.c:1784 block_deallocate
span := block_get_span(block);
page := span_get_page_from_block(span, block);
#if HEAP_STATISTICS { // 2.0.1: rpmalloc.c:1789
if span.heap {
// A huge block subtracts its capacity, which its allocation added (#jai fix U13).
if span.page_type <= .LARGE heap_stat_sub(*span.heap.stats.allocated_size, page.block_size);
else heap_stat_sub(*span.heap.stats.allocated_size, cast(u64) span.page_size * span.page_count);
}
}
// Fast path: a free by the heap's owner, of a block in a page that is not huge, not full and has
// no aligned block; heap_fast encodes all of that by matching the owned heap.
if page.heap_fast == owned_heap {
block.next = page.local_free;
page.local_free = block;
stat_add_free(page.heap, page.size_class, 1);
page.block_used -= 1;
if page.block_used == 0 page_available_to_free(page);
} else {
span_deallocate_block(span, page, block);
}
}
block_deallocate :: inline (block: *Block) {
block_deallocate_heap(block, get_thread_heap());
}
block_usable_size :: inline (block: *void) -> u64 { // 2.0.1: rpmalloc.c:1819
span := block_get_span(block);
if span.page_type <= .LARGE {
page := span_get_page_from_block(span, block);
// #jai develop #411: the offset within a page is below 2^32; a 32-bit modulo is several times
// cheaper than a 64-bit one.
block_offset := cast(u32) pointer_diff(block, page) - page_block_start_offset(page); // #jai PR #424
return page.block_size - (block_offset % page.block_size);
}
return cast(u64) span.page_size * span.page_count - cast(u64) pointer_diff(block, span);
}
////////////
///
/// Heap interface (2.0.1: rpmalloc.c:1830-2509)
///
//////
heap_lock_acquire :: inline () { // 2.0.1: rpmalloc.c:1837
this_lock := get_thread_id();
spin: u32;
while true {
swapped := compare_and_swap(*global_heap_lock, 0, this_lock);
if swapped break;
wait_spin(*spin);
}
}
heap_lock_release :: inline () { // 2.0.1: rpmalloc.c:1848
rpmalloc_assert(atomic_load(*global_heap_lock) == get_thread_id(), "Bad heap lock");
atomic_store(*global_heap_lock, 0);
}
// A heap control structure is much smaller than the mapping page size (and far smaller than a huge
// page), so one page-aligned mapping is carved into as many heaps as fit: the first is returned to
// the caller and the extras are queued for reuse. Only the first heap records the mapping offset
// and size; it owns the shared mapping and is the one heap that unmaps it at finalize.
//
// Block creation is serialised through global_heap_creating: when no heap is available, one thread
// maps and carves a block while concurrent thread starts wait for the carved heaps. Ordinary
// requests reuse a released heap first, then a pristine carved one. First-class requests need a heap
// with no prior allocations, so they take only pristine heaps; a released first-class heap that holds
// nothing goes back to the pristine list (#jai fix U20).
heap_allocate_new :: (first_class: bool) -> *Heap { // 2.0.1: rpmalloc.c:1875
// #jai fix U11: upstream took page_size != 0 to mean "initialised", but page_size is set early in
// initialisation and never cleared by finalize, so a heap acquired after finalize was mapped and
// used on a finalized allocator (W-10). Check the init state itself.
if atomic_load(*global_rpmalloc_init_state) != RPMALLOC_INIT_DONE initialize(null);
heap_lock_acquire();
spin: u32;
while true {
if !first_class && global_heap_queue {
heap := global_heap_queue;
global_heap_queue = heap.next;
// #jai fix E4: take ownership under the lock. Frees into a released heap check its owner
// under the heap's free_lock (page_free_into_released_heap; #jai fix E6), so take that too;
// heap_allocate set the owner only afterwards. Lock order: global, then the heap's.
heap_free_lock_acquire(heap);
heap.owner_thread = get_thread_id();
heap.released = false;
heap_free_lock_release(heap);
heap_lock_release();
if heap.release_trim { // #jai develop #419: the trim deferred at release.
heap.release_trim = 0;
heap_release_free_pages(heap);
}
return heap;
}
if global_heap_pristine {
heap := global_heap_pristine;
global_heap_pristine = heap.next;
heap_lock_release();
return heap;
}
if !global_heap_creating break;
heap_lock_release();
wait_spin(*spin);
heap_lock_acquire();
}
global_heap_creating = 1;
heap_lock_release();
aligned_heap_size := cast(u64) HEAP_ALIGNMENT * ((size_of(Heap) + (HEAP_ALIGNMENT - 1)) / HEAP_ALIGNMENT);
block_size := get_page_aligned_size(aligned_heap_size);
offset, mapped_size: u64;
block := global_memory_interface.memory_map(block_size, 0, *offset, *mapped_size);
#if ENABLE_DECOMMIT {
if block && os_commit_on_demand() && global_memory_interface.memory_commit(block, block_size) != 0 {
global_memory_interface.memory_unmap(block, offset, mapped_size);
block = null;
}
}
if !block {
heap_lock_acquire();
global_heap_creating = 0;
heap_lock_release();
return null;
}
heap := heap_initialize(block);
heap.offset = cast(u32) offset;
heap.mapped_size = mapped_size;
// Carve the remaining heaps into a local chain, then splice it onto the pristine pool in one
// step, so the lock is not held across the per-heap initialisation. heap_initialize zeroes each
// structure, leaving mapped_size 0, so the extras are not mapping owners.
heap_count := mapped_size / aligned_heap_size;
chain_head: *Heap;
chain_tail: *Heap;
extra_heap: *Heap = pointer_offset(heap, aligned_heap_size);
for iheap: 1..cast(s64) heap_count - 1 {
heap_initialize(extra_heap);
if !chain_tail chain_tail = extra_heap;
extra_heap.next = chain_head;
chain_head = extra_heap;
extra_heap = pointer_offset(extra_heap, aligned_heap_size);
}
heap_lock_acquire();
if chain_tail {
chain_tail.next = global_heap_pristine;
global_heap_pristine = chain_head;
}
global_heap_creating = 0;
heap_lock_release();
return heap;
}
// Unmaps the shared block owned by this heap. Invariant: heap blocks are only ever unmapped here,
// at global finalize, never while the allocator is running, because several heaps share a block.
heap_unmap :: (heap: *Heap) { // 2.0.1: rpmalloc.c:1959
global_memory_interface.memory_unmap(heap, heap.offset, heap.mapped_size);
}
heap_allocate :: (first_class: bool) -> *Heap { // 2.0.1: rpmalloc.c:1964
heap := heap_allocate_new(first_class);
if heap {
current_thread_id := get_thread_id();
heap_lock_acquire();
heap.next = global_heap_used;
heap.prev = null;
if global_heap_used global_heap_used.prev = heap;
global_heap_used = heap;
heap_lock_release();
heap.owner_thread = current_thread_id;
heap.is_first_class = first_class;
heap.released = false; // #jai fix U14
// Counters belong to the heap, not the thread, so alloc_current reflects the blocks live in
// the heap's pages whichever thread owns it.
#if ENABLE_STATISTICS atomic_add(*global_counters.heap_count, 1);
}
return heap;
}
// Upstream heap_release, renamed: the public rpmalloc_heap_release is heap_release in this port.
// #jai fix U20: a first-class heap must start with no blocks, so heap_acquire takes only pristine
// heaps, and upstream queued every released one for thread heaps: create/destroy cycles used up a new
// heap structure each time (HEAP-13), and on macOS ran out of address space (THR-02). A heap that
// holds nothing, as after DESTROY_HEAP's heap_free_all, is as good as pristine: it is reset (keeping
// the mapping it may own) and goes back to the pristine list.
heap_is_empty :: (heap: *Heap) -> bool {
for 0..3 {
if heap.span_partial[it] || heap.page_free[it] || atomic_load(*heap.thread_free[it]) return false;
}
for heap.span_used if it return false;
return true;
}
heap_release_to_pristine :: (heap: *Heap) {
heap_lock_acquire();
if heap.released { heap_lock_release(); return; } // #jai fix U14: already released.
#if ENABLE_STATISTICS statistics_sub_saturating(*global_counters.heap_count, 1);
if heap.prev heap.prev.next = heap.next;
if heap.next heap.next.prev = heap.prev;
if global_heap_used == heap global_heap_used = heap.next;
offset, mapped_size := heap.offset, heap.mapped_size; // Only the mapping's owner has these.
heap_initialize(heap);
heap.offset, heap.mapped_size = offset, mapped_size;
heap.released = true;
heap.next = global_heap_pristine;
global_heap_pristine = heap;
heap_lock_release();
}
// #jai fix U14: upstream left a released first-class heap with owner_thread 0, which the free path
// reads as "first-class, the caller owns it", so frees into it after release took the unsynchronised
// local path while the heap was queued for another thread (W-13); and a heap made current by a thread
// was queued twice when that thread exited and the program released it too (HEAP-14). A released heap
// now gets an owner id no thread has, so frees into it are cross-thread frees, and a second release
// of a heap that is already released is ignored. The adopting thread sets its own id.
HEAP_RELEASED_OWNER :: 0xFFFF_FFFF_FFFF_FFFF;
// #jai fix E6: the lock for frees into a released heap (E4) and for adopting one; see Heap.free_lock.
heap_free_lock_acquire :: inline (heap: *Heap) {
spin: u32;
while !compare_and_swap(*heap.free_lock, 0, 1) wait_spin(*spin);
}
heap_free_lock_release :: inline (heap: *Heap) {
atomic_store(*heap.free_lock, 0);
}
heap_release_to_queue :: inline (heap: *Heap) { // 2.0.1: rpmalloc.c:1986 heap_release
heap.thread_current = false; // #jai fix E10
heap_lock_acquire();
if heap.released { heap_lock_release(); return; } // #jai fix U14: already released.
heap.released = true;
heap.owner_thread = HEAP_RELEASED_OWNER; // #jai fix U14
#if ENABLE_STATISTICS statistics_sub_saturating(*global_counters.heap_count, 1);
if heap.prev heap.prev.next = heap.next;
if heap.next heap.next.prev = heap.prev;
if global_heap_used == heap global_heap_used = heap.next;
heap.next = global_heap_queue;
global_heap_queue = heap;
heap_lock_release();
}
heap_initialize :: (block: *void) -> *Heap { // 2.0.1: rpmalloc.c:1855
heap := cast(*Heap) block;
memset(heap, 0, size_of(Heap));
heap.id = 1 + atomic_add(*global_heap_id, 1);
return heap;
}
// #jai fix (U6 part, needed by develop #413): #413 keeps a size class's last available page when it
// empties, so the next allocation of the class need not rebuild it. Every class used then pins one
// empty, committed page until the heap is released: 226 MiB after RET-02's burst, where the large
// classes use 16 MiB pages. thread_collect retires those pages through the normal free path, where
// the decommit thresholds apply. An empty page (block_used 0) has no blocks on the heap's local free
// list and none pending from other threads, so retiring it is safe.
// #jai fix U6 (rest): blocks freed by other threads sit on the heap's thread_free lists (full
// pages) or in a page's own thread-free token (pages that are not full) until the owner next needs a
// page of that type; an idle owner never does, so the memory stays committed (RET-03). thread_collect
// drains both: the heap lists are freed locally (pages that empty go to the free list), and every
// available page adopts its deferred frees, retiring it if it is now empty.
heap_drain_deferred_frees :: (heap: *Heap) {
for itype: 0..3 {
if !atomic_load(*heap.thread_free[itype]) continue;
block := cast(*Block) atomic_exchange(*heap.thread_free[itype], 0);
while block {
next_block := block.next;
block_deallocate(block);
block = next_block;
}
}
for iclass: 0..SIZE_CLASS_COUNT - 1 {
page := heap.page_available[iclass];
while page {
next_page := page.next;
if atomic_load(*page.thread_free) {
page_adopt_thread_free_merging(page); // #jai fix E2
if page.block_used == 0 page_available_to_free(page, keep_last = false);
}
page = next_page;
}
}
}
// #jai fix E2: page_adopt_thread_free_block_list leaves a page alone while its local free list is
// non-empty, because claiming the thread-free list replaces that list (the allocation path only needs
// a list when the local one is empty). The drain needs every pending block: a page freed partly by its
// owner and partly by other threads kept its pending blocks, so it stayed in use and committed through
// thread_collect and trim() (RET-16), and at thread exit (RET-15). Here the claimed list is linked in
// front of the local one.
page_adopt_thread_free_merging :: (page: *Page) {
if !page.local_free {
page_adopt_thread_free_block_list(page);
return;
}
if !atomic_load(*page.thread_free) return;
token := atomic_exchange(*page.thread_free, 0);
head: *Block;
count := page_block_from_thread_free_list(page, token, *head);
if !count return;
tail := head;
for 2..count tail = tail.next;
tail.next = page.local_free;
page.local_free = head;
rpmalloc_assert(count <= page.block_used, "Page thread free list count internal failure");
page.block_used -= count;
stat_add_free(page.heap, page.size_class, count);
}
heap_retire_kept_pages :: (heap: *Heap) {
for iclass: 0..SIZE_CLASS_COUNT - 1 {
page := heap.page_available[iclass];
if page && !page.next && page.block_used == 0 page_available_to_free(page, keep_last = false);
}
}
heap_page_free_decommit :: (heap: *Heap, page_type: u32, page_retain_count: u32) { // 2.0.1: rpmalloc.c:2003
page := heap.page_free[page_type];
while page && page_retain_count {
page = page.next;
page_retain_count -= 1;
}
while page && !page_has(page, .IS_DECOMMITTED) {
page_decommit_memory_pages(page);
// Decommit can be refused (decommit disabled, page type not larger than the configured
// page size, or the kernel rejecting decommit of huge page mappings); then the remaining
// pages cannot be decommitted either.
if !page_has(page, .IS_DECOMMITTED) break;
heap.page_free_commit_count[page_type] -= 1;
page = page.next;
}
}
// #jai develop #416 (851e255): decommit by time, not by count. Called when the free committed page
// count of a page type is at or above the overflow threshold. The excess is decommitted only once
// the count has stayed above the threshold for PAGE_DECOMMIT_DELAY_MS (dropping below restarts the
// delay), or at once at PAGE_DECOMMIT_LIMIT_FACTOR times the threshold. Decommitting on the count
// alone made every allocate/free phase larger than the threshold decommit its pages and fault them
// back in on the next phase (RET-10). thread_collect decommits the excess immediately.
PAGE_DECOMMIT_DELAY_MS :: 250;
PAGE_DECOMMIT_LIMIT_FACTOR :: 4;
// #jai upstream PR #420 (fb4288f): decommits the most recently freed page, at the head of the free
// list, and moves it to the tail behind the committed pages, so the list keeps its committed pages
// first for reuse.
heap_page_free_decommit_head :: (heap: *Heap, page_type: u32) {
page := heap.page_free[page_type];
if !page || page_has(page, .IS_DECOMMITTED) return;
page_decommit_memory_pages(page);
if !page_has(page, .IS_DECOMMITTED) return;
heap.page_free_commit_count[page_type] -= 1;
if page.next {
heap.page_free[page_type] = page.next;
heap.page_free_tail[page_type].next = page;
page.next = null;
heap.page_free_tail[page_type] = page;
}
}
// The decommit delay (#416) and, #jai PR #420, the hard limit within it: a page freed with the count at
// PAGE_DECOMMIT_LIMIT_FACTOR times the threshold is decommitted on its own. #416 decommitted every free
// page down to the retain count there, including the pages the next phase reuses (W-12).
heap_page_free_overflow :: no_inline (heap: *Heap, page_type: u32) {
limit := cast(u64) global_page_free_overflow[page_type] * PAGE_DECOMMIT_LIMIT_FACTOR;
now := os_monotonic_time_ms();
since := heap.page_free_overflow_ms[page_type];
if !since {
since = ifx now then now else 1;
heap.page_free_overflow_ms[page_type] = since;
}
if elapsed_ms(now, since) < PAGE_DECOMMIT_DELAY_MS {
if heap.page_free_commit_count[page_type] >= limit heap_page_free_decommit_head(heap, page_type);
return;
}
heap.page_free_overflow_ms[page_type] = 0;
heap_page_free_decommit(heap, page_type, global_page_free_retain[page_type]);
}
// #jai fix U5 (second half): the decommit delay was checked only when another page was freed, so a
// heap that stopped freeing kept its free pages over the threshold (up to 4x) for good (RET-12).
// The allocation slow path, which runs whenever a page is needed, checks it too: one test of four
// timestamps while no delay is running.
heap_recheck_decommit_delay :: inline (heap: *Heap) {
if !(heap.page_free_overflow_ms[0] | heap.page_free_overflow_ms[1] | heap.page_free_overflow_ms[2] | heap.page_free_overflow_ms[3])
return;
heap_recheck_decommit_delay_now(heap);
}
heap_recheck_decommit_delay_now :: no_inline (heap: *Heap) {
now := os_monotonic_time_ms();
for itype: 0..3 {
since := heap.page_free_overflow_ms[itype];
if !since || elapsed_ms(now, since) < PAGE_DECOMMIT_DELAY_MS continue;
heap.page_free_overflow_ms[itype] = 0;
heap_page_free_decommit(heap, cast(u32) itype, global_page_free_retain[itype]);
}
}
// #jai develop #418 (4b83f74), #419 (1d5730b): trims the memory a released heap holds for reuse.
// Until another thread adopts the heap nothing reuses that memory or runs the decommit delay, so a
// heap passed along by many short-lived threads would keep all of it (RET-11). The empty
// medium-large and large pages kept as the last available page of their class are retired, and free
// pages of those types are decommitted to below the overflow threshold, the most a heap holds without
// the delay. Small and medium-small pages are cheap to keep and costly to fault back in; they are
// left to the decommit delay (#419).
heap_release_free_pages :: (heap: *Heap) {
for iclass: 0..SIZE_CLASS_COUNT - 1 {
page := heap.page_available[iclass];
if !page || page.next || page.block_used || page_get_type(page) < .MEDIUM_LARGE continue;
heap.page_available[iclass] = null;
page_set(page, .IS_FREE, true);
page_set(page, .IS_ZERO, false);
page_type := cast(u32) page_get_type(page);
page.next = heap.page_free[page_type];
heap.page_free[page_type] = page;
if !page.next heap.page_free_tail[page_type] = page; // #jai PR #420
heap.page_free_commit_count[page_type] += 1;
}
for itype: cast(s64) Page_Type.MEDIUM_LARGE..cast(s64) Page_Type.LARGE {
heap.page_free_overflow_ms[itype] = 0;
if heap.page_free_commit_count[itype] >= global_page_free_overflow[itype]
heap_page_free_decommit(heap, cast(u32) itype, global_page_free_overflow[itype] - 1);
}
}
// #jai develop #419: releases a heap (by its exiting thread, or a first-class heap) for reuse. With
// other heaps already queued it is likely to stay unused for a while and is trimmed at once. With
// the queue empty a thread is likely to adopt it shortly, so the trim is left to the adopter rather
// than delaying the release while threads wait for a heap.
heap_release_trim :: (heap: *Heap) {
heap_lock_acquire();
queued := global_heap_queue != null;
heap_lock_release();
// With the queue empty the trim waits for an adopter, which may never come (U5); threads that
// exit therefore use heap_release_on_thread_exit instead. Only first-class heaps come here.
if queued heap_release_free_pages(heap);
else heap.release_trim = 1;
heap_release_to_queue(heap);
}
// #jai fix U5 (first half): a thread that exits trims its heap at once, over every page type,
// instead of leaving it to an adopter (#419), which may never come: when the last worker of a
// program exits, its heap's memory stayed committed (RET-11). Kept empty pages of every class are
// retired and free pages decommitted to below the overflow threshold. The rest of U5, re-checking
// the decommit delay on the allocation slow path, comes with the retention fixes.
// #jai fix E1 (found in Ex4 1.45.0): that trim kept free pages up to just under each type's overflow
// threshold (about 36 MiB) for an adopter, and blocks other threads had freed into the heap stayed
// pending, keeping their pages in use. With many worker threads that exit and no thread adopting
// their heaps, all of it stayed committed: 625 MiB after RET-15's 8 threads, and 709 vs 139 MB of
// private commit in Ex4's index build with 24 workers. The exiting thread now does for its heap what
// trim() does: drain the pending frees (with fix E2), retire the kept empty pages, and decommit every
// free page. An adopter recommits what it uses. Frees that arrive after the exit still wait for one.
// Callers make the heap the thread heap first: the drain frees through the thread heap's paths.
heap_release_on_thread_exit :: (heap: *Heap) {
heap_drain_deferred_frees(heap);
heap_retire_kept_pages(heap);
for itype: 0..3 {
heap.page_free_overflow_ms[itype] = 0;
if heap.page_free_commit_count[itype] heap_page_free_decommit(heap, cast(u32) itype, 0);
}
heap_release_to_queue(heap);
}
heap_make_free_page_available :: inline (heap: *Heap, size_class: u32, page: *Page) -> s32 { // 2.0.1: rpmalloc.c:2022
// #jai fix U4: recommit before anything else. Upstream linked the page into page_available
// first, so when the recommit failed the still-decommitted page stayed available and the next
// allocation wrote into memory that was not committed (an access violation on Windows; C 2.0.1
// does the same in OS-09). On failure nothing is changed; the caller keeps the page.
if page_has(page, .IS_DECOMMITTED) && page_commit_memory_pages(page) != 0 return 1;
#if TEST_HOOKS atomic_add(*test_page_reinits, 1);
page.size_class = size_class;
page.block_size = global_size_class[size_class].block_size;
page.block_count = global_size_class[size_class].block_count;
page.block_used = 0;
page.block_initialized = 0;
page.local_free = null;
page_set(page, .IS_FULL, false);
page_set(page, .IS_FREE, false);
page_set(page, .HAS_ALIGNED_BLOCK, false);
page.heap = heap;
page.heap_fast = heap; // #jai PR #422
head := heap.page_available[size_class];
page.next = head;
page.prev = null;
atomic_store(*page.thread_free, 0);
if head head.prev = page;
heap.page_available[size_class] = page;
return 0;
}
// Find or allocate a span for the given page type.
heap_get_span :: inline (heap: *Heap, page_type: Page_Type) -> *Span { // 2.0.1: rpmalloc.c:2049
// Fast path: an available span for the page type.
if heap.span_partial[page_type] return heap.span_partial[page_type];
// Fallback path: map more memory.
offset, mapped_size: u64;
span := cast(*Span) global_memory_interface.memory_map(SPAN_SIZE, SPAN_SIZE, *offset, *mapped_size);
if span {
#if HEAP_STATISTICS heap_stat_add(*heap.stats.mapped_size, mapped_size); // #jai fix U13: only when the map succeeded.
rpmalloc_assert(!(cast(u64) span & (SPAN_SIZE - 1)), "memory_map returned a block not aligned to the span size");
stat_inc_span_map(heap, page_type);
page_count: u32;
page_size: u32;
page_address_mask: u64;
if page_type == .SMALL {
page_count = SPAN_SIZE / SMALL_PAGE_SIZE;
page_size = SMALL_PAGE_SIZE;
page_address_mask = SMALL_PAGE_MASK;
} else if page_type == .MEDIUM_SMALL {
page_count = SPAN_SIZE / MEDIUM_SMALL_PAGE_SIZE;
page_size = MEDIUM_SMALL_PAGE_SIZE;
page_address_mask = MEDIUM_SMALL_PAGE_MASK;
} else if page_type == .MEDIUM_LARGE {
page_count = SPAN_SIZE / MEDIUM_LARGE_PAGE_SIZE;
page_size = MEDIUM_LARGE_PAGE_SIZE;
page_address_mask = MEDIUM_LARGE_PAGE_MASK;
} else {
page_count = SPAN_SIZE / LARGE_PAGE_SIZE;
page_size = LARGE_PAGE_SIZE;
page_address_mask = LARGE_PAGE_MASK;
}
// Commit the first page for reserve-only spans (fully committed spans commit nothing here);
// record the commit model on the span only after its header is committed.
commit_on_demand := os_commit_on_demand();
commit_prefix_shift: u32 = 0;
while (cast(u64) 1 << commit_prefix_shift) < global_config.page_size commit_prefix_shift += 1;
// #jai fix E3: the first chunk of page 0 only; page_initialize_blocks commits the rest.
first_commit := page_commit_chunk_size(page_size, cast(u64) 1 << commit_prefix_shift);
#if ENABLE_DECOMMIT {
if commit_on_demand {
if global_memory_interface.memory_commit(span, first_commit) != 0 {
global_memory_interface.memory_unmap(span, offset, mapped_size);
return null;
}
}
}
page_set(*span.page, .COMMIT_ON_DEMAND, commit_on_demand);
page_set_commit_prefix_shift(*span.page, commit_prefix_shift);
page_set_committed_chunks(*span.page, 1);
#if HEAP_STATISTICS {
#if ENABLE_DECOMMIT heap_stat_add(*heap.stats.committed_size, ifx commit_on_demand then first_commit else page_size);
else heap_stat_add(*heap.stats.committed_size, mapped_size);
}
span.heap = heap;
span.page_type = page_type;
span.page_count = page_count;
span.page_size = page_size;
span.page_address_mask = page_address_mask;
span.offset = cast(u32) offset;
span.mapped_size = mapped_size;
span_register(span); // #jai for IS_THIS_YOURS
heap.span_partial[page_type] = span;
}
return span;
}
heap_get_page_generic :: (heap: *Heap, size_class: u32) -> *Page { // 2.0.1: rpmalloc.c:2129
page_type := get_page_type(size_class);
heap_recheck_decommit_delay(heap); // #jai fix U5
#if HUGE_CACHE_SLOT_COUNT huge_cache_purge_if_due(); // #jai fix E5
// Check for a free page from multithreaded deallocations.
block_mt := atomic_load(*heap.thread_free[page_type]);
if block_mt != 0 {
// #jai develop #415: claim the whole list in one exchange; concurrent pushes land either
// before it (and are claimed) or after.
block_mt = atomic_exchange(*heap.thread_free[page_type], 0);
block := cast(*Block) block_mt;
while block {
next_block := block.next;
block_deallocate(block);
block = next_block;
}
// Retry after processing the deferred thread frees.
return heap_get_page(heap, size_class);
}
// Check for a free page.
page := heap.page_free[page_type];
if page {
heap.page_free[page_type] = page.next;
if !page.next heap.page_free_tail[page_type] = null; // #jai PR #420
if !page_has(page, .IS_DECOMMITTED) {
rpmalloc_assert(heap.page_free_commit_count[page_type] > 0, "Free committed page count out of sync");
heap.page_free_commit_count[page_type] -= 1;
// #jai develop #416: dropping below the threshold restarts the decommit delay.
if heap.page_free_commit_count[page_type] < global_page_free_overflow[page_type]
heap.page_free_overflow_ms[page_type] = 0;
}
if heap_make_free_page_available(heap, size_class, page) != 0 {
// #jai fix U4: the recommit failed; put the (still decommitted) page back for later, at the
// tail behind the committed pages (#jai PR #420).
page.next = null;
if heap.page_free_tail[page_type] heap.page_free_tail[page_type].next = page;
else heap.page_free[page_type] = page;
heap.page_free_tail[page_type] = page;
return null;
}
return page;
}
rpmalloc_assert(heap.page_free_commit_count[page_type] == 0, "Free committed page count out of sync");
if heap.id == 0 {
// The thread is not initialised yet: assign a heap and try again.
initialize(null);
return heap_get_page(get_thread_heap(), size_class);
}
// Fallback path: find or allocate a span for the size class. If the thread was not
// initialised, the heap for the new span differs from the local heap variable here (the
// default heap), so use the span page heap instead.
span := heap_get_span(heap, page_type);
if span {
page = span_allocate_page(span);
if !page return null; // #jai fix U4: the page's commit failed; upstream dereferenced null.
if heap_make_free_page_available(page.heap, size_class, page) != 0 return null;
}
return page;
}
// Find or allocate a page for the given size class.
heap_get_page :: (heap: *Heap, size_class: u32) -> *Page { // 2.0.1: rpmalloc.c:2186
// Fast path: an available page for the size class.
page := heap.page_available[size_class];
if page return page;
return heap_get_page_generic(heap, size_class);
}
// Pop a block from the heap local free list.
heap_pop_local_free :: inline (heap: *Heap, size_class: u32) -> *Block { // 2.0.1: rpmalloc.c:2196
free_list := *heap.local_free[size_class];
block := free_list.*;
if block free_list.* = block.next;
return block;
}
// Generic allocation path from heap pages, spans or a new mapping.
heap_allocate_block_small_to_large :: no_inline (heap: *Heap, size_class: u32, zero: bool) -> *void { // 2.0.1: rpmalloc.c:2206
page := heap_get_page(heap, size_class);
if page {
block := page_allocate_block(page, zero);
// Count on the page owner, which may differ from heap when an uninitialised thread is
// routed to its real heap, so the allocation and its later free count on the same heap.
// #jai fix E3: only once the block exists; committing it can fail.
if block stat_inc_alloc(page.heap, size_class);
return block;
}
return null;
}
heap_allocate_block_huge :: no_inline (heap: *Heap, size: u64, zero: bool) -> *void { // 2.0.1: rpmalloc.c:2219
if heap.id == 0 {
// The thread is not initialised yet: assign a heap and try again.
initialize(null);
heap = get_thread_heap();
}
// #jai fix U1: upstream checks the size only under ENABLE_VALIDATE_ARGS, so without it
// size + SPAN_HEADER_SIZE and the page rounding wrap, and alloc(-1) returns a tiny block.
// Refusing here bounds every path into a mapping; the cost is one compare on a path that maps.
if size >= MAX_ALLOC_SIZE return null;
alloc_size := get_page_aligned_size(size + SPAN_HEADER_SIZE);
offset, mapped_size: u64;
block: *void;
from_cache := false;
#if HUGE_CACHE_SLOT_COUNT {
// Reuse a recently freed huge mapping of fitting size if there is one.
huge_cache_purge_if_due(); // #jai fix E5: huge allocations age the cache too.
block = huge_cache_pop(alloc_size);
from_cache = block != null;
}
committed_at_map := false;
if !block {
// #jai fix E9: with the OS's own interface, map and commit in one call (see os_mmap_committed).
// A custom interface keeps the separate map and commit calls.
if global_memory_interface.memory_map == os_mmap && os_commit_on_demand() {
block = os_mmap_committed(alloc_size, SPAN_SIZE, *offset, *mapped_size);
committed_at_map = true;
} else {
block = global_memory_interface.memory_map(alloc_size, SPAN_SIZE, *offset, *mapped_size);
}
}
if block {
span := cast(*Span) block;
rpmalloc_assert(!(cast(u64) span & (SPAN_SIZE - 1)), "memory_map returned a block not aligned to the span size");
if !from_cache stat_inc_span_map(heap, .HUGE);
#if ENABLE_DECOMMIT {
if !from_cache && !committed_at_map && os_commit_on_demand() && global_memory_interface.memory_commit(span, alloc_size) != 0 {
global_memory_interface.memory_unmap(block, offset, mapped_size);
return null;
}
}
span.heap = heap;
span.page_type = .HUGE;
span.page_address_mask = LARGE_PAGE_MASK;
// Huge blocks never use the per-page commit path; set explicitly so a reused cached
// mapping carries no stale value.
page_set(*span.page, .COMMIT_ON_DEMAND, false);
if !from_cache {
span.page_size = cast(u32) global_config.page_size;
span.page_count = cast(u32)(alloc_size / global_config.page_size);
span.offset = cast(u32) offset;
span.mapped_size = mapped_size;
}
#if HEAP_STATISTICS {
// #jai fix U13: from the span's own fields, so a mapping reused from the huge cache counts
// its real size (upstream added 0 mapped bytes and the request's commit for those), and
// the free subtracts exactly this.
heap_stat_add(*heap.stats.mapped_size, span.mapped_size);
#if ENABLE_DECOMMIT heap_stat_add(*heap.stats.committed_size, cast(u64) span.page_count * span.page_size);
else heap_stat_add(*heap.stats.committed_size, span.mapped_size);
}
span.page.heap = heap;
page_set(*span.page, .IS_FULL, true);
span.page.heap_fast = null; // #jai PR #422: huge blocks take the slow path.
page_set_type(*span.page, .HUGE);
#if ENABLE_STATISTICS statistics_add_with_peak(*global_counters.huge_alloc, *global_counters.huge_alloc_peak, cast(u64) span.page_count * span.page_size);
// Keep track of the span if this is a first-class heap.
if !heap.owner_thread {
span.next = heap.span_used[Page_Type.HUGE]; // Unlinked again on free (#jai fix U3).
heap.span_used[Page_Type.HUGE] = span;
}
if !from_cache span_register(span); // #jai for IS_THIS_YOURS
ptr := pointer_offset(block, SPAN_HEADER_SIZE);
// #jai develop #406 (4301fb8): a fresh mapping from the OS interface is already zero
// (anonymous mmap and VirtualAlloc guarantee it); clearing it again faults in and commits every
// page. Reused cached mappings and custom memory interfaces give no such guarantee.
if zero && (from_cache || global_memory_interface.memory_map != os_mmap) memset(ptr, 0, cast(s64) size);
// #jai fix U13: the capacity, which the free subtracts; upstream added the requested size and
// subtracted the page-rounded one, so allocated_size drifted (API-06).
#if HEAP_STATISTICS heap_stat_add(*heap.stats.allocated_size, cast(u64) span.page_size * span.page_count);
return ptr;
}
return null;
}
heap_allocate_block_generic :: no_inline (heap: *Heap, size: u64, zero: bool) -> *void { // 2.0.1: rpmalloc.c:2305
size_class := get_size_class(size);
if size_class < SIZE_CLASS_COUNT {
#if HEAP_STATISTICS heap_stat_add(*heap.stats.allocated_size, global_size_class[size_class].block_size);
block := heap_pop_local_free(heap, size_class);
if block {
// Fast track with a small block available in the heap local free list; a hit means
// this heap owns the block.
stat_inc_alloc(heap, size_class);
if zero memset(block, 0, global_size_class[size_class].block_size);
return block;
}
return heap_allocate_block_small_to_large(heap, size_class, zero);
}
return heap_allocate_block_huge(heap, size, zero);
}
// Find or allocate a block of the given size.
heap_allocate_block :: inline (heap: *Heap, size: u64, zero: bool) -> *void { // 2.0.1: rpmalloc.c:2330
if size <= SMALL_GRANULARITY * 64 {
size_class := get_size_class_tiny(size);
block := heap_pop_local_free(heap, size_class);
if block {
// Fast track with a small block available in the heap local free list.
if zero memset(block, 0, global_size_class[size_class].block_size);
#if HEAP_STATISTICS heap_stat_add(*heap.stats.allocated_size, global_size_class[size_class].block_size);
stat_inc_alloc(heap, size_class);
return block;
}
#if HEAP_STATISTICS heap_stat_add(*heap.stats.allocated_size, global_size_class[size_class].block_size);
// The size class is already known: refill directly, skipping the generic path's size
// class recomputation and dead local free list pop.
return heap_allocate_block_small_to_large(heap, size_class, zero);
}
return heap_allocate_block_generic(heap, size, zero);
}
// An aligned request allocates size + alignment and moves the pointer up to the alignment. The page
// is then marked, so that freeing the moved pointer finds the block start (page_block_realign).
heap_allocate_block_aligned :: (heap: *Heap, alignment: u64, size: u64, zero: bool) -> *void { // 2.0.1: rpmalloc.c:2356
if alignment <= SMALL_GRANULARITY return heap_allocate_block(heap, size, zero);
// #jai fix U1: upstream makes these two checks only under ENABLE_VALIDATE_ARGS. Without them
// size + alignment wraps and a tiny block comes back, and a non-power-of-two alignment gives a
// misaligned block. This path over-allocates anyway; the checks cost two compares.
// #jai Without computing size + alignment, which traps with arithmetic_overflow_check.
if size > 0xFFFF_FFFF_FFFF_FFFF - alignment return null;
if alignment & (alignment - 1) return null;
if alignment >= RPMALLOC_MAX_ALIGNMENT return null;
align_mask := alignment - 1;
// #jai develop #405 (a3abe2e): blocks start PAGE_HEADER_SIZE bytes into a page (or span) whose
// start is aligned to at least the page size. Rounding the size up to a multiple of the alignment
// (and at least the alignment) selects a size class whose block size is a multiple of the
// alignment: up to 1 KiB every multiple of the granularity is a class, above that every class size
// is a multiple of 256. Every block of that class is then naturally aligned: no over-allocation,
// no realignment, and the page keeps the fast free path.
// #jai PR #424: up to NATURAL_ALIGNMENT_LIMIT, a class whose block size is a multiple of the
// alignment starts its blocks at an offset that is a multiple of it too, so the plain allocation of
// the size rounded up to the alignment is aligned.
if alignment > PAGE_HEADER_SIZE && alignment <= NATURAL_ALIGNMENT_LIMIT {
aligned_size := (size + align_mask) & ~align_mask;
if aligned_size >= size && aligned_size <= LARGE_BLOCK_SIZE_LIMIT {
size_class := get_size_class(aligned_size);
block_size := ifx size_class < SIZE_CLASS_COUNT then global_size_class[size_class].block_size else 0;
if cast(u64) lowest_set_bit(block_size) >= alignment return heap_allocate_block(heap, aligned_size, zero);
}
}
if alignment <= PAGE_HEADER_SIZE {
aligned_size := (size + align_mask) & ~align_mask;
if aligned_size < alignment aligned_size = alignment;
if aligned_size >= size {
block := heap_allocate_block(heap, aligned_size, zero);
rpmalloc_assert(!(cast(u64) block & align_mask), "Natural block alignment internal failure");
return block;
}
}
block := heap_allocate_block(heap, size + alignment, zero);
if cast(u64) block & align_mask {
block = cast(*void)((cast(u64) block & ~align_mask) + alignment);
// Mark the page as having aligned blocks.
span := block_get_span(block);
page := span_get_page_from_block(span, block);
// #jai PR #424: write the flag word only on the page's first aligned block.
if !page_has(page, .HAS_ALIGNED_BLOCK) {
page_set(page, .HAS_ALIGNED_BLOCK, true);
page.heap_fast = null; // #jai PR #422
}
}
return block;
}
heap_reallocate_block :: no_inline (heap: *Heap, block: *void, size: u64, old_size: u64, flags: Realloc_Flags) -> *void { // 2.0.1: rpmalloc.c:2389
if block {
// Grab the span through the guaranteed span alignment.
span := block_get_span(block);
if span.page_type <= .LARGE {
// Normal sized block.
page := span_get_page_from_block(span, block);
// #jai upstream PR #423 (951f903): a pointer from the allocator is at its block start
// unless the page holds aligned blocks, so the division locating the block start is only
// needed for those pages.
block_origin := block;
if page_has(page, .HAS_ALIGNED_BLOCK) block_origin = page_block_realign(page, block);
if !old_size old_size = cast(u64)(cast(s64) page.block_size - pointer_diff(block, block_origin));
if cast(u64) page.block_size >= size {
// Still fits in the block: never mind saving memory, but preserve the data if the
// alignment changed.
if block != block_origin && !(flags & .NO_PRESERVE)
memmove_rp(block_origin, block, cast(s64) old_size);
return block_origin;
}
} else {
// Huge block.
block_start := pointer_offset(span, SPAN_HEADER_SIZE);
capacity := cast(u64) span.page_size * span.page_count - cast(u64) pointer_diff(block, span);
if !old_size old_size = capacity;
// #jai fix U12: upstream judged the fit on old_size, which is also the caller's live size,
// with size < old_size, so a same-size request with the real old size reallocated (RE-06).
// The fit is judged on the capacity, and an equal size fits.
if size <= capacity && size >= capacity / 2 && size > LARGE_BLOCK_SIZE_LIMIT {
// Still fits, still huge, and saves less than half the memory: keep it, but
// preserve the data if the alignment changed.
if block_start != block && !(flags & .NO_PRESERVE)
memmove_rp(block_start, block, cast(s64) old_size);
return block_start;
}
}
} else {
old_size = 0;
}
if flags & .GROW_OR_FAIL return null;
// The size is greater than the block size, or shrinking saves enough memory: allocate a new
// block and free the old one. Avoid hysteresis by overallocating small increases (below 37%).
lower_bound := old_size + (old_size >> 2) + (old_size >> 3);
new_size := ifx size > lower_bound then size else (ifx size > old_size then lower_bound else size);
old_block := block;
block = heap_allocate_block(heap, new_size, false);
if block && old_block {
if !(flags & .NO_PRESERVE)
memcpy(block, old_block, cast(s64) ifx old_size < new_size then old_size else new_size);
block_deallocate(old_block);
}
return block;
}
heap_reallocate_block_aligned :: (heap: *Heap, block: *void, alignment: u64, size: u64, old_size: u64, flags: Realloc_Flags) -> *void { // 2.0.1: rpmalloc.c:2444
if alignment <= SMALL_GRANULARITY return heap_reallocate_block(heap, block, size, old_size, flags);
no_alloc := (flags & .GROW_OR_FAIL) != 0;
usable_size := ifx block then block_usable_size(block) else 0;
if usable_size >= size && !(cast(u64) block & (alignment - 1)) {
if no_alloc || size >= usable_size / 2 return block;
}
// The aligned allocation marks the page as having aligned blocks.
old_block := block;
block = ifx !no_alloc then heap_allocate_block_aligned(heap, alignment, size, false) else null;
if block {
if !(flags & .NO_PRESERVE) && old_block {
if !old_size old_size = usable_size;
memcpy(block, old_block, cast(s64) ifx old_size < size then old_size else size);
}
if old_block block_deallocate(old_block);
}
return block;
}
// Upstream heap_free_all, renamed: the public rpmalloc_heap_free_all is heap_free_all in this port.
heap_unmap_all :: (heap: *Heap) { // 2.0.1: rpmalloc.c:2471 heap_free_all
for itype: 0..3 {
span := heap.span_partial[itype];
while span {
span_next := span.next;
span_unmap(span);
span = span_next;
}
heap.span_partial[itype] = null;
heap.page_free[itype] = null;
heap.page_free_tail[itype] = null; // #jai PR #420
heap.page_free_commit_count[itype] = 0;
heap.page_free_overflow_ms[itype] = 0; // #jai develop #416
atomic_store(*heap.thread_free[itype], 0);
}
for itype: 0..4 {
span := heap.span_used[itype];
while span {
span_next := span.next;
#if ENABLE_STATISTICS {
if itype == cast(s64) Page_Type.HUGE statistics_sub_saturating(*global_counters.huge_alloc, cast(u64) span.page_count * span.page_size);
}
span_unmap(span); // Only live huge blocks are listed (#jai fix U3).
span = span_next;
}
heap.span_used[itype] = null;
}
memset(heap.local_free.data, 0, size_of(type_of(heap.local_free)));
memset(heap.page_available.data, 0, size_of(type_of(heap.page_available)));
#if ENABLE_STATISTICS {
// Every block in this heap has been released, so reverse the per size class counters.
for * heap.size_use {
it.free_total += it.alloc_current;
it.alloc_current = 0;
}
}
}
////////////
///
/// Extern interface (2.0.1: rpmalloc.c:2510-2660)
///
//////
// rpmalloc_is_thread_initialized:2517, rpmalloc:2522, rpzalloc:2534, rpfree:2546,
// rpcalloc:2553, rprealloc:2577, rpaligned_realloc:2589, rpaligned_alloc:2601,
// rpaligned_zalloc:2607, rpaligned_calloc:2613, rpmemalign:2637, rpposix_memalign:2643,
// rpmalloc_usable_size:2653. (rpmalloc_linker_reference:2658 is C-only; not ported.)
////////////
///
/// Initialization and finalization (2.0.1: rpmalloc.c:2661-3421)
///
//////
// The exported procedures of this section are further down, after #scope_export.
// os_read_system_file:2684 lives in os_posix.jai. (malloc.c and rpnew.h, the libc / C++
// overrides, are not ported.)
// Runs at thread exit through the FLS / pthread key (2.0.1: rpmalloc.c:2668). #c_call: the OS calls
// it, so it has no Jai context of its own.
rpmalloc_thread_destructor :: (value: *void) #c_call {
fresh: #Context;
push_context fresh {
// On the main thread this means rpmalloc_finalize has not been called and shutdown is
// forced (through _exit) or unclean.
if get_thread_id() == global_main_thread_id return;
// #jai fix U22: when another thread calls exit, Windows stops every other thread and then
// runs this destructor on the exiting one; releasing its heap takes global_heap_lock, which
// a stopped thread may hold, and the process never exits (LIFE-05). The process is ending
// anyway, so keep the heap.
if os_process_exiting() return;
// #jai upstream calls rpmalloc_thread_finalize, which reads its _Thread_local heap. Here the
// key itself holds the heap, and POSIX clears it before calling the destructor, so
// thread_finalize would find no heap and release nothing (every exiting POSIX thread leaked
// its heap: RET-04/RET-05 on macOS). Release the heap passed in, and leave the key alone:
// storing the default heap would make POSIX run the destructor again.
heap := cast(*Heap) value;
if heap && heap != global_heap_default {
// #jai fix E1: the release drains the frees pending in the heap, and frees take the owner's
// path only when the heap is the thread heap. POSIX has cleared the key, so set it for the
// release, and clear it again after (POSIX runs the destructor again for a non-null value).
// Windows still has the value here.
#if OS != .WINDOWS os_tls_set(heap);
heap_release_on_thread_exit(heap); // #jai fix U5
#if OS != .WINDOWS os_tls_set(null);
} else if heap == global_heap_default {
heap_release_parked_on_exit(); // #jai fix E7: the thread exits with its heap parked.
}
}
}
#if ENABLE_LEAK_DETECTION {
// Size class blocks still allocated across every heap. Non-zero once all threads have finalized
// means the application has not freed all its allocations (a leak or a double free).
statistics_outstanding_blocks :: () -> s64 { // 2.0.1: rpmalloc.c:2975
lists := *Heap.[global_heap_used, global_heap_queue, global_heap_pristine];
outstanding: s64;
for list: lists {
heap := list;
while heap {
for heap.size_use outstanding += it.alloc_current;
heap = heap.next;
}
}
return outstanding;
}
// Cross-thread freed blocks parked in a span's per page deferred lists, over every initialised page
// (a full page is not reachable from page_available but can still hold deferred frees). The block
// count is in the token's high 32 bits.
span_deferred_block_count :: (span: *Span) -> s64 { // 2.0.1: rpmalloc.c:2992
deferred: s64;
for ipage: 0..cast(s64) span.page_initialized - 1 {
page := cast(*Page) pointer_offset(span, cast(u64) span.page_size * cast(u64) ipage);
token := atomic_load(*page.thread_free);
deferred += cast(s64)((token >> 32) & 0x7FFF_FFFF); // #jai fix PR #426: mask the full flag.
}
return deferred;
}
// Blocks freed cross-thread but not yet applied to the owning heap's alloc_current. Subtracting
// them from the outstanding count gives the blocks the application has really leaked.
statistics_pending_deferred_blocks :: () -> s64 { // 2.0.1: rpmalloc.c:3006
lists := *Heap.[global_heap_used, global_heap_queue, global_heap_pristine];
deferred: s64;
for list: lists {
heap := list;
while heap {
// Page types small to large carry the heap's full page deferred list and the per page
// deferred lists of every span. Huge blocks are never deferred per block.
for itype: 0..3 {
block := cast(*Block) atomic_load(*heap.thread_free[itype]);
while block {
deferred += 1;
block = block.next;
}
if heap.span_partial[itype] deferred += span_deferred_block_count(heap.span_partial[itype]);
span := heap.span_used[itype];
while span {
deferred += span_deferred_block_count(span);
span = span.next;
}
}
heap = heap.next;
}
}
return deferred;
}
} // #if ENABLE_LEAK_DETECTION
//
// Test hooks (TEST_HOOKS): fault injection inside the OS layer, so tests exercise the real
// error paths, plus direct access to the OS entry points for the white-box tests.
//
#if TEST_HOOKS {
test_fail_maps: s64; // The next N os_mmap calls fail.
test_fail_commits: s64; // The next N os_mcommit calls fail.
test_pages_decommitted: [4] s64; // Pages decommitted / recommitted so far, per page type.
test_before_mark_full: (page: *void); // Called just before the owner marks a page full.
test_page_reinits: s64; // Pages (re)initialised for a size class (#413).
test_pages_recommitted: [4] s64;
test_take_fault :: (counter: *s64) -> bool {
while true {
n := atomic_load(counter);
if n <= 0 return false;
if compare_and_swap(counter, n, n - 1) return true;
}
return false;
}
}
// The count * size product of the calloc family; ok is false for a refused request.
// #jai fix #417 (as upstream develop 06b18dd0): the product is checked whatever ENABLE_VALIDATE_ARGS
// says. Unchecked, calloc(2^33, 2^33) wraps to 0 and returns a small block.
calloc_total :: inline (num: s64, size: s64) -> total: u64, ok: bool {
n := cast,no_check(u64) num;
s := cast,no_check(u64) size;
if n && s > 0xFFFF_FFFF_FFFF_FFFF / n return 0, false; // 2.0.1: SizeTMult / __builtin_umull_overflow
total := n * s;
if total >= MAX_ALLOC_SIZE return 0, false;
return total, true;
}
// #jai upstream PR #423 (951f903): a block from a page without aligned blocks that still fits the
// size is kept as it is, without the call to heap_reallocate_block.
realloc_fits_in_place :: inline (ptr: *void, size: u64) -> bool {
if !ptr return false;
span := block_get_span(ptr);
if span.page_type > .LARGE return false;
page := span_get_page_from_block(span, ptr);
return !page_has(page, .HAS_ALIGNED_BLOCK) && cast(u64) page.block_size >= size;
}
// Jai Allocator protocol helpers used by rpmalloc_allocator_proc (module.jai).
// RESIZE passes the caller's old_size through to heap_reallocate_block (2.0.1: rpmalloc.c:2389).
allocator_proc_resize :: (heap: *Heap, ptr: *void, size: s64, old_size: s64) -> *void {
#if ENABLE_VALIDATE_ARGS {
if cast,no_check(u64) size >= MAX_ALLOC_SIZE return null; // #jai null, not ptr (upstream L6, see rprealloc).
}
// #jai PR #423: the Allocator protocol's RESIZE is this port's realloc, so it gets the fast path.
if realloc_fits_in_place(ptr, cast,no_check(u64) size) return ptr;
if !heap heap = get_thread_heap();
// #jai the protocol's old_size goes through (upstream's rprealloc passes 0 and derives the block's
// whole capacity), so a moved block copies only the caller's live bytes. Safe since the U12 fix.
live := ifx old_size > 0 then cast(u64) old_size else 0;
return heap_reallocate_block(heap, ptr, cast,no_check(u64) size, live, 0);
}
#scope_export
#if TEST_HOOKS {
test_fail_next_maps :: (count: s64) { atomic_store(*test_fail_maps, count); }
test_fail_next_commits :: (count: s64) { atomic_store(*test_fail_commits, count); }
test_os_page_size :: () -> u64 { os_initialize(); return os_page_size; }
test_os_map_granularity :: () -> u64 { os_initialize(); return os_map_granularity; }
test_os_mmap :: (size: u64, alignment: u64, offset: *u64, mapped_size: *u64) -> *void { return os_mmap(size, alignment, offset, mapped_size); }
test_os_mcommit :: (address: *void, size: u64) -> s32 { return os_mcommit(address, size); }
test_os_mdecommit :: (address: *void, size: u64) -> s32 { return os_mdecommit(address, size); }
test_os_munmap :: (address: *void, offset: u64, mapped_size: u64) { os_munmap(address, offset, mapped_size); }
test_os_commit_on_demand :: () -> bool { return os_commit_on_demand(); }
// Size classes and geometry.
test_size_class_count :: () -> s64 { return SIZE_CLASS_COUNT; }
test_size_class_block_size :: (index: s64) -> u32 { return global_size_class[index].block_size; }
test_size_class_block_count :: (index: s64) -> u32 { return global_size_class[index].block_count; }
test_get_size_class :: (size: u64) -> u32 { return get_size_class(size); }
test_get_size_class_tiny :: (size: u64) -> u32 { return get_size_class_tiny(size); }
test_get_page_type :: (size_class: u32) -> u32 { return cast(u32) get_page_type(size_class); }
test_page_size_of_type :: (page_type: u32) -> u64 {
if page_type == {
case 0; return SMALL_PAGE_SIZE;
case 1; return MEDIUM_SMALL_PAGE_SIZE;
case 2; return MEDIUM_LARGE_PAGE_SIZE;
case 3; return LARGE_PAGE_SIZE;
}
return 0;
}
test_span_size :: () -> u64 { return SPAN_SIZE; }
test_page_header_size :: () -> u64 { return PAGE_HEADER_SIZE; }
test_msb_index :: (x: u64) -> u32 { return msb_index(x); }
test_usable_huge_page_size :: (reported: u64) -> u64 { return usable_huge_page_size(reported); }
test_huge_cache_purge_aged :: (now: u64) { #if HUGE_CACHE_SLOT_COUNT huge_cache_purge_aged(now); }
// The huge cache now: committed bytes, committed entries, decommitted entries.
test_huge_cache_state :: () -> committed: u64, committed_entries: s64, decommitted_entries: s64 {
#if HUGE_CACHE_SLOT_COUNT {
committed_entries, decommitted_entries: s64;
huge_cache_lock_acquire();
for global_huge_cache.slot {
if !it.span continue;
if it.decommitted decommitted_entries += 1;
else committed_entries += 1;
}
committed := global_huge_cache.committed;
huge_cache_lock_release();
return committed, committed_entries, decommitted_entries;
} else {
return 0, 0, 0;
}
}
test_heap_stat_add :: (field: *u64, value: u64) { heap_stat_add(field, value); }
test_heap_stat_sub :: (field: *u64, value: u64) { heap_stat_sub(field, value); }
// Ownership: would freeing this block from the calling thread take the local (unsynchronised) path?
test_block_is_thread_local :: (block: *void) -> bool {
span := block_get_span(block);
return page_is_thread_heap(span_get_page_from_block(span, block));
}
// The thread-free race of PR #426: a callback between "no cross-thread frees" and "mark full",
// and the resulting page state.
test_set_before_mark_full :: (callback: (page: *void)) { test_before_mark_full = callback; }
test_page_full_state :: (block: *void) -> is_full: bool, pending_cross_thread_frees: u32 {
span := block_get_span(block);
page := span_get_page_from_block(span, block);
unused: *Block;
return page_has(page, .IS_FULL), page_block_from_thread_free_list(page, atomic_load(*page.thread_free), *unused);
}
// Heap layout (#409): where the fields other threads access end, where the owner's begin.
test_heap_layout :: () -> remote_end: s64, first_owner_field: s64, heap_size: s64 {
remote_end := 0;
first_owner := size_of(Heap);
for type_info(Heap).members {
end := it.offset_in_bytes + it.type.runtime_size;
if it.name == "owner_thread" || it.name == "thread_free" || it.name == "free_lock" { // free_lock: #jai fix E6
if end > remote_end remote_end = end;
} else if it.name != "remote_pad" {
if it.offset_in_bytes < first_owner first_owner = it.offset_in_bytes;
}
}
return remote_end, first_owner, size_of(Heap);
}
test_page_reinit_count :: () -> s64 { return atomic_load(*test_page_reinits); }
test_is_initialized :: () -> bool { return atomic_load(*global_rpmalloc_init_state) == RPMALLOC_INIT_DONE; }
test_hold_heap_lock :: () { heap_lock_acquire(); } // Never released: the U22 scenario.
// Decommit activity.
test_pages_decommitted_of_type :: (page_type: u32) -> s64 { return atomic_load(*test_pages_decommitted[page_type]); }
test_pages_recommitted_of_type :: (page_type: u32) -> s64 { return atomic_load(*test_pages_recommitted[page_type]); }
}
// IS_THIS_YOURS (Jai only; no upstream equivalent).
//
// Every span is mapped at a SPAN_SIZE (256 MiB) aligned address, huge blocks included, so one bit
// per possible span base records which bases are live mappings of this allocator: 2^20 bits
// (128 KiB) cover a 48-bit address space. is_this_yours reads a span header only when its bit is
// set, so it never dereferences memory that is not ours (review §5.7). Walking the heaps' span lists
// instead would race with their owners, miss the huge blocks of thread heaps (which no list
// tracks), and follow the stale huge entries of first-class heaps (U3).
SPAN_BITMAP_BITS :: 1 << (48 - SPAN_SIZE_SHIFT);
SPAN_BITMAP_BYTES :: SPAN_BITMAP_BITS / 8;
global_span_bitmap: *u64; // Mapped once at the first initialize, kept for the life of the process.
span_bitmap_map :: () {
if global_span_bitmap return;
offset, mapped_size: u64;
bitmap := os_mmap(SPAN_BITMAP_BYTES, 0, *offset, *mapped_size);
if bitmap && os_mcommit(bitmap, SPAN_BITMAP_BYTES) != 0 bitmap = null; // Without it, IS_THIS_YOURS says no.
global_span_bitmap = bitmap;
#if ENABLE_STATISTICS {
// Bookkeeping, not allocator memory: take it back out of the counters os_mmap and
// os_mcommit just raised (the peaks keep it).
page_count := mapped_size / global_config.page_size;
statistics_sub_saturating(*global_counters.page_mapped, page_count);
if bitmap {
statistics_sub_saturating(*global_counters.page_commit, SPAN_BITMAP_BYTES / global_config.page_size);
statistics_sub_saturating(*global_counters.page_active, SPAN_BITMAP_BYTES / global_config.page_size);
}
}
}
span_bitmap_slot :: inline (span: *Span) -> word: *u64, bit: u64 {
index := (cast(u64) span) >> SPAN_SIZE_SHIFT;
if !global_span_bitmap || index >= SPAN_BITMAP_BITS return null, 0; // Not tracked: IS_THIS_YOURS says no.
return global_span_bitmap + (index >> 6), (cast(u64) 1) << (index & 63);
}
span_register :: (span: *Span) {
word, bit := span_bitmap_slot(span);
if word atomic_or(word, bit);
}
span_unmap :: (span: *Span) {
word, bit := span_bitmap_slot(span);
if word atomic_and(word, ~bit);
global_memory_interface.memory_unmap(span, span.offset, span.mapped_size);
}
// With a heap: is ptr inside a page of that heap? Without (the default allocator): is it inside a
// page of any thread heap? Blocks of first-class heaps are not the default allocator's. A pointer
// into a free part of an owned page counts as owned: this is about address ranges, not live blocks.
// Asking about a huge block while another thread frees it is a use after free by the caller.
is_this_yours :: (heap: *Heap, ptr: *void) -> bool {
if !ptr || atomic_load(*global_rpmalloc_init_state) != RPMALLOC_INIT_DONE return false;
span := cast(*Span)(cast(u64) ptr & SPAN_MASK);
word, bit := span_bitmap_slot(span);
if !word || !(atomic_load(word) & bit) return false;
address := cast(u64) ptr;
page: *Page;
if span.page_type == .HUGE {
if address < cast(u64) span + SPAN_HEADER_SIZE || address >= cast(u64) span + cast(u64) span.page_count * span.page_size
return false;
page = *span.page;
} else {
if address >= cast(u64) span + cast(u64) atomic_load(*span.page_initialized) * span.page_size return false;
page = span_get_page_from_block(span, ptr);
if address < cast(u64) page + page_block_start_offset(page) return false; // #jai PR #424
}
owner := page.heap;
if !owner return false; // A cached huge mapping.
if heap return owner == heap;
return !owner.is_first_class;
}
//
// Extern interface (2.0.1: rpmalloc.c:2510-2660).
//
// #jai refused arguments return null (or the documented error code) without setting errno: Jai
// code has no errno convention, and on Windows it would mean linking the C runtime.
//
// Sizes are s64 at the Jai API (the Allocator protocol's type) and size_t (u64) inside, like C.
rpmalloc :: (size: s64) -> *void { // 2.0.1: rpmalloc.c:2522
#if ENABLE_VALIDATE_ARGS {
if cast,no_check(u64) size >= MAX_ALLOC_SIZE return null;
}
heap := get_thread_heap();
return heap_allocate_block(heap, cast,no_check(u64) size, false);
}
rpzalloc :: (size: s64) -> *void { // 2.0.1: rpmalloc.c:2534
#if ENABLE_VALIDATE_ARGS {
if cast,no_check(u64) size >= MAX_ALLOC_SIZE return null;
}
heap := get_thread_heap();
return heap_allocate_block(heap, cast,no_check(u64) size, true);
}
rpfree :: (ptr: *void) { // 2.0.1: rpmalloc.c:2546
if !ptr return;
block_deallocate(ptr);
}
rpcalloc :: (num: s64, size: s64) -> *void { // 2.0.1: rpmalloc.c:2553
total, ok := calloc_total(num, size);
if !ok return null;
heap := get_thread_heap();
return heap_allocate_block(heap, total, true);
}
rprealloc :: (ptr: *void, size: s64) -> *void { // 2.0.1: rpmalloc.c:2577
#if ENABLE_VALIDATE_ARGS {
// UPSTREAM-BUG L6: returns the old pointer for a refused size, which callers read as success.
if cast,no_check(u64) size >= MAX_ALLOC_SIZE return ptr;
}
if realloc_fits_in_place(ptr, cast,no_check(u64) size) return ptr; // #jai PR #423
heap := get_thread_heap();
return heap_reallocate_block(heap, ptr, cast,no_check(u64) size, 0, 0);
}
rpaligned_realloc :: (ptr: *void, alignment: s64, size: s64, old_size: s64, flags: Realloc_Flags = 0) -> *void { // 2.0.1: rpmalloc.c:2589
#if ENABLE_VALIDATE_ARGS {
// Upstream refuses alignments above SMALL_PAGE_SIZE here, while rpaligned_alloc accepts
// anything below RPMALLOC_MAX_ALIGNMENT (256 KiB).
// #jai upstream's size + alignment < size, without the wrapping sum (arithmetic_overflow_check).
if cast,no_check(u64) size > 0xFFFF_FFFF_FFFF_FFFF - cast,no_check(u64) alignment || cast,no_check(u64) alignment > SMALL_PAGE_SIZE
return null;
}
heap := get_thread_heap();
return heap_reallocate_block_aligned(heap, ptr, cast,no_check(u64) alignment, cast,no_check(u64) size, cast,no_check(u64) old_size, flags);
}
rpaligned_alloc :: (alignment: s64, size: s64) -> *void { // 2.0.1: rpmalloc.c:2601
heap := get_thread_heap();
return heap_allocate_block_aligned(heap, cast,no_check(u64) alignment, cast,no_check(u64) size, false);
}
rpaligned_zalloc :: (alignment: s64, size: s64) -> *void { // 2.0.1: rpmalloc.c:2607
heap := get_thread_heap();
return heap_allocate_block_aligned(heap, cast,no_check(u64) alignment, cast,no_check(u64) size, true);
}
rpaligned_calloc :: (alignment: s64, num: s64, size: s64) -> *void { // 2.0.1: rpmalloc.c:2613
total, ok := calloc_total(num, size);
if !ok return null;
heap := get_thread_heap();
return heap_allocate_block_aligned(heap, cast,no_check(u64) alignment, total, true);
}
rpmemalign :: (alignment: s64, size: s64) -> *void { // 2.0.1: rpmalloc.c:2637
heap := get_thread_heap();
return heap_allocate_block_aligned(heap, cast,no_check(u64) alignment, cast,no_check(u64) size, false);
}
// Returns 0, EINVAL (memptr is null) or ENOMEM, like posix_memalign.
rpposix_memalign :: (memptr: **void, alignment: s64, size: s64) -> s32 { // 2.0.1: rpmalloc.c:2643
EINVAL :: 22;
ENOMEM :: 12;
heap := get_thread_heap();
if !memptr return EINVAL;
memptr.* = heap_allocate_block_aligned(heap, cast,no_check(u64) alignment, cast,no_check(u64) size, false);
return ifx memptr.* then 0 else ENOMEM;
}
usable_size :: (ptr: *void) -> s64 { // 2.0.1: rpmalloc.c:2653
return ifx ptr then cast(s64) block_usable_size(ptr) else 0;
}
//
// Initialization and finalization (2.0.1: rpmalloc.c:2661-3421).
//
// #jai fix U10: upstream's initialize_config wrote global_config before initialize claimed the init
// state, so it could race a concurrent lazy initialisation. The config is now applied only by the
// thread that claims the state; if the allocator is already initialised it is not applied, as
// upstream.
initialize_config :: (memory_interface: *Memory_Interface, config: *Config) -> s32 { // 2.0.1: rpmalloc.c:2704
result := initialize_with_config(memory_interface, config);
if config config.* = global_config;
return result;
}
initialize :: (memory_interface: *Memory_Interface = null) -> s32 { // 2.0.1: rpmalloc.c:2724
return initialize_with_config(memory_interface, null);
}
#scope_module
initialize_with_config :: (memory_interface: *Memory_Interface, config: *Config) -> s32 {
while true {
state := atomic_load(*global_rpmalloc_init_state);
if state == RPMALLOC_INIT_DONE {
thread_initialize();
return 0;
}
if state == RPMALLOC_INIT_RUNNING {
while atomic_load(*global_rpmalloc_init_state) == RPMALLOC_INIT_RUNNING thread_yield();
continue;
}
if compare_and_swap(*global_rpmalloc_init_state, RPMALLOC_INIT_UNINIT, RPMALLOC_INIT_RUNNING) break;
}
if config global_config = config.*; // #jai fix U10: only after claiming the init state.
// Remember whether the caller asked for huge pages; detection below overwrites
// global_config.enable_huge_pages with what is actually available.
huge_pages_requested := global_config.enable_huge_pages;
// #jai fix U8: upstream installed the OS defaults only when map or unmap was missing, so an
// interface with map and unmap but no commit / decommit was called through null; and it wrote
// the defaults into the caller's struct. Now the caller's interface is copied, and a null commit
// or decommit falls back to the OS. Without map or unmap, all four are the OS's, as upstream.
if memory_interface {
global_memory_interface_copy = memory_interface.*;
global_memory_interface = *global_memory_interface_copy;
if !global_memory_interface.memory_commit global_memory_interface.memory_commit = os_mcommit;
if !global_memory_interface.memory_decommit global_memory_interface.memory_decommit = os_mdecommit;
} else {
global_memory_interface = *global_memory_interface_default;
}
if !global_memory_interface.memory_map || !global_memory_interface.memory_unmap {
global_memory_interface.memory_map = os_mmap;
global_memory_interface.memory_commit = os_mcommit;
global_memory_interface.memory_decommit = os_mdecommit;
global_memory_interface.memory_unmap = os_munmap;
}
// (ENABLE_OVERRIDE's unmap_on_finalize override is C-only; not ported.)
// 2.0.1: rpmalloc.c:2765-2785. #jai os_initialize also serves the test hooks, so force a fresh
// read here, as upstream does on every initialisation.
os_page_size = 0;
os_initialize();
os_huge_pages = false;
os_thp = false;
os_huge_page_shift = 0;
rpmalloc_assert(!global_config.page_size || !(global_config.page_size & (global_config.page_size - 1)), "Configured page size must be a power of two"); // #jai 0 is valid (the OS page size), and 0 - 1 would wrap
// Establish a valid (normal) page size up front, so the allocator never sees a zero page size
// during the rest of initialisation. Huge page detection may raise it.
if !memory_interface || global_config.page_size < os_page_size global_config.page_size = os_page_size;
if global_config.enable_huge_pages os_detect_huge_pages(); // 2.0.1: rpmalloc.c:2797-2893
if huge_pages_requested && !os_huge_pages {
// Huge pages were requested but are not available: fail rather than silently use normal
// pages behind a huge page size. Leave the allocator uninitialised (and the flag cleared)
// so the caller can retry without huge pages.
global_config.enable_huge_pages = false;
atomic_store(*global_rpmalloc_init_state, RPMALLOC_INIT_UNINIT);
return -1;
}
global_config.enable_huge_pages = os_huge_pages;
if os_huge_pages {
while (cast(u64) 1 << os_huge_page_shift) < os_page_size os_huge_page_shift += 1;
}
if !memory_interface || global_config.page_size < os_page_size global_config.page_size = os_page_size;
#if OS == .WINDOWS {
// Large pages on Windows must be committed when mapped and can never be decommitted.
if global_config.enable_huge_pages global_config.disable_decommit = true;
}
os_configure_thp(); // 2.0.1: rpmalloc.c:2921-2936
global_config.enable_thp = os_thp;
// Decommit of free pages only applies to page types larger than the configured page size:
// disable the free page overflow pass for page types where it can never apply (always the case
// with huge pages or a large page size).
page_free_overflow_default :: u32.[16, 8, 4, 2];
builtin_page_size :: u64.[SMALL_PAGE_SIZE, MEDIUM_SMALL_PAGE_SIZE, MEDIUM_LARGE_PAGE_SIZE, LARGE_PAGE_SIZE];
for itype: 0..3 {
page_can_decommit := !global_config.disable_decommit && builtin_page_size[itype] > global_config.page_size;
global_page_free_overflow[itype] = ifx page_can_decommit then page_free_overflow_default[itype] else 0xFFFF_FFFF;
}
span_bitmap_map(); // #jai for IS_THIS_YOURS; needs the final page size (statistics).
os_tls_create(rpmalloc_thread_destructor); // 2.0.1: rpmalloc.c:2951-2955
global_main_thread_id = get_thread_id();
atomic_store(*global_rpmalloc_init_state, RPMALLOC_INIT_DONE);
thread_initialize();
return 0;
}
#scope_export
config :: () -> *Config { // 2.0.1: rpmalloc.c:2967
return *global_config;
}
finalize :: () { // 2.0.1: rpmalloc.c:3031
thread_finalize();
#if HUGE_CACHE_SLOT_COUNT huge_cache_flush(); // 2.0.1: rpmalloc.c:3037
#if ENABLE_LEAK_DETECTION && ENABLE_ASSERTS { // 2.0.1: rpmalloc.c:3040-3047
// Every thread has finalized, so blocks still allocated and not merely waiting in a deferred
// cross-thread free list are blocks the application never freed (or freed twice).
rpmalloc_assert(statistics_outstanding_blocks() <= statistics_pending_deferred_blocks(), "Memory leak detected");
rpmalloc_assert(atomic_load(*global_counters.huge_alloc) == 0, "Memory leak detected");
}
if global_config.unmap_on_finalize {
// Heaps can share a mapping (heap_allocate_new); only the owner has mapped_size != 0.
// Resources are freed while every heap is still mapped, then owners are unmapped in a
// separate pass, so no heap is touched after its shared mapping is gone.
owners: *Heap;
collect :: (list: **Heap) #expand {
heap := list.*;
list.* = null;
while heap {
heap_next := heap.next;
heap_unmap_all(heap);
if heap.mapped_size {
heap.next = `owners;
`owners = heap;
}
heap = heap_next;
}
}
collect(*global_heap_pristine);
collect(*global_heap_queue);
collect(*global_heap_used);
while owners {
owner_next := owners.next;
heap_unmap(owners);
owners = owner_next;
}
#if ENABLE_STATISTICS memset(*global_counters, 0, size_of(Rpmalloc_Statistics)); // 2.0.1: rpmalloc.c:3094
}
os_tls_destroy(); // 2.0.1: rpmalloc.c:3098-3104
// #jai every span is unmapped or abandoned now: forget them all (IS_THIS_YOURS).
if global_span_bitmap memset(global_span_bitmap, 0, SPAN_BITMAP_BYTES);
global_heap_creating = 0;
global_main_thread_id = 0;
atomic_store(*global_rpmalloc_init_state, RPMALLOC_INIT_UNINIT);
}
thread_initialize :: () { // 2.0.1: rpmalloc.c:3112
if get_thread_heap() == global_heap_default get_thread_heap_allocate();
}
thread_finalize :: () { // 2.0.1: rpmalloc.c:3118
heap := get_thread_heap();
if heap != global_heap_default {
heap_release_on_thread_exit(heap); // #jai fix U5
set_thread_heap(global_heap_default);
} else {
heap_release_parked_on_exit(); // #jai fix E7
}
}
thread_collect :: () { // 2.0.1: rpmalloc.c:3127 (empty in 2.0.1; develop #416 fills it in)
heap := get_thread_heap();
if !heap || !heap.id return;
// #jai fix U23: idle huge cache entries were unmapped only when another huge block was freed, so a
// program that stopped using huge blocks kept up to 128 MiB committed for good (RET-08).
#if HUGE_CACHE_SLOT_COUNT huge_cache_purge_aged(os_monotonic_time_ms());
heap_drain_deferred_frees(heap); // #jai fix U6
heap_retire_kept_pages(heap); // #jai fix (U6 part, with #413)
// #jai develop #416: decommit the free pages held above the retain count now, instead of
// waiting for the decommit delay.
for itype: 0..3 {
if global_page_free_overflow[itype] == 0xFFFF_FFFF continue;
heap.page_free_overflow_ms[itype] = 0;
if heap.page_free_commit_count[itype] > global_page_free_retain[itype]
heap_page_free_decommit(heap, cast(u32) itype, global_page_free_retain[itype]);
}
}
// #jai (upstream issue #116): returns as much memory as possible to the OS now. thread_collect keeps
// the usual retain counts per page type and the young huge cache entries; trim also decommits every
// free page of the calling thread's heap and unmaps the whole huge cache. Pages that hold live blocks
// stay, and other threads' heaps are not touched (each thread can call trim itself).
trim :: () {
heap := get_thread_heap();
if heap && heap.id {
heap_drain_deferred_frees(heap);
heap_retire_kept_pages(heap);
for itype: 0..3 {
heap.page_free_overflow_ms[itype] = 0;
if heap.page_free_commit_count[itype] heap_page_free_decommit(heap, cast(u32) itype, 0);
}
}
#if HUGE_CACHE_SLOT_COUNT huge_cache_flush();
}
// #jai fix E7 (Ex4 1.45.0, items 1 and 3 of its analysis): frees by other threads into a heap queue up
// for its owner, which takes them in only on its next allocation, and a heap keeps some free pages
// committed for reuse. A pool thread that goes idle makes no allocator call, possibly for a long time,
// so its heap kept the pages of every block other threads freed into it, and its free pages: 185 MB
// and 35 MB in Ex4, with 24 idle walk workers (RET-20). Only the thread knows it is going idle, so it
// says so: thread_idle gives back what it can, as trim() does for its heap (pending frees taken in,
// kept empty pages retired, every free page decommitted, aged huge cache entries decommitted), and
// parks the heap. While parked, frees by other threads into it are freed at once under the heap's
// lock, as into an exited thread's heap (E4, E6), so their pages are decommitted as they empty. The
// thread's next allocator call takes the heap back. Call it where a pool thread waits for work; it
// costs a few OS calls, and the first allocation after it takes the slow path once.
thread_idle :: () {
heap := get_thread_heap();
if !heap || !heap.id || heap.is_first_class return; // No thread heap (or a first-class one made current).
heap_drain_deferred_frees(heap);
heap_retire_kept_pages(heap);
for itype: 0..3 {
heap.page_free_overflow_ms[itype] = 0;
if heap.page_free_commit_count[itype] heap_page_free_decommit(heap, cast(u32) itype, 0);
}
#if HUGE_CACHE_SLOT_COUNT huge_cache_purge_aged(os_monotonic_time_ms());
me := get_thread_id();
heap_lock_acquire();
heap_free_lock_acquire(heap);
heap.parked_thread = me;
heap.owner_thread = HEAP_RELEASED_OWNER; // Frees by any thread take the released-heap path.
heap_free_lock_release(heap);
atomic_add(*global_parked_count, 1);
heap_lock_release();
// The fallback heap: the fast paths find nothing, and the next call unparks (heap_take_parked).
// Non-null, so the POSIX thread exit destructor still runs and releases the parked heap.
heap.thread_current = false; // #jai fix E10: no Context cache may hand it out while parked.
os_tls_set(global_heap_default);
}
#scope_module
// A thread that exits (or sends THREAD_STOP) with its heap parked: the heap goes to the queue for
// reuse. It was trimmed when it was parked, and frees since were applied at once.
heap_release_parked_on_exit :: () {
heap := heap_take_parked();
if !heap return;
heap_retire_kept_pages(heap);
for itype: 0..3 {
heap.page_free_overflow_ms[itype] = 0;
if heap.page_free_commit_count[itype] heap_page_free_decommit(heap, cast(u32) itype, 0);
}
heap_release_to_queue(heap);
}
#scope_export
is_thread_initialized :: () -> bool { // 2.0.1: rpmalloc.c:2517
return get_thread_heap() != global_heap_default;
}
thread_statistics :: (stats: *Thread_Statistics) { // 2.0.1: rpmalloc.c:3131
memset(stats, 0, size_of(Thread_Statistics));
#if ENABLE_STATISTICS {
heap := get_thread_heap();
if !heap || heap.id == 0 return;
// Size class cache: blocks available without mapping a new page. A page's block_used counts
// the blocks held on the heap free list, so those are counted from the heap list, and each
// page adds its remaining blocks (its free list plus the not yet initialised tail).
for iclass: 0..SIZE_CLASS_COUNT - 1 {
block_size := cast(u64) global_size_class[iclass].block_size;
free_count: u64;
block := heap.local_free[iclass];
while block {
free_count += 1;
block = block.next;
}
page := heap.page_available[iclass];
while page {
free_count += cast(u64)(page.block_count - page.block_used);
page = page.next;
}
stats.sizecache += free_count * block_size;
su := *heap.size_use[iclass];
// #jai fix U13: a transiently negative count (frees counted before the matching
// allocation is adopted) reads as 0, not as a huge size_t.
stats.size_use[iclass].alloc_current = cast(u64) max(su.alloc_current, 0);
stats.size_use[iclass].alloc_peak = cast(u64) max(su.alloc_peak, 0);
stats.size_use[iclass].alloc_total = cast(u64) max(su.alloc_total, 0);
stats.size_use[iclass].free_total = cast(u64) max(su.free_total, 0);
}
// Span cache: free but still committed pages kept per page type.
page_type_size :: u64.[SMALL_PAGE_SIZE, MEDIUM_SMALL_PAGE_SIZE, MEDIUM_LARGE_PAGE_SIZE, LARGE_PAGE_SIZE];
for itype: 0..3 stats.spancache += cast(u64) heap.page_free_commit_count[itype] * page_type_size[itype];
// Span use per page type: current spans counted live (partial plus full); map calls counted.
for itype: 0..4 {
current: u64;
if itype < 4 && heap.span_partial[itype] current += 1;
span := heap.span_used[itype];
while span {
current += 1;
span = span.next;
}
stats.span_use[itype].current = current;
stats.span_use[itype].map_calls = heap.span_map_calls[itype];
}
}
}
// #jai appends to a String_Builder instead of writing to a FILE*.
dump_statistics :: (builder: *String_Builder) { // 2.0.1: rpmalloc.c:3176
#if ENABLE_STATISTICS {
g := *global_counters;
print_to_builder(builder, "Mapped pages: %\n", atomic_load(*g.page_mapped));
print_to_builder(builder, "Mapped pages (peak): %\n", atomic_load(*g.page_mapped_peak));
print_to_builder(builder, "Active pages: %\n", atomic_load(*g.page_active));
print_to_builder(builder, "Active pages (peak): %\n", atomic_load(*g.page_active_peak));
print_to_builder(builder, "Pages committed: %\n", atomic_load(*g.page_commit));
print_to_builder(builder, "Pages decommitted: %\n", atomic_load(*g.page_decommit));
print_to_builder(builder, "Huge bytes: %\n", atomic_load(*g.huge_alloc));
print_to_builder(builder, "Huge bytes (peak): %\n", atomic_load(*g.huge_alloc_peak));
print_to_builder(builder, "Heaps in use: %\n", atomic_load(*g.heap_count));
#if ENABLE_LEAK_DETECTION
print_to_builder(builder, "Outstanding blocks: %\n", statistics_outstanding_blocks() - statistics_pending_deferred_blocks());
// Aggregate the per size class and per page type counters across all in-use heaps. This
// walks live heaps without stopping the world, so the values are a best-effort snapshot.
size_use: [SIZE_CLASS_COUNT] Heap_Size_Use;
span_map_calls: [5] u64;
heap_lock_acquire();
heap := global_heap_used;
while heap {
for iclass: 0..SIZE_CLASS_COUNT - 1 {
size_use[iclass].alloc_current += heap.size_use[iclass].alloc_current;
size_use[iclass].alloc_peak += heap.size_use[iclass].alloc_peak;
size_use[iclass].alloc_total += heap.size_use[iclass].alloc_total;
size_use[iclass].free_total += heap.size_use[iclass].free_total;
}
for itype: 0..4 span_map_calls[itype] += heap.span_map_calls[itype];
heap = heap.next;
}
heap_lock_release();
print_to_builder(builder, "SizeClass CurAlloc PeakAlloc TotAlloc TotFree BlockSize\n");
for size_use {
if !it.alloc_total continue;
print_to_builder(builder, "% % % % % %\n", formatInt(it_index, minimum_digits = 9, padding = #char " "),
formatInt(it.alloc_current, minimum_digits = 9, padding = #char " "), formatInt(it.alloc_peak, minimum_digits = 9, padding = #char " "),
formatInt(it.alloc_total, minimum_digits = 9, padding = #char " "), formatInt(it.free_total, minimum_digits = 9, padding = #char " "),
formatInt(global_size_class[it_index].block_size, minimum_digits = 9, padding = #char " "));
}
print_to_builder(builder, "PageType MapCalls\n");
for span_map_calls {
if !it continue;
print_to_builder(builder, "% %\n", formatInt(it_index, minimum_digits = 8, padding = #char " "), formatInt(it, minimum_digits = 9, padding = #char " "));
}
}
}
global_statistics :: (stats: *Global_Statistics) { // 2.0.1: rpmalloc.c:3240
#if ENABLE_STATISTICS {
g := *global_counters;
page_size := global_config.page_size;
stats.mapped = page_size * atomic_load(*g.page_mapped);
stats.mapped_peak = page_size * atomic_load(*g.page_mapped_peak);
stats.committed = page_size * atomic_load(*g.page_commit);
stats.decommitted = page_size * atomic_load(*g.page_decommit);
stats.active = page_size * atomic_load(*g.page_active);
stats.active_peak = page_size * atomic_load(*g.page_active_peak);
stats.huge_alloc = atomic_load(*g.huge_alloc);
stats.huge_alloc_peak = atomic_load(*g.huge_alloc_peak);
stats.heap_count = atomic_load(*g.heap_count);
} else {
memset(stats, 0, size_of(Global_Statistics));
}
}
#if FIRST_CLASS_HEAPS {
// First-class heaps (2.0.1: rpmalloc.c:3255-3418). A first-class heap has no owner thread
// (owner_thread 0): only one thread may use it at a time, and heap_free_all releases every block
// it ever handed out.
heap_acquire :: () -> *Heap { // 2.0.1: rpmalloc.c:3259
// Must be a pristine heap, or else blocks could already be allocated from it, which
// heap_free_all would (wrongly) release. heap_allocate(true) returns a never-used heap carved
// from a shared block, or a freshly mapped one (including heaps released empty, #jai fix U20).
heap := heap_allocate(true);
// #jai fix U9: upstream's rpmalloc_assume(heap != 0) turned out-of-memory into a null
// dereference. Return null, as CREATE_HEAP's protocol expects.
if !heap return null;
heap.owner_thread = 0;
return heap;
}
heap_release :: (heap: *Heap) { // 2.0.1: rpmalloc.c:3271
// #jai fix U14: see heap_release_to_queue. The released flag is checked again under the lock.
if heap && heap != global_heap_default && !heap.released { // #jai fix U2: never the fallback heap.
if heap_is_empty(heap) heap_release_to_pristine(heap); // #jai fix U20
else heap_release_trim(heap); // #jai develop #418, #419
}
}
heap_alloc :: (heap: *Heap, size: s64) -> *void { // 2.0.1: rpmalloc.c:3277
#if ENABLE_VALIDATE_ARGS {
if cast,no_check(u64) size >= MAX_ALLOC_SIZE return null;
}
return heap_allocate_block(heap, cast,no_check(u64) size, false);
}
heap_aligned_alloc :: (heap: *Heap, alignment: s64, size: s64) -> *void { // 2.0.1: rpmalloc.c:3288
#if ENABLE_VALIDATE_ARGS {
if cast,no_check(u64) size >= MAX_ALLOC_SIZE return null;
}
return heap_allocate_block_aligned(heap, cast,no_check(u64) alignment, cast,no_check(u64) size, false);
}
heap_aligned_zalloc :: (heap: *Heap, alignment: s64, size: s64) -> *void { // 2.0.1: rpmalloc.c:3299
#if ENABLE_VALIDATE_ARGS {
if cast,no_check(u64) size >= MAX_ALLOC_SIZE return null;
}
return heap_allocate_block_aligned(heap, cast,no_check(u64) alignment, cast,no_check(u64) size, true);
}
heap_calloc :: (heap: *Heap, num: s64, size: s64) -> *void { // 2.0.1: rpmalloc.c:3310
total, ok := calloc_total(num, size);
if !ok return null;
return heap_allocate_block(heap, total, true);
}
heap_aligned_calloc :: (heap: *Heap, alignment: s64, num: s64, size: s64) -> *void { // 2.0.1: rpmalloc.c:3333
total, ok := calloc_total(num, size);
if !ok return null;
return heap_allocate_block_aligned(heap, cast,no_check(u64) alignment, total, true);
}
heap_realloc :: (heap: *Heap, ptr: *void, size: s64, flags: Realloc_Flags = 0) -> *void { // 2.0.1: rpmalloc.c:3356
#if ENABLE_VALIDATE_ARGS {
// UPSTREAM-BUG L6: returns the old pointer for a refused size, which callers read as success.
if cast,no_check(u64) size >= MAX_ALLOC_SIZE return ptr;
}
return heap_reallocate_block(heap, ptr, cast,no_check(u64) size, 0, flags);
}
heap_aligned_realloc :: (heap: *Heap, ptr: *void, alignment: s64, size: s64, flags: Realloc_Flags = 0) -> *void { // 2.0.1: rpmalloc.c:3367
#if ENABLE_VALIDATE_ARGS {
// #jai upstream's size + alignment < size, without the wrapping sum (arithmetic_overflow_check).
if cast,no_check(u64) size > 0xFFFF_FFFF_FFFF_FFFF - cast,no_check(u64) alignment || cast,no_check(u64) alignment > SMALL_PAGE_SIZE
return null;
}
return heap_reallocate_block_aligned(heap, ptr, cast,no_check(u64) alignment, cast,no_check(u64) size, 0, flags);
}
heap_free :: (heap: *Heap, ptr: *void) { // 2.0.1: rpmalloc.c:3378
// The heap argument is unused: the block's page knows its heap.
// #jai fix U18: rpfree checks for null, upstream's heap_free did not and dereferenced 0.
if !ptr return;
// #jai PR #422 compares the page with the given heap, which upstream trusts. A non-owner thread
// passing the owner's thread heap (heap_free(get_heap_for_ptr(p), p) reads naturally) would then
// take the owner's unsynchronised local path (HEAP-09). Only a first-class heap, which has no
// owner thread, or the caller's own thread heap is trusted; anything else frees as rpfree does.
owned := get_thread_heap();
if heap && !heap.owner_thread owned = heap;
block_deallocate_heap(ptr, owned);
}
// Frees every block the heap has handed out.
heap_free_all :: (heap: *Heap) { // 2.0.1: rpmalloc.c:3385
heap_unmap_all(heap);
}
heap_statistics :: (heap: *Heap) -> Heap_Statistics { // 2.0.1: rpmalloc.c:3390
#if HEAP_STATISTICS {
if heap return heap.stats;
}
return .{};
}
heap_thread_set_current :: (heap: *Heap) { // 2.0.1: rpmalloc.c:3402
prev_heap := get_thread_heap();
if prev_heap != heap {
// set_thread_heap stamps owner_thread on a first-class heap, which makes it a thread heap
// that the thread's exit releases; a later heap_release of it is ignored (#jai fix U14).
set_thread_heap(heap);
// #jai fix U2: a thread that has not allocated yet still has the static fallback heap as its
// current heap. Upstream queued it for reuse like a real heap, and later new threads were
// handed it (or, with the queue now inconsistent, a heap another thread owns).
if prev_heap && prev_heap != global_heap_default heap_release_to_queue(prev_heap);
}
}
// The heap that owns the page of `ptr`. Dereferences the span header of any pointer it is given,
// so it is only for pointers known to come from this allocator (IS_THIS_YOURS does not use it).
get_heap_for_ptr :: (ptr: *void) -> *Heap { // 2.0.1: rpmalloc.c:3412
span := cast(*Span)(cast(u64) ptr & SPAN_MASK);
if span return span_get_page_from_block(span, ptr).heap;
return null;
}
} // #if FIRST_CLASS_HEAPS
#scope_module
#import "Basic";
#import "Bit_Operations";
#import "Atomics"; // atomic_add returns the previous value, like C atomic_fetch_add.
Reference in New Issue View Git Blame Copy Permalink