// The allocator core: a port of rpmalloc 2.0.1 rpmalloc.c.
//
// Layout follows the C file section by section, in the same order. Procedures keep the C name
// (minus rpmalloc_ prefixes) and carry a `// 2.0.1: rpmalloc.c:NNNN` reference, so the port can be
// compared with upstream and later upstream fixes applied mechanically.
//
// Deliberate differences from C are marked `// #jai` with the reason. Known upstream bugs that are
// ported as-is, until their fix lands as a separate commit, are marked `// UPSTREAM-BUG <id>`; the
// ids and the fix order are in PLAN.md.

////////////
///
/// Build time configurable limits (2.0.1: rpmalloc.c:135-190)
///
//////
// Module parameters replace the C #defines (ENABLE_VALIDATE_ARGS, ENABLE_ASSERTS, ENABLE_UNMAP,
// ENABLE_DECOMMIT, ENABLE_STATISTICS, ENABLE_LEAK_DETECTION); see module.jai. ENABLE_OVERRIDE
// and ENABLE_DYNAMIC_LINK are C-only and are not ported.

////////////
///
/// Built in size configurations (2.0.1: rpmalloc.c:191-241)
///
//////

PAGE_HEADER_SIZE :: 128;                 // 2.0.1: rpmalloc.c:197
SPAN_HEADER_SIZE :: PAGE_HEADER_SIZE;    // 2.0.1: rpmalloc.c:198

// Alignment of each heap control structure within a shared mapping, so adjacent heaps sit on
// separate cache lines (128 bytes covers Apple Silicon and x86 destructive interference).
HEAP_ALIGNMENT :: 128;                   // 2.0.1: rpmalloc.c:204

SMALL_GRANULARITY :: 16;                 // 2.0.1: rpmalloc.c:206

SMALL_BLOCK_SIZE_LIMIT        :: 4 * 1024;           // 2.0.1: rpmalloc.c:208
MEDIUM_SMALL_BLOCK_SIZE_LIMIT :: 32 * 1024;          // 2.0.1: rpmalloc.c:209
MEDIUM_LARGE_BLOCK_SIZE_LIMIT :: 256 * 1024;         // 2.0.1: rpmalloc.c:210
LARGE_BLOCK_SIZE_LIMIT        :: 2 * 1024 * 1024;    // 2.0.1: rpmalloc.c:211
NATURAL_ALIGNMENT_LIMIT       :: 2048;               // #jai upstream PR #424: largest alignment a class gets from its start offset.

SMALL_SIZE_CLASS_COUNT        :: 73;     // 2.0.1: rpmalloc.c:213
MEDIUM_SMALL_SIZE_CLASS_COUNT :: 12;     // 2.0.1: rpmalloc.c:214
MEDIUM_LARGE_SIZE_CLASS_COUNT :: 12;     // 2.0.1: rpmalloc.c:215
LARGE_SIZE_CLASS_COUNT        :: 12;     // 2.0.1: rpmalloc.c:216
SIZE_CLASS_COUNT :: SMALL_SIZE_CLASS_COUNT + MEDIUM_SMALL_SIZE_CLASS_COUNT + MEDIUM_LARGE_SIZE_CLASS_COUNT + LARGE_SIZE_CLASS_COUNT;   // 2.0.1: rpmalloc.c:217

SMALL_PAGE_SIZE_SHIFT        :: 16;                                           // 2.0.1: rpmalloc.c:220
SMALL_PAGE_SIZE              :: 1 << SMALL_PAGE_SIZE_SHIFT;
SMALL_PAGE_MASK              :: ~cast(u64)(SMALL_PAGE_SIZE - 1);
MEDIUM_SMALL_PAGE_SIZE_SHIFT :: 20;                                           // 2.0.1: rpmalloc.c:223
MEDIUM_SMALL_PAGE_SIZE       :: 1 << MEDIUM_SMALL_PAGE_SIZE_SHIFT;
MEDIUM_SMALL_PAGE_MASK       :: ~cast(u64)(MEDIUM_SMALL_PAGE_SIZE - 1);
MEDIUM_LARGE_PAGE_SIZE_SHIFT :: 22;                                           // 2.0.1: rpmalloc.c:226
MEDIUM_LARGE_PAGE_SIZE       :: 1 << MEDIUM_LARGE_PAGE_SIZE_SHIFT;
MEDIUM_LARGE_PAGE_MASK       :: ~cast(u64)(MEDIUM_LARGE_PAGE_SIZE - 1);
LARGE_PAGE_SIZE_SHIFT        :: 24;                                           // 2.0.1: rpmalloc.c:229
LARGE_PAGE_SIZE              :: 1 << LARGE_PAGE_SIZE_SHIFT;
LARGE_PAGE_MASK              :: ~cast(u64)(LARGE_PAGE_SIZE - 1);

SPAN_SIZE :: 256 * 1024 * 1024;          // 2.0.1: rpmalloc.c:233
SPAN_MASK :: ~cast(u64)(SPAN_SIZE - 1);  // 2.0.1: rpmalloc.c:234
SPAN_SIZE_SHIFT :: 28;                    // #jai for the IS_THIS_YOURS span registry
#assert (1 << SPAN_SIZE_SHIFT) == SPAN_SIZE;

// Maximum allocation size to avoid integer overflow (2.0.1: rpmalloc.c:236-240; upstream only
// defines it with ENABLE_VALIDATE_ARGS).
MAX_ALLOC_SIZE :: 0xFFFF_FFFF_FFFF_FFFF - cast(u64) SPAN_SIZE;

// Geometry invariants. Upstream has none, so changing SPAN_SIZE silently breaks the allocator
// (upstream issue #364). #jai checked at compile time.
#assert (SMALL_PAGE_SIZE & (SMALL_PAGE_SIZE - 1)) == 0 && (SPAN_SIZE & (SPAN_SIZE - 1)) == 0  "page and span sizes must be powers of two";
#assert SMALL_PAGE_SIZE < MEDIUM_SMALL_PAGE_SIZE && MEDIUM_SMALL_PAGE_SIZE < MEDIUM_LARGE_PAGE_SIZE && MEDIUM_LARGE_PAGE_SIZE < LARGE_PAGE_SIZE  "page sizes must increase by page type";
#assert LARGE_PAGE_SIZE <= SPAN_SIZE  "a span must hold at least one large page";
#assert SPAN_SIZE <= 0xFFFF_FFFF  "page offsets within a span must fit in 32 bits";
#assert RPMALLOC_MAX_ALIGNMENT < LARGE_PAGE_SIZE  "huge aligned blocks must stay within the span header's reach";
#assert PAGE_HEADER_SIZE % SMALL_GRANULARITY == 0  "blocks must start 16-byte aligned";

////////////
///
/// Utility macros (2.0.1: rpmalloc.c:242-281)
///
//////
// EXPECTED / UNEXPECTED have no Jai equivalent and are dropped.

pointer_offset :: inline (ptr: *void, offset: $T) -> *void {   // 2.0.1: rpmalloc.c:132
    return cast(*u8) ptr + cast,no_check(s64) offset;
}

pointer_diff :: inline (first: *void, second: *void) -> s64 {   // 2.0.1: rpmalloc.c:133
    return cast(*u8) first - cast(*u8) second;
}

// 2.0.1: rpmalloc.c:256-268. #jai reports through the memory interface's error_callback when one
// is set (upstream declares that callback but never calls it), otherwise through
// context.assertion_failed.
#if ENABLE_ASSERTS {
    rpmalloc_assert :: (truth: bool, message: string, loc := #caller_location) {
        if truth  return;
        iface := global_memory_interface;
        if iface && iface.error_callback {
            iface.error_callback(message);
            return;
        }
        context.handling_assertion_failure = true;
        context.assertion_failed(loc, message);
        context.handling_assertion_failure = false;
    }
} else {
    rpmalloc_assert :: ($truth: Code, message: string) #expand {}
}

////////////
///
/// Statistics (2.0.1: rpmalloc.c:282-360)
///
//////

#if ENABLE_STATISTICS {

Rpmalloc_Statistics :: struct {   // 2.0.1: rpmalloc.c:290
    page_mapped:      u64;
    page_mapped_peak: u64;
    page_commit:      u64;
    page_decommit:    u64;
    page_active:      u64;
    page_active_peak: u64;
    huge_alloc:       u64;
    huge_alloc_peak:  u64;
    heap_count:       u64;
}

global_counters: Rpmalloc_Statistics;   // 2.0.1: rpmalloc.c:302 global_statistics (renamed: global_statistics is the public procedure)

// Subtracts from a counter, clamping at zero. Counters are in units of the configured page size,
// and heaps with committed pages are kept across finalize and initialize cycles where the page
// size can change, so strict accounting is not possible.
statistics_sub_saturating :: (counter: *u64, value: u64) {   // 2.0.1: rpmalloc.c:308
    current := atomic_load(counter);
    while true {
        next := ifx current > value then current - value else 0;
        swapped, actual := compare_and_swap(counter, current, next);
        if swapped  break;
        current = actual;
    }
}

// #jai the add-then-raise-the-peak loop that upstream writes out at each site.
statistics_add_with_peak :: (counter: *u64, peak: *u64, value: u64) {
    current := atomic_add(counter, value) + value;
    observed := atomic_load(peak);
    while current > observed {
        swapped, actual := compare_and_swap(peak, observed, current);
        if swapped  break;
        observed = actual;
    }
}

// Per size class allocation counters, updated only by the owning thread (cross-thread frees are
// deferred and counted when the owner processes them), so plain integers suffice.
// #jai fix U13: 64-bit; upstream's 32-bit counters wrap on long runs (undefined behaviour in C).
Heap_Size_Use :: struct {   // 2.0.1: rpmalloc.c:319
    alloc_current: s64;
    alloc_peak:    s64;
    alloc_total:   s64;
    free_total:    s64;
}

}   // #if ENABLE_STATISTICS

// #jai fix U13: heap statistics are updated by the threads that free into a heap too, so every
// update is atomic (upstream used plain += and -=). Only HEAP_STATISTICS builds pay for it.
// Subtraction adds the two's complement, and a counter can be below zero for a moment (the first
// allocation already sees -16), so the sums wrap on purpose, as upstream's unsigned arithmetic does.
heap_stat_add :: inline (field: *u64, value: u64) { atomic_add_wrapping(field, value); }
heap_stat_sub :: inline (field: *u64, value: u64) { atomic_add_wrapping(field, cast,no_check(u64) -cast,no_check(s64) value); }

// #jai Jai's atomic_add wraps on X64 (lock xadd), but on other CPUs it computes old + value in Jai,
// which traps in a program built with arithmetic_overflow_check = .FATAL.
atomic_add_wrapping :: inline (field: *u64, value: u64) {
    #if CPU == .X64 {
        atomic_add(field, value);
    } else {
        old := atomic_load(field);
        while true {
            new := ifx old > 0xFFFF_FFFF_FFFF_FFFF - value then old - (0xFFFF_FFFF_FFFF_FFFF - value) - 1 else old + value;
            swapped, actual := compare_and_swap(field, old, new);
            if swapped  break;
            old = actual;
        }
    }
}

// The _rpmalloc_stat_* macros (2.0.1: rpmalloc.c:327-360); no-ops without ENABLE_STATISTICS.
stat_inc_alloc :: inline (heap: *Heap, class_idx: u32) {
    #if ENABLE_STATISTICS {
        su := *heap.size_use[class_idx];
        su.alloc_current += 1;
        if su.alloc_current > su.alloc_peak  su.alloc_peak = su.alloc_current;
        su.alloc_total += 1;
    }
}

stat_add_free :: inline (heap: *Heap, class_idx: u32, count: u32) {
    #if ENABLE_STATISTICS {
        su := *heap.size_use[class_idx];
        su.alloc_current -= cast(s64) count;
        su.free_total += cast(s64) count;
    }
}

stat_inc_span_map :: inline (heap: *Heap, page_type: Page_Type) {
    #if ENABLE_STATISTICS  heap.span_map_calls[cast(u32) page_type] += 1;
}

// The OS layer's counters (2.0.1: rpmalloc.c:1023-1033, 1067-1078, 1129-1133, 1154-1158).
// #jai each skips counting while the page size is still unknown: the white-box tests (W-05, W-06)
// call the OS layer before initialisation, which upstream never does.
statistics_os_map :: inline (map_size: u64) {
    if !global_config.page_size  return;
    #if ENABLE_STATISTICS  statistics_add_with_peak(*global_counters.page_mapped, *global_counters.page_mapped_peak, map_size / global_config.page_size);
}

statistics_os_commit :: inline (size: u64) {
    if !global_config.page_size  return;
    #if ENABLE_STATISTICS {
        page_count := size / global_config.page_size;
        atomic_add(*global_counters.page_commit, page_count);
        statistics_add_with_peak(*global_counters.page_active, *global_counters.page_active_peak, page_count);
    }
}

statistics_os_decommit :: inline (size: u64) {
    if !global_config.page_size  return;
    #if ENABLE_STATISTICS {
        page_count := size / global_config.page_size;
        atomic_add(*global_counters.page_decommit, page_count);
        statistics_sub_saturating(*global_counters.page_active, page_count);
    }
}

statistics_os_unmap :: inline (mapped_size: u64) {
    if !global_config.page_size  return;
    #if ENABLE_STATISTICS {
        page_count := mapped_size / global_config.page_size;
        statistics_sub_saturating(*global_counters.page_mapped, page_count);
        statistics_sub_saturating(*global_counters.page_active, page_count);
    }
}

////////////
///
/// Low level abstractions (2.0.1: rpmalloc.c:361-476)
///
//////
// memset_const / memcpy_const are plain memset / memcpy.

// Yield the rest of this thread's quantum, so another runnable thread (for example a lock holder
// that was preempted) can run. A real scheduler yield, unlike the CPU hint in wait_spin.
thread_yield :: inline () {   // 2.0.1: rpmalloc.c:396
    os_thread_yield();
}

// Number of cheap CPU-pause spins before wait_spin escalates to an OS scheduler yield.
WAIT_SPIN_YIELD_THRESHOLD :: 100;   // 2.0.1: rpmalloc.c:406

wait_spin :: inline (spin_count: *u32) {   // 2.0.1: rpmalloc.c:410
    spin_count.* += 1;
    if spin_count.* >= WAIT_SPIN_YIELD_THRESHOLD {
        thread_yield();
        return;
    }
    #if CPU == .X64 {
        #asm { pause; }
    } else #if CPU == .ARM64 {
        #bytes .[0x3F, 0x20, 0x03, 0xD5];   // YIELD
    }
}

// #jai Basic has no memmove, and faking it with memcpy was review finding M7. Only the
// realloc-of-an-aligned-block paths move overlapping data, so a simple version is enough.
memmove_rp :: (dest: *void, source: *void, count: s64) {
    if dest == source || count <= 0  return;
    d := cast(*u8) dest;
    s := cast(*u8) source;
    if d + count <= s || s + count <= d {
        memcpy(d, s, count);
    } else if d < s {
        for i: 0..count-1  d[i] = s[i];
    } else {
        for < i: count-1..0  d[i] = s[i];
    }
}

// Index of the most significant set bit of a non-zero value. Replaces rpmalloc_clz
// (2.0.1: rpmalloc.c:368), which get_size_class uses as 63 - clz(x).
// #jai bsr on x64, which needs no LZCNT support (the MSVC build's __lzcnt64 silently gives wrong
// answers on CPUs without LZCNT, see upstream issue #404). Elsewhere a six-step binary search,
// because Bit_Operations' non-x64 fallback scans bit by bit.
msb_index :: inline (x: u64) -> u32 {
    #if CPU == .X64 {
        return cast(u32)(bit_scan_reverse(x) - 1);
    } else {
        n: u32 = 0;
        if x >= 1 << 32 { x >>= 32; n += 32; }
        if x >= 1 << 16 { x >>= 16; n += 16; }
        if x >= 1 << 8  { x >>= 8;  n += 8;  }
        if x >= 1 << 4  { x >>= 4;  n += 4;  }
        if x >= 1 << 2  { x >>= 2;  n += 2;  }
        if x >= 1 << 1  {           n += 1;  }
        return n;
    }
}

////////////
///
/// Data types (2.0.1: rpmalloc.c:477-637)
///
//////

Page_Type :: enum u8 {   // 2.0.1: rpmalloc.c:495 page_type_t
    SMALL;          // 64 KiB pages
    MEDIUM_SMALL;   // 1 MiB pages
    MEDIUM_LARGE;   // 4 MiB pages
    LARGE;          // 16 MiB pages
    HUGE;
}

Size_Class :: struct {    // 2.0.1: rpmalloc.c:504 size_class_t
    block_size:  u32;     // Size of blocks in this class.
    block_count: u32;     // Number of blocks in each page.
}


Block :: struct {   // 2.0.1: rpmalloc.c:512 block_t
    next: *Block;
}

// The packed flag word of page_t (2.0.1: rpmalloc.c:532-552). Bits 0-6 are flags; bits 8-13 hold
// commit_prefix_shift (see page_commit_prefix_shift). Packing keeps Page at 72 bytes and Span at
// exactly SPAN_HEADER_SIZE, as upstream.
Page_Flags :: enum_flags u32 {
    IS_FULL;            // Part of the heap full list.
    IS_FREE;            // Part of the heap free list.
    IS_ZERO;            // Blocks are zero initialised.
    IS_DECOMMITTED;     // Memory pages have been decommitted.
    // Has aligned blocks. Note: has_aligned_block and is_full are read on the cross-thread free
    // path while the owner thread writes this word. The owner is the only writer, the word is
    // naturally aligned and the flags read are monotonic/tolerant, so the race is benign
    // (upstream comment, rpmalloc.c:539-545).
    HAS_ALIGNED_BLOCK;
    COMMIT_ON_DEMAND;   // The span maps reserve-only; pages commit on demand and may be decommitted.
}
COMMIT_PREFIX_SHIFT_BIT  :: 8;
COMMIT_PREFIX_SHIFT_MASK :: 0x3F;

Page :: struct {   // 2.0.1: rpmalloc.c:518 page_t
    size_class:        u32;
    block_size:        u32;
    block_count:       u32;
    block_initialized: u32;
    block_used:        u32;
    // Flags, commit_prefix_shift and (#jai PR #422) the page type, see Page_Flags. The page type moved
    // into this word, as upstream's PR #422 does, so Page.heap_fast fits: Span stays 128 bytes.
    flags:             Page_Flags;
    local_free:        *Block;       // Local free list.
    heap:              *Heap;        // Owning heap.
    // #jai upstream PR #422 (867ea07): the owning heap while the page takes the fast free path, null
    // for huge, full or aligned-block pages, so the free fast path checks ownership and eligibility
    // with one compare. (Replaces 2.0.1's generic_free flag and local_free_count, which always equals
    // block_initialized - block_used.)
    heap_fast:         *Heap;
    next:              *Page;
    prev:              *Page;
    // Multithreaded free list: block index in the low 32 bits, list count in the high 32 bits.
    thread_free:       u64;
}

Span :: struct {   // 2.0.1: rpmalloc.c:568 span_t
    page:              Page;         // Page header of page 0.
    // Owning heap. Not deduplicated into page.heap, which is the owner of page 0 as a memory page,
    // while the span owner must stay stable.
    heap:              *Heap;
    page_address_mask: u64;
    page_initialized:  u32;          // Number of pages initialized.
    page_count:        u32;          // Number of pages in use (huge: number of OS pages).
    page_size:         u32;          // Bytes per page.
    page_type:         Page_Type;
    offset:            u32;          // Offset to the start of the mapped region.
    mapped_size:       u64;
    next:              *Span;
}

// Control structure for a heap: a thread heap, or a first-class heap.
Heap :: struct {   // 2.0.1: rpmalloc.c:594 heap_t
    // #jai develop #409 (3302251): the first HEAP_ALIGNMENT bytes hold the fields other threads
    // access. owner_thread is read and thread_free is written by every cross-thread free into a full
    // page; keeping them apart from the owner's hot allocation state avoids invalidating the owner's
    // local free lists on every remote free, and the remote CAS line being pulled away by owner writes.
    owner_thread:           u64;                          // Owning thread id (0: first-class heap).
    thread_free:            [4] u64;                      // Multithreaded free list (atomic) per type.
    // #jai fix E6: taken by frees into the heap while it is released (E4), and by the thread adopting
    // it, so those frees no longer take the global heap lock. On the line other threads write anyway.
    free_lock:              u64;
    remote_pad:             [HEAP_ALIGNMENT - 2 * size_of(u64) - 4 * size_of(u64)] u8;
    local_free:             [SIZE_CLASS_COUNT] *Block;    // Heap local free list per size class.
    page_available:         [SIZE_CLASS_COUNT] *Page;     // Non-full pages per size class.
    page_free:              [4] *Page;                    // Free pages per page type, committed pages first.
    page_free_tail:         [4] *Page;                    // #jai PR #420: last page of each free list.
    page_free_commit_count: [4] u32;                      // Free but still committed pages per type.
    // #jai develop #416: when (monotonic ms) the free committed page count of each page type first
    // went over the overflow threshold; zero while below it.
    page_free_overflow_ms:  [4] u64;
    span_partial:           [4] *Span;                    // Partially initialized spans per type.
    span_used:              [5] *Span;                    // Spans in full use per type.
    next:                   *Heap;
    prev:                   *Heap;
    id:                     u32;
    finalize:               u32;
    release_trim:           u32;   // #jai develop #419: released untrimmed; the adopting thread trims it first.
    released:               bool;  // #jai fix U14: queued or back in the pristine list; set under the heap lock.
    // #jai fix E7: the id of the thread that parked the heap with thread_idle, 0 when not parked. Set
    // and cleared under the global heap lock and the heap's free_lock.
    parked_thread:          u64;
    offset:                 u32;                          // Memory map region offset.
    mapped_size:            u64;                          // Memory map size.
    #if HEAP_STATISTICS  stats: Heap_Statistics;              // 2.0.1: rpmalloc.c:624
    #if ENABLE_STATISTICS {
        size_use:       [SIZE_CLASS_COUNT] Heap_Size_Use;          // 2.0.1: rpmalloc.c:628
        span_map_calls: [5] u32;                                   // 2.0.1: rpmalloc.c:630
    }
    // #jai handed out by heap_acquire. IS_THIS_YOURS on the default allocator must not claim the
    // blocks of first-class heaps; owner_thread cannot tell them apart from released thread heaps.
    is_first_class:         bool;
    // #jai fix E10: for the Context cache of the thread heap. The stack range of the thread whose heap
    // this is, and whether it is that thread's heap now (cleared when the thread parks the heap, sets
    // another current, or releases it). Written by the owning thread.
    stack_low:              u64;
    stack_high:             u64;
    thread_current:         bool;
}

#assert size_of(Page) <= PAGE_HEADER_SIZE  "Invalid page header size";   // 2.0.1: rpmalloc.c:634
#assert #run every_class_fits_after_its_start_offset()  "PR #424: a size class no longer fits its page after its block start offset";

every_class_fits_after_its_start_offset :: () -> bool {
    page_size_of :: (index: s64) -> u64 {
        if index < SMALL_SIZE_CLASS_COUNT  return SMALL_PAGE_SIZE;
        if index < SMALL_SIZE_CLASS_COUNT + MEDIUM_SMALL_SIZE_CLASS_COUNT  return MEDIUM_SMALL_PAGE_SIZE;
        if index < SMALL_SIZE_CLASS_COUNT + MEDIUM_SMALL_SIZE_CLASS_COUNT + MEDIUM_LARGE_SIZE_CLASS_COUNT  return MEDIUM_LARGE_PAGE_SIZE;
        return LARGE_PAGE_SIZE;
    }
    for global_size_class {
        if it.block_size && cast(u64) it.block_count * it.block_size + block_start_offset(it.block_size) > page_size_of(it_index)  return false;
    }
    return true;
}
#assert size_of(Span) <= SPAN_HEADER_SIZE  "Invalid span header size";   // 2.0.1: rpmalloc.c:635
// 2.0.1: rpmalloc.c:636. #jai with ENABLE_STATISTICS the 64-bit size class counters (fix U13) make the
// heap larger than 4 KiB; heap_allocate_new rounds each heap up to whole pages, so a statistics build
// simply uses more of the mapping per heap.
#assert ENABLE_STATISTICS || size_of(Heap) <= 4096  "Invalid heap size";
#assert #run heap_member_offset("local_free") == HEAP_ALIGNMENT  "Remotely accessed heap fields must fill one line";   // develop #409

heap_member_offset :: (name: string) -> s64 {
    for type_info(Heap).members  if it.name == name  return it.offset_in_bytes;
    return -1;
}

page_has :: inline (page: *Page, flag: Page_Flags) -> bool {
    return (page.flags & flag) != 0;
}

page_set :: inline (page: *Page, flag: Page_Flags, on: bool) {
    if on  page.flags |= flag;
    else   page.flags &= ~flag;
}

PAGE_TYPE_BIT  :: 16;
PAGE_TYPE_MASK :: 0x7;

page_get_type :: inline (page: *Page) -> Page_Type {
    return cast(Page_Type)(((cast(u32) page.flags) >> PAGE_TYPE_BIT) & PAGE_TYPE_MASK);
}

page_set_type :: inline (page: *Page, page_type: Page_Type) {
    bits := (cast(u32) page.flags) & ~((cast(u32) PAGE_TYPE_MASK) << PAGE_TYPE_BIT);
    page.flags = cast(Page_Flags)(bits | ((cast(u32) page_type) << PAGE_TYPE_BIT));
}

page_commit_prefix_shift :: inline (page: *Page) -> u32 {
    return ((cast(u32) page.flags) >> COMMIT_PREFIX_SHIFT_BIT) & COMMIT_PREFIX_SHIFT_MASK;
}

page_set_commit_prefix_shift :: inline (page: *Page, shift: u32) {
    bits := (cast(u32) page.flags) & ~((cast(u32) COMMIT_PREFIX_SHIFT_MASK) << COMMIT_PREFIX_SHIFT_BIT);
    page.flags = cast(Page_Flags)(bits | ((shift & COMMIT_PREFIX_SHIFT_MASK) << COMMIT_PREFIX_SHIFT_BIT));
}

// #jai fix E3: commit a page as it fills. Upstream commits a whole page (64 KiB to 16 MiB) when it is
// first used, so a page holding a few long-lived blocks at its start kept all of it committed: in Ex4
// 1.45.0, 84 MB of pages for 8 MB of live blocks, left by 24 short-lived walk threads, whose pages had
// been carved only to 11 MB (RET-17). A page of a reserve-only span is committed in chunks of 1/16 of
// the page, at least 64 KiB (so a small page is one chunk, as before), as page_initialize_blocks carves
// blocks into it. The number of committed chunks is kept in the flags word (Span has no room for a
// field): 0 means only the commit prefix, the first OS page holding the header, is committed (a
// decommitted page). It survives reuse of the page until the page is decommitted.
COMMITTED_CHUNKS_BIT  :: 20;
COMMITTED_CHUNKS_MASK :: 0x1F;
COMMIT_CHUNKS_PER_PAGE :: 16;
COMMIT_CHUNK_MIN      :: 64 * 1024;

page_committed_chunks :: inline (page: *Page) -> u32 {
    return ((cast(u32) page.flags) >> COMMITTED_CHUNKS_BIT) & COMMITTED_CHUNKS_MASK;
}

page_set_committed_chunks :: inline (page: *Page, chunks: u32) {
    bits := (cast(u32) page.flags) & ~((cast(u32) COMMITTED_CHUNKS_MASK) << COMMITTED_CHUNKS_BIT);
    page.flags = cast(Page_Flags)(bits | ((chunks & COMMITTED_CHUNKS_MASK) << COMMITTED_CHUNKS_BIT));
}

// The commit chunk of a page of this size: 1/16 of it, at least COMMIT_CHUNK_MIN and the commit prefix,
// at most the page.
page_commit_chunk_size :: inline (page_size: u64, commit_prefix: u64) -> u64 {
    chunk := page_size / COMMIT_CHUNKS_PER_PAGE;
    if chunk < COMMIT_CHUNK_MIN  chunk = COMMIT_CHUNK_MIN;
    if chunk < commit_prefix     chunk = commit_prefix;
    if chunk > page_size         chunk = page_size;
    return chunk;
}

// Commits the page up to `extent` bytes from its start (rounded up to a chunk), if it is not yet.
// Non-zero on failure, with nothing changed. Only for pages of reserve-only spans.
page_commit_to :: (page: *Page, extent: u64) -> s32 {
    page_size := page_get_size(page);
    commit_prefix := cast(u64) 1 << page_commit_prefix_shift(page);
    chunk := page_commit_chunk_size(page_size, commit_prefix);
    needed := cast(u32)((extent + chunk - 1) / chunk);
    have := page_committed_chunks(page);
    if needed <= have  return 0;
    from := ifx have then cast(u64) have * chunk else commit_prefix;
    to := cast(u64) needed * chunk;
    if to > page_size  to = page_size;
    if to > from {
        if global_memory_interface.memory_commit(pointer_offset(page, from), to - from) != 0  return 1;
        #if HEAP_STATISTICS && ENABLE_DECOMMIT {
            if page.heap  heap_stat_add(*page.heap.stats.committed_size, to - from);
        }
    }
    page_set_committed_chunks(page, needed);
    return 0;
}

// Bytes of the page committed now: what decommit gives back beyond the prefix.
page_committed_size :: inline (page: *Page) -> u64 {
    page_size := page_get_size(page);
    commit_prefix := cast(u64) 1 << page_commit_prefix_shift(page);
    have := page_committed_chunks(page);
    if !have  return commit_prefix;
    size := cast(u64) have * page_commit_chunk_size(page_size, commit_prefix);
    return ifx size < page_size then size else page_size;
}

////////////
///
/// Global data (2.0.1: rpmalloc.c:638-728)
///
//////

global_heap_id: u32 = 1;   // 2.0.1: rpmalloc.c:664 (atomic) Heap id counter.

// Size classes (2.0.1: rpmalloc.c:685-709). The table is the same list of block sizes in units of
// SMALL_GRANULARITY as upstream's SCLASS/MSCLASS/MLCLASS/LCLASS entries; block counts are computed
// the same way, (page size of the class's page type - PAGE_HEADER_SIZE) / block size.
SIZE_CLASS_UNITS :: u32.[
    1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26,
    27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50,
    51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 80, 96, 112, 128, 160, 192, 224, 256,   // SCLASS
    320, 384, 448, 512, 640, 768, 896, 1024, 1280, 1536, 1792, 2048,                               // MSCLASS
    2560, 3072, 3584, 4096, 5120, 6144, 7168, 8192, 10240, 12288, 14336, 16384,                     // MLCLASS
    20480, 24576, 28672, 32768, 40960, 49152, 57344, 65536, 81920, 98304, 114688, 131072,          // LCLASS
];
#assert SIZE_CLASS_UNITS.count == SIZE_CLASS_COUNT  "size class table must have SIZE_CLASS_COUNT entries";

global_size_class :: #run make_size_class_table();   // 2.0.1: rpmalloc.c:693

make_size_class_table :: () -> [SIZE_CLASS_COUNT] Size_Class {
    table: [SIZE_CLASS_COUNT] Size_Class;
    for units, index: SIZE_CLASS_UNITS {
        block_size := units * SMALL_GRANULARITY;
        page_size: u32;
        if index < SMALL_SIZE_CLASS_COUNT                                                       page_size = SMALL_PAGE_SIZE;
        else if index < SMALL_SIZE_CLASS_COUNT + MEDIUM_SMALL_SIZE_CLASS_COUNT                  page_size = MEDIUM_SMALL_PAGE_SIZE;
        else if index < SMALL_SIZE_CLASS_COUNT + MEDIUM_SMALL_SIZE_CLASS_COUNT + MEDIUM_LARGE_SIZE_CLASS_COUNT   page_size = MEDIUM_LARGE_PAGE_SIZE;
        else                                                                                     page_size = LARGE_PAGE_SIZE;
        table[index] = .{block_size, (page_size - PAGE_HEADER_SIZE) / block_size};
    }
    return table;
}

// Threshold number of free pages per page type at which free pages are decommitted, and the
// number of pages to retain when that happens (2.0.1: rpmalloc.c:711-715). Not constant:
// initialisation adjusts the thresholds to the OS page size.
global_page_free_overflow: [5] u32 = .[16, 8, 4, 2, 0];
global_page_free_retain:   [5] u32 = .[4, 2, 1, 1, 0];

global_memory_interface:         *Memory_Interface;   // 2.0.1: rpmalloc.c:676
global_memory_interface_default: Memory_Interface;    // 2.0.1: rpmalloc.c:678
global_memory_interface_copy:    Memory_Interface;    // #jai fix U8: the caller's interface, with OS defaults filled in.
global_config:                   Config;              // 2.0.1: rpmalloc.c:680

os_huge_pages:      bool;   // 2.0.1: rpmalloc.c:718  Huge (large) pages are in use.
os_thp:             bool;   // 2.0.1: rpmalloc.c:721  Transparent huge pages are in use (Linux).
os_huge_page_shift: u64;    // 2.0.1: rpmalloc.c:723  log2 of the huge page size, for MAP_HUGE_SHIFT.
os_map_granularity: u64;    // 2.0.1: rpmalloc.c:725  Granularity of OS mappings.
os_page_size:       u64;    // 2.0.1: rpmalloc.c:727  OS page size (0 until os_initialize).

////////////
///
/// Thread local heap and ID (2.0.1: rpmalloc.c:729-877)
///
//////
// #jai the thread heap lives in OS thread-local storage (see os_tls_* in os_windows.jai /
// os_posix.jai): this Jai version has no #thread_local, and the Context can be copied between
// threads (review H3).
// The fallback heap: every thread starts on it. It is all zero (id 0), so the allocation fast path
// finds nothing and drops into the slow path, which initialises the allocator and the thread's real
// heap (heap_get_page_generic and heap_allocate_block_huge check heap.id == 0).
global_heap_fallback: Heap;                       // 2.0.1: rpmalloc.c:645
global_heap_default := *global_heap_fallback;     // 2.0.1: rpmalloc.c:647
global_heap_queue:    *Heap;   // 2.0.1: rpmalloc.c:650  Released heaps; only handed to thread heaps.
global_heap_pristine: *Heap;   // 2.0.1: rpmalloc.c:653  Never-used heaps carved from a shared block.
global_heap_used:     *Heap;   // 2.0.1: rpmalloc.c:655  Heaps in use.
// Set while a thread maps and carves a new shared heap block, so concurrent thread starts wait for
// the carved heaps instead of each mapping a block. A user memory interface callback must not
// allocate through rpmalloc on the creating thread: that would wait on itself.
global_heap_creating: s32;     // 2.0.1: rpmalloc.c:660
global_heap_lock:     u64;     // 2.0.1: rpmalloc.c:662 (atomic)

// Global initialisation state machine (2.0.1: rpmalloc.c:665-674). The first allocation on any
// thread may initialise, so exactly one thread does the global setup while the others wait; none
// may see a half-written global_config.
RPMALLOC_INIT_UNINIT  :: 0;
RPMALLOC_INIT_RUNNING :: 1;
RPMALLOC_INIT_DONE    :: 2;
global_rpmalloc_init_state: s32;   // atomic
global_main_thread_id: u64;        // 2.0.1: rpmalloc.c:682

// Fast thread id (2.0.1: rpmalloc.c:760). #jai see os_thread_id: GetCurrentThreadId / pthread_self
// instead of the TEB / thread pointer.
get_thread_id :: inline () -> u64 {
    return os_thread_id();
}

// Set the current thread heap (2.0.1: rpmalloc.c:796).
set_thread_heap :: (heap: *Heap) {
    // #jai fix E10: the heap this thread had is no longer its heap, whatever Context caches it.
    if atomic_load(*global_rpmalloc_init_state) == RPMALLOC_INIT_DONE {
        previous := cast(*Heap) os_tls_get();
        if previous && previous != heap  previous.thread_current = false;
    }
    if heap && heap.id != 0 {
        rpmalloc_assert(heap.id != 0, "Default heap being used");
        heap.owner_thread = get_thread_id();
        heap.stack_low, heap.stack_high = os_thread_stack_range();   // #jai fix E10
        heap.thread_current = true;
    }
    // #jai one OS TLS store carries both the heap pointer (upstream's _Thread_local variable) and
    // the FLS / pthread key value that drives the thread exit destructor.
    os_tls_set(heap);
}

get_thread_heap_allocate :: () -> *Heap {   // 2.0.1: rpmalloc.c:810
    heap := heap_take_parked();   // #jai fix E7: a thread waking from thread_idle gets its own heap back.
    if !heap  heap = heap_allocate(false);
    set_thread_heap(heap);
    return heap;
}

// #jai fix E7: parked heaps (thread_idle), so thread starts skip the search when there are none.
global_parked_count: s64;   // atomic

// The heap the calling thread parked, unparked and its own again (its owner id set), or null. The
// owner's thread-local slot holds the fallback heap while it is parked, so its first allocator call
// after thread_idle comes here through the uninitialised-thread path.
heap_take_parked :: () -> *Heap {
    if !atomic_load(*global_parked_count)  return null;
    me := get_thread_id();
    heap_lock_acquire();
    heap := global_heap_used;
    while heap && heap.parked_thread != me  heap = heap.next;
    if heap {
        heap_free_lock_acquire(heap);   // Frees into it (as a released heap) take this lock (E6).
        heap.parked_thread = 0;
        heap.owner_thread = me;
        heap_free_lock_release(heap);
        atomic_add(*global_parked_count, -1);
    }
    heap_lock_release();
    return heap;
}

// Get the current thread heap (2.0.1: rpmalloc.c:818).
// #jai until the first initialisation creates the TLS slot, and on any thread that has not stored
// a heap yet, this returns the fallback heap: upstream's _Thread_local default value.
// #jai fix E10: every allocation and free looks up the thread heap, and the OS thread-local storage
// lookup is a call (TlsGetValue, pthread_getspecific): 1.3 ns of the 2.3 ns per allocate+free pair the
// port was behind C 2.0.1 (which reads a _Thread_local variable inline). Jai has no thread-local
// variables, but each thread runs with its own Context, so the heap is cached there too. The Context
// can be copied to another thread, or the heap stop being this thread's, so the cache is used only if
// the stack in use is the stack of the heap's thread and the heap is still that thread's heap; otherwise
// the OS lookup runs and refreshes it. A fiber, whose stack is not its thread's, always takes the OS
// lookup. Benchmarks: pair_*, churn_small and mt_churn_* 4-9% faster.
get_thread_heap :: inline () -> *Heap {
    #if CONTEXT_HEAP_CACHE {
        cached := cast(*Heap) context_thread_heap().*;
        if cached {
            here := cast(u64) *cached;   // An address on the stack in use.
            if here >= cached.stack_low && here < cached.stack_high && cached.thread_current  return cached;
        }
    }
    return get_thread_heap_from_os();
}

get_thread_heap_from_os :: no_inline () -> *Heap {
    if atomic_load(*global_rpmalloc_init_state) == RPMALLOC_INIT_DONE {
        heap := cast(*Heap) os_tls_get();
        if heap {
            #if CONTEXT_HEAP_CACHE  if heap.thread_current  context_thread_heap().* = heap;
            return heap;
        }
    }
    return global_heap_default;
}

// Size class for tiny blocks: sizes up to 16 times the minimum granularity (1024 bytes).
get_size_class_tiny :: inline (size: u64) -> u32 {   // 2.0.1: rpmalloc.c:824
    return (cast,trunc(u32) size + (SMALL_GRANULARITY - 1)) / SMALL_GRANULARITY;
}

get_size_class :: inline (size: u64) -> u32 {   // 2.0.1: rpmalloc.c:830
    // #jai The block count rounded up without 2.0.1's size + 15, which wraps for sizes within 15 of
    // U64_MAX (and traps with arithmetic_overflow_check). Such sizes get a class >= SIZE_CLASS_COUNT,
    // and the huge path refuses them (#jai fix U1, heap_allocate_block_huge).
    minblock_count := size / SMALL_GRANULARITY + cast(u64) ((size & (SMALL_GRANULARITY - 1)) != 0);
    // Up to 64 times the minimum granularity (1024 bytes) the size class is the block count.
    if size <= SMALL_GRANULARITY * 64 {
        rpmalloc_assert(global_size_class[minblock_count].block_size >= size, "Size class misconfiguration");
        return cast(u32) ifx minblock_count then minblock_count else 1;
    }
    minblock_count -= 1;
    // minblock_count is now > 64, so its most significant bit is at position 6 or above.
    most_significant_bit := msb_index(minblock_count);
    // Class sizes have the bit format [..]000xxx000[..]: the most significant bit, then the two
    // bits after it select the subclass.
    subclass_bits := cast(u32)(minblock_count >> (most_significant_bit - 2)) & 0x03;
    class_idx := (most_significant_bit << 2) + subclass_bits + 41;
    rpmalloc_assert(class_idx >= SIZE_CLASS_COUNT || global_size_class[class_idx].block_size >= size, "Size class misconfiguration");
    rpmalloc_assert(class_idx >= SIZE_CLASS_COUNT || global_size_class[class_idx - 1].block_size < size, "Size class misconfiguration");
    return class_idx;
}

get_page_type :: inline (size_class: u32) -> Page_Type {   // 2.0.1: rpmalloc.c:858
    if size_class < SMALL_SIZE_CLASS_COUNT  return .SMALL;
    else if size_class < SMALL_SIZE_CLASS_COUNT + MEDIUM_SMALL_SIZE_CLASS_COUNT  return .MEDIUM_SMALL;
    else if size_class < SMALL_SIZE_CLASS_COUNT + MEDIUM_SMALL_SIZE_CLASS_COUNT + MEDIUM_LARGE_SIZE_CLASS_COUNT  return .MEDIUM_LARGE;
    else if size_class < SIZE_CLASS_COUNT  return .LARGE;
    return .HUGE;
}

get_page_aligned_size :: inline (size: u64) -> u64 {   // 2.0.1: rpmalloc.c:871
    unalign := size % global_config.page_size;
    if unalign  size += global_config.page_size - unalign;   // Callers keep size below MAX_ALLOC_SIZE (#jai fix U1).
    return size;
}

////////////
///
/// OS entry points (2.0.1: rpmalloc.c:878-1161)
///
//////
// os_set_page_name:885, os_mmap:900, os_commit_on_demand:1040, os_mcommit:1049,
// os_mdecommit:1086, os_munmap:1142. These live in os_windows.jai / os_posix.jai.

////////////
///
/// Page interface (2.0.1: rpmalloc.c:1162-1500)
///
//////

page_get_span :: inline (page: *Page) -> *Span {   // 2.0.1: rpmalloc.c:1169
    return cast(*Span)(cast(u64) page & SPAN_MASK);
}

page_get_size :: inline (page: *Page) -> u64 {   // 2.0.1: rpmalloc.c:1174
    if page_get_type(page) == {
      case .SMALL;         return SMALL_PAGE_SIZE;
      case .MEDIUM_SMALL;  return MEDIUM_SMALL_PAGE_SIZE;
      case .MEDIUM_LARGE;  return MEDIUM_LARGE_PAGE_SIZE;
      case .LARGE;         return LARGE_PAGE_SIZE;
    }
    return page_get_span(page).page_size;
}

// #jai fix #407 (as upstream develop 30bc8859): ownership by heap, not by thread id. 2.0.1 compared
// owner_thread with the caller's thread id, and a released heap keeps its owner_thread, so a thread
// freeing after its THREAD_STOP (or a new thread reusing its id) took the unsynchronised local path
// on a heap queued for or adopted by another thread.
page_is_thread_heap :: inline (page: *Page) -> bool {   // 2.0.1: rpmalloc.c:1188
    #if FIRST_CLASS_HEAPS  return page.heap == get_thread_heap() || !page.heap.owner_thread;
    else                   return page.heap == get_thread_heap();
}

// #jai upstream PR #424 (2759a3e): the first block of a page starts at the largest power of two
// dividing the block size, at least the page header size and at most NATURAL_ALIGNMENT_LIMIT, so every
// block of a class is aligned to that power of two. The tail slack of every page is larger than the
// extra offset, so no class loses a block (checked at compile time below). The offset is derived from
// the block size, not stored in the page.
block_start_offset :: inline (block_size: u32) -> u32 {
    offset := lowest_set_bit(block_size);
    if offset < PAGE_HEADER_SIZE  return PAGE_HEADER_SIZE;
    if offset > NATURAL_ALIGNMENT_LIMIT  return NATURAL_ALIGNMENT_LIMIT;
    return offset;
}

// #jai The largest power of two dividing x; 0 for 0. C's x & -x relies on unsigned wraparound, which a
// Jai program built with arithmetic_overflow_check = .FATAL aborts on: x - 1 cannot wrap once x is
// non-zero.
lowest_set_bit :: inline (x: u32) -> u32 {
    if !x  return 0;
    return x & ~(x - 1);
}

// Sanity check of the huge page size the OS reports (Linux Hugepagesize; 2.0.1: rpmalloc.c:2850-2856).
// The builtin page geometry cannot use huge pages larger than the largest builtin page size (for
// example 1 GiB defaults). The huge page size is passed to mmap explicitly, so a non-default 2 MiB pool
// can still be used. 0 means no usable huge pages. #jai here rather than in os_posix.jai, so the tests
// reach it on every platform.
usable_huge_page_size :: (reported: u64) -> u64 {
    size := reported;
    if size && (size & (size - 1))  size = 0;   // #jai 0 first: 0 - 1 would wrap (arithmetic_overflow_check)
    if size > LARGE_PAGE_SIZE  size = 2 * 1024 * 1024;
    return size;
}

page_block_start_offset :: inline (page: *Page) -> u32 {
    return block_start_offset(page.block_size);
}

page_block_start :: inline (page: *Page) -> *Block {   // 2.0.1: rpmalloc.c:1197
    return pointer_offset(page, page_block_start_offset(page));
}

page_block :: inline (page: *Page, block_index: u32) -> *Block {   // 2.0.1: rpmalloc.c:1202
    return pointer_offset(page, page_block_start_offset(page) + cast(u64) page.block_size * block_index);
}

// #jai develop #411 (03537c6): the thread-free token holds the head block's byte offset from the
// page start instead of its index, so a cross-thread free needs no division and a decode no
// multiplication. Pages are at most LARGE_PAGE_SIZE (16 MiB), so the offset fits in 32 bits.
page_block_offset :: inline (page: *Page, block: *Block) -> u32 {   // 2.0.1: rpmalloc.c:1207 page_block_index
    return cast(u32) pointer_diff(block, page);
}

// #jai fix PR #426 (upstream 0aaf830, adapted to the 2.0.1 token): set in the thread-free token of a
// full page, whose list is then empty. The list count takes the 31 bits below it.
THREAD_FREE_FULL :: cast(u64) 1 << 63;

page_block_from_thread_free_list :: inline (page: *Page, token: u64, block: **Block) -> u32 {   // 2.0.1: rpmalloc.c:1213
    block_offset := cast(u32)(token & 0xFFFF_FFFF);
    list_count   := cast(u32)((token >> 32) & 0x7FFF_FFFF);
    block.* = ifx list_count then cast(*Block) pointer_offset(page, block_offset) else null;
    return list_count;
}

page_block_to_thread_free_list :: inline (page: *Page, block_offset: u32, list_count: u32) -> u64 {   // 2.0.1: rpmalloc.c:1221
    // #jai fix P-2 (port bug): `cast(u64) list_count << 32` casts the shifted u32, so the count was
    // lost and every cross-thread free into a page that was not full went missing.
    return ((cast(u64) list_count) << 32) | cast(u64) block_offset;
}

// #jai PR #424: the modulo is taken on the offset from the page start, then adjusted by the block start
// offset. Aligned blocks come from classes at least the alignment larger than the request, whose block
// start offset is at most the block size.
page_block_realign :: inline (page: *Page, block: *Block) -> *Block {   // 2.0.1: rpmalloc.c:1227
    block_size := page.block_size;
    start_offset := page_block_start_offset(page);
    rpmalloc_assert(start_offset <= block_size, "Block start offset exceeds aligned block size");
    page_offset := (cast(u32) pointer_diff(block, page)) % block_size;
    block_offset := ifx page_offset >= start_offset then page_offset - start_offset else page_offset + block_size - start_offset;
    return pointer_offset(block, -cast(s64) block_offset);
}

page_get_local_free_block :: (page: *Page) -> *Block {   // 2.0.1: rpmalloc.c:1234
    block := page.local_free;
    page.local_free = block.next;
    page.block_used += 1;
    return block;
}

page_decommit_memory_pages :: inline (page: *Page) {   // 2.0.1: rpmalloc.c:1243
    if page_has(page, .IS_DECOMMITTED)  return;
    // Only reserve-only spans support decommit.
    if !page_has(page, .COMMIT_ON_DEMAND)  return;
    // Keep the prefix (holding the page header) committed and decommit the rest. The prefix is
    // the span's fixed map-time page size, so commit and decommit always use the same one.
    commit_prefix := cast(u64) 1 << page_commit_prefix_shift(page);
    page_size := page_get_size(page);
    if page_size <= commit_prefix  return;
    // #jai fix E3: only the chunks committed so far.
    extra_page := pointer_offset(page, commit_prefix);
    extra_page_size := page_committed_size(page) - commit_prefix;
    if extra_page_size && global_memory_interface.memory_decommit(extra_page, extra_page_size) != 0  return;
    #if HEAP_STATISTICS && ENABLE_DECOMMIT {
        if page.heap  heap_stat_sub(*page.heap.stats.committed_size, extra_page_size);
    }
    page_set(page, .IS_DECOMMITTED, true);
    page_set_committed_chunks(page, 0);
    #if TEST_HOOKS  atomic_add(*test_pages_decommitted[cast(u32) page_get_type(page)], 1);
}

page_commit_memory_pages :: inline (page: *Page) -> s32 {   // 2.0.1: rpmalloc.c:1267
    if !page_has(page, .IS_DECOMMITTED)  return 0;
    commit_prefix := cast(u64) 1 << page_commit_prefix_shift(page);
    // #jai fix E3: only the first chunk; page_initialize_blocks commits the rest as it carves.
    if page_commit_to(page, commit_prefix + 1) != 0  return 1;
    page_set(page, .IS_DECOMMITTED, false);
    #if TEST_HOOKS  atomic_add(*test_pages_recommitted[cast(u32) page_get_type(page)], 1);
    #if ENABLE_DECOMMIT {
        #if OS != .MACOS {
            // When a page is recommitted, the blocks from the second memory page onwards are
            // zeroed by the OS: take advantage in zalloc/calloc, and zero the first page here.
            first_page := pointer_offset(page, PAGE_HEADER_SIZE);
            memset(first_page, 0, cast(s64)(commit_prefix - PAGE_HEADER_SIZE));
            page_set(page, .IS_ZERO, true);
        }
    }
    return 0;
}

page_available_to_free :: (page: *Page, keep_last := true) {   // 2.0.1: rpmalloc.c:1294
    rpmalloc_assert(!page_has(page, .IS_FULL), "Page full flag internal failure");
    rpmalloc_assert(!page_has(page, .IS_DECOMMITTED), "Page decommitted flag internal failure");
    heap := page.heap;
    if heap.page_available[page.size_class] == page {
        // #jai develop #413 (857888a): keep the last available page of a size class in place.
        // Retiring it made the next allocation of the class take the generic path and reinitialise
        // the page from scratch, which alternating alloc/free then repeated on every pair (W-11).
        // At most one empty page per size class stays committed this way.
        // keep_last is false only from heap_retire_kept_pages (#jai fix, see there).
        if keep_last && !page.next  return;
        heap.page_available[page.size_class] = page.next;
    } else {
        page.prev.next = page.next;
        if page.next  page.next.prev = page.prev;
    }
    page_set(page, .IS_FREE, true);
    page_set(page, .IS_ZERO, false);
    page_type := cast(u32) page_get_type(page);
    page.next = heap.page_free[page_type];
    heap.page_free[page_type] = page;
    if !page.next  heap.page_free_tail[page_type] = page;   // #jai PR #420
    heap.page_free_commit_count[page_type] += 1;
    if heap.page_free_commit_count[page_type] >= global_page_free_overflow[page_type]
        heap_page_free_overflow(heap, page_type);   // #jai develop #416
}

page_full_to_available :: (page: *Page) {   // 2.0.1: rpmalloc.c:1314
    rpmalloc_assert(page_has(page, .IS_FULL), "Page full flag internal failure");
    rpmalloc_assert(!page_has(page, .IS_DECOMMITTED), "Page decommitted flag internal failure");
    heap := page.heap;
    page.next = heap.page_available[page.size_class];
    if page.next  page.next.prev = page;
    heap.page_available[page.size_class] = page;
    page_set(page, .IS_FULL, false);
    if !page_has(page, .HAS_ALIGNED_BLOCK)  page.heap_fast = heap;   // #jai PR #422
    // #jai fix PR #426: other threads do not change the token of a page flagged full; it holds only
    // the flag.
    atomic_store(*page.thread_free, 0);
}

// page_full_to_free_on_new_heap (2.0.1: rpmalloc.c:1328) is dead code upstream; not ported.

page_available_to_full :: (page: *Page) {   // 2.0.1: rpmalloc.c:1343
    heap := page.heap;
    if heap.page_available[page.size_class] == page {
        heap.page_available[page.size_class] = page.next;
    } else {
        page.prev.next = page.next;
        if page.next  page.next.prev = page.prev;
    }
    page_set(page, .IS_FULL, true);
    page_set(page, .IS_ZERO, false);
    page.heap_fast = null;   // #jai PR #422
}

page_put_local_free_block :: inline (page: *Page, block: *Block) {   // 2.0.1: rpmalloc.c:1358
    block.next = page.local_free;
    page.local_free = block;
    stat_add_free(page.heap, page.size_class, 1);
    page.block_used -= 1;
    if page.block_used == 0 {
        page_available_to_free(page);
    } else if page_has(page, .IS_FULL) {
        page_full_to_available(page);
    }
}

page_adopt_thread_free_block_list :: no_inline (page: *Page) {   // 2.0.1: rpmalloc.c:1371
    if page.local_free  return;
    thread_free := atomic_load(*page.thread_free);
    if thread_free != 0 {
        // Other threads can only replace it with another valid list head; it never changes to 0
        // in other threads. #jai develop #415 (686917f): so one exchange claims the whole list, with
        // no retry loop or backoff on the owner's path.
        thread_free = atomic_exchange(*page.thread_free, 0);
        thread_free_count := page_block_from_thread_free_list(page, thread_free, *page.local_free);
        rpmalloc_assert(thread_free_count <= page.block_used, "Page thread free list count internal failure");
        page.block_used -= thread_free_count;
        stat_add_free(page.heap, page.size_class, thread_free_count);
    }
}

// Pushes a block freed into a full page to the heap-level thread-free list; otherwise the owner would
// not pick it up until a thread-local free happens on that page.
// #jai develop #412 (50fd91a): the uncontended case is a single CAS; the spin and yield retry loops
// live in separate cold procedures, so the common path stays small.
heap_put_thread_free_block :: (page: *Page, block: *Block) {
    list := *page.heap.thread_free[cast(u32) page_get_type(page)];
    prev_head := atomic_load(list);
    block.next = cast(*Block) prev_head;
    swapped, actual := compare_and_swap(list, prev_head, cast(u64) block);
    if !swapped  heap_put_thread_free_block_retry(list, block, actual);
}

heap_put_thread_free_block_retry :: no_inline (list: *u64, block: *Block, prev_head: u64) {
    spin: u32;
    while true {
        block.next = cast(*Block) prev_head;
        swapped, actual := compare_and_swap(list, prev_head, cast(u64) block);
        if swapped  return;
        prev_head = actual;
        wait_spin(*spin);
    }
}

// Pushes a block to another thread's deferred free list.
// #jai fix PR #426: 2.0.1 chose between the page's list and the heap's by reading the owner's
// is_full flag, while the owner marked the page full with a plain store after finding the page's
// list empty. A free in between landed in the page's list of a page then marked full, and if it was
// among the page's last frees the page never became available or free again (W-09). The owner now
// marks a page full by setting THREAD_FREE_FULL in the token with a compare and swap, which fails
// if a free got in first, and frees route by that flag.
page_put_thread_free_block :: no_inline (page: *Page, block: *Block) {   // 2.0.1: rpmalloc.c:1389
    prev_thread_free := atomic_load(*page.thread_free);
    if prev_thread_free & THREAD_FREE_FULL {
        heap_put_thread_free_block(page, block);
        return;
    }
    block_offset := page_block_offset(page, block);
    rpmalloc_assert(!((block_offset - page_block_start_offset(page)) % page.block_size), "Block pointer is not aligned to start of block");
    list_size := page_block_from_thread_free_list(page, prev_thread_free, *block.next) + 1;
    thread_free := page_block_to_thread_free_list(page, block_offset, list_size);
    swapped, actual := compare_and_swap(*page.thread_free, prev_thread_free, thread_free);
    if !swapped  page_put_thread_free_block_retry(page, block, block_offset, actual);   // #jai develop #412
}

// The contended retry path for pushing to a page's thread-free list, or to the heap-level list once the
// owner has flagged the page full (#jai develop #412, with PR #426).
page_put_thread_free_block_retry :: no_inline (page: *Page, block: *Block, block_offset: u32, prev_thread_free: u64) {
    spin: u32;
    while true {
        if prev_thread_free & THREAD_FREE_FULL {
            heap_put_thread_free_block(page, block);
            return;
        }
        list_size := page_block_from_thread_free_list(page, prev_thread_free, *block.next) + 1;
        thread_free := page_block_to_thread_free_list(page, block_offset, list_size);
        wait_spin(*spin);
        swapped, actual := compare_and_swap(*page.thread_free, prev_thread_free, thread_free);
        if swapped  return;
        prev_thread_free = actual;
    }
}

page_push_local_free_to_heap :: (page: *Page) {   // 2.0.1: rpmalloc.c:1420
    // Push the page free list as the fast-track list of free blocks for the heap.
    page.heap.local_free[page.size_class] = page.local_free;
    // Every initialised block not counted as used is on the local free list, so after handing the
    // whole list to the heap every initialised block is in use (#jai PR #422).
    page.block_used = page.block_initialized;
    page.local_free = null;
}

page_initialize_blocks :: no_inline (page: *Page) -> *Block {   // 2.0.1: rpmalloc.c:1429
    rpmalloc_assert(page.block_initialized < page.block_count, "Block initialization internal failure");
    // #jai fix E3: commit the page up to the end of the new block first (a small page is one chunk,
    // so the free list linked below stays inside it). Null when the OS refuses; nothing has changed.
    if page_has(page, .COMMIT_ON_DEMAND) {
        extent := cast(u64) page_block_start_offset(page) + (cast(u64) page.block_initialized + 1) * page.block_size;
        if page_commit_to(page, extent) != 0  return null;
    }
    block := page_block(page, page.block_initialized);
    page.block_initialized += 1;
    page.block_used += 1;

    if page_get_type(page) == .SMALL && cast(u64) page.block_size < (global_config.page_size >> 1) {
        // Link up blocks until the next memory page into the free list.
        memory_page_start := cast(*void)(cast(u64) block & ~(global_config.page_size - 1));
        memory_page_next := pointer_offset(memory_page_start, global_config.page_size);
        free_block: *Block = pointer_offset(block, page.block_size);
        first_block := free_block;
        last_block := free_block;
        list_count: u32 = 0;
        max_list_count := page.block_count - page.block_initialized;
        while cast(*void) free_block < memory_page_next && list_count < max_list_count {
            last_block = free_block;
            free_block.next = pointer_offset(free_block, page.block_size);
            free_block = free_block.next;
            list_count += 1;
        }
        if list_count {
            last_block.next = null;
            page.local_free = first_block;
            page.block_initialized += list_count;
        }
    }
    return block;
}

page_allocate_block :: inline (page: *Page, zero: bool) -> *void {   // 2.0.1: rpmalloc.c:1462
    is_zero := false;
    block := ifx page.local_free then page_get_local_free_block(page) else null;
    if block == null {
        if atomic_load(*page.thread_free) != 0 {
            page_adopt_thread_free_block_list(page);
            block = ifx page.local_free then page_get_local_free_block(page) else null;
        }
        if block == null {
            block = page_initialize_blocks(page);
            if !block  return null;   // #jai fix E3: the commit for the new block failed.
            is_zero = page_has(page, .IS_ZERO);
        }
    }

    rpmalloc_assert(page.block_used <= page.block_count, "Page block use counter out of sync");
    if page.local_free && !page.heap.local_free[page.size_class]
        page_push_local_free_to_heap(page);

    // The page might be full once its free list has been pushed to the heap local free list:
    // check whether there is a thread free list to adopt.
    if page.block_used == page.block_count {
        page_adopt_thread_free_block_list(page);
        if page.block_used == page.block_count {
            // The page is now fully used.
            rpmalloc_assert(!page_has(page, .IS_FULL), "Page block use counter out of sync with full flag");
            #if TEST_HOOKS  if test_before_mark_full  test_before_mark_full(page);
            // #jai fix PR #426: flag the page full in the token, unless a cross-thread free just
            // added a block; then the page stays available and the block is adopted on the next
            // refill.
            swapped := compare_and_swap(*page.thread_free, 0, THREAD_FREE_FULL);
            if swapped  page_available_to_full(page);
        }
    }

    if zero {
        if !is_zero  memset(block, 0, page.block_size);
        else         (cast(*u64) block).* = 0;
    }
    return block;
}

////////////
///
/// Span interface (2.0.1: rpmalloc.c:1501-1771)
///
//////

// #jai fix E8: milliseconds from `since` to `now`, 0 when `since` is later. A thread reads the clock and
// then compares under a lock, so another thread may have stored a later reading in between: the huge
// cache (an entry cached after this thread read the clock) and, since E4 and E7 let other threads free
// into a heap, a heap's decommit delay. Upstream's now - since wraps there, and the entry merely counts
// as aged early; in a program built with arithmetic_overflow_check it trapped (THR-08 on macOS, W-17).
elapsed_ms :: inline (now: u64, since: u64) -> u64 {
    return ifx now > since then now - since else 0;
}

// The huge block cache (2.0.1: rpmalloc.c:1561-1734). Recently freed huge mappings are kept for reuse
// instead of paying munmap + mmap + page faults per huge allocation cycle. Entries are bounded by a
// committed byte budget and unmapped after an idle epoch. A cached mapping is reused for requests down
// to 1 / HUGE_CACHE_REUSE_FACTOR of its size (#jai PR #427; 25% in 2.0.1).
HUGE_CACHE_COMMITTED_LIMIT :: cast(u64) HUGE_CACHE_LIMIT_MB * 1024 * 1024;   // 2.0.1: rpmalloc.c:1570 (128 MiB; #jai fix E11: a module parameter)
#assert HUGE_CACHE_LIMIT_MB > 0  "HUGE_CACHE_LIMIT_MB must be positive; HUGE_CACHE_SLOT_COUNT = 0 disables the cache";
// The largest single mapping the cache holds, so one rare giant cannot take the whole budget from
// the smaller, more often cycled sizes.
HUGE_CACHE_ENTRY_LIMIT     :: HUGE_CACHE_COMMITTED_LIMIT / 4;  // 2.0.1: rpmalloc.c:1576
HUGE_CACHE_EPOCH_MS        :: 1000;                            // 2.0.1: rpmalloc.c:1579
// #jai upstream PR #427 (eb6d48b), as a constant: a cached mapping serves requests down to 1 / this
// of its size. 2.0.1 reused a mapping only within 25% of the request (1.25), so buffers of varying
// sizes mostly mapped fresh memory and faulted every page in (RET-14). A larger factor reuses more
// and wastes up to that much of a mapping; upstream measured malloc-large 10.0 -> 6.8 s, peak RSS
// 527 -> 557 MiB with 2.
HUGE_CACHE_REUSE_FACTOR    :: 2;

#if HUGE_CACHE_SLOT_COUNT {

Huge_Cache :: struct {   // 2.0.1: rpmalloc.c:1600
    lock:      u64;   // Cache lock.
    committed: u64;   // Committed bytes currently cached (decommitted entries do not count).
    // #jai fix E5: when the oldest committed entry was cached (0: none), so the allocation slow path
    // can tell without the lock whether an entry is due for decommit. Written under the lock.
    oldest_committed_ms: u64;
    slot: [HUGE_CACHE_SLOT_COUNT] struct {
        span:        *Span;
        free_ms:     u64;    // When the mapping was cached, or (decommitted) when it was decommitted.
        decommitted: bool;   // #jai fix E5: only the header page is committed; the address space is kept.
    };
}

global_huge_cache: Huge_Cache;   // 2.0.1: rpmalloc.c:1612

huge_cache_lock_acquire :: inline () {   // 2.0.1: rpmalloc.c:1615
    spin: u32;
    while !compare_and_swap(*global_huge_cache.lock, 0, 1)  wait_spin(*spin);
}

huge_cache_lock_release :: inline () {   // 2.0.1: rpmalloc.c:1626
    atomic_store(*global_huge_cache.lock, 0);
}

huge_cache_entry_size :: inline (span: *Span) -> u64 {
    return cast(u64) span.page_count * span.page_size;
}

// Under the lock.
huge_cache_update_oldest :: () {
    oldest: u64 = 0;
    for global_huge_cache.slot  if it.span && !it.decommitted && (!oldest || it.free_ms < oldest)  oldest = it.free_ms;
    atomic_store(*global_huge_cache.oldest_committed_ms, oldest);
}

// #jai fix E5: Ex4 1.45.0's cache filled to its 128 MiB limit and stayed committed, because idle
// entries were let go only at the next huge free (huge_cache_push) or thread_collect (U23), and a
// program that stops freeing huge blocks does neither (RET-19). An entry idle for an epoch now has its
// contents decommitted rather than its mapping unmapped: reusing it skips the mapping (most of the
// cache's value; the pages fault in again either way), and the reservation costs no memory. Without
// decommit (ENABLE_DECOMMIT off, or decommit disabled) it is unmapped, as before. The OS calls are made
// outside the lock. Runs from huge_cache_push, huge allocations, thread_collect and, through
// huge_cache_purge_if_due, the allocation slow path.
huge_cache_purge_aged :: (now: u64) {   // 2.0.1: rpmalloc.c:1633
    aged: [HUGE_CACHE_SLOT_COUNT] *Span = ---;
    aged_count := 0;
    huge_cache_lock_acquire();
    for * slot: global_huge_cache.slot {
        span := slot.span;
        if span && !slot.decommitted && elapsed_ms(now, slot.free_ms) >= HUGE_CACHE_EPOCH_MS {
            global_huge_cache.committed -= huge_cache_entry_size(span);
            slot.span = null;
            aged[aged_count] = span;
            aged_count += 1;
        }
    }
    huge_cache_update_oldest();
    huge_cache_lock_release();
    for ispan: 0..aged_count - 1 {
        span := aged[ispan];
        page_size := cast(u64) span.page_size;
        size := huge_cache_entry_size(span);
        kept := false;
        if os_commit_on_demand() && size > page_size &&
           global_memory_interface.memory_decommit(pointer_offset(span, page_size), size - page_size) == 0 {
            // Back in, decommitted, if a slot is still free; otherwise let it go.
            huge_cache_lock_acquire();
            for * slot: global_huge_cache.slot {
                if !slot.span {
                    slot.span = span;
                    slot.free_ms = now;
                    slot.decommitted = true;
                    kept = true;
                    break;
                }
            }
            huge_cache_lock_release();
        }
        if !kept  span_unmap(span);
    }
}

// The allocation slow path's check: purge only when a committed entry has been idle for an epoch. One
// load while nothing committed is cached.
huge_cache_purge_if_due :: inline () {
    oldest := atomic_load(*global_huge_cache.oldest_committed_ms);
    if !oldest  return;
    now := os_monotonic_time_ms();
    if elapsed_ms(now, oldest) >= HUGE_CACHE_EPOCH_MS  huge_cache_purge_aged(now);
}

// Unmaps every cached mapping. The cache must not outlive allocator finalization.
huge_cache_flush :: () {   // 2.0.1: rpmalloc.c:1652
    purge: [HUGE_CACHE_SLOT_COUNT] *Span = ---;
    purge_count := 0;
    huge_cache_lock_acquire();
    for * slot: global_huge_cache.slot {
        span := slot.span;
        if span {
            if !slot.decommitted  global_huge_cache.committed -= huge_cache_entry_size(span);
            slot.span = null;
            purge[purge_count] = span;
            purge_count += 1;
        }
    }
    huge_cache_update_oldest();
    huge_cache_lock_release();
    for ispan: 0..purge_count - 1
        span_unmap(purge[ispan]);
}

// Tries to cache a freed huge mapping; returns true if it was cached.
huge_cache_push :: (span: *Span) -> bool {   // 2.0.1: rpmalloc.c:1671
    committed_size := huge_cache_entry_size(span);
    if committed_size > HUGE_CACHE_ENTRY_LIMIT  return false;
    now := os_monotonic_time_ms();
    if !now  now = 1;   // 0 marks "no committed entry" in oldest_committed_ms.
    huge_cache_purge_aged(now);
    cached := false;
    evicted: *Span;
    huge_cache_lock_acquire();
    if global_huge_cache.committed + committed_size <= HUGE_CACHE_COMMITTED_LIMIT {
        // A free slot, or else (#jai fix E5) the slot of the longest decommitted entry.
        target: s64 = -1;
        for global_huge_cache.slot  if !it.span { target = it_index; break; }
        if target < 0 {
            for global_huge_cache.slot {
                if it.decommitted && (target < 0 || it.free_ms < global_huge_cache.slot[target].free_ms)  target = it_index;
            }
            if target >= 0  evicted = global_huge_cache.slot[target].span;
        }
        if target >= 0 {
            slot := *global_huge_cache.slot[target];
            // #jai a cached mapping belongs to no heap until it is handed out again
            // (heap_allocate_block_huge sets both fields), so IS_THIS_YOURS rejects it.
            span.heap = null;
            span.page.heap = null;
            slot.span = span;
            slot.free_ms = now;
            slot.decommitted = false;
            global_huge_cache.committed += committed_size;
            cached = true;
        }
    }
    huge_cache_update_oldest();
    huge_cache_lock_release();
    if evicted  span_unmap(evicted);
    // #jai E11 kept a block beyond the budget decommitted instead of letting the caller unmap it. On
    // macOS that made churn_huge slower: 4.5 us per step before, 12.4 with it, 6.3 even with E12
    // (decommit + recommit cost about 20 us there per block, more than an munmap and a fresh mapping).
    // A program that frees huge blocks often does better with a larger HUGE_CACHE_LIMIT_MB.
    return cached;
}

// Takes the smallest cached mapping that fits the request within the size class spacing; null on a
// miss. The cached span keeps its own offset, mapped_size and page_count. #jai fix E5: a decommitted
// entry is committed again here; if the OS refuses, it is unmapped and the caller maps afresh.
// #jai fix E12: a committed entry that fits goes before any decommitted one, not only one of the same
// size. Taking the smallest let committed entries sit idle while decommitted ones were recommitted:
// while E11 kept blocks beyond the budget decommitted, churn_huge on macOS paid a decommit and a
// recommit on 58% of steps (21% with this order; pattern simulated). Entries decommitted after idling
// (E5) are still reused this way. A committed entry costs nothing to hand out, and taking it frees
// committed budget.
huge_cache_pop :: (alloc_size: u64) -> *Span {   // 2.0.1: rpmalloc.c:1697
    // A racy emptiness peek is fine: a missed entry only costs a fresh mapping.
    empty := true;
    for global_huge_cache.slot {
        if it.span {
            empty = false;
            break;
        }
    }
    if empty  return null;
    size_limit := alloc_size * HUGE_CACHE_REUSE_FACTOR;   // #jai PR #427
    best: *Span;
    best_size: u64;
    best_slot: s64;
    best_decommitted: bool;
    huge_cache_lock_acquire();
    for global_huge_cache.slot {
        span := it.span;
        if !span  continue;
        size := huge_cache_entry_size(span);
        if size < alloc_size || size > size_limit  continue;
        better := !best || (best_decommitted && !it.decommitted) ||
                  (best_decommitted == it.decommitted && size < best_size);
        if better {
            best = span;
            best_size = size;
            best_slot = it_index;
            best_decommitted = it.decommitted;
        }
    }
    if best {
        global_huge_cache.slot[best_slot].span = null;
        if !best_decommitted  global_huge_cache.committed -= best_size;
        huge_cache_update_oldest();
    }
    huge_cache_lock_release();
    if best && best_decommitted {
        page_size := cast(u64) best.page_size;
        if global_memory_interface.memory_commit(pointer_offset(best, page_size), best_size - page_size) != 0 {
            span_unmap(best);
            return null;
        }
    }
    return best;
}

}   // #if HUGE_CACHE_SLOT_COUNT

span_is_thread_heap :: inline (span: *Span) -> bool {   // 2.0.1: rpmalloc.c:1508 (#jai fix #407, as page_is_thread_heap)
    #if FIRST_CLASS_HEAPS  return span.heap == get_thread_heap() || !span.heap.owner_thread;
    else                   return span.heap == get_thread_heap();
}

span_get_page_from_block :: inline (span: *Span, block: *void) -> *Page {   // 2.0.1: rpmalloc.c:1517
    return cast(*Page)(cast(u64) block & span.page_address_mask);
}

// Initialize a new page from the given span.
span_allocate_page :: inline (span: *Span) -> *Page {   // 2.0.1: rpmalloc.c:1523
    rpmalloc_assert(span.page_initialized < span.page_count, "Page initialization internal failure");
    heap := span.heap;
    page: *Page = pointer_offset(span, cast(u64) span.page_size * span.page_initialized);

    #if ENABLE_DECOMMIT {
        // Reserve-only spans commit later pages on demand; fully committed spans are already
        // committed from the map and must not be committed per page.
        // #jai fix E3: the first chunk only; page_initialize_blocks commits the rest as it carves.
        if span.page_initialized && page_has(*span.page, .COMMIT_ON_DEMAND) {
            chunk := page_commit_chunk_size(span.page_size, cast(u64) 1 << page_commit_prefix_shift(*span.page));
            if global_memory_interface.memory_commit(page, chunk) != 0  return null;
            #if HEAP_STATISTICS  heap_stat_add(*heap.stats.committed_size, chunk);
        }
    }
    span.page_initialized += 1;

    page_set_type(page, span.page_type);
    page_set(page, .COMMIT_ON_DEMAND, page_has(*span.page, .COMMIT_ON_DEMAND));
    page_set_commit_prefix_shift(page, page_commit_prefix_shift(*span.page));
    page_set_committed_chunks(page, 1);   // #jai fix E3 (page 0: committed by the span map)
    page_set(page, .IS_ZERO, true);
    page.heap = heap;
    rpmalloc_assert(page_is_thread_heap(page), "Page owner thread mismatch");

    if span.page_initialized == span.page_count {
        // Span fully used.
        page_type := cast(u32) span.page_type;
        rpmalloc_assert(span == heap.span_partial[page_type], "Span partial tracking out of sync");
        heap.span_partial[page_type] = null;
        span.next = heap.span_used[page_type];
        heap.span_used[page_type] = span;
    }
    return page;
}

// #jai fix U3: removes a freed huge span from its first-class heap's list. A first-class heap is used
// by one thread at a time, which is also the thread that frees its blocks, so the list needs no lock.
huge_span_unlink :: (heap: *Heap, span: *Span) {
    link := *heap.span_used[Page_Type.HUGE];
    while link.* {
        if link.* == span {
            link.* = span.next;
            span.next = null;
            return;
        }
        link = *link.*.next;
    }
}

// #jai upstream PR #425 (d668610): the huge block free and the free of a block in a page holding
// aligned blocks are out of line, so the generic free path, taken by every cross-thread free, needs no
// stack frame.
span_deallocate_huge :: no_inline (span: *Span) {
    #if ENABLE_STATISTICS  statistics_sub_saturating(*global_counters.huge_alloc, cast(u64) span.page_count * span.page_size);
    #if HEAP_STATISTICS {
        if span.heap {
            heap_stat_sub(*span.heap.stats.mapped_size, span.mapped_size);
            #if ENABLE_DECOMMIT  heap_stat_sub(*span.heap.stats.committed_size, cast(u64) span.page_count * span.page_size);
            else                 heap_stat_sub(*span.heap.stats.committed_size, span.mapped_size);
        }
    }
    // #jai fix U3: a first-class heap lists its huge spans in span_used[HUGE] (so heap_free_all can
    // release them), but upstream never unlinks one when it is freed. The span then goes to the
    // huge cache or is unmapped, and a later heap_free_all unmaps it again, possibly while another
    // heap uses the cached mapping. The list is singly linked and Span has no room for a back
    // pointer, so this walks it: O(live huge blocks of that heap) on a path that makes an OS call.
    // Thread heaps never link huge spans, so for them the list is empty.
    if span.heap  huge_span_unlink(span.heap, span);
    #if HUGE_CACHE_SLOT_COUNT {
        if huge_cache_push(span)  return;
    }
    span_unmap(span);
}

span_deallocate_aligned_block :: no_inline (page: *Page, block: *void) {
    block = page_block_realign(page, block);
    if page_is_thread_heap(page)  page_put_local_free_block(page, block);
    else                          page_put_thread_free_block(page, block);
}

span_deallocate_block :: no_inline (span: *Span, page: *Page, block: *void) {   // 2.0.1: rpmalloc.c:1736
    if page_get_type(page) == .HUGE {
        span_deallocate_huge(span);
        return;
    }
    if page_has(page, .HAS_ALIGNED_BLOCK) {
        span_deallocate_aligned_block(page, block);
        return;
    }
    if page_is_thread_heap(page) {
        page_put_local_free_block(page, block);
    } else {
        // #jai fix E4: a released heap has no owner to take in a deferred block until a thread
        // adopts the heap, which may be never.
        if page.heap.owner_thread == HEAP_RELEASED_OWNER && page_free_into_released_heap(page, block)  return;
        // Multithreaded deallocation: push to the deferred deallocation list.
        page_put_thread_free_block(page, block);
    }
}

// #jai fix E4: a block freed into a released heap (its thread has exited, or heap_release) went on a
// deferred list that only a thread adopting the heap drains, so its page stayed in use and committed
// until then: in Ex4, buffers that walk threads grew and the service thread freed after the walkers
// had exited (RET-18). Under the heap lock, which adoption also takes (heap_allocate_new), nobody owns
// a heap that is still released, so the block is freed as its owner would free it, the page takes in
// the frees already pending in it, and a page left empty is retired and decommitted, as at thread exit
// (E1). False when the heap has been adopted meanwhile: the caller defers the block as usual.
// #jai fix E6: under the heap's own free_lock, not the global heap lock, which every thread start and
// exit takes; frees into different released heaps no longer serialise either.
page_free_into_released_heap :: no_inline (page: *Page, block: *Block) -> bool {
    heap := page.heap;
    heap_free_lock_acquire(heap);
    if heap.owner_thread != HEAP_RELEASED_OWNER {
        heap_free_lock_release(heap);
        return false;
    }
    page_put_local_free_block(page, block);
    if !page_has(page, .IS_FREE) && !page_has(page, .IS_FULL) {
        page_adopt_thread_free_merging(page);
        // Empty: retire it, also when it is the page #413 keeps as the last of its class.
        if page.block_used == 0  page_available_to_free(page, keep_last = false);
    }
    page_type := cast(u32) page_get_type(page);
    if heap.page_free_commit_count[page_type] {
        heap.page_free_overflow_ms[page_type] = 0;
        heap_page_free_decommit(heap, page_type, 0);
    }
    heap_free_lock_release(heap);
    return true;
}

////////////
///
/// Block interface (2.0.1: rpmalloc.c:1772-1829)
///
//////

block_get_span :: inline (block: *void) -> *Span {   // 2.0.1: rpmalloc.c:1779
    return cast(*Span)(cast(u64) block & SPAN_MASK);
}

// #jai PR #422: frees a block, taking the fast path when the page's heap_fast is the given heap. The
// caller owns that heap: it is the thread heap, or the first-class heap the block came from.
block_deallocate_heap :: inline (block: *Block, owned_heap: *Heap) {   // 2.0.1: rpmalloc.c:1784 block_deallocate
    span := block_get_span(block);
    page := span_get_page_from_block(span, block);

    #if HEAP_STATISTICS {   // 2.0.1: rpmalloc.c:1789
        if span.heap {
            // A huge block subtracts its capacity, which its allocation added (#jai fix U13).
            if span.page_type <= .LARGE  heap_stat_sub(*span.heap.stats.allocated_size, page.block_size);
            else                         heap_stat_sub(*span.heap.stats.allocated_size, cast(u64) span.page_size * span.page_count);
        }
    }

    // Fast path: a free by the heap's owner, of a block in a page that is not huge, not full and has
    // no aligned block; heap_fast encodes all of that by matching the owned heap.
    if page.heap_fast == owned_heap {
        block.next = page.local_free;
        page.local_free = block;
        stat_add_free(page.heap, page.size_class, 1);
        page.block_used -= 1;
        if page.block_used == 0  page_available_to_free(page);
    } else {
        span_deallocate_block(span, page, block);
    }
}

block_deallocate :: inline (block: *Block) {
    block_deallocate_heap(block, get_thread_heap());
}

block_usable_size :: inline (block: *void) -> u64 {   // 2.0.1: rpmalloc.c:1819
    span := block_get_span(block);
    if span.page_type <= .LARGE {
        page := span_get_page_from_block(span, block);
        // #jai develop #411: the offset within a page is below 2^32; a 32-bit modulo is several times
        // cheaper than a 64-bit one.
        block_offset := cast(u32) pointer_diff(block, page) - page_block_start_offset(page);   // #jai PR #424
        return page.block_size - (block_offset % page.block_size);
    }
    return cast(u64) span.page_size * span.page_count - cast(u64) pointer_diff(block, span);
}

////////////
///
/// Heap interface (2.0.1: rpmalloc.c:1830-2509)
///
//////

heap_lock_acquire :: inline () {   // 2.0.1: rpmalloc.c:1837
    this_lock := get_thread_id();
    spin: u32;
    while true {
        swapped := compare_and_swap(*global_heap_lock, 0, this_lock);
        if swapped  break;
        wait_spin(*spin);
    }
}

heap_lock_release :: inline () {   // 2.0.1: rpmalloc.c:1848
    rpmalloc_assert(atomic_load(*global_heap_lock) == get_thread_id(), "Bad heap lock");
    atomic_store(*global_heap_lock, 0);
}

// A heap control structure is much smaller than the mapping page size (and far smaller than a huge
// page), so one page-aligned mapping is carved into as many heaps as fit: the first is returned to
// the caller and the extras are queued for reuse. Only the first heap records the mapping offset
// and size; it owns the shared mapping and is the one heap that unmaps it at finalize.
//
// Block creation is serialised through global_heap_creating: when no heap is available, one thread
// maps and carves a block while concurrent thread starts wait for the carved heaps. Ordinary
// requests reuse a released heap first, then a pristine carved one. First-class requests need a heap
// with no prior allocations, so they take only pristine heaps; a released first-class heap that holds
// nothing goes back to the pristine list (#jai fix U20).
heap_allocate_new :: (first_class: bool) -> *Heap {   // 2.0.1: rpmalloc.c:1875
    // #jai fix U11: upstream took page_size != 0 to mean "initialised", but page_size is set early in
    // initialisation and never cleared by finalize, so a heap acquired after finalize was mapped and
    // used on a finalized allocator (W-10). Check the init state itself.
    if atomic_load(*global_rpmalloc_init_state) != RPMALLOC_INIT_DONE  initialize(null);

    heap_lock_acquire();
    spin: u32;
    while true {
        if !first_class && global_heap_queue {
            heap := global_heap_queue;
            global_heap_queue = heap.next;
            // #jai fix E4: take ownership under the lock. Frees into a released heap check its owner
            // under the heap's free_lock (page_free_into_released_heap; #jai fix E6), so take that too;
            // heap_allocate set the owner only afterwards. Lock order: global, then the heap's.
            heap_free_lock_acquire(heap);
            heap.owner_thread = get_thread_id();
            heap.released = false;
            heap_free_lock_release(heap);
            heap_lock_release();
            if heap.release_trim {   // #jai develop #419: the trim deferred at release.
                heap.release_trim = 0;
                heap_release_free_pages(heap);
            }
            return heap;
        }
        if global_heap_pristine {
            heap := global_heap_pristine;
            global_heap_pristine = heap.next;
            heap_lock_release();
            return heap;
        }
        if !global_heap_creating  break;
        heap_lock_release();
        wait_spin(*spin);
        heap_lock_acquire();
    }
    global_heap_creating = 1;
    heap_lock_release();

    aligned_heap_size := cast(u64) HEAP_ALIGNMENT * ((size_of(Heap) + (HEAP_ALIGNMENT - 1)) / HEAP_ALIGNMENT);
    block_size := get_page_aligned_size(aligned_heap_size);
    offset, mapped_size: u64;
    block := global_memory_interface.memory_map(block_size, 0, *offset, *mapped_size);
    #if ENABLE_DECOMMIT {
        if block && os_commit_on_demand() && global_memory_interface.memory_commit(block, block_size) != 0 {
            global_memory_interface.memory_unmap(block, offset, mapped_size);
            block = null;
        }
    }
    if !block {
        heap_lock_acquire();
        global_heap_creating = 0;
        heap_lock_release();
        return null;
    }

    heap := heap_initialize(block);
    heap.offset = cast(u32) offset;
    heap.mapped_size = mapped_size;

    // Carve the remaining heaps into a local chain, then splice it onto the pristine pool in one
    // step, so the lock is not held across the per-heap initialisation. heap_initialize zeroes each
    // structure, leaving mapped_size 0, so the extras are not mapping owners.
    heap_count := mapped_size / aligned_heap_size;
    chain_head: *Heap;
    chain_tail: *Heap;
    extra_heap: *Heap = pointer_offset(heap, aligned_heap_size);
    for iheap: 1..cast(s64) heap_count - 1 {
        heap_initialize(extra_heap);
        if !chain_tail  chain_tail = extra_heap;
        extra_heap.next = chain_head;
        chain_head = extra_heap;
        extra_heap = pointer_offset(extra_heap, aligned_heap_size);
    }

    heap_lock_acquire();
    if chain_tail {
        chain_tail.next = global_heap_pristine;
        global_heap_pristine = chain_head;
    }
    global_heap_creating = 0;
    heap_lock_release();
    return heap;
}

// Unmaps the shared block owned by this heap. Invariant: heap blocks are only ever unmapped here,
// at global finalize, never while the allocator is running, because several heaps share a block.
heap_unmap :: (heap: *Heap) {   // 2.0.1: rpmalloc.c:1959
    global_memory_interface.memory_unmap(heap, heap.offset, heap.mapped_size);
}

heap_allocate :: (first_class: bool) -> *Heap {   // 2.0.1: rpmalloc.c:1964
    heap := heap_allocate_new(first_class);
    if heap {
        current_thread_id := get_thread_id();
        heap_lock_acquire();
        heap.next = global_heap_used;
        heap.prev = null;
        if global_heap_used  global_heap_used.prev = heap;
        global_heap_used = heap;
        heap_lock_release();
        heap.owner_thread = current_thread_id;
        heap.is_first_class = first_class;
        heap.released = false;   // #jai fix U14
        // Counters belong to the heap, not the thread, so alloc_current reflects the blocks live in
        // the heap's pages whichever thread owns it.
        #if ENABLE_STATISTICS  atomic_add(*global_counters.heap_count, 1);
    }
    return heap;
}

// Upstream heap_release, renamed: the public rpmalloc_heap_release is heap_release in this port.
// #jai fix U20: a first-class heap must start with no blocks, so heap_acquire takes only pristine
// heaps, and upstream queued every released one for thread heaps: create/destroy cycles used up a new
// heap structure each time (HEAP-13), and on macOS ran out of address space (THR-02). A heap that
// holds nothing, as after DESTROY_HEAP's heap_free_all, is as good as pristine: it is reset (keeping
// the mapping it may own) and goes back to the pristine list.
heap_is_empty :: (heap: *Heap) -> bool {
    for 0..3 {
        if heap.span_partial[it] || heap.page_free[it] || atomic_load(*heap.thread_free[it])  return false;
    }
    for heap.span_used  if it  return false;
    return true;
}

heap_release_to_pristine :: (heap: *Heap) {
    heap_lock_acquire();
    if heap.released { heap_lock_release(); return; }   // #jai fix U14: already released.
    #if ENABLE_STATISTICS  statistics_sub_saturating(*global_counters.heap_count, 1);
    if heap.prev  heap.prev.next = heap.next;
    if heap.next  heap.next.prev = heap.prev;
    if global_heap_used == heap  global_heap_used = heap.next;
    offset, mapped_size := heap.offset, heap.mapped_size;   // Only the mapping's owner has these.
    heap_initialize(heap);
    heap.offset, heap.mapped_size = offset, mapped_size;
    heap.released = true;
    heap.next = global_heap_pristine;
    global_heap_pristine = heap;
    heap_lock_release();
}

// #jai fix U14: upstream left a released first-class heap with owner_thread 0, which the free path
// reads as "first-class, the caller owns it", so frees into it after release took the unsynchronised
// local path while the heap was queued for another thread (W-13); and a heap made current by a thread
// was queued twice when that thread exited and the program released it too (HEAP-14). A released heap
// now gets an owner id no thread has, so frees into it are cross-thread frees, and a second release
// of a heap that is already released is ignored. The adopting thread sets its own id.
HEAP_RELEASED_OWNER :: 0xFFFF_FFFF_FFFF_FFFF;

// #jai fix E6: the lock for frees into a released heap (E4) and for adopting one; see Heap.free_lock.
heap_free_lock_acquire :: inline (heap: *Heap) {
    spin: u32;
    while !compare_and_swap(*heap.free_lock, 0, 1)  wait_spin(*spin);
}

heap_free_lock_release :: inline (heap: *Heap) {
    atomic_store(*heap.free_lock, 0);
}

heap_release_to_queue :: inline (heap: *Heap) {   // 2.0.1: rpmalloc.c:1986 heap_release
    heap.thread_current = false;   // #jai fix E10
    heap_lock_acquire();
    if heap.released { heap_lock_release(); return; }   // #jai fix U14: already released.
    heap.released = true;
    heap.owner_thread = HEAP_RELEASED_OWNER;            // #jai fix U14
    #if ENABLE_STATISTICS  statistics_sub_saturating(*global_counters.heap_count, 1);
    if heap.prev  heap.prev.next = heap.next;
    if heap.next  heap.next.prev = heap.prev;
    if global_heap_used == heap  global_heap_used = heap.next;
    heap.next = global_heap_queue;
    global_heap_queue = heap;
    heap_lock_release();
}

heap_initialize :: (block: *void) -> *Heap {   // 2.0.1: rpmalloc.c:1855
    heap := cast(*Heap) block;
    memset(heap, 0, size_of(Heap));
    heap.id = 1 + atomic_add(*global_heap_id, 1);
    return heap;
}

// #jai fix (U6 part, needed by develop #413): #413 keeps a size class's last available page when it
// empties, so the next allocation of the class need not rebuild it. Every class used then pins one
// empty, committed page until the heap is released: 226 MiB after RET-02's burst, where the large
// classes use 16 MiB pages. thread_collect retires those pages through the normal free path, where
// the decommit thresholds apply. An empty page (block_used 0) has no blocks on the heap's local free
// list and none pending from other threads, so retiring it is safe.
// #jai fix U6 (rest): blocks freed by other threads sit on the heap's thread_free lists (full
// pages) or in a page's own thread-free token (pages that are not full) until the owner next needs a
// page of that type; an idle owner never does, so the memory stays committed (RET-03). thread_collect
// drains both: the heap lists are freed locally (pages that empty go to the free list), and every
// available page adopts its deferred frees, retiring it if it is now empty.
heap_drain_deferred_frees :: (heap: *Heap) {
    for itype: 0..3 {
        if !atomic_load(*heap.thread_free[itype])  continue;
        block := cast(*Block) atomic_exchange(*heap.thread_free[itype], 0);
        while block {
            next_block := block.next;
            block_deallocate(block);
            block = next_block;
        }
    }
    for iclass: 0..SIZE_CLASS_COUNT - 1 {
        page := heap.page_available[iclass];
        while page {
            next_page := page.next;
            if atomic_load(*page.thread_free) {
                page_adopt_thread_free_merging(page);   // #jai fix E2
                if page.block_used == 0  page_available_to_free(page, keep_last = false);
            }
            page = next_page;
        }
    }
}

// #jai fix E2: page_adopt_thread_free_block_list leaves a page alone while its local free list is
// non-empty, because claiming the thread-free list replaces that list (the allocation path only needs
// a list when the local one is empty). The drain needs every pending block: a page freed partly by its
// owner and partly by other threads kept its pending blocks, so it stayed in use and committed through
// thread_collect and trim() (RET-16), and at thread exit (RET-15). Here the claimed list is linked in
// front of the local one.
page_adopt_thread_free_merging :: (page: *Page) {
    if !page.local_free {
        page_adopt_thread_free_block_list(page);
        return;
    }
    if !atomic_load(*page.thread_free)  return;
    token := atomic_exchange(*page.thread_free, 0);
    head: *Block;
    count := page_block_from_thread_free_list(page, token, *head);
    if !count  return;
    tail := head;
    for 2..count  tail = tail.next;
    tail.next = page.local_free;
    page.local_free = head;
    rpmalloc_assert(count <= page.block_used, "Page thread free list count internal failure");
    page.block_used -= count;
    stat_add_free(page.heap, page.size_class, count);
}

heap_retire_kept_pages :: (heap: *Heap) {
    for iclass: 0..SIZE_CLASS_COUNT - 1 {
        page := heap.page_available[iclass];
        if page && !page.next && page.block_used == 0  page_available_to_free(page, keep_last = false);
    }
}

heap_page_free_decommit :: (heap: *Heap, page_type: u32, page_retain_count: u32) {   // 2.0.1: rpmalloc.c:2003
    page := heap.page_free[page_type];
    while page && page_retain_count {
        page = page.next;
        page_retain_count -= 1;
    }
    while page && !page_has(page, .IS_DECOMMITTED) {
        page_decommit_memory_pages(page);
        // Decommit can be refused (decommit disabled, page type not larger than the configured
        // page size, or the kernel rejecting decommit of huge page mappings); then the remaining
        // pages cannot be decommitted either.
        if !page_has(page, .IS_DECOMMITTED)  break;
        heap.page_free_commit_count[page_type] -= 1;
        page = page.next;
    }
}

// #jai develop #416 (851e255): decommit by time, not by count. Called when the free committed page
// count of a page type is at or above the overflow threshold. The excess is decommitted only once
// the count has stayed above the threshold for PAGE_DECOMMIT_DELAY_MS (dropping below restarts the
// delay), or at once at PAGE_DECOMMIT_LIMIT_FACTOR times the threshold. Decommitting on the count
// alone made every allocate/free phase larger than the threshold decommit its pages and fault them
// back in on the next phase (RET-10). thread_collect decommits the excess immediately.
PAGE_DECOMMIT_DELAY_MS     :: 250;
PAGE_DECOMMIT_LIMIT_FACTOR :: 4;

// #jai upstream PR #420 (fb4288f): decommits the most recently freed page, at the head of the free
// list, and moves it to the tail behind the committed pages, so the list keeps its committed pages
// first for reuse.
heap_page_free_decommit_head :: (heap: *Heap, page_type: u32) {
    page := heap.page_free[page_type];
    if !page || page_has(page, .IS_DECOMMITTED)  return;
    page_decommit_memory_pages(page);
    if !page_has(page, .IS_DECOMMITTED)  return;
    heap.page_free_commit_count[page_type] -= 1;
    if page.next {
        heap.page_free[page_type] = page.next;
        heap.page_free_tail[page_type].next = page;
        page.next = null;
        heap.page_free_tail[page_type] = page;
    }
}

// The decommit delay (#416) and, #jai PR #420, the hard limit within it: a page freed with the count at
// PAGE_DECOMMIT_LIMIT_FACTOR times the threshold is decommitted on its own. #416 decommitted every free
// page down to the retain count there, including the pages the next phase reuses (W-12).
heap_page_free_overflow :: no_inline (heap: *Heap, page_type: u32) {
    limit := cast(u64) global_page_free_overflow[page_type] * PAGE_DECOMMIT_LIMIT_FACTOR;
    now := os_monotonic_time_ms();
    since := heap.page_free_overflow_ms[page_type];
    if !since {
        since = ifx now then now else 1;
        heap.page_free_overflow_ms[page_type] = since;
    }
    if elapsed_ms(now, since) < PAGE_DECOMMIT_DELAY_MS {
        if heap.page_free_commit_count[page_type] >= limit  heap_page_free_decommit_head(heap, page_type);
        return;
    }
    heap.page_free_overflow_ms[page_type] = 0;
    heap_page_free_decommit(heap, page_type, global_page_free_retain[page_type]);
}

// #jai fix U5 (second half): the decommit delay was checked only when another page was freed, so a
// heap that stopped freeing kept its free pages over the threshold (up to 4x) for good (RET-12).
// The allocation slow path, which runs whenever a page is needed, checks it too: one test of four
// timestamps while no delay is running.
heap_recheck_decommit_delay :: inline (heap: *Heap) {
    if !(heap.page_free_overflow_ms[0] | heap.page_free_overflow_ms[1] | heap.page_free_overflow_ms[2] | heap.page_free_overflow_ms[3])
        return;
    heap_recheck_decommit_delay_now(heap);
}

heap_recheck_decommit_delay_now :: no_inline (heap: *Heap) {
    now := os_monotonic_time_ms();
    for itype: 0..3 {
        since := heap.page_free_overflow_ms[itype];
        if !since || elapsed_ms(now, since) < PAGE_DECOMMIT_DELAY_MS  continue;
        heap.page_free_overflow_ms[itype] = 0;
        heap_page_free_decommit(heap, cast(u32) itype, global_page_free_retain[itype]);
    }
}

// #jai develop #418 (4b83f74), #419 (1d5730b): trims the memory a released heap holds for reuse.
// Until another thread adopts the heap nothing reuses that memory or runs the decommit delay, so a
// heap passed along by many short-lived threads would keep all of it (RET-11). The empty
// medium-large and large pages kept as the last available page of their class are retired, and free
// pages of those types are decommitted to below the overflow threshold, the most a heap holds without
// the delay. Small and medium-small pages are cheap to keep and costly to fault back in; they are
// left to the decommit delay (#419).
heap_release_free_pages :: (heap: *Heap) {
    for iclass: 0..SIZE_CLASS_COUNT - 1 {
        page := heap.page_available[iclass];
        if !page || page.next || page.block_used || page_get_type(page) < .MEDIUM_LARGE  continue;
        heap.page_available[iclass] = null;
        page_set(page, .IS_FREE, true);
        page_set(page, .IS_ZERO, false);
        page_type := cast(u32) page_get_type(page);
        page.next = heap.page_free[page_type];
        heap.page_free[page_type] = page;
        if !page.next  heap.page_free_tail[page_type] = page;   // #jai PR #420
        heap.page_free_commit_count[page_type] += 1;
    }
    for itype: cast(s64) Page_Type.MEDIUM_LARGE..cast(s64) Page_Type.LARGE {
        heap.page_free_overflow_ms[itype] = 0;
        if heap.page_free_commit_count[itype] >= global_page_free_overflow[itype]
            heap_page_free_decommit(heap, cast(u32) itype, global_page_free_overflow[itype] - 1);
    }
}

// #jai develop #419: releases a heap (by its exiting thread, or a first-class heap) for reuse. With
// other heaps already queued it is likely to stay unused for a while and is trimmed at once. With
// the queue empty a thread is likely to adopt it shortly, so the trim is left to the adopter rather
// than delaying the release while threads wait for a heap.
heap_release_trim :: (heap: *Heap) {
    heap_lock_acquire();
    queued := global_heap_queue != null;
    heap_lock_release();
    // With the queue empty the trim waits for an adopter, which may never come (U5); threads that
    // exit therefore use heap_release_on_thread_exit instead. Only first-class heaps come here.
    if queued  heap_release_free_pages(heap);
    else       heap.release_trim = 1;
    heap_release_to_queue(heap);
}

// #jai fix U5 (first half): a thread that exits trims its heap at once, over every page type,
// instead of leaving it to an adopter (#419), which may never come: when the last worker of a
// program exits, its heap's memory stayed committed (RET-11). Kept empty pages of every class are
// retired and free pages decommitted to below the overflow threshold. The rest of U5, re-checking
// the decommit delay on the allocation slow path, comes with the retention fixes.
// #jai fix E1 (found in Ex4 1.45.0): that trim kept free pages up to just under each type's overflow
// threshold (about 36 MiB) for an adopter, and blocks other threads had freed into the heap stayed
// pending, keeping their pages in use. With many worker threads that exit and no thread adopting
// their heaps, all of it stayed committed: 625 MiB after RET-15's 8 threads, and 709 vs 139 MB of
// private commit in Ex4's index build with 24 workers. The exiting thread now does for its heap what
// trim() does: drain the pending frees (with fix E2), retire the kept empty pages, and decommit every
// free page. An adopter recommits what it uses. Frees that arrive after the exit still wait for one.
// Callers make the heap the thread heap first: the drain frees through the thread heap's paths.
heap_release_on_thread_exit :: (heap: *Heap) {
    heap_drain_deferred_frees(heap);
    heap_retire_kept_pages(heap);
    for itype: 0..3 {
        heap.page_free_overflow_ms[itype] = 0;
        if heap.page_free_commit_count[itype]  heap_page_free_decommit(heap, cast(u32) itype, 0);
    }
    heap_release_to_queue(heap);
}

heap_make_free_page_available :: inline (heap: *Heap, size_class: u32, page: *Page) -> s32 {   // 2.0.1: rpmalloc.c:2022
    // #jai fix U4: recommit before anything else. Upstream linked the page into page_available
    // first, so when the recommit failed the still-decommitted page stayed available and the next
    // allocation wrote into memory that was not committed (an access violation on Windows; C 2.0.1
    // does the same in OS-09). On failure nothing is changed; the caller keeps the page.
    if page_has(page, .IS_DECOMMITTED) && page_commit_memory_pages(page) != 0  return 1;
    #if TEST_HOOKS  atomic_add(*test_page_reinits, 1);
    page.size_class = size_class;
    page.block_size = global_size_class[size_class].block_size;
    page.block_count = global_size_class[size_class].block_count;
    page.block_used = 0;
    page.block_initialized = 0;
    page.local_free = null;
    page_set(page, .IS_FULL, false);
    page_set(page, .IS_FREE, false);
    page_set(page, .HAS_ALIGNED_BLOCK, false);
    page.heap = heap;
    page.heap_fast = heap;   // #jai PR #422
    head := heap.page_available[size_class];
    page.next = head;
    page.prev = null;
    atomic_store(*page.thread_free, 0);
    if head  head.prev = page;
    heap.page_available[size_class] = page;
    return 0;
}

// Find or allocate a span for the given page type.
heap_get_span :: inline (heap: *Heap, page_type: Page_Type) -> *Span {   // 2.0.1: rpmalloc.c:2049
    // Fast path: an available span for the page type.
    if heap.span_partial[page_type]  return heap.span_partial[page_type];

    // Fallback path: map more memory.
    offset, mapped_size: u64;
    span := cast(*Span) global_memory_interface.memory_map(SPAN_SIZE, SPAN_SIZE, *offset, *mapped_size);
    if span {
        #if HEAP_STATISTICS  heap_stat_add(*heap.stats.mapped_size, mapped_size);   // #jai fix U13: only when the map succeeded.
        rpmalloc_assert(!(cast(u64) span & (SPAN_SIZE - 1)), "memory_map returned a block not aligned to the span size");
        stat_inc_span_map(heap, page_type);
        page_count: u32;
        page_size: u32;
        page_address_mask: u64;
        if page_type == .SMALL {
            page_count = SPAN_SIZE / SMALL_PAGE_SIZE;
            page_size = SMALL_PAGE_SIZE;
            page_address_mask = SMALL_PAGE_MASK;
        } else if page_type == .MEDIUM_SMALL {
            page_count = SPAN_SIZE / MEDIUM_SMALL_PAGE_SIZE;
            page_size = MEDIUM_SMALL_PAGE_SIZE;
            page_address_mask = MEDIUM_SMALL_PAGE_MASK;
        } else if page_type == .MEDIUM_LARGE {
            page_count = SPAN_SIZE / MEDIUM_LARGE_PAGE_SIZE;
            page_size = MEDIUM_LARGE_PAGE_SIZE;
            page_address_mask = MEDIUM_LARGE_PAGE_MASK;
        } else {
            page_count = SPAN_SIZE / LARGE_PAGE_SIZE;
            page_size = LARGE_PAGE_SIZE;
            page_address_mask = LARGE_PAGE_MASK;
        }
        // Commit the first page for reserve-only spans (fully committed spans commit nothing here);
        // record the commit model on the span only after its header is committed.
        commit_on_demand := os_commit_on_demand();
        commit_prefix_shift: u32 = 0;
        while (cast(u64) 1 << commit_prefix_shift) < global_config.page_size  commit_prefix_shift += 1;
        // #jai fix E3: the first chunk of page 0 only; page_initialize_blocks commits the rest.
        first_commit := page_commit_chunk_size(page_size, cast(u64) 1 << commit_prefix_shift);
        #if ENABLE_DECOMMIT {
            if commit_on_demand {
                if global_memory_interface.memory_commit(span, first_commit) != 0 {
                    global_memory_interface.memory_unmap(span, offset, mapped_size);
                    return null;
                }
            }
        }
        page_set(*span.page, .COMMIT_ON_DEMAND, commit_on_demand);
        page_set_commit_prefix_shift(*span.page, commit_prefix_shift);
        page_set_committed_chunks(*span.page, 1);
        #if HEAP_STATISTICS {
            #if ENABLE_DECOMMIT  heap_stat_add(*heap.stats.committed_size, ifx commit_on_demand then first_commit else page_size);
            else                 heap_stat_add(*heap.stats.committed_size, mapped_size);
        }
        span.heap = heap;
        span.page_type = page_type;
        span.page_count = page_count;
        span.page_size = page_size;
        span.page_address_mask = page_address_mask;
        span.offset = cast(u32) offset;
        span.mapped_size = mapped_size;
        span_register(span);   // #jai for IS_THIS_YOURS

        heap.span_partial[page_type] = span;
    }
    return span;
}

heap_get_page_generic :: (heap: *Heap, size_class: u32) -> *Page {   // 2.0.1: rpmalloc.c:2129
    page_type := get_page_type(size_class);
    heap_recheck_decommit_delay(heap);   // #jai fix U5
    #if HUGE_CACHE_SLOT_COUNT  huge_cache_purge_if_due();   // #jai fix E5

    // Check for a free page from multithreaded deallocations.
    block_mt := atomic_load(*heap.thread_free[page_type]);
    if block_mt != 0 {
        // #jai develop #415: claim the whole list in one exchange; concurrent pushes land either
        // before it (and are claimed) or after.
        block_mt = atomic_exchange(*heap.thread_free[page_type], 0);
        block := cast(*Block) block_mt;
        while block {
            next_block := block.next;
            block_deallocate(block);
            block = next_block;
        }
        // Retry after processing the deferred thread frees.
        return heap_get_page(heap, size_class);
    }

    // Check for a free page.
    page := heap.page_free[page_type];
    if page {
        heap.page_free[page_type] = page.next;
        if !page.next  heap.page_free_tail[page_type] = null;   // #jai PR #420
        if !page_has(page, .IS_DECOMMITTED) {
            rpmalloc_assert(heap.page_free_commit_count[page_type] > 0, "Free committed page count out of sync");
            heap.page_free_commit_count[page_type] -= 1;
            // #jai develop #416: dropping below the threshold restarts the decommit delay.
            if heap.page_free_commit_count[page_type] < global_page_free_overflow[page_type]
                heap.page_free_overflow_ms[page_type] = 0;
        }
        if heap_make_free_page_available(heap, size_class, page) != 0 {
            // #jai fix U4: the recommit failed; put the (still decommitted) page back for later, at the
            // tail behind the committed pages (#jai PR #420).
            page.next = null;
            if heap.page_free_tail[page_type]  heap.page_free_tail[page_type].next = page;
            else                               heap.page_free[page_type] = page;
            heap.page_free_tail[page_type] = page;
            return null;
        }
        return page;
    }
    rpmalloc_assert(heap.page_free_commit_count[page_type] == 0, "Free committed page count out of sync");

    if heap.id == 0 {
        // The thread is not initialised yet: assign a heap and try again.
        initialize(null);
        return heap_get_page(get_thread_heap(), size_class);
    }

    // Fallback path: find or allocate a span for the size class. If the thread was not
    // initialised, the heap for the new span differs from the local heap variable here (the
    // default heap), so use the span page heap instead.
    span := heap_get_span(heap, page_type);
    if span {
        page = span_allocate_page(span);
        if !page  return null;   // #jai fix U4: the page's commit failed; upstream dereferenced null.
        if heap_make_free_page_available(page.heap, size_class, page) != 0  return null;
    }
    return page;
}

// Find or allocate a page for the given size class.
heap_get_page :: (heap: *Heap, size_class: u32) -> *Page {   // 2.0.1: rpmalloc.c:2186
    // Fast path: an available page for the size class.
    page := heap.page_available[size_class];
    if page  return page;
    return heap_get_page_generic(heap, size_class);
}

// Pop a block from the heap local free list.
heap_pop_local_free :: inline (heap: *Heap, size_class: u32) -> *Block {   // 2.0.1: rpmalloc.c:2196
    free_list := *heap.local_free[size_class];
    block := free_list.*;
    if block  free_list.* = block.next;
    return block;
}

// Generic allocation path from heap pages, spans or a new mapping.
heap_allocate_block_small_to_large :: no_inline (heap: *Heap, size_class: u32, zero: bool) -> *void {   // 2.0.1: rpmalloc.c:2206
    page := heap_get_page(heap, size_class);
    if page {
        block := page_allocate_block(page, zero);
        // Count on the page owner, which may differ from heap when an uninitialised thread is
        // routed to its real heap, so the allocation and its later free count on the same heap.
        // #jai fix E3: only once the block exists; committing it can fail.
        if block  stat_inc_alloc(page.heap, size_class);
        return block;
    }
    return null;
}

heap_allocate_block_huge :: no_inline (heap: *Heap, size: u64, zero: bool) -> *void {   // 2.0.1: rpmalloc.c:2219
    if heap.id == 0 {
        // The thread is not initialised yet: assign a heap and try again.
        initialize(null);
        heap = get_thread_heap();
    }
    // #jai fix U1: upstream checks the size only under ENABLE_VALIDATE_ARGS, so without it
    // size + SPAN_HEADER_SIZE and the page rounding wrap, and alloc(-1) returns a tiny block.
    // Refusing here bounds every path into a mapping; the cost is one compare on a path that maps.
    if size >= MAX_ALLOC_SIZE  return null;
    alloc_size := get_page_aligned_size(size + SPAN_HEADER_SIZE);
    offset, mapped_size: u64;
    block: *void;
    from_cache := false;
    #if HUGE_CACHE_SLOT_COUNT {
        // Reuse a recently freed huge mapping of fitting size if there is one.
        huge_cache_purge_if_due();   // #jai fix E5: huge allocations age the cache too.
        block = huge_cache_pop(alloc_size);
        from_cache = block != null;
    }
    committed_at_map := false;
    if !block {
        // #jai fix E9: with the OS's own interface, map and commit in one call (see os_mmap_committed).
        // A custom interface keeps the separate map and commit calls.
        if global_memory_interface.memory_map == os_mmap && os_commit_on_demand() {
            block = os_mmap_committed(alloc_size, SPAN_SIZE, *offset, *mapped_size);
            committed_at_map = true;
        } else {
            block = global_memory_interface.memory_map(alloc_size, SPAN_SIZE, *offset, *mapped_size);
        }
    }
    if block {
        span := cast(*Span) block;
        rpmalloc_assert(!(cast(u64) span & (SPAN_SIZE - 1)), "memory_map returned a block not aligned to the span size");
        if !from_cache  stat_inc_span_map(heap, .HUGE);
        #if ENABLE_DECOMMIT {
            if !from_cache && !committed_at_map && os_commit_on_demand() && global_memory_interface.memory_commit(span, alloc_size) != 0 {
                global_memory_interface.memory_unmap(block, offset, mapped_size);
                return null;
            }
        }
        span.heap = heap;
        span.page_type = .HUGE;
        span.page_address_mask = LARGE_PAGE_MASK;
        // Huge blocks never use the per-page commit path; set explicitly so a reused cached
        // mapping carries no stale value.
        page_set(*span.page, .COMMIT_ON_DEMAND, false);
        if !from_cache {
            span.page_size = cast(u32) global_config.page_size;
            span.page_count = cast(u32)(alloc_size / global_config.page_size);
            span.offset = cast(u32) offset;
            span.mapped_size = mapped_size;
        }
        #if HEAP_STATISTICS {
            // #jai fix U13: from the span's own fields, so a mapping reused from the huge cache counts
            // its real size (upstream added 0 mapped bytes and the request's commit for those), and
            // the free subtracts exactly this.
            heap_stat_add(*heap.stats.mapped_size, span.mapped_size);
            #if ENABLE_DECOMMIT  heap_stat_add(*heap.stats.committed_size, cast(u64) span.page_count * span.page_size);
            else                 heap_stat_add(*heap.stats.committed_size, span.mapped_size);
        }
        span.page.heap = heap;
        page_set(*span.page, .IS_FULL, true);
        span.page.heap_fast = null;   // #jai PR #422: huge blocks take the slow path.
        page_set_type(*span.page, .HUGE);
        #if ENABLE_STATISTICS  statistics_add_with_peak(*global_counters.huge_alloc, *global_counters.huge_alloc_peak, cast(u64) span.page_count * span.page_size);
        // Keep track of the span if this is a first-class heap.
        if !heap.owner_thread {
            span.next = heap.span_used[Page_Type.HUGE];   // Unlinked again on free (#jai fix U3).
            heap.span_used[Page_Type.HUGE] = span;
        }
        if !from_cache  span_register(span);   // #jai for IS_THIS_YOURS
        ptr := pointer_offset(block, SPAN_HEADER_SIZE);
        // #jai develop #406 (4301fb8): a fresh mapping from the OS interface is already zero
        // (anonymous mmap and VirtualAlloc guarantee it); clearing it again faults in and commits every
        // page. Reused cached mappings and custom memory interfaces give no such guarantee.
        if zero && (from_cache || global_memory_interface.memory_map != os_mmap)  memset(ptr, 0, cast(s64) size);
        // #jai fix U13: the capacity, which the free subtracts; upstream added the requested size and
        // subtracted the page-rounded one, so allocated_size drifted (API-06).
        #if HEAP_STATISTICS  heap_stat_add(*heap.stats.allocated_size, cast(u64) span.page_size * span.page_count);
        return ptr;
    }
    return null;
}

heap_allocate_block_generic :: no_inline (heap: *Heap, size: u64, zero: bool) -> *void {   // 2.0.1: rpmalloc.c:2305
    size_class := get_size_class(size);
    if size_class < SIZE_CLASS_COUNT {
        #if HEAP_STATISTICS  heap_stat_add(*heap.stats.allocated_size, global_size_class[size_class].block_size);
        block := heap_pop_local_free(heap, size_class);
        if block {
            // Fast track with a small block available in the heap local free list; a hit means
            // this heap owns the block.
            stat_inc_alloc(heap, size_class);
            if zero  memset(block, 0, global_size_class[size_class].block_size);
            return block;
        }
        return heap_allocate_block_small_to_large(heap, size_class, zero);
    }
    return heap_allocate_block_huge(heap, size, zero);
}

// Find or allocate a block of the given size.
heap_allocate_block :: inline (heap: *Heap, size: u64, zero: bool) -> *void {   // 2.0.1: rpmalloc.c:2330
    if size <= SMALL_GRANULARITY * 64 {
        size_class := get_size_class_tiny(size);
        block := heap_pop_local_free(heap, size_class);
        if block {
            // Fast track with a small block available in the heap local free list.
            if zero  memset(block, 0, global_size_class[size_class].block_size);
            #if HEAP_STATISTICS  heap_stat_add(*heap.stats.allocated_size, global_size_class[size_class].block_size);
            stat_inc_alloc(heap, size_class);
            return block;
        }
        #if HEAP_STATISTICS  heap_stat_add(*heap.stats.allocated_size, global_size_class[size_class].block_size);
        // The size class is already known: refill directly, skipping the generic path's size
        // class recomputation and dead local free list pop.
        return heap_allocate_block_small_to_large(heap, size_class, zero);
    }
    return heap_allocate_block_generic(heap, size, zero);
}

// An aligned request allocates size + alignment and moves the pointer up to the alignment. The page
// is then marked, so that freeing the moved pointer finds the block start (page_block_realign).
heap_allocate_block_aligned :: (heap: *Heap, alignment: u64, size: u64, zero: bool) -> *void {   // 2.0.1: rpmalloc.c:2356
    if alignment <= SMALL_GRANULARITY  return heap_allocate_block(heap, size, zero);

    // #jai fix U1: upstream makes these two checks only under ENABLE_VALIDATE_ARGS. Without them
    // size + alignment wraps and a tiny block comes back, and a non-power-of-two alignment gives a
    // misaligned block. This path over-allocates anyway; the checks cost two compares.
    // #jai Without computing size + alignment, which traps with arithmetic_overflow_check.
    if size > 0xFFFF_FFFF_FFFF_FFFF - alignment  return null;
    if alignment & (alignment - 1)  return null;
    if alignment >= RPMALLOC_MAX_ALIGNMENT  return null;

    align_mask := alignment - 1;
    // #jai develop #405 (a3abe2e): blocks start PAGE_HEADER_SIZE bytes into a page (or span) whose
    // start is aligned to at least the page size. Rounding the size up to a multiple of the alignment
    // (and at least the alignment) selects a size class whose block size is a multiple of the
    // alignment: up to 1 KiB every multiple of the granularity is a class, above that every class size
    // is a multiple of 256. Every block of that class is then naturally aligned: no over-allocation,
    // no realignment, and the page keeps the fast free path.
    // #jai PR #424: up to NATURAL_ALIGNMENT_LIMIT, a class whose block size is a multiple of the
    // alignment starts its blocks at an offset that is a multiple of it too, so the plain allocation of
    // the size rounded up to the alignment is aligned.
    if alignment > PAGE_HEADER_SIZE && alignment <= NATURAL_ALIGNMENT_LIMIT {
        aligned_size := (size + align_mask) & ~align_mask;
        if aligned_size >= size && aligned_size <= LARGE_BLOCK_SIZE_LIMIT {
            size_class := get_size_class(aligned_size);
            block_size := ifx size_class < SIZE_CLASS_COUNT then global_size_class[size_class].block_size else 0;
            if cast(u64) lowest_set_bit(block_size) >= alignment  return heap_allocate_block(heap, aligned_size, zero);
        }
    }
    if alignment <= PAGE_HEADER_SIZE {
        aligned_size := (size + align_mask) & ~align_mask;
        if aligned_size < alignment  aligned_size = alignment;
        if aligned_size >= size {
            block := heap_allocate_block(heap, aligned_size, zero);
            rpmalloc_assert(!(cast(u64) block & align_mask), "Natural block alignment internal failure");
            return block;
        }
    }
    block := heap_allocate_block(heap, size + alignment, zero);
    if cast(u64) block & align_mask {
        block = cast(*void)((cast(u64) block & ~align_mask) + alignment);
        // Mark the page as having aligned blocks.
        span := block_get_span(block);
        page := span_get_page_from_block(span, block);
        // #jai PR #424: write the flag word only on the page's first aligned block.
        if !page_has(page, .HAS_ALIGNED_BLOCK) {
            page_set(page, .HAS_ALIGNED_BLOCK, true);
            page.heap_fast = null;   // #jai PR #422
        }
    }
    return block;
}

heap_reallocate_block :: no_inline (heap: *Heap, block: *void, size: u64, old_size: u64, flags: Realloc_Flags) -> *void {   // 2.0.1: rpmalloc.c:2389
    if block {
        // Grab the span through the guaranteed span alignment.
        span := block_get_span(block);
        if span.page_type <= .LARGE {
            // Normal sized block.
            page := span_get_page_from_block(span, block);
            // #jai upstream PR #423 (951f903): a pointer from the allocator is at its block start
            // unless the page holds aligned blocks, so the division locating the block start is only
            // needed for those pages.
            block_origin := block;
            if page_has(page, .HAS_ALIGNED_BLOCK)  block_origin = page_block_realign(page, block);
            if !old_size  old_size = cast(u64)(cast(s64) page.block_size - pointer_diff(block, block_origin));
            if cast(u64) page.block_size >= size {
                // Still fits in the block: never mind saving memory, but preserve the data if the
                // alignment changed.
                if block != block_origin && !(flags & .NO_PRESERVE)
                    memmove_rp(block_origin, block, cast(s64) old_size);
                return block_origin;
            }
        } else {
            // Huge block.
            block_start := pointer_offset(span, SPAN_HEADER_SIZE);
            capacity := cast(u64) span.page_size * span.page_count - cast(u64) pointer_diff(block, span);
            if !old_size  old_size = capacity;
            // #jai fix U12: upstream judged the fit on old_size, which is also the caller's live size,
            // with size < old_size, so a same-size request with the real old size reallocated (RE-06).
            // The fit is judged on the capacity, and an equal size fits.
            if size <= capacity && size >= capacity / 2 && size > LARGE_BLOCK_SIZE_LIMIT {
                // Still fits, still huge, and saves less than half the memory: keep it, but
                // preserve the data if the alignment changed.
                if block_start != block && !(flags & .NO_PRESERVE)
                    memmove_rp(block_start, block, cast(s64) old_size);
                return block_start;
            }
        }
    } else {
        old_size = 0;
    }

    if flags & .GROW_OR_FAIL  return null;

    // The size is greater than the block size, or shrinking saves enough memory: allocate a new
    // block and free the old one. Avoid hysteresis by overallocating small increases (below 37%).
    lower_bound := old_size + (old_size >> 2) + (old_size >> 3);
    new_size := ifx size > lower_bound then size else (ifx size > old_size then lower_bound else size);
    old_block := block;
    block = heap_allocate_block(heap, new_size, false);
    if block && old_block {
        if !(flags & .NO_PRESERVE)
            memcpy(block, old_block, cast(s64) ifx old_size < new_size then old_size else new_size);
        block_deallocate(old_block);
    }
    return block;
}

heap_reallocate_block_aligned :: (heap: *Heap, block: *void, alignment: u64, size: u64, old_size: u64, flags: Realloc_Flags) -> *void {   // 2.0.1: rpmalloc.c:2444
    if alignment <= SMALL_GRANULARITY  return heap_reallocate_block(heap, block, size, old_size, flags);

    no_alloc := (flags & .GROW_OR_FAIL) != 0;
    usable_size := ifx block then block_usable_size(block) else 0;
    if usable_size >= size && !(cast(u64) block & (alignment - 1)) {
        if no_alloc || size >= usable_size / 2  return block;
    }
    // The aligned allocation marks the page as having aligned blocks.
    old_block := block;
    block = ifx !no_alloc then heap_allocate_block_aligned(heap, alignment, size, false) else null;
    if block {
        if !(flags & .NO_PRESERVE) && old_block {
            if !old_size  old_size = usable_size;
            memcpy(block, old_block, cast(s64) ifx old_size < size then old_size else size);
        }
        if old_block  block_deallocate(old_block);
    }
    return block;
}

// Upstream heap_free_all, renamed: the public rpmalloc_heap_free_all is heap_free_all in this port.
heap_unmap_all :: (heap: *Heap) {   // 2.0.1: rpmalloc.c:2471 heap_free_all
    for itype: 0..3 {
        span := heap.span_partial[itype];
        while span {
            span_next := span.next;
            span_unmap(span);
            span = span_next;
        }
        heap.span_partial[itype] = null;
        heap.page_free[itype] = null;
        heap.page_free_tail[itype] = null;   // #jai PR #420
        heap.page_free_commit_count[itype] = 0;
        heap.page_free_overflow_ms[itype] = 0;   // #jai develop #416
        atomic_store(*heap.thread_free[itype], 0);
    }
    for itype: 0..4 {
        span := heap.span_used[itype];
        while span {
            span_next := span.next;
            #if ENABLE_STATISTICS {
                if itype == cast(s64) Page_Type.HUGE  statistics_sub_saturating(*global_counters.huge_alloc, cast(u64) span.page_count * span.page_size);
            }
            span_unmap(span);   // Only live huge blocks are listed (#jai fix U3).
            span = span_next;
        }
        heap.span_used[itype] = null;
    }
    memset(heap.local_free.data, 0, size_of(type_of(heap.local_free)));
    memset(heap.page_available.data, 0, size_of(type_of(heap.page_available)));
    #if ENABLE_STATISTICS {
        // Every block in this heap has been released, so reverse the per size class counters.
        for * heap.size_use {
            it.free_total += it.alloc_current;
            it.alloc_current = 0;
        }
    }
}

////////////
///
/// Extern interface (2.0.1: rpmalloc.c:2510-2660)
///
//////
// rpmalloc_is_thread_initialized:2517, rpmalloc:2522, rpzalloc:2534, rpfree:2546,
// rpcalloc:2553, rprealloc:2577, rpaligned_realloc:2589, rpaligned_alloc:2601,
// rpaligned_zalloc:2607, rpaligned_calloc:2613, rpmemalign:2637, rpposix_memalign:2643,
// rpmalloc_usable_size:2653. (rpmalloc_linker_reference:2658 is C-only; not ported.)

////////////
///
/// Initialization and finalization (2.0.1: rpmalloc.c:2661-3421)
///
//////
// The exported procedures of this section are further down, after #scope_export.
// os_read_system_file:2684 lives in os_posix.jai. (malloc.c and rpnew.h, the libc / C++
// overrides, are not ported.)

// Runs at thread exit through the FLS / pthread key (2.0.1: rpmalloc.c:2668). #c_call: the OS calls
// it, so it has no Jai context of its own.
rpmalloc_thread_destructor :: (value: *void) #c_call {
    fresh: #Context;
    push_context fresh {
        // On the main thread this means rpmalloc_finalize has not been called and shutdown is
        // forced (through _exit) or unclean.
        if get_thread_id() == global_main_thread_id  return;
        // #jai fix U22: when another thread calls exit, Windows stops every other thread and then
        // runs this destructor on the exiting one; releasing its heap takes global_heap_lock, which
        // a stopped thread may hold, and the process never exits (LIFE-05). The process is ending
        // anyway, so keep the heap.
        if os_process_exiting()  return;
        // #jai upstream calls rpmalloc_thread_finalize, which reads its _Thread_local heap. Here the
        // key itself holds the heap, and POSIX clears it before calling the destructor, so
        // thread_finalize would find no heap and release nothing (every exiting POSIX thread leaked
        // its heap: RET-04/RET-05 on macOS). Release the heap passed in, and leave the key alone:
        // storing the default heap would make POSIX run the destructor again.
        heap := cast(*Heap) value;
        if heap && heap != global_heap_default {
            // #jai fix E1: the release drains the frees pending in the heap, and frees take the owner's
            // path only when the heap is the thread heap. POSIX has cleared the key, so set it for the
            // release, and clear it again after (POSIX runs the destructor again for a non-null value).
            // Windows still has the value here.
            #if OS != .WINDOWS  os_tls_set(heap);
            heap_release_on_thread_exit(heap);   // #jai fix U5
            #if OS != .WINDOWS  os_tls_set(null);
        } else if heap == global_heap_default {
            heap_release_parked_on_exit();   // #jai fix E7: the thread exits with its heap parked.
        }
    }
}

#if ENABLE_LEAK_DETECTION {

// Size class blocks still allocated across every heap. Non-zero once all threads have finalized
// means the application has not freed all its allocations (a leak or a double free).
statistics_outstanding_blocks :: () -> s64 {   // 2.0.1: rpmalloc.c:2975
    lists := *Heap.[global_heap_used, global_heap_queue, global_heap_pristine];
    outstanding: s64;
    for list: lists {
        heap := list;
        while heap {
            for heap.size_use  outstanding += it.alloc_current;
            heap = heap.next;
        }
    }
    return outstanding;
}

// Cross-thread freed blocks parked in a span's per page deferred lists, over every initialised page
// (a full page is not reachable from page_available but can still hold deferred frees). The block
// count is in the token's high 32 bits.
span_deferred_block_count :: (span: *Span) -> s64 {   // 2.0.1: rpmalloc.c:2992
    deferred: s64;
    for ipage: 0..cast(s64) span.page_initialized - 1 {
        page := cast(*Page) pointer_offset(span, cast(u64) span.page_size * cast(u64) ipage);
        token := atomic_load(*page.thread_free);
        deferred += cast(s64)((token >> 32) & 0x7FFF_FFFF);   // #jai fix PR #426: mask the full flag.
    }
    return deferred;
}

// Blocks freed cross-thread but not yet applied to the owning heap's alloc_current. Subtracting
// them from the outstanding count gives the blocks the application has really leaked.
statistics_pending_deferred_blocks :: () -> s64 {   // 2.0.1: rpmalloc.c:3006
    lists := *Heap.[global_heap_used, global_heap_queue, global_heap_pristine];
    deferred: s64;
    for list: lists {
        heap := list;
        while heap {
            // Page types small to large carry the heap's full page deferred list and the per page
            // deferred lists of every span. Huge blocks are never deferred per block.
            for itype: 0..3 {
                block := cast(*Block) atomic_load(*heap.thread_free[itype]);
                while block {
                    deferred += 1;
                    block = block.next;
                }
                if heap.span_partial[itype]  deferred += span_deferred_block_count(heap.span_partial[itype]);
                span := heap.span_used[itype];
                while span {
                    deferred += span_deferred_block_count(span);
                    span = span.next;
                }
            }
            heap = heap.next;
        }
    }
    return deferred;
}

}   // #if ENABLE_LEAK_DETECTION

//
// Test hooks (TEST_HOOKS): fault injection inside the OS layer, so tests exercise the real
// error paths, plus direct access to the OS entry points for the white-box tests.
//

#if TEST_HOOKS {
    test_fail_maps:    s64;   // The next N os_mmap calls fail.
    test_fail_commits: s64;   // The next N os_mcommit calls fail.
    test_pages_decommitted: [4] s64;   // Pages decommitted / recommitted so far, per page type.
    test_before_mark_full: (page: *void);   // Called just before the owner marks a page full.
    test_page_reinits: s64;                 // Pages (re)initialised for a size class (#413).
    test_pages_recommitted: [4] s64;

    test_take_fault :: (counter: *s64) -> bool {
        while true {
            n := atomic_load(counter);
            if n <= 0  return false;
            if compare_and_swap(counter, n, n - 1)  return true;
        }
        return false;
    }
}

// The count * size product of the calloc family; ok is false for a refused request.
// #jai fix #417 (as upstream develop 06b18dd0): the product is checked whatever ENABLE_VALIDATE_ARGS
// says. Unchecked, calloc(2^33, 2^33) wraps to 0 and returns a small block.
calloc_total :: inline (num: s64, size: s64) -> total: u64, ok: bool {
    n := cast,no_check(u64) num;
    s := cast,no_check(u64) size;
    if n && s > 0xFFFF_FFFF_FFFF_FFFF / n  return 0, false;   // 2.0.1: SizeTMult / __builtin_umull_overflow
    total := n * s;
    if total >= MAX_ALLOC_SIZE  return 0, false;
    return total, true;
}

// #jai upstream PR #423 (951f903): a block from a page without aligned blocks that still fits the
// size is kept as it is, without the call to heap_reallocate_block.
realloc_fits_in_place :: inline (ptr: *void, size: u64) -> bool {
    if !ptr  return false;
    span := block_get_span(ptr);
    if span.page_type > .LARGE  return false;
    page := span_get_page_from_block(span, ptr);
    return !page_has(page, .HAS_ALIGNED_BLOCK) && cast(u64) page.block_size >= size;
}

// Jai Allocator protocol helpers used by rpmalloc_allocator_proc (module.jai).
// RESIZE passes the caller's old_size through to heap_reallocate_block (2.0.1: rpmalloc.c:2389).
allocator_proc_resize :: (heap: *Heap, ptr: *void, size: s64, old_size: s64) -> *void {
    #if ENABLE_VALIDATE_ARGS {
        if cast,no_check(u64) size >= MAX_ALLOC_SIZE  return null;   // #jai null, not ptr (upstream L6, see rprealloc).
    }
    // #jai PR #423: the Allocator protocol's RESIZE is this port's realloc, so it gets the fast path.
    if realloc_fits_in_place(ptr, cast,no_check(u64) size)  return ptr;
    if !heap  heap = get_thread_heap();
    // #jai the protocol's old_size goes through (upstream's rprealloc passes 0 and derives the block's
    // whole capacity), so a moved block copies only the caller's live bytes. Safe since the U12 fix.
    live := ifx old_size > 0 then cast(u64) old_size else 0;
    return heap_reallocate_block(heap, ptr, cast,no_check(u64) size, live, 0);
}

#scope_export

#if TEST_HOOKS {
    test_fail_next_maps    :: (count: s64) { atomic_store(*test_fail_maps, count); }
    test_fail_next_commits :: (count: s64) { atomic_store(*test_fail_commits, count); }
    test_os_page_size      :: () -> u64 { os_initialize(); return os_page_size; }
    test_os_map_granularity :: () -> u64 { os_initialize(); return os_map_granularity; }
    test_os_mmap      :: (size: u64, alignment: u64, offset: *u64, mapped_size: *u64) -> *void { return os_mmap(size, alignment, offset, mapped_size); }
    test_os_mcommit   :: (address: *void, size: u64) -> s32 { return os_mcommit(address, size); }
    test_os_mdecommit :: (address: *void, size: u64) -> s32 { return os_mdecommit(address, size); }
    test_os_munmap    :: (address: *void, offset: u64, mapped_size: u64) { os_munmap(address, offset, mapped_size); }
    test_os_commit_on_demand :: () -> bool { return os_commit_on_demand(); }

    // Size classes and geometry.
    test_size_class_count      :: () -> s64 { return SIZE_CLASS_COUNT; }
    test_size_class_block_size  :: (index: s64) -> u32 { return global_size_class[index].block_size; }
    test_size_class_block_count :: (index: s64) -> u32 { return global_size_class[index].block_count; }
    test_get_size_class      :: (size: u64) -> u32 { return get_size_class(size); }
    test_get_size_class_tiny :: (size: u64) -> u32 { return get_size_class_tiny(size); }
    test_get_page_type       :: (size_class: u32) -> u32 { return cast(u32) get_page_type(size_class); }
    test_page_size_of_type   :: (page_type: u32) -> u64 {
        if page_type == {
          case 0;  return SMALL_PAGE_SIZE;
          case 1;  return MEDIUM_SMALL_PAGE_SIZE;
          case 2;  return MEDIUM_LARGE_PAGE_SIZE;
          case 3;  return LARGE_PAGE_SIZE;
        }
        return 0;
    }
    test_span_size        :: () -> u64 { return SPAN_SIZE; }
    test_page_header_size :: () -> u64 { return PAGE_HEADER_SIZE; }
    test_msb_index        :: (x: u64) -> u32 { return msb_index(x); }
    test_usable_huge_page_size :: (reported: u64) -> u64 { return usable_huge_page_size(reported); }
    test_huge_cache_purge_aged :: (now: u64) { #if HUGE_CACHE_SLOT_COUNT  huge_cache_purge_aged(now); }
    // The huge cache now: committed bytes, committed entries, decommitted entries.
    test_huge_cache_state :: () -> committed: u64, committed_entries: s64, decommitted_entries: s64 {
        #if HUGE_CACHE_SLOT_COUNT {
            committed_entries, decommitted_entries: s64;
            huge_cache_lock_acquire();
            for global_huge_cache.slot {
                if !it.span  continue;
                if it.decommitted  decommitted_entries += 1;
                else               committed_entries += 1;
            }
            committed := global_huge_cache.committed;
            huge_cache_lock_release();
            return committed, committed_entries, decommitted_entries;
        } else {
            return 0, 0, 0;
        }
    }
    test_heap_stat_add    :: (field: *u64, value: u64) { heap_stat_add(field, value); }
    test_heap_stat_sub    :: (field: *u64, value: u64) { heap_stat_sub(field, value); }

    // Ownership: would freeing this block from the calling thread take the local (unsynchronised) path?
    test_block_is_thread_local :: (block: *void) -> bool {
        span := block_get_span(block);
        return page_is_thread_heap(span_get_page_from_block(span, block));
    }

    // The thread-free race of PR #426: a callback between "no cross-thread frees" and "mark full",
    // and the resulting page state.
    test_set_before_mark_full :: (callback: (page: *void)) { test_before_mark_full = callback; }
    test_page_full_state :: (block: *void) -> is_full: bool, pending_cross_thread_frees: u32 {
        span := block_get_span(block);
        page := span_get_page_from_block(span, block);
        unused: *Block;
        return page_has(page, .IS_FULL), page_block_from_thread_free_list(page, atomic_load(*page.thread_free), *unused);
    }

    // Heap layout (#409): where the fields other threads access end, where the owner's begin.
    test_heap_layout :: () -> remote_end: s64, first_owner_field: s64, heap_size: s64 {
        remote_end := 0;
        first_owner := size_of(Heap);
        for type_info(Heap).members {
            end := it.offset_in_bytes + it.type.runtime_size;
            if it.name == "owner_thread" || it.name == "thread_free" || it.name == "free_lock" {   // free_lock: #jai fix E6
                if end > remote_end  remote_end = end;
            } else if it.name != "remote_pad" {
                if it.offset_in_bytes < first_owner  first_owner = it.offset_in_bytes;
            }
        }
        return remote_end, first_owner, size_of(Heap);
    }

    test_page_reinit_count :: () -> s64 { return atomic_load(*test_page_reinits); }

    test_is_initialized :: () -> bool { return atomic_load(*global_rpmalloc_init_state) == RPMALLOC_INIT_DONE; }
    test_hold_heap_lock :: () { heap_lock_acquire(); }   // Never released: the U22 scenario.

    // Decommit activity.
    test_pages_decommitted_of_type :: (page_type: u32) -> s64 { return atomic_load(*test_pages_decommitted[page_type]); }
    test_pages_recommitted_of_type :: (page_type: u32) -> s64 { return atomic_load(*test_pages_recommitted[page_type]); }
}

// IS_THIS_YOURS (Jai only; no upstream equivalent).
//
// Every span is mapped at a SPAN_SIZE (256 MiB) aligned address, huge blocks included, so one bit
// per possible span base records which bases are live mappings of this allocator: 2^20 bits
// (128 KiB) cover a 48-bit address space. is_this_yours reads a span header only when its bit is
// set, so it never dereferences memory that is not ours (review §5.7). Walking the heaps' span lists
// instead would race with their owners, miss the huge blocks of thread heaps (which no list
// tracks), and follow the stale huge entries of first-class heaps (U3).

SPAN_BITMAP_BITS  :: 1 << (48 - SPAN_SIZE_SHIFT);
SPAN_BITMAP_BYTES :: SPAN_BITMAP_BITS / 8;

global_span_bitmap: *u64;   // Mapped once at the first initialize, kept for the life of the process.

span_bitmap_map :: () {
    if global_span_bitmap  return;
    offset, mapped_size: u64;
    bitmap := os_mmap(SPAN_BITMAP_BYTES, 0, *offset, *mapped_size);
    if bitmap && os_mcommit(bitmap, SPAN_BITMAP_BYTES) != 0  bitmap = null;   // Without it, IS_THIS_YOURS says no.
    global_span_bitmap = bitmap;
    #if ENABLE_STATISTICS {
        // Bookkeeping, not allocator memory: take it back out of the counters os_mmap and
        // os_mcommit just raised (the peaks keep it).
        page_count := mapped_size / global_config.page_size;
        statistics_sub_saturating(*global_counters.page_mapped, page_count);
        if bitmap {
            statistics_sub_saturating(*global_counters.page_commit, SPAN_BITMAP_BYTES / global_config.page_size);
            statistics_sub_saturating(*global_counters.page_active, SPAN_BITMAP_BYTES / global_config.page_size);
        }
    }
}

span_bitmap_slot :: inline (span: *Span) -> word: *u64, bit: u64 {
    index := (cast(u64) span) >> SPAN_SIZE_SHIFT;
    if !global_span_bitmap || index >= SPAN_BITMAP_BITS  return null, 0;   // Not tracked: IS_THIS_YOURS says no.
    return global_span_bitmap + (index >> 6), (cast(u64) 1) << (index & 63);
}

span_register :: (span: *Span) {
    word, bit := span_bitmap_slot(span);
    if word  atomic_or(word, bit);
}

span_unmap :: (span: *Span) {
    word, bit := span_bitmap_slot(span);
    if word  atomic_and(word, ~bit);
    global_memory_interface.memory_unmap(span, span.offset, span.mapped_size);
}

// With a heap: is ptr inside a page of that heap? Without (the default allocator): is it inside a
// page of any thread heap? Blocks of first-class heaps are not the default allocator's. A pointer
// into a free part of an owned page counts as owned: this is about address ranges, not live blocks.
// Asking about a huge block while another thread frees it is a use after free by the caller.
is_this_yours :: (heap: *Heap, ptr: *void) -> bool {
    if !ptr || atomic_load(*global_rpmalloc_init_state) != RPMALLOC_INIT_DONE  return false;
    span := cast(*Span)(cast(u64) ptr & SPAN_MASK);
    word, bit := span_bitmap_slot(span);
    if !word || !(atomic_load(word) & bit)  return false;

    address := cast(u64) ptr;
    page: *Page;
    if span.page_type == .HUGE {
        if address < cast(u64) span + SPAN_HEADER_SIZE || address >= cast(u64) span + cast(u64) span.page_count * span.page_size
            return false;
        page = *span.page;
    } else {
        if address >= cast(u64) span + cast(u64) atomic_load(*span.page_initialized) * span.page_size  return false;
        page = span_get_page_from_block(span, ptr);
        if address < cast(u64) page + page_block_start_offset(page)  return false;   // #jai PR #424
    }
    owner := page.heap;
    if !owner  return false;   // A cached huge mapping.
    if heap  return owner == heap;
    return !owner.is_first_class;
}

//
// Extern interface (2.0.1: rpmalloc.c:2510-2660).
//
// #jai refused arguments return null (or the documented error code) without setting errno: Jai
// code has no errno convention, and on Windows it would mean linking the C runtime.
//


// Sizes are s64 at the Jai API (the Allocator protocol's type) and size_t (u64) inside, like C.

rpmalloc :: (size: s64) -> *void {   // 2.0.1: rpmalloc.c:2522
    #if ENABLE_VALIDATE_ARGS {
        if cast,no_check(u64) size >= MAX_ALLOC_SIZE  return null;
    }
    heap := get_thread_heap();
    return heap_allocate_block(heap, cast,no_check(u64) size, false);
}

rpzalloc :: (size: s64) -> *void {   // 2.0.1: rpmalloc.c:2534
    #if ENABLE_VALIDATE_ARGS {
        if cast,no_check(u64) size >= MAX_ALLOC_SIZE  return null;
    }
    heap := get_thread_heap();
    return heap_allocate_block(heap, cast,no_check(u64) size, true);
}

rpfree :: (ptr: *void) {   // 2.0.1: rpmalloc.c:2546
    if !ptr  return;
    block_deallocate(ptr);
}
rpcalloc :: (num: s64, size: s64) -> *void {   // 2.0.1: rpmalloc.c:2553
    total, ok := calloc_total(num, size);
    if !ok  return null;
    heap := get_thread_heap();
    return heap_allocate_block(heap, total, true);
}

rprealloc :: (ptr: *void, size: s64) -> *void {   // 2.0.1: rpmalloc.c:2577
    #if ENABLE_VALIDATE_ARGS {
        // UPSTREAM-BUG L6: returns the old pointer for a refused size, which callers read as success.
        if cast,no_check(u64) size >= MAX_ALLOC_SIZE  return ptr;
    }
    if realloc_fits_in_place(ptr, cast,no_check(u64) size)  return ptr;   // #jai PR #423
    heap := get_thread_heap();
    return heap_reallocate_block(heap, ptr, cast,no_check(u64) size, 0, 0);
}

rpaligned_realloc :: (ptr: *void, alignment: s64, size: s64, old_size: s64, flags: Realloc_Flags = 0) -> *void {   // 2.0.1: rpmalloc.c:2589
    #if ENABLE_VALIDATE_ARGS {
        // Upstream refuses alignments above SMALL_PAGE_SIZE here, while rpaligned_alloc accepts
        // anything below RPMALLOC_MAX_ALIGNMENT (256 KiB).
        // #jai upstream's size + alignment < size, without the wrapping sum (arithmetic_overflow_check).
        if cast,no_check(u64) size > 0xFFFF_FFFF_FFFF_FFFF - cast,no_check(u64) alignment || cast,no_check(u64) alignment > SMALL_PAGE_SIZE
            return null;
    }
    heap := get_thread_heap();
    return heap_reallocate_block_aligned(heap, ptr, cast,no_check(u64) alignment, cast,no_check(u64) size, cast,no_check(u64) old_size, flags);
}

rpaligned_alloc :: (alignment: s64, size: s64) -> *void {   // 2.0.1: rpmalloc.c:2601
    heap := get_thread_heap();
    return heap_allocate_block_aligned(heap, cast,no_check(u64) alignment, cast,no_check(u64) size, false);
}

rpaligned_zalloc :: (alignment: s64, size: s64) -> *void {   // 2.0.1: rpmalloc.c:2607
    heap := get_thread_heap();
    return heap_allocate_block_aligned(heap, cast,no_check(u64) alignment, cast,no_check(u64) size, true);
}

rpaligned_calloc :: (alignment: s64, num: s64, size: s64) -> *void {   // 2.0.1: rpmalloc.c:2613
    total, ok := calloc_total(num, size);
    if !ok  return null;
    heap := get_thread_heap();
    return heap_allocate_block_aligned(heap, cast,no_check(u64) alignment, total, true);
}

rpmemalign :: (alignment: s64, size: s64) -> *void {   // 2.0.1: rpmalloc.c:2637
    heap := get_thread_heap();
    return heap_allocate_block_aligned(heap, cast,no_check(u64) alignment, cast,no_check(u64) size, false);
}

// Returns 0, EINVAL (memptr is null) or ENOMEM, like posix_memalign.
rpposix_memalign :: (memptr: **void, alignment: s64, size: s64) -> s32 {   // 2.0.1: rpmalloc.c:2643
    EINVAL :: 22;
    ENOMEM :: 12;
    heap := get_thread_heap();
    if !memptr  return EINVAL;
    memptr.* = heap_allocate_block_aligned(heap, cast,no_check(u64) alignment, cast,no_check(u64) size, false);
    return ifx memptr.* then 0 else ENOMEM;
}

usable_size :: (ptr: *void) -> s64 {   // 2.0.1: rpmalloc.c:2653
    return ifx ptr then cast(s64) block_usable_size(ptr) else 0;
}

//
// Initialization and finalization (2.0.1: rpmalloc.c:2661-3421).
//

// #jai fix U10: upstream's initialize_config wrote global_config before initialize claimed the init
// state, so it could race a concurrent lazy initialisation. The config is now applied only by the
// thread that claims the state; if the allocator is already initialised it is not applied, as
// upstream.
initialize_config :: (memory_interface: *Memory_Interface, config: *Config) -> s32 {   // 2.0.1: rpmalloc.c:2704
    result := initialize_with_config(memory_interface, config);
    if config  config.* = global_config;
    return result;
}

initialize :: (memory_interface: *Memory_Interface = null) -> s32 {   // 2.0.1: rpmalloc.c:2724
    return initialize_with_config(memory_interface, null);
}

#scope_module

initialize_with_config :: (memory_interface: *Memory_Interface, config: *Config) -> s32 {
    while true {
        state := atomic_load(*global_rpmalloc_init_state);
        if state == RPMALLOC_INIT_DONE {
            thread_initialize();
            return 0;
        }
        if state == RPMALLOC_INIT_RUNNING {
            while atomic_load(*global_rpmalloc_init_state) == RPMALLOC_INIT_RUNNING  thread_yield();
            continue;
        }
        if compare_and_swap(*global_rpmalloc_init_state, RPMALLOC_INIT_UNINIT, RPMALLOC_INIT_RUNNING)  break;
    }
    if config  global_config = config.*;   // #jai fix U10: only after claiming the init state.

    // Remember whether the caller asked for huge pages; detection below overwrites
    // global_config.enable_huge_pages with what is actually available.
    huge_pages_requested := global_config.enable_huge_pages;

    // #jai fix U8: upstream installed the OS defaults only when map or unmap was missing, so an
    // interface with map and unmap but no commit / decommit was called through null; and it wrote
    // the defaults into the caller's struct. Now the caller's interface is copied, and a null commit
    // or decommit falls back to the OS. Without map or unmap, all four are the OS's, as upstream.
    if memory_interface {
        global_memory_interface_copy = memory_interface.*;
        global_memory_interface = *global_memory_interface_copy;
        if !global_memory_interface.memory_commit    global_memory_interface.memory_commit   = os_mcommit;
        if !global_memory_interface.memory_decommit  global_memory_interface.memory_decommit = os_mdecommit;
    } else {
        global_memory_interface = *global_memory_interface_default;
    }
    if !global_memory_interface.memory_map || !global_memory_interface.memory_unmap {
        global_memory_interface.memory_map      = os_mmap;
        global_memory_interface.memory_commit   = os_mcommit;
        global_memory_interface.memory_decommit = os_mdecommit;
        global_memory_interface.memory_unmap    = os_munmap;
    }
    // (ENABLE_OVERRIDE's unmap_on_finalize override is C-only; not ported.)

    // 2.0.1: rpmalloc.c:2765-2785. #jai os_initialize also serves the test hooks, so force a fresh
    // read here, as upstream does on every initialisation.
    os_page_size = 0;
    os_initialize();
    os_huge_pages = false;
    os_thp = false;
    os_huge_page_shift = 0;

    rpmalloc_assert(!global_config.page_size || !(global_config.page_size & (global_config.page_size - 1)), "Configured page size must be a power of two");   // #jai 0 is valid (the OS page size), and 0 - 1 would wrap

    // Establish a valid (normal) page size up front, so the allocator never sees a zero page size
    // during the rest of initialisation. Huge page detection may raise it.
    if !memory_interface || global_config.page_size < os_page_size  global_config.page_size = os_page_size;

    if global_config.enable_huge_pages  os_detect_huge_pages();   // 2.0.1: rpmalloc.c:2797-2893

    if huge_pages_requested && !os_huge_pages {
        // Huge pages were requested but are not available: fail rather than silently use normal
        // pages behind a huge page size. Leave the allocator uninitialised (and the flag cleared)
        // so the caller can retry without huge pages.
        global_config.enable_huge_pages = false;
        atomic_store(*global_rpmalloc_init_state, RPMALLOC_INIT_UNINIT);
        return -1;
    }

    global_config.enable_huge_pages = os_huge_pages;

    if os_huge_pages {
        while (cast(u64) 1 << os_huge_page_shift) < os_page_size  os_huge_page_shift += 1;
    }

    if !memory_interface || global_config.page_size < os_page_size  global_config.page_size = os_page_size;

    #if OS == .WINDOWS {
        // Large pages on Windows must be committed when mapped and can never be decommitted.
        if global_config.enable_huge_pages  global_config.disable_decommit = true;
    }

    os_configure_thp();   // 2.0.1: rpmalloc.c:2921-2936
    global_config.enable_thp = os_thp;

    // Decommit of free pages only applies to page types larger than the configured page size:
    // disable the free page overflow pass for page types where it can never apply (always the case
    // with huge pages or a large page size).
    page_free_overflow_default :: u32.[16, 8, 4, 2];
    builtin_page_size :: u64.[SMALL_PAGE_SIZE, MEDIUM_SMALL_PAGE_SIZE, MEDIUM_LARGE_PAGE_SIZE, LARGE_PAGE_SIZE];
    for itype: 0..3 {
        page_can_decommit := !global_config.disable_decommit && builtin_page_size[itype] > global_config.page_size;
        global_page_free_overflow[itype] = ifx page_can_decommit then page_free_overflow_default[itype] else 0xFFFF_FFFF;
    }

    span_bitmap_map();   // #jai for IS_THIS_YOURS; needs the final page size (statistics).

    os_tls_create(rpmalloc_thread_destructor);   // 2.0.1: rpmalloc.c:2951-2955

    global_main_thread_id = get_thread_id();

    atomic_store(*global_rpmalloc_init_state, RPMALLOC_INIT_DONE);

    thread_initialize();
    return 0;
}

#scope_export

config :: () -> *Config {   // 2.0.1: rpmalloc.c:2967
    return *global_config;
}

finalize :: () {   // 2.0.1: rpmalloc.c:3031
    thread_finalize();

    #if HUGE_CACHE_SLOT_COUNT  huge_cache_flush();   // 2.0.1: rpmalloc.c:3037
    #if ENABLE_LEAK_DETECTION && ENABLE_ASSERTS {   // 2.0.1: rpmalloc.c:3040-3047
        // Every thread has finalized, so blocks still allocated and not merely waiting in a deferred
        // cross-thread free list are blocks the application never freed (or freed twice).
        rpmalloc_assert(statistics_outstanding_blocks() <= statistics_pending_deferred_blocks(), "Memory leak detected");
        rpmalloc_assert(atomic_load(*global_counters.huge_alloc) == 0, "Memory leak detected");
    }

    if global_config.unmap_on_finalize {
        // Heaps can share a mapping (heap_allocate_new); only the owner has mapped_size != 0.
        // Resources are freed while every heap is still mapped, then owners are unmapped in a
        // separate pass, so no heap is touched after its shared mapping is gone.
        owners: *Heap;
        collect :: (list: **Heap) #expand {
            heap := list.*;
            list.* = null;
            while heap {
                heap_next := heap.next;
                heap_unmap_all(heap);
                if heap.mapped_size {
                    heap.next = `owners;
                    `owners = heap;
                }
                heap = heap_next;
            }
        }
        collect(*global_heap_pristine);
        collect(*global_heap_queue);
        collect(*global_heap_used);
        while owners {
            owner_next := owners.next;
            heap_unmap(owners);
            owners = owner_next;
        }
        #if ENABLE_STATISTICS  memset(*global_counters, 0, size_of(Rpmalloc_Statistics));   // 2.0.1: rpmalloc.c:3094
    }

    os_tls_destroy();   // 2.0.1: rpmalloc.c:3098-3104

    // #jai every span is unmapped or abandoned now: forget them all (IS_THIS_YOURS).
    if global_span_bitmap  memset(global_span_bitmap, 0, SPAN_BITMAP_BYTES);

    global_heap_creating = 0;
    global_main_thread_id = 0;
    atomic_store(*global_rpmalloc_init_state, RPMALLOC_INIT_UNINIT);
}

thread_initialize :: () {   // 2.0.1: rpmalloc.c:3112
    if get_thread_heap() == global_heap_default  get_thread_heap_allocate();
}

thread_finalize :: () {   // 2.0.1: rpmalloc.c:3118
    heap := get_thread_heap();
    if heap != global_heap_default {
        heap_release_on_thread_exit(heap);   // #jai fix U5
        set_thread_heap(global_heap_default);
    } else {
        heap_release_parked_on_exit();   // #jai fix E7
    }
}

thread_collect :: () {   // 2.0.1: rpmalloc.c:3127 (empty in 2.0.1; develop #416 fills it in)
    heap := get_thread_heap();
    if !heap || !heap.id  return;
    // #jai fix U23: idle huge cache entries were unmapped only when another huge block was freed, so a
    // program that stopped using huge blocks kept up to 128 MiB committed for good (RET-08).
    #if HUGE_CACHE_SLOT_COUNT  huge_cache_purge_aged(os_monotonic_time_ms());
    heap_drain_deferred_frees(heap);   // #jai fix U6
    heap_retire_kept_pages(heap);   // #jai fix (U6 part, with #413)
    // #jai develop #416: decommit the free pages held above the retain count now, instead of
    // waiting for the decommit delay.
    for itype: 0..3 {
        if global_page_free_overflow[itype] == 0xFFFF_FFFF  continue;
        heap.page_free_overflow_ms[itype] = 0;
        if heap.page_free_commit_count[itype] > global_page_free_retain[itype]
            heap_page_free_decommit(heap, cast(u32) itype, global_page_free_retain[itype]);
    }
}

// #jai (upstream issue #116): returns as much memory as possible to the OS now. thread_collect keeps
// the usual retain counts per page type and the young huge cache entries; trim also decommits every
// free page of the calling thread's heap and unmaps the whole huge cache. Pages that hold live blocks
// stay, and other threads' heaps are not touched (each thread can call trim itself).
trim :: () {
    heap := get_thread_heap();
    if heap && heap.id {
        heap_drain_deferred_frees(heap);
        heap_retire_kept_pages(heap);
        for itype: 0..3 {
            heap.page_free_overflow_ms[itype] = 0;
            if heap.page_free_commit_count[itype]  heap_page_free_decommit(heap, cast(u32) itype, 0);
        }
    }
    #if HUGE_CACHE_SLOT_COUNT  huge_cache_flush();
}

// #jai fix E7 (Ex4 1.45.0, items 1 and 3 of its analysis): frees by other threads into a heap queue up
// for its owner, which takes them in only on its next allocation, and a heap keeps some free pages
// committed for reuse. A pool thread that goes idle makes no allocator call, possibly for a long time,
// so its heap kept the pages of every block other threads freed into it, and its free pages: 185 MB
// and 35 MB in Ex4, with 24 idle walk workers (RET-20). Only the thread knows it is going idle, so it
// says so: thread_idle gives back what it can, as trim() does for its heap (pending frees taken in,
// kept empty pages retired, every free page decommitted, aged huge cache entries decommitted), and
// parks the heap. While parked, frees by other threads into it are freed at once under the heap's
// lock, as into an exited thread's heap (E4, E6), so their pages are decommitted as they empty. The
// thread's next allocator call takes the heap back. Call it where a pool thread waits for work; it
// costs a few OS calls, and the first allocation after it takes the slow path once.
thread_idle :: () {
    heap := get_thread_heap();
    if !heap || !heap.id || heap.is_first_class  return;   // No thread heap (or a first-class one made current).
    heap_drain_deferred_frees(heap);
    heap_retire_kept_pages(heap);
    for itype: 0..3 {
        heap.page_free_overflow_ms[itype] = 0;
        if heap.page_free_commit_count[itype]  heap_page_free_decommit(heap, cast(u32) itype, 0);
    }
    #if HUGE_CACHE_SLOT_COUNT  huge_cache_purge_aged(os_monotonic_time_ms());
    me := get_thread_id();
    heap_lock_acquire();
    heap_free_lock_acquire(heap);
    heap.parked_thread = me;
    heap.owner_thread = HEAP_RELEASED_OWNER;   // Frees by any thread take the released-heap path.
    heap_free_lock_release(heap);
    atomic_add(*global_parked_count, 1);
    heap_lock_release();
    // The fallback heap: the fast paths find nothing, and the next call unparks (heap_take_parked).
    // Non-null, so the POSIX thread exit destructor still runs and releases the parked heap.
    heap.thread_current = false;   // #jai fix E10: no Context cache may hand it out while parked.
    os_tls_set(global_heap_default);
}

#scope_module

// A thread that exits (or sends THREAD_STOP) with its heap parked: the heap goes to the queue for
// reuse. It was trimmed when it was parked, and frees since were applied at once.
heap_release_parked_on_exit :: () {
    heap := heap_take_parked();
    if !heap  return;
    heap_retire_kept_pages(heap);
    for itype: 0..3 {
        heap.page_free_overflow_ms[itype] = 0;
        if heap.page_free_commit_count[itype]  heap_page_free_decommit(heap, cast(u32) itype, 0);
    }
    heap_release_to_queue(heap);
}

#scope_export

is_thread_initialized :: () -> bool {   // 2.0.1: rpmalloc.c:2517
    return get_thread_heap() != global_heap_default;
}

thread_statistics :: (stats: *Thread_Statistics) {   // 2.0.1: rpmalloc.c:3131
    memset(stats, 0, size_of(Thread_Statistics));
    #if ENABLE_STATISTICS {
        heap := get_thread_heap();
        if !heap || heap.id == 0  return;

        // Size class cache: blocks available without mapping a new page. A page's block_used counts
        // the blocks held on the heap free list, so those are counted from the heap list, and each
        // page adds its remaining blocks (its free list plus the not yet initialised tail).
        for iclass: 0..SIZE_CLASS_COUNT - 1 {
            block_size := cast(u64) global_size_class[iclass].block_size;
            free_count: u64;
            block := heap.local_free[iclass];
            while block {
                free_count += 1;
                block = block.next;
            }
            page := heap.page_available[iclass];
            while page {
                free_count += cast(u64)(page.block_count - page.block_used);
                page = page.next;
            }
            stats.sizecache += free_count * block_size;

            su := *heap.size_use[iclass];
            // #jai fix U13: a transiently negative count (frees counted before the matching
            // allocation is adopted) reads as 0, not as a huge size_t.
            stats.size_use[iclass].alloc_current = cast(u64) max(su.alloc_current, 0);
            stats.size_use[iclass].alloc_peak    = cast(u64) max(su.alloc_peak, 0);
            stats.size_use[iclass].alloc_total   = cast(u64) max(su.alloc_total, 0);
            stats.size_use[iclass].free_total    = cast(u64) max(su.free_total, 0);
        }

        // Span cache: free but still committed pages kept per page type.
        page_type_size :: u64.[SMALL_PAGE_SIZE, MEDIUM_SMALL_PAGE_SIZE, MEDIUM_LARGE_PAGE_SIZE, LARGE_PAGE_SIZE];
        for itype: 0..3  stats.spancache += cast(u64) heap.page_free_commit_count[itype] * page_type_size[itype];

        // Span use per page type: current spans counted live (partial plus full); map calls counted.
        for itype: 0..4 {
            current: u64;
            if itype < 4 && heap.span_partial[itype]  current += 1;
            span := heap.span_used[itype];
            while span {
                current += 1;
                span = span.next;
            }
            stats.span_use[itype].current = current;
            stats.span_use[itype].map_calls = heap.span_map_calls[itype];
        }
    }
}

// #jai appends to a String_Builder instead of writing to a FILE*.
dump_statistics :: (builder: *String_Builder) {   // 2.0.1: rpmalloc.c:3176
    #if ENABLE_STATISTICS {
        g := *global_counters;
        print_to_builder(builder, "Mapped pages:        %\n", atomic_load(*g.page_mapped));
        print_to_builder(builder, "Mapped pages (peak): %\n", atomic_load(*g.page_mapped_peak));
        print_to_builder(builder, "Active pages:        %\n", atomic_load(*g.page_active));
        print_to_builder(builder, "Active pages (peak): %\n", atomic_load(*g.page_active_peak));
        print_to_builder(builder, "Pages committed:     %\n", atomic_load(*g.page_commit));
        print_to_builder(builder, "Pages decommitted:   %\n", atomic_load(*g.page_decommit));
        print_to_builder(builder, "Huge bytes:          %\n", atomic_load(*g.huge_alloc));
        print_to_builder(builder, "Huge bytes (peak):   %\n", atomic_load(*g.huge_alloc_peak));
        print_to_builder(builder, "Heaps in use:        %\n", atomic_load(*g.heap_count));
        #if ENABLE_LEAK_DETECTION
            print_to_builder(builder, "Outstanding blocks:  %\n", statistics_outstanding_blocks() - statistics_pending_deferred_blocks());

        // Aggregate the per size class and per page type counters across all in-use heaps. This
        // walks live heaps without stopping the world, so the values are a best-effort snapshot.
        size_use: [SIZE_CLASS_COUNT] Heap_Size_Use;
        span_map_calls: [5] u64;
        heap_lock_acquire();
        heap := global_heap_used;
        while heap {
            for iclass: 0..SIZE_CLASS_COUNT - 1 {
                size_use[iclass].alloc_current += heap.size_use[iclass].alloc_current;
                size_use[iclass].alloc_peak    += heap.size_use[iclass].alloc_peak;
                size_use[iclass].alloc_total   += heap.size_use[iclass].alloc_total;
                size_use[iclass].free_total    += heap.size_use[iclass].free_total;
            }
            for itype: 0..4  span_map_calls[itype] += heap.span_map_calls[itype];
            heap = heap.next;
        }
        heap_lock_release();

        print_to_builder(builder, "SizeClass  CurAlloc PeakAlloc  TotAlloc   TotFree BlockSize\n");
        for size_use {
            if !it.alloc_total  continue;
            print_to_builder(builder, "% % % % % %\n", formatInt(it_index, minimum_digits = 9, padding = #char " "),
                formatInt(it.alloc_current, minimum_digits = 9, padding = #char " "), formatInt(it.alloc_peak, minimum_digits = 9, padding = #char " "),
                formatInt(it.alloc_total, minimum_digits = 9, padding = #char " "), formatInt(it.free_total, minimum_digits = 9, padding = #char " "),
                formatInt(global_size_class[it_index].block_size, minimum_digits = 9, padding = #char " "));
        }
        print_to_builder(builder, "PageType  MapCalls\n");
        for span_map_calls {
            if !it  continue;
            print_to_builder(builder, "% %\n", formatInt(it_index, minimum_digits = 8, padding = #char " "), formatInt(it, minimum_digits = 9, padding = #char " "));
        }
    }
}

global_statistics :: (stats: *Global_Statistics) {   // 2.0.1: rpmalloc.c:3240
    #if ENABLE_STATISTICS {
        g := *global_counters;
        page_size := global_config.page_size;
        stats.mapped          = page_size * atomic_load(*g.page_mapped);
        stats.mapped_peak     = page_size * atomic_load(*g.page_mapped_peak);
        stats.committed       = page_size * atomic_load(*g.page_commit);
        stats.decommitted     = page_size * atomic_load(*g.page_decommit);
        stats.active          = page_size * atomic_load(*g.page_active);
        stats.active_peak     = page_size * atomic_load(*g.page_active_peak);
        stats.huge_alloc      = atomic_load(*g.huge_alloc);
        stats.huge_alloc_peak = atomic_load(*g.huge_alloc_peak);
        stats.heap_count      = atomic_load(*g.heap_count);
    } else {
        memset(stats, 0, size_of(Global_Statistics));
    }
}

#if FIRST_CLASS_HEAPS {

// First-class heaps (2.0.1: rpmalloc.c:3255-3418). A first-class heap has no owner thread
// (owner_thread 0): only one thread may use it at a time, and heap_free_all releases every block
// it ever handed out.

heap_acquire :: () -> *Heap {   // 2.0.1: rpmalloc.c:3259
    // Must be a pristine heap, or else blocks could already be allocated from it, which
    // heap_free_all would (wrongly) release. heap_allocate(true) returns a never-used heap carved
    // from a shared block, or a freshly mapped one (including heaps released empty, #jai fix U20).
    heap := heap_allocate(true);
    // #jai fix U9: upstream's rpmalloc_assume(heap != 0) turned out-of-memory into a null
    // dereference. Return null, as CREATE_HEAP's protocol expects.
    if !heap  return null;
    heap.owner_thread = 0;
    return heap;
}

heap_release :: (heap: *Heap) {   // 2.0.1: rpmalloc.c:3271
    // #jai fix U14: see heap_release_to_queue. The released flag is checked again under the lock.
    if heap && heap != global_heap_default && !heap.released {   // #jai fix U2: never the fallback heap.
        if heap_is_empty(heap)  heap_release_to_pristine(heap);   // #jai fix U20
        else                    heap_release_trim(heap);           // #jai develop #418, #419
    }
}

heap_alloc :: (heap: *Heap, size: s64) -> *void {   // 2.0.1: rpmalloc.c:3277
    #if ENABLE_VALIDATE_ARGS {
        if cast,no_check(u64) size >= MAX_ALLOC_SIZE  return null;
    }
    return heap_allocate_block(heap, cast,no_check(u64) size, false);
}

heap_aligned_alloc :: (heap: *Heap, alignment: s64, size: s64) -> *void {   // 2.0.1: rpmalloc.c:3288
    #if ENABLE_VALIDATE_ARGS {
        if cast,no_check(u64) size >= MAX_ALLOC_SIZE  return null;
    }
    return heap_allocate_block_aligned(heap, cast,no_check(u64) alignment, cast,no_check(u64) size, false);
}

heap_aligned_zalloc :: (heap: *Heap, alignment: s64, size: s64) -> *void {   // 2.0.1: rpmalloc.c:3299
    #if ENABLE_VALIDATE_ARGS {
        if cast,no_check(u64) size >= MAX_ALLOC_SIZE  return null;
    }
    return heap_allocate_block_aligned(heap, cast,no_check(u64) alignment, cast,no_check(u64) size, true);
}

heap_calloc :: (heap: *Heap, num: s64, size: s64) -> *void {   // 2.0.1: rpmalloc.c:3310
    total, ok := calloc_total(num, size);
    if !ok  return null;
    return heap_allocate_block(heap, total, true);
}

heap_aligned_calloc :: (heap: *Heap, alignment: s64, num: s64, size: s64) -> *void {   // 2.0.1: rpmalloc.c:3333
    total, ok := calloc_total(num, size);
    if !ok  return null;
    return heap_allocate_block_aligned(heap, cast,no_check(u64) alignment, total, true);
}

heap_realloc :: (heap: *Heap, ptr: *void, size: s64, flags: Realloc_Flags = 0) -> *void {   // 2.0.1: rpmalloc.c:3356
    #if ENABLE_VALIDATE_ARGS {
        // UPSTREAM-BUG L6: returns the old pointer for a refused size, which callers read as success.
        if cast,no_check(u64) size >= MAX_ALLOC_SIZE  return ptr;
    }
    return heap_reallocate_block(heap, ptr, cast,no_check(u64) size, 0, flags);
}

heap_aligned_realloc :: (heap: *Heap, ptr: *void, alignment: s64, size: s64, flags: Realloc_Flags = 0) -> *void {   // 2.0.1: rpmalloc.c:3367
    #if ENABLE_VALIDATE_ARGS {
        // #jai upstream's size + alignment < size, without the wrapping sum (arithmetic_overflow_check).
        if cast,no_check(u64) size > 0xFFFF_FFFF_FFFF_FFFF - cast,no_check(u64) alignment || cast,no_check(u64) alignment > SMALL_PAGE_SIZE
            return null;
    }
    return heap_reallocate_block_aligned(heap, ptr, cast,no_check(u64) alignment, cast,no_check(u64) size, 0, flags);
}

heap_free :: (heap: *Heap, ptr: *void) {   // 2.0.1: rpmalloc.c:3378
    // The heap argument is unused: the block's page knows its heap.
    // #jai fix U18: rpfree checks for null, upstream's heap_free did not and dereferenced 0.
    if !ptr  return;
    // #jai PR #422 compares the page with the given heap, which upstream trusts. A non-owner thread
    // passing the owner's thread heap (heap_free(get_heap_for_ptr(p), p) reads naturally) would then
    // take the owner's unsynchronised local path (HEAP-09). Only a first-class heap, which has no
    // owner thread, or the caller's own thread heap is trusted; anything else frees as rpfree does.
    owned := get_thread_heap();
    if heap && !heap.owner_thread  owned = heap;
    block_deallocate_heap(ptr, owned);
}

// Frees every block the heap has handed out.
heap_free_all :: (heap: *Heap) {   // 2.0.1: rpmalloc.c:3385
    heap_unmap_all(heap);
}

heap_statistics :: (heap: *Heap) -> Heap_Statistics {   // 2.0.1: rpmalloc.c:3390
    #if HEAP_STATISTICS {
        if heap  return heap.stats;
    }
    return .{};
}

heap_thread_set_current :: (heap: *Heap) {   // 2.0.1: rpmalloc.c:3402
    prev_heap := get_thread_heap();
    if prev_heap != heap {
        // set_thread_heap stamps owner_thread on a first-class heap, which makes it a thread heap
        // that the thread's exit releases; a later heap_release of it is ignored (#jai fix U14).
        set_thread_heap(heap);
        // #jai fix U2: a thread that has not allocated yet still has the static fallback heap as its
        // current heap. Upstream queued it for reuse like a real heap, and later new threads were
        // handed it (or, with the queue now inconsistent, a heap another thread owns).
        if prev_heap && prev_heap != global_heap_default  heap_release_to_queue(prev_heap);
    }
}

// The heap that owns the page of `ptr`. Dereferences the span header of any pointer it is given,
// so it is only for pointers known to come from this allocator (IS_THIS_YOURS does not use it).
get_heap_for_ptr :: (ptr: *void) -> *Heap {   // 2.0.1: rpmalloc.c:3412
    span := cast(*Span)(cast(u64) ptr & SPAN_MASK);
    if span  return span_get_page_from_block(span, ptr).heap;
    return null;
}

}   // #if FIRST_CLASS_HEAPS

#scope_module

#import "Basic";
#import "Bit_Operations";
#import "Atomics";   // atomic_add returns the previous value, like C atomic_fetch_add.