From 22cad985003c9bd269b291910564551e4caadcab Mon Sep 17 00:00:00 2001 From: Musa Mahmood Date: Tue, 13 Oct 2026 07:26:42 +0100 Subject: [PATCH] [0.1.15] Startup: a catalog cache, the catalog on the background threads, rpmalloc, and [startup] timings The catalog is kept in .catalog-cache in the library, each entry with the size and modified time of the file it was read from; a start takes the entries whose file is unchanged from it and reads only the rest. It is saved once the catalog is complete and has changed (at most every 30 s) and at shutdown. After a reboot the catalog was complete 145 ms after launch, against 3.5 s reading every file. The catalog loads on the background Thread_Group, every worker of it, instead of the four preview workers. rpmalloc (modules/Default_Allocator, 2.0.1-jai.16) takes the place of Jai's single locked heap: entries from the cache 380 -> 50 ms, and reading the files now scales with threads. thread_idle is called where workers idle. check_targets.jai checks other OSes with Jai's own allocator, as rpmalloc runs at compile time with the target's OS layer. -cold drops the library from the system's file cache first; -no-catalog-cache and -catalog-threads N are for measuring. [startup] log lines time the listing, the first frame, the cache, the first preview and the complete catalog. --- README.md | 4 + check_targets.jai | 22 +- first.jai | 2 +- modules/Default_Allocator/DIFFERENCES.md | 148 + modules/Default_Allocator/LICENSE | 21 + modules/Default_Allocator/LICENSES/MIT.txt | 21 + .../Default_Allocator/LICENSES/Unlicense.txt | 24 + modules/Default_Allocator/README.md | 59 + modules/Default_Allocator/module.jai | 303 ++ modules/Default_Allocator/os_posix.jai | 408 ++ modules/Default_Allocator/os_windows.jai | 334 ++ modules/Default_Allocator/rpmalloc.jai | 3490 +++++++++++++++++ modules/Default_Allocator/version.jai | 48 + src/App.jai | 5 + src/app/background.jai | 85 +- src/app/catalog.jai | 350 +- src/app/main.jai | 9 + src/app/ocr_ui.jai | 4 +- src/app/segment_ui.jai | 2 +- src/app/viewer.jai | 123 +- src/base/platform/platform_linux.jai | 9 + src/base/platform/platform_macos.jai | 8 +- src/base/platform/platform_windows.jai | 12 + src/test/catalog_tests.jai | 78 +- 24 files changed, 5522 insertions(+), 47 deletions(-) create mode 100644 modules/Default_Allocator/DIFFERENCES.md create mode 100644 modules/Default_Allocator/LICENSE create mode 100644 modules/Default_Allocator/LICENSES/MIT.txt create mode 100644 modules/Default_Allocator/LICENSES/Unlicense.txt create mode 100644 modules/Default_Allocator/README.md create mode 100644 modules/Default_Allocator/module.jai create mode 100644 modules/Default_Allocator/os_posix.jai create mode 100644 modules/Default_Allocator/os_windows.jai create mode 100644 modules/Default_Allocator/rpmalloc.jai create mode 100644 modules/Default_Allocator/version.jai diff --git a/README.md b/README.md index 964f5e3..6a3dcb7 100644 --- a/README.md +++ b/README.md @@ -350,6 +350,9 @@ then in that order instead of by date. | `-library FOLDER` | Shows the `.mcap` files in FOLDER (and its folders) instead of `D:\McapIndex\mcap`. | | `-import SOURCE DEST` | Imports the folder SOURCE into the library folder DEST, as the Import window does, then exits: 0, or 1 if any file failed. | | `-search QUERY` | Starts with QUERY in the search box. | +| `-cold` | Loads the library as on a cold start: its files and the catalog cache are dropped from the system's file cache first (Windows and Linux; not macOS, which allows it only with `sudo purge`). The `[startup]` lines in the log time each step. With `-stats`, a startup benchmark: `bin\mcap.exe -cold -stats`. | +| `-no-catalog-cache` | Reads every capture from its file, as without the catalog cache (for measuring); the cache is still written. | +| `-catalog-threads N` | Loads the catalog on N threads instead of all the background group's (for measuring). | | `-scan` | Scans the library (the chosen OCR model, and labels when the label model is up) once it has loaded, then exits: 0, or 1 if any capture failed. | | `-stats` | Logs the Stats window's numbers once the catalog has loaded, then exits; with `-search`, for the matches, and the first ten are logged. | | `-record SECONDS` | Records the screen under the cursor that long (no overlay), puts it in the library, then exits: 0, or 1 if it failed. `-record-region X Y W H` records that region instead. `-record-quit` quits while still recording, to check the recording is kept. `-record-ui-visible` leaves the border and toolbar in screenshots, to look at them. | @@ -373,6 +376,7 @@ scripts and for trying things on a copy of the library: `bin\mcap.exe -library D | Settings (theme, hotkeys, OCR model) | `%APPDATA%\mcap\settings.config` | | Logs | `logs\\` beside the program | | The hash of every capture's image, for finding duplicates | `.import-index` in the library folder | +| The catalog cache: every capture's catalog entry, so a start reads one file instead of every capture (an entry is used while its file's size and modified time are unchanged; deleting it is safe) | `.catalog-cache` in the library folder | | The folder each library folder was imported from | `.import-sources` in the library folder | | Which library folders are segmented | `.library-folders` in the library folder | diff --git a/check_targets.jai b/check_targets.jai index 8eebe97..06c3a2d 100644 --- a/check_targets.jai +++ b/check_targets.jai @@ -20,6 +20,24 @@ options.output_executable_name = "check"; options.output_path = ".build"; + // rpmalloc (modules/Default_Allocator) also runs at compile time, with the target's OS layer, + // which crashes on another OS. A check for another OS uses Jai's own allocator instead, through a + // module of the same name ahead of modules/ (with the thread_idle the app calls, doing nothing). + if target != OS { + jai_modules := trim_right(options.import_path[options.import_path.count - 1], "/"); + shim := tprint("%/.build/check_modules", get_working_directory()); + make_directory_if_it_does_not_exist(tprint("%/Default_Allocator", shim), recursive = true); + write_entire_file(tprint("%/Default_Allocator/module.jai", shim), tprint(#string SHIM +// Written by check_targets.jai: Jai's own allocator, for type-checking another OS. +#load "%/Default_Allocator/module.jai"; +thread_idle :: () {} +SHIM, jai_modules)); + paths: [..] string; + array_add(*paths, shim); + array_add(*paths, ..options.import_path); + options.import_path = paths; + } + w := compiler_create_workspace("check"); set_build_options(options, w); compiler_begin_intercept(w); @@ -29,7 +47,7 @@ APP_NAME :: "mcap"; APP_FOLDER_NAME :: "mcap"; GIT_BRANCH :: "check"; GIT_HASH :: "-"; -VERSION :: "0.1.14"; +VERSION :: "0.1.15"; BUILD_DATE :: "-"; DEBUG :: true; COMPILE_TESTS :: true; @@ -49,3 +67,5 @@ S, w); #import "Basic"; #import "Compiler"; +#import "File"; +#import "String"; diff --git a/first.jai b/first.jai index ad243b7..cdee0a5 100644 --- a/first.jai +++ b/first.jai @@ -8,7 +8,7 @@ APP_NAME :: "mcap"; APP_FOLDER_NAME :: "mcap"; APP_BUNDLE_IDENTIFIER :: "com.example.mcap"; -VERSION :: "0.1.14"; +VERSION :: "0.1.15"; // Where everything goes: bin/, or bin-headless/ for builds made with no_crash_dialog, so automated // builds never write over the ones you run by hand. build_directory := "bin"; diff --git a/modules/Default_Allocator/DIFFERENCES.md b/modules/Default_Allocator/DIFFERENCES.md new file mode 100644 index 0000000..9c8c7e8 --- /dev/null +++ b/modules/Default_Allocator/DIFFERENCES.md @@ -0,0 +1,148 @@ +# How this Default_Allocator differs from Jai's + +This module replaces Jai's `modules/Default_Allocator` with the rpmalloc 2.0.1 port from `my_modules/rpmalloc` (installed with its `install.jai`). Jai finds the default allocator by module name, so the replacement is the directory that comes first on the import path. This document lists everything that behaves differently. Facts about Jai's module refer to Jai beta 0.2.030. + +Everything here is tested by the port's suite, `my_modules/rpmalloc/tests`: +- `jai build.jai - swap` installs this module with `install.jai`, builds a program whose only allocator is the installed copy, runs that program, and type-checks it for macOS and Linux. +- The port's own suite (89 tests) covers the allocator code itself. + +## 1. Which allocator you get + +| Platform | Jai's Default_Allocator | This module | +|---|---|---| +| Windows x64 | Stripped-down rpmalloc 1.4.4: one heap shared by every thread behind one lock, and no thread caches | rpmalloc 2.0.1 (port), one heap per thread (**tested**) | +| Linux x64 | The same as on Windows x64 | The same as on Windows (**type-checked only**; there is no Linux machine) | +| macOS ARM64 | A thin wrapper over the C runtime's `malloc` / `realloc` / `free` | rpmalloc 2.0.1 (port) (**tested**) | +| macOS x64, Linux ARM64 | Linux ARM64: C runtime wrapper; macOS x64: stripped-down rpmalloc 1.4.4 | Builds, but untested | +| PS5, Switch 2, iOS, Android, WASM, 32-bit | Their own branches (rpmalloc 1.4.4 on PS5 and Switch 2; the C runtime wrapper elsewhere) | **Not supported.** The module stops with an `#assert`. Take `my_modules` off the import path for these targets. | + +The CAPS string tells you which allocator is in use: `"rpmalloc 2.0.1 (Jai port) as Default_Allocator"`. Jai's reports `"stripped-down rpmalloc 1.4.4 intended as Default_Allocator"`, or `"Jai wrapper for C’s malloc/free/realloc functions."`. + +## 2. Exported interface + +| Name | Jai's | This module | +|---|---|---| +| `allocator` | Yes (rpmalloc branch only) | Yes | +| `allocator_proc` | Yes | Yes, same signature | +| `unsafe_reset()` | rpmalloc branch: forgets all state and starts over. C runtime branch: does nothing. | Releases every lock and keeps all state (§6) | +| `initialize_if_needed() -> bool` | Yes | Yes | +| `is_initialized() -> bool` | Yes | Yes | +| `finalize()` | Yes | Yes (the port's `finalize`) | +| `usable_size(ptr) -> s64` | Yes | Yes | +| `is_this_yours(heap, ptr) -> bool` | Walks the heap's span lists. It misses older large blocks (finding F4/M2, test HEAP-05) and the blocks of exited threads (HEAP-10). With a null heap it looks only in the one default heap. | Looks the address up in a bitmap of live spans and never reads memory it does not own. With a null heap it checks every thread heap (not first-class heaps). | +| `heap_free_all(heap)` | Yes | Yes | +| `global_statistics()` | A placeholder struct of 7 fields, always zero | rpmalloc's `Global_Statistics` (9 fields). It is zero unless `ENABLE_STATISTICS`. The memory committed now is `active`; `committed` and `decommitted` are running totals that only grow, as upstream. The pointer form `global_statistics(*stats)` also exists. | +| `heap_statistics(heap)` | `{sizecache, spancache}` | `{allocated_size, committed_size, mapped_size}`, zero unless `HEAP_STATISTICS` | +| `allocator_default_heap` | A global `*Heap` | **Not provided.** There is no single default heap; each thread has its own. | +| The rest of the rpmalloc API | No | `initialize`, `thread_collect`, `trim`, `rpmalloc`, `rpaligned_alloc`, `heap_acquire`, and the rest (see `my_modules/rpmalloc/module.jai`) | + +**Module parameters.** `ENABLE_ASSERTS` and `ENABLE_VALIDATE_ARGS` have the same names and defaults (both `true`). The port's other parameters are added: statistics, decommit, unmap, first-class heaps, the huge cache and test hooks. `Runtime_Support` imports the module without parameters, so the context allocator always uses the defaults. Importing it with other parameters creates a separate instance with its own state; the context allocator is not affected. + +## 3. Allocator protocol + +| Mode | Jai's | This module | +|---|---|---| +| ALLOCATE / RESIZE / FREE with `data == null` | rpmalloc branch: the one default heap, behind a global lock. C runtime branch: `malloc`. | The calling thread's own heap, with no lock on the fast path. A block freed by another thread is handed back to its owning heap without blocking. | +| Out of memory | **Asserts** with asserts on (N1, tests SIZE-01 and SIZE-07) | Returns null | +| Negative or impossibly large size | Not caught (F11, SIZE-02 and SIZE-03). The C runtime wrapper panics in debug builds (N5). | Returns null; a refused RESIZE leaves the block intact | +| RESIZE `old_size` | Ignored | When the block moves, `old_size` bytes are copied; 0 means the whole old block. **Passing a positive `old_size` smaller than the live data loses the rest.** | +| RESIZE to the same size | Can move the block (I1, RE-02) | Keeps it | +| A refused RESIZE | Returns the old pointer (L6, RE-04) | Returns null | +| STARTUP | Does nothing; returns null | `initialize()`; returns null. `Process/posix.jai` stores this return value in `context.allocator.data`, where null means "the thread's heap". | +| SHUTDOWN | Forgets the default heap; later use fails (F3, LIFE-01) | `finalize()`; the next allocation initialises again | +| THREAD_START / THREAD_STOP | Do nothing | Set up / release the thread's heap. Both are optional: a thread gets its heap on its first allocation, and the heap is released when the thread exits. Jai's `Thread` module sends these only on Windows (N4). | +| CREATE_HEAP | rpmalloc branch: yes. C runtime branch: **asserts**. | Yes (with `FIRST_CLASS_HEAPS`, the default) | +| DESTROY_HEAP | Releases the heap; a new heap can then claim the destroyed heap's blocks (F5, HEAP-01). Blocks still in it leak: 1,970 MiB over 190 heaps (HEAP-11). | Frees every block in the heap, then releases it | +| IS_THIS_YOURS | rpmalloc branch: see §2. C runtime branch: **asserts**. | See §2 | +| CAPS flags | rpmalloc branch: general heap, threads, CREATE_HEAP, FREE, ACTUALLY_RESIZE, IS_THIS_YOURS. C runtime branch: no heaps, no IS_THIS_YOURS. | The same as the rpmalloc branch, on every platform | + +**First-class heaps.** A heap from CREATE_HEAP is for one thread at a time, as in upstream rpmalloc. Putting it in a Context that several threads copy lets them race on it. + +## 4. Threads + +- **Heap per thread.** Each thread allocates from its own heap, kept in OS thread-local storage (`TlsAlloc`/`FlsAlloc`, or a pthread key). A Context copied to another thread does not share the heap. +- **Heaps of exited threads are reused.** A thread's heap is released when the thread exits, even without THREAD_STOP, and the next new thread takes it over. With Jai's module on Windows and Linux, all threads share one heap behind one lock: there is no per-thread state, but every allocation contends for that lock. +- **Freeing another thread's block.** A block freed by a thread other than its owner is queued without a lock. Its owner reuses it on that owner's next allocation, or on `thread_collect()`. If the owner has exited, or is idle in `thread_idle()`, the block is freed at once instead. +- **Thread pools: call `thread_idle()`.** A pool thread that sleeps makes no allocator calls, so blocks other threads free into its heap, and the heap's free pages, would stay committed until it wakes. Calling `Default_Allocator.thread_idle()` where a pool thread waits for work gives that memory back and parks the heap until the thread's next allocation. Jai's module has no per-thread state, so it needs no such call. +- **Partly used pages per thread.** Each heap keeps its own partly used pages, so a program with many threads holds some more memory in partly filled small pages (64 KiB each) than one shared heap would. Larger pages are committed as they fill. + +## 5. Memory use + +- **Retention.** The port keeps freed pages per thread and huge blocks in a cache of up to 128 MiB committed (`HUGE_CACHE_LIMIT_MB`; a program that frees and allocates many huge blocks may run faster with more, holding up to that much while it does). Cached entries idle for a second are decommitted; blocks beyond the budget are unmapped. It decommits free pages only after a delay. Jai's rpmalloc branch returns memory eagerly, and pays page faults for it on reuse (F1, RET-01: about 112 faults per warm cycle). `trim()` returns every free page of the calling thread's heap, plus the whole huge cache, at once. `thread_collect()` does the same for pages past their delay. +- **Address space.** Memory is reserved in 256 MiB spans and committed page by page. Virtual size therefore looks large; the committed size is what costs memory. A 128 KiB bitmap for IS_THIS_YOURS is mapped once. +- **Measured footprint** (MiB with the benchmark's live set allocated; `my_modules/rpmalloc/tests/results/BENCH-*.md`): + + | Benchmark | Windows: Jai's | Windows: this | macOS: Jai's (C runtime) | macOS: this | + |---|---|---|---|---| + | churn_medium | 204 | 170 | 148 | 66 | + | churn_large | 157 | 260 | 128 | 7.8 | + | churn_huge | 296 | 452 | 28.5 | 4.2 | + +- **macOS.** The port decommits with `MADV_FREE_REUSABLE`, so freed memory leaves the process's footprint. + +## 6. fork() + +`Process/posix.jai` calls `unsafe_reset()` in the child, then `allocator_proc(.STARTUP)`. Jai's rpmalloc branch forgets its whole state at that point, so every block allocated before the fork leaks in the child. This module instead releases each lock that another thread of the parent might have held: the global heap lock, the heap-creation flag, the huge-cache lock, and an initialisation another thread had started. Everything else stays as it was. The forking thread's heap and its blocks remain valid in the child. Like Jai's, it cannot repair a list another thread was in the middle of changing; that window is a few instructions under one of those locks. `jai build.jai - swap` checks this: without `unsafe_reset` the smoke test hangs, and with it the test passes. + +## 7. Compile time + +Code in `#run` allocates through this module as well; the swap smoke test checks this. Jai's module also works at compile time. + +## 8. Build + +- This module imports `Basic`, `Bit_Operations` and `Atomics`; Jai's imports nothing. +- Its code is about 4,000 lines; Jai's rpmalloc branch is about 2,000. +- Unsupported platforms fail to compile (§1). +- **One Context field.** The module adds `rpmalloc_default_allocator_thread_heap` (a pointer) to every program's `Context`, a cache of the thread's heap, so allocations and frees need no OS thread-local storage call. A second import of the module with other parameters must pass `CONTEXT_HEAP_CACHE = false`, since two imports cannot add the same field. Jai's module adds nothing to the Context. +- It works in programs built with `arithmetic_overflow_check = .FATAL`: no code relies on unsigned wraparound. In `my_modules/rpmalloc/tests`, `jai build.jai - swap overflow=fatal` and the port's suite with `overflow=fatal` test this. Both pass on Windows and macOS; Linux is type-checked only. + +## 9. Speed + +Median ns per operation, module defaults, from `my_modules/rpmalloc/tests/results/BENCH-*.md`; lower is better. + +| Benchmark | Windows: Jai's | Windows: this | macOS: Jai's (C runtime) | macOS: this | +|---|---|---|---|---| +| pair_64 (alloc + free of 64 B) | 34.3 | 13.1 | 9.7 | 12.2 | +| pair_mixed (16..1024 B) | 256 | 15.5 | 23.6 | 13.4 | +| churn_medium (1..64 KiB) | 1059 | 51.7 | 74.0 | 53.2 | +| churn_large (64 KiB..1 MiB) | 4600 | 32.3 | 122 | 30.9 | +| **churn_huge (2..32 MiB)** | 9526 | **20245** | 618 | **4751** | +| realloc_double (64 B to 8 MiB) | 184662 | 15492 | 6373 | 6601 | +| mt_churn_8 (8 threads) | 515 | 2.3 | 18.0 | 1.9 | +| mt_producer (cross-thread frees) | 484 | 46.4 | 122 | 87.3 | + +On Windows this module is faster everywhere except **churn_huge** (replacing blocks of 2 MiB and up), where it is 2× slower. On macOS, where Jai's module is the system `malloc`, it is level or faster except on pair_64 (0.8×) and churn_huge (7× slower). That gap is shelved in the plan. + +## 10. Test results + +The test suite was run against Jai's module and against the port: + +| Platform | Jai's Default_Allocator | This module's code (the port) | +|---|---|---| +| Windows x64 (Jai's: rpmalloc 1.4.4) | 30 pass, 11 fail, 4 crash (41 skipped: capabilities it lacks) | 82 pass; every other test is SKIP or N/A | +| macOS ARM64 (Jai's: C runtime `malloc`) | 30 pass, 6 fail, 2 crash (48 skipped) | 83 pass; every other test is SKIP or N/A | + +What Jai's module fails, by kind: + +- **Correctness, Windows:** + - SIZE-01 and SIZE-07 assert when the OS refuses memory. + - SIZE-02 and SIZE-03 accept negative sizes. + - RE-02: a same-size resize moves the block. + - RE-04: a refused resize returns the old pointer. + - HEAP-01: a new heap claims a destroyed heap's blocks. + - HEAP-05 and HEAP-10: IS_THIS_YOURS misses blocks it owns. + - HEAP-11: destroyed heaps leak their blocks. + - LIFE-01: the allocator is unusable after SHUTDOWN. +- **Correctness, macOS:** SIZE-02 crashes on a negative size, and RE-04 is the same as on Windows. +- **Memory behaviour, Windows:** RET-01, RET-06, RET-10 and RET-14 fail on page faults. Freed memory goes back to the OS and faults in again on every reuse (up to 2,053 faults per 8 MiB cycle). +- **Memory behaviour, macOS:** OS-07, RET-02, RET-07, RET-11 and RET-12 fail because the system `malloc` keeps freed memory private (258 of 258 MiB after freeing everything). THR-09: blocks freed by another thread are not reused first. + +One upstream bug remains open in the port: U15 (POSIX only; at most one heap per exiting thread can leak, in a rare destructor order). See `my_modules/rpmalloc/module.jai`. + +## 11. Before you switch a project + +- Check every `RESIZE` / `realloc` call for a correct `old_size` (§3). Jai's module ignores it; this one relies on it. +- Expect a larger footprint on Windows for large and huge blocks, or call `trim()` at quiet points. +- Workloads with many huge blocks (2 MiB and up) are slower than with Jai's module (§9). +- Supported platforms are Windows and macOS; Linux is type-checked only. +- A block must still be freed by the allocator that made it. Each program and DLL compiled with this module has its own instance, just as with Jai's. diff --git a/modules/Default_Allocator/LICENSE b/modules/Default_Allocator/LICENSE new file mode 100644 index 0000000..74f442e --- /dev/null +++ b/modules/Default_Allocator/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 Mattias Jansson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/modules/Default_Allocator/LICENSES/MIT.txt b/modules/Default_Allocator/LICENSES/MIT.txt new file mode 100644 index 0000000..74f442e --- /dev/null +++ b/modules/Default_Allocator/LICENSES/MIT.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 Mattias Jansson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/modules/Default_Allocator/LICENSES/Unlicense.txt b/modules/Default_Allocator/LICENSES/Unlicense.txt new file mode 100644 index 0000000..efb9808 --- /dev/null +++ b/modules/Default_Allocator/LICENSES/Unlicense.txt @@ -0,0 +1,24 @@ +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to diff --git a/modules/Default_Allocator/README.md b/modules/Default_Allocator/README.md new file mode 100644 index 0000000..3459fc8 --- /dev/null +++ b/modules/Default_Allocator/README.md @@ -0,0 +1,59 @@ +# Default_Allocator (rpmalloc 2.0.1 port) + +A drop-in replacement for Jai's `modules/Default_Allocator`: the rpmalloc 2.0.1 port, wrapped in the interface Jai's runtime expects. This directory holds only what differs from the port's other flavour. `install.jai` at the repository root puts it together with the shared allocator files. + +## Installing it + +From the repository root (`my_modules/rpmalloc`): + +```sh +jai install.jai - Default_Allocator PROJECT/modules +``` + +Run the same command again to update the copy when the repository has a newer `VERSION_REVISION`. Add `check` to see what it would do. The repository's `README.md` describes every case. + +## Using it + +Jai picks its default allocator by module name: `Runtime_Support.jai` imports `"Default_Allocator"`. `PROJECT/modules` has to come ahead of Jai's own `modules` directory on the import path: + +- Jai searches the working directory's `modules/` folder first, so building from the project's root is enough: `jai main.jai`. +- From elsewhere, or to be explicit: `jai main.jai -import_dir PROJECT/modules`, or in a build program prepend the directory to `Build_Options.import_path`: + + ```jai + paths: [..] string; + array_add(*paths, "PROJECT/modules"); + array_add(*paths, ..options.import_path); + options.import_path = paths; + ``` + +Every allocation the program makes then goes through this allocator, from the first one, and at compile time too. Nothing needs to set `context.allocator`. + +**Thread pools:** call `Default_Allocator.thread_idle()` where a pool thread waits for work. Each thread has its own heap, and an idle thread makes no allocator calls, so without it the memory other threads free into that heap stays committed until the thread wakes (DIFFERENCES.md, section 4). + +To check which allocator, and which version, a program is using: + +```jai +name: string; +context.allocator.proc(.CAPS, 0, 0, *name, context.allocator.data); +// "rpmalloc 2.0.1-jai.N (Jai port) as Default_Allocator", N being VERSION_REVISION +``` + +`Default_Allocator.VERSION` holds the same version (`version.jai` in the installed copy). + +**Supported:** 64-bit Windows and macOS (tested), and Linux (type-checked only). Other targets stop with an `#assert`; build them without this module on the import path. + +**Read [DIFFERENCES.md](DIFFERENCES.md) before switching a project.** In particular, RESIZE relies on `old_size`, memory retention differs, and huge-block churn is slower. + +## Files + +| File | In the repository | Installed copy | +|---|---|---| +| `module.jai` | `Default_Allocator/module.jai`: Jai's `Default_Allocator` interface (`allocator`, `allocator_proc`, `unsafe_reset`, `initialize_if_needed`, …) and the port's public types | `module.jai` | +| `README.md`, `DIFFERENCES.md` | `Default_Allocator/` | the same names | +| `rpmalloc.jai`, `os_windows.jai`, `os_posix.jai`, `version.jai`, `LICENSE`, `LICENSES/` | the repository root, shared with the `rpmalloc2` flavour | the same names | + +`module.jai` loads the shared files from its own directory, so it builds only in an installed copy. In the repository, `tests/` checks it with `jai build.jai - swap`, which installs this flavour into a scratch directory and runs a program on it. + +When the port's `module.jai` changes the protocol procedure or the public types, bring this `module.jai` up to date by hand. It has its own copy of both. + +License: rpmalloc is dual licensed Unlicense / MIT (`LICENSE`, `LICENSES/`). diff --git a/modules/Default_Allocator/module.jai b/modules/Default_Allocator/module.jai new file mode 100644 index 0000000..52acfe0 --- /dev/null +++ b/modules/Default_Allocator/module.jai @@ -0,0 +1,303 @@ +// Default_Allocator: rpmalloc 2.0.1 for Jai, as a drop-in replacement for Jai's modules/Default_Allocator. +// +// Jai finds the default allocator by module name: Runtime_Support.jai sets the default context's +// allocator to Default_Allocator.allocator_proc, and Process/posix.jai calls +// Default_Allocator.unsafe_reset() and allocator_proc(.STARTUP) in a forked child. Put the directory +// that holds this module ahead of Jai's modules directory on the import path, and every allocation +// of the program, from the first one the runtime makes, goes through this allocator. +// +// The allocator is the port in my_modules/rpmalloc (tested by its tests/ suite). This file is the +// repository's Default_Allocator/module.jai; install.jai copies it next to the port's shared files +// (rpmalloc.jai, os_*.jai, version.jai) in PROJECT/modules/Default_Allocator, and only there does it +// build. Upstream is dual licensed Unlicense / MIT; see LICENSE and LICENSES/. +// +// Read DIFFERENCES.md before switching a project over: it lists everything that behaves differently +// from Jai's Default_Allocator. + +#module_parameters ( + // The first two are Jai's Default_Allocator parameters, with the same names and defaults. + + // Internal consistency asserts, reported through Memory_Interface.error_callback if set, + // otherwise context.assertion_failed. Upstream rpmalloc default: off. + ENABLE_ASSERTS := true, + // Refuse bad arguments at the public entry points: sizes at or above MAX_ALLOC_SIZE (negative + // sizes included), count * size overflow in the calloc family, size + alignment overflow and + // non-power-of-two alignments. Upstream rpmalloc default: off. + ENABLE_VALIDATE_ARGS := true, + + // The rest are the port's (see my_modules/rpmalloc/module.jai). Runtime_Support imports this + // module without parameters, so the defaults below are what a program's context allocator uses. + + // Global, per-thread and (with HEAP_STATISTICS) per-heap counters: global_statistics, + // thread_statistics, dump_statistics, heap_statistics. + ENABLE_STATISTICS := false, + // At finalize, assert that every block was freed. Needs ENABLE_STATISTICS. + ENABLE_LEAK_DETECTION := false, + // Commit pages on demand, and decommit free pages beyond a per page type threshold. + ENABLE_DECOMMIT := true, + // Return address space to the OS (huge blocks; everything at finalize with unmap_on_finalize). + ENABLE_UNMAP := true, + // The heap_* API and Allocator_Mode.CREATE_HEAP / DESTROY_HEAP. + FIRST_CLASS_HEAPS := true, + // Per first-class heap allocated / committed / mapped sizes (heap_statistics). + HEAP_STATISTICS := false, + // Recently freed huge blocks kept for reuse (at most 128 MiB, 32 MiB each); 0 disables the cache. + HUGE_CACHE_SLOT_COUNT := 32, + // How much freed huge-block memory the cache may keep committed, in MiB; a single entry may take a + // quarter of it. Upstream's 128. A program that frees and allocates huge blocks often runs faster + // with more (churn_huge misses the cache on 26% of steps at 128, 5% at 256, 1% at 512), at the cost of + // up to this much memory held while it does; entries idle for a second are decommitted (#jai fix E11). + HUGE_CACHE_LIMIT_MB := 128, + // Keep the thread heap in the Jai Context (field rpmalloc_default_allocator_thread_heap) as well as in OS thread-local storage, so + // allocations and frees need no OS call to find it (#jai fix E10). Two imports of this module cannot + // both add the field: turn it off for a second import with other parameters. + CONTEXT_HEAP_CACHE := true, + // Export internals and fault injection for the port's test suite. Adds exports only. + TEST_HOOKS := false +)(); + +#assert !ENABLE_LEAK_DETECTION || ENABLE_STATISTICS "ENABLE_LEAK_DETECTION needs ENABLE_STATISTICS"; + +// #jai Jai's Default_Allocator also builds for PS5, Switch 2, iOS, Android and 32-bit targets (as a +// CRT malloc wrapper on most of them). The port has an OS layer for Windows, macOS and Linux only. +#assert (OS == .WINDOWS || OS == .MACOS || OS == .LINUX) && (CPU == .X64 || CPU == .ARM64) + "This Default_Allocator (rpmalloc 2.0.1 port) supports 64-bit Windows, macOS and Linux only. For other targets, take my_modules off the import path so Jai's own Default_Allocator is used."; + +/* + + Jai's Default_Allocator interface, as Runtime_Support, Process and other code use it: + + allocator Allocator.{allocator_proc, null} + allocator_proc The Allocator protocol, below. + unsafe_reset() For a forked child only, before it allocates. + initialize_if_needed() Initialise now; true if this call did it. + is_initialized() + finalize() + usable_size(ptr) + is_this_yours(heap, ptr) + heap_free_all(heap) + global_statistics() Also global_statistics(*stats), the rpmalloc form. + heap_statistics(heap) + + Not provided: allocator_default_heap. There is no single default heap; each thread has its own. + + Allocator protocol (allocator_proc): + + ALLOCATE, RESIZE, FREE The calling thread's heap, or the first-class heap in allocator_data. + Out of memory returns null. With ENABLE_VALIDATE_ARGS, so does an + impossible size (negative or near S64_MAX), and such a RESIZE leaves + the block intact. FREE of null is a no-op. RESIZE copies old_size + bytes when the block moves (all of the old block when old_size is 0). + STARTUP Optional: initialize(). Returns null. + SHUTDOWN finalize(). Allocating afterwards initialises again. + THREAD_START, THREAD_STOP thread_initialize(), thread_finalize(). Both optional: a thread gets + its heap on its first allocation, and gives it back at thread exit. + CREATE_HEAP heap_acquire(): a first-class heap, to put in allocator_data. + DESTROY_HEAP Frees every block still in the heap, then releases it. + IS_THIS_YOURS With a heap: is the address inside a page of that heap? Without: + inside a page of any thread heap (not of a first-class heap)? + CAPS Everything above; CREATE_HEAP only with FIRST_CLASS_HEAPS. + + For a thread pool: call thread_idle() where a pool thread waits for work. It gives back what + the thread's heap holds free and parks the heap, so blocks other threads free into it while it + sleeps are given back at once; the thread's next allocation takes the heap back. + + The whole rpmalloc API of the port is exported as well: initialize, thread_collect, trim, thread_idle, + rpmalloc, rpaligned_alloc, heap_acquire, and the rest (my_modules/rpmalloc/module.jai lists it). + +*/ + +allocator :: Allocator.{allocator_proc, null}; + +allocator_proc :: (mode: Allocator_Mode, requested_size: s64, old_size: s64, old_memory: *void, allocator_data: *void) -> *void { + heap := cast(*Heap) allocator_data; + + if #complete mode == { + case .ALLOCATE; + #if FIRST_CLASS_HEAPS if heap return heap_alloc(heap, requested_size); + return rpmalloc(requested_size); + + case .RESIZE; + return allocator_proc_resize(heap, old_memory, requested_size, old_size); + + case .FREE; + #if FIRST_CLASS_HEAPS if heap { heap_free(heap, old_memory); return null; } // Null is a no-op (#jai fix U18). + rpfree(old_memory); + return null; + + case .STARTUP; + initialize(); + return null; // Process/posix.jai stores this in context.allocator.data: null means the thread heap. + + case .SHUTDOWN; + finalize(); + return null; + + case .THREAD_START; + thread_initialize(); + return null; + + case .THREAD_STOP; + thread_finalize(); + return null; + + case .CREATE_HEAP; + #if FIRST_CLASS_HEAPS return heap_acquire(); + else return null; + + case .DESTROY_HEAP; + // #jai fix M4: destroying a heap frees every block still in it. + #if FIRST_CLASS_HEAPS if heap { + heap_free_all(heap); + heap_release(heap); + } + return null; + + case .IS_THIS_YOURS; + return cast(*void) cast(s64) is_this_yours(heap, old_memory); + + case .CAPS; + if old_memory { (cast(*string) old_memory).* = CAPS_VERSION_STRING; } + caps := Allocator_Caps.HINT_I_AM_A_GENERAL_HEAP_ALLOCATOR | .MULTIPLE_THREADS | .FREE | .ACTUALLY_RESIZE | .IS_THIS_YOURS; + #if FIRST_CLASS_HEAPS caps |= .CREATE_HEAP; + return cast(*void) caps; + + case; + context.handling_assertion_failure = true; + context.assertion_failed(#location(), "Invalid or corrupt mode passed to Default_Allocator.allocator_proc.\n"); + context.handling_assertion_failure = false; + return null; + } +} + +// Jai's Default_Allocator interface: initialise now, unless already done. True if this call did it. +initialize_if_needed :: () -> did_init: bool { + if is_initialized() return false; + initialize(); + return true; +} + +is_initialized :: inline () -> bool { + return atomic_load(*global_rpmalloc_init_state) == RPMALLOC_INIT_DONE; +} + +// Jai's Default_Allocator form of global_statistics. With ENABLE_STATISTICS off (the default) every +// field is 0, as with Jai's. +global_statistics :: () -> Global_Statistics { + stats: Global_Statistics; + global_statistics(*stats); + return stats; +} + +// :AllocatorAfterFork: Only for the child of fork(), before it allocates (Process/posix.jai). The +// child has one thread, the one that called fork; any lock another thread of the parent held at that +// moment would never be released, so the next allocation would wait forever. This releases them. +// +// #jai Jai's Default_Allocator forgets its whole state here and starts afresh. This keeps it: the +// forking thread's heap and every block it holds stay valid, and the heaps of the parent's other +// threads stay allocated (in the child nobody frees them, as with fork and any allocator). Like Jai's, +// it cannot repair a list another thread was in the middle of changing; the window is a few +// instructions under the global heap lock or the huge cache lock. +unsafe_reset :: () { + atomic_store(*global_heap_lock, 0); + global_heap_creating = 0; // Another thread was mapping a block of heaps; the child would wait for it. + #if HUGE_CACHE_SLOT_COUNT atomic_store(*global_huge_cache.lock, 0); + // Another thread was initialising: start again (the forking thread was not, it was in fork). + if atomic_load(*global_rpmalloc_init_state) == RPMALLOC_INIT_RUNNING + atomic_store(*global_rpmalloc_init_state, RPMALLOC_INIT_UNINIT); +} + +// +// Public types (2.0.1: rpmalloc.h). Kept identical to my_modules/rpmalloc/module.jai. +// + +#load "version.jai"; // VERSION, VERSION_REVISION, RPMALLOC_VERSION + +RPMALLOC_MAX_ALIGNMENT :: 256 * 1024; // 2.0.1: rpmalloc.h:69 + +// Flags for rpaligned_realloc / heap_realloc (2.0.1: rpmalloc.h:79-84). +Realloc_Flags :: enum_flags u32 { + NO_PRESERVE :: 1; // Do not preserve content in reallocation. + GROW_OR_FAIL :: 2; // Fail (return null) if the block cannot grow in place; the original stays valid. +} + +// 2.0.1: rpmalloc.h rpmalloc_interface_t. Pass one to initialize / initialize_config to replace +// the OS calls; it is copied, so later changes to it have no effect. If memory_map or memory_unmap +// is null, all four OS defaults are used; otherwise a null memory_commit or memory_decommit uses +// the OS's (#jai fix U8). +Memory_Interface :: struct { + // Map `size` bytes aligned to `alignment` (0 or the span size). May store an alignment offset + // and the mapped size, both passed back to memory_unmap. + memory_map: (size: u64, alignment: u64, offset: *u64, mapped_size: *u64) -> *void; + memory_commit: (address: *void, size: u64) -> s32; // Non-zero on failure. + memory_decommit: (address: *void, size: u64) -> s32; // Non-zero on failure. + memory_unmap: (address: *void, offset: u64, mapped_size: u64); + map_fail_callback: (size: u64) -> s32; // Non-zero: retry the map. + error_callback: (message: string); +} + +// 2.0.1: rpmalloc.h rpmalloc_config_t. +Config :: struct { + page_size: u64; // 0: the OS page size. + enable_huge_pages: bool; + enable_thp: bool; // Linux / Android only. + disable_decommit: bool; + page_name: string; + huge_page_name: string; + unmap_on_finalize: bool; + disable_thp: bool; // Linux / Android only. +} + +// 2.0.1: rpmalloc.h rpmalloc_global_statistics_t. All zero without ENABLE_STATISTICS. The memory +// committed now is active, not committed: committed and decommitted are running totals, as upstream. +Global_Statistics :: struct { + mapped: u64; // Address space mapped now, committed or not. + mapped_peak: u64; + committed: u64; // Running total of bytes ever committed; it only grows. + decommitted: u64; // Running total of bytes ever decommitted; it only grows. + active: u64; // Bytes committed now: what the OS counts (API-07 checks it). + active_peak: u64; + huge_alloc: u64; // Bytes in huge blocks allocated now. + huge_alloc_peak: u64; + heap_count: u64; // Heaps in use now. +} + +// 2.0.1: rpmalloc.h rpmalloc_thread_statistics_t. +Thread_Statistics :: struct { + sizecache: u64; // Bytes available in thread size class caches. + spancache: u64; // Bytes available in thread page caches. + span_use: [5] struct { + current: u64; + map_calls: u64; + }; + size_use: [128] struct { + alloc_current: u64; + alloc_peak: u64; + alloc_total: u64; + free_total: u64; + }; +} + +// 2.0.1: rpmalloc.h rpmalloc_heap_statistics_t. +Heap_Statistics :: struct { + allocated_size: u64; + committed_size: u64; + mapped_size: u64; +} + +#scope_module + +// #jai fix E10: the thread heap, cached in the Context (see get_thread_heap in rpmalloc.jai). +#if CONTEXT_HEAP_CACHE { + #add_context rpmalloc_default_allocator_thread_heap: *void; + context_thread_heap :: () -> **void #expand { return *context.rpmalloc_default_allocator_thread_heap; } +} + +// A literal, not built from VERSION with #run: a #run would wait for the default allocator, which is +// this module, whose CAPS needs this string. The assert keeps it in step with version.jai. +CAPS_VERSION_STRING :: "rpmalloc 2.0.1-jai.16 (Jai port) as Default_Allocator"; +#assert #run text_is(CAPS_VERSION_STRING, "rpmalloc ", VERSION, " (Jai port) as Default_Allocator") "Update CAPS_VERSION_STRING to the VERSION in version.jai"; + +#load "rpmalloc.jai"; +#if OS == .WINDOWS #load "os_windows.jai"; +else #load "os_posix.jai"; diff --git a/modules/Default_Allocator/os_posix.jai b/modules/Default_Allocator/os_posix.jai new file mode 100644 index 0000000..9527fdc --- /dev/null +++ b/modules/Default_Allocator/os_posix.jai @@ -0,0 +1,408 @@ +// OS entry points for macOS and Linux (2.0.1: rpmalloc.c:878-1161, the POSIX branches), plus +// the POSIX part of rpmalloc_initialize (2.0.1: rpmalloc.c:2776-2782). +// +// Everything goes through libc on every CPU. The 1.4.4 Jai port's raw syscalls caused review +// findings H4, H7 and M1; none of that code exists here. +// +// #jai deliberate differences from upstream: +// - A failed map returns null without asserting (finding N1). +// - macOS decommits with MADV_FREE_REUSABLE, retried on EAGAIN, and commits with +// MADV_FREE_REUSE. Upstream prefers MADV_DONTNEED, which exists on macOS but does not reduce +// the process footprint there, so decommit is a no-op on macOS (upstream bug U21). +// - Every entry point works before initialize(), like os_windows.jai. + +// Fills os_map_granularity and os_page_size. Idempotent. +// 2.0.1: rpmalloc.c:2776-2782 (the POSIX part of rpmalloc_initialize) +os_initialize :: () { + if os_page_size return; + os_map_granularity = cast(u64) sysconf_p(_SC_PAGESIZE); + os_page_size = os_map_granularity; +} + +os_set_page_name :: (address: *void, size: u64) { // 2.0.1: rpmalloc.c:885 + #if OS == .LINUX { + name := ifx os_huge_pages then global_config.huge_page_name else global_config.page_name; + if address == MAP_FAILED_P || !name return; + // prctl wants a C string; names are short, copy into a terminated buffer. + buffer: [80] u8; + n := min(name.count, buffer.count - 1); + memcpy(buffer.data, name.data, n); + buffer[n] = 0; + // If the kernel lacks CONFIG_ANON_VMA_NAME, or the name is invalid, this does nothing. + prctl_p(PR_SET_VMA, PR_SET_VMA_ANON_NAME, cast(u64) address, size, cast(u64) buffer.data); + } +} + +// #jai fix E9: a mapping that is committed as well. On POSIX a fresh anonymous mapping is usable as it +// is, so this only counts the commit (no madvise); see os_windows.jai for why huge blocks use it. +os_mmap_committed :: (size: u64, alignment: u64, offset: *u64, mapped_size: *u64) -> *void { + ptr := os_mmap(size, alignment, offset, mapped_size); + #if ENABLE_DECOMMIT if ptr statistics_os_commit(size); + return ptr; +} + +os_mmap :: (size: u64, alignment: u64, offset: *u64, mapped_size: *u64) -> *void { // 2.0.1: rpmalloc.c:900 + os_initialize(); + #if TEST_HOOKS if test_take_fault(*test_fail_maps) return os_map_failed(size, alignment, offset, mapped_size); + + map_size := size + alignment; // UPSTREAM-BUG U1: wraps for sizes near U64_MAX. + flags: s32 = MAP_PRIVATE_P | MAP_ANONYMOUS_P | MAP_UNINITIALIZED_P; + #if OS == .MACOS { + fd := VM_MAKE_TAG_240; + if os_huge_pages fd |= VM_FLAGS_SUPERPAGE_SIZE_2MB; + ptr := mmap_p(null, map_size, PROT_READ_WRITE_P, flags, fd, 0); + if (ptr == MAP_FAILED_P || !ptr) && os_huge_pages { + // Superpage allocations can fail through physical memory fragmentation; fall back + // to regular pages. + ptr = mmap_p(null, map_size, PROT_READ_WRITE_P, flags, VM_MAKE_TAG_240, 0); + } + } else { + huge_flags: s32 = 0; + if os_huge_pages { + huge_flags = MAP_HUGETLB_P; + // Explicitly request the detected huge page size; the system default huge page size + // can be a different (unusable) size like 1 GiB. + if os_huge_page_shift huge_flags |= cast(s32)(os_huge_page_shift << MAP_HUGE_SHIFT_P); + } + ptr := mmap_p(null, map_size, PROT_READ_WRITE_P, huge_flags | flags, -1, 0); + if (ptr == MAP_FAILED_P || !ptr) && os_huge_pages { + // Huge page allocations can fail: fall back to normal pages and advise transparent + // huge pages (best effort; for example EINVAL when THP is disabled system-wide). + ptr = mmap_p(null, map_size, PROT_READ_WRITE_P, flags, -1, 0); + if ptr && ptr != MAP_FAILED_P madvise_p(ptr, map_size, MADV_HUGEPAGE_P); + } else if os_thp && ptr && ptr != MAP_FAILED_P { + // Transparent huge page mode: ask for huge pages without a preallocated pool. + madvise_p(ptr, map_size, MADV_HUGEPAGE_P); + } + os_set_page_name(ptr, map_size); + } + if ptr == MAP_FAILED_P ptr = null; + if !ptr return os_map_failed(size, alignment, offset, mapped_size); + + if alignment { + padding := cast(u64) ptr & (alignment - 1); + if padding padding = alignment - padding; + rpmalloc_assert(padding <= alignment, "Internal failure in padding"); + rpmalloc_assert(!(padding % 8), "Internal failure in padding"); + // Unmap the alignment padding head and tail, so no unusable address space (or huge page + // reservation) is kept. Both the mmap result and the alignment are multiples of the + // mapping page size, so head and tail stay page aligned. + if padding munmap_p(ptr, padding); + ptr = pointer_offset(ptr, padding); + if alignment - padding munmap_p(pointer_offset(ptr, size), alignment - padding); + offset.* = 0; + map_size = size; + } + mapped_size.* = map_size; + statistics_os_map(map_size); // 2.0.1: rpmalloc.c:1023-1033 + return ptr; +} + +// The failure tail of os_mmap (2.0.1: rpmalloc.c:990-998). +// #jai returns null without asserting when there is no map_fail_callback (finding N1). +os_map_failed :: (size: u64, alignment: u64, offset: *u64, mapped_size: *u64) -> *void { + iface := global_memory_interface; + if iface && iface.map_fail_callback && iface.map_fail_callback(size + alignment) + return os_mmap(size, alignment, offset, mapped_size); + return null; +} + +os_commit_on_demand :: inline () -> bool { // 2.0.1: rpmalloc.c:1040 + #if ENABLE_DECOMMIT return !global_config.disable_decommit; + else return false; +} + +os_mcommit :: (address: *void, size: u64) -> s32 { // 2.0.1: rpmalloc.c:1049 + #if ENABLE_DECOMMIT { + #if TEST_HOOKS if test_take_fault(*test_fail_commits) return 1; + // POSIX commits on first touch, so upstream does nothing here (its mprotect is commented + // out). #jai on macOS, tell the kernel the pages given back with MADV_FREE_REUSABLE are + // in use again, so the footprint accounting stays right (the pairing Apple documents). + #if OS == .MACOS madvise_p(address, size, MADV_FREE_REUSE_P); + statistics_os_commit(size); // 2.0.1: rpmalloc.c:1067-1078 + } + return 0; +} + +os_mdecommit :: (address: *void, size: u64) -> s32 { // 2.0.1: rpmalloc.c:1086 + #if ENABLE_DECOMMIT { + // Unlike os_mcommit, honor the live flag (the Linux fallback below can set it mid-run). + if global_config.disable_decommit return 1; + #if OS == .MACOS { + // #jai MADV_FREE_REUSABLE, not MADV_DONTNEED (upstream bug U21). + ret := madvise_p(address, size, MADV_FREE_REUSABLE_P); + while ret == -1 && errno_p().* == EAGAIN_P { + errno_p().* = 0; + ret = madvise_p(address, size, MADV_FREE_REUSABLE_P); + } + failed := ret == -1; + } else { + failed := madvise_p(address, size, MADV_DONTNEED_P) != 0; + if failed && os_huge_pages { + // MADV_DONTNEED on huge page mappings needs Linux 5.18 or later: keep pages + // committed and stop further decommit attempts. + global_config.disable_decommit = true; + return 1; + } + } + if failed { + rpmalloc_assert(false, "Failed to decommit virtual memory block"); + return 1; + } + statistics_os_decommit(size); // 2.0.1: rpmalloc.c:1129-1133 + } + return 0; +} + +os_munmap :: (address: *void, offset: u64, mapped_size: u64) { // 2.0.1: rpmalloc.c:1142 + address = pointer_offset(address, -cast(s64) offset); + #if ENABLE_UNMAP { + if munmap_p(address, mapped_size) + rpmalloc_assert(false, "Failed to unmap virtual memory block"); + statistics_os_unmap(mapped_size); // 2.0.1: rpmalloc.c:1154-1158 + } +} + +os_thread_yield :: () { // 2.0.1: rpmalloc.c:396 (sched_yield) + sched_yield_p(); +} + +// Coarse monotonic time for the huge block cache. +os_monotonic_time_ms :: () -> u64 { // 2.0.1: rpmalloc.c:1586 + ts: timespec_p; + clock_gettime_p(CLOCK_MONOTONIC_FOR_CACHE, *ts); + return cast(u64) ts.tv_sec * 1000 + cast(u64) ts.tv_nsec / 1_000_000; +} + +// +// Thread-local heap storage, thread identity and the thread exit hook. +// +// #jai upstream keeps the thread heap in a C11 _Thread_local variable and uses a pthread key only +// for the thread exit destructor. This Jai version has no #thread_local, and keeping the heap in +// the Context would let copied contexts share a heap between threads (review H3), so one pthread +// key does both: it stores the heap pointer and runs the destructor at thread exit. The destructor +// matters more here than upstream: Jai's Thread module never sends THREAD_STOP on POSIX (finding +// N4), so it is what returns a finished Jai thread's heap. + +os_tls_create :: (destructor: (value: *void) #c_call) -> bool { + return pthread_key_create_p(*tls_key, destructor) == 0; // 2.0.1: rpmalloc.c:2954 +} + +os_tls_destroy :: () { + pthread_key_delete_p(tls_key); // 2.0.1: rpmalloc.c:3102 + tls_key = 0; +} + +// #jai fix E10: the calling thread's stack, [low, high); 0, 0 if it cannot be found out (the Context +// cache of the thread heap is then never used). +os_thread_stack_range :: () -> low: u64, high: u64 { + self := pthread_self_p(); + #if OS == .MACOS { + high := cast(u64) pthread_get_stackaddr_np_p(self); // The top: the stack grows down. + size := pthread_get_stacksize_np_p(self); + if !high || size > high return 0, 0; + return high - size, high; + } else { + attr: [8] u64; // pthread_attr_t: 56 bytes on x86-64, 64 on ARM64 glibc. + if pthread_getattr_np_p(self, attr.data) != 0 return 0, 0; + address: *void; + size: u64; + ok := pthread_attr_getstack_p(attr.data, *address, *size) == 0; + pthread_attr_destroy_p(attr.data); + if !ok return 0, 0; + return cast(u64) address, cast(u64) address + size; + } +} + +os_tls_get :: inline () -> *void { + return pthread_getspecific_p(tls_key); +} + +os_tls_set :: (value: *void) { + pthread_setspecific_p(tls_key, value); // 2.0.1: rpmalloc.c:805 +} + +// #jai upstream reads the thread pointer (__builtin_thread_pointer or fs/gs/tpidr registers, +// rpmalloc.c:763-790). pthread_self is equally unique among live threads and never 0. +// See os_windows.jai (#jai fix U22). POSIX exit() does not run other threads' key destructors, and +// the exiting thread's own run only from pthread_exit, so there is nothing to skip here. +os_process_exiting :: inline () -> bool { return false; } + +os_thread_id :: inline () -> u64 { + return cast(u64) pthread_self_p(); +} + +// Huge page detection (2.0.1: rpmalloc.c:2831-2892, the Linux and Apple branches). +os_detect_huge_pages :: () { + #if OS == .LINUX { + huge_page_size: u64 = 0; + meminfo: [4096] u8; + n := os_read_system_file("/proc/meminfo", meminfo.data, meminfo.count); + if n { + text := string.{n, meminfo.data}; + at := find_substring(text, "Hugepagesize:"); + if at >= 0 huge_page_size = parse_decimal(advance_string(text, at + 13)) * 1024; + } + huge_page_size = usable_huge_page_size(huge_page_size); + if huge_page_size { + // Hugepagesize only reports the configured size, not whether any huge pages are + // available. Probe by mapping one huge page with the flags real mappings use. + huge_shift: u64 = 0; + while (cast(u64) 1 << huge_shift) < huge_page_size huge_shift += 1; + probe_flags := MAP_PRIVATE_P | MAP_ANONYMOUS_P | MAP_HUGETLB_P | cast(s32)(huge_shift << MAP_HUGE_SHIFT_P); + probe := mmap_p(null, huge_page_size, PROT_READ_WRITE_P, probe_flags, -1, 0); + if probe != MAP_FAILED_P && probe { + munmap_p(probe, huge_page_size); + os_huge_pages = true; + os_huge_page_shift = huge_shift; + os_page_size = huge_page_size; + os_map_granularity = huge_page_size; + } + } + } else #if OS == .MACOS && CPU == .X64 { + // Superpages are only supported on x86-64; Apple silicon has no huge page support. + os_huge_pages = true; + os_page_size = 2 * 1024 * 1024; + os_map_granularity = os_page_size; + } +} + +// Transparent huge pages (2.0.1: rpmalloc.c:2921-2936, Linux only). +os_configure_thp :: () { + #if OS == .LINUX { + if global_config.enable_thp && !os_huge_pages && !global_config.disable_thp { + // With THP disabled system-wide ("never") madvise(MADV_HUGEPAGE) does nothing, so + // report THP as disabled rather than claim it is in use. + state: [64] u8; + n := os_read_system_file("/sys/kernel/mm/transparent_hugepage/enabled", state.data, state.count); + thp_never := n && find_substring(string.{n, state.data}, "[never]") >= 0; + if !thp_never os_thp = true; + } + if global_config.disable_thp prctl_p(PR_SET_THP_DISABLE_P, 1, 0, 0, 0); + } +} + +#if OS == .LINUX { + // Read a small (pseudo) file such as /proc/meminfo into a caller buffer with raw open/read, so + // nothing allocates during initialisation. Returns the bytes read (the buffer is always zero + // terminated), or 0. + os_read_system_file :: (path: string, buffer: *u8, capacity: s64) -> s64 { // 2.0.1: rpmalloc.c:2684 + if capacity < 2 return 0; + fd := open_p(path.data, O_RDONLY_P | O_CLOEXEC_P); // path is a zero-terminated literal + if fd < 0 return 0; + total: s64 = 0; + while total + 1 < capacity { + got := read_p(fd, buffer + total, cast(u64)((capacity - 1) - total)); + if got <= 0 break; + total += got; + } + close_p(fd); + buffer[total] = 0; + return total; + } + + find_substring :: (text: string, pattern: string) -> s64 { + if pattern.count > text.count return -1; + for i: 0..text.count - pattern.count { + match := true; + for j: 0..pattern.count-1 if text[i + j] != pattern[j] { match = false; break; } + if match return i; + } + return -1; + } + + advance_string :: (text: string, n: s64) -> string { + if n >= text.count return ""; + return string.{text.count - n, text.data + n}; + } + + // strtol(..., 10) for a non-negative number: skips leading blanks, stops at the first non-digit. + parse_decimal :: (text: string) -> u64 { + i := 0; + while i < text.count && (text[i] == #char " " || text[i] == #char "\t") i += 1; + value: u64 = 0; + while i < text.count && text[i] >= #char "0" && text[i] <= #char "9" { + value = value * 10 + (text[i] - #char "0"); + i += 1; + } + return value; + } +} + +#scope_file + +libc_rpm :: #library,system "libc"; + +mmap_p :: (address: *void, length: u64, prot: s32, flags: s32, fd: s32, offset: s64) -> *void #foreign libc_rpm "mmap"; +munmap_p :: (address: *void, length: u64) -> s32 #foreign libc_rpm "munmap"; +madvise_p :: (address: *void, length: u64, advice: s32) -> s32 #foreign libc_rpm "madvise"; +sysconf_p :: (name: s32) -> s64 #foreign libc_rpm "sysconf"; +sched_yield_p :: () -> s32 #foreign libc_rpm "sched_yield"; + +timespec_p :: struct { + tv_sec: s64; + tv_nsec: s64; +} +clock_gettime_p :: (clock_id: s32, ts: *timespec_p) -> s32 #foreign libc_rpm "clock_gettime"; +// 6 is CLOCK_MONOTONIC_COARSE on Linux (read through the vDSO, as upstream prefers) and +// CLOCK_MONOTONIC on macOS, which has no coarse clock. +CLOCK_MONOTONIC_FOR_CACHE :: 6; + +MAP_FAILED_P :: cast(*void) -1; +PROT_READ_WRITE_P :: 0x1 | 0x2; +MAP_PRIVATE_P :: 0x0002; + +#if OS == .MACOS { + errno_p :: () -> *s32 #foreign libc_rpm "__error"; + MAP_ANONYMOUS_P :: 0x1000; + MAP_UNINITIALIZED_P :: 0; + _SC_PAGESIZE :: 29; + EAGAIN_P :: 35; + MADV_FREE_REUSABLE_P :: 7; + MADV_FREE_REUSE_P :: 8; + VM_MAKE_TAG_240 :: cast(s32)(240 << 24); // VM_MAKE_TAG(240U) + VM_FLAGS_SUPERPAGE_SIZE_2MB :: 2 << 16; // VM_FLAGS_SUPERPAGE_SIZE_2MB (x86-64 macOS) +} else { + prctl_p :: (option: s32, arg2: u64, arg3: u64, arg4: u64, arg5: u64) -> s32 #foreign libc_rpm "prctl"; + MAP_ANONYMOUS_P :: 0x20; + MAP_UNINITIALIZED_P :: 0x400_0000; // Only honoured on no-MMU kernels. + MAP_HUGETLB_P :: 0x4_0000; + MAP_HUGE_SHIFT_P :: 26; + MADV_DONTNEED_P :: 4; + MADV_HUGEPAGE_P :: 14; + _SC_PAGESIZE :: 30; + PR_SET_VMA :: 0x5356_4d41; + PR_SET_VMA_ANON_NAME :: 0; +} + +// Thread-local storage and huge page detection. + +libc_tls :: #library,system "libc"; + +#if OS == .MACOS pthread_key_t_p :: u64; // unsigned long +else pthread_key_t_p :: u32; // unsigned int + +tls_key: pthread_key_t_p; + +pthread_key_create_p :: (key: *pthread_key_t_p, destructor: (value: *void) #c_call) -> s32 #foreign libc_tls "pthread_key_create"; +pthread_key_delete_p :: (key: pthread_key_t_p) -> s32 #foreign libc_tls "pthread_key_delete"; +pthread_getspecific_p :: (key: pthread_key_t_p) -> *void #foreign libc_tls "pthread_getspecific"; +pthread_setspecific_p :: (key: pthread_key_t_p, value: *void) -> s32 #foreign libc_tls "pthread_setspecific"; +pthread_self_p :: () -> *void #foreign libc_tls "pthread_self"; +#if OS == .MACOS { + pthread_get_stackaddr_np_p :: (thread: *void) -> *void #foreign libc_tls "pthread_get_stackaddr_np"; + pthread_get_stacksize_np_p :: (thread: *void) -> u64 #foreign libc_tls "pthread_get_stacksize_np"; +} else { + pthread_getattr_np_p :: (thread: *void, attr: *u64) -> s32 #foreign libc_tls "pthread_getattr_np"; + pthread_attr_getstack_p :: (attr: *u64, address: **void, size: *u64) -> s32 #foreign libc_tls "pthread_attr_getstack"; + pthread_attr_destroy_p :: (attr: *u64) -> s32 #foreign libc_tls "pthread_attr_destroy"; +} + +#if OS == .LINUX { + open_p :: (path: *u8, flags: s32) -> s32 #foreign libc_tls "open"; + read_p :: (fd: s32, buffer: *u8, count: u64) -> s64 #foreign libc_tls "read"; + close_p :: (fd: s32) -> s32 #foreign libc_tls "close"; + O_RDONLY_P :: 0; + O_CLOEXEC_P :: 0x8_0000; + PR_SET_THP_DISABLE_P :: 41; +} diff --git a/modules/Default_Allocator/os_windows.jai b/modules/Default_Allocator/os_windows.jai new file mode 100644 index 0000000..09f069c --- /dev/null +++ b/modules/Default_Allocator/os_windows.jai @@ -0,0 +1,334 @@ +// OS entry points for Windows (2.0.1: rpmalloc.c:878-1161, the PLATFORM_WINDOWS branches), +// plus the Windows part of rpmalloc_initialize (2.0.1: rpmalloc.c:2765-2782). +// +// #jai deliberate differences from upstream, all in the error paths: +// - A failed map or commit returns null / non-zero without asserting (finding N1). Upstream +// asserts when no map_fail_callback is set and ENABLE_ASSERTS is on. +// - Every entry point works before initialize(): it initialises the OS globals on demand and +// tolerates a null global_memory_interface (used by the test hooks). + +// Fills os_map_granularity, os_page_size and os_virtualalloc2. Idempotent. +// 2.0.1: rpmalloc.c:2765-2782 (the PLATFORM_WINDOWS part of rpmalloc_initialize) +os_initialize :: () { + if os_page_size return; + info: SYSTEM_INFO_W; + GetSystemInfo_w(*info); + os_map_granularity = info.dwAllocationGranularity; + os_virtualalloc2 = null; + // #jai GetModuleHandleA with an ASCII name; upstream uses the W variant. Same module. + kernelbase := GetModuleHandleA_w("kernelbase.dll".data); + if kernelbase os_virtualalloc2 = cast(VirtualAlloc2_Proc) GetProcAddress_w(kernelbase, "VirtualAlloc2".data); + os_page_size = info.dwPageSize; +} + +os_set_page_name :: (address: *void, size: u64) { // 2.0.1: rpmalloc.c:885 (Linux only; a no-op here) +} + +os_mmap :: (size: u64, alignment: u64, offset: *u64, mapped_size: *u64) -> *void { // 2.0.1: rpmalloc.c:900 + return os_map(size, alignment, offset, mapped_size, false); +} + +// #jai fix E9: a mapping that is committed as well, in the same call. Committing a range that is only +// reserved costs about 15 us per 24 MiB, and makes releasing it cost about 48 us instead of 7 us +// (MEM_RESERVE | MEM_COMMIT in one VirtualAlloc: 0 and 7 us). Huge blocks are committed whole at once +// anyway, so heap_allocate_block_huge maps them this way: churn_huge missed the huge cache on a quarter +// of its steps and spent 13 of its 19.6 us per step releasing such mappings. +os_mmap_committed :: (size: u64, alignment: u64, offset: *u64, mapped_size: *u64) -> *void { + ptr := os_map(size, alignment, offset, mapped_size, true); + #if ENABLE_DECOMMIT if ptr statistics_os_commit(size); + return ptr; +} + +os_map :: (size: u64, alignment: u64, offset: *u64, mapped_size: *u64, commit: bool) -> *void { + os_initialize(); + #if TEST_HOOKS if test_take_fault(*test_fail_maps) return os_map_failed(size, alignment, offset, mapped_size); + + map_size := size + alignment; // UPSTREAM-BUG U1: wraps for sizes near U64_MAX. + // Ok to MEM_COMMIT: according to MSDN, "actual physical pages are not allocated unless/until the + // virtual addresses are actually accessed". But with decommit enabled it is better not to commit + // immediately and instead commit per page, to avoid saturating the OS commit limit. + #if ENABLE_DECOMMIT { + do_commit: u32 = ifx commit || global_config.disable_decommit then cast(u32) MEM_COMMIT_W else 0; + } else { + do_commit: u32 = MEM_COMMIT_W; + } + base_flags: u32 = MEM_RESERVE_W | do_commit; + large_flag: u32 = ifx os_huge_pages then cast(u32) MEM_LARGE_PAGES_W else 0; + ptr: *void; + if alignment && os_virtualalloc2 { + // Map an aligned region directly, so no alignment padding is kept per mapping. + reqs: MEM_ADDRESS_REQUIREMENTS_W; + reqs.Alignment = alignment; + param: MEM_EXTENDED_PARAMETER_W; + param.Type = MemExtendedParameterAddressRequirements; + param.Pointer = *reqs; + map_size = size; + ptr = os_virtualalloc2(null, null, map_size, large_flag | base_flags, PAGE_READWRITE_W, *param, 1); + if !ptr && large_flag + ptr = os_virtualalloc2(null, null, map_size, base_flags, PAGE_READWRITE_W, *param, 1); + } else { + // Pre Windows 10: over-allocate and offset for alignment below. + ptr = VirtualAlloc_w(null, map_size, large_flag | base_flags, PAGE_READWRITE_W); + if !ptr && large_flag + ptr = VirtualAlloc_w(null, map_size, base_flags, PAGE_READWRITE_W); + } + if !ptr return os_map_failed(size, alignment, offset, mapped_size); + + if alignment { + padding := cast(u64) ptr & (alignment - 1); + if padding padding = alignment - padding; + rpmalloc_assert(padding <= alignment, "Internal failure in padding"); + rpmalloc_assert(!(padding % 8), "Internal failure in padding"); + ptr = pointer_offset(ptr, padding); + offset.* = padding; + } + mapped_size.* = map_size; + statistics_os_map(map_size); // 2.0.1: rpmalloc.c:1023-1033 + return ptr; +} + +// The failure tail of os_mmap (2.0.1: rpmalloc.c:990-998). +// #jai returns null without asserting when there is no map_fail_callback (finding N1). +os_map_failed :: (size: u64, alignment: u64, offset: *u64, mapped_size: *u64) -> *void { + iface := global_memory_interface; + if iface && iface.map_fail_callback && iface.map_fail_callback(size + alignment) + return os_mmap(size, alignment, offset, mapped_size); + return null; +} + +os_commit_on_demand :: inline () -> bool { // 2.0.1: rpmalloc.c:1040 + #if ENABLE_DECOMMIT return !global_config.disable_decommit; + else return false; +} + +os_mcommit :: (address: *void, size: u64) -> s32 { // 2.0.1: rpmalloc.c:1049 + #if ENABLE_DECOMMIT { + // Gated by callers on the per-span commit_on_demand flag. Spans committed in full at map + // time (including large pages, which cannot be partially committed) never reach here. + failed := false; + #if TEST_HOOKS failed = test_take_fault(*test_fail_commits); + if !failed failed = !VirtualAlloc_w(address, size, MEM_COMMIT_W, PAGE_READWRITE_W); + if failed { + iface := global_memory_interface; + if iface && iface.map_fail_callback && iface.map_fail_callback(size) + return os_mcommit(address, size); + // #jai no assert (finding N1); the caller handles the failure. + return 1; + } + statistics_os_commit(size); // 2.0.1: rpmalloc.c:1067-1078 + } + return 0; +} + +os_mdecommit :: (address: *void, size: u64) -> s32 { // 2.0.1: rpmalloc.c:1086 + #if ENABLE_DECOMMIT { + // Unlike os_mcommit, honor the live flag (the Linux fallback can set it mid-run). + if global_config.disable_decommit return 1; + if !VirtualFree_w(address, size, MEM_DECOMMIT_W) { + rpmalloc_assert(false, "Failed to decommit virtual memory block"); + return 1; + } + statistics_os_decommit(size); // 2.0.1: rpmalloc.c:1129-1133 + } + return 0; +} + +os_munmap :: (address: *void, offset: u64, mapped_size: u64) { // 2.0.1: rpmalloc.c:1142 + address = pointer_offset(address, -cast(s64) offset); + #if ENABLE_UNMAP { + if !VirtualFree_w(address, 0, MEM_RELEASE_W) + rpmalloc_assert(false, "Failed to unmap virtual memory block"); + statistics_os_unmap(mapped_size); // 2.0.1: rpmalloc.c:1154-1158 + } +} + +// VirtualAlloc2 is resolved at run time from kernelbase.dll (Windows 10 and later). +VirtualAlloc2_Proc :: #type (process: *void, base_address: *void, size: u64, allocation_type: u32, page_protection: u32, + parameters: *MEM_EXTENDED_PARAMETER_W, parameter_count: u32) -> *void #c_call; +os_virtualalloc2: VirtualAlloc2_Proc; // 2.0.1: rpmalloc.c:77 + +MEM_ADDRESS_REQUIREMENTS_W :: struct { + LowestStartingAddress: *void; + HighestEndingAddress: *void; + Alignment: u64; +} + +// MEM_EXTENDED_PARAMETER: a 64-bit word (Type:8, Reserved:56) followed by a pointer-sized union. +MEM_EXTENDED_PARAMETER_W :: struct { + Type: u64; + Pointer: *void; +} + +os_thread_yield :: () { // 2.0.1: rpmalloc.c:396 (SwitchToThread) + SwitchToThread_w(); +} + +// Coarse monotonic time for the huge block cache. +os_monotonic_time_ms :: inline () -> u64 { // 2.0.1: rpmalloc.c:1586 + return GetTickCount64_w(); +} + +// +// Thread-local heap storage, thread identity and the thread exit hook. +// +// #jai upstream keeps the thread heap in a C11 _Thread_local variable and uses FLS only for the +// thread exit destructor. This Jai version has no #thread_local, and keeping the heap in the +// Context would let copied contexts share a heap between threads (review H3), so the heap pointer +// lives in an OS TLS slot (TlsAlloc). An FLS slot (FlsAlloc) carries the exit destructor, as +// upstream. Possible performance follow-up: read the TEB slot directly instead of calling TlsGetValue. + +os_tls_create :: (destructor: (value: *void) #c_call) -> bool { + tls_index = TlsAlloc_w(); + if tls_index == TLS_OUT_OF_INDEXES_W return false; + fls_index = FlsAlloc_w(destructor); // 2.0.1: rpmalloc.c:2952 + return true; +} + +os_tls_destroy :: () { + FlsFree_w(fls_index); // 2.0.1: rpmalloc.c:3099 + fls_index = 0; + TlsFree_w(tls_index); + tls_index = TLS_OUT_OF_INDEXES_W; +} + +// #jai fix E10: the calling thread's stack, [low, high). +os_thread_stack_range :: () -> low: u64, high: u64 { + low, high: u64; + GetCurrentThreadStackLimits_w(*low, *high); + return low, high; +} + +os_tls_get :: inline () -> *void { + return TlsGetValue_w(tls_index); +} + +os_tls_set :: (value: *void) { + TlsSetValue_w(tls_index, value); + FlsSetValue_w(fls_index, value); // 2.0.1: rpmalloc.c:803 +} + +// #jai upstream returns the TEB address (NtCurrentTeb, rpmalloc.c:762). GetCurrentThreadId is +// equally unique among live threads, never 0, and needs no inline assembly. +// #jai fix U22: true while the process is exiting. ExitProcess stops every other thread first, so a +// lock one of them held stays held; the thread-exit destructor must not wait for it then. +os_process_exiting :: () -> bool { + return RtlDllShutdownInProgress_w() != 0; +} + +os_thread_id :: inline () -> u64 { + return GetCurrentThreadId_w(); +} + +// Large page detection (2.0.1: rpmalloc.c:2798-2830). +os_detect_huge_pages :: () { + token: *void; + large_page_minimum := GetLargePageMinimum_w(); + if large_page_minimum OpenProcessToken_w(GetCurrentProcess_w(), TOKEN_ADJUST_PRIVILEGES_W | TOKEN_QUERY_W, *token); + if token { + luid: LUID_W; + if LookupPrivilegeValueA_w(null, "SeLockMemoryPrivilege".data, *luid) { + privileges: TOKEN_PRIVILEGES_W; + privileges.PrivilegeCount = 1; + privileges.Privileges[0].Luid = luid; + privileges.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED_W; + if AdjustTokenPrivileges_w(token, 0, *privileges, 0, null, null) { + if GetLastError_w() == 0 os_huge_pages = true; // ERROR_SUCCESS + } + } + CloseHandle_w(token); + } + // Disable huge pages if the large page size is not a power of two that fits the builtin geometry. + if os_huge_pages && ((large_page_minimum & (large_page_minimum - 1)) || large_page_minimum > LARGE_PAGE_SIZE) + os_huge_pages = false; + if os_huge_pages { + if large_page_minimum > os_page_size os_page_size = large_page_minimum; + if large_page_minimum > os_map_granularity os_map_granularity = large_page_minimum; + } +} + +// Transparent huge pages are Linux only (2.0.1: rpmalloc.c:2921-2936). +os_configure_thp :: () { } + +#scope_file + +kernel32_rpm :: #library,system "kernel32"; + +SYSTEM_INFO_W :: struct { + wProcessorArchitecture: u16; + wReserved: u16; + dwPageSize: u32; + lpMinimumApplicationAddress: *void; + lpMaximumApplicationAddress: *void; + dwActiveProcessorMask: u64; + dwNumberOfProcessors: u32; + dwProcessorType: u32; + dwAllocationGranularity: u32; + wProcessorLevel: u16; + wProcessorRevision: u16; +} + +MemExtendedParameterAddressRequirements :: 1; + +MEM_COMMIT_W :: 0x0000_1000; +MEM_RESERVE_W :: 0x0000_2000; +MEM_DECOMMIT_W :: 0x0000_4000; +MEM_RELEASE_W :: 0x0000_8000; +MEM_LARGE_PAGES_W :: 0x2000_0000; +PAGE_READWRITE_W :: 0x04; + +VirtualAlloc_w :: (address: *void, size: u64, type: u32, protect: u32) -> *void #foreign kernel32_rpm "VirtualAlloc"; +VirtualFree_w :: (address: *void, size: u64, type: u32) -> s32 #foreign kernel32_rpm "VirtualFree"; +GetSystemInfo_w :: (info: *SYSTEM_INFO_W) #foreign kernel32_rpm "GetSystemInfo"; +GetModuleHandleA_w :: (name: *u8) -> *void #foreign kernel32_rpm "GetModuleHandleA"; +GetProcAddress_w :: (module: *void, name: *u8) -> *void #foreign kernel32_rpm "GetProcAddress"; +SwitchToThread_w :: () -> s32 #foreign kernel32_rpm "SwitchToThread"; +GetTickCount64_w :: () -> u64 #foreign kernel32_rpm "GetTickCount64"; + +// Thread-local storage and huge page detection. + +TLS_OUT_OF_INDEXES_W :: 0xFFFF_FFFF; +tls_index: u32 = TLS_OUT_OF_INDEXES_W; +fls_index: u32; + +// The Windows LUID is {DWORD LowPart; LONG HighPart}: 4-byte aligned, so LUID_AND_ATTRIBUTES is 12 +// bytes and TOKEN_PRIVILEGES.Privileges starts at offset 4. Declaring the LUID as a u64 shifts it +// to offset 8, which made large pages silently fail in the 1.4.4 Jai port (review M5). +LUID_W :: struct { + LowPart: u32; + HighPart: s32; +} +LUID_AND_ATTRIBUTES_W :: struct { + Luid: LUID_W; + Attributes: u32; +} +TOKEN_PRIVILEGES_W :: struct { + PrivilegeCount: u32; + Privileges: [1] LUID_AND_ATTRIBUTES_W; +} +#assert size_of(TOKEN_PRIVILEGES_W) == 16 "TOKEN_PRIVILEGES layout (review M5)"; + +TOKEN_ADJUST_PRIVILEGES_W :: 0x0020; +TOKEN_QUERY_W :: 0x0008; +SE_PRIVILEGE_ENABLED_W :: 0x0002; + +kernel32_tls :: #library,system "kernel32"; +GetCurrentThreadStackLimits_w :: (low: *u64, high: *u64) #foreign kernel32_tls "GetCurrentThreadStackLimits"; +advapi32_tls :: #library,system "advapi32"; +ntdll_tls :: #library,system "ntdll"; +RtlDllShutdownInProgress_w :: () -> u8 #foreign ntdll_tls "RtlDllShutdownInProgress"; + +TlsAlloc_w :: () -> u32 #foreign kernel32_tls "TlsAlloc"; +TlsFree_w :: (index: u32) -> s32 #foreign kernel32_tls "TlsFree"; +TlsGetValue_w :: (index: u32) -> *void #foreign kernel32_tls "TlsGetValue"; +TlsSetValue_w :: (index: u32, value: *void) -> s32 #foreign kernel32_tls "TlsSetValue"; +FlsAlloc_w :: (callback: (value: *void) #c_call) -> u32 #foreign kernel32_tls "FlsAlloc"; +FlsFree_w :: (index: u32) -> s32 #foreign kernel32_tls "FlsFree"; +FlsSetValue_w :: (index: u32, value: *void) -> s32 #foreign kernel32_tls "FlsSetValue"; +GetCurrentThreadId_w :: () -> u32 #foreign kernel32_tls "GetCurrentThreadId"; +GetCurrentProcess_w :: () -> *void #foreign kernel32_tls "GetCurrentProcess"; +GetLargePageMinimum_w :: () -> u64 #foreign kernel32_tls "GetLargePageMinimum"; +GetLastError_w :: () -> u32 #foreign kernel32_tls "GetLastError"; +CloseHandle_w :: (handle: *void) -> s32 #foreign kernel32_tls "CloseHandle"; +OpenProcessToken_w :: (process: *void, access: u32, token: **void) -> s32 #foreign advapi32_tls "OpenProcessToken"; +LookupPrivilegeValueA_w :: (system: *u8, name: *u8, luid: *LUID_W) -> s32 #foreign advapi32_tls "LookupPrivilegeValueA"; +AdjustTokenPrivileges_w :: (token: *void, disable_all: s32, new_state: *TOKEN_PRIVILEGES_W, buffer_length: u32, previous: *void, return_length: *u32) -> s32 #foreign advapi32_tls "AdjustTokenPrivileges"; diff --git a/modules/Default_Allocator/rpmalloc.jai b/modules/Default_Allocator/rpmalloc.jai new file mode 100644 index 0000000..7974ddb --- /dev/null +++ b/modules/Default_Allocator/rpmalloc.jai @@ -0,0 +1,3490 @@ +// The allocator core: a port of rpmalloc 2.0.1 rpmalloc.c. +// +// Layout follows the C file section by section, in the same order. Procedures keep the C name +// (minus rpmalloc_ prefixes) and carry a `// 2.0.1: rpmalloc.c:NNNN` reference, so the port can be +// compared with upstream and later upstream fixes applied mechanically. +// +// Deliberate differences from C are marked `// #jai` with the reason. Known upstream bugs that are +// ported as-is, until their fix lands as a separate commit, are marked `// UPSTREAM-BUG `; the +// ids and the fix order are in PLAN.md. + +//////////// +/// +/// Build time configurable limits (2.0.1: rpmalloc.c:135-190) +/// +////// +// Module parameters replace the C #defines (ENABLE_VALIDATE_ARGS, ENABLE_ASSERTS, ENABLE_UNMAP, +// ENABLE_DECOMMIT, ENABLE_STATISTICS, ENABLE_LEAK_DETECTION); see module.jai. ENABLE_OVERRIDE +// and ENABLE_DYNAMIC_LINK are C-only and are not ported. + +//////////// +/// +/// Built in size configurations (2.0.1: rpmalloc.c:191-241) +/// +////// + +PAGE_HEADER_SIZE :: 128; // 2.0.1: rpmalloc.c:197 +SPAN_HEADER_SIZE :: PAGE_HEADER_SIZE; // 2.0.1: rpmalloc.c:198 + +// Alignment of each heap control structure within a shared mapping, so adjacent heaps sit on +// separate cache lines (128 bytes covers Apple Silicon and x86 destructive interference). +HEAP_ALIGNMENT :: 128; // 2.0.1: rpmalloc.c:204 + +SMALL_GRANULARITY :: 16; // 2.0.1: rpmalloc.c:206 + +SMALL_BLOCK_SIZE_LIMIT :: 4 * 1024; // 2.0.1: rpmalloc.c:208 +MEDIUM_SMALL_BLOCK_SIZE_LIMIT :: 32 * 1024; // 2.0.1: rpmalloc.c:209 +MEDIUM_LARGE_BLOCK_SIZE_LIMIT :: 256 * 1024; // 2.0.1: rpmalloc.c:210 +LARGE_BLOCK_SIZE_LIMIT :: 2 * 1024 * 1024; // 2.0.1: rpmalloc.c:211 +NATURAL_ALIGNMENT_LIMIT :: 2048; // #jai upstream PR #424: largest alignment a class gets from its start offset. + +SMALL_SIZE_CLASS_COUNT :: 73; // 2.0.1: rpmalloc.c:213 +MEDIUM_SMALL_SIZE_CLASS_COUNT :: 12; // 2.0.1: rpmalloc.c:214 +MEDIUM_LARGE_SIZE_CLASS_COUNT :: 12; // 2.0.1: rpmalloc.c:215 +LARGE_SIZE_CLASS_COUNT :: 12; // 2.0.1: rpmalloc.c:216 +SIZE_CLASS_COUNT :: SMALL_SIZE_CLASS_COUNT + MEDIUM_SMALL_SIZE_CLASS_COUNT + MEDIUM_LARGE_SIZE_CLASS_COUNT + LARGE_SIZE_CLASS_COUNT; // 2.0.1: rpmalloc.c:217 + +SMALL_PAGE_SIZE_SHIFT :: 16; // 2.0.1: rpmalloc.c:220 +SMALL_PAGE_SIZE :: 1 << SMALL_PAGE_SIZE_SHIFT; +SMALL_PAGE_MASK :: ~cast(u64)(SMALL_PAGE_SIZE - 1); +MEDIUM_SMALL_PAGE_SIZE_SHIFT :: 20; // 2.0.1: rpmalloc.c:223 +MEDIUM_SMALL_PAGE_SIZE :: 1 << MEDIUM_SMALL_PAGE_SIZE_SHIFT; +MEDIUM_SMALL_PAGE_MASK :: ~cast(u64)(MEDIUM_SMALL_PAGE_SIZE - 1); +MEDIUM_LARGE_PAGE_SIZE_SHIFT :: 22; // 2.0.1: rpmalloc.c:226 +MEDIUM_LARGE_PAGE_SIZE :: 1 << MEDIUM_LARGE_PAGE_SIZE_SHIFT; +MEDIUM_LARGE_PAGE_MASK :: ~cast(u64)(MEDIUM_LARGE_PAGE_SIZE - 1); +LARGE_PAGE_SIZE_SHIFT :: 24; // 2.0.1: rpmalloc.c:229 +LARGE_PAGE_SIZE :: 1 << LARGE_PAGE_SIZE_SHIFT; +LARGE_PAGE_MASK :: ~cast(u64)(LARGE_PAGE_SIZE - 1); + +SPAN_SIZE :: 256 * 1024 * 1024; // 2.0.1: rpmalloc.c:233 +SPAN_MASK :: ~cast(u64)(SPAN_SIZE - 1); // 2.0.1: rpmalloc.c:234 +SPAN_SIZE_SHIFT :: 28; // #jai for the IS_THIS_YOURS span registry +#assert (1 << SPAN_SIZE_SHIFT) == SPAN_SIZE; + +// Maximum allocation size to avoid integer overflow (2.0.1: rpmalloc.c:236-240; upstream only +// defines it with ENABLE_VALIDATE_ARGS). +MAX_ALLOC_SIZE :: 0xFFFF_FFFF_FFFF_FFFF - cast(u64) SPAN_SIZE; + +// Geometry invariants. Upstream has none, so changing SPAN_SIZE silently breaks the allocator +// (upstream issue #364). #jai checked at compile time. +#assert (SMALL_PAGE_SIZE & (SMALL_PAGE_SIZE - 1)) == 0 && (SPAN_SIZE & (SPAN_SIZE - 1)) == 0 "page and span sizes must be powers of two"; +#assert SMALL_PAGE_SIZE < MEDIUM_SMALL_PAGE_SIZE && MEDIUM_SMALL_PAGE_SIZE < MEDIUM_LARGE_PAGE_SIZE && MEDIUM_LARGE_PAGE_SIZE < LARGE_PAGE_SIZE "page sizes must increase by page type"; +#assert LARGE_PAGE_SIZE <= SPAN_SIZE "a span must hold at least one large page"; +#assert SPAN_SIZE <= 0xFFFF_FFFF "page offsets within a span must fit in 32 bits"; +#assert RPMALLOC_MAX_ALIGNMENT < LARGE_PAGE_SIZE "huge aligned blocks must stay within the span header's reach"; +#assert PAGE_HEADER_SIZE % SMALL_GRANULARITY == 0 "blocks must start 16-byte aligned"; + +//////////// +/// +/// Utility macros (2.0.1: rpmalloc.c:242-281) +/// +////// +// EXPECTED / UNEXPECTED have no Jai equivalent and are dropped. + +pointer_offset :: inline (ptr: *void, offset: $T) -> *void { // 2.0.1: rpmalloc.c:132 + return cast(*u8) ptr + cast,no_check(s64) offset; +} + +pointer_diff :: inline (first: *void, second: *void) -> s64 { // 2.0.1: rpmalloc.c:133 + return cast(*u8) first - cast(*u8) second; +} + +// 2.0.1: rpmalloc.c:256-268. #jai reports through the memory interface's error_callback when one +// is set (upstream declares that callback but never calls it), otherwise through +// context.assertion_failed. +#if ENABLE_ASSERTS { + rpmalloc_assert :: (truth: bool, message: string, loc := #caller_location) { + if truth return; + iface := global_memory_interface; + if iface && iface.error_callback { + iface.error_callback(message); + return; + } + context.handling_assertion_failure = true; + context.assertion_failed(loc, message); + context.handling_assertion_failure = false; + } +} else { + rpmalloc_assert :: ($truth: Code, message: string) #expand {} +} + +//////////// +/// +/// Statistics (2.0.1: rpmalloc.c:282-360) +/// +////// + +#if ENABLE_STATISTICS { + +Rpmalloc_Statistics :: struct { // 2.0.1: rpmalloc.c:290 + page_mapped: u64; + page_mapped_peak: u64; + page_commit: u64; + page_decommit: u64; + page_active: u64; + page_active_peak: u64; + huge_alloc: u64; + huge_alloc_peak: u64; + heap_count: u64; +} + +global_counters: Rpmalloc_Statistics; // 2.0.1: rpmalloc.c:302 global_statistics (renamed: global_statistics is the public procedure) + +// Subtracts from a counter, clamping at zero. Counters are in units of the configured page size, +// and heaps with committed pages are kept across finalize and initialize cycles where the page +// size can change, so strict accounting is not possible. +statistics_sub_saturating :: (counter: *u64, value: u64) { // 2.0.1: rpmalloc.c:308 + current := atomic_load(counter); + while true { + next := ifx current > value then current - value else 0; + swapped, actual := compare_and_swap(counter, current, next); + if swapped break; + current = actual; + } +} + +// #jai the add-then-raise-the-peak loop that upstream writes out at each site. +statistics_add_with_peak :: (counter: *u64, peak: *u64, value: u64) { + current := atomic_add(counter, value) + value; + observed := atomic_load(peak); + while current > observed { + swapped, actual := compare_and_swap(peak, observed, current); + if swapped break; + observed = actual; + } +} + +// Per size class allocation counters, updated only by the owning thread (cross-thread frees are +// deferred and counted when the owner processes them), so plain integers suffice. +// #jai fix U13: 64-bit; upstream's 32-bit counters wrap on long runs (undefined behaviour in C). +Heap_Size_Use :: struct { // 2.0.1: rpmalloc.c:319 + alloc_current: s64; + alloc_peak: s64; + alloc_total: s64; + free_total: s64; +} + +} // #if ENABLE_STATISTICS + +// #jai fix U13: heap statistics are updated by the threads that free into a heap too, so every +// update is atomic (upstream used plain += and -=). Only HEAP_STATISTICS builds pay for it. +// Subtraction adds the two's complement, and a counter can be below zero for a moment (the first +// allocation already sees -16), so the sums wrap on purpose, as upstream's unsigned arithmetic does. +heap_stat_add :: inline (field: *u64, value: u64) { atomic_add_wrapping(field, value); } +heap_stat_sub :: inline (field: *u64, value: u64) { atomic_add_wrapping(field, cast,no_check(u64) -cast,no_check(s64) value); } + +// #jai Jai's atomic_add wraps on X64 (lock xadd), but on other CPUs it computes old + value in Jai, +// which traps in a program built with arithmetic_overflow_check = .FATAL. +atomic_add_wrapping :: inline (field: *u64, value: u64) { + #if CPU == .X64 { + atomic_add(field, value); + } else { + old := atomic_load(field); + while true { + new := ifx old > 0xFFFF_FFFF_FFFF_FFFF - value then old - (0xFFFF_FFFF_FFFF_FFFF - value) - 1 else old + value; + swapped, actual := compare_and_swap(field, old, new); + if swapped break; + old = actual; + } + } +} + +// The _rpmalloc_stat_* macros (2.0.1: rpmalloc.c:327-360); no-ops without ENABLE_STATISTICS. +stat_inc_alloc :: inline (heap: *Heap, class_idx: u32) { + #if ENABLE_STATISTICS { + su := *heap.size_use[class_idx]; + su.alloc_current += 1; + if su.alloc_current > su.alloc_peak su.alloc_peak = su.alloc_current; + su.alloc_total += 1; + } +} + +stat_add_free :: inline (heap: *Heap, class_idx: u32, count: u32) { + #if ENABLE_STATISTICS { + su := *heap.size_use[class_idx]; + su.alloc_current -= cast(s64) count; + su.free_total += cast(s64) count; + } +} + +stat_inc_span_map :: inline (heap: *Heap, page_type: Page_Type) { + #if ENABLE_STATISTICS heap.span_map_calls[cast(u32) page_type] += 1; +} + +// The OS layer's counters (2.0.1: rpmalloc.c:1023-1033, 1067-1078, 1129-1133, 1154-1158). +// #jai each skips counting while the page size is still unknown: the white-box tests (W-05, W-06) +// call the OS layer before initialisation, which upstream never does. +statistics_os_map :: inline (map_size: u64) { + if !global_config.page_size return; + #if ENABLE_STATISTICS statistics_add_with_peak(*global_counters.page_mapped, *global_counters.page_mapped_peak, map_size / global_config.page_size); +} + +statistics_os_commit :: inline (size: u64) { + if !global_config.page_size return; + #if ENABLE_STATISTICS { + page_count := size / global_config.page_size; + atomic_add(*global_counters.page_commit, page_count); + statistics_add_with_peak(*global_counters.page_active, *global_counters.page_active_peak, page_count); + } +} + +statistics_os_decommit :: inline (size: u64) { + if !global_config.page_size return; + #if ENABLE_STATISTICS { + page_count := size / global_config.page_size; + atomic_add(*global_counters.page_decommit, page_count); + statistics_sub_saturating(*global_counters.page_active, page_count); + } +} + +statistics_os_unmap :: inline (mapped_size: u64) { + if !global_config.page_size return; + #if ENABLE_STATISTICS { + page_count := mapped_size / global_config.page_size; + statistics_sub_saturating(*global_counters.page_mapped, page_count); + statistics_sub_saturating(*global_counters.page_active, page_count); + } +} + +//////////// +/// +/// Low level abstractions (2.0.1: rpmalloc.c:361-476) +/// +////// +// memset_const / memcpy_const are plain memset / memcpy. + +// Yield the rest of this thread's quantum, so another runnable thread (for example a lock holder +// that was preempted) can run. A real scheduler yield, unlike the CPU hint in wait_spin. +thread_yield :: inline () { // 2.0.1: rpmalloc.c:396 + os_thread_yield(); +} + +// Number of cheap CPU-pause spins before wait_spin escalates to an OS scheduler yield. +WAIT_SPIN_YIELD_THRESHOLD :: 100; // 2.0.1: rpmalloc.c:406 + +wait_spin :: inline (spin_count: *u32) { // 2.0.1: rpmalloc.c:410 + spin_count.* += 1; + if spin_count.* >= WAIT_SPIN_YIELD_THRESHOLD { + thread_yield(); + return; + } + #if CPU == .X64 { + #asm { pause; } + } else #if CPU == .ARM64 { + #bytes .[0x3F, 0x20, 0x03, 0xD5]; // YIELD + } +} + +// #jai Basic has no memmove, and faking it with memcpy was review finding M7. Only the +// realloc-of-an-aligned-block paths move overlapping data, so a simple version is enough. +memmove_rp :: (dest: *void, source: *void, count: s64) { + if dest == source || count <= 0 return; + d := cast(*u8) dest; + s := cast(*u8) source; + if d + count <= s || s + count <= d { + memcpy(d, s, count); + } else if d < s { + for i: 0..count-1 d[i] = s[i]; + } else { + for < i: count-1..0 d[i] = s[i]; + } +} + +// Index of the most significant set bit of a non-zero value. Replaces rpmalloc_clz +// (2.0.1: rpmalloc.c:368), which get_size_class uses as 63 - clz(x). +// #jai bsr on x64, which needs no LZCNT support (the MSVC build's __lzcnt64 silently gives wrong +// answers on CPUs without LZCNT, see upstream issue #404). Elsewhere a six-step binary search, +// because Bit_Operations' non-x64 fallback scans bit by bit. +msb_index :: inline (x: u64) -> u32 { + #if CPU == .X64 { + return cast(u32)(bit_scan_reverse(x) - 1); + } else { + n: u32 = 0; + if x >= 1 << 32 { x >>= 32; n += 32; } + if x >= 1 << 16 { x >>= 16; n += 16; } + if x >= 1 << 8 { x >>= 8; n += 8; } + if x >= 1 << 4 { x >>= 4; n += 4; } + if x >= 1 << 2 { x >>= 2; n += 2; } + if x >= 1 << 1 { n += 1; } + return n; + } +} + +//////////// +/// +/// Data types (2.0.1: rpmalloc.c:477-637) +/// +////// + +Page_Type :: enum u8 { // 2.0.1: rpmalloc.c:495 page_type_t + SMALL; // 64 KiB pages + MEDIUM_SMALL; // 1 MiB pages + MEDIUM_LARGE; // 4 MiB pages + LARGE; // 16 MiB pages + HUGE; +} + +Size_Class :: struct { // 2.0.1: rpmalloc.c:504 size_class_t + block_size: u32; // Size of blocks in this class. + block_count: u32; // Number of blocks in each page. +} + + +Block :: struct { // 2.0.1: rpmalloc.c:512 block_t + next: *Block; +} + +// The packed flag word of page_t (2.0.1: rpmalloc.c:532-552). Bits 0-6 are flags; bits 8-13 hold +// commit_prefix_shift (see page_commit_prefix_shift). Packing keeps Page at 72 bytes and Span at +// exactly SPAN_HEADER_SIZE, as upstream. +Page_Flags :: enum_flags u32 { + IS_FULL; // Part of the heap full list. + IS_FREE; // Part of the heap free list. + IS_ZERO; // Blocks are zero initialised. + IS_DECOMMITTED; // Memory pages have been decommitted. + // Has aligned blocks. Note: has_aligned_block and is_full are read on the cross-thread free + // path while the owner thread writes this word. The owner is the only writer, the word is + // naturally aligned and the flags read are monotonic/tolerant, so the race is benign + // (upstream comment, rpmalloc.c:539-545). + HAS_ALIGNED_BLOCK; + COMMIT_ON_DEMAND; // The span maps reserve-only; pages commit on demand and may be decommitted. +} +COMMIT_PREFIX_SHIFT_BIT :: 8; +COMMIT_PREFIX_SHIFT_MASK :: 0x3F; + +Page :: struct { // 2.0.1: rpmalloc.c:518 page_t + size_class: u32; + block_size: u32; + block_count: u32; + block_initialized: u32; + block_used: u32; + // Flags, commit_prefix_shift and (#jai PR #422) the page type, see Page_Flags. The page type moved + // into this word, as upstream's PR #422 does, so Page.heap_fast fits: Span stays 128 bytes. + flags: Page_Flags; + local_free: *Block; // Local free list. + heap: *Heap; // Owning heap. + // #jai upstream PR #422 (867ea07): the owning heap while the page takes the fast free path, null + // for huge, full or aligned-block pages, so the free fast path checks ownership and eligibility + // with one compare. (Replaces 2.0.1's generic_free flag and local_free_count, which always equals + // block_initialized - block_used.) + heap_fast: *Heap; + next: *Page; + prev: *Page; + // Multithreaded free list: block index in the low 32 bits, list count in the high 32 bits. + thread_free: u64; +} + +Span :: struct { // 2.0.1: rpmalloc.c:568 span_t + page: Page; // Page header of page 0. + // Owning heap. Not deduplicated into page.heap, which is the owner of page 0 as a memory page, + // while the span owner must stay stable. + heap: *Heap; + page_address_mask: u64; + page_initialized: u32; // Number of pages initialized. + page_count: u32; // Number of pages in use (huge: number of OS pages). + page_size: u32; // Bytes per page. + page_type: Page_Type; + offset: u32; // Offset to the start of the mapped region. + mapped_size: u64; + next: *Span; +} + +// Control structure for a heap: a thread heap, or a first-class heap. +Heap :: struct { // 2.0.1: rpmalloc.c:594 heap_t + // #jai develop #409 (3302251): the first HEAP_ALIGNMENT bytes hold the fields other threads + // access. owner_thread is read and thread_free is written by every cross-thread free into a full + // page; keeping them apart from the owner's hot allocation state avoids invalidating the owner's + // local free lists on every remote free, and the remote CAS line being pulled away by owner writes. + owner_thread: u64; // Owning thread id (0: first-class heap). + thread_free: [4] u64; // Multithreaded free list (atomic) per type. + // #jai fix E6: taken by frees into the heap while it is released (E4), and by the thread adopting + // it, so those frees no longer take the global heap lock. On the line other threads write anyway. + free_lock: u64; + remote_pad: [HEAP_ALIGNMENT - 2 * size_of(u64) - 4 * size_of(u64)] u8; + local_free: [SIZE_CLASS_COUNT] *Block; // Heap local free list per size class. + page_available: [SIZE_CLASS_COUNT] *Page; // Non-full pages per size class. + page_free: [4] *Page; // Free pages per page type, committed pages first. + page_free_tail: [4] *Page; // #jai PR #420: last page of each free list. + page_free_commit_count: [4] u32; // Free but still committed pages per type. + // #jai develop #416: when (monotonic ms) the free committed page count of each page type first + // went over the overflow threshold; zero while below it. + page_free_overflow_ms: [4] u64; + span_partial: [4] *Span; // Partially initialized spans per type. + span_used: [5] *Span; // Spans in full use per type. + next: *Heap; + prev: *Heap; + id: u32; + finalize: u32; + release_trim: u32; // #jai develop #419: released untrimmed; the adopting thread trims it first. + released: bool; // #jai fix U14: queued or back in the pristine list; set under the heap lock. + // #jai fix E7: the id of the thread that parked the heap with thread_idle, 0 when not parked. Set + // and cleared under the global heap lock and the heap's free_lock. + parked_thread: u64; + offset: u32; // Memory map region offset. + mapped_size: u64; // Memory map size. + #if HEAP_STATISTICS stats: Heap_Statistics; // 2.0.1: rpmalloc.c:624 + #if ENABLE_STATISTICS { + size_use: [SIZE_CLASS_COUNT] Heap_Size_Use; // 2.0.1: rpmalloc.c:628 + span_map_calls: [5] u32; // 2.0.1: rpmalloc.c:630 + } + // #jai handed out by heap_acquire. IS_THIS_YOURS on the default allocator must not claim the + // blocks of first-class heaps; owner_thread cannot tell them apart from released thread heaps. + is_first_class: bool; + // #jai fix E10: for the Context cache of the thread heap. The stack range of the thread whose heap + // this is, and whether it is that thread's heap now (cleared when the thread parks the heap, sets + // another current, or releases it). Written by the owning thread. + stack_low: u64; + stack_high: u64; + thread_current: bool; +} + +#assert size_of(Page) <= PAGE_HEADER_SIZE "Invalid page header size"; // 2.0.1: rpmalloc.c:634 +#assert #run every_class_fits_after_its_start_offset() "PR #424: a size class no longer fits its page after its block start offset"; + +every_class_fits_after_its_start_offset :: () -> bool { + page_size_of :: (index: s64) -> u64 { + if index < SMALL_SIZE_CLASS_COUNT return SMALL_PAGE_SIZE; + if index < SMALL_SIZE_CLASS_COUNT + MEDIUM_SMALL_SIZE_CLASS_COUNT return MEDIUM_SMALL_PAGE_SIZE; + if index < SMALL_SIZE_CLASS_COUNT + MEDIUM_SMALL_SIZE_CLASS_COUNT + MEDIUM_LARGE_SIZE_CLASS_COUNT return MEDIUM_LARGE_PAGE_SIZE; + return LARGE_PAGE_SIZE; + } + for global_size_class { + if it.block_size && cast(u64) it.block_count * it.block_size + block_start_offset(it.block_size) > page_size_of(it_index) return false; + } + return true; +} +#assert size_of(Span) <= SPAN_HEADER_SIZE "Invalid span header size"; // 2.0.1: rpmalloc.c:635 +// 2.0.1: rpmalloc.c:636. #jai with ENABLE_STATISTICS the 64-bit size class counters (fix U13) make the +// heap larger than 4 KiB; heap_allocate_new rounds each heap up to whole pages, so a statistics build +// simply uses more of the mapping per heap. +#assert ENABLE_STATISTICS || size_of(Heap) <= 4096 "Invalid heap size"; +#assert #run heap_member_offset("local_free") == HEAP_ALIGNMENT "Remotely accessed heap fields must fill one line"; // develop #409 + +heap_member_offset :: (name: string) -> s64 { + for type_info(Heap).members if it.name == name return it.offset_in_bytes; + return -1; +} + +page_has :: inline (page: *Page, flag: Page_Flags) -> bool { + return (page.flags & flag) != 0; +} + +page_set :: inline (page: *Page, flag: Page_Flags, on: bool) { + if on page.flags |= flag; + else page.flags &= ~flag; +} + +PAGE_TYPE_BIT :: 16; +PAGE_TYPE_MASK :: 0x7; + +page_get_type :: inline (page: *Page) -> Page_Type { + return cast(Page_Type)(((cast(u32) page.flags) >> PAGE_TYPE_BIT) & PAGE_TYPE_MASK); +} + +page_set_type :: inline (page: *Page, page_type: Page_Type) { + bits := (cast(u32) page.flags) & ~((cast(u32) PAGE_TYPE_MASK) << PAGE_TYPE_BIT); + page.flags = cast(Page_Flags)(bits | ((cast(u32) page_type) << PAGE_TYPE_BIT)); +} + +page_commit_prefix_shift :: inline (page: *Page) -> u32 { + return ((cast(u32) page.flags) >> COMMIT_PREFIX_SHIFT_BIT) & COMMIT_PREFIX_SHIFT_MASK; +} + +page_set_commit_prefix_shift :: inline (page: *Page, shift: u32) { + bits := (cast(u32) page.flags) & ~((cast(u32) COMMIT_PREFIX_SHIFT_MASK) << COMMIT_PREFIX_SHIFT_BIT); + page.flags = cast(Page_Flags)(bits | ((shift & COMMIT_PREFIX_SHIFT_MASK) << COMMIT_PREFIX_SHIFT_BIT)); +} + +// #jai fix E3: commit a page as it fills. Upstream commits a whole page (64 KiB to 16 MiB) when it is +// first used, so a page holding a few long-lived blocks at its start kept all of it committed: in Ex4 +// 1.45.0, 84 MB of pages for 8 MB of live blocks, left by 24 short-lived walk threads, whose pages had +// been carved only to 11 MB (RET-17). A page of a reserve-only span is committed in chunks of 1/16 of +// the page, at least 64 KiB (so a small page is one chunk, as before), as page_initialize_blocks carves +// blocks into it. The number of committed chunks is kept in the flags word (Span has no room for a +// field): 0 means only the commit prefix, the first OS page holding the header, is committed (a +// decommitted page). It survives reuse of the page until the page is decommitted. +COMMITTED_CHUNKS_BIT :: 20; +COMMITTED_CHUNKS_MASK :: 0x1F; +COMMIT_CHUNKS_PER_PAGE :: 16; +COMMIT_CHUNK_MIN :: 64 * 1024; + +page_committed_chunks :: inline (page: *Page) -> u32 { + return ((cast(u32) page.flags) >> COMMITTED_CHUNKS_BIT) & COMMITTED_CHUNKS_MASK; +} + +page_set_committed_chunks :: inline (page: *Page, chunks: u32) { + bits := (cast(u32) page.flags) & ~((cast(u32) COMMITTED_CHUNKS_MASK) << COMMITTED_CHUNKS_BIT); + page.flags = cast(Page_Flags)(bits | ((chunks & COMMITTED_CHUNKS_MASK) << COMMITTED_CHUNKS_BIT)); +} + +// The commit chunk of a page of this size: 1/16 of it, at least COMMIT_CHUNK_MIN and the commit prefix, +// at most the page. +page_commit_chunk_size :: inline (page_size: u64, commit_prefix: u64) -> u64 { + chunk := page_size / COMMIT_CHUNKS_PER_PAGE; + if chunk < COMMIT_CHUNK_MIN chunk = COMMIT_CHUNK_MIN; + if chunk < commit_prefix chunk = commit_prefix; + if chunk > page_size chunk = page_size; + return chunk; +} + +// Commits the page up to `extent` bytes from its start (rounded up to a chunk), if it is not yet. +// Non-zero on failure, with nothing changed. Only for pages of reserve-only spans. +page_commit_to :: (page: *Page, extent: u64) -> s32 { + page_size := page_get_size(page); + commit_prefix := cast(u64) 1 << page_commit_prefix_shift(page); + chunk := page_commit_chunk_size(page_size, commit_prefix); + needed := cast(u32)((extent + chunk - 1) / chunk); + have := page_committed_chunks(page); + if needed <= have return 0; + from := ifx have then cast(u64) have * chunk else commit_prefix; + to := cast(u64) needed * chunk; + if to > page_size to = page_size; + if to > from { + if global_memory_interface.memory_commit(pointer_offset(page, from), to - from) != 0 return 1; + #if HEAP_STATISTICS && ENABLE_DECOMMIT { + if page.heap heap_stat_add(*page.heap.stats.committed_size, to - from); + } + } + page_set_committed_chunks(page, needed); + return 0; +} + +// Bytes of the page committed now: what decommit gives back beyond the prefix. +page_committed_size :: inline (page: *Page) -> u64 { + page_size := page_get_size(page); + commit_prefix := cast(u64) 1 << page_commit_prefix_shift(page); + have := page_committed_chunks(page); + if !have return commit_prefix; + size := cast(u64) have * page_commit_chunk_size(page_size, commit_prefix); + return ifx size < page_size then size else page_size; +} + +//////////// +/// +/// Global data (2.0.1: rpmalloc.c:638-728) +/// +////// + +global_heap_id: u32 = 1; // 2.0.1: rpmalloc.c:664 (atomic) Heap id counter. + +// Size classes (2.0.1: rpmalloc.c:685-709). The table is the same list of block sizes in units of +// SMALL_GRANULARITY as upstream's SCLASS/MSCLASS/MLCLASS/LCLASS entries; block counts are computed +// the same way, (page size of the class's page type - PAGE_HEADER_SIZE) / block size. +SIZE_CLASS_UNITS :: u32.[ + 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, + 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 80, 96, 112, 128, 160, 192, 224, 256, // SCLASS + 320, 384, 448, 512, 640, 768, 896, 1024, 1280, 1536, 1792, 2048, // MSCLASS + 2560, 3072, 3584, 4096, 5120, 6144, 7168, 8192, 10240, 12288, 14336, 16384, // MLCLASS + 20480, 24576, 28672, 32768, 40960, 49152, 57344, 65536, 81920, 98304, 114688, 131072, // LCLASS +]; +#assert SIZE_CLASS_UNITS.count == SIZE_CLASS_COUNT "size class table must have SIZE_CLASS_COUNT entries"; + +global_size_class :: #run make_size_class_table(); // 2.0.1: rpmalloc.c:693 + +make_size_class_table :: () -> [SIZE_CLASS_COUNT] Size_Class { + table: [SIZE_CLASS_COUNT] Size_Class; + for units, index: SIZE_CLASS_UNITS { + block_size := units * SMALL_GRANULARITY; + page_size: u32; + if index < SMALL_SIZE_CLASS_COUNT page_size = SMALL_PAGE_SIZE; + else if index < SMALL_SIZE_CLASS_COUNT + MEDIUM_SMALL_SIZE_CLASS_COUNT page_size = MEDIUM_SMALL_PAGE_SIZE; + else if index < SMALL_SIZE_CLASS_COUNT + MEDIUM_SMALL_SIZE_CLASS_COUNT + MEDIUM_LARGE_SIZE_CLASS_COUNT page_size = MEDIUM_LARGE_PAGE_SIZE; + else page_size = LARGE_PAGE_SIZE; + table[index] = .{block_size, (page_size - PAGE_HEADER_SIZE) / block_size}; + } + return table; +} + +// Threshold number of free pages per page type at which free pages are decommitted, and the +// number of pages to retain when that happens (2.0.1: rpmalloc.c:711-715). Not constant: +// initialisation adjusts the thresholds to the OS page size. +global_page_free_overflow: [5] u32 = .[16, 8, 4, 2, 0]; +global_page_free_retain: [5] u32 = .[4, 2, 1, 1, 0]; + +global_memory_interface: *Memory_Interface; // 2.0.1: rpmalloc.c:676 +global_memory_interface_default: Memory_Interface; // 2.0.1: rpmalloc.c:678 +global_memory_interface_copy: Memory_Interface; // #jai fix U8: the caller's interface, with OS defaults filled in. +global_config: Config; // 2.0.1: rpmalloc.c:680 + +os_huge_pages: bool; // 2.0.1: rpmalloc.c:718 Huge (large) pages are in use. +os_thp: bool; // 2.0.1: rpmalloc.c:721 Transparent huge pages are in use (Linux). +os_huge_page_shift: u64; // 2.0.1: rpmalloc.c:723 log2 of the huge page size, for MAP_HUGE_SHIFT. +os_map_granularity: u64; // 2.0.1: rpmalloc.c:725 Granularity of OS mappings. +os_page_size: u64; // 2.0.1: rpmalloc.c:727 OS page size (0 until os_initialize). + +//////////// +/// +/// Thread local heap and ID (2.0.1: rpmalloc.c:729-877) +/// +////// +// #jai the thread heap lives in OS thread-local storage (see os_tls_* in os_windows.jai / +// os_posix.jai): this Jai version has no #thread_local, and the Context can be copied between +// threads (review H3). +// The fallback heap: every thread starts on it. It is all zero (id 0), so the allocation fast path +// finds nothing and drops into the slow path, which initialises the allocator and the thread's real +// heap (heap_get_page_generic and heap_allocate_block_huge check heap.id == 0). +global_heap_fallback: Heap; // 2.0.1: rpmalloc.c:645 +global_heap_default := *global_heap_fallback; // 2.0.1: rpmalloc.c:647 +global_heap_queue: *Heap; // 2.0.1: rpmalloc.c:650 Released heaps; only handed to thread heaps. +global_heap_pristine: *Heap; // 2.0.1: rpmalloc.c:653 Never-used heaps carved from a shared block. +global_heap_used: *Heap; // 2.0.1: rpmalloc.c:655 Heaps in use. +// Set while a thread maps and carves a new shared heap block, so concurrent thread starts wait for +// the carved heaps instead of each mapping a block. A user memory interface callback must not +// allocate through rpmalloc on the creating thread: that would wait on itself. +global_heap_creating: s32; // 2.0.1: rpmalloc.c:660 +global_heap_lock: u64; // 2.0.1: rpmalloc.c:662 (atomic) + +// Global initialisation state machine (2.0.1: rpmalloc.c:665-674). The first allocation on any +// thread may initialise, so exactly one thread does the global setup while the others wait; none +// may see a half-written global_config. +RPMALLOC_INIT_UNINIT :: 0; +RPMALLOC_INIT_RUNNING :: 1; +RPMALLOC_INIT_DONE :: 2; +global_rpmalloc_init_state: s32; // atomic +global_main_thread_id: u64; // 2.0.1: rpmalloc.c:682 + +// Fast thread id (2.0.1: rpmalloc.c:760). #jai see os_thread_id: GetCurrentThreadId / pthread_self +// instead of the TEB / thread pointer. +get_thread_id :: inline () -> u64 { + return os_thread_id(); +} + +// Set the current thread heap (2.0.1: rpmalloc.c:796). +set_thread_heap :: (heap: *Heap) { + // #jai fix E10: the heap this thread had is no longer its heap, whatever Context caches it. + if atomic_load(*global_rpmalloc_init_state) == RPMALLOC_INIT_DONE { + previous := cast(*Heap) os_tls_get(); + if previous && previous != heap previous.thread_current = false; + } + if heap && heap.id != 0 { + rpmalloc_assert(heap.id != 0, "Default heap being used"); + heap.owner_thread = get_thread_id(); + heap.stack_low, heap.stack_high = os_thread_stack_range(); // #jai fix E10 + heap.thread_current = true; + } + // #jai one OS TLS store carries both the heap pointer (upstream's _Thread_local variable) and + // the FLS / pthread key value that drives the thread exit destructor. + os_tls_set(heap); +} + +get_thread_heap_allocate :: () -> *Heap { // 2.0.1: rpmalloc.c:810 + heap := heap_take_parked(); // #jai fix E7: a thread waking from thread_idle gets its own heap back. + if !heap heap = heap_allocate(false); + set_thread_heap(heap); + return heap; +} + +// #jai fix E7: parked heaps (thread_idle), so thread starts skip the search when there are none. +global_parked_count: s64; // atomic + +// The heap the calling thread parked, unparked and its own again (its owner id set), or null. The +// owner's thread-local slot holds the fallback heap while it is parked, so its first allocator call +// after thread_idle comes here through the uninitialised-thread path. +heap_take_parked :: () -> *Heap { + if !atomic_load(*global_parked_count) return null; + me := get_thread_id(); + heap_lock_acquire(); + heap := global_heap_used; + while heap && heap.parked_thread != me heap = heap.next; + if heap { + heap_free_lock_acquire(heap); // Frees into it (as a released heap) take this lock (E6). + heap.parked_thread = 0; + heap.owner_thread = me; + heap_free_lock_release(heap); + atomic_add(*global_parked_count, -1); + } + heap_lock_release(); + return heap; +} + +// Get the current thread heap (2.0.1: rpmalloc.c:818). +// #jai until the first initialisation creates the TLS slot, and on any thread that has not stored +// a heap yet, this returns the fallback heap: upstream's _Thread_local default value. +// #jai fix E10: every allocation and free looks up the thread heap, and the OS thread-local storage +// lookup is a call (TlsGetValue, pthread_getspecific): 1.3 ns of the 2.3 ns per allocate+free pair the +// port was behind C 2.0.1 (which reads a _Thread_local variable inline). Jai has no thread-local +// variables, but each thread runs with its own Context, so the heap is cached there too. The Context +// can be copied to another thread, or the heap stop being this thread's, so the cache is used only if +// the stack in use is the stack of the heap's thread and the heap is still that thread's heap; otherwise +// the OS lookup runs and refreshes it. A fiber, whose stack is not its thread's, always takes the OS +// lookup. Benchmarks: pair_*, churn_small and mt_churn_* 4-9% faster. +get_thread_heap :: inline () -> *Heap { + #if CONTEXT_HEAP_CACHE { + cached := cast(*Heap) context_thread_heap().*; + if cached { + here := cast(u64) *cached; // An address on the stack in use. + if here >= cached.stack_low && here < cached.stack_high && cached.thread_current return cached; + } + } + return get_thread_heap_from_os(); +} + +get_thread_heap_from_os :: no_inline () -> *Heap { + if atomic_load(*global_rpmalloc_init_state) == RPMALLOC_INIT_DONE { + heap := cast(*Heap) os_tls_get(); + if heap { + #if CONTEXT_HEAP_CACHE if heap.thread_current context_thread_heap().* = heap; + return heap; + } + } + return global_heap_default; +} + +// Size class for tiny blocks: sizes up to 16 times the minimum granularity (1024 bytes). +get_size_class_tiny :: inline (size: u64) -> u32 { // 2.0.1: rpmalloc.c:824 + return (cast,trunc(u32) size + (SMALL_GRANULARITY - 1)) / SMALL_GRANULARITY; +} + +get_size_class :: inline (size: u64) -> u32 { // 2.0.1: rpmalloc.c:830 + // #jai The block count rounded up without 2.0.1's size + 15, which wraps for sizes within 15 of + // U64_MAX (and traps with arithmetic_overflow_check). Such sizes get a class >= SIZE_CLASS_COUNT, + // and the huge path refuses them (#jai fix U1, heap_allocate_block_huge). + minblock_count := size / SMALL_GRANULARITY + cast(u64) ((size & (SMALL_GRANULARITY - 1)) != 0); + // Up to 64 times the minimum granularity (1024 bytes) the size class is the block count. + if size <= SMALL_GRANULARITY * 64 { + rpmalloc_assert(global_size_class[minblock_count].block_size >= size, "Size class misconfiguration"); + return cast(u32) ifx minblock_count then minblock_count else 1; + } + minblock_count -= 1; + // minblock_count is now > 64, so its most significant bit is at position 6 or above. + most_significant_bit := msb_index(minblock_count); + // Class sizes have the bit format [..]000xxx000[..]: the most significant bit, then the two + // bits after it select the subclass. + subclass_bits := cast(u32)(minblock_count >> (most_significant_bit - 2)) & 0x03; + class_idx := (most_significant_bit << 2) + subclass_bits + 41; + rpmalloc_assert(class_idx >= SIZE_CLASS_COUNT || global_size_class[class_idx].block_size >= size, "Size class misconfiguration"); + rpmalloc_assert(class_idx >= SIZE_CLASS_COUNT || global_size_class[class_idx - 1].block_size < size, "Size class misconfiguration"); + return class_idx; +} + +get_page_type :: inline (size_class: u32) -> Page_Type { // 2.0.1: rpmalloc.c:858 + if size_class < SMALL_SIZE_CLASS_COUNT return .SMALL; + else if size_class < SMALL_SIZE_CLASS_COUNT + MEDIUM_SMALL_SIZE_CLASS_COUNT return .MEDIUM_SMALL; + else if size_class < SMALL_SIZE_CLASS_COUNT + MEDIUM_SMALL_SIZE_CLASS_COUNT + MEDIUM_LARGE_SIZE_CLASS_COUNT return .MEDIUM_LARGE; + else if size_class < SIZE_CLASS_COUNT return .LARGE; + return .HUGE; +} + +get_page_aligned_size :: inline (size: u64) -> u64 { // 2.0.1: rpmalloc.c:871 + unalign := size % global_config.page_size; + if unalign size += global_config.page_size - unalign; // Callers keep size below MAX_ALLOC_SIZE (#jai fix U1). + return size; +} + +//////////// +/// +/// OS entry points (2.0.1: rpmalloc.c:878-1161) +/// +////// +// os_set_page_name:885, os_mmap:900, os_commit_on_demand:1040, os_mcommit:1049, +// os_mdecommit:1086, os_munmap:1142. These live in os_windows.jai / os_posix.jai. + +//////////// +/// +/// Page interface (2.0.1: rpmalloc.c:1162-1500) +/// +////// + +page_get_span :: inline (page: *Page) -> *Span { // 2.0.1: rpmalloc.c:1169 + return cast(*Span)(cast(u64) page & SPAN_MASK); +} + +page_get_size :: inline (page: *Page) -> u64 { // 2.0.1: rpmalloc.c:1174 + if page_get_type(page) == { + case .SMALL; return SMALL_PAGE_SIZE; + case .MEDIUM_SMALL; return MEDIUM_SMALL_PAGE_SIZE; + case .MEDIUM_LARGE; return MEDIUM_LARGE_PAGE_SIZE; + case .LARGE; return LARGE_PAGE_SIZE; + } + return page_get_span(page).page_size; +} + +// #jai fix #407 (as upstream develop 30bc8859): ownership by heap, not by thread id. 2.0.1 compared +// owner_thread with the caller's thread id, and a released heap keeps its owner_thread, so a thread +// freeing after its THREAD_STOP (or a new thread reusing its id) took the unsynchronised local path +// on a heap queued for or adopted by another thread. +page_is_thread_heap :: inline (page: *Page) -> bool { // 2.0.1: rpmalloc.c:1188 + #if FIRST_CLASS_HEAPS return page.heap == get_thread_heap() || !page.heap.owner_thread; + else return page.heap == get_thread_heap(); +} + +// #jai upstream PR #424 (2759a3e): the first block of a page starts at the largest power of two +// dividing the block size, at least the page header size and at most NATURAL_ALIGNMENT_LIMIT, so every +// block of a class is aligned to that power of two. The tail slack of every page is larger than the +// extra offset, so no class loses a block (checked at compile time below). The offset is derived from +// the block size, not stored in the page. +block_start_offset :: inline (block_size: u32) -> u32 { + offset := lowest_set_bit(block_size); + if offset < PAGE_HEADER_SIZE return PAGE_HEADER_SIZE; + if offset > NATURAL_ALIGNMENT_LIMIT return NATURAL_ALIGNMENT_LIMIT; + return offset; +} + +// #jai The largest power of two dividing x; 0 for 0. C's x & -x relies on unsigned wraparound, which a +// Jai program built with arithmetic_overflow_check = .FATAL aborts on: x - 1 cannot wrap once x is +// non-zero. +lowest_set_bit :: inline (x: u32) -> u32 { + if !x return 0; + return x & ~(x - 1); +} + +// Sanity check of the huge page size the OS reports (Linux Hugepagesize; 2.0.1: rpmalloc.c:2850-2856). +// The builtin page geometry cannot use huge pages larger than the largest builtin page size (for +// example 1 GiB defaults). The huge page size is passed to mmap explicitly, so a non-default 2 MiB pool +// can still be used. 0 means no usable huge pages. #jai here rather than in os_posix.jai, so the tests +// reach it on every platform. +usable_huge_page_size :: (reported: u64) -> u64 { + size := reported; + if size && (size & (size - 1)) size = 0; // #jai 0 first: 0 - 1 would wrap (arithmetic_overflow_check) + if size > LARGE_PAGE_SIZE size = 2 * 1024 * 1024; + return size; +} + +page_block_start_offset :: inline (page: *Page) -> u32 { + return block_start_offset(page.block_size); +} + +page_block_start :: inline (page: *Page) -> *Block { // 2.0.1: rpmalloc.c:1197 + return pointer_offset(page, page_block_start_offset(page)); +} + +page_block :: inline (page: *Page, block_index: u32) -> *Block { // 2.0.1: rpmalloc.c:1202 + return pointer_offset(page, page_block_start_offset(page) + cast(u64) page.block_size * block_index); +} + +// #jai develop #411 (03537c6): the thread-free token holds the head block's byte offset from the +// page start instead of its index, so a cross-thread free needs no division and a decode no +// multiplication. Pages are at most LARGE_PAGE_SIZE (16 MiB), so the offset fits in 32 bits. +page_block_offset :: inline (page: *Page, block: *Block) -> u32 { // 2.0.1: rpmalloc.c:1207 page_block_index + return cast(u32) pointer_diff(block, page); +} + +// #jai fix PR #426 (upstream 0aaf830, adapted to the 2.0.1 token): set in the thread-free token of a +// full page, whose list is then empty. The list count takes the 31 bits below it. +THREAD_FREE_FULL :: cast(u64) 1 << 63; + +page_block_from_thread_free_list :: inline (page: *Page, token: u64, block: **Block) -> u32 { // 2.0.1: rpmalloc.c:1213 + block_offset := cast(u32)(token & 0xFFFF_FFFF); + list_count := cast(u32)((token >> 32) & 0x7FFF_FFFF); + block.* = ifx list_count then cast(*Block) pointer_offset(page, block_offset) else null; + return list_count; +} + +page_block_to_thread_free_list :: inline (page: *Page, block_offset: u32, list_count: u32) -> u64 { // 2.0.1: rpmalloc.c:1221 + // #jai fix P-2 (port bug): `cast(u64) list_count << 32` casts the shifted u32, so the count was + // lost and every cross-thread free into a page that was not full went missing. + return ((cast(u64) list_count) << 32) | cast(u64) block_offset; +} + +// #jai PR #424: the modulo is taken on the offset from the page start, then adjusted by the block start +// offset. Aligned blocks come from classes at least the alignment larger than the request, whose block +// start offset is at most the block size. +page_block_realign :: inline (page: *Page, block: *Block) -> *Block { // 2.0.1: rpmalloc.c:1227 + block_size := page.block_size; + start_offset := page_block_start_offset(page); + rpmalloc_assert(start_offset <= block_size, "Block start offset exceeds aligned block size"); + page_offset := (cast(u32) pointer_diff(block, page)) % block_size; + block_offset := ifx page_offset >= start_offset then page_offset - start_offset else page_offset + block_size - start_offset; + return pointer_offset(block, -cast(s64) block_offset); +} + +page_get_local_free_block :: (page: *Page) -> *Block { // 2.0.1: rpmalloc.c:1234 + block := page.local_free; + page.local_free = block.next; + page.block_used += 1; + return block; +} + +page_decommit_memory_pages :: inline (page: *Page) { // 2.0.1: rpmalloc.c:1243 + if page_has(page, .IS_DECOMMITTED) return; + // Only reserve-only spans support decommit. + if !page_has(page, .COMMIT_ON_DEMAND) return; + // Keep the prefix (holding the page header) committed and decommit the rest. The prefix is + // the span's fixed map-time page size, so commit and decommit always use the same one. + commit_prefix := cast(u64) 1 << page_commit_prefix_shift(page); + page_size := page_get_size(page); + if page_size <= commit_prefix return; + // #jai fix E3: only the chunks committed so far. + extra_page := pointer_offset(page, commit_prefix); + extra_page_size := page_committed_size(page) - commit_prefix; + if extra_page_size && global_memory_interface.memory_decommit(extra_page, extra_page_size) != 0 return; + #if HEAP_STATISTICS && ENABLE_DECOMMIT { + if page.heap heap_stat_sub(*page.heap.stats.committed_size, extra_page_size); + } + page_set(page, .IS_DECOMMITTED, true); + page_set_committed_chunks(page, 0); + #if TEST_HOOKS atomic_add(*test_pages_decommitted[cast(u32) page_get_type(page)], 1); +} + +page_commit_memory_pages :: inline (page: *Page) -> s32 { // 2.0.1: rpmalloc.c:1267 + if !page_has(page, .IS_DECOMMITTED) return 0; + commit_prefix := cast(u64) 1 << page_commit_prefix_shift(page); + // #jai fix E3: only the first chunk; page_initialize_blocks commits the rest as it carves. + if page_commit_to(page, commit_prefix + 1) != 0 return 1; + page_set(page, .IS_DECOMMITTED, false); + #if TEST_HOOKS atomic_add(*test_pages_recommitted[cast(u32) page_get_type(page)], 1); + #if ENABLE_DECOMMIT { + #if OS != .MACOS { + // When a page is recommitted, the blocks from the second memory page onwards are + // zeroed by the OS: take advantage in zalloc/calloc, and zero the first page here. + first_page := pointer_offset(page, PAGE_HEADER_SIZE); + memset(first_page, 0, cast(s64)(commit_prefix - PAGE_HEADER_SIZE)); + page_set(page, .IS_ZERO, true); + } + } + return 0; +} + +page_available_to_free :: (page: *Page, keep_last := true) { // 2.0.1: rpmalloc.c:1294 + rpmalloc_assert(!page_has(page, .IS_FULL), "Page full flag internal failure"); + rpmalloc_assert(!page_has(page, .IS_DECOMMITTED), "Page decommitted flag internal failure"); + heap := page.heap; + if heap.page_available[page.size_class] == page { + // #jai develop #413 (857888a): keep the last available page of a size class in place. + // Retiring it made the next allocation of the class take the generic path and reinitialise + // the page from scratch, which alternating alloc/free then repeated on every pair (W-11). + // At most one empty page per size class stays committed this way. + // keep_last is false only from heap_retire_kept_pages (#jai fix, see there). + if keep_last && !page.next return; + heap.page_available[page.size_class] = page.next; + } else { + page.prev.next = page.next; + if page.next page.next.prev = page.prev; + } + page_set(page, .IS_FREE, true); + page_set(page, .IS_ZERO, false); + page_type := cast(u32) page_get_type(page); + page.next = heap.page_free[page_type]; + heap.page_free[page_type] = page; + if !page.next heap.page_free_tail[page_type] = page; // #jai PR #420 + heap.page_free_commit_count[page_type] += 1; + if heap.page_free_commit_count[page_type] >= global_page_free_overflow[page_type] + heap_page_free_overflow(heap, page_type); // #jai develop #416 +} + +page_full_to_available :: (page: *Page) { // 2.0.1: rpmalloc.c:1314 + rpmalloc_assert(page_has(page, .IS_FULL), "Page full flag internal failure"); + rpmalloc_assert(!page_has(page, .IS_DECOMMITTED), "Page decommitted flag internal failure"); + heap := page.heap; + page.next = heap.page_available[page.size_class]; + if page.next page.next.prev = page; + heap.page_available[page.size_class] = page; + page_set(page, .IS_FULL, false); + if !page_has(page, .HAS_ALIGNED_BLOCK) page.heap_fast = heap; // #jai PR #422 + // #jai fix PR #426: other threads do not change the token of a page flagged full; it holds only + // the flag. + atomic_store(*page.thread_free, 0); +} + +// page_full_to_free_on_new_heap (2.0.1: rpmalloc.c:1328) is dead code upstream; not ported. + +page_available_to_full :: (page: *Page) { // 2.0.1: rpmalloc.c:1343 + heap := page.heap; + if heap.page_available[page.size_class] == page { + heap.page_available[page.size_class] = page.next; + } else { + page.prev.next = page.next; + if page.next page.next.prev = page.prev; + } + page_set(page, .IS_FULL, true); + page_set(page, .IS_ZERO, false); + page.heap_fast = null; // #jai PR #422 +} + +page_put_local_free_block :: inline (page: *Page, block: *Block) { // 2.0.1: rpmalloc.c:1358 + block.next = page.local_free; + page.local_free = block; + stat_add_free(page.heap, page.size_class, 1); + page.block_used -= 1; + if page.block_used == 0 { + page_available_to_free(page); + } else if page_has(page, .IS_FULL) { + page_full_to_available(page); + } +} + +page_adopt_thread_free_block_list :: no_inline (page: *Page) { // 2.0.1: rpmalloc.c:1371 + if page.local_free return; + thread_free := atomic_load(*page.thread_free); + if thread_free != 0 { + // Other threads can only replace it with another valid list head; it never changes to 0 + // in other threads. #jai develop #415 (686917f): so one exchange claims the whole list, with + // no retry loop or backoff on the owner's path. + thread_free = atomic_exchange(*page.thread_free, 0); + thread_free_count := page_block_from_thread_free_list(page, thread_free, *page.local_free); + rpmalloc_assert(thread_free_count <= page.block_used, "Page thread free list count internal failure"); + page.block_used -= thread_free_count; + stat_add_free(page.heap, page.size_class, thread_free_count); + } +} + +// Pushes a block freed into a full page to the heap-level thread-free list; otherwise the owner would +// not pick it up until a thread-local free happens on that page. +// #jai develop #412 (50fd91a): the uncontended case is a single CAS; the spin and yield retry loops +// live in separate cold procedures, so the common path stays small. +heap_put_thread_free_block :: (page: *Page, block: *Block) { + list := *page.heap.thread_free[cast(u32) page_get_type(page)]; + prev_head := atomic_load(list); + block.next = cast(*Block) prev_head; + swapped, actual := compare_and_swap(list, prev_head, cast(u64) block); + if !swapped heap_put_thread_free_block_retry(list, block, actual); +} + +heap_put_thread_free_block_retry :: no_inline (list: *u64, block: *Block, prev_head: u64) { + spin: u32; + while true { + block.next = cast(*Block) prev_head; + swapped, actual := compare_and_swap(list, prev_head, cast(u64) block); + if swapped return; + prev_head = actual; + wait_spin(*spin); + } +} + +// Pushes a block to another thread's deferred free list. +// #jai fix PR #426: 2.0.1 chose between the page's list and the heap's by reading the owner's +// is_full flag, while the owner marked the page full with a plain store after finding the page's +// list empty. A free in between landed in the page's list of a page then marked full, and if it was +// among the page's last frees the page never became available or free again (W-09). The owner now +// marks a page full by setting THREAD_FREE_FULL in the token with a compare and swap, which fails +// if a free got in first, and frees route by that flag. +page_put_thread_free_block :: no_inline (page: *Page, block: *Block) { // 2.0.1: rpmalloc.c:1389 + prev_thread_free := atomic_load(*page.thread_free); + if prev_thread_free & THREAD_FREE_FULL { + heap_put_thread_free_block(page, block); + return; + } + block_offset := page_block_offset(page, block); + rpmalloc_assert(!((block_offset - page_block_start_offset(page)) % page.block_size), "Block pointer is not aligned to start of block"); + list_size := page_block_from_thread_free_list(page, prev_thread_free, *block.next) + 1; + thread_free := page_block_to_thread_free_list(page, block_offset, list_size); + swapped, actual := compare_and_swap(*page.thread_free, prev_thread_free, thread_free); + if !swapped page_put_thread_free_block_retry(page, block, block_offset, actual); // #jai develop #412 +} + +// The contended retry path for pushing to a page's thread-free list, or to the heap-level list once the +// owner has flagged the page full (#jai develop #412, with PR #426). +page_put_thread_free_block_retry :: no_inline (page: *Page, block: *Block, block_offset: u32, prev_thread_free: u64) { + spin: u32; + while true { + if prev_thread_free & THREAD_FREE_FULL { + heap_put_thread_free_block(page, block); + return; + } + list_size := page_block_from_thread_free_list(page, prev_thread_free, *block.next) + 1; + thread_free := page_block_to_thread_free_list(page, block_offset, list_size); + wait_spin(*spin); + swapped, actual := compare_and_swap(*page.thread_free, prev_thread_free, thread_free); + if swapped return; + prev_thread_free = actual; + } +} + +page_push_local_free_to_heap :: (page: *Page) { // 2.0.1: rpmalloc.c:1420 + // Push the page free list as the fast-track list of free blocks for the heap. + page.heap.local_free[page.size_class] = page.local_free; + // Every initialised block not counted as used is on the local free list, so after handing the + // whole list to the heap every initialised block is in use (#jai PR #422). + page.block_used = page.block_initialized; + page.local_free = null; +} + +page_initialize_blocks :: no_inline (page: *Page) -> *Block { // 2.0.1: rpmalloc.c:1429 + rpmalloc_assert(page.block_initialized < page.block_count, "Block initialization internal failure"); + // #jai fix E3: commit the page up to the end of the new block first (a small page is one chunk, + // so the free list linked below stays inside it). Null when the OS refuses; nothing has changed. + if page_has(page, .COMMIT_ON_DEMAND) { + extent := cast(u64) page_block_start_offset(page) + (cast(u64) page.block_initialized + 1) * page.block_size; + if page_commit_to(page, extent) != 0 return null; + } + block := page_block(page, page.block_initialized); + page.block_initialized += 1; + page.block_used += 1; + + if page_get_type(page) == .SMALL && cast(u64) page.block_size < (global_config.page_size >> 1) { + // Link up blocks until the next memory page into the free list. + memory_page_start := cast(*void)(cast(u64) block & ~(global_config.page_size - 1)); + memory_page_next := pointer_offset(memory_page_start, global_config.page_size); + free_block: *Block = pointer_offset(block, page.block_size); + first_block := free_block; + last_block := free_block; + list_count: u32 = 0; + max_list_count := page.block_count - page.block_initialized; + while cast(*void) free_block < memory_page_next && list_count < max_list_count { + last_block = free_block; + free_block.next = pointer_offset(free_block, page.block_size); + free_block = free_block.next; + list_count += 1; + } + if list_count { + last_block.next = null; + page.local_free = first_block; + page.block_initialized += list_count; + } + } + return block; +} + +page_allocate_block :: inline (page: *Page, zero: bool) -> *void { // 2.0.1: rpmalloc.c:1462 + is_zero := false; + block := ifx page.local_free then page_get_local_free_block(page) else null; + if block == null { + if atomic_load(*page.thread_free) != 0 { + page_adopt_thread_free_block_list(page); + block = ifx page.local_free then page_get_local_free_block(page) else null; + } + if block == null { + block = page_initialize_blocks(page); + if !block return null; // #jai fix E3: the commit for the new block failed. + is_zero = page_has(page, .IS_ZERO); + } + } + + rpmalloc_assert(page.block_used <= page.block_count, "Page block use counter out of sync"); + if page.local_free && !page.heap.local_free[page.size_class] + page_push_local_free_to_heap(page); + + // The page might be full once its free list has been pushed to the heap local free list: + // check whether there is a thread free list to adopt. + if page.block_used == page.block_count { + page_adopt_thread_free_block_list(page); + if page.block_used == page.block_count { + // The page is now fully used. + rpmalloc_assert(!page_has(page, .IS_FULL), "Page block use counter out of sync with full flag"); + #if TEST_HOOKS if test_before_mark_full test_before_mark_full(page); + // #jai fix PR #426: flag the page full in the token, unless a cross-thread free just + // added a block; then the page stays available and the block is adopted on the next + // refill. + swapped := compare_and_swap(*page.thread_free, 0, THREAD_FREE_FULL); + if swapped page_available_to_full(page); + } + } + + if zero { + if !is_zero memset(block, 0, page.block_size); + else (cast(*u64) block).* = 0; + } + return block; +} + +//////////// +/// +/// Span interface (2.0.1: rpmalloc.c:1501-1771) +/// +////// + +// #jai fix E8: milliseconds from `since` to `now`, 0 when `since` is later. A thread reads the clock and +// then compares under a lock, so another thread may have stored a later reading in between: the huge +// cache (an entry cached after this thread read the clock) and, since E4 and E7 let other threads free +// into a heap, a heap's decommit delay. Upstream's now - since wraps there, and the entry merely counts +// as aged early; in a program built with arithmetic_overflow_check it trapped (THR-08 on macOS, W-17). +elapsed_ms :: inline (now: u64, since: u64) -> u64 { + return ifx now > since then now - since else 0; +} + +// The huge block cache (2.0.1: rpmalloc.c:1561-1734). Recently freed huge mappings are kept for reuse +// instead of paying munmap + mmap + page faults per huge allocation cycle. Entries are bounded by a +// committed byte budget and unmapped after an idle epoch. A cached mapping is reused for requests down +// to 1 / HUGE_CACHE_REUSE_FACTOR of its size (#jai PR #427; 25% in 2.0.1). +HUGE_CACHE_COMMITTED_LIMIT :: cast(u64) HUGE_CACHE_LIMIT_MB * 1024 * 1024; // 2.0.1: rpmalloc.c:1570 (128 MiB; #jai fix E11: a module parameter) +#assert HUGE_CACHE_LIMIT_MB > 0 "HUGE_CACHE_LIMIT_MB must be positive; HUGE_CACHE_SLOT_COUNT = 0 disables the cache"; +// The largest single mapping the cache holds, so one rare giant cannot take the whole budget from +// the smaller, more often cycled sizes. +HUGE_CACHE_ENTRY_LIMIT :: HUGE_CACHE_COMMITTED_LIMIT / 4; // 2.0.1: rpmalloc.c:1576 +HUGE_CACHE_EPOCH_MS :: 1000; // 2.0.1: rpmalloc.c:1579 +// #jai upstream PR #427 (eb6d48b), as a constant: a cached mapping serves requests down to 1 / this +// of its size. 2.0.1 reused a mapping only within 25% of the request (1.25), so buffers of varying +// sizes mostly mapped fresh memory and faulted every page in (RET-14). A larger factor reuses more +// and wastes up to that much of a mapping; upstream measured malloc-large 10.0 -> 6.8 s, peak RSS +// 527 -> 557 MiB with 2. +HUGE_CACHE_REUSE_FACTOR :: 2; + +#if HUGE_CACHE_SLOT_COUNT { + +Huge_Cache :: struct { // 2.0.1: rpmalloc.c:1600 + lock: u64; // Cache lock. + committed: u64; // Committed bytes currently cached (decommitted entries do not count). + // #jai fix E5: when the oldest committed entry was cached (0: none), so the allocation slow path + // can tell without the lock whether an entry is due for decommit. Written under the lock. + oldest_committed_ms: u64; + slot: [HUGE_CACHE_SLOT_COUNT] struct { + span: *Span; + free_ms: u64; // When the mapping was cached, or (decommitted) when it was decommitted. + decommitted: bool; // #jai fix E5: only the header page is committed; the address space is kept. + }; +} + +global_huge_cache: Huge_Cache; // 2.0.1: rpmalloc.c:1612 + +huge_cache_lock_acquire :: inline () { // 2.0.1: rpmalloc.c:1615 + spin: u32; + while !compare_and_swap(*global_huge_cache.lock, 0, 1) wait_spin(*spin); +} + +huge_cache_lock_release :: inline () { // 2.0.1: rpmalloc.c:1626 + atomic_store(*global_huge_cache.lock, 0); +} + +huge_cache_entry_size :: inline (span: *Span) -> u64 { + return cast(u64) span.page_count * span.page_size; +} + +// Under the lock. +huge_cache_update_oldest :: () { + oldest: u64 = 0; + for global_huge_cache.slot if it.span && !it.decommitted && (!oldest || it.free_ms < oldest) oldest = it.free_ms; + atomic_store(*global_huge_cache.oldest_committed_ms, oldest); +} + +// #jai fix E5: Ex4 1.45.0's cache filled to its 128 MiB limit and stayed committed, because idle +// entries were let go only at the next huge free (huge_cache_push) or thread_collect (U23), and a +// program that stops freeing huge blocks does neither (RET-19). An entry idle for an epoch now has its +// contents decommitted rather than its mapping unmapped: reusing it skips the mapping (most of the +// cache's value; the pages fault in again either way), and the reservation costs no memory. Without +// decommit (ENABLE_DECOMMIT off, or decommit disabled) it is unmapped, as before. The OS calls are made +// outside the lock. Runs from huge_cache_push, huge allocations, thread_collect and, through +// huge_cache_purge_if_due, the allocation slow path. +huge_cache_purge_aged :: (now: u64) { // 2.0.1: rpmalloc.c:1633 + aged: [HUGE_CACHE_SLOT_COUNT] *Span = ---; + aged_count := 0; + huge_cache_lock_acquire(); + for * slot: global_huge_cache.slot { + span := slot.span; + if span && !slot.decommitted && elapsed_ms(now, slot.free_ms) >= HUGE_CACHE_EPOCH_MS { + global_huge_cache.committed -= huge_cache_entry_size(span); + slot.span = null; + aged[aged_count] = span; + aged_count += 1; + } + } + huge_cache_update_oldest(); + huge_cache_lock_release(); + for ispan: 0..aged_count - 1 { + span := aged[ispan]; + page_size := cast(u64) span.page_size; + size := huge_cache_entry_size(span); + kept := false; + if os_commit_on_demand() && size > page_size && + global_memory_interface.memory_decommit(pointer_offset(span, page_size), size - page_size) == 0 { + // Back in, decommitted, if a slot is still free; otherwise let it go. + huge_cache_lock_acquire(); + for * slot: global_huge_cache.slot { + if !slot.span { + slot.span = span; + slot.free_ms = now; + slot.decommitted = true; + kept = true; + break; + } + } + huge_cache_lock_release(); + } + if !kept span_unmap(span); + } +} + +// The allocation slow path's check: purge only when a committed entry has been idle for an epoch. One +// load while nothing committed is cached. +huge_cache_purge_if_due :: inline () { + oldest := atomic_load(*global_huge_cache.oldest_committed_ms); + if !oldest return; + now := os_monotonic_time_ms(); + if elapsed_ms(now, oldest) >= HUGE_CACHE_EPOCH_MS huge_cache_purge_aged(now); +} + +// Unmaps every cached mapping. The cache must not outlive allocator finalization. +huge_cache_flush :: () { // 2.0.1: rpmalloc.c:1652 + purge: [HUGE_CACHE_SLOT_COUNT] *Span = ---; + purge_count := 0; + huge_cache_lock_acquire(); + for * slot: global_huge_cache.slot { + span := slot.span; + if span { + if !slot.decommitted global_huge_cache.committed -= huge_cache_entry_size(span); + slot.span = null; + purge[purge_count] = span; + purge_count += 1; + } + } + huge_cache_update_oldest(); + huge_cache_lock_release(); + for ispan: 0..purge_count - 1 + span_unmap(purge[ispan]); +} + +// Tries to cache a freed huge mapping; returns true if it was cached. +huge_cache_push :: (span: *Span) -> bool { // 2.0.1: rpmalloc.c:1671 + committed_size := huge_cache_entry_size(span); + if committed_size > HUGE_CACHE_ENTRY_LIMIT return false; + now := os_monotonic_time_ms(); + if !now now = 1; // 0 marks "no committed entry" in oldest_committed_ms. + huge_cache_purge_aged(now); + cached := false; + evicted: *Span; + huge_cache_lock_acquire(); + if global_huge_cache.committed + committed_size <= HUGE_CACHE_COMMITTED_LIMIT { + // A free slot, or else (#jai fix E5) the slot of the longest decommitted entry. + target: s64 = -1; + for global_huge_cache.slot if !it.span { target = it_index; break; } + if target < 0 { + for global_huge_cache.slot { + if it.decommitted && (target < 0 || it.free_ms < global_huge_cache.slot[target].free_ms) target = it_index; + } + if target >= 0 evicted = global_huge_cache.slot[target].span; + } + if target >= 0 { + slot := *global_huge_cache.slot[target]; + // #jai a cached mapping belongs to no heap until it is handed out again + // (heap_allocate_block_huge sets both fields), so IS_THIS_YOURS rejects it. + span.heap = null; + span.page.heap = null; + slot.span = span; + slot.free_ms = now; + slot.decommitted = false; + global_huge_cache.committed += committed_size; + cached = true; + } + } + huge_cache_update_oldest(); + huge_cache_lock_release(); + if evicted span_unmap(evicted); + // #jai E11 kept a block beyond the budget decommitted instead of letting the caller unmap it. On + // macOS that made churn_huge slower: 4.5 us per step before, 12.4 with it, 6.3 even with E12 + // (decommit + recommit cost about 20 us there per block, more than an munmap and a fresh mapping). + // A program that frees huge blocks often does better with a larger HUGE_CACHE_LIMIT_MB. + return cached; +} + +// Takes the smallest cached mapping that fits the request within the size class spacing; null on a +// miss. The cached span keeps its own offset, mapped_size and page_count. #jai fix E5: a decommitted +// entry is committed again here; if the OS refuses, it is unmapped and the caller maps afresh. +// #jai fix E12: a committed entry that fits goes before any decommitted one, not only one of the same +// size. Taking the smallest let committed entries sit idle while decommitted ones were recommitted: +// while E11 kept blocks beyond the budget decommitted, churn_huge on macOS paid a decommit and a +// recommit on 58% of steps (21% with this order; pattern simulated). Entries decommitted after idling +// (E5) are still reused this way. A committed entry costs nothing to hand out, and taking it frees +// committed budget. +huge_cache_pop :: (alloc_size: u64) -> *Span { // 2.0.1: rpmalloc.c:1697 + // A racy emptiness peek is fine: a missed entry only costs a fresh mapping. + empty := true; + for global_huge_cache.slot { + if it.span { + empty = false; + break; + } + } + if empty return null; + size_limit := alloc_size * HUGE_CACHE_REUSE_FACTOR; // #jai PR #427 + best: *Span; + best_size: u64; + best_slot: s64; + best_decommitted: bool; + huge_cache_lock_acquire(); + for global_huge_cache.slot { + span := it.span; + if !span continue; + size := huge_cache_entry_size(span); + if size < alloc_size || size > size_limit continue; + better := !best || (best_decommitted && !it.decommitted) || + (best_decommitted == it.decommitted && size < best_size); + if better { + best = span; + best_size = size; + best_slot = it_index; + best_decommitted = it.decommitted; + } + } + if best { + global_huge_cache.slot[best_slot].span = null; + if !best_decommitted global_huge_cache.committed -= best_size; + huge_cache_update_oldest(); + } + huge_cache_lock_release(); + if best && best_decommitted { + page_size := cast(u64) best.page_size; + if global_memory_interface.memory_commit(pointer_offset(best, page_size), best_size - page_size) != 0 { + span_unmap(best); + return null; + } + } + return best; +} + +} // #if HUGE_CACHE_SLOT_COUNT + +span_is_thread_heap :: inline (span: *Span) -> bool { // 2.0.1: rpmalloc.c:1508 (#jai fix #407, as page_is_thread_heap) + #if FIRST_CLASS_HEAPS return span.heap == get_thread_heap() || !span.heap.owner_thread; + else return span.heap == get_thread_heap(); +} + +span_get_page_from_block :: inline (span: *Span, block: *void) -> *Page { // 2.0.1: rpmalloc.c:1517 + return cast(*Page)(cast(u64) block & span.page_address_mask); +} + +// Initialize a new page from the given span. +span_allocate_page :: inline (span: *Span) -> *Page { // 2.0.1: rpmalloc.c:1523 + rpmalloc_assert(span.page_initialized < span.page_count, "Page initialization internal failure"); + heap := span.heap; + page: *Page = pointer_offset(span, cast(u64) span.page_size * span.page_initialized); + + #if ENABLE_DECOMMIT { + // Reserve-only spans commit later pages on demand; fully committed spans are already + // committed from the map and must not be committed per page. + // #jai fix E3: the first chunk only; page_initialize_blocks commits the rest as it carves. + if span.page_initialized && page_has(*span.page, .COMMIT_ON_DEMAND) { + chunk := page_commit_chunk_size(span.page_size, cast(u64) 1 << page_commit_prefix_shift(*span.page)); + if global_memory_interface.memory_commit(page, chunk) != 0 return null; + #if HEAP_STATISTICS heap_stat_add(*heap.stats.committed_size, chunk); + } + } + span.page_initialized += 1; + + page_set_type(page, span.page_type); + page_set(page, .COMMIT_ON_DEMAND, page_has(*span.page, .COMMIT_ON_DEMAND)); + page_set_commit_prefix_shift(page, page_commit_prefix_shift(*span.page)); + page_set_committed_chunks(page, 1); // #jai fix E3 (page 0: committed by the span map) + page_set(page, .IS_ZERO, true); + page.heap = heap; + rpmalloc_assert(page_is_thread_heap(page), "Page owner thread mismatch"); + + if span.page_initialized == span.page_count { + // Span fully used. + page_type := cast(u32) span.page_type; + rpmalloc_assert(span == heap.span_partial[page_type], "Span partial tracking out of sync"); + heap.span_partial[page_type] = null; + span.next = heap.span_used[page_type]; + heap.span_used[page_type] = span; + } + return page; +} + +// #jai fix U3: removes a freed huge span from its first-class heap's list. A first-class heap is used +// by one thread at a time, which is also the thread that frees its blocks, so the list needs no lock. +huge_span_unlink :: (heap: *Heap, span: *Span) { + link := *heap.span_used[Page_Type.HUGE]; + while link.* { + if link.* == span { + link.* = span.next; + span.next = null; + return; + } + link = *link.*.next; + } +} + +// #jai upstream PR #425 (d668610): the huge block free and the free of a block in a page holding +// aligned blocks are out of line, so the generic free path, taken by every cross-thread free, needs no +// stack frame. +span_deallocate_huge :: no_inline (span: *Span) { + #if ENABLE_STATISTICS statistics_sub_saturating(*global_counters.huge_alloc, cast(u64) span.page_count * span.page_size); + #if HEAP_STATISTICS { + if span.heap { + heap_stat_sub(*span.heap.stats.mapped_size, span.mapped_size); + #if ENABLE_DECOMMIT heap_stat_sub(*span.heap.stats.committed_size, cast(u64) span.page_count * span.page_size); + else heap_stat_sub(*span.heap.stats.committed_size, span.mapped_size); + } + } + // #jai fix U3: a first-class heap lists its huge spans in span_used[HUGE] (so heap_free_all can + // release them), but upstream never unlinks one when it is freed. The span then goes to the + // huge cache or is unmapped, and a later heap_free_all unmaps it again, possibly while another + // heap uses the cached mapping. The list is singly linked and Span has no room for a back + // pointer, so this walks it: O(live huge blocks of that heap) on a path that makes an OS call. + // Thread heaps never link huge spans, so for them the list is empty. + if span.heap huge_span_unlink(span.heap, span); + #if HUGE_CACHE_SLOT_COUNT { + if huge_cache_push(span) return; + } + span_unmap(span); +} + +span_deallocate_aligned_block :: no_inline (page: *Page, block: *void) { + block = page_block_realign(page, block); + if page_is_thread_heap(page) page_put_local_free_block(page, block); + else page_put_thread_free_block(page, block); +} + +span_deallocate_block :: no_inline (span: *Span, page: *Page, block: *void) { // 2.0.1: rpmalloc.c:1736 + if page_get_type(page) == .HUGE { + span_deallocate_huge(span); + return; + } + if page_has(page, .HAS_ALIGNED_BLOCK) { + span_deallocate_aligned_block(page, block); + return; + } + if page_is_thread_heap(page) { + page_put_local_free_block(page, block); + } else { + // #jai fix E4: a released heap has no owner to take in a deferred block until a thread + // adopts the heap, which may be never. + if page.heap.owner_thread == HEAP_RELEASED_OWNER && page_free_into_released_heap(page, block) return; + // Multithreaded deallocation: push to the deferred deallocation list. + page_put_thread_free_block(page, block); + } +} + +// #jai fix E4: a block freed into a released heap (its thread has exited, or heap_release) went on a +// deferred list that only a thread adopting the heap drains, so its page stayed in use and committed +// until then: in Ex4, buffers that walk threads grew and the service thread freed after the walkers +// had exited (RET-18). Under the heap lock, which adoption also takes (heap_allocate_new), nobody owns +// a heap that is still released, so the block is freed as its owner would free it, the page takes in +// the frees already pending in it, and a page left empty is retired and decommitted, as at thread exit +// (E1). False when the heap has been adopted meanwhile: the caller defers the block as usual. +// #jai fix E6: under the heap's own free_lock, not the global heap lock, which every thread start and +// exit takes; frees into different released heaps no longer serialise either. +page_free_into_released_heap :: no_inline (page: *Page, block: *Block) -> bool { + heap := page.heap; + heap_free_lock_acquire(heap); + if heap.owner_thread != HEAP_RELEASED_OWNER { + heap_free_lock_release(heap); + return false; + } + page_put_local_free_block(page, block); + if !page_has(page, .IS_FREE) && !page_has(page, .IS_FULL) { + page_adopt_thread_free_merging(page); + // Empty: retire it, also when it is the page #413 keeps as the last of its class. + if page.block_used == 0 page_available_to_free(page, keep_last = false); + } + page_type := cast(u32) page_get_type(page); + if heap.page_free_commit_count[page_type] { + heap.page_free_overflow_ms[page_type] = 0; + heap_page_free_decommit(heap, page_type, 0); + } + heap_free_lock_release(heap); + return true; +} + +//////////// +/// +/// Block interface (2.0.1: rpmalloc.c:1772-1829) +/// +////// + +block_get_span :: inline (block: *void) -> *Span { // 2.0.1: rpmalloc.c:1779 + return cast(*Span)(cast(u64) block & SPAN_MASK); +} + +// #jai PR #422: frees a block, taking the fast path when the page's heap_fast is the given heap. The +// caller owns that heap: it is the thread heap, or the first-class heap the block came from. +block_deallocate_heap :: inline (block: *Block, owned_heap: *Heap) { // 2.0.1: rpmalloc.c:1784 block_deallocate + span := block_get_span(block); + page := span_get_page_from_block(span, block); + + #if HEAP_STATISTICS { // 2.0.1: rpmalloc.c:1789 + if span.heap { + // A huge block subtracts its capacity, which its allocation added (#jai fix U13). + if span.page_type <= .LARGE heap_stat_sub(*span.heap.stats.allocated_size, page.block_size); + else heap_stat_sub(*span.heap.stats.allocated_size, cast(u64) span.page_size * span.page_count); + } + } + + // Fast path: a free by the heap's owner, of a block in a page that is not huge, not full and has + // no aligned block; heap_fast encodes all of that by matching the owned heap. + if page.heap_fast == owned_heap { + block.next = page.local_free; + page.local_free = block; + stat_add_free(page.heap, page.size_class, 1); + page.block_used -= 1; + if page.block_used == 0 page_available_to_free(page); + } else { + span_deallocate_block(span, page, block); + } +} + +block_deallocate :: inline (block: *Block) { + block_deallocate_heap(block, get_thread_heap()); +} + +block_usable_size :: inline (block: *void) -> u64 { // 2.0.1: rpmalloc.c:1819 + span := block_get_span(block); + if span.page_type <= .LARGE { + page := span_get_page_from_block(span, block); + // #jai develop #411: the offset within a page is below 2^32; a 32-bit modulo is several times + // cheaper than a 64-bit one. + block_offset := cast(u32) pointer_diff(block, page) - page_block_start_offset(page); // #jai PR #424 + return page.block_size - (block_offset % page.block_size); + } + return cast(u64) span.page_size * span.page_count - cast(u64) pointer_diff(block, span); +} + +//////////// +/// +/// Heap interface (2.0.1: rpmalloc.c:1830-2509) +/// +////// + +heap_lock_acquire :: inline () { // 2.0.1: rpmalloc.c:1837 + this_lock := get_thread_id(); + spin: u32; + while true { + swapped := compare_and_swap(*global_heap_lock, 0, this_lock); + if swapped break; + wait_spin(*spin); + } +} + +heap_lock_release :: inline () { // 2.0.1: rpmalloc.c:1848 + rpmalloc_assert(atomic_load(*global_heap_lock) == get_thread_id(), "Bad heap lock"); + atomic_store(*global_heap_lock, 0); +} + +// A heap control structure is much smaller than the mapping page size (and far smaller than a huge +// page), so one page-aligned mapping is carved into as many heaps as fit: the first is returned to +// the caller and the extras are queued for reuse. Only the first heap records the mapping offset +// and size; it owns the shared mapping and is the one heap that unmaps it at finalize. +// +// Block creation is serialised through global_heap_creating: when no heap is available, one thread +// maps and carves a block while concurrent thread starts wait for the carved heaps. Ordinary +// requests reuse a released heap first, then a pristine carved one. First-class requests need a heap +// with no prior allocations, so they take only pristine heaps; a released first-class heap that holds +// nothing goes back to the pristine list (#jai fix U20). +heap_allocate_new :: (first_class: bool) -> *Heap { // 2.0.1: rpmalloc.c:1875 + // #jai fix U11: upstream took page_size != 0 to mean "initialised", but page_size is set early in + // initialisation and never cleared by finalize, so a heap acquired after finalize was mapped and + // used on a finalized allocator (W-10). Check the init state itself. + if atomic_load(*global_rpmalloc_init_state) != RPMALLOC_INIT_DONE initialize(null); + + heap_lock_acquire(); + spin: u32; + while true { + if !first_class && global_heap_queue { + heap := global_heap_queue; + global_heap_queue = heap.next; + // #jai fix E4: take ownership under the lock. Frees into a released heap check its owner + // under the heap's free_lock (page_free_into_released_heap; #jai fix E6), so take that too; + // heap_allocate set the owner only afterwards. Lock order: global, then the heap's. + heap_free_lock_acquire(heap); + heap.owner_thread = get_thread_id(); + heap.released = false; + heap_free_lock_release(heap); + heap_lock_release(); + if heap.release_trim { // #jai develop #419: the trim deferred at release. + heap.release_trim = 0; + heap_release_free_pages(heap); + } + return heap; + } + if global_heap_pristine { + heap := global_heap_pristine; + global_heap_pristine = heap.next; + heap_lock_release(); + return heap; + } + if !global_heap_creating break; + heap_lock_release(); + wait_spin(*spin); + heap_lock_acquire(); + } + global_heap_creating = 1; + heap_lock_release(); + + aligned_heap_size := cast(u64) HEAP_ALIGNMENT * ((size_of(Heap) + (HEAP_ALIGNMENT - 1)) / HEAP_ALIGNMENT); + block_size := get_page_aligned_size(aligned_heap_size); + offset, mapped_size: u64; + block := global_memory_interface.memory_map(block_size, 0, *offset, *mapped_size); + #if ENABLE_DECOMMIT { + if block && os_commit_on_demand() && global_memory_interface.memory_commit(block, block_size) != 0 { + global_memory_interface.memory_unmap(block, offset, mapped_size); + block = null; + } + } + if !block { + heap_lock_acquire(); + global_heap_creating = 0; + heap_lock_release(); + return null; + } + + heap := heap_initialize(block); + heap.offset = cast(u32) offset; + heap.mapped_size = mapped_size; + + // Carve the remaining heaps into a local chain, then splice it onto the pristine pool in one + // step, so the lock is not held across the per-heap initialisation. heap_initialize zeroes each + // structure, leaving mapped_size 0, so the extras are not mapping owners. + heap_count := mapped_size / aligned_heap_size; + chain_head: *Heap; + chain_tail: *Heap; + extra_heap: *Heap = pointer_offset(heap, aligned_heap_size); + for iheap: 1..cast(s64) heap_count - 1 { + heap_initialize(extra_heap); + if !chain_tail chain_tail = extra_heap; + extra_heap.next = chain_head; + chain_head = extra_heap; + extra_heap = pointer_offset(extra_heap, aligned_heap_size); + } + + heap_lock_acquire(); + if chain_tail { + chain_tail.next = global_heap_pristine; + global_heap_pristine = chain_head; + } + global_heap_creating = 0; + heap_lock_release(); + return heap; +} + +// Unmaps the shared block owned by this heap. Invariant: heap blocks are only ever unmapped here, +// at global finalize, never while the allocator is running, because several heaps share a block. +heap_unmap :: (heap: *Heap) { // 2.0.1: rpmalloc.c:1959 + global_memory_interface.memory_unmap(heap, heap.offset, heap.mapped_size); +} + +heap_allocate :: (first_class: bool) -> *Heap { // 2.0.1: rpmalloc.c:1964 + heap := heap_allocate_new(first_class); + if heap { + current_thread_id := get_thread_id(); + heap_lock_acquire(); + heap.next = global_heap_used; + heap.prev = null; + if global_heap_used global_heap_used.prev = heap; + global_heap_used = heap; + heap_lock_release(); + heap.owner_thread = current_thread_id; + heap.is_first_class = first_class; + heap.released = false; // #jai fix U14 + // Counters belong to the heap, not the thread, so alloc_current reflects the blocks live in + // the heap's pages whichever thread owns it. + #if ENABLE_STATISTICS atomic_add(*global_counters.heap_count, 1); + } + return heap; +} + +// Upstream heap_release, renamed: the public rpmalloc_heap_release is heap_release in this port. +// #jai fix U20: a first-class heap must start with no blocks, so heap_acquire takes only pristine +// heaps, and upstream queued every released one for thread heaps: create/destroy cycles used up a new +// heap structure each time (HEAP-13), and on macOS ran out of address space (THR-02). A heap that +// holds nothing, as after DESTROY_HEAP's heap_free_all, is as good as pristine: it is reset (keeping +// the mapping it may own) and goes back to the pristine list. +heap_is_empty :: (heap: *Heap) -> bool { + for 0..3 { + if heap.span_partial[it] || heap.page_free[it] || atomic_load(*heap.thread_free[it]) return false; + } + for heap.span_used if it return false; + return true; +} + +heap_release_to_pristine :: (heap: *Heap) { + heap_lock_acquire(); + if heap.released { heap_lock_release(); return; } // #jai fix U14: already released. + #if ENABLE_STATISTICS statistics_sub_saturating(*global_counters.heap_count, 1); + if heap.prev heap.prev.next = heap.next; + if heap.next heap.next.prev = heap.prev; + if global_heap_used == heap global_heap_used = heap.next; + offset, mapped_size := heap.offset, heap.mapped_size; // Only the mapping's owner has these. + heap_initialize(heap); + heap.offset, heap.mapped_size = offset, mapped_size; + heap.released = true; + heap.next = global_heap_pristine; + global_heap_pristine = heap; + heap_lock_release(); +} + +// #jai fix U14: upstream left a released first-class heap with owner_thread 0, which the free path +// reads as "first-class, the caller owns it", so frees into it after release took the unsynchronised +// local path while the heap was queued for another thread (W-13); and a heap made current by a thread +// was queued twice when that thread exited and the program released it too (HEAP-14). A released heap +// now gets an owner id no thread has, so frees into it are cross-thread frees, and a second release +// of a heap that is already released is ignored. The adopting thread sets its own id. +HEAP_RELEASED_OWNER :: 0xFFFF_FFFF_FFFF_FFFF; + +// #jai fix E6: the lock for frees into a released heap (E4) and for adopting one; see Heap.free_lock. +heap_free_lock_acquire :: inline (heap: *Heap) { + spin: u32; + while !compare_and_swap(*heap.free_lock, 0, 1) wait_spin(*spin); +} + +heap_free_lock_release :: inline (heap: *Heap) { + atomic_store(*heap.free_lock, 0); +} + +heap_release_to_queue :: inline (heap: *Heap) { // 2.0.1: rpmalloc.c:1986 heap_release + heap.thread_current = false; // #jai fix E10 + heap_lock_acquire(); + if heap.released { heap_lock_release(); return; } // #jai fix U14: already released. + heap.released = true; + heap.owner_thread = HEAP_RELEASED_OWNER; // #jai fix U14 + #if ENABLE_STATISTICS statistics_sub_saturating(*global_counters.heap_count, 1); + if heap.prev heap.prev.next = heap.next; + if heap.next heap.next.prev = heap.prev; + if global_heap_used == heap global_heap_used = heap.next; + heap.next = global_heap_queue; + global_heap_queue = heap; + heap_lock_release(); +} + +heap_initialize :: (block: *void) -> *Heap { // 2.0.1: rpmalloc.c:1855 + heap := cast(*Heap) block; + memset(heap, 0, size_of(Heap)); + heap.id = 1 + atomic_add(*global_heap_id, 1); + return heap; +} + +// #jai fix (U6 part, needed by develop #413): #413 keeps a size class's last available page when it +// empties, so the next allocation of the class need not rebuild it. Every class used then pins one +// empty, committed page until the heap is released: 226 MiB after RET-02's burst, where the large +// classes use 16 MiB pages. thread_collect retires those pages through the normal free path, where +// the decommit thresholds apply. An empty page (block_used 0) has no blocks on the heap's local free +// list and none pending from other threads, so retiring it is safe. +// #jai fix U6 (rest): blocks freed by other threads sit on the heap's thread_free lists (full +// pages) or in a page's own thread-free token (pages that are not full) until the owner next needs a +// page of that type; an idle owner never does, so the memory stays committed (RET-03). thread_collect +// drains both: the heap lists are freed locally (pages that empty go to the free list), and every +// available page adopts its deferred frees, retiring it if it is now empty. +heap_drain_deferred_frees :: (heap: *Heap) { + for itype: 0..3 { + if !atomic_load(*heap.thread_free[itype]) continue; + block := cast(*Block) atomic_exchange(*heap.thread_free[itype], 0); + while block { + next_block := block.next; + block_deallocate(block); + block = next_block; + } + } + for iclass: 0..SIZE_CLASS_COUNT - 1 { + page := heap.page_available[iclass]; + while page { + next_page := page.next; + if atomic_load(*page.thread_free) { + page_adopt_thread_free_merging(page); // #jai fix E2 + if page.block_used == 0 page_available_to_free(page, keep_last = false); + } + page = next_page; + } + } +} + +// #jai fix E2: page_adopt_thread_free_block_list leaves a page alone while its local free list is +// non-empty, because claiming the thread-free list replaces that list (the allocation path only needs +// a list when the local one is empty). The drain needs every pending block: a page freed partly by its +// owner and partly by other threads kept its pending blocks, so it stayed in use and committed through +// thread_collect and trim() (RET-16), and at thread exit (RET-15). Here the claimed list is linked in +// front of the local one. +page_adopt_thread_free_merging :: (page: *Page) { + if !page.local_free { + page_adopt_thread_free_block_list(page); + return; + } + if !atomic_load(*page.thread_free) return; + token := atomic_exchange(*page.thread_free, 0); + head: *Block; + count := page_block_from_thread_free_list(page, token, *head); + if !count return; + tail := head; + for 2..count tail = tail.next; + tail.next = page.local_free; + page.local_free = head; + rpmalloc_assert(count <= page.block_used, "Page thread free list count internal failure"); + page.block_used -= count; + stat_add_free(page.heap, page.size_class, count); +} + +heap_retire_kept_pages :: (heap: *Heap) { + for iclass: 0..SIZE_CLASS_COUNT - 1 { + page := heap.page_available[iclass]; + if page && !page.next && page.block_used == 0 page_available_to_free(page, keep_last = false); + } +} + +heap_page_free_decommit :: (heap: *Heap, page_type: u32, page_retain_count: u32) { // 2.0.1: rpmalloc.c:2003 + page := heap.page_free[page_type]; + while page && page_retain_count { + page = page.next; + page_retain_count -= 1; + } + while page && !page_has(page, .IS_DECOMMITTED) { + page_decommit_memory_pages(page); + // Decommit can be refused (decommit disabled, page type not larger than the configured + // page size, or the kernel rejecting decommit of huge page mappings); then the remaining + // pages cannot be decommitted either. + if !page_has(page, .IS_DECOMMITTED) break; + heap.page_free_commit_count[page_type] -= 1; + page = page.next; + } +} + +// #jai develop #416 (851e255): decommit by time, not by count. Called when the free committed page +// count of a page type is at or above the overflow threshold. The excess is decommitted only once +// the count has stayed above the threshold for PAGE_DECOMMIT_DELAY_MS (dropping below restarts the +// delay), or at once at PAGE_DECOMMIT_LIMIT_FACTOR times the threshold. Decommitting on the count +// alone made every allocate/free phase larger than the threshold decommit its pages and fault them +// back in on the next phase (RET-10). thread_collect decommits the excess immediately. +PAGE_DECOMMIT_DELAY_MS :: 250; +PAGE_DECOMMIT_LIMIT_FACTOR :: 4; + +// #jai upstream PR #420 (fb4288f): decommits the most recently freed page, at the head of the free +// list, and moves it to the tail behind the committed pages, so the list keeps its committed pages +// first for reuse. +heap_page_free_decommit_head :: (heap: *Heap, page_type: u32) { + page := heap.page_free[page_type]; + if !page || page_has(page, .IS_DECOMMITTED) return; + page_decommit_memory_pages(page); + if !page_has(page, .IS_DECOMMITTED) return; + heap.page_free_commit_count[page_type] -= 1; + if page.next { + heap.page_free[page_type] = page.next; + heap.page_free_tail[page_type].next = page; + page.next = null; + heap.page_free_tail[page_type] = page; + } +} + +// The decommit delay (#416) and, #jai PR #420, the hard limit within it: a page freed with the count at +// PAGE_DECOMMIT_LIMIT_FACTOR times the threshold is decommitted on its own. #416 decommitted every free +// page down to the retain count there, including the pages the next phase reuses (W-12). +heap_page_free_overflow :: no_inline (heap: *Heap, page_type: u32) { + limit := cast(u64) global_page_free_overflow[page_type] * PAGE_DECOMMIT_LIMIT_FACTOR; + now := os_monotonic_time_ms(); + since := heap.page_free_overflow_ms[page_type]; + if !since { + since = ifx now then now else 1; + heap.page_free_overflow_ms[page_type] = since; + } + if elapsed_ms(now, since) < PAGE_DECOMMIT_DELAY_MS { + if heap.page_free_commit_count[page_type] >= limit heap_page_free_decommit_head(heap, page_type); + return; + } + heap.page_free_overflow_ms[page_type] = 0; + heap_page_free_decommit(heap, page_type, global_page_free_retain[page_type]); +} + +// #jai fix U5 (second half): the decommit delay was checked only when another page was freed, so a +// heap that stopped freeing kept its free pages over the threshold (up to 4x) for good (RET-12). +// The allocation slow path, which runs whenever a page is needed, checks it too: one test of four +// timestamps while no delay is running. +heap_recheck_decommit_delay :: inline (heap: *Heap) { + if !(heap.page_free_overflow_ms[0] | heap.page_free_overflow_ms[1] | heap.page_free_overflow_ms[2] | heap.page_free_overflow_ms[3]) + return; + heap_recheck_decommit_delay_now(heap); +} + +heap_recheck_decommit_delay_now :: no_inline (heap: *Heap) { + now := os_monotonic_time_ms(); + for itype: 0..3 { + since := heap.page_free_overflow_ms[itype]; + if !since || elapsed_ms(now, since) < PAGE_DECOMMIT_DELAY_MS continue; + heap.page_free_overflow_ms[itype] = 0; + heap_page_free_decommit(heap, cast(u32) itype, global_page_free_retain[itype]); + } +} + +// #jai develop #418 (4b83f74), #419 (1d5730b): trims the memory a released heap holds for reuse. +// Until another thread adopts the heap nothing reuses that memory or runs the decommit delay, so a +// heap passed along by many short-lived threads would keep all of it (RET-11). The empty +// medium-large and large pages kept as the last available page of their class are retired, and free +// pages of those types are decommitted to below the overflow threshold, the most a heap holds without +// the delay. Small and medium-small pages are cheap to keep and costly to fault back in; they are +// left to the decommit delay (#419). +heap_release_free_pages :: (heap: *Heap) { + for iclass: 0..SIZE_CLASS_COUNT - 1 { + page := heap.page_available[iclass]; + if !page || page.next || page.block_used || page_get_type(page) < .MEDIUM_LARGE continue; + heap.page_available[iclass] = null; + page_set(page, .IS_FREE, true); + page_set(page, .IS_ZERO, false); + page_type := cast(u32) page_get_type(page); + page.next = heap.page_free[page_type]; + heap.page_free[page_type] = page; + if !page.next heap.page_free_tail[page_type] = page; // #jai PR #420 + heap.page_free_commit_count[page_type] += 1; + } + for itype: cast(s64) Page_Type.MEDIUM_LARGE..cast(s64) Page_Type.LARGE { + heap.page_free_overflow_ms[itype] = 0; + if heap.page_free_commit_count[itype] >= global_page_free_overflow[itype] + heap_page_free_decommit(heap, cast(u32) itype, global_page_free_overflow[itype] - 1); + } +} + +// #jai develop #419: releases a heap (by its exiting thread, or a first-class heap) for reuse. With +// other heaps already queued it is likely to stay unused for a while and is trimmed at once. With +// the queue empty a thread is likely to adopt it shortly, so the trim is left to the adopter rather +// than delaying the release while threads wait for a heap. +heap_release_trim :: (heap: *Heap) { + heap_lock_acquire(); + queued := global_heap_queue != null; + heap_lock_release(); + // With the queue empty the trim waits for an adopter, which may never come (U5); threads that + // exit therefore use heap_release_on_thread_exit instead. Only first-class heaps come here. + if queued heap_release_free_pages(heap); + else heap.release_trim = 1; + heap_release_to_queue(heap); +} + +// #jai fix U5 (first half): a thread that exits trims its heap at once, over every page type, +// instead of leaving it to an adopter (#419), which may never come: when the last worker of a +// program exits, its heap's memory stayed committed (RET-11). Kept empty pages of every class are +// retired and free pages decommitted to below the overflow threshold. The rest of U5, re-checking +// the decommit delay on the allocation slow path, comes with the retention fixes. +// #jai fix E1 (found in Ex4 1.45.0): that trim kept free pages up to just under each type's overflow +// threshold (about 36 MiB) for an adopter, and blocks other threads had freed into the heap stayed +// pending, keeping their pages in use. With many worker threads that exit and no thread adopting +// their heaps, all of it stayed committed: 625 MiB after RET-15's 8 threads, and 709 vs 139 MB of +// private commit in Ex4's index build with 24 workers. The exiting thread now does for its heap what +// trim() does: drain the pending frees (with fix E2), retire the kept empty pages, and decommit every +// free page. An adopter recommits what it uses. Frees that arrive after the exit still wait for one. +// Callers make the heap the thread heap first: the drain frees through the thread heap's paths. +heap_release_on_thread_exit :: (heap: *Heap) { + heap_drain_deferred_frees(heap); + heap_retire_kept_pages(heap); + for itype: 0..3 { + heap.page_free_overflow_ms[itype] = 0; + if heap.page_free_commit_count[itype] heap_page_free_decommit(heap, cast(u32) itype, 0); + } + heap_release_to_queue(heap); +} + +heap_make_free_page_available :: inline (heap: *Heap, size_class: u32, page: *Page) -> s32 { // 2.0.1: rpmalloc.c:2022 + // #jai fix U4: recommit before anything else. Upstream linked the page into page_available + // first, so when the recommit failed the still-decommitted page stayed available and the next + // allocation wrote into memory that was not committed (an access violation on Windows; C 2.0.1 + // does the same in OS-09). On failure nothing is changed; the caller keeps the page. + if page_has(page, .IS_DECOMMITTED) && page_commit_memory_pages(page) != 0 return 1; + #if TEST_HOOKS atomic_add(*test_page_reinits, 1); + page.size_class = size_class; + page.block_size = global_size_class[size_class].block_size; + page.block_count = global_size_class[size_class].block_count; + page.block_used = 0; + page.block_initialized = 0; + page.local_free = null; + page_set(page, .IS_FULL, false); + page_set(page, .IS_FREE, false); + page_set(page, .HAS_ALIGNED_BLOCK, false); + page.heap = heap; + page.heap_fast = heap; // #jai PR #422 + head := heap.page_available[size_class]; + page.next = head; + page.prev = null; + atomic_store(*page.thread_free, 0); + if head head.prev = page; + heap.page_available[size_class] = page; + return 0; +} + +// Find or allocate a span for the given page type. +heap_get_span :: inline (heap: *Heap, page_type: Page_Type) -> *Span { // 2.0.1: rpmalloc.c:2049 + // Fast path: an available span for the page type. + if heap.span_partial[page_type] return heap.span_partial[page_type]; + + // Fallback path: map more memory. + offset, mapped_size: u64; + span := cast(*Span) global_memory_interface.memory_map(SPAN_SIZE, SPAN_SIZE, *offset, *mapped_size); + if span { + #if HEAP_STATISTICS heap_stat_add(*heap.stats.mapped_size, mapped_size); // #jai fix U13: only when the map succeeded. + rpmalloc_assert(!(cast(u64) span & (SPAN_SIZE - 1)), "memory_map returned a block not aligned to the span size"); + stat_inc_span_map(heap, page_type); + page_count: u32; + page_size: u32; + page_address_mask: u64; + if page_type == .SMALL { + page_count = SPAN_SIZE / SMALL_PAGE_SIZE; + page_size = SMALL_PAGE_SIZE; + page_address_mask = SMALL_PAGE_MASK; + } else if page_type == .MEDIUM_SMALL { + page_count = SPAN_SIZE / MEDIUM_SMALL_PAGE_SIZE; + page_size = MEDIUM_SMALL_PAGE_SIZE; + page_address_mask = MEDIUM_SMALL_PAGE_MASK; + } else if page_type == .MEDIUM_LARGE { + page_count = SPAN_SIZE / MEDIUM_LARGE_PAGE_SIZE; + page_size = MEDIUM_LARGE_PAGE_SIZE; + page_address_mask = MEDIUM_LARGE_PAGE_MASK; + } else { + page_count = SPAN_SIZE / LARGE_PAGE_SIZE; + page_size = LARGE_PAGE_SIZE; + page_address_mask = LARGE_PAGE_MASK; + } + // Commit the first page for reserve-only spans (fully committed spans commit nothing here); + // record the commit model on the span only after its header is committed. + commit_on_demand := os_commit_on_demand(); + commit_prefix_shift: u32 = 0; + while (cast(u64) 1 << commit_prefix_shift) < global_config.page_size commit_prefix_shift += 1; + // #jai fix E3: the first chunk of page 0 only; page_initialize_blocks commits the rest. + first_commit := page_commit_chunk_size(page_size, cast(u64) 1 << commit_prefix_shift); + #if ENABLE_DECOMMIT { + if commit_on_demand { + if global_memory_interface.memory_commit(span, first_commit) != 0 { + global_memory_interface.memory_unmap(span, offset, mapped_size); + return null; + } + } + } + page_set(*span.page, .COMMIT_ON_DEMAND, commit_on_demand); + page_set_commit_prefix_shift(*span.page, commit_prefix_shift); + page_set_committed_chunks(*span.page, 1); + #if HEAP_STATISTICS { + #if ENABLE_DECOMMIT heap_stat_add(*heap.stats.committed_size, ifx commit_on_demand then first_commit else page_size); + else heap_stat_add(*heap.stats.committed_size, mapped_size); + } + span.heap = heap; + span.page_type = page_type; + span.page_count = page_count; + span.page_size = page_size; + span.page_address_mask = page_address_mask; + span.offset = cast(u32) offset; + span.mapped_size = mapped_size; + span_register(span); // #jai for IS_THIS_YOURS + + heap.span_partial[page_type] = span; + } + return span; +} + +heap_get_page_generic :: (heap: *Heap, size_class: u32) -> *Page { // 2.0.1: rpmalloc.c:2129 + page_type := get_page_type(size_class); + heap_recheck_decommit_delay(heap); // #jai fix U5 + #if HUGE_CACHE_SLOT_COUNT huge_cache_purge_if_due(); // #jai fix E5 + + // Check for a free page from multithreaded deallocations. + block_mt := atomic_load(*heap.thread_free[page_type]); + if block_mt != 0 { + // #jai develop #415: claim the whole list in one exchange; concurrent pushes land either + // before it (and are claimed) or after. + block_mt = atomic_exchange(*heap.thread_free[page_type], 0); + block := cast(*Block) block_mt; + while block { + next_block := block.next; + block_deallocate(block); + block = next_block; + } + // Retry after processing the deferred thread frees. + return heap_get_page(heap, size_class); + } + + // Check for a free page. + page := heap.page_free[page_type]; + if page { + heap.page_free[page_type] = page.next; + if !page.next heap.page_free_tail[page_type] = null; // #jai PR #420 + if !page_has(page, .IS_DECOMMITTED) { + rpmalloc_assert(heap.page_free_commit_count[page_type] > 0, "Free committed page count out of sync"); + heap.page_free_commit_count[page_type] -= 1; + // #jai develop #416: dropping below the threshold restarts the decommit delay. + if heap.page_free_commit_count[page_type] < global_page_free_overflow[page_type] + heap.page_free_overflow_ms[page_type] = 0; + } + if heap_make_free_page_available(heap, size_class, page) != 0 { + // #jai fix U4: the recommit failed; put the (still decommitted) page back for later, at the + // tail behind the committed pages (#jai PR #420). + page.next = null; + if heap.page_free_tail[page_type] heap.page_free_tail[page_type].next = page; + else heap.page_free[page_type] = page; + heap.page_free_tail[page_type] = page; + return null; + } + return page; + } + rpmalloc_assert(heap.page_free_commit_count[page_type] == 0, "Free committed page count out of sync"); + + if heap.id == 0 { + // The thread is not initialised yet: assign a heap and try again. + initialize(null); + return heap_get_page(get_thread_heap(), size_class); + } + + // Fallback path: find or allocate a span for the size class. If the thread was not + // initialised, the heap for the new span differs from the local heap variable here (the + // default heap), so use the span page heap instead. + span := heap_get_span(heap, page_type); + if span { + page = span_allocate_page(span); + if !page return null; // #jai fix U4: the page's commit failed; upstream dereferenced null. + if heap_make_free_page_available(page.heap, size_class, page) != 0 return null; + } + return page; +} + +// Find or allocate a page for the given size class. +heap_get_page :: (heap: *Heap, size_class: u32) -> *Page { // 2.0.1: rpmalloc.c:2186 + // Fast path: an available page for the size class. + page := heap.page_available[size_class]; + if page return page; + return heap_get_page_generic(heap, size_class); +} + +// Pop a block from the heap local free list. +heap_pop_local_free :: inline (heap: *Heap, size_class: u32) -> *Block { // 2.0.1: rpmalloc.c:2196 + free_list := *heap.local_free[size_class]; + block := free_list.*; + if block free_list.* = block.next; + return block; +} + +// Generic allocation path from heap pages, spans or a new mapping. +heap_allocate_block_small_to_large :: no_inline (heap: *Heap, size_class: u32, zero: bool) -> *void { // 2.0.1: rpmalloc.c:2206 + page := heap_get_page(heap, size_class); + if page { + block := page_allocate_block(page, zero); + // Count on the page owner, which may differ from heap when an uninitialised thread is + // routed to its real heap, so the allocation and its later free count on the same heap. + // #jai fix E3: only once the block exists; committing it can fail. + if block stat_inc_alloc(page.heap, size_class); + return block; + } + return null; +} + +heap_allocate_block_huge :: no_inline (heap: *Heap, size: u64, zero: bool) -> *void { // 2.0.1: rpmalloc.c:2219 + if heap.id == 0 { + // The thread is not initialised yet: assign a heap and try again. + initialize(null); + heap = get_thread_heap(); + } + // #jai fix U1: upstream checks the size only under ENABLE_VALIDATE_ARGS, so without it + // size + SPAN_HEADER_SIZE and the page rounding wrap, and alloc(-1) returns a tiny block. + // Refusing here bounds every path into a mapping; the cost is one compare on a path that maps. + if size >= MAX_ALLOC_SIZE return null; + alloc_size := get_page_aligned_size(size + SPAN_HEADER_SIZE); + offset, mapped_size: u64; + block: *void; + from_cache := false; + #if HUGE_CACHE_SLOT_COUNT { + // Reuse a recently freed huge mapping of fitting size if there is one. + huge_cache_purge_if_due(); // #jai fix E5: huge allocations age the cache too. + block = huge_cache_pop(alloc_size); + from_cache = block != null; + } + committed_at_map := false; + if !block { + // #jai fix E9: with the OS's own interface, map and commit in one call (see os_mmap_committed). + // A custom interface keeps the separate map and commit calls. + if global_memory_interface.memory_map == os_mmap && os_commit_on_demand() { + block = os_mmap_committed(alloc_size, SPAN_SIZE, *offset, *mapped_size); + committed_at_map = true; + } else { + block = global_memory_interface.memory_map(alloc_size, SPAN_SIZE, *offset, *mapped_size); + } + } + if block { + span := cast(*Span) block; + rpmalloc_assert(!(cast(u64) span & (SPAN_SIZE - 1)), "memory_map returned a block not aligned to the span size"); + if !from_cache stat_inc_span_map(heap, .HUGE); + #if ENABLE_DECOMMIT { + if !from_cache && !committed_at_map && os_commit_on_demand() && global_memory_interface.memory_commit(span, alloc_size) != 0 { + global_memory_interface.memory_unmap(block, offset, mapped_size); + return null; + } + } + span.heap = heap; + span.page_type = .HUGE; + span.page_address_mask = LARGE_PAGE_MASK; + // Huge blocks never use the per-page commit path; set explicitly so a reused cached + // mapping carries no stale value. + page_set(*span.page, .COMMIT_ON_DEMAND, false); + if !from_cache { + span.page_size = cast(u32) global_config.page_size; + span.page_count = cast(u32)(alloc_size / global_config.page_size); + span.offset = cast(u32) offset; + span.mapped_size = mapped_size; + } + #if HEAP_STATISTICS { + // #jai fix U13: from the span's own fields, so a mapping reused from the huge cache counts + // its real size (upstream added 0 mapped bytes and the request's commit for those), and + // the free subtracts exactly this. + heap_stat_add(*heap.stats.mapped_size, span.mapped_size); + #if ENABLE_DECOMMIT heap_stat_add(*heap.stats.committed_size, cast(u64) span.page_count * span.page_size); + else heap_stat_add(*heap.stats.committed_size, span.mapped_size); + } + span.page.heap = heap; + page_set(*span.page, .IS_FULL, true); + span.page.heap_fast = null; // #jai PR #422: huge blocks take the slow path. + page_set_type(*span.page, .HUGE); + #if ENABLE_STATISTICS statistics_add_with_peak(*global_counters.huge_alloc, *global_counters.huge_alloc_peak, cast(u64) span.page_count * span.page_size); + // Keep track of the span if this is a first-class heap. + if !heap.owner_thread { + span.next = heap.span_used[Page_Type.HUGE]; // Unlinked again on free (#jai fix U3). + heap.span_used[Page_Type.HUGE] = span; + } + if !from_cache span_register(span); // #jai for IS_THIS_YOURS + ptr := pointer_offset(block, SPAN_HEADER_SIZE); + // #jai develop #406 (4301fb8): a fresh mapping from the OS interface is already zero + // (anonymous mmap and VirtualAlloc guarantee it); clearing it again faults in and commits every + // page. Reused cached mappings and custom memory interfaces give no such guarantee. + if zero && (from_cache || global_memory_interface.memory_map != os_mmap) memset(ptr, 0, cast(s64) size); + // #jai fix U13: the capacity, which the free subtracts; upstream added the requested size and + // subtracted the page-rounded one, so allocated_size drifted (API-06). + #if HEAP_STATISTICS heap_stat_add(*heap.stats.allocated_size, cast(u64) span.page_size * span.page_count); + return ptr; + } + return null; +} + +heap_allocate_block_generic :: no_inline (heap: *Heap, size: u64, zero: bool) -> *void { // 2.0.1: rpmalloc.c:2305 + size_class := get_size_class(size); + if size_class < SIZE_CLASS_COUNT { + #if HEAP_STATISTICS heap_stat_add(*heap.stats.allocated_size, global_size_class[size_class].block_size); + block := heap_pop_local_free(heap, size_class); + if block { + // Fast track with a small block available in the heap local free list; a hit means + // this heap owns the block. + stat_inc_alloc(heap, size_class); + if zero memset(block, 0, global_size_class[size_class].block_size); + return block; + } + return heap_allocate_block_small_to_large(heap, size_class, zero); + } + return heap_allocate_block_huge(heap, size, zero); +} + +// Find or allocate a block of the given size. +heap_allocate_block :: inline (heap: *Heap, size: u64, zero: bool) -> *void { // 2.0.1: rpmalloc.c:2330 + if size <= SMALL_GRANULARITY * 64 { + size_class := get_size_class_tiny(size); + block := heap_pop_local_free(heap, size_class); + if block { + // Fast track with a small block available in the heap local free list. + if zero memset(block, 0, global_size_class[size_class].block_size); + #if HEAP_STATISTICS heap_stat_add(*heap.stats.allocated_size, global_size_class[size_class].block_size); + stat_inc_alloc(heap, size_class); + return block; + } + #if HEAP_STATISTICS heap_stat_add(*heap.stats.allocated_size, global_size_class[size_class].block_size); + // The size class is already known: refill directly, skipping the generic path's size + // class recomputation and dead local free list pop. + return heap_allocate_block_small_to_large(heap, size_class, zero); + } + return heap_allocate_block_generic(heap, size, zero); +} + +// An aligned request allocates size + alignment and moves the pointer up to the alignment. The page +// is then marked, so that freeing the moved pointer finds the block start (page_block_realign). +heap_allocate_block_aligned :: (heap: *Heap, alignment: u64, size: u64, zero: bool) -> *void { // 2.0.1: rpmalloc.c:2356 + if alignment <= SMALL_GRANULARITY return heap_allocate_block(heap, size, zero); + + // #jai fix U1: upstream makes these two checks only under ENABLE_VALIDATE_ARGS. Without them + // size + alignment wraps and a tiny block comes back, and a non-power-of-two alignment gives a + // misaligned block. This path over-allocates anyway; the checks cost two compares. + // #jai Without computing size + alignment, which traps with arithmetic_overflow_check. + if size > 0xFFFF_FFFF_FFFF_FFFF - alignment return null; + if alignment & (alignment - 1) return null; + if alignment >= RPMALLOC_MAX_ALIGNMENT return null; + + align_mask := alignment - 1; + // #jai develop #405 (a3abe2e): blocks start PAGE_HEADER_SIZE bytes into a page (or span) whose + // start is aligned to at least the page size. Rounding the size up to a multiple of the alignment + // (and at least the alignment) selects a size class whose block size is a multiple of the + // alignment: up to 1 KiB every multiple of the granularity is a class, above that every class size + // is a multiple of 256. Every block of that class is then naturally aligned: no over-allocation, + // no realignment, and the page keeps the fast free path. + // #jai PR #424: up to NATURAL_ALIGNMENT_LIMIT, a class whose block size is a multiple of the + // alignment starts its blocks at an offset that is a multiple of it too, so the plain allocation of + // the size rounded up to the alignment is aligned. + if alignment > PAGE_HEADER_SIZE && alignment <= NATURAL_ALIGNMENT_LIMIT { + aligned_size := (size + align_mask) & ~align_mask; + if aligned_size >= size && aligned_size <= LARGE_BLOCK_SIZE_LIMIT { + size_class := get_size_class(aligned_size); + block_size := ifx size_class < SIZE_CLASS_COUNT then global_size_class[size_class].block_size else 0; + if cast(u64) lowest_set_bit(block_size) >= alignment return heap_allocate_block(heap, aligned_size, zero); + } + } + if alignment <= PAGE_HEADER_SIZE { + aligned_size := (size + align_mask) & ~align_mask; + if aligned_size < alignment aligned_size = alignment; + if aligned_size >= size { + block := heap_allocate_block(heap, aligned_size, zero); + rpmalloc_assert(!(cast(u64) block & align_mask), "Natural block alignment internal failure"); + return block; + } + } + block := heap_allocate_block(heap, size + alignment, zero); + if cast(u64) block & align_mask { + block = cast(*void)((cast(u64) block & ~align_mask) + alignment); + // Mark the page as having aligned blocks. + span := block_get_span(block); + page := span_get_page_from_block(span, block); + // #jai PR #424: write the flag word only on the page's first aligned block. + if !page_has(page, .HAS_ALIGNED_BLOCK) { + page_set(page, .HAS_ALIGNED_BLOCK, true); + page.heap_fast = null; // #jai PR #422 + } + } + return block; +} + +heap_reallocate_block :: no_inline (heap: *Heap, block: *void, size: u64, old_size: u64, flags: Realloc_Flags) -> *void { // 2.0.1: rpmalloc.c:2389 + if block { + // Grab the span through the guaranteed span alignment. + span := block_get_span(block); + if span.page_type <= .LARGE { + // Normal sized block. + page := span_get_page_from_block(span, block); + // #jai upstream PR #423 (951f903): a pointer from the allocator is at its block start + // unless the page holds aligned blocks, so the division locating the block start is only + // needed for those pages. + block_origin := block; + if page_has(page, .HAS_ALIGNED_BLOCK) block_origin = page_block_realign(page, block); + if !old_size old_size = cast(u64)(cast(s64) page.block_size - pointer_diff(block, block_origin)); + if cast(u64) page.block_size >= size { + // Still fits in the block: never mind saving memory, but preserve the data if the + // alignment changed. + if block != block_origin && !(flags & .NO_PRESERVE) + memmove_rp(block_origin, block, cast(s64) old_size); + return block_origin; + } + } else { + // Huge block. + block_start := pointer_offset(span, SPAN_HEADER_SIZE); + capacity := cast(u64) span.page_size * span.page_count - cast(u64) pointer_diff(block, span); + if !old_size old_size = capacity; + // #jai fix U12: upstream judged the fit on old_size, which is also the caller's live size, + // with size < old_size, so a same-size request with the real old size reallocated (RE-06). + // The fit is judged on the capacity, and an equal size fits. + if size <= capacity && size >= capacity / 2 && size > LARGE_BLOCK_SIZE_LIMIT { + // Still fits, still huge, and saves less than half the memory: keep it, but + // preserve the data if the alignment changed. + if block_start != block && !(flags & .NO_PRESERVE) + memmove_rp(block_start, block, cast(s64) old_size); + return block_start; + } + } + } else { + old_size = 0; + } + + if flags & .GROW_OR_FAIL return null; + + // The size is greater than the block size, or shrinking saves enough memory: allocate a new + // block and free the old one. Avoid hysteresis by overallocating small increases (below 37%). + lower_bound := old_size + (old_size >> 2) + (old_size >> 3); + new_size := ifx size > lower_bound then size else (ifx size > old_size then lower_bound else size); + old_block := block; + block = heap_allocate_block(heap, new_size, false); + if block && old_block { + if !(flags & .NO_PRESERVE) + memcpy(block, old_block, cast(s64) ifx old_size < new_size then old_size else new_size); + block_deallocate(old_block); + } + return block; +} + +heap_reallocate_block_aligned :: (heap: *Heap, block: *void, alignment: u64, size: u64, old_size: u64, flags: Realloc_Flags) -> *void { // 2.0.1: rpmalloc.c:2444 + if alignment <= SMALL_GRANULARITY return heap_reallocate_block(heap, block, size, old_size, flags); + + no_alloc := (flags & .GROW_OR_FAIL) != 0; + usable_size := ifx block then block_usable_size(block) else 0; + if usable_size >= size && !(cast(u64) block & (alignment - 1)) { + if no_alloc || size >= usable_size / 2 return block; + } + // The aligned allocation marks the page as having aligned blocks. + old_block := block; + block = ifx !no_alloc then heap_allocate_block_aligned(heap, alignment, size, false) else null; + if block { + if !(flags & .NO_PRESERVE) && old_block { + if !old_size old_size = usable_size; + memcpy(block, old_block, cast(s64) ifx old_size < size then old_size else size); + } + if old_block block_deallocate(old_block); + } + return block; +} + +// Upstream heap_free_all, renamed: the public rpmalloc_heap_free_all is heap_free_all in this port. +heap_unmap_all :: (heap: *Heap) { // 2.0.1: rpmalloc.c:2471 heap_free_all + for itype: 0..3 { + span := heap.span_partial[itype]; + while span { + span_next := span.next; + span_unmap(span); + span = span_next; + } + heap.span_partial[itype] = null; + heap.page_free[itype] = null; + heap.page_free_tail[itype] = null; // #jai PR #420 + heap.page_free_commit_count[itype] = 0; + heap.page_free_overflow_ms[itype] = 0; // #jai develop #416 + atomic_store(*heap.thread_free[itype], 0); + } + for itype: 0..4 { + span := heap.span_used[itype]; + while span { + span_next := span.next; + #if ENABLE_STATISTICS { + if itype == cast(s64) Page_Type.HUGE statistics_sub_saturating(*global_counters.huge_alloc, cast(u64) span.page_count * span.page_size); + } + span_unmap(span); // Only live huge blocks are listed (#jai fix U3). + span = span_next; + } + heap.span_used[itype] = null; + } + memset(heap.local_free.data, 0, size_of(type_of(heap.local_free))); + memset(heap.page_available.data, 0, size_of(type_of(heap.page_available))); + #if ENABLE_STATISTICS { + // Every block in this heap has been released, so reverse the per size class counters. + for * heap.size_use { + it.free_total += it.alloc_current; + it.alloc_current = 0; + } + } +} + +//////////// +/// +/// Extern interface (2.0.1: rpmalloc.c:2510-2660) +/// +////// +// rpmalloc_is_thread_initialized:2517, rpmalloc:2522, rpzalloc:2534, rpfree:2546, +// rpcalloc:2553, rprealloc:2577, rpaligned_realloc:2589, rpaligned_alloc:2601, +// rpaligned_zalloc:2607, rpaligned_calloc:2613, rpmemalign:2637, rpposix_memalign:2643, +// rpmalloc_usable_size:2653. (rpmalloc_linker_reference:2658 is C-only; not ported.) + +//////////// +/// +/// Initialization and finalization (2.0.1: rpmalloc.c:2661-3421) +/// +////// +// The exported procedures of this section are further down, after #scope_export. +// os_read_system_file:2684 lives in os_posix.jai. (malloc.c and rpnew.h, the libc / C++ +// overrides, are not ported.) + +// Runs at thread exit through the FLS / pthread key (2.0.1: rpmalloc.c:2668). #c_call: the OS calls +// it, so it has no Jai context of its own. +rpmalloc_thread_destructor :: (value: *void) #c_call { + fresh: #Context; + push_context fresh { + // On the main thread this means rpmalloc_finalize has not been called and shutdown is + // forced (through _exit) or unclean. + if get_thread_id() == global_main_thread_id return; + // #jai fix U22: when another thread calls exit, Windows stops every other thread and then + // runs this destructor on the exiting one; releasing its heap takes global_heap_lock, which + // a stopped thread may hold, and the process never exits (LIFE-05). The process is ending + // anyway, so keep the heap. + if os_process_exiting() return; + // #jai upstream calls rpmalloc_thread_finalize, which reads its _Thread_local heap. Here the + // key itself holds the heap, and POSIX clears it before calling the destructor, so + // thread_finalize would find no heap and release nothing (every exiting POSIX thread leaked + // its heap: RET-04/RET-05 on macOS). Release the heap passed in, and leave the key alone: + // storing the default heap would make POSIX run the destructor again. + heap := cast(*Heap) value; + if heap && heap != global_heap_default { + // #jai fix E1: the release drains the frees pending in the heap, and frees take the owner's + // path only when the heap is the thread heap. POSIX has cleared the key, so set it for the + // release, and clear it again after (POSIX runs the destructor again for a non-null value). + // Windows still has the value here. + #if OS != .WINDOWS os_tls_set(heap); + heap_release_on_thread_exit(heap); // #jai fix U5 + #if OS != .WINDOWS os_tls_set(null); + } else if heap == global_heap_default { + heap_release_parked_on_exit(); // #jai fix E7: the thread exits with its heap parked. + } + } +} + +#if ENABLE_LEAK_DETECTION { + +// Size class blocks still allocated across every heap. Non-zero once all threads have finalized +// means the application has not freed all its allocations (a leak or a double free). +statistics_outstanding_blocks :: () -> s64 { // 2.0.1: rpmalloc.c:2975 + lists := *Heap.[global_heap_used, global_heap_queue, global_heap_pristine]; + outstanding: s64; + for list: lists { + heap := list; + while heap { + for heap.size_use outstanding += it.alloc_current; + heap = heap.next; + } + } + return outstanding; +} + +// Cross-thread freed blocks parked in a span's per page deferred lists, over every initialised page +// (a full page is not reachable from page_available but can still hold deferred frees). The block +// count is in the token's high 32 bits. +span_deferred_block_count :: (span: *Span) -> s64 { // 2.0.1: rpmalloc.c:2992 + deferred: s64; + for ipage: 0..cast(s64) span.page_initialized - 1 { + page := cast(*Page) pointer_offset(span, cast(u64) span.page_size * cast(u64) ipage); + token := atomic_load(*page.thread_free); + deferred += cast(s64)((token >> 32) & 0x7FFF_FFFF); // #jai fix PR #426: mask the full flag. + } + return deferred; +} + +// Blocks freed cross-thread but not yet applied to the owning heap's alloc_current. Subtracting +// them from the outstanding count gives the blocks the application has really leaked. +statistics_pending_deferred_blocks :: () -> s64 { // 2.0.1: rpmalloc.c:3006 + lists := *Heap.[global_heap_used, global_heap_queue, global_heap_pristine]; + deferred: s64; + for list: lists { + heap := list; + while heap { + // Page types small to large carry the heap's full page deferred list and the per page + // deferred lists of every span. Huge blocks are never deferred per block. + for itype: 0..3 { + block := cast(*Block) atomic_load(*heap.thread_free[itype]); + while block { + deferred += 1; + block = block.next; + } + if heap.span_partial[itype] deferred += span_deferred_block_count(heap.span_partial[itype]); + span := heap.span_used[itype]; + while span { + deferred += span_deferred_block_count(span); + span = span.next; + } + } + heap = heap.next; + } + } + return deferred; +} + +} // #if ENABLE_LEAK_DETECTION + +// +// Test hooks (TEST_HOOKS): fault injection inside the OS layer, so tests exercise the real +// error paths, plus direct access to the OS entry points for the white-box tests. +// + +#if TEST_HOOKS { + test_fail_maps: s64; // The next N os_mmap calls fail. + test_fail_commits: s64; // The next N os_mcommit calls fail. + test_pages_decommitted: [4] s64; // Pages decommitted / recommitted so far, per page type. + test_before_mark_full: (page: *void); // Called just before the owner marks a page full. + test_page_reinits: s64; // Pages (re)initialised for a size class (#413). + test_pages_recommitted: [4] s64; + + test_take_fault :: (counter: *s64) -> bool { + while true { + n := atomic_load(counter); + if n <= 0 return false; + if compare_and_swap(counter, n, n - 1) return true; + } + return false; + } +} + +// The count * size product of the calloc family; ok is false for a refused request. +// #jai fix #417 (as upstream develop 06b18dd0): the product is checked whatever ENABLE_VALIDATE_ARGS +// says. Unchecked, calloc(2^33, 2^33) wraps to 0 and returns a small block. +calloc_total :: inline (num: s64, size: s64) -> total: u64, ok: bool { + n := cast,no_check(u64) num; + s := cast,no_check(u64) size; + if n && s > 0xFFFF_FFFF_FFFF_FFFF / n return 0, false; // 2.0.1: SizeTMult / __builtin_umull_overflow + total := n * s; + if total >= MAX_ALLOC_SIZE return 0, false; + return total, true; +} + +// #jai upstream PR #423 (951f903): a block from a page without aligned blocks that still fits the +// size is kept as it is, without the call to heap_reallocate_block. +realloc_fits_in_place :: inline (ptr: *void, size: u64) -> bool { + if !ptr return false; + span := block_get_span(ptr); + if span.page_type > .LARGE return false; + page := span_get_page_from_block(span, ptr); + return !page_has(page, .HAS_ALIGNED_BLOCK) && cast(u64) page.block_size >= size; +} + +// Jai Allocator protocol helpers used by rpmalloc_allocator_proc (module.jai). +// RESIZE passes the caller's old_size through to heap_reallocate_block (2.0.1: rpmalloc.c:2389). +allocator_proc_resize :: (heap: *Heap, ptr: *void, size: s64, old_size: s64) -> *void { + #if ENABLE_VALIDATE_ARGS { + if cast,no_check(u64) size >= MAX_ALLOC_SIZE return null; // #jai null, not ptr (upstream L6, see rprealloc). + } + // #jai PR #423: the Allocator protocol's RESIZE is this port's realloc, so it gets the fast path. + if realloc_fits_in_place(ptr, cast,no_check(u64) size) return ptr; + if !heap heap = get_thread_heap(); + // #jai the protocol's old_size goes through (upstream's rprealloc passes 0 and derives the block's + // whole capacity), so a moved block copies only the caller's live bytes. Safe since the U12 fix. + live := ifx old_size > 0 then cast(u64) old_size else 0; + return heap_reallocate_block(heap, ptr, cast,no_check(u64) size, live, 0); +} + +#scope_export + +#if TEST_HOOKS { + test_fail_next_maps :: (count: s64) { atomic_store(*test_fail_maps, count); } + test_fail_next_commits :: (count: s64) { atomic_store(*test_fail_commits, count); } + test_os_page_size :: () -> u64 { os_initialize(); return os_page_size; } + test_os_map_granularity :: () -> u64 { os_initialize(); return os_map_granularity; } + test_os_mmap :: (size: u64, alignment: u64, offset: *u64, mapped_size: *u64) -> *void { return os_mmap(size, alignment, offset, mapped_size); } + test_os_mcommit :: (address: *void, size: u64) -> s32 { return os_mcommit(address, size); } + test_os_mdecommit :: (address: *void, size: u64) -> s32 { return os_mdecommit(address, size); } + test_os_munmap :: (address: *void, offset: u64, mapped_size: u64) { os_munmap(address, offset, mapped_size); } + test_os_commit_on_demand :: () -> bool { return os_commit_on_demand(); } + + // Size classes and geometry. + test_size_class_count :: () -> s64 { return SIZE_CLASS_COUNT; } + test_size_class_block_size :: (index: s64) -> u32 { return global_size_class[index].block_size; } + test_size_class_block_count :: (index: s64) -> u32 { return global_size_class[index].block_count; } + test_get_size_class :: (size: u64) -> u32 { return get_size_class(size); } + test_get_size_class_tiny :: (size: u64) -> u32 { return get_size_class_tiny(size); } + test_get_page_type :: (size_class: u32) -> u32 { return cast(u32) get_page_type(size_class); } + test_page_size_of_type :: (page_type: u32) -> u64 { + if page_type == { + case 0; return SMALL_PAGE_SIZE; + case 1; return MEDIUM_SMALL_PAGE_SIZE; + case 2; return MEDIUM_LARGE_PAGE_SIZE; + case 3; return LARGE_PAGE_SIZE; + } + return 0; + } + test_span_size :: () -> u64 { return SPAN_SIZE; } + test_page_header_size :: () -> u64 { return PAGE_HEADER_SIZE; } + test_msb_index :: (x: u64) -> u32 { return msb_index(x); } + test_usable_huge_page_size :: (reported: u64) -> u64 { return usable_huge_page_size(reported); } + test_huge_cache_purge_aged :: (now: u64) { #if HUGE_CACHE_SLOT_COUNT huge_cache_purge_aged(now); } + // The huge cache now: committed bytes, committed entries, decommitted entries. + test_huge_cache_state :: () -> committed: u64, committed_entries: s64, decommitted_entries: s64 { + #if HUGE_CACHE_SLOT_COUNT { + committed_entries, decommitted_entries: s64; + huge_cache_lock_acquire(); + for global_huge_cache.slot { + if !it.span continue; + if it.decommitted decommitted_entries += 1; + else committed_entries += 1; + } + committed := global_huge_cache.committed; + huge_cache_lock_release(); + return committed, committed_entries, decommitted_entries; + } else { + return 0, 0, 0; + } + } + test_heap_stat_add :: (field: *u64, value: u64) { heap_stat_add(field, value); } + test_heap_stat_sub :: (field: *u64, value: u64) { heap_stat_sub(field, value); } + + // Ownership: would freeing this block from the calling thread take the local (unsynchronised) path? + test_block_is_thread_local :: (block: *void) -> bool { + span := block_get_span(block); + return page_is_thread_heap(span_get_page_from_block(span, block)); + } + + // The thread-free race of PR #426: a callback between "no cross-thread frees" and "mark full", + // and the resulting page state. + test_set_before_mark_full :: (callback: (page: *void)) { test_before_mark_full = callback; } + test_page_full_state :: (block: *void) -> is_full: bool, pending_cross_thread_frees: u32 { + span := block_get_span(block); + page := span_get_page_from_block(span, block); + unused: *Block; + return page_has(page, .IS_FULL), page_block_from_thread_free_list(page, atomic_load(*page.thread_free), *unused); + } + + // Heap layout (#409): where the fields other threads access end, where the owner's begin. + test_heap_layout :: () -> remote_end: s64, first_owner_field: s64, heap_size: s64 { + remote_end := 0; + first_owner := size_of(Heap); + for type_info(Heap).members { + end := it.offset_in_bytes + it.type.runtime_size; + if it.name == "owner_thread" || it.name == "thread_free" || it.name == "free_lock" { // free_lock: #jai fix E6 + if end > remote_end remote_end = end; + } else if it.name != "remote_pad" { + if it.offset_in_bytes < first_owner first_owner = it.offset_in_bytes; + } + } + return remote_end, first_owner, size_of(Heap); + } + + test_page_reinit_count :: () -> s64 { return atomic_load(*test_page_reinits); } + + test_is_initialized :: () -> bool { return atomic_load(*global_rpmalloc_init_state) == RPMALLOC_INIT_DONE; } + test_hold_heap_lock :: () { heap_lock_acquire(); } // Never released: the U22 scenario. + + // Decommit activity. + test_pages_decommitted_of_type :: (page_type: u32) -> s64 { return atomic_load(*test_pages_decommitted[page_type]); } + test_pages_recommitted_of_type :: (page_type: u32) -> s64 { return atomic_load(*test_pages_recommitted[page_type]); } +} + +// IS_THIS_YOURS (Jai only; no upstream equivalent). +// +// Every span is mapped at a SPAN_SIZE (256 MiB) aligned address, huge blocks included, so one bit +// per possible span base records which bases are live mappings of this allocator: 2^20 bits +// (128 KiB) cover a 48-bit address space. is_this_yours reads a span header only when its bit is +// set, so it never dereferences memory that is not ours (review §5.7). Walking the heaps' span lists +// instead would race with their owners, miss the huge blocks of thread heaps (which no list +// tracks), and follow the stale huge entries of first-class heaps (U3). + +SPAN_BITMAP_BITS :: 1 << (48 - SPAN_SIZE_SHIFT); +SPAN_BITMAP_BYTES :: SPAN_BITMAP_BITS / 8; + +global_span_bitmap: *u64; // Mapped once at the first initialize, kept for the life of the process. + +span_bitmap_map :: () { + if global_span_bitmap return; + offset, mapped_size: u64; + bitmap := os_mmap(SPAN_BITMAP_BYTES, 0, *offset, *mapped_size); + if bitmap && os_mcommit(bitmap, SPAN_BITMAP_BYTES) != 0 bitmap = null; // Without it, IS_THIS_YOURS says no. + global_span_bitmap = bitmap; + #if ENABLE_STATISTICS { + // Bookkeeping, not allocator memory: take it back out of the counters os_mmap and + // os_mcommit just raised (the peaks keep it). + page_count := mapped_size / global_config.page_size; + statistics_sub_saturating(*global_counters.page_mapped, page_count); + if bitmap { + statistics_sub_saturating(*global_counters.page_commit, SPAN_BITMAP_BYTES / global_config.page_size); + statistics_sub_saturating(*global_counters.page_active, SPAN_BITMAP_BYTES / global_config.page_size); + } + } +} + +span_bitmap_slot :: inline (span: *Span) -> word: *u64, bit: u64 { + index := (cast(u64) span) >> SPAN_SIZE_SHIFT; + if !global_span_bitmap || index >= SPAN_BITMAP_BITS return null, 0; // Not tracked: IS_THIS_YOURS says no. + return global_span_bitmap + (index >> 6), (cast(u64) 1) << (index & 63); +} + +span_register :: (span: *Span) { + word, bit := span_bitmap_slot(span); + if word atomic_or(word, bit); +} + +span_unmap :: (span: *Span) { + word, bit := span_bitmap_slot(span); + if word atomic_and(word, ~bit); + global_memory_interface.memory_unmap(span, span.offset, span.mapped_size); +} + +// With a heap: is ptr inside a page of that heap? Without (the default allocator): is it inside a +// page of any thread heap? Blocks of first-class heaps are not the default allocator's. A pointer +// into a free part of an owned page counts as owned: this is about address ranges, not live blocks. +// Asking about a huge block while another thread frees it is a use after free by the caller. +is_this_yours :: (heap: *Heap, ptr: *void) -> bool { + if !ptr || atomic_load(*global_rpmalloc_init_state) != RPMALLOC_INIT_DONE return false; + span := cast(*Span)(cast(u64) ptr & SPAN_MASK); + word, bit := span_bitmap_slot(span); + if !word || !(atomic_load(word) & bit) return false; + + address := cast(u64) ptr; + page: *Page; + if span.page_type == .HUGE { + if address < cast(u64) span + SPAN_HEADER_SIZE || address >= cast(u64) span + cast(u64) span.page_count * span.page_size + return false; + page = *span.page; + } else { + if address >= cast(u64) span + cast(u64) atomic_load(*span.page_initialized) * span.page_size return false; + page = span_get_page_from_block(span, ptr); + if address < cast(u64) page + page_block_start_offset(page) return false; // #jai PR #424 + } + owner := page.heap; + if !owner return false; // A cached huge mapping. + if heap return owner == heap; + return !owner.is_first_class; +} + +// +// Extern interface (2.0.1: rpmalloc.c:2510-2660). +// +// #jai refused arguments return null (or the documented error code) without setting errno: Jai +// code has no errno convention, and on Windows it would mean linking the C runtime. +// + + +// Sizes are s64 at the Jai API (the Allocator protocol's type) and size_t (u64) inside, like C. + +rpmalloc :: (size: s64) -> *void { // 2.0.1: rpmalloc.c:2522 + #if ENABLE_VALIDATE_ARGS { + if cast,no_check(u64) size >= MAX_ALLOC_SIZE return null; + } + heap := get_thread_heap(); + return heap_allocate_block(heap, cast,no_check(u64) size, false); +} + +rpzalloc :: (size: s64) -> *void { // 2.0.1: rpmalloc.c:2534 + #if ENABLE_VALIDATE_ARGS { + if cast,no_check(u64) size >= MAX_ALLOC_SIZE return null; + } + heap := get_thread_heap(); + return heap_allocate_block(heap, cast,no_check(u64) size, true); +} + +rpfree :: (ptr: *void) { // 2.0.1: rpmalloc.c:2546 + if !ptr return; + block_deallocate(ptr); +} +rpcalloc :: (num: s64, size: s64) -> *void { // 2.0.1: rpmalloc.c:2553 + total, ok := calloc_total(num, size); + if !ok return null; + heap := get_thread_heap(); + return heap_allocate_block(heap, total, true); +} + +rprealloc :: (ptr: *void, size: s64) -> *void { // 2.0.1: rpmalloc.c:2577 + #if ENABLE_VALIDATE_ARGS { + // UPSTREAM-BUG L6: returns the old pointer for a refused size, which callers read as success. + if cast,no_check(u64) size >= MAX_ALLOC_SIZE return ptr; + } + if realloc_fits_in_place(ptr, cast,no_check(u64) size) return ptr; // #jai PR #423 + heap := get_thread_heap(); + return heap_reallocate_block(heap, ptr, cast,no_check(u64) size, 0, 0); +} + +rpaligned_realloc :: (ptr: *void, alignment: s64, size: s64, old_size: s64, flags: Realloc_Flags = 0) -> *void { // 2.0.1: rpmalloc.c:2589 + #if ENABLE_VALIDATE_ARGS { + // Upstream refuses alignments above SMALL_PAGE_SIZE here, while rpaligned_alloc accepts + // anything below RPMALLOC_MAX_ALIGNMENT (256 KiB). + // #jai upstream's size + alignment < size, without the wrapping sum (arithmetic_overflow_check). + if cast,no_check(u64) size > 0xFFFF_FFFF_FFFF_FFFF - cast,no_check(u64) alignment || cast,no_check(u64) alignment > SMALL_PAGE_SIZE + return null; + } + heap := get_thread_heap(); + return heap_reallocate_block_aligned(heap, ptr, cast,no_check(u64) alignment, cast,no_check(u64) size, cast,no_check(u64) old_size, flags); +} + +rpaligned_alloc :: (alignment: s64, size: s64) -> *void { // 2.0.1: rpmalloc.c:2601 + heap := get_thread_heap(); + return heap_allocate_block_aligned(heap, cast,no_check(u64) alignment, cast,no_check(u64) size, false); +} + +rpaligned_zalloc :: (alignment: s64, size: s64) -> *void { // 2.0.1: rpmalloc.c:2607 + heap := get_thread_heap(); + return heap_allocate_block_aligned(heap, cast,no_check(u64) alignment, cast,no_check(u64) size, true); +} + +rpaligned_calloc :: (alignment: s64, num: s64, size: s64) -> *void { // 2.0.1: rpmalloc.c:2613 + total, ok := calloc_total(num, size); + if !ok return null; + heap := get_thread_heap(); + return heap_allocate_block_aligned(heap, cast,no_check(u64) alignment, total, true); +} + +rpmemalign :: (alignment: s64, size: s64) -> *void { // 2.0.1: rpmalloc.c:2637 + heap := get_thread_heap(); + return heap_allocate_block_aligned(heap, cast,no_check(u64) alignment, cast,no_check(u64) size, false); +} + +// Returns 0, EINVAL (memptr is null) or ENOMEM, like posix_memalign. +rpposix_memalign :: (memptr: **void, alignment: s64, size: s64) -> s32 { // 2.0.1: rpmalloc.c:2643 + EINVAL :: 22; + ENOMEM :: 12; + heap := get_thread_heap(); + if !memptr return EINVAL; + memptr.* = heap_allocate_block_aligned(heap, cast,no_check(u64) alignment, cast,no_check(u64) size, false); + return ifx memptr.* then 0 else ENOMEM; +} + +usable_size :: (ptr: *void) -> s64 { // 2.0.1: rpmalloc.c:2653 + return ifx ptr then cast(s64) block_usable_size(ptr) else 0; +} + +// +// Initialization and finalization (2.0.1: rpmalloc.c:2661-3421). +// + +// #jai fix U10: upstream's initialize_config wrote global_config before initialize claimed the init +// state, so it could race a concurrent lazy initialisation. The config is now applied only by the +// thread that claims the state; if the allocator is already initialised it is not applied, as +// upstream. +initialize_config :: (memory_interface: *Memory_Interface, config: *Config) -> s32 { // 2.0.1: rpmalloc.c:2704 + result := initialize_with_config(memory_interface, config); + if config config.* = global_config; + return result; +} + +initialize :: (memory_interface: *Memory_Interface = null) -> s32 { // 2.0.1: rpmalloc.c:2724 + return initialize_with_config(memory_interface, null); +} + +#scope_module + +initialize_with_config :: (memory_interface: *Memory_Interface, config: *Config) -> s32 { + while true { + state := atomic_load(*global_rpmalloc_init_state); + if state == RPMALLOC_INIT_DONE { + thread_initialize(); + return 0; + } + if state == RPMALLOC_INIT_RUNNING { + while atomic_load(*global_rpmalloc_init_state) == RPMALLOC_INIT_RUNNING thread_yield(); + continue; + } + if compare_and_swap(*global_rpmalloc_init_state, RPMALLOC_INIT_UNINIT, RPMALLOC_INIT_RUNNING) break; + } + if config global_config = config.*; // #jai fix U10: only after claiming the init state. + + // Remember whether the caller asked for huge pages; detection below overwrites + // global_config.enable_huge_pages with what is actually available. + huge_pages_requested := global_config.enable_huge_pages; + + // #jai fix U8: upstream installed the OS defaults only when map or unmap was missing, so an + // interface with map and unmap but no commit / decommit was called through null; and it wrote + // the defaults into the caller's struct. Now the caller's interface is copied, and a null commit + // or decommit falls back to the OS. Without map or unmap, all four are the OS's, as upstream. + if memory_interface { + global_memory_interface_copy = memory_interface.*; + global_memory_interface = *global_memory_interface_copy; + if !global_memory_interface.memory_commit global_memory_interface.memory_commit = os_mcommit; + if !global_memory_interface.memory_decommit global_memory_interface.memory_decommit = os_mdecommit; + } else { + global_memory_interface = *global_memory_interface_default; + } + if !global_memory_interface.memory_map || !global_memory_interface.memory_unmap { + global_memory_interface.memory_map = os_mmap; + global_memory_interface.memory_commit = os_mcommit; + global_memory_interface.memory_decommit = os_mdecommit; + global_memory_interface.memory_unmap = os_munmap; + } + // (ENABLE_OVERRIDE's unmap_on_finalize override is C-only; not ported.) + + // 2.0.1: rpmalloc.c:2765-2785. #jai os_initialize also serves the test hooks, so force a fresh + // read here, as upstream does on every initialisation. + os_page_size = 0; + os_initialize(); + os_huge_pages = false; + os_thp = false; + os_huge_page_shift = 0; + + rpmalloc_assert(!global_config.page_size || !(global_config.page_size & (global_config.page_size - 1)), "Configured page size must be a power of two"); // #jai 0 is valid (the OS page size), and 0 - 1 would wrap + + // Establish a valid (normal) page size up front, so the allocator never sees a zero page size + // during the rest of initialisation. Huge page detection may raise it. + if !memory_interface || global_config.page_size < os_page_size global_config.page_size = os_page_size; + + if global_config.enable_huge_pages os_detect_huge_pages(); // 2.0.1: rpmalloc.c:2797-2893 + + if huge_pages_requested && !os_huge_pages { + // Huge pages were requested but are not available: fail rather than silently use normal + // pages behind a huge page size. Leave the allocator uninitialised (and the flag cleared) + // so the caller can retry without huge pages. + global_config.enable_huge_pages = false; + atomic_store(*global_rpmalloc_init_state, RPMALLOC_INIT_UNINIT); + return -1; + } + + global_config.enable_huge_pages = os_huge_pages; + + if os_huge_pages { + while (cast(u64) 1 << os_huge_page_shift) < os_page_size os_huge_page_shift += 1; + } + + if !memory_interface || global_config.page_size < os_page_size global_config.page_size = os_page_size; + + #if OS == .WINDOWS { + // Large pages on Windows must be committed when mapped and can never be decommitted. + if global_config.enable_huge_pages global_config.disable_decommit = true; + } + + os_configure_thp(); // 2.0.1: rpmalloc.c:2921-2936 + global_config.enable_thp = os_thp; + + // Decommit of free pages only applies to page types larger than the configured page size: + // disable the free page overflow pass for page types where it can never apply (always the case + // with huge pages or a large page size). + page_free_overflow_default :: u32.[16, 8, 4, 2]; + builtin_page_size :: u64.[SMALL_PAGE_SIZE, MEDIUM_SMALL_PAGE_SIZE, MEDIUM_LARGE_PAGE_SIZE, LARGE_PAGE_SIZE]; + for itype: 0..3 { + page_can_decommit := !global_config.disable_decommit && builtin_page_size[itype] > global_config.page_size; + global_page_free_overflow[itype] = ifx page_can_decommit then page_free_overflow_default[itype] else 0xFFFF_FFFF; + } + + span_bitmap_map(); // #jai for IS_THIS_YOURS; needs the final page size (statistics). + + os_tls_create(rpmalloc_thread_destructor); // 2.0.1: rpmalloc.c:2951-2955 + + global_main_thread_id = get_thread_id(); + + atomic_store(*global_rpmalloc_init_state, RPMALLOC_INIT_DONE); + + thread_initialize(); + return 0; +} + +#scope_export + +config :: () -> *Config { // 2.0.1: rpmalloc.c:2967 + return *global_config; +} + +finalize :: () { // 2.0.1: rpmalloc.c:3031 + thread_finalize(); + + #if HUGE_CACHE_SLOT_COUNT huge_cache_flush(); // 2.0.1: rpmalloc.c:3037 + #if ENABLE_LEAK_DETECTION && ENABLE_ASSERTS { // 2.0.1: rpmalloc.c:3040-3047 + // Every thread has finalized, so blocks still allocated and not merely waiting in a deferred + // cross-thread free list are blocks the application never freed (or freed twice). + rpmalloc_assert(statistics_outstanding_blocks() <= statistics_pending_deferred_blocks(), "Memory leak detected"); + rpmalloc_assert(atomic_load(*global_counters.huge_alloc) == 0, "Memory leak detected"); + } + + if global_config.unmap_on_finalize { + // Heaps can share a mapping (heap_allocate_new); only the owner has mapped_size != 0. + // Resources are freed while every heap is still mapped, then owners are unmapped in a + // separate pass, so no heap is touched after its shared mapping is gone. + owners: *Heap; + collect :: (list: **Heap) #expand { + heap := list.*; + list.* = null; + while heap { + heap_next := heap.next; + heap_unmap_all(heap); + if heap.mapped_size { + heap.next = `owners; + `owners = heap; + } + heap = heap_next; + } + } + collect(*global_heap_pristine); + collect(*global_heap_queue); + collect(*global_heap_used); + while owners { + owner_next := owners.next; + heap_unmap(owners); + owners = owner_next; + } + #if ENABLE_STATISTICS memset(*global_counters, 0, size_of(Rpmalloc_Statistics)); // 2.0.1: rpmalloc.c:3094 + } + + os_tls_destroy(); // 2.0.1: rpmalloc.c:3098-3104 + + // #jai every span is unmapped or abandoned now: forget them all (IS_THIS_YOURS). + if global_span_bitmap memset(global_span_bitmap, 0, SPAN_BITMAP_BYTES); + + global_heap_creating = 0; + global_main_thread_id = 0; + atomic_store(*global_rpmalloc_init_state, RPMALLOC_INIT_UNINIT); +} + +thread_initialize :: () { // 2.0.1: rpmalloc.c:3112 + if get_thread_heap() == global_heap_default get_thread_heap_allocate(); +} + +thread_finalize :: () { // 2.0.1: rpmalloc.c:3118 + heap := get_thread_heap(); + if heap != global_heap_default { + heap_release_on_thread_exit(heap); // #jai fix U5 + set_thread_heap(global_heap_default); + } else { + heap_release_parked_on_exit(); // #jai fix E7 + } +} + +thread_collect :: () { // 2.0.1: rpmalloc.c:3127 (empty in 2.0.1; develop #416 fills it in) + heap := get_thread_heap(); + if !heap || !heap.id return; + // #jai fix U23: idle huge cache entries were unmapped only when another huge block was freed, so a + // program that stopped using huge blocks kept up to 128 MiB committed for good (RET-08). + #if HUGE_CACHE_SLOT_COUNT huge_cache_purge_aged(os_monotonic_time_ms()); + heap_drain_deferred_frees(heap); // #jai fix U6 + heap_retire_kept_pages(heap); // #jai fix (U6 part, with #413) + // #jai develop #416: decommit the free pages held above the retain count now, instead of + // waiting for the decommit delay. + for itype: 0..3 { + if global_page_free_overflow[itype] == 0xFFFF_FFFF continue; + heap.page_free_overflow_ms[itype] = 0; + if heap.page_free_commit_count[itype] > global_page_free_retain[itype] + heap_page_free_decommit(heap, cast(u32) itype, global_page_free_retain[itype]); + } +} + +// #jai (upstream issue #116): returns as much memory as possible to the OS now. thread_collect keeps +// the usual retain counts per page type and the young huge cache entries; trim also decommits every +// free page of the calling thread's heap and unmaps the whole huge cache. Pages that hold live blocks +// stay, and other threads' heaps are not touched (each thread can call trim itself). +trim :: () { + heap := get_thread_heap(); + if heap && heap.id { + heap_drain_deferred_frees(heap); + heap_retire_kept_pages(heap); + for itype: 0..3 { + heap.page_free_overflow_ms[itype] = 0; + if heap.page_free_commit_count[itype] heap_page_free_decommit(heap, cast(u32) itype, 0); + } + } + #if HUGE_CACHE_SLOT_COUNT huge_cache_flush(); +} + +// #jai fix E7 (Ex4 1.45.0, items 1 and 3 of its analysis): frees by other threads into a heap queue up +// for its owner, which takes them in only on its next allocation, and a heap keeps some free pages +// committed for reuse. A pool thread that goes idle makes no allocator call, possibly for a long time, +// so its heap kept the pages of every block other threads freed into it, and its free pages: 185 MB +// and 35 MB in Ex4, with 24 idle walk workers (RET-20). Only the thread knows it is going idle, so it +// says so: thread_idle gives back what it can, as trim() does for its heap (pending frees taken in, +// kept empty pages retired, every free page decommitted, aged huge cache entries decommitted), and +// parks the heap. While parked, frees by other threads into it are freed at once under the heap's +// lock, as into an exited thread's heap (E4, E6), so their pages are decommitted as they empty. The +// thread's next allocator call takes the heap back. Call it where a pool thread waits for work; it +// costs a few OS calls, and the first allocation after it takes the slow path once. +thread_idle :: () { + heap := get_thread_heap(); + if !heap || !heap.id || heap.is_first_class return; // No thread heap (or a first-class one made current). + heap_drain_deferred_frees(heap); + heap_retire_kept_pages(heap); + for itype: 0..3 { + heap.page_free_overflow_ms[itype] = 0; + if heap.page_free_commit_count[itype] heap_page_free_decommit(heap, cast(u32) itype, 0); + } + #if HUGE_CACHE_SLOT_COUNT huge_cache_purge_aged(os_monotonic_time_ms()); + me := get_thread_id(); + heap_lock_acquire(); + heap_free_lock_acquire(heap); + heap.parked_thread = me; + heap.owner_thread = HEAP_RELEASED_OWNER; // Frees by any thread take the released-heap path. + heap_free_lock_release(heap); + atomic_add(*global_parked_count, 1); + heap_lock_release(); + // The fallback heap: the fast paths find nothing, and the next call unparks (heap_take_parked). + // Non-null, so the POSIX thread exit destructor still runs and releases the parked heap. + heap.thread_current = false; // #jai fix E10: no Context cache may hand it out while parked. + os_tls_set(global_heap_default); +} + +#scope_module + +// A thread that exits (or sends THREAD_STOP) with its heap parked: the heap goes to the queue for +// reuse. It was trimmed when it was parked, and frees since were applied at once. +heap_release_parked_on_exit :: () { + heap := heap_take_parked(); + if !heap return; + heap_retire_kept_pages(heap); + for itype: 0..3 { + heap.page_free_overflow_ms[itype] = 0; + if heap.page_free_commit_count[itype] heap_page_free_decommit(heap, cast(u32) itype, 0); + } + heap_release_to_queue(heap); +} + +#scope_export + +is_thread_initialized :: () -> bool { // 2.0.1: rpmalloc.c:2517 + return get_thread_heap() != global_heap_default; +} + +thread_statistics :: (stats: *Thread_Statistics) { // 2.0.1: rpmalloc.c:3131 + memset(stats, 0, size_of(Thread_Statistics)); + #if ENABLE_STATISTICS { + heap := get_thread_heap(); + if !heap || heap.id == 0 return; + + // Size class cache: blocks available without mapping a new page. A page's block_used counts + // the blocks held on the heap free list, so those are counted from the heap list, and each + // page adds its remaining blocks (its free list plus the not yet initialised tail). + for iclass: 0..SIZE_CLASS_COUNT - 1 { + block_size := cast(u64) global_size_class[iclass].block_size; + free_count: u64; + block := heap.local_free[iclass]; + while block { + free_count += 1; + block = block.next; + } + page := heap.page_available[iclass]; + while page { + free_count += cast(u64)(page.block_count - page.block_used); + page = page.next; + } + stats.sizecache += free_count * block_size; + + su := *heap.size_use[iclass]; + // #jai fix U13: a transiently negative count (frees counted before the matching + // allocation is adopted) reads as 0, not as a huge size_t. + stats.size_use[iclass].alloc_current = cast(u64) max(su.alloc_current, 0); + stats.size_use[iclass].alloc_peak = cast(u64) max(su.alloc_peak, 0); + stats.size_use[iclass].alloc_total = cast(u64) max(su.alloc_total, 0); + stats.size_use[iclass].free_total = cast(u64) max(su.free_total, 0); + } + + // Span cache: free but still committed pages kept per page type. + page_type_size :: u64.[SMALL_PAGE_SIZE, MEDIUM_SMALL_PAGE_SIZE, MEDIUM_LARGE_PAGE_SIZE, LARGE_PAGE_SIZE]; + for itype: 0..3 stats.spancache += cast(u64) heap.page_free_commit_count[itype] * page_type_size[itype]; + + // Span use per page type: current spans counted live (partial plus full); map calls counted. + for itype: 0..4 { + current: u64; + if itype < 4 && heap.span_partial[itype] current += 1; + span := heap.span_used[itype]; + while span { + current += 1; + span = span.next; + } + stats.span_use[itype].current = current; + stats.span_use[itype].map_calls = heap.span_map_calls[itype]; + } + } +} + +// #jai appends to a String_Builder instead of writing to a FILE*. +dump_statistics :: (builder: *String_Builder) { // 2.0.1: rpmalloc.c:3176 + #if ENABLE_STATISTICS { + g := *global_counters; + print_to_builder(builder, "Mapped pages: %\n", atomic_load(*g.page_mapped)); + print_to_builder(builder, "Mapped pages (peak): %\n", atomic_load(*g.page_mapped_peak)); + print_to_builder(builder, "Active pages: %\n", atomic_load(*g.page_active)); + print_to_builder(builder, "Active pages (peak): %\n", atomic_load(*g.page_active_peak)); + print_to_builder(builder, "Pages committed: %\n", atomic_load(*g.page_commit)); + print_to_builder(builder, "Pages decommitted: %\n", atomic_load(*g.page_decommit)); + print_to_builder(builder, "Huge bytes: %\n", atomic_load(*g.huge_alloc)); + print_to_builder(builder, "Huge bytes (peak): %\n", atomic_load(*g.huge_alloc_peak)); + print_to_builder(builder, "Heaps in use: %\n", atomic_load(*g.heap_count)); + #if ENABLE_LEAK_DETECTION + print_to_builder(builder, "Outstanding blocks: %\n", statistics_outstanding_blocks() - statistics_pending_deferred_blocks()); + + // Aggregate the per size class and per page type counters across all in-use heaps. This + // walks live heaps without stopping the world, so the values are a best-effort snapshot. + size_use: [SIZE_CLASS_COUNT] Heap_Size_Use; + span_map_calls: [5] u64; + heap_lock_acquire(); + heap := global_heap_used; + while heap { + for iclass: 0..SIZE_CLASS_COUNT - 1 { + size_use[iclass].alloc_current += heap.size_use[iclass].alloc_current; + size_use[iclass].alloc_peak += heap.size_use[iclass].alloc_peak; + size_use[iclass].alloc_total += heap.size_use[iclass].alloc_total; + size_use[iclass].free_total += heap.size_use[iclass].free_total; + } + for itype: 0..4 span_map_calls[itype] += heap.span_map_calls[itype]; + heap = heap.next; + } + heap_lock_release(); + + print_to_builder(builder, "SizeClass CurAlloc PeakAlloc TotAlloc TotFree BlockSize\n"); + for size_use { + if !it.alloc_total continue; + print_to_builder(builder, "% % % % % %\n", formatInt(it_index, minimum_digits = 9, padding = #char " "), + formatInt(it.alloc_current, minimum_digits = 9, padding = #char " "), formatInt(it.alloc_peak, minimum_digits = 9, padding = #char " "), + formatInt(it.alloc_total, minimum_digits = 9, padding = #char " "), formatInt(it.free_total, minimum_digits = 9, padding = #char " "), + formatInt(global_size_class[it_index].block_size, minimum_digits = 9, padding = #char " ")); + } + print_to_builder(builder, "PageType MapCalls\n"); + for span_map_calls { + if !it continue; + print_to_builder(builder, "% %\n", formatInt(it_index, minimum_digits = 8, padding = #char " "), formatInt(it, minimum_digits = 9, padding = #char " ")); + } + } +} + +global_statistics :: (stats: *Global_Statistics) { // 2.0.1: rpmalloc.c:3240 + #if ENABLE_STATISTICS { + g := *global_counters; + page_size := global_config.page_size; + stats.mapped = page_size * atomic_load(*g.page_mapped); + stats.mapped_peak = page_size * atomic_load(*g.page_mapped_peak); + stats.committed = page_size * atomic_load(*g.page_commit); + stats.decommitted = page_size * atomic_load(*g.page_decommit); + stats.active = page_size * atomic_load(*g.page_active); + stats.active_peak = page_size * atomic_load(*g.page_active_peak); + stats.huge_alloc = atomic_load(*g.huge_alloc); + stats.huge_alloc_peak = atomic_load(*g.huge_alloc_peak); + stats.heap_count = atomic_load(*g.heap_count); + } else { + memset(stats, 0, size_of(Global_Statistics)); + } +} + +#if FIRST_CLASS_HEAPS { + +// First-class heaps (2.0.1: rpmalloc.c:3255-3418). A first-class heap has no owner thread +// (owner_thread 0): only one thread may use it at a time, and heap_free_all releases every block +// it ever handed out. + +heap_acquire :: () -> *Heap { // 2.0.1: rpmalloc.c:3259 + // Must be a pristine heap, or else blocks could already be allocated from it, which + // heap_free_all would (wrongly) release. heap_allocate(true) returns a never-used heap carved + // from a shared block, or a freshly mapped one (including heaps released empty, #jai fix U20). + heap := heap_allocate(true); + // #jai fix U9: upstream's rpmalloc_assume(heap != 0) turned out-of-memory into a null + // dereference. Return null, as CREATE_HEAP's protocol expects. + if !heap return null; + heap.owner_thread = 0; + return heap; +} + +heap_release :: (heap: *Heap) { // 2.0.1: rpmalloc.c:3271 + // #jai fix U14: see heap_release_to_queue. The released flag is checked again under the lock. + if heap && heap != global_heap_default && !heap.released { // #jai fix U2: never the fallback heap. + if heap_is_empty(heap) heap_release_to_pristine(heap); // #jai fix U20 + else heap_release_trim(heap); // #jai develop #418, #419 + } +} + +heap_alloc :: (heap: *Heap, size: s64) -> *void { // 2.0.1: rpmalloc.c:3277 + #if ENABLE_VALIDATE_ARGS { + if cast,no_check(u64) size >= MAX_ALLOC_SIZE return null; + } + return heap_allocate_block(heap, cast,no_check(u64) size, false); +} + +heap_aligned_alloc :: (heap: *Heap, alignment: s64, size: s64) -> *void { // 2.0.1: rpmalloc.c:3288 + #if ENABLE_VALIDATE_ARGS { + if cast,no_check(u64) size >= MAX_ALLOC_SIZE return null; + } + return heap_allocate_block_aligned(heap, cast,no_check(u64) alignment, cast,no_check(u64) size, false); +} + +heap_aligned_zalloc :: (heap: *Heap, alignment: s64, size: s64) -> *void { // 2.0.1: rpmalloc.c:3299 + #if ENABLE_VALIDATE_ARGS { + if cast,no_check(u64) size >= MAX_ALLOC_SIZE return null; + } + return heap_allocate_block_aligned(heap, cast,no_check(u64) alignment, cast,no_check(u64) size, true); +} + +heap_calloc :: (heap: *Heap, num: s64, size: s64) -> *void { // 2.0.1: rpmalloc.c:3310 + total, ok := calloc_total(num, size); + if !ok return null; + return heap_allocate_block(heap, total, true); +} + +heap_aligned_calloc :: (heap: *Heap, alignment: s64, num: s64, size: s64) -> *void { // 2.0.1: rpmalloc.c:3333 + total, ok := calloc_total(num, size); + if !ok return null; + return heap_allocate_block_aligned(heap, cast,no_check(u64) alignment, total, true); +} + +heap_realloc :: (heap: *Heap, ptr: *void, size: s64, flags: Realloc_Flags = 0) -> *void { // 2.0.1: rpmalloc.c:3356 + #if ENABLE_VALIDATE_ARGS { + // UPSTREAM-BUG L6: returns the old pointer for a refused size, which callers read as success. + if cast,no_check(u64) size >= MAX_ALLOC_SIZE return ptr; + } + return heap_reallocate_block(heap, ptr, cast,no_check(u64) size, 0, flags); +} + +heap_aligned_realloc :: (heap: *Heap, ptr: *void, alignment: s64, size: s64, flags: Realloc_Flags = 0) -> *void { // 2.0.1: rpmalloc.c:3367 + #if ENABLE_VALIDATE_ARGS { + // #jai upstream's size + alignment < size, without the wrapping sum (arithmetic_overflow_check). + if cast,no_check(u64) size > 0xFFFF_FFFF_FFFF_FFFF - cast,no_check(u64) alignment || cast,no_check(u64) alignment > SMALL_PAGE_SIZE + return null; + } + return heap_reallocate_block_aligned(heap, ptr, cast,no_check(u64) alignment, cast,no_check(u64) size, 0, flags); +} + +heap_free :: (heap: *Heap, ptr: *void) { // 2.0.1: rpmalloc.c:3378 + // The heap argument is unused: the block's page knows its heap. + // #jai fix U18: rpfree checks for null, upstream's heap_free did not and dereferenced 0. + if !ptr return; + // #jai PR #422 compares the page with the given heap, which upstream trusts. A non-owner thread + // passing the owner's thread heap (heap_free(get_heap_for_ptr(p), p) reads naturally) would then + // take the owner's unsynchronised local path (HEAP-09). Only a first-class heap, which has no + // owner thread, or the caller's own thread heap is trusted; anything else frees as rpfree does. + owned := get_thread_heap(); + if heap && !heap.owner_thread owned = heap; + block_deallocate_heap(ptr, owned); +} + +// Frees every block the heap has handed out. +heap_free_all :: (heap: *Heap) { // 2.0.1: rpmalloc.c:3385 + heap_unmap_all(heap); +} + +heap_statistics :: (heap: *Heap) -> Heap_Statistics { // 2.0.1: rpmalloc.c:3390 + #if HEAP_STATISTICS { + if heap return heap.stats; + } + return .{}; +} + +heap_thread_set_current :: (heap: *Heap) { // 2.0.1: rpmalloc.c:3402 + prev_heap := get_thread_heap(); + if prev_heap != heap { + // set_thread_heap stamps owner_thread on a first-class heap, which makes it a thread heap + // that the thread's exit releases; a later heap_release of it is ignored (#jai fix U14). + set_thread_heap(heap); + // #jai fix U2: a thread that has not allocated yet still has the static fallback heap as its + // current heap. Upstream queued it for reuse like a real heap, and later new threads were + // handed it (or, with the queue now inconsistent, a heap another thread owns). + if prev_heap && prev_heap != global_heap_default heap_release_to_queue(prev_heap); + } +} + +// The heap that owns the page of `ptr`. Dereferences the span header of any pointer it is given, +// so it is only for pointers known to come from this allocator (IS_THIS_YOURS does not use it). +get_heap_for_ptr :: (ptr: *void) -> *Heap { // 2.0.1: rpmalloc.c:3412 + span := cast(*Span)(cast(u64) ptr & SPAN_MASK); + if span return span_get_page_from_block(span, ptr).heap; + return null; +} + +} // #if FIRST_CLASS_HEAPS + +#scope_module + +#import "Basic"; +#import "Bit_Operations"; +#import "Atomics"; // atomic_add returns the previous value, like C atomic_fetch_add. diff --git a/modules/Default_Allocator/version.jai b/modules/Default_Allocator/version.jai new file mode 100644 index 0000000..cce2414 --- /dev/null +++ b/modules/Default_Allocator/version.jai @@ -0,0 +1,48 @@ +// The version of this port, the same for both flavours (rpmalloc2 and Default_Allocator). +// +// VERSION is upstream's version followed by this port's revision. Raise VERSION_REVISION with every +// change to a file that install.jai installs: an installed copy whose revision is lower is behind, +// and install.jai updates it. Raise it here, in VERSION, and in CAPS_VERSION_STRING in module.jai and +// Default_Allocator/module.jai; compile-time asserts fail the build until all four agree. To see what +// an installed copy is, read this file in it, or the CAPS name its allocator reports +// ("rpmalloc 2.0.1-jai.N (Jai port)"). + +VERSION :: "2.0.1-jai.16"; +VERSION_REVISION :: 16; +RPMALLOC_VERSION :: "2.0.1"; // The upstream rpmalloc release this is a port of. + +#assert #run version_is(VERSION, RPMALLOC_VERSION, VERSION_REVISION) + "VERSION must be RPMALLOC_VERSION \"-jai.\" VERSION_REVISION"; + +#scope_module + +// For the compile-time version asserts, here and in both module.jai files. #jai not tprint: they run +// as the target's code, and the x64 tprint uses SIMD assembly that a compiler on another CPU cannot +// execute (a Linux x64 build on Apple silicon failed). Plain loops run anywhere, and nothing allocates. + +// Is `version` exactly upstream "-jai." revision? +version_is :: (version: string, upstream: string, revision: int) -> bool { + PREFIX :: "-jai."; + if !text_starts(version, 0, upstream) || !text_starts(version, upstream.count, PREFIX) return false; + at := upstream.count + PREFIX.count; + if at >= version.count return false; + n := 0; + for i: at..version.count-1 { + c := version[i]; + if c < #char "0" || c > #char "9" return false; + n = n * 10 + (c - #char "0"); + } + return n == revision; +} + +// Is `text` exactly before, middle, after joined? +text_is :: (text: string, before: string, middle: string, after: string) -> bool { + if text.count != before.count + middle.count + after.count return false; + return text_starts(text, 0, before) && text_starts(text, before.count, middle) && text_starts(text, before.count + middle.count, after); +} + +text_starts :: (text: string, at: int, part: string) -> bool { + if at + part.count > text.count return false; + for 0..part.count-1 if text[at + it] != part[it] return false; + return true; +} diff --git a/src/App.jai b/src/App.jai index 49c2801..c4cb546 100644 --- a/src/App.jai +++ b/src/App.jai @@ -63,6 +63,7 @@ USING_WINDOWS :: OS == .WINDOWS; #import "Thread"; #import "Hash_Table"; #import "Hash"; // fnv1a_hash, for the label list's hash +Default_Allocator :: #import "Default_Allocator"; // rpmalloc (modules/Default_Allocator), for thread_idle at the workers' waits Shared_Channel :: #import "Shared_Memory_Channel"; #import "stb_image"; #import "stb_image_resize"; @@ -198,6 +199,10 @@ App :: struct { library_folder: string; // -library FOLDER: the .mcap folder the viewer shows, in place of VIEWER_FOLDER scan_and_exit: bool; // -scan: scan the library with the chosen OCR model once it has loaded, then exit (0, or 1 if any capture failed) stats_and_exit: bool; // -stats: log the Statistics window's numbers once the catalog has loaded, then exit + catalog_threads: s64; // -catalog-threads N: workers loading the catalog at once, in place of all of the background group's (for measuring) + cold_start: bool; + no_catalog_cache: bool; // -no-catalog-cache: read every capture from its file, as without the catalog cache (for measuring) + launch_time: Apollo_Time; // monotonic, when main() started: the [startup] log lines count from it // -cold: load the library as on a cold start, its files dropped from the system's file cache first start_search: string; // -search QUERY: the search box's text at startup erase_and_exit: string; // -erase ocr|labels|segments|all: erase that from the whole library once it has loaded, then exit record_and_exit: float64; // -record SECONDS: record the screen under the cursor that long, put it in the library, then exit (0, or 1 if it failed) diff --git a/src/app/background.jai b/src/app/background.jai index 3d90686..87f9e84 100644 --- a/src/app/background.jai +++ b/src/app/background.jai @@ -14,6 +14,24 @@ BACKGROUND_SEARCH_CHUNK :: 256; // captures matched per job +// Captures whose catalog entries are wanted, shared by the jobs loading them. Heap; freed by the UI +// thread when the last job is back. +Catalog_Batch :: struct { + items: [] Catalog_Item; + cache: string; // the catalog cache's bytes, for items with an offset; heap, freed with the batch + next: s64; // the next item a job takes (atomic) + jobs_left: s64; // UI thread +} + +// A capture whose catalog entry is wanted. path is the capture's, never freed. +Catalog_Item :: struct { + index: s32; + path: string; + size: s64 = -1; // its file's, as listed; -1 when not known (catalog_load looks) + modified: s64; // nanoseconds + offset: s64 = -1; // where its entry starts in the batch's cache; -1: read it from the file +} + Search_Item :: struct { index: s32; entry: *Catalog_Entry; // null while not loaded @@ -30,11 +48,14 @@ background: struct { stats_generation: s64; // the newest Statistics job stats_out: bool; retired: [..] *Catalog_Entry; // replaced while jobs were out + catalog_jobs_out: s64; // UI thread: catalog loads handed out; they read no entries, so jobs_out leaves them out + threads: s64; }; background_startup :: () { background.retired.allocator = context.default_allocator; threads := clamp(get_number_of_processors(.ALL_LOGICAL) - 2, 2, 16); + background.threads = threads; // Statistics count words in a table in temporary storage: a few MB on the full library. init(*background.group, cast(s32) threads, background_proc, temporary_storage_size = 4 * 1024 * 1024, group_name = "background"); background.group.logging = false; @@ -71,8 +92,18 @@ background_frame :: () { if !background.started return; for work: get_completed_work(*background.group) { job := cast(*Background_Job) work; - background.jobs_out -= 1; + if job.kind == .CATALOG_LOAD || job.kind == .CATALOG_CACHE background.catalog_jobs_out -= 1; + else background.jobs_out -= 1; if job.kind == { + case .CATALOG_CACHE; catalog_cache_indexed(job.cache_index); + case .CATALOG_SAVE; catalog_cache_save_done(job.cache_save); + case .CATALOG_LOAD; + job.catalog.jobs_left -= 1; + if !job.catalog.jobs_left { + array_free(job.catalog.items,, context.default_allocator); + free(job.catalog.cache,, context.default_allocator); + free(job.catalog,, context.default_allocator); + } case .SEARCH_EMBED; request := job.request; if request.generation == background.search_generation search_dispatch_chunks(request); @@ -185,9 +216,47 @@ background_stats_start :: (items: [] Search_Item, model: string) { background_add(job); } +// The catalog entries of these captures, loaded by background_catalog_threads() jobs at once +// (app/catalog.jai, catalog_load_job). The entries come back through viewer.catalog_updates. +background_catalog_load :: (items: [] Catalog_Item, cache := "") { + if !items.count return; + batch := New(Catalog_Batch,, context.default_allocator); + batch.items = array_copy(items,, context.default_allocator); + batch.cache = cache; + jobs := min(background_catalog_threads(), items.count); + batch.jobs_left = jobs; + for 1..jobs { + job := New(Background_Job,, context.default_allocator); + job.* = .{kind = .CATALOG_LOAD, catalog = batch}; + background.catalog_jobs_out += 1; + add_work(*background.group, job); + } +} + +// The startup job reading the catalog cache (app/catalog.jai); catalog_cache_indexed takes it back. +background_catalog_cache :: (index: *Catalog_Cache_Index) { + job := New(Background_Job,, context.default_allocator); + job.* = .{kind = .CATALOG_CACHE, cache_index = index}; + background.catalog_jobs_out += 1; + add_work(*background.group, job); +} + +// A save of the catalog cache. It reads entries, so it counts in jobs_out. +background_catalog_save :: (save: *Catalog_Cache_Save) { + job := New(Background_Job,, context.default_allocator); + job.* = .{kind = .CATALOG_SAVE, cache_save = save}; + background_add(job); +} + +// Every worker of the group by default: with rpmalloc's per-thread heaps, catalog loads scale with +// threads (reading 10k files cold: 4 threads 850 ms, 8 450 ms, 16 250 ms). -catalog-threads N, for measuring. +background_catalog_threads :: () -> s64 { + return clamp(ifx app.catalog_threads > 0 then app.catalog_threads else background.threads, 1, background.threads); +} + #scope_file -Background_Kind :: enum u8 { SEARCH_EMBED; SEARCH_CHUNK; SEARCH_MERGE; STATS; } +Background_Kind :: enum u8 { SEARCH_EMBED; SEARCH_CHUNK; SEARCH_MERGE; STATS; CATALOG_LOAD; CATALOG_CACHE; CATALOG_SAVE; } Background_Job :: struct { kind: Background_Kind; @@ -200,6 +269,11 @@ Background_Job :: struct { matches: [..] Search_Item; visible: [..] s32; + // CATALOG_LOAD, CATALOG_CACHE, CATALOG_SAVE + catalog: *Catalog_Batch; + cache_index: *Catalog_Cache_Index; + cache_save: *Catalog_Cache_Save; + // STATS items: [] Search_Item; model: string; @@ -249,7 +323,14 @@ background_proc :: (group: *Thread_Group, thread: *Thread, work: *void) -> Threa case .SEARCH_CHUNK; search_chunk(job); case .SEARCH_MERGE; search_merge(job); case .STATS; job.stats = stats_compute(job.items, job.model); + case .CATALOG_LOAD; catalog_load_job(job.catalog); + case .CATALOG_CACHE; catalog_cache_index_job(job.cache_index); + case .CATALOG_SAVE; catalog_cache_save_job(job.cache_save); } + // The catalog's jobs are long and come in bursts (startup, a save), after which the workers idle: + // what this thread's heap holds free goes back now. Not after each search chunk, which come many + // a second. + if job.kind == .CATALOG_LOAD || job.kind == .CATALOG_CACHE || job.kind == .CATALOG_SAVE Default_Allocator.thread_idle(); reset_temporary_storage(); return .CONTINUE; } diff --git a/src/app/catalog.jai b/src/app/catalog.jai index a561c7e..2922c7c 100644 --- a/src/app/catalog.jai +++ b/src/app/catalog.jai @@ -1,5 +1,11 @@ // The catalog: what search and sorting need from every capture, loaded once from the front of each -// file (its Capture Text and OCR text, not its images) on the viewer's worker threads. +// file (its Capture Text and OCR text, not its images) on the background thread group. +// +// The catalog cache (CATALOG_CACHE_NAME, in the library folder) keeps every entry with the size and +// modified time of the file it was read from, so a start reads one file instead of every capture: +// an entry whose file is unchanged comes from the cache, the rest from their files. It is written +// once the catalog is complete and has changed, at most every CATALOG_CACHE_SAVE_SECONDS, and at +// shutdown. // // Entries are made on the workers and handed over through viewer.catalog_done; only the main thread // sets or frees viewer.captures[i].catalog, and only it reads them, so no entry is freed while in use. @@ -25,6 +31,11 @@ Catalog_Entry :: struct { segment_count: s32; segments: [..] Catalog_Segment; // largest first + // The file it was read from, when it was listed or just before it was read: the catalog cache + // gives this entry again only while the file's size and modified time are still these. + file_size: s64; + file_modified: s64; // nanoseconds + // Lowercased copies, for matching without case. find_title, find_url, find_app, find_notes, find_ocr, find_labels, find_folder: string; find_segments: string; // each class in it once, between newlines: "|person|dog|" with | a newline, for has: @@ -67,8 +78,14 @@ catalog_entry_free :: (entry: *Catalog_Entry) { free(entry); } -// The entry for the .mcap at path, or null with why. On any thread. -catalog_load :: (path: string) -> *Catalog_Entry, error: string { +// The entry for the .mcap at path, or null with why. On any thread. size and modified are the file's +// as listed, when known; otherwise it is looked at first. +catalog_load :: (path: string, size: s64 = -1, modified: s64 = 0) -> *Catalog_Entry, error: string { + if size < 0 { + time, stat_size, stat_ok := file_modtime_and_size(path); + if stat_ok { size = stat_size; modified = to_nanoseconds(time); } + else { size = 0; modified = 0; } // matches no listing: never given again by the cache + } front: Mcap; ok: bool; error: string; @@ -80,6 +97,7 @@ catalog_load :: (path: string) -> *Catalog_Entry, error: string { push_allocator(context.default_allocator); entry := New(Catalog_Entry); + entry.file_size, entry.file_modified = size, modified; // The metadata and the annotations, from the Capture Text. doc: Capture_Document; @@ -158,6 +176,14 @@ catalog_load :: (path: string) -> *Catalog_Entry, error: string { } } + catalog_finish(entry, path); + return entry, ""; +} + +// What an entry derives from the rest and from where its file is: its folder, and the lowercased +// copies search matches. For entries read from files and from the cache alike. +catalog_finish :: (entry: *Catalog_Entry, path: string) { + push_allocator(context.default_allocator); library := viewer_folder(); if path.count > library.count + 1 && begins_with_nocase(path, library) { folder := path_strip_filename(advance(path, library.count + 1)); @@ -182,7 +208,41 @@ catalog_load :: (path: string) -> *Catalog_Entry, error: string { append(*all_labels, "\n"); } entry.find_labels = to_lower_copy(builder_to_string(*all_labels,, temp)); - return entry, ""; +} + +// On a background worker (app/background.jai): the batch's entries, taken one at a time with the +// other jobs on it, each handed to the main thread as it is made. Stops early when the viewer is +// shutting down. +catalog_load_job :: (batch: *Catalog_Batch) { + while true { + n := atomic_add(*batch.next, 1); + if n >= batch.items.count return; + it := batch.items[n]; + entry: *Catalog_Entry; + from_cache := false; + if it.offset >= 0 { + d := Deserializer.{count = batch.cache.count, data = batch.cache.data, cursor = it.offset}; + entry = catalog_cache_read_entry(*d); + if entry { + entry.file_size, entry.file_modified = it.size, it.modified; + catalog_finish(entry, it.path); + from_cache = true; + } else { + log_error("[catalog] The cache's entry for % is damaged; reading the file.", it.path); + } + } + if !entry { + error: string; + entry, error = catalog_load(it.path, it.size, it.modified); + if !entry log_error("[catalog] % %", it.path, error); + } + reset_temporary_storage(); + lock(*viewer.mutex); + array_add(*viewer.catalog_updates, .{it.index, entry, from_cache}); + quit := viewer.quit; + unlock(*viewer.mutex); + if quit return; + } } catalog_section :: (entry: *Catalog_Entry, model: string) -> *Mcap_Ocr_Section { @@ -191,6 +251,288 @@ catalog_section :: (entry: *Catalog_Entry, model: string) -> *Mcap_Ocr_Section { return null; } +// ---------------------------------------------------------------- the catalog cache +// +// "MCAPCATL", u32 CATALOG_CACHE_VERSION, LABELS_MODEL and SEGMENT_MODEL (u32-sized strings), u32 count, +// then count records: u32 length of what follows, the path relative to the library, the file's +// size and modified time (s64, s64), and the entry's own fields (catalog_cache_add_entry). Its +// folder and lowercased copies are not kept: catalog_finish makes them again. + +CATALOG_CACHE_NAME :: ".catalog-cache"; // in the library folder; hidden, so the watcher and listing leave it out +CATALOG_CACHE_MAGIC :: "MCAPCATL"; +CATALOG_CACHE_VERSION :: 1; // raise whenever Catalog_Entry, or what catalog_load takes from a file, changes +CATALOG_CACHE_SAVE_SECONDS :: 30.0; // the least time between saves while the app runs + +catalog_cache: struct { + dirty: bool; // UI thread: entries changed since the cache was last written + saving: bool; // UI thread: a save job is out + saved_at: Apollo_Time; // monotonic; zero before the first save +}; + +catalog_cache_path :: () -> string { + return tprint("%/%", viewer_folder(), CATALOG_CACHE_NAME); +} + +// The startup job's: the cache read, and the captures sorted into those it has a current entry for +// (items, each with where its entry starts in data) and the rest (misses). All heap. +Catalog_Cache_Index :: struct { + path: string; + captures: [] Catalog_Item; // every capture, with its file's size and modified time as listed + data: string; // the cache file; handed to the batch that reads items + items: [..] Catalog_Item; + misses: [..] Catalog_Item; + records, stale: s64; // records in the cache, and those for files changed or gone + read_ms, index_ms: float64; + problem: string; // why some or all of it was not used (a literal); "" when all was +} + +// On a background worker. +catalog_cache_index_job :: (job: *Catalog_Cache_Index) { + push_allocator(context.default_allocator); + job.items.allocator = context.default_allocator; + job.misses.allocator = context.default_allocator; + start := current_time_monotonic(); + data, read_ok := read_entire_file(job.path, log_errors = false); + job.read_ms = to_float64_seconds(current_time_monotonic() - start) * 1000.0; + + start = current_time_monotonic(); + matched := NewArray(job.captures.count, bool,, temp); + if read_ok { + job.data = data; + job.problem = catalog_cache_index(job, matched); + } else { + job.problem = "there is none yet"; + } + for job.captures if !matched[it_index] array_add(*job.misses, it); + job.index_ms = to_float64_seconds(current_time_monotonic() - start) * 1000.0; +} + +// The records whose file is unchanged, into job.items. A damaged record ends the walk: those before +// it are good, as each says how long it is. +catalog_cache_index :: (job: *Catalog_Cache_Index, matched: [] bool) -> problem: string { + d := to_deserializer(job.data); + if ReadStringView(*d, CATALOG_CACHE_MAGIC.count) != CATALOG_CACHE_MAGIC return "it is not a catalog cache"; + if Read(*d, u32) != CATALOG_CACHE_VERSION return "another version of mcap wrote it"; + labels_model := ReadSizedString(*d, u32); + segment_model := ReadSizedString(*d, u32); + if labels_model != LABELS_MODEL || segment_model != SEGMENT_MODEL return "it was made with other label or segmentation models"; + count := Read(*d, u32); + if !d.ok return "it is damaged"; + + folder := viewer_folder(); + by_path: Table(string, s64); + by_path.allocator = temp; + for job.captures table_set(*by_path, advance(it.path, folder.count + 1), it_index); + for 1..count { + length := Read(*d, u32); + end := d.cursor + length; + relative := ReadSizedString(*d, u32); + size := Read(*d, s64); + modified := Read(*d, s64); + if !d.ok || end > d.count return "it ends early; the rest is read from the files"; + job.records += 1; + found, position := table_find(*by_path, relative); + if found && !matched[position] && job.captures[position].size == size && job.captures[position].modified == modified { + matched[position] = true; + item := job.captures[position]; + item.offset = d.cursor; + array_add(*job.items, item); + } else { + job.stale += 1; + } + d.cursor = end; + } + return ""; +} + +// UI thread: the startup job is back. The current entries are read from the cache and the rest from +// their files, both on the workers. +catalog_cache_indexed :: (job: *Catalog_Cache_Index) { + log("[startup] Catalog cache at % ms: % MB read in % ms, sorted in % ms; % of % captures from it, % to read from their files (% of its records stale or gone)%.", + FF(startup_ms(), 1), FF(job.data.count / (1024.0 * 1024.0), 1), FF(job.read_ms, 1), FF(job.index_ms, 1), + job.items.count, job.captures.count, job.misses.count, job.stale, ifx job.problem then tprint("; %", job.problem) else ""); + if job.stale || job.problem catalog_cache.dirty = true; // the next save leaves the stale ones out + background_catalog_load(job.misses); + if job.items.count background_catalog_load(job.items, job.data); // the batch frees data + else free(job.data,, context.default_allocator); + push_allocator(context.default_allocator); + free(job.path); + array_free(job.captures); + array_free(job.items); + array_free(job.misses); + free(job); +} + +// The entry at d's cursor, from the default allocator, without its folder and lowercased copies +// (catalog_finish). Null when d runs out. +catalog_cache_read_entry :: (d: *Deserializer) -> *Catalog_Entry { + push_allocator(context.default_allocator); + read_string :: () -> string #expand { return copy_string(ReadSizedString(d, u32)); } + entry := New(Catalog_Entry); + entry.date = read_string(); + entry.title = read_string(); + entry.url = read_string(); + entry.app = read_string(); + entry.notes = read_string(); + entry.label_list = read_string(); + entry.video_codec = read_string(); + entry.segment_classes = read_string(); + entry.media = Read(d, Catalog_Media); + entry.duration = Read(d, float64); + entry.frame_rate = Read(d, float64); + entry.segment_count = Read(d, s32); + entry.embedding = array_copy(ReadSizedArray(d, u16, u32)); + + entry.labels.allocator = context.default_allocator; + array_resize(*entry.labels, Read(d, u32)); + for * entry.labels { it.score = Read(d, float); it.name = read_string(); } + entry.segments.allocator = context.default_allocator; + array_resize(*entry.segments, Read(d, u32)); + for * entry.segments { it.class = read_string(); it.score = Read(d, float); it.area = Read(d, float); } + entry.ocr.allocator = context.default_allocator; + array_resize(*entry.ocr, Read(d, u32)); + for * section: entry.ocr { + section.model = read_string(); + section.engine = read_string(); + section.image = read_string(); + section.date = read_string(); + section.image_width = Read(d, s32); + section.image_height = Read(d, s32); + section.lines.allocator = context.default_allocator; + array_resize(*section.lines, Read(d, u32)); + for * line: section.lines { + line.x = Read(d, s32); + line.y = Read(d, s32); + line.width = Read(d, s32); + line.height = Read(d, s32); + line.confidence = Read(d, float); + line.corners = Read(d, [8] s32); + line.text = read_string(); + if !d.ok break; + } + if !d.ok break; + } + if !d.ok { catalog_entry_free(entry); return null; } + return entry; +} + +catalog_cache_add_entry :: (s: *Serializer, entry: *Catalog_Entry) { + for string.[entry.date, entry.title, entry.url, entry.app, entry.notes, entry.label_list, entry.video_codec, entry.segment_classes] AddString(s, it, u32); + Add(s, entry.media); + Add(s, entry.duration); + Add(s, entry.frame_rate); + Add(s, entry.segment_count); + AddSizedArray(s, entry.embedding, u32); + Add(s, cast(u32) entry.labels.count); + for entry.labels { Add(s, it.score); AddString(s, it.name, u32); } + Add(s, cast(u32) entry.segments.count); + for entry.segments { AddString(s, it.class, u32); Add(s, it.score); Add(s, it.area); } + Add(s, cast(u32) entry.ocr.count); + for section: entry.ocr { + for string.[section.model, section.engine, section.image, section.date] AddString(s, it, u32); + Add(s, section.image_width); + Add(s, section.image_height); + Add(s, cast(u32) section.lines.count); + for line: section.lines { + Add(s, line.x); Add(s, line.y); Add(s, line.width); Add(s, line.height); + Add(s, line.confidence); + Add(s, line.corners); + AddString(s, line.text, u32); + } + } +} + +Catalog_Cache_Record :: struct { + relative: string; // the capture's path relative to the library: a view into it, never freed + entry: *Catalog_Entry; +} + +// The cache file's bytes for these records. Default allocator. +catalog_cache_serialize :: (records: [] Catalog_Cache_Record) -> Serializer { + push_allocator(context.default_allocator); + s := new_serializer(records.count * 4096 + 1024); + AddArray_NoSize(*s, cast([] u8) CATALOG_CACHE_MAGIC); + Add(*s, cast(u32) CATALOG_CACHE_VERSION); + AddString(*s, LABELS_MODEL, u32); + AddString(*s, SEGMENT_MODEL, u32); + Add(*s, cast(u32) records.count); + for records { + length_at := s.count; + Add(*s, cast(u32) 0); // its length, filled in below + AddString(*s, it.relative, u32); + Add(*s, it.entry.file_size); + Add(*s, it.entry.file_modified); + catalog_cache_add_entry(*s, it.entry); + length := cast(u32) (s.count - length_at - size_of(u32)); + memcpy(s.data + length_at, *length, size_of(u32)); + } + return s; +} + +// A save: the entries as they were when it started. Heap. A job holding one counts in +// background.jobs_out, so no entry it reads is freed while it runs (background_retire). +Catalog_Cache_Save :: struct { + path: string; + records: [] Catalog_Cache_Record; + bytes: s64; + ms: float64; + ok: bool; +} + +// UI thread, each frame: saves once the catalog is complete and has changed. +catalog_cache_frame :: () { + if !catalog_cache.dirty || catalog_cache.saving || viewer.catalog_loaded < viewer.captures.count return; + if catalog_cache.saved_at != .{} && to_float64_seconds(current_time_monotonic() - catalog_cache.saved_at) < CATALOG_CACHE_SAVE_SECONDS return; + catalog_cache.dirty = false; + catalog_cache.saving = true; + background_catalog_save(catalog_cache_snapshot()); +} + +catalog_cache_snapshot :: () -> *Catalog_Cache_Save { + push_allocator(context.default_allocator); + save := New(Catalog_Cache_Save); + save.path = copy_string(catalog_cache_path()); + records: [..] Catalog_Cache_Record; + array_reserve(*records, viewer.captures.count); + folder := viewer_folder(); + for viewer.captures if !it.removed && it.catalog array_add(*records, .{advance(it.path, folder.count + 1), it.catalog}); + save.records = records; + return save; +} + +// On a background worker, or the shutdown thread: written beside itself, then swapped in. +catalog_cache_save_job :: (save: *Catalog_Cache_Save) { + start := current_time_monotonic(); + s := catalog_cache_serialize(save.records); + defer array_free(s); + temp_path := tprint("%.tmp", save.path); + save.ok = write_entire_file(temp_path, s.data, s.count) && atomic_file_replace_or_just_rename_if_destination_file_does_not_exist(temp_path, save.path); + save.bytes = s.count; + save.ms = to_float64_seconds(current_time_monotonic() - start) * 1000.0; +} + +// UI thread (or the shutdown thread): a save is done. +catalog_cache_save_done :: (save: *Catalog_Cache_Save) { + catalog_cache.saving = false; + catalog_cache.saved_at = current_time_monotonic(); + if save.ok log("[catalog] Saved the cache: % entries, % MB in % ms.", save.records.count, FF(save.bytes / (1024.0 * 1024.0), 1), FF(save.ms, 1)); + else log_error("[catalog] Could not write %; the next start reads the captures from their files.", save.path); + push_allocator(context.default_allocator); + free(save.path); + array_free(save.records); + free(save); +} + +// On the shutdown thread, once the background group has stopped: the last changes, when the +// catalog is complete (an incomplete one would drop what the cache holds of the rest). +catalog_cache_save_at_shutdown :: () { + if !catalog_cache.dirty && !catalog_cache.saving return; + if viewer.catalog_loaded < viewer.captures.count return; + save := catalog_cache_snapshot(); + catalog_cache_save_job(save); + catalog_cache_save_done(save); +} + // ---------------------------------------------------------------- search // LIKE is by meaning: the captures whose image is nearest the phrase, ranked by the label model diff --git a/src/app/main.jai b/src/app/main.jai index dd5c04d..995cf3b 100644 --- a/src/app/main.jai +++ b/src/app/main.jai @@ -1,6 +1,7 @@ main :: () { using platform; using ImGui; // #Startup: Program Startup + app.launch_time = current_time_monotonic(); app.args = get_command_line_arguments(); print_version_and_exit := array_find(app.args, "-version"); @@ -31,6 +32,9 @@ main :: () { app.record_region = .{x, y, x + cast(s32) string_to_int(app.args[it_index + 3]), y + cast(s32) string_to_int(app.args[it_index + 4])}; } app.stats_and_exit = array_find(app.args, "-stats"); + app.cold_start = array_find(app.args, "-cold"); + app.no_catalog_cache = array_find(app.args, "-no-catalog-cache"); + for app.args if it == "-catalog-threads" && it_index + 1 < app.args.count app.catalog_threads = string_to_int(app.args[it_index + 1]); for app.args if it == "-search" && it_index + 1 < app.args.count app.start_search = app.args[it_index + 1]; for app.args if it == "-erase" && it_index + 1 < app.args.count app.erase_and_exit = app.args[it_index + 1]; for app.args if it == "-import" && it_index + 2 < app.args.count { app.import_source = app.args[it_index + 1]; app.import_dest = app.args[it_index + 2]; } @@ -366,6 +370,11 @@ main :: () { if app.exit_code exit(app.exit_code); // e.g. a headless test run with failures } +// Milliseconds since main() started, for the [startup] log lines. +startup_ms :: () -> float64 { + return to_float64_seconds(current_time_monotonic() - app.launch_time) * 1000.0; +} + // Returns false when the main loop should exit. app_frame :: (in_live_resize := false) -> bool { using platform; using ImGui; diff --git a/src/app/ocr_ui.jai b/src/app/ocr_ui.jai index 98e6671..fcd9215 100644 --- a/src/app/ocr_ui.jai +++ b/src/app/ocr_ui.jai @@ -362,7 +362,7 @@ ocr_ui_thread_proc :: (thread: *Thread) -> s64 { entry, load_error := catalog_load(item.path); if entry { lock(*viewer.mutex); - array_add(*viewer.catalog_updates, .{item.index, entry}); + array_add(*viewer.catalog_updates, .{item.index, entry, false}); unlock(*viewer.mutex); } else { log_error("[Erase] % was erased, but could not be read back: %", item.path, load_error); @@ -508,7 +508,7 @@ ocr_read_and_store :: (items: [] Scan_Item, set: *Label_Set) -> [] string { entry, load_error := catalog_load(items[n].path); if !entry { errors[n] = sprint("stored, but could not be read back: %", load_error); continue; } lock(*viewer.mutex); - array_add(*viewer.catalog_updates, .{items[n].index, entry}); + array_add(*viewer.catalog_updates, .{items[n].index, entry, false}); unlock(*viewer.mutex); } return errors; diff --git a/src/app/segment_ui.jai b/src/app/segment_ui.jai index 8455e32..93edc71 100644 --- a/src/app/segment_ui.jai +++ b/src/app/segment_ui.jai @@ -271,7 +271,7 @@ segment_read_and_store :: (engine: *Segment_Engine, items: [] Scan_Item) -> [] s entry, load_error := catalog_load(item.path); if entry { lock(*viewer.mutex); - array_add(*viewer.catalog_updates, .{item.index, entry}); + array_add(*viewer.catalog_updates, .{item.index, entry, false}); unlock(*viewer.mutex); } else { errors[n] = sprint("was segmented, but could not be read back: %", load_error); diff --git a/src/app/viewer.jai b/src/app/viewer.jai index 96c0562..255bd52 100644 --- a/src/app/viewer.jai +++ b/src/app/viewer.jai @@ -3,8 +3,8 @@ // those in view first and then the rows around them, and are uploaded a few per frame. Textures // for previews far out of view are released once there are more than VIEWER_MAX_TEXTURES. // -// When the workers have no previews to decode they load the catalog (app/catalog.jai), what search -// and sorting need from each capture. The grid shows viewer.visible: the captures in display order, +// The catalog (app/catalog.jai), what search and sorting need from each capture, loads on the +// background thread group (app/background.jai), so previews keep these workers to themselves. The grid shows viewer.visible: the captures in display order, // newest first once the catalog is complete, and only those matching the search. Searching runs on // background workers (app/background.jai), so the UI thread never waits on it. // @@ -57,12 +57,15 @@ Viewer_Capture :: struct { catalog: *Catalog_Entry; // main thread only; null until loaded removed: bool; // main thread only: its file was gone at a refresh; shown nowhere catalog_failed: bool; // main thread only: its catalog entry could not be loaded + file_size: s64 = -1; // its file's, as listed at startup or a refresh; -1 when not known + file_modified: s64; // nanoseconds } // A catalog entry made off the main thread, for the main thread to put in place. Catalog_Update :: struct { index: s32; entry: *Catalog_Entry; // null when it could not be loaded + from_cache: bool; // given by the catalog cache, not read from the file } // How the full-size window sizes its image. @@ -110,7 +113,6 @@ viewer: struct { quit: bool; workers: [VIEWER_WORKER_COUNT] Thread; workers_started: s64; - catalog_next: s64; // the next capture whose catalog entry a worker loads catalog_updates: [..] Catalog_Update; // made, waiting for the main thread // Display order and search; main thread only. @@ -118,6 +120,12 @@ viewer: struct { visible_dirty := true; visible_rebuilt_at: Apollo_Time; catalog_loaded: s64; // entries put in place, or found unreadable + catalog_started: Apollo_Time; // when the workers started on it, for the log line once it is complete + catalog_timed: bool; // that line has been logged + catalog_from_cache: s64; // entries the catalog cache gave + catalog_from_files: s64; // entries read from their files + first_preview_logged: bool; + first_frame_logged: bool; removed_count: s64; // captures marked removed delete_index: s32 = -1; // the capture whose delete is being confirmed delete_without_asking: bool; // until the app restarts @@ -172,13 +180,16 @@ viewer_startup :: () { start := current_time_monotonic(); for viewer_list_files() { capture := array_add(*viewer.captures); - viewer_capture_init(capture, it); + viewer_capture_init(capture, it.path); + capture.file_size, capture.file_modified = it.size, it.modified; } viewer_update_folders(); // Newest first, roughly: dated names (2026-...) sort after the older {GUID} ones. quick_sort(viewer.captures, (a: Viewer_Capture, b: Viewer_Capture) -> s64 { return compare(b.name, a.name); }); if !viewer.captures.count viewer.scan_error = sprint("No .% files in %.", MCAP_EXTENSION, viewer_folder()); - log("[viewer] Found % captures in % in % ms.", viewer.captures.count, viewer_folder(), FF(to_float64_seconds(current_time_monotonic() - start) * 1000.0, 1)); + log("[startup] Found % captures in % at % ms, listing them in % ms.", viewer.captures.count, viewer_folder(), FF(startup_ms(), 1), FF(to_float64_seconds(current_time_monotonic() - start) * 1000.0, 1)); + if app.cold_start viewer_evict_file_cache(); + viewer.catalog_started = current_time_monotonic(); for * viewer.workers { // Reading a front takes 64 KB of temporary storage, and a full image several MB. @@ -190,6 +201,23 @@ viewer_startup :: () { viewer.workers_started += 1; } background_startup(); + { + // The catalog: from the cache what it has a current entry for, the rest from the files. + items := NewArray(viewer.captures.count, Catalog_Item,, temp); + for * items { + capture := *viewer.captures[it_index]; + it.* = .{index = cast(s32) it_index, path = capture.path, size = capture.file_size, modified = capture.file_modified}; + } + if app.no_catalog_cache { + log("[startup] -no-catalog-cache: every capture is read from its file."); + background_catalog_load(items); + } else { + index := New(Catalog_Cache_Index,, context.default_allocator); + index.path = copy_string(catalog_cache_path(),, context.default_allocator); + index.captures = array_copy(items,, context.default_allocator); + background_catalog_cache(index); + } + } if app.start_search viewer_set_search(app.start_search); } @@ -202,6 +230,7 @@ viewer_shutdown :: () { for 0..viewer.workers_started - 1 thread_is_done(*viewer.workers[it], -1); if viewer.open_started thread_is_done(*viewer.open_thread, -1); background_shutdown(); + catalog_cache_save_at_shutdown(); // the workers have stopped: nothing changes the entries now viewer_watch_stop(); } @@ -213,7 +242,6 @@ viewer_worker :: (thread: *Thread) -> s64 { while true { index: s32 = -1; full: bool; - catalog: bool; request: u64; path: string; @@ -234,15 +262,13 @@ viewer_worker :: (thread: *Thread) -> s64 { } else if viewer.queue_next < viewer.queue.count { index = viewer.queue[viewer.queue_next]; viewer.queue_next += 1; - } else if viewer.catalog_next < viewer.captures.count { - // Nothing to show is waiting: load the next catalog entry. - catalog = true; - index = cast(s32) viewer.catalog_next; - viewer.catalog_next += 1; } if index >= 0 path = viewer.captures[index].path; // the array grows only under the mutex, and paths are never freed unlock(*viewer.mutex); - if index < 0 break; // nothing to do: wait to be woken + if index < 0 { // nothing to do: wait to be woken, having given back what this thread's heap holds free + Default_Allocator.thread_idle(); + break; + } if segments { overlay := segment_overlay_read(path); @@ -260,16 +286,6 @@ viewer_worker :: (thread: *Thread) -> s64 { continue; } - if catalog { - entry, error := catalog_load(path); - if !entry log_error("[catalog] % %", path, error); - reset_temporary_storage(); - lock(*viewer.mutex); - array_add(*viewer.catalog_updates, .{index, entry}); - unlock(*viewer.mutex); - continue; - } - result := ifx full then viewer_decode_full(path) else viewer_decode_preview(path); result.index = index; @@ -422,6 +438,10 @@ viewer_frame_gpu_work :: () { } else if !capture.texture.handle { capture.texture = gpu_texture_create(d.pixels, d.width, d.height); if capture.texture.handle array_add(*viewer.with_texture, d.index); + if capture.texture.handle && !viewer.first_preview_logged { + viewer.first_preview_logged = true; + log("[startup] First preview shown at % ms.", FF(startup_ms(), 1)); + } else capture.failed = true; } viewer_free_pixels(*d); @@ -429,6 +449,7 @@ viewer_frame_gpu_work :: () { viewer_evict(); viewer_apply_catalog_updates(); + catalog_cache_frame(); // saves the catalog cache once it is complete and has changed (app/catalog.jai) viewer_open_frame(); segment_overlay_frame(); // the Capture window's segments: what arrived, and its texture (app/segment_ui.jai) background_frame(); // takes back search results, and starts a search when visible_dirty (app/background.jai) @@ -446,7 +467,6 @@ viewer_apply_catalog_updates :: () { lock(*viewer.mutex); updates := array_copy(viewer.catalog_updates,, temp); array_reset_keeping_memory(*viewer.catalog_updates); - more_to_load := viewer.catalog_next < viewer.captures.count; unlock(*viewer.mutex); // Whether the date order can change: a capture's first entry (one a refresh added), one lost, or a @@ -457,6 +477,8 @@ viewer_apply_catalog_updates :: () { capture := *viewer.captures[it.index]; if !capture.catalog viewer.catalog_loaded += 1; if !capture.catalog capture.catalog_failed = !it.entry; + if it.from_cache viewer.catalog_from_cache += 1; + else { viewer.catalog_from_files += 1; catalog_cache.dirty = true; } if !capture.catalog || !it.entry || it.entry.date != capture.catalog.date order_changes = true; if it.entry || !capture.catalog { background_retire(capture.catalog); // a search or Stats job may be reading it @@ -470,9 +492,15 @@ viewer_apply_catalog_updates :: () { if !viewer.sorted_by_date && viewer.catalog_loaded >= viewer.captures.count { viewer.sorted_by_date = true; viewer.visible_dirty = true; + if !viewer.catalog_timed { + viewer.catalog_timed = true; + failed: s64; + for viewer.captures if it.catalog_failed failed += 1; + log("[startup] Catalog complete at % ms: % entries in % ms, % from the cache and % read from their files on % threads (% could not be read)%.", + FF(startup_ms(), 1), viewer.captures.count, FF(to_float64_seconds(current_time_monotonic() - viewer.catalog_started) * 1000.0, 1), + viewer.catalog_from_cache, viewer.catalog_from_files, background_catalog_threads(), failed, ifx app.cold_start then ", from a cold start" else ""); + } } - // The workers sleep when there is nothing to show; wake them while there is catalog to load. - if more_to_load signal(*viewer.wake); } // The captures in display order: by date, newest first, once every date is known (by name until @@ -484,7 +512,7 @@ viewer_refresh :: (quiet := false) { start := current_time_monotonic(); present: Table(string, bool); // the folder's .mcap files now present.allocator = temp; - for viewer_list_files() table_set(*present, it, true); + for viewer_list_files() table_set(*present, it.path, true); removed, added: s64; known: Table(string, bool); // the captures still there @@ -496,16 +524,19 @@ viewer_refresh :: (quiet := false) { removed += 1; } + new_items: [..] Catalog_Item; + new_items.allocator = temp; lock(*viewer.mutex); for _, path: present { if table_contains(*known, path) continue; capture := array_add(*viewer.captures); capture.* = .{}; viewer_capture_init(capture, path); + array_add(*new_items, .{index = cast(s32) (viewer.captures.count - 1), path = capture.path}); added += 1; } unlock(*viewer.mutex); - if added signal(*viewer.wake); // their catalog entries + background_catalog_load(new_items); viewer_update_folders(); free(viewer.scan_error); @@ -531,7 +562,7 @@ viewer_add_capture :: (path: string) -> s32 { viewer_capture_init(capture, path); index = cast(s32) (viewer.captures.count - 1); unlock(*viewer.mutex); - signal(*viewer.wake); // its catalog entry + background_catalog_load(.[.{index = index, path = capture.path}]); if viewer.scan_error { free(viewer.scan_error); viewer.scan_error = ""; } if !array_find(viewer.folders, capture.folder) viewer_update_folders(); viewer.sorted_by_date = false; // until its entry is in @@ -539,13 +570,21 @@ viewer_add_capture :: (path: string) -> s32 { return index; } -// Every .mcap in the library, in its folders too, but not in hidden ones. Temp. -viewer_list_files :: () -> [..] string { - files: [..] string; +Listed_File :: struct { + path: string; + size: s64; + modified: s64; // nanoseconds +} + +// Every .mcap in the library, in its folders too, but not in hidden ones, with its size and modified +// time (which listing a folder gives at no extra cost on Windows and Unix). Temp. +viewer_list_files :: () -> [..] Listed_File { + files: [..] Listed_File; files.allocator = temp; - for file_list(viewer_folder(), recursive = true,, temp) { - if viewer_is_library_file(it) array_add(*files, it); - } + visit_files(viewer_folder(), true, *files, (info: *File_Visit_Info, files: *[..] Listed_File) { + if !viewer_is_library_file(info.full_name) return; + array_add(files, .{copy_string(info.full_name,, temp), info.size, to_nanoseconds(info.modification_time)}); + }); return files; } @@ -671,6 +710,20 @@ viewer_watch_callback :: (watcher: *Watcher.File_Watcher(void), change: *Watcher for viewer.captures if !it.removed && begins_with(it.path, prefix) { viewer_watch.rescan = true; return; } } +// -cold: every capture's file dropped from the system's file cache, so the catalog and previews are +// read from the disk as on the first start after a reboot. +viewer_evict_file_cache :: () { + start := current_time_monotonic(); + evicted: s64; + for viewer.captures if platform_evict_file_cache(it.path) evicted += 1; + platform_evict_file_cache(catalog_cache_path()); + if !evicted && viewer.captures.count { + log("[viewer] -cold: the system's file cache cannot be emptied here; the library is read from whatever it holds."); + return; + } + log("[viewer] -cold: dropped % of % captures from the system's file cache in % ms.", evicted, viewer.captures.count, FF(to_float64_seconds(current_time_monotonic() - start) * 1000.0, 1)); +} + viewer_capture_init :: (capture: *Viewer_Capture, path: string) { push_allocator(context.default_allocator); capture.path = copy_string(path); @@ -704,6 +757,7 @@ viewer_mark_removed :: (index: s32) { if capture.removed return; capture.removed = true; viewer.removed_count += 1; + catalog_cache.dirty = true; // the next save leaves it out if capture.texture.handle { gpu_texture_release(*capture.texture); found, position := array_find(viewer.with_texture, index); @@ -971,6 +1025,7 @@ viewer_request_full :: (index: s32) { // ---------------------------------------------------------------- drawing viewer_ui :: () { using ImGui; + if !viewer.first_frame_logged { viewer.first_frame_logged = true; log("[startup] First frame of the grid at % ms.", FF(startup_ms(), 1)); } viewer_watch_frame(); // files added or deleted outside the app viewer_frame_gpu_work(); diff --git a/src/base/platform/platform_linux.jai b/src/base/platform/platform_linux.jai index 4dcc95e..e9303c0 100644 --- a/src/base/platform/platform_linux.jai +++ b/src/base/platform/platform_linux.jai @@ -133,3 +133,12 @@ platform_set_thread_affinity :: (thread: *Thread, affinity: CPUID_Type) { } XDG :: #import "XDG"; + +// Drops what the page cache holds of the file, so the next read comes from the disk (-cold). Clean +// pages go at once; pages not yet written back stay. False when it cannot be opened. +platform_evict_file_cache :: (path: string) -> bool { + fd := POSIX.open(temp_c_string(path), POSIX.O_RDONLY | POSIX.O_CLOEXEC); + if fd < 0 return false; + defer POSIX.close(fd); + return POSIX.posix_fadvise(fd, 0, 0, POSIX.POSIX_FADV_DONTNEED) == 0; +} diff --git a/src/base/platform/platform_macos.jai b/src/base/platform/platform_macos.jai index 7ead4cd..953d8a1 100644 --- a/src/base/platform/platform_macos.jai +++ b/src/base/platform/platform_macos.jai @@ -73,4 +73,10 @@ platform_cjk_font_candidates :: () -> [] Cjk_Font { Mach :: #import "macOS"; // task_info #import "Objective_C"; -#import "Objective_C/AppKit"; \ No newline at end of file +#import "Objective_C/AppKit"; + +// macOS lets no unprivileged program empty its file cache of a file (only `sudo purge`, all of it), +// so -cold reads whatever is cached. +platform_evict_file_cache :: (path: string) -> bool { + return false; +} diff --git a/src/base/platform/platform_windows.jai b/src/base/platform/platform_windows.jai index 686bec9..6708b8f 100644 --- a/src/base/platform/platform_windows.jai +++ b/src/base/platform/platform_windows.jai @@ -646,6 +646,18 @@ platform_map_file_read_only :: (path: string) -> (data: string, ok: bool) { return string.{size, cast(*u8) view}, true; } +// Drops what the system's file cache holds of the file, so the next read comes from the disk (-cold). +// Opening a file unbuffered makes the cache manager flush and purge its pages, when nothing has the +// file mapped. False when it cannot be opened. +platform_evict_file_cache :: (path: string) -> bool { + FILE_FLAG_NO_BUFFERING :: 0x20000000; + wide := utf8_to_wide(path,, temp); + handle := CreateFileW(wide, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, null, OPEN_EXISTING, FILE_FLAG_NO_BUFFERING); + if handle == INVALID_HANDLE_VALUE return false; + CloseHandle(handle); + return true; +} + // "" when the variable is unset or empty. In the current allocator. windows_environment_variable :: (name: string) -> string { name_w := utf8_to_wide(name,, temp); diff --git a/src/test/catalog_tests.jai b/src/test/catalog_tests.jai index 5503738..0412fb8 100644 --- a/src/test/catalog_tests.jai +++ b/src/test/catalog_tests.jai @@ -1,5 +1,5 @@ // The catalog's pieces on their own (app/catalog.jai, app/viewer.jai): the search query's parsing and -// matching, the RTF stripper on Snagit's annotations, and the grid's order by date. +// matching, the RTF stripper on Snagit's annotations, the grid's order by date, and the catalog cache. #scope_file @@ -136,6 +136,81 @@ order_tests :: () { expect(viewer_display_order("2025-01-01 00:00:00", "b", "2025-01-01 00:00:00", "a") < 0, "the same date: by name, newest first"); } +// The catalog cache (app/catalog.jai): an entry written and read back is the same, and only the +// records whose file is unchanged are given again. +cache_tests :: () { + group("cache"); + folder := viewer_folder(); + entry := New(Catalog_Entry); + entry.date = copy_string("2026-10-07 08:11:15"); entry.title = copy_string("KiCad"); entry.url = copy_string("https://x"); + entry.app = copy_string("kicad"); entry.notes = copy_string("late\nnote"); entry.label_list = copy_string("f6d0"); + entry.video_codec = copy_string("h264"); entry.segment_classes = copy_string("ade20k"); + entry.media = .VIDEO; entry.duration = 12.5; entry.frame_rate = 30; entry.segment_count = 2; + entry.embedding = array_copy(u16.[0x3C00, 0x4000, 0x4200]); + array_add(*entry.labels, .{0.9, copy_string("Schematic")}, .{0.4, copy_string("PCB layout")}); + array_add(*entry.segments, .{copy_string("person"), 0.8, 0.25}); + section := array_add(*entry.ocr); + section.* = .{model = copy_string("ppocr"), engine = copy_string("ort"), image = copy_string("rendered"), image_width = 640, image_height = 480, date = copy_string("2026-10-08T21:04:11Z")}; + array_add(*section.lines, .{1, 2, 3, 4, 0.75, copy_string("0.1uF : Capacitor_SMD"), .[1, 2, 3, 4, 5, 6, 7, 8]}); + entry.file_size, entry.file_modified = 1234, 5678; + + records := Catalog_Cache_Record.[.{"A/one.mcap", entry}, .{"A/two.mcap", entry}]; + bytes := catalog_cache_serialize(records); + defer array_free(bytes); + + // one.mcap unchanged; two.mcap changed since; three.mcap not in the cache. + job: Catalog_Cache_Index; + job.data = cast(string) bytes; + captures := Catalog_Item.[ + .{index = 0, path = tprint("%/A/one.mcap", folder), size = 1234, modified = 5678}, + .{index = 1, path = tprint("%/A/two.mcap", folder), size = 1234, modified = 9999}, + .{index = 2, path = tprint("%/A/three.mcap", folder), size = 1, modified = 1}]; + job.captures = captures; + matched: [3] bool; + problem := catalog_cache_index(*job, matched); + expect(problem == "", "a cache it wrote is read whole: %", problem); + expect(job.records == 2 && job.items.count == 1 && job.items[0].index == 0 && job.stale == 1, "only the unchanged file's record is given: % records, % items, % stale", job.records, job.items.count, job.stale); + expect(matched[0] && !matched[1] && !matched[2], "the others are read from their files"); + + if job.items.count { + d := Deserializer.{count = bytes.count, data = bytes.data, cursor = job.items[0].offset}; + back := catalog_cache_read_entry(*d); + expect(back != null, "the entry reads back"); + if back { + catalog_finish(back, captures[0].path); + expect(back.date == entry.date && back.title == entry.title && back.url == entry.url && back.app == entry.app && back.notes == entry.notes, "its texts"); + expect(back.media == .VIDEO && back.duration == 12.5 && back.frame_rate == 30 && back.video_codec == "h264", "its media"); + expect(back.label_list == "f6d0" && back.labels.count == 2 && back.labels[1].name == "PCB layout" && back.labels[0].score == 0.9, "its labels"); + expect(back.embedding.count == 3 && back.embedding[2] == 0x4200, "its image vector"); + expect(back.segment_count == 2 && back.segments.count == 1 && back.segments[0].class == "person" && back.segments[0].area == 0.25, "its segments"); + ok := back.ocr.count == 1 && back.ocr[0].model == "ppocr" && back.ocr[0].image_height == 480 && back.ocr[0].lines.count == 1; + if ok { line := back.ocr[0].lines[0]; ok = line.text == "0.1uF : Capacitor_SMD" && line.height == 4 && line.confidence == 0.75 && line.corners[7] == 8; } + expect(ok, "its OCR lines"); + expect(back.find_ocr == "0.1uf : capacitor_smd\n" && back.find_folder == "a" && back.find_labels == "schematic\n", "catalog_finish makes its folder and lowercased copies: %, %, %", back.find_ocr, back.find_folder, back.find_labels); + catalog_entry_free(back); + } + } + array_free(job.items); + array_free(job.misses); + + // A cache from another version is not used at all. + (cast(*u32) (bytes.data + CATALOG_CACHE_MAGIC.count)).* = CATALOG_CACHE_VERSION + 1; + other: Catalog_Cache_Index; + other.data = cast(string) bytes; + other.captures = captures; + matched = .[false, false, false]; + expect(catalog_cache_index(*other, matched) != "" && !other.items.count, "another version's cache gives nothing"); + // Truncated: the records before the damage are still good. + (cast(*u32) (bytes.data + CATALOG_CACHE_MAGIC.count)).* = CATALOG_CACHE_VERSION; + short: Catalog_Cache_Index; + short.data = string.{bytes.count - 10, bytes.data}; + short.captures = captures; + matched = .[false, false, false]; + expect(catalog_cache_index(*short, matched) != "" && short.items.count == 1, "a cache cut short gives the records before the cut"); + array_free(other.items); array_free(short.items); + catalog_entry_free(entry); +} + #scope_export run_catalog_tests :: (builder: *String_Builder = null) -> bool { @@ -146,5 +221,6 @@ run_catalog_tests :: (builder: *String_Builder = null) -> bool { match_tests(); rtf_tests(); order_tests(); + cache_tests(); return report(); }