mcap/src/test/testing.jai

// Asynchronous testing module. To add a suite: a Test_Type, an add_suite_test line in
// run_all_tests, and a case in test_thread_group_proc. Run with: <executable> run_tests
// #NOTE: use all cpus for testing. see: get_number_of_processors(.ALL_PYHSICAL)
test_arena: *Arena;
test_data: *Test_Data;
test_thread_group: *Thread_Group;

Test_Data :: struct {
  work_added: s32;
  work_completed: s32;
  work_failed: s32;
  
  log_successful_tests: bool;
  assert_on_fail: bool;
  hide_test_logs: bool; // override logger in Thread_Group with a null logger to hide all output
  
  all_tests: [..]*Test_Task;
  // #note: store general results? successful vs failed tests?
  
  start_time: Apollo_Time;
  complete_time: Apollo_Time;
}

Test_Task :: struct { // Generic Test Task
  test_type: Test_Type;
  arena: *FixedArena;
  module: string;
  id: string;
  pass: bool;
  // Non-null means the test writes its report here instead of to stdout, and testing_cleanup
  // prints it whole on the main thread. Without this, concurrent tests interleave line by line.
  output: *String_Builder;
}

Test_Type :: enum s32 {
  none          :: 0;
  arena_suite;
  fixed_arena_suite;
  secure_memory_suite;
  logger_suite;
  catalog_suite;
  ocr_suite;
  labels_suite;
  import_suite;
}

testing_started :: () -> bool {
  return (test_arena != null) && (test_data.work_added > 0);
}

testing_init :: (assert_on_fail:=false, log_successful_tests:=false) -> bool {
  if testing_started() 
    return false;
  
  thread_count := cpu_info.physical_cpu_count;
  
  test_arena = bootstrap_arena(.Size_64G, label="Testing Module Arena");
  push_arena(test_arena);
  
  test_data         = New(Test_Data);
  test_thread_group = New(Thread_Group);
  
  init(test_thread_group, thread_count, test_thread_group_proc, 
    enable_work_stealing=true, temporary_storage_size=xx MB(1), group_name="tests");
  
  test_thread_group.allocator = get_allocator(test_arena);
  
  test_data.log_successful_tests = log_successful_tests;
  test_data.assert_on_fail       = assert_on_fail;
  
  start(test_thread_group);
  
  log("Started testing module with % threads.", thread_count);
  
  return true;
}

run_all_tests :: () {
  assert(test_thread_group != null);

  add_suite_test(.arena_suite,       "arena");
  add_suite_test(.fixed_arena_suite, "fixed_arena");
  add_suite_test(.secure_memory_suite, "secure_memory");
  add_suite_test(.logger_suite,        "logger");
  add_suite_test(.catalog_suite,       "catalog");
  add_suite_test(.ocr_suite,           "ocr");
  add_suite_test(.labels_suite,        "labels");
  add_suite_test(.import_suite,        "import");
}

// One work item per suite. Each suite keeps its tallies in file-scope globals, so the whole
// suite is the smallest unit that can run on a thread of its own without racing them.
add_suite_test :: (test_type: Test_Type, module: string) {
  arena := bootstrap_fixed_arena(KB(64)); // holds the task, the builder, and the whole report
  push_arena(arena);

  task := New(Test_Task);
  task.test_type = test_type;
  task.arena     = arena;
  task.module    = copy_string(module);
  task.id        = copy_string(module);

  // The builder allocates from this task's private arena, not from test_arena: three workers
  // calling arena_alloc on the one shared Arena is not safe.
  task.output = New(String_Builder);
  task.output.allocator = get_allocator(arena); // must be set before init; init only defaults it
  init_string_builder(task.output);

  add_new_test(task);
}

add_new_test :: (new_test: *void) {
  // !NOT THREADSAFE, ONLY CALL FROM MAIN THREAD.
  test_data.work_added += 1;
  add_work(test_thread_group, new_test);
}

testing_cleanup :: () {
  if test_thread_group && test_arena {
    cw := cast,force([]*Test_Task)get_completed_work(test_thread_group);
    test_data.work_completed += cast(s32)cw.count;

    for tt: cw {
      // A buffered task reported nothing from the worker. Print its output and then its
      // verdict, in that order, so a failing suite's report is visible before we assert.
      if tt.output {
        write_builder(tt.output);

        if tt.pass {
          log("[TEST PASSED][%] suite passed", tt.module);
        } else {
          log_error("[TEST FAILED][%] suite reported failures", tt.module);
          test_data.work_failed += 1;
          if test_data.assert_on_fail  assert(false, tprint("Test suite % failed.", tt.module));
        }
      }

      delete_fixed_arena(tt.arena);
    }

    if cw.count {
      log("Completed % tests this frame", cw.count);
    }
    
    if test_data.work_added == test_data.work_completed {
      log("All % test tasks complete.", test_data.work_completed);
      // Headless (run_tests -headless), the app exits once the tests are done: 0 if every suite
      // passed, 1 otherwise.
      if app.headless {
        if test_data.work_failed  log_error("% of % test suites failed.", test_data.work_failed, test_data.work_completed);
        app.exit_code = ifx test_data.work_failed then cast(s32) 1 else 0;
        app.should_exit = true;
      }
      testing_finish();
    }
  }
}

testing_finish :: () {
  success := shutdown(test_thread_group,, get_allocator(test_arena));
  if !success then return;
  
  test_thread_group = null;
  
  arena_delete(test_arena);
  
  test_arena = null;
  test_data  = null;
}

test_thread_group_proc :: (group: *Thread_Group, thread: *Thread, work: *void) -> Thread_Continue_Status {
  // thread_group_run never resets it, and these threads have no frame boundary to do it on,
  // so reclaim whatever the previous task on this worker left behind.
  reset_temporary_storage();

  task := thread_group_task(Test_Task);

  assert(task.test_type != .none, "test_type not set for Test_Task with id % @ %", task.id, work);
    
  if #complete task.test_type == {
    case .none;

    // The suites keep their own tallies and write their report into task.output. The verdict
    // lands in task.pass so that testing_cleanup can print the report before acting on a failure.
    case .arena_suite;
      task.pass = run_arena_tests(task.output);
    case .fixed_arena_suite;
      task.pass = run_fixed_arena_tests(task.output);
    case .secure_memory_suite;
      task.pass = run_secure_memory_tests(task.output);
    case .logger_suite;
      task.pass = run_logger_tests(task.output);
    case .catalog_suite;
      task.pass = run_catalog_tests(task.output);
    case .ocr_suite;
      task.pass = run_ocr_tests(task.output);
    case .labels_suite;
      task.pass = run_labels_tests(task.output);
    case .import_suite;
      task.pass = run_import_tests(task.output);
  }

  return .CONTINUE;
}