MusaMahmood/mcap
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity
5dd62ac201
mcap/modules/Screen_Record/macos.jai

527 lines
21 KiB
Plaintext
Raw Blame History

// The macOS recorder (module.jai has what is shared; windows.jai is the Windows one, which this follows).
// A ScreenCaptureKit stream (modules/Screen_Record/macos/capture.m) of one display's region gives the
// frames, already scaled to the region's pixels and with the cursor drawn in when asked, and the sound:
// what the computer plays, and on macOS 15 and later the microphone, both from the same stream, so
// there is no device to open here. mcap's own sound is left out.
//
// Three threads, as on Windows. The capture thread takes the newest frame, at most fps a second and
// one a second while nothing changes, so players can seek; each carries the host time it was shown,
// less the time paused. The encoder thread hands them to FFmpeg (encoder.jai, with VideoToolbox), the
// oldest dropped when it falls RECORD_QUEUE behind. The audio thread puts each source's packets on the
// same timeline into the mix (audio_mix.jai), and turns the sources on and off in the stream's
// configuration as record_set_audio asks.
//
// Times are the host clock's (mach_absolute_time, CLOCK_UPTIME_RAW) in nanoseconds, which
// ScreenCaptureKit's sample times are on too.
AUDIO_RATE :: 48000;
AUDIO_CHANNELS :: 2;
Audio_Source_Kind :: enum u8 { SYSTEM; MICROPHONE; }
// A source of sound in the stream: open while the stream is set to give it.
Audio_Source :: struct {
kind: Audio_Source_Kind;
name: string; // "System sound", "Microphone"
open: bool;
lost_at: s64; // the mix's; never lost here
}
Audio_Packet :: struct {
samples: [] float; // interleaved stereo, AUDIO_CHANNELS a frame
frames: s64;
ticks: s64; // the host time its first frame was heard, in nanoseconds
silent: bool;
}
Record_Options :: struct {
path: string; // the .mp4 written
display_id: u32; // the display (CGDirectDisplayID)
x, y, w, h: float64; // the region, in the display's points from its top left
width, height: s32; // and in its pixels: rounded down to even here
fps: s32 = 30;
cursor := true;
encoder: string; // "" tries RECORD_ENCODERS
system_audio: bool; // what the computer plays
microphone: bool; // the default input (macOS 15 and later); both at the start: record_set_audio turns them on and off
audio_track: bool; // a sound track whatever is on at the start, so a source can be turned on later
exclude_windows: [] u32; // window numbers kept out of the frames: the border and toolbar
}
Record_Stats :: struct {
source: string; // "ScreenCaptureKit, display 1"
fell_back_from: string; // never, here
encoder: string;
width, height: s32;
captured: s64; // new frames from the stream
pointer_moves: s64;
emitted: s64; // frames sent to be encoded
repeats: s64; // of them, the once-a-second repeat of an unchanged screen
written: s64; // encoded
dropped: s64; // the encoder was too far behind
lost: s64;
paused_seconds: float64; // left out of the video
audio: string; // the sound recorded: "System sound; Microphone", or ""
audio_frames: s64; // AAC frames encoded
audio_errors: string; // sources that could not be turned on
bytes: s64; // encoded
seconds: float64; // from start to stop
}
Recorder :: struct {
options: Record_Options;
stream: *Mcap_Sck_Stream;
encoder: Encoder;
width, height: s32;
frequency: s64 = 1_000_000_000; // ticks a second: nanoseconds
start_ticks: s64;
capture_thread, encoder_thread, audio_thread: Thread;
capture_started, encoder_started, audio_started: bool;
audio_sources: [2] Audio_Source;
audio_open: bool; // the MP4 has sound
audio_on: [2] bool; // each source is wanted (by Audio_Source_Kind): record_set_audio
audio_failed: [2] bool; // it could not be turned on: record_audio_failed says once
zero_ticks: s64; // the recording's time 0
paused_ticks: s64; // time paused so far; set by record_pause
pause_began: s64;
running: bool;
stopping: bool;
paused: bool;
capture_ended: bool;
mutex: Mutex;
ready: Semaphore; // signalled for each frame queued, and once when capture ends
latest: [] u8; // the stream's newest frame, BGRA: the capture thread's
frames: [RECORD_BUFFERS] Record_Frame;
free: [RECORD_BUFFERS] s32;
free_count: s32;
queue: [RECORD_BUFFERS] s32; // oldest first
queue_count: s32;
last_frame: [] u8; // the last frame queued, for record_last_frame
stats: Record_Stats;
error: string; // why it stopped by itself; heap
}
Record_Frame :: struct {
pixels: [] u8; // BGRA, rows packed
ticks: s64;
}
// Starts the stream, the encoder and the file, and starts recording. fall_back is never set: there is
// one way of taking the screen here.
record_start :: (r: *Recorder, options: Record_Options) -> ok: bool, error: string, fall_back: bool {
r.* = .{};
r.options = options;
r.options.path = copy_string(options.path);
r.options.encoder = copy_string(options.encoder);
r.options.exclude_windows = array_copy(options.exclude_windows);
if r.options.fps <= 0 r.options.fps = 30;
if !record_size_allowed(options.width, options.height)
return false, tprint("% x % is smaller than % x %", options.width, options.height, RECORD_MIN_WIDTH, RECORD_MIN_HEIGHT), false;
// Even both ways: H.264's 4:2:0 colour needs it. The region in points shrinks to match.
r.width = options.width & cast(s32) ~1;
r.height = options.height & cast(s32) ~1;
scale := options.w / options.width;
w := r.width * scale;
h := r.height * scale;
r.stats.width, r.stats.height = r.width, r.height;
init(*r.mutex, "record");
init(*r.ready);
for * r.frames it.pixels = NewArray(cast(s64) r.width * r.height * 4, u8, initialized = false);
for 0..RECORD_BUFFERS - 1 r.free[it] = cast(s32) it;
r.free_count = RECORD_BUFFERS;
r.latest = NewArray(cast(s64) r.width * r.height * 4, u8);
r.last_frame = NewArray(cast(s64) r.width * r.height * 4, u8);
// The microphone only where the stream can give it.
microphone := options.microphone && mcap_sck_microphone_supported() != 0;
if options.microphone && !microphone {
r.stats.audio_errors = copy_string("microphone: it needs macOS 15 or later");
r.audio_failed[Audio_Source_Kind.MICROPHONE] = true;
}
error_text: [512] u8;
excluded := r.options.exclude_windows;
r.stream = mcap_sck_stream_start(options.display_id, options.x, options.y, w, h, r.width, r.height, r.options.fps,
xx ifx options.cursor then 1 else 0, xx ifx options.system_audio then 1 else 0, xx ifx microphone then 1 else 0,
excluded.data, xx excluded.count, error_text.data, error_text.count);
if !r.stream {
error := tprint("ScreenCaptureKit could not record: %", to_string(error_text.data));
record_free(r);
return false, error, false;
}
r.stats.source = sprint("ScreenCaptureKit, display %", options.display_id);
r.start_ticks = ticks_now();
r.zero_ticks = r.start_ticks;
r.audio_sources[0] = .{kind = .SYSTEM, name = "System sound", open = options.system_audio};
r.audio_sources[1] = .{kind = .MICROPHONE, name = "Microphone", open = microphone};
r.audio_on = .[options.system_audio, microphone];
r.audio_open = options.system_audio || microphone || options.audio_track;
record_note_audio(r);
ok, error, tried := encoder_open(*r.encoder, r.options.path, r.width, r.height, r.options.fps, r.options.encoder, audio = r.audio_open);
if tried log("[record] Encoders that did not open: %", tried);
if !ok {
error = copy_string(error,, temp);
record_free(r);
return false, error, false;
}
r.stats.encoder = copy_string(r.encoder.name);
r.running = true;
thread_init(*r.capture_thread, record_capture_proc, name = "record capture");
r.capture_started = true;
thread_start(*r.capture_thread, r);
thread_init(*r.encoder_thread, record_encoder_proc, name = "record encoder");
r.encoder_started = true;
thread_start(*r.encoder_thread, r);
if r.audio_open {
thread_init(*r.audio_thread, record_audio_proc, name = "record audio");
r.audio_started = true;
thread_start(*r.audio_thread, r);
}
return true, "", false;
}
// Stops, finishes the file and frees everything. ok is false when recording stopped by itself on an
// error, or the file could not be finished; what was recorded up to then is in the file all the same.
record_stop :: (r: *Recorder) -> ok: bool, error: string {
if !r.running return false, "not recording";
atomic_swap(*r.stopping, true);
thread_is_done(*r.capture_thread, -1);
thread_is_done(*r.encoder_thread, -1);
if r.audio_started thread_is_done(*r.audio_thread, -1);
r.running = false;
closed, close_error := encoder_close(*r.encoder);
r.stats.seconds = cast(float64) (ticks_now() - r.start_ticks) / r.frequency;
r.stats.bytes = r.encoder.bytes;
error := ifx r.error then copy_string(r.error,, temp) else "";
record_free(r);
if error return false, error;
if !closed return false, close_error;
return true, "";
}
// Pauses or resumes; the time paused is left out of the video, sound and all.
record_pause :: (r: *Recorder, pause: bool) {
if !r.running || atomic_read(*r.paused) == pause return;
now := ticks_now();
if pause {
atomic_write(*r.pause_began, now);
atomic_swap(*r.paused, true);
} else {
atomic_add(*r.paused_ticks, now - atomic_read(*r.pause_began));
atomic_swap(*r.paused, false);
}
}
// Turns a source of sound on or off while recording; the stream is told on the audio thread. False
// when the recording has no sound track to put it in.
record_set_audio :: (r: *Recorder, kind: Audio_Source_Kind, on: bool) -> bool {
if !r.running || !r.audio_open return false;
atomic_swap(*r.audio_on[kind], on);
return true;
}
record_audio_on :: (r: *Recorder, kind: Audio_Source_Kind) -> bool {
return r.running && atomic_read(*r.audio_on[kind]);
}
// Whether a source could not be turned on (it is off again then), and why; once.
record_audio_failed :: (r: *Recorder, kind: Audio_Source_Kind) -> failed: bool, error: string {
if !r.running || !atomic_read(*r.audio_failed[kind]) return false, "";
atomic_swap(*r.audio_failed[kind], false);
lock(*r.mutex);
defer unlock(*r.mutex);
return true, copy_string(r.stats.audio_errors,, temp);
}
// Ticks on the recording's timeline: time 0 at its start, less the time paused.
record_timeline :: (r: *Recorder, ticks: s64) -> s64 {
return ticks - r.zero_ticks - atomic_read(*r.paused_ticks);
}
record_paused :: (r: *Recorder) -> bool {
return r.running && atomic_read(*r.paused);
}
record_stats :: (r: *Recorder) -> Record_Stats {
if !r.running return r.stats;
lock(*r.mutex);
defer unlock(*r.mutex);
return r.stats;
}
// The last frame queued, BGRA (temp), cursor and all.
record_last_frame :: (r: *Recorder) -> pixels: [] u8, width: s32, height: s32 {
if r.running lock(*r.mutex);
pixels := array_copy(r.last_frame,, temp);
if r.running unlock(*r.mutex);
return pixels, r.width, r.height;
}
record_failed :: (r: *Recorder) -> bool, string {
lock(*r.mutex);
defer unlock(*r.mutex);
return r.error.count > 0, r.error;
}
// The host clock, in nanoseconds: ScreenCaptureKit's sample times are on it.
ticks_now :: () -> s64 {
CLOCK_UPTIME_RAW :: 8;
return cast(s64) clock_gettime_nsec_np(CLOCK_UPTIME_RAW);
}
#scope_file
// The sources heard, for the stats: "System sound; Microphone".
record_note_audio :: (r: *Recorder) {
names: [..] string;
names.allocator = temp;
for r.audio_sources if it.open array_add(*names, it.name);
lock(*r.mutex);
free(r.stats.audio);
r.stats.audio = join(..names, "; ");
unlock(*r.mutex);
}
record_free :: (r: *Recorder) {
atomic_swap(*r.stopping, true);
if r.capture_started { thread_is_done(*r.capture_thread, -1); thread_deinit(*r.capture_thread); r.capture_started = false; }
if r.encoder_started { thread_is_done(*r.encoder_thread, -1); thread_deinit(*r.encoder_thread); r.encoder_started = false; }
if r.audio_started { thread_is_done(*r.audio_thread, -1); thread_deinit(*r.audio_thread); r.audio_started = false; }
if r.stream { mcap_sck_stream_stop(r.stream); r.stream = null; }
if r.encoder.format encoder_close(*r.encoder);
for * r.frames array_free(it.pixels);
array_free(r.latest);
array_free(r.last_frame);
r.latest, r.last_frame = .[], .[];
array_free(r.options.exclude_windows);
r.options.exclude_windows = .[];
destroy(*r.ready);
destroy(*r.mutex);
free(r.error);
r.error = "";
}
// Takes the stream's frames as they come, and sends one when its turn is due: at most fps a second
// (a tenth of a frame early counts as on time), and one a second while nothing changes.
record_capture_proc :: (thread: *Thread) -> s64 {
r := cast(*Recorder) thread.data;
period := r.frequency / r.options.fps;
early := period / 10;
turn := ticks_now();
last_emit: s64 = -r.frequency;
have_image, changed := false, false;
frame_time: s64;
was_paused := false;
error_text: [512] u8;
while !atomic_read(*r.stopping) {
now := ticks_now();
due := ifx changed then turn - early else last_emit + r.frequency;
wait := cast(s32) clamp((due - now) * 1000 / r.frequency, 1, 100);
shown: s64;
got := mcap_sck_next_frame(r.stream, r.latest.data, *shown, wait, error_text.data, error_text.count);
if got < 0 {
record_fail(r, tprint("ScreenCaptureKit stopped: %", to_string(error_text.data)));
break;
}
if got > 0 {
have_image, changed = true, true;
frame_time = shown;
lock(*r.mutex); r.stats.captured += 1; unlock(*r.mutex);
}
now = ticks_now();
if atomic_read(*r.paused) {
was_paused = true;
lock(*r.mutex); r.stats.paused_seconds = cast(float64) (atomic_read(*r.paused_ticks) + now - atomic_read(*r.pause_began)) / r.frequency; unlock(*r.mutex);
continue;
}
if was_paused {
was_paused = false;
changed = true;
turn = now;
frame_time = now; // the screen as it is, at the moment of resuming
lock(*r.mutex); r.stats.paused_seconds = cast(float64) atomic_read(*r.paused_ticks) / r.frequency; unlock(*r.mutex);
}
if !have_image continue;
repeat := !changed && now - last_emit >= r.frequency;
if !(changed && now >= turn - early) && !repeat continue;
turn = max(turn + period, now - period);
record_queue_frame(r, record_timeline(r, ifx repeat then now else frame_time));
last_emit = now;
changed = false;
lock(*r.mutex);
r.stats.emitted += 1;
if repeat r.stats.repeats += 1;
unlock(*r.mutex);
}
atomic_swap(*r.capture_ended, true);
signal(*r.ready);
return 0;
}
// The newest frame into a free buffer, queued for the encoder; the oldest waiting dropped when every
// buffer is taken.
record_queue_frame :: (r: *Recorder, ticks: s64) {
lock(*r.mutex);
if !r.free_count {
r.free[r.free_count] = r.queue[0];
r.free_count += 1;
for 1..r.queue_count - 1 r.queue[it - 1] = r.queue[it];
r.queue_count -= 1;
r.stats.dropped += 1;
}
r.free_count -= 1;
index := r.free[r.free_count];
unlock(*r.mutex);
frame := *r.frames[index];
memcpy(frame.pixels.data, r.latest.data, frame.pixels.count);
frame.ticks = ticks;
lock(*r.mutex);
r.queue[r.queue_count] = index;
r.queue_count += 1;
memcpy(r.last_frame.data, frame.pixels.data, frame.pixels.count);
unlock(*r.mutex);
signal(*r.ready);
}
record_encoder_proc :: (thread: *Thread) -> s64 {
r := cast(*Recorder) thread.data;
while true {
wait_for(*r.ready, 100);
lock(*r.mutex);
index: s32 = -1;
if r.queue_count {
index = r.queue[0];
for 1..r.queue_count - 1 r.queue[it - 1] = r.queue[it];
r.queue_count -= 1;
}
unlock(*r.mutex);
if index < 0 {
if atomic_read(*r.capture_ended) break;
continue;
}
frame := *r.frames[index];
pts := max(frame.ticks, 0) * 90000 / r.frequency; // on the timeline already
ok, error := encoder_write(*r.encoder, frame.pixels.data, pts);
lock(*r.mutex);
r.free[r.free_count] = index;
r.free_count += 1;
if ok r.stats.written += 1;
r.stats.bytes = r.encoder.bytes;
unlock(*r.mutex);
if !ok {
record_fail(r, error);
while !atomic_read(*r.capture_ended) {
wait_for(*r.ready, 100);
lock(*r.mutex);
for 0..r.queue_count - 1 { r.free[r.free_count] = r.queue[it]; r.free_count += 1; }
r.queue_count = 0;
unlock(*r.mutex);
}
break;
}
}
return 0;
}
// Every 10 ms: the sources turned on or off as asked, their packets into the mix, and what every
// source has covered encoded.
record_audio_proc :: (thread: *Thread) -> s64 {
r := cast(*Recorder) thread.data;
if !r.encoder.audio return 0;
mix: Audio_Mix;
mix.frame_size = r.encoder.audio_frame_size;
defer array_free(mix.samples);
MAX_FRAMES :: 4096;
samples := NewArray(MAX_FRAMES * AUDIO_CHANNELS, float);
defer array_free(samples);
error_text: [512] u8;
while true {
stopping := atomic_read(*r.stopping);
// The stream's sources as the toolbar wants them.
want := r.audio_on;
for * want it.* = atomic_read(it);
if !stopping && (want[0] != r.audio_sources[0].open || want[1] != r.audio_sources[1].open) {
if want[1] && !r.audio_sources[1].open && !mcap_sck_microphone_supported() {
atomic_swap(*r.audio_on[Audio_Source_Kind.MICROPHONE], false);
lock(*r.mutex); free(r.stats.audio_errors); r.stats.audio_errors = copy_string("microphone: it needs macOS 15 or later"); unlock(*r.mutex);
atomic_swap(*r.audio_failed[Audio_Source_Kind.MICROPHONE], true);
want[1] = false;
}
if mcap_sck_stream_set_audio(r.stream, xx ifx want[0] then 1 else 0, xx ifx want[1] then 1 else 0, error_text.data, error_text.count) {
for 0..1 {
if want[it] && !r.audio_sources[it].open mix.started[it] = false;
r.audio_sources[it].open = want[it];
}
record_note_audio(r);
log("[record] Sound now: %.", ifx r.stats.audio then r.stats.audio else "none");
} else {
why := to_string(error_text.data);
log_error("[record] The sound could not be changed: %", why);
for 0..1 if want[it] && !r.audio_sources[it].open {
atomic_swap(*r.audio_on[it], false);
lock(*r.mutex); free(r.stats.audio_errors); r.stats.audio_errors = sprint("% could not be turned on: %", r.audio_sources[it].name, why); unlock(*r.mutex);
atomic_swap(*r.audio_failed[it], true);
}
}
}
for * source, index: r.audio_sources {
if !source.open continue;
while true {
shown: s64;
frames := mcap_sck_next_audio(r.stream, xx index, samples.data, MAX_FRAMES, *shown);
if frames <= 0 break;
if atomic_read(*r.paused) continue;
packet := Audio_Packet.{samples = array_view(samples, 0, frames * AUDIO_CHANNELS), frames = frames, ticks = shown};
audio_mix_add(*mix, r, index, packet);
}
}
ok, error := audio_mix_encode(*mix, r, final = stopping);
if !ok { record_fail(r, error); break; }
if stopping break;
reset_temporary_storage();
sleep_milliseconds(10);
}
return 0;
}
record_fail :: (r: *Recorder, error: string) {
log_error("[record] %", error);
lock(*r.mutex);
if !r.error r.error = copy_string(error);
unlock(*r.mutex);
atomic_swap(*r.stopping, true);
}
#scope_module
// The shim (macos/capture.h).
Mcap_Sck_Stream :: struct { _: u8; }
mcap_capture :: #library,no_dll "macos/capture";
mcap_sck_stream_start :: (display_id: u32, x: float64, y: float64, w: float64, h: float64, width: s32, height: s32, fps: s32, show_cursor: s32,
system_audio: s32, microphone: s32, excluded: *u32, excluded_count: s32, error: *u8, error_size: s32) -> *Mcap_Sck_Stream #foreign mcap_capture;
mcap_sck_next_frame :: (s: *Mcap_Sck_Stream, pixels: *u8, host_time_ns: *s64, timeout_ms: s32, error: *u8, error_size: s32) -> s32 #foreign mcap_capture;
mcap_sck_next_audio :: (s: *Mcap_Sck_Stream, source: s32, samples: *float, max_frames: s32, host_time_ns: *s64) -> s32 #foreign mcap_capture;
mcap_sck_stream_set_audio :: (s: *Mcap_Sck_Stream, system_audio: s32, microphone: s32, error: *u8, error_size: s32) -> s32 #foreign mcap_capture;
mcap_sck_microphone_supported :: () -> s32 #foreign mcap_capture;
mcap_sck_stream_stop :: (s: *Mcap_Sck_Stream) #foreign mcap_capture;
ScreenCaptureKit_Framework :: #library,system,link_always "ScreenCaptureKit";
CoreMedia_Framework :: #library,system,link_always "CoreMedia";
CoreVideo_Framework :: #library,system,link_always "CoreVideo";
clock_gettime_nsec_np :: (clock: s32) -> u64 #foreign libc_for_clock;
libc_for_clock :: #library,system "libc";
Reference in New Issue View Git Blame Copy Permalink