MusaMahmood/mcap
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

[0.1.27] macOS phase 3: HEIC, AVIF, WebP, TIFF and RAW through ImageIO

Browse Source
This commit is contained in:
Musa Mahmood 2026-10-16 15:36:21 -04:00
parent 5571c21d0b
commit a66928b223
8 changed files with 214 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
README.md
View File

@ -262,9 +262,10 @@ What becomes of each file:
way up (by its EXIF orientation), and where it came from: its file name (searched by `title:`), the
camera or phone (`app:`), and the date it was taken, from its EXIF, else a date in its name
(`IMG_20240512_101530`, `Screenshot 2024-05-12 at 10.15.30`), else the file's time. It is named
`2024-05-12_10-15-30_1a2b3c4d.mcap`, the date and the start of its hash. HEIC, AVIF, WebP and RAW
are read by Windows' own codecs: the HEIF, AV1, WebP and Raw Image Extensions from the Microsoft
Store.
`2024-05-12_10-15-30_1a2b3c4d.mcap`, the date and the start of its hash. HEIC, AVIF, WebP, TIFF and RAW
are read by the system's own codecs: on Windows the HEIF, AV1, WebP and Raw Image Extensions from
the Microsoft Store, and on macOS ImageIO, which has them all built in. Linux reads only PNG, JPEG,
GIF, BMP, PSD and TGA.
- **Animated GIFs** are images; their thumbnail is marked **GIF**, and the Capture window says how
long they run and opens them in the default viewer.
- **Videos** (MP4, MOV, M4V, 3GP, MKV, WebM, AVI, WMV, MPEG, MTS and M2TS, FLV), read with FFmpeg
@ -455,7 +456,8 @@ converted copy.
library index up to date (hashing the image in any `.mcap` it has not seen), then imports on
several threads; captures written are handed to the grid each frame.
- `modules/Image_Decode`: any image file's bytes to pixels, the right way up: stb_image for PNG,
JPEG, GIF, BMP, PSD and TGA, Windows Imaging Component for the rest; EXIF (from JPEG, PNG, WebP,
JPEG, GIF, BMP, PSD and TGA, and for the rest Windows Imaging Component (`wic.jai`) or macOS's
ImageIO (`imageio.jai`); EXIF (from JPEG, PNG, WebP,
HEIF and TIFF); previews; and videos' frames and what their containers say, with FFmpeg
(`video.jai`), and how many frames a GIF has. `bin\image-probe FILE...` (`jai first.jai -
image_probe`) shows what it makes of files.

2
check_targets.jai
View File

@ -49,7 +49,7 @@ APP_NAME :: "mcap";
APP_FOLDER_NAME :: "mcap";
GIT_BRANCH :: "check";
GIT_HASH :: "-";
VERSION :: "0.1.26";
VERSION :: "0.1.27";
BUILD_DATE :: "-";
DEBUG :: true;
COMPILE_TESTS :: true;

2
first.jai
View File

@ -8,7 +8,7 @@ APP_NAME :: "mcap";
APP_FOLDER_NAME :: "mcap";
APP_BUNDLE_IDENTIFIER :: "com.example.mcap";
VERSION :: "0.1.26";
VERSION :: "0.1.27";
// Where everything goes: bin/, or bin-headless/ for builds made with no_crash_dialog, so automated
// builds never write over the ones you run by hand.
build_directory := "bin";

2
modules/Image_Decode/exif.jai
View File

@ -258,7 +258,7 @@ heif_info :: (bytes: [] u8) -> Heif_Info {
box := properties[index - 1];
type := cast(string) slice_bytes(box, 4, 4);
if type == "ispe" && box.count >= 20 { info.width = be32(box, 12); info.height = be32(box, 16); }
if type == "irot" && box.count >= 9 info.rotation = (box[8] & 3) * 90;
if type == "irot" && box.count >= 9 info.rotation = cast(s64) (box[8] & 3) * 90; // as a u8, 270 wrapped to 14
if type == "imir" && box.count >= 9 info.mirror = box[8] & 1;
}
}

149
modules/Image_Decode/imageio.jai Normal file
View File

@ -0,0 +1,149 @@
// ImageIO, macOS's own codecs: HEIC and AVIF, WebP, TIFF and the camera RAW formats, as WIC reads them
// on Windows. Only the few Core Foundation, ImageIO and Core Graphics calls used are declared.
// The primary image in bytes, as non-premultiplied RGBA (default allocator), as stored: not turned.
// orientation is what ImageIO says the image's orientation is (1-8, from a HEIF's irot and imir, or
// its EXIF), for image_decode to apply where exif.jai cannot read it.
//
// The pixels keep the image's own RGB color space, as WIC and stb_image leave them; only gray, CMYK
// and the like are converted, to sRGB.
imageio_decode :: (bytes: [] u8) -> Image, ok: bool, error: string, orientation: s64 {
image: Image;
// Not copied: the source is let go of before this returns.
data := CFDataCreateWithBytesNoCopy(null, bytes.data, bytes.count, kCFAllocatorNull);
if !data return image, false, "ImageIO could not take the bytes", 1;
defer CFRelease(data);
source := CGImageSourceCreateWithData(data, null);
if !source return image, false, "ImageIO cannot read it", 1;
defer CFRelease(source);
if CGImageSourceGetCount(source) < 1 return image, false, "ImageIO finds no image in it", 1;
index := CGImageSourceGetPrimaryImageIndex(source);
cg := CGImageSourceCreateImageAtIndex(source, index, null);
if !cg return image, false, "ImageIO cannot decode it", 1;
defer CGImageRelease(cg);
orientation: s32 = 1;
properties := CGImageSourceCopyPropertiesAtIndex(source, index, null);
if properties {
number := CFDictionaryGetValue(properties, kCGImagePropertyOrientation);
if number CFNumberGetValue(number, kCFNumberSInt32Type, *orientation);
CFRelease(properties);
}
w := CGImageGetWidth(cg);
h := CGImageGetHeight(cg);
if !w || !h return image, false, "ImageIO gives it no size", orientation;
if w * h * 4 > U32_MAX return image, false, tprint("it is % x %, too large to decode in one piece", w, h), orientation;
// Opaque images are drawn as 8-bit RGB with the fourth byte left out, so their pixels are exact.
// Others are drawn premultiplied, the only RGBA a bitmap context takes, at 16 bits a channel so
// that dividing back gives the 8-bit colors again, even where alpha is small.
alpha_info := CGImageGetAlphaInfo(cg);
opaque := alpha_info == kCGImageAlphaNone || alpha_info == kCGImageAlphaNoneSkipLast || alpha_info == kCGImageAlphaNoneSkipFirst;
bits: u64 = 16;
bitmap_info: u32 = kCGBitmapByteOrder16Little | kCGImageAlphaPremultipliedLast;
if opaque { bits = 8; bitmap_info = kCGBitmapByteOrder32Big | kCGImageAlphaNoneSkipLast; }
row_bytes := w * bits / 2;
count := cast(s64) (w * h);
pixels := cast(*u8) alloc(count * 4,, context.default_allocator);
drawn := pixels;
if !opaque drawn = cast(*u8) alloc(count * 8,, context.default_allocator);
defer if drawn != pixels free(drawn,, context.default_allocator);
memset(drawn, 0, cast(s64) (row_bytes * h));
space := CGImageGetColorSpace(cg);
own_space := space && CGColorSpaceGetModel(space) == kCGColorSpaceModelRGB;
bitmap: CGContextRef;
if own_space bitmap = CGBitmapContextCreate(drawn, w, h, bits, row_bytes, space, bitmap_info);
if !bitmap {
// Not RGB, or a space a bitmap context cannot draw in (an extended range one): converted to sRGB.
srgb := CGColorSpaceCreateWithName(kCGColorSpaceSRGB);
defer CGColorSpaceRelease(srgb);
bitmap = CGBitmapContextCreate(drawn, w, h, bits, row_bytes, srgb, bitmap_info);
}
if !bitmap {
free(pixels,, context.default_allocator);
return image, false, "Core Graphics could not make a bitmap to draw it in", orientation;
}
CGContextSetBlendMode(bitmap, kCGBlendModeCopy);
CGContextDrawImage(bitmap, .{.{0, 0}, .{cast(float64) w, cast(float64) h}}, cg);
CGContextRelease(bitmap);
if opaque {
for 0..count - 1 pixels[it * 4 + 3] = 255;
} else {
wide := cast(*u16) drawn;
for 0..count - 1 {
in := wide + it * 4;
out := pixels + it * 4;
a := cast(u32) in[3];
out[3] = cast(u8) ((a * 255 + 32767) / 65535);
if a == 0 { out[0], out[1], out[2] = 0; continue; }
for c: 0..2 out[c] = cast(u8) min(255, (cast(u32) in[c] * 255 + a / 2) / a);
}
}
image = .{pixels = pixels, width = cast(s32) w, height = cast(s32) h};
return image, true, "", orientation;
}
#scope_file
CFTypeRef :: *void;
CFDataRef :: *void;
CFDictionaryRef :: *void;
CFStringRef :: *void;
CFAllocatorRef :: *void;
CGImageSourceRef :: *void;
CGImageRef :: *void;
CGColorSpaceRef :: *void;
CGContextRef :: *void;
CGPoint :: struct { x, y: float64; }
CGSize :: struct { width, height: float64; }
CGRect :: struct { origin: CGPoint; size: CGSize; }
kCFNumberSInt32Type :: 3;
kCGColorSpaceModelRGB :: 1;
kCGBlendModeCopy :: 17;
kCGImageAlphaNone :: 0;
kCGImageAlphaPremultipliedLast :: 1;
kCGImageAlphaNoneSkipLast :: 5;
kCGImageAlphaNoneSkipFirst :: 6;
kCGBitmapByteOrder16Little :: 1 << 12;
kCGBitmapByteOrder32Big :: 4 << 12;
CoreFoundation :: #library,system "CoreFoundation";
ImageIO :: #library,system "ImageIO";
CoreGraphics :: #library,system "CoreGraphics";
kCFAllocatorNull: CFAllocatorRef #elsewhere CoreFoundation;
kCGImagePropertyOrientation: CFStringRef #elsewhere ImageIO;
kCGColorSpaceSRGB: CFStringRef #elsewhere CoreGraphics;
CFDataCreateWithBytesNoCopy :: (allocator: CFAllocatorRef, bytes: *u8, length: s64, deallocator: CFAllocatorRef) -> CFDataRef #foreign CoreFoundation;
CFRelease :: (object: CFTypeRef) #foreign CoreFoundation;
CFDictionaryGetValue :: (dictionary: CFDictionaryRef, key: *void) -> *void #foreign CoreFoundation;
CFNumberGetValue :: (number: *void, type: s64, value: *void) -> u8 #foreign CoreFoundation;
CGImageSourceCreateWithData :: (data: CFDataRef, options: CFDictionaryRef) -> CGImageSourceRef #foreign ImageIO;
CGImageSourceGetCount :: (source: CGImageSourceRef) -> u64 #foreign ImageIO;
CGImageSourceGetPrimaryImageIndex :: (source: CGImageSourceRef) -> u64 #foreign ImageIO;
CGImageSourceCreateImageAtIndex :: (source: CGImageSourceRef, index: u64, options: CFDictionaryRef) -> CGImageRef #foreign ImageIO;
CGImageSourceCopyPropertiesAtIndex :: (source: CGImageSourceRef, index: u64, options: CFDictionaryRef) -> CFDictionaryRef #foreign ImageIO;
CGImageGetWidth :: (image: CGImageRef) -> u64 #foreign CoreGraphics;
CGImageGetHeight :: (image: CGImageRef) -> u64 #foreign CoreGraphics;
CGImageGetAlphaInfo :: (image: CGImageRef) -> u32 #foreign CoreGraphics;
CGImageGetColorSpace :: (image: CGImageRef) -> CGColorSpaceRef #foreign CoreGraphics;
CGImageRelease :: (image: CGImageRef) #foreign CoreGraphics;
CGColorSpaceGetModel :: (space: CGColorSpaceRef) -> s32 #foreign CoreGraphics;
CGColorSpaceCreateWithName :: (name: CFStringRef) -> CGColorSpaceRef #foreign CoreGraphics;
CGColorSpaceRelease :: (space: CGColorSpaceRef) #foreign CoreGraphics;
CGBitmapContextCreate :: (data: *void, width: u64, height: u64, bits_per_component: u64, bytes_per_row: u64, space: CGColorSpaceRef, bitmap_info: u32) -> CGContextRef #foreign CoreGraphics;
CGContextSetBlendMode :: (context_ref: CGContextRef, mode: s32) #foreign CoreGraphics;
CGContextDrawImage :: (context_ref: CGContextRef, rect: CGRect, image: CGImageRef) #foreign CoreGraphics;
CGContextRelease :: (context_ref: CGContextRef) #foreign CoreGraphics;

27
modules/Image_Decode/module.jai
View File

@ -1,7 +1,8 @@
// Image_Decode: any image file's bytes to RGBA pixels, the right way up, and the PNG previews .mcap
// files hold. stb_image reads PNG, JPEG, GIF, BMP, PSD and TGA; everything else (HEIC, AVIF, WebP,
// TIFF, camera RAW) goes to Windows' own codecs through WIC, which read what the installed Store
// extensions (HEIF, WebP, Raw Image) can. A photo's EXIF orientation is applied, so a phone's portrait
// TIFF, camera RAW) goes to the system's own codecs: on Windows WIC, which reads what the installed
// Store extensions (HEIF, WebP, Raw Image) can, and on macOS ImageIO (imageio.jai). Linux reads only
// what stb_image does. A photo's EXIF orientation is applied, so a phone's portrait
// photo comes out standing up.
//
// What the bytes are comes from their first bytes (image_kind), never from a file name, except for
@ -10,12 +11,13 @@
// Videos (video.jai) are told apart here too, by their first bytes or their extension; video_poster
// reads a frame from one with FFmpeg, on Windows, and gif_info counts an animated GIF's frames.
//
// Thread-safe: every call stands alone, and WIC is set up and released on the calling thread.
// Thread-safe: every call stands alone, and WIC is set up and released on the calling thread; ImageIO
// needs no setting up.
Image_Kind :: enum u8 {
UNKNOWN;
PNG; JPEG; GIF; BMP; PSD; TGA; // stb_image
WEBP; HEIF; AVIF; TIFF; RAW; // WIC
WEBP; HEIF; AVIF; TIFF; RAW; // WIC on Windows, ImageIO on macOS
VIDEO; // MP4, QuickTime, Matroska and WebM, AVI, WMV, MPEG: see video_poster
}
@ -154,6 +156,7 @@ image_decode :: (bytes: [] u8, kind := Image_Kind.UNKNOWN, orient := true) -> Im
error = to_string(stbi_failure_reason());
}
}
container_orientation := 0; // ImageIO's, for HEIF and AVIF on macOS; 0 when not known
if !decoded {
#if OS == .WINDOWS {
wic_ok: bool;
@ -163,14 +166,25 @@ image_decode :: (bytes: [] u8, kind := Image_Kind.UNKNOWN, orient := true) -> Im
if error return image, false, tprint("% (stb_image: %)", wic_error, error);
return image, false, wic_error;
}
} else #if OS == .MACOS {
imageio_ok: bool;
imageio_error: string;
image, imageio_ok, imageio_error, container_orientation = imageio_decode(bytes);
if !imageio_ok {
if error return image, false, tprint("% (stb_image: %)", imageio_error, error);
return image, false, imageio_error;
}
} else {
return image, false, ifx error then tprint("stb_image: %", error) else tprint("% images are only read on Windows", image_kind_name(kind));
return image, false, ifx error then tprint("stb_image: %", error) else tprint("% images are only read on Windows and macOS", image_kind_name(kind));
}
}
if orient {
orientation := 1;
if kind == .HEIF || kind == .AVIF {
if (kind == .HEIF || kind == .AVIF) && container_orientation {
// ImageIO gives the image as stored, and the container's irot and imir as an orientation.
orientation = container_orientation;
} else if kind == .HEIF || kind == .AVIF {
// The container's irot and imir are the orientation; the EXIF tag only repeats it. WIC applies
// them (HEIF requires a reader to); a quarter turn it left out shows as the sides not swapped.
heif := heif_info(bytes);
@ -251,6 +265,7 @@ png_size :: (png: [] u8) -> width: s64, height: s64, ok: bool {
#load "exif.jai";
#load "video.jai";
#if OS == .WINDOWS #load "wic.jai";
#if OS == .MACOS #load "imageio.jai";
#if OS == .WINDOWS #load "video_ffmpeg.jai";
#scope_module

34
src/test/import_tests.jai
View File

@ -152,6 +152,39 @@ exif_tests :: () {
expect(exif_read(cast([] u8) "\xFF\xD8\xFF\xDA").orientation == 1, "a JPEG with none is upright");
}
// heif_info: a HEIF's primary item's size, rotation and mirroring, from a meta box made here.
heif_tests :: () {
group("heif");
box :: (type: string, contents: [] u8) -> [] u8 {
out: [..] u8;
out.allocator = temp;
size := contents.count + 8;
array_add(*out, cast(u8) (size >> 24), cast(u8) (size >> 16), cast(u8) (size >> 8), cast(u8) size);
array_add(*out, ..cast([] u8) type);
array_add(*out, ..contents);
return out;
}
join :: (parts: ..[] u8) -> [] u8 {
out: [..] u8;
out.allocator = temp;
for parts array_add(*out, ..it);
return out;
}
for turns: 0..3 {
ispe := box("ispe", u8.[0,0,0,0, 0,0,0,64, 0,0,0,48]);
irot := box("irot", u8.[cast(u8) turns]);
imir := box("imir", u8.[1]);
ipco := box("ipco", join(ispe, irot, imir));
ipma := box("ipma", u8.[0,0,0,0, 0,0,0,1, 0,1, 3, 0x81, 0x82, 0x03]); // item 1: properties 1-3
pitm := box("pitm", u8.[0,0,0,0, 0,1]);
meta := box("meta", join(u8.[0,0,0,0], pitm, box("iprp", join(ipco, ipma))));
heif := heif_info(join(box("ftyp", cast([] u8) "heic\0\0\0\0mif1heic"), meta));
expect(heif.width == 64 && heif.height == 48, "irot %: the size, % x %", turns, heif.width, heif.height);
expect(heif.rotation == turns * 90, "irot %: % degrees (expected %)", turns, heif.rotation, turns * 90);
expect(heif.mirror == 1, "irot %: imir read", turns);
}
}
orientation_tests :: () {
group("orientation");
// 3 x 2, pixels numbered by where they are: 0 1 2 / 3 4 5. Each orientation's result, row by row.
@ -861,6 +894,7 @@ run_import_tests :: (builder: *String_Builder = null) -> bool {
push_allocator(context.default_allocator);
kind_tests();
exif_tests();
heif_tests();
orientation_tests();
date_tests();
folder_tests();

2
tools/image_probe.jai
View File

@ -66,7 +66,7 @@ main :: () {
start := current_time_monotonic();
image, decoded, error := image_decode(bytes, kind);
if !decoded { print(" cannot decode: %\n", error); failures += 1; continue; }
print(" decoded: % x % in % ms (%)\n", image.width, image.height, FF(to_float64_seconds(current_time_monotonic() - start) * 1000, 1), ifx image.from_stb then "stb_image" else "WIC or turned");
print(" decoded: % x % in % ms (%)\n", image.width, image.height, FF(to_float64_seconds(current_time_monotonic() - start) * 1000, 1), ifx image.from_stb then "stb_image" else "WIC, ImageIO or turned");
if out_folder {
name := path_strip_extension(path_filename(path));
full := stbi_write_png(temp_c_string(tprint("%/%.png", out_folder, name)), image.width, image.height, 4, image.pixels, image.width * 4);