mcap/modules/Snagx/module.jai

// Snagx: Snagit's .snagx captures (ZIP archives of JSON and PNG) as .mcap files, as SnagitConverter's
// `snagit-converter mcap` makes them, byte for byte: the JSON as Capture Text, the page image as the
// original, thumbnail.png as the rendered image, and a preview. Copied from
// D:\McapIndex\SnagitConverter (src/snagx.jai, src/main.jai, src/json_convert.jai), reading only.

Archive_Entry :: struct {
  name: string;
  data: string;   // inflated
}

SNAGX_RENDERED_NAME :: "thumbnail.png";

// A .snagx's bytes as an .mcap (its strings temp), checked by reading the serialized bytes back,
// which are returned too. On failure, why.
snagx_to_mcap :: (data: string) -> m: Mcap, bytes: [] u8, ok: bool, error: string {
  entries, error := snagx_read_entries_from_memory(data);
  if error  return .{}, .[], false, error;
  m, bytes, ok, mcap_error := snagx_mcap_from_entries(entries);
  return m, bytes, ok, mcap_error;
}

// The entries of a .snagx, in archive order. Temp. On failure, the entries are empty and the error says why.
snagx_read_entries :: (path: string) -> [..] Archive_Entry, error: string {
  data, ok := read_entire_file(path, log_errors = false,, temp);
  if !ok {
    entries: [..] Archive_Entry;
    return entries, "cannot be read";
  }
  entries, error := snagx_read_entries_from_memory(data);
  return entries, error;
}

snagx_read_entries_from_memory :: (data: string) -> [..] Archive_Entry, error: string {
  entries: [..] Archive_Entry;
  entries.allocator = temp;
  if data.count < EOCD_SIZE  return entries, "is too short to be a ZIP archive";

  // The end of central directory record is the last thing in the file, followed only by a comment
  // of up to 65535 bytes.
  tail_start := max(0, data.count - EOCD_SIZE - 65535);
  eocd := -1;
  i := data.count - EOCD_SIZE;
  while i >= tail_start {
    if zip_u32(data, i) == EOCD_SIGNATURE { eocd = i; break; }
    i -= 1;
  }
  if eocd < 0  return entries, "is not a ZIP archive (no end of central directory)";

  entry_count := zip_u16(data, eocd + 10);
  cd_size     := cast(s64) zip_u32(data, eocd + 12);
  cd_offset   := cast(s64) zip_u32(data, eocd + 16);
  if entry_count == 0xFFFF || cd_offset == 0xFFFFFFFF  return entries, "is a ZIP64 archive, which is not supported";
  if cd_offset + cd_size > data.count  return entries, "has a central directory past its end";
  cd := slice(data, cd_offset, cd_size);

  at := 0;
  for 1..entry_count {
    if at + CD_ENTRY_SIZE > cd.count || zip_u32(cd, at) != CD_ENTRY_SIGNATURE  return entries, "has a damaged central directory";
    method          := zip_u16(cd, at + 10);
    compressed_size := cast(s64) zip_u32(cd, at + 20);
    size            := cast(s64) zip_u32(cd, at + 24);
    name_length     := zip_u16(cd, at + 28);
    extra_length    := zip_u16(cd, at + 30);
    comment_length  := zip_u16(cd, at + 32);
    local_offset    := cast(s64) zip_u32(cd, at + 42);
    if at + CD_ENTRY_SIZE + name_length > cd.count  return entries, "has a damaged central directory";
    name := slice(cd, at + CD_ENTRY_SIZE, name_length);
    at += CD_ENTRY_SIZE + name_length + extra_length + comment_length;
    if size > MAX_ENTRY_SIZE  return entries, tprint("has a % byte entry %, too large", size, name);

    // The data starts after the local header, whose name and extra field lengths can differ from
    // the central directory's.
    if local_offset + LOCAL_HEADER_SIZE > data.count || zip_u32(data, local_offset) != LOCAL_HEADER_SIGNATURE  return entries, tprint("has a damaged local header for %", name);
    data_offset := local_offset + LOCAL_HEADER_SIZE + zip_u16(data, local_offset + 26) + zip_u16(data, local_offset + 28);
    if data_offset + compressed_size > data.count  return entries, tprint("cannot read entry %", name);
    compressed := slice(data, data_offset, compressed_size);

    text: string;
    if method == 0 {
      if compressed_size != size  return entries, tprint("has a stored entry % whose sizes differ", name);
      text = compressed;
    } else if method == 8 {
      text = talloc_string(size);
      decoded := stbi_zlib_decode_noheader_buffer(text.data, cast(s32) size, compressed.data, cast(s32) compressed_size);
      if decoded != cast(s32) size  return entries, tprint("has an entry % that does not inflate to its % bytes", name, size);
    } else {
      return entries, tprint("has an entry % compressed with method %, which is not supported", name, method);
    }
    array_add(*entries, .{copy_string(name,, temp), text});
  }
  return entries, "";
}

// A capture's entries as an .mcap: its strings and bytes temp, checked by reading the bytes back.
snagx_mcap_from_entries :: (entries: [] Archive_Entry) -> m: Mcap, bytes: [] u8, ok: bool, error: string {
  m: Mcap;
  json_entries: [..] Archive_Entry;
  json_entries.allocator = temp;
  original: *Archive_Entry;
  rendered: *Archive_Entry;
  for * entries {
    if ends_with_nocase(it.name, ".json")             array_add(*json_entries, it.*);
    else if it.name == SNAGX_RENDERED_NAME            rendered = it;
    else if ends_with_nocase(it.name, ".backup.png")  {}   // the image before an edit: left out
    else if ends_with_nocase(it.name, ".png") {
      if original  return m, .[], false, tprint("has two page images, % and %; an .mcap holds one page", original.name, it.name);
      original = it;
    } else {
      return m, .[], false, tprint("has an entry % that an .mcap has no place for", it.name);
    }
  }
  if !original  return m, .[], false, "has no page image";

  text, text_ok, error := capture_text_from_json(json_entries);
  if !text_ok  return m, .[], false, error;

  m.text          = text;
  m.original_name = original.name;
  m.original      = cast([] u8) original.data;
  if !rendered                              m.rendered_kind = .NONE;
  else if rendered.data == original.data    m.rendered_kind = .SAME_AS_ORIGINAL;
  else { m.rendered_kind = .OWN; m.rendered = cast([] u8) rendered.data; }

  preview_ok: bool;
  m.preview, preview_ok, error = snagx_preview(mcap_rendered(m));
  if !preview_ok  return m, .[], false, error;

  push_allocator(temp);
  serializer, serialized, serialize_error := mcap_serialize(m);
  if !serialized  return m, .[], false, serialize_error;
  bytes := cast([] u8) string.{serializer.count, serializer.data};

  // The check: the bytes read back as exactly what went in.
  back, parsed, parse_error := mcap_parse(bytes);
  if !parsed  return m, .[], false, tprint("the .mcap it was written as does not read back: %", parse_error);
  if back.text != m.text  return m, .[], false, "the text came back different";
  if back.original_name != original.name || cast(string) back.original != original.data  return m, .[], false, "the original image came back different";
  if back.rendered_kind != m.rendered_kind || cast(string) mcap_rendered(back) != cast(string) mcap_rendered(m)  return m, .[], false, "the rendered image came back different";
  width, height, size_ok := png_size(back.preview.png);
  if back.preview.png.count && (!size_ok || width != back.preview.width || height != back.preview.height)  return m, .[], false, "the preview's stated size is not its own";
  return m, bytes, true, "";
}

// The preview, as SnagitConverter makes it: the PNG itself when it is small enough, else scaled down.
snagx_preview :: (png: [] u8) -> Mcap_Image, ok: bool, error: string {
  preview: Mcap_Image;
  if !png.count  return preview, true, "";
  image, decoded, error := image_decode(png, .PNG, orient = false);
  if !decoded  return preview, false, tprint("its image cannot be decoded: %", error);
  defer image_free(*image);
  if max(image.width, image.height) <= MCAP_PREVIEW_MAX_SIDE {
    preview = .{width = cast(u16) image.width, height = cast(u16) image.height, png = png};
    return preview, true, "";
  }
  scaled, w, h, scaled_ok, scale_error := image_preview_png(image, MCAP_PREVIEW_MAX_SIDE);
  if !scaled_ok  return preview, false, scale_error;
  preview = .{width = cast(u16) w, height = cast(u16) h, png = scaled};
  return preview, true, "";
}

// ---------------------------------------------------------------- JSON to Capture Text

// The JSON entries as Capture Text (temp), checked: the text reads back without a mistake, holds
// exactly the JSON, and writes out the same.
capture_text_from_json :: (json_entries: [] Archive_Entry) -> text: string, ok: bool, error: string {
  // jaison allocates from the context; its values only need to last for this capture.
  push_allocator(temp);
  parsed: [..] Parsed_Entry;
  for json_entries {
    success, json := json_parse_string(it.data);
    if !success  return "", false, tprint("% is not valid JSON", it.name);
    if json.type != .OBJECT  return "", false, tprint("% holds a JSON %, not an object", it.name, json.type);
    array_add(*parsed, .{name = it.name, json = json});
  }
  if !parsed.count  return "", false, "holds no JSON entries";
  order_entries(parsed);

  doc: Capture_Document;
  capture_document_init(*doc, temp);
  for parsed {
    root, ok, error := capture_from_json(*doc, it.json);
    if !ok  return "", false, tprint("%: %", it.name, error);
    file := capture_add_file(*doc, it.name);
    file.root = root;
  }
  text := capture_text_write(*doc);

  // Comparing with the JSON cannot catch what the JSON reader itself got wrong, as with emoji
  // before jaison joined surrogate pairs; text that is not UTF-8 is a sign of it.
  if !is_valid_utf8(text)  return "", false, "the text is not valid UTF-8 (a lone surrogate escape in the JSON?)";
  back: Capture_Document;
  capture_document_init(*back, temp);
  if !capture_text_parse(*back, "the converted text", text) || back.mistakes  return "", false, "the text it was written as does not read back cleanly (see above)";
  if back.files.count != parsed.count  return "", false, tprint("% JSON files went in, % came back", parsed.count, back.files.count);
  for parsed {
    file := capture_find_file(*back, it.name);
    if !file  return "", false, tprint("% did not come back", it.name);
    if !capture_equals_json(file.root, it.json)  return "", false, tprint("% came back with different data", it.name);
  }
  if capture_text_write(*back) != text  return "", false, "the text read back writes out differently";
  return text, true, "";
}

#scope_file

Parsed_Entry :: struct {
  name: string;
  json: JSON_Value;
  rank: s64;   // where it goes in the text
}

// index.json, metadata.json, the pages in the order index.json lists them, then any other files by name.
order_entries :: (parsed: [] Parsed_Entry) {
  pages: [] JSON_Value;
  for parsed {
    if it.name != "index.json"  continue;
    found, value := table_find(it.json.object, "Pages");
    if found && value.type == .ARRAY  pages = value.array;
  }
  for * parsed {
    if it.name == "index.json"         { it.rank = 0; continue; }
    if it.name == "metadata.json"      { it.rank = 1; continue; }
    it.rank = 2 + pages.count;   // after every page
    for page: pages {
      if page.type == .STRING && page.str == it.name { it.rank = 2 + it_index; break; }
    }
  }
  quick_sort(parsed, (a: Parsed_Entry, b: Parsed_Entry) -> s64 {
    if a.rank != b.rank  return ifx a.rank < b.rank then -1 else 1;
    return compare(a.name, b.name);
  });
}

// Object keys are sorted, as Snagit writes them; jaison keeps them in a hash table, so their order in
// the file is gone by now anyway.
capture_from_json :: (doc: *Capture_Document, json: JSON_Value) -> Capture_Value, ok: bool, error: string {
  if #complete json.type == {
    case .NULL;     return capture_null(), true, "";
    case .BOOLEAN;  return capture_boolean(json.boolean), true, "";
    case .NUMBER;   return capture_number(json.number), true, "";
    case .STRING;   return capture_string(doc, json.str), true, "";

    case .ARRAY;
      result := capture_array(doc);
      for json.array {
        item, ok, error := capture_from_json(doc, it);
        if !ok  return result, false, error;
        array_add(*result.items, item);
      }
      return result, true, "";

    case .OBJECT;
      result := capture_object(doc);
      keys: [..] string;
      keys.allocator = temp;
      for value, key: json.object.*  array_add(*keys, key);
      intro_sort(keys, compare);
      for keys {
        if !capture_key_is_writable(it)  return result, false, tprint("the key \"%\" holds a control character, which Capture Text cannot write", it);
        _, json_value := table_find(json.object, it);
        value, ok, error := capture_from_json(doc, json_value);
        if !ok  return result, false, error;
        capture_object_add(doc, *result, it, value);
      }
      return result, true, "";
  }
  return .{}, false, "unknown JSON type";
}

// Whether value holds exactly what json does: the same types, keys, strings byte for byte, and
// numbers bit for bit.
capture_equals_json :: (value: Capture_Value, json: JSON_Value) -> bool {
  if #complete json.type == {
    case .NULL;     return value.kind == .NULL;
    case .BOOLEAN;  return value.kind == .BOOLEAN && value.boolean == json.boolean;
    case .NUMBER;
      number := json.number;
      return value.kind == .NUMBER && (cast(*u64) *value.number).* == (cast(*u64) *number).*;
    case .STRING;   return value.kind == .STRING && value.text == json.str;

    case .ARRAY;
      if value.kind != .ARRAY || value.items.count != json.array.count  return false;
      for json.array  if !capture_equals_json(value.items[it_index], it)  return false;
      return true;

    case .OBJECT;
      if value.kind != .OBJECT || value.fields.count != json.object.count  return false;
      for json_value, key: json.object.* {
        found := capture_object_find(*value, key);
        if !found || !capture_equals_json(found.*, json_value)  return false;
      }
      return true;
  }
  return false;
}

EOCD_SIGNATURE         :: 0x06054b50;
CD_ENTRY_SIGNATURE     :: 0x02014b50;
LOCAL_HEADER_SIGNATURE :: 0x04034b50;
EOCD_SIZE         :: 22;
CD_ENTRY_SIZE     :: 46;
LOCAL_HEADER_SIZE :: 30;
MAX_ENTRY_SIZE    :: 256 * 1024 * 1024;

// Each byte's cast is parenthesized: a bare cast(u32) applies to the whole | expression, which is then
// worked out in u8 and keeps only the first byte.
zip_u16 :: (s: string, at: s64) -> u16 {
  return (cast(u16) s[at]) | ((cast(u16) s[at + 1]) << 8);
}

zip_u32 :: (s: string, at: s64) -> u32 {
  return (cast(u32) s[at]) | ((cast(u32) s[at + 1]) << 8) | ((cast(u32) s[at + 2]) << 16) | ((cast(u32) s[at + 3]) << 24);
}

#import "Basic";
#import "String";
#import "File";
#import "Sort";
#import "IntroSort";
#import "Hash_Table";
#import "stb_image";
#import "jaison";
#import,dir "../jaison/unicode_utils";   // is_valid_utf8
#import "Capture_Text";
#import "Capture_File";
#import "Image_Decode";