mcap/modules/Image_Decode/video_ffmpeg.jai

// video_poster on Windows, with FFmpeg (modules/ffmpeg). See video.jai.

video_poster :: (path: string) -> Image, Video_Info, ok: bool, error: string {
  image: Image;
  info: Video_Info;
  av_log_set_level(AV_LOG_ERROR);   // FFmpeg's notes would go to the console

  format: *AVFormatContext;
  result := avformat_open_input(*format, temp_c_string(path), null, null);   // UTF-8 paths, on Windows too
  if result < 0  return image, info, false, av_text("FFmpeg cannot open it", result);
  defer avformat_close_input(*format);
  result = avformat_find_stream_info(format, null);
  if result < 0  return image, info, false, av_text("FFmpeg cannot find its streams", result);

  codec: *AVCodec;
  index := av_find_best_stream(format, .VIDEO, -1, -1, *codec, 0);
  if index < 0 || !codec  return image, info, false, ifx index == AVERROR_DECODER_NOT_FOUND then "FFmpeg has no decoder for its video" else "it has no video stream";
  stream := format.streams[index];
  for 0..format.nb_streams - 1  if format.streams[it].codecpar.codec_type == .AUDIO  info.has_audio = true;

  info.codec = to_string(avcodec_get_name(stream.codecpar.codec_id));
  if format.duration > 0 && format.duration != AV_NOPTS                 info.duration = cast(float64) format.duration / AV_TIME_BASE;
  else if stream.duration > 0 && stream.duration != AV_NOPTS           info.duration = cast(float64) stream.duration * rational(stream.time_base);
  if stream.avg_frame_rate.num > 0 && stream.avg_frame_rate.den > 0    info.frame_rate = rational(stream.avg_frame_rate);
  else if stream.r_frame_rate.num > 0 && stream.r_frame_rate.den > 0   info.frame_rate = rational(stream.r_frame_rate);

  side_size: u64;
  matrix := cast(*s32) av_stream_get_side_data(stream, .DISPLAYMATRIX, *side_size);
  if matrix && side_size >= 9 * size_of(s32) {
    m: [9] s32;
    memcpy(m.data, matrix, size_of(type_of(m)));
    info.rotation = display_matrix_rotation(m);
  }

  meta :: (format: *AVFormatContext, stream: *AVStream, key: string) -> string {
    for dictionary: *AVDictionary.[format.metadata, stream.metadata] {
      entry := av_dict_get(dictionary, temp_c_string(key), null, 0);
      if entry && entry.value && entry.value[0]  return copy_string(to_string(entry.value),, temp);
    }
    return "";
  }
  info.creation_date = meta(format, stream, "com.apple.quicktime.creationdate");
  info.creation_time = meta(format, stream, "creation_time");
  info.make          = meta(format, stream, "com.apple.quicktime.make");
  if !info.make  info.make = meta(format, stream, "make");
  if !info.make  info.make = meta(format, stream, "com.android.manufacturer");
  info.model         = meta(format, stream, "com.apple.quicktime.model");
  if !info.model  info.model = meta(format, stream, "model");
  if !info.model  info.model = meta(format, stream, "com.android.model");
  info.software      = meta(format, stream, "com.apple.quicktime.software");
  if !info.software  info.software = meta(format, stream, "encoder");
  info.location      = meta(format, stream, "com.apple.quicktime.location.ISO6709");
  if !info.location  info.location = meta(format, stream, "location");

  // The decoder.
  decoder := avcodec_alloc_context3(codec);
  if !decoder  return image, info, false, "FFmpeg could not make a decoder";
  defer avcodec_free_context(*decoder);
  result = avcodec_parameters_to_context(decoder, stream.codecpar);
  if result < 0  return image, info, false, av_text("FFmpeg cannot set up its decoder", result);
  decoder.thread_count = 2;   // several imports run at once
  result = avcodec_open2(decoder, codec, null);
  if result < 0  return image, info, false, av_text(tprint("FFmpeg cannot open its % decoder", info.codec), result);

  // Seek to the poster's time (to the key frame before it), then decode up to it.
  if info.duration >= 1  info.poster_time = min(info.duration / 10, VIDEO_POSTER_MAX_SECONDS);
  target: s64 = AV_NOPTS;
  if info.poster_time > 0 && stream.time_base.num > 0 {
    target = cast(s64) (info.poster_time / rational(stream.time_base));
    if stream.start_time != AV_NOPTS  target += stream.start_time;
    if av_seek_frame(format, index, target, AVSEEK_FLAG_BACKWARD) < 0 {
      // Cannot seek (some AVI, MPEG): the first frame, then.
      target = AV_NOPTS;
      info.poster_time = 0;
      av_seek_frame(format, index, 0, AVSEEK_FLAG_BACKWARD);
    }
    avcodec_flush_buffers(decoder);
  }

  packet := av_packet_alloc();
  defer av_packet_free(*packet);
  frame := av_frame_alloc();
  defer av_frame_free(*frame);
  got := av_frame_alloc();      // the last frame decoded: the poster, once it reaches the target
  defer av_frame_free(*got);
  have := false;
  packets := 0;
  ended := false;   // the end of the file was sent to the decoder
  while true {
    // The frames the decoder has, until it wants another packet.
    if avcodec_receive_frame(decoder, frame) == 0 {
      av_frame_unref(got);
      av_frame_move_ref(got, frame);
      have = true;
      if target == AV_NOPTS || got.best_effort_timestamp == AV_NOPTS || got.best_effort_timestamp >= target  break;
      continue;
    }
    if ended || packets >= VIDEO_MAX_PACKETS  break;

    // The next packet of the video.
    while true {
      if av_read_frame(format, packet) < 0 {
        avcodec_send_packet(decoder, null);   // the end: the frames the decoder still holds
        ended = true;
        break;
      }
      ours := packet.stream_index == index;
      if ours  avcodec_send_packet(decoder, packet);
      av_packet_unref(packet);
      if ours { packets += 1; break; }
    }
  }
  if !have  return image, info, false, tprint("FFmpeg decoded no frame of its % video", info.codec);
  if got.best_effort_timestamp != AV_NOPTS && stream.time_base.num > 0 {
    start := ifx stream.start_time != AV_NOPTS then stream.start_time else 0;
    info.poster_time = max(cast(float64) 0, cast(float64) (got.best_effort_timestamp - start) * rational(stream.time_base));
  }

  // To RGBA, stretched to square pixels when they are not.
  w, h := got.width, got.height;
  if w <= 0 || h <= 0  return image, info, false, "its frame has no size";
  out_w, out_h := w, h;
  aspect := got.sample_aspect_ratio;
  if aspect.num > 0 && aspect.den > 0 && aspect.num != aspect.den  out_w = max(cast(s32) 1, cast(s32) (cast(float64) w * rational(aspect) + 0.5));
  if cast(s64) out_w * out_h * 4 > S32_MAX  return image, info, false, tprint("its frames are % x %, too large", out_w, out_h);
  scaler := sws_getContext(w, h, cast(AVPixelFormat) got.format, out_w, out_h, .RGBA, SWS_BILINEAR, null, null, null);
  if !scaler  return image, info, false, tprint("FFmpeg cannot convert its frames (pixel format %)", got.format);
  defer sws_freeContext(scaler);
  // swscale writes with SIMD, a row at a time, past the last pixel of a row and of the image: into a
  // buffer of its own allocator (aligned), its rows padded and a spare row after them, then copied
  // out packed. An exact-sized buffer of ours let it write past the end, a crash in swscale.
  packed_stride := cast(s64) out_w * 4;
  stride := (packed_stride + SCALE_ALIGN - 1) / SCALE_ALIGN * SCALE_ALIGN;
  scaled := cast(*u8) av_malloc(cast(u64) (stride * (out_h + 1) + SCALE_ALIGN));
  if !scaled  return image, info, false, tprint("there is no memory for its % x % frame", out_w, out_h);
  defer av_free(scaled);
  planes: [4] *u8;
  strides: [4] s32;
  planes[0] = scaled;
  strides[0] = cast(s32) stride;
  rows := sws_scale(scaler, got.data.data, got.linesize.data, 0, h, planes.data, strides.data);
  if rows <= 0  return image, info, false, "FFmpeg could not convert its frame";
  pixels := cast(*u8) alloc(packed_stride * out_h,, context.default_allocator);
  for y: 0..out_h - 1  memcpy(pixels + y * packed_stride, scaled + y * stride, packed_stride);
  image = .{pixels = pixels, width = out_w, height = out_h};

  if info.rotation == {
    case 90;   image_orient(*image, 6);   // turned clockwise, as EXIF orientation 6
    case 180;  image_orient(*image, 3);
    case 270;  image_orient(*image, 8);
  }
  info.width, info.height = image.width, image.height;
  return image, info, true, "";
}

#scope_file

SCALE_ALIGN :: 64;           // bytes: swscale's widest SIMD writes
VIDEO_MAX_PACKETS :: 3000;   // packets decoded to reach the poster at most: a key frame every few seconds is usual

AV_NOPTS :: cast,no_check(s64) AV_NOPTS_VALUE;
AVERROR_DECODER_NOT_FOUND :: -1128613112;   // FFERRTAG(0xF8, 'D', 'E', 'C')

rational :: (r: AVRational) -> float64 {
  if r.den == 0  return 0;
  return cast(float64) r.num / r.den;
}

av_text :: (what: string, error: s32) -> string {
  buffer: [256] u8;
  av_strerror(error, buffer.data, buffer.count);
  return tprint("% (%)", what, to_string(buffer.data));
}

#import "ffmpeg";