如何用原生Lua创建、读取MP4视频帧及排查代码问题

我正在开展一个项目，需完全使用Lua创建MP4视频并控制其帧，不依赖任何外部API，目标是实现视频帧的生成与读取。具体需要以下指导：

• 在Lua中创建视频帧
• 将这些帧组装为MP4文件
• 从现有MP4文件中读取帧

希望了解从头实现的必要步骤及可用的Lua函数或技术。

我编写了一段Lua代码来创建带帧的MP4文件，但生成的文件无法正常运行，怀疑是MP4 box结构或视频帧组装方式存在问题。代码如下：

local function create_ftyp_box()
  local major_brand = "isom"
  local minor_version = "\0\0\2\0" -- Version 2.0
  local compatible_brands = "isom" .. "iso2" .. "avc1" .. "mp41"
  local ftyp_data = major_brand .. minor_version .. compatible_brands
  return ftyp_data
end

local function create_mvhd_box()
  local version = "\0"  -- version 0
  local flags = "\0\0\0"  -- flags
  local creation_time = "\0\0\0\0"
  local modification_time = "\0\0\0\0"
  local timescale = "\0\0\03\xe8"  -- 1000 units per second
  local duration = "\0\0\03\xe8"  -- 1 second duration
  local rate = "\0\0\01\00"  -- 1.0
  local volume = "\0\01\00\00"  -- full volume
  local reserved = "\0\0" .. string.rep("\0", 10)
  local matrix = string.rep("\0", 36)  -- identity matrix
  local pre_defined = string.rep("\0", 24)
  local next_track_ID = "\0\0\0\2"
  return version .. flags .. creation_time .. modification_time .. timescale .. duration ..
         rate .. volume .. reserved .. matrix .. pre_defined .. next_track_ID
end

local function create_tkhd_box()
  local version = "\0"  -- version 0
  local flags = "\0\0\7"  -- track enabled, track in movie, track in preview
  local creation_time = "\0\0\0\0"
  local modification_time = "\0\0\0\0"
  local track_ID = "\0\0\0\1"
  local reserved = "\0\0\0\0"
  local duration = "\0\0\03\xe8"  -- 1 second duration
  local reserved2 = string.rep("\0", 8)
  local layer = "\0\0"
  local alternate_group = "\0\0"
  local volume = "\0\01\00\00"
  local reserved3 = "\0\0"
  local matrix = string.rep("\0", 36)  -- identity matrix
  local width = "\0\01\00\00"  -- 1.0
  local height = "\0\01\00\00"  -- 1.0
  return version .. flags .. creation_time .. modification_time .. track_ID .. reserved .. duration ..
         reserved2 .. layer .. alternate_group .. volume .. reserved3 .. matrix .. width .. height
end

local function create_mdhd_box()
  local version = "\0"  -- version 0
  local flags = "\0\0\0"
  local creation_time = "\0\0\0\0"
  local modification_time = "\0\0\0\0"
  local timescale = "\0\0\03\xe8"  -- 1000 units per second
  local duration = "\0\0\03\xe8"  -- 1 second duration
  local language = "\x55\xc4"  -- undetermined language
  local pre_defined = "\0\0"
  return version .. flags .. creation_time .. modification_time .. timescale .. duration .. language .. pre_defined
end

local function create_hdlr_box()
  local version = "\0"  -- version 0
  local flags = "\0\0\0"
  local pre_defined = "\0\0\0\0"
  local handler_type = "vide"  -- video track
  local reserved = string.rep("\0", 12)
  local name = "VideoHandler\0"
  return version .. flags .. pre_defined .. handler_type .. reserved .. name
end

local function create_vmhd_box()
  local version = "\0"  -- version 0
  local flags = "\0\0\1"  -- default flags
  local graphics_mode = "\0\0"
  local opcolor = "\0\0\0\0\0\0"
  return version .. flags .. graphics_mode .. opcolor
end

local function create_dinf_box()
  local version = "\0"  -- version 0
  local flags = "\0\0\0"
  local entry_count = "\0\0\0\1"
  local data_reference_box = "\0\0\0\x0c" .. "url " .. "\0\0\0\1"
  return version .. flags .. entry_count .. data_reference_box
end

local function create_stsd_box()
  local version = "\0"  -- version 0
  local flags = "\0\0\0"
  local entry_count = "\0\0\0\1"
  local sample_entry = "\0\0\0\x86" .. "avc1" .. string.rep("\0", 24) .. "\0\0\0\1" .. "\0\0\0\0" .. "\0\0\0\0"
  return version .. flags .. entry_count .. sample_entry
end

local function create_stts_box()
  local version = "\0"  -- version 0
  local flags = "\0\0\0"
  local entry_count = "\0\0\0\1"
  local sample_count = "\0\0\0\1"
  local sample_delta = "\0\0\03\xe8"
  return version .. flags .. entry_count .. sample_count .. sample_delta
end

local function create_stsc_box()
  local version = "\0"  -- version 0
  local flags = "\0\0\0"
  local entry_count = "\0\0\0\1"
  local first_chunk = "\0\0\0\1"
  local samples_per_chunk = "\0\0\0\1"
  local sample_description_index = "\0\0\0\1"
  return version .. flags .. entry_count .. first_chunk .. samples_per_chunk .. sample_description_index
end

local function create_stsz_box()
  local version = "\0"  -- version 0
  local flags = "\0\0\0"
  local sample_size = "\0\0\0\0"  -- non-uniform sample sizes
  local sample_count = "\0\0\0\1"
  local sample_sizes = "\0\0\0\1"
  return version .. flags .. sample_size .. sample_count .. sample_sizes
end

local function create_stco_box()
  local version = "\0"  -- version 0
  local flags = "\0\0\0"
  local entry_count = "\0\0\0\1"
  local chunk_offset = "\0\0\0\0"  -- placeholder offset
  return version .. flags .. entry_count .. chunk_offset
end

local function create_minf_box()
  local vmhd = create_vmhd_box()
  local dinf = create_dinf_box()
  local stbl = create_stsd_box() .. create_stts_box() .. create_stsc_box() .. create_stsz_box() .. create_stco_box()
  return vmhd .. dinf .. stbl
end

local function create_mdia_box()
  local mdhd = create_mdhd_box()
  local hdlr = create_hdlr_box()
  local minf = create_minf_box()
  return mdhd .. hdlr .. minf
end

local function create_trak_box()
  local tkhd = create_tkhd_box()
  local mdia = create_mdia_box()
  return tkhd .. mdia
end

local function create_moov_box()
  local mvhd = create_mvhd_box()
  local trak = create_trak_box()
  return mvhd .. trak
end

local function create_mdat_box()
  local data = string.rep("\0", 4)  -- placeholder for actual media data
  return data
end

local function create_frame_data(color)
  local width = 1280
  local height = 720
  local bytes_per_pixel = 3 -- RGB
  local frame_size = width * height * bytes_per_pixel

  local data = ""

  if color == "white" then
    for i = 1, frame_size do
      data = data .. "\xff"
    end
  elseif color == "black" then
    for i = 1, frame_size do
      data = data .. "\x00"
    end
  end

  return data
end

local function write_box(f, box_type, data)
  local size = 8 + #data
  f:write(string.pack(">>I4", size))  -- Box size
  f:write(box_type)  -- Box type
  f:write(data)  -- Box data
end

local function create_mp4(filename, color)
  local f = assert(io.open(filename, "wb"))

  -- Write ftyp box
  write_box(f, "ftyp", create_ftyp_box())

  -- Write moov box
  write_box(f, "moov", create_moov_box())

  -- Write mdat box
  local frame_data = create_frame_data(color)
  write_box(f, "mdat", frame_data)

  f:close()
end

create_mp4("black_frame.mp4", "black")
create_mp4("white_frame.mp4", "white")

你的代码存在多个核心问题，导致播放器无法解析生成的MP4文件，以下是关键问题和修复方案：

核心问题 #

1. Box嵌套错误：MP4的box是层级结构，子box必须作为父box的内容写入，但你的代码直接拼接字符串（比如dinf里的url box），导致结构混乱。
2. 编码不匹配：声明使用avc1（H.264）编码，但实际写入的是原始RGB数据，播放器无法解码。
3. 参数错误：尺寸（width/height）使用了错误的定点数格式，chunk偏移量为占位符0，sample大小与实际帧数据不匹配。
4. 缺失必要子box：stsd里的avc1条目缺少avcc子box（存储H.264参数），无法让播放器识别编码参数。

修复后的代码 #

我们改用无压缩的mp4v格式简化实现，同时修复所有结构和参数错误：

-- 辅助函数：将整数转换为大端4字节
local function uint32_be(n)
  return string.pack(">I4", n)
end

-- 辅助函数：将整数转换为大端2字节
local function uint16_be(n)
  return string.pack(">I2", n)
end

-- 辅助函数：将16.16定点数转换为4字节大端
local function fixed16_16(n)
  return string.pack(">I4", math.floor(n * 65536))
end

-- 创建ftyp box
local function create_ftyp_box()
  local major_brand = "isom"
  local minor_version = uint32_be(0x00020000) -- Version 2.0
  local compatible_brands = "isom" .. "mp41" .. "mp42" .. "avc1"
  return major_brand .. minor_version .. compatible_brands
end

-- 创建mvhd box
local function create_mvhd_box(duration, timescale)
  local version = "\0"
  local flags = "\0\0\0"
  local creation_time = uint32_be(0)
  local modification_time = uint32_be(0)
  timescale = uint32_be(timescale or 1000)
  duration = uint32_be(duration or 1000) -- 1秒
  local rate = fixed16_16(1.0) -- 播放速率
  local volume = fixed16_16(1.0) -- 音量
  local reserved = string.rep("\0", 12)
  -- 单位矩阵
  local matrix = fixed16_16(1.0) .. string.rep("\0", 8) ..
                 string.rep("\0", 8) .. fixed16_16(1.0) ..
                 string.rep("\0", 8) .. fixed16_16(1.0) ..
                 string.rep("\0", 12)
  local pre_defined = string.rep("\0", 24)
  local next_track_ID = uint32_be(2)
  return version .. flags .. creation_time .. modification_time .. timescale .. duration ..
         rate .. volume .. reserved .. matrix .. pre_defined .. next_track_ID
end

-- 创建tkhd box
local function create_tkhd_box(width, height, duration, timescale)
  local version = "\0"
  local flags = "\0\0\7" -- 启用轨道
  local creation_time = uint32_be(0)
  local modification_time = uint32_be(0)
  local track_ID = uint32_be(1)
  local reserved = uint32_be(0)
  duration = uint32_be(duration or 1000)
  local reserved2 = string.rep("\0", 8)
  local layer = uint16_be(0)
  local alternate_group = uint16_be(0)
  local volume = fixed16_16(0.0) -- 视频轨道音量为0
  local reserved3 = uint16_be(0)
  -- 单位矩阵
  local matrix = fixed16_16(1.0) .. string.rep("\0", 8) ..
                 string.rep("\0", 8) .. fixed16_16(1.0) ..
                 string.rep("\0", 8) .. fixed16_16(1.0) ..
                 string.rep("\0", 12)
  width = fixed16_16(width)
  height = fixed16_16(height)
  return version .. flags .. creation_time .. modification_time .. track_ID .. reserved .. duration ..
         reserved2 .. layer .. alternate_group .. volume .. reserved3 .. matrix .. width .. height
end

-- 创建mdhd box
local function create_mdhd_box(duration, timescale)
  local version = "\0"
  local flags = "\0\0\0"
  local creation_time = uint32_be(0)
  local modification_time = uint32_be(0)
  timescale = uint32_be(timescale or 1000)
  duration = uint32_be(duration or 1000)
  local language = "\x55\xc4" -- 未指定语言
  local pre_defined = uint16_be(0)
  return version .. flags .. creation_time .. modification_time .. timescale .. duration .. language .. pre_defined
end

-- 创建hdlr box
local function create_hdlr_box()
  local version = "\0"
  local flags = "\0\0\0"
  local pre_defined = uint32_be(0)
  local handler_type = "vide" -- 视频轨道
  local reserved = string.rep("\0", 12)
  local name = "VideoHandler\0"
  return version .. flags .. pre_defined .. handler_type .. reserved .. name
end

-- 创建vmhd box
local function create_vmhd_box()
  local version = "\0"
  local flags = "\0\0\1"
  local graphics_mode = uint16_be(0)
  local opcolor = string.rep("\0", 6)
  return version .. flags .. graphics_mode .. opcolor
end

-- 创建url box（dinf的子box）
local function create_url_box()
  local version = "\0"
  local flags = "\0\0\1" -- 引用本地数据
  return version .. flags
end

-- 创建dinf box
local function create_dinf_box()
  local version = "\0"
  local flags = "\0\0\0"
  local entry_count = uint32_be(1)
  -- 创建url子box
  local url_data = create_url_box()
  local url_box = uint32_be(8 + #url_data) .. "url " .. url_data
  return version .. flags .. entry_count .. url_box
end

-- 创建mp4v sample entry（stsd的子box）
local function create_mp4v_sample_entry(width, height)
  local size = 8 + 6 + 16 + 8 + 2 + 2 + 4 + 4 + 4 + 4 + 4 + 4
  local entry_size = uint32_be(size)
  local entry_type = "mp4v"
  local reserved = string.rep("\0", 6)
  local data