diff --git a/lib/membrane_mp4/container/schema_default.ex b/lib/membrane_mp4/container/schema_default.ex index 343b48ce..32416312 100644 --- a/lib/membrane_mp4/container/schema_default.ex +++ b/lib/membrane_mp4/container/schema_default.ex @@ -50,6 +50,20 @@ defmodule Membrane.MP4.Container.Schema.Default do ] ] + @av1_schema [ + version: 0, + fields: @visual_sample_entry, + av1C: [ + black_box?: true + ], + pasp: [ + fields: [ + h_spacing: :uint32, + v_spacing: :uint32 + ] + ] + ] + @schema_def ftyp: [ fields: [ major_brand: :str32, @@ -197,6 +211,7 @@ defmodule Membrane.MP4.Container.Schema.Default do avc3: @avc_schema, hvc1: @hevc_schema, hev1: @hevc_schema, + av01: @av1_schema, mp4a: [ fields: [ reserved: <<0::6*8>>, diff --git a/lib/membrane_mp4/demuxer/isom.ex b/lib/membrane_mp4/demuxer/isom.ex index 60cbd144..d4095a01 100644 --- a/lib/membrane_mp4/demuxer/isom.ex +++ b/lib/membrane_mp4/demuxer/isom.ex @@ -50,7 +50,8 @@ defmodule Membrane.MP4.Demuxer.ISOM do stream_structure: {_hevc, _dcr}, alignment: :au }, - %Membrane.Opus{self_delimiting?: false} + %Membrane.Opus{self_delimiting?: false}, + %Membrane.AV1{alignment: :tu} ), availability: :on_request, options: [ @@ -517,6 +518,7 @@ defmodule Membrane.MP4.Demuxer.ISOM do defp sample_description_to_kind(%Membrane.H264{}), do: :video defp sample_description_to_kind(%Membrane.H265{}), do: :video + defp sample_description_to_kind(%Membrane.AV1{}), do: :video defp sample_description_to_kind(%Membrane.AAC{}), do: :audio defp sample_description_to_kind(%Membrane.Opus{}), do: :audio diff --git a/lib/membrane_mp4/demuxer/isom/samples_info.ex b/lib/membrane_mp4/demuxer/isom/samples_info.ex index 3fbe81aa..d4fc5946 100644 --- a/lib/membrane_mp4/demuxer/isom/samples_info.ex +++ b/lib/membrane_mp4/demuxer/isom/samples_info.ex @@ -5,11 +5,13 @@ defmodule Membrane.MP4.Demuxer.ISOM.SamplesInfo do # generating output buffers using that structure. # The samples' description is generated from the the `moov` box, which describes how the data is stored inside the `mdat` box. - alias Membrane.{Buffer, Time} + alias Membrane.{AV1, Buffer, Time} alias Membrane.MP4.Container alias Membrane.MP4.MovieBox.SampleTableBox alias Membrane.MP4.Track.SampleTable + @av1_temporal_delimiter_obu <<0x12, 0x00>> + @enforce_keys [ :samples, :tracks_number, @@ -77,7 +79,7 @@ defmodule Membrane.MP4.Demuxer.ISOM.SamplesInfo do buffer = {%Buffer{ - payload: payload, + payload: maybe_prepend_temporal_delimiter(samples_info, track_id, payload), dts: dts, pts: pts }, track_id} @@ -95,6 +97,16 @@ defmodule Membrane.MP4.Demuxer.ISOM.SamplesInfo do end end + # The AV1 ISOBMFF binding requires samples to be stored without temporal delimiter OBUs, + # while `Membrane.AV1` with `alignment: :tu` requires each buffer to start with one, + # so it is restored here. + defp maybe_prepend_temporal_delimiter(samples_info, track_id, payload) do + case samples_info.sample_tables[track_id].sample_description do + %AV1{} -> @av1_temporal_delimiter_obu <> payload + _other -> payload + end + end + defp get_dts_and_pts(samples_info, %{ sample_delta: delta, track_id: track_id, diff --git a/lib/membrane_mp4/helper.ex b/lib/membrane_mp4/helper.ex index 915d10a3..637b78cd 100644 --- a/lib/membrane_mp4/helper.ex +++ b/lib/membrane_mp4/helper.ex @@ -17,5 +17,6 @@ defmodule Membrane.MP4.Helper do @spec key_frame?(Membrane.Buffer.metadata()) :: boolean() def key_frame?(%{h264: %{key_frame?: false}}), do: false def key_frame?(%{h265: %{key_frame?: false}}), do: false + def key_frame?(%{av1: %{key_frame?: false}}), do: false def key_frame?(_metadata), do: true end diff --git a/lib/membrane_mp4/movie_box/sample_table_box.ex b/lib/membrane_mp4/movie_box/sample_table_box.ex index 2b8b5573..9eddf3fd 100644 --- a/lib/membrane_mp4/movie_box/sample_table_box.ex +++ b/lib/membrane_mp4/movie_box/sample_table_box.ex @@ -3,7 +3,7 @@ defmodule Membrane.MP4.MovieBox.SampleTableBox do require Membrane.H264 require Membrane.Logger - alias Membrane.{AAC, H264, H265, Opus} + alias Membrane.{AAC, AV1, H264, H265, Opus} alias Membrane.MP4.{Container, Helper, Track.SampleTable} @spec assemble(SampleTable.t()) :: Container.t() @@ -184,6 +184,79 @@ defmodule Membrane.MP4.MovieBox.SampleTableBox do ] end + defp assemble_sample_description(%AV1{width: width, height: height} = format) do + av1c_content = build_av1c(format) + + [ + {:av01, + %{ + children: [ + av1C: %{ + content: av1c_content + }, + pasp: %{ + children: [], + fields: %{h_spacing: 1, v_spacing: 1} + } + ], + fields: %{ + compressor_name: <<0::size(32)-unit(8)>>, + depth: 24, + flags: 0, + frame_count: 1, + height: height || 0, + horizresolution: {0, 0}, + num_of_entries: 1, + version: 0, + vertresolution: {0, 0}, + width: width || 0 + } + }} + ] + end + + # Build av1C (AV1CodecConfigurationRecord) per ISO/IEC 14496-15 + defp build_av1c(%AV1{} = format) do + config = av1c_config(format) + + << + # Byte 0: marker (1) | version (7) + 1::1, + 1::7, + # Byte 1: seq_profile (3) | seq_level_idx_0 (5) + config.profile::3, + config.level_idx::5, + # Byte 2: seq_tier_0 (1) | high_bitdepth (1) | twelve_bit (1) | monochrome (1) | + # chroma_subsampling_x (1) | chroma_subsampling_y (1) | chroma_sample_position (2) + config.tier::1, + config.high_bitdepth::1, + config.twelve_bit::1, + config.monochrome::1, + config.chroma_subsampling_x::1, + config.chroma_subsampling_y::1, + config.chroma_sample_position::2, + # Byte 3: reserved (3) | initial_presentation_delay_present (1) | reserved (4) + 0::3, + 0::1, + 0::4 + >> + end + + defp av1c_config(%AV1{} = format) do + %{ + profile: AV1.profile_to_seq_profile(format.profile || :main), + level_idx: AV1.level_to_seq_level_idx(format.level || :"4.0"), + tier: AV1.tier_to_seq_tier(format.tier || :main), + # Defaults for 8-bit, 4:2:0 + high_bitdepth: 0, + twelve_bit: 0, + monochrome: 0, + chroma_subsampling_x: 1, + chroma_subsampling_y: 1, + chroma_sample_position: 0 + } + end + defp assemble_sample_deltas(%{timescale: timescale, decoding_deltas: decoding_deltas}), do: Enum.map(decoding_deltas, fn %{sample_count: count, sample_delta: delta} -> @@ -341,6 +414,20 @@ defmodule Membrane.MP4.MovieBox.SampleTableBox do %Opus{channels: boxes[:dOps].fields.output_channel_count, self_delimiting?: false} end + defp unpack_sample_description(%{children: [{:av01, %{children: boxes, fields: fields}}]}) do + <<_marker::1, _version::7, seq_profile::3, seq_level_idx::5, seq_tier::1, _rest::bitstring>> = + boxes[:av1C].content + + %AV1{ + alignment: :tu, + width: fields.width, + height: fields.height, + profile: AV1.seq_profile_to_profile(seq_profile), + level: AV1.seq_level_idx_to_level(seq_level_idx), + tier: AV1.seq_tier_to_tier(seq_tier) + } + end + defp unpack_sample_description(%{children: [{sample_type, _sample_metadata}]}) do Membrane.Logger.warning("Unknown sample type: #{inspect(sample_type)}") nil diff --git a/lib/membrane_mp4/movie_box/track_box.ex b/lib/membrane_mp4/movie_box/track_box.ex index 882af44c..cd24323a 100644 --- a/lib/membrane_mp4/movie_box/track_box.ex +++ b/lib/membrane_mp4/movie_box/track_box.ex @@ -13,7 +13,8 @@ defmodule Membrane.MP4.MovieBox.TrackBox do defguardp is_audio(track) when not is_struct(track.stream_format, Membrane.H264) and - not is_struct(track.stream_format, Membrane.H265) + not is_struct(track.stream_format, Membrane.H265) and + not is_struct(track.stream_format, Membrane.AV1) @spec assemble(Track.t()) :: Container.t() def assemble(track) do diff --git a/lib/membrane_mp4/muxer/isom.ex b/lib/membrane_mp4/muxer/isom.ex index 924963be..8b702f09 100644 --- a/lib/membrane_mp4/muxer/isom.ex +++ b/lib/membrane_mp4/muxer/isom.ex @@ -24,7 +24,8 @@ defmodule Membrane.MP4.Muxer.ISOM do stream_structure: {_hevc, _dcr}, alignment: :au }, - %Membrane.Opus{self_delimiting?: false} + %Membrane.Opus{self_delimiting?: false}, + %Membrane.AV1{alignment: :tu} ), availability: :on_request @@ -182,11 +183,23 @@ defmodule Membrane.MP4.Muxer.ISOM do {[event: {:output, event}], state} end - defp handle_queue_item({Pad.ref(:input, pad_ref), {:buffer, buffer}}, state) do + defp handle_queue_item({Pad.ref(:input, pad_ref), {:buffer, %Buffer{} = buffer}}, state) do # In case DTS is not set, use PTS. This is the case for audio tracks or H264 originated # from an RTP stream. ISO base media file format specification uses DTS for calculating # decoding deltas, and so is the implementation of sample table in this plugin. - buffer = %{buffer | dts: Buffer.get_dts_or_pts(buffer)} + buffer = %Buffer{buffer | dts: Buffer.get_dts_or_pts(buffer)} + + # For AV1 tracks, strip temporal delimiter OBUs from the payload + track = get_in(state, [:pad_to_track, pad_ref]) + + buffer = + case track.stream_format do + %Membrane.AV1{} -> + %Buffer{buffer | payload: strip_av1_temporal_delimiter_obu(buffer.payload)} + + _other -> + buffer + end state |> update_in([:pad_to_track, pad_ref], &Track.store_sample(&1, buffer)) @@ -286,4 +299,11 @@ defmodule Membrane.MP4.Muxer.ISOM do defp shift_left([]), do: [] defp shift_left([first | rest]), do: rest ++ [first] + + # Strip temporal delimiter OBUs from AV1 payload + # Temporal delimiter OBU: header=0x12 (type=2, has_size=1), size=0x00 + defp strip_av1_temporal_delimiter_obu(<<0x12, 0x00, rest::binary>>), + do: strip_av1_temporal_delimiter_obu(rest) + + defp strip_av1_temporal_delimiter_obu(payload), do: payload end diff --git a/lib/membrane_mp4/track.ex b/lib/membrane_mp4/track.ex index d3a16283..00e4c1a4 100644 --- a/lib/membrane_mp4/track.ex +++ b/lib/membrane_mp4/track.ex @@ -8,7 +8,7 @@ defmodule Membrane.MP4.Track do """ require Membrane.{H264, H265} alias __MODULE__.SampleTable - alias Membrane.{AAC, H264, H265} + alias Membrane.{AAC, AV1, H264, H265} alias Membrane.MP4.Helper @type t :: %__MODULE__{ @@ -90,6 +90,8 @@ defmodule Membrane.MP4.Track do | {:avc1, %{profile: binary(), compatibility: binary(), level: binary()}} | {:hvc1, %{profile: non_neg_integer(), tier: non_neg_integer(), level: non_neg_integer()}} + | {:av01, + %{profile: non_neg_integer(), level: non_neg_integer(), tier: non_neg_integer()}} | nil def get_encoding_info(%__MODULE__{ @@ -139,6 +141,16 @@ defmodule Membrane.MP4.Track do {hevc, map} end + def get_encoding_info(%__MODULE__{stream_format: %AV1{} = format}) do + map = %{ + profile: AV1.profile_to_seq_profile(format.profile || :main), + level: AV1.level_to_seq_level_idx(format.level || :"4.0"), + tier: AV1.tier_to_seq_tier(format.tier || :main) + } + + {:av01, map} + end + def get_encoding_info(_unknown), do: nil defp get_timescale(stream_format) do @@ -149,13 +161,13 @@ defmodule Membrane.MP4.Track do %Membrane.AAC{sample_rate: sample_rate} -> sample_rate - %module{framerate: nil} when module in [H264, H265] -> + %module{framerate: nil} when module in [AV1, H264, H265] -> 30 * 1024 - %module{framerate: {0, _denominator}} when module in [H264, H265] -> + %module{framerate: {0, _denominator}} when module in [AV1, H264, H265] -> 30 * 1024 - %module{framerate: {nominator, _denominator}} when module in [H264, H265] -> + %module{framerate: {nominator, _denominator}} when module in [AV1, H264, H265] -> nominator * 1024 end end diff --git a/lib/membrane_mp4/track/sample_table.ex b/lib/membrane_mp4/track/sample_table.ex index 78f092d9..6c941239 100644 --- a/lib/membrane_mp4/track/sample_table.ex +++ b/lib/membrane_mp4/track/sample_table.ex @@ -177,6 +177,11 @@ defmodule Membrane.MP4.Track.SampleTable do Map.update!(sample_table, :sync_samples, &[sample_table.sample_count | &1]) end + defp maybe_store_sync_sample(sample_table, %Buffer{metadata: %{av1: av1}}) + when av1.key_frame? do + Map.update!(sample_table, :sync_samples, &[sample_table.sample_count | &1]) + end + defp maybe_store_sync_sample(sample_table, _buffer), do: sample_table defp store_last_dts(sample_table, %Buffer{dts: dts}), diff --git a/mix.exs b/mix.exs index f3c1036f..77c67bd7 100644 --- a/mix.exs +++ b/mix.exs @@ -45,6 +45,7 @@ defmodule Membrane.MP4.Plugin.MixProject do {:membrane_h264_format, "~> 0.6.1"}, {:membrane_h265_format, "~> 0.2.0"}, {:membrane_opus_format, "~> 0.3.0"}, + {:membrane_av1_format, "~> 0.3.1"}, {:membrane_file_plugin, "~> 0.17.0"}, {:membrane_timestamp_queue, "~> 0.2.1"}, {:bunch, "~> 1.5"}, @@ -53,6 +54,7 @@ defmodule Membrane.MP4.Plugin.MixProject do {:membrane_opus_plugin, "~> 0.19.0", only: :test}, {:membrane_stream_plugin, "~> 0.4.0", only: :test}, {:membrane_fake_plugin, "~> 0.11.0", only: :test}, + {:membrane_ivf_plugin, "~> 0.9.2", only: :test}, {:ex_doc, ">= 0.40.0", only: :dev, runtime: false}, {:dialyxir, ">= 0.0.0", only: :dev, runtime: false}, {:credo, ">= 0.0.0", only: :dev, runtime: false} diff --git a/mix.lock b/mix.lock index 93a87fd4..e68a2a16 100644 --- a/mix.lock +++ b/mix.lock @@ -1,6 +1,6 @@ %{ "bimap": {:hex, :bimap, "1.3.0", "3ea4832e58dc83a9b5b407c6731e7bae87458aa618e6d11d8e12114a17afa4b3", [:mix], [], "hexpm", "bf5a2b078528465aa705f405a5c638becd63e41d280ada41e0f77e6d255a10b4"}, - "bunch": {:hex, :bunch, "1.6.3", "341cd27f16506c3d27cebdd737ab65ee5177215e2c5d74db336ef8d8ca92e412", [:mix], [], "hexpm", "3afd0c5a3eaa5d3d16e221f67297d2f29337a7a9614df7b5c3584b8b485e8765"}, + "bunch": {:hex, :bunch, "1.6.4", "b23ecefdcbc1d8b9723044055daa2ad8573a0ebe6ef35c0ac4f3bad243273ba3", [:mix], [], "hexpm", "cacbe7437dd5299c9e665011d32127e0ce92b26ab4b9f64ebd2389f57cc7c141"}, "bunch_native": {:hex, :bunch_native, "0.5.1", "6928ed1b2239009b04d04bd67d048d2aa7a785b5200301498c04a5b27a200a57", [:mix], [{:bundlex, "~> 1.0", [hex: :bundlex, repo: "hexpm", optional: false]}], "hexpm", "fc979bbac2244b0fce102246122893a46570d38da389a2097faac7a386f6c72b"}, "bundlex": {:hex, :bundlex, "1.5.7", "0be2ec83820df0e285c26e00fda24d1dab410ae3c435bd3257aea83860da8766", [:mix], [{:bunch, "~> 1.0", [hex: :bunch, repo: "hexpm", optional: false]}, {:elixir_uuid, "~> 1.2", [hex: :elixir_uuid, repo: "hexpm", optional: false]}, {:qex, "~> 0.5", [hex: :qex, repo: "hexpm", optional: false]}, {:req, ">= 0.4.0", [hex: :req, repo: "hexpm", optional: false]}, {:zarex, "~> 1.0", [hex: :zarex, repo: "hexpm", optional: false]}], "hexpm", "8ed29816031274077342003fd5816a108e40190333c76497663da074ddd02a89"}, "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, @@ -22,6 +22,7 @@ "makeup_erlang": {:hex, :makeup_erlang, "1.1.0", "835f7e60792e08824cda445639555d7bf1bbbddb1b60b306e33cb6f6db24dc74", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "1cd6780fb1dd1a03979abaed0fe82712b0625118fd5257d3ebbf73f960c73c3c"}, "membrane_aac_format": {:hex, :membrane_aac_format, "0.8.0", "515631eabd6e584e0e9af2cea80471fee6246484dbbefc4726c1d93ece8e0838", [:mix], [{:bimap, "~> 1.1", [hex: :bimap, repo: "hexpm", optional: false]}], "hexpm", "a30176a94491033ed32be45e51d509fc70a5ee6e751f12fd6c0d60bd637013f6"}, "membrane_aac_plugin": {:hex, :membrane_aac_plugin, "0.18.2", "7da3f73e60ad12178623379fd62e0a90e127d65541ab9c46f35e996642db4786", [:mix], [{:bunch, "~> 1.0", [hex: :bunch, repo: "hexpm", optional: false]}, {:membrane_aac_format, "~> 0.8.0", [hex: :membrane_aac_format, repo: "hexpm", optional: false]}, {:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}], "hexpm", "24060500f83b0697cccbe8528ba48c0fbc225b88c87057c413aa3bf5d0dcb6a6"}, + "membrane_av1_format": {:hex, :membrane_av1_format, "0.3.1", "d7d52d11ef359f723f443a12836624ab19235353732f18e8b0adf23b1d1a5062", [:mix], [{:bimap, "~> 1.3", [hex: :bimap, repo: "hexpm", optional: false]}, {:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}], "hexpm", "dcea3c1f255a28d188e85a68a8729b258a3fb1a151254266a872000254f85b90"}, "membrane_cmaf_format": {:hex, :membrane_cmaf_format, "0.7.1", "9ea858faefdcb181cdfa8001be827c35c5f854e9809ad57d7062cff1f0f703fd", [:mix], [], "hexpm", "3c7b4ed2a986e27f6f336d2f19e9442cb31d93b3142fc024c019572faca54a73"}, "membrane_common_c": {:hex, :membrane_common_c, "0.16.0", "caf3f29d2f5a1d32d8c2c122866110775866db2726e4272be58e66dfdf4bce40", [:mix], [{:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}, {:shmex, "~> 0.5.0", [hex: :shmex, repo: "hexpm", optional: false]}, {:unifex, "~> 1.0", [hex: :unifex, repo: "hexpm", optional: false]}], "hexpm", "a3c7e91de1ce1f8b23b9823188a5d13654d317235ea0ca781c05353ed3be9b1c"}, "membrane_core": {:hex, :membrane_core, "1.3.4", "c79944e8a98a965dcb83c1a452d47adb73436f07a9d3cdb21963a8f74cc2c435", [:mix], [{:bunch, "~> 1.6", [hex: :bunch, repo: "hexpm", optional: false]}, {:qex, "~> 0.3", [hex: :qex, repo: "hexpm", optional: false]}, {:ratio, "~> 3.0 or ~> 4.0", [hex: :ratio, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "610e98a5e98834c99171ce78715953919c3014430d0ec8610a166fa58c25f872"}, @@ -30,6 +31,7 @@ "membrane_h264_format": {:hex, :membrane_h264_format, "0.6.1", "44836cd9de0abe989b146df1e114507787efc0cf0da2368f17a10c47b4e0738c", [:mix], [], "hexpm", "4b79be56465a876d2eac2c3af99e115374bbdc03eb1dea4f696ee9a8033cd4b0"}, "membrane_h265_format": {:hex, :membrane_h265_format, "0.2.0", "1903c072cf7b0980c4d0c117ab61a2cd33e88782b696290de29570a7fab34819", [:mix], [], "hexpm", "6df418bdf242c0d9f7dbf2e5aea4c2d182e34ac9ad5a8b8cef2610c290002e83"}, "membrane_h26x_plugin": {:hex, :membrane_h26x_plugin, "0.10.7", "c1c3f6287498ae17cf9a8c904bc594096e2b1ea4d96a5de48086d0e3568d42a1", [:mix], [{:bunch, "~> 1.4", [hex: :bunch, repo: "hexpm", optional: false]}, {:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}, {:membrane_h264_format, "~> 0.6.0", [hex: :membrane_h264_format, repo: "hexpm", optional: false]}, {:membrane_h265_format, "~> 0.2.0", [hex: :membrane_h265_format, repo: "hexpm", optional: false]}], "hexpm", "c9a44ada6ff4827848419660a19062afff2aff0b73d889cc7721426c0d985741"}, + "membrane_ivf_plugin": {:hex, :membrane_ivf_plugin, "0.9.2", "1d3756bac67bde235999cdef7863241b169faade161539ffcb8547e93ea02721", [:mix], [{:membrane_av1_format, "~> 0.3.0", [hex: :membrane_av1_format, repo: "hexpm", optional: false]}, {:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}, {:membrane_file_plugin, "~> 0.17.0", [hex: :membrane_file_plugin, repo: "hexpm", optional: false]}, {:membrane_vp8_format, "~> 0.5.0", [hex: :membrane_vp8_format, repo: "hexpm", optional: false]}, {:membrane_vp9_format, "~> 0.5.0", [hex: :membrane_vp9_format, repo: "hexpm", optional: false]}], "hexpm", "989f1076a3230a21a35b4621614b294af8aa089c480890b6a8216739e4e47a7c"}, "membrane_mp4_format": {:hex, :membrane_mp4_format, "0.8.0", "8c6e7d68829228117d333b4fbb030e7be829aab49dd8cb047fdc664db1812e6a", [:mix], [], "hexpm", "148dea678a1f82ccfd44dbde6f936d2f21255f496cb45a22cc6eec427f025522"}, "membrane_opus_format": {:hex, :membrane_opus_format, "0.3.0", "3804d9916058b7cfa2baa0131a644d8186198d64f52d592ae09e0942513cb4c2", [:mix], [], "hexpm", "8fc89c97be50de23ded15f2050fe603dcce732566fe6fdd15a2de01cb6b81afe"}, "membrane_opus_plugin": {:hex, :membrane_opus_plugin, "0.19.3", "af398a10c84d27e49b9a68ec78a54f123f2637441dd380857a3da4bb492eca5c", [:mix], [{:bunch, "~> 1.3", [hex: :bunch, repo: "hexpm", optional: false]}, {:bundlex, "~> 1.2", [hex: :bundlex, repo: "hexpm", optional: false]}, {:membrane_common_c, "~> 0.16.0", [hex: :membrane_common_c, repo: "hexpm", optional: false]}, {:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}, {:membrane_opus_format, "~> 0.3.0", [hex: :membrane_opus_format, repo: "hexpm", optional: false]}, {:membrane_precompiled_dependency_provider, "~> 0.1.0", [hex: :membrane_precompiled_dependency_provider, repo: "hexpm", optional: false]}, {:membrane_raw_audio_format, "~> 0.12.0", [hex: :membrane_raw_audio_format, repo: "hexpm", optional: false]}, {:unifex, "~> 1.0", [hex: :unifex, repo: "hexpm", optional: false]}], "hexpm", "172d5637233e4e7cb2c464be34ea85b487188887381ef5ff98d5c110fdf44f5b"}, @@ -37,6 +39,8 @@ "membrane_raw_audio_format": {:hex, :membrane_raw_audio_format, "0.12.2", "61aa22442ec7780fbee862fab0522d1069fa4e27e02b87d07a7545ffa8e3d2e0", [:mix], [{:bimap, "~> 1.1", [hex: :bimap, repo: "hexpm", optional: false]}, {:bunch, "~> 1.0", [hex: :bunch, repo: "hexpm", optional: false]}, {:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}], "hexpm", "5ceef5595976e16b3688fe543b08ad34806687c379c61f7f26696bf144a5f782"}, "membrane_stream_plugin": {:hex, :membrane_stream_plugin, "0.4.1", "06503630f52c95c3e3ab792514d2afc674beb707eb3a2483930fbaf86c08bd6e", [:mix], [{:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}], "hexpm", "f6d68f7d42a499bfdf940f71f117c8acdb47ffe84d54f80cd84e41780603d3a1"}, "membrane_timestamp_queue": {:hex, :membrane_timestamp_queue, "0.2.2", "1c831b2273d018a6548654aa9f7fa7c4b683f71d96ffe164934ef55f9d11f693", [:mix], [{:heap, "~> 2.0", [hex: :heap, repo: "hexpm", optional: false]}, {:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}], "hexpm", "7c830e760baaced0988421671cd2c83c7cda8d1bd2b61fd05332711675d1204f"}, + "membrane_vp8_format": {:hex, :membrane_vp8_format, "0.5.1", "82f0b28f936f8c6e719f4c93161f6c028dd5b1352105feb65e1124c36a687376", [:mix], [], "hexpm", "07d8d674878ff8e0146f1b513d2a5f071f9a747504bcdfb2f790eda85def4db2"}, + "membrane_vp9_format": {:hex, :membrane_vp9_format, "0.5.1", "575693fad234794e897848aaa2b28203e157c91b096b9e1304f9e87bc0789372", [:mix], [], "hexpm", "7857f28716cb48adea6dc27f531a0343db03d0708f8bda7555a8fa0437a8297a"}, "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"}, "mint": {:hex, :mint, "1.9.0", "d6f534c2a3e98b2a8cc749b4796eb77e9e3af79a76f96e4c74035a827de0d318", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "007154c7d8c43916aed3c93afd1f11aebbaa9c5ff4b7ba55ebe0d17ee0296042"}, "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, diff --git a/test/fixtures/in_video.ivf b/test/fixtures/in_video.ivf new file mode 100644 index 00000000..1fb6820b Binary files /dev/null and b/test/fixtures/in_video.ivf differ diff --git a/test/membrane_mp4/container_test.exs b/test/membrane_mp4/container_test.exs index 31c6a1f0..975c8b8a 100644 --- a/test/membrane_mp4/container_test.exs +++ b/test/membrane_mp4/container_test.exs @@ -14,7 +14,7 @@ defmodule Membrane.MP4.ContainerTest do defp test_partial(file, boxes_expected) do data = @isom_fixtures |> Path.join(file) |> File.read!() data_size = byte_size(data) - 1 - <> = data + <> = data assert {:ok, boxes, <<_rest::binary>>} = Container.parse(partial) assert boxes |> Enum.map(&elem(&1, 0)) == boxes_expected end diff --git a/test/membrane_mp4/demuxer/isom/integration_test.exs b/test/membrane_mp4/demuxer/isom/integration_test.exs index 62a8b753..130432bd 100644 --- a/test/membrane_mp4/demuxer/isom/integration_test.exs +++ b/test/membrane_mp4/demuxer/isom/integration_test.exs @@ -39,6 +39,16 @@ defmodule Membrane.MP4.Demuxer.ISOM.IntegrationTest do test "non-fast-start H264 and AAC tracks", %{tmp_dir: dir} do perform_test([:H264, :AAC], false, dir) end + + @tag :tmp_dir + test "a single fast start AV1 track", %{tmp_dir: dir} do + perform_test([:AV1], true, dir) + end + + @tag :tmp_dir + test "a single non-fast-start AV1 track", %{tmp_dir: dir} do + perform_test([:AV1], false, dir) + end end test "the PTS and DTS are properly read" do @@ -186,6 +196,52 @@ defmodule Membrane.MP4.Demuxer.ISOM.IntegrationTest do assert_files_equal(out_audio_path, in_audio_path) end + defp perform_test([:AV1], fast_start, dir) do + in_path = "test/fixtures/in_video.ivf" + mp4_path = Path.join(dir, "out.mp4") + + muxing_spec = [ + child(:file, %Membrane.File.Source{location: in_path}) + |> child(:deserializer, Membrane.IVF.Deserializer) + |> child(:muxer, %Membrane.MP4.Muxer.ISOM{ + chunk_duration: Membrane.Time.seconds(1), + fast_start: fast_start + }) + |> child(:sink, %Membrane.File.Sink{location: mp4_path}) + ] + + Pipeline.start_link_supervised!(spec: muxing_spec) |> wait_for_pipeline_termination() + + ref_spec = [ + child(:file, %Membrane.File.Source{location: in_path}) + |> child(:deserializer, Membrane.IVF.Deserializer) + |> child(:sink, Membrane.Testing.Sink) + ] + + Pipeline.start_link_supervised!(spec: ref_spec) |> wait_for_pipeline_termination() + ref_payloads = flush_buffers() |> Enum.map(& &1.payload) + + demuxing_spec = [ + child(:file, %Membrane.File.Source{location: mp4_path}) + |> child(:demuxer, Membrane.MP4.Demuxer.ISOM) + |> via_out(Pad.ref(:output, 1)) + |> child(:sink, Membrane.Testing.Sink) + ] + + pipeline = Pipeline.start_link_supervised!(spec: demuxing_spec) + + assert_sink_stream_format(pipeline, :sink, %Membrane.AV1{ + alignment: :tu, + width: 1080, + height: 720 + }) + + wait_for_pipeline_termination(pipeline) + demuxed_payloads = flush_buffers() |> Enum.map(& &1.payload) + + assert demuxed_payloads == ref_payloads + end + defp flush_buffers(acc \\ []) do receive do {Membrane.Testing.Pipeline, _pid, {:handle_child_notification, {{:buffer, buffer}, :sink}}} -> diff --git a/test/membrane_mp4/muxer/isom/integration_test.exs b/test/membrane_mp4/muxer/isom/integration_test.exs index 710e611a..d06d7b35 100644 --- a/test/membrane_mp4/muxer/isom/integration_test.exs +++ b/test/membrane_mp4/muxer/isom/integration_test.exs @@ -44,6 +44,11 @@ defmodule Membrane.MP4.Muxer.ISOM.IntegrationTest do end describe "Muxer.ISOM should mux" do + @tag :tmp_dir + test "single AV1 track", %{tmp_dir: dir} do + perform_av1_test(false, dir) + end + test "single H264 track" do prepare_test("video") @@ -161,6 +166,11 @@ defmodule Membrane.MP4.Muxer.ISOM.IntegrationTest do end describe "Muxer.ISOM with fast_start enabled should mux" do + @tag :tmp_dir + test "single AV1 track", %{tmp_dir: dir} do + perform_av1_test(true, dir) + end + test "single H264 track" do prepare_test("video_fast_start") @@ -255,4 +265,91 @@ defmodule Membrane.MP4.Muxer.ISOM.IntegrationTest do refute Container.get_box(parsed_out, [:moov, :trak, :mdia, :minf, :stbl, :ctts]) end end + + defp perform_av1_test(fast_start, dir) do + output_path = Path.join(dir, "av1.mp4") + temporal_delimiter = <<0x12, 0x00>> + + stored_payloads = [ + <<0xA1, 0xA2>>, + <<0xB1, 0x12, 0x00, 0xB2>>, + <<0xC1>> + ] + + [first_payload, second_payload, third_payload] = stored_payloads + + buffers = [ + %Membrane.Buffer{ + payload: temporal_delimiter <> temporal_delimiter <> first_payload, + pts: 0, + metadata: %{av1: %{key_frame?: true}} + }, + %Membrane.Buffer{ + payload: temporal_delimiter <> second_payload, + pts: Time.milliseconds(33), + metadata: %{av1: %{key_frame?: false}} + }, + %Membrane.Buffer{ + payload: temporal_delimiter <> third_payload, + pts: Time.milliseconds(66), + metadata: %{av1: %{key_frame?: true}} + } + ] + + stream_format = %Membrane.AV1{ + alignment: :tu, + width: 1080, + height: 720, + framerate: {30, 1}, + profile: :main, + level: :"4.1", + tier: :high + } + + structure = + child(:source, %Membrane.Testing.Source{output: buffers, stream_format: stream_format}) + |> child(:muxer, %Membrane.MP4.Muxer.ISOM{ + chunk_duration: Time.seconds(1), + fast_start: fast_start + }) + |> child(:sink, %Membrane.File.Sink{location: output_path}) + + pid = Pipeline.start_link_supervised!(spec: structure) + + assert_end_of_stream(pid, :sink, :input) + assert :ok == Pipeline.terminate(pid) + + output = File.read!(output_path) + assert {parsed, <<>>} = Container.parse!(output) + + expected_box_order = + if fast_start, do: [:ftyp, :moov, :mdat], else: [:ftyp, :mdat, :moov] + + assert Enum.map(parsed, &elem(&1, 0)) == expected_box_order + + expected_mdat = IO.iodata_to_binary(stored_payloads) + assert parsed[:mdat].content == expected_mdat + + av01 = Container.get_box(parsed, [:moov, :trak, :mdia, :minf, :stbl, :stsd, :av01]) + assert av01.fields.width == 1080 + assert av01.fields.height == 720 + assert av01.children[:av1C].content == <<0x81, 0x09, 0x8C, 0x00>> + + assert Container.get_box(parsed, [:moov, :trak, :mdia, :hdlr]).fields.handler_type == + "vide" + + assert Container.get_box(parsed, [:moov, :trak, :mdia, :minf, :vmhd]) + + stss = Container.get_box(parsed, [:moov, :trak, :mdia, :minf, :stbl, :stss]) + assert stss.fields.entry_list == [%{sample_number: 1}, %{sample_number: 3}] + + stsz = Container.get_box(parsed, [:moov, :trak, :mdia, :minf, :stbl, :stsz]) + + assert stsz.fields.entry_list == + Enum.map(stored_payloads, &%{entry_size: byte_size(&1)}) + + stco = Container.get_box(parsed, [:moov, :trak, :mdia, :minf, :stbl, :stco]) + assert [%{chunk_offset: chunk_offset}] = stco.fields.entry_list + assert binary_part(output, chunk_offset, byte_size(expected_mdat)) == expected_mdat + end end diff --git a/test/support/demuxer/multi_file_source.ex b/test/support/demuxer/multi_file_source.ex index 78aa3308..fa9613fb 100644 --- a/test/support/demuxer/multi_file_source.ex +++ b/test/support/demuxer/multi_file_source.ex @@ -25,7 +25,7 @@ defmodule Membrane.MP4.Demuxer.MultiFileSource do @impl true def handle_demand(:output, demand_size, :bytes, ctx, state) do case state.binary do - <> -> + <> -> {[buffer: {:output, %Membrane.Buffer{payload: first}}], %{state | binary: rest}} other ->