Expose DISPLAYMATRIX side data

Co-authored-by: Sebastien Ehrhardt <sebastien.ehrhardt@onfido.com>
Co-authored-by: WyattBlue <wyattblue@auto-editor.com>

Author: 3 people

av/stream.pxd

@@ -12,13 +12,16 @@ cdef class Stream:
     # Stream attributes.
     cdef readonly Container container
     cdef readonly dict metadata
+    cdef readonly int nb_side_data
+    cdef readonly dict side_data
 
     # CodecContext attributes.
     cdef readonly CodecContext codec_context
 
     # Private API.
     cdef _init(self, Container, lib.AVStream*, CodecContext)
     cdef _finalize_for_output(self)
+    cdef _get_side_data(self, lib.AVStream *stream)
     cdef _set_time_base(self, value)
     cdef _set_id(self, value)
 

av/stream.pyx

@@ -1,7 +1,9 @@
 import warnings
 
+from libc.stdint cimport int32_t
 cimport libav as lib
 
+from av.enum cimport define_enum
 from av.error cimport err_check
 from av.packet cimport Packet
 from av.utils cimport (
@@ -17,6 +19,12 @@ from av.deprecation import AVDeprecationWarning
 cdef object _cinit_bypass_sentinel = object()
 
 
+# If necessary more can be added from
+# https://ffmpeg.org/doxygen/trunk/group__lavc__packet.html#ga9a80bfcacc586b483a973272800edb97
+SideData = define_enum("SideData", __name__, (
+    ("DISPLAYMATRIX", lib.AV_PKT_DATA_DISPLAYMATRIX, "Display Matrix"),
+))
+
 cdef Stream wrap_stream(Container container, lib.AVStream *c_stream, CodecContext codec_context):
     """Build an av.Stream for an existing AVStream.
 
@@ -79,6 +87,8 @@ cdef class Stream:
         if self.codec_context:
             self.codec_context.stream_index = stream.index
 
+        self.nb_side_data, self.side_data = self._get_side_data(stream)
+        
         self.metadata = avdict_to_dict(
             stream.metadata,
             encoding=self.container.metadata_encoding,
@@ -103,6 +113,11 @@ cdef class Stream:
                 AVDeprecationWarning
             )
 
+        if name == "side_data":
+            return self.side_data
+        elif name == "nb_side_data":
+            return self.nb_side_data
+
         # Convenience getter for codec context properties.
         if self.codec_context is not None:
             return getattr(self.codec_context, name)
@@ -167,6 +182,20 @@ cdef class Stream:
 
         return self.codec_context.decode(packet)
 
+    cdef _get_side_data(self, lib.AVStream *stream):
+        # Get DISPLAYMATRIX SideDate from a lib.AVStream object.
+        # Returns: tuple[int, dict[str, Any]]
+
+        nb_side_data = stream.nb_side_data
+        side_data = {}
+        
+        for i in range(nb_side_data):
+            # Based on: https://www.ffmpeg.org/doxygen/trunk/dump_8c_source.html#l00430
+            if stream.side_data[i].type == lib.AV_PKT_DATA_DISPLAYMATRIX:
+                side_data["DISPLAYMATRIX"] = lib.av_display_rotation_get(<const int32_t *>stream.side_data[i].data)
+
+        return nb_side_data, side_data
+
     property id:
         """
         The format-specific ID of this stream.

include/libavcodec/avcodec.pxd

@@ -258,6 +258,46 @@ cdef extern from "libavcodec/avcodec.h" nogil:
 
     cdef int AV_NUM_DATA_POINTERS
 
+    cdef enum AVPacketSideDataType:
+        AV_PKT_DATA_PALETTE
+        AV_PKT_DATA_NEW_EXTRADATA
+        AV_PKT_DATA_PARAM_CHANGE
+        AV_PKT_DATA_H263_MB_INFO
+        AV_PKT_DATA_REPLAYGAIN
+        AV_PKT_DATA_DISPLAYMATRIX
+        AV_PKT_DATA_STEREO3D
+        AV_PKT_DATA_AUDIO_SERVICE_TYPE
+        AV_PKT_DATA_QUALITY_STATS
+        AV_PKT_DATA_FALLBACK_TRACK
+        AV_PKT_DATA_CPB_PROPERTIES
+        AV_PKT_DATA_SKIP_SAMPLES
+        AV_PKT_DATA_JP_DUALMONO
+        AV_PKT_DATA_STRINGS_METADATA
+        AV_PKT_DATA_SUBTITLE_POSITION
+        AV_PKT_DATA_MATROSKA_BLOCKADDITIONAL
+        AV_PKT_DATA_WEBVTT_IDENTIFIER
+        AV_PKT_DATA_WEBVTT_SETTINGS
+        AV_PKT_DATA_METADATA_UPDATE
+        AV_PKT_DATA_MPEGTS_STREAM_ID
+        AV_PKT_DATA_MASTERING_DISPLAY_METADATA
+        AV_PKT_DATA_SPHERICAL
+        AV_PKT_DATA_CONTENT_LIGHT_LEVEL
+        AV_PKT_DATA_A53_CC
+        AV_PKT_DATA_ENCRYPTION_INIT_INFO
+        AV_PKT_DATA_ENCRYPTION_INFO
+        AV_PKT_DATA_AFD
+        AV_PKT_DATA_PRFT
+        AV_PKT_DATA_ICC_PROFILE
+        AV_PKT_DATA_DOVI_CONF
+        AV_PKT_DATA_S12M_TIMECODE
+        AV_PKT_DATA_DYNAMIC_HDR10_PLUS
+        AV_PKT_DATA_NB
+
+    cdef struct AVPacketSideData:
+        uint8_t *data;
+        size_t size;
+        AVPacketSideDataType type;
+
     cdef enum AVFrameSideDataType:
         AV_FRAME_DATA_PANSCAN
         AV_FRAME_DATA_A53_CC

include/libavformat/avformat.pxd

@@ -47,6 +47,10 @@ cdef extern from "libavformat/avformat.h" nogil:
         AVRational r_frame_rate
         AVRational sample_aspect_ratio
 
+        int nb_side_data
+        AVPacketSideData *side_data
+
+
     # http://ffmpeg.org/doxygen/trunk/structAVIOContext.html
     cdef struct AVIOContext:
         unsigned char* buffer

include/libavutil/avutil.pxd

@@ -1,9 +1,12 @@
-from libc.stdint cimport int64_t, uint8_t, uint64_t
+from libc.stdint cimport int64_t, uint8_t, uint64_t, int32_t
 
 
 cdef extern from "libavutil/mathematics.h" nogil:
     pass
 
+cdef extern from "libavutil/display.h" nogil:
+    cdef double av_display_rotation_get(const int32_t matrix[9])
+
 cdef extern from "libavutil/rational.h" nogil:
     cdef int av_reduce(int *dst_num, int *dst_den, int64_t num, int64_t den, int64_t max)
 

tests/test_streams.py

@@ -24,3 +24,16 @@ def test_selection(self):
         self.assertEqual([video], container.streams.get(video=(0,)))
 
         # TODO: Find something in the fate suite with video, audio, and subtitles.
+
+    def test_noside_data(self):
+        container = av.open(fate_suite("h264/interlaced_crop.mp4"))
+        video = container.streams.video[0]
+
+        self.assertEqual(video.nb_side_data, 0)
+
+    def test_side_data(self):
+        container = av.open(fate_suite("mov/displaymatrix.mov"))
+        video = container.streams.video[0]
+
+        self.assertEqual(video.nb_side_data, 1)
+        self.assertEqual(video.side_data["DISPLAYMATRIX"], -90.0)