by Hans Verkuil

[permalink] [raw]

Subject: Re: [PATCH v3 01/19] media: uapi: h264: Update reference lists

On 14/08/2020 15:36, Ezequiel Garcia wrote:
> From: Jernej Skrabec <[email protected]>
>
> When dealing with with interlaced frames, reference lists must tell if
> each particular reference is meant for top or bottom field. This info
> is currently not provided at all in the H264 related controls.
>
> Make reference lists hold a structure which will also hold an
> enumerator type along index into DPB array. The enumerator must
> be used to specify if reference is for top or bottom field.
>
> Currently the only user of these lists is Cedrus which is just compile
> fixed here. Actual usage of will come in a following commit.
>
> Signed-off-by: Jernej Skrabec <[email protected]>
> Signed-off-by: Ezequiel Garcia <[email protected]>
> ---
> v3:
> * Rename to avoid mentioning the DPB.
> v2:
> * As pointed out by Jonas, enum v4l2_h264_dpb_reference here.
> ---
> .../media/v4l/ext-ctrls-codec.rst | 44 ++++++++++++++++++-
> .../staging/media/sunxi/cedrus/cedrus_h264.c | 6 +--
> include/media/h264-ctrls.h | 23 +++++++---
> 3 files changed, 62 insertions(+), 11 deletions(-)
>
> diff --git a/Documentation/userspace-api/media/v4l/ext-ctrls-codec.rst b/Documentation/userspace-api/media/v4l/ext-ctrls-codec.rst
> index d0d506a444b1..b9b2617c3bda 100644
> --- a/Documentation/userspace-api/media/v4l/ext-ctrls-codec.rst
> +++ b/Documentation/userspace-api/media/v4l/ext-ctrls-codec.rst
> @@ -1843,10 +1843,10 @@ enum v4l2_mpeg_video_h264_hierarchical_coding_type -
> * - __u32
> - ``slice_group_change_cycle``
> -
> - * - __u8
> + * - struct :c:type:`v4l2_h264_reference`
> - ``ref_pic_list0[32]``
> - Reference picture list after applying the per-slice modifications
> - * - __u8
> + * - struct :c:type:`v4l2_h264_reference`
> - ``ref_pic_list1[32]``
> - Reference picture list after applying the per-slice modifications
> * - __u32
> @@ -1926,6 +1926,46 @@ enum v4l2_mpeg_video_h264_hierarchical_coding_type -
> - ``chroma_offset[32][2]``
> -
>
> +``Picture Reference``
> +
> +.. c:type:: v4l2_h264_reference
> +
> +.. cssclass:: longtable
> +
> +.. flat-table:: struct v4l2_h264_reference
> + :header-rows: 0
> + :stub-columns: 0
> + :widths: 1 1 2
> +
> + * - enum :c:type:`v4l2_h264_field_reference`
> + - ``reference``
> + - Specifies how the picture is referenced.
> + * - __u8
> + - ``index``
> + - Index into the :c:type:`v4l2_ctrl_h264_decode_params`.dpb array.
> +
> +.. c:type:: v4l2_h264_field_reference
> +
> +.. cssclass:: longtable
> +
> +.. flat-table::
> + :header-rows: 0
> + :stub-columns: 0
> + :widths: 1 1 2
> +
> + * - ``V4L2_H264_TOP_FIELD_REF``
> + - 0x1
> + - The top field in field pair is used for
> + short-term reference.
> + * - ``V4L2_H264_BOTTOM_FIELD_REF``
> + - 0x2
> + - The bottom field in field pair is used for
> + short-term reference.
> + * - ``V4L2_H264_FRAME_REF``
> + - 0x3
> + - The frame (or the top/bottom fields, if it's a field pair)
> + is used for short-term reference.
> +
> ``V4L2_CID_MPEG_VIDEO_H264_DECODE_PARAMS (struct)``
> Specifies the decode parameters (as extracted from the bitstream)
> for the associated H264 slice data. This includes the necessary
> diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_h264.c b/drivers/staging/media/sunxi/cedrus/cedrus_h264.c
> index 54ee2aa423e2..cce527bbdf86 100644
> --- a/drivers/staging/media/sunxi/cedrus/cedrus_h264.c
> +++ b/drivers/staging/media/sunxi/cedrus/cedrus_h264.c
> @@ -166,8 +166,8 @@ static void cedrus_write_frame_list(struct cedrus_ctx *ctx,
>
> static void _cedrus_write_ref_list(struct cedrus_ctx *ctx,
> struct cedrus_run *run,
> - const u8 *ref_list, u8 num_ref,
> - enum cedrus_h264_sram_off sram)
> + const struct v4l2_h264_reference *ref_list,
> + u8 num_ref, enum cedrus_h264_sram_off sram)
> {
> const struct v4l2_ctrl_h264_decode_params *decode = run->h264.decode_params;
> struct vb2_queue *cap_q;
> @@ -188,7 +188,7 @@ static void _cedrus_write_ref_list(struct cedrus_ctx *ctx,
> int buf_idx;
> u8 dpb_idx;
>
> - dpb_idx = ref_list[i];
> + dpb_idx = ref_list[i].index;
> dpb = &decode->dpb[dpb_idx];
>
> if (!(dpb->flags & V4L2_H264_DPB_ENTRY_FLAG_ACTIVE))
> diff --git a/include/media/h264-ctrls.h b/include/media/h264-ctrls.h
> index 080fd1293c42..5f635e8d25e2 100644
> --- a/include/media/h264-ctrls.h
> +++ b/include/media/h264-ctrls.h
> @@ -19,6 +19,8 @@
> */
> #define V4L2_H264_NUM_DPB_ENTRIES 16
>
> +#define V4L2_H264_REF_LIST_LEN (2 * V4L2_H264_NUM_DPB_ENTRIES)
> +
> /* Our pixel format isn't stable at the moment */
> #define V4L2_PIX_FMT_H264_SLICE v4l2_fourcc('S', '2', '6', '4') /* H264 parsed slices */
>
> @@ -140,6 +142,19 @@ struct v4l2_h264_pred_weight_table {
> #define V4L2_H264_SLICE_FLAG_DIRECT_SPATIAL_MV_PRED 0x04
> #define V4L2_H264_SLICE_FLAG_SP_FOR_SWITCH 0x08
>
> +enum v4l2_h264_field_reference {
> + V4L2_H264_TOP_FIELD_REF = 0x1,
> + V4L2_H264_BOTTOM_FIELD_REF = 0x2,
> + V4L2_H264_FRAME_REF = 0x3,
> +};
> +
> +struct v4l2_h264_reference {
> + enum v4l2_h264_field_reference fields;
> +
> + /* Index into v4l2_ctrl_h264_decode_params.dpb[] */
> + __u8 index;
> +};

This introducing 3 bytes of padding at the end of this struct...

> +
> struct v4l2_ctrl_h264_slice_params {
> /* Size in bytes, including header */
> __u32 size;
> @@ -178,12 +193,8 @@ struct v4l2_ctrl_h264_slice_params {
> __u8 num_ref_idx_l1_active_minus1;
> __u32 slice_group_change_cycle;
>
> - /*
> - * Entries on each list are indices into
> - * v4l2_ctrl_h264_decode_params.dpb[].
> - */
> - __u8 ref_pic_list0[32];
> - __u8 ref_pic_list1[32];
> + struct v4l2_h264_reference ref_pic_list0[V4L2_H264_REF_LIST_LEN];
> + struct v4l2_h264_reference ref_pic_list1[V4L2_H264_REF_LIST_LEN];

...which leads to a lot of holes of uninitialized data in these arrays.

I recommend to replace 'enum v4l2_h264_field_reference fields;' in struct v4l2_h264_reference
by '__u8 fields; /* enum v4l2_h264_field_reference */'.

This also saves a lot of memory (2*32*6 = 384 bytes!) and avoids the padding issue.

Regards,

Hans

>
> __u32 flags;
> };
>

2020-08-20 09:14:09

by Hans Verkuil

[permalink] [raw]

Subject: Re: [PATCH v3 06/19] media: uapi: h264: Clean DPB entry interface

On 14/08/2020 15:36, Ezequiel Garcia wrote:
> As discussed recently, the current interface for the
> Decoded Picture Buffer is not enough to properly
> support field coding.
>
> This commit introduces enough semantics to support
> frame and field coding, and to signal how DPB entries
> are "used for reference".
>
> Signed-off-by: Ezequiel Garcia <[email protected]>
> ---
> v3:
> * Port to renamed types.
> v2:
> * Fix rkvdec usage of fields flags as noted by Jonas.
> ---
> .../media/v4l/ext-ctrls-codec.rst | 24 ++++++-------------
> drivers/media/v4l2-core/v4l2-h264.c | 4 ++--
> drivers/staging/media/rkvdec/rkvdec-h264.c | 17 ++++++-------
> include/media/h264-ctrls.h | 2 +-
> 4 files changed, 19 insertions(+), 28 deletions(-)
>
> diff --git a/Documentation/userspace-api/media/v4l/ext-ctrls-codec.rst b/Documentation/userspace-api/media/v4l/ext-ctrls-codec.rst
> index 714a8d9ae6a0..d14da8325382 100644
> --- a/Documentation/userspace-api/media/v4l/ext-ctrls-codec.rst
> +++ b/Documentation/userspace-api/media/v4l/ext-ctrls-codec.rst
> @@ -2063,6 +2063,9 @@ enum v4l2_mpeg_video_h264_hierarchical_coding_type -
> * - __s32
> - ``bottom_field_order_cnt``
> -
> + * - enum :c:type:`v4l2_h264_field_reference`
> + - ``reference``
> + - Specifies how the DPB entry is referenced.
> * - __u32
> - ``flags``
> - See :ref:`DPB Entry Flags <h264_dpb_flags>`
> @@ -2080,29 +2083,16 @@ enum v4l2_mpeg_video_h264_hierarchical_coding_type -
>
> * - ``V4L2_H264_DPB_ENTRY_FLAG_VALID``
> - 0x00000001
> - - The DPB entry is valid and should be considered
> + - The DPB entry is valid (non-empty) and should be considered.
> * - ``V4L2_H264_DPB_ENTRY_FLAG_ACTIVE``
> - 0x00000002
> - - The DPB entry is currently being used as a reference frame
> + - The DPB entry is used for reference.
> * - ``V4L2_H264_DPB_ENTRY_FLAG_LONG_TERM``
> - 0x00000004
> - - The DPB entry is a long term reference frame
> + - The DPB entry is used for long-term reference.
> * - ``V4L2_H264_DPB_ENTRY_FLAG_FIELD``
> - 0x00000008
> - - The DPB entry is a field reference, which means only one of the field
> - will be used when decoding the new frame/field. When not set the DPB
> - entry is a frame reference (both fields will be used). Note that this
> - flag does not say anything about the number of fields contained in the
> - reference frame, it just describes the one used to decode the new
> - field/frame
> - * - ``V4L2_H264_DPB_ENTRY_FLAG_BOTTOM_FIELD``
> - - 0x00000010
> - - The DPB entry is a bottom field reference (only the bottom field of the
> - reference frame is needed to decode the new frame/field). Only valid if
> - V4L2_H264_DPB_ENTRY_FLAG_FIELD is set. When
> - V4L2_H264_DPB_ENTRY_FLAG_FIELD is set but
> - V4L2_H264_DPB_ENTRY_FLAG_BOTTOM_FIELD is not, that means the
> - DPB entry is a top field reference
> + - The DPB entry is a single field or a complementary field pair.
>
> ``V4L2_CID_MPEG_VIDEO_H264_DECODE_MODE (enum)``
> Specifies the decoding mode to use. Currently exposes slice-based and
> diff --git a/drivers/media/v4l2-core/v4l2-h264.c b/drivers/media/v4l2-core/v4l2-h264.c
> index edf6225f0522..12b751c09016 100644
> --- a/drivers/media/v4l2-core/v4l2-h264.c
> +++ b/drivers/media/v4l2-core/v4l2-h264.c
> @@ -66,10 +66,10 @@ v4l2_h264_init_reflist_builder(struct v4l2_h264_reflist_builder *b,
> else
> b->refs[i].frame_num = dpb[i].frame_num;
>
> - if (!(dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_FIELD))
> + if (dpb[i].reference == V4L2_H264_FRAME_REF)
> pic_order_count = min(dpb[i].top_field_order_cnt,
> dpb[i].bottom_field_order_cnt);
> - else if (dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_BOTTOM_FIELD)
> + else if (dpb[i].reference & V4L2_H264_BOTTOM_FIELD_REF)
> pic_order_count = dpb[i].bottom_field_order_cnt;
> else
> pic_order_count = dpb[i].top_field_order_cnt;
> diff --git a/drivers/staging/media/rkvdec/rkvdec-h264.c b/drivers/staging/media/rkvdec/rkvdec-h264.c
> index 7b66e2743a4f..07a80e9a9df2 100644
> --- a/drivers/staging/media/rkvdec/rkvdec-h264.c
> +++ b/drivers/staging/media/rkvdec/rkvdec-h264.c
> @@ -949,16 +949,17 @@ static void config_registers(struct rkvdec_ctx *ctx,
> for (i = 0; i < ARRAY_SIZE(dec_params->dpb); i++) {
> struct vb2_buffer *vb_buf = get_ref_buf(ctx, run, i);
>
> - refer_addr = vb2_dma_contig_plane_dma_addr(vb_buf, 0) |
> - RKVDEC_COLMV_USED_FLAG_REF;
> + refer_addr = vb2_dma_contig_plane_dma_addr(vb_buf, 0);
>
> - if (!(dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_FIELD))
> - refer_addr |= RKVDEC_TOPFIELD_USED_REF |
> - RKVDEC_BOTFIELD_USED_REF;
> - else if (dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_BOTTOM_FIELD)
> - refer_addr |= RKVDEC_BOTFIELD_USED_REF;
> - else
> + if (dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_ACTIVE)
> + refer_addr |= RKVDEC_COLMV_USED_FLAG_REF;
> + if (dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_FIELD)
> + refer_addr |= RKVDEC_FIELD_REF;
> +
> + if (dpb[i].reference & V4L2_H264_TOP_FIELD_REF)
> refer_addr |= RKVDEC_TOPFIELD_USED_REF;
> + if (dpb[i].reference & V4L2_H264_BOTTOM_FIELD_REF)
> + refer_addr |= RKVDEC_BOTFIELD_USED_REF;
>
> writel_relaxed(dpb[i].top_field_order_cnt,
> rkvdec->regs + poc_reg_tbl_top_field[i]);
> diff --git a/include/media/h264-ctrls.h b/include/media/h264-ctrls.h
> index 9ff085fdc9ab..4447697e9465 100644
> --- a/include/media/h264-ctrls.h
> +++ b/include/media/h264-ctrls.h
> @@ -212,7 +212,6 @@ struct v4l2_ctrl_h264_slice_params {
> #define V4L2_H264_DPB_ENTRY_FLAG_ACTIVE 0x02
> #define V4L2_H264_DPB_ENTRY_FLAG_LONG_TERM 0x04
> #define V4L2_H264_DPB_ENTRY_FLAG_FIELD 0x08
> -#define V4L2_H264_DPB_ENTRY_FLAG_BOTTOM_FIELD 0x10
>
> struct v4l2_h264_dpb_entry {
> __u64 reference_ts;
> @@ -221,6 +220,7 @@ struct v4l2_h264_dpb_entry {
> /* Note that field is indicated by v4l2_buffer.field */
> __s32 top_field_order_cnt;
> __s32 bottom_field_order_cnt;
> + enum v4l2_h264_field_reference reference;

Here it is called 'references', but the same enum field in struct v4l2_h264_reference
is called 'fields' there. It's a good idea to use the same names.

Regards,

Hans

> __u32 flags; /* V4L2_H264_DPB_ENTRY_FLAG_* */
> };
>
>