target/linux/bcm27xx/patches-5.4/950-0500-media-v4l2-mem2mem-allow-request-job-buffer-processi.patch

   1 From b8ae9d55d468a9f55524296247dba93531c29c99 Mon Sep 17 00:00:00 2001
   2 From: John Cox <jc@kynesim.co.uk>
   3 Date: Thu, 5 Mar 2020 14:46:54 +0000
   4 Subject: [PATCH] media: v4l2-mem2mem: allow request job buffer
   5  processing after job finish
   6
   7 Allow the capture buffer to be detached from a v4l2 request job such
   8 that another job can start before the capture buffer is returned. This
   9 allows h/w codecs that can process multiple requests at the same time
  10 to operate more efficiently.
  11
  12 Signed-off-by: John Cox <jc@kynesim.co.uk>
  13 ---
  14  drivers/media/v4l2-core/v4l2-mem2mem.c | 105 +++++++++++++++++++++++--
  15  include/media/v4l2-mem2mem.h           |  47 +++++++++++
  16  include/media/videobuf2-v4l2.h         |   3 +
  17  3 files changed, 149 insertions(+), 6 deletions(-)
  18
  19 --- a/drivers/media/v4l2-core/v4l2-mem2mem.c
  20 +++ b/drivers/media/v4l2-core/v4l2-mem2mem.c
  21 @@ -399,15 +399,18 @@ static void v4l2_m2m_cancel_job(struct v
  22  {
  23         struct v4l2_m2m_dev *m2m_dev;
  24         unsigned long flags;
  25 +       bool det_abort_req;
  26
  27         m2m_dev = m2m_ctx->m2m_dev;
  28         spin_lock_irqsave(&m2m_dev->job_spinlock, flags);
  29
  30 +       det_abort_req = !list_empty(&m2m_ctx->det_list);
  31         m2m_ctx->job_flags |= TRANS_ABORT;
  32         if (m2m_ctx->job_flags & TRANS_RUNNING) {
  33                 spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
  34                 if (m2m_dev->m2m_ops->job_abort)
  35                         m2m_dev->m2m_ops->job_abort(m2m_ctx->priv);
  36 +               det_abort_req = false;
  37                 dprintk("m2m_ctx %p running, will wait to complete\n", m2m_ctx);
  38                 wait_event(m2m_ctx->finished,
  39                                 !(m2m_ctx->job_flags & TRANS_RUNNING));
  40 @@ -421,6 +424,11 @@ static void v4l2_m2m_cancel_job(struct v
  41                 /* Do nothing, was not on queue/running */
  42                 spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
  43         }
  44 +
  45 +       /* Wait for detached buffers to come back too */
  46 +       if (det_abort_req && m2m_dev->m2m_ops->job_abort)
  47 +               m2m_dev->m2m_ops->job_abort(m2m_ctx->priv);
  48 +       wait_event(m2m_ctx->det_empty, list_empty(&m2m_ctx->det_list));
  49  }
  50
  51  /*
  52 @@ -458,6 +466,7 @@ static bool _v4l2_m2m_job_finish(struct
  53
  54         list_del(&m2m_dev->curr_ctx->queue);
  55         m2m_dev->curr_ctx->job_flags &= ~(TRANS_QUEUED | TRANS_RUNNING);
  56 +       m2m_ctx->cap_detached = false;
  57         wake_up(&m2m_dev->curr_ctx->finished);
  58         m2m_dev->curr_ctx = NULL;
  59         return true;
  60 @@ -485,6 +494,80 @@ void v4l2_m2m_job_finish(struct v4l2_m2m
  61  }
  62  EXPORT_SYMBOL(v4l2_m2m_job_finish);
  63
  64 +struct vb2_v4l2_buffer *_v4l2_m2m_cap_buf_detach(struct v4l2_m2m_ctx *m2m_ctx)
  65 +{
  66 +       struct vb2_v4l2_buffer *buf;
  67 +
  68 +       buf = v4l2_m2m_dst_buf_remove(m2m_ctx);
  69 +       list_add_tail(&container_of(buf, struct v4l2_m2m_buffer, vb)->list,
  70 +                     &m2m_ctx->det_list);
  71 +       m2m_ctx->cap_detached = true;
  72 +       buf->is_held = true;
  73 +       buf->det_state = VB2_BUF_STATE_ACTIVE;
  74 +
  75 +       return buf;
  76 +}
  77 +
  78 +struct vb2_v4l2_buffer *v4l2_m2m_cap_buf_detach(struct v4l2_m2m_dev *m2m_dev,
  79 +                                               struct v4l2_m2m_ctx *m2m_ctx)
  80 +{
  81 +       unsigned long flags;
  82 +       struct vb2_v4l2_buffer *src_buf, *dst_buf;
  83 +
  84 +       spin_lock_irqsave(&m2m_dev->job_spinlock, flags);
  85 +
  86 +       dst_buf = NULL;
  87 +       src_buf = v4l2_m2m_next_src_buf(m2m_ctx);
  88 +
  89 +       if (!(src_buf->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF) &&
  90 +           !m2m_ctx->cap_detached)
  91 +               dst_buf = _v4l2_m2m_cap_buf_detach(m2m_ctx);
  92 +
  93 +       spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
  94 +       return dst_buf;
  95 +}
  96 +EXPORT_SYMBOL(v4l2_m2m_cap_buf_detach);
  97 +
  98 +static void _v4l2_m2m_cap_buf_return(struct v4l2_m2m_ctx *m2m_ctx,
  99 +                                    struct vb2_v4l2_buffer *buf,
 100 +                                    enum vb2_buffer_state state)
 101 +{
 102 +       buf->det_state = state;
 103 +
 104 +       /*
 105 +        * Always signal done in the order we got stuff
 106 +        * Stop if we find a buf that is still in use
 107 +        */
 108 +       while (!list_empty(&m2m_ctx->det_list)) {
 109 +               buf = &list_first_entry(&m2m_ctx->det_list,
 110 +                                       struct v4l2_m2m_buffer, list)->vb;
 111 +               state = buf->det_state;
 112 +               if (state != VB2_BUF_STATE_DONE &&
 113 +                   state != VB2_BUF_STATE_ERROR)
 114 +                       return;
 115 +               list_del(&container_of(buf, struct v4l2_m2m_buffer, vb)->list);
 116 +               buf->det_state = VB2_BUF_STATE_DEQUEUED;
 117 +               v4l2_m2m_buf_done(buf, state);
 118 +       }
 119 +       wake_up(&m2m_ctx->det_empty);
 120 +}
 121 +
 122 +void v4l2_m2m_cap_buf_return(struct v4l2_m2m_dev *m2m_dev,
 123 +                            struct v4l2_m2m_ctx *m2m_ctx,
 124 +                            struct vb2_v4l2_buffer *buf,
 125 +                            enum vb2_buffer_state state)
 126 +{
 127 +       unsigned long flags;
 128 +
 129 +       if (!buf)
 130 +               return;
 131 +
 132 +       spin_lock_irqsave(&m2m_dev->job_spinlock, flags);
 133 +       _v4l2_m2m_cap_buf_return(m2m_ctx, buf, state);
 134 +       spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
 135 +}
 136 +EXPORT_SYMBOL(v4l2_m2m_cap_buf_return);
 137 +
 138  void v4l2_m2m_buf_done_and_job_finish(struct v4l2_m2m_dev *m2m_dev,
 139                                       struct v4l2_m2m_ctx *m2m_ctx,
 140                                       enum vb2_buffer_state state)
 141 @@ -495,15 +578,23 @@ void v4l2_m2m_buf_done_and_job_finish(st
 142
 143         spin_lock_irqsave(&m2m_dev->job_spinlock, flags);
 144         src_buf = v4l2_m2m_src_buf_remove(m2m_ctx);
 145 -       dst_buf = v4l2_m2m_next_dst_buf(m2m_ctx);
 146
 147 -       if (WARN_ON(!src_buf || !dst_buf))
 148 +       if (WARN_ON(!src_buf))
 149                 goto unlock;
 150         v4l2_m2m_buf_done(src_buf, state);
 151 -       dst_buf->is_held = src_buf->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF;
 152 -       if (!dst_buf->is_held) {
 153 -               v4l2_m2m_dst_buf_remove(m2m_ctx);
 154 -               v4l2_m2m_buf_done(dst_buf, state);
 155 +
 156 +       if (!m2m_ctx->cap_detached) {
 157 +               dst_buf = v4l2_m2m_next_dst_buf(m2m_ctx);
 158 +               if (WARN_ON(!dst_buf))
 159 +                       goto unlock;
 160 +
 161 +               dst_buf->is_held = src_buf->flags
 162 +                                   & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF;
 163 +
 164 +               if (!dst_buf->is_held) {
 165 +                       dst_buf = _v4l2_m2m_cap_buf_detach(m2m_ctx);
 166 +                       _v4l2_m2m_cap_buf_return(m2m_ctx, dst_buf, state);
 167 +               }
 168         }
 169         schedule_next = _v4l2_m2m_job_finish(m2m_dev, m2m_ctx);
 170  unlock:
 171 @@ -981,12 +1072,14 @@ struct v4l2_m2m_ctx *v4l2_m2m_ctx_init(s
 172         m2m_ctx->priv = drv_priv;
 173         m2m_ctx->m2m_dev = m2m_dev;
 174         init_waitqueue_head(&m2m_ctx->finished);
 175 +       init_waitqueue_head(&m2m_ctx->det_empty);
 176
 177         out_q_ctx = &m2m_ctx->out_q_ctx;
 178         cap_q_ctx = &m2m_ctx->cap_q_ctx;
 179
 180         INIT_LIST_HEAD(&out_q_ctx->rdy_queue);
 181         INIT_LIST_HEAD(&cap_q_ctx->rdy_queue);
 182 +       INIT_LIST_HEAD(&m2m_ctx->det_list);
 183         spin_lock_init(&out_q_ctx->rdy_spinlock);
 184         spin_lock_init(&cap_q_ctx->rdy_spinlock);
 185
 186 --- a/include/media/v4l2-mem2mem.h
 187 +++ b/include/media/v4l2-mem2mem.h
 188 @@ -88,6 +88,9 @@ struct v4l2_m2m_queue_ctx {
 189   *             %TRANS_QUEUED, %TRANS_RUNNING and %TRANS_ABORT.
 190   * @finished: Wait queue used to signalize when a job queue finished.
 191   * @priv: Instance private data
 192 + * @cap_detached: Current job's capture buffer has been detached
 193 + * @det_list: List of detached (post-job but still in flight) capture buffers
 194 + * @det_empty: Wait queue signalled when det_list goes empty
 195   *
 196   * The memory to memory context is specific to a file handle, NOT to e.g.
 197   * a device.
 198 @@ -111,6 +114,11 @@ struct v4l2_m2m_ctx {
 199         wait_queue_head_t               finished;
 200
 201         void                            *priv;
 202 +
 203 +       /* Detached buffer handling */
 204 +       bool    cap_detached;
 205 +       struct list_head                det_list;
 206 +       wait_queue_head_t               det_empty;
 207  };
 208
 209  /**
 210 @@ -216,6 +224,45 @@ v4l2_m2m_buf_done(struct vb2_v4l2_buffer
 211  }
 212
 213  /**
 214 + * v4l2_m2m_cap_buf_detach() - detach the capture buffer from the job and
 215 + * return it.
 216 + *
 217 + * @m2m_dev: opaque pointer to the internal data to handle M2M context
 218 + * @m2m_ctx: m2m context assigned to the instance given by struct &v4l2_m2m_ctx
 219 + *
 220 + * This function is designed to be used in conjunction with
 221 + * v4l2_m2m_buf_done_and_job_finish(). It allows the next job to start
 222 + * execution before the capture buffer is returned to the user which can be
 223 + * important if the underlying processing has multiple phases that are more
 224 + * efficiently executed in parallel.
 225 + *
 226 + * If used then it must be called before v4l2_m2m_buf_done_and_job_finish()
 227 + * as otherwise the buffer will have already gone.
 228 + *
 229 + * It is the callers reponsibilty to ensure that all detached buffers are
 230 + * returned.
 231 + */
 232 +struct vb2_v4l2_buffer *v4l2_m2m_cap_buf_detach(struct v4l2_m2m_dev *m2m_dev,
 233 +                                               struct v4l2_m2m_ctx *m2m_ctx);
 234 +
 235 +/**
 236 + * v4l2_m2m_cap_buf_return() - return a capture buffer, previously detached
 237 + * with v4l2_m2m_cap_buf_detach() to the user.
 238 + *
 239 + * @m2m_dev: opaque pointer to the internal data to handle M2M context
 240 + * @m2m_ctx: m2m context assigned to the instance given by struct &v4l2_m2m_ctx
 241 + * @buf: the buffer to return
 242 + * @state: vb2 buffer state passed to v4l2_m2m_buf_done().
 243 + *
 244 + * Buffers returned by this function will be returned to the user in the order
 245 + * of the original jobs rather than the order in which this function is called.
 246 + */
 247 +void v4l2_m2m_cap_buf_return(struct v4l2_m2m_dev *m2m_dev,
 248 +                            struct v4l2_m2m_ctx *m2m_ctx,
 249 +                            struct vb2_v4l2_buffer *buf,
 250 +                            enum vb2_buffer_state state);
 251 +
 252 +/**
 253   * v4l2_m2m_reqbufs() - multi-queue-aware REQBUFS multiplexer
 254   *
 255   * @file: pointer to struct &file
 256 --- a/include/media/videobuf2-v4l2.h
 257 +++ b/include/media/videobuf2-v4l2.h
 258 @@ -35,6 +35,8 @@
 259   * @request_fd:        the request_fd associated with this buffer
 260   * @is_held:   if true, then this capture buffer was held
 261   * @planes:    plane information (userptr/fd, length, bytesused, data_offset).
 262 + * @det_state: if a detached request capture buffer then this contains its
 263 + *             current state
 264   *
 265   * Should contain enough information to be able to cover all the fields
 266   * of &struct v4l2_buffer at ``videodev2.h``.
 267 @@ -49,6 +51,7 @@ struct vb2_v4l2_buffer {
 268         __s32                   request_fd;
 269         bool                    is_held;
 270         struct vb2_plane        planes[VB2_MAX_PLANES];
 271 +       enum vb2_buffer_state   det_state;
 272  };
 273
 274  /* VB2 V4L2 flags as set in vb2_queue.subsystem_flags */