drivers/dma-buf/dma-fence-array.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * dma-fence-array: aggregate fences to be waited together
   4  *
   5  * Copyright (C) 2016 Collabora Ltd
   6  * Copyright (C) 2016 Advanced Micro Devices, Inc.
   7  * Authors:
   8  *      Gustavo Padovan <[email protected]>
   9  *      Christian König <[email protected]>
  10  */
  11
  12 #include <linux/export.h>
  13 #include <linux/slab.h>
  14 #include <linux/dma-fence-array.h>
  15
  16 #define PENDING_ERROR 1
  17
  18 static const char *dma_fence_array_get_driver_name(struct dma_fence *fence)
  19 {
  20         return "dma_fence_array";
  21 }
  22
  23 static const char *dma_fence_array_get_timeline_name(struct dma_fence *fence)
  24 {
  25         return "unbound";
  26 }
  27
  28 static void dma_fence_array_set_pending_error(struct dma_fence_array *array,
  29                                               int error)
  30 {
  31         /*
  32          * Propagate the first error reported by any of our fences, but only
  33          * before we ourselves are signaled.
  34          */
  35         if (error)
  36                 cmpxchg(&array->base.error, PENDING_ERROR, error);
  37 }
  38
  39 static void dma_fence_array_clear_pending_error(struct dma_fence_array *array)
  40 {
  41         /* Clear the error flag if not actually set. */
  42         cmpxchg(&array->base.error, PENDING_ERROR, 0);
  43 }
  44
  45 static void irq_dma_fence_array_work(struct irq_work *wrk)
  46 {
  47         struct dma_fence_array *array = container_of(wrk, typeof(*array), work);
  48
  49         dma_fence_array_clear_pending_error(array);
  50
  51         dma_fence_signal(&array->base);
  52         dma_fence_put(&array->base);
  53 }
  54
  55 static void dma_fence_array_cb_func(struct dma_fence *f,
  56                                     struct dma_fence_cb *cb)
  57 {
  58         struct dma_fence_array_cb *array_cb =
  59                 container_of(cb, struct dma_fence_array_cb, cb);
  60         struct dma_fence_array *array = array_cb->array;
  61
  62         dma_fence_array_set_pending_error(array, f->error);
  63
  64         if (atomic_dec_and_test(&array->num_pending))
  65                 irq_work_queue(&array->work);
  66         else
  67                 dma_fence_put(&array->base);
  68 }
  69
  70 static bool dma_fence_array_enable_signaling(struct dma_fence *fence)
  71 {
  72         struct dma_fence_array *array = to_dma_fence_array(fence);
  73         struct dma_fence_array_cb *cb = array->callbacks;
  74         unsigned i;
  75
  76         for (i = 0; i < array->num_fences; ++i) {
  77                 cb[i].array = array;
  78                 /*
  79                  * As we may report that the fence is signaled before all
  80                  * callbacks are complete, we need to take an additional
  81                  * reference count on the array so that we do not free it too
  82                  * early. The core fence handling will only hold the reference
  83                  * until we signal the array as complete (but that is now
  84                  * insufficient).
  85                  */
  86                 dma_fence_get(&array->base);
  87                 if (dma_fence_add_callback(array->fences[i], &cb[i].cb,
  88                                            dma_fence_array_cb_func)) {
  89                         int error = array->fences[i]->error;
  90
  91                         dma_fence_array_set_pending_error(array, error);
  92                         dma_fence_put(&array->base);
  93                         if (atomic_dec_and_test(&array->num_pending)) {
  94                                 dma_fence_array_clear_pending_error(array);
  95                                 return false;
  96                         }
  97                 }
  98         }
  99
 100         return true;
 101 }
 102
 103 static bool dma_fence_array_signaled(struct dma_fence *fence)
 104 {
 105         struct dma_fence_array *array = to_dma_fence_array(fence);
 106         int num_pending;
 107         unsigned int i;
 108
 109         /*
 110          * We need to read num_pending before checking the enable_signal bit
 111          * to avoid racing with the enable_signaling() implementation, which
 112          * might decrement the counter, and cause a partial check.
 113          * atomic_read_acquire() pairs with atomic_dec_and_test() in
 114          * dma_fence_array_enable_signaling()
 115          *
 116          * The !--num_pending check is here to account for the any_signaled case
 117          * if we race with enable_signaling(), that means the !num_pending check
 118          * in the is_signalling_enabled branch might be outdated (num_pending
 119          * might have been decremented), but that's fine. The user will get the
 120          * right value when testing again later.
 121          */
 122         num_pending = atomic_read_acquire(&array->num_pending);
 123         if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &array->base.flags)) {
 124                 if (num_pending <= 0)
 125                         goto signal;
 126                 return false;
 127         }
 128
 129         for (i = 0; i < array->num_fences; ++i) {
 130                 if (dma_fence_is_signaled(array->fences[i]) && !--num_pending)
 131                         goto signal;
 132         }
 133         return false;
 134
 135 signal:
 136         dma_fence_array_clear_pending_error(array);
 137         return true;
 138 }
 139
 140 static void dma_fence_array_release(struct dma_fence *fence)
 141 {
 142         struct dma_fence_array *array = to_dma_fence_array(fence);
 143         unsigned i;
 144
 145         for (i = 0; i < array->num_fences; ++i)
 146                 dma_fence_put(array->fences[i]);
 147
 148         kfree(array->fences);
 149         dma_fence_free(fence);
 150 }
 151
 152 static void dma_fence_array_set_deadline(struct dma_fence *fence,
 153                                          ktime_t deadline)
 154 {
 155         struct dma_fence_array *array = to_dma_fence_array(fence);
 156         unsigned i;
 157
 158         for (i = 0; i < array->num_fences; ++i)
 159                 dma_fence_set_deadline(array->fences[i], deadline);
 160 }
 161
 162 const struct dma_fence_ops dma_fence_array_ops = {
 163         .get_driver_name = dma_fence_array_get_driver_name,
 164         .get_timeline_name = dma_fence_array_get_timeline_name,
 165         .enable_signaling = dma_fence_array_enable_signaling,
 166         .signaled = dma_fence_array_signaled,
 167         .release = dma_fence_array_release,
 168         .set_deadline = dma_fence_array_set_deadline,
 169 };
 170 EXPORT_SYMBOL(dma_fence_array_ops);
 171
 172 /**
 173  * dma_fence_array_alloc - Allocate a custom fence array
 174  * @num_fences:         [in]    number of fences to add in the array
 175  *
 176  * Return dma fence array on success, NULL on failure
 177  */
 178 struct dma_fence_array *dma_fence_array_alloc(int num_fences)
 179 {
 180         struct dma_fence_array *array;
 181
 182         return kzalloc(struct_size(array, callbacks, num_fences), GFP_KERNEL);
 183 }
 184 EXPORT_SYMBOL(dma_fence_array_alloc);
 185
 186 /**
 187  * dma_fence_array_init - Init a custom fence array
 188  * @array:              [in]    dma fence array to arm
 189  * @num_fences:         [in]    number of fences to add in the array
 190  * @fences:             [in]    array containing the fences
 191  * @context:            [in]    fence context to use
 192  * @seqno:              [in]    sequence number to use
 193  * @signal_on_any:      [in]    signal on any fence in the array
 194  *
 195  * Implementation of @dma_fence_array_create without allocation. Useful to init
 196  * a preallocated dma fence array in the path of reclaim or dma fence signaling.
 197  */
 198 void dma_fence_array_init(struct dma_fence_array *array,
 199                           int num_fences, struct dma_fence **fences,
 200                           u64 context, unsigned seqno,
 201                           bool signal_on_any)
 202 {
 203         WARN_ON(!num_fences || !fences);
 204
 205         array->num_fences = num_fences;
 206
 207         spin_lock_init(&array->lock);
 208         dma_fence_init(&array->base, &dma_fence_array_ops, &array->lock,
 209                        context, seqno);
 210         init_irq_work(&array->work, irq_dma_fence_array_work);
 211
 212         atomic_set(&array->num_pending, signal_on_any ? 1 : num_fences);
 213         array->fences = fences;
 214
 215         array->base.error = PENDING_ERROR;
 216
 217         /*
 218          * dma_fence_array objects should never contain any other fence
 219          * containers or otherwise we run into recursion and potential kernel
 220          * stack overflow on operations on the dma_fence_array.
 221          *
 222          * The correct way of handling this is to flatten out the array by the
 223          * caller instead.
 224          *
 225          * Enforce this here by checking that we don't create a dma_fence_array
 226          * with any container inside.
 227          */
 228         while (num_fences--)
 229                 WARN_ON(dma_fence_is_container(fences[num_fences]));
 230 }
 231 EXPORT_SYMBOL(dma_fence_array_init);
 232
 233 /**
 234  * dma_fence_array_create - Create a custom fence array
 235  * @num_fences:         [in]    number of fences to add in the array
 236  * @fences:             [in]    array containing the fences
 237  * @context:            [in]    fence context to use
 238  * @seqno:              [in]    sequence number to use
 239  * @signal_on_any:      [in]    signal on any fence in the array
 240  *
 241  * Allocate a dma_fence_array object and initialize the base fence with
 242  * dma_fence_init().
 243  * In case of error it returns NULL.
 244  *
 245  * The caller should allocate the fences array with num_fences size
 246  * and fill it with the fences it wants to add to the object. Ownership of this
 247  * array is taken and dma_fence_put() is used on each fence on release.
 248  *
 249  * If @signal_on_any is true the fence array signals if any fence in the array
 250  * signals, otherwise it signals when all fences in the array signal.
 251  */
 252 struct dma_fence_array *dma_fence_array_create(int num_fences,
 253                                                struct dma_fence **fences,
 254                                                u64 context, unsigned seqno,
 255                                                bool signal_on_any)
 256 {
 257         struct dma_fence_array *array;
 258
 259         array = dma_fence_array_alloc(num_fences);
 260         if (!array)
 261                 return NULL;
 262
 263         dma_fence_array_init(array, num_fences, fences,
 264                              context, seqno, signal_on_any);
 265
 266         return array;
 267 }
 268 EXPORT_SYMBOL(dma_fence_array_create);
 269
 270 /**
 271  * dma_fence_match_context - Check if all fences are from the given context
 272  * @fence:              [in]    fence or fence array
 273  * @context:            [in]    fence context to check all fences against
 274  *
 275  * Checks the provided fence or, for a fence array, all fences in the array
 276  * against the given context. Returns false if any fence is from a different
 277  * context.
 278  */
 279 bool dma_fence_match_context(struct dma_fence *fence, u64 context)
 280 {
 281         struct dma_fence_array *array = to_dma_fence_array(fence);
 282         unsigned i;
 283
 284         if (!dma_fence_is_array(fence))
 285                 return fence->context == context;
 286
 287         for (i = 0; i < array->num_fences; i++) {
 288                 if (array->fences[i]->context != context)
 289                         return false;
 290         }
 291
 292         return true;
 293 }
 294 EXPORT_SYMBOL(dma_fence_match_context);
 295
 296 struct dma_fence *dma_fence_array_first(struct dma_fence *head)
 297 {
 298         struct dma_fence_array *array;
 299
 300         if (!head)
 301                 return NULL;
 302
 303         array = to_dma_fence_array(head);
 304         if (!array)
 305                 return head;
 306
 307         if (!array->num_fences)
 308                 return NULL;
 309
 310         return array->fences[0];
 311 }
 312 EXPORT_SYMBOL(dma_fence_array_first);
 313
 314 struct dma_fence *dma_fence_array_next(struct dma_fence *head,
 315                                        unsigned int index)
 316 {
 317         struct dma_fence_array *array = to_dma_fence_array(head);
 318
 319         if (!array || index >= array->num_fences)
 320                 return NULL;
 321
 322         return array->fences[index];
 323 }
 324 EXPORT_SYMBOL(dma_fence_array_next);