drivers/gpu/drm/v3d/v3d_gem.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /* Copyright (C) 2014-2018 Broadcom */
   3
   4 #include <linux/device.h>
   5 #include <linux/dma-mapping.h>
   6 #include <linux/io.h>
   7 #include <linux/module.h>
   8 #include <linux/platform_device.h>
   9 #include <linux/reset.h>
  10 #include <linux/sched/signal.h>
  11 #include <linux/uaccess.h>
  12
  13 #include <drm/drm_syncobj.h>
  14 #include <uapi/drm/v3d_drm.h>
  15
  16 #include "v3d_drv.h"
  17 #include "v3d_regs.h"
  18 #include "v3d_trace.h"
  19
  20 static void
  21 v3d_init_core(struct v3d_dev *v3d, int core)
  22 {
  23         /* Set OVRTMUOUT, which means that the texture sampler uniform
  24          * configuration's tmu output type field is used, instead of
  25          * using the hardware default behavior based on the texture
  26          * type.  If you want the default behavior, you can still put
  27          * "2" in the indirect texture state's output_type field.
  28          */
  29         if (v3d->ver < 40)
  30                 V3D_CORE_WRITE(core, V3D_CTL_MISCCFG, V3D_MISCCFG_OVRTMUOUT);
  31
  32         /* Whenever we flush the L2T cache, we always want to flush
  33          * the whole thing.
  34          */
  35         V3D_CORE_WRITE(core, V3D_CTL_L2TFLSTA, 0);
  36         V3D_CORE_WRITE(core, V3D_CTL_L2TFLEND, ~0);
  37 }
  38
  39 /* Sets invariant state for the HW. */
  40 static void
  41 v3d_init_hw_state(struct v3d_dev *v3d)
  42 {
  43         v3d_init_core(v3d, 0);
  44 }
  45
  46 static void
  47 v3d_idle_axi(struct v3d_dev *v3d, int core)
  48 {
  49         V3D_CORE_WRITE(core, V3D_GMP_CFG, V3D_GMP_CFG_STOP_REQ);
  50
  51         if (wait_for((V3D_CORE_READ(core, V3D_GMP_STATUS) &
  52                       (V3D_GMP_STATUS_RD_COUNT_MASK |
  53                        V3D_GMP_STATUS_WR_COUNT_MASK |
  54                        V3D_GMP_STATUS_CFG_BUSY)) == 0, 100)) {
  55                 DRM_ERROR("Failed to wait for safe GMP shutdown\n");
  56         }
  57 }
  58
  59 static void
  60 v3d_idle_gca(struct v3d_dev *v3d)
  61 {
  62         if (v3d->ver >= 41)
  63                 return;
  64
  65         V3D_GCA_WRITE(V3D_GCA_SAFE_SHUTDOWN, V3D_GCA_SAFE_SHUTDOWN_EN);
  66
  67         if (wait_for((V3D_GCA_READ(V3D_GCA_SAFE_SHUTDOWN_ACK) &
  68                       V3D_GCA_SAFE_SHUTDOWN_ACK_ACKED) ==
  69                      V3D_GCA_SAFE_SHUTDOWN_ACK_ACKED, 100)) {
  70                 DRM_ERROR("Failed to wait for safe GCA shutdown\n");
  71         }
  72 }
  73
  74 static void
  75 v3d_reset_by_bridge(struct v3d_dev *v3d)
  76 {
  77         int version = V3D_BRIDGE_READ(V3D_TOP_GR_BRIDGE_REVISION);
  78
  79         if (V3D_GET_FIELD(version, V3D_TOP_GR_BRIDGE_MAJOR) == 2) {
  80                 V3D_BRIDGE_WRITE(V3D_TOP_GR_BRIDGE_SW_INIT_0,
  81                                  V3D_TOP_GR_BRIDGE_SW_INIT_0_V3D_CLK_108_SW_INIT);
  82                 V3D_BRIDGE_WRITE(V3D_TOP_GR_BRIDGE_SW_INIT_0, 0);
  83
  84                 /* GFXH-1383: The SW_INIT may cause a stray write to address 0
  85                  * of the unit, so reset it to its power-on value here.
  86                  */
  87                 V3D_WRITE(V3D_HUB_AXICFG, V3D_HUB_AXICFG_MAX_LEN_MASK);
  88         } else {
  89                 WARN_ON_ONCE(V3D_GET_FIELD(version,
  90                                            V3D_TOP_GR_BRIDGE_MAJOR) != 7);
  91                 V3D_BRIDGE_WRITE(V3D_TOP_GR_BRIDGE_SW_INIT_1,
  92                                  V3D_TOP_GR_BRIDGE_SW_INIT_1_V3D_CLK_108_SW_INIT);
  93                 V3D_BRIDGE_WRITE(V3D_TOP_GR_BRIDGE_SW_INIT_1, 0);
  94         }
  95 }
  96
  97 static void
  98 v3d_reset_v3d(struct v3d_dev *v3d)
  99 {
 100         if (v3d->reset)
 101                 reset_control_reset(v3d->reset);
 102         else
 103                 v3d_reset_by_bridge(v3d);
 104
 105         v3d_init_hw_state(v3d);
 106 }
 107
 108 void
 109 v3d_reset(struct v3d_dev *v3d)
 110 {
 111         struct drm_device *dev = &v3d->drm;
 112
 113         DRM_DEV_ERROR(dev->dev, "Resetting GPU for hang.\n");
 114         DRM_DEV_ERROR(dev->dev, "V3D_ERR_STAT: 0x%08x\n",
 115                       V3D_CORE_READ(0, V3D_ERR_STAT));
 116         trace_v3d_reset_begin(dev);
 117
 118         /* XXX: only needed for safe powerdown, not reset. */
 119         if (false)
 120                 v3d_idle_axi(v3d, 0);
 121
 122         v3d_idle_gca(v3d);
 123         v3d_reset_v3d(v3d);
 124
 125         v3d_mmu_set_page_table(v3d);
 126         v3d_irq_reset(v3d);
 127
 128         v3d_perfmon_stop(v3d, v3d->active_perfmon, false);
 129
 130         trace_v3d_reset_end(dev);
 131 }
 132
 133 static void
 134 v3d_flush_l3(struct v3d_dev *v3d)
 135 {
 136         if (v3d->ver < 41) {
 137                 u32 gca_ctrl = V3D_GCA_READ(V3D_GCA_CACHE_CTRL);
 138
 139                 V3D_GCA_WRITE(V3D_GCA_CACHE_CTRL,
 140                               gca_ctrl | V3D_GCA_CACHE_CTRL_FLUSH);
 141
 142                 if (v3d->ver < 33) {
 143                         V3D_GCA_WRITE(V3D_GCA_CACHE_CTRL,
 144                                       gca_ctrl & ~V3D_GCA_CACHE_CTRL_FLUSH);
 145                 }
 146         }
 147 }
 148
 149 /* Invalidates the (read-only) L2C cache.  This was the L2 cache for
 150  * uniforms and instructions on V3D 3.2.
 151  */
 152 static void
 153 v3d_invalidate_l2c(struct v3d_dev *v3d, int core)
 154 {
 155         if (v3d->ver > 32)
 156                 return;
 157
 158         V3D_CORE_WRITE(core, V3D_CTL_L2CACTL,
 159                        V3D_L2CACTL_L2CCLR |
 160                        V3D_L2CACTL_L2CENA);
 161 }
 162
 163 /* Invalidates texture L2 cachelines */
 164 static void
 165 v3d_flush_l2t(struct v3d_dev *v3d, int core)
 166 {
 167         /* While there is a busy bit (V3D_L2TCACTL_L2TFLS), we don't
 168          * need to wait for completion before dispatching the job --
 169          * L2T accesses will be stalled until the flush has completed.
 170          * However, we do need to make sure we don't try to trigger a
 171          * new flush while the L2_CLEAN queue is trying to
 172          * synchronously clean after a job.
 173          */
 174         mutex_lock(&v3d->cache_clean_lock);
 175         V3D_CORE_WRITE(core, V3D_CTL_L2TCACTL,
 176                        V3D_L2TCACTL_L2TFLS |
 177                        V3D_SET_FIELD(V3D_L2TCACTL_FLM_FLUSH, V3D_L2TCACTL_FLM));
 178         mutex_unlock(&v3d->cache_clean_lock);
 179 }
 180
 181 /* Cleans texture L1 and L2 cachelines (writing back dirty data).
 182  *
 183  * For cleaning, which happens from the CACHE_CLEAN queue after CSD has
 184  * executed, we need to make sure that the clean is done before
 185  * signaling job completion.  So, we synchronously wait before
 186  * returning, and we make sure that L2 invalidates don't happen in the
 187  * meantime to confuse our are-we-done checks.
 188  */
 189 void
 190 v3d_clean_caches(struct v3d_dev *v3d)
 191 {
 192         struct drm_device *dev = &v3d->drm;
 193         int core = 0;
 194
 195         trace_v3d_cache_clean_begin(dev);
 196
 197         V3D_CORE_WRITE(core, V3D_CTL_L2TCACTL, V3D_L2TCACTL_TMUWCF);
 198         if (wait_for(!(V3D_CORE_READ(core, V3D_CTL_L2TCACTL) &
 199                        V3D_L2TCACTL_TMUWCF), 100)) {
 200                 DRM_ERROR("Timeout waiting for TMU write combiner flush\n");
 201         }
 202
 203         mutex_lock(&v3d->cache_clean_lock);
 204         V3D_CORE_WRITE(core, V3D_CTL_L2TCACTL,
 205                        V3D_L2TCACTL_L2TFLS |
 206                        V3D_SET_FIELD(V3D_L2TCACTL_FLM_CLEAN, V3D_L2TCACTL_FLM));
 207
 208         if (wait_for(!(V3D_CORE_READ(core, V3D_CTL_L2TCACTL) &
 209                        V3D_L2TCACTL_L2TFLS), 100)) {
 210                 DRM_ERROR("Timeout waiting for L2T clean\n");
 211         }
 212
 213         mutex_unlock(&v3d->cache_clean_lock);
 214
 215         trace_v3d_cache_clean_end(dev);
 216 }
 217
 218 /* Invalidates the slice caches.  These are read-only caches. */
 219 static void
 220 v3d_invalidate_slices(struct v3d_dev *v3d, int core)
 221 {
 222         V3D_CORE_WRITE(core, V3D_CTL_SLCACTL,
 223                        V3D_SET_FIELD(0xf, V3D_SLCACTL_TVCCS) |
 224                        V3D_SET_FIELD(0xf, V3D_SLCACTL_TDCCS) |
 225                        V3D_SET_FIELD(0xf, V3D_SLCACTL_UCC) |
 226                        V3D_SET_FIELD(0xf, V3D_SLCACTL_ICC));
 227 }
 228
 229 void
 230 v3d_invalidate_caches(struct v3d_dev *v3d)
 231 {
 232         /* Invalidate the caches from the outside in.  That way if
 233          * another CL's concurrent use of nearby memory were to pull
 234          * an invalidated cacheline back in, we wouldn't leave stale
 235          * data in the inner cache.
 236          */
 237         v3d_flush_l3(v3d);
 238         v3d_invalidate_l2c(v3d, 0);
 239         v3d_flush_l2t(v3d, 0);
 240         v3d_invalidate_slices(v3d, 0);
 241 }
 242
 243 /* Takes the reservation lock on all the BOs being referenced, so that
 244  * at queue submit time we can update the reservations.
 245  *
 246  * We don't lock the RCL the tile alloc/state BOs, or overflow memory
 247  * (all of which are on exec->unref_list).  They're entirely private
 248  * to v3d, so we don't attach dma-buf fences to them.
 249  */
 250 static int
 251 v3d_lock_bo_reservations(struct v3d_job *job,
 252                          struct ww_acquire_ctx *acquire_ctx)
 253 {
 254         int i, ret;
 255
 256         ret = drm_gem_lock_reservations(job->bo, job->bo_count, acquire_ctx);
 257         if (ret)
 258                 return ret;
 259
 260         for (i = 0; i < job->bo_count; i++) {
 261                 ret = dma_resv_reserve_fences(job->bo[i]->resv, 1);
 262                 if (ret)
 263                         goto fail;
 264
 265                 ret = drm_sched_job_add_implicit_dependencies(&job->base,
 266                                                               job->bo[i], true);
 267                 if (ret)
 268                         goto fail;
 269         }
 270
 271         return 0;
 272
 273 fail:
 274         drm_gem_unlock_reservations(job->bo, job->bo_count, acquire_ctx);
 275         return ret;
 276 }
 277
 278 /**
 279  * v3d_lookup_bos() - Sets up job->bo[] with the GEM objects
 280  * referenced by the job.
 281  * @dev: DRM device
 282  * @file_priv: DRM file for this fd
 283  * @job: V3D job being set up
 284  * @bo_handles: GEM handles
 285  * @bo_count: Number of GEM handles passed in
 286  *
 287  * The command validator needs to reference BOs by their index within
 288  * the submitted job's BO list.  This does the validation of the job's
 289  * BO list and reference counting for the lifetime of the job.
 290  *
 291  * Note that this function doesn't need to unreference the BOs on
 292  * failure, because that will happen at v3d_exec_cleanup() time.
 293  */
 294 static int
 295 v3d_lookup_bos(struct drm_device *dev,
 296                struct drm_file *file_priv,
 297                struct v3d_job *job,
 298                u64 bo_handles,
 299                u32 bo_count)
 300 {
 301         u32 *handles;
 302         int ret = 0;
 303         int i;
 304
 305         job->bo_count = bo_count;
 306
 307         if (!job->bo_count) {
 308                 /* See comment on bo_index for why we have to check
 309                  * this.
 310                  */
 311                 DRM_DEBUG("Rendering requires BOs\n");
 312                 return -EINVAL;
 313         }
 314
 315         job->bo = kvmalloc_array(job->bo_count,
 316                                  sizeof(struct drm_gem_cma_object *),
 317                                  GFP_KERNEL | __GFP_ZERO);
 318         if (!job->bo) {
 319                 DRM_DEBUG("Failed to allocate validated BO pointers\n");
 320                 return -ENOMEM;
 321         }
 322
 323         handles = kvmalloc_array(job->bo_count, sizeof(u32), GFP_KERNEL);
 324         if (!handles) {
 325                 ret = -ENOMEM;
 326                 DRM_DEBUG("Failed to allocate incoming GEM handles\n");
 327                 goto fail;
 328         }
 329
 330         if (copy_from_user(handles,
 331                            (void __user *)(uintptr_t)bo_handles,
 332                            job->bo_count * sizeof(u32))) {
 333                 ret = -EFAULT;
 334                 DRM_DEBUG("Failed to copy in GEM handles\n");
 335                 goto fail;
 336         }
 337
 338         spin_lock(&file_priv->table_lock);
 339         for (i = 0; i < job->bo_count; i++) {
 340                 struct drm_gem_object *bo = idr_find(&file_priv->object_idr,
 341                                                      handles[i]);
 342                 if (!bo) {
 343                         DRM_DEBUG("Failed to look up GEM BO %d: %d\n",
 344                                   i, handles[i]);
 345                         ret = -ENOENT;
 346                         spin_unlock(&file_priv->table_lock);
 347                         goto fail;
 348                 }
 349                 drm_gem_object_get(bo);
 350                 job->bo[i] = bo;
 351         }
 352         spin_unlock(&file_priv->table_lock);
 353
 354 fail:
 355         kvfree(handles);
 356         return ret;
 357 }
 358
 359 static void
 360 v3d_job_free(struct kref *ref)
 361 {
 362         struct v3d_job *job = container_of(ref, struct v3d_job, refcount);
 363         int i;
 364
 365         for (i = 0; i < job->bo_count; i++) {
 366                 if (job->bo[i])
 367                         drm_gem_object_put(job->bo[i]);
 368         }
 369         kvfree(job->bo);
 370
 371         dma_fence_put(job->irq_fence);
 372         dma_fence_put(job->done_fence);
 373
 374         if (job->perfmon)
 375                 v3d_perfmon_put(job->perfmon);
 376
 377         kfree(job);
 378 }
 379
 380 static void
 381 v3d_render_job_free(struct kref *ref)
 382 {
 383         struct v3d_render_job *job = container_of(ref, struct v3d_render_job,
 384                                                   base.refcount);
 385         struct v3d_bo *bo, *save;
 386
 387         list_for_each_entry_safe(bo, save, &job->unref_list, unref_head) {
 388                 drm_gem_object_put(&bo->base.base);
 389         }
 390
 391         v3d_job_free(ref);
 392 }
 393
 394 void v3d_job_cleanup(struct v3d_job *job)
 395 {
 396         if (!job)
 397                 return;
 398
 399         drm_sched_job_cleanup(&job->base);
 400         v3d_job_put(job);
 401 }
 402
 403 void v3d_job_put(struct v3d_job *job)
 404 {
 405         kref_put(&job->refcount, job->free);
 406 }
 407
 408 int
 409 v3d_wait_bo_ioctl(struct drm_device *dev, void *data,
 410                   struct drm_file *file_priv)
 411 {
 412         int ret;
 413         struct drm_v3d_wait_bo *args = data;
 414         ktime_t start = ktime_get();
 415         u64 delta_ns;
 416         unsigned long timeout_jiffies =
 417                 nsecs_to_jiffies_timeout(args->timeout_ns);
 418
 419         if (args->pad != 0)
 420                 return -EINVAL;
 421
 422         ret = drm_gem_dma_resv_wait(file_priv, args->handle,
 423                                     true, timeout_jiffies);
 424
 425         /* Decrement the user's timeout, in case we got interrupted
 426          * such that the ioctl will be restarted.
 427          */
 428         delta_ns = ktime_to_ns(ktime_sub(ktime_get(), start));
 429         if (delta_ns < args->timeout_ns)
 430                 args->timeout_ns -= delta_ns;
 431         else
 432                 args->timeout_ns = 0;
 433
 434         /* Asked to wait beyond the jiffie/scheduler precision? */
 435         if (ret == -ETIME && args->timeout_ns)
 436                 ret = -EAGAIN;
 437
 438         return ret;
 439 }
 440
 441 static int
 442 v3d_job_add_deps(struct drm_file *file_priv, struct v3d_job *job,
 443                  u32 in_sync, u32 point)
 444 {
 445         struct dma_fence *in_fence = NULL;
 446         int ret;
 447
 448         ret = drm_syncobj_find_fence(file_priv, in_sync, point, 0, &in_fence);
 449         if (ret == -EINVAL)
 450                 return ret;
 451
 452         return drm_sched_job_add_dependency(&job->base, in_fence);
 453 }
 454
 455 static int
 456 v3d_job_init(struct v3d_dev *v3d, struct drm_file *file_priv,
 457              void **container, size_t size, void (*free)(struct kref *ref),
 458              u32 in_sync, struct v3d_submit_ext *se, enum v3d_queue queue)
 459 {
 460         struct v3d_file_priv *v3d_priv = file_priv->driver_priv;
 461         struct v3d_job *job;
 462         bool has_multisync = se && (se->flags & DRM_V3D_EXT_ID_MULTI_SYNC);
 463         int ret, i;
 464
 465         *container = kcalloc(1, size, GFP_KERNEL);
 466         if (!*container) {
 467                 DRM_ERROR("Cannot allocate memory for v3d job.");
 468                 return -ENOMEM;
 469         }
 470
 471         job = *container;
 472         job->v3d = v3d;
 473         job->free = free;
 474
 475         ret = drm_sched_job_init(&job->base, &v3d_priv->sched_entity[queue],
 476                                  v3d_priv);
 477         if (ret)
 478                 goto fail;
 479
 480         if (has_multisync) {
 481                 if (se->in_sync_count && se->wait_stage == queue) {
 482                         struct drm_v3d_sem __user *handle = u64_to_user_ptr(se->in_syncs);
 483
 484                         for (i = 0; i < se->in_sync_count; i++) {
 485                                 struct drm_v3d_sem in;
 486
 487                                 if (copy_from_user(&in, handle++, sizeof(in))) {
 488                                         ret = -EFAULT;
 489                                         DRM_DEBUG("Failed to copy wait dep handle.\n");
 490                                         goto fail_deps;
 491                                 }
 492                                 ret = v3d_job_add_deps(file_priv, job, in.handle, 0);
 493                                 if (ret)
 494                                         goto fail_deps;
 495                         }
 496                 }
 497         } else {
 498                 ret = v3d_job_add_deps(file_priv, job, in_sync, 0);
 499                 if (ret)
 500                         goto fail_deps;
 501         }
 502
 503         kref_init(&job->refcount);
 504
 505         return 0;
 506
 507 fail_deps:
 508         drm_sched_job_cleanup(&job->base);
 509 fail:
 510         kfree(*container);
 511         *container = NULL;
 512
 513         return ret;
 514 }
 515
 516 static void
 517 v3d_push_job(struct v3d_job *job)
 518 {
 519         drm_sched_job_arm(&job->base);
 520
 521         job->done_fence = dma_fence_get(&job->base.s_fence->finished);
 522
 523         /* put by scheduler job completion */
 524         kref_get(&job->refcount);
 525
 526         drm_sched_entity_push_job(&job->base);
 527 }
 528
 529 static void
 530 v3d_attach_fences_and_unlock_reservation(struct drm_file *file_priv,
 531                                          struct v3d_job *job,
 532                                          struct ww_acquire_ctx *acquire_ctx,
 533                                          u32 out_sync,
 534                                          struct v3d_submit_ext *se,
 535                                          struct dma_fence *done_fence)
 536 {
 537         struct drm_syncobj *sync_out;
 538         bool has_multisync = se && (se->flags & DRM_V3D_EXT_ID_MULTI_SYNC);
 539         int i;
 540
 541         for (i = 0; i < job->bo_count; i++) {
 542                 /* XXX: Use shared fences for read-only objects. */
 543                 dma_resv_add_fence(job->bo[i]->resv, job->done_fence,
 544                                    DMA_RESV_USAGE_WRITE);
 545         }
 546
 547         drm_gem_unlock_reservations(job->bo, job->bo_count, acquire_ctx);
 548
 549         /* Update the return sync object for the job */
 550         /* If it only supports a single signal semaphore*/
 551         if (!has_multisync) {
 552                 sync_out = drm_syncobj_find(file_priv, out_sync);
 553                 if (sync_out) {
 554                         drm_syncobj_replace_fence(sync_out, done_fence);
 555                         drm_syncobj_put(sync_out);
 556                 }
 557                 return;
 558         }
 559
 560         /* If multiple semaphores extension is supported */
 561         if (se->out_sync_count) {
 562                 for (i = 0; i < se->out_sync_count; i++) {
 563                         drm_syncobj_replace_fence(se->out_syncs[i].syncobj,
 564                                                   done_fence);
 565                         drm_syncobj_put(se->out_syncs[i].syncobj);
 566                 }
 567                 kvfree(se->out_syncs);
 568         }
 569 }
 570
 571 static void
 572 v3d_put_multisync_post_deps(struct v3d_submit_ext *se)
 573 {
 574         unsigned int i;
 575
 576         if (!(se && se->out_sync_count))
 577                 return;
 578
 579         for (i = 0; i < se->out_sync_count; i++)
 580                 drm_syncobj_put(se->out_syncs[i].syncobj);
 581         kvfree(se->out_syncs);
 582 }
 583
 584 static int
 585 v3d_get_multisync_post_deps(struct drm_file *file_priv,
 586                             struct v3d_submit_ext *se,
 587                             u32 count, u64 handles)
 588 {
 589         struct drm_v3d_sem __user *post_deps;
 590         int i, ret;
 591
 592         if (!count)
 593                 return 0;
 594
 595         se->out_syncs = (struct v3d_submit_outsync *)
 596                         kvmalloc_array(count,
 597                                        sizeof(struct v3d_submit_outsync),
 598                                        GFP_KERNEL);
 599         if (!se->out_syncs)
 600                 return -ENOMEM;
 601
 602         post_deps = u64_to_user_ptr(handles);
 603
 604         for (i = 0; i < count; i++) {
 605                 struct drm_v3d_sem out;
 606
 607                 if (copy_from_user(&out, post_deps++, sizeof(out))) {
 608                         ret = -EFAULT;
 609                         DRM_DEBUG("Failed to copy post dep handles\n");
 610                         goto fail;
 611                 }
 612
 613                 se->out_syncs[i].syncobj = drm_syncobj_find(file_priv,
 614                                                             out.handle);
 615                 if (!se->out_syncs[i].syncobj) {
 616                         ret = -EINVAL;
 617                         goto fail;
 618                 }
 619         }
 620         se->out_sync_count = count;
 621
 622         return 0;
 623
 624 fail:
 625         for (i--; i >= 0; i--)
 626                 drm_syncobj_put(se->out_syncs[i].syncobj);
 627         kvfree(se->out_syncs);
 628
 629         return ret;
 630 }
 631
 632 /* Get data for multiple binary semaphores synchronization. Parse syncobj
 633  * to be signaled when job completes (out_sync).
 634  */
 635 static int
 636 v3d_get_multisync_submit_deps(struct drm_file *file_priv,
 637                               struct drm_v3d_extension __user *ext,
 638                               void *data)
 639 {
 640         struct drm_v3d_multi_sync multisync;
 641         struct v3d_submit_ext *se = data;
 642         int ret;
 643
 644         if (copy_from_user(&multisync, ext, sizeof(multisync)))
 645                 return -EFAULT;
 646
 647         if (multisync.pad)
 648                 return -EINVAL;
 649
 650         ret = v3d_get_multisync_post_deps(file_priv, data, multisync.out_sync_count,
 651                                           multisync.out_syncs);
 652         if (ret)
 653                 return ret;
 654
 655         se->in_sync_count = multisync.in_sync_count;
 656         se->in_syncs = multisync.in_syncs;
 657         se->flags |= DRM_V3D_EXT_ID_MULTI_SYNC;
 658         se->wait_stage = multisync.wait_stage;
 659
 660         return 0;
 661 }
 662
 663 /* Whenever userspace sets ioctl extensions, v3d_get_extensions parses data
 664  * according to the extension id (name).
 665  */
 666 static int
 667 v3d_get_extensions(struct drm_file *file_priv,
 668                    u64 ext_handles,
 669                    void *data)
 670 {
 671         struct drm_v3d_extension __user *user_ext;
 672         int ret;
 673
 674         user_ext = u64_to_user_ptr(ext_handles);
 675         while (user_ext) {
 676                 struct drm_v3d_extension ext;
 677
 678                 if (copy_from_user(&ext, user_ext, sizeof(ext))) {
 679                         DRM_DEBUG("Failed to copy submit extension\n");
 680                         return -EFAULT;
 681                 }
 682
 683                 switch (ext.id) {
 684                 case DRM_V3D_EXT_ID_MULTI_SYNC:
 685                         ret = v3d_get_multisync_submit_deps(file_priv, user_ext, data);
 686                         if (ret)
 687                                 return ret;
 688                         break;
 689                 default:
 690                         DRM_DEBUG_DRIVER("Unknown extension id: %d\n", ext.id);
 691                         return -EINVAL;
 692                 }
 693
 694                 user_ext = u64_to_user_ptr(ext.next);
 695         }
 696
 697         return 0;
 698 }
 699
 700 /**
 701  * v3d_submit_cl_ioctl() - Submits a job (frame) to the V3D.
 702  * @dev: DRM device
 703  * @data: ioctl argument
 704  * @file_priv: DRM file for this fd
 705  *
 706  * This is the main entrypoint for userspace to submit a 3D frame to
 707  * the GPU.  Userspace provides the binner command list (if
 708  * applicable), and the kernel sets up the render command list to draw
 709  * to the framebuffer described in the ioctl, using the command lists
 710  * that the 3D engine's binner will produce.
 711  */
 712 int
 713 v3d_submit_cl_ioctl(struct drm_device *dev, void *data,
 714                     struct drm_file *file_priv)
 715 {
 716         struct v3d_dev *v3d = to_v3d_dev(dev);
 717         struct v3d_file_priv *v3d_priv = file_priv->driver_priv;
 718         struct drm_v3d_submit_cl *args = data;
 719         struct v3d_submit_ext se = {0};
 720         struct v3d_bin_job *bin = NULL;
 721         struct v3d_render_job *render = NULL;
 722         struct v3d_job *clean_job = NULL;
 723         struct v3d_job *last_job;
 724         struct ww_acquire_ctx acquire_ctx;
 725         int ret = 0;
 726
 727         trace_v3d_submit_cl_ioctl(&v3d->drm, args->rcl_start, args->rcl_end);
 728
 729         if (args->pad)
 730                 return -EINVAL;
 731
 732         if (args->flags &&
 733             args->flags & ~(DRM_V3D_SUBMIT_CL_FLUSH_CACHE |
 734                             DRM_V3D_SUBMIT_EXTENSION)) {
 735                 DRM_INFO("invalid flags: %d\n", args->flags);
 736                 return -EINVAL;
 737         }
 738
 739         if (args->flags & DRM_V3D_SUBMIT_EXTENSION) {
 740                 ret = v3d_get_extensions(file_priv, args->extensions, &se);
 741                 if (ret) {
 742                         DRM_DEBUG("Failed to get extensions.\n");
 743                         return ret;
 744                 }
 745         }
 746
 747         ret = v3d_job_init(v3d, file_priv, (void *)&render, sizeof(*render),
 748                            v3d_render_job_free, args->in_sync_rcl, &se, V3D_RENDER);
 749         if (ret)
 750                 goto fail;
 751
 752         render->start = args->rcl_start;
 753         render->end = args->rcl_end;
 754         INIT_LIST_HEAD(&render->unref_list);
 755
 756         if (args->bcl_start != args->bcl_end) {
 757                 ret = v3d_job_init(v3d, file_priv, (void *)&bin, sizeof(*bin),
 758                                    v3d_job_free, args->in_sync_bcl, &se, V3D_BIN);
 759                 if (ret)
 760                         goto fail;
 761
 762                 bin->start = args->bcl_start;
 763                 bin->end = args->bcl_end;
 764                 bin->qma = args->qma;
 765                 bin->qms = args->qms;
 766                 bin->qts = args->qts;
 767                 bin->render = render;
 768         }
 769
 770         if (args->flags & DRM_V3D_SUBMIT_CL_FLUSH_CACHE) {
 771                 ret = v3d_job_init(v3d, file_priv, (void *)&clean_job, sizeof(*clean_job),
 772                                    v3d_job_free, 0, NULL, V3D_CACHE_CLEAN);
 773                 if (ret)
 774                         goto fail;
 775
 776                 last_job = clean_job;
 777         } else {
 778                 last_job = &render->base;
 779         }
 780
 781         ret = v3d_lookup_bos(dev, file_priv, last_job,
 782                              args->bo_handles, args->bo_handle_count);
 783         if (ret)
 784                 goto fail;
 785
 786         ret = v3d_lock_bo_reservations(last_job, &acquire_ctx);
 787         if (ret)
 788                 goto fail;
 789
 790         if (args->perfmon_id) {
 791                 render->base.perfmon = v3d_perfmon_find(v3d_priv,
 792                                                         args->perfmon_id);
 793
 794                 if (!render->base.perfmon) {
 795                         ret = -ENOENT;
 796                         goto fail_perfmon;
 797                 }
 798         }
 799
 800         mutex_lock(&v3d->sched_lock);
 801         if (bin) {
 802                 bin->base.perfmon = render->base.perfmon;
 803                 v3d_perfmon_get(bin->base.perfmon);
 804                 v3d_push_job(&bin->base);
 805
 806                 ret = drm_sched_job_add_dependency(&render->base.base,
 807                                                    dma_fence_get(bin->base.done_fence));
 808                 if (ret)
 809                         goto fail_unreserve;
 810         }
 811
 812         v3d_push_job(&render->base);
 813
 814         if (clean_job) {
 815                 struct dma_fence *render_fence =
 816                         dma_fence_get(render->base.done_fence);
 817                 ret = drm_sched_job_add_dependency(&clean_job->base,
 818                                                    render_fence);
 819                 if (ret)
 820                         goto fail_unreserve;
 821                 clean_job->perfmon = render->base.perfmon;
 822                 v3d_perfmon_get(clean_job->perfmon);
 823                 v3d_push_job(clean_job);
 824         }
 825
 826         mutex_unlock(&v3d->sched_lock);
 827
 828         v3d_attach_fences_and_unlock_reservation(file_priv,
 829                                                  last_job,
 830                                                  &acquire_ctx,
 831                                                  args->out_sync,
 832                                                  &se,
 833                                                  last_job->done_fence);
 834
 835         if (bin)
 836                 v3d_job_put(&bin->base);
 837         v3d_job_put(&render->base);
 838         if (clean_job)
 839                 v3d_job_put(clean_job);
 840
 841         return 0;
 842
 843 fail_unreserve:
 844         mutex_unlock(&v3d->sched_lock);
 845 fail_perfmon:
 846         drm_gem_unlock_reservations(last_job->bo,
 847                                     last_job->bo_count, &acquire_ctx);
 848 fail:
 849         v3d_job_cleanup((void *)bin);
 850         v3d_job_cleanup((void *)render);
 851         v3d_job_cleanup(clean_job);
 852         v3d_put_multisync_post_deps(&se);
 853
 854         return ret;
 855 }
 856
 857 /**
 858  * v3d_submit_tfu_ioctl() - Submits a TFU (texture formatting) job to the V3D.
 859  * @dev: DRM device
 860  * @data: ioctl argument
 861  * @file_priv: DRM file for this fd
 862  *
 863  * Userspace provides the register setup for the TFU, which we don't
 864  * need to validate since the TFU is behind the MMU.
 865  */
 866 int
 867 v3d_submit_tfu_ioctl(struct drm_device *dev, void *data,
 868                      struct drm_file *file_priv)
 869 {
 870         struct v3d_dev *v3d = to_v3d_dev(dev);
 871         struct drm_v3d_submit_tfu *args = data;
 872         struct v3d_submit_ext se = {0};
 873         struct v3d_tfu_job *job = NULL;
 874         struct ww_acquire_ctx acquire_ctx;
 875         int ret = 0;
 876
 877         trace_v3d_submit_tfu_ioctl(&v3d->drm, args->iia);
 878
 879         if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) {
 880                 DRM_DEBUG("invalid flags: %d\n", args->flags);
 881                 return -EINVAL;
 882         }
 883
 884         if (args->flags & DRM_V3D_SUBMIT_EXTENSION) {
 885                 ret = v3d_get_extensions(file_priv, args->extensions, &se);
 886                 if (ret) {
 887                         DRM_DEBUG("Failed to get extensions.\n");
 888                         return ret;
 889                 }
 890         }
 891
 892         ret = v3d_job_init(v3d, file_priv, (void *)&job, sizeof(*job),
 893                            v3d_job_free, args->in_sync, &se, V3D_TFU);
 894         if (ret)
 895                 goto fail;
 896
 897         job->base.bo = kcalloc(ARRAY_SIZE(args->bo_handles),
 898                                sizeof(*job->base.bo), GFP_KERNEL);
 899         if (!job->base.bo) {
 900                 ret = -ENOMEM;
 901                 goto fail;
 902         }
 903
 904         job->args = *args;
 905
 906         spin_lock(&file_priv->table_lock);
 907         for (job->base.bo_count = 0;
 908              job->base.bo_count < ARRAY_SIZE(args->bo_handles);
 909              job->base.bo_count++) {
 910                 struct drm_gem_object *bo;
 911
 912                 if (!args->bo_handles[job->base.bo_count])
 913                         break;
 914
 915                 bo = idr_find(&file_priv->object_idr,
 916                               args->bo_handles[job->base.bo_count]);
 917                 if (!bo) {
 918                         DRM_DEBUG("Failed to look up GEM BO %d: %d\n",
 919                                   job->base.bo_count,
 920                                   args->bo_handles[job->base.bo_count]);
 921                         ret = -ENOENT;
 922                         spin_unlock(&file_priv->table_lock);
 923                         goto fail;
 924                 }
 925                 drm_gem_object_get(bo);
 926                 job->base.bo[job->base.bo_count] = bo;
 927         }
 928         spin_unlock(&file_priv->table_lock);
 929
 930         ret = v3d_lock_bo_reservations(&job->base, &acquire_ctx);
 931         if (ret)
 932                 goto fail;
 933
 934         mutex_lock(&v3d->sched_lock);
 935         v3d_push_job(&job->base);
 936         mutex_unlock(&v3d->sched_lock);
 937
 938         v3d_attach_fences_and_unlock_reservation(file_priv,
 939                                                  &job->base, &acquire_ctx,
 940                                                  args->out_sync,
 941                                                  &se,
 942                                                  job->base.done_fence);
 943
 944         v3d_job_put(&job->base);
 945
 946         return 0;
 947
 948 fail:
 949         v3d_job_cleanup((void *)job);
 950         v3d_put_multisync_post_deps(&se);
 951
 952         return ret;
 953 }
 954
 955 /**
 956  * v3d_submit_csd_ioctl() - Submits a CSD (texture formatting) job to the V3D.
 957  * @dev: DRM device
 958  * @data: ioctl argument
 959  * @file_priv: DRM file for this fd
 960  *
 961  * Userspace provides the register setup for the CSD, which we don't
 962  * need to validate since the CSD is behind the MMU.
 963  */
 964 int
 965 v3d_submit_csd_ioctl(struct drm_device *dev, void *data,
 966                      struct drm_file *file_priv)
 967 {
 968         struct v3d_dev *v3d = to_v3d_dev(dev);
 969         struct v3d_file_priv *v3d_priv = file_priv->driver_priv;
 970         struct drm_v3d_submit_csd *args = data;
 971         struct v3d_submit_ext se = {0};
 972         struct v3d_csd_job *job = NULL;
 973         struct v3d_job *clean_job = NULL;
 974         struct ww_acquire_ctx acquire_ctx;
 975         int ret;
 976
 977         trace_v3d_submit_csd_ioctl(&v3d->drm, args->cfg[5], args->cfg[6]);
 978
 979         if (args->pad)
 980                 return -EINVAL;
 981
 982         if (!v3d_has_csd(v3d)) {
 983                 DRM_DEBUG("Attempting CSD submit on non-CSD hardware\n");
 984                 return -EINVAL;
 985         }
 986
 987         if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) {
 988                 DRM_INFO("invalid flags: %d\n", args->flags);
 989                 return -EINVAL;
 990         }
 991
 992         if (args->flags & DRM_V3D_SUBMIT_EXTENSION) {
 993                 ret = v3d_get_extensions(file_priv, args->extensions, &se);
 994                 if (ret) {
 995                         DRM_DEBUG("Failed to get extensions.\n");
 996                         return ret;
 997                 }
 998         }
 999
1000         ret = v3d_job_init(v3d, file_priv, (void *)&job, sizeof(*job),
1001                            v3d_job_free, args->in_sync, &se, V3D_CSD);
1002         if (ret)
1003                 goto fail;
1004
1005         ret = v3d_job_init(v3d, file_priv, (void *)&clean_job, sizeof(*clean_job),
1006                            v3d_job_free, 0, NULL, V3D_CACHE_CLEAN);
1007         if (ret)
1008                 goto fail;
1009
1010         job->args = *args;
1011
1012         ret = v3d_lookup_bos(dev, file_priv, clean_job,
1013                              args->bo_handles, args->bo_handle_count);
1014         if (ret)
1015                 goto fail;
1016
1017         ret = v3d_lock_bo_reservations(clean_job, &acquire_ctx);
1018         if (ret)
1019                 goto fail;
1020
1021         if (args->perfmon_id) {
1022                 job->base.perfmon = v3d_perfmon_find(v3d_priv,
1023                                                      args->perfmon_id);
1024                 if (!job->base.perfmon) {
1025                         ret = -ENOENT;
1026                         goto fail_perfmon;
1027                 }
1028         }
1029
1030         mutex_lock(&v3d->sched_lock);
1031         v3d_push_job(&job->base);
1032
1033         ret = drm_sched_job_add_dependency(&clean_job->base,
1034                                            dma_fence_get(job->base.done_fence));
1035         if (ret)
1036                 goto fail_unreserve;
1037
1038         v3d_push_job(clean_job);
1039         mutex_unlock(&v3d->sched_lock);
1040
1041         v3d_attach_fences_and_unlock_reservation(file_priv,
1042                                                  clean_job,
1043                                                  &acquire_ctx,
1044                                                  args->out_sync,
1045                                                  &se,
1046                                                  clean_job->done_fence);
1047
1048         v3d_job_put(&job->base);
1049         v3d_job_put(clean_job);
1050
1051         return 0;
1052
1053 fail_unreserve:
1054         mutex_unlock(&v3d->sched_lock);
1055 fail_perfmon:
1056         drm_gem_unlock_reservations(clean_job->bo, clean_job->bo_count,
1057                                     &acquire_ctx);
1058 fail:
1059         v3d_job_cleanup((void *)job);
1060         v3d_job_cleanup(clean_job);
1061         v3d_put_multisync_post_deps(&se);
1062
1063         return ret;
1064 }
1065
1066 int
1067 v3d_gem_init(struct drm_device *dev)
1068 {
1069         struct v3d_dev *v3d = to_v3d_dev(dev);
1070         u32 pt_size = 4096 * 1024;
1071         int ret, i;
1072
1073         for (i = 0; i < V3D_MAX_QUEUES; i++)
1074                 v3d->queue[i].fence_context = dma_fence_context_alloc(1);
1075
1076         spin_lock_init(&v3d->mm_lock);
1077         spin_lock_init(&v3d->job_lock);
1078         mutex_init(&v3d->bo_lock);
1079         mutex_init(&v3d->reset_lock);
1080         mutex_init(&v3d->sched_lock);
1081         mutex_init(&v3d->cache_clean_lock);
1082
1083         /* Note: We don't allocate address 0.  Various bits of HW
1084          * treat 0 as special, such as the occlusion query counters
1085          * where 0 means "disabled".
1086          */
1087         drm_mm_init(&v3d->mm, 1, pt_size / sizeof(u32) - 1);
1088
1089         v3d->pt = dma_alloc_wc(v3d->drm.dev, pt_size,
1090                                &v3d->pt_paddr,
1091                                GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO);
1092         if (!v3d->pt) {
1093                 drm_mm_takedown(&v3d->mm);
1094                 dev_err(v3d->drm.dev,
1095                         "Failed to allocate page tables. Please ensure you have CMA enabled.\n");
1096                 return -ENOMEM;
1097         }
1098
1099         v3d_init_hw_state(v3d);
1100         v3d_mmu_set_page_table(v3d);
1101
1102         ret = v3d_sched_init(v3d);
1103         if (ret) {
1104                 drm_mm_takedown(&v3d->mm);
1105                 dma_free_coherent(v3d->drm.dev, 4096 * 1024, (void *)v3d->pt,
1106                                   v3d->pt_paddr);
1107         }
1108
1109         return 0;
1110 }
1111
1112 void
1113 v3d_gem_destroy(struct drm_device *dev)
1114 {
1115         struct v3d_dev *v3d = to_v3d_dev(dev);
1116
1117         v3d_sched_fini(v3d);
1118
1119         /* Waiting for jobs to finish would need to be done before
1120          * unregistering V3D.
1121          */
1122         WARN_ON(v3d->bin_job);
1123         WARN_ON(v3d->render_job);
1124
1125         drm_mm_takedown(&v3d->mm);
1126
1127         dma_free_coherent(v3d->drm.dev, 4096 * 1024, (void *)v3d->pt,
1128                           v3d->pt_paddr);
1129 }