drivers/gpu/host1x/hw/cdma_hw.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Tegra host1x Command DMA
   4  *
   5  * Copyright (c) 2010-2013, NVIDIA Corporation.
   6  */
   7
   8 #include <linux/slab.h>
   9 #include <linux/scatterlist.h>
  10 #include <linux/dma-mapping.h>
  11
  12 #include "../cdma.h"
  13 #include "../channel.h"
  14 #include "../dev.h"
  15 #include "../debug.h"
  16
  17 /*
  18  * Put the restart at the end of pushbuffer memory
  19  */
  20 static void push_buffer_init(struct push_buffer *pb)
  21 {
  22         *(u32 *)(pb->mapped + pb->size) = host1x_opcode_restart(0);
  23 }
  24
  25 /*
  26  * Increment timedout buffer's syncpt via CPU.
  27  */
  28 static void cdma_timeout_cpu_incr(struct host1x_cdma *cdma, u32 getptr,
  29                                 u32 syncpt_incrs, u32 syncval, u32 nr_slots)
  30 {
  31         unsigned int i;
  32
  33         for (i = 0; i < syncpt_incrs; i++)
  34                 host1x_syncpt_incr(cdma->timeout.syncpt);
  35
  36         /* after CPU incr, ensure shadow is up to date */
  37         host1x_syncpt_load(cdma->timeout.syncpt);
  38 }
  39
  40 /*
  41  * Start channel DMA
  42  */
  43 static void cdma_start(struct host1x_cdma *cdma)
  44 {
  45         struct host1x_channel *ch = cdma_to_channel(cdma);
  46         u64 start, end;
  47
  48         if (cdma->running)
  49                 return;
  50
  51         cdma->last_pos = cdma->push_buffer.pos;
  52         start = cdma->push_buffer.dma;
  53         end = cdma->push_buffer.size + 4;
  54
  55         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
  56                          HOST1X_CHANNEL_DMACTRL);
  57
  58         /* set base, put and end pointer */
  59         host1x_ch_writel(ch, lower_32_bits(start), HOST1X_CHANNEL_DMASTART);
  60 #if HOST1X_HW >= 6
  61         host1x_ch_writel(ch, upper_32_bits(start), HOST1X_CHANNEL_DMASTART_HI);
  62 #endif
  63         host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT);
  64 #if HOST1X_HW >= 6
  65         host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMAPUT_HI);
  66 #endif
  67         host1x_ch_writel(ch, lower_32_bits(end), HOST1X_CHANNEL_DMAEND);
  68 #if HOST1X_HW >= 6
  69         host1x_ch_writel(ch, upper_32_bits(end), HOST1X_CHANNEL_DMAEND_HI);
  70 #endif
  71
  72         /* reset GET */
  73         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP |
  74                          HOST1X_CHANNEL_DMACTRL_DMAGETRST |
  75                          HOST1X_CHANNEL_DMACTRL_DMAINITGET,
  76                          HOST1X_CHANNEL_DMACTRL);
  77
  78         /* start the command DMA */
  79         host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL);
  80
  81         cdma->running = true;
  82 }
  83
  84 /*
  85  * Similar to cdma_start(), but rather than starting from an idle
  86  * state (where DMA GET is set to DMA PUT), on a timeout we restore
  87  * DMA GET from an explicit value (so DMA may again be pending).
  88  */
  89 static void cdma_timeout_restart(struct host1x_cdma *cdma, u32 getptr)
  90 {
  91         struct host1x *host1x = cdma_to_host1x(cdma);
  92         struct host1x_channel *ch = cdma_to_channel(cdma);
  93         u64 start, end;
  94
  95         if (cdma->running)
  96                 return;
  97
  98         cdma->last_pos = cdma->push_buffer.pos;
  99
 100         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 101                          HOST1X_CHANNEL_DMACTRL);
 102
 103         start = cdma->push_buffer.dma;
 104         end = cdma->push_buffer.size + 4;
 105
 106         /* set base, end pointer (all of memory) */
 107         host1x_ch_writel(ch, lower_32_bits(start), HOST1X_CHANNEL_DMASTART);
 108 #if HOST1X_HW >= 6
 109         host1x_ch_writel(ch, upper_32_bits(start), HOST1X_CHANNEL_DMASTART_HI);
 110 #endif
 111         host1x_ch_writel(ch, lower_32_bits(end), HOST1X_CHANNEL_DMAEND);
 112 #if HOST1X_HW >= 6
 113         host1x_ch_writel(ch, upper_32_bits(end), HOST1X_CHANNEL_DMAEND_HI);
 114 #endif
 115
 116         /* set GET, by loading the value in PUT (then reset GET) */
 117         host1x_ch_writel(ch, getptr, HOST1X_CHANNEL_DMAPUT);
 118         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP |
 119                          HOST1X_CHANNEL_DMACTRL_DMAGETRST |
 120                          HOST1X_CHANNEL_DMACTRL_DMAINITGET,
 121                          HOST1X_CHANNEL_DMACTRL);
 122
 123         dev_dbg(host1x->dev,
 124                 "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n", __func__,
 125                 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET),
 126                 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT),
 127                 cdma->last_pos);
 128
 129         /* deassert GET reset and set PUT */
 130         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 131                          HOST1X_CHANNEL_DMACTRL);
 132         host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT);
 133
 134         /* start the command DMA */
 135         host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL);
 136
 137         cdma->running = true;
 138 }
 139
 140 /*
 141  * Kick channel DMA into action by writing its PUT offset (if it has changed)
 142  */
 143 static void cdma_flush(struct host1x_cdma *cdma)
 144 {
 145         struct host1x_channel *ch = cdma_to_channel(cdma);
 146
 147         if (cdma->push_buffer.pos != cdma->last_pos) {
 148                 host1x_ch_writel(ch, cdma->push_buffer.pos,
 149                                  HOST1X_CHANNEL_DMAPUT);
 150                 cdma->last_pos = cdma->push_buffer.pos;
 151         }
 152 }
 153
 154 static void cdma_stop(struct host1x_cdma *cdma)
 155 {
 156         struct host1x_channel *ch = cdma_to_channel(cdma);
 157
 158         mutex_lock(&cdma->lock);
 159
 160         if (cdma->running) {
 161                 host1x_cdma_wait_locked(cdma, CDMA_EVENT_SYNC_QUEUE_EMPTY);
 162                 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 163                                  HOST1X_CHANNEL_DMACTRL);
 164                 cdma->running = false;
 165         }
 166
 167         mutex_unlock(&cdma->lock);
 168 }
 169
 170 static void cdma_hw_cmdproc_stop(struct host1x *host, struct host1x_channel *ch,
 171                                  bool stop)
 172 {
 173 #if HOST1X_HW >= 6
 174         host1x_ch_writel(ch, stop ? 0x1 : 0x0, HOST1X_CHANNEL_CMDPROC_STOP);
 175 #else
 176         u32 cmdproc_stop = host1x_sync_readl(host, HOST1X_SYNC_CMDPROC_STOP);
 177         if (stop)
 178                 cmdproc_stop |= BIT(ch->id);
 179         else
 180                 cmdproc_stop &= ~BIT(ch->id);
 181         host1x_sync_writel(host, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP);
 182 #endif
 183 }
 184
 185 static void cdma_hw_teardown(struct host1x *host, struct host1x_channel *ch)
 186 {
 187 #if HOST1X_HW >= 6
 188         host1x_ch_writel(ch, 0x1, HOST1X_CHANNEL_TEARDOWN);
 189 #else
 190         host1x_sync_writel(host, BIT(ch->id), HOST1X_SYNC_CH_TEARDOWN);
 191 #endif
 192 }
 193
 194 /*
 195  * Stops both channel's command processor and CDMA immediately.
 196  * Also, tears down the channel and resets corresponding module.
 197  */
 198 static void cdma_freeze(struct host1x_cdma *cdma)
 199 {
 200         struct host1x *host = cdma_to_host1x(cdma);
 201         struct host1x_channel *ch = cdma_to_channel(cdma);
 202
 203         if (cdma->torndown && !cdma->running) {
 204                 dev_warn(host->dev, "Already torn down\n");
 205                 return;
 206         }
 207
 208         dev_dbg(host->dev, "freezing channel (id %d)\n", ch->id);
 209
 210         cdma_hw_cmdproc_stop(host, ch, true);
 211
 212         dev_dbg(host->dev, "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n",
 213                 __func__, host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET),
 214                 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT),
 215                 cdma->last_pos);
 216
 217         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 218                          HOST1X_CHANNEL_DMACTRL);
 219
 220         cdma_hw_teardown(host, ch);
 221
 222         cdma->running = false;
 223         cdma->torndown = true;
 224 }
 225
 226 static void cdma_resume(struct host1x_cdma *cdma, u32 getptr)
 227 {
 228         struct host1x *host1x = cdma_to_host1x(cdma);
 229         struct host1x_channel *ch = cdma_to_channel(cdma);
 230
 231         dev_dbg(host1x->dev,
 232                 "resuming channel (id %u, DMAGET restart = 0x%x)\n",
 233                 ch->id, getptr);
 234
 235         cdma_hw_cmdproc_stop(host1x, ch, false);
 236
 237         cdma->torndown = false;
 238         cdma_timeout_restart(cdma, getptr);
 239 }
 240
 241 static void timeout_release_mlock(struct host1x_cdma *cdma)
 242 {
 243 #if HOST1X_HW >= 8
 244         /* Tegra186 and Tegra194 require a more complicated MLOCK release
 245          * sequence. Furthermore, those chips by default don't enforce MLOCKs,
 246          * so it turns out that if we don't /actually/ need MLOCKs, we can just
 247          * ignore them.
 248          *
 249          * As such, for now just implement this on Tegra234 where things are
 250          * stricter but also easy to implement.
 251          */
 252         struct host1x_channel *ch = cdma_to_channel(cdma);
 253         struct host1x *host1x = cdma_to_host1x(cdma);
 254         u32 offset;
 255
 256         switch (ch->client->class) {
 257         case HOST1X_CLASS_NVJPG1:
 258                 offset = HOST1X_COMMON_NVJPG1_MLOCK;
 259                 break;
 260         case HOST1X_CLASS_NVENC:
 261                 offset = HOST1X_COMMON_NVENC_MLOCK;
 262                 break;
 263         case HOST1X_CLASS_VIC:
 264                 offset = HOST1X_COMMON_VIC_MLOCK;
 265                 break;
 266         case HOST1X_CLASS_NVJPG:
 267                 offset = HOST1X_COMMON_NVJPG_MLOCK;
 268                 break;
 269         case HOST1X_CLASS_NVDEC:
 270                 offset = HOST1X_COMMON_NVDEC_MLOCK;
 271                 break;
 272         case HOST1X_CLASS_OFA:
 273                 offset = HOST1X_COMMON_OFA_MLOCK;
 274                 break;
 275         default:
 276                 WARN(1, "%s was not updated for class %u", __func__, ch->client->class);
 277                 return;
 278         }
 279
 280         host1x_common_writel(host1x, 0x0, offset);
 281 #endif
 282 }
 283
 284 /*
 285  * If this timeout fires, it indicates the current sync_queue entry has
 286  * exceeded its TTL and the userctx should be timed out and remaining
 287  * submits already issued cleaned up (future submits return an error).
 288  */
 289 static void cdma_timeout_handler(struct work_struct *work)
 290 {
 291         u32 syncpt_val;
 292         struct host1x_cdma *cdma;
 293         struct host1x *host1x;
 294         struct host1x_channel *ch;
 295
 296         cdma = container_of(to_delayed_work(work), struct host1x_cdma,
 297                             timeout.wq);
 298         host1x = cdma_to_host1x(cdma);
 299         ch = cdma_to_channel(cdma);
 300
 301         host1x_debug_dump(cdma_to_host1x(cdma));
 302
 303         mutex_lock(&cdma->lock);
 304
 305         if (!cdma->timeout.client) {
 306                 dev_dbg(host1x->dev,
 307                         "cdma_timeout: expired, but has no clientid\n");
 308                 mutex_unlock(&cdma->lock);
 309                 return;
 310         }
 311
 312         /* stop processing to get a clean snapshot */
 313         cdma_hw_cmdproc_stop(host1x, ch, true);
 314
 315         syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt);
 316
 317         /* has buffer actually completed? */
 318         if ((s32)(syncpt_val - cdma->timeout.syncpt_val) >= 0) {
 319                 dev_dbg(host1x->dev,
 320                         "cdma_timeout: expired, but buffer had completed\n");
 321                 /* restore */
 322                 cdma_hw_cmdproc_stop(host1x, ch, false);
 323                 mutex_unlock(&cdma->lock);
 324                 return;
 325         }
 326
 327         dev_warn(host1x->dev, "%s: timeout: %u (%s), HW thresh %d, done %d\n",
 328                  __func__, cdma->timeout.syncpt->id, cdma->timeout.syncpt->name,
 329                  syncpt_val, cdma->timeout.syncpt_val);
 330
 331         /* stop HW, resetting channel/module */
 332         host1x_hw_cdma_freeze(host1x, cdma);
 333
 334         /* release any held MLOCK */
 335         timeout_release_mlock(cdma);
 336
 337         host1x_cdma_update_sync_queue(cdma, ch->dev);
 338         mutex_unlock(&cdma->lock);
 339 }
 340
 341 /*
 342  * Init timeout resources
 343  */
 344 static int cdma_timeout_init(struct host1x_cdma *cdma)
 345 {
 346         INIT_DELAYED_WORK(&cdma->timeout.wq, cdma_timeout_handler);
 347         cdma->timeout.initialized = true;
 348
 349         return 0;
 350 }
 351
 352 /*
 353  * Clean up timeout resources
 354  */
 355 static void cdma_timeout_destroy(struct host1x_cdma *cdma)
 356 {
 357         if (cdma->timeout.initialized)
 358                 cancel_delayed_work(&cdma->timeout.wq);
 359
 360         cdma->timeout.initialized = false;
 361 }
 362
 363 static const struct host1x_cdma_ops host1x_cdma_ops = {
 364         .start = cdma_start,
 365         .stop = cdma_stop,
 366         .flush = cdma_flush,
 367
 368         .timeout_init = cdma_timeout_init,
 369         .timeout_destroy = cdma_timeout_destroy,
 370         .freeze = cdma_freeze,
 371         .resume = cdma_resume,
 372         .timeout_cpu_incr = cdma_timeout_cpu_incr,
 373 };
 374
 375 static const struct host1x_pushbuffer_ops host1x_pushbuffer_ops = {
 376         .init = push_buffer_init,
 377 };