drivers/gpu/drm/savage/savage_state.c

   1 /* savage_state.c -- State and drawing support for Savage
   2  *
   3  * Copyright 2004  Felix Kuehling
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sub license,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the
  14  * next paragraph) shall be included in all copies or substantial portions
  15  * of the Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  18  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  19  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  20  * NON-INFRINGEMENT. IN NO EVENT SHALL FELIX KUEHLING BE LIABLE FOR
  21  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
  22  * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  23  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  24  */
  25 #include <drm/drmP.h>
  26 #include <drm/savage_drm.h>
  27 #include "savage_drv.h"
  28
  29 void savage_emit_clip_rect_s3d(drm_savage_private_t * dev_priv,
  30                                const struct drm_clip_rect * pbox)
  31 {
  32         uint32_t scstart = dev_priv->state.s3d.new_scstart;
  33         uint32_t scend = dev_priv->state.s3d.new_scend;
  34         scstart = (scstart & ~SAVAGE_SCISSOR_MASK_S3D) |
  35             ((uint32_t) pbox->x1 & 0x000007ff) |
  36             (((uint32_t) pbox->y1 << 16) & 0x07ff0000);
  37         scend = (scend & ~SAVAGE_SCISSOR_MASK_S3D) |
  38             (((uint32_t) pbox->x2 - 1) & 0x000007ff) |
  39             ((((uint32_t) pbox->y2 - 1) << 16) & 0x07ff0000);
  40         if (scstart != dev_priv->state.s3d.scstart ||
  41             scend != dev_priv->state.s3d.scend) {
  42                 DMA_LOCALS;
  43                 BEGIN_DMA(4);
  44                 DMA_WRITE(BCI_CMD_WAIT | BCI_CMD_WAIT_3D);
  45                 DMA_SET_REGISTERS(SAVAGE_SCSTART_S3D, 2);
  46                 DMA_WRITE(scstart);
  47                 DMA_WRITE(scend);
  48                 dev_priv->state.s3d.scstart = scstart;
  49                 dev_priv->state.s3d.scend = scend;
  50                 dev_priv->waiting = 1;
  51                 DMA_COMMIT();
  52         }
  53 }
  54
  55 void savage_emit_clip_rect_s4(drm_savage_private_t * dev_priv,
  56                               const struct drm_clip_rect * pbox)
  57 {
  58         uint32_t drawctrl0 = dev_priv->state.s4.new_drawctrl0;
  59         uint32_t drawctrl1 = dev_priv->state.s4.new_drawctrl1;
  60         drawctrl0 = (drawctrl0 & ~SAVAGE_SCISSOR_MASK_S4) |
  61             ((uint32_t) pbox->x1 & 0x000007ff) |
  62             (((uint32_t) pbox->y1 << 12) & 0x00fff000);
  63         drawctrl1 = (drawctrl1 & ~SAVAGE_SCISSOR_MASK_S4) |
  64             (((uint32_t) pbox->x2 - 1) & 0x000007ff) |
  65             ((((uint32_t) pbox->y2 - 1) << 12) & 0x00fff000);
  66         if (drawctrl0 != dev_priv->state.s4.drawctrl0 ||
  67             drawctrl1 != dev_priv->state.s4.drawctrl1) {
  68                 DMA_LOCALS;
  69                 BEGIN_DMA(4);
  70                 DMA_WRITE(BCI_CMD_WAIT | BCI_CMD_WAIT_3D);
  71                 DMA_SET_REGISTERS(SAVAGE_DRAWCTRL0_S4, 2);
  72                 DMA_WRITE(drawctrl0);
  73                 DMA_WRITE(drawctrl1);
  74                 dev_priv->state.s4.drawctrl0 = drawctrl0;
  75                 dev_priv->state.s4.drawctrl1 = drawctrl1;
  76                 dev_priv->waiting = 1;
  77                 DMA_COMMIT();
  78         }
  79 }
  80
  81 static int savage_verify_texaddr(drm_savage_private_t * dev_priv, int unit,
  82                                  uint32_t addr)
  83 {
  84         if ((addr & 6) != 2) {  /* reserved bits */
  85                 DRM_ERROR("bad texAddr%d %08x (reserved bits)\n", unit, addr);
  86                 return -EINVAL;
  87         }
  88         if (!(addr & 1)) {      /* local */
  89                 addr &= ~7;
  90                 if (addr < dev_priv->texture_offset ||
  91                     addr >= dev_priv->texture_offset + dev_priv->texture_size) {
  92                         DRM_ERROR
  93                             ("bad texAddr%d %08x (local addr out of range)\n",
  94                              unit, addr);
  95                         return -EINVAL;
  96                 }
  97         } else {                /* AGP */
  98                 if (!dev_priv->agp_textures) {
  99                         DRM_ERROR("bad texAddr%d %08x (AGP not available)\n",
 100                                   unit, addr);
 101                         return -EINVAL;
 102                 }
 103                 addr &= ~7;
 104                 if (addr < dev_priv->agp_textures->offset ||
 105                     addr >= (dev_priv->agp_textures->offset +
 106                              dev_priv->agp_textures->size)) {
 107                         DRM_ERROR
 108                             ("bad texAddr%d %08x (AGP addr out of range)\n",
 109                              unit, addr);
 110                         return -EINVAL;
 111                 }
 112         }
 113         return 0;
 114 }
 115
 116 #define SAVE_STATE(reg,where)                   \
 117         if(start <= reg && start+count > reg)   \
 118                 dev_priv->state.where = regs[reg - start]
 119 #define SAVE_STATE_MASK(reg,where,mask) do {                    \
 120         if(start <= reg && start+count > reg) {                 \
 121                 uint32_t tmp;                                   \
 122                 tmp = regs[reg - start];                        \
 123                 dev_priv->state.where = (tmp & (mask)) |        \
 124                         (dev_priv->state.where & ~(mask));      \
 125         }                                                       \
 126 } while (0)
 127
 128 static int savage_verify_state_s3d(drm_savage_private_t * dev_priv,
 129                                    unsigned int start, unsigned int count,
 130                                    const uint32_t *regs)
 131 {
 132         if (start < SAVAGE_TEXPALADDR_S3D ||
 133             start + count - 1 > SAVAGE_DESTTEXRWWATERMARK_S3D) {
 134                 DRM_ERROR("invalid register range (0x%04x-0x%04x)\n",
 135                           start, start + count - 1);
 136                 return -EINVAL;
 137         }
 138
 139         SAVE_STATE_MASK(SAVAGE_SCSTART_S3D, s3d.new_scstart,
 140                         ~SAVAGE_SCISSOR_MASK_S3D);
 141         SAVE_STATE_MASK(SAVAGE_SCEND_S3D, s3d.new_scend,
 142                         ~SAVAGE_SCISSOR_MASK_S3D);
 143
 144         /* if any texture regs were changed ... */
 145         if (start <= SAVAGE_TEXCTRL_S3D &&
 146             start + count > SAVAGE_TEXPALADDR_S3D) {
 147                 /* ... check texture state */
 148                 SAVE_STATE(SAVAGE_TEXCTRL_S3D, s3d.texctrl);
 149                 SAVE_STATE(SAVAGE_TEXADDR_S3D, s3d.texaddr);
 150                 if (dev_priv->state.s3d.texctrl & SAVAGE_TEXCTRL_TEXEN_MASK)
 151                         return savage_verify_texaddr(dev_priv, 0,
 152                                                 dev_priv->state.s3d.texaddr);
 153         }
 154
 155         return 0;
 156 }
 157
 158 static int savage_verify_state_s4(drm_savage_private_t * dev_priv,
 159                                   unsigned int start, unsigned int count,
 160                                   const uint32_t *regs)
 161 {
 162         int ret = 0;
 163
 164         if (start < SAVAGE_DRAWLOCALCTRL_S4 ||
 165             start + count - 1 > SAVAGE_TEXBLENDCOLOR_S4) {
 166                 DRM_ERROR("invalid register range (0x%04x-0x%04x)\n",
 167                           start, start + count - 1);
 168                 return -EINVAL;
 169         }
 170
 171         SAVE_STATE_MASK(SAVAGE_DRAWCTRL0_S4, s4.new_drawctrl0,
 172                         ~SAVAGE_SCISSOR_MASK_S4);
 173         SAVE_STATE_MASK(SAVAGE_DRAWCTRL1_S4, s4.new_drawctrl1,
 174                         ~SAVAGE_SCISSOR_MASK_S4);
 175
 176         /* if any texture regs were changed ... */
 177         if (start <= SAVAGE_TEXDESCR_S4 &&
 178             start + count > SAVAGE_TEXPALADDR_S4) {
 179                 /* ... check texture state */
 180                 SAVE_STATE(SAVAGE_TEXDESCR_S4, s4.texdescr);
 181                 SAVE_STATE(SAVAGE_TEXADDR0_S4, s4.texaddr0);
 182                 SAVE_STATE(SAVAGE_TEXADDR1_S4, s4.texaddr1);
 183                 if (dev_priv->state.s4.texdescr & SAVAGE_TEXDESCR_TEX0EN_MASK)
 184                         ret |= savage_verify_texaddr(dev_priv, 0,
 185                                                 dev_priv->state.s4.texaddr0);
 186                 if (dev_priv->state.s4.texdescr & SAVAGE_TEXDESCR_TEX1EN_MASK)
 187                         ret |= savage_verify_texaddr(dev_priv, 1,
 188                                                 dev_priv->state.s4.texaddr1);
 189         }
 190
 191         return ret;
 192 }
 193
 194 #undef SAVE_STATE
 195 #undef SAVE_STATE_MASK
 196
 197 static int savage_dispatch_state(drm_savage_private_t * dev_priv,
 198                                  const drm_savage_cmd_header_t * cmd_header,
 199                                  const uint32_t *regs)
 200 {
 201         unsigned int count = cmd_header->state.count;
 202         unsigned int start = cmd_header->state.start;
 203         unsigned int count2 = 0;
 204         unsigned int bci_size;
 205         int ret;
 206         DMA_LOCALS;
 207
 208         if (!count)
 209                 return 0;
 210
 211         if (S3_SAVAGE3D_SERIES(dev_priv->chipset)) {
 212                 ret = savage_verify_state_s3d(dev_priv, start, count, regs);
 213                 if (ret != 0)
 214                         return ret;
 215                 /* scissor regs are emitted in savage_dispatch_draw */
 216                 if (start < SAVAGE_SCSTART_S3D) {
 217                         if (start + count > SAVAGE_SCEND_S3D + 1)
 218                                 count2 = count - (SAVAGE_SCEND_S3D + 1 - start);
 219                         if (start + count > SAVAGE_SCSTART_S3D)
 220                                 count = SAVAGE_SCSTART_S3D - start;
 221                 } else if (start <= SAVAGE_SCEND_S3D) {
 222                         if (start + count > SAVAGE_SCEND_S3D + 1) {
 223                                 count -= SAVAGE_SCEND_S3D + 1 - start;
 224                                 start = SAVAGE_SCEND_S3D + 1;
 225                         } else
 226                                 return 0;
 227                 }
 228         } else {
 229                 ret = savage_verify_state_s4(dev_priv, start, count, regs);
 230                 if (ret != 0)
 231                         return ret;
 232                 /* scissor regs are emitted in savage_dispatch_draw */
 233                 if (start < SAVAGE_DRAWCTRL0_S4) {
 234                         if (start + count > SAVAGE_DRAWCTRL1_S4 + 1)
 235                                 count2 = count -
 236                                          (SAVAGE_DRAWCTRL1_S4 + 1 - start);
 237                         if (start + count > SAVAGE_DRAWCTRL0_S4)
 238                                 count = SAVAGE_DRAWCTRL0_S4 - start;
 239                 } else if (start <= SAVAGE_DRAWCTRL1_S4) {
 240                         if (start + count > SAVAGE_DRAWCTRL1_S4 + 1) {
 241                                 count -= SAVAGE_DRAWCTRL1_S4 + 1 - start;
 242                                 start = SAVAGE_DRAWCTRL1_S4 + 1;
 243                         } else
 244                                 return 0;
 245                 }
 246         }
 247
 248         bci_size = count + (count + 254) / 255 + count2 + (count2 + 254) / 255;
 249
 250         if (cmd_header->state.global) {
 251                 BEGIN_DMA(bci_size + 1);
 252                 DMA_WRITE(BCI_CMD_WAIT | BCI_CMD_WAIT_3D);
 253                 dev_priv->waiting = 1;
 254         } else {
 255                 BEGIN_DMA(bci_size);
 256         }
 257
 258         do {
 259                 while (count > 0) {
 260                         unsigned int n = count < 255 ? count : 255;
 261                         DMA_SET_REGISTERS(start, n);
 262                         DMA_COPY(regs, n);
 263                         count -= n;
 264                         start += n;
 265                         regs += n;
 266                 }
 267                 start += 2;
 268                 regs += 2;
 269                 count = count2;
 270                 count2 = 0;
 271         } while (count);
 272
 273         DMA_COMMIT();
 274
 275         return 0;
 276 }
 277
 278 static int savage_dispatch_dma_prim(drm_savage_private_t * dev_priv,
 279                                     const drm_savage_cmd_header_t * cmd_header,
 280                                     const struct drm_buf * dmabuf)
 281 {
 282         unsigned char reorder = 0;
 283         unsigned int prim = cmd_header->prim.prim;
 284         unsigned int skip = cmd_header->prim.skip;
 285         unsigned int n = cmd_header->prim.count;
 286         unsigned int start = cmd_header->prim.start;
 287         unsigned int i;
 288         BCI_LOCALS;
 289
 290         if (!dmabuf) {
 291                 DRM_ERROR("called without dma buffers!\n");
 292                 return -EINVAL;
 293         }
 294
 295         if (!n)
 296                 return 0;
 297
 298         switch (prim) {
 299         case SAVAGE_PRIM_TRILIST_201:
 300                 reorder = 1;
 301                 prim = SAVAGE_PRIM_TRILIST;
 302                 /* fall through */
 303         case SAVAGE_PRIM_TRILIST:
 304                 if (n % 3 != 0) {
 305                         DRM_ERROR("wrong number of vertices %u in TRILIST\n",
 306                                   n);
 307                         return -EINVAL;
 308                 }
 309                 break;
 310         case SAVAGE_PRIM_TRISTRIP:
 311         case SAVAGE_PRIM_TRIFAN:
 312                 if (n < 3) {
 313                         DRM_ERROR
 314                             ("wrong number of vertices %u in TRIFAN/STRIP\n",
 315                              n);
 316                         return -EINVAL;
 317                 }
 318                 break;
 319         default:
 320                 DRM_ERROR("invalid primitive type %u\n", prim);
 321                 return -EINVAL;
 322         }
 323
 324         if (S3_SAVAGE3D_SERIES(dev_priv->chipset)) {
 325                 if (skip != 0) {
 326                         DRM_ERROR("invalid skip flags 0x%04x for DMA\n", skip);
 327                         return -EINVAL;
 328                 }
 329         } else {
 330                 unsigned int size = 10 - (skip & 1) - (skip >> 1 & 1) -
 331                     (skip >> 2 & 1) - (skip >> 3 & 1) - (skip >> 4 & 1) -
 332                     (skip >> 5 & 1) - (skip >> 6 & 1) - (skip >> 7 & 1);
 333                 if (skip > SAVAGE_SKIP_ALL_S4 || size != 8) {
 334                         DRM_ERROR("invalid skip flags 0x%04x for DMA\n", skip);
 335                         return -EINVAL;
 336                 }
 337                 if (reorder) {
 338                         DRM_ERROR("TRILIST_201 used on Savage4 hardware\n");
 339                         return -EINVAL;
 340                 }
 341         }
 342
 343         if (start + n > dmabuf->total / 32) {
 344                 DRM_ERROR("vertex indices (%u-%u) out of range (0-%u)\n",
 345                           start, start + n - 1, dmabuf->total / 32);
 346                 return -EINVAL;
 347         }
 348
 349         /* Vertex DMA doesn't work with command DMA at the same time,
 350          * so we use BCI_... to submit commands here. Flush buffered
 351          * faked DMA first. */
 352         DMA_FLUSH();
 353
 354         if (dmabuf->bus_address != dev_priv->state.common.vbaddr) {
 355                 BEGIN_BCI(2);
 356                 BCI_SET_REGISTERS(SAVAGE_VERTBUFADDR, 1);
 357                 BCI_WRITE(dmabuf->bus_address | dev_priv->dma_type);
 358                 dev_priv->state.common.vbaddr = dmabuf->bus_address;
 359         }
 360         if (S3_SAVAGE3D_SERIES(dev_priv->chipset) && dev_priv->waiting) {
 361                 /* Workaround for what looks like a hardware bug. If a
 362                  * WAIT_3D_IDLE was emitted some time before the
 363                  * indexed drawing command then the engine will lock
 364                  * up. There are two known workarounds:
 365                  * WAIT_IDLE_EMPTY or emit at least 63 NOPs. */
 366                 BEGIN_BCI(63);
 367                 for (i = 0; i < 63; ++i)
 368                         BCI_WRITE(BCI_CMD_WAIT);
 369                 dev_priv->waiting = 0;
 370         }
 371
 372         prim <<= 25;
 373         while (n != 0) {
 374                 /* Can emit up to 255 indices (85 triangles) at once. */
 375                 unsigned int count = n > 255 ? 255 : n;
 376                 if (reorder) {
 377                         /* Need to reorder indices for correct flat
 378                          * shading while preserving the clock sense
 379                          * for correct culling. Only on Savage3D. */
 380                         int reorder[3] = { -1, -1, -1 };
 381                         reorder[start % 3] = 2;
 382
 383                         BEGIN_BCI((count + 1 + 1) / 2);
 384                         BCI_DRAW_INDICES_S3D(count, prim, start + 2);
 385
 386                         for (i = start + 1; i + 1 < start + count; i += 2)
 387                                 BCI_WRITE((i + reorder[i % 3]) |
 388                                           ((i + 1 +
 389                                             reorder[(i + 1) % 3]) << 16));
 390                         if (i < start + count)
 391                                 BCI_WRITE(i + reorder[i % 3]);
 392                 } else if (S3_SAVAGE3D_SERIES(dev_priv->chipset)) {
 393                         BEGIN_BCI((count + 1 + 1) / 2);
 394                         BCI_DRAW_INDICES_S3D(count, prim, start);
 395
 396                         for (i = start + 1; i + 1 < start + count; i += 2)
 397                                 BCI_WRITE(i | ((i + 1) << 16));
 398                         if (i < start + count)
 399                                 BCI_WRITE(i);
 400                 } else {
 401                         BEGIN_BCI((count + 2 + 1) / 2);
 402                         BCI_DRAW_INDICES_S4(count, prim, skip);
 403
 404                         for (i = start; i + 1 < start + count; i += 2)
 405                                 BCI_WRITE(i | ((i + 1) << 16));
 406                         if (i < start + count)
 407                                 BCI_WRITE(i);
 408                 }
 409
 410                 start += count;
 411                 n -= count;
 412
 413                 prim |= BCI_CMD_DRAW_CONT;
 414         }
 415
 416         return 0;
 417 }
 418
 419 static int savage_dispatch_vb_prim(drm_savage_private_t * dev_priv,
 420                                    const drm_savage_cmd_header_t * cmd_header,
 421                                    const uint32_t *vtxbuf, unsigned int vb_size,
 422                                    unsigned int vb_stride)
 423 {
 424         unsigned char reorder = 0;
 425         unsigned int prim = cmd_header->prim.prim;
 426         unsigned int skip = cmd_header->prim.skip;
 427         unsigned int n = cmd_header->prim.count;
 428         unsigned int start = cmd_header->prim.start;
 429         unsigned int vtx_size;
 430         unsigned int i;
 431         DMA_LOCALS;
 432
 433         if (!n)
 434                 return 0;
 435
 436         switch (prim) {
 437         case SAVAGE_PRIM_TRILIST_201:
 438                 reorder = 1;
 439                 prim = SAVAGE_PRIM_TRILIST;
 440                 /* fall through */
 441         case SAVAGE_PRIM_TRILIST:
 442                 if (n % 3 != 0) {
 443                         DRM_ERROR("wrong number of vertices %u in TRILIST\n",
 444                                   n);
 445                         return -EINVAL;
 446                 }
 447                 break;
 448         case SAVAGE_PRIM_TRISTRIP:
 449         case SAVAGE_PRIM_TRIFAN:
 450                 if (n < 3) {
 451                         DRM_ERROR
 452                             ("wrong number of vertices %u in TRIFAN/STRIP\n",
 453                              n);
 454                         return -EINVAL;
 455                 }
 456                 break;
 457         default:
 458                 DRM_ERROR("invalid primitive type %u\n", prim);
 459                 return -EINVAL;
 460         }
 461
 462         if (S3_SAVAGE3D_SERIES(dev_priv->chipset)) {
 463                 if (skip > SAVAGE_SKIP_ALL_S3D) {
 464                         DRM_ERROR("invalid skip flags 0x%04x\n", skip);
 465                         return -EINVAL;
 466                 }
 467                 vtx_size = 8;   /* full vertex */
 468         } else {
 469                 if (skip > SAVAGE_SKIP_ALL_S4) {
 470                         DRM_ERROR("invalid skip flags 0x%04x\n", skip);
 471                         return -EINVAL;
 472                 }
 473                 vtx_size = 10;  /* full vertex */
 474         }
 475
 476         vtx_size -= (skip & 1) + (skip >> 1 & 1) +
 477             (skip >> 2 & 1) + (skip >> 3 & 1) + (skip >> 4 & 1) +
 478             (skip >> 5 & 1) + (skip >> 6 & 1) + (skip >> 7 & 1);
 479
 480         if (vtx_size > vb_stride) {
 481                 DRM_ERROR("vertex size greater than vb stride (%u > %u)\n",
 482                           vtx_size, vb_stride);
 483                 return -EINVAL;
 484         }
 485
 486         if (start + n > vb_size / (vb_stride * 4)) {
 487                 DRM_ERROR("vertex indices (%u-%u) out of range (0-%u)\n",
 488                           start, start + n - 1, vb_size / (vb_stride * 4));
 489                 return -EINVAL;
 490         }
 491
 492         prim <<= 25;
 493         while (n != 0) {
 494                 /* Can emit up to 255 vertices (85 triangles) at once. */
 495                 unsigned int count = n > 255 ? 255 : n;
 496                 if (reorder) {
 497                         /* Need to reorder vertices for correct flat
 498                          * shading while preserving the clock sense
 499                          * for correct culling. Only on Savage3D. */
 500                         int reorder[3] = { -1, -1, -1 };
 501                         reorder[start % 3] = 2;
 502
 503                         BEGIN_DMA(count * vtx_size + 1);
 504                         DMA_DRAW_PRIMITIVE(count, prim, skip);
 505
 506                         for (i = start; i < start + count; ++i) {
 507                                 unsigned int j = i + reorder[i % 3];
 508                                 DMA_COPY(&vtxbuf[vb_stride * j], vtx_size);
 509                         }
 510
 511                         DMA_COMMIT();
 512                 } else {
 513                         BEGIN_DMA(count * vtx_size + 1);
 514                         DMA_DRAW_PRIMITIVE(count, prim, skip);
 515
 516                         if (vb_stride == vtx_size) {
 517                                 DMA_COPY(&vtxbuf[vb_stride * start],
 518                                          vtx_size * count);
 519                         } else {
 520                                 for (i = start; i < start + count; ++i) {
 521                                         DMA_COPY(&vtxbuf [vb_stride * i],
 522                                                  vtx_size);
 523                                 }
 524                         }
 525
 526                         DMA_COMMIT();
 527                 }
 528
 529                 start += count;
 530                 n -= count;
 531
 532                 prim |= BCI_CMD_DRAW_CONT;
 533         }
 534
 535         return 0;
 536 }
 537
 538 static int savage_dispatch_dma_idx(drm_savage_private_t * dev_priv,
 539                                    const drm_savage_cmd_header_t * cmd_header,
 540                                    const uint16_t *idx,
 541                                    const struct drm_buf * dmabuf)
 542 {
 543         unsigned char reorder = 0;
 544         unsigned int prim = cmd_header->idx.prim;
 545         unsigned int skip = cmd_header->idx.skip;
 546         unsigned int n = cmd_header->idx.count;
 547         unsigned int i;
 548         BCI_LOCALS;
 549
 550         if (!dmabuf) {
 551                 DRM_ERROR("called without dma buffers!\n");
 552                 return -EINVAL;
 553         }
 554
 555         if (!n)
 556                 return 0;
 557
 558         switch (prim) {
 559         case SAVAGE_PRIM_TRILIST_201:
 560                 reorder = 1;
 561                 prim = SAVAGE_PRIM_TRILIST;
 562                 /* fall through */
 563         case SAVAGE_PRIM_TRILIST:
 564                 if (n % 3 != 0) {
 565                         DRM_ERROR("wrong number of indices %u in TRILIST\n", n);
 566                         return -EINVAL;
 567                 }
 568                 break;
 569         case SAVAGE_PRIM_TRISTRIP:
 570         case SAVAGE_PRIM_TRIFAN:
 571                 if (n < 3) {
 572                         DRM_ERROR
 573                             ("wrong number of indices %u in TRIFAN/STRIP\n", n);
 574                         return -EINVAL;
 575                 }
 576                 break;
 577         default:
 578                 DRM_ERROR("invalid primitive type %u\n", prim);
 579                 return -EINVAL;
 580         }
 581
 582         if (S3_SAVAGE3D_SERIES(dev_priv->chipset)) {
 583                 if (skip != 0) {
 584                         DRM_ERROR("invalid skip flags 0x%04x for DMA\n", skip);
 585                         return -EINVAL;
 586                 }
 587         } else {
 588                 unsigned int size = 10 - (skip & 1) - (skip >> 1 & 1) -
 589                     (skip >> 2 & 1) - (skip >> 3 & 1) - (skip >> 4 & 1) -
 590                     (skip >> 5 & 1) - (skip >> 6 & 1) - (skip >> 7 & 1);
 591                 if (skip > SAVAGE_SKIP_ALL_S4 || size != 8) {
 592                         DRM_ERROR("invalid skip flags 0x%04x for DMA\n", skip);
 593                         return -EINVAL;
 594                 }
 595                 if (reorder) {
 596                         DRM_ERROR("TRILIST_201 used on Savage4 hardware\n");
 597                         return -EINVAL;
 598                 }
 599         }
 600
 601         /* Vertex DMA doesn't work with command DMA at the same time,
 602          * so we use BCI_... to submit commands here. Flush buffered
 603          * faked DMA first. */
 604         DMA_FLUSH();
 605
 606         if (dmabuf->bus_address != dev_priv->state.common.vbaddr) {
 607                 BEGIN_BCI(2);
 608                 BCI_SET_REGISTERS(SAVAGE_VERTBUFADDR, 1);
 609                 BCI_WRITE(dmabuf->bus_address | dev_priv->dma_type);
 610                 dev_priv->state.common.vbaddr = dmabuf->bus_address;
 611         }
 612         if (S3_SAVAGE3D_SERIES(dev_priv->chipset) && dev_priv->waiting) {
 613                 /* Workaround for what looks like a hardware bug. If a
 614                  * WAIT_3D_IDLE was emitted some time before the
 615                  * indexed drawing command then the engine will lock
 616                  * up. There are two known workarounds:
 617                  * WAIT_IDLE_EMPTY or emit at least 63 NOPs. */
 618                 BEGIN_BCI(63);
 619                 for (i = 0; i < 63; ++i)
 620                         BCI_WRITE(BCI_CMD_WAIT);
 621                 dev_priv->waiting = 0;
 622         }
 623
 624         prim <<= 25;
 625         while (n != 0) {
 626                 /* Can emit up to 255 indices (85 triangles) at once. */
 627                 unsigned int count = n > 255 ? 255 : n;
 628
 629                 /* check indices */
 630                 for (i = 0; i < count; ++i) {
 631                         if (idx[i] > dmabuf->total / 32) {
 632                                 DRM_ERROR("idx[%u]=%u out of range (0-%u)\n",
 633                                           i, idx[i], dmabuf->total / 32);
 634                                 return -EINVAL;
 635                         }
 636                 }
 637
 638                 if (reorder) {
 639                         /* Need to reorder indices for correct flat
 640                          * shading while preserving the clock sense
 641                          * for correct culling. Only on Savage3D. */
 642                         int reorder[3] = { 2, -1, -1 };
 643
 644                         BEGIN_BCI((count + 1 + 1) / 2);
 645                         BCI_DRAW_INDICES_S3D(count, prim, idx[2]);
 646
 647                         for (i = 1; i + 1 < count; i += 2)
 648                                 BCI_WRITE(idx[i + reorder[i % 3]] |
 649                                           (idx[i + 1 +
 650                                            reorder[(i + 1) % 3]] << 16));
 651                         if (i < count)
 652                                 BCI_WRITE(idx[i + reorder[i % 3]]);
 653                 } else if (S3_SAVAGE3D_SERIES(dev_priv->chipset)) {
 654                         BEGIN_BCI((count + 1 + 1) / 2);
 655                         BCI_DRAW_INDICES_S3D(count, prim, idx[0]);
 656
 657                         for (i = 1; i + 1 < count; i += 2)
 658                                 BCI_WRITE(idx[i] | (idx[i + 1] << 16));
 659                         if (i < count)
 660                                 BCI_WRITE(idx[i]);
 661                 } else {
 662                         BEGIN_BCI((count + 2 + 1) / 2);
 663                         BCI_DRAW_INDICES_S4(count, prim, skip);
 664
 665                         for (i = 0; i + 1 < count; i += 2)
 666                                 BCI_WRITE(idx[i] | (idx[i + 1] << 16));
 667                         if (i < count)
 668                                 BCI_WRITE(idx[i]);
 669                 }
 670
 671                 idx += count;
 672                 n -= count;
 673
 674                 prim |= BCI_CMD_DRAW_CONT;
 675         }
 676
 677         return 0;
 678 }
 679
 680 static int savage_dispatch_vb_idx(drm_savage_private_t * dev_priv,
 681                                   const drm_savage_cmd_header_t * cmd_header,
 682                                   const uint16_t *idx,
 683                                   const uint32_t *vtxbuf,
 684                                   unsigned int vb_size, unsigned int vb_stride)
 685 {
 686         unsigned char reorder = 0;
 687         unsigned int prim = cmd_header->idx.prim;
 688         unsigned int skip = cmd_header->idx.skip;
 689         unsigned int n = cmd_header->idx.count;
 690         unsigned int vtx_size;
 691         unsigned int i;
 692         DMA_LOCALS;
 693
 694         if (!n)
 695                 return 0;
 696
 697         switch (prim) {
 698         case SAVAGE_PRIM_TRILIST_201:
 699                 reorder = 1;
 700                 prim = SAVAGE_PRIM_TRILIST;
 701                 /* fall through */
 702         case SAVAGE_PRIM_TRILIST:
 703                 if (n % 3 != 0) {
 704                         DRM_ERROR("wrong number of indices %u in TRILIST\n", n);
 705                         return -EINVAL;
 706                 }
 707                 break;
 708         case SAVAGE_PRIM_TRISTRIP:
 709         case SAVAGE_PRIM_TRIFAN:
 710                 if (n < 3) {
 711                         DRM_ERROR
 712                             ("wrong number of indices %u in TRIFAN/STRIP\n", n);
 713                         return -EINVAL;
 714                 }
 715                 break;
 716         default:
 717                 DRM_ERROR("invalid primitive type %u\n", prim);
 718                 return -EINVAL;
 719         }
 720
 721         if (S3_SAVAGE3D_SERIES(dev_priv->chipset)) {
 722                 if (skip > SAVAGE_SKIP_ALL_S3D) {
 723                         DRM_ERROR("invalid skip flags 0x%04x\n", skip);
 724                         return -EINVAL;
 725                 }
 726                 vtx_size = 8;   /* full vertex */
 727         } else {
 728                 if (skip > SAVAGE_SKIP_ALL_S4) {
 729                         DRM_ERROR("invalid skip flags 0x%04x\n", skip);
 730                         return -EINVAL;
 731                 }
 732                 vtx_size = 10;  /* full vertex */
 733         }
 734
 735         vtx_size -= (skip & 1) + (skip >> 1 & 1) +
 736             (skip >> 2 & 1) + (skip >> 3 & 1) + (skip >> 4 & 1) +
 737             (skip >> 5 & 1) + (skip >> 6 & 1) + (skip >> 7 & 1);
 738
 739         if (vtx_size > vb_stride) {
 740                 DRM_ERROR("vertex size greater than vb stride (%u > %u)\n",
 741                           vtx_size, vb_stride);
 742                 return -EINVAL;
 743         }
 744
 745         prim <<= 25;
 746         while (n != 0) {
 747                 /* Can emit up to 255 vertices (85 triangles) at once. */
 748                 unsigned int count = n > 255 ? 255 : n;
 749
 750                 /* Check indices */
 751                 for (i = 0; i < count; ++i) {
 752                         if (idx[i] > vb_size / (vb_stride * 4)) {
 753                                 DRM_ERROR("idx[%u]=%u out of range (0-%u)\n",
 754                                           i, idx[i], vb_size / (vb_stride * 4));
 755                                 return -EINVAL;
 756                         }
 757                 }
 758
 759                 if (reorder) {
 760                         /* Need to reorder vertices for correct flat
 761                          * shading while preserving the clock sense
 762                          * for correct culling. Only on Savage3D. */
 763                         int reorder[3] = { 2, -1, -1 };
 764
 765                         BEGIN_DMA(count * vtx_size + 1);
 766                         DMA_DRAW_PRIMITIVE(count, prim, skip);
 767
 768                         for (i = 0; i < count; ++i) {
 769                                 unsigned int j = idx[i + reorder[i % 3]];
 770                                 DMA_COPY(&vtxbuf[vb_stride * j], vtx_size);
 771                         }
 772
 773                         DMA_COMMIT();
 774                 } else {
 775                         BEGIN_DMA(count * vtx_size + 1);
 776                         DMA_DRAW_PRIMITIVE(count, prim, skip);
 777
 778                         for (i = 0; i < count; ++i) {
 779                                 unsigned int j = idx[i];
 780                                 DMA_COPY(&vtxbuf[vb_stride * j], vtx_size);
 781                         }
 782
 783                         DMA_COMMIT();
 784                 }
 785
 786                 idx += count;
 787                 n -= count;
 788
 789                 prim |= BCI_CMD_DRAW_CONT;
 790         }
 791
 792         return 0;
 793 }
 794
 795 static int savage_dispatch_clear(drm_savage_private_t * dev_priv,
 796                                  const drm_savage_cmd_header_t * cmd_header,
 797                                  const drm_savage_cmd_header_t *data,
 798                                  unsigned int nbox,
 799                                  const struct drm_clip_rect *boxes)
 800 {
 801         unsigned int flags = cmd_header->clear0.flags;
 802         unsigned int clear_cmd;
 803         unsigned int i, nbufs;
 804         DMA_LOCALS;
 805
 806         if (nbox == 0)
 807                 return 0;
 808
 809         clear_cmd = BCI_CMD_RECT | BCI_CMD_RECT_XP | BCI_CMD_RECT_YP |
 810             BCI_CMD_SEND_COLOR | BCI_CMD_DEST_PBD_NEW;
 811         BCI_CMD_SET_ROP(clear_cmd, 0xCC);
 812
 813         nbufs = ((flags & SAVAGE_FRONT) ? 1 : 0) +
 814             ((flags & SAVAGE_BACK) ? 1 : 0) + ((flags & SAVAGE_DEPTH) ? 1 : 0);
 815         if (nbufs == 0)
 816                 return 0;
 817
 818         if (data->clear1.mask != 0xffffffff) {
 819                 /* set mask */
 820                 BEGIN_DMA(2);
 821                 DMA_SET_REGISTERS(SAVAGE_BITPLANEWTMASK, 1);
 822                 DMA_WRITE(data->clear1.mask);
 823                 DMA_COMMIT();
 824         }
 825         for (i = 0; i < nbox; ++i) {
 826                 unsigned int x, y, w, h;
 827                 unsigned int buf;
 828                 x = boxes[i].x1, y = boxes[i].y1;
 829                 w = boxes[i].x2 - boxes[i].x1;
 830                 h = boxes[i].y2 - boxes[i].y1;
 831                 BEGIN_DMA(nbufs * 6);
 832                 for (buf = SAVAGE_FRONT; buf <= SAVAGE_DEPTH; buf <<= 1) {
 833                         if (!(flags & buf))
 834                                 continue;
 835                         DMA_WRITE(clear_cmd);
 836                         switch (buf) {
 837                         case SAVAGE_FRONT:
 838                                 DMA_WRITE(dev_priv->front_offset);
 839                                 DMA_WRITE(dev_priv->front_bd);
 840                                 break;
 841                         case SAVAGE_BACK:
 842                                 DMA_WRITE(dev_priv->back_offset);
 843                                 DMA_WRITE(dev_priv->back_bd);
 844                                 break;
 845                         case SAVAGE_DEPTH:
 846                                 DMA_WRITE(dev_priv->depth_offset);
 847                                 DMA_WRITE(dev_priv->depth_bd);
 848                                 break;
 849                         }
 850                         DMA_WRITE(data->clear1.value);
 851                         DMA_WRITE(BCI_X_Y(x, y));
 852                         DMA_WRITE(BCI_W_H(w, h));
 853                 }
 854                 DMA_COMMIT();
 855         }
 856         if (data->clear1.mask != 0xffffffff) {
 857                 /* reset mask */
 858                 BEGIN_DMA(2);
 859                 DMA_SET_REGISTERS(SAVAGE_BITPLANEWTMASK, 1);
 860                 DMA_WRITE(0xffffffff);
 861                 DMA_COMMIT();
 862         }
 863
 864         return 0;
 865 }
 866
 867 static int savage_dispatch_swap(drm_savage_private_t * dev_priv,
 868                                 unsigned int nbox, const struct drm_clip_rect *boxes)
 869 {
 870         unsigned int swap_cmd;
 871         unsigned int i;
 872         DMA_LOCALS;
 873
 874         if (nbox == 0)
 875                 return 0;
 876
 877         swap_cmd = BCI_CMD_RECT | BCI_CMD_RECT_XP | BCI_CMD_RECT_YP |
 878             BCI_CMD_SRC_PBD_COLOR_NEW | BCI_CMD_DEST_GBD;
 879         BCI_CMD_SET_ROP(swap_cmd, 0xCC);
 880
 881         for (i = 0; i < nbox; ++i) {
 882                 BEGIN_DMA(6);
 883                 DMA_WRITE(swap_cmd);
 884                 DMA_WRITE(dev_priv->back_offset);
 885                 DMA_WRITE(dev_priv->back_bd);
 886                 DMA_WRITE(BCI_X_Y(boxes[i].x1, boxes[i].y1));
 887                 DMA_WRITE(BCI_X_Y(boxes[i].x1, boxes[i].y1));
 888                 DMA_WRITE(BCI_W_H(boxes[i].x2 - boxes[i].x1,
 889                                   boxes[i].y2 - boxes[i].y1));
 890                 DMA_COMMIT();
 891         }
 892
 893         return 0;
 894 }
 895
 896 static int savage_dispatch_draw(drm_savage_private_t * dev_priv,
 897                                 const drm_savage_cmd_header_t *start,
 898                                 const drm_savage_cmd_header_t *end,
 899                                 const struct drm_buf * dmabuf,
 900                                 const unsigned int *vtxbuf,
 901                                 unsigned int vb_size, unsigned int vb_stride,
 902                                 unsigned int nbox,
 903                                 const struct drm_clip_rect *boxes)
 904 {
 905         unsigned int i, j;
 906         int ret;
 907
 908         for (i = 0; i < nbox; ++i) {
 909                 const drm_savage_cmd_header_t *cmdbuf;
 910                 dev_priv->emit_clip_rect(dev_priv, &boxes[i]);
 911
 912                 cmdbuf = start;
 913                 while (cmdbuf < end) {
 914                         drm_savage_cmd_header_t cmd_header;
 915                         cmd_header = *cmdbuf;
 916                         cmdbuf++;
 917                         switch (cmd_header.cmd.cmd) {
 918                         case SAVAGE_CMD_DMA_PRIM:
 919                                 ret = savage_dispatch_dma_prim(
 920                                         dev_priv, &cmd_header, dmabuf);
 921                                 break;
 922                         case SAVAGE_CMD_VB_PRIM:
 923                                 ret = savage_dispatch_vb_prim(
 924                                         dev_priv, &cmd_header,
 925                                         vtxbuf, vb_size, vb_stride);
 926                                 break;
 927                         case SAVAGE_CMD_DMA_IDX:
 928                                 j = (cmd_header.idx.count + 3) / 4;
 929                                 /* j was check in savage_bci_cmdbuf */
 930                                 ret = savage_dispatch_dma_idx(dev_priv,
 931                                         &cmd_header, (const uint16_t *)cmdbuf,
 932                                         dmabuf);
 933                                 cmdbuf += j;
 934                                 break;
 935                         case SAVAGE_CMD_VB_IDX:
 936                                 j = (cmd_header.idx.count + 3) / 4;
 937                                 /* j was check in savage_bci_cmdbuf */
 938                                 ret = savage_dispatch_vb_idx(dev_priv,
 939                                         &cmd_header, (const uint16_t *)cmdbuf,
 940                                         (const uint32_t *)vtxbuf, vb_size,
 941                                         vb_stride);
 942                                 cmdbuf += j;
 943                                 break;
 944                         default:
 945                                 /* What's the best return code? EFAULT? */
 946                                 DRM_ERROR("IMPLEMENTATION ERROR: "
 947                                           "non-drawing-command %d\n",
 948                                           cmd_header.cmd.cmd);
 949                                 return -EINVAL;
 950                         }
 951
 952                         if (ret != 0)
 953                                 return ret;
 954                 }
 955         }
 956
 957         return 0;
 958 }
 959
 960 int savage_bci_cmdbuf(struct drm_device *dev, void *data, struct drm_file *file_priv)
 961 {
 962         drm_savage_private_t *dev_priv = dev->dev_private;
 963         struct drm_device_dma *dma = dev->dma;
 964         struct drm_buf *dmabuf;
 965         drm_savage_cmdbuf_t *cmdbuf = data;
 966         drm_savage_cmd_header_t *kcmd_addr = NULL;
 967         drm_savage_cmd_header_t *first_draw_cmd;
 968         unsigned int *kvb_addr = NULL;
 969         struct drm_clip_rect *kbox_addr = NULL;
 970         unsigned int i, j;
 971         int ret = 0;
 972
 973         DRM_DEBUG("\n");
 974
 975         LOCK_TEST_WITH_RETURN(dev, file_priv);
 976
 977         if (dma && dma->buflist) {
 978                 if (cmdbuf->dma_idx >= dma->buf_count) {
 979                         DRM_ERROR
 980                             ("vertex buffer index %u out of range (0-%u)\n",
 981                              cmdbuf->dma_idx, dma->buf_count - 1);
 982                         return -EINVAL;
 983                 }
 984                 dmabuf = dma->buflist[cmdbuf->dma_idx];
 985         } else {
 986                 dmabuf = NULL;
 987         }
 988
 989         /* Copy the user buffers into kernel temporary areas.  This hasn't been
 990          * a performance loss compared to VERIFYAREA_READ/
 991          * COPY_FROM_USER_UNCHECKED when done in other drivers, and is correct
 992          * for locking on FreeBSD.
 993          */
 994         if (cmdbuf->size) {
 995                 kcmd_addr = kmalloc_array(cmdbuf->size, 8, GFP_KERNEL);
 996                 if (kcmd_addr == NULL)
 997                         return -ENOMEM;
 998
 999                 if (copy_from_user(kcmd_addr, cmdbuf->cmd_addr,
1000                                        cmdbuf->size * 8))
1001                 {
1002                         kfree(kcmd_addr);
1003                         return -EFAULT;
1004                 }
1005                 cmdbuf->cmd_addr = kcmd_addr;
1006         }
1007         if (cmdbuf->vb_size) {
1008                 kvb_addr = memdup_user(cmdbuf->vb_addr, cmdbuf->vb_size);
1009                 if (IS_ERR(kvb_addr)) {
1010                         ret = PTR_ERR(kvb_addr);
1011                         kvb_addr = NULL;
1012                         goto done;
1013                 }
1014                 cmdbuf->vb_addr = kvb_addr;
1015         }
1016         if (cmdbuf->nbox) {
1017                 kbox_addr = kmalloc_array(cmdbuf->nbox, sizeof(struct drm_clip_rect),
1018                                           GFP_KERNEL);
1019                 if (kbox_addr == NULL) {
1020                         ret = -ENOMEM;
1021                         goto done;
1022                 }
1023
1024                 if (copy_from_user(kbox_addr, cmdbuf->box_addr,
1025                                        cmdbuf->nbox * sizeof(struct drm_clip_rect))) {
1026                         ret = -EFAULT;
1027                         goto done;
1028                 }
1029         cmdbuf->box_addr = kbox_addr;
1030         }
1031
1032         /* Make sure writes to DMA buffers are finished before sending
1033          * DMA commands to the graphics hardware. */
1034         mb();
1035
1036         /* Coming from user space. Don't know if the Xserver has
1037          * emitted wait commands. Assuming the worst. */
1038         dev_priv->waiting = 1;
1039
1040         i = 0;
1041         first_draw_cmd = NULL;
1042         while (i < cmdbuf->size) {
1043                 drm_savage_cmd_header_t cmd_header;
1044                 cmd_header = *(drm_savage_cmd_header_t *)cmdbuf->cmd_addr;
1045                 cmdbuf->cmd_addr++;
1046                 i++;
1047
1048                 /* Group drawing commands with same state to minimize
1049                  * iterations over clip rects. */
1050                 j = 0;
1051                 switch (cmd_header.cmd.cmd) {
1052                 case SAVAGE_CMD_DMA_IDX:
1053                 case SAVAGE_CMD_VB_IDX:
1054                         j = (cmd_header.idx.count + 3) / 4;
1055                         if (i + j > cmdbuf->size) {
1056                                 DRM_ERROR("indexed drawing command extends "
1057                                           "beyond end of command buffer\n");
1058                                 DMA_FLUSH();
1059                                 ret = -EINVAL;
1060                                 goto done;
1061                         }
1062                         /* fall through */
1063                 case SAVAGE_CMD_DMA_PRIM:
1064                 case SAVAGE_CMD_VB_PRIM:
1065                         if (!first_draw_cmd)
1066                                 first_draw_cmd = cmdbuf->cmd_addr - 1;
1067                         cmdbuf->cmd_addr += j;
1068                         i += j;
1069                         break;
1070                 default:
1071                         if (first_draw_cmd) {
1072                                 ret = savage_dispatch_draw(
1073                                       dev_priv, first_draw_cmd,
1074                                       cmdbuf->cmd_addr - 1,
1075                                       dmabuf, cmdbuf->vb_addr, cmdbuf->vb_size,
1076                                       cmdbuf->vb_stride,
1077                                       cmdbuf->nbox, cmdbuf->box_addr);
1078                                 if (ret != 0)
1079                                         goto done;
1080                                 first_draw_cmd = NULL;
1081                         }
1082                 }
1083                 if (first_draw_cmd)
1084                         continue;
1085
1086                 switch (cmd_header.cmd.cmd) {
1087                 case SAVAGE_CMD_STATE:
1088                         j = (cmd_header.state.count + 1) / 2;
1089                         if (i + j > cmdbuf->size) {
1090                                 DRM_ERROR("command SAVAGE_CMD_STATE extends "
1091                                           "beyond end of command buffer\n");
1092                                 DMA_FLUSH();
1093                                 ret = -EINVAL;
1094                                 goto done;
1095                         }
1096                         ret = savage_dispatch_state(dev_priv, &cmd_header,
1097                                 (const uint32_t *)cmdbuf->cmd_addr);
1098                         cmdbuf->cmd_addr += j;
1099                         i += j;
1100                         break;
1101                 case SAVAGE_CMD_CLEAR:
1102                         if (i + 1 > cmdbuf->size) {
1103                                 DRM_ERROR("command SAVAGE_CMD_CLEAR extends "
1104                                           "beyond end of command buffer\n");
1105                                 DMA_FLUSH();
1106                                 ret = -EINVAL;
1107                                 goto done;
1108                         }
1109                         ret = savage_dispatch_clear(dev_priv, &cmd_header,
1110                                                     cmdbuf->cmd_addr,
1111                                                     cmdbuf->nbox,
1112                                                     cmdbuf->box_addr);
1113                         cmdbuf->cmd_addr++;
1114                         i++;
1115                         break;
1116                 case SAVAGE_CMD_SWAP:
1117                         ret = savage_dispatch_swap(dev_priv, cmdbuf->nbox,
1118                                                    cmdbuf->box_addr);
1119                         break;
1120                 default:
1121                         DRM_ERROR("invalid command 0x%x\n",
1122                                   cmd_header.cmd.cmd);
1123                         DMA_FLUSH();
1124                         ret = -EINVAL;
1125                         goto done;
1126                 }
1127
1128                 if (ret != 0) {
1129                         DMA_FLUSH();
1130                         goto done;
1131                 }
1132         }
1133
1134         if (first_draw_cmd) {
1135                 ret = savage_dispatch_draw (
1136                         dev_priv, first_draw_cmd, cmdbuf->cmd_addr, dmabuf,
1137                         cmdbuf->vb_addr, cmdbuf->vb_size, cmdbuf->vb_stride,
1138                         cmdbuf->nbox, cmdbuf->box_addr);
1139                 if (ret != 0) {
1140                         DMA_FLUSH();
1141                         goto done;
1142                 }
1143         }
1144
1145         DMA_FLUSH();
1146
1147         if (dmabuf && cmdbuf->discard) {
1148                 drm_savage_buf_priv_t *buf_priv = dmabuf->dev_private;
1149                 uint16_t event;
1150                 event = savage_bci_emit_event(dev_priv, SAVAGE_WAIT_3D);
1151                 SET_AGE(&buf_priv->age, event, dev_priv->event_wrap);
1152                 savage_freelist_put(dev, dmabuf);
1153         }
1154
1155 done:
1156         /* If we didn't need to allocate them, these'll be NULL */
1157         kfree(kcmd_addr);
1158         kfree(kvb_addr);
1159         kfree(kbox_addr);
1160
1161         return ret;
1162 }