drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2014-2019 Intel Corporation
   4  */
   5
   6 #include <linux/bsearch.h>
   7
   8 #include "gem/i915_gem_lmem.h"
   9 #include "gt/intel_engine_regs.h"
  10 #include "gt/intel_gt.h"
  11 #include "gt/intel_gt_mcr.h"
  12 #include "gt/intel_gt_regs.h"
  13 #include "gt/intel_lrc.h"
  14 #include "gt/shmem_utils.h"
  15 #include "intel_guc_ads.h"
  16 #include "intel_guc_capture.h"
  17 #include "intel_guc_fwif.h"
  18 #include "intel_guc_print.h"
  19 #include "intel_uc.h"
  20 #include "i915_drv.h"
  21
  22 /*
  23  * The Additional Data Struct (ADS) has pointers for different buffers used by
  24  * the GuC. One single gem object contains the ADS struct itself (guc_ads) and
  25  * all the extra buffers indirectly linked via the ADS struct's entries.
  26  *
  27  * Layout of the ADS blob allocated for the GuC:
  28  *
  29  *      +---------------------------------------+ <== base
  30  *      | guc_ads                               |
  31  *      +---------------------------------------+
  32  *      | guc_policies                          |
  33  *      +---------------------------------------+
  34  *      | guc_gt_system_info                    |
  35  *      +---------------------------------------+
  36  *      | guc_engine_usage                      |
  37  *      +---------------------------------------+ <== static
  38  *      | guc_mmio_reg[countA] (engine 0.0)     |
  39  *      | guc_mmio_reg[countB] (engine 0.1)     |
  40  *      | guc_mmio_reg[countC] (engine 1.0)     |
  41  *      |   ...                                 |
  42  *      +---------------------------------------+ <== dynamic
  43  *      | padding                               |
  44  *      +---------------------------------------+ <== 4K aligned
  45  *      | golden contexts                       |
  46  *      +---------------------------------------+
  47  *      | padding                               |
  48  *      +---------------------------------------+ <== 4K aligned
  49  *      | w/a KLVs                              |
  50  *      +---------------------------------------+
  51  *      | padding                               |
  52  *      +---------------------------------------+ <== 4K aligned
  53  *      | capture lists                         |
  54  *      +---------------------------------------+
  55  *      | padding                               |
  56  *      +---------------------------------------+ <== 4K aligned
  57  *      | private data                          |
  58  *      +---------------------------------------+
  59  *      | padding                               |
  60  *      +---------------------------------------+ <== 4K aligned
  61  */
  62 struct __guc_ads_blob {
  63         struct guc_ads ads;
  64         struct guc_policies policies;
  65         struct guc_gt_system_info system_info;
  66         struct guc_engine_usage engine_usage;
  67         /* From here on, location is dynamic! Refer to above diagram. */
  68         struct guc_mmio_reg regset[];
  69 } __packed;
  70
  71 #define ads_blob_read(guc_, field_)                                     \
  72         iosys_map_rd_field(&(guc_)->ads_map, 0, struct __guc_ads_blob, field_)
  73
  74 #define ads_blob_write(guc_, field_, val_)                              \
  75         iosys_map_wr_field(&(guc_)->ads_map, 0, struct __guc_ads_blob,  \
  76                            field_, val_)
  77
  78 #define info_map_write(map_, field_, val_) \
  79         iosys_map_wr_field(map_, 0, struct guc_gt_system_info, field_, val_)
  80
  81 #define info_map_read(map_, field_) \
  82         iosys_map_rd_field(map_, 0, struct guc_gt_system_info, field_)
  83
  84 static u32 guc_ads_regset_size(struct intel_guc *guc)
  85 {
  86         GEM_BUG_ON(!guc->ads_regset_size);
  87         return guc->ads_regset_size;
  88 }
  89
  90 static u32 guc_ads_golden_ctxt_size(struct intel_guc *guc)
  91 {
  92         return PAGE_ALIGN(guc->ads_golden_ctxt_size);
  93 }
  94
  95 static u32 guc_ads_waklv_size(struct intel_guc *guc)
  96 {
  97         return PAGE_ALIGN(guc->ads_waklv_size);
  98 }
  99
 100 static u32 guc_ads_capture_size(struct intel_guc *guc)
 101 {
 102         return PAGE_ALIGN(guc->ads_capture_size);
 103 }
 104
 105 static u32 guc_ads_private_data_size(struct intel_guc *guc)
 106 {
 107         return PAGE_ALIGN(guc->fw.private_data_size);
 108 }
 109
 110 static u32 guc_ads_regset_offset(struct intel_guc *guc)
 111 {
 112         return offsetof(struct __guc_ads_blob, regset);
 113 }
 114
 115 static u32 guc_ads_golden_ctxt_offset(struct intel_guc *guc)
 116 {
 117         u32 offset;
 118
 119         offset = guc_ads_regset_offset(guc) +
 120                  guc_ads_regset_size(guc);
 121
 122         return PAGE_ALIGN(offset);
 123 }
 124
 125 static u32 guc_ads_waklv_offset(struct intel_guc *guc)
 126 {
 127         u32 offset;
 128
 129         offset = guc_ads_golden_ctxt_offset(guc) +
 130                  guc_ads_golden_ctxt_size(guc);
 131
 132         return PAGE_ALIGN(offset);
 133 }
 134
 135 static u32 guc_ads_capture_offset(struct intel_guc *guc)
 136 {
 137         u32 offset;
 138
 139         offset = guc_ads_waklv_offset(guc) +
 140                  guc_ads_waklv_size(guc);
 141
 142         return PAGE_ALIGN(offset);
 143 }
 144
 145 static u32 guc_ads_private_data_offset(struct intel_guc *guc)
 146 {
 147         u32 offset;
 148
 149         offset = guc_ads_capture_offset(guc) +
 150                  guc_ads_capture_size(guc);
 151
 152         return PAGE_ALIGN(offset);
 153 }
 154
 155 static u32 guc_ads_blob_size(struct intel_guc *guc)
 156 {
 157         return guc_ads_private_data_offset(guc) +
 158                guc_ads_private_data_size(guc);
 159 }
 160
 161 static void guc_policies_init(struct intel_guc *guc)
 162 {
 163         struct intel_gt *gt = guc_to_gt(guc);
 164         struct drm_i915_private *i915 = gt->i915;
 165         u32 global_flags = 0;
 166
 167         ads_blob_write(guc, policies.dpc_promote_time,
 168                        GLOBAL_POLICY_DEFAULT_DPC_PROMOTE_TIME_US);
 169         ads_blob_write(guc, policies.max_num_work_items,
 170                        GLOBAL_POLICY_MAX_NUM_WI);
 171
 172         if (i915->params.reset < 2)
 173                 global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET;
 174
 175         ads_blob_write(guc, policies.global_flags, global_flags);
 176         ads_blob_write(guc, policies.is_valid, 1);
 177 }
 178
 179 void intel_guc_ads_print_policy_info(struct intel_guc *guc,
 180                                      struct drm_printer *dp)
 181 {
 182         if (unlikely(iosys_map_is_null(&guc->ads_map)))
 183                 return;
 184
 185         drm_printf(dp, "Global scheduling policies:\n");
 186         drm_printf(dp, "  DPC promote time   = %u\n",
 187                    ads_blob_read(guc, policies.dpc_promote_time));
 188         drm_printf(dp, "  Max num work items = %u\n",
 189                    ads_blob_read(guc, policies.max_num_work_items));
 190         drm_printf(dp, "  Flags              = %u\n",
 191                    ads_blob_read(guc, policies.global_flags));
 192 }
 193
 194 static int guc_action_policies_update(struct intel_guc *guc, u32 policy_offset)
 195 {
 196         u32 action[] = {
 197                 INTEL_GUC_ACTION_GLOBAL_SCHED_POLICY_CHANGE,
 198                 policy_offset
 199         };
 200
 201         return intel_guc_send_busy_loop(guc, action, ARRAY_SIZE(action), 0, true);
 202 }
 203
 204 int intel_guc_global_policies_update(struct intel_guc *guc)
 205 {
 206         struct intel_gt *gt = guc_to_gt(guc);
 207         u32 scheduler_policies;
 208         intel_wakeref_t wakeref;
 209         int ret;
 210
 211         if (iosys_map_is_null(&guc->ads_map))
 212                 return -EOPNOTSUPP;
 213
 214         scheduler_policies = ads_blob_read(guc, ads.scheduler_policies);
 215         GEM_BUG_ON(!scheduler_policies);
 216
 217         guc_policies_init(guc);
 218
 219         if (!intel_guc_is_ready(guc))
 220                 return 0;
 221
 222         with_intel_runtime_pm(&gt->i915->runtime_pm, wakeref)
 223                 ret = guc_action_policies_update(guc, scheduler_policies);
 224
 225         return ret;
 226 }
 227
 228 static void guc_mapping_table_init(struct intel_gt *gt,
 229                                    struct iosys_map *info_map)
 230 {
 231         unsigned int i, j;
 232         struct intel_engine_cs *engine;
 233         enum intel_engine_id id;
 234
 235         /* Table must be set to invalid values for entries not used */
 236         for (i = 0; i < GUC_MAX_ENGINE_CLASSES; ++i)
 237                 for (j = 0; j < GUC_MAX_INSTANCES_PER_CLASS; ++j)
 238                         info_map_write(info_map, mapping_table[i][j],
 239                                        GUC_MAX_INSTANCES_PER_CLASS);
 240
 241         for_each_engine(engine, gt, id) {
 242                 u8 guc_class = engine_class_to_guc_class(engine->class);
 243
 244                 info_map_write(info_map, mapping_table[guc_class][ilog2(engine->logical_mask)],
 245                                engine->instance);
 246         }
 247 }
 248
 249 /*
 250  * The save/restore register list must be pre-calculated to a temporary
 251  * buffer before it can be copied inside the ADS.
 252  */
 253 struct temp_regset {
 254         /*
 255          * ptr to the section of the storage for the engine currently being
 256          * worked on
 257          */
 258         struct guc_mmio_reg *registers;
 259         /* ptr to the base of the allocated storage for all engines */
 260         struct guc_mmio_reg *storage;
 261         u32 storage_used;
 262         u32 storage_max;
 263 };
 264
 265 static int guc_mmio_reg_cmp(const void *a, const void *b)
 266 {
 267         const struct guc_mmio_reg *ra = a;
 268         const struct guc_mmio_reg *rb = b;
 269
 270         return (int)ra->offset - (int)rb->offset;
 271 }
 272
 273 static struct guc_mmio_reg * __must_check
 274 __mmio_reg_add(struct temp_regset *regset, struct guc_mmio_reg *reg)
 275 {
 276         u32 pos = regset->storage_used;
 277         struct guc_mmio_reg *slot;
 278
 279         if (pos >= regset->storage_max) {
 280                 size_t size = ALIGN((pos + 1) * sizeof(*slot), PAGE_SIZE);
 281                 struct guc_mmio_reg *r = krealloc(regset->storage,
 282                                                   size, GFP_KERNEL);
 283                 if (!r) {
 284                         WARN_ONCE(1, "Incomplete regset list: can't add register (%d)\n",
 285                                   -ENOMEM);
 286                         return ERR_PTR(-ENOMEM);
 287                 }
 288
 289                 regset->registers = r + (regset->registers - regset->storage);
 290                 regset->storage = r;
 291                 regset->storage_max = size / sizeof(*slot);
 292         }
 293
 294         slot = &regset->storage[pos];
 295         regset->storage_used++;
 296         *slot = *reg;
 297
 298         return slot;
 299 }
 300
 301 static long __must_check guc_mmio_reg_add(struct intel_gt *gt,
 302                                           struct temp_regset *regset,
 303                                           u32 offset, u32 flags)
 304 {
 305         u32 count = regset->storage_used - (regset->registers - regset->storage);
 306         struct guc_mmio_reg entry = {
 307                 .offset = offset,
 308                 .flags = flags,
 309         };
 310         struct guc_mmio_reg *slot;
 311
 312         /*
 313          * The mmio list is built using separate lists within the driver.
 314          * It's possible that at some point we may attempt to add the same
 315          * register more than once. Do not consider this an error; silently
 316          * move on if the register is already in the list.
 317          */
 318         if (bsearch(&entry, regset->registers, count,
 319                     sizeof(entry), guc_mmio_reg_cmp))
 320                 return 0;
 321
 322         slot = __mmio_reg_add(regset, &entry);
 323         if (IS_ERR(slot))
 324                 return PTR_ERR(slot);
 325
 326         while (slot-- > regset->registers) {
 327                 GEM_BUG_ON(slot[0].offset == slot[1].offset);
 328                 if (slot[1].offset > slot[0].offset)
 329                         break;
 330
 331                 swap(slot[1], slot[0]);
 332         }
 333
 334         return 0;
 335 }
 336
 337 #define GUC_MMIO_REG_ADD(gt, regset, reg, masked) \
 338         guc_mmio_reg_add(gt, \
 339                          regset, \
 340                          i915_mmio_reg_offset(reg), \
 341                          (masked) ? GUC_REGSET_MASKED : 0)
 342
 343 #define GUC_REGSET_STEERING(group, instance) ( \
 344         FIELD_PREP(GUC_REGSET_STEERING_GROUP, (group)) | \
 345         FIELD_PREP(GUC_REGSET_STEERING_INSTANCE, (instance)) | \
 346         GUC_REGSET_NEEDS_STEERING \
 347 )
 348
 349 static long __must_check guc_mcr_reg_add(struct intel_gt *gt,
 350                                          struct temp_regset *regset,
 351                                          i915_mcr_reg_t reg, u32 flags)
 352 {
 353         u8 group, inst;
 354
 355         /*
 356          * The GuC doesn't have a default steering, so we need to explicitly
 357          * steer all registers that need steering. However, we do not keep track
 358          * of all the steering ranges, only of those that have a chance of using
 359          * a non-default steering from the i915 pov. Instead of adding such
 360          * tracking, it is easier to just program the default steering for all
 361          * regs that don't need a non-default one.
 362          */
 363         intel_gt_mcr_get_nonterminated_steering(gt, reg, &group, &inst);
 364         flags |= GUC_REGSET_STEERING(group, inst);
 365
 366         return guc_mmio_reg_add(gt, regset, i915_mmio_reg_offset(reg), flags);
 367 }
 368
 369 #define GUC_MCR_REG_ADD(gt, regset, reg, masked) \
 370         guc_mcr_reg_add(gt, \
 371                          regset, \
 372                          (reg), \
 373                          (masked) ? GUC_REGSET_MASKED : 0)
 374
 375 static int guc_mmio_regset_init(struct temp_regset *regset,
 376                                 struct intel_engine_cs *engine)
 377 {
 378         struct intel_gt *gt = engine->gt;
 379         const u32 base = engine->mmio_base;
 380         struct i915_wa_list *wal = &engine->wa_list;
 381         struct i915_wa *wa;
 382         unsigned int i;
 383         int ret = 0;
 384
 385         /*
 386          * Each engine's registers point to a new start relative to
 387          * storage
 388          */
 389         regset->registers = regset->storage + regset->storage_used;
 390
 391         ret |= GUC_MMIO_REG_ADD(gt, regset, RING_MODE_GEN7(base), true);
 392         ret |= GUC_MMIO_REG_ADD(gt, regset, RING_HWS_PGA(base), false);
 393         ret |= GUC_MMIO_REG_ADD(gt, regset, RING_IMR(base), false);
 394
 395         if ((engine->flags & I915_ENGINE_FIRST_RENDER_COMPUTE) &&
 396             CCS_MASK(engine->gt))
 397                 ret |= GUC_MMIO_REG_ADD(gt, regset, GEN12_RCU_MODE, true);
 398
 399         /*
 400          * some of the WA registers are MCR registers. As it is safe to
 401          * use MCR form for non-MCR registers, for code simplicity, all
 402          * WA registers are added with MCR form.
 403          */
 404         for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
 405                 ret |= GUC_MCR_REG_ADD(gt, regset, wa->mcr_reg, wa->masked_reg);
 406
 407         /* Be extra paranoid and include all whitelist registers. */
 408         for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++)
 409                 ret |= GUC_MMIO_REG_ADD(gt, regset,
 410                                         RING_FORCE_TO_NONPRIV(base, i),
 411                                         false);
 412
 413         /* add in local MOCS registers */
 414         for (i = 0; i < LNCFCMOCS_REG_COUNT; i++)
 415                 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55))
 416                         ret |= GUC_MCR_REG_ADD(gt, regset, XEHP_LNCFCMOCS(i), false);
 417                 else
 418                         ret |= GUC_MMIO_REG_ADD(gt, regset, GEN9_LNCFCMOCS(i), false);
 419
 420         if (GRAPHICS_VER(engine->i915) >= 12) {
 421                 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL0)), false);
 422                 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL1)), false);
 423                 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL2)), false);
 424                 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL3)), false);
 425                 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL4)), false);
 426                 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL5)), false);
 427                 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL6)), false);
 428         }
 429
 430         return ret ? -1 : 0;
 431 }
 432
 433 static long guc_mmio_reg_state_create(struct intel_guc *guc)
 434 {
 435         struct intel_gt *gt = guc_to_gt(guc);
 436         struct intel_engine_cs *engine;
 437         enum intel_engine_id id;
 438         struct temp_regset temp_set = {};
 439         long total = 0;
 440         long ret;
 441
 442         for_each_engine(engine, gt, id) {
 443                 u32 used = temp_set.storage_used;
 444
 445                 ret = guc_mmio_regset_init(&temp_set, engine);
 446                 if (ret < 0)
 447                         goto fail_regset_init;
 448
 449                 guc->ads_regset_count[id] = temp_set.storage_used - used;
 450                 total += guc->ads_regset_count[id];
 451         }
 452
 453         guc->ads_regset = temp_set.storage;
 454
 455         guc_dbg(guc, "Used %zu KB for temporary ADS regset\n",
 456                 (temp_set.storage_max * sizeof(struct guc_mmio_reg)) >> 10);
 457
 458         return total * sizeof(struct guc_mmio_reg);
 459
 460 fail_regset_init:
 461         kfree(temp_set.storage);
 462         return ret;
 463 }
 464
 465 static void guc_mmio_reg_state_init(struct intel_guc *guc)
 466 {
 467         struct intel_gt *gt = guc_to_gt(guc);
 468         struct intel_engine_cs *engine;
 469         enum intel_engine_id id;
 470         u32 addr_ggtt, offset;
 471
 472         offset = guc_ads_regset_offset(guc);
 473         addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset;
 474
 475         iosys_map_memcpy_to(&guc->ads_map, offset, guc->ads_regset,
 476                             guc->ads_regset_size);
 477
 478         for_each_engine(engine, gt, id) {
 479                 u32 count = guc->ads_regset_count[id];
 480                 u8 guc_class;
 481
 482                 /* Class index is checked in class converter */
 483                 GEM_BUG_ON(engine->instance >= GUC_MAX_INSTANCES_PER_CLASS);
 484
 485                 guc_class = engine_class_to_guc_class(engine->class);
 486
 487                 if (!count) {
 488                         ads_blob_write(guc,
 489                                        ads.reg_state_list[guc_class][engine->instance].address,
 490                                        0);
 491                         ads_blob_write(guc,
 492                                        ads.reg_state_list[guc_class][engine->instance].count,
 493                                        0);
 494                         continue;
 495                 }
 496
 497                 ads_blob_write(guc,
 498                                ads.reg_state_list[guc_class][engine->instance].address,
 499                                addr_ggtt);
 500                 ads_blob_write(guc,
 501                                ads.reg_state_list[guc_class][engine->instance].count,
 502                                count);
 503
 504                 addr_ggtt += count * sizeof(struct guc_mmio_reg);
 505         }
 506 }
 507
 508 static void fill_engine_enable_masks(struct intel_gt *gt,
 509                                      struct iosys_map *info_map)
 510 {
 511         info_map_write(info_map, engine_enabled_masks[GUC_RENDER_CLASS], RCS_MASK(gt));
 512         info_map_write(info_map, engine_enabled_masks[GUC_COMPUTE_CLASS], CCS_MASK(gt));
 513         info_map_write(info_map, engine_enabled_masks[GUC_BLITTER_CLASS], BCS_MASK(gt));
 514         info_map_write(info_map, engine_enabled_masks[GUC_VIDEO_CLASS], VDBOX_MASK(gt));
 515         info_map_write(info_map, engine_enabled_masks[GUC_VIDEOENHANCE_CLASS], VEBOX_MASK(gt));
 516
 517         /* The GSC engine is an instance (6) of OTHER_CLASS */
 518         if (gt->engine[GSC0])
 519                 info_map_write(info_map, engine_enabled_masks[GUC_GSC_OTHER_CLASS],
 520                                BIT(gt->engine[GSC0]->instance));
 521 }
 522
 523 #define LR_HW_CONTEXT_SIZE (80 * sizeof(u32))
 524 #define XEHP_LR_HW_CONTEXT_SIZE (96 * sizeof(u32))
 525 #define LR_HW_CONTEXT_SZ(i915) (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55) ? \
 526                                     XEHP_LR_HW_CONTEXT_SIZE : \
 527                                     LR_HW_CONTEXT_SIZE)
 528 #define LRC_SKIP_SIZE(i915) (LRC_PPHWSP_SZ * PAGE_SIZE + LR_HW_CONTEXT_SZ(i915))
 529 static int guc_prep_golden_context(struct intel_guc *guc)
 530 {
 531         struct intel_gt *gt = guc_to_gt(guc);
 532         u32 addr_ggtt, offset;
 533         u32 total_size = 0, alloc_size, real_size;
 534         u8 engine_class, guc_class;
 535         struct guc_gt_system_info local_info;
 536         struct iosys_map info_map;
 537
 538         /*
 539          * Reserve the memory for the golden contexts and point GuC at it but
 540          * leave it empty for now. The context data will be filled in later
 541          * once there is something available to put there.
 542          *
 543          * Note that the HWSP and ring context are not included.
 544          *
 545          * Note also that the storage must be pinned in the GGTT, so that the
 546          * address won't change after GuC has been told where to find it. The
 547          * GuC will also validate that the LRC base + size fall within the
 548          * allowed GGTT range.
 549          */
 550         if (!iosys_map_is_null(&guc->ads_map)) {
 551                 offset = guc_ads_golden_ctxt_offset(guc);
 552                 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset;
 553                 info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map,
 554                                                  offsetof(struct __guc_ads_blob, system_info));
 555         } else {
 556                 memset(&local_info, 0, sizeof(local_info));
 557                 iosys_map_set_vaddr(&info_map, &local_info);
 558                 fill_engine_enable_masks(gt, &info_map);
 559         }
 560
 561         for (engine_class = 0; engine_class <= MAX_ENGINE_CLASS; ++engine_class) {
 562                 guc_class = engine_class_to_guc_class(engine_class);
 563
 564                 if (!info_map_read(&info_map, engine_enabled_masks[guc_class]))
 565                         continue;
 566
 567                 real_size = intel_engine_context_size(gt, engine_class);
 568                 alloc_size = PAGE_ALIGN(real_size);
 569                 total_size += alloc_size;
 570
 571                 if (iosys_map_is_null(&guc->ads_map))
 572                         continue;
 573
 574                 /*
 575                  * This interface is slightly confusing. We need to pass the
 576                  * base address of the full golden context and the size of just
 577                  * the engine state, which is the section of the context image
 578                  * that starts after the execlists context. This is required to
 579                  * allow the GuC to restore just the engine state when a
 580                  * watchdog reset occurs.
 581                  * We calculate the engine state size by removing the size of
 582                  * what comes before it in the context image (which is identical
 583                  * on all engines).
 584                  */
 585                 ads_blob_write(guc, ads.eng_state_size[guc_class],
 586                                real_size - LRC_SKIP_SIZE(gt->i915));
 587                 ads_blob_write(guc, ads.golden_context_lrca[guc_class],
 588                                addr_ggtt);
 589
 590                 addr_ggtt += alloc_size;
 591         }
 592
 593         /* Make sure current size matches what we calculated previously */
 594         if (guc->ads_golden_ctxt_size)
 595                 GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size);
 596
 597         return total_size;
 598 }
 599
 600 static struct intel_engine_cs *find_engine_state(struct intel_gt *gt, u8 engine_class)
 601 {
 602         struct intel_engine_cs *engine;
 603         enum intel_engine_id id;
 604
 605         for_each_engine(engine, gt, id) {
 606                 if (engine->class != engine_class)
 607                         continue;
 608
 609                 if (!engine->default_state)
 610                         continue;
 611
 612                 return engine;
 613         }
 614
 615         return NULL;
 616 }
 617
 618 static void guc_init_golden_context(struct intel_guc *guc)
 619 {
 620         struct intel_engine_cs *engine;
 621         struct intel_gt *gt = guc_to_gt(guc);
 622         unsigned long offset;
 623         u32 addr_ggtt, total_size = 0, alloc_size, real_size;
 624         u8 engine_class, guc_class;
 625
 626         if (!intel_uc_uses_guc_submission(&gt->uc))
 627                 return;
 628
 629         GEM_BUG_ON(iosys_map_is_null(&guc->ads_map));
 630
 631         /*
 632          * Go back and fill in the golden context data now that it is
 633          * available.
 634          */
 635         offset = guc_ads_golden_ctxt_offset(guc);
 636         addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset;
 637
 638         for (engine_class = 0; engine_class <= MAX_ENGINE_CLASS; ++engine_class) {
 639                 guc_class = engine_class_to_guc_class(engine_class);
 640                 if (!ads_blob_read(guc, system_info.engine_enabled_masks[guc_class]))
 641                         continue;
 642
 643                 real_size = intel_engine_context_size(gt, engine_class);
 644                 alloc_size = PAGE_ALIGN(real_size);
 645                 total_size += alloc_size;
 646
 647                 engine = find_engine_state(gt, engine_class);
 648                 if (!engine) {
 649                         guc_err(guc, "No engine state recorded for class %d!\n",
 650                                 engine_class);
 651                         ads_blob_write(guc, ads.eng_state_size[guc_class], 0);
 652                         ads_blob_write(guc, ads.golden_context_lrca[guc_class], 0);
 653                         continue;
 654                 }
 655
 656                 GEM_BUG_ON(ads_blob_read(guc, ads.eng_state_size[guc_class]) !=
 657                            real_size - LRC_SKIP_SIZE(gt->i915));
 658                 GEM_BUG_ON(ads_blob_read(guc, ads.golden_context_lrca[guc_class]) != addr_ggtt);
 659
 660                 addr_ggtt += alloc_size;
 661
 662                 shmem_read_to_iosys_map(engine->default_state, 0, &guc->ads_map,
 663                                         offset, real_size);
 664                 offset += alloc_size;
 665         }
 666
 667         GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size);
 668 }
 669
 670 static u32 guc_get_capture_engine_mask(struct iosys_map *info_map, u32 capture_class)
 671 {
 672         u32 mask;
 673
 674         switch (capture_class) {
 675         case GUC_CAPTURE_LIST_CLASS_RENDER_COMPUTE:
 676                 mask = info_map_read(info_map, engine_enabled_masks[GUC_RENDER_CLASS]);
 677                 mask |= info_map_read(info_map, engine_enabled_masks[GUC_COMPUTE_CLASS]);
 678                 break;
 679
 680         case GUC_CAPTURE_LIST_CLASS_VIDEO:
 681                 mask = info_map_read(info_map, engine_enabled_masks[GUC_VIDEO_CLASS]);
 682                 break;
 683
 684         case GUC_CAPTURE_LIST_CLASS_VIDEOENHANCE:
 685                 mask = info_map_read(info_map, engine_enabled_masks[GUC_VIDEOENHANCE_CLASS]);
 686                 break;
 687
 688         case GUC_CAPTURE_LIST_CLASS_BLITTER:
 689                 mask = info_map_read(info_map, engine_enabled_masks[GUC_BLITTER_CLASS]);
 690                 break;
 691
 692         case GUC_CAPTURE_LIST_CLASS_GSC_OTHER:
 693                 mask = info_map_read(info_map, engine_enabled_masks[GUC_GSC_OTHER_CLASS]);
 694                 break;
 695
 696         default:
 697                 mask = 0;
 698         }
 699
 700         return mask;
 701 }
 702
 703 static int
 704 guc_capture_prep_lists(struct intel_guc *guc)
 705 {
 706         struct intel_gt *gt = guc_to_gt(guc);
 707         u32 ads_ggtt, capture_offset, null_ggtt, total_size = 0;
 708         struct guc_gt_system_info local_info;
 709         struct iosys_map info_map;
 710         bool ads_is_mapped;
 711         size_t size = 0;
 712         void *ptr;
 713         int i, j;
 714
 715         ads_is_mapped = !iosys_map_is_null(&guc->ads_map);
 716         if (ads_is_mapped) {
 717                 capture_offset = guc_ads_capture_offset(guc);
 718                 ads_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma);
 719                 info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map,
 720                                                  offsetof(struct __guc_ads_blob, system_info));
 721         } else {
 722                 memset(&local_info, 0, sizeof(local_info));
 723                 iosys_map_set_vaddr(&info_map, &local_info);
 724                 fill_engine_enable_masks(gt, &info_map);
 725         }
 726
 727         /* first, set aside the first page for a capture_list with zero descriptors */
 728         total_size = PAGE_SIZE;
 729         if (ads_is_mapped) {
 730                 if (!intel_guc_capture_getnullheader(guc, &ptr, &size))
 731                         iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size);
 732                 null_ggtt = ads_ggtt + capture_offset;
 733                 capture_offset += PAGE_SIZE;
 734         }
 735
 736         for (i = 0; i < GUC_CAPTURE_LIST_INDEX_MAX; i++) {
 737                 for (j = 0; j < GUC_MAX_ENGINE_CLASSES; j++) {
 738                         u32 engine_mask = guc_get_capture_engine_mask(&info_map, j);
 739
 740                         /* null list if we dont have said engine or list */
 741                         if (!engine_mask) {
 742                                 if (ads_is_mapped) {
 743                                         ads_blob_write(guc, ads.capture_class[i][j], null_ggtt);
 744                                         ads_blob_write(guc, ads.capture_instance[i][j], null_ggtt);
 745                                 }
 746                                 continue;
 747                         }
 748                         if (intel_guc_capture_getlistsize(guc, i,
 749                                                           GUC_CAPTURE_LIST_TYPE_ENGINE_CLASS,
 750                                                           j, &size)) {
 751                                 if (ads_is_mapped)
 752                                         ads_blob_write(guc, ads.capture_class[i][j], null_ggtt);
 753                                 goto engine_instance_list;
 754                         }
 755                         total_size += size;
 756                         if (ads_is_mapped) {
 757                                 if (total_size > guc->ads_capture_size ||
 758                                     intel_guc_capture_getlist(guc, i,
 759                                                               GUC_CAPTURE_LIST_TYPE_ENGINE_CLASS,
 760                                                               j, &ptr)) {
 761                                         ads_blob_write(guc, ads.capture_class[i][j], null_ggtt);
 762                                         continue;
 763                                 }
 764                                 ads_blob_write(guc, ads.capture_class[i][j], ads_ggtt +
 765                                                capture_offset);
 766                                 iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size);
 767                                 capture_offset += size;
 768                         }
 769 engine_instance_list:
 770                         if (intel_guc_capture_getlistsize(guc, i,
 771                                                           GUC_CAPTURE_LIST_TYPE_ENGINE_INSTANCE,
 772                                                           j, &size)) {
 773                                 if (ads_is_mapped)
 774                                         ads_blob_write(guc, ads.capture_instance[i][j], null_ggtt);
 775                                 continue;
 776                         }
 777                         total_size += size;
 778                         if (ads_is_mapped) {
 779                                 if (total_size > guc->ads_capture_size ||
 780                                     intel_guc_capture_getlist(guc, i,
 781                                                               GUC_CAPTURE_LIST_TYPE_ENGINE_INSTANCE,
 782                                                               j, &ptr)) {
 783                                         ads_blob_write(guc, ads.capture_instance[i][j], null_ggtt);
 784                                         continue;
 785                                 }
 786                                 ads_blob_write(guc, ads.capture_instance[i][j], ads_ggtt +
 787                                                capture_offset);
 788                                 iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size);
 789                                 capture_offset += size;
 790                         }
 791                 }
 792                 if (intel_guc_capture_getlistsize(guc, i, GUC_CAPTURE_LIST_TYPE_GLOBAL, 0, &size)) {
 793                         if (ads_is_mapped)
 794                                 ads_blob_write(guc, ads.capture_global[i], null_ggtt);
 795                         continue;
 796                 }
 797                 total_size += size;
 798                 if (ads_is_mapped) {
 799                         if (total_size > guc->ads_capture_size ||
 800                             intel_guc_capture_getlist(guc, i, GUC_CAPTURE_LIST_TYPE_GLOBAL, 0,
 801                                                       &ptr)) {
 802                                 ads_blob_write(guc, ads.capture_global[i], null_ggtt);
 803                                 continue;
 804                         }
 805                         ads_blob_write(guc, ads.capture_global[i], ads_ggtt + capture_offset);
 806                         iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size);
 807                         capture_offset += size;
 808                 }
 809         }
 810
 811         if (guc->ads_capture_size && guc->ads_capture_size != PAGE_ALIGN(total_size))
 812                 guc_warn(guc, "ADS capture alloc size changed from %d to %d\n",
 813                          guc->ads_capture_size, PAGE_ALIGN(total_size));
 814
 815         return PAGE_ALIGN(total_size);
 816 }
 817
 818 static void guc_waklv_enable_simple(struct intel_guc *guc, u32 *offset, u32 *remain, u32 klv_id)
 819 {
 820         u32 size;
 821         u32 klv_entry[] = {
 822                 /* 16:16 key/length */
 823                 FIELD_PREP(GUC_KLV_0_KEY, klv_id) |
 824                 FIELD_PREP(GUC_KLV_0_LEN, 0),
 825                 /* 0 dwords data */
 826         };
 827
 828         size = sizeof(klv_entry);
 829         GEM_BUG_ON(*remain < size);
 830
 831         iosys_map_memcpy_to(&guc->ads_map, *offset, klv_entry, size);
 832         *offset += size;
 833         *remain -= size;
 834 }
 835
 836 static void guc_waklv_init(struct intel_guc *guc)
 837 {
 838         struct intel_gt *gt = guc_to_gt(guc);
 839         u32 offset, addr_ggtt, remain, size;
 840
 841         if (!intel_uc_uses_guc_submission(&gt->uc))
 842                 return;
 843
 844         if (GUC_FIRMWARE_VER(guc) < MAKE_GUC_VER(70, 10, 0))
 845                 return;
 846
 847         GEM_BUG_ON(iosys_map_is_null(&guc->ads_map));
 848         offset = guc_ads_waklv_offset(guc);
 849         remain = guc_ads_waklv_size(guc);
 850
 851         /* Wa_14019159160 */
 852         if (IS_GFX_GT_IP_RANGE(gt, IP_VER(12, 70), IP_VER(12, 74))) {
 853                 guc_waklv_enable_simple(guc, &offset, &remain,
 854                                         GUC_WORKAROUND_KLV_SERIALIZED_RA_MODE);
 855                 guc_waklv_enable_simple(guc, &offset, &remain,
 856                                         GUC_WORKAROUND_KLV_AVOID_GFX_CLEAR_WHILE_ACTIVE);
 857         }
 858
 859         /* Wa_16021333562 */
 860         if ((GUC_FIRMWARE_VER(guc) >= MAKE_GUC_VER(70, 21, 1)) &&
 861             (IS_GFX_GT_IP_RANGE(gt, IP_VER(12, 70), IP_VER(12, 74)) ||
 862              IS_MEDIA_GT_IP_RANGE(gt, IP_VER(13, 0), IP_VER(13, 0)) ||
 863              IS_DG2(gt->i915)))
 864                 guc_waklv_enable_simple(guc, &offset, &remain,
 865                                         GUC_WORKAROUND_KLV_BLOCK_INTERRUPTS_WHEN_MGSR_BLOCKED);
 866
 867         size = guc_ads_waklv_size(guc) - remain;
 868         if (!size)
 869                 return;
 870
 871         offset = guc_ads_waklv_offset(guc);
 872         addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset;
 873
 874         ads_blob_write(guc, ads.wa_klv_addr_lo, addr_ggtt);
 875         ads_blob_write(guc, ads.wa_klv_addr_hi, 0);
 876         ads_blob_write(guc, ads.wa_klv_size, size);
 877 }
 878
 879 static int guc_prep_waklv(struct intel_guc *guc)
 880 {
 881         /* Fudge something chunky for now: */
 882         return PAGE_SIZE;
 883 }
 884
 885 static void __guc_ads_init(struct intel_guc *guc)
 886 {
 887         struct intel_gt *gt = guc_to_gt(guc);
 888         struct drm_i915_private *i915 = gt->i915;
 889         struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map,
 890                         offsetof(struct __guc_ads_blob, system_info));
 891         u32 base;
 892
 893         /* GuC scheduling policies */
 894         guc_policies_init(guc);
 895
 896         /* System info */
 897         fill_engine_enable_masks(gt, &info_map);
 898
 899         ads_blob_write(guc, system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_SLICE_ENABLED],
 900                        hweight8(gt->info.sseu.slice_mask));
 901         ads_blob_write(guc, system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_VDBOX_SFC_SUPPORT_MASK],
 902                        gt->info.vdbox_sfc_access);
 903
 904         if (GRAPHICS_VER(i915) >= 12 && !IS_DGFX(i915)) {
 905                 u32 distdbreg = intel_uncore_read(gt->uncore,
 906                                                   GEN12_DIST_DBS_POPULATED);
 907                 ads_blob_write(guc,
 908                                system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI],
 909                                ((distdbreg >> GEN12_DOORBELLS_PER_SQIDI_SHIFT)
 910                                 & GEN12_DOORBELLS_PER_SQIDI) + 1);
 911         }
 912
 913         /* Golden contexts for re-initialising after a watchdog reset */
 914         guc_prep_golden_context(guc);
 915
 916         guc_mapping_table_init(guc_to_gt(guc), &info_map);
 917
 918         base = intel_guc_ggtt_offset(guc, guc->ads_vma);
 919
 920         /* Lists for error capture debug */
 921         guc_capture_prep_lists(guc);
 922
 923         /* ADS */
 924         ads_blob_write(guc, ads.scheduler_policies, base +
 925                        offsetof(struct __guc_ads_blob, policies));
 926         ads_blob_write(guc, ads.gt_system_info, base +
 927                        offsetof(struct __guc_ads_blob, system_info));
 928
 929         /* MMIO save/restore list */
 930         guc_mmio_reg_state_init(guc);
 931
 932         /* Workaround KLV list */
 933         guc_waklv_init(guc);
 934
 935         /* Private Data */
 936         ads_blob_write(guc, ads.private_data, base +
 937                        guc_ads_private_data_offset(guc));
 938
 939         i915_gem_object_flush_map(guc->ads_vma->obj);
 940 }
 941
 942 /**
 943  * intel_guc_ads_create() - allocates and initializes GuC ADS.
 944  * @guc: intel_guc struct
 945  *
 946  * GuC needs memory block (Additional Data Struct), where it will store
 947  * some data. Allocate and initialize such memory block for GuC use.
 948  */
 949 int intel_guc_ads_create(struct intel_guc *guc)
 950 {
 951         void *ads_blob;
 952         u32 size;
 953         int ret;
 954
 955         GEM_BUG_ON(guc->ads_vma);
 956
 957         /*
 958          * Create reg state size dynamically on system memory to be copied to
 959          * the final ads blob on gt init/reset
 960          */
 961         ret = guc_mmio_reg_state_create(guc);
 962         if (ret < 0)
 963                 return ret;
 964         guc->ads_regset_size = ret;
 965
 966         /* Likewise the golden contexts: */
 967         ret = guc_prep_golden_context(guc);
 968         if (ret < 0)
 969                 return ret;
 970         guc->ads_golden_ctxt_size = ret;
 971
 972         /* Likewise the capture lists: */
 973         ret = guc_capture_prep_lists(guc);
 974         if (ret < 0)
 975                 return ret;
 976         guc->ads_capture_size = ret;
 977
 978         /* And don't forget the workaround KLVs: */
 979         ret = guc_prep_waklv(guc);
 980         if (ret < 0)
 981                 return ret;
 982         guc->ads_waklv_size = ret;
 983
 984         /* Now the total size can be determined: */
 985         size = guc_ads_blob_size(guc);
 986
 987         ret = intel_guc_allocate_and_map_vma(guc, size, &guc->ads_vma,
 988                                              &ads_blob);
 989         if (ret)
 990                 return ret;
 991
 992         if (i915_gem_object_is_lmem(guc->ads_vma->obj))
 993                 iosys_map_set_vaddr_iomem(&guc->ads_map, (void __iomem *)ads_blob);
 994         else
 995                 iosys_map_set_vaddr(&guc->ads_map, ads_blob);
 996
 997         __guc_ads_init(guc);
 998
 999         return 0;
1000 }
1001
1002 void intel_guc_ads_init_late(struct intel_guc *guc)
1003 {
1004         /*
1005          * The golden context setup requires the saved engine state from
1006          * __engines_record_defaults(). However, that requires engines to be
1007          * operational which means the ADS must already have been configured.
1008          * Fortunately, the golden context state is not needed until a hang
1009          * occurs, so it can be filled in during this late init phase.
1010          */
1011         guc_init_golden_context(guc);
1012 }
1013
1014 void intel_guc_ads_destroy(struct intel_guc *guc)
1015 {
1016         i915_vma_unpin_and_release(&guc->ads_vma, I915_VMA_RELEASE_MAP);
1017         iosys_map_clear(&guc->ads_map);
1018         kfree(guc->ads_regset);
1019 }
1020
1021 static void guc_ads_private_data_reset(struct intel_guc *guc)
1022 {
1023         u32 size;
1024
1025         size = guc_ads_private_data_size(guc);
1026         if (!size)
1027                 return;
1028
1029         iosys_map_memset(&guc->ads_map, guc_ads_private_data_offset(guc),
1030                          0, size);
1031 }
1032
1033 /**
1034  * intel_guc_ads_reset() - prepares GuC Additional Data Struct for reuse
1035  * @guc: intel_guc struct
1036  *
1037  * GuC stores some data in ADS, which might be stale after a reset.
1038  * Reinitialize whole ADS in case any part of it was corrupted during
1039  * previous GuC run.
1040  */
1041 void intel_guc_ads_reset(struct intel_guc *guc)
1042 {
1043         if (!guc->ads_vma)
1044                 return;
1045
1046         __guc_ads_init(guc);
1047
1048         guc_ads_private_data_reset(guc);
1049 }
1050
1051 u32 intel_guc_engine_usage_offset(struct intel_guc *guc)
1052 {
1053         return intel_guc_ggtt_offset(guc, guc->ads_vma) +
1054                 offsetof(struct __guc_ads_blob, engine_usage);
1055 }
1056
1057 struct iosys_map intel_guc_engine_usage_record_map(struct intel_engine_cs *engine)
1058 {
1059         struct intel_guc *guc = gt_to_guc(engine->gt);
1060         u8 guc_class = engine_class_to_guc_class(engine->class);
1061         size_t offset = offsetof(struct __guc_ads_blob,
1062                                  engine_usage.engines[guc_class][ilog2(engine->logical_mask)]);
1063
1064         return IOSYS_MAP_INIT_OFFSET(&guc->ads_map, offset);
1065 }