drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2014-2019 Intel Corporation
   4  */
   5
   6 #include <linux/bsearch.h>
   7
   8 #include "gem/i915_gem_lmem.h"
   9 #include "gt/intel_engine_regs.h"
  10 #include "gt/intel_gt.h"
  11 #include "gt/intel_gt_mcr.h"
  12 #include "gt/intel_gt_regs.h"
  13 #include "gt/intel_lrc.h"
  14 #include "gt/shmem_utils.h"
  15 #include "intel_guc_ads.h"
  16 #include "intel_guc_capture.h"
  17 #include "intel_guc_fwif.h"
  18 #include "intel_guc_print.h"
  19 #include "intel_uc.h"
  20 #include "i915_drv.h"
  21
  22 /*
  23  * The Additional Data Struct (ADS) has pointers for different buffers used by
  24  * the GuC. One single gem object contains the ADS struct itself (guc_ads) and
  25  * all the extra buffers indirectly linked via the ADS struct's entries.
  26  *
  27  * Layout of the ADS blob allocated for the GuC:
  28  *
  29  *      +---------------------------------------+ <== base
  30  *      | guc_ads                               |
  31  *      +---------------------------------------+
  32  *      | guc_policies                          |
  33  *      +---------------------------------------+
  34  *      | guc_gt_system_info                    |
  35  *      +---------------------------------------+
  36  *      | guc_engine_usage                      |
  37  *      +---------------------------------------+ <== static
  38  *      | guc_mmio_reg[countA] (engine 0.0)     |
  39  *      | guc_mmio_reg[countB] (engine 0.1)     |
  40  *      | guc_mmio_reg[countC] (engine 1.0)     |
  41  *      |   ...                                 |
  42  *      +---------------------------------------+ <== dynamic
  43  *      | padding                               |
  44  *      +---------------------------------------+ <== 4K aligned
  45  *      | golden contexts                       |
  46  *      +---------------------------------------+
  47  *      | padding                               |
  48  *      +---------------------------------------+ <== 4K aligned
  49  *      | capture lists                         |
  50  *      +---------------------------------------+
  51  *      | padding                               |
  52  *      +---------------------------------------+ <== 4K aligned
  53  *      | private data                          |
  54  *      +---------------------------------------+
  55  *      | padding                               |
  56  *      +---------------------------------------+ <== 4K aligned
  57  */
  58 struct __guc_ads_blob {
  59         struct guc_ads ads;
  60         struct guc_policies policies;
  61         struct guc_gt_system_info system_info;
  62         struct guc_engine_usage engine_usage;
  63         /* From here on, location is dynamic! Refer to above diagram. */
  64         struct guc_mmio_reg regset[];
  65 } __packed;
  66
  67 #define ads_blob_read(guc_, field_)                                     \
  68         iosys_map_rd_field(&(guc_)->ads_map, 0, struct __guc_ads_blob, field_)
  69
  70 #define ads_blob_write(guc_, field_, val_)                              \
  71         iosys_map_wr_field(&(guc_)->ads_map, 0, struct __guc_ads_blob,  \
  72                            field_, val_)
  73
  74 #define info_map_write(map_, field_, val_) \
  75         iosys_map_wr_field(map_, 0, struct guc_gt_system_info, field_, val_)
  76
  77 #define info_map_read(map_, field_) \
  78         iosys_map_rd_field(map_, 0, struct guc_gt_system_info, field_)
  79
  80 static u32 guc_ads_regset_size(struct intel_guc *guc)
  81 {
  82         GEM_BUG_ON(!guc->ads_regset_size);
  83         return guc->ads_regset_size;
  84 }
  85
  86 static u32 guc_ads_golden_ctxt_size(struct intel_guc *guc)
  87 {
  88         return PAGE_ALIGN(guc->ads_golden_ctxt_size);
  89 }
  90
  91 static u32 guc_ads_capture_size(struct intel_guc *guc)
  92 {
  93         return PAGE_ALIGN(guc->ads_capture_size);
  94 }
  95
  96 static u32 guc_ads_private_data_size(struct intel_guc *guc)
  97 {
  98         return PAGE_ALIGN(guc->fw.private_data_size);
  99 }
 100
 101 static u32 guc_ads_regset_offset(struct intel_guc *guc)
 102 {
 103         return offsetof(struct __guc_ads_blob, regset);
 104 }
 105
 106 static u32 guc_ads_golden_ctxt_offset(struct intel_guc *guc)
 107 {
 108         u32 offset;
 109
 110         offset = guc_ads_regset_offset(guc) +
 111                  guc_ads_regset_size(guc);
 112
 113         return PAGE_ALIGN(offset);
 114 }
 115
 116 static u32 guc_ads_capture_offset(struct intel_guc *guc)
 117 {
 118         u32 offset;
 119
 120         offset = guc_ads_golden_ctxt_offset(guc) +
 121                  guc_ads_golden_ctxt_size(guc);
 122
 123         return PAGE_ALIGN(offset);
 124 }
 125
 126 static u32 guc_ads_private_data_offset(struct intel_guc *guc)
 127 {
 128         u32 offset;
 129
 130         offset = guc_ads_capture_offset(guc) +
 131                  guc_ads_capture_size(guc);
 132
 133         return PAGE_ALIGN(offset);
 134 }
 135
 136 static u32 guc_ads_blob_size(struct intel_guc *guc)
 137 {
 138         return guc_ads_private_data_offset(guc) +
 139                guc_ads_private_data_size(guc);
 140 }
 141
 142 static void guc_policies_init(struct intel_guc *guc)
 143 {
 144         struct intel_gt *gt = guc_to_gt(guc);
 145         struct drm_i915_private *i915 = gt->i915;
 146         u32 global_flags = 0;
 147
 148         ads_blob_write(guc, policies.dpc_promote_time,
 149                        GLOBAL_POLICY_DEFAULT_DPC_PROMOTE_TIME_US);
 150         ads_blob_write(guc, policies.max_num_work_items,
 151                        GLOBAL_POLICY_MAX_NUM_WI);
 152
 153         if (i915->params.reset < 2)
 154                 global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET;
 155
 156         ads_blob_write(guc, policies.global_flags, global_flags);
 157         ads_blob_write(guc, policies.is_valid, 1);
 158 }
 159
 160 void intel_guc_ads_print_policy_info(struct intel_guc *guc,
 161                                      struct drm_printer *dp)
 162 {
 163         if (unlikely(iosys_map_is_null(&guc->ads_map)))
 164                 return;
 165
 166         drm_printf(dp, "Global scheduling policies:\n");
 167         drm_printf(dp, "  DPC promote time   = %u\n",
 168                    ads_blob_read(guc, policies.dpc_promote_time));
 169         drm_printf(dp, "  Max num work items = %u\n",
 170                    ads_blob_read(guc, policies.max_num_work_items));
 171         drm_printf(dp, "  Flags              = %u\n",
 172                    ads_blob_read(guc, policies.global_flags));
 173 }
 174
 175 static int guc_action_policies_update(struct intel_guc *guc, u32 policy_offset)
 176 {
 177         u32 action[] = {
 178                 INTEL_GUC_ACTION_GLOBAL_SCHED_POLICY_CHANGE,
 179                 policy_offset
 180         };
 181
 182         return intel_guc_send_busy_loop(guc, action, ARRAY_SIZE(action), 0, true);
 183 }
 184
 185 int intel_guc_global_policies_update(struct intel_guc *guc)
 186 {
 187         struct intel_gt *gt = guc_to_gt(guc);
 188         u32 scheduler_policies;
 189         intel_wakeref_t wakeref;
 190         int ret;
 191
 192         if (iosys_map_is_null(&guc->ads_map))
 193                 return -EOPNOTSUPP;
 194
 195         scheduler_policies = ads_blob_read(guc, ads.scheduler_policies);
 196         GEM_BUG_ON(!scheduler_policies);
 197
 198         guc_policies_init(guc);
 199
 200         if (!intel_guc_is_ready(guc))
 201                 return 0;
 202
 203         with_intel_runtime_pm(&gt->i915->runtime_pm, wakeref)
 204                 ret = guc_action_policies_update(guc, scheduler_policies);
 205
 206         return ret;
 207 }
 208
 209 static void guc_mapping_table_init(struct intel_gt *gt,
 210                                    struct iosys_map *info_map)
 211 {
 212         unsigned int i, j;
 213         struct intel_engine_cs *engine;
 214         enum intel_engine_id id;
 215
 216         /* Table must be set to invalid values for entries not used */
 217         for (i = 0; i < GUC_MAX_ENGINE_CLASSES; ++i)
 218                 for (j = 0; j < GUC_MAX_INSTANCES_PER_CLASS; ++j)
 219                         info_map_write(info_map, mapping_table[i][j],
 220                                        GUC_MAX_INSTANCES_PER_CLASS);
 221
 222         for_each_engine(engine, gt, id) {
 223                 u8 guc_class = engine_class_to_guc_class(engine->class);
 224
 225                 info_map_write(info_map, mapping_table[guc_class][ilog2(engine->logical_mask)],
 226                                engine->instance);
 227         }
 228 }
 229
 230 /*
 231  * The save/restore register list must be pre-calculated to a temporary
 232  * buffer before it can be copied inside the ADS.
 233  */
 234 struct temp_regset {
 235         /*
 236          * ptr to the section of the storage for the engine currently being
 237          * worked on
 238          */
 239         struct guc_mmio_reg *registers;
 240         /* ptr to the base of the allocated storage for all engines */
 241         struct guc_mmio_reg *storage;
 242         u32 storage_used;
 243         u32 storage_max;
 244 };
 245
 246 static int guc_mmio_reg_cmp(const void *a, const void *b)
 247 {
 248         const struct guc_mmio_reg *ra = a;
 249         const struct guc_mmio_reg *rb = b;
 250
 251         return (int)ra->offset - (int)rb->offset;
 252 }
 253
 254 static struct guc_mmio_reg * __must_check
 255 __mmio_reg_add(struct temp_regset *regset, struct guc_mmio_reg *reg)
 256 {
 257         u32 pos = regset->storage_used;
 258         struct guc_mmio_reg *slot;
 259
 260         if (pos >= regset->storage_max) {
 261                 size_t size = ALIGN((pos + 1) * sizeof(*slot), PAGE_SIZE);
 262                 struct guc_mmio_reg *r = krealloc(regset->storage,
 263                                                   size, GFP_KERNEL);
 264                 if (!r) {
 265                         WARN_ONCE(1, "Incomplete regset list: can't add register (%d)\n",
 266                                   -ENOMEM);
 267                         return ERR_PTR(-ENOMEM);
 268                 }
 269
 270                 regset->registers = r + (regset->registers - regset->storage);
 271                 regset->storage = r;
 272                 regset->storage_max = size / sizeof(*slot);
 273         }
 274
 275         slot = &regset->storage[pos];
 276         regset->storage_used++;
 277         *slot = *reg;
 278
 279         return slot;
 280 }
 281
 282 static long __must_check guc_mmio_reg_add(struct intel_gt *gt,
 283                                           struct temp_regset *regset,
 284                                           u32 offset, u32 flags)
 285 {
 286         u32 count = regset->storage_used - (regset->registers - regset->storage);
 287         struct guc_mmio_reg entry = {
 288                 .offset = offset,
 289                 .flags = flags,
 290         };
 291         struct guc_mmio_reg *slot;
 292
 293         /*
 294          * The mmio list is built using separate lists within the driver.
 295          * It's possible that at some point we may attempt to add the same
 296          * register more than once. Do not consider this an error; silently
 297          * move on if the register is already in the list.
 298          */
 299         if (bsearch(&entry, regset->registers, count,
 300                     sizeof(entry), guc_mmio_reg_cmp))
 301                 return 0;
 302
 303         slot = __mmio_reg_add(regset, &entry);
 304         if (IS_ERR(slot))
 305                 return PTR_ERR(slot);
 306
 307         while (slot-- > regset->registers) {
 308                 GEM_BUG_ON(slot[0].offset == slot[1].offset);
 309                 if (slot[1].offset > slot[0].offset)
 310                         break;
 311
 312                 swap(slot[1], slot[0]);
 313         }
 314
 315         return 0;
 316 }
 317
 318 #define GUC_MMIO_REG_ADD(gt, regset, reg, masked) \
 319         guc_mmio_reg_add(gt, \
 320                          regset, \
 321                          i915_mmio_reg_offset(reg), \
 322                          (masked) ? GUC_REGSET_MASKED : 0)
 323
 324 #define GUC_REGSET_STEERING(group, instance) ( \
 325         FIELD_PREP(GUC_REGSET_STEERING_GROUP, (group)) | \
 326         FIELD_PREP(GUC_REGSET_STEERING_INSTANCE, (instance)) | \
 327         GUC_REGSET_NEEDS_STEERING \
 328 )
 329
 330 static long __must_check guc_mcr_reg_add(struct intel_gt *gt,
 331                                          struct temp_regset *regset,
 332                                          i915_mcr_reg_t reg, u32 flags)
 333 {
 334         u8 group, inst;
 335
 336         /*
 337          * The GuC doesn't have a default steering, so we need to explicitly
 338          * steer all registers that need steering. However, we do not keep track
 339          * of all the steering ranges, only of those that have a chance of using
 340          * a non-default steering from the i915 pov. Instead of adding such
 341          * tracking, it is easier to just program the default steering for all
 342          * regs that don't need a non-default one.
 343          */
 344         intel_gt_mcr_get_nonterminated_steering(gt, reg, &group, &inst);
 345         flags |= GUC_REGSET_STEERING(group, inst);
 346
 347         return guc_mmio_reg_add(gt, regset, i915_mmio_reg_offset(reg), flags);
 348 }
 349
 350 #define GUC_MCR_REG_ADD(gt, regset, reg, masked) \
 351         guc_mcr_reg_add(gt, \
 352                          regset, \
 353                          (reg), \
 354                          (masked) ? GUC_REGSET_MASKED : 0)
 355
 356 static int guc_mmio_regset_init(struct temp_regset *regset,
 357                                 struct intel_engine_cs *engine)
 358 {
 359         struct intel_gt *gt = engine->gt;
 360         const u32 base = engine->mmio_base;
 361         struct i915_wa_list *wal = &engine->wa_list;
 362         struct i915_wa *wa;
 363         unsigned int i;
 364         int ret = 0;
 365
 366         /*
 367          * Each engine's registers point to a new start relative to
 368          * storage
 369          */
 370         regset->registers = regset->storage + regset->storage_used;
 371
 372         ret |= GUC_MMIO_REG_ADD(gt, regset, RING_MODE_GEN7(base), true);
 373         ret |= GUC_MMIO_REG_ADD(gt, regset, RING_HWS_PGA(base), false);
 374         ret |= GUC_MMIO_REG_ADD(gt, regset, RING_IMR(base), false);
 375
 376         if ((engine->flags & I915_ENGINE_FIRST_RENDER_COMPUTE) &&
 377             CCS_MASK(engine->gt))
 378                 ret |= GUC_MMIO_REG_ADD(gt, regset, GEN12_RCU_MODE, true);
 379
 380         /*
 381          * some of the WA registers are MCR registers. As it is safe to
 382          * use MCR form for non-MCR registers, for code simplicity, all
 383          * WA registers are added with MCR form.
 384          */
 385         for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
 386                 ret |= GUC_MCR_REG_ADD(gt, regset, wa->mcr_reg, wa->masked_reg);
 387
 388         /* Be extra paranoid and include all whitelist registers. */
 389         for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++)
 390                 ret |= GUC_MMIO_REG_ADD(gt, regset,
 391                                         RING_FORCE_TO_NONPRIV(base, i),
 392                                         false);
 393
 394         /* add in local MOCS registers */
 395         for (i = 0; i < LNCFCMOCS_REG_COUNT; i++)
 396                 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50))
 397                         ret |= GUC_MCR_REG_ADD(gt, regset, XEHP_LNCFCMOCS(i), false);
 398                 else
 399                         ret |= GUC_MMIO_REG_ADD(gt, regset, GEN9_LNCFCMOCS(i), false);
 400
 401         if (GRAPHICS_VER(engine->i915) >= 12) {
 402                 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL0)), false);
 403                 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL1)), false);
 404                 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL2)), false);
 405                 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL3)), false);
 406                 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL4)), false);
 407                 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL5)), false);
 408                 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL6)), false);
 409         }
 410
 411         return ret ? -1 : 0;
 412 }
 413
 414 static long guc_mmio_reg_state_create(struct intel_guc *guc)
 415 {
 416         struct intel_gt *gt = guc_to_gt(guc);
 417         struct intel_engine_cs *engine;
 418         enum intel_engine_id id;
 419         struct temp_regset temp_set = {};
 420         long total = 0;
 421         long ret;
 422
 423         for_each_engine(engine, gt, id) {
 424                 u32 used = temp_set.storage_used;
 425
 426                 ret = guc_mmio_regset_init(&temp_set, engine);
 427                 if (ret < 0)
 428                         goto fail_regset_init;
 429
 430                 guc->ads_regset_count[id] = temp_set.storage_used - used;
 431                 total += guc->ads_regset_count[id];
 432         }
 433
 434         guc->ads_regset = temp_set.storage;
 435
 436         guc_dbg(guc, "Used %zu KB for temporary ADS regset\n",
 437                 (temp_set.storage_max * sizeof(struct guc_mmio_reg)) >> 10);
 438
 439         return total * sizeof(struct guc_mmio_reg);
 440
 441 fail_regset_init:
 442         kfree(temp_set.storage);
 443         return ret;
 444 }
 445
 446 static void guc_mmio_reg_state_init(struct intel_guc *guc)
 447 {
 448         struct intel_gt *gt = guc_to_gt(guc);
 449         struct intel_engine_cs *engine;
 450         enum intel_engine_id id;
 451         u32 addr_ggtt, offset;
 452
 453         offset = guc_ads_regset_offset(guc);
 454         addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset;
 455
 456         iosys_map_memcpy_to(&guc->ads_map, offset, guc->ads_regset,
 457                             guc->ads_regset_size);
 458
 459         for_each_engine(engine, gt, id) {
 460                 u32 count = guc->ads_regset_count[id];
 461                 u8 guc_class;
 462
 463                 /* Class index is checked in class converter */
 464                 GEM_BUG_ON(engine->instance >= GUC_MAX_INSTANCES_PER_CLASS);
 465
 466                 guc_class = engine_class_to_guc_class(engine->class);
 467
 468                 if (!count) {
 469                         ads_blob_write(guc,
 470                                        ads.reg_state_list[guc_class][engine->instance].address,
 471                                        0);
 472                         ads_blob_write(guc,
 473                                        ads.reg_state_list[guc_class][engine->instance].count,
 474                                        0);
 475                         continue;
 476                 }
 477
 478                 ads_blob_write(guc,
 479                                ads.reg_state_list[guc_class][engine->instance].address,
 480                                addr_ggtt);
 481                 ads_blob_write(guc,
 482                                ads.reg_state_list[guc_class][engine->instance].count,
 483                                count);
 484
 485                 addr_ggtt += count * sizeof(struct guc_mmio_reg);
 486         }
 487 }
 488
 489 static void fill_engine_enable_masks(struct intel_gt *gt,
 490                                      struct iosys_map *info_map)
 491 {
 492         info_map_write(info_map, engine_enabled_masks[GUC_RENDER_CLASS], RCS_MASK(gt));
 493         info_map_write(info_map, engine_enabled_masks[GUC_COMPUTE_CLASS], CCS_MASK(gt));
 494         info_map_write(info_map, engine_enabled_masks[GUC_BLITTER_CLASS], BCS_MASK(gt));
 495         info_map_write(info_map, engine_enabled_masks[GUC_VIDEO_CLASS], VDBOX_MASK(gt));
 496         info_map_write(info_map, engine_enabled_masks[GUC_VIDEOENHANCE_CLASS], VEBOX_MASK(gt));
 497
 498         /* The GSC engine is an instance (6) of OTHER_CLASS */
 499         if (gt->engine[GSC0])
 500                 info_map_write(info_map, engine_enabled_masks[GUC_GSC_OTHER_CLASS],
 501                                BIT(gt->engine[GSC0]->instance));
 502 }
 503
 504 #define LR_HW_CONTEXT_SIZE (80 * sizeof(u32))
 505 #define XEHP_LR_HW_CONTEXT_SIZE (96 * sizeof(u32))
 506 #define LR_HW_CONTEXT_SZ(i915) (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50) ? \
 507                                     XEHP_LR_HW_CONTEXT_SIZE : \
 508                                     LR_HW_CONTEXT_SIZE)
 509 #define LRC_SKIP_SIZE(i915) (LRC_PPHWSP_SZ * PAGE_SIZE + LR_HW_CONTEXT_SZ(i915))
 510 static int guc_prep_golden_context(struct intel_guc *guc)
 511 {
 512         struct intel_gt *gt = guc_to_gt(guc);
 513         u32 addr_ggtt, offset;
 514         u32 total_size = 0, alloc_size, real_size;
 515         u8 engine_class, guc_class;
 516         struct guc_gt_system_info local_info;
 517         struct iosys_map info_map;
 518
 519         /*
 520          * Reserve the memory for the golden contexts and point GuC at it but
 521          * leave it empty for now. The context data will be filled in later
 522          * once there is something available to put there.
 523          *
 524          * Note that the HWSP and ring context are not included.
 525          *
 526          * Note also that the storage must be pinned in the GGTT, so that the
 527          * address won't change after GuC has been told where to find it. The
 528          * GuC will also validate that the LRC base + size fall within the
 529          * allowed GGTT range.
 530          */
 531         if (!iosys_map_is_null(&guc->ads_map)) {
 532                 offset = guc_ads_golden_ctxt_offset(guc);
 533                 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset;
 534                 info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map,
 535                                                  offsetof(struct __guc_ads_blob, system_info));
 536         } else {
 537                 memset(&local_info, 0, sizeof(local_info));
 538                 iosys_map_set_vaddr(&info_map, &local_info);
 539                 fill_engine_enable_masks(gt, &info_map);
 540         }
 541
 542         for (engine_class = 0; engine_class <= MAX_ENGINE_CLASS; ++engine_class) {
 543                 guc_class = engine_class_to_guc_class(engine_class);
 544
 545                 if (!info_map_read(&info_map, engine_enabled_masks[guc_class]))
 546                         continue;
 547
 548                 real_size = intel_engine_context_size(gt, engine_class);
 549                 alloc_size = PAGE_ALIGN(real_size);
 550                 total_size += alloc_size;
 551
 552                 if (iosys_map_is_null(&guc->ads_map))
 553                         continue;
 554
 555                 /*
 556                  * This interface is slightly confusing. We need to pass the
 557                  * base address of the full golden context and the size of just
 558                  * the engine state, which is the section of the context image
 559                  * that starts after the execlists context. This is required to
 560                  * allow the GuC to restore just the engine state when a
 561                  * watchdog reset occurs.
 562                  * We calculate the engine state size by removing the size of
 563                  * what comes before it in the context image (which is identical
 564                  * on all engines).
 565                  */
 566                 ads_blob_write(guc, ads.eng_state_size[guc_class],
 567                                real_size - LRC_SKIP_SIZE(gt->i915));
 568                 ads_blob_write(guc, ads.golden_context_lrca[guc_class],
 569                                addr_ggtt);
 570
 571                 addr_ggtt += alloc_size;
 572         }
 573
 574         /* Make sure current size matches what we calculated previously */
 575         if (guc->ads_golden_ctxt_size)
 576                 GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size);
 577
 578         return total_size;
 579 }
 580
 581 static struct intel_engine_cs *find_engine_state(struct intel_gt *gt, u8 engine_class)
 582 {
 583         struct intel_engine_cs *engine;
 584         enum intel_engine_id id;
 585
 586         for_each_engine(engine, gt, id) {
 587                 if (engine->class != engine_class)
 588                         continue;
 589
 590                 if (!engine->default_state)
 591                         continue;
 592
 593                 return engine;
 594         }
 595
 596         return NULL;
 597 }
 598
 599 static void guc_init_golden_context(struct intel_guc *guc)
 600 {
 601         struct intel_engine_cs *engine;
 602         struct intel_gt *gt = guc_to_gt(guc);
 603         unsigned long offset;
 604         u32 addr_ggtt, total_size = 0, alloc_size, real_size;
 605         u8 engine_class, guc_class;
 606
 607         if (!intel_uc_uses_guc_submission(&gt->uc))
 608                 return;
 609
 610         GEM_BUG_ON(iosys_map_is_null(&guc->ads_map));
 611
 612         /*
 613          * Go back and fill in the golden context data now that it is
 614          * available.
 615          */
 616         offset = guc_ads_golden_ctxt_offset(guc);
 617         addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset;
 618
 619         for (engine_class = 0; engine_class <= MAX_ENGINE_CLASS; ++engine_class) {
 620                 guc_class = engine_class_to_guc_class(engine_class);
 621                 if (!ads_blob_read(guc, system_info.engine_enabled_masks[guc_class]))
 622                         continue;
 623
 624                 real_size = intel_engine_context_size(gt, engine_class);
 625                 alloc_size = PAGE_ALIGN(real_size);
 626                 total_size += alloc_size;
 627
 628                 engine = find_engine_state(gt, engine_class);
 629                 if (!engine) {
 630                         guc_err(guc, "No engine state recorded for class %d!\n",
 631                                 engine_class);
 632                         ads_blob_write(guc, ads.eng_state_size[guc_class], 0);
 633                         ads_blob_write(guc, ads.golden_context_lrca[guc_class], 0);
 634                         continue;
 635                 }
 636
 637                 GEM_BUG_ON(ads_blob_read(guc, ads.eng_state_size[guc_class]) !=
 638                            real_size - LRC_SKIP_SIZE(gt->i915));
 639                 GEM_BUG_ON(ads_blob_read(guc, ads.golden_context_lrca[guc_class]) != addr_ggtt);
 640
 641                 addr_ggtt += alloc_size;
 642
 643                 shmem_read_to_iosys_map(engine->default_state, 0, &guc->ads_map,
 644                                         offset, real_size);
 645                 offset += alloc_size;
 646         }
 647
 648         GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size);
 649 }
 650
 651 static u32 guc_get_capture_engine_mask(struct iosys_map *info_map, u32 capture_class)
 652 {
 653         u32 mask;
 654
 655         switch (capture_class) {
 656         case GUC_CAPTURE_LIST_CLASS_RENDER_COMPUTE:
 657                 mask = info_map_read(info_map, engine_enabled_masks[GUC_RENDER_CLASS]);
 658                 mask |= info_map_read(info_map, engine_enabled_masks[GUC_COMPUTE_CLASS]);
 659                 break;
 660
 661         case GUC_CAPTURE_LIST_CLASS_VIDEO:
 662                 mask = info_map_read(info_map, engine_enabled_masks[GUC_VIDEO_CLASS]);
 663                 break;
 664
 665         case GUC_CAPTURE_LIST_CLASS_VIDEOENHANCE:
 666                 mask = info_map_read(info_map, engine_enabled_masks[GUC_VIDEOENHANCE_CLASS]);
 667                 break;
 668
 669         case GUC_CAPTURE_LIST_CLASS_BLITTER:
 670                 mask = info_map_read(info_map, engine_enabled_masks[GUC_BLITTER_CLASS]);
 671                 break;
 672
 673         case GUC_CAPTURE_LIST_CLASS_GSC_OTHER:
 674                 mask = info_map_read(info_map, engine_enabled_masks[GUC_GSC_OTHER_CLASS]);
 675                 break;
 676
 677         default:
 678                 mask = 0;
 679         }
 680
 681         return mask;
 682 }
 683
 684 static int
 685 guc_capture_prep_lists(struct intel_guc *guc)
 686 {
 687         struct intel_gt *gt = guc_to_gt(guc);
 688         u32 ads_ggtt, capture_offset, null_ggtt, total_size = 0;
 689         struct guc_gt_system_info local_info;
 690         struct iosys_map info_map;
 691         bool ads_is_mapped;
 692         size_t size = 0;
 693         void *ptr;
 694         int i, j;
 695
 696         ads_is_mapped = !iosys_map_is_null(&guc->ads_map);
 697         if (ads_is_mapped) {
 698                 capture_offset = guc_ads_capture_offset(guc);
 699                 ads_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma);
 700                 info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map,
 701                                                  offsetof(struct __guc_ads_blob, system_info));
 702         } else {
 703                 memset(&local_info, 0, sizeof(local_info));
 704                 iosys_map_set_vaddr(&info_map, &local_info);
 705                 fill_engine_enable_masks(gt, &info_map);
 706         }
 707
 708         /* first, set aside the first page for a capture_list with zero descriptors */
 709         total_size = PAGE_SIZE;
 710         if (ads_is_mapped) {
 711                 if (!intel_guc_capture_getnullheader(guc, &ptr, &size))
 712                         iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size);
 713                 null_ggtt = ads_ggtt + capture_offset;
 714                 capture_offset += PAGE_SIZE;
 715         }
 716
 717         for (i = 0; i < GUC_CAPTURE_LIST_INDEX_MAX; i++) {
 718                 for (j = 0; j < GUC_MAX_ENGINE_CLASSES; j++) {
 719                         u32 engine_mask = guc_get_capture_engine_mask(&info_map, j);
 720
 721                         /* null list if we dont have said engine or list */
 722                         if (!engine_mask) {
 723                                 if (ads_is_mapped) {
 724                                         ads_blob_write(guc, ads.capture_class[i][j], null_ggtt);
 725                                         ads_blob_write(guc, ads.capture_instance[i][j], null_ggtt);
 726                                 }
 727                                 continue;
 728                         }
 729                         if (intel_guc_capture_getlistsize(guc, i,
 730                                                           GUC_CAPTURE_LIST_TYPE_ENGINE_CLASS,
 731                                                           j, &size)) {
 732                                 if (ads_is_mapped)
 733                                         ads_blob_write(guc, ads.capture_class[i][j], null_ggtt);
 734                                 goto engine_instance_list;
 735                         }
 736                         total_size += size;
 737                         if (ads_is_mapped) {
 738                                 if (total_size > guc->ads_capture_size ||
 739                                     intel_guc_capture_getlist(guc, i,
 740                                                               GUC_CAPTURE_LIST_TYPE_ENGINE_CLASS,
 741                                                               j, &ptr)) {
 742                                         ads_blob_write(guc, ads.capture_class[i][j], null_ggtt);
 743                                         continue;
 744                                 }
 745                                 ads_blob_write(guc, ads.capture_class[i][j], ads_ggtt +
 746                                                capture_offset);
 747                                 iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size);
 748                                 capture_offset += size;
 749                         }
 750 engine_instance_list:
 751                         if (intel_guc_capture_getlistsize(guc, i,
 752                                                           GUC_CAPTURE_LIST_TYPE_ENGINE_INSTANCE,
 753                                                           j, &size)) {
 754                                 if (ads_is_mapped)
 755                                         ads_blob_write(guc, ads.capture_instance[i][j], null_ggtt);
 756                                 continue;
 757                         }
 758                         total_size += size;
 759                         if (ads_is_mapped) {
 760                                 if (total_size > guc->ads_capture_size ||
 761                                     intel_guc_capture_getlist(guc, i,
 762                                                               GUC_CAPTURE_LIST_TYPE_ENGINE_INSTANCE,
 763                                                               j, &ptr)) {
 764                                         ads_blob_write(guc, ads.capture_instance[i][j], null_ggtt);
 765                                         continue;
 766                                 }
 767                                 ads_blob_write(guc, ads.capture_instance[i][j], ads_ggtt +
 768                                                capture_offset);
 769                                 iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size);
 770                                 capture_offset += size;
 771                         }
 772                 }
 773                 if (intel_guc_capture_getlistsize(guc, i, GUC_CAPTURE_LIST_TYPE_GLOBAL, 0, &size)) {
 774                         if (ads_is_mapped)
 775                                 ads_blob_write(guc, ads.capture_global[i], null_ggtt);
 776                         continue;
 777                 }
 778                 total_size += size;
 779                 if (ads_is_mapped) {
 780                         if (total_size > guc->ads_capture_size ||
 781                             intel_guc_capture_getlist(guc, i, GUC_CAPTURE_LIST_TYPE_GLOBAL, 0,
 782                                                       &ptr)) {
 783                                 ads_blob_write(guc, ads.capture_global[i], null_ggtt);
 784                                 continue;
 785                         }
 786                         ads_blob_write(guc, ads.capture_global[i], ads_ggtt + capture_offset);
 787                         iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size);
 788                         capture_offset += size;
 789                 }
 790         }
 791
 792         if (guc->ads_capture_size && guc->ads_capture_size != PAGE_ALIGN(total_size))
 793                 guc_warn(guc, "ADS capture alloc size changed from %d to %d\n",
 794                          guc->ads_capture_size, PAGE_ALIGN(total_size));
 795
 796         return PAGE_ALIGN(total_size);
 797 }
 798
 799 static void __guc_ads_init(struct intel_guc *guc)
 800 {
 801         struct intel_gt *gt = guc_to_gt(guc);
 802         struct drm_i915_private *i915 = gt->i915;
 803         struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map,
 804                         offsetof(struct __guc_ads_blob, system_info));
 805         u32 base;
 806
 807         /* GuC scheduling policies */
 808         guc_policies_init(guc);
 809
 810         /* System info */
 811         fill_engine_enable_masks(gt, &info_map);
 812
 813         ads_blob_write(guc, system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_SLICE_ENABLED],
 814                        hweight8(gt->info.sseu.slice_mask));
 815         ads_blob_write(guc, system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_VDBOX_SFC_SUPPORT_MASK],
 816                        gt->info.vdbox_sfc_access);
 817
 818         if (GRAPHICS_VER(i915) >= 12 && !IS_DGFX(i915)) {
 819                 u32 distdbreg = intel_uncore_read(gt->uncore,
 820                                                   GEN12_DIST_DBS_POPULATED);
 821                 ads_blob_write(guc,
 822                                system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI],
 823                                ((distdbreg >> GEN12_DOORBELLS_PER_SQIDI_SHIFT)
 824                                 & GEN12_DOORBELLS_PER_SQIDI) + 1);
 825         }
 826
 827         /* Golden contexts for re-initialising after a watchdog reset */
 828         guc_prep_golden_context(guc);
 829
 830         guc_mapping_table_init(guc_to_gt(guc), &info_map);
 831
 832         base = intel_guc_ggtt_offset(guc, guc->ads_vma);
 833
 834         /* Lists for error capture debug */
 835         guc_capture_prep_lists(guc);
 836
 837         /* ADS */
 838         ads_blob_write(guc, ads.scheduler_policies, base +
 839                        offsetof(struct __guc_ads_blob, policies));
 840         ads_blob_write(guc, ads.gt_system_info, base +
 841                        offsetof(struct __guc_ads_blob, system_info));
 842
 843         /* MMIO save/restore list */
 844         guc_mmio_reg_state_init(guc);
 845
 846         /* Private Data */
 847         ads_blob_write(guc, ads.private_data, base +
 848                        guc_ads_private_data_offset(guc));
 849
 850         i915_gem_object_flush_map(guc->ads_vma->obj);
 851 }
 852
 853 /**
 854  * intel_guc_ads_create() - allocates and initializes GuC ADS.
 855  * @guc: intel_guc struct
 856  *
 857  * GuC needs memory block (Additional Data Struct), where it will store
 858  * some data. Allocate and initialize such memory block for GuC use.
 859  */
 860 int intel_guc_ads_create(struct intel_guc *guc)
 861 {
 862         void *ads_blob;
 863         u32 size;
 864         int ret;
 865
 866         GEM_BUG_ON(guc->ads_vma);
 867
 868         /*
 869          * Create reg state size dynamically on system memory to be copied to
 870          * the final ads blob on gt init/reset
 871          */
 872         ret = guc_mmio_reg_state_create(guc);
 873         if (ret < 0)
 874                 return ret;
 875         guc->ads_regset_size = ret;
 876
 877         /* Likewise the golden contexts: */
 878         ret = guc_prep_golden_context(guc);
 879         if (ret < 0)
 880                 return ret;
 881         guc->ads_golden_ctxt_size = ret;
 882
 883         /* Likewise the capture lists: */
 884         ret = guc_capture_prep_lists(guc);
 885         if (ret < 0)
 886                 return ret;
 887         guc->ads_capture_size = ret;
 888
 889         /* Now the total size can be determined: */
 890         size = guc_ads_blob_size(guc);
 891
 892         ret = intel_guc_allocate_and_map_vma(guc, size, &guc->ads_vma,
 893                                              &ads_blob);
 894         if (ret)
 895                 return ret;
 896
 897         if (i915_gem_object_is_lmem(guc->ads_vma->obj))
 898                 iosys_map_set_vaddr_iomem(&guc->ads_map, (void __iomem *)ads_blob);
 899         else
 900                 iosys_map_set_vaddr(&guc->ads_map, ads_blob);
 901
 902         __guc_ads_init(guc);
 903
 904         return 0;
 905 }
 906
 907 void intel_guc_ads_init_late(struct intel_guc *guc)
 908 {
 909         /*
 910          * The golden context setup requires the saved engine state from
 911          * __engines_record_defaults(). However, that requires engines to be
 912          * operational which means the ADS must already have been configured.
 913          * Fortunately, the golden context state is not needed until a hang
 914          * occurs, so it can be filled in during this late init phase.
 915          */
 916         guc_init_golden_context(guc);
 917 }
 918
 919 void intel_guc_ads_destroy(struct intel_guc *guc)
 920 {
 921         i915_vma_unpin_and_release(&guc->ads_vma, I915_VMA_RELEASE_MAP);
 922         iosys_map_clear(&guc->ads_map);
 923         kfree(guc->ads_regset);
 924 }
 925
 926 static void guc_ads_private_data_reset(struct intel_guc *guc)
 927 {
 928         u32 size;
 929
 930         size = guc_ads_private_data_size(guc);
 931         if (!size)
 932                 return;
 933
 934         iosys_map_memset(&guc->ads_map, guc_ads_private_data_offset(guc),
 935                          0, size);
 936 }
 937
 938 /**
 939  * intel_guc_ads_reset() - prepares GuC Additional Data Struct for reuse
 940  * @guc: intel_guc struct
 941  *
 942  * GuC stores some data in ADS, which might be stale after a reset.
 943  * Reinitialize whole ADS in case any part of it was corrupted during
 944  * previous GuC run.
 945  */
 946 void intel_guc_ads_reset(struct intel_guc *guc)
 947 {
 948         if (!guc->ads_vma)
 949                 return;
 950
 951         __guc_ads_init(guc);
 952
 953         guc_ads_private_data_reset(guc);
 954 }
 955
 956 u32 intel_guc_engine_usage_offset(struct intel_guc *guc)
 957 {
 958         return intel_guc_ggtt_offset(guc, guc->ads_vma) +
 959                 offsetof(struct __guc_ads_blob, engine_usage);
 960 }
 961
 962 struct iosys_map intel_guc_engine_usage_record_map(struct intel_engine_cs *engine)
 963 {
 964         struct intel_guc *guc = &engine->gt->uc.guc;
 965         u8 guc_class = engine_class_to_guc_class(engine->class);
 966         size_t offset = offsetof(struct __guc_ads_blob,
 967                                  engine_usage.engines[guc_class][ilog2(engine->logical_mask)]);
 968
 969         return IOSYS_MAP_INIT_OFFSET(&guc->ads_map, offset);
 970 }