drivers/gpu/drm/i915/gt/uc/intel_uc.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2016-2019 Intel Corporation
   4  */
   5
   6 #include <linux/string_helpers.h>
   7
   8 #include "gt/intel_gt.h"
   9 #include "gt/intel_gt_print.h"
  10 #include "gt/intel_reset.h"
  11 #include "intel_gsc_fw.h"
  12 #include "intel_gsc_uc.h"
  13 #include "intel_guc.h"
  14 #include "intel_guc_ads.h"
  15 #include "intel_guc_print.h"
  16 #include "intel_guc_submission.h"
  17 #include "gt/intel_rps.h"
  18 #include "intel_uc.h"
  19
  20 #include "i915_drv.h"
  21 #include "i915_hwmon.h"
  22
  23 static const struct intel_uc_ops uc_ops_off;
  24 static const struct intel_uc_ops uc_ops_on;
  25
  26 static void uc_expand_default_options(struct intel_uc *uc)
  27 {
  28         struct drm_i915_private *i915 = uc_to_gt(uc)->i915;
  29
  30         if (i915->params.enable_guc != -1)
  31                 return;
  32
  33         /* Don't enable GuC/HuC on pre-Gen12 */
  34         if (GRAPHICS_VER(i915) < 12) {
  35                 i915->params.enable_guc = 0;
  36                 return;
  37         }
  38
  39         /* Don't enable GuC/HuC on older Gen12 platforms */
  40         if (IS_TIGERLAKE(i915) || IS_ROCKETLAKE(i915)) {
  41                 i915->params.enable_guc = 0;
  42                 return;
  43         }
  44
  45         /* Intermediate platforms are HuC authentication only */
  46         if (IS_ALDERLAKE_S(i915) && !IS_RAPTORLAKE_S(i915)) {
  47                 i915->params.enable_guc = ENABLE_GUC_LOAD_HUC;
  48                 return;
  49         }
  50
  51         /* Default: enable HuC authentication and GuC submission */
  52         i915->params.enable_guc = ENABLE_GUC_LOAD_HUC | ENABLE_GUC_SUBMISSION;
  53 }
  54
  55 /* Reset GuC providing us with fresh state for both GuC and HuC.
  56  */
  57 static int __intel_uc_reset_hw(struct intel_uc *uc)
  58 {
  59         struct intel_gt *gt = uc_to_gt(uc);
  60         int ret;
  61         u32 guc_status;
  62
  63         ret = i915_inject_probe_error(gt->i915, -ENXIO);
  64         if (ret)
  65                 return ret;
  66
  67         ret = intel_reset_guc(gt);
  68         if (ret) {
  69                 gt_err(gt, "Failed to reset GuC, ret = %d\n", ret);
  70                 return ret;
  71         }
  72
  73         guc_status = intel_uncore_read(gt->uncore, GUC_STATUS);
  74         gt_WARN(gt, !(guc_status & GS_MIA_IN_RESET),
  75                 "GuC status: 0x%x, MIA core expected to be in reset\n",
  76                 guc_status);
  77
  78         return ret;
  79 }
  80
  81 static void __confirm_options(struct intel_uc *uc)
  82 {
  83         struct intel_gt *gt = uc_to_gt(uc);
  84         struct drm_i915_private *i915 = gt->i915;
  85
  86         gt_dbg(gt, "enable_guc=%d (guc:%s submission:%s huc:%s slpc:%s)\n",
  87                i915->params.enable_guc,
  88                str_yes_no(intel_uc_wants_guc(uc)),
  89                str_yes_no(intel_uc_wants_guc_submission(uc)),
  90                str_yes_no(intel_uc_wants_huc(uc)),
  91                str_yes_no(intel_uc_wants_guc_slpc(uc)));
  92
  93         if (i915->params.enable_guc == 0) {
  94                 GEM_BUG_ON(intel_uc_wants_guc(uc));
  95                 GEM_BUG_ON(intel_uc_wants_guc_submission(uc));
  96                 GEM_BUG_ON(intel_uc_wants_huc(uc));
  97                 GEM_BUG_ON(intel_uc_wants_guc_slpc(uc));
  98                 return;
  99         }
 100
 101         if (!intel_uc_supports_guc(uc))
 102                 gt_info(gt, "Incompatible option enable_guc=%d - %s\n",
 103                         i915->params.enable_guc, "GuC is not supported!");
 104
 105         if (i915->params.enable_guc & ENABLE_GUC_SUBMISSION &&
 106             !intel_uc_supports_guc_submission(uc))
 107                 gt_info(gt, "Incompatible option enable_guc=%d - %s\n",
 108                         i915->params.enable_guc, "GuC submission is N/A");
 109
 110         if (i915->params.enable_guc & ~ENABLE_GUC_MASK)
 111                 gt_info(gt, "Incompatible option enable_guc=%d - %s\n",
 112                         i915->params.enable_guc, "undocumented flag");
 113 }
 114
 115 void intel_uc_init_early(struct intel_uc *uc)
 116 {
 117         uc_expand_default_options(uc);
 118
 119         intel_guc_init_early(&uc->guc);
 120         intel_huc_init_early(&uc->huc);
 121         intel_gsc_uc_init_early(&uc->gsc);
 122
 123         __confirm_options(uc);
 124
 125         if (intel_uc_wants_guc(uc))
 126                 uc->ops = &uc_ops_on;
 127         else
 128                 uc->ops = &uc_ops_off;
 129 }
 130
 131 void intel_uc_init_late(struct intel_uc *uc)
 132 {
 133         intel_guc_init_late(&uc->guc);
 134         intel_gsc_uc_load_start(&uc->gsc);
 135 }
 136
 137 void intel_uc_driver_late_release(struct intel_uc *uc)
 138 {
 139 }
 140
 141 /**
 142  * intel_uc_init_mmio - setup uC MMIO access
 143  * @uc: the intel_uc structure
 144  *
 145  * Setup minimal state necessary for MMIO accesses later in the
 146  * initialization sequence.
 147  */
 148 void intel_uc_init_mmio(struct intel_uc *uc)
 149 {
 150         intel_guc_init_send_regs(&uc->guc);
 151 }
 152
 153 static void __uc_capture_load_err_log(struct intel_uc *uc)
 154 {
 155         struct intel_guc *guc = &uc->guc;
 156
 157         if (guc->log.vma && !uc->load_err_log)
 158                 uc->load_err_log = i915_gem_object_get(guc->log.vma->obj);
 159 }
 160
 161 static void __uc_free_load_err_log(struct intel_uc *uc)
 162 {
 163         struct drm_i915_gem_object *log = fetch_and_zero(&uc->load_err_log);
 164
 165         if (log)
 166                 i915_gem_object_put(log);
 167 }
 168
 169 void intel_uc_driver_remove(struct intel_uc *uc)
 170 {
 171         intel_uc_fini_hw(uc);
 172         intel_uc_fini(uc);
 173         __uc_free_load_err_log(uc);
 174 }
 175
 176 /*
 177  * Events triggered while CT buffers are disabled are logged in the SCRATCH_15
 178  * register using the same bits used in the CT message payload. Since our
 179  * communication channel with guc is turned off at this point, we can save the
 180  * message and handle it after we turn it back on.
 181  */
 182 static void guc_clear_mmio_msg(struct intel_guc *guc)
 183 {
 184         intel_uncore_write(guc_to_gt(guc)->uncore, SOFT_SCRATCH(15), 0);
 185 }
 186
 187 static void guc_get_mmio_msg(struct intel_guc *guc)
 188 {
 189         u32 val;
 190
 191         spin_lock_irq(&guc->irq_lock);
 192
 193         val = intel_uncore_read(guc_to_gt(guc)->uncore, SOFT_SCRATCH(15));
 194         guc->mmio_msg |= val & guc->msg_enabled_mask;
 195
 196         /*
 197          * clear all events, including the ones we're not currently servicing,
 198          * to make sure we don't try to process a stale message if we enable
 199          * handling of more events later.
 200          */
 201         guc_clear_mmio_msg(guc);
 202
 203         spin_unlock_irq(&guc->irq_lock);
 204 }
 205
 206 static void guc_handle_mmio_msg(struct intel_guc *guc)
 207 {
 208         /* we need communication to be enabled to reply to GuC */
 209         GEM_BUG_ON(!intel_guc_ct_enabled(&guc->ct));
 210
 211         spin_lock_irq(&guc->irq_lock);
 212         if (guc->mmio_msg) {
 213                 intel_guc_to_host_process_recv_msg(guc, &guc->mmio_msg, 1);
 214                 guc->mmio_msg = 0;
 215         }
 216         spin_unlock_irq(&guc->irq_lock);
 217 }
 218
 219 static int guc_enable_communication(struct intel_guc *guc)
 220 {
 221         struct intel_gt *gt = guc_to_gt(guc);
 222         struct drm_i915_private *i915 = gt->i915;
 223         int ret;
 224
 225         GEM_BUG_ON(intel_guc_ct_enabled(&guc->ct));
 226
 227         ret = i915_inject_probe_error(i915, -ENXIO);
 228         if (ret)
 229                 return ret;
 230
 231         ret = intel_guc_ct_enable(&guc->ct);
 232         if (ret)
 233                 return ret;
 234
 235         /* check for mmio messages received before/during the CT enable */
 236         guc_get_mmio_msg(guc);
 237         guc_handle_mmio_msg(guc);
 238
 239         intel_guc_enable_interrupts(guc);
 240
 241         /* check for CT messages received before we enabled interrupts */
 242         spin_lock_irq(gt->irq_lock);
 243         intel_guc_ct_event_handler(&guc->ct);
 244         spin_unlock_irq(gt->irq_lock);
 245
 246         guc_dbg(guc, "communication enabled\n");
 247
 248         return 0;
 249 }
 250
 251 static void guc_disable_communication(struct intel_guc *guc)
 252 {
 253         /*
 254          * Events generated during or after CT disable are logged by guc in
 255          * via mmio. Make sure the register is clear before disabling CT since
 256          * all events we cared about have already been processed via CT.
 257          */
 258         guc_clear_mmio_msg(guc);
 259
 260         intel_guc_disable_interrupts(guc);
 261
 262         intel_guc_ct_disable(&guc->ct);
 263
 264         /*
 265          * Check for messages received during/after the CT disable. We do not
 266          * expect any messages to have arrived via CT between the interrupt
 267          * disable and the CT disable because GuC should've been idle until we
 268          * triggered the CT disable protocol.
 269          */
 270         guc_get_mmio_msg(guc);
 271
 272         guc_dbg(guc, "communication disabled\n");
 273 }
 274
 275 static void __uc_fetch_firmwares(struct intel_uc *uc)
 276 {
 277         struct intel_gt *gt = uc_to_gt(uc);
 278         int err;
 279
 280         GEM_BUG_ON(!intel_uc_wants_guc(uc));
 281
 282         err = intel_uc_fw_fetch(&uc->guc.fw);
 283         if (err) {
 284                 /* Make sure we transition out of transient "SELECTED" state */
 285                 if (intel_uc_wants_huc(uc)) {
 286                         gt_dbg(gt, "Failed to fetch GuC fw (%pe) disabling HuC\n", ERR_PTR(err));
 287                         intel_uc_fw_change_status(&uc->huc.fw,
 288                                                   INTEL_UC_FIRMWARE_ERROR);
 289                 }
 290
 291                 if (intel_uc_wants_gsc_uc(uc)) {
 292                         gt_dbg(gt, "Failed to fetch GuC fw (%pe) disabling GSC\n", ERR_PTR(err));
 293                         intel_uc_fw_change_status(&uc->gsc.fw,
 294                                                   INTEL_UC_FIRMWARE_ERROR);
 295                 }
 296
 297                 return;
 298         }
 299
 300         if (intel_uc_wants_huc(uc))
 301                 intel_uc_fw_fetch(&uc->huc.fw);
 302
 303         if (intel_uc_wants_gsc_uc(uc))
 304                 intel_uc_fw_fetch(&uc->gsc.fw);
 305 }
 306
 307 static void __uc_cleanup_firmwares(struct intel_uc *uc)
 308 {
 309         intel_uc_fw_cleanup_fetch(&uc->gsc.fw);
 310         intel_uc_fw_cleanup_fetch(&uc->huc.fw);
 311         intel_uc_fw_cleanup_fetch(&uc->guc.fw);
 312 }
 313
 314 static int __uc_init(struct intel_uc *uc)
 315 {
 316         struct intel_guc *guc = &uc->guc;
 317         struct intel_huc *huc = &uc->huc;
 318         int ret;
 319
 320         GEM_BUG_ON(!intel_uc_wants_guc(uc));
 321
 322         if (!intel_uc_uses_guc(uc))
 323                 return 0;
 324
 325         if (i915_inject_probe_failure(uc_to_gt(uc)->i915))
 326                 return -ENOMEM;
 327
 328         ret = intel_guc_init(guc);
 329         if (ret)
 330                 return ret;
 331
 332         if (intel_uc_uses_huc(uc))
 333                 intel_huc_init(huc);
 334
 335         if (intel_uc_uses_gsc_uc(uc))
 336                 intel_gsc_uc_init(&uc->gsc);
 337
 338         return 0;
 339 }
 340
 341 static void __uc_fini(struct intel_uc *uc)
 342 {
 343         intel_gsc_uc_fini(&uc->gsc);
 344         intel_huc_fini(&uc->huc);
 345         intel_guc_fini(&uc->guc);
 346 }
 347
 348 static int __uc_sanitize(struct intel_uc *uc)
 349 {
 350         struct intel_guc *guc = &uc->guc;
 351         struct intel_huc *huc = &uc->huc;
 352
 353         GEM_BUG_ON(!intel_uc_supports_guc(uc));
 354
 355         intel_huc_sanitize(huc);
 356         intel_guc_sanitize(guc);
 357
 358         return __intel_uc_reset_hw(uc);
 359 }
 360
 361 /* Initialize and verify the uC regs related to uC positioning in WOPCM */
 362 static int uc_init_wopcm(struct intel_uc *uc)
 363 {
 364         struct intel_gt *gt = uc_to_gt(uc);
 365         struct intel_uncore *uncore = gt->uncore;
 366         u32 base = intel_wopcm_guc_base(&gt->wopcm);
 367         u32 size = intel_wopcm_guc_size(&gt->wopcm);
 368         u32 huc_agent = intel_uc_uses_huc(uc) ? HUC_LOADING_AGENT_GUC : 0;
 369         u32 mask;
 370         int err;
 371
 372         if (unlikely(!base || !size)) {
 373                 gt_probe_error(gt, "Unsuccessful WOPCM partitioning\n");
 374                 return -E2BIG;
 375         }
 376
 377         GEM_BUG_ON(!intel_uc_supports_guc(uc));
 378         GEM_BUG_ON(!(base & GUC_WOPCM_OFFSET_MASK));
 379         GEM_BUG_ON(base & ~GUC_WOPCM_OFFSET_MASK);
 380         GEM_BUG_ON(!(size & GUC_WOPCM_SIZE_MASK));
 381         GEM_BUG_ON(size & ~GUC_WOPCM_SIZE_MASK);
 382
 383         err = i915_inject_probe_error(gt->i915, -ENXIO);
 384         if (err)
 385                 return err;
 386
 387         mask = GUC_WOPCM_SIZE_MASK | GUC_WOPCM_SIZE_LOCKED;
 388         err = intel_uncore_write_and_verify(uncore, GUC_WOPCM_SIZE, size, mask,
 389                                             size | GUC_WOPCM_SIZE_LOCKED);
 390         if (err)
 391                 goto err_out;
 392
 393         mask = GUC_WOPCM_OFFSET_MASK | GUC_WOPCM_OFFSET_VALID | huc_agent;
 394         err = intel_uncore_write_and_verify(uncore, DMA_GUC_WOPCM_OFFSET,
 395                                             base | huc_agent, mask,
 396                                             base | huc_agent |
 397                                             GUC_WOPCM_OFFSET_VALID);
 398         if (err)
 399                 goto err_out;
 400
 401         return 0;
 402
 403 err_out:
 404         gt_probe_error(gt, "Failed to init uC WOPCM registers!\n");
 405         gt_probe_error(gt, "%s(%#x)=%#x\n", "DMA_GUC_WOPCM_OFFSET",
 406                        i915_mmio_reg_offset(DMA_GUC_WOPCM_OFFSET),
 407                        intel_uncore_read(uncore, DMA_GUC_WOPCM_OFFSET));
 408         gt_probe_error(gt, "%s(%#x)=%#x\n", "GUC_WOPCM_SIZE",
 409                        i915_mmio_reg_offset(GUC_WOPCM_SIZE),
 410                        intel_uncore_read(uncore, GUC_WOPCM_SIZE));
 411
 412         return err;
 413 }
 414
 415 static bool uc_is_wopcm_locked(struct intel_uc *uc)
 416 {
 417         struct intel_gt *gt = uc_to_gt(uc);
 418         struct intel_uncore *uncore = gt->uncore;
 419
 420         return (intel_uncore_read(uncore, GUC_WOPCM_SIZE) & GUC_WOPCM_SIZE_LOCKED) ||
 421                (intel_uncore_read(uncore, DMA_GUC_WOPCM_OFFSET) & GUC_WOPCM_OFFSET_VALID);
 422 }
 423
 424 static int __uc_check_hw(struct intel_uc *uc)
 425 {
 426         if (uc->fw_table_invalid)
 427                 return -EIO;
 428
 429         if (!intel_uc_supports_guc(uc))
 430                 return 0;
 431
 432         /*
 433          * We can silently continue without GuC only if it was never enabled
 434          * before on this system after reboot, otherwise we risk GPU hangs.
 435          * To check if GuC was loaded before we look at WOPCM registers.
 436          */
 437         if (uc_is_wopcm_locked(uc))
 438                 return -EIO;
 439
 440         return 0;
 441 }
 442
 443 static void print_fw_ver(struct intel_gt *gt, struct intel_uc_fw *fw)
 444 {
 445         gt_info(gt, "%s firmware %s version %u.%u.%u\n",
 446                 intel_uc_fw_type_repr(fw->type), fw->file_selected.path,
 447                 fw->file_selected.ver.major,
 448                 fw->file_selected.ver.minor,
 449                 fw->file_selected.ver.patch);
 450 }
 451
 452 static int __uc_init_hw(struct intel_uc *uc)
 453 {
 454         struct intel_gt *gt = uc_to_gt(uc);
 455         struct drm_i915_private *i915 = gt->i915;
 456         struct intel_guc *guc = &uc->guc;
 457         struct intel_huc *huc = &uc->huc;
 458         int ret, attempts;
 459         bool pl1en = false;
 460
 461         GEM_BUG_ON(!intel_uc_supports_guc(uc));
 462         GEM_BUG_ON(!intel_uc_wants_guc(uc));
 463
 464         print_fw_ver(gt, &guc->fw);
 465
 466         if (intel_uc_uses_huc(uc))
 467                 print_fw_ver(gt, &huc->fw);
 468
 469         if (!intel_uc_fw_is_loadable(&guc->fw)) {
 470                 ret = __uc_check_hw(uc) ||
 471                       intel_uc_fw_is_overridden(&guc->fw) ||
 472                       intel_uc_wants_guc_submission(uc) ?
 473                       intel_uc_fw_status_to_error(guc->fw.status) : 0;
 474                 goto err_out;
 475         }
 476
 477         ret = uc_init_wopcm(uc);
 478         if (ret)
 479                 goto err_out;
 480
 481         intel_guc_reset_interrupts(guc);
 482
 483         /* WaEnableuKernelHeaderValidFix:skl */
 484         /* WaEnableGuCBootHashCheckNotSet:skl,bxt,kbl */
 485         if (GRAPHICS_VER(i915) == 9)
 486                 attempts = 3;
 487         else
 488                 attempts = 1;
 489
 490         /* Disable a potentially low PL1 power limit to allow freq to be raised */
 491         i915_hwmon_power_max_disable(gt->i915, &pl1en);
 492
 493         intel_rps_raise_unslice(&uc_to_gt(uc)->rps);
 494
 495         while (attempts--) {
 496                 /*
 497                  * Always reset the GuC just before (re)loading, so
 498                  * that the state and timing are fairly predictable
 499                  */
 500                 ret = __uc_sanitize(uc);
 501                 if (ret)
 502                         goto err_rps;
 503
 504                 intel_huc_fw_upload(huc);
 505                 intel_guc_ads_reset(guc);
 506                 intel_guc_write_params(guc);
 507                 ret = intel_guc_fw_upload(guc);
 508                 if (ret == 0)
 509                         break;
 510
 511                 gt_dbg(gt, "GuC fw load failed (%pe) will reset and retry %d more time(s)\n",
 512                        ERR_PTR(ret), attempts);
 513         }
 514
 515         /* Did we succeded or run out of retries? */
 516         if (ret)
 517                 goto err_log_capture;
 518
 519         ret = guc_enable_communication(guc);
 520         if (ret)
 521                 goto err_log_capture;
 522
 523         /*
 524          * GSC-loaded HuC is authenticated by the GSC, so we don't need to
 525          * trigger the auth here. However, given that the HuC loaded this way
 526          * survive GT reset, we still need to update our SW bookkeeping to make
 527          * sure it reflects the correct HW status.
 528          */
 529         if (intel_huc_is_loaded_by_gsc(huc))
 530                 intel_huc_update_auth_status(huc);
 531         else
 532                 intel_huc_auth(huc, INTEL_HUC_AUTH_BY_GUC);
 533
 534         if (intel_uc_uses_guc_submission(uc)) {
 535                 ret = intel_guc_submission_enable(guc);
 536                 if (ret)
 537                         goto err_log_capture;
 538         }
 539
 540         if (intel_uc_uses_guc_slpc(uc)) {
 541                 ret = intel_guc_slpc_enable(&guc->slpc);
 542                 if (ret)
 543                         goto err_submission;
 544         } else {
 545                 /* Restore GT back to RPn for non-SLPC path */
 546                 intel_rps_lower_unslice(&uc_to_gt(uc)->rps);
 547         }
 548
 549         i915_hwmon_power_max_restore(gt->i915, pl1en);
 550
 551         guc_info(guc, "submission %s\n", str_enabled_disabled(intel_uc_uses_guc_submission(uc)));
 552         guc_info(guc, "SLPC %s\n", str_enabled_disabled(intel_uc_uses_guc_slpc(uc)));
 553
 554         return 0;
 555
 556         /*
 557          * We've failed to load the firmware :(
 558          */
 559 err_submission:
 560         intel_guc_submission_disable(guc);
 561 err_log_capture:
 562         __uc_capture_load_err_log(uc);
 563 err_rps:
 564         /* Return GT back to RPn */
 565         intel_rps_lower_unslice(&uc_to_gt(uc)->rps);
 566
 567         i915_hwmon_power_max_restore(gt->i915, pl1en);
 568 err_out:
 569         __uc_sanitize(uc);
 570
 571         if (!ret) {
 572                 gt_notice(gt, "GuC is uninitialized\n");
 573                 /* We want to run without GuC submission */
 574                 return 0;
 575         }
 576
 577         gt_probe_error(gt, "GuC initialization failed %pe\n", ERR_PTR(ret));
 578
 579         /* We want to keep KMS alive */
 580         return -EIO;
 581 }
 582
 583 static void __uc_fini_hw(struct intel_uc *uc)
 584 {
 585         struct intel_guc *guc = &uc->guc;
 586
 587         if (!intel_guc_is_fw_running(guc))
 588                 return;
 589
 590         if (intel_uc_uses_guc_submission(uc))
 591                 intel_guc_submission_disable(guc);
 592
 593         __uc_sanitize(uc);
 594 }
 595
 596 /**
 597  * intel_uc_reset_prepare - Prepare for reset
 598  * @uc: the intel_uc structure
 599  *
 600  * Preparing for full gpu reset.
 601  */
 602 void intel_uc_reset_prepare(struct intel_uc *uc)
 603 {
 604         struct intel_guc *guc = &uc->guc;
 605
 606         uc->reset_in_progress = true;
 607
 608         /* Nothing to do if GuC isn't supported */
 609         if (!intel_uc_supports_guc(uc))
 610                 return;
 611
 612         /* Firmware expected to be running when this function is called */
 613         if (!intel_guc_is_ready(guc))
 614                 goto sanitize;
 615
 616         if (intel_uc_uses_guc_submission(uc))
 617                 intel_guc_submission_reset_prepare(guc);
 618
 619 sanitize:
 620         __uc_sanitize(uc);
 621 }
 622
 623 void intel_uc_reset(struct intel_uc *uc, intel_engine_mask_t stalled)
 624 {
 625         struct intel_guc *guc = &uc->guc;
 626
 627         /* Firmware can not be running when this function is called  */
 628         if (intel_uc_uses_guc_submission(uc))
 629                 intel_guc_submission_reset(guc, stalled);
 630 }
 631
 632 void intel_uc_reset_finish(struct intel_uc *uc)
 633 {
 634         struct intel_guc *guc = &uc->guc;
 635
 636         /*
 637          * NB: The wedge code path results in prepare -> prepare -> finish -> finish.
 638          * So this function is sometimes called with the in-progress flag not set.
 639          */
 640         uc->reset_in_progress = false;
 641
 642         /* Firmware expected to be running when this function is called */
 643         if (intel_uc_uses_guc_submission(uc))
 644                 intel_guc_submission_reset_finish(guc);
 645 }
 646
 647 void intel_uc_cancel_requests(struct intel_uc *uc)
 648 {
 649         struct intel_guc *guc = &uc->guc;
 650
 651         /* Firmware can not be running when this function is called  */
 652         if (intel_uc_uses_guc_submission(uc))
 653                 intel_guc_submission_cancel_requests(guc);
 654 }
 655
 656 void intel_uc_runtime_suspend(struct intel_uc *uc)
 657 {
 658         struct intel_guc *guc = &uc->guc;
 659
 660         if (!intel_guc_is_ready(guc)) {
 661                 guc->interrupts.enabled = false;
 662                 return;
 663         }
 664
 665         /*
 666          * Wait for any outstanding CTB before tearing down communication /w the
 667          * GuC.
 668          */
 669 #define OUTSTANDING_CTB_TIMEOUT_PERIOD  (HZ / 5)
 670         intel_guc_wait_for_pending_msg(guc, &guc->outstanding_submission_g2h,
 671                                        false, OUTSTANDING_CTB_TIMEOUT_PERIOD);
 672         GEM_WARN_ON(atomic_read(&guc->outstanding_submission_g2h));
 673
 674         guc_disable_communication(guc);
 675 }
 676
 677 void intel_uc_suspend(struct intel_uc *uc)
 678 {
 679         struct intel_guc *guc = &uc->guc;
 680         intel_wakeref_t wakeref;
 681         int err;
 682
 683         /* flush the GSC worker */
 684         intel_gsc_uc_flush_work(&uc->gsc);
 685
 686         wake_up_all_tlb_invalidate(guc);
 687
 688         if (!intel_guc_is_ready(guc)) {
 689                 guc->interrupts.enabled = false;
 690                 return;
 691         }
 692
 693         intel_guc_submission_flush_work(guc);
 694
 695         with_intel_runtime_pm(&uc_to_gt(uc)->i915->runtime_pm, wakeref) {
 696                 err = intel_guc_suspend(guc);
 697                 if (err)
 698                         guc_dbg(guc, "Failed to suspend, %pe", ERR_PTR(err));
 699         }
 700 }
 701
 702 static void __uc_resume_mappings(struct intel_uc *uc)
 703 {
 704         intel_uc_fw_resume_mapping(&uc->guc.fw);
 705         intel_uc_fw_resume_mapping(&uc->huc.fw);
 706 }
 707
 708 static int __uc_resume(struct intel_uc *uc, bool enable_communication)
 709 {
 710         struct intel_guc *guc = &uc->guc;
 711         struct intel_gt *gt = guc_to_gt(guc);
 712         int err;
 713
 714         if (!intel_guc_is_fw_running(guc))
 715                 return 0;
 716
 717         /* Make sure we enable communication if and only if it's disabled */
 718         GEM_BUG_ON(enable_communication == intel_guc_ct_enabled(&guc->ct));
 719
 720         if (enable_communication)
 721                 guc_enable_communication(guc);
 722
 723         /* If we are only resuming GuC communication but not reloading
 724          * GuC, we need to ensure the ARAT timer interrupt is enabled
 725          * again. In case of GuC reload, it is enabled during SLPC enable.
 726          */
 727         if (enable_communication && intel_uc_uses_guc_slpc(uc))
 728                 intel_guc_pm_intrmsk_enable(gt);
 729
 730         err = intel_guc_resume(guc);
 731         if (err) {
 732                 guc_dbg(guc, "Failed to resume, %pe", ERR_PTR(err));
 733                 return err;
 734         }
 735
 736         intel_gsc_uc_resume(&uc->gsc);
 737
 738         if (intel_guc_tlb_invalidation_is_available(guc)) {
 739                 intel_guc_invalidate_tlb_engines(guc);
 740                 intel_guc_invalidate_tlb_guc(guc);
 741         }
 742
 743         return 0;
 744 }
 745
 746 int intel_uc_resume(struct intel_uc *uc)
 747 {
 748         /*
 749          * When coming out of S3/S4 we sanitize and re-init the HW, so
 750          * communication is already re-enabled at this point.
 751          */
 752         return __uc_resume(uc, false);
 753 }
 754
 755 int intel_uc_runtime_resume(struct intel_uc *uc)
 756 {
 757         /*
 758          * During runtime resume we don't sanitize, so we need to re-init
 759          * communication as well.
 760          */
 761         return __uc_resume(uc, true);
 762 }
 763
 764 static const struct intel_uc_ops uc_ops_off = {
 765         .init_hw = __uc_check_hw,
 766         .fini = __uc_fini, /* to clean-up the init_early initialization */
 767 };
 768
 769 static const struct intel_uc_ops uc_ops_on = {
 770         .sanitize = __uc_sanitize,
 771
 772         .init_fw = __uc_fetch_firmwares,
 773         .fini_fw = __uc_cleanup_firmwares,
 774
 775         .init = __uc_init,
 776         .fini = __uc_fini,
 777
 778         .init_hw = __uc_init_hw,
 779         .fini_hw = __uc_fini_hw,
 780
 781         .resume_mappings = __uc_resume_mappings,
 782 };