drivers/gpu/drm/i915/gt/uc/intel_guc.h

   1 /* SPDX-License-Identifier: MIT */
   2 /*
   3  * Copyright © 2014-2019 Intel Corporation
   4  */
   5
   6 #ifndef _INTEL_GUC_H_
   7 #define _INTEL_GUC_H_
   8
   9 #include <linux/delay.h>
  10 #include <linux/iosys-map.h>
  11 #include <linux/xarray.h>
  12
  13 #include "intel_guc_ct.h"
  14 #include "intel_guc_fw.h"
  15 #include "intel_guc_fwif.h"
  16 #include "intel_guc_log.h"
  17 #include "intel_guc_reg.h"
  18 #include "intel_guc_slpc_types.h"
  19 #include "intel_uc_fw.h"
  20 #include "intel_uncore.h"
  21 #include "i915_utils.h"
  22 #include "i915_vma.h"
  23
  24 struct __guc_ads_blob;
  25 struct intel_guc_state_capture;
  26
  27 /**
  28  * struct intel_guc - Top level structure of GuC.
  29  *
  30  * It handles firmware loading and manages client pool. intel_guc owns an
  31  * i915_sched_engine for submission.
  32  */
  33 struct intel_guc {
  34         /** @fw: the GuC firmware */
  35         struct intel_uc_fw fw;
  36         /** @log: sub-structure containing GuC log related data and objects */
  37         struct intel_guc_log log;
  38         /** @ct: the command transport communication channel */
  39         struct intel_guc_ct ct;
  40         /** @slpc: sub-structure containing SLPC related data and objects */
  41         struct intel_guc_slpc slpc;
  42         /** @capture: the error-state-capture module's data and objects */
  43         struct intel_guc_state_capture *capture;
  44
  45         /** @dbgfs_node: debugfs node */
  46         struct dentry *dbgfs_node;
  47
  48         /** @sched_engine: Global engine used to submit requests to GuC */
  49         struct i915_sched_engine *sched_engine;
  50         /**
  51          * @stalled_request: if GuC can't process a request for any reason, we
  52          * save it until GuC restarts processing. No other request can be
  53          * submitted until the stalled request is processed.
  54          */
  55         struct i915_request *stalled_request;
  56         /**
  57          * @submission_stall_reason: reason why submission is stalled
  58          */
  59         enum {
  60                 STALL_NONE,
  61                 STALL_REGISTER_CONTEXT,
  62                 STALL_MOVE_LRC_TAIL,
  63                 STALL_ADD_REQUEST,
  64         } submission_stall_reason;
  65
  66         /* intel_guc_recv interrupt related state */
  67         /** @irq_lock: protects GuC irq state */
  68         spinlock_t irq_lock;
  69         /**
  70          * @msg_enabled_mask: mask of events that are processed when receiving
  71          * an INTEL_GUC_ACTION_DEFAULT G2H message.
  72          */
  73         unsigned int msg_enabled_mask;
  74
  75         /**
  76          * @outstanding_submission_g2h: number of outstanding GuC to Host
  77          * responses related to GuC submission, used to determine if the GT is
  78          * idle
  79          */
  80         atomic_t outstanding_submission_g2h;
  81
  82         /** @tlb_lookup: xarray to store all pending TLB invalidation requests */
  83         struct xarray tlb_lookup;
  84
  85         /**
  86          * @serial_slot: id to the initial waiter created in tlb_lookup,
  87          * which is used only when failed to allocate new waiter.
  88          */
  89         u32 serial_slot;
  90
  91         /** @next_seqno: the next id (sequence number) to allocate. */
  92         u32 next_seqno;
  93
  94         /** @interrupts: pointers to GuC interrupt-managing functions. */
  95         struct {
  96                 bool enabled;
  97                 void (*reset)(struct intel_guc *guc);
  98                 void (*enable)(struct intel_guc *guc);
  99                 void (*disable)(struct intel_guc *guc);
 100         } interrupts;
 101
 102         /**
 103          * @submission_state: sub-structure for submission state protected by
 104          * single lock
 105          */
 106         struct {
 107                 /**
 108                  * @submission_state.lock: protects everything in
 109                  * submission_state, ce->guc_id.id, and ce->guc_id.ref
 110                  * when transitioning in and out of zero
 111                  */
 112                 spinlock_t lock;
 113                 /**
 114                  * @submission_state.guc_ids: used to allocate new
 115                  * guc_ids, single-lrc
 116                  */
 117                 struct ida guc_ids;
 118                 /**
 119                  * @submission_state.num_guc_ids: Number of guc_ids, selftest
 120                  * feature to be able to reduce this number while testing.
 121                  */
 122                 int num_guc_ids;
 123                 /**
 124                  * @submission_state.guc_ids_bitmap: used to allocate
 125                  * new guc_ids, multi-lrc
 126                  */
 127                 unsigned long *guc_ids_bitmap;
 128                 /**
 129                  * @submission_state.guc_id_list: list of intel_context
 130                  * with valid guc_ids but no refs
 131                  */
 132                 struct list_head guc_id_list;
 133                 /**
 134                  * @submission_state.guc_ids_in_use: Number single-lrc
 135                  * guc_ids in use
 136                  */
 137                 unsigned int guc_ids_in_use;
 138                 /**
 139                  * @submission_state.destroyed_contexts: list of contexts
 140                  * waiting to be destroyed (deregistered with the GuC)
 141                  */
 142                 struct list_head destroyed_contexts;
 143                 /**
 144                  * @submission_state.destroyed_worker: worker to deregister
 145                  * contexts, need as we need to take a GT PM reference and
 146                  * can't from destroy function as it might be in an atomic
 147                  * context (no sleeping)
 148                  */
 149                 struct work_struct destroyed_worker;
 150                 /**
 151                  * @submission_state.reset_fail_worker: worker to trigger
 152                  * a GT reset after an engine reset fails
 153                  */
 154                 struct work_struct reset_fail_worker;
 155                 /**
 156                  * @submission_state.reset_fail_mask: mask of engines that
 157                  * failed to reset
 158                  */
 159                 intel_engine_mask_t reset_fail_mask;
 160                 /**
 161                  * @submission_state.sched_disable_delay_ms: schedule
 162                  * disable delay, in ms, for contexts
 163                  */
 164                 unsigned int sched_disable_delay_ms;
 165                 /**
 166                  * @submission_state.sched_disable_gucid_threshold:
 167                  * threshold of min remaining available guc_ids before
 168                  * we start bypassing the schedule disable delay
 169                  */
 170                 unsigned int sched_disable_gucid_threshold;
 171         } submission_state;
 172
 173         /**
 174          * @submission_supported: tracks whether we support GuC submission on
 175          * the current platform
 176          */
 177         bool submission_supported;
 178         /** @submission_selected: tracks whether the user enabled GuC submission */
 179         bool submission_selected;
 180         /** @submission_initialized: tracks whether GuC submission has been initialised */
 181         bool submission_initialized;
 182         /** @submission_version: Submission API version of the currently loaded firmware */
 183         struct intel_uc_fw_ver submission_version;
 184
 185         /**
 186          * @rc_supported: tracks whether we support GuC rc on the current platform
 187          */
 188         bool rc_supported;
 189         /** @rc_selected: tracks whether the user enabled GuC rc */
 190         bool rc_selected;
 191
 192         /** @ads_vma: object allocated to hold the GuC ADS */
 193         struct i915_vma *ads_vma;
 194         /** @ads_map: contents of the GuC ADS */
 195         struct iosys_map ads_map;
 196         /** @ads_regset_size: size of the save/restore regsets in the ADS */
 197         u32 ads_regset_size;
 198         /**
 199          * @ads_regset_count: number of save/restore registers in the ADS for
 200          * each engine
 201          */
 202         u32 ads_regset_count[I915_NUM_ENGINES];
 203         /** @ads_regset: save/restore regsets in the ADS */
 204         struct guc_mmio_reg *ads_regset;
 205         /** @ads_golden_ctxt_size: size of the golden contexts in the ADS */
 206         u32 ads_golden_ctxt_size;
 207         /** @ads_capture_size: size of register lists in the ADS used for error capture */
 208         u32 ads_capture_size;
 209
 210         /** @lrc_desc_pool_v69: object allocated to hold the GuC LRC descriptor pool */
 211         struct i915_vma *lrc_desc_pool_v69;
 212         /** @lrc_desc_pool_vaddr_v69: contents of the GuC LRC descriptor pool */
 213         void *lrc_desc_pool_vaddr_v69;
 214
 215         /**
 216          * @context_lookup: used to resolve intel_context from guc_id, if a
 217          * context is present in this structure it is registered with the GuC
 218          */
 219         struct xarray context_lookup;
 220
 221         /** @params: Control params for fw initialization */
 222         u32 params[GUC_CTL_MAX_DWORDS];
 223
 224         /** @send_regs: GuC's FW specific registers used for sending MMIO H2G */
 225         struct {
 226                 u32 base;
 227                 unsigned int count;
 228                 enum forcewake_domains fw_domains;
 229         } send_regs;
 230
 231         /** @notify_reg: register used to send interrupts to the GuC FW */
 232         i915_reg_t notify_reg;
 233
 234         /**
 235          * @mmio_msg: notification bitmask that the GuC writes in one of its
 236          * registers when the CT channel is disabled, to be processed when the
 237          * channel is back up.
 238          */
 239         u32 mmio_msg;
 240
 241         /** @send_mutex: used to serialize the intel_guc_send actions */
 242         struct mutex send_mutex;
 243
 244         /**
 245          * @timestamp: GT timestamp object that stores a copy of the timestamp
 246          * and adjusts it for overflow using a worker.
 247          */
 248         struct {
 249                 /**
 250                  * @timestamp.lock: Lock protecting the below fields and
 251                  * the engine stats.
 252                  */
 253                 spinlock_t lock;
 254
 255                 /**
 256                  * @timestamp.gt_stamp: 64-bit extended value of the GT
 257                  * timestamp.
 258                  */
 259                 u64 gt_stamp;
 260
 261                 /**
 262                  * @timestamp.ping_delay: Period for polling the GT
 263                  * timestamp for overflow.
 264                  */
 265                 unsigned long ping_delay;
 266
 267                 /**
 268                  * @timestamp.work: Periodic work to adjust GT timestamp,
 269                  * engine and context usage for overflows.
 270                  */
 271                 struct delayed_work work;
 272
 273                 /**
 274                  * @timestamp.shift: Right shift value for the gpm timestamp
 275                  */
 276                 u32 shift;
 277
 278                 /**
 279                  * @timestamp.last_stat_jiffies: jiffies at last actual
 280                  * stats collection time. We use this timestamp to ensure
 281                  * we don't oversample the stats because runtime power
 282                  * management events can trigger stats collection at much
 283                  * higher rates than required.
 284                  */
 285                 unsigned long last_stat_jiffies;
 286         } timestamp;
 287
 288         /**
 289          * @dead_guc_worker: Asynchronous worker thread for forcing a GuC reset.
 290          * Specifically used when the G2H handler wants to issue a reset. Resets
 291          * require flushing the G2H queue. So, the G2H processing itself must not
 292          * trigger a reset directly. Instead, go via this worker.
 293          */
 294         struct work_struct dead_guc_worker;
 295         /**
 296          * @last_dead_guc_jiffies: timestamp of previous 'dead guc' occurrance
 297          * used to prevent a fundamentally broken system from continuously
 298          * reloading the GuC.
 299          */
 300         unsigned long last_dead_guc_jiffies;
 301
 302 #ifdef CONFIG_DRM_I915_SELFTEST
 303         /**
 304          * @number_guc_id_stolen: The number of guc_ids that have been stolen
 305          */
 306         int number_guc_id_stolen;
 307         /**
 308          * @fast_response_selftest: Backdoor to CT handler for fast response selftest
 309          */
 310         u32 fast_response_selftest;
 311 #endif
 312 };
 313
 314 struct intel_guc_tlb_wait {
 315         struct wait_queue_head wq;
 316         bool busy;
 317 };
 318
 319 /*
 320  * GuC version number components are only 8-bit, so converting to a 32bit 8.8.8
 321  * integer works.
 322  */
 323 #define MAKE_GUC_VER(maj, min, pat)     (((maj) << 16) | ((min) << 8) | (pat))
 324 #define MAKE_GUC_VER_STRUCT(ver)        MAKE_GUC_VER((ver).major, (ver).minor, (ver).patch)
 325 #define GUC_SUBMIT_VER(guc)             MAKE_GUC_VER_STRUCT((guc)->submission_version)
 326 #define GUC_FIRMWARE_VER(guc)           MAKE_GUC_VER_STRUCT((guc)->fw.file_selected.ver)
 327
 328 static inline struct intel_guc *log_to_guc(struct intel_guc_log *log)
 329 {
 330         return container_of(log, struct intel_guc, log);
 331 }
 332
 333 static
 334 inline int intel_guc_send(struct intel_guc *guc, const u32 *action, u32 len)
 335 {
 336         return intel_guc_ct_send(&guc->ct, action, len, NULL, 0, 0);
 337 }
 338
 339 static
 340 inline int intel_guc_send_nb(struct intel_guc *guc, const u32 *action, u32 len,
 341                              u32 g2h_len_dw)
 342 {
 343         return intel_guc_ct_send(&guc->ct, action, len, NULL, 0,
 344                                  MAKE_SEND_FLAGS(g2h_len_dw));
 345 }
 346
 347 static inline int
 348 intel_guc_send_and_receive(struct intel_guc *guc, const u32 *action, u32 len,
 349                            u32 *response_buf, u32 response_buf_size)
 350 {
 351         return intel_guc_ct_send(&guc->ct, action, len,
 352                                  response_buf, response_buf_size, 0);
 353 }
 354
 355 static inline int intel_guc_send_busy_loop(struct intel_guc *guc,
 356                                            const u32 *action,
 357                                            u32 len,
 358                                            u32 g2h_len_dw,
 359                                            bool loop)
 360 {
 361         int err;
 362         unsigned int sleep_period_ms = 1;
 363         bool not_atomic = !in_atomic() && !irqs_disabled();
 364
 365         /*
 366          * FIXME: Have caller pass in if we are in an atomic context to avoid
 367          * using in_atomic(). It is likely safe here as we check for irqs
 368          * disabled which basically all the spin locks in the i915 do but
 369          * regardless this should be cleaned up.
 370          */
 371
 372         /* No sleeping with spin locks, just busy loop */
 373         might_sleep_if(loop && not_atomic);
 374
 375 retry:
 376         err = intel_guc_send_nb(guc, action, len, g2h_len_dw);
 377         if (unlikely(err == -EBUSY && loop)) {
 378                 if (likely(not_atomic)) {
 379                         if (msleep_interruptible(sleep_period_ms))
 380                                 return -EINTR;
 381                         sleep_period_ms = sleep_period_ms << 1;
 382                 } else {
 383                         cpu_relax();
 384                 }
 385                 goto retry;
 386         }
 387
 388         return err;
 389 }
 390
 391 /* Only call this from the interrupt handler code */
 392 static inline void intel_guc_to_host_event_handler(struct intel_guc *guc)
 393 {
 394         if (guc->interrupts.enabled)
 395                 intel_guc_ct_event_handler(&guc->ct);
 396 }
 397
 398 /* GuC addresses above GUC_GGTT_TOP also don't map through the GTT */
 399 #define GUC_GGTT_TOP    0xFEE00000
 400
 401 /**
 402  * intel_guc_ggtt_offset() - Get and validate the GGTT offset of @vma
 403  * @guc: intel_guc structure.
 404  * @vma: i915 graphics virtual memory area.
 405  *
 406  * GuC does not allow any gfx GGTT address that falls into range
 407  * [0, ggtt.pin_bias), which is reserved for Boot ROM, SRAM and WOPCM.
 408  * Currently, in order to exclude [0, ggtt.pin_bias) address space from
 409  * GGTT, all gfx objects used by GuC are allocated with intel_guc_allocate_vma()
 410  * and pinned with PIN_OFFSET_BIAS along with the value of ggtt.pin_bias.
 411  *
 412  * Return: GGTT offset of the @vma.
 413  */
 414 static inline u32 intel_guc_ggtt_offset(struct intel_guc *guc,
 415                                         struct i915_vma *vma)
 416 {
 417         u32 offset = i915_ggtt_offset(vma);
 418
 419         GEM_BUG_ON(offset < i915_ggtt_pin_bias(vma));
 420         GEM_BUG_ON(range_overflows_t(u64, offset, vma->size, GUC_GGTT_TOP));
 421
 422         return offset;
 423 }
 424
 425 void intel_guc_init_early(struct intel_guc *guc);
 426 void intel_guc_init_late(struct intel_guc *guc);
 427 void intel_guc_init_send_regs(struct intel_guc *guc);
 428 void intel_guc_write_params(struct intel_guc *guc);
 429 int intel_guc_init(struct intel_guc *guc);
 430 void intel_guc_fini(struct intel_guc *guc);
 431 void intel_guc_notify(struct intel_guc *guc);
 432 int intel_guc_send_mmio(struct intel_guc *guc, const u32 *action, u32 len,
 433                         u32 *response_buf, u32 response_buf_size);
 434 int intel_guc_to_host_process_recv_msg(struct intel_guc *guc,
 435                                        const u32 *payload, u32 len);
 436 int intel_guc_auth_huc(struct intel_guc *guc, u32 rsa_offset);
 437 int intel_guc_suspend(struct intel_guc *guc);
 438 int intel_guc_resume(struct intel_guc *guc);
 439 struct i915_vma *intel_guc_allocate_vma(struct intel_guc *guc, u32 size);
 440 int intel_guc_allocate_and_map_vma(struct intel_guc *guc, u32 size,
 441                                    struct i915_vma **out_vma, void **out_vaddr);
 442 int intel_guc_self_cfg32(struct intel_guc *guc, u16 key, u32 value);
 443 int intel_guc_self_cfg64(struct intel_guc *guc, u16 key, u64 value);
 444
 445 static inline bool intel_guc_is_supported(struct intel_guc *guc)
 446 {
 447         return intel_uc_fw_is_supported(&guc->fw);
 448 }
 449
 450 static inline bool intel_guc_is_wanted(struct intel_guc *guc)
 451 {
 452         return intel_uc_fw_is_enabled(&guc->fw);
 453 }
 454
 455 static inline bool intel_guc_is_used(struct intel_guc *guc)
 456 {
 457         GEM_BUG_ON(__intel_uc_fw_status(&guc->fw) == INTEL_UC_FIRMWARE_SELECTED);
 458         return intel_uc_fw_is_available(&guc->fw);
 459 }
 460
 461 static inline bool intel_guc_is_fw_running(struct intel_guc *guc)
 462 {
 463         return intel_uc_fw_is_running(&guc->fw);
 464 }
 465
 466 static inline bool intel_guc_is_ready(struct intel_guc *guc)
 467 {
 468         return intel_guc_is_fw_running(guc) && intel_guc_ct_enabled(&guc->ct);
 469 }
 470
 471 static inline void intel_guc_reset_interrupts(struct intel_guc *guc)
 472 {
 473         guc->interrupts.reset(guc);
 474 }
 475
 476 static inline void intel_guc_enable_interrupts(struct intel_guc *guc)
 477 {
 478         guc->interrupts.enable(guc);
 479 }
 480
 481 static inline void intel_guc_disable_interrupts(struct intel_guc *guc)
 482 {
 483         guc->interrupts.disable(guc);
 484 }
 485
 486 static inline int intel_guc_sanitize(struct intel_guc *guc)
 487 {
 488         intel_uc_fw_sanitize(&guc->fw);
 489         intel_guc_disable_interrupts(guc);
 490         intel_guc_ct_sanitize(&guc->ct);
 491         guc->mmio_msg = 0;
 492
 493         return 0;
 494 }
 495
 496 static inline void intel_guc_enable_msg(struct intel_guc *guc, u32 mask)
 497 {
 498         spin_lock_irq(&guc->irq_lock);
 499         guc->msg_enabled_mask |= mask;
 500         spin_unlock_irq(&guc->irq_lock);
 501 }
 502
 503 static inline void intel_guc_disable_msg(struct intel_guc *guc, u32 mask)
 504 {
 505         spin_lock_irq(&guc->irq_lock);
 506         guc->msg_enabled_mask &= ~mask;
 507         spin_unlock_irq(&guc->irq_lock);
 508 }
 509
 510 int intel_guc_wait_for_idle(struct intel_guc *guc, long timeout);
 511
 512 int intel_guc_deregister_done_process_msg(struct intel_guc *guc,
 513                                           const u32 *msg, u32 len);
 514 int intel_guc_sched_done_process_msg(struct intel_guc *guc,
 515                                      const u32 *msg, u32 len);
 516 int intel_guc_context_reset_process_msg(struct intel_guc *guc,
 517                                         const u32 *msg, u32 len);
 518 int intel_guc_engine_failure_process_msg(struct intel_guc *guc,
 519                                          const u32 *msg, u32 len);
 520 int intel_guc_error_capture_process_msg(struct intel_guc *guc,
 521                                         const u32 *msg, u32 len);
 522 int intel_guc_crash_process_msg(struct intel_guc *guc, u32 action);
 523
 524 struct intel_engine_cs *
 525 intel_guc_lookup_engine(struct intel_guc *guc, u8 guc_class, u8 instance);
 526
 527 void intel_guc_find_hung_context(struct intel_engine_cs *engine);
 528
 529 int intel_guc_global_policies_update(struct intel_guc *guc);
 530
 531 void intel_guc_context_ban(struct intel_context *ce, struct i915_request *rq);
 532
 533 void intel_guc_submission_reset_prepare(struct intel_guc *guc);
 534 void intel_guc_submission_reset(struct intel_guc *guc, intel_engine_mask_t stalled);
 535 void intel_guc_submission_reset_finish(struct intel_guc *guc);
 536 void intel_guc_submission_cancel_requests(struct intel_guc *guc);
 537
 538 void intel_guc_load_status(struct intel_guc *guc, struct drm_printer *p);
 539
 540 void intel_guc_write_barrier(struct intel_guc *guc);
 541
 542 void intel_guc_dump_time_info(struct intel_guc *guc, struct drm_printer *p);
 543
 544 int intel_guc_sched_disable_gucid_threshold_max(struct intel_guc *guc);
 545
 546 bool intel_guc_tlb_invalidation_is_available(struct intel_guc *guc);
 547 int intel_guc_invalidate_tlb_engines(struct intel_guc *guc);
 548 int intel_guc_invalidate_tlb_guc(struct intel_guc *guc);
 549 int intel_guc_tlb_invalidation_done(struct intel_guc *guc,
 550                                     const u32 *payload, u32 len);
 551 void wake_up_all_tlb_invalidate(struct intel_guc *guc);
 552 #endif