drivers/gpu/drm/i915/i915_pci.c

   1 /*
   2  * Copyright © 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <drm/drm_color_mgmt.h>
  26 #include <drm/drm_drv.h>
  27 #include <drm/i915_pciids.h>
  28
  29 #include "display/intel_display.h"
  30 #include "display/intel_display_driver.h"
  31 #include "gt/intel_gt_regs.h"
  32 #include "gt/intel_sa_media.h"
  33 #include "gem/i915_gem_object_types.h"
  34
  35 #include "i915_driver.h"
  36 #include "i915_drv.h"
  37 #include "i915_pci.h"
  38 #include "i915_reg.h"
  39 #include "intel_pci_config.h"
  40
  41 __diag_push();
  42 __diag_ignore_all("-Woverride-init", "Allow overriding inherited members");
  43
  44 #define PLATFORM(x) .platform = (x)
  45 #define GEN(x) \
  46         .__runtime.graphics.ip.ver = (x), \
  47         .__runtime.media.ip.ver = (x)
  48
  49 #define LEGACY_CACHELEVEL \
  50         .cachelevel_to_pat = { \
  51                 [I915_CACHE_NONE]   = 0, \
  52                 [I915_CACHE_LLC]    = 1, \
  53                 [I915_CACHE_L3_LLC] = 2, \
  54                 [I915_CACHE_WT]     = 3, \
  55         }
  56
  57 #define TGL_CACHELEVEL \
  58         .cachelevel_to_pat = { \
  59                 [I915_CACHE_NONE]   = 3, \
  60                 [I915_CACHE_LLC]    = 0, \
  61                 [I915_CACHE_L3_LLC] = 0, \
  62                 [I915_CACHE_WT]     = 2, \
  63         }
  64
  65 #define PVC_CACHELEVEL \
  66         .cachelevel_to_pat = { \
  67                 [I915_CACHE_NONE]   = 0, \
  68                 [I915_CACHE_LLC]    = 3, \
  69                 [I915_CACHE_L3_LLC] = 3, \
  70                 [I915_CACHE_WT]     = 2, \
  71         }
  72
  73 #define MTL_CACHELEVEL \
  74         .cachelevel_to_pat = { \
  75                 [I915_CACHE_NONE]   = 2, \
  76                 [I915_CACHE_LLC]    = 3, \
  77                 [I915_CACHE_L3_LLC] = 3, \
  78                 [I915_CACHE_WT]     = 1, \
  79         }
  80
  81 /* Keep in gen based order, and chronological order within a gen */
  82
  83 #define GEN_DEFAULT_PAGE_SIZES \
  84         .__runtime.page_sizes = I915_GTT_PAGE_SIZE_4K
  85
  86 #define GEN_DEFAULT_REGIONS \
  87         .__runtime.memory_regions = REGION_SMEM | REGION_STOLEN_SMEM
  88
  89 #define I830_FEATURES \
  90         GEN(2), \
  91         .is_mobile = 1, \
  92         .gpu_reset_clobbers_display = true, \
  93         .has_3d_pipeline = 1, \
  94         .hws_needs_physical = 1, \
  95         .unfenced_needs_alignment = 1, \
  96         .__runtime.platform_engine_mask = BIT(RCS0), \
  97         .has_snoop = true, \
  98         .has_coherent_ggtt = false, \
  99         .dma_mask_size = 32, \
 100         .max_pat_index = 3, \
 101         GEN_DEFAULT_PAGE_SIZES, \
 102         GEN_DEFAULT_REGIONS, \
 103         LEGACY_CACHELEVEL
 104
 105 #define I845_FEATURES \
 106         GEN(2), \
 107         .has_3d_pipeline = 1, \
 108         .gpu_reset_clobbers_display = true, \
 109         .hws_needs_physical = 1, \
 110         .unfenced_needs_alignment = 1, \
 111         .__runtime.platform_engine_mask = BIT(RCS0), \
 112         .has_snoop = true, \
 113         .has_coherent_ggtt = false, \
 114         .dma_mask_size = 32, \
 115         .max_pat_index = 3, \
 116         GEN_DEFAULT_PAGE_SIZES, \
 117         GEN_DEFAULT_REGIONS, \
 118         LEGACY_CACHELEVEL
 119
 120 static const struct intel_device_info i830_info = {
 121         I830_FEATURES,
 122         PLATFORM(INTEL_I830),
 123 };
 124
 125 static const struct intel_device_info i845g_info = {
 126         I845_FEATURES,
 127         PLATFORM(INTEL_I845G),
 128 };
 129
 130 static const struct intel_device_info i85x_info = {
 131         I830_FEATURES,
 132         PLATFORM(INTEL_I85X),
 133 };
 134
 135 static const struct intel_device_info i865g_info = {
 136         I845_FEATURES,
 137         PLATFORM(INTEL_I865G),
 138 };
 139
 140 #define GEN3_FEATURES \
 141         GEN(3), \
 142         .gpu_reset_clobbers_display = true, \
 143         .__runtime.platform_engine_mask = BIT(RCS0), \
 144         .has_3d_pipeline = 1, \
 145         .has_snoop = true, \
 146         .has_coherent_ggtt = true, \
 147         .dma_mask_size = 32, \
 148         .max_pat_index = 3, \
 149         GEN_DEFAULT_PAGE_SIZES, \
 150         GEN_DEFAULT_REGIONS, \
 151         LEGACY_CACHELEVEL
 152
 153 static const struct intel_device_info i915g_info = {
 154         GEN3_FEATURES,
 155         PLATFORM(INTEL_I915G),
 156         .has_coherent_ggtt = false,
 157         .hws_needs_physical = 1,
 158         .unfenced_needs_alignment = 1,
 159 };
 160
 161 static const struct intel_device_info i915gm_info = {
 162         GEN3_FEATURES,
 163         PLATFORM(INTEL_I915GM),
 164         .is_mobile = 1,
 165         .hws_needs_physical = 1,
 166         .unfenced_needs_alignment = 1,
 167 };
 168
 169 static const struct intel_device_info i945g_info = {
 170         GEN3_FEATURES,
 171         PLATFORM(INTEL_I945G),
 172         .hws_needs_physical = 1,
 173         .unfenced_needs_alignment = 1,
 174 };
 175
 176 static const struct intel_device_info i945gm_info = {
 177         GEN3_FEATURES,
 178         PLATFORM(INTEL_I945GM),
 179         .is_mobile = 1,
 180         .hws_needs_physical = 1,
 181         .unfenced_needs_alignment = 1,
 182 };
 183
 184 static const struct intel_device_info g33_info = {
 185         GEN3_FEATURES,
 186         PLATFORM(INTEL_G33),
 187         .dma_mask_size = 36,
 188 };
 189
 190 static const struct intel_device_info pnv_g_info = {
 191         GEN3_FEATURES,
 192         PLATFORM(INTEL_PINEVIEW),
 193         .dma_mask_size = 36,
 194 };
 195
 196 static const struct intel_device_info pnv_m_info = {
 197         GEN3_FEATURES,
 198         PLATFORM(INTEL_PINEVIEW),
 199         .is_mobile = 1,
 200         .dma_mask_size = 36,
 201 };
 202
 203 #define GEN4_FEATURES \
 204         GEN(4), \
 205         .gpu_reset_clobbers_display = true, \
 206         .__runtime.platform_engine_mask = BIT(RCS0), \
 207         .has_3d_pipeline = 1, \
 208         .has_snoop = true, \
 209         .has_coherent_ggtt = true, \
 210         .dma_mask_size = 36, \
 211         .max_pat_index = 3, \
 212         GEN_DEFAULT_PAGE_SIZES, \
 213         GEN_DEFAULT_REGIONS, \
 214         LEGACY_CACHELEVEL
 215
 216 static const struct intel_device_info i965g_info = {
 217         GEN4_FEATURES,
 218         PLATFORM(INTEL_I965G),
 219         .hws_needs_physical = 1,
 220         .has_snoop = false,
 221 };
 222
 223 static const struct intel_device_info i965gm_info = {
 224         GEN4_FEATURES,
 225         PLATFORM(INTEL_I965GM),
 226         .is_mobile = 1,
 227         .hws_needs_physical = 1,
 228         .has_snoop = false,
 229 };
 230
 231 static const struct intel_device_info g45_info = {
 232         GEN4_FEATURES,
 233         PLATFORM(INTEL_G45),
 234         .__runtime.platform_engine_mask = BIT(RCS0) | BIT(VCS0),
 235         .gpu_reset_clobbers_display = false,
 236 };
 237
 238 static const struct intel_device_info gm45_info = {
 239         GEN4_FEATURES,
 240         PLATFORM(INTEL_GM45),
 241         .is_mobile = 1,
 242         .__runtime.platform_engine_mask = BIT(RCS0) | BIT(VCS0),
 243         .gpu_reset_clobbers_display = false,
 244 };
 245
 246 #define GEN5_FEATURES \
 247         GEN(5), \
 248         .__runtime.platform_engine_mask = BIT(RCS0) | BIT(VCS0), \
 249         .has_3d_pipeline = 1, \
 250         .has_snoop = true, \
 251         .has_coherent_ggtt = true, \
 252         /* ilk does support rc6, but we do not implement [power] contexts */ \
 253         .has_rc6 = 0, \
 254         .dma_mask_size = 36, \
 255         .max_pat_index = 3, \
 256         GEN_DEFAULT_PAGE_SIZES, \
 257         GEN_DEFAULT_REGIONS, \
 258         LEGACY_CACHELEVEL
 259
 260 static const struct intel_device_info ilk_d_info = {
 261         GEN5_FEATURES,
 262         PLATFORM(INTEL_IRONLAKE),
 263 };
 264
 265 static const struct intel_device_info ilk_m_info = {
 266         GEN5_FEATURES,
 267         PLATFORM(INTEL_IRONLAKE),
 268         .is_mobile = 1,
 269         .has_rps = true,
 270 };
 271
 272 #define GEN6_FEATURES \
 273         GEN(6), \
 274         .__runtime.platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0), \
 275         .has_3d_pipeline = 1, \
 276         .has_coherent_ggtt = true, \
 277         .has_llc = 1, \
 278         .has_rc6 = 1, \
 279         /* snb does support rc6p, but enabling it causes various issues */ \
 280         .has_rc6p = 0, \
 281         .has_rps = true, \
 282         .dma_mask_size = 40, \
 283         .max_pat_index = 3, \
 284         .__runtime.ppgtt_type = INTEL_PPGTT_ALIASING, \
 285         .__runtime.ppgtt_size = 31, \
 286         GEN_DEFAULT_PAGE_SIZES, \
 287         GEN_DEFAULT_REGIONS, \
 288         LEGACY_CACHELEVEL
 289
 290 #define SNB_D_PLATFORM \
 291         GEN6_FEATURES, \
 292         PLATFORM(INTEL_SANDYBRIDGE)
 293
 294 static const struct intel_device_info snb_d_gt1_info = {
 295         SNB_D_PLATFORM,
 296         .gt = 1,
 297 };
 298
 299 static const struct intel_device_info snb_d_gt2_info = {
 300         SNB_D_PLATFORM,
 301         .gt = 2,
 302 };
 303
 304 #define SNB_M_PLATFORM \
 305         GEN6_FEATURES, \
 306         PLATFORM(INTEL_SANDYBRIDGE), \
 307         .is_mobile = 1
 308
 309
 310 static const struct intel_device_info snb_m_gt1_info = {
 311         SNB_M_PLATFORM,
 312         .gt = 1,
 313 };
 314
 315 static const struct intel_device_info snb_m_gt2_info = {
 316         SNB_M_PLATFORM,
 317         .gt = 2,
 318 };
 319
 320 #define GEN7_FEATURES  \
 321         GEN(7), \
 322         .__runtime.platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0), \
 323         .has_3d_pipeline = 1, \
 324         .has_coherent_ggtt = true, \
 325         .has_llc = 1, \
 326         .has_rc6 = 1, \
 327         .has_rc6p = 1, \
 328         .has_reset_engine = true, \
 329         .has_rps = true, \
 330         .dma_mask_size = 40, \
 331         .max_pat_index = 3, \
 332         .__runtime.ppgtt_type = INTEL_PPGTT_ALIASING, \
 333         .__runtime.ppgtt_size = 31, \
 334         GEN_DEFAULT_PAGE_SIZES, \
 335         GEN_DEFAULT_REGIONS, \
 336         LEGACY_CACHELEVEL
 337
 338 #define IVB_D_PLATFORM \
 339         GEN7_FEATURES, \
 340         PLATFORM(INTEL_IVYBRIDGE), \
 341         .has_l3_dpf = 1
 342
 343 static const struct intel_device_info ivb_d_gt1_info = {
 344         IVB_D_PLATFORM,
 345         .gt = 1,
 346 };
 347
 348 static const struct intel_device_info ivb_d_gt2_info = {
 349         IVB_D_PLATFORM,
 350         .gt = 2,
 351 };
 352
 353 #define IVB_M_PLATFORM \
 354         GEN7_FEATURES, \
 355         PLATFORM(INTEL_IVYBRIDGE), \
 356         .is_mobile = 1, \
 357         .has_l3_dpf = 1
 358
 359 static const struct intel_device_info ivb_m_gt1_info = {
 360         IVB_M_PLATFORM,
 361         .gt = 1,
 362 };
 363
 364 static const struct intel_device_info ivb_m_gt2_info = {
 365         IVB_M_PLATFORM,
 366         .gt = 2,
 367 };
 368
 369 static const struct intel_device_info ivb_q_info = {
 370         GEN7_FEATURES,
 371         PLATFORM(INTEL_IVYBRIDGE),
 372         .gt = 2,
 373         .has_l3_dpf = 1,
 374 };
 375
 376 static const struct intel_device_info vlv_info = {
 377         PLATFORM(INTEL_VALLEYVIEW),
 378         GEN(7),
 379         .is_lp = 1,
 380         .has_runtime_pm = 1,
 381         .has_rc6 = 1,
 382         .has_reset_engine = true,
 383         .has_rps = true,
 384         .dma_mask_size = 40,
 385         .max_pat_index = 3,
 386         .__runtime.ppgtt_type = INTEL_PPGTT_ALIASING,
 387         .__runtime.ppgtt_size = 31,
 388         .has_snoop = true,
 389         .has_coherent_ggtt = false,
 390         .__runtime.platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0),
 391         GEN_DEFAULT_PAGE_SIZES,
 392         GEN_DEFAULT_REGIONS,
 393         LEGACY_CACHELEVEL,
 394 };
 395
 396 #define G75_FEATURES  \
 397         GEN7_FEATURES, \
 398         .__runtime.platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0), \
 399         .has_rc6p = 0 /* RC6p removed-by HSW */, \
 400         .has_runtime_pm = 1
 401
 402 #define HSW_PLATFORM \
 403         G75_FEATURES, \
 404         PLATFORM(INTEL_HASWELL), \
 405         .has_l3_dpf = 1
 406
 407 static const struct intel_device_info hsw_gt1_info = {
 408         HSW_PLATFORM,
 409         .gt = 1,
 410 };
 411
 412 static const struct intel_device_info hsw_gt2_info = {
 413         HSW_PLATFORM,
 414         .gt = 2,
 415 };
 416
 417 static const struct intel_device_info hsw_gt3_info = {
 418         HSW_PLATFORM,
 419         .gt = 3,
 420 };
 421
 422 #define GEN8_FEATURES \
 423         G75_FEATURES, \
 424         GEN(8), \
 425         .has_logical_ring_contexts = 1, \
 426         .dma_mask_size = 39, \
 427         .__runtime.ppgtt_type = INTEL_PPGTT_FULL, \
 428         .__runtime.ppgtt_size = 48, \
 429         .has_64bit_reloc = 1
 430
 431 #define BDW_PLATFORM \
 432         GEN8_FEATURES, \
 433         PLATFORM(INTEL_BROADWELL)
 434
 435 static const struct intel_device_info bdw_gt1_info = {
 436         BDW_PLATFORM,
 437         .gt = 1,
 438 };
 439
 440 static const struct intel_device_info bdw_gt2_info = {
 441         BDW_PLATFORM,
 442         .gt = 2,
 443 };
 444
 445 static const struct intel_device_info bdw_rsvd_info = {
 446         BDW_PLATFORM,
 447         .gt = 3,
 448         /* According to the device ID those devices are GT3, they were
 449          * previously treated as not GT3, keep it like that.
 450          */
 451 };
 452
 453 static const struct intel_device_info bdw_gt3_info = {
 454         BDW_PLATFORM,
 455         .gt = 3,
 456         .__runtime.platform_engine_mask =
 457                 BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS1),
 458 };
 459
 460 static const struct intel_device_info chv_info = {
 461         PLATFORM(INTEL_CHERRYVIEW),
 462         GEN(8),
 463         .is_lp = 1,
 464         .__runtime.platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0),
 465         .has_64bit_reloc = 1,
 466         .has_runtime_pm = 1,
 467         .has_rc6 = 1,
 468         .has_rps = true,
 469         .has_logical_ring_contexts = 1,
 470         .dma_mask_size = 39,
 471         .max_pat_index = 3,
 472         .__runtime.ppgtt_type = INTEL_PPGTT_FULL,
 473         .__runtime.ppgtt_size = 32,
 474         .has_reset_engine = 1,
 475         .has_snoop = true,
 476         .has_coherent_ggtt = false,
 477         GEN_DEFAULT_PAGE_SIZES,
 478         GEN_DEFAULT_REGIONS,
 479         LEGACY_CACHELEVEL,
 480 };
 481
 482 #define GEN9_DEFAULT_PAGE_SIZES \
 483         .__runtime.page_sizes = I915_GTT_PAGE_SIZE_4K | \
 484                 I915_GTT_PAGE_SIZE_64K
 485
 486 #define GEN9_FEATURES \
 487         GEN8_FEATURES, \
 488         GEN(9), \
 489         GEN9_DEFAULT_PAGE_SIZES, \
 490         .has_gt_uc = 1
 491
 492 #define SKL_PLATFORM \
 493         GEN9_FEATURES, \
 494         PLATFORM(INTEL_SKYLAKE)
 495
 496 static const struct intel_device_info skl_gt1_info = {
 497         SKL_PLATFORM,
 498         .gt = 1,
 499 };
 500
 501 static const struct intel_device_info skl_gt2_info = {
 502         SKL_PLATFORM,
 503         .gt = 2,
 504 };
 505
 506 #define SKL_GT3_PLUS_PLATFORM \
 507         SKL_PLATFORM, \
 508         .__runtime.platform_engine_mask = \
 509                 BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS1)
 510
 511
 512 static const struct intel_device_info skl_gt3_info = {
 513         SKL_GT3_PLUS_PLATFORM,
 514         .gt = 3,
 515 };
 516
 517 static const struct intel_device_info skl_gt4_info = {
 518         SKL_GT3_PLUS_PLATFORM,
 519         .gt = 4,
 520 };
 521
 522 #define GEN9_LP_FEATURES \
 523         GEN(9), \
 524         .is_lp = 1, \
 525         .__runtime.platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0), \
 526         .has_3d_pipeline = 1, \
 527         .has_64bit_reloc = 1, \
 528         .has_runtime_pm = 1, \
 529         .has_rc6 = 1, \
 530         .has_rps = true, \
 531         .has_logical_ring_contexts = 1, \
 532         .has_gt_uc = 1, \
 533         .dma_mask_size = 39, \
 534         .__runtime.ppgtt_type = INTEL_PPGTT_FULL, \
 535         .__runtime.ppgtt_size = 48, \
 536         .has_reset_engine = 1, \
 537         .has_snoop = true, \
 538         .has_coherent_ggtt = false, \
 539         .max_pat_index = 3, \
 540         GEN9_DEFAULT_PAGE_SIZES, \
 541         GEN_DEFAULT_REGIONS, \
 542         LEGACY_CACHELEVEL
 543
 544 static const struct intel_device_info bxt_info = {
 545         GEN9_LP_FEATURES,
 546         PLATFORM(INTEL_BROXTON),
 547 };
 548
 549 static const struct intel_device_info glk_info = {
 550         GEN9_LP_FEATURES,
 551         PLATFORM(INTEL_GEMINILAKE),
 552 };
 553
 554 #define KBL_PLATFORM \
 555         GEN9_FEATURES, \
 556         PLATFORM(INTEL_KABYLAKE)
 557
 558 static const struct intel_device_info kbl_gt1_info = {
 559         KBL_PLATFORM,
 560         .gt = 1,
 561 };
 562
 563 static const struct intel_device_info kbl_gt2_info = {
 564         KBL_PLATFORM,
 565         .gt = 2,
 566 };
 567
 568 static const struct intel_device_info kbl_gt3_info = {
 569         KBL_PLATFORM,
 570         .gt = 3,
 571         .__runtime.platform_engine_mask =
 572                 BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS1),
 573 };
 574
 575 #define CFL_PLATFORM \
 576         GEN9_FEATURES, \
 577         PLATFORM(INTEL_COFFEELAKE)
 578
 579 static const struct intel_device_info cfl_gt1_info = {
 580         CFL_PLATFORM,
 581         .gt = 1,
 582 };
 583
 584 static const struct intel_device_info cfl_gt2_info = {
 585         CFL_PLATFORM,
 586         .gt = 2,
 587 };
 588
 589 static const struct intel_device_info cfl_gt3_info = {
 590         CFL_PLATFORM,
 591         .gt = 3,
 592         .__runtime.platform_engine_mask =
 593                 BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS1),
 594 };
 595
 596 #define CML_PLATFORM \
 597         GEN9_FEATURES, \
 598         PLATFORM(INTEL_COMETLAKE)
 599
 600 static const struct intel_device_info cml_gt1_info = {
 601         CML_PLATFORM,
 602         .gt = 1,
 603 };
 604
 605 static const struct intel_device_info cml_gt2_info = {
 606         CML_PLATFORM,
 607         .gt = 2,
 608 };
 609
 610 #define GEN11_DEFAULT_PAGE_SIZES \
 611         .__runtime.page_sizes = I915_GTT_PAGE_SIZE_4K | \
 612                 I915_GTT_PAGE_SIZE_64K |                \
 613                 I915_GTT_PAGE_SIZE_2M
 614
 615 #define GEN11_FEATURES \
 616         GEN9_FEATURES, \
 617         GEN11_DEFAULT_PAGE_SIZES, \
 618         GEN(11), \
 619         .has_coherent_ggtt = false, \
 620         .has_logical_ring_elsq = 1
 621
 622 static const struct intel_device_info icl_info = {
 623         GEN11_FEATURES,
 624         PLATFORM(INTEL_ICELAKE),
 625         .__runtime.platform_engine_mask =
 626                 BIT(RCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS0) | BIT(VCS2),
 627 };
 628
 629 static const struct intel_device_info ehl_info = {
 630         GEN11_FEATURES,
 631         PLATFORM(INTEL_ELKHARTLAKE),
 632         .__runtime.platform_engine_mask = BIT(RCS0) | BIT(BCS0) | BIT(VCS0) | BIT(VECS0),
 633         .__runtime.ppgtt_size = 36,
 634 };
 635
 636 static const struct intel_device_info jsl_info = {
 637         GEN11_FEATURES,
 638         PLATFORM(INTEL_JASPERLAKE),
 639         .__runtime.platform_engine_mask = BIT(RCS0) | BIT(BCS0) | BIT(VCS0) | BIT(VECS0),
 640         .__runtime.ppgtt_size = 36,
 641 };
 642
 643 #define GEN12_FEATURES \
 644         GEN11_FEATURES, \
 645         GEN(12), \
 646         TGL_CACHELEVEL, \
 647         .has_global_mocs = 1, \
 648         .has_pxp = 1, \
 649         .max_pat_index = 3
 650
 651 static const struct intel_device_info tgl_info = {
 652         GEN12_FEATURES,
 653         PLATFORM(INTEL_TIGERLAKE),
 654         .__runtime.platform_engine_mask =
 655                 BIT(RCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS0) | BIT(VCS2),
 656 };
 657
 658 static const struct intel_device_info rkl_info = {
 659         GEN12_FEATURES,
 660         PLATFORM(INTEL_ROCKETLAKE),
 661         .__runtime.platform_engine_mask =
 662                 BIT(RCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS0),
 663 };
 664
 665 #define DGFX_FEATURES \
 666         .__runtime.memory_regions = REGION_SMEM | REGION_LMEM | REGION_STOLEN_LMEM, \
 667         .has_llc = 0, \
 668         .has_pxp = 0, \
 669         .has_snoop = 1, \
 670         .is_dgfx = 1, \
 671         .has_heci_gscfi = 1
 672
 673 static const struct intel_device_info dg1_info = {
 674         GEN12_FEATURES,
 675         DGFX_FEATURES,
 676         .__runtime.graphics.ip.rel = 10,
 677         PLATFORM(INTEL_DG1),
 678         .require_force_probe = 1,
 679         .__runtime.platform_engine_mask =
 680                 BIT(RCS0) | BIT(BCS0) | BIT(VECS0) |
 681                 BIT(VCS0) | BIT(VCS2),
 682         /* Wa_16011227922 */
 683         .__runtime.ppgtt_size = 47,
 684 };
 685
 686 static const struct intel_device_info adl_s_info = {
 687         GEN12_FEATURES,
 688         PLATFORM(INTEL_ALDERLAKE_S),
 689         .__runtime.platform_engine_mask =
 690                 BIT(RCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS0) | BIT(VCS2),
 691         .dma_mask_size = 39,
 692 };
 693
 694 static const struct intel_device_info adl_p_info = {
 695         GEN12_FEATURES,
 696         PLATFORM(INTEL_ALDERLAKE_P),
 697         .__runtime.platform_engine_mask =
 698                 BIT(RCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS0) | BIT(VCS2),
 699         .__runtime.ppgtt_size = 48,
 700         .dma_mask_size = 39,
 701 };
 702
 703 #undef GEN
 704
 705 #define XE_HP_PAGE_SIZES \
 706         .__runtime.page_sizes = I915_GTT_PAGE_SIZE_4K | \
 707                 I915_GTT_PAGE_SIZE_64K |                \
 708                 I915_GTT_PAGE_SIZE_2M
 709
 710 #define XE_HP_FEATURES \
 711         .__runtime.graphics.ip.ver = 12, \
 712         .__runtime.graphics.ip.rel = 50, \
 713         XE_HP_PAGE_SIZES, \
 714         TGL_CACHELEVEL, \
 715         .dma_mask_size = 46, \
 716         .has_3d_pipeline = 1, \
 717         .has_64bit_reloc = 1, \
 718         .has_flat_ccs = 1, \
 719         .has_4tile = 1, \
 720         .has_global_mocs = 1, \
 721         .has_gt_uc = 1, \
 722         .has_llc = 1, \
 723         .has_logical_ring_contexts = 1, \
 724         .has_logical_ring_elsq = 1, \
 725         .has_mslice_steering = 1, \
 726         .has_oa_bpc_reporting = 1, \
 727         .has_oa_slice_contrib_limits = 1, \
 728         .has_oam = 1, \
 729         .has_rc6 = 1, \
 730         .has_reset_engine = 1, \
 731         .has_rps = 1, \
 732         .has_runtime_pm = 1, \
 733         .max_pat_index = 3, \
 734         .__runtime.ppgtt_size = 48, \
 735         .__runtime.ppgtt_type = INTEL_PPGTT_FULL
 736
 737 #define XE_HPM_FEATURES \
 738         .__runtime.media.ip.ver = 12, \
 739         .__runtime.media.ip.rel = 50
 740
 741 __maybe_unused
 742 static const struct intel_device_info xehpsdv_info = {
 743         XE_HP_FEATURES,
 744         XE_HPM_FEATURES,
 745         DGFX_FEATURES,
 746         PLATFORM(INTEL_XEHPSDV),
 747         .has_64k_pages = 1,
 748         .has_media_ratio_mode = 1,
 749         .__runtime.platform_engine_mask =
 750                 BIT(RCS0) | BIT(BCS0) |
 751                 BIT(VECS0) | BIT(VECS1) | BIT(VECS2) | BIT(VECS3) |
 752                 BIT(VCS0) | BIT(VCS1) | BIT(VCS2) | BIT(VCS3) |
 753                 BIT(VCS4) | BIT(VCS5) | BIT(VCS6) | BIT(VCS7) |
 754                 BIT(CCS0) | BIT(CCS1) | BIT(CCS2) | BIT(CCS3),
 755         .require_force_probe = 1,
 756 };
 757
 758 #define DG2_FEATURES \
 759         XE_HP_FEATURES, \
 760         XE_HPM_FEATURES, \
 761         DGFX_FEATURES, \
 762         .__runtime.graphics.ip.rel = 55, \
 763         .__runtime.media.ip.rel = 55, \
 764         PLATFORM(INTEL_DG2), \
 765         .has_64k_pages = 1, \
 766         .has_guc_deprivilege = 1, \
 767         .has_heci_pxp = 1, \
 768         .has_media_ratio_mode = 1, \
 769         .__runtime.platform_engine_mask = \
 770                 BIT(RCS0) | BIT(BCS0) | \
 771                 BIT(VECS0) | BIT(VECS1) | \
 772                 BIT(VCS0) | BIT(VCS2) | \
 773                 BIT(CCS0) | BIT(CCS1) | BIT(CCS2) | BIT(CCS3)
 774
 775 static const struct intel_device_info dg2_info = {
 776         DG2_FEATURES,
 777 };
 778
 779 static const struct intel_device_info ats_m_info = {
 780         DG2_FEATURES,
 781         .require_force_probe = 1,
 782         .tuning_thread_rr_after_dep = 1,
 783 };
 784
 785 #define XE_HPC_FEATURES \
 786         XE_HP_FEATURES, \
 787         .dma_mask_size = 52, \
 788         .has_3d_pipeline = 0, \
 789         .has_guc_deprivilege = 1, \
 790         .has_l3_ccs_read = 1, \
 791         .has_mslice_steering = 0, \
 792         .has_one_eu_per_fuse_bit = 1
 793
 794 __maybe_unused
 795 static const struct intel_device_info pvc_info = {
 796         XE_HPC_FEATURES,
 797         XE_HPM_FEATURES,
 798         DGFX_FEATURES,
 799         .__runtime.graphics.ip.rel = 60,
 800         .__runtime.media.ip.rel = 60,
 801         PLATFORM(INTEL_PONTEVECCHIO),
 802         .has_flat_ccs = 0,
 803         .max_pat_index = 7,
 804         .__runtime.platform_engine_mask =
 805                 BIT(BCS0) |
 806                 BIT(VCS0) |
 807                 BIT(CCS0) | BIT(CCS1) | BIT(CCS2) | BIT(CCS3),
 808         .require_force_probe = 1,
 809         PVC_CACHELEVEL,
 810 };
 811
 812 static const struct intel_gt_definition xelpmp_extra_gt[] = {
 813         {
 814                 .type = GT_MEDIA,
 815                 .name = "Standalone Media GT",
 816                 .gsi_offset = MTL_MEDIA_GSI_BASE,
 817                 .engine_mask = BIT(VECS0) | BIT(VCS0) | BIT(VCS2) | BIT(GSC0),
 818         },
 819         {}
 820 };
 821
 822 static const struct intel_device_info mtl_info = {
 823         XE_HP_FEATURES,
 824         /*
 825          * Real graphics IP version will be obtained from hardware GMD_ID
 826          * register.  Value provided here is just for sanity checking.
 827          */
 828         .__runtime.graphics.ip.ver = 12,
 829         .__runtime.graphics.ip.rel = 70,
 830         .__runtime.media.ip.ver = 13,
 831         PLATFORM(INTEL_METEORLAKE),
 832         .extra_gt_list = xelpmp_extra_gt,
 833         .has_flat_ccs = 0,
 834         .has_gmd_id = 1,
 835         .has_guc_deprivilege = 1,
 836         .has_llc = 0,
 837         .has_mslice_steering = 0,
 838         .has_snoop = 1,
 839         .max_pat_index = 4,
 840         .has_pxp = 1,
 841         .__runtime.memory_regions = REGION_SMEM | REGION_STOLEN_LMEM,
 842         .__runtime.platform_engine_mask = BIT(RCS0) | BIT(BCS0) | BIT(CCS0),
 843         .require_force_probe = 1,
 844         MTL_CACHELEVEL,
 845 };
 846
 847 #undef PLATFORM
 848
 849 __diag_pop();
 850
 851 /*
 852  * Make sure any device matches here are from most specific to most
 853  * general.  For example, since the Quanta match is based on the subsystem
 854  * and subvendor IDs, we need it to come before the more general IVB
 855  * PCI ID matches, otherwise we'll use the wrong info struct above.
 856  */
 857 static const struct pci_device_id pciidlist[] = {
 858         INTEL_I830_IDS(&i830_info),
 859         INTEL_I845G_IDS(&i845g_info),
 860         INTEL_I85X_IDS(&i85x_info),
 861         INTEL_I865G_IDS(&i865g_info),
 862         INTEL_I915G_IDS(&i915g_info),
 863         INTEL_I915GM_IDS(&i915gm_info),
 864         INTEL_I945G_IDS(&i945g_info),
 865         INTEL_I945GM_IDS(&i945gm_info),
 866         INTEL_I965G_IDS(&i965g_info),
 867         INTEL_G33_IDS(&g33_info),
 868         INTEL_I965GM_IDS(&i965gm_info),
 869         INTEL_GM45_IDS(&gm45_info),
 870         INTEL_G45_IDS(&g45_info),
 871         INTEL_PINEVIEW_G_IDS(&pnv_g_info),
 872         INTEL_PINEVIEW_M_IDS(&pnv_m_info),
 873         INTEL_IRONLAKE_D_IDS(&ilk_d_info),
 874         INTEL_IRONLAKE_M_IDS(&ilk_m_info),
 875         INTEL_SNB_D_GT1_IDS(&snb_d_gt1_info),
 876         INTEL_SNB_D_GT2_IDS(&snb_d_gt2_info),
 877         INTEL_SNB_M_GT1_IDS(&snb_m_gt1_info),
 878         INTEL_SNB_M_GT2_IDS(&snb_m_gt2_info),
 879         INTEL_IVB_Q_IDS(&ivb_q_info), /* must be first IVB */
 880         INTEL_IVB_M_GT1_IDS(&ivb_m_gt1_info),
 881         INTEL_IVB_M_GT2_IDS(&ivb_m_gt2_info),
 882         INTEL_IVB_D_GT1_IDS(&ivb_d_gt1_info),
 883         INTEL_IVB_D_GT2_IDS(&ivb_d_gt2_info),
 884         INTEL_HSW_GT1_IDS(&hsw_gt1_info),
 885         INTEL_HSW_GT2_IDS(&hsw_gt2_info),
 886         INTEL_HSW_GT3_IDS(&hsw_gt3_info),
 887         INTEL_VLV_IDS(&vlv_info),
 888         INTEL_BDW_GT1_IDS(&bdw_gt1_info),
 889         INTEL_BDW_GT2_IDS(&bdw_gt2_info),
 890         INTEL_BDW_GT3_IDS(&bdw_gt3_info),
 891         INTEL_BDW_RSVD_IDS(&bdw_rsvd_info),
 892         INTEL_CHV_IDS(&chv_info),
 893         INTEL_SKL_GT1_IDS(&skl_gt1_info),
 894         INTEL_SKL_GT2_IDS(&skl_gt2_info),
 895         INTEL_SKL_GT3_IDS(&skl_gt3_info),
 896         INTEL_SKL_GT4_IDS(&skl_gt4_info),
 897         INTEL_BXT_IDS(&bxt_info),
 898         INTEL_GLK_IDS(&glk_info),
 899         INTEL_KBL_GT1_IDS(&kbl_gt1_info),
 900         INTEL_KBL_GT2_IDS(&kbl_gt2_info),
 901         INTEL_KBL_GT3_IDS(&kbl_gt3_info),
 902         INTEL_KBL_GT4_IDS(&kbl_gt3_info),
 903         INTEL_AML_KBL_GT2_IDS(&kbl_gt2_info),
 904         INTEL_CFL_S_GT1_IDS(&cfl_gt1_info),
 905         INTEL_CFL_S_GT2_IDS(&cfl_gt2_info),
 906         INTEL_CFL_H_GT1_IDS(&cfl_gt1_info),
 907         INTEL_CFL_H_GT2_IDS(&cfl_gt2_info),
 908         INTEL_CFL_U_GT2_IDS(&cfl_gt2_info),
 909         INTEL_CFL_U_GT3_IDS(&cfl_gt3_info),
 910         INTEL_WHL_U_GT1_IDS(&cfl_gt1_info),
 911         INTEL_WHL_U_GT2_IDS(&cfl_gt2_info),
 912         INTEL_AML_CFL_GT2_IDS(&cfl_gt2_info),
 913         INTEL_WHL_U_GT3_IDS(&cfl_gt3_info),
 914         INTEL_CML_GT1_IDS(&cml_gt1_info),
 915         INTEL_CML_GT2_IDS(&cml_gt2_info),
 916         INTEL_CML_U_GT1_IDS(&cml_gt1_info),
 917         INTEL_CML_U_GT2_IDS(&cml_gt2_info),
 918         INTEL_ICL_11_IDS(&icl_info),
 919         INTEL_EHL_IDS(&ehl_info),
 920         INTEL_JSL_IDS(&jsl_info),
 921         INTEL_TGL_12_IDS(&tgl_info),
 922         INTEL_RKL_IDS(&rkl_info),
 923         INTEL_ADLS_IDS(&adl_s_info),
 924         INTEL_ADLP_IDS(&adl_p_info),
 925         INTEL_ADLN_IDS(&adl_p_info),
 926         INTEL_DG1_IDS(&dg1_info),
 927         INTEL_RPLS_IDS(&adl_s_info),
 928         INTEL_RPLP_IDS(&adl_p_info),
 929         INTEL_DG2_IDS(&dg2_info),
 930         INTEL_ATS_M_IDS(&ats_m_info),
 931         INTEL_MTL_IDS(&mtl_info),
 932         {0, 0, 0}
 933 };
 934 MODULE_DEVICE_TABLE(pci, pciidlist);
 935
 936 static void i915_pci_remove(struct pci_dev *pdev)
 937 {
 938         struct drm_i915_private *i915;
 939
 940         i915 = pci_get_drvdata(pdev);
 941         if (!i915) /* driver load aborted, nothing to cleanup */
 942                 return;
 943
 944         i915_driver_remove(i915);
 945         pci_set_drvdata(pdev, NULL);
 946 }
 947
 948 /* is device_id present in comma separated list of ids */
 949 static bool device_id_in_list(u16 device_id, const char *devices, bool negative)
 950 {
 951         char *s, *p, *tok;
 952         bool ret;
 953
 954         if (!devices || !*devices)
 955                 return false;
 956
 957         /* match everything */
 958         if (negative && strcmp(devices, "!*") == 0)
 959                 return true;
 960         if (!negative && strcmp(devices, "*") == 0)
 961                 return true;
 962
 963         s = kstrdup(devices, GFP_KERNEL);
 964         if (!s)
 965                 return false;
 966
 967         for (p = s, ret = false; (tok = strsep(&p, ",")) != NULL; ) {
 968                 u16 val;
 969
 970                 if (negative && tok[0] == '!')
 971                         tok++;
 972                 else if ((negative && tok[0] != '!') ||
 973                          (!negative && tok[0] == '!'))
 974                         continue;
 975
 976                 if (kstrtou16(tok, 16, &val) == 0 && val == device_id) {
 977                         ret = true;
 978                         break;
 979                 }
 980         }
 981
 982         kfree(s);
 983
 984         return ret;
 985 }
 986
 987 static bool id_forced(u16 device_id)
 988 {
 989         return device_id_in_list(device_id, i915_modparams.force_probe, false);
 990 }
 991
 992 static bool id_blocked(u16 device_id)
 993 {
 994         return device_id_in_list(device_id, i915_modparams.force_probe, true);
 995 }
 996
 997 bool i915_pci_resource_valid(struct pci_dev *pdev, int bar)
 998 {
 999         if (!pci_resource_flags(pdev, bar))
1000                 return false;
1001
1002         if (pci_resource_flags(pdev, bar) & IORESOURCE_UNSET)
1003                 return false;
1004
1005         if (!pci_resource_len(pdev, bar))
1006                 return false;
1007
1008         return true;
1009 }
1010
1011 static bool intel_mmio_bar_valid(struct pci_dev *pdev, struct intel_device_info *intel_info)
1012 {
1013         return i915_pci_resource_valid(pdev, intel_mmio_bar(intel_info->__runtime.graphics.ip.ver));
1014 }
1015
1016 static int i915_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1017 {
1018         struct intel_device_info *intel_info =
1019                 (struct intel_device_info *) ent->driver_data;
1020         int err;
1021
1022         if (intel_info->require_force_probe && !id_forced(pdev->device)) {
1023                 dev_info(&pdev->dev,
1024                          "Your graphics device %04x is not properly supported by i915 in this\n"
1025                          "kernel version. To force driver probe anyway, use i915.force_probe=%04x\n"
1026                          "module parameter or CONFIG_DRM_I915_FORCE_PROBE=%04x configuration option,\n"
1027                          "or (recommended) check for kernel updates.\n",
1028                          pdev->device, pdev->device, pdev->device);
1029                 return -ENODEV;
1030         }
1031
1032         if (id_blocked(pdev->device)) {
1033                 dev_info(&pdev->dev, "I915 probe blocked for Device ID %04x.\n",
1034                          pdev->device);
1035                 return -ENODEV;
1036         }
1037
1038         if (intel_info->require_force_probe) {
1039                 dev_info(&pdev->dev, "Force probing unsupported Device ID %04x, tainting kernel\n",
1040                          pdev->device);
1041                 add_taint(TAINT_USER, LOCKDEP_STILL_OK);
1042         }
1043
1044         /* Only bind to function 0 of the device. Early generations
1045          * used function 1 as a placeholder for multi-head. This causes
1046          * us confusion instead, especially on the systems where both
1047          * functions have the same PCI-ID!
1048          */
1049         if (PCI_FUNC(pdev->devfn))
1050                 return -ENODEV;
1051
1052         if (!intel_mmio_bar_valid(pdev, intel_info))
1053                 return -ENXIO;
1054
1055         /* Detect if we need to wait for other drivers early on */
1056         if (intel_display_driver_probe_defer(pdev))
1057                 return -EPROBE_DEFER;
1058
1059         err = i915_driver_probe(pdev, ent);
1060         if (err)
1061                 return err;
1062
1063         if (i915_inject_probe_failure(pci_get_drvdata(pdev))) {
1064                 i915_pci_remove(pdev);
1065                 return -ENODEV;
1066         }
1067
1068         err = i915_live_selftests(pdev);
1069         if (err) {
1070                 i915_pci_remove(pdev);
1071                 return err > 0 ? -ENOTTY : err;
1072         }
1073
1074         err = i915_perf_selftests(pdev);
1075         if (err) {
1076                 i915_pci_remove(pdev);
1077                 return err > 0 ? -ENOTTY : err;
1078         }
1079
1080         return 0;
1081 }
1082
1083 static void i915_pci_shutdown(struct pci_dev *pdev)
1084 {
1085         struct drm_i915_private *i915 = pci_get_drvdata(pdev);
1086
1087         i915_driver_shutdown(i915);
1088 }
1089
1090 static struct pci_driver i915_pci_driver = {
1091         .name = DRIVER_NAME,
1092         .id_table = pciidlist,
1093         .probe = i915_pci_probe,
1094         .remove = i915_pci_remove,
1095         .shutdown = i915_pci_shutdown,
1096         .driver.pm = &i915_pm_ops,
1097 };
1098
1099 int i915_pci_register_driver(void)
1100 {
1101         return pci_register_driver(&i915_pci_driver);
1102 }
1103
1104 void i915_pci_unregister_driver(void)
1105 {
1106         pci_unregister_driver(&i915_pci_driver);
1107 }