drivers/irqchip/irq-gic-v3.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved.
   4  * Author: Marc Zyngier <[email protected]>
   5  */
   6
   7 #define pr_fmt(fmt)     "GICv3: " fmt
   8
   9 #include <linux/acpi.h>
  10 #include <linux/cpu.h>
  11 #include <linux/cpu_pm.h>
  12 #include <linux/delay.h>
  13 #include <linux/interrupt.h>
  14 #include <linux/irqdomain.h>
  15 #include <linux/kstrtox.h>
  16 #include <linux/of.h>
  17 #include <linux/of_address.h>
  18 #include <linux/of_irq.h>
  19 #include <linux/percpu.h>
  20 #include <linux/refcount.h>
  21 #include <linux/slab.h>
  22
  23 #include <linux/irqchip.h>
  24 #include <linux/irqchip/arm-gic-common.h>
  25 #include <linux/irqchip/arm-gic-v3.h>
  26 #include <linux/irqchip/irq-partition-percpu.h>
  27 #include <linux/bitfield.h>
  28 #include <linux/bits.h>
  29 #include <linux/arm-smccc.h>
  30
  31 #include <asm/cputype.h>
  32 #include <asm/exception.h>
  33 #include <asm/smp_plat.h>
  34 #include <asm/virt.h>
  35
  36 #include "irq-gic-common.h"
  37
  38 #define GICD_INT_NMI_PRI        (GICD_INT_DEF_PRI & ~0x80)
  39
  40 #define FLAGS_WORKAROUND_GICR_WAKER_MSM8996     (1ULL << 0)
  41 #define FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539   (1ULL << 1)
  42 #define FLAGS_WORKAROUND_ASR_ERRATUM_8601001    (1ULL << 2)
  43
  44 #define GIC_IRQ_TYPE_PARTITION  (GIC_IRQ_TYPE_LPI + 1)
  45
  46 struct redist_region {
  47         void __iomem            *redist_base;
  48         phys_addr_t             phys_base;
  49         bool                    single_redist;
  50 };
  51
  52 struct gic_chip_data {
  53         struct fwnode_handle    *fwnode;
  54         phys_addr_t             dist_phys_base;
  55         void __iomem            *dist_base;
  56         struct redist_region    *redist_regions;
  57         struct rdists           rdists;
  58         struct irq_domain       *domain;
  59         u64                     redist_stride;
  60         u32                     nr_redist_regions;
  61         u64                     flags;
  62         bool                    has_rss;
  63         unsigned int            ppi_nr;
  64         struct partition_desc   **ppi_descs;
  65 };
  66
  67 #define T241_CHIPS_MAX          4
  68 static void __iomem *t241_dist_base_alias[T241_CHIPS_MAX] __read_mostly;
  69 static DEFINE_STATIC_KEY_FALSE(gic_nvidia_t241_erratum);
  70
  71 static DEFINE_STATIC_KEY_FALSE(gic_arm64_2941627_erratum);
  72
  73 static struct gic_chip_data gic_data __read_mostly;
  74 static DEFINE_STATIC_KEY_TRUE(supports_deactivate_key);
  75
  76 #define GIC_ID_NR       (1U << GICD_TYPER_ID_BITS(gic_data.rdists.gicd_typer))
  77 #define GIC_LINE_NR     min(GICD_TYPER_SPIS(gic_data.rdists.gicd_typer), 1020U)
  78 #define GIC_ESPI_NR     GICD_TYPER_ESPIS(gic_data.rdists.gicd_typer)
  79
  80 /*
  81  * There are 16 SGIs, though we only actually use 8 in Linux. The other 8 SGIs
  82  * are potentially stolen by the secure side. Some code, especially code dealing
  83  * with hwirq IDs, is simplified by accounting for all 16.
  84  */
  85 #define SGI_NR          16
  86
  87 /*
  88  * The behaviours of RPR and PMR registers differ depending on the value of
  89  * SCR_EL3.FIQ, and the behaviour of non-secure priority registers of the
  90  * distributor and redistributors depends on whether security is enabled in the
  91  * GIC.
  92  *
  93  * When security is enabled, non-secure priority values from the (re)distributor
  94  * are presented to the GIC CPUIF as follow:
  95  *     (GIC_(R)DIST_PRI[irq] >> 1) | 0x80;
  96  *
  97  * If SCR_EL3.FIQ == 1, the values written to/read from PMR and RPR at non-secure
  98  * EL1 are subject to a similar operation thus matching the priorities presented
  99  * from the (re)distributor when security is enabled. When SCR_EL3.FIQ == 0,
 100  * these values are unchanged by the GIC.
 101  *
 102  * see GICv3/GICv4 Architecture Specification (IHI0069D):
 103  * - section 4.8.1 Non-secure accesses to register fields for Secure interrupt
 104  *   priorities.
 105  * - Figure 4-7 Secure read of the priority field for a Non-secure Group 1
 106  *   interrupt.
 107  */
 108 static DEFINE_STATIC_KEY_FALSE(supports_pseudo_nmis);
 109
 110 DEFINE_STATIC_KEY_FALSE(gic_nonsecure_priorities);
 111 EXPORT_SYMBOL(gic_nonsecure_priorities);
 112
 113 /*
 114  * When the Non-secure world has access to group 0 interrupts (as a
 115  * consequence of SCR_EL3.FIQ == 0), reading the ICC_RPR_EL1 register will
 116  * return the Distributor's view of the interrupt priority.
 117  *
 118  * When GIC security is enabled (GICD_CTLR.DS == 0), the interrupt priority
 119  * written by software is moved to the Non-secure range by the Distributor.
 120  *
 121  * If both are true (which is when gic_nonsecure_priorities gets enabled),
 122  * we need to shift down the priority programmed by software to match it
 123  * against the value returned by ICC_RPR_EL1.
 124  */
 125 #define GICD_INT_RPR_PRI(priority)                                      \
 126         ({                                                              \
 127                 u32 __priority = (priority);                            \
 128                 if (static_branch_unlikely(&gic_nonsecure_priorities))  \
 129                         __priority = 0x80 | (__priority >> 1);          \
 130                                                                         \
 131                 __priority;                                             \
 132         })
 133
 134 /* rdist_nmi_refs[n] == number of cpus having the rdist interrupt n set as NMI */
 135 static refcount_t *rdist_nmi_refs;
 136
 137 static struct gic_kvm_info gic_v3_kvm_info __initdata;
 138 static DEFINE_PER_CPU(bool, has_rss);
 139
 140 #define MPIDR_RS(mpidr)                 (((mpidr) & 0xF0UL) >> 4)
 141 #define gic_data_rdist()                (this_cpu_ptr(gic_data.rdists.rdist))
 142 #define gic_data_rdist_rd_base()        (gic_data_rdist()->rd_base)
 143 #define gic_data_rdist_sgi_base()       (gic_data_rdist_rd_base() + SZ_64K)
 144
 145 /* Our default, arbitrary priority value. Linux only uses one anyway. */
 146 #define DEFAULT_PMR_VALUE       0xf0
 147
 148 enum gic_intid_range {
 149         SGI_RANGE,
 150         PPI_RANGE,
 151         SPI_RANGE,
 152         EPPI_RANGE,
 153         ESPI_RANGE,
 154         LPI_RANGE,
 155         __INVALID_RANGE__
 156 };
 157
 158 static enum gic_intid_range __get_intid_range(irq_hw_number_t hwirq)
 159 {
 160         switch (hwirq) {
 161         case 0 ... 15:
 162                 return SGI_RANGE;
 163         case 16 ... 31:
 164                 return PPI_RANGE;
 165         case 32 ... 1019:
 166                 return SPI_RANGE;
 167         case EPPI_BASE_INTID ... (EPPI_BASE_INTID + 63):
 168                 return EPPI_RANGE;
 169         case ESPI_BASE_INTID ... (ESPI_BASE_INTID + 1023):
 170                 return ESPI_RANGE;
 171         case 8192 ... GENMASK(23, 0):
 172                 return LPI_RANGE;
 173         default:
 174                 return __INVALID_RANGE__;
 175         }
 176 }
 177
 178 static enum gic_intid_range get_intid_range(struct irq_data *d)
 179 {
 180         return __get_intid_range(d->hwirq);
 181 }
 182
 183 static inline unsigned int gic_irq(struct irq_data *d)
 184 {
 185         return d->hwirq;
 186 }
 187
 188 static inline bool gic_irq_in_rdist(struct irq_data *d)
 189 {
 190         switch (get_intid_range(d)) {
 191         case SGI_RANGE:
 192         case PPI_RANGE:
 193         case EPPI_RANGE:
 194                 return true;
 195         default:
 196                 return false;
 197         }
 198 }
 199
 200 static inline void __iomem *gic_dist_base_alias(struct irq_data *d)
 201 {
 202         if (static_branch_unlikely(&gic_nvidia_t241_erratum)) {
 203                 irq_hw_number_t hwirq = irqd_to_hwirq(d);
 204                 u32 chip;
 205
 206                 /*
 207                  * For the erratum T241-FABRIC-4, read accesses to GICD_In{E}
 208                  * registers are directed to the chip that owns the SPI. The
 209                  * the alias region can also be used for writes to the
 210                  * GICD_In{E} except GICD_ICENABLERn. Each chip has support
 211                  * for 320 {E}SPIs. Mappings for all 4 chips:
 212                  *    Chip0 = 32-351
 213                  *    Chip1 = 352-671
 214                  *    Chip2 = 672-991
 215                  *    Chip3 = 4096-4415
 216                  */
 217                 switch (__get_intid_range(hwirq)) {
 218                 case SPI_RANGE:
 219                         chip = (hwirq - 32) / 320;
 220                         break;
 221                 case ESPI_RANGE:
 222                         chip = 3;
 223                         break;
 224                 default:
 225                         unreachable();
 226                 }
 227                 return t241_dist_base_alias[chip];
 228         }
 229
 230         return gic_data.dist_base;
 231 }
 232
 233 static inline void __iomem *gic_dist_base(struct irq_data *d)
 234 {
 235         switch (get_intid_range(d)) {
 236         case SGI_RANGE:
 237         case PPI_RANGE:
 238         case EPPI_RANGE:
 239                 /* SGI+PPI -> SGI_base for this CPU */
 240                 return gic_data_rdist_sgi_base();
 241
 242         case SPI_RANGE:
 243         case ESPI_RANGE:
 244                 /* SPI -> dist_base */
 245                 return gic_data.dist_base;
 246
 247         default:
 248                 return NULL;
 249         }
 250 }
 251
 252 static void gic_do_wait_for_rwp(void __iomem *base, u32 bit)
 253 {
 254         u32 count = 1000000;    /* 1s! */
 255
 256         while (readl_relaxed(base + GICD_CTLR) & bit) {
 257                 count--;
 258                 if (!count) {
 259                         pr_err_ratelimited("RWP timeout, gone fishing\n");
 260                         return;
 261                 }
 262                 cpu_relax();
 263                 udelay(1);
 264         }
 265 }
 266
 267 /* Wait for completion of a distributor change */
 268 static void gic_dist_wait_for_rwp(void)
 269 {
 270         gic_do_wait_for_rwp(gic_data.dist_base, GICD_CTLR_RWP);
 271 }
 272
 273 /* Wait for completion of a redistributor change */
 274 static void gic_redist_wait_for_rwp(void)
 275 {
 276         gic_do_wait_for_rwp(gic_data_rdist_rd_base(), GICR_CTLR_RWP);
 277 }
 278
 279 static void gic_enable_redist(bool enable)
 280 {
 281         void __iomem *rbase;
 282         u32 count = 1000000;    /* 1s! */
 283         u32 val;
 284
 285         if (gic_data.flags & FLAGS_WORKAROUND_GICR_WAKER_MSM8996)
 286                 return;
 287
 288         rbase = gic_data_rdist_rd_base();
 289
 290         val = readl_relaxed(rbase + GICR_WAKER);
 291         if (enable)
 292                 /* Wake up this CPU redistributor */
 293                 val &= ~GICR_WAKER_ProcessorSleep;
 294         else
 295                 val |= GICR_WAKER_ProcessorSleep;
 296         writel_relaxed(val, rbase + GICR_WAKER);
 297
 298         if (!enable) {          /* Check that GICR_WAKER is writeable */
 299                 val = readl_relaxed(rbase + GICR_WAKER);
 300                 if (!(val & GICR_WAKER_ProcessorSleep))
 301                         return; /* No PM support in this redistributor */
 302         }
 303
 304         while (--count) {
 305                 val = readl_relaxed(rbase + GICR_WAKER);
 306                 if (enable ^ (bool)(val & GICR_WAKER_ChildrenAsleep))
 307                         break;
 308                 cpu_relax();
 309                 udelay(1);
 310         }
 311         if (!count)
 312                 pr_err_ratelimited("redistributor failed to %s...\n",
 313                                    enable ? "wakeup" : "sleep");
 314 }
 315
 316 /*
 317  * Routines to disable, enable, EOI and route interrupts
 318  */
 319 static u32 convert_offset_index(struct irq_data *d, u32 offset, u32 *index)
 320 {
 321         switch (get_intid_range(d)) {
 322         case SGI_RANGE:
 323         case PPI_RANGE:
 324         case SPI_RANGE:
 325                 *index = d->hwirq;
 326                 return offset;
 327         case EPPI_RANGE:
 328                 /*
 329                  * Contrary to the ESPI range, the EPPI range is contiguous
 330                  * to the PPI range in the registers, so let's adjust the
 331                  * displacement accordingly. Consistency is overrated.
 332                  */
 333                 *index = d->hwirq - EPPI_BASE_INTID + 32;
 334                 return offset;
 335         case ESPI_RANGE:
 336                 *index = d->hwirq - ESPI_BASE_INTID;
 337                 switch (offset) {
 338                 case GICD_ISENABLER:
 339                         return GICD_ISENABLERnE;
 340                 case GICD_ICENABLER:
 341                         return GICD_ICENABLERnE;
 342                 case GICD_ISPENDR:
 343                         return GICD_ISPENDRnE;
 344                 case GICD_ICPENDR:
 345                         return GICD_ICPENDRnE;
 346                 case GICD_ISACTIVER:
 347                         return GICD_ISACTIVERnE;
 348                 case GICD_ICACTIVER:
 349                         return GICD_ICACTIVERnE;
 350                 case GICD_IPRIORITYR:
 351                         return GICD_IPRIORITYRnE;
 352                 case GICD_ICFGR:
 353                         return GICD_ICFGRnE;
 354                 case GICD_IROUTER:
 355                         return GICD_IROUTERnE;
 356                 default:
 357                         break;
 358                 }
 359                 break;
 360         default:
 361                 break;
 362         }
 363
 364         WARN_ON(1);
 365         *index = d->hwirq;
 366         return offset;
 367 }
 368
 369 static int gic_peek_irq(struct irq_data *d, u32 offset)
 370 {
 371         void __iomem *base;
 372         u32 index, mask;
 373
 374         offset = convert_offset_index(d, offset, &index);
 375         mask = 1 << (index % 32);
 376
 377         if (gic_irq_in_rdist(d))
 378                 base = gic_data_rdist_sgi_base();
 379         else
 380                 base = gic_dist_base_alias(d);
 381
 382         return !!(readl_relaxed(base + offset + (index / 32) * 4) & mask);
 383 }
 384
 385 static void gic_poke_irq(struct irq_data *d, u32 offset)
 386 {
 387         void __iomem *base;
 388         u32 index, mask;
 389
 390         offset = convert_offset_index(d, offset, &index);
 391         mask = 1 << (index % 32);
 392
 393         if (gic_irq_in_rdist(d))
 394                 base = gic_data_rdist_sgi_base();
 395         else
 396                 base = gic_data.dist_base;
 397
 398         writel_relaxed(mask, base + offset + (index / 32) * 4);
 399 }
 400
 401 static void gic_mask_irq(struct irq_data *d)
 402 {
 403         gic_poke_irq(d, GICD_ICENABLER);
 404         if (gic_irq_in_rdist(d))
 405                 gic_redist_wait_for_rwp();
 406         else
 407                 gic_dist_wait_for_rwp();
 408 }
 409
 410 static void gic_eoimode1_mask_irq(struct irq_data *d)
 411 {
 412         gic_mask_irq(d);
 413         /*
 414          * When masking a forwarded interrupt, make sure it is
 415          * deactivated as well.
 416          *
 417          * This ensures that an interrupt that is getting
 418          * disabled/masked will not get "stuck", because there is
 419          * noone to deactivate it (guest is being terminated).
 420          */
 421         if (irqd_is_forwarded_to_vcpu(d))
 422                 gic_poke_irq(d, GICD_ICACTIVER);
 423 }
 424
 425 static void gic_unmask_irq(struct irq_data *d)
 426 {
 427         gic_poke_irq(d, GICD_ISENABLER);
 428 }
 429
 430 static inline bool gic_supports_nmi(void)
 431 {
 432         return IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) &&
 433                static_branch_likely(&supports_pseudo_nmis);
 434 }
 435
 436 static int gic_irq_set_irqchip_state(struct irq_data *d,
 437                                      enum irqchip_irq_state which, bool val)
 438 {
 439         u32 reg;
 440
 441         if (d->hwirq >= 8192) /* SGI/PPI/SPI only */
 442                 return -EINVAL;
 443
 444         switch (which) {
 445         case IRQCHIP_STATE_PENDING:
 446                 reg = val ? GICD_ISPENDR : GICD_ICPENDR;
 447                 break;
 448
 449         case IRQCHIP_STATE_ACTIVE:
 450                 reg = val ? GICD_ISACTIVER : GICD_ICACTIVER;
 451                 break;
 452
 453         case IRQCHIP_STATE_MASKED:
 454                 if (val) {
 455                         gic_mask_irq(d);
 456                         return 0;
 457                 }
 458                 reg = GICD_ISENABLER;
 459                 break;
 460
 461         default:
 462                 return -EINVAL;
 463         }
 464
 465         gic_poke_irq(d, reg);
 466         return 0;
 467 }
 468
 469 static int gic_irq_get_irqchip_state(struct irq_data *d,
 470                                      enum irqchip_irq_state which, bool *val)
 471 {
 472         if (d->hwirq >= 8192) /* PPI/SPI only */
 473                 return -EINVAL;
 474
 475         switch (which) {
 476         case IRQCHIP_STATE_PENDING:
 477                 *val = gic_peek_irq(d, GICD_ISPENDR);
 478                 break;
 479
 480         case IRQCHIP_STATE_ACTIVE:
 481                 *val = gic_peek_irq(d, GICD_ISACTIVER);
 482                 break;
 483
 484         case IRQCHIP_STATE_MASKED:
 485                 *val = !gic_peek_irq(d, GICD_ISENABLER);
 486                 break;
 487
 488         default:
 489                 return -EINVAL;
 490         }
 491
 492         return 0;
 493 }
 494
 495 static void gic_irq_set_prio(struct irq_data *d, u8 prio)
 496 {
 497         void __iomem *base = gic_dist_base(d);
 498         u32 offset, index;
 499
 500         offset = convert_offset_index(d, GICD_IPRIORITYR, &index);
 501
 502         writeb_relaxed(prio, base + offset + index);
 503 }
 504
 505 static u32 __gic_get_ppi_index(irq_hw_number_t hwirq)
 506 {
 507         switch (__get_intid_range(hwirq)) {
 508         case PPI_RANGE:
 509                 return hwirq - 16;
 510         case EPPI_RANGE:
 511                 return hwirq - EPPI_BASE_INTID + 16;
 512         default:
 513                 unreachable();
 514         }
 515 }
 516
 517 static u32 __gic_get_rdist_index(irq_hw_number_t hwirq)
 518 {
 519         switch (__get_intid_range(hwirq)) {
 520         case SGI_RANGE:
 521         case PPI_RANGE:
 522                 return hwirq;
 523         case EPPI_RANGE:
 524                 return hwirq - EPPI_BASE_INTID + 32;
 525         default:
 526                 unreachable();
 527         }
 528 }
 529
 530 static u32 gic_get_rdist_index(struct irq_data *d)
 531 {
 532         return __gic_get_rdist_index(d->hwirq);
 533 }
 534
 535 static int gic_irq_nmi_setup(struct irq_data *d)
 536 {
 537         struct irq_desc *desc = irq_to_desc(d->irq);
 538
 539         if (!gic_supports_nmi())
 540                 return -EINVAL;
 541
 542         if (gic_peek_irq(d, GICD_ISENABLER)) {
 543                 pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq);
 544                 return -EINVAL;
 545         }
 546
 547         /*
 548          * A secondary irq_chip should be in charge of LPI request,
 549          * it should not be possible to get there
 550          */
 551         if (WARN_ON(gic_irq(d) >= 8192))
 552                 return -EINVAL;
 553
 554         /* desc lock should already be held */
 555         if (gic_irq_in_rdist(d)) {
 556                 u32 idx = gic_get_rdist_index(d);
 557
 558                 /*
 559                  * Setting up a percpu interrupt as NMI, only switch handler
 560                  * for first NMI
 561                  */
 562                 if (!refcount_inc_not_zero(&rdist_nmi_refs[idx])) {
 563                         refcount_set(&rdist_nmi_refs[idx], 1);
 564                         desc->handle_irq = handle_percpu_devid_fasteoi_nmi;
 565                 }
 566         } else {
 567                 desc->handle_irq = handle_fasteoi_nmi;
 568         }
 569
 570         gic_irq_set_prio(d, GICD_INT_NMI_PRI);
 571
 572         return 0;
 573 }
 574
 575 static void gic_irq_nmi_teardown(struct irq_data *d)
 576 {
 577         struct irq_desc *desc = irq_to_desc(d->irq);
 578
 579         if (WARN_ON(!gic_supports_nmi()))
 580                 return;
 581
 582         if (gic_peek_irq(d, GICD_ISENABLER)) {
 583                 pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq);
 584                 return;
 585         }
 586
 587         /*
 588          * A secondary irq_chip should be in charge of LPI request,
 589          * it should not be possible to get there
 590          */
 591         if (WARN_ON(gic_irq(d) >= 8192))
 592                 return;
 593
 594         /* desc lock should already be held */
 595         if (gic_irq_in_rdist(d)) {
 596                 u32 idx = gic_get_rdist_index(d);
 597
 598                 /* Tearing down NMI, only switch handler for last NMI */
 599                 if (refcount_dec_and_test(&rdist_nmi_refs[idx]))
 600                         desc->handle_irq = handle_percpu_devid_irq;
 601         } else {
 602                 desc->handle_irq = handle_fasteoi_irq;
 603         }
 604
 605         gic_irq_set_prio(d, GICD_INT_DEF_PRI);
 606 }
 607
 608 static bool gic_arm64_erratum_2941627_needed(struct irq_data *d)
 609 {
 610         enum gic_intid_range range;
 611
 612         if (!static_branch_unlikely(&gic_arm64_2941627_erratum))
 613                 return false;
 614
 615         range = get_intid_range(d);
 616
 617         /*
 618          * The workaround is needed if the IRQ is an SPI and
 619          * the target cpu is different from the one we are
 620          * executing on.
 621          */
 622         return (range == SPI_RANGE || range == ESPI_RANGE) &&
 623                 !cpumask_test_cpu(raw_smp_processor_id(),
 624                                   irq_data_get_effective_affinity_mask(d));
 625 }
 626
 627 static void gic_eoi_irq(struct irq_data *d)
 628 {
 629         write_gicreg(gic_irq(d), ICC_EOIR1_EL1);
 630         isb();
 631
 632         if (gic_arm64_erratum_2941627_needed(d)) {
 633                 /*
 634                  * Make sure the GIC stream deactivate packet
 635                  * issued by ICC_EOIR1_EL1 has completed before
 636                  * deactivating through GICD_IACTIVER.
 637                  */
 638                 dsb(sy);
 639                 gic_poke_irq(d, GICD_ICACTIVER);
 640         }
 641 }
 642
 643 static void gic_eoimode1_eoi_irq(struct irq_data *d)
 644 {
 645         /*
 646          * No need to deactivate an LPI, or an interrupt that
 647          * is is getting forwarded to a vcpu.
 648          */
 649         if (gic_irq(d) >= 8192 || irqd_is_forwarded_to_vcpu(d))
 650                 return;
 651
 652         if (!gic_arm64_erratum_2941627_needed(d))
 653                 gic_write_dir(gic_irq(d));
 654         else
 655                 gic_poke_irq(d, GICD_ICACTIVER);
 656 }
 657
 658 static int gic_set_type(struct irq_data *d, unsigned int type)
 659 {
 660         enum gic_intid_range range;
 661         unsigned int irq = gic_irq(d);
 662         void __iomem *base;
 663         u32 offset, index;
 664         int ret;
 665
 666         range = get_intid_range(d);
 667
 668         /* Interrupt configuration for SGIs can't be changed */
 669         if (range == SGI_RANGE)
 670                 return type != IRQ_TYPE_EDGE_RISING ? -EINVAL : 0;
 671
 672         /* SPIs have restrictions on the supported types */
 673         if ((range == SPI_RANGE || range == ESPI_RANGE) &&
 674             type != IRQ_TYPE_LEVEL_HIGH && type != IRQ_TYPE_EDGE_RISING)
 675                 return -EINVAL;
 676
 677         if (gic_irq_in_rdist(d))
 678                 base = gic_data_rdist_sgi_base();
 679         else
 680                 base = gic_dist_base_alias(d);
 681
 682         offset = convert_offset_index(d, GICD_ICFGR, &index);
 683
 684         ret = gic_configure_irq(index, type, base + offset, NULL);
 685         if (ret && (range == PPI_RANGE || range == EPPI_RANGE)) {
 686                 /* Misconfigured PPIs are usually not fatal */
 687                 pr_warn("GIC: PPI INTID%d is secure or misconfigured\n", irq);
 688                 ret = 0;
 689         }
 690
 691         return ret;
 692 }
 693
 694 static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu)
 695 {
 696         if (get_intid_range(d) == SGI_RANGE)
 697                 return -EINVAL;
 698
 699         if (vcpu)
 700                 irqd_set_forwarded_to_vcpu(d);
 701         else
 702                 irqd_clr_forwarded_to_vcpu(d);
 703         return 0;
 704 }
 705
 706 static u64 gic_cpu_to_affinity(int cpu)
 707 {
 708         u64 mpidr = cpu_logical_map(cpu);
 709         u64 aff;
 710
 711         /* ASR8601 needs to have its affinities shifted down... */
 712         if (unlikely(gic_data.flags & FLAGS_WORKAROUND_ASR_ERRATUM_8601001))
 713                 mpidr = (MPIDR_AFFINITY_LEVEL(mpidr, 1) |
 714                          (MPIDR_AFFINITY_LEVEL(mpidr, 2) << 8));
 715
 716         aff = ((u64)MPIDR_AFFINITY_LEVEL(mpidr, 3) << 32 |
 717                MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
 718                MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8  |
 719                MPIDR_AFFINITY_LEVEL(mpidr, 0));
 720
 721         return aff;
 722 }
 723
 724 static void gic_deactivate_unhandled(u32 irqnr)
 725 {
 726         if (static_branch_likely(&supports_deactivate_key)) {
 727                 if (irqnr < 8192)
 728                         gic_write_dir(irqnr);
 729         } else {
 730                 write_gicreg(irqnr, ICC_EOIR1_EL1);
 731                 isb();
 732         }
 733 }
 734
 735 /*
 736  * Follow a read of the IAR with any HW maintenance that needs to happen prior
 737  * to invoking the relevant IRQ handler. We must do two things:
 738  *
 739  * (1) Ensure instruction ordering between a read of IAR and subsequent
 740  *     instructions in the IRQ handler using an ISB.
 741  *
 742  *     It is possible for the IAR to report an IRQ which was signalled *after*
 743  *     the CPU took an IRQ exception as multiple interrupts can race to be
 744  *     recognized by the GIC, earlier interrupts could be withdrawn, and/or
 745  *     later interrupts could be prioritized by the GIC.
 746  *
 747  *     For devices which are tightly coupled to the CPU, such as PMUs, a
 748  *     context synchronization event is necessary to ensure that system
 749  *     register state is not stale, as these may have been indirectly written
 750  *     *after* exception entry.
 751  *
 752  * (2) Deactivate the interrupt when EOI mode 1 is in use.
 753  */
 754 static inline void gic_complete_ack(u32 irqnr)
 755 {
 756         if (static_branch_likely(&supports_deactivate_key))
 757                 write_gicreg(irqnr, ICC_EOIR1_EL1);
 758
 759         isb();
 760 }
 761
 762 static bool gic_rpr_is_nmi_prio(void)
 763 {
 764         if (!gic_supports_nmi())
 765                 return false;
 766
 767         return unlikely(gic_read_rpr() == GICD_INT_RPR_PRI(GICD_INT_NMI_PRI));
 768 }
 769
 770 static bool gic_irqnr_is_special(u32 irqnr)
 771 {
 772         return irqnr >= 1020 && irqnr <= 1023;
 773 }
 774
 775 static void __gic_handle_irq(u32 irqnr, struct pt_regs *regs)
 776 {
 777         if (gic_irqnr_is_special(irqnr))
 778                 return;
 779
 780         gic_complete_ack(irqnr);
 781
 782         if (generic_handle_domain_irq(gic_data.domain, irqnr)) {
 783                 WARN_ONCE(true, "Unexpected interrupt (irqnr %u)\n", irqnr);
 784                 gic_deactivate_unhandled(irqnr);
 785         }
 786 }
 787
 788 static void __gic_handle_nmi(u32 irqnr, struct pt_regs *regs)
 789 {
 790         if (gic_irqnr_is_special(irqnr))
 791                 return;
 792
 793         gic_complete_ack(irqnr);
 794
 795         if (generic_handle_domain_nmi(gic_data.domain, irqnr)) {
 796                 WARN_ONCE(true, "Unexpected pseudo-NMI (irqnr %u)\n", irqnr);
 797                 gic_deactivate_unhandled(irqnr);
 798         }
 799 }
 800
 801 /*
 802  * An exception has been taken from a context with IRQs enabled, and this could
 803  * be an IRQ or an NMI.
 804  *
 805  * The entry code called us with DAIF.IF set to keep NMIs masked. We must clear
 806  * DAIF.IF (and update ICC_PMR_EL1 to mask regular IRQs) prior to returning,
 807  * after handling any NMI but before handling any IRQ.
 808  *
 809  * The entry code has performed IRQ entry, and if an NMI is detected we must
 810  * perform NMI entry/exit around invoking the handler.
 811  */
 812 static void __gic_handle_irq_from_irqson(struct pt_regs *regs)
 813 {
 814         bool is_nmi;
 815         u32 irqnr;
 816
 817         irqnr = gic_read_iar();
 818
 819         is_nmi = gic_rpr_is_nmi_prio();
 820
 821         if (is_nmi) {
 822                 nmi_enter();
 823                 __gic_handle_nmi(irqnr, regs);
 824                 nmi_exit();
 825         }
 826
 827         if (gic_prio_masking_enabled()) {
 828                 gic_pmr_mask_irqs();
 829                 gic_arch_enable_irqs();
 830         }
 831
 832         if (!is_nmi)
 833                 __gic_handle_irq(irqnr, regs);
 834 }
 835
 836 /*
 837  * An exception has been taken from a context with IRQs disabled, which can only
 838  * be an NMI.
 839  *
 840  * The entry code called us with DAIF.IF set to keep NMIs masked. We must leave
 841  * DAIF.IF (and ICC_PMR_EL1) unchanged.
 842  *
 843  * The entry code has performed NMI entry.
 844  */
 845 static void __gic_handle_irq_from_irqsoff(struct pt_regs *regs)
 846 {
 847         u64 pmr;
 848         u32 irqnr;
 849
 850         /*
 851          * We were in a context with IRQs disabled. However, the
 852          * entry code has set PMR to a value that allows any
 853          * interrupt to be acknowledged, and not just NMIs. This can
 854          * lead to surprising effects if the NMI has been retired in
 855          * the meantime, and that there is an IRQ pending. The IRQ
 856          * would then be taken in NMI context, something that nobody
 857          * wants to debug twice.
 858          *
 859          * Until we sort this, drop PMR again to a level that will
 860          * actually only allow NMIs before reading IAR, and then
 861          * restore it to what it was.
 862          */
 863         pmr = gic_read_pmr();
 864         gic_pmr_mask_irqs();
 865         isb();
 866         irqnr = gic_read_iar();
 867         gic_write_pmr(pmr);
 868
 869         __gic_handle_nmi(irqnr, regs);
 870 }
 871
 872 static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs)
 873 {
 874         if (unlikely(gic_supports_nmi() && !interrupts_enabled(regs)))
 875                 __gic_handle_irq_from_irqsoff(regs);
 876         else
 877                 __gic_handle_irq_from_irqson(regs);
 878 }
 879
 880 static u32 gic_get_pribits(void)
 881 {
 882         u32 pribits;
 883
 884         pribits = gic_read_ctlr();
 885         pribits &= ICC_CTLR_EL1_PRI_BITS_MASK;
 886         pribits >>= ICC_CTLR_EL1_PRI_BITS_SHIFT;
 887         pribits++;
 888
 889         return pribits;
 890 }
 891
 892 static bool gic_has_group0(void)
 893 {
 894         u32 val;
 895         u32 old_pmr;
 896
 897         old_pmr = gic_read_pmr();
 898
 899         /*
 900          * Let's find out if Group0 is under control of EL3 or not by
 901          * setting the highest possible, non-zero priority in PMR.
 902          *
 903          * If SCR_EL3.FIQ is set, the priority gets shifted down in
 904          * order for the CPU interface to set bit 7, and keep the
 905          * actual priority in the non-secure range. In the process, it
 906          * looses the least significant bit and the actual priority
 907          * becomes 0x80. Reading it back returns 0, indicating that
 908          * we're don't have access to Group0.
 909          */
 910         gic_write_pmr(BIT(8 - gic_get_pribits()));
 911         val = gic_read_pmr();
 912
 913         gic_write_pmr(old_pmr);
 914
 915         return val != 0;
 916 }
 917
 918 static void __init gic_dist_init(void)
 919 {
 920         unsigned int i;
 921         u64 affinity;
 922         void __iomem *base = gic_data.dist_base;
 923         u32 val;
 924
 925         /* Disable the distributor */
 926         writel_relaxed(0, base + GICD_CTLR);
 927         gic_dist_wait_for_rwp();
 928
 929         /*
 930          * Configure SPIs as non-secure Group-1. This will only matter
 931          * if the GIC only has a single security state. This will not
 932          * do the right thing if the kernel is running in secure mode,
 933          * but that's not the intended use case anyway.
 934          */
 935         for (i = 32; i < GIC_LINE_NR; i += 32)
 936                 writel_relaxed(~0, base + GICD_IGROUPR + i / 8);
 937
 938         /* Extended SPI range, not handled by the GICv2/GICv3 common code */
 939         for (i = 0; i < GIC_ESPI_NR; i += 32) {
 940                 writel_relaxed(~0U, base + GICD_ICENABLERnE + i / 8);
 941                 writel_relaxed(~0U, base + GICD_ICACTIVERnE + i / 8);
 942         }
 943
 944         for (i = 0; i < GIC_ESPI_NR; i += 32)
 945                 writel_relaxed(~0U, base + GICD_IGROUPRnE + i / 8);
 946
 947         for (i = 0; i < GIC_ESPI_NR; i += 16)
 948                 writel_relaxed(0, base + GICD_ICFGRnE + i / 4);
 949
 950         for (i = 0; i < GIC_ESPI_NR; i += 4)
 951                 writel_relaxed(GICD_INT_DEF_PRI_X4, base + GICD_IPRIORITYRnE + i);
 952
 953         /* Now do the common stuff */
 954         gic_dist_config(base, GIC_LINE_NR, NULL);
 955
 956         val = GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1;
 957         if (gic_data.rdists.gicd_typer2 & GICD_TYPER2_nASSGIcap) {
 958                 pr_info("Enabling SGIs without active state\n");
 959                 val |= GICD_CTLR_nASSGIreq;
 960         }
 961
 962         /* Enable distributor with ARE, Group1, and wait for it to drain */
 963         writel_relaxed(val, base + GICD_CTLR);
 964         gic_dist_wait_for_rwp();
 965
 966         /*
 967          * Set all global interrupts to the boot CPU only. ARE must be
 968          * enabled.
 969          */
 970         affinity = gic_cpu_to_affinity(smp_processor_id());
 971         for (i = 32; i < GIC_LINE_NR; i++)
 972                 gic_write_irouter(affinity, base + GICD_IROUTER + i * 8);
 973
 974         for (i = 0; i < GIC_ESPI_NR; i++)
 975                 gic_write_irouter(affinity, base + GICD_IROUTERnE + i * 8);
 976 }
 977
 978 static int gic_iterate_rdists(int (*fn)(struct redist_region *, void __iomem *))
 979 {
 980         int ret = -ENODEV;
 981         int i;
 982
 983         for (i = 0; i < gic_data.nr_redist_regions; i++) {
 984                 void __iomem *ptr = gic_data.redist_regions[i].redist_base;
 985                 u64 typer;
 986                 u32 reg;
 987
 988                 reg = readl_relaxed(ptr + GICR_PIDR2) & GIC_PIDR2_ARCH_MASK;
 989                 if (reg != GIC_PIDR2_ARCH_GICv3 &&
 990                     reg != GIC_PIDR2_ARCH_GICv4) { /* We're in trouble... */
 991                         pr_warn("No redistributor present @%p\n", ptr);
 992                         break;
 993                 }
 994
 995                 do {
 996                         typer = gic_read_typer(ptr + GICR_TYPER);
 997                         ret = fn(gic_data.redist_regions + i, ptr);
 998                         if (!ret)
 999                                 return 0;
1000
1001                         if (gic_data.redist_regions[i].single_redist)
1002                                 break;
1003
1004                         if (gic_data.redist_stride) {
1005                                 ptr += gic_data.redist_stride;
1006                         } else {
1007                                 ptr += SZ_64K * 2; /* Skip RD_base + SGI_base */
1008                                 if (typer & GICR_TYPER_VLPIS)
1009                                         ptr += SZ_64K * 2; /* Skip VLPI_base + reserved page */
1010                         }
1011                 } while (!(typer & GICR_TYPER_LAST));
1012         }
1013
1014         return ret ? -ENODEV : 0;
1015 }
1016
1017 static int __gic_populate_rdist(struct redist_region *region, void __iomem *ptr)
1018 {
1019         unsigned long mpidr;
1020         u64 typer;
1021         u32 aff;
1022
1023         /*
1024          * Convert affinity to a 32bit value that can be matched to
1025          * GICR_TYPER bits [63:32].
1026          */
1027         mpidr = gic_cpu_to_affinity(smp_processor_id());
1028
1029         aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 |
1030                MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
1031                MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 |
1032                MPIDR_AFFINITY_LEVEL(mpidr, 0));
1033
1034         typer = gic_read_typer(ptr + GICR_TYPER);
1035         if ((typer >> 32) == aff) {
1036                 u64 offset = ptr - region->redist_base;
1037                 raw_spin_lock_init(&gic_data_rdist()->rd_lock);
1038                 gic_data_rdist_rd_base() = ptr;
1039                 gic_data_rdist()->phys_base = region->phys_base + offset;
1040
1041                 pr_info("CPU%d: found redistributor %lx region %d:%pa\n",
1042                         smp_processor_id(), mpidr,
1043                         (int)(region - gic_data.redist_regions),
1044                         &gic_data_rdist()->phys_base);
1045                 return 0;
1046         }
1047
1048         /* Try next one */
1049         return 1;
1050 }
1051
1052 static int gic_populate_rdist(void)
1053 {
1054         if (gic_iterate_rdists(__gic_populate_rdist) == 0)
1055                 return 0;
1056
1057         /* We couldn't even deal with ourselves... */
1058         WARN(true, "CPU%d: mpidr %lx has no re-distributor!\n",
1059              smp_processor_id(),
1060              (unsigned long)cpu_logical_map(smp_processor_id()));
1061         return -ENODEV;
1062 }
1063
1064 static int __gic_update_rdist_properties(struct redist_region *region,
1065                                          void __iomem *ptr)
1066 {
1067         u64 typer = gic_read_typer(ptr + GICR_TYPER);
1068         u32 ctlr = readl_relaxed(ptr + GICR_CTLR);
1069
1070         /* Boot-time cleanup */
1071         if ((typer & GICR_TYPER_VLPIS) && (typer & GICR_TYPER_RVPEID)) {
1072                 u64 val;
1073
1074                 /* Deactivate any present vPE */
1075                 val = gicr_read_vpendbaser(ptr + SZ_128K + GICR_VPENDBASER);
1076                 if (val & GICR_VPENDBASER_Valid)
1077                         gicr_write_vpendbaser(GICR_VPENDBASER_PendingLast,
1078                                               ptr + SZ_128K + GICR_VPENDBASER);
1079
1080                 /* Mark the VPE table as invalid */
1081                 val = gicr_read_vpropbaser(ptr + SZ_128K + GICR_VPROPBASER);
1082                 val &= ~GICR_VPROPBASER_4_1_VALID;
1083                 gicr_write_vpropbaser(val, ptr + SZ_128K + GICR_VPROPBASER);
1084         }
1085
1086         gic_data.rdists.has_vlpis &= !!(typer & GICR_TYPER_VLPIS);
1087
1088         /*
1089          * TYPER.RVPEID implies some form of DirectLPI, no matter what the
1090          * doc says... :-/ And CTLR.IR implies another subset of DirectLPI
1091          * that the ITS driver can make use of for LPIs (and not VLPIs).
1092          *
1093          * These are 3 different ways to express the same thing, depending
1094          * on the revision of the architecture and its relaxations over
1095          * time. Just group them under the 'direct_lpi' banner.
1096          */
1097         gic_data.rdists.has_rvpeid &= !!(typer & GICR_TYPER_RVPEID);
1098         gic_data.rdists.has_direct_lpi &= (!!(typer & GICR_TYPER_DirectLPIS) |
1099                                            !!(ctlr & GICR_CTLR_IR) |
1100                                            gic_data.rdists.has_rvpeid);
1101         gic_data.rdists.has_vpend_valid_dirty &= !!(typer & GICR_TYPER_DIRTY);
1102
1103         /* Detect non-sensical configurations */
1104         if (WARN_ON_ONCE(gic_data.rdists.has_rvpeid && !gic_data.rdists.has_vlpis)) {
1105                 gic_data.rdists.has_direct_lpi = false;
1106                 gic_data.rdists.has_vlpis = false;
1107                 gic_data.rdists.has_rvpeid = false;
1108         }
1109
1110         gic_data.ppi_nr = min(GICR_TYPER_NR_PPIS(typer), gic_data.ppi_nr);
1111
1112         return 1;
1113 }
1114
1115 static void gic_update_rdist_properties(void)
1116 {
1117         gic_data.ppi_nr = UINT_MAX;
1118         gic_iterate_rdists(__gic_update_rdist_properties);
1119         if (WARN_ON(gic_data.ppi_nr == UINT_MAX))
1120                 gic_data.ppi_nr = 0;
1121         pr_info("GICv3 features: %d PPIs%s%s\n",
1122                 gic_data.ppi_nr,
1123                 gic_data.has_rss ? ", RSS" : "",
1124                 gic_data.rdists.has_direct_lpi ? ", DirectLPI" : "");
1125
1126         if (gic_data.rdists.has_vlpis)
1127                 pr_info("GICv4 features: %s%s%s\n",
1128                         gic_data.rdists.has_direct_lpi ? "DirectLPI " : "",
1129                         gic_data.rdists.has_rvpeid ? "RVPEID " : "",
1130                         gic_data.rdists.has_vpend_valid_dirty ? "Valid+Dirty " : "");
1131 }
1132
1133 /* Check whether it's single security state view */
1134 static inline bool gic_dist_security_disabled(void)
1135 {
1136         return readl_relaxed(gic_data.dist_base + GICD_CTLR) & GICD_CTLR_DS;
1137 }
1138
1139 static void gic_cpu_sys_reg_init(void)
1140 {
1141         int i, cpu = smp_processor_id();
1142         u64 mpidr = gic_cpu_to_affinity(cpu);
1143         u64 need_rss = MPIDR_RS(mpidr);
1144         bool group0;
1145         u32 pribits;
1146
1147         /*
1148          * Need to check that the SRE bit has actually been set. If
1149          * not, it means that SRE is disabled at EL2. We're going to
1150          * die painfully, and there is nothing we can do about it.
1151          *
1152          * Kindly inform the luser.
1153          */
1154         if (!gic_enable_sre())
1155                 pr_err("GIC: unable to set SRE (disabled at EL2), panic ahead\n");
1156
1157         pribits = gic_get_pribits();
1158
1159         group0 = gic_has_group0();
1160
1161         /* Set priority mask register */
1162         if (!gic_prio_masking_enabled()) {
1163                 write_gicreg(DEFAULT_PMR_VALUE, ICC_PMR_EL1);
1164         } else if (gic_supports_nmi()) {
1165                 /*
1166                  * Mismatch configuration with boot CPU, the system is likely
1167                  * to die as interrupt masking will not work properly on all
1168                  * CPUs
1169                  *
1170                  * The boot CPU calls this function before enabling NMI support,
1171                  * and as a result we'll never see this warning in the boot path
1172                  * for that CPU.
1173                  */
1174                 if (static_branch_unlikely(&gic_nonsecure_priorities))
1175                         WARN_ON(!group0 || gic_dist_security_disabled());
1176                 else
1177                         WARN_ON(group0 && !gic_dist_security_disabled());
1178         }
1179
1180         /*
1181          * Some firmwares hand over to the kernel with the BPR changed from
1182          * its reset value (and with a value large enough to prevent
1183          * any pre-emptive interrupts from working at all). Writing a zero
1184          * to BPR restores is reset value.
1185          */
1186         gic_write_bpr1(0);
1187
1188         if (static_branch_likely(&supports_deactivate_key)) {
1189                 /* EOI drops priority only (mode 1) */
1190                 gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop);
1191         } else {
1192                 /* EOI deactivates interrupt too (mode 0) */
1193                 gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop_dir);
1194         }
1195
1196         /* Always whack Group0 before Group1 */
1197         if (group0) {
1198                 switch(pribits) {
1199                 case 8:
1200                 case 7:
1201                         write_gicreg(0, ICC_AP0R3_EL1);
1202                         write_gicreg(0, ICC_AP0R2_EL1);
1203                         fallthrough;
1204                 case 6:
1205                         write_gicreg(0, ICC_AP0R1_EL1);
1206                         fallthrough;
1207                 case 5:
1208                 case 4:
1209                         write_gicreg(0, ICC_AP0R0_EL1);
1210                 }
1211
1212                 isb();
1213         }
1214
1215         switch(pribits) {
1216         case 8:
1217         case 7:
1218                 write_gicreg(0, ICC_AP1R3_EL1);
1219                 write_gicreg(0, ICC_AP1R2_EL1);
1220                 fallthrough;
1221         case 6:
1222                 write_gicreg(0, ICC_AP1R1_EL1);
1223                 fallthrough;
1224         case 5:
1225         case 4:
1226                 write_gicreg(0, ICC_AP1R0_EL1);
1227         }
1228
1229         isb();
1230
1231         /* ... and let's hit the road... */
1232         gic_write_grpen1(1);
1233
1234         /* Keep the RSS capability status in per_cpu variable */
1235         per_cpu(has_rss, cpu) = !!(gic_read_ctlr() & ICC_CTLR_EL1_RSS);
1236
1237         /* Check all the CPUs have capable of sending SGIs to other CPUs */
1238         for_each_online_cpu(i) {
1239                 bool have_rss = per_cpu(has_rss, i) && per_cpu(has_rss, cpu);
1240
1241                 need_rss |= MPIDR_RS(gic_cpu_to_affinity(i));
1242                 if (need_rss && (!have_rss))
1243                         pr_crit("CPU%d (%lx) can't SGI CPU%d (%lx), no RSS\n",
1244                                 cpu, (unsigned long)mpidr,
1245                                 i, (unsigned long)gic_cpu_to_affinity(i));
1246         }
1247
1248         /**
1249          * GIC spec says, when ICC_CTLR_EL1.RSS==1 and GICD_TYPER.RSS==0,
1250          * writing ICC_ASGI1R_EL1 register with RS != 0 is a CONSTRAINED
1251          * UNPREDICTABLE choice of :
1252          *   - The write is ignored.
1253          *   - The RS field is treated as 0.
1254          */
1255         if (need_rss && (!gic_data.has_rss))
1256                 pr_crit_once("RSS is required but GICD doesn't support it\n");
1257 }
1258
1259 static bool gicv3_nolpi;
1260
1261 static int __init gicv3_nolpi_cfg(char *buf)
1262 {
1263         return kstrtobool(buf, &gicv3_nolpi);
1264 }
1265 early_param("irqchip.gicv3_nolpi", gicv3_nolpi_cfg);
1266
1267 static int gic_dist_supports_lpis(void)
1268 {
1269         return (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) &&
1270                 !!(readl_relaxed(gic_data.dist_base + GICD_TYPER) & GICD_TYPER_LPIS) &&
1271                 !gicv3_nolpi);
1272 }
1273
1274 static void gic_cpu_init(void)
1275 {
1276         void __iomem *rbase;
1277         int i;
1278
1279         /* Register ourselves with the rest of the world */
1280         if (gic_populate_rdist())
1281                 return;
1282
1283         gic_enable_redist(true);
1284
1285         WARN((gic_data.ppi_nr > 16 || GIC_ESPI_NR != 0) &&
1286              !(gic_read_ctlr() & ICC_CTLR_EL1_ExtRange),
1287              "Distributor has extended ranges, but CPU%d doesn't\n",
1288              smp_processor_id());
1289
1290         rbase = gic_data_rdist_sgi_base();
1291
1292         /* Configure SGIs/PPIs as non-secure Group-1 */
1293         for (i = 0; i < gic_data.ppi_nr + SGI_NR; i += 32)
1294                 writel_relaxed(~0, rbase + GICR_IGROUPR0 + i / 8);
1295
1296         gic_cpu_config(rbase, gic_data.ppi_nr + SGI_NR, gic_redist_wait_for_rwp);
1297
1298         /* initialise system registers */
1299         gic_cpu_sys_reg_init();
1300 }
1301
1302 #ifdef CONFIG_SMP
1303
1304 #define MPIDR_TO_SGI_RS(mpidr)  (MPIDR_RS(mpidr) << ICC_SGI1R_RS_SHIFT)
1305 #define MPIDR_TO_SGI_CLUSTER_ID(mpidr)  ((mpidr) & ~0xFUL)
1306
1307 static int gic_starting_cpu(unsigned int cpu)
1308 {
1309         gic_cpu_init();
1310
1311         if (gic_dist_supports_lpis())
1312                 its_cpu_init();
1313
1314         return 0;
1315 }
1316
1317 static u16 gic_compute_target_list(int *base_cpu, const struct cpumask *mask,
1318                                    unsigned long cluster_id)
1319 {
1320         int next_cpu, cpu = *base_cpu;
1321         unsigned long mpidr;
1322         u16 tlist = 0;
1323
1324         mpidr = gic_cpu_to_affinity(cpu);
1325
1326         while (cpu < nr_cpu_ids) {
1327                 tlist |= 1 << (mpidr & 0xf);
1328
1329                 next_cpu = cpumask_next(cpu, mask);
1330                 if (next_cpu >= nr_cpu_ids)
1331                         goto out;
1332                 cpu = next_cpu;
1333
1334                 mpidr = gic_cpu_to_affinity(cpu);
1335
1336                 if (cluster_id != MPIDR_TO_SGI_CLUSTER_ID(mpidr)) {
1337                         cpu--;
1338                         goto out;
1339                 }
1340         }
1341 out:
1342         *base_cpu = cpu;
1343         return tlist;
1344 }
1345
1346 #define MPIDR_TO_SGI_AFFINITY(cluster_id, level) \
1347         (MPIDR_AFFINITY_LEVEL(cluster_id, level) \
1348                 << ICC_SGI1R_AFFINITY_## level ##_SHIFT)
1349
1350 static void gic_send_sgi(u64 cluster_id, u16 tlist, unsigned int irq)
1351 {
1352         u64 val;
1353
1354         val = (MPIDR_TO_SGI_AFFINITY(cluster_id, 3)     |
1355                MPIDR_TO_SGI_AFFINITY(cluster_id, 2)     |
1356                irq << ICC_SGI1R_SGI_ID_SHIFT            |
1357                MPIDR_TO_SGI_AFFINITY(cluster_id, 1)     |
1358                MPIDR_TO_SGI_RS(cluster_id)              |
1359                tlist << ICC_SGI1R_TARGET_LIST_SHIFT);
1360
1361         pr_devel("CPU%d: ICC_SGI1R_EL1 %llx\n", smp_processor_id(), val);
1362         gic_write_sgi1r(val);
1363 }
1364
1365 static void gic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask)
1366 {
1367         int cpu;
1368
1369         if (WARN_ON(d->hwirq >= 16))
1370                 return;
1371
1372         /*
1373          * Ensure that stores to Normal memory are visible to the
1374          * other CPUs before issuing the IPI.
1375          */
1376         dsb(ishst);
1377
1378         for_each_cpu(cpu, mask) {
1379                 u64 cluster_id = MPIDR_TO_SGI_CLUSTER_ID(gic_cpu_to_affinity(cpu));
1380                 u16 tlist;
1381
1382                 tlist = gic_compute_target_list(&cpu, mask, cluster_id);
1383                 gic_send_sgi(cluster_id, tlist, d->hwirq);
1384         }
1385
1386         /* Force the above writes to ICC_SGI1R_EL1 to be executed */
1387         isb();
1388 }
1389
1390 static void __init gic_smp_init(void)
1391 {
1392         struct irq_fwspec sgi_fwspec = {
1393                 .fwnode         = gic_data.fwnode,
1394                 .param_count    = 1,
1395         };
1396         int base_sgi;
1397
1398         cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_GIC_STARTING,
1399                                   "irqchip/arm/gicv3:starting",
1400                                   gic_starting_cpu, NULL);
1401
1402         /* Register all 8 non-secure SGIs */
1403         base_sgi = irq_domain_alloc_irqs(gic_data.domain, 8, NUMA_NO_NODE, &sgi_fwspec);
1404         if (WARN_ON(base_sgi <= 0))
1405                 return;
1406
1407         set_smp_ipi_range(base_sgi, 8);
1408 }
1409
1410 static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
1411                             bool force)
1412 {
1413         unsigned int cpu;
1414         u32 offset, index;
1415         void __iomem *reg;
1416         int enabled;
1417         u64 val;
1418
1419         if (force)
1420                 cpu = cpumask_first(mask_val);
1421         else
1422                 cpu = cpumask_any_and(mask_val, cpu_online_mask);
1423
1424         if (cpu >= nr_cpu_ids)
1425                 return -EINVAL;
1426
1427         if (gic_irq_in_rdist(d))
1428                 return -EINVAL;
1429
1430         /* If interrupt was enabled, disable it first */
1431         enabled = gic_peek_irq(d, GICD_ISENABLER);
1432         if (enabled)
1433                 gic_mask_irq(d);
1434
1435         offset = convert_offset_index(d, GICD_IROUTER, &index);
1436         reg = gic_dist_base(d) + offset + (index * 8);
1437         val = gic_cpu_to_affinity(cpu);
1438
1439         gic_write_irouter(val, reg);
1440
1441         /*
1442          * If the interrupt was enabled, enabled it again. Otherwise,
1443          * just wait for the distributor to have digested our changes.
1444          */
1445         if (enabled)
1446                 gic_unmask_irq(d);
1447
1448         irq_data_update_effective_affinity(d, cpumask_of(cpu));
1449
1450         return IRQ_SET_MASK_OK_DONE;
1451 }
1452 #else
1453 #define gic_set_affinity        NULL
1454 #define gic_ipi_send_mask       NULL
1455 #define gic_smp_init()          do { } while(0)
1456 #endif
1457
1458 static int gic_retrigger(struct irq_data *data)
1459 {
1460         return !gic_irq_set_irqchip_state(data, IRQCHIP_STATE_PENDING, true);
1461 }
1462
1463 #ifdef CONFIG_CPU_PM
1464 static int gic_cpu_pm_notifier(struct notifier_block *self,
1465                                unsigned long cmd, void *v)
1466 {
1467         if (cmd == CPU_PM_EXIT) {
1468                 if (gic_dist_security_disabled())
1469                         gic_enable_redist(true);
1470                 gic_cpu_sys_reg_init();
1471         } else if (cmd == CPU_PM_ENTER && gic_dist_security_disabled()) {
1472                 gic_write_grpen1(0);
1473                 gic_enable_redist(false);
1474         }
1475         return NOTIFY_OK;
1476 }
1477
1478 static struct notifier_block gic_cpu_pm_notifier_block = {
1479         .notifier_call = gic_cpu_pm_notifier,
1480 };
1481
1482 static void gic_cpu_pm_init(void)
1483 {
1484         cpu_pm_register_notifier(&gic_cpu_pm_notifier_block);
1485 }
1486
1487 #else
1488 static inline void gic_cpu_pm_init(void) { }
1489 #endif /* CONFIG_CPU_PM */
1490
1491 static struct irq_chip gic_chip = {
1492         .name                   = "GICv3",
1493         .irq_mask               = gic_mask_irq,
1494         .irq_unmask             = gic_unmask_irq,
1495         .irq_eoi                = gic_eoi_irq,
1496         .irq_set_type           = gic_set_type,
1497         .irq_set_affinity       = gic_set_affinity,
1498         .irq_retrigger          = gic_retrigger,
1499         .irq_get_irqchip_state  = gic_irq_get_irqchip_state,
1500         .irq_set_irqchip_state  = gic_irq_set_irqchip_state,
1501         .irq_nmi_setup          = gic_irq_nmi_setup,
1502         .irq_nmi_teardown       = gic_irq_nmi_teardown,
1503         .ipi_send_mask          = gic_ipi_send_mask,
1504         .flags                  = IRQCHIP_SET_TYPE_MASKED |
1505                                   IRQCHIP_SKIP_SET_WAKE |
1506                                   IRQCHIP_MASK_ON_SUSPEND,
1507 };
1508
1509 static struct irq_chip gic_eoimode1_chip = {
1510         .name                   = "GICv3",
1511         .irq_mask               = gic_eoimode1_mask_irq,
1512         .irq_unmask             = gic_unmask_irq,
1513         .irq_eoi                = gic_eoimode1_eoi_irq,
1514         .irq_set_type           = gic_set_type,
1515         .irq_set_affinity       = gic_set_affinity,
1516         .irq_retrigger          = gic_retrigger,
1517         .irq_get_irqchip_state  = gic_irq_get_irqchip_state,
1518         .irq_set_irqchip_state  = gic_irq_set_irqchip_state,
1519         .irq_set_vcpu_affinity  = gic_irq_set_vcpu_affinity,
1520         .irq_nmi_setup          = gic_irq_nmi_setup,
1521         .irq_nmi_teardown       = gic_irq_nmi_teardown,
1522         .ipi_send_mask          = gic_ipi_send_mask,
1523         .flags                  = IRQCHIP_SET_TYPE_MASKED |
1524                                   IRQCHIP_SKIP_SET_WAKE |
1525                                   IRQCHIP_MASK_ON_SUSPEND,
1526 };
1527
1528 static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq,
1529                               irq_hw_number_t hw)
1530 {
1531         struct irq_chip *chip = &gic_chip;
1532         struct irq_data *irqd = irq_desc_get_irq_data(irq_to_desc(irq));
1533
1534         if (static_branch_likely(&supports_deactivate_key))
1535                 chip = &gic_eoimode1_chip;
1536
1537         switch (__get_intid_range(hw)) {
1538         case SGI_RANGE:
1539         case PPI_RANGE:
1540         case EPPI_RANGE:
1541                 irq_set_percpu_devid(irq);
1542                 irq_domain_set_info(d, irq, hw, chip, d->host_data,
1543                                     handle_percpu_devid_irq, NULL, NULL);
1544                 break;
1545
1546         case SPI_RANGE:
1547         case ESPI_RANGE:
1548                 irq_domain_set_info(d, irq, hw, chip, d->host_data,
1549                                     handle_fasteoi_irq, NULL, NULL);
1550                 irq_set_probe(irq);
1551                 irqd_set_single_target(irqd);
1552                 break;
1553
1554         case LPI_RANGE:
1555                 if (!gic_dist_supports_lpis())
1556                         return -EPERM;
1557                 irq_domain_set_info(d, irq, hw, chip, d->host_data,
1558                                     handle_fasteoi_irq, NULL, NULL);
1559                 break;
1560
1561         default:
1562                 return -EPERM;
1563         }
1564
1565         /* Prevents SW retriggers which mess up the ACK/EOI ordering */
1566         irqd_set_handle_enforce_irqctx(irqd);
1567         return 0;
1568 }
1569
1570 static int gic_irq_domain_translate(struct irq_domain *d,
1571                                     struct irq_fwspec *fwspec,
1572                                     unsigned long *hwirq,
1573                                     unsigned int *type)
1574 {
1575         if (fwspec->param_count == 1 && fwspec->param[0] < 16) {
1576                 *hwirq = fwspec->param[0];
1577                 *type = IRQ_TYPE_EDGE_RISING;
1578                 return 0;
1579         }
1580
1581         if (is_of_node(fwspec->fwnode)) {
1582                 if (fwspec->param_count < 3)
1583                         return -EINVAL;
1584
1585                 switch (fwspec->param[0]) {
1586                 case 0:                 /* SPI */
1587                         *hwirq = fwspec->param[1] + 32;
1588                         break;
1589                 case 1:                 /* PPI */
1590                         *hwirq = fwspec->param[1] + 16;
1591                         break;
1592                 case 2:                 /* ESPI */
1593                         *hwirq = fwspec->param[1] + ESPI_BASE_INTID;
1594                         break;
1595                 case 3:                 /* EPPI */
1596                         *hwirq = fwspec->param[1] + EPPI_BASE_INTID;
1597                         break;
1598                 case GIC_IRQ_TYPE_LPI:  /* LPI */
1599                         *hwirq = fwspec->param[1];
1600                         break;
1601                 case GIC_IRQ_TYPE_PARTITION:
1602                         *hwirq = fwspec->param[1];
1603                         if (fwspec->param[1] >= 16)
1604                                 *hwirq += EPPI_BASE_INTID - 16;
1605                         else
1606                                 *hwirq += 16;
1607                         break;
1608                 default:
1609                         return -EINVAL;
1610                 }
1611
1612                 *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK;
1613
1614                 /*
1615                  * Make it clear that broken DTs are... broken.
1616                  * Partitioned PPIs are an unfortunate exception.
1617                  */
1618                 WARN_ON(*type == IRQ_TYPE_NONE &&
1619                         fwspec->param[0] != GIC_IRQ_TYPE_PARTITION);
1620                 return 0;
1621         }
1622
1623         if (is_fwnode_irqchip(fwspec->fwnode)) {
1624                 if(fwspec->param_count != 2)
1625                         return -EINVAL;
1626
1627                 if (fwspec->param[0] < 16) {
1628                         pr_err(FW_BUG "Illegal GSI%d translation request\n",
1629                                fwspec->param[0]);
1630                         return -EINVAL;
1631                 }
1632
1633                 *hwirq = fwspec->param[0];
1634                 *type = fwspec->param[1];
1635
1636                 WARN_ON(*type == IRQ_TYPE_NONE);
1637                 return 0;
1638         }
1639
1640         return -EINVAL;
1641 }
1642
1643 static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
1644                                 unsigned int nr_irqs, void *arg)
1645 {
1646         int i, ret;
1647         irq_hw_number_t hwirq;
1648         unsigned int type = IRQ_TYPE_NONE;
1649         struct irq_fwspec *fwspec = arg;
1650
1651         ret = gic_irq_domain_translate(domain, fwspec, &hwirq, &type);
1652         if (ret)
1653                 return ret;
1654
1655         for (i = 0; i < nr_irqs; i++) {
1656                 ret = gic_irq_domain_map(domain, virq + i, hwirq + i);
1657                 if (ret)
1658                         return ret;
1659         }
1660
1661         return 0;
1662 }
1663
1664 static void gic_irq_domain_free(struct irq_domain *domain, unsigned int virq,
1665                                 unsigned int nr_irqs)
1666 {
1667         int i;
1668
1669         for (i = 0; i < nr_irqs; i++) {
1670                 struct irq_data *d = irq_domain_get_irq_data(domain, virq + i);
1671                 irq_set_handler(virq + i, NULL);
1672                 irq_domain_reset_irq_data(d);
1673         }
1674 }
1675
1676 static bool fwspec_is_partitioned_ppi(struct irq_fwspec *fwspec,
1677                                       irq_hw_number_t hwirq)
1678 {
1679         enum gic_intid_range range;
1680
1681         if (!gic_data.ppi_descs)
1682                 return false;
1683
1684         if (!is_of_node(fwspec->fwnode))
1685                 return false;
1686
1687         if (fwspec->param_count < 4 || !fwspec->param[3])
1688                 return false;
1689
1690         range = __get_intid_range(hwirq);
1691         if (range != PPI_RANGE && range != EPPI_RANGE)
1692                 return false;
1693
1694         return true;
1695 }
1696
1697 static int gic_irq_domain_select(struct irq_domain *d,
1698                                  struct irq_fwspec *fwspec,
1699                                  enum irq_domain_bus_token bus_token)
1700 {
1701         unsigned int type, ret, ppi_idx;
1702         irq_hw_number_t hwirq;
1703
1704         /* Not for us */
1705         if (fwspec->fwnode != d->fwnode)
1706                 return 0;
1707
1708         /* If this is not DT, then we have a single domain */
1709         if (!is_of_node(fwspec->fwnode))
1710                 return 1;
1711
1712         ret = gic_irq_domain_translate(d, fwspec, &hwirq, &type);
1713         if (WARN_ON_ONCE(ret))
1714                 return 0;
1715
1716         if (!fwspec_is_partitioned_ppi(fwspec, hwirq))
1717                 return d == gic_data.domain;
1718
1719         /*
1720          * If this is a PPI and we have a 4th (non-null) parameter,
1721          * then we need to match the partition domain.
1722          */
1723         ppi_idx = __gic_get_ppi_index(hwirq);
1724         return d == partition_get_domain(gic_data.ppi_descs[ppi_idx]);
1725 }
1726
1727 static const struct irq_domain_ops gic_irq_domain_ops = {
1728         .translate = gic_irq_domain_translate,
1729         .alloc = gic_irq_domain_alloc,
1730         .free = gic_irq_domain_free,
1731         .select = gic_irq_domain_select,
1732 };
1733
1734 static int partition_domain_translate(struct irq_domain *d,
1735                                       struct irq_fwspec *fwspec,
1736                                       unsigned long *hwirq,
1737                                       unsigned int *type)
1738 {
1739         unsigned long ppi_intid;
1740         struct device_node *np;
1741         unsigned int ppi_idx;
1742         int ret;
1743
1744         if (!gic_data.ppi_descs)
1745                 return -ENOMEM;
1746
1747         np = of_find_node_by_phandle(fwspec->param[3]);
1748         if (WARN_ON(!np))
1749                 return -EINVAL;
1750
1751         ret = gic_irq_domain_translate(d, fwspec, &ppi_intid, type);
1752         if (WARN_ON_ONCE(ret))
1753                 return 0;
1754
1755         ppi_idx = __gic_get_ppi_index(ppi_intid);
1756         ret = partition_translate_id(gic_data.ppi_descs[ppi_idx],
1757                                      of_node_to_fwnode(np));
1758         if (ret < 0)
1759                 return ret;
1760
1761         *hwirq = ret;
1762         *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK;
1763
1764         return 0;
1765 }
1766
1767 static const struct irq_domain_ops partition_domain_ops = {
1768         .translate = partition_domain_translate,
1769         .select = gic_irq_domain_select,
1770 };
1771
1772 static bool gic_enable_quirk_msm8996(void *data)
1773 {
1774         struct gic_chip_data *d = data;
1775
1776         d->flags |= FLAGS_WORKAROUND_GICR_WAKER_MSM8996;
1777
1778         return true;
1779 }
1780
1781 static bool gic_enable_quirk_cavium_38539(void *data)
1782 {
1783         struct gic_chip_data *d = data;
1784
1785         d->flags |= FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539;
1786
1787         return true;
1788 }
1789
1790 static bool gic_enable_quirk_hip06_07(void *data)
1791 {
1792         struct gic_chip_data *d = data;
1793
1794         /*
1795          * HIP06 GICD_IIDR clashes with GIC-600 product number (despite
1796          * not being an actual ARM implementation). The saving grace is
1797          * that GIC-600 doesn't have ESPI, so nothing to do in that case.
1798          * HIP07 doesn't even have a proper IIDR, and still pretends to
1799          * have ESPI. In both cases, put them right.
1800          */
1801         if (d->rdists.gicd_typer & GICD_TYPER_ESPI) {
1802                 /* Zero both ESPI and the RES0 field next to it... */
1803                 d->rdists.gicd_typer &= ~GENMASK(9, 8);
1804                 return true;
1805         }
1806
1807         return false;
1808 }
1809
1810 #define T241_CHIPN_MASK         GENMASK_ULL(45, 44)
1811 #define T241_CHIP_GICDA_OFFSET  0x1580000
1812 #define SMCCC_SOC_ID_T241       0x036b0241
1813
1814 static bool gic_enable_quirk_nvidia_t241(void *data)
1815 {
1816         s32 soc_id = arm_smccc_get_soc_id_version();
1817         unsigned long chip_bmask = 0;
1818         phys_addr_t phys;
1819         u32 i;
1820
1821         /* Check JEP106 code for NVIDIA T241 chip (036b:0241) */
1822         if ((soc_id < 0) || (soc_id != SMCCC_SOC_ID_T241))
1823                 return false;
1824
1825         /* Find the chips based on GICR regions PHYS addr */
1826         for (i = 0; i < gic_data.nr_redist_regions; i++) {
1827                 chip_bmask |= BIT(FIELD_GET(T241_CHIPN_MASK,
1828                                   (u64)gic_data.redist_regions[i].phys_base));
1829         }
1830
1831         if (hweight32(chip_bmask) < 3)
1832                 return false;
1833
1834         /* Setup GICD alias regions */
1835         for (i = 0; i < ARRAY_SIZE(t241_dist_base_alias); i++) {
1836                 if (chip_bmask & BIT(i)) {
1837                         phys = gic_data.dist_phys_base + T241_CHIP_GICDA_OFFSET;
1838                         phys |= FIELD_PREP(T241_CHIPN_MASK, i);
1839                         t241_dist_base_alias[i] = ioremap(phys, SZ_64K);
1840                         WARN_ON_ONCE(!t241_dist_base_alias[i]);
1841                 }
1842         }
1843         static_branch_enable(&gic_nvidia_t241_erratum);
1844         return true;
1845 }
1846
1847 static bool gic_enable_quirk_asr8601(void *data)
1848 {
1849         struct gic_chip_data *d = data;
1850
1851         d->flags |= FLAGS_WORKAROUND_ASR_ERRATUM_8601001;
1852
1853         return true;
1854 }
1855
1856 static bool gic_enable_quirk_arm64_2941627(void *data)
1857 {
1858         static_branch_enable(&gic_arm64_2941627_erratum);
1859         return true;
1860 }
1861
1862 static bool rd_set_non_coherent(void *data)
1863 {
1864         struct gic_chip_data *d = data;
1865
1866         d->rdists.flags |= RDIST_FLAGS_FORCE_NON_SHAREABLE;
1867         return true;
1868 }
1869
1870 static const struct gic_quirk gic_quirks[] = {
1871         {
1872                 .desc   = "GICv3: Qualcomm MSM8996 broken firmware",
1873                 .compatible = "qcom,msm8996-gic-v3",
1874                 .init   = gic_enable_quirk_msm8996,
1875         },
1876         {
1877                 .desc   = "GICv3: ASR erratum 8601001",
1878                 .compatible = "asr,asr8601-gic-v3",
1879                 .init   = gic_enable_quirk_asr8601,
1880         },
1881         {
1882                 .desc   = "GICv3: HIP06 erratum 161010803",
1883                 .iidr   = 0x0204043b,
1884                 .mask   = 0xffffffff,
1885                 .init   = gic_enable_quirk_hip06_07,
1886         },
1887         {
1888                 .desc   = "GICv3: HIP07 erratum 161010803",
1889                 .iidr   = 0x00000000,
1890                 .mask   = 0xffffffff,
1891                 .init   = gic_enable_quirk_hip06_07,
1892         },
1893         {
1894                 /*
1895                  * Reserved register accesses generate a Synchronous
1896                  * External Abort. This erratum applies to:
1897                  * - ThunderX: CN88xx
1898                  * - OCTEON TX: CN83xx, CN81xx
1899                  * - OCTEON TX2: CN93xx, CN96xx, CN98xx, CNF95xx*
1900                  */
1901                 .desc   = "GICv3: Cavium erratum 38539",
1902                 .iidr   = 0xa000034c,
1903                 .mask   = 0xe8f00fff,
1904                 .init   = gic_enable_quirk_cavium_38539,
1905         },
1906         {
1907                 .desc   = "GICv3: NVIDIA erratum T241-FABRIC-4",
1908                 .iidr   = 0x0402043b,
1909                 .mask   = 0xffffffff,
1910                 .init   = gic_enable_quirk_nvidia_t241,
1911         },
1912         {
1913                 /*
1914                  * GIC-700: 2941627 workaround - IP variant [0,1]
1915                  *
1916                  */
1917                 .desc   = "GICv3: ARM64 erratum 2941627",
1918                 .iidr   = 0x0400043b,
1919                 .mask   = 0xff0e0fff,
1920                 .init   = gic_enable_quirk_arm64_2941627,
1921         },
1922         {
1923                 /*
1924                  * GIC-700: 2941627 workaround - IP variant [2]
1925                  */
1926                 .desc   = "GICv3: ARM64 erratum 2941627",
1927                 .iidr   = 0x0402043b,
1928                 .mask   = 0xff0f0fff,
1929                 .init   = gic_enable_quirk_arm64_2941627,
1930         },
1931         {
1932                 .desc   = "GICv3: non-coherent attribute",
1933                 .property = "dma-noncoherent",
1934                 .init   = rd_set_non_coherent,
1935         },
1936         {
1937         }
1938 };
1939
1940 static void gic_enable_nmi_support(void)
1941 {
1942         int i;
1943
1944         if (!gic_prio_masking_enabled())
1945                 return;
1946
1947         rdist_nmi_refs = kcalloc(gic_data.ppi_nr + SGI_NR,
1948                                  sizeof(*rdist_nmi_refs), GFP_KERNEL);
1949         if (!rdist_nmi_refs)
1950                 return;
1951
1952         for (i = 0; i < gic_data.ppi_nr + SGI_NR; i++)
1953                 refcount_set(&rdist_nmi_refs[i], 0);
1954
1955         pr_info("Pseudo-NMIs enabled using %s ICC_PMR_EL1 synchronisation\n",
1956                 gic_has_relaxed_pmr_sync() ? "relaxed" : "forced");
1957
1958         /*
1959          * How priority values are used by the GIC depends on two things:
1960          * the security state of the GIC (controlled by the GICD_CTRL.DS bit)
1961          * and if Group 0 interrupts can be delivered to Linux in the non-secure
1962          * world as FIQs (controlled by the SCR_EL3.FIQ bit). These affect the
1963          * ICC_PMR_EL1 register and the priority that software assigns to
1964          * interrupts:
1965          *
1966          * GICD_CTRL.DS | SCR_EL3.FIQ | ICC_PMR_EL1 | Group 1 priority
1967          * -----------------------------------------------------------
1968          *      1       |      -      |  unchanged  |    unchanged
1969          * -----------------------------------------------------------
1970          *      0       |      1      |  non-secure |    non-secure
1971          * -----------------------------------------------------------
1972          *      0       |      0      |  unchanged  |    non-secure
1973          *
1974          * where non-secure means that the value is right-shifted by one and the
1975          * MSB bit set, to make it fit in the non-secure priority range.
1976          *
1977          * In the first two cases, where ICC_PMR_EL1 and the interrupt priority
1978          * are both either modified or unchanged, we can use the same set of
1979          * priorities.
1980          *
1981          * In the last case, where only the interrupt priorities are modified to
1982          * be in the non-secure range, we use a different PMR value to mask IRQs
1983          * and the rest of the values that we use remain unchanged.
1984          */
1985         if (gic_has_group0() && !gic_dist_security_disabled())
1986                 static_branch_enable(&gic_nonsecure_priorities);
1987
1988         static_branch_enable(&supports_pseudo_nmis);
1989
1990         if (static_branch_likely(&supports_deactivate_key))
1991                 gic_eoimode1_chip.flags |= IRQCHIP_SUPPORTS_NMI;
1992         else
1993                 gic_chip.flags |= IRQCHIP_SUPPORTS_NMI;
1994 }
1995
1996 static int __init gic_init_bases(phys_addr_t dist_phys_base,
1997                                  void __iomem *dist_base,
1998                                  struct redist_region *rdist_regs,
1999                                  u32 nr_redist_regions,
2000                                  u64 redist_stride,
2001                                  struct fwnode_handle *handle)
2002 {
2003         u32 typer;
2004         int err;
2005
2006         if (!is_hyp_mode_available())
2007                 static_branch_disable(&supports_deactivate_key);
2008
2009         if (static_branch_likely(&supports_deactivate_key))
2010                 pr_info("GIC: Using split EOI/Deactivate mode\n");
2011
2012         gic_data.fwnode = handle;
2013         gic_data.dist_phys_base = dist_phys_base;
2014         gic_data.dist_base = dist_base;
2015         gic_data.redist_regions = rdist_regs;
2016         gic_data.nr_redist_regions = nr_redist_regions;
2017         gic_data.redist_stride = redist_stride;
2018
2019         /*
2020          * Find out how many interrupts are supported.
2021          */
2022         typer = readl_relaxed(gic_data.dist_base + GICD_TYPER);
2023         gic_data.rdists.gicd_typer = typer;
2024
2025         gic_enable_quirks(readl_relaxed(gic_data.dist_base + GICD_IIDR),
2026                           gic_quirks, &gic_data);
2027
2028         pr_info("%d SPIs implemented\n", GIC_LINE_NR - 32);
2029         pr_info("%d Extended SPIs implemented\n", GIC_ESPI_NR);
2030
2031         /*
2032          * ThunderX1 explodes on reading GICD_TYPER2, in violation of the
2033          * architecture spec (which says that reserved registers are RES0).
2034          */
2035         if (!(gic_data.flags & FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539))
2036                 gic_data.rdists.gicd_typer2 = readl_relaxed(gic_data.dist_base + GICD_TYPER2);
2037
2038         gic_data.domain = irq_domain_create_tree(handle, &gic_irq_domain_ops,
2039                                                  &gic_data);
2040         gic_data.rdists.rdist = alloc_percpu(typeof(*gic_data.rdists.rdist));
2041         if (!static_branch_unlikely(&gic_nvidia_t241_erratum)) {
2042                 /* Disable GICv4.x features for the erratum T241-FABRIC-4 */
2043                 gic_data.rdists.has_rvpeid = true;
2044                 gic_data.rdists.has_vlpis = true;
2045                 gic_data.rdists.has_direct_lpi = true;
2046                 gic_data.rdists.has_vpend_valid_dirty = true;
2047         }
2048
2049         if (WARN_ON(!gic_data.domain) || WARN_ON(!gic_data.rdists.rdist)) {
2050                 err = -ENOMEM;
2051                 goto out_free;
2052         }
2053
2054         irq_domain_update_bus_token(gic_data.domain, DOMAIN_BUS_WIRED);
2055
2056         gic_data.has_rss = !!(typer & GICD_TYPER_RSS);
2057
2058         if (typer & GICD_TYPER_MBIS) {
2059                 err = mbi_init(handle, gic_data.domain);
2060                 if (err)
2061                         pr_err("Failed to initialize MBIs\n");
2062         }
2063
2064         set_handle_irq(gic_handle_irq);
2065
2066         gic_update_rdist_properties();
2067
2068         gic_dist_init();
2069         gic_cpu_init();
2070         gic_enable_nmi_support();
2071         gic_smp_init();
2072         gic_cpu_pm_init();
2073
2074         if (gic_dist_supports_lpis()) {
2075                 its_init(handle, &gic_data.rdists, gic_data.domain);
2076                 its_cpu_init();
2077                 its_lpi_memreserve_init();
2078         } else {
2079                 if (IS_ENABLED(CONFIG_ARM_GIC_V2M))
2080                         gicv2m_init(handle, gic_data.domain);
2081         }
2082
2083         return 0;
2084
2085 out_free:
2086         if (gic_data.domain)
2087                 irq_domain_remove(gic_data.domain);
2088         free_percpu(gic_data.rdists.rdist);
2089         return err;
2090 }
2091
2092 static int __init gic_validate_dist_version(void __iomem *dist_base)
2093 {
2094         u32 reg = readl_relaxed(dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK;
2095
2096         if (reg != GIC_PIDR2_ARCH_GICv3 && reg != GIC_PIDR2_ARCH_GICv4)
2097                 return -ENODEV;
2098
2099         return 0;
2100 }
2101
2102 /* Create all possible partitions at boot time */
2103 static void __init gic_populate_ppi_partitions(struct device_node *gic_node)
2104 {
2105         struct device_node *parts_node, *child_part;
2106         int part_idx = 0, i;
2107         int nr_parts;
2108         struct partition_affinity *parts;
2109
2110         parts_node = of_get_child_by_name(gic_node, "ppi-partitions");
2111         if (!parts_node)
2112                 return;
2113
2114         gic_data.ppi_descs = kcalloc(gic_data.ppi_nr, sizeof(*gic_data.ppi_descs), GFP_KERNEL);
2115         if (!gic_data.ppi_descs)
2116                 goto out_put_node;
2117
2118         nr_parts = of_get_child_count(parts_node);
2119
2120         if (!nr_parts)
2121                 goto out_put_node;
2122
2123         parts = kcalloc(nr_parts, sizeof(*parts), GFP_KERNEL);
2124         if (WARN_ON(!parts))
2125                 goto out_put_node;
2126
2127         for_each_child_of_node(parts_node, child_part) {
2128                 struct partition_affinity *part;
2129                 int n;
2130
2131                 part = &parts[part_idx];
2132
2133                 part->partition_id = of_node_to_fwnode(child_part);
2134
2135                 pr_info("GIC: PPI partition %pOFn[%d] { ",
2136                         child_part, part_idx);
2137
2138                 n = of_property_count_elems_of_size(child_part, "affinity",
2139                                                     sizeof(u32));
2140                 WARN_ON(n <= 0);
2141
2142                 for (i = 0; i < n; i++) {
2143                         int err, cpu;
2144                         u32 cpu_phandle;
2145                         struct device_node *cpu_node;
2146
2147                         err = of_property_read_u32_index(child_part, "affinity",
2148                                                          i, &cpu_phandle);
2149                         if (WARN_ON(err))
2150                                 continue;
2151
2152                         cpu_node = of_find_node_by_phandle(cpu_phandle);
2153                         if (WARN_ON(!cpu_node))
2154                                 continue;
2155
2156                         cpu = of_cpu_node_to_id(cpu_node);
2157                         if (WARN_ON(cpu < 0)) {
2158                                 of_node_put(cpu_node);
2159                                 continue;
2160                         }
2161
2162                         pr_cont("%pOF[%d] ", cpu_node, cpu);
2163
2164                         cpumask_set_cpu(cpu, &part->mask);
2165                         of_node_put(cpu_node);
2166                 }
2167
2168                 pr_cont("}\n");
2169                 part_idx++;
2170         }
2171
2172         for (i = 0; i < gic_data.ppi_nr; i++) {
2173                 unsigned int irq;
2174                 struct partition_desc *desc;
2175                 struct irq_fwspec ppi_fwspec = {
2176                         .fwnode         = gic_data.fwnode,
2177                         .param_count    = 3,
2178                         .param          = {
2179                                 [0]     = GIC_IRQ_TYPE_PARTITION,
2180                                 [1]     = i,
2181                                 [2]     = IRQ_TYPE_NONE,
2182                         },
2183                 };
2184
2185                 irq = irq_create_fwspec_mapping(&ppi_fwspec);
2186                 if (WARN_ON(!irq))
2187                         continue;
2188                 desc = partition_create_desc(gic_data.fwnode, parts, nr_parts,
2189                                              irq, &partition_domain_ops);
2190                 if (WARN_ON(!desc))
2191                         continue;
2192
2193                 gic_data.ppi_descs[i] = desc;
2194         }
2195
2196 out_put_node:
2197         of_node_put(parts_node);
2198 }
2199
2200 static void __init gic_of_setup_kvm_info(struct device_node *node)
2201 {
2202         int ret;
2203         struct resource r;
2204         u32 gicv_idx;
2205
2206         gic_v3_kvm_info.type = GIC_V3;
2207
2208         gic_v3_kvm_info.maint_irq = irq_of_parse_and_map(node, 0);
2209         if (!gic_v3_kvm_info.maint_irq)
2210                 return;
2211
2212         if (of_property_read_u32(node, "#redistributor-regions",
2213                                  &gicv_idx))
2214                 gicv_idx = 1;
2215
2216         gicv_idx += 3;  /* Also skip GICD, GICC, GICH */
2217         ret = of_address_to_resource(node, gicv_idx, &r);
2218         if (!ret)
2219                 gic_v3_kvm_info.vcpu = r;
2220
2221         gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis;
2222         gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid;
2223         vgic_set_kvm_info(&gic_v3_kvm_info);
2224 }
2225
2226 static void gic_request_region(resource_size_t base, resource_size_t size,
2227                                const char *name)
2228 {
2229         if (!request_mem_region(base, size, name))
2230                 pr_warn_once(FW_BUG "%s region %pa has overlapping address\n",
2231                              name, &base);
2232 }
2233
2234 static void __iomem *gic_of_iomap(struct device_node *node, int idx,
2235                                   const char *name, struct resource *res)
2236 {
2237         void __iomem *base;
2238         int ret;
2239
2240         ret = of_address_to_resource(node, idx, res);
2241         if (ret)
2242                 return IOMEM_ERR_PTR(ret);
2243
2244         gic_request_region(res->start, resource_size(res), name);
2245         base = of_iomap(node, idx);
2246
2247         return base ?: IOMEM_ERR_PTR(-ENOMEM);
2248 }
2249
2250 static int __init gic_of_init(struct device_node *node, struct device_node *parent)
2251 {
2252         phys_addr_t dist_phys_base;
2253         void __iomem *dist_base;
2254         struct redist_region *rdist_regs;
2255         struct resource res;
2256         u64 redist_stride;
2257         u32 nr_redist_regions;
2258         int err, i;
2259
2260         dist_base = gic_of_iomap(node, 0, "GICD", &res);
2261         if (IS_ERR(dist_base)) {
2262                 pr_err("%pOF: unable to map gic dist registers\n", node);
2263                 return PTR_ERR(dist_base);
2264         }
2265
2266         dist_phys_base = res.start;
2267
2268         err = gic_validate_dist_version(dist_base);
2269         if (err) {
2270                 pr_err("%pOF: no distributor detected, giving up\n", node);
2271                 goto out_unmap_dist;
2272         }
2273
2274         if (of_property_read_u32(node, "#redistributor-regions", &nr_redist_regions))
2275                 nr_redist_regions = 1;
2276
2277         rdist_regs = kcalloc(nr_redist_regions, sizeof(*rdist_regs),
2278                              GFP_KERNEL);
2279         if (!rdist_regs) {
2280                 err = -ENOMEM;
2281                 goto out_unmap_dist;
2282         }
2283
2284         for (i = 0; i < nr_redist_regions; i++) {
2285                 rdist_regs[i].redist_base = gic_of_iomap(node, 1 + i, "GICR", &res);
2286                 if (IS_ERR(rdist_regs[i].redist_base)) {
2287                         pr_err("%pOF: couldn't map region %d\n", node, i);
2288                         err = -ENODEV;
2289                         goto out_unmap_rdist;
2290                 }
2291                 rdist_regs[i].phys_base = res.start;
2292         }
2293
2294         if (of_property_read_u64(node, "redistributor-stride", &redist_stride))
2295                 redist_stride = 0;
2296
2297         gic_enable_of_quirks(node, gic_quirks, &gic_data);
2298
2299         err = gic_init_bases(dist_phys_base, dist_base, rdist_regs,
2300                              nr_redist_regions, redist_stride, &node->fwnode);
2301         if (err)
2302                 goto out_unmap_rdist;
2303
2304         gic_populate_ppi_partitions(node);
2305
2306         if (static_branch_likely(&supports_deactivate_key))
2307                 gic_of_setup_kvm_info(node);
2308         return 0;
2309
2310 out_unmap_rdist:
2311         for (i = 0; i < nr_redist_regions; i++)
2312                 if (rdist_regs[i].redist_base && !IS_ERR(rdist_regs[i].redist_base))
2313                         iounmap(rdist_regs[i].redist_base);
2314         kfree(rdist_regs);
2315 out_unmap_dist:
2316         iounmap(dist_base);
2317         return err;
2318 }
2319
2320 IRQCHIP_DECLARE(gic_v3, "arm,gic-v3", gic_of_init);
2321
2322 #ifdef CONFIG_ACPI
2323 static struct
2324 {
2325         void __iomem *dist_base;
2326         struct redist_region *redist_regs;
2327         u32 nr_redist_regions;
2328         bool single_redist;
2329         int enabled_rdists;
2330         u32 maint_irq;
2331         int maint_irq_mode;
2332         phys_addr_t vcpu_base;
2333 } acpi_data __initdata;
2334
2335 static void __init
2336 gic_acpi_register_redist(phys_addr_t phys_base, void __iomem *redist_base)
2337 {
2338         static int count = 0;
2339
2340         acpi_data.redist_regs[count].phys_base = phys_base;
2341         acpi_data.redist_regs[count].redist_base = redist_base;
2342         acpi_data.redist_regs[count].single_redist = acpi_data.single_redist;
2343         count++;
2344 }
2345
2346 static int __init
2347 gic_acpi_parse_madt_redist(union acpi_subtable_headers *header,
2348                            const unsigned long end)
2349 {
2350         struct acpi_madt_generic_redistributor *redist =
2351                         (struct acpi_madt_generic_redistributor *)header;
2352         void __iomem *redist_base;
2353
2354         redist_base = ioremap(redist->base_address, redist->length);
2355         if (!redist_base) {
2356                 pr_err("Couldn't map GICR region @%llx\n", redist->base_address);
2357                 return -ENOMEM;
2358         }
2359         gic_request_region(redist->base_address, redist->length, "GICR");
2360
2361         gic_acpi_register_redist(redist->base_address, redist_base);
2362         return 0;
2363 }
2364
2365 static int __init
2366 gic_acpi_parse_madt_gicc(union acpi_subtable_headers *header,
2367                          const unsigned long end)
2368 {
2369         struct acpi_madt_generic_interrupt *gicc =
2370                                 (struct acpi_madt_generic_interrupt *)header;
2371         u32 reg = readl_relaxed(acpi_data.dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK;
2372         u32 size = reg == GIC_PIDR2_ARCH_GICv4 ? SZ_64K * 4 : SZ_64K * 2;
2373         void __iomem *redist_base;
2374
2375         if (!acpi_gicc_is_usable(gicc))
2376                 return 0;
2377
2378         redist_base = ioremap(gicc->gicr_base_address, size);
2379         if (!redist_base)
2380                 return -ENOMEM;
2381         gic_request_region(gicc->gicr_base_address, size, "GICR");
2382
2383         gic_acpi_register_redist(gicc->gicr_base_address, redist_base);
2384         return 0;
2385 }
2386
2387 static int __init gic_acpi_collect_gicr_base(void)
2388 {
2389         acpi_tbl_entry_handler redist_parser;
2390         enum acpi_madt_type type;
2391
2392         if (acpi_data.single_redist) {
2393                 type = ACPI_MADT_TYPE_GENERIC_INTERRUPT;
2394                 redist_parser = gic_acpi_parse_madt_gicc;
2395         } else {
2396                 type = ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR;
2397                 redist_parser = gic_acpi_parse_madt_redist;
2398         }
2399
2400         /* Collect redistributor base addresses in GICR entries */
2401         if (acpi_table_parse_madt(type, redist_parser, 0) > 0)
2402                 return 0;
2403
2404         pr_info("No valid GICR entries exist\n");
2405         return -ENODEV;
2406 }
2407
2408 static int __init gic_acpi_match_gicr(union acpi_subtable_headers *header,
2409                                   const unsigned long end)
2410 {
2411         /* Subtable presence means that redist exists, that's it */
2412         return 0;
2413 }
2414
2415 static int __init gic_acpi_match_gicc(union acpi_subtable_headers *header,
2416                                       const unsigned long end)
2417 {
2418         struct acpi_madt_generic_interrupt *gicc =
2419                                 (struct acpi_madt_generic_interrupt *)header;
2420
2421         /*
2422          * If GICC is enabled and has valid gicr base address, then it means
2423          * GICR base is presented via GICC
2424          */
2425         if (acpi_gicc_is_usable(gicc) && gicc->gicr_base_address) {
2426                 acpi_data.enabled_rdists++;
2427                 return 0;
2428         }
2429
2430         /*
2431          * It's perfectly valid firmware can pass disabled GICC entry, driver
2432          * should not treat as errors, skip the entry instead of probe fail.
2433          */
2434         if (!acpi_gicc_is_usable(gicc))
2435                 return 0;
2436
2437         return -ENODEV;
2438 }
2439
2440 static int __init gic_acpi_count_gicr_regions(void)
2441 {
2442         int count;
2443
2444         /*
2445          * Count how many redistributor regions we have. It is not allowed
2446          * to mix redistributor description, GICR and GICC subtables have to be
2447          * mutually exclusive.
2448          */
2449         count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR,
2450                                       gic_acpi_match_gicr, 0);
2451         if (count > 0) {
2452                 acpi_data.single_redist = false;
2453                 return count;
2454         }
2455
2456         count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
2457                                       gic_acpi_match_gicc, 0);
2458         if (count > 0) {
2459                 acpi_data.single_redist = true;
2460                 count = acpi_data.enabled_rdists;
2461         }
2462
2463         return count;
2464 }
2465
2466 static bool __init acpi_validate_gic_table(struct acpi_subtable_header *header,
2467                                            struct acpi_probe_entry *ape)
2468 {
2469         struct acpi_madt_generic_distributor *dist;
2470         int count;
2471
2472         dist = (struct acpi_madt_generic_distributor *)header;
2473         if (dist->version != ape->driver_data)
2474                 return false;
2475
2476         /* We need to do that exercise anyway, the sooner the better */
2477         count = gic_acpi_count_gicr_regions();
2478         if (count <= 0)
2479                 return false;
2480
2481         acpi_data.nr_redist_regions = count;
2482         return true;
2483 }
2484
2485 static int __init gic_acpi_parse_virt_madt_gicc(union acpi_subtable_headers *header,
2486                                                 const unsigned long end)
2487 {
2488         struct acpi_madt_generic_interrupt *gicc =
2489                 (struct acpi_madt_generic_interrupt *)header;
2490         int maint_irq_mode;
2491         static int first_madt = true;
2492
2493         if (!acpi_gicc_is_usable(gicc))
2494                 return 0;
2495
2496         maint_irq_mode = (gicc->flags & ACPI_MADT_VGIC_IRQ_MODE) ?
2497                 ACPI_EDGE_SENSITIVE : ACPI_LEVEL_SENSITIVE;
2498
2499         if (first_madt) {
2500                 first_madt = false;
2501
2502                 acpi_data.maint_irq = gicc->vgic_interrupt;
2503                 acpi_data.maint_irq_mode = maint_irq_mode;
2504                 acpi_data.vcpu_base = gicc->gicv_base_address;
2505
2506                 return 0;
2507         }
2508
2509         /*
2510          * The maintenance interrupt and GICV should be the same for every CPU
2511          */
2512         if ((acpi_data.maint_irq != gicc->vgic_interrupt) ||
2513             (acpi_data.maint_irq_mode != maint_irq_mode) ||
2514             (acpi_data.vcpu_base != gicc->gicv_base_address))
2515                 return -EINVAL;
2516
2517         return 0;
2518 }
2519
2520 static bool __init gic_acpi_collect_virt_info(void)
2521 {
2522         int count;
2523
2524         count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
2525                                       gic_acpi_parse_virt_madt_gicc, 0);
2526
2527         return (count > 0);
2528 }
2529
2530 #define ACPI_GICV3_DIST_MEM_SIZE (SZ_64K)
2531 #define ACPI_GICV2_VCTRL_MEM_SIZE       (SZ_4K)
2532 #define ACPI_GICV2_VCPU_MEM_SIZE        (SZ_8K)
2533
2534 static void __init gic_acpi_setup_kvm_info(void)
2535 {
2536         int irq;
2537
2538         if (!gic_acpi_collect_virt_info()) {
2539                 pr_warn("Unable to get hardware information used for virtualization\n");
2540                 return;
2541         }
2542
2543         gic_v3_kvm_info.type = GIC_V3;
2544
2545         irq = acpi_register_gsi(NULL, acpi_data.maint_irq,
2546                                 acpi_data.maint_irq_mode,
2547                                 ACPI_ACTIVE_HIGH);
2548         if (irq <= 0)
2549                 return;
2550
2551         gic_v3_kvm_info.maint_irq = irq;
2552
2553         if (acpi_data.vcpu_base) {
2554                 struct resource *vcpu = &gic_v3_kvm_info.vcpu;
2555
2556                 vcpu->flags = IORESOURCE_MEM;
2557                 vcpu->start = acpi_data.vcpu_base;
2558                 vcpu->end = vcpu->start + ACPI_GICV2_VCPU_MEM_SIZE - 1;
2559         }
2560
2561         gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis;
2562         gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid;
2563         vgic_set_kvm_info(&gic_v3_kvm_info);
2564 }
2565
2566 static struct fwnode_handle *gsi_domain_handle;
2567
2568 static struct fwnode_handle *gic_v3_get_gsi_domain_id(u32 gsi)
2569 {
2570         return gsi_domain_handle;
2571 }
2572
2573 static int __init
2574 gic_acpi_init(union acpi_subtable_headers *header, const unsigned long end)
2575 {
2576         struct acpi_madt_generic_distributor *dist;
2577         size_t size;
2578         int i, err;
2579
2580         /* Get distributor base address */
2581         dist = (struct acpi_madt_generic_distributor *)header;
2582         acpi_data.dist_base = ioremap(dist->base_address,
2583                                       ACPI_GICV3_DIST_MEM_SIZE);
2584         if (!acpi_data.dist_base) {
2585                 pr_err("Unable to map GICD registers\n");
2586                 return -ENOMEM;
2587         }
2588         gic_request_region(dist->base_address, ACPI_GICV3_DIST_MEM_SIZE, "GICD");
2589
2590         err = gic_validate_dist_version(acpi_data.dist_base);
2591         if (err) {
2592                 pr_err("No distributor detected at @%p, giving up\n",
2593                        acpi_data.dist_base);
2594                 goto out_dist_unmap;
2595         }
2596
2597         size = sizeof(*acpi_data.redist_regs) * acpi_data.nr_redist_regions;
2598         acpi_data.redist_regs = kzalloc(size, GFP_KERNEL);
2599         if (!acpi_data.redist_regs) {
2600                 err = -ENOMEM;
2601                 goto out_dist_unmap;
2602         }
2603
2604         err = gic_acpi_collect_gicr_base();
2605         if (err)
2606                 goto out_redist_unmap;
2607
2608         gsi_domain_handle = irq_domain_alloc_fwnode(&dist->base_address);
2609         if (!gsi_domain_handle) {
2610                 err = -ENOMEM;
2611                 goto out_redist_unmap;
2612         }
2613
2614         err = gic_init_bases(dist->base_address, acpi_data.dist_base,
2615                              acpi_data.redist_regs, acpi_data.nr_redist_regions,
2616                              0, gsi_domain_handle);
2617         if (err)
2618                 goto out_fwhandle_free;
2619
2620         acpi_set_irq_model(ACPI_IRQ_MODEL_GIC, gic_v3_get_gsi_domain_id);
2621
2622         if (static_branch_likely(&supports_deactivate_key))
2623                 gic_acpi_setup_kvm_info();
2624
2625         return 0;
2626
2627 out_fwhandle_free:
2628         irq_domain_free_fwnode(gsi_domain_handle);
2629 out_redist_unmap:
2630         for (i = 0; i < acpi_data.nr_redist_regions; i++)
2631                 if (acpi_data.redist_regs[i].redist_base)
2632                         iounmap(acpi_data.redist_regs[i].redist_base);
2633         kfree(acpi_data.redist_regs);
2634 out_dist_unmap:
2635         iounmap(acpi_data.dist_base);
2636         return err;
2637 }
2638 IRQCHIP_ACPI_DECLARE(gic_v3, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR,
2639                      acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V3,
2640                      gic_acpi_init);
2641 IRQCHIP_ACPI_DECLARE(gic_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR,
2642                      acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V4,
2643                      gic_acpi_init);
2644 IRQCHIP_ACPI_DECLARE(gic_v3_or_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR,
2645                      acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_NONE,
2646                      gic_acpi_init);
2647 #endif