drivers/irqchip/irq-riscv-imsic-platform.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2021 Western Digital Corporation or its affiliates.
   4  * Copyright (C) 2022 Ventana Micro Systems Inc.
   5  */
   6
   7 #define pr_fmt(fmt) "riscv-imsic: " fmt
   8 #include <linux/bitmap.h>
   9 #include <linux/cpu.h>
  10 #include <linux/interrupt.h>
  11 #include <linux/io.h>
  12 #include <linux/irq.h>
  13 #include <linux/irqchip.h>
  14 #include <linux/irqdomain.h>
  15 #include <linux/module.h>
  16 #include <linux/msi.h>
  17 #include <linux/pci.h>
  18 #include <linux/platform_device.h>
  19 #include <linux/spinlock.h>
  20 #include <linux/smp.h>
  21
  22 #include "irq-riscv-imsic-state.h"
  23
  24 static bool imsic_cpu_page_phys(unsigned int cpu, unsigned int guest_index,
  25                                 phys_addr_t *out_msi_pa)
  26 {
  27         struct imsic_global_config *global;
  28         struct imsic_local_config *local;
  29
  30         global = &imsic->global;
  31         local = per_cpu_ptr(global->local, cpu);
  32
  33         if (BIT(global->guest_index_bits) <= guest_index)
  34                 return false;
  35
  36         if (out_msi_pa)
  37                 *out_msi_pa = local->msi_pa + (guest_index * IMSIC_MMIO_PAGE_SZ);
  38
  39         return true;
  40 }
  41
  42 static void imsic_irq_mask(struct irq_data *d)
  43 {
  44         imsic_vector_mask(irq_data_get_irq_chip_data(d));
  45 }
  46
  47 static void imsic_irq_unmask(struct irq_data *d)
  48 {
  49         imsic_vector_unmask(irq_data_get_irq_chip_data(d));
  50 }
  51
  52 static int imsic_irq_retrigger(struct irq_data *d)
  53 {
  54         struct imsic_vector *vec = irq_data_get_irq_chip_data(d);
  55         struct imsic_local_config *local;
  56
  57         if (WARN_ON(!vec))
  58                 return -ENOENT;
  59
  60         local = per_cpu_ptr(imsic->global.local, vec->cpu);
  61         writel_relaxed(vec->local_id, local->msi_va);
  62         return 0;
  63 }
  64
  65 static void imsic_irq_compose_vector_msg(struct imsic_vector *vec, struct msi_msg *msg)
  66 {
  67         phys_addr_t msi_addr;
  68
  69         if (WARN_ON(!vec))
  70                 return;
  71
  72         if (WARN_ON(!imsic_cpu_page_phys(vec->cpu, 0, &msi_addr)))
  73                 return;
  74
  75         msg->address_hi = upper_32_bits(msi_addr);
  76         msg->address_lo = lower_32_bits(msi_addr);
  77         msg->data = vec->local_id;
  78 }
  79
  80 static void imsic_irq_compose_msg(struct irq_data *d, struct msi_msg *msg)
  81 {
  82         imsic_irq_compose_vector_msg(irq_data_get_irq_chip_data(d), msg);
  83 }
  84
  85 #ifdef CONFIG_SMP
  86 static void imsic_msi_update_msg(struct irq_data *d, struct imsic_vector *vec)
  87 {
  88         struct msi_msg msg = { };
  89
  90         imsic_irq_compose_vector_msg(vec, &msg);
  91         irq_data_get_irq_chip(d)->irq_write_msi_msg(d, &msg);
  92 }
  93
  94 static int imsic_irq_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
  95                                   bool force)
  96 {
  97         struct imsic_vector *old_vec, *new_vec;
  98         struct irq_data *pd = d->parent_data;
  99
 100         old_vec = irq_data_get_irq_chip_data(pd);
 101         if (WARN_ON(!old_vec))
 102                 return -ENOENT;
 103
 104         /* If old vector cpu belongs to the target cpumask then do nothing */
 105         if (cpumask_test_cpu(old_vec->cpu, mask_val))
 106                 return IRQ_SET_MASK_OK_DONE;
 107
 108         /* If move is already in-flight then return failure */
 109         if (imsic_vector_get_move(old_vec))
 110                 return -EBUSY;
 111
 112         /* Get a new vector on the desired set of CPUs */
 113         new_vec = imsic_vector_alloc(old_vec->hwirq, mask_val);
 114         if (!new_vec)
 115                 return -ENOSPC;
 116
 117         /* Point device to the new vector */
 118         imsic_msi_update_msg(d, new_vec);
 119
 120         /* Update irq descriptors with the new vector */
 121         pd->chip_data = new_vec;
 122
 123         /* Update effective affinity of parent irq data */
 124         irq_data_update_effective_affinity(pd, cpumask_of(new_vec->cpu));
 125
 126         /* Move state of the old vector to the new vector */
 127         imsic_vector_move(old_vec, new_vec);
 128
 129         return IRQ_SET_MASK_OK_DONE;
 130 }
 131 #endif
 132
 133 static struct irq_chip imsic_irq_base_chip = {
 134         .name                   = "IMSIC",
 135         .irq_mask               = imsic_irq_mask,
 136         .irq_unmask             = imsic_irq_unmask,
 137         .irq_retrigger          = imsic_irq_retrigger,
 138         .irq_compose_msi_msg    = imsic_irq_compose_msg,
 139         .flags                  = IRQCHIP_SKIP_SET_WAKE |
 140                                   IRQCHIP_MASK_ON_SUSPEND,
 141 };
 142
 143 static int imsic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
 144                                   unsigned int nr_irqs, void *args)
 145 {
 146         struct imsic_vector *vec;
 147
 148         /* Multi-MSI is not supported yet. */
 149         if (nr_irqs > 1)
 150                 return -EOPNOTSUPP;
 151
 152         vec = imsic_vector_alloc(virq, cpu_online_mask);
 153         if (!vec)
 154                 return -ENOSPC;
 155
 156         irq_domain_set_info(domain, virq, virq, &imsic_irq_base_chip, vec,
 157                             handle_simple_irq, NULL, NULL);
 158         irq_set_noprobe(virq);
 159         irq_set_affinity(virq, cpu_online_mask);
 160         irq_data_update_effective_affinity(irq_get_irq_data(virq), cpumask_of(vec->cpu));
 161
 162         return 0;
 163 }
 164
 165 static void imsic_irq_domain_free(struct irq_domain *domain, unsigned int virq,
 166                                   unsigned int nr_irqs)
 167 {
 168         struct irq_data *d = irq_domain_get_irq_data(domain, virq);
 169
 170         imsic_vector_free(irq_data_get_irq_chip_data(d));
 171         irq_domain_free_irqs_parent(domain, virq, nr_irqs);
 172 }
 173
 174 static int imsic_irq_domain_select(struct irq_domain *domain, struct irq_fwspec *fwspec,
 175                                    enum irq_domain_bus_token bus_token)
 176 {
 177         const struct msi_parent_ops *ops = domain->msi_parent_ops;
 178         u32 busmask = BIT(bus_token);
 179
 180         if (fwspec->fwnode != domain->fwnode || fwspec->param_count != 0)
 181                 return 0;
 182
 183         /* Handle pure domain searches */
 184         if (bus_token == ops->bus_select_token)
 185                 return 1;
 186
 187         return !!(ops->bus_select_mask & busmask);
 188 }
 189
 190 #ifdef CONFIG_GENERIC_IRQ_DEBUGFS
 191 static void imsic_irq_debug_show(struct seq_file *m, struct irq_domain *d,
 192                                  struct irq_data *irqd, int ind)
 193 {
 194         if (!irqd) {
 195                 imsic_vector_debug_show_summary(m, ind);
 196                 return;
 197         }
 198
 199         imsic_vector_debug_show(m, irq_data_get_irq_chip_data(irqd), ind);
 200 }
 201 #endif
 202
 203 static const struct irq_domain_ops imsic_base_domain_ops = {
 204         .alloc          = imsic_irq_domain_alloc,
 205         .free           = imsic_irq_domain_free,
 206         .select         = imsic_irq_domain_select,
 207 #ifdef CONFIG_GENERIC_IRQ_DEBUGFS
 208         .debug_show     = imsic_irq_debug_show,
 209 #endif
 210 };
 211
 212 #ifdef CONFIG_RISCV_IMSIC_PCI
 213
 214 static void imsic_pci_mask_irq(struct irq_data *d)
 215 {
 216         pci_msi_mask_irq(d);
 217         irq_chip_mask_parent(d);
 218 }
 219
 220 static void imsic_pci_unmask_irq(struct irq_data *d)
 221 {
 222         irq_chip_unmask_parent(d);
 223         pci_msi_unmask_irq(d);
 224 }
 225
 226 #define MATCH_PCI_MSI           BIT(DOMAIN_BUS_PCI_MSI)
 227
 228 #else
 229
 230 #define MATCH_PCI_MSI           0
 231
 232 #endif
 233
 234 static bool imsic_init_dev_msi_info(struct device *dev,
 235                                     struct irq_domain *domain,
 236                                     struct irq_domain *real_parent,
 237                                     struct msi_domain_info *info)
 238 {
 239         const struct msi_parent_ops *pops = real_parent->msi_parent_ops;
 240
 241         /* MSI parent domain specific settings */
 242         switch (real_parent->bus_token) {
 243         case DOMAIN_BUS_NEXUS:
 244                 if (WARN_ON_ONCE(domain != real_parent))
 245                         return false;
 246 #ifdef CONFIG_SMP
 247                 info->chip->irq_set_affinity = imsic_irq_set_affinity;
 248 #endif
 249                 break;
 250         default:
 251                 WARN_ON_ONCE(1);
 252                 return false;
 253         }
 254
 255         /* Is the target supported? */
 256         switch (info->bus_token) {
 257 #ifdef CONFIG_RISCV_IMSIC_PCI
 258         case DOMAIN_BUS_PCI_DEVICE_MSI:
 259         case DOMAIN_BUS_PCI_DEVICE_MSIX:
 260                 info->chip->irq_mask = imsic_pci_mask_irq;
 261                 info->chip->irq_unmask = imsic_pci_unmask_irq;
 262                 break;
 263 #endif
 264         case DOMAIN_BUS_DEVICE_MSI:
 265                 /*
 266                  * Per-device MSI should never have any MSI feature bits
 267                  * set. It's sole purpose is to create a dumb interrupt
 268                  * chip which has a device specific irq_write_msi_msg()
 269                  * callback.
 270                  */
 271                 if (WARN_ON_ONCE(info->flags))
 272                         return false;
 273
 274                 /* Core managed MSI descriptors */
 275                 info->flags |= MSI_FLAG_ALLOC_SIMPLE_MSI_DESCS |
 276                                MSI_FLAG_FREE_MSI_DESCS;
 277                 break;
 278         case DOMAIN_BUS_WIRED_TO_MSI:
 279                 break;
 280         default:
 281                 WARN_ON_ONCE(1);
 282                 return false;
 283         }
 284
 285         /* Use hierarchial chip operations re-trigger */
 286         info->chip->irq_retrigger = irq_chip_retrigger_hierarchy;
 287
 288         /*
 289          * Mask out the domain specific MSI feature flags which are not
 290          * supported by the real parent.
 291          */
 292         info->flags &= pops->supported_flags;
 293
 294         /* Enforce the required flags */
 295         info->flags |= pops->required_flags;
 296
 297         return true;
 298 }
 299
 300 #define MATCH_PLATFORM_MSI              BIT(DOMAIN_BUS_PLATFORM_MSI)
 301
 302 static const struct msi_parent_ops imsic_msi_parent_ops = {
 303         .supported_flags        = MSI_GENERIC_FLAGS_MASK |
 304                                   MSI_FLAG_PCI_MSIX,
 305         .required_flags         = MSI_FLAG_USE_DEF_DOM_OPS |
 306                                   MSI_FLAG_USE_DEF_CHIP_OPS,
 307         .bus_select_token       = DOMAIN_BUS_NEXUS,
 308         .bus_select_mask        = MATCH_PCI_MSI | MATCH_PLATFORM_MSI,
 309         .init_dev_msi_info      = imsic_init_dev_msi_info,
 310 };
 311
 312 int imsic_irqdomain_init(void)
 313 {
 314         struct imsic_global_config *global;
 315
 316         if (!imsic || !imsic->fwnode) {
 317                 pr_err("early driver not probed\n");
 318                 return -ENODEV;
 319         }
 320
 321         if (imsic->base_domain) {
 322                 pr_err("%pfwP: irq domain already created\n", imsic->fwnode);
 323                 return -ENODEV;
 324         }
 325
 326         /* Create Base IRQ domain */
 327         imsic->base_domain = irq_domain_create_tree(imsic->fwnode,
 328                                                     &imsic_base_domain_ops, imsic);
 329         if (!imsic->base_domain) {
 330                 pr_err("%pfwP: failed to create IMSIC base domain\n", imsic->fwnode);
 331                 return -ENOMEM;
 332         }
 333         imsic->base_domain->flags |= IRQ_DOMAIN_FLAG_MSI_PARENT;
 334         imsic->base_domain->msi_parent_ops = &imsic_msi_parent_ops;
 335
 336         irq_domain_update_bus_token(imsic->base_domain, DOMAIN_BUS_NEXUS);
 337
 338         global = &imsic->global;
 339         pr_info("%pfwP:  hart-index-bits: %d,  guest-index-bits: %d\n",
 340                 imsic->fwnode, global->hart_index_bits, global->guest_index_bits);
 341         pr_info("%pfwP: group-index-bits: %d, group-index-shift: %d\n",
 342                 imsic->fwnode, global->group_index_bits, global->group_index_shift);
 343         pr_info("%pfwP: per-CPU IDs %d at base PPN %pa\n",
 344                 imsic->fwnode, global->nr_ids, &global->base_addr);
 345         pr_info("%pfwP: total %d interrupts available\n",
 346                 imsic->fwnode, num_possible_cpus() * (global->nr_ids - 1));
 347
 348         return 0;
 349 }
 350
 351 static int imsic_platform_probe(struct platform_device *pdev)
 352 {
 353         struct device *dev = &pdev->dev;
 354
 355         if (imsic && imsic->fwnode != dev->fwnode) {
 356                 dev_err(dev, "fwnode mismatch\n");
 357                 return -ENODEV;
 358         }
 359
 360         return imsic_irqdomain_init();
 361 }
 362
 363 static const struct of_device_id imsic_platform_match[] = {
 364         { .compatible = "riscv,imsics" },
 365         {}
 366 };
 367
 368 static struct platform_driver imsic_platform_driver = {
 369         .driver = {
 370                 .name           = "riscv-imsic",
 371                 .of_match_table = imsic_platform_match,
 372         },
 373         .probe = imsic_platform_probe,
 374 };
 375 builtin_platform_driver(imsic_platform_driver);