arch/loongarch/kernel/paravirt.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <linux/export.h>
   3 #include <linux/types.h>
   4 #include <linux/interrupt.h>
   5 #include <linux/irq_work.h>
   6 #include <linux/jump_label.h>
   7 #include <linux/kvm_para.h>
   8 #include <linux/reboot.h>
   9 #include <linux/static_call.h>
  10 #include <asm/paravirt.h>
  11
  12 static int has_steal_clock;
  13 struct static_key paravirt_steal_enabled;
  14 struct static_key paravirt_steal_rq_enabled;
  15 static DEFINE_PER_CPU(struct kvm_steal_time, steal_time) __aligned(64);
  16 DEFINE_STATIC_KEY_FALSE(virt_spin_lock_key);
  17
  18 static u64 native_steal_clock(int cpu)
  19 {
  20         return 0;
  21 }
  22
  23 DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock);
  24
  25 static bool steal_acc = true;
  26
  27 static int __init parse_no_stealacc(char *arg)
  28 {
  29         steal_acc = false;
  30         return 0;
  31 }
  32 early_param("no-steal-acc", parse_no_stealacc);
  33
  34 static u64 paravt_steal_clock(int cpu)
  35 {
  36         int version;
  37         u64 steal;
  38         struct kvm_steal_time *src;
  39
  40         src = &per_cpu(steal_time, cpu);
  41         do {
  42
  43                 version = src->version;
  44                 virt_rmb(); /* Make sure that the version is read before the steal */
  45                 steal = src->steal;
  46                 virt_rmb(); /* Make sure that the steal is read before the next version */
  47
  48         } while ((version & 1) || (version != src->version));
  49
  50         return steal;
  51 }
  52
  53 #ifdef CONFIG_SMP
  54 static struct smp_ops native_ops;
  55
  56 static void pv_send_ipi_single(int cpu, unsigned int action)
  57 {
  58         int min, old;
  59         irq_cpustat_t *info = &per_cpu(irq_stat, cpu);
  60
  61         if (unlikely(action == ACTION_BOOT_CPU)) {
  62                 native_ops.send_ipi_single(cpu, action);
  63                 return;
  64         }
  65
  66         old = atomic_fetch_or(BIT(action), &info->message);
  67         if (old)
  68                 return;
  69
  70         min = cpu_logical_map(cpu);
  71         kvm_hypercall3(KVM_HCALL_FUNC_IPI, 1, 0, min);
  72 }
  73
  74 #define KVM_IPI_CLUSTER_SIZE    (2 * BITS_PER_LONG)
  75
  76 static void pv_send_ipi_mask(const struct cpumask *mask, unsigned int action)
  77 {
  78         int i, cpu, min = 0, max = 0, old;
  79         __uint128_t bitmap = 0;
  80         irq_cpustat_t *info;
  81
  82         if (cpumask_empty(mask))
  83                 return;
  84
  85         if (unlikely(action == ACTION_BOOT_CPU)) {
  86                 native_ops.send_ipi_mask(mask, action);
  87                 return;
  88         }
  89
  90         action = BIT(action);
  91         for_each_cpu(i, mask) {
  92                 info = &per_cpu(irq_stat, i);
  93                 old = atomic_fetch_or(action, &info->message);
  94                 if (old)
  95                         continue;
  96
  97                 cpu = cpu_logical_map(i);
  98                 if (!bitmap) {
  99                         min = max = cpu;
 100                 } else if (cpu < min && cpu > (max - KVM_IPI_CLUSTER_SIZE)) {
 101                         /* cpu < min, and bitmap still enough */
 102                         bitmap <<= min - cpu;
 103                         min = cpu;
 104                 } else if (cpu > min && cpu < (min + KVM_IPI_CLUSTER_SIZE)) {
 105                         /* cpu > min, and bitmap still enough */
 106                         max = cpu > max ? cpu : max;
 107                 } else {
 108                         /*
 109                          * With cpu, bitmap will exceed KVM_IPI_CLUSTER_SIZE,
 110                          * send IPI here directly and skip the remaining CPUs.
 111                          */
 112                         kvm_hypercall3(KVM_HCALL_FUNC_IPI, (unsigned long)bitmap,
 113                                       (unsigned long)(bitmap >> BITS_PER_LONG), min);
 114                         min = max = cpu;
 115                         bitmap = 0;
 116                 }
 117                 __set_bit(cpu - min, (unsigned long *)&bitmap);
 118         }
 119
 120         if (bitmap)
 121                 kvm_hypercall3(KVM_HCALL_FUNC_IPI, (unsigned long)bitmap,
 122                               (unsigned long)(bitmap >> BITS_PER_LONG), min);
 123 }
 124
 125 static irqreturn_t pv_ipi_interrupt(int irq, void *dev)
 126 {
 127         u32 action;
 128         irq_cpustat_t *info;
 129
 130         /* Clear SWI interrupt */
 131         clear_csr_estat(1 << INT_SWI0);
 132         info = this_cpu_ptr(&irq_stat);
 133         action = atomic_xchg(&info->message, 0);
 134
 135         if (action & SMP_RESCHEDULE) {
 136                 scheduler_ipi();
 137                 info->ipi_irqs[IPI_RESCHEDULE]++;
 138         }
 139
 140         if (action & SMP_CALL_FUNCTION) {
 141                 generic_smp_call_function_interrupt();
 142                 info->ipi_irqs[IPI_CALL_FUNCTION]++;
 143         }
 144
 145         if (action & SMP_IRQ_WORK) {
 146                 irq_work_run();
 147                 info->ipi_irqs[IPI_IRQ_WORK]++;
 148         }
 149
 150         if (action & SMP_CLEAR_VECTOR) {
 151                 complete_irq_moving();
 152                 info->ipi_irqs[IPI_CLEAR_VECTOR]++;
 153         }
 154
 155         return IRQ_HANDLED;
 156 }
 157
 158 static void pv_init_ipi(void)
 159 {
 160         int r, swi;
 161
 162         /* Init native ipi irq for ACTION_BOOT_CPU */
 163         native_ops.init_ipi();
 164         swi = get_percpu_irq(INT_SWI0);
 165         if (swi < 0)
 166                 panic("SWI0 IRQ mapping failed\n");
 167         irq_set_percpu_devid(swi);
 168         r = request_percpu_irq(swi, pv_ipi_interrupt, "SWI0-IPI", &irq_stat);
 169         if (r < 0)
 170                 panic("SWI0 IRQ request failed\n");
 171 }
 172 #endif
 173
 174 bool kvm_para_available(void)
 175 {
 176         int config;
 177         static int hypervisor_type;
 178
 179         if (!cpu_has_hypervisor)
 180                 return false;
 181
 182         if (!hypervisor_type) {
 183                 config = read_cpucfg(CPUCFG_KVM_SIG);
 184                 if (!memcmp(&config, KVM_SIGNATURE, 4))
 185                         hypervisor_type = HYPERVISOR_KVM;
 186         }
 187
 188         return hypervisor_type == HYPERVISOR_KVM;
 189 }
 190
 191 unsigned int kvm_arch_para_features(void)
 192 {
 193         static unsigned int feature;
 194
 195         if (!kvm_para_available())
 196                 return 0;
 197
 198         if (!feature)
 199                 feature = read_cpucfg(CPUCFG_KVM_FEATURE);
 200
 201         return feature;
 202 }
 203
 204 int __init pv_ipi_init(void)
 205 {
 206         if (!kvm_para_has_feature(KVM_FEATURE_IPI))
 207                 return 0;
 208
 209 #ifdef CONFIG_SMP
 210         native_ops              = mp_ops;
 211         mp_ops.init_ipi         = pv_init_ipi;
 212         mp_ops.send_ipi_single  = pv_send_ipi_single;
 213         mp_ops.send_ipi_mask    = pv_send_ipi_mask;
 214 #endif
 215
 216         return 0;
 217 }
 218
 219 static int pv_enable_steal_time(void)
 220 {
 221         int cpu = smp_processor_id();
 222         unsigned long addr;
 223         struct kvm_steal_time *st;
 224
 225         if (!has_steal_clock)
 226                 return -EPERM;
 227
 228         st = &per_cpu(steal_time, cpu);
 229         addr = per_cpu_ptr_to_phys(st);
 230
 231         /* The whole structure kvm_steal_time should be in one page */
 232         if (PFN_DOWN(addr) != PFN_DOWN(addr + sizeof(*st))) {
 233                 pr_warn("Illegal PV steal time addr %lx\n", addr);
 234                 return -EFAULT;
 235         }
 236
 237         addr |= KVM_STEAL_PHYS_VALID;
 238         kvm_hypercall2(KVM_HCALL_FUNC_NOTIFY, BIT(KVM_FEATURE_STEAL_TIME), addr);
 239
 240         return 0;
 241 }
 242
 243 static void pv_disable_steal_time(void)
 244 {
 245         if (has_steal_clock)
 246                 kvm_hypercall2(KVM_HCALL_FUNC_NOTIFY, BIT(KVM_FEATURE_STEAL_TIME), 0);
 247 }
 248
 249 #ifdef CONFIG_SMP
 250 static int pv_time_cpu_online(unsigned int cpu)
 251 {
 252         unsigned long flags;
 253
 254         local_irq_save(flags);
 255         pv_enable_steal_time();
 256         local_irq_restore(flags);
 257
 258         return 0;
 259 }
 260
 261 static int pv_time_cpu_down_prepare(unsigned int cpu)
 262 {
 263         unsigned long flags;
 264
 265         local_irq_save(flags);
 266         pv_disable_steal_time();
 267         local_irq_restore(flags);
 268
 269         return 0;
 270 }
 271 #endif
 272
 273 static void pv_cpu_reboot(void *unused)
 274 {
 275         pv_disable_steal_time();
 276 }
 277
 278 static int pv_reboot_notify(struct notifier_block *nb, unsigned long code, void *unused)
 279 {
 280         on_each_cpu(pv_cpu_reboot, NULL, 1);
 281         return NOTIFY_DONE;
 282 }
 283
 284 static struct notifier_block pv_reboot_nb = {
 285         .notifier_call  = pv_reboot_notify,
 286 };
 287
 288 int __init pv_time_init(void)
 289 {
 290         int r;
 291
 292         if (!kvm_para_has_feature(KVM_FEATURE_STEAL_TIME))
 293                 return 0;
 294
 295         has_steal_clock = 1;
 296         r = pv_enable_steal_time();
 297         if (r < 0) {
 298                 has_steal_clock = 0;
 299                 return 0;
 300         }
 301         register_reboot_notifier(&pv_reboot_nb);
 302
 303 #ifdef CONFIG_SMP
 304         r = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
 305                                       "loongarch/pv_time:online",
 306                                       pv_time_cpu_online, pv_time_cpu_down_prepare);
 307         if (r < 0) {
 308                 has_steal_clock = 0;
 309                 pr_err("Failed to install cpu hotplug callbacks\n");
 310                 return r;
 311         }
 312 #endif
 313
 314         static_call_update(pv_steal_clock, paravt_steal_clock);
 315
 316         static_key_slow_inc(&paravirt_steal_enabled);
 317 #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
 318         if (steal_acc)
 319                 static_key_slow_inc(&paravirt_steal_rq_enabled);
 320 #endif
 321
 322         pr_info("Using paravirt steal-time\n");
 323
 324         return 0;
 325 }
 326
 327 int __init pv_spinlock_init(void)
 328 {
 329         if (!cpu_has_hypervisor)
 330                 return 0;
 331
 332         static_branch_enable(&virt_spin_lock_key);
 333
 334         return 0;
 335 }