arch/arm64/kernel/cpu_errata.c

   1 /*
   2  * Contains CPU specific errata definitions
   3  *
   4  * Copyright (C) 2014 ARM Ltd.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17  */
  18
  19 #include <linux/types.h>
  20 #include <asm/cpu.h>
  21 #include <asm/cputype.h>
  22 #include <asm/cpufeature.h>
  23
  24 static bool __maybe_unused
  25 is_affected_midr_range(const struct arm64_cpu_capabilities *entry, int scope)
  26 {
  27         const struct arm64_midr_revidr *fix;
  28         u32 midr = read_cpuid_id(), revidr;
  29
  30         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
  31         if (!is_midr_in_range(midr, &entry->midr_range))
  32                 return false;
  33
  34         midr &= MIDR_REVISION_MASK | MIDR_VARIANT_MASK;
  35         revidr = read_cpuid(REVIDR_EL1);
  36         for (fix = entry->fixed_revs; fix && fix->revidr_mask; fix++)
  37                 if (midr == fix->midr_rv && (revidr & fix->revidr_mask))
  38                         return false;
  39
  40         return true;
  41 }
  42
  43 static bool __maybe_unused
  44 is_affected_midr_range_list(const struct arm64_cpu_capabilities *entry,
  45                             int scope)
  46 {
  47         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
  48         return is_midr_in_range_list(read_cpuid_id(), entry->midr_range_list);
  49 }
  50
  51 static bool __maybe_unused
  52 is_kryo_midr(const struct arm64_cpu_capabilities *entry, int scope)
  53 {
  54         u32 model;
  55
  56         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
  57
  58         model = read_cpuid_id();
  59         model &= MIDR_IMPLEMENTOR_MASK | (0xf00 << MIDR_PARTNUM_SHIFT) |
  60                  MIDR_ARCHITECTURE_MASK;
  61
  62         return model == entry->midr_range.model;
  63 }
  64
  65 static bool
  66 has_mismatched_cache_line_size(const struct arm64_cpu_capabilities *entry,
  67                                 int scope)
  68 {
  69         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
  70         return (read_cpuid_cachetype() & arm64_ftr_reg_ctrel0.strict_mask) !=
  71                 (arm64_ftr_reg_ctrel0.sys_val & arm64_ftr_reg_ctrel0.strict_mask);
  72 }
  73
  74 static void
  75 cpu_enable_trap_ctr_access(const struct arm64_cpu_capabilities *__unused)
  76 {
  77         /* Clear SCTLR_EL1.UCT */
  78         config_sctlr_el1(SCTLR_EL1_UCT, 0);
  79 }
  80
  81 atomic_t arm64_el2_vector_last_slot = ATOMIC_INIT(-1);
  82
  83 #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
  84 #include <asm/mmu_context.h>
  85 #include <asm/cacheflush.h>
  86
  87 DEFINE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data);
  88
  89 #ifdef CONFIG_KVM_INDIRECT_VECTORS
  90 extern char __smccc_workaround_1_smc_start[];
  91 extern char __smccc_workaround_1_smc_end[];
  92
  93 static void __copy_hyp_vect_bpi(int slot, const char *hyp_vecs_start,
  94                                 const char *hyp_vecs_end)
  95 {
  96         void *dst = lm_alias(__bp_harden_hyp_vecs_start + slot * SZ_2K);
  97         int i;
  98
  99         for (i = 0; i < SZ_2K; i += 0x80)
 100                 memcpy(dst + i, hyp_vecs_start, hyp_vecs_end - hyp_vecs_start);
 101
 102         flush_icache_range((uintptr_t)dst, (uintptr_t)dst + SZ_2K);
 103 }
 104
 105 static void __install_bp_hardening_cb(bp_hardening_cb_t fn,
 106                                       const char *hyp_vecs_start,
 107                                       const char *hyp_vecs_end)
 108 {
 109         static DEFINE_SPINLOCK(bp_lock);
 110         int cpu, slot = -1;
 111
 112         spin_lock(&bp_lock);
 113         for_each_possible_cpu(cpu) {
 114                 if (per_cpu(bp_hardening_data.fn, cpu) == fn) {
 115                         slot = per_cpu(bp_hardening_data.hyp_vectors_slot, cpu);
 116                         break;
 117                 }
 118         }
 119
 120         if (slot == -1) {
 121                 slot = atomic_inc_return(&arm64_el2_vector_last_slot);
 122                 BUG_ON(slot >= BP_HARDEN_EL2_SLOTS);
 123                 __copy_hyp_vect_bpi(slot, hyp_vecs_start, hyp_vecs_end);
 124         }
 125
 126         __this_cpu_write(bp_hardening_data.hyp_vectors_slot, slot);
 127         __this_cpu_write(bp_hardening_data.fn, fn);
 128         spin_unlock(&bp_lock);
 129 }
 130 #else
 131 #define __smccc_workaround_1_smc_start          NULL
 132 #define __smccc_workaround_1_smc_end            NULL
 133
 134 static void __install_bp_hardening_cb(bp_hardening_cb_t fn,
 135                                       const char *hyp_vecs_start,
 136                                       const char *hyp_vecs_end)
 137 {
 138         __this_cpu_write(bp_hardening_data.fn, fn);
 139 }
 140 #endif  /* CONFIG_KVM_INDIRECT_VECTORS */
 141
 142 static void  install_bp_hardening_cb(const struct arm64_cpu_capabilities *entry,
 143                                      bp_hardening_cb_t fn,
 144                                      const char *hyp_vecs_start,
 145                                      const char *hyp_vecs_end)
 146 {
 147         u64 pfr0;
 148
 149         if (!entry->matches(entry, SCOPE_LOCAL_CPU))
 150                 return;
 151
 152         pfr0 = read_cpuid(ID_AA64PFR0_EL1);
 153         if (cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_CSV2_SHIFT))
 154                 return;
 155
 156         __install_bp_hardening_cb(fn, hyp_vecs_start, hyp_vecs_end);
 157 }
 158
 159 #include <uapi/linux/psci.h>
 160 #include <linux/arm-smccc.h>
 161 #include <linux/psci.h>
 162
 163 static void call_smc_arch_workaround_1(void)
 164 {
 165         arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
 166 }
 167
 168 static void call_hvc_arch_workaround_1(void)
 169 {
 170         arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
 171 }
 172
 173 static void qcom_link_stack_sanitization(void)
 174 {
 175         u64 tmp;
 176
 177         asm volatile("mov       %0, x30         \n"
 178                      ".rept     16              \n"
 179                      "bl        . + 4           \n"
 180                      ".endr                     \n"
 181                      "mov       x30, %0         \n"
 182                      : "=&r" (tmp));
 183 }
 184
 185 static void
 186 enable_smccc_arch_workaround_1(const struct arm64_cpu_capabilities *entry)
 187 {
 188         bp_hardening_cb_t cb;
 189         void *smccc_start, *smccc_end;
 190         struct arm_smccc_res res;
 191         u32 midr = read_cpuid_id();
 192
 193         if (!entry->matches(entry, SCOPE_LOCAL_CPU))
 194                 return;
 195
 196         if (psci_ops.smccc_version == SMCCC_VERSION_1_0)
 197                 return;
 198
 199         switch (psci_ops.conduit) {
 200         case PSCI_CONDUIT_HVC:
 201                 arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
 202                                   ARM_SMCCC_ARCH_WORKAROUND_1, &res);
 203                 if ((int)res.a0 < 0)
 204                         return;
 205                 cb = call_hvc_arch_workaround_1;
 206                 /* This is a guest, no need to patch KVM vectors */
 207                 smccc_start = NULL;
 208                 smccc_end = NULL;
 209                 break;
 210
 211         case PSCI_CONDUIT_SMC:
 212                 arm_smccc_1_1_smc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
 213                                   ARM_SMCCC_ARCH_WORKAROUND_1, &res);
 214                 if ((int)res.a0 < 0)
 215                         return;
 216                 cb = call_smc_arch_workaround_1;
 217                 smccc_start = __smccc_workaround_1_smc_start;
 218                 smccc_end = __smccc_workaround_1_smc_end;
 219                 break;
 220
 221         default:
 222                 return;
 223         }
 224
 225         if (((midr & MIDR_CPU_MODEL_MASK) == MIDR_QCOM_FALKOR) ||
 226             ((midr & MIDR_CPU_MODEL_MASK) == MIDR_QCOM_FALKOR_V1))
 227                 cb = qcom_link_stack_sanitization;
 228
 229         install_bp_hardening_cb(entry, cb, smccc_start, smccc_end);
 230
 231         return;
 232 }
 233 #endif  /* CONFIG_HARDEN_BRANCH_PREDICTOR */
 234
 235 #define CAP_MIDR_RANGE(model, v_min, r_min, v_max, r_max)       \
 236         .matches = is_affected_midr_range,                      \
 237         .midr_range = MIDR_RANGE(model, v_min, r_min, v_max, r_max)
 238
 239 #define CAP_MIDR_ALL_VERSIONS(model)                                    \
 240         .matches = is_affected_midr_range,                              \
 241         .midr_range = MIDR_ALL_VERSIONS(model)
 242
 243 #define MIDR_FIXED(rev, revidr_mask) \
 244         .fixed_revs = (struct arm64_midr_revidr[]){{ (rev), (revidr_mask) }, {}}
 245
 246 #define ERRATA_MIDR_RANGE(model, v_min, r_min, v_max, r_max)            \
 247         .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,                         \
 248         CAP_MIDR_RANGE(model, v_min, r_min, v_max, r_max)
 249
 250 #define CAP_MIDR_RANGE_LIST(list)                               \
 251         .matches = is_affected_midr_range_list,                 \
 252         .midr_range_list = list
 253
 254 /* Errata affecting a range of revisions of  given model variant */
 255 #define ERRATA_MIDR_REV_RANGE(m, var, r_min, r_max)      \
 256         ERRATA_MIDR_RANGE(m, var, r_min, var, r_max)
 257
 258 /* Errata affecting a single variant/revision of a model */
 259 #define ERRATA_MIDR_REV(model, var, rev)        \
 260         ERRATA_MIDR_RANGE(model, var, rev, var, rev)
 261
 262 /* Errata affecting all variants/revisions of a given a model */
 263 #define ERRATA_MIDR_ALL_VERSIONS(model)                         \
 264         .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,                 \
 265         CAP_MIDR_ALL_VERSIONS(model)
 266
 267 /* Errata affecting a list of midr ranges, with same work around */
 268 #define ERRATA_MIDR_RANGE_LIST(midr_list)                       \
 269         .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,                 \
 270         CAP_MIDR_RANGE_LIST(midr_list)
 271
 272 /*
 273  * Generic helper for handling capabilties with multiple (match,enable) pairs
 274  * of call backs, sharing the same capability bit.
 275  * Iterate over each entry to see if at least one matches.
 276  */
 277 static bool __maybe_unused
 278 multi_entry_cap_matches(const struct arm64_cpu_capabilities *entry, int scope)
 279 {
 280         const struct arm64_cpu_capabilities *caps;
 281
 282         for (caps = entry->match_list; caps->matches; caps++)
 283                 if (caps->matches(caps, scope))
 284                         return true;
 285
 286         return false;
 287 }
 288
 289 /*
 290  * Take appropriate action for all matching entries in the shared capability
 291  * entry.
 292  */
 293 static void __maybe_unused
 294 multi_entry_cap_cpu_enable(const struct arm64_cpu_capabilities *entry)
 295 {
 296         const struct arm64_cpu_capabilities *caps;
 297
 298         for (caps = entry->match_list; caps->matches; caps++)
 299                 if (caps->matches(caps, SCOPE_LOCAL_CPU) &&
 300                     caps->cpu_enable)
 301                         caps->cpu_enable(caps);
 302 }
 303
 304 #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
 305
 306 /*
 307  * List of CPUs where we need to issue a psci call to
 308  * harden the branch predictor.
 309  */
 310 static const struct midr_range arm64_bp_harden_smccc_cpus[] = {
 311         MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
 312         MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
 313         MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
 314         MIDR_ALL_VERSIONS(MIDR_CORTEX_A75),
 315         MIDR_ALL_VERSIONS(MIDR_BRCM_VULCAN),
 316         MIDR_ALL_VERSIONS(MIDR_CAVIUM_THUNDERX2),
 317         MIDR_ALL_VERSIONS(MIDR_QCOM_FALKOR_V1),
 318         MIDR_ALL_VERSIONS(MIDR_QCOM_FALKOR),
 319         MIDR_ALL_VERSIONS(MIDR_NVIDIA_DENVER),
 320         {},
 321 };
 322
 323 #endif
 324
 325 #ifdef CONFIG_HARDEN_EL2_VECTORS
 326
 327 static const struct midr_range arm64_harden_el2_vectors[] = {
 328         MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
 329         MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
 330         {},
 331 };
 332
 333 #endif
 334
 335 const struct arm64_cpu_capabilities arm64_errata[] = {
 336 #if     defined(CONFIG_ARM64_ERRATUM_826319) || \
 337         defined(CONFIG_ARM64_ERRATUM_827319) || \
 338         defined(CONFIG_ARM64_ERRATUM_824069)
 339         {
 340         /* Cortex-A53 r0p[012] */
 341                 .desc = "ARM errata 826319, 827319, 824069",
 342                 .capability = ARM64_WORKAROUND_CLEAN_CACHE,
 343                 ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 2),
 344                 .cpu_enable = cpu_enable_cache_maint_trap,
 345         },
 346 #endif
 347 #ifdef CONFIG_ARM64_ERRATUM_819472
 348         {
 349         /* Cortex-A53 r0p[01] */
 350                 .desc = "ARM errata 819472",
 351                 .capability = ARM64_WORKAROUND_CLEAN_CACHE,
 352                 ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 1),
 353                 .cpu_enable = cpu_enable_cache_maint_trap,
 354         },
 355 #endif
 356 #ifdef CONFIG_ARM64_ERRATUM_832075
 357         {
 358         /* Cortex-A57 r0p0 - r1p2 */
 359                 .desc = "ARM erratum 832075",
 360                 .capability = ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE,
 361                 ERRATA_MIDR_RANGE(MIDR_CORTEX_A57,
 362                                   0, 0,
 363                                   1, 2),
 364         },
 365 #endif
 366 #ifdef CONFIG_ARM64_ERRATUM_834220
 367         {
 368         /* Cortex-A57 r0p0 - r1p2 */
 369                 .desc = "ARM erratum 834220",
 370                 .capability = ARM64_WORKAROUND_834220,
 371                 ERRATA_MIDR_RANGE(MIDR_CORTEX_A57,
 372                                   0, 0,
 373                                   1, 2),
 374         },
 375 #endif
 376 #ifdef CONFIG_ARM64_ERRATUM_843419
 377         {
 378         /* Cortex-A53 r0p[01234] */
 379                 .desc = "ARM erratum 843419",
 380                 .capability = ARM64_WORKAROUND_843419,
 381                 ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 4),
 382                 MIDR_FIXED(0x4, BIT(8)),
 383         },
 384 #endif
 385 #ifdef CONFIG_ARM64_ERRATUM_845719
 386         {
 387         /* Cortex-A53 r0p[01234] */
 388                 .desc = "ARM erratum 845719",
 389                 .capability = ARM64_WORKAROUND_845719,
 390                 ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 4),
 391         },
 392 #endif
 393 #ifdef CONFIG_CAVIUM_ERRATUM_23154
 394         {
 395         /* Cavium ThunderX, pass 1.x */
 396                 .desc = "Cavium erratum 23154",
 397                 .capability = ARM64_WORKAROUND_CAVIUM_23154,
 398                 ERRATA_MIDR_REV_RANGE(MIDR_THUNDERX, 0, 0, 1),
 399         },
 400 #endif
 401 #ifdef CONFIG_CAVIUM_ERRATUM_27456
 402         {
 403         /* Cavium ThunderX, T88 pass 1.x - 2.1 */
 404                 .desc = "Cavium erratum 27456",
 405                 .capability = ARM64_WORKAROUND_CAVIUM_27456,
 406                 ERRATA_MIDR_RANGE(MIDR_THUNDERX,
 407                                   0, 0,
 408                                   1, 1),
 409         },
 410         {
 411         /* Cavium ThunderX, T81 pass 1.0 */
 412                 .desc = "Cavium erratum 27456",
 413                 .capability = ARM64_WORKAROUND_CAVIUM_27456,
 414                 ERRATA_MIDR_REV(MIDR_THUNDERX_81XX, 0, 0),
 415         },
 416 #endif
 417 #ifdef CONFIG_CAVIUM_ERRATUM_30115
 418         {
 419         /* Cavium ThunderX, T88 pass 1.x - 2.2 */
 420                 .desc = "Cavium erratum 30115",
 421                 .capability = ARM64_WORKAROUND_CAVIUM_30115,
 422                 ERRATA_MIDR_RANGE(MIDR_THUNDERX,
 423                                       0, 0,
 424                                       1, 2),
 425         },
 426         {
 427         /* Cavium ThunderX, T81 pass 1.0 - 1.2 */
 428                 .desc = "Cavium erratum 30115",
 429                 .capability = ARM64_WORKAROUND_CAVIUM_30115,
 430                 ERRATA_MIDR_REV_RANGE(MIDR_THUNDERX_81XX, 0, 0, 2),
 431         },
 432         {
 433         /* Cavium ThunderX, T83 pass 1.0 */
 434                 .desc = "Cavium erratum 30115",
 435                 .capability = ARM64_WORKAROUND_CAVIUM_30115,
 436                 ERRATA_MIDR_REV(MIDR_THUNDERX_83XX, 0, 0),
 437         },
 438 #endif
 439         {
 440                 .desc = "Mismatched cache line size",
 441                 .capability = ARM64_MISMATCHED_CACHE_LINE_SIZE,
 442                 .matches = has_mismatched_cache_line_size,
 443                 .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
 444                 .cpu_enable = cpu_enable_trap_ctr_access,
 445         },
 446 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1003
 447         {
 448                 .desc = "Qualcomm Technologies Falkor erratum 1003",
 449                 .capability = ARM64_WORKAROUND_QCOM_FALKOR_E1003,
 450                 ERRATA_MIDR_REV(MIDR_QCOM_FALKOR_V1, 0, 0),
 451         },
 452         {
 453                 .desc = "Qualcomm Technologies Kryo erratum 1003",
 454                 .capability = ARM64_WORKAROUND_QCOM_FALKOR_E1003,
 455                 .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
 456                 .midr_range.model = MIDR_QCOM_KRYO,
 457                 .matches = is_kryo_midr,
 458         },
 459 #endif
 460 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1009
 461         {
 462                 .desc = "Qualcomm Technologies Falkor erratum 1009",
 463                 .capability = ARM64_WORKAROUND_REPEAT_TLBI,
 464                 ERRATA_MIDR_REV(MIDR_QCOM_FALKOR_V1, 0, 0),
 465         },
 466 #endif
 467 #ifdef CONFIG_ARM64_ERRATUM_858921
 468         {
 469         /* Cortex-A73 all versions */
 470                 .desc = "ARM erratum 858921",
 471                 .capability = ARM64_WORKAROUND_858921,
 472                 ERRATA_MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
 473         },
 474 #endif
 475 #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
 476         {
 477                 .capability = ARM64_HARDEN_BRANCH_PREDICTOR,
 478                 .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
 479                 .cpu_enable = enable_smccc_arch_workaround_1,
 480                 ERRATA_MIDR_RANGE_LIST(arm64_bp_harden_smccc_cpus),
 481         },
 482 #endif
 483 #ifdef CONFIG_HARDEN_EL2_VECTORS
 484         {
 485                 .desc = "EL2 vector hardening",
 486                 .capability = ARM64_HARDEN_EL2_VECTORS,
 487                 .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
 488                 ERRATA_MIDR_RANGE_LIST(arm64_harden_el2_vectors),
 489         },
 490 #endif
 491         {
 492         }
 493 };