arch/arm/mm/proc-v7.S

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  *  linux/arch/arm/mm/proc-v7.S
   4  *
   5  *  Copyright (C) 2001 Deep Blue Solutions Ltd.
   6  *
   7  *  This is the "shell" of the ARMv7 processor support.
   8  */
   9 #include <linux/arm-smccc.h>
  10 #include <linux/cfi_types.h>
  11 #include <linux/init.h>
  12 #include <linux/linkage.h>
  13 #include <linux/pgtable.h>
  14 #include <asm/assembler.h>
  15 #include <asm/asm-offsets.h>
  16 #include <asm/hwcap.h>
  17 #include <asm/pgtable-hwdef.h>
  18 #include <asm/page.h>
  19
  20 #include "proc-macros.S"
  21
  22 #ifdef CONFIG_ARM_LPAE
  23 #include "proc-v7-3level.S"
  24 #else
  25 #include "proc-v7-2level.S"
  26 #endif
  27
  28 .arch armv7-a
  29
  30 SYM_TYPED_FUNC_START(cpu_v7_proc_init)
  31         ret     lr
  32 SYM_FUNC_END(cpu_v7_proc_init)
  33
  34 SYM_TYPED_FUNC_START(cpu_v7_proc_fin)
  35         mrc     p15, 0, r0, c1, c0, 0           @ ctrl register
  36         bic     r0, r0, #0x1000                 @ ...i............
  37         bic     r0, r0, #0x0006                 @ .............ca.
  38         mcr     p15, 0, r0, c1, c0, 0           @ disable caches
  39         ret     lr
  40 SYM_FUNC_END(cpu_v7_proc_fin)
  41
  42 /*
  43  *      cpu_v7_reset(loc, hyp)
  44  *
  45  *      Perform a soft reset of the system.  Put the CPU into the
  46  *      same state as it would be if it had been reset, and branch
  47  *      to what would be the reset vector.
  48  *
  49  *      - loc   - location to jump to for soft reset
  50  *      - hyp   - indicate if restart occurs in HYP mode
  51  *
  52  *      This code must be executed using a flat identity mapping with
  53  *      caches disabled.
  54  */
  55         .align  5
  56         .pushsection    .idmap.text, "ax"
  57 SYM_TYPED_FUNC_START(cpu_v7_reset)
  58         mrc     p15, 0, r2, c1, c0, 0           @ ctrl register
  59         bic     r2, r2, #0x1                    @ ...............m
  60  THUMB( bic     r2, r2, #1 << 30 )              @ SCTLR.TE (Thumb exceptions)
  61         mcr     p15, 0, r2, c1, c0, 0           @ disable MMU
  62         isb
  63 #ifdef CONFIG_ARM_VIRT_EXT
  64         teq     r1, #0
  65         bne     __hyp_soft_restart
  66 #endif
  67         bx      r0
  68 SYM_FUNC_END(cpu_v7_reset)
  69         .popsection
  70
  71 /*
  72  *      cpu_v7_do_idle()
  73  *
  74  *      Idle the processor (eg, wait for interrupt).
  75  *
  76  *      IRQs are already disabled.
  77  */
  78 SYM_TYPED_FUNC_START(cpu_v7_do_idle)
  79         dsb                                     @ WFI may enter a low-power mode
  80         wfi
  81         ret     lr
  82 SYM_FUNC_END(cpu_v7_do_idle)
  83
  84 SYM_TYPED_FUNC_START(cpu_v7_dcache_clean_area)
  85         ALT_SMP(W(nop))                 @ MP extensions imply L1 PTW
  86         ALT_UP_B(1f)
  87         ret     lr
  88 1:      dcache_line_size r2, r3
  89 2:      mcr     p15, 0, r0, c7, c10, 1          @ clean D entry
  90         add     r0, r0, r2
  91         subs    r1, r1, r2
  92         bhi     2b
  93         dsb     ishst
  94         ret     lr
  95 SYM_FUNC_END(cpu_v7_dcache_clean_area)
  96
  97 #if defined(CONFIG_ARM_PSCI) && defined(CONFIG_HARDEN_BRANCH_PREDICTOR)
  98         .arch_extension sec
  99 SYM_TYPED_FUNC_START(cpu_v7_smc_switch_mm)
 100         stmfd   sp!, {r0 - r3}
 101         movw    r0, #:lower16:ARM_SMCCC_ARCH_WORKAROUND_1
 102         movt    r0, #:upper16:ARM_SMCCC_ARCH_WORKAROUND_1
 103         smc     #0
 104         ldmfd   sp!, {r0 - r3}
 105         b       cpu_v7_switch_mm
 106 SYM_FUNC_END(cpu_v7_smc_switch_mm)
 107         .arch_extension virt
 108 SYM_TYPED_FUNC_START(cpu_v7_hvc_switch_mm)
 109         stmfd   sp!, {r0 - r3}
 110         movw    r0, #:lower16:ARM_SMCCC_ARCH_WORKAROUND_1
 111         movt    r0, #:upper16:ARM_SMCCC_ARCH_WORKAROUND_1
 112         hvc     #0
 113         ldmfd   sp!, {r0 - r3}
 114         b       cpu_v7_switch_mm
 115 SYM_FUNC_END(cpu_v7_hvc_switch_mm)
 116 #endif
 117
 118 SYM_TYPED_FUNC_START(cpu_v7_iciallu_switch_mm)
 119         mov     r3, #0
 120         mcr     p15, 0, r3, c7, c5, 0           @ ICIALLU
 121         b       cpu_v7_switch_mm
 122 SYM_FUNC_END(cpu_v7_iciallu_switch_mm)
 123 SYM_TYPED_FUNC_START(cpu_v7_bpiall_switch_mm)
 124         mov     r3, #0
 125         mcr     p15, 0, r3, c7, c5, 6           @ flush BTAC/BTB
 126         b       cpu_v7_switch_mm
 127 SYM_FUNC_END(cpu_v7_bpiall_switch_mm)
 128
 129         string  cpu_v7_name, "ARMv7 Processor"
 130         .align
 131
 132 /* Suspend/resume support: derived from arch/arm/mach-s5pv210/sleep.S */
 133 .globl  cpu_v7_suspend_size
 134 .equ    cpu_v7_suspend_size, 4 * 9
 135 #ifdef CONFIG_ARM_CPU_SUSPEND
 136 SYM_TYPED_FUNC_START(cpu_v7_do_suspend)
 137         stmfd   sp!, {r4 - r11, lr}
 138         mrc     p15, 0, r4, c13, c0, 0  @ FCSE/PID
 139         mrc     p15, 0, r5, c13, c0, 3  @ User r/o thread ID
 140         stmia   r0!, {r4 - r5}
 141 #ifdef CONFIG_MMU
 142         mrc     p15, 0, r6, c3, c0, 0   @ Domain ID
 143 #ifdef CONFIG_ARM_LPAE
 144         mrrc    p15, 1, r5, r7, c2      @ TTB 1
 145 #else
 146         mrc     p15, 0, r7, c2, c0, 1   @ TTB 1
 147 #endif
 148         mrc     p15, 0, r11, c2, c0, 2  @ TTB control register
 149 #endif
 150         mrc     p15, 0, r8, c1, c0, 0   @ Control register
 151         mrc     p15, 0, r9, c1, c0, 1   @ Auxiliary control register
 152         mrc     p15, 0, r10, c1, c0, 2  @ Co-processor access control
 153         stmia   r0, {r5 - r11}
 154         ldmfd   sp!, {r4 - r11, pc}
 155 SYM_FUNC_END(cpu_v7_do_suspend)
 156
 157 SYM_TYPED_FUNC_START(cpu_v7_do_resume)
 158         mov     ip, #0
 159         mcr     p15, 0, ip, c7, c5, 0   @ invalidate I cache
 160         mcr     p15, 0, ip, c13, c0, 1  @ set reserved context ID
 161         ldmia   r0!, {r4 - r5}
 162         mcr     p15, 0, r4, c13, c0, 0  @ FCSE/PID
 163         mcr     p15, 0, r5, c13, c0, 3  @ User r/o thread ID
 164         ldmia   r0, {r5 - r11}
 165 #ifdef CONFIG_MMU
 166         mcr     p15, 0, ip, c8, c7, 0   @ invalidate TLBs
 167         mcr     p15, 0, r6, c3, c0, 0   @ Domain ID
 168 #ifdef CONFIG_ARM_LPAE
 169         mcrr    p15, 0, r1, ip, c2      @ TTB 0
 170         mcrr    p15, 1, r5, r7, c2      @ TTB 1
 171 #else
 172         ALT_SMP(orr     r1, r1, #TTB_FLAGS_SMP)
 173         ALT_UP(orr      r1, r1, #TTB_FLAGS_UP)
 174         mcr     p15, 0, r1, c2, c0, 0   @ TTB 0
 175         mcr     p15, 0, r7, c2, c0, 1   @ TTB 1
 176 #endif
 177         mcr     p15, 0, r11, c2, c0, 2  @ TTB control register
 178         ldr     r4, =PRRR               @ PRRR
 179         ldr     r5, =NMRR               @ NMRR
 180         mcr     p15, 0, r4, c10, c2, 0  @ write PRRR
 181         mcr     p15, 0, r5, c10, c2, 1  @ write NMRR
 182 #endif  /* CONFIG_MMU */
 183         mrc     p15, 0, r4, c1, c0, 1   @ Read Auxiliary control register
 184         teq     r4, r9                  @ Is it already set?
 185         mcrne   p15, 0, r9, c1, c0, 1   @ No, so write it
 186         mcr     p15, 0, r10, c1, c0, 2  @ Co-processor access control
 187         isb
 188         dsb
 189         mov     r0, r8                  @ control register
 190         b       cpu_resume_mmu
 191 SYM_FUNC_END(cpu_v7_do_resume)
 192 #endif
 193
 194 .globl  cpu_ca9mp_suspend_size
 195 .equ    cpu_ca9mp_suspend_size, cpu_v7_suspend_size + 4 * 2
 196 #ifdef CONFIG_ARM_CPU_SUSPEND
 197 SYM_TYPED_FUNC_START(cpu_ca9mp_do_suspend)
 198         stmfd   sp!, {r4 - r5}
 199         mrc     p15, 0, r4, c15, c0, 1          @ Diagnostic register
 200         mrc     p15, 0, r5, c15, c0, 0          @ Power register
 201         stmia   r0!, {r4 - r5}
 202         ldmfd   sp!, {r4 - r5}
 203         b       cpu_v7_do_suspend
 204 SYM_FUNC_END(cpu_ca9mp_do_suspend)
 205
 206 SYM_TYPED_FUNC_START(cpu_ca9mp_do_resume)
 207         ldmia   r0!, {r4 - r5}
 208         mrc     p15, 0, r10, c15, c0, 1         @ Read Diagnostic register
 209         teq     r4, r10                         @ Already restored?
 210         mcrne   p15, 0, r4, c15, c0, 1          @ No, so restore it
 211         mrc     p15, 0, r10, c15, c0, 0         @ Read Power register
 212         teq     r5, r10                         @ Already restored?
 213         mcrne   p15, 0, r5, c15, c0, 0          @ No, so restore it
 214         b       cpu_v7_do_resume
 215 SYM_FUNC_END(cpu_ca9mp_do_resume)
 216 #endif
 217
 218 #ifdef CONFIG_CPU_PJ4B
 219         globl_equ       cpu_pj4b_switch_mm,     cpu_v7_switch_mm
 220         globl_equ       cpu_pj4b_set_pte_ext,   cpu_v7_set_pte_ext
 221         globl_equ       cpu_pj4b_proc_init,     cpu_v7_proc_init
 222         globl_equ       cpu_pj4b_proc_fin,      cpu_v7_proc_fin
 223         globl_equ       cpu_pj4b_reset,         cpu_v7_reset
 224 #ifdef CONFIG_PJ4B_ERRATA_4742
 225 SYM_TYPED_FUNC_START(cpu_pj4b_do_idle)
 226         dsb                                     @ WFI may enter a low-power mode
 227         wfi
 228         dsb                                     @barrier
 229         ret     lr
 230 SYM_FUNC_END(cpu_pj4b_do_idle)
 231 #else
 232         globl_equ       cpu_pj4b_do_idle,       cpu_v7_do_idle
 233 #endif
 234         globl_equ       cpu_pj4b_dcache_clean_area,     cpu_v7_dcache_clean_area
 235 #ifdef CONFIG_ARM_CPU_SUSPEND
 236 SYM_TYPED_FUNC_START(cpu_pj4b_do_suspend)
 237         stmfd   sp!, {r6 - r10}
 238         mrc     p15, 1, r6, c15, c1, 0  @ save CP15 - extra features
 239         mrc     p15, 1, r7, c15, c2, 0  @ save CP15 - Aux Func Modes Ctrl 0
 240         mrc     p15, 1, r8, c15, c1, 2  @ save CP15 - Aux Debug Modes Ctrl 2
 241         mrc     p15, 1, r9, c15, c1, 1  @ save CP15 - Aux Debug Modes Ctrl 1
 242         mrc     p15, 0, r10, c9, c14, 0  @ save CP15 - PMC
 243         stmia   r0!, {r6 - r10}
 244         ldmfd   sp!, {r6 - r10}
 245         b cpu_v7_do_suspend
 246 SYM_FUNC_END(cpu_pj4b_do_suspend)
 247
 248 SYM_TYPED_FUNC_START(cpu_pj4b_do_resume)
 249         ldmia   r0!, {r6 - r10}
 250         mcr     p15, 1, r6, c15, c1, 0  @ restore CP15 - extra features
 251         mcr     p15, 1, r7, c15, c2, 0  @ restore CP15 - Aux Func Modes Ctrl 0
 252         mcr     p15, 1, r8, c15, c1, 2  @ restore CP15 - Aux Debug Modes Ctrl 2
 253         mcr     p15, 1, r9, c15, c1, 1  @ restore CP15 - Aux Debug Modes Ctrl 1
 254         mcr     p15, 0, r10, c9, c14, 0  @ restore CP15 - PMC
 255         b cpu_v7_do_resume
 256 SYM_FUNC_END(cpu_pj4b_do_resume)
 257 #endif
 258 .globl  cpu_pj4b_suspend_size
 259 .equ    cpu_pj4b_suspend_size, cpu_v7_suspend_size + 4 * 5
 260
 261 #endif
 262
 263         @
 264         @ Invoke the v7_invalidate_l1() function, which adheres to the AAPCS
 265         @ rules, and so it may corrupt registers that we need to preserve.
 266         @
 267         .macro  do_invalidate_l1
 268         mov     r6, r1
 269         mov     r7, r2
 270         mov     r10, lr
 271         bl      v7_invalidate_l1                @ corrupts {r0-r3, ip, lr}
 272         mov     r1, r6
 273         mov     r2, r7
 274         mov     lr, r10
 275         .endm
 276
 277 /*
 278  *      __v7_setup
 279  *
 280  *      Initialise TLB, Caches, and MMU state ready to switch the MMU
 281  *      on.  Return in r0 the new CP15 C1 control register setting.
 282  *
 283  *      r1, r2, r4, r5, r9, r13 must be preserved - r13 is not a stack
 284  *      r4: TTBR0 (low word)
 285  *      r5: TTBR0 (high word if LPAE)
 286  *      r8: TTBR1
 287  *      r9: Main ID register
 288  *
 289  *      This should be able to cover all ARMv7 cores.
 290  *
 291  *      It is assumed that:
 292  *      - cache type register is implemented
 293  */
 294 __v7_ca5mp_setup:
 295 __v7_ca9mp_setup:
 296 __v7_cr7mp_setup:
 297 __v7_cr8mp_setup:
 298         do_invalidate_l1
 299         mov     r10, #(1 << 0)                  @ Cache/TLB ops broadcasting
 300         b       1f
 301 __v7_ca7mp_setup:
 302 __v7_ca12mp_setup:
 303 __v7_ca15mp_setup:
 304 __v7_b15mp_setup:
 305 __v7_ca17mp_setup:
 306         do_invalidate_l1
 307         mov     r10, #0
 308 1:
 309 #ifdef CONFIG_SMP
 310         orr     r10, r10, #(1 << 6)             @ Enable SMP/nAMP mode
 311         ALT_SMP(mrc     p15, 0, r0, c1, c0, 1)
 312         ALT_UP(mov      r0, r10)                @ fake it for UP
 313         orr     r10, r10, r0                    @ Set required bits
 314         teq     r10, r0                         @ Were they already set?
 315         mcrne   p15, 0, r10, c1, c0, 1          @ No, update register
 316 #endif
 317         b       __v7_setup_cont
 318
 319 /*
 320  * Errata:
 321  *  r0, r10 available for use
 322  *  r1, r2, r4, r5, r9, r13: must be preserved
 323  *  r3: contains MIDR rX number in bits 23-20
 324  *  r6: contains MIDR rXpY as 8-bit XY number
 325  *  r9: MIDR
 326  */
 327 __ca8_errata:
 328 #if defined(CONFIG_ARM_ERRATA_430973) && !defined(CONFIG_ARCH_MULTIPLATFORM)
 329         teq     r3, #0x00100000                 @ only present in r1p*
 330         mrceq   p15, 0, r0, c1, c0, 1           @ read aux control register
 331         orreq   r0, r0, #(1 << 6)               @ set IBE to 1
 332         mcreq   p15, 0, r0, c1, c0, 1           @ write aux control register
 333 #endif
 334 #ifdef CONFIG_ARM_ERRATA_458693
 335         teq     r6, #0x20                       @ only present in r2p0
 336         mrceq   p15, 0, r0, c1, c0, 1           @ read aux control register
 337         orreq   r0, r0, #(1 << 5)               @ set L1NEON to 1
 338         orreq   r0, r0, #(1 << 9)               @ set PLDNOP to 1
 339         mcreq   p15, 0, r0, c1, c0, 1           @ write aux control register
 340 #endif
 341 #ifdef CONFIG_ARM_ERRATA_460075
 342         teq     r6, #0x20                       @ only present in r2p0
 343         mrceq   p15, 1, r0, c9, c0, 2           @ read L2 cache aux ctrl register
 344         tsteq   r0, #1 << 22
 345         orreq   r0, r0, #(1 << 22)              @ set the Write Allocate disable bit
 346         mcreq   p15, 1, r0, c9, c0, 2           @ write the L2 cache aux ctrl register
 347 #endif
 348         b       __errata_finish
 349
 350 __ca9_errata:
 351 #ifdef CONFIG_ARM_ERRATA_742230
 352         cmp     r6, #0x22                       @ only present up to r2p2
 353         mrcle   p15, 0, r0, c15, c0, 1          @ read diagnostic register
 354         orrle   r0, r0, #1 << 4                 @ set bit #4
 355         mcrle   p15, 0, r0, c15, c0, 1          @ write diagnostic register
 356 #endif
 357 #ifdef CONFIG_ARM_ERRATA_742231
 358         teq     r6, #0x20                       @ present in r2p0
 359         teqne   r6, #0x21                       @ present in r2p1
 360         teqne   r6, #0x22                       @ present in r2p2
 361         mrceq   p15, 0, r0, c15, c0, 1          @ read diagnostic register
 362         orreq   r0, r0, #1 << 12                @ set bit #12
 363         orreq   r0, r0, #1 << 22                @ set bit #22
 364         mcreq   p15, 0, r0, c15, c0, 1          @ write diagnostic register
 365 #endif
 366 #ifdef CONFIG_ARM_ERRATA_743622
 367         teq     r3, #0x00200000                 @ only present in r2p*
 368         mrceq   p15, 0, r0, c15, c0, 1          @ read diagnostic register
 369         orreq   r0, r0, #1 << 6                 @ set bit #6
 370         mcreq   p15, 0, r0, c15, c0, 1          @ write diagnostic register
 371 #endif
 372 #if defined(CONFIG_ARM_ERRATA_751472) && defined(CONFIG_SMP)
 373         ALT_SMP(cmp r6, #0x30)                  @ present prior to r3p0
 374         ALT_UP_B(1f)
 375         mrclt   p15, 0, r0, c15, c0, 1          @ read diagnostic register
 376         orrlt   r0, r0, #1 << 11                @ set bit #11
 377         mcrlt   p15, 0, r0, c15, c0, 1          @ write diagnostic register
 378 1:
 379 #endif
 380         b       __errata_finish
 381
 382 __ca15_errata:
 383 #ifdef CONFIG_ARM_ERRATA_773022
 384         cmp     r6, #0x4                        @ only present up to r0p4
 385         mrcle   p15, 0, r0, c1, c0, 1           @ read aux control register
 386         orrle   r0, r0, #1 << 1                 @ disable loop buffer
 387         mcrle   p15, 0, r0, c1, c0, 1           @ write aux control register
 388 #endif
 389         b       __errata_finish
 390
 391 __ca12_errata:
 392 #ifdef CONFIG_ARM_ERRATA_818325_852422
 393         mrc     p15, 0, r10, c15, c0, 1         @ read diagnostic register
 394         orr     r10, r10, #1 << 12              @ set bit #12
 395         mcr     p15, 0, r10, c15, c0, 1         @ write diagnostic register
 396 #endif
 397 #ifdef CONFIG_ARM_ERRATA_821420
 398         mrc     p15, 0, r10, c15, c0, 2         @ read internal feature reg
 399         orr     r10, r10, #1 << 1               @ set bit #1
 400         mcr     p15, 0, r10, c15, c0, 2         @ write internal feature reg
 401 #endif
 402 #ifdef CONFIG_ARM_ERRATA_825619
 403         mrc     p15, 0, r10, c15, c0, 1         @ read diagnostic register
 404         orr     r10, r10, #1 << 24              @ set bit #24
 405         mcr     p15, 0, r10, c15, c0, 1         @ write diagnostic register
 406 #endif
 407 #ifdef CONFIG_ARM_ERRATA_857271
 408         mrc     p15, 0, r10, c15, c0, 1         @ read diagnostic register
 409         orr     r10, r10, #3 << 10              @ set bits #10 and #11
 410         mcr     p15, 0, r10, c15, c0, 1         @ write diagnostic register
 411 #endif
 412         b       __errata_finish
 413
 414 __ca17_errata:
 415 #ifdef CONFIG_ARM_ERRATA_852421
 416         cmp     r6, #0x12                       @ only present up to r1p2
 417         mrcle   p15, 0, r10, c15, c0, 1         @ read diagnostic register
 418         orrle   r10, r10, #1 << 24              @ set bit #24
 419         mcrle   p15, 0, r10, c15, c0, 1         @ write diagnostic register
 420 #endif
 421 #ifdef CONFIG_ARM_ERRATA_852423
 422         cmp     r6, #0x12                       @ only present up to r1p2
 423         mrcle   p15, 0, r10, c15, c0, 1         @ read diagnostic register
 424         orrle   r10, r10, #1 << 12              @ set bit #12
 425         mcrle   p15, 0, r10, c15, c0, 1         @ write diagnostic register
 426 #endif
 427 #ifdef CONFIG_ARM_ERRATA_857272
 428         mrc     p15, 0, r10, c15, c0, 1         @ read diagnostic register
 429         orr     r10, r10, #3 << 10              @ set bits #10 and #11
 430         mcr     p15, 0, r10, c15, c0, 1         @ write diagnostic register
 431 #endif
 432         b       __errata_finish
 433
 434 __v7_pj4b_setup:
 435 #ifdef CONFIG_CPU_PJ4B
 436
 437 /* Auxiliary Debug Modes Control 1 Register */
 438 #define PJ4B_STATIC_BP (1 << 2) /* Enable Static BP */
 439 #define PJ4B_INTER_PARITY (1 << 8) /* Disable Internal Parity Handling */
 440 #define PJ4B_CLEAN_LINE (1 << 16) /* Disable data transfer for clean line */
 441
 442 /* Auxiliary Debug Modes Control 2 Register */
 443 #define PJ4B_FAST_LDR (1 << 23) /* Disable fast LDR */
 444 #define PJ4B_SNOOP_DATA (1 << 25) /* Do not interleave write and snoop data */
 445 #define PJ4B_CWF (1 << 27) /* Disable Critical Word First feature */
 446 #define PJ4B_OUTSDNG_NC (1 << 29) /* Disable outstanding non cacheable rqst */
 447 #define PJ4B_L1_REP_RR (1 << 30) /* L1 replacement - Strict round robin */
 448 #define PJ4B_AUX_DBG_CTRL2 (PJ4B_SNOOP_DATA | PJ4B_CWF |\
 449                             PJ4B_OUTSDNG_NC | PJ4B_L1_REP_RR)
 450
 451 /* Auxiliary Functional Modes Control Register 0 */
 452 #define PJ4B_SMP_CFB (1 << 1) /* Set SMP mode. Join the coherency fabric */
 453 #define PJ4B_L1_PAR_CHK (1 << 2) /* Support L1 parity checking */
 454 #define PJ4B_BROADCAST_CACHE (1 << 8) /* Broadcast Cache and TLB maintenance */
 455
 456 /* Auxiliary Debug Modes Control 0 Register */
 457 #define PJ4B_WFI_WFE (1 << 22) /* WFI/WFE - serve the DVM and back to idle */
 458
 459         /* Auxiliary Debug Modes Control 1 Register */
 460         mrc     p15, 1, r0, c15, c1, 1
 461         orr     r0, r0, #PJ4B_CLEAN_LINE
 462         orr     r0, r0, #PJ4B_INTER_PARITY
 463         bic     r0, r0, #PJ4B_STATIC_BP
 464         mcr     p15, 1, r0, c15, c1, 1
 465
 466         /* Auxiliary Debug Modes Control 2 Register */
 467         mrc     p15, 1, r0, c15, c1, 2
 468         bic     r0, r0, #PJ4B_FAST_LDR
 469         orr     r0, r0, #PJ4B_AUX_DBG_CTRL2
 470         mcr     p15, 1, r0, c15, c1, 2
 471
 472         /* Auxiliary Functional Modes Control Register 0 */
 473         mrc     p15, 1, r0, c15, c2, 0
 474 #ifdef CONFIG_SMP
 475         orr     r0, r0, #PJ4B_SMP_CFB
 476 #endif
 477         orr     r0, r0, #PJ4B_L1_PAR_CHK
 478         orr     r0, r0, #PJ4B_BROADCAST_CACHE
 479         mcr     p15, 1, r0, c15, c2, 0
 480
 481         /* Auxiliary Debug Modes Control 0 Register */
 482         mrc     p15, 1, r0, c15, c1, 0
 483         orr     r0, r0, #PJ4B_WFI_WFE
 484         mcr     p15, 1, r0, c15, c1, 0
 485
 486 #endif /* CONFIG_CPU_PJ4B */
 487
 488 __v7_setup:
 489         do_invalidate_l1
 490
 491 __v7_setup_cont:
 492         and     r0, r9, #0xff000000             @ ARM?
 493         teq     r0, #0x41000000
 494         bne     __errata_finish
 495         and     r3, r9, #0x00f00000             @ variant
 496         and     r6, r9, #0x0000000f             @ revision
 497         orr     r6, r6, r3, lsr #20-4           @ combine variant and revision
 498         ubfx    r0, r9, #4, #12                 @ primary part number
 499
 500         /* Cortex-A8 Errata */
 501         ldr     r10, =0x00000c08                @ Cortex-A8 primary part number
 502         teq     r0, r10
 503         beq     __ca8_errata
 504
 505         /* Cortex-A9 Errata */
 506         ldr     r10, =0x00000c09                @ Cortex-A9 primary part number
 507         teq     r0, r10
 508         beq     __ca9_errata
 509
 510         /* Cortex-A12 Errata */
 511         ldr     r10, =0x00000c0d                @ Cortex-A12 primary part number
 512         teq     r0, r10
 513         beq     __ca12_errata
 514
 515         /* Cortex-A17 Errata */
 516         ldr     r10, =0x00000c0e                @ Cortex-A17 primary part number
 517         teq     r0, r10
 518         beq     __ca17_errata
 519
 520         /* Cortex-A15 Errata */
 521         ldr     r10, =0x00000c0f                @ Cortex-A15 primary part number
 522         teq     r0, r10
 523         beq     __ca15_errata
 524
 525 __errata_finish:
 526         mov     r10, #0
 527         mcr     p15, 0, r10, c7, c5, 0          @ I+BTB cache invalidate
 528 #ifdef CONFIG_MMU
 529         mcr     p15, 0, r10, c8, c7, 0          @ invalidate I + D TLBs
 530         v7_ttb_setup r10, r4, r5, r8, r3        @ TTBCR, TTBRx setup
 531         ldr     r3, =PRRR                       @ PRRR
 532         ldr     r6, =NMRR                       @ NMRR
 533         mcr     p15, 0, r3, c10, c2, 0          @ write PRRR
 534         mcr     p15, 0, r6, c10, c2, 1          @ write NMRR
 535 #endif
 536         dsb                                     @ Complete invalidations
 537 #ifndef CONFIG_ARM_THUMBEE
 538         mrc     p15, 0, r0, c0, c1, 0           @ read ID_PFR0 for ThumbEE
 539         and     r0, r0, #(0xf << 12)            @ ThumbEE enabled field
 540         teq     r0, #(1 << 12)                  @ check if ThumbEE is present
 541         bne     1f
 542         mov     r3, #0
 543         mcr     p14, 6, r3, c1, c0, 0           @ Initialize TEEHBR to 0
 544         mrc     p14, 6, r0, c0, c0, 0           @ load TEECR
 545         orr     r0, r0, #1                      @ set the 1st bit in order to
 546         mcr     p14, 6, r0, c0, c0, 0           @ stop userspace TEEHBR access
 547 1:
 548 #endif
 549         adr     r3, v7_crval
 550         ldmia   r3, {r3, r6}
 551  ARM_BE8(orr    r6, r6, #1 << 25)               @ big-endian page tables
 552 #ifdef CONFIG_SWP_EMULATE
 553         orr     r3, r3, #(1 << 10)              @ set SW bit in "clear"
 554         bic     r6, r6, #(1 << 10)              @ clear it in "mmuset"
 555 #endif
 556         mrc     p15, 0, r0, c1, c0, 0           @ read control register
 557         bic     r0, r0, r3                      @ clear bits them
 558         orr     r0, r0, r6                      @ set them
 559  THUMB( orr     r0, r0, #1 << 30        )       @ Thumb exceptions
 560         ret     lr                              @ return to head.S:__ret
 561 ENDPROC(__v7_setup)
 562
 563         __INITDATA
 564
 565         .weak cpu_v7_bugs_init
 566
 567         @ define struct processor (see <asm/proc-fns.h> and proc-macros.S)
 568         define_processor_functions v7, dabort=v7_early_abort, pabort=v7_pabort, suspend=1, bugs=cpu_v7_bugs_init
 569
 570 #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
 571         @ generic v7 bpiall on context switch
 572         globl_equ       cpu_v7_bpiall_proc_init,        cpu_v7_proc_init
 573         globl_equ       cpu_v7_bpiall_proc_fin,         cpu_v7_proc_fin
 574         globl_equ       cpu_v7_bpiall_reset,            cpu_v7_reset
 575         globl_equ       cpu_v7_bpiall_do_idle,          cpu_v7_do_idle
 576         globl_equ       cpu_v7_bpiall_dcache_clean_area, cpu_v7_dcache_clean_area
 577         globl_equ       cpu_v7_bpiall_set_pte_ext,      cpu_v7_set_pte_ext
 578         globl_equ       cpu_v7_bpiall_suspend_size,     cpu_v7_suspend_size
 579 #ifdef CONFIG_ARM_CPU_SUSPEND
 580         globl_equ       cpu_v7_bpiall_do_suspend,       cpu_v7_do_suspend
 581         globl_equ       cpu_v7_bpiall_do_resume,        cpu_v7_do_resume
 582 #endif
 583         define_processor_functions v7_bpiall, dabort=v7_early_abort, pabort=v7_pabort, suspend=1, bugs=cpu_v7_bugs_init
 584
 585 #define HARDENED_BPIALL_PROCESSOR_FUNCTIONS v7_bpiall_processor_functions
 586 #else
 587 #define HARDENED_BPIALL_PROCESSOR_FUNCTIONS v7_processor_functions
 588 #endif
 589
 590 #ifndef CONFIG_ARM_LPAE
 591         @ Cortex-A8 - always needs bpiall switch_mm implementation
 592         globl_equ       cpu_ca8_proc_init,      cpu_v7_proc_init
 593         globl_equ       cpu_ca8_proc_fin,       cpu_v7_proc_fin
 594         globl_equ       cpu_ca8_reset,          cpu_v7_reset
 595         globl_equ       cpu_ca8_do_idle,        cpu_v7_do_idle
 596         globl_equ       cpu_ca8_dcache_clean_area, cpu_v7_dcache_clean_area
 597         globl_equ       cpu_ca8_set_pte_ext,    cpu_v7_set_pte_ext
 598         globl_equ       cpu_ca8_switch_mm,      cpu_v7_bpiall_switch_mm
 599         globl_equ       cpu_ca8_suspend_size,   cpu_v7_suspend_size
 600 #ifdef CONFIG_ARM_CPU_SUSPEND
 601         globl_equ       cpu_ca8_do_suspend,     cpu_v7_do_suspend
 602         globl_equ       cpu_ca8_do_resume,      cpu_v7_do_resume
 603 #endif
 604         define_processor_functions ca8, dabort=v7_early_abort, pabort=v7_pabort, suspend=1, bugs=cpu_v7_ca8_ibe
 605
 606         @ Cortex-A9 - needs more registers preserved across suspend/resume
 607         @ and bpiall switch_mm for hardening
 608         globl_equ       cpu_ca9mp_proc_init,    cpu_v7_proc_init
 609         globl_equ       cpu_ca9mp_proc_fin,     cpu_v7_proc_fin
 610         globl_equ       cpu_ca9mp_reset,        cpu_v7_reset
 611         globl_equ       cpu_ca9mp_do_idle,      cpu_v7_do_idle
 612         globl_equ       cpu_ca9mp_dcache_clean_area, cpu_v7_dcache_clean_area
 613 #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
 614         globl_equ       cpu_ca9mp_switch_mm,    cpu_v7_bpiall_switch_mm
 615 #else
 616         globl_equ       cpu_ca9mp_switch_mm,    cpu_v7_switch_mm
 617 #endif
 618         globl_equ       cpu_ca9mp_set_pte_ext,  cpu_v7_set_pte_ext
 619         define_processor_functions ca9mp, dabort=v7_early_abort, pabort=v7_pabort, suspend=1, bugs=cpu_v7_bugs_init
 620 #endif
 621
 622         @ Cortex-A15 - needs iciallu switch_mm for hardening
 623         globl_equ       cpu_ca15_proc_init,     cpu_v7_proc_init
 624         globl_equ       cpu_ca15_proc_fin,      cpu_v7_proc_fin
 625         globl_equ       cpu_ca15_reset,         cpu_v7_reset
 626         globl_equ       cpu_ca15_do_idle,       cpu_v7_do_idle
 627         globl_equ       cpu_ca15_dcache_clean_area, cpu_v7_dcache_clean_area
 628 #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
 629         globl_equ       cpu_ca15_switch_mm,     cpu_v7_iciallu_switch_mm
 630 #else
 631         globl_equ       cpu_ca15_switch_mm,     cpu_v7_switch_mm
 632 #endif
 633         globl_equ       cpu_ca15_set_pte_ext,   cpu_v7_set_pte_ext
 634         globl_equ       cpu_ca15_suspend_size,  cpu_v7_suspend_size
 635         globl_equ       cpu_ca15_do_suspend,    cpu_v7_do_suspend
 636         globl_equ       cpu_ca15_do_resume,     cpu_v7_do_resume
 637         define_processor_functions ca15, dabort=v7_early_abort, pabort=v7_pabort, suspend=1, bugs=cpu_v7_ca15_ibe
 638 #ifdef CONFIG_CPU_PJ4B
 639         define_processor_functions pj4b, dabort=v7_early_abort, pabort=v7_pabort, suspend=1
 640 #endif
 641
 642         .section ".rodata"
 643
 644         string  cpu_arch_name, "armv7"
 645         string  cpu_elf_name, "v7"
 646         .align
 647
 648         .section ".proc.info.init", "a"
 649
 650         /*
 651          * Standard v7 proc info content
 652          */
 653 .macro __v7_proc name, initfunc, mm_mmuflags = 0, io_mmuflags = 0, hwcaps = 0, proc_fns = v7_processor_functions, cache_fns = v7_cache_fns
 654         ALT_SMP(.long   PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_AP_READ | \
 655                         PMD_SECT_AF | PMD_FLAGS_SMP | \mm_mmuflags)
 656         ALT_UP(.long    PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_AP_READ | \
 657                         PMD_SECT_AF | PMD_FLAGS_UP | \mm_mmuflags)
 658         .long   PMD_TYPE_SECT | PMD_SECT_AP_WRITE | \
 659                 PMD_SECT_AP_READ | PMD_SECT_AF | \io_mmuflags
 660         initfn  \initfunc, \name
 661         .long   cpu_arch_name
 662         .long   cpu_elf_name
 663         .long   HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB | HWCAP_FAST_MULT | \
 664                 HWCAP_EDSP | HWCAP_TLS | \hwcaps
 665         .long   cpu_v7_name
 666         .long   \proc_fns
 667         .long   v7wbi_tlb_fns
 668         .long   v6_user_fns
 669         .long   \cache_fns
 670 .endm
 671
 672 #ifndef CONFIG_ARM_LPAE
 673         /*
 674          * ARM Ltd. Cortex A5 processor.
 675          */
 676         .type   __v7_ca5mp_proc_info, #object
 677 __v7_ca5mp_proc_info:
 678         .long   0x410fc050
 679         .long   0xff0ffff0
 680         __v7_proc __v7_ca5mp_proc_info, __v7_ca5mp_setup
 681         .size   __v7_ca5mp_proc_info, . - __v7_ca5mp_proc_info
 682
 683         /*
 684          * ARM Ltd. Cortex A9 processor.
 685          */
 686         .type   __v7_ca9mp_proc_info, #object
 687 __v7_ca9mp_proc_info:
 688         .long   0x410fc090
 689         .long   0xff0ffff0
 690         __v7_proc __v7_ca9mp_proc_info, __v7_ca9mp_setup, proc_fns = ca9mp_processor_functions
 691         .size   __v7_ca9mp_proc_info, . - __v7_ca9mp_proc_info
 692
 693         /*
 694          * ARM Ltd. Cortex A8 processor.
 695          */
 696         .type   __v7_ca8_proc_info, #object
 697 __v7_ca8_proc_info:
 698         .long   0x410fc080
 699         .long   0xff0ffff0
 700         __v7_proc __v7_ca8_proc_info, __v7_setup, proc_fns = ca8_processor_functions
 701         .size   __v7_ca8_proc_info, . - __v7_ca8_proc_info
 702
 703 #endif  /* CONFIG_ARM_LPAE */
 704
 705         /*
 706          * Marvell PJ4B processor.
 707          */
 708 #ifdef CONFIG_CPU_PJ4B
 709         .type   __v7_pj4b_proc_info, #object
 710 __v7_pj4b_proc_info:
 711         .long   0x560f5800
 712         .long   0xff0fff00
 713         __v7_proc __v7_pj4b_proc_info, __v7_pj4b_setup, proc_fns = pj4b_processor_functions
 714         .size   __v7_pj4b_proc_info, . - __v7_pj4b_proc_info
 715 #endif
 716
 717         /*
 718          * ARM Ltd. Cortex R7 processor.
 719          */
 720         .type   __v7_cr7mp_proc_info, #object
 721 __v7_cr7mp_proc_info:
 722         .long   0x410fc170
 723         .long   0xff0ffff0
 724         __v7_proc __v7_cr7mp_proc_info, __v7_cr7mp_setup
 725         .size   __v7_cr7mp_proc_info, . - __v7_cr7mp_proc_info
 726
 727         /*
 728          * ARM Ltd. Cortex R8 processor.
 729          */
 730         .type   __v7_cr8mp_proc_info, #object
 731 __v7_cr8mp_proc_info:
 732         .long   0x410fc180
 733         .long   0xff0ffff0
 734         __v7_proc __v7_cr8mp_proc_info, __v7_cr8mp_setup
 735         .size   __v7_cr8mp_proc_info, . - __v7_cr8mp_proc_info
 736
 737         /*
 738          * ARM Ltd. Cortex A7 processor.
 739          */
 740         .type   __v7_ca7mp_proc_info, #object
 741 __v7_ca7mp_proc_info:
 742         .long   0x410fc070
 743         .long   0xff0ffff0
 744         __v7_proc __v7_ca7mp_proc_info, __v7_ca7mp_setup
 745         .size   __v7_ca7mp_proc_info, . - __v7_ca7mp_proc_info
 746
 747         /*
 748          * ARM Ltd. Cortex A12 processor.
 749          */
 750         .type   __v7_ca12mp_proc_info, #object
 751 __v7_ca12mp_proc_info:
 752         .long   0x410fc0d0
 753         .long   0xff0ffff0
 754         __v7_proc __v7_ca12mp_proc_info, __v7_ca12mp_setup, proc_fns = HARDENED_BPIALL_PROCESSOR_FUNCTIONS
 755         .size   __v7_ca12mp_proc_info, . - __v7_ca12mp_proc_info
 756
 757         /*
 758          * ARM Ltd. Cortex A15 processor.
 759          */
 760         .type   __v7_ca15mp_proc_info, #object
 761 __v7_ca15mp_proc_info:
 762         .long   0x410fc0f0
 763         .long   0xff0ffff0
 764         __v7_proc __v7_ca15mp_proc_info, __v7_ca15mp_setup, proc_fns = ca15_processor_functions
 765         .size   __v7_ca15mp_proc_info, . - __v7_ca15mp_proc_info
 766
 767         /*
 768          * Broadcom Corporation Brahma-B15 processor.
 769          */
 770         .type   __v7_b15mp_proc_info, #object
 771 __v7_b15mp_proc_info:
 772         .long   0x420f00f0
 773         .long   0xff0ffff0
 774         __v7_proc __v7_b15mp_proc_info, __v7_b15mp_setup, proc_fns = ca15_processor_functions, cache_fns = b15_cache_fns
 775         .size   __v7_b15mp_proc_info, . - __v7_b15mp_proc_info
 776
 777         /*
 778          * ARM Ltd. Cortex A17 processor.
 779          */
 780         .type   __v7_ca17mp_proc_info, #object
 781 __v7_ca17mp_proc_info:
 782         .long   0x410fc0e0
 783         .long   0xff0ffff0
 784         __v7_proc __v7_ca17mp_proc_info, __v7_ca17mp_setup, proc_fns = HARDENED_BPIALL_PROCESSOR_FUNCTIONS
 785         .size   __v7_ca17mp_proc_info, . - __v7_ca17mp_proc_info
 786
 787         /* ARM Ltd. Cortex A73 processor */
 788         .type   __v7_ca73_proc_info, #object
 789 __v7_ca73_proc_info:
 790         .long   0x410fd090
 791         .long   0xff0ffff0
 792         __v7_proc __v7_ca73_proc_info, __v7_setup, proc_fns = HARDENED_BPIALL_PROCESSOR_FUNCTIONS
 793         .size   __v7_ca73_proc_info, . - __v7_ca73_proc_info
 794
 795         /* ARM Ltd. Cortex A75 processor */
 796         .type   __v7_ca75_proc_info, #object
 797 __v7_ca75_proc_info:
 798         .long   0x410fd0a0
 799         .long   0xff0ffff0
 800         __v7_proc __v7_ca75_proc_info, __v7_setup, proc_fns = HARDENED_BPIALL_PROCESSOR_FUNCTIONS
 801         .size   __v7_ca75_proc_info, . - __v7_ca75_proc_info
 802
 803         /*
 804          * Qualcomm Inc. Krait processors.
 805          */
 806         .type   __krait_proc_info, #object
 807 __krait_proc_info:
 808         .long   0x510f0400              @ Required ID value
 809         .long   0xff0ffc00              @ Mask for ID
 810         /*
 811          * Some Krait processors don't indicate support for SDIV and UDIV
 812          * instructions in the ARM instruction set, even though they actually
 813          * do support them. They also don't indicate support for fused multiply
 814          * instructions even though they actually do support them.
 815          */
 816         __v7_proc __krait_proc_info, __v7_setup, hwcaps = HWCAP_IDIV | HWCAP_VFPv4
 817         .size   __krait_proc_info, . - __krait_proc_info
 818
 819         /*
 820          * Match any ARMv7 processor core.
 821          */
 822         .type   __v7_proc_info, #object
 823 __v7_proc_info:
 824         .long   0x000f0000              @ Required ID value
 825         .long   0x000f0000              @ Mask for ID
 826         __v7_proc __v7_proc_info, __v7_setup
 827         .size   __v7_proc_info, . - __v7_proc_info