lib/cpus/aarch64/wa_cve_2017_5715_bpiall.S

   1 /*
   2  * Copyright (c) 2017-2018, ARM Limited and Contributors. All rights reserved.
   3  *
   4  * SPDX-License-Identifier: BSD-3-Clause
   5  */
   6
   7 #include <arch.h>
   8 #include <asm_macros.S>
   9 #include <context.h>
  10 #include <services/arm_arch_svc.h>
  11
  12         .globl  wa_cve_2017_5715_bpiall_vbar
  13
  14 #define EMIT_BPIALL             0xee070fd5
  15 #define EMIT_SMC                0xe1600070
  16 #define ESR_EL3_A64_SMC0        0x5e000000
  17
  18         .macro  apply_cve_2017_5715_wa _from_vector
  19         /*
  20          * Save register state to enable a call to AArch32 S-EL1 and return
  21          * Identify the original calling vector in w2 (==_from_vector)
  22          * Use w3-w6 for additional register state preservation while in S-EL1
  23          */
  24
  25         /* Save GP regs */
  26         stp     x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0]
  27         stp     x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2]
  28         stp     x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4]
  29         stp     x6, x7, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X6]
  30         stp     x8, x9, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X8]
  31         stp     x10, x11, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X10]
  32         stp     x12, x13, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X12]
  33         stp     x14, x15, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X14]
  34         stp     x16, x17, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X16]
  35         stp     x18, x19, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X18]
  36         stp     x20, x21, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X20]
  37         stp     x22, x23, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X22]
  38         stp     x24, x25, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X24]
  39         stp     x26, x27, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X26]
  40         stp     x28, x29, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X28]
  41
  42         /* Identify the original exception vector */
  43         mov     w2, \_from_vector
  44
  45         /* Preserve 32-bit system registers in GP registers through the workaround */
  46         mrs     x3, esr_el3
  47         mrs     x4, spsr_el3
  48         mrs     x5, scr_el3
  49         mrs     x6, sctlr_el1
  50
  51         /*
  52          * Preserve LR and ELR_EL3 registers in the GP regs context.
  53          * Temporarily use the CTX_GPREG_SP_EL0 slot to preserve ELR_EL3
  54          * through the workaround. This is OK because at this point the
  55          * current state for this context's SP_EL0 is in the live system
  56          * register, which is unmodified by the workaround.
  57          */
  58         mrs     x7, elr_el3
  59         stp     x30, x7, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_LR]
  60
  61         /*
  62          * Load system registers for entry to S-EL1.
  63          */
  64
  65         /* Mask all interrupts and set AArch32 Supervisor mode */
  66         movz    w8, SPSR_MODE32(MODE32_svc, SPSR_T_ARM, SPSR_E_LITTLE, SPSR_AIF_MASK)
  67
  68         /* Switch EL3 exception vectors while the workaround is executing. */
  69         adr     x9, wa_cve_2017_5715_bpiall_ret_vbar
  70
  71         /* Setup SCTLR_EL1 with MMU off and I$ on */
  72         ldr     x10, stub_sel1_sctlr
  73
  74         /* Land at the S-EL1 workaround stub */
  75         adr     x11, aarch32_stub
  76
  77         /*
  78          * Setting SCR_EL3 to all zeroes means that the NS, RW
  79          * and SMD bits are configured as expected.
  80          */
  81         msr     scr_el3, xzr
  82         msr     spsr_el3, x8
  83         msr     vbar_el3, x9
  84         msr     sctlr_el1, x10
  85         msr     elr_el3, x11
  86
  87         eret
  88         .endm
  89
  90         /* ---------------------------------------------------------------------
  91          * This vector table is used at runtime to enter the workaround at
  92          * AArch32 S-EL1 for Sync/IRQ/FIQ/SError exceptions.  If the workaround
  93          * is not enabled, the existing runtime exception vector table is used.
  94          * ---------------------------------------------------------------------
  95          */
  96 vector_base wa_cve_2017_5715_bpiall_vbar
  97
  98         /* ---------------------------------------------------------------------
  99          * Current EL with SP_EL0 : 0x0 - 0x200
 100          * ---------------------------------------------------------------------
 101          */
 102 vector_entry bpiall_sync_exception_sp_el0
 103         b       sync_exception_sp_el0
 104         nop     /* to force 8 byte alignment for the following stub */
 105
 106         /*
 107          * Since each vector table entry is 128 bytes, we can store the
 108          * stub context in the unused space to minimize memory footprint.
 109          */
 110 stub_sel1_sctlr:
 111         .quad   SCTLR_AARCH32_EL1_RES1 | SCTLR_I_BIT
 112
 113 aarch32_stub:
 114         .word   EMIT_BPIALL
 115         .word   EMIT_SMC
 116
 117 end_vector_entry bpiall_sync_exception_sp_el0
 118
 119 vector_entry bpiall_irq_sp_el0
 120         b       irq_sp_el0
 121 end_vector_entry bpiall_irq_sp_el0
 122
 123 vector_entry bpiall_fiq_sp_el0
 124         b       fiq_sp_el0
 125 end_vector_entry bpiall_fiq_sp_el0
 126
 127 vector_entry bpiall_serror_sp_el0
 128         b       serror_sp_el0
 129 end_vector_entry bpiall_serror_sp_el0
 130
 131         /* ---------------------------------------------------------------------
 132          * Current EL with SP_ELx: 0x200 - 0x400
 133          * ---------------------------------------------------------------------
 134          */
 135 vector_entry bpiall_sync_exception_sp_elx
 136         b       sync_exception_sp_elx
 137 end_vector_entry bpiall_sync_exception_sp_elx
 138
 139 vector_entry bpiall_irq_sp_elx
 140         b       irq_sp_elx
 141 end_vector_entry bpiall_irq_sp_elx
 142
 143 vector_entry bpiall_fiq_sp_elx
 144         b       fiq_sp_elx
 145 end_vector_entry bpiall_fiq_sp_elx
 146
 147 vector_entry bpiall_serror_sp_elx
 148         b       serror_sp_elx
 149 end_vector_entry bpiall_serror_sp_elx
 150
 151         /* ---------------------------------------------------------------------
 152          * Lower EL using AArch64 : 0x400 - 0x600
 153          * ---------------------------------------------------------------------
 154          */
 155 vector_entry bpiall_sync_exception_aarch64
 156         apply_cve_2017_5715_wa 1
 157 end_vector_entry bpiall_sync_exception_aarch64
 158
 159 vector_entry bpiall_irq_aarch64
 160         apply_cve_2017_5715_wa 2
 161 end_vector_entry bpiall_irq_aarch64
 162
 163 vector_entry bpiall_fiq_aarch64
 164         apply_cve_2017_5715_wa 4
 165 end_vector_entry bpiall_fiq_aarch64
 166
 167 vector_entry bpiall_serror_aarch64
 168         apply_cve_2017_5715_wa 8
 169 end_vector_entry bpiall_serror_aarch64
 170
 171         /* ---------------------------------------------------------------------
 172          * Lower EL using AArch32 : 0x600 - 0x800
 173          * ---------------------------------------------------------------------
 174          */
 175 vector_entry bpiall_sync_exception_aarch32
 176         apply_cve_2017_5715_wa 1
 177 end_vector_entry bpiall_sync_exception_aarch32
 178
 179 vector_entry bpiall_irq_aarch32
 180         apply_cve_2017_5715_wa 2
 181 end_vector_entry bpiall_irq_aarch32
 182
 183 vector_entry bpiall_fiq_aarch32
 184         apply_cve_2017_5715_wa 4
 185 end_vector_entry bpiall_fiq_aarch32
 186
 187 vector_entry bpiall_serror_aarch32
 188         apply_cve_2017_5715_wa 8
 189 end_vector_entry bpiall_serror_aarch32
 190
 191         /* ---------------------------------------------------------------------
 192          * This vector table is used while the workaround is executing.  It
 193          * installs a simple SMC handler to allow the Sync/IRQ/FIQ/SError
 194          * workaround stubs to enter EL3 from S-EL1.  It restores the previous
 195          * EL3 state before proceeding with the normal runtime exception vector.
 196          * ---------------------------------------------------------------------
 197          */
 198 vector_base wa_cve_2017_5715_bpiall_ret_vbar
 199
 200         /* ---------------------------------------------------------------------
 201          * Current EL with SP_EL0 : 0x0 - 0x200 (UNUSED)
 202          * ---------------------------------------------------------------------
 203          */
 204 vector_entry bpiall_ret_sync_exception_sp_el0
 205         b       report_unhandled_exception
 206 end_vector_entry bpiall_ret_sync_exception_sp_el0
 207
 208 vector_entry bpiall_ret_irq_sp_el0
 209         b       report_unhandled_interrupt
 210 end_vector_entry bpiall_ret_irq_sp_el0
 211
 212 vector_entry bpiall_ret_fiq_sp_el0
 213         b       report_unhandled_interrupt
 214 end_vector_entry bpiall_ret_fiq_sp_el0
 215
 216 vector_entry bpiall_ret_serror_sp_el0
 217         b       report_unhandled_exception
 218 end_vector_entry bpiall_ret_serror_sp_el0
 219
 220         /* ---------------------------------------------------------------------
 221          * Current EL with SP_ELx: 0x200 - 0x400 (UNUSED)
 222          * ---------------------------------------------------------------------
 223          */
 224 vector_entry bpiall_ret_sync_exception_sp_elx
 225         b       report_unhandled_exception
 226 end_vector_entry bpiall_ret_sync_exception_sp_elx
 227
 228 vector_entry bpiall_ret_irq_sp_elx
 229         b       report_unhandled_interrupt
 230 end_vector_entry bpiall_ret_irq_sp_elx
 231
 232 vector_entry bpiall_ret_fiq_sp_elx
 233         b       report_unhandled_interrupt
 234 end_vector_entry bpiall_ret_fiq_sp_elx
 235
 236 vector_entry bpiall_ret_serror_sp_elx
 237         b       report_unhandled_exception
 238 end_vector_entry bpiall_ret_serror_sp_elx
 239
 240         /* ---------------------------------------------------------------------
 241          * Lower EL using AArch64 : 0x400 - 0x600 (UNUSED)
 242          * ---------------------------------------------------------------------
 243          */
 244 vector_entry bpiall_ret_sync_exception_aarch64
 245         b       report_unhandled_exception
 246 end_vector_entry bpiall_ret_sync_exception_aarch64
 247
 248 vector_entry bpiall_ret_irq_aarch64
 249         b       report_unhandled_interrupt
 250 end_vector_entry bpiall_ret_irq_aarch64
 251
 252 vector_entry bpiall_ret_fiq_aarch64
 253         b       report_unhandled_interrupt
 254 end_vector_entry bpiall_ret_fiq_aarch64
 255
 256 vector_entry bpiall_ret_serror_aarch64
 257         b       report_unhandled_exception
 258 end_vector_entry bpiall_ret_serror_aarch64
 259
 260         /* ---------------------------------------------------------------------
 261          * Lower EL using AArch32 : 0x600 - 0x800
 262          * ---------------------------------------------------------------------
 263          */
 264 vector_entry bpiall_ret_sync_exception_aarch32
 265         /*
 266          * w2 indicates which SEL1 stub was run and thus which original vector was used
 267          * w3-w6 contain saved system register state (esr_el3 in w3)
 268          * Restore LR and ELR_EL3 register state from the GP regs context
 269          */
 270         ldp     x30, x7, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_LR]
 271
 272         /* Apply the restored system register state */
 273         msr     esr_el3, x3
 274         msr     spsr_el3, x4
 275         msr     scr_el3, x5
 276         msr     sctlr_el1, x6
 277         msr     elr_el3, x7
 278
 279         /*
 280          * Workaround is complete, so swap VBAR_EL3 to point
 281          * to workaround entry table in preparation for subsequent
 282          * Sync/IRQ/FIQ/SError exceptions.
 283          */
 284         adr     x0, wa_cve_2017_5715_bpiall_vbar
 285         msr     vbar_el3, x0
 286
 287         /*
 288          * Restore all GP regs except x2 and x3 (esr).  The value in x2
 289          * indicates the type of the original exception.
 290          */
 291         ldp     x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0]
 292         ldp     x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4]
 293         ldp     x6, x7, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X6]
 294         ldp     x8, x9, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X8]
 295         ldp     x10, x11, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X10]
 296         ldp     x12, x13, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X12]
 297         ldp     x14, x15, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X14]
 298         ldp     x16, x17, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X16]
 299         ldp     x18, x19, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X18]
 300         ldp     x20, x21, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X20]
 301         ldp     x22, x23, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X22]
 302         ldp     x24, x25, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X24]
 303         ldp     x26, x27, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X26]
 304         ldp     x28, x29, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X28]
 305
 306         /* Fast path Sync exceptions.  Static predictor will fall through. */
 307         tbz     w2, #0, workaround_not_sync
 308
 309         /*
 310          * Check if SMC is coming from A64 state on #0
 311          * with W0 = SMCCC_ARCH_WORKAROUND_1
 312          *
 313          * This sequence evaluates as:
 314          *    (W0==SMCCC_ARCH_WORKAROUND_1) ? (ESR_EL3==SMC#0) : (NE)
 315          * allowing use of a single branch operation
 316          */
 317         orr     w2, wzr, #SMCCC_ARCH_WORKAROUND_1
 318         cmp     w0, w2
 319         mov_imm w2, ESR_EL3_A64_SMC0
 320         ccmp    w3, w2, #0, eq
 321         /* Static predictor will predict a fall through */
 322         bne     1f
 323         eret
 324 1:
 325         ldp     x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2]
 326         b       sync_exception_aarch64
 327 end_vector_entry bpiall_ret_sync_exception_aarch32
 328
 329 vector_entry bpiall_ret_irq_aarch32
 330         b       report_unhandled_interrupt
 331
 332         /*
 333          * Post-workaround fan-out for non-sync exceptions
 334          */
 335 workaround_not_sync:
 336         tbnz    w2, #3, bpiall_ret_serror
 337         tbnz    w2, #2, bpiall_ret_fiq
 338         /* IRQ */
 339         ldp     x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2]
 340         b       irq_aarch64
 341
 342 bpiall_ret_fiq:
 343         ldp     x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2]
 344         b       fiq_aarch64
 345
 346 bpiall_ret_serror:
 347         ldp     x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2]
 348         b       serror_aarch64
 349 end_vector_entry bpiall_ret_irq_aarch32
 350
 351 vector_entry bpiall_ret_fiq_aarch32
 352         b       report_unhandled_interrupt
 353 end_vector_entry bpiall_ret_fiq_aarch32
 354
 355 vector_entry bpiall_ret_serror_aarch32
 356         b       report_unhandled_exception
 357 end_vector_entry bpiall_ret_serror_aarch32