lib/cpus/aarch64/neoverse_n1.S

   1 /*
   2  * Copyright (c) 2017-2019, ARM Limited and Contributors. All rights reserved.
   3  *
   4  * SPDX-License-Identifier: BSD-3-Clause
   5  */
   6
   7 #include <arch.h>
   8 #include <asm_macros.S>
   9 #include <neoverse_n1.h>
  10 #include <cpuamu.h>
  11 #include <cpu_macros.S>
  12 #include <context.h>
  13
  14 /* Hardware handled coherency */
  15 #if HW_ASSISTED_COHERENCY == 0
  16 #error "Neoverse N1 must be compiled with HW_ASSISTED_COHERENCY enabled"
  17 #endif
  18
  19 /* 64-bit only core */
  20 #if CTX_INCLUDE_AARCH32_REGS == 1
  21 #error "Neoverse-N1 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0"
  22 #endif
  23
  24 #if ERRATA_N1_IC_TRAP
  25         .global neoverse_n1_errata_ic_trap_handler
  26 #endif
  27
  28 /* --------------------------------------------------
  29  * Errata Workaround for Neoverse N1 Erratum 1043202.
  30  * This applies to revision r0p0 and r1p0 of Neoverse N1.
  31  * Inputs:
  32  * x0: variant[4:7] and revision[0:3] of current cpu.
  33  * Shall clobber: x0-x17
  34  * --------------------------------------------------
  35  */
  36 func errata_n1_1043202_wa
  37         /* Compare x0 against revision r1p0 */
  38         mov     x17, x30
  39         bl      check_errata_1043202
  40         cbz     x0, 1f
  41
  42         /* Apply instruction patching sequence */
  43         ldr     x0, =0x0
  44         msr     CPUPSELR_EL3, x0
  45         ldr     x0, =0xF3BF8F2F
  46         msr     CPUPOR_EL3, x0
  47         ldr     x0, =0xFFFFFFFF
  48         msr     CPUPMR_EL3, x0
  49         ldr     x0, =0x800200071
  50         msr     CPUPCR_EL3, x0
  51         isb
  52 1:
  53         ret     x17
  54 endfunc errata_n1_1043202_wa
  55
  56 func check_errata_1043202
  57         /* Applies to r0p0 and r1p0 */
  58         mov     x1, #0x10
  59         b       cpu_rev_var_ls
  60 endfunc check_errata_1043202
  61
  62 /* --------------------------------------------------
  63  * Disable speculative loads if Neoverse N1 supports
  64  * SSBS.
  65  *
  66  * Shall clobber: x0.
  67  * --------------------------------------------------
  68  */
  69 func neoverse_n1_disable_speculative_loads
  70         /* Check if the PE implements SSBS */
  71         mrs     x0, id_aa64pfr1_el1
  72         tst     x0, #(ID_AA64PFR1_EL1_SSBS_MASK << ID_AA64PFR1_EL1_SSBS_SHIFT)
  73         b.eq    1f
  74
  75         /* Disable speculative loads */
  76         msr     SSBS, xzr
  77
  78 1:
  79         ret
  80 endfunc neoverse_n1_disable_speculative_loads
  81
  82 /* --------------------------------------------------
  83  * Errata Workaround for Neoverse N1 Errata #1073348
  84  * This applies to revision r0p0 and r1p0 of Neoverse N1.
  85  * Inputs:
  86  * x0: variant[4:7] and revision[0:3] of current cpu.
  87  * Shall clobber: x0-x17
  88  * --------------------------------------------------
  89  */
  90 func errata_n1_1073348_wa
  91         /* Compare x0 against revision r1p0 */
  92         mov     x17, x30
  93         bl      check_errata_1073348
  94         cbz     x0, 1f
  95         mrs     x1, NEOVERSE_N1_CPUACTLR_EL1
  96         orr     x1, x1, NEOVERSE_N1_CPUACTLR_EL1_BIT_6
  97         msr     NEOVERSE_N1_CPUACTLR_EL1, x1
  98 1:
  99         ret     x17
 100 endfunc errata_n1_1073348_wa
 101
 102 func check_errata_1073348
 103         /* Applies to r0p0 and r1p0 */
 104         mov     x1, #0x10
 105         b       cpu_rev_var_ls
 106 endfunc check_errata_1073348
 107
 108 /* --------------------------------------------------
 109  * Errata Workaround for Neoverse N1 Errata #1130799
 110  * This applies to revision <=r2p0 of Neoverse N1.
 111  * Inputs:
 112  * x0: variant[4:7] and revision[0:3] of current cpu.
 113  * Shall clobber: x0-x17
 114  * --------------------------------------------------
 115  */
 116 func errata_n1_1130799_wa
 117         /* Compare x0 against revision r2p0 */
 118         mov     x17, x30
 119         bl      check_errata_1130799
 120         cbz     x0, 1f
 121         mrs     x1, NEOVERSE_N1_CPUACTLR2_EL1
 122         orr     x1, x1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_59
 123         msr     NEOVERSE_N1_CPUACTLR2_EL1, x1
 124 1:
 125         ret     x17
 126 endfunc errata_n1_1130799_wa
 127
 128 func check_errata_1130799
 129         /* Applies to <=r2p0 */
 130         mov     x1, #0x20
 131         b       cpu_rev_var_ls
 132 endfunc check_errata_1130799
 133
 134 /* --------------------------------------------------
 135  * Errata Workaround for Neoverse N1 Errata #1165347
 136  * This applies to revision <=r2p0 of Neoverse N1.
 137  * Inputs:
 138  * x0: variant[4:7] and revision[0:3] of current cpu.
 139  * Shall clobber: x0-x17
 140  * --------------------------------------------------
 141  */
 142 func errata_n1_1165347_wa
 143         /* Compare x0 against revision r2p0 */
 144         mov     x17, x30
 145         bl      check_errata_1165347
 146         cbz     x0, 1f
 147         mrs     x1, NEOVERSE_N1_CPUACTLR2_EL1
 148         orr     x1, x1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_0
 149         orr     x1, x1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_15
 150         msr     NEOVERSE_N1_CPUACTLR2_EL1, x1
 151 1:
 152         ret     x17
 153 endfunc errata_n1_1165347_wa
 154
 155 func check_errata_1165347
 156         /* Applies to <=r2p0 */
 157         mov     x1, #0x20
 158         b       cpu_rev_var_ls
 159 endfunc check_errata_1165347
 160
 161 /* --------------------------------------------------
 162  * Errata Workaround for Neoverse N1 Errata #1207823
 163  * This applies to revision <=r2p0 of Neoverse N1.
 164  * Inputs:
 165  * x0: variant[4:7] and revision[0:3] of current cpu.
 166  * Shall clobber: x0-x17
 167  * --------------------------------------------------
 168  */
 169 func errata_n1_1207823_wa
 170         /* Compare x0 against revision r2p0 */
 171         mov     x17, x30
 172         bl      check_errata_1207823
 173         cbz     x0, 1f
 174         mrs     x1, NEOVERSE_N1_CPUACTLR2_EL1
 175         orr     x1, x1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_11
 176         msr     NEOVERSE_N1_CPUACTLR2_EL1, x1
 177 1:
 178         ret     x17
 179 endfunc errata_n1_1207823_wa
 180
 181 func check_errata_1207823
 182         /* Applies to <=r2p0 */
 183         mov     x1, #0x20
 184         b       cpu_rev_var_ls
 185 endfunc check_errata_1207823
 186
 187 /* --------------------------------------------------
 188  * Errata Workaround for Neoverse N1 Errata #1220197
 189  * This applies to revision <=r2p0 of Neoverse N1.
 190  * Inputs:
 191  * x0: variant[4:7] and revision[0:3] of current cpu.
 192  * Shall clobber: x0-x17
 193  * --------------------------------------------------
 194  */
 195 func errata_n1_1220197_wa
 196         /* Compare x0 against revision r2p0 */
 197         mov     x17, x30
 198         bl      check_errata_1220197
 199         cbz     x0, 1f
 200         mrs     x1, NEOVERSE_N1_CPUECTLR_EL1
 201         orr     x1, x1, NEOVERSE_N1_WS_THR_L2_MASK
 202         msr     NEOVERSE_N1_CPUECTLR_EL1, x1
 203 1:
 204         ret     x17
 205 endfunc errata_n1_1220197_wa
 206
 207 func check_errata_1220197
 208         /* Applies to <=r2p0 */
 209         mov     x1, #0x20
 210         b       cpu_rev_var_ls
 211 endfunc check_errata_1220197
 212
 213 /* --------------------------------------------------
 214  * Errata Workaround for Neoverse N1 Errata #1257314
 215  * This applies to revision <=r3p0 of Neoverse N1.
 216  * Inputs:
 217  * x0: variant[4:7] and revision[0:3] of current cpu.
 218  * Shall clobber: x0-x17
 219  * --------------------------------------------------
 220  */
 221 func errata_n1_1257314_wa
 222         /* Compare x0 against revision r3p0 */
 223         mov     x17, x30
 224         bl      check_errata_1257314
 225         cbz     x0, 1f
 226         mrs     x1, NEOVERSE_N1_CPUACTLR3_EL1
 227         orr     x1, x1, NEOVERSE_N1_CPUACTLR3_EL1_BIT_10
 228         msr     NEOVERSE_N1_CPUACTLR3_EL1, x1
 229 1:
 230         ret     x17
 231 endfunc errata_n1_1257314_wa
 232
 233 func check_errata_1257314
 234         /* Applies to <=r3p0 */
 235         mov     x1, #0x30
 236         b       cpu_rev_var_ls
 237 endfunc check_errata_1257314
 238
 239 /* --------------------------------------------------
 240  * Errata Workaround for Neoverse N1 Errata #1262606
 241  * This applies to revision <=r3p0 of Neoverse N1.
 242  * Inputs:
 243  * x0: variant[4:7] and revision[0:3] of current cpu.
 244  * Shall clobber: x0-x17
 245  * --------------------------------------------------
 246  */
 247 func errata_n1_1262606_wa
 248         /* Compare x0 against revision r3p0 */
 249         mov     x17, x30
 250         bl      check_errata_1262606
 251         cbz     x0, 1f
 252         mrs     x1, NEOVERSE_N1_CPUACTLR_EL1
 253         orr     x1, x1, NEOVERSE_N1_CPUACTLR_EL1_BIT_13
 254         msr     NEOVERSE_N1_CPUACTLR_EL1, x1
 255 1:
 256         ret     x17
 257 endfunc errata_n1_1262606_wa
 258
 259 func check_errata_1262606
 260         /* Applies to <=r3p0 */
 261         mov     x1, #0x30
 262         b       cpu_rev_var_ls
 263 endfunc check_errata_1262606
 264
 265 /* --------------------------------------------------
 266  * Errata Workaround for Neoverse N1 Errata #1262888
 267  * This applies to revision <=r3p0 of Neoverse N1.
 268  * Inputs:
 269  * x0: variant[4:7] and revision[0:3] of current cpu.
 270  * Shall clobber: x0-x17
 271  * --------------------------------------------------
 272  */
 273 func errata_n1_1262888_wa
 274         /* Compare x0 against revision r3p0 */
 275         mov     x17, x30
 276         bl      check_errata_1262888
 277         cbz     x0, 1f
 278         mrs     x1, NEOVERSE_N1_CPUECTLR_EL1
 279         orr     x1, x1, NEOVERSE_N1_CPUECTLR_EL1_MM_TLBPF_DIS_BIT
 280         msr     NEOVERSE_N1_CPUECTLR_EL1, x1
 281 1:
 282         ret     x17
 283 endfunc errata_n1_1262888_wa
 284
 285 func check_errata_1262888
 286         /* Applies to <=r3p0 */
 287         mov     x1, #0x30
 288         b       cpu_rev_var_ls
 289 endfunc check_errata_1262888
 290
 291 /* --------------------------------------------------
 292  * Errata Workaround for Neoverse N1 Errata #1275112
 293  * This applies to revision <=r3p0 of Neoverse N1.
 294  * Inputs:
 295  * x0: variant[4:7] and revision[0:3] of current cpu.
 296  * Shall clobber: x0-x17
 297  * --------------------------------------------------
 298  */
 299 func errata_n1_1275112_wa
 300         /* Compare x0 against revision r3p0 */
 301         mov     x17, x30
 302         bl      check_errata_1275112
 303         cbz     x0, 1f
 304         mrs     x1, NEOVERSE_N1_CPUACTLR_EL1
 305         orr     x1, x1, NEOVERSE_N1_CPUACTLR_EL1_BIT_13
 306         msr     NEOVERSE_N1_CPUACTLR_EL1, x1
 307 1:
 308         ret     x17
 309 endfunc errata_n1_1275112_wa
 310
 311 func check_errata_1275112
 312         /* Applies to <=r3p0 */
 313         mov     x1, #0x30
 314         b       cpu_rev_var_ls
 315 endfunc check_errata_1275112
 316
 317 /* --------------------------------------------------
 318  * Errata Workaround for Neoverse N1 Erratum 1315703.
 319  * This applies to revision <= r3p0 of Neoverse N1.
 320  * Inputs:
 321  * x0: variant[4:7] and revision[0:3] of current cpu.
 322  * Shall clobber: x0-x17
 323  * --------------------------------------------------
 324  */
 325 func errata_n1_1315703_wa
 326         /* Compare x0 against revision r3p1 */
 327         mov     x17, x30
 328         bl      check_errata_1315703
 329         cbz     x0, 1f
 330
 331         mrs     x0, NEOVERSE_N1_CPUACTLR2_EL1
 332         orr     x0, x0, #NEOVERSE_N1_CPUACTLR2_EL1_BIT_16
 333         msr     NEOVERSE_N1_CPUACTLR2_EL1, x0
 334
 335 1:
 336         ret     x17
 337 endfunc errata_n1_1315703_wa
 338
 339 func check_errata_1315703
 340         /* Applies to everything <= r3p0. */
 341         mov     x1, #0x30
 342         b       cpu_rev_var_ls
 343 endfunc check_errata_1315703
 344
 345 /* --------------------------------------------------
 346  * Errata Workaround for Neoverse N1 Erratum 1542419.
 347  * This applies to revisions r3p0 - r4p0 of Neoverse N1
 348  * Inputs:
 349  * x0: variant[4:7] and revision[0:3] of current cpu.
 350  * Shall clobber: x0-x17
 351  * --------------------------------------------------
 352  */
 353 func errata_n1_1542419_wa
 354         /* Compare x0 against revision r3p0 and r4p0 */
 355         mov     x17, x30
 356         bl      check_errata_1542419
 357         cbz     x0, 1f
 358
 359         /* Apply instruction patching sequence */
 360         ldr     x0, =0x0
 361         msr     CPUPSELR_EL3, x0
 362         ldr     x0, =0xEE670D35
 363         msr     CPUPOR_EL3, x0
 364         ldr     x0, =0xFFFF0FFF
 365         msr     CPUPMR_EL3, x0
 366         ldr     x0, =0x08000020007D
 367         msr     CPUPCR_EL3, x0
 368         isb
 369 1:
 370         ret     x17
 371 endfunc errata_n1_1542419_wa
 372
 373 func check_errata_1542419
 374         /* Applies to everything r3p0 - r4p0. */
 375         mov     x1, #0x30
 376         mov     x2, #0x40
 377         b       cpu_rev_var_range
 378 endfunc check_errata_1542419
 379
 380 func neoverse_n1_reset_func
 381         mov     x19, x30
 382
 383         bl neoverse_n1_disable_speculative_loads
 384
 385         /* Forces all cacheable atomic instructions to be near */
 386         mrs     x0, NEOVERSE_N1_CPUACTLR2_EL1
 387         orr     x0, x0, #NEOVERSE_N1_CPUACTLR2_EL1_BIT_2
 388         msr     NEOVERSE_N1_CPUACTLR2_EL1, x0
 389         isb
 390
 391         bl      cpu_get_rev_var
 392         mov     x18, x0
 393
 394 #if ERRATA_N1_1043202
 395         mov     x0, x18
 396         bl      errata_n1_1043202_wa
 397 #endif
 398
 399 #if ERRATA_N1_1073348
 400         mov     x0, x18
 401         bl      errata_n1_1073348_wa
 402 #endif
 403
 404 #if ERRATA_N1_1130799
 405         mov     x0, x18
 406         bl      errata_n1_1130799_wa
 407 #endif
 408
 409 #if ERRATA_N1_1165347
 410         mov     x0, x18
 411         bl      errata_n1_1165347_wa
 412 #endif
 413
 414 #if ERRATA_N1_1207823
 415         mov     x0, x18
 416         bl      errata_n1_1207823_wa
 417 #endif
 418
 419 #if ERRATA_N1_1220197
 420         mov     x0, x18
 421         bl      errata_n1_1220197_wa
 422 #endif
 423
 424 #if ERRATA_N1_1257314
 425         mov     x0, x18
 426         bl      errata_n1_1257314_wa
 427 #endif
 428
 429 #if ERRATA_N1_1262606
 430         mov     x0, x18
 431         bl      errata_n1_1262606_wa
 432 #endif
 433
 434 #if ERRATA_N1_1262888
 435         mov     x0, x18
 436         bl      errata_n1_1262888_wa
 437 #endif
 438
 439 #if ERRATA_N1_1275112
 440         mov     x0, x18
 441         bl      errata_n1_1275112_wa
 442 #endif
 443
 444 #if ERRATA_N1_1315703
 445         mov     x0, x18
 446         bl      errata_n1_1315703_wa
 447 #endif
 448
 449 #if ERRATA_N1_1542419
 450         mov     x0, x18
 451         bl      errata_n1_1542419_wa
 452 #endif
 453
 454 #if ENABLE_AMU
 455         /* Make sure accesses from EL0/EL1 and EL2 are not trapped to EL3 */
 456         mrs     x0, actlr_el3
 457         orr     x0, x0, #NEOVERSE_N1_ACTLR_AMEN_BIT
 458         msr     actlr_el3, x0
 459
 460         /* Make sure accesses from EL0/EL1 are not trapped to EL2 */
 461         mrs     x0, actlr_el2
 462         orr     x0, x0, #NEOVERSE_N1_ACTLR_AMEN_BIT
 463         msr     actlr_el2, x0
 464
 465         /* Enable group0 counters */
 466         mov     x0, #NEOVERSE_N1_AMU_GROUP0_MASK
 467         msr     CPUAMCNTENSET_EL0, x0
 468 #endif
 469
 470 #if ERRATA_DSU_936184
 471         bl      errata_dsu_936184_wa
 472 #endif
 473
 474         isb
 475         ret     x19
 476 endfunc neoverse_n1_reset_func
 477
 478         /* ---------------------------------------------
 479          * HW will do the cache maintenance while powering down
 480          * ---------------------------------------------
 481          */
 482 func neoverse_n1_core_pwr_dwn
 483         /* ---------------------------------------------
 484          * Enable CPU power down bit in power control register
 485          * ---------------------------------------------
 486          */
 487         mrs     x0, NEOVERSE_N1_CPUPWRCTLR_EL1
 488         orr     x0, x0, #NEOVERSE_N1_CORE_PWRDN_EN_MASK
 489         msr     NEOVERSE_N1_CPUPWRCTLR_EL1, x0
 490         isb
 491         ret
 492 endfunc neoverse_n1_core_pwr_dwn
 493
 494 #if REPORT_ERRATA
 495 /*
 496  * Errata printing function for Neoverse N1. Must follow AAPCS.
 497  */
 498 func neoverse_n1_errata_report
 499         stp     x8, x30, [sp, #-16]!
 500
 501         bl      cpu_get_rev_var
 502         mov     x8, x0
 503
 504         /*
 505          * Report all errata. The revision-variant information is passed to
 506          * checking functions of each errata.
 507          */
 508         report_errata ERRATA_N1_1043202, neoverse_n1, 1043202
 509         report_errata ERRATA_N1_1073348, neoverse_n1, 1073348
 510         report_errata ERRATA_N1_1130799, neoverse_n1, 1130799
 511         report_errata ERRATA_N1_1165347, neoverse_n1, 1165347
 512         report_errata ERRATA_N1_1207823, neoverse_n1, 1207823
 513         report_errata ERRATA_N1_1220197, neoverse_n1, 1220197
 514         report_errata ERRATA_N1_1257314, neoverse_n1, 1257314
 515         report_errata ERRATA_N1_1262606, neoverse_n1, 1262606
 516         report_errata ERRATA_N1_1262888, neoverse_n1, 1262888
 517         report_errata ERRATA_N1_1275112, neoverse_n1, 1275112
 518         report_errata ERRATA_N1_1315703, neoverse_n1, 1315703
 519         report_errata ERRATA_N1_1542419, neoverse_n1, 1542419
 520         report_errata ERRATA_DSU_936184, neoverse_n1, dsu_936184
 521
 522         ldp     x8, x30, [sp], #16
 523         ret
 524 endfunc neoverse_n1_errata_report
 525 #endif
 526
 527 /*
 528  * Handle trap of EL0 IC IVAU instructions to EL3 by executing a TLB
 529  * inner-shareable invalidation to an arbitrary address followed by a DSB.
 530  *
 531  * x1: Exception Syndrome
 532  */
 533 func neoverse_n1_errata_ic_trap_handler
 534         cmp     x1, #NEOVERSE_N1_EC_IC_TRAP
 535         b.ne    1f
 536         tlbi    vae3is, xzr
 537         dsb     sy
 538
 539         # Skip the IC instruction itself
 540         mrs     x3, elr_el3
 541         add     x3, x3, #4
 542         msr     elr_el3, x3
 543
 544         ldp     x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0]
 545         ldp     x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2]
 546         ldp     x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4]
 547         ldr     x30, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_LR]
 548
 549 #if IMAGE_BL31 && RAS_EXTENSION
 550         /*
 551          * Issue Error Synchronization Barrier to synchronize SErrors before
 552          * exiting EL3. We're running with EAs unmasked, so any synchronized
 553          * errors would be taken immediately; therefore no need to inspect
 554          * DISR_EL1 register.
 555          */
 556         esb
 557 #endif
 558         eret
 559 1:
 560         ret
 561 endfunc neoverse_n1_errata_ic_trap_handler
 562
 563         /* ---------------------------------------------
 564          * This function provides neoverse_n1 specific
 565          * register information for crash reporting.
 566          * It needs to return with x6 pointing to
 567          * a list of register names in ascii and
 568          * x8 - x15 having values of registers to be
 569          * reported.
 570          * ---------------------------------------------
 571          */
 572 .section .rodata.neoverse_n1_regs, "aS"
 573 neoverse_n1_regs:  /* The ascii list of register names to be reported */
 574         .asciz  "cpuectlr_el1", ""
 575
 576 func neoverse_n1_cpu_reg_dump
 577         adr     x6, neoverse_n1_regs
 578         mrs     x8, NEOVERSE_N1_CPUECTLR_EL1
 579         ret
 580 endfunc neoverse_n1_cpu_reg_dump
 581
 582 declare_cpu_ops_eh neoverse_n1, NEOVERSE_N1_MIDR, \
 583         neoverse_n1_reset_func, \
 584         neoverse_n1_errata_ic_trap_handler, \
 585         neoverse_n1_core_pwr_dwn