From 80942622fe760c23f0a677eac48aff37e90f4251 Mon Sep 17 00:00:00 2001 From: laurenw-arm Date: Tue, 20 Aug 2019 15:51:24 -0500 Subject: [PATCH] Neoverse N1 Errata Workaround 1542419 Coherent I-cache is causing a prefetch violation where when the core executes an instruction that has recently been modified, the core might fetch a stale instruction which violates the ordering of instruction fetches. The workaround includes an instruction sequence to implementation defined registers to trap all EL0 IC IVAU instructions to EL3 and a trap handler to execute a TLB inner-shareable invalidation to an arbitrary address followed by a DSB. Signed-off-by: Lauren Wehrmeister Change-Id: Ic3b7cbb11cf2eaf9005523ef5578a372593ae4d6 --- bl31/aarch64/ea_delegate.S | 39 +++++++++-- docs/design/cpu-specific-build-macros.rst | 3 + include/lib/cpus/aarch64/cpu_macros.S | 19 +++-- include/lib/cpus/aarch64/neoverse_n1.h | 3 + lib/cpus/aarch64/cpu_helpers.S | 21 ++++++ lib/cpus/aarch64/neoverse_n1.S | 85 ++++++++++++++++++++++- lib/cpus/cpu-ops.mk | 8 +++ 7 files changed, 167 insertions(+), 11 deletions(-) diff --git a/bl31/aarch64/ea_delegate.S b/bl31/aarch64/ea_delegate.S index 6e71a063..3cc4d56a 100644 --- a/bl31/aarch64/ea_delegate.S +++ b/bl31/aarch64/ea_delegate.S @@ -11,7 +11,8 @@ #include #include #include - +#include +#include .globl handle_lower_el_ea_esb .globl enter_lower_el_sync_ea @@ -35,9 +36,9 @@ endfunc handle_lower_el_ea_esb /* * This function forms the tail end of Synchronous Exception entry from lower - * EL, and expects to handle only Synchronous External Aborts from lower EL. If - * any other kind of exception is detected, then this function reports unhandled - * exception. + * EL, and expects to handle Synchronous External Aborts from lower EL and CPU + * Implementation Defined Exceptions. If any other kind of exception is detected, + * then this function reports unhandled exception. * * Since it's part of exception vector, this function doesn't expect any GP * registers to have been saved. It delegates the handling of the EA to platform @@ -58,12 +59,33 @@ func enter_lower_el_sync_ea b.eq 1f cmp x30, #EC_DABORT_LOWER_EL - b.ne 2f + b.eq 1f + + /* Save GP registers */ + stp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0] + stp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2] + stp x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4] + + /* Get the cpu_ops pointer */ + bl get_cpu_ops_ptr + + /* Get the cpu_ops exception handler */ + ldr x0, [x0, #CPU_E_HANDLER_FUNC] + + /* + * If the reserved function pointer is NULL, this CPU does not have an + * implementation defined exception handler function + */ + cbz x0, 2f + mrs x1, esr_el3 + ubfx x1, x1, #ESR_EC_SHIFT, #ESR_EC_LENGTH + blr x0 + b 2f 1: /* Test for EA bit in the instruction syndrome */ mrs x30, esr_el3 - tbz x30, #ESR_ISS_EABORT_EA_BIT, 2f + tbz x30, #ESR_ISS_EABORT_EA_BIT, 3f /* * Save general purpose and ARMv8.3-PAuth registers (if enabled). @@ -84,6 +106,11 @@ func enter_lower_el_sync_ea b delegate_sync_ea 2: + ldp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0] + ldp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2] + ldp x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4] + +3: /* Synchronous exceptions other than the above are assumed to be EA */ ldr x30, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_LR] no_ret report_unhandled_exception diff --git a/docs/design/cpu-specific-build-macros.rst b/docs/design/cpu-specific-build-macros.rst index d3fe89d6..a392eea2 100644 --- a/docs/design/cpu-specific-build-macros.rst +++ b/docs/design/cpu-specific-build-macros.rst @@ -258,6 +258,9 @@ For Neoverse N1, the following errata build flags are defined : - ``ERRATA_N1_1315703``: This applies errata 1315703 workaround to Neoverse-N1 CPU. This needs to be enabled only for revision <= r3p0 of the CPU. +- ``ERRATA_N1_1542419``: This applies errata 1542419 workaround to Neoverse-N1 + CPU. This needs to be enabled only for revisions r3p0 - r4p0 of the CPU. + DSU Errata Workarounds ---------------------- diff --git a/include/lib/cpus/aarch64/cpu_macros.S b/include/lib/cpus/aarch64/cpu_macros.S index 044aacaf..c83824d7 100644 --- a/include/lib/cpus/aarch64/cpu_macros.S +++ b/include/lib/cpus/aarch64/cpu_macros.S @@ -43,6 +43,7 @@ .equ CPU_MIDR_SIZE, CPU_WORD_SIZE .equ CPU_EXTRA1_FUNC_SIZE, CPU_WORD_SIZE .equ CPU_EXTRA2_FUNC_SIZE, CPU_WORD_SIZE + .equ CPU_E_HANDLER_FUNC_SIZE, CPU_WORD_SIZE .equ CPU_RESET_FUNC_SIZE, CPU_WORD_SIZE .equ CPU_PWR_DWN_OPS_SIZE, CPU_WORD_SIZE * CPU_MAX_PWR_DWN_OPS .equ CPU_ERRATA_FUNC_SIZE, CPU_WORD_SIZE @@ -83,7 +84,8 @@ .equ CPU_RESET_FUNC, CPU_MIDR + CPU_MIDR_SIZE .equ CPU_EXTRA1_FUNC, CPU_RESET_FUNC + CPU_RESET_FUNC_SIZE .equ CPU_EXTRA2_FUNC, CPU_EXTRA1_FUNC + CPU_EXTRA1_FUNC_SIZE - .equ CPU_PWR_DWN_OPS, CPU_EXTRA2_FUNC + CPU_EXTRA2_FUNC_SIZE + .equ CPU_E_HANDLER_FUNC, CPU_EXTRA2_FUNC + CPU_EXTRA2_FUNC_SIZE + .equ CPU_PWR_DWN_OPS, CPU_E_HANDLER_FUNC + CPU_E_HANDLER_FUNC_SIZE .equ CPU_ERRATA_FUNC, CPU_PWR_DWN_OPS + CPU_PWR_DWN_OPS_SIZE .equ CPU_ERRATA_LOCK, CPU_ERRATA_FUNC + CPU_ERRATA_FUNC_SIZE .equ CPU_ERRATA_PRINTED, CPU_ERRATA_LOCK + CPU_ERRATA_LOCK_SIZE @@ -139,6 +141,8 @@ * This is a placeholder for future per CPU operations. Currently * some CPUs use this entry to set a function to disable the * workaround for CVE-2018-3639. + * _e_handler: + * This is a placeholder for future per CPU exception handlers. * _power_down_ops: * Comma-separated list of functions to perform power-down * operatios on the CPU. At least one, and up to @@ -149,7 +153,7 @@ * used to handle power down at subsequent levels */ .macro declare_cpu_ops_base _name:req, _midr:req, _resetfunc:req, \ - _extra1:req, _extra2:req, _power_down_ops:vararg + _extra1:req, _extra2:req, _e_handler:req, _power_down_ops:vararg .section cpu_ops, "a" .align 3 .type cpu_ops_\_name, %object @@ -159,6 +163,7 @@ #endif .quad \_extra1 .quad \_extra2 + .quad \_e_handler #ifdef IMAGE_BL31 /* Insert list of functions */ fill_constants CPU_MAX_PWR_DWN_OPS, \_power_down_ops @@ -203,15 +208,21 @@ .macro declare_cpu_ops _name:req, _midr:req, _resetfunc:req, \ _power_down_ops:vararg - declare_cpu_ops_base \_name, \_midr, \_resetfunc, 0, 0, \ + declare_cpu_ops_base \_name, \_midr, \_resetfunc, 0, 0, 0, \ \_power_down_ops .endm + .macro declare_cpu_ops_eh _name:req, _midr:req, _resetfunc:req, \ + _e_handler:req, _power_down_ops:vararg + declare_cpu_ops_base \_name, \_midr, \_resetfunc, \ + 0, 0, \_e_handler, \_power_down_ops + .endm + .macro declare_cpu_ops_wa _name:req, _midr:req, \ _resetfunc:req, _extra1:req, _extra2:req, \ _power_down_ops:vararg declare_cpu_ops_base \_name, \_midr, \_resetfunc, \ - \_extra1, \_extra2, \_power_down_ops + \_extra1, \_extra2, 0, \_power_down_ops .endm #if REPORT_ERRATA diff --git a/include/lib/cpus/aarch64/neoverse_n1.h b/include/lib/cpus/aarch64/neoverse_n1.h index f90aa2ea..fa733ce1 100644 --- a/include/lib/cpus/aarch64/neoverse_n1.h +++ b/include/lib/cpus/aarch64/neoverse_n1.h @@ -12,6 +12,9 @@ /* Neoverse N1 MIDR for revision 0 */ #define NEOVERSE_N1_MIDR U(0x410fd0c0) +/* Exception Syndrome register EC code for IC Trap */ +#define NEOVERSE_N1_EC_IC_TRAP U(0x1f) + /******************************************************************************* * CPU Power Control register specific definitions. ******************************************************************************/ diff --git a/lib/cpus/aarch64/cpu_helpers.S b/lib/cpus/aarch64/cpu_helpers.S index de1177c3..808c7f80 100644 --- a/lib/cpus/aarch64/cpu_helpers.S +++ b/lib/cpus/aarch64/cpu_helpers.S @@ -227,6 +227,27 @@ func cpu_rev_var_hs ret endfunc cpu_rev_var_hs +/* + * Compare the CPU's revision-variant (x0) with a given range (x1 - x2), for errata + * application purposes. If the revision-variant is between or includes the given + * values, this indicates that errata applies; otherwise not. + * + * Shall clobber: x0-x4 + */ + .globl cpu_rev_var_range +func cpu_rev_var_range + mov x3, #ERRATA_APPLIES + mov x4, #ERRATA_NOT_APPLIES + cmp x0, x1 + csel x1, x3, x4, hs + cbz x1, 1f + cmp x0, x2 + csel x1, x3, x4, ls +1: + mov x0, x1 + ret +endfunc cpu_rev_var_range + #if REPORT_ERRATA /* * void print_errata_status(void); diff --git a/lib/cpus/aarch64/neoverse_n1.S b/lib/cpus/aarch64/neoverse_n1.S index 31e7a3a7..c9bb005e 100644 --- a/lib/cpus/aarch64/neoverse_n1.S +++ b/lib/cpus/aarch64/neoverse_n1.S @@ -9,6 +9,7 @@ #include #include #include +#include /* Hardware handled coherency */ #if HW_ASSISTED_COHERENCY == 0 @@ -20,6 +21,10 @@ #error "Neoverse-N1 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" #endif +#if ERRATA_N1_IC_TRAP + .global neoverse_n1_errata_ic_trap_handler +#endif + /* -------------------------------------------------- * Errata Workaround for Neoverse N1 Erratum 1043202. * This applies to revision r0p0 and r1p0 of Neoverse N1. @@ -337,6 +342,41 @@ func check_errata_1315703 b cpu_rev_var_ls endfunc check_errata_1315703 +/* -------------------------------------------------- + * Errata Workaround for Neoverse N1 Erratum 1542419. + * This applies to revisions r3p0 - r4p0 of Neoverse N1 + * Inputs: + * x0: variant[4:7] and revision[0:3] of current cpu. + * Shall clobber: x0-x17 + * -------------------------------------------------- + */ +func errata_n1_1542419_wa + /* Compare x0 against revision r3p0 and r4p0 */ + mov x17, x30 + bl check_errata_1542419 + cbz x0, 1f + + /* Apply instruction patching sequence */ + ldr x0, =0x0 + msr CPUPSELR_EL3, x0 + ldr x0, =0xEE670D35 + msr CPUPOR_EL3, x0 + ldr x0, =0xFFFF0FFF + msr CPUPMR_EL3, x0 + ldr x0, =0x08000020007D + msr CPUPCR_EL3, x0 + isb +1: + ret x17 +endfunc errata_n1_1542419_wa + +func check_errata_1542419 + /* Applies to everything r3p0 - r4p0. */ + mov x1, #0x30 + mov x2, #0x40 + b cpu_rev_var_range +endfunc check_errata_1542419 + func neoverse_n1_reset_func mov x19, x30 @@ -406,6 +446,11 @@ func neoverse_n1_reset_func bl errata_n1_1315703_wa #endif +#if ERRATA_N1_1542419 + mov x0, x18 + bl errata_n1_1542419_wa +#endif + #if ENABLE_AMU /* Make sure accesses from EL0/EL1 and EL2 are not trapped to EL3 */ mrs x0, actlr_el3 @@ -471,6 +516,7 @@ func neoverse_n1_errata_report report_errata ERRATA_N1_1262888, neoverse_n1, 1262888 report_errata ERRATA_N1_1275112, neoverse_n1, 1275112 report_errata ERRATA_N1_1315703, neoverse_n1, 1315703 + report_errata ERRATA_N1_1542419, neoverse_n1, 1542419 report_errata ERRATA_DSU_936184, neoverse_n1, dsu_936184 ldp x8, x30, [sp], #16 @@ -478,6 +524,42 @@ func neoverse_n1_errata_report endfunc neoverse_n1_errata_report #endif +/* + * Handle trap of EL0 IC IVAU instructions to EL3 by executing a TLB + * inner-shareable invalidation to an arbitrary address followed by a DSB. + * + * x1: Exception Syndrome + */ +func neoverse_n1_errata_ic_trap_handler + cmp x1, #NEOVERSE_N1_EC_IC_TRAP + b.ne 1f + tlbi vae3is, xzr + dsb sy + + # Skip the IC instruction itself + mrs x3, elr_el3 + add x3, x3, #4 + msr elr_el3, x3 + + ldp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0] + ldp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2] + ldp x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4] + ldr x30, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_LR] + +#if IMAGE_BL31 && RAS_EXTENSION + /* + * Issue Error Synchronization Barrier to synchronize SErrors before + * exiting EL3. We're running with EAs unmasked, so any synchronized + * errors would be taken immediately; therefore no need to inspect + * DISR_EL1 register. + */ + esb +#endif + eret +1: + ret +endfunc neoverse_n1_errata_ic_trap_handler + /* --------------------------------------------- * This function provides neoverse_n1 specific * register information for crash reporting. @@ -497,6 +579,7 @@ func neoverse_n1_cpu_reg_dump ret endfunc neoverse_n1_cpu_reg_dump -declare_cpu_ops neoverse_n1, NEOVERSE_N1_MIDR, \ +declare_cpu_ops_eh neoverse_n1, NEOVERSE_N1_MIDR, \ neoverse_n1_reset_func, \ + neoverse_n1_errata_ic_trap_handler, \ neoverse_n1_core_pwr_dwn diff --git a/lib/cpus/cpu-ops.mk b/lib/cpus/cpu-ops.mk index 26040235..078888eb 100644 --- a/lib/cpus/cpu-ops.mk +++ b/lib/cpus/cpu-ops.mk @@ -278,6 +278,10 @@ ERRATA_N1_1275112 ?=0 # to revisions before r3p1 of the Neoverse N1 cpu. ERRATA_N1_1315703 ?=1 +# Flag to apply erratum 1542419 workaround during reset. This erratum applies +# to revisions r3p0 - r4p0 of the Neoverse N1 cpu. +ERRATA_N1_1542419 ?=0 + # Flag to apply DSU erratum 798953. This erratum applies to DSUs revision r0p0. # Applying the workaround results in higher DSU power consumption on idle. ERRATA_DSU_798953 ?=0 @@ -507,6 +511,10 @@ $(eval $(call add_define,ERRATA_N1_1275112)) $(eval $(call assert_boolean,ERRATA_N1_1315703)) $(eval $(call add_define,ERRATA_N1_1315703)) +# Process ERRATA_N1_1542419 flag +$(eval $(call assert_boolean,ERRATA_N1_1542419)) +$(eval $(call add_define,ERRATA_N1_1542419)) + # Process ERRATA_DSU_798953 flag $(eval $(call assert_boolean,ERRATA_DSU_798953)) $(eval $(call add_define,ERRATA_DSU_798953)) -- 2.30.2