diff --git a/src/arch/aarch32/cache_helpers.S b/src/arch/aarch32/cache_helpers.S new file mode 100644 index 0000000000000000000000000000000000000000..37b31305c8f6d4a407e77fa5c3ed81beef91f058 --- /dev/null +++ b/src/arch/aarch32/cache_helpers.S @@ -0,0 +1,233 @@ +/* + * Copyright (c) 2016-2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include +#include + + .globl flush_dcache_range + .globl clean_dcache_range + .globl inv_dcache_range + .globl dcsw_op_louis + .globl dcsw_op_all + .globl dcsw_op_level1 + .globl dcsw_op_level2 + .globl dcsw_op_level3 + +/* + * This macro can be used for implementing various data cache operations `op` + */ +.macro do_dcache_maintenance_by_mva op, coproc, opc1, CRn, CRm, opc2 + /* Exit early if size is zero */ + cmp r1, #0 + beq exit_loop_\op + dcache_line_size r2, r3 + add r1, r0, r1 + sub r3, r2, #1 + bic r0, r0, r3 +loop_\op: + stcopr r0, \coproc, \opc1, \CRn, \CRm, \opc2 + add r0, r0, r2 + cmp r0, r1 + blo loop_\op + dsb sy +exit_loop_\op: + bx lr +.endm + + /* ------------------------------------------ + * Clean+Invalidate from base address till + * size. 'r0' = addr, 'r1' = size + * ------------------------------------------ + */ +func flush_dcache_range + do_dcache_maintenance_by_mva cimvac, DCCIMVAC +endfunc flush_dcache_range + + /* ------------------------------------------ + * Clean from base address till size. + * 'r0' = addr, 'r1' = size + * ------------------------------------------ + */ +func clean_dcache_range + do_dcache_maintenance_by_mva cmvac, DCCMVAC +endfunc clean_dcache_range + + /* ------------------------------------------ + * Invalidate from base address till + * size. 'r0' = addr, 'r1' = size + * ------------------------------------------ + */ +func inv_dcache_range + do_dcache_maintenance_by_mva imvac, DCIMVAC +endfunc inv_dcache_range + + /* ---------------------------------------------------------------- + * Data cache operations by set/way to the level specified + * + * The main function, do_dcsw_op requires: + * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), + * as defined in arch.h + * r1: The cache level to begin operation from + * r2: clidr_el1 + * r3: The last cache level to operate on + * and will carry out the operation on each data cache from level 0 + * to the level in r3 in sequence + * + * The dcsw_op macro sets up the r2 and r3 parameters based on + * clidr_el1 cache information before invoking the main function + * ---------------------------------------------------------------- + */ + + .macro dcsw_op shift, fw, ls + ldcopr r2, CLIDR + ubfx r3, r2, \shift, \fw + lsl r3, r3, \ls + mov r1, #0 + b do_dcsw_op + .endm + +func do_dcsw_op + push {r4-r12, lr} + ldcopr r8, ID_MMFR4 // stash FEAT_CCIDX identifier in r8 + ubfx r8, r8, #ID_MMFR4_CCIDX_SHIFT, #ID_MMFR4_CCIDX_LENGTH + adr r11, dcsw_loop_table // compute cache op based on the operation type + add r6, r11, r0, lsl #3 // cache op is 2x32-bit instructions +loop1: + add r10, r1, r1, LSR #1 // Work out 3x current cache level + mov r12, r2, LSR r10 // extract cache type bits from clidr + and r12, r12, #7 // mask the bits for current cache only + cmp r12, #2 // see what cache we have at this level + blo level_done // no cache or only instruction cache at this level + + stcopr r1, CSSELR // select current cache level in csselr + isb // isb to sych the new cssr&csidr + ldcopr r12, CCSIDR // read the new ccsidr + and r10, r12, #7 // extract the length of the cache lines + add r10, r10, #4 // add 4 (r10 = line length offset) + + cmp r8, #0 // check for FEAT_CCIDX for Associativity + beq 1f + ubfx r4, r12, #3, #21 // r4 = associativity CCSIDR[23:3] + b 2f +1: + ubfx r4, r12, #3, #10 // r4 = associativity CCSIDR[12:3] +2: + clz r5, r4 // r5 = the bit position of the way size increment + mov r9, r4 // r9 working copy of the aligned max way number + +loop2: + cmp r8, #0 // check for FEAT_CCIDX for NumSets + beq 3f + ldcopr r12, CCSIDR2 // FEAT_CCIDX numsets is in CCSIDR2 + ubfx r7, r12, #0, #24 // r7 = numsets CCSIDR2[23:0] + b loop3 +3: + ubfx r7, r12, #13, #15 // r7 = numsets CCSIDR[27:13] +loop3: + orr r0, r1, r9, LSL r5 // factor in the way number and cache level into r0 + orr r0, r0, r7, LSL r10 // factor in the set number + + blx r6 + subs r7, r7, #1 // decrement the set number + bhs loop3 + subs r9, r9, #1 // decrement the way number + bhs loop2 +level_done: + add r1, r1, #2 // increment the cache number + cmp r3, r1 + // Ensure completion of previous cache maintenance instruction. Note + // this also mitigates erratum 814220 on Cortex-A7 + dsb sy + bhi loop1 + + mov r6, #0 + stcopr r6, CSSELR //select cache level 0 in csselr + dsb sy + isb + pop {r4-r12, pc} + +dcsw_loop_table: + stcopr r0, DCISW + bx lr + stcopr r0, DCCISW + bx lr + stcopr r0, DCCSW + bx lr + +endfunc do_dcsw_op + + /* --------------------------------------------------------------- + * Data cache operations by set/way till PoU. + * + * The function requires : + * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), + * as defined in arch.h + * --------------------------------------------------------------- + */ +func dcsw_op_louis + dcsw_op #LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT +endfunc dcsw_op_louis + + /* --------------------------------------------------------------- + * Data cache operations by set/way till PoC. + * + * The function requires : + * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), + * as defined in arch.h + * --------------------------------------------------------------- + */ +func dcsw_op_all + dcsw_op #LOC_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT +endfunc dcsw_op_all + + + /* --------------------------------------------------------------- + * Helper macro for data cache operations by set/way for the + * level specified + * --------------------------------------------------------------- + */ + .macro dcsw_op_level level + ldcopr r2, CLIDR + mov r3, \level + sub r1, r3, #2 + b do_dcsw_op + .endm + + /* --------------------------------------------------------------- + * Data cache operations by set/way for level 1 cache + * + * The main function, do_dcsw_op requires: + * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), + * as defined in arch.h + * --------------------------------------------------------------- + */ +func dcsw_op_level1 + dcsw_op_level #(1 << LEVEL_SHIFT) +endfunc dcsw_op_level1 + + /* --------------------------------------------------------------- + * Data cache operations by set/way for level 2 cache + * + * The main function, do_dcsw_op requires: + * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), + * as defined in arch.h + * --------------------------------------------------------------- + */ +func dcsw_op_level2 + dcsw_op_level #(2 << LEVEL_SHIFT) +endfunc dcsw_op_level2 + + /* --------------------------------------------------------------- + * Data cache operations by set/way for level 3 cache + * + * The main function, do_dcsw_op requires: + * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), + * as defined in arch.h + * --------------------------------------------------------------- + */ +func dcsw_op_level3 + dcsw_op_level #(3 << LEVEL_SHIFT) +endfunc dcsw_op_level3 diff --git a/src/arch/aarch32/inc/arch/cacheflush.h b/src/arch/aarch32/inc/arch/cacheflush.h new file mode 100644 index 0000000000000000000000000000000000000000..48005589b5601ab8a428e169e9604c5051c591af --- /dev/null +++ b/src/arch/aarch32/inc/arch/cacheflush.h @@ -0,0 +1,9 @@ +#ifndef _CACHEFLUSH_H_ +#define _CACHEFLUSH_H_ +#include + +void flush_dcache_range(uintptr_t addr, size_t size); +void inv_dcache_range(uintptr_t addr, size_t size); +void clean_dcache_range(uintptr_t addr, size_t size); + +#endif /* _CACHEFLUSH_H_ */ diff --git a/src/arch/aarch32/sources.mk b/src/arch/aarch32/sources.mk index 7baa248c9fd32394f0d4871829d3b876438fb942..b086c6a0a111d9150351bb5110f262b07ee5e81c 100644 --- a/src/arch/aarch32/sources.mk +++ b/src/arch/aarch32/sources.mk @@ -1,4 +1,4 @@ arch_c_srcs:= interrupts.c cpu.c gicv3.c arm_mpu.c -arch_s_srcs:= boot.S plat_helpers.S spinlock.S exceptions.S +arch_s_srcs:= boot.S plat_helpers.S spinlock.S exceptions.S cache_helpers.S -include $(ARCH_DIR)/mpu/sources.mk diff --git a/src/arch/aarch64/cache_helpers.S b/src/arch/aarch64/cache_helpers.S new file mode 100644 index 0000000000000000000000000000000000000000..ad15c5d37a0fc8bed98dd47e8559fd30985e7b0a --- /dev/null +++ b/src/arch/aarch64/cache_helpers.S @@ -0,0 +1,254 @@ +/* + * Copyright (c) 2013-2022, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include +#include + + .globl flush_dcache_range + .globl flush_dcache_to_popa_range + .globl clean_dcache_range + .globl inv_dcache_range + .globl dcsw_op_louis + .globl dcsw_op_all + .globl dcsw_op_level1 + .globl dcsw_op_level2 + .globl dcsw_op_level3 + +/* + * This macro can be used for implementing various data cache operations `op` + */ +.macro do_dcache_maintenance_by_mva op + /* Exit early if size is zero */ + cbz x1, exit_loop_\op + dcache_line_size x2, x3 + add x1, x0, x1 + sub x3, x2, #1 + bic x0, x0, x3 +loop_\op: + dc \op, x0 + add x0, x0, x2 + cmp x0, x1 + b.lo loop_\op + dsb sy +exit_loop_\op: + ret +.endm + /* ------------------------------------------ + * Clean+Invalidate from base address till + * size. 'x0' = addr, 'x1' = size + * ------------------------------------------ + */ +func flush_dcache_range + do_dcache_maintenance_by_mva civac +endfunc flush_dcache_range + + /* ------------------------------------------ + * Clean from base address till size. + * 'x0' = addr, 'x1' = size + * ------------------------------------------ + */ +func clean_dcache_range + do_dcache_maintenance_by_mva cvac +endfunc clean_dcache_range + + /* ------------------------------------------ + * Invalidate from base address till + * size. 'x0' = addr, 'x1' = size + * ------------------------------------------ + */ +func inv_dcache_range + do_dcache_maintenance_by_mva ivac +endfunc inv_dcache_range + + + /* + * On implementations with FEAT_MTE2, + * Root firmware must issue DC_CIGDPAPA instead of DC_CIPAPA , + * in order to additionally clean and invalidate Allocation Tags + * associated with the affected locations. + * + * ------------------------------------------ + * Clean+Invalidate by PA to POPA + * from base address till size. + * 'x0' = addr, 'x1' = size + * ------------------------------------------ + */ +func flush_dcache_to_popa_range + /* Exit early if size is zero */ + cbz x1, exit_loop_dc_cipapa + dcache_line_size x2, x3 + sub x3, x2, #1 + bic x0, x0, x3 + add x1, x1, x0 +loop_dc_cipapa: + sys #6, c7, c14, #1, x0 /* DC CIPAPA, */ + add x0, x0, x2 + cmp x0, x1 + b.lo loop_dc_cipapa + dsb osh +exit_loop_dc_cipapa: + ret +endfunc flush_dcache_to_popa_range + + /* --------------------------------------------------------------- + * Data cache operations by set/way to the level specified + * + * The main function, do_dcsw_op requires: + * x0: The operation type (0-2), as defined in arch.h + * x3: The last cache level to operate on + * x9: clidr_el1 + * x10: The cache level to begin operation from + * and will carry out the operation on each data cache from level 0 + * to the level in x3 in sequence + * + * The dcsw_op macro sets up the x3 and x9 parameters based on + * clidr_el1 cache information before invoking the main function + * --------------------------------------------------------------- + */ + + .macro dcsw_op shift, fw, ls + mrs x9, clidr_el1 + ubfx x3, x9, \shift, \fw + lsl x3, x3, \ls + mov x10, xzr + b do_dcsw_op + .endm + +func do_dcsw_op + cbz x3, exit + mrs x12, ID_AA64MMFR2_EL1 // stash FEAT_CCIDX identifier in x12 + ubfx x12, x12, #ID_AA64MMFR2_EL1_CCIDX_SHIFT, #ID_AA64MMFR2_EL1_CCIDX_LENGTH + adr x14, dcsw_loop_table // compute inner loop address + add x14, x14, x0, lsl #5 // inner loop is 8x32-bit instructions +#if ENABLE_BTI + add x14, x14, x0, lsl #2 // inner loop is + "bti j" instruction +#endif + mov x0, x9 + mov w8, #1 +loop1: + add x2, x10, x10, lsr #1 // work out 3x current cache level + lsr x1, x0, x2 // extract cache type bits from clidr + and x1, x1, #7 // mask the bits for current cache only + cmp x1, #2 // see what cache we have at this level + b.lo level_done // nothing to do if no cache or icache + + msr csselr_el1, x10 // select current cache level in csselr + isb // isb to sych the new cssr&csidr + mrs x1, ccsidr_el1 // read the new ccsidr + and x2, x1, #7 // extract the length of the cache lines + add x2, x2, #4 // add 4 (line length offset) + + cbz x12, 1f // check for FEAT_CCIDX for Associativity + ubfx x4, x1, #3, #21 // x4 = associativity CCSIDR_EL1[23:3] + b 2f +1: + ubfx x4, x1, #3, #10 // x4 = associativity CCSIDR_EL1[12:3] +2: + clz w5, w4 // bit position of way size increment + lsl w9, w4, w5 // w9 = aligned max way number + lsl w16, w8, w5 // w16 = way number loop decrement + orr w9, w10, w9 // w9 = combine way and cache number + + cbz x12, 3f // check for FEAT_CCIDX for NumSets + ubfx x6, x1, #32, #24 // x6 (w6) = numsets CCSIDR_EL1[55:32] + // ISA will not allow x->w ubfx + b 4f +3: + ubfx w6, w1, #13, #15 // w6 = numsets CCSIDR_EL1[27:13] +4: + lsl w17, w8, w2 // w17 = set number loop decrement + dsb sy // barrier before we start this level + br x14 // jump to DC operation specific loop + + .macro dcsw_loop _op +#if ENABLE_BTI + bti j +#endif +loop2_\_op: + lsl w7, w6, w2 // w7 = aligned max set number + +loop3_\_op: + orr w11, w9, w7 // combine cache, way and set number + dc \_op, x11 + subs w7, w7, w17 // decrement set number + b.hs loop3_\_op + + subs x9, x9, x16 // decrement way number + b.hs loop2_\_op + + b level_done + .endm + +level_done: + add x10, x10, #2 // increment cache number + cmp x3, x10 + b.hi loop1 + msr csselr_el1, xzr // select cache level 0 in csselr + dsb sy // barrier to complete final cache operation + isb +exit: + ret +endfunc do_dcsw_op + +dcsw_loop_table: + dcsw_loop isw + dcsw_loop cisw + dcsw_loop csw + + +func dcsw_op_louis + dcsw_op #LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT +endfunc dcsw_op_louis + + +func dcsw_op_all + dcsw_op #LOC_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT +endfunc dcsw_op_all + + /* --------------------------------------------------------------- + * Helper macro for data cache operations by set/way for the + * level specified + * --------------------------------------------------------------- + */ + .macro dcsw_op_level level + mrs x9, clidr_el1 + mov x3, \level + sub x10, x3, #2 + b do_dcsw_op + .endm + + /* --------------------------------------------------------------- + * Data cache operations by set/way for level 1 cache + * + * The main function, do_dcsw_op requires: + * x0: The operation type (0-2), as defined in arch.h + * --------------------------------------------------------------- + */ +func dcsw_op_level1 + dcsw_op_level #(1 << LEVEL_SHIFT) +endfunc dcsw_op_level1 + + /* --------------------------------------------------------------- + * Data cache operations by set/way for level 2 cache + * + * The main function, do_dcsw_op requires: + * x0: The operation type (0-2), as defined in arch.h + * --------------------------------------------------------------- + */ +func dcsw_op_level2 + dcsw_op_level #(2 << LEVEL_SHIFT) +endfunc dcsw_op_level2 + + /* --------------------------------------------------------------- + * Data cache operations by set/way for level 3 cache + * + * The main function, do_dcsw_op requires: + * x0: The operation type (0-2), as defined in arch.h + * --------------------------------------------------------------- + */ +func dcsw_op_level3 + dcsw_op_level #(3 << LEVEL_SHIFT) +endfunc dcsw_op_level3 diff --git a/src/arch/aarch64/inc/arch.h b/src/arch/aarch64/inc/arch.h index 24503b75dccd5144a6a8f59cdab458f555d4c014..03582088010393ccc9fbdbcc57a31e9c1a52c33f 100644 --- a/src/arch/aarch64/inc/arch.h +++ b/src/arch/aarch64/inc/arch.h @@ -156,6 +156,9 @@ #define ID_AA64PFR0_GIC_SHIFT 24 #define ID_AA64PFR0_GIC_WIDTH 4 #define ID_AA64PFR0_GIC_MASK ((1 << ID_AA64PFR0_GIC_WIDTH) - 1) +#define ID_AA64MMFR2_EL1_CCIDX_SHIFT U(20) +#define ID_AA64MMFR2_EL1_CCIDX_MASK ULL(0xf) +#define ID_AA64MMFR2_EL1_CCIDX_LENGTH U(4) /* ID_PFR1_EL1 definitions */ #define ID_PFR1_VIRTEXT_SHIFT 12 diff --git a/src/arch/aarch64/inc/arch/cacheflush.h b/src/arch/aarch64/inc/arch/cacheflush.h new file mode 100644 index 0000000000000000000000000000000000000000..48005589b5601ab8a428e169e9604c5051c591af --- /dev/null +++ b/src/arch/aarch64/inc/arch/cacheflush.h @@ -0,0 +1,9 @@ +#ifndef _CACHEFLUSH_H_ +#define _CACHEFLUSH_H_ +#include + +void flush_dcache_range(uintptr_t addr, size_t size); +void inv_dcache_range(uintptr_t addr, size_t size); +void clean_dcache_range(uintptr_t addr, size_t size); + +#endif /* _CACHEFLUSH_H_ */ diff --git a/src/arch/aarch64/inc/asm_macros.h b/src/arch/aarch64/inc/asm_macros.h index 459ed9c1ed0001b5ab21f28f017076f65980210d..943ba37720980bf7a1543deef87220b6b0f8b682 100644 --- a/src/arch/aarch64/inc/asm_macros.h +++ b/src/arch/aarch64/inc/asm_macros.h @@ -1,6 +1,21 @@ #ifndef __ASM_MACROS_S__ #define __ASM_MACROS_S__ + .macro dcache_line_size reg, tmp + mrs \tmp, ctr_el0 + ubfx \tmp, \tmp, #16, #4 + mov \reg, #4 + lsl \reg, \reg, \tmp + .endm + + + .macro icache_line_size reg, tmp + mrs \tmp, ctr_el0 + and \tmp, \tmp, #0xf + mov \reg, #4 + lsl \reg, \reg, \tmp + .endm + /* * This macro is used to create a function label and place the diff --git a/src/arch/aarch64/sources.mk b/src/arch/aarch64/sources.mk index 8624cac4492d59bd40f40c76bcaff99a3978751d..1ae01621f679ea39cd39a4c31e608fb5de6f1fc8 100644 --- a/src/arch/aarch64/sources.mk +++ b/src/arch/aarch64/sources.mk @@ -1,2 +1,2 @@ arch_c_srcs:= gicv3.c cpu.c interrupts.c mmu.c -arch_s_srcs:= boot.S exceptions.S plat_helpers.S debug.S page_tables.S +arch_s_srcs:= boot.S exceptions.S plat_helpers.S debug.S page_tables.S cache_helpers.S diff --git a/tests/quick_start/7_cache_maint.c b/tests/quick_start/7_cache_maint.c new file mode 100644 index 0000000000000000000000000000000000000000..17d8a57b1afa18598f9f2c37faaf96a184937005 --- /dev/null +++ b/tests/quick_start/7_cache_maint.c @@ -0,0 +1,54 @@ +/* + * IO访问(寄存器 内存)示例 + */ +/* C头文件，支持标准C库, 可以跨平台 */ +#include +#include +#include +#include +#include +/* + * tests_lib.h包含了测试框架的定义 + * 可以在测例中使用断言(assertion) + */ +#include +/* + * 访问寄存器/内存，需要引用中的定义，用*p方式访问会有跨平台问题 + */ +#include +/* + * cache maintenace + */ +#include + +static int cache_maint_run(void) +{ + uintptr_t phys_addr = 0x40000000; /* 这个地址只是示意 */ + uint8_t val8; + uint16_t val16; + uint32_t val32; + + clean_dcache_range(phys_addr, 0x10000); + inv_dcache_range(phys_addr, 0x10000); + flush_dcache_range(phys_addr, 0x10000); + + val8 = 0x12; + mmio_write_8(phys_addr, val8); + ASSERT_EQ(val8, mmio_read_8(phys_addr)); + + val16 = 0x1234; + mmio_write_16(phys_addr, val16); + ASSERT_EQ(val16, mmio_read_16(phys_addr)); + + val32 = 0x12345678; + mmio_write_32(phys_addr, val32); + ASSERT_EQ(val32, mmio_read_32(phys_addr)); + +} + +DECLARE_TEST_CASE( /* 声明了一个单元测例, 测试框架会依次调用下面的init(), run(), exit()函数 */ + cache_maint_example, /* 测例名字，不用带引号"" */ + NULL, /* 测例预初始化函数，若不需要可以为NULL. 比如可以把复位和时钟初始化放这里 */ + cache_maint_run, /* 测例执行主体函数，必须定义 */ + NULL /* 测例退出函数，若不需要可以为NULL. 比如结束测例后需要关时钟 */ +); diff --git a/tests/quick_start/sources.mk b/tests/quick_start/sources.mk index 2ebb6727c32ccda9e06ab454bae316b2a72d85a4..8cd73632550ebcbc3ff3f7c992d9b28d1a10e917 100644 --- a/tests/quick_start/sources.mk +++ b/tests/quick_start/sources.mk @@ -10,5 +10,6 @@ test_c_srcs += $(addprefix quick_start/, \ 4_smp_basic.c \ 5_smp_synchronization.c \ 6_embedded_images.c \ + 7_cache_maint.c \ )