6#ifndef ZEPHYR_INCLUDE_ARCH_ARM64_CACHE_H_
7#define ZEPHYR_INCLUDE_ARCH_ARM64_CACHE_H_
21#define K_CACHE_WB BIT(0)
22#define K_CACHE_INVD BIT(1)
23#define K_CACHE_WB_INVD (K_CACHE_WB | K_CACHE_INVD)
25#if defined(CONFIG_DCACHE)
27#define CTR_EL0_DMINLINE_SHIFT 16
28#define CTR_EL0_DMINLINE_MASK BIT_MASK(4)
29#define CTR_EL0_CWG_SHIFT 24
30#define CTR_EL0_CWG_MASK BIT_MASK(4)
33#define CLIDR_EL1_LOC_SHIFT 24
34#define CLIDR_EL1_LOC_MASK BIT_MASK(3)
35#define CLIDR_EL1_CTYPE_SHIFT(level) ((level) * 3)
36#define CLIDR_EL1_CTYPE_MASK BIT_MASK(3)
39#define CCSIDR_EL1_LN_SZ_SHIFT 0
40#define CCSIDR_EL1_LN_SZ_MASK BIT_MASK(3)
41#define CCSIDR_EL1_WAYS_SHIFT 3
42#define CCSIDR_EL1_WAYS_MASK BIT_MASK(10)
43#define CCSIDR_EL1_SETS_SHIFT 13
44#define CCSIDR_EL1_SETS_MASK BIT_MASK(15)
46#define dc_ops(op, val) \
48 __asm__ volatile ("dc " op ", %0" :: "r" (val) : "memory"); \
51static size_t dcache_line_size;
58 if (dcache_line_size) {
59 return dcache_line_size;
64 dminline = (ctr_el0 >> CTR_EL0_DMINLINE_SHIFT) & CTR_EL0_DMINLINE_MASK;
66 dcache_line_size = 4 << dminline;
68 return dcache_line_size;
77static ALWAYS_INLINE int arm64_dcache_range(
void *addr,
size_t size,
int op)
106 if (end_addr & (line_size - 1)) {
107 end_addr &= ~(line_size - 1);
108 dc_ops(
"civac", end_addr);
111 if (start_addr & (line_size - 1)) {
112 start_addr &= ~(line_size - 1);
113 if (start_addr == end_addr) {
116 dc_ops(
"civac", start_addr);
117 start_addr += line_size;
122 start_addr &= ~(line_size - 1);
124 while (start_addr < end_addr) {
126 dc_ops(
"ivac", start_addr);
128 dc_ops(
"cvac", start_addr);
130 dc_ops(
"civac", start_addr);
133 start_addr += line_size;
142#ifdef CONFIG_ARM64_DCACHE_ALL_OPS
152 uint32_t clidr_el1, csselr_el1, ccsidr_el1;
153 uint8_t loc, ctype, cache_level, line_size, way_pos;
154 uint32_t max_ways, max_sets, dc_val, set, way;
165 loc = (clidr_el1 >> CLIDR_EL1_LOC_SHIFT) & CLIDR_EL1_LOC_MASK;
170 for (cache_level = 0; cache_level < loc; cache_level++) {
171 ctype = (clidr_el1 >> CLIDR_EL1_CTYPE_SHIFT(cache_level)) & CLIDR_EL1_CTYPE_MASK;
178 csselr_el1 = cache_level << 1;
183 line_size = (ccsidr_el1 >> CCSIDR_EL1_LN_SZ_SHIFT & CCSIDR_EL1_LN_SZ_MASK) + 4;
184 max_ways = (ccsidr_el1 >> CCSIDR_EL1_WAYS_SHIFT) & CCSIDR_EL1_WAYS_MASK;
185 max_sets = (ccsidr_el1 >> CCSIDR_EL1_SETS_SHIFT) & CCSIDR_EL1_SETS_MASK;
187 way_pos = __builtin_clz(max_ways);
189 for (set = 0; set <= max_sets; set++) {
190 for (way = 0; way <= max_ways; way++) {
192 dc_val = way << way_pos;
194 dc_val |= csselr_el1;
196 dc_val |= set << line_size;
199 dc_ops(
"isw", dc_val);
201 dc_ops(
"cisw", dc_val);
203 dc_ops(
"csw", dc_val);
253 return arm64_dcache_range(addr, size,
K_CACHE_WB);
278#if defined(CONFIG_ICACHE)
static ALWAYS_INLINE uint64_t read_clidr_el1(void)
Definition lib_helpers.h:58
static ALWAYS_INLINE uint64_t read_ccsidr_el1(void)
Definition lib_helpers.h:57
static ALWAYS_INLINE void write_csselr_el1(uint64_t val)
Definition lib_helpers.h:68
static ALWAYS_INLINE void arch_cache_init(void)
Definition cache.h:333
#define K_CACHE_INVD
Definition cache.h:22
#define K_CACHE_WB
Definition cache.h:21
#define K_CACHE_WB_INVD
Definition cache.h:23
#define read_sysreg(reg)
Definition lib_helpers.h:100
static ALWAYS_INLINE void barrier_isync_fence_full(void)
Full/sequentially-consistent instruction synchronization barrier.
Definition barrier.h:78
static ALWAYS_INLINE void barrier_dsync_fence_full(void)
Full/sequentially-consistent data synchronization barrier.
Definition barrier.h:59
void arch_dcache_disable(void)
Disable the d-cache.
void arch_icache_disable(void)
Disable the i-cache.
int arch_dcache_invd_range(void *addr, size_t size)
Invalidate an address range in the d-cache.
int arch_icache_flush_and_invd_all(void)
Flush and Invalidate the i-cache.
int arch_icache_flush_all(void)
Flush the i-cache.
int arch_icache_invd_all(void)
Invalidate the i-cache.
int arch_dcache_flush_range(void *addr, size_t size)
Flush an address range in the d-cache.
size_t arch_dcache_line_size_get(void)
Get the d-cache line size.
size_t arch_icache_line_size_get(void)
Get the i-cache line size.
int arch_icache_flush_range(void *addr, size_t size)
Flush an address range in the i-cache.
int arch_icache_invd_range(void *addr, size_t size)
Invalidate an address range in the i-cache.
int arch_dcache_flush_all(void)
Flush the d-cache.
int arch_icache_flush_and_invd_range(void *addr, size_t size)
Flush and Invalidate an address range in the i-cache.
int arch_dcache_flush_and_invd_all(void)
Flush and Invalidate the d-cache.
int arch_dcache_invd_all(void)
Invalidate the d-cache.
void arch_icache_enable(void)
Enable the i-cache.
int arch_dcache_flush_and_invd_range(void *addr, size_t size)
Flush and Invalidate an address range in the d-cache.
void arch_dcache_enable(void)
Enable the d-cache.
#define ENOTSUP
Unsupported value.
Definition errno.h:114
__UINT32_TYPE__ uint32_t
Definition stdint.h:90
__UINT64_TYPE__ uint64_t
Definition stdint.h:91
__UINT8_TYPE__ uint8_t
Definition stdint.h:88
__UINTPTR_TYPE__ uintptr_t
Definition stdint.h:105