Zephyr Project API 4.1.99
A Scalable Open Source RTOS
Loading...
Searching...
No Matches
cache.h
Go to the documentation of this file.
1/*
2 * Copyright 2022 Carlo Caione <ccaione@baylibre.com>
3 *
4 * SPDX-License-Identifier: Apache-2.0
5 */
6#ifndef ZEPHYR_INCLUDE_ARCH_ARM64_CACHE_H_
7#define ZEPHYR_INCLUDE_ARCH_ARM64_CACHE_H_
8
9#ifndef _ASMLANGUAGE
10
11#include <zephyr/types.h>
12#include <zephyr/sys/util.h>
13#include <zephyr/sys/barrier.h>
14#include <zephyr/arch/cpu.h>
15#include <errno.h>
16
17#ifdef __cplusplus
18extern "C" {
19#endif
20
21#define K_CACHE_WB BIT(0)
22#define K_CACHE_INVD BIT(1)
23#define K_CACHE_WB_INVD (K_CACHE_WB | K_CACHE_INVD)
24
25#if defined(CONFIG_DCACHE)
26
27#define CTR_EL0_DMINLINE_SHIFT 16
28#define CTR_EL0_DMINLINE_MASK BIT_MASK(4)
29#define CTR_EL0_CWG_SHIFT 24
30#define CTR_EL0_CWG_MASK BIT_MASK(4)
31
32/* clidr_el1 */
33#define CLIDR_EL1_LOC_SHIFT 24
34#define CLIDR_EL1_LOC_MASK BIT_MASK(3)
35#define CLIDR_EL1_CTYPE_SHIFT(level) ((level) * 3)
36#define CLIDR_EL1_CTYPE_MASK BIT_MASK(3)
37
38/* ccsidr_el1 */
39#define CCSIDR_EL1_LN_SZ_SHIFT 0
40#define CCSIDR_EL1_LN_SZ_MASK BIT_MASK(3)
41#define CCSIDR_EL1_WAYS_SHIFT 3
42#define CCSIDR_EL1_WAYS_MASK BIT_MASK(10)
43#define CCSIDR_EL1_SETS_SHIFT 13
44#define CCSIDR_EL1_SETS_MASK BIT_MASK(15)
45
46#define dc_ops(op, val) \
47({ \
48 __asm__ volatile ("dc " op ", %0" :: "r" (val) : "memory"); \
49})
50
51static size_t dcache_line_size;
52
54{
55 uint64_t ctr_el0;
56 uint32_t dminline;
57
58 if (dcache_line_size) {
59 return dcache_line_size;
60 }
61
62 ctr_el0 = read_sysreg(CTR_EL0);
63
64 dminline = (ctr_el0 >> CTR_EL0_DMINLINE_SHIFT) & CTR_EL0_DMINLINE_MASK;
65
66 dcache_line_size = 4 << dminline;
67
68 return dcache_line_size;
69}
70
71/*
72 * operation for data cache by virtual address to PoC
73 * ops: K_CACHE_INVD: invalidate
74 * K_CACHE_WB: clean
75 * K_CACHE_WB_INVD: clean and invalidate
76 */
77static ALWAYS_INLINE int arm64_dcache_range(void *addr, size_t size, int op)
78{
79 size_t line_size;
80 uintptr_t start_addr = (uintptr_t)addr;
81 uintptr_t end_addr = start_addr + size;
82
83 if (op != K_CACHE_INVD && op != K_CACHE_WB && op != K_CACHE_WB_INVD) {
84 return -ENOTSUP;
85 }
86
87 line_size = arch_dcache_line_size_get();
88
89 /*
90 * For the data cache invalidate operation, clean and invalidate
91 * the partial cache lines at both ends of the given range to
92 * prevent data corruption.
93 *
94 * For example (assume cache line size is 64 bytes):
95 * There are 2 consecutive 32-byte buffers, which can be cached in
96 * one line like below.
97 * +------------------+------------------+
98 * Cache line: | buffer 0 (dirty) | buffer 1 |
99 * +------------------+------------------+
100 * For the start address not aligned case, when invalidate the
101 * buffer 1, the full cache line will be invalidated, if the buffer
102 * 0 is dirty, its data will be lost.
103 * The same logic applies to the not aligned end address.
104 */
105 if (op == K_CACHE_INVD) {
106 if (end_addr & (line_size - 1)) {
107 end_addr &= ~(line_size - 1);
108 dc_ops("civac", end_addr);
109 }
110
111 if (start_addr & (line_size - 1)) {
112 start_addr &= ~(line_size - 1);
113 if (start_addr == end_addr) {
114 goto done;
115 }
116 dc_ops("civac", start_addr);
117 start_addr += line_size;
118 }
119 }
120
121 /* Align address to line size */
122 start_addr &= ~(line_size - 1);
123
124 while (start_addr < end_addr) {
125 if (op == K_CACHE_INVD) {
126 dc_ops("ivac", start_addr);
127 } else if (op == K_CACHE_WB) {
128 dc_ops("cvac", start_addr);
129 } else if (op == K_CACHE_WB_INVD) {
130 dc_ops("civac", start_addr);
131 }
132
133 start_addr += line_size;
134 }
135
136done:
138
139 return 0;
140}
141
142#ifdef CONFIG_ARM64_DCACHE_ALL_OPS
143
144/*
145 * operation for all data cache
146 * ops: K_CACHE_INVD: invalidate
147 * K_CACHE_WB: clean
148 * K_CACHE_WB_INVD: clean and invalidate
149 */
150static ALWAYS_INLINE int arm64_dcache_all(int op)
151{
152 uint32_t clidr_el1, csselr_el1, ccsidr_el1;
153 uint8_t loc, ctype, cache_level, line_size, way_pos;
154 uint32_t max_ways, max_sets, dc_val, set, way;
155
156 if (op != K_CACHE_INVD && op != K_CACHE_WB && op != K_CACHE_WB_INVD) {
157 return -ENOTSUP;
158 }
159
160 /* Data barrier before start */
162
163 clidr_el1 = read_clidr_el1();
164
165 loc = (clidr_el1 >> CLIDR_EL1_LOC_SHIFT) & CLIDR_EL1_LOC_MASK;
166 if (!loc) {
167 return 0;
168 }
169
170 for (cache_level = 0; cache_level < loc; cache_level++) {
171 ctype = (clidr_el1 >> CLIDR_EL1_CTYPE_SHIFT(cache_level)) & CLIDR_EL1_CTYPE_MASK;
172 /* No data cache, continue */
173 if (ctype < 2) {
174 continue;
175 }
176
177 /* select cache level */
178 csselr_el1 = cache_level << 1;
179 write_csselr_el1(csselr_el1);
181
182 ccsidr_el1 = read_ccsidr_el1();
183 line_size = (ccsidr_el1 >> CCSIDR_EL1_LN_SZ_SHIFT & CCSIDR_EL1_LN_SZ_MASK) + 4;
184 max_ways = (ccsidr_el1 >> CCSIDR_EL1_WAYS_SHIFT) & CCSIDR_EL1_WAYS_MASK;
185 max_sets = (ccsidr_el1 >> CCSIDR_EL1_SETS_SHIFT) & CCSIDR_EL1_SETS_MASK;
186 /* 32-log2(ways), bit position of way in DC operand */
187 way_pos = __builtin_clz(max_ways);
188
189 for (set = 0; set <= max_sets; set++) {
190 for (way = 0; way <= max_ways; way++) {
191 /* way number, aligned to pos in DC operand */
192 dc_val = way << way_pos;
193 /* cache level, aligned to pos in DC operand */
194 dc_val |= csselr_el1;
195 /* set number, aligned to pos in DC operand */
196 dc_val |= set << line_size;
197
198 if (op == K_CACHE_INVD) {
199 dc_ops("isw", dc_val);
200 } else if (op == K_CACHE_WB_INVD) {
201 dc_ops("cisw", dc_val);
202 } else if (op == K_CACHE_WB) {
203 dc_ops("csw", dc_val);
204 }
205 }
206 }
207 }
208
209 /* Restore csselr_el1 to level 0 */
213
214 return 0;
215}
216
218{
219 return arm64_dcache_all(K_CACHE_WB);
220}
221
222static ALWAYS_INLINE int arch_dcache_invd_all(void)
223{
224 return arm64_dcache_all(K_CACHE_INVD);
225}
226
228{
229 return arm64_dcache_all(K_CACHE_WB_INVD);
230}
231
232#else
233
235{
236 return -ENOTSUP;
237}
238
239static ALWAYS_INLINE int arch_dcache_invd_all(void)
240{
241 return -ENOTSUP;
242}
243
245{
246 return -ENOTSUP;
247}
248
249#endif /* CONFIG_ARM64_DCACHE_ALL_OPS */
250
251static ALWAYS_INLINE int arch_dcache_flush_range(void *addr, size_t size)
252{
253 return arm64_dcache_range(addr, size, K_CACHE_WB);
254}
255
256static ALWAYS_INLINE int arch_dcache_invd_range(void *addr, size_t size)
257{
258 return arm64_dcache_range(addr, size, K_CACHE_INVD);
259}
260
261static ALWAYS_INLINE int arch_dcache_flush_and_invd_range(void *addr, size_t size)
262{
263 return arm64_dcache_range(addr, size, K_CACHE_WB_INVD);
264}
265
266static ALWAYS_INLINE void arch_dcache_enable(void)
267{
268 /* nothing */
269}
270
271static ALWAYS_INLINE void arch_dcache_disable(void)
272{
273 /* nothing */
274}
275
276#endif /* CONFIG_DCACHE */
277
278#if defined(CONFIG_ICACHE)
279
281{
282 return -ENOTSUP;
283}
284
286{
287 return -ENOTSUP;
288}
289
290static ALWAYS_INLINE int arch_icache_invd_all(void)
291{
292 return -ENOTSUP;
293}
294
296{
297 return -ENOTSUP;
298}
299
300static ALWAYS_INLINE int arch_icache_flush_range(void *addr, size_t size)
301{
302 ARG_UNUSED(addr);
303 ARG_UNUSED(size);
304 return -ENOTSUP;
305}
306
307static ALWAYS_INLINE int arch_icache_invd_range(void *addr, size_t size)
308{
309 ARG_UNUSED(addr);
310 ARG_UNUSED(size);
311 return -ENOTSUP;
312}
313
314static ALWAYS_INLINE int arch_icache_flush_and_invd_range(void *addr, size_t size)
315{
316 ARG_UNUSED(addr);
317 ARG_UNUSED(size);
318 return -ENOTSUP;
319}
320
321static ALWAYS_INLINE void arch_icache_enable(void)
322{
323 /* nothing */
324}
325
326static ALWAYS_INLINE void arch_icache_disable(void)
327{
328 /* nothing */
329}
330
331#endif /* CONFIG_ICACHE */
332
334{
335}
336
337#ifdef __cplusplus
338}
339#endif
340
341#endif /* _ASMLANGUAGE */
342
343#endif /* ZEPHYR_INCLUDE_ARCH_ARM64_CACHE_H_ */
static ALWAYS_INLINE uint64_t read_clidr_el1(void)
Definition lib_helpers.h:58
static ALWAYS_INLINE uint64_t read_ccsidr_el1(void)
Definition lib_helpers.h:57
static ALWAYS_INLINE void write_csselr_el1(uint64_t val)
Definition lib_helpers.h:68
static ALWAYS_INLINE void arch_cache_init(void)
Definition cache.h:333
#define K_CACHE_INVD
Definition cache.h:22
#define K_CACHE_WB
Definition cache.h:21
#define K_CACHE_WB_INVD
Definition cache.h:23
#define read_sysreg(reg)
Definition lib_helpers.h:100
System error numbers.
static ALWAYS_INLINE void barrier_isync_fence_full(void)
Full/sequentially-consistent instruction synchronization barrier.
Definition barrier.h:78
static ALWAYS_INLINE void barrier_dsync_fence_full(void)
Full/sequentially-consistent data synchronization barrier.
Definition barrier.h:59
void arch_dcache_disable(void)
Disable the d-cache.
void arch_icache_disable(void)
Disable the i-cache.
int arch_dcache_invd_range(void *addr, size_t size)
Invalidate an address range in the d-cache.
int arch_icache_flush_and_invd_all(void)
Flush and Invalidate the i-cache.
int arch_icache_flush_all(void)
Flush the i-cache.
int arch_icache_invd_all(void)
Invalidate the i-cache.
int arch_dcache_flush_range(void *addr, size_t size)
Flush an address range in the d-cache.
size_t arch_dcache_line_size_get(void)
Get the d-cache line size.
size_t arch_icache_line_size_get(void)
Get the i-cache line size.
int arch_icache_flush_range(void *addr, size_t size)
Flush an address range in the i-cache.
int arch_icache_invd_range(void *addr, size_t size)
Invalidate an address range in the i-cache.
int arch_dcache_flush_all(void)
Flush the d-cache.
int arch_icache_flush_and_invd_range(void *addr, size_t size)
Flush and Invalidate an address range in the i-cache.
int arch_dcache_flush_and_invd_all(void)
Flush and Invalidate the d-cache.
int arch_dcache_invd_all(void)
Invalidate the d-cache.
void arch_icache_enable(void)
Enable the i-cache.
int arch_dcache_flush_and_invd_range(void *addr, size_t size)
Flush and Invalidate an address range in the d-cache.
void arch_dcache_enable(void)
Enable the d-cache.
#define ENOTSUP
Unsupported value.
Definition errno.h:114
#define ALWAYS_INLINE
Definition common.h:160
__UINT32_TYPE__ uint32_t
Definition stdint.h:90
__UINT64_TYPE__ uint64_t
Definition stdint.h:91
__UINT8_TYPE__ uint8_t
Definition stdint.h:88
__UINTPTR_TYPE__ uintptr_t
Definition stdint.h:105
Misc utilities.