1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2020 Loongson Technology Corporation Limited
4 */
5 #include <linux/fs.h>
6 #include <linux/kernel.h>
7 #include <linux/linkage.h>
8 #include <linux/export.h>
9 #include <linux/sched.h>
10 #include <linux/syscalls.h>
11 #include <linux/mm.h>
12 #include <linux/cacheinfo.h>
13 #include <linux/cpu_pm.h>
14 #include <linux/highmem.h>
15
16 #include <asm/cacheflush.h>
17 #include <asm/processor.h>
18 #include <asm/cpu.h>
19 #include <asm/cpu-features.h>
20 #include <asm/numa.h>
21 #include <asm/setup.h>
22 #include <asm/dma.h>
23 #include <loongson.h>
24
cache_error_setup(void)25 void cache_error_setup(void)
26 {
27 extern char __weak except_vec_cex;
28 set_merr_handler(0x0, &except_vec_cex, 0x80);
29 }
30
31 /*
32 * LoongArch maintains ICache/DCache coherency by hardware,
33 * we just need "ibar" to avoid instruction hazard here.
34 */
local_flush_icache_range(unsigned long start,unsigned long end)35 void local_flush_icache_range(unsigned long start, unsigned long end)
36 {
37 asm volatile ("\tibar 0\n"::);
38 }
39 EXPORT_SYMBOL(local_flush_icache_range);
40
flush_cache_leaf(unsigned int leaf)41 static void flush_cache_leaf(unsigned int leaf)
42 {
43 int i, j, nr_nodes;
44 uint64_t addr = CSR_DMW0_BASE;
45 struct cache_desc *cdesc = current_cpu_data.cache_leaves + leaf;
46
47 nr_nodes = cache_private(cdesc) ? 1 : loongson_sysconf.nr_nodes;
48
49 do {
50 for (i = 0; i < cdesc->sets; i++) {
51 for (j = 0; j < cdesc->ways; j++) {
52 flush_cache_line(leaf, addr);
53 addr++;
54 }
55
56 addr -= cdesc->ways;
57 addr += cdesc->linesz;
58 }
59 addr += (1ULL << NODE_ADDRSPACE_SHIFT);
60 } while (--nr_nodes > 0);
61 }
62
__flush_cache_all(void)63 asmlinkage __visible void __flush_cache_all(void)
64 {
65 int leaf;
66 struct cache_desc *cdesc = current_cpu_data.cache_leaves;
67 unsigned int cache_present = current_cpu_data.cache_leaves_present;
68
69 leaf = cache_present - 1;
70 if (cache_inclusive(cdesc + leaf)) {
71 flush_cache_leaf(leaf);
72 return;
73 }
74
75 for (leaf = 0; leaf < cache_present; leaf++)
76 flush_cache_leaf(leaf);
77 }
78
79 #define L1IUPRE (1 << 0)
80 #define L1IUUNIFY (1 << 1)
81 #define L1DPRE (1 << 2)
82
83 #define LXIUPRE (1 << 0)
84 #define LXIUUNIFY (1 << 1)
85 #define LXIUPRIV (1 << 2)
86 #define LXIUINCL (1 << 3)
87 #define LXDPRE (1 << 4)
88 #define LXDPRIV (1 << 5)
89 #define LXDINCL (1 << 6)
90
91 #define populate_cache_properties(cfg0, cdesc, level, leaf) \
92 do { \
93 unsigned int cfg1; \
94 \
95 cfg1 = read_cpucfg(LOONGARCH_CPUCFG17 + leaf); \
96 if (level == 1) { \
97 cdesc->flags |= CACHE_PRIVATE; \
98 } else { \
99 if (cfg0 & LXIUPRIV) \
100 cdesc->flags |= CACHE_PRIVATE; \
101 if (cfg0 & LXIUINCL) \
102 cdesc->flags |= CACHE_INCLUSIVE; \
103 } \
104 cdesc->level = level; \
105 cdesc->flags |= CACHE_PRESENT; \
106 cdesc->ways = ((cfg1 & CPUCFG_CACHE_WAYS_M) >> CPUCFG_CACHE_WAYS) + 1; \
107 cdesc->sets = 1 << ((cfg1 & CPUCFG_CACHE_SETS_M) >> CPUCFG_CACHE_SETS); \
108 cdesc->linesz = 1 << ((cfg1 & CPUCFG_CACHE_LSIZE_M) >> CPUCFG_CACHE_LSIZE); \
109 cdesc++; leaf++; \
110 } while (0)
111
cpu_cache_init(void)112 void cpu_cache_init(void)
113 {
114 unsigned int leaf = 0, level = 1;
115 unsigned int config = read_cpucfg(LOONGARCH_CPUCFG16);
116 struct cache_desc *cdesc = current_cpu_data.cache_leaves;
117
118 if (config & L1IUPRE) {
119 if (config & L1IUUNIFY)
120 cdesc->type = CACHE_TYPE_UNIFIED;
121 else
122 cdesc->type = CACHE_TYPE_INST;
123 populate_cache_properties(config, cdesc, level, leaf);
124 }
125
126 if (config & L1DPRE) {
127 cdesc->type = CACHE_TYPE_DATA;
128 populate_cache_properties(config, cdesc, level, leaf);
129 }
130
131 config = config >> 3;
132 for (level = 2; level <= CACHE_LEVEL_MAX; level++) {
133 if (!config)
134 break;
135
136 if (config & LXIUPRE) {
137 if (config & LXIUUNIFY)
138 cdesc->type = CACHE_TYPE_UNIFIED;
139 else
140 cdesc->type = CACHE_TYPE_INST;
141 populate_cache_properties(config, cdesc, level, leaf);
142 }
143
144 if (config & LXDPRE) {
145 cdesc->type = CACHE_TYPE_DATA;
146 populate_cache_properties(config, cdesc, level, leaf);
147 }
148
149 config = config >> 7;
150 }
151
152 BUG_ON(leaf > CACHE_LEAVES_MAX);
153
154 current_cpu_data.cache_leaves_present = leaf;
155 current_cpu_data.options |= LOONGARCH_CPU_PREFETCH;
156 }
157