1 /*
2 * "Cell Reference Set" HTAB support.
3 *
4 * (C) Copyright 2006-2007 TOSHIBA CORPORATION
5 *
6 * This code is based on arch/powerpc/platforms/pseries/lpar.c:
7 * Copyright (C) 2001 Todd Inglett, IBM Corporation
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation; either version 2 of the License, or
12 * (at your option) any later version.
13 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
18 *
19 * You should have received a copy of the GNU General Public License along
20 * with this program; if not, write to the Free Software Foundation, Inc.,
21 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
22 */
23
24 #undef DEBUG_LOW
25
26 #include <linux/kernel.h>
27 #include <linux/spinlock.h>
28
29 #include <asm/mmu.h>
30 #include <asm/page.h>
31 #include <asm/pgtable.h>
32 #include <asm/machdep.h>
33 #include <asm/udbg.h>
34
35 #include "beat_wrapper.h"
36
37 #ifdef DEBUG_LOW
38 #define DBG_LOW(fmt...) do { udbg_printf(fmt); } while (0)
39 #else
40 #define DBG_LOW(fmt...) do { } while (0)
41 #endif
42
43 static DEFINE_RAW_SPINLOCK(beat_htab_lock);
44
beat_read_mask(unsigned hpte_group)45 static inline unsigned int beat_read_mask(unsigned hpte_group)
46 {
47 unsigned long rmask = 0;
48 u64 hpte_v[5];
49
50 beat_read_htab_entries(0, hpte_group + 0, hpte_v);
51 if (!(hpte_v[0] & HPTE_V_BOLTED))
52 rmask |= 0x8000;
53 if (!(hpte_v[1] & HPTE_V_BOLTED))
54 rmask |= 0x4000;
55 if (!(hpte_v[2] & HPTE_V_BOLTED))
56 rmask |= 0x2000;
57 if (!(hpte_v[3] & HPTE_V_BOLTED))
58 rmask |= 0x1000;
59 beat_read_htab_entries(0, hpte_group + 4, hpte_v);
60 if (!(hpte_v[0] & HPTE_V_BOLTED))
61 rmask |= 0x0800;
62 if (!(hpte_v[1] & HPTE_V_BOLTED))
63 rmask |= 0x0400;
64 if (!(hpte_v[2] & HPTE_V_BOLTED))
65 rmask |= 0x0200;
66 if (!(hpte_v[3] & HPTE_V_BOLTED))
67 rmask |= 0x0100;
68 hpte_group = ~hpte_group & (htab_hash_mask * HPTES_PER_GROUP);
69 beat_read_htab_entries(0, hpte_group + 0, hpte_v);
70 if (!(hpte_v[0] & HPTE_V_BOLTED))
71 rmask |= 0x80;
72 if (!(hpte_v[1] & HPTE_V_BOLTED))
73 rmask |= 0x40;
74 if (!(hpte_v[2] & HPTE_V_BOLTED))
75 rmask |= 0x20;
76 if (!(hpte_v[3] & HPTE_V_BOLTED))
77 rmask |= 0x10;
78 beat_read_htab_entries(0, hpte_group + 4, hpte_v);
79 if (!(hpte_v[0] & HPTE_V_BOLTED))
80 rmask |= 0x08;
81 if (!(hpte_v[1] & HPTE_V_BOLTED))
82 rmask |= 0x04;
83 if (!(hpte_v[2] & HPTE_V_BOLTED))
84 rmask |= 0x02;
85 if (!(hpte_v[3] & HPTE_V_BOLTED))
86 rmask |= 0x01;
87 return rmask;
88 }
89
beat_lpar_hpte_insert(unsigned long hpte_group,unsigned long vpn,unsigned long pa,unsigned long rflags,unsigned long vflags,int psize,int apsize,int ssize)90 static long beat_lpar_hpte_insert(unsigned long hpte_group,
91 unsigned long vpn, unsigned long pa,
92 unsigned long rflags, unsigned long vflags,
93 int psize, int apsize, int ssize)
94 {
95 unsigned long lpar_rc;
96 u64 hpte_v, hpte_r, slot;
97
98 if (vflags & HPTE_V_SECONDARY)
99 return -1;
100
101 if (!(vflags & HPTE_V_BOLTED))
102 DBG_LOW("hpte_insert(group=%lx, va=%016lx, pa=%016lx, "
103 "rflags=%lx, vflags=%lx, psize=%d)\n",
104 hpte_group, va, pa, rflags, vflags, psize);
105
106 hpte_v = hpte_encode_v(vpn, psize, apsize, MMU_SEGSIZE_256M) |
107 vflags | HPTE_V_VALID;
108 hpte_r = hpte_encode_r(pa, psize, apsize) | rflags;
109
110 if (!(vflags & HPTE_V_BOLTED))
111 DBG_LOW(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v, hpte_r);
112
113 if (rflags & _PAGE_NO_CACHE)
114 hpte_r &= ~HPTE_R_M;
115
116 raw_spin_lock(&beat_htab_lock);
117 lpar_rc = beat_read_mask(hpte_group);
118 if (lpar_rc == 0) {
119 if (!(vflags & HPTE_V_BOLTED))
120 DBG_LOW(" full\n");
121 raw_spin_unlock(&beat_htab_lock);
122 return -1;
123 }
124
125 lpar_rc = beat_insert_htab_entry(0, hpte_group, lpar_rc << 48,
126 hpte_v, hpte_r, &slot);
127 raw_spin_unlock(&beat_htab_lock);
128
129 /*
130 * Since we try and ioremap PHBs we don't own, the pte insert
131 * will fail. However we must catch the failure in hash_page
132 * or we will loop forever, so return -2 in this case.
133 */
134 if (unlikely(lpar_rc != 0)) {
135 if (!(vflags & HPTE_V_BOLTED))
136 DBG_LOW(" lpar err %lx\n", lpar_rc);
137 return -2;
138 }
139 if (!(vflags & HPTE_V_BOLTED))
140 DBG_LOW(" -> slot: %lx\n", slot);
141
142 /* We have to pass down the secondary bucket bit here as well */
143 return (slot ^ hpte_group) & 15;
144 }
145
beat_lpar_hpte_remove(unsigned long hpte_group)146 static long beat_lpar_hpte_remove(unsigned long hpte_group)
147 {
148 DBG_LOW("hpte_remove(group=%lx)\n", hpte_group);
149 return -1;
150 }
151
beat_lpar_hpte_getword0(unsigned long slot)152 static unsigned long beat_lpar_hpte_getword0(unsigned long slot)
153 {
154 unsigned long dword0;
155 unsigned long lpar_rc;
156 u64 dword[5];
157
158 lpar_rc = beat_read_htab_entries(0, slot & ~3UL, dword);
159
160 dword0 = dword[slot&3];
161
162 BUG_ON(lpar_rc != 0);
163
164 return dword0;
165 }
166
beat_lpar_hptab_clear(void)167 static void beat_lpar_hptab_clear(void)
168 {
169 unsigned long size_bytes = 1UL << ppc64_pft_size;
170 unsigned long hpte_count = size_bytes >> 4;
171 int i;
172 u64 dummy0, dummy1;
173
174 /* TODO: Use bulk call */
175 for (i = 0; i < hpte_count; i++)
176 beat_write_htab_entry(0, i, 0, 0, -1UL, -1UL, &dummy0, &dummy1);
177 }
178
179 /*
180 * NOTE: for updatepp ops we are fortunate that the linux "newpp" bits and
181 * the low 3 bits of flags happen to line up. So no transform is needed.
182 * We can probably optimize here and assume the high bits of newpp are
183 * already zero. For now I am paranoid.
184 */
beat_lpar_hpte_updatepp(unsigned long slot,unsigned long newpp,unsigned long vpn,int psize,int apsize,int ssize,int local)185 static long beat_lpar_hpte_updatepp(unsigned long slot,
186 unsigned long newpp,
187 unsigned long vpn,
188 int psize, int apsize,
189 int ssize, int local)
190 {
191 unsigned long lpar_rc;
192 u64 dummy0, dummy1;
193 unsigned long want_v;
194
195 want_v = hpte_encode_avpn(vpn, psize, MMU_SEGSIZE_256M);
196
197 DBG_LOW(" update: "
198 "avpnv=%016lx, slot=%016lx, psize: %d, newpp %016lx ... ",
199 want_v & HPTE_V_AVPN, slot, psize, newpp);
200
201 raw_spin_lock(&beat_htab_lock);
202 dummy0 = beat_lpar_hpte_getword0(slot);
203 if ((dummy0 & ~0x7FUL) != (want_v & ~0x7FUL)) {
204 DBG_LOW("not found !\n");
205 raw_spin_unlock(&beat_htab_lock);
206 return -1;
207 }
208
209 lpar_rc = beat_write_htab_entry(0, slot, 0, newpp, 0, 7, &dummy0,
210 &dummy1);
211 raw_spin_unlock(&beat_htab_lock);
212 if (lpar_rc != 0 || dummy0 == 0) {
213 DBG_LOW("not found !\n");
214 return -1;
215 }
216
217 DBG_LOW("ok %lx %lx\n", dummy0, dummy1);
218
219 BUG_ON(lpar_rc != 0);
220
221 return 0;
222 }
223
beat_lpar_hpte_find(unsigned long vpn,int psize)224 static long beat_lpar_hpte_find(unsigned long vpn, int psize)
225 {
226 unsigned long hash;
227 unsigned long i, j;
228 long slot;
229 unsigned long want_v, hpte_v;
230
231 hash = hpt_hash(vpn, mmu_psize_defs[psize].shift, MMU_SEGSIZE_256M);
232 want_v = hpte_encode_avpn(vpn, psize, MMU_SEGSIZE_256M);
233
234 for (j = 0; j < 2; j++) {
235 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
236 for (i = 0; i < HPTES_PER_GROUP; i++) {
237 hpte_v = beat_lpar_hpte_getword0(slot);
238
239 if (HPTE_V_COMPARE(hpte_v, want_v)
240 && (hpte_v & HPTE_V_VALID)
241 && (!!(hpte_v & HPTE_V_SECONDARY) == j)) {
242 /* HPTE matches */
243 if (j)
244 slot = -slot;
245 return slot;
246 }
247 ++slot;
248 }
249 hash = ~hash;
250 }
251
252 return -1;
253 }
254
beat_lpar_hpte_updateboltedpp(unsigned long newpp,unsigned long ea,int psize,int ssize)255 static void beat_lpar_hpte_updateboltedpp(unsigned long newpp,
256 unsigned long ea,
257 int psize, int ssize)
258 {
259 unsigned long vpn;
260 unsigned long lpar_rc, slot, vsid;
261 u64 dummy0, dummy1;
262
263 vsid = get_kernel_vsid(ea, MMU_SEGSIZE_256M);
264 vpn = hpt_vpn(ea, vsid, MMU_SEGSIZE_256M);
265
266 raw_spin_lock(&beat_htab_lock);
267 slot = beat_lpar_hpte_find(vpn, psize);
268 BUG_ON(slot == -1);
269
270 lpar_rc = beat_write_htab_entry(0, slot, 0, newpp, 0, 7,
271 &dummy0, &dummy1);
272 raw_spin_unlock(&beat_htab_lock);
273
274 BUG_ON(lpar_rc != 0);
275 }
276
beat_lpar_hpte_invalidate(unsigned long slot,unsigned long vpn,int psize,int apsize,int ssize,int local)277 static void beat_lpar_hpte_invalidate(unsigned long slot, unsigned long vpn,
278 int psize, int apsize,
279 int ssize, int local)
280 {
281 unsigned long want_v;
282 unsigned long lpar_rc;
283 u64 dummy1, dummy2;
284 unsigned long flags;
285
286 DBG_LOW(" inval : slot=%lx, va=%016lx, psize: %d, local: %d\n",
287 slot, va, psize, local);
288 want_v = hpte_encode_avpn(vpn, psize, MMU_SEGSIZE_256M);
289
290 raw_spin_lock_irqsave(&beat_htab_lock, flags);
291 dummy1 = beat_lpar_hpte_getword0(slot);
292
293 if ((dummy1 & ~0x7FUL) != (want_v & ~0x7FUL)) {
294 DBG_LOW("not found !\n");
295 raw_spin_unlock_irqrestore(&beat_htab_lock, flags);
296 return;
297 }
298
299 lpar_rc = beat_write_htab_entry(0, slot, 0, 0, HPTE_V_VALID, 0,
300 &dummy1, &dummy2);
301 raw_spin_unlock_irqrestore(&beat_htab_lock, flags);
302
303 BUG_ON(lpar_rc != 0);
304 }
305
hpte_init_beat(void)306 void __init hpte_init_beat(void)
307 {
308 ppc_md.hpte_invalidate = beat_lpar_hpte_invalidate;
309 ppc_md.hpte_updatepp = beat_lpar_hpte_updatepp;
310 ppc_md.hpte_updateboltedpp = beat_lpar_hpte_updateboltedpp;
311 ppc_md.hpte_insert = beat_lpar_hpte_insert;
312 ppc_md.hpte_remove = beat_lpar_hpte_remove;
313 ppc_md.hpte_clear_all = beat_lpar_hptab_clear;
314 }
315
beat_lpar_hpte_insert_v3(unsigned long hpte_group,unsigned long vpn,unsigned long pa,unsigned long rflags,unsigned long vflags,int psize,int apsize,int ssize)316 static long beat_lpar_hpte_insert_v3(unsigned long hpte_group,
317 unsigned long vpn, unsigned long pa,
318 unsigned long rflags, unsigned long vflags,
319 int psize, int apsize, int ssize)
320 {
321 unsigned long lpar_rc;
322 u64 hpte_v, hpte_r, slot;
323
324 if (vflags & HPTE_V_SECONDARY)
325 return -1;
326
327 if (!(vflags & HPTE_V_BOLTED))
328 DBG_LOW("hpte_insert(group=%lx, vpn=%016lx, pa=%016lx, "
329 "rflags=%lx, vflags=%lx, psize=%d)\n",
330 hpte_group, vpn, pa, rflags, vflags, psize);
331
332 hpte_v = hpte_encode_v(vpn, psize, apsize, MMU_SEGSIZE_256M) |
333 vflags | HPTE_V_VALID;
334 hpte_r = hpte_encode_r(pa, psize, apsize) | rflags;
335
336 if (!(vflags & HPTE_V_BOLTED))
337 DBG_LOW(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v, hpte_r);
338
339 if (rflags & _PAGE_NO_CACHE)
340 hpte_r &= ~HPTE_R_M;
341
342 /* insert into not-volted entry */
343 lpar_rc = beat_insert_htab_entry3(0, hpte_group, hpte_v, hpte_r,
344 HPTE_V_BOLTED, 0, &slot);
345 /*
346 * Since we try and ioremap PHBs we don't own, the pte insert
347 * will fail. However we must catch the failure in hash_page
348 * or we will loop forever, so return -2 in this case.
349 */
350 if (unlikely(lpar_rc != 0)) {
351 if (!(vflags & HPTE_V_BOLTED))
352 DBG_LOW(" lpar err %lx\n", lpar_rc);
353 return -2;
354 }
355 if (!(vflags & HPTE_V_BOLTED))
356 DBG_LOW(" -> slot: %lx\n", slot);
357
358 /* We have to pass down the secondary bucket bit here as well */
359 return (slot ^ hpte_group) & 15;
360 }
361
362 /*
363 * NOTE: for updatepp ops we are fortunate that the linux "newpp" bits and
364 * the low 3 bits of flags happen to line up. So no transform is needed.
365 * We can probably optimize here and assume the high bits of newpp are
366 * already zero. For now I am paranoid.
367 */
beat_lpar_hpte_updatepp_v3(unsigned long slot,unsigned long newpp,unsigned long vpn,int psize,int apsize,int ssize,int local)368 static long beat_lpar_hpte_updatepp_v3(unsigned long slot,
369 unsigned long newpp,
370 unsigned long vpn,
371 int psize, int apsize,
372 int ssize, int local)
373 {
374 unsigned long lpar_rc;
375 unsigned long want_v;
376 unsigned long pss;
377
378 want_v = hpte_encode_avpn(vpn, psize, MMU_SEGSIZE_256M);
379 pss = (psize == MMU_PAGE_4K) ? -1UL : mmu_psize_defs[psize].penc[psize];
380
381 DBG_LOW(" update: "
382 "avpnv=%016lx, slot=%016lx, psize: %d, newpp %016lx ... ",
383 want_v & HPTE_V_AVPN, slot, psize, newpp);
384
385 lpar_rc = beat_update_htab_permission3(0, slot, want_v, pss, 7, newpp);
386
387 if (lpar_rc == 0xfffffff7) {
388 DBG_LOW("not found !\n");
389 return -1;
390 }
391
392 DBG_LOW("ok\n");
393
394 BUG_ON(lpar_rc != 0);
395
396 return 0;
397 }
398
beat_lpar_hpte_invalidate_v3(unsigned long slot,unsigned long vpn,int psize,int apsize,int ssize,int local)399 static void beat_lpar_hpte_invalidate_v3(unsigned long slot, unsigned long vpn,
400 int psize, int apsize,
401 int ssize, int local)
402 {
403 unsigned long want_v;
404 unsigned long lpar_rc;
405 unsigned long pss;
406
407 DBG_LOW(" inval : slot=%lx, vpn=%016lx, psize: %d, local: %d\n",
408 slot, vpn, psize, local);
409 want_v = hpte_encode_avpn(vpn, psize, MMU_SEGSIZE_256M);
410 pss = (psize == MMU_PAGE_4K) ? -1UL : mmu_psize_defs[psize].penc[psize];
411
412 lpar_rc = beat_invalidate_htab_entry3(0, slot, want_v, pss);
413
414 /* E_busy can be valid output: page may be already replaced */
415 BUG_ON(lpar_rc != 0 && lpar_rc != 0xfffffff7);
416 }
417
_beat_lpar_hptab_clear_v3(void)418 static int64_t _beat_lpar_hptab_clear_v3(void)
419 {
420 return beat_clear_htab3(0);
421 }
422
beat_lpar_hptab_clear_v3(void)423 static void beat_lpar_hptab_clear_v3(void)
424 {
425 _beat_lpar_hptab_clear_v3();
426 }
427
hpte_init_beat_v3(void)428 void __init hpte_init_beat_v3(void)
429 {
430 if (_beat_lpar_hptab_clear_v3() == 0) {
431 ppc_md.hpte_invalidate = beat_lpar_hpte_invalidate_v3;
432 ppc_md.hpte_updatepp = beat_lpar_hpte_updatepp_v3;
433 ppc_md.hpte_updateboltedpp = beat_lpar_hpte_updateboltedpp;
434 ppc_md.hpte_insert = beat_lpar_hpte_insert_v3;
435 ppc_md.hpte_remove = beat_lpar_hpte_remove;
436 ppc_md.hpte_clear_all = beat_lpar_hptab_clear_v3;
437 } else {
438 ppc_md.hpte_invalidate = beat_lpar_hpte_invalidate;
439 ppc_md.hpte_updatepp = beat_lpar_hpte_updatepp;
440 ppc_md.hpte_updateboltedpp = beat_lpar_hpte_updateboltedpp;
441 ppc_md.hpte_insert = beat_lpar_hpte_insert;
442 ppc_md.hpte_remove = beat_lpar_hpte_remove;
443 ppc_md.hpte_clear_all = beat_lpar_hptab_clear;
444 }
445 }
446