• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * "Cell Reference Set" HTAB support.
3  *
4  * (C) Copyright 2006-2007 TOSHIBA CORPORATION
5  *
6  * This code is based on arch/powerpc/platforms/pseries/lpar.c:
7  *  Copyright (C) 2001 Todd Inglett, IBM Corporation
8  *
9  * This program is free software; you can redistribute it and/or modify
10  * it under the terms of the GNU General Public License as published by
11  * the Free Software Foundation; either version 2 of the License, or
12  * (at your option) any later version.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17  * GNU General Public License for more details.
18  *
19  * You should have received a copy of the GNU General Public License along
20  * with this program; if not, write to the Free Software Foundation, Inc.,
21  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
22  */
23 
24 #undef DEBUG_LOW
25 
26 #include <linux/kernel.h>
27 #include <linux/spinlock.h>
28 
29 #include <asm/mmu.h>
30 #include <asm/page.h>
31 #include <asm/pgtable.h>
32 #include <asm/machdep.h>
33 #include <asm/udbg.h>
34 
35 #include "beat_wrapper.h"
36 
37 #ifdef DEBUG_LOW
38 #define DBG_LOW(fmt...) do { udbg_printf(fmt); } while (0)
39 #else
40 #define DBG_LOW(fmt...) do { } while (0)
41 #endif
42 
43 static DEFINE_SPINLOCK(beat_htab_lock);
44 
beat_read_mask(unsigned hpte_group)45 static inline unsigned int beat_read_mask(unsigned hpte_group)
46 {
47 	unsigned long rmask = 0;
48 	u64 hpte_v[5];
49 
50 	beat_read_htab_entries(0, hpte_group + 0, hpte_v);
51 	if (!(hpte_v[0] & HPTE_V_BOLTED))
52 		rmask |= 0x8000;
53 	if (!(hpte_v[1] & HPTE_V_BOLTED))
54 		rmask |= 0x4000;
55 	if (!(hpte_v[2] & HPTE_V_BOLTED))
56 		rmask |= 0x2000;
57 	if (!(hpte_v[3] & HPTE_V_BOLTED))
58 		rmask |= 0x1000;
59 	beat_read_htab_entries(0, hpte_group + 4, hpte_v);
60 	if (!(hpte_v[0] & HPTE_V_BOLTED))
61 		rmask |= 0x0800;
62 	if (!(hpte_v[1] & HPTE_V_BOLTED))
63 		rmask |= 0x0400;
64 	if (!(hpte_v[2] & HPTE_V_BOLTED))
65 		rmask |= 0x0200;
66 	if (!(hpte_v[3] & HPTE_V_BOLTED))
67 		rmask |= 0x0100;
68 	hpte_group = ~hpte_group & (htab_hash_mask * HPTES_PER_GROUP);
69 	beat_read_htab_entries(0, hpte_group + 0, hpte_v);
70 	if (!(hpte_v[0] & HPTE_V_BOLTED))
71 		rmask |= 0x80;
72 	if (!(hpte_v[1] & HPTE_V_BOLTED))
73 		rmask |= 0x40;
74 	if (!(hpte_v[2] & HPTE_V_BOLTED))
75 		rmask |= 0x20;
76 	if (!(hpte_v[3] & HPTE_V_BOLTED))
77 		rmask |= 0x10;
78 	beat_read_htab_entries(0, hpte_group + 4, hpte_v);
79 	if (!(hpte_v[0] & HPTE_V_BOLTED))
80 		rmask |= 0x08;
81 	if (!(hpte_v[1] & HPTE_V_BOLTED))
82 		rmask |= 0x04;
83 	if (!(hpte_v[2] & HPTE_V_BOLTED))
84 		rmask |= 0x02;
85 	if (!(hpte_v[3] & HPTE_V_BOLTED))
86 		rmask |= 0x01;
87 	return rmask;
88 }
89 
beat_lpar_hpte_insert(unsigned long hpte_group,unsigned long va,unsigned long pa,unsigned long rflags,unsigned long vflags,int psize,int ssize)90 static long beat_lpar_hpte_insert(unsigned long hpte_group,
91 				  unsigned long va, unsigned long pa,
92 				  unsigned long rflags, unsigned long vflags,
93 				  int psize, int ssize)
94 {
95 	unsigned long lpar_rc;
96 	u64 hpte_v, hpte_r, slot;
97 
98 	/* same as iseries */
99 	if (vflags & HPTE_V_SECONDARY)
100 		return -1;
101 
102 	if (!(vflags & HPTE_V_BOLTED))
103 		DBG_LOW("hpte_insert(group=%lx, va=%016lx, pa=%016lx, "
104 			"rflags=%lx, vflags=%lx, psize=%d)\n",
105 		hpte_group, va, pa, rflags, vflags, psize);
106 
107 	hpte_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M) |
108 		vflags | HPTE_V_VALID;
109 	hpte_r = hpte_encode_r(pa, psize) | rflags;
110 
111 	if (!(vflags & HPTE_V_BOLTED))
112 		DBG_LOW(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v, hpte_r);
113 
114 	if (rflags & _PAGE_NO_CACHE)
115 		hpte_r &= ~_PAGE_COHERENT;
116 
117 	spin_lock(&beat_htab_lock);
118 	lpar_rc = beat_read_mask(hpte_group);
119 	if (lpar_rc == 0) {
120 		if (!(vflags & HPTE_V_BOLTED))
121 			DBG_LOW(" full\n");
122 		spin_unlock(&beat_htab_lock);
123 		return -1;
124 	}
125 
126 	lpar_rc = beat_insert_htab_entry(0, hpte_group, lpar_rc << 48,
127 		hpte_v, hpte_r, &slot);
128 	spin_unlock(&beat_htab_lock);
129 
130 	/*
131 	 * Since we try and ioremap PHBs we don't own, the pte insert
132 	 * will fail. However we must catch the failure in hash_page
133 	 * or we will loop forever, so return -2 in this case.
134 	 */
135 	if (unlikely(lpar_rc != 0)) {
136 		if (!(vflags & HPTE_V_BOLTED))
137 			DBG_LOW(" lpar err %lx\n", lpar_rc);
138 		return -2;
139 	}
140 	if (!(vflags & HPTE_V_BOLTED))
141 		DBG_LOW(" -> slot: %lx\n", slot);
142 
143 	/* We have to pass down the secondary bucket bit here as well */
144 	return (slot ^ hpte_group) & 15;
145 }
146 
beat_lpar_hpte_remove(unsigned long hpte_group)147 static long beat_lpar_hpte_remove(unsigned long hpte_group)
148 {
149 	DBG_LOW("hpte_remove(group=%lx)\n", hpte_group);
150 	return -1;
151 }
152 
beat_lpar_hpte_getword0(unsigned long slot)153 static unsigned long beat_lpar_hpte_getword0(unsigned long slot)
154 {
155 	unsigned long dword0;
156 	unsigned long lpar_rc;
157 	u64 dword[5];
158 
159 	lpar_rc = beat_read_htab_entries(0, slot & ~3UL, dword);
160 
161 	dword0 = dword[slot&3];
162 
163 	BUG_ON(lpar_rc != 0);
164 
165 	return dword0;
166 }
167 
beat_lpar_hptab_clear(void)168 static void beat_lpar_hptab_clear(void)
169 {
170 	unsigned long size_bytes = 1UL << ppc64_pft_size;
171 	unsigned long hpte_count = size_bytes >> 4;
172 	int i;
173 	u64 dummy0, dummy1;
174 
175 	/* TODO: Use bulk call */
176 	for (i = 0; i < hpte_count; i++)
177 		beat_write_htab_entry(0, i, 0, 0, -1UL, -1UL, &dummy0, &dummy1);
178 }
179 
180 /*
181  * NOTE: for updatepp ops we are fortunate that the linux "newpp" bits and
182  * the low 3 bits of flags happen to line up.  So no transform is needed.
183  * We can probably optimize here and assume the high bits of newpp are
184  * already zero.  For now I am paranoid.
185  */
beat_lpar_hpte_updatepp(unsigned long slot,unsigned long newpp,unsigned long va,int psize,int ssize,int local)186 static long beat_lpar_hpte_updatepp(unsigned long slot,
187 				    unsigned long newpp,
188 				    unsigned long va,
189 				    int psize, int ssize, int local)
190 {
191 	unsigned long lpar_rc;
192 	u64 dummy0, dummy1;
193 	unsigned long want_v;
194 
195 	want_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M);
196 
197 	DBG_LOW("    update: "
198 		"avpnv=%016lx, slot=%016lx, psize: %d, newpp %016lx ... ",
199 		want_v & HPTE_V_AVPN, slot, psize, newpp);
200 
201 	spin_lock(&beat_htab_lock);
202 	dummy0 = beat_lpar_hpte_getword0(slot);
203 	if ((dummy0 & ~0x7FUL) != (want_v & ~0x7FUL)) {
204 		DBG_LOW("not found !\n");
205 		spin_unlock(&beat_htab_lock);
206 		return -1;
207 	}
208 
209 	lpar_rc = beat_write_htab_entry(0, slot, 0, newpp, 0, 7, &dummy0,
210 					&dummy1);
211 	spin_unlock(&beat_htab_lock);
212 	if (lpar_rc != 0 || dummy0 == 0) {
213 		DBG_LOW("not found !\n");
214 		return -1;
215 	}
216 
217 	DBG_LOW("ok %lx %lx\n", dummy0, dummy1);
218 
219 	BUG_ON(lpar_rc != 0);
220 
221 	return 0;
222 }
223 
beat_lpar_hpte_find(unsigned long va,int psize)224 static long beat_lpar_hpte_find(unsigned long va, int psize)
225 {
226 	unsigned long hash;
227 	unsigned long i, j;
228 	long slot;
229 	unsigned long want_v, hpte_v;
230 
231 	hash = hpt_hash(va, mmu_psize_defs[psize].shift, MMU_SEGSIZE_256M);
232 	want_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M);
233 
234 	for (j = 0; j < 2; j++) {
235 		slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
236 		for (i = 0; i < HPTES_PER_GROUP; i++) {
237 			hpte_v = beat_lpar_hpte_getword0(slot);
238 
239 			if (HPTE_V_COMPARE(hpte_v, want_v)
240 			    && (hpte_v & HPTE_V_VALID)
241 			    && (!!(hpte_v & HPTE_V_SECONDARY) == j)) {
242 				/* HPTE matches */
243 				if (j)
244 					slot = -slot;
245 				return slot;
246 			}
247 			++slot;
248 		}
249 		hash = ~hash;
250 	}
251 
252 	return -1;
253 }
254 
beat_lpar_hpte_updateboltedpp(unsigned long newpp,unsigned long ea,int psize,int ssize)255 static void beat_lpar_hpte_updateboltedpp(unsigned long newpp,
256 					  unsigned long ea,
257 					  int psize, int ssize)
258 {
259 	unsigned long lpar_rc, slot, vsid, va;
260 	u64 dummy0, dummy1;
261 
262 	vsid = get_kernel_vsid(ea, MMU_SEGSIZE_256M);
263 	va = (vsid << 28) | (ea & 0x0fffffff);
264 
265 	spin_lock(&beat_htab_lock);
266 	slot = beat_lpar_hpte_find(va, psize);
267 	BUG_ON(slot == -1);
268 
269 	lpar_rc = beat_write_htab_entry(0, slot, 0, newpp, 0, 7,
270 		&dummy0, &dummy1);
271 	spin_unlock(&beat_htab_lock);
272 
273 	BUG_ON(lpar_rc != 0);
274 }
275 
beat_lpar_hpte_invalidate(unsigned long slot,unsigned long va,int psize,int ssize,int local)276 static void beat_lpar_hpte_invalidate(unsigned long slot, unsigned long va,
277 					 int psize, int ssize, int local)
278 {
279 	unsigned long want_v;
280 	unsigned long lpar_rc;
281 	u64 dummy1, dummy2;
282 	unsigned long flags;
283 
284 	DBG_LOW("    inval : slot=%lx, va=%016lx, psize: %d, local: %d\n",
285 		slot, va, psize, local);
286 	want_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M);
287 
288 	spin_lock_irqsave(&beat_htab_lock, flags);
289 	dummy1 = beat_lpar_hpte_getword0(slot);
290 
291 	if ((dummy1 & ~0x7FUL) != (want_v & ~0x7FUL)) {
292 		DBG_LOW("not found !\n");
293 		spin_unlock_irqrestore(&beat_htab_lock, flags);
294 		return;
295 	}
296 
297 	lpar_rc = beat_write_htab_entry(0, slot, 0, 0, HPTE_V_VALID, 0,
298 		&dummy1, &dummy2);
299 	spin_unlock_irqrestore(&beat_htab_lock, flags);
300 
301 	BUG_ON(lpar_rc != 0);
302 }
303 
hpte_init_beat(void)304 void __init hpte_init_beat(void)
305 {
306 	ppc_md.hpte_invalidate	= beat_lpar_hpte_invalidate;
307 	ppc_md.hpte_updatepp	= beat_lpar_hpte_updatepp;
308 	ppc_md.hpte_updateboltedpp = beat_lpar_hpte_updateboltedpp;
309 	ppc_md.hpte_insert	= beat_lpar_hpte_insert;
310 	ppc_md.hpte_remove	= beat_lpar_hpte_remove;
311 	ppc_md.hpte_clear_all	= beat_lpar_hptab_clear;
312 }
313 
beat_lpar_hpte_insert_v3(unsigned long hpte_group,unsigned long va,unsigned long pa,unsigned long rflags,unsigned long vflags,int psize,int ssize)314 static long beat_lpar_hpte_insert_v3(unsigned long hpte_group,
315 				  unsigned long va, unsigned long pa,
316 				  unsigned long rflags, unsigned long vflags,
317 				  int psize, int ssize)
318 {
319 	unsigned long lpar_rc;
320 	u64 hpte_v, hpte_r, slot;
321 
322 	/* same as iseries */
323 	if (vflags & HPTE_V_SECONDARY)
324 		return -1;
325 
326 	if (!(vflags & HPTE_V_BOLTED))
327 		DBG_LOW("hpte_insert(group=%lx, va=%016lx, pa=%016lx, "
328 			"rflags=%lx, vflags=%lx, psize=%d)\n",
329 		hpte_group, va, pa, rflags, vflags, psize);
330 
331 	hpte_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M) |
332 		vflags | HPTE_V_VALID;
333 	hpte_r = hpte_encode_r(pa, psize) | rflags;
334 
335 	if (!(vflags & HPTE_V_BOLTED))
336 		DBG_LOW(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v, hpte_r);
337 
338 	if (rflags & _PAGE_NO_CACHE)
339 		hpte_r &= ~_PAGE_COHERENT;
340 
341 	/* insert into not-volted entry */
342 	lpar_rc = beat_insert_htab_entry3(0, hpte_group, hpte_v, hpte_r,
343 		HPTE_V_BOLTED, 0, &slot);
344 	/*
345 	 * Since we try and ioremap PHBs we don't own, the pte insert
346 	 * will fail. However we must catch the failure in hash_page
347 	 * or we will loop forever, so return -2 in this case.
348 	 */
349 	if (unlikely(lpar_rc != 0)) {
350 		if (!(vflags & HPTE_V_BOLTED))
351 			DBG_LOW(" lpar err %lx\n", lpar_rc);
352 		return -2;
353 	}
354 	if (!(vflags & HPTE_V_BOLTED))
355 		DBG_LOW(" -> slot: %lx\n", slot);
356 
357 	/* We have to pass down the secondary bucket bit here as well */
358 	return (slot ^ hpte_group) & 15;
359 }
360 
361 /*
362  * NOTE: for updatepp ops we are fortunate that the linux "newpp" bits and
363  * the low 3 bits of flags happen to line up.  So no transform is needed.
364  * We can probably optimize here and assume the high bits of newpp are
365  * already zero.  For now I am paranoid.
366  */
beat_lpar_hpte_updatepp_v3(unsigned long slot,unsigned long newpp,unsigned long va,int psize,int ssize,int local)367 static long beat_lpar_hpte_updatepp_v3(unsigned long slot,
368 				    unsigned long newpp,
369 				    unsigned long va,
370 				    int psize, int ssize, int local)
371 {
372 	unsigned long lpar_rc;
373 	unsigned long want_v;
374 	unsigned long pss;
375 
376 	want_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M);
377 	pss = (psize == MMU_PAGE_4K) ? -1UL : mmu_psize_defs[psize].penc;
378 
379 	DBG_LOW("    update: "
380 		"avpnv=%016lx, slot=%016lx, psize: %d, newpp %016lx ... ",
381 		want_v & HPTE_V_AVPN, slot, psize, newpp);
382 
383 	lpar_rc = beat_update_htab_permission3(0, slot, want_v, pss, 7, newpp);
384 
385 	if (lpar_rc == 0xfffffff7) {
386 		DBG_LOW("not found !\n");
387 		return -1;
388 	}
389 
390 	DBG_LOW("ok\n");
391 
392 	BUG_ON(lpar_rc != 0);
393 
394 	return 0;
395 }
396 
beat_lpar_hpte_invalidate_v3(unsigned long slot,unsigned long va,int psize,int ssize,int local)397 static void beat_lpar_hpte_invalidate_v3(unsigned long slot, unsigned long va,
398 					 int psize, int ssize, int local)
399 {
400 	unsigned long want_v;
401 	unsigned long lpar_rc;
402 	unsigned long pss;
403 
404 	DBG_LOW("    inval : slot=%lx, va=%016lx, psize: %d, local: %d\n",
405 		slot, va, psize, local);
406 	want_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M);
407 	pss = (psize == MMU_PAGE_4K) ? -1UL : mmu_psize_defs[psize].penc;
408 
409 	lpar_rc = beat_invalidate_htab_entry3(0, slot, want_v, pss);
410 
411 	/* E_busy can be valid output: page may be already replaced */
412 	BUG_ON(lpar_rc != 0 && lpar_rc != 0xfffffff7);
413 }
414 
_beat_lpar_hptab_clear_v3(void)415 static int64_t _beat_lpar_hptab_clear_v3(void)
416 {
417 	return beat_clear_htab3(0);
418 }
419 
beat_lpar_hptab_clear_v3(void)420 static void beat_lpar_hptab_clear_v3(void)
421 {
422 	_beat_lpar_hptab_clear_v3();
423 }
424 
hpte_init_beat_v3(void)425 void __init hpte_init_beat_v3(void)
426 {
427 	if (_beat_lpar_hptab_clear_v3() == 0) {
428 		ppc_md.hpte_invalidate	= beat_lpar_hpte_invalidate_v3;
429 		ppc_md.hpte_updatepp	= beat_lpar_hpte_updatepp_v3;
430 		ppc_md.hpte_updateboltedpp = beat_lpar_hpte_updateboltedpp;
431 		ppc_md.hpte_insert	= beat_lpar_hpte_insert_v3;
432 		ppc_md.hpte_remove	= beat_lpar_hpte_remove;
433 		ppc_md.hpte_clear_all	= beat_lpar_hptab_clear_v3;
434 	} else {
435 		ppc_md.hpte_invalidate	= beat_lpar_hpte_invalidate;
436 		ppc_md.hpte_updatepp	= beat_lpar_hpte_updatepp;
437 		ppc_md.hpte_updateboltedpp = beat_lpar_hpte_updateboltedpp;
438 		ppc_md.hpte_insert	= beat_lpar_hpte_insert;
439 		ppc_md.hpte_remove	= beat_lpar_hpte_remove;
440 		ppc_md.hpte_clear_all	= beat_lpar_hptab_clear;
441 	}
442 }
443