1 /*
2 * Copyright (C) 2021 HiSilicon (Shanghai) Technologies CO., LIMITED.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version 2
7 * of the License, or (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17 */
18
19 #include <linux/fs.h>
20 #include <linux/slab.h>
21 #include <linux/init.h>
22 #include <linux/delay.h>
23 #include <asm/uaccess.h>
24 #include <asm/io.h>
25 #include <linux/list.h>
26 #include <asm/cacheflush.h>
27 #include <linux/version.h>
28 #include "securec.h"
29 #include "allocator.h"
30
31 extern struct osal_list_head g_mmz_list;
32
33 extern int anony;
34
35 unsigned long g_hi_max_malloc_size = 0x40000000UL; /* 1GB */
36
_strtoul_ex(const char * s,char ** ep,unsigned int base)37 static unsigned long _strtoul_ex(const char *s, char **ep, unsigned int base)
38 {
39 char *__end_p = NULL;
40 unsigned long __value;
41
42 __value = simple_strtoul(s, &__end_p, base);
43
44 switch (*__end_p) {
45 case 'm':
46 case 'M':
47 __value <<= 10; /* 10: 1M=1024k, left shift 10bit */
48 /* fall-through */
49 case 'k':
50 case 'K':
51 __value <<= 10; /* 10: 1K=1024Byte, left shift 10bit */
52 if (ep != NULL) {
53 (*ep) = __end_p + 1;
54 }
55 /* fall-through */
56 default:
57 break;
58 }
59
60 return __value;
61 }
62
find_fixed_region(unsigned long * region_len,hil_mmz_t * mmz,unsigned long size,unsigned long align)63 static unsigned long find_fixed_region(unsigned long *region_len,
64 hil_mmz_t *mmz,
65 unsigned long size,
66 unsigned long align)
67 {
68 unsigned long start;
69 unsigned long fixed_start = 0;
70 unsigned long fixed_len = -1;
71 unsigned long len;
72 unsigned long blank_len;
73 hil_mmb_t *p = NULL;
74
75 mmz_trace_func();
76 align = mmz_grain_align(align);
77 if (align == 0) {
78 align = MMZ_GRAIN;
79 }
80 start = mmz_align2(mmz->phys_start, align);
81 len = mmz_grain_align(size);
82
83 list_for_each_entry(p, &mmz->mmb_list, list) {
84 hil_mmb_t *next = NULL;
85 mmz_trace(4, "p->phys_addr=0x%08lX p->length = %luKB \t", /* 4: log debug level */
86 p->phys_addr, p->length / SZ_1K);
87 next = list_entry(p->list.next, typeof(*p), list);
88 mmz_trace(4, ",next = 0x%08lX\n\n", next->phys_addr); /* 4: log debug level */
89 /*
90 * if p is the first entry or not.
91 */
92 if (list_first_entry(&mmz->mmb_list, typeof(*p), list) == p) {
93 blank_len = p->phys_addr - start;
94 if ((blank_len < fixed_len) && (blank_len >= len)) {
95 fixed_len = blank_len;
96 fixed_start = start;
97 mmz_trace(4, "%d: fixed_region: start=0x%08lX, len=%luKB\n", /* 4: log debug level */
98 __LINE__, fixed_start, fixed_len / SZ_1K);
99 }
100 }
101 start = mmz_align2((p->phys_addr + p->length), align);
102 /* if aglin is larger than mmz->nbytes, it would trigger the BUG_ON */
103 /* if we have to alloc after the last node. */
104 if (osal_list_is_last(&p->list, &mmz->mmb_list)) {
105 blank_len = mmz->phys_start + mmz->nbytes - start;
106 if ((blank_len < fixed_len) && (blank_len >= len)) {
107 fixed_len = blank_len;
108 fixed_start = start;
109 mmz_trace(4, "fixed_region: start=0x%08lX, len=%luKB\n", /* 4: log debug level */
110 fixed_start, fixed_len / SZ_1K);
111 break;
112 } else {
113 if (fixed_len != -1UL) {
114 goto out;
115 }
116 fixed_start = 0;
117 mmz_trace(4, "fixed_region: start=0x%08lX, len=%luKB\n", /* 4: log debug level */
118 fixed_start, fixed_len / SZ_1K);
119 goto out;
120 }
121 }
122 /* blank is too small */
123 if ((start + len) > next->phys_addr) {
124 mmz_trace(4, "start=0x%08lX ,len=%lu,next=0x%08lX\n", /* 4: log debug level */
125 start, len, next->phys_addr);
126 continue;
127 }
128 blank_len = next->phys_addr - start;
129 if ((blank_len < fixed_len) && (blank_len >= len)) {
130 fixed_len = blank_len;
131 fixed_start = start;
132 mmz_trace(4, "fixed_region: start=0x%08lX, len=%luKB\n", /* 4: log debug level */
133 fixed_start, fixed_len / SZ_1K);
134 }
135 }
136
137 if ((mmz_grain_align(start + len) <= (mmz->phys_start + mmz->nbytes)) &&
138 (start >= mmz->phys_start) &&
139 (start < (mmz->phys_start + mmz->nbytes))) {
140 fixed_len = len;
141 fixed_start = start;
142 mmz_trace(4, "fixed_region: start=0x%08lX, len=%luKB\n", /* 4: log debug level */
143 fixed_start, fixed_len / SZ_1K);
144 } else {
145 fixed_start = 0;
146 mmz_trace(4, "fixed_region: start=0x%08lX, len=%luKB\n", /* 4: log debug level */
147 fixed_start, len / SZ_1K);
148 }
149 out:
150 *region_len = len;
151 return fixed_start;
152 }
153
find_fixed_region_from_highaddr(unsigned long * region_len,hil_mmz_t * mmz,unsigned long size,unsigned long align)154 static unsigned long find_fixed_region_from_highaddr(unsigned long *region_len,
155 hil_mmz_t *mmz, unsigned long size, unsigned long align)
156 {
157 int j;
158 unsigned int i;
159 unsigned long fixed_start = 0;
160 unsigned long fixed_len = ~1;
161
162 mmz_trace_func();
163
164 i = mmz_length2grain(mmz->nbytes);
165
166 for (; i > 0; i--) {
167 unsigned long start;
168 unsigned long len;
169 unsigned long start_highaddr;
170
171 if (mmz_get_bit(mmz, i)) {
172 continue;
173 }
174
175 len = 0;
176 start_highaddr = mmz_pos2phy_addr(mmz, i);
177 for (; i > 0; i--) {
178 if (mmz_get_bit(mmz, i)) {
179 break;
180 }
181
182 len += MMZ_GRAIN;
183 }
184
185 if (len >= size) {
186 j = mmz_phy_addr2pos(mmz, mmz_align2low(start_highaddr - size, align));
187 start = mmz_pos2phy_addr(mmz, j);
188 if ((start_highaddr - len <= start) && (start <= start_highaddr - size)) {
189 fixed_len = len;
190 fixed_start = start;
191 break;
192 }
193
194 mmz_trace(1, "fixed_region: start=0x%08lX, len=%luKB",
195 fixed_start, fixed_len / SZ_1K);
196 }
197 }
198
199 *region_len = fixed_len;
200
201 return fixed_start;
202 }
203
do_mmb_alloc(hil_mmb_t * mmb)204 static int do_mmb_alloc(hil_mmb_t *mmb)
205 {
206 hil_mmb_t *p = NULL;
207 mmz_trace_func();
208
209 /* add mmb sorted */
210 osal_list_for_each_entry(p, &mmb->zone->mmb_list, list) {
211 if (mmb->phys_addr < p->phys_addr) {
212 break;
213 }
214 if (mmb->phys_addr == p->phys_addr) {
215 osal_trace(KERN_ERR "ERROR: media-mem allocator bad in %s! (%s, %d)",
216 mmb->zone->name, __FUNCTION__, __LINE__);
217 }
218 }
219 osal_list_add(&mmb->list, p->list.prev);
220
221 mmz_trace(1, HIL_MMB_FMT_S, hil_mmb_fmt_arg(mmb));
222
223 return 0;
224 }
225
__mmb_alloc(const char * name,unsigned long size,unsigned long align,unsigned long gfp,const char * mmz_name,hil_mmz_t * _user_mmz)226 static hil_mmb_t *__mmb_alloc(const char *name,
227 unsigned long size,
228 unsigned long align,
229 unsigned long gfp,
230 const char *mmz_name,
231 hil_mmz_t *_user_mmz)
232 {
233 hil_mmz_t *mmz = NULL;
234 hil_mmb_t *mmb = NULL;
235
236 unsigned long start;
237 unsigned long region_len;
238
239 unsigned long fixed_start = 0;
240 unsigned long fixed_len = ~1;
241 hil_mmz_t *fixed_mmz = NULL;
242 errno_t err_value;
243
244 mmz_trace_func();
245
246 if ((size == 0) || (size > g_hi_max_malloc_size)) {
247 return NULL;
248 }
249 if (align == 0) {
250 align = MMZ_GRAIN;
251 }
252
253 size = mmz_grain_align(size);
254
255 mmz_trace(1, "size=%luKB, align=%lu", size / SZ_1K, align);
256
257 begin_list_for_each_mmz(mmz, gfp, mmz_name)
258
259 if ((_user_mmz != NULL) && (_user_mmz != mmz)) {
260 continue;
261 }
262
263 start = find_fixed_region(®ion_len, mmz, size, align);
264 if ((fixed_len > region_len) && (start != 0)) {
265 fixed_len = region_len;
266 fixed_start = start;
267 fixed_mmz = mmz;
268 }
269 end_list_for_each_mmz()
270
271 if (fixed_mmz == NULL) {
272 return NULL;
273 }
274
275 mmb = kmalloc(sizeof(hil_mmb_t), GFP_KERNEL);
276 if (mmb == NULL) {
277 return NULL;
278 }
279
280 (void)memset_s(mmb, sizeof(hil_mmb_t), 0, sizeof(hil_mmb_t));
281 mmb->zone = fixed_mmz;
282 mmb->phys_addr = fixed_start;
283 mmb->length = size;
284 if (name != NULL) {
285 err_value = strncpy_s(mmb->name, HIL_MMB_NAME_LEN, name, HIL_MMB_NAME_LEN - 1);
286 } else {
287 err_value = strncpy_s(mmb->name, HIL_MMB_NAME_LEN, "<null>", HIL_MMB_NAME_LEN - 1);
288 }
289
290 if ((err_value != EOK) || do_mmb_alloc(mmb)) {
291 kfree(mmb);
292 mmb = NULL;
293 }
294
295 return mmb;
296 }
297
__mmb_alloc_v2(const char * name,unsigned long size,unsigned long align,unsigned long gfp,const char * mmz_name,hil_mmz_t * _user_mmz,unsigned int order)298 static hil_mmb_t *__mmb_alloc_v2(const char *name,
299 unsigned long size,
300 unsigned long align,
301 unsigned long gfp,
302 const char *mmz_name,
303 hil_mmz_t *_user_mmz,
304 unsigned int order)
305 {
306 hil_mmz_t *mmz = NULL;
307 hil_mmb_t *mmb = NULL;
308 unsigned int i;
309
310 unsigned long start = 0;
311 unsigned long region_len = 0;
312
313 unsigned long fixed_start = 0;
314 unsigned long fixed_len = ~1;
315 hil_mmz_t *fixed_mmz = NULL;
316 errno_t err_value;
317
318 mmz_trace_func();
319
320 if ((size == 0) || (size > 0x40000000UL)) {
321 return NULL;
322 }
323 if (align == 0) {
324 align = 1;
325 }
326
327 size = mmz_grain_align(size);
328
329 mmz_trace(1, "size=%luKB, align=%lu", size / SZ_1K, align);
330
331 begin_list_for_each_mmz(mmz, gfp, mmz_name)
332
333 if ((_user_mmz != NULL) && (_user_mmz != mmz)) {
334 continue;
335 }
336
337 if (mmz->alloc_type == SLAB_ALLOC) {
338 if ((size - 1) & size) {
339 for (i = 1; i < 32; i++) { /* 32: the max size is 2^(32-1) */
340 if (!((size >> i) & ~0)) {
341 size = 1 << i;
342 break;
343 }
344 }
345 }
346 } else if (mmz->alloc_type == EQ_BLOCK_ALLOC) {
347 size = mmz_align2(size, mmz->block_align);
348 }
349
350 if (order == LOW_TO_HIGH) {
351 start = find_fixed_region(®ion_len, mmz, size, align);
352 } else if (order == HIGH_TO_LOW) {
353 start = find_fixed_region_from_highaddr(®ion_len, mmz, size, align);
354 }
355
356 if ((fixed_len > region_len) && (start != 0)) {
357 fixed_len = region_len;
358 fixed_start = start;
359 fixed_mmz = mmz;
360 }
361
362 end_list_for_each_mmz()
363
364 if (fixed_mmz == NULL) {
365 return NULL;
366 }
367
368 mmb = kmalloc(sizeof(hil_mmb_t), GFP_KERNEL);
369 if (mmb == NULL) {
370 return NULL;
371 }
372
373 (void)memset_s(mmb, sizeof(hil_mmb_t), 0, sizeof(hil_mmb_t));
374 mmb->zone = fixed_mmz;
375 mmb->phys_addr = fixed_start;
376 mmb->length = size;
377 mmb->order = order;
378
379 if (name != NULL) {
380 err_value = strncpy_s(mmb->name, HIL_MMB_NAME_LEN, name, HIL_MMB_NAME_LEN - 1);
381 } else {
382 err_value = strncpy_s(mmb->name, HIL_MMB_NAME_LEN, "<null>", HIL_MMB_NAME_LEN - 1);
383 }
384
385 if ((err_value != EOK) || do_mmb_alloc(mmb)) {
386 kfree(mmb);
387 mmb = NULL;
388 }
389
390 return mmb;
391 }
392
__mmb_map2kern(hil_mmb_t * mmb,int cached)393 static void *__mmb_map2kern(hil_mmb_t *mmb, int cached)
394 {
395 /*
396 * already mapped? no need to remap again,
397 * just return mmb's kernel virtual address.
398 */
399 if (mmb->flags & HIL_MMB_MAP2KERN) {
400 if ((!!cached * HIL_MMB_MAP2KERN_CACHED) != (mmb->flags & HIL_MMB_MAP2KERN_CACHED)) {
401 osal_trace(KERN_ERR "mmb<%s> has been kernel-mapped as %s, can not be re-mapped as %s.",
402 mmb->name,
403 (mmb->flags & HIL_MMB_MAP2KERN_CACHED) ? "cached" : "non-cached",
404 (cached) ? "cached" : "non-cached");
405 return NULL;
406 }
407
408 mmb->map_ref++;
409
410 return mmb->kvirt;
411 }
412
413 if (cached) {
414 mmb->flags |= HIL_MMB_MAP2KERN_CACHED;
415 mmb->kvirt = ioremap_cache(mmb->phys_addr, mmb->length);
416 } else {
417 mmb->flags &= ~HIL_MMB_MAP2KERN_CACHED;
418 /* ioremap_wc has better performance */
419 mmb->kvirt = ioremap_wc(mmb->phys_addr, mmb->length);
420 }
421
422 if (mmb->kvirt) {
423 mmb->flags |= HIL_MMB_MAP2KERN;
424 mmb->map_ref++;
425 } else {
426 mmb->flags &= ~HIL_MMB_MAP2KERN_CACHED;
427 }
428
429 return mmb->kvirt;
430 }
431
__mmb_free(hil_mmb_t * mmb)432 static void __mmb_free(hil_mmb_t *mmb)
433 {
434 if (mmb->flags & HIL_MMB_MAP2KERN_CACHED) {
435 #ifdef CONFIG_64BIT
436 __flush_dcache_area((void *)mmb->kvirt, (size_t)mmb->length);
437 #else
438 __cpuc_flush_dcache_area((void *)mmb->kvirt, (size_t)mmb->length);
439 outer_flush_range(mmb->phys_addr, mmb->phys_addr + mmb->length);
440 #endif
441 }
442
443 osal_list_del(&mmb->list);
444 kfree(mmb);
445 }
446
__mmb_unmap(hil_mmb_t * mmb)447 static int __mmb_unmap(hil_mmb_t *mmb)
448 {
449 int ref;
450
451 if (mmb->flags & HIL_MMB_MAP2KERN_CACHED) {
452 #ifdef CONFIG_64BIT
453 __flush_dcache_area((void *)mmb->kvirt, (size_t)mmb->length);
454 #else
455 __cpuc_flush_dcache_area((void *)mmb->kvirt, (size_t)mmb->length);
456 outer_flush_range(mmb->phys_addr, mmb->phys_addr + mmb->length);
457 #endif
458 }
459
460 if (mmb->flags & HIL_MMB_MAP2KERN) {
461 ref = --mmb->map_ref;
462 if (mmb->map_ref != 0) {
463 return ref;
464 }
465 iounmap(mmb->kvirt);
466 }
467
468 mmb->kvirt = NULL;
469 mmb->flags &= ~HIL_MMB_MAP2KERN;
470 mmb->flags &= ~HIL_MMB_MAP2KERN_CACHED;
471
472 if ((mmb->flags & HIL_MMB_RELEASED) && (mmb->phy_ref == 0)) {
473 __mmb_free(mmb);
474 }
475
476 return 0;
477 }
478
__mmf_map(phys_addr_t phys,int len,int cache)479 static void *__mmf_map(phys_addr_t phys, int len, int cache)
480 {
481 void *virt = NULL;
482 if (cache) {
483 virt = ioremap_cache(phys, len);
484 } else {
485 virt = ioremap_wc(phys, len);
486 }
487
488 return virt;
489 }
490
__mmf_unmap(void * virt)491 static void __mmf_unmap(void *virt)
492 {
493 if (virt != NULL) {
494 iounmap(virt);
495 }
496 }
497
__allocator_init(char * s)498 static int __allocator_init(char *s)
499 {
500 hil_mmz_t *zone = NULL;
501 char *line = NULL;
502 unsigned long phys_end;
503
504 while ((line = strsep(&s, ":")) != NULL) {
505 int i;
506 char *argv[6]; /* 6: cmdline include 6 arguments */
507
508 for (i = 0; (argv[i] = strsep(&line, ",")) != NULL;) {
509 if (++i == ARRAY_SIZE(argv)) {
510 break;
511 }
512 }
513
514 if (i == 4) { /* 4: had parse four args */
515 zone = hil_mmz_create("null", 0, 0, 0);
516 if (zone == NULL) {
517 continue;
518 }
519
520 /* 0: the first args */
521 if (strncpy_s(zone->name, HIL_MMZ_NAME_LEN, argv[0], HIL_MMZ_NAME_LEN - 1) != EOK) {
522 osal_trace("%s - strncpy_s failed!\n", __FUNCTION__);
523 hil_mmz_destroy(zone);
524 return -1;
525 }
526 zone->gfp = _strtoul_ex(argv[1], NULL, 0); /* 1: the second args */
527 zone->phys_start = _strtoul_ex(argv[2], NULL, 0); /* 2: the third args */
528 zone->nbytes = _strtoul_ex(argv[3], NULL, 0); /* 3: the fourth args */
529 if (zone->nbytes > g_hi_max_malloc_size) {
530 g_hi_max_malloc_size = zone->nbytes;
531 }
532 } else if (i == 6) { /* 6: had parse six args */
533 zone = hil_mmz_create_v2("null", 0, 0, 0, 0, 0);
534 if (zone == NULL) {
535 continue;
536 }
537
538 /* 0: the first args */
539 if (strncpy_s(zone->name, HIL_MMZ_NAME_LEN, argv[0], HIL_MMZ_NAME_LEN - 1) != EOK) {
540 osal_trace("%s - strncpy_s failed!\n", __FUNCTION__);
541 hil_mmz_destroy(zone);
542 return -1;
543 }
544 zone->gfp = _strtoul_ex(argv[1], NULL, 0); /* 1: the second args */
545 zone->phys_start = _strtoul_ex(argv[2], NULL, 0); /* 2: the third args */
546 zone->nbytes = _strtoul_ex(argv[3], NULL, 0); /* 3: the fourth args */
547 zone->alloc_type = _strtoul_ex(argv[4], NULL, 0); /* 4: the fifth args */
548 zone->block_align = _strtoul_ex(argv[5], NULL, 0); /* 5: the sixth args */
549 if (zone->nbytes > g_hi_max_malloc_size) {
550 g_hi_max_malloc_size = zone->nbytes;
551 }
552 } else {
553 osal_trace(KERN_ERR "error parameters\n");
554 return -EINVAL;
555 }
556
557 if (hil_mmz_register(zone)) {
558 osal_trace(KERN_WARNING "Add MMZ failed: " HIL_MMZ_FMT_S "\n", hil_mmz_fmt_arg(zone));
559 hil_mmz_destroy(zone);
560 return -1;
561 }
562
563 /* if phys_end is maximum value (ex, 0xFFFFFFFF 32bit) */
564 phys_end = (zone->phys_start + zone->nbytes);
565
566 if ((phys_end == 0) && (zone->nbytes >= PAGE_SIZE)) {
567 /* reserve last PAGE_SIZE memory */
568 zone->nbytes = zone->nbytes - PAGE_SIZE;
569 }
570
571 /* if phys_end exceed 0xFFFFFFFF (32bit), wrapping error */
572 if ((zone->phys_start > phys_end) && (phys_end != 0)) {
573 osal_trace(KERN_ERR "MMZ: parameter is not correct! Address exceeds 0xFFFFFFFF\n");
574 hil_mmz_unregister(zone);
575 hil_mmz_destroy(zone);
576 return -1;
577 }
578 zone = NULL;
579 }
580
581 return 0;
582 }
583
hisi_allocator_setopt(struct mmz_allocator * allocator)584 int hisi_allocator_setopt(struct mmz_allocator *allocator)
585 {
586 allocator->init = __allocator_init;
587 allocator->mmb_alloc = __mmb_alloc;
588 allocator->mmb_alloc_v2 = __mmb_alloc_v2;
589 allocator->mmb_map2kern = __mmb_map2kern;
590 allocator->mmb_unmap = __mmb_unmap;
591 allocator->mmb_free = __mmb_free;
592 allocator->mmf_map = __mmf_map;
593 allocator->mmf_unmap = __mmf_unmap;
594 return 0;
595 }
596