Lines Matching refs:mem
215 struct io_tlb_mem *mem = &io_tlb_default_mem; in swiotlb_print_info() local
217 if (!mem->nslabs) { in swiotlb_print_info()
222 pr_info("mapped [mem %pa-%pa] (%luMB)\n", &mem->start, &mem->end, in swiotlb_print_info()
223 (mem->nslabs << IO_TLB_SHIFT) >> 20); in swiotlb_print_info()
242 static void *swiotlb_mem_remap(struct io_tlb_mem *mem, unsigned long bytes) in swiotlb_mem_remap() argument
247 phys_addr_t paddr = mem->start + swiotlb_unencrypted_base; in swiotlb_mem_remap()
258 static void *swiotlb_mem_remap(struct io_tlb_mem *mem, unsigned long bytes) in swiotlb_mem_remap() argument
272 struct io_tlb_mem *mem = &io_tlb_default_mem; in swiotlb_update_mem_attributes() local
276 if (!mem->nslabs || mem->late_alloc) in swiotlb_update_mem_attributes()
278 vaddr = phys_to_virt(mem->start); in swiotlb_update_mem_attributes()
279 bytes = PAGE_ALIGN(mem->nslabs << IO_TLB_SHIFT); in swiotlb_update_mem_attributes()
282 mem->vaddr = swiotlb_mem_remap(mem, bytes); in swiotlb_update_mem_attributes()
283 if (!mem->vaddr) in swiotlb_update_mem_attributes()
284 mem->vaddr = vaddr; in swiotlb_update_mem_attributes()
287 static void swiotlb_init_io_tlb_mem(struct io_tlb_mem *mem, phys_addr_t start, in swiotlb_init_io_tlb_mem() argument
294 mem->nslabs = nslabs; in swiotlb_init_io_tlb_mem()
295 mem->start = start; in swiotlb_init_io_tlb_mem()
296 mem->end = mem->start + bytes; in swiotlb_init_io_tlb_mem()
297 mem->late_alloc = late_alloc; in swiotlb_init_io_tlb_mem()
298 mem->nareas = nareas; in swiotlb_init_io_tlb_mem()
299 mem->area_nslabs = nslabs / mem->nareas; in swiotlb_init_io_tlb_mem()
301 mem->force_bounce = swiotlb_force_bounce || (flags & SWIOTLB_FORCE); in swiotlb_init_io_tlb_mem()
303 for (i = 0; i < mem->nareas; i++) { in swiotlb_init_io_tlb_mem()
304 spin_lock_init(&mem->areas[i].lock); in swiotlb_init_io_tlb_mem()
305 mem->areas[i].index = 0; in swiotlb_init_io_tlb_mem()
306 mem->areas[i].used = 0; in swiotlb_init_io_tlb_mem()
309 for (i = 0; i < mem->nslabs; i++) { in swiotlb_init_io_tlb_mem()
310 mem->slots[i].list = IO_TLB_SEGSIZE - io_tlb_offset(i); in swiotlb_init_io_tlb_mem()
311 mem->slots[i].orig_addr = INVALID_PHYS_ADDR; in swiotlb_init_io_tlb_mem()
312 mem->slots[i].alloc_size = 0; in swiotlb_init_io_tlb_mem()
323 mem->vaddr = vaddr; in swiotlb_init_io_tlb_mem()
366 struct io_tlb_mem *mem = &io_tlb_default_mem; in swiotlb_init_remap() local
395 alloc_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), nslabs)); in swiotlb_init_remap()
396 mem->slots = memblock_alloc(alloc_size, PAGE_SIZE); in swiotlb_init_remap()
397 if (!mem->slots) { in swiotlb_init_remap()
403 mem->areas = memblock_alloc(array_size(sizeof(struct io_tlb_area), in swiotlb_init_remap()
405 if (!mem->areas) { in swiotlb_init_remap()
410 swiotlb_init_io_tlb_mem(mem, __pa(tlb), nslabs, flags, false, in swiotlb_init_remap()
430 struct io_tlb_mem *mem = &io_tlb_default_mem; in swiotlb_init_late() local
479 area_order = get_order(array_size(sizeof(*mem->areas), nareas)); in swiotlb_init_late()
480 mem->areas = (struct io_tlb_area *) in swiotlb_init_late()
482 if (!mem->areas) in swiotlb_init_late()
485 mem->slots = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, in swiotlb_init_late()
486 get_order(array_size(sizeof(*mem->slots), nslabs))); in swiotlb_init_late()
487 if (!mem->slots) in swiotlb_init_late()
492 swiotlb_init_io_tlb_mem(mem, virt_to_phys(vstart), nslabs, 0, true, in swiotlb_init_late()
499 free_pages((unsigned long)mem->areas, area_order); in swiotlb_init_late()
507 struct io_tlb_mem *mem = &io_tlb_default_mem; in swiotlb_exit() local
515 if (!mem->nslabs) in swiotlb_exit()
519 tbl_vaddr = (unsigned long)phys_to_virt(mem->start); in swiotlb_exit()
520 tbl_size = PAGE_ALIGN(mem->end - mem->start); in swiotlb_exit()
521 slots_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), mem->nslabs)); in swiotlb_exit()
524 if (mem->late_alloc) { in swiotlb_exit()
525 area_order = get_order(array_size(sizeof(*mem->areas), in swiotlb_exit()
526 mem->nareas)); in swiotlb_exit()
527 free_pages((unsigned long)mem->areas, area_order); in swiotlb_exit()
529 free_pages((unsigned long)mem->slots, get_order(slots_size)); in swiotlb_exit()
531 memblock_free_late(__pa(mem->areas), in swiotlb_exit()
532 array_size(sizeof(*mem->areas), mem->nareas)); in swiotlb_exit()
533 memblock_free_late(mem->start, tbl_size); in swiotlb_exit()
534 memblock_free_late(__pa(mem->slots), slots_size); in swiotlb_exit()
537 memset(mem, 0, sizeof(*mem)); in swiotlb_exit()
554 struct io_tlb_mem *mem = dev->dma_io_tlb_mem; in swiotlb_bounce() local
555 int index = (tlb_addr - mem->start) >> IO_TLB_SHIFT; in swiotlb_bounce()
556 phys_addr_t orig_addr = mem->slots[index].orig_addr; in swiotlb_bounce()
557 size_t alloc_size = mem->slots[index].alloc_size; in swiotlb_bounce()
559 unsigned char *vaddr = mem->vaddr + tlb_addr - mem->start; in swiotlb_bounce()
636 static unsigned int wrap_area_index(struct io_tlb_mem *mem, unsigned int index) in wrap_area_index() argument
638 if (index >= mem->area_nslabs) in wrap_area_index()
651 struct io_tlb_mem *mem = dev->dma_io_tlb_mem; in swiotlb_do_find_slots() local
652 struct io_tlb_area *area = mem->areas + area_index; in swiotlb_do_find_slots()
655 phys_to_dma_unencrypted(dev, mem->start) & boundary_mask; in swiotlb_do_find_slots()
667 BUG_ON(area_index >= mem->nareas); in swiotlb_do_find_slots()
680 if (unlikely(nslots > mem->area_nslabs - area->used)) in swiotlb_do_find_slots()
683 slot_base = area_index * mem->area_nslabs; in swiotlb_do_find_slots()
684 index = wrap = wrap_area_index(mem, ALIGN(area->index, stride)); in swiotlb_do_find_slots()
692 index = wrap_area_index(mem, index + 1); in swiotlb_do_find_slots()
704 if (mem->slots[slot_index].list >= nslots) in swiotlb_do_find_slots()
707 index = wrap_area_index(mem, index + stride); in swiotlb_do_find_slots()
716 mem->slots[i].list = 0; in swiotlb_do_find_slots()
717 mem->slots[i].alloc_size = alloc_size - (offset + in swiotlb_do_find_slots()
722 mem->slots[i].list; i--) in swiotlb_do_find_slots()
723 mem->slots[i].list = ++count; in swiotlb_do_find_slots()
728 if (index + nslots < mem->area_nslabs) in swiotlb_do_find_slots()
740 struct io_tlb_mem *mem = dev->dma_io_tlb_mem; in swiotlb_find_slots() local
741 int start = raw_smp_processor_id() & (mem->nareas - 1); in swiotlb_find_slots()
749 if (++i >= mem->nareas) in swiotlb_find_slots()
756 static unsigned long mem_used(struct io_tlb_mem *mem) in mem_used() argument
761 for (i = 0; i < mem->nareas; i++) in mem_used()
762 used += mem->areas[i].used; in mem_used()
771 struct io_tlb_mem *mem = dev->dma_io_tlb_mem; in swiotlb_tbl_map_single() local
777 if (!mem || !mem->nslabs) { in swiotlb_tbl_map_single()
798 alloc_size, mem->nslabs, mem_used(mem)); in swiotlb_tbl_map_single()
808 mem->slots[index + i].orig_addr = slot_addr(orig_addr, i); in swiotlb_tbl_map_single()
809 tlb_addr = slot_addr(mem->start, index) + offset; in swiotlb_tbl_map_single()
823 struct io_tlb_mem *mem = dev->dma_io_tlb_mem; in swiotlb_release_slots() local
826 int index = (tlb_addr - offset - mem->start) >> IO_TLB_SHIFT; in swiotlb_release_slots()
827 int nslots = nr_slots(mem->slots[index].alloc_size + offset); in swiotlb_release_slots()
828 int aindex = index / mem->area_nslabs; in swiotlb_release_slots()
829 struct io_tlb_area *area = &mem->areas[aindex]; in swiotlb_release_slots()
838 BUG_ON(aindex >= mem->nareas); in swiotlb_release_slots()
842 count = mem->slots[index + nslots].list; in swiotlb_release_slots()
851 mem->slots[i].list = ++count; in swiotlb_release_slots()
852 mem->slots[i].orig_addr = INVALID_PHYS_ADDR; in swiotlb_release_slots()
853 mem->slots[i].alloc_size = 0; in swiotlb_release_slots()
861 io_tlb_offset(i) != IO_TLB_SEGSIZE - 1 && mem->slots[i].list; in swiotlb_release_slots()
863 mem->slots[i].list = ++count; in swiotlb_release_slots()
954 struct io_tlb_mem *mem = dev->dma_io_tlb_mem; in is_swiotlb_active() local
956 return mem && mem->nslabs; in is_swiotlb_active()
962 struct io_tlb_mem *mem = data; in io_tlb_used_get() local
964 *val = mem_used(mem); in io_tlb_used_get()
969 static void swiotlb_create_debugfs_files(struct io_tlb_mem *mem, in swiotlb_create_debugfs_files() argument
972 mem->debugfs = debugfs_create_dir(dirname, io_tlb_default_mem.debugfs); in swiotlb_create_debugfs_files()
973 if (!mem->nslabs) in swiotlb_create_debugfs_files()
976 debugfs_create_ulong("io_tlb_nslabs", 0400, mem->debugfs, &mem->nslabs); in swiotlb_create_debugfs_files()
977 debugfs_create_file("io_tlb_used", 0400, mem->debugfs, mem, in swiotlb_create_debugfs_files()
995 struct io_tlb_mem *mem = dev->dma_io_tlb_mem; in swiotlb_alloc() local
999 if (!mem) in swiotlb_alloc()
1006 tlb_addr = slot_addr(mem->start, index); in swiotlb_alloc()
1026 struct io_tlb_mem *mem = rmem->priv; in rmem_swiotlb_device_init() local
1042 if (!mem) { in rmem_swiotlb_device_init()
1043 mem = kzalloc(sizeof(*mem), GFP_KERNEL); in rmem_swiotlb_device_init()
1044 if (!mem) in rmem_swiotlb_device_init()
1047 mem->slots = kcalloc(nslabs, sizeof(*mem->slots), GFP_KERNEL); in rmem_swiotlb_device_init()
1048 if (!mem->slots) { in rmem_swiotlb_device_init()
1049 kfree(mem); in rmem_swiotlb_device_init()
1053 mem->areas = kcalloc(nareas, sizeof(*mem->areas), in rmem_swiotlb_device_init()
1055 if (!mem->areas) { in rmem_swiotlb_device_init()
1056 kfree(mem->slots); in rmem_swiotlb_device_init()
1057 kfree(mem); in rmem_swiotlb_device_init()
1063 swiotlb_init_io_tlb_mem(mem, rmem->base, nslabs, SWIOTLB_FORCE, in rmem_swiotlb_device_init()
1065 mem->for_alloc = true; in rmem_swiotlb_device_init()
1067 rmem->priv = mem; in rmem_swiotlb_device_init()
1069 swiotlb_create_debugfs_files(mem, rmem->name); in rmem_swiotlb_device_init()
1072 dev->dma_io_tlb_mem = mem; in rmem_swiotlb_device_init()