Lines Matching refs:mem
60 struct mm_iommu_table_group_mem_t *mem, *mem2; in mm_iommu_do_alloc() local
73 mem = kzalloc(sizeof(*mem), GFP_KERNEL); in mm_iommu_do_alloc()
74 if (!mem) { in mm_iommu_do_alloc()
80 mem->pageshift = __ffs(dev_hpa | (entries << PAGE_SHIFT)); in mm_iommu_do_alloc()
81 mem->dev_hpa = dev_hpa; in mm_iommu_do_alloc()
84 mem->dev_hpa = MM_IOMMU_TABLE_INVALID_HPA; in mm_iommu_do_alloc()
91 mem->pageshift = __ffs(ua | (entries << PAGE_SHIFT)); in mm_iommu_do_alloc()
92 mem->hpas = vzalloc(array_size(entries, sizeof(mem->hpas[0]))); in mm_iommu_do_alloc()
93 if (!mem->hpas) { in mm_iommu_do_alloc()
94 kfree(mem); in mm_iommu_do_alloc()
108 mem->hpages + entry, NULL); in mm_iommu_do_alloc()
126 struct page *page = mem->hpages[i]; in mm_iommu_do_alloc()
132 if ((mem->pageshift > PAGE_SHIFT) && PageHuge(page)) in mm_iommu_do_alloc()
134 mem->pageshift = min(mem->pageshift, pageshift); in mm_iommu_do_alloc()
139 mem->hpas[i] = page_to_pfn(page) << PAGE_SHIFT; in mm_iommu_do_alloc()
143 atomic64_set(&mem->mapped, 1); in mm_iommu_do_alloc()
144 mem->used = 1; in mm_iommu_do_alloc()
145 mem->ua = ua; in mm_iommu_do_alloc()
146 mem->entries = entries; in mm_iommu_do_alloc()
161 list_add_rcu(&mem->next, &mm->context.iommu_group_mem_list); in mm_iommu_do_alloc()
165 *pmem = mem; in mm_iommu_do_alloc()
172 put_page(mem->hpages[i]); in mm_iommu_do_alloc()
174 vfree(mem->hpas); in mm_iommu_do_alloc()
175 kfree(mem); in mm_iommu_do_alloc()
199 static void mm_iommu_unpin(struct mm_iommu_table_group_mem_t *mem) in mm_iommu_unpin() argument
204 if (!mem->hpas) in mm_iommu_unpin()
207 for (i = 0; i < mem->entries; ++i) { in mm_iommu_unpin()
208 if (!mem->hpas[i]) in mm_iommu_unpin()
211 page = pfn_to_page(mem->hpas[i] >> PAGE_SHIFT); in mm_iommu_unpin()
215 if (mem->hpas[i] & MM_IOMMU_TABLE_GROUP_PAGE_DIRTY) in mm_iommu_unpin()
219 mem->hpas[i] = 0; in mm_iommu_unpin()
223 static void mm_iommu_do_free(struct mm_iommu_table_group_mem_t *mem) in mm_iommu_do_free() argument
226 mm_iommu_unpin(mem); in mm_iommu_do_free()
227 vfree(mem->hpas); in mm_iommu_do_free()
228 kfree(mem); in mm_iommu_do_free()
233 struct mm_iommu_table_group_mem_t *mem = container_of(head, in mm_iommu_free() local
236 mm_iommu_do_free(mem); in mm_iommu_free()
239 static void mm_iommu_release(struct mm_iommu_table_group_mem_t *mem) in mm_iommu_release() argument
241 list_del_rcu(&mem->next); in mm_iommu_release()
242 call_rcu(&mem->rcu, mm_iommu_free); in mm_iommu_release()
245 long mm_iommu_put(struct mm_struct *mm, struct mm_iommu_table_group_mem_t *mem) in mm_iommu_put() argument
252 if (mem->used == 0) { in mm_iommu_put()
257 --mem->used; in mm_iommu_put()
259 if (mem->used) in mm_iommu_put()
263 if (atomic_cmpxchg(&mem->mapped, 1, 0) != 1) { in mm_iommu_put()
264 ++mem->used; in mm_iommu_put()
269 if (mem->dev_hpa == MM_IOMMU_TABLE_INVALID_HPA) in mm_iommu_put()
270 unlock_entries = mem->entries; in mm_iommu_put()
273 mm_iommu_release(mem); in mm_iommu_put()
287 struct mm_iommu_table_group_mem_t *mem, *ret = NULL; in mm_iommu_lookup() local
289 list_for_each_entry_rcu(mem, &mm->context.iommu_group_mem_list, next) { in mm_iommu_lookup()
290 if ((mem->ua <= ua) && in mm_iommu_lookup()
291 (ua + size <= mem->ua + in mm_iommu_lookup()
292 (mem->entries << PAGE_SHIFT))) { in mm_iommu_lookup()
293 ret = mem; in mm_iommu_lookup()
305 struct mm_iommu_table_group_mem_t *mem, *ret = NULL; in mm_iommu_lookup_rm() local
307 list_for_each_entry_lockless(mem, &mm->context.iommu_group_mem_list, in mm_iommu_lookup_rm()
309 if ((mem->ua <= ua) && in mm_iommu_lookup_rm()
310 (ua + size <= mem->ua + in mm_iommu_lookup_rm()
311 (mem->entries << PAGE_SHIFT))) { in mm_iommu_lookup_rm()
312 ret = mem; in mm_iommu_lookup_rm()
323 struct mm_iommu_table_group_mem_t *mem, *ret = NULL; in mm_iommu_get() local
327 list_for_each_entry_rcu(mem, &mm->context.iommu_group_mem_list, next) { in mm_iommu_get()
328 if ((mem->ua == ua) && (mem->entries == entries)) { in mm_iommu_get()
329 ret = mem; in mm_iommu_get()
330 ++mem->used; in mm_iommu_get()
341 long mm_iommu_ua_to_hpa(struct mm_iommu_table_group_mem_t *mem, in mm_iommu_ua_to_hpa() argument
344 const long entry = (ua - mem->ua) >> PAGE_SHIFT; in mm_iommu_ua_to_hpa()
347 if (entry >= mem->entries) in mm_iommu_ua_to_hpa()
350 if (pageshift > mem->pageshift) in mm_iommu_ua_to_hpa()
353 if (!mem->hpas) { in mm_iommu_ua_to_hpa()
354 *hpa = mem->dev_hpa + (ua - mem->ua); in mm_iommu_ua_to_hpa()
358 va = &mem->hpas[entry]; in mm_iommu_ua_to_hpa()
365 long mm_iommu_ua_to_hpa_rm(struct mm_iommu_table_group_mem_t *mem, in mm_iommu_ua_to_hpa_rm() argument
368 const long entry = (ua - mem->ua) >> PAGE_SHIFT; in mm_iommu_ua_to_hpa_rm()
371 if (entry >= mem->entries) in mm_iommu_ua_to_hpa_rm()
374 if (pageshift > mem->pageshift) in mm_iommu_ua_to_hpa_rm()
377 if (!mem->hpas) { in mm_iommu_ua_to_hpa_rm()
378 *hpa = mem->dev_hpa + (ua - mem->ua); in mm_iommu_ua_to_hpa_rm()
382 pa = (void *) vmalloc_to_phys(&mem->hpas[entry]); in mm_iommu_ua_to_hpa_rm()
393 struct mm_iommu_table_group_mem_t *mem; in mm_iommu_ua_mark_dirty_rm() local
398 mem = mm_iommu_lookup_rm(mm, ua, PAGE_SIZE); in mm_iommu_ua_mark_dirty_rm()
399 if (!mem) in mm_iommu_ua_mark_dirty_rm()
402 if (mem->dev_hpa != MM_IOMMU_TABLE_INVALID_HPA) in mm_iommu_ua_mark_dirty_rm()
405 entry = (ua - mem->ua) >> PAGE_SHIFT; in mm_iommu_ua_mark_dirty_rm()
406 va = &mem->hpas[entry]; in mm_iommu_ua_mark_dirty_rm()
418 struct mm_iommu_table_group_mem_t *mem; in mm_iommu_is_devmem() local
421 list_for_each_entry_rcu(mem, &mm->context.iommu_group_mem_list, next) { in mm_iommu_is_devmem()
422 if (mem->dev_hpa == MM_IOMMU_TABLE_INVALID_HPA) in mm_iommu_is_devmem()
425 end = mem->dev_hpa + (mem->entries << PAGE_SHIFT); in mm_iommu_is_devmem()
426 if ((mem->dev_hpa <= hpa) && (hpa < end)) { in mm_iommu_is_devmem()
442 long mm_iommu_mapped_inc(struct mm_iommu_table_group_mem_t *mem) in mm_iommu_mapped_inc() argument
444 if (atomic64_inc_not_zero(&mem->mapped)) in mm_iommu_mapped_inc()
452 void mm_iommu_mapped_dec(struct mm_iommu_table_group_mem_t *mem) in mm_iommu_mapped_dec() argument
454 atomic64_add_unless(&mem->mapped, -1, 1); in mm_iommu_mapped_dec()