1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */ 3 4 #include <linux/slab.h> 5 #include <linux/dma-mapping.h> 6 7 #include "lima_device.h" 8 #include "lima_vm.h" 9 #include "lima_gem.h" 10 #include "lima_regs.h" 11 12 struct lima_bo_va { 13 struct list_head list; 14 unsigned int ref_count; 15 16 struct drm_mm_node node; 17 18 struct lima_vm *vm; 19 }; 20 21 #define LIMA_VM_PD_SHIFT 22 22 #define LIMA_VM_PT_SHIFT 12 23 #define LIMA_VM_PB_SHIFT (LIMA_VM_PD_SHIFT + LIMA_VM_NUM_PT_PER_BT_SHIFT) 24 #define LIMA_VM_BT_SHIFT LIMA_VM_PT_SHIFT 25 26 #define LIMA_VM_PT_MASK ((1 << LIMA_VM_PD_SHIFT) - 1) 27 #define LIMA_VM_BT_MASK ((1 << LIMA_VM_PB_SHIFT) - 1) 28 29 #define LIMA_PDE(va) (va >> LIMA_VM_PD_SHIFT) 30 #define LIMA_PTE(va) ((va & LIMA_VM_PT_MASK) >> LIMA_VM_PT_SHIFT) 31 #define LIMA_PBE(va) (va >> LIMA_VM_PB_SHIFT) 32 #define LIMA_BTE(va) ((va & LIMA_VM_BT_MASK) >> LIMA_VM_BT_SHIFT) 33 34 lima_vm_unmap_range(struct lima_vm * vm,u32 start,u32 end)35 static void lima_vm_unmap_range(struct lima_vm *vm, u32 start, u32 end) 36 { 37 u32 addr; 38 39 for (addr = start; addr <= end; addr += LIMA_PAGE_SIZE) { 40 u32 pbe = LIMA_PBE(addr); 41 u32 bte = LIMA_BTE(addr); 42 43 vm->bts[pbe].cpu[bte] = 0; 44 } 45 } 46 lima_vm_map_page(struct lima_vm * vm,dma_addr_t pa,u32 va)47 static int lima_vm_map_page(struct lima_vm *vm, dma_addr_t pa, u32 va) 48 { 49 u32 pbe = LIMA_PBE(va); 50 u32 bte = LIMA_BTE(va); 51 52 if (!vm->bts[pbe].cpu) { 53 dma_addr_t pts; 54 u32 *pd; 55 int j; 56 57 vm->bts[pbe].cpu = dma_alloc_wc( 58 vm->dev->dev, LIMA_PAGE_SIZE << LIMA_VM_NUM_PT_PER_BT_SHIFT, 59 &vm->bts[pbe].dma, GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO); 60 if (!vm->bts[pbe].cpu) 61 return -ENOMEM; 62 63 pts = vm->bts[pbe].dma; 64 pd = vm->pd.cpu + (pbe << LIMA_VM_NUM_PT_PER_BT_SHIFT); 65 for (j = 0; j < LIMA_VM_NUM_PT_PER_BT; j++) { 66 pd[j] = pts | LIMA_VM_FLAG_PRESENT; 67 pts += LIMA_PAGE_SIZE; 68 } 69 } 70 71 vm->bts[pbe].cpu[bte] = pa | LIMA_VM_FLAGS_CACHE; 72 73 return 0; 74 } 75 76 static struct lima_bo_va * lima_vm_bo_find(struct lima_vm * vm,struct lima_bo * bo)77 lima_vm_bo_find(struct lima_vm *vm, struct lima_bo *bo) 78 { 79 struct lima_bo_va *bo_va, *ret = NULL; 80 81 list_for_each_entry(bo_va, &bo->va, list) { 82 if (bo_va->vm == vm) { 83 ret = bo_va; 84 break; 85 } 86 } 87 88 return ret; 89 } 90 lima_vm_bo_add(struct lima_vm * vm,struct lima_bo * bo,bool create)91 int lima_vm_bo_add(struct lima_vm *vm, struct lima_bo *bo, bool create) 92 { 93 struct lima_bo_va *bo_va; 94 struct sg_dma_page_iter sg_iter; 95 int offset = 0, err; 96 97 mutex_lock(&bo->lock); 98 99 bo_va = lima_vm_bo_find(vm, bo); 100 if (bo_va) { 101 bo_va->ref_count++; 102 mutex_unlock(&bo->lock); 103 return 0; 104 } 105 106 /* should not create new bo_va if not asked by caller */ 107 if (!create) { 108 mutex_unlock(&bo->lock); 109 return -ENOENT; 110 } 111 112 bo_va = kzalloc(sizeof(*bo_va), GFP_KERNEL); 113 if (!bo_va) { 114 err = -ENOMEM; 115 goto err_out0; 116 } 117 118 bo_va->vm = vm; 119 bo_va->ref_count = 1; 120 121 mutex_lock(&vm->lock); 122 123 err = drm_mm_insert_node(&vm->mm, &bo_va->node, lima_bo_size(bo)); 124 if (err) 125 goto err_out1; 126 127 for_each_sgtable_dma_page(bo->base.sgt, &sg_iter, 0) { 128 err = lima_vm_map_page(vm, sg_page_iter_dma_address(&sg_iter), 129 bo_va->node.start + offset); 130 if (err) 131 goto err_out2; 132 133 offset += PAGE_SIZE; 134 } 135 136 mutex_unlock(&vm->lock); 137 138 list_add_tail(&bo_va->list, &bo->va); 139 140 mutex_unlock(&bo->lock); 141 return 0; 142 143 err_out2: 144 if (offset) 145 lima_vm_unmap_range(vm, bo_va->node.start, bo_va->node.start + offset - 1); 146 drm_mm_remove_node(&bo_va->node); 147 err_out1: 148 mutex_unlock(&vm->lock); 149 kfree(bo_va); 150 err_out0: 151 mutex_unlock(&bo->lock); 152 return err; 153 } 154 lima_vm_bo_del(struct lima_vm * vm,struct lima_bo * bo)155 void lima_vm_bo_del(struct lima_vm *vm, struct lima_bo *bo) 156 { 157 struct lima_bo_va *bo_va; 158 u32 size; 159 160 mutex_lock(&bo->lock); 161 162 bo_va = lima_vm_bo_find(vm, bo); 163 if (--bo_va->ref_count > 0) { 164 mutex_unlock(&bo->lock); 165 return; 166 } 167 168 mutex_lock(&vm->lock); 169 170 size = bo->heap_size ? bo->heap_size : bo_va->node.size; 171 lima_vm_unmap_range(vm, bo_va->node.start, 172 bo_va->node.start + size - 1); 173 174 drm_mm_remove_node(&bo_va->node); 175 176 mutex_unlock(&vm->lock); 177 178 list_del(&bo_va->list); 179 180 mutex_unlock(&bo->lock); 181 182 kfree(bo_va); 183 } 184 lima_vm_get_va(struct lima_vm * vm,struct lima_bo * bo)185 u32 lima_vm_get_va(struct lima_vm *vm, struct lima_bo *bo) 186 { 187 struct lima_bo_va *bo_va; 188 u32 ret; 189 190 mutex_lock(&bo->lock); 191 192 bo_va = lima_vm_bo_find(vm, bo); 193 ret = bo_va->node.start; 194 195 mutex_unlock(&bo->lock); 196 197 return ret; 198 } 199 lima_vm_create(struct lima_device * dev)200 struct lima_vm *lima_vm_create(struct lima_device *dev) 201 { 202 struct lima_vm *vm; 203 204 vm = kzalloc(sizeof(*vm), GFP_KERNEL); 205 if (!vm) 206 return NULL; 207 208 vm->dev = dev; 209 mutex_init(&vm->lock); 210 kref_init(&vm->refcount); 211 212 vm->pd.cpu = dma_alloc_wc(dev->dev, LIMA_PAGE_SIZE, &vm->pd.dma, 213 GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO); 214 if (!vm->pd.cpu) 215 goto err_out0; 216 217 if (dev->dlbu_cpu) { 218 int err = lima_vm_map_page( 219 vm, dev->dlbu_dma, LIMA_VA_RESERVE_DLBU); 220 if (err) 221 goto err_out1; 222 } 223 224 drm_mm_init(&vm->mm, dev->va_start, dev->va_end - dev->va_start); 225 226 return vm; 227 228 err_out1: 229 dma_free_wc(dev->dev, LIMA_PAGE_SIZE, vm->pd.cpu, vm->pd.dma); 230 err_out0: 231 kfree(vm); 232 return NULL; 233 } 234 lima_vm_release(struct kref * kref)235 void lima_vm_release(struct kref *kref) 236 { 237 struct lima_vm *vm = container_of(kref, struct lima_vm, refcount); 238 int i; 239 240 drm_mm_takedown(&vm->mm); 241 242 for (i = 0; i < LIMA_VM_NUM_BT; i++) { 243 if (vm->bts[i].cpu) 244 dma_free_wc(vm->dev->dev, LIMA_PAGE_SIZE << LIMA_VM_NUM_PT_PER_BT_SHIFT, 245 vm->bts[i].cpu, vm->bts[i].dma); 246 } 247 248 if (vm->pd.cpu) 249 dma_free_wc(vm->dev->dev, LIMA_PAGE_SIZE, vm->pd.cpu, vm->pd.dma); 250 251 kfree(vm); 252 } 253 lima_vm_print(struct lima_vm * vm)254 void lima_vm_print(struct lima_vm *vm) 255 { 256 int i, j, k; 257 u32 *pd, *pt; 258 259 if (!vm->pd.cpu) 260 return; 261 262 pd = vm->pd.cpu; 263 for (i = 0; i < LIMA_VM_NUM_BT; i++) { 264 if (!vm->bts[i].cpu) 265 continue; 266 267 pt = vm->bts[i].cpu; 268 for (j = 0; j < LIMA_VM_NUM_PT_PER_BT; j++) { 269 int idx = (i << LIMA_VM_NUM_PT_PER_BT_SHIFT) + j; 270 271 printk(KERN_INFO "lima vm pd %03x:%08x\n", idx, pd[idx]); 272 273 for (k = 0; k < LIMA_PAGE_ENT_NUM; k++) { 274 u32 pte = *pt++; 275 276 if (pte) 277 printk(KERN_INFO " pt %03x:%08x\n", k, pte); 278 } 279 } 280 } 281 } 282 lima_vm_map_bo(struct lima_vm * vm,struct lima_bo * bo,int pageoff)283 int lima_vm_map_bo(struct lima_vm *vm, struct lima_bo *bo, int pageoff) 284 { 285 struct lima_bo_va *bo_va; 286 struct sg_dma_page_iter sg_iter; 287 int offset = 0, err; 288 u32 base; 289 290 mutex_lock(&bo->lock); 291 292 bo_va = lima_vm_bo_find(vm, bo); 293 if (!bo_va) { 294 err = -ENOENT; 295 goto err_out0; 296 } 297 298 mutex_lock(&vm->lock); 299 300 base = bo_va->node.start + (pageoff << PAGE_SHIFT); 301 for_each_sgtable_dma_page(bo->base.sgt, &sg_iter, pageoff) { 302 err = lima_vm_map_page(vm, sg_page_iter_dma_address(&sg_iter), 303 base + offset); 304 if (err) 305 goto err_out1; 306 307 offset += PAGE_SIZE; 308 } 309 310 mutex_unlock(&vm->lock); 311 312 mutex_unlock(&bo->lock); 313 return 0; 314 315 err_out1: 316 if (offset) 317 lima_vm_unmap_range(vm, base, base + offset - 1); 318 mutex_unlock(&vm->lock); 319 err_out0: 320 mutex_unlock(&bo->lock); 321 return err; 322 } 323