• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*******************************************************************************
2 *
3 * Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses.  You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenFabrics.org BSD license below:
10 *
11 *   Redistribution and use in source and binary forms, with or
12 *   without modification, are permitted provided that the following
13 *   conditions are met:
14 *
15 *    - Redistributions of source code must retain the above
16 *	copyright notice, this list of conditions and the following
17 *	disclaimer.
18 *
19 *    - Redistributions in binary form must reproduce the above
20 *	copyright notice, this list of conditions and the following
21 *	disclaimer in the documentation and/or other materials
22 *	provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 *
33 *******************************************************************************/
34 
35 #include "i40iw_status.h"
36 #include "i40iw_osdep.h"
37 #include "i40iw_register.h"
38 #include "i40iw_hmc.h"
39 
40 #include "i40iw_d.h"
41 #include "i40iw_type.h"
42 #include "i40iw_p.h"
43 
44 #include <linux/pci.h>
45 #include <linux/genalloc.h>
46 #include <linux/vmalloc.h>
47 #include "i40iw_pble.h"
48 #include "i40iw.h"
49 
50 struct i40iw_device;
51 static enum i40iw_status_code add_pble_pool(struct i40iw_sc_dev *dev,
52 					    struct i40iw_hmc_pble_rsrc *pble_rsrc);
53 static void i40iw_free_vmalloc_mem(struct i40iw_hw *hw, struct i40iw_chunk *chunk);
54 
55 /**
56  * i40iw_destroy_pble_pool - destroy pool during module unload
57  * @pble_rsrc:	pble resources
58  */
i40iw_destroy_pble_pool(struct i40iw_sc_dev * dev,struct i40iw_hmc_pble_rsrc * pble_rsrc)59 void i40iw_destroy_pble_pool(struct i40iw_sc_dev *dev, struct i40iw_hmc_pble_rsrc *pble_rsrc)
60 {
61 	struct list_head *clist;
62 	struct list_head *tlist;
63 	struct i40iw_chunk *chunk;
64 	struct i40iw_pble_pool *pinfo = &pble_rsrc->pinfo;
65 
66 	if (pinfo->pool) {
67 		list_for_each_safe(clist, tlist, &pinfo->clist) {
68 			chunk = list_entry(clist, struct i40iw_chunk, list);
69 			if (chunk->type == I40IW_VMALLOC)
70 				i40iw_free_vmalloc_mem(dev->hw, chunk);
71 			kfree(chunk);
72 		}
73 		gen_pool_destroy(pinfo->pool);
74 	}
75 }
76 
77 /**
78  * i40iw_hmc_init_pble - Initialize pble resources during module load
79  * @dev: i40iw_sc_dev struct
80  * @pble_rsrc:	pble resources
81  */
i40iw_hmc_init_pble(struct i40iw_sc_dev * dev,struct i40iw_hmc_pble_rsrc * pble_rsrc)82 enum i40iw_status_code i40iw_hmc_init_pble(struct i40iw_sc_dev *dev,
83 					   struct i40iw_hmc_pble_rsrc *pble_rsrc)
84 {
85 	struct i40iw_hmc_info *hmc_info;
86 	u32 fpm_idx = 0;
87 
88 	hmc_info = dev->hmc_info;
89 	pble_rsrc->fpm_base_addr = hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].base;
90 	/* Now start the pble' on 4k boundary */
91 	if (pble_rsrc->fpm_base_addr & 0xfff)
92 		fpm_idx = (PAGE_SIZE - (pble_rsrc->fpm_base_addr & 0xfff)) >> 3;
93 
94 	pble_rsrc->unallocated_pble =
95 	    hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt - fpm_idx;
96 	pble_rsrc->next_fpm_addr = pble_rsrc->fpm_base_addr + (fpm_idx << 3);
97 
98 	pble_rsrc->pinfo.pool_shift = POOL_SHIFT;
99 	pble_rsrc->pinfo.pool = gen_pool_create(pble_rsrc->pinfo.pool_shift, -1);
100 	INIT_LIST_HEAD(&pble_rsrc->pinfo.clist);
101 	if (!pble_rsrc->pinfo.pool)
102 		goto error;
103 
104 	if (add_pble_pool(dev, pble_rsrc))
105 		goto error;
106 
107 	return 0;
108 
109  error:i40iw_destroy_pble_pool(dev, pble_rsrc);
110 	return I40IW_ERR_NO_MEMORY;
111 }
112 
113 /**
114  * get_sd_pd_idx -  Returns sd index, pd index and rel_pd_idx from fpm address
115  * @ pble_rsrc:	structure containing fpm address
116  * @ idx: where to return indexes
117  */
get_sd_pd_idx(struct i40iw_hmc_pble_rsrc * pble_rsrc,struct sd_pd_idx * idx)118 static inline void get_sd_pd_idx(struct i40iw_hmc_pble_rsrc *pble_rsrc,
119 				 struct sd_pd_idx *idx)
120 {
121 	idx->sd_idx = (u32)(pble_rsrc->next_fpm_addr) / I40IW_HMC_DIRECT_BP_SIZE;
122 	idx->pd_idx = (u32)(pble_rsrc->next_fpm_addr) / I40IW_HMC_PAGED_BP_SIZE;
123 	idx->rel_pd_idx = (idx->pd_idx % I40IW_HMC_PD_CNT_IN_SD);
124 }
125 
126 /**
127  * add_sd_direct - add sd direct for pble
128  * @dev: hardware control device structure
129  * @pble_rsrc: pble resource ptr
130  * @info: page info for sd
131  */
add_sd_direct(struct i40iw_sc_dev * dev,struct i40iw_hmc_pble_rsrc * pble_rsrc,struct i40iw_add_page_info * info)132 static enum i40iw_status_code add_sd_direct(struct i40iw_sc_dev *dev,
133 					    struct i40iw_hmc_pble_rsrc *pble_rsrc,
134 					    struct i40iw_add_page_info *info)
135 {
136 	enum i40iw_status_code ret_code = 0;
137 	struct sd_pd_idx *idx = &info->idx;
138 	struct i40iw_chunk *chunk = info->chunk;
139 	struct i40iw_hmc_info *hmc_info = info->hmc_info;
140 	struct i40iw_hmc_sd_entry *sd_entry = info->sd_entry;
141 	u32 offset = 0;
142 
143 	if (!sd_entry->valid) {
144 		if (dev->is_pf) {
145 			ret_code = i40iw_add_sd_table_entry(dev->hw, hmc_info,
146 							    info->idx.sd_idx,
147 							    I40IW_SD_TYPE_DIRECT,
148 							    I40IW_HMC_DIRECT_BP_SIZE);
149 			if (ret_code)
150 				return ret_code;
151 			chunk->type = I40IW_DMA_COHERENT;
152 		}
153 	}
154 	offset = idx->rel_pd_idx << I40IW_HMC_PAGED_BP_SHIFT;
155 	chunk->size = info->pages << I40IW_HMC_PAGED_BP_SHIFT;
156 	chunk->vaddr = ((u8 *)sd_entry->u.bp.addr.va + offset);
157 	chunk->fpm_addr = pble_rsrc->next_fpm_addr;
158 	i40iw_debug(dev, I40IW_DEBUG_PBLE, "chunk_size[%d] = 0x%x vaddr=%p fpm_addr = %llx\n",
159 		    chunk->size, chunk->size, chunk->vaddr, chunk->fpm_addr);
160 	return 0;
161 }
162 
163 /**
164  * i40iw_free_vmalloc_mem - free vmalloc during close
165  * @hw: hw struct
166  * @chunk: chunk information for vmalloc
167  */
i40iw_free_vmalloc_mem(struct i40iw_hw * hw,struct i40iw_chunk * chunk)168 static void i40iw_free_vmalloc_mem(struct i40iw_hw *hw, struct i40iw_chunk *chunk)
169 {
170 	struct pci_dev *pcidev = hw->pcidev;
171 	int i;
172 
173 	if (!chunk->pg_cnt)
174 		goto done;
175 	for (i = 0; i < chunk->pg_cnt; i++)
176 		dma_unmap_page(&pcidev->dev, chunk->dmaaddrs[i], PAGE_SIZE, DMA_BIDIRECTIONAL);
177 
178  done:
179 	kfree(chunk->dmaaddrs);
180 	chunk->dmaaddrs = NULL;
181 	vfree(chunk->vaddr);
182 	chunk->vaddr = NULL;
183 	chunk->type = 0;
184 }
185 
186 /**
187  * i40iw_get_vmalloc_mem - get 2M page for sd
188  * @hw: hardware address
189  * @chunk: chunk to adf
190  * @pg_cnt: #of 4 K pages
191  */
i40iw_get_vmalloc_mem(struct i40iw_hw * hw,struct i40iw_chunk * chunk,int pg_cnt)192 static enum i40iw_status_code i40iw_get_vmalloc_mem(struct i40iw_hw *hw,
193 						    struct i40iw_chunk *chunk,
194 						    int pg_cnt)
195 {
196 	struct pci_dev *pcidev = hw->pcidev;
197 	struct page *page;
198 	u8 *addr;
199 	u32 size;
200 	int i;
201 
202 	chunk->dmaaddrs = kzalloc(pg_cnt << 3, GFP_KERNEL);
203 	if (!chunk->dmaaddrs)
204 		return I40IW_ERR_NO_MEMORY;
205 	size = PAGE_SIZE * pg_cnt;
206 	chunk->vaddr = vmalloc(size);
207 	if (!chunk->vaddr) {
208 		kfree(chunk->dmaaddrs);
209 		chunk->dmaaddrs = NULL;
210 		return I40IW_ERR_NO_MEMORY;
211 	}
212 	chunk->size = size;
213 	addr = (u8 *)chunk->vaddr;
214 	for (i = 0; i < pg_cnt; i++) {
215 		page = vmalloc_to_page((void *)addr);
216 		if (!page)
217 			break;
218 		chunk->dmaaddrs[i] = dma_map_page(&pcidev->dev, page, 0,
219 						  PAGE_SIZE, DMA_BIDIRECTIONAL);
220 		if (dma_mapping_error(&pcidev->dev, chunk->dmaaddrs[i]))
221 			break;
222 		addr += PAGE_SIZE;
223 	}
224 
225 	chunk->pg_cnt = i;
226 	chunk->type = I40IW_VMALLOC;
227 	if (i == pg_cnt)
228 		return 0;
229 
230 	i40iw_free_vmalloc_mem(hw, chunk);
231 	return I40IW_ERR_NO_MEMORY;
232 }
233 
234 /**
235  * fpm_to_idx - given fpm address, get pble index
236  * @pble_rsrc: pble resource management
237  * @addr: fpm address for index
238  */
fpm_to_idx(struct i40iw_hmc_pble_rsrc * pble_rsrc,u64 addr)239 static inline u32 fpm_to_idx(struct i40iw_hmc_pble_rsrc *pble_rsrc, u64 addr)
240 {
241 	return (addr - (pble_rsrc->fpm_base_addr)) >> 3;
242 }
243 
244 /**
245  * add_bp_pages - add backing pages for sd
246  * @dev: hardware control device structure
247  * @pble_rsrc: pble resource management
248  * @info: page info for sd
249  */
add_bp_pages(struct i40iw_sc_dev * dev,struct i40iw_hmc_pble_rsrc * pble_rsrc,struct i40iw_add_page_info * info)250 static enum i40iw_status_code add_bp_pages(struct i40iw_sc_dev *dev,
251 					   struct i40iw_hmc_pble_rsrc *pble_rsrc,
252 					   struct i40iw_add_page_info *info)
253 {
254 	u8 *addr;
255 	struct i40iw_dma_mem mem;
256 	struct i40iw_hmc_pd_entry *pd_entry;
257 	struct i40iw_hmc_sd_entry *sd_entry = info->sd_entry;
258 	struct i40iw_hmc_info *hmc_info = info->hmc_info;
259 	struct i40iw_chunk *chunk = info->chunk;
260 	struct i40iw_manage_vf_pble_info vf_pble_info;
261 	enum i40iw_status_code status = 0;
262 	u32 rel_pd_idx = info->idx.rel_pd_idx;
263 	u32 pd_idx = info->idx.pd_idx;
264 	u32 i;
265 
266 	status = i40iw_get_vmalloc_mem(dev->hw, chunk, info->pages);
267 	if (status)
268 		return I40IW_ERR_NO_MEMORY;
269 	status = i40iw_add_sd_table_entry(dev->hw, hmc_info,
270 					  info->idx.sd_idx, I40IW_SD_TYPE_PAGED,
271 					  I40IW_HMC_DIRECT_BP_SIZE);
272 	if (status)
273 		goto error;
274 	if (!dev->is_pf) {
275 		status = i40iw_vchnl_vf_add_hmc_objs(dev, I40IW_HMC_IW_PBLE,
276 						     fpm_to_idx(pble_rsrc,
277 								pble_rsrc->next_fpm_addr),
278 						     (info->pages << PBLE_512_SHIFT));
279 		if (status) {
280 			i40iw_pr_err("allocate PBLEs in the PF.  Error %i\n", status);
281 			goto error;
282 		}
283 	}
284 	addr = chunk->vaddr;
285 	for (i = 0; i < info->pages; i++) {
286 		mem.pa = chunk->dmaaddrs[i];
287 		mem.size = PAGE_SIZE;
288 		mem.va = (void *)(addr);
289 		pd_entry = &sd_entry->u.pd_table.pd_entry[rel_pd_idx++];
290 		if (!pd_entry->valid) {
291 			status = i40iw_add_pd_table_entry(dev->hw, hmc_info, pd_idx++, &mem);
292 			if (status)
293 				goto error;
294 			addr += PAGE_SIZE;
295 		} else {
296 			i40iw_pr_err("pd entry is valid expecting to be invalid\n");
297 		}
298 	}
299 	if (!dev->is_pf) {
300 		vf_pble_info.first_pd_index = info->idx.rel_pd_idx;
301 		vf_pble_info.inv_pd_ent = false;
302 		vf_pble_info.pd_entry_cnt = PBLE_PER_PAGE;
303 		vf_pble_info.pd_pl_pba = sd_entry->u.pd_table.pd_page_addr.pa;
304 		vf_pble_info.sd_index = info->idx.sd_idx;
305 		status = i40iw_hw_manage_vf_pble_bp(dev->back_dev,
306 						    &vf_pble_info, true);
307 		if (status) {
308 			i40iw_pr_err("CQP manage VF PBLE BP failed.  %i\n", status);
309 			goto error;
310 		}
311 	}
312 	chunk->fpm_addr = pble_rsrc->next_fpm_addr;
313 	return 0;
314 error:
315 	i40iw_free_vmalloc_mem(dev->hw, chunk);
316 	return status;
317 }
318 
319 /**
320  * add_pble_pool - add a sd entry for pble resoure
321  * @dev: hardware control device structure
322  * @pble_rsrc: pble resource management
323  */
add_pble_pool(struct i40iw_sc_dev * dev,struct i40iw_hmc_pble_rsrc * pble_rsrc)324 static enum i40iw_status_code add_pble_pool(struct i40iw_sc_dev *dev,
325 					    struct i40iw_hmc_pble_rsrc *pble_rsrc)
326 {
327 	struct i40iw_hmc_sd_entry *sd_entry;
328 	struct i40iw_hmc_info *hmc_info;
329 	struct i40iw_chunk *chunk;
330 	struct i40iw_add_page_info info;
331 	struct sd_pd_idx *idx = &info.idx;
332 	enum i40iw_status_code ret_code = 0;
333 	enum i40iw_sd_entry_type sd_entry_type;
334 	u64 sd_reg_val = 0;
335 	u32 pages;
336 
337 	if (pble_rsrc->unallocated_pble < PBLE_PER_PAGE)
338 		return I40IW_ERR_NO_MEMORY;
339 	if (pble_rsrc->next_fpm_addr & 0xfff) {
340 		i40iw_pr_err("next fpm_addr %llx\n", pble_rsrc->next_fpm_addr);
341 		return I40IW_ERR_INVALID_PAGE_DESC_INDEX;
342 	}
343 	chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
344 	if (!chunk)
345 		return I40IW_ERR_NO_MEMORY;
346 	hmc_info = dev->hmc_info;
347 	chunk->fpm_addr = pble_rsrc->next_fpm_addr;
348 	get_sd_pd_idx(pble_rsrc, idx);
349 	sd_entry = &hmc_info->sd_table.sd_entry[idx->sd_idx];
350 	pages = (idx->rel_pd_idx) ? (I40IW_HMC_PD_CNT_IN_SD -
351 			idx->rel_pd_idx) : I40IW_HMC_PD_CNT_IN_SD;
352 	pages = min(pages, pble_rsrc->unallocated_pble >> PBLE_512_SHIFT);
353 	info.chunk = chunk;
354 	info.hmc_info = hmc_info;
355 	info.pages = pages;
356 	info.sd_entry = sd_entry;
357 	if (!sd_entry->valid) {
358 		sd_entry_type = (!idx->rel_pd_idx &&
359 				 (pages == I40IW_HMC_PD_CNT_IN_SD) &&
360 				 dev->is_pf) ? I40IW_SD_TYPE_DIRECT : I40IW_SD_TYPE_PAGED;
361 	} else {
362 		sd_entry_type = sd_entry->entry_type;
363 	}
364 	i40iw_debug(dev, I40IW_DEBUG_PBLE,
365 		    "pages = %d, unallocated_pble[%u] current_fpm_addr = %llx\n",
366 		    pages, pble_rsrc->unallocated_pble, pble_rsrc->next_fpm_addr);
367 	i40iw_debug(dev, I40IW_DEBUG_PBLE, "sd_entry_type = %d sd_entry valid = %d\n",
368 		    sd_entry_type, sd_entry->valid);
369 
370 	if (sd_entry_type == I40IW_SD_TYPE_DIRECT)
371 		ret_code = add_sd_direct(dev, pble_rsrc, &info);
372 	if (ret_code)
373 		sd_entry_type = I40IW_SD_TYPE_PAGED;
374 	else
375 		pble_rsrc->stats_direct_sds++;
376 
377 	if (sd_entry_type == I40IW_SD_TYPE_PAGED) {
378 		ret_code = add_bp_pages(dev, pble_rsrc, &info);
379 		if (ret_code)
380 			goto error;
381 		else
382 			pble_rsrc->stats_paged_sds++;
383 	}
384 
385 	if (gen_pool_add_virt(pble_rsrc->pinfo.pool, (unsigned long)chunk->vaddr,
386 			      (phys_addr_t)chunk->fpm_addr, chunk->size, -1)) {
387 		i40iw_pr_err("could not allocate memory by gen_pool_addr_virt()\n");
388 		ret_code = I40IW_ERR_NO_MEMORY;
389 		goto error;
390 	}
391 	pble_rsrc->next_fpm_addr += chunk->size;
392 	i40iw_debug(dev, I40IW_DEBUG_PBLE, "next_fpm_addr = %llx chunk_size[%u] = 0x%x\n",
393 		    pble_rsrc->next_fpm_addr, chunk->size, chunk->size);
394 	pble_rsrc->unallocated_pble -= (chunk->size >> 3);
395 	sd_reg_val = (sd_entry_type == I40IW_SD_TYPE_PAGED) ?
396 			sd_entry->u.pd_table.pd_page_addr.pa : sd_entry->u.bp.addr.pa;
397 	if (dev->is_pf && !sd_entry->valid) {
398 		ret_code = i40iw_hmc_sd_one(dev, hmc_info->hmc_fn_id,
399 					    sd_reg_val, idx->sd_idx,
400 					    sd_entry->entry_type, true);
401 		if (ret_code) {
402 			i40iw_pr_err("cqp cmd failed for sd (pbles)\n");
403 			goto error;
404 		}
405 	}
406 
407 	sd_entry->valid = true;
408 	list_add(&chunk->list, &pble_rsrc->pinfo.clist);
409 	return 0;
410  error:
411 	kfree(chunk);
412 	return ret_code;
413 }
414 
415 /**
416  * free_lvl2 - fee level 2 pble
417  * @pble_rsrc: pble resource management
418  * @palloc: level 2 pble allocation
419  */
free_lvl2(struct i40iw_hmc_pble_rsrc * pble_rsrc,struct i40iw_pble_alloc * palloc)420 static void free_lvl2(struct i40iw_hmc_pble_rsrc *pble_rsrc,
421 		      struct i40iw_pble_alloc *palloc)
422 {
423 	u32 i;
424 	struct gen_pool *pool;
425 	struct i40iw_pble_level2 *lvl2 = &palloc->level2;
426 	struct i40iw_pble_info *root = &lvl2->root;
427 	struct i40iw_pble_info *leaf = lvl2->leaf;
428 
429 	pool = pble_rsrc->pinfo.pool;
430 
431 	for (i = 0; i < lvl2->leaf_cnt; i++, leaf++) {
432 		if (leaf->addr)
433 			gen_pool_free(pool, leaf->addr, (leaf->cnt << 3));
434 		else
435 			break;
436 	}
437 
438 	if (root->addr)
439 		gen_pool_free(pool, root->addr, (root->cnt << 3));
440 
441 	kfree(lvl2->leaf);
442 	lvl2->leaf = NULL;
443 }
444 
445 /**
446  * get_lvl2_pble - get level 2 pble resource
447  * @pble_rsrc: pble resource management
448  * @palloc: level 2 pble allocation
449  * @pool: pool pointer
450  */
get_lvl2_pble(struct i40iw_hmc_pble_rsrc * pble_rsrc,struct i40iw_pble_alloc * palloc,struct gen_pool * pool)451 static enum i40iw_status_code get_lvl2_pble(struct i40iw_hmc_pble_rsrc *pble_rsrc,
452 					    struct i40iw_pble_alloc *palloc,
453 					    struct gen_pool *pool)
454 {
455 	u32 lf4k, lflast, total, i;
456 	u32 pblcnt = PBLE_PER_PAGE;
457 	u64 *addr;
458 	struct i40iw_pble_level2 *lvl2 = &palloc->level2;
459 	struct i40iw_pble_info *root = &lvl2->root;
460 	struct i40iw_pble_info *leaf;
461 
462 	/* number of full 512 (4K) leafs) */
463 	lf4k = palloc->total_cnt >> 9;
464 	lflast = palloc->total_cnt % PBLE_PER_PAGE;
465 	total = (lflast == 0) ? lf4k : lf4k + 1;
466 	lvl2->leaf_cnt = total;
467 
468 	leaf = kzalloc((sizeof(*leaf) * total), GFP_ATOMIC);
469 	if (!leaf)
470 		return I40IW_ERR_NO_MEMORY;
471 	lvl2->leaf = leaf;
472 	/* allocate pbles for the root */
473 	root->addr = gen_pool_alloc(pool, (total << 3));
474 	if (!root->addr) {
475 		kfree(lvl2->leaf);
476 		lvl2->leaf = NULL;
477 		return I40IW_ERR_NO_MEMORY;
478 	}
479 	root->idx = fpm_to_idx(pble_rsrc,
480 			       (u64)gen_pool_virt_to_phys(pool, root->addr));
481 	root->cnt = total;
482 	addr = (u64 *)root->addr;
483 	for (i = 0; i < total; i++, leaf++) {
484 		pblcnt = (lflast && ((i + 1) == total)) ? lflast : PBLE_PER_PAGE;
485 		leaf->addr = gen_pool_alloc(pool, (pblcnt << 3));
486 		if (!leaf->addr)
487 			goto error;
488 		leaf->idx = fpm_to_idx(pble_rsrc, (u64)gen_pool_virt_to_phys(pool, leaf->addr));
489 
490 		leaf->cnt = pblcnt;
491 		*addr = (u64)leaf->idx;
492 		addr++;
493 	}
494 	palloc->level = I40IW_LEVEL_2;
495 	pble_rsrc->stats_lvl2++;
496 	return 0;
497  error:
498 	free_lvl2(pble_rsrc, palloc);
499 	return I40IW_ERR_NO_MEMORY;
500 }
501 
502 /**
503  * get_lvl1_pble - get level 1 pble resource
504  * @dev: hardware control device structure
505  * @pble_rsrc: pble resource management
506  * @palloc: level 1 pble allocation
507  */
get_lvl1_pble(struct i40iw_sc_dev * dev,struct i40iw_hmc_pble_rsrc * pble_rsrc,struct i40iw_pble_alloc * palloc)508 static enum i40iw_status_code get_lvl1_pble(struct i40iw_sc_dev *dev,
509 					    struct i40iw_hmc_pble_rsrc *pble_rsrc,
510 					    struct i40iw_pble_alloc *palloc)
511 {
512 	u64 *addr;
513 	struct gen_pool *pool;
514 	struct i40iw_pble_info *lvl1 = &palloc->level1;
515 
516 	pool = pble_rsrc->pinfo.pool;
517 	addr = (u64 *)gen_pool_alloc(pool, (palloc->total_cnt << 3));
518 
519 	if (!addr)
520 		return I40IW_ERR_NO_MEMORY;
521 
522 	palloc->level = I40IW_LEVEL_1;
523 	lvl1->addr = (unsigned long)addr;
524 	lvl1->idx = fpm_to_idx(pble_rsrc, (u64)gen_pool_virt_to_phys(pool,
525 			       (unsigned long)addr));
526 	lvl1->cnt = palloc->total_cnt;
527 	pble_rsrc->stats_lvl1++;
528 	return 0;
529 }
530 
531 /**
532  * get_lvl1_lvl2_pble - calls get_lvl1 and get_lvl2 pble routine
533  * @dev: i40iw_sc_dev struct
534  * @pble_rsrc:	pble resources
535  * @palloc: contains all inforamtion regarding pble (idx + pble addr)
536  * @pool: pointer to general purpose special memory pool descriptor
537  */
get_lvl1_lvl2_pble(struct i40iw_sc_dev * dev,struct i40iw_hmc_pble_rsrc * pble_rsrc,struct i40iw_pble_alloc * palloc,struct gen_pool * pool)538 static inline enum i40iw_status_code get_lvl1_lvl2_pble(struct i40iw_sc_dev *dev,
539 							struct i40iw_hmc_pble_rsrc *pble_rsrc,
540 							struct i40iw_pble_alloc *palloc,
541 							struct gen_pool *pool)
542 {
543 	enum i40iw_status_code status = 0;
544 
545 	status = get_lvl1_pble(dev, pble_rsrc, palloc);
546 	if (status && (palloc->total_cnt > PBLE_PER_PAGE))
547 		status = get_lvl2_pble(pble_rsrc, palloc, pool);
548 	return status;
549 }
550 
551 /**
552  * i40iw_get_pble - allocate pbles from the pool
553  * @dev: i40iw_sc_dev struct
554  * @pble_rsrc:	pble resources
555  * @palloc: contains all inforamtion regarding pble (idx + pble addr)
556  * @pble_cnt: #of pbles requested
557  */
i40iw_get_pble(struct i40iw_sc_dev * dev,struct i40iw_hmc_pble_rsrc * pble_rsrc,struct i40iw_pble_alloc * palloc,u32 pble_cnt)558 enum i40iw_status_code i40iw_get_pble(struct i40iw_sc_dev *dev,
559 				      struct i40iw_hmc_pble_rsrc *pble_rsrc,
560 				      struct i40iw_pble_alloc *palloc,
561 				      u32 pble_cnt)
562 {
563 	struct gen_pool *pool;
564 	enum i40iw_status_code status = 0;
565 	u32 max_sds = 0;
566 	int i;
567 
568 	pool = pble_rsrc->pinfo.pool;
569 	palloc->total_cnt = pble_cnt;
570 	palloc->level = I40IW_LEVEL_0;
571 	/*check first to see if we can get pble's without acquiring additional sd's */
572 	status = get_lvl1_lvl2_pble(dev, pble_rsrc, palloc, pool);
573 	if (!status)
574 		goto exit;
575 	max_sds = (palloc->total_cnt >> 18) + 1;
576 	for (i = 0; i < max_sds; i++) {
577 		status = add_pble_pool(dev, pble_rsrc);
578 		if (status)
579 			break;
580 		status = get_lvl1_lvl2_pble(dev, pble_rsrc, palloc, pool);
581 		if (!status)
582 			break;
583 	}
584 exit:
585 	if (!status)
586 		pble_rsrc->stats_alloc_ok++;
587 	else
588 		pble_rsrc->stats_alloc_fail++;
589 
590 	return status;
591 }
592 
593 /**
594  * i40iw_free_pble - put pbles back into pool
595  * @pble_rsrc:	pble resources
596  * @palloc: contains all inforamtion regarding pble resource being freed
597  */
i40iw_free_pble(struct i40iw_hmc_pble_rsrc * pble_rsrc,struct i40iw_pble_alloc * palloc)598 void i40iw_free_pble(struct i40iw_hmc_pble_rsrc *pble_rsrc,
599 		     struct i40iw_pble_alloc *palloc)
600 {
601 	struct gen_pool *pool;
602 
603 	pool = pble_rsrc->pinfo.pool;
604 	if (palloc->level == I40IW_LEVEL_2)
605 		free_lvl2(pble_rsrc, palloc);
606 	else
607 		gen_pool_free(pool, palloc->level1.addr,
608 			      (palloc->level1.cnt << 3));
609 	pble_rsrc->stats_alloc_freed++;
610 }
611