1 /*
2 * Copyright (c) 2016 Hisilicon Limited.
3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34 #include <linux/platform_device.h>
35 #include "hns_roce_device.h"
36 #include "hns_roce_hem.h"
37 #include "hns_roce_common.h"
38
39 #define DMA_ADDR_T_SHIFT 12
40 #define BT_BA_SHIFT 32
41
hns_roce_check_whether_mhop(struct hns_roce_dev * hr_dev,u32 type)42 bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type)
43 {
44 if ((hr_dev->caps.qpc_hop_num && type == HEM_TYPE_QPC) ||
45 (hr_dev->caps.mpt_hop_num && type == HEM_TYPE_MTPT) ||
46 (hr_dev->caps.cqc_hop_num && type == HEM_TYPE_CQC) ||
47 (hr_dev->caps.srqc_hop_num && type == HEM_TYPE_SRQC) ||
48 (hr_dev->caps.cqe_hop_num && type == HEM_TYPE_CQE) ||
49 (hr_dev->caps.mtt_hop_num && type == HEM_TYPE_MTT))
50 return true;
51
52 return false;
53 }
54 EXPORT_SYMBOL_GPL(hns_roce_check_whether_mhop);
55
hns_roce_check_hem_null(struct hns_roce_hem ** hem,u64 start_idx,u32 bt_chunk_num,u64 hem_max_num)56 static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 start_idx,
57 u32 bt_chunk_num, u64 hem_max_num)
58 {
59 u64 check_max_num = start_idx + bt_chunk_num;
60 u64 i;
61
62 for (i = start_idx; (i < check_max_num) && (i < hem_max_num); i++)
63 if (hem[i])
64 return false;
65
66 return true;
67 }
68
hns_roce_check_bt_null(u64 ** bt,u64 start_idx,u32 bt_chunk_num)69 static bool hns_roce_check_bt_null(u64 **bt, u64 start_idx, u32 bt_chunk_num)
70 {
71 int i;
72
73 for (i = 0; i < bt_chunk_num; i++)
74 if (bt[start_idx + i])
75 return false;
76
77 return true;
78 }
79
hns_roce_get_bt_num(u32 table_type,u32 hop_num)80 static int hns_roce_get_bt_num(u32 table_type, u32 hop_num)
81 {
82 if (check_whether_bt_num_3(table_type, hop_num))
83 return 3;
84 else if (check_whether_bt_num_2(table_type, hop_num))
85 return 2;
86 else if (check_whether_bt_num_1(table_type, hop_num))
87 return 1;
88 else
89 return 0;
90 }
91
hns_roce_calc_hem_mhop(struct hns_roce_dev * hr_dev,struct hns_roce_hem_table * table,unsigned long * obj,struct hns_roce_hem_mhop * mhop)92 int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,
93 struct hns_roce_hem_table *table, unsigned long *obj,
94 struct hns_roce_hem_mhop *mhop)
95 {
96 struct device *dev = hr_dev->dev;
97 u32 chunk_ba_num;
98 u32 table_idx;
99 u32 bt_num;
100 u32 chunk_size;
101
102 switch (table->type) {
103 case HEM_TYPE_QPC:
104 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz
105 + PAGE_SHIFT);
106 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz
107 + PAGE_SHIFT);
108 mhop->ba_l0_num = hr_dev->caps.qpc_bt_num;
109 mhop->hop_num = hr_dev->caps.qpc_hop_num;
110 break;
111 case HEM_TYPE_MTPT:
112 mhop->buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz
113 + PAGE_SHIFT);
114 mhop->bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz
115 + PAGE_SHIFT);
116 mhop->ba_l0_num = hr_dev->caps.mpt_bt_num;
117 mhop->hop_num = hr_dev->caps.mpt_hop_num;
118 break;
119 case HEM_TYPE_CQC:
120 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz
121 + PAGE_SHIFT);
122 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz
123 + PAGE_SHIFT);
124 mhop->ba_l0_num = hr_dev->caps.cqc_bt_num;
125 mhop->hop_num = hr_dev->caps.cqc_hop_num;
126 break;
127 case HEM_TYPE_SRQC:
128 mhop->buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz
129 + PAGE_SHIFT);
130 mhop->bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz
131 + PAGE_SHIFT);
132 mhop->ba_l0_num = hr_dev->caps.srqc_bt_num;
133 mhop->hop_num = hr_dev->caps.srqc_hop_num;
134 break;
135 case HEM_TYPE_MTT:
136 mhop->buf_chunk_size = 1 << (hr_dev->caps.mtt_buf_pg_sz
137 + PAGE_SHIFT);
138 mhop->bt_chunk_size = 1 << (hr_dev->caps.mtt_ba_pg_sz
139 + PAGE_SHIFT);
140 mhop->ba_l0_num = mhop->bt_chunk_size / 8;
141 mhop->hop_num = hr_dev->caps.mtt_hop_num;
142 break;
143 case HEM_TYPE_CQE:
144 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqe_buf_pg_sz
145 + PAGE_SHIFT);
146 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqe_ba_pg_sz
147 + PAGE_SHIFT);
148 mhop->ba_l0_num = mhop->bt_chunk_size / 8;
149 mhop->hop_num = hr_dev->caps.cqe_hop_num;
150 break;
151 default:
152 dev_err(dev, "Table %d not support multi-hop addressing!\n",
153 table->type);
154 return -EINVAL;
155 }
156
157 if (!obj)
158 return 0;
159
160 /*
161 * QPC/MTPT/CQC/SRQC alloc hem for buffer pages.
162 * MTT/CQE alloc hem for bt pages.
163 */
164 bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);
165 chunk_ba_num = mhop->bt_chunk_size / 8;
166 chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size :
167 mhop->bt_chunk_size;
168 table_idx = (*obj & (table->num_obj - 1)) /
169 (chunk_size / table->obj_size);
170 switch (bt_num) {
171 case 3:
172 mhop->l2_idx = table_idx & (chunk_ba_num - 1);
173 mhop->l1_idx = table_idx / chunk_ba_num & (chunk_ba_num - 1);
174 mhop->l0_idx = (table_idx / chunk_ba_num) / chunk_ba_num;
175 break;
176 case 2:
177 mhop->l1_idx = table_idx & (chunk_ba_num - 1);
178 mhop->l0_idx = table_idx / chunk_ba_num;
179 break;
180 case 1:
181 mhop->l0_idx = table_idx;
182 break;
183 default:
184 dev_err(dev, "Table %d not support hop_num = %d!\n",
185 table->type, mhop->hop_num);
186 return -EINVAL;
187 }
188 if (mhop->l0_idx >= mhop->ba_l0_num)
189 mhop->l0_idx %= mhop->ba_l0_num;
190
191 return 0;
192 }
193 EXPORT_SYMBOL_GPL(hns_roce_calc_hem_mhop);
194
hns_roce_alloc_hem(struct hns_roce_dev * hr_dev,int npages,unsigned long hem_alloc_size,gfp_t gfp_mask)195 static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev,
196 int npages,
197 unsigned long hem_alloc_size,
198 gfp_t gfp_mask)
199 {
200 struct hns_roce_hem_chunk *chunk = NULL;
201 struct hns_roce_hem *hem;
202 struct scatterlist *mem;
203 int order;
204 void *buf;
205
206 WARN_ON(gfp_mask & __GFP_HIGHMEM);
207
208 hem = kmalloc(sizeof(*hem),
209 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
210 if (!hem)
211 return NULL;
212
213 hem->refcount = 0;
214 INIT_LIST_HEAD(&hem->chunk_list);
215
216 order = get_order(hem_alloc_size);
217
218 while (npages > 0) {
219 if (!chunk) {
220 chunk = kmalloc(sizeof(*chunk),
221 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
222 if (!chunk)
223 goto fail;
224
225 sg_init_table(chunk->mem, HNS_ROCE_HEM_CHUNK_LEN);
226 chunk->npages = 0;
227 chunk->nsg = 0;
228 memset(chunk->buf, 0, sizeof(chunk->buf));
229 list_add_tail(&chunk->list, &hem->chunk_list);
230 }
231
232 while (1 << order > npages)
233 --order;
234
235 /*
236 * Alloc memory one time. If failed, don't alloc small block
237 * memory, directly return fail.
238 */
239 mem = &chunk->mem[chunk->npages];
240 buf = dma_alloc_coherent(hr_dev->dev, PAGE_SIZE << order,
241 &sg_dma_address(mem), gfp_mask);
242 if (!buf)
243 goto fail;
244
245 chunk->buf[chunk->npages] = buf;
246 sg_dma_len(mem) = PAGE_SIZE << order;
247
248 ++chunk->npages;
249 ++chunk->nsg;
250 npages -= 1 << order;
251 }
252
253 return hem;
254
255 fail:
256 hns_roce_free_hem(hr_dev, hem);
257 return NULL;
258 }
259
hns_roce_free_hem(struct hns_roce_dev * hr_dev,struct hns_roce_hem * hem)260 void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem)
261 {
262 struct hns_roce_hem_chunk *chunk, *tmp;
263 int i;
264
265 if (!hem)
266 return;
267
268 list_for_each_entry_safe(chunk, tmp, &hem->chunk_list, list) {
269 for (i = 0; i < chunk->npages; ++i)
270 dma_free_coherent(hr_dev->dev,
271 sg_dma_len(&chunk->mem[i]),
272 chunk->buf[i],
273 sg_dma_address(&chunk->mem[i]));
274 kfree(chunk);
275 }
276
277 kfree(hem);
278 }
279
hns_roce_set_hem(struct hns_roce_dev * hr_dev,struct hns_roce_hem_table * table,unsigned long obj)280 static int hns_roce_set_hem(struct hns_roce_dev *hr_dev,
281 struct hns_roce_hem_table *table, unsigned long obj)
282 {
283 spinlock_t *lock = &hr_dev->bt_cmd_lock;
284 struct device *dev = hr_dev->dev;
285 unsigned long end = 0;
286 unsigned long flags;
287 struct hns_roce_hem_iter iter;
288 void __iomem *bt_cmd;
289 u32 bt_cmd_h_val = 0;
290 u32 bt_cmd_val[2];
291 u32 bt_cmd_l = 0;
292 u64 bt_ba = 0;
293 int ret = 0;
294
295 /* Find the HEM(Hardware Entry Memory) entry */
296 unsigned long i = (obj & (table->num_obj - 1)) /
297 (table->table_chunk_size / table->obj_size);
298
299 switch (table->type) {
300 case HEM_TYPE_QPC:
301 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
302 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_QPC);
303 break;
304 case HEM_TYPE_MTPT:
305 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
306 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S,
307 HEM_TYPE_MTPT);
308 break;
309 case HEM_TYPE_CQC:
310 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
311 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_CQC);
312 break;
313 case HEM_TYPE_SRQC:
314 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
315 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S,
316 HEM_TYPE_SRQC);
317 break;
318 default:
319 return ret;
320 }
321 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M,
322 ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj);
323 roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0);
324 roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1);
325
326 /* Currently iter only a chunk */
327 for (hns_roce_hem_first(table->hem[i], &iter);
328 !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) {
329 bt_ba = hns_roce_hem_addr(&iter) >> DMA_ADDR_T_SHIFT;
330
331 spin_lock_irqsave(lock, flags);
332
333 bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG;
334
335 end = msecs_to_jiffies(HW_SYNC_TIMEOUT_MSECS) + jiffies;
336 while (1) {
337 if (readl(bt_cmd) >> BT_CMD_SYNC_SHIFT) {
338 if (!(time_before(jiffies, end))) {
339 dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n");
340 spin_unlock_irqrestore(lock, flags);
341 return -EBUSY;
342 }
343 } else {
344 break;
345 }
346 mdelay(HW_SYNC_SLEEP_TIME_INTERVAL);
347 }
348
349 bt_cmd_l = (u32)bt_ba;
350 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M,
351 ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S,
352 bt_ba >> BT_BA_SHIFT);
353
354 bt_cmd_val[0] = bt_cmd_l;
355 bt_cmd_val[1] = bt_cmd_h_val;
356 hns_roce_write64_k(bt_cmd_val,
357 hr_dev->reg_base + ROCEE_BT_CMD_L_REG);
358 spin_unlock_irqrestore(lock, flags);
359 }
360
361 return ret;
362 }
363
hns_roce_table_mhop_get(struct hns_roce_dev * hr_dev,struct hns_roce_hem_table * table,unsigned long obj)364 static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev,
365 struct hns_roce_hem_table *table,
366 unsigned long obj)
367 {
368 struct device *dev = hr_dev->dev;
369 struct hns_roce_hem_mhop mhop;
370 struct hns_roce_hem_iter iter;
371 u32 buf_chunk_size;
372 u32 bt_chunk_size;
373 u32 chunk_ba_num;
374 u32 hop_num;
375 u32 size;
376 u32 bt_num;
377 u64 hem_idx;
378 u64 bt_l1_idx = 0;
379 u64 bt_l0_idx = 0;
380 u64 bt_ba;
381 unsigned long mhop_obj = obj;
382 int bt_l1_allocated = 0;
383 int bt_l0_allocated = 0;
384 int step_idx;
385 int ret;
386
387 ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
388 if (ret)
389 return ret;
390
391 buf_chunk_size = mhop.buf_chunk_size;
392 bt_chunk_size = mhop.bt_chunk_size;
393 hop_num = mhop.hop_num;
394 chunk_ba_num = bt_chunk_size / 8;
395
396 bt_num = hns_roce_get_bt_num(table->type, hop_num);
397 switch (bt_num) {
398 case 3:
399 hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num +
400 mhop.l1_idx * chunk_ba_num + mhop.l2_idx;
401 bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
402 bt_l0_idx = mhop.l0_idx;
403 break;
404 case 2:
405 hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
406 bt_l0_idx = mhop.l0_idx;
407 break;
408 case 1:
409 hem_idx = mhop.l0_idx;
410 break;
411 default:
412 dev_err(dev, "Table %d not support hop_num = %d!\n",
413 table->type, hop_num);
414 return -EINVAL;
415 }
416
417 if (unlikely(hem_idx >= table->num_hem)) {
418 dev_err(dev, "Table %d exceed hem limt idx = %llu,max = %lu!\n",
419 table->type, hem_idx, table->num_hem);
420 return -EINVAL;
421 }
422
423 mutex_lock(&table->mutex);
424
425 if (table->hem[hem_idx]) {
426 ++table->hem[hem_idx]->refcount;
427 goto out;
428 }
429
430 /* alloc L1 BA's chunk */
431 if ((check_whether_bt_num_3(table->type, hop_num) ||
432 check_whether_bt_num_2(table->type, hop_num)) &&
433 !table->bt_l0[bt_l0_idx]) {
434 table->bt_l0[bt_l0_idx] = dma_alloc_coherent(dev, bt_chunk_size,
435 &(table->bt_l0_dma_addr[bt_l0_idx]),
436 GFP_KERNEL);
437 if (!table->bt_l0[bt_l0_idx]) {
438 ret = -ENOMEM;
439 goto out;
440 }
441 bt_l0_allocated = 1;
442
443 /* set base address to hardware */
444 if (table->type < HEM_TYPE_MTT) {
445 step_idx = 0;
446 if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) {
447 ret = -ENODEV;
448 dev_err(dev, "set HEM base address to HW failed!\n");
449 goto err_dma_alloc_l1;
450 }
451 }
452 }
453
454 /* alloc L2 BA's chunk */
455 if (check_whether_bt_num_3(table->type, hop_num) &&
456 !table->bt_l1[bt_l1_idx]) {
457 table->bt_l1[bt_l1_idx] = dma_alloc_coherent(dev, bt_chunk_size,
458 &(table->bt_l1_dma_addr[bt_l1_idx]),
459 GFP_KERNEL);
460 if (!table->bt_l1[bt_l1_idx]) {
461 ret = -ENOMEM;
462 goto err_dma_alloc_l1;
463 }
464 bt_l1_allocated = 1;
465 *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) =
466 table->bt_l1_dma_addr[bt_l1_idx];
467
468 /* set base address to hardware */
469 step_idx = 1;
470 if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) {
471 ret = -ENODEV;
472 dev_err(dev, "set HEM base address to HW failed!\n");
473 goto err_alloc_hem_buf;
474 }
475 }
476
477 /*
478 * alloc buffer space chunk for QPC/MTPT/CQC/SRQC.
479 * alloc bt space chunk for MTT/CQE.
480 */
481 size = table->type < HEM_TYPE_MTT ? buf_chunk_size : bt_chunk_size;
482 table->hem[hem_idx] = hns_roce_alloc_hem(hr_dev,
483 size >> PAGE_SHIFT,
484 size,
485 (table->lowmem ? GFP_KERNEL :
486 GFP_HIGHUSER) | __GFP_NOWARN);
487 if (!table->hem[hem_idx]) {
488 ret = -ENOMEM;
489 goto err_alloc_hem_buf;
490 }
491
492 hns_roce_hem_first(table->hem[hem_idx], &iter);
493 bt_ba = hns_roce_hem_addr(&iter);
494
495 if (table->type < HEM_TYPE_MTT) {
496 if (hop_num == 2) {
497 *(table->bt_l1[bt_l1_idx] + mhop.l2_idx) = bt_ba;
498 step_idx = 2;
499 } else if (hop_num == 1) {
500 *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba;
501 step_idx = 1;
502 } else if (hop_num == HNS_ROCE_HOP_NUM_0) {
503 step_idx = 0;
504 } else {
505 ret = -EINVAL;
506 goto err_dma_alloc_l1;
507 }
508
509 /* set HEM base address to hardware */
510 if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) {
511 ret = -ENODEV;
512 dev_err(dev, "set HEM base address to HW failed!\n");
513 goto err_alloc_hem_buf;
514 }
515 } else if (hop_num == 2) {
516 *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba;
517 }
518
519 ++table->hem[hem_idx]->refcount;
520 goto out;
521
522 err_alloc_hem_buf:
523 if (bt_l1_allocated) {
524 dma_free_coherent(dev, bt_chunk_size, table->bt_l1[bt_l1_idx],
525 table->bt_l1_dma_addr[bt_l1_idx]);
526 table->bt_l1[bt_l1_idx] = NULL;
527 }
528
529 err_dma_alloc_l1:
530 if (bt_l0_allocated) {
531 dma_free_coherent(dev, bt_chunk_size, table->bt_l0[bt_l0_idx],
532 table->bt_l0_dma_addr[bt_l0_idx]);
533 table->bt_l0[bt_l0_idx] = NULL;
534 }
535
536 out:
537 mutex_unlock(&table->mutex);
538 return ret;
539 }
540
hns_roce_table_get(struct hns_roce_dev * hr_dev,struct hns_roce_hem_table * table,unsigned long obj)541 int hns_roce_table_get(struct hns_roce_dev *hr_dev,
542 struct hns_roce_hem_table *table, unsigned long obj)
543 {
544 struct device *dev = hr_dev->dev;
545 int ret = 0;
546 unsigned long i;
547
548 if (hns_roce_check_whether_mhop(hr_dev, table->type))
549 return hns_roce_table_mhop_get(hr_dev, table, obj);
550
551 i = (obj & (table->num_obj - 1)) / (table->table_chunk_size /
552 table->obj_size);
553
554 mutex_lock(&table->mutex);
555
556 if (table->hem[i]) {
557 ++table->hem[i]->refcount;
558 goto out;
559 }
560
561 table->hem[i] = hns_roce_alloc_hem(hr_dev,
562 table->table_chunk_size >> PAGE_SHIFT,
563 table->table_chunk_size,
564 (table->lowmem ? GFP_KERNEL :
565 GFP_HIGHUSER) | __GFP_NOWARN);
566 if (!table->hem[i]) {
567 ret = -ENOMEM;
568 goto out;
569 }
570
571 /* Set HEM base address(128K/page, pa) to Hardware */
572 if (hns_roce_set_hem(hr_dev, table, obj)) {
573 hns_roce_free_hem(hr_dev, table->hem[i]);
574 table->hem[i] = NULL;
575 ret = -ENODEV;
576 dev_err(dev, "set HEM base address to HW failed.\n");
577 goto out;
578 }
579
580 ++table->hem[i]->refcount;
581 out:
582 mutex_unlock(&table->mutex);
583 return ret;
584 }
585
hns_roce_table_mhop_put(struct hns_roce_dev * hr_dev,struct hns_roce_hem_table * table,unsigned long obj,int check_refcount)586 static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev,
587 struct hns_roce_hem_table *table,
588 unsigned long obj,
589 int check_refcount)
590 {
591 struct device *dev = hr_dev->dev;
592 struct hns_roce_hem_mhop mhop;
593 unsigned long mhop_obj = obj;
594 u32 bt_chunk_size;
595 u32 chunk_ba_num;
596 u32 hop_num;
597 u32 start_idx;
598 u32 bt_num;
599 u64 hem_idx;
600 u64 bt_l1_idx = 0;
601 int ret;
602
603 ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
604 if (ret)
605 return;
606
607 bt_chunk_size = mhop.bt_chunk_size;
608 hop_num = mhop.hop_num;
609 chunk_ba_num = bt_chunk_size / 8;
610
611 bt_num = hns_roce_get_bt_num(table->type, hop_num);
612 switch (bt_num) {
613 case 3:
614 hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num +
615 mhop.l1_idx * chunk_ba_num + mhop.l2_idx;
616 bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
617 break;
618 case 2:
619 hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
620 break;
621 case 1:
622 hem_idx = mhop.l0_idx;
623 break;
624 default:
625 dev_err(dev, "Table %d not support hop_num = %d!\n",
626 table->type, hop_num);
627 return;
628 }
629
630 mutex_lock(&table->mutex);
631
632 if (check_refcount && (--table->hem[hem_idx]->refcount > 0)) {
633 mutex_unlock(&table->mutex);
634 return;
635 }
636
637 if (table->type < HEM_TYPE_MTT && hop_num == 1) {
638 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1))
639 dev_warn(dev, "Clear HEM base address failed.\n");
640 } else if (table->type < HEM_TYPE_MTT && hop_num == 2) {
641 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 2))
642 dev_warn(dev, "Clear HEM base address failed.\n");
643 } else if (table->type < HEM_TYPE_MTT &&
644 hop_num == HNS_ROCE_HOP_NUM_0) {
645 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
646 dev_warn(dev, "Clear HEM base address failed.\n");
647 }
648
649 /*
650 * free buffer space chunk for QPC/MTPT/CQC/SRQC.
651 * free bt space chunk for MTT/CQE.
652 */
653 hns_roce_free_hem(hr_dev, table->hem[hem_idx]);
654 table->hem[hem_idx] = NULL;
655
656 if (check_whether_bt_num_2(table->type, hop_num)) {
657 start_idx = mhop.l0_idx * chunk_ba_num;
658 if (hns_roce_check_hem_null(table->hem, start_idx,
659 chunk_ba_num, table->num_hem)) {
660 if (table->type < HEM_TYPE_MTT &&
661 hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
662 dev_warn(dev, "Clear HEM base address failed.\n");
663
664 dma_free_coherent(dev, bt_chunk_size,
665 table->bt_l0[mhop.l0_idx],
666 table->bt_l0_dma_addr[mhop.l0_idx]);
667 table->bt_l0[mhop.l0_idx] = NULL;
668 }
669 } else if (check_whether_bt_num_3(table->type, hop_num)) {
670 start_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num +
671 mhop.l1_idx * chunk_ba_num;
672 if (hns_roce_check_hem_null(table->hem, start_idx,
673 chunk_ba_num, table->num_hem)) {
674 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1))
675 dev_warn(dev, "Clear HEM base address failed.\n");
676
677 dma_free_coherent(dev, bt_chunk_size,
678 table->bt_l1[bt_l1_idx],
679 table->bt_l1_dma_addr[bt_l1_idx]);
680 table->bt_l1[bt_l1_idx] = NULL;
681
682 start_idx = mhop.l0_idx * chunk_ba_num;
683 if (hns_roce_check_bt_null(table->bt_l1, start_idx,
684 chunk_ba_num)) {
685 if (hr_dev->hw->clear_hem(hr_dev, table, obj,
686 0))
687 dev_warn(dev, "Clear HEM base address failed.\n");
688
689 dma_free_coherent(dev, bt_chunk_size,
690 table->bt_l0[mhop.l0_idx],
691 table->bt_l0_dma_addr[mhop.l0_idx]);
692 table->bt_l0[mhop.l0_idx] = NULL;
693 }
694 }
695 }
696
697 mutex_unlock(&table->mutex);
698 }
699
hns_roce_table_put(struct hns_roce_dev * hr_dev,struct hns_roce_hem_table * table,unsigned long obj)700 void hns_roce_table_put(struct hns_roce_dev *hr_dev,
701 struct hns_roce_hem_table *table, unsigned long obj)
702 {
703 struct device *dev = hr_dev->dev;
704 unsigned long i;
705
706 if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
707 hns_roce_table_mhop_put(hr_dev, table, obj, 1);
708 return;
709 }
710
711 i = (obj & (table->num_obj - 1)) /
712 (table->table_chunk_size / table->obj_size);
713
714 mutex_lock(&table->mutex);
715
716 if (--table->hem[i]->refcount == 0) {
717 /* Clear HEM base address */
718 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
719 dev_warn(dev, "Clear HEM base address failed.\n");
720
721 hns_roce_free_hem(hr_dev, table->hem[i]);
722 table->hem[i] = NULL;
723 }
724
725 mutex_unlock(&table->mutex);
726 }
727
hns_roce_table_find(struct hns_roce_dev * hr_dev,struct hns_roce_hem_table * table,unsigned long obj,dma_addr_t * dma_handle)728 void *hns_roce_table_find(struct hns_roce_dev *hr_dev,
729 struct hns_roce_hem_table *table,
730 unsigned long obj, dma_addr_t *dma_handle)
731 {
732 struct hns_roce_hem_chunk *chunk;
733 struct hns_roce_hem_mhop mhop;
734 struct hns_roce_hem *hem;
735 void *addr = NULL;
736 unsigned long mhop_obj = obj;
737 unsigned long obj_per_chunk;
738 unsigned long idx_offset;
739 int offset, dma_offset;
740 int length;
741 int i, j;
742 u32 hem_idx = 0;
743
744 if (!table->lowmem)
745 return NULL;
746
747 mutex_lock(&table->mutex);
748
749 if (!hns_roce_check_whether_mhop(hr_dev, table->type)) {
750 obj_per_chunk = table->table_chunk_size / table->obj_size;
751 hem = table->hem[(obj & (table->num_obj - 1)) / obj_per_chunk];
752 idx_offset = (obj & (table->num_obj - 1)) % obj_per_chunk;
753 dma_offset = offset = idx_offset * table->obj_size;
754 } else {
755 u32 seg_size = 64; /* 8 bytes per BA and 8 BA per segment */
756
757 hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
758 /* mtt mhop */
759 i = mhop.l0_idx;
760 j = mhop.l1_idx;
761 if (mhop.hop_num == 2)
762 hem_idx = i * (mhop.bt_chunk_size / 8) + j;
763 else if (mhop.hop_num == 1 ||
764 mhop.hop_num == HNS_ROCE_HOP_NUM_0)
765 hem_idx = i;
766
767 hem = table->hem[hem_idx];
768 dma_offset = offset = (obj & (table->num_obj - 1)) * seg_size %
769 mhop.bt_chunk_size;
770 if (mhop.hop_num == 2)
771 dma_offset = offset = 0;
772 }
773
774 if (!hem)
775 goto out;
776
777 list_for_each_entry(chunk, &hem->chunk_list, list) {
778 for (i = 0; i < chunk->npages; ++i) {
779 length = sg_dma_len(&chunk->mem[i]);
780 if (dma_handle && dma_offset >= 0) {
781 if (length > (u32)dma_offset)
782 *dma_handle = sg_dma_address(
783 &chunk->mem[i]) + dma_offset;
784 dma_offset -= length;
785 }
786
787 if (length > (u32)offset) {
788 addr = chunk->buf[i] + offset;
789 goto out;
790 }
791 offset -= length;
792 }
793 }
794
795 out:
796 mutex_unlock(&table->mutex);
797 return addr;
798 }
799 EXPORT_SYMBOL_GPL(hns_roce_table_find);
800
hns_roce_table_get_range(struct hns_roce_dev * hr_dev,struct hns_roce_hem_table * table,unsigned long start,unsigned long end)801 int hns_roce_table_get_range(struct hns_roce_dev *hr_dev,
802 struct hns_roce_hem_table *table,
803 unsigned long start, unsigned long end)
804 {
805 struct hns_roce_hem_mhop mhop;
806 unsigned long inc = table->table_chunk_size / table->obj_size;
807 unsigned long i;
808 int ret;
809
810 if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
811 hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop);
812 inc = mhop.bt_chunk_size / table->obj_size;
813 }
814
815 /* Allocate MTT entry memory according to chunk(128K) */
816 for (i = start; i <= end; i += inc) {
817 ret = hns_roce_table_get(hr_dev, table, i);
818 if (ret)
819 goto fail;
820 }
821
822 return 0;
823
824 fail:
825 while (i > start) {
826 i -= inc;
827 hns_roce_table_put(hr_dev, table, i);
828 }
829 return ret;
830 }
831
hns_roce_table_put_range(struct hns_roce_dev * hr_dev,struct hns_roce_hem_table * table,unsigned long start,unsigned long end)832 void hns_roce_table_put_range(struct hns_roce_dev *hr_dev,
833 struct hns_roce_hem_table *table,
834 unsigned long start, unsigned long end)
835 {
836 struct hns_roce_hem_mhop mhop;
837 unsigned long inc = table->table_chunk_size / table->obj_size;
838 unsigned long i;
839
840 if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
841 hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop);
842 inc = mhop.bt_chunk_size / table->obj_size;
843 }
844
845 for (i = start; i <= end; i += inc)
846 hns_roce_table_put(hr_dev, table, i);
847 }
848
hns_roce_init_hem_table(struct hns_roce_dev * hr_dev,struct hns_roce_hem_table * table,u32 type,unsigned long obj_size,unsigned long nobj,int use_lowmem)849 int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev,
850 struct hns_roce_hem_table *table, u32 type,
851 unsigned long obj_size, unsigned long nobj,
852 int use_lowmem)
853 {
854 struct device *dev = hr_dev->dev;
855 unsigned long obj_per_chunk;
856 unsigned long num_hem;
857
858 if (!hns_roce_check_whether_mhop(hr_dev, type)) {
859 table->table_chunk_size = hr_dev->caps.chunk_sz;
860 obj_per_chunk = table->table_chunk_size / obj_size;
861 num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk;
862
863 table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL);
864 if (!table->hem)
865 return -ENOMEM;
866 } else {
867 unsigned long buf_chunk_size;
868 unsigned long bt_chunk_size;
869 unsigned long bt_chunk_num;
870 unsigned long num_bt_l0 = 0;
871 u32 hop_num;
872
873 switch (type) {
874 case HEM_TYPE_QPC:
875 buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz
876 + PAGE_SHIFT);
877 bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz
878 + PAGE_SHIFT);
879 num_bt_l0 = hr_dev->caps.qpc_bt_num;
880 hop_num = hr_dev->caps.qpc_hop_num;
881 break;
882 case HEM_TYPE_MTPT:
883 buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz
884 + PAGE_SHIFT);
885 bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz
886 + PAGE_SHIFT);
887 num_bt_l0 = hr_dev->caps.mpt_bt_num;
888 hop_num = hr_dev->caps.mpt_hop_num;
889 break;
890 case HEM_TYPE_CQC:
891 buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz
892 + PAGE_SHIFT);
893 bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz
894 + PAGE_SHIFT);
895 num_bt_l0 = hr_dev->caps.cqc_bt_num;
896 hop_num = hr_dev->caps.cqc_hop_num;
897 break;
898 case HEM_TYPE_SRQC:
899 buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz
900 + PAGE_SHIFT);
901 bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz
902 + PAGE_SHIFT);
903 num_bt_l0 = hr_dev->caps.srqc_bt_num;
904 hop_num = hr_dev->caps.srqc_hop_num;
905 break;
906 case HEM_TYPE_MTT:
907 buf_chunk_size = 1 << (hr_dev->caps.mtt_ba_pg_sz
908 + PAGE_SHIFT);
909 bt_chunk_size = buf_chunk_size;
910 hop_num = hr_dev->caps.mtt_hop_num;
911 break;
912 case HEM_TYPE_CQE:
913 buf_chunk_size = 1 << (hr_dev->caps.cqe_ba_pg_sz
914 + PAGE_SHIFT);
915 bt_chunk_size = buf_chunk_size;
916 hop_num = hr_dev->caps.cqe_hop_num;
917 break;
918 default:
919 dev_err(dev,
920 "Table %d not support to init hem table here!\n",
921 type);
922 return -EINVAL;
923 }
924 obj_per_chunk = buf_chunk_size / obj_size;
925 num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk;
926 bt_chunk_num = bt_chunk_size / 8;
927 if (type >= HEM_TYPE_MTT)
928 num_bt_l0 = bt_chunk_num;
929
930 table->hem = kcalloc(num_hem, sizeof(*table->hem),
931 GFP_KERNEL);
932 if (!table->hem)
933 goto err_kcalloc_hem_buf;
934
935 if (check_whether_bt_num_3(type, hop_num)) {
936 unsigned long num_bt_l1;
937
938 num_bt_l1 = (num_hem + bt_chunk_num - 1) /
939 bt_chunk_num;
940 table->bt_l1 = kcalloc(num_bt_l1,
941 sizeof(*table->bt_l1),
942 GFP_KERNEL);
943 if (!table->bt_l1)
944 goto err_kcalloc_bt_l1;
945
946 table->bt_l1_dma_addr = kcalloc(num_bt_l1,
947 sizeof(*table->bt_l1_dma_addr),
948 GFP_KERNEL);
949
950 if (!table->bt_l1_dma_addr)
951 goto err_kcalloc_l1_dma;
952 }
953
954 if (check_whether_bt_num_2(type, hop_num) ||
955 check_whether_bt_num_3(type, hop_num)) {
956 table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0),
957 GFP_KERNEL);
958 if (!table->bt_l0)
959 goto err_kcalloc_bt_l0;
960
961 table->bt_l0_dma_addr = kcalloc(num_bt_l0,
962 sizeof(*table->bt_l0_dma_addr),
963 GFP_KERNEL);
964 if (!table->bt_l0_dma_addr)
965 goto err_kcalloc_l0_dma;
966 }
967 }
968
969 table->type = type;
970 table->num_hem = num_hem;
971 table->num_obj = nobj;
972 table->obj_size = obj_size;
973 table->lowmem = use_lowmem;
974 mutex_init(&table->mutex);
975
976 return 0;
977
978 err_kcalloc_l0_dma:
979 kfree(table->bt_l0);
980 table->bt_l0 = NULL;
981
982 err_kcalloc_bt_l0:
983 kfree(table->bt_l1_dma_addr);
984 table->bt_l1_dma_addr = NULL;
985
986 err_kcalloc_l1_dma:
987 kfree(table->bt_l1);
988 table->bt_l1 = NULL;
989
990 err_kcalloc_bt_l1:
991 kfree(table->hem);
992 table->hem = NULL;
993
994 err_kcalloc_hem_buf:
995 return -ENOMEM;
996 }
997
hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev * hr_dev,struct hns_roce_hem_table * table)998 static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev *hr_dev,
999 struct hns_roce_hem_table *table)
1000 {
1001 struct hns_roce_hem_mhop mhop;
1002 u32 buf_chunk_size;
1003 int i;
1004 u64 obj;
1005
1006 hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop);
1007 buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size :
1008 mhop.bt_chunk_size;
1009
1010 for (i = 0; i < table->num_hem; ++i) {
1011 obj = i * buf_chunk_size / table->obj_size;
1012 if (table->hem[i])
1013 hns_roce_table_mhop_put(hr_dev, table, obj, 0);
1014 }
1015
1016 kfree(table->hem);
1017 table->hem = NULL;
1018 kfree(table->bt_l1);
1019 table->bt_l1 = NULL;
1020 kfree(table->bt_l1_dma_addr);
1021 table->bt_l1_dma_addr = NULL;
1022 kfree(table->bt_l0);
1023 table->bt_l0 = NULL;
1024 kfree(table->bt_l0_dma_addr);
1025 table->bt_l0_dma_addr = NULL;
1026 }
1027
hns_roce_cleanup_hem_table(struct hns_roce_dev * hr_dev,struct hns_roce_hem_table * table)1028 void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev,
1029 struct hns_roce_hem_table *table)
1030 {
1031 struct device *dev = hr_dev->dev;
1032 unsigned long i;
1033
1034 if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
1035 hns_roce_cleanup_mhop_hem_table(hr_dev, table);
1036 return;
1037 }
1038
1039 for (i = 0; i < table->num_hem; ++i)
1040 if (table->hem[i]) {
1041 if (hr_dev->hw->clear_hem(hr_dev, table,
1042 i * table->table_chunk_size / table->obj_size, 0))
1043 dev_err(dev, "Clear HEM base address failed.\n");
1044
1045 hns_roce_free_hem(hr_dev, table->hem[i]);
1046 }
1047
1048 kfree(table->hem);
1049 }
1050
hns_roce_cleanup_hem(struct hns_roce_dev * hr_dev)1051 void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev)
1052 {
1053 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table);
1054 if (hr_dev->caps.trrl_entry_sz)
1055 hns_roce_cleanup_hem_table(hr_dev,
1056 &hr_dev->qp_table.trrl_table);
1057 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table);
1058 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table);
1059 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table);
1060 if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))
1061 hns_roce_cleanup_hem_table(hr_dev,
1062 &hr_dev->mr_table.mtt_cqe_table);
1063 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtt_table);
1064 }
1065