Lines Matching refs:table
204 struct hns_roce_hem_table *table, unsigned long *obj, in hns_roce_calc_hem_mhop() argument
213 if (get_hem_table_config(hr_dev, mhop, table->type)) in hns_roce_calc_hem_mhop()
223 bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num); in hns_roce_calc_hem_mhop()
225 chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : in hns_roce_calc_hem_mhop()
227 table_idx = *obj / (chunk_size / table->obj_size); in hns_roce_calc_hem_mhop()
243 table->type, mhop->hop_num); in hns_roce_calc_hem_mhop()
337 struct hns_roce_hem_table *table, unsigned long obj, in calc_hem_config() argument
348 ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, mhop); in calc_hem_config()
356 bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num); in calc_hem_config()
373 table->type, mhop->hop_num); in calc_hem_config()
377 if (unlikely(index->buf >= table->num_hem)) { in calc_hem_config()
379 table->type, index->buf, table->num_hem); in calc_hem_config()
387 struct hns_roce_hem_table *table, in free_mhop_hem() argument
395 hns_roce_free_hem(hr_dev, table->hem[index->buf]); in free_mhop_hem()
396 table->hem[index->buf] = NULL; in free_mhop_hem()
400 dma_free_coherent(dev, bt_size, table->bt_l1[index->l1], in free_mhop_hem()
401 table->bt_l1_dma_addr[index->l1]); in free_mhop_hem()
402 table->bt_l1[index->l1] = NULL; in free_mhop_hem()
406 dma_free_coherent(dev, bt_size, table->bt_l0[index->l0], in free_mhop_hem()
407 table->bt_l0_dma_addr[index->l0]); in free_mhop_hem()
408 table->bt_l0[index->l0] = NULL; in free_mhop_hem()
413 struct hns_roce_hem_table *table, in alloc_mhop_hem() argument
426 if ((check_whether_bt_num_3(table->type, mhop->hop_num) || in alloc_mhop_hem()
427 check_whether_bt_num_2(table->type, mhop->hop_num)) && in alloc_mhop_hem()
428 !table->bt_l0[index->l0]) { in alloc_mhop_hem()
429 table->bt_l0[index->l0] = dma_alloc_coherent(dev, bt_size, in alloc_mhop_hem()
430 &table->bt_l0_dma_addr[index->l0], in alloc_mhop_hem()
432 if (!table->bt_l0[index->l0]) { in alloc_mhop_hem()
440 if (check_whether_bt_num_3(table->type, mhop->hop_num) && in alloc_mhop_hem()
441 !table->bt_l1[index->l1]) { in alloc_mhop_hem()
442 table->bt_l1[index->l1] = dma_alloc_coherent(dev, bt_size, in alloc_mhop_hem()
443 &table->bt_l1_dma_addr[index->l1], in alloc_mhop_hem()
445 if (!table->bt_l1[index->l1]) { in alloc_mhop_hem()
450 *(table->bt_l0[index->l0] + mhop->l1_idx) = in alloc_mhop_hem()
451 table->bt_l1_dma_addr[index->l1]; in alloc_mhop_hem()
458 size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : bt_size; in alloc_mhop_hem()
459 flag = (table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) | __GFP_NOWARN; in alloc_mhop_hem()
460 table->hem[index->buf] = hns_roce_alloc_hem(hr_dev, size >> PAGE_SHIFT, in alloc_mhop_hem()
462 if (!table->hem[index->buf]) { in alloc_mhop_hem()
468 hns_roce_hem_first(table->hem[index->buf], &iter); in alloc_mhop_hem()
470 if (table->type < HEM_TYPE_MTT) { in alloc_mhop_hem()
472 *(table->bt_l1[index->l1] + mhop->l2_idx) = bt_ba; in alloc_mhop_hem()
474 *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba; in alloc_mhop_hem()
476 *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba; in alloc_mhop_hem()
481 free_mhop_hem(hr_dev, table, mhop, index); in alloc_mhop_hem()
487 struct hns_roce_hem_table *table, unsigned long obj, in set_mhop_hem() argument
496 ret = hr_dev->hw->set_hem(hr_dev, table, obj, 0); in set_mhop_hem()
504 ret = hr_dev->hw->set_hem(hr_dev, table, obj, 1); in set_mhop_hem()
516 ret = hr_dev->hw->set_hem(hr_dev, table, obj, step_idx); in set_mhop_hem()
525 struct hns_roce_hem_table *table, in hns_roce_table_mhop_get() argument
533 ret = calc_hem_config(hr_dev, table, obj, &mhop, &index); in hns_roce_table_mhop_get()
539 mutex_lock(&table->mutex); in hns_roce_table_mhop_get()
540 if (table->hem[index.buf]) { in hns_roce_table_mhop_get()
541 refcount_inc(&table->hem[index.buf]->refcount); in hns_roce_table_mhop_get()
545 ret = alloc_mhop_hem(hr_dev, table, &mhop, &index); in hns_roce_table_mhop_get()
552 if (table->type < HEM_TYPE_MTT) { in hns_roce_table_mhop_get()
553 ret = set_mhop_hem(hr_dev, table, obj, &mhop, &index); in hns_roce_table_mhop_get()
560 refcount_set(&table->hem[index.buf]->refcount, 1); in hns_roce_table_mhop_get()
564 free_mhop_hem(hr_dev, table, &mhop, &index); in hns_roce_table_mhop_get()
566 mutex_unlock(&table->mutex); in hns_roce_table_mhop_get()
571 struct hns_roce_hem_table *table, unsigned long obj) in hns_roce_table_get() argument
577 if (hns_roce_check_whether_mhop(hr_dev, table->type)) in hns_roce_table_get()
578 return hns_roce_table_mhop_get(hr_dev, table, obj); in hns_roce_table_get()
580 i = obj / (table->table_chunk_size / table->obj_size); in hns_roce_table_get()
582 mutex_lock(&table->mutex); in hns_roce_table_get()
584 if (table->hem[i]) { in hns_roce_table_get()
585 refcount_inc(&table->hem[i]->refcount); in hns_roce_table_get()
589 table->hem[i] = hns_roce_alloc_hem(hr_dev, in hns_roce_table_get()
590 table->table_chunk_size >> PAGE_SHIFT, in hns_roce_table_get()
591 table->table_chunk_size, in hns_roce_table_get()
592 (table->lowmem ? GFP_KERNEL : in hns_roce_table_get()
594 if (!table->hem[i]) { in hns_roce_table_get()
600 ret = hr_dev->hw->set_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT); in hns_roce_table_get()
602 hns_roce_free_hem(hr_dev, table->hem[i]); in hns_roce_table_get()
603 table->hem[i] = NULL; in hns_roce_table_get()
609 refcount_set(&table->hem[i]->refcount, 1); in hns_roce_table_get()
611 mutex_unlock(&table->mutex); in hns_roce_table_get()
616 struct hns_roce_hem_table *table, unsigned long obj, in clear_mhop_hem() argument
627 if (check_whether_bt_num_2(table->type, hop_num)) { in clear_mhop_hem()
628 if (hns_roce_check_hem_null(table->hem, index->buf, in clear_mhop_hem()
629 chunk_ba_num, table->num_hem)) in clear_mhop_hem()
631 } else if (check_whether_bt_num_3(table->type, hop_num)) { in clear_mhop_hem()
632 if (hns_roce_check_hem_null(table->hem, index->buf, in clear_mhop_hem()
633 chunk_ba_num, table->num_hem)) { in clear_mhop_hem()
635 if (hns_roce_check_bt_null(table->bt_l1, index->l1, in clear_mhop_hem()
641 if (table->type < HEM_TYPE_MTT) { in clear_mhop_hem()
647 if (hr_dev->hw->clear_hem(hr_dev, table, obj, step_idx)) in clear_mhop_hem()
651 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1)) in clear_mhop_hem()
655 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0)) in clear_mhop_hem()
661 struct hns_roce_hem_table *table, in hns_roce_table_mhop_put() argument
670 ret = calc_hem_config(hr_dev, table, obj, &mhop, &index); in hns_roce_table_mhop_put()
677 mutex_lock(&table->mutex); in hns_roce_table_mhop_put()
678 else if (!refcount_dec_and_mutex_lock(&table->hem[index.buf]->refcount, in hns_roce_table_mhop_put()
679 &table->mutex)) in hns_roce_table_mhop_put()
682 clear_mhop_hem(hr_dev, table, obj, &mhop, &index); in hns_roce_table_mhop_put()
683 free_mhop_hem(hr_dev, table, &mhop, &index); in hns_roce_table_mhop_put()
685 mutex_unlock(&table->mutex); in hns_roce_table_mhop_put()
689 struct hns_roce_hem_table *table, unsigned long obj) in hns_roce_table_put() argument
694 if (hns_roce_check_whether_mhop(hr_dev, table->type)) { in hns_roce_table_put()
695 hns_roce_table_mhop_put(hr_dev, table, obj, 1); in hns_roce_table_put()
699 i = obj / (table->table_chunk_size / table->obj_size); in hns_roce_table_put()
701 if (!refcount_dec_and_mutex_lock(&table->hem[i]->refcount, in hns_roce_table_put()
702 &table->mutex)) in hns_roce_table_put()
705 if (hr_dev->hw->clear_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT)) in hns_roce_table_put()
708 hns_roce_free_hem(hr_dev, table->hem[i]); in hns_roce_table_put()
709 table->hem[i] = NULL; in hns_roce_table_put()
711 mutex_unlock(&table->mutex); in hns_roce_table_put()
715 struct hns_roce_hem_table *table, in hns_roce_table_find() argument
730 if (!table->lowmem) in hns_roce_table_find()
733 mutex_lock(&table->mutex); in hns_roce_table_find()
735 if (!hns_roce_check_whether_mhop(hr_dev, table->type)) { in hns_roce_table_find()
736 obj_per_chunk = table->table_chunk_size / table->obj_size; in hns_roce_table_find()
737 hem = table->hem[obj / obj_per_chunk]; in hns_roce_table_find()
739 dma_offset = offset = idx_offset * table->obj_size; in hns_roce_table_find()
743 if (hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop)) in hns_roce_table_find()
754 hem = table->hem[hem_idx]; in hns_roce_table_find()
782 mutex_unlock(&table->mutex); in hns_roce_table_find()
787 struct hns_roce_hem_table *table, u32 type, in hns_roce_init_hem_table() argument
795 table->table_chunk_size = hr_dev->caps.chunk_sz; in hns_roce_init_hem_table()
796 obj_per_chunk = table->table_chunk_size / obj_size; in hns_roce_init_hem_table()
799 table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL); in hns_roce_init_hem_table()
800 if (!table->hem) in hns_roce_init_hem_table()
825 table->hem = kcalloc(num_hem, sizeof(*table->hem), in hns_roce_init_hem_table()
827 if (!table->hem) in hns_roce_init_hem_table()
834 table->bt_l1 = kcalloc(num_bt_l1, in hns_roce_init_hem_table()
835 sizeof(*table->bt_l1), in hns_roce_init_hem_table()
837 if (!table->bt_l1) in hns_roce_init_hem_table()
840 table->bt_l1_dma_addr = kcalloc(num_bt_l1, in hns_roce_init_hem_table()
841 sizeof(*table->bt_l1_dma_addr), in hns_roce_init_hem_table()
844 if (!table->bt_l1_dma_addr) in hns_roce_init_hem_table()
850 table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0), in hns_roce_init_hem_table()
852 if (!table->bt_l0) in hns_roce_init_hem_table()
855 table->bt_l0_dma_addr = kcalloc(num_bt_l0, in hns_roce_init_hem_table()
856 sizeof(*table->bt_l0_dma_addr), in hns_roce_init_hem_table()
858 if (!table->bt_l0_dma_addr) in hns_roce_init_hem_table()
863 table->type = type; in hns_roce_init_hem_table()
864 table->num_hem = num_hem; in hns_roce_init_hem_table()
865 table->obj_size = obj_size; in hns_roce_init_hem_table()
866 table->lowmem = use_lowmem; in hns_roce_init_hem_table()
867 mutex_init(&table->mutex); in hns_roce_init_hem_table()
872 kfree(table->bt_l0); in hns_roce_init_hem_table()
873 table->bt_l0 = NULL; in hns_roce_init_hem_table()
876 kfree(table->bt_l1_dma_addr); in hns_roce_init_hem_table()
877 table->bt_l1_dma_addr = NULL; in hns_roce_init_hem_table()
880 kfree(table->bt_l1); in hns_roce_init_hem_table()
881 table->bt_l1 = NULL; in hns_roce_init_hem_table()
884 kfree(table->hem); in hns_roce_init_hem_table()
885 table->hem = NULL; in hns_roce_init_hem_table()
892 struct hns_roce_hem_table *table) in hns_roce_cleanup_mhop_hem_table() argument
899 if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop)) in hns_roce_cleanup_mhop_hem_table()
901 buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size : in hns_roce_cleanup_mhop_hem_table()
904 for (i = 0; i < table->num_hem; ++i) { in hns_roce_cleanup_mhop_hem_table()
905 obj = i * buf_chunk_size / table->obj_size; in hns_roce_cleanup_mhop_hem_table()
906 if (table->hem[i]) in hns_roce_cleanup_mhop_hem_table()
907 hns_roce_table_mhop_put(hr_dev, table, obj, 0); in hns_roce_cleanup_mhop_hem_table()
910 kfree(table->hem); in hns_roce_cleanup_mhop_hem_table()
911 table->hem = NULL; in hns_roce_cleanup_mhop_hem_table()
912 kfree(table->bt_l1); in hns_roce_cleanup_mhop_hem_table()
913 table->bt_l1 = NULL; in hns_roce_cleanup_mhop_hem_table()
914 kfree(table->bt_l1_dma_addr); in hns_roce_cleanup_mhop_hem_table()
915 table->bt_l1_dma_addr = NULL; in hns_roce_cleanup_mhop_hem_table()
916 kfree(table->bt_l0); in hns_roce_cleanup_mhop_hem_table()
917 table->bt_l0 = NULL; in hns_roce_cleanup_mhop_hem_table()
918 kfree(table->bt_l0_dma_addr); in hns_roce_cleanup_mhop_hem_table()
919 table->bt_l0_dma_addr = NULL; in hns_roce_cleanup_mhop_hem_table()
923 struct hns_roce_hem_table *table) in hns_roce_cleanup_hem_table() argument
928 if (hns_roce_check_whether_mhop(hr_dev, table->type)) { in hns_roce_cleanup_hem_table()
929 hns_roce_cleanup_mhop_hem_table(hr_dev, table); in hns_roce_cleanup_hem_table()
933 for (i = 0; i < table->num_hem; ++i) in hns_roce_cleanup_hem_table()
934 if (table->hem[i]) { in hns_roce_cleanup_hem_table()
935 if (hr_dev->hw->clear_hem(hr_dev, table, in hns_roce_cleanup_hem_table()
936 i * table->table_chunk_size / table->obj_size, 0)) in hns_roce_cleanup_hem_table()
939 hns_roce_free_hem(hr_dev, table->hem[i]); in hns_roce_cleanup_hem_table()
942 kfree(table->hem); in hns_roce_cleanup_hem_table()
949 &hr_dev->srq_table.table); in hns_roce_cleanup_hem()
950 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table); in hns_roce_cleanup_hem()