Lines Matching refs:rcd
129 struct hfi1_ctxtdata *rcd; in hfi1_create_kctxt() local
135 ret = hfi1_create_ctxtdata(ppd, dd->node, &rcd); in hfi1_create_kctxt()
146 rcd->flags = HFI1_CAP_KGET(MULTI_PKT_EGR) | in hfi1_create_kctxt()
152 if (rcd->ctxt == HFI1_CTRL_CTXT) in hfi1_create_kctxt()
153 rcd->flags |= HFI1_CAP_DMA_RTAIL; in hfi1_create_kctxt()
154 rcd->fast_handler = get_dma_rtail_setting(rcd) ? in hfi1_create_kctxt()
158 hfi1_set_seq_cnt(rcd, 1); in hfi1_create_kctxt()
160 rcd->sc = sc_alloc(dd, SC_ACK, rcd->rcvhdrqentsize, dd->node); in hfi1_create_kctxt()
161 if (!rcd->sc) { in hfi1_create_kctxt()
165 hfi1_init_ctxt(rcd->sc); in hfi1_create_kctxt()
178 dd->rcd = kcalloc_node(dd->num_rcv_contexts, sizeof(*dd->rcd), in hfi1_create_kctxts()
180 if (!dd->rcd) in hfi1_create_kctxts()
191 for (i = 0; dd->rcd && i < dd->first_dyn_alloc_ctxt; ++i) in hfi1_create_kctxts()
192 hfi1_free_ctxt(dd->rcd[i]); in hfi1_create_kctxts()
195 kfree(dd->rcd); in hfi1_create_kctxts()
196 dd->rcd = NULL; in hfi1_create_kctxts()
203 static void hfi1_rcd_init(struct hfi1_ctxtdata *rcd) in hfi1_rcd_init() argument
205 kref_init(&rcd->kref); in hfi1_rcd_init()
216 struct hfi1_ctxtdata *rcd = in hfi1_rcd_free() local
219 spin_lock_irqsave(&rcd->dd->uctxt_lock, flags); in hfi1_rcd_free()
220 rcd->dd->rcd[rcd->ctxt] = NULL; in hfi1_rcd_free()
221 spin_unlock_irqrestore(&rcd->dd->uctxt_lock, flags); in hfi1_rcd_free()
223 hfi1_free_ctxtdata(rcd->dd, rcd); in hfi1_rcd_free()
225 kfree(rcd); in hfi1_rcd_free()
234 int hfi1_rcd_put(struct hfi1_ctxtdata *rcd) in hfi1_rcd_put() argument
236 if (rcd) in hfi1_rcd_put()
237 return kref_put(&rcd->kref, hfi1_rcd_free); in hfi1_rcd_put()
251 int hfi1_rcd_get(struct hfi1_ctxtdata *rcd) in hfi1_rcd_get() argument
253 return kref_get_unless_zero(&rcd->kref); in hfi1_rcd_get()
267 struct hfi1_ctxtdata *rcd, u16 *index) in allocate_rcd_index() argument
274 if (!dd->rcd[ctxt]) in allocate_rcd_index()
278 rcd->ctxt = ctxt; in allocate_rcd_index()
279 dd->rcd[ctxt] = rcd; in allocate_rcd_index()
280 hfi1_rcd_init(rcd); in allocate_rcd_index()
328 struct hfi1_ctxtdata *rcd = NULL; in hfi1_rcd_get_by_index() local
331 if (dd->rcd[ctxt]) { in hfi1_rcd_get_by_index()
332 rcd = dd->rcd[ctxt]; in hfi1_rcd_get_by_index()
333 if (!hfi1_rcd_get(rcd)) in hfi1_rcd_get_by_index()
334 rcd = NULL; in hfi1_rcd_get_by_index()
338 return rcd; in hfi1_rcd_get_by_index()
349 struct hfi1_ctxtdata *rcd; in hfi1_create_ctxtdata() local
357 rcd = kzalloc_node(sizeof(*rcd), GFP_KERNEL, numa); in hfi1_create_ctxtdata()
358 if (rcd) { in hfi1_create_ctxtdata()
363 ret = allocate_rcd_index(dd, rcd, &ctxt); in hfi1_create_ctxtdata()
366 kfree(rcd); in hfi1_create_ctxtdata()
370 INIT_LIST_HEAD(&rcd->qp_wait_list); in hfi1_create_ctxtdata()
371 hfi1_exp_tid_group_init(rcd); in hfi1_create_ctxtdata()
372 rcd->ppd = ppd; in hfi1_create_ctxtdata()
373 rcd->dd = dd; in hfi1_create_ctxtdata()
374 rcd->numa_id = numa; in hfi1_create_ctxtdata()
375 rcd->rcv_array_groups = dd->rcv_entries.ngroups; in hfi1_create_ctxtdata()
376 rcd->rhf_rcv_function_map = normal_rhf_rcv_functions; in hfi1_create_ctxtdata()
377 rcd->slow_handler = handle_receive_interrupt; in hfi1_create_ctxtdata()
378 rcd->do_interrupt = rcd->slow_handler; in hfi1_create_ctxtdata()
379 rcd->msix_intr = CCE_NUM_MSIX_VECTORS; in hfi1_create_ctxtdata()
381 mutex_init(&rcd->exp_mutex); in hfi1_create_ctxtdata()
382 spin_lock_init(&rcd->exp_lock); in hfi1_create_ctxtdata()
383 INIT_LIST_HEAD(&rcd->flow_queue.queue_head); in hfi1_create_ctxtdata()
384 INIT_LIST_HEAD(&rcd->rarr_queue.queue_head); in hfi1_create_ctxtdata()
386 hfi1_cdbg(PROC, "setting up context %u\n", rcd->ctxt); in hfi1_create_ctxtdata()
398 rcd->rcv_array_groups++; in hfi1_create_ctxtdata()
410 rcd->rcv_array_groups++; in hfi1_create_ctxtdata()
416 rcd->eager_base = base * dd->rcv_entries.group_size; in hfi1_create_ctxtdata()
418 rcd->rcvhdrq_cnt = rcvhdrcnt; in hfi1_create_ctxtdata()
419 rcd->rcvhdrqentsize = hfi1_hdrq_entsize; in hfi1_create_ctxtdata()
420 rcd->rhf_offset = in hfi1_create_ctxtdata()
421 rcd->rcvhdrqentsize - sizeof(u64) / sizeof(u32); in hfi1_create_ctxtdata()
433 max_entries = rcd->rcv_array_groups * in hfi1_create_ctxtdata()
436 rcd->egrbufs.count = round_down(rcvtids, in hfi1_create_ctxtdata()
438 if (rcd->egrbufs.count > MAX_EAGER_ENTRIES) { in hfi1_create_ctxtdata()
440 rcd->ctxt); in hfi1_create_ctxtdata()
441 rcd->egrbufs.count = MAX_EAGER_ENTRIES; in hfi1_create_ctxtdata()
445 rcd->ctxt, rcd->egrbufs.count); in hfi1_create_ctxtdata()
455 rcd->egrbufs.buffers = in hfi1_create_ctxtdata()
456 kcalloc_node(rcd->egrbufs.count, in hfi1_create_ctxtdata()
457 sizeof(*rcd->egrbufs.buffers), in hfi1_create_ctxtdata()
459 if (!rcd->egrbufs.buffers) in hfi1_create_ctxtdata()
461 rcd->egrbufs.rcvtids = in hfi1_create_ctxtdata()
462 kcalloc_node(rcd->egrbufs.count, in hfi1_create_ctxtdata()
463 sizeof(*rcd->egrbufs.rcvtids), in hfi1_create_ctxtdata()
465 if (!rcd->egrbufs.rcvtids) in hfi1_create_ctxtdata()
467 rcd->egrbufs.size = eager_buffer_size; in hfi1_create_ctxtdata()
473 if (rcd->egrbufs.size < hfi1_max_mtu) { in hfi1_create_ctxtdata()
474 rcd->egrbufs.size = __roundup_pow_of_two(hfi1_max_mtu); in hfi1_create_ctxtdata()
477 rcd->ctxt, rcd->egrbufs.size); in hfi1_create_ctxtdata()
479 rcd->egrbufs.rcvtid_size = HFI1_MAX_EAGER_BUFFER_SIZE; in hfi1_create_ctxtdata()
483 rcd->opstats = kzalloc_node(sizeof(*rcd->opstats), in hfi1_create_ctxtdata()
485 if (!rcd->opstats) in hfi1_create_ctxtdata()
489 hfi1_kern_init_ctxt_generations(rcd); in hfi1_create_ctxtdata()
492 *context = rcd; in hfi1_create_ctxtdata()
498 hfi1_free_ctxt(rcd); in hfi1_create_ctxtdata()
512 void hfi1_free_ctxt(struct hfi1_ctxtdata *rcd) in hfi1_free_ctxt() argument
514 hfi1_rcd_put(rcd); in hfi1_free_ctxt()
721 struct hfi1_ctxtdata *rcd; in init_after_reset() local
728 rcd = hfi1_rcd_get_by_index(dd, i); in init_after_reset()
731 HFI1_RCVCTRL_TAILUPD_DIS, rcd); in init_after_reset()
732 hfi1_rcd_put(rcd); in init_after_reset()
743 struct hfi1_ctxtdata *rcd; in enable_chip() local
755 rcd = hfi1_rcd_get_by_index(dd, i); in enable_chip()
756 if (!rcd) in enable_chip()
759 rcvmask |= HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL) ? in enable_chip()
761 if (!HFI1_CAP_KGET_MASK(rcd->flags, MULTI_PKT_EGR)) in enable_chip()
763 if (HFI1_CAP_KGET_MASK(rcd->flags, NODROP_RHQ_FULL)) in enable_chip()
765 if (HFI1_CAP_KGET_MASK(rcd->flags, NODROP_EGR_FULL)) in enable_chip()
769 hfi1_rcvctrl(dd, rcvmask, rcd); in enable_chip()
770 sc_enable(rcd->sc); in enable_chip()
771 hfi1_rcd_put(rcd); in enable_chip()
889 struct hfi1_ctxtdata *rcd; in hfi1_init() local
920 for (i = 0; dd->rcd && i < dd->first_dyn_alloc_ctxt; ++i) { in hfi1_init()
927 rcd = hfi1_rcd_get_by_index(dd, i); in hfi1_init()
928 if (!rcd) in hfi1_init()
931 lastfail = hfi1_create_rcvhdrq(dd, rcd); in hfi1_init()
933 lastfail = hfi1_setup_eagerbufs(rcd); in hfi1_init()
935 lastfail = hfi1_kern_exp_rcv_init(rcd, reinit); in hfi1_init()
942 hfi1_rcd_put(rcd); in hfi1_init()
1048 struct hfi1_ctxtdata *rcd; in shutdown_device() local
1073 rcd = hfi1_rcd_get_by_index(dd, i); in shutdown_device()
1078 HFI1_RCVCTRL_ONE_PKT_EGR_DIS, rcd); in shutdown_device()
1079 hfi1_rcd_put(rcd); in shutdown_device()
1127 void hfi1_free_ctxtdata(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd) in hfi1_free_ctxtdata() argument
1131 if (!rcd) in hfi1_free_ctxtdata()
1134 if (rcd->rcvhdrq) { in hfi1_free_ctxtdata()
1135 dma_free_coherent(&dd->pcidev->dev, rcvhdrq_size(rcd), in hfi1_free_ctxtdata()
1136 rcd->rcvhdrq, rcd->rcvhdrq_dma); in hfi1_free_ctxtdata()
1137 rcd->rcvhdrq = NULL; in hfi1_free_ctxtdata()
1138 if (hfi1_rcvhdrtail_kvaddr(rcd)) { in hfi1_free_ctxtdata()
1140 (void *)hfi1_rcvhdrtail_kvaddr(rcd), in hfi1_free_ctxtdata()
1141 rcd->rcvhdrqtailaddr_dma); in hfi1_free_ctxtdata()
1142 rcd->rcvhdrtail_kvaddr = NULL; in hfi1_free_ctxtdata()
1147 kfree(rcd->egrbufs.rcvtids); in hfi1_free_ctxtdata()
1148 rcd->egrbufs.rcvtids = NULL; in hfi1_free_ctxtdata()
1150 for (e = 0; e < rcd->egrbufs.alloced; e++) { in hfi1_free_ctxtdata()
1151 if (rcd->egrbufs.buffers[e].addr) in hfi1_free_ctxtdata()
1153 rcd->egrbufs.buffers[e].len, in hfi1_free_ctxtdata()
1154 rcd->egrbufs.buffers[e].addr, in hfi1_free_ctxtdata()
1155 rcd->egrbufs.buffers[e].dma); in hfi1_free_ctxtdata()
1157 kfree(rcd->egrbufs.buffers); in hfi1_free_ctxtdata()
1158 rcd->egrbufs.alloced = 0; in hfi1_free_ctxtdata()
1159 rcd->egrbufs.buffers = NULL; in hfi1_free_ctxtdata()
1161 sc_free(rcd->sc); in hfi1_free_ctxtdata()
1162 rcd->sc = NULL; in hfi1_free_ctxtdata()
1164 vfree(rcd->subctxt_uregbase); in hfi1_free_ctxtdata()
1165 vfree(rcd->subctxt_rcvegrbuf); in hfi1_free_ctxtdata()
1166 vfree(rcd->subctxt_rcvhdr_base); in hfi1_free_ctxtdata()
1167 kfree(rcd->opstats); in hfi1_free_ctxtdata()
1169 rcd->subctxt_uregbase = NULL; in hfi1_free_ctxtdata()
1170 rcd->subctxt_rcvegrbuf = NULL; in hfi1_free_ctxtdata()
1171 rcd->subctxt_rcvhdr_base = NULL; in hfi1_free_ctxtdata()
1172 rcd->opstats = NULL; in hfi1_free_ctxtdata()
1554 for (ctxt = 0; dd->rcd && ctxt < dd->num_rcv_contexts; ctxt++) { in cleanup_device_data()
1555 struct hfi1_ctxtdata *rcd = dd->rcd[ctxt]; in cleanup_device_data() local
1557 if (rcd) { in cleanup_device_data()
1558 hfi1_free_ctxt_rcv_groups(rcd); in cleanup_device_data()
1559 hfi1_free_ctxt(rcd); in cleanup_device_data()
1563 kfree(dd->rcd); in cleanup_device_data()
1564 dd->rcd = NULL; in cleanup_device_data()
1800 int hfi1_create_rcvhdrq(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd) in hfi1_create_rcvhdrq() argument
1804 if (!rcd->rcvhdrq) { in hfi1_create_rcvhdrq()
1807 amt = rcvhdrq_size(rcd); in hfi1_create_rcvhdrq()
1809 if (rcd->ctxt < dd->first_dyn_alloc_ctxt || rcd->is_vnic) in hfi1_create_rcvhdrq()
1813 rcd->rcvhdrq = dma_alloc_coherent(&dd->pcidev->dev, amt, in hfi1_create_rcvhdrq()
1814 &rcd->rcvhdrq_dma, in hfi1_create_rcvhdrq()
1817 if (!rcd->rcvhdrq) { in hfi1_create_rcvhdrq()
1820 amt, rcd->ctxt); in hfi1_create_rcvhdrq()
1824 if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL) || in hfi1_create_rcvhdrq()
1825 HFI1_CAP_UGET_MASK(rcd->flags, DMA_RTAIL)) { in hfi1_create_rcvhdrq()
1826 rcd->rcvhdrtail_kvaddr = dma_alloc_coherent(&dd->pcidev->dev, in hfi1_create_rcvhdrq()
1828 &rcd->rcvhdrqtailaddr_dma, in hfi1_create_rcvhdrq()
1830 if (!rcd->rcvhdrtail_kvaddr) in hfi1_create_rcvhdrq()
1835 set_hdrq_regs(rcd->dd, rcd->ctxt, rcd->rcvhdrqentsize, in hfi1_create_rcvhdrq()
1836 rcd->rcvhdrq_cnt); in hfi1_create_rcvhdrq()
1843 rcd->ctxt); in hfi1_create_rcvhdrq()
1844 dma_free_coherent(&dd->pcidev->dev, amt, rcd->rcvhdrq, in hfi1_create_rcvhdrq()
1845 rcd->rcvhdrq_dma); in hfi1_create_rcvhdrq()
1846 rcd->rcvhdrq = NULL; in hfi1_create_rcvhdrq()
1860 int hfi1_setup_eagerbufs(struct hfi1_ctxtdata *rcd) in hfi1_setup_eagerbufs() argument
1862 struct hfi1_devdata *dd = rcd->dd; in hfi1_setup_eagerbufs()
1884 if (rcd->egrbufs.size < (round_mtu * dd->rcv_entries.group_size)) in hfi1_setup_eagerbufs()
1885 rcd->egrbufs.size = round_mtu * dd->rcv_entries.group_size; in hfi1_setup_eagerbufs()
1890 if (!HFI1_CAP_KGET_MASK(rcd->flags, MULTI_PKT_EGR)) in hfi1_setup_eagerbufs()
1891 rcd->egrbufs.rcvtid_size = round_mtu; in hfi1_setup_eagerbufs()
1897 if (rcd->egrbufs.size <= (1 << 20)) in hfi1_setup_eagerbufs()
1898 rcd->egrbufs.rcvtid_size = max((unsigned long)round_mtu, in hfi1_setup_eagerbufs()
1899 rounddown_pow_of_two(rcd->egrbufs.size / 8)); in hfi1_setup_eagerbufs()
1901 while (alloced_bytes < rcd->egrbufs.size && in hfi1_setup_eagerbufs()
1902 rcd->egrbufs.alloced < rcd->egrbufs.count) { in hfi1_setup_eagerbufs()
1903 rcd->egrbufs.buffers[idx].addr = in hfi1_setup_eagerbufs()
1905 rcd->egrbufs.rcvtid_size, in hfi1_setup_eagerbufs()
1906 &rcd->egrbufs.buffers[idx].dma, in hfi1_setup_eagerbufs()
1908 if (rcd->egrbufs.buffers[idx].addr) { in hfi1_setup_eagerbufs()
1909 rcd->egrbufs.buffers[idx].len = in hfi1_setup_eagerbufs()
1910 rcd->egrbufs.rcvtid_size; in hfi1_setup_eagerbufs()
1911 rcd->egrbufs.rcvtids[rcd->egrbufs.alloced].addr = in hfi1_setup_eagerbufs()
1912 rcd->egrbufs.buffers[idx].addr; in hfi1_setup_eagerbufs()
1913 rcd->egrbufs.rcvtids[rcd->egrbufs.alloced].dma = in hfi1_setup_eagerbufs()
1914 rcd->egrbufs.buffers[idx].dma; in hfi1_setup_eagerbufs()
1915 rcd->egrbufs.alloced++; in hfi1_setup_eagerbufs()
1916 alloced_bytes += rcd->egrbufs.rcvtid_size; in hfi1_setup_eagerbufs()
1928 if (rcd->egrbufs.rcvtid_size == round_mtu || in hfi1_setup_eagerbufs()
1929 !HFI1_CAP_KGET_MASK(rcd->flags, MULTI_PKT_EGR)) { in hfi1_setup_eagerbufs()
1931 rcd->ctxt); in hfi1_setup_eagerbufs()
1936 new_size = rcd->egrbufs.rcvtid_size / 2; in hfi1_setup_eagerbufs()
1944 rcd->egrbufs.rcvtid_size = new_size; in hfi1_setup_eagerbufs()
1952 rcd->egrbufs.alloced = 0; in hfi1_setup_eagerbufs()
1954 if (i >= rcd->egrbufs.count) in hfi1_setup_eagerbufs()
1956 rcd->egrbufs.rcvtids[i].dma = in hfi1_setup_eagerbufs()
1957 rcd->egrbufs.buffers[j].dma + offset; in hfi1_setup_eagerbufs()
1958 rcd->egrbufs.rcvtids[i].addr = in hfi1_setup_eagerbufs()
1959 rcd->egrbufs.buffers[j].addr + offset; in hfi1_setup_eagerbufs()
1960 rcd->egrbufs.alloced++; in hfi1_setup_eagerbufs()
1961 if ((rcd->egrbufs.buffers[j].dma + offset + in hfi1_setup_eagerbufs()
1963 (rcd->egrbufs.buffers[j].dma + in hfi1_setup_eagerbufs()
1964 rcd->egrbufs.buffers[j].len)) { in hfi1_setup_eagerbufs()
1971 rcd->egrbufs.rcvtid_size = new_size; in hfi1_setup_eagerbufs()
1974 rcd->egrbufs.numbufs = idx; in hfi1_setup_eagerbufs()
1975 rcd->egrbufs.size = alloced_bytes; in hfi1_setup_eagerbufs()
1979 rcd->ctxt, rcd->egrbufs.alloced, in hfi1_setup_eagerbufs()
1980 rcd->egrbufs.rcvtid_size / 1024, rcd->egrbufs.size / 1024); in hfi1_setup_eagerbufs()
1987 rcd->egrbufs.threshold = in hfi1_setup_eagerbufs()
1988 rounddown_pow_of_two(rcd->egrbufs.alloced / 2); in hfi1_setup_eagerbufs()
1994 max_entries = rcd->rcv_array_groups * dd->rcv_entries.group_size; in hfi1_setup_eagerbufs()
1995 egrtop = roundup(rcd->egrbufs.alloced, dd->rcv_entries.group_size); in hfi1_setup_eagerbufs()
1996 rcd->expected_count = max_entries - egrtop; in hfi1_setup_eagerbufs()
1997 if (rcd->expected_count > MAX_TID_PAIR_ENTRIES * 2) in hfi1_setup_eagerbufs()
1998 rcd->expected_count = MAX_TID_PAIR_ENTRIES * 2; in hfi1_setup_eagerbufs()
2000 rcd->expected_base = rcd->eager_base + egrtop; in hfi1_setup_eagerbufs()
2002 rcd->ctxt, rcd->egrbufs.alloced, rcd->expected_count, in hfi1_setup_eagerbufs()
2003 rcd->eager_base, rcd->expected_base); in hfi1_setup_eagerbufs()
2005 if (!hfi1_rcvbuf_validate(rcd->egrbufs.rcvtid_size, PT_EAGER, &order)) { in hfi1_setup_eagerbufs()
2008 rcd->ctxt, rcd->egrbufs.rcvtid_size); in hfi1_setup_eagerbufs()
2013 for (idx = 0; idx < rcd->egrbufs.alloced; idx++) { in hfi1_setup_eagerbufs()
2014 hfi1_put_tid(dd, rcd->eager_base + idx, PT_EAGER, in hfi1_setup_eagerbufs()
2015 rcd->egrbufs.rcvtids[idx].dma, order); in hfi1_setup_eagerbufs()
2022 for (idx = 0; idx < rcd->egrbufs.alloced && in hfi1_setup_eagerbufs()
2023 rcd->egrbufs.buffers[idx].addr; in hfi1_setup_eagerbufs()
2026 rcd->egrbufs.buffers[idx].len, in hfi1_setup_eagerbufs()
2027 rcd->egrbufs.buffers[idx].addr, in hfi1_setup_eagerbufs()
2028 rcd->egrbufs.buffers[idx].dma); in hfi1_setup_eagerbufs()
2029 rcd->egrbufs.buffers[idx].addr = NULL; in hfi1_setup_eagerbufs()
2030 rcd->egrbufs.buffers[idx].dma = 0; in hfi1_setup_eagerbufs()
2031 rcd->egrbufs.buffers[idx].len = 0; in hfi1_setup_eagerbufs()