97 	void (*tx_handler) (struct ntb_transport_qp *qp, void *qp_data,
106 	void (*rx_handler) (struct ntb_transport_qp *qp, void *qp_data,
186 #define QP_TO_MW(qp)		((qp) % NTB_NUM_MW)  argument
381 	struct ntb_transport_qp *qp;  in debugfs_read()  local
391 	qp = filp->private_data;  in debugfs_read()
396 			       "rx_bytes - \t%llu\n", qp->rx_bytes);  in debugfs_read()
398 			       "rx_pkts - \t%llu\n", qp->rx_pkts);  in debugfs_read()
400 			       "rx_ring_empty - %llu\n", qp->rx_ring_empty);  in debugfs_read()
402 			       "rx_err_no_buf - %llu\n", qp->rx_err_no_buf);  in debugfs_read()
404 			       "rx_err_oflow - \t%llu\n", qp->rx_err_oflow);  in debugfs_read()
406 			       "rx_err_ver - \t%llu\n", qp->rx_err_ver);  in debugfs_read()
408 			       "rx_buff - \t%p\n", qp->rx_buff);  in debugfs_read()
410 			       "rx_index - \t%u\n", qp->rx_index);  in debugfs_read()
412 			       "rx_max_entry - \t%u\n", qp->rx_max_entry);  in debugfs_read()
415 			       "tx_bytes - \t%llu\n", qp->tx_bytes);  in debugfs_read()
417 			       "tx_pkts - \t%llu\n", qp->tx_pkts);  in debugfs_read()
419 			       "tx_ring_full - \t%llu\n", qp->tx_ring_full);  in debugfs_read()
421 			       "tx_mw - \t%p\n", qp->tx_mw);  in debugfs_read()
423 			       "tx_index - \t%u\n", qp->tx_index);  in debugfs_read()
425 			       "tx_max_entry - \t%u\n", qp->tx_max_entry);  in debugfs_read()
428 			       "\nQP Link %s\n", (qp->qp_link == NTB_LINK_UP) ?  in debugfs_read()
476 	struct ntb_transport_qp *qp = &nt->qps[qp_num];  in ntb_transport_setup_qp_mw()  local
489 	qp->remote_rx_info = nt->mw[mw_num].virt_addr +  in ntb_transport_setup_qp_mw()
493 	qp->rx_buff = qp->remote_rx_info + 1;  in ntb_transport_setup_qp_mw()
495 	qp->rx_max_frame = min(transport_mtu, rx_size / 2);  in ntb_transport_setup_qp_mw()
496 	qp->rx_max_entry = rx_size / qp->rx_max_frame;  in ntb_transport_setup_qp_mw()
497 	qp->rx_index = 0;  in ntb_transport_setup_qp_mw()
499 	qp->remote_rx_info->entry = qp->rx_max_entry - 1;  in ntb_transport_setup_qp_mw()
502 	for (i = 0; i < qp->rx_max_entry; i++) {  in ntb_transport_setup_qp_mw()
503 		void *offset = qp->rx_buff + qp->rx_max_frame * (i + 1) -  in ntb_transport_setup_qp_mw()
508 	qp->rx_pkts = 0;  in ntb_transport_setup_qp_mw()
509 	qp->tx_pkts = 0;  in ntb_transport_setup_qp_mw()
510 	qp->tx_index = 0;  in ntb_transport_setup_qp_mw()
557 	struct ntb_transport_qp *qp = container_of(work,  in ntb_qp_link_cleanup()  local
560 	struct ntb_transport *nt = qp->transport;  in ntb_qp_link_cleanup()
563 	if (qp->qp_link == NTB_LINK_DOWN) {  in ntb_qp_link_cleanup()
564 		cancel_delayed_work_sync(&qp->link_work);  in ntb_qp_link_cleanup()
568 	if (qp->event_handler)  in ntb_qp_link_cleanup()
569 		qp->event_handler(qp->cb_data, NTB_LINK_DOWN);  in ntb_qp_link_cleanup()
571 	dev_info(&pdev->dev, "qp %d: Link Down\n", qp->qp_num);  in ntb_qp_link_cleanup()
572 	qp->qp_link = NTB_LINK_DOWN;  in ntb_qp_link_cleanup()
575 		schedule_delayed_work(&qp->link_work,  in ntb_qp_link_cleanup()
579 static void ntb_qp_link_down(struct ntb_transport_qp *qp)  in ntb_qp_link_down()  argument
581 	schedule_work(&qp->link_cleanup);  in ntb_qp_link_down()
737 		struct ntb_transport_qp *qp = &nt->qps[i];  in ntb_transport_link_work()  local
741 		if (qp->client_ready == NTB_LINK_UP)  in ntb_transport_link_work()
742 			schedule_delayed_work(&qp->link_work, 0);  in ntb_transport_link_work()
758 	struct ntb_transport_qp *qp = container_of(work,  in ntb_qp_link_work()  local
761 	struct pci_dev *pdev = ntb_query_pdev(qp->ndev);  in ntb_qp_link_work()
762 	struct ntb_transport *nt = qp->transport;  in ntb_qp_link_work()
773 	rc = ntb_write_remote_spad(nt->ndev, QP_LINKS, val | 1 << qp->qp_num);  in ntb_qp_link_work()
776 			val | 1 << qp->qp_num, QP_LINKS);  in ntb_qp_link_work()
786 	if (1 << qp->qp_num & val) {  in ntb_qp_link_work()
787 		qp->qp_link = NTB_LINK_UP;  in ntb_qp_link_work()
789 		dev_info(&pdev->dev, "qp %d: Link Up\n", qp->qp_num);  in ntb_qp_link_work()
790 		if (qp->event_handler)  in ntb_qp_link_work()
791 			qp->event_handler(qp->cb_data, NTB_LINK_UP);  in ntb_qp_link_work()
793 		schedule_delayed_work(&qp->link_work,  in ntb_qp_link_work()
800 	struct ntb_transport_qp *qp;  in ntb_transport_init_queue()  local
804 	qp = &nt->qps[qp_num];  in ntb_transport_init_queue()
805 	qp->qp_num = qp_num;  in ntb_transport_init_queue()
806 	qp->transport = nt;  in ntb_transport_init_queue()
807 	qp->ndev = nt->ndev;  in ntb_transport_init_queue()
808 	qp->qp_link = NTB_LINK_DOWN;  in ntb_transport_init_queue()
809 	qp->client_ready = NTB_LINK_DOWN;  in ntb_transport_init_queue()
810 	qp->event_handler = NULL;  in ntb_transport_init_queue()
817 	tx_size = (unsigned int) ntb_get_mw_size(qp->ndev, mw_num) / num_qps_mw;  in ntb_transport_init_queue()
818 	qp->rx_info = ntb_get_mw_vbase(nt->ndev, mw_num) +  in ntb_transport_init_queue()
822 	qp->tx_mw = qp->rx_info + 1;  in ntb_transport_init_queue()
824 	qp->tx_max_frame = min(transport_mtu, tx_size / 2);  in ntb_transport_init_queue()
825 	qp->tx_max_entry = tx_size / qp->tx_max_frame;  in ntb_transport_init_queue()
831 		qp->debugfs_dir = debugfs_create_dir(debugfs_name,  in ntb_transport_init_queue()
834 		qp->debugfs_stats = debugfs_create_file("stats", S_IRUSR,  in ntb_transport_init_queue()
835 							qp->debugfs_dir, qp,  in ntb_transport_init_queue()
839 	INIT_DELAYED_WORK(&qp->link_work, ntb_qp_link_work);  in ntb_transport_init_queue()
840 	INIT_WORK(&qp->link_cleanup, ntb_qp_link_cleanup);  in ntb_transport_init_queue()
842 	spin_lock_init(&qp->ntb_rx_pend_q_lock);  in ntb_transport_init_queue()
843 	spin_lock_init(&qp->ntb_rx_free_q_lock);  in ntb_transport_init_queue()
844 	spin_lock_init(&qp->ntb_tx_free_q_lock);  in ntb_transport_init_queue()
846 	INIT_LIST_HEAD(&qp->rx_pend_q);  in ntb_transport_init_queue()
847 	INIT_LIST_HEAD(&qp->rx_free_q);  in ntb_transport_init_queue()
848 	INIT_LIST_HEAD(&qp->tx_free_q);  in ntb_transport_init_queue()
946 static void ntb_rx_copy_task(struct ntb_transport_qp *qp,  in ntb_rx_copy_task()  argument
954 	ntb_list_add(&qp->ntb_rx_free_q_lock, &entry->entry, &qp->rx_free_q);  in ntb_rx_copy_task()
956 	if (qp->rx_handler && qp->client_ready == NTB_LINK_UP)  in ntb_rx_copy_task()
957 		qp->rx_handler(qp, qp->cb_data, cb_data, len);  in ntb_rx_copy_task()
960 static int ntb_process_rxc(struct ntb_transport_qp *qp)  in ntb_process_rxc()  argument
966 	offset = qp->rx_buff + qp->rx_max_frame * qp->rx_index;  in ntb_process_rxc()
967 	hdr = offset + qp->rx_max_frame - sizeof(struct ntb_payload_header);  in ntb_process_rxc()
969 	entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q);  in ntb_process_rxc()
971 		dev_dbg(&ntb_query_pdev(qp->ndev)->dev,  in ntb_process_rxc()
974 		qp->rx_err_no_buf++;  in ntb_process_rxc()
979 		ntb_list_add(&qp->ntb_rx_pend_q_lock, &entry->entry,  in ntb_process_rxc()
980 			     &qp->rx_pend_q);  in ntb_process_rxc()
981 		qp->rx_ring_empty++;  in ntb_process_rxc()
985 	if (hdr->ver != (u32) qp->rx_pkts) {  in ntb_process_rxc()
986 		dev_dbg(&ntb_query_pdev(qp->ndev)->dev,  in ntb_process_rxc()
988 			qp->qp_num, qp->rx_pkts, hdr->ver);  in ntb_process_rxc()
989 		ntb_list_add(&qp->ntb_rx_pend_q_lock, &entry->entry,  in ntb_process_rxc()
990 			     &qp->rx_pend_q);  in ntb_process_rxc()
991 		qp->rx_err_ver++;  in ntb_process_rxc()
996 		ntb_qp_link_down(qp);  in ntb_process_rxc()
998 		ntb_list_add(&qp->ntb_rx_pend_q_lock, &entry->entry,  in ntb_process_rxc()
999 			     &qp->rx_pend_q);  in ntb_process_rxc()
1003 	dev_dbg(&ntb_query_pdev(qp->ndev)->dev,  in ntb_process_rxc()
1005 		qp->rx_index, hdr->ver, hdr->len, entry->len);  in ntb_process_rxc()
1009 		ntb_rx_copy_task(qp, entry, offset);  in ntb_process_rxc()
1011 		ntb_list_add(&qp->ntb_rx_pend_q_lock, &entry->entry,  in ntb_process_rxc()
1012 			     &qp->rx_pend_q);  in ntb_process_rxc()
1014 		qp->rx_err_oflow++;  in ntb_process_rxc()
1015 		dev_dbg(&ntb_query_pdev(qp->ndev)->dev,  in ntb_process_rxc()
1020 	qp->rx_bytes += hdr->len;  in ntb_process_rxc()
1021 	qp->rx_pkts++;  in ntb_process_rxc()
1027 	iowrite32(qp->rx_index, &qp->rx_info->entry);  in ntb_process_rxc()
1029 	qp->rx_index++;  in ntb_process_rxc()
1030 	qp->rx_index %= qp->rx_max_entry;  in ntb_process_rxc()
1037 	struct ntb_transport_qp *qp = (struct ntb_transport_qp *)data;  in ntb_transport_rx()  local
1043 	for (i = 0; i < qp->rx_max_entry; i++) {  in ntb_transport_rx()
1044 		rc = ntb_process_rxc(qp);  in ntb_transport_rx()
1052 	struct ntb_transport_qp *qp = data;  in ntb_transport_rxc_db()  local
1054 	dev_dbg(&ntb_query_pdev(qp->ndev)->dev, "%s: doorbell %d received\n",  in ntb_transport_rxc_db()
1057 	tasklet_schedule(&qp->rx_work);  in ntb_transport_rxc_db()
1060 static void ntb_tx_copy_task(struct ntb_transport_qp *qp,  in ntb_tx_copy_task()  argument
1068 	hdr = offset + qp->tx_max_frame - sizeof(struct ntb_payload_header);  in ntb_tx_copy_task()
1070 	iowrite32((u32) qp->tx_pkts, &hdr->ver);  in ntb_tx_copy_task()
1076 	ntb_ring_sdb(qp->ndev, qp->qp_num);  in ntb_tx_copy_task()
1083 		qp->tx_bytes += entry->len;  in ntb_tx_copy_task()
1085 		if (qp->tx_handler)  in ntb_tx_copy_task()
1086 			qp->tx_handler(qp, qp->cb_data, entry->cb_data,  in ntb_tx_copy_task()
1090 	ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, &qp->tx_free_q);  in ntb_tx_copy_task()
1093 static int ntb_process_tx(struct ntb_transport_qp *qp,  in ntb_process_tx()  argument
1098 	offset = qp->tx_mw + qp->tx_max_frame * qp->tx_index;  in ntb_process_tx()
1100 	dev_dbg(&ntb_query_pdev(qp->ndev)->dev, "%lld - offset %p, tx %u, entry len %d flags %x buff %p\n",  in ntb_process_tx()
1101 		qp->tx_pkts, offset, qp->tx_index, entry->len, entry->flags,  in ntb_process_tx()
1103 	if (qp->tx_index == qp->remote_rx_info->entry) {  in ntb_process_tx()
1104 		qp->tx_ring_full++;  in ntb_process_tx()
1108 	if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) {  in ntb_process_tx()
1109 		if (qp->tx_handler)  in ntb_process_tx()
1110 			qp->tx_handler(qp->cb_data, qp, NULL, -EIO);  in ntb_process_tx()
1112 		ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry,  in ntb_process_tx()
1113 			     &qp->tx_free_q);  in ntb_process_tx()
1117 	ntb_tx_copy_task(qp, entry, offset);  in ntb_process_tx()
1119 	qp->tx_index++;  in ntb_process_tx()
1120 	qp->tx_index %= qp->tx_max_entry;  in ntb_process_tx()
1122 	qp->tx_pkts++;  in ntb_process_tx()
1127 static void ntb_send_link_down(struct ntb_transport_qp *qp)  in ntb_send_link_down()  argument
1129 	struct pci_dev *pdev = ntb_query_pdev(qp->ndev);  in ntb_send_link_down()
1133 	if (qp->qp_link == NTB_LINK_DOWN)  in ntb_send_link_down()
1136 	qp->qp_link = NTB_LINK_DOWN;  in ntb_send_link_down()
1137 	dev_info(&pdev->dev, "qp %d: Link Down\n", qp->qp_num);  in ntb_send_link_down()
1140 		entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q);  in ntb_send_link_down()
1154 	rc = ntb_process_tx(qp, entry);  in ntb_send_link_down()
1157 			qp->qp_num);  in ntb_send_link_down()
1179 	struct ntb_transport_qp *qp;  in ntb_transport_create_queue()  local
1197 	qp = &nt->qps[free_queue];  in ntb_transport_create_queue()
1198 	qp->cb_data = data;  in ntb_transport_create_queue()
1199 	qp->rx_handler = handlers->rx_handler;  in ntb_transport_create_queue()
1200 	qp->tx_handler = handlers->tx_handler;  in ntb_transport_create_queue()
1201 	qp->event_handler = handlers->event_handler;  in ntb_transport_create_queue()
1208 		ntb_list_add(&qp->ntb_rx_free_q_lock, &entry->entry,  in ntb_transport_create_queue()
1209 			     &qp->rx_free_q);  in ntb_transport_create_queue()
1217 		ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry,  in ntb_transport_create_queue()
1218 			     &qp->tx_free_q);  in ntb_transport_create_queue()
1221 	tasklet_init(&qp->rx_work, ntb_transport_rx, (unsigned long) qp);  in ntb_transport_create_queue()
1223 	rc = ntb_register_db_callback(qp->ndev, free_queue, qp,  in ntb_transport_create_queue()
1228 	dev_info(&pdev->dev, "NTB Transport QP %d created\n", qp->qp_num);  in ntb_transport_create_queue()
1230 	return qp;  in ntb_transport_create_queue()
1233 	tasklet_disable(&qp->rx_work);  in ntb_transport_create_queue()
1235 	while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q)))  in ntb_transport_create_queue()
1238 	while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q)))  in ntb_transport_create_queue()
1252 void ntb_transport_free_queue(struct ntb_transport_qp *qp)  in ntb_transport_free_queue()  argument
1257 	if (!qp)  in ntb_transport_free_queue()
1260 	pdev = ntb_query_pdev(qp->ndev);  in ntb_transport_free_queue()
1262 	cancel_delayed_work_sync(&qp->link_work);  in ntb_transport_free_queue()
1264 	ntb_unregister_db_callback(qp->ndev, qp->qp_num);  in ntb_transport_free_queue()
1265 	tasklet_disable(&qp->rx_work);  in ntb_transport_free_queue()
1267 	while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q)))  in ntb_transport_free_queue()
1270 	while ((entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q))) {  in ntb_transport_free_queue()
1275 	while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q)))  in ntb_transport_free_queue()
1278 	set_bit(qp->qp_num, &qp->transport->qp_bitmap);  in ntb_transport_free_queue()
1280 	dev_info(&pdev->dev, "NTB Transport QP %d freed\n", qp->qp_num);  in ntb_transport_free_queue()
1294 void *ntb_transport_rx_remove(struct ntb_transport_qp *qp, unsigned int *len)  in ntb_transport_rx_remove()  argument
1299 	if (!qp || qp->client_ready == NTB_LINK_UP)  in ntb_transport_rx_remove()
1302 	entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q);  in ntb_transport_rx_remove()
1309 	ntb_list_add(&qp->ntb_rx_free_q_lock, &entry->entry, &qp->rx_free_q);  in ntb_transport_rx_remove()
1327 int ntb_transport_rx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data,  in ntb_transport_rx_enqueue()  argument
1332 	if (!qp)  in ntb_transport_rx_enqueue()
1335 	entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q);  in ntb_transport_rx_enqueue()
1343 	ntb_list_add(&qp->ntb_rx_pend_q_lock, &entry->entry, &qp->rx_pend_q);  in ntb_transport_rx_enqueue()
1362 int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data,  in ntb_transport_tx_enqueue()  argument
1368 	if (!qp || qp->qp_link != NTB_LINK_UP || !len)  in ntb_transport_tx_enqueue()
1371 	entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q);  in ntb_transport_tx_enqueue()
1380 	rc = ntb_process_tx(qp, entry);  in ntb_transport_tx_enqueue()
1382 		ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry,  in ntb_transport_tx_enqueue()
1383 			     &qp->tx_free_q);  in ntb_transport_tx_enqueue()
1395 void ntb_transport_link_up(struct ntb_transport_qp *qp)  in ntb_transport_link_up()  argument
1397 	if (!qp)  in ntb_transport_link_up()
1400 	qp->client_ready = NTB_LINK_UP;  in ntb_transport_link_up()
1402 	if (qp->transport->transport_link == NTB_LINK_UP)  in ntb_transport_link_up()
1403 		schedule_delayed_work(&qp->link_work, 0);  in ntb_transport_link_up()
1415 void ntb_transport_link_down(struct ntb_transport_qp *qp)  in ntb_transport_link_down()  argument
1420 	if (!qp)  in ntb_transport_link_down()
1423 	pdev = ntb_query_pdev(qp->ndev);  in ntb_transport_link_down()
1424 	qp->client_ready = NTB_LINK_DOWN;  in ntb_transport_link_down()
1426 	rc = ntb_read_local_spad(qp->ndev, QP_LINKS, &val);  in ntb_transport_link_down()
1432 	rc = ntb_write_remote_spad(qp->ndev, QP_LINKS,  in ntb_transport_link_down()
1433 				   val & ~(1 << qp->qp_num));  in ntb_transport_link_down()
1436 			val & ~(1 << qp->qp_num), QP_LINKS);  in ntb_transport_link_down()
1438 	if (qp->qp_link == NTB_LINK_UP)  in ntb_transport_link_down()
1439 		ntb_send_link_down(qp);  in ntb_transport_link_down()
1441 		cancel_delayed_work_sync(&qp->link_work);  in ntb_transport_link_down()
1453 bool ntb_transport_link_query(struct ntb_transport_qp *qp)  in ntb_transport_link_query()  argument
1455 	if (!qp)  in ntb_transport_link_query()
1458 	return qp->qp_link == NTB_LINK_UP;  in ntb_transport_link_query()
1470 unsigned char ntb_transport_qp_num(struct ntb_transport_qp *qp)  in ntb_transport_qp_num()  argument
1472 	if (!qp)  in ntb_transport_qp_num()
1475 	return qp->qp_num;  in ntb_transport_qp_num()
1487 unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp)  in ntb_transport_max_size()  argument
1489 	if (!qp)  in ntb_transport_max_size()
1492 	return qp->tx_max_frame - sizeof(struct ntb_payload_header);  in ntb_transport_max_size()