1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Intel MIC Platform Software Stack (MPSS)
4 *
5 * Copyright(c) 2014 Intel Corporation.
6 *
7 * Intel SCIF driver.
8 */
9 #include <linux/module.h>
10 #include <linux/idr.h>
11
12 #include <linux/mic_common.h>
13 #include "../common/mic_dev.h"
14 #include "../bus/scif_bus.h"
15 #include "scif_peer_bus.h"
16 #include "scif_main.h"
17 #include "scif_map.h"
18
19 struct scif_info scif_info = {
20 .mdev = {
21 .minor = MISC_DYNAMIC_MINOR,
22 .name = "scif",
23 .fops = &scif_fops,
24 }
25 };
26
27 struct scif_dev *scif_dev;
28 struct kmem_cache *unaligned_cache;
29 static atomic_t g_loopb_cnt;
30
31 /* Runs in the context of intr_wq */
scif_intr_bh_handler(struct work_struct * work)32 static void scif_intr_bh_handler(struct work_struct *work)
33 {
34 struct scif_dev *scifdev =
35 container_of(work, struct scif_dev, intr_bh);
36
37 if (scifdev_self(scifdev))
38 scif_loopb_msg_handler(scifdev, scifdev->qpairs);
39 else
40 scif_nodeqp_intrhandler(scifdev, scifdev->qpairs);
41 }
42
scif_setup_intr_wq(struct scif_dev * scifdev)43 int scif_setup_intr_wq(struct scif_dev *scifdev)
44 {
45 if (!scifdev->intr_wq) {
46 snprintf(scifdev->intr_wqname, sizeof(scifdev->intr_wqname),
47 "SCIF INTR %d", scifdev->node);
48 scifdev->intr_wq =
49 alloc_ordered_workqueue(scifdev->intr_wqname, 0);
50 if (!scifdev->intr_wq)
51 return -ENOMEM;
52 INIT_WORK(&scifdev->intr_bh, scif_intr_bh_handler);
53 }
54 return 0;
55 }
56
scif_destroy_intr_wq(struct scif_dev * scifdev)57 void scif_destroy_intr_wq(struct scif_dev *scifdev)
58 {
59 if (scifdev->intr_wq) {
60 destroy_workqueue(scifdev->intr_wq);
61 scifdev->intr_wq = NULL;
62 }
63 }
64
scif_intr_handler(int irq,void * data)65 irqreturn_t scif_intr_handler(int irq, void *data)
66 {
67 struct scif_dev *scifdev = data;
68 struct scif_hw_dev *sdev = scifdev->sdev;
69
70 sdev->hw_ops->ack_interrupt(sdev, scifdev->db);
71 queue_work(scifdev->intr_wq, &scifdev->intr_bh);
72 return IRQ_HANDLED;
73 }
74
scif_qp_setup_handler(struct work_struct * work)75 static void scif_qp_setup_handler(struct work_struct *work)
76 {
77 struct scif_dev *scifdev = container_of(work, struct scif_dev,
78 qp_dwork.work);
79 struct scif_hw_dev *sdev = scifdev->sdev;
80 dma_addr_t da = 0;
81 int err;
82
83 if (scif_is_mgmt_node()) {
84 struct mic_bootparam *bp = sdev->dp;
85
86 da = bp->scif_card_dma_addr;
87 scifdev->rdb = bp->h2c_scif_db;
88 } else {
89 struct mic_bootparam __iomem *bp = sdev->rdp;
90
91 da = readq(&bp->scif_host_dma_addr);
92 scifdev->rdb = ioread8(&bp->c2h_scif_db);
93 }
94 if (da) {
95 err = scif_qp_response(da, scifdev);
96 if (err)
97 dev_err(&scifdev->sdev->dev,
98 "scif_qp_response err %d\n", err);
99 } else {
100 schedule_delayed_work(&scifdev->qp_dwork,
101 msecs_to_jiffies(1000));
102 }
103 }
104
scif_setup_scifdev(void)105 static int scif_setup_scifdev(void)
106 {
107 /* We support a maximum of 129 SCIF nodes including the mgmt node */
108 #define MAX_SCIF_NODES 129
109 int i;
110 u8 num_nodes = MAX_SCIF_NODES;
111
112 scif_dev = kcalloc(num_nodes, sizeof(*scif_dev), GFP_KERNEL);
113 if (!scif_dev)
114 return -ENOMEM;
115 for (i = 0; i < num_nodes; i++) {
116 struct scif_dev *scifdev = &scif_dev[i];
117
118 scifdev->node = i;
119 scifdev->exit = OP_IDLE;
120 init_waitqueue_head(&scifdev->disconn_wq);
121 mutex_init(&scifdev->lock);
122 INIT_WORK(&scifdev->peer_add_work, scif_add_peer_device);
123 INIT_DELAYED_WORK(&scifdev->p2p_dwork,
124 scif_poll_qp_state);
125 INIT_DELAYED_WORK(&scifdev->qp_dwork,
126 scif_qp_setup_handler);
127 INIT_LIST_HEAD(&scifdev->p2p);
128 RCU_INIT_POINTER(scifdev->spdev, NULL);
129 }
130 return 0;
131 }
132
scif_destroy_scifdev(void)133 static void scif_destroy_scifdev(void)
134 {
135 kfree(scif_dev);
136 scif_dev = NULL;
137 }
138
scif_probe(struct scif_hw_dev * sdev)139 static int scif_probe(struct scif_hw_dev *sdev)
140 {
141 struct scif_dev *scifdev = &scif_dev[sdev->dnode];
142 int rc;
143
144 dev_set_drvdata(&sdev->dev, sdev);
145 scifdev->sdev = sdev;
146
147 if (1 == atomic_add_return(1, &g_loopb_cnt)) {
148 struct scif_dev *loopb_dev = &scif_dev[sdev->snode];
149
150 loopb_dev->sdev = sdev;
151 rc = scif_setup_loopback_qp(loopb_dev);
152 if (rc)
153 goto exit;
154 }
155
156 rc = scif_setup_intr_wq(scifdev);
157 if (rc)
158 goto destroy_loopb;
159 rc = scif_setup_qp(scifdev);
160 if (rc)
161 goto destroy_intr;
162 scifdev->db = sdev->hw_ops->next_db(sdev);
163 scifdev->cookie = sdev->hw_ops->request_irq(sdev, scif_intr_handler,
164 "SCIF_INTR", scifdev,
165 scifdev->db);
166 if (IS_ERR(scifdev->cookie)) {
167 rc = PTR_ERR(scifdev->cookie);
168 goto free_qp;
169 }
170 if (scif_is_mgmt_node()) {
171 struct mic_bootparam *bp = sdev->dp;
172
173 bp->c2h_scif_db = scifdev->db;
174 bp->scif_host_dma_addr = scifdev->qp_dma_addr;
175 } else {
176 struct mic_bootparam __iomem *bp = sdev->rdp;
177
178 iowrite8(scifdev->db, &bp->h2c_scif_db);
179 writeq(scifdev->qp_dma_addr, &bp->scif_card_dma_addr);
180 }
181 schedule_delayed_work(&scifdev->qp_dwork,
182 msecs_to_jiffies(1000));
183 return rc;
184 free_qp:
185 scif_free_qp(scifdev);
186 destroy_intr:
187 scif_destroy_intr_wq(scifdev);
188 destroy_loopb:
189 if (atomic_dec_and_test(&g_loopb_cnt))
190 scif_destroy_loopback_qp(&scif_dev[sdev->snode]);
191 exit:
192 return rc;
193 }
194
scif_stop(struct scif_dev * scifdev)195 void scif_stop(struct scif_dev *scifdev)
196 {
197 struct scif_dev *dev;
198 int i;
199
200 for (i = scif_info.maxid; i >= 0; i--) {
201 dev = &scif_dev[i];
202 if (scifdev_self(dev))
203 continue;
204 scif_handle_remove_node(i);
205 }
206 }
207
scif_remove(struct scif_hw_dev * sdev)208 static void scif_remove(struct scif_hw_dev *sdev)
209 {
210 struct scif_dev *scifdev = &scif_dev[sdev->dnode];
211
212 if (scif_is_mgmt_node()) {
213 struct mic_bootparam *bp = sdev->dp;
214
215 bp->c2h_scif_db = -1;
216 bp->scif_host_dma_addr = 0x0;
217 } else {
218 struct mic_bootparam __iomem *bp = sdev->rdp;
219
220 iowrite8(-1, &bp->h2c_scif_db);
221 writeq(0x0, &bp->scif_card_dma_addr);
222 }
223 if (scif_is_mgmt_node()) {
224 scif_disconnect_node(scifdev->node, true);
225 } else {
226 scif_info.card_initiated_exit = true;
227 scif_stop(scifdev);
228 }
229 if (atomic_dec_and_test(&g_loopb_cnt))
230 scif_destroy_loopback_qp(&scif_dev[sdev->snode]);
231 if (scifdev->cookie) {
232 sdev->hw_ops->free_irq(sdev, scifdev->cookie, scifdev);
233 scifdev->cookie = NULL;
234 }
235 scif_destroy_intr_wq(scifdev);
236 cancel_delayed_work(&scifdev->qp_dwork);
237 scif_free_qp(scifdev);
238 scifdev->rdb = -1;
239 scifdev->sdev = NULL;
240 }
241
242 static struct scif_hw_dev_id id_table[] = {
243 { MIC_SCIF_DEV, SCIF_DEV_ANY_ID },
244 { 0 },
245 };
246
247 static struct scif_driver scif_driver = {
248 .driver.name = KBUILD_MODNAME,
249 .driver.owner = THIS_MODULE,
250 .id_table = id_table,
251 .probe = scif_probe,
252 .remove = scif_remove,
253 };
254
_scif_init(void)255 static int _scif_init(void)
256 {
257 int rc;
258
259 mutex_init(&scif_info.eplock);
260 spin_lock_init(&scif_info.rmalock);
261 spin_lock_init(&scif_info.nb_connect_lock);
262 spin_lock_init(&scif_info.port_lock);
263 mutex_init(&scif_info.conflock);
264 mutex_init(&scif_info.connlock);
265 mutex_init(&scif_info.fencelock);
266 INIT_LIST_HEAD(&scif_info.uaccept);
267 INIT_LIST_HEAD(&scif_info.listen);
268 INIT_LIST_HEAD(&scif_info.zombie);
269 INIT_LIST_HEAD(&scif_info.connected);
270 INIT_LIST_HEAD(&scif_info.disconnected);
271 INIT_LIST_HEAD(&scif_info.rma);
272 INIT_LIST_HEAD(&scif_info.rma_tc);
273 INIT_LIST_HEAD(&scif_info.mmu_notif_cleanup);
274 INIT_LIST_HEAD(&scif_info.fence);
275 INIT_LIST_HEAD(&scif_info.nb_connect_list);
276 init_waitqueue_head(&scif_info.exitwq);
277 scif_info.rma_tc_limit = SCIF_RMA_TEMP_CACHE_LIMIT;
278 scif_info.en_msg_log = 0;
279 scif_info.p2p_enable = 1;
280 rc = scif_setup_scifdev();
281 if (rc)
282 goto error;
283 unaligned_cache = kmem_cache_create("Unaligned_DMA",
284 SCIF_KMEM_UNALIGNED_BUF_SIZE,
285 0, SLAB_HWCACHE_ALIGN, NULL);
286 if (!unaligned_cache) {
287 rc = -ENOMEM;
288 goto free_sdev;
289 }
290 INIT_WORK(&scif_info.misc_work, scif_misc_handler);
291 INIT_WORK(&scif_info.mmu_notif_work, scif_mmu_notif_handler);
292 INIT_WORK(&scif_info.conn_work, scif_conn_handler);
293 idr_init(&scif_ports);
294 return 0;
295 free_sdev:
296 scif_destroy_scifdev();
297 error:
298 return rc;
299 }
300
_scif_exit(void)301 static void _scif_exit(void)
302 {
303 idr_destroy(&scif_ports);
304 kmem_cache_destroy(unaligned_cache);
305 scif_destroy_scifdev();
306 }
307
scif_init(void)308 static int __init scif_init(void)
309 {
310 struct miscdevice *mdev = &scif_info.mdev;
311 int rc;
312
313 _scif_init();
314 iova_cache_get();
315 rc = scif_peer_bus_init();
316 if (rc)
317 goto exit;
318 rc = scif_register_driver(&scif_driver);
319 if (rc)
320 goto peer_bus_exit;
321 rc = misc_register(mdev);
322 if (rc)
323 goto unreg_scif;
324 scif_init_debugfs();
325 return 0;
326 unreg_scif:
327 scif_unregister_driver(&scif_driver);
328 peer_bus_exit:
329 scif_peer_bus_exit();
330 exit:
331 _scif_exit();
332 return rc;
333 }
334
scif_exit(void)335 static void __exit scif_exit(void)
336 {
337 scif_exit_debugfs();
338 misc_deregister(&scif_info.mdev);
339 scif_unregister_driver(&scif_driver);
340 scif_peer_bus_exit();
341 iova_cache_put();
342 _scif_exit();
343 }
344
345 module_init(scif_init);
346 module_exit(scif_exit);
347
348 MODULE_DEVICE_TABLE(scif, id_table);
349 MODULE_AUTHOR("Intel Corporation");
350 MODULE_DESCRIPTION("Intel(R) SCIF driver");
351 MODULE_LICENSE("GPL v2");
352