1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
3 #ifndef _IDXD_H_
4 #define _IDXD_H_
5
6 #include <linux/sbitmap.h>
7 #include <linux/dmaengine.h>
8 #include <linux/percpu-rwsem.h>
9 #include <linux/wait.h>
10 #include <linux/cdev.h>
11 #include "registers.h"
12
13 #define IDXD_DRIVER_VERSION "1.00"
14
15 extern struct kmem_cache *idxd_desc_pool;
16
17 struct idxd_device;
18 struct idxd_wq;
19
20 #define IDXD_REG_TIMEOUT 50
21 #define IDXD_DRAIN_TIMEOUT 5000
22
23 enum idxd_type {
24 IDXD_TYPE_UNKNOWN = -1,
25 IDXD_TYPE_DSA = 0,
26 IDXD_TYPE_MAX
27 };
28
29 #define IDXD_NAME_SIZE 128
30
31 struct idxd_device_driver {
32 struct device_driver drv;
33 };
34
35 struct idxd_irq_entry {
36 struct idxd_device *idxd;
37 int id;
38 struct llist_head pending_llist;
39 struct list_head work_list;
40 };
41
42 struct idxd_group {
43 struct device conf_dev;
44 struct idxd_device *idxd;
45 struct grpcfg grpcfg;
46 int id;
47 int num_engines;
48 int num_wqs;
49 bool use_token_limit;
50 u8 tokens_allowed;
51 u8 tokens_reserved;
52 int tc_a;
53 int tc_b;
54 };
55
56 #define IDXD_MAX_PRIORITY 0xf
57
58 enum idxd_wq_state {
59 IDXD_WQ_DISABLED = 0,
60 IDXD_WQ_ENABLED,
61 };
62
63 enum idxd_wq_flag {
64 WQ_FLAG_DEDICATED = 0,
65 };
66
67 enum idxd_wq_type {
68 IDXD_WQT_NONE = 0,
69 IDXD_WQT_KERNEL,
70 IDXD_WQT_USER,
71 };
72
73 struct idxd_cdev {
74 struct idxd_wq *wq;
75 struct cdev cdev;
76 struct device dev;
77 int minor;
78 };
79
80 #define IDXD_ALLOCATED_BATCH_SIZE 128U
81 #define WQ_NAME_SIZE 1024
82 #define WQ_TYPE_SIZE 10
83
84 enum idxd_op_type {
85 IDXD_OP_BLOCK = 0,
86 IDXD_OP_NONBLOCK = 1,
87 };
88
89 enum idxd_complete_type {
90 IDXD_COMPLETE_NORMAL = 0,
91 IDXD_COMPLETE_ABORT,
92 };
93
94 struct idxd_dma_chan {
95 struct dma_chan chan;
96 struct idxd_wq *wq;
97 };
98
99 struct idxd_wq {
100 void __iomem *dportal;
101 struct device conf_dev;
102 struct idxd_cdev *idxd_cdev;
103 struct wait_queue_head err_queue;
104 struct idxd_device *idxd;
105 int id;
106 enum idxd_wq_type type;
107 struct idxd_group *group;
108 int client_count;
109 struct mutex wq_lock; /* mutex for workqueue */
110 u32 size;
111 u32 threshold;
112 u32 priority;
113 enum idxd_wq_state state;
114 unsigned long flags;
115 union wqcfg *wqcfg;
116 u32 vec_ptr; /* interrupt steering */
117 struct dsa_hw_desc **hw_descs;
118 int num_descs;
119 struct dsa_completion_record *compls;
120 dma_addr_t compls_addr;
121 int compls_size;
122 struct idxd_desc **descs;
123 struct sbitmap_queue sbq;
124 struct idxd_dma_chan *idxd_chan;
125 char name[WQ_NAME_SIZE + 1];
126 u64 max_xfer_bytes;
127 u32 max_batch_size;
128 };
129
130 struct idxd_engine {
131 struct device conf_dev;
132 int id;
133 struct idxd_group *group;
134 struct idxd_device *idxd;
135 };
136
137 /* shadow registers */
138 struct idxd_hw {
139 u32 version;
140 union gen_cap_reg gen_cap;
141 union wq_cap_reg wq_cap;
142 union group_cap_reg group_cap;
143 union engine_cap_reg engine_cap;
144 struct opcap opcap;
145 };
146
147 enum idxd_device_state {
148 IDXD_DEV_HALTED = -1,
149 IDXD_DEV_DISABLED = 0,
150 IDXD_DEV_CONF_READY,
151 IDXD_DEV_ENABLED,
152 };
153
154 enum idxd_device_flag {
155 IDXD_FLAG_CONFIGURABLE = 0,
156 IDXD_FLAG_CMD_RUNNING,
157 };
158
159 struct idxd_dma_dev {
160 struct idxd_device *idxd;
161 struct dma_device dma;
162 };
163
164 struct idxd_device {
165 enum idxd_type type;
166 struct device conf_dev;
167 struct list_head list;
168 struct idxd_hw hw;
169 enum idxd_device_state state;
170 unsigned long flags;
171 int id;
172 int major;
173 u8 cmd_status;
174
175 struct pci_dev *pdev;
176 void __iomem *reg_base;
177
178 spinlock_t dev_lock; /* spinlock for device */
179 struct completion *cmd_done;
180 struct idxd_group *groups;
181 struct idxd_wq *wqs;
182 struct idxd_engine *engines;
183
184 int num_groups;
185
186 u32 msix_perm_offset;
187 u32 wqcfg_offset;
188 u32 grpcfg_offset;
189 u32 perfmon_offset;
190
191 u64 max_xfer_bytes;
192 u32 max_batch_size;
193 int max_groups;
194 int max_engines;
195 int max_tokens;
196 int max_wqs;
197 int max_wq_size;
198 int token_limit;
199 int nr_tokens; /* non-reserved tokens */
200 unsigned int wqcfg_size;
201
202 union sw_err_reg sw_err;
203 wait_queue_head_t cmd_waitq;
204 struct msix_entry *msix_entries;
205 int num_wq_irqs;
206 struct idxd_irq_entry *irq_entries;
207
208 struct idxd_dma_dev *idxd_dma;
209 struct workqueue_struct *wq;
210 struct work_struct work;
211 };
212
213 /* IDXD software descriptor */
214 struct idxd_desc {
215 struct dsa_hw_desc *hw;
216 dma_addr_t desc_dma;
217 struct dsa_completion_record *completion;
218 dma_addr_t compl_dma;
219 struct dma_async_tx_descriptor txd;
220 struct llist_node llnode;
221 struct list_head list;
222 int id;
223 int cpu;
224 struct idxd_wq *wq;
225 };
226
227 #define confdev_to_idxd(dev) container_of(dev, struct idxd_device, conf_dev)
228 #define confdev_to_wq(dev) container_of(dev, struct idxd_wq, conf_dev)
229
230 extern struct bus_type dsa_bus_type;
231
wq_dedicated(struct idxd_wq * wq)232 static inline bool wq_dedicated(struct idxd_wq *wq)
233 {
234 return test_bit(WQ_FLAG_DEDICATED, &wq->flags);
235 }
236
237 enum idxd_portal_prot {
238 IDXD_PORTAL_UNLIMITED = 0,
239 IDXD_PORTAL_LIMITED,
240 };
241
idxd_get_wq_portal_offset(enum idxd_portal_prot prot)242 static inline int idxd_get_wq_portal_offset(enum idxd_portal_prot prot)
243 {
244 return prot * 0x1000;
245 }
246
idxd_get_wq_portal_full_offset(int wq_id,enum idxd_portal_prot prot)247 static inline int idxd_get_wq_portal_full_offset(int wq_id,
248 enum idxd_portal_prot prot)
249 {
250 return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot);
251 }
252
idxd_set_type(struct idxd_device * idxd)253 static inline void idxd_set_type(struct idxd_device *idxd)
254 {
255 struct pci_dev *pdev = idxd->pdev;
256
257 if (pdev->device == PCI_DEVICE_ID_INTEL_DSA_SPR0)
258 idxd->type = IDXD_TYPE_DSA;
259 else
260 idxd->type = IDXD_TYPE_UNKNOWN;
261 }
262
idxd_wq_get(struct idxd_wq * wq)263 static inline void idxd_wq_get(struct idxd_wq *wq)
264 {
265 wq->client_count++;
266 }
267
idxd_wq_put(struct idxd_wq * wq)268 static inline void idxd_wq_put(struct idxd_wq *wq)
269 {
270 wq->client_count--;
271 }
272
idxd_wq_refcount(struct idxd_wq * wq)273 static inline int idxd_wq_refcount(struct idxd_wq *wq)
274 {
275 return wq->client_count;
276 };
277
278 const char *idxd_get_dev_name(struct idxd_device *idxd);
279 int idxd_register_bus_type(void);
280 void idxd_unregister_bus_type(void);
281 int idxd_setup_sysfs(struct idxd_device *idxd);
282 void idxd_cleanup_sysfs(struct idxd_device *idxd);
283 int idxd_register_driver(void);
284 void idxd_unregister_driver(void);
285 struct bus_type *idxd_get_bus_type(struct idxd_device *idxd);
286
287 /* device interrupt control */
288 irqreturn_t idxd_irq_handler(int vec, void *data);
289 irqreturn_t idxd_misc_thread(int vec, void *data);
290 irqreturn_t idxd_wq_thread(int irq, void *data);
291 void idxd_mask_error_interrupts(struct idxd_device *idxd);
292 void idxd_unmask_error_interrupts(struct idxd_device *idxd);
293 void idxd_mask_msix_vectors(struct idxd_device *idxd);
294 void idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id);
295 void idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id);
296
297 /* device control */
298 int idxd_device_init_reset(struct idxd_device *idxd);
299 int idxd_device_enable(struct idxd_device *idxd);
300 int idxd_device_disable(struct idxd_device *idxd);
301 void idxd_device_reset(struct idxd_device *idxd);
302 void idxd_device_cleanup(struct idxd_device *idxd);
303 int idxd_device_config(struct idxd_device *idxd);
304 void idxd_device_wqs_clear_state(struct idxd_device *idxd);
305
306 /* work queue control */
307 int idxd_wq_alloc_resources(struct idxd_wq *wq);
308 void idxd_wq_free_resources(struct idxd_wq *wq);
309 int idxd_wq_enable(struct idxd_wq *wq);
310 int idxd_wq_disable(struct idxd_wq *wq);
311 void idxd_wq_drain(struct idxd_wq *wq);
312 void idxd_wq_reset(struct idxd_wq *wq);
313 int idxd_wq_map_portal(struct idxd_wq *wq);
314 void idxd_wq_unmap_portal(struct idxd_wq *wq);
315 void idxd_wq_disable_cleanup(struct idxd_wq *wq);
316
317 /* submission */
318 int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc);
319 struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype);
320 void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc);
321
322 /* dmaengine */
323 int idxd_register_dma_device(struct idxd_device *idxd);
324 void idxd_unregister_dma_device(struct idxd_device *idxd);
325 int idxd_register_dma_channel(struct idxd_wq *wq);
326 void idxd_unregister_dma_channel(struct idxd_wq *wq);
327 void idxd_parse_completion_status(u8 status, enum dmaengine_tx_result *res);
328 void idxd_dma_complete_txd(struct idxd_desc *desc,
329 enum idxd_complete_type comp_type);
330
331 /* cdev */
332 int idxd_cdev_register(void);
333 void idxd_cdev_remove(void);
334 int idxd_cdev_get_major(struct idxd_device *idxd);
335 int idxd_wq_add_cdev(struct idxd_wq *wq);
336 void idxd_wq_del_cdev(struct idxd_wq *wq);
337
338 #endif
339