1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3 * Copyright 2016-17 IBM Corp.
4 */
5
6 #ifndef _VAS_H
7 #define _VAS_H
8 #include <linux/atomic.h>
9 #include <linux/idr.h>
10 #include <asm/vas.h>
11 #include <linux/io.h>
12 #include <linux/dcache.h>
13 #include <linux/mutex.h>
14 #include <linux/stringify.h>
15
16 /*
17 * Overview of Virtual Accelerator Switchboard (VAS).
18 *
19 * VAS is a hardware "switchboard" that allows senders and receivers to
20 * exchange messages with _minimal_ kernel involvment. The receivers are
21 * typically NX coprocessor engines that perform compression or encryption
22 * in hardware, but receivers can also be other software threads.
23 *
24 * Senders are user/kernel threads that submit compression/encryption or
25 * other requests to the receivers. Senders must format their messages as
26 * Coprocessor Request Blocks (CRB)s and submit them using the "copy" and
27 * "paste" instructions which were introduced in Power9.
28 *
29 * A Power node can have (upto?) 8 Power chips. There is one instance of
30 * VAS in each Power9 chip. Each instance of VAS has 64K windows or ports,
31 * Senders and receivers must each connect to a separate window before they
32 * can exchange messages through the switchboard.
33 *
34 * Each window is described by two types of window contexts:
35 *
36 * Hypervisor Window Context (HVWC) of size VAS_HVWC_SIZE bytes
37 *
38 * OS/User Window Context (UWC) of size VAS_UWC_SIZE bytes.
39 *
40 * A window context can be viewed as a set of 64-bit registers. The settings
41 * in these registers configure/control/determine the behavior of the VAS
42 * hardware when messages are sent/received through the window. The registers
43 * in the HVWC are configured by the kernel while the registers in the UWC can
44 * be configured by the kernel or by the user space application that is using
45 * the window.
46 *
47 * The HVWCs for all windows on a specific instance of VAS are in a contiguous
48 * range of hardware addresses or Base address region (BAR) referred to as the
49 * HVWC BAR for the instance. Similarly the UWCs for all windows on an instance
50 * are referred to as the UWC BAR for the instance.
51 *
52 * The two BARs for each instance are defined Power9 MMIO Ranges spreadsheet
53 * and available to the kernel in the VAS node's "reg" property in the device
54 * tree:
55 *
56 * /proc/device-tree/vasm@.../reg
57 *
58 * (see vas_probe() for details on the reg property).
59 *
60 * The kernel maps the HVWC and UWC BAR regions into the kernel address
61 * space (hvwc_map and uwc_map). The kernel can then access the window
62 * contexts of a specific window using:
63 *
64 * hvwc = hvwc_map + winid * VAS_HVWC_SIZE.
65 * uwc = uwc_map + winid * VAS_UWC_SIZE.
66 *
67 * where winid is the window index (0..64K).
68 *
69 * As mentioned, a window context is used to "configure" a window. Besides
70 * this configuration address, each _send_ window also has a unique hardware
71 * "paste" address that is used to submit requests/CRBs (see vas_paste_crb()).
72 *
73 * The hardware paste address for a window is computed using the "paste
74 * base address" and "paste win id shift" reg properties in the VAS device
75 * tree node using:
76 *
77 * paste_addr = paste_base + ((winid << paste_win_id_shift))
78 *
79 * (again, see vas_probe() for ->paste_base_addr and ->paste_win_id_shift).
80 *
81 * The kernel maps this hardware address into the sender's address space
82 * after which they can use the 'paste' instruction (new in Power9) to
83 * send a message (submit a request aka CRB) to the coprocessor.
84 *
85 * NOTE: In the initial version, senders can only in-kernel drivers/threads.
86 * Support for user space threads will be added in follow-on patches.
87 *
88 * TODO: Do we need to map the UWC into user address space so they can return
89 * credits? Its NA for NX but may be needed for other receive windows.
90 *
91 */
92
93 #define VAS_WINDOWS_PER_CHIP (64 << 10)
94
95 /*
96 * Hypervisor and OS/USer Window Context sizes
97 */
98 #define VAS_HVWC_SIZE 512
99 #define VAS_UWC_SIZE PAGE_SIZE
100
101 /*
102 * Initial per-process credits.
103 * Max send window credits: 4K-1 (12-bits in VAS_TX_WCRED)
104 * Max receive window credits: 64K-1 (16 bits in VAS_LRX_WCRED)
105 *
106 * TODO: Needs tuning for per-process credits
107 */
108 #define VAS_RX_WCREDS_MAX ((64 << 10) - 1)
109 #define VAS_TX_WCREDS_MAX ((4 << 10) - 1)
110 #define VAS_WCREDS_DEFAULT (1 << 10)
111
112 /*
113 * VAS Window Context Register Offsets and bitmasks.
114 * See Section 3.1.4 of VAS Work book
115 */
116 #define VAS_LPID_OFFSET 0x010
117 #define VAS_LPID PPC_BITMASK(0, 11)
118
119 #define VAS_PID_OFFSET 0x018
120 #define VAS_PID_ID PPC_BITMASK(0, 19)
121
122 #define VAS_XLATE_MSR_OFFSET 0x020
123 #define VAS_XLATE_MSR_DR PPC_BIT(0)
124 #define VAS_XLATE_MSR_TA PPC_BIT(1)
125 #define VAS_XLATE_MSR_PR PPC_BIT(2)
126 #define VAS_XLATE_MSR_US PPC_BIT(3)
127 #define VAS_XLATE_MSR_HV PPC_BIT(4)
128 #define VAS_XLATE_MSR_SF PPC_BIT(5)
129
130 #define VAS_XLATE_LPCR_OFFSET 0x028
131 #define VAS_XLATE_LPCR_PAGE_SIZE PPC_BITMASK(0, 2)
132 #define VAS_XLATE_LPCR_ISL PPC_BIT(3)
133 #define VAS_XLATE_LPCR_TC PPC_BIT(4)
134 #define VAS_XLATE_LPCR_SC PPC_BIT(5)
135
136 #define VAS_XLATE_CTL_OFFSET 0x030
137 #define VAS_XLATE_MODE PPC_BITMASK(0, 1)
138
139 #define VAS_AMR_OFFSET 0x040
140 #define VAS_AMR PPC_BITMASK(0, 63)
141
142 #define VAS_SEIDR_OFFSET 0x048
143 #define VAS_SEIDR PPC_BITMASK(0, 63)
144
145 #define VAS_FAULT_TX_WIN_OFFSET 0x050
146 #define VAS_FAULT_TX_WIN PPC_BITMASK(48, 63)
147
148 #define VAS_OSU_INTR_SRC_RA_OFFSET 0x060
149 #define VAS_OSU_INTR_SRC_RA PPC_BITMASK(8, 63)
150
151 #define VAS_HV_INTR_SRC_RA_OFFSET 0x070
152 #define VAS_HV_INTR_SRC_RA PPC_BITMASK(8, 63)
153
154 #define VAS_PSWID_OFFSET 0x078
155 #define VAS_PSWID_EA_HANDLE PPC_BITMASK(0, 31)
156
157 #define VAS_SPARE1_OFFSET 0x080
158 #define VAS_SPARE2_OFFSET 0x088
159 #define VAS_SPARE3_OFFSET 0x090
160 #define VAS_SPARE4_OFFSET 0x130
161 #define VAS_SPARE5_OFFSET 0x160
162 #define VAS_SPARE6_OFFSET 0x188
163
164 #define VAS_LFIFO_BAR_OFFSET 0x0A0
165 #define VAS_LFIFO_BAR PPC_BITMASK(8, 53)
166 #define VAS_PAGE_MIGRATION_SELECT PPC_BITMASK(54, 56)
167
168 #define VAS_LDATA_STAMP_CTL_OFFSET 0x0A8
169 #define VAS_LDATA_STAMP PPC_BITMASK(0, 1)
170 #define VAS_XTRA_WRITE PPC_BIT(2)
171
172 #define VAS_LDMA_CACHE_CTL_OFFSET 0x0B0
173 #define VAS_LDMA_TYPE PPC_BITMASK(0, 1)
174 #define VAS_LDMA_FIFO_DISABLE PPC_BIT(2)
175
176 #define VAS_LRFIFO_PUSH_OFFSET 0x0B8
177 #define VAS_LRFIFO_PUSH PPC_BITMASK(0, 15)
178
179 #define VAS_CURR_MSG_COUNT_OFFSET 0x0C0
180 #define VAS_CURR_MSG_COUNT PPC_BITMASK(0, 7)
181
182 #define VAS_LNOTIFY_AFTER_COUNT_OFFSET 0x0C8
183 #define VAS_LNOTIFY_AFTER_COUNT PPC_BITMASK(0, 7)
184
185 #define VAS_LRX_WCRED_OFFSET 0x0E0
186 #define VAS_LRX_WCRED PPC_BITMASK(0, 15)
187
188 #define VAS_LRX_WCRED_ADDER_OFFSET 0x190
189 #define VAS_LRX_WCRED_ADDER PPC_BITMASK(0, 15)
190
191 #define VAS_TX_WCRED_OFFSET 0x0F0
192 #define VAS_TX_WCRED PPC_BITMASK(4, 15)
193
194 #define VAS_TX_WCRED_ADDER_OFFSET 0x1A0
195 #define VAS_TX_WCRED_ADDER PPC_BITMASK(4, 15)
196
197 #define VAS_LFIFO_SIZE_OFFSET 0x100
198 #define VAS_LFIFO_SIZE PPC_BITMASK(0, 3)
199
200 #define VAS_WINCTL_OFFSET 0x108
201 #define VAS_WINCTL_OPEN PPC_BIT(0)
202 #define VAS_WINCTL_REJ_NO_CREDIT PPC_BIT(1)
203 #define VAS_WINCTL_PIN PPC_BIT(2)
204 #define VAS_WINCTL_TX_WCRED_MODE PPC_BIT(3)
205 #define VAS_WINCTL_RX_WCRED_MODE PPC_BIT(4)
206 #define VAS_WINCTL_TX_WORD_MODE PPC_BIT(5)
207 #define VAS_WINCTL_RX_WORD_MODE PPC_BIT(6)
208 #define VAS_WINCTL_RSVD_TXBUF PPC_BIT(7)
209 #define VAS_WINCTL_THRESH_CTL PPC_BITMASK(8, 9)
210 #define VAS_WINCTL_FAULT_WIN PPC_BIT(10)
211 #define VAS_WINCTL_NX_WIN PPC_BIT(11)
212
213 #define VAS_WIN_STATUS_OFFSET 0x110
214 #define VAS_WIN_BUSY PPC_BIT(1)
215
216 #define VAS_WIN_CTX_CACHING_CTL_OFFSET 0x118
217 #define VAS_CASTOUT_REQ PPC_BIT(0)
218 #define VAS_PUSH_TO_MEM PPC_BIT(1)
219 #define VAS_WIN_CACHE_STATUS PPC_BIT(4)
220
221 #define VAS_TX_RSVD_BUF_COUNT_OFFSET 0x120
222 #define VAS_RXVD_BUF_COUNT PPC_BITMASK(58, 63)
223
224 #define VAS_LRFIFO_WIN_PTR_OFFSET 0x128
225 #define VAS_LRX_WIN_ID PPC_BITMASK(0, 15)
226
227 /*
228 * Local Notification Control Register controls what happens in _response_
229 * to a paste command and hence applies only to receive windows.
230 */
231 #define VAS_LNOTIFY_CTL_OFFSET 0x138
232 #define VAS_NOTIFY_DISABLE PPC_BIT(0)
233 #define VAS_INTR_DISABLE PPC_BIT(1)
234 #define VAS_NOTIFY_EARLY PPC_BIT(2)
235 #define VAS_NOTIFY_OSU_INTR PPC_BIT(3)
236
237 #define VAS_LNOTIFY_PID_OFFSET 0x140
238 #define VAS_LNOTIFY_PID PPC_BITMASK(0, 19)
239
240 #define VAS_LNOTIFY_LPID_OFFSET 0x148
241 #define VAS_LNOTIFY_LPID PPC_BITMASK(0, 11)
242
243 #define VAS_LNOTIFY_TID_OFFSET 0x150
244 #define VAS_LNOTIFY_TID PPC_BITMASK(0, 15)
245
246 #define VAS_LNOTIFY_SCOPE_OFFSET 0x158
247 #define VAS_LNOTIFY_MIN_SCOPE PPC_BITMASK(0, 1)
248 #define VAS_LNOTIFY_MAX_SCOPE PPC_BITMASK(2, 3)
249
250 #define VAS_NX_UTIL_OFFSET 0x1B0
251 #define VAS_NX_UTIL PPC_BITMASK(0, 63)
252
253 /* SE: Side effects */
254 #define VAS_NX_UTIL_SE_OFFSET 0x1B8
255 #define VAS_NX_UTIL_SE PPC_BITMASK(0, 63)
256
257 #define VAS_NX_UTIL_ADDER_OFFSET 0x180
258 #define VAS_NX_UTIL_ADDER PPC_BITMASK(32, 63)
259
260 /*
261 * VREG(x):
262 * Expand a register's short name (eg: LPID) into two parameters:
263 * - the register's short name in string form ("LPID"), and
264 * - the name of the macro (eg: VAS_LPID_OFFSET), defining the
265 * register's offset in the window context
266 */
267 #define VREG_SFX(n, s) __stringify(n), VAS_##n##s
268 #define VREG(r) VREG_SFX(r, _OFFSET)
269
270 /*
271 * Local Notify Scope Control Register. (Receive windows only).
272 */
273 enum vas_notify_scope {
274 VAS_SCOPE_LOCAL,
275 VAS_SCOPE_GROUP,
276 VAS_SCOPE_VECTORED_GROUP,
277 VAS_SCOPE_UNUSED,
278 };
279
280 /*
281 * Local DMA Cache Control Register (Receive windows only).
282 */
283 enum vas_dma_type {
284 VAS_DMA_TYPE_INJECT,
285 VAS_DMA_TYPE_WRITE,
286 };
287
288 /*
289 * Local Notify Scope Control Register. (Receive windows only).
290 * Not applicable to NX receive windows.
291 */
292 enum vas_notify_after_count {
293 VAS_NOTIFY_AFTER_256 = 0,
294 VAS_NOTIFY_NONE,
295 VAS_NOTIFY_AFTER_2
296 };
297
298 /*
299 * One per instance of VAS. Each instance will have a separate set of
300 * receive windows, one per coprocessor type.
301 *
302 * See also function header of set_vinst_win() for details on ->windows[]
303 * and ->rxwin[] tables.
304 */
305 struct vas_instance {
306 int vas_id;
307 struct ida ida;
308 struct list_head node;
309 struct platform_device *pdev;
310
311 u64 hvwc_bar_start;
312 u64 uwc_bar_start;
313 u64 paste_base_addr;
314 u64 paste_win_id_shift;
315
316 struct mutex mutex;
317 struct vas_window *rxwin[VAS_COP_TYPE_MAX];
318 struct vas_window *windows[VAS_WINDOWS_PER_CHIP];
319
320 char *dbgname;
321 struct dentry *dbgdir;
322 };
323
324 /*
325 * In-kernel state a VAS window. One per window.
326 */
327 struct vas_window {
328 /* Fields common to send and receive windows */
329 struct vas_instance *vinst;
330 int winid;
331 bool tx_win; /* True if send window */
332 bool nx_win; /* True if NX window */
333 bool user_win; /* True if user space window */
334 void *hvwc_map; /* HV window context */
335 void *uwc_map; /* OS/User window context */
336 pid_t pid; /* Linux process id of owner */
337 int wcreds_max; /* Window credits */
338
339 char *dbgname;
340 struct dentry *dbgdir;
341
342 /* Fields applicable only to send windows */
343 void *paste_kaddr;
344 char *paste_addr_name;
345 struct vas_window *rxwin;
346
347 /* Feilds applicable only to receive windows */
348 enum vas_cop_type cop;
349 atomic_t num_txwins;
350 };
351
352 /*
353 * Container for the hardware state of a window. One per-window.
354 *
355 * A VAS Window context is a 512-byte area in the hardware that contains
356 * a set of 64-bit registers. Individual bit-fields in these registers
357 * determine the configuration/operation of the hardware. struct vas_winctx
358 * is a container for the register fields in the window context.
359 */
360 struct vas_winctx {
361 void *rx_fifo;
362 int rx_fifo_size;
363 int wcreds_max;
364 int rsvd_txbuf_count;
365
366 bool user_win;
367 bool nx_win;
368 bool fault_win;
369 bool rsvd_txbuf_enable;
370 bool pin_win;
371 bool rej_no_credit;
372 bool tx_wcred_mode;
373 bool rx_wcred_mode;
374 bool tx_word_mode;
375 bool rx_word_mode;
376 bool data_stamp;
377 bool xtra_write;
378 bool notify_disable;
379 bool intr_disable;
380 bool fifo_disable;
381 bool notify_early;
382 bool notify_os_intr_reg;
383
384 int lpid;
385 int pidr; /* value from SPRN_PID, not linux pid */
386 int lnotify_lpid;
387 int lnotify_pid;
388 int lnotify_tid;
389 u32 pswid;
390 int rx_win_id;
391 int fault_win_id;
392 int tc_mode;
393
394 u64 irq_port;
395
396 enum vas_dma_type dma_type;
397 enum vas_notify_scope min_scope;
398 enum vas_notify_scope max_scope;
399 enum vas_notify_after_count notify_after_count;
400 };
401
402 extern struct mutex vas_mutex;
403
404 extern struct vas_instance *find_vas_instance(int vasid);
405 extern void vas_init_dbgdir(void);
406 extern void vas_instance_init_dbgdir(struct vas_instance *vinst);
407 extern void vas_window_init_dbgdir(struct vas_window *win);
408 extern void vas_window_free_dbgdir(struct vas_window *win);
409
vas_log_write(struct vas_window * win,char * name,void * regptr,u64 val)410 static inline void vas_log_write(struct vas_window *win, char *name,
411 void *regptr, u64 val)
412 {
413 if (val)
414 pr_debug("%swin #%d: %s reg %p, val 0x%016llx\n",
415 win->tx_win ? "Tx" : "Rx", win->winid, name,
416 regptr, val);
417 }
418
write_uwc_reg(struct vas_window * win,char * name,s32 reg,u64 val)419 static inline void write_uwc_reg(struct vas_window *win, char *name,
420 s32 reg, u64 val)
421 {
422 void *regptr;
423
424 regptr = win->uwc_map + reg;
425 vas_log_write(win, name, regptr, val);
426
427 out_be64(regptr, val);
428 }
429
write_hvwc_reg(struct vas_window * win,char * name,s32 reg,u64 val)430 static inline void write_hvwc_reg(struct vas_window *win, char *name,
431 s32 reg, u64 val)
432 {
433 void *regptr;
434
435 regptr = win->hvwc_map + reg;
436 vas_log_write(win, name, regptr, val);
437
438 out_be64(regptr, val);
439 }
440
read_hvwc_reg(struct vas_window * win,char * name __maybe_unused,s32 reg)441 static inline u64 read_hvwc_reg(struct vas_window *win,
442 char *name __maybe_unused, s32 reg)
443 {
444 return in_be64(win->hvwc_map+reg);
445 }
446
decode_pswid(u32 pswid,int * vasid,int * winid)447 static inline void decode_pswid(u32 pswid, int *vasid, int *winid)
448 {
449 if (vasid)
450 *vasid = pswid >> (31 - 7) & 0xFF;
451
452 if (winid)
453 *winid = pswid & 0xFFFF;
454 }
455 #endif /* _VAS_H */
456