1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Driver for HiSilicon PCIe tune and trace device
4 *
5 * Copyright (c) 2022 HiSilicon Technologies Co., Ltd.
6 * Author: Yicong Yang <yangyicong@hisilicon.com>
7 */
8
9 #include <linux/bitfield.h>
10 #include <linux/bitops.h>
11 #include <linux/cpuhotplug.h>
12 #include <linux/delay.h>
13 #include <linux/dma-mapping.h>
14 #include <linux/interrupt.h>
15 #include <linux/io.h>
16 #include <linux/iommu.h>
17 #include <linux/iopoll.h>
18 #include <linux/module.h>
19 #include <linux/sysfs.h>
20 #include <linux/vmalloc.h>
21
22 #include "hisi_ptt.h"
23
24 /* Dynamic CPU hotplug state used by PTT */
25 static enum cpuhp_state hisi_ptt_pmu_online;
26
hisi_ptt_wait_tuning_finish(struct hisi_ptt * hisi_ptt)27 static bool hisi_ptt_wait_tuning_finish(struct hisi_ptt *hisi_ptt)
28 {
29 u32 val;
30
31 return !readl_poll_timeout(hisi_ptt->iobase + HISI_PTT_TUNING_INT_STAT,
32 val, !(val & HISI_PTT_TUNING_INT_STAT_MASK),
33 HISI_PTT_WAIT_POLL_INTERVAL_US,
34 HISI_PTT_WAIT_TUNE_TIMEOUT_US);
35 }
36
hisi_ptt_tune_attr_show(struct device * dev,struct device_attribute * attr,char * buf)37 static ssize_t hisi_ptt_tune_attr_show(struct device *dev,
38 struct device_attribute *attr,
39 char *buf)
40 {
41 struct hisi_ptt *hisi_ptt = to_hisi_ptt(dev_get_drvdata(dev));
42 struct dev_ext_attribute *ext_attr;
43 struct hisi_ptt_tune_desc *desc;
44 u32 reg;
45 u16 val;
46
47 ext_attr = container_of(attr, struct dev_ext_attribute, attr);
48 desc = ext_attr->var;
49
50 mutex_lock(&hisi_ptt->tune_lock);
51
52 reg = readl(hisi_ptt->iobase + HISI_PTT_TUNING_CTRL);
53 reg &= ~(HISI_PTT_TUNING_CTRL_CODE | HISI_PTT_TUNING_CTRL_SUB);
54 reg |= FIELD_PREP(HISI_PTT_TUNING_CTRL_CODE | HISI_PTT_TUNING_CTRL_SUB,
55 desc->event_code);
56 writel(reg, hisi_ptt->iobase + HISI_PTT_TUNING_CTRL);
57
58 /* Write all 1 to indicates it's the read process */
59 writel(~0U, hisi_ptt->iobase + HISI_PTT_TUNING_DATA);
60
61 if (!hisi_ptt_wait_tuning_finish(hisi_ptt)) {
62 mutex_unlock(&hisi_ptt->tune_lock);
63 return -ETIMEDOUT;
64 }
65
66 reg = readl(hisi_ptt->iobase + HISI_PTT_TUNING_DATA);
67 reg &= HISI_PTT_TUNING_DATA_VAL_MASK;
68 val = FIELD_GET(HISI_PTT_TUNING_DATA_VAL_MASK, reg);
69
70 mutex_unlock(&hisi_ptt->tune_lock);
71 return sysfs_emit(buf, "%u\n", val);
72 }
73
hisi_ptt_tune_attr_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)74 static ssize_t hisi_ptt_tune_attr_store(struct device *dev,
75 struct device_attribute *attr,
76 const char *buf, size_t count)
77 {
78 struct hisi_ptt *hisi_ptt = to_hisi_ptt(dev_get_drvdata(dev));
79 struct dev_ext_attribute *ext_attr;
80 struct hisi_ptt_tune_desc *desc;
81 u32 reg;
82 u16 val;
83
84 ext_attr = container_of(attr, struct dev_ext_attribute, attr);
85 desc = ext_attr->var;
86
87 if (kstrtou16(buf, 10, &val))
88 return -EINVAL;
89
90 mutex_lock(&hisi_ptt->tune_lock);
91
92 reg = readl(hisi_ptt->iobase + HISI_PTT_TUNING_CTRL);
93 reg &= ~(HISI_PTT_TUNING_CTRL_CODE | HISI_PTT_TUNING_CTRL_SUB);
94 reg |= FIELD_PREP(HISI_PTT_TUNING_CTRL_CODE | HISI_PTT_TUNING_CTRL_SUB,
95 desc->event_code);
96 writel(reg, hisi_ptt->iobase + HISI_PTT_TUNING_CTRL);
97 writel(FIELD_PREP(HISI_PTT_TUNING_DATA_VAL_MASK, val),
98 hisi_ptt->iobase + HISI_PTT_TUNING_DATA);
99
100 if (!hisi_ptt_wait_tuning_finish(hisi_ptt)) {
101 mutex_unlock(&hisi_ptt->tune_lock);
102 return -ETIMEDOUT;
103 }
104
105 mutex_unlock(&hisi_ptt->tune_lock);
106 return count;
107 }
108
109 #define HISI_PTT_TUNE_ATTR(_name, _val, _show, _store) \
110 static struct hisi_ptt_tune_desc _name##_desc = { \
111 .name = #_name, \
112 .event_code = (_val), \
113 }; \
114 static struct dev_ext_attribute hisi_ptt_##_name##_attr = { \
115 .attr = __ATTR(_name, 0600, _show, _store), \
116 .var = &_name##_desc, \
117 }
118
119 #define HISI_PTT_TUNE_ATTR_COMMON(_name, _val) \
120 HISI_PTT_TUNE_ATTR(_name, _val, \
121 hisi_ptt_tune_attr_show, \
122 hisi_ptt_tune_attr_store)
123
124 /*
125 * The value of the tuning event are composed of two parts: main event code
126 * in BIT[0,15] and subevent code in BIT[16,23]. For example, qox_tx_cpl is
127 * a subevent of 'Tx path QoS control' which for tuning the weight of Tx
128 * completion TLPs. See hisi_ptt.rst documentation for more information.
129 */
130 #define HISI_PTT_TUNE_QOS_TX_CPL (0x4 | (3 << 16))
131 #define HISI_PTT_TUNE_QOS_TX_NP (0x4 | (4 << 16))
132 #define HISI_PTT_TUNE_QOS_TX_P (0x4 | (5 << 16))
133 #define HISI_PTT_TUNE_RX_ALLOC_BUF_LEVEL (0x5 | (6 << 16))
134 #define HISI_PTT_TUNE_TX_ALLOC_BUF_LEVEL (0x5 | (7 << 16))
135
136 HISI_PTT_TUNE_ATTR_COMMON(qos_tx_cpl, HISI_PTT_TUNE_QOS_TX_CPL);
137 HISI_PTT_TUNE_ATTR_COMMON(qos_tx_np, HISI_PTT_TUNE_QOS_TX_NP);
138 HISI_PTT_TUNE_ATTR_COMMON(qos_tx_p, HISI_PTT_TUNE_QOS_TX_P);
139 HISI_PTT_TUNE_ATTR_COMMON(rx_alloc_buf_level, HISI_PTT_TUNE_RX_ALLOC_BUF_LEVEL);
140 HISI_PTT_TUNE_ATTR_COMMON(tx_alloc_buf_level, HISI_PTT_TUNE_TX_ALLOC_BUF_LEVEL);
141
142 static struct attribute *hisi_ptt_tune_attrs[] = {
143 &hisi_ptt_qos_tx_cpl_attr.attr.attr,
144 &hisi_ptt_qos_tx_np_attr.attr.attr,
145 &hisi_ptt_qos_tx_p_attr.attr.attr,
146 &hisi_ptt_rx_alloc_buf_level_attr.attr.attr,
147 &hisi_ptt_tx_alloc_buf_level_attr.attr.attr,
148 NULL,
149 };
150
151 static struct attribute_group hisi_ptt_tune_group = {
152 .name = "tune",
153 .attrs = hisi_ptt_tune_attrs,
154 };
155
hisi_ptt_get_filter_val(u16 devid,bool is_port)156 static u16 hisi_ptt_get_filter_val(u16 devid, bool is_port)
157 {
158 if (is_port)
159 return BIT(HISI_PCIE_CORE_PORT_ID(devid & 0xff));
160
161 return devid;
162 }
163
hisi_ptt_wait_trace_hw_idle(struct hisi_ptt * hisi_ptt)164 static bool hisi_ptt_wait_trace_hw_idle(struct hisi_ptt *hisi_ptt)
165 {
166 u32 val;
167
168 return !readl_poll_timeout_atomic(hisi_ptt->iobase + HISI_PTT_TRACE_STS,
169 val, val & HISI_PTT_TRACE_IDLE,
170 HISI_PTT_WAIT_POLL_INTERVAL_US,
171 HISI_PTT_WAIT_TRACE_TIMEOUT_US);
172 }
173
hisi_ptt_wait_dma_reset_done(struct hisi_ptt * hisi_ptt)174 static void hisi_ptt_wait_dma_reset_done(struct hisi_ptt *hisi_ptt)
175 {
176 u32 val;
177
178 readl_poll_timeout_atomic(hisi_ptt->iobase + HISI_PTT_TRACE_WR_STS,
179 val, !val, HISI_PTT_RESET_POLL_INTERVAL_US,
180 HISI_PTT_RESET_TIMEOUT_US);
181 }
182
hisi_ptt_trace_end(struct hisi_ptt * hisi_ptt)183 static void hisi_ptt_trace_end(struct hisi_ptt *hisi_ptt)
184 {
185 writel(0, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL);
186 hisi_ptt->trace_ctrl.started = false;
187 }
188
hisi_ptt_trace_start(struct hisi_ptt * hisi_ptt)189 static int hisi_ptt_trace_start(struct hisi_ptt *hisi_ptt)
190 {
191 struct hisi_ptt_trace_ctrl *ctrl = &hisi_ptt->trace_ctrl;
192 u32 val;
193 int i;
194
195 /* Check device idle before start trace */
196 if (!hisi_ptt_wait_trace_hw_idle(hisi_ptt)) {
197 pci_err(hisi_ptt->pdev, "Failed to start trace, the device is still busy\n");
198 return -EBUSY;
199 }
200
201 ctrl->started = true;
202
203 /* Reset the DMA before start tracing */
204 val = readl(hisi_ptt->iobase + HISI_PTT_TRACE_CTRL);
205 val |= HISI_PTT_TRACE_CTRL_RST;
206 writel(val, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL);
207
208 hisi_ptt_wait_dma_reset_done(hisi_ptt);
209
210 val = readl(hisi_ptt->iobase + HISI_PTT_TRACE_CTRL);
211 val &= ~HISI_PTT_TRACE_CTRL_RST;
212 writel(val, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL);
213
214 /* Reset the index of current buffer */
215 hisi_ptt->trace_ctrl.buf_index = 0;
216
217 /* Zero the trace buffers */
218 for (i = 0; i < HISI_PTT_TRACE_BUF_CNT; i++)
219 memset(ctrl->trace_buf[i].addr, 0, HISI_PTT_TRACE_BUF_SIZE);
220
221 /* Clear the interrupt status */
222 writel(HISI_PTT_TRACE_INT_STAT_MASK, hisi_ptt->iobase + HISI_PTT_TRACE_INT_STAT);
223 writel(0, hisi_ptt->iobase + HISI_PTT_TRACE_INT_MASK);
224
225 /* Set the trace control register */
226 val = FIELD_PREP(HISI_PTT_TRACE_CTRL_TYPE_SEL, ctrl->type);
227 val |= FIELD_PREP(HISI_PTT_TRACE_CTRL_RXTX_SEL, ctrl->direction);
228 val |= FIELD_PREP(HISI_PTT_TRACE_CTRL_DATA_FORMAT, ctrl->format);
229 val |= FIELD_PREP(HISI_PTT_TRACE_CTRL_TARGET_SEL, hisi_ptt->trace_ctrl.filter);
230 if (!hisi_ptt->trace_ctrl.is_port)
231 val |= HISI_PTT_TRACE_CTRL_FILTER_MODE;
232
233 /* Start the Trace */
234 val |= HISI_PTT_TRACE_CTRL_EN;
235 writel(val, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL);
236
237 return 0;
238 }
239
hisi_ptt_update_aux(struct hisi_ptt * hisi_ptt,int index,bool stop)240 static int hisi_ptt_update_aux(struct hisi_ptt *hisi_ptt, int index, bool stop)
241 {
242 struct hisi_ptt_trace_ctrl *ctrl = &hisi_ptt->trace_ctrl;
243 struct perf_output_handle *handle = &ctrl->handle;
244 struct perf_event *event = handle->event;
245 struct hisi_ptt_pmu_buf *buf;
246 size_t size;
247 void *addr;
248
249 buf = perf_get_aux(handle);
250 if (!buf || !handle->size)
251 return -EINVAL;
252
253 addr = ctrl->trace_buf[ctrl->buf_index].addr;
254
255 /*
256 * If we're going to stop, read the size of already traced data from
257 * HISI_PTT_TRACE_WR_STS. Otherwise we're coming from the interrupt,
258 * the data size is always HISI_PTT_TRACE_BUF_SIZE.
259 */
260 if (stop) {
261 u32 reg;
262
263 reg = readl(hisi_ptt->iobase + HISI_PTT_TRACE_WR_STS);
264 size = FIELD_GET(HISI_PTT_TRACE_WR_STS_WRITE, reg);
265 } else {
266 size = HISI_PTT_TRACE_BUF_SIZE;
267 }
268
269 memcpy(buf->base + buf->pos, addr, size);
270 buf->pos += size;
271
272 /*
273 * Just commit the traced data if we're going to stop. Otherwise if the
274 * resident AUX buffer cannot contain the data of next trace buffer,
275 * apply a new one.
276 */
277 if (stop) {
278 perf_aux_output_end(handle, buf->pos);
279 } else if (buf->length - buf->pos < HISI_PTT_TRACE_BUF_SIZE) {
280 perf_aux_output_end(handle, buf->pos);
281
282 buf = perf_aux_output_begin(handle, event);
283 if (!buf)
284 return -EINVAL;
285
286 buf->pos = handle->head % buf->length;
287 if (buf->length - buf->pos < HISI_PTT_TRACE_BUF_SIZE) {
288 perf_aux_output_end(handle, 0);
289 return -EINVAL;
290 }
291 }
292
293 return 0;
294 }
295
hisi_ptt_isr(int irq,void * context)296 static irqreturn_t hisi_ptt_isr(int irq, void *context)
297 {
298 struct hisi_ptt *hisi_ptt = context;
299 u32 status, buf_idx;
300
301 status = readl(hisi_ptt->iobase + HISI_PTT_TRACE_INT_STAT);
302 if (!(status & HISI_PTT_TRACE_INT_STAT_MASK))
303 return IRQ_NONE;
304
305 buf_idx = ffs(status) - 1;
306
307 /* Clear the interrupt status of buffer @buf_idx */
308 writel(status, hisi_ptt->iobase + HISI_PTT_TRACE_INT_STAT);
309
310 /*
311 * Update the AUX buffer and cache the current buffer index,
312 * as we need to know this and save the data when the trace
313 * is ended out of the interrupt handler. End the trace
314 * if the updating fails.
315 */
316 if (hisi_ptt_update_aux(hisi_ptt, buf_idx, false))
317 hisi_ptt_trace_end(hisi_ptt);
318 else
319 hisi_ptt->trace_ctrl.buf_index = (buf_idx + 1) % HISI_PTT_TRACE_BUF_CNT;
320
321 return IRQ_HANDLED;
322 }
323
hisi_ptt_irq_free_vectors(void * pdev)324 static void hisi_ptt_irq_free_vectors(void *pdev)
325 {
326 pci_free_irq_vectors(pdev);
327 }
328
hisi_ptt_register_irq(struct hisi_ptt * hisi_ptt)329 static int hisi_ptt_register_irq(struct hisi_ptt *hisi_ptt)
330 {
331 struct pci_dev *pdev = hisi_ptt->pdev;
332 int ret;
333
334 ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI);
335 if (ret < 0) {
336 pci_err(pdev, "failed to allocate irq vector, ret = %d\n", ret);
337 return ret;
338 }
339
340 ret = devm_add_action_or_reset(&pdev->dev, hisi_ptt_irq_free_vectors, pdev);
341 if (ret < 0)
342 return ret;
343
344 hisi_ptt->trace_irq = pci_irq_vector(pdev, HISI_PTT_TRACE_DMA_IRQ);
345 ret = devm_request_irq(&pdev->dev, hisi_ptt->trace_irq, hisi_ptt_isr,
346 IRQF_NOBALANCING | IRQF_NO_THREAD, DRV_NAME,
347 hisi_ptt);
348 if (ret) {
349 pci_err(pdev, "failed to request irq %d, ret = %d\n",
350 hisi_ptt->trace_irq, ret);
351 return ret;
352 }
353
354 return 0;
355 }
356
hisi_ptt_init_filters(struct pci_dev * pdev,void * data)357 static int hisi_ptt_init_filters(struct pci_dev *pdev, void *data)
358 {
359 struct pci_dev *root_port = pcie_find_root_port(pdev);
360 struct hisi_ptt_filter_desc *filter;
361 struct hisi_ptt *hisi_ptt = data;
362 u32 port_devid;
363
364 if (!root_port)
365 return 0;
366
367 port_devid = PCI_DEVID(root_port->bus->number, root_port->devfn);
368 if (port_devid < hisi_ptt->lower_bdf ||
369 port_devid > hisi_ptt->upper_bdf)
370 return 0;
371
372 /*
373 * We won't fail the probe if filter allocation failed here. The filters
374 * should be partial initialized and users would know which filter fails
375 * through the log. Other functions of PTT device are still available.
376 */
377 filter = kzalloc(sizeof(*filter), GFP_KERNEL);
378 if (!filter) {
379 pci_err(hisi_ptt->pdev, "failed to add filter %s\n", pci_name(pdev));
380 return -ENOMEM;
381 }
382
383 filter->devid = PCI_DEVID(pdev->bus->number, pdev->devfn);
384
385 if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT) {
386 filter->is_port = true;
387 list_add_tail(&filter->list, &hisi_ptt->port_filters);
388
389 /* Update the available port mask */
390 hisi_ptt->port_mask |= hisi_ptt_get_filter_val(filter->devid, true);
391 } else {
392 list_add_tail(&filter->list, &hisi_ptt->req_filters);
393 }
394
395 return 0;
396 }
397
hisi_ptt_release_filters(void * data)398 static void hisi_ptt_release_filters(void *data)
399 {
400 struct hisi_ptt_filter_desc *filter, *tmp;
401 struct hisi_ptt *hisi_ptt = data;
402
403 list_for_each_entry_safe(filter, tmp, &hisi_ptt->req_filters, list) {
404 list_del(&filter->list);
405 kfree(filter);
406 }
407
408 list_for_each_entry_safe(filter, tmp, &hisi_ptt->port_filters, list) {
409 list_del(&filter->list);
410 kfree(filter);
411 }
412 }
413
hisi_ptt_config_trace_buf(struct hisi_ptt * hisi_ptt)414 static int hisi_ptt_config_trace_buf(struct hisi_ptt *hisi_ptt)
415 {
416 struct hisi_ptt_trace_ctrl *ctrl = &hisi_ptt->trace_ctrl;
417 struct device *dev = &hisi_ptt->pdev->dev;
418 int i;
419
420 ctrl->trace_buf = devm_kcalloc(dev, HISI_PTT_TRACE_BUF_CNT,
421 sizeof(*ctrl->trace_buf), GFP_KERNEL);
422 if (!ctrl->trace_buf)
423 return -ENOMEM;
424
425 for (i = 0; i < HISI_PTT_TRACE_BUF_CNT; ++i) {
426 ctrl->trace_buf[i].addr = dmam_alloc_coherent(dev, HISI_PTT_TRACE_BUF_SIZE,
427 &ctrl->trace_buf[i].dma,
428 GFP_KERNEL);
429 if (!ctrl->trace_buf[i].addr)
430 return -ENOMEM;
431 }
432
433 /* Configure the trace DMA buffer */
434 for (i = 0; i < HISI_PTT_TRACE_BUF_CNT; i++) {
435 writel(lower_32_bits(ctrl->trace_buf[i].dma),
436 hisi_ptt->iobase + HISI_PTT_TRACE_ADDR_BASE_LO_0 +
437 i * HISI_PTT_TRACE_ADDR_STRIDE);
438 writel(upper_32_bits(ctrl->trace_buf[i].dma),
439 hisi_ptt->iobase + HISI_PTT_TRACE_ADDR_BASE_HI_0 +
440 i * HISI_PTT_TRACE_ADDR_STRIDE);
441 }
442 writel(HISI_PTT_TRACE_BUF_SIZE, hisi_ptt->iobase + HISI_PTT_TRACE_ADDR_SIZE);
443
444 return 0;
445 }
446
hisi_ptt_init_ctrls(struct hisi_ptt * hisi_ptt)447 static int hisi_ptt_init_ctrls(struct hisi_ptt *hisi_ptt)
448 {
449 struct pci_dev *pdev = hisi_ptt->pdev;
450 struct pci_bus *bus;
451 int ret;
452 u32 reg;
453
454 INIT_LIST_HEAD(&hisi_ptt->port_filters);
455 INIT_LIST_HEAD(&hisi_ptt->req_filters);
456
457 ret = hisi_ptt_config_trace_buf(hisi_ptt);
458 if (ret)
459 return ret;
460
461 /*
462 * The device range register provides the information about the root
463 * ports which the RCiEP can control and trace. The RCiEP and the root
464 * ports which it supports are on the same PCIe core, with same domain
465 * number but maybe different bus number. The device range register
466 * will tell us which root ports we can support, Bit[31:16] indicates
467 * the upper BDF numbers of the root port, while Bit[15:0] indicates
468 * the lower.
469 */
470 reg = readl(hisi_ptt->iobase + HISI_PTT_DEVICE_RANGE);
471 hisi_ptt->upper_bdf = FIELD_GET(HISI_PTT_DEVICE_RANGE_UPPER, reg);
472 hisi_ptt->lower_bdf = FIELD_GET(HISI_PTT_DEVICE_RANGE_LOWER, reg);
473
474 bus = pci_find_bus(pci_domain_nr(pdev->bus), PCI_BUS_NUM(hisi_ptt->upper_bdf));
475 if (bus)
476 pci_walk_bus(bus, hisi_ptt_init_filters, hisi_ptt);
477
478 ret = devm_add_action_or_reset(&pdev->dev, hisi_ptt_release_filters, hisi_ptt);
479 if (ret)
480 return ret;
481
482 hisi_ptt->trace_ctrl.on_cpu = -1;
483 return 0;
484 }
485
cpumask_show(struct device * dev,struct device_attribute * attr,char * buf)486 static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr,
487 char *buf)
488 {
489 struct hisi_ptt *hisi_ptt = to_hisi_ptt(dev_get_drvdata(dev));
490 const cpumask_t *cpumask = cpumask_of_node(dev_to_node(&hisi_ptt->pdev->dev));
491
492 return cpumap_print_to_pagebuf(true, buf, cpumask);
493 }
494 static DEVICE_ATTR_RO(cpumask);
495
496 static struct attribute *hisi_ptt_cpumask_attrs[] = {
497 &dev_attr_cpumask.attr,
498 NULL
499 };
500
501 static const struct attribute_group hisi_ptt_cpumask_attr_group = {
502 .attrs = hisi_ptt_cpumask_attrs,
503 };
504
505 /*
506 * Bit 19 indicates the filter type, 1 for Root Port filter and 0 for Requester
507 * filter. Bit[15:0] indicates the filter value, for Root Port filter it's
508 * a bit mask of desired ports and for Requester filter it's the Requester ID
509 * of the desired PCIe function. Bit[18:16] is reserved for extension.
510 *
511 * See hisi_ptt.rst documentation for detailed information.
512 */
513 PMU_FORMAT_ATTR(filter, "config:0-19");
514 PMU_FORMAT_ATTR(direction, "config:20-23");
515 PMU_FORMAT_ATTR(type, "config:24-31");
516 PMU_FORMAT_ATTR(format, "config:32-35");
517
518 static struct attribute *hisi_ptt_pmu_format_attrs[] = {
519 &format_attr_filter.attr,
520 &format_attr_direction.attr,
521 &format_attr_type.attr,
522 &format_attr_format.attr,
523 NULL
524 };
525
526 static struct attribute_group hisi_ptt_pmu_format_group = {
527 .name = "format",
528 .attrs = hisi_ptt_pmu_format_attrs,
529 };
530
531 static const struct attribute_group *hisi_ptt_pmu_groups[] = {
532 &hisi_ptt_cpumask_attr_group,
533 &hisi_ptt_pmu_format_group,
534 &hisi_ptt_tune_group,
535 NULL
536 };
537
hisi_ptt_trace_valid_direction(u32 val)538 static int hisi_ptt_trace_valid_direction(u32 val)
539 {
540 /*
541 * The direction values have different effects according to the data
542 * format (specified in the parentheses). TLP set A/B means different
543 * set of TLP types. See hisi_ptt.rst documentation for more details.
544 */
545 static const u32 hisi_ptt_trace_available_direction[] = {
546 0, /* inbound(4DW) or reserved(8DW) */
547 1, /* outbound(4DW) */
548 2, /* {in, out}bound(4DW) or inbound(8DW), TLP set A */
549 3, /* {in, out}bound(4DW) or inbound(8DW), TLP set B */
550 };
551 int i;
552
553 for (i = 0; i < ARRAY_SIZE(hisi_ptt_trace_available_direction); i++) {
554 if (val == hisi_ptt_trace_available_direction[i])
555 return 0;
556 }
557
558 return -EINVAL;
559 }
560
hisi_ptt_trace_valid_type(u32 val)561 static int hisi_ptt_trace_valid_type(u32 val)
562 {
563 /* Different types can be set simultaneously */
564 static const u32 hisi_ptt_trace_available_type[] = {
565 1, /* posted_request */
566 2, /* non-posted_request */
567 4, /* completion */
568 };
569 int i;
570
571 if (!val)
572 return -EINVAL;
573
574 /*
575 * Walk the available list and clear the valid bits of
576 * the config. If there is any resident bit after the
577 * walk then the config is invalid.
578 */
579 for (i = 0; i < ARRAY_SIZE(hisi_ptt_trace_available_type); i++)
580 val &= ~hisi_ptt_trace_available_type[i];
581
582 if (val)
583 return -EINVAL;
584
585 return 0;
586 }
587
hisi_ptt_trace_valid_format(u32 val)588 static int hisi_ptt_trace_valid_format(u32 val)
589 {
590 static const u32 hisi_ptt_trace_availble_format[] = {
591 0, /* 4DW */
592 1, /* 8DW */
593 };
594 int i;
595
596 for (i = 0; i < ARRAY_SIZE(hisi_ptt_trace_availble_format); i++) {
597 if (val == hisi_ptt_trace_availble_format[i])
598 return 0;
599 }
600
601 return -EINVAL;
602 }
603
hisi_ptt_trace_valid_filter(struct hisi_ptt * hisi_ptt,u64 config)604 static int hisi_ptt_trace_valid_filter(struct hisi_ptt *hisi_ptt, u64 config)
605 {
606 unsigned long val, port_mask = hisi_ptt->port_mask;
607 struct hisi_ptt_filter_desc *filter;
608
609 hisi_ptt->trace_ctrl.is_port = FIELD_GET(HISI_PTT_PMU_FILTER_IS_PORT, config);
610 val = FIELD_GET(HISI_PTT_PMU_FILTER_VAL_MASK, config);
611
612 /*
613 * Port filters are defined as bit mask. For port filters, check
614 * the bits in the @val are within the range of hisi_ptt->port_mask
615 * and whether it's empty or not, otherwise user has specified
616 * some unsupported root ports.
617 *
618 * For Requester ID filters, walk the available filter list to see
619 * whether we have one matched.
620 */
621 if (!hisi_ptt->trace_ctrl.is_port) {
622 list_for_each_entry(filter, &hisi_ptt->req_filters, list) {
623 if (val == hisi_ptt_get_filter_val(filter->devid, filter->is_port))
624 return 0;
625 }
626 } else if (bitmap_subset(&val, &port_mask, BITS_PER_LONG)) {
627 return 0;
628 }
629
630 return -EINVAL;
631 }
632
hisi_ptt_pmu_init_configs(struct hisi_ptt * hisi_ptt,struct perf_event * event)633 static void hisi_ptt_pmu_init_configs(struct hisi_ptt *hisi_ptt, struct perf_event *event)
634 {
635 struct hisi_ptt_trace_ctrl *ctrl = &hisi_ptt->trace_ctrl;
636 u32 val;
637
638 val = FIELD_GET(HISI_PTT_PMU_FILTER_VAL_MASK, event->attr.config);
639 hisi_ptt->trace_ctrl.filter = val;
640
641 val = FIELD_GET(HISI_PTT_PMU_DIRECTION_MASK, event->attr.config);
642 ctrl->direction = val;
643
644 val = FIELD_GET(HISI_PTT_PMU_TYPE_MASK, event->attr.config);
645 ctrl->type = val;
646
647 val = FIELD_GET(HISI_PTT_PMU_FORMAT_MASK, event->attr.config);
648 ctrl->format = val;
649 }
650
hisi_ptt_pmu_event_init(struct perf_event * event)651 static int hisi_ptt_pmu_event_init(struct perf_event *event)
652 {
653 struct hisi_ptt *hisi_ptt = to_hisi_ptt(event->pmu);
654 int ret;
655 u32 val;
656
657 if (event->cpu < 0) {
658 dev_dbg(event->pmu->dev, "Per-task mode not supported\n");
659 return -EOPNOTSUPP;
660 }
661
662 if (event->attach_state & PERF_ATTACH_TASK)
663 return -EOPNOTSUPP;
664
665 if (event->attr.type != hisi_ptt->hisi_ptt_pmu.type)
666 return -ENOENT;
667
668 ret = hisi_ptt_trace_valid_filter(hisi_ptt, event->attr.config);
669 if (ret < 0)
670 return ret;
671
672 val = FIELD_GET(HISI_PTT_PMU_DIRECTION_MASK, event->attr.config);
673 ret = hisi_ptt_trace_valid_direction(val);
674 if (ret < 0)
675 return ret;
676
677 val = FIELD_GET(HISI_PTT_PMU_TYPE_MASK, event->attr.config);
678 ret = hisi_ptt_trace_valid_type(val);
679 if (ret < 0)
680 return ret;
681
682 val = FIELD_GET(HISI_PTT_PMU_FORMAT_MASK, event->attr.config);
683 return hisi_ptt_trace_valid_format(val);
684 }
685
hisi_ptt_pmu_setup_aux(struct perf_event * event,void ** pages,int nr_pages,bool overwrite)686 static void *hisi_ptt_pmu_setup_aux(struct perf_event *event, void **pages,
687 int nr_pages, bool overwrite)
688 {
689 struct hisi_ptt_pmu_buf *buf;
690 struct page **pagelist;
691 int i;
692
693 if (overwrite) {
694 dev_warn(event->pmu->dev, "Overwrite mode is not supported\n");
695 return NULL;
696 }
697
698 /* If the pages size less than buffers, we cannot start trace */
699 if (nr_pages < HISI_PTT_TRACE_TOTAL_BUF_SIZE / PAGE_SIZE)
700 return NULL;
701
702 buf = kzalloc(sizeof(*buf), GFP_KERNEL);
703 if (!buf)
704 return NULL;
705
706 pagelist = kcalloc(nr_pages, sizeof(*pagelist), GFP_KERNEL);
707 if (!pagelist)
708 goto err;
709
710 for (i = 0; i < nr_pages; i++)
711 pagelist[i] = virt_to_page(pages[i]);
712
713 buf->base = vmap(pagelist, nr_pages, VM_MAP, PAGE_KERNEL);
714 if (!buf->base) {
715 kfree(pagelist);
716 goto err;
717 }
718
719 buf->nr_pages = nr_pages;
720 buf->length = nr_pages * PAGE_SIZE;
721 buf->pos = 0;
722
723 kfree(pagelist);
724 return buf;
725 err:
726 kfree(buf);
727 return NULL;
728 }
729
hisi_ptt_pmu_free_aux(void * aux)730 static void hisi_ptt_pmu_free_aux(void *aux)
731 {
732 struct hisi_ptt_pmu_buf *buf = aux;
733
734 vunmap(buf->base);
735 kfree(buf);
736 }
737
hisi_ptt_pmu_start(struct perf_event * event,int flags)738 static void hisi_ptt_pmu_start(struct perf_event *event, int flags)
739 {
740 struct hisi_ptt *hisi_ptt = to_hisi_ptt(event->pmu);
741 struct perf_output_handle *handle = &hisi_ptt->trace_ctrl.handle;
742 struct hw_perf_event *hwc = &event->hw;
743 struct device *dev = event->pmu->dev;
744 struct hisi_ptt_pmu_buf *buf;
745 int cpu = event->cpu;
746 int ret;
747
748 hwc->state = 0;
749
750 /* Serialize the perf process if user specified several CPUs */
751 spin_lock(&hisi_ptt->pmu_lock);
752 if (hisi_ptt->trace_ctrl.started) {
753 dev_dbg(dev, "trace has already started\n");
754 goto stop;
755 }
756
757 /*
758 * Handle the interrupt on the same cpu which starts the trace to avoid
759 * context mismatch. Otherwise we'll trigger the WARN from the perf
760 * core in event_function_local(). If CPU passed is offline we'll fail
761 * here, just log it since we can do nothing here.
762 */
763 ret = irq_set_affinity(hisi_ptt->trace_irq, cpumask_of(cpu));
764 if (ret)
765 dev_warn(dev, "failed to set the affinity of trace interrupt\n");
766
767 hisi_ptt->trace_ctrl.on_cpu = cpu;
768
769 buf = perf_aux_output_begin(handle, event);
770 if (!buf) {
771 dev_dbg(dev, "aux output begin failed\n");
772 goto stop;
773 }
774
775 buf->pos = handle->head % buf->length;
776
777 hisi_ptt_pmu_init_configs(hisi_ptt, event);
778
779 ret = hisi_ptt_trace_start(hisi_ptt);
780 if (ret) {
781 dev_dbg(dev, "trace start failed, ret = %d\n", ret);
782 perf_aux_output_end(handle, 0);
783 goto stop;
784 }
785
786 spin_unlock(&hisi_ptt->pmu_lock);
787 return;
788 stop:
789 event->hw.state |= PERF_HES_STOPPED;
790 spin_unlock(&hisi_ptt->pmu_lock);
791 }
792
hisi_ptt_pmu_stop(struct perf_event * event,int flags)793 static void hisi_ptt_pmu_stop(struct perf_event *event, int flags)
794 {
795 struct hisi_ptt *hisi_ptt = to_hisi_ptt(event->pmu);
796 struct hw_perf_event *hwc = &event->hw;
797
798 if (hwc->state & PERF_HES_STOPPED)
799 return;
800
801 spin_lock(&hisi_ptt->pmu_lock);
802 if (hisi_ptt->trace_ctrl.started) {
803 hisi_ptt_trace_end(hisi_ptt);
804
805 if (!hisi_ptt_wait_trace_hw_idle(hisi_ptt))
806 dev_warn(event->pmu->dev, "Device is still busy\n");
807
808 hisi_ptt_update_aux(hisi_ptt, hisi_ptt->trace_ctrl.buf_index, true);
809 }
810 spin_unlock(&hisi_ptt->pmu_lock);
811
812 hwc->state |= PERF_HES_STOPPED;
813 perf_event_update_userpage(event);
814 hwc->state |= PERF_HES_UPTODATE;
815 }
816
hisi_ptt_pmu_add(struct perf_event * event,int flags)817 static int hisi_ptt_pmu_add(struct perf_event *event, int flags)
818 {
819 struct hisi_ptt *hisi_ptt = to_hisi_ptt(event->pmu);
820 struct hw_perf_event *hwc = &event->hw;
821 int cpu = event->cpu;
822
823 /* Only allow the cpus on the device's node to add the event */
824 if (!cpumask_test_cpu(cpu, cpumask_of_node(dev_to_node(&hisi_ptt->pdev->dev))))
825 return 0;
826
827 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
828
829 if (flags & PERF_EF_START) {
830 hisi_ptt_pmu_start(event, PERF_EF_RELOAD);
831 if (hwc->state & PERF_HES_STOPPED)
832 return -EINVAL;
833 }
834
835 return 0;
836 }
837
hisi_ptt_pmu_del(struct perf_event * event,int flags)838 static void hisi_ptt_pmu_del(struct perf_event *event, int flags)
839 {
840 hisi_ptt_pmu_stop(event, PERF_EF_UPDATE);
841 }
842
hisi_ptt_pmu_read(struct perf_event * event)843 static void hisi_ptt_pmu_read(struct perf_event *event)
844 {
845 }
846
hisi_ptt_remove_cpuhp_instance(void * hotplug_node)847 static void hisi_ptt_remove_cpuhp_instance(void *hotplug_node)
848 {
849 cpuhp_state_remove_instance_nocalls(hisi_ptt_pmu_online, hotplug_node);
850 }
851
hisi_ptt_unregister_pmu(void * pmu)852 static void hisi_ptt_unregister_pmu(void *pmu)
853 {
854 perf_pmu_unregister(pmu);
855 }
856
hisi_ptt_register_pmu(struct hisi_ptt * hisi_ptt)857 static int hisi_ptt_register_pmu(struct hisi_ptt *hisi_ptt)
858 {
859 u16 core_id, sicl_id;
860 char *pmu_name;
861 u32 reg;
862 int ret;
863
864 ret = cpuhp_state_add_instance_nocalls(hisi_ptt_pmu_online,
865 &hisi_ptt->hotplug_node);
866 if (ret)
867 return ret;
868
869 ret = devm_add_action_or_reset(&hisi_ptt->pdev->dev,
870 hisi_ptt_remove_cpuhp_instance,
871 &hisi_ptt->hotplug_node);
872 if (ret)
873 return ret;
874
875 mutex_init(&hisi_ptt->tune_lock);
876 spin_lock_init(&hisi_ptt->pmu_lock);
877
878 hisi_ptt->hisi_ptt_pmu = (struct pmu) {
879 .module = THIS_MODULE,
880 .capabilities = PERF_PMU_CAP_EXCLUSIVE | PERF_PMU_CAP_ITRACE,
881 .task_ctx_nr = perf_sw_context,
882 .attr_groups = hisi_ptt_pmu_groups,
883 .event_init = hisi_ptt_pmu_event_init,
884 .setup_aux = hisi_ptt_pmu_setup_aux,
885 .free_aux = hisi_ptt_pmu_free_aux,
886 .start = hisi_ptt_pmu_start,
887 .stop = hisi_ptt_pmu_stop,
888 .add = hisi_ptt_pmu_add,
889 .del = hisi_ptt_pmu_del,
890 .read = hisi_ptt_pmu_read,
891 };
892
893 reg = readl(hisi_ptt->iobase + HISI_PTT_LOCATION);
894 core_id = FIELD_GET(HISI_PTT_CORE_ID, reg);
895 sicl_id = FIELD_GET(HISI_PTT_SICL_ID, reg);
896
897 pmu_name = devm_kasprintf(&hisi_ptt->pdev->dev, GFP_KERNEL, "hisi_ptt%u_%u",
898 sicl_id, core_id);
899 if (!pmu_name)
900 return -ENOMEM;
901
902 ret = perf_pmu_register(&hisi_ptt->hisi_ptt_pmu, pmu_name, -1);
903 if (ret)
904 return ret;
905
906 return devm_add_action_or_reset(&hisi_ptt->pdev->dev,
907 hisi_ptt_unregister_pmu,
908 &hisi_ptt->hisi_ptt_pmu);
909 }
910
911 /*
912 * The DMA of PTT trace can only use direct mappings due to some
913 * hardware restriction. Check whether there is no IOMMU or the
914 * policy of the IOMMU domain is passthrough, otherwise the trace
915 * cannot work.
916 *
917 * The PTT device is supposed to behind an ARM SMMUv3, which
918 * should have passthrough the device by a quirk.
919 */
hisi_ptt_check_iommu_mapping(struct pci_dev * pdev)920 static int hisi_ptt_check_iommu_mapping(struct pci_dev *pdev)
921 {
922 struct iommu_domain *iommu_domain;
923
924 iommu_domain = iommu_get_domain_for_dev(&pdev->dev);
925 if (!iommu_domain || iommu_domain->type == IOMMU_DOMAIN_IDENTITY)
926 return 0;
927
928 return -EOPNOTSUPP;
929 }
930
hisi_ptt_probe(struct pci_dev * pdev,const struct pci_device_id * id)931 static int hisi_ptt_probe(struct pci_dev *pdev,
932 const struct pci_device_id *id)
933 {
934 struct hisi_ptt *hisi_ptt;
935 int ret;
936
937 ret = hisi_ptt_check_iommu_mapping(pdev);
938 if (ret) {
939 pci_err(pdev, "requires direct DMA mappings\n");
940 return ret;
941 }
942
943 hisi_ptt = devm_kzalloc(&pdev->dev, sizeof(*hisi_ptt), GFP_KERNEL);
944 if (!hisi_ptt)
945 return -ENOMEM;
946
947 hisi_ptt->pdev = pdev;
948 pci_set_drvdata(pdev, hisi_ptt);
949
950 ret = pcim_enable_device(pdev);
951 if (ret) {
952 pci_err(pdev, "failed to enable device, ret = %d\n", ret);
953 return ret;
954 }
955
956 ret = pcim_iomap_regions(pdev, BIT(2), DRV_NAME);
957 if (ret) {
958 pci_err(pdev, "failed to remap io memory, ret = %d\n", ret);
959 return ret;
960 }
961
962 hisi_ptt->iobase = pcim_iomap_table(pdev)[2];
963
964 ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
965 if (ret) {
966 pci_err(pdev, "failed to set 64 bit dma mask, ret = %d\n", ret);
967 return ret;
968 }
969
970 pci_set_master(pdev);
971
972 ret = hisi_ptt_register_irq(hisi_ptt);
973 if (ret)
974 return ret;
975
976 ret = hisi_ptt_init_ctrls(hisi_ptt);
977 if (ret) {
978 pci_err(pdev, "failed to init controls, ret = %d\n", ret);
979 return ret;
980 }
981
982 ret = hisi_ptt_register_pmu(hisi_ptt);
983 if (ret) {
984 pci_err(pdev, "failed to register PMU device, ret = %d", ret);
985 return ret;
986 }
987
988 return 0;
989 }
990
991 static const struct pci_device_id hisi_ptt_id_tbl[] = {
992 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa12e) },
993 { }
994 };
995 MODULE_DEVICE_TABLE(pci, hisi_ptt_id_tbl);
996
997 static struct pci_driver hisi_ptt_driver = {
998 .name = DRV_NAME,
999 .id_table = hisi_ptt_id_tbl,
1000 .probe = hisi_ptt_probe,
1001 };
1002
hisi_ptt_cpu_teardown(unsigned int cpu,struct hlist_node * node)1003 static int hisi_ptt_cpu_teardown(unsigned int cpu, struct hlist_node *node)
1004 {
1005 struct hisi_ptt *hisi_ptt;
1006 struct device *dev;
1007 int target, src;
1008
1009 hisi_ptt = hlist_entry_safe(node, struct hisi_ptt, hotplug_node);
1010 src = hisi_ptt->trace_ctrl.on_cpu;
1011 dev = hisi_ptt->hisi_ptt_pmu.dev;
1012
1013 if (!hisi_ptt->trace_ctrl.started || src != cpu)
1014 return 0;
1015
1016 target = cpumask_any_but(cpumask_of_node(dev_to_node(&hisi_ptt->pdev->dev)), cpu);
1017 if (target >= nr_cpu_ids) {
1018 dev_err(dev, "no available cpu for perf context migration\n");
1019 return 0;
1020 }
1021
1022 perf_pmu_migrate_context(&hisi_ptt->hisi_ptt_pmu, src, target);
1023
1024 /*
1025 * Also make sure the interrupt bind to the migrated CPU as well. Warn
1026 * the user on failure here.
1027 */
1028 if (irq_set_affinity(hisi_ptt->trace_irq, cpumask_of(target)))
1029 dev_warn(dev, "failed to set the affinity of trace interrupt\n");
1030
1031 hisi_ptt->trace_ctrl.on_cpu = target;
1032 return 0;
1033 }
1034
hisi_ptt_init(void)1035 static int __init hisi_ptt_init(void)
1036 {
1037 int ret;
1038
1039 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, DRV_NAME, NULL,
1040 hisi_ptt_cpu_teardown);
1041 if (ret < 0)
1042 return ret;
1043 hisi_ptt_pmu_online = ret;
1044
1045 ret = pci_register_driver(&hisi_ptt_driver);
1046 if (ret)
1047 cpuhp_remove_multi_state(hisi_ptt_pmu_online);
1048
1049 return ret;
1050 }
1051 module_init(hisi_ptt_init);
1052
hisi_ptt_exit(void)1053 static void __exit hisi_ptt_exit(void)
1054 {
1055 pci_unregister_driver(&hisi_ptt_driver);
1056 cpuhp_remove_multi_state(hisi_ptt_pmu_online);
1057 }
1058 module_exit(hisi_ptt_exit);
1059
1060 MODULE_LICENSE("GPL");
1061 MODULE_AUTHOR("Yicong Yang <yangyicong@hisilicon.com>");
1062 MODULE_DESCRIPTION("Driver for HiSilicon PCIe tune and trace device");
1063