1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright 2019 Collabora Ltd */
3
4 #include <drm/drm_file.h>
5 #include <drm/drm_gem_shmem_helper.h>
6 #include <drm/panfrost_drm.h>
7 #include <linux/completion.h>
8 #include <linux/dma-buf-map.h>
9 #include <linux/iopoll.h>
10 #include <linux/pm_runtime.h>
11 #include <linux/slab.h>
12 #include <linux/uaccess.h>
13
14 #include "panfrost_device.h"
15 #include "panfrost_features.h"
16 #include "panfrost_gem.h"
17 #include "panfrost_issues.h"
18 #include "panfrost_job.h"
19 #include "panfrost_mmu.h"
20 #include "panfrost_perfcnt.h"
21 #include "panfrost_regs.h"
22
23 #define COUNTERS_PER_BLOCK 64
24 #define BYTES_PER_COUNTER 4
25 #define BLOCKS_PER_COREGROUP 8
26 #define V4_SHADERS_PER_COREGROUP 4
27
28 struct panfrost_perfcnt {
29 struct panfrost_gem_mapping *mapping;
30 size_t bosize;
31 void *buf;
32 struct panfrost_file_priv *user;
33 struct mutex lock;
34 struct completion dump_comp;
35 };
36
panfrost_perfcnt_clean_cache_done(struct panfrost_device * pfdev)37 void panfrost_perfcnt_clean_cache_done(struct panfrost_device *pfdev)
38 {
39 complete(&pfdev->perfcnt->dump_comp);
40 }
41
panfrost_perfcnt_sample_done(struct panfrost_device * pfdev)42 void panfrost_perfcnt_sample_done(struct panfrost_device *pfdev)
43 {
44 gpu_write(pfdev, GPU_CMD, GPU_CMD_CLEAN_CACHES);
45 }
46
panfrost_perfcnt_dump_locked(struct panfrost_device * pfdev)47 static int panfrost_perfcnt_dump_locked(struct panfrost_device *pfdev)
48 {
49 u64 gpuva;
50 int ret;
51
52 reinit_completion(&pfdev->perfcnt->dump_comp);
53 gpuva = pfdev->perfcnt->mapping->mmnode.start << PAGE_SHIFT;
54 gpu_write(pfdev, GPU_PERFCNT_BASE_LO, gpuva);
55 gpu_write(pfdev, GPU_PERFCNT_BASE_HI, gpuva >> 32);
56 gpu_write(pfdev, GPU_INT_CLEAR,
57 GPU_IRQ_CLEAN_CACHES_COMPLETED |
58 GPU_IRQ_PERFCNT_SAMPLE_COMPLETED);
59 gpu_write(pfdev, GPU_CMD, GPU_CMD_PERFCNT_SAMPLE);
60 ret = wait_for_completion_interruptible_timeout(&pfdev->perfcnt->dump_comp,
61 msecs_to_jiffies(1000));
62 if (!ret)
63 ret = -ETIMEDOUT;
64 else if (ret > 0)
65 ret = 0;
66
67 return ret;
68 }
69
panfrost_perfcnt_enable_locked(struct panfrost_device * pfdev,struct drm_file * file_priv,unsigned int counterset)70 static int panfrost_perfcnt_enable_locked(struct panfrost_device *pfdev,
71 struct drm_file *file_priv,
72 unsigned int counterset)
73 {
74 struct panfrost_file_priv *user = file_priv->driver_priv;
75 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
76 struct dma_buf_map map;
77 struct drm_gem_shmem_object *bo;
78 u32 cfg, as;
79 int ret;
80
81 if (user == perfcnt->user)
82 return 0;
83 else if (perfcnt->user)
84 return -EBUSY;
85
86 ret = pm_runtime_get_sync(pfdev->dev);
87 if (ret < 0)
88 goto err_put_pm;
89
90 bo = drm_gem_shmem_create(pfdev->ddev, perfcnt->bosize);
91 if (IS_ERR(bo)) {
92 ret = PTR_ERR(bo);
93 goto err_put_pm;
94 }
95
96 /* Map the perfcnt buf in the address space attached to file_priv. */
97 ret = panfrost_gem_open(&bo->base, file_priv);
98 if (ret)
99 goto err_put_bo;
100
101 perfcnt->mapping = panfrost_gem_mapping_get(to_panfrost_bo(&bo->base),
102 user);
103 if (!perfcnt->mapping) {
104 ret = -EINVAL;
105 goto err_close_bo;
106 }
107
108 ret = drm_gem_shmem_vmap(bo, &map);
109 if (ret)
110 goto err_put_mapping;
111 perfcnt->buf = map.vaddr;
112
113 /*
114 * Invalidate the cache and clear the counters to start from a fresh
115 * state.
116 */
117 reinit_completion(&pfdev->perfcnt->dump_comp);
118 gpu_write(pfdev, GPU_INT_CLEAR,
119 GPU_IRQ_CLEAN_CACHES_COMPLETED |
120 GPU_IRQ_PERFCNT_SAMPLE_COMPLETED);
121 gpu_write(pfdev, GPU_CMD, GPU_CMD_PERFCNT_CLEAR);
122 gpu_write(pfdev, GPU_CMD, GPU_CMD_CLEAN_INV_CACHES);
123 ret = wait_for_completion_timeout(&pfdev->perfcnt->dump_comp,
124 msecs_to_jiffies(1000));
125 if (!ret) {
126 ret = -ETIMEDOUT;
127 goto err_vunmap;
128 }
129
130 perfcnt->user = user;
131
132 as = panfrost_mmu_as_get(pfdev, perfcnt->mapping->mmu);
133 cfg = GPU_PERFCNT_CFG_AS(as) |
134 GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_MANUAL);
135
136 /*
137 * Bifrost GPUs have 2 set of counters, but we're only interested by
138 * the first one for now.
139 */
140 if (panfrost_model_is_bifrost(pfdev))
141 cfg |= GPU_PERFCNT_CFG_SETSEL(counterset);
142
143 gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0xffffffff);
144 gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0xffffffff);
145 gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0xffffffff);
146
147 /*
148 * Due to PRLAM-8186 we need to disable the Tiler before we enable HW
149 * counters.
150 */
151 if (panfrost_has_hw_issue(pfdev, HW_ISSUE_8186))
152 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0);
153 else
154 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0xffffffff);
155
156 gpu_write(pfdev, GPU_PERFCNT_CFG, cfg);
157
158 if (panfrost_has_hw_issue(pfdev, HW_ISSUE_8186))
159 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0xffffffff);
160
161 /* The BO ref is retained by the mapping. */
162 drm_gem_object_put(&bo->base);
163
164 return 0;
165
166 err_vunmap:
167 drm_gem_shmem_vunmap(bo, &map);
168 err_put_mapping:
169 panfrost_gem_mapping_put(perfcnt->mapping);
170 err_close_bo:
171 panfrost_gem_close(&bo->base, file_priv);
172 err_put_bo:
173 drm_gem_object_put(&bo->base);
174 err_put_pm:
175 pm_runtime_put(pfdev->dev);
176 return ret;
177 }
178
panfrost_perfcnt_disable_locked(struct panfrost_device * pfdev,struct drm_file * file_priv)179 static int panfrost_perfcnt_disable_locked(struct panfrost_device *pfdev,
180 struct drm_file *file_priv)
181 {
182 struct panfrost_file_priv *user = file_priv->driver_priv;
183 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
184 struct dma_buf_map map = DMA_BUF_MAP_INIT_VADDR(perfcnt->buf);
185
186 if (user != perfcnt->user)
187 return -EINVAL;
188
189 gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0x0);
190 gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0x0);
191 gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0x0);
192 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0);
193 gpu_write(pfdev, GPU_PERFCNT_CFG,
194 GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF));
195
196 perfcnt->user = NULL;
197 drm_gem_shmem_vunmap(&perfcnt->mapping->obj->base, &map);
198 perfcnt->buf = NULL;
199 panfrost_gem_close(&perfcnt->mapping->obj->base.base, file_priv);
200 panfrost_mmu_as_put(pfdev, perfcnt->mapping->mmu);
201 panfrost_gem_mapping_put(perfcnt->mapping);
202 perfcnt->mapping = NULL;
203 pm_runtime_mark_last_busy(pfdev->dev);
204 pm_runtime_put_autosuspend(pfdev->dev);
205
206 return 0;
207 }
208
panfrost_ioctl_perfcnt_enable(struct drm_device * dev,void * data,struct drm_file * file_priv)209 int panfrost_ioctl_perfcnt_enable(struct drm_device *dev, void *data,
210 struct drm_file *file_priv)
211 {
212 struct panfrost_device *pfdev = dev->dev_private;
213 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
214 struct drm_panfrost_perfcnt_enable *req = data;
215 int ret;
216
217 ret = panfrost_unstable_ioctl_check();
218 if (ret)
219 return ret;
220
221 /* Only Bifrost GPUs have 2 set of counters. */
222 if (req->counterset > (panfrost_model_is_bifrost(pfdev) ? 1 : 0))
223 return -EINVAL;
224
225 mutex_lock(&perfcnt->lock);
226 if (req->enable)
227 ret = panfrost_perfcnt_enable_locked(pfdev, file_priv,
228 req->counterset);
229 else
230 ret = panfrost_perfcnt_disable_locked(pfdev, file_priv);
231 mutex_unlock(&perfcnt->lock);
232
233 return ret;
234 }
235
panfrost_ioctl_perfcnt_dump(struct drm_device * dev,void * data,struct drm_file * file_priv)236 int panfrost_ioctl_perfcnt_dump(struct drm_device *dev, void *data,
237 struct drm_file *file_priv)
238 {
239 struct panfrost_device *pfdev = dev->dev_private;
240 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
241 struct drm_panfrost_perfcnt_dump *req = data;
242 void __user *user_ptr = (void __user *)(uintptr_t)req->buf_ptr;
243 int ret;
244
245 ret = panfrost_unstable_ioctl_check();
246 if (ret)
247 return ret;
248
249 mutex_lock(&perfcnt->lock);
250 if (perfcnt->user != file_priv->driver_priv) {
251 ret = -EINVAL;
252 goto out;
253 }
254
255 ret = panfrost_perfcnt_dump_locked(pfdev);
256 if (ret)
257 goto out;
258
259 if (copy_to_user(user_ptr, perfcnt->buf, perfcnt->bosize))
260 ret = -EFAULT;
261
262 out:
263 mutex_unlock(&perfcnt->lock);
264
265 return ret;
266 }
267
panfrost_perfcnt_close(struct drm_file * file_priv)268 void panfrost_perfcnt_close(struct drm_file *file_priv)
269 {
270 struct panfrost_file_priv *pfile = file_priv->driver_priv;
271 struct panfrost_device *pfdev = pfile->pfdev;
272 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
273
274 pm_runtime_get_sync(pfdev->dev);
275 mutex_lock(&perfcnt->lock);
276 if (perfcnt->user == pfile)
277 panfrost_perfcnt_disable_locked(pfdev, file_priv);
278 mutex_unlock(&perfcnt->lock);
279 pm_runtime_mark_last_busy(pfdev->dev);
280 pm_runtime_put_autosuspend(pfdev->dev);
281 }
282
panfrost_perfcnt_init(struct panfrost_device * pfdev)283 int panfrost_perfcnt_init(struct panfrost_device *pfdev)
284 {
285 struct panfrost_perfcnt *perfcnt;
286 size_t size;
287
288 if (panfrost_has_hw_feature(pfdev, HW_FEATURE_V4)) {
289 unsigned int ncoregroups;
290
291 ncoregroups = hweight64(pfdev->features.l2_present);
292 size = ncoregroups * BLOCKS_PER_COREGROUP *
293 COUNTERS_PER_BLOCK * BYTES_PER_COUNTER;
294 } else {
295 unsigned int nl2c, ncores;
296
297 /*
298 * TODO: define a macro to extract the number of l2 caches from
299 * mem_features.
300 */
301 nl2c = ((pfdev->features.mem_features >> 8) & GENMASK(3, 0)) + 1;
302
303 /*
304 * shader_present might be sparse, but the counters layout
305 * forces to dump unused regions too, hence the fls64() call
306 * instead of hweight64().
307 */
308 ncores = fls64(pfdev->features.shader_present);
309
310 /*
311 * There's always one JM and one Tiler block, hence the '+ 2'
312 * here.
313 */
314 size = (nl2c + ncores + 2) *
315 COUNTERS_PER_BLOCK * BYTES_PER_COUNTER;
316 }
317
318 perfcnt = devm_kzalloc(pfdev->dev, sizeof(*perfcnt), GFP_KERNEL);
319 if (!perfcnt)
320 return -ENOMEM;
321
322 perfcnt->bosize = size;
323
324 /* Start with everything disabled. */
325 gpu_write(pfdev, GPU_PERFCNT_CFG,
326 GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF));
327 gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0);
328 gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0);
329 gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0);
330 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0);
331
332 init_completion(&perfcnt->dump_comp);
333 mutex_init(&perfcnt->lock);
334 pfdev->perfcnt = perfcnt;
335
336 return 0;
337 }
338
panfrost_perfcnt_fini(struct panfrost_device * pfdev)339 void panfrost_perfcnt_fini(struct panfrost_device *pfdev)
340 {
341 /* Disable everything before leaving. */
342 gpu_write(pfdev, GPU_PERFCNT_CFG,
343 GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF));
344 gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0);
345 gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0);
346 gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0);
347 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0);
348 }
349