1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * The On Chip Memory (OCMEM) allocator allows various clients to allocate
4 * memory from OCMEM based on performance, latency and power requirements.
5 * This is typically used by the GPU, camera/video, and audio components on
6 * some Snapdragon SoCs.
7 *
8 * Copyright (C) 2019 Brian Masney <masneyb@onstation.org>
9 * Copyright (C) 2015 Red Hat. Author: Rob Clark <robdclark@gmail.com>
10 */
11
12 #include <linux/bitfield.h>
13 #include <linux/clk.h>
14 #include <linux/io.h>
15 #include <linux/kernel.h>
16 #include <linux/module.h>
17 #include <linux/of_device.h>
18 #include <linux/platform_device.h>
19 #include <linux/qcom_scm.h>
20 #include <linux/sizes.h>
21 #include <linux/slab.h>
22 #include <linux/types.h>
23 #include <soc/qcom/ocmem.h>
24
25 enum region_mode {
26 WIDE_MODE = 0x0,
27 THIN_MODE,
28 MODE_DEFAULT = WIDE_MODE,
29 };
30
31 enum ocmem_macro_state {
32 PASSTHROUGH = 0,
33 PERI_ON = 1,
34 CORE_ON = 2,
35 CLK_OFF = 4,
36 };
37
38 struct ocmem_region {
39 bool interleaved;
40 enum region_mode mode;
41 unsigned int num_macros;
42 enum ocmem_macro_state macro_state[4];
43 unsigned long macro_size;
44 unsigned long region_size;
45 };
46
47 struct ocmem_config {
48 uint8_t num_regions;
49 unsigned long macro_size;
50 };
51
52 struct ocmem {
53 struct device *dev;
54 const struct ocmem_config *config;
55 struct resource *memory;
56 void __iomem *mmio;
57 unsigned int num_ports;
58 unsigned int num_macros;
59 bool interleaved;
60 struct ocmem_region *regions;
61 unsigned long active_allocations;
62 };
63
64 #define OCMEM_MIN_ALIGN SZ_64K
65 #define OCMEM_MIN_ALLOC SZ_64K
66
67 #define OCMEM_REG_HW_VERSION 0x00000000
68 #define OCMEM_REG_HW_PROFILE 0x00000004
69
70 #define OCMEM_REG_REGION_MODE_CTL 0x00001000
71 #define OCMEM_REGION_MODE_CTL_REG0_THIN 0x00000001
72 #define OCMEM_REGION_MODE_CTL_REG1_THIN 0x00000002
73 #define OCMEM_REGION_MODE_CTL_REG2_THIN 0x00000004
74 #define OCMEM_REGION_MODE_CTL_REG3_THIN 0x00000008
75
76 #define OCMEM_REG_GFX_MPU_START 0x00001004
77 #define OCMEM_REG_GFX_MPU_END 0x00001008
78
79 #define OCMEM_HW_VERSION_MAJOR(val) FIELD_GET(GENMASK(31, 28), val)
80 #define OCMEM_HW_VERSION_MINOR(val) FIELD_GET(GENMASK(27, 16), val)
81 #define OCMEM_HW_VERSION_STEP(val) FIELD_GET(GENMASK(15, 0), val)
82
83 #define OCMEM_HW_PROFILE_NUM_PORTS(val) FIELD_GET(0x0000000f, (val))
84 #define OCMEM_HW_PROFILE_NUM_MACROS(val) FIELD_GET(0x00003f00, (val))
85
86 #define OCMEM_HW_PROFILE_LAST_REGN_HALFSIZE 0x00010000
87 #define OCMEM_HW_PROFILE_INTERLEAVING 0x00020000
88 #define OCMEM_REG_GEN_STATUS 0x0000000c
89
90 #define OCMEM_REG_PSGSC_STATUS 0x00000038
91 #define OCMEM_REG_PSGSC_CTL(i0) (0x0000003c + 0x1*(i0))
92
93 #define OCMEM_PSGSC_CTL_MACRO0_MODE(val) FIELD_PREP(0x00000007, (val))
94 #define OCMEM_PSGSC_CTL_MACRO1_MODE(val) FIELD_PREP(0x00000070, (val))
95 #define OCMEM_PSGSC_CTL_MACRO2_MODE(val) FIELD_PREP(0x00000700, (val))
96 #define OCMEM_PSGSC_CTL_MACRO3_MODE(val) FIELD_PREP(0x00007000, (val))
97
98 #define OCMEM_CLK_CORE_IDX 0
99 static struct clk_bulk_data ocmem_clks[] = {
100 {
101 .id = "core",
102 },
103 {
104 .id = "iface",
105 },
106 };
107
ocmem_write(struct ocmem * ocmem,u32 reg,u32 data)108 static inline void ocmem_write(struct ocmem *ocmem, u32 reg, u32 data)
109 {
110 writel(data, ocmem->mmio + reg);
111 }
112
ocmem_read(struct ocmem * ocmem,u32 reg)113 static inline u32 ocmem_read(struct ocmem *ocmem, u32 reg)
114 {
115 return readl(ocmem->mmio + reg);
116 }
117
update_ocmem(struct ocmem * ocmem)118 static void update_ocmem(struct ocmem *ocmem)
119 {
120 uint32_t region_mode_ctrl = 0x0;
121 int i;
122
123 if (!qcom_scm_ocmem_lock_available()) {
124 for (i = 0; i < ocmem->config->num_regions; i++) {
125 struct ocmem_region *region = &ocmem->regions[i];
126
127 if (region->mode == THIN_MODE)
128 region_mode_ctrl |= BIT(i);
129 }
130
131 dev_dbg(ocmem->dev, "ocmem_region_mode_control %x\n",
132 region_mode_ctrl);
133 ocmem_write(ocmem, OCMEM_REG_REGION_MODE_CTL, region_mode_ctrl);
134 }
135
136 for (i = 0; i < ocmem->config->num_regions; i++) {
137 struct ocmem_region *region = &ocmem->regions[i];
138 u32 data;
139
140 data = OCMEM_PSGSC_CTL_MACRO0_MODE(region->macro_state[0]) |
141 OCMEM_PSGSC_CTL_MACRO1_MODE(region->macro_state[1]) |
142 OCMEM_PSGSC_CTL_MACRO2_MODE(region->macro_state[2]) |
143 OCMEM_PSGSC_CTL_MACRO3_MODE(region->macro_state[3]);
144
145 ocmem_write(ocmem, OCMEM_REG_PSGSC_CTL(i), data);
146 }
147 }
148
phys_to_offset(struct ocmem * ocmem,unsigned long addr)149 static unsigned long phys_to_offset(struct ocmem *ocmem,
150 unsigned long addr)
151 {
152 if (addr < ocmem->memory->start || addr >= ocmem->memory->end)
153 return 0;
154
155 return addr - ocmem->memory->start;
156 }
157
device_address(struct ocmem * ocmem,enum ocmem_client client,unsigned long addr)158 static unsigned long device_address(struct ocmem *ocmem,
159 enum ocmem_client client,
160 unsigned long addr)
161 {
162 WARN_ON(client != OCMEM_GRAPHICS);
163
164 /* TODO: gpu uses phys_to_offset, but others do not.. */
165 return phys_to_offset(ocmem, addr);
166 }
167
update_range(struct ocmem * ocmem,struct ocmem_buf * buf,enum ocmem_macro_state mstate,enum region_mode rmode)168 static void update_range(struct ocmem *ocmem, struct ocmem_buf *buf,
169 enum ocmem_macro_state mstate, enum region_mode rmode)
170 {
171 unsigned long offset = 0;
172 int i, j;
173
174 for (i = 0; i < ocmem->config->num_regions; i++) {
175 struct ocmem_region *region = &ocmem->regions[i];
176
177 if (buf->offset <= offset && offset < buf->offset + buf->len)
178 region->mode = rmode;
179
180 for (j = 0; j < region->num_macros; j++) {
181 if (buf->offset <= offset &&
182 offset < buf->offset + buf->len)
183 region->macro_state[j] = mstate;
184
185 offset += region->macro_size;
186 }
187 }
188
189 update_ocmem(ocmem);
190 }
191
of_get_ocmem(struct device * dev)192 struct ocmem *of_get_ocmem(struct device *dev)
193 {
194 struct platform_device *pdev;
195 struct device_node *devnode;
196 struct ocmem *ocmem;
197
198 devnode = of_parse_phandle(dev->of_node, "sram", 0);
199 if (!devnode || !devnode->parent) {
200 dev_err(dev, "Cannot look up sram phandle\n");
201 of_node_put(devnode);
202 return ERR_PTR(-ENODEV);
203 }
204
205 pdev = of_find_device_by_node(devnode->parent);
206 if (!pdev) {
207 dev_err(dev, "Cannot find device node %s\n", devnode->name);
208 of_node_put(devnode);
209 return ERR_PTR(-EPROBE_DEFER);
210 }
211 of_node_put(devnode);
212
213 ocmem = platform_get_drvdata(pdev);
214 if (!ocmem) {
215 dev_err(dev, "Cannot get ocmem\n");
216 put_device(&pdev->dev);
217 return ERR_PTR(-ENODEV);
218 }
219 return ocmem;
220 }
221 EXPORT_SYMBOL(of_get_ocmem);
222
ocmem_allocate(struct ocmem * ocmem,enum ocmem_client client,unsigned long size)223 struct ocmem_buf *ocmem_allocate(struct ocmem *ocmem, enum ocmem_client client,
224 unsigned long size)
225 {
226 struct ocmem_buf *buf;
227 int ret;
228
229 /* TODO: add support for other clients... */
230 if (WARN_ON(client != OCMEM_GRAPHICS))
231 return ERR_PTR(-ENODEV);
232
233 if (size < OCMEM_MIN_ALLOC || !IS_ALIGNED(size, OCMEM_MIN_ALIGN))
234 return ERR_PTR(-EINVAL);
235
236 if (test_and_set_bit_lock(BIT(client), &ocmem->active_allocations))
237 return ERR_PTR(-EBUSY);
238
239 buf = kzalloc(sizeof(*buf), GFP_KERNEL);
240 if (!buf) {
241 ret = -ENOMEM;
242 goto err_unlock;
243 }
244
245 buf->offset = 0;
246 buf->addr = device_address(ocmem, client, buf->offset);
247 buf->len = size;
248
249 update_range(ocmem, buf, CORE_ON, WIDE_MODE);
250
251 if (qcom_scm_ocmem_lock_available()) {
252 ret = qcom_scm_ocmem_lock(QCOM_SCM_OCMEM_GRAPHICS_ID,
253 buf->offset, buf->len, WIDE_MODE);
254 if (ret) {
255 dev_err(ocmem->dev, "could not lock: %d\n", ret);
256 ret = -EINVAL;
257 goto err_kfree;
258 }
259 } else {
260 ocmem_write(ocmem, OCMEM_REG_GFX_MPU_START, buf->offset);
261 ocmem_write(ocmem, OCMEM_REG_GFX_MPU_END,
262 buf->offset + buf->len);
263 }
264
265 dev_dbg(ocmem->dev, "using %ldK of OCMEM at 0x%08lx for client %d\n",
266 size / 1024, buf->addr, client);
267
268 return buf;
269
270 err_kfree:
271 kfree(buf);
272 err_unlock:
273 clear_bit_unlock(BIT(client), &ocmem->active_allocations);
274
275 return ERR_PTR(ret);
276 }
277 EXPORT_SYMBOL(ocmem_allocate);
278
ocmem_free(struct ocmem * ocmem,enum ocmem_client client,struct ocmem_buf * buf)279 void ocmem_free(struct ocmem *ocmem, enum ocmem_client client,
280 struct ocmem_buf *buf)
281 {
282 /* TODO: add support for other clients... */
283 if (WARN_ON(client != OCMEM_GRAPHICS))
284 return;
285
286 update_range(ocmem, buf, CLK_OFF, MODE_DEFAULT);
287
288 if (qcom_scm_ocmem_lock_available()) {
289 int ret;
290
291 ret = qcom_scm_ocmem_unlock(QCOM_SCM_OCMEM_GRAPHICS_ID,
292 buf->offset, buf->len);
293 if (ret)
294 dev_err(ocmem->dev, "could not unlock: %d\n", ret);
295 } else {
296 ocmem_write(ocmem, OCMEM_REG_GFX_MPU_START, 0x0);
297 ocmem_write(ocmem, OCMEM_REG_GFX_MPU_END, 0x0);
298 }
299
300 kfree(buf);
301
302 clear_bit_unlock(BIT(client), &ocmem->active_allocations);
303 }
304 EXPORT_SYMBOL(ocmem_free);
305
ocmem_dev_probe(struct platform_device * pdev)306 static int ocmem_dev_probe(struct platform_device *pdev)
307 {
308 struct device *dev = &pdev->dev;
309 unsigned long reg, region_size;
310 int i, j, ret, num_banks;
311 struct ocmem *ocmem;
312
313 if (!qcom_scm_is_available())
314 return -EPROBE_DEFER;
315
316 ocmem = devm_kzalloc(dev, sizeof(*ocmem), GFP_KERNEL);
317 if (!ocmem)
318 return -ENOMEM;
319
320 ocmem->dev = dev;
321 ocmem->config = device_get_match_data(dev);
322
323 ret = devm_clk_bulk_get(dev, ARRAY_SIZE(ocmem_clks), ocmem_clks);
324 if (ret) {
325 if (ret != -EPROBE_DEFER)
326 dev_err(dev, "Unable to get clocks\n");
327
328 return ret;
329 }
330
331 ocmem->mmio = devm_platform_ioremap_resource_byname(pdev, "ctrl");
332 if (IS_ERR(ocmem->mmio)) {
333 dev_err(&pdev->dev, "Failed to ioremap ocmem_ctrl resource\n");
334 return PTR_ERR(ocmem->mmio);
335 }
336
337 ocmem->memory = platform_get_resource_byname(pdev, IORESOURCE_MEM,
338 "mem");
339 if (!ocmem->memory) {
340 dev_err(dev, "Could not get mem region\n");
341 return -ENXIO;
342 }
343
344 /* The core clock is synchronous with graphics */
345 WARN_ON(clk_set_rate(ocmem_clks[OCMEM_CLK_CORE_IDX].clk, 1000) < 0);
346
347 ret = clk_bulk_prepare_enable(ARRAY_SIZE(ocmem_clks), ocmem_clks);
348 if (ret) {
349 dev_info(ocmem->dev, "Failed to enable clocks\n");
350 return ret;
351 }
352
353 if (qcom_scm_restore_sec_cfg_available()) {
354 dev_dbg(dev, "configuring scm\n");
355 ret = qcom_scm_restore_sec_cfg(QCOM_SCM_OCMEM_DEV_ID, 0);
356 if (ret) {
357 dev_err(dev, "Could not enable secure configuration\n");
358 goto err_clk_disable;
359 }
360 }
361
362 reg = ocmem_read(ocmem, OCMEM_REG_HW_VERSION);
363 dev_dbg(dev, "OCMEM hardware version: %lu.%lu.%lu\n",
364 OCMEM_HW_VERSION_MAJOR(reg),
365 OCMEM_HW_VERSION_MINOR(reg),
366 OCMEM_HW_VERSION_STEP(reg));
367
368 reg = ocmem_read(ocmem, OCMEM_REG_HW_PROFILE);
369 ocmem->num_ports = OCMEM_HW_PROFILE_NUM_PORTS(reg);
370 ocmem->num_macros = OCMEM_HW_PROFILE_NUM_MACROS(reg);
371 ocmem->interleaved = !!(reg & OCMEM_HW_PROFILE_INTERLEAVING);
372
373 num_banks = ocmem->num_ports / 2;
374 region_size = ocmem->config->macro_size * num_banks;
375
376 dev_info(dev, "%u ports, %u regions, %u macros, %sinterleaved\n",
377 ocmem->num_ports, ocmem->config->num_regions,
378 ocmem->num_macros, ocmem->interleaved ? "" : "not ");
379
380 ocmem->regions = devm_kcalloc(dev, ocmem->config->num_regions,
381 sizeof(struct ocmem_region), GFP_KERNEL);
382 if (!ocmem->regions) {
383 ret = -ENOMEM;
384 goto err_clk_disable;
385 }
386
387 for (i = 0; i < ocmem->config->num_regions; i++) {
388 struct ocmem_region *region = &ocmem->regions[i];
389
390 if (WARN_ON(num_banks > ARRAY_SIZE(region->macro_state))) {
391 ret = -EINVAL;
392 goto err_clk_disable;
393 }
394
395 region->mode = MODE_DEFAULT;
396 region->num_macros = num_banks;
397
398 if (i == (ocmem->config->num_regions - 1) &&
399 reg & OCMEM_HW_PROFILE_LAST_REGN_HALFSIZE) {
400 region->macro_size = ocmem->config->macro_size / 2;
401 region->region_size = region_size / 2;
402 } else {
403 region->macro_size = ocmem->config->macro_size;
404 region->region_size = region_size;
405 }
406
407 for (j = 0; j < ARRAY_SIZE(region->macro_state); j++)
408 region->macro_state[j] = CLK_OFF;
409 }
410
411 platform_set_drvdata(pdev, ocmem);
412
413 return 0;
414
415 err_clk_disable:
416 clk_bulk_disable_unprepare(ARRAY_SIZE(ocmem_clks), ocmem_clks);
417 return ret;
418 }
419
ocmem_dev_remove(struct platform_device * pdev)420 static int ocmem_dev_remove(struct platform_device *pdev)
421 {
422 clk_bulk_disable_unprepare(ARRAY_SIZE(ocmem_clks), ocmem_clks);
423
424 return 0;
425 }
426
427 static const struct ocmem_config ocmem_8974_config = {
428 .num_regions = 3,
429 .macro_size = SZ_128K,
430 };
431
432 static const struct of_device_id ocmem_of_match[] = {
433 { .compatible = "qcom,msm8974-ocmem", .data = &ocmem_8974_config },
434 { }
435 };
436
437 MODULE_DEVICE_TABLE(of, ocmem_of_match);
438
439 static struct platform_driver ocmem_driver = {
440 .probe = ocmem_dev_probe,
441 .remove = ocmem_dev_remove,
442 .driver = {
443 .name = "ocmem",
444 .of_match_table = ocmem_of_match,
445 },
446 };
447
448 module_platform_driver(ocmem_driver);
449
450 MODULE_DESCRIPTION("On Chip Memory (OCMEM) allocator for some Snapdragon SoCs");
451 MODULE_LICENSE("GPL v2");
452