• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Tegra host1x Job
3  *
4  * Copyright (c) 2010-2013, NVIDIA Corporation.
5  *
6  * This program is free software; you can redistribute it and/or modify it
7  * under the terms and conditions of the GNU General Public License,
8  * version 2, as published by the Free Software Foundation.
9  *
10  * This program is distributed in the hope it will be useful, but WITHOUT
11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
13  * more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
17  */
18 
19 #include <linux/dma-mapping.h>
20 #include <linux/err.h>
21 #include <linux/host1x.h>
22 #include <linux/kref.h>
23 #include <linux/module.h>
24 #include <linux/scatterlist.h>
25 #include <linux/slab.h>
26 #include <linux/vmalloc.h>
27 #include <trace/events/host1x.h>
28 
29 #include "channel.h"
30 #include "dev.h"
31 #include "job.h"
32 #include "syncpt.h"
33 
host1x_job_alloc(struct host1x_channel * ch,u32 num_cmdbufs,u32 num_relocs,u32 num_waitchks)34 struct host1x_job *host1x_job_alloc(struct host1x_channel *ch,
35 				    u32 num_cmdbufs, u32 num_relocs,
36 				    u32 num_waitchks)
37 {
38 	struct host1x_job *job = NULL;
39 	unsigned int num_unpins = num_cmdbufs + num_relocs;
40 	u64 total;
41 	void *mem;
42 
43 	/* Check that we're not going to overflow */
44 	total = sizeof(struct host1x_job) +
45 		(u64)num_relocs * sizeof(struct host1x_reloc) +
46 		(u64)num_unpins * sizeof(struct host1x_job_unpin_data) +
47 		(u64)num_waitchks * sizeof(struct host1x_waitchk) +
48 		(u64)num_cmdbufs * sizeof(struct host1x_job_gather) +
49 		(u64)num_unpins * sizeof(dma_addr_t) +
50 		(u64)num_unpins * sizeof(u32 *);
51 	if (total > ULONG_MAX)
52 		return NULL;
53 
54 	mem = job = kzalloc(total, GFP_KERNEL);
55 	if (!job)
56 		return NULL;
57 
58 	kref_init(&job->ref);
59 	job->channel = ch;
60 
61 	/* Redistribute memory to the structs  */
62 	mem += sizeof(struct host1x_job);
63 	job->relocarray = num_relocs ? mem : NULL;
64 	mem += num_relocs * sizeof(struct host1x_reloc);
65 	job->unpins = num_unpins ? mem : NULL;
66 	mem += num_unpins * sizeof(struct host1x_job_unpin_data);
67 	job->waitchk = num_waitchks ? mem : NULL;
68 	mem += num_waitchks * sizeof(struct host1x_waitchk);
69 	job->gathers = num_cmdbufs ? mem : NULL;
70 	mem += num_cmdbufs * sizeof(struct host1x_job_gather);
71 	job->addr_phys = num_unpins ? mem : NULL;
72 
73 	job->reloc_addr_phys = job->addr_phys;
74 	job->gather_addr_phys = &job->addr_phys[num_relocs];
75 
76 	return job;
77 }
78 EXPORT_SYMBOL(host1x_job_alloc);
79 
host1x_job_get(struct host1x_job * job)80 struct host1x_job *host1x_job_get(struct host1x_job *job)
81 {
82 	kref_get(&job->ref);
83 	return job;
84 }
85 EXPORT_SYMBOL(host1x_job_get);
86 
job_free(struct kref * ref)87 static void job_free(struct kref *ref)
88 {
89 	struct host1x_job *job = container_of(ref, struct host1x_job, ref);
90 
91 	kfree(job);
92 }
93 
host1x_job_put(struct host1x_job * job)94 void host1x_job_put(struct host1x_job *job)
95 {
96 	kref_put(&job->ref, job_free);
97 }
98 EXPORT_SYMBOL(host1x_job_put);
99 
host1x_job_add_gather(struct host1x_job * job,struct host1x_bo * bo,u32 words,u32 offset)100 void host1x_job_add_gather(struct host1x_job *job, struct host1x_bo *bo,
101 			   u32 words, u32 offset)
102 {
103 	struct host1x_job_gather *cur_gather = &job->gathers[job->num_gathers];
104 
105 	cur_gather->words = words;
106 	cur_gather->bo = bo;
107 	cur_gather->offset = offset;
108 	job->num_gathers++;
109 }
110 EXPORT_SYMBOL(host1x_job_add_gather);
111 
112 /*
113  * NULL an already satisfied WAIT_SYNCPT host method, by patching its
114  * args in the command stream. The method data is changed to reference
115  * a reserved (never given out or incr) HOST1X_SYNCPT_RESERVED syncpt
116  * with a matching threshold value of 0, so is guaranteed to be popped
117  * by the host HW.
118  */
host1x_syncpt_patch_offset(struct host1x_syncpt * sp,struct host1x_bo * h,u32 offset)119 static void host1x_syncpt_patch_offset(struct host1x_syncpt *sp,
120 				       struct host1x_bo *h, u32 offset)
121 {
122 	void *patch_addr = NULL;
123 
124 	/* patch the wait */
125 	patch_addr = host1x_bo_kmap(h, offset >> PAGE_SHIFT);
126 	if (patch_addr) {
127 		host1x_syncpt_patch_wait(sp,
128 					 patch_addr + (offset & ~PAGE_MASK));
129 		host1x_bo_kunmap(h, offset >> PAGE_SHIFT, patch_addr);
130 	} else
131 		pr_err("Could not map cmdbuf for wait check\n");
132 }
133 
134 /*
135  * Check driver supplied waitchk structs for syncpt thresholds
136  * that have already been satisfied and NULL the comparison (to
137  * avoid a wrap condition in the HW).
138  */
do_waitchks(struct host1x_job * job,struct host1x * host,struct host1x_bo * patch)139 static int do_waitchks(struct host1x_job *job, struct host1x *host,
140 		       struct host1x_bo *patch)
141 {
142 	int i;
143 
144 	/* compare syncpt vs wait threshold */
145 	for (i = 0; i < job->num_waitchk; i++) {
146 		struct host1x_waitchk *wait = &job->waitchk[i];
147 		struct host1x_syncpt *sp =
148 			host1x_syncpt_get(host, wait->syncpt_id);
149 
150 		/* validate syncpt id */
151 		if (wait->syncpt_id > host1x_syncpt_nb_pts(host))
152 			continue;
153 
154 		/* skip all other gathers */
155 		if (patch != wait->bo)
156 			continue;
157 
158 		trace_host1x_syncpt_wait_check(wait->bo, wait->offset,
159 					       wait->syncpt_id, wait->thresh,
160 					       host1x_syncpt_read_min(sp));
161 
162 		if (host1x_syncpt_is_expired(sp, wait->thresh)) {
163 			dev_dbg(host->dev,
164 				"drop WAIT id %d (%s) thresh 0x%x, min 0x%x\n",
165 				wait->syncpt_id, sp->name, wait->thresh,
166 				host1x_syncpt_read_min(sp));
167 
168 			host1x_syncpt_patch_offset(sp, patch, wait->offset);
169 		}
170 
171 		wait->bo = NULL;
172 	}
173 
174 	return 0;
175 }
176 
pin_job(struct host1x_job * job)177 static unsigned int pin_job(struct host1x_job *job)
178 {
179 	unsigned int i;
180 
181 	job->num_unpins = 0;
182 
183 	for (i = 0; i < job->num_relocs; i++) {
184 		struct host1x_reloc *reloc = &job->relocarray[i];
185 		struct sg_table *sgt;
186 		dma_addr_t phys_addr;
187 
188 		reloc->target.bo = host1x_bo_get(reloc->target.bo);
189 		if (!reloc->target.bo)
190 			goto unpin;
191 
192 		phys_addr = host1x_bo_pin(reloc->target.bo, &sgt);
193 		if (!phys_addr)
194 			goto unpin;
195 
196 		job->addr_phys[job->num_unpins] = phys_addr;
197 		job->unpins[job->num_unpins].bo = reloc->target.bo;
198 		job->unpins[job->num_unpins].sgt = sgt;
199 		job->num_unpins++;
200 	}
201 
202 	for (i = 0; i < job->num_gathers; i++) {
203 		struct host1x_job_gather *g = &job->gathers[i];
204 		struct sg_table *sgt;
205 		dma_addr_t phys_addr;
206 
207 		g->bo = host1x_bo_get(g->bo);
208 		if (!g->bo)
209 			goto unpin;
210 
211 		phys_addr = host1x_bo_pin(g->bo, &sgt);
212 		if (!phys_addr)
213 			goto unpin;
214 
215 		job->addr_phys[job->num_unpins] = phys_addr;
216 		job->unpins[job->num_unpins].bo = g->bo;
217 		job->unpins[job->num_unpins].sgt = sgt;
218 		job->num_unpins++;
219 	}
220 
221 	return job->num_unpins;
222 
223 unpin:
224 	host1x_job_unpin(job);
225 	return 0;
226 }
227 
do_relocs(struct host1x_job * job,struct host1x_bo * cmdbuf)228 static unsigned int do_relocs(struct host1x_job *job, struct host1x_bo *cmdbuf)
229 {
230 	int i = 0;
231 	u32 last_page = ~0;
232 	void *cmdbuf_page_addr = NULL;
233 
234 	/* pin & patch the relocs for one gather */
235 	for (i = 0; i < job->num_relocs; i++) {
236 		struct host1x_reloc *reloc = &job->relocarray[i];
237 		u32 reloc_addr = (job->reloc_addr_phys[i] +
238 				  reloc->target.offset) >> reloc->shift;
239 		u32 *target;
240 
241 		/* skip all other gathers */
242 		if (cmdbuf != reloc->cmdbuf.bo)
243 			continue;
244 
245 		if (last_page != reloc->cmdbuf.offset >> PAGE_SHIFT) {
246 			if (cmdbuf_page_addr)
247 				host1x_bo_kunmap(cmdbuf, last_page,
248 						 cmdbuf_page_addr);
249 
250 			cmdbuf_page_addr = host1x_bo_kmap(cmdbuf,
251 					reloc->cmdbuf.offset >> PAGE_SHIFT);
252 			last_page = reloc->cmdbuf.offset >> PAGE_SHIFT;
253 
254 			if (unlikely(!cmdbuf_page_addr)) {
255 				pr_err("Could not map cmdbuf for relocation\n");
256 				return -ENOMEM;
257 			}
258 		}
259 
260 		target = cmdbuf_page_addr + (reloc->cmdbuf.offset & ~PAGE_MASK);
261 		*target = reloc_addr;
262 	}
263 
264 	if (cmdbuf_page_addr)
265 		host1x_bo_kunmap(cmdbuf, last_page, cmdbuf_page_addr);
266 
267 	return 0;
268 }
269 
check_reloc(struct host1x_reloc * reloc,struct host1x_bo * cmdbuf,unsigned int offset)270 static bool check_reloc(struct host1x_reloc *reloc, struct host1x_bo *cmdbuf,
271 			unsigned int offset)
272 {
273 	offset *= sizeof(u32);
274 
275 	if (reloc->cmdbuf.bo != cmdbuf || reloc->cmdbuf.offset != offset)
276 		return false;
277 
278 	return true;
279 }
280 
281 struct host1x_firewall {
282 	struct host1x_job *job;
283 	struct device *dev;
284 
285 	unsigned int num_relocs;
286 	struct host1x_reloc *reloc;
287 
288 	struct host1x_bo *cmdbuf;
289 	unsigned int offset;
290 
291 	u32 words;
292 	u32 class;
293 	u32 reg;
294 	u32 mask;
295 	u32 count;
296 };
297 
check_register(struct host1x_firewall * fw,unsigned long offset)298 static int check_register(struct host1x_firewall *fw, unsigned long offset)
299 {
300 	if (fw->job->is_addr_reg(fw->dev, fw->class, offset)) {
301 		if (!fw->num_relocs)
302 			return -EINVAL;
303 
304 		if (!check_reloc(fw->reloc, fw->cmdbuf, fw->offset))
305 			return -EINVAL;
306 
307 		fw->num_relocs--;
308 		fw->reloc++;
309 	}
310 
311 	return 0;
312 }
313 
check_mask(struct host1x_firewall * fw)314 static int check_mask(struct host1x_firewall *fw)
315 {
316 	u32 mask = fw->mask;
317 	u32 reg = fw->reg;
318 	int ret;
319 
320 	while (mask) {
321 		if (fw->words == 0)
322 			return -EINVAL;
323 
324 		if (mask & 1) {
325 			ret = check_register(fw, reg);
326 			if (ret < 0)
327 				return ret;
328 
329 			fw->words--;
330 			fw->offset++;
331 		}
332 		mask >>= 1;
333 		reg++;
334 	}
335 
336 	return 0;
337 }
338 
check_incr(struct host1x_firewall * fw)339 static int check_incr(struct host1x_firewall *fw)
340 {
341 	u32 count = fw->count;
342 	u32 reg = fw->reg;
343 	int ret;
344 
345 	while (count) {
346 		if (fw->words == 0)
347 			return -EINVAL;
348 
349 		ret = check_register(fw, reg);
350 		if (ret < 0)
351 			return ret;
352 
353 		reg++;
354 		fw->words--;
355 		fw->offset++;
356 		count--;
357 	}
358 
359 	return 0;
360 }
361 
check_nonincr(struct host1x_firewall * fw)362 static int check_nonincr(struct host1x_firewall *fw)
363 {
364 	u32 count = fw->count;
365 	int ret;
366 
367 	while (count) {
368 		if (fw->words == 0)
369 			return -EINVAL;
370 
371 		ret = check_register(fw, fw->reg);
372 		if (ret < 0)
373 			return ret;
374 
375 		fw->words--;
376 		fw->offset++;
377 		count--;
378 	}
379 
380 	return 0;
381 }
382 
validate(struct host1x_firewall * fw,struct host1x_job_gather * g)383 static int validate(struct host1x_firewall *fw, struct host1x_job_gather *g)
384 {
385 	u32 *cmdbuf_base = (u32 *)fw->job->gather_copy_mapped +
386 		(g->offset / sizeof(u32));
387 	int err = 0;
388 
389 	if (!fw->job->is_addr_reg)
390 		return 0;
391 
392 	fw->words = g->words;
393 	fw->cmdbuf = g->bo;
394 	fw->offset = 0;
395 
396 	while (fw->words && !err) {
397 		u32 word = cmdbuf_base[fw->offset];
398 		u32 opcode = (word & 0xf0000000) >> 28;
399 
400 		fw->mask = 0;
401 		fw->reg = 0;
402 		fw->count = 0;
403 		fw->words--;
404 		fw->offset++;
405 
406 		switch (opcode) {
407 		case 0:
408 			fw->class = word >> 6 & 0x3ff;
409 			fw->mask = word & 0x3f;
410 			fw->reg = word >> 16 & 0xfff;
411 			err = check_mask(fw);
412 			if (err)
413 				goto out;
414 			break;
415 		case 1:
416 			fw->reg = word >> 16 & 0xfff;
417 			fw->count = word & 0xffff;
418 			err = check_incr(fw);
419 			if (err)
420 				goto out;
421 			break;
422 
423 		case 2:
424 			fw->reg = word >> 16 & 0xfff;
425 			fw->count = word & 0xffff;
426 			err = check_nonincr(fw);
427 			if (err)
428 				goto out;
429 			break;
430 
431 		case 3:
432 			fw->mask = word & 0xffff;
433 			fw->reg = word >> 16 & 0xfff;
434 			err = check_mask(fw);
435 			if (err)
436 				goto out;
437 			break;
438 		case 4:
439 		case 5:
440 		case 14:
441 			break;
442 		default:
443 			err = -EINVAL;
444 			break;
445 		}
446 	}
447 
448 out:
449 	return err;
450 }
451 
copy_gathers(struct host1x_job * job,struct device * dev)452 static inline int copy_gathers(struct host1x_job *job, struct device *dev)
453 {
454 	struct host1x_firewall fw;
455 	size_t size = 0;
456 	size_t offset = 0;
457 	int i;
458 
459 	fw.job = job;
460 	fw.dev = dev;
461 	fw.reloc = job->relocarray;
462 	fw.num_relocs = job->num_relocs;
463 	fw.class = 0;
464 
465 	for (i = 0; i < job->num_gathers; i++) {
466 		struct host1x_job_gather *g = &job->gathers[i];
467 		size += g->words * sizeof(u32);
468 	}
469 
470 	job->gather_copy_mapped = dma_alloc_writecombine(dev, size,
471 							 &job->gather_copy,
472 							 GFP_KERNEL);
473 	if (!job->gather_copy_mapped) {
474 		job->gather_copy_mapped = NULL;
475 		return -ENOMEM;
476 	}
477 
478 	job->gather_copy_size = size;
479 
480 	for (i = 0; i < job->num_gathers; i++) {
481 		struct host1x_job_gather *g = &job->gathers[i];
482 		void *gather;
483 
484 		/* Copy the gather */
485 		gather = host1x_bo_mmap(g->bo);
486 		memcpy(job->gather_copy_mapped + offset, gather + g->offset,
487 		       g->words * sizeof(u32));
488 		host1x_bo_munmap(g->bo, gather);
489 
490 		/* Store the location in the buffer */
491 		g->base = job->gather_copy;
492 		g->offset = offset;
493 
494 		/* Validate the job */
495 		if (validate(&fw, g))
496 			return -EINVAL;
497 
498 		offset += g->words * sizeof(u32);
499 	}
500 
501 	/* No relocs should remain at this point */
502 	if (fw.num_relocs)
503 		return -EINVAL;
504 
505 	return 0;
506 }
507 
host1x_job_pin(struct host1x_job * job,struct device * dev)508 int host1x_job_pin(struct host1x_job *job, struct device *dev)
509 {
510 	int err;
511 	unsigned int i, j;
512 	struct host1x *host = dev_get_drvdata(dev->parent);
513 	DECLARE_BITMAP(waitchk_mask, host1x_syncpt_nb_pts(host));
514 
515 	bitmap_zero(waitchk_mask, host1x_syncpt_nb_pts(host));
516 	for (i = 0; i < job->num_waitchk; i++) {
517 		u32 syncpt_id = job->waitchk[i].syncpt_id;
518 		if (syncpt_id < host1x_syncpt_nb_pts(host))
519 			set_bit(syncpt_id, waitchk_mask);
520 	}
521 
522 	/* get current syncpt values for waitchk */
523 	for_each_set_bit(i, waitchk_mask, host1x_syncpt_nb_pts(host))
524 		host1x_syncpt_load(host->syncpt + i);
525 
526 	/* pin memory */
527 	err = pin_job(job);
528 	if (!err)
529 		goto out;
530 
531 	/* patch gathers */
532 	for (i = 0; i < job->num_gathers; i++) {
533 		struct host1x_job_gather *g = &job->gathers[i];
534 
535 		/* process each gather mem only once */
536 		if (g->handled)
537 			continue;
538 
539 		g->base = job->gather_addr_phys[i];
540 
541 		for (j = i + 1; j < job->num_gathers; j++)
542 			if (job->gathers[j].bo == g->bo)
543 				job->gathers[j].handled = true;
544 
545 		err = do_relocs(job, g->bo);
546 		if (err)
547 			break;
548 
549 		err = do_waitchks(job, host, g->bo);
550 		if (err)
551 			break;
552 	}
553 
554 	if (IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL) && !err) {
555 		err = copy_gathers(job, dev);
556 		if (err) {
557 			host1x_job_unpin(job);
558 			return err;
559 		}
560 	}
561 
562 out:
563 	wmb();
564 
565 	return err;
566 }
567 EXPORT_SYMBOL(host1x_job_pin);
568 
host1x_job_unpin(struct host1x_job * job)569 void host1x_job_unpin(struct host1x_job *job)
570 {
571 	unsigned int i;
572 
573 	for (i = 0; i < job->num_unpins; i++) {
574 		struct host1x_job_unpin_data *unpin = &job->unpins[i];
575 		host1x_bo_unpin(unpin->bo, unpin->sgt);
576 		host1x_bo_put(unpin->bo);
577 	}
578 	job->num_unpins = 0;
579 
580 	if (job->gather_copy_size)
581 		dma_free_writecombine(job->channel->dev, job->gather_copy_size,
582 				      job->gather_copy_mapped,
583 				      job->gather_copy);
584 }
585 EXPORT_SYMBOL(host1x_job_unpin);
586 
587 /*
588  * Debug routine used to dump job entries
589  */
host1x_job_dump(struct device * dev,struct host1x_job * job)590 void host1x_job_dump(struct device *dev, struct host1x_job *job)
591 {
592 	dev_dbg(dev, "    SYNCPT_ID   %d\n", job->syncpt_id);
593 	dev_dbg(dev, "    SYNCPT_VAL  %d\n", job->syncpt_end);
594 	dev_dbg(dev, "    FIRST_GET   0x%x\n", job->first_get);
595 	dev_dbg(dev, "    TIMEOUT     %d\n", job->timeout);
596 	dev_dbg(dev, "    NUM_SLOTS   %d\n", job->num_slots);
597 	dev_dbg(dev, "    NUM_HANDLES %d\n", job->num_unpins);
598 }
599