94 static struct dma_fence *panfrost_fence_create(struct panfrost_device *pfdev, int js_num)  in panfrost_fence_create()  argument
97 	struct panfrost_job_slot *js = pfdev->js;  in panfrost_fence_create()
103 	fence->dev = pfdev->ddev;  in panfrost_fence_create()
125 		    (job->pfdev->features.nr_core_groups == 2))  in panfrost_job_get_slot()
127 		if (panfrost_has_hw_issue(job->pfdev, HW_ISSUE_8987))  in panfrost_job_get_slot()
134 static void panfrost_job_write_affinity(struct panfrost_device *pfdev,  in panfrost_job_write_affinity()  argument
145 	affinity = pfdev->features.shader_present;  in panfrost_job_write_affinity()
147 	job_write(pfdev, JS_AFFINITY_NEXT_LO(js), affinity & 0xFFFFFFFF);  in panfrost_job_write_affinity()
148 	job_write(pfdev, JS_AFFINITY_NEXT_HI(js), affinity >> 32);  in panfrost_job_write_affinity()
153 	struct panfrost_device *pfdev = job->pfdev;  in panfrost_job_hw_submit()  local
158 	panfrost_devfreq_record_busy(&pfdev->pfdevfreq);  in panfrost_job_hw_submit()
160 	ret = pm_runtime_get_sync(pfdev->dev);  in panfrost_job_hw_submit()
164 	if (WARN_ON(job_read(pfdev, JS_COMMAND_NEXT(js)))) {  in panfrost_job_hw_submit()
168 	cfg = panfrost_mmu_as_get(pfdev, job->file_priv->mmu);  in panfrost_job_hw_submit()
170 	job_write(pfdev, JS_HEAD_NEXT_LO(js), jc_head & 0xFFFFFFFF);  in panfrost_job_hw_submit()
171 	job_write(pfdev, JS_HEAD_NEXT_HI(js), jc_head >> 32);  in panfrost_job_hw_submit()
173 	panfrost_job_write_affinity(pfdev, job->requirements, js);  in panfrost_job_hw_submit()
181 	if (panfrost_has_hw_feature(pfdev, HW_FEATURE_FLUSH_REDUCTION))  in panfrost_job_hw_submit()
184 	if (panfrost_has_hw_issue(pfdev, HW_ISSUE_10649))  in panfrost_job_hw_submit()
187 	job_write(pfdev, JS_CONFIG_NEXT(js), cfg);  in panfrost_job_hw_submit()
189 	if (panfrost_has_hw_feature(pfdev, HW_FEATURE_FLUSH_REDUCTION))  in panfrost_job_hw_submit()
190 		job_write(pfdev, JS_FLUSH_ID_NEXT(js), job->flush_id);  in panfrost_job_hw_submit()
193 	dev_dbg(pfdev->dev, "JS: Submitting atom %p to js[%d] with head=0x%llx",  in panfrost_job_hw_submit()
196 	job_write(pfdev, JS_COMMAND_NEXT(js), JS_COMMAND_START);  in panfrost_job_hw_submit()
221 	struct panfrost_device *pfdev = job->pfdev;  in panfrost_job_push()  local
227 	mutex_lock(&pfdev->sched_lock);  in panfrost_job_push()
232 		mutex_unlock(&pfdev->sched_lock);  in panfrost_job_push()
238 		mutex_unlock(&pfdev->sched_lock);  in panfrost_job_push()
251 	mutex_unlock(&pfdev->sched_lock);  in panfrost_job_push()
347 	struct panfrost_device *pfdev = job->pfdev;  in panfrost_job_run()  local
354 	pfdev->jobs[slot] = job;  in panfrost_job_run()
356 	fence = panfrost_fence_create(pfdev, slot);  in panfrost_job_run()
369 void panfrost_job_enable_interrupts(struct panfrost_device *pfdev)  in panfrost_job_enable_interrupts()  argument
378 	job_write(pfdev, JOB_INT_CLEAR, irq_mask);  in panfrost_job_enable_interrupts()
379 	job_write(pfdev, JOB_INT_MASK, irq_mask);  in panfrost_job_enable_interrupts()
438 	struct panfrost_device *pfdev = job->pfdev;  in panfrost_job_timedout()  local
448 …dev_err(pfdev->dev, "gpu sched timeout, js=%d, config=0x%x, status=0x%x, head=0x%x, tail=0x%x, sch…  in panfrost_job_timedout()
450 		job_read(pfdev, JS_CONFIG(js)),  in panfrost_job_timedout()
451 		job_read(pfdev, JS_STATUS(js)),  in panfrost_job_timedout()
452 		job_read(pfdev, JS_HEAD_LO(js)),  in panfrost_job_timedout()
453 		job_read(pfdev, JS_TAIL_LO(js)),  in panfrost_job_timedout()
457 	if (!panfrost_scheduler_stop(&pfdev->js->queue[js], sched_job))  in panfrost_job_timedout()
461 	if (!atomic_xchg(&pfdev->reset.pending, 1))  in panfrost_job_timedout()
462 		schedule_work(&pfdev->reset.work);  in panfrost_job_timedout()
474 	struct panfrost_device *pfdev = data;  in panfrost_job_irq_handler()  local
475 	u32 status = job_read(pfdev, JOB_INT_STAT);  in panfrost_job_irq_handler()
478 	dev_dbg(pfdev->dev, "jobslot irq status=%x\n", status);  in panfrost_job_irq_handler()
483 	pm_runtime_mark_last_busy(pfdev->dev);  in panfrost_job_irq_handler()
491 		job_write(pfdev, JOB_INT_CLEAR, mask);  in panfrost_job_irq_handler()
496 			job_write(pfdev, JS_COMMAND_NEXT(j), JS_COMMAND_NOP);  in panfrost_job_irq_handler()
498 			dev_err(pfdev->dev, "js fault, js=%d, status=%s, head=0x%x, tail=0x%x",  in panfrost_job_irq_handler()
500 				panfrost_exception_name(pfdev, job_read(pfdev, JS_STATUS(j))),  in panfrost_job_irq_handler()
501 				job_read(pfdev, JS_HEAD_LO(j)),  in panfrost_job_irq_handler()
502 				job_read(pfdev, JS_TAIL_LO(j)));  in panfrost_job_irq_handler()
511 			old_status = atomic_cmpxchg(&pfdev->js->queue[j].status,  in panfrost_job_irq_handler()
515 				drm_sched_fault(&pfdev->js->queue[j].sched);  in panfrost_job_irq_handler()
521 			spin_lock(&pfdev->js->job_lock);  in panfrost_job_irq_handler()
522 			job = pfdev->jobs[j];  in panfrost_job_irq_handler()
525 				pfdev->jobs[j] = NULL;  in panfrost_job_irq_handler()
527 				panfrost_mmu_as_put(pfdev, job->file_priv->mmu);  in panfrost_job_irq_handler()
528 				panfrost_devfreq_record_idle(&pfdev->pfdevfreq);  in panfrost_job_irq_handler()
531 				pm_runtime_put_autosuspend(pfdev->dev);  in panfrost_job_irq_handler()
533 			spin_unlock(&pfdev->js->job_lock);  in panfrost_job_irq_handler()
544 	struct panfrost_device *pfdev = container_of(work,  in panfrost_reset()  local
569 		pfdev->js->queue[i].sched.timeout = MAX_SCHEDULE_TIMEOUT - 1;  in panfrost_reset()
570 		cancel_delayed_work_sync(&pfdev->js->queue[i].sched.work_tdr);  in panfrost_reset()
571 		panfrost_scheduler_stop(&pfdev->js->queue[i], NULL);  in panfrost_reset()
575 	atomic_set(&pfdev->reset.pending, 0);  in panfrost_reset()
577 	spin_lock_irqsave(&pfdev->js->job_lock, flags);  in panfrost_reset()
579 		if (pfdev->jobs[i]) {  in panfrost_reset()
580 			pm_runtime_put_noidle(pfdev->dev);  in panfrost_reset()
581 			panfrost_devfreq_record_idle(&pfdev->pfdevfreq);  in panfrost_reset()
582 			pfdev->jobs[i] = NULL;  in panfrost_reset()
585 	spin_unlock_irqrestore(&pfdev->js->job_lock, flags);  in panfrost_reset()
587 	panfrost_device_reset(pfdev);  in panfrost_reset()
590 		panfrost_scheduler_start(&pfdev->js->queue[i]);  in panfrost_reset()
595 int panfrost_job_init(struct panfrost_device *pfdev)  in panfrost_job_init()  argument
600 	INIT_WORK(&pfdev->reset.work, panfrost_reset);  in panfrost_job_init()
602 	pfdev->js = js = devm_kzalloc(pfdev->dev, sizeof(*js), GFP_KERNEL);  in panfrost_job_init()
608 	irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "job");  in panfrost_job_init()
612 	ret = devm_request_irq(pfdev->dev, irq, panfrost_job_irq_handler,  in panfrost_job_init()
613 			       IRQF_SHARED, KBUILD_MODNAME "-job", pfdev);  in panfrost_job_init()
615 		dev_err(pfdev->dev, "failed to request job irq");  in panfrost_job_init()
629 			dev_err(pfdev->dev, "Failed to create scheduler: %d.", ret);  in panfrost_job_init()
634 	panfrost_job_enable_interrupts(pfdev);  in panfrost_job_init()
645 void panfrost_job_fini(struct panfrost_device *pfdev)  in panfrost_job_fini()  argument
647 	struct panfrost_job_slot *js = pfdev->js;  in panfrost_job_fini()
650 	job_write(pfdev, JOB_INT_MASK, 0);  in panfrost_job_fini()
661 	struct panfrost_device *pfdev = panfrost_priv->pfdev;  in panfrost_job_open()  local
662 	struct panfrost_job_slot *js = pfdev->js;  in panfrost_job_open()
685 int panfrost_job_is_idle(struct panfrost_device *pfdev)  in panfrost_job_is_idle()  argument
687 	struct panfrost_job_slot *js = pfdev->js;  in panfrost_job_is_idle()