36 	struct tegra_bpmp *bpmp = channel->bpmp;  in channel_to_ops()  local
38 	return bpmp->soc->ops;  in channel_to_ops()
44 	struct tegra_bpmp *bpmp;  in tegra_bpmp_get()  local
53 		bpmp = ERR_PTR(-ENODEV);  in tegra_bpmp_get()
57 	bpmp = platform_get_drvdata(pdev);  in tegra_bpmp_get()
58 	if (!bpmp) {  in tegra_bpmp_get()
59 		bpmp = ERR_PTR(-EPROBE_DEFER);  in tegra_bpmp_get()
66 	return bpmp;  in tegra_bpmp_get()
70 void tegra_bpmp_put(struct tegra_bpmp *bpmp)  in tegra_bpmp_put()  argument
72 	if (bpmp)  in tegra_bpmp_put()
73 		put_device(bpmp->dev);  in tegra_bpmp_put()
80 	struct tegra_bpmp *bpmp = channel->bpmp;  in tegra_bpmp_channel_get_thread_index()  local
84 	count = bpmp->soc->channels.thread.count;  in tegra_bpmp_channel_get_thread_index()
86 	index = channel - channel->bpmp->threaded_channels;  in tegra_bpmp_channel_get_thread_index()
117 	unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;  in tegra_bpmp_wait_response()
163 	unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;  in tegra_bpmp_wait_request_channel_free()
192 static int tegra_bpmp_ring_doorbell(struct tegra_bpmp *bpmp)  in tegra_bpmp_ring_doorbell()  argument
194 	return bpmp->soc->ops->ring_doorbell(bpmp);  in tegra_bpmp_ring_doorbell()
217 	struct tegra_bpmp *bpmp = channel->bpmp;  in tegra_bpmp_channel_read()  local
228 	spin_lock_irqsave(&bpmp->lock, flags);  in tegra_bpmp_channel_read()
230 	clear_bit(index, bpmp->threaded.allocated);  in tegra_bpmp_channel_read()
231 	spin_unlock_irqrestore(&bpmp->lock, flags);  in tegra_bpmp_channel_read()
234 	up(&bpmp->threaded.lock);  in tegra_bpmp_channel_read()
253 tegra_bpmp_write_threaded(struct tegra_bpmp *bpmp, unsigned int mrq,  in tegra_bpmp_write_threaded()  argument
256 	unsigned long timeout = bpmp->soc->channels.thread.timeout;  in tegra_bpmp_write_threaded()
257 	unsigned int count = bpmp->soc->channels.thread.count;  in tegra_bpmp_write_threaded()
263 	err = down_timeout(&bpmp->threaded.lock, usecs_to_jiffies(timeout));  in tegra_bpmp_write_threaded()
267 	spin_lock_irqsave(&bpmp->lock, flags);  in tegra_bpmp_write_threaded()
269 	index = find_first_zero_bit(bpmp->threaded.allocated, count);  in tegra_bpmp_write_threaded()
275 	channel = &bpmp->threaded_channels[index];  in tegra_bpmp_write_threaded()
282 	set_bit(index, bpmp->threaded.allocated);  in tegra_bpmp_write_threaded()
289 	set_bit(index, bpmp->threaded.busy);  in tegra_bpmp_write_threaded()
291 	spin_unlock_irqrestore(&bpmp->lock, flags);  in tegra_bpmp_write_threaded()
295 	clear_bit(index, bpmp->threaded.allocated);  in tegra_bpmp_write_threaded()
297 	spin_unlock_irqrestore(&bpmp->lock, flags);  in tegra_bpmp_write_threaded()
298 	up(&bpmp->threaded.lock);  in tegra_bpmp_write_threaded()
316 int tegra_bpmp_transfer_atomic(struct tegra_bpmp *bpmp,  in tegra_bpmp_transfer_atomic()  argument
328 	channel = bpmp->tx_channel;  in tegra_bpmp_transfer_atomic()
330 	spin_lock(&bpmp->atomic_tx_lock);  in tegra_bpmp_transfer_atomic()
335 		spin_unlock(&bpmp->atomic_tx_lock);  in tegra_bpmp_transfer_atomic()
339 	spin_unlock(&bpmp->atomic_tx_lock);  in tegra_bpmp_transfer_atomic()
341 	err = tegra_bpmp_ring_doorbell(bpmp);  in tegra_bpmp_transfer_atomic()
354 int tegra_bpmp_transfer(struct tegra_bpmp *bpmp,  in tegra_bpmp_transfer()  argument
367 	channel = tegra_bpmp_write_threaded(bpmp, msg->mrq, msg->tx.data,  in tegra_bpmp_transfer()
372 	err = tegra_bpmp_ring_doorbell(bpmp);  in tegra_bpmp_transfer()
376 	timeout = usecs_to_jiffies(bpmp->soc->channels.thread.timeout);  in tegra_bpmp_transfer()
387 static struct tegra_bpmp_mrq *tegra_bpmp_find_mrq(struct tegra_bpmp *bpmp,  in tegra_bpmp_find_mrq()  argument
392 	list_for_each_entry(entry, &bpmp->mrqs, list)  in tegra_bpmp_find_mrq()
403 	struct tegra_bpmp *bpmp = channel->bpmp;  in tegra_bpmp_mrq_return()  local
429 		err = tegra_bpmp_ring_doorbell(bpmp);  in tegra_bpmp_mrq_return()
436 static void tegra_bpmp_handle_mrq(struct tegra_bpmp *bpmp,  in tegra_bpmp_handle_mrq()  argument
443 	spin_lock(&bpmp->lock);  in tegra_bpmp_handle_mrq()
445 	entry = tegra_bpmp_find_mrq(bpmp, mrq);  in tegra_bpmp_handle_mrq()
447 		spin_unlock(&bpmp->lock);  in tegra_bpmp_handle_mrq()
454 	spin_unlock(&bpmp->lock);  in tegra_bpmp_handle_mrq()
457 int tegra_bpmp_request_mrq(struct tegra_bpmp *bpmp, unsigned int mrq,  in tegra_bpmp_request_mrq()  argument
466 	entry = devm_kzalloc(bpmp->dev, sizeof(*entry), GFP_KERNEL);  in tegra_bpmp_request_mrq()
470 	spin_lock_irqsave(&bpmp->lock, flags);  in tegra_bpmp_request_mrq()
475 	list_add(&entry->list, &bpmp->mrqs);  in tegra_bpmp_request_mrq()
477 	spin_unlock_irqrestore(&bpmp->lock, flags);  in tegra_bpmp_request_mrq()
483 void tegra_bpmp_free_mrq(struct tegra_bpmp *bpmp, unsigned int mrq, void *data)  in tegra_bpmp_free_mrq()  argument
488 	spin_lock_irqsave(&bpmp->lock, flags);  in tegra_bpmp_free_mrq()
490 	entry = tegra_bpmp_find_mrq(bpmp, mrq);  in tegra_bpmp_free_mrq()
495 	devm_kfree(bpmp->dev, entry);  in tegra_bpmp_free_mrq()
498 	spin_unlock_irqrestore(&bpmp->lock, flags);  in tegra_bpmp_free_mrq()
502 bool tegra_bpmp_mrq_is_supported(struct tegra_bpmp *bpmp, unsigned int mrq)  in tegra_bpmp_mrq_is_supported()  argument
519 	ret = tegra_bpmp_transfer(bpmp, &msg);  in tegra_bpmp_mrq_is_supported()
542 static int tegra_bpmp_ping(struct tegra_bpmp *bpmp)  in tegra_bpmp_ping()  argument
565 	err = tegra_bpmp_transfer_atomic(bpmp, &msg);  in tegra_bpmp_ping()
570 		dev_dbg(bpmp->dev,  in tegra_bpmp_ping()
579 static int tegra_bpmp_get_firmware_tag_old(struct tegra_bpmp *bpmp, char *tag,  in tegra_bpmp_get_firmware_tag_old()  argument
592 	virt = dma_alloc_coherent(bpmp->dev, TAG_SZ, &phys,  in tegra_bpmp_get_firmware_tag_old()
606 	err = tegra_bpmp_transfer_atomic(bpmp, &msg);  in tegra_bpmp_get_firmware_tag_old()
612 	dma_free_coherent(bpmp->dev, TAG_SZ, virt, phys);  in tegra_bpmp_get_firmware_tag_old()
617 static int tegra_bpmp_get_firmware_tag(struct tegra_bpmp *bpmp, char *tag,  in tegra_bpmp_get_firmware_tag()  argument
620 	if (tegra_bpmp_mrq_is_supported(bpmp, MRQ_QUERY_FW_TAG)) {  in tegra_bpmp_get_firmware_tag()
634 		err = tegra_bpmp_transfer(bpmp, &msg);  in tegra_bpmp_get_firmware_tag()
645 	return tegra_bpmp_get_firmware_tag_old(bpmp, tag, size);  in tegra_bpmp_get_firmware_tag()
658 void tegra_bpmp_handle_rx(struct tegra_bpmp *bpmp)  in tegra_bpmp_handle_rx()  argument
664 	channel = bpmp->rx_channel;  in tegra_bpmp_handle_rx()
665 	count = bpmp->soc->channels.thread.count;  in tegra_bpmp_handle_rx()
666 	busy = bpmp->threaded.busy;  in tegra_bpmp_handle_rx()
669 		tegra_bpmp_handle_mrq(bpmp, channel->ib->code, channel);  in tegra_bpmp_handle_rx()
671 	spin_lock(&bpmp->lock);  in tegra_bpmp_handle_rx()
676 		channel = &bpmp->threaded_channels[i];  in tegra_bpmp_handle_rx()
684 	spin_unlock(&bpmp->lock);  in tegra_bpmp_handle_rx()
689 	struct tegra_bpmp *bpmp;  in tegra_bpmp_probe()  local
694 	bpmp = devm_kzalloc(&pdev->dev, sizeof(*bpmp), GFP_KERNEL);  in tegra_bpmp_probe()
695 	if (!bpmp)  in tegra_bpmp_probe()
698 	bpmp->soc = of_device_get_match_data(&pdev->dev);  in tegra_bpmp_probe()
699 	bpmp->dev = &pdev->dev;  in tegra_bpmp_probe()
701 	INIT_LIST_HEAD(&bpmp->mrqs);  in tegra_bpmp_probe()
702 	spin_lock_init(&bpmp->lock);  in tegra_bpmp_probe()
704 	bpmp->threaded.count = bpmp->soc->channels.thread.count;  in tegra_bpmp_probe()
705 	sema_init(&bpmp->threaded.lock, bpmp->threaded.count);  in tegra_bpmp_probe()
707 	size = BITS_TO_LONGS(bpmp->threaded.count) * sizeof(long);  in tegra_bpmp_probe()
709 	bpmp->threaded.allocated = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);  in tegra_bpmp_probe()
710 	if (!bpmp->threaded.allocated)  in tegra_bpmp_probe()
713 	bpmp->threaded.busy = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);  in tegra_bpmp_probe()
714 	if (!bpmp->threaded.busy)  in tegra_bpmp_probe()
717 	spin_lock_init(&bpmp->atomic_tx_lock);  in tegra_bpmp_probe()
718 	bpmp->tx_channel = devm_kzalloc(&pdev->dev, sizeof(*bpmp->tx_channel),  in tegra_bpmp_probe()
720 	if (!bpmp->tx_channel)  in tegra_bpmp_probe()
723 	bpmp->rx_channel = devm_kzalloc(&pdev->dev, sizeof(*bpmp->rx_channel),  in tegra_bpmp_probe()
725 	if (!bpmp->rx_channel)  in tegra_bpmp_probe()
728 	bpmp->threaded_channels = devm_kcalloc(&pdev->dev, bpmp->threaded.count,  in tegra_bpmp_probe()
729 					       sizeof(*bpmp->threaded_channels),  in tegra_bpmp_probe()
731 	if (!bpmp->threaded_channels)  in tegra_bpmp_probe()
734 	err = bpmp->soc->ops->init(bpmp);  in tegra_bpmp_probe()
738 	err = tegra_bpmp_request_mrq(bpmp, MRQ_PING,  in tegra_bpmp_probe()
739 				     tegra_bpmp_mrq_handle_ping, bpmp);  in tegra_bpmp_probe()
743 	err = tegra_bpmp_ping(bpmp);  in tegra_bpmp_probe()
749 	err = tegra_bpmp_get_firmware_tag(bpmp, tag, sizeof(tag));  in tegra_bpmp_probe()
757 	platform_set_drvdata(pdev, bpmp);  in tegra_bpmp_probe()
764 		err = tegra_bpmp_init_clocks(bpmp);  in tegra_bpmp_probe()
770 		err = tegra_bpmp_init_resets(bpmp);  in tegra_bpmp_probe()
776 		err = tegra_bpmp_init_powergates(bpmp);  in tegra_bpmp_probe()
781 	err = tegra_bpmp_init_debugfs(bpmp);  in tegra_bpmp_probe()
788 	tegra_bpmp_free_mrq(bpmp, MRQ_PING, bpmp);  in tegra_bpmp_probe()
790 	if (bpmp->soc->ops->deinit)  in tegra_bpmp_probe()
791 		bpmp->soc->ops->deinit(bpmp);  in tegra_bpmp_probe()
798 	struct tegra_bpmp *bpmp = dev_get_drvdata(dev);  in tegra_bpmp_resume()  local
800 	if (bpmp->soc->ops->resume)  in tegra_bpmp_resume()
801 		return bpmp->soc->ops->resume(bpmp);  in tegra_bpmp_resume()