arm64/mm/dma-mapping-noalias.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Support for uncached DMA mappings.
 * Part of Cortex-A510 erratum 2454944 workaround.
 *
 * Copyright (C) 2022-2023 ARM Ltd.
 * Author: Robin Murphy <robin.murphy@arm.com>
 *	   Activating swiotlb + disabling lazy vunmap: Beata Michalska
 */
#include <linux/dma-direct.h>
#include <linux/dma-map-ops.h>
#include <linux/iommu.h>
#include <linux/slab.h>
#include <linux/swiotlb.h>
#include <linux/vmalloc.h>
#include <asm/cacheflush.h>

/*
 * Bits [58:55] of the translation table descriptor are being reserved
 * by the architecture for software use purposes. With the assumption that
 * those should not be used on linear map addresses (which is not without
 * any guarantee though), those bits are being leveraged to trace potential
 * cacheable aliases. This is still far from being perfect, to say at least:
 * ... categorically the worst, but oh well, needs must...
 */
#define REFCOUNT_INC BIT(55)
#define PTE_REFCOUNT(pte) (((pte) >> 55) & 0xf)

static int pte_set_nc(pte_t *ptep, unsigned long addr, void *data)
{
	pteval_t old_pte, new_pte, pte;
	unsigned int refcount;

	pte = pte_val(READ_ONCE(*ptep));
	do {
		/* Avoid racing against the transient invalid state */
		old_pte = pte | PTE_VALID;
		new_pte = old_pte + REFCOUNT_INC;
		refcount = PTE_REFCOUNT(pte);
		if (WARN_ON(refcount == 15))
			return -EINVAL;
		if (refcount == 0) {
			new_pte &= ~(PTE_ATTRINDX_MASK | PTE_VALID);
			new_pte |= PTE_ATTRINDX(MT_NORMAL_NC);
		}
		pte = cmpxchg_relaxed(&pte_val(*ptep), old_pte, new_pte);
	} while (pte != old_pte);

	*(unsigned int *)data = refcount;
	if (refcount)
		return 0;

	flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
	WRITE_ONCE(*ptep, __pte(new_pte | PTE_VALID));
	return 0;
}

static int pte_clear_nc(pte_t *ptep, unsigned long addr, void *data)
{
	pteval_t old_pte, new_pte, pte;
	unsigned int refcount;

	pte = pte_val(READ_ONCE(*ptep));
	do {
		old_pte = pte | PTE_VALID;
		new_pte = old_pte - REFCOUNT_INC;
		refcount = PTE_REFCOUNT(pte);
		if (WARN_ON(refcount == 0))
			return -EINVAL;
		if (refcount == 1) {
			new_pte &= ~(PTE_ATTRINDX_MASK | PTE_VALID);
			new_pte |= PTE_ATTRINDX(MT_NORMAL_TAGGED);
		}
		pte = cmpxchg_relaxed(&pte_val(*ptep), old_pte, new_pte);
	} while (pte != old_pte);

	if (refcount > 1)
		return 0;

	flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
	WRITE_ONCE(*ptep, __pte(new_pte | PTE_VALID));
	return 0;
}

static int set_nc(void *addr, size_t size)
{
	unsigned int count;
	int ret = apply_to_existing_page_range(&init_mm, (unsigned long)addr,
					       size, pte_set_nc, &count);

	WARN_RATELIMIT(count == 0 && page_mapped(virt_to_page(addr)),
		       "changing linear mapping but cacheable aliases may still exist\n");
	dsb(ishst);
	isb();
	__flush_dcache_area(addr, size);
	return ret;
}

static int clear_nc(void *addr, size_t size)
{
	int ret = apply_to_existing_page_range(&init_mm, (unsigned long)addr,
					       size, pte_clear_nc, NULL);
	dsb(ishst);
	isb();
	__inval_dcache_area(addr, size);
	return ret;
}

static phys_addr_t __arm64_noalias_map(struct device *dev, phys_addr_t phys,
				       size_t size, enum dma_data_direction dir,
				       unsigned long attrs, bool bounce)
{
	bounce = bounce || (phys | size) & ~PAGE_MASK;
	if (bounce) {
		phys = swiotlb_tbl_map_single(dev, phys, size, PAGE_ALIGN(size),
					      dir, attrs);
		if (phys == DMA_MAPPING_ERROR)
			return DMA_MAPPING_ERROR;
	}
	if (set_nc(phys_to_virt(phys & PAGE_MASK), PAGE_ALIGN(size)))
		goto out_unmap;

	return phys;
out_unmap:
	if (bounce)
		swiotlb_tbl_unmap_single(dev, phys, size, PAGE_ALIGN(size), dir,
					 attrs | DMA_ATTR_SKIP_CPU_SYNC);
	return DMA_MAPPING_ERROR;

}

static void __arm64_noalias_unmap(struct device *dev, phys_addr_t phys, size_t size,
				  enum dma_data_direction dir, unsigned long attrs)
{
	clear_nc(phys_to_virt(phys & PAGE_MASK), PAGE_ALIGN(size));
	if (is_swiotlb_buffer(phys))
		swiotlb_tbl_unmap_single(dev, phys, size, PAGE_ALIGN(size), dir, attrs);
}

static void __arm64_noalias_sync_for_device(struct device *dev, phys_addr_t phys,
					    size_t size, enum dma_data_direction dir)
{
	if (is_swiotlb_buffer(phys))
		swiotlb_tbl_sync_single(dev, phys, size, dir, SYNC_FOR_DEVICE);
	else
		arch_sync_dma_for_device(phys, size, dir);
}

static void __arm64_noalias_sync_for_cpu(struct device *dev, phys_addr_t phys,
					 size_t size, enum dma_data_direction dir)
{
	if (is_swiotlb_buffer(phys))
		swiotlb_tbl_sync_single(dev, phys, size, dir, SYNC_FOR_CPU);
	else
		arch_sync_dma_for_cpu(phys, size, dir);
}

static void *arm64_noalias_alloc(struct device *dev, size_t size,
				 dma_addr_t *dma_addr, gfp_t gfp, unsigned long attrs)
{
	struct page *page;
	void *ret;

	if (attrs & DMA_ATTR_NO_WARN)
		gfp |= __GFP_NOWARN;

	size = PAGE_ALIGN(size);
	page = dma_direct_alloc_pages(dev, size, dma_addr, 0, gfp & ~__GFP_ZERO);
	if (!page)
		return NULL;

	ret = page_address(page);
	if (set_nc(ret, size)) {
		dma_direct_free_pages(dev, size, page, *dma_addr, 0);
		return NULL;
	}
	return ret;
}

static void arm64_noalias_free(struct device *dev, size_t size, void *cpu_addr,
			       dma_addr_t dma_addr, unsigned long attrs)
{
	size = PAGE_ALIGN(size);
	clear_nc(cpu_addr, size);
	dma_direct_free_pages(dev, size, virt_to_page(cpu_addr), dma_addr, 0);
}

static dma_addr_t arm64_noalias_map_page(struct device *dev, struct page *page,
					 unsigned long offset, size_t size,
					 enum dma_data_direction dir, unsigned long attrs)
{
	phys_addr_t phys = page_to_phys(page) + offset;
	bool bounce = !dma_capable(dev, phys_to_dma(dev, phys), size, true);

	if (!bounce && dir == DMA_TO_DEVICE) {
		arch_sync_dma_for_device(phys, size, dir);
		return phys_to_dma(dev, phys);
	}

	bounce = bounce || page_mapped(page);
	phys = __arm64_noalias_map(dev, phys, size, dir, attrs, bounce);
	if (phys == DMA_MAPPING_ERROR)
		return DMA_MAPPING_ERROR;

	return phys_to_dma(dev, phys);
}

static void arm64_noalias_unmap_page(struct device *dev, dma_addr_t dma_addr,
				     size_t size, enum dma_data_direction dir,
				     unsigned long attrs)
{
	if (dir == DMA_TO_DEVICE)
		return;
	__arm64_noalias_unmap(dev, dma_to_phys(dev, dma_addr), size, dir, attrs);
}

static void arm64_noalias_unmap_sg(struct device *dev, struct scatterlist *sgl, int nents,
				   enum dma_data_direction dir, unsigned long attrs)
{
	struct scatterlist *sg;
	int i;

	if (dir == DMA_TO_DEVICE)
		return;
	for_each_sg(sgl, sg, nents, i)
		__arm64_noalias_unmap(dev, dma_to_phys(dev, sg->dma_address),
				      sg->length, dir, attrs);
}

static int arm64_noalias_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
				enum dma_data_direction dir, unsigned long attrs)
{
	int i;
	struct scatterlist *sg;

	for_each_sg(sgl, sg, nents, i) {
		sg->dma_address = arm64_noalias_map_page(dev, sg_page(sg), sg->offset,
							 sg->length, dir, attrs);
		if (sg->dma_address == DMA_MAPPING_ERROR)
			goto out_unmap;
		sg->dma_length = sg->length;
	}

	return nents;

out_unmap:
	arm64_noalias_unmap_sg(dev, sgl, i, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC);
	return 0;
}

static void arm64_noalias_sync_single_for_device(struct device *dev, dma_addr_t addr,
						 size_t size, enum dma_data_direction dir)
{
	__arm64_noalias_sync_for_device(dev, dma_to_phys(dev, addr), size, dir);
}

static void arm64_noalias_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
					      size_t size, enum dma_data_direction dir)
{
	__arm64_noalias_sync_for_cpu(dev, dma_to_phys(dev, addr), size, dir);
}

static void arm64_noalias_sync_sg_for_device(struct device *dev, struct scatterlist *sgl,
					     int nents, enum dma_data_direction dir)
{
	struct scatterlist *sg;
	int i;

	for_each_sg(sgl, sg, nents, i)
		arm64_noalias_sync_single_for_device(dev, sg->dma_address, sg->length, dir);
}

static void arm64_noalias_sync_sg_for_cpu(struct device *dev, struct scatterlist *sgl,
					  int nents, enum dma_data_direction dir)
{
	struct scatterlist *sg;
	int i;

	for_each_sg(sgl, sg, nents, i)
		arm64_noalias_sync_single_for_cpu(dev, sg->dma_address, sg->length, dir);
}

static const struct dma_map_ops arm64_noalias_ops = {
	.alloc = arm64_noalias_alloc,
	.free = arm64_noalias_free,
	.alloc_pages = dma_common_alloc_pages,
	.free_pages = dma_common_free_pages,
	.mmap = dma_common_mmap,
	.get_sgtable = dma_common_get_sgtable,
	.map_page = arm64_noalias_map_page,
	.unmap_page = arm64_noalias_unmap_page,
	.map_sg = arm64_noalias_map_sg,
	.unmap_sg = arm64_noalias_unmap_sg,
	.sync_single_for_cpu = arm64_noalias_sync_single_for_cpu,
	.sync_single_for_device = arm64_noalias_sync_single_for_device,
	.sync_sg_for_cpu = arm64_noalias_sync_sg_for_cpu,
	.sync_sg_for_device = arm64_noalias_sync_sg_for_device,
	.dma_supported = dma_direct_supported,
	.get_required_mask = dma_direct_get_required_mask,
	.max_mapping_size = swiotlb_max_mapping_size,
};

#ifdef CONFIG_IOMMU_DMA
static const struct dma_map_ops *iommu_dma_ops;

static void *arm64_iommu_alloc(struct device *dev, size_t size,
			       dma_addr_t *dma_addr, gfp_t gfp, unsigned long attrs)
{
	struct page **pages;
	void *ret;
	int i;

	size = PAGE_ALIGN(size);
	if (!gfpflags_allow_blocking(gfp) || (attrs & DMA_ATTR_FORCE_CONTIGUOUS)) {
		ret = dma_common_alloc_pages(dev, size, dma_addr, 0, gfp);
		return ret ? page_address(ret) : NULL;
	}

	ret = iommu_dma_ops->alloc(dev, size, dma_addr, gfp, attrs);
	if (ret) {
		pages = dma_common_find_pages(ret);
		for (i = 0; i < size / PAGE_SIZE; i++)
			if (set_nc(page_address(pages[i]), PAGE_SIZE))
				goto err;
	}
	return ret;

err:
	while (i--)
		clear_nc(page_address(pages[i]), PAGE_SIZE);
	iommu_dma_ops->free(dev, size, ret, *dma_addr, attrs);
	return NULL;
}

static void arm64_iommu_free(struct device *dev, size_t size, void *cpu_addr,
			     dma_addr_t dma_addr, unsigned long attrs)
{
	struct page **pages = dma_common_find_pages(cpu_addr);
	int i;

	size = PAGE_ALIGN(size);
	if (!pages)
		return dma_common_free_pages(dev, size, virt_to_page(cpu_addr), dma_addr, 0);

	for (i = 0; i < size / PAGE_SIZE; i++)
		clear_nc(page_address(pages[i]), PAGE_SIZE);
	iommu_dma_ops->free(dev, size, cpu_addr, dma_addr, attrs);
}

static dma_addr_t arm64_iommu_map_page(struct device *dev, struct page *page,
				       unsigned long offset, size_t size,
				       enum dma_data_direction dir, unsigned long attrs)
{
	phys_addr_t phys = page_to_phys(page) + offset;
	dma_addr_t ret;

	if (dir == DMA_TO_DEVICE)
		return iommu_dma_ops->map_page(dev, page, offset, size, dir, attrs);

	phys = __arm64_noalias_map(dev, phys, size, dir, attrs, page_mapped(page));
	if (phys == DMA_MAPPING_ERROR)
		return DMA_MAPPING_ERROR;

	attrs |= DMA_ATTR_SKIP_CPU_SYNC;
	ret = iommu_dma_ops->map_page(dev, phys_to_page(phys), offset_in_page(phys),
				       size, dir, attrs);
	if (ret == DMA_MAPPING_ERROR)
		__arm64_noalias_unmap(dev, phys, size, dir, attrs);
	return ret;
}

static void arm64_iommu_unmap_page(struct device *dev, dma_addr_t addr, size_t size,
				   enum dma_data_direction dir, unsigned long attrs)
{
	phys_addr_t phys;

	if (dir == DMA_TO_DEVICE)
		return iommu_dma_ops->unmap_page(dev, addr, size, dir, attrs);

	phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), addr);
	iommu_dma_ops->unmap_page(dev, addr, size, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC);
	__arm64_noalias_unmap(dev, phys, size, dir, attrs);
}

static int arm64_iommu_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
			      enum dma_data_direction dir, unsigned long attrs)
{
	int i, ret;
	struct scatterlist *sg;
	phys_addr_t *orig_phys;

	if (dir == DMA_TO_DEVICE)
		return iommu_dma_ops->map_sg(dev, sgl, nents, dir, attrs);

	orig_phys = kmalloc_array(nents, sizeof(*orig_phys), GFP_ATOMIC);
	if (!orig_phys)
		return 0;

	for_each_sg(sgl, sg, nents, i) {
		phys_addr_t phys = sg_phys(sg);
		/*
		 * Note we do not have the page_mapped() check here, since
		 * bouncing plays complete havoc with dma-buf imports. Those
		 * may well be mapped in userspace, but we hope and pray that
		 * it's via dma_mmap_attrs() so any such mappings are safely
		 * non-cacheable. DO NOT allow a block device or other similar
		 * scatterlist user to get here (disable IOMMUs if necessary),
		 * since we can't mitigate for both conflicting use-cases.
		 */
		phys = __arm64_noalias_map(dev, phys, sg->length, dir, attrs, false);
		if (phys == DMA_MAPPING_ERROR)
			goto out_unmap;

		orig_phys[i] = sg_phys(sg);
		sg_assign_page(sg, phys_to_page(phys));
		sg->offset = offset_in_page(phys);
	}
	ret = iommu_dma_ops->map_sg(dev, sgl, nents, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC);
	if (ret <= 0)
		goto out_unmap;

	for_each_sg(sgl, sg, nents, i) {
		sg_assign_page(sg, phys_to_page(orig_phys[i]));
		sg->offset = offset_in_page(orig_phys[i]);
	}

	kfree(orig_phys);
	return ret;

out_unmap:
	for_each_sg(sgl, sg, nents, i) {
		__arm64_noalias_unmap(dev, sg_phys(sg), sg->length, dir, attrs);
		sg_assign_page(sg, phys_to_page(orig_phys[i]));
		sg->offset = offset_in_page(orig_phys[i]);
	}
	kfree(orig_phys);
	return 0;
}

static void arm64_iommu_unmap_sg(struct device *dev, struct scatterlist *sgl, int nents,
				 enum dma_data_direction dir, unsigned long attrs)
{
	struct iommu_domain *domain;
	struct scatterlist *sg, *tmp;
	dma_addr_t iova;
	int i;

	if (dir == DMA_TO_DEVICE)
		return iommu_dma_ops->unmap_sg(dev, sgl, nents, dir, attrs);

	domain = iommu_get_dma_domain(dev);
	iova = sgl->dma_address;
	tmp = sgl;
	for_each_sg(sgl, sg, nents, i) {
		phys_addr_t phys = iommu_iova_to_phys(domain, iova);

		__arm64_noalias_unmap(dev, phys, sg->length, dir, attrs);
		iova += sg->length;
		if (iova == tmp->dma_address + tmp->dma_length && !sg_is_last(tmp)) {
			tmp = sg_next(tmp);
			iova = tmp->dma_address;
		}
	}
	iommu_dma_ops->unmap_sg(dev, sgl, nents, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC);
}

static void arm64_iommu_sync_single_for_device(struct device *dev, dma_addr_t addr,
					       size_t size, enum dma_data_direction dir)
{
	phys_addr_t phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), addr);

	__arm64_noalias_sync_for_device(dev, phys, size, dir);
}

static void arm64_iommu_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
					    size_t size, enum dma_data_direction dir)
{
	phys_addr_t phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), addr);

	__arm64_noalias_sync_for_cpu(dev, phys, size, dir);
}

static void arm64_iommu_sync_sg_for_device(struct device *dev, struct scatterlist *sgl,
					   int nents, enum dma_data_direction dir)
{
	struct iommu_domain *domain = iommu_get_dma_domain(dev);
	struct scatterlist *sg, *tmp = sgl;
	dma_addr_t iova = sgl->dma_address;
	int i;

	for_each_sg(sgl, sg, nents, i) {
		phys_addr_t phys = iommu_iova_to_phys(domain, iova);

		__arm64_noalias_sync_for_device(dev, phys, sg->length, dir);
		iova += sg->length;
		if (iova == tmp->dma_address + tmp->dma_length && !sg_is_last(tmp)) {
			tmp = sg_next(tmp);
			iova = tmp->dma_address;
		}
	}
}

static void arm64_iommu_sync_sg_for_cpu(struct device *dev, struct scatterlist *sgl,
					int nents, enum dma_data_direction dir)
{
	struct iommu_domain *domain = iommu_get_dma_domain(dev);
	struct scatterlist *sg, *tmp = sgl;
	dma_addr_t iova = sgl->dma_address;
	int i;

	for_each_sg(sgl, sg, nents, i) {
		phys_addr_t phys = iommu_iova_to_phys(domain, iova);

		__arm64_noalias_sync_for_cpu(dev, phys, sg->length, dir);
		iova += sg->length;
		if (iova == tmp->dma_address + tmp->dma_length && !sg_is_last(tmp)) {
			tmp = sg_next(tmp);
			iova = tmp->dma_address;
		}
	}
}

static struct dma_map_ops arm64_iommu_ops = {
	.alloc = arm64_iommu_alloc,
	.free = arm64_iommu_free,
	.alloc_pages = dma_common_alloc_pages,
	.free_pages = dma_common_free_pages,
	.map_page = arm64_iommu_map_page,
	.unmap_page = arm64_iommu_unmap_page,
	.map_sg = arm64_iommu_map_sg,
	.unmap_sg = arm64_iommu_unmap_sg,
	.sync_single_for_cpu = arm64_iommu_sync_single_for_cpu,
	.sync_single_for_device = arm64_iommu_sync_single_for_device,
	.sync_sg_for_cpu = arm64_iommu_sync_sg_for_cpu,
	.sync_sg_for_device = arm64_iommu_sync_sg_for_device,
};

#endif /* CONFIG_IOMMU_DMA */

static inline void arm64_noalias_prepare(void)
{
	if (!is_swiotlb_active())
		swiotlb_late_init_with_default_size(swiotlb_size_or_default());
	if (lazy_vunmap_enable) {
		lazy_vunmap_enable = false;
		vm_unmap_aliases();
	}
}

void arm64_noalias_setup_dma_ops(struct device *dev)
{
	if (dev_is_dma_coherent(dev))
		return;

	dev_info(dev, "applying no-alias DMA workaround\n");
	if (!dev->dma_ops) {
		dev->dma_ops = &arm64_noalias_ops;
		goto done;
	}

	if (IS_ENABLED(CONFIG_IOMMU_DMA)) {
		dev->dma_ops = &arm64_iommu_ops;
		if (iommu_dma_ops)
			goto done;

		iommu_dma_ops = dev->dma_ops;
		arm64_iommu_ops.mmap = iommu_dma_ops->mmap;
		arm64_iommu_ops.get_sgtable = iommu_dma_ops->get_sgtable;
		arm64_iommu_ops.map_resource = iommu_dma_ops->map_resource;
		arm64_iommu_ops.unmap_resource = iommu_dma_ops->unmap_resource;
		arm64_iommu_ops.get_merge_boundary = iommu_dma_ops->get_merge_boundary;
	}
done:
	arm64_noalias_prepare();
}
EXPORT_SYMBOL_GPL(arm64_noalias_setup_dma_ops);