• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Access to PCI I/O memory from user space programs.
4  *
5  * Copyright IBM Corp. 2014
6  * Author(s): Alexey Ishchuk <aishchuk@linux.vnet.ibm.com>
7  */
8 #include <linux/kernel.h>
9 #include <linux/syscalls.h>
10 #include <linux/init.h>
11 #include <linux/mm.h>
12 #include <linux/errno.h>
13 #include <linux/pci.h>
14 #include <asm/pci_io.h>
15 #include <asm/pci_debug.h>
16 
zpci_err_mmio(u8 cc,u8 status,u64 offset)17 static inline void zpci_err_mmio(u8 cc, u8 status, u64 offset)
18 {
19 	struct {
20 		u64 offset;
21 		u8 cc;
22 		u8 status;
23 	} data = {offset, cc, status};
24 
25 	zpci_err_hex(&data, sizeof(data));
26 }
27 
__pcistb_mio_inuser(void __iomem * ioaddr,const void __user * src,u64 len,u8 * status)28 static inline int __pcistb_mio_inuser(
29 		void __iomem *ioaddr, const void __user *src,
30 		u64 len, u8 *status)
31 {
32 	int cc = -ENXIO;
33 
34 	asm volatile (
35 		"       sacf 256\n"
36 		"0:     .insn   rsy,0xeb00000000d4,%[len],%[ioaddr],%[src]\n"
37 		"1:     ipm     %[cc]\n"
38 		"       srl     %[cc],28\n"
39 		"2:     sacf 768\n"
40 		EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
41 		: [cc] "+d" (cc), [len] "+d" (len)
42 		: [ioaddr] "a" (ioaddr), [src] "Q" (*((u8 __force *)src))
43 		: "cc", "memory");
44 	*status = len >> 24 & 0xff;
45 	return cc;
46 }
47 
__pcistg_mio_inuser(void __iomem * ioaddr,const void __user * src,u64 ulen,u8 * status)48 static inline int __pcistg_mio_inuser(
49 		void __iomem *ioaddr, const void __user *src,
50 		u64 ulen, u8 *status)
51 {
52 	register u64 addr asm("2") = (u64 __force) ioaddr;
53 	register u64 len asm("3") = ulen;
54 	int cc = -ENXIO;
55 	u64 val = 0;
56 	u64 cnt = ulen;
57 	u8 tmp;
58 
59 	/*
60 	 * copy 0 < @len <= 8 bytes from @src into the right most bytes of
61 	 * a register, then store it to PCI at @ioaddr while in secondary
62 	 * address space. pcistg then uses the user mappings.
63 	 */
64 	asm volatile (
65 		"       sacf    256\n"
66 		"0:     llgc    %[tmp],0(%[src])\n"
67 		"       sllg    %[val],%[val],8\n"
68 		"       aghi    %[src],1\n"
69 		"       ogr     %[val],%[tmp]\n"
70 		"       brctg   %[cnt],0b\n"
71 		"1:     .insn   rre,0xb9d40000,%[val],%[ioaddr]\n"
72 		"2:     ipm     %[cc]\n"
73 		"       srl     %[cc],28\n"
74 		"3:     sacf    768\n"
75 		EX_TABLE(0b, 3b) EX_TABLE(1b, 3b) EX_TABLE(2b, 3b)
76 		:
77 		[src] "+a" (src), [cnt] "+d" (cnt),
78 		[val] "+d" (val), [tmp] "=d" (tmp),
79 		[len] "+d" (len), [cc] "+d" (cc),
80 		[ioaddr] "+a" (addr)
81 		:: "cc", "memory");
82 	*status = len >> 24 & 0xff;
83 
84 	/* did we read everything from user memory? */
85 	if (!cc && cnt != 0)
86 		cc = -EFAULT;
87 
88 	return cc;
89 }
90 
__memcpy_toio_inuser(void __iomem * dst,const void __user * src,size_t n)91 static inline int __memcpy_toio_inuser(void __iomem *dst,
92 				   const void __user *src, size_t n)
93 {
94 	int size, rc = 0;
95 	u8 status = 0;
96 	mm_segment_t old_fs;
97 
98 	if (!src)
99 		return -EINVAL;
100 
101 	old_fs = enable_sacf_uaccess();
102 	while (n > 0) {
103 		size = zpci_get_max_write_size((u64 __force) dst,
104 					       (u64 __force) src, n,
105 					       ZPCI_MAX_WRITE_SIZE);
106 		if (size > 8) /* main path */
107 			rc = __pcistb_mio_inuser(dst, src, size, &status);
108 		else
109 			rc = __pcistg_mio_inuser(dst, src, size, &status);
110 		if (rc)
111 			break;
112 		src += size;
113 		dst += size;
114 		n -= size;
115 	}
116 	disable_sacf_uaccess(old_fs);
117 	if (rc)
118 		zpci_err_mmio(rc, status, (__force u64) dst);
119 	return rc;
120 }
121 
get_pfn(unsigned long user_addr,unsigned long access,unsigned long * pfn)122 static long get_pfn(unsigned long user_addr, unsigned long access,
123 		    unsigned long *pfn)
124 {
125 	struct vm_area_struct *vma;
126 	long ret;
127 
128 	mmap_read_lock(current->mm);
129 	ret = -EINVAL;
130 	vma = find_vma(current->mm, user_addr);
131 	if (!vma || user_addr < vma->vm_start)
132 		goto out;
133 	ret = -EACCES;
134 	if (!(vma->vm_flags & access))
135 		goto out;
136 	ret = follow_pfn(vma, user_addr, pfn);
137 out:
138 	mmap_read_unlock(current->mm);
139 	return ret;
140 }
141 
SYSCALL_DEFINE3(s390_pci_mmio_write,unsigned long,mmio_addr,const void __user *,user_buffer,size_t,length)142 SYSCALL_DEFINE3(s390_pci_mmio_write, unsigned long, mmio_addr,
143 		const void __user *, user_buffer, size_t, length)
144 {
145 	u8 local_buf[64];
146 	void __iomem *io_addr;
147 	void *buf;
148 	unsigned long pfn;
149 	long ret;
150 
151 	if (!zpci_is_enabled())
152 		return -ENODEV;
153 
154 	if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length)
155 		return -EINVAL;
156 
157 	/*
158 	 * We only support write access to MIO capable devices if we are on
159 	 * a MIO enabled system. Otherwise we would have to check for every
160 	 * address if it is a special ZPCI_ADDR and would have to do
161 	 * a get_pfn() which we don't need for MIO capable devices.  Currently
162 	 * ISM devices are the only devices without MIO support and there is no
163 	 * known need for accessing these from userspace.
164 	 */
165 	if (static_branch_likely(&have_mio)) {
166 		ret = __memcpy_toio_inuser((void  __iomem *) mmio_addr,
167 					user_buffer,
168 					length);
169 		return ret;
170 	}
171 
172 	if (length > 64) {
173 		buf = kmalloc(length, GFP_KERNEL);
174 		if (!buf)
175 			return -ENOMEM;
176 	} else
177 		buf = local_buf;
178 
179 	ret = get_pfn(mmio_addr, VM_WRITE, &pfn);
180 	if (ret)
181 		goto out;
182 	io_addr = (void __iomem *)((pfn << PAGE_SHIFT) |
183 			(mmio_addr & ~PAGE_MASK));
184 
185 	ret = -EFAULT;
186 	if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE)
187 		goto out;
188 
189 	if (copy_from_user(buf, user_buffer, length))
190 		goto out;
191 
192 	ret = zpci_memcpy_toio(io_addr, buf, length);
193 out:
194 	if (buf != local_buf)
195 		kfree(buf);
196 	return ret;
197 }
198 
__pcilg_mio_inuser(void __user * dst,const void __iomem * ioaddr,u64 ulen,u8 * status)199 static inline int __pcilg_mio_inuser(
200 		void __user *dst, const void __iomem *ioaddr,
201 		u64 ulen, u8 *status)
202 {
203 	register u64 addr asm("2") = (u64 __force) ioaddr;
204 	register u64 len asm("3") = ulen;
205 	u64 cnt = ulen;
206 	int shift = ulen * 8;
207 	int cc = -ENXIO;
208 	u64 val, tmp;
209 
210 	/*
211 	 * read 0 < @len <= 8 bytes from the PCI memory mapped at @ioaddr (in
212 	 * user space) into a register using pcilg then store these bytes at
213 	 * user address @dst
214 	 */
215 	asm volatile (
216 		"       sacf    256\n"
217 		"0:     .insn   rre,0xb9d60000,%[val],%[ioaddr]\n"
218 		"1:     ipm     %[cc]\n"
219 		"       srl     %[cc],28\n"
220 		"       ltr     %[cc],%[cc]\n"
221 		"       jne     4f\n"
222 		"2:     ahi     %[shift],-8\n"
223 		"       srlg    %[tmp],%[val],0(%[shift])\n"
224 		"3:     stc     %[tmp],0(%[dst])\n"
225 		"       aghi    %[dst],1\n"
226 		"       brctg   %[cnt],2b\n"
227 		"4:     sacf    768\n"
228 		EX_TABLE(0b, 4b) EX_TABLE(1b, 4b) EX_TABLE(3b, 4b)
229 		:
230 		[cc] "+d" (cc), [val] "=d" (val), [len] "+d" (len),
231 		[dst] "+a" (dst), [cnt] "+d" (cnt), [tmp] "=d" (tmp),
232 		[shift] "+d" (shift)
233 		:
234 		[ioaddr] "a" (addr)
235 		: "cc", "memory");
236 
237 	/* did we write everything to the user space buffer? */
238 	if (!cc && cnt != 0)
239 		cc = -EFAULT;
240 
241 	*status = len >> 24 & 0xff;
242 	return cc;
243 }
244 
__memcpy_fromio_inuser(void __user * dst,const void __iomem * src,unsigned long n)245 static inline int __memcpy_fromio_inuser(void __user *dst,
246 				     const void __iomem *src,
247 				     unsigned long n)
248 {
249 	int size, rc = 0;
250 	u8 status;
251 	mm_segment_t old_fs;
252 
253 	old_fs = enable_sacf_uaccess();
254 	while (n > 0) {
255 		size = zpci_get_max_write_size((u64 __force) src,
256 					       (u64 __force) dst, n,
257 					       ZPCI_MAX_READ_SIZE);
258 		rc = __pcilg_mio_inuser(dst, src, size, &status);
259 		if (rc)
260 			break;
261 		src += size;
262 		dst += size;
263 		n -= size;
264 	}
265 	disable_sacf_uaccess(old_fs);
266 	if (rc)
267 		zpci_err_mmio(rc, status, (__force u64) dst);
268 	return rc;
269 }
270 
SYSCALL_DEFINE3(s390_pci_mmio_read,unsigned long,mmio_addr,void __user *,user_buffer,size_t,length)271 SYSCALL_DEFINE3(s390_pci_mmio_read, unsigned long, mmio_addr,
272 		void __user *, user_buffer, size_t, length)
273 {
274 	u8 local_buf[64];
275 	void __iomem *io_addr;
276 	void *buf;
277 	unsigned long pfn;
278 	long ret;
279 
280 	if (!zpci_is_enabled())
281 		return -ENODEV;
282 
283 	if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length)
284 		return -EINVAL;
285 
286 	/*
287 	 * We only support read access to MIO capable devices if we are on
288 	 * a MIO enabled system. Otherwise we would have to check for every
289 	 * address if it is a special ZPCI_ADDR and would have to do
290 	 * a get_pfn() which we don't need for MIO capable devices.  Currently
291 	 * ISM devices are the only devices without MIO support and there is no
292 	 * known need for accessing these from userspace.
293 	 */
294 	if (static_branch_likely(&have_mio)) {
295 		ret = __memcpy_fromio_inuser(
296 				user_buffer, (const void __iomem *)mmio_addr,
297 				length);
298 		return ret;
299 	}
300 
301 	if (length > 64) {
302 		buf = kmalloc(length, GFP_KERNEL);
303 		if (!buf)
304 			return -ENOMEM;
305 	} else {
306 		buf = local_buf;
307 	}
308 
309 	ret = get_pfn(mmio_addr, VM_READ, &pfn);
310 	if (ret)
311 		goto out;
312 	io_addr = (void __iomem *)((pfn << PAGE_SHIFT) | (mmio_addr & ~PAGE_MASK));
313 
314 	if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE) {
315 		ret = -EFAULT;
316 		goto out;
317 	}
318 	ret = zpci_memcpy_fromio(buf, io_addr, length);
319 	if (ret)
320 		goto out;
321 	if (copy_to_user(user_buffer, buf, length))
322 		ret = -EFAULT;
323 
324 out:
325 	if (buf != local_buf)
326 		kfree(buf);
327 	return ret;
328 }
329