1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Access to PCI I/O memory from user space programs.
4 *
5 * Copyright IBM Corp. 2014
6 * Author(s): Alexey Ishchuk <aishchuk@linux.vnet.ibm.com>
7 */
8 #include <linux/kernel.h>
9 #include <linux/syscalls.h>
10 #include <linux/init.h>
11 #include <linux/mm.h>
12 #include <linux/errno.h>
13 #include <linux/pci.h>
14 #include <asm/asm-extable.h>
15 #include <asm/pci_io.h>
16 #include <asm/pci_debug.h>
17
zpci_err_mmio(u8 cc,u8 status,u64 offset)18 static inline void zpci_err_mmio(u8 cc, u8 status, u64 offset)
19 {
20 struct {
21 u64 offset;
22 u8 cc;
23 u8 status;
24 } data = {offset, cc, status};
25
26 zpci_err_hex(&data, sizeof(data));
27 }
28
__pcistb_mio_inuser(void __iomem * ioaddr,const void __user * src,u64 len,u8 * status)29 static inline int __pcistb_mio_inuser(
30 void __iomem *ioaddr, const void __user *src,
31 u64 len, u8 *status)
32 {
33 int cc = -ENXIO;
34
35 asm volatile (
36 " sacf 256\n"
37 "0: .insn rsy,0xeb00000000d4,%[len],%[ioaddr],%[src]\n"
38 "1: ipm %[cc]\n"
39 " srl %[cc],28\n"
40 "2: sacf 768\n"
41 EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
42 : [cc] "+d" (cc), [len] "+d" (len)
43 : [ioaddr] "a" (ioaddr), [src] "Q" (*((u8 __force *)src))
44 : "cc", "memory");
45 *status = len >> 24 & 0xff;
46 return cc;
47 }
48
__pcistg_mio_inuser(void __iomem * ioaddr,const void __user * src,u64 ulen,u8 * status)49 static inline int __pcistg_mio_inuser(
50 void __iomem *ioaddr, const void __user *src,
51 u64 ulen, u8 *status)
52 {
53 union register_pair ioaddr_len = {.even = (u64 __force)ioaddr, .odd = ulen};
54 int cc = -ENXIO;
55 u64 val = 0;
56 u64 cnt = ulen;
57 u8 tmp;
58
59 /*
60 * copy 0 < @len <= 8 bytes from @src into the right most bytes of
61 * a register, then store it to PCI at @ioaddr while in secondary
62 * address space. pcistg then uses the user mappings.
63 */
64 asm volatile (
65 " sacf 256\n"
66 "0: llgc %[tmp],0(%[src])\n"
67 "4: sllg %[val],%[val],8\n"
68 " aghi %[src],1\n"
69 " ogr %[val],%[tmp]\n"
70 " brctg %[cnt],0b\n"
71 "1: .insn rre,0xb9d40000,%[val],%[ioaddr_len]\n"
72 "2: ipm %[cc]\n"
73 " srl %[cc],28\n"
74 "3: sacf 768\n"
75 EX_TABLE(0b, 3b) EX_TABLE(4b, 3b) EX_TABLE(1b, 3b) EX_TABLE(2b, 3b)
76 :
77 [src] "+a" (src), [cnt] "+d" (cnt),
78 [val] "+d" (val), [tmp] "=d" (tmp),
79 [cc] "+d" (cc), [ioaddr_len] "+&d" (ioaddr_len.pair)
80 :: "cc", "memory");
81 *status = ioaddr_len.odd >> 24 & 0xff;
82
83 /* did we read everything from user memory? */
84 if (!cc && cnt != 0)
85 cc = -EFAULT;
86
87 return cc;
88 }
89
__memcpy_toio_inuser(void __iomem * dst,const void __user * src,size_t n)90 static inline int __memcpy_toio_inuser(void __iomem *dst,
91 const void __user *src, size_t n)
92 {
93 int size, rc = 0;
94 u8 status = 0;
95
96 if (!src)
97 return -EINVAL;
98
99 while (n > 0) {
100 size = zpci_get_max_io_size((u64 __force) dst,
101 (u64 __force) src, n,
102 ZPCI_MAX_WRITE_SIZE);
103 if (size > 8) /* main path */
104 rc = __pcistb_mio_inuser(dst, src, size, &status);
105 else
106 rc = __pcistg_mio_inuser(dst, src, size, &status);
107 if (rc)
108 break;
109 src += size;
110 dst += size;
111 n -= size;
112 }
113 if (rc)
114 zpci_err_mmio(rc, status, (__force u64) dst);
115 return rc;
116 }
117
SYSCALL_DEFINE3(s390_pci_mmio_write,unsigned long,mmio_addr,const void __user *,user_buffer,size_t,length)118 SYSCALL_DEFINE3(s390_pci_mmio_write, unsigned long, mmio_addr,
119 const void __user *, user_buffer, size_t, length)
120 {
121 struct follow_pfnmap_args args = { };
122 u8 local_buf[64];
123 void __iomem *io_addr;
124 void *buf;
125 struct vm_area_struct *vma;
126 long ret;
127
128 if (!zpci_is_enabled())
129 return -ENODEV;
130
131 if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length)
132 return -EINVAL;
133
134 /*
135 * We only support write access to MIO capable devices if we are on
136 * a MIO enabled system. Otherwise we would have to check for every
137 * address if it is a special ZPCI_ADDR and would have to do
138 * a pfn lookup which we don't need for MIO capable devices. Currently
139 * ISM devices are the only devices without MIO support and there is no
140 * known need for accessing these from userspace.
141 */
142 if (static_branch_likely(&have_mio)) {
143 ret = __memcpy_toio_inuser((void __iomem *) mmio_addr,
144 user_buffer,
145 length);
146 return ret;
147 }
148
149 if (length > 64) {
150 buf = kmalloc(length, GFP_KERNEL);
151 if (!buf)
152 return -ENOMEM;
153 } else
154 buf = local_buf;
155
156 ret = -EFAULT;
157 if (copy_from_user(buf, user_buffer, length))
158 goto out_free;
159
160 mmap_read_lock(current->mm);
161 ret = -EINVAL;
162 vma = vma_lookup(current->mm, mmio_addr);
163 if (!vma)
164 goto out_unlock_mmap;
165 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
166 goto out_unlock_mmap;
167 ret = -EACCES;
168 if (!(vma->vm_flags & VM_WRITE))
169 goto out_unlock_mmap;
170
171 args.address = mmio_addr;
172 args.vma = vma;
173 ret = follow_pfnmap_start(&args);
174 if (ret) {
175 fixup_user_fault(current->mm, mmio_addr, FAULT_FLAG_WRITE, NULL);
176 ret = follow_pfnmap_start(&args);
177 if (ret)
178 goto out_unlock_mmap;
179 }
180
181 io_addr = (void __iomem *)((args.pfn << PAGE_SHIFT) |
182 (mmio_addr & ~PAGE_MASK));
183
184 if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE)
185 goto out_unlock_pt;
186
187 ret = zpci_memcpy_toio(io_addr, buf, length);
188 out_unlock_pt:
189 follow_pfnmap_end(&args);
190 out_unlock_mmap:
191 mmap_read_unlock(current->mm);
192 out_free:
193 if (buf != local_buf)
194 kfree(buf);
195 return ret;
196 }
197
__pcilg_mio_inuser(void __user * dst,const void __iomem * ioaddr,u64 ulen,u8 * status)198 static inline int __pcilg_mio_inuser(
199 void __user *dst, const void __iomem *ioaddr,
200 u64 ulen, u8 *status)
201 {
202 union register_pair ioaddr_len = {.even = (u64 __force)ioaddr, .odd = ulen};
203 u64 cnt = ulen;
204 int shift = ulen * 8;
205 int cc = -ENXIO;
206 u64 val, tmp;
207
208 /*
209 * read 0 < @len <= 8 bytes from the PCI memory mapped at @ioaddr (in
210 * user space) into a register using pcilg then store these bytes at
211 * user address @dst
212 */
213 asm volatile (
214 " sacf 256\n"
215 "0: .insn rre,0xb9d60000,%[val],%[ioaddr_len]\n"
216 "1: ipm %[cc]\n"
217 " srl %[cc],28\n"
218 " ltr %[cc],%[cc]\n"
219 " jne 4f\n"
220 "2: ahi %[shift],-8\n"
221 " srlg %[tmp],%[val],0(%[shift])\n"
222 "3: stc %[tmp],0(%[dst])\n"
223 "5: aghi %[dst],1\n"
224 " brctg %[cnt],2b\n"
225 "4: sacf 768\n"
226 EX_TABLE(0b, 4b) EX_TABLE(1b, 4b) EX_TABLE(3b, 4b) EX_TABLE(5b, 4b)
227 :
228 [ioaddr_len] "+&d" (ioaddr_len.pair),
229 [cc] "+d" (cc), [val] "=d" (val),
230 [dst] "+a" (dst), [cnt] "+d" (cnt), [tmp] "=d" (tmp),
231 [shift] "+a" (shift)
232 :: "cc", "memory");
233
234 /* did we write everything to the user space buffer? */
235 if (!cc && cnt != 0)
236 cc = -EFAULT;
237
238 *status = ioaddr_len.odd >> 24 & 0xff;
239 return cc;
240 }
241
__memcpy_fromio_inuser(void __user * dst,const void __iomem * src,unsigned long n)242 static inline int __memcpy_fromio_inuser(void __user *dst,
243 const void __iomem *src,
244 unsigned long n)
245 {
246 int size, rc = 0;
247 u8 status;
248
249 while (n > 0) {
250 size = zpci_get_max_io_size((u64 __force) src,
251 (u64 __force) dst, n,
252 ZPCI_MAX_READ_SIZE);
253 rc = __pcilg_mio_inuser(dst, src, size, &status);
254 if (rc)
255 break;
256 src += size;
257 dst += size;
258 n -= size;
259 }
260 if (rc)
261 zpci_err_mmio(rc, status, (__force u64) dst);
262 return rc;
263 }
264
SYSCALL_DEFINE3(s390_pci_mmio_read,unsigned long,mmio_addr,void __user *,user_buffer,size_t,length)265 SYSCALL_DEFINE3(s390_pci_mmio_read, unsigned long, mmio_addr,
266 void __user *, user_buffer, size_t, length)
267 {
268 struct follow_pfnmap_args args = { };
269 u8 local_buf[64];
270 void __iomem *io_addr;
271 void *buf;
272 struct vm_area_struct *vma;
273 long ret;
274
275 if (!zpci_is_enabled())
276 return -ENODEV;
277
278 if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length)
279 return -EINVAL;
280
281 /*
282 * We only support read access to MIO capable devices if we are on
283 * a MIO enabled system. Otherwise we would have to check for every
284 * address if it is a special ZPCI_ADDR and would have to do
285 * a pfn lookup which we don't need for MIO capable devices. Currently
286 * ISM devices are the only devices without MIO support and there is no
287 * known need for accessing these from userspace.
288 */
289 if (static_branch_likely(&have_mio)) {
290 ret = __memcpy_fromio_inuser(
291 user_buffer, (const void __iomem *)mmio_addr,
292 length);
293 return ret;
294 }
295
296 if (length > 64) {
297 buf = kmalloc(length, GFP_KERNEL);
298 if (!buf)
299 return -ENOMEM;
300 } else {
301 buf = local_buf;
302 }
303
304 mmap_read_lock(current->mm);
305 ret = -EINVAL;
306 vma = vma_lookup(current->mm, mmio_addr);
307 if (!vma)
308 goto out_unlock_mmap;
309 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
310 goto out_unlock_mmap;
311 ret = -EACCES;
312 if (!(vma->vm_flags & VM_READ))
313 goto out_unlock_mmap;
314
315 args.vma = vma;
316 args.address = mmio_addr;
317 ret = follow_pfnmap_start(&args);
318 if (ret) {
319 fixup_user_fault(current->mm, mmio_addr, 0, NULL);
320 ret = follow_pfnmap_start(&args);
321 if (ret)
322 goto out_unlock_mmap;
323 }
324
325 io_addr = (void __iomem *)((args.pfn << PAGE_SHIFT) |
326 (mmio_addr & ~PAGE_MASK));
327
328 if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE) {
329 ret = -EFAULT;
330 goto out_unlock_pt;
331 }
332 ret = zpci_memcpy_fromio(buf, io_addr, length);
333
334 out_unlock_pt:
335 follow_pfnmap_end(&args);
336 out_unlock_mmap:
337 mmap_read_unlock(current->mm);
338
339 if (!ret && copy_to_user(user_buffer, buf, length))
340 ret = -EFAULT;
341
342 if (buf != local_buf)
343 kfree(buf);
344 return ret;
345 }
346