1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
3 */
4 #ifndef __IOMMUFD_PRIVATE_H
5 #define __IOMMUFD_PRIVATE_H
6
7 #include <linux/rwsem.h>
8 #include <linux/xarray.h>
9 #include <linux/refcount.h>
10 #include <linux/uaccess.h>
11
12 struct iommu_domain;
13 struct iommu_group;
14 struct iommu_option;
15 struct iommufd_device;
16
17 struct iommufd_ctx {
18 struct file *file;
19 struct xarray objects;
20 struct xarray groups;
21
22 u8 account_mode;
23 /* Compatibility with VFIO no iommu */
24 u8 no_iommu_mode;
25 struct iommufd_ioas *vfio_ioas;
26 };
27
28 /*
29 * The IOVA to PFN map. The map automatically copies the PFNs into multiple
30 * domains and permits sharing of PFNs between io_pagetable instances. This
31 * supports both a design where IOAS's are 1:1 with a domain (eg because the
32 * domain is HW customized), or where the IOAS is 1:N with multiple generic
33 * domains. The io_pagetable holds an interval tree of iopt_areas which point
34 * to shared iopt_pages which hold the pfns mapped to the page table.
35 *
36 * The locking order is domains_rwsem -> iova_rwsem -> pages::mutex
37 */
38 struct io_pagetable {
39 struct rw_semaphore domains_rwsem;
40 struct xarray domains;
41 struct xarray access_list;
42 unsigned int next_domain_id;
43
44 struct rw_semaphore iova_rwsem;
45 struct rb_root_cached area_itree;
46 /* IOVA that cannot become reserved, struct iopt_allowed */
47 struct rb_root_cached allowed_itree;
48 /* IOVA that cannot be allocated, struct iopt_reserved */
49 struct rb_root_cached reserved_itree;
50 u8 disable_large_pages;
51 unsigned long iova_alignment;
52 };
53
54 void iopt_init_table(struct io_pagetable *iopt);
55 void iopt_destroy_table(struct io_pagetable *iopt);
56 int iopt_get_pages(struct io_pagetable *iopt, unsigned long iova,
57 unsigned long length, struct list_head *pages_list);
58 void iopt_free_pages_list(struct list_head *pages_list);
59 enum {
60 IOPT_ALLOC_IOVA = 1 << 0,
61 };
62 int iopt_map_user_pages(struct iommufd_ctx *ictx, struct io_pagetable *iopt,
63 unsigned long *iova, void __user *uptr,
64 unsigned long length, int iommu_prot,
65 unsigned int flags);
66 int iopt_map_pages(struct io_pagetable *iopt, struct list_head *pages_list,
67 unsigned long length, unsigned long *dst_iova,
68 int iommu_prot, unsigned int flags);
69 int iopt_unmap_iova(struct io_pagetable *iopt, unsigned long iova,
70 unsigned long length, unsigned long *unmapped);
71 int iopt_unmap_all(struct io_pagetable *iopt, unsigned long *unmapped);
72
73 void iommufd_access_notify_unmap(struct io_pagetable *iopt, unsigned long iova,
74 unsigned long length);
75 int iopt_table_add_domain(struct io_pagetable *iopt,
76 struct iommu_domain *domain);
77 void iopt_table_remove_domain(struct io_pagetable *iopt,
78 struct iommu_domain *domain);
79 int iopt_table_enforce_dev_resv_regions(struct io_pagetable *iopt,
80 struct device *dev,
81 phys_addr_t *sw_msi_start);
82 int iopt_set_allow_iova(struct io_pagetable *iopt,
83 struct rb_root_cached *allowed_iova);
84 int iopt_reserve_iova(struct io_pagetable *iopt, unsigned long start,
85 unsigned long last, void *owner);
86 void iopt_remove_reserved_iova(struct io_pagetable *iopt, void *owner);
87 int iopt_cut_iova(struct io_pagetable *iopt, unsigned long *iovas,
88 size_t num_iovas);
89 void iopt_enable_large_pages(struct io_pagetable *iopt);
90 int iopt_disable_large_pages(struct io_pagetable *iopt);
91
92 struct iommufd_ucmd {
93 struct iommufd_ctx *ictx;
94 void __user *ubuffer;
95 u32 user_size;
96 void *cmd;
97 };
98
99 int iommufd_vfio_ioctl(struct iommufd_ctx *ictx, unsigned int cmd,
100 unsigned long arg);
101
102 /* Copy the response in ucmd->cmd back to userspace. */
iommufd_ucmd_respond(struct iommufd_ucmd * ucmd,size_t cmd_len)103 static inline int iommufd_ucmd_respond(struct iommufd_ucmd *ucmd,
104 size_t cmd_len)
105 {
106 if (copy_to_user(ucmd->ubuffer, ucmd->cmd,
107 min_t(size_t, ucmd->user_size, cmd_len)))
108 return -EFAULT;
109 return 0;
110 }
111
112 enum iommufd_object_type {
113 IOMMUFD_OBJ_NONE,
114 IOMMUFD_OBJ_ANY = IOMMUFD_OBJ_NONE,
115 IOMMUFD_OBJ_DEVICE,
116 IOMMUFD_OBJ_HW_PAGETABLE,
117 IOMMUFD_OBJ_IOAS,
118 IOMMUFD_OBJ_ACCESS,
119 #ifdef CONFIG_IOMMUFD_TEST
120 IOMMUFD_OBJ_SELFTEST,
121 #endif
122 IOMMUFD_OBJ_MAX,
123 };
124
125 /* Base struct for all objects with a userspace ID handle. */
126 struct iommufd_object {
127 struct rw_semaphore destroy_rwsem;
128 refcount_t users;
129 enum iommufd_object_type type;
130 unsigned int id;
131 };
132
iommufd_lock_obj(struct iommufd_object * obj)133 static inline bool iommufd_lock_obj(struct iommufd_object *obj)
134 {
135 if (!down_read_trylock(&obj->destroy_rwsem))
136 return false;
137 if (!refcount_inc_not_zero(&obj->users)) {
138 up_read(&obj->destroy_rwsem);
139 return false;
140 }
141 return true;
142 }
143
144 struct iommufd_object *iommufd_get_object(struct iommufd_ctx *ictx, u32 id,
145 enum iommufd_object_type type);
iommufd_put_object(struct iommufd_object * obj)146 static inline void iommufd_put_object(struct iommufd_object *obj)
147 {
148 refcount_dec(&obj->users);
149 up_read(&obj->destroy_rwsem);
150 }
151
152 void iommufd_object_abort(struct iommufd_ctx *ictx, struct iommufd_object *obj);
153 void iommufd_object_abort_and_destroy(struct iommufd_ctx *ictx,
154 struct iommufd_object *obj);
155 void iommufd_object_finalize(struct iommufd_ctx *ictx,
156 struct iommufd_object *obj);
157 void __iommufd_object_destroy_user(struct iommufd_ctx *ictx,
158 struct iommufd_object *obj, bool allow_fail);
iommufd_object_destroy_user(struct iommufd_ctx * ictx,struct iommufd_object * obj)159 static inline void iommufd_object_destroy_user(struct iommufd_ctx *ictx,
160 struct iommufd_object *obj)
161 {
162 __iommufd_object_destroy_user(ictx, obj, false);
163 }
iommufd_object_deref_user(struct iommufd_ctx * ictx,struct iommufd_object * obj)164 static inline void iommufd_object_deref_user(struct iommufd_ctx *ictx,
165 struct iommufd_object *obj)
166 {
167 __iommufd_object_destroy_user(ictx, obj, true);
168 }
169
170 struct iommufd_object *_iommufd_object_alloc(struct iommufd_ctx *ictx,
171 size_t size,
172 enum iommufd_object_type type);
173
174 #define iommufd_object_alloc(ictx, ptr, type) \
175 container_of(_iommufd_object_alloc( \
176 ictx, \
177 sizeof(*(ptr)) + BUILD_BUG_ON_ZERO( \
178 offsetof(typeof(*(ptr)), \
179 obj) != 0), \
180 type), \
181 typeof(*(ptr)), obj)
182
183 /*
184 * The IO Address Space (IOAS) pagetable is a virtual page table backed by the
185 * io_pagetable object. It is a user controlled mapping of IOVA -> PFNs. The
186 * mapping is copied into all of the associated domains and made available to
187 * in-kernel users.
188 *
189 * Every iommu_domain that is created is wrapped in a iommufd_hw_pagetable
190 * object. When we go to attach a device to an IOAS we need to get an
191 * iommu_domain and wrapping iommufd_hw_pagetable for it.
192 *
193 * An iommu_domain & iommfd_hw_pagetable will be automatically selected
194 * for a device based on the hwpt_list. If no suitable iommu_domain
195 * is found a new iommu_domain will be created.
196 */
197 struct iommufd_ioas {
198 struct iommufd_object obj;
199 struct io_pagetable iopt;
200 struct mutex mutex;
201 struct list_head hwpt_list;
202 };
203
iommufd_get_ioas(struct iommufd_ctx * ictx,u32 id)204 static inline struct iommufd_ioas *iommufd_get_ioas(struct iommufd_ctx *ictx,
205 u32 id)
206 {
207 return container_of(iommufd_get_object(ictx, id,
208 IOMMUFD_OBJ_IOAS),
209 struct iommufd_ioas, obj);
210 }
211
212 struct iommufd_ioas *iommufd_ioas_alloc(struct iommufd_ctx *ictx);
213 int iommufd_ioas_alloc_ioctl(struct iommufd_ucmd *ucmd);
214 void iommufd_ioas_destroy(struct iommufd_object *obj);
215 int iommufd_ioas_iova_ranges(struct iommufd_ucmd *ucmd);
216 int iommufd_ioas_allow_iovas(struct iommufd_ucmd *ucmd);
217 int iommufd_ioas_map(struct iommufd_ucmd *ucmd);
218 int iommufd_ioas_copy(struct iommufd_ucmd *ucmd);
219 int iommufd_ioas_unmap(struct iommufd_ucmd *ucmd);
220 int iommufd_ioas_option(struct iommufd_ucmd *ucmd);
221 int iommufd_option_rlimit_mode(struct iommu_option *cmd,
222 struct iommufd_ctx *ictx);
223
224 int iommufd_vfio_ioas(struct iommufd_ucmd *ucmd);
225
226 /*
227 * A HW pagetable is called an iommu_domain inside the kernel. This user object
228 * allows directly creating and inspecting the domains. Domains that have kernel
229 * owned page tables will be associated with an iommufd_ioas that provides the
230 * IOVA to PFN map.
231 */
232 struct iommufd_hw_pagetable {
233 struct iommufd_object obj;
234 struct iommufd_ioas *ioas;
235 struct iommu_domain *domain;
236 bool auto_domain : 1;
237 bool enforce_cache_coherency : 1;
238 bool msi_cookie : 1;
239 /* Head at iommufd_ioas::hwpt_list */
240 struct list_head hwpt_item;
241 };
242
243 struct iommufd_hw_pagetable *
244 iommufd_hw_pagetable_alloc(struct iommufd_ctx *ictx, struct iommufd_ioas *ioas,
245 struct iommufd_device *idev, bool immediate_attach);
246 int iommufd_hw_pagetable_enforce_cc(struct iommufd_hw_pagetable *hwpt);
247 int iommufd_hw_pagetable_attach(struct iommufd_hw_pagetable *hwpt,
248 struct iommufd_device *idev);
249 struct iommufd_hw_pagetable *
250 iommufd_hw_pagetable_detach(struct iommufd_device *idev);
251 void iommufd_hw_pagetable_destroy(struct iommufd_object *obj);
252 void iommufd_hw_pagetable_abort(struct iommufd_object *obj);
253 int iommufd_hwpt_alloc(struct iommufd_ucmd *ucmd);
254
iommufd_hw_pagetable_put(struct iommufd_ctx * ictx,struct iommufd_hw_pagetable * hwpt)255 static inline void iommufd_hw_pagetable_put(struct iommufd_ctx *ictx,
256 struct iommufd_hw_pagetable *hwpt)
257 {
258 lockdep_assert_not_held(&hwpt->ioas->mutex);
259 if (hwpt->auto_domain)
260 iommufd_object_deref_user(ictx, &hwpt->obj);
261 else
262 refcount_dec(&hwpt->obj.users);
263 }
264
265 struct iommufd_group {
266 struct kref ref;
267 struct mutex lock;
268 struct iommufd_ctx *ictx;
269 struct iommu_group *group;
270 struct iommufd_hw_pagetable *hwpt;
271 struct list_head device_list;
272 phys_addr_t sw_msi_start;
273 };
274
275 /*
276 * A iommufd_device object represents the binding relationship between a
277 * consuming driver and the iommufd. These objects are created/destroyed by
278 * external drivers, not by userspace.
279 */
280 struct iommufd_device {
281 struct iommufd_object obj;
282 struct iommufd_ctx *ictx;
283 struct iommufd_group *igroup;
284 struct list_head group_item;
285 /* always the physical device */
286 struct device *dev;
287 bool enforce_cache_coherency;
288 };
289
290 static inline struct iommufd_device *
iommufd_get_device(struct iommufd_ucmd * ucmd,u32 id)291 iommufd_get_device(struct iommufd_ucmd *ucmd, u32 id)
292 {
293 return container_of(iommufd_get_object(ucmd->ictx, id,
294 IOMMUFD_OBJ_DEVICE),
295 struct iommufd_device, obj);
296 }
297
298 void iommufd_device_destroy(struct iommufd_object *obj);
299 int iommufd_get_hw_info(struct iommufd_ucmd *ucmd);
300
301 struct iommufd_access {
302 struct iommufd_object obj;
303 struct iommufd_ctx *ictx;
304 struct iommufd_ioas *ioas;
305 struct iommufd_ioas *ioas_unpin;
306 struct mutex ioas_lock;
307 const struct iommufd_access_ops *ops;
308 void *data;
309 unsigned long iova_alignment;
310 u32 iopt_access_list_id;
311 };
312
313 int iopt_add_access(struct io_pagetable *iopt, struct iommufd_access *access);
314 void iopt_remove_access(struct io_pagetable *iopt,
315 struct iommufd_access *access,
316 u32 iopt_access_list_id);
317 void iommufd_access_destroy_object(struct iommufd_object *obj);
318
319 #ifdef CONFIG_IOMMUFD_TEST
320 int iommufd_test(struct iommufd_ucmd *ucmd);
321 void iommufd_selftest_destroy(struct iommufd_object *obj);
322 extern size_t iommufd_test_memory_limit;
323 void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd,
324 unsigned int ioas_id, u64 *iova, u32 *flags);
325 bool iommufd_should_fail(void);
326 int __init iommufd_test_init(void);
327 void iommufd_test_exit(void);
328 bool iommufd_selftest_is_mock_dev(struct device *dev);
329 #else
iommufd_test_syz_conv_iova_id(struct iommufd_ucmd * ucmd,unsigned int ioas_id,u64 * iova,u32 * flags)330 static inline void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd,
331 unsigned int ioas_id,
332 u64 *iova, u32 *flags)
333 {
334 }
iommufd_should_fail(void)335 static inline bool iommufd_should_fail(void)
336 {
337 return false;
338 }
iommufd_test_init(void)339 static inline int __init iommufd_test_init(void)
340 {
341 return 0;
342 }
iommufd_test_exit(void)343 static inline void iommufd_test_exit(void)
344 {
345 }
iommufd_selftest_is_mock_dev(struct device * dev)346 static inline bool iommufd_selftest_is_mock_dev(struct device *dev)
347 {
348 return false;
349 }
350 #endif
351 #endif
352