• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0
2 #define _GNU_SOURCE
3 #include <getopt.h>
4 #include <limits.h>
5 #include <string.h>
6 #include <poll.h>
7 #include <sys/eventfd.h>
8 #include <stdlib.h>
9 #include <assert.h>
10 #include <unistd.h>
11 #include <sys/ioctl.h>
12 #include <sys/stat.h>
13 #include <sys/types.h>
14 #include <fcntl.h>
15 #include <stdbool.h>
16 #include <linux/virtio_types.h>
17 #include <linux/vhost.h>
18 #include <linux/virtio.h>
19 #include <linux/virtio_ring.h>
20 #include "../../drivers/vhost/test.h"
21 
22 #define RANDOM_BATCH -1
23 
24 /* Unused */
25 void *__kmalloc_fake, *__kfree_ignore_start, *__kfree_ignore_end;
26 
27 struct vq_info {
28 	int kick;
29 	int call;
30 	int num;
31 	int idx;
32 	void *ring;
33 	/* copy used for control */
34 	struct vring vring;
35 	struct virtqueue *vq;
36 };
37 
38 struct vdev_info {
39 	struct virtio_device vdev;
40 	int control;
41 	struct pollfd fds[1];
42 	struct vq_info vqs[1];
43 	int nvqs;
44 	void *buf;
45 	size_t buf_size;
46 	struct vhost_memory *mem;
47 };
48 
49 static const struct vhost_vring_file no_backend = { .fd = -1 },
50 				     backend = { .fd = 1 };
51 static const struct vhost_vring_state null_state = {};
52 
vq_notify(struct virtqueue * vq)53 bool vq_notify(struct virtqueue *vq)
54 {
55 	struct vq_info *info = vq->priv;
56 	unsigned long long v = 1;
57 	int r;
58 	r = write(info->kick, &v, sizeof v);
59 	assert(r == sizeof v);
60 	return true;
61 }
62 
vq_callback(struct virtqueue * vq)63 void vq_callback(struct virtqueue *vq)
64 {
65 }
66 
67 
vhost_vq_setup(struct vdev_info * dev,struct vq_info * info)68 void vhost_vq_setup(struct vdev_info *dev, struct vq_info *info)
69 {
70 	struct vhost_vring_state state = { .index = info->idx };
71 	struct vhost_vring_file file = { .index = info->idx };
72 	unsigned long long features = dev->vdev.features;
73 	struct vhost_vring_addr addr = {
74 		.index = info->idx,
75 		.desc_user_addr = (uint64_t)(unsigned long)info->vring.desc,
76 		.avail_user_addr = (uint64_t)(unsigned long)info->vring.avail,
77 		.used_user_addr = (uint64_t)(unsigned long)info->vring.used,
78 	};
79 	int r;
80 	r = ioctl(dev->control, VHOST_SET_FEATURES, &features);
81 	assert(r >= 0);
82 	state.num = info->vring.num;
83 	r = ioctl(dev->control, VHOST_SET_VRING_NUM, &state);
84 	assert(r >= 0);
85 	state.num = 0;
86 	r = ioctl(dev->control, VHOST_SET_VRING_BASE, &state);
87 	assert(r >= 0);
88 	r = ioctl(dev->control, VHOST_SET_VRING_ADDR, &addr);
89 	assert(r >= 0);
90 	file.fd = info->kick;
91 	r = ioctl(dev->control, VHOST_SET_VRING_KICK, &file);
92 	assert(r >= 0);
93 	file.fd = info->call;
94 	r = ioctl(dev->control, VHOST_SET_VRING_CALL, &file);
95 	assert(r >= 0);
96 }
97 
vq_reset(struct vq_info * info,int num,struct virtio_device * vdev)98 static void vq_reset(struct vq_info *info, int num, struct virtio_device *vdev)
99 {
100 	if (info->vq)
101 		vring_del_virtqueue(info->vq);
102 
103 	memset(info->ring, 0, vring_size(num, 4096));
104 	vring_init(&info->vring, num, info->ring, 4096);
105 	info->vq = __vring_new_virtqueue(info->idx, info->vring, vdev, true,
106 					 false, vq_notify, vq_callback, "test");
107 	assert(info->vq);
108 	info->vq->priv = info;
109 }
110 
vq_info_add(struct vdev_info * dev,int num)111 static void vq_info_add(struct vdev_info *dev, int num)
112 {
113 	struct vq_info *info = &dev->vqs[dev->nvqs];
114 	int r;
115 	info->idx = dev->nvqs;
116 	info->kick = eventfd(0, EFD_NONBLOCK);
117 	info->call = eventfd(0, EFD_NONBLOCK);
118 	r = posix_memalign(&info->ring, 4096, vring_size(num, 4096));
119 	assert(r >= 0);
120 	vq_reset(info, num, &dev->vdev);
121 	vhost_vq_setup(dev, info);
122 	dev->fds[info->idx].fd = info->call;
123 	dev->fds[info->idx].events = POLLIN;
124 	dev->nvqs++;
125 }
126 
vdev_info_init(struct vdev_info * dev,unsigned long long features)127 static void vdev_info_init(struct vdev_info* dev, unsigned long long features)
128 {
129 	int r;
130 	memset(dev, 0, sizeof *dev);
131 	dev->vdev.features = features;
132 	INIT_LIST_HEAD(&dev->vdev.vqs);
133 	spin_lock_init(&dev->vdev.vqs_list_lock);
134 	dev->buf_size = 1024;
135 	dev->buf = malloc(dev->buf_size);
136 	assert(dev->buf);
137         dev->control = open("/dev/vhost-test", O_RDWR);
138 	assert(dev->control >= 0);
139 	r = ioctl(dev->control, VHOST_SET_OWNER, NULL);
140 	assert(r >= 0);
141 	dev->mem = malloc(offsetof(struct vhost_memory, regions) +
142 			  sizeof dev->mem->regions[0]);
143 	assert(dev->mem);
144 	memset(dev->mem, 0, offsetof(struct vhost_memory, regions) +
145                           sizeof dev->mem->regions[0]);
146 	dev->mem->nregions = 1;
147 	dev->mem->regions[0].guest_phys_addr = (long)dev->buf;
148 	dev->mem->regions[0].userspace_addr = (long)dev->buf;
149 	dev->mem->regions[0].memory_size = dev->buf_size;
150 	r = ioctl(dev->control, VHOST_SET_MEM_TABLE, dev->mem);
151 	assert(r >= 0);
152 }
153 
154 /* TODO: this is pretty bad: we get a cache line bounce
155  * for the wait queue on poll and another one on read,
156  * plus the read which is there just to clear the
157  * current state. */
wait_for_interrupt(struct vdev_info * dev)158 static void wait_for_interrupt(struct vdev_info *dev)
159 {
160 	int i;
161 	unsigned long long val;
162 	poll(dev->fds, dev->nvqs, -1);
163 	for (i = 0; i < dev->nvqs; ++i)
164 		if (dev->fds[i].revents & POLLIN) {
165 			read(dev->fds[i].fd, &val, sizeof val);
166 		}
167 }
168 
run_test(struct vdev_info * dev,struct vq_info * vq,bool delayed,int batch,int reset_n,int bufs)169 static void run_test(struct vdev_info *dev, struct vq_info *vq,
170 		     bool delayed, int batch, int reset_n, int bufs)
171 {
172 	struct scatterlist sl;
173 	long started = 0, completed = 0, next_reset = reset_n;
174 	long completed_before, started_before;
175 	int r, test = 1;
176 	unsigned len;
177 	long long spurious = 0;
178 	const bool random_batch = batch == RANDOM_BATCH;
179 
180 	r = ioctl(dev->control, VHOST_TEST_RUN, &test);
181 	assert(r >= 0);
182 	if (!reset_n) {
183 		next_reset = INT_MAX;
184 	}
185 
186 	for (;;) {
187 		virtqueue_disable_cb(vq->vq);
188 		completed_before = completed;
189 		started_before = started;
190 		do {
191 			const bool reset = completed > next_reset;
192 			if (random_batch)
193 				batch = (random() % vq->vring.num) + 1;
194 
195 			while (started < bufs &&
196 			       (started - completed) < batch) {
197 				sg_init_one(&sl, dev->buf, dev->buf_size);
198 				r = virtqueue_add_outbuf(vq->vq, &sl, 1,
199 							 dev->buf + started,
200 							 GFP_ATOMIC);
201 				if (unlikely(r != 0)) {
202 					if (r == -ENOSPC &&
203 					    started > started_before)
204 						r = 0;
205 					else
206 						r = -1;
207 					break;
208 				}
209 
210 				++started;
211 
212 				if (unlikely(!virtqueue_kick(vq->vq))) {
213 					r = -1;
214 					break;
215 				}
216 			}
217 
218 			if (started >= bufs)
219 				r = -1;
220 
221 			if (reset) {
222 				r = ioctl(dev->control, VHOST_TEST_SET_BACKEND,
223 					  &no_backend);
224 				assert(!r);
225 			}
226 
227 			/* Flush out completed bufs if any */
228 			while (virtqueue_get_buf(vq->vq, &len)) {
229 				++completed;
230 				r = 0;
231 			}
232 
233 			if (reset) {
234 				struct vhost_vring_state s = { .index = 0 };
235 
236 				vq_reset(vq, vq->vring.num, &dev->vdev);
237 
238 				r = ioctl(dev->control, VHOST_GET_VRING_BASE,
239 					  &s);
240 				assert(!r);
241 
242 				s.num = 0;
243 				r = ioctl(dev->control, VHOST_SET_VRING_BASE,
244 					  &null_state);
245 				assert(!r);
246 
247 				r = ioctl(dev->control, VHOST_TEST_SET_BACKEND,
248 					  &backend);
249 				assert(!r);
250 
251 				started = completed;
252 				while (completed > next_reset)
253 					next_reset += completed;
254 			}
255 		} while (r == 0);
256 		if (completed == completed_before && started == started_before)
257 			++spurious;
258 		assert(completed <= bufs);
259 		assert(started <= bufs);
260 		if (completed == bufs)
261 			break;
262 		if (delayed) {
263 			if (virtqueue_enable_cb_delayed(vq->vq))
264 				wait_for_interrupt(dev);
265 		} else {
266 			if (virtqueue_enable_cb(vq->vq))
267 				wait_for_interrupt(dev);
268 		}
269 	}
270 	test = 0;
271 	r = ioctl(dev->control, VHOST_TEST_RUN, &test);
272 	assert(r >= 0);
273 	fprintf(stderr,
274 		"spurious wakeups: 0x%llx started=0x%lx completed=0x%lx\n",
275 		spurious, started, completed);
276 }
277 
278 const char optstring[] = "h";
279 const struct option longopts[] = {
280 	{
281 		.name = "help",
282 		.val = 'h',
283 	},
284 	{
285 		.name = "event-idx",
286 		.val = 'E',
287 	},
288 	{
289 		.name = "no-event-idx",
290 		.val = 'e',
291 	},
292 	{
293 		.name = "indirect",
294 		.val = 'I',
295 	},
296 	{
297 		.name = "no-indirect",
298 		.val = 'i',
299 	},
300 	{
301 		.name = "virtio-1",
302 		.val = '1',
303 	},
304 	{
305 		.name = "no-virtio-1",
306 		.val = '0',
307 	},
308 	{
309 		.name = "delayed-interrupt",
310 		.val = 'D',
311 	},
312 	{
313 		.name = "no-delayed-interrupt",
314 		.val = 'd',
315 	},
316 	{
317 		.name = "batch",
318 		.val = 'b',
319 		.has_arg = required_argument,
320 	},
321 	{
322 		.name = "reset",
323 		.val = 'r',
324 		.has_arg = optional_argument,
325 	},
326 	{
327 	}
328 };
329 
help(void)330 static void help(void)
331 {
332 	fprintf(stderr, "Usage: virtio_test [--help]"
333 		" [--no-indirect]"
334 		" [--no-event-idx]"
335 		" [--no-virtio-1]"
336 		" [--delayed-interrupt]"
337 		" [--batch=random/N]"
338 		" [--reset=N]"
339 		"\n");
340 }
341 
main(int argc,char ** argv)342 int main(int argc, char **argv)
343 {
344 	struct vdev_info dev;
345 	unsigned long long features = (1ULL << VIRTIO_RING_F_INDIRECT_DESC) |
346 		(1ULL << VIRTIO_RING_F_EVENT_IDX) | (1ULL << VIRTIO_F_VERSION_1);
347 	long batch = 1, reset = 0;
348 	int o;
349 	bool delayed = false;
350 
351 	for (;;) {
352 		o = getopt_long(argc, argv, optstring, longopts, NULL);
353 		switch (o) {
354 		case -1:
355 			goto done;
356 		case '?':
357 			help();
358 			exit(2);
359 		case 'e':
360 			features &= ~(1ULL << VIRTIO_RING_F_EVENT_IDX);
361 			break;
362 		case 'h':
363 			help();
364 			goto done;
365 		case 'i':
366 			features &= ~(1ULL << VIRTIO_RING_F_INDIRECT_DESC);
367 			break;
368 		case '0':
369 			features &= ~(1ULL << VIRTIO_F_VERSION_1);
370 			break;
371 		case 'D':
372 			delayed = true;
373 			break;
374 		case 'b':
375 			if (0 == strcmp(optarg, "random")) {
376 				batch = RANDOM_BATCH;
377 			} else {
378 				batch = strtol(optarg, NULL, 10);
379 				assert(batch > 0);
380 				assert(batch < (long)INT_MAX + 1);
381 			}
382 			break;
383 		case 'r':
384 			if (!optarg) {
385 				reset = 1;
386 			} else {
387 				reset = strtol(optarg, NULL, 10);
388 				assert(reset > 0);
389 				assert(reset < (long)INT_MAX + 1);
390 			}
391 			break;
392 		default:
393 			assert(0);
394 			break;
395 		}
396 	}
397 
398 done:
399 	vdev_info_init(&dev, features);
400 	vq_info_add(&dev, 256);
401 	run_test(&dev, &dev.vqs[0], delayed, batch, reset, 0x100000);
402 	return 0;
403 }
404