1 #include <linux/kernel.h>
2 #include <linux/mm.h>
3 #include <linux/slab.h>
4 #include <linux/uaccess.h>
5 #include <linux/ktime.h>
6 #include <linux/debugfs.h>
7
8 #define GUP_FAST_BENCHMARK _IOWR('g', 1, struct gup_benchmark)
9 #define GUP_BENCHMARK _IOWR('g', 2, struct gup_benchmark)
10 #define PIN_FAST_BENCHMARK _IOWR('g', 3, struct gup_benchmark)
11 #define PIN_BENCHMARK _IOWR('g', 4, struct gup_benchmark)
12 #define PIN_LONGTERM_BENCHMARK _IOWR('g', 5, struct gup_benchmark)
13
14 struct gup_benchmark {
15 __u64 get_delta_usec;
16 __u64 put_delta_usec;
17 __u64 addr;
18 __u64 size;
19 __u32 nr_pages_per_call;
20 __u32 flags;
21 __u64 expansion[10]; /* For future use */
22 };
23
put_back_pages(unsigned int cmd,struct page ** pages,unsigned long nr_pages)24 static void put_back_pages(unsigned int cmd, struct page **pages,
25 unsigned long nr_pages)
26 {
27 unsigned long i;
28
29 switch (cmd) {
30 case GUP_FAST_BENCHMARK:
31 case GUP_BENCHMARK:
32 for (i = 0; i < nr_pages; i++)
33 put_page(pages[i]);
34 break;
35
36 case PIN_FAST_BENCHMARK:
37 case PIN_BENCHMARK:
38 case PIN_LONGTERM_BENCHMARK:
39 unpin_user_pages(pages, nr_pages);
40 break;
41 }
42 }
43
verify_dma_pinned(unsigned int cmd,struct page ** pages,unsigned long nr_pages)44 static void verify_dma_pinned(unsigned int cmd, struct page **pages,
45 unsigned long nr_pages)
46 {
47 unsigned long i;
48 struct page *page;
49
50 switch (cmd) {
51 case PIN_FAST_BENCHMARK:
52 case PIN_BENCHMARK:
53 case PIN_LONGTERM_BENCHMARK:
54 for (i = 0; i < nr_pages; i++) {
55 page = pages[i];
56 if (WARN(!page_maybe_dma_pinned(page),
57 "pages[%lu] is NOT dma-pinned\n", i)) {
58
59 dump_page(page, "gup_benchmark failure");
60 break;
61 }
62 }
63 break;
64 }
65 }
66
__gup_benchmark_ioctl(unsigned int cmd,struct gup_benchmark * gup)67 static int __gup_benchmark_ioctl(unsigned int cmd,
68 struct gup_benchmark *gup)
69 {
70 ktime_t start_time, end_time;
71 unsigned long i, nr_pages, addr, next;
72 int nr;
73 struct page **pages;
74 int ret = 0;
75 bool needs_mmap_lock =
76 cmd != GUP_FAST_BENCHMARK && cmd != PIN_FAST_BENCHMARK;
77
78 if (gup->size > ULONG_MAX)
79 return -EINVAL;
80
81 nr_pages = gup->size / PAGE_SIZE;
82 pages = kvcalloc(nr_pages, sizeof(void *), GFP_KERNEL);
83 if (!pages)
84 return -ENOMEM;
85
86 if (needs_mmap_lock && mmap_read_lock_killable(current->mm)) {
87 ret = -EINTR;
88 goto free_pages;
89 }
90
91 i = 0;
92 nr = gup->nr_pages_per_call;
93 start_time = ktime_get();
94 for (addr = gup->addr; addr < gup->addr + gup->size; addr = next) {
95 if (nr != gup->nr_pages_per_call)
96 break;
97
98 next = addr + nr * PAGE_SIZE;
99 if (next > gup->addr + gup->size) {
100 next = gup->addr + gup->size;
101 nr = (next - addr) / PAGE_SIZE;
102 }
103
104 /* Filter out most gup flags: only allow a tiny subset here: */
105 gup->flags &= FOLL_WRITE;
106
107 switch (cmd) {
108 case GUP_FAST_BENCHMARK:
109 nr = get_user_pages_fast(addr, nr, gup->flags,
110 pages + i);
111 break;
112 case GUP_BENCHMARK:
113 nr = get_user_pages(addr, nr, gup->flags, pages + i,
114 NULL);
115 break;
116 case PIN_FAST_BENCHMARK:
117 nr = pin_user_pages_fast(addr, nr, gup->flags,
118 pages + i);
119 break;
120 case PIN_BENCHMARK:
121 nr = pin_user_pages(addr, nr, gup->flags, pages + i,
122 NULL);
123 break;
124 case PIN_LONGTERM_BENCHMARK:
125 nr = pin_user_pages(addr, nr,
126 gup->flags | FOLL_LONGTERM,
127 pages + i, NULL);
128 break;
129 default:
130 ret = -EINVAL;
131 goto unlock;
132 }
133
134 if (nr <= 0)
135 break;
136 i += nr;
137 }
138 end_time = ktime_get();
139
140 /* Shifting the meaning of nr_pages: now it is actual number pinned: */
141 nr_pages = i;
142
143 gup->get_delta_usec = ktime_us_delta(end_time, start_time);
144 gup->size = addr - gup->addr;
145
146 /*
147 * Take an un-benchmark-timed moment to verify DMA pinned
148 * state: print a warning if any non-dma-pinned pages are found:
149 */
150 verify_dma_pinned(cmd, pages, nr_pages);
151
152 start_time = ktime_get();
153
154 put_back_pages(cmd, pages, nr_pages);
155
156 end_time = ktime_get();
157 gup->put_delta_usec = ktime_us_delta(end_time, start_time);
158
159 unlock:
160 if (needs_mmap_lock)
161 mmap_read_unlock(current->mm);
162 free_pages:
163 kvfree(pages);
164 return ret;
165 }
166
gup_benchmark_ioctl(struct file * filep,unsigned int cmd,unsigned long arg)167 static long gup_benchmark_ioctl(struct file *filep, unsigned int cmd,
168 unsigned long arg)
169 {
170 struct gup_benchmark gup;
171 int ret;
172
173 switch (cmd) {
174 case GUP_FAST_BENCHMARK:
175 case GUP_BENCHMARK:
176 case PIN_FAST_BENCHMARK:
177 case PIN_BENCHMARK:
178 case PIN_LONGTERM_BENCHMARK:
179 break;
180 default:
181 return -EINVAL;
182 }
183
184 if (copy_from_user(&gup, (void __user *)arg, sizeof(gup)))
185 return -EFAULT;
186
187 ret = __gup_benchmark_ioctl(cmd, &gup);
188 if (ret)
189 return ret;
190
191 if (copy_to_user((void __user *)arg, &gup, sizeof(gup)))
192 return -EFAULT;
193
194 return 0;
195 }
196
197 static const struct file_operations gup_benchmark_fops = {
198 .open = nonseekable_open,
199 .unlocked_ioctl = gup_benchmark_ioctl,
200 };
201
gup_benchmark_init(void)202 static int gup_benchmark_init(void)
203 {
204 debugfs_create_file_unsafe("gup_benchmark", 0600, NULL, NULL,
205 &gup_benchmark_fops);
206
207 return 0;
208 }
209
210 late_initcall(gup_benchmark_init);
211