1 #ifndef FIO_OS_LINUX_H
2 #define FIO_OS_LINUX_H
3
4 #define FIO_OS os_linux
5
6 #include <sys/ioctl.h>
7 #include <sys/uio.h>
8 #include <sys/syscall.h>
9 #include <sys/sysmacros.h>
10 #include <sys/vfs.h>
11 #include <sys/mman.h>
12 #include <unistd.h>
13 #include <fcntl.h>
14 #include <errno.h>
15 #include <sched.h>
16 #include <linux/unistd.h>
17 #include <linux/raw.h>
18 #include <linux/major.h>
19
20 #include "./os-linux-syscall.h"
21 #include "binject.h"
22 #include "../file.h"
23
24 #ifndef __has_builtin // Optional of course.
25 #define __has_builtin(x) 0 // Compatibility with non-clang compilers.
26 #endif
27
28 #define FIO_HAVE_CPU_AFFINITY
29 #define FIO_HAVE_DISK_UTIL
30 #define FIO_HAVE_SGIO
31 #define FIO_HAVE_IOPRIO
32 #define FIO_HAVE_IOPRIO_CLASS
33 #define FIO_HAVE_IOSCHED_SWITCH
34 #define FIO_HAVE_ODIRECT
35 #define FIO_HAVE_HUGETLB
36 #define FIO_HAVE_RAWBIND
37 #define FIO_HAVE_BLKTRACE
38 #define FIO_HAVE_CL_SIZE
39 #define FIO_HAVE_CGROUPS
40 #define FIO_HAVE_FS_STAT
41 #define FIO_HAVE_TRIM
42 #define FIO_HAVE_BINJECT
43 #define FIO_HAVE_GETTID
44 #define FIO_USE_GENERIC_INIT_RANDOM_STATE
45 #define FIO_HAVE_PWRITEV2
46 #define FIO_HAVE_SHM_ATTACH_REMOVED
47
48 #ifdef MAP_HUGETLB
49 #define FIO_HAVE_MMAP_HUGE
50 #endif
51
52 #define OS_MAP_ANON MAP_ANONYMOUS
53
54 typedef cpu_set_t os_cpu_mask_t;
55
56 typedef struct drand48_data os_random_state_t;
57
58 #ifdef CONFIG_3ARG_AFFINITY
59 #define fio_setaffinity(pid, cpumask) \
60 sched_setaffinity((pid), sizeof(cpumask), &(cpumask))
61 #define fio_getaffinity(pid, ptr) \
62 sched_getaffinity((pid), sizeof(cpu_set_t), (ptr))
63 #elif defined(CONFIG_2ARG_AFFINITY)
64 #define fio_setaffinity(pid, cpumask) \
65 sched_setaffinity((pid), &(cpumask))
66 #define fio_getaffinity(pid, ptr) \
67 sched_getaffinity((pid), (ptr))
68 #endif
69
70 #define fio_cpu_clear(mask, cpu) (void) CPU_CLR((cpu), (mask))
71 #define fio_cpu_set(mask, cpu) (void) CPU_SET((cpu), (mask))
72 #define fio_cpu_isset(mask, cpu) CPU_ISSET((cpu), (mask))
73 #define fio_cpu_count(mask) CPU_COUNT((mask))
74
fio_cpuset_init(os_cpu_mask_t * mask)75 static inline int fio_cpuset_init(os_cpu_mask_t *mask)
76 {
77 CPU_ZERO(mask);
78 return 0;
79 }
80
fio_cpuset_exit(os_cpu_mask_t * mask)81 static inline int fio_cpuset_exit(os_cpu_mask_t *mask)
82 {
83 return 0;
84 }
85
86 #define FIO_MAX_CPUS CPU_SETSIZE
87
88 enum {
89 IOPRIO_CLASS_NONE,
90 IOPRIO_CLASS_RT,
91 IOPRIO_CLASS_BE,
92 IOPRIO_CLASS_IDLE,
93 };
94
95 enum {
96 IOPRIO_WHO_PROCESS = 1,
97 IOPRIO_WHO_PGRP,
98 IOPRIO_WHO_USER,
99 };
100
101 #define IOPRIO_BITS 16
102 #define IOPRIO_CLASS_SHIFT 13
103
104 #define IOPRIO_MIN_PRIO 0 /* highest priority */
105 #define IOPRIO_MAX_PRIO 7 /* lowest priority */
106
107 #define IOPRIO_MIN_PRIO_CLASS 0
108 #define IOPRIO_MAX_PRIO_CLASS 3
109
ioprio_set(int which,int who,int ioprio_class,int ioprio)110 static inline int ioprio_set(int which, int who, int ioprio_class, int ioprio)
111 {
112 /*
113 * If no class is set, assume BE
114 */
115 if (!ioprio_class)
116 ioprio_class = IOPRIO_CLASS_BE;
117
118 ioprio |= ioprio_class << IOPRIO_CLASS_SHIFT;
119 return syscall(__NR_ioprio_set, which, who, ioprio);
120 }
121
gettid(void)122 static inline int gettid(void)
123 {
124 return syscall(__NR_gettid);
125 }
126
127 #define SPLICE_DEF_SIZE (64*1024)
128
129 #ifndef BLKGETSIZE64
130 #define BLKGETSIZE64 _IOR(0x12,114,size_t)
131 #endif
132
133 #ifndef BLKFLSBUF
134 #define BLKFLSBUF _IO(0x12,97)
135 #endif
136
137 #ifndef BLKDISCARD
138 #define BLKDISCARD _IO(0x12,119)
139 #endif
140
blockdev_invalidate_cache(struct fio_file * f)141 static inline int blockdev_invalidate_cache(struct fio_file *f)
142 {
143 return ioctl(f->fd, BLKFLSBUF);
144 }
145
blockdev_size(struct fio_file * f,unsigned long long * bytes)146 static inline int blockdev_size(struct fio_file *f, unsigned long long *bytes)
147 {
148 if (!ioctl(f->fd, BLKGETSIZE64, bytes))
149 return 0;
150
151 return errno;
152 }
153
os_phys_mem(void)154 static inline unsigned long long os_phys_mem(void)
155 {
156 long pagesize, pages;
157
158 pagesize = sysconf(_SC_PAGESIZE);
159 pages = sysconf(_SC_PHYS_PAGES);
160 if (pages == -1 || pagesize == -1)
161 return 0;
162
163 return (unsigned long long) pages * (unsigned long long) pagesize;
164 }
165
os_random_seed(unsigned long seed,os_random_state_t * rs)166 static inline void os_random_seed(unsigned long seed, os_random_state_t *rs)
167 {
168 srand48_r(seed, rs);
169 }
170
os_random_long(os_random_state_t * rs)171 static inline long os_random_long(os_random_state_t *rs)
172 {
173 long val;
174
175 lrand48_r(rs, &val);
176 return val;
177 }
178
fio_lookup_raw(dev_t dev,int * majdev,int * mindev)179 static inline int fio_lookup_raw(dev_t dev, int *majdev, int *mindev)
180 {
181 struct raw_config_request rq;
182 int fd;
183
184 if (major(dev) != RAW_MAJOR)
185 return 1;
186
187 /*
188 * we should be able to find /dev/rawctl or /dev/raw/rawctl
189 */
190 fd = open("/dev/rawctl", O_RDONLY);
191 if (fd < 0) {
192 fd = open("/dev/raw/rawctl", O_RDONLY);
193 if (fd < 0)
194 return 1;
195 }
196
197 rq.raw_minor = minor(dev);
198 if (ioctl(fd, RAW_GETBIND, &rq) < 0) {
199 close(fd);
200 return 1;
201 }
202
203 close(fd);
204 *majdev = rq.block_major;
205 *mindev = rq.block_minor;
206 return 0;
207 }
208
209 #ifdef O_NOATIME
210 #define FIO_O_NOATIME O_NOATIME
211 #else
212 #define FIO_O_NOATIME 0
213 #endif
214
215 #ifdef O_ATOMIC
216 #define OS_O_ATOMIC O_ATOMIC
217 #else
218 #define OS_O_ATOMIC 040000000
219 #endif
220
221 #ifdef MADV_REMOVE
222 #define FIO_MADV_FREE MADV_REMOVE
223 #endif
224
225 /* Check for GCC or Clang byte swap intrinsics */
226 #if (__has_builtin(__builtin_bswap16) && __has_builtin(__builtin_bswap32) \
227 && __has_builtin(__builtin_bswap64)) || (__GNUC__ > 4 \
228 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) /* fio_swapN */
229 #define fio_swap16(x) __builtin_bswap16(x)
230 #define fio_swap32(x) __builtin_bswap32(x)
231 #define fio_swap64(x) __builtin_bswap64(x)
232 #else
233 #include <byteswap.h>
234 #define fio_swap16(x) bswap_16(x)
235 #define fio_swap32(x) bswap_32(x)
236 #define fio_swap64(x) bswap_64(x)
237 #endif /* fio_swapN */
238
239 #define CACHE_LINE_FILE \
240 "/sys/devices/system/cpu/cpu0/cache/index0/coherency_line_size"
241
arch_cache_line_size(void)242 static inline int arch_cache_line_size(void)
243 {
244 char size[32];
245 int fd, ret;
246
247 fd = open(CACHE_LINE_FILE, O_RDONLY);
248 if (fd < 0)
249 return -1;
250
251 ret = read(fd, size, sizeof(size));
252
253 close(fd);
254
255 if (ret <= 0)
256 return -1;
257 else
258 return atoi(size);
259 }
260
get_fs_free_size(const char * path)261 static inline unsigned long long get_fs_free_size(const char *path)
262 {
263 unsigned long long ret;
264 struct statfs s;
265
266 if (statfs(path, &s) < 0)
267 return -1ULL;
268
269 ret = s.f_bsize;
270 ret *= (unsigned long long) s.f_bfree;
271 return ret;
272 }
273
os_trim(int fd,unsigned long long start,unsigned long long len)274 static inline int os_trim(int fd, unsigned long long start,
275 unsigned long long len)
276 {
277 uint64_t range[2];
278
279 range[0] = start;
280 range[1] = len;
281
282 if (!ioctl(fd, BLKDISCARD, range))
283 return 0;
284
285 return errno;
286 }
287
288 #ifdef CONFIG_SCHED_IDLE
fio_set_sched_idle(void)289 static inline int fio_set_sched_idle(void)
290 {
291 struct sched_param p = { .sched_priority = 0, };
292 return sched_setscheduler(gettid(), SCHED_IDLE, &p);
293 }
294 #endif
295
296 #ifndef POSIX_FADV_STREAMID
297 #define POSIX_FADV_STREAMID 8
298 #endif
299
300 #define FIO_HAVE_STREAMID
301
302 #ifndef RWF_HIPRI
303 #define RWF_HIPRI 0x00000001
304 #endif
305 #ifndef RWF_DSYNC
306 #define RWF_DSYNC 0x00000002
307 #endif
308 #ifndef RWF_SYNC
309 #define RWF_SYNC 0x00000004
310 #endif
311
312 #ifndef CONFIG_PWRITEV2
313 #ifdef __NR_preadv2
make_pos_h_l(unsigned long * pos_h,unsigned long * pos_l,off_t offset)314 static inline void make_pos_h_l(unsigned long *pos_h, unsigned long *pos_l,
315 off_t offset)
316 {
317 #if BITS_PER_LONG == 64
318 *pos_l = offset;
319 *pos_h = 0;
320 #else
321 *pos_l = offset & 0xffffffff;
322 *pos_h = ((uint64_t) offset) >> 32;
323 #endif
324 }
preadv2(int fd,const struct iovec * iov,int iovcnt,off_t offset,unsigned int flags)325 static inline ssize_t preadv2(int fd, const struct iovec *iov, int iovcnt,
326 off_t offset, unsigned int flags)
327 {
328 unsigned long pos_l, pos_h;
329
330 make_pos_h_l(&pos_h, &pos_l, offset);
331 return syscall(__NR_preadv2, fd, iov, iovcnt, pos_l, pos_h, flags);
332 }
pwritev2(int fd,const struct iovec * iov,int iovcnt,off_t offset,unsigned int flags)333 static inline ssize_t pwritev2(int fd, const struct iovec *iov, int iovcnt,
334 off_t offset, unsigned int flags)
335 {
336 unsigned long pos_l, pos_h;
337
338 make_pos_h_l(&pos_h, &pos_l, offset);
339 return syscall(__NR_pwritev2, fd, iov, iovcnt, pos_l, pos_h, flags);
340 }
341 #else
preadv2(int fd,const struct iovec * iov,int iovcnt,off_t offset,unsigned int flags)342 static inline ssize_t preadv2(int fd, const struct iovec *iov, int iovcnt,
343 off_t offset, unsigned int flags)
344 {
345 errno = ENOSYS;
346 return -1;
347 }
pwritev2(int fd,const struct iovec * iov,int iovcnt,off_t offset,unsigned int flags)348 static inline ssize_t pwritev2(int fd, const struct iovec *iov, int iovcnt,
349 off_t offset, unsigned int flags)
350 {
351 errno = ENOSYS;
352 return -1;
353 }
354 #endif /* __NR_preadv2 */
355 #endif /* CONFIG_PWRITEV2 */
356
shm_attach_to_open_removed(void)357 static inline int shm_attach_to_open_removed(void)
358 {
359 return 1;
360 }
361
362 #endif
363