• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2001 Jens Axboe <axboe@suse.de>
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License version 2 as
6  * published by the Free Software Foundation.
7  *
8  * This program is distributed in the hope that it will be useful,
9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
10  *
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public Licens
15  * along with this program; if not, write to the Free Software
16  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-
17  */
18 #ifndef __LINUX_BIO_H
19 #define __LINUX_BIO_H
20 
21 #include <linux/highmem.h>
22 #include <linux/mempool.h>
23 #include <linux/ioprio.h>
24 #include <linux/bug.h>
25 
26 #ifdef CONFIG_BLOCK
27 
28 #include <asm/io.h>
29 
30 /* struct bio, bio_vec and BIO_* flags are defined in blk_types.h */
31 #include <linux/blk_types.h>
32 
33 #define BIO_DEBUG
34 
35 #ifdef BIO_DEBUG
36 #define BIO_BUG_ON	BUG_ON
37 #else
38 #define BIO_BUG_ON
39 #endif
40 
41 #define BIO_MAX_PAGES		256
42 
43 #define bio_prio(bio)			(bio)->bi_ioprio
44 #define bio_set_prio(bio, prio)		((bio)->bi_ioprio = prio)
45 
46 #define bio_iter_iovec(bio, iter)				\
47 	bvec_iter_bvec((bio)->bi_io_vec, (iter))
48 
49 #define bio_iter_page(bio, iter)				\
50 	bvec_iter_page((bio)->bi_io_vec, (iter))
51 #define bio_iter_len(bio, iter)					\
52 	bvec_iter_len((bio)->bi_io_vec, (iter))
53 #define bio_iter_offset(bio, iter)				\
54 	bvec_iter_offset((bio)->bi_io_vec, (iter))
55 
56 #define bio_page(bio)		bio_iter_page((bio), (bio)->bi_iter)
57 #define bio_offset(bio)		bio_iter_offset((bio), (bio)->bi_iter)
58 #define bio_iovec(bio)		bio_iter_iovec((bio), (bio)->bi_iter)
59 
60 #define bio_multiple_segments(bio)				\
61 	((bio)->bi_iter.bi_size != bio_iovec(bio).bv_len)
62 #define bio_sectors(bio)	((bio)->bi_iter.bi_size >> 9)
63 #define bio_end_sector(bio)	((bio)->bi_iter.bi_sector + bio_sectors((bio)))
64 
65 /*
66  * Check whether this bio carries any data or not. A NULL bio is allowed.
67  */
bio_has_data(struct bio * bio)68 static inline bool bio_has_data(struct bio *bio)
69 {
70 	if (bio &&
71 	    bio->bi_iter.bi_size &&
72 	    bio_op(bio) != REQ_OP_DISCARD &&
73 	    bio_op(bio) != REQ_OP_SECURE_ERASE)
74 		return true;
75 
76 	return false;
77 }
78 
bio_no_advance_iter(struct bio * bio)79 static inline bool bio_no_advance_iter(struct bio *bio)
80 {
81 	return bio_op(bio) == REQ_OP_DISCARD ||
82 	       bio_op(bio) == REQ_OP_SECURE_ERASE ||
83 	       bio_op(bio) == REQ_OP_WRITE_SAME;
84 }
85 
bio_is_rw(struct bio * bio)86 static inline bool bio_is_rw(struct bio *bio)
87 {
88 	if (!bio_has_data(bio))
89 		return false;
90 
91 	if (bio_no_advance_iter(bio))
92 		return false;
93 
94 	return true;
95 }
96 
bio_mergeable(struct bio * bio)97 static inline bool bio_mergeable(struct bio *bio)
98 {
99 	if (bio->bi_opf & REQ_NOMERGE_FLAGS)
100 		return false;
101 
102 	return true;
103 }
104 
bio_cur_bytes(struct bio * bio)105 static inline unsigned int bio_cur_bytes(struct bio *bio)
106 {
107 	if (bio_has_data(bio))
108 		return bio_iovec(bio).bv_len;
109 	else /* dataless requests such as discard */
110 		return bio->bi_iter.bi_size;
111 }
112 
bio_data(struct bio * bio)113 static inline void *bio_data(struct bio *bio)
114 {
115 	if (bio_has_data(bio))
116 		return page_address(bio_page(bio)) + bio_offset(bio);
117 
118 	return NULL;
119 }
120 
121 /*
122  * will die
123  */
124 #define bio_to_phys(bio)	(page_to_phys(bio_page((bio))) + (unsigned long) bio_offset((bio)))
125 #define bvec_to_phys(bv)	(page_to_phys((bv)->bv_page) + (unsigned long) (bv)->bv_offset)
126 
127 /*
128  * queues that have highmem support enabled may still need to revert to
129  * PIO transfers occasionally and thus map high pages temporarily. For
130  * permanent PIO fall back, user is probably better off disabling highmem
131  * I/O completely on that queue (see ide-dma for example)
132  */
133 #define __bio_kmap_atomic(bio, iter)				\
134 	(kmap_atomic(bio_iter_iovec((bio), (iter)).bv_page) +	\
135 		bio_iter_iovec((bio), (iter)).bv_offset)
136 
137 #define __bio_kunmap_atomic(addr)	kunmap_atomic(addr)
138 
139 /*
140  * merge helpers etc
141  */
142 
143 /* Default implementation of BIOVEC_PHYS_MERGEABLE */
144 #define __BIOVEC_PHYS_MERGEABLE(vec1, vec2)	\
145 	((bvec_to_phys((vec1)) + (vec1)->bv_len) == bvec_to_phys((vec2)))
146 
147 /*
148  * allow arch override, for eg virtualized architectures (put in asm/io.h)
149  */
150 #ifndef BIOVEC_PHYS_MERGEABLE
151 #define BIOVEC_PHYS_MERGEABLE(vec1, vec2)	\
152 	__BIOVEC_PHYS_MERGEABLE(vec1, vec2)
153 #endif
154 
155 #define __BIO_SEG_BOUNDARY(addr1, addr2, mask) \
156 	(((addr1) | (mask)) == (((addr2) - 1) | (mask)))
157 #define BIOVEC_SEG_BOUNDARY(q, b1, b2) \
158 	__BIO_SEG_BOUNDARY(bvec_to_phys((b1)), bvec_to_phys((b2)) + (b2)->bv_len, queue_segment_boundary((q)))
159 
160 /*
161  * drivers should _never_ use the all version - the bio may have been split
162  * before it got to the driver and the driver won't own all of it
163  */
164 #define bio_for_each_segment_all(bvl, bio, i)				\
165 	for (i = 0, bvl = (bio)->bi_io_vec; i < (bio)->bi_vcnt; i++, bvl++)
166 
bio_advance_iter(struct bio * bio,struct bvec_iter * iter,unsigned bytes)167 static inline void bio_advance_iter(struct bio *bio, struct bvec_iter *iter,
168 				    unsigned bytes)
169 {
170 	iter->bi_sector += bytes >> 9;
171 
172 	if (bio_no_advance_iter(bio))
173 		iter->bi_size -= bytes;
174 	else
175 		bvec_iter_advance(bio->bi_io_vec, iter, bytes);
176 }
177 
178 #define __bio_for_each_segment(bvl, bio, iter, start)			\
179 	for (iter = (start);						\
180 	     (iter).bi_size &&						\
181 		((bvl = bio_iter_iovec((bio), (iter))), 1);		\
182 	     bio_advance_iter((bio), &(iter), (bvl).bv_len))
183 
184 #define bio_for_each_segment(bvl, bio, iter)				\
185 	__bio_for_each_segment(bvl, bio, iter, (bio)->bi_iter)
186 
187 #define bio_iter_last(bvec, iter) ((iter).bi_size == (bvec).bv_len)
188 
bio_segments(struct bio * bio)189 static inline unsigned bio_segments(struct bio *bio)
190 {
191 	unsigned segs = 0;
192 	struct bio_vec bv;
193 	struct bvec_iter iter;
194 
195 	/*
196 	 * We special case discard/write same, because they interpret bi_size
197 	 * differently:
198 	 */
199 
200 	if (bio_op(bio) == REQ_OP_DISCARD)
201 		return 1;
202 
203 	if (bio_op(bio) == REQ_OP_SECURE_ERASE)
204 		return 1;
205 
206 	if (bio_op(bio) == REQ_OP_WRITE_SAME)
207 		return 1;
208 
209 	bio_for_each_segment(bv, bio, iter)
210 		segs++;
211 
212 	return segs;
213 }
214 
215 /*
216  * get a reference to a bio, so it won't disappear. the intended use is
217  * something like:
218  *
219  * bio_get(bio);
220  * submit_bio(rw, bio);
221  * if (bio->bi_flags ...)
222  *	do_something
223  * bio_put(bio);
224  *
225  * without the bio_get(), it could potentially complete I/O before submit_bio
226  * returns. and then bio would be freed memory when if (bio->bi_flags ...)
227  * runs
228  */
bio_get(struct bio * bio)229 static inline void bio_get(struct bio *bio)
230 {
231 	bio->bi_flags |= (1 << BIO_REFFED);
232 	smp_mb__before_atomic();
233 	atomic_inc(&bio->__bi_cnt);
234 }
235 
bio_cnt_set(struct bio * bio,unsigned int count)236 static inline void bio_cnt_set(struct bio *bio, unsigned int count)
237 {
238 	if (count != 1) {
239 		bio->bi_flags |= (1 << BIO_REFFED);
240 		smp_mb__before_atomic();
241 	}
242 	atomic_set(&bio->__bi_cnt, count);
243 }
244 
bio_flagged(struct bio * bio,unsigned int bit)245 static inline bool bio_flagged(struct bio *bio, unsigned int bit)
246 {
247 	return (bio->bi_flags & (1U << bit)) != 0;
248 }
249 
bio_set_flag(struct bio * bio,unsigned int bit)250 static inline void bio_set_flag(struct bio *bio, unsigned int bit)
251 {
252 	bio->bi_flags |= (1U << bit);
253 }
254 
bio_clear_flag(struct bio * bio,unsigned int bit)255 static inline void bio_clear_flag(struct bio *bio, unsigned int bit)
256 {
257 	bio->bi_flags &= ~(1U << bit);
258 }
259 
bio_get_first_bvec(struct bio * bio,struct bio_vec * bv)260 static inline void bio_get_first_bvec(struct bio *bio, struct bio_vec *bv)
261 {
262 	*bv = bio_iovec(bio);
263 }
264 
bio_get_last_bvec(struct bio * bio,struct bio_vec * bv)265 static inline void bio_get_last_bvec(struct bio *bio, struct bio_vec *bv)
266 {
267 	struct bvec_iter iter = bio->bi_iter;
268 	int idx;
269 
270 	if (unlikely(!bio_multiple_segments(bio))) {
271 		*bv = bio_iovec(bio);
272 		return;
273 	}
274 
275 	bio_advance_iter(bio, &iter, iter.bi_size);
276 
277 	if (!iter.bi_bvec_done)
278 		idx = iter.bi_idx - 1;
279 	else	/* in the middle of bvec */
280 		idx = iter.bi_idx;
281 
282 	*bv = bio->bi_io_vec[idx];
283 
284 	/*
285 	 * iter.bi_bvec_done records actual length of the last bvec
286 	 * if this bio ends in the middle of one io vector
287 	 */
288 	if (iter.bi_bvec_done)
289 		bv->bv_len = iter.bi_bvec_done;
290 }
291 
292 enum bip_flags {
293 	BIP_BLOCK_INTEGRITY	= 1 << 0, /* block layer owns integrity data */
294 	BIP_MAPPED_INTEGRITY	= 1 << 1, /* ref tag has been remapped */
295 	BIP_CTRL_NOCHECK	= 1 << 2, /* disable HBA integrity checking */
296 	BIP_DISK_NOCHECK	= 1 << 3, /* disable disk integrity checking */
297 	BIP_IP_CHECKSUM		= 1 << 4, /* IP checksum */
298 };
299 
300 /*
301  * bio integrity payload
302  */
303 struct bio_integrity_payload {
304 	struct bio		*bip_bio;	/* parent bio */
305 
306 	struct bvec_iter	bip_iter;
307 
308 	bio_end_io_t		*bip_end_io;	/* saved I/O completion fn */
309 
310 	unsigned short		bip_slab;	/* slab the bip came from */
311 	unsigned short		bip_vcnt;	/* # of integrity bio_vecs */
312 	unsigned short		bip_max_vcnt;	/* integrity bio_vec slots */
313 	unsigned short		bip_flags;	/* control flags */
314 
315 	struct work_struct	bip_work;	/* I/O completion */
316 
317 	struct bio_vec		*bip_vec;
318 	struct bio_vec		bip_inline_vecs[0];/* embedded bvec array */
319 };
320 
321 #if defined(CONFIG_BLK_DEV_INTEGRITY)
322 
bio_integrity(struct bio * bio)323 static inline struct bio_integrity_payload *bio_integrity(struct bio *bio)
324 {
325 	if (bio->bi_opf & REQ_INTEGRITY)
326 		return bio->bi_integrity;
327 
328 	return NULL;
329 }
330 
bio_integrity_flagged(struct bio * bio,enum bip_flags flag)331 static inline bool bio_integrity_flagged(struct bio *bio, enum bip_flags flag)
332 {
333 	struct bio_integrity_payload *bip = bio_integrity(bio);
334 
335 	if (bip)
336 		return bip->bip_flags & flag;
337 
338 	return false;
339 }
340 
bip_get_seed(struct bio_integrity_payload * bip)341 static inline sector_t bip_get_seed(struct bio_integrity_payload *bip)
342 {
343 	return bip->bip_iter.bi_sector;
344 }
345 
bip_set_seed(struct bio_integrity_payload * bip,sector_t seed)346 static inline void bip_set_seed(struct bio_integrity_payload *bip,
347 				sector_t seed)
348 {
349 	bip->bip_iter.bi_sector = seed;
350 }
351 
352 #endif /* CONFIG_BLK_DEV_INTEGRITY */
353 
354 extern void bio_trim(struct bio *bio, int offset, int size);
355 extern struct bio *bio_split(struct bio *bio, int sectors,
356 			     gfp_t gfp, struct bio_set *bs);
357 
358 /**
359  * bio_next_split - get next @sectors from a bio, splitting if necessary
360  * @bio:	bio to split
361  * @sectors:	number of sectors to split from the front of @bio
362  * @gfp:	gfp mask
363  * @bs:		bio set to allocate from
364  *
365  * Returns a bio representing the next @sectors of @bio - if the bio is smaller
366  * than @sectors, returns the original bio unchanged.
367  */
bio_next_split(struct bio * bio,int sectors,gfp_t gfp,struct bio_set * bs)368 static inline struct bio *bio_next_split(struct bio *bio, int sectors,
369 					 gfp_t gfp, struct bio_set *bs)
370 {
371 	if (sectors >= bio_sectors(bio))
372 		return bio;
373 
374 	return bio_split(bio, sectors, gfp, bs);
375 }
376 
377 extern struct bio_set *bioset_create(unsigned int, unsigned int);
378 extern struct bio_set *bioset_create_nobvec(unsigned int, unsigned int);
379 extern void bioset_free(struct bio_set *);
380 extern mempool_t *biovec_create_pool(int pool_entries);
381 
382 extern struct bio *bio_alloc_bioset(gfp_t, int, struct bio_set *);
383 extern void bio_put(struct bio *);
384 
385 extern void __bio_clone_fast(struct bio *, struct bio *);
386 extern struct bio *bio_clone_fast(struct bio *, gfp_t, struct bio_set *);
387 extern struct bio *bio_clone_bioset(struct bio *, gfp_t, struct bio_set *bs);
388 
389 extern struct bio_set *fs_bio_set;
390 
bio_alloc(gfp_t gfp_mask,unsigned int nr_iovecs)391 static inline struct bio *bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs)
392 {
393 	return bio_alloc_bioset(gfp_mask, nr_iovecs, fs_bio_set);
394 }
395 
bio_clone(struct bio * bio,gfp_t gfp_mask)396 static inline struct bio *bio_clone(struct bio *bio, gfp_t gfp_mask)
397 {
398 	return bio_clone_bioset(bio, gfp_mask, fs_bio_set);
399 }
400 
bio_kmalloc(gfp_t gfp_mask,unsigned int nr_iovecs)401 static inline struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned int nr_iovecs)
402 {
403 	return bio_alloc_bioset(gfp_mask, nr_iovecs, NULL);
404 }
405 
bio_clone_kmalloc(struct bio * bio,gfp_t gfp_mask)406 static inline struct bio *bio_clone_kmalloc(struct bio *bio, gfp_t gfp_mask)
407 {
408 	return bio_clone_bioset(bio, gfp_mask, NULL);
409 
410 }
411 
412 extern void bio_endio(struct bio *);
413 
bio_io_error(struct bio * bio)414 static inline void bio_io_error(struct bio *bio)
415 {
416 	bio->bi_error = -EIO;
417 	bio_endio(bio);
418 }
419 
420 struct request_queue;
421 extern int bio_phys_segments(struct request_queue *, struct bio *);
422 
423 extern int submit_bio_wait(struct bio *bio);
424 extern void bio_advance(struct bio *, unsigned);
425 
426 extern void bio_init(struct bio *);
427 extern void bio_reset(struct bio *);
428 void bio_chain(struct bio *, struct bio *);
429 
430 extern int bio_add_page(struct bio *, struct page *, unsigned int,unsigned int);
431 extern int bio_add_pc_page(struct request_queue *, struct bio *, struct page *,
432 			   unsigned int, unsigned int);
433 struct rq_map_data;
434 extern struct bio *bio_map_user_iov(struct request_queue *,
435 				    const struct iov_iter *, gfp_t);
436 extern void bio_unmap_user(struct bio *);
437 extern struct bio *bio_map_kern(struct request_queue *, void *, unsigned int,
438 				gfp_t);
439 extern struct bio *bio_copy_kern(struct request_queue *, void *, unsigned int,
440 				 gfp_t, int);
441 extern void bio_set_pages_dirty(struct bio *bio);
442 extern void bio_check_pages_dirty(struct bio *bio);
443 
444 void generic_start_io_acct(int rw, unsigned long sectors,
445 			   struct hd_struct *part);
446 void generic_end_io_acct(int rw, struct hd_struct *part,
447 			 unsigned long start_time);
448 
449 #ifndef ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
450 # error	"You should define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE for your platform"
451 #endif
452 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
453 extern void bio_flush_dcache_pages(struct bio *bi);
454 #else
bio_flush_dcache_pages(struct bio * bi)455 static inline void bio_flush_dcache_pages(struct bio *bi)
456 {
457 }
458 #endif
459 
460 extern void bio_copy_data(struct bio *dst, struct bio *src);
461 extern int bio_alloc_pages(struct bio *bio, gfp_t gfp);
462 extern void bio_free_pages(struct bio *bio);
463 
464 extern struct bio *bio_copy_user_iov(struct request_queue *,
465 				     struct rq_map_data *,
466 				     const struct iov_iter *,
467 				     gfp_t);
468 extern int bio_uncopy_user(struct bio *);
469 void zero_fill_bio(struct bio *bio);
470 extern struct bio_vec *bvec_alloc(gfp_t, int, unsigned long *, mempool_t *);
471 extern void bvec_free(mempool_t *, struct bio_vec *, unsigned int);
472 extern unsigned int bvec_nr_vecs(unsigned short idx);
473 
474 #ifdef CONFIG_BLK_CGROUP
475 int bio_associate_blkcg(struct bio *bio, struct cgroup_subsys_state *blkcg_css);
476 int bio_associate_current(struct bio *bio);
477 void bio_disassociate_task(struct bio *bio);
478 void bio_clone_blkcg_association(struct bio *dst, struct bio *src);
479 #else	/* CONFIG_BLK_CGROUP */
bio_associate_blkcg(struct bio * bio,struct cgroup_subsys_state * blkcg_css)480 static inline int bio_associate_blkcg(struct bio *bio,
481 			struct cgroup_subsys_state *blkcg_css) { return 0; }
bio_associate_current(struct bio * bio)482 static inline int bio_associate_current(struct bio *bio) { return -ENOENT; }
bio_disassociate_task(struct bio * bio)483 static inline void bio_disassociate_task(struct bio *bio) { }
bio_clone_blkcg_association(struct bio * dst,struct bio * src)484 static inline void bio_clone_blkcg_association(struct bio *dst,
485 			struct bio *src) { }
486 #endif	/* CONFIG_BLK_CGROUP */
487 
488 #ifdef CONFIG_HIGHMEM
489 /*
490  * remember never ever reenable interrupts between a bvec_kmap_irq and
491  * bvec_kunmap_irq!
492  */
bvec_kmap_irq(struct bio_vec * bvec,unsigned long * flags)493 static inline char *bvec_kmap_irq(struct bio_vec *bvec, unsigned long *flags)
494 {
495 	unsigned long addr;
496 
497 	/*
498 	 * might not be a highmem page, but the preempt/irq count
499 	 * balancing is a lot nicer this way
500 	 */
501 	local_irq_save(*flags);
502 	addr = (unsigned long) kmap_atomic(bvec->bv_page);
503 
504 	BUG_ON(addr & ~PAGE_MASK);
505 
506 	return (char *) addr + bvec->bv_offset;
507 }
508 
bvec_kunmap_irq(char * buffer,unsigned long * flags)509 static inline void bvec_kunmap_irq(char *buffer, unsigned long *flags)
510 {
511 	unsigned long ptr = (unsigned long) buffer & PAGE_MASK;
512 
513 	kunmap_atomic((void *) ptr);
514 	local_irq_restore(*flags);
515 }
516 
517 #else
bvec_kmap_irq(struct bio_vec * bvec,unsigned long * flags)518 static inline char *bvec_kmap_irq(struct bio_vec *bvec, unsigned long *flags)
519 {
520 	return page_address(bvec->bv_page) + bvec->bv_offset;
521 }
522 
bvec_kunmap_irq(char * buffer,unsigned long * flags)523 static inline void bvec_kunmap_irq(char *buffer, unsigned long *flags)
524 {
525 	*flags = 0;
526 }
527 #endif
528 
__bio_kmap_irq(struct bio * bio,struct bvec_iter iter,unsigned long * flags)529 static inline char *__bio_kmap_irq(struct bio *bio, struct bvec_iter iter,
530 				   unsigned long *flags)
531 {
532 	return bvec_kmap_irq(&bio_iter_iovec(bio, iter), flags);
533 }
534 #define __bio_kunmap_irq(buf, flags)	bvec_kunmap_irq(buf, flags)
535 
536 #define bio_kmap_irq(bio, flags) \
537 	__bio_kmap_irq((bio), (bio)->bi_iter, (flags))
538 #define bio_kunmap_irq(buf,flags)	__bio_kunmap_irq(buf, flags)
539 
540 /*
541  * BIO list management for use by remapping drivers (e.g. DM or MD) and loop.
542  *
543  * A bio_list anchors a singly-linked list of bios chained through the bi_next
544  * member of the bio.  The bio_list also caches the last list member to allow
545  * fast access to the tail.
546  */
547 struct bio_list {
548 	struct bio *head;
549 	struct bio *tail;
550 };
551 
bio_list_empty(const struct bio_list * bl)552 static inline int bio_list_empty(const struct bio_list *bl)
553 {
554 	return bl->head == NULL;
555 }
556 
bio_list_init(struct bio_list * bl)557 static inline void bio_list_init(struct bio_list *bl)
558 {
559 	bl->head = bl->tail = NULL;
560 }
561 
562 #define BIO_EMPTY_LIST	{ NULL, NULL }
563 
564 #define bio_list_for_each(bio, bl) \
565 	for (bio = (bl)->head; bio; bio = bio->bi_next)
566 
bio_list_size(const struct bio_list * bl)567 static inline unsigned bio_list_size(const struct bio_list *bl)
568 {
569 	unsigned sz = 0;
570 	struct bio *bio;
571 
572 	bio_list_for_each(bio, bl)
573 		sz++;
574 
575 	return sz;
576 }
577 
bio_list_add(struct bio_list * bl,struct bio * bio)578 static inline void bio_list_add(struct bio_list *bl, struct bio *bio)
579 {
580 	bio->bi_next = NULL;
581 
582 	if (bl->tail)
583 		bl->tail->bi_next = bio;
584 	else
585 		bl->head = bio;
586 
587 	bl->tail = bio;
588 }
589 
bio_list_add_head(struct bio_list * bl,struct bio * bio)590 static inline void bio_list_add_head(struct bio_list *bl, struct bio *bio)
591 {
592 	bio->bi_next = bl->head;
593 
594 	bl->head = bio;
595 
596 	if (!bl->tail)
597 		bl->tail = bio;
598 }
599 
bio_list_merge(struct bio_list * bl,struct bio_list * bl2)600 static inline void bio_list_merge(struct bio_list *bl, struct bio_list *bl2)
601 {
602 	if (!bl2->head)
603 		return;
604 
605 	if (bl->tail)
606 		bl->tail->bi_next = bl2->head;
607 	else
608 		bl->head = bl2->head;
609 
610 	bl->tail = bl2->tail;
611 }
612 
bio_list_merge_head(struct bio_list * bl,struct bio_list * bl2)613 static inline void bio_list_merge_head(struct bio_list *bl,
614 				       struct bio_list *bl2)
615 {
616 	if (!bl2->head)
617 		return;
618 
619 	if (bl->head)
620 		bl2->tail->bi_next = bl->head;
621 	else
622 		bl->tail = bl2->tail;
623 
624 	bl->head = bl2->head;
625 }
626 
bio_list_peek(struct bio_list * bl)627 static inline struct bio *bio_list_peek(struct bio_list *bl)
628 {
629 	return bl->head;
630 }
631 
bio_list_pop(struct bio_list * bl)632 static inline struct bio *bio_list_pop(struct bio_list *bl)
633 {
634 	struct bio *bio = bl->head;
635 
636 	if (bio) {
637 		bl->head = bl->head->bi_next;
638 		if (!bl->head)
639 			bl->tail = NULL;
640 
641 		bio->bi_next = NULL;
642 	}
643 
644 	return bio;
645 }
646 
bio_list_get(struct bio_list * bl)647 static inline struct bio *bio_list_get(struct bio_list *bl)
648 {
649 	struct bio *bio = bl->head;
650 
651 	bl->head = bl->tail = NULL;
652 
653 	return bio;
654 }
655 
656 /*
657  * Increment chain count for the bio. Make sure the CHAIN flag update
658  * is visible before the raised count.
659  */
bio_inc_remaining(struct bio * bio)660 static inline void bio_inc_remaining(struct bio *bio)
661 {
662 	bio_set_flag(bio, BIO_CHAIN);
663 	smp_mb__before_atomic();
664 	atomic_inc(&bio->__bi_remaining);
665 }
666 
667 /*
668  * bio_set is used to allow other portions of the IO system to
669  * allocate their own private memory pools for bio and iovec structures.
670  * These memory pools in turn all allocate from the bio_slab
671  * and the bvec_slabs[].
672  */
673 #define BIO_POOL_SIZE 2
674 
675 struct bio_set {
676 	struct kmem_cache *bio_slab;
677 	unsigned int front_pad;
678 
679 	mempool_t *bio_pool;
680 	mempool_t *bvec_pool;
681 #if defined(CONFIG_BLK_DEV_INTEGRITY)
682 	mempool_t *bio_integrity_pool;
683 	mempool_t *bvec_integrity_pool;
684 #endif
685 
686 	/*
687 	 * Deadlock avoidance for stacking block drivers: see comments in
688 	 * bio_alloc_bioset() for details
689 	 */
690 	spinlock_t		rescue_lock;
691 	struct bio_list		rescue_list;
692 	struct work_struct	rescue_work;
693 	struct workqueue_struct	*rescue_workqueue;
694 };
695 
696 struct biovec_slab {
697 	int nr_vecs;
698 	char *name;
699 	struct kmem_cache *slab;
700 };
701 
702 /*
703  * a small number of entries is fine, not going to be performance critical.
704  * basically we just need to survive
705  */
706 #define BIO_SPLIT_ENTRIES 2
707 
708 #if defined(CONFIG_BLK_DEV_INTEGRITY)
709 
710 #define bip_for_each_vec(bvl, bip, iter)				\
711 	for_each_bvec(bvl, (bip)->bip_vec, iter, (bip)->bip_iter)
712 
713 #define bio_for_each_integrity_vec(_bvl, _bio, _iter)			\
714 	for_each_bio(_bio)						\
715 		bip_for_each_vec(_bvl, _bio->bi_integrity, _iter)
716 
717 extern struct bio_integrity_payload *bio_integrity_alloc(struct bio *, gfp_t, unsigned int);
718 extern void bio_integrity_free(struct bio *);
719 extern int bio_integrity_add_page(struct bio *, struct page *, unsigned int, unsigned int);
720 extern bool bio_integrity_enabled(struct bio *bio);
721 extern int bio_integrity_prep(struct bio *);
722 extern void bio_integrity_endio(struct bio *);
723 extern void bio_integrity_advance(struct bio *, unsigned int);
724 extern void bio_integrity_trim(struct bio *, unsigned int, unsigned int);
725 extern int bio_integrity_clone(struct bio *, struct bio *, gfp_t);
726 extern int bioset_integrity_create(struct bio_set *, int);
727 extern void bioset_integrity_free(struct bio_set *);
728 extern void bio_integrity_init(void);
729 
730 #else /* CONFIG_BLK_DEV_INTEGRITY */
731 
bio_integrity(struct bio * bio)732 static inline void *bio_integrity(struct bio *bio)
733 {
734 	return NULL;
735 }
736 
bio_integrity_enabled(struct bio * bio)737 static inline bool bio_integrity_enabled(struct bio *bio)
738 {
739 	return false;
740 }
741 
bioset_integrity_create(struct bio_set * bs,int pool_size)742 static inline int bioset_integrity_create(struct bio_set *bs, int pool_size)
743 {
744 	return 0;
745 }
746 
bioset_integrity_free(struct bio_set * bs)747 static inline void bioset_integrity_free (struct bio_set *bs)
748 {
749 	return;
750 }
751 
bio_integrity_prep(struct bio * bio)752 static inline int bio_integrity_prep(struct bio *bio)
753 {
754 	return 0;
755 }
756 
bio_integrity_free(struct bio * bio)757 static inline void bio_integrity_free(struct bio *bio)
758 {
759 	return;
760 }
761 
bio_integrity_clone(struct bio * bio,struct bio * bio_src,gfp_t gfp_mask)762 static inline int bio_integrity_clone(struct bio *bio, struct bio *bio_src,
763 				      gfp_t gfp_mask)
764 {
765 	return 0;
766 }
767 
bio_integrity_advance(struct bio * bio,unsigned int bytes_done)768 static inline void bio_integrity_advance(struct bio *bio,
769 					 unsigned int bytes_done)
770 {
771 	return;
772 }
773 
bio_integrity_trim(struct bio * bio,unsigned int offset,unsigned int sectors)774 static inline void bio_integrity_trim(struct bio *bio, unsigned int offset,
775 				      unsigned int sectors)
776 {
777 	return;
778 }
779 
bio_integrity_init(void)780 static inline void bio_integrity_init(void)
781 {
782 	return;
783 }
784 
bio_integrity_flagged(struct bio * bio,enum bip_flags flag)785 static inline bool bio_integrity_flagged(struct bio *bio, enum bip_flags flag)
786 {
787 	return false;
788 }
789 
bio_integrity_alloc(struct bio * bio,gfp_t gfp,unsigned int nr)790 static inline void *bio_integrity_alloc(struct bio * bio, gfp_t gfp,
791 								unsigned int nr)
792 {
793 	return ERR_PTR(-EINVAL);
794 }
795 
bio_integrity_add_page(struct bio * bio,struct page * page,unsigned int len,unsigned int offset)796 static inline int bio_integrity_add_page(struct bio *bio, struct page *page,
797 					unsigned int len, unsigned int offset)
798 {
799 	return 0;
800 }
801 
802 #endif /* CONFIG_BLK_DEV_INTEGRITY */
803 
804 #endif /* CONFIG_BLOCK */
805 #endif /* __LINUX_BIO_H */
806