Lines Matching refs:bio
27 #define bio_prio(bio) (bio)->bi_ioprio argument
28 #define bio_set_prio(bio, prio) ((bio)->bi_ioprio = prio) argument
30 #define bio_iter_iovec(bio, iter) \ argument
31 bvec_iter_bvec((bio)->bi_io_vec, (iter))
33 #define bio_iter_page(bio, iter) \ argument
34 bvec_iter_page((bio)->bi_io_vec, (iter))
35 #define bio_iter_len(bio, iter) \ argument
36 bvec_iter_len((bio)->bi_io_vec, (iter))
37 #define bio_iter_offset(bio, iter) \ argument
38 bvec_iter_offset((bio)->bi_io_vec, (iter))
40 #define bio_page(bio) bio_iter_page((bio), (bio)->bi_iter) argument
41 #define bio_offset(bio) bio_iter_offset((bio), (bio)->bi_iter) argument
42 #define bio_iovec(bio) bio_iter_iovec((bio), (bio)->bi_iter) argument
44 #define bio_multiple_segments(bio) \ argument
45 ((bio)->bi_iter.bi_size != bio_iovec(bio).bv_len)
50 #define bio_sectors(bio) bvec_iter_sectors((bio)->bi_iter) argument
51 #define bio_end_sector(bio) bvec_iter_end_sector((bio)->bi_iter) argument
56 #define bio_data_dir(bio) \ argument
57 (op_is_write(bio_op(bio)) ? WRITE : READ)
62 static inline bool bio_has_data(struct bio *bio) in bio_has_data() argument
64 if (bio && in bio_has_data()
65 bio->bi_iter.bi_size && in bio_has_data()
66 bio_op(bio) != REQ_OP_DISCARD && in bio_has_data()
67 bio_op(bio) != REQ_OP_SECURE_ERASE && in bio_has_data()
68 bio_op(bio) != REQ_OP_WRITE_ZEROES) in bio_has_data()
74 static inline bool bio_no_advance_iter(struct bio *bio) in bio_no_advance_iter() argument
76 return bio_op(bio) == REQ_OP_DISCARD || in bio_no_advance_iter()
77 bio_op(bio) == REQ_OP_SECURE_ERASE || in bio_no_advance_iter()
78 bio_op(bio) == REQ_OP_WRITE_SAME || in bio_no_advance_iter()
79 bio_op(bio) == REQ_OP_WRITE_ZEROES; in bio_no_advance_iter()
82 static inline bool bio_mergeable(struct bio *bio) in bio_mergeable() argument
84 if (bio->bi_opf & REQ_NOMERGE_FLAGS) in bio_mergeable()
90 static inline unsigned int bio_cur_bytes(struct bio *bio) in bio_cur_bytes() argument
92 if (bio_has_data(bio)) in bio_cur_bytes()
93 return bio_iovec(bio).bv_len; in bio_cur_bytes()
95 return bio->bi_iter.bi_size; in bio_cur_bytes()
98 static inline void *bio_data(struct bio *bio) in bio_data() argument
100 if (bio_has_data(bio)) in bio_data()
101 return page_address(bio_page(bio)) + bio_offset(bio); in bio_data()
114 static inline bool bio_full(struct bio *bio, unsigned len) in bio_full() argument
116 if (bio->bi_vcnt >= bio->bi_max_vecs) in bio_full()
119 if (bio->bi_iter.bi_size > UINT_MAX - len) in bio_full()
125 static inline bool bio_next_segment(const struct bio *bio, in bio_next_segment() argument
128 if (iter->idx >= bio->bi_vcnt) in bio_next_segment()
131 bvec_advance(&bio->bi_io_vec[iter->idx], iter); in bio_next_segment()
139 #define bio_for_each_segment_all(bvl, bio, iter) \ argument
140 for (bvl = bvec_init_iter_all(&iter); bio_next_segment((bio), &iter); )
142 static inline void bio_advance_iter(struct bio *bio, struct bvec_iter *iter, in bio_advance_iter() argument
147 if (bio_no_advance_iter(bio)) in bio_advance_iter()
150 bvec_iter_advance(bio->bi_io_vec, iter, bytes); in bio_advance_iter()
154 #define __bio_for_each_segment(bvl, bio, iter, start) \ argument
157 ((bvl = bio_iter_iovec((bio), (iter))), 1); \
158 bio_advance_iter((bio), &(iter), (bvl).bv_len))
160 #define bio_for_each_segment(bvl, bio, iter) \ argument
161 __bio_for_each_segment(bvl, bio, iter, (bio)->bi_iter)
163 #define __bio_for_each_bvec(bvl, bio, iter, start) \ argument
166 ((bvl = mp_bvec_iter_bvec((bio)->bi_io_vec, (iter))), 1); \
167 bio_advance_iter((bio), &(iter), (bvl).bv_len))
170 #define bio_for_each_bvec(bvl, bio, iter) \ argument
171 __bio_for_each_bvec(bvl, bio, iter, (bio)->bi_iter)
175 static inline unsigned bio_segments(struct bio *bio) in bio_segments() argument
186 switch (bio_op(bio)) { in bio_segments()
197 bio_for_each_segment(bv, bio, iter) in bio_segments()
217 static inline void bio_get(struct bio *bio) in bio_get() argument
219 bio->bi_flags |= (1 << BIO_REFFED); in bio_get()
221 atomic_inc(&bio->__bi_cnt); in bio_get()
224 static inline void bio_cnt_set(struct bio *bio, unsigned int count) in bio_cnt_set() argument
227 bio->bi_flags |= (1 << BIO_REFFED); in bio_cnt_set()
230 atomic_set(&bio->__bi_cnt, count); in bio_cnt_set()
233 static inline bool bio_flagged(struct bio *bio, unsigned int bit) in bio_flagged() argument
235 return (bio->bi_flags & (1U << bit)) != 0; in bio_flagged()
238 static inline void bio_set_flag(struct bio *bio, unsigned int bit) in bio_set_flag() argument
240 bio->bi_flags |= (1U << bit); in bio_set_flag()
243 static inline void bio_clear_flag(struct bio *bio, unsigned int bit) in bio_clear_flag() argument
245 bio->bi_flags &= ~(1U << bit); in bio_clear_flag()
248 static inline void bio_get_first_bvec(struct bio *bio, struct bio_vec *bv) in bio_get_first_bvec() argument
250 *bv = bio_iovec(bio); in bio_get_first_bvec()
253 static inline void bio_get_last_bvec(struct bio *bio, struct bio_vec *bv) in bio_get_last_bvec() argument
255 struct bvec_iter iter = bio->bi_iter; in bio_get_last_bvec()
258 if (unlikely(!bio_multiple_segments(bio))) { in bio_get_last_bvec()
259 *bv = bio_iovec(bio); in bio_get_last_bvec()
263 bio_advance_iter(bio, &iter, iter.bi_size); in bio_get_last_bvec()
270 *bv = bio->bi_io_vec[idx]; in bio_get_last_bvec()
280 static inline struct bio_vec *bio_first_bvec_all(struct bio *bio) in bio_first_bvec_all() argument
282 WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)); in bio_first_bvec_all()
283 return bio->bi_io_vec; in bio_first_bvec_all()
286 static inline struct page *bio_first_page_all(struct bio *bio) in bio_first_page_all() argument
288 return bio_first_bvec_all(bio)->bv_page; in bio_first_page_all()
291 static inline struct bio_vec *bio_last_bvec_all(struct bio *bio) in bio_last_bvec_all() argument
293 WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)); in bio_last_bvec_all()
294 return &bio->bi_io_vec[bio->bi_vcnt - 1]; in bio_last_bvec_all()
309 struct bio *bip_bio; /* parent bio */
328 static inline struct bio_integrity_payload *bio_integrity(struct bio *bio) in bio_integrity() argument
330 if (bio->bi_opf & REQ_INTEGRITY) in bio_integrity()
331 return bio->bi_integrity; in bio_integrity()
336 static inline bool bio_integrity_flagged(struct bio *bio, enum bip_flags flag) in bio_integrity_flagged() argument
338 struct bio_integrity_payload *bip = bio_integrity(bio); in bio_integrity_flagged()
359 extern void bio_trim(struct bio *bio, int offset, int size);
360 extern struct bio *bio_split(struct bio *bio, int sectors,
373 static inline struct bio *bio_next_split(struct bio *bio, int sectors, in bio_next_split() argument
376 if (sectors >= bio_sectors(bio)) in bio_next_split()
377 return bio; in bio_next_split()
379 return bio_split(bio, sectors, gfp, bs); in bio_next_split()
391 extern struct bio *bio_alloc_bioset(gfp_t, unsigned int, struct bio_set *);
392 extern void bio_put(struct bio *);
394 extern void __bio_clone_fast(struct bio *, struct bio *);
395 extern struct bio *bio_clone_fast(struct bio *, gfp_t, struct bio_set *);
399 static inline struct bio *bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs) in bio_alloc()
404 static inline struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned int nr_iovecs) in bio_kmalloc()
409 extern blk_qc_t submit_bio(struct bio *);
411 extern void bio_endio(struct bio *);
413 static inline void bio_io_error(struct bio *bio) in bio_io_error() argument
415 bio->bi_status = BLK_STS_IOERR; in bio_io_error()
416 bio_endio(bio); in bio_io_error()
419 static inline void bio_wouldblock_error(struct bio *bio) in bio_wouldblock_error() argument
421 bio->bi_status = BLK_STS_AGAIN; in bio_wouldblock_error()
422 bio_endio(bio); in bio_wouldblock_error()
427 extern int submit_bio_wait(struct bio *bio);
428 extern void bio_advance(struct bio *, unsigned);
430 extern void bio_init(struct bio *bio, struct bio_vec *table,
432 extern void bio_uninit(struct bio *);
433 extern void bio_reset(struct bio *);
434 void bio_chain(struct bio *, struct bio *);
436 extern int bio_add_page(struct bio *, struct page *, unsigned int,unsigned int);
437 extern int bio_add_pc_page(struct request_queue *, struct bio *, struct page *,
439 bool __bio_try_merge_page(struct bio *bio, struct page *page,
441 void __bio_add_page(struct bio *bio, struct page *page,
443 int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter);
444 void bio_release_pages(struct bio *bio, bool mark_dirty);
446 extern struct bio *bio_map_user_iov(struct request_queue *,
448 extern void bio_unmap_user(struct bio *);
449 extern struct bio *bio_map_kern(struct request_queue *, void *, unsigned int,
451 extern struct bio *bio_copy_kern(struct request_queue *, void *, unsigned int,
453 extern void bio_set_pages_dirty(struct bio *bio);
454 extern void bio_check_pages_dirty(struct bio *bio);
462 extern void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter,
463 struct bio *src, struct bvec_iter *src_iter);
464 extern void bio_copy_data(struct bio *dst, struct bio *src);
465 extern void bio_list_copy_data(struct bio *dst, struct bio *src);
466 extern void bio_free_pages(struct bio *bio);
468 extern struct bio *bio_copy_user_iov(struct request_queue *,
472 extern int bio_uncopy_user(struct bio *);
473 void zero_fill_bio_iter(struct bio *bio, struct bvec_iter iter);
474 void bio_truncate(struct bio *bio, unsigned new_size);
476 static inline void zero_fill_bio(struct bio *bio) in zero_fill_bio() argument
478 zero_fill_bio_iter(bio, bio->bi_iter); in zero_fill_bio()
484 extern const char *bio_devname(struct bio *bio, char *buffer);
486 #define bio_set_dev(bio, bdev) \ argument
488 if ((bio)->bi_disk != (bdev)->bd_disk) \
489 bio_clear_flag(bio, BIO_THROTTLED);\
490 (bio)->bi_disk = (bdev)->bd_disk; \
491 (bio)->bi_partno = (bdev)->bd_partno; \
492 bio_associate_blkg(bio); \
502 #define bio_dev(bio) \ argument
503 disk_devt((bio)->bi_disk)
506 void bio_associate_blkg_from_page(struct bio *bio, struct page *page);
508 static inline void bio_associate_blkg_from_page(struct bio *bio, in bio_associate_blkg_from_page() argument
513 void bio_disassociate_blkg(struct bio *bio);
514 void bio_associate_blkg(struct bio *bio);
515 void bio_associate_blkg_from_css(struct bio *bio,
517 void bio_clone_blkg_association(struct bio *dst, struct bio *src);
519 static inline void bio_disassociate_blkg(struct bio *bio) { } in bio_disassociate_blkg() argument
520 static inline void bio_associate_blkg(struct bio *bio) { } in bio_associate_blkg() argument
521 static inline void bio_associate_blkg_from_css(struct bio *bio, in bio_associate_blkg_from_css() argument
524 static inline void bio_clone_blkg_association(struct bio *dst, in bio_clone_blkg_association()
525 struct bio *src) { } in bio_clone_blkg_association()
577 struct bio *head;
578 struct bio *tail;
593 #define bio_list_for_each(bio, bl) \ argument
594 for (bio = (bl)->head; bio; bio = bio->bi_next)
599 struct bio *bio; in bio_list_size() local
601 bio_list_for_each(bio, bl) in bio_list_size()
607 static inline void bio_list_add(struct bio_list *bl, struct bio *bio) in bio_list_add() argument
609 bio->bi_next = NULL; in bio_list_add()
612 bl->tail->bi_next = bio; in bio_list_add()
614 bl->head = bio; in bio_list_add()
616 bl->tail = bio; in bio_list_add()
619 static inline void bio_list_add_head(struct bio_list *bl, struct bio *bio) in bio_list_add_head() argument
621 bio->bi_next = bl->head; in bio_list_add_head()
623 bl->head = bio; in bio_list_add_head()
626 bl->tail = bio; in bio_list_add_head()
656 static inline struct bio *bio_list_peek(struct bio_list *bl) in bio_list_peek()
661 static inline struct bio *bio_list_pop(struct bio_list *bl) in bio_list_pop()
663 struct bio *bio = bl->head; in bio_list_pop() local
665 if (bio) { in bio_list_pop()
670 bio->bi_next = NULL; in bio_list_pop()
673 return bio; in bio_list_pop()
676 static inline struct bio *bio_list_get(struct bio_list *bl) in bio_list_get()
678 struct bio *bio = bl->head; in bio_list_get() local
682 return bio; in bio_list_get()
689 static inline void bio_inc_remaining(struct bio *bio) in bio_inc_remaining() argument
691 bio_set_flag(bio, BIO_CHAIN); in bio_inc_remaining()
693 atomic_inc(&bio->__bi_remaining); in bio_inc_remaining()
751 extern struct bio_integrity_payload *bio_integrity_alloc(struct bio *, gfp_t, unsigned int);
752 extern int bio_integrity_add_page(struct bio *, struct page *, unsigned int, unsigned int);
753 extern bool bio_integrity_prep(struct bio *);
754 extern void bio_integrity_advance(struct bio *, unsigned int);
755 extern void bio_integrity_trim(struct bio *);
756 extern int bio_integrity_clone(struct bio *, struct bio *, gfp_t);
763 static inline void *bio_integrity(struct bio *bio) in bio_integrity() argument
778 static inline bool bio_integrity_prep(struct bio *bio) in bio_integrity_prep() argument
783 static inline int bio_integrity_clone(struct bio *bio, struct bio *bio_src, in bio_integrity_clone() argument
789 static inline void bio_integrity_advance(struct bio *bio, in bio_integrity_advance() argument
795 static inline void bio_integrity_trim(struct bio *bio) in bio_integrity_trim() argument
805 static inline bool bio_integrity_flagged(struct bio *bio, enum bip_flags flag) in bio_integrity_flagged() argument
810 static inline void *bio_integrity_alloc(struct bio * bio, gfp_t gfp, in bio_integrity_alloc() argument
816 static inline int bio_integrity_add_page(struct bio *bio, struct page *page, in bio_integrity_add_page() argument
831 static inline void bio_set_polled(struct bio *bio, struct kiocb *kiocb) in bio_set_polled() argument
833 bio->bi_opf |= REQ_HIPRI; in bio_set_polled()
835 bio->bi_opf |= REQ_NOWAIT; in bio_set_polled()