99 static unsigned int blk_flush_policy(unsigned long fflags, struct request *rq)  in blk_flush_policy()  argument
103 	if (blk_rq_sectors(rq))  in blk_flush_policy()
107 		if (rq->cmd_flags & REQ_PREFLUSH)  in blk_flush_policy()
110 		    (rq->cmd_flags & REQ_FUA))  in blk_flush_policy()
116 static unsigned int blk_flush_cur_seq(struct request *rq)  in blk_flush_cur_seq()  argument
118 	return 1 << ffz(rq->flush.seq);  in blk_flush_cur_seq()
121 static void blk_flush_restore_request(struct request *rq)  in blk_flush_restore_request()  argument
128 	rq->bio = rq->biotail;  in blk_flush_restore_request()
131 	rq->rq_flags &= ~RQF_FLUSH_SEQ;  in blk_flush_restore_request()
132 	rq->end_io = rq->flush.saved_end_io;  in blk_flush_restore_request()
135 static void blk_flush_queue_rq(struct request *rq, bool add_front)  in blk_flush_queue_rq()  argument
137 	blk_mq_add_to_requeue_list(rq, add_front, true);  in blk_flush_queue_rq()
140 static void blk_account_io_flush(struct request *rq)  in blk_account_io_flush()  argument
142 	struct hd_struct *part = &rq->rq_disk->part0;  in blk_account_io_flush()
147 		      ktime_get_ns() - rq->start_time_ns);  in blk_account_io_flush()
164 static void blk_flush_complete_seq(struct request *rq,  in blk_flush_complete_seq()  argument
168 	struct request_queue *q = rq->q;  in blk_flush_complete_seq()
172 	BUG_ON(rq->flush.seq & seq);  in blk_flush_complete_seq()
173 	rq->flush.seq |= seq;  in blk_flush_complete_seq()
174 	cmd_flags = rq->cmd_flags;  in blk_flush_complete_seq()
177 		seq = blk_flush_cur_seq(rq);  in blk_flush_complete_seq()
187 		list_move_tail(&rq->flush.list, pending);  in blk_flush_complete_seq()
191 		list_move_tail(&rq->flush.list, &fq->flush_data_in_flight);  in blk_flush_complete_seq()
192 		blk_flush_queue_rq(rq, true);  in blk_flush_complete_seq()
202 		BUG_ON(!list_empty(&rq->queuelist));  in blk_flush_complete_seq()
203 		list_del_init(&rq->flush.list);  in blk_flush_complete_seq()
204 		blk_flush_restore_request(rq);  in blk_flush_complete_seq()
205 		blk_mq_end_request(rq, error);  in blk_flush_complete_seq()
219 	struct request *rq, *n;  in flush_end_io()  local
258 	list_for_each_entry_safe(rq, n, running, flush.list) {  in flush_end_io()
259 		unsigned int seq = blk_flush_cur_seq(rq);  in flush_end_io()
262 		blk_flush_complete_seq(rq, fq, seq, error);  in flush_end_io()
268 bool is_flush_rq(struct request *rq)  in is_flush_rq()  argument
270 	return rq->end_io == flush_end_io;  in is_flush_rq()
352 static void mq_flush_data_end_io(struct request *rq, blk_status_t error)  in mq_flush_data_end_io()  argument
354 	struct request_queue *q = rq->q;  in mq_flush_data_end_io()
355 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;  in mq_flush_data_end_io()
356 	struct blk_mq_ctx *ctx = rq->mq_ctx;  in mq_flush_data_end_io()
361 		WARN_ON(rq->tag < 0);  in mq_flush_data_end_io()
362 		blk_mq_put_driver_tag(rq);  in mq_flush_data_end_io()
370 	blk_flush_complete_seq(rq, fq, REQ_FSEQ_DATA, error);  in mq_flush_data_end_io()
385 void blk_insert_flush(struct request *rq)  in blk_insert_flush()  argument
387 	struct request_queue *q = rq->q;  in blk_insert_flush()
389 	unsigned int policy = blk_flush_policy(fflags, rq);  in blk_insert_flush()
390 	struct blk_flush_queue *fq = blk_get_flush_queue(q, rq->mq_ctx);  in blk_insert_flush()
396 	rq->cmd_flags &= ~REQ_PREFLUSH;  in blk_insert_flush()
398 		rq->cmd_flags &= ~REQ_FUA;  in blk_insert_flush()
405 	rq->cmd_flags |= REQ_SYNC;  in blk_insert_flush()
414 		blk_mq_end_request(rq, 0);  in blk_insert_flush()
418 	BUG_ON(rq->bio != rq->biotail); /*assumes zero or single bio rq */  in blk_insert_flush()
427 		blk_mq_request_bypass_insert(rq, false, false);  in blk_insert_flush()
435 	memset(&rq->flush, 0, sizeof(rq->flush));  in blk_insert_flush()
436 	INIT_LIST_HEAD(&rq->flush.list);  in blk_insert_flush()
437 	rq->rq_flags |= RQF_FLUSH_SEQ;  in blk_insert_flush()
438 	rq->flush.saved_end_io = rq->end_io; /* Usually NULL */  in blk_insert_flush()
440 	rq->end_io = mq_flush_data_end_io;  in blk_insert_flush()
443 	blk_flush_complete_seq(rq, fq, REQ_FSEQ_ACTIONS & ~policy, 0);  in blk_insert_flush()