98 static unsigned int blk_flush_policy(unsigned int fflags, struct request *rq)  in blk_flush_policy()  argument
102 	if (blk_rq_sectors(rq))  in blk_flush_policy()
106 		if (rq->cmd_flags & REQ_FLUSH)  in blk_flush_policy()
108 		if (!(fflags & REQ_FUA) && (rq->cmd_flags & REQ_FUA))  in blk_flush_policy()
114 static unsigned int blk_flush_cur_seq(struct request *rq)  in blk_flush_cur_seq()  argument
116 	return 1 << ffz(rq->flush.seq);  in blk_flush_cur_seq()
119 static void blk_flush_restore_request(struct request *rq)  in blk_flush_restore_request()  argument
126 	rq->bio = rq->biotail;  in blk_flush_restore_request()
129 	rq->cmd_flags &= ~REQ_FLUSH_SEQ;  in blk_flush_restore_request()
130 	rq->end_io = rq->flush.saved_end_io;  in blk_flush_restore_request()
133 static bool blk_flush_queue_rq(struct request *rq, bool add_front)  in blk_flush_queue_rq()  argument
135 	if (rq->q->mq_ops) {  in blk_flush_queue_rq()
136 		struct request_queue *q = rq->q;  in blk_flush_queue_rq()
138 		blk_mq_add_to_requeue_list(rq, add_front);  in blk_flush_queue_rq()
143 			list_add(&rq->queuelist, &rq->q->queue_head);  in blk_flush_queue_rq()
145 			list_add_tail(&rq->queuelist, &rq->q->queue_head);  in blk_flush_queue_rq()
166 static bool blk_flush_complete_seq(struct request *rq,  in blk_flush_complete_seq()  argument
170 	struct request_queue *q = rq->q;  in blk_flush_complete_seq()
174 	BUG_ON(rq->flush.seq & seq);  in blk_flush_complete_seq()
175 	rq->flush.seq |= seq;  in blk_flush_complete_seq()
178 		seq = blk_flush_cur_seq(rq);  in blk_flush_complete_seq()
188 		list_move_tail(&rq->flush.list, pending);  in blk_flush_complete_seq()
192 		list_move_tail(&rq->flush.list, &fq->flush_data_in_flight);  in blk_flush_complete_seq()
193 		queued = blk_flush_queue_rq(rq, true);  in blk_flush_complete_seq()
203 		BUG_ON(!list_empty(&rq->queuelist));  in blk_flush_complete_seq()
204 		list_del_init(&rq->flush.list);  in blk_flush_complete_seq()
205 		blk_flush_restore_request(rq);  in blk_flush_complete_seq()
207 			blk_mq_end_request(rq, error);  in blk_flush_complete_seq()
209 			__blk_end_request_all(rq, error);  in blk_flush_complete_seq()
225 	struct request *rq, *n;  in flush_end_io()  local
249 	list_for_each_entry_safe(rq, n, running, flush.list) {  in flush_end_io()
250 		unsigned int seq = blk_flush_cur_seq(rq);  in flush_end_io()
253 		queued |= blk_flush_complete_seq(rq, fq, seq, error);  in flush_end_io()
339 static void flush_data_end_io(struct request *rq, int error)  in flush_data_end_io()  argument
341 	struct request_queue *q = rq->q;  in flush_data_end_io()
348 	if (blk_flush_complete_seq(rq, fq, REQ_FSEQ_DATA, error))  in flush_data_end_io()
352 static void mq_flush_data_end_io(struct request *rq, int error)  in mq_flush_data_end_io()  argument
354 	struct request_queue *q = rq->q;  in mq_flush_data_end_io()
356 	struct blk_mq_ctx *ctx = rq->mq_ctx;  in mq_flush_data_end_io()
367 	if (blk_flush_complete_seq(rq, fq, REQ_FSEQ_DATA, error))  in mq_flush_data_end_io()
384 void blk_insert_flush(struct request *rq)  in blk_insert_flush()  argument
386 	struct request_queue *q = rq->q;  in blk_insert_flush()
388 	unsigned int policy = blk_flush_policy(fflags, rq);  in blk_insert_flush()
389 	struct blk_flush_queue *fq = blk_get_flush_queue(q, rq->mq_ctx);  in blk_insert_flush()
395 	rq->cmd_flags &= ~REQ_FLUSH;  in blk_insert_flush()
397 		rq->cmd_flags &= ~REQ_FUA;  in blk_insert_flush()
407 			blk_mq_end_request(rq, 0);  in blk_insert_flush()
409 			__blk_end_bidi_request(rq, 0, 0, 0);  in blk_insert_flush()
413 	BUG_ON(rq->bio != rq->biotail); /*assumes zero or single bio rq */  in blk_insert_flush()
423 			blk_mq_insert_request(rq, false, false, true);  in blk_insert_flush()
425 			list_add_tail(&rq->queuelist, &q->queue_head);  in blk_insert_flush()
433 	memset(&rq->flush, 0, sizeof(rq->flush));  in blk_insert_flush()
434 	INIT_LIST_HEAD(&rq->flush.list);  in blk_insert_flush()
435 	rq->cmd_flags |= REQ_FLUSH_SEQ;  in blk_insert_flush()
436 	rq->flush.saved_end_io = rq->end_io; /* Usually NULL */  in blk_insert_flush()
438 		rq->end_io = mq_flush_data_end_io;  in blk_insert_flush()
441 		blk_flush_complete_seq(rq, fq, REQ_FSEQ_ACTIONS & ~policy, 0);  in blk_insert_flush()
445 	rq->end_io = flush_data_end_io;  in blk_insert_flush()
447 	blk_flush_complete_seq(rq, fq, REQ_FSEQ_ACTIONS & ~policy, 0);  in blk_insert_flush()