330 static int __blk_trace_remove(struct request_queue *q)  in __blk_trace_remove()  argument
334 	bt = xchg(&q->blk_trace, NULL);  in __blk_trace_remove()
344 int blk_trace_remove(struct request_queue *q)  in blk_trace_remove()  argument
348 	mutex_lock(&q->blk_trace_mutex);  in blk_trace_remove()
349 	ret = __blk_trace_remove(q);  in blk_trace_remove()
350 	mutex_unlock(&q->blk_trace_mutex);  in blk_trace_remove()
467 int do_blk_trace_setup(struct request_queue *q, char *name, dev_t dev,  in do_blk_trace_setup()  argument
491 	if (q->blk_trace) {  in do_blk_trace_setup()
563 	if (cmpxchg(&q->blk_trace, NULL, bt))  in do_blk_trace_setup()
574 static int __blk_trace_setup(struct request_queue *q, char *name, dev_t dev,  in __blk_trace_setup()  argument
584 	ret = do_blk_trace_setup(q, name, dev, bdev, &buts);  in __blk_trace_setup()
589 		__blk_trace_remove(q);  in __blk_trace_setup()
595 int blk_trace_setup(struct request_queue *q, char *name, dev_t dev,  in blk_trace_setup()  argument
601 	mutex_lock(&q->blk_trace_mutex);  in blk_trace_setup()
602 	ret = __blk_trace_setup(q, name, dev, bdev, arg);  in blk_trace_setup()
603 	mutex_unlock(&q->blk_trace_mutex);  in blk_trace_setup()
610 static int compat_blk_trace_setup(struct request_queue *q, char *name,  in compat_blk_trace_setup()  argument
630 	ret = do_blk_trace_setup(q, name, dev, bdev, &buts);  in compat_blk_trace_setup()
635 		__blk_trace_remove(q);  in compat_blk_trace_setup()
643 static int __blk_trace_startstop(struct request_queue *q, int start)  in __blk_trace_startstop()  argument
648 	bt = rcu_dereference_protected(q->blk_trace,  in __blk_trace_startstop()
649 				       lockdep_is_held(&q->blk_trace_mutex));  in __blk_trace_startstop()
685 int blk_trace_startstop(struct request_queue *q, int start)  in blk_trace_startstop()  argument
689 	mutex_lock(&q->blk_trace_mutex);  in blk_trace_startstop()
690 	ret = __blk_trace_startstop(q, start);  in blk_trace_startstop()
691 	mutex_unlock(&q->blk_trace_mutex);  in blk_trace_startstop()
712 	struct request_queue *q;  in blk_trace_ioctl()  local
716 	q = bdev_get_queue(bdev);  in blk_trace_ioctl()
717 	if (!q)  in blk_trace_ioctl()
720 	mutex_lock(&q->blk_trace_mutex);  in blk_trace_ioctl()
725 		ret = __blk_trace_setup(q, b, bdev->bd_dev, bdev, arg);  in blk_trace_ioctl()
730 		ret = compat_blk_trace_setup(q, b, bdev->bd_dev, bdev, arg);  in blk_trace_ioctl()
736 		ret = __blk_trace_startstop(q, start);  in blk_trace_ioctl()
739 		ret = __blk_trace_remove(q);  in blk_trace_ioctl()
746 	mutex_unlock(&q->blk_trace_mutex);  in blk_trace_ioctl()
755 void blk_trace_shutdown(struct request_queue *q)  in blk_trace_shutdown()  argument
757 	mutex_lock(&q->blk_trace_mutex);  in blk_trace_shutdown()
758 	if (rcu_dereference_protected(q->blk_trace,  in blk_trace_shutdown()
759 				      lockdep_is_held(&q->blk_trace_mutex))) {  in blk_trace_shutdown()
760 		__blk_trace_startstop(q, 0);  in blk_trace_shutdown()
761 		__blk_trace_remove(q);  in blk_trace_shutdown()
764 	mutex_unlock(&q->blk_trace_mutex);  in blk_trace_shutdown()
782 static void blk_add_trace_rq(struct request_queue *q, struct request *rq,  in blk_add_trace_rq()  argument
788 	bt = rcu_dereference(q->blk_trace);  in blk_add_trace_rq()
807 				   struct request_queue *q, struct request *rq)  in blk_add_trace_rq_abort()  argument
809 	blk_add_trace_rq(q, rq, blk_rq_bytes(rq), BLK_TA_ABORT);  in blk_add_trace_rq_abort()
813 				    struct request_queue *q, struct request *rq)  in blk_add_trace_rq_insert()  argument
815 	blk_add_trace_rq(q, rq, blk_rq_bytes(rq), BLK_TA_INSERT);  in blk_add_trace_rq_insert()
819 				   struct request_queue *q, struct request *rq)  in blk_add_trace_rq_issue()  argument
821 	blk_add_trace_rq(q, rq, blk_rq_bytes(rq), BLK_TA_ISSUE);  in blk_add_trace_rq_issue()
825 				     struct request_queue *q,  in blk_add_trace_rq_requeue()  argument
828 	blk_add_trace_rq(q, rq, blk_rq_bytes(rq), BLK_TA_REQUEUE);  in blk_add_trace_rq_requeue()
832 				      struct request_queue *q,  in blk_add_trace_rq_complete()  argument
836 	blk_add_trace_rq(q, rq, nr_bytes, BLK_TA_COMPLETE);  in blk_add_trace_rq_complete()
850 static void blk_add_trace_bio(struct request_queue *q, struct bio *bio,  in blk_add_trace_bio()  argument
856 	bt = rcu_dereference(q->blk_trace);  in blk_add_trace_bio()
868 				     struct request_queue *q, struct bio *bio)  in blk_add_trace_bio_bounce()  argument
870 	blk_add_trace_bio(q, bio, BLK_TA_BOUNCE, 0);  in blk_add_trace_bio_bounce()
874 				       struct request_queue *q, struct bio *bio,  in blk_add_trace_bio_complete()  argument
877 	blk_add_trace_bio(q, bio, BLK_TA_COMPLETE, error);  in blk_add_trace_bio_complete()
881 					struct request_queue *q,  in blk_add_trace_bio_backmerge()  argument
885 	blk_add_trace_bio(q, bio, BLK_TA_BACKMERGE, 0);  in blk_add_trace_bio_backmerge()
889 					 struct request_queue *q,  in blk_add_trace_bio_frontmerge()  argument
893 	blk_add_trace_bio(q, bio, BLK_TA_FRONTMERGE, 0);  in blk_add_trace_bio_frontmerge()
897 				    struct request_queue *q, struct bio *bio)  in blk_add_trace_bio_queue()  argument
899 	blk_add_trace_bio(q, bio, BLK_TA_QUEUE, 0);  in blk_add_trace_bio_queue()
903 				struct request_queue *q,  in blk_add_trace_getrq()  argument
907 		blk_add_trace_bio(q, bio, BLK_TA_GETRQ, 0);  in blk_add_trace_getrq()
912 		bt = rcu_dereference(q->blk_trace);  in blk_add_trace_getrq()
921 				  struct request_queue *q,  in blk_add_trace_sleeprq()  argument
925 		blk_add_trace_bio(q, bio, BLK_TA_SLEEPRQ, 0);  in blk_add_trace_sleeprq()
930 		bt = rcu_dereference(q->blk_trace);  in blk_add_trace_sleeprq()
938 static void blk_add_trace_plug(void *ignore, struct request_queue *q)  in blk_add_trace_plug()  argument
943 	bt = rcu_dereference(q->blk_trace);  in blk_add_trace_plug()
949 static void blk_add_trace_unplug(void *ignore, struct request_queue *q,  in blk_add_trace_unplug()  argument
955 	bt = rcu_dereference(q->blk_trace);  in blk_add_trace_unplug()
971 				struct request_queue *q, struct bio *bio,  in blk_add_trace_split()  argument
977 	bt = rcu_dereference(q->blk_trace);  in blk_add_trace_split()
1002 				    struct request_queue *q, struct bio *bio,  in blk_add_trace_bio_remap()  argument
1009 	bt = rcu_dereference(q->blk_trace);  in blk_add_trace_bio_remap()
1039 				   struct request_queue *q,  in blk_add_trace_rq_remap()  argument
1047 	bt = rcu_dereference(q->blk_trace);  in blk_add_trace_rq_remap()
1074 void blk_add_driver_data(struct request_queue *q,  in blk_add_driver_data()  argument
1081 	bt = rcu_dereference(q->blk_trace);  in blk_add_driver_data()
1576 static int blk_trace_remove_queue(struct request_queue *q)  in blk_trace_remove_queue()  argument
1580 	bt = xchg(&q->blk_trace, NULL);  in blk_trace_remove_queue()
1601 static int blk_trace_setup_queue(struct request_queue *q,  in blk_trace_setup_queue()  argument
1621 	if (cmpxchg(&q->blk_trace, NULL, bt))  in blk_trace_setup_queue()
1752 	struct request_queue *q;  in sysfs_blk_trace_attr_show()  local
1761 	q = blk_trace_get_queue(bdev);  in sysfs_blk_trace_attr_show()
1762 	if (q == NULL)  in sysfs_blk_trace_attr_show()
1765 	mutex_lock(&q->blk_trace_mutex);  in sysfs_blk_trace_attr_show()
1767 	bt = rcu_dereference_protected(q->blk_trace,  in sysfs_blk_trace_attr_show()
1768 				       lockdep_is_held(&q->blk_trace_mutex));  in sysfs_blk_trace_attr_show()
1786 	mutex_unlock(&q->blk_trace_mutex);  in sysfs_blk_trace_attr_show()
1798 	struct request_queue *q;  in sysfs_blk_trace_attr_store()  local
1825 	q = blk_trace_get_queue(bdev);  in sysfs_blk_trace_attr_store()
1826 	if (q == NULL)  in sysfs_blk_trace_attr_store()
1829 	mutex_lock(&q->blk_trace_mutex);  in sysfs_blk_trace_attr_store()
1831 	bt = rcu_dereference_protected(q->blk_trace,  in sysfs_blk_trace_attr_store()
1832 				       lockdep_is_held(&q->blk_trace_mutex));  in sysfs_blk_trace_attr_store()
1839 			ret = blk_trace_setup_queue(q, bdev);  in sysfs_blk_trace_attr_store()
1841 			ret = blk_trace_remove_queue(q);  in sysfs_blk_trace_attr_store()
1847 		ret = blk_trace_setup_queue(q, bdev);  in sysfs_blk_trace_attr_store()
1848 		bt = rcu_dereference_protected(q->blk_trace,  in sysfs_blk_trace_attr_store()
1849 				lockdep_is_held(&q->blk_trace_mutex));  in sysfs_blk_trace_attr_store()
1864 	mutex_unlock(&q->blk_trace_mutex);  in sysfs_blk_trace_attr_store()