Lines Matching full:lc
137 static inline sector_t bio_to_dev_sectors(struct log_writes_c *lc, in bio_to_dev_sectors() argument
140 return sectors >> (lc->sectorshift - SECTOR_SHIFT); in bio_to_dev_sectors()
143 static inline sector_t dev_to_bio_sectors(struct log_writes_c *lc, in dev_to_bio_sectors() argument
146 return sectors << (lc->sectorshift - SECTOR_SHIFT); in dev_to_bio_sectors()
149 static void put_pending_block(struct log_writes_c *lc) in put_pending_block() argument
151 if (atomic_dec_and_test(&lc->pending_blocks)) { in put_pending_block()
153 if (waitqueue_active(&lc->wait)) in put_pending_block()
154 wake_up(&lc->wait); in put_pending_block()
158 static void put_io_block(struct log_writes_c *lc) in put_io_block() argument
160 if (atomic_dec_and_test(&lc->io_blocks)) { in put_io_block()
162 if (waitqueue_active(&lc->wait)) in put_io_block()
163 wake_up(&lc->wait); in put_io_block()
169 struct log_writes_c *lc = bio->bi_private; in log_end_io() local
175 spin_lock_irqsave(&lc->blocks_lock, flags); in log_end_io()
176 lc->logging_enabled = false; in log_end_io()
177 spin_unlock_irqrestore(&lc->blocks_lock, flags); in log_end_io()
181 put_io_block(lc); in log_end_io()
187 struct log_writes_c *lc = bio->bi_private; in log_end_super() local
189 complete(&lc->super_done); in log_end_super()
197 static void free_pending_block(struct log_writes_c *lc, in free_pending_block() argument
208 put_pending_block(lc); in free_pending_block()
211 static int write_metadata(struct log_writes_c *lc, void *entry, in write_metadata() argument
227 bio_set_dev(bio, lc->logdev->bdev); in write_metadata()
230 bio->bi_private = lc; in write_metadata()
245 lc->sectorsize - entrylen - datalen); in write_metadata()
248 ret = bio_add_page(bio, page, lc->sectorsize, 0); in write_metadata()
249 if (ret != lc->sectorsize) { in write_metadata()
259 put_io_block(lc); in write_metadata()
263 static int write_inline_data(struct log_writes_c *lc, void *entry, in write_inline_data() argument
277 atomic_inc(&lc->io_blocks); in write_inline_data()
287 bio_set_dev(bio, lc->logdev->bdev); in write_inline_data()
289 bio->bi_private = lc; in write_inline_data()
294 pg_sectorlen = ALIGN(pg_datalen, lc->sectorsize); in write_inline_data()
327 put_io_block(lc); in write_inline_data()
331 static int log_one_block(struct log_writes_c *lc, in log_one_block() argument
345 if (write_metadata(lc, &entry, sizeof(entry), block->data, in log_one_block()
347 free_pending_block(lc, block); in log_one_block()
351 sector += dev_to_bio_sectors(lc, 1); in log_one_block()
354 if (write_inline_data(lc, &entry, sizeof(entry), block->data, in log_one_block()
356 free_pending_block(lc, block); in log_one_block()
366 atomic_inc(&lc->io_blocks); in log_one_block()
374 bio_set_dev(bio, lc->logdev->bdev); in log_one_block()
376 bio->bi_private = lc; in log_one_block()
387 atomic_inc(&lc->io_blocks); in log_one_block()
396 bio_set_dev(bio, lc->logdev->bdev); in log_one_block()
398 bio->bi_private = lc; in log_one_block()
415 put_pending_block(lc); in log_one_block()
418 free_pending_block(lc, block); in log_one_block()
419 put_io_block(lc); in log_one_block()
423 static int log_super(struct log_writes_c *lc) in log_super() argument
429 super.nr_entries = cpu_to_le64(lc->logged_entries); in log_super()
430 super.sectorsize = cpu_to_le32(lc->sectorsize); in log_super()
432 if (write_metadata(lc, &super, sizeof(super), NULL, 0, in log_super()
442 wait_for_completion_io(&lc->super_done); in log_super()
447 static inline sector_t logdev_last_sector(struct log_writes_c *lc) in logdev_last_sector() argument
449 return i_size_read(lc->logdev->bdev->bd_inode) >> SECTOR_SHIFT; in logdev_last_sector()
454 struct log_writes_c *lc = (struct log_writes_c *)arg; in log_writes_kthread() local
463 spin_lock_irq(&lc->blocks_lock); in log_writes_kthread()
464 if (!list_empty(&lc->logging_blocks)) { in log_writes_kthread()
465 block = list_first_entry(&lc->logging_blocks, in log_writes_kthread()
468 if (!lc->logging_enabled) in log_writes_kthread()
471 sector = lc->next_sector; in log_writes_kthread()
473 lc->next_sector += dev_to_bio_sectors(lc, block->nr_sectors); in log_writes_kthread()
474 lc->next_sector += dev_to_bio_sectors(lc, 1); in log_writes_kthread()
480 if (!lc->end_sector) in log_writes_kthread()
481 lc->end_sector = logdev_last_sector(lc); in log_writes_kthread()
482 if (lc->end_sector && in log_writes_kthread()
483 lc->next_sector >= lc->end_sector) { in log_writes_kthread()
485 lc->logging_enabled = false; in log_writes_kthread()
488 lc->logged_entries++; in log_writes_kthread()
489 atomic_inc(&lc->io_blocks); in log_writes_kthread()
493 atomic_inc(&lc->io_blocks); in log_writes_kthread()
496 logging_enabled = lc->logging_enabled; in log_writes_kthread()
497 spin_unlock_irq(&lc->blocks_lock); in log_writes_kthread()
500 ret = log_one_block(lc, block, sector); in log_writes_kthread()
502 ret = log_super(lc); in log_writes_kthread()
504 spin_lock_irq(&lc->blocks_lock); in log_writes_kthread()
505 lc->logging_enabled = false; in log_writes_kthread()
506 spin_unlock_irq(&lc->blocks_lock); in log_writes_kthread()
509 free_pending_block(lc, block); in log_writes_kthread()
516 list_empty(&lc->logging_blocks)) in log_writes_kthread()
530 struct log_writes_c *lc; in log_writes_ctr() local
543 lc = kzalloc(sizeof(struct log_writes_c), GFP_KERNEL); in log_writes_ctr()
544 if (!lc) { in log_writes_ctr()
548 spin_lock_init(&lc->blocks_lock); in log_writes_ctr()
549 INIT_LIST_HEAD(&lc->unflushed_blocks); in log_writes_ctr()
550 INIT_LIST_HEAD(&lc->logging_blocks); in log_writes_ctr()
551 init_waitqueue_head(&lc->wait); in log_writes_ctr()
552 init_completion(&lc->super_done); in log_writes_ctr()
553 atomic_set(&lc->io_blocks, 0); in log_writes_ctr()
554 atomic_set(&lc->pending_blocks, 0); in log_writes_ctr()
557 ret = dm_get_device(ti, devname, dm_table_get_mode(ti->table), &lc->dev); in log_writes_ctr()
565 &lc->logdev); in log_writes_ctr()
568 dm_put_device(ti, lc->dev); in log_writes_ctr()
572 lc->sectorsize = bdev_logical_block_size(lc->dev->bdev); in log_writes_ctr()
573 lc->sectorshift = ilog2(lc->sectorsize); in log_writes_ctr()
574 lc->log_kthread = kthread_run(log_writes_kthread, lc, "log-write"); in log_writes_ctr()
575 if (IS_ERR(lc->log_kthread)) { in log_writes_ctr()
576 ret = PTR_ERR(lc->log_kthread); in log_writes_ctr()
578 dm_put_device(ti, lc->dev); in log_writes_ctr()
579 dm_put_device(ti, lc->logdev); in log_writes_ctr()
588 lc->next_sector = lc->sectorsize >> SECTOR_SHIFT; in log_writes_ctr()
589 lc->logging_enabled = true; in log_writes_ctr()
590 lc->end_sector = logdev_last_sector(lc); in log_writes_ctr()
591 lc->device_supports_discard = true; in log_writes_ctr()
598 ti->private = lc; in log_writes_ctr()
602 kfree(lc); in log_writes_ctr()
606 static int log_mark(struct log_writes_c *lc, char *data) in log_mark() argument
609 size_t maxsize = lc->sectorsize - sizeof(struct log_write_entry); in log_mark()
623 atomic_inc(&lc->pending_blocks); in log_mark()
626 spin_lock_irq(&lc->blocks_lock); in log_mark()
627 list_add_tail(&block->list, &lc->logging_blocks); in log_mark()
628 spin_unlock_irq(&lc->blocks_lock); in log_mark()
629 wake_up_process(lc->log_kthread); in log_mark()
635 struct log_writes_c *lc = ti->private; in log_writes_dtr() local
637 spin_lock_irq(&lc->blocks_lock); in log_writes_dtr()
638 list_splice_init(&lc->unflushed_blocks, &lc->logging_blocks); in log_writes_dtr()
639 spin_unlock_irq(&lc->blocks_lock); in log_writes_dtr()
645 log_mark(lc, "dm-log-writes-end"); in log_writes_dtr()
646 wake_up_process(lc->log_kthread); in log_writes_dtr()
647 wait_event(lc->wait, !atomic_read(&lc->io_blocks) && in log_writes_dtr()
648 !atomic_read(&lc->pending_blocks)); in log_writes_dtr()
649 kthread_stop(lc->log_kthread); in log_writes_dtr()
651 WARN_ON(!list_empty(&lc->logging_blocks)); in log_writes_dtr()
652 WARN_ON(!list_empty(&lc->unflushed_blocks)); in log_writes_dtr()
653 dm_put_device(ti, lc->dev); in log_writes_dtr()
654 dm_put_device(ti, lc->logdev); in log_writes_dtr()
655 kfree(lc); in log_writes_dtr()
660 struct log_writes_c *lc = ti->private; in normal_map_bio() local
662 bio_set_dev(bio, lc->dev->bdev); in normal_map_bio()
667 struct log_writes_c *lc = ti->private; in log_writes_map() local
682 if (!lc->logging_enabled) in log_writes_map()
707 spin_lock_irq(&lc->blocks_lock); in log_writes_map()
708 lc->logging_enabled = false; in log_writes_map()
709 spin_unlock_irq(&lc->blocks_lock); in log_writes_map()
714 atomic_inc(&lc->pending_blocks); in log_writes_map()
725 block->sector = bio_to_dev_sectors(lc, bio->bi_iter.bi_sector); in log_writes_map()
726 block->nr_sectors = bio_to_dev_sectors(lc, bio_sectors(bio)); in log_writes_map()
731 if (lc->device_supports_discard) in log_writes_map()
739 spin_lock_irq(&lc->blocks_lock); in log_writes_map()
740 list_splice_init(&lc->unflushed_blocks, &block->list); in log_writes_map()
741 spin_unlock_irq(&lc->blocks_lock); in log_writes_map()
761 free_pending_block(lc, block); in log_writes_map()
762 spin_lock_irq(&lc->blocks_lock); in log_writes_map()
763 lc->logging_enabled = false; in log_writes_map()
764 spin_unlock_irq(&lc->blocks_lock); in log_writes_map()
781 spin_lock_irq(&lc->blocks_lock); in log_writes_map()
782 list_splice_init(&lc->unflushed_blocks, &block->list); in log_writes_map()
783 spin_unlock_irq(&lc->blocks_lock); in log_writes_map()
793 struct log_writes_c *lc = ti->private; in normal_end_io() local
800 spin_lock_irqsave(&lc->blocks_lock, flags); in normal_end_io()
802 list_splice_tail_init(&block->list, &lc->logging_blocks); in normal_end_io()
803 list_add_tail(&block->list, &lc->logging_blocks); in normal_end_io()
804 wake_up_process(lc->log_kthread); in normal_end_io()
806 list_add_tail(&block->list, &lc->logging_blocks); in normal_end_io()
807 wake_up_process(lc->log_kthread); in normal_end_io()
809 list_add_tail(&block->list, &lc->unflushed_blocks); in normal_end_io()
810 spin_unlock_irqrestore(&lc->blocks_lock, flags); in normal_end_io()
824 struct log_writes_c *lc = ti->private; in log_writes_status() local
828 DMEMIT("%llu %llu", lc->logged_entries, in log_writes_status()
829 (unsigned long long)lc->next_sector - 1); in log_writes_status()
830 if (!lc->logging_enabled) in log_writes_status()
835 DMEMIT("%s %s", lc->dev->name, lc->logdev->name); in log_writes_status()
843 struct log_writes_c *lc = ti->private; in log_writes_prepare_ioctl() local
844 struct dm_dev *dev = lc->dev; in log_writes_prepare_ioctl()
859 struct log_writes_c *lc = ti->private; in log_writes_iterate_devices() local
861 return fn(ti, lc->dev, 0, ti->len, data); in log_writes_iterate_devices()
872 struct log_writes_c *lc = ti->private; in log_writes_message() local
880 r = log_mark(lc, argv[1]); in log_writes_message()
889 struct log_writes_c *lc = ti->private; in log_writes_io_hints() local
890 struct request_queue *q = bdev_get_queue(lc->dev->bdev); in log_writes_io_hints()
893 lc->device_supports_discard = false; in log_writes_io_hints()
894 limits->discard_granularity = lc->sectorsize; in log_writes_io_hints()
897 limits->logical_block_size = bdev_logical_block_size(lc->dev->bdev); in log_writes_io_hints()
898 limits->physical_block_size = bdev_physical_block_size(lc->dev->bdev); in log_writes_io_hints()
903 static int log_dax(struct log_writes_c *lc, sector_t sector, size_t bytes, in log_dax() argument
936 block->sector = bio_to_dev_sectors(lc, sector); in log_dax()
937 block->nr_sectors = ALIGN(bytes, lc->sectorsize) >> lc->sectorshift; in log_dax()
939 atomic_inc(&lc->pending_blocks); in log_dax()
940 spin_lock_irq(&lc->blocks_lock); in log_dax()
941 list_add_tail(&block->list, &lc->unflushed_blocks); in log_dax()
942 spin_unlock_irq(&lc->blocks_lock); in log_dax()
943 wake_up_process(lc->log_kthread); in log_dax()
951 struct log_writes_c *lc = ti->private; in log_writes_dax_direct_access() local
955 ret = bdev_dax_pgoff(lc->dev->bdev, sector, nr_pages * PAGE_SIZE, &pgoff); in log_writes_dax_direct_access()
958 return dax_direct_access(lc->dev->dax_dev, pgoff, nr_pages, kaddr, pfn); in log_writes_dax_direct_access()
965 struct log_writes_c *lc = ti->private; in log_writes_dax_copy_from_iter() local
969 if (bdev_dax_pgoff(lc->dev->bdev, sector, ALIGN(bytes, PAGE_SIZE), &pgoff)) in log_writes_dax_copy_from_iter()
973 if (!lc->logging_enabled) in log_writes_dax_copy_from_iter()
976 err = log_dax(lc, sector, bytes, i); in log_writes_dax_copy_from_iter()
982 return dax_copy_from_iter(lc->dev->dax_dev, pgoff, addr, bytes, i); in log_writes_dax_copy_from_iter()
989 struct log_writes_c *lc = ti->private; in log_writes_dax_copy_to_iter() local
992 if (bdev_dax_pgoff(lc->dev->bdev, sector, ALIGN(bytes, PAGE_SIZE), &pgoff)) in log_writes_dax_copy_to_iter()
994 return dax_copy_to_iter(lc->dev->dax_dev, pgoff, addr, bytes, i); in log_writes_dax_copy_to_iter()
1001 struct log_writes_c *lc = ti->private; in log_writes_dax_zero_page_range() local
1004 ret = bdev_dax_pgoff(lc->dev->bdev, sector, nr_pages << PAGE_SHIFT, in log_writes_dax_zero_page_range()
1008 return dax_zero_page_range(lc->dev->dax_dev, pgoff, in log_writes_dax_zero_page_range()