Lines Matching refs:pool
223 struct pool { struct
278 static enum pool_mode get_pool_mode(struct pool *pool); argument
279 static void metadata_operation_failed(struct pool *pool, const char *op, int r);
286 struct pool *pool; member
306 struct pool *pool; member
327 static bool block_size_is_power_of_two(struct pool *pool) in block_size_is_power_of_two() argument
329 return pool->sectors_per_block_shift >= 0; in block_size_is_power_of_two()
332 static sector_t block_to_sectors(struct pool *pool, dm_block_t b) in block_to_sectors() argument
334 return block_size_is_power_of_two(pool) ? in block_to_sectors()
335 (b << pool->sectors_per_block_shift) : in block_to_sectors()
336 (b * pool->sectors_per_block); in block_to_sectors()
361 sector_t s = block_to_sectors(tc->pool, data_b); in issue_discard()
362 sector_t len = block_to_sectors(tc->pool, data_e - data_b); in issue_discard()
397 static void wake_worker(struct pool *pool) in wake_worker() argument
399 queue_work(pool->wq, &pool->worker); in wake_worker()
404 static int bio_detain(struct pool *pool, struct dm_cell_key *key, struct bio *bio, in bio_detain() argument
414 cell_prealloc = dm_bio_prison_alloc_cell(pool->prison, GFP_NOIO); in bio_detain()
416 r = dm_bio_detain(pool->prison, key, bio, cell_prealloc, cell_result); in bio_detain()
422 dm_bio_prison_free_cell(pool->prison, cell_prealloc); in bio_detain()
427 static void cell_release(struct pool *pool, in cell_release() argument
431 dm_cell_release(pool->prison, cell, bios); in cell_release()
432 dm_bio_prison_free_cell(pool->prison, cell); in cell_release()
435 static void cell_visit_release(struct pool *pool, in cell_visit_release() argument
440 dm_cell_visit_release(pool->prison, fn, context, cell); in cell_visit_release()
441 dm_bio_prison_free_cell(pool->prison, cell); in cell_visit_release()
444 static void cell_release_no_holder(struct pool *pool, in cell_release_no_holder() argument
448 dm_cell_release_no_holder(pool->prison, cell, bios); in cell_release_no_holder()
449 dm_bio_prison_free_cell(pool->prison, cell); in cell_release_no_holder()
452 static void cell_error_with_code(struct pool *pool, in cell_error_with_code() argument
455 dm_cell_error(pool->prison, cell, error_code); in cell_error_with_code()
456 dm_bio_prison_free_cell(pool->prison, cell); in cell_error_with_code()
459 static int get_pool_io_error_code(struct pool *pool) in get_pool_io_error_code() argument
461 return pool->out_of_data_space ? -ENOSPC : -EIO; in get_pool_io_error_code()
464 static void cell_error(struct pool *pool, struct dm_bio_prison_cell *cell) in cell_error() argument
466 int error = get_pool_io_error_code(pool); in cell_error()
468 cell_error_with_code(pool, cell, error); in cell_error()
471 static void cell_success(struct pool *pool, struct dm_bio_prison_cell *cell) in cell_success() argument
473 cell_error_with_code(pool, cell, 0); in cell_success()
476 static void cell_requeue(struct pool *pool, struct dm_bio_prison_cell *cell) in cell_requeue() argument
478 cell_error_with_code(pool, cell, DM_ENDIO_REQUEUE); in cell_requeue()
497 static void __pool_table_insert(struct pool *pool) in __pool_table_insert() argument
500 list_add(&pool->list, &dm_thin_pool_table.pools); in __pool_table_insert()
503 static void __pool_table_remove(struct pool *pool) in __pool_table_remove() argument
506 list_del(&pool->list); in __pool_table_remove()
509 static struct pool *__pool_table_lookup(struct mapped_device *md) in __pool_table_lookup()
511 struct pool *pool = NULL, *tmp; in __pool_table_lookup() local
517 pool = tmp; in __pool_table_lookup()
522 return pool; in __pool_table_lookup()
525 static struct pool *__pool_table_lookup_metadata_dev(struct block_device *md_dev) in __pool_table_lookup_metadata_dev()
527 struct pool *pool = NULL, *tmp; in __pool_table_lookup_metadata_dev() local
533 pool = tmp; in __pool_table_lookup_metadata_dev()
538 return pool; in __pool_table_lookup_metadata_dev()
584 struct pool *pool = tc->pool; in requeue_deferred_cells() local
596 cell_requeue(pool, cell); in requeue_deferred_cells()
615 static void error_retry_list_with_code(struct pool *pool, int error) in error_retry_list_with_code() argument
620 list_for_each_entry_rcu(tc, &pool->active_thins, list) in error_retry_list_with_code()
625 static void error_retry_list(struct pool *pool) in error_retry_list() argument
627 int error = get_pool_io_error_code(pool); in error_retry_list()
629 error_retry_list_with_code(pool, error); in error_retry_list()
641 struct pool *pool = tc->pool; in get_bio_block() local
644 if (block_size_is_power_of_two(pool)) in get_bio_block()
645 block_nr >>= pool->sectors_per_block_shift; in get_bio_block()
647 (void) sector_div(block_nr, pool->sectors_per_block); in get_bio_block()
658 struct pool *pool = tc->pool; in get_bio_block_range() local
662 b += pool->sectors_per_block - 1ull; /* so we round up */ in get_bio_block_range()
664 if (block_size_is_power_of_two(pool)) { in get_bio_block_range()
665 b >>= pool->sectors_per_block_shift; in get_bio_block_range()
666 e >>= pool->sectors_per_block_shift; in get_bio_block_range()
668 (void) sector_div(b, pool->sectors_per_block); in get_bio_block_range()
669 (void) sector_div(e, pool->sectors_per_block); in get_bio_block_range()
682 struct pool *pool = tc->pool; in remap() local
686 if (block_size_is_power_of_two(pool)) in remap()
688 (block << pool->sectors_per_block_shift) | in remap()
689 (bi_sector & (pool->sectors_per_block - 1)); in remap()
691 bio->bi_iter.bi_sector = (block * pool->sectors_per_block) + in remap()
692 sector_div(bi_sector, pool->sectors_per_block); in remap()
706 static void inc_all_io_entry(struct pool *pool, struct bio *bio) in inc_all_io_entry() argument
714 h->all_io_entry = dm_deferred_entry_inc(pool->all_io_ds); in inc_all_io_entry()
719 struct pool *pool = tc->pool; in issue() local
741 spin_lock_irqsave(&pool->lock, flags); in issue()
742 bio_list_add(&pool->deferred_flush_bios, bio); in issue()
743 spin_unlock_irqrestore(&pool->lock, flags); in issue()
795 struct pool *pool = m->tc->pool; in __complete_mapping_preparation() local
798 list_add_tail(&m->list, &pool->prepared_mappings); in __complete_mapping_preparation()
799 wake_worker(pool); in __complete_mapping_preparation()
806 struct pool *pool = m->tc->pool; in complete_mapping_preparation() local
808 spin_lock_irqsave(&pool->lock, flags); in complete_mapping_preparation()
810 spin_unlock_irqrestore(&pool->lock, flags); in complete_mapping_preparation()
848 struct pool *pool = tc->pool; in cell_defer_no_holder() local
852 cell_release_no_holder(pool, cell, &tc->deferred_bio_list); in cell_defer_no_holder()
855 wake_worker(pool); in cell_defer_no_holder()
877 inc_all_io_entry(info->tc->pool, bio); in __inc_remap_and_issue_cell()
905 cell_visit_release(tc->pool, __inc_remap_and_issue_cell, in inc_remap_and_issue_cell()
917 cell_error(m->tc->pool, m->cell); in process_prepared_mapping_fail()
919 mempool_free(m, m->tc->pool->mapping_pool); in process_prepared_mapping_fail()
925 struct pool *pool = tc->pool; in process_prepared_mapping() local
930 cell_error(pool, m->cell); in process_prepared_mapping()
941 metadata_operation_failed(pool, "dm_thin_insert_block", r); in process_prepared_mapping()
942 cell_error(pool, m->cell); in process_prepared_mapping()
956 inc_all_io_entry(tc->pool, m->cell->holder); in process_prepared_mapping()
963 mempool_free(m, pool->mapping_pool); in process_prepared_mapping()
973 mempool_free(m, tc->pool->mapping_pool); in free_discard_mapping()
995 metadata_operation_failed(tc->pool, "dm_thin_remove_range", r); in process_prepared_discard_no_passdown()
1001 mempool_free(m, tc->pool->mapping_pool); in process_prepared_discard_no_passdown()
1016 struct pool *pool = tc->pool; in passdown_double_checking_shared_status() local
1024 r = dm_pool_block_is_used(pool->pmd, b, &used); in passdown_double_checking_shared_status()
1037 r = dm_pool_block_is_used(pool->pmd, e, &used); in passdown_double_checking_shared_status()
1058 struct pool *pool = m->tc->pool; in queue_passdown_pt2() local
1060 spin_lock_irqsave(&pool->lock, flags); in queue_passdown_pt2()
1061 list_add_tail(&m->list, &pool->prepared_discards_pt2); in queue_passdown_pt2()
1062 spin_unlock_irqrestore(&pool->lock, flags); in queue_passdown_pt2()
1063 wake_worker(pool); in queue_passdown_pt2()
1080 struct pool *pool = tc->pool; in process_prepared_discard_passdown_pt1() local
1091 metadata_operation_failed(pool, "dm_thin_remove_range", r); in process_prepared_discard_passdown_pt1()
1094 mempool_free(m, pool->mapping_pool); in process_prepared_discard_passdown_pt1()
1102 r = dm_pool_inc_data_range(pool->pmd, m->data_block, data_end); in process_prepared_discard_passdown_pt1()
1104 metadata_operation_failed(pool, "dm_pool_inc_data_range", r); in process_prepared_discard_passdown_pt1()
1107 mempool_free(m, pool->mapping_pool); in process_prepared_discard_passdown_pt1()
1114 dm_device_name(tc->pool->pool_md)); in process_prepared_discard_passdown_pt1()
1137 struct pool *pool = tc->pool; in process_prepared_discard_passdown_pt2() local
1143 r = dm_pool_dec_data_range(pool->pmd, m->data_block, in process_prepared_discard_passdown_pt2()
1146 metadata_operation_failed(pool, "dm_pool_dec_data_range", r); in process_prepared_discard_passdown_pt2()
1152 mempool_free(m, pool->mapping_pool); in process_prepared_discard_passdown_pt2()
1155 static void process_prepared(struct pool *pool, struct list_head *head, in process_prepared() argument
1163 spin_lock_irqsave(&pool->lock, flags); in process_prepared()
1165 spin_unlock_irqrestore(&pool->lock, flags); in process_prepared()
1174 static int io_overlaps_block(struct pool *pool, struct bio *bio) in io_overlaps_block() argument
1177 (pool->sectors_per_block << SECTOR_SHIFT); in io_overlaps_block()
1180 static int io_overwrites_block(struct pool *pool, struct bio *bio) in io_overwrites_block() argument
1183 io_overlaps_block(pool, bio); in io_overwrites_block()
1193 static int ensure_next_mapping(struct pool *pool) in ensure_next_mapping() argument
1195 if (pool->next_mapping) in ensure_next_mapping()
1198 pool->next_mapping = mempool_alloc(pool->mapping_pool, GFP_ATOMIC); in ensure_next_mapping()
1200 return pool->next_mapping ? 0 : -ENOMEM; in ensure_next_mapping()
1203 static struct dm_thin_new_mapping *get_next_mapping(struct pool *pool) in get_next_mapping() argument
1205 struct dm_thin_new_mapping *m = pool->next_mapping; in get_next_mapping()
1207 BUG_ON(!pool->next_mapping); in get_next_mapping()
1213 pool->next_mapping = NULL; in get_next_mapping()
1228 r = dm_kcopyd_zero(tc->pool->copier, 1, &to, 0, copy_complete, m); in ll_zero()
1239 struct pool *pool = tc->pool; in remap_and_issue_overwrite() local
1245 inc_all_io_entry(pool, bio); in remap_and_issue_overwrite()
1259 struct pool *pool = tc->pool; in schedule_copy() local
1260 struct dm_thin_new_mapping *m = get_next_mapping(pool); in schedule_copy()
1275 if (!dm_deferred_set_add_work(pool->shared_read_ds, &m->list)) in schedule_copy()
1284 if (io_overwrites_block(pool, bio)) in schedule_copy()
1290 from.sector = data_origin * pool->sectors_per_block; in schedule_copy()
1294 to.sector = data_dest * pool->sectors_per_block; in schedule_copy()
1297 r = dm_kcopyd_copy(pool->copier, &from, 1, &to, in schedule_copy()
1314 if (len < pool->sectors_per_block && pool->pf.zero_new_blocks) { in schedule_copy()
1317 data_dest * pool->sectors_per_block + len, in schedule_copy()
1318 (data_dest + 1) * pool->sectors_per_block); in schedule_copy()
1331 tc->pool->sectors_per_block); in schedule_internal_copy()
1338 struct pool *pool = tc->pool; in schedule_zero() local
1339 struct dm_thin_new_mapping *m = get_next_mapping(pool); in schedule_zero()
1353 if (pool->pf.zero_new_blocks) { in schedule_zero()
1354 if (io_overwrites_block(pool, bio)) in schedule_zero()
1357 ll_zero(tc, m, data_block * pool->sectors_per_block, in schedule_zero()
1358 (data_block + 1) * pool->sectors_per_block); in schedule_zero()
1367 struct pool *pool = tc->pool; in schedule_external_copy() local
1368 sector_t virt_block_begin = virt_block * pool->sectors_per_block; in schedule_external_copy()
1369 sector_t virt_block_end = (virt_block + 1) * pool->sectors_per_block; in schedule_external_copy()
1374 pool->sectors_per_block); in schedule_external_copy()
1385 static void set_pool_mode(struct pool *pool, enum pool_mode new_mode);
1387 static void check_for_space(struct pool *pool) in check_for_space() argument
1392 if (get_pool_mode(pool) != PM_OUT_OF_DATA_SPACE) in check_for_space()
1395 r = dm_pool_get_free_block_count(pool->pmd, &nr_free); in check_for_space()
1400 set_pool_mode(pool, PM_WRITE); in check_for_space()
1407 static int commit(struct pool *pool) in commit() argument
1411 if (get_pool_mode(pool) >= PM_READ_ONLY) in commit()
1414 r = dm_pool_commit_metadata(pool->pmd); in commit()
1416 metadata_operation_failed(pool, "dm_pool_commit_metadata", r); in commit()
1418 check_for_space(pool); in commit()
1423 static void check_low_water_mark(struct pool *pool, dm_block_t free_blocks) in check_low_water_mark() argument
1427 if (free_blocks <= pool->low_water_blocks && !pool->low_water_triggered) { in check_low_water_mark()
1429 dm_device_name(pool->pool_md)); in check_low_water_mark()
1430 spin_lock_irqsave(&pool->lock, flags); in check_low_water_mark()
1431 pool->low_water_triggered = true; in check_low_water_mark()
1432 spin_unlock_irqrestore(&pool->lock, flags); in check_low_water_mark()
1433 dm_table_event(pool->ti->table); in check_low_water_mark()
1441 struct pool *pool = tc->pool; in alloc_data_block() local
1443 if (WARN_ON(get_pool_mode(pool) != PM_WRITE)) in alloc_data_block()
1446 r = dm_pool_get_free_block_count(pool->pmd, &free_blocks); in alloc_data_block()
1448 metadata_operation_failed(pool, "dm_pool_get_free_block_count", r); in alloc_data_block()
1452 check_low_water_mark(pool, free_blocks); in alloc_data_block()
1459 r = commit(pool); in alloc_data_block()
1463 r = dm_pool_get_free_block_count(pool->pmd, &free_blocks); in alloc_data_block()
1465 metadata_operation_failed(pool, "dm_pool_get_free_block_count", r); in alloc_data_block()
1470 set_pool_mode(pool, PM_OUT_OF_DATA_SPACE); in alloc_data_block()
1475 r = dm_pool_alloc_data_block(pool->pmd, result); in alloc_data_block()
1477 metadata_operation_failed(pool, "dm_pool_alloc_data_block", r); in alloc_data_block()
1499 static int should_error_unserviceable_bio(struct pool *pool) in should_error_unserviceable_bio() argument
1501 enum pool_mode m = get_pool_mode(pool); in should_error_unserviceable_bio()
1510 return pool->pf.error_if_no_space ? -ENOSPC : 0; in should_error_unserviceable_bio()
1522 static void handle_unserviceable_bio(struct pool *pool, struct bio *bio) in handle_unserviceable_bio() argument
1524 int error = should_error_unserviceable_bio(pool); in handle_unserviceable_bio()
1533 static void retry_bios_on_resume(struct pool *pool, struct dm_bio_prison_cell *cell) in retry_bios_on_resume() argument
1539 error = should_error_unserviceable_bio(pool); in retry_bios_on_resume()
1541 cell_error_with_code(pool, cell, error); in retry_bios_on_resume()
1546 cell_release(pool, cell, &bios); in retry_bios_on_resume()
1555 struct pool *pool = tc->pool; in process_discard_cell_no_passdown() local
1556 struct dm_thin_new_mapping *m = get_next_mapping(pool); in process_discard_cell_no_passdown()
1568 if (!dm_deferred_set_add_work(pool->all_io_ds, &m->list)) in process_discard_cell_no_passdown()
1569 pool->process_prepared_discard(m); in process_discard_cell_no_passdown()
1575 struct pool *pool = tc->pool; in break_up_discard_bio() local
1585 r = ensure_next_mapping(pool); in break_up_discard_bio()
1600 if (bio_detain(tc->pool, &data_key, NULL, &data_cell)) { in break_up_discard_bio()
1610 m = get_next_mapping(pool); in break_up_discard_bio()
1628 if (!dm_deferred_set_add_work(pool->all_io_ds, &m->list)) in break_up_discard_bio()
1629 pool->process_prepared_discard(m); in break_up_discard_bio()
1672 if (bio_detain(tc->pool, &virt_key, bio, &virt_cell)) in process_discard_bio()
1682 tc->pool->process_discard_cell(tc, virt_cell); in process_discard_bio()
1692 struct pool *pool = tc->pool; in break_sharing() local
1702 retry_bios_on_resume(pool, cell); in break_sharing()
1708 cell_error(pool, cell); in break_sharing()
1727 h->shared_read_entry = dm_deferred_entry_inc(info->tc->pool->shared_read_ds); in __remap_and_issue_shared_cell()
1728 inc_all_io_entry(info->tc->pool, bio); in __remap_and_issue_shared_cell()
1745 cell_visit_release(tc->pool, __remap_and_issue_shared_cell, in remap_and_issue_shared_cell()
1761 struct pool *pool = tc->pool; in process_shared_bio() local
1769 if (bio_detain(pool, &key, bio, &data_cell)) { in process_shared_bio()
1780 h->shared_read_entry = dm_deferred_entry_inc(pool->shared_read_ds); in process_shared_bio()
1781 inc_all_io_entry(pool, bio); in process_shared_bio()
1794 struct pool *pool = tc->pool; in provision_block() local
1800 inc_all_io_entry(pool, bio); in provision_block()
1827 retry_bios_on_resume(pool, cell); in provision_block()
1833 cell_error(pool, cell); in provision_block()
1841 struct pool *pool = tc->pool; in process_cell() local
1847 cell_requeue(pool, cell); in process_cell()
1857 inc_all_io_entry(pool, bio); in process_cell()
1865 inc_all_io_entry(pool, bio); in process_cell()
1895 struct pool *pool = tc->pool; in process_bio() local
1905 if (bio_detain(pool, &key, bio, &cell)) in process_bio()
1923 handle_unserviceable_bio(tc->pool, bio); in __process_bio_read_only()
1927 inc_all_io_entry(tc->pool, bio); in __process_bio_read_only()
1938 handle_unserviceable_bio(tc->pool, bio); in __process_bio_read_only()
1943 inc_all_io_entry(tc->pool, bio); in __process_bio_read_only()
1984 cell_success(tc->pool, cell); in process_cell_success()
1989 cell_error(tc->pool, cell); in process_cell_fail()
1996 static int need_commit_due_to_time(struct pool *pool) in need_commit_due_to_time() argument
1998 return !time_in_range(jiffies, pool->last_commit_jiffies, in need_commit_due_to_time()
1999 pool->last_commit_jiffies + COMMIT_PERIOD); in need_commit_due_to_time()
2068 struct pool *pool = tc->pool; in process_thin_deferred_bios() local
2103 if (ensure_next_mapping(pool)) { in process_thin_deferred_bios()
2112 pool->process_discard(tc, bio); in process_thin_deferred_bios()
2114 pool->process_bio(tc, bio); in process_thin_deferred_bios()
2117 throttle_work_update(&pool->throttle); in process_thin_deferred_bios()
2118 dm_pool_issue_prefetches(pool->pmd); in process_thin_deferred_bios()
2141 static unsigned sort_cells(struct pool *pool, struct list_head *cells) in sort_cells() argument
2150 pool->cell_sort_array[count++] = cell; in sort_cells()
2154 sort(pool->cell_sort_array, count, sizeof(cell), cmp_cells, NULL); in sort_cells()
2161 struct pool *pool = tc->pool; in process_thin_deferred_cells() local
2177 count = sort_cells(tc->pool, &cells); in process_thin_deferred_cells()
2180 cell = pool->cell_sort_array[i]; in process_thin_deferred_cells()
2188 if (ensure_next_mapping(pool)) { in process_thin_deferred_cells()
2190 list_add(&pool->cell_sort_array[j]->user_list, &cells); in process_thin_deferred_cells()
2199 pool->process_discard_cell(tc, cell); in process_thin_deferred_cells()
2201 pool->process_cell(tc, cell); in process_thin_deferred_cells()
2214 static struct thin_c *get_first_thin(struct pool *pool) in get_first_thin() argument
2219 if (!list_empty(&pool->active_thins)) { in get_first_thin()
2220 tc = list_entry_rcu(pool->active_thins.next, struct thin_c, list); in get_first_thin()
2228 static struct thin_c *get_next_thin(struct pool *pool, struct thin_c *tc) in get_next_thin() argument
2233 list_for_each_entry_continue_rcu(tc, &pool->active_thins, list) { in get_next_thin()
2245 static void process_deferred_bios(struct pool *pool) in process_deferred_bios() argument
2252 tc = get_first_thin(pool); in process_deferred_bios()
2256 tc = get_next_thin(pool, tc); in process_deferred_bios()
2264 spin_lock_irqsave(&pool->lock, flags); in process_deferred_bios()
2265 bio_list_merge(&bios, &pool->deferred_flush_bios); in process_deferred_bios()
2266 bio_list_init(&pool->deferred_flush_bios); in process_deferred_bios()
2267 spin_unlock_irqrestore(&pool->lock, flags); in process_deferred_bios()
2270 !(dm_pool_changed_this_transaction(pool->pmd) && need_commit_due_to_time(pool))) in process_deferred_bios()
2273 if (commit(pool)) { in process_deferred_bios()
2278 pool->last_commit_jiffies = jiffies; in process_deferred_bios()
2286 struct pool *pool = container_of(ws, struct pool, worker); in do_worker() local
2288 throttle_work_start(&pool->throttle); in do_worker()
2289 dm_pool_issue_prefetches(pool->pmd); in do_worker()
2290 throttle_work_update(&pool->throttle); in do_worker()
2291 process_prepared(pool, &pool->prepared_mappings, &pool->process_prepared_mapping); in do_worker()
2292 throttle_work_update(&pool->throttle); in do_worker()
2293 process_prepared(pool, &pool->prepared_discards, &pool->process_prepared_discard); in do_worker()
2294 throttle_work_update(&pool->throttle); in do_worker()
2295 process_prepared(pool, &pool->prepared_discards_pt2, &pool->process_prepared_discard_pt2); in do_worker()
2296 throttle_work_update(&pool->throttle); in do_worker()
2297 process_deferred_bios(pool); in do_worker()
2298 throttle_work_complete(&pool->throttle); in do_worker()
2307 struct pool *pool = container_of(to_delayed_work(ws), struct pool, waker); in do_waker() local
2308 wake_worker(pool); in do_waker()
2309 queue_delayed_work(pool->wq, &pool->waker, COMMIT_PERIOD); in do_waker()
2312 static void notify_of_pool_mode_change_to_oods(struct pool *pool);
2321 struct pool *pool = container_of(to_delayed_work(ws), struct pool, in do_no_space_timeout() local
2324 if (get_pool_mode(pool) == PM_OUT_OF_DATA_SPACE && !pool->pf.error_if_no_space) { in do_no_space_timeout()
2325 pool->pf.error_if_no_space = true; in do_no_space_timeout()
2326 notify_of_pool_mode_change_to_oods(pool); in do_no_space_timeout()
2327 error_retry_list_with_code(pool, -ENOSPC); in do_no_space_timeout()
2348 static void pool_work_wait(struct pool_work *pw, struct pool *pool, in pool_work_wait() argument
2353 queue_work(pool->wq, &pw->worker); in pool_work_wait()
2389 pool_work_wait(&w.pw, tc->pool, fn); in noflush_work()
2394 static enum pool_mode get_pool_mode(struct pool *pool) in get_pool_mode() argument
2396 return pool->pf.mode; in get_pool_mode()
2399 static void notify_of_pool_mode_change(struct pool *pool, const char *new_mode) in notify_of_pool_mode_change() argument
2401 dm_table_event(pool->ti->table); in notify_of_pool_mode_change()
2403 dm_device_name(pool->pool_md), new_mode); in notify_of_pool_mode_change()
2406 static void notify_of_pool_mode_change_to_oods(struct pool *pool) in notify_of_pool_mode_change_to_oods() argument
2408 if (!pool->pf.error_if_no_space) in notify_of_pool_mode_change_to_oods()
2409 notify_of_pool_mode_change(pool, "out-of-data-space (queue IO)"); in notify_of_pool_mode_change_to_oods()
2411 notify_of_pool_mode_change(pool, "out-of-data-space (error IO)"); in notify_of_pool_mode_change_to_oods()
2419 static void set_discard_callbacks(struct pool *pool) in set_discard_callbacks() argument
2421 struct pool_c *pt = pool->ti->private; in set_discard_callbacks()
2424 pool->process_discard_cell = process_discard_cell_passdown; in set_discard_callbacks()
2425 pool->process_prepared_discard = process_prepared_discard_passdown_pt1; in set_discard_callbacks()
2426 pool->process_prepared_discard_pt2 = process_prepared_discard_passdown_pt2; in set_discard_callbacks()
2428 pool->process_discard_cell = process_discard_cell_no_passdown; in set_discard_callbacks()
2429 pool->process_prepared_discard = process_prepared_discard_no_passdown; in set_discard_callbacks()
2433 static void set_pool_mode(struct pool *pool, enum pool_mode new_mode) in set_pool_mode() argument
2435 struct pool_c *pt = pool->ti->private; in set_pool_mode()
2436 bool needs_check = dm_pool_metadata_needs_check(pool->pmd); in set_pool_mode()
2437 enum pool_mode old_mode = get_pool_mode(pool); in set_pool_mode()
2446 dm_device_name(pool->pool_md)); in set_pool_mode()
2463 notify_of_pool_mode_change(pool, "failure"); in set_pool_mode()
2464 dm_pool_metadata_read_only(pool->pmd); in set_pool_mode()
2465 pool->process_bio = process_bio_fail; in set_pool_mode()
2466 pool->process_discard = process_bio_fail; in set_pool_mode()
2467 pool->process_cell = process_cell_fail; in set_pool_mode()
2468 pool->process_discard_cell = process_cell_fail; in set_pool_mode()
2469 pool->process_prepared_mapping = process_prepared_mapping_fail; in set_pool_mode()
2470 pool->process_prepared_discard = process_prepared_discard_fail; in set_pool_mode()
2472 error_retry_list(pool); in set_pool_mode()
2477 notify_of_pool_mode_change(pool, "read-only"); in set_pool_mode()
2478 dm_pool_metadata_read_only(pool->pmd); in set_pool_mode()
2479 pool->process_bio = process_bio_read_only; in set_pool_mode()
2480 pool->process_discard = process_bio_success; in set_pool_mode()
2481 pool->process_cell = process_cell_read_only; in set_pool_mode()
2482 pool->process_discard_cell = process_cell_success; in set_pool_mode()
2483 pool->process_prepared_mapping = process_prepared_mapping_fail; in set_pool_mode()
2484 pool->process_prepared_discard = process_prepared_discard_success; in set_pool_mode()
2486 error_retry_list(pool); in set_pool_mode()
2499 notify_of_pool_mode_change_to_oods(pool); in set_pool_mode()
2500 pool->out_of_data_space = true; in set_pool_mode()
2501 pool->process_bio = process_bio_read_only; in set_pool_mode()
2502 pool->process_discard = process_discard_bio; in set_pool_mode()
2503 pool->process_cell = process_cell_read_only; in set_pool_mode()
2504 pool->process_prepared_mapping = process_prepared_mapping; in set_pool_mode()
2505 set_discard_callbacks(pool); in set_pool_mode()
2507 if (!pool->pf.error_if_no_space && no_space_timeout) in set_pool_mode()
2508 queue_delayed_work(pool->wq, &pool->no_space_timeout, no_space_timeout); in set_pool_mode()
2513 notify_of_pool_mode_change(pool, "write"); in set_pool_mode()
2514 pool->out_of_data_space = false; in set_pool_mode()
2515 pool->pf.error_if_no_space = pt->requested_pf.error_if_no_space; in set_pool_mode()
2516 dm_pool_metadata_read_write(pool->pmd); in set_pool_mode()
2517 pool->process_bio = process_bio; in set_pool_mode()
2518 pool->process_discard = process_discard_bio; in set_pool_mode()
2519 pool->process_cell = process_cell; in set_pool_mode()
2520 pool->process_prepared_mapping = process_prepared_mapping; in set_pool_mode()
2521 set_discard_callbacks(pool); in set_pool_mode()
2525 pool->pf.mode = new_mode; in set_pool_mode()
2533 static void abort_transaction(struct pool *pool) in abort_transaction() argument
2535 const char *dev_name = dm_device_name(pool->pool_md); in abort_transaction()
2538 if (dm_pool_abort_metadata(pool->pmd)) { in abort_transaction()
2540 set_pool_mode(pool, PM_FAIL); in abort_transaction()
2543 if (dm_pool_metadata_set_needs_check(pool->pmd)) { in abort_transaction()
2545 set_pool_mode(pool, PM_FAIL); in abort_transaction()
2549 static void metadata_operation_failed(struct pool *pool, const char *op, int r) in metadata_operation_failed() argument
2552 dm_device_name(pool->pool_md), op, r); in metadata_operation_failed()
2554 abort_transaction(pool); in metadata_operation_failed()
2555 set_pool_mode(pool, PM_READ_ONLY); in metadata_operation_failed()
2570 struct pool *pool = tc->pool; in thin_defer_bio() local
2576 wake_worker(pool); in thin_defer_bio()
2581 struct pool *pool = tc->pool; in thin_defer_bio_with_throttle() local
2583 throttle_lock(&pool->throttle); in thin_defer_bio_with_throttle()
2585 throttle_unlock(&pool->throttle); in thin_defer_bio_with_throttle()
2591 struct pool *pool = tc->pool; in thin_defer_cell() local
2593 throttle_lock(&pool->throttle); in thin_defer_cell()
2597 throttle_unlock(&pool->throttle); in thin_defer_cell()
2599 wake_worker(pool); in thin_defer_cell()
2634 if (get_pool_mode(tc->pool) == PM_FAIL) { in thin_bio_map()
2650 if (bio_detain(tc->pool, &key, bio, &virt_cell)) in thin_bio_map()
2680 if (bio_detain(tc->pool, &key, bio, &data_cell)) { in thin_bio_map()
2685 inc_all_io_entry(tc->pool, bio); in thin_bio_map()
2714 if (get_pool_mode(pt->pool) == PM_OUT_OF_DATA_SPACE) in pool_is_congested()
2721 static void requeue_bios(struct pool *pool) in requeue_bios() argument
2727 list_for_each_entry_rcu(tc, &pool->active_thins, list) { in requeue_bios()
2757 struct pool *pool = pt->pool; in disable_passdown_if_not_supported() local
2769 else if (data_limits->max_discard_sectors < pool->sectors_per_block) in disable_passdown_if_not_supported()
2778 static int bind_control_target(struct pool *pool, struct dm_target *ti) in bind_control_target() argument
2785 enum pool_mode old_mode = get_pool_mode(pool); in bind_control_target()
2795 pool->ti = ti; in bind_control_target()
2796 pool->pf = pt->adjusted_pf; in bind_control_target()
2797 pool->low_water_blocks = pt->low_water_blocks; in bind_control_target()
2799 set_pool_mode(pool, new_mode); in bind_control_target()
2804 static void unbind_control_target(struct pool *pool, struct dm_target *ti) in unbind_control_target() argument
2806 if (pool->ti == ti) in unbind_control_target()
2807 pool->ti = NULL; in unbind_control_target()
2823 static void __pool_destroy(struct pool *pool) in __pool_destroy() argument
2825 __pool_table_remove(pool); in __pool_destroy()
2827 vfree(pool->cell_sort_array); in __pool_destroy()
2828 if (dm_pool_metadata_close(pool->pmd) < 0) in __pool_destroy()
2831 dm_bio_prison_destroy(pool->prison); in __pool_destroy()
2832 dm_kcopyd_client_destroy(pool->copier); in __pool_destroy()
2834 if (pool->wq) in __pool_destroy()
2835 destroy_workqueue(pool->wq); in __pool_destroy()
2837 if (pool->next_mapping) in __pool_destroy()
2838 mempool_free(pool->next_mapping, pool->mapping_pool); in __pool_destroy()
2839 mempool_destroy(pool->mapping_pool); in __pool_destroy()
2840 dm_deferred_set_destroy(pool->shared_read_ds); in __pool_destroy()
2841 dm_deferred_set_destroy(pool->all_io_ds); in __pool_destroy()
2842 kfree(pool); in __pool_destroy()
2847 static struct pool *pool_create(struct mapped_device *pool_md, in pool_create()
2854 struct pool *pool; in pool_create() local
2861 return (struct pool *)pmd; in pool_create()
2864 pool = kmalloc(sizeof(*pool), GFP_KERNEL); in pool_create()
2865 if (!pool) { in pool_create()
2871 pool->pmd = pmd; in pool_create()
2872 pool->sectors_per_block = block_size; in pool_create()
2874 pool->sectors_per_block_shift = -1; in pool_create()
2876 pool->sectors_per_block_shift = __ffs(block_size); in pool_create()
2877 pool->low_water_blocks = 0; in pool_create()
2878 pool_features_init(&pool->pf); in pool_create()
2879 pool->prison = dm_bio_prison_create(); in pool_create()
2880 if (!pool->prison) { in pool_create()
2886 pool->copier = dm_kcopyd_client_create(&dm_kcopyd_throttle); in pool_create()
2887 if (IS_ERR(pool->copier)) { in pool_create()
2888 r = PTR_ERR(pool->copier); in pool_create()
2898 pool->wq = alloc_ordered_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM); in pool_create()
2899 if (!pool->wq) { in pool_create()
2905 throttle_init(&pool->throttle); in pool_create()
2906 INIT_WORK(&pool->worker, do_worker); in pool_create()
2907 INIT_DELAYED_WORK(&pool->waker, do_waker); in pool_create()
2908 INIT_DELAYED_WORK(&pool->no_space_timeout, do_no_space_timeout); in pool_create()
2909 spin_lock_init(&pool->lock); in pool_create()
2910 bio_list_init(&pool->deferred_flush_bios); in pool_create()
2911 INIT_LIST_HEAD(&pool->prepared_mappings); in pool_create()
2912 INIT_LIST_HEAD(&pool->prepared_discards); in pool_create()
2913 INIT_LIST_HEAD(&pool->prepared_discards_pt2); in pool_create()
2914 INIT_LIST_HEAD(&pool->active_thins); in pool_create()
2915 pool->low_water_triggered = false; in pool_create()
2916 pool->suspended = true; in pool_create()
2917 pool->out_of_data_space = false; in pool_create()
2919 pool->shared_read_ds = dm_deferred_set_create(); in pool_create()
2920 if (!pool->shared_read_ds) { in pool_create()
2926 pool->all_io_ds = dm_deferred_set_create(); in pool_create()
2927 if (!pool->all_io_ds) { in pool_create()
2933 pool->next_mapping = NULL; in pool_create()
2934 pool->mapping_pool = mempool_create_slab_pool(MAPPING_POOL_SIZE, in pool_create()
2936 if (!pool->mapping_pool) { in pool_create()
2942 pool->cell_sort_array = vmalloc(sizeof(*pool->cell_sort_array) * CELL_SORT_ARRAY_SIZE); in pool_create()
2943 if (!pool->cell_sort_array) { in pool_create()
2949 pool->ref_count = 1; in pool_create()
2950 pool->last_commit_jiffies = jiffies; in pool_create()
2951 pool->pool_md = pool_md; in pool_create()
2952 pool->md_dev = metadata_dev; in pool_create()
2953 __pool_table_insert(pool); in pool_create()
2955 return pool; in pool_create()
2958 mempool_destroy(pool->mapping_pool); in pool_create()
2960 dm_deferred_set_destroy(pool->all_io_ds); in pool_create()
2962 dm_deferred_set_destroy(pool->shared_read_ds); in pool_create()
2964 destroy_workqueue(pool->wq); in pool_create()
2966 dm_kcopyd_client_destroy(pool->copier); in pool_create()
2968 dm_bio_prison_destroy(pool->prison); in pool_create()
2970 kfree(pool); in pool_create()
2978 static void __pool_inc(struct pool *pool) in __pool_inc() argument
2981 pool->ref_count++; in __pool_inc()
2984 static void __pool_dec(struct pool *pool) in __pool_dec() argument
2987 BUG_ON(!pool->ref_count); in __pool_dec()
2988 if (!--pool->ref_count) in __pool_dec()
2989 __pool_destroy(pool); in __pool_dec()
2992 static struct pool *__pool_find(struct mapped_device *pool_md, in __pool_find()
2997 struct pool *pool = __pool_table_lookup_metadata_dev(metadata_dev); in __pool_find() local
2999 if (pool) { in __pool_find()
3000 if (pool->pool_md != pool_md) { in __pool_find()
3004 __pool_inc(pool); in __pool_find()
3007 pool = __pool_table_lookup(pool_md); in __pool_find()
3008 if (pool) { in __pool_find()
3009 if (pool->md_dev != metadata_dev) { in __pool_find()
3013 __pool_inc(pool); in __pool_find()
3016 pool = pool_create(pool_md, metadata_dev, block_size, read_only, error); in __pool_find()
3021 return pool; in __pool_find()
3033 unbind_control_target(pt->pool, ti); in pool_dtr()
3034 __pool_dec(pt->pool); in pool_dtr()
3094 struct pool *pool = context; in metadata_low_callback() local
3097 dm_device_name(pool->pool_md)); in metadata_low_callback()
3099 dm_table_event(pool->ti->table); in metadata_low_callback()
3170 struct pool *pool; in pool_ctr() local
3238 pool = __pool_find(dm_table_get_md(ti->table), metadata_dev->bdev, in pool_ctr()
3240 if (IS_ERR(pool)) { in pool_ctr()
3241 r = PTR_ERR(pool); in pool_ctr()
3251 if (!pool_created && pf.discard_enabled != pool->pf.discard_enabled) { in pool_ctr()
3257 pt->pool = pool; in pool_ctr()
3283 r = dm_pool_register_metadata_threshold(pt->pool->pmd, in pool_ctr()
3286 pool); in pool_ctr()
3298 __pool_dec(pool); in pool_ctr()
3315 struct pool *pool = pt->pool; in pool_map() local
3321 spin_lock_irqsave(&pool->lock, flags); in pool_map()
3324 spin_unlock_irqrestore(&pool->lock, flags); in pool_map()
3333 struct pool *pool = pt->pool; in maybe_resize_data_dev() local
3339 (void) sector_div(data_size, pool->sectors_per_block); in maybe_resize_data_dev()
3341 r = dm_pool_get_data_dev_size(pool->pmd, &sb_data_size); in maybe_resize_data_dev()
3344 dm_device_name(pool->pool_md)); in maybe_resize_data_dev()
3350 dm_device_name(pool->pool_md), in maybe_resize_data_dev()
3355 if (dm_pool_metadata_needs_check(pool->pmd)) { in maybe_resize_data_dev()
3357 dm_device_name(pool->pool_md)); in maybe_resize_data_dev()
3363 dm_device_name(pool->pool_md), in maybe_resize_data_dev()
3365 r = dm_pool_resize_data_dev(pool->pmd, data_size); in maybe_resize_data_dev()
3367 metadata_operation_failed(pool, "dm_pool_resize_data_dev", r); in maybe_resize_data_dev()
3381 struct pool *pool = pt->pool; in maybe_resize_metadata_dev() local
3386 metadata_dev_size = get_metadata_dev_size_in_blocks(pool->md_dev); in maybe_resize_metadata_dev()
3388 r = dm_pool_get_metadata_dev_size(pool->pmd, &sb_metadata_dev_size); in maybe_resize_metadata_dev()
3391 dm_device_name(pool->pool_md)); in maybe_resize_metadata_dev()
3397 dm_device_name(pool->pool_md), in maybe_resize_metadata_dev()
3402 if (dm_pool_metadata_needs_check(pool->pmd)) { in maybe_resize_metadata_dev()
3404 dm_device_name(pool->pool_md)); in maybe_resize_metadata_dev()
3408 warn_if_metadata_device_too_big(pool->md_dev); in maybe_resize_metadata_dev()
3410 dm_device_name(pool->pool_md), in maybe_resize_metadata_dev()
3412 r = dm_pool_resize_metadata_dev(pool->pmd, metadata_dev_size); in maybe_resize_metadata_dev()
3414 metadata_operation_failed(pool, "dm_pool_resize_metadata_dev", r); in maybe_resize_metadata_dev()
3440 struct pool *pool = pt->pool; in pool_preresume() local
3445 r = bind_control_target(pool, ti); in pool_preresume()
3458 (void) commit(pool); in pool_preresume()
3463 static void pool_suspend_active_thins(struct pool *pool) in pool_suspend_active_thins() argument
3468 tc = get_first_thin(pool); in pool_suspend_active_thins()
3471 tc = get_next_thin(pool, tc); in pool_suspend_active_thins()
3475 static void pool_resume_active_thins(struct pool *pool) in pool_resume_active_thins() argument
3480 tc = get_first_thin(pool); in pool_resume_active_thins()
3483 tc = get_next_thin(pool, tc); in pool_resume_active_thins()
3490 struct pool *pool = pt->pool; in pool_resume() local
3497 requeue_bios(pool); in pool_resume()
3498 pool_resume_active_thins(pool); in pool_resume()
3500 spin_lock_irqsave(&pool->lock, flags); in pool_resume()
3501 pool->low_water_triggered = false; in pool_resume()
3502 pool->suspended = false; in pool_resume()
3503 spin_unlock_irqrestore(&pool->lock, flags); in pool_resume()
3505 do_waker(&pool->waker.work); in pool_resume()
3511 struct pool *pool = pt->pool; in pool_presuspend() local
3514 spin_lock_irqsave(&pool->lock, flags); in pool_presuspend()
3515 pool->suspended = true; in pool_presuspend()
3516 spin_unlock_irqrestore(&pool->lock, flags); in pool_presuspend()
3518 pool_suspend_active_thins(pool); in pool_presuspend()
3524 struct pool *pool = pt->pool; in pool_presuspend_undo() local
3527 pool_resume_active_thins(pool); in pool_presuspend_undo()
3529 spin_lock_irqsave(&pool->lock, flags); in pool_presuspend_undo()
3530 pool->suspended = false; in pool_presuspend_undo()
3531 spin_unlock_irqrestore(&pool->lock, flags); in pool_presuspend_undo()
3537 struct pool *pool = pt->pool; in pool_postsuspend() local
3539 cancel_delayed_work_sync(&pool->waker); in pool_postsuspend()
3540 cancel_delayed_work_sync(&pool->no_space_timeout); in pool_postsuspend()
3541 flush_workqueue(pool->wq); in pool_postsuspend()
3542 (void) commit(pool); in pool_postsuspend()
3568 static int process_create_thin_mesg(unsigned argc, char **argv, struct pool *pool) in process_create_thin_mesg() argument
3581 r = dm_pool_create_thin(pool->pmd, dev_id); in process_create_thin_mesg()
3591 static int process_create_snap_mesg(unsigned argc, char **argv, struct pool *pool) in process_create_snap_mesg() argument
3609 r = dm_pool_create_snap(pool->pmd, dev_id, origin_dev_id); in process_create_snap_mesg()
3619 static int process_delete_mesg(unsigned argc, char **argv, struct pool *pool) in process_delete_mesg() argument
3632 r = dm_pool_delete_thin_device(pool->pmd, dev_id); in process_delete_mesg()
3639 static int process_set_transaction_id_mesg(unsigned argc, char **argv, struct pool *pool) in process_set_transaction_id_mesg() argument
3658 r = dm_pool_set_metadata_transaction_id(pool->pmd, old_id, new_id); in process_set_transaction_id_mesg()
3668 static int process_reserve_metadata_snap_mesg(unsigned argc, char **argv, struct pool *pool) in process_reserve_metadata_snap_mesg() argument
3676 (void) commit(pool); in process_reserve_metadata_snap_mesg()
3678 r = dm_pool_reserve_metadata_snap(pool->pmd); in process_reserve_metadata_snap_mesg()
3685 static int process_release_metadata_snap_mesg(unsigned argc, char **argv, struct pool *pool) in process_release_metadata_snap_mesg() argument
3693 r = dm_pool_release_metadata_snap(pool->pmd); in process_release_metadata_snap_mesg()
3713 struct pool *pool = pt->pool; in pool_message() local
3715 if (get_pool_mode(pool) >= PM_READ_ONLY) { in pool_message()
3717 dm_device_name(pool->pool_md)); in pool_message()
3722 r = process_create_thin_mesg(argc, argv, pool); in pool_message()
3725 r = process_create_snap_mesg(argc, argv, pool); in pool_message()
3728 r = process_delete_mesg(argc, argv, pool); in pool_message()
3731 r = process_set_transaction_id_mesg(argc, argv, pool); in pool_message()
3734 r = process_reserve_metadata_snap_mesg(argc, argv, pool); in pool_message()
3737 r = process_release_metadata_snap_mesg(argc, argv, pool); in pool_message()
3743 (void) commit(pool); in pool_message()
3792 struct pool *pool = pt->pool; in pool_status() local
3796 if (get_pool_mode(pool) == PM_FAIL) { in pool_status()
3803 (void) commit(pool); in pool_status()
3805 r = dm_pool_get_metadata_transaction_id(pool->pmd, &transaction_id); in pool_status()
3808 dm_device_name(pool->pool_md), r); in pool_status()
3812 r = dm_pool_get_free_metadata_block_count(pool->pmd, &nr_free_blocks_metadata); in pool_status()
3815 dm_device_name(pool->pool_md), r); in pool_status()
3819 r = dm_pool_get_metadata_dev_size(pool->pmd, &nr_blocks_metadata); in pool_status()
3822 dm_device_name(pool->pool_md), r); in pool_status()
3826 r = dm_pool_get_free_block_count(pool->pmd, &nr_free_blocks_data); in pool_status()
3829 dm_device_name(pool->pool_md), r); in pool_status()
3833 r = dm_pool_get_data_dev_size(pool->pmd, &nr_blocks_data); in pool_status()
3836 dm_device_name(pool->pool_md), r); in pool_status()
3840 r = dm_pool_get_metadata_snap(pool->pmd, &held_root); in pool_status()
3843 dm_device_name(pool->pool_md), r); in pool_status()
3859 if (pool->pf.mode == PM_OUT_OF_DATA_SPACE) in pool_status()
3861 else if (pool->pf.mode == PM_READ_ONLY) in pool_status()
3866 if (!pool->pf.discard_enabled) in pool_status()
3868 else if (pool->pf.discard_passdown) in pool_status()
3873 if (pool->pf.error_if_no_space) in pool_status()
3878 if (dm_pool_metadata_needs_check(pool->pmd)) in pool_status()
3889 (unsigned long)pool->sectors_per_block, in pool_status()
3911 struct pool *pool = pt->pool; in pool_io_hints() local
3923 if (limits->max_sectors < pool->sectors_per_block) { in pool_io_hints()
3924 while (!is_factor(pool->sectors_per_block, limits->max_sectors)) { in pool_io_hints()
3935 if (io_opt_sectors < pool->sectors_per_block || in pool_io_hints()
3936 !is_factor(io_opt_sectors, pool->sectors_per_block)) { in pool_io_hints()
3937 if (is_factor(pool->sectors_per_block, limits->max_sectors)) in pool_io_hints()
3940 blk_limits_io_min(limits, pool->sectors_per_block << SECTOR_SHIFT); in pool_io_hints()
3941 blk_limits_io_opt(limits, pool->sectors_per_block << SECTOR_SHIFT); in pool_io_hints()
4007 spin_lock_irqsave(&tc->pool->lock, flags); in thin_dtr()
4009 spin_unlock_irqrestore(&tc->pool->lock, flags); in thin_dtr()
4017 __pool_dec(tc->pool); in thin_dtr()
4097 tc->pool = __pool_table_lookup(pool_md); in thin_ctr()
4098 if (!tc->pool) { in thin_ctr()
4103 __pool_inc(tc->pool); in thin_ctr()
4105 if (get_pool_mode(tc->pool) == PM_FAIL) { in thin_ctr()
4111 r = dm_pool_open_thin_device(tc->pool->pmd, tc->dev_id, &tc->td); in thin_ctr()
4117 r = dm_set_target_max_io_len(ti, tc->pool->sectors_per_block); in thin_ctr()
4127 if (tc->pool->pf.discard_enabled) { in thin_ctr()
4135 spin_lock_irqsave(&tc->pool->lock, flags); in thin_ctr()
4136 if (tc->pool->suspended) { in thin_ctr()
4137 spin_unlock_irqrestore(&tc->pool->lock, flags); in thin_ctr()
4145 list_add_tail_rcu(&tc->list, &tc->pool->active_thins); in thin_ctr()
4146 spin_unlock_irqrestore(&tc->pool->lock, flags); in thin_ctr()
4162 __pool_dec(tc->pool); in thin_ctr()
4191 struct pool *pool = h->tc->pool; in thin_endio() local
4197 spin_lock_irqsave(&pool->lock, flags); in thin_endio()
4202 spin_unlock_irqrestore(&pool->lock, flags); in thin_endio()
4209 spin_lock_irqsave(&pool->lock, flags); in thin_endio()
4211 list_add_tail(&m->list, &pool->prepared_discards); in thin_endio()
4212 spin_unlock_irqrestore(&pool->lock, flags); in thin_endio()
4213 wake_worker(pool); in thin_endio()
4264 if (get_pool_mode(tc->pool) == PM_FAIL) { in thin_status()
4286 DMEMIT("%llu ", mapped * tc->pool->sectors_per_block); in thin_status()
4289 tc->pool->sectors_per_block) - 1); in thin_status()
4315 struct pool *pool = tc->pool; in thin_iterate_devices() local
4321 if (!pool->ti) in thin_iterate_devices()
4324 blocks = pool->ti->len; in thin_iterate_devices()
4325 (void) sector_div(blocks, pool->sectors_per_block); in thin_iterate_devices()
4327 return fn(ti, tc->pool_dev, 0, pool->sectors_per_block * blocks, data); in thin_iterate_devices()
4335 struct pool *pool = tc->pool; in thin_io_hints() local
4337 if (!pool->pf.discard_enabled) in thin_io_hints()
4340 limits->discard_granularity = pool->sectors_per_block << SECTOR_SHIFT; in thin_io_hints()