Lines Matching refs:pblk
52 struct pblk *pblk = bio->bi_disk->queue->queuedata; in pblk_submit_bio() local
55 pblk_discard(pblk, bio); in pblk_submit_bio()
67 pblk_submit_read(pblk, bio); in pblk_submit_bio()
73 if (pblk_get_secs(bio) > pblk_rl_max_io(&pblk->rl)) in pblk_submit_bio()
76 pblk_write_to_cache(pblk, bio, PBLK_IOTYPE_USER); in pblk_submit_bio()
88 static size_t pblk_trans_map_size(struct pblk *pblk) in pblk_trans_map_size() argument
92 if (pblk->addrf_len < 32) in pblk_trans_map_size()
95 return entry_size * pblk->capacity; in pblk_trans_map_size()
99 static u32 pblk_l2p_crc(struct pblk *pblk) in pblk_l2p_crc() argument
104 map_size = pblk_trans_map_size(pblk); in pblk_l2p_crc()
105 crc = crc32_le(crc, pblk->trans_map, map_size); in pblk_l2p_crc()
110 static void pblk_l2p_free(struct pblk *pblk) in pblk_l2p_free() argument
112 vfree(pblk->trans_map); in pblk_l2p_free()
115 static int pblk_l2p_recover(struct pblk *pblk, bool factory_init) in pblk_l2p_recover() argument
120 guid_gen(&pblk->instance_uuid); in pblk_l2p_recover()
122 line = pblk_recov_l2p(pblk); in pblk_l2p_recover()
124 pblk_err(pblk, "could not recover l2p table\n"); in pblk_l2p_recover()
130 pblk_info(pblk, "init: L2P CRC: %x\n", pblk_l2p_crc(pblk)); in pblk_l2p_recover()
134 pblk_gc_free_full_lines(pblk); in pblk_l2p_recover()
138 line = pblk_line_get_first_data(pblk); in pblk_l2p_recover()
146 static int pblk_l2p_init(struct pblk *pblk, bool factory_init) in pblk_l2p_init() argument
153 map_size = pblk_trans_map_size(pblk); in pblk_l2p_init()
154 pblk->trans_map = __vmalloc(map_size, GFP_KERNEL | __GFP_NOWARN | in pblk_l2p_init()
156 if (!pblk->trans_map) { in pblk_l2p_init()
157 pblk_err(pblk, "failed to allocate L2P (need %zu of memory)\n", in pblk_l2p_init()
164 for (i = 0; i < pblk->capacity; i++) in pblk_l2p_init()
165 pblk_trans_map_set(pblk, i, ppa); in pblk_l2p_init()
167 ret = pblk_l2p_recover(pblk, factory_init); in pblk_l2p_init()
169 vfree(pblk->trans_map); in pblk_l2p_init()
174 static void pblk_rwb_free(struct pblk *pblk) in pblk_rwb_free() argument
176 if (pblk_rb_tear_down_check(&pblk->rwb)) in pblk_rwb_free()
177 pblk_err(pblk, "write buffer error on tear down\n"); in pblk_rwb_free()
179 pblk_rb_free(&pblk->rwb); in pblk_rwb_free()
182 static int pblk_rwb_init(struct pblk *pblk) in pblk_rwb_init() argument
184 struct nvm_tgt_dev *dev = pblk->dev; in pblk_rwb_init()
198 return pblk_rb_init(&pblk->rwb, buffer_size, threshold, geo->csecs); in pblk_rwb_init()
201 static int pblk_set_addrf_12(struct pblk *pblk, struct nvm_geo *geo, in pblk_set_addrf_12() argument
210 pblk_err(pblk, "supports only power-of-two channel config.\n"); in pblk_set_addrf_12()
217 pblk_err(pblk, "supports only power-of-two LUN config.\n"); in pblk_set_addrf_12()
274 static int pblk_set_addrf(struct pblk *pblk) in pblk_set_addrf() argument
276 struct nvm_tgt_dev *dev = pblk->dev; in pblk_set_addrf()
282 div_u64_rem(geo->clba, pblk->min_write_pgs, &mod); in pblk_set_addrf()
284 pblk_err(pblk, "bad configuration of sectors/pages\n"); in pblk_set_addrf()
288 pblk->addrf_len = pblk_set_addrf_12(pblk, geo, in pblk_set_addrf()
289 (void *)&pblk->addrf); in pblk_set_addrf()
292 pblk->addrf_len = pblk_set_addrf_20(geo, (void *)&pblk->addrf, in pblk_set_addrf()
293 &pblk->uaddrf); in pblk_set_addrf()
296 pblk_err(pblk, "OCSSD revision not supported (%d)\n", in pblk_set_addrf()
376 static int pblk_core_init(struct pblk *pblk) in pblk_core_init() argument
378 struct nvm_tgt_dev *dev = pblk->dev; in pblk_core_init()
382 atomic64_set(&pblk->user_wa, 0); in pblk_core_init()
383 atomic64_set(&pblk->pad_wa, 0); in pblk_core_init()
384 atomic64_set(&pblk->gc_wa, 0); in pblk_core_init()
385 pblk->user_rst_wa = 0; in pblk_core_init()
386 pblk->pad_rst_wa = 0; in pblk_core_init()
387 pblk->gc_rst_wa = 0; in pblk_core_init()
389 atomic64_set(&pblk->nr_flush, 0); in pblk_core_init()
390 pblk->nr_flush_rst = 0; in pblk_core_init()
392 pblk->min_write_pgs = geo->ws_opt; in pblk_core_init()
393 pblk->min_write_pgs_data = pblk->min_write_pgs; in pblk_core_init()
394 max_write_ppas = pblk->min_write_pgs * geo->all_luns; in pblk_core_init()
395 pblk->max_write_pgs = min_t(int, max_write_ppas, NVM_MAX_VLBA); in pblk_core_init()
396 pblk->max_write_pgs = min_t(int, pblk->max_write_pgs, in pblk_core_init()
398 pblk_set_sec_per_write(pblk, pblk->min_write_pgs); in pblk_core_init()
400 pblk->oob_meta_size = geo->sos; in pblk_core_init()
401 if (!pblk_is_oob_meta_supported(pblk)) { in pblk_core_init()
409 if (pblk->min_write_pgs in pblk_core_init()
419 pblk_err(pblk, "Not supported min write size\n"); in pblk_core_init()
428 pblk->max_write_pgs = pblk->min_write_pgs; in pblk_core_init()
429 pblk->min_write_pgs_data = pblk->min_write_pgs - 1; in pblk_core_init()
432 pblk->pad_dist = kcalloc(pblk->min_write_pgs - 1, sizeof(atomic64_t), in pblk_core_init()
434 if (!pblk->pad_dist) in pblk_core_init()
441 ret = mempool_init_page_pool(&pblk->page_bio_pool, NVM_MAX_VLBA, 0); in pblk_core_init()
445 ret = mempool_init_slab_pool(&pblk->gen_ws_pool, PBLK_GEN_WS_POOL_SIZE, in pblk_core_init()
450 ret = mempool_init_slab_pool(&pblk->rec_pool, geo->all_luns, in pblk_core_init()
455 ret = mempool_init_slab_pool(&pblk->r_rq_pool, geo->all_luns, in pblk_core_init()
460 ret = mempool_init_slab_pool(&pblk->e_rq_pool, geo->all_luns, in pblk_core_init()
465 ret = mempool_init_slab_pool(&pblk->w_rq_pool, geo->all_luns, in pblk_core_init()
470 pblk->close_wq = alloc_workqueue("pblk-close-wq", in pblk_core_init()
472 if (!pblk->close_wq) in pblk_core_init()
475 pblk->bb_wq = alloc_workqueue("pblk-bb-wq", in pblk_core_init()
477 if (!pblk->bb_wq) in pblk_core_init()
480 pblk->r_end_wq = alloc_workqueue("pblk-read-end-wq", in pblk_core_init()
482 if (!pblk->r_end_wq) in pblk_core_init()
485 if (pblk_set_addrf(pblk)) in pblk_core_init()
488 INIT_LIST_HEAD(&pblk->compl_list); in pblk_core_init()
489 INIT_LIST_HEAD(&pblk->resubmit_list); in pblk_core_init()
494 destroy_workqueue(pblk->r_end_wq); in pblk_core_init()
496 destroy_workqueue(pblk->bb_wq); in pblk_core_init()
498 destroy_workqueue(pblk->close_wq); in pblk_core_init()
500 mempool_exit(&pblk->w_rq_pool); in pblk_core_init()
502 mempool_exit(&pblk->e_rq_pool); in pblk_core_init()
504 mempool_exit(&pblk->r_rq_pool); in pblk_core_init()
506 mempool_exit(&pblk->rec_pool); in pblk_core_init()
508 mempool_exit(&pblk->gen_ws_pool); in pblk_core_init()
510 mempool_exit(&pblk->page_bio_pool); in pblk_core_init()
514 kfree(pblk->pad_dist); in pblk_core_init()
518 static void pblk_core_free(struct pblk *pblk) in pblk_core_free() argument
520 if (pblk->close_wq) in pblk_core_free()
521 destroy_workqueue(pblk->close_wq); in pblk_core_free()
523 if (pblk->r_end_wq) in pblk_core_free()
524 destroy_workqueue(pblk->r_end_wq); in pblk_core_free()
526 if (pblk->bb_wq) in pblk_core_free()
527 destroy_workqueue(pblk->bb_wq); in pblk_core_free()
529 mempool_exit(&pblk->page_bio_pool); in pblk_core_free()
530 mempool_exit(&pblk->gen_ws_pool); in pblk_core_free()
531 mempool_exit(&pblk->rec_pool); in pblk_core_free()
532 mempool_exit(&pblk->r_rq_pool); in pblk_core_free()
533 mempool_exit(&pblk->e_rq_pool); in pblk_core_free()
534 mempool_exit(&pblk->w_rq_pool); in pblk_core_free()
537 kfree(pblk->pad_dist); in pblk_core_free()
540 static void pblk_line_mg_free(struct pblk *pblk) in pblk_line_mg_free() argument
542 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_line_mg_free()
572 static void pblk_lines_free(struct pblk *pblk) in pblk_lines_free() argument
574 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_lines_free()
579 line = &pblk->lines[i]; in pblk_lines_free()
585 pblk_line_mg_free(pblk); in pblk_lines_free()
587 kfree(pblk->luns); in pblk_lines_free()
588 kfree(pblk->lines); in pblk_lines_free()
591 static int pblk_luns_init(struct pblk *pblk) in pblk_luns_init() argument
593 struct nvm_tgt_dev *dev = pblk->dev; in pblk_luns_init()
600 pblk_err(pblk, "unbalanced LUN config.\n"); in pblk_luns_init()
604 pblk->luns = kcalloc(geo->all_luns, sizeof(struct pblk_lun), in pblk_luns_init()
606 if (!pblk->luns) in pblk_luns_init()
615 rlun = &pblk->luns[i]; in pblk_luns_init()
625 static unsigned int calc_emeta_len(struct pblk *pblk) in calc_emeta_len() argument
627 struct pblk_line_meta *lm = &pblk->lm; in calc_emeta_len()
628 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in calc_emeta_len()
629 struct nvm_tgt_dev *dev = pblk->dev; in calc_emeta_len()
653 static int pblk_set_provision(struct pblk *pblk, int nr_free_chks) in pblk_set_provision() argument
655 struct nvm_tgt_dev *dev = pblk->dev; in pblk_set_provision()
656 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_set_provision()
657 struct pblk_line_meta *lm = &pblk->lm; in pblk_set_provision()
664 pblk->op = PBLK_DEFAULT_OP; in pblk_set_provision()
666 pblk->op = geo->op; in pblk_set_provision()
668 minimum = pblk_get_min_chks(pblk); in pblk_set_provision()
670 provisioned *= (100 - pblk->op); in pblk_set_provision()
675 pblk_err(pblk, "OP too small to create a sane instance\n"); in pblk_set_provision()
684 pblk->op = (100 * minimum) / nr_free_chks; in pblk_set_provision()
685 pblk_info(pblk, "Default OP insufficient, adjusting OP to %d\n", in pblk_set_provision()
686 pblk->op); in pblk_set_provision()
689 pblk->op_blks = nr_free_chks - provisioned; in pblk_set_provision()
694 pblk->rl.total_blocks = nr_free_chks; in pblk_set_provision()
700 clba = (geo->clba / pblk->min_write_pgs) * pblk->min_write_pgs_data; in pblk_set_provision()
701 pblk->capacity = (provisioned - blk_meta) * clba; in pblk_set_provision()
703 atomic_set(&pblk->rl.free_blocks, nr_free_chks); in pblk_set_provision()
704 atomic_set(&pblk->rl.free_user_blocks, nr_free_chks); in pblk_set_provision()
709 static int pblk_setup_line_meta_chk(struct pblk *pblk, struct pblk_line *line, in pblk_setup_line_meta_chk() argument
712 struct nvm_tgt_dev *dev = pblk->dev; in pblk_setup_line_meta_chk()
714 struct pblk_line_meta *lm = &pblk->lm; in pblk_setup_line_meta_chk()
718 struct pblk_lun *rlun = &pblk->luns[i]; in pblk_setup_line_meta_chk()
729 chunk_meta = pblk_chunk_get_off(pblk, meta, ppa); in pblk_setup_line_meta_chk()
738 trace_pblk_chunk_state(pblk_disk_name(pblk), &ppa, in pblk_setup_line_meta_chk()
756 static long pblk_setup_line_meta(struct pblk *pblk, struct pblk_line *line, in pblk_setup_line_meta() argument
759 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_setup_line_meta()
760 struct pblk_line_meta *lm = &pblk->lm; in pblk_setup_line_meta()
763 line->pblk = pblk; in pblk_setup_line_meta()
771 nr_bad_chks = pblk_setup_line_meta_chk(pblk, line, chunk_meta); in pblk_setup_line_meta()
788 static int pblk_alloc_line_meta(struct pblk *pblk, struct pblk_line *line) in pblk_alloc_line_meta() argument
790 struct pblk_line_meta *lm = &pblk->lm; in pblk_alloc_line_meta()
821 static int pblk_line_mg_init(struct pblk *pblk) in pblk_line_mg_init() argument
823 struct nvm_tgt_dev *dev = pblk->dev; in pblk_line_mg_init()
825 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_line_mg_init()
826 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_mg_init()
938 static int pblk_line_meta_init(struct pblk *pblk) in pblk_line_meta_init() argument
940 struct nvm_tgt_dev *dev = pblk->dev; in pblk_line_meta_init()
942 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_meta_init()
953 lm->meta_distance = (geo->all_luns / 2) * pblk->min_write_pgs; in pblk_line_meta_init()
977 emeta_len = calc_emeta_len(pblk); in pblk_line_meta_init()
991 pblk_err(pblk, "config. not supported. Min. LUN in line:%d\n", in pblk_line_meta_init()
999 static int pblk_lines_init(struct pblk *pblk) in pblk_lines_init() argument
1001 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_lines_init()
1007 ret = pblk_line_meta_init(pblk); in pblk_lines_init()
1011 ret = pblk_line_mg_init(pblk); in pblk_lines_init()
1015 ret = pblk_luns_init(pblk); in pblk_lines_init()
1019 chunk_meta = pblk_get_chunk_meta(pblk); in pblk_lines_init()
1025 pblk->lines = kcalloc(l_mg->nr_lines, sizeof(struct pblk_line), in pblk_lines_init()
1027 if (!pblk->lines) { in pblk_lines_init()
1033 line = &pblk->lines[i]; in pblk_lines_init()
1035 ret = pblk_alloc_line_meta(pblk, line); in pblk_lines_init()
1039 nr_free_chks += pblk_setup_line_meta(pblk, line, chunk_meta, i); in pblk_lines_init()
1041 trace_pblk_line_state(pblk_disk_name(pblk), line->id, in pblk_lines_init()
1046 pblk_err(pblk, "too many bad blocks prevent for sane instance\n"); in pblk_lines_init()
1051 ret = pblk_set_provision(pblk, nr_free_chks); in pblk_lines_init()
1060 pblk_line_meta_free(l_mg, &pblk->lines[i]); in pblk_lines_init()
1061 kfree(pblk->lines); in pblk_lines_init()
1065 kfree(pblk->luns); in pblk_lines_init()
1067 pblk_line_mg_free(pblk); in pblk_lines_init()
1072 static int pblk_writer_init(struct pblk *pblk) in pblk_writer_init() argument
1074 pblk->writer_ts = kthread_create(pblk_write_ts, pblk, "pblk-writer-t"); in pblk_writer_init()
1075 if (IS_ERR(pblk->writer_ts)) { in pblk_writer_init()
1076 int err = PTR_ERR(pblk->writer_ts); in pblk_writer_init()
1079 pblk_err(pblk, "could not allocate writer kthread (%d)\n", in pblk_writer_init()
1084 timer_setup(&pblk->wtimer, pblk_write_timer_fn, 0); in pblk_writer_init()
1085 mod_timer(&pblk->wtimer, jiffies + msecs_to_jiffies(100)); in pblk_writer_init()
1090 static void pblk_writer_stop(struct pblk *pblk) in pblk_writer_stop() argument
1095 WARN(pblk_rb_read_count(&pblk->rwb), in pblk_writer_stop()
1098 WARN(pblk_rb_sync_count(&pblk->rwb), in pblk_writer_stop()
1101 del_timer_sync(&pblk->wtimer); in pblk_writer_stop()
1102 if (pblk->writer_ts) in pblk_writer_stop()
1103 kthread_stop(pblk->writer_ts); in pblk_writer_stop()
1106 static void pblk_free(struct pblk *pblk) in pblk_free() argument
1108 pblk_lines_free(pblk); in pblk_free()
1109 pblk_l2p_free(pblk); in pblk_free()
1110 pblk_rwb_free(pblk); in pblk_free()
1111 pblk_core_free(pblk); in pblk_free()
1113 kfree(pblk); in pblk_free()
1116 static void pblk_tear_down(struct pblk *pblk, bool graceful) in pblk_tear_down() argument
1119 __pblk_pipeline_flush(pblk); in pblk_tear_down()
1120 __pblk_pipeline_stop(pblk); in pblk_tear_down()
1121 pblk_writer_stop(pblk); in pblk_tear_down()
1122 pblk_rb_sync_l2p(&pblk->rwb); in pblk_tear_down()
1123 pblk_rl_free(&pblk->rl); in pblk_tear_down()
1125 pblk_debug(pblk, "consistent tear down (graceful:%d)\n", graceful); in pblk_tear_down()
1130 struct pblk *pblk = private; in pblk_exit() local
1132 pblk_gc_exit(pblk, graceful); in pblk_exit()
1133 pblk_tear_down(pblk, graceful); in pblk_exit()
1136 pblk_info(pblk, "exit: L2P CRC: %x\n", pblk_l2p_crc(pblk)); in pblk_exit()
1139 pblk_free(pblk); in pblk_exit()
1144 struct pblk *pblk = private; in pblk_capacity() local
1146 return pblk->capacity * NR_PHY_IN_LOG; in pblk_capacity()
1155 struct pblk *pblk; in pblk_init() local
1158 pblk = kzalloc(sizeof(struct pblk), GFP_KERNEL); in pblk_init()
1159 if (!pblk) in pblk_init()
1162 pblk->dev = dev; in pblk_init()
1163 pblk->disk = tdisk; in pblk_init()
1164 pblk->state = PBLK_STATE_RUNNING; in pblk_init()
1165 trace_pblk_state(pblk_disk_name(pblk), pblk->state); in pblk_init()
1166 pblk->gc.gc_enabled = 0; in pblk_init()
1170 pblk_err(pblk, "OCSSD version not supported (%u)\n", in pblk_init()
1172 kfree(pblk); in pblk_init()
1177 pblk_err(pblk, "extended metadata not supported\n"); in pblk_init()
1178 kfree(pblk); in pblk_init()
1182 spin_lock_init(&pblk->resubmit_lock); in pblk_init()
1183 spin_lock_init(&pblk->trans_lock); in pblk_init()
1184 spin_lock_init(&pblk->lock); in pblk_init()
1187 atomic_long_set(&pblk->inflight_writes, 0); in pblk_init()
1188 atomic_long_set(&pblk->padded_writes, 0); in pblk_init()
1189 atomic_long_set(&pblk->padded_wb, 0); in pblk_init()
1190 atomic_long_set(&pblk->req_writes, 0); in pblk_init()
1191 atomic_long_set(&pblk->sub_writes, 0); in pblk_init()
1192 atomic_long_set(&pblk->sync_writes, 0); in pblk_init()
1193 atomic_long_set(&pblk->inflight_reads, 0); in pblk_init()
1194 atomic_long_set(&pblk->cache_reads, 0); in pblk_init()
1195 atomic_long_set(&pblk->sync_reads, 0); in pblk_init()
1196 atomic_long_set(&pblk->recov_writes, 0); in pblk_init()
1197 atomic_long_set(&pblk->recov_writes, 0); in pblk_init()
1198 atomic_long_set(&pblk->recov_gc_writes, 0); in pblk_init()
1199 atomic_long_set(&pblk->recov_gc_reads, 0); in pblk_init()
1202 atomic_long_set(&pblk->read_failed, 0); in pblk_init()
1203 atomic_long_set(&pblk->read_empty, 0); in pblk_init()
1204 atomic_long_set(&pblk->read_high_ecc, 0); in pblk_init()
1205 atomic_long_set(&pblk->read_failed_gc, 0); in pblk_init()
1206 atomic_long_set(&pblk->write_failed, 0); in pblk_init()
1207 atomic_long_set(&pblk->erase_failed, 0); in pblk_init()
1209 ret = pblk_core_init(pblk); in pblk_init()
1211 pblk_err(pblk, "could not initialize core\n"); in pblk_init()
1215 ret = pblk_lines_init(pblk); in pblk_init()
1217 pblk_err(pblk, "could not initialize lines\n"); in pblk_init()
1221 ret = pblk_rwb_init(pblk); in pblk_init()
1223 pblk_err(pblk, "could not initialize write buffer\n"); in pblk_init()
1227 ret = pblk_l2p_init(pblk, flags & NVM_TARGET_FACTORY); in pblk_init()
1229 pblk_err(pblk, "could not initialize maps\n"); in pblk_init()
1233 ret = pblk_writer_init(pblk); in pblk_init()
1236 pblk_err(pblk, "could not initialize write thread\n"); in pblk_init()
1240 ret = pblk_gc_init(pblk); in pblk_init()
1242 pblk_err(pblk, "could not initialize gc\n"); in pblk_init()
1257 pblk_info(pblk, "luns:%u, lines:%d, secs:%llu, buf entries:%u\n", in pblk_init()
1258 geo->all_luns, pblk->l_mg.nr_lines, in pblk_init()
1259 (unsigned long long)pblk->capacity, in pblk_init()
1260 pblk->rwb.nr_entries); in pblk_init()
1262 wake_up_process(pblk->writer_ts); in pblk_init()
1265 pblk_gc_should_kick(pblk); in pblk_init()
1267 return pblk; in pblk_init()
1270 pblk_writer_stop(pblk); in pblk_init()
1272 pblk_l2p_free(pblk); in pblk_init()
1274 pblk_rwb_free(pblk); in pblk_init()
1276 pblk_lines_free(pblk); in pblk_init()
1278 pblk_core_free(pblk); in pblk_init()
1280 kfree(pblk); in pblk_init()