Lines Matching refs:pblk
24 int pblk_recov_check_emeta(struct pblk *pblk, struct line_emeta *emeta_buf) in pblk_recov_check_emeta() argument
28 crc = pblk_calc_emeta_crc(pblk, emeta_buf); in pblk_recov_check_emeta()
38 static int pblk_recov_l2p_from_emeta(struct pblk *pblk, struct pblk_line *line) in pblk_recov_l2p_from_emeta() argument
40 struct nvm_tgt_dev *dev = pblk->dev; in pblk_recov_l2p_from_emeta()
42 struct pblk_line_meta *lm = &pblk->lm; in pblk_recov_l2p_from_emeta()
50 lba_list = emeta_to_lbas(pblk, emeta_buf); in pblk_recov_l2p_from_emeta()
54 data_start = pblk_line_smeta_start(pblk, line) + lm->smeta_sec; in pblk_recov_l2p_from_emeta()
62 ppa = addr_to_gen_ppa(pblk, i, line->id); in pblk_recov_l2p_from_emeta()
80 pblk_update_map(pblk, le64_to_cpu(lba_list[i]), ppa); in pblk_recov_l2p_from_emeta()
85 pblk_err(pblk, "line %d - inconsistent lba list(%llu/%llu)\n", in pblk_recov_l2p_from_emeta()
93 static void pblk_update_line_wp(struct pblk *pblk, struct pblk_line *line, in pblk_update_line_wp() argument
96 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_update_line_wp()
99 for (i = 0; i < written_secs; i += pblk->min_write_pgs) in pblk_update_line_wp()
100 __pblk_alloc_page(pblk, line, pblk->min_write_pgs); in pblk_update_line_wp()
116 static u64 pblk_sec_in_open_line(struct pblk *pblk, struct pblk_line *line) in pblk_sec_in_open_line() argument
118 struct pblk_line_meta *lm = &pblk->lm; in pblk_sec_in_open_line()
135 pblk_err(pblk, "recovery line %d is bad\n", line->id); in pblk_sec_in_open_line()
137 pblk_update_line_wp(pblk, line, written_secs - lm->smeta_sec); in pblk_sec_in_open_line()
162 struct pblk *pblk = pad_rq->pblk; in pblk_end_io_recov() local
164 pblk_up_chunk(pblk, ppa_list[0]); in pblk_end_io_recov()
166 pblk_free_rqd(pblk, rqd, PBLK_WRITE_INT); in pblk_end_io_recov()
168 atomic_dec(&pblk->inflight_io); in pblk_end_io_recov()
173 static int pblk_recov_pad_line(struct pblk *pblk, struct pblk_line *line, in pblk_recov_pad_line() argument
176 struct nvm_tgt_dev *dev = pblk->dev; in pblk_recov_pad_line()
183 __le64 *lba_list = emeta_to_lbas(pblk, line->emeta->buf); in pblk_recov_pad_line()
197 data = vzalloc(array_size(pblk->max_write_pgs, geo->csecs)); in pblk_recov_pad_line()
203 pad_rq->pblk = pblk; in pblk_recov_pad_line()
208 rq_ppas = pblk_calc_secs(pblk, left_ppas, 0, false); in pblk_recov_pad_line()
209 if (rq_ppas < pblk->min_write_pgs) { in pblk_recov_pad_line()
210 pblk_err(pblk, "corrupted pad line %d\n", line->id); in pblk_recov_pad_line()
214 rqd = pblk_alloc_rqd(pblk, PBLK_WRITE_INT); in pblk_recov_pad_line()
216 ret = pblk_alloc_rqd_meta(pblk, rqd); in pblk_recov_pad_line()
218 pblk_free_rqd(pblk, rqd, PBLK_WRITE_INT); in pblk_recov_pad_line()
236 w_ptr = pblk_alloc_page(pblk, line, pblk->min_write_pgs); in pblk_recov_pad_line()
237 ppa = addr_to_gen_ppa(pblk, w_ptr, line->id); in pblk_recov_pad_line()
241 w_ptr += pblk->min_write_pgs; in pblk_recov_pad_line()
242 ppa = addr_to_gen_ppa(pblk, w_ptr, line->id); in pblk_recov_pad_line()
246 for (j = 0; j < pblk->min_write_pgs; j++, i++, w_ptr++) { in pblk_recov_pad_line()
251 dev_ppa = addr_to_gen_ppa(pblk, w_ptr, line->id); in pblk_recov_pad_line()
253 pblk_map_invalidate(pblk, dev_ppa); in pblk_recov_pad_line()
255 meta = pblk_get_meta(pblk, meta_list, i); in pblk_recov_pad_line()
262 pblk_down_chunk(pblk, ppa_list[0]); in pblk_recov_pad_line()
264 ret = pblk_submit_io(pblk, rqd, data); in pblk_recov_pad_line()
266 pblk_err(pblk, "I/O submission failed: %d\n", ret); in pblk_recov_pad_line()
267 pblk_up_chunk(pblk, ppa_list[0]); in pblk_recov_pad_line()
269 pblk_free_rqd(pblk, rqd, PBLK_WRITE_INT); in pblk_recov_pad_line()
283 pblk_err(pblk, "corrupted padded line: %d\n", line->id); in pblk_recov_pad_line()
291 static int pblk_pad_distance(struct pblk *pblk, struct pblk_line *line) in pblk_pad_distance() argument
293 struct nvm_tgt_dev *dev = pblk->dev; in pblk_pad_distance()
301 static struct nvm_chk_meta *pblk_get_stripe_chunk(struct pblk *pblk, in pblk_get_stripe_chunk() argument
305 struct nvm_tgt_dev *dev = pblk->dev; in pblk_get_stripe_chunk()
311 rlun = &pblk->luns[index]; in pblk_get_stripe_chunk()
318 static int pblk_line_wps_are_unbalanced(struct pblk *pblk, in pblk_line_wps_are_unbalanced() argument
321 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_wps_are_unbalanced()
335 chunk = pblk_get_stripe_chunk(pblk, line, i); in pblk_line_wps_are_unbalanced()
337 if (max_wp > pblk->max_write_pgs) in pblk_line_wps_are_unbalanced()
338 min_wp = max_wp - pblk->max_write_pgs; in pblk_line_wps_are_unbalanced()
344 chunk = pblk_get_stripe_chunk(pblk, line, i); in pblk_line_wps_are_unbalanced()
354 static int pblk_recov_scan_oob(struct pblk *pblk, struct pblk_line *line, in pblk_recov_scan_oob() argument
357 struct nvm_tgt_dev *dev = pblk->dev; in pblk_recov_scan_oob()
358 struct pblk_line_meta *lm = &pblk->lm; in pblk_recov_scan_oob()
366 u64 paddr = pblk_line_smeta_start(pblk, line) + lm->smeta_sec; in pblk_recov_scan_oob()
371 u64 left_ppas = pblk_sec_in_open_line(pblk, line) - lm->smeta_sec; in pblk_recov_scan_oob()
373 if (pblk_line_wps_are_unbalanced(pblk, line)) in pblk_recov_scan_oob()
374 pblk_warn(pblk, "recovering unbalanced line (%d)\n", line->id); in pblk_recov_scan_oob()
383 lba_list = emeta_to_lbas(pblk, line->emeta->buf); in pblk_recov_scan_oob()
388 rq_ppas = pblk_calc_secs(pblk, left_ppas, 0, false); in pblk_recov_scan_oob()
390 rq_ppas = pblk->min_write_pgs; in pblk_recov_scan_oob()
402 if (pblk_io_aligned(pblk, rq_ppas)) in pblk_recov_scan_oob()
409 ppa = addr_to_gen_ppa(pblk, paddr, line->id); in pblk_recov_scan_oob()
413 paddr += pblk->min_write_pgs; in pblk_recov_scan_oob()
414 ppa = addr_to_gen_ppa(pblk, paddr, line->id); in pblk_recov_scan_oob()
418 for (j = 0; j < pblk->min_write_pgs; j++, i++) in pblk_recov_scan_oob()
420 addr_to_gen_ppa(pblk, paddr + j, line->id); in pblk_recov_scan_oob()
423 ret = pblk_submit_io_sync(pblk, rqd, data); in pblk_recov_scan_oob()
425 pblk_err(pblk, "I/O submission failed: %d\n", ret); in pblk_recov_scan_oob()
429 atomic_dec(&pblk->inflight_io); in pblk_recov_scan_oob()
436 pblk_log_read_err(pblk, rqd); in pblk_recov_scan_oob()
440 pad_distance = pblk_pad_distance(pblk, line); in pblk_recov_scan_oob()
441 ret = pblk_recov_pad_line(pblk, line, pad_distance); in pblk_recov_scan_oob()
450 pblk_get_packed_meta(pblk, rqd); in pblk_recov_scan_oob()
453 struct pblk_sec_meta *meta = pblk_get_meta(pblk, meta_list, i); in pblk_recov_scan_oob()
458 if (lba == ADDR_EMPTY || lba >= pblk->capacity) in pblk_recov_scan_oob()
462 pblk_update_map(pblk, lba, ppa_list[i]); in pblk_recov_scan_oob()
477 static int pblk_recov_l2p_from_oob(struct pblk *pblk, struct pblk_line *line) in pblk_recov_l2p_from_oob() argument
479 struct nvm_tgt_dev *dev = pblk->dev; in pblk_recov_l2p_from_oob()
493 ppa_list = (void *)(meta_list) + pblk_dma_meta_size(pblk); in pblk_recov_l2p_from_oob()
494 dma_ppa_list = dma_meta_list + pblk_dma_meta_size(pblk); in pblk_recov_l2p_from_oob()
496 data = kcalloc(pblk->max_write_pgs, geo->csecs, GFP_KERNEL); in pblk_recov_l2p_from_oob()
502 rqd = mempool_alloc(&pblk->r_rq_pool, GFP_KERNEL); in pblk_recov_l2p_from_oob()
512 ret = pblk_recov_scan_oob(pblk, line, p); in pblk_recov_l2p_from_oob()
514 pblk_err(pblk, "could not recover L2P form OOB\n"); in pblk_recov_l2p_from_oob()
519 pblk_line_recov_close(pblk, line); in pblk_recov_l2p_from_oob()
522 mempool_free(rqd, &pblk->r_rq_pool); in pblk_recov_l2p_from_oob()
543 static u64 pblk_line_emeta_start(struct pblk *pblk, struct pblk_line *line) in pblk_line_emeta_start() argument
545 struct nvm_tgt_dev *dev = pblk->dev; in pblk_line_emeta_start()
547 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_emeta_start()
558 ppa = addr_to_gen_ppa(pblk, emeta_start, line->id); in pblk_line_emeta_start()
567 static int pblk_recov_check_line_version(struct pblk *pblk, in pblk_recov_check_line_version() argument
573 pblk_err(pblk, "line major version mismatch: %d, expected: %d\n", in pblk_recov_check_line_version()
580 pblk_info(pblk, "newer line minor version found: %d\n", in pblk_recov_check_line_version()
587 static void pblk_recov_wa_counters(struct pblk *pblk, in pblk_recov_wa_counters() argument
590 struct pblk_line_meta *lm = &pblk->lm; in pblk_recov_wa_counters()
600 atomic64_set(&pblk->user_wa, user); in pblk_recov_wa_counters()
601 atomic64_set(&pblk->pad_wa, pad); in pblk_recov_wa_counters()
602 atomic64_set(&pblk->gc_wa, gc); in pblk_recov_wa_counters()
604 pblk->user_rst_wa = user; in pblk_recov_wa_counters()
605 pblk->pad_rst_wa = pad; in pblk_recov_wa_counters()
606 pblk->gc_rst_wa = gc; in pblk_recov_wa_counters()
611 struct pblk *pblk) in pblk_line_was_written() argument
614 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_was_written()
615 struct nvm_tgt_dev *dev = pblk->dev; in pblk_line_was_written()
628 bppa = pblk->luns[smeta_blk].bppa; in pblk_line_was_written()
639 static bool pblk_line_is_open(struct pblk *pblk, struct pblk_line *line) in pblk_line_is_open() argument
641 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_is_open()
651 struct pblk_line *pblk_recov_l2p(struct pblk *pblk) in pblk_recov_l2p() argument
653 struct pblk_line_meta *lm = &pblk->lm; in pblk_recov_l2p()
654 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_recov_l2p()
680 line = &pblk->lines[i]; in pblk_recov_l2p()
687 if (!pblk_line_was_written(line, pblk)) in pblk_recov_l2p()
691 if (pblk_line_smeta_read(pblk, line)) in pblk_recov_l2p()
694 crc = pblk_calc_smeta_crc(pblk, smeta_buf); in pblk_recov_l2p()
702 pblk_err(pblk, "found incompatible line version %u\n", in pblk_recov_l2p()
709 guid_copy(&pblk->instance_uuid, in pblk_recov_l2p()
714 if (!guid_equal(&pblk->instance_uuid, in pblk_recov_l2p()
716 pblk_debug(pblk, "ignore line %u due to uuid mismatch\n", in pblk_recov_l2p()
735 if (pblk_line_recov_alloc(pblk, line)) in pblk_recov_l2p()
740 pblk_debug(pblk, "recovering data line %d, seq:%llu\n", in pblk_recov_l2p()
745 guid_gen(&pblk->instance_uuid); in pblk_recov_l2p()
759 line->emeta_ssec = pblk_line_emeta_start(pblk, line); in pblk_recov_l2p()
763 if (pblk_line_is_open(pblk, line)) { in pblk_recov_l2p()
764 pblk_recov_l2p_from_oob(pblk, line); in pblk_recov_l2p()
768 if (pblk_line_emeta_read(pblk, line, line->emeta->buf)) { in pblk_recov_l2p()
769 pblk_recov_l2p_from_oob(pblk, line); in pblk_recov_l2p()
773 if (pblk_recov_check_emeta(pblk, line->emeta->buf)) { in pblk_recov_l2p()
774 pblk_recov_l2p_from_oob(pblk, line); in pblk_recov_l2p()
778 if (pblk_recov_check_line_version(pblk, line->emeta->buf)) in pblk_recov_l2p()
781 pblk_recov_wa_counters(pblk, line->emeta->buf); in pblk_recov_l2p()
783 if (pblk_recov_l2p_from_emeta(pblk, line)) in pblk_recov_l2p()
784 pblk_recov_l2p_from_oob(pblk, line); in pblk_recov_l2p()
792 trace_pblk_line_state(pblk_disk_name(pblk), line->id, in pblk_recov_l2p()
794 move_list = pblk_line_gc_list(pblk, line); in pblk_recov_l2p()
813 trace_pblk_line_state(pblk_disk_name(pblk), line->id, in pblk_recov_l2p()
832 l_mg->data_next = pblk_line_get(pblk); in pblk_recov_l2p()
842 pblk_line_erase(pblk, l_mg->data_next); in pblk_recov_l2p()
846 pblk_err(pblk, "failed to recover all found lines %d/%d\n", in pblk_recov_l2p()
855 int pblk_recov_pad(struct pblk *pblk) in pblk_recov_pad() argument
858 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_recov_pad()
867 ret = pblk_recov_pad_line(pblk, line, left_msecs); in pblk_recov_pad()
869 pblk_err(pblk, "tear down padding failed (%d)\n", ret); in pblk_recov_pad()
873 pblk_line_close_meta(pblk, line); in pblk_recov_pad()