28 static int pblk_read_from_cache(struct pblk *pblk, struct bio *bio,  in pblk_read_from_cache()  argument
37 	return pblk_rb_copy_to_bio(&pblk->rwb, bio, lba, ppa);  in pblk_read_from_cache()
40 static int pblk_read_ppalist_rq(struct pblk *pblk, struct nvm_rq *rqd,  in pblk_read_ppalist_rq()  argument
48 	nr_secs = pblk_lookup_l2p_seq(pblk, rqd->ppa_list, blba, rqd->nr_ppas,  in pblk_read_ppalist_rq()
55 		struct pblk_sec_meta *meta = pblk_get_meta(pblk, meta_list, i);  in pblk_read_ppalist_rq()
68 			if (!pblk_read_from_cache(pblk, bio, lba,  in pblk_read_ppalist_rq()
91 			atomic_long_inc(&pblk->cache_reads);  in pblk_read_ppalist_rq()
98 	if (pblk_io_aligned(pblk, nr_secs))  in pblk_read_ppalist_rq()
102 	atomic_long_add(nr_secs, &pblk->inflight_reads);  in pblk_read_ppalist_rq()
109 static void pblk_read_check_seq(struct pblk *pblk, struct nvm_rq *rqd,  in pblk_read_check_seq()  argument
116 	if (!pblk_is_oob_meta_supported(pblk))  in pblk_read_check_seq()
120 		struct pblk_sec_meta *meta = pblk_get_meta(pblk, meta_list, i);  in pblk_read_check_seq()
130 			print_ppa(pblk, &ppa_list[i], "seq", i);  in pblk_read_check_seq()
132 			pblk_err(pblk, "corrupted read LBA (%llu/%llu)\n",  in pblk_read_check_seq()
142 static void pblk_read_check_rand(struct pblk *pblk, struct nvm_rq *rqd,  in pblk_read_check_rand()  argument
148 	if (!pblk_is_oob_meta_supported(pblk))  in pblk_read_check_rand()
152 		struct pblk_sec_meta *meta = pblk_get_meta(pblk,  in pblk_read_check_rand()
166 			print_ppa(pblk, &ppa_list[j], "rnd", j);  in pblk_read_check_rand()
168 			pblk_err(pblk, "corrupted read LBA (%llu/%llu)\n",  in pblk_read_check_rand()
187 static void __pblk_end_io_read(struct pblk *pblk, struct nvm_rq *rqd,  in __pblk_end_io_read()  argument
197 		pblk_log_read_err(pblk, rqd);  in __pblk_end_io_read()
199 	pblk_read_check_seq(pblk, rqd, r_ctx->lba);  in __pblk_end_io_read()
203 		pblk_rq_to_line_put(pblk, rqd);  in __pblk_end_io_read()
206 	atomic_long_add(rqd->nr_ppas, &pblk->sync_reads);  in __pblk_end_io_read()
207 	atomic_long_sub(rqd->nr_ppas, &pblk->inflight_reads);  in __pblk_end_io_read()
210 	pblk_free_rqd(pblk, rqd, PBLK_READ);  in __pblk_end_io_read()
211 	atomic_dec(&pblk->inflight_io);  in __pblk_end_io_read()
216 	struct pblk *pblk = rqd->private;  in pblk_end_io_read()  local
221 	__pblk_end_io_read(pblk, rqd, true);  in pblk_end_io_read()
224 static void pblk_read_rq(struct pblk *pblk, struct nvm_rq *rqd, struct bio *bio,  in pblk_read_rq()  argument
227 	struct pblk_sec_meta *meta = pblk_get_meta(pblk, rqd->meta_list, 0);  in pblk_read_rq()
230 	pblk_lookup_l2p_seq(pblk, &ppa, lba, 1, from_cache);  in pblk_read_rq()
233 	atomic_long_inc(&pblk->inflight_reads);  in pblk_read_rq()
248 		if (!pblk_read_from_cache(pblk, bio, lba, ppa)) {  in pblk_read_rq()
249 			pblk_lookup_l2p_seq(pblk, &ppa, lba, 1, from_cache);  in pblk_read_rq()
256 		atomic_long_inc(&pblk->cache_reads);  in pblk_read_rq()
263 void pblk_submit_read(struct pblk *pblk, struct bio *bio)  in pblk_submit_read()  argument
275 	rqd = pblk_alloc_rqd(pblk, PBLK_READ);  in pblk_submit_read()
279 	rqd->private = pblk;  in pblk_submit_read()
286 	if (pblk_alloc_rqd_meta(pblk, rqd)) {  in pblk_submit_read()
288 		pblk_free_rqd(pblk, rqd, PBLK_READ);  in pblk_submit_read()
299 		nr_secs = pblk_read_ppalist_rq(pblk, rqd, int_bio, blba,  in pblk_submit_read()
302 		pblk_read_rq(pblk, rqd, int_bio, blba, &from_cache);  in pblk_submit_read()
311 		atomic_inc(&pblk->inflight_io);  in pblk_submit_read()
312 		__pblk_end_io_read(pblk, rqd, false);  in pblk_submit_read()
342 	} else if (pblk_submit_io(pblk, rqd, NULL)) {  in pblk_submit_read()
349 static int read_ppalist_rq_gc(struct pblk *pblk, struct nvm_rq *rqd,  in read_ppalist_rq_gc()  argument
358 	pblk_lookup_l2p_rand(pblk, ppa_list_l2p, lba_list, nr_secs);  in read_ppalist_rq_gc()
364 		ppa_gc = addr_to_gen_ppa(pblk, paddr_list_gc[i], line->id);  in read_ppalist_rq_gc()
374 	atomic_long_add(valid_secs, &pblk->inflight_reads);  in read_ppalist_rq_gc()
380 static int read_rq_gc(struct pblk *pblk, struct nvm_rq *rqd,  in read_rq_gc()  argument
391 	if (lba >= pblk->capacity) {  in read_rq_gc()
396 	spin_lock(&pblk->trans_lock);  in read_rq_gc()
397 	ppa_l2p = pblk_trans_map_get(pblk, lba);  in read_rq_gc()
398 	spin_unlock(&pblk->trans_lock);  in read_rq_gc()
400 	ppa_gc = addr_to_gen_ppa(pblk, paddr_gc, line->id);  in read_rq_gc()
408 	atomic_long_inc(&pblk->inflight_reads);  in read_rq_gc()
415 int pblk_submit_read_gc(struct pblk *pblk, struct pblk_gc_rq *gc_rq)  in pblk_submit_read_gc()  argument
422 	ret = pblk_alloc_rqd_meta(pblk, &rqd);  in pblk_submit_read_gc()
427 		gc_rq->secs_to_gc = read_ppalist_rq_gc(pblk, &rqd, gc_rq->line,  in pblk_submit_read_gc()
434 		gc_rq->secs_to_gc = read_rq_gc(pblk, &rqd, gc_rq->line,  in pblk_submit_read_gc()
445 	if (pblk_submit_io_sync(pblk, &rqd, gc_rq->data)) {  in pblk_submit_read_gc()
450 	pblk_read_check_rand(pblk, &rqd, gc_rq->lba_list, gc_rq->nr_secs);  in pblk_submit_read_gc()
452 	atomic_dec(&pblk->inflight_io);  in pblk_submit_read_gc()
455 		atomic_long_inc(&pblk->read_failed_gc);  in pblk_submit_read_gc()
457 		pblk_print_failed_rqd(pblk, &rqd, rqd.error);  in pblk_submit_read_gc()
462 	atomic_long_add(gc_rq->secs_to_gc, &pblk->sync_reads);  in pblk_submit_read_gc()
463 	atomic_long_add(gc_rq->secs_to_gc, &pblk->recov_gc_reads);  in pblk_submit_read_gc()
464 	atomic_long_sub(gc_rq->secs_to_gc, &pblk->inflight_reads);  in pblk_submit_read_gc()
468 	pblk_free_rqd_meta(pblk, &rqd);  in pblk_submit_read_gc()
472 	pblk_free_rqd_meta(pblk, &rqd);  in pblk_submit_read_gc()