28 static int pblk_read_from_cache(struct pblk *pblk, struct bio *bio,  in pblk_read_from_cache()  argument
37 	return pblk_rb_copy_to_bio(&pblk->rwb, bio, lba, ppa);  in pblk_read_from_cache()
40 static int pblk_read_ppalist_rq(struct pblk *pblk, struct nvm_rq *rqd,  in pblk_read_ppalist_rq()  argument
48 	nr_secs = pblk_lookup_l2p_seq(pblk, rqd->ppa_list, blba, rqd->nr_ppas,  in pblk_read_ppalist_rq()
55 		struct pblk_sec_meta *meta = pblk_get_meta(pblk, meta_list, i);  in pblk_read_ppalist_rq()
68 			if (!pblk_read_from_cache(pblk, bio, lba,  in pblk_read_ppalist_rq()
91 			atomic_long_inc(&pblk->cache_reads);  in pblk_read_ppalist_rq()
98 	if (pblk_io_aligned(pblk, nr_secs))  in pblk_read_ppalist_rq()
102 	atomic_long_add(nr_secs, &pblk->inflight_reads);  in pblk_read_ppalist_rq()
109 static void pblk_read_check_seq(struct pblk *pblk, struct nvm_rq *rqd,  in pblk_read_check_seq()  argument
116 	if (!pblk_is_oob_meta_supported(pblk))  in pblk_read_check_seq()
120 		struct pblk_sec_meta *meta = pblk_get_meta(pblk, meta_list, i);  in pblk_read_check_seq()
130 			print_ppa(pblk, &ppa_list[i], "seq", i);  in pblk_read_check_seq()
132 			pblk_err(pblk, "corrupted read LBA (%llu/%llu)\n",  in pblk_read_check_seq()
142 static void pblk_read_check_rand(struct pblk *pblk, struct nvm_rq *rqd,  in pblk_read_check_rand()  argument
148 	if (!pblk_is_oob_meta_supported(pblk))  in pblk_read_check_rand()
152 		struct pblk_sec_meta *meta = pblk_get_meta(pblk,  in pblk_read_check_rand()
166 			print_ppa(pblk, &ppa_list[j], "rnd", j);  in pblk_read_check_rand()
168 			pblk_err(pblk, "corrupted read LBA (%llu/%llu)\n",  in pblk_read_check_rand()
187 static void __pblk_end_io_read(struct pblk *pblk, struct nvm_rq *rqd,  in __pblk_end_io_read()  argument
190 	struct nvm_tgt_dev *dev = pblk->dev;  in __pblk_end_io_read()
195 	generic_end_io_acct(dev->q, REQ_OP_READ, &pblk->disk->part0, start_time);  in __pblk_end_io_read()
198 		pblk_log_read_err(pblk, rqd);  in __pblk_end_io_read()
200 	pblk_read_check_seq(pblk, rqd, r_ctx->lba);  in __pblk_end_io_read()
204 		pblk_rq_to_line_put(pblk, rqd);  in __pblk_end_io_read()
207 	atomic_long_add(rqd->nr_ppas, &pblk->sync_reads);  in __pblk_end_io_read()
208 	atomic_long_sub(rqd->nr_ppas, &pblk->inflight_reads);  in __pblk_end_io_read()
211 	pblk_free_rqd(pblk, rqd, PBLK_READ);  in __pblk_end_io_read()
212 	atomic_dec(&pblk->inflight_io);  in __pblk_end_io_read()
217 	struct pblk *pblk = rqd->private;  in pblk_end_io_read()  local
222 	__pblk_end_io_read(pblk, rqd, true);  in pblk_end_io_read()
225 static void pblk_read_rq(struct pblk *pblk, struct nvm_rq *rqd, struct bio *bio,  in pblk_read_rq()  argument
228 	struct pblk_sec_meta *meta = pblk_get_meta(pblk, rqd->meta_list, 0);  in pblk_read_rq()
231 	pblk_lookup_l2p_seq(pblk, &ppa, lba, 1, from_cache);  in pblk_read_rq()
234 	atomic_long_inc(&pblk->inflight_reads);  in pblk_read_rq()
249 		if (!pblk_read_from_cache(pblk, bio, lba, ppa)) {  in pblk_read_rq()
250 			pblk_lookup_l2p_seq(pblk, &ppa, lba, 1, from_cache);  in pblk_read_rq()
257 		atomic_long_inc(&pblk->cache_reads);  in pblk_read_rq()
264 void pblk_submit_read(struct pblk *pblk, struct bio *bio)  in pblk_submit_read()  argument
266 	struct nvm_tgt_dev *dev = pblk->dev;  in pblk_submit_read()
276 			      &pblk->disk->part0);  in pblk_submit_read()
278 	rqd = pblk_alloc_rqd(pblk, PBLK_READ);  in pblk_submit_read()
282 	rqd->private = pblk;  in pblk_submit_read()
289 	if (pblk_alloc_rqd_meta(pblk, rqd)) {  in pblk_submit_read()
291 		pblk_free_rqd(pblk, rqd, PBLK_READ);  in pblk_submit_read()
302 		nr_secs = pblk_read_ppalist_rq(pblk, rqd, int_bio, blba,  in pblk_submit_read()
305 		pblk_read_rq(pblk, rqd, int_bio, blba, &from_cache);  in pblk_submit_read()
314 		atomic_inc(&pblk->inflight_io);  in pblk_submit_read()
315 		__pblk_end_io_read(pblk, rqd, false);  in pblk_submit_read()
345 	} else if (pblk_submit_io(pblk, rqd, NULL)) {  in pblk_submit_read()
352 static int read_ppalist_rq_gc(struct pblk *pblk, struct nvm_rq *rqd,  in read_ppalist_rq_gc()  argument
361 	pblk_lookup_l2p_rand(pblk, ppa_list_l2p, lba_list, nr_secs);  in read_ppalist_rq_gc()
367 		ppa_gc = addr_to_gen_ppa(pblk, paddr_list_gc[i], line->id);  in read_ppalist_rq_gc()
377 	atomic_long_add(valid_secs, &pblk->inflight_reads);  in read_ppalist_rq_gc()
383 static int read_rq_gc(struct pblk *pblk, struct nvm_rq *rqd,  in read_rq_gc()  argument
394 	if (lba >= pblk->capacity) {  in read_rq_gc()
399 	spin_lock(&pblk->trans_lock);  in read_rq_gc()
400 	ppa_l2p = pblk_trans_map_get(pblk, lba);  in read_rq_gc()
401 	spin_unlock(&pblk->trans_lock);  in read_rq_gc()
403 	ppa_gc = addr_to_gen_ppa(pblk, paddr_gc, line->id);  in read_rq_gc()
411 	atomic_long_inc(&pblk->inflight_reads);  in read_rq_gc()
418 int pblk_submit_read_gc(struct pblk *pblk, struct pblk_gc_rq *gc_rq)  in pblk_submit_read_gc()  argument
425 	ret = pblk_alloc_rqd_meta(pblk, &rqd);  in pblk_submit_read_gc()
430 		gc_rq->secs_to_gc = read_ppalist_rq_gc(pblk, &rqd, gc_rq->line,  in pblk_submit_read_gc()
437 		gc_rq->secs_to_gc = read_rq_gc(pblk, &rqd, gc_rq->line,  in pblk_submit_read_gc()
448 	if (pblk_submit_io_sync(pblk, &rqd, gc_rq->data)) {  in pblk_submit_read_gc()
453 	pblk_read_check_rand(pblk, &rqd, gc_rq->lba_list, gc_rq->nr_secs);  in pblk_submit_read_gc()
455 	atomic_dec(&pblk->inflight_io);  in pblk_submit_read_gc()
458 		atomic_long_inc(&pblk->read_failed_gc);  in pblk_submit_read_gc()
460 		pblk_print_failed_rqd(pblk, &rqd, rqd.error);  in pblk_submit_read_gc()
465 	atomic_long_add(gc_rq->secs_to_gc, &pblk->sync_reads);  in pblk_submit_read_gc()
466 	atomic_long_add(gc_rq->secs_to_gc, &pblk->recov_gc_reads);  in pblk_submit_read_gc()
467 	atomic_long_sub(gc_rq->secs_to_gc, &pblk->inflight_reads);  in pblk_submit_read_gc()
471 	pblk_free_rqd_meta(pblk, &rqd);  in pblk_submit_read_gc()
475 	pblk_free_rqd_meta(pblk, &rqd);  in pblk_submit_read_gc()