31 static int pblk_gc_write(struct pblk *pblk)  in pblk_gc_write()  argument
33 	struct pblk_gc *gc = &pblk->gc;  in pblk_gc_write()
48 		pblk_write_gc_to_cache(pblk, gc_rq);  in pblk_gc_write()
62 void pblk_put_line_back(struct pblk *pblk, struct pblk_line *line)  in pblk_put_line_back()  argument
64 	struct pblk_line_mgmt *l_mg = &pblk->l_mg;  in pblk_put_line_back()
71 	trace_pblk_line_state(pblk_disk_name(pblk), line->id,  in pblk_put_line_back()
80 	move_list = pblk_line_gc_list(pblk, line);  in pblk_put_line_back()
90 	struct pblk *pblk = gc_rq_ws->pblk;  in pblk_gc_line_ws()  local
91 	struct pblk_gc *gc = &pblk->gc;  in pblk_gc_line_ws()
99 	ret = pblk_submit_read_gc(pblk, gc_rq);  in pblk_gc_line_ws()
112 		pblk_gc_writer_kick(&pblk->gc);  in pblk_gc_line_ws()
120 	pblk_gc_writer_kick(&pblk->gc);  in pblk_gc_line_ws()
131 static __le64 *get_lba_list_from_emeta(struct pblk *pblk,  in get_lba_list_from_emeta()  argument
135 	struct pblk_line_meta *lm = &pblk->lm;  in get_lba_list_from_emeta()
144 	ret = pblk_line_emeta_read(pblk, line, emeta_buf);  in get_lba_list_from_emeta()
146 		pblk_err(pblk, "line %d read emeta failed (%d)\n",  in get_lba_list_from_emeta()
158 	ret = pblk_recov_check_emeta(pblk, emeta_buf);  in get_lba_list_from_emeta()
160 		pblk_err(pblk, "inconsistent emeta (line %d)\n",  in get_lba_list_from_emeta()
169 		memcpy(lba_list, emeta_to_lbas(pblk, emeta_buf), lba_list_size);  in get_lba_list_from_emeta()
180 	struct pblk *pblk = line_ws->pblk;  in pblk_gc_line_prepare_ws()  local
182 	struct pblk_line_meta *lm = &pblk->lm;  in pblk_gc_line_prepare_ws()
183 	struct nvm_tgt_dev *dev = pblk->dev;  in pblk_gc_line_prepare_ws()
185 	struct pblk_gc *gc = &pblk->gc;  in pblk_gc_line_prepare_ws()
200 		lba_list = get_lba_list_from_emeta(pblk, line);  in pblk_gc_line_prepare_ws()
202 			pblk_err(pblk, "could not interpret emeta (line %d)\n",  in pblk_gc_line_prepare_ws()
214 		pblk_err(pblk, "corrupted GC line (%d)\n", line->id);  in pblk_gc_line_prepare_ws()
233 	} while (nr_secs < pblk->max_write_pgs);  in pblk_gc_line_prepare_ws()
251 	gc_rq_ws->pblk = pblk;  in pblk_gc_line_prepare_ws()
296 	pblk_put_line_back(pblk, line);  in pblk_gc_line_prepare_ws()
299 	pblk_err(pblk, "failed to GC line %d\n", line->id);  in pblk_gc_line_prepare_ws()
302 static int pblk_gc_line(struct pblk *pblk, struct pblk_line *line)  in pblk_gc_line()  argument
304 	struct pblk_gc *gc = &pblk->gc;  in pblk_gc_line()
307 	pblk_debug(pblk, "line '%d' being reclaimed for GC\n", line->id);  in pblk_gc_line()
313 	line_ws->pblk = pblk;  in pblk_gc_line()
328 static void pblk_gc_kick(struct pblk *pblk)  in pblk_gc_kick()  argument
330 	struct pblk_gc *gc = &pblk->gc;  in pblk_gc_kick()
343 static int pblk_gc_read(struct pblk *pblk)  in pblk_gc_read()  argument
345 	struct pblk_gc *gc = &pblk->gc;  in pblk_gc_read()
358 	pblk_gc_kick(pblk);  in pblk_gc_read()
360 	if (pblk_gc_line(pblk, line)) {  in pblk_gc_read()
361 		pblk_err(pblk, "failed to GC line %d\n", line->id);  in pblk_gc_read()
371 static struct pblk_line *pblk_gc_get_victim_line(struct pblk *pblk,  in pblk_gc_get_victim_line()  argument
407 void pblk_gc_free_full_lines(struct pblk *pblk)  in pblk_gc_free_full_lines()  argument
409 	struct pblk_line_mgmt *l_mg = &pblk->l_mg;  in pblk_gc_free_full_lines()
410 	struct pblk_gc *gc = &pblk->gc;  in pblk_gc_free_full_lines()
426 		trace_pblk_line_state(pblk_disk_name(pblk), line->id,  in pblk_gc_free_full_lines()
444 static void pblk_gc_run(struct pblk *pblk)  in pblk_gc_run()  argument
446 	struct pblk_line_mgmt *l_mg = &pblk->l_mg;  in pblk_gc_run()
447 	struct pblk_gc *gc = &pblk->gc;  in pblk_gc_run()
453 	pblk_gc_free_full_lines(pblk);  in pblk_gc_run()
455 	run_gc = pblk_gc_should_run(&pblk->gc, &pblk->rl);  in pblk_gc_run()
465 		line = pblk_gc_get_victim_line(pblk, group_list);  in pblk_gc_run()
474 		trace_pblk_line_state(pblk_disk_name(pblk), line->id,  in pblk_gc_run()
491 		run_gc = pblk_gc_should_run(&pblk->gc, &pblk->rl);  in pblk_gc_run()
496 	if (!prev_group && pblk->rl.rb_state > gc_group &&  in pblk_gc_run()
503 	struct pblk *pblk = from_timer(pblk, t, gc.gc_timer);  in pblk_gc_timer()  local
505 	pblk_gc_kick(pblk);  in pblk_gc_timer()
510 	struct pblk *pblk = data;  in pblk_gc_ts()  local
513 		pblk_gc_run(pblk);  in pblk_gc_ts()
523 	struct pblk *pblk = data;  in pblk_gc_writer_ts()  local
526 		if (!pblk_gc_write(pblk))  in pblk_gc_writer_ts()
537 	struct pblk *pblk = data;  in pblk_gc_reader_ts()  local
538 	struct pblk_gc *gc = &pblk->gc;  in pblk_gc_reader_ts()
541 		if (!pblk_gc_read(pblk))  in pblk_gc_reader_ts()
548 	pblk_info(pblk, "flushing gc pipeline, %d lines left\n",  in pblk_gc_reader_ts()
562 static void pblk_gc_start(struct pblk *pblk)  in pblk_gc_start()  argument
564 	pblk->gc.gc_active = 1;  in pblk_gc_start()
565 	pblk_debug(pblk, "gc start\n");  in pblk_gc_start()
568 void pblk_gc_should_start(struct pblk *pblk)  in pblk_gc_should_start()  argument
570 	struct pblk_gc *gc = &pblk->gc;  in pblk_gc_should_start()
573 		pblk_gc_start(pblk);  in pblk_gc_should_start()
574 		pblk_gc_kick(pblk);  in pblk_gc_should_start()
578 void pblk_gc_should_stop(struct pblk *pblk)  in pblk_gc_should_stop()  argument
580 	struct pblk_gc *gc = &pblk->gc;  in pblk_gc_should_stop()
586 void pblk_gc_should_kick(struct pblk *pblk)  in pblk_gc_should_kick()  argument
588 	pblk_rl_update_rates(&pblk->rl);  in pblk_gc_should_kick()
591 void pblk_gc_sysfs_state_show(struct pblk *pblk, int *gc_enabled,  in pblk_gc_sysfs_state_show()  argument
594 	struct pblk_gc *gc = &pblk->gc;  in pblk_gc_sysfs_state_show()
602 int pblk_gc_sysfs_force(struct pblk *pblk, int force)  in pblk_gc_sysfs_force()  argument
604 	struct pblk_gc *gc = &pblk->gc;  in pblk_gc_sysfs_force()
618 	pblk_gc_should_start(pblk);  in pblk_gc_sysfs_force()
623 int pblk_gc_init(struct pblk *pblk)  in pblk_gc_init()  argument
625 	struct pblk_gc *gc = &pblk->gc;  in pblk_gc_init()
628 	gc->gc_ts = kthread_create(pblk_gc_ts, pblk, "pblk-gc-ts");  in pblk_gc_init()
630 		pblk_err(pblk, "could not allocate GC main kthread\n");  in pblk_gc_init()
634 	gc->gc_writer_ts = kthread_create(pblk_gc_writer_ts, pblk,  in pblk_gc_init()
637 		pblk_err(pblk, "could not allocate GC writer kthread\n");  in pblk_gc_init()
642 	gc->gc_reader_ts = kthread_create(pblk_gc_reader_ts, pblk,  in pblk_gc_init()
645 		pblk_err(pblk, "could not allocate GC reader kthread\n");  in pblk_gc_init()
666 		pblk_err(pblk, "could not allocate GC line reader workqueue\n");  in pblk_gc_init()
675 		pblk_err(pblk, "could not allocate GC reader workqueue\n");  in pblk_gc_init()
703 void pblk_gc_exit(struct pblk *pblk, bool graceful)  in pblk_gc_exit()  argument
705 	struct pblk_gc *gc = &pblk->gc;  in pblk_gc_exit()