Lines Matching full:line
32 struct pblk_line *line; in pblk_line_mark_bb() local
35 line = &pblk->lines[pblk_ppa_to_line(*ppa)]; in pblk_line_mark_bb()
38 pblk_err(pblk, "failed to mark bb, line:%d, pos:%d\n", in pblk_line_mark_bb()
39 line->id, pos); in pblk_line_mark_bb()
46 static void pblk_mark_bb(struct pblk *pblk, struct pblk_line *line, in pblk_mark_bb() argument
54 pblk_debug(pblk, "erase failed: line:%d, pos:%d\n", line->id, pos); in pblk_mark_bb()
57 atomic_dec(&line->blk_in_line); in pblk_mark_bb()
58 if (test_and_set_bit(pos, line->blk_bitmap)) in pblk_mark_bb()
59 pblk_err(pblk, "attempted to erase bb: line:%d, pos:%d\n", in pblk_mark_bb()
60 line->id, pos); in pblk_mark_bb()
80 struct pblk_line *line; in __pblk_end_io_erase() local
83 line = &pblk->lines[pblk_ppa_to_line(rqd->ppa_addr)]; in __pblk_end_io_erase()
85 chunk = &line->chks[pos]; in __pblk_end_io_erase()
87 atomic_dec(&line->left_seblks); in __pblk_end_io_erase()
91 pblk_mark_bb(pblk, line, rqd->ppa_addr); in __pblk_end_io_erase()
151 void __pblk_map_invalidate(struct pblk *pblk, struct pblk_line *line, in __pblk_map_invalidate() argument
161 spin_lock(&line->lock); in __pblk_map_invalidate()
162 WARN_ON(line->state == PBLK_LINESTATE_FREE); in __pblk_map_invalidate()
164 if (test_and_set_bit(paddr, line->invalid_bitmap)) { in __pblk_map_invalidate()
166 spin_unlock(&line->lock); in __pblk_map_invalidate()
169 le32_add_cpu(line->vsc, -1); in __pblk_map_invalidate()
171 if (line->state == PBLK_LINESTATE_CLOSED) in __pblk_map_invalidate()
172 move_list = pblk_line_gc_list(pblk, line); in __pblk_map_invalidate()
173 spin_unlock(&line->lock); in __pblk_map_invalidate()
177 spin_lock(&line->lock); in __pblk_map_invalidate()
178 /* Prevent moving a line that has just been chosen for GC */ in __pblk_map_invalidate()
179 if (line->state == PBLK_LINESTATE_GC) { in __pblk_map_invalidate()
180 spin_unlock(&line->lock); in __pblk_map_invalidate()
184 spin_unlock(&line->lock); in __pblk_map_invalidate()
186 list_move_tail(&line->list, move_list); in __pblk_map_invalidate()
193 struct pblk_line *line; in pblk_map_invalidate() local
204 line = &pblk->lines[line_id]; in pblk_map_invalidate()
207 __pblk_map_invalidate(pblk, line, paddr); in pblk_map_invalidate()
372 struct list_head *pblk_line_gc_list(struct pblk *pblk, struct pblk_line *line) in pblk_line_gc_list() argument
377 int vsc = le32_to_cpu(*line->vsc); in pblk_line_gc_list()
379 lockdep_assert_held(&line->lock); in pblk_line_gc_list()
381 if (line->w_err_gc->has_write_err) { in pblk_line_gc_list()
382 if (line->gc_group != PBLK_LINEGC_WERR) { in pblk_line_gc_list()
383 line->gc_group = PBLK_LINEGC_WERR; in pblk_line_gc_list()
388 if (line->gc_group != PBLK_LINEGC_FULL) { in pblk_line_gc_list()
389 line->gc_group = PBLK_LINEGC_FULL; in pblk_line_gc_list()
393 if (line->gc_group != PBLK_LINEGC_HIGH) { in pblk_line_gc_list()
394 line->gc_group = PBLK_LINEGC_HIGH; in pblk_line_gc_list()
398 if (line->gc_group != PBLK_LINEGC_MID) { in pblk_line_gc_list()
399 line->gc_group = PBLK_LINEGC_MID; in pblk_line_gc_list()
402 } else if (vsc < line->sec_in_line) { in pblk_line_gc_list()
403 if (line->gc_group != PBLK_LINEGC_LOW) { in pblk_line_gc_list()
404 line->gc_group = PBLK_LINEGC_LOW; in pblk_line_gc_list()
407 } else if (vsc == line->sec_in_line) { in pblk_line_gc_list()
408 if (line->gc_group != PBLK_LINEGC_EMPTY) { in pblk_line_gc_list()
409 line->gc_group = PBLK_LINEGC_EMPTY; in pblk_line_gc_list()
413 line->state = PBLK_LINESTATE_CORRUPT; in pblk_line_gc_list()
414 line->gc_group = PBLK_LINEGC_NONE; in pblk_line_gc_list()
416 pblk_err(pblk, "corrupted vsc for line %d, vsc:%d (%d/%d/%d)\n", in pblk_line_gc_list()
417 line->id, vsc, in pblk_line_gc_list()
418 line->sec_in_line, in pblk_line_gc_list()
562 void pblk_dealloc_page(struct pblk *pblk, struct pblk_line *line, int nr_secs) in pblk_dealloc_page() argument
567 spin_lock(&line->lock); in pblk_dealloc_page()
568 addr = find_next_zero_bit(line->map_bitmap, in pblk_dealloc_page()
569 pblk->lm.sec_per_line, line->cur_sec); in pblk_dealloc_page()
570 line->cur_sec = addr - nr_secs; in pblk_dealloc_page()
572 for (i = 0; i < nr_secs; i++, line->cur_sec--) in pblk_dealloc_page()
573 WARN_ON(!test_and_clear_bit(line->cur_sec, line->map_bitmap)); in pblk_dealloc_page()
574 spin_unlock(&line->lock); in pblk_dealloc_page()
577 u64 __pblk_alloc_page(struct pblk *pblk, struct pblk_line *line, int nr_secs) in __pblk_alloc_page() argument
582 lockdep_assert_held(&line->lock); in __pblk_alloc_page()
585 if (line->cur_sec + nr_secs > pblk->lm.sec_per_line) { in __pblk_alloc_page()
587 nr_secs = pblk->lm.sec_per_line - line->cur_sec; in __pblk_alloc_page()
590 line->cur_sec = addr = find_next_zero_bit(line->map_bitmap, in __pblk_alloc_page()
591 pblk->lm.sec_per_line, line->cur_sec); in __pblk_alloc_page()
592 for (i = 0; i < nr_secs; i++, line->cur_sec++) in __pblk_alloc_page()
593 WARN_ON(test_and_set_bit(line->cur_sec, line->map_bitmap)); in __pblk_alloc_page()
598 u64 pblk_alloc_page(struct pblk *pblk, struct pblk_line *line, int nr_secs) in pblk_alloc_page() argument
605 spin_lock(&line->lock); in pblk_alloc_page()
606 addr = __pblk_alloc_page(pblk, line, nr_secs); in pblk_alloc_page()
607 line->left_msecs -= nr_secs; in pblk_alloc_page()
608 WARN(line->left_msecs < 0, "pblk: page allocation out of bounds\n"); in pblk_alloc_page()
609 spin_unlock(&line->lock); in pblk_alloc_page()
614 u64 pblk_lookup_page(struct pblk *pblk, struct pblk_line *line) in pblk_lookup_page() argument
618 spin_lock(&line->lock); in pblk_lookup_page()
619 paddr = find_next_zero_bit(line->map_bitmap, in pblk_lookup_page()
620 pblk->lm.sec_per_line, line->cur_sec); in pblk_lookup_page()
621 spin_unlock(&line->lock); in pblk_lookup_page()
627 * Submit emeta to one LUN in the raid line at the time to avoid a deadlock when
630 static int pblk_line_submit_emeta_io(struct pblk *pblk, struct pblk_line *line, in pblk_line_submit_emeta_io() argument
643 int id = line->id; in pblk_line_submit_emeta_io()
695 spin_lock(&line->lock); in pblk_line_submit_emeta_io()
696 paddr = __pblk_alloc_page(pblk, line, min); in pblk_line_submit_emeta_io()
697 spin_unlock(&line->lock); in pblk_line_submit_emeta_io()
714 while (test_bit(pos, line->blk_bitmap)) { in pblk_line_submit_emeta_io()
717 pblk_err(pblk, "corrupt emeta line:%d\n", in pblk_line_submit_emeta_io()
718 line->id); in pblk_line_submit_emeta_io()
729 pblk_err(pblk, "corrupt emeta line:%d\n", in pblk_line_submit_emeta_io()
730 line->id); in pblk_line_submit_emeta_io()
738 addr_to_gen_ppa(pblk, paddr, line->id); in pblk_line_submit_emeta_io()
767 u64 pblk_line_smeta_start(struct pblk *pblk, struct pblk_line *line) in pblk_line_smeta_start() argument
775 bit = find_first_zero_bit(line->blk_bitmap, lm->blk_per_line); in pblk_line_smeta_start()
782 static int pblk_line_submit_smeta_io(struct pblk *pblk, struct pblk_line *line, in pblk_line_submit_smeta_io() argument
798 lba_list = emeta_to_lbas(pblk, line->emeta->buf); in pblk_line_submit_smeta_io()
816 bio = bio_map_kern(dev->q, line->smeta, lm->smeta_len, GFP_KERNEL); in pblk_line_submit_smeta_io()
833 rqd.ppa_list[i] = addr_to_gen_ppa(pblk, paddr, line->id); in pblk_line_submit_smeta_io()
843 * This I/O is sent by the write thread when a line is replace. Since in pblk_line_submit_smeta_io()
870 int pblk_line_read_smeta(struct pblk *pblk, struct pblk_line *line) in pblk_line_read_smeta() argument
872 u64 bpaddr = pblk_line_smeta_start(pblk, line); in pblk_line_read_smeta()
874 return pblk_line_submit_smeta_io(pblk, line, bpaddr, PBLK_READ_RECOV); in pblk_line_read_smeta()
877 int pblk_line_read_emeta(struct pblk *pblk, struct pblk_line *line, in pblk_line_read_emeta() argument
880 return pblk_line_submit_emeta_io(pblk, line, emeta_buf, in pblk_line_read_emeta()
881 line->emeta_ssec, PBLK_READ); in pblk_line_read_emeta()
911 int pblk_line_erase(struct pblk *pblk, struct pblk_line *line) in pblk_line_erase() argument
919 spin_lock(&line->lock); in pblk_line_erase()
920 bit = find_next_zero_bit(line->erase_bitmap, lm->blk_per_line, in pblk_line_erase()
923 spin_unlock(&line->lock); in pblk_line_erase()
928 ppa.a.blk = line->id; in pblk_line_erase()
930 atomic_dec(&line->left_eblks); in pblk_line_erase()
931 WARN_ON(test_and_set_bit(bit, line->erase_bitmap)); in pblk_line_erase()
932 spin_unlock(&line->lock); in pblk_line_erase()
936 pblk_err(pblk, "failed to erase line %d\n", line->id); in pblk_line_erase()
944 static void pblk_line_setup_metadata(struct pblk_line *line, in pblk_line_setup_metadata() argument
962 line->meta_line = meta_line; in pblk_line_setup_metadata()
964 line->smeta = l_mg->sline_meta[meta_line]; in pblk_line_setup_metadata()
965 line->emeta = l_mg->eline_meta[meta_line]; in pblk_line_setup_metadata()
967 memset(line->smeta, 0, lm->smeta_len); in pblk_line_setup_metadata()
968 memset(line->emeta->buf, 0, lm->emeta_len[0]); in pblk_line_setup_metadata()
970 line->emeta->mem = 0; in pblk_line_setup_metadata()
971 atomic_set(&line->emeta->sync, 0); in pblk_line_setup_metadata()
977 static int pblk_line_init_metadata(struct pblk *pblk, struct pblk_line *line, in pblk_line_init_metadata() argument
984 struct pblk_emeta *emeta = line->emeta; in pblk_line_init_metadata()
986 struct line_smeta *smeta_buf = (struct line_smeta *)line->smeta; in pblk_line_init_metadata()
989 /* After erasing the line, new bad blocks might appear and we risk in pblk_line_init_metadata()
990 * having an invalid line in pblk_line_init_metadata()
993 bitmap_weight(line->blk_bitmap, lm->blk_per_line); in pblk_line_init_metadata()
996 spin_lock(&line->lock); in pblk_line_init_metadata()
997 line->state = PBLK_LINESTATE_BAD; in pblk_line_init_metadata()
998 spin_unlock(&line->lock); in pblk_line_init_metadata()
1000 list_add_tail(&line->list, &l_mg->bad_list); in pblk_line_init_metadata()
1003 pblk_debug(pblk, "line %d is bad\n", line->id); in pblk_line_init_metadata()
1009 line->lun_bitmap = ((void *)(smeta_buf)) + sizeof(struct line_smeta); in pblk_line_init_metadata()
1011 /* Mark LUNs allocated in this line (all for now) */ in pblk_line_init_metadata()
1012 bitmap_set(line->lun_bitmap, 0, lm->lun_bitmap_len); in pblk_line_init_metadata()
1016 smeta_buf->header.id = cpu_to_le32(line->id); in pblk_line_init_metadata()
1017 smeta_buf->header.type = cpu_to_le16(line->type); in pblk_line_init_metadata()
1022 smeta_buf->seq_nr = cpu_to_le64(line->seq_nr); in pblk_line_init_metadata()
1027 memcpy(line->lun_bitmap, cur->lun_bitmap, lm->lun_bitmap_len); in pblk_line_init_metadata()
1029 cur->emeta->buf->next_id = cpu_to_le32(line->id); in pblk_line_init_metadata()
1048 emeta_buf->seq_nr = cpu_to_le64(line->seq_nr); in pblk_line_init_metadata()
1049 emeta_buf->nr_lbas = cpu_to_le64(line->sec_in_line); in pblk_line_init_metadata()
1058 static int pblk_line_alloc_bitmaps(struct pblk *pblk, struct pblk_line *line) in pblk_line_alloc_bitmaps() argument
1062 line->map_bitmap = kzalloc(lm->sec_bitmap_len, GFP_KERNEL); in pblk_line_alloc_bitmaps()
1063 if (!line->map_bitmap) in pblk_line_alloc_bitmaps()
1067 line->invalid_bitmap = kmalloc(lm->sec_bitmap_len, GFP_KERNEL); in pblk_line_alloc_bitmaps()
1068 if (!line->invalid_bitmap) { in pblk_line_alloc_bitmaps()
1069 kfree(line->map_bitmap); in pblk_line_alloc_bitmaps()
1070 line->map_bitmap = NULL; in pblk_line_alloc_bitmaps()
1080 static int pblk_line_init_bb(struct pblk *pblk, struct pblk_line *line, in pblk_line_init_bb() argument
1091 line->sec_in_line = lm->sec_per_line; in pblk_line_init_bb()
1093 /* Capture bad block information on line mapping bitmaps */ in pblk_line_init_bb()
1094 while ((bit = find_next_bit(line->blk_bitmap, lm->blk_per_line, in pblk_line_init_bb()
1099 bitmap_or(line->map_bitmap, line->map_bitmap, l_mg->bb_aux, in pblk_line_init_bb()
1101 line->sec_in_line -= geo->clba; in pblk_line_init_bb()
1105 bit = find_first_zero_bit(line->blk_bitmap, lm->blk_per_line); in pblk_line_init_bb()
1107 bitmap_set(line->map_bitmap, off, lm->smeta_sec); in pblk_line_init_bb()
1108 line->sec_in_line -= lm->smeta_sec; in pblk_line_init_bb()
1109 line->smeta_ssec = off; in pblk_line_init_bb()
1110 line->cur_sec = off + lm->smeta_sec; in pblk_line_init_bb()
1112 if (init && pblk_line_submit_smeta_io(pblk, line, off, PBLK_WRITE)) { in pblk_line_init_bb()
1113 pblk_debug(pblk, "line smeta I/O failed. Retry\n"); in pblk_line_init_bb()
1117 bitmap_copy(line->invalid_bitmap, line->map_bitmap, lm->sec_per_line); in pblk_line_init_bb()
1126 if (!test_bit(off, line->invalid_bitmap)) { in pblk_line_init_bb()
1127 bitmap_set(line->invalid_bitmap, off, geo->ws_opt); in pblk_line_init_bb()
1132 line->emeta_ssec = off; in pblk_line_init_bb()
1133 line->sec_in_line -= lm->emeta_sec[0]; in pblk_line_init_bb()
1134 line->nr_valid_lbas = 0; in pblk_line_init_bb()
1135 line->left_msecs = line->sec_in_line; in pblk_line_init_bb()
1136 *line->vsc = cpu_to_le32(line->sec_in_line); in pblk_line_init_bb()
1138 if (lm->sec_per_line - line->sec_in_line != in pblk_line_init_bb()
1139 bitmap_weight(line->invalid_bitmap, lm->sec_per_line)) { in pblk_line_init_bb()
1140 spin_lock(&line->lock); in pblk_line_init_bb()
1141 line->state = PBLK_LINESTATE_BAD; in pblk_line_init_bb()
1142 spin_unlock(&line->lock); in pblk_line_init_bb()
1144 list_add_tail(&line->list, &l_mg->bad_list); in pblk_line_init_bb()
1145 pblk_err(pblk, "unexpected line %d is bad\n", line->id); in pblk_line_init_bb()
1153 static int pblk_prepare_new_line(struct pblk *pblk, struct pblk_line *line) in pblk_prepare_new_line() argument
1158 int blk_to_erase = atomic_read(&line->blk_in_line); in pblk_prepare_new_line()
1164 int state = line->chks[pos].state; in pblk_prepare_new_line()
1169 line->erase_bitmap); in pblk_prepare_new_line()
1177 static int pblk_line_prepare(struct pblk *pblk, struct pblk_line *line) in pblk_line_prepare() argument
1180 int blk_in_line = atomic_read(&line->blk_in_line); in pblk_line_prepare()
1184 bitmap_copy(line->erase_bitmap, line->blk_bitmap, lm->blk_per_line); in pblk_line_prepare()
1186 spin_lock(&line->lock); in pblk_line_prepare()
1188 /* If we have not written to this line, we need to mark up free chunks in pblk_line_prepare()
1191 if (line->state == PBLK_LINESTATE_NEW) { in pblk_line_prepare()
1192 blk_to_erase = pblk_prepare_new_line(pblk, line); in pblk_line_prepare()
1193 line->state = PBLK_LINESTATE_FREE; in pblk_line_prepare()
1199 spin_unlock(&line->lock); in pblk_line_prepare()
1203 if (line->state != PBLK_LINESTATE_FREE) { in pblk_line_prepare()
1204 WARN(1, "pblk: corrupted line %d, state %d\n", in pblk_line_prepare()
1205 line->id, line->state); in pblk_line_prepare()
1206 spin_unlock(&line->lock); in pblk_line_prepare()
1210 line->state = PBLK_LINESTATE_OPEN; in pblk_line_prepare()
1212 atomic_set(&line->left_eblks, blk_to_erase); in pblk_line_prepare()
1213 atomic_set(&line->left_seblks, blk_to_erase); in pblk_line_prepare()
1215 line->meta_distance = lm->meta_distance; in pblk_line_prepare()
1216 spin_unlock(&line->lock); in pblk_line_prepare()
1218 kref_init(&line->ref); in pblk_line_prepare()
1223 int pblk_line_recov_alloc(struct pblk *pblk, struct pblk_line *line) in pblk_line_recov_alloc() argument
1229 l_mg->data_line = line; in pblk_line_recov_alloc()
1230 list_del(&line->list); in pblk_line_recov_alloc()
1232 ret = pblk_line_prepare(pblk, line); in pblk_line_recov_alloc()
1234 list_add(&line->list, &l_mg->free_list); in pblk_line_recov_alloc()
1240 ret = pblk_line_alloc_bitmaps(pblk, line); in pblk_line_recov_alloc()
1244 if (!pblk_line_init_bb(pblk, line, 0)) { in pblk_line_recov_alloc()
1249 pblk_rl_free_lines_dec(&pblk->rl, line, true); in pblk_line_recov_alloc()
1254 list_add(&line->list, &l_mg->free_list); in pblk_line_recov_alloc()
1260 void pblk_line_recov_close(struct pblk *pblk, struct pblk_line *line) in pblk_line_recov_close() argument
1262 kfree(line->map_bitmap); in pblk_line_recov_close()
1263 line->map_bitmap = NULL; in pblk_line_recov_close()
1264 line->smeta = NULL; in pblk_line_recov_close()
1265 line->emeta = NULL; in pblk_line_recov_close()
1268 static void pblk_line_reinit(struct pblk_line *line) in pblk_line_reinit() argument
1270 *line->vsc = cpu_to_le32(EMPTY_ENTRY); in pblk_line_reinit()
1272 line->map_bitmap = NULL; in pblk_line_reinit()
1273 line->invalid_bitmap = NULL; in pblk_line_reinit()
1274 line->smeta = NULL; in pblk_line_reinit()
1275 line->emeta = NULL; in pblk_line_reinit()
1278 void pblk_line_free(struct pblk_line *line) in pblk_line_free() argument
1280 kfree(line->map_bitmap); in pblk_line_free()
1281 kfree(line->invalid_bitmap); in pblk_line_free()
1283 pblk_line_reinit(line); in pblk_line_free()
1290 struct pblk_line *line; in pblk_line_get() local
1301 line = list_first_entry(&l_mg->free_list, struct pblk_line, list); in pblk_line_get()
1302 list_del(&line->list); in pblk_line_get()
1305 bit = find_first_zero_bit(line->blk_bitmap, lm->blk_per_line); in pblk_line_get()
1307 spin_lock(&line->lock); in pblk_line_get()
1308 line->state = PBLK_LINESTATE_BAD; in pblk_line_get()
1309 spin_unlock(&line->lock); in pblk_line_get()
1311 list_add_tail(&line->list, &l_mg->bad_list); in pblk_line_get()
1313 pblk_debug(pblk, "line %d is bad\n", line->id); in pblk_line_get()
1317 ret = pblk_line_prepare(pblk, line); in pblk_line_get()
1321 list_add(&line->list, &l_mg->bad_list); in pblk_line_get()
1324 list_add(&line->list, &l_mg->corrupt_list); in pblk_line_get()
1327 pblk_err(pblk, "failed to prepare line %d\n", line->id); in pblk_line_get()
1328 list_add(&line->list, &l_mg->free_list); in pblk_line_get()
1334 return line; in pblk_line_get()
1338 struct pblk_line *line) in pblk_line_retry() argument
1352 retry_line->map_bitmap = line->map_bitmap; in pblk_line_retry()
1353 retry_line->invalid_bitmap = line->invalid_bitmap; in pblk_line_retry()
1354 retry_line->smeta = line->smeta; in pblk_line_retry()
1355 retry_line->emeta = line->emeta; in pblk_line_retry()
1356 retry_line->meta_line = line->meta_line; in pblk_line_retry()
1358 pblk_line_reinit(line); in pblk_line_retry()
1363 pblk_rl_free_lines_dec(&pblk->rl, line, false); in pblk_line_retry()
1381 struct pblk_line *line; in pblk_line_get_first_data() local
1384 line = pblk_line_get(pblk); in pblk_line_get_first_data()
1385 if (!line) { in pblk_line_get_first_data()
1390 line->seq_nr = l_mg->d_seq_nr++; in pblk_line_get_first_data()
1391 line->type = PBLK_LINETYPE_DATA; in pblk_line_get_first_data()
1392 l_mg->data_line = line; in pblk_line_get_first_data()
1394 pblk_line_setup_metadata(line, l_mg, &pblk->lm); in pblk_line_get_first_data()
1396 /* Allocate next line for preparation */ in pblk_line_get_first_data()
1399 /* If we cannot get a new line, we need to stop the pipeline. in pblk_line_get_first_data()
1412 if (pblk_line_alloc_bitmaps(pblk, line)) in pblk_line_get_first_data()
1415 if (pblk_line_erase(pblk, line)) { in pblk_line_get_first_data()
1416 line = pblk_line_retry(pblk, line); in pblk_line_get_first_data()
1417 if (!line) in pblk_line_get_first_data()
1422 if (!pblk_line_init_metadata(pblk, line, NULL)) { in pblk_line_get_first_data()
1423 line = pblk_line_retry(pblk, line); in pblk_line_get_first_data()
1424 if (!line) in pblk_line_get_first_data()
1430 if (!pblk_line_init_bb(pblk, line, 1)) { in pblk_line_get_first_data()
1431 line = pblk_line_retry(pblk, line); in pblk_line_get_first_data()
1432 if (!line) in pblk_line_get_first_data()
1438 pblk_rl_free_lines_dec(&pblk->rl, line, true); in pblk_line_get_first_data()
1440 return line; in pblk_line_get_first_data()
1443 static void pblk_stop_writes(struct pblk *pblk, struct pblk_line *line) in pblk_stop_writes() argument
1455 struct pblk_line *line, *tline; in pblk_line_close_meta_sync() local
1467 list_for_each_entry_safe(line, tline, &list, list) { in pblk_line_close_meta_sync()
1468 struct pblk_emeta *emeta = line->emeta; in pblk_line_close_meta_sync()
1473 ret = pblk_submit_meta_io(pblk, line); in pblk_line_close_meta_sync()
1475 pblk_err(pblk, "sync meta line %d failed (%d)\n", in pblk_line_close_meta_sync()
1476 line->id, ret); in pblk_line_close_meta_sync()
1550 /* If line is not fully erased, erase it */ in pblk_line_replace_data()
1582 /* Allocate next line for preparation */ in pblk_line_replace_data()
1586 /* If we cannot get a new line, we need to stop the pipeline. in pblk_line_replace_data()
1602 static void __pblk_line_put(struct pblk *pblk, struct pblk_line *line) in __pblk_line_put() argument
1607 spin_lock(&line->lock); in __pblk_line_put()
1608 WARN_ON(line->state != PBLK_LINESTATE_GC); in __pblk_line_put()
1609 line->state = PBLK_LINESTATE_FREE; in __pblk_line_put()
1610 line->gc_group = PBLK_LINEGC_NONE; in __pblk_line_put()
1611 pblk_line_free(line); in __pblk_line_put()
1613 if (line->w_err_gc->has_write_err) { in __pblk_line_put()
1615 line->w_err_gc->has_write_err = 0; in __pblk_line_put()
1618 spin_unlock(&line->lock); in __pblk_line_put()
1622 list_add_tail(&line->list, &l_mg->free_list); in __pblk_line_put()
1626 pblk_rl_free_lines_inc(&pblk->rl, line); in __pblk_line_put()
1634 struct pblk_line *line = line_put_ws->line; in pblk_line_put_ws() local
1636 __pblk_line_put(pblk, line); in pblk_line_put_ws()
1642 struct pblk_line *line = container_of(ref, struct pblk_line, ref); in pblk_line_put() local
1643 struct pblk *pblk = line->pblk; in pblk_line_put()
1645 __pblk_line_put(pblk, line); in pblk_line_put()
1650 struct pblk_line *line = container_of(ref, struct pblk_line, ref); in pblk_line_put_wq() local
1651 struct pblk *pblk = line->pblk; in pblk_line_put_wq()
1659 line_put_ws->line = line; in pblk_line_put_wq()
1686 pblk_err(pblk, "could not async erase line:%d,blk:%d\n", in pblk_blk_erase_async()
1699 /* For now, always erase next line */
1705 int pblk_line_is_full(struct pblk_line *line) in pblk_line_is_full() argument
1707 return (line->left_msecs == 0); in pblk_line_is_full()
1716 void pblk_line_close(struct pblk *pblk, struct pblk_line *line) in pblk_line_close() argument
1726 WARN(!bitmap_full(line->map_bitmap, lm->sec_per_line), in pblk_line_close()
1727 "pblk: corrupt closed line %d\n", line->id); in pblk_line_close()
1731 WARN_ON(!test_and_clear_bit(line->meta_line, &l_mg->meta_bitmap)); in pblk_line_close()
1735 spin_lock(&line->lock); in pblk_line_close()
1736 WARN_ON(line->state != PBLK_LINESTATE_OPEN); in pblk_line_close()
1737 line->state = PBLK_LINESTATE_CLOSED; in pblk_line_close()
1738 move_list = pblk_line_gc_list(pblk, line); in pblk_line_close()
1740 list_add_tail(&line->list, move_list); in pblk_line_close()
1742 kfree(line->map_bitmap); in pblk_line_close()
1743 line->map_bitmap = NULL; in pblk_line_close()
1744 line->smeta = NULL; in pblk_line_close()
1745 line->emeta = NULL; in pblk_line_close()
1750 int state = line->chks[pos].state; in pblk_line_close()
1756 spin_unlock(&line->lock); in pblk_line_close()
1760 void pblk_line_close_meta(struct pblk *pblk, struct pblk_line *line) in pblk_line_close_meta() argument
1764 struct pblk_emeta *emeta = line->emeta; in pblk_line_close_meta()
1768 /* No need for exact vsc value; avoid a big line lock and take aprox. */ in pblk_line_close_meta()
1770 memcpy(emeta_to_bb(emeta_buf), line->blk_bitmap, lm->blk_bitmap_len); in pblk_line_close_meta()
1779 emeta_buf->header.id = cpu_to_le32(line->id); in pblk_line_close_meta()
1780 emeta_buf->header.type = cpu_to_le16(line->type); in pblk_line_close_meta()
1787 emeta_buf->nr_valid_lbas = cpu_to_le64(line->nr_valid_lbas); in pblk_line_close_meta()
1791 spin_lock(&line->lock); in pblk_line_close_meta()
1796 if (line->emeta_ssec != line->cur_sec) in pblk_line_close_meta()
1797 line->emeta_ssec = line->cur_sec; in pblk_line_close_meta()
1799 list_add_tail(&line->list, &l_mg->emeta_list); in pblk_line_close_meta()
1800 spin_unlock(&line->lock); in pblk_line_close_meta()
1806 static void pblk_save_lba_list(struct pblk *pblk, struct pblk_line *line) in pblk_save_lba_list() argument
1811 struct pblk_w_err_gc *w_err_gc = line->w_err_gc; in pblk_save_lba_list()
1812 struct pblk_emeta *emeta = line->emeta; in pblk_save_lba_list()
1825 struct pblk_line *line = line_ws->line; in pblk_line_close_ws() local
1826 struct pblk_w_err_gc *w_err_gc = line->w_err_gc; in pblk_line_close_ws()
1829 * so keep a copy of the lba list until we've gc'd the line in pblk_line_close_ws()
1832 pblk_save_lba_list(pblk, line); in pblk_line_close_ws()
1834 pblk_line_close(pblk, line); in pblk_line_close_ws()
1838 void pblk_gen_run_ws(struct pblk *pblk, struct pblk_line *line, void *priv, in pblk_gen_run_ws() argument
1847 line_ws->line = line; in pblk_gen_run_ws()
2065 /* If the L2P entry maps to a line, the reference is valid */ in pblk_lookup_l2p_seq()
2068 struct pblk_line *line = &pblk->lines[line_id]; in pblk_lookup_l2p_seq() local
2070 kref_get(&line->ref); in pblk_lookup_l2p_seq()