Lines Matching refs:v
21 bool verity_fec_is_enabled(struct dm_verity *v) in verity_fec_is_enabled() argument
23 return v->fec && v->fec->dev; in verity_fec_is_enabled()
32 return (struct dm_verity_fec_io *) verity_io_digest_end(io->v, io); in fec_io()
38 static inline u64 fec_interleave(struct dm_verity *v, u64 offset) in fec_interleave() argument
42 mod = do_div(offset, v->fec->rsn); in fec_interleave()
43 return offset + mod * (v->fec->rounds << v->data_dev_block_bits); in fec_interleave()
49 static int fec_decode_rs8(struct dm_verity *v, struct dm_verity_fec_io *fio, in fec_decode_rs8() argument
55 for (i = 0; i < v->fec->roots; i++) in fec_decode_rs8()
58 return decode_rs8(fio->rs, data, par, v->fec->rsn, NULL, neras, in fec_decode_rs8()
66 static u8 *fec_read_parity(struct dm_verity *v, u64 rsb, int index, in fec_read_parity() argument
72 position = (index + rsb) * v->fec->roots; in fec_read_parity()
73 block = position >> v->data_dev_block_bits; in fec_read_parity()
74 *offset = (unsigned)(position - (block << v->data_dev_block_bits)); in fec_read_parity()
76 res = dm_bufio_read(v->fec->bufio, v->fec->start + block, buf); in fec_read_parity()
79 v->data_dev->name, (unsigned long long)rsb, in fec_read_parity()
80 (unsigned long long)(v->fec->start + block), in fec_read_parity()
109 static inline u8 *fec_buffer_rs_block(struct dm_verity *v, in fec_buffer_rs_block() argument
113 return &fio->bufs[i][j * v->fec->rsn]; in fec_buffer_rs_block()
129 static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio, in fec_decode_bufs() argument
138 par = fec_read_parity(v, rsb, block_offset, &offset, &buf); in fec_decode_bufs()
147 block = fec_buffer_rs_block(v, fio, n, i); in fec_decode_bufs()
148 res = fec_decode_rs8(v, fio, block, &par[offset], neras); in fec_decode_bufs()
160 if (block_offset >= 1 << v->data_dev_block_bits) in fec_decode_bufs()
164 offset += v->fec->roots; in fec_decode_bufs()
165 if (offset >= 1 << v->data_dev_block_bits) { in fec_decode_bufs()
168 par = fec_read_parity(v, rsb, block_offset, &offset, &buf); in fec_decode_bufs()
178 v->data_dev->name, (unsigned long long)rsb, r); in fec_decode_bufs()
181 v->data_dev->name, (unsigned long long)rsb, r); in fec_decode_bufs()
182 atomic_add_unless(&v->fec->corrected, 1, INT_MAX); in fec_decode_bufs()
191 static int fec_is_erasure(struct dm_verity *v, struct dm_verity_io *io, in fec_is_erasure() argument
194 if (unlikely(verity_hash(v, verity_io_hash_desc(v, io), in fec_is_erasure()
195 data, 1 << v->data_dev_block_bits, in fec_is_erasure()
196 verity_io_real_digest(v, io)))) in fec_is_erasure()
199 return memcmp(verity_io_real_digest(v, io), want_digest, in fec_is_erasure()
200 v->digest_size) != 0; in fec_is_erasure()
207 static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io, in fec_read_bufs() argument
218 u8 want_digest[v->digest_size]; in fec_read_bufs()
228 for (i = 0; i < v->fec->rsn; i++) { in fec_read_bufs()
229 ileaved = fec_interleave(v, rsb * v->fec->rsn + i); in fec_read_bufs()
238 block = ileaved >> v->data_dev_block_bits; in fec_read_bufs()
239 bufio = v->fec->data_bufio; in fec_read_bufs()
241 if (block >= v->data_blocks) { in fec_read_bufs()
242 block -= v->data_blocks; in fec_read_bufs()
248 if (unlikely(block >= v->fec->hash_blocks)) in fec_read_bufs()
251 block += v->hash_start; in fec_read_bufs()
252 bufio = v->bufio; in fec_read_bufs()
258 v->data_dev->name, in fec_read_bufs()
263 if (neras && *neras <= v->fec->roots) in fec_read_bufs()
270 if (bufio == v->fec->data_bufio && in fec_read_bufs()
271 verity_hash_for_block(v, io, block, want_digest, in fec_read_bufs()
281 if (neras && *neras <= v->fec->roots && in fec_read_bufs()
282 fec_is_erasure(v, io, want_digest, bbuf)) in fec_read_bufs()
293 if (k >= 1 << v->data_dev_block_bits) in fec_read_bufs()
296 rs_block = fec_buffer_rs_block(v, fio, n, j); in fec_read_bufs()
310 static int fec_alloc_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio) in fec_alloc_bufs() argument
315 fio->rs = mempool_alloc(v->fec->rs_pool, 0); in fec_alloc_bufs()
326 fio->bufs[n] = mempool_alloc(v->fec->prealloc_pool, GFP_NOIO); in fec_alloc_bufs()
338 fio->bufs[n] = mempool_alloc(v->fec->extra_pool, GFP_NOIO); in fec_alloc_bufs()
346 fio->output = mempool_alloc(v->fec->output_pool, GFP_NOIO); in fec_alloc_bufs()
361 static void fec_init_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio) in fec_init_bufs() argument
366 memset(fio->bufs[n], 0, v->fec->rsn << DM_VERITY_FEC_BUF_RS_BITS); in fec_init_bufs()
376 static int fec_decode_rsb(struct dm_verity *v, struct dm_verity_io *io, in fec_decode_rsb() argument
383 r = fec_alloc_bufs(v, fio); in fec_decode_rsb()
387 for (pos = 0; pos < 1 << v->data_dev_block_bits; ) { in fec_decode_rsb()
388 fec_init_bufs(v, fio); in fec_decode_rsb()
390 r = fec_read_bufs(v, io, rsb, offset, pos, in fec_decode_rsb()
395 r = fec_decode_bufs(v, fio, rsb, r, pos, neras); in fec_decode_rsb()
403 r = verity_hash(v, verity_io_hash_desc(v, io), fio->output, in fec_decode_rsb()
404 1 << v->data_dev_block_bits, in fec_decode_rsb()
405 verity_io_real_digest(v, io)); in fec_decode_rsb()
409 if (memcmp(verity_io_real_digest(v, io), verity_io_want_digest(v, io), in fec_decode_rsb()
410 v->digest_size)) { in fec_decode_rsb()
412 v->data_dev->name, (unsigned long long)rsb, neras); in fec_decode_rsb()
419 static int fec_bv_copy(struct dm_verity *v, struct dm_verity_io *io, u8 *data, in fec_bv_copy() argument
434 int verity_fec_decode(struct dm_verity *v, struct dm_verity_io *io, in verity_fec_decode() argument
442 if (!verity_fec_is_enabled(v)) in verity_fec_decode()
446 DMWARN_LIMIT("%s: FEC: recursion too deep", v->data_dev->name); in verity_fec_decode()
453 block += v->data_blocks; in verity_fec_decode()
465 offset = block << v->data_dev_block_bits; in verity_fec_decode()
466 res = div64_u64(offset, v->fec->rounds << v->data_dev_block_bits); in verity_fec_decode()
472 rsb = offset - res * (v->fec->rounds << v->data_dev_block_bits); in verity_fec_decode()
479 r = fec_decode_rsb(v, io, fio, rsb, offset, false); in verity_fec_decode()
481 r = fec_decode_rsb(v, io, fio, rsb, offset, true); in verity_fec_decode()
487 memcpy(dest, fio->output, 1 << v->data_dev_block_bits); in verity_fec_decode()
490 r = verity_for_bv_block(v, io, iter, fec_bv_copy); in verity_fec_decode()
504 struct dm_verity_fec *f = io->v->fec; in verity_fec_finish_io()
507 if (!verity_fec_is_enabled(io->v)) in verity_fec_finish_io()
528 if (!verity_fec_is_enabled(io->v)) in verity_fec_init_io()
541 unsigned verity_fec_status_table(struct dm_verity *v, unsigned sz, in verity_fec_status_table() argument
544 if (!verity_fec_is_enabled(v)) in verity_fec_status_table()
551 v->fec->dev->name, in verity_fec_status_table()
552 (unsigned long long)v->fec->blocks, in verity_fec_status_table()
553 (unsigned long long)v->fec->start, in verity_fec_status_table()
554 v->fec->roots); in verity_fec_status_table()
559 void verity_fec_dtr(struct dm_verity *v) in verity_fec_dtr() argument
561 struct dm_verity_fec *f = v->fec; in verity_fec_dtr()
564 if (!verity_fec_is_enabled(v)) in verity_fec_dtr()
578 dm_put_device(v->ti, f->dev); in verity_fec_dtr()
587 v->fec = NULL; in verity_fec_dtr()
592 struct dm_verity *v = (struct dm_verity *)pool_data; in fec_rs_alloc() local
594 return init_rs(8, 0x11d, 0, 1, v->fec->roots); in fec_rs_alloc()
613 int verity_fec_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v, in verity_fec_parse_opt_args() argument
617 struct dm_target *ti = v->ti; in verity_fec_parse_opt_args()
632 r = dm_get_device(ti, arg_value, FMODE_READ, &v->fec->dev); in verity_fec_parse_opt_args()
640 ((sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) in verity_fec_parse_opt_args()
641 >> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll)) { in verity_fec_parse_opt_args()
645 v->fec->blocks = num_ll; in verity_fec_parse_opt_args()
649 ((sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) >> in verity_fec_parse_opt_args()
650 (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll)) { in verity_fec_parse_opt_args()
654 v->fec->start = num_ll; in verity_fec_parse_opt_args()
663 v->fec->roots = num_c; in verity_fec_parse_opt_args()
698 int verity_fec_ctr_alloc(struct dm_verity *v) in verity_fec_ctr_alloc() argument
704 v->ti->error = "Cannot allocate FEC structure"; in verity_fec_ctr_alloc()
707 v->fec = f; in verity_fec_ctr_alloc()
716 int verity_fec_ctr(struct dm_verity *v) in verity_fec_ctr() argument
719 struct dm_verity_fec *f = v->fec; in verity_fec_ctr()
720 struct dm_target *ti = v->ti; in verity_fec_ctr()
724 if (!verity_fec_is_enabled(v)) { in verity_fec_ctr()
725 verity_fec_dtr(v); in verity_fec_ctr()
754 hash_blocks = v->hash_blocks - v->hash_start; in verity_fec_ctr()
760 if (v->data_dev_block_bits != v->hash_dev_block_bits) { in verity_fec_ctr()
784 if (f->blocks < v->data_blocks + hash_blocks || !f->rounds) { in verity_fec_ctr()
793 f->hash_blocks = f->blocks - v->data_blocks; in verity_fec_ctr()
794 if (dm_bufio_get_device_size(v->bufio) < f->hash_blocks) { in verity_fec_ctr()
801 1 << v->data_dev_block_bits, in verity_fec_ctr()
809 ((f->start + f->rounds * f->roots) >> v->data_dev_block_bits)) { in verity_fec_ctr()
814 f->data_bufio = dm_bufio_client_create(v->data_dev->bdev, in verity_fec_ctr()
815 1 << v->data_dev_block_bits, in verity_fec_ctr()
822 if (dm_bufio_get_device_size(f->data_bufio) < v->data_blocks) { in verity_fec_ctr()
829 fec_rs_free, (void *) v); in verity_fec_ctr()
860 1 << v->data_dev_block_bits); in verity_fec_ctr()