Lines Matching refs:v
16 bool verity_fec_is_enabled(struct dm_verity *v) in verity_fec_is_enabled() argument
18 return v->fec && v->fec->dev; in verity_fec_is_enabled()
28 ((char *)io + io->v->ti->per_io_data_size - sizeof(struct dm_verity_fec_io)); in fec_io()
34 static inline u64 fec_interleave(struct dm_verity *v, u64 offset) in fec_interleave() argument
38 mod = do_div(offset, v->fec->rsn); in fec_interleave()
39 return offset + mod * (v->fec->rounds << v->data_dev_block_bits); in fec_interleave()
45 static int fec_decode_rs8(struct dm_verity *v, struct dm_verity_fec_io *fio, in fec_decode_rs8() argument
51 for (i = 0; i < v->fec->roots; i++) in fec_decode_rs8()
54 return decode_rs8(fio->rs, data, par, v->fec->rsn, NULL, neras, in fec_decode_rs8()
62 static u8 *fec_read_parity(struct dm_verity *v, u64 rsb, int index, in fec_read_parity() argument
68 position = (index + rsb) * v->fec->roots; in fec_read_parity()
69 block = div64_u64_rem(position, v->fec->io_size, &rem); in fec_read_parity()
72 res = dm_bufio_read(v->fec->bufio, block, buf); in fec_read_parity()
75 v->data_dev->name, (unsigned long long)rsb, in fec_read_parity()
104 static inline u8 *fec_buffer_rs_block(struct dm_verity *v, in fec_buffer_rs_block() argument
108 return &fio->bufs[i][j * v->fec->rsn]; in fec_buffer_rs_block()
124 static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio, in fec_decode_bufs() argument
133 par = fec_read_parity(v, rsb, block_offset, &offset, &buf); in fec_decode_bufs()
142 block = fec_buffer_rs_block(v, fio, n, i); in fec_decode_bufs()
143 res = fec_decode_rs8(v, fio, block, &par[offset], neras); in fec_decode_bufs()
153 if (block_offset >= 1 << v->data_dev_block_bits) in fec_decode_bufs()
157 offset += v->fec->roots; in fec_decode_bufs()
158 if (offset >= v->fec->io_size) { in fec_decode_bufs()
161 par = fec_read_parity(v, rsb, block_offset, &offset, &buf); in fec_decode_bufs()
173 v->data_dev->name, (unsigned long long)rsb, r); in fec_decode_bufs()
176 v->data_dev->name, (unsigned long long)rsb, r); in fec_decode_bufs()
184 static int fec_is_erasure(struct dm_verity *v, struct dm_verity_io *io, in fec_is_erasure() argument
187 if (unlikely(verity_hash(v, verity_io_hash_req(v, io), in fec_is_erasure()
188 data, 1 << v->data_dev_block_bits, in fec_is_erasure()
189 verity_io_real_digest(v, io), true))) in fec_is_erasure()
192 return memcmp(verity_io_real_digest(v, io), want_digest, in fec_is_erasure()
193 v->digest_size) != 0; in fec_is_erasure()
200 static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io, in fec_read_bufs() argument
217 if (WARN_ON(v->digest_size > sizeof(want_digest))) in fec_read_bufs()
224 for (i = 0; i < v->fec->rsn; i++) { in fec_read_bufs()
225 ileaved = fec_interleave(v, rsb * v->fec->rsn + i); in fec_read_bufs()
234 block = ileaved >> v->data_dev_block_bits; in fec_read_bufs()
235 bufio = v->fec->data_bufio; in fec_read_bufs()
237 if (block >= v->data_blocks) { in fec_read_bufs()
238 block -= v->data_blocks; in fec_read_bufs()
244 if (unlikely(block >= v->fec->hash_blocks)) in fec_read_bufs()
247 block += v->hash_start; in fec_read_bufs()
248 bufio = v->bufio; in fec_read_bufs()
254 v->data_dev->name, in fec_read_bufs()
259 if (neras && *neras <= v->fec->roots) in fec_read_bufs()
266 if (bufio == v->fec->data_bufio && in fec_read_bufs()
267 verity_hash_for_block(v, io, block, want_digest, in fec_read_bufs()
277 if (neras && *neras <= v->fec->roots && in fec_read_bufs()
278 fec_is_erasure(v, io, want_digest, bbuf)) in fec_read_bufs()
289 if (k >= 1 << v->data_dev_block_bits) in fec_read_bufs()
292 rs_block = fec_buffer_rs_block(v, fio, n, j); in fec_read_bufs()
306 static int fec_alloc_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio) in fec_alloc_bufs() argument
311 fio->rs = mempool_alloc(&v->fec->rs_pool, GFP_NOIO); in fec_alloc_bufs()
317 fio->bufs[n] = mempool_alloc(&v->fec->prealloc_pool, GFP_NOWAIT); in fec_alloc_bufs()
329 fio->bufs[n] = mempool_alloc(&v->fec->extra_pool, GFP_NOWAIT); in fec_alloc_bufs()
337 fio->output = mempool_alloc(&v->fec->output_pool, GFP_NOIO); in fec_alloc_bufs()
346 static void fec_init_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio) in fec_init_bufs() argument
351 memset(fio->bufs[n], 0, v->fec->rsn << DM_VERITY_FEC_BUF_RS_BITS); in fec_init_bufs()
361 static int fec_decode_rsb(struct dm_verity *v, struct dm_verity_io *io, in fec_decode_rsb() argument
368 r = fec_alloc_bufs(v, fio); in fec_decode_rsb()
372 for (pos = 0; pos < 1 << v->data_dev_block_bits; ) { in fec_decode_rsb()
373 fec_init_bufs(v, fio); in fec_decode_rsb()
375 r = fec_read_bufs(v, io, rsb, offset, pos, in fec_decode_rsb()
380 r = fec_decode_bufs(v, fio, rsb, r, pos, neras); in fec_decode_rsb()
388 r = verity_hash(v, verity_io_hash_req(v, io), fio->output, in fec_decode_rsb()
389 1 << v->data_dev_block_bits, in fec_decode_rsb()
390 verity_io_real_digest(v, io), true); in fec_decode_rsb()
394 if (memcmp(verity_io_real_digest(v, io), verity_io_want_digest(v, io), in fec_decode_rsb()
395 v->digest_size)) { in fec_decode_rsb()
397 v->data_dev->name, (unsigned long long)rsb, neras); in fec_decode_rsb()
404 static int fec_bv_copy(struct dm_verity *v, struct dm_verity_io *io, u8 *data, in fec_bv_copy() argument
419 int verity_fec_decode(struct dm_verity *v, struct dm_verity_io *io, in verity_fec_decode() argument
427 if (!verity_fec_is_enabled(v)) in verity_fec_decode()
431 DMWARN_LIMIT("%s: FEC: recursion too deep", v->data_dev->name); in verity_fec_decode()
438 block = block - v->hash_start + v->data_blocks; in verity_fec_decode()
450 offset = block << v->data_dev_block_bits; in verity_fec_decode()
451 res = div64_u64(offset, v->fec->rounds << v->data_dev_block_bits); in verity_fec_decode()
457 rsb = offset - res * (v->fec->rounds << v->data_dev_block_bits); in verity_fec_decode()
464 r = fec_decode_rsb(v, io, fio, rsb, offset, false); in verity_fec_decode()
466 r = fec_decode_rsb(v, io, fio, rsb, offset, true); in verity_fec_decode()
472 memcpy(dest, fio->output, 1 << v->data_dev_block_bits); in verity_fec_decode()
475 r = verity_for_bv_block(v, io, iter, fec_bv_copy); in verity_fec_decode()
489 struct dm_verity_fec *f = io->v->fec; in verity_fec_finish_io()
492 if (!verity_fec_is_enabled(io->v)) in verity_fec_finish_io()
513 if (!verity_fec_is_enabled(io->v)) in verity_fec_init_io()
526 unsigned int verity_fec_status_table(struct dm_verity *v, unsigned int sz, in verity_fec_status_table() argument
529 if (!verity_fec_is_enabled(v)) in verity_fec_status_table()
536 v->fec->dev->name, in verity_fec_status_table()
537 (unsigned long long)v->fec->blocks, in verity_fec_status_table()
538 (unsigned long long)v->fec->start, in verity_fec_status_table()
539 v->fec->roots); in verity_fec_status_table()
544 void verity_fec_dtr(struct dm_verity *v) in verity_fec_dtr() argument
546 struct dm_verity_fec *f = v->fec; in verity_fec_dtr()
548 if (!verity_fec_is_enabled(v)) in verity_fec_dtr()
563 dm_put_device(v->ti, f->dev); in verity_fec_dtr()
566 v->fec = NULL; in verity_fec_dtr()
571 struct dm_verity *v = (struct dm_verity *)pool_data; in fec_rs_alloc() local
573 return init_rs_gfp(8, 0x11d, 0, 1, v->fec->roots, gfp_mask); in fec_rs_alloc()
592 int verity_fec_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v, in verity_fec_parse_opt_args() argument
596 struct dm_target *ti = v->ti; in verity_fec_parse_opt_args()
611 r = dm_get_device(ti, arg_value, FMODE_READ, &v->fec->dev); in verity_fec_parse_opt_args()
619 ((sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) in verity_fec_parse_opt_args()
620 >> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll)) { in verity_fec_parse_opt_args()
624 v->fec->blocks = num_ll; in verity_fec_parse_opt_args()
628 ((sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) >> in verity_fec_parse_opt_args()
629 (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll)) { in verity_fec_parse_opt_args()
633 v->fec->start = num_ll; in verity_fec_parse_opt_args()
642 v->fec->roots = num_c; in verity_fec_parse_opt_args()
655 int verity_fec_ctr_alloc(struct dm_verity *v) in verity_fec_ctr_alloc() argument
661 v->ti->error = "Cannot allocate FEC structure"; in verity_fec_ctr_alloc()
664 v->fec = f; in verity_fec_ctr_alloc()
673 int verity_fec_ctr(struct dm_verity *v) in verity_fec_ctr() argument
675 struct dm_verity_fec *f = v->fec; in verity_fec_ctr()
676 struct dm_target *ti = v->ti; in verity_fec_ctr()
680 if (!verity_fec_is_enabled(v)) { in verity_fec_ctr()
681 verity_fec_dtr(v); in verity_fec_ctr()
700 hash_blocks = v->hash_blocks - v->hash_start; in verity_fec_ctr()
706 if (v->data_dev_block_bits != v->hash_dev_block_bits) { in verity_fec_ctr()
730 if (f->blocks < v->data_blocks + hash_blocks || !f->rounds) { in verity_fec_ctr()
739 f->hash_blocks = f->blocks - v->data_blocks; in verity_fec_ctr()
740 if (dm_bufio_get_device_size(v->bufio) < f->hash_blocks) { in verity_fec_ctr()
746 if ((f->roots << SECTOR_SHIFT) & ((1 << v->data_dev_block_bits) - 1)) in verity_fec_ctr()
747 f->io_size = 1 << v->data_dev_block_bits; in verity_fec_ctr()
749 f->io_size = v->fec->roots << SECTOR_SHIFT; in verity_fec_ctr()
759 dm_bufio_set_sector_offset(f->bufio, f->start << (v->data_dev_block_bits - SECTOR_SHIFT)); in verity_fec_ctr()
761 fec_blocks = div64_u64(f->rounds * f->roots, v->fec->roots << SECTOR_SHIFT); in verity_fec_ctr()
767 f->data_bufio = dm_bufio_client_create(v->data_dev->bdev, in verity_fec_ctr()
768 1 << v->data_dev_block_bits, in verity_fec_ctr()
775 if (dm_bufio_get_device_size(f->data_bufio) < v->data_blocks) { in verity_fec_ctr()
782 fec_rs_free, (void *) v); in verity_fec_ctr()
813 1 << v->data_dev_block_bits); in verity_fec_ctr()