1 // SPDX-License-Identifier: GPL-2.0
3  * Data verification functions, i.e. hooks for ->readahead()
19 static bool is_hash_block_verified(struct fsverity_info *vi, struct page *hpage,  in is_hash_block_verified()
27 	 * When the Merkle tree block size and page size are the same, then the  in is_hash_block_verified()
28 	 * ->hash_block_verified bitmap isn't allocated, and we use PG_checked  in is_hash_block_verified()
29 	 * to directly indicate whether the page's block has been verified.  in is_hash_block_verified()
31 	 * Using PG_checked also guarantees that we re-verify hash pages that  in is_hash_block_verified()
32 	 * get evicted and re-instantiated from the backing storage, as new  in is_hash_block_verified()
35 	if (!vi->hash_block_verified)  in is_hash_block_verified()
39 	 * When the Merkle tree block size and page size differ, we use a bitmap  in is_hash_block_verified()
43 	 * re-instantiated from the backing storage are re-verified.  To do  in is_hash_block_verified()
46 	 * the hash page is newly instantiated or not.  in is_hash_block_verified()
59 		return test_bit(hblock_idx, vi->hash_block_verified);  in is_hash_block_verified()
61 	spin_lock(&vi->hash_page_init_lock);  in is_hash_block_verified()
63 		verified = test_bit(hblock_idx, vi->hash_block_verified);  in is_hash_block_verified()
65 		blocks_per_page = vi->tree_params.blocks_per_page;  in is_hash_block_verified()
68 			clear_bit(hblock_idx + i, vi->hash_block_verified);  in is_hash_block_verified()
77 	spin_unlock(&vi->hash_page_init_lock);  in is_hash_block_verified()
86  * only ascend the tree until an already-verified hash block is seen, and then
95 	const struct merkle_tree_params *params = &vi->tree_params;  in verify_data_block()
96 	const unsigned int hsize = params->digest_size;  in verify_data_block()
97 	int level;  in verify_data_block()  local
101 	/* The hash blocks that are traversed, indexed by level */  in verify_data_block()
103 		/* Page containing the hash block */  in verify_data_block()
104 		struct page *page;  in verify_data_block()  member
105 		/* Mapped address of the hash block (will be within @page) */  in verify_data_block()
113 	 * The index of the previous level's block within that level; also the  in verify_data_block()
114 	 * index of that block's hash within the current level.  in verify_data_block()
116 	u64 hidx = data_pos >> params->log_blocksize;  in verify_data_block()
121 	if (unlikely(data_pos >= inode->i_size)) {  in verify_data_block()
123 		 * This can happen in the data page spanning EOF when the Merkle  in verify_data_block()
124 		 * tree block size is less than the page size.  The Merkle tree  in verify_data_block()
126 		 * page spanning EOF can be visible to userspace via a mmap, and  in verify_data_block()
130 		if (memchr_inv(data, 0, params->block_size)) {  in verify_data_block()
139 	if (data_pos >= vi->verified_data_size) {  in verify_data_block()
141 			data_pos, vi->verified_data_size);  in verify_data_block()
146 	 * Starting at the leaf level, ascend the tree saving hash blocks along  in verify_data_block()
150 	for (level = 0; level < params->num_levels; level++) {  in verify_data_block()
156 		struct page *hpage;  in verify_data_block()
160 		 * The index of the block in the current level; also the index  in verify_data_block()
161 		 * of that block's hash within the next level.  in verify_data_block()
163 		next_hidx = hidx >> params->log_arity;  in verify_data_block()
166 		hblock_idx = params->level_start[level] + next_hidx;  in verify_data_block()
168 		/* Index of the hash page in the tree overall */  in verify_data_block()
169 		hpage_idx = hblock_idx >> params->log_blocks_per_page;  in verify_data_block()
171 		/* Byte offset of the hash block within the page */  in verify_data_block()
173 			(hblock_idx << params->log_blocksize) & ~PAGE_MASK;  in verify_data_block()
176 		hoffset = (hidx << params->log_digestsize) &  in verify_data_block()
177 			  (params->block_size - 1);  in verify_data_block()
179 		hpage = inode->i_sb->s_vop->read_merkle_tree_page(inode,  in verify_data_block()
180 				hpage_idx, level == 0 ? min(max_ra_pages,  in verify_data_block()
181 					params->tree_pages - hpage_idx) : 0);  in verify_data_block()
184 				     "Error %ld reading Merkle tree page %lu",  in verify_data_block()
196 		hblocks[level].page = hpage;  in verify_data_block()
197 		hblocks[level].addr = haddr;  in verify_data_block()
198 		hblocks[level].index = hblock_idx;  in verify_data_block()
199 		hblocks[level].hoffset = hoffset;  in verify_data_block()
203 	want_hash = vi->root_hash;  in verify_data_block()
206 	for (; level > 0; level--) {  in verify_data_block()
207 		struct page *hpage = hblocks[level - 1].page;  in verify_data_block()
208 		const void *haddr = hblocks[level - 1].addr;  in verify_data_block()
209 		unsigned long hblock_idx = hblocks[level - 1].index;  in verify_data_block()
210 		unsigned int hoffset = hblocks[level - 1].hoffset;  in verify_data_block()
221 		if (vi->hash_block_verified)  in verify_data_block()
222 			set_bit(hblock_idx, vi->hash_block_verified);  in verify_data_block()
240 		     "FILE CORRUPTED! pos=%llu, level=%d, want_hash=%s:%*phN, real_hash=%s:%*phN",  in verify_data_block()
241 		     data_pos, level - 1,  in verify_data_block()
242 		     params->hash_alg->name, hsize, want_hash,  in verify_data_block()
243 		     params->hash_alg->name, hsize, real_hash);  in verify_data_block()
245 	for (; level > 0; level--) {  in verify_data_block()
246 		kunmap_local(hblocks[level - 1].addr);  in verify_data_block()
247 		put_page(hblocks[level - 1].page);  in verify_data_block()
256 	struct inode *inode = data_folio->mapping->host;  in verify_data_blocks()
257 	struct fsverity_info *vi = inode->i_verity_info;  in verify_data_blocks()
258 	const unsigned int block_size = vi->tree_params.block_size;  in verify_data_blocks()
259 	u64 pos = (u64)data_folio->index << PAGE_SHIFT;  in verify_data_blocks()
277 		len -= block_size;  in verify_data_blocks()
283  * fsverity_verify_blocks() - verify data in a folio
302  * fsverity_verify_bio() - verify a 'read' bio that has just completed
307  * verification, then bio->bi_status is set to an error status.
309  * This is a helper function for use by the ->readahead() method of filesystems
310  * that issue bios to read data directly into the page cache.  Filesystems that
311  * populate the page cache without issuing bios (e.g. non block-based
312  * filesystems) must instead call fsverity_verify_page() directly on each page.
320 	if (bio->bi_opf & REQ_RAHEAD) {  in fsverity_verify_bio()
323 		 * of the first (largest) level of the Merkle tree.  Namely,  in fsverity_verify_bio()
324 		 * when a Merkle tree page is read, we also try to piggy-back on  in fsverity_verify_bio()
325 		 * some additional pages -- up to 1/4 the number of data pages.  in fsverity_verify_bio()
330 		max_ra_pages = bio->bi_iter.bi_size >> (PAGE_SHIFT + 2);  in fsverity_verify_bio()
336 			bio->bi_status = BLK_STS_IOERR;  in fsverity_verify_bio()
345  * fsverity_get_verified_data_size() - get verified data size of a verity file
353 	return fsverity_get_info(inode)->verified_data_size;  in fsverity_get_verified_data_size()
355 	return inode->i_size;  in fsverity_get_verified_data_size()
360  * fsverity_enqueue_verify_work() - enqueue work on the fs-verity workqueue
374 	 * Use a high-priority workqueue to prioritize verification work, which  in fsverity_init_workqueue()