1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright 2019 Google LLC
4 */
5 #include <linux/crc32.h>
6 #include <linux/file.h>
7 #include <linux/fsverity.h>
8 #include <linux/gfp.h>
9 #include <linux/kobject.h>
10 #include <linux/ktime.h>
11 #include <linux/lz4.h>
12 #include <linux/mm.h>
13 #include <linux/namei.h>
14 #include <linux/pagemap.h>
15 #include <linux/slab.h>
16 #include <linux/types.h>
17 #include <linux/workqueue.h>
18
19 #include "data_mgmt.h"
20 #include "format.h"
21 #include "integrity.h"
22 #include "sysfs.h"
23 #include "verity.h"
24
25 static int incfs_scan_metadata_chain(struct data_file *df);
26
log_wake_up_all(struct work_struct * work)27 static void log_wake_up_all(struct work_struct *work)
28 {
29 struct delayed_work *dw = container_of(work, struct delayed_work, work);
30 struct read_log *rl = container_of(dw, struct read_log, ml_wakeup_work);
31 wake_up_all(&rl->ml_notif_wq);
32 }
33
zstd_free_workspace(struct work_struct * work)34 static void zstd_free_workspace(struct work_struct *work)
35 {
36 struct delayed_work *dw = container_of(work, struct delayed_work, work);
37 struct mount_info *mi =
38 container_of(dw, struct mount_info, mi_zstd_cleanup_work);
39
40 mutex_lock(&mi->mi_zstd_workspace_mutex);
41 kvfree(mi->mi_zstd_workspace);
42 mi->mi_zstd_workspace = NULL;
43 mi->mi_zstd_stream = NULL;
44 mutex_unlock(&mi->mi_zstd_workspace_mutex);
45 }
46
incfs_alloc_mount_info(struct super_block * sb,struct mount_options * options,struct path * backing_dir_path)47 struct mount_info *incfs_alloc_mount_info(struct super_block *sb,
48 struct mount_options *options,
49 struct path *backing_dir_path)
50 {
51 struct mount_info *mi = NULL;
52 int error = 0;
53 struct incfs_sysfs_node *node;
54
55 mi = kzalloc(sizeof(*mi), GFP_NOFS);
56 if (!mi)
57 return ERR_PTR(-ENOMEM);
58
59 mi->mi_sb = sb;
60 mi->mi_backing_dir_path = *backing_dir_path;
61 mi->mi_owner = get_current_cred();
62 path_get(&mi->mi_backing_dir_path);
63 mutex_init(&mi->mi_dir_struct_mutex);
64 init_waitqueue_head(&mi->mi_pending_reads_notif_wq);
65 init_waitqueue_head(&mi->mi_log.ml_notif_wq);
66 init_waitqueue_head(&mi->mi_blocks_written_notif_wq);
67 atomic_set(&mi->mi_blocks_written, 0);
68 INIT_DELAYED_WORK(&mi->mi_log.ml_wakeup_work, log_wake_up_all);
69 spin_lock_init(&mi->mi_log.rl_lock);
70 spin_lock_init(&mi->pending_read_lock);
71 INIT_LIST_HEAD(&mi->mi_reads_list_head);
72 spin_lock_init(&mi->mi_per_uid_read_timeouts_lock);
73 mutex_init(&mi->mi_zstd_workspace_mutex);
74 INIT_DELAYED_WORK(&mi->mi_zstd_cleanup_work, zstd_free_workspace);
75 mutex_init(&mi->mi_le_mutex);
76
77 node = incfs_add_sysfs_node(options->sysfs_name, mi);
78 if (IS_ERR(node)) {
79 error = PTR_ERR(node);
80 goto err;
81 }
82 mi->mi_sysfs_node = node;
83
84 error = incfs_realloc_mount_info(mi, options);
85 if (error)
86 goto err;
87
88 return mi;
89
90 err:
91 incfs_free_mount_info(mi);
92 return ERR_PTR(error);
93 }
94
incfs_realloc_mount_info(struct mount_info * mi,struct mount_options * options)95 int incfs_realloc_mount_info(struct mount_info *mi,
96 struct mount_options *options)
97 {
98 void *new_buffer = NULL;
99 void *old_buffer;
100 size_t new_buffer_size = 0;
101
102 if (options->read_log_pages != mi->mi_options.read_log_pages) {
103 struct read_log_state log_state;
104 /*
105 * Even though having two buffers allocated at once isn't
106 * usually good, allocating a multipage buffer under a spinlock
107 * is even worse, so let's optimize for the shorter lock
108 * duration. It's not end of the world if we fail to increase
109 * the buffer size anyway.
110 */
111 if (options->read_log_pages > 0) {
112 new_buffer_size = PAGE_SIZE * options->read_log_pages;
113 new_buffer = kzalloc(new_buffer_size, GFP_NOFS);
114 if (!new_buffer)
115 return -ENOMEM;
116 }
117
118 spin_lock(&mi->mi_log.rl_lock);
119 old_buffer = mi->mi_log.rl_ring_buf;
120 mi->mi_log.rl_ring_buf = new_buffer;
121 mi->mi_log.rl_size = new_buffer_size;
122 log_state = (struct read_log_state){
123 .generation_id = mi->mi_log.rl_head.generation_id + 1,
124 };
125 mi->mi_log.rl_head = log_state;
126 mi->mi_log.rl_tail = log_state;
127 spin_unlock(&mi->mi_log.rl_lock);
128
129 kfree(old_buffer);
130 }
131
132 if (options->sysfs_name && !mi->mi_sysfs_node)
133 mi->mi_sysfs_node = incfs_add_sysfs_node(options->sysfs_name,
134 mi);
135 else if (!options->sysfs_name && mi->mi_sysfs_node) {
136 incfs_free_sysfs_node(mi->mi_sysfs_node);
137 mi->mi_sysfs_node = NULL;
138 } else if (options->sysfs_name &&
139 strcmp(options->sysfs_name,
140 kobject_name(&mi->mi_sysfs_node->isn_sysfs_node))) {
141 incfs_free_sysfs_node(mi->mi_sysfs_node);
142 mi->mi_sysfs_node = incfs_add_sysfs_node(options->sysfs_name,
143 mi);
144 }
145
146 if (IS_ERR(mi->mi_sysfs_node)) {
147 int err = PTR_ERR(mi->mi_sysfs_node);
148
149 mi->mi_sysfs_node = NULL;
150 return err;
151 }
152
153 mi->mi_options = *options;
154 return 0;
155 }
156
incfs_free_mount_info(struct mount_info * mi)157 void incfs_free_mount_info(struct mount_info *mi)
158 {
159 int i;
160 if (!mi)
161 return;
162
163 flush_delayed_work(&mi->mi_log.ml_wakeup_work);
164 flush_delayed_work(&mi->mi_zstd_cleanup_work);
165
166 dput(mi->mi_index_dir);
167 dput(mi->mi_incomplete_dir);
168 path_put(&mi->mi_backing_dir_path);
169 mutex_destroy(&mi->mi_dir_struct_mutex);
170 mutex_destroy(&mi->mi_zstd_workspace_mutex);
171 put_cred(mi->mi_owner);
172 kfree(mi->mi_log.rl_ring_buf);
173 for (i = 0; i < ARRAY_SIZE(mi->pseudo_file_xattr); ++i)
174 kfree(mi->pseudo_file_xattr[i].data);
175 kfree(mi->mi_per_uid_read_timeouts);
176 incfs_free_sysfs_node(mi->mi_sysfs_node);
177 kfree(mi);
178 }
179
data_file_segment_init(struct data_file_segment * segment)180 static void data_file_segment_init(struct data_file_segment *segment)
181 {
182 init_waitqueue_head(&segment->new_data_arrival_wq);
183 init_rwsem(&segment->rwsem);
184 INIT_LIST_HEAD(&segment->reads_list_head);
185 }
186
file_id_to_str(incfs_uuid_t id)187 char *file_id_to_str(incfs_uuid_t id)
188 {
189 char *result = kmalloc(1 + sizeof(id.bytes) * 2, GFP_NOFS);
190 char *end;
191
192 if (!result)
193 return NULL;
194
195 end = bin2hex(result, id.bytes, sizeof(id.bytes));
196 *end = 0;
197 return result;
198 }
199
incfs_lookup_dentry(struct dentry * parent,const char * name)200 struct dentry *incfs_lookup_dentry(struct dentry *parent, const char *name)
201 {
202 struct inode *inode;
203 struct dentry *result = NULL;
204
205 if (!parent)
206 return ERR_PTR(-EFAULT);
207
208 inode = d_inode(parent);
209 inode_lock_nested(inode, I_MUTEX_PARENT);
210 result = lookup_one_len(name, parent, strlen(name));
211 inode_unlock(inode);
212
213 if (IS_ERR(result))
214 pr_warn("%s err:%ld\n", __func__, PTR_ERR(result));
215
216 return result;
217 }
218
handle_mapped_file(struct mount_info * mi,struct data_file * df)219 static struct data_file *handle_mapped_file(struct mount_info *mi,
220 struct data_file *df)
221 {
222 char *file_id_str;
223 struct dentry *index_file_dentry;
224 struct path path;
225 struct file *bf;
226 struct data_file *result = NULL;
227 const struct cred *old_cred;
228
229 file_id_str = file_id_to_str(df->df_id);
230 if (!file_id_str)
231 return ERR_PTR(-ENOENT);
232
233 index_file_dentry = incfs_lookup_dentry(mi->mi_index_dir,
234 file_id_str);
235 kfree(file_id_str);
236 if (!index_file_dentry)
237 return ERR_PTR(-ENOENT);
238 if (IS_ERR(index_file_dentry))
239 return (struct data_file *)index_file_dentry;
240 if (!d_really_is_positive(index_file_dentry)) {
241 result = ERR_PTR(-ENOENT);
242 goto out;
243 }
244
245 path = (struct path) {
246 .mnt = mi->mi_backing_dir_path.mnt,
247 .dentry = index_file_dentry
248 };
249
250 old_cred = override_creds(mi->mi_owner);
251 bf = dentry_open(&path, O_RDWR | O_NOATIME | O_LARGEFILE,
252 current_cred());
253 revert_creds(old_cred);
254
255 if (IS_ERR(bf)) {
256 result = (struct data_file *)bf;
257 goto out;
258 }
259
260 result = incfs_open_data_file(mi, bf);
261 fput(bf);
262 if (IS_ERR(result))
263 goto out;
264
265 result->df_mapped_offset = df->df_metadata_off;
266
267 out:
268 dput(index_file_dentry);
269 return result;
270 }
271
incfs_open_data_file(struct mount_info * mi,struct file * bf)272 struct data_file *incfs_open_data_file(struct mount_info *mi, struct file *bf)
273 {
274 struct data_file *df = NULL;
275 struct backing_file_context *bfc = NULL;
276 int md_records;
277 u64 size;
278 int error = 0;
279 int i;
280
281 if (!bf || !mi)
282 return ERR_PTR(-EFAULT);
283
284 if (!S_ISREG(bf->f_inode->i_mode))
285 return ERR_PTR(-EBADF);
286
287 bfc = incfs_alloc_bfc(mi, bf);
288 if (IS_ERR(bfc))
289 return ERR_CAST(bfc);
290
291 df = kzalloc(sizeof(*df), GFP_NOFS);
292 if (!df) {
293 error = -ENOMEM;
294 goto out;
295 }
296
297 mutex_init(&df->df_enable_verity);
298
299 df->df_backing_file_context = bfc;
300 df->df_mount_info = mi;
301 for (i = 0; i < ARRAY_SIZE(df->df_segments); i++)
302 data_file_segment_init(&df->df_segments[i]);
303
304 error = incfs_read_file_header(bfc, &df->df_metadata_off, &df->df_id,
305 &size, &df->df_header_flags);
306
307 if (error)
308 goto out;
309
310 df->df_size = size;
311 if (size > 0)
312 df->df_data_block_count = get_blocks_count_for_size(size);
313
314 if (df->df_header_flags & INCFS_FILE_MAPPED) {
315 struct data_file *mapped_df = handle_mapped_file(mi, df);
316
317 incfs_free_data_file(df);
318 return mapped_df;
319 }
320
321 md_records = incfs_scan_metadata_chain(df);
322 if (md_records < 0)
323 error = md_records;
324
325 out:
326 if (error) {
327 incfs_free_bfc(bfc);
328 if (df)
329 df->df_backing_file_context = NULL;
330 incfs_free_data_file(df);
331 return ERR_PTR(error);
332 }
333 return df;
334 }
335
incfs_free_data_file(struct data_file * df)336 void incfs_free_data_file(struct data_file *df)
337 {
338 u32 data_blocks_written, hash_blocks_written;
339
340 if (!df)
341 return;
342
343 data_blocks_written = atomic_read(&df->df_data_blocks_written);
344 hash_blocks_written = atomic_read(&df->df_hash_blocks_written);
345
346 if (data_blocks_written != df->df_initial_data_blocks_written ||
347 hash_blocks_written != df->df_initial_hash_blocks_written) {
348 struct backing_file_context *bfc = df->df_backing_file_context;
349 int error = -1;
350
351 if (bfc && !mutex_lock_interruptible(&bfc->bc_mutex)) {
352 error = incfs_write_status_to_backing_file(
353 df->df_backing_file_context,
354 df->df_status_offset,
355 data_blocks_written,
356 hash_blocks_written);
357 mutex_unlock(&bfc->bc_mutex);
358 }
359
360 if (error)
361 /* Nothing can be done, just warn */
362 pr_warn("incfs: failed to write status to backing file\n");
363 }
364
365 incfs_free_mtree(df->df_hash_tree);
366 incfs_free_bfc(df->df_backing_file_context);
367 kfree(df->df_signature);
368 kfree(df->df_verity_file_digest.data);
369 kfree(df->df_verity_signature);
370 mutex_destroy(&df->df_enable_verity);
371 kfree(df);
372 }
373
make_inode_ready_for_data_ops(struct mount_info * mi,struct inode * inode,struct file * backing_file)374 int make_inode_ready_for_data_ops(struct mount_info *mi,
375 struct inode *inode,
376 struct file *backing_file)
377 {
378 struct inode_info *node = get_incfs_node(inode);
379 struct data_file *df = NULL;
380 int err = 0;
381
382 inode_lock(inode);
383 if (S_ISREG(inode->i_mode)) {
384 if (!node->n_file) {
385 df = incfs_open_data_file(mi, backing_file);
386
387 if (IS_ERR(df))
388 err = PTR_ERR(df);
389 else
390 node->n_file = df;
391 }
392 } else
393 err = -EBADF;
394 inode_unlock(inode);
395 return err;
396 }
397
incfs_open_dir_file(struct mount_info * mi,struct file * bf)398 struct dir_file *incfs_open_dir_file(struct mount_info *mi, struct file *bf)
399 {
400 struct dir_file *dir = NULL;
401
402 if (!S_ISDIR(bf->f_inode->i_mode))
403 return ERR_PTR(-EBADF);
404
405 dir = kzalloc(sizeof(*dir), GFP_NOFS);
406 if (!dir)
407 return ERR_PTR(-ENOMEM);
408
409 dir->backing_dir = get_file(bf);
410 dir->mount_info = mi;
411 return dir;
412 }
413
incfs_free_dir_file(struct dir_file * dir)414 void incfs_free_dir_file(struct dir_file *dir)
415 {
416 if (!dir)
417 return;
418 if (dir->backing_dir)
419 fput(dir->backing_dir);
420 kfree(dir);
421 }
422
zstd_decompress_safe(struct mount_info * mi,struct mem_range src,struct mem_range dst)423 static ssize_t zstd_decompress_safe(struct mount_info *mi,
424 struct mem_range src, struct mem_range dst)
425 {
426 ssize_t result;
427 ZSTD_inBuffer inbuf = {.src = src.data, .size = src.len};
428 ZSTD_outBuffer outbuf = {.dst = dst.data, .size = dst.len};
429
430 result = mutex_lock_interruptible(&mi->mi_zstd_workspace_mutex);
431 if (result)
432 return result;
433
434 if (!mi->mi_zstd_stream) {
435 unsigned int workspace_size = ZSTD_DStreamWorkspaceBound(
436 INCFS_DATA_FILE_BLOCK_SIZE);
437 void *workspace = kvmalloc(workspace_size, GFP_NOFS);
438 ZSTD_DStream *stream;
439
440 if (!workspace) {
441 result = -ENOMEM;
442 goto out;
443 }
444
445 stream = ZSTD_initDStream(INCFS_DATA_FILE_BLOCK_SIZE, workspace,
446 workspace_size);
447 if (!stream) {
448 kvfree(workspace);
449 result = -EIO;
450 goto out;
451 }
452
453 mi->mi_zstd_workspace = workspace;
454 mi->mi_zstd_stream = stream;
455 }
456
457 result = ZSTD_decompressStream(mi->mi_zstd_stream, &outbuf, &inbuf) ?
458 -EBADMSG : outbuf.pos;
459
460 mod_delayed_work(system_wq, &mi->mi_zstd_cleanup_work,
461 msecs_to_jiffies(5000));
462
463 out:
464 mutex_unlock(&mi->mi_zstd_workspace_mutex);
465 return result;
466 }
467
decompress(struct mount_info * mi,struct mem_range src,struct mem_range dst,int alg)468 static ssize_t decompress(struct mount_info *mi,
469 struct mem_range src, struct mem_range dst, int alg)
470 {
471 int result;
472
473 switch (alg) {
474 case INCFS_BLOCK_COMPRESSED_LZ4:
475 result = LZ4_decompress_safe(src.data, dst.data, src.len,
476 dst.len);
477 if (result < 0)
478 return -EBADMSG;
479 return result;
480
481 case INCFS_BLOCK_COMPRESSED_ZSTD:
482 return zstd_decompress_safe(mi, src, dst);
483
484 default:
485 WARN_ON(true);
486 return -EOPNOTSUPP;
487 }
488 }
489
log_read_one_record(struct read_log * rl,struct read_log_state * rs)490 static void log_read_one_record(struct read_log *rl, struct read_log_state *rs)
491 {
492 union log_record *record =
493 (union log_record *)((u8 *)rl->rl_ring_buf + rs->next_offset);
494 size_t record_size;
495
496 switch (record->full_record.type) {
497 case FULL:
498 rs->base_record = record->full_record;
499 record_size = sizeof(record->full_record);
500 break;
501
502 case SAME_FILE:
503 rs->base_record.block_index =
504 record->same_file.block_index;
505 rs->base_record.absolute_ts_us +=
506 record->same_file.relative_ts_us;
507 rs->base_record.uid = record->same_file.uid;
508 record_size = sizeof(record->same_file);
509 break;
510
511 case SAME_FILE_CLOSE_BLOCK:
512 rs->base_record.block_index +=
513 record->same_file_close_block.block_index_delta;
514 rs->base_record.absolute_ts_us +=
515 record->same_file_close_block.relative_ts_us;
516 record_size = sizeof(record->same_file_close_block);
517 break;
518
519 case SAME_FILE_CLOSE_BLOCK_SHORT:
520 rs->base_record.block_index +=
521 record->same_file_close_block_short.block_index_delta;
522 rs->base_record.absolute_ts_us +=
523 record->same_file_close_block_short.relative_ts_tens_us * 10;
524 record_size = sizeof(record->same_file_close_block_short);
525 break;
526
527 case SAME_FILE_NEXT_BLOCK:
528 ++rs->base_record.block_index;
529 rs->base_record.absolute_ts_us +=
530 record->same_file_next_block.relative_ts_us;
531 record_size = sizeof(record->same_file_next_block);
532 break;
533
534 case SAME_FILE_NEXT_BLOCK_SHORT:
535 ++rs->base_record.block_index;
536 rs->base_record.absolute_ts_us +=
537 record->same_file_next_block_short.relative_ts_tens_us * 10;
538 record_size = sizeof(record->same_file_next_block_short);
539 break;
540 }
541
542 rs->next_offset += record_size;
543 if (rs->next_offset > rl->rl_size - sizeof(*record)) {
544 rs->next_offset = 0;
545 ++rs->current_pass_no;
546 }
547 ++rs->current_record_no;
548 }
549
log_block_read(struct mount_info * mi,incfs_uuid_t * id,int block_index)550 static void log_block_read(struct mount_info *mi, incfs_uuid_t *id,
551 int block_index)
552 {
553 struct read_log *log = &mi->mi_log;
554 struct read_log_state *head, *tail;
555 s64 now_us;
556 s64 relative_us;
557 union log_record record;
558 size_t record_size;
559 uid_t uid = current_uid().val;
560 int block_delta;
561 bool same_file, same_uid;
562 bool next_block, close_block, very_close_block;
563 bool close_time, very_close_time, very_very_close_time;
564
565 /*
566 * This may read the old value, but it's OK to delay the logging start
567 * right after the configuration update.
568 */
569 if (READ_ONCE(log->rl_size) == 0)
570 return;
571
572 now_us = ktime_to_us(ktime_get());
573
574 spin_lock(&log->rl_lock);
575 if (log->rl_size == 0) {
576 spin_unlock(&log->rl_lock);
577 return;
578 }
579
580 head = &log->rl_head;
581 tail = &log->rl_tail;
582 relative_us = now_us - head->base_record.absolute_ts_us;
583
584 same_file = !memcmp(id, &head->base_record.file_id,
585 sizeof(incfs_uuid_t));
586 same_uid = uid == head->base_record.uid;
587
588 block_delta = block_index - head->base_record.block_index;
589 next_block = block_delta == 1;
590 very_close_block = block_delta >= S8_MIN && block_delta <= S8_MAX;
591 close_block = block_delta >= S16_MIN && block_delta <= S16_MAX;
592
593 very_very_close_time = relative_us < (1 << 5) * 10;
594 very_close_time = relative_us < (1 << 13);
595 close_time = relative_us < (1 << 16);
596
597 if (same_file && same_uid && next_block && very_very_close_time) {
598 record.same_file_next_block_short =
599 (struct same_file_next_block_short){
600 .type = SAME_FILE_NEXT_BLOCK_SHORT,
601 .relative_ts_tens_us = div_s64(relative_us, 10),
602 };
603 record_size = sizeof(struct same_file_next_block_short);
604 } else if (same_file && same_uid && next_block && very_close_time) {
605 record.same_file_next_block = (struct same_file_next_block){
606 .type = SAME_FILE_NEXT_BLOCK,
607 .relative_ts_us = relative_us,
608 };
609 record_size = sizeof(struct same_file_next_block);
610 } else if (same_file && same_uid && very_close_block &&
611 very_very_close_time) {
612 record.same_file_close_block_short =
613 (struct same_file_close_block_short){
614 .type = SAME_FILE_CLOSE_BLOCK_SHORT,
615 .relative_ts_tens_us = div_s64(relative_us, 10),
616 .block_index_delta = block_delta,
617 };
618 record_size = sizeof(struct same_file_close_block_short);
619 } else if (same_file && same_uid && close_block && very_close_time) {
620 record.same_file_close_block = (struct same_file_close_block){
621 .type = SAME_FILE_CLOSE_BLOCK,
622 .relative_ts_us = relative_us,
623 .block_index_delta = block_delta,
624 };
625 record_size = sizeof(struct same_file_close_block);
626 } else if (same_file && close_time) {
627 record.same_file = (struct same_file){
628 .type = SAME_FILE,
629 .block_index = block_index,
630 .relative_ts_us = relative_us,
631 .uid = uid,
632 };
633 record_size = sizeof(struct same_file);
634 } else {
635 record.full_record = (struct full_record){
636 .type = FULL,
637 .block_index = block_index,
638 .file_id = *id,
639 .absolute_ts_us = now_us,
640 .uid = uid,
641 };
642 head->base_record.file_id = *id;
643 record_size = sizeof(struct full_record);
644 }
645
646 head->base_record.block_index = block_index;
647 head->base_record.absolute_ts_us = now_us;
648
649 /* Advance tail beyond area we are going to overwrite */
650 while (tail->current_pass_no < head->current_pass_no &&
651 tail->next_offset < head->next_offset + record_size)
652 log_read_one_record(log, tail);
653
654 memcpy(((u8 *)log->rl_ring_buf) + head->next_offset, &record,
655 record_size);
656 head->next_offset += record_size;
657 if (head->next_offset > log->rl_size - sizeof(record)) {
658 head->next_offset = 0;
659 ++head->current_pass_no;
660 }
661 ++head->current_record_no;
662
663 spin_unlock(&log->rl_lock);
664 schedule_delayed_work(&log->ml_wakeup_work, msecs_to_jiffies(16));
665 }
666
validate_hash_tree(struct backing_file_context * bfc,struct file * f,int block_index,struct mem_range data,u8 * buf)667 static int validate_hash_tree(struct backing_file_context *bfc, struct file *f,
668 int block_index, struct mem_range data, u8 *buf)
669 {
670 struct data_file *df = get_incfs_data_file(f);
671 u8 stored_digest[INCFS_MAX_HASH_SIZE] = {};
672 u8 calculated_digest[INCFS_MAX_HASH_SIZE] = {};
673 struct mtree *tree = NULL;
674 struct incfs_df_signature *sig = NULL;
675 int digest_size;
676 int hash_block_index = block_index;
677 int lvl;
678 int res;
679 loff_t hash_block_offset[INCFS_MAX_MTREE_LEVELS];
680 size_t hash_offset_in_block[INCFS_MAX_MTREE_LEVELS];
681 int hash_per_block;
682 pgoff_t file_pages;
683
684 /*
685 * Memory barrier to make sure tree is fully present if added via enable
686 * verity
687 */
688 tree = smp_load_acquire(&df->df_hash_tree);
689 sig = df->df_signature;
690 if (!tree || !sig)
691 return 0;
692
693 digest_size = tree->alg->digest_size;
694 hash_per_block = INCFS_DATA_FILE_BLOCK_SIZE / digest_size;
695 for (lvl = 0; lvl < tree->depth; lvl++) {
696 loff_t lvl_off = tree->hash_level_suboffset[lvl];
697
698 hash_block_offset[lvl] =
699 lvl_off + round_down(hash_block_index * digest_size,
700 INCFS_DATA_FILE_BLOCK_SIZE);
701 hash_offset_in_block[lvl] = hash_block_index * digest_size %
702 INCFS_DATA_FILE_BLOCK_SIZE;
703 hash_block_index /= hash_per_block;
704 }
705
706 memcpy(stored_digest, tree->root_hash, digest_size);
707
708 file_pages = DIV_ROUND_UP(df->df_size, INCFS_DATA_FILE_BLOCK_SIZE);
709 for (lvl = tree->depth - 1; lvl >= 0; lvl--) {
710 pgoff_t hash_page =
711 file_pages +
712 hash_block_offset[lvl] / INCFS_DATA_FILE_BLOCK_SIZE;
713 struct page *page = find_get_page_flags(
714 f->f_inode->i_mapping, hash_page, FGP_ACCESSED);
715
716 if (page && PageChecked(page)) {
717 u8 *addr = kmap_atomic(page);
718
719 memcpy(stored_digest, addr + hash_offset_in_block[lvl],
720 digest_size);
721
722 kunmap_atomic(addr);
723 put_page(page);
724 continue;
725 }
726
727 if (page)
728 put_page(page);
729
730 res = incfs_kread(bfc, buf, INCFS_DATA_FILE_BLOCK_SIZE,
731 hash_block_offset[lvl] + sig->hash_offset);
732 if (res < 0)
733 return res;
734 if (res != INCFS_DATA_FILE_BLOCK_SIZE)
735 return -EIO;
736 res = incfs_calc_digest(tree->alg,
737 range(buf, INCFS_DATA_FILE_BLOCK_SIZE),
738 range(calculated_digest, digest_size));
739 if (res)
740 return res;
741
742 if (memcmp(stored_digest, calculated_digest, digest_size)) {
743 int i;
744 bool zero = true;
745
746 pr_warn("incfs: Hash mismatch lvl:%d blk:%d\n",
747 lvl, block_index);
748 for (i = 0; i < digest_size; i++)
749 if (stored_digest[i]) {
750 zero = false;
751 break;
752 }
753
754 if (zero)
755 pr_debug("Note saved_digest all zero - did you forget to load the hashes?\n");
756 return -EBADMSG;
757 }
758
759 memcpy(stored_digest, buf + hash_offset_in_block[lvl],
760 digest_size);
761
762 page = grab_cache_page(f->f_inode->i_mapping, hash_page);
763 if (page) {
764 u8 *addr = kmap_atomic(page);
765
766 memcpy(addr, buf, INCFS_DATA_FILE_BLOCK_SIZE);
767 kunmap_atomic(addr);
768 SetPageChecked(page);
769 SetPageUptodate(page);
770 unlock_page(page);
771 put_page(page);
772 }
773 }
774
775 res = incfs_calc_digest(tree->alg, data,
776 range(calculated_digest, digest_size));
777 if (res)
778 return res;
779
780 if (memcmp(stored_digest, calculated_digest, digest_size)) {
781 pr_debug("Leaf hash mismatch blk:%d\n", block_index);
782 return -EBADMSG;
783 }
784
785 return 0;
786 }
787
get_file_segment(struct data_file * df,int block_index)788 static struct data_file_segment *get_file_segment(struct data_file *df,
789 int block_index)
790 {
791 int seg_idx = block_index % ARRAY_SIZE(df->df_segments);
792
793 return &df->df_segments[seg_idx];
794 }
795
is_data_block_present(struct data_file_block * block)796 static bool is_data_block_present(struct data_file_block *block)
797 {
798 return (block->db_backing_file_data_offset != 0) &&
799 (block->db_stored_size != 0);
800 }
801
convert_data_file_block(struct incfs_blockmap_entry * bme,struct data_file_block * res_block)802 static void convert_data_file_block(struct incfs_blockmap_entry *bme,
803 struct data_file_block *res_block)
804 {
805 u16 flags = le16_to_cpu(bme->me_flags);
806
807 res_block->db_backing_file_data_offset =
808 le16_to_cpu(bme->me_data_offset_hi);
809 res_block->db_backing_file_data_offset <<= 32;
810 res_block->db_backing_file_data_offset |=
811 le32_to_cpu(bme->me_data_offset_lo);
812 res_block->db_stored_size = le16_to_cpu(bme->me_data_size);
813 res_block->db_comp_alg = flags & INCFS_BLOCK_COMPRESSED_MASK;
814 }
815
get_data_file_block(struct data_file * df,int index,struct data_file_block * res_block)816 static int get_data_file_block(struct data_file *df, int index,
817 struct data_file_block *res_block)
818 {
819 struct incfs_blockmap_entry bme = {};
820 struct backing_file_context *bfc = NULL;
821 loff_t blockmap_off = 0;
822 int error = 0;
823
824 if (!df || !res_block)
825 return -EFAULT;
826
827 blockmap_off = df->df_blockmap_off;
828 bfc = df->df_backing_file_context;
829
830 if (index < 0 || blockmap_off == 0)
831 return -EINVAL;
832
833 error = incfs_read_blockmap_entry(bfc, index, blockmap_off, &bme);
834 if (error)
835 return error;
836
837 convert_data_file_block(&bme, res_block);
838 return 0;
839 }
840
check_room_for_one_range(u32 size,u32 size_out)841 static int check_room_for_one_range(u32 size, u32 size_out)
842 {
843 if (size_out + sizeof(struct incfs_filled_range) > size)
844 return -ERANGE;
845 return 0;
846 }
847
copy_one_range(struct incfs_filled_range * range,void __user * buffer,u32 size,u32 * size_out)848 static int copy_one_range(struct incfs_filled_range *range, void __user *buffer,
849 u32 size, u32 *size_out)
850 {
851 int error = check_room_for_one_range(size, *size_out);
852 if (error)
853 return error;
854
855 if (copy_to_user(((char __user *)buffer) + *size_out, range,
856 sizeof(*range)))
857 return -EFAULT;
858
859 *size_out += sizeof(*range);
860 return 0;
861 }
862
863 #define READ_BLOCKMAP_ENTRIES 512
incfs_get_filled_blocks(struct data_file * df,struct incfs_file_data * fd,struct incfs_get_filled_blocks_args * arg)864 int incfs_get_filled_blocks(struct data_file *df,
865 struct incfs_file_data *fd,
866 struct incfs_get_filled_blocks_args *arg)
867 {
868 int error = 0;
869 bool in_range = false;
870 struct incfs_filled_range range;
871 void __user *buffer = u64_to_user_ptr(arg->range_buffer);
872 u32 size = arg->range_buffer_size;
873 u32 end_index =
874 arg->end_index ? arg->end_index : df->df_total_block_count;
875 u32 *size_out = &arg->range_buffer_size_out;
876 int i = READ_BLOCKMAP_ENTRIES - 1;
877 int entries_read = 0;
878 struct incfs_blockmap_entry *bme;
879 int data_blocks_filled = 0;
880 int hash_blocks_filled = 0;
881
882 *size_out = 0;
883 if (end_index > df->df_total_block_count)
884 end_index = df->df_total_block_count;
885 arg->total_blocks_out = df->df_total_block_count;
886 arg->data_blocks_out = df->df_data_block_count;
887
888 if (atomic_read(&df->df_data_blocks_written) ==
889 df->df_data_block_count) {
890 pr_debug("File marked full, fast get_filled_blocks");
891 if (arg->start_index > end_index) {
892 arg->index_out = arg->start_index;
893 return 0;
894 }
895 arg->index_out = arg->start_index;
896
897 error = check_room_for_one_range(size, *size_out);
898 if (error)
899 return error;
900
901 range = (struct incfs_filled_range){
902 .begin = arg->start_index,
903 .end = end_index,
904 };
905
906 error = copy_one_range(&range, buffer, size, size_out);
907 if (error)
908 return error;
909 arg->index_out = end_index;
910 return 0;
911 }
912
913 bme = kzalloc(sizeof(*bme) * READ_BLOCKMAP_ENTRIES,
914 GFP_NOFS | __GFP_COMP);
915 if (!bme)
916 return -ENOMEM;
917
918 for (arg->index_out = arg->start_index; arg->index_out < end_index;
919 ++arg->index_out) {
920 struct data_file_block dfb;
921
922 if (++i == READ_BLOCKMAP_ENTRIES) {
923 entries_read = incfs_read_blockmap_entries(
924 df->df_backing_file_context, bme,
925 arg->index_out, READ_BLOCKMAP_ENTRIES,
926 df->df_blockmap_off);
927 if (entries_read < 0) {
928 error = entries_read;
929 break;
930 }
931
932 i = 0;
933 }
934
935 if (i >= entries_read) {
936 error = -EIO;
937 break;
938 }
939
940 convert_data_file_block(bme + i, &dfb);
941
942 if (is_data_block_present(&dfb)) {
943 if (arg->index_out >= df->df_data_block_count)
944 ++hash_blocks_filled;
945 else
946 ++data_blocks_filled;
947 }
948
949 if (is_data_block_present(&dfb) == in_range)
950 continue;
951
952 if (!in_range) {
953 error = check_room_for_one_range(size, *size_out);
954 if (error)
955 break;
956 in_range = true;
957 range.begin = arg->index_out;
958 } else {
959 range.end = arg->index_out;
960 error = copy_one_range(&range, buffer, size, size_out);
961 if (error) {
962 /* there will be another try out of the loop,
963 * it will reset the index_out if it fails too
964 */
965 break;
966 }
967 in_range = false;
968 }
969 }
970
971 if (in_range) {
972 range.end = arg->index_out;
973 error = copy_one_range(&range, buffer, size, size_out);
974 if (error)
975 arg->index_out = range.begin;
976 }
977
978 if (arg->start_index == 0) {
979 fd->fd_get_block_pos = 0;
980 fd->fd_filled_data_blocks = 0;
981 fd->fd_filled_hash_blocks = 0;
982 }
983
984 if (arg->start_index == fd->fd_get_block_pos) {
985 fd->fd_get_block_pos = arg->index_out + 1;
986 fd->fd_filled_data_blocks += data_blocks_filled;
987 fd->fd_filled_hash_blocks += hash_blocks_filled;
988 }
989
990 if (fd->fd_get_block_pos == df->df_total_block_count + 1) {
991 if (fd->fd_filled_data_blocks >
992 atomic_read(&df->df_data_blocks_written))
993 atomic_set(&df->df_data_blocks_written,
994 fd->fd_filled_data_blocks);
995
996 if (fd->fd_filled_hash_blocks >
997 atomic_read(&df->df_hash_blocks_written))
998 atomic_set(&df->df_hash_blocks_written,
999 fd->fd_filled_hash_blocks);
1000 }
1001
1002 kfree(bme);
1003 return error;
1004 }
1005
is_read_done(struct pending_read * read)1006 static bool is_read_done(struct pending_read *read)
1007 {
1008 return atomic_read_acquire(&read->done) != 0;
1009 }
1010
set_read_done(struct pending_read * read)1011 static void set_read_done(struct pending_read *read)
1012 {
1013 atomic_set_release(&read->done, 1);
1014 }
1015
1016 /*
1017 * Notifies a given data file about pending read from a given block.
1018 * Returns a new pending read entry.
1019 */
add_pending_read(struct data_file * df,int block_index)1020 static struct pending_read *add_pending_read(struct data_file *df,
1021 int block_index)
1022 {
1023 struct pending_read *result = NULL;
1024 struct data_file_segment *segment = NULL;
1025 struct mount_info *mi = NULL;
1026
1027 segment = get_file_segment(df, block_index);
1028 mi = df->df_mount_info;
1029
1030 result = kzalloc(sizeof(*result), GFP_NOFS);
1031 if (!result)
1032 return NULL;
1033
1034 result->file_id = df->df_id;
1035 result->block_index = block_index;
1036 result->timestamp_us = ktime_to_us(ktime_get());
1037 result->uid = current_uid().val;
1038
1039 spin_lock(&mi->pending_read_lock);
1040
1041 result->serial_number = ++mi->mi_last_pending_read_number;
1042 mi->mi_pending_reads_count++;
1043
1044 list_add_rcu(&result->mi_reads_list, &mi->mi_reads_list_head);
1045 list_add_rcu(&result->segment_reads_list, &segment->reads_list_head);
1046
1047 spin_unlock(&mi->pending_read_lock);
1048
1049 wake_up_all(&mi->mi_pending_reads_notif_wq);
1050 return result;
1051 }
1052
free_pending_read_entry(struct rcu_head * entry)1053 static void free_pending_read_entry(struct rcu_head *entry)
1054 {
1055 struct pending_read *read;
1056
1057 read = container_of(entry, struct pending_read, rcu);
1058
1059 kfree(read);
1060 }
1061
1062 /* Notifies a given data file that pending read is completed. */
remove_pending_read(struct data_file * df,struct pending_read * read)1063 static void remove_pending_read(struct data_file *df, struct pending_read *read)
1064 {
1065 struct mount_info *mi = NULL;
1066
1067 if (!df || !read) {
1068 WARN_ON(!df);
1069 WARN_ON(!read);
1070 return;
1071 }
1072
1073 mi = df->df_mount_info;
1074
1075 spin_lock(&mi->pending_read_lock);
1076
1077 list_del_rcu(&read->mi_reads_list);
1078 list_del_rcu(&read->segment_reads_list);
1079
1080 mi->mi_pending_reads_count--;
1081
1082 spin_unlock(&mi->pending_read_lock);
1083
1084 /* Don't free. Wait for readers */
1085 call_rcu(&read->rcu, free_pending_read_entry);
1086 }
1087
notify_pending_reads(struct mount_info * mi,struct data_file_segment * segment,int index)1088 static void notify_pending_reads(struct mount_info *mi,
1089 struct data_file_segment *segment,
1090 int index)
1091 {
1092 struct pending_read *entry = NULL;
1093
1094 /* Notify pending reads waiting for this block. */
1095 rcu_read_lock();
1096 list_for_each_entry_rcu(entry, &segment->reads_list_head,
1097 segment_reads_list) {
1098 if (entry->block_index == index)
1099 set_read_done(entry);
1100 }
1101 rcu_read_unlock();
1102 wake_up_all(&segment->new_data_arrival_wq);
1103
1104 atomic_inc(&mi->mi_blocks_written);
1105 wake_up_all(&mi->mi_blocks_written_notif_wq);
1106 }
1107
wait_for_data_block(struct data_file * df,int block_index,struct data_file_block * res_block,struct incfs_read_data_file_timeouts * timeouts,unsigned int * delayed_min_us)1108 static int wait_for_data_block(struct data_file *df, int block_index,
1109 struct data_file_block *res_block,
1110 struct incfs_read_data_file_timeouts *timeouts,
1111 unsigned int *delayed_min_us)
1112 {
1113 struct data_file_block block = {};
1114 struct data_file_segment *segment = NULL;
1115 struct pending_read *read = NULL;
1116 struct mount_info *mi = NULL;
1117 int error;
1118 int wait_res = 0;
1119 unsigned int delayed_pending_us = 0;
1120 bool delayed_pending = false;
1121
1122 if (!df || !res_block)
1123 return -EFAULT;
1124
1125 if (block_index < 0 || block_index >= df->df_data_block_count)
1126 return -EINVAL;
1127
1128 if (df->df_blockmap_off <= 0 || !df->df_mount_info)
1129 return -ENODATA;
1130
1131 mi = df->df_mount_info;
1132 segment = get_file_segment(df, block_index);
1133
1134 error = down_read_killable(&segment->rwsem);
1135 if (error)
1136 return error;
1137
1138 /* Look up the given block */
1139 error = get_data_file_block(df, block_index, &block);
1140
1141 up_read(&segment->rwsem);
1142
1143 if (error)
1144 return error;
1145
1146 /* If the block was found, just return it. No need to wait. */
1147 if (is_data_block_present(&block)) {
1148 *res_block = block;
1149 if (timeouts && timeouts->min_time_us) {
1150 *delayed_min_us = timeouts->min_time_us;
1151 goto out;
1152 }
1153 return 0;
1154 } else {
1155 /* If it's not found, create a pending read */
1156 if (timeouts && timeouts->max_pending_time_us) {
1157 read = add_pending_read(df, block_index);
1158 if (!read)
1159 return -ENOMEM;
1160 } else {
1161 log_block_read(mi, &df->df_id, block_index);
1162 return -ETIME;
1163 }
1164 }
1165
1166 /* Rest of function only applies if timeouts != NULL */
1167 if (!timeouts) {
1168 pr_warn("incfs: timeouts unexpectedly NULL\n");
1169 return -EFSCORRUPTED;
1170 }
1171
1172 /* Wait for notifications about block's arrival */
1173 wait_res =
1174 wait_event_interruptible_timeout(segment->new_data_arrival_wq,
1175 (is_read_done(read)),
1176 usecs_to_jiffies(timeouts->max_pending_time_us));
1177
1178 /* Woke up, the pending read is no longer needed. */
1179 remove_pending_read(df, read);
1180
1181 if (wait_res == 0) {
1182 /* Wait has timed out */
1183 log_block_read(mi, &df->df_id, block_index);
1184 return -ETIME;
1185 }
1186 if (wait_res < 0) {
1187 /*
1188 * Only ERESTARTSYS is really expected here when a signal
1189 * comes while we wait.
1190 */
1191 return wait_res;
1192 }
1193
1194 delayed_pending = true;
1195 delayed_pending_us = timeouts->max_pending_time_us -
1196 jiffies_to_usecs(wait_res);
1197 if (timeouts->min_pending_time_us > delayed_pending_us)
1198 *delayed_min_us = timeouts->min_pending_time_us -
1199 delayed_pending_us;
1200
1201 error = down_read_killable(&segment->rwsem);
1202 if (error)
1203 return error;
1204
1205 /*
1206 * Re-read blocks info now, it has just arrived and
1207 * should be available.
1208 */
1209 error = get_data_file_block(df, block_index, &block);
1210 if (!error) {
1211 if (is_data_block_present(&block))
1212 *res_block = block;
1213 else {
1214 /*
1215 * Somehow wait finished successfully but block still
1216 * can't be found. It's not normal.
1217 */
1218 pr_warn("incfs: Wait succeeded but block not found.\n");
1219 error = -ENODATA;
1220 }
1221 }
1222 up_read(&segment->rwsem);
1223
1224 out:
1225 if (error)
1226 return error;
1227
1228 if (delayed_pending) {
1229 mi->mi_reads_delayed_pending++;
1230 mi->mi_reads_delayed_pending_us +=
1231 delayed_pending_us;
1232 }
1233
1234 if (delayed_min_us && *delayed_min_us) {
1235 mi->mi_reads_delayed_min++;
1236 mi->mi_reads_delayed_min_us += *delayed_min_us;
1237 }
1238
1239 return 0;
1240 }
1241
incfs_update_sysfs_error(struct file * file,int index,int result,struct mount_info * mi,struct data_file * df)1242 static int incfs_update_sysfs_error(struct file *file, int index, int result,
1243 struct mount_info *mi, struct data_file *df)
1244 {
1245 int error;
1246
1247 if (result >= 0)
1248 return 0;
1249
1250 error = mutex_lock_interruptible(&mi->mi_le_mutex);
1251 if (error)
1252 return error;
1253
1254 mi->mi_le_file_id = df->df_id;
1255 mi->mi_le_time_us = ktime_to_us(ktime_get());
1256 mi->mi_le_page = index;
1257 mi->mi_le_errno = result;
1258 mi->mi_le_uid = current_uid().val;
1259 mutex_unlock(&mi->mi_le_mutex);
1260
1261 return 0;
1262 }
1263
incfs_read_data_file_block(struct mem_range dst,struct file * f,int index,struct mem_range tmp,struct incfs_read_data_file_timeouts * timeouts,unsigned int * delayed_min_us)1264 ssize_t incfs_read_data_file_block(struct mem_range dst, struct file *f,
1265 int index, struct mem_range tmp,
1266 struct incfs_read_data_file_timeouts *timeouts,
1267 unsigned int *delayed_min_us)
1268 {
1269 loff_t pos;
1270 ssize_t result;
1271 size_t bytes_to_read;
1272 struct mount_info *mi = NULL;
1273 struct backing_file_context *bfc = NULL;
1274 struct data_file_block block = {};
1275 struct data_file *df = get_incfs_data_file(f);
1276
1277 if (!dst.data || !df || !tmp.data)
1278 return -EFAULT;
1279
1280 if (tmp.len < 2 * INCFS_DATA_FILE_BLOCK_SIZE)
1281 return -ERANGE;
1282
1283 mi = df->df_mount_info;
1284 bfc = df->df_backing_file_context;
1285
1286 result = wait_for_data_block(df, index, &block, timeouts,
1287 delayed_min_us);
1288 if (result < 0)
1289 goto out;
1290
1291 pos = block.db_backing_file_data_offset;
1292 if (block.db_comp_alg == COMPRESSION_NONE) {
1293 bytes_to_read = min(dst.len, block.db_stored_size);
1294 result = incfs_kread(bfc, dst.data, bytes_to_read, pos);
1295
1296 /* Some data was read, but not enough */
1297 if (result >= 0 && result != bytes_to_read)
1298 result = -EIO;
1299 } else {
1300 bytes_to_read = min(tmp.len, block.db_stored_size);
1301 result = incfs_kread(bfc, tmp.data, bytes_to_read, pos);
1302 if (result == bytes_to_read) {
1303 result =
1304 decompress(mi, range(tmp.data, bytes_to_read),
1305 dst, block.db_comp_alg);
1306 if (result < 0) {
1307 const char *name =
1308 bfc->bc_file->f_path.dentry->d_name.name;
1309
1310 pr_warn_once("incfs: Decompression error. %s",
1311 name);
1312 }
1313 } else if (result >= 0) {
1314 /* Some data was read, but not enough */
1315 result = -EIO;
1316 }
1317 }
1318
1319 if (result > 0) {
1320 int err = validate_hash_tree(bfc, f, index, dst, tmp.data);
1321
1322 if (err < 0)
1323 result = err;
1324 }
1325
1326 if (result >= 0)
1327 log_block_read(mi, &df->df_id, index);
1328
1329 out:
1330 if (result == -ETIME)
1331 mi->mi_reads_failed_timed_out++;
1332 else if (result == -EBADMSG)
1333 mi->mi_reads_failed_hash_verification++;
1334 else if (result < 0)
1335 mi->mi_reads_failed_other++;
1336
1337 incfs_update_sysfs_error(f, index, result, mi, df);
1338
1339 return result;
1340 }
1341
incfs_read_merkle_tree_blocks(struct mem_range dst,struct data_file * df,size_t offset)1342 ssize_t incfs_read_merkle_tree_blocks(struct mem_range dst,
1343 struct data_file *df, size_t offset)
1344 {
1345 struct backing_file_context *bfc = NULL;
1346 struct incfs_df_signature *sig = NULL;
1347 size_t to_read = dst.len;
1348
1349 if (!dst.data || !df)
1350 return -EFAULT;
1351
1352 sig = df->df_signature;
1353 bfc = df->df_backing_file_context;
1354
1355 if (offset > sig->hash_size)
1356 return -ERANGE;
1357
1358 if (offset + to_read > sig->hash_size)
1359 to_read = sig->hash_size - offset;
1360
1361 return incfs_kread(bfc, dst.data, to_read, sig->hash_offset + offset);
1362 }
1363
incfs_process_new_data_block(struct data_file * df,struct incfs_fill_block * block,u8 * data,bool * complete)1364 int incfs_process_new_data_block(struct data_file *df,
1365 struct incfs_fill_block *block, u8 *data,
1366 bool *complete)
1367 {
1368 struct mount_info *mi = NULL;
1369 struct backing_file_context *bfc = NULL;
1370 struct data_file_segment *segment = NULL;
1371 struct data_file_block existing_block = {};
1372 u16 flags = 0;
1373 int error = 0;
1374
1375 if (!df || !block)
1376 return -EFAULT;
1377
1378 bfc = df->df_backing_file_context;
1379 mi = df->df_mount_info;
1380
1381 if (block->block_index >= df->df_data_block_count)
1382 return -ERANGE;
1383
1384 segment = get_file_segment(df, block->block_index);
1385 if (!segment)
1386 return -EFAULT;
1387
1388 if (block->compression == COMPRESSION_LZ4)
1389 flags |= INCFS_BLOCK_COMPRESSED_LZ4;
1390 else if (block->compression == COMPRESSION_ZSTD)
1391 flags |= INCFS_BLOCK_COMPRESSED_ZSTD;
1392 else if (block->compression)
1393 return -EINVAL;
1394
1395 error = down_read_killable(&segment->rwsem);
1396 if (error)
1397 return error;
1398
1399 error = get_data_file_block(df, block->block_index, &existing_block);
1400
1401 up_read(&segment->rwsem);
1402
1403 if (error)
1404 return error;
1405 if (is_data_block_present(&existing_block))
1406 /* Block is already present, nothing to do here */
1407 return 0;
1408
1409 error = down_write_killable(&segment->rwsem);
1410 if (error)
1411 return error;
1412
1413 /* Recheck inside write lock */
1414 error = get_data_file_block(df, block->block_index, &existing_block);
1415 if (error)
1416 goto out_up_write;
1417
1418 if (is_data_block_present(&existing_block))
1419 goto out_up_write;
1420
1421 error = mutex_lock_interruptible(&bfc->bc_mutex);
1422 if (error)
1423 goto out_up_write;
1424
1425 error = incfs_write_data_block_to_backing_file(bfc,
1426 range(data, block->data_len), block->block_index,
1427 df->df_blockmap_off, flags);
1428 if (error)
1429 goto out_mutex_unlock;
1430
1431 if (atomic_inc_return(&df->df_data_blocks_written)
1432 >= df->df_data_block_count)
1433 *complete = true;
1434
1435 out_mutex_unlock:
1436 mutex_unlock(&bfc->bc_mutex);
1437 if (!error)
1438 notify_pending_reads(mi, segment, block->block_index);
1439
1440 out_up_write:
1441 up_write(&segment->rwsem);
1442
1443 if (error)
1444 pr_debug("%d error: %d\n", block->block_index, error);
1445 return error;
1446 }
1447
incfs_read_file_signature(struct data_file * df,struct mem_range dst)1448 int incfs_read_file_signature(struct data_file *df, struct mem_range dst)
1449 {
1450 struct backing_file_context *bfc = df->df_backing_file_context;
1451 struct incfs_df_signature *sig;
1452 int read_res = 0;
1453
1454 if (!dst.data)
1455 return -EFAULT;
1456
1457 sig = df->df_signature;
1458 if (!sig)
1459 return 0;
1460
1461 if (dst.len < sig->sig_size)
1462 return -E2BIG;
1463
1464 read_res = incfs_kread(bfc, dst.data, sig->sig_size, sig->sig_offset);
1465
1466 if (read_res < 0)
1467 return read_res;
1468
1469 if (read_res != sig->sig_size)
1470 return -EIO;
1471
1472 return read_res;
1473 }
1474
incfs_process_new_hash_block(struct data_file * df,struct incfs_fill_block * block,u8 * data)1475 int incfs_process_new_hash_block(struct data_file *df,
1476 struct incfs_fill_block *block, u8 *data)
1477 {
1478 struct backing_file_context *bfc = NULL;
1479 struct mount_info *mi = NULL;
1480 struct mtree *hash_tree = NULL;
1481 struct incfs_df_signature *sig = NULL;
1482 loff_t hash_area_base = 0;
1483 loff_t hash_area_size = 0;
1484 int error = 0;
1485
1486 if (!df || !block)
1487 return -EFAULT;
1488
1489 if (!(block->flags & INCFS_BLOCK_FLAGS_HASH))
1490 return -EINVAL;
1491
1492 bfc = df->df_backing_file_context;
1493 mi = df->df_mount_info;
1494
1495 if (!df)
1496 return -ENOENT;
1497
1498 hash_tree = df->df_hash_tree;
1499 sig = df->df_signature;
1500 if (!hash_tree || !sig || sig->hash_offset == 0)
1501 return -ENOTSUPP;
1502
1503 hash_area_base = sig->hash_offset;
1504 hash_area_size = sig->hash_size;
1505 if (hash_area_size < block->block_index * INCFS_DATA_FILE_BLOCK_SIZE
1506 + block->data_len) {
1507 /* Hash block goes beyond dedicated hash area of this file. */
1508 return -ERANGE;
1509 }
1510
1511 error = mutex_lock_interruptible(&bfc->bc_mutex);
1512 if (!error) {
1513 error = incfs_write_hash_block_to_backing_file(
1514 bfc, range(data, block->data_len), block->block_index,
1515 hash_area_base, df->df_blockmap_off, df->df_size);
1516 mutex_unlock(&bfc->bc_mutex);
1517 }
1518 if (!error)
1519 atomic_inc(&df->df_hash_blocks_written);
1520
1521 return error;
1522 }
1523
process_blockmap_md(struct incfs_blockmap * bm,struct metadata_handler * handler)1524 static int process_blockmap_md(struct incfs_blockmap *bm,
1525 struct metadata_handler *handler)
1526 {
1527 struct data_file *df = handler->context;
1528 int error = 0;
1529 loff_t base_off = le64_to_cpu(bm->m_base_offset);
1530 u32 block_count = le32_to_cpu(bm->m_block_count);
1531
1532 if (!df)
1533 return -EFAULT;
1534
1535 if (df->df_data_block_count > block_count)
1536 return -EBADMSG;
1537
1538 df->df_total_block_count = block_count;
1539 df->df_blockmap_off = base_off;
1540 return error;
1541 }
1542
process_file_signature_md(struct incfs_file_signature * sg,struct metadata_handler * handler)1543 static int process_file_signature_md(struct incfs_file_signature *sg,
1544 struct metadata_handler *handler)
1545 {
1546 struct data_file *df = handler->context;
1547 struct mtree *hash_tree = NULL;
1548 int error = 0;
1549 struct incfs_df_signature *signature =
1550 kzalloc(sizeof(*signature), GFP_NOFS);
1551 void *buf = NULL;
1552 ssize_t read;
1553
1554 if (!signature)
1555 return -ENOMEM;
1556
1557 if (!df || !df->df_backing_file_context ||
1558 !df->df_backing_file_context->bc_file) {
1559 error = -ENOENT;
1560 goto out;
1561 }
1562
1563 signature->hash_offset = le64_to_cpu(sg->sg_hash_tree_offset);
1564 signature->hash_size = le32_to_cpu(sg->sg_hash_tree_size);
1565 signature->sig_offset = le64_to_cpu(sg->sg_sig_offset);
1566 signature->sig_size = le32_to_cpu(sg->sg_sig_size);
1567
1568 buf = kzalloc(signature->sig_size, GFP_NOFS);
1569 if (!buf) {
1570 error = -ENOMEM;
1571 goto out;
1572 }
1573
1574 read = incfs_kread(df->df_backing_file_context, buf,
1575 signature->sig_size, signature->sig_offset);
1576 if (read < 0) {
1577 error = read;
1578 goto out;
1579 }
1580
1581 if (read != signature->sig_size) {
1582 error = -EINVAL;
1583 goto out;
1584 }
1585
1586 hash_tree = incfs_alloc_mtree(range(buf, signature->sig_size),
1587 df->df_data_block_count);
1588 if (IS_ERR(hash_tree)) {
1589 error = PTR_ERR(hash_tree);
1590 hash_tree = NULL;
1591 goto out;
1592 }
1593 if (hash_tree->hash_tree_area_size != signature->hash_size) {
1594 error = -EINVAL;
1595 goto out;
1596 }
1597 if (signature->hash_size > 0 &&
1598 handler->md_record_offset <= signature->hash_offset) {
1599 error = -EINVAL;
1600 goto out;
1601 }
1602 if (handler->md_record_offset <= signature->sig_offset) {
1603 error = -EINVAL;
1604 goto out;
1605 }
1606 df->df_hash_tree = hash_tree;
1607 hash_tree = NULL;
1608 df->df_signature = signature;
1609 signature = NULL;
1610 out:
1611 incfs_free_mtree(hash_tree);
1612 kfree(signature);
1613 kfree(buf);
1614
1615 return error;
1616 }
1617
process_status_md(struct incfs_status * is,struct metadata_handler * handler)1618 static int process_status_md(struct incfs_status *is,
1619 struct metadata_handler *handler)
1620 {
1621 struct data_file *df = handler->context;
1622
1623 df->df_initial_data_blocks_written =
1624 le32_to_cpu(is->is_data_blocks_written);
1625 atomic_set(&df->df_data_blocks_written,
1626 df->df_initial_data_blocks_written);
1627
1628 df->df_initial_hash_blocks_written =
1629 le32_to_cpu(is->is_hash_blocks_written);
1630 atomic_set(&df->df_hash_blocks_written,
1631 df->df_initial_hash_blocks_written);
1632
1633 df->df_status_offset = handler->md_record_offset;
1634 return 0;
1635 }
1636
process_file_verity_signature_md(struct incfs_file_verity_signature * vs,struct metadata_handler * handler)1637 static int process_file_verity_signature_md(
1638 struct incfs_file_verity_signature *vs,
1639 struct metadata_handler *handler)
1640 {
1641 struct data_file *df = handler->context;
1642 struct incfs_df_verity_signature *verity_signature;
1643
1644 if (!df)
1645 return -EFAULT;
1646
1647 verity_signature = kzalloc(sizeof(*verity_signature), GFP_NOFS);
1648 if (!verity_signature)
1649 return -ENOMEM;
1650
1651 verity_signature->offset = le64_to_cpu(vs->vs_offset);
1652 verity_signature->size = le32_to_cpu(vs->vs_size);
1653 if (verity_signature->size > FS_VERITY_MAX_SIGNATURE_SIZE) {
1654 kfree(verity_signature);
1655 return -EFAULT;
1656 }
1657
1658 df->df_verity_signature = verity_signature;
1659 return 0;
1660 }
1661
incfs_scan_metadata_chain(struct data_file * df)1662 static int incfs_scan_metadata_chain(struct data_file *df)
1663 {
1664 struct metadata_handler *handler = NULL;
1665 int result = 0;
1666 int records_count = 0;
1667 int error = 0;
1668 struct backing_file_context *bfc = NULL;
1669 int nondata_block_count;
1670
1671 if (!df || !df->df_backing_file_context)
1672 return -EFAULT;
1673
1674 bfc = df->df_backing_file_context;
1675
1676 handler = kzalloc(sizeof(*handler), GFP_NOFS);
1677 if (!handler)
1678 return -ENOMEM;
1679
1680 handler->md_record_offset = df->df_metadata_off;
1681 handler->context = df;
1682 handler->handle_blockmap = process_blockmap_md;
1683 handler->handle_signature = process_file_signature_md;
1684 handler->handle_status = process_status_md;
1685 handler->handle_verity_signature = process_file_verity_signature_md;
1686
1687 while (handler->md_record_offset > 0) {
1688 error = incfs_read_next_metadata_record(bfc, handler);
1689 if (error) {
1690 pr_warn("incfs: Error during reading incfs-metadata record. Offset: %lld Record #%d Error code: %d\n",
1691 handler->md_record_offset, records_count + 1,
1692 -error);
1693 break;
1694 }
1695 records_count++;
1696 }
1697 if (error) {
1698 pr_warn("incfs: Error %d after reading %d incfs-metadata records.\n",
1699 -error, records_count);
1700 result = error;
1701 } else
1702 result = records_count;
1703
1704 nondata_block_count = df->df_total_block_count -
1705 df->df_data_block_count;
1706 if (df->df_hash_tree) {
1707 int hash_block_count = get_blocks_count_for_size(
1708 df->df_hash_tree->hash_tree_area_size);
1709
1710 /*
1711 * Files that were created with a hash tree have the hash tree
1712 * included in the block map, i.e. nondata_block_count ==
1713 * hash_block_count. Files whose hash tree was added by
1714 * FS_IOC_ENABLE_VERITY will still have the original block
1715 * count, i.e. nondata_block_count == 0.
1716 */
1717 if (nondata_block_count != hash_block_count &&
1718 nondata_block_count != 0)
1719 result = -EINVAL;
1720 } else if (nondata_block_count != 0) {
1721 result = -EINVAL;
1722 }
1723
1724 kfree(handler);
1725 return result;
1726 }
1727
1728 /*
1729 * Quickly checks if there are pending reads with a serial number larger
1730 * than a given one.
1731 */
incfs_fresh_pending_reads_exist(struct mount_info * mi,int last_number)1732 bool incfs_fresh_pending_reads_exist(struct mount_info *mi, int last_number)
1733 {
1734 bool result = false;
1735
1736 spin_lock(&mi->pending_read_lock);
1737 result = (mi->mi_last_pending_read_number > last_number) &&
1738 (mi->mi_pending_reads_count > 0);
1739 spin_unlock(&mi->pending_read_lock);
1740 return result;
1741 }
1742
incfs_collect_pending_reads(struct mount_info * mi,int sn_lowerbound,struct incfs_pending_read_info * reads,struct incfs_pending_read_info2 * reads2,int reads_size,int * new_max_sn)1743 int incfs_collect_pending_reads(struct mount_info *mi, int sn_lowerbound,
1744 struct incfs_pending_read_info *reads,
1745 struct incfs_pending_read_info2 *reads2,
1746 int reads_size, int *new_max_sn)
1747 {
1748 int reported_reads = 0;
1749 struct pending_read *entry = NULL;
1750
1751 if (!mi)
1752 return -EFAULT;
1753
1754 if (reads_size <= 0)
1755 return 0;
1756
1757 if (!incfs_fresh_pending_reads_exist(mi, sn_lowerbound))
1758 return 0;
1759
1760 rcu_read_lock();
1761
1762 list_for_each_entry_rcu(entry, &mi->mi_reads_list_head, mi_reads_list) {
1763 if (entry->serial_number <= sn_lowerbound)
1764 continue;
1765
1766 if (reads) {
1767 reads[reported_reads].file_id = entry->file_id;
1768 reads[reported_reads].block_index = entry->block_index;
1769 reads[reported_reads].serial_number =
1770 entry->serial_number;
1771 reads[reported_reads].timestamp_us =
1772 entry->timestamp_us;
1773 }
1774
1775 if (reads2) {
1776 reads2[reported_reads].file_id = entry->file_id;
1777 reads2[reported_reads].block_index = entry->block_index;
1778 reads2[reported_reads].serial_number =
1779 entry->serial_number;
1780 reads2[reported_reads].timestamp_us =
1781 entry->timestamp_us;
1782 reads2[reported_reads].uid = entry->uid;
1783 }
1784
1785 if (entry->serial_number > *new_max_sn)
1786 *new_max_sn = entry->serial_number;
1787
1788 reported_reads++;
1789 if (reported_reads >= reads_size)
1790 break;
1791 }
1792
1793 rcu_read_unlock();
1794
1795 return reported_reads;
1796 }
1797
incfs_get_log_state(struct mount_info * mi)1798 struct read_log_state incfs_get_log_state(struct mount_info *mi)
1799 {
1800 struct read_log *log = &mi->mi_log;
1801 struct read_log_state result;
1802
1803 spin_lock(&log->rl_lock);
1804 result = log->rl_head;
1805 spin_unlock(&log->rl_lock);
1806 return result;
1807 }
1808
incfs_get_uncollected_logs_count(struct mount_info * mi,const struct read_log_state * state)1809 int incfs_get_uncollected_logs_count(struct mount_info *mi,
1810 const struct read_log_state *state)
1811 {
1812 struct read_log *log = &mi->mi_log;
1813 u32 generation;
1814 u64 head_no, tail_no;
1815
1816 spin_lock(&log->rl_lock);
1817 tail_no = log->rl_tail.current_record_no;
1818 head_no = log->rl_head.current_record_no;
1819 generation = log->rl_head.generation_id;
1820 spin_unlock(&log->rl_lock);
1821
1822 if (generation != state->generation_id)
1823 return head_no - tail_no;
1824 else
1825 return head_no - max_t(u64, tail_no, state->current_record_no);
1826 }
1827
incfs_collect_logged_reads(struct mount_info * mi,struct read_log_state * state,struct incfs_pending_read_info * reads,struct incfs_pending_read_info2 * reads2,int reads_size)1828 int incfs_collect_logged_reads(struct mount_info *mi,
1829 struct read_log_state *state,
1830 struct incfs_pending_read_info *reads,
1831 struct incfs_pending_read_info2 *reads2,
1832 int reads_size)
1833 {
1834 int dst_idx;
1835 struct read_log *log = &mi->mi_log;
1836 struct read_log_state *head, *tail;
1837
1838 spin_lock(&log->rl_lock);
1839 head = &log->rl_head;
1840 tail = &log->rl_tail;
1841
1842 if (state->generation_id != head->generation_id) {
1843 pr_debug("read ptr is wrong generation: %u/%u",
1844 state->generation_id, head->generation_id);
1845
1846 *state = (struct read_log_state){
1847 .generation_id = head->generation_id,
1848 };
1849 }
1850
1851 if (state->current_record_no < tail->current_record_no) {
1852 pr_debug("read ptr is behind, moving: %u/%u -> %u/%u\n",
1853 (u32)state->next_offset,
1854 (u32)state->current_pass_no,
1855 (u32)tail->next_offset, (u32)tail->current_pass_no);
1856
1857 *state = *tail;
1858 }
1859
1860 for (dst_idx = 0; dst_idx < reads_size; dst_idx++) {
1861 if (state->current_record_no == head->current_record_no)
1862 break;
1863
1864 log_read_one_record(log, state);
1865
1866 if (reads)
1867 reads[dst_idx] = (struct incfs_pending_read_info) {
1868 .file_id = state->base_record.file_id,
1869 .block_index = state->base_record.block_index,
1870 .serial_number = state->current_record_no,
1871 .timestamp_us =
1872 state->base_record.absolute_ts_us,
1873 };
1874
1875 if (reads2)
1876 reads2[dst_idx] = (struct incfs_pending_read_info2) {
1877 .file_id = state->base_record.file_id,
1878 .block_index = state->base_record.block_index,
1879 .serial_number = state->current_record_no,
1880 .timestamp_us =
1881 state->base_record.absolute_ts_us,
1882 .uid = state->base_record.uid,
1883 };
1884 }
1885
1886 spin_unlock(&log->rl_lock);
1887 return dst_idx;
1888 }
1889
1890