1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright 2019 Google LLC
4 */
5 #include <linux/gfp.h>
6 #include <linux/types.h>
7 #include <linux/slab.h>
8 #include <linux/file.h>
9 #include <linux/ktime.h>
10 #include <linux/mm.h>
11 #include <linux/lz4.h>
12 #include <linux/crc32.h>
13
14 #include "data_mgmt.h"
15 #include "format.h"
16 #include "integrity.h"
17
incfs_alloc_mount_info(struct super_block * sb,struct mount_options * options,struct path * backing_dir_path)18 struct mount_info *incfs_alloc_mount_info(struct super_block *sb,
19 struct mount_options *options,
20 struct path *backing_dir_path)
21 {
22 struct mount_info *mi = NULL;
23 int error = 0;
24
25 mi = kzalloc(sizeof(*mi), GFP_NOFS);
26 if (!mi)
27 return ERR_PTR(-ENOMEM);
28
29 mi->mi_sb = sb;
30 mi->mi_options = *options;
31 mi->mi_backing_dir_path = *backing_dir_path;
32 mi->mi_owner = get_current_cred();
33 path_get(&mi->mi_backing_dir_path);
34 mutex_init(&mi->mi_dir_struct_mutex);
35 mutex_init(&mi->mi_pending_reads_mutex);
36 init_waitqueue_head(&mi->mi_pending_reads_notif_wq);
37 INIT_LIST_HEAD(&mi->mi_reads_list_head);
38
39 if (options->read_log_pages != 0) {
40 size_t buf_size = PAGE_SIZE * options->read_log_pages;
41
42 spin_lock_init(&mi->mi_log.rl_writer_lock);
43 init_waitqueue_head(&mi->mi_log.ml_notif_wq);
44
45 mi->mi_log.rl_size = buf_size / sizeof(*mi->mi_log.rl_ring_buf);
46 mi->mi_log.rl_ring_buf = kzalloc(buf_size, GFP_NOFS);
47 if (!mi->mi_log.rl_ring_buf) {
48 error = -ENOMEM;
49 goto err;
50 }
51 }
52
53 return mi;
54
55 err:
56 incfs_free_mount_info(mi);
57 return ERR_PTR(error);
58 }
59
incfs_free_mount_info(struct mount_info * mi)60 void incfs_free_mount_info(struct mount_info *mi)
61 {
62 if (!mi)
63 return;
64
65 dput(mi->mi_index_dir);
66 path_put(&mi->mi_backing_dir_path);
67 mutex_destroy(&mi->mi_dir_struct_mutex);
68 mutex_destroy(&mi->mi_pending_reads_mutex);
69 put_cred(mi->mi_owner);
70 kfree(mi->mi_log.rl_ring_buf);
71 kfree(mi);
72 }
73
data_file_segment_init(struct data_file_segment * segment)74 static void data_file_segment_init(struct data_file_segment *segment)
75 {
76 init_waitqueue_head(&segment->new_data_arrival_wq);
77 mutex_init(&segment->blockmap_mutex);
78 INIT_LIST_HEAD(&segment->reads_list_head);
79 }
80
data_file_segment_destroy(struct data_file_segment * segment)81 static void data_file_segment_destroy(struct data_file_segment *segment)
82 {
83 mutex_destroy(&segment->blockmap_mutex);
84 }
85
incfs_open_data_file(struct mount_info * mi,struct file * bf)86 struct data_file *incfs_open_data_file(struct mount_info *mi, struct file *bf)
87 {
88 struct data_file *df = NULL;
89 struct backing_file_context *bfc = NULL;
90 int md_records;
91 u64 size;
92 int error = 0;
93 int i;
94
95 if (!bf || !mi)
96 return ERR_PTR(-EFAULT);
97
98 if (!S_ISREG(bf->f_inode->i_mode))
99 return ERR_PTR(-EBADF);
100
101 bfc = incfs_alloc_bfc(bf);
102 if (IS_ERR(bfc))
103 return ERR_CAST(bfc);
104
105 df = kzalloc(sizeof(*df), GFP_NOFS);
106 if (!df) {
107 error = -ENOMEM;
108 goto out;
109 }
110
111 df->df_backing_file_context = bfc;
112 df->df_mount_info = mi;
113 for (i = 0; i < ARRAY_SIZE(df->df_segments); i++)
114 data_file_segment_init(&df->df_segments[i]);
115
116 error = mutex_lock_interruptible(&bfc->bc_mutex);
117 if (error)
118 goto out;
119 error = incfs_read_file_header(bfc, &df->df_metadata_off,
120 &df->df_id, &size);
121 mutex_unlock(&bfc->bc_mutex);
122
123 if (error)
124 goto out;
125
126 df->df_size = size;
127 if (size > 0)
128 df->df_block_count = get_blocks_count_for_size(size);
129
130 md_records = incfs_scan_metadata_chain(df);
131 if (md_records < 0)
132 error = md_records;
133
134 out:
135 if (error) {
136 incfs_free_bfc(bfc);
137 df->df_backing_file_context = NULL;
138 incfs_free_data_file(df);
139 return ERR_PTR(error);
140 }
141 return df;
142 }
143
incfs_free_data_file(struct data_file * df)144 void incfs_free_data_file(struct data_file *df)
145 {
146 int i;
147
148 if (!df)
149 return;
150
151 incfs_free_mtree(df->df_hash_tree);
152 for (i = 0; i < ARRAY_SIZE(df->df_segments); i++)
153 data_file_segment_destroy(&df->df_segments[i]);
154 incfs_free_bfc(df->df_backing_file_context);
155 kfree(df);
156 }
157
make_inode_ready_for_data_ops(struct mount_info * mi,struct inode * inode,struct file * backing_file)158 int make_inode_ready_for_data_ops(struct mount_info *mi,
159 struct inode *inode,
160 struct file *backing_file)
161 {
162 struct inode_info *node = get_incfs_node(inode);
163 struct data_file *df = NULL;
164 int err = 0;
165
166 inode_lock(inode);
167 if (S_ISREG(inode->i_mode)) {
168 if (!node->n_file) {
169 df = incfs_open_data_file(mi, backing_file);
170
171 if (IS_ERR(df))
172 err = PTR_ERR(df);
173 else
174 node->n_file = df;
175 }
176 } else
177 err = -EBADF;
178 inode_unlock(inode);
179 return err;
180 }
181
incfs_open_dir_file(struct mount_info * mi,struct file * bf)182 struct dir_file *incfs_open_dir_file(struct mount_info *mi, struct file *bf)
183 {
184 struct dir_file *dir = NULL;
185
186 if (!S_ISDIR(bf->f_inode->i_mode))
187 return ERR_PTR(-EBADF);
188
189 dir = kzalloc(sizeof(*dir), GFP_NOFS);
190 if (!dir)
191 return ERR_PTR(-ENOMEM);
192
193 dir->backing_dir = get_file(bf);
194 dir->mount_info = mi;
195 return dir;
196 }
197
incfs_free_dir_file(struct dir_file * dir)198 void incfs_free_dir_file(struct dir_file *dir)
199 {
200 if (!dir)
201 return;
202 if (dir->backing_dir)
203 fput(dir->backing_dir);
204 kfree(dir);
205 }
206
decompress(struct mem_range src,struct mem_range dst)207 static ssize_t decompress(struct mem_range src, struct mem_range dst)
208 {
209 int result = LZ4_decompress_safe(src.data, dst.data, src.len, dst.len);
210
211 if (result < 0)
212 return -EBADMSG;
213
214 return result;
215 }
216
log_block_read(struct mount_info * mi,incfs_uuid_t * id,int block_index,bool timed_out)217 static void log_block_read(struct mount_info *mi, incfs_uuid_t *id,
218 int block_index, bool timed_out)
219 {
220 struct read_log *log = &mi->mi_log;
221 struct read_log_state state;
222 s64 now_us = ktime_to_us(ktime_get());
223 struct read_log_record record = {
224 .file_id = *id,
225 .block_index = block_index,
226 .timed_out = timed_out,
227 .timestamp_us = now_us
228 };
229
230 if (log->rl_size == 0)
231 return;
232
233 spin_lock(&log->rl_writer_lock);
234 state = READ_ONCE(log->rl_state);
235 log->rl_ring_buf[state.next_index] = record;
236 if (++state.next_index == log->rl_size) {
237 state.next_index = 0;
238 ++state.current_pass_no;
239 }
240 WRITE_ONCE(log->rl_state, state);
241 spin_unlock(&log->rl_writer_lock);
242
243 wake_up_all(&log->ml_notif_wq);
244 }
245
validate_hash_tree(struct file * bf,struct data_file * df,int block_index,struct mem_range data,u8 * buf)246 static int validate_hash_tree(struct file *bf, struct data_file *df,
247 int block_index, struct mem_range data, u8 *buf)
248 {
249 u8 digest[INCFS_MAX_HASH_SIZE] = {};
250 struct mtree *tree = NULL;
251 struct ondisk_signature *sig = NULL;
252 struct mem_range calc_digest_rng;
253 struct mem_range saved_digest_rng;
254 struct mem_range root_hash_rng;
255 int digest_size;
256 int hash_block_index = block_index;
257 int hash_per_block;
258 int lvl = 0;
259 int res;
260
261 tree = df->df_hash_tree;
262 sig = df->df_signature;
263 if (!tree || !sig)
264 return 0;
265
266 digest_size = tree->alg->digest_size;
267 hash_per_block = INCFS_DATA_FILE_BLOCK_SIZE / digest_size;
268 calc_digest_rng = range(digest, digest_size);
269 res = incfs_calc_digest(tree->alg, data, calc_digest_rng);
270 if (res)
271 return res;
272
273 for (lvl = 0; lvl < tree->depth; lvl++) {
274 loff_t lvl_off = tree->hash_level_suboffset[lvl] +
275 sig->mtree_offset;
276 loff_t hash_block_off = lvl_off +
277 round_down(hash_block_index * digest_size,
278 INCFS_DATA_FILE_BLOCK_SIZE);
279 size_t hash_off_in_block = hash_block_index * digest_size
280 % INCFS_DATA_FILE_BLOCK_SIZE;
281 struct mem_range buf_range = range(buf,
282 INCFS_DATA_FILE_BLOCK_SIZE);
283 ssize_t read_res = incfs_kread(bf, buf,
284 INCFS_DATA_FILE_BLOCK_SIZE, hash_block_off);
285
286 if (read_res < 0)
287 return read_res;
288 if (read_res != INCFS_DATA_FILE_BLOCK_SIZE)
289 return -EIO;
290
291 saved_digest_rng = range(buf + hash_off_in_block, digest_size);
292 if (!incfs_equal_ranges(calc_digest_rng, saved_digest_rng)) {
293 int i;
294 bool zero = true;
295
296 pr_debug("incfs: Hash mismatch lvl:%d blk:%d\n",
297 lvl, block_index);
298 for (i = 0; i < saved_digest_rng.len; ++i)
299 if (saved_digest_rng.data[i]) {
300 zero = false;
301 break;
302 }
303
304 if (zero)
305 pr_debug("incfs: Note saved_digest all zero - did you forget to load the hashes?\n");
306 return -EBADMSG;
307 }
308
309 res = incfs_calc_digest(tree->alg, buf_range, calc_digest_rng);
310 if (res)
311 return res;
312 hash_block_index /= hash_per_block;
313 }
314
315 root_hash_rng = range(tree->root_hash, digest_size);
316 if (!incfs_equal_ranges(calc_digest_rng, root_hash_rng)) {
317 pr_debug("incfs: Root hash mismatch blk:%d\n", block_index);
318 return -EBADMSG;
319 }
320 return 0;
321 }
322
revalidate_signature(struct file * bf,struct data_file * df)323 static int revalidate_signature(struct file *bf, struct data_file *df)
324 {
325 struct ondisk_signature *sig = df->df_signature;
326 struct mem_range root_hash = {};
327 int result = 0;
328 u8 *sig_buf = NULL;
329 u8 *add_data_buf = NULL;
330 ssize_t read_res;
331
332 /* File has no signature. */
333 if (!sig || !df->df_hash_tree || sig->sig_size == 0)
334 return 0;
335
336 /* Signature has already been validated. */
337 if (df->df_signature_validated)
338 return 0;
339
340 add_data_buf = kzalloc(sig->add_data_size, GFP_NOFS);
341 if (!add_data_buf) {
342 result = -ENOMEM;
343 goto out;
344 }
345
346 read_res = incfs_kread(bf, add_data_buf, sig->add_data_size,
347 sig->add_data_offset);
348 if (read_res < 0) {
349 result = read_res;
350 goto out;
351 }
352 if (read_res != sig->add_data_size) {
353 result = -EIO;
354 goto out;
355 }
356
357 sig_buf = kzalloc(sig->sig_size, GFP_NOFS);
358 if (!sig_buf) {
359 result = -ENOMEM;
360 goto out;
361 }
362
363 read_res = incfs_kread(bf, sig_buf, sig->sig_size, sig->sig_offset);
364 if (read_res < 0) {
365 result = read_res;
366 goto out;
367 }
368 if (read_res != sig->sig_size) {
369 result = -EIO;
370 goto out;
371 }
372
373 root_hash = range(df->df_hash_tree->root_hash,
374 df->df_hash_tree->alg->digest_size);
375
376 result = incfs_validate_pkcs7_signature(
377 range(sig_buf, sig->sig_size),
378 root_hash,
379 range(add_data_buf, sig->add_data_size));
380
381 if (result == 0)
382 df->df_signature_validated = true;
383 out:
384 kfree(sig_buf);
385 kfree(add_data_buf);
386 return result;
387 }
388
get_file_segment(struct data_file * df,int block_index)389 static struct data_file_segment *get_file_segment(struct data_file *df,
390 int block_index)
391 {
392 int seg_idx = block_index % ARRAY_SIZE(df->df_segments);
393
394 return &df->df_segments[seg_idx];
395 }
396
is_data_block_present(struct data_file_block * block)397 static bool is_data_block_present(struct data_file_block *block)
398 {
399 return (block->db_backing_file_data_offset != 0) &&
400 (block->db_stored_size != 0);
401 }
402
get_data_file_block(struct data_file * df,int index,struct data_file_block * res_block)403 static int get_data_file_block(struct data_file *df, int index,
404 struct data_file_block *res_block)
405 {
406 struct incfs_blockmap_entry bme = {};
407 struct backing_file_context *bfc = NULL;
408 loff_t blockmap_off = 0;
409 u16 flags = 0;
410 int error = 0;
411
412 if (!df || !res_block)
413 return -EFAULT;
414
415 blockmap_off = df->df_blockmap_off;
416 bfc = df->df_backing_file_context;
417
418 if (index < 0 || index >= df->df_block_count || blockmap_off == 0)
419 return -EINVAL;
420
421 error = incfs_read_blockmap_entry(bfc, index, blockmap_off, &bme);
422 if (error)
423 return error;
424
425 flags = le16_to_cpu(bme.me_flags);
426 res_block->db_backing_file_data_offset =
427 le16_to_cpu(bme.me_data_offset_hi);
428 res_block->db_backing_file_data_offset <<= 32;
429 res_block->db_backing_file_data_offset |=
430 le32_to_cpu(bme.me_data_offset_lo);
431 res_block->db_stored_size = le16_to_cpu(bme.me_data_size);
432 res_block->db_comp_alg = (flags & INCFS_BLOCK_COMPRESSED_LZ4) ?
433 COMPRESSION_LZ4 :
434 COMPRESSION_NONE;
435 return 0;
436 }
437
is_read_done(struct pending_read * read)438 static bool is_read_done(struct pending_read *read)
439 {
440 return atomic_read_acquire(&read->done) != 0;
441 }
442
set_read_done(struct pending_read * read)443 static void set_read_done(struct pending_read *read)
444 {
445 atomic_set_release(&read->done, 1);
446 }
447
448 /*
449 * Notifies a given data file about pending read from a given block.
450 * Returns a new pending read entry.
451 */
add_pending_read(struct data_file * df,int block_index)452 static struct pending_read *add_pending_read(struct data_file *df,
453 int block_index)
454 {
455 struct pending_read *result = NULL;
456 struct data_file_segment *segment = NULL;
457 struct mount_info *mi = NULL;
458
459 segment = get_file_segment(df, block_index);
460 mi = df->df_mount_info;
461
462 result = kzalloc(sizeof(*result), GFP_NOFS);
463 if (!result)
464 return NULL;
465
466 result->file_id = df->df_id;
467 result->block_index = block_index;
468 result->timestamp_us = ktime_to_us(ktime_get());
469
470 mutex_lock(&mi->mi_pending_reads_mutex);
471
472 result->serial_number = ++mi->mi_last_pending_read_number;
473 mi->mi_pending_reads_count++;
474
475 list_add(&result->mi_reads_list, &mi->mi_reads_list_head);
476 list_add(&result->segment_reads_list, &segment->reads_list_head);
477 mutex_unlock(&mi->mi_pending_reads_mutex);
478
479 wake_up_all(&mi->mi_pending_reads_notif_wq);
480 return result;
481 }
482
483 /* Notifies a given data file that pending read is completed. */
remove_pending_read(struct data_file * df,struct pending_read * read)484 static void remove_pending_read(struct data_file *df, struct pending_read *read)
485 {
486 struct mount_info *mi = NULL;
487
488 if (!df || !read) {
489 WARN_ON(!df);
490 WARN_ON(!read);
491 return;
492 }
493
494 mi = df->df_mount_info;
495
496 mutex_lock(&mi->mi_pending_reads_mutex);
497 list_del(&read->mi_reads_list);
498 list_del(&read->segment_reads_list);
499
500 mi->mi_pending_reads_count--;
501 mutex_unlock(&mi->mi_pending_reads_mutex);
502
503 kfree(read);
504 }
505
notify_pending_reads(struct mount_info * mi,struct data_file_segment * segment,int index)506 static void notify_pending_reads(struct mount_info *mi,
507 struct data_file_segment *segment,
508 int index)
509 {
510 struct pending_read *entry = NULL;
511
512 /* Notify pending reads waiting for this block. */
513 mutex_lock(&mi->mi_pending_reads_mutex);
514 list_for_each_entry(entry, &segment->reads_list_head,
515 segment_reads_list) {
516 if (entry->block_index == index)
517 set_read_done(entry);
518 }
519 mutex_unlock(&mi->mi_pending_reads_mutex);
520 wake_up_all(&segment->new_data_arrival_wq);
521 }
522
wait_for_data_block(struct data_file * df,int block_index,int timeout_ms,struct data_file_block * res_block)523 static int wait_for_data_block(struct data_file *df, int block_index,
524 int timeout_ms,
525 struct data_file_block *res_block)
526 {
527 struct data_file_block block = {};
528 struct data_file_segment *segment = NULL;
529 struct pending_read *read = NULL;
530 struct mount_info *mi = NULL;
531 int error = 0;
532 int wait_res = 0;
533
534 if (!df || !res_block)
535 return -EFAULT;
536
537 if (block_index < 0 || block_index >= df->df_block_count)
538 return -EINVAL;
539
540 if (df->df_blockmap_off <= 0)
541 return -ENODATA;
542
543 segment = get_file_segment(df, block_index);
544 error = mutex_lock_interruptible(&segment->blockmap_mutex);
545 if (error)
546 return error;
547
548 /* Look up the given block */
549 error = get_data_file_block(df, block_index, &block);
550
551 /* If it's not found, create a pending read */
552 if (!error && !is_data_block_present(&block) && timeout_ms != 0)
553 read = add_pending_read(df, block_index);
554
555 mutex_unlock(&segment->blockmap_mutex);
556 if (error)
557 return error;
558
559 /* If the block was found, just return it. No need to wait. */
560 if (is_data_block_present(&block)) {
561 *res_block = block;
562 return 0;
563 }
564
565 mi = df->df_mount_info;
566
567 if (timeout_ms == 0) {
568 log_block_read(mi, &df->df_id, block_index,
569 true /*timed out*/);
570 return -ETIME;
571 }
572
573 if (!read)
574 return -ENOMEM;
575
576 /* Wait for notifications about block's arrival */
577 wait_res =
578 wait_event_interruptible_timeout(segment->new_data_arrival_wq,
579 (is_read_done(read)),
580 msecs_to_jiffies(timeout_ms));
581
582 /* Woke up, the pending read is no longer needed. */
583 remove_pending_read(df, read);
584 read = NULL;
585
586 if (wait_res == 0) {
587 /* Wait has timed out */
588 log_block_read(mi, &df->df_id, block_index,
589 true /*timed out*/);
590 return -ETIME;
591 }
592 if (wait_res < 0) {
593 /*
594 * Only ERESTARTSYS is really expected here when a signal
595 * comes while we wait.
596 */
597 return wait_res;
598 }
599
600 error = mutex_lock_interruptible(&segment->blockmap_mutex);
601 if (error)
602 return error;
603
604 /*
605 * Re-read block's info now, it has just arrived and
606 * should be available.
607 */
608 error = get_data_file_block(df, block_index, &block);
609 if (!error) {
610 if (is_data_block_present(&block))
611 *res_block = block;
612 else {
613 /*
614 * Somehow wait finished successfully bug block still
615 * can't be found. It's not normal.
616 */
617 pr_warn("incfs:Wait succeeded, but block not found.\n");
618 error = -ENODATA;
619 }
620 }
621
622 mutex_unlock(&segment->blockmap_mutex);
623 return error;
624 }
625
incfs_read_data_file_block(struct mem_range dst,struct data_file * df,int index,int timeout_ms,struct mem_range tmp)626 ssize_t incfs_read_data_file_block(struct mem_range dst, struct data_file *df,
627 int index, int timeout_ms,
628 struct mem_range tmp)
629 {
630 loff_t pos;
631 ssize_t result;
632 size_t bytes_to_read;
633 struct mount_info *mi = NULL;
634 struct file *bf = NULL;
635 struct data_file_block block = {};
636
637 if (!dst.data || !df)
638 return -EFAULT;
639
640 if (tmp.len < 2 * INCFS_DATA_FILE_BLOCK_SIZE)
641 return -ERANGE;
642
643 mi = df->df_mount_info;
644 bf = df->df_backing_file_context->bc_file;
645
646 result = wait_for_data_block(df, index, timeout_ms, &block);
647 if (result < 0)
648 goto out;
649
650 pos = block.db_backing_file_data_offset;
651 if (block.db_comp_alg == COMPRESSION_NONE) {
652 bytes_to_read = min(dst.len, block.db_stored_size);
653 result = incfs_kread(bf, dst.data, bytes_to_read, pos);
654
655 /* Some data was read, but not enough */
656 if (result >= 0 && result != bytes_to_read)
657 result = -EIO;
658 } else {
659 bytes_to_read = min(tmp.len, block.db_stored_size);
660 result = incfs_kread(bf, tmp.data, bytes_to_read, pos);
661 if (result == bytes_to_read) {
662 result =
663 decompress(range(tmp.data, bytes_to_read), dst);
664 if (result < 0) {
665 const char *name =
666 bf->f_path.dentry->d_name.name;
667
668 pr_warn_once("incfs: Decompression error. %s",
669 name);
670 }
671 } else if (result >= 0) {
672 /* Some data was read, but not enough */
673 result = -EIO;
674 }
675 }
676
677 if (result > 0) {
678 int err = validate_hash_tree(bf, df, index, dst, tmp.data);
679
680 if (err < 0)
681 result = err;
682 }
683
684 if (result > 0) {
685 int err = revalidate_signature(bf, df);
686
687 if (err < 0)
688 result = err;
689 }
690
691 if (result >= 0)
692 log_block_read(mi, &df->df_id, index, false /*timed out*/);
693
694 out:
695 return result;
696 }
697
incfs_process_new_data_block(struct data_file * df,struct incfs_new_data_block * block,u8 * data)698 int incfs_process_new_data_block(struct data_file *df,
699 struct incfs_new_data_block *block, u8 *data)
700 {
701 struct mount_info *mi = NULL;
702 struct backing_file_context *bfc = NULL;
703 struct data_file_segment *segment = NULL;
704 struct data_file_block existing_block = {};
705 u16 flags = 0;
706 int error = 0;
707
708 if (!df || !block)
709 return -EFAULT;
710
711 bfc = df->df_backing_file_context;
712 mi = df->df_mount_info;
713
714 if (block->block_index >= df->df_block_count)
715 return -ERANGE;
716
717 segment = get_file_segment(df, block->block_index);
718 if (!segment)
719 return -EFAULT;
720 if (block->compression == COMPRESSION_LZ4)
721 flags |= INCFS_BLOCK_COMPRESSED_LZ4;
722
723 error = mutex_lock_interruptible(&segment->blockmap_mutex);
724 if (error)
725 return error;
726
727 error = get_data_file_block(df, block->block_index, &existing_block);
728 if (error)
729 goto unlock;
730 if (is_data_block_present(&existing_block)) {
731 /* Block is already present, nothing to do here */
732 goto unlock;
733 }
734
735 error = mutex_lock_interruptible(&bfc->bc_mutex);
736 if (!error) {
737 error = incfs_write_data_block_to_backing_file(
738 bfc, range(data, block->data_len), block->block_index,
739 df->df_blockmap_off, flags);
740 mutex_unlock(&bfc->bc_mutex);
741 }
742 if (!error)
743 notify_pending_reads(mi, segment, block->block_index);
744
745 unlock:
746 mutex_unlock(&segment->blockmap_mutex);
747 if (error)
748 pr_debug("incfs: %s %d error: %d\n", __func__,
749 block->block_index, error);
750 return error;
751 }
752
incfs_read_file_signature(struct data_file * df,struct mem_range dst)753 int incfs_read_file_signature(struct data_file *df, struct mem_range dst)
754 {
755 struct file *bf = df->df_backing_file_context->bc_file;
756 struct ondisk_signature *sig;
757 int read_res = 0;
758
759 if (!dst.data)
760 return -EFAULT;
761
762 sig = df->df_signature;
763 if (!sig)
764 return 0;
765
766 if (dst.len < sig->sig_size)
767 return -E2BIG;
768
769 read_res = incfs_kread(bf, dst.data, sig->sig_size, sig->sig_offset);
770
771 if (read_res < 0)
772 return read_res;
773
774 if (read_res != sig->sig_size)
775 return -EIO;
776
777 return read_res;
778 }
779
incfs_process_new_hash_block(struct data_file * df,struct incfs_new_data_block * block,u8 * data)780 int incfs_process_new_hash_block(struct data_file *df,
781 struct incfs_new_data_block *block, u8 *data)
782 {
783 struct backing_file_context *bfc = NULL;
784 struct mount_info *mi = NULL;
785 struct mtree *hash_tree = NULL;
786 struct ondisk_signature *sig = NULL;
787 loff_t hash_area_base = 0;
788 loff_t hash_area_size = 0;
789 int error = 0;
790
791 if (!df || !block)
792 return -EFAULT;
793
794 if (!(block->flags & INCFS_BLOCK_FLAGS_HASH))
795 return -EINVAL;
796
797 bfc = df->df_backing_file_context;
798 mi = df->df_mount_info;
799
800 if (!df)
801 return -ENOENT;
802
803 hash_tree = df->df_hash_tree;
804 sig = df->df_signature;
805 if (!hash_tree || !sig || sig->mtree_offset == 0)
806 return -ENOTSUPP;
807
808 hash_area_base = sig->mtree_offset;
809 hash_area_size = sig->mtree_size;
810 if (hash_area_size < block->block_index * INCFS_DATA_FILE_BLOCK_SIZE
811 + block->data_len) {
812 /* Hash block goes beyond dedicated hash area of this file. */
813 return -ERANGE;
814 }
815
816 error = mutex_lock_interruptible(&bfc->bc_mutex);
817 if (!error)
818 error = incfs_write_hash_block_to_backing_file(
819 bfc, range(data, block->data_len), block->block_index,
820 hash_area_base);
821 mutex_unlock(&bfc->bc_mutex);
822 return error;
823 }
824
process_blockmap_md(struct incfs_blockmap * bm,struct metadata_handler * handler)825 static int process_blockmap_md(struct incfs_blockmap *bm,
826 struct metadata_handler *handler)
827 {
828 struct data_file *df = handler->context;
829 int error = 0;
830 loff_t base_off = le64_to_cpu(bm->m_base_offset);
831 u32 block_count = le32_to_cpu(bm->m_block_count);
832
833 if (!df)
834 return -EFAULT;
835
836 if (df->df_block_count != block_count)
837 return -EBADMSG;
838
839 df->df_blockmap_off = base_off;
840 return error;
841 }
842
process_file_attr_md(struct incfs_file_attr * fa,struct metadata_handler * handler)843 static int process_file_attr_md(struct incfs_file_attr *fa,
844 struct metadata_handler *handler)
845 {
846 struct data_file *df = handler->context;
847 u16 attr_size = le16_to_cpu(fa->fa_size);
848
849 if (!df)
850 return -EFAULT;
851
852 if (attr_size > INCFS_MAX_FILE_ATTR_SIZE)
853 return -E2BIG;
854
855 df->n_attr.fa_value_offset = le64_to_cpu(fa->fa_offset);
856 df->n_attr.fa_value_size = attr_size;
857 df->n_attr.fa_crc = le32_to_cpu(fa->fa_crc);
858
859 return 0;
860 }
861
process_file_signature_md(struct incfs_file_signature * sg,struct metadata_handler * handler)862 static int process_file_signature_md(struct incfs_file_signature *sg,
863 struct metadata_handler *handler)
864 {
865 struct data_file *df = handler->context;
866 struct mtree *hash_tree = NULL;
867 struct ondisk_signature *signature = NULL;
868 int error = 0;
869 loff_t base_tree_off = le64_to_cpu(sg->sg_hash_tree_offset);
870 u32 tree_size = le32_to_cpu(sg->sg_hash_tree_size);
871 loff_t sig_off = le64_to_cpu(sg->sg_sig_offset);
872 u32 sig_size = le32_to_cpu(sg->sg_sig_size);
873 loff_t add_data_off = le64_to_cpu(sg->sg_add_data_offset);
874 u32 add_data_size = le32_to_cpu(sg->sg_add_data_size);
875
876 if (!df)
877 return -ENOENT;
878
879 signature = kzalloc(sizeof(*signature), GFP_NOFS);
880 if (!signature) {
881 error = -ENOMEM;
882 goto out;
883 }
884
885 signature->add_data_offset = add_data_off;
886 signature->add_data_size = add_data_size;
887 signature->sig_offset = sig_off;
888 signature->sig_size = sig_size;
889 signature->mtree_offset = base_tree_off;
890 signature->mtree_size = tree_size;
891
892 hash_tree = incfs_alloc_mtree(sg->sg_hash_alg, df->df_block_count,
893 range(sg->sg_root_hash, sizeof(sg->sg_root_hash)));
894 if (IS_ERR(hash_tree)) {
895 error = PTR_ERR(hash_tree);
896 hash_tree = NULL;
897 goto out;
898 }
899 if (hash_tree->hash_tree_area_size != tree_size) {
900 error = -EINVAL;
901 goto out;
902 }
903 if (tree_size > 0 && handler->md_record_offset <= base_tree_off) {
904 error = -EINVAL;
905 goto out;
906 }
907 if (handler->md_record_offset <= signature->add_data_offset ||
908 handler->md_record_offset <= signature->sig_offset) {
909 error = -EINVAL;
910 goto out;
911 }
912 df->df_hash_tree = hash_tree;
913 df->df_signature = signature;
914 out:
915 if (error) {
916 incfs_free_mtree(hash_tree);
917 kfree(signature);
918 }
919
920 return error;
921 }
922
incfs_scan_metadata_chain(struct data_file * df)923 int incfs_scan_metadata_chain(struct data_file *df)
924 {
925 struct metadata_handler *handler = NULL;
926 int result = 0;
927 int records_count = 0;
928 int error = 0;
929 struct backing_file_context *bfc = NULL;
930
931 if (!df || !df->df_backing_file_context)
932 return -EFAULT;
933
934 bfc = df->df_backing_file_context;
935
936 handler = kzalloc(sizeof(*handler), GFP_NOFS);
937 if (!handler)
938 return -ENOMEM;
939
940 /* No writing to the backing file while it's being scanned. */
941 error = mutex_lock_interruptible(&bfc->bc_mutex);
942 if (error)
943 goto out;
944
945 /* Reading superblock */
946 handler->md_record_offset = df->df_metadata_off;
947 handler->context = df;
948 handler->handle_blockmap = process_blockmap_md;
949 handler->handle_file_attr = process_file_attr_md;
950 handler->handle_signature = process_file_signature_md;
951
952 pr_debug("incfs: Starting reading incfs-metadata records at offset %lld\n",
953 handler->md_record_offset);
954 while (handler->md_record_offset > 0) {
955 error = incfs_read_next_metadata_record(bfc, handler);
956 if (error) {
957 pr_warn("incfs: Error during reading incfs-metadata record. Offset: %lld Record #%d Error code: %d\n",
958 handler->md_record_offset, records_count + 1,
959 -error);
960 break;
961 }
962 records_count++;
963 }
964 if (error) {
965 pr_debug("incfs: Error %d after reading %d incfs-metadata records.\n",
966 -error, records_count);
967 result = error;
968 } else {
969 pr_debug("incfs: Finished reading %d incfs-metadata records.\n",
970 records_count);
971 result = records_count;
972 }
973 mutex_unlock(&bfc->bc_mutex);
974 out:
975 kfree(handler);
976 return result;
977 }
978
979 /*
980 * Quickly checks if there are pending reads with a serial number larger
981 * than a given one.
982 */
incfs_fresh_pending_reads_exist(struct mount_info * mi,int last_number)983 bool incfs_fresh_pending_reads_exist(struct mount_info *mi, int last_number)
984 {
985 bool result = false;
986
987 mutex_lock(&mi->mi_pending_reads_mutex);
988 result = (mi->mi_last_pending_read_number > last_number) &&
989 (mi->mi_pending_reads_count > 0);
990 mutex_unlock(&mi->mi_pending_reads_mutex);
991 return result;
992 }
993
incfs_collect_pending_reads(struct mount_info * mi,int sn_lowerbound,struct incfs_pending_read_info * reads,int reads_size)994 int incfs_collect_pending_reads(struct mount_info *mi, int sn_lowerbound,
995 struct incfs_pending_read_info *reads,
996 int reads_size)
997 {
998 int reported_reads = 0;
999 struct pending_read *entry = NULL;
1000
1001 if (!mi)
1002 return -EFAULT;
1003
1004 if (reads_size <= 0)
1005 return 0;
1006
1007 mutex_lock(&mi->mi_pending_reads_mutex);
1008
1009 if (mi->mi_last_pending_read_number <= sn_lowerbound
1010 || mi->mi_pending_reads_count == 0)
1011 goto unlock;
1012
1013 list_for_each_entry(entry, &mi->mi_reads_list_head, mi_reads_list) {
1014 if (entry->serial_number <= sn_lowerbound)
1015 continue;
1016
1017 reads[reported_reads].file_id = entry->file_id;
1018 reads[reported_reads].block_index = entry->block_index;
1019 reads[reported_reads].serial_number = entry->serial_number;
1020 reads[reported_reads].timestamp_us = entry->timestamp_us;
1021 /* reads[reported_reads].kind = INCFS_READ_KIND_PENDING; */
1022
1023 reported_reads++;
1024 if (reported_reads >= reads_size)
1025 break;
1026 }
1027
1028 unlock:
1029 mutex_unlock(&mi->mi_pending_reads_mutex);
1030
1031 return reported_reads;
1032 }
1033
incfs_get_log_state(struct mount_info * mi)1034 struct read_log_state incfs_get_log_state(struct mount_info *mi)
1035 {
1036 struct read_log *log = &mi->mi_log;
1037 struct read_log_state result;
1038
1039 spin_lock(&log->rl_writer_lock);
1040 result = READ_ONCE(log->rl_state);
1041 spin_unlock(&log->rl_writer_lock);
1042 return result;
1043 }
1044
calc_record_count(const struct read_log_state * state,int rl_size)1045 static u64 calc_record_count(const struct read_log_state *state, int rl_size)
1046 {
1047 return state->current_pass_no * (u64)rl_size + state->next_index;
1048 }
1049
incfs_get_uncollected_logs_count(struct mount_info * mi,struct read_log_state state)1050 int incfs_get_uncollected_logs_count(struct mount_info *mi,
1051 struct read_log_state state)
1052 {
1053 struct read_log *log = &mi->mi_log;
1054
1055 u64 count = calc_record_count(&log->rl_state, log->rl_size) -
1056 calc_record_count(&state, log->rl_size);
1057 return min_t(int, count, log->rl_size);
1058 }
1059
fill_pending_read_from_log_record(struct incfs_pending_read_info * dest,const struct read_log_record * src,struct read_log_state * state,u64 log_size)1060 static void fill_pending_read_from_log_record(
1061 struct incfs_pending_read_info *dest, const struct read_log_record *src,
1062 struct read_log_state *state, u64 log_size)
1063 {
1064 dest->file_id = src->file_id;
1065 dest->block_index = src->block_index;
1066 dest->serial_number =
1067 state->current_pass_no * log_size + state->next_index;
1068 dest->timestamp_us = src->timestamp_us;
1069 }
1070
incfs_collect_logged_reads(struct mount_info * mi,struct read_log_state * reader_state,struct incfs_pending_read_info * reads,int reads_size)1071 int incfs_collect_logged_reads(struct mount_info *mi,
1072 struct read_log_state *reader_state,
1073 struct incfs_pending_read_info *reads,
1074 int reads_size)
1075 {
1076 struct read_log *log = &mi->mi_log;
1077 struct read_log_state live_state = incfs_get_log_state(mi);
1078 u64 read_count = calc_record_count(reader_state, log->rl_size);
1079 u64 written_count = calc_record_count(&live_state, log->rl_size);
1080 int dst_idx;
1081
1082 if (reader_state->next_index >= log->rl_size ||
1083 read_count > written_count)
1084 return -ERANGE;
1085
1086 if (read_count == written_count)
1087 return 0;
1088
1089 if (read_count > written_count) {
1090 /* This reader is somehow ahead of the writer. */
1091 pr_debug("incfs: Log reader is ahead of writer\n");
1092 *reader_state = live_state;
1093 }
1094
1095 if (written_count - read_count > log->rl_size) {
1096 /*
1097 * Reading pointer is too far behind,
1098 * start from the record following the write pointer.
1099 */
1100 pr_debug("incfs: read pointer is behind, moving: %u/%u -> %u/%u / %u\n",
1101 (u32)reader_state->next_index,
1102 (u32)reader_state->current_pass_no,
1103 (u32)live_state.next_index,
1104 (u32)live_state.current_pass_no - 1, (u32)log->rl_size);
1105
1106 *reader_state = (struct read_log_state){
1107 .next_index = live_state.next_index,
1108 .current_pass_no = live_state.current_pass_no - 1,
1109 };
1110 }
1111
1112 for (dst_idx = 0; dst_idx < reads_size; dst_idx++) {
1113 if (reader_state->next_index == live_state.next_index &&
1114 reader_state->current_pass_no == live_state.current_pass_no)
1115 break;
1116
1117 fill_pending_read_from_log_record(
1118 &reads[dst_idx],
1119 &log->rl_ring_buf[reader_state->next_index],
1120 reader_state, log->rl_size);
1121
1122 reader_state->next_index++;
1123 if (reader_state->next_index == log->rl_size) {
1124 reader_state->next_index = 0;
1125 reader_state->current_pass_no++;
1126 }
1127 }
1128 return dst_idx;
1129 }
1130
incfs_equal_ranges(struct mem_range lhs,struct mem_range rhs)1131 bool incfs_equal_ranges(struct mem_range lhs, struct mem_range rhs)
1132 {
1133 if (lhs.len != rhs.len)
1134 return false;
1135 return memcmp(lhs.data, rhs.data, lhs.len) == 0;
1136 }
1137