1 /*
2 * This contains encryption functions for per-file encryption.
3 *
4 * Copyright (C) 2015, Google, Inc.
5 * Copyright (C) 2015, Motorola Mobility
6 *
7 * Written by Michael Halcrow, 2014.
8 *
9 * Filename encryption additions
10 * Uday Savagaonkar, 2014
11 * Encryption policy handling additions
12 * Ildar Muslukhov, 2014
13 * Add fscrypt_pullback_bio_page()
14 * Jaegeuk Kim, 2015.
15 *
16 * This has not yet undergone a rigorous security audit.
17 *
18 * The usage of AES-XTS should conform to recommendations in NIST
19 * Special Publication 800-38E and IEEE P1619/D16.
20 */
21
22 #include <linux/pagemap.h>
23 #include <linux/mempool.h>
24 #include <linux/module.h>
25 #include <linux/scatterlist.h>
26 #include <linux/ratelimit.h>
27 #include <linux/dcache.h>
28 #include <linux/namei.h>
29 #include <crypto/aes.h>
30 #include <crypto/skcipher.h>
31 #include "fscrypt_private.h"
32
33 static unsigned int num_prealloc_crypto_pages = 32;
34 static unsigned int num_prealloc_crypto_ctxs = 128;
35
36 module_param(num_prealloc_crypto_pages, uint, 0444);
37 MODULE_PARM_DESC(num_prealloc_crypto_pages,
38 "Number of crypto pages to preallocate");
39 module_param(num_prealloc_crypto_ctxs, uint, 0444);
40 MODULE_PARM_DESC(num_prealloc_crypto_ctxs,
41 "Number of crypto contexts to preallocate");
42
43 static mempool_t *fscrypt_bounce_page_pool = NULL;
44
45 static LIST_HEAD(fscrypt_free_ctxs);
46 static DEFINE_SPINLOCK(fscrypt_ctx_lock);
47
48 static struct workqueue_struct *fscrypt_read_workqueue;
49 static DEFINE_MUTEX(fscrypt_init_mutex);
50
51 static struct kmem_cache *fscrypt_ctx_cachep;
52 struct kmem_cache *fscrypt_info_cachep;
53
fscrypt_enqueue_decrypt_work(struct work_struct * work)54 void fscrypt_enqueue_decrypt_work(struct work_struct *work)
55 {
56 queue_work(fscrypt_read_workqueue, work);
57 }
58 EXPORT_SYMBOL(fscrypt_enqueue_decrypt_work);
59
60 /**
61 * fscrypt_release_ctx() - Releases an encryption context
62 * @ctx: The encryption context to release.
63 *
64 * If the encryption context was allocated from the pre-allocated pool, returns
65 * it to that pool. Else, frees it.
66 *
67 * If there's a bounce page in the context, this frees that.
68 */
fscrypt_release_ctx(struct fscrypt_ctx * ctx)69 void fscrypt_release_ctx(struct fscrypt_ctx *ctx)
70 {
71 unsigned long flags;
72
73 if (ctx->flags & FS_CTX_HAS_BOUNCE_BUFFER_FL && ctx->w.bounce_page) {
74 mempool_free(ctx->w.bounce_page, fscrypt_bounce_page_pool);
75 ctx->w.bounce_page = NULL;
76 }
77 ctx->w.control_page = NULL;
78 if (ctx->flags & FS_CTX_REQUIRES_FREE_ENCRYPT_FL) {
79 kmem_cache_free(fscrypt_ctx_cachep, ctx);
80 } else {
81 spin_lock_irqsave(&fscrypt_ctx_lock, flags);
82 list_add(&ctx->free_list, &fscrypt_free_ctxs);
83 spin_unlock_irqrestore(&fscrypt_ctx_lock, flags);
84 }
85 }
86 EXPORT_SYMBOL(fscrypt_release_ctx);
87
88 /**
89 * fscrypt_get_ctx() - Gets an encryption context
90 * @inode: The inode for which we are doing the crypto
91 * @gfp_flags: The gfp flag for memory allocation
92 *
93 * Allocates and initializes an encryption context.
94 *
95 * Return: An allocated and initialized encryption context on success; error
96 * value or NULL otherwise.
97 */
fscrypt_get_ctx(const struct inode * inode,gfp_t gfp_flags)98 struct fscrypt_ctx *fscrypt_get_ctx(const struct inode *inode, gfp_t gfp_flags)
99 {
100 struct fscrypt_ctx *ctx = NULL;
101 struct fscrypt_info *ci = inode->i_crypt_info;
102 unsigned long flags;
103
104 if (ci == NULL)
105 return ERR_PTR(-ENOKEY);
106
107 /*
108 * We first try getting the ctx from a free list because in
109 * the common case the ctx will have an allocated and
110 * initialized crypto tfm, so it's probably a worthwhile
111 * optimization. For the bounce page, we first try getting it
112 * from the kernel allocator because that's just about as fast
113 * as getting it from a list and because a cache of free pages
114 * should generally be a "last resort" option for a filesystem
115 * to be able to do its job.
116 */
117 spin_lock_irqsave(&fscrypt_ctx_lock, flags);
118 ctx = list_first_entry_or_null(&fscrypt_free_ctxs,
119 struct fscrypt_ctx, free_list);
120 if (ctx)
121 list_del(&ctx->free_list);
122 spin_unlock_irqrestore(&fscrypt_ctx_lock, flags);
123 if (!ctx) {
124 ctx = kmem_cache_zalloc(fscrypt_ctx_cachep, gfp_flags);
125 if (!ctx)
126 return ERR_PTR(-ENOMEM);
127 ctx->flags |= FS_CTX_REQUIRES_FREE_ENCRYPT_FL;
128 } else {
129 ctx->flags &= ~FS_CTX_REQUIRES_FREE_ENCRYPT_FL;
130 }
131 ctx->flags &= ~FS_CTX_HAS_BOUNCE_BUFFER_FL;
132 return ctx;
133 }
134 EXPORT_SYMBOL(fscrypt_get_ctx);
135
fscrypt_do_page_crypto(const struct inode * inode,fscrypt_direction_t rw,u64 lblk_num,struct page * src_page,struct page * dest_page,unsigned int len,unsigned int offs,gfp_t gfp_flags)136 int fscrypt_do_page_crypto(const struct inode *inode, fscrypt_direction_t rw,
137 u64 lblk_num, struct page *src_page,
138 struct page *dest_page, unsigned int len,
139 unsigned int offs, gfp_t gfp_flags)
140 {
141 struct {
142 __le64 index;
143 u8 padding[FS_IV_SIZE - sizeof(__le64)];
144 } iv;
145 struct skcipher_request *req = NULL;
146 DECLARE_CRYPTO_WAIT(wait);
147 struct scatterlist dst, src;
148 struct fscrypt_info *ci = inode->i_crypt_info;
149 struct crypto_skcipher *tfm = ci->ci_ctfm;
150 int res = 0;
151
152 BUG_ON(len == 0);
153
154 BUILD_BUG_ON(sizeof(iv) != FS_IV_SIZE);
155 BUILD_BUG_ON(AES_BLOCK_SIZE != FS_IV_SIZE);
156 iv.index = cpu_to_le64(lblk_num);
157 memset(iv.padding, 0, sizeof(iv.padding));
158
159 if (ci->ci_essiv_tfm != NULL) {
160 crypto_cipher_encrypt_one(ci->ci_essiv_tfm, (u8 *)&iv,
161 (u8 *)&iv);
162 }
163
164 req = skcipher_request_alloc(tfm, gfp_flags);
165 if (!req) {
166 printk_ratelimited(KERN_ERR
167 "%s: crypto_request_alloc() failed\n",
168 __func__);
169 return -ENOMEM;
170 }
171
172 skcipher_request_set_callback(
173 req, CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,
174 crypto_req_done, &wait);
175
176 sg_init_table(&dst, 1);
177 sg_set_page(&dst, dest_page, len, offs);
178 sg_init_table(&src, 1);
179 sg_set_page(&src, src_page, len, offs);
180 skcipher_request_set_crypt(req, &src, &dst, len, &iv);
181 if (rw == FS_DECRYPT)
182 res = crypto_wait_req(crypto_skcipher_decrypt(req), &wait);
183 else
184 res = crypto_wait_req(crypto_skcipher_encrypt(req), &wait);
185 skcipher_request_free(req);
186 if (res) {
187 printk_ratelimited(KERN_ERR
188 "%s: crypto_skcipher_encrypt() returned %d\n",
189 __func__, res);
190 return res;
191 }
192 return 0;
193 }
194
fscrypt_alloc_bounce_page(struct fscrypt_ctx * ctx,gfp_t gfp_flags)195 struct page *fscrypt_alloc_bounce_page(struct fscrypt_ctx *ctx,
196 gfp_t gfp_flags)
197 {
198 ctx->w.bounce_page = mempool_alloc(fscrypt_bounce_page_pool, gfp_flags);
199 if (ctx->w.bounce_page == NULL)
200 return ERR_PTR(-ENOMEM);
201 ctx->flags |= FS_CTX_HAS_BOUNCE_BUFFER_FL;
202 return ctx->w.bounce_page;
203 }
204
205 /**
206 * fscypt_encrypt_page() - Encrypts a page
207 * @inode: The inode for which the encryption should take place
208 * @page: The page to encrypt. Must be locked for bounce-page
209 * encryption.
210 * @len: Length of data to encrypt in @page and encrypted
211 * data in returned page.
212 * @offs: Offset of data within @page and returned
213 * page holding encrypted data.
214 * @lblk_num: Logical block number. This must be unique for multiple
215 * calls with same inode, except when overwriting
216 * previously written data.
217 * @gfp_flags: The gfp flag for memory allocation
218 *
219 * Encrypts @page using the ctx encryption context. Performs encryption
220 * either in-place or into a newly allocated bounce page.
221 * Called on the page write path.
222 *
223 * Bounce page allocation is the default.
224 * In this case, the contents of @page are encrypted and stored in an
225 * allocated bounce page. @page has to be locked and the caller must call
226 * fscrypt_restore_control_page() on the returned ciphertext page to
227 * release the bounce buffer and the encryption context.
228 *
229 * In-place encryption is used by setting the FS_CFLG_OWN_PAGES flag in
230 * fscrypt_operations. Here, the input-page is returned with its content
231 * encrypted.
232 *
233 * Return: A page with the encrypted content on success. Else, an
234 * error value or NULL.
235 */
fscrypt_encrypt_page(const struct inode * inode,struct page * page,unsigned int len,unsigned int offs,u64 lblk_num,gfp_t gfp_flags)236 struct page *fscrypt_encrypt_page(const struct inode *inode,
237 struct page *page,
238 unsigned int len,
239 unsigned int offs,
240 u64 lblk_num, gfp_t gfp_flags)
241
242 {
243 struct fscrypt_ctx *ctx;
244 struct page *ciphertext_page = page;
245 int err;
246
247 BUG_ON(len % FS_CRYPTO_BLOCK_SIZE != 0);
248
249 if (inode->i_sb->s_cop->flags & FS_CFLG_OWN_PAGES) {
250 /* with inplace-encryption we just encrypt the page */
251 err = fscrypt_do_page_crypto(inode, FS_ENCRYPT, lblk_num, page,
252 ciphertext_page, len, offs,
253 gfp_flags);
254 if (err)
255 return ERR_PTR(err);
256
257 return ciphertext_page;
258 }
259
260 BUG_ON(!PageLocked(page));
261
262 ctx = fscrypt_get_ctx(inode, gfp_flags);
263 if (IS_ERR(ctx))
264 return (struct page *)ctx;
265
266 /* The encryption operation will require a bounce page. */
267 ciphertext_page = fscrypt_alloc_bounce_page(ctx, gfp_flags);
268 if (IS_ERR(ciphertext_page))
269 goto errout;
270
271 ctx->w.control_page = page;
272 err = fscrypt_do_page_crypto(inode, FS_ENCRYPT, lblk_num,
273 page, ciphertext_page, len, offs,
274 gfp_flags);
275 if (err) {
276 ciphertext_page = ERR_PTR(err);
277 goto errout;
278 }
279 SetPagePrivate(ciphertext_page);
280 set_page_private(ciphertext_page, (unsigned long)ctx);
281 lock_page(ciphertext_page);
282 return ciphertext_page;
283
284 errout:
285 fscrypt_release_ctx(ctx);
286 return ciphertext_page;
287 }
288 EXPORT_SYMBOL(fscrypt_encrypt_page);
289
290 /**
291 * fscrypt_decrypt_page() - Decrypts a page in-place
292 * @inode: The corresponding inode for the page to decrypt.
293 * @page: The page to decrypt. Must be locked in case
294 * it is a writeback page (FS_CFLG_OWN_PAGES unset).
295 * @len: Number of bytes in @page to be decrypted.
296 * @offs: Start of data in @page.
297 * @lblk_num: Logical block number.
298 *
299 * Decrypts page in-place using the ctx encryption context.
300 *
301 * Called from the read completion callback.
302 *
303 * Return: Zero on success, non-zero otherwise.
304 */
fscrypt_decrypt_page(const struct inode * inode,struct page * page,unsigned int len,unsigned int offs,u64 lblk_num)305 int fscrypt_decrypt_page(const struct inode *inode, struct page *page,
306 unsigned int len, unsigned int offs, u64 lblk_num)
307 {
308 if (!(inode->i_sb->s_cop->flags & FS_CFLG_OWN_PAGES))
309 BUG_ON(!PageLocked(page));
310
311 return fscrypt_do_page_crypto(inode, FS_DECRYPT, lblk_num, page, page,
312 len, offs, GFP_NOFS);
313 }
314 EXPORT_SYMBOL(fscrypt_decrypt_page);
315
316 /*
317 * Validate dentries for encrypted directories to make sure we aren't
318 * potentially caching stale data after a key has been added or
319 * removed.
320 */
fscrypt_d_revalidate(struct dentry * dentry,unsigned int flags)321 static int fscrypt_d_revalidate(struct dentry *dentry, unsigned int flags)
322 {
323 struct dentry *dir;
324 int dir_has_key, cached_with_key;
325
326 if (flags & LOOKUP_RCU)
327 return -ECHILD;
328
329 dir = dget_parent(dentry);
330 if (!IS_ENCRYPTED(d_inode(dir))) {
331 dput(dir);
332 return 0;
333 }
334
335 /* this should eventually be an flag in d_flags */
336 spin_lock(&dentry->d_lock);
337 cached_with_key = dentry->d_flags & DCACHE_ENCRYPTED_WITH_KEY;
338 spin_unlock(&dentry->d_lock);
339 dir_has_key = (d_inode(dir)->i_crypt_info != NULL);
340 dput(dir);
341
342 /*
343 * If the dentry was cached without the key, and it is a
344 * negative dentry, it might be a valid name. We can't check
345 * if the key has since been made available due to locking
346 * reasons, so we fail the validation so ext4_lookup() can do
347 * this check.
348 *
349 * We also fail the validation if the dentry was created with
350 * the key present, but we no longer have the key, or vice versa.
351 */
352 if ((!cached_with_key && d_is_negative(dentry)) ||
353 (!cached_with_key && dir_has_key) ||
354 (cached_with_key && !dir_has_key))
355 return 0;
356 return 1;
357 }
358
359 const struct dentry_operations fscrypt_d_ops = {
360 .d_revalidate = fscrypt_d_revalidate,
361 };
362 EXPORT_SYMBOL(fscrypt_d_ops);
363
fscrypt_restore_control_page(struct page * page)364 void fscrypt_restore_control_page(struct page *page)
365 {
366 struct fscrypt_ctx *ctx;
367
368 ctx = (struct fscrypt_ctx *)page_private(page);
369 set_page_private(page, (unsigned long)NULL);
370 ClearPagePrivate(page);
371 unlock_page(page);
372 fscrypt_release_ctx(ctx);
373 }
374 EXPORT_SYMBOL(fscrypt_restore_control_page);
375
fscrypt_destroy(void)376 static void fscrypt_destroy(void)
377 {
378 struct fscrypt_ctx *pos, *n;
379
380 list_for_each_entry_safe(pos, n, &fscrypt_free_ctxs, free_list)
381 kmem_cache_free(fscrypt_ctx_cachep, pos);
382 INIT_LIST_HEAD(&fscrypt_free_ctxs);
383 mempool_destroy(fscrypt_bounce_page_pool);
384 fscrypt_bounce_page_pool = NULL;
385 }
386
387 /**
388 * fscrypt_initialize() - allocate major buffers for fs encryption.
389 * @cop_flags: fscrypt operations flags
390 *
391 * We only call this when we start accessing encrypted files, since it
392 * results in memory getting allocated that wouldn't otherwise be used.
393 *
394 * Return: Zero on success, non-zero otherwise.
395 */
fscrypt_initialize(unsigned int cop_flags)396 int fscrypt_initialize(unsigned int cop_flags)
397 {
398 int i, res = -ENOMEM;
399
400 /* No need to allocate a bounce page pool if this FS won't use it. */
401 if (cop_flags & FS_CFLG_OWN_PAGES)
402 return 0;
403
404 mutex_lock(&fscrypt_init_mutex);
405 if (fscrypt_bounce_page_pool)
406 goto already_initialized;
407
408 for (i = 0; i < num_prealloc_crypto_ctxs; i++) {
409 struct fscrypt_ctx *ctx;
410
411 ctx = kmem_cache_zalloc(fscrypt_ctx_cachep, GFP_NOFS);
412 if (!ctx)
413 goto fail;
414 list_add(&ctx->free_list, &fscrypt_free_ctxs);
415 }
416
417 fscrypt_bounce_page_pool =
418 mempool_create_page_pool(num_prealloc_crypto_pages, 0);
419 if (!fscrypt_bounce_page_pool)
420 goto fail;
421
422 already_initialized:
423 mutex_unlock(&fscrypt_init_mutex);
424 return 0;
425 fail:
426 fscrypt_destroy();
427 mutex_unlock(&fscrypt_init_mutex);
428 return res;
429 }
430
431 /**
432 * fscrypt_init() - Set up for fs encryption.
433 */
fscrypt_init(void)434 static int __init fscrypt_init(void)
435 {
436 fscrypt_read_workqueue = alloc_workqueue("fscrypt_read_queue",
437 WQ_HIGHPRI, 0);
438 if (!fscrypt_read_workqueue)
439 goto fail;
440
441 fscrypt_ctx_cachep = KMEM_CACHE(fscrypt_ctx, SLAB_RECLAIM_ACCOUNT);
442 if (!fscrypt_ctx_cachep)
443 goto fail_free_queue;
444
445 fscrypt_info_cachep = KMEM_CACHE(fscrypt_info, SLAB_RECLAIM_ACCOUNT);
446 if (!fscrypt_info_cachep)
447 goto fail_free_ctx;
448
449 return 0;
450
451 fail_free_ctx:
452 kmem_cache_destroy(fscrypt_ctx_cachep);
453 fail_free_queue:
454 destroy_workqueue(fscrypt_read_workqueue);
455 fail:
456 return -ENOMEM;
457 }
module_init(fscrypt_init)458 module_init(fscrypt_init)
459
460 /**
461 * fscrypt_exit() - Shutdown the fs encryption system
462 */
463 static void __exit fscrypt_exit(void)
464 {
465 fscrypt_destroy();
466
467 if (fscrypt_read_workqueue)
468 destroy_workqueue(fscrypt_read_workqueue);
469 kmem_cache_destroy(fscrypt_ctx_cachep);
470 kmem_cache_destroy(fscrypt_info_cachep);
471
472 fscrypt_essiv_cleanup();
473 }
474 module_exit(fscrypt_exit);
475
476 MODULE_LICENSE("GPL");
477