1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
|
// SPDX-License-Identifier: GPL-2.0
/*
* Inline encryption support for fscrypt
*
* Copyright 2019 Google LLC
*/
/*
* With "inline encryption", the block layer handles the decryption/encryption
* as part of the bio, instead of the filesystem doing the crypto itself via
* crypto API. See Documentation/block/inline-encryption.rst. fscrypt still
* provides the key and IV to use.
*/
#include <linux/blk-crypto.h>
#include <linux/blkdev.h>
#include <linux/buffer_head.h>
#include <linux/sched/mm.h>
#include "fscrypt_private.h"
struct fscrypt_blk_crypto_key {
struct blk_crypto_key base;
int num_devs;
struct request_queue *devs[];
};
static int fscrypt_get_num_devices(struct super_block *sb)
{
if (sb->s_cop->get_num_devices)
return sb->s_cop->get_num_devices(sb);
return 1;
}
static void fscrypt_get_devices(struct super_block *sb, int num_devs,
struct request_queue **devs)
{
if (num_devs == 1)
devs[0] = bdev_get_queue(sb->s_bdev);
else
sb->s_cop->get_devices(sb, devs);
}
static unsigned int fscrypt_get_dun_bytes(const struct fscrypt_info *ci)
{
struct super_block *sb = ci->ci_inode->i_sb;
unsigned int flags = fscrypt_policy_flags(&ci->ci_policy);
int ino_bits = 64, lblk_bits = 64;
if (flags & FSCRYPT_POLICY_FLAG_DIRECT_KEY)
return offsetofend(union fscrypt_iv, nonce);
if (flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64)
return sizeof(__le64);
if (flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32)
return sizeof(__le32);
/* Default case: IVs are just the file logical block number */
if (sb->s_cop->get_ino_and_lblk_bits)
sb->s_cop->get_ino_and_lblk_bits(sb, &ino_bits, &lblk_bits);
return DIV_ROUND_UP(lblk_bits, 8);
}
/* Enable inline encryption for this file if supported. */
int fscrypt_select_encryption_impl(struct fscrypt_info *ci)
{
const struct inode *inode = ci->ci_inode;
struct super_block *sb = inode->i_sb;
struct blk_crypto_config crypto_cfg;
int num_devs;
struct request_queue **devs;
int i;
/* The file must need contents encryption, not filenames encryption */
if (!fscrypt_needs_contents_encryption(inode))
return 0;
/* The crypto mode must have a blk-crypto counterpart */
if (ci->ci_mode->blk_crypto_mode == BLK_ENCRYPTION_MODE_INVALID)
return 0;
/* The filesystem must be mounted with -o inlinecrypt */
if (!(sb->s_flags & SB_INLINECRYPT))
return 0;
/*
* When a page contains multiple logically contiguous filesystem blocks,
* some filesystem code only calls fscrypt_mergeable_bio() for the first
* block in the page. This is fine for most of fscrypt's IV generation
* strategies, where contiguous blocks imply contiguous IVs. But it
* doesn't work with IV_INO_LBLK_32. For now, simply exclude
* IV_INO_LBLK_32 with blocksize != PAGE_SIZE from inline encryption.
*/
if ((fscrypt_policy_flags(&ci->ci_policy) &
FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32) &&
sb->s_blocksize != PAGE_SIZE)
return 0;
/*
* On all the filesystem's devices, blk-crypto must support the crypto
* configuration that the file would use.
*/
crypto_cfg.crypto_mode = ci->ci_mode->blk_crypto_mode;
crypto_cfg.data_unit_size = sb->s_blocksize;
crypto_cfg.dun_bytes = fscrypt_get_dun_bytes(ci);
num_devs = fscrypt_get_num_devices(sb);
devs = kmalloc_array(num_devs, sizeof(*devs), GFP_NOFS);
if (!devs)
return -ENOMEM;
fscrypt_get_devices(sb, num_devs, devs);
for (i = 0; i < num_devs; i++) {
if (!blk_crypto_config_supported(devs[i], &crypto_cfg))
goto out_free_devs;
}
ci->ci_inlinecrypt = true;
out_free_devs:
kfree(devs);
return 0;
}
int fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key,
const u8 *raw_key,
const struct fscrypt_info *ci)
{
const struct inode *inode = ci->ci_inode;
struct super_block *sb = inode->i_sb;
enum blk_crypto_mode_num crypto_mode = ci->ci_mode->blk_crypto_mode;
int num_devs = fscrypt_get_num_devices(sb);
int queue_refs = 0;
struct fscrypt_blk_crypto_key *blk_key;
int err;
int i;
unsigned int flags;
blk_key = kzalloc(struct_size(blk_key, devs, num_devs), GFP_NOFS);
if (!blk_key)
return -ENOMEM;
blk_key->num_devs = num_devs;
fscrypt_get_devices(sb, num_devs, blk_key->devs);
err = blk_crypto_init_key(&blk_key->base, raw_key, crypto_mode,
fscrypt_get_dun_bytes(ci), sb->s_blocksize);
if (err) {
fscrypt_err(inode, "error %d initializing blk-crypto key", err);
goto fail;
}
/*
* We have to start using blk-crypto on all the filesystem's devices.
* We also have to save all the request_queue's for later so that the
* key can be evicted from them. This is needed because some keys
* aren't destroyed until after the filesystem was already unmounted
* (namely, the per-mode keys in struct fscrypt_master_key).
*/
for (i = 0; i < num_devs; i++) {
if (!blk_get_queue(blk_key->devs[i])) {
fscrypt_err(inode, "couldn't get request_queue");
err = -EAGAIN;
goto fail;
}
queue_refs++;
flags = memalloc_nofs_save();
err = blk_crypto_start_using_key(&blk_key->base,
blk_key->devs[i]);
memalloc_nofs_restore(flags);
if (err) {
fscrypt_err(inode,
"error %d starting to use blk-crypto", err);
goto fail;
}
}
/*
* Pairs with READ_ONCE() in fscrypt_is_key_prepared(). (Only matters
* for the per-mode keys, which are shared by multiple inodes.)
*/
smp_store_release(&prep_key->blk_key, blk_key);
return 0;
fail:
for (i = 0; i < queue_refs; i++)
blk_put_queue(blk_key->devs[i]);
kzfree(blk_key);
return err;
}
void fscrypt_destroy_inline_crypt_key(struct fscrypt_prepared_key *prep_key)
{
struct fscrypt_blk_crypto_key *blk_key = prep_key->blk_key;
int i;
if (blk_key) {
for (i = 0; i < blk_key->num_devs; i++) {
blk_crypto_evict_key(blk_key->devs[i], &blk_key->base);
blk_put_queue(blk_key->devs[i]);
}
kzfree(blk_key);
}
}
bool __fscrypt_inode_uses_inline_crypto(const struct inode *inode)
{
return inode->i_crypt_info->ci_inlinecrypt;
}
EXPORT_SYMBOL_GPL(__fscrypt_inode_uses_inline_crypto);
static void fscrypt_generate_dun(const struct fscrypt_info *ci, u64 lblk_num,
u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE])
{
union fscrypt_iv iv;
int i;
fscrypt_generate_iv(&iv, lblk_num, ci);
BUILD_BUG_ON(FSCRYPT_MAX_IV_SIZE > BLK_CRYPTO_MAX_IV_SIZE);
memset(dun, 0, BLK_CRYPTO_MAX_IV_SIZE);
for (i = 0; i < ci->ci_mode->ivsize/sizeof(dun[0]); i++)
dun[i] = le64_to_cpu(iv.dun[i]);
}
/**
* fscrypt_set_bio_crypt_ctx() - prepare a file contents bio for inline crypto
* @bio: a bio which will eventually be submitted to the file
* @inode: the file's inode
* @first_lblk: the first file logical block number in the I/O
* @gfp_mask: memory allocation flags - these must be a waiting mask so that
* bio_crypt_set_ctx can't fail.
*
* If the contents of the file should be encrypted (or decrypted) with inline
* encryption, then assign the appropriate encryption context to the bio.
*
* Normally the bio should be newly allocated (i.e. no pages added yet), as
* otherwise fscrypt_mergeable_bio() won't work as intended.
*
* The encryption context will be freed automatically when the bio is freed.
*/
void fscrypt_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode,
u64 first_lblk, gfp_t gfp_mask)
{
const struct fscrypt_info *ci = inode->i_crypt_info;
u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE];
if (!fscrypt_inode_uses_inline_crypto(inode))
return;
fscrypt_generate_dun(ci, first_lblk, dun);
bio_crypt_set_ctx(bio, &ci->ci_enc_key.blk_key->base, dun, gfp_mask);
}
EXPORT_SYMBOL_GPL(fscrypt_set_bio_crypt_ctx);
/* Extract the inode and logical block number from a buffer_head. */
static bool bh_get_inode_and_lblk_num(const struct buffer_head *bh,
const struct inode **inode_ret,
u64 *lblk_num_ret)
{
struct page *page = bh->b_page;
const struct address_space *mapping;
const struct inode *inode;
/*
* The ext4 journal (jbd2) can submit a buffer_head it directly created
* for a non-pagecache page. fscrypt doesn't care about these.
*/
mapping = page_mapping(page);
if (!mapping)
return false;
inode = mapping->host;
*inode_ret = inode;
*lblk_num_ret = ((u64)page->index << (P
|