1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
|
// SPDX-License-Identifier: GPL-2.0+
/*
* PowerPC Memory Protection Keys management
*
* Copyright 2017, Ram Pai, IBM Corporation.
*/
#include <asm/mman.h>
#include <asm/mmu_context.h>
#include <asm/mmu.h>
#include <asm/setup.h>
#include <linux/pkeys.h>
#include <linux/of_device.h>
DEFINE_STATIC_KEY_FALSE(pkey_disabled);
int num_pkey; /* Max number of pkeys supported */
/*
* Keys marked in the reservation list cannot be allocated by userspace
*/
u32 reserved_allocation_mask __ro_after_init;
/* Bits set for the initially allocated keys */
static u32 initial_allocation_mask __ro_after_init;
static bool pkey_execute_disable_supported;
/*
* Even if we allocate keys with sys_pkey_alloc(), we need to make sure
* other thread still find the access denied using the same keys.
*/
static u64 default_amr = ~0x0UL;
static u64 default_iamr = 0x5555555555555555UL;
/* Allow all keys to be modified by default */
static u64 default_uamor = ~0x0UL;
/*
* Key used to implement PROT_EXEC mmap. Denies READ/WRITE
* We pick key 2 because 0 is special key and 1 is reserved as per ISA.
*/
static int execute_only_key = 2;
#define AMR_BITS_PER_PKEY 2
#define AMR_RD_BIT 0x1UL
#define AMR_WR_BIT 0x2UL
#define IAMR_EX_BIT 0x1UL
#define PKEY_REG_BITS (sizeof(u64) * 8)
#define pkeyshift(pkey) (PKEY_REG_BITS - ((pkey+1) * AMR_BITS_PER_PKEY))
static int scan_pkey_feature(void)
{
u32 vals[2];
int pkeys_total = 0;
struct device_node *cpu;
/*
* Pkey is not supported with Radix translation.
*/
if (radix_enabled())
return 0;
cpu = of_find_node_by_type(NULL, "cpu");
if (!cpu)
return 0;
if (of_property_read_u32_array(cpu,
"ibm,processor-storage-keys", vals, 2) == 0) {
/*
* Since any pkey can be used for data or execute, we will
* just treat all keys as equal and track them as one entity.
*/
pkeys_total = vals[0];
} else {
/*
* Let's assume 32 pkeys on P8/P9 bare metal, if its not defined by device
* tree. We make this exception since some version of skiboot forgot to
* expose this property on power8/9.
*/
if (!firmware_has_feature(FW_FEATURE_LPAR)) {
unsigned long pvr = mfspr(SPRN_PVR);
if (PVR_VER(pvr) == PVR_POWER8 || PVR_VER(pvr) == PVR_POWER8E ||
PVR_VER(pvr) == PVR_POWER8NVL || PVR_VER(pvr) == PVR_POWER9)
pkeys_total = 32;
}
}
/*
* Adjust the upper limit, based on the number of bits supported by
* arch-neutral code.
*/
pkeys_total = min_t(int, pkeys_total,
((ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT) + 1));
return pkeys_total;
}
static int pkey_initialize(void)
{
int pkeys_total, i;
/*
* We define PKEY_DISABLE_EXECUTE in addition to the arch-neutral
* generic defines for PKEY_DISABLE_ACCESS and PKEY_DISABLE_WRITE.
* Ensure that the bits a distinct.
*/
BUILD_BUG_ON(PKEY_DISABLE_EXECUTE &
(PKEY_DISABLE_ACCESS | PKEY_DISABLE_WRITE));
/*
* pkey_to_vmflag_bits() assumes that the pkey bits are contiguous
* in the vmaflag. Make sure that is really the case.
*/
BUILD_BUG_ON(__builtin_clzl(ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT) +
__builtin_popcountl(ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT)
!= (sizeof(u64) * BITS_PER_BYTE));
/* scan the device tree for pkey feature */
pkeys_total = scan_pkey_feature();
if (!pkeys_total) {
/* No support for pkey. Mark it disabled */
static_branch_enable(&pkey_disabled);
return 0;
}
/*
* The device tree cannot be relied to indicate support for
* execute_disable support. Instead we use a PVR check.
*/
if (pvr_version_is(PVR_POWER7) || pvr_version_is(PVR_POWER7p))
pkey_execute_disable_supported = false;
else
pkey_execute_disable_supported = true;
#ifdef CONFIG_PPC_4K_PAGES
/*
* The OS can manage only 8 pkeys due to its inability to represent them
* in the Linux 4K PTE. Mark all other keys reserved.
*/
num_pkey = min(8, pkeys_total);
#else
num_pkey = pkeys_total;
#endif
if (unlikely(num_pkey <= execute_only_key)) {
/*
* Insufficient number of keys to support
* execute only key. Mark it unavailable.
*/
execute_only_key = -1;
} else {
/*
* Mark the execute_only_pkey as not available for
* user allocation via pkey_alloc.
*/
reserved_allocation_mask |= (0x1 << execute_only_key);
/*
* Deny READ/WRITE for execute_only_key.
* Allow execute in IAMR.
*/
default_amr |= (0x3ul << pkeyshift(execute_only_key));
default_iamr &= ~(0x1ul << pkeyshift(execute_only_key));
/*
* Clear the uamor bits for this key.
*/
default_uamor &= ~(0x3ul << pkeyshift(execute_only_key));
}
/*
* Allow access for only key 0. And prevent any other modification.
*/
default_amr &= ~(0x3ul << pkeyshift(0));
default_iamr &= ~(0x1ul << pkeyshift(0));
default_uamor &= ~(0x3ul << pkeyshift(0));
/*
* key 0 is special in that we want to consider it an allocated
* key which is preallocated. We don't allow changing AMR bits
* w.r.t key 0. But one can pkey_free(key0)
*/
initial_allocation_mask |= (0x1 << 0);
/*
* key 1 is recommended not to be used. PowerISA(3.0) page 1015,
* programming note.
*/
reserved_allocation_mask |= (0x1 << 1);
default_uamor &= ~(0x3ul << pkeyshift(1));
/*
* Prevent the usage of OS reserved keys. Update UAMOR
* for those keys.
*/
for (i = num_pkey; i < pkeys_total; i++) {
reserved_allocation_mask |= (0x1 << i);
default_uamor &= ~(0x3ul << pkeyshift(i));
}
/*
* Prevent the allocation of reserved keys too.
*/
initial_allocation_mask |= reserved_allocation_mask;
return 0;
}
arch_initcall(pkey_initialize);
void pkey_mm_init(struct mm_struct *mm)
{
if (static_branch_likely(&pkey_disabled))
return;
mm_pkey_allocation_map(mm) = initial_allocation_mask;
mm->context.execute_only_pkey = execute_only_key;
}
static inline u64 read_amr(void)
{
return mfspr(SPRN_AMR);
}
static inline void write_amr(u64 value)
{
mtspr(SPRN_AMR, value);
}
static inline u64 read_iamr(void)
{
if (!likely(pkey_execute_disable_supported))
return 0x0UL;
return mfspr(SPRN_IAMR);
}
static inline void write_iamr(u64 value)
{
if (!likely(pkey_execute_disable_supported))
return;
mtspr(SPRN_IAMR, value);
}
static inline u64 read_uamor(void)
{
return mfspr(SPRN_UAMOR);
}
static inline void write_uamor(u64 value)
{
mtspr(SPRN_UAMOR, value);
}
static bool is_pkey_enabled(int pkey)
{
u64 uamor = read_uamor();
u64 pkey_bits = 0x3ul << pkeyshift(pkey);
u64 uamor_pkey_bits = (uamor & pkey_bits);
/*
* Both the bits in UAMOR corresponding to the key should be set or
* reset.
*/
WARN_ON(uamor_pkey_bits && (uamor_pkey_bits != pkey_bits));
return !!(uamor_pkey_bits);
}
static inline void init_amr(int pkey, u8 init_bits)
{
u64 new_amr_bits = (((u64)init_bits & 0x3UL) << pkeyshift(pkey));
u64 old_amr = read_amr() & ~((u64)(0x3ul) << pkeyshift(pkey));
write_amr(old_amr | new_amr_bits);
}
static inline void init_iamr(int pkey, u8 init_bits)
{
u64 new_iamr_bits = (((u64)init_bits & 0x1UL) << pkeyshift(pkey));
u64 old_iamr = read_iamr() & ~((u64)(0x1ul) << pkeyshift(pkey));
write_iamr(old_iamr | new_iamr_bits);
}
/*
* Set the access rights in AMR IAMR and UAMOR registers for @pkey to that
* specified in @init_val.
*/
int __arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
unsigned long init_val)
{
u64 new_amr_bits = 0x0ul;
u64 new_iamr_bits = 0x0ul;
if (!is_pkey_enabled(pkey))
return -EINVAL;
if (init_val & PKEY_DISABLE_EXECUTE) {
if (!pkey_execute_disable_supported)
return -EINVAL;
new_iamr_bits |= IAMR_EX_BIT;
}
init_iamr(pkey, new_iamr_bits);
/* Set the bits we need in AMR: */
if (init_val & PKEY_DISABLE_ACCESS)
new_amr_bits |= AMR_RD_BIT | AMR_WR_BIT;
else if (init_val & PKEY_DISABLE_WRITE)
new_amr_bits |= AMR_WR_BIT;
init_amr(pkey, new_amr_bits);
return 0;
}
void thread_pkey_regs_save(struct thread_struct *thread)
{
if (static_branch_likely(&pkey_disabled))
return;
/*
* TODO: Skip saving registers if @thread hasn't used any keys yet.
*/
thread-&g
|