1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
|
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2014 Linaro Ltd. <ard.biesheuvel@linaro.org>
*/
#ifndef __ASM_CPUFEATURE_H
#define __ASM_CPUFEATURE_H
#include <asm/cpucaps.h>
#include <asm/cputype.h>
#include <asm/hwcap.h>
#include <asm/sysreg.h>
#define MAX_CPU_FEATURES 64
#define cpu_feature(x) KERNEL_HWCAP_ ## x
#ifndef __ASSEMBLY__
#include <linux/bug.h>
#include <linux/jump_label.h>
#include <linux/kernel.h>
/*
* CPU feature register tracking
*
* The safe value of a CPUID feature field is dependent on the implications
* of the values assigned to it by the architecture. Based on the relationship
* between the values, the features are classified into 3 types - LOWER_SAFE,
* HIGHER_SAFE and EXACT.
*
* The lowest value of all the CPUs is chosen for LOWER_SAFE and highest
* for HIGHER_SAFE. It is expected that all CPUs have the same value for
* a field when EXACT is specified, failing which, the safe value specified
* in the table is chosen.
*/
enum ftr_type {
FTR_EXACT, /* Use a predefined safe value */
FTR_LOWER_SAFE, /* Smaller value is safe */
FTR_HIGHER_SAFE, /* Bigger value is safe */
FTR_HIGHER_OR_ZERO_SAFE, /* Bigger value is safe, but 0 is biggest */
};
#define FTR_STRICT true /* SANITY check strict matching required */
#define FTR_NONSTRICT false /* SANITY check ignored */
#define FTR_SIGNED true /* Value should be treated as signed */
#define FTR_UNSIGNED false /* Value should be treated as unsigned */
#define FTR_VISIBLE true /* Feature visible to the user space */
#define FTR_HIDDEN false /* Feature is hidden from the user */
#define FTR_VISIBLE_IF_IS_ENABLED(config) \
(IS_ENABLED(config) ? FTR_VISIBLE : FTR_HIDDEN)
struct arm64_ftr_bits {
bool sign; /* Value is signed ? */
bool visible;
bool strict; /* CPU Sanity check: strict matching required ? */
enum ftr_type type;
u8 shift;
u8 width;
s64 safe_val; /* safe value for FTR_EXACT features */
};
/*
* @arm64_ftr_reg - Feature register
* @strict_mask Bits which should match across all CPUs for sanity.
* @sys_val Safe value across the CPUs (system view)
*/
struct arm64_ftr_reg {
const char *name;
u64 strict_mask;
u64 user_mask;
u64 sys_val;
u64 user_val;
const struct arm64_ftr_bits *ftr_bits;
};
extern struct arm64_ftr_reg arm64_ftr_reg_ctrel0;
/*
* CPU capabilities:
*
* We use arm64_cpu_capabilities to represent system features, errata work
* arounds (both used internally by kernel and tracked in cpu_hwcaps) and
* ELF HWCAPs (which are exposed to user).
*
* To support systems with heterogeneous CPUs, we need to make sure that we
* detect the capabilities correctly on the system and take appropriate
* measures to ensure there are no incompatibilities.
*
* This comment tries to explain how we treat the capabilities.
* Each capability has the following list of attributes :
*
* 1) Scope of Detection : The system detects a given capability by
* performing some checks at runtime. This could be, e.g, checking the
* value of a field in CPU ID feature register or checking the cpu
* model. The capability provides a call back ( @matches() ) to
* perform the check. Scope defines how the checks should be performed.
* There are three cases:
*
* a) SCOPE_LOCAL_CPU: check all the CPUs and "detect" if at least one
* matches. This implies, we have to run the check on all the
* booting CPUs, until the system decides that state of the
* capability is finalised. (See section 2 below)
* Or
* b) SCOPE_SYSTEM: check all the CPUs and "detect" if all the CPUs
* matches. This implies, we run the check only once, when the
* system decides to finalise the state of the capability. If the
* capability relies on a field in one of the CPU ID feature
* registers, we use the sanitised value of the register from the
* CPU feature infrastructure to make the decision.
* Or
* c) SCOPE_BOOT_CPU: Check only on the primary boot CPU to detect the
* feature. This category is for features that are "finalised"
* (or used) by the kernel very early even before the SMP cpus
* are brought up.
*
* The process of detection is usually denoted by "update" capability
* state in the code.
*
* 2) Finalise the state : The kernel should finalise the state of a
* capability at some point during its execution and take necessary
* actions if any. Usually, this is done, after all the boot-time
* enabled CPUs are brought up by the kernel, so that it can make
* better decision based on the available set of CPUs. However, there
* are some special cases, where the action is taken during the early
* boot by the primary boot CPU. (e.g, running the kernel at EL2 with
* Virtualisation Host Extensions). The kernel usually disallows any
* changes to the state of a capability once it finalises the capability
* and takes any action, as it may be impossible to execute the actions
* safely. A CPU brought up after a capability is "finalised" is
* referred to as "Late CPU" w.r.t the capability. e.g, all secondary
* CPUs are treated "late CPUs" for capabilities determined by the boot
* CPU.
*
* At the moment there are two passes of finalising the capabilities.
* a) Boot CPU scope capabilities - Finalised by primary boot CPU via
* setup_boot_cpu_capabilities().
* b) Everything except (a) - Run via setup_system_capabilities().
*
* 3) Verification: When a CPU is brought online (e.g, by user or by the
* kernel), the kernel should make sure that it is safe to use the CPU,
* by verifying that the CPU is compliant with the state of the
* capabilities finalised already. This happens via :
*
* secondary_start_kernel()-> check_local_cpu_capabilities()
*
* As explained in (2) above, capabilities could be finalised at
* different points in the execution. Each newly booted CPU is verified
* against the capabilities that have been finalised by the time it
* boots.
*
* a) SCOPE_BOOT_CPU : All CPUs are verified against the capability
* except for the primary boot CPU.
*
* b) SCOPE_LOCAL_CPU, SCOPE_SYSTEM: All CPUs hotplugged on by the
* user after the kernel boot are verified against the capability.
*
* If there is a conflict, the kernel takes an action, based on the
* severity (e.g, a CPU could be prevented from booting or cause a
* kernel panic). The CPU is allowed to "affect" the state of the
* capability, if it has not been finalised already. See section 5
* for more details on conflicts.
*
* 4) Action: As mentioned in (2), the kernel can take an action for each
* detected capability, on all CPUs on the system. Appropriate actions
* include, turning on an architectural feature, modifying the control
* registers (e.g, SCTLR, TCR etc.) or patching the kernel via
* alternatives. The kernel patching is batched and performed at later
* point. The actions are always initiated only after the capability
* is finalised. This is usally denoted by "enabling" the capability.
* The actions are initiated as follows :
* a) Action is triggered on all online CPUs, after the capability is
* finalised, invoked within the stop_machine() context from
* enable_cpu_capabilitie().
*
* b) Any late CPU, brought up after (1), the action is triggered via:
*
* check_local_cpu_capabilities() -> verify_local_cpu_capabilities()
*
* 5) Conflicts: Based on the state of the capability on a late CPU vs.
* the system state, we could have the following combinations :
*
* x-----------------------------x
* | Type | System | Late CPU |
* |-----------------------------|
* | a | y | n |
* |-----------------------------|
* | b | n | y |
* x-----------------------------x
*
* Two separate flag bits are defined to indicate whether each kind of
* conflict can be allowed:
* ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU - Case(a) is allowed
* ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU - Case(b) is allowed
*
* Case (a) is not permitted for a capability that the system requires
* all CPUs to have in order for the capability to be enabled. This is
* typical for capabilities that represent enhanced functionality.
*
* Case (b) is not permitted for a capability that must be enabled
* during boot if any CPU in the system requires it in order to run
* safely. This is typical for erratum work arounds that cannot be
* enabled after the corresponding capability is finalised.
*
* In some non-typical cases either both (a) and (b), or neither,
* should be permitted. This can be described by including neither
* or both flags in the capability's type field.
*/
/*
* Decide how the capability is detected.
* On any local CPU vs System wide vs the primary boot CPU
*/
#define ARM64_CPUCAP_SCOPE_LOCAL_CPU ((u16)BIT(0))
#define ARM64_CPUCAP_SCOPE_SYSTEM ((u16)BIT(1))
/*
* The capabilitiy is detected on the Boot CPU and is used by kernel
* during early boot. i.e, the capability should be "detected" and
* "enabled" as early as possibly on all booting CPUs.
*/
#define ARM64_CPUCAP_SCOPE_BOOT_CPU ((u16)BIT(2))
#define ARM64_CPUCAP_SCOPE_MASK \
(ARM64_CPUCAP_SCOPE_SYSTEM | \
ARM64_CPUCAP_SCOPE_LOCAL_CPU | \
ARM64_CPUCAP_SCOPE_BOOT_CPU)
#define SCOPE_SYSTEM ARM64_CPUCAP_SCOPE_SYSTEM
#define SCOPE_LOCAL_CPU ARM64_CPUCAP_SCOPE_LOCAL_CPU
#define SCOPE_BOOT_CPU ARM64_CPUCAP_SCOPE_BOOT_CPU
#define SCOPE_ALL ARM64_CPUCAP_SCOPE_MASK
/*
* Is it permitted for a late CPU to have this capability when system
* hasn't already enabled it ?
*/
#define ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU ((u16)BIT(4))
/* Is it safe for a late CPU to miss this capability when system has it */
#define ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU ((u16)BIT(5))
/*
* CPU errata workarounds that need to be enabled at boot time if one or
* more CPUs in the system requires it. When one of these capabilities
* has been enabled, it is safe to allow any CPU to boot that doesn't
* require the workaround. However, it is not safe if a "late" CPU
* requires a workaround and the system hasn't enabled it already.
*/
#define ARM64_CPUCAP_LOCAL_CPU_ERRATUM \
(ARM64_CPUCAP_SCOPE_LOCAL_CPU | ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU)
/*
* CPU feature detected at boot time based on system-wide value of a
* feature. It is safe for a late CPU to have this feature even though
* the system hasn't enabled it, although the feature will not be used
* by Linux in this case. If the system has enabled this feature already,
* then every late CPU must have it.
*/
#define ARM64_CPUCAP_SYSTEM_FEATURE \
(ARM64_CPUCAP_SCOPE_SYSTEM | ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU)
/*
* CPU feature detected at boot time based on feature of one or more CPUs.
* All possible conflicts for a late CPU are ignored.
*/
#define ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE \
(ARM64_CPUCAP_SCOPE_LOCAL_CPU | \
ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU | \
ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU)
/*
* CPU feature detected at boot time, on one or more CPUs. A late CPU
* is not allowed to have the capability when the system doesn't have it.
* It is Ok for a late CPU to miss the feature.
*/
#define ARM64_CPUCAP_BOOT_RESTRICTED_CPU_LOCAL_FEATURE \
(ARM64_CPUCAP_SCOPE_LOCAL_CPU | \
ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU)
/*
* CPU feature used early in the boot based on the boot CPU. All secondary
* CPUs must match the state of the capability as detected by the boot CPU.
*/
#define ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE ARM64_CPUCAP_SCOPE_BOOT_CPU
struct arm64_cpu_capabilities {
const char *desc;
u16 capability;
u16 type;
bool (*matches)(const struct arm64_cpu_capabilities *caps, int scope);
/*
* Take the appropriate actions to enable this capability for this CPU.
* For each successfully booted CPU, this method is called for each
* globally detected capability.
*/
void (*cpu_enable)(const struct arm64_cpu_capabilities *cap);
union {
struct { /* To be used for erratum handling only */
struct midr_range midr_range;
const struct arm64_midr_revidr {
u32 midr_rv; /* revision/variant */
u32 revidr_mask;
} * const fixed_revs;
};
const struct midr_range *midr_range_list;
struct { /* Feature register checking */
u32 sys_reg;
u8 field_pos;
u8 min_field_value;
u8 hwcap_type;
bool sign;
unsigned long hwcap;
};
};
/*
* An optional list of "matches/cpu_enable" pair for the same
* "capability" of the same "type" as described by the parent.
* Only matches(), cpu_enable() and fields relevant to these
* methods are significant in the list. The cpu_enable is
* invoked only if the corresponding entry "matches()".
* However, if a cpu_enable() method is associated
* with multiple matches(), care should be taken that either
* the match criteria are mutually exclusive, or that the
* method is robust against being called multiple times.
*/
const struct arm64_cpu_capabilities *match_list;
};
static inline int cpucap_default_scope(const struct arm64_cpu_capabilities *cap)
{
return cap->type & ARM64_CPUCAP_SCOPE_MASK;
}
static inline bool
cpucap_late_cpu_optional(const struct arm64_cpu_capabilities *cap)
{
return !!(cap->type & ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU);
}
static inline bool
cpucap_late_cpu_permitted(const struct arm64_cpu_capabilities *cap)
{
return !!(cap->type & ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU);
}
/*
* Generic helper for handling capabilties with multiple (match,enable) pairs
* of call backs, sharing the same capability bit.
* Iterate over each entry to see if at least one matches.
*/
static inline bool
cpucap_multi_entry_cap_matches(const struct arm64_cpu_capabilities *entry,
int scope)
{
const struct arm64_cpu_capabilities *caps;
for (caps = entry->match_list; caps->matches; caps++)
if (caps->matches(caps, scope))
return true;
return false;
}
/*
* Take appropriate action for all matching entries in the shared capability
* entry.
*/
static inline void
cpucap_multi_entry_cap_cpu_enable(const struct arm64_cpu_capabilities *entry)
{
const struct arm64_cpu_capabilities *caps;
for (caps = entry->match_list; caps->matches; caps++)
if (caps->matches(caps, SCOPE_LOCAL_CPU) &&
caps->cpu_enable
|