1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
|
/*
* Copyright 2013, Michael (Ellerman|Neuling), IBM Corporation.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version
* 2 of the License, or (at your option) any later version.
*/
#define pr_fmt(fmt) "powernv: " fmt
#include <linux/kernel.h>
#include <linux/cpu.h>
#include <linux/cpumask.h>
#include <linux/device.h>
#include <linux/gfp.h>
#include <linux/smp.h>
#include <linux/stop_machine.h>
#include <asm/cputhreads.h>
#include <asm/cpuidle.h>
#include <asm/kvm_ppc.h>
#include <asm/machdep.h>
#include <asm/opal.h>
#include <asm/smp.h>
#include "subcore.h"
#include "powernv.h"
/*
* Split/unsplit procedure:
*
* A core can be in one of three states, unsplit, 2-way split, and 4-way split.
*
* The mapping to subcores_per_core is simple:
*
* State | subcores_per_core
* ------------|------------------
* Unsplit | 1
* 2-way split | 2
* 4-way split | 4
*
* The core is split along thread boundaries, the mapping between subcores and
* threads is as follows:
*
* Unsplit:
* ----------------------------
* Subcore | 0 |
* ----------------------------
* Thread | 0 1 2 3 4 5 6 7 |
* ----------------------------
*
* 2-way split:
* -------------------------------------
* Subcore | 0 | 1 |
* -------------------------------------
* Thread | 0 1 2 3 | 4 5 6 7 |
* -------------------------------------
*
* 4-way split:
* -----------------------------------------
* Subcore | 0 | 1 | 2 | 3 |
* -----------------------------------------
* Thread | 0 1 | 2 3 | 4 5 | 6 7 |
* -----------------------------------------
*
*
* Transitions
* -----------
*
* It is not possible to transition between either of the split states, the
* core must first be unsplit. The legal transitions are:
*
* ----------- ---------------
* | | <----> | 2-way split |
* | | ---------------
* | Unsplit |
* | | ---------------
* | | <----> | 4-way split |
* ----------- ---------------
*
* Unsplitting
* -----------
*
* Unsplitting is the simpler procedure. It requires thread 0 to request the
* unsplit while all other threads NAP.
*
* Thread 0 clears HID0_POWER8_DYNLPARDIS (Dynamic LPAR Disable). This tells
* the hardware that if all threads except 0 are napping, the hardware should
* unsplit the core.
*
* Non-zero threads are sent to a NAP loop, they don't exit the loop until they
* see the core unsplit.
*
* Core 0 spins waiting for the hardware to see all the other threads napping
* and perform the unsplit.
*
* Once thread 0 sees the unsplit, it IPIs the secondary threads to wake them
* out of NAP. They will then see the core unsplit and exit the NAP loop.
*
* Splitting
* ---------
*
* The basic splitting procedure is fairly straight forward. However it is
* complicated by the fact that after the split occurs, the newly created
* subcores are not in a fully initialised state.
*
* Most notably the subcores do not have the correct value for SDR1, which
* means they must not be running in virtual mode when the split occurs. The
* subcores have separate timebases SPRs but these are pre-synchronised by
* opal.
*
* To begin with secondary threads are sent to an assembly routine. There they
* switch to real mode, so they are immune to the uninitialised SDR1 value.
* Once in real mode they indicate that they are in real mode, and spin waiting
* to see the core split.
*
* Thread 0 waits to see that all secondaries are in real mode, and then begins
* the splitting procedure. It firstly sets HID0_POWER8_DYNLPARDIS, which
* prevents the hardware from unsplitting. Then it sets the appropriate HID bit
* to request the split, and spins waiting to see that the split has happened.
*
* Concurrently the secondaries will notice the split. When they do they set up
* their SPRs, notably SDR1, and then they can return to virtual mode and exit
* the procedure.
*/
/* Initialised at boot by subcore_init() */
static int subcores_per_core;
/*
* Used to communicate to offline cpus that we want them to pop out of the
* offline loop and do a split or unsplit.
*
* 0 - no split happening
* 1 - unsplit in progress
* 2 - split to 2 in progress
* 4 - split to 4 in progress
*/
static int new_split_mode;
static cpumask_var_t cpu_offline_mask;
struct split_state {
u8 step;
u8 master;
};
static DEFINE_PER_CPU(struct split_state, split_state);
static void wait_for_sync_step(int step)
{
int i, cpu = smp_processor_id();
for (i = cpu + 1; i < cpu + threads_per_core; i++)
while(per_cpu(split_state, i).step < step)
barrier();
/* Order the wait loop vs any subsequent loads/stores. */
mb();
}
static void update_hid_in_slw(u64 hid0)
{
u64 idle_states = pnv_get_supported_cpuidle_states();
if (idle_states & OPAL_PM_WINKLE_ENABLED) {
/* OPAL call to patch slw with the new HID0 value */
u64 cpu_pir = hard_smp_processor_id();
opal_slw_set_reg(cpu_pir, SPRN_HID0, hid0);
}
}
static void unsplit_core(void)
{
u64 hid0, mask;
int i, cpu;
mask = HID0_POWER8_2LPARMODE | HID0_POWER8_4LPARMODE;
cpu = smp_processor_id();
if (cpu_thread_in_core(cpu) != 0) {
while (mfspr(SPRN_HID0) & mask)
power7_idle_insn(PNV_THREAD_NAP);
per_cpu(split_state, cpu).step = SYNC_STEP_UNSPLIT;
return;
}
hid0 = mfspr(SPRN_HID0);
hid0 &= ~HID0_POWER8_DYNLPARDIS;
update_power8_hid0(hid0);
update_hid_in_slw(hid0);
while (mfspr(SPRN_HID0) & mask)
cpu_relax();
/* Wake secondaries out of NAP */
for (i = cpu + 1; i < cpu + threads_per_core; i++)
smp_send_reschedule(i);
wait_for_sync_step(SYNC_STEP_UNSPLIT);
}
static void split_core(int new_mode)
{
struct { u64 value; u64 mask; } split_parms[2] = {
{ HID0_POWER8_1TO2LPAR, HID0_POWER8_2LPARMODE },
{ HID0_POWER8_1TO4LPAR, HID0_POWER8_4LPARMODE }
};
int i, cpu;
u64 hid0;
/* Convert new_mode (2 or 4) into an index into our parms array */
i = (new_mode >> 1) - 1;
BUG_ON(i < 0 || i > 1);
cpu = smp_processor_id();
if (cpu_thread_in_core(cpu) != 0) {
split_core_secondary_loop(&per_cpu(split_state, cpu).step);
return;
}
wait_for_sync_step(SYNC_STEP_REAL_MODE);
/* Write new mode */
hid0 = mfspr(SPRN_HID0);
hid0 |= HID0_POWER8_DYNLPARDIS | split_parms[i].value;
update_power8_hid0(hid0);
update_hid_in_slw(hid0);
/* Wait for it to happen */
while (!(mfspr(SPRN_HID0) & split_parms[i].mask))
cpu_relax();
}
static void cpu_do_split(int new_mode)
{
/*
* At boot subcores_per_core will be 0, so we will always unsplit at
* boot. In the usual case where the core is already unsplit it's a
* nop, and this just ensures the kernel's notion of the mode is
* consistent with the hardware.
*/
if (subcores_per_core != 1)
unsplit_core();
if (new_mode != 1)
split_core(new_mode);
mb();
per_cpu(split_state, smp_processor_id()).step = SYNC_STEP_FINISHED;
}
bool cpu_core_split_required(void)
{
smp_rmb();
if (!new_split_mode)
return false;
cpu_do_split(new_split_mode);
return true;
}
void update_subcore_sibling_mask(void)
{
int cpu;
/*
* sibling mask for the first cpu. Left shift this by required bits
* to get sibling mask for the rest of the cpus.
*/
int sibling_mask_first_cpu = (1 << threads_per_subcore) - 1;
for_each_possible_cpu(cpu) {
int tid = cpu_thread_in_core(cpu);
int offset = (tid / threads_per_subcore) * threads_per_subcore;
int mask = sibling_mask_first_cpu << offset;
paca[cpu].subcore_sibling_mask = mask;
}
}
static int cpu_update_split_mode(void *data)
{
int cpu, new_mode = *(int *)data;
if (this_cpu_ptr(&split_state)->master) {
new_split_mode = new_mode;
smp_wmb();
cpumask_andnot(cpu_offline_mask, cpu_present_mask,
cpu_online_mask);
/* This should work even though the cpu is offline */
for_each_cpu(cpu, cpu_offline_mask)
smp_send_reschedule(cpu);
}
cpu_do_split(new_mode);
if (this_cpu_ptr(&split_state)->master) {
/* Wait for all cpus to finish before we touch subcores_per_core */
for_each_present_cpu(cpu) {
if (cpu >= setup_max_cpus)
break;
while(per_cpu(split_state, cpu).step < SYNC_STEP_FINISHED)
barrier();
}
new_split_mode = 0;
/* Make the new mode public */
subcores_per_core = new_mode;
threads_per_subcore = threads_per_core / subcores_per_core;
update_subcore_sibling_mask();
/* Make sure the new mode is written before we exit */
mb();
}
return 0;
}
static int set_subcores_per_core(int new_mode)
{
struct split_state *state;
int cpu;
if (kvm_hv_mode_active()) {
pr_err("Unable to change split core mode while KVM active.\n");
return -EBUSY;
}
/*
* We are only called at boot, or from the sysfs write. If that ever
* changes we'll need a lock here.
*/
BUG_ON(new_mode < 1 || new_mode > 4 || new_mode == 3);
for_each_present_cpu(cpu) {
state = &per_cpu(split_state, cpu);
state->step = SYNC_STEP_INITIAL;
state->master = 0;
}
cpus_read_lock();
/* This cpu will update the globals before exiting stop machine */
this_cpu_ptr(&split_state)->master = 1;
/* Ensure state is consistent before we call the other cpus */
mb();
stop_machine_cpuslocked(cpu_update_split_mode, &new_mode,
cpu_online_mask);
cpus_read_unlock();
return 0;
}
static ssize_t __used store_subcores_per_core(struct device *dev,
struct device_attribute *attr, const char *buf,
size_t count)
{
unsigned long val;
int rc;
/* We are serialised by the attribute lock */
rc = sscanf(buf, "%lx", &val);
if (rc != 1)
return -EINVAL;
switch (val) {
|