1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
|
// SPDX-License-Identifier: GPL-2.0+
/* Copyright (C) 2018 Broadcom */
/**
* DOC: Broadcom V3D scheduling
*
* The shared DRM GPU scheduler is used to coordinate submitting jobs
* to the hardware. Each DRM fd (roughly a client process) gets its
* own scheduler entity, which will process jobs in order. The GPU
* scheduler will round-robin between clients to submit the next job.
*
* For simplicity, and in order to keep latency low for interactive
* jobs when bulk background jobs are queued up, we submit a new job
* to the HW only when it has completed the last one, instead of
* filling up the CT[01]Q FIFOs with jobs. Similarly, we use
* v3d_job_dependency() to manage the dependency between bin and
* render, instead of having the clients submit jobs using the HW's
* semaphores to interlock between them.
*/
#include <linux/kthread.h>
#include "v3d_drv.h"
#include "v3d_regs.h"
#include "v3d_trace.h"
static struct v3d_job *
to_v3d_job(struct drm_sched_job *sched_job)
{
return container_of(sched_job, struct v3d_job, base);
}
static struct v3d_tfu_job *
to_tfu_job(struct drm_sched_job *sched_job)
{
return container_of(sched_job, struct v3d_tfu_job, base);
}
static void
v3d_job_free(struct drm_sched_job *sched_job)
{
struct v3d_job *job = to_v3d_job(sched_job);
drm_sched_job_cleanup(sched_job);
v3d_exec_put(job->exec);
}
static void
v3d_tfu_job_free(struct drm_sched_job *sched_job)
{
struct v3d_tfu_job *job = to_tfu_job(sched_job);
drm_sched_job_cleanup(sched_job);
v3d_tfu_job_put(job);
}
/**
* Returns the fences that the bin or render job depends on, one by one.
* v3d_job_run() won't be called until all of them have been signaled.
*/
static struct dma_fence *
v3d_job_dependency(struct drm_sched_job *sched_job,
struct drm_sched_entity *s_entity)
{
struct v3d_job *job = to_v3d_job(sched_job);
struct v3d_exec_info *exec = job->exec;
enum v3d_queue q = job == &exec->bin ? V3D_BIN : V3D_RENDER;
struct dma_fence *fence;
fence = job->in_fence;
if (fence) {
job->in_fence = NULL;
return fence;
}
if (q == V3D_RENDER) {
/* If we had a bin job, the render job definitely depends on
* it. We first have to wait for bin to be scheduled, so that
* its done_fence is created.
*/
fence = exec->bin_done_fence;
if (fence) {
exec->bin_done_fence = NULL;
return fence;
}
}
/* XXX: Wait on a fence for switching the GMP if necessary,
* and then do so.
*/
return fence;
}
/**
* Returns the fences that the TFU job depends on, one by one.
* v3d_tfu_job_run() won't be called until all of them have been
* signaled.
*/
static struct dma_fence *
v3d_tfu_job_dependency(struct drm_sched_job *sched_job,
struct drm_sched_entity *s_entity)
{
struct v3d_tfu_job *job = to_tfu_job(sched_job);
struct dma_fence *fence;
fence = job->in_fence;
if (fence) {
job->in_fence = NULL;
return fence;
}
return NULL;
}
static struct dma_fence *v3d_job_run(struct drm_sched_job *sched_job)
{
struct v3d_job *job = to_v3d_job(sched_job);
struct v3d_exec_info *exec = job->exec;
enum v3d_queue q = job == &exec->bin ? V3D_BIN : V3D_RENDER;
struct v3d_dev *v3d = exec->v3d;
struct drm_device *dev = &v3d->drm;
struct dma_fence *fence;
unsigned long irqflags;
if (unlikely(job->base.s_fence->finished.error))
return NULL;
/* Lock required around bin_job update vs
* v3d_overflow_mem_work().
*/
spin_lock_irqsave(&v3d->job_lock, irqflags);
if (q == V3D_BIN) {
v3d->bin_job = job->exec;
/* Clear out the overflow allocation, so we don't
* reuse the overflow attached to a previous job.
*/
V3D_CORE_WRITE(0, V3D_PTB_BPOS, 0);
} else {
v3d->render_job = job->exec;
}
spin_unlock_irqrestore(&v3d->job_lock, irqflags);
/* Can we avoid this flush when q==RENDER? We need to be
* careful of scheduling, though -- imagine job0 rendering to
* texture and job1 reading, and them being executed as bin0,
* bin1, render0, render1, so that render1's flush at bin time
* wasn't enough.
*/
v3d_invalidate_caches(v3d);
fence = v3d_fence_create(v3d, q);
if (IS_ERR(fence))
return NULL;
if (job->done_fence)
dma_fence_put(job->done_fence);
job->done_fence = dma_fence_get(fence);
trace_v3d_submit_cl(dev, q == V3D_RENDER, to_v3d_fence(fence)->seqno,
job->start, job->end);
if (q == V3D_BIN) {
if (exec->qma) {
V3D_CORE_WRITE(0, V3D_CLE_CT0QMA, exec->qma);
V3D_CORE_WRITE(0, V3D_CLE_CT0QMS, exec->qms);
}
if (exec->qts) {
V3D_CORE_WRITE(0, V3D_CLE_CT0QTS,
V3D_CLE_CT0QTS_ENABLE |
exec->qts);
}
} else {
/* XXX: Set the QCFG */
}
/* Set the current and end address of the control list.
* Writing the end register is what starts the job.
*/
V3D_CORE_WRITE(0, V3D_CLE_CTNQBA(q), job->start);
V3D_CORE_WRITE(0, V3D_CLE_CTNQEA(q), job->end);
return fence;
}
static struct dma_fence *
v3d_tfu_job_run(struct drm_sched_job *sched_job)
{
struct v3d_tfu_job *job = to_tfu_job(sched_job);
struct v3d_dev *v3d = job->v3d;
struct drm_device *dev = &v3d->drm;
struct dma_fence *fence;
fence = v3d_fence_create(v3d, V3D_TFU);
if (IS_ERR(fence))
return NULL;
v3d->tfu_job = job;
if (job->done_fence)
dma_fence_put(job->done_fence);
job->done_fence = dma_fence_get(fence);
trace_v3d_submit_tfu(dev, to_v3d_fence(fence)->seqno);
V3D_WRITE(V3D_TFU_IIA, job->args.iia);
V3D_WRITE(V3D_TFU_IIS, job->args.iis);
V3D_WRITE(V3D_TFU_ICA, job->args.ica);
V3D_WRITE(V3D_TFU_IUA, job->args.iua);
V3D_WRITE(V3D_TFU_IOA, job->args.ioa);
V3D_WRITE(V3D_TFU_IOS, job->args.ios);
V3D_WRITE(V3D_TFU_COEF0, job->args.coef[0]);
if (job->args.coef[0] & V3D_TFU_COEF0_USECOEF) {
V3D_WRITE(V3D_TFU_COEF1, job->args.coef[1]);
V3D_WRITE(V3D_TFU_COEF2, job->args.coef[2]);
V3D_WRITE(V3D_TFU_COEF3, job->args.coef[3]);
}
/* ICFG kicks off the job. */
V3D_WRITE(V3D_TFU_ICFG, job->args.icfg | V3D_TFU_ICFG_IOC);
return fence;
}
static void
v3d_gpu_reset_for_timeout(struct v3d_dev *v3d, struct drm_sched_job *sched_job)
{
enum v3d_queue q;
mutex_lock(&v3d->reset_lock);
/* block scheduler */
for (q = 0; q < V3D_MAX_QUEUES; q++) {
struct drm_gpu_scheduler *sched = &v3d->queue[q].sched;
kthread_park(sched->thread);
drm_sched_hw_job_reset(sched, (sched_job->sched == sched ?
sched_job : NULL));
}
/* get the GPU back into the init state */
v3d_reset(v3d);
/* Unblock schedulers and restart their jobs. */
for (q = 0; q < V3D_MAX_QUEUES; q++) {
drm_sched_job_recovery(&v3d->queue[q].sched);
kthread_unpark(v3d->queue[q].sched.thread);
}
mutex_unlock(&v3d->reset_lock);
}
static void
v3d_job_timedout(struct drm_sched_job *sched_job)
{
struct v3d_job *job = to_v3d_job(sched_job);
struct v3d_exec_info *exec = job->exec;
struct v3d_dev *v3d = exec->v3d;
enum v3d_queue job_q = job == &exec->bin ? V3D_BIN : V3D_RENDER;
u32 ctca = V3D_CORE_READ(0, V3D_CLE_CTNCA(job_q));
u32 ctra = V3D_CORE_READ(0, V3D_CLE_CTNRA(job_q));
/* If the current address or return address have changed, then
* the GPU has probably made progress and we should delay the
* reset. This could fail if the GPU got in an infinite loop
* in the CL, but that is pretty unlikely outside of an i-g-t
* testcase.
*/
if (job->timedout_ctca != ctca || job->timedout_ctra != ctra) {
job->timedout_ctca = ctca;
job->timedout_ctra = ctra;
return;
}
v3d_gpu_reset_for_timeout(v3d, sched_job);
}
static void
v3d_tfu_job_timedout(struct drm_sched_job *sched_job)
{
struct v3d_tfu_job *job = to_tfu_job(sched_job);
v3d_gpu_reset_for_timeout(job->v3d,
|