1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
|
/*
* DMA-able FIFO implementation
*
* Copyright (C) 2012 Peter Hurley <peter@hurleysoftware.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*/
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/list.h>
#include <linux/bug.h>
#include "dma_fifo.h"
#ifdef DEBUG_TRACING
#define df_trace(s, args...) pr_debug(s, ##args)
#else
#define df_trace(s, args...)
#endif
#define FAIL(fifo, condition, format...) ({ \
fifo->corrupt = !!(condition); \
WARN(fifo->corrupt, format); \
})
/*
* private helper fn to determine if check is in open interval (lo,hi)
*/
static bool addr_check(unsigned int check, unsigned int lo, unsigned int hi)
{
return check - (lo + 1) < (hi - 1) - lo;
}
/**
* dma_fifo_init: initialize the fifo to a valid but inoperative state
* @fifo: address of in-place "struct dma_fifo" object
*/
void dma_fifo_init(struct dma_fifo *fifo)
{
memset(fifo, 0, sizeof(*fifo));
INIT_LIST_HEAD(&fifo->pending);
}
/**
* dma_fifo_alloc - initialize and allocate dma_fifo
* @fifo: address of in-place "struct dma_fifo" object
* @size: 'apparent' size, in bytes, of fifo
* @align: dma alignment to maintain (should be at least cpu cache alignment),
* must be power of 2
* @tx_limit: maximum # of bytes transmissible per dma (rounded down to
* multiple of alignment, but at least align size)
* @open_limit: maximum # of outstanding dma transactions allowed
* @gfp_mask: get_free_pages mask, passed to kmalloc()
*
* The 'apparent' size will be rounded up to next greater aligned size.
* Returns 0 if no error, otherwise an error code
*/
int dma_fifo_alloc(struct dma_fifo *fifo, int size, unsigned int align,
int tx_limit, int open_limit, gfp_t gfp_mask)
{
int capacity;
if (!is_power_of_2(align) || size < 0)
return -EINVAL;
size = round_up(size, align);
capacity = size + align * open_limit + align * DMA_FIFO_GUARD;
fifo->data = kmalloc(capacity, gfp_mask);
if (!fifo->data)
return -ENOMEM;
fifo->in = 0;
fifo->out = 0;
fifo->done = 0;
fifo->size = size;
fifo->avail = size;
fifo->align = align;
fifo->tx_limit = max_t(int, round_down(tx_limit, align), align);
fifo->open = 0;
fifo->open_limit = open_limit;
fifo->guard = size + align * open_limit;
fifo->capacity = capacity;
fifo->corrupt = 0;
return 0;
}
/**
* dma_fifo_free - frees the fifo
* @fifo: address of in-place "struct dma_fifo" to free
*
* Also reinits the fifo to a valid but inoperative state. This
* allows the fifo to be reused with a different target requiring
* different fifo parameters.
*/
void dma_fifo_free(struct dma_fifo *fifo)
{
struct dma_pending *pending, *next;
if (!fifo->data)
return;
list_for_each_entry_safe(pending, next, &fifo->pending, link)
list_del_init(&pending->link);
kfree(fifo->data);
fifo->data = NULL;
}
/**
* dma_fifo_reset - dumps the fifo contents and reinits for reuse
* @fifo: address of in-place "struct dma_fifo" to reset
*/
void dma_fifo_reset(struct dma_fifo *fifo)
{
struct dma_pending *pending, *next;
if (!fifo->data)
return;
list_for_each_entry_safe(pending, next, &fifo->pending, link)
list_del_init(&pending->link);
fifo->in = 0;
fifo->out = 0;
fifo->done = 0;
fifo->avail = fifo->size;
fifo->open = 0;
fifo->corrupt = 0;
}
/**
* dma_fifo_in - copies data into the fifo
* @fifo: address of in-place "struct dma_fifo" to write to
* @src: buffer to copy from
* @n: # of bytes to copy
*
* Returns the # of bytes actually copied, which can be less than requested if
* the fifo becomes full. If < 0, return is error code.
*/
int dma_fifo_in(struct dma_fifo *fifo, const void *src, int n)
{
int ofs, l;
if (!fifo->data)
return -ENOENT;
if (fifo->corrupt)
return -ENXIO;
if (n > fifo->avail)
n = fifo->avail;
if (n <= 0)
return 0;
ofs = fifo->in % fifo->capacity;
l = min(n, fifo->capacity - ofs);
memcpy(fifo->data + ofs, src, l);
memcpy(fifo->data, src + l, n - l);
if (FAIL(fifo, addr_check(fifo->done, fifo->in, fifo->in + n) ||
fifo->avail < n,
"fifo corrupt: in:%u out:%u done:%u n:%d avail:%d",
fifo->in, fifo->out, fifo->done, n, fifo->avail))
return -ENXIO;
fifo->in += n;
fifo->avail -= n;
df_trace("in:%u out:%u done:%u n:%d avail:%d", fifo->in, fifo->out,
fifo->done, n, fifo->avail);
return n;
}
/**
* dma_fifo_out_pend - gets address/len of next avail read and marks as pended
* @fifo: address of in-place "struct dma_fifo" to read from
* @pended: address of structure to fill with read address/len
* The data/len fields will be NULL/0 if no dma is pended.
*
* Returns the # of used bytes remaining in fifo (ie, if > 0, more data
* remains in the fifo that was not pended). If < 0, return is error code.
*/
int dma_fifo_out_pend(struct dma_fifo *fifo, struct dma_pending *pended)
{
unsigned int len, n, ofs, l, limit;
if (!fifo->data)
return -ENOENT;
if (fifo->corrupt)
return -ENXIO;
pended->len = 0;
pended->data = NULL;
pended->out = fifo->out;
len = fifo->in - fifo->out;
if (!len)
return -ENODATA;
if (fifo->open == fifo->open_limit)
return -EAGAIN;
n = len;
ofs = fifo->out % fifo->capacity;
l = fifo->capacity - ofs;
limit = min_t(unsigned int, l, fifo->tx_limit);
if (n > limit) {
n = limit;
fifo->out += limit;
} else if (ofs + n > fifo->guard) {
fifo->out += l;
fifo->in = fifo->out;
} else {
fifo->out += round_up(n, fifo->align);
fifo->in = fifo->out;
}
df_trace("in: %u out: %u done: %u n: %d len: %u avail: %d", fifo->in,
fifo->out, fifo->done, n, len, fifo->avail);
pended->len = n;
pended->data = fifo->data + ofs;
pended->next = fifo->out;
list_add_tail(&pended->link, &fifo->pending);
++fifo->open;
if (FAIL(fifo, fifo->open > fifo->open_limit,
"past open limit:%d (limit:%d)",
fifo->open, fifo->open_limit))
return -ENXIO;
if (FAIL(fifo, fifo->out & (fifo->align - 1),
"fifo out unaligned:%u (align:%u)",
fifo->out, fifo->align))
return -ENXIO;
return len - n;
}
/**
* dma_fifo_out_complete - marks pended dma as completed
* @fifo: address of in-place "struct dma_fifo" which was read from
* @complete: address of structure for previously pended dma to mark completed
*/
int dma_fifo_out_complete(struct dma_fifo *fifo, struct dma_pending *complete)
{
struct dma_pending *pending, *next, *tmp;
if (!fifo->data)
return -ENOENT;
if (fifo->corrupt)
return -ENXIO;
if (list_empty(&fifo->pending) && fifo->open == 0)
return -EINVAL;
if (FAIL(fifo, list_empty(&fifo->pending) != (fifo->open == 0),
"pending list disagrees with open count:%d",
fifo->open))
return -ENXIO;
tmp = complete->data;
*tmp = *complete;
list_replace(&complete->link, &tmp->link);
dp_mark_completed(tmp);
/* Only update the fifo in the original pended order */
list_for_each_entry_safe(pending, next, &fifo->pending, link) {
if (!dp_is_completed(pending)) {
df_trace("still pending: saved out: %u len: %d",
pending->out, pending->len);
break;
}
if (FAIL(fifo, pending->out !=
|