1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
|
/*
* Copyright (C) 2007-2008 Advanced Micro Devices, Inc.
* Author: Joerg Roedel <joerg.roedel@amd.com>
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 as published
* by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
#define pr_fmt(fmt) "%s: " fmt, __func__
#include <linux/device.h>
#include <linux/kernel.h>
#include <linux/bug.h>
#include <linux/types.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/errno.h>
#include <linux/iommu.h>
static ssize_t show_iommu_group(struct device *dev,
struct device_attribute *attr, char *buf)
{
unsigned int groupid;
if (iommu_device_group(dev, &groupid))
return 0;
return sprintf(buf, "%u", groupid);
}
static DEVICE_ATTR(iommu_group, S_IRUGO, show_iommu_group, NULL);
static int add_iommu_group(struct device *dev, void *data)
{
unsigned int groupid;
if (iommu_device_group(dev, &groupid) == 0)
return device_create_file(dev, &dev_attr_iommu_group);
return 0;
}
static int remove_iommu_group(struct device *dev)
{
unsigned int groupid;
if (iommu_device_group(dev, &groupid) == 0)
device_remove_file(dev, &dev_attr_iommu_group);
return 0;
}
static int iommu_device_notifier(struct notifier_block *nb,
unsigned long action, void *data)
{
struct device *dev = data;
if (action == BUS_NOTIFY_ADD_DEVICE)
return add_iommu_group(dev, NULL);
else if (action == BUS_NOTIFY_DEL_DEVICE)
return remove_iommu_group(dev);
return 0;
}
static struct notifier_block iommu_device_nb = {
.notifier_call = iommu_device_notifier,
};
static void iommu_bus_init(struct bus_type *bus, struct iommu_ops *ops)
{
bus_register_notifier(bus, &iommu_device_nb);
bus_for_each_dev(bus, NULL, NULL, add_iommu_group);
}
/**
* bus_set_iommu - set iommu-callbacks for the bus
* @bus: bus.
* @ops: the callbacks provided by the iommu-driver
*
* This function is called by an iommu driver to set the iommu methods
* used for a particular bus. Drivers for devices on that bus can use
* the iommu-api after these ops are registered.
* This special function is needed because IOMMUs are usually devices on
* the bus itself, so the iommu drivers are not initialized when the bus
* is set up. With this function the iommu-driver can set the iommu-ops
* afterwards.
*/
int bus_set_iommu(struct bus_type *bus, struct iommu_ops *ops)
{
if (bus->iommu_ops != NULL)
return -EBUSY;
bus->iommu_ops = ops;
/* Do IOMMU specific setup for this bus-type */
iommu_bus_init(bus, ops);
return 0;
}
EXPORT_SYMBOL_GPL(bus_set_iommu);
bool iommu_present(struct bus_type *bus)
{
return bus->iommu_ops != NULL;
}
EXPORT_SYMBOL_GPL(iommu_present);
/**
* iommu_set_fault_handler() - set a fault handler for an iommu domain
* @domain: iommu domain
* @handler: fault handler
*
* This function should be used by IOMMU users which want to be notified
* whenever an IOMMU fault happens.
*
* The fault handler itself should return 0 on success, and an appropriate
* error code otherwise.
*/
void iommu_set_fault_handler(struct iommu_domain *domain,
iommu_fault_handler_t handler)
{
BUG_ON(!domain);
domain->handler = handler;
}
EXPORT_SYMBOL_GPL(iommu_set_fault_handler);
struct iommu_domain *iommu_domain_alloc(struct bus_type *bus)
{
struct iommu_domain *domain;
int ret;
if (bus == NULL || bus->iommu_ops == NULL)
return NULL;
domain = kzalloc(sizeof(*domain), GFP_KERNEL);
if (!domain)
return NULL;
domain->ops = bus->iommu_ops;
ret = domain->ops->domain_init(domain);
if (ret)
goto out_free;
return domain;
out_free:
kfree(domain);
return NULL;
}
EXPORT_SYMBOL_GPL(iommu_domain_alloc);
void iommu_domain_free(struct iommu_domain *domain)
{
if (likely(domain->ops->domain_destroy != NULL))
domain->ops->domain_destroy(domain);
kfree(domain);
}
EXPORT_SYMBOL_GPL(iommu_domain_free);
int iommu_attach_device(struct iommu_domain *domain, struct device *dev)
{
if (unlikely(domain->ops->attach_dev == NULL))
return -ENODEV;
return domain->ops->attach_dev(domain, dev);
}
EXPORT_SYMBOL_GPL(iommu_attach_device);
void iommu_detach_device(struct iommu_domain *domain, struct device *dev)
{
if (unlikely(domain->ops->detach_dev == NULL))
return;
domain->ops->detach_dev(domain, dev);
}
EXPORT_SYMBOL_GPL(iommu_detach_device);
phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain,
unsigned long iova)
{
if (unlikely(domain->ops->iova_to_phys == NULL))
return 0;
return domain->ops->iova_to_phys(domain, iova);
}
EXPORT_SYMBOL_GPL(iommu_iova_to_phys);
int iommu_domain_has_cap(struct iommu_domain *domain,
unsigned long cap)
{
if (unlikely(domain->ops->domain_has_cap == NULL))
return 0;
return domain->ops->domain_has_cap(domain, cap);
}
EXPORT_SYMBOL_GPL(iommu_domain_has_cap);
int iommu_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t size, int prot)
{
unsigned long orig_iova = iova;
unsigned int min_pagesz;
size_t orig_size = size;
int ret = 0;
if (unlikely(domain->ops->map == NULL))
return -ENODEV;
/* find out the minimum page size supported */
min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap);
/*
* both the virtual address and the physical one, as well as
* the size of the mapping, must be aligned (at least) to the
* size of the smallest page supported by the hardware
*/
if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) {
pr_err("unaligned: iova 0x%lx pa 0x%lx size 0x%lx min_pagesz "
"0x%x\n", iova, (unsigned long)paddr,
(unsigned long)size, min_pagesz);
return -EINVAL;
}
pr_debug("map: iova 0x%lx pa 0x%lx size 0x%lx\n", iova,
(unsigned long)paddr, (unsigned long)size);
while (size) {
unsigned long pgsize, addr_merge = iova | paddr;
unsigned int pgsize_idx;
/* Max page size that still fits into 'size' */
pgsize_idx = __fls(size);
/* need to consider alignment requirements ? */
if (likely(addr_merge)) {
/* Max page size allowed by both iova and paddr */
unsigned int align_pgsize_idx = __ffs(addr_merge);
pgsize_idx = min(pgsize_idx, align_pgsize_idx);
}
/* build a mask of acceptable page sizes */
pgsize = (1UL << (pgsize_idx + 1)) - 1;
/* throw away page sizes not supported by the hardware */
pgsize &= domain->ops->pgsize_bitmap;
/* make sure we're still sane */
BUG_ON(!pgsize);
/* pick the biggest page */
pgsize_idx = __fls(pgsize);
pgsize = 1UL << pgsize_idx;
pr_debug("mapping: iova 0x%lx pa 0x%lx pgsize %lu\n", iova,
(unsigned long)paddr, pgsize);
ret = domain->ops->map(domain, iova, paddr, pgsize, prot);
if (ret)
break;
iova += pgsize;
paddr += pgsize;
size -= pgsize;
}
/* unroll mapping in case something went wrong */
if (ret)
iommu_unmap(domain, orig_iova, orig_size - size);
return ret;
}
EXPORT_SYMBOL_GPL(iommu_map);
size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
{
size_t unmapped_page, unmapped = 0;
unsigned int min_pagesz;
if (unlikely(domain->ops->unmap == NULL))
return -ENODEV;
/* find out the minimum page size supported */
min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap);
/*
* The virtual address, as well as the size of the mapping, must be
* aligned (at least) to the size of the smallest page supported
* by the hardware
*/
if (!IS_ALIGNED(iova | size, min_pagesz)) {
pr_err("unaligned: iova 0x%lx size 0x%lx min_pagesz 0x%x\n",
iova, (unsigned long)size, min_pagesz);
return -EINVAL;
}
pr_debug("unmap this: iova 0x%lx size 0x%lx\n", iova,
(unsigned long)size);
/*
* Keep iterating until we either unmap 'size' bytes (or more)
* or we hit an area that isn't mapped.
*/
while (unmapped
|