/*
* Copyright 2014 Advanced Micro Devices, Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#include <linux/mm_types.h>
#include <linux/slab.h>
#include <linux/types.h>
#include <linux/sched/signal.h>
#include <linux/sched/mm.h>
#include <linux/uaccess.h>
#include <linux/mman.h>
#include <linux/memory.h>
#include "kfd_priv.h"
#include "kfd_events.h"
#include "kfd_iommu.h"
#include <linux/device.h>
/*
* Wrapper around wait_queue_entry_t
*/
struct kfd_event_waiter {
wait_queue_entry_t wait;
struct kfd_event *event; /* Event to wait for */
bool activated; /* Becomes true when event is signaled */
};
/*
* Each signal event needs a 64-bit signal slot where the signaler will write
* a 1 before sending an interrupt. (This is needed because some interrupts
* do not contain enough spare data bits to identify an event.)
* We get whole pages and map them to the process VA.
* Individual signal events use their event_id as slot index.
*/
struct kfd_signal_page {
uint64_t *kernel_address;
uint64_t __user *user_address;
bool need_to_free_pages;
};
static uint64_t *page_slots(struct kfd_signal_page *page)
{
return page->kernel_address;
}
static struct kfd_signal_page *allocate_signal_page(struct kfd_process *p)
{
void *backing_store;
struct kfd_signal_page *page;
page = kzalloc(sizeof(*page), GFP_KERNEL);
if (!page)
return NULL;
backing_store = (void *) __get_free_pages(GFP_KERNEL,
get_order(KFD_SIGNAL_EVENT_LIMIT * 8));
if (!backing_store)
goto fail_alloc_signal_store;
/* Initialize all events to unsignaled */
memset(backing_store, (uint8_t) UNSIGNALED_EVENT_SLOT,
KFD_SIGNAL_EVENT_LIMIT * 8);
page->kernel_address = backing_store;
page->need_to_free_pages = true;
pr_debug("Allocated new event signal page at %p, for process %p\n",
page, p);
return page;
fail_alloc_signal_store:
kfree(page);
return NULL;
}
static int allocate_event_notification_slot(struct kfd_process *p,
struct kfd_event *ev)
{
int id;
if (!p->signal_page) {
p->signal_page = allocate_signal_page(p);
if (!p->signal_page)
return -ENOMEM;
/* Oldest user mode expects 256 event slots */
p->signal_mapped_size = 256*8;
}
/*
* Compatibility with old user mode: Only use signal slots
* user mode has mapped, may be less than
* KFD_SIGNAL_EVENT_LIMIT. This also allows future increase
* of the event limit without breaking user mode.
*/
id = idr_alloc(&p->event_idr, ev, 0, p->signal_mapped_size / 8,
GFP_KERNEL);
if (id < 0)
return id;
ev->event_id = id;
page_slots(p->signal_page)[id] = UNSIGNALED_EVENT_SLOT;
return 0;
}
/*
* Assumes that p->event_mutex is held and of course that p is not going
* away (current or locked).
*/
static struct kfd_event *lookup_event_by_id(struct kfd_process *p, uint32_t id)
{
return idr_find(&p->event_idr, id);
}
/**
* lookup_signaled_event_by_partial_id - Lookup signaled event from partial ID
* @p: Pointer to struct kfd_process
* @id: ID to look up
* @bits: Number of valid bits in @id
*
* Finds the first signaled event with a matching partial ID. If no
* matching signaled event is found, returns NULL. In that case the
* caller should assume that the partial ID is invalid and do an
* exhaustive search of all siglaned events.
*
* If multiple events with the same partial ID signal at the same
* time, they will be found one interrupt at a time, not necessarily
* in the same order the interrupts occurred. As long as the number of
* interrupts is correct, all signaled events will be seen by the
* driver.
*/
static struct kfd_event *lookup_signaled_event_by_partial_id(
struct kfd_process *p, uint32_t id, uint32_t bits)
{
struct kfd_event *ev;
if (!p->signal_page || id >= KFD_SIGNAL_EVENT_LIMIT)
return NULL;
/* Fast path for the common case that @id is not a partial ID
* and we only need a single lookup.
*/
if (bits > 31 || (1U << bits) >= KFD_SIGNAL_EVENT_LIMIT) {
if (page_slots(p->signal_page)[id] == UNSIGNALED_EVENT_SLOT)
return NULL;
return idr_find(&p->event_idr, id);
}
/* General case for partial IDs: Iterate over all matching IDs
* and find the first one that has signaled.
*/
for (ev = NULL; id < KFD_SIGNAL_EVENT_LIMIT && !ev; id += 1U <<