[IA64] move XP and XPC to drivers/misc/sgi-xp

Move XPC and XPNET from arch/ia64/sn/kernel to drivers/misc/sgi-xp.

Signed-off-by: Dean Nelson <dcn@sgi.com>
Signed-off-by: Tony Luck <tony.luck@intel.com>

1 files changed, 2379 insertions, 0 deletions

diff --git a/drivers/misc/sgi-xp/xpc_channel.c b/drivers/misc/sgi-xp/xpc_channel.c
new file mode 100644
index 000000000000..d7a215eeaaf6
--- /dev/null
+++ b/drivers/misc/sgi-xp/xpc_channel.c
@@ -0,0 +1,2379 @@
+/*
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * Copyright (c) 2004-2008 Silicon Graphics, Inc.  All Rights Reserved.
+ */
+
+
+/*
+ * Cross Partition Communication (XPC) channel support.
+ *
+ *	This is the part of XPC that manages the channels and
+ *	sends/receives messages across them to/from other partitions.
+ *
+ */
+
+
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/sched.h>
+#include <linux/cache.h>
+#include <linux/interrupt.h>
+#include <linux/mutex.h>
+#include <linux/completion.h>
+#include <asm/sn/bte.h>
+#include <asm/sn/sn_sal.h>
+#include "xpc.h"
+
+
+/*
+ * Guarantee that the kzalloc'd memory is cacheline aligned.
+ */
+static void *
+xpc_kzalloc_cacheline_aligned(size_t size, gfp_t flags, void **base)
+{
+	/* see if kzalloc will give us cachline aligned memory by default */
+	*base = kzalloc(size, flags);
+	if (*base == NULL) {
+		return NULL;
+	}
+	if ((u64) *base == L1_CACHE_ALIGN((u64) *base)) {
+		return *base;
+	}
+	kfree(*base);
+
+	/* nope, we'll have to do it ourselves */
+	*base = kzalloc(size + L1_CACHE_BYTES, flags);
+	if (*base == NULL) {
+		return NULL;
+	}
+	return (void *) L1_CACHE_ALIGN((u64) *base);
+}
+
+
+/*
+ * Set up the initial values for the XPartition Communication channels.
+ */
+static void
+xpc_initialize_channels(struct xpc_partition *part, partid_t partid)
+{
+	int ch_number;
+	struct xpc_channel *ch;
+
+
+	for (ch_number = 0; ch_number < part->nchannels; ch_number++) {
+		ch = &part->channels[ch_number];
+
+		ch->partid = partid;
+		ch->number = ch_number;
+		ch->flags = XPC_C_DISCONNECTED;
+
+		ch->local_GP = &part->local_GPs[ch_number];
+		ch->local_openclose_args =
+					&part->local_openclose_args[ch_number];
+
+		atomic_set(&ch->kthreads_assigned, 0);
+		atomic_set(&ch->kthreads_idle, 0);
+		atomic_set(&ch->kthreads_active, 0);
+
+		atomic_set(&ch->references, 0);
+		atomic_set(&ch->n_to_notify, 0);
+
+		spin_lock_init(&ch->lock);
+		mutex_init(&ch->msg_to_pull_mutex);
+		init_completion(&ch->wdisconnect_wait);
+
+		atomic_set(&ch->n_on_msg_allocate_wq, 0);
+		init_waitqueue_head(&ch->msg_allocate_wq);
+		init_waitqueue_head(&ch->idle_wq);
+	}
+}
+
+
+/*
+ * Setup the infrastructure necessary to support XPartition Communication
+ * between the specified remote partition and the local one.
+ */
+enum xpc_retval
+xpc_setup_infrastructure(struct xpc_partition *part)
+{
+	int ret, cpuid;
+	struct timer_list *timer;
+	partid_t partid = XPC_PARTID(part);
+
+
+	/*
+	 * Zero out MOST of the entry for this partition. Only the fields
+	 * starting with `nchannels' will be zeroed. The preceding fields must
+	 * remain `viable' across partition ups and downs, since they may be
+	 * referenced during this memset() operation.
+	 */
+	memset(&part->nchannels, 0, sizeof(struct xpc_partition) -
+				offsetof(struct xpc_partition, nchannels));
+
+	/*
+	 * Allocate all of the channel structures as a contiguous chunk of
+	 * memory.
+	 */
+	part->channels = kzalloc(sizeof(struct xpc_channel) * XPC_NCHANNELS,
+								GFP_KERNEL);
+	if (part->channels == NULL) {
+		dev_err(xpc_chan, "can't get memory for channels\n");
+		return xpcNoMemory;
+	}
+
+	part->nchannels = XPC_NCHANNELS;
+
+
+	/* allocate all the required GET/PUT values */
+
+	part->local_GPs = xpc_kzalloc_cacheline_aligned(XPC_GP_SIZE,
+					GFP_KERNEL, &part->local_GPs_base);
+	if (part->local_GPs == NULL) {
+		kfree(part->channels);
+		part->channels = NULL;
+		dev_err(xpc_chan, "can't get memory for local get/put "
+			"values\n");
+		return xpcNoMemory;
+	}
+
+	part->remote_GPs = xpc_kzalloc_cacheline_aligned(XPC_GP_SIZE,
+					GFP_KERNEL, &part->remote_GPs_base);
+	if (part->remote_GPs == NULL) {
+		dev_err(xpc_chan, "can't get memory for remote get/put "
+			"values\n");
+		kfree(part->local_GPs_base);
+		part->local_GPs = NULL;
+		kfree(part->channels);
+		part->channels = NULL;
+		return xpcNoMemory;
+	}
+
+
+	/* allocate all the required open and close args */
+
+	part->local_openclose_args = xpc_kzalloc_cacheline_aligned(
+					XPC_OPENCLOSE_ARGS_SIZE, GFP_KERNEL,
+					&part->local_openclose_args_base);
+	if (part->local_openclose_args == NULL) {
+		dev_err(xpc_chan, "can't get memory for local connect args\n");
+		kfree(part->remote_GPs_base);
+		part->remote_GPs = NULL;
+		kfree(part->local_GPs_base);
+		part->local_GPs = NULL;
+		kfree(part->channels);
+		part->channels = NULL;
+		return xpcNoMemory;
+	}
+
+	part->remote_openclose_args = xpc_kzalloc_cacheline_aligned(
+					XPC_OPENCLOSE_ARGS_SIZE, GFP_KERNEL,
+					&part->remote_openclose_args_base);
+	if (part->remote_openclose_args == NULL) {
+		dev_err(xpc_chan, "can't get memory for remote connect args\n");
+		kfree(part->local_openclose_args_base);
+		part->local_openclose_args = NULL;
+		kfree(part->remote_GPs_base);
+		part->remote_GPs = NULL;
+		kfree(part->local_GPs_base);
+		part->local_GPs = NULL;
+		kfree(part->channels);
+		part->channels = NULL;
+		return xpcNoMemory;
+	}
+
+
+	xpc_initialize_channels(part, partid);
+
+	atomic_set(&part->nchannels_active, 0);
+	atomic_set(&part->nchannels_engaged, 0);
+
+
+	/* local_IPI_amo were set to 0 by an earlier memset() */
+
+	/* Initialize this partitions AMO_t structure */
+	part->local_IPI_amo_va = xpc_IPI_init(partid);
+
+	spin_lock_init(&part->IPI_lock);
+
+	atomic_set(&part->channel_mgr_requests, 1);
+	init_waitqueue_head(&part->channel_mgr_wq);
+
+	sprintf(part->IPI_owner, "xpc%02d", partid);
+	ret = request_irq(SGI_XPC_NOTIFY, xpc_notify_IRQ_handler, IRQF_SHARED,
+				part->IPI_owner, (void *) (u64) partid);
+	if (ret != 0) {
+		dev_err(xpc_chan, "can't register NOTIFY IRQ handler, "
+			"errno=%d\n", -ret);
+		kfree(part->remote_openclose_args_base);
+		part->remote_openclose_args = NULL;
+		kfree(part->local_openclose_args_base);
+		part->local_openclose_args = NULL;
+		kfree(part->remote_GPs_base);
+		part->remote_GPs = NULL;
+		kfree(part->local_GPs_base);
+		part->local_GPs = NULL;
+		kfree(part->channels);
+		part->channels = NULL;
+		return xpcLackOfResources;
+	}
+
+	/* Setup a timer to check for dropped IPIs */
+	timer = &part->dropped_IPI_timer;
+	init_timer(timer);
+	timer->function = (void (*)(unsigned long)) xpc_dropped_IPI_check;
+	timer->data = (unsigned long) part;
+	timer->expires = jiffies + XPC_P_DROPPED_IPI_WAIT;
+	add_timer(timer);
+
+	/*
+	 * With the setting of the partition setup_state to XPC_P_SETUP, we're
+	 * declaring that this partition is ready to go.
+	 */
+	part->setup_state = XPC_P_SETUP;
+
+
+	/*
+	 * Setup the per partition specific variables required by the
+	 * remote partition to establish channel connections with us.
+	 *
+	 * The setting of the magic # indicates that these per partition
+	 * specific variables are ready to be used.
+	 */
+	xpc_vars_part[partid].GPs_pa = __pa(part->local_GPs);
+	xpc_vars_part[partid].openclose_args_pa =
+					__pa(part->local_openclose_args);
+	xpc_vars_part[partid].IPI_amo_pa = __pa(part->local_IPI_amo_va);
+	cpuid = raw_smp_processor_id();	/* any CPU in this partition will do */
+	xpc_vars_part[partid].IPI_nasid = cpuid_to_nasid(cpuid);
+	xpc_vars_part[partid].IPI_phys_cpuid = cpu_physical_id(cpuid);
+	xpc_vars_part[partid].nchannels = part->nchannels;
+	xpc_vars_part[partid].magic = XPC_VP_MAGIC1;
+
+	return xpcSuccess;
+}
+
+
+/*
+ * Create a wrapper that hides the underlying mechanism for pulling a cacheline
+ * (or multiple cachelines) from a remote partition.
+ *
+ * src must be a cacheline aligned physical address on the remote partition.
+ * dst must be a cacheline aligned virtual address on this partition.
+ * cnt must be an cacheline sized
+ */
+static enum xpc_retval
+xpc_pull_remote_cachelines(struct xpc_partition *part, void *dst,
+				const void *src, size_t cnt)
+{
+	bte_result_t bte_ret;
+
+
+	DBUG_ON((u64) src != L1_CACHE_ALIGN((u64) src));
+	DBUG_ON((u64) dst != L1_CACHE_ALIGN((u64) dst));
+	DBUG_ON(cnt != L1_CACHE_ALIGN(cnt));
+
+	if (part->act_state == XPC_P_DEACTIVATING) {
+		return part->reason;
+	}
+
+	bte_ret = xp_bte_copy((u64) src, (u64) dst, (u64) cnt,
+					(BTE_NORMAL | BTE_WACQUIRE), NULL);
+	if (bte_ret == BTE_SUCCESS) {
+		return xpcSuccess;
+	}
+
+	dev_dbg(xpc_chan, "xp_bte_copy() from partition %d failed, ret=%d\n",
+		XPC_PARTID(part), bte_ret);
+
+	return xpc_map_bte_errors(bte_ret);
+}
+
+
+/*
+ * Pull the remote per partition specific variables from the specified
+ * partition.
+ */
+enum xpc_retval
+xpc_pull_remote_vars_part(struct xpc_partition *part)
+{
+	u8 buffer[L1_CACHE_BYTES * 2];
+	struct xpc_vars_part *pulled_entry_cacheline =
+			(struct xpc_vars_part *) L1_CACHE_ALIGN((u64) buffer);
+	struct xpc_vars_part *pulled_entry;
+	u64 remote_entry_cacheline_pa, remote_entry_pa;
+	partid_t partid = XPC_PARTID(part);
+	enum xpc_retval ret;
+
+
+	/* pull the cacheline that contains the variables we're interested in */
+
+	DBUG_ON(part->remote_vars_part_pa !=
+				L1_CACHE_ALIGN(part->remote_vars_part_pa));
+	DBUG_ON(sizeof(struct xpc_vars_part) != L1_CACHE_BYTES / 2);
+
+	remote_entry_pa = part->remote_vars_part_pa +
+			sn_partition_id * sizeof(struct xpc_vars_part);
+
+	remote_entry_cacheline_pa = (remote_entry_pa & ~(L1_CACHE_BYTES - 1));
+
+	pulled_entry = (struct xpc_vars_part *) ((u64) pulled_entry_cacheline +
+				(remote_entry_pa & (L1_CACHE_BYTES - 1)));
+
+	ret = xpc_pull_remote_cachelines(part, pulled_entry_cacheline,
+					(void *) remote_entry_cacheline_pa,
+					L1_CACHE_BYTES);
+	if (ret != xpcSuccess) {
+		dev_dbg(xpc_chan, "failed to pull XPC vars_part from "
+			"partition %d, ret=%d\n", partid, ret);
+		return ret;
+	}
+
+
+	/* see if they've been set up yet */
+
+	if (pulled_entry->magic != XPC_VP_MAGIC1 &&
+				pulled_entry->magic != XPC_VP_MAGIC2) {
+
+		if (pulled_entry->magic != 0) {
+			dev_dbg(xpc_chan, "partition %d's XPC vars_part for "
+				"partition %d has bad magic value (=0x%lx)\n",
+				partid, sn_partition_id, pulled_entry->magic);
+			return xpcBadMagic;
+		}
+
+		/* they've not been initialized yet */
+		return xpcRetry;
+	}
+
+	if (xpc_vars_part[partid].magic == XPC_VP_MAGIC1) {
+
+		/* validate the variables */
+
+		if (pulled_entry->GPs_pa == 0 ||
+				pulled_entry->openclose_args_pa == 0 ||
+					pulled_entry->IPI_amo_pa == 0) {
+
+			dev_err(xpc_chan, "partition %d's XPC vars_part for "
+				"partition %d are not valid\n", partid,
+				sn_partition_id);
+			return xpcInvalidAddress;
+		}
+
+		/* the variables we imported look to be valid */
+
+		part->remote_GPs_pa = pulled_entry->GPs_pa;
+		part->remote_openclose_args_pa =
+					pulled_entry->openclose_args_pa;
+		part->remote_IPI_amo_va =
+				      (AMO_t *) __va(pulled_entry->IPI_amo_pa);
+		part->remote_IPI_nasid = pulled_entry->IPI_nasid;
+		part->remote_IPI_phys_cpuid = pulled_entry->IPI_phys_cpuid;
+
+		if (part->nchannels > pulled_entry->nchannels) {
+			part->nchannels = pulled_entry->nchannels;
+		}
+
+		/* let the other side know that we've pulled their variables */
+
+		xpc_vars_part[partid].magic = XPC_VP_MAGIC2;
+	}
+
+	if (pulled_entry->magic == XPC_VP_MAGIC1) {
+		return xpcRetry;
+	}
+
+	return xpcSuccess;
+}
+
+
+/*
+ * Get the IPI flags and pull the openclose args and/or remote GPs as needed.
+ */
+static u64
+xpc_get_IPI_flags(struct xpc_partition *part)
+{
+	unsigned long irq_flags;
+	u64 IPI_amo;
+	enum xpc_retval ret;
+
+
+	/*
+	 * See if there are any IPI flags to be handled.
+	 */
+
+	spin_lock_irqsave(&part->IPI_lock, irq_flags);
+	if ((IPI_amo = part->local_IPI_amo) != 0) {
+		part->local_IPI_amo = 0;
+	}
+	spin_unlock_irqrestore(&part->IPI_lock, irq_flags);
+
+
+	if (XPC_ANY_OPENCLOSE_IPI_FLAGS_SET(IPI_amo)) {
+		ret = xpc_pull_remote_cachelines(part,
+					part->remote_openclose_args,
+					(void *) part->remote_openclose_args_pa,
+					XPC_OPENCLOSE_ARGS_SIZE);
+		if (ret != xpcSuccess) {
+			XPC_DEACTIVATE_PARTITION(part, ret);
+
+			dev_dbg(xpc_chan, "failed to pull openclose args from "
+				"partition %d, ret=%d\n", XPC_PARTID(part),
+				ret);
+
+			/* don't bother processing IPIs anymore */
+			IPI_amo = 0;
+		}
+	}
+
+	if (XPC_ANY_MSG_IPI_FLAGS_SET(IPI_amo)) {
+		ret = xpc_pull_remote_cachelines(part, part->remote_GPs,
+						(void *) part->remote_GPs_pa,
+						XPC_GP_SIZE);
+		if (ret != xpcSuccess) {
+			XPC_DEACTIVATE_PARTITION(part, ret);
+
+			dev_dbg(xpc_chan, "failed to pull GPs from partition "
+				"%d, ret=%d\n", XPC_PARTID(part), ret);
+
+			/* don't bother processing IPIs anymore */
+			IPI_amo = 0;
+		}
+	}
+
+	return IPI_amo;
+}
+
+
+/*
+ * Allocate the local message queue and the notify queue.
+ */
+static enum xpc_retval
+xpc_allocate_local_msgqueue(struct xpc_channel *ch)
+{
+	unsigned long irq_flags;
+	int nentries;
+	size_t nbytes;
+
+
+	// >>> may want to check for ch->flags & XPC_C_DISCONNECTING between
+	// >>> iterations of the for-loop, bail if set?
+
+	// >>> should we impose a minimum #of entries? like 4 or 8?
+	for (nentries = ch->local_nentries; nentries > 0; nentries--) {
+
+		nbytes = nentries * ch->msg_size;
+		ch->local_msgqueue = xpc_kzalloc_cacheline_aligned(nbytes,
+						GFP_KERNEL,
+						&ch->local_msgqueue_base);
+		if (ch->local_msgqueue == NULL) {
+			continue;
+		}
+
+		nbytes = nentries * sizeof(struct xpc_notify);
+		ch->notify_queue = kzalloc(nbytes, GFP_KERNEL);
+		if (ch->notify_queue == NULL) {
+			kfree(ch->local_msgqueue_base);
+			ch->local_msgqueue = NULL;
+			continue;
+		}
+
+		spin_lock_irqsave(&ch->lock, irq_flags);
+		if (nentries < ch->local_nentries) {
+			dev_dbg(xpc_chan, "nentries=%d local_nentries=%d, "
+				"partid=%d, channel=%d\n", nentries,
+				ch->local_nentries, ch->partid, ch->number);
+
+			ch->local_nentries = nentries;
+		}
+		spin_unlock_irqrestore(&ch->lock, irq_flags);
+		return xpcSuccess;
+	}
+
+	dev_dbg(xpc_chan, "can't get memory for local message queue and notify "
+		"queue, partid=%d, channel=%d\n", ch->partid, ch->number);
+	return xpcNoMemory;
+}
+
+
+/*
+ * Allocate the cached remote message queue.
+ */
+static enum xpc_retval
+xpc_allocate_remote_msgqueue(struct xpc_channel *ch)
+{
+	unsigned long irq_flags;
+	int nentries;
+	size_t nbytes;
+
+
+	DBUG_ON(ch->remote_nentries <= 0);
+
+	// >>> may want to check for ch->flags & XPC_C_DISCONNECTING between
+	// >>> iterations of the for-loop, bail if set?
+
+	// >>> should we impose a minimum #of entries? like 4 or 8?
+	for (nentries = ch->remote_nentries; nentries > 0; nentries--) {
+
+		nbytes = nentries * ch->msg_size;
+		ch->remote_msgqueue = xpc_kzalloc_cacheline_aligned(nbytes,
+						GFP_KERNEL,
+						&ch->remote_msgqueue_base);
+		if (ch->remote_msgqueue == NULL) {
+			continue;
+		}
+
+		spin_lock_irqsave(&ch->lock, irq_flags);
+		if (nentries < ch->remote_nentries) {
+			dev_dbg(xpc_chan, "nentries=%d remote_nentries=%d, "
+				"partid=%d, channel=%d\n", nentries,
+				ch->remote_nentries, ch->partid, ch->number);
+
+			ch->remote_nentries = nentries;
+		}
+		spin_unlock_irqrestore(&ch->lock, irq_flags);
+		return xpcSuccess;
+	}
+
+	dev_dbg(xpc_chan, "can't get memory for cached remote message queue, "
+		"partid=%d, channel=%d\n", ch->partid, ch->number);
+	return xpcNoMemory;
+}
+
+
+/*
+ * Allocate message queues and other stuff associated with a channel.
+ *
+ * Note: Assumes all of the channel sizes are filled in.
+ */
+static enum xpc_retval
+xpc_allocate_msgqueues(struct xpc_channel *ch)
+{
+	unsigned long irq_flags;
+	enum xpc_retval ret;
+
+
+	DBUG_ON(ch->flags & XPC_C_SETUP);
+
+	if ((ret = xpc_allocate_local_msgqueue(ch)) != xpcSuccess) {
+		return ret;
+	}
+
+	if ((ret = xpc_allocate_remote_msgqueue(ch)) != xpcSuccess) {
+		kfree(ch->local_msgqueue_base);
+		ch->local_msgqueue = NULL;
+		kfree(ch->notify_queue);
+		ch->notify_queue = NULL;
+		return ret;
+	}
+
+	spin_lock_irqsave(&ch->lock, irq_flags);
+	ch->flags |= XPC_C_SETUP;
+	spin_unlock_irqrestore(&ch->lock, irq_flags);
+
+	return xpcSuccess;
+}
+
+
+/*
+ * Process a connect message from a remote partition.
+ *
+ * Note: xpc_process_connect() is expecting to be called with the
+ * spin_lock_irqsave held and will leave it locked upon return.
+ */
+static void
+xpc_process_connect(struct xpc_channel *ch, unsigned long *irq_flags)
+{
+	enum xpc_retval ret;
+
+
+	DBUG_ON(!spin_is_locked(&ch->lock));
+
+	if (!(ch->flags & XPC_C_OPENREQUEST) ||
+				!(ch->flags & XPC_C_ROPENREQUEST)) {
+		/* nothing more to do for now */
+		return;
+	}
+	DBUG_ON(!(ch->flags & XPC_C_CONNECTING));
+
+	if (!(ch->flags & XPC_C_SETUP)) {
+		spin_unlock_irqrestore(&ch->lock, *irq_flags);
+		ret = xpc_allocate_msgqueues(ch);
+		spin_lock_irqsave(&ch->lock, *irq_flags);
+
+		if (ret != xpcSuccess) {
+			XPC_DISCONNECT_CHANNEL(ch, ret, irq_flags);
+		}
+		if (ch->flags & (XPC_C_CONNECTED | XPC_C_DISCONNECTING)) {
+			return;
+		}
+
+		DBUG_ON(!(ch->flags & XPC_C_SETUP));
+		DBUG_ON(ch->local_msgqueue == NULL);
+		DBUG_ON(ch->remote_msgqueue == NULL);
+	}
+
+	if (!(ch->flags & XPC_C_OPENREPLY)) {
+		ch->flags |= XPC_C_OPENREPLY;
+		xpc_IPI_send_openreply(ch, irq_flags);
+	}
+
+	if (!(ch->flags & XPC_C_ROPENREPLY)) {
+		return;
+	}
+
+	DBUG_ON(ch->remote_msgqueue_pa == 0);
+
+	ch->flags = (XPC_C_CONNECTED | XPC_C_SETUP);	/* clear all else */
+
+	dev_info(xpc_chan, "channel %d to partition %d connected\n",
+		ch->number, ch->partid);
+
+	spin_unlock_irqrestore(&ch->lock, *irq_flags);
+	xpc_create_kthreads(ch, 1, 0);
+	spin_lock_irqsave(&ch->lock, *irq_flags);
+}
+
+
+/*
+ * Notify those who wanted to be notified upon delivery of their message.
+ */
+static void
+xpc_notify_senders(struct xpc_channel *ch, enum xpc_retval reason, s64 put)
+{
+	struct xpc_notify *notify;
+	u8 notify_type;
+	s64 get = ch->w_remote_GP.get - 1;
+
+
+	while (++get < put && atomic_read(&ch->n_to_notify) > 0) {
+
+		notify = &ch->notify_queue[get % ch->local_nentries];
+
+		/*
+		 * See if the notify entry indicates it was associated with
+		 * a message who's sender wants to be notified. It is possible
+		 * that it is, but someone else is doing or has done the
+		 * notification.
+		 */
+		notify_type = notify->type;
+		if (notify_type == 0 ||
+				cmpxchg(&notify->type, notify_type, 0) !=
+								notify_type) {
+			continue;
+		}
+
+		DBUG_ON(notify_type != XPC_N_CALL);
+
+		atomic_dec(&ch->n_to_notify);
+
+		if (notify->func != NULL) {
+			dev_dbg(xpc_chan, "notify->func() called, notify=0x%p, "
+				"msg_number=%ld, partid=%d, channel=%d\n",
+				(void *) notify, get, ch->partid, ch->number);
+
+			notify->func(reason, ch->partid, ch->number,
+								notify->key);
+
+			dev_dbg(xpc_chan, "notify->func() returned, "
+				"notify=0x%p, msg_number=%ld, partid=%d, "
+				"channel=%d\n", (void *) notify, get,
+				ch->partid, ch->number);
+		}
+	}
+}
+
+
+/*
+ * Free up message queues and other stuff that were allocated for the specified
+ * channel.
+ *
+ * Note: ch->reason and ch->reason_line are left set for debugging purposes,
+ * they're cleared when XPC_C_DISCONNECTED is cleared.
+ */
+static void
+xpc_free_msgqueues(struct xpc_channel *ch)
+{
+	DBUG_ON(!spin_is_locked(&ch->lock));
+	DBUG_ON(atomic_read(&ch->n_to_notify) != 0);
+
+	ch->remote_msgqueue_pa = 0;
+	ch->func = NULL;
+	ch->key = NULL;
+	ch->msg_size = 0;
+	ch->local_nentries = 0;
+	ch->remote_nentries = 0;
+	ch->kthreads_assigned_limit = 0;
+	ch->kthreads_idle_limit = 0;
+
+	ch->local_GP->get = 0;
+	ch->local_GP->put = 0;
+	ch->remote_GP.get = 0;
+	ch->remote_GP.put = 0;
+	ch->w_local_GP.get = 0;
+	ch->w_local_GP.put = 0;
+	ch->w_remote_GP.get = 0;
+	ch->w_remote_GP.put = 0;
+	ch->next_msg_to_pull = 0;
+
+	if (ch->flags & XPC_C_SETUP) {
+		ch->flags &= ~XPC_C_SETUP;
+
+		dev_dbg(xpc_chan, "ch->flags=0x%x, partid=%d, channel=%d\n",
+			ch->flags, ch->partid, ch->number);
+
+		kfree(ch->local_msgqueue_base);
+		ch->local_msgqueue = NULL;
+		kfree(ch->remote_msgqueue_base);
+		ch->remote_msgqueue = NULL;
+		kfree(ch->notify_queue);
+		ch->notify_queue = NULL;
+	}
+}
+
+
+/*
+ * spin_lock_irqsave() is expected to be held on entry.
+ */
+static void
+xpc_process_disconnect(struct xpc_channel *ch, unsigned long *irq_flags)
+{
+	struct xpc_partition *part = &xpc_partitions[ch->partid];
+	u32 channel_was_connected = (ch->flags & XPC_C_WASCONNECTED);
+
+
+	DBUG_ON(!spin_is_locked(&ch->lock));
+
+	if (!(ch->flags & XPC_C_DISCONNECTING)) {
+		return;
+	}
+
+	DBUG_ON(!(ch->flags & XPC_C_CLOSEREQUEST));
+
+	/* make sure all activity has settled down first */
+
+	if (atomic_read(&ch->kthreads_assigned) > 0 ||
+				atomic_read(&ch->references) > 0) {
+		return;
+	}
+	DBUG_ON((ch->flags & XPC_C_CONNECTEDCALLOUT_MADE) &&
+			!(ch->flags & XPC_C_DISCONNECTINGCALLOUT_MADE));
+
+	if (part->act_state == XPC_P_DEACTIVATING) {
+		/* can't proceed until the other side disengages from us */
+		if (xpc_partition_engaged(1UL << ch->partid)) {
+			return;
+		}
+
+	} else {
+
+		/* as long as the other side is up do the full protocol */
+
+		if (!(ch->flags & XPC_C_RCLOSEREQUEST)) {
+			return;
+		}
+
+		if (!(ch->flags & XPC_C_CLOSEREPLY)) {
+			ch->flags |= XPC_C_CLOSEREPLY;
+			xpc_IPI_send_closereply(ch, irq_flags);
+		}
+
+		if (!(ch->flags & XPC_C_RCLOSEREPLY)) {
+			return;
+		}
+	}
+
+	/* wake those waiting for notify completion */
+	if (atomic_read(&ch->n_to_notify) > 0) {
+		/* >>> we do callout while holding ch->lock */
+		xpc_notify_senders(ch, ch->reason, ch->w_local_GP.put);
+	}
+
+	/* both sides are disconnected now */
+
+	if (ch->flags & XPC_C_DISCONNECTINGCALLOUT_MADE) {
+		spin_unlock_irqrestore(&ch->lock, *irq_flags);
+		xpc_disconnect_callout(ch, xpcDisconnected);
+		spin_lock_irqsave(&ch->lock, *irq_flags);
+	}
+
+	/* it's now safe to free the channel's message queues */
+	xpc_free_msgqueues(ch);
+
+	/* mark disconnected, clear all other flags except XPC_C_WDISCONNECT */
+	ch->flags = (XPC_C_DISCONNECTED | (ch->flags & XPC_C_WDISCONNECT));
+
+	atomic_dec(&part->nchannels_active);
+
+	if (channel_was_connected) {
+		dev_info(xpc_chan, "channel %d to partition %d disconnected, "
+			"reason=%d\n", ch->number, ch->partid, ch->reason);
+	}
+
+	if (ch->flags & XPC_C_WDISCONNECT) {
+		/* we won't lose the CPU since we're holding ch->lock */
+		complete(&ch->wdisconnect_wait);
+	} else if (ch->delayed_IPI_flags) {
+		if (part->act_state != XPC_P_DEACTIVATING) {
+			/* time to take action on any delayed IPI flags */
+			spin_lock(&part->IPI_lock);
+			XPC_SET_IPI_FLAGS(part->local_IPI_amo, ch->number,
+							ch->delayed_IPI_flags);
+			spin_unlock(&part->IPI_lock);
+		}
+		ch->delayed_IPI_flags = 0;
+	}
+}
+
+
+/*
+ * Process a change in the channel's remote connection state.
+ */
+static void
+xpc_process_openclose_IPI(struct xpc_partition *part, int ch_number,
+				u8 IPI_flags)
+{
+	unsigned long irq_flags;
+	struct xpc_openclose_args *args =
+				&part->remote_openclose_args[ch_number];
+	struct xpc_channel *ch = &part->channels[ch_number];
+	enum xpc_retval reason;
+
+
+
+	spin_lock_irqsave(&ch->lock, irq_flags);
+
+again:
+
+	if ((ch->flags & XPC_C_DISCONNECTED) &&
+					(ch->flags & XPC_C_WDISCONNECT)) {
+		/*
+		 * Delay processing IPI flags until thread waiting disconnect
+		 * has had a chance to see that the channel is disconnected.
+		 */
+		ch->delayed_IPI_flags |= IPI_flags;
+		spin_unlock_irqrestore(&ch->lock, irq_flags);
+		return;
+	}
+
+
+	if (IPI_flags & XPC_IPI_CLOSEREQUEST) {
+
+		dev_dbg(xpc_chan, "XPC_IPI_CLOSEREQUEST (reason=%d) received "
+			"from partid=%d, channel=%d\n", args->reason,
+			ch->partid, ch->number);
+
+		/*
+		 * If RCLOSEREQUEST is set, we're probably waiting for
+		 * RCLOSEREPLY. We should find it and a ROPENREQUEST packed
+		 * with this RCLOSEREQUEST in the IPI_flags.
+		 */
+
+		if (ch->flags & XPC_C_RCLOSEREQUEST) {
+			DBUG_ON(!(ch->flags & XPC_C_DISCONNECTING));
+			DBUG_ON(!(ch->flags & XPC_C_CLOSEREQUEST));
+			DBUG_ON(!(ch->flags & XPC_C_CLOSEREPLY));
+			DBUG_ON(ch->flags & XPC_C_RCLOSEREPLY);
+
+			DBUG_ON(!(IPI_flags & XPC_IPI_CLOSEREPLY));
+			IPI_flags &= ~XPC_IPI_CLOSEREPLY;
+			ch->flags |= XPC_C_RCLOSEREPLY;
+
+			/* both sides have finished disconnecting */
+			xpc_process_disconnect(ch, &irq_flags);
+			DBUG_ON(!(ch->flags & XPC_C_DISCONNECTED));
+			goto again;
+		}
+
+		if (ch->flags & XPC_C_DISCONNECTED) {
+			if (!(IPI_flags & XPC_IPI_OPENREQUEST)) {
+				if ((XPC_GET_IPI_FLAGS(part->local_IPI_amo,
+					 ch_number) & XPC_IPI_OPENREQUEST)) {
+
+					DBUG_ON(ch->delayed_IPI_flags != 0);
+					spin_lock(&part->IPI_lock);
+					XPC_SET_IPI_FLAGS(part->local_IPI_amo,
+							ch_number,
+							XPC_IPI_CLOSEREQUEST);
+					spin_unlock(&part->IPI_lock);
+				}
+				spin_unlock_irqrestore(&ch->lock, irq_flags);
+				return;
+			}
+
+			XPC_SET_REASON(ch, 0, 0);
+			ch->flags &= ~XPC_C_DISCONNECTED;
+
+			atomic_inc(&part->nchannels_active);
+			ch->flags |= (XPC_C_CONNECTING | XPC_C_ROPENREQUEST);
+		}
+
+		IPI_flags &= ~(XPC_IPI_OPENREQUEST | XPC_IPI_OPENREPLY);
+
+		/*
+		 * The meaningful CLOSEREQUEST connection state fields are:
+		 *      reason = reason connection is to be closed
+		 */
+
+		ch->flags |= XPC_C_RCLOSEREQUEST;
+
+		if (!(ch->flags & XPC_C_DISCONNECTING)) {
+			reason = args->reason;
+			if (reason <= xpcSuccess || reason > xpcUnknownReason) {
+				reason = xpcUnknownReason;
+			} else if (reason == xpcUnregistering) {
+				reason = xpcOtherUnregistering;
+			}
+
+			XPC_DISCONNECT_CHANNEL(ch, reason, &irq_flags);
+
+			DBUG_ON(IPI_flags & XPC_IPI_CLOSEREPLY);
+			spin_unlock_irqrestore(&ch->lock, irq_flags);
+			return;
+		}
+
+		xpc_process_disconnect(ch, &irq_flags);
+	}
+
+
+	if (IPI_flags & XPC_IPI_CLOSEREPLY) {
+
+		dev_dbg(xpc_chan, "XPC_IPI_CLOSEREPLY received from partid=%d,"
+			" channel=%d\n", ch->partid, ch->number);
+
+		if (ch->flags & XPC_C_DISCONNECTED) {
+			DBUG_ON(part->act_state != XPC_P_DEACTIVATING);
+			spin_unlock_irqrestore(&ch->lock, irq_flags);
+			return;
+		}
+
+		DBUG_ON(!(ch->flags & XPC_C_CLOSEREQUEST));
+
+		if (!(ch->flags & XPC_C_RCLOSEREQUEST)) {
+			if ((XPC_GET_IPI_FLAGS(part->local_IPI_amo, ch_number)
+						& XPC_IPI_CLOSEREQUEST)) {
+
+				DBUG_ON(ch->delayed_IPI_flags != 0);
+				spin_lock(&part->IPI_lock);
+				XPC_SET_IPI_FLAGS(part->local_IPI_amo,
+						ch_number, XPC_IPI_CLOSEREPLY);
+				spin_unlock(&part->IPI_lock);
+			}
+			spin_unlock_irqrestore(&ch->lock, irq_flags);
+			return;
+		}
+
+		ch->flags |= XPC_C_RCLOSEREPLY;
+
+		if (ch->flags & XPC_C_CLOSEREPLY) {
+			/* both sides have finished disconnecting */
+			xpc_process_disconnect(ch, &irq_flags);
+		}
+	}
+
+
+	if (IPI_flags & XPC_IPI_OPENREQUEST) {
+
+		dev_dbg(xpc_chan, "XPC_IPI_OPENREQUEST (msg_size=%d, "
+			"local_nentries=%d) received from partid=%d, "
+			"channel=%d\n", args->msg_size, args->local_nentries,
+			ch->partid, ch->number);
+
+		if (part->act_state == XPC_P_DEACTIVATING ||
+					(ch->flags & XPC_C_ROPENREQUEST)) {
+			spin_unlock_irqrestore(&ch->lock, irq_flags);
+			return;
+		}
+
+		if (ch->flags & (XPC_C_DISCONNECTING | XPC_C_WDISCONNECT)) {
+			ch->delayed_IPI_flags |= XPC_IPI_OPENREQUEST;
+			spin_unlock_irqrestore(&ch->lock, irq_flags);
+			return;
+		}
+		DBUG_ON(!(ch->flags & (XPC_C_DISCONNECTED |
+							XPC_C_OPENREQUEST)));
+		DBUG_ON(ch->flags & (XPC_C_ROPENREQUEST | XPC_C_ROPENREPLY |
+					XPC_C_OPENREPLY | XPC_C_CONNECTED));
+
+		/*
+		 * The meaningful OPENREQUEST connection state fields are:
+		 *      msg_size = size of channel's messages in bytes
+		 *      local_nentries = remote partition's local_nentries
+		 */
+		if (args->msg_size == 0 || args->local_nentries == 0) {
+			/* assume OPENREQUEST was delayed by mistake */
+			spin_unlock_irqrestore(&ch->lock, irq_flags);
+			return;
+		}
+
+		ch->flags |= (XPC_C_ROPENREQUEST | XPC_C_CONNECTING);
+		ch->remote_nentries = args->local_nentries;
+
+
+		if (ch->flags & XPC_C_OPENREQUEST) {
+			if (args->msg_size != ch->msg_size) {
+				XPC_DISCONNECT_CHANNEL(ch, xpcUnequalMsgSizes,
+								&irq_flags);
+				spin_unlock_irqrestore(&ch->lock, irq_flags);
+				return;
+			}
+		} else {
+			ch->msg_size = args->msg_size;
+
+			XPC_SET_REASON(ch, 0, 0);
+			ch->flags &= ~XPC_C_DISCONNECTED;
+
+			atomic_inc(&part->nchannels_active);
+		}
+
+		xpc_process_connect(ch, &irq_flags);
+	}
+
+
+	if (IPI_flags & XPC_IPI_OPENREPLY) {
+
+		dev_dbg(xpc_chan, "XPC_IPI_OPENREPLY (local_msgqueue_pa=0x%lx, "
+			"local_nentries=%d, remote_nentries=%d) received from "
+			"partid=%d, channel=%d\n", args->local_msgqueue_pa,
+			args->local_nentries, args->remote_nentries,
+			ch->partid, ch->number);
+
+		if (ch->flags & (XPC_C_DISCONNECTING | XPC_C_DISCONNECTED)) {
+			spin_unlock_irqrestore(&ch->lock, irq_flags);
+			return;
+		}
+		if (!(ch->flags & XPC_C_OPENREQUEST)) {
+			XPC_DISCONNECT_CHANNEL(ch, xpcOpenCloseError,
+								&irq_flags);
+			spin_unlock_irqrestore(&ch->lock, irq_flags);
+			return;
+		}
+
+		DBUG_ON(!(ch->flags & XPC_C_ROPENREQUEST));
+		DBUG_ON(ch->flags & XPC_C_CONNECTED);
+
+		/*
+		 * The meaningful OPENREPLY connection state fields are:
+		 *      local_msgqueue_pa = physical address of remote
+		 *			    partition's local_msgqueue
+		 *      local_nentries = remote partition's local_nentries
+		 *      remote_nentries = remote partition's remote_nentries
+		 */
+		DBUG_ON(args->local_msgqueue_pa == 0);
+		DBUG_ON(args->local_nentries == 0);
+		DBUG_ON(args->remote_nentries == 0);
+
+		ch->flags |= XPC_C_ROPENREPLY;
+		ch->remote_msgqueue_pa = args->local_msgqueue_pa;
+
+		if (args->local_nentries < ch->remote_nentries) {
+			dev_dbg(xpc_chan, "XPC_IPI_OPENREPLY: new "
+				"remote_nentries=%d, old remote_nentries=%d, "
+				"partid=%d, channel=%d\n",
+				args->local_nentries, ch->remote_nentries,
+				ch->partid, ch->number);
+
+			ch->remote_nentries = args->local_nentries;
+		}
+		if (args->remote_nentries < ch->local_nentries) {
+			dev_dbg(xpc_chan, "XPC_IPI_OPENREPLY: new "
+				"local_nentries=%d, old local_nentries=%d, "
+				"partid=%d, channel=%d\n",
+				args->remote_nentries, ch->local_nentries,
+				ch->partid, ch->number);
+
+			ch->local_nentries = args->remote_nentries;
+		}
+
+		xpc_process_connect(ch, &irq_flags);
+	}
+
+	spin_unlock_irqrestore(&ch->lock, irq_flags);
+}
+
+
+/*
+ * Attempt to establish a channel connection to a remote partition.
+ */
+static enum xpc_retval
+xpc_connect_channel(struct xpc_channel *ch)
+{
+	unsigned long irq_flags;
+	struct xpc_registration *registration = &xpc_registrations[ch->number];
+
+
+	if (mutex_trylock(&registration->mutex) == 0) {
+		return xpcRetry;
+	}
+
+	if (!XPC_CHANNEL_REGISTERED(ch->number)) {
+		mutex_unlock(&registration->mutex);
+		return xpcUnregistered;
+	}
+
+	spin_lock_irqsave(&ch->lock, irq_flags);
+
+	DBUG_ON(ch->flags & XPC_C_CONNECTED);
+	DBUG_ON(ch->flags & XPC_C_OPENREQUEST);
+
+	if (ch->flags & XPC_C_DISCONNECTING) {
+		spin_unlock_irqrestore(&ch->lock, irq_flags);
+		mutex_unlock(&registration->mutex);
+		return