ibmvmc.c

// SPDX-License-Identifier: GPL-2.0+
/*
 * IBM Power Systems Virtual Management Channel Support.
 *
 * Copyright (c) 2004, 2018 IBM Corp.
 *   Dave Engebretsen engebret@us.ibm.com
 *   Steven Royer seroyer@linux.vnet.ibm.com
 *   Adam Reznechek adreznec@linux.vnet.ibm.com
 *   Bryant G. Ly <bryantly@linux.vnet.ibm.com>
 */

#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/kthread.h>
#include <linux/major.h>
#include <linux/string.h>
#include <linux/fcntl.h>
#include <linux/slab.h>
#include <linux/poll.h>
#include <linux/init.h>
#include <linux/fs.h>
#include <linux/interrupt.h>
#include <linux/spinlock.h>
#include <linux/percpu.h>
#include <linux/delay.h>
#include <linux/uaccess.h>
#include <linux/io.h>
#include <linux/miscdevice.h>
#include <linux/sched/signal.h>

#include <asm/byteorder.h>
#include <asm/irq.h>
#include <asm/vio.h>

#include "ibmvmc.h"

#define IBMVMC_DRIVER_VERSION "1.0"

/*
 * Static global variables
 */
static DECLARE_WAIT_QUEUE_HEAD(ibmvmc_read_wait);

static const char ibmvmc_driver_name[] = "ibmvmc";

static struct ibmvmc_struct ibmvmc;
static struct ibmvmc_hmc hmcs[MAX_HMCS];
static struct crq_server_adapter ibmvmc_adapter;

static int ibmvmc_max_buf_pool_size = DEFAULT_BUF_POOL_SIZE;
static int ibmvmc_max_hmcs = DEFAULT_HMCS;
static int ibmvmc_max_mtu = DEFAULT_MTU;

static inline long h_copy_rdma(s64 length, u64 sliobn, u64 slioba,
			       u64 dliobn, u64 dlioba)
{
	long rc = 0;

	/* Ensure all writes to source memory are visible before hcall */
	dma_wmb();
	pr_debug("ibmvmc: h_copy_rdma(0x%llx, 0x%llx, 0x%llx, 0x%llx, 0x%llx\n",
		 length, sliobn, slioba, dliobn, dlioba);
	rc = plpar_hcall_norets(H_COPY_RDMA, length, sliobn, slioba,
				dliobn, dlioba);
	pr_debug("ibmvmc: h_copy_rdma rc = 0x%lx\n", rc);

	return rc;
}

static inline void h_free_crq(uint32_t unit_address)
{
	long rc = 0;

	do {
		if (H_IS_LONG_BUSY(rc))
			msleep(get_longbusy_msecs(rc));

		rc = plpar_hcall_norets(H_FREE_CRQ, unit_address);
	} while ((rc == H_BUSY) || (H_IS_LONG_BUSY(rc)));
}

/**
 * h_request_vmc: - request a hypervisor virtual management channel device
 * @vmc_index: drc index of the vmc device created
 *
 * Requests the hypervisor create a new virtual management channel device,
 * allowing this partition to send hypervisor virtualization control
 * commands.
 *
 * Return:
 *	0 - Success
 *	Non-zero - Failure
 */
static inline long h_request_vmc(u32 *vmc_index)
{
	long rc = 0;
	unsigned long retbuf[PLPAR_HCALL_BUFSIZE];

	do {
		if (H_IS_LONG_BUSY(rc))
			msleep(get_longbusy_msecs(rc));

		/* Call to request the VMC device from phyp */
		rc = plpar_hcall(H_REQUEST_VMC, retbuf);
		pr_debug("ibmvmc: %s rc = 0x%lx\n", __func__, rc);
		*vmc_index = retbuf[0];
	} while ((rc == H_BUSY) || (H_IS_LONG_BUSY(rc)));

	return rc;
}

/* routines for managing a command/response queue */
/**
 * ibmvmc_handle_event: - Interrupt handler for crq events
 * @irq:        number of irq to handle, not used
 * @dev_instance: crq_server_adapter that received interrupt
 *
 * Disables interrupts and schedules ibmvmc_task
 *
 * Always returns IRQ_HANDLED
 */
static irqreturn_t ibmvmc_handle_event(int irq, void *dev_instance)
{
	struct crq_server_adapter *adapter =
		(struct crq_server_adapter *)dev_instance;

	vio_disable_interrupts(to_vio_dev(adapter->dev));
	tasklet_schedule(&adapter->work_task);

	return IRQ_HANDLED;
}

/**
 * ibmvmc_release_crq_queue - Release CRQ Queue
 *
 * @adapter:	crq_server_adapter struct
 *
 * Return:
 *	0 - Success
 *	Non-Zero - Failure
 */
static void ibmvmc_release_crq_queue(struct crq_server_adapter *adapter)
{
	struct vio_dev *vdev = to_vio_dev(adapter->dev);
	struct crq_queue *queue = &adapter->queue;

	free_irq(vdev->irq, (void *)adapter);
	tasklet_kill(&adapter->work_task);

	if (adapter->reset_task)
		kthread_stop(adapter->reset_task);

	h_free_crq(vdev->unit_address);
	dma_unmap_single(adapter->dev,
			 queue->msg_token,
			 queue->size * sizeof(*queue->msgs), DMA_BIDIRECTIONAL);
	free_page((unsigned long)queue->msgs);
}

/**
 * ibmvmc_reset_crq_queue - Reset CRQ Queue
 *
 * @adapter:	crq_server_adapter struct
 *
 * This function calls h_free_crq and then calls H_REG_CRQ and does all the
 * bookkeeping to get us back to where we can communicate.
 *
 * Return:
 *	0 - Success
 *	Non-Zero - Failure
 */
static int ibmvmc_reset_crq_queue(struct crq_server_adapter *adapter)
{
	struct vio_dev *vdev = to_vio_dev(adapter->dev);
	struct crq_queue *queue = &adapter->queue;
	int rc = 0;

	/* Close the CRQ */
	h_free_crq(vdev->unit_address);

	/* Clean out the queue */
	memset(queue->msgs, 0x00, PAGE_SIZE);
	queue->cur = 0;

	/* And re-open it again */
	rc = plpar_hcall_norets(H_REG_CRQ,
				vdev->unit_address,
				queue->msg_token, PAGE_SIZE);
	if (rc == 2)
		/* Adapter is good, but other end is not ready */
		dev_warn(adapter->dev, "Partner adapter not ready\n");
	else if (rc != 0)
		dev_err(adapter->dev, "couldn't register crq--rc 0x%x\n", rc);

	return rc;
}

/**
 * crq_queue_next_crq: - Returns the next entry in message queue
 * @queue:      crq_queue to use
 *
 * Returns pointer to next entry in queue, or NULL if there are no new
 * entried in the CRQ.
 */
static struct ibmvmc_crq_msg *crq_queue_next_crq(struct crq_queue *queue)
{
	struct ibmvmc_crq_msg *crq;
	unsigned long flags;

	spin_lock_irqsave(&queue->lock, flags);
	crq = &queue->msgs[queue->cur];
	if (crq->valid & 0x80) {
		if (++queue->cur == queue->size)
			queue->cur = 0;

		/* Ensure the read of the valid bit occurs before reading any
		 * other bits of the CRQ entry
		 */
		dma_rmb();
	} else {
		crq = NULL;
	}

	spin_unlock_irqrestore(&queue->lock, flags);

	return crq;
}

/**
 * ibmvmc_send_crq - Send CRQ
 *
 * @adapter:	crq_server_adapter struct
 * @word1:	Word1 Data field
 * @word2:	Word2 Data field
 *
 * Return:
 *	0 - Success
 *	Non-Zero - Failure
 */
static long ibmvmc_send_crq(struct crq_server_adapter *adapter,
			    u64 word1, u64 word2)
{
	struct vio_dev *vdev = to_vio_dev(adapter->dev);
	long rc = 0;

	dev_dbg(adapter->dev, "(0x%x, 0x%016llx, 0x%016llx)\n",
		vdev->unit_address, word1, word2);

	/*
	 * Ensure the command buffer is flushed to memory before handing it
	 * over to the other side to prevent it from fetching any stale data.
	 */
	dma_wmb();
	rc = plpar_hcall_norets(H_SEND_CRQ, vdev->unit_address, word1, word2);
	dev_dbg(adapter->dev, "rc = 0x%lx\n", rc);

	return rc;
}

/**
 * alloc_dma_buffer - Create DMA Buffer
 *
 * @vdev:	vio_dev struct
 * @size:	Size field
 * @dma_handle:	DMA address field
 *
 * Allocates memory for the command queue and maps remote memory into an
 * ioba.
 *
 * Returns a pointer to the buffer
 */
static void *alloc_dma_buffer(struct vio_dev *vdev, size_t size,
			      dma_addr_t *dma_handle)
{
	/* allocate memory */
	void *buffer = kzalloc(size, GFP_ATOMIC);

	if (!buffer) {
		*dma_handle = 0;
		return NULL;
	}

	/* DMA map */
	*dma_handle = dma_map_single(&vdev->dev, buffer, size,
				     DMA_BIDIRECTIONAL);

	if (dma_mapping_error(&vdev->dev, *dma_handle)) {
		*dma_handle = 0;
		kfree_sensitive(buffer);
		return NULL;
	}

	return buffer;
}

/**
 * free_dma_buffer - Free DMA Buffer
 *
 * @vdev:	vio_dev struct
 * @size:	Size field
 * @vaddr:	Address field
 * @dma_handle:	DMA address field
 *
 * Releases memory for a command queue and unmaps mapped remote memory.
 */
static void free_dma_buffer(struct vio_dev *vdev, size_t size, void *vaddr,
			    dma_addr_t dma_handle)
{
	/* DMA unmap */
	dma_unmap_single(&vdev->dev, dma_handle, size, DMA_BIDIRECTIONAL);

	/* deallocate memory */
	kfree_sensitive(vaddr);
}

/**
 * ibmvmc_get_valid_hmc_buffer - Retrieve Valid HMC Buffer
 *
 * @hmc_index:	HMC Index Field
 *
 * Return:
 *	Pointer to ibmvmc_buffer
 */
static struct ibmvmc_buffer *ibmvmc_get_valid_hmc_buffer(u8 hmc_index)
{
	struct ibmvmc_buffer *buffer;
	struct ibmvmc_buffer *ret_buf = NULL;
	unsigned long i;

	if (hmc_index > ibmvmc.max_hmc_index)
		return NULL;

	buffer = hmcs[hmc_index].buffer;

	for (i = 0; i < ibmvmc_max_buf_pool_size; i++) {
		if (buffer[i].valid && buffer[i].free &&
		    buffer[i].owner == VMC_BUF_OWNER_ALPHA) {
			buffer[i].free = 0;
			ret_buf = &buffer[i];
			break;
		}
	}

	return ret_buf;
}

/**
 * ibmvmc_get_free_hmc_buffer - Get Free HMC Buffer
 *
 * @adapter:	crq_server_adapter struct
 * @hmc_index:	Hmc Index field
 *
 * Return:
 *	Pointer to ibmvmc_buffer
 */
static struct ibmvmc_buffer *ibmvmc_get_free_hmc_buffer(struct crq_server_adapter *adapter,
							u8 hmc_index)
{
	struct ibmvmc_buffer *buffer;
	struct ibmvmc_buffer *ret_buf = NULL;
	unsigned long i;

	if (hmc_index > ibmvmc.max_hmc_index) {
		dev_info(adapter->dev, "get_free_hmc_buffer: invalid hmc_index=0x%x\n",
			 hmc_index);
		return NULL;
	}

	buffer = hmcs[hmc_index].buffer;

	for (i = 0; i < ibmvmc_max_buf_pool_size; i++) {
		if (buffer[i].free &&
		    buffer[i].owner == VMC_BUF_OWNER_ALPHA) {
			buffer[i].free = 0;
			ret_buf = &buffer[i];
			break;
		}
	}

	return ret_buf;
}

/**
 * ibmvmc_free_hmc_buffer - Free an HMC Buffer
 *
 * @hmc:	ibmvmc_hmc struct
 * @buffer:	ibmvmc_buffer struct
 *
 */
static void ibmvmc_free_hmc_buffer(struct ibmvmc_hmc *hmc,
				   struct ibmvmc_buffer *buffer)
{
	unsigned long flags;

	spin_lock_irqsave(&hmc->lock, flags);
	buffer->free = 1;
	spin_unlock_irqrestore(&hmc->lock, flags);
}

/**
 * ibmvmc_count_hmc_buffers - Count HMC Buffers
 *
 * @hmc_index:	HMC Index field
 * @valid:	Valid number of buffers field
 * @free:	Free number of buffers field
 *
 */
static void ibmvmc_count_hmc_buffers(u8 hmc_index, unsigned int *valid,
				     unsigned int *free)
{
	struct ibmvmc_buffer *buffer;
	unsigned long i;
	unsigned long flags;

	if (hmc_index > ibmvmc.max_hmc_index)
		return;

	if (!valid || !free)
		return;

	*valid = 0; *free = 0;

	buffer = hmcs[hmc_index].buffer;
	spin_lock_irqsave(&hmcs[hmc_index].lock, flags);

	for (i = 0; i < ibmvmc_max_buf_pool_size; i++) {
		if (buffer[i].valid) {
			*valid = *valid + 1;
			if (buffer[i].free)
				*free = *free + 1;
		}
	}

	spin_unlock_irqrestore(&hmcs[hmc_index].lock, flags);
}

/**
 * ibmvmc_get_free_hmc - Get Free HMC
 *
 * Return:
 *	Pointer to an available HMC Connection
 *	Null otherwise
 */
static struct ibmvmc_hmc *ibmvmc_get_free_hmc(void)
{
	unsigned long i;
	unsigned long flags;

	/*
	 * Find an available HMC connection.
	 */
	for (i = 0; i <= ibmvmc.max_hmc_index; i++) {
		spin_lock_irqsave(&hmcs[i].lock, flags);
		if (hmcs[i].state == ibmhmc_state_free) {
			hmcs[i].index = i;
			hmcs[i].state = ibmhmc_state_initial;
			spin_unlock_irqrestore(&hmcs[i].lock, flags);
			return &hmcs[i];
		}
		spin_unlock_irqrestore(&hmcs[i].lock, flags);
	}

	return NULL;
}

/**
 * ibmvmc_return_hmc - Return an HMC Connection
 *
 * @hmc:		ibmvmc_hmc struct
 * @release_readers:	Number of readers connected to session
 *
 * This function releases the HMC connections back into the pool.
 *
 * Return:
 *	0 - Success
 *	Non-zero - Failure
 */
static int ibmvmc_return_hmc(struct ibmvmc_hmc *hmc, bool release_readers)
{
	struct ibmvmc_buffer *buffer;
	struct crq_server_adapter *adapter;
	struct vio_dev *vdev;
	unsigned long i;
	unsigned long flags;

	if (!hmc || !hmc->adapter)
		return -EIO;

	if (release_readers) {
		if (hmc->file_session) {
			struct ibmvmc_file_session *session = hmc->file_session;

			session->valid = 0;
			wake_up_interruptible(&ibmvmc_read_wait);
		}
	}

	adapter = hmc->adapter;
	vdev = to_vio_dev(adapter->dev);

	spin_lock_irqsave(&hmc->lock, flags);
	hmc->index = 0;
	hmc->state = ibmhmc_state_free;
	hmc->queue_head = 0;
	hmc->queue_tail = 0;
	buffer = hmc->buffer;
	for (i = 0; i < ibmvmc_max_buf_pool_size; i++) {
		if (buffer[i].valid) {
			free_dma_buffer(vdev,
					ibmvmc.max_mtu,
					buffer[i].real_addr_local,
					buffer[i].dma_addr_local);
			dev_dbg(adapter->dev, "Forgot buffer id 0x%lx\n", i);
		}
		memset(&buffer[i], 0, sizeof(struct ibmvmc_buffer));

		hmc->queue_outbound_msgs[i] = VMC_INVALID_BUFFER_ID;
	}

	spin_unlock_irqrestore(&hmc->lock, flags);

	return 0;
}

/**
 * ibmvmc_send_open - Interface Open
 * @buffer: Pointer to ibmvmc_buffer struct
 * @hmc: Pointer to ibmvmc_hmc struct
 *
 * This command is sent by the management partition as the result of a
 * management partition device request. It causes the hypervisor to
 * prepare a set of data buffers for the management application connection
 * indicated HMC idx. A unique HMC Idx would be used if multiple management
 * applications running concurrently were desired. Before responding to this
 * command, the hypervisor must provide the management partition with at
 * least one of these new buffers via the Add Buffer. This indicates whether
 * the messages are inbound or outbound from the hypervisor.
 *
 * Return:
 *	0 - Success
 *	Non-zero - Failure
 */
static int ibmvmc_send_open(struct ibmvmc_buffer *buffer,
			    struct ibmvmc_hmc *hmc)
{
	struct ibmvmc_crq_msg crq_msg;
	struct crq_server_adapter *adapter;
	__be64 *crq_as_u64 = (__be64 *)&crq_msg;
	int rc = 0;

	if (!hmc || !hmc->adapter)
		return -EIO;

	adapter = hmc->adapter;

	dev_dbg(adapter->dev, "send_open: 0x%lx 0x%lx 0x%lx 0x%lx 0x%lx\n",
		(unsigned long)buffer->size, (unsigned long)adapter->liobn,
		(unsigned long)buffer->dma_addr_local,
		(unsigned long)adapter->riobn,
		(unsigned long)buffer->dma_addr_remote);

	rc = h_copy_rdma(buffer->size,
			 adapter->liobn,
			 buffer->dma_addr_local,
			 adapter->riobn,
			 buffer->dma_addr_remote);
	if (rc) {
		dev_err(adapter->dev, "Error: In send_open, h_copy_rdma rc 0x%x\n",
			rc);
		return -EIO;
	}

	hmc->state = ibmhmc_state_opening;

	crq_msg.valid = 0x80;
	crq_msg.type = VMC_MSG_OPEN;
	crq_msg.status = 0;
	crq_msg.var1.rsvd = 0;
	crq_msg.hmc_session = hmc->session;
	crq_msg.hmc_index = hmc->index;
	crq_msg.var2.buffer_id = cpu_to_be16(buffer->id);
	crq_msg.rsvd = 0;
	crq_msg.var3.rsvd = 0;

	ibmvmc_send_crq(adapter, be64_to_cpu(crq_as_u64[0]),
			be64_to_cpu(crq_as_u64[1]));

	return rc;
}

/**
 * ibmvmc_send_close - Interface Close
 * @hmc: Pointer to ibmvmc_hmc struct
 *
 * This command is sent by the management partition to terminate a
 * management application to hypervisor connection. When this command is
 * sent, the management partition has quiesced all I/O operations to all
 * buffers associated with this management application connection, and
 * has freed any storage for these buffers.
 *
 * Return:
 *	0 - Success
 *	Non-zero - Failure
 */
static int ibmvmc_send_close(struct ibmvmc_hmc *hmc)
{
	struct ibmvmc_crq_msg crq_msg;
	struct crq_server_adapter *adapter;
	__be64 *crq_as_u64 = (__be64 *)&crq_msg;
	int rc = 0;

	if (!hmc || !hmc->adapter)
		return -EIO;

	adapter = hmc->adapter;

	dev_info(adapter->dev, "CRQ send: close\n");

	crq_msg.valid = 0x80;
	crq_msg.type = VMC_MSG_CLOSE;
	crq_msg.status = 0;
	crq_msg.var1.rsvd = 0;
	crq_msg.hmc_session = hmc->session;
	crq_msg.hmc_index = hmc->index;
	crq_msg.var2.rsvd = 0;
	crq_msg.rsvd = 0;
	crq_msg.var3.rsvd = 0;

	ibmvmc_send_crq(adapter, be64_to_cpu(crq_as_u64[0]),
			be64_to_cpu(crq_as_u64[1]));

	return rc;
}

/**
 * ibmvmc_send_capabilities - Send VMC Capabilities
 *
 * @adapter:	crq_server_adapter struct
 *
 * The capabilities message is an administrative message sent after the CRQ
 * initialization sequence of messages and is used to exchange VMC capabilities
 * between the management partition and the hypervisor. The management
 * partition must send this message and the hypervisor must respond with VMC
 * capabilities Response message before HMC interface message can begin. Any
 * HMC interface messages received before the exchange of capabilities has
 * complete are dropped.
 *
 * Return:
 *	0 - Success
 */
static int ibmvmc_send_capabilities(struct crq_server_adapter *adapter)
{
	struct ibmvmc_admin_crq_msg crq_msg;
	__be64 *crq_as_u64 = (__be64 *)&crq_msg;

	dev_dbg(adapter->dev, "ibmvmc: CRQ send: capabilities\n");
	crq_msg.valid = 0x80;
	crq_msg.type = VMC_MSG_CAP;
	crq_msg.status = 0;
	crq_msg.rsvd[0] = 0;
	crq_msg.rsvd[1] = 0;
	crq_msg.max_hmc = ibmvmc_max_hmcs;
	crq_msg.max_mtu = cpu_to_be32(ibmvmc_max_mtu);
	crq_msg.pool_size = cpu_to_be16(ibmvmc_max_buf_pool_size);
	crq_msg.crq_size = cpu_to_be16(adapter->queue.size);
	crq_msg.version = cpu_to_be16(IBMVMC_PROTOCOL_VERSION);

	ibmvmc_send_crq(adapter, be64_to_cpu(crq_as_u64[0]),
			be64_to_cpu(crq_as_u64[1]));

	ibmvmc.state = ibmvmc_state_capabilities;

	return 0;
}

/**
 * ibmvmc_send_add_buffer_resp - Add Buffer Response
 *
 * @adapter:	crq_server_adapter struct
 * @status:	Status field
 * @hmc_session: HMC Session field
 * @hmc_index:	HMC Index field
 * @buffer_id:	Buffer Id field
 *
 * This command is sent by the management partition to the hypervisor in
 * response to the Add Buffer message. The Status field indicates the result of
 * the command.
 *
 * Return:
 *	0 - Success
 */
static int ibmvmc_send_add_buffer_resp(struct crq_server_adapter *adapter,
				       u8 status, u8 hmc_session,
				       u8 hmc_index, u16 buffer_id)
{
	struct ibmvmc_crq_msg crq_msg;
	__be64 *crq_as_u64 = (__be64 *)&crq_msg;

	dev_dbg(adapter->dev, "CRQ send: add_buffer_resp\n");
	crq_msg.valid = 0x80;
	crq_msg.type = VMC_MSG_ADD_BUF_RESP;
	crq_msg.status = status;
	crq_msg.var1.rsvd = 0;
	crq_msg.hmc_session = hmc_session;
	crq_msg.hmc_index = hmc_index;
	crq_msg.var2.buffer_id = cpu_to_be16(buffer_id);
	crq_msg.rsvd = 0;
	crq_msg.var3.rsvd = 0;

	ibmvmc_send_crq(adapter, be64_to_cpu(crq_as_u64[0]),
			be64_to_cpu(crq_as_u64[1]));

	return 0;
}

/**
 * ibmvmc_send_rem_buffer_resp - Remove Buffer Response
 *
 * @adapter:	crq_server_adapter struct
 * @status:	Status field
 * @hmc_session: HMC Session field
 * @hmc_index:	HMC Index field
 * @buffer_id:	Buffer Id field
 *
 * This command is sent by the management partition to the hypervisor in
 * response to the Remove Buffer message. The Buffer ID field indicates
 * which buffer the management partition selected to remove. The Status
 * field indicates the result of the command.
 *
 * Return:
 *	0 - Success
 */
static int ibmvmc_send_rem_buffer_resp(struct crq_server_adapter *adapter,
				       u8 status, u8 hmc_session,
				       u8 hmc_index, u16 buffer_id)
{
	struct ibmvmc_crq_msg crq_msg;
	__be64 *crq_as_u64 = (__be64 *)&crq_msg;

	dev_dbg(adapter->dev, "CRQ send: rem_buffer_resp\n");
	crq_msg.valid = 0x80;
	crq_msg.type = VMC_MSG_REM_BUF_RESP;
	crq_msg.status = status;
	crq_msg.var1.rsvd = 0;
	crq_msg.hmc_session = hmc_session;
	crq_msg.hmc_index = hmc_index;
	crq_msg.var2.buffer_id = cpu_to_be16(buffer_id);
	crq_msg.rsvd = 0;
	crq_msg.var3.rsvd = 0;

	ibmvmc_send_crq(adapter, be64_to_cpu(crq_as_u64[0]),
			be64_to_cpu(crq_as_u64[1]));

	return 0;
}

/**
 * ibmvmc_send_msg - Signal Message
 *
 * @adapter:	crq_server_adapter struct
 * @buffer:	ibmvmc_buffer struct
 * @hmc:	ibmvmc_hmc struct
 * @msg_len:	message length field
 *
 * This command is sent between the management partition and the hypervisor
 * in order to signal the arrival of an HMC protocol message. The command
 * can be sent by both the management partition and the hypervisor. It is
 * used for all traffic between the management application and the hypervisor,
 * regardless of who initiated the communication.
 *
 * There is no response to this message.
 *
 * Return:
 *	0 - Success
 *	Non-zero - Failure
 */
static int ibmvmc_send_msg(struct crq_server_adapter *adapter,
			   struct ibmvmc_buffer *buffer,
			   struct ibmvmc_hmc *hmc, int msg_len)
{
	struct ibmvmc_crq_msg crq_msg;
	__be64 *crq_as_u64 = (__be64 *)&crq_msg;
	int rc = 0;

	dev_dbg(adapter->dev, "CRQ send: rdma to HV\n");
	rc = h_copy_rdma(msg_len,
			 adapter->liobn,
			 buffer->dma_addr_local,
			 adapter->riobn,
			 buffer->dma_addr_remote);
	if (rc) {
		dev_err(adapter->dev, "Error in send_msg, h_copy_rdma rc 0x%x\n",
			rc);
		return rc;
	}

	crq_msg.valid = 0x80;
	crq_msg.type = VMC_MSG_SIGNAL;
	crq_msg.status = 0;
	crq_msg.var1.rsvd = 0;
	crq_msg.hmc_session = hmc->session;
	crq_msg.hmc_index = hmc->index;
	crq_msg.var2.buffer_id = cpu_to_be16(buffer->id);
	crq_msg.var3.msg_len = cpu_to_be32(msg_len);
	dev_dbg(adapter->dev, "CRQ send: msg to HV 0x%llx 0x%llx\n",
		be64_to_cpu(crq_as_u64[0]), be64_to_cpu(crq_as_u64[1]));

	buffer->owner = VMC_BUF_OWNER_HV;
	ibmvmc_send_crq(adapter, be64_to_cpu(crq_as_u64[0]),
			be64_to_cpu(crq_as_u64[1]));

	return rc;
}

/**
 * ibmvmc_open - Open Session
 *
 * @inode:	inode struct
 * @file:	file struct
 *
 * Return:
 *	0 - Success
 *	Non-zero - Failure
 */
static int ibmvmc_open(struct inode *inode, struct file *file)
{
	struct ibmvmc_file_session *session;

	pr_debug("%s: inode = 0x%lx, file = 0x%lx, state = 0x%x\n", __func__,
		 (unsigned long)inode, (unsigned long)file,
		 ibmvmc.state);

	session = kzalloc(sizeof(*session), GFP_KERNEL);
	if (!session)
		return -ENOMEM;

	session->file = file;
	file->private_data = session;

	return 0;
}

/**
 * ibmvmc_close - Close Session
 *
 * @inode:	inode struct
 * @file:	file struct
 *
 * Return:
 *	0 - Success
 *	Non-zero - Failure
 */
static int ibmvmc_close(struct inode *inode, struct file *file)
{
	struct ibmvmc_file_session *session;
	struct ibmvmc_hmc *hmc;
	int rc = 0;
	unsigned long flags;

	pr_debug("%s: file = 0x%lx, state = 0x%x\n", __func__,
		 (unsigned long)file, ibmvmc.state);

	session = file->private_data;
	if (!session)
		return -EIO;

	hmc = session->hmc;
	if (hmc) {
		if (!hmc->adapter)
			return -EIO;

		if (ibmvmc.state == ibmvmc_state_failed) {
			dev_warn(hmc->adapter->dev, "close: state_failed\n");
			return -EIO;
		}

		spin_lock_irqsave(&hmc->lock, flags);
		if (hmc->state >= ibmhmc_state_opening) {
			rc = ibmvmc_send_close(hmc);
			if (rc)
				dev_warn(hmc->adapter->dev, "close: send_close failed.\n");
		}
		spin_unlock_irqrestore(&hmc->lock, flags);
	}

	kfree_sensitive(session);

	return rc;
}

/**
 * ibmvmc_read - Read
 *
 * @file:	file struct
 * @buf:	Character buffer
 * @nbytes:	Size in bytes
 * @ppos:	Offset
 *
 * Return:
 *	0 - Success
 *	Non-zero - Failure
 */
static ssize_t ibmvmc_read(struct file *file, char *buf, size_t nbytes,
			   loff_t *ppos)
{
	struct ibmvmc_file_session *session;
	struct ibmvmc_hmc *hmc;
	struct crq_server_adapter *adapter;
	struct ibmvmc_buffer *buffer;
	ssize_t n;
	ssize_t retval = 0;
	unsigned long flags;
	DEFINE_WAIT(wait);

	pr_debug("ibmvmc: read: file = 0x%lx, buf = 0x%lx, nbytes = 0x%lx\n",
		 (unsigned long)file, (unsigned long)buf,
		 (unsigned long)nbytes);

	if (nbytes == 0)
		return 0;

	if (nbytes > ibmvmc.max_mtu) {
		pr_warn("ibmvmc: read: nbytes invalid 0x%x\n",
			(unsigned int)nbytes);
		return -EINVAL;
	}

	session = file->private_data;
	if (!session) {
		pr_warn("ibmvmc: read: no session\n");
		return -EIO;
	}

	hmc = session->hmc;
	if (!hmc) {
		pr_warn("ibmvmc: read: no hmc\n");
		return -EIO;
	}

	adapter = hmc->adapter;
	if (!adapter) {
		pr_warn("ibmvmc: read: no adapter\n");
		return -EIO;
	}

	do {
		prepare_to_wait(&ibmvmc_read_wait, &wait, TASK_INTERRUPTIBLE);

		spin_lock_irqsave(&hmc->lock, flags);
		if (hmc->queue_tail != hmc->queue_head)
			/* Data is available */
			break;

		spin_unlock_irqrestore(&hmc->lock, flags);

		if (!session->valid) {
			retval = -EBADFD;
			goto out;
		}
		if (file->f_flags & O_NONBLOCK) {
			retval = -EAGAIN;
			goto out;
		}

		schedule();

		if (signal_pending(current)) {
			retval = -ERESTARTSYS;
			goto out;
		}
	} while (1);

	buffer = &(hmc->buffer[hmc->queue_outbound_msgs[hmc->queue_tail]]);
	hmc->queue_tail++;
	if (hmc->queue_tail == ibmvmc_max_buf_pool_size)
		hmc->queue_tail = 0;
	spin_unlock_irqrestore(&hmc->lock, flags);

	nbytes = min_t(size_t, nbytes, buffer->msg_len);
	n = copy_to_user((void *)buf, buffer->real_addr_local, nbytes);
	dev_dbg(adapter->dev, "read: copy to user nbytes = 0x%lx.\n", nbytes);
	ibmvmc_free_hmc_buffer(hmc, buffer);
	retval = nbytes;

	if (n) {
		dev_warn(adapter->dev, "read: copy to user failed.\n");
		retval = -EFAULT;
	}

 out:
	finish_wait(&ibmvmc_read_wait, &wait);
	dev_dbg(adapter->dev, "read: out %ld\n", retval);
	return retval;
}

/**
 * ibmvmc_poll - Poll
 *
 * @file:	file struct
 * @wait:	Poll Table
 *
 * Return:
 *	poll.h return values
 */
static unsigned int ibmvmc_poll(struct file *file, poll_table *wait)
{
	struct ibmvmc_file_session *session;
	struct ibmvmc_hmc *hmc;
	unsigned int mask = 0;

	session = file->private_data;
	if (!session)
		return 0;

	hmc = session->hmc;
	if (!hmc)
		return 0;

	poll_wait(file, &ibmvmc_read_wait, wait);

	if (hmc->queue_head != hmc->queue_tail)
		mask |= POLLIN | POLLRDNORM;

	return mask;
}

/**
 * ibmvmc_write - Write
 *
 * @file:	file struct
 * @buffer:	Character buffer
 * @count:	Count field
 * @ppos:	Offset
 *
 * Return:
 *	0 - Success
 *	Non-zero - Failure
 */
static ssize_t ibmvmc_write(struct file *file, const char *buffer,
			    size_t count, loff_t *ppos)
{
	struct inode *inode;
	struct ibmvmc_buffer *vmc_buffer;
	struct ibmvmc_file_session *session;
	struct crq_server_adapter *adapter;
	struct ibmvmc_hmc *hmc;
	unsigned char *buf;
	unsigned long flags;
	size_t bytes;
	const char *p = buffer;
	size_t c = count;
	int ret = 0;

	session = file->private_data;
	if (!session)
		return -EIO;

	hmc = session->hmc;
	if (!hmc)
		return -EIO;

	spin_lock_irqsave(&hmc->lock, flags);
	if (hmc->state == ibmhmc_state_free) {
		/* HMC connection is not valid (possibly was reset under us). */
		ret = -EIO;
		goto out;
	}

	adapter = hmc->adapter;
	if (!adapter) {
		ret = -EIO;
		goto out;
	}

	if (count > ibmvmc.max_mtu) {
		dev_warn(adapter->dev, "invalid buffer size 0x%lx\n",
			 (unsigned long)count);
		ret = -EIO;
		goto out;
	}

	/* Waiting for the open resp message to the ioctl(1) - retry */
	if (hmc->state == ibmhmc_state_opening) {
		ret = -EBUSY;
		goto out;
	}

	/* Make sure the ioctl() was called & the open msg sent, and that
	 * the HMC connection has not failed.
	 */
	if (hmc->state != ibmhmc_state_ready) {
		ret = -EIO;
		goto out;
	}

	vmc_buffer = ibmvmc_get_valid_hmc_buffer(hmc->index);
	if (!vmc_buffer) {
		/* No buffer available for the msg send, or we have not yet
		 * completed the open/open_resp sequence.  Retry until this is
		 * complete.
		 */
		ret = -EBUSY;
		goto out;
	}
	if (!vmc_buffer->real_addr_local) {
		dev_err(adapter->dev, "no buffer storage assigned\n");
		ret = -EIO;
		goto out;
	}
	buf = vmc_buffer->real_addr_local;

	while (c > 0) {
		bytes = min_t(size_t, c, vmc_buffer->size);

		bytes -= copy_from_user(buf, p, bytes);
		if (!bytes) {
			ret = -EFAULT;
			goto out;
		}
		c -= bytes;
		p += bytes;
	}
	if (p == buffer)
		goto out;

	inode = file_inode(file);
	inode->i_mtime = current_time(inode);
	mark_inode_dirty(inode);

	dev_dbg(adapter->dev, "write: file = 0x%lx, count = 0x%lx\n",
		(unsigned long)file, (unsigned long)count);

	ibmvmc_send_msg(adapter, vmc_buffer, hmc, count);
	ret = p - buffer;
 out:
	spin_unlock_irqrestore(&hmc->lock, flags);
	return (ssize_t)(ret);
}

/**
 * ibmvmc_setup_hmc - Setup the HMC
 *
 * @session:	ibmvmc_file_session struct
 *
 * Return:
 *	0 - Success
 *	Non-zero - Failure
 */
static long ibmvmc_setup_hmc(struct ibmvmc_file_session *session)
{
	struct ibmvmc_hmc *hmc;
	unsigned int valid, free, index;

	if (ibmvmc.state == ibmvmc_state_failed) {
		pr_warn("ibmvmc: Reserve HMC: state_failed\n");
		return -EIO;
	}

	if (ibmvmc.state < ibmvmc_state_ready) {
		pr_warn("ibmvmc: Reserve HMC: not state_ready\n");
		return -EAGAIN;
	}

	/* Device is busy until capabilities have been exchanged and we
	 * have a generic buffer for each possible HMC connection.
	 */
	for (index = 0; index <= ibmvmc.max_hmc_index; index++) {
		valid = 0;
		ibmvmc_count_hmc_buffers(index, &valid, &free);
		if (valid == 0) {
			pr_warn("ibmvmc: buffers not ready for index %d\n",
				index);
			return -ENOBUFS;
		}
	}

	/* Get an hmc object, and transition to ibmhmc_state_initial */
	hmc = ibmvmc_get_free_hmc();
	if (!hmc) {
		pr_warn("%s: free hmc not found\n", __func__);
		return -EBUSY;
	}

	hmc->session = hmc->session + 1;
	if (hmc->session == 0xff)
		hmc->session = 1;

	session->hmc = hmc;
	hmc->adapter = &ibmvmc_adapter;
	hmc->file_session = session;
	session->valid = 1;

	return 0;
}

/**
 * ibmvmc_ioctl_sethmcid - IOCTL Set HMC ID
 *
 * @session:	ibmvmc_file_session struct
 * @new_hmc_id:	HMC id field
 *
 * IOCTL command to setup the hmc id
 *
 * Return:
 *	0 - Success
 *	Non-zero - Failure
 */
static long ibmvmc_ioctl_sethmcid(struct ibmvmc_file_session *session,
				  unsigned char __user *new_hmc_id)
{
	struct ibmvmc_hmc *hmc;
	struct ibmvmc_buffer *buffer;
	size_t bytes;
	char print_buffer[HMC_ID_LEN + 1];
	unsigned long flags;
	long rc = 0;

	/* Reserve HMC session */
	hmc = session->hmc;
	if (!hmc) {
		rc = ibmvmc_setup_hmc(session);
		if (rc)
			return rc;

		hmc = session->hmc;
		if (!hmc) {
			pr_err("ibmvmc: setup_hmc success but no hmc\n");
			return -EIO;
		}
	}

	if (hmc->state != ibmhmc_state_initial) {
		pr_warn("ibmvmc: sethmcid: invalid state to send open 0x%x\n",
			hmc->state);
		return -EIO;
	}

	bytes = copy_from_user(hmc->hmc_id, new_hmc_id, HMC_ID_LEN);
	if (bytes)
		return -EFAULT;

	/* Send Open Session command */
	spin_lock_irqsave(&hmc->lock, flags);
	buffer = ibmvmc_get_valid_hmc_buffer(hmc->index);
	spin_unlock_irqrestore(&hmc->lock, flags);

	if (!buffer || !buffer->real_addr_local) {
		pr_warn("ibmvmc: sethmcid: no buffer available\n");
		return -EIO;
	}

	/* Make sure buffer is NULL terminated before trying to print it */
	memset(print_buffer, 0, HMC_ID_LEN + 1);
	strncpy(print_buffer, hmc->hmc_id, HMC_ID_LEN);
	pr_info("ibmvmc: sethmcid: Set HMC ID: \"%s\"\n", print_buffer);

	memcpy(buffer->real_addr_local, hmc->hmc_id, HMC_ID_LEN);
	/* RDMA over ID, send open msg, change state to ibmhmc_state_opening */
	rc = ibmvmc_send_open(buffer, hmc);

	return rc;
}

/**
 * ibmvmc_ioctl_query - IOCTL Query
 *
 * @session:	ibmvmc_file_session struct
 * @ret_struct:	ibmvmc_query_struct
 *
 * Return:
 *	0 - Success
 *	Non-zero - Failure
 */
static long ibmvmc_ioctl_query(struct ibmvmc_file_session *session,
			       struct ibmvmc_query_struct __user *ret_struct)
{
	struct ibmvmc_query_struct query_struct;
	size_t bytes;

	memset(&query_struct, 0, sizeof(query_struct));
	query_struct.have_vmc = (ibmvmc.state > ibmvmc_state_initial);
	query_struct.state = ibmvmc.state;
	query_struct.vmc_drc_index = ibmvmc.vmc_drc_index;

	bytes = copy_to_user(ret_struct, &query_struct,
			     sizeof(query_struct));
	if (bytes)
		return -EFAULT;

	return 0;
}

/**
 * ibmvmc_ioctl_requestvmc - IOCTL Request VMC
 *
 * @session:	ibmvmc_file_session struct
 * @ret_vmc_index:	VMC Index
 *
 * Return:
 *	0 - Success
 *	Non-zero - Failure
 */
static long ibmvmc_ioctl_requestvmc(struct ibmvmc_file_session *session,
				    u32 __user *ret_vmc_index)
{
	/* TODO: (adreznec) Add locking to control multiple process access */
	size_t bytes;
	long rc;
	u32 vmc_drc_index;

	/* Call to request the VMC device from phyp*/
	rc = h_request_vmc(&vmc_drc_index);
	pr_debug("ibmvmc: requestvmc: H_REQUEST_VMC rc = 0x%lx\n", rc);

	if (rc == H_SUCCESS) {
		rc = 0;
	} else if (rc == H_FUNCTION) {
		pr_err("ibmvmc: requestvmc: h_request_vmc not supported\n");
		return -EPERM;
	} else if (rc == H_AUTHORITY) {
		pr_err("ibmvmc: requestvmc: hypervisor denied vmc request\n");
		return -EPERM;
	} else if (rc == H_HARDWARE) {
		pr_err("ibmvmc: requestvmc: hypervisor hardware fault\n");
		return -EIO;
	} else if (rc == H_RESOURCE) {
		pr_err("ibmvmc: requestvmc: vmc resource unavailable\n");
		return -ENODEV;
	} else if (rc == H_NOT_AVAILABLE) {
		pr_err("ibmvmc: requestvmc: system cannot be vmc managed\n");
		return -EPERM;
	} else if (rc == H_PARAMETER) {
		pr_err("ibmvmc: requestvmc: invalid parameter\n");
		return -EINVAL;
	}

	/* Success, set the vmc index in global struct */
	ibmvmc.vmc_drc_index = vmc_drc_index;

	bytes = copy_to_user(ret_vmc_index, &vmc_drc_index,
			     sizeof(*ret_vmc_index));
	if (bytes) {
		pr_warn("ibmvmc: requestvmc: copy to user failed.\n");
		return -EFAULT;
	}
	return rc;
}

/**
 * ibmvmc_ioctl - IOCTL
 *
 * @file:	file information
 * @cmd:	cmd field
 * @arg:	Argument field
 *
 * Return:
 *	0 - Success
 *	Non-zero - Failure
 */
static long ibmvmc_ioctl(struct file *file,
			 unsigned int cmd, unsigned long arg)
{
	struct ibmvmc_file_session *session = file->private_data;

	pr_debug("ibmvmc: ioctl file=0x%lx, cmd=0x%x, arg=0x%lx, ses=0x%lx\n",
		 (unsigned long)file, cmd, arg,
		 (unsigned long)session);

	if (!session) {
		pr_warn("ibmvmc: ioctl: no session\n");
		return -EIO;
	}

	switch (cmd) {
	case VMC_IOCTL_SETHMCID:
		return ibmvmc_ioctl_sethmcid(session,
			(unsigned char __user *)arg);
	case VMC_IOCTL_QUERY:
		return ibmvmc_ioctl_query(session,
			(struct ibmvmc_query_struct __user *)arg);
	case VMC_IOCTL_REQUESTVMC:
		return ibmvmc_ioctl_requestvmc(session,
			(unsigned int __user *)arg);
	default:
		pr_warn("ibmvmc: unknown ioctl 0x%x\n", cmd);
		return -EINVAL;
	}
}

static const struct file_operations ibmvmc_fops = {
	.owner		= THIS_MODULE,
	.read		= ibmvmc_read,
	.write		= ibmvmc_write,
	.poll		= ibmvmc_poll,
	.unlocked_ioctl	= ibmvmc_ioctl,
	.open           = ibmvmc_open,
	.release        = ibmvmc_close,
};

/**
 * ibmvmc_add_buffer - Add Buffer
 *
 * @adapter: crq_server_adapter struct
 * @crq:	ibmvmc_crq_msg struct
 *
 * This message transfers a buffer from hypervisor ownership to management
 * partition ownership. The LIOBA is obtained from the virtual TCE table
 * associated with the hypervisor side of the VMC device, and points to a
 * buffer of size MTU (as established in the capabilities exchange).
 *
 * Typical flow for ading buffers:
 * 1. A new management application connection is opened by the management
 *	partition.
 * 2. The hypervisor assigns new buffers for the traffic associated with
 *	that connection.
 * 3. The hypervisor sends VMC Add Buffer messages to the management
 *	partition, informing it of the new buffers.
 * 4. The hypervisor sends an HMC protocol message (to the management
 *	application) notifying it of the new buffers. This informs the
 *	application that it has buffers available for sending HMC
 *	commands.
 *
 * Return:
 *	0 - Success
 *	Non-zero - Failure
 */
static int ibmvmc_add_buffer(struct crq_server_adapter *adapter,
			     struct ibmvmc_crq_msg *crq)
{
	struct ibmvmc_buffer *buffer;
	u8 hmc_index;
	u8 hmc_session;
	u16 buffer_id;
	unsigned long flags;
	int rc = 0;

	if (!crq)
		return -1;

	hmc_session = crq->hmc_session;
	hmc_index = crq->hmc_index;
	buffer_id = be16_to_cpu(crq->var2.buffer_id);

	if (hmc_index > ibmvmc.max_hmc_index) {
		dev_err(adapter->dev, "add_buffer: invalid hmc_index = 0x%x\n",
			hmc_index);
		ibmvmc_send_add_buffer_resp(adapter, VMC_MSG_INVALID_HMC_INDEX,
					    hmc_session, hmc_index, buffer_id);
		return -1;
	}

	if (buffer_id >= ibmvmc.max_buffer_pool_size) {
		dev_err(adapter->dev, "add_buffer: invalid buffer_id = 0x%x\n",
			buffer_id);
		ibmvmc_send_add_buffer_resp(adapter, VMC_MSG_INVALID_BUFFER_ID,
					    hmc_session, hmc_index, buffer_id);
		return -1;
	}

	spin_lock_irqsave(&hmcs[hmc_index].lock, flags);
	buffer = &hmcs[hmc_index].buffer[buffer_id];

	if (buffer->real_addr_local || buffer->dma_addr_local) {
		dev_warn(adapter->dev, "add_buffer: already allocated id = 0x%lx\n",
			 (unsigned long)buffer_id);
		spin_unlock_irqrestore(&hmcs[hmc_index].lock, flags);
		ibmvmc_send_add_buffer_resp(adapter, VMC_MSG_INVALID_BUFFER_ID,
					    hmc_session, hmc_index, buffer_id);
		return -1;
	}

	buffer->real_addr_local = alloc_dma_buffer(to_vio_dev(adapter->dev),
						   ibmvmc.max_mtu,
						   &buffer->dma_addr_local);

	if (!buffer->real_addr_local) {
		dev_err(adapter->dev, "add_buffer: alloc_dma_buffer failed.\n");
		spin_unlock_irqrestore(&hmcs[hmc_index].lock, flags);
		ibmvmc_send_add_buffer_resp(adapter, VMC_MSG_INTERFACE_FAILURE,
					    hmc_session, hmc_index, buffer_id);
		return -1;
	}

	buffer->dma_addr_remote = be32_to_cpu(crq->var3.lioba);
	buffer->size = ibmvmc.max_mtu;
	buffer->owner = crq->var1.owner;
	buffer->free = 1;
	/* Must ensure valid==1 is observable only after all other fields are */
	dma_wmb();
	buffer->valid = 1;
	buffer->id = buffer_id;

	dev_dbg(adapter->dev, "add_buffer: successfully added a buffer:\n");
	dev_dbg(adapter->dev, "   index: %d, session: %d, buffer: 0x%x, owner: %d\n",
		hmc_index, hmc_session, buffer_id, buffer->owner);
	dev_dbg(adapter->dev, "   local: 0x%x, remote: 0x%x\n",
		(u32)buffer->dma_addr_local,
		(u32)buffer->dma_addr_remote);
	spin_unlock_irqrestore(&hmcs[hmc_index].lock, flags);

	ibmvmc_send_add_buffer_resp(adapter, VMC_MSG_SUCCESS, hmc_session,
				    hmc_index, buffer_id);

	return rc;
}

/**
 * ibmvmc_rem_buffer - Remove Buffer
 *
 * @adapter: crq_server_adapter struct
 * @crq:	ibmvmc_crq_msg struct
 *
 * This message requests an HMC buffer to be transferred from management
 * partition ownership to hypervisor ownership. The management partition may
 * not be able to satisfy the request at a particular point in time if all its
 * buffers are in use. The management partition requires a depth of at least
 * one inbound buffer to allow management application commands to flow to the
 * hypervisor. It is, therefore, an interface error for the hypervisor to
 * attempt to remove the management partition's last buffer.
 *
 * The hypervisor is expected to manage buffer usage with the management
 * application directly and inform the management partition when buffers may be
 * removed. The typical flow for removing buffers:
 *
 * 1. The management application no longer needs a communication path to a
 *	particular hypervisor function. That function is closed.
 * 2. The hypervisor and the management application quiesce all traffic to that
 *	function. The hypervisor requests a reduction in buffer pool size.
 * 3. The management application acknowledges the reduction in buffer pool size.
 * 4. The hypervisor sends a Remove Buffer message to the management partition,
 *	informing it of the reduction in buffers.
 * 5. The management partition verifies it can remove the buffer. This is
 *	possible if buffers have been quiesced.
 *
 * Return:
 *	0 - Success
 *	Non-zero - Failure
 */
/*
 * The hypervisor requested that we pick an unused buffer, and return it.
 * Before sending the buffer back, we free any storage associated with the
 * buffer.
 */
static int ibmvmc_rem_buffer(struct crq_server_adapter *adapter,
			     struct ibmvmc_crq_msg *crq)
{
	struct ibmvmc_buffer *buffer;
	u8 hmc_index;
	u8 hmc_session;
	u16 buffer_id = 0;
	unsigned long flags;
	int rc = 0;

	if (!crq)
		return -1;

	hmc_session = crq->hmc_session;
	hmc_index = crq->hmc_index;

	if (hmc_index > ibmvmc.max_hmc_index) {
		dev_warn(adapter->dev, "rem_buffer: invalid hmc_index = 0x%x\n",
			 hmc_index);
		ibmvmc_send_rem_buffer_resp(adapter, VMC_MSG_INVALID_HMC_INDEX,
					    hmc_session, hmc_index, buffer_id);
		return -1;
	}

	spin_lock_irqsave(&hmcs[hmc_index].lock, flags);
	buffer = ibmvmc_get_free_hmc_buffer(adapter, hmc_index);
	if (!buffer) {
		dev_info(adapter->dev, "rem_buffer: no buffer to remove\n");
		spin_unlock_irqrestore(&hmcs[hmc_index].lock, flags);
		ibmvmc_send_rem_buffer_resp(adapter, VMC_MSG_NO_BUFFER,
					    hmc_session, hmc_index,
					    VMC_INVALID_BUFFER_ID);
		return -1;
	}

	buffer_id = buffer->id;

	if (buffer->valid)
		free_dma_buffer(to_vio_dev(adapter->dev),
				ibmvmc.max_mtu,
				buffer->real_addr_local,
				buffer->dma_addr_local);

	memset(buffer, 0, sizeof(struct ibmvmc_buffer));
	spin_unlock_irqrestore(&hmcs[hmc_index].lock, flags);

	dev_dbg(adapter->dev, "rem_buffer: removed buffer 0x%x.\n", buffer_id);
	ibmvmc_send_rem_buffer_resp(adapter, VMC_MSG_SUCCESS, hmc_session,
				    hmc_index, buffer_id);

	return rc;
}

static int ibmvmc_recv_msg(struct crq_server_adapter *adapter,
			   struct ibmvmc_crq_msg *crq)
{
	struct ibmvmc_buffer *buffer;
	struct ibmvmc_hmc *hmc;
	unsigned long msg_len;
	u8 hmc_index;
	u8 hmc_session;
	u16 buffer_id;
	unsigned long flags;
	int rc = 0;

	if (!crq)
		return -1;

	/* Hypervisor writes CRQs directly into our memory in big endian */
	dev_dbg(adapter->dev, "Recv_msg: msg from HV 0x%016llx 0x%016llx\n",
		be64_to_cpu(*((unsigned long *)crq)),
		be64_to_cpu(*(((unsigned long *)crq) + 1)));

	hmc_session = crq->hmc_session;
	hmc_index = crq->hmc_index;
	buffer_id = be16_to_cpu(crq->var2.buffer_id);
	msg_len = be32_to_cpu(crq->var3.msg_len);

	if (hmc_index > ibmvmc.max_hmc_index) {
		dev_err(adapter->dev, "Recv_msg: invalid hmc_index = 0x%x\n",
			hmc_index);
		ibmvmc_send_add_buffer_resp(adapter, VMC_MSG_INVALID_HMC_INDEX,
					    hmc_session, hmc_index, buffer_id);
		return -1;
	}

	if (buffer_id >= ibmvmc.max_buffer_pool_size) {
		dev_err(adapter->dev, "Recv_msg: invalid buffer_id = 0x%x\n",
			buffer_id);
		ibmvmc_send_add_buffer_resp(adapter, VMC_MSG_INVALID_BUFFER_ID,
					    hmc_session, hmc_index, buffer_id);
		return -1;
	}

	hmc = &hmcs[hmc_index];
	spin_lock_irqsave(&hmc->lock, flags);

	if (hmc->state == ibmhmc_state_free) {
		dev_err(adapter->dev, "Recv_msg: invalid hmc state = 0x%x\n",
			hmc->state);
		/* HMC connection is not valid (possibly was reset under us). */
		spin_unlock_irqrestore(&hmc->lock, flags);
		return -1;
	}

	buffer = &hmc->buffer[buffer_id];

	if (buffer->valid == 0 || buffer->owner == VMC_BUF_OWNER_ALPHA) {
		dev_err(adapter->dev, "Recv_msg: not valid, or not HV.  0x%x 0x%x\n",
			buffer->valid, buffer->owner);
		spin_unlock_irqrestore(&hmc->lock, flags);
		return -1;
	}

	/* RDMA the data into the partition. */
	rc = h_copy_rdma(msg_len,
			 adapter->riobn,
			 buffer->dma_addr_remote,
			 adapter->liobn,
			 buffer->dma_addr_local);

	dev_dbg(adapter->dev, "Recv_msg: msg_len = 0x%x, buffer_id = 0x%x, queue_head = 0x%x, hmc_idx = 0x%x\n",
		(unsigned int)msg_len, (unsigned int)buffer_id,
		(unsigned int)hmc->queue_head, (unsigned int)hmc_index);
	buffer->msg_len = msg_len;
	buffer->free = 0;
	buffer->owner = VMC_BUF_OWNER_ALPHA;

	if (rc) {
		dev_err(adapter->dev, "Failure in recv_msg: h_copy_rdma = 0x%x\n",
			rc);
		spin_unlock_irqrestore(&hmc->lock, flags);
		return -1;
	}

	/* Must be locked because read operates on the same data */
	hmc->queue_outbound_msgs[hmc->queue_head] = buffer_id;
	hmc->queue_head++;
	if (hmc->queue_head == ibmvmc_max_buf_pool_size)
		hmc->queue_head = 0;

	if (hmc->queue_head == hmc->queue_tail)
		dev_err(adapter->dev, "outbound buffer queue wrapped.\n");

	spin_unlock_irqrestore(&hmc->lock, flags);

	wake_up_interruptible(&ibmvmc_read_wait);

	return 0;
}

/**
 * ibmvmc_process_capabilities - Process Capabilities
 *
 * @adapter:	crq_server_adapter struct
 * @crqp:	ibmvmc_crq_msg struct
 *
 */
static void ibmvmc_process_capabilities(struct crq_server_adapter *adapter,
					struct ibmvmc_crq_msg *crqp)
{
	struct ibmvmc_admin_crq_msg *crq = (struct ibmvmc_admin_crq_msg *)crqp;

	if ((be16_to_cpu(crq->version) >> 8) !=
			(IBMVMC_PROTOCOL_VERSION >> 8)) {
		dev_err(adapter->dev, "init failed, incompatible versions 0x%x 0x%x\n",
			be16_to_cpu(crq->version),
			IBMVMC_PROTOCOL_VERSION);
		ibmvmc.state = ibmvmc_state_failed;
		return;
	}

	ibmvmc.max_mtu = min_t(u32, ibmvmc_max_mtu, be32_to_cpu(crq->max_mtu));
	ibmvmc.max_buffer_pool_size = min_t(u16, ibmvmc_max_buf_pool_size,
					    be16_to_cpu(crq->pool_size));
	ibmvmc.max_hmc_index = min_t(u8, ibmvmc_max_hmcs, crq->max_hmc) - 1;
	ibmvmc.state = ibmvmc_state_ready;

	dev_info(adapter->dev, "Capabilities: mtu=0x%x, pool_size=0x%x, max_hmc=0x%x\n",
		 ibmvmc.max_mtu, ibmvmc.max_buffer_pool_size,
		 ibmvmc.max_hmc_index);
}

/**
 * ibmvmc_validate_hmc_session - Validate HMC Session
 *
 * @adapter:	crq_server_adapter struct
 * @crq:	ibmvmc_crq_msg struct
 *
 * Return:
 *	0 - Success
 *	Non-zero - Failure
 */
static int ibmvmc_validate_hmc_session(struct crq_server_adapter *adapter,
				       struct ibmvmc_crq_msg *crq)
{
	unsigned char hmc_index;

	hmc_index = crq->hmc_index;

	if (crq->hmc_session == 0)
		return 0;

	if (hmc_index > ibmvmc.max_hmc_index)
		return -1;

	if (hmcs[hmc_index].session != crq->hmc_session) {
		dev_warn(adapter->dev, "Drop, bad session: expected 0x%x, recv 0x%x\n",
			 hmcs[hmc_index].session, crq->hmc_session);
		return -1;
	}

	return 0;
}

/**
 * ibmvmc_reset - Reset
 *
 * @adapter:	crq_server_adapter struct
 * @xport_event:	export_event field
 *
 * Closes all HMC sessions and conditionally schedules a CRQ reset.
 * @xport_event: If true, the partner closed their CRQ; we don't need to reset.
 *               If false, we need to schedule a CRQ reset.
 */
static void ibmvmc_reset(struct crq_server_adapter *adapter, bool xport_event)
{
	int i;

	if (ibmvmc.state != ibmvmc_state_sched_reset) {
		dev_info(adapter->dev, "*** Reset to initial state.\n");
		for (i = 0; i < ibmvmc_max_hmcs; i++)
			ibmvmc_return_hmc(&hmcs[i], xport_event);

		if (xport_event) {
			/* CRQ was closed by the partner.  We don't need to do
			 * anything except set ourself to the correct state to
			 * handle init msgs.
			 */
			ibmvmc.state = ibmvmc_state_crqinit;
		} else {
			/* The partner did not close their CRQ - instead, we're
			 * closing the CRQ on our end. Need to schedule this
			 * for process context, because CRQ reset may require a
			 * sleep.
			 *
			 * Setting ibmvmc.state here immediately prevents
			 * ibmvmc_open from completing until the reset
			 * completes in process context.
			 */
			ibmvmc.state = ibmvmc_state_sched_reset;
			dev_dbg(adapter->dev, "Device reset scheduled");
			wake_up_interruptible(&adapter->reset_wait_queue);
		}
	}
}

/**
 * ibmvmc_reset_task - Reset Task
 *
 * @data:	Data field
 *
 * Performs a CRQ reset of the VMC device in process context.
 * NOTE: This function should not be called directly, use ibmvmc_reset.
 */
static int ibmvmc_reset_task(void *data)
{
	struct crq_server_adapter *adapter = data;
	int rc;

	set_user_nice(current, -20);

	while (!kthread_should_stop()) {
		wait_event_interruptible(adapter->reset_wait_queue,
			(ibmvmc.state == ibmvmc_state_sched_reset) ||
			kthread_should_stop());

		if (kthread_should_stop())
			break;

		dev_dbg(adapter->dev, "CRQ resetting in process context");
		tasklet_disable(&adapter->work_task);

		rc = ibmvmc_reset_crq_queue(adapter);

		if (rc != H_SUCCESS && rc != H_RESOURCE) {
			dev_err(adapter->dev, "Error initializing CRQ.  rc = 0x%x\n",
				rc);
			ibmvmc.state = ibmvmc_state_failed;
		} else {
			ibmvmc.state = ibmvmc_state_crqinit;

			if (ibmvmc_send_crq(adapter, 0xC001000000000000LL, 0)
			    != 0 && rc != H_RESOURCE)
				dev_warn(adapter->dev, "Failed to send initialize CRQ message\n");
		}

		vio_enable_interrupts(to_vio_dev(adapter->dev));
		tasklet_enable(&adapter->work_task);
	}

	return 0;
}

/**
 * ibmvmc_process_open_resp - Process Open Response
 *
 * @crq: ibmvmc_crq_msg struct
 * @adapter:    crq_server_adapter struct
 *
 * This command is sent by the hypervisor in response to the Interface
 * Open message. When this message is received, the indicated buffer is
 * again available for management partition use.
 */
static void ibmvmc_process_open_resp(struct ibmvmc_crq_msg *crq,
				     struct crq_server_adapter *adapter)
{
	unsigned char hmc_index;
	unsigned short buffer_id;

	hmc_index = crq->hmc_index;
	if (hmc_index > ibmvmc.max_hmc_index) {
		/* Why would PHYP give an index > max negotiated? */
		ibmvmc_reset(adapter, false);
		return;
	}

	if (crq->status) {
		dev_warn(adapter->dev, "open_resp: failed - status 0x%x\n",
			 crq->status);
		ibmvmc_return_hmc(&hmcs[hmc_index], false);
		return;
	}

	if (hmcs[hmc_index].state == ibmhmc_state_opening) {
		buffer_id = be16_to_cpu(crq->var2.buffer_id);
		if (buffer_id >= ibmvmc.max_buffer_pool_size) {
			dev_err(adapter->dev, "open_resp: invalid buffer_id = 0x%x\n",
				buffer_id);
			hmcs[hmc_index].state = ibmhmc_state_failed;
		} else {
			ibmvmc_free_hmc_buffer(&hmcs[hmc_index],
					       &hmcs[hmc_index].buffer[buffer_id]);
			hmcs[hmc_index].state = ibmhmc_state_ready;
			dev_dbg(adapter->dev, "open_resp: set hmc state = ready\n");
		}
	} else {
		dev_warn(adapter->dev, "open_resp: invalid hmc state (0x%x)\n",
			 hmcs[hmc_index].state);
	}
}

/**
 * ibmvmc_process_close_resp - Process Close Response
 *
 * @crq: ibmvmc_crq_msg struct
 * @adapter:    crq_server_adapter struct
 *
 * This command is sent by the hypervisor in response to the managemant
 * application Interface Close message.
 *
 * If the close fails, simply reset the entire driver as the state of the VMC
 * must be in tough shape.
 */
static void ibmvmc_process_close_resp(struct ibmvmc_crq_msg *crq,
				      struct crq_server_adapter *adapter)
{
	unsigned char hmc_index;

	hmc_index = crq->hmc_index;
	if (hmc_index > ibmvmc.max_hmc_index) {
		ibmvmc_reset(adapter, false);
		return;
	}

	if (crq->status) {
		dev_warn(adapter->dev, "close_resp: failed - status 0x%x\n",
			 crq->status);
		ibmvmc_reset(adapter, false);
		return;
	}

	ibmvmc_return_hmc(&hmcs[hmc_index], false);
}

/**
 * ibmvmc_crq_process - Process CRQ
 *
 * @adapter:    crq_server_adapter struct
 * @crq:	ibmvmc_crq_msg struct
 *
 * Process the CRQ message based upon the type of message received.
 *
 */
static void ibmvmc_crq_process(struct crq_server_adapter *adapter,
			       struct ibmvmc_crq_msg *crq)
{
	switch (crq->type) {
	case VMC_MSG_CAP_RESP:
		dev_dbg(adapter->dev, "CRQ recv: capabilities resp (0x%x)\n",
			crq->type);
		if (ibmvmc.state == ibmvmc_state_capabilities)
			ibmvmc_process_capabilities(adapter, crq);
		else
			dev_warn(adapter->dev, "caps msg invalid in state 0x%x\n",
				 ibmvmc.state);
		break;
	case VMC_MSG_OPEN_RESP:
		dev_dbg(adapter->dev, "CRQ recv: open resp (0x%x)\n",
			crq->type);
		if (ibmvmc_validate_hmc_session(adapter, crq) == 0)
			ibmvmc_process_open_resp(crq, adapter);
		break;
	case VMC_MSG_ADD_BUF:
		dev_dbg(adapter->dev, "CRQ recv: add buf (0x%x)\n",
			crq->type);
		if (ibmvmc_validate_hmc_session(adapter, crq) == 0)
			ibmvmc_add_buffer(adapter, crq);
		break;
	case VMC_MSG_REM_BUF:
		dev_dbg(adapter->dev, "CRQ recv: rem buf (0x%x)\n",
			crq->type);
		if (ibmvmc_validate_hmc_session(adapter, crq) == 0)
			ibmvmc_rem_buffer(adapter, crq);
		break;
	case VMC_MSG_SIGNAL:
		dev_dbg(adapter->dev, "CRQ recv: signal msg (0x%x)\n",
			crq->type);
		if (ibmvmc_validate_hmc_session(adapter, crq) == 0)
			ibmvmc_recv_msg(adapter, crq);
		break;
	case VMC_MSG_CLOSE_RESP:
		dev_dbg(adapter->dev, "CRQ recv: close resp (0x%x)\n",
			crq->type);
		if (ibmvmc_validate_hmc_session(adapter, crq) == 0)
			ibmvmc_process_close_resp(crq, adapter);
		break;
	case VMC_MSG_CAP:
	case VMC_MSG_OPEN:
	case VMC_MSG_CLOSE:
	case VMC_MSG_ADD_BUF_RESP:
	case VMC_MSG_REM_BUF_RESP:
		dev_warn(adapter->dev, "CRQ recv: unexpected msg (0x%x)\n",
			 crq->type);
		break;
	default:
		dev_warn(adapter->dev, "CRQ recv: unknown msg (0x%x)\n",
			 crq->type);
		break;
	}
}

/**
 * ibmvmc_handle_crq_init - Handle CRQ Init
 *
 * @crq:	ibmvmc_crq_msg struct
 * @adapter:	crq_server_adapter struct
 *
 * Handle the type of crq initialization based on whether
 * it is a message or a response.
 *
 */
static void ibmvmc_handle_crq_init(struct ibmvmc_crq_msg *crq,
				   struct crq_server_adapter *adapter)
{
	switch (crq->type) {
	case 0x01:	/* Initialization message */
		dev_dbg(adapter->dev, "CRQ recv: CRQ init msg - state 0x%x\n",
			ibmvmc.state);
		if (ibmvmc.state == ibmvmc_state_crqinit) {
			/* Send back a response */
			if (ibmvmc_send_crq(adapter, 0xC002000000000000,
					    0) == 0)
				ibmvmc_send_capabilities(adapter);
			else
				dev_err(adapter->dev, " Unable to send init rsp\n");
		} else {
			dev_err(adapter->dev, "Invalid state 0x%x mtu = 0x%x\n",
				ibmvmc.state, ibmvmc.max_mtu);
		}

		break;
	case 0x02:	/* Initialization response */
		dev_dbg(adapter->dev, "CRQ recv: initialization resp msg - state 0x%x\n",
			ibmvmc.state);
		if (ibmvmc.state == ibmvmc_state_crqinit)
			ibmvmc_send_capabilities(adapter);
		break;
	default:
		dev_warn(adapter->dev, "Unknown crq message type 0x%lx\n",
			 (unsigned long)crq->type);
	}
}

/**
 * ibmvmc_handle_crq - Handle CRQ
 *
 * @crq:	ibmvmc_crq_msg struct
 * @adapter:	crq_server_adapter struct
 *
 * Read the command elements from the command queue and execute the
 * requests based upon the type of crq message.
 *
 */
static void ibmvmc_handle_crq(struct ibmvmc_crq_msg *crq,
			      struct crq_server_adapter *adapter)
{
	switch (crq->valid) {
	case 0xC0:		/* initialization */
		ibmvmc_handle_crq_init(crq, adapter);
		break;
	case 0xFF:	/* Hypervisor telling us the connection is closed */
		dev_warn(adapter->dev, "CRQ recv: virtual adapter failed - resetting.\n");
		ibmvmc_reset(adapter, true);
		break;
	case 0x80:	/* real payload */
		ibmvmc_crq_process(adapter, crq);
		break;
	default:
		dev_warn(adapter->dev, "CRQ recv: unknown msg 0x%02x.\n",
			 crq->valid);
		break;
	}
}

static void ibmvmc_task(unsigned long data)
{
	struct crq_server_adapter *adapter =
		(struct crq_server_adapter *)data;
	struct vio_dev *vdev = to_vio_dev(adapter->dev);
	struct ibmvmc_crq_msg *crq;
	int done = 0;

	while (!done) {
		/* Pull all the valid messages off the CRQ */
		while ((crq = crq_queue_next_crq(&adapter->queue)) != NULL) {
			ibmvmc_handle_crq(crq, adapter);
			crq->valid = 0x00;
			/* CRQ reset was requested, stop processing CRQs.
			 * Interrupts will be re-enabled by the reset task.
			 */
			if (ibmvmc.state == ibmvmc_state_sched_reset)
				return;
		}

		vio_enable_interrupts(vdev);
		crq = crq_queue_next_crq(&adapter->queue);
		if (crq) {
			vio_disable_interrupts(vdev);
			ibmvmc_handle_crq(crq, adapter);
			crq->valid = 0x00;
			/* CRQ reset was requested, stop processing CRQs.
			 * Interrupts will be re-enabled by the reset task.
			 */
			if (ibmvmc.state == ibmvmc_state_sched_reset)
				return;
		} else {
			done = 1;
		}
	}
}

/**
 * ibmvmc_init_crq_queue - Init CRQ Queue
 *
 * @adapter:	crq_server_adapter struct
 *
 * Return:
 *	0 - Success
 *	Non-zero - Failure
 */
static int ibmvmc_init_crq_queue(struct crq_server_adapter *adapter)
{
	struct vio_dev *vdev = to_vio_dev(adapter->dev);
	struct crq_queue *queue = &adapter->queue;
	int rc = 0;
	int retrc = 0;

	queue->msgs = (struct ibmvmc_crq_msg *)get_zeroed_page(GFP_KERNEL);

	if (!queue->msgs)
		goto malloc_failed;

	queue->size = PAGE_SIZE / sizeof(*queue->msgs);

	queue->msg_token = dma_map_single(adapter->dev, queue->msgs,
					  queue->size * sizeof(*queue->msgs),
					  DMA_BIDIRECTIONAL);

	if (dma_mapping_error(adapter->dev, queue->msg_token))
		goto map_failed;

	retrc = plpar_hcall_norets(H_REG_CRQ,
				   vdev->unit_address,
				   queue->msg_token, PAGE_SIZE);
	rc = retrc;

	if (rc == H_RESOURCE)
		rc = ibmvmc_reset_crq_queue(adapter);

	if (rc == 2) {
		dev_warn(adapter->dev, "Partner adapter not ready\n");
		retrc = 0;
	} else if (rc != 0) {
		dev_err(adapter->dev, "Error %d opening adapter\n", rc);
		goto reg_crq_failed;
	}

	queue->cur = 0;
	spin_lock_init(&queue->lock);

	tasklet_init(&adapter->work_task, ibmvmc_task, (unsigned long)adapter);

	if (request_irq(vdev->irq,
			ibmvmc_handle_event,
			0, "ibmvmc", (void *)adapter) != 0) {
		dev_err(adapter->dev, "couldn't register irq 0x%x\n",
			vdev->irq);
		goto req_irq_failed;
	}

	rc = vio_enable_interrupts(vdev);
	if (rc != 0) {
		dev_err(adapter->dev, "Error %d enabling interrupts!!!\n", rc);
		goto req_irq_failed;
	}

	return retrc;

req_irq_failed:
	/* Cannot have any work since we either never got our IRQ registered,
	 * or never got interrupts enabled
	 */
	tasklet_kill(&adapter->work_task);
	h_free_crq(vdev->unit_address);
reg_crq_failed:
	dma_unmap_single(adapter->dev,
			 queue->msg_token,
			 queue->size * sizeof(*queue->msgs), DMA_BIDIRECTIONAL);
map_failed:
	free_page((unsigned long)queue->msgs);
malloc_failed:
	return -ENOMEM;
}

/* Fill in the liobn and riobn fields on the adapter */
static int read_dma_window(struct vio_dev *vdev,
			   struct crq_server_adapter *adapter)
{
	const __be32 *dma_window;
	const __be32 *prop;

	/* TODO Using of_parse_dma_window would be better, but it doesn't give
	 * a way to read multiple windows without already knowing the size of
	 * a window or the number of windows
	 */
	dma_window =
		(const __be32 *)vio_get_attribute(vdev, "ibm,my-dma-window",
						NULL);
	if (!dma_window) {
		dev_warn(adapter->dev, "Couldn't find ibm,my-dma-window property\n");
		return -1;
	}

	adapter->liobn = be32_to_cpu(*dma_window);
	dma_window++;

	prop = (const __be32 *)vio_get_attribute(vdev, "ibm,#dma-address-cells",
						NULL);
	if (!prop) {
		dev_warn(adapter->dev, "Couldn't find ibm,#dma-address-cells property\n");
		dma_window++;
	} else {
		dma_window += be32_to_cpu(*prop);
	}

	prop = (const __be32 *)vio_get_attribute(vdev, "ibm,#dma-size-cells",
						NULL);
	if (!prop) {
		dev_warn(adapter->dev, "Couldn't find ibm,#dma-size-cells property\n");
		dma_window++;
	} else {
		dma_window += be32_to_cpu(*prop);
	}

	/* dma_window should point to the second window now */
	adapter->riobn = be32_to_cpu(*dma_window);

	return 0;
}

static int ibmvmc_probe(struct vio_dev *vdev, const struct vio_device_id *id)
{
	struct crq_server_adapter *adapter = &ibmvmc_adapter;
	int rc;

	dev_set_drvdata(&vdev->dev, NULL);
	memset(adapter, 0, sizeof(*adapter));
	adapter->dev = &vdev->dev;

	dev_info(adapter->dev, "Probe for UA 0x%x\n", vdev->unit_address);

	rc = read_dma_window(vdev, adapter);
	if (rc != 0) {
		ibmvmc.state = ibmvmc_state_failed;
		return -1;
	}

	dev_dbg(adapter->dev, "Probe: liobn 0x%x, riobn 0x%x\n",
		adapter->liobn, adapter->riobn);

	init_waitqueue_head(&adapter->reset_wait_queue);
	adapter->reset_task = kthread_run(ibmvmc_reset_task, adapter, "ibmvmc");
	if (IS_ERR(adapter->reset_task)) {
		dev_err(adapter->dev, "Failed to start reset thread\n");
		ibmvmc.state = ibmvmc_state_failed;
		rc = PTR_ERR(adapter->reset_task);
		adapter->reset_task = NULL;
		return rc;
	}

	rc = ibmvmc_init_crq_queue(adapter);
	if (rc != 0 && rc != H_RESOURCE) {
		dev_err(adapter->dev, "Error initializing CRQ.  rc = 0x%x\n",
			rc);
		ibmvmc.state = ibmvmc_state_failed;
		goto crq_failed;
	}

	ibmvmc.state = ibmvmc_state_crqinit;

	/* Try to send an initialization message.  Note that this is allowed
	 * to fail if the other end is not acive.  In that case we just wait
	 * for the other side to initialize.
	 */
	if (ibmvmc_send_crq(adapter, 0xC001000000000000LL, 0) != 0 &&
	    rc != H_RESOURCE)
		dev_warn(adapter->dev, "Failed to send initialize CRQ message\n");

	dev_set_drvdata(&vdev->dev, adapter);

	return 0;

crq_failed:
	kthread_stop(adapter->reset_task);
	adapter->reset_task = NULL;
	return -EPERM;
}

static void ibmvmc_remove(struct vio_dev *vdev)
{
	struct crq_server_adapter *adapter = dev_get_drvdata(&vdev->dev);

	dev_info(adapter->dev, "Entering remove for UA 0x%x\n",
		 vdev->unit_address);
	ibmvmc_release_crq_queue(adapter);
}

static struct vio_device_id ibmvmc_device_table[] = {
	{ "ibm,vmc", "IBM,vmc" },
	{ "", "" }
};
MODULE_DEVICE_TABLE(vio, ibmvmc_device_table);

static struct vio_driver ibmvmc_driver = {
	.name        = ibmvmc_driver_name,
	.id_table    = ibmvmc_device_table,
	.probe       = ibmvmc_probe,
	.remove      = ibmvmc_remove,
};

static void __init ibmvmc_scrub_module_parms(void)
{
	if (ibmvmc_max_mtu > MAX_MTU) {
		pr_warn("ibmvmc: Max MTU reduced to %d\n", MAX_MTU);
		ibmvmc_max_mtu = MAX_MTU;
	} else if (ibmvmc_max_mtu < MIN_MTU) {
		pr_warn("ibmvmc: Max MTU increased to %d\n", MIN_MTU);
		ibmvmc_max_mtu = MIN_MTU;
	}

	if (ibmvmc_max_buf_pool_size > MAX_BUF_POOL_SIZE) {
		pr_warn("ibmvmc: Max buffer pool size reduced to %d\n",
			MAX_BUF_POOL_SIZE);
		ibmvmc_max_buf_pool_size = MAX_BUF_POOL_SIZE;
	} else if (ibmvmc_max_buf_pool_size < MIN_BUF_POOL_SIZE) {
		pr_warn("ibmvmc: Max buffer pool size increased to %d\n",
			MIN_BUF_POOL_SIZE);
		ibmvmc_max_buf_pool_size = MIN_BUF_POOL_SIZE;
	}

	if (ibmvmc_max_hmcs > MAX_HMCS) {
		pr_warn("ibmvmc: Max HMCs reduced to %d\n", MAX_HMCS);
		ibmvmc_max_hmcs = MAX_HMCS;
	} else if (ibmvmc_max_hmcs < MIN_HMCS) {
		pr_warn("ibmvmc: Max HMCs increased to %d\n", MIN_HMCS);
		ibmvmc_max_hmcs = MIN_HMCS;
	}
}

static struct miscdevice ibmvmc_miscdev = {
	.name = ibmvmc_driver_name,
	.minor = MISC_DYNAMIC_MINOR,
	.fops = &ibmvmc_fops,
};

static int __init ibmvmc_module_init(void)
{
	int rc, i, j;

	ibmvmc.state = ibmvmc_state_initial;
	pr_info("ibmvmc: version %s\n", IBMVMC_DRIVER_VERSION);

	rc = misc_register(&ibmvmc_miscdev);
	if (rc) {
		pr_err("ibmvmc: misc registration failed\n");
		goto misc_register_failed;
	}
	pr_info("ibmvmc: node %d:%d\n", MISC_MAJOR,
		ibmvmc_miscdev.minor);

	/* Initialize data structures */
	memset(hmcs, 0, sizeof(struct ibmvmc_hmc) * MAX_HMCS);
	for (i = 0; i < MAX_HMCS; i++) {
		spin_lock_init(&hmcs[i].lock);
		hmcs[i].state = ibmhmc_state_free;
		for (j = 0; j < MAX_BUF_POOL_SIZE; j++)
			hmcs[i].queue_outbound_msgs[j] = VMC_INVALID_BUFFER_ID;
	}

	/* Sanity check module parms */
	ibmvmc_scrub_module_parms();

	/*
	 * Initialize some reasonable values.  Might be negotiated smaller
	 * values during the capabilities exchange.
	 */
	ibmvmc.max_mtu = ibmvmc_max_mtu;
	ibmvmc.max_buffer_pool_size = ibmvmc_max_buf_pool_size;
	ibmvmc.max_hmc_index = ibmvmc_max_hmcs - 1;

	rc = vio_register_driver(&ibmvmc_driver);

	if (rc) {
		pr_err("ibmvmc: rc %d from vio_register_driver\n", rc);
		goto vio_reg_failed;
	}

	return 0;

vio_reg_failed:
	misc_deregister(&ibmvmc_miscdev);
misc_register_failed:
	return rc;
}

static void __exit ibmvmc_module_exit(void)
{
	pr_info("ibmvmc: module exit\n");
	vio_unregister_driver(&ibmvmc_driver);
	misc_deregister(&ibmvmc_miscdev);
}

module_init(ibmvmc_module_init);
module_exit(ibmvmc_module_exit);

module_param_named(buf_pool_size, ibmvmc_max_buf_pool_size,
		   int, 0644);
MODULE_PARM_DESC(buf_pool_size, "Buffer pool size");
module_param_named(max_hmcs, ibmvmc_max_hmcs, int, 0644);
MODULE_PARM_DESC(max_hmcs, "Max HMCs");
module_param_named(max_mtu, ibmvmc_max_mtu, int, 0644);
MODULE_PARM_DESC(max_mtu, "Max MTU");

MODULE_AUTHOR("Steven Royer <seroyer@linux.vnet.ibm.com>");
MODULE_DESCRIPTION("IBM VMC");
MODULE_VERSION(IBMVMC_DRIVER_VERSION);
MODULE_LICENSE("GPL v2");