#include <linux/module.h>
#include <linux/list.h>
#include <linux/pci.h>
+#include <linux/slab.h>
#include <linux/interrupt.h>
#include <linux/virtio.h>
#include <linux/virtio_config.h>
char (*msix_names)[256];
/* Number of available vectors */
unsigned msix_vectors;
- /* Vectors allocated */
+ /* Vectors allocated, excluding per-vq vectors if any */
unsigned msix_used_vectors;
+ /* Whether we have vector per vq */
+ bool per_vq_vectors;
};
/* Constants for MSI-X */
struct list_head node;
/* MSI-X vector (or none) */
- unsigned vector;
+ unsigned msix_vector;
};
/* Qumranet donated their vendor ID for devices 0x1000 thru 0x10FF. */
vp_dev->msix_entries = NULL;
}
-static int vp_enable_msix(struct pci_dev *dev, struct msix_entry *entries,
- int *options, int noptions)
-{
- int i;
- for (i = 0; i < noptions; ++i)
- if (!pci_enable_msix(dev, entries, options[i]))
- return options[i];
- return -EBUSY;
-}
-
-static int vp_request_vectors(struct virtio_device *vdev, unsigned max_vqs)
+static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors,
+ bool per_vq_vectors)
{
struct virtio_pci_device *vp_dev = to_vp_device(vdev);
const char *name = dev_name(&vp_dev->vdev.dev);
unsigned i, v;
int err = -ENOMEM;
- /* We want at most one vector per queue and one for config changes.
- * Fallback to separate vectors for config and a shared for queues.
- * Finally fall back to regular interrupts. */
- int options[] = { max_vqs + 1, 2 };
- int nvectors = max(options[0], options[1]);
vp_dev->msix_entries = kmalloc(nvectors * sizeof *vp_dev->msix_entries,
GFP_KERNEL);
for (i = 0; i < nvectors; ++i)
vp_dev->msix_entries[i].entry = i;
- err = vp_enable_msix(vp_dev->pci_dev, vp_dev->msix_entries,
- options, ARRAY_SIZE(options));
- if (err < 0) {
- /* Can't allocate enough MSI-X vectors, use regular interrupt */
- vp_dev->msix_vectors = 0;
- err = request_irq(vp_dev->pci_dev->irq, vp_interrupt,
- IRQF_SHARED, name, vp_dev);
- if (err)
- goto error;
- vp_dev->intx_enabled = 1;
- } else {
- vp_dev->msix_vectors = err;
- vp_dev->msix_enabled = 1;
-
- /* Set the vector used for configuration */
- v = vp_dev->msix_used_vectors;
- snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
- "%s-config", name);
- err = request_irq(vp_dev->msix_entries[v].vector,
- vp_config_changed, 0, vp_dev->msix_names[v],
- vp_dev);
- if (err)
- goto error;
- ++vp_dev->msix_used_vectors;
+ /* pci_enable_msix returns positive if we can't get this many. */
+ err = pci_enable_msix(vp_dev->pci_dev, vp_dev->msix_entries, nvectors);
+ if (err > 0)
+ err = -ENOSPC;
+ if (err)
+ goto error;
+ vp_dev->msix_vectors = nvectors;
+ vp_dev->msix_enabled = 1;
+
+ /* Set the vector used for configuration */
+ v = vp_dev->msix_used_vectors;
+ snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
+ "%s-config", name);
+ err = request_irq(vp_dev->msix_entries[v].vector,
+ vp_config_changed, 0, vp_dev->msix_names[v],
+ vp_dev);
+ if (err)
+ goto error;
+ ++vp_dev->msix_used_vectors;
- iowrite16(v, vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
- /* Verify we had enough resources to assign the vector */
- v = ioread16(vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
- if (v == VIRTIO_MSI_NO_VECTOR) {
- err = -EBUSY;
- goto error;
- }
+ iowrite16(v, vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
+ /* Verify we had enough resources to assign the vector */
+ v = ioread16(vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
+ if (v == VIRTIO_MSI_NO_VECTOR) {
+ err = -EBUSY;
+ goto error;
}
- if (vp_dev->msix_vectors && vp_dev->msix_vectors != max_vqs + 1) {
+ if (!per_vq_vectors) {
/* Shared vector for all VQs */
v = vp_dev->msix_used_vectors;
snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
return err;
}
-static struct virtqueue *vp_find_vq(struct virtio_device *vdev, unsigned index,
- void (*callback)(struct virtqueue *vq),
- const char *name)
+static int vp_request_intx(struct virtio_device *vdev)
+{
+ int err;
+ struct virtio_pci_device *vp_dev = to_vp_device(vdev);
+
+ err = request_irq(vp_dev->pci_dev->irq, vp_interrupt,
+ IRQF_SHARED, dev_name(&vdev->dev), vp_dev);
+ if (!err)
+ vp_dev->intx_enabled = 1;
+ return err;
+}
+
+static struct virtqueue *setup_vq(struct virtio_device *vdev, unsigned index,
+ void (*callback)(struct virtqueue *vq),
+ const char *name,
+ u16 msix_vec)
{
struct virtio_pci_device *vp_dev = to_vp_device(vdev);
struct virtio_pci_vq_info *info;
struct virtqueue *vq;
unsigned long flags, size;
- u16 num, vector;
+ u16 num;
int err;
/* Select the queue we're interested in */
info->queue_index = index;
info->num = num;
- info->vector = VIRTIO_MSI_NO_VECTOR;
+ info->msix_vector = msix_vec;
size = PAGE_ALIGN(vring_size(num, VIRTIO_PCI_VRING_ALIGN));
info->queue = alloc_pages_exact(size, GFP_KERNEL|__GFP_ZERO);
vq->priv = info;
info->vq = vq;
- /* allocate per-vq vector if available and necessary */
- if (callback && vp_dev->msix_used_vectors < vp_dev->msix_vectors) {
- vector = vp_dev->msix_used_vectors;
- snprintf(vp_dev->msix_names[vector], sizeof *vp_dev->msix_names,
- "%s-%s", dev_name(&vp_dev->vdev.dev), name);
- err = request_irq(vp_dev->msix_entries[vector].vector,
- vring_interrupt, 0,
- vp_dev->msix_names[vector], vq);
- if (err)
- goto out_request_irq;
- info->vector = vector;
- ++vp_dev->msix_used_vectors;
- } else
- vector = VP_MSIX_VQ_VECTOR;
-
- if (callback && vp_dev->msix_enabled) {
- iowrite16(vector, vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
- vector = ioread16(vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
- if (vector == VIRTIO_MSI_NO_VECTOR) {
+ if (msix_vec != VIRTIO_MSI_NO_VECTOR) {
+ iowrite16(msix_vec, vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
+ msix_vec = ioread16(vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
+ if (msix_vec == VIRTIO_MSI_NO_VECTOR) {
err = -EBUSY;
goto out_assign;
}
return vq;
out_assign:
- if (info->vector != VIRTIO_MSI_NO_VECTOR) {
- free_irq(vp_dev->msix_entries[info->vector].vector, vq);
- --vp_dev->msix_used_vectors;
- }
-out_request_irq:
vring_del_virtqueue(vq);
out_activate_queue:
iowrite32(0, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
iowrite16(info->queue_index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_SEL);
- if (info->vector != VIRTIO_MSI_NO_VECTOR)
- free_irq(vp_dev->msix_entries[info->vector].vector, vq);
-
if (vp_dev->msix_enabled) {
iowrite16(VIRTIO_MSI_NO_VECTOR,
vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
/* the config->del_vqs() implementation */
static void vp_del_vqs(struct virtio_device *vdev)
{
+ struct virtio_pci_device *vp_dev = to_vp_device(vdev);
struct virtqueue *vq, *n;
+ struct virtio_pci_vq_info *info;
- list_for_each_entry_safe(vq, n, &vdev->vqs, list)
+ list_for_each_entry_safe(vq, n, &vdev->vqs, list) {
+ info = vq->priv;
+ if (vp_dev->per_vq_vectors &&
+ info->msix_vector != VIRTIO_MSI_NO_VECTOR)
+ free_irq(vp_dev->msix_entries[info->msix_vector].vector,
+ vq);
vp_del_vq(vq);
+ }
+ vp_dev->per_vq_vectors = false;
vp_free_vectors(vdev);
}
-/* the config->find_vqs() implementation */
-static int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs,
- struct virtqueue *vqs[],
- vq_callback_t *callbacks[],
- const char *names[])
+static int vp_try_to_find_vqs(struct virtio_device *vdev, unsigned nvqs,
+ struct virtqueue *vqs[],
+ vq_callback_t *callbacks[],
+ const char *names[],
+ bool use_msix,
+ bool per_vq_vectors)
{
- int vectors = 0;
- int i, err;
+ struct virtio_pci_device *vp_dev = to_vp_device(vdev);
+ u16 msix_vec;
+ int i, err, nvectors, allocated_vectors;
- /* How many vectors would we like? */
- for (i = 0; i < nvqs; ++i)
- if (callbacks[i])
- ++vectors;
+ if (!use_msix) {
+ /* Old style: one normal interrupt for change and all vqs. */
+ err = vp_request_intx(vdev);
+ if (err)
+ goto error_request;
+ } else {
+ if (per_vq_vectors) {
+ /* Best option: one for change interrupt, one per vq. */
+ nvectors = 1;
+ for (i = 0; i < nvqs; ++i)
+ if (callbacks[i])
+ ++nvectors;
+ } else {
+ /* Second best: one for change, shared for all vqs. */
+ nvectors = 2;
+ }
- err = vp_request_vectors(vdev, vectors);
- if (err)
- goto error_request;
+ err = vp_request_msix_vectors(vdev, nvectors, per_vq_vectors);
+ if (err)
+ goto error_request;
+ }
+ vp_dev->per_vq_vectors = per_vq_vectors;
+ allocated_vectors = vp_dev->msix_used_vectors;
for (i = 0; i < nvqs; ++i) {
- vqs[i] = vp_find_vq(vdev, i, callbacks[i], names[i]);
- if (IS_ERR(vqs[i]))
+ if (!callbacks[i] || !vp_dev->msix_enabled)
+ msix_vec = VIRTIO_MSI_NO_VECTOR;
+ else if (vp_dev->per_vq_vectors)
+ msix_vec = allocated_vectors++;
+ else
+ msix_vec = VP_MSIX_VQ_VECTOR;
+ vqs[i] = setup_vq(vdev, i, callbacks[i], names[i], msix_vec);
+ if (IS_ERR(vqs[i])) {
+ err = PTR_ERR(vqs[i]);
goto error_find;
+ }
+
+ if (!vp_dev->per_vq_vectors || msix_vec == VIRTIO_MSI_NO_VECTOR)
+ continue;
+
+ /* allocate per-vq irq if available and necessary */
+ snprintf(vp_dev->msix_names[msix_vec],
+ sizeof *vp_dev->msix_names,
+ "%s-%s",
+ dev_name(&vp_dev->vdev.dev), names[i]);
+ err = request_irq(vp_dev->msix_entries[msix_vec].vector,
+ vring_interrupt, 0,
+ vp_dev->msix_names[msix_vec],
+ vqs[i]);
+ if (err) {
+ vp_del_vq(vqs[i]);
+ goto error_find;
+ }
}
return 0;
vp_del_vqs(vdev);
error_request:
- return PTR_ERR(vqs[i]);
+ return err;
+}
+
+/* the config->find_vqs() implementation */
+static int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs,
+ struct virtqueue *vqs[],
+ vq_callback_t *callbacks[],
+ const char *names[])
+{
+ int err;
+
+ /* Try MSI-X with one vector per queue. */
+ err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names, true, true);
+ if (!err)
+ return 0;
+ /* Fallback: MSI-X with one vector for config, one shared for queues. */
+ err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names,
+ true, false);
+ if (!err)
+ return 0;
+ /* Finally fall back to regular interrupts. */
+ return vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names,
+ false, false);
}
static struct virtio_config_ops virtio_pci_config_ops = {
goto out_req_regions;
pci_set_drvdata(pci_dev, vp_dev);
+ pci_set_master(pci_dev);
/* we use the subsystem vendor/device id as the virtio vendor/device
* id. this allows us to use the same PCI vendor/device id for all
* virtio devices and to identify the particular virtio driver by
- * the subsytem ids */
+ * the subsystem ids */
vp_dev->vdev.id.vendor = pci_dev->subsystem_vendor;
vp_dev->vdev.id.device = pci_dev->subsystem_device;
.name = "virtio-pci",
.id_table = virtio_pci_id_table,
.probe = virtio_pci_probe,
- .remove = virtio_pci_remove,
+ .remove = __devexit_p(virtio_pci_remove),
#ifdef CONFIG_PM
.suspend = virtio_pci_suspend,
.resume = virtio_pci_resume,
git://ftp.safe.ca / safe / jmp / linux-2.6 / blobdiff