#include <linux/errno.h>
#include <linux/io.h>
#include <linux/slab.h>
+#include <linux/irqdomain.h>
#include "pci.h"
static int pci_msi_enable = 1;
+int pci_msi_ignore_mask;
#define msix_table_size(flags) ((flags & PCI_MSIX_FLAGS_QSIZE) + 1)
+#ifdef CONFIG_PCI_MSI_IRQ_DOMAIN
+static struct irq_domain *pci_msi_default_domain;
+static DEFINE_MUTEX(pci_msi_domain_lock);
+
+struct irq_domain * __weak arch_get_pci_msi_domain(struct pci_dev *dev)
+{
+ return pci_msi_default_domain;
+}
+
+static struct irq_domain *pci_msi_get_domain(struct pci_dev *dev)
+{
+ struct irq_domain *domain = NULL;
+
+ if (dev->bus->msi)
+ domain = dev->bus->msi->domain;
+ if (!domain)
+ domain = arch_get_pci_msi_domain(dev);
+
+ return domain;
+}
+
+static int pci_msi_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
+{
+ struct irq_domain *domain;
+
+ domain = pci_msi_get_domain(dev);
+ if (domain)
+ return pci_msi_domain_alloc_irqs(domain, dev, nvec, type);
+
+ return arch_setup_msi_irqs(dev, nvec, type);
+}
+
+static void pci_msi_teardown_msi_irqs(struct pci_dev *dev)
+{
+ struct irq_domain *domain;
+
+ domain = pci_msi_get_domain(dev);
+ if (domain)
+ pci_msi_domain_free_irqs(domain, dev);
+ else
+ arch_teardown_msi_irqs(dev);
+}
+#else
+#define pci_msi_setup_msi_irqs arch_setup_msi_irqs
+#define pci_msi_teardown_msi_irqs arch_teardown_msi_irqs
+#endif
/* Arch hooks */
+struct msi_controller * __weak pcibios_msi_controller(struct pci_dev *dev)
+{
+ return NULL;
+}
+
+static struct msi_controller *pci_msi_controller(struct pci_dev *dev)
+{
+ struct msi_controller *msi_ctrl = dev->bus->msi;
+
+ if (msi_ctrl)
+ return msi_ctrl;
+
+ return pcibios_msi_controller(dev);
+}
+
int __weak arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc)
{
- struct msi_chip *chip = dev->bus->msi;
+ struct msi_controller *chip = pci_msi_controller(dev);
int err;
if (!chip || !chip->setup_irq)
void __weak arch_teardown_msi_irq(unsigned int irq)
{
- struct msi_chip *chip = irq_get_chip_data(irq);
+ struct msi_controller *chip = irq_get_chip_data(irq);
if (!chip || !chip->teardown_irq)
return;
*/
void default_teardown_msi_irqs(struct pci_dev *dev)
{
+ int i;
struct msi_desc *entry;
- list_for_each_entry(entry, &dev->msi_list, list) {
- int i, nvec;
- if (entry->irq == 0)
- continue;
- if (entry->nvec_used)
- nvec = entry->nvec_used;
- else
- nvec = 1 << entry->msi_attrib.multiple;
- for (i = 0; i < nvec; i++)
- arch_teardown_msi_irq(entry->irq + i);
- }
+ list_for_each_entry(entry, &dev->msi_list, list)
+ if (entry->irq)
+ for (i = 0; i < entry->nvec_used; i++)
+ arch_teardown_msi_irq(entry->irq + i);
}
void __weak arch_teardown_msi_irqs(struct pci_dev *dev)
}
if (entry)
- __write_msi_msg(entry, &entry->msg);
+ __pci_write_msi_msg(entry, &entry->msg);
}
void __weak arch_restore_msi_irqs(struct pci_dev *dev)
* reliably as devices without an INTx disable bit will then generate a
* level IRQ which will never be cleared.
*/
-u32 default_msi_mask_irq(struct msi_desc *desc, u32 mask, u32 flag)
+u32 __pci_msi_desc_mask_irq(struct msi_desc *desc, u32 mask, u32 flag)
{
u32 mask_bits = desc->masked;
- if (!desc->msi_attrib.maskbit)
+ if (pci_msi_ignore_mask || !desc->msi_attrib.maskbit)
return 0;
mask_bits &= ~mask;
return mask_bits;
}
-__weak u32 arch_msi_mask_irq(struct msi_desc *desc, u32 mask, u32 flag)
-{
- return default_msi_mask_irq(desc, mask, flag);
-}
-
static void msi_mask_irq(struct msi_desc *desc, u32 mask, u32 flag)
{
- desc->masked = arch_msi_mask_irq(desc, mask, flag);
+ desc->masked = __pci_msi_desc_mask_irq(desc, mask, flag);
}
/*
* file. This saves a few milliseconds when initialising devices with lots
* of MSI-X interrupts.
*/
-u32 default_msix_mask_irq(struct msi_desc *desc, u32 flag)
+u32 __pci_msix_desc_mask_irq(struct msi_desc *desc, u32 flag)
{
u32 mask_bits = desc->masked;
unsigned offset = desc->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE +
PCI_MSIX_ENTRY_VECTOR_CTRL;
+
+ if (pci_msi_ignore_mask)
+ return 0;
+
mask_bits &= ~PCI_MSIX_ENTRY_CTRL_MASKBIT;
if (flag)
mask_bits |= PCI_MSIX_ENTRY_CTRL_MASKBIT;
return mask_bits;
}
-__weak u32 arch_msix_mask_irq(struct msi_desc *desc, u32 flag)
-{
- return default_msix_mask_irq(desc, flag);
-}
-
static void msix_mask_irq(struct msi_desc *desc, u32 flag)
{
- desc->masked = arch_msix_mask_irq(desc, flag);
+ desc->masked = __pci_msix_desc_mask_irq(desc, flag);
}
static void msi_set_mask_bit(struct irq_data *data, u32 flag)
}
}
-void mask_msi_irq(struct irq_data *data)
+/**
+ * pci_msi_mask_irq - Generic irq chip callback to mask PCI/MSI interrupts
+ * @data: pointer to irqdata associated to that interrupt
+ */
+void pci_msi_mask_irq(struct irq_data *data)
{
msi_set_mask_bit(data, 1);
}
-void unmask_msi_irq(struct irq_data *data)
+/**
+ * pci_msi_unmask_irq - Generic irq chip callback to unmask PCI/MSI interrupts
+ * @data: pointer to irqdata associated to that interrupt
+ */
+void pci_msi_unmask_irq(struct irq_data *data)
{
msi_set_mask_bit(data, 0);
}
{
struct msi_desc *entry;
- list_for_each_entry(entry, &dev->msi_list, list) {
+ list_for_each_entry(entry, &dev->msi_list, list)
default_restore_msi_irq(dev, entry->irq);
- }
}
-void __read_msi_msg(struct msi_desc *entry, struct msi_msg *msg)
+void __pci_read_msi_msg(struct msi_desc *entry, struct msi_msg *msg)
{
BUG_ON(entry->dev->current_state != PCI_D0);
}
}
-void read_msi_msg(unsigned int irq, struct msi_msg *msg)
-{
- struct msi_desc *entry = irq_get_msi_desc(irq);
-
- __read_msi_msg(entry, msg);
-}
-
-void __get_cached_msi_msg(struct msi_desc *entry, struct msi_msg *msg)
-{
- /* Assert that the cache is valid, assuming that
- * valid messages are not all-zeroes. */
- BUG_ON(!(entry->msg.address_hi | entry->msg.address_lo |
- entry->msg.data));
-
- *msg = entry->msg;
-}
-
-void get_cached_msi_msg(unsigned int irq, struct msi_msg *msg)
-{
- struct msi_desc *entry = irq_get_msi_desc(irq);
-
- __get_cached_msi_msg(entry, msg);
-}
-EXPORT_SYMBOL_GPL(get_cached_msi_msg);
-
-void __write_msi_msg(struct msi_desc *entry, struct msi_msg *msg)
+void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg)
{
if (entry->dev->current_state != PCI_D0) {
/* Don't touch the hardware now */
entry->msg = *msg;
}
-void write_msi_msg(unsigned int irq, struct msi_msg *msg)
+void pci_write_msi_msg(unsigned int irq, struct msi_msg *msg)
{
struct msi_desc *entry = irq_get_msi_desc(irq);
- __write_msi_msg(entry, msg);
+ __pci_write_msi_msg(entry, msg);
}
-EXPORT_SYMBOL_GPL(write_msi_msg);
+EXPORT_SYMBOL_GPL(pci_write_msi_msg);
static void free_msi_irqs(struct pci_dev *dev)
{
struct msi_desc *entry, *tmp;
struct attribute **msi_attrs;
struct device_attribute *dev_attr;
- int count = 0;
+ int i, count = 0;
- list_for_each_entry(entry, &dev->msi_list, list) {
- int i, nvec;
- if (!entry->irq)
- continue;
- if (entry->nvec_used)
- nvec = entry->nvec_used;
- else
- nvec = 1 << entry->msi_attrib.multiple;
- for (i = 0; i < nvec; i++)
- BUG_ON(irq_has_action(entry->irq + i));
- }
+ list_for_each_entry(entry, &dev->msi_list, list)
+ if (entry->irq)
+ for (i = 0; i < entry->nvec_used; i++)
+ BUG_ON(irq_has_action(entry->irq + i));
- arch_teardown_msi_irqs(dev);
+ pci_msi_teardown_msi_irqs(dev);
list_for_each_entry_safe(entry, tmp, &dev->msi_list, list) {
if (entry->msi_attrib.is_msix) {
PCI_MSIX_FLAGS_ENABLE | PCI_MSIX_FLAGS_MASKALL);
arch_restore_msi_irqs(dev);
- list_for_each_entry(entry, &dev->msi_list, list) {
+ list_for_each_entry(entry, &dev->msi_list, list)
msix_mask_irq(entry, entry->masked);
- }
msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0);
}
int count = 0;
/* Determine how many msi entries we have */
- list_for_each_entry(entry, &pdev->msi_list, list) {
+ list_for_each_entry(entry, &pdev->msi_list, list)
++num_msi;
- }
if (!num_msi)
return 0;
return ret;
}
-static struct msi_desc *msi_setup_entry(struct pci_dev *dev)
+static struct msi_desc *msi_setup_entry(struct pci_dev *dev, int nvec)
{
u16 control;
struct msi_desc *entry;
entry->msi_attrib.maskbit = !!(control & PCI_MSI_FLAGS_MASKBIT);
entry->msi_attrib.default_irq = dev->irq; /* Save IOAPIC IRQ */
entry->msi_attrib.multi_cap = (control & PCI_MSI_FLAGS_QMASK) >> 1;
+ entry->msi_attrib.multiple = ilog2(__roundup_pow_of_two(nvec));
+ entry->nvec_used = nvec;
if (control & PCI_MSI_FLAGS_64BIT)
entry->mask_pos = dev->msi_cap + PCI_MSI_MASK_64;
msi_set_enable(dev, 0); /* Disable MSI during set up */
- entry = msi_setup_entry(dev);
+ entry = msi_setup_entry(dev, nvec);
if (!entry)
return -ENOMEM;
list_add_tail(&entry->list, &dev->msi_list);
/* Configure MSI capability structure */
- ret = arch_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSI);
+ ret = pci_msi_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSI);
if (ret) {
msi_mask_irq(entry, mask, ~mask);
free_msi_irqs(dev);
{
resource_size_t phys_addr;
u32 table_offset;
+ unsigned long flags;
u8 bir;
pci_read_config_dword(dev, dev->msix_cap + PCI_MSIX_TABLE,
&table_offset);
bir = (u8)(table_offset & PCI_MSIX_TABLE_BIR);
+ flags = pci_resource_flags(dev, bir);
+ if (!flags || (flags & IORESOURCE_UNSET))
+ return NULL;
+
table_offset &= PCI_MSIX_TABLE_OFFSET;
phys_addr = pci_resource_start(dev, bir) + table_offset;
entry->msi_attrib.entry_nr = entries[i].entry;
entry->msi_attrib.default_irq = dev->irq;
entry->mask_base = base;
+ entry->nvec_used = 1;
list_add_tail(&entry->list, &dev->msi_list);
}
PCI_MSIX_ENTRY_VECTOR_CTRL;
entries[i].vector = entry->irq;
- irq_set_msi_desc(entry->irq, entry);
entry->masked = readl(entry->mask_base + offset);
msix_mask_irq(entry, 1);
i++;
if (ret)
return ret;
- ret = arch_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSIX);
+ ret = pci_msi_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSIX);
if (ret)
goto out_avail;
/* Return the device with MSI unmasked as initial states */
mask = msi_mask(desc->msi_attrib.multi_cap);
/* Keep cached state to be restored */
- arch_msi_mask_irq(desc, mask, ~mask);
+ __pci_msi_desc_mask_irq(desc, mask, ~mask);
/* Restore dev->irq to its default pin-assertion irq */
dev->irq = desc->msi_attrib.default_irq;
/* Return the device with MSI-X masked as initial states */
list_for_each_entry(entry, &dev->msi_list, list) {
/* Keep cached states to be restored */
- arch_msix_mask_irq(entry, 1);
+ __pci_msix_desc_mask_irq(entry, 1);
}
msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_ENABLE, 0);
return nvec;
}
EXPORT_SYMBOL(pci_enable_msix_range);
+
+#ifdef CONFIG_PCI_MSI_IRQ_DOMAIN
+/**
+ * pci_msi_domain_write_msg - Helper to write MSI message to PCI config space
+ * @irq_data: Pointer to interrupt data of the MSI interrupt
+ * @msg: Pointer to the message
+ */
+void pci_msi_domain_write_msg(struct irq_data *irq_data, struct msi_msg *msg)
+{
+ struct msi_desc *desc = irq_data->msi_desc;
+
+ /*
+ * For MSI-X desc->irq is always equal to irq_data->irq. For
+ * MSI only the first interrupt of MULTI MSI passes the test.
+ */
+ if (desc->irq == irq_data->irq)
+ __pci_write_msi_msg(desc, msg);
+}
+
+/**
+ * pci_msi_domain_calc_hwirq - Generate a unique ID for an MSI source
+ * @dev: Pointer to the PCI device
+ * @desc: Pointer to the msi descriptor
+ *
+ * The ID number is only used within the irqdomain.
+ */
+irq_hw_number_t pci_msi_domain_calc_hwirq(struct pci_dev *dev,
+ struct msi_desc *desc)
+{
+ return (irq_hw_number_t)desc->msi_attrib.entry_nr |
+ PCI_DEVID(dev->bus->number, dev->devfn) << 11 |
+ (pci_domain_nr(dev->bus) & 0xFFFFFFFF) << 27;
+}
+
+static inline bool pci_msi_desc_is_multi_msi(struct msi_desc *desc)
+{
+ return !desc->msi_attrib.is_msix && desc->nvec_used > 1;
+}
+
+/**
+ * pci_msi_domain_check_cap - Verify that @domain supports the capabilities for @dev
+ * @domain: The interrupt domain to check
+ * @info: The domain info for verification
+ * @dev: The device to check
+ *
+ * Returns:
+ * 0 if the functionality is supported
+ * 1 if Multi MSI is requested, but the domain does not support it
+ * -ENOTSUPP otherwise
+ */
+int pci_msi_domain_check_cap(struct irq_domain *domain,
+ struct msi_domain_info *info, struct device *dev)
+{
+ struct msi_desc *desc = first_pci_msi_entry(to_pci_dev(dev));
+
+ /* Special handling to support pci_enable_msi_range() */
+ if (pci_msi_desc_is_multi_msi(desc) &&
+ !(info->flags & MSI_FLAG_MULTI_PCI_MSI))
+ return 1;
+ else if (desc->msi_attrib.is_msix && !(info->flags & MSI_FLAG_PCI_MSIX))
+ return -ENOTSUPP;
+
+ return 0;
+}
+
+static int pci_msi_domain_handle_error(struct irq_domain *domain,
+ struct msi_desc *desc, int error)
+{
+ /* Special handling to support pci_enable_msi_range() */
+ if (pci_msi_desc_is_multi_msi(desc) && error == -ENOSPC)
+ return 1;
+
+ return error;
+}
+
+#ifdef GENERIC_MSI_DOMAIN_OPS
+static void pci_msi_domain_set_desc(msi_alloc_info_t *arg,
+ struct msi_desc *desc)
+{
+ arg->desc = desc;
+ arg->hwirq = pci_msi_domain_calc_hwirq(msi_desc_to_pci_dev(desc),
+ desc);
+}
+#else
+#define pci_msi_domain_set_desc NULL
+#endif
+
+static struct msi_domain_ops pci_msi_domain_ops_default = {
+ .set_desc = pci_msi_domain_set_desc,
+ .msi_check = pci_msi_domain_check_cap,
+ .handle_error = pci_msi_domain_handle_error,
+};
+
+static void pci_msi_domain_update_dom_ops(struct msi_domain_info *info)
+{
+ struct msi_domain_ops *ops = info->ops;
+
+ if (ops == NULL) {
+ info->ops = &pci_msi_domain_ops_default;
+ } else {
+ if (ops->set_desc == NULL)
+ ops->set_desc = pci_msi_domain_set_desc;
+ if (ops->msi_check == NULL)
+ ops->msi_check = pci_msi_domain_check_cap;
+ if (ops->handle_error == NULL)
+ ops->handle_error = pci_msi_domain_handle_error;
+ }
+}
+
+static void pci_msi_domain_update_chip_ops(struct msi_domain_info *info)
+{
+ struct irq_chip *chip = info->chip;
+
+ BUG_ON(!chip);
+ if (!chip->irq_write_msi_msg)
+ chip->irq_write_msi_msg = pci_msi_domain_write_msg;
+}
+
+/**
+ * pci_msi_create_irq_domain - Creat a MSI interrupt domain
+ * @node: Optional device-tree node of the interrupt controller
+ * @info: MSI domain info
+ * @parent: Parent irq domain
+ *
+ * Updates the domain and chip ops and creates a MSI interrupt domain.
+ *
+ * Returns:
+ * A domain pointer or NULL in case of failure.
+ */
+struct irq_domain *pci_msi_create_irq_domain(struct device_node *node,
+ struct msi_domain_info *info,
+ struct irq_domain *parent)
+{
+ if (info->flags & MSI_FLAG_USE_DEF_DOM_OPS)
+ pci_msi_domain_update_dom_ops(info);
+ if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS)
+ pci_msi_domain_update_chip_ops(info);
+
+ return msi_create_irq_domain(node, info, parent);
+}
+
+/**
+ * pci_msi_domain_alloc_irqs - Allocate interrupts for @dev in @domain
+ * @domain: The interrupt domain to allocate from
+ * @dev: The device for which to allocate
+ * @nvec: The number of interrupts to allocate
+ * @type: Unused to allow simpler migration from the arch_XXX interfaces
+ *
+ * Returns:
+ * A virtual interrupt number or an error code in case of failure
+ */
+int pci_msi_domain_alloc_irqs(struct irq_domain *domain, struct pci_dev *dev,
+ int nvec, int type)
+{
+ return msi_domain_alloc_irqs(domain, &dev->dev, nvec);
+}
+
+/**
+ * pci_msi_domain_free_irqs - Free interrupts for @dev in @domain
+ * @domain: The interrupt domain
+ * @dev: The device for which to free interrupts
+ */
+void pci_msi_domain_free_irqs(struct irq_domain *domain, struct pci_dev *dev)
+{
+ msi_domain_free_irqs(domain, &dev->dev);
+}
+
+/**
+ * pci_msi_create_default_irq_domain - Create a default MSI interrupt domain
+ * @node: Optional device-tree node of the interrupt controller
+ * @info: MSI domain info
+ * @parent: Parent irq domain
+ *
+ * Returns: A domain pointer or NULL in case of failure. If successful
+ * the default PCI/MSI irqdomain pointer is updated.
+ */
+struct irq_domain *pci_msi_create_default_irq_domain(struct device_node *node,
+ struct msi_domain_info *info, struct irq_domain *parent)
+{
+ struct irq_domain *domain;
+
+ mutex_lock(&pci_msi_domain_lock);
+ if (pci_msi_default_domain) {
+ pr_err("PCI: default irq domain for PCI MSI has already been created.\n");
+ domain = NULL;
+ } else {
+ domain = pci_msi_create_irq_domain(node, info, parent);
+ pci_msi_default_domain = domain;
+ }
+ mutex_unlock(&pci_msi_domain_lock);
+
+ return domain;
+}
+#endif /* CONFIG_PCI_MSI_IRQ_DOMAIN */
projects / linux-drm-fsl-dcu.git / blobdiff