From 343b7258687ecfbb363bfda8833a7cf641aac524 Mon Sep 17 00:00:00 2001 From: Max Gurtovoy Date: Thu, 26 Aug 2021 13:39:08 +0300 Subject: PCI: Add 'override_only' field to struct pci_device_id Add 'override_only' field to struct pci_device_id to be used as part of pci_match_device(). When set, a driver only matches the entry when dev->driver_override is set to that driver. In addition, add a helper macro named 'PCI_DEVICE_DRIVER_OVERRIDE' to enable setting some data on it. Next patch from this series will use the above functionality. Signed-off-by: Max Gurtovoy Signed-off-by: Jason Gunthorpe Acked-by: Bjorn Helgaas Signed-off-by: Yishai Hadas Link: https://lore.kernel.org/r/20210826103912.128972-10-yishaih@nvidia.com Signed-off-by: Alex Williamson --- include/linux/mod_devicetable.h | 2 ++ include/linux/pci.h | 15 +++++++++++++++ 2 files changed, 17 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mod_devicetable.h b/include/linux/mod_devicetable.h index 8e291cfdaf06..2e3ba6d9ece0 100644 --- a/include/linux/mod_devicetable.h +++ b/include/linux/mod_devicetable.h @@ -34,12 +34,14 @@ typedef unsigned long kernel_ulong_t; * Best practice is to use driver_data as an index * into a static list of equivalent device types, * instead of using it as a pointer. + * @override_only: Match only when dev->driver_override is this driver. */ struct pci_device_id { __u32 vendor, device; /* Vendor and device ID or PCI_ANY_ID*/ __u32 subvendor, subdevice; /* Subsystem ID's or PCI_ANY_ID */ __u32 class, class_mask; /* (class,subclass,prog-if) triplet */ kernel_ulong_t driver_data; /* Data private to the driver */ + __u32 override_only; }; diff --git a/include/linux/pci.h b/include/linux/pci.h index 540b377ca8f6..0506b1a8c921 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -901,6 +901,21 @@ struct pci_driver { .vendor = (vend), .device = (dev), \ .subvendor = PCI_ANY_ID, .subdevice = PCI_ANY_ID +/** + * PCI_DEVICE_DRIVER_OVERRIDE - macro used to describe a PCI device with + * override_only flags. + * @vend: the 16 bit PCI Vendor ID + * @dev: the 16 bit PCI Device ID + * @driver_override: the 32 bit PCI Device override_only + * + * This macro is used to create a struct pci_device_id that matches only a + * driver_override device. The subvendor and subdevice fields will be set to + * PCI_ANY_ID. + */ +#define PCI_DEVICE_DRIVER_OVERRIDE(vend, dev, driver_override) \ + .vendor = (vend), .device = (dev), .subvendor = PCI_ANY_ID, \ + .subdevice = PCI_ANY_ID, .override_only = (driver_override) + /** * PCI_DEVICE_SUB - macro used to describe a specific PCI device with subsystem * @vend: the 16 bit PCI Vendor ID -- cgit From cc6711b0bf36de068b10490198d05ac168377989 Mon Sep 17 00:00:00 2001 From: Max Gurtovoy Date: Thu, 26 Aug 2021 13:39:09 +0300 Subject: PCI / VFIO: Add 'override_only' support for VFIO PCI sub system Expose an 'override_only' helper macro (i.e. PCI_DRIVER_OVERRIDE_DEVICE_VFIO) for VFIO PCI sub system and add the required code to prefix its matching entries with "vfio_" in modules.alias file. It allows VFIO device drivers to include match entries in the modules.alias file produced by kbuild that are not used for normal driver autoprobing and module autoloading. Drivers using these match entries can be connected to the PCI device manually, by userspace, using the existing driver_override sysfs. For example the resulting modules.alias may have: alias pci:v000015B3d00001021sv*sd*bc*sc*i* mlx5_core alias vfio_pci:v000015B3d00001021sv*sd*bc*sc*i* mlx5_vfio_pci alias vfio_pci:v*d*sv*sd*bc*sc*i* vfio_pci In this example mlx5_core and mlx5_vfio_pci match to the same PCI device. The kernel will autoload and autobind to mlx5_core but the kernel and udev mechanisms will ignore mlx5_vfio_pci. When userspace wants to change a device to the VFIO subsystem it can implement a generic algorithm: 1) Identify the sysfs path to the device: /sys/devices/pci0000:00/0000:00:01.0/0000:01:00.0 2) Get the modalias string from the kernel: $ cat /sys/bus/pci/devices/0000:01:00.0/modalias pci:v000015B3d00001021sv000015B3sd00000001bc02sc00i00 3) Prefix it with vfio_: vfio_pci:v000015B3d00001021sv000015B3sd00000001bc02sc00i00 4) Search modules.alias for the above string and select the entry that has the fewest *'s: alias vfio_pci:v000015B3d00001021sv*sd*bc*sc*i* mlx5_vfio_pci 5) modprobe the matched module name: $ modprobe mlx5_vfio_pci 6) cat the matched module name to driver_override: echo mlx5_vfio_pci > /sys/bus/pci/devices/0000:01:00.0/driver_override 7) unbind device from original module echo 0000:01:00.0 > /sys/bus/pci/devices/0000:01:00.0/driver/unbind 8) probe PCI drivers (or explicitly bind to mlx5_vfio_pci) echo 0000:01:00.0 > /sys/bus/pci/drivers_probe The algorithm is independent of bus type. In future the other buses with VFIO device drivers, like platform and ACPI, can use this algorithm as well. This patch is the infrastructure to provide the information in the modules.alias to userspace. Convert the only VFIO pci_driver which results in one new line in the modules.alias: alias vfio_pci:v*d*sv*sd*bc*sc*i* vfio_pci Later series introduce additional HW specific VFIO PCI drivers, such as mlx5_vfio_pci. Signed-off-by: Max Gurtovoy Signed-off-by: Jason Gunthorpe Acked-by: Bjorn Helgaas # for pci.h Signed-off-by: Yishai Hadas Link: https://lore.kernel.org/r/20210826103912.128972-11-yishaih@nvidia.com Signed-off-by: Alex Williamson --- drivers/vfio/pci/vfio_pci.c | 9 ++++++++- include/linux/mod_devicetable.h | 4 ++++ include/linux/pci.h | 14 ++++++++++++++ scripts/mod/devicetable-offsets.c | 1 + scripts/mod/file2alias.c | 17 +++++++++++++++-- 5 files changed, 42 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c index 163e560c4495..85fd638a5955 100644 --- a/drivers/vfio/pci/vfio_pci.c +++ b/drivers/vfio/pci/vfio_pci.c @@ -178,9 +178,16 @@ static int vfio_pci_sriov_configure(struct pci_dev *pdev, int nr_virtfn) return vfio_pci_core_sriov_configure(pdev, nr_virtfn); } +static const struct pci_device_id vfio_pci_table[] = { + { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_ANY_ID, PCI_ANY_ID) }, /* match all by default */ + {} +}; + +MODULE_DEVICE_TABLE(pci, vfio_pci_table); + static struct pci_driver vfio_pci_driver = { .name = "vfio-pci", - .id_table = NULL, /* only dynamic ids */ + .id_table = vfio_pci_table, .probe = vfio_pci_probe, .remove = vfio_pci_remove, .sriov_configure = vfio_pci_sriov_configure, diff --git a/include/linux/mod_devicetable.h b/include/linux/mod_devicetable.h index 2e3ba6d9ece0..ae2e75d15b21 100644 --- a/include/linux/mod_devicetable.h +++ b/include/linux/mod_devicetable.h @@ -16,6 +16,10 @@ typedef unsigned long kernel_ulong_t; #define PCI_ANY_ID (~0) +enum { + PCI_ID_F_VFIO_DRIVER_OVERRIDE = 1, +}; + /** * struct pci_device_id - PCI device ID structure * @vendor: Vendor ID to match (or PCI_ANY_ID) diff --git a/include/linux/pci.h b/include/linux/pci.h index 0506b1a8c921..527a1dfd1d06 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -916,6 +916,20 @@ struct pci_driver { .vendor = (vend), .device = (dev), .subvendor = PCI_ANY_ID, \ .subdevice = PCI_ANY_ID, .override_only = (driver_override) +/** + * PCI_DRIVER_OVERRIDE_DEVICE_VFIO - macro used to describe a VFIO + * "driver_override" PCI device. + * @vend: the 16 bit PCI Vendor ID + * @dev: the 16 bit PCI Device ID + * + * This macro is used to create a struct pci_device_id that matches a + * specific device. The subvendor and subdevice fields will be set to + * PCI_ANY_ID and the driver_override will be set to + * PCI_ID_F_VFIO_DRIVER_OVERRIDE. + */ +#define PCI_DRIVER_OVERRIDE_DEVICE_VFIO(vend, dev) \ + PCI_DEVICE_DRIVER_OVERRIDE(vend, dev, PCI_ID_F_VFIO_DRIVER_OVERRIDE) + /** * PCI_DEVICE_SUB - macro used to describe a specific PCI device with subsystem * @vend: the 16 bit PCI Vendor ID diff --git a/scripts/mod/devicetable-offsets.c b/scripts/mod/devicetable-offsets.c index 9bb6c7edccc4..cc3625617a0e 100644 --- a/scripts/mod/devicetable-offsets.c +++ b/scripts/mod/devicetable-offsets.c @@ -42,6 +42,7 @@ int main(void) DEVID_FIELD(pci_device_id, subdevice); DEVID_FIELD(pci_device_id, class); DEVID_FIELD(pci_device_id, class_mask); + DEVID_FIELD(pci_device_id, override_only); DEVID(ccw_device_id); DEVID_FIELD(ccw_device_id, match_flags); diff --git a/scripts/mod/file2alias.c b/scripts/mod/file2alias.c index 7c97fa8e36bc..49aba862073e 100644 --- a/scripts/mod/file2alias.c +++ b/scripts/mod/file2alias.c @@ -426,7 +426,7 @@ static int do_ieee1394_entry(const char *filename, return 1; } -/* Looks like: pci:vNdNsvNsdNbcNscNiN. */ +/* Looks like: pci:vNdNsvNsdNbcNscNiN or _pci:vNdNsvNsdNbcNscNiN. */ static int do_pci_entry(const char *filename, void *symval, char *alias) { @@ -440,8 +440,21 @@ static int do_pci_entry(const char *filename, DEF_FIELD(symval, pci_device_id, subdevice); DEF_FIELD(symval, pci_device_id, class); DEF_FIELD(symval, pci_device_id, class_mask); + DEF_FIELD(symval, pci_device_id, override_only); + + switch (override_only) { + case 0: + strcpy(alias, "pci:"); + break; + case PCI_ID_F_VFIO_DRIVER_OVERRIDE: + strcpy(alias, "vfio_pci:"); + break; + default: + warn("Unknown PCI driver_override alias %08X\n", + override_only); + return 0; + } - strcpy(alias, "pci:"); ADD(alias, "v", vendor != PCI_ANY_ID, vendor); ADD(alias, "d", device != PCI_ANY_ID, device); ADD(alias, "sv", subvendor != PCI_ANY_ID, subvendor); -- cgit From 7fa005caa35ed92563b9e9d88d319b2623763a77 Mon Sep 17 00:00:00 2001 From: Max Gurtovoy Date: Thu, 26 Aug 2021 13:39:12 +0300 Subject: vfio/pci: Introduce vfio_pci_core.ko Now that vfio_pci has been split into two source modules, one focusing on the "struct pci_driver" (vfio_pci.c) and a toolbox library of code (vfio_pci_core.c), complete the split and move them into two different kernel modules. As before vfio_pci.ko continues to present the same interface under sysfs and this change will have no functional impact. Splitting into another module and adding exports allows creating new HW specific VFIO PCI drivers that can implement device specific functionality, such as VFIO migration interfaces or specialized device requirements. Signed-off-by: Max Gurtovoy Signed-off-by: Jason Gunthorpe Reviewed-by: Christoph Hellwig Signed-off-by: Yishai Hadas Link: https://lore.kernel.org/r/20210826103912.128972-14-yishaih@nvidia.com Signed-off-by: Alex Williamson --- MAINTAINERS | 1 + drivers/vfio/pci/Kconfig | 33 ++--- drivers/vfio/pci/Makefile | 8 +- drivers/vfio/pci/vfio_pci.c | 14 +-- drivers/vfio/pci/vfio_pci_config.c | 2 +- drivers/vfio/pci/vfio_pci_core.c | 39 +++++- drivers/vfio/pci/vfio_pci_core.h | 241 ------------------------------------- drivers/vfio/pci/vfio_pci_igd.c | 2 +- drivers/vfio/pci/vfio_pci_intrs.c | 2 +- drivers/vfio/pci/vfio_pci_rdwr.c | 2 +- drivers/vfio/pci/vfio_pci_zdev.c | 2 +- include/linux/vfio_pci_core.h | 239 ++++++++++++++++++++++++++++++++++++ 12 files changed, 304 insertions(+), 281 deletions(-) delete mode 100644 drivers/vfio/pci/vfio_pci_core.h create mode 100644 include/linux/vfio_pci_core.h (limited to 'include/linux') diff --git a/MAINTAINERS b/MAINTAINERS index c9467d2839f5..7f0fcaa8ee67 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -19466,6 +19466,7 @@ T: git git://github.com/awilliam/linux-vfio.git F: Documentation/driver-api/vfio.rst F: drivers/vfio/ F: include/linux/vfio.h +F: include/linux/vfio_pci_core.h F: include/uapi/linux/vfio.h VFIO FSL-MC DRIVER diff --git a/drivers/vfio/pci/Kconfig b/drivers/vfio/pci/Kconfig index afdab7d71e98..860424ccda1b 100644 --- a/drivers/vfio/pci/Kconfig +++ b/drivers/vfio/pci/Kconfig @@ -1,19 +1,28 @@ # SPDX-License-Identifier: GPL-2.0-only -config VFIO_PCI - tristate "VFIO support for PCI devices" - depends on PCI - depends on MMU +if PCI && MMU +config VFIO_PCI_CORE + tristate select VFIO_VIRQFD select IRQ_BYPASS_MANAGER + +config VFIO_PCI_MMAP + def_bool y if !S390 + +config VFIO_PCI_INTX + def_bool y if !S390 + +config VFIO_PCI + tristate "Generic VFIO support for any PCI device" + select VFIO_PCI_CORE help - Support for the PCI VFIO bus driver. This is required to make - use of PCI drivers using the VFIO framework. + Support for the generic PCI VFIO bus driver which can connect any + PCI device to the VFIO framework. If you don't know what to do here, say N. if VFIO_PCI config VFIO_PCI_VGA - bool "VFIO PCI support for VGA devices" + bool "Generic VFIO PCI support for VGA devices" depends on X86 && VGA_ARB help Support for VGA extension to VFIO PCI. This exposes an additional @@ -22,14 +31,8 @@ config VFIO_PCI_VGA If you don't know what to do here, say N. -config VFIO_PCI_MMAP - def_bool y if !S390 - -config VFIO_PCI_INTX - def_bool y if !S390 - config VFIO_PCI_IGD - bool "VFIO PCI extensions for Intel graphics (GVT-d)" + bool "Generic VFIO PCI extensions for Intel graphics (GVT-d)" depends on X86 default y help @@ -39,5 +42,5 @@ config VFIO_PCI_IGD and LPC bridge config space. To enable Intel IGD assignment through vfio-pci, say Y. - +endif endif diff --git a/drivers/vfio/pci/Makefile b/drivers/vfio/pci/Makefile index 8aa517b4b671..349d68d242b4 100644 --- a/drivers/vfio/pci/Makefile +++ b/drivers/vfio/pci/Makefile @@ -1,7 +1,9 @@ # SPDX-License-Identifier: GPL-2.0-only -vfio-pci-y := vfio_pci.o vfio_pci_core.o vfio_pci_intrs.o vfio_pci_rdwr.o vfio_pci_config.o -vfio-pci-$(CONFIG_VFIO_PCI_IGD) += vfio_pci_igd.o -vfio-pci-$(CONFIG_S390) += vfio_pci_zdev.o +vfio-pci-core-y := vfio_pci_core.o vfio_pci_intrs.o vfio_pci_rdwr.o vfio_pci_config.o +vfio-pci-core-$(CONFIG_S390) += vfio_pci_zdev.o +obj-$(CONFIG_VFIO_PCI_CORE) += vfio-pci-core.o +vfio-pci-y := vfio_pci.o +vfio-pci-$(CONFIG_VFIO_PCI_IGD) += vfio_pci_igd.o obj-$(CONFIG_VFIO_PCI) += vfio-pci.o diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c index 85fd638a5955..a5ce92beb655 100644 --- a/drivers/vfio/pci/vfio_pci.c +++ b/drivers/vfio/pci/vfio_pci.c @@ -25,7 +25,7 @@ #include #include -#include "vfio_pci_core.h" +#include #define DRIVER_AUTHOR "Alex Williamson " #define DRIVER_DESC "VFIO PCI - User Level meta-driver" @@ -153,6 +153,7 @@ static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) ret = vfio_pci_core_register_device(vdev); if (ret) goto out_free; + dev_set_drvdata(&pdev->dev, vdev); return 0; out_free: @@ -246,14 +247,10 @@ static int __init vfio_pci_init(void) vfio_pci_core_set_params(nointxmask, is_disable_vga, disable_idle_d3); - ret = vfio_pci_core_init(); - if (ret) - return ret; - /* Register and scan for devices */ ret = pci_register_driver(&vfio_pci_driver); if (ret) - goto out; + return ret; vfio_pci_fill_ids(); @@ -261,17 +258,12 @@ static int __init vfio_pci_init(void) pr_warn("device denylist disabled.\n"); return 0; - -out: - vfio_pci_core_cleanup(); - return ret; } module_init(vfio_pci_init); static void __exit vfio_pci_cleanup(void) { pci_unregister_driver(&vfio_pci_driver); - vfio_pci_core_cleanup(); } module_exit(vfio_pci_cleanup); diff --git a/drivers/vfio/pci/vfio_pci_config.c b/drivers/vfio/pci/vfio_pci_config.c index 1f034f768a27..6e58b4bf7a60 100644 --- a/drivers/vfio/pci/vfio_pci_config.c +++ b/drivers/vfio/pci/vfio_pci_config.c @@ -26,7 +26,7 @@ #include #include -#include "vfio_pci_core.h" +#include /* Fake capability ID for standard config space */ #define PCI_CAP_ID_BASIC 0 diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c index 65eafaafb2e0..675616e08897 100644 --- a/drivers/vfio/pci/vfio_pci_core.c +++ b/drivers/vfio/pci/vfio_pci_core.c @@ -8,6 +8,8 @@ * Author: Tom Lyon, pugs@cisco.com */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include #include #include @@ -25,7 +27,10 @@ #include #include -#include "vfio_pci_core.h" +#include + +#define DRIVER_AUTHOR "Alex Williamson " +#define DRIVER_DESC "core driver for VFIO based PCI devices" static bool nointxmask; static bool disable_vga; @@ -306,6 +311,7 @@ int vfio_pci_core_enable(struct vfio_pci_core_device *vdev) return 0; } +EXPORT_SYMBOL_GPL(vfio_pci_core_enable); void vfio_pci_core_disable(struct vfio_pci_core_device *vdev) { @@ -403,6 +409,7 @@ out: if (!vfio_pci_dev_set_try_reset(vdev->vdev.dev_set) && !disable_idle_d3) vfio_pci_set_power_state(vdev, PCI_D3hot); } +EXPORT_SYMBOL_GPL(vfio_pci_core_disable); static struct vfio_pci_core_device *get_pf_vdev(struct vfio_pci_core_device *vdev) { @@ -459,6 +466,7 @@ void vfio_pci_core_close_device(struct vfio_device *core_vdev) } mutex_unlock(&vdev->igate); } +EXPORT_SYMBOL_GPL(vfio_pci_core_close_device); void vfio_pci_core_finish_enable(struct vfio_pci_core_device *vdev) { @@ -466,6 +474,7 @@ void vfio_pci_core_finish_enable(struct vfio_pci_core_device *vdev) vfio_spapr_pci_eeh_open(vdev->pdev); vfio_pci_vf_token_user_add(vdev, 1); } +EXPORT_SYMBOL_GPL(vfio_pci_core_finish_enable); static int vfio_pci_get_irq_count(struct vfio_pci_core_device *vdev, int irq_type) { @@ -624,6 +633,7 @@ int vfio_pci_register_dev_region(struct vfio_pci_core_device *vdev, return 0; } +EXPORT_SYMBOL_GPL(vfio_pci_register_dev_region); long vfio_pci_core_ioctl(struct vfio_device *core_vdev, unsigned int cmd, unsigned long arg) @@ -1168,6 +1178,7 @@ hot_reset_release: return -ENOTTY; } +EXPORT_SYMBOL_GPL(vfio_pci_core_ioctl); static ssize_t vfio_pci_rw(struct vfio_pci_core_device *vdev, char __user *buf, size_t count, loff_t *ppos, bool iswrite) @@ -1211,6 +1222,7 @@ ssize_t vfio_pci_core_read(struct vfio_device *core_vdev, char __user *buf, return vfio_pci_rw(vdev, buf, count, ppos, false); } +EXPORT_SYMBOL_GPL(vfio_pci_core_read); ssize_t vfio_pci_core_write(struct vfio_device *core_vdev, const char __user *buf, size_t count, loff_t *ppos) @@ -1223,6 +1235,7 @@ ssize_t vfio_pci_core_write(struct vfio_device *core_vdev, const char __user *bu return vfio_pci_rw(vdev, (char __user *)buf, count, ppos, true); } +EXPORT_SYMBOL_GPL(vfio_pci_core_write); /* Return 1 on zap and vma_lock acquired, 0 on contention (only with @try) */ static int vfio_pci_zap_and_vma_lock(struct vfio_pci_core_device *vdev, bool try) @@ -1501,6 +1514,7 @@ int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma return 0; } +EXPORT_SYMBOL_GPL(vfio_pci_core_mmap); void vfio_pci_core_request(struct vfio_device *core_vdev, unsigned int count) { @@ -1523,6 +1537,7 @@ void vfio_pci_core_request(struct vfio_device *core_vdev, unsigned int count) mutex_unlock(&vdev->igate); } +EXPORT_SYMBOL_GPL(vfio_pci_core_request); static int vfio_pci_validate_vf_token(struct vfio_pci_core_device *vdev, bool vf_token, uuid_t *uuid) @@ -1667,6 +1682,7 @@ int vfio_pci_core_match(struct vfio_device *core_vdev, char *buf) return 1; /* Match */ } +EXPORT_SYMBOL_GPL(vfio_pci_core_match); static int vfio_pci_bus_notifier(struct notifier_block *nb, unsigned long action, void *data) @@ -1775,6 +1791,7 @@ void vfio_pci_core_init_device(struct vfio_pci_core_device *vdev, INIT_LIST_HEAD(&vdev->vma_list); init_rwsem(&vdev->memory_lock); } +EXPORT_SYMBOL_GPL(vfio_pci_core_init_device); void vfio_pci_core_uninit_device(struct vfio_pci_core_device *vdev) { @@ -1785,6 +1802,7 @@ void vfio_pci_core_uninit_device(struct vfio_pci_core_device *vdev) kfree(vdev->region); kfree(vdev->pm_save); } +EXPORT_SYMBOL_GPL(vfio_pci_core_uninit_device); int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev) { @@ -1852,7 +1870,6 @@ int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev) ret = vfio_register_group_dev(&vdev->vdev); if (ret) goto out_power; - dev_set_drvdata(&pdev->dev, vdev); return 0; out_power: @@ -1864,6 +1881,7 @@ out_group_put: vfio_iommu_group_put(group, &pdev->dev); return ret; } +EXPORT_SYMBOL_GPL(vfio_pci_core_register_device); void vfio_pci_core_unregister_device(struct vfio_pci_core_device *vdev) { @@ -1881,6 +1899,7 @@ void vfio_pci_core_unregister_device(struct vfio_pci_core_device *vdev) if (!disable_idle_d3) vfio_pci_set_power_state(vdev, PCI_D0); } +EXPORT_SYMBOL_GPL(vfio_pci_core_unregister_device); static pci_ers_result_t vfio_pci_aer_err_detected(struct pci_dev *pdev, pci_channel_state_t state) @@ -1924,10 +1943,12 @@ int vfio_pci_core_sriov_configure(struct pci_dev *pdev, int nr_virtfn) return ret < 0 ? ret : nr_virtfn; } +EXPORT_SYMBOL_GPL(vfio_pci_core_sriov_configure); const struct pci_error_handlers vfio_pci_core_err_handlers = { .error_detected = vfio_pci_aer_err_detected, }; +EXPORT_SYMBOL_GPL(vfio_pci_core_err_handlers); static bool vfio_dev_in_groups(struct vfio_pci_core_device *vdev, struct vfio_pci_group_info *groups) @@ -2116,16 +2137,22 @@ void vfio_pci_core_set_params(bool is_nointxmask, bool is_disable_vga, disable_vga = is_disable_vga; disable_idle_d3 = is_disable_idle_d3; } +EXPORT_SYMBOL_GPL(vfio_pci_core_set_params); -/* This will become the __exit function of vfio_pci_core.ko */ -void vfio_pci_core_cleanup(void) +static void vfio_pci_core_cleanup(void) { vfio_pci_uninit_perm_bits(); } -/* This will become the __init function of vfio_pci_core.ko */ -int __init vfio_pci_core_init(void) +static int __init vfio_pci_core_init(void) { /* Allocate shared config space permission data used by all devices */ return vfio_pci_init_perm_bits(); } + +module_init(vfio_pci_core_init); +module_exit(vfio_pci_core_cleanup); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR(DRIVER_AUTHOR); +MODULE_DESCRIPTION(DRIVER_DESC); diff --git a/drivers/vfio/pci/vfio_pci_core.h b/drivers/vfio/pci/vfio_pci_core.h deleted file mode 100644 index 7a2da1e14de3..000000000000 --- a/drivers/vfio/pci/vfio_pci_core.h +++ /dev/null @@ -1,241 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2012 Red Hat, Inc. All rights reserved. - * Author: Alex Williamson - * - * Derived from original vfio: - * Copyright 2010 Cisco Systems, Inc. All rights reserved. - * Author: Tom Lyon, pugs@cisco.com - */ - -#include -#include -#include -#include -#include -#include -#include - -#ifndef VFIO_PCI_CORE_H -#define VFIO_PCI_CORE_H - -#define VFIO_PCI_OFFSET_SHIFT 40 - -#define VFIO_PCI_OFFSET_TO_INDEX(off) (off >> VFIO_PCI_OFFSET_SHIFT) -#define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT) -#define VFIO_PCI_OFFSET_MASK (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1) - -/* Special capability IDs predefined access */ -#define PCI_CAP_ID_INVALID 0xFF /* default raw access */ -#define PCI_CAP_ID_INVALID_VIRT 0xFE /* default virt access */ - -/* Cap maximum number of ioeventfds per device (arbitrary) */ -#define VFIO_PCI_IOEVENTFD_MAX 1000 - -struct vfio_pci_ioeventfd { - struct list_head next; - struct vfio_pci_core_device *vdev; - struct virqfd *virqfd; - void __iomem *addr; - uint64_t data; - loff_t pos; - int bar; - int count; - bool test_mem; -}; - -struct vfio_pci_irq_ctx { - struct eventfd_ctx *trigger; - struct virqfd *unmask; - struct virqfd *mask; - char *name; - bool masked; - struct irq_bypass_producer producer; -}; - -struct vfio_pci_core_device; -struct vfio_pci_region; - -struct vfio_pci_regops { - ssize_t (*rw)(struct vfio_pci_core_device *vdev, char __user *buf, - size_t count, loff_t *ppos, bool iswrite); - void (*release)(struct vfio_pci_core_device *vdev, - struct vfio_pci_region *region); - int (*mmap)(struct vfio_pci_core_device *vdev, - struct vfio_pci_region *region, - struct vm_area_struct *vma); - int (*add_capability)(struct vfio_pci_core_device *vdev, - struct vfio_pci_region *region, - struct vfio_info_cap *caps); -}; - -struct vfio_pci_region { - u32 type; - u32 subtype; - const struct vfio_pci_regops *ops; - void *data; - size_t size; - u32 flags; -}; - -struct vfio_pci_dummy_resource { - struct resource resource; - int index; - struct list_head res_next; -}; - -struct vfio_pci_vf_token { - struct mutex lock; - uuid_t uuid; - int users; -}; - -struct vfio_pci_mmap_vma { - struct vm_area_struct *vma; - struct list_head vma_next; -}; - -struct vfio_pci_core_device { - struct vfio_device vdev; - struct pci_dev *pdev; - void __iomem *barmap[PCI_STD_NUM_BARS]; - bool bar_mmap_supported[PCI_STD_NUM_BARS]; - u8 *pci_config_map; - u8 *vconfig; - struct perm_bits *msi_perm; - spinlock_t irqlock; - struct mutex igate; - struct vfio_pci_irq_ctx *ctx; - int num_ctx; - int irq_type; - int num_regions; - struct vfio_pci_region *region; - u8 msi_qmax; - u8 msix_bar; - u16 msix_size; - u32 msix_offset; - u32 rbar[7]; - bool pci_2_3; - bool virq_disabled; - bool reset_works; - bool extended_caps; - bool bardirty; - bool has_vga; - bool needs_reset; - bool nointx; - bool needs_pm_restore; - struct pci_saved_state *pci_saved_state; - struct pci_saved_state *pm_save; - int ioeventfds_nr; - struct eventfd_ctx *err_trigger; - struct eventfd_ctx *req_trigger; - struct list_head dummy_resources_list; - struct mutex ioeventfds_lock; - struct list_head ioeventfds_list; - struct vfio_pci_vf_token *vf_token; - struct notifier_block nb; - struct mutex vma_lock; - struct list_head vma_list; - struct rw_semaphore memory_lock; -}; - -#define is_intx(vdev) (vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX) -#define is_msi(vdev) (vdev->irq_type == VFIO_PCI_MSI_IRQ_INDEX) -#define is_msix(vdev) (vdev->irq_type == VFIO_PCI_MSIX_IRQ_INDEX) -#define is_irq_none(vdev) (!(is_intx(vdev) || is_msi(vdev) || is_msix(vdev))) -#define irq_is(vdev, type) (vdev->irq_type == type) - -extern void vfio_pci_intx_mask(struct vfio_pci_core_device *vdev); -extern void vfio_pci_intx_unmask(struct vfio_pci_core_device *vdev); - -extern int vfio_pci_set_irqs_ioctl(struct vfio_pci_core_device *vdev, - uint32_t flags, unsigned index, - unsigned start, unsigned count, void *data); - -extern ssize_t vfio_pci_config_rw(struct vfio_pci_core_device *vdev, - char __user *buf, size_t count, - loff_t *ppos, bool iswrite); - -extern ssize_t vfio_pci_bar_rw(struct vfio_pci_core_device *vdev, char __user *buf, - size_t count, loff_t *ppos, bool iswrite); - -extern ssize_t vfio_pci_vga_rw(struct vfio_pci_core_device *vdev, char __user *buf, - size_t count, loff_t *ppos, bool iswrite); - -extern long vfio_pci_ioeventfd(struct vfio_pci_core_device *vdev, loff_t offset, - uint64_t data, int count, int fd); - -extern int vfio_pci_init_perm_bits(void); -extern void vfio_pci_uninit_perm_bits(void); - -extern int vfio_config_init(struct vfio_pci_core_device *vdev); -extern void vfio_config_free(struct vfio_pci_core_device *vdev); - -extern int vfio_pci_register_dev_region(struct vfio_pci_core_device *vdev, - unsigned int type, unsigned int subtype, - const struct vfio_pci_regops *ops, - size_t size, u32 flags, void *data); - -extern int vfio_pci_set_power_state(struct vfio_pci_core_device *vdev, - pci_power_t state); - -extern bool __vfio_pci_memory_enabled(struct vfio_pci_core_device *vdev); -extern void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_core_device - *vdev); -extern u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_core_device *vdev); -extern void vfio_pci_memory_unlock_and_restore(struct vfio_pci_core_device *vdev, - u16 cmd); - -#ifdef CONFIG_VFIO_PCI_IGD -extern int vfio_pci_igd_init(struct vfio_pci_core_device *vdev); -#else -static inline int vfio_pci_igd_init(struct vfio_pci_core_device *vdev) -{ - return -ENODEV; -} -#endif - -#ifdef CONFIG_S390 -extern int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev, - struct vfio_info_cap *caps); -#else -static inline int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev, - struct vfio_info_cap *caps) -{ - return -ENODEV; -} -#endif - -/* Will be exported for vfio pci drivers usage */ -void vfio_pci_core_cleanup(void); -int vfio_pci_core_init(void); -void vfio_pci_core_set_params(bool nointxmask, bool is_disable_vga, - bool is_disable_idle_d3); -void vfio_pci_core_close_device(struct vfio_device *core_vdev); -void vfio_pci_core_init_device(struct vfio_pci_core_device *vdev, - struct pci_dev *pdev, - const struct vfio_device_ops *vfio_pci_ops); -int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev); -void vfio_pci_core_uninit_device(struct vfio_pci_core_device *vdev); -void vfio_pci_core_unregister_device(struct vfio_pci_core_device *vdev); -int vfio_pci_core_sriov_configure(struct pci_dev *pdev, int nr_virtfn); -extern const struct pci_error_handlers vfio_pci_core_err_handlers; -long vfio_pci_core_ioctl(struct vfio_device *core_vdev, unsigned int cmd, - unsigned long arg); -ssize_t vfio_pci_core_read(struct vfio_device *core_vdev, char __user *buf, - size_t count, loff_t *ppos); -ssize_t vfio_pci_core_write(struct vfio_device *core_vdev, const char __user *buf, - size_t count, loff_t *ppos); -int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma); -void vfio_pci_core_request(struct vfio_device *core_vdev, unsigned int count); -int vfio_pci_core_match(struct vfio_device *core_vdev, char *buf); -int vfio_pci_core_enable(struct vfio_pci_core_device *vdev); -void vfio_pci_core_disable(struct vfio_pci_core_device *vdev); -void vfio_pci_core_finish_enable(struct vfio_pci_core_device *vdev); - -static inline bool vfio_pci_is_vga(struct pci_dev *pdev) -{ - return (pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA; -} - -#endif /* VFIO_PCI_CORE_H */ diff --git a/drivers/vfio/pci/vfio_pci_igd.c b/drivers/vfio/pci/vfio_pci_igd.c index a324ca7e6b5a..7ca4109bba48 100644 --- a/drivers/vfio/pci/vfio_pci_igd.c +++ b/drivers/vfio/pci/vfio_pci_igd.c @@ -15,7 +15,7 @@ #include #include -#include "vfio_pci_core.h" +#include #define OPREGION_SIGNATURE "IntelGraphicsMem" #define OPREGION_SIZE (8 * 1024) diff --git a/drivers/vfio/pci/vfio_pci_intrs.c b/drivers/vfio/pci/vfio_pci_intrs.c index 945ddbdf4d11..6069a11fb51a 100644 --- a/drivers/vfio/pci/vfio_pci_intrs.c +++ b/drivers/vfio/pci/vfio_pci_intrs.c @@ -20,7 +20,7 @@ #include #include -#include "vfio_pci_core.h" +#include /* * INTx diff --git a/drivers/vfio/pci/vfio_pci_rdwr.c b/drivers/vfio/pci/vfio_pci_rdwr.c index 8fff4689dd44..57d3b2cbbd8e 100644 --- a/drivers/vfio/pci/vfio_pci_rdwr.c +++ b/drivers/vfio/pci/vfio_pci_rdwr.c @@ -17,7 +17,7 @@ #include #include -#include "vfio_pci_core.h" +#include #ifdef __LITTLE_ENDIAN #define vfio_ioread64 ioread64 diff --git a/drivers/vfio/pci/vfio_pci_zdev.c b/drivers/vfio/pci/vfio_pci_zdev.c index 2ffbdc11f089..fe4def9ffffb 100644 --- a/drivers/vfio/pci/vfio_pci_zdev.c +++ b/drivers/vfio/pci/vfio_pci_zdev.c @@ -19,7 +19,7 @@ #include #include -#include "vfio_pci_core.h" +#include /* * Add the Base PCI Function information to the device info region. diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h new file mode 100644 index 000000000000..ef9a44b6cf5d --- /dev/null +++ b/include/linux/vfio_pci_core.h @@ -0,0 +1,239 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2012 Red Hat, Inc. All rights reserved. + * Author: Alex Williamson + * + * Derived from original vfio: + * Copyright 2010 Cisco Systems, Inc. All rights reserved. + * Author: Tom Lyon, pugs@cisco.com + */ + +#include +#include +#include +#include +#include +#include +#include + +#ifndef VFIO_PCI_CORE_H +#define VFIO_PCI_CORE_H + +#define VFIO_PCI_OFFSET_SHIFT 40 + +#define VFIO_PCI_OFFSET_TO_INDEX(off) (off >> VFIO_PCI_OFFSET_SHIFT) +#define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT) +#define VFIO_PCI_OFFSET_MASK (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1) + +/* Special capability IDs predefined access */ +#define PCI_CAP_ID_INVALID 0xFF /* default raw access */ +#define PCI_CAP_ID_INVALID_VIRT 0xFE /* default virt access */ + +/* Cap maximum number of ioeventfds per device (arbitrary) */ +#define VFIO_PCI_IOEVENTFD_MAX 1000 + +struct vfio_pci_ioeventfd { + struct list_head next; + struct vfio_pci_core_device *vdev; + struct virqfd *virqfd; + void __iomem *addr; + uint64_t data; + loff_t pos; + int bar; + int count; + bool test_mem; +}; + +struct vfio_pci_irq_ctx { + struct eventfd_ctx *trigger; + struct virqfd *unmask; + struct virqfd *mask; + char *name; + bool masked; + struct irq_bypass_producer producer; +}; + +struct vfio_pci_core_device; +struct vfio_pci_region; + +struct vfio_pci_regops { + ssize_t (*rw)(struct vfio_pci_core_device *vdev, char __user *buf, + size_t count, loff_t *ppos, bool iswrite); + void (*release)(struct vfio_pci_core_device *vdev, + struct vfio_pci_region *region); + int (*mmap)(struct vfio_pci_core_device *vdev, + struct vfio_pci_region *region, + struct vm_area_struct *vma); + int (*add_capability)(struct vfio_pci_core_device *vdev, + struct vfio_pci_region *region, + struct vfio_info_cap *caps); +}; + +struct vfio_pci_region { + u32 type; + u32 subtype; + const struct vfio_pci_regops *ops; + void *data; + size_t size; + u32 flags; +}; + +struct vfio_pci_dummy_resource { + struct resource resource; + int index; + struct list_head res_next; +}; + +struct vfio_pci_vf_token { + struct mutex lock; + uuid_t uuid; + int users; +}; + +struct vfio_pci_mmap_vma { + struct vm_area_struct *vma; + struct list_head vma_next; +}; + +struct vfio_pci_core_device { + struct vfio_device vdev; + struct pci_dev *pdev; + void __iomem *barmap[PCI_STD_NUM_BARS]; + bool bar_mmap_supported[PCI_STD_NUM_BARS]; + u8 *pci_config_map; + u8 *vconfig; + struct perm_bits *msi_perm; + spinlock_t irqlock; + struct mutex igate; + struct vfio_pci_irq_ctx *ctx; + int num_ctx; + int irq_type; + int num_regions; + struct vfio_pci_region *region; + u8 msi_qmax; + u8 msix_bar; + u16 msix_size; + u32 msix_offset; + u32 rbar[7]; + bool pci_2_3; + bool virq_disabled; + bool reset_works; + bool extended_caps; + bool bardirty; + bool has_vga; + bool needs_reset; + bool nointx; + bool needs_pm_restore; + struct pci_saved_state *pci_saved_state; + struct pci_saved_state *pm_save; + int ioeventfds_nr; + struct eventfd_ctx *err_trigger; + struct eventfd_ctx *req_trigger; + struct list_head dummy_resources_list; + struct mutex ioeventfds_lock; + struct list_head ioeventfds_list; + struct vfio_pci_vf_token *vf_token; + struct notifier_block nb; + struct mutex vma_lock; + struct list_head vma_list; + struct rw_semaphore memory_lock; +}; + +#define is_intx(vdev) (vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX) +#define is_msi(vdev) (vdev->irq_type == VFIO_PCI_MSI_IRQ_INDEX) +#define is_msix(vdev) (vdev->irq_type == VFIO_PCI_MSIX_IRQ_INDEX) +#define is_irq_none(vdev) (!(is_intx(vdev) || is_msi(vdev) || is_msix(vdev))) +#define irq_is(vdev, type) (vdev->irq_type == type) + +extern void vfio_pci_intx_mask(struct vfio_pci_core_device *vdev); +extern void vfio_pci_intx_unmask(struct vfio_pci_core_device *vdev); + +extern int vfio_pci_set_irqs_ioctl(struct vfio_pci_core_device *vdev, + uint32_t flags, unsigned index, + unsigned start, unsigned count, void *data); + +extern ssize_t vfio_pci_config_rw(struct vfio_pci_core_device *vdev, + char __user *buf, size_t count, + loff_t *ppos, bool iswrite); + +extern ssize_t vfio_pci_bar_rw(struct vfio_pci_core_device *vdev, char __user *buf, + size_t count, loff_t *ppos, bool iswrite); + +extern ssize_t vfio_pci_vga_rw(struct vfio_pci_core_device *vdev, char __user *buf, + size_t count, loff_t *ppos, bool iswrite); + +extern long vfio_pci_ioeventfd(struct vfio_pci_core_device *vdev, loff_t offset, + uint64_t data, int count, int fd); + +extern int vfio_pci_init_perm_bits(void); +extern void vfio_pci_uninit_perm_bits(void); + +extern int vfio_config_init(struct vfio_pci_core_device *vdev); +extern void vfio_config_free(struct vfio_pci_core_device *vdev); + +extern int vfio_pci_register_dev_region(struct vfio_pci_core_device *vdev, + unsigned int type, unsigned int subtype, + const struct vfio_pci_regops *ops, + size_t size, u32 flags, void *data); + +extern int vfio_pci_set_power_state(struct vfio_pci_core_device *vdev, + pci_power_t state); + +extern bool __vfio_pci_memory_enabled(struct vfio_pci_core_device *vdev); +extern void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_core_device + *vdev); +extern u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_core_device *vdev); +extern void vfio_pci_memory_unlock_and_restore(struct vfio_pci_core_device *vdev, + u16 cmd); + +#ifdef CONFIG_VFIO_PCI_IGD +extern int vfio_pci_igd_init(struct vfio_pci_core_device *vdev); +#else +static inline int vfio_pci_igd_init(struct vfio_pci_core_device *vdev) +{ + return -ENODEV; +} +#endif + +#ifdef CONFIG_S390 +extern int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev, + struct vfio_info_cap *caps); +#else +static inline int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev, + struct vfio_info_cap *caps) +{ + return -ENODEV; +} +#endif + +/* Will be exported for vfio pci drivers usage */ +void vfio_pci_core_set_params(bool nointxmask, bool is_disable_vga, + bool is_disable_idle_d3); +void vfio_pci_core_close_device(struct vfio_device *core_vdev); +void vfio_pci_core_init_device(struct vfio_pci_core_device *vdev, + struct pci_dev *pdev, + const struct vfio_device_ops *vfio_pci_ops); +int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev); +void vfio_pci_core_uninit_device(struct vfio_pci_core_device *vdev); +void vfio_pci_core_unregister_device(struct vfio_pci_core_device *vdev); +int vfio_pci_core_sriov_configure(struct pci_dev *pdev, int nr_virtfn); +extern const struct pci_error_handlers vfio_pci_core_err_handlers; +long vfio_pci_core_ioctl(struct vfio_device *core_vdev, unsigned int cmd, + unsigned long arg); +ssize_t vfio_pci_core_read(struct vfio_device *core_vdev, char __user *buf, + size_t count, loff_t *ppos); +ssize_t vfio_pci_core_write(struct vfio_device *core_vdev, const char __user *buf, + size_t count, loff_t *ppos); +int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma); +void vfio_pci_core_request(struct vfio_device *core_vdev, unsigned int count); +int vfio_pci_core_match(struct vfio_device *core_vdev, char *buf); +int vfio_pci_core_enable(struct vfio_pci_core_device *vdev); +void vfio_pci_core_disable(struct vfio_pci_core_device *vdev); +void vfio_pci_core_finish_enable(struct vfio_pci_core_device *vdev); + +static inline bool vfio_pci_is_vga(struct pci_dev *pdev) +{ + return (pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA; +} + +#endif /* VFIO_PCI_CORE_H */ -- cgit