From e7c8cc35a64d1d21e6fb811c519eadbda30f4f77 Mon Sep 17 00:00:00 2001 From: Matej Genci Date: Wed, 11 Sep 2019 12:49:53 +0000 Subject: [PATCH 01/40] virtio: add VIRTIO_RING_NO_LEGACY Add macro to disable legacy vring functions. Signed-off-by: Matej Genci Link: https://lore.kernel.org/r/20190911124942.243713-1-matej.genci@nutanix.com Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_pci_modern.c | 1 + include/uapi/linux/virtio_ring.h | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c index 7abcc50838b8..db93cedd262f 100644 --- a/drivers/virtio/virtio_pci_modern.c +++ b/drivers/virtio/virtio_pci_modern.c @@ -16,6 +16,7 @@ #include #define VIRTIO_PCI_NO_LEGACY +#define VIRTIO_RING_NO_LEGACY #include "virtio_pci_common.h" /* diff --git a/include/uapi/linux/virtio_ring.h b/include/uapi/linux/virtio_ring.h index 559f42e73315..9223c3a5c46a 100644 --- a/include/uapi/linux/virtio_ring.h +++ b/include/uapi/linux/virtio_ring.h @@ -135,6 +135,8 @@ struct vring { #define VRING_USED_ALIGN_SIZE 4 #define VRING_DESC_ALIGN_SIZE 16 +#ifndef VIRTIO_RING_NO_LEGACY + /* The standard layout for the ring is a continuous chunk of memory which looks * like this. We assume num is a power of 2. * @@ -181,6 +183,8 @@ static inline unsigned vring_size(unsigned int num, unsigned long align) + sizeof(__virtio16) * 3 + sizeof(struct vring_used_elem) * num; } +#endif /* VIRTIO_RING_NO_LEGACY */ + /* The following is used with USED_EVENT_IDX and AVAIL_EVENT_IDX */ /* Assuming a given event_idx value from the other side, if * we have just incremented index from old to new_idx, From 0c35c67412f0ae9ebe1a87cb83bc9de8143438b7 Mon Sep 17 00:00:00 2001 From: Markus Elfring Date: Sun, 5 Apr 2020 19:14:10 +0200 Subject: [PATCH 02/40] virtio-mmio: Delete an error message in vm_find_vqs() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The function “platform_get_irq” can log an error already. Thus omit a redundant message for the exception handling in the calling function. This issue was detected by using the Coccinelle software. Signed-off-by: Markus Elfring Link: https://lore.kernel.org/r/9e27bc4a-cfa1-7818-dc25-8ad308816b30@web.de Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_mmio.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c index 97d5725fd9a2..9d16aaffca9d 100644 --- a/drivers/virtio/virtio_mmio.c +++ b/drivers/virtio/virtio_mmio.c @@ -466,10 +466,8 @@ static int vm_find_vqs(struct virtio_device *vdev, unsigned nvqs, int irq = platform_get_irq(vm_dev->pdev, 0); int i, err, queue_idx = 0; - if (irq < 0) { - dev_err(&vdev->dev, "Cannot get IRQ resource\n"); + if (irq < 0) return irq; - } err = request_irq(irq, vm_interrupt, IRQF_SHARED, dev_name(&vdev->dev), vm_dev); From a865e420b9561235851c3f5d483c82ef389d29bd Mon Sep 17 00:00:00 2001 From: "Michael S. Tsirkin" Date: Mon, 6 Apr 2020 08:42:55 -0400 Subject: [PATCH 03/40] virtio: force spec specified alignment on types The ring element addresses are passed between components with different alignments assumptions. Thus, if guest/userspace selects a pointer and host then gets and dereferences it, we might need to decrease the compiler-selected alignment to prevent compiler on the host from assuming pointer is aligned. This actually triggers on ARM with -mabi=apcs-gnu - which is a deprecated configuration, but it seems safer to handle this generally. Note that userspace that allocates the memory is actually OK and does not need to be fixed, but userspace that gets it from guest or another process does need to be fixed. The later doesn't generally talk to the kernel so while it might be buggy it's not talking to the kernel in the buggy way - it's just using the header in the buggy way - so fixing header and asking userspace to recompile is the best we can do. I verified that the produced kernel binary on x86 is exactly identical before and after the change. Signed-off-by: Michael S. Tsirkin Acked-by: Jason Wang --- drivers/vhost/vhost.c | 8 +++--- drivers/vhost/vhost.h | 6 ++--- drivers/vhost/vringh.c | 6 ++--- include/linux/vringh.h | 6 ++--- include/uapi/linux/virtio_ring.h | 46 ++++++++++++++++++++++++-------- 5 files changed, 48 insertions(+), 24 deletions(-) diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index 21a59b598ed8..96d9871fa0cb 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -1244,9 +1244,9 @@ static int vhost_iotlb_miss(struct vhost_virtqueue *vq, u64 iova, int access) } static bool vq_access_ok(struct vhost_virtqueue *vq, unsigned int num, - struct vring_desc __user *desc, - struct vring_avail __user *avail, - struct vring_used __user *used) + vring_desc_t __user *desc, + vring_avail_t __user *avail, + vring_used_t __user *used) { return access_ok(desc, vhost_get_desc_size(vq, num)) && @@ -2301,7 +2301,7 @@ static int __vhost_add_used_n(struct vhost_virtqueue *vq, struct vring_used_elem *heads, unsigned count) { - struct vring_used_elem __user *used; + vring_used_elem_t __user *used; u16 old, new; int start; diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h index f8403bd46b85..60cab4c78229 100644 --- a/drivers/vhost/vhost.h +++ b/drivers/vhost/vhost.h @@ -67,9 +67,9 @@ struct vhost_virtqueue { /* The actual ring of buffers. */ struct mutex mutex; unsigned int num; - struct vring_desc __user *desc; - struct vring_avail __user *avail; - struct vring_used __user *used; + vring_desc_t __user *desc; + vring_avail_t __user *avail; + vring_used_t __user *used; const struct vhost_iotlb_map *meta_iotlb[VHOST_NUM_ADDRS]; struct file *kick; struct eventfd_ctx *call_ctx; diff --git a/drivers/vhost/vringh.c b/drivers/vhost/vringh.c index ba8e0d6cfd97..e059a9a47cdf 100644 --- a/drivers/vhost/vringh.c +++ b/drivers/vhost/vringh.c @@ -620,9 +620,9 @@ static inline int xfer_to_user(const struct vringh *vrh, */ int vringh_init_user(struct vringh *vrh, u64 features, unsigned int num, bool weak_barriers, - struct vring_desc __user *desc, - struct vring_avail __user *avail, - struct vring_used __user *used) + vring_desc_t __user *desc, + vring_avail_t __user *avail, + vring_used_t __user *used) { /* Sane power of 2 please! */ if (!num || num > 0xffff || (num & (num - 1))) { diff --git a/include/linux/vringh.h b/include/linux/vringh.h index 9e2763d7c159..59bd50f99291 100644 --- a/include/linux/vringh.h +++ b/include/linux/vringh.h @@ -105,9 +105,9 @@ struct vringh_kiov { /* Helpers for userspace vrings. */ int vringh_init_user(struct vringh *vrh, u64 features, unsigned int num, bool weak_barriers, - struct vring_desc __user *desc, - struct vring_avail __user *avail, - struct vring_used __user *used); + vring_desc_t __user *desc, + vring_avail_t __user *avail, + vring_used_t __user *used); static inline void vringh_iov_init(struct vringh_iov *iov, struct iovec *iovec, unsigned num) diff --git a/include/uapi/linux/virtio_ring.h b/include/uapi/linux/virtio_ring.h index 9223c3a5c46a..476d3e5c0fe7 100644 --- a/include/uapi/linux/virtio_ring.h +++ b/include/uapi/linux/virtio_ring.h @@ -86,6 +86,13 @@ * at the end of the used ring. Guest should ignore the used->flags field. */ #define VIRTIO_RING_F_EVENT_IDX 29 +/* Alignment requirements for vring elements. + * When using pre-virtio 1.0 layout, these fall out naturally. + */ +#define VRING_AVAIL_ALIGN_SIZE 2 +#define VRING_USED_ALIGN_SIZE 4 +#define VRING_DESC_ALIGN_SIZE 16 + /* Virtio ring descriptors: 16 bytes. These can chain together via "next". */ struct vring_desc { /* Address (guest-physical). */ @@ -112,29 +119,46 @@ struct vring_used_elem { __virtio32 len; }; +typedef struct vring_used_elem __attribute__((aligned(VRING_USED_ALIGN_SIZE))) + vring_used_elem_t; + struct vring_used { __virtio16 flags; __virtio16 idx; - struct vring_used_elem ring[]; + vring_used_elem_t ring[]; }; +/* + * The ring element addresses are passed between components with different + * alignments assumptions. Thus, we might need to decrease the compiler-selected + * alignment, and so must use a typedef to make sure the aligned attribute + * actually takes hold: + * + * https://gcc.gnu.org/onlinedocs//gcc/Common-Type-Attributes.html#Common-Type-Attributes + * + * When used on a struct, or struct member, the aligned attribute can only + * increase the alignment; in order to decrease it, the packed attribute must + * be specified as well. When used as part of a typedef, the aligned attribute + * can both increase and decrease alignment, and specifying the packed + * attribute generates a warning. + */ +typedef struct vring_desc __attribute__((aligned(VRING_DESC_ALIGN_SIZE))) + vring_desc_t; +typedef struct vring_avail __attribute__((aligned(VRING_AVAIL_ALIGN_SIZE))) + vring_avail_t; +typedef struct vring_used __attribute__((aligned(VRING_USED_ALIGN_SIZE))) + vring_used_t; + struct vring { unsigned int num; - struct vring_desc *desc; + vring_desc_t *desc; - struct vring_avail *avail; + vring_avail_t *avail; - struct vring_used *used; + vring_used_t *used; }; -/* Alignment requirements for vring elements. - * When using pre-virtio 1.0 layout, these fall out naturally. - */ -#define VRING_AVAIL_ALIGN_SIZE 2 -#define VRING_USED_ALIGN_SIZE 4 -#define VRING_DESC_ALIGN_SIZE 16 - #ifndef VIRTIO_RING_NO_LEGACY /* The standard layout for the ring is a continuous chunk of memory which looks From 213e77213867e4edbdb342c37ea822feedf4608f Mon Sep 17 00:00:00 2001 From: "Michael S. Tsirkin" Date: Thu, 23 Apr 2020 08:36:57 -0400 Subject: [PATCH 04/40] vhost: revert "vhost: disable for OABI" This reverts commit d085eb8ce727 ("vhost: disable for OABI") With commit "virtio: force spec specified alignment on types" in place, we force proper alignment for all structures, so there's no longer a reason to blacklist OABI. Signed-off-by: Michael S. Tsirkin --- drivers/misc/mic/Kconfig | 2 +- drivers/net/caif/Kconfig | 2 +- drivers/vdpa/Kconfig | 2 +- drivers/vhost/Kconfig | 17 ++++------------- 4 files changed, 7 insertions(+), 16 deletions(-) diff --git a/drivers/misc/mic/Kconfig b/drivers/misc/mic/Kconfig index 3bfe72c59864..8f201d019f5a 100644 --- a/drivers/misc/mic/Kconfig +++ b/drivers/misc/mic/Kconfig @@ -116,7 +116,7 @@ config MIC_COSM config VOP tristate "VOP Driver" - depends on VOP_BUS && VHOST_DPN + depends on VOP_BUS select VHOST_RING select VIRTIO help diff --git a/drivers/net/caif/Kconfig b/drivers/net/caif/Kconfig index 661c25eb1c46..9db0570c5beb 100644 --- a/drivers/net/caif/Kconfig +++ b/drivers/net/caif/Kconfig @@ -50,7 +50,7 @@ config CAIF_HSI config CAIF_VIRTIO tristate "CAIF virtio transport driver" - depends on CAIF && HAS_DMA && VHOST_DPN + depends on CAIF && HAS_DMA select VHOST_RING select VIRTIO select GENERIC_ALLOCATOR diff --git a/drivers/vdpa/Kconfig b/drivers/vdpa/Kconfig index e8140065c8a5..3e1ceb8e9f2b 100644 --- a/drivers/vdpa/Kconfig +++ b/drivers/vdpa/Kconfig @@ -10,7 +10,7 @@ if VDPA config VDPA_SIM tristate "vDPA device simulator" - depends on RUNTIME_TESTING_MENU && HAS_DMA && VHOST_DPN + depends on RUNTIME_TESTING_MENU && HAS_DMA select VHOST_RING default n help diff --git a/drivers/vhost/Kconfig b/drivers/vhost/Kconfig index c4f273793595..2c75d164b827 100644 --- a/drivers/vhost/Kconfig +++ b/drivers/vhost/Kconfig @@ -13,15 +13,6 @@ config VHOST_RING This option is selected by any driver which needs to access the host side of a virtio ring. -config VHOST_DPN - bool - depends on !ARM || AEABI - default y - help - Anything selecting VHOST or VHOST_RING must depend on VHOST_DPN. - This excludes the deprecated ARM ABI since that forces a 4 byte - alignment on all structs - incompatible with virtio spec requirements. - config VHOST tristate select VHOST_IOTLB @@ -37,7 +28,7 @@ if VHOST_MENU config VHOST_NET tristate "Host kernel accelerator for virtio net" - depends on NET && EVENTFD && (TUN || !TUN) && (TAP || !TAP) && VHOST_DPN + depends on NET && EVENTFD && (TUN || !TUN) && (TAP || !TAP) select VHOST ---help--- This kernel module can be loaded in host kernel to accelerate @@ -49,7 +40,7 @@ config VHOST_NET config VHOST_SCSI tristate "VHOST_SCSI TCM fabric driver" - depends on TARGET_CORE && EVENTFD && VHOST_DPN + depends on TARGET_CORE && EVENTFD select VHOST default n ---help--- @@ -58,7 +49,7 @@ config VHOST_SCSI config VHOST_VSOCK tristate "vhost virtio-vsock driver" - depends on VSOCKETS && EVENTFD && VHOST_DPN + depends on VSOCKETS && EVENTFD select VHOST select VIRTIO_VSOCKETS_COMMON default n @@ -72,7 +63,7 @@ config VHOST_VSOCK config VHOST_VDPA tristate "Vhost driver for vDPA-based backend" - depends on EVENTFD && VHOST_DPN + depends on EVENTFD select VHOST depends on VDPA help From 5c1bd89b45d4c82316833a8d7ed36720ab89ee3d Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Tue, 10 Mar 2020 12:54:11 +0100 Subject: [PATCH 05/40] MAINTAINERS: Add myself as virtio-balloon co-maintainer As suggested by Michael, let's add me as co-maintainer of virtio-balloon. While at it, also add "include/linux/balloon_compaction.h" to the file list. Cc: Michael S. Tsirkin Cc: Jason Wang Cc: Greg Kroah-Hartman Signed-off-by: David Hildenbrand Link: https://lore.kernel.org/r/20200310115411.12760-1-david@redhat.com Signed-off-by: Michael S. Tsirkin --- MAINTAINERS | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index 50659d76976b..b5106b7d6d0c 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -17909,9 +17909,18 @@ F: drivers/virtio/ F: include/linux/vdpa.h F: include/linux/virtio*.h F: include/uapi/linux/virtio_*.h -F: mm/balloon_compaction.c F: tools/virtio/ +VIRTIO BALLOON +M: "Michael S. Tsirkin" +M: David Hildenbrand +L: virtualization@lists.linux-foundation.org +S: Maintained +F: drivers/virtio/virtio_balloon.c +F: include/uapi/linux/virtio_balloon.h +F: include/linux/balloon_compaction.h +F: mm/balloon_compaction.c + VIRTIO CRYPTO DRIVER M: Gonglei L: virtualization@lists.linux-foundation.org From 01fcb1cbc88effb3493c6197efc96b69b9f4823a Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Fri, 29 May 2020 16:02:58 +0800 Subject: [PATCH 06/40] vhost: allow device that does not depend on vhost worker vDPA device currently relays the eventfd via vhost worker. This is inefficient due the latency of wakeup and scheduling, so this patch tries to introduce a use_worker attribute for the vhost device. When use_worker is not set with vhost_dev_init(), vhost won't try to allocate a worker thread and the vhost_poll will be processed directly in the wakeup function. This help for vDPA since it reduces the latency caused by vhost worker. In my testing, it saves 0.2 ms in pings between VMs on a mutual host. Signed-off-by: Zhu Lingshan Signed-off-by: Jason Wang Link: https://lore.kernel.org/r/20200529080303.15449-2-jasowang@redhat.com Signed-off-by: Michael S. Tsirkin --- drivers/vhost/net.c | 2 +- drivers/vhost/scsi.c | 2 +- drivers/vhost/vdpa.c | 2 +- drivers/vhost/vhost.c | 40 ++++++++++++++++++++++++++-------------- drivers/vhost/vhost.h | 2 ++ drivers/vhost/vsock.c | 2 +- 6 files changed, 32 insertions(+), 18 deletions(-) diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c index 2927f02cc7e1..bf5e1d81ae25 100644 --- a/drivers/vhost/net.c +++ b/drivers/vhost/net.c @@ -1326,7 +1326,7 @@ static int vhost_net_open(struct inode *inode, struct file *f) } vhost_dev_init(dev, vqs, VHOST_NET_VQ_MAX, UIO_MAXIOV + VHOST_NET_BATCH, - VHOST_NET_PKT_WEIGHT, VHOST_NET_WEIGHT, + VHOST_NET_PKT_WEIGHT, VHOST_NET_WEIGHT, true, NULL); vhost_poll_init(n->poll + VHOST_NET_VQ_TX, handle_tx_net, EPOLLOUT, dev); diff --git a/drivers/vhost/scsi.c b/drivers/vhost/scsi.c index c39952243fd3..0cbaa0b3893d 100644 --- a/drivers/vhost/scsi.c +++ b/drivers/vhost/scsi.c @@ -1628,7 +1628,7 @@ static int vhost_scsi_open(struct inode *inode, struct file *f) vs->vqs[i].vq.handle_kick = vhost_scsi_handle_kick; } vhost_dev_init(&vs->dev, vqs, VHOST_SCSI_MAX_VQ, UIO_MAXIOV, - VHOST_SCSI_WEIGHT, 0, NULL); + VHOST_SCSI_WEIGHT, 0, true, NULL); vhost_scsi_init_inflight(vs, NULL); diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c index 0968361e3b77..f22bb316b8c4 100644 --- a/drivers/vhost/vdpa.c +++ b/drivers/vhost/vdpa.c @@ -694,7 +694,7 @@ static int vhost_vdpa_open(struct inode *inode, struct file *filep) vqs[i] = &v->vqs[i]; vqs[i]->handle_kick = handle_vq_kick; } - vhost_dev_init(dev, vqs, nvqs, 0, 0, 0, + vhost_dev_init(dev, vqs, nvqs, 0, 0, 0, false, vhost_vdpa_process_iotlb_msg); dev->iotlb = vhost_iotlb_alloc(0, 0); diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index 96d9871fa0cb..80da9d9662f2 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -166,11 +166,16 @@ static int vhost_poll_wakeup(wait_queue_entry_t *wait, unsigned mode, int sync, void *key) { struct vhost_poll *poll = container_of(wait, struct vhost_poll, wait); + struct vhost_work *work = &poll->work; if (!(key_to_poll(key) & poll->mask)) return 0; - vhost_poll_queue(poll); + if (!poll->dev->use_worker) + work->fn(work); + else + vhost_poll_queue(poll); + return 0; } @@ -454,6 +459,7 @@ static size_t vhost_get_desc_size(struct vhost_virtqueue *vq, void vhost_dev_init(struct vhost_dev *dev, struct vhost_virtqueue **vqs, int nvqs, int iov_limit, int weight, int byte_weight, + bool use_worker, int (*msg_handler)(struct vhost_dev *dev, struct vhost_iotlb_msg *msg)) { @@ -471,6 +477,7 @@ void vhost_dev_init(struct vhost_dev *dev, dev->iov_limit = iov_limit; dev->weight = weight; dev->byte_weight = byte_weight; + dev->use_worker = use_worker; dev->msg_handler = msg_handler; init_llist_head(&dev->work_list); init_waitqueue_head(&dev->wait); @@ -549,27 +556,32 @@ long vhost_dev_set_owner(struct vhost_dev *dev) /* No owner, become one */ dev->mm = get_task_mm(current); dev->kcov_handle = kcov_common_handle(); - worker = kthread_create(vhost_worker, dev, "vhost-%d", current->pid); - if (IS_ERR(worker)) { - err = PTR_ERR(worker); - goto err_worker; + if (dev->use_worker) { + worker = kthread_create(vhost_worker, dev, + "vhost-%d", current->pid); + if (IS_ERR(worker)) { + err = PTR_ERR(worker); + goto err_worker; + } + + dev->worker = worker; + wake_up_process(worker); /* avoid contributing to loadavg */ + + err = vhost_attach_cgroups(dev); + if (err) + goto err_cgroup; } - dev->worker = worker; - wake_up_process(worker); /* avoid contributing to loadavg */ - - err = vhost_attach_cgroups(dev); - if (err) - goto err_cgroup; - err = vhost_dev_alloc_iovecs(dev); if (err) goto err_cgroup; return 0; err_cgroup: - kthread_stop(worker); - dev->worker = NULL; + if (dev->worker) { + kthread_stop(dev->worker); + dev->worker = NULL; + } err_worker: if (dev->mm) mmput(dev->mm); diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h index 60cab4c78229..c8e96a095d3b 100644 --- a/drivers/vhost/vhost.h +++ b/drivers/vhost/vhost.h @@ -154,6 +154,7 @@ struct vhost_dev { int weight; int byte_weight; u64 kcov_handle; + bool use_worker; int (*msg_handler)(struct vhost_dev *dev, struct vhost_iotlb_msg *msg); }; @@ -161,6 +162,7 @@ struct vhost_dev { bool vhost_exceeds_weight(struct vhost_virtqueue *vq, int pkts, int total_len); void vhost_dev_init(struct vhost_dev *, struct vhost_virtqueue **vqs, int nvqs, int iov_limit, int weight, int byte_weight, + bool use_worker, int (*msg_handler)(struct vhost_dev *dev, struct vhost_iotlb_msg *msg)); long vhost_dev_set_owner(struct vhost_dev *dev); diff --git a/drivers/vhost/vsock.c b/drivers/vhost/vsock.c index fb4e944c4d0d..a483cec31d5c 100644 --- a/drivers/vhost/vsock.c +++ b/drivers/vhost/vsock.c @@ -632,7 +632,7 @@ static int vhost_vsock_dev_open(struct inode *inode, struct file *file) vhost_dev_init(&vsock->dev, vqs, ARRAY_SIZE(vsock->vqs), UIO_MAXIOV, VHOST_VSOCK_PKT_WEIGHT, - VHOST_VSOCK_WEIGHT, NULL); + VHOST_VSOCK_WEIGHT, true, NULL); file->private_data = vsock; spin_lock_init(&vsock->send_pkt_list_lock); From 5ce995f313ce56c0c62425c3ddc37c5c50fc33db Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Fri, 29 May 2020 16:02:59 +0800 Subject: [PATCH 07/40] vhost: use mmgrab() instead of mmget() for non worker device For the device that doesn't use vhost worker and use_mm(), mmget() is too heavy weight and it may brings troubles for implementing mmap() support for vDPA device. This is because, an reference to the address space was held via mm_get() in vhost_dev_set_owner() and an reference to the file was held in mmap(). This means when process exits, the mm can not be released thus we can not release the file. This patch tries to use mmgrab() instead of mmget(), which allows the address space to be destroy in process exit without releasing the mm structure itself. This is sufficient for vDPA device which pin user pages and does not depend on the address space to work. Signed-off-by: Jason Wang Link: https://lore.kernel.org/r/20200529080303.15449-3-jasowang@redhat.com Signed-off-by: Michael S. Tsirkin --- drivers/vhost/vhost.c | 42 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index 80da9d9662f2..a40d16bdebb5 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -541,6 +541,36 @@ bool vhost_dev_has_owner(struct vhost_dev *dev) } EXPORT_SYMBOL_GPL(vhost_dev_has_owner); +static void vhost_attach_mm(struct vhost_dev *dev) +{ + /* No owner, become one */ + if (dev->use_worker) { + dev->mm = get_task_mm(current); + } else { + /* vDPA device does not use worker thead, so there's + * no need to hold the address space for mm. This help + * to avoid deadlock in the case of mmap() which may + * held the refcnt of the file and depends on release + * method to remove vma. + */ + dev->mm = current->mm; + mmgrab(dev->mm); + } +} + +static void vhost_detach_mm(struct vhost_dev *dev) +{ + if (!dev->mm) + return; + + if (dev->use_worker) + mmput(dev->mm); + else + mmdrop(dev->mm); + + dev->mm = NULL; +} + /* Caller should have device mutex */ long vhost_dev_set_owner(struct vhost_dev *dev) { @@ -553,8 +583,8 @@ long vhost_dev_set_owner(struct vhost_dev *dev) goto err_mm; } - /* No owner, become one */ - dev->mm = get_task_mm(current); + vhost_attach_mm(dev); + dev->kcov_handle = kcov_common_handle(); if (dev->use_worker) { worker = kthread_create(vhost_worker, dev, @@ -583,9 +613,7 @@ err_cgroup: dev->worker = NULL; } err_worker: - if (dev->mm) - mmput(dev->mm); - dev->mm = NULL; + vhost_detach_mm(dev); dev->kcov_handle = 0; err_mm: return err; @@ -682,9 +710,7 @@ void vhost_dev_cleanup(struct vhost_dev *dev) dev->worker = NULL; dev->kcov_handle = 0; } - if (dev->mm) - mmput(dev->mm); - dev->mm = NULL; + vhost_detach_mm(dev); } EXPORT_SYMBOL_GPL(vhost_dev_cleanup); From c25a26e653a61b93339dcd1b734c889df60b3eef Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Fri, 29 May 2020 16:03:00 +0800 Subject: [PATCH 08/40] vdpa: introduce get_vq_notification method This patch introduces a new method in the vdpa_config_ops which reports the physical address and the size of the doorbell for a specific virtqueue. This will be used by the future patches that maps doorbell to userspace. Signed-off-by: Jason Wang Link: https://lore.kernel.org/r/20200529080303.15449-4-jasowang@redhat.com Signed-off-by: Michael S. Tsirkin --- include/linux/vdpa.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h index 5453af87a33e..239db794357c 100644 --- a/include/linux/vdpa.h +++ b/include/linux/vdpa.h @@ -17,6 +17,16 @@ struct vdpa_callback { void *private; }; +/** + * vDPA notification area + * @addr: base address of the notification area + * @size: size of the notification area + */ +struct vdpa_notification_area { + resource_size_t addr; + resource_size_t size; +}; + /** * vDPA device - representation of a vDPA device * @dev: underlying device @@ -73,6 +83,10 @@ struct vdpa_device { * @vdev: vdpa device * @idx: virtqueue index * Returns virtqueue state (last_avail_idx) + * @get_vq_notification: Get the notification area for a virtqueue + * @vdev: vdpa device + * @idx: virtqueue index + * Returns the notifcation area * @get_vq_align: Get the virtqueue align requirement * for the device * @vdev: vdpa device @@ -162,6 +176,8 @@ struct vdpa_config_ops { bool (*get_vq_ready)(struct vdpa_device *vdev, u16 idx); int (*set_vq_state)(struct vdpa_device *vdev, u16 idx, u64 state); u64 (*get_vq_state)(struct vdpa_device *vdev, u16 idx); + struct vdpa_notification_area + (*get_vq_notification)(struct vdpa_device *vdev, u16 idx); /* Device ops */ u32 (*get_vq_align)(struct vdpa_device *vdev); From ddd89d0a059d8e9740c75a97e0efe9bf07ee51f9 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Fri, 29 May 2020 16:03:01 +0800 Subject: [PATCH 09/40] vhost_vdpa: support doorbell mapping via mmap Currently the doorbell is relayed via eventfd which may have significant overhead because of the cost of vmexits or syscall. This patch introduces mmap() based doorbell mapping which can eliminate the overhead caused by vmexit or syscall. To ease the userspace modeling of the doorbell layout (usually virtio-pci), this patch starts from a doorbell per page model. Vhost-vdpa only support the hardware doorbell that sit at the boundary of a page and does not share the page with other registers. Doorbell of each virtqueue must be mapped separately, pgoff is the index of the virtqueue. This allows userspace to map a subset of the doorbell which may be useful for the implementation of software assisted virtqueue (control vq) in the future. Signed-off-by: Jason Wang Link: https://lore.kernel.org/r/20200529080303.15449-5-jasowang@redhat.com Signed-off-by: Michael S. Tsirkin --- drivers/vhost/vdpa.c | 59 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c index f22bb316b8c4..3d163ec6fb28 100644 --- a/drivers/vhost/vdpa.c +++ b/drivers/vhost/vdpa.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -739,12 +740,70 @@ static int vhost_vdpa_release(struct inode *inode, struct file *filep) return 0; } +static vm_fault_t vhost_vdpa_fault(struct vm_fault *vmf) +{ + struct vhost_vdpa *v = vmf->vma->vm_file->private_data; + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + struct vdpa_notification_area notify; + struct vm_area_struct *vma = vmf->vma; + u16 index = vma->vm_pgoff; + + notify = ops->get_vq_notification(vdpa, index); + + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + if (remap_pfn_range(vma, vmf->address & PAGE_MASK, + notify.addr >> PAGE_SHIFT, PAGE_SIZE, + vma->vm_page_prot)) + return VM_FAULT_SIGBUS; + + return VM_FAULT_NOPAGE; +} + +static const struct vm_operations_struct vhost_vdpa_vm_ops = { + .fault = vhost_vdpa_fault, +}; + +static int vhost_vdpa_mmap(struct file *file, struct vm_area_struct *vma) +{ + struct vhost_vdpa *v = vma->vm_file->private_data; + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + struct vdpa_notification_area notify; + int index = vma->vm_pgoff; + + if (vma->vm_end - vma->vm_start != PAGE_SIZE) + return -EINVAL; + if ((vma->vm_flags & VM_SHARED) == 0) + return -EINVAL; + if (vma->vm_flags & VM_READ) + return -EINVAL; + if (index > 65535) + return -EINVAL; + if (!ops->get_vq_notification) + return -ENOTSUPP; + + /* To be safe and easily modelled by userspace, We only + * support the doorbell which sits on the page boundary and + * does not share the page with other registers. + */ + notify = ops->get_vq_notification(vdpa, index); + if (notify.addr & (PAGE_SIZE - 1)) + return -EINVAL; + if (vma->vm_end - vma->vm_start != notify.size) + return -ENOTSUPP; + + vma->vm_ops = &vhost_vdpa_vm_ops; + return 0; +} + static const struct file_operations vhost_vdpa_fops = { .owner = THIS_MODULE, .open = vhost_vdpa_open, .release = vhost_vdpa_release, .write_iter = vhost_vdpa_chr_write_iter, .unlocked_ioctl = vhost_vdpa_unlocked_ioctl, + .mmap = vhost_vdpa_mmap, .compat_ioctl = compat_ptr_ioctl, }; From 4b4e4867d92205158c524842f59b1c1caeb969fe Mon Sep 17 00:00:00 2001 From: "Michael S. Tsirkin" Date: Thu, 4 Jun 2020 14:47:29 -0400 Subject: [PATCH 10/40] vhost_vdpa: disable doorbell mapping for !MMU There could be ways to support doorbell mapping with !MMU, but things like pgprot_noncached are not universally supported. Fixable, but just disable this for now. Signed-off-by: Michael S. Tsirkin --- drivers/vhost/vdpa.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c index 3d163ec6fb28..6ca7660ee6b5 100644 --- a/drivers/vhost/vdpa.c +++ b/drivers/vhost/vdpa.c @@ -740,6 +740,7 @@ static int vhost_vdpa_release(struct inode *inode, struct file *filep) return 0; } +#ifdef CONFIG_MMU static vm_fault_t vhost_vdpa_fault(struct vm_fault *vmf) { struct vhost_vdpa *v = vmf->vma->vm_file->private_data; @@ -796,6 +797,7 @@ static int vhost_vdpa_mmap(struct file *file, struct vm_area_struct *vma) vma->vm_ops = &vhost_vdpa_vm_ops; return 0; } +#endif /* CONFIG_MMU */ static const struct file_operations vhost_vdpa_fops = { .owner = THIS_MODULE, @@ -803,7 +805,9 @@ static const struct file_operations vhost_vdpa_fops = { .release = vhost_vdpa_release, .write_iter = vhost_vdpa_chr_write_iter, .unlocked_ioctl = vhost_vdpa_unlocked_ioctl, +#ifdef CONFIG_MMU .mmap = vhost_vdpa_mmap, +#endif /* CONFIG_MMU */ .compat_ioctl = compat_ptr_ioctl, }; From fb69c2c896fc8289b0d9e2c0791472e7cd398bca Mon Sep 17 00:00:00 2001 From: Alexander Duyck Date: Fri, 8 May 2020 10:40:06 -0700 Subject: [PATCH 11/40] virtio-balloon: Disable free page reporting if page poison reporting is not enabled We should disable free page reporting if page poisoning is enabled but we cannot report it via the balloon interface. This way we can avoid the possibility of corrupting guest memory. Normally the page poisoning feature should always be present when free page reporting is enabled on the hypervisor, however this allows us to correctly handle a case of the virtio-balloon device being possibly misconfigured. Fixes: 5d757c8d518d ("virtio-balloon: add support for providing free page reports to host") Cc: stable@vger.kernel.org Acked-by: David Hildenbrand Signed-off-by: Alexander Duyck Link: https://lore.kernel.org/r/20200508173732.17877.85060.stgit@localhost.localdomain Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_balloon.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c index 51086a5afdd4..1f157d2f4952 100644 --- a/drivers/virtio/virtio_balloon.c +++ b/drivers/virtio/virtio_balloon.c @@ -1107,11 +1107,18 @@ static int virtballoon_restore(struct virtio_device *vdev) static int virtballoon_validate(struct virtio_device *vdev) { - /* Tell the host whether we care about poisoned pages. */ + /* + * Inform the hypervisor that our pages are poisoned or + * initialized. If we cannot do that then we should disable + * page reporting as it could potentially change the contents + * of our free pages. + */ if (!want_init_on_free() && (IS_ENABLED(CONFIG_PAGE_POISONING_NO_SANITY) || !page_poisoning_enabled())) __virtio_clear_bit(vdev, VIRTIO_BALLOON_F_PAGE_POISON); + else if (!virtio_has_feature(vdev, VIRTIO_BALLOON_F_PAGE_POISON)) + __virtio_clear_bit(vdev, VIRTIO_BALLOON_F_REPORTING); __virtio_clear_bit(vdev, VIRTIO_F_IOMMU_PLATFORM); return 0; From b02989f37fc5e865ceeee9070907e4493b3a21e2 Mon Sep 17 00:00:00 2001 From: "Longpeng(Mike)" Date: Tue, 2 Jun 2020 15:04:59 +0800 Subject: [PATCH 12/40] crypto: virtio: Fix src/dst scatterlist calculation in __virtio_crypto_skcipher_do_req() The system will crash when the users insmod crypto/tcrypt.ko with mode=38 ( testing "cts(cbc(aes))" ). Usually the next entry of one sg will be @sg@ + 1, but if this sg element is part of a chained scatterlist, it could jump to the start of a new scatterlist array. Fix it by sg_next() on calculation of src/dst scatterlist. Fixes: dbaf0624ffa5 ("crypto: add virtio-crypto driver") Reported-by: LABBE Corentin Cc: Herbert Xu Cc: "Michael S. Tsirkin" Cc: Jason Wang Cc: "David S. Miller" Cc: virtualization@lists.linux-foundation.org Cc: linux-kernel@vger.kernel.org Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20200123101000.GB24255@Red Signed-off-by: Gonglei Signed-off-by: Longpeng(Mike) Link: https://lore.kernel.org/r/20200602070501.2023-2-longpeng2@huawei.com Signed-off-by: Michael S. Tsirkin --- drivers/crypto/virtio/virtio_crypto_algs.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/drivers/crypto/virtio/virtio_crypto_algs.c b/drivers/crypto/virtio/virtio_crypto_algs.c index fd045e64972a..5f8243563009 100644 --- a/drivers/crypto/virtio/virtio_crypto_algs.c +++ b/drivers/crypto/virtio/virtio_crypto_algs.c @@ -350,13 +350,18 @@ __virtio_crypto_skcipher_do_req(struct virtio_crypto_sym_request *vc_sym_req, int err; unsigned long flags; struct scatterlist outhdr, iv_sg, status_sg, **sgs; - int i; u64 dst_len; unsigned int num_out = 0, num_in = 0; int sg_total; uint8_t *iv; + struct scatterlist *sg; src_nents = sg_nents_for_len(req->src, req->cryptlen); + if (src_nents < 0) { + pr_err("Invalid number of src SG.\n"); + return src_nents; + } + dst_nents = sg_nents(req->dst); pr_debug("virtio_crypto: Number of sgs (src_nents: %d, dst_nents: %d)\n", @@ -442,12 +447,12 @@ __virtio_crypto_skcipher_do_req(struct virtio_crypto_sym_request *vc_sym_req, vc_sym_req->iv = iv; /* Source data */ - for (i = 0; i < src_nents; i++) - sgs[num_out++] = &req->src[i]; + for (sg = req->src; src_nents; sg = sg_next(sg), src_nents--) + sgs[num_out++] = sg; /* Destination data */ - for (i = 0; i < dst_nents; i++) - sgs[num_out + num_in++] = &req->dst[i]; + for (sg = req->dst; sg; sg = sg_next(sg)) + sgs[num_out + num_in++] = sg; /* Status */ sg_init_one(&status_sg, &vc_req->status, sizeof(vc_req->status)); From 8c855f0720ff006d75d0a2512c7f6c4f60ff60ee Mon Sep 17 00:00:00 2001 From: "Longpeng(Mike)" Date: Tue, 2 Jun 2020 15:05:00 +0800 Subject: [PATCH 13/40] crypto: virtio: Fix use-after-free in virtio_crypto_skcipher_finalize_req() The system'll crash when the users insmod crypto/tcrypto.ko with mode=155 ( testing "authenc(hmac(sha1),cbc(aes))" ). It's caused by reuse the memory of request structure. In crypto_authenc_init_tfm(), the reqsize is set to: [PART 1] sizeof(authenc_request_ctx) + [PART 2] ictx->reqoff + [PART 3] MAX(ahash part, skcipher part) and the 'PART 3' is used by both ahash and skcipher in turn. When the virtio_crypto driver finish skcipher req, it'll call ->complete callback(in crypto_finalize_skcipher_request) and then free its resources whose pointers are recorded in 'skcipher parts'. However, the ->complete is 'crypto_authenc_encrypt_done' in this case, it will use the 'ahash part' of the request and change its content, so virtio_crypto driver will get the wrong pointer after ->complete finish and mistakenly free some other's memory. So the system will crash when these memory will be used again. The resources which need to be cleaned up are not used any more. But the pointers of these resources may be changed in the function "crypto_finalize_skcipher_request". Thus release specific resources before calling this function. Fixes: dbaf0624ffa5 ("crypto: add virtio-crypto driver") Reported-by: LABBE Corentin Cc: Gonglei Cc: Herbert Xu Cc: "Michael S. Tsirkin" Cc: Jason Wang Cc: "David S. Miller" Cc: virtualization@lists.linux-foundation.org Cc: linux-kernel@vger.kernel.org Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20200123101000.GB24255@Red Acked-by: Gonglei Signed-off-by: Longpeng(Mike) Link: https://lore.kernel.org/r/20200602070501.2023-3-longpeng2@huawei.com Signed-off-by: Michael S. Tsirkin --- drivers/crypto/virtio/virtio_crypto_algs.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/virtio/virtio_crypto_algs.c b/drivers/crypto/virtio/virtio_crypto_algs.c index 5f8243563009..52261b6c247e 100644 --- a/drivers/crypto/virtio/virtio_crypto_algs.c +++ b/drivers/crypto/virtio/virtio_crypto_algs.c @@ -582,10 +582,11 @@ static void virtio_crypto_skcipher_finalize_req( scatterwalk_map_and_copy(req->iv, req->dst, req->cryptlen - AES_BLOCK_SIZE, AES_BLOCK_SIZE, 0); - crypto_finalize_skcipher_request(vc_sym_req->base.dataq->engine, - req, err); kzfree(vc_sym_req->iv); virtcrypto_clear_request(&vc_sym_req->base); + + crypto_finalize_skcipher_request(vc_sym_req->base.dataq->engine, + req, err); } static struct virtio_crypto_algo virtio_crypto_algs[] = { { From d90ca42012db2863a9a30b564a2ace6016594bda Mon Sep 17 00:00:00 2001 From: "Longpeng(Mike)" Date: Tue, 2 Jun 2020 15:05:01 +0800 Subject: [PATCH 14/40] crypto: virtio: Fix dest length calculation in __virtio_crypto_skcipher_do_req() The src/dst length is not aligned with AES_BLOCK_SIZE(which is 16) in some testcases in tcrypto.ko. For example, the src/dst length of one of cts(cbc(aes))'s testcase is 17, the crypto_virtio driver will set @src_data_len=16 but @dst_data_len=17 in this case and get a wrong at then end. SRC: pp pp pp pp pp pp pp pp pp pp pp pp pp pp pp pp pp (17 bytes) EXP: cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc pp (17 bytes) DST: cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc 00 (pollute the last bytes) (pp: plaintext cc:ciphertext) Fix this issue by limit the length of dest buffer. Fixes: dbaf0624ffa5 ("crypto: add virtio-crypto driver") Cc: Gonglei Cc: Herbert Xu Cc: "Michael S. Tsirkin" Cc: Jason Wang Cc: "David S. Miller" Cc: virtualization@lists.linux-foundation.org Cc: linux-kernel@vger.kernel.org Cc: stable@vger.kernel.org Signed-off-by: Longpeng(Mike) Link: https://lore.kernel.org/r/20200602070501.2023-4-longpeng2@huawei.com Signed-off-by: Michael S. Tsirkin --- drivers/crypto/virtio/virtio_crypto_algs.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/crypto/virtio/virtio_crypto_algs.c b/drivers/crypto/virtio/virtio_crypto_algs.c index 52261b6c247e..cb8a6ea2a4bc 100644 --- a/drivers/crypto/virtio/virtio_crypto_algs.c +++ b/drivers/crypto/virtio/virtio_crypto_algs.c @@ -407,6 +407,7 @@ __virtio_crypto_skcipher_do_req(struct virtio_crypto_sym_request *vc_sym_req, goto free; } + dst_len = min_t(unsigned int, req->cryptlen, dst_len); pr_debug("virtio_crypto: src_len: %u, dst_len: %llu\n", req->cryptlen, dst_len); From 002ef18eff43708be1d914cb11dbdd84fb149474 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Wed, 27 May 2020 20:05:38 +0200 Subject: [PATCH 15/40] vhost: (cosmetic) remove a superfluous variable initialisation Even the compiler is able to figure out that in this case the initialisation is superfluous. Signed-off-by: Guennadi Liakhovetski Link: https://lore.kernel.org/r/20200527180541.5570-3-guennadi.liakhovetski@linux.intel.com Signed-off-by: Michael S. Tsirkin --- drivers/vhost/vhost.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index a40d16bdebb5..534d1267b761 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -920,7 +920,7 @@ static inline void __user *__vhost_get_user(struct vhost_virtqueue *vq, #define vhost_put_user(vq, x, ptr) \ ({ \ - int ret = -EFAULT; \ + int ret; \ if (!vq->iotlb) { \ ret = __put_user(x, ptr); \ } else { \ From 7dd793f37e2ab4ca6585df8cfb1a1fcf44f61248 Mon Sep 17 00:00:00 2001 From: Zhu Lingshan Date: Tue, 12 May 2020 16:00:44 +0800 Subject: [PATCH 16/40] ifcvf: move IRQ request/free to status change handlers This commit move IRQ request and free operations from probe() to VIRTIO status change handler to comply with VIRTIO spec. VIRTIO spec 1.1, section 2.1.2 Device Requirements: Device Status Field The device MUST NOT consume buffers or send any used buffer notifications to the driver before DRIVER_OK. Signed-off-by: Zhu Lingshan Link: https://lore.kernel.org/r/1589270444-3669-1-git-send-email-lingshan.zhu@intel.com Signed-off-by: Michael S. Tsirkin Acked-by: Jason Wang --- drivers/vdpa/ifcvf/ifcvf_main.c | 120 +++++++++++++++++++------------- 1 file changed, 73 insertions(+), 47 deletions(-) diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c index abf6a061cab6..d529ed681fe6 100644 --- a/drivers/vdpa/ifcvf/ifcvf_main.c +++ b/drivers/vdpa/ifcvf/ifcvf_main.c @@ -28,6 +28,60 @@ static irqreturn_t ifcvf_intr_handler(int irq, void *arg) return IRQ_HANDLED; } +static void ifcvf_free_irq_vectors(void *data) +{ + pci_free_irq_vectors(data); +} + +static void ifcvf_free_irq(struct ifcvf_adapter *adapter, int queues) +{ + struct pci_dev *pdev = adapter->pdev; + struct ifcvf_hw *vf = &adapter->vf; + int i; + + + for (i = 0; i < queues; i++) + devm_free_irq(&pdev->dev, vf->vring[i].irq, &vf->vring[i]); + + ifcvf_free_irq_vectors(pdev); +} + +static int ifcvf_request_irq(struct ifcvf_adapter *adapter) +{ + struct pci_dev *pdev = adapter->pdev; + struct ifcvf_hw *vf = &adapter->vf; + int vector, i, ret, irq; + + ret = pci_alloc_irq_vectors(pdev, IFCVF_MAX_INTR, + IFCVF_MAX_INTR, PCI_IRQ_MSIX); + if (ret < 0) { + IFCVF_ERR(pdev, "Failed to alloc IRQ vectors\n"); + return ret; + } + + for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) { + snprintf(vf->vring[i].msix_name, 256, "ifcvf[%s]-%d\n", + pci_name(pdev), i); + vector = i + IFCVF_MSI_QUEUE_OFF; + irq = pci_irq_vector(pdev, vector); + ret = devm_request_irq(&pdev->dev, irq, + ifcvf_intr_handler, 0, + vf->vring[i].msix_name, + &vf->vring[i]); + if (ret) { + IFCVF_ERR(pdev, + "Failed to request irq for vq %d\n", i); + ifcvf_free_irq(adapter, i); + + return ret; + } + + vf->vring[i].irq = irq; + } + + return 0; +} + static int ifcvf_start_datapath(void *private) { struct ifcvf_hw *vf = ifcvf_private_to_vf(private); @@ -118,17 +172,34 @@ static void ifcvf_vdpa_set_status(struct vdpa_device *vdpa_dev, u8 status) { struct ifcvf_adapter *adapter; struct ifcvf_hw *vf; + u8 status_old; + int ret; vf = vdpa_to_vf(vdpa_dev); adapter = dev_get_drvdata(vdpa_dev->dev.parent); + status_old = ifcvf_get_status(vf); + + if ((status_old & VIRTIO_CONFIG_S_DRIVER_OK) && + !(status & VIRTIO_CONFIG_S_DRIVER_OK)) { + ifcvf_stop_datapath(adapter); + ifcvf_free_irq(adapter, IFCVF_MAX_QUEUE_PAIRS * 2); + } if (status == 0) { - ifcvf_stop_datapath(adapter); ifcvf_reset_vring(adapter); return; } - if (status & VIRTIO_CONFIG_S_DRIVER_OK) { + if ((status & VIRTIO_CONFIG_S_DRIVER_OK) && + !(status_old & VIRTIO_CONFIG_S_DRIVER_OK)) { + ret = ifcvf_request_irq(adapter); + if (ret) { + status = ifcvf_get_status(vf); + status |= VIRTIO_CONFIG_S_FAILED; + ifcvf_set_status(vf, status); + return; + } + if (ifcvf_start_datapath(adapter) < 0) IFCVF_ERR(adapter->pdev, "Failed to set ifcvf vdpa status %u\n", @@ -284,38 +355,6 @@ static const struct vdpa_config_ops ifc_vdpa_ops = { .set_config_cb = ifcvf_vdpa_set_config_cb, }; -static int ifcvf_request_irq(struct ifcvf_adapter *adapter) -{ - struct pci_dev *pdev = adapter->pdev; - struct ifcvf_hw *vf = &adapter->vf; - int vector, i, ret, irq; - - - for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) { - snprintf(vf->vring[i].msix_name, 256, "ifcvf[%s]-%d\n", - pci_name(pdev), i); - vector = i + IFCVF_MSI_QUEUE_OFF; - irq = pci_irq_vector(pdev, vector); - ret = devm_request_irq(&pdev->dev, irq, - ifcvf_intr_handler, 0, - vf->vring[i].msix_name, - &vf->vring[i]); - if (ret) { - IFCVF_ERR(pdev, - "Failed to request irq for vq %d\n", i); - return ret; - } - vf->vring[i].irq = irq; - } - - return 0; -} - -static void ifcvf_free_irq_vectors(void *data) -{ - pci_free_irq_vectors(data); -} - static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id) { struct device *dev = &pdev->dev; @@ -349,13 +388,6 @@ static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id) return ret; } - ret = pci_alloc_irq_vectors(pdev, IFCVF_MAX_INTR, - IFCVF_MAX_INTR, PCI_IRQ_MSIX); - if (ret < 0) { - IFCVF_ERR(pdev, "Failed to alloc irq vectors\n"); - return ret; - } - ret = devm_add_action_or_reset(dev, ifcvf_free_irq_vectors, pdev); if (ret) { IFCVF_ERR(pdev, @@ -379,12 +411,6 @@ static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id) adapter->pdev = pdev; adapter->vdpa.dma_dev = &pdev->dev; - ret = ifcvf_request_irq(adapter); - if (ret) { - IFCVF_ERR(pdev, "Failed to request MSI-X irq\n"); - goto err; - } - ret = ifcvf_init_hw(vf, pdev); if (ret) { IFCVF_ERR(pdev, "Failed to init IFCVF hw\n"); From 18e643cd6c4dcdb09510a4bf739438e7cd1ae1f2 Mon Sep 17 00:00:00 2001 From: Samuel Zou Date: Sat, 9 May 2020 10:20:02 +0800 Subject: [PATCH 17/40] vdpasim: Fix some coccinelle warnings Fix below warnings reported by coccicheck: drivers/vdpa/vdpa_sim/vdpa_sim.c:104:1-10: WARNING: Assignment of 0/1 to bool variable drivers/vdpa/vdpa_sim/vdpa_sim.c:164:7-11: WARNING: Unsigned expression compared with zero: read <= 0 drivers/vdpa/vdpa_sim/vdpa_sim.c:169:7-12: WARNING: Unsigned expression compared with zero: write <= 0 1. The 'ready' variable in vdpasim_virtqueue struct is bool type. It is better to initialize vq->ready to false 2. Modify 'read' and 'write' variables type from size_t to ssize_t. And preserve the reverse christmas tree ordering of local variables. Fixes: 2c53d0f64c06 ("vdpasim: vDPA device simulator") Reported-by: Hulk Robot Signed-off-by: Samuel Zou Link: https://lore.kernel.org/r/1588990802-28451-1-git-send-email-zou_wei@huawei.com Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/vdpa_sim/vdpa_sim.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c index 01c456f7c1f7..c7334cc65bb2 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim.c +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c @@ -101,7 +101,7 @@ static void vdpasim_queue_ready(struct vdpasim *vdpasim, unsigned int idx) static void vdpasim_vq_reset(struct vdpasim_virtqueue *vq) { - vq->ready = 0; + vq->ready = false; vq->desc_addr = 0; vq->driver_addr = 0; vq->device_addr = 0; @@ -131,9 +131,10 @@ static void vdpasim_work(struct work_struct *work) vdpasim, work); struct vdpasim_virtqueue *txq = &vdpasim->vqs[1]; struct vdpasim_virtqueue *rxq = &vdpasim->vqs[0]; - size_t read, write, total_write; - int err; + ssize_t read, write; + size_t total_write; int pkts = 0; + int err; spin_lock(&vdpasim->lock); From 5f1f79bbc9e26fa9412fa9522f957bb8f030c442 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Thu, 7 May 2020 16:01:25 +0200 Subject: [PATCH 18/40] virtio-mem: Paravirtualized memory hotplug Each virtio-mem device owns exactly one memory region. It is responsible for adding/removing memory from that memory region on request. When the device driver starts up, the requested amount of memory is queried and then plugged to Linux. On request, further memory can be plugged or unplugged. This patch only implements the plugging part. On x86-64, memory can currently be plugged in 4MB ("subblock") granularity. When required, a new memory block will be added (e.g., usually 128MB on x86-64) in order to plug more subblocks. Only x86-64 was tested for now. The online_page callback is used to keep unplugged subblocks offline when onlining memory - similar to the Hyper-V balloon driver. Unplugged pages are marked PG_offline, to tell dump tools (e.g., makedumpfile) to skip them. User space is usually responsible for onlining the added memory. The memory hotplug notifier is used to synchronize virtio-mem activity against memory onlining/offlining. Each virtio-mem device can belong to a NUMA node, which allows us to easily add/remove small chunks of memory to/from a specific NUMA node by using multiple virtio-mem devices. Something that works even when the guest has no idea about the NUMA topology. One way to view virtio-mem is as a "resizable DIMM" or a DIMM with many "sub-DIMMS". This patch directly introduces the basic infrastructure to implement memory unplug. Especially the memory block states and subblock bitmaps will be heavily used there. Notes: - In case memory is to be onlined by user space, we limit the amount of offline memory blocks, to not run out of memory. This is esp. an issue if memory is added faster than it is getting onlined. - Suspend/Hibernate is not supported due to the way virtio-mem devices behave. Limited support might be possible in the future. - Reloading the device driver is not supported. Reviewed-by: Pankaj Gupta Tested-by: Pankaj Gupta Cc: "Michael S. Tsirkin" Cc: Jason Wang Cc: Oscar Salvador Cc: Michal Hocko Cc: Igor Mammedov Cc: Dave Young Cc: Andrew Morton Cc: Dan Williams Cc: Pavel Tatashin Cc: Stefan Hajnoczi Cc: Vlastimil Babka Cc: "Rafael J. Wysocki" Cc: Len Brown Cc: linux-acpi@vger.kernel.org Signed-off-by: David Hildenbrand Link: https://lore.kernel.org/r/20200507140139.17083-2-david@redhat.com Signed-off-by: Michael S. Tsirkin --- drivers/virtio/Kconfig | 16 + drivers/virtio/Makefile | 1 + drivers/virtio/virtio_mem.c | 1533 +++++++++++++++++++++++++++++++ include/uapi/linux/virtio_ids.h | 1 + include/uapi/linux/virtio_mem.h | 200 ++++ 5 files changed, 1751 insertions(+) create mode 100644 drivers/virtio/virtio_mem.c create mode 100644 include/uapi/linux/virtio_mem.h diff --git a/drivers/virtio/Kconfig b/drivers/virtio/Kconfig index 69a32dfc318a..d6dde7d2cf76 100644 --- a/drivers/virtio/Kconfig +++ b/drivers/virtio/Kconfig @@ -78,6 +78,22 @@ config VIRTIO_BALLOON If unsure, say M. +config VIRTIO_MEM + tristate "Virtio mem driver" + default m + depends on X86_64 + depends on VIRTIO + depends on MEMORY_HOTPLUG_SPARSE + depends on MEMORY_HOTREMOVE + help + This driver provides access to virtio-mem paravirtualized memory + devices, allowing to hotplug and hotunplug memory. + + This driver was only tested under x86-64, but should theoretically + work on all architectures that support memory hotplug and hotremove. + + If unsure, say M. + config VIRTIO_INPUT tristate "Virtio input driver" depends on VIRTIO diff --git a/drivers/virtio/Makefile b/drivers/virtio/Makefile index 29a1386ecc03..4d993791f2d7 100644 --- a/drivers/virtio/Makefile +++ b/drivers/virtio/Makefile @@ -7,3 +7,4 @@ virtio_pci-$(CONFIG_VIRTIO_PCI_LEGACY) += virtio_pci_legacy.o obj-$(CONFIG_VIRTIO_BALLOON) += virtio_balloon.o obj-$(CONFIG_VIRTIO_INPUT) += virtio_input.o obj-$(CONFIG_VIRTIO_VDPA) += virtio_vdpa.o +obj-$(CONFIG_VIRTIO_MEM) += virtio_mem.o diff --git a/drivers/virtio/virtio_mem.c b/drivers/virtio/virtio_mem.c new file mode 100644 index 000000000000..5d1dcaa6fc42 --- /dev/null +++ b/drivers/virtio/virtio_mem.c @@ -0,0 +1,1533 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Virtio-mem device driver. + * + * Copyright Red Hat, Inc. 2020 + * + * Author(s): David Hildenbrand + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum virtio_mem_mb_state { + /* Unplugged, not added to Linux. Can be reused later. */ + VIRTIO_MEM_MB_STATE_UNUSED = 0, + /* (Partially) plugged, not added to Linux. Error on add_memory(). */ + VIRTIO_MEM_MB_STATE_PLUGGED, + /* Fully plugged, fully added to Linux, offline. */ + VIRTIO_MEM_MB_STATE_OFFLINE, + /* Partially plugged, fully added to Linux, offline. */ + VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL, + /* Fully plugged, fully added to Linux, online (!ZONE_MOVABLE). */ + VIRTIO_MEM_MB_STATE_ONLINE, + /* Partially plugged, fully added to Linux, online (!ZONE_MOVABLE). */ + VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL, + /* + * Fully plugged, fully added to Linux, online (ZONE_MOVABLE). + * We are not allowed to allocate (unplug) parts of this block that + * are not movable (similar to gigantic pages). We will never allow + * to online OFFLINE_PARTIAL to ZONE_MOVABLE (as they would contain + * unmovable parts). + */ + VIRTIO_MEM_MB_STATE_ONLINE_MOVABLE, + VIRTIO_MEM_MB_STATE_COUNT +}; + +struct virtio_mem { + struct virtio_device *vdev; + + /* We might first have to unplug all memory when starting up. */ + bool unplug_all_required; + + /* Workqueue that processes the plug/unplug requests. */ + struct work_struct wq; + atomic_t config_changed; + + /* Virtqueue for guest->host requests. */ + struct virtqueue *vq; + + /* Wait for a host response to a guest request. */ + wait_queue_head_t host_resp; + + /* Space for one guest request and the host response. */ + struct virtio_mem_req req; + struct virtio_mem_resp resp; + + /* The current size of the device. */ + uint64_t plugged_size; + /* The requested size of the device. */ + uint64_t requested_size; + + /* The device block size (for communicating with the device). */ + uint32_t device_block_size; + /* Physical start address of the memory region. */ + uint64_t addr; + /* Maximum region size in bytes. */ + uint64_t region_size; + + /* The subblock size. */ + uint32_t subblock_size; + /* The number of subblocks per memory block. */ + uint32_t nb_sb_per_mb; + + /* Id of the first memory block of this device. */ + unsigned long first_mb_id; + /* Id of the last memory block of this device. */ + unsigned long last_mb_id; + /* Id of the last usable memory block of this device. */ + unsigned long last_usable_mb_id; + /* Id of the next memory bock to prepare when needed. */ + unsigned long next_mb_id; + + /* Summary of all memory block states. */ + unsigned long nb_mb_state[VIRTIO_MEM_MB_STATE_COUNT]; +#define VIRTIO_MEM_NB_OFFLINE_THRESHOLD 10 + + /* + * One byte state per memory block. + * + * Allocated via vmalloc(). When preparing new blocks, resized + * (alloc+copy+free) when needed (crossing pages with the next mb). + * (when crossing pages). + * + * With 128MB memory blocks, we have states for 512GB of memory in one + * page. + */ + uint8_t *mb_state; + + /* + * $nb_sb_per_mb bit per memory block. Handled similar to mb_state. + * + * With 4MB subblocks, we manage 128GB of memory in one page. + */ + unsigned long *sb_bitmap; + + /* + * Mutex that protects the nb_mb_state, mb_state, and sb_bitmap. + * + * When this lock is held the pointers can't change, ONLINE and + * OFFLINE blocks can't change the state and no subblocks will get + * plugged. + */ + struct mutex hotplug_mutex; + bool hotplug_active; + + /* An error occurred we cannot handle - stop processing requests. */ + bool broken; + + /* The driver is being removed. */ + spinlock_t removal_lock; + bool removing; + + /* Timer for retrying to plug/unplug memory. */ + struct hrtimer retry_timer; +#define VIRTIO_MEM_RETRY_TIMER_MS 30000 + + /* Memory notifier (online/offline events). */ + struct notifier_block memory_notifier; + + /* Next device in the list of virtio-mem devices. */ + struct list_head next; +}; + +/* + * We have to share a single online_page callback among all virtio-mem + * devices. We use RCU to iterate the list in the callback. + */ +static DEFINE_MUTEX(virtio_mem_mutex); +static LIST_HEAD(virtio_mem_devices); + +static void virtio_mem_online_page_cb(struct page *page, unsigned int order); + +/* + * Register a virtio-mem device so it will be considered for the online_page + * callback. + */ +static int register_virtio_mem_device(struct virtio_mem *vm) +{ + int rc = 0; + + /* First device registers the callback. */ + mutex_lock(&virtio_mem_mutex); + if (list_empty(&virtio_mem_devices)) + rc = set_online_page_callback(&virtio_mem_online_page_cb); + if (!rc) + list_add_rcu(&vm->next, &virtio_mem_devices); + mutex_unlock(&virtio_mem_mutex); + + return rc; +} + +/* + * Unregister a virtio-mem device so it will no longer be considered for the + * online_page callback. + */ +static void unregister_virtio_mem_device(struct virtio_mem *vm) +{ + /* Last device unregisters the callback. */ + mutex_lock(&virtio_mem_mutex); + list_del_rcu(&vm->next); + if (list_empty(&virtio_mem_devices)) + restore_online_page_callback(&virtio_mem_online_page_cb); + mutex_unlock(&virtio_mem_mutex); + + synchronize_rcu(); +} + +/* + * Calculate the memory block id of a given address. + */ +static unsigned long virtio_mem_phys_to_mb_id(unsigned long addr) +{ + return addr / memory_block_size_bytes(); +} + +/* + * Calculate the physical start address of a given memory block id. + */ +static unsigned long virtio_mem_mb_id_to_phys(unsigned long mb_id) +{ + return mb_id * memory_block_size_bytes(); +} + +/* + * Calculate the subblock id of a given address. + */ +static unsigned long virtio_mem_phys_to_sb_id(struct virtio_mem *vm, + unsigned long addr) +{ + const unsigned long mb_id = virtio_mem_phys_to_mb_id(addr); + const unsigned long mb_addr = virtio_mem_mb_id_to_phys(mb_id); + + return (addr - mb_addr) / vm->subblock_size; +} + +/* + * Set the state of a memory block, taking care of the state counter. + */ +static void virtio_mem_mb_set_state(struct virtio_mem *vm, unsigned long mb_id, + enum virtio_mem_mb_state state) +{ + const unsigned long idx = mb_id - vm->first_mb_id; + enum virtio_mem_mb_state old_state; + + old_state = vm->mb_state[idx]; + vm->mb_state[idx] = state; + + BUG_ON(vm->nb_mb_state[old_state] == 0); + vm->nb_mb_state[old_state]--; + vm->nb_mb_state[state]++; +} + +/* + * Get the state of a memory block. + */ +static enum virtio_mem_mb_state virtio_mem_mb_get_state(struct virtio_mem *vm, + unsigned long mb_id) +{ + const unsigned long idx = mb_id - vm->first_mb_id; + + return vm->mb_state[idx]; +} + +/* + * Prepare the state array for the next memory block. + */ +static int virtio_mem_mb_state_prepare_next_mb(struct virtio_mem *vm) +{ + unsigned long old_bytes = vm->next_mb_id - vm->first_mb_id + 1; + unsigned long new_bytes = vm->next_mb_id - vm->first_mb_id + 2; + int old_pages = PFN_UP(old_bytes); + int new_pages = PFN_UP(new_bytes); + uint8_t *new_mb_state; + + if (vm->mb_state && old_pages == new_pages) + return 0; + + new_mb_state = vzalloc(new_pages * PAGE_SIZE); + if (!new_mb_state) + return -ENOMEM; + + mutex_lock(&vm->hotplug_mutex); + if (vm->mb_state) + memcpy(new_mb_state, vm->mb_state, old_pages * PAGE_SIZE); + vfree(vm->mb_state); + vm->mb_state = new_mb_state; + mutex_unlock(&vm->hotplug_mutex); + + return 0; +} + +#define virtio_mem_for_each_mb_state(_vm, _mb_id, _state) \ + for (_mb_id = _vm->first_mb_id; \ + _mb_id < _vm->next_mb_id && _vm->nb_mb_state[_state]; \ + _mb_id++) \ + if (virtio_mem_mb_get_state(_vm, _mb_id) == _state) + +/* + * Mark all selected subblocks plugged. + * + * Will not modify the state of the memory block. + */ +static void virtio_mem_mb_set_sb_plugged(struct virtio_mem *vm, + unsigned long mb_id, int sb_id, + int count) +{ + const int bit = (mb_id - vm->first_mb_id) * vm->nb_sb_per_mb + sb_id; + + __bitmap_set(vm->sb_bitmap, bit, count); +} + +/* + * Mark all selected subblocks unplugged. + * + * Will not modify the state of the memory block. + */ +static void virtio_mem_mb_set_sb_unplugged(struct virtio_mem *vm, + unsigned long mb_id, int sb_id, + int count) +{ + const int bit = (mb_id - vm->first_mb_id) * vm->nb_sb_per_mb + sb_id; + + __bitmap_clear(vm->sb_bitmap, bit, count); +} + +/* + * Test if all selected subblocks are plugged. + */ +static bool virtio_mem_mb_test_sb_plugged(struct virtio_mem *vm, + unsigned long mb_id, int sb_id, + int count) +{ + const int bit = (mb_id - vm->first_mb_id) * vm->nb_sb_per_mb + sb_id; + + if (count == 1) + return test_bit(bit, vm->sb_bitmap); + + /* TODO: Helper similar to bitmap_set() */ + return find_next_zero_bit(vm->sb_bitmap, bit + count, bit) >= + bit + count; +} + +/* + * Find the first plugged subblock. Returns vm->nb_sb_per_mb in case there is + * none. + */ +static int virtio_mem_mb_first_plugged_sb(struct virtio_mem *vm, + unsigned long mb_id) +{ + const int bit = (mb_id - vm->first_mb_id) * vm->nb_sb_per_mb; + + return find_next_bit(vm->sb_bitmap, bit + vm->nb_sb_per_mb, bit) - bit; +} + +/* + * Find the first unplugged subblock. Returns vm->nb_sb_per_mb in case there is + * none. + */ +static int virtio_mem_mb_first_unplugged_sb(struct virtio_mem *vm, + unsigned long mb_id) +{ + const int bit = (mb_id - vm->first_mb_id) * vm->nb_sb_per_mb; + + return find_next_zero_bit(vm->sb_bitmap, bit + vm->nb_sb_per_mb, bit) - + bit; +} + +/* + * Prepare the subblock bitmap for the next memory block. + */ +static int virtio_mem_sb_bitmap_prepare_next_mb(struct virtio_mem *vm) +{ + const unsigned long old_nb_mb = vm->next_mb_id - vm->first_mb_id; + const unsigned long old_nb_bits = old_nb_mb * vm->nb_sb_per_mb; + const unsigned long new_nb_bits = (old_nb_mb + 1) * vm->nb_sb_per_mb; + int old_pages = PFN_UP(BITS_TO_LONGS(old_nb_bits) * sizeof(long)); + int new_pages = PFN_UP(BITS_TO_LONGS(new_nb_bits) * sizeof(long)); + unsigned long *new_sb_bitmap, *old_sb_bitmap; + + if (vm->sb_bitmap && old_pages == new_pages) + return 0; + + new_sb_bitmap = vzalloc(new_pages * PAGE_SIZE); + if (!new_sb_bitmap) + return -ENOMEM; + + mutex_lock(&vm->hotplug_mutex); + if (new_sb_bitmap) + memcpy(new_sb_bitmap, vm->sb_bitmap, old_pages * PAGE_SIZE); + + old_sb_bitmap = vm->sb_bitmap; + vm->sb_bitmap = new_sb_bitmap; + mutex_unlock(&vm->hotplug_mutex); + + vfree(old_sb_bitmap); + return 0; +} + +/* + * Try to add a memory block to Linux. This will usually only fail + * if out of memory. + * + * Must not be called with the vm->hotplug_mutex held (possible deadlock with + * onlining code). + * + * Will not modify the state of the memory block. + */ +static int virtio_mem_mb_add(struct virtio_mem *vm, unsigned long mb_id) +{ + const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id); + int nid = memory_add_physaddr_to_nid(addr); + + dev_dbg(&vm->vdev->dev, "adding memory block: %lu\n", mb_id); + return add_memory(nid, addr, memory_block_size_bytes()); +} + +/* + * Try to remove a memory block from Linux. Will only fail if the memory block + * is not offline. + * + * Must not be called with the vm->hotplug_mutex held (possible deadlock with + * onlining code). + * + * Will not modify the state of the memory block. + */ +static int virtio_mem_mb_remove(struct virtio_mem *vm, unsigned long mb_id) +{ + const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id); + int nid = memory_add_physaddr_to_nid(addr); + + dev_dbg(&vm->vdev->dev, "removing memory block: %lu\n", mb_id); + return remove_memory(nid, addr, memory_block_size_bytes()); +} + +/* + * Trigger the workqueue so the device can perform its magic. + */ +static void virtio_mem_retry(struct virtio_mem *vm) +{ + unsigned long flags; + + spin_lock_irqsave(&vm->removal_lock, flags); + if (!vm->removing) + queue_work(system_freezable_wq, &vm->wq); + spin_unlock_irqrestore(&vm->removal_lock, flags); +} + +/* + * Test if a virtio-mem device overlaps with the given range. Can be called + * from (notifier) callbacks lockless. + */ +static bool virtio_mem_overlaps_range(struct virtio_mem *vm, + unsigned long start, unsigned long size) +{ + unsigned long dev_start = virtio_mem_mb_id_to_phys(vm->first_mb_id); + unsigned long dev_end = virtio_mem_mb_id_to_phys(vm->last_mb_id) + + memory_block_size_bytes(); + + return start < dev_end && dev_start < start + size; +} + +/* + * Test if a virtio-mem device owns a memory block. Can be called from + * (notifier) callbacks lockless. + */ +static bool virtio_mem_owned_mb(struct virtio_mem *vm, unsigned long mb_id) +{ + return mb_id >= vm->first_mb_id && mb_id <= vm->last_mb_id; +} + +static int virtio_mem_notify_going_online(struct virtio_mem *vm, + unsigned long mb_id, + enum zone_type zone) +{ + switch (virtio_mem_mb_get_state(vm, mb_id)) { + case VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL: + /* + * We won't allow to online a partially plugged memory block + * to the MOVABLE zone - it would contain unmovable parts. + */ + if (zone == ZONE_MOVABLE) { + dev_warn_ratelimited(&vm->vdev->dev, + "memory block has holes, MOVABLE not supported\n"); + return NOTIFY_BAD; + } + return NOTIFY_OK; + case VIRTIO_MEM_MB_STATE_OFFLINE: + return NOTIFY_OK; + default: + break; + } + dev_warn_ratelimited(&vm->vdev->dev, + "memory block onlining denied\n"); + return NOTIFY_BAD; +} + +static void virtio_mem_notify_offline(struct virtio_mem *vm, + unsigned long mb_id) +{ + switch (virtio_mem_mb_get_state(vm, mb_id)) { + case VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL: + virtio_mem_mb_set_state(vm, mb_id, + VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL); + break; + case VIRTIO_MEM_MB_STATE_ONLINE: + case VIRTIO_MEM_MB_STATE_ONLINE_MOVABLE: + virtio_mem_mb_set_state(vm, mb_id, + VIRTIO_MEM_MB_STATE_OFFLINE); + break; + default: + BUG(); + break; + } +} + +static void virtio_mem_notify_online(struct virtio_mem *vm, unsigned long mb_id, + enum zone_type zone) +{ + unsigned long nb_offline; + + switch (virtio_mem_mb_get_state(vm, mb_id)) { + case VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL: + BUG_ON(zone == ZONE_MOVABLE); + virtio_mem_mb_set_state(vm, mb_id, + VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL); + break; + case VIRTIO_MEM_MB_STATE_OFFLINE: + if (zone == ZONE_MOVABLE) + virtio_mem_mb_set_state(vm, mb_id, + VIRTIO_MEM_MB_STATE_ONLINE_MOVABLE); + else + virtio_mem_mb_set_state(vm, mb_id, + VIRTIO_MEM_MB_STATE_ONLINE); + break; + default: + BUG(); + break; + } + nb_offline = vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE] + + vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL]; + + /* see if we can add new blocks now that we onlined one block */ + if (nb_offline == VIRTIO_MEM_NB_OFFLINE_THRESHOLD - 1) + virtio_mem_retry(vm); +} + +/* + * This callback will either be called synchronously from add_memory() or + * asynchronously (e.g., triggered via user space). We have to be careful + * with locking when calling add_memory(). + */ +static int virtio_mem_memory_notifier_cb(struct notifier_block *nb, + unsigned long action, void *arg) +{ + struct virtio_mem *vm = container_of(nb, struct virtio_mem, + memory_notifier); + struct memory_notify *mhp = arg; + const unsigned long start = PFN_PHYS(mhp->start_pfn); + const unsigned long size = PFN_PHYS(mhp->nr_pages); + const unsigned long mb_id = virtio_mem_phys_to_mb_id(start); + enum zone_type zone; + int rc = NOTIFY_OK; + + if (!virtio_mem_overlaps_range(vm, start, size)) + return NOTIFY_DONE; + + /* + * Memory is onlined/offlined in memory block granularity. We cannot + * cross virtio-mem device boundaries and memory block boundaries. Bail + * out if this ever changes. + */ + if (WARN_ON_ONCE(size != memory_block_size_bytes() || + !IS_ALIGNED(start, memory_block_size_bytes()))) + return NOTIFY_BAD; + + /* + * Avoid circular locking lockdep warnings. We lock the mutex + * e.g., in MEM_GOING_ONLINE and unlock it in MEM_ONLINE. The + * blocking_notifier_call_chain() has it's own lock, which gets unlocked + * between both notifier calls and will bail out. False positive. + */ + lockdep_off(); + + switch (action) { + case MEM_GOING_OFFLINE: + mutex_lock(&vm->hotplug_mutex); + if (vm->removing) { + rc = notifier_from_errno(-EBUSY); + mutex_unlock(&vm->hotplug_mutex); + break; + } + vm->hotplug_active = true; + break; + case MEM_GOING_ONLINE: + mutex_lock(&vm->hotplug_mutex); + if (vm->removing) { + rc = notifier_from_errno(-EBUSY); + mutex_unlock(&vm->hotplug_mutex); + break; + } + vm->hotplug_active = true; + zone = page_zonenum(pfn_to_page(mhp->start_pfn)); + rc = virtio_mem_notify_going_online(vm, mb_id, zone); + break; + case MEM_OFFLINE: + virtio_mem_notify_offline(vm, mb_id); + vm->hotplug_active = false; + mutex_unlock(&vm->hotplug_mutex); + break; + case MEM_ONLINE: + zone = page_zonenum(pfn_to_page(mhp->start_pfn)); + virtio_mem_notify_online(vm, mb_id, zone); + vm->hotplug_active = false; + mutex_unlock(&vm->hotplug_mutex); + break; + case MEM_CANCEL_OFFLINE: + case MEM_CANCEL_ONLINE: + if (!vm->hotplug_active) + break; + vm->hotplug_active = false; + mutex_unlock(&vm->hotplug_mutex); + break; + default: + break; + } + + lockdep_on(); + + return rc; +} + +/* + * Set a range of pages PG_offline. + */ +static void virtio_mem_set_fake_offline(unsigned long pfn, + unsigned int nr_pages) +{ + for (; nr_pages--; pfn++) + __SetPageOffline(pfn_to_page(pfn)); +} + +/* + * Clear PG_offline from a range of pages. + */ +static void virtio_mem_clear_fake_offline(unsigned long pfn, + unsigned int nr_pages) +{ + for (; nr_pages--; pfn++) + __ClearPageOffline(pfn_to_page(pfn)); +} + +/* + * Release a range of fake-offline pages to the buddy, effectively + * fake-onlining them. + */ +static void virtio_mem_fake_online(unsigned long pfn, unsigned int nr_pages) +{ + const int order = MAX_ORDER - 1; + int i; + + /* + * We are always called with subblock granularity, which is at least + * aligned to MAX_ORDER - 1. + */ + virtio_mem_clear_fake_offline(pfn, nr_pages); + + for (i = 0; i < nr_pages; i += 1 << order) + generic_online_page(pfn_to_page(pfn + i), order); +} + +static void virtio_mem_online_page_cb(struct page *page, unsigned int order) +{ + const unsigned long addr = page_to_phys(page); + const unsigned long mb_id = virtio_mem_phys_to_mb_id(addr); + struct virtio_mem *vm; + int sb_id; + + /* + * We exploit here that subblocks have at least MAX_ORDER - 1 + * size/alignment and that this callback is is called with such a + * size/alignment. So we cannot cross subblocks and therefore + * also not memory blocks. + */ + rcu_read_lock(); + list_for_each_entry_rcu(vm, &virtio_mem_devices, next) { + if (!virtio_mem_owned_mb(vm, mb_id)) + continue; + + sb_id = virtio_mem_phys_to_sb_id(vm, addr); + /* + * If plugged, online the pages, otherwise, set them fake + * offline (PageOffline). + */ + if (virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id, 1)) + generic_online_page(page, order); + else + virtio_mem_set_fake_offline(PFN_DOWN(addr), 1 << order); + rcu_read_unlock(); + return; + } + rcu_read_unlock(); + + /* not virtio-mem memory, but e.g., a DIMM. online it */ + generic_online_page(page, order); +} + +static uint64_t virtio_mem_send_request(struct virtio_mem *vm, + const struct virtio_mem_req *req) +{ + struct scatterlist *sgs[2], sg_req, sg_resp; + unsigned int len; + int rc; + + /* don't use the request residing on the stack (vaddr) */ + vm->req = *req; + + /* out: buffer for request */ + sg_init_one(&sg_req, &vm->req, sizeof(vm->req)); + sgs[0] = &sg_req; + + /* in: buffer for response */ + sg_init_one(&sg_resp, &vm->resp, sizeof(vm->resp)); + sgs[1] = &sg_resp; + + rc = virtqueue_add_sgs(vm->vq, sgs, 1, 1, vm, GFP_KERNEL); + if (rc < 0) + return rc; + + virtqueue_kick(vm->vq); + + /* wait for a response */ + wait_event(vm->host_resp, virtqueue_get_buf(vm->vq, &len)); + + return virtio16_to_cpu(vm->vdev, vm->resp.type); +} + +static int virtio_mem_send_plug_request(struct virtio_mem *vm, uint64_t addr, + uint64_t size) +{ + const uint64_t nb_vm_blocks = size / vm->device_block_size; + const struct virtio_mem_req req = { + .type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_PLUG), + .u.plug.addr = cpu_to_virtio64(vm->vdev, addr), + .u.plug.nb_blocks = cpu_to_virtio16(vm->vdev, nb_vm_blocks), + }; + + if (atomic_read(&vm->config_changed)) + return -EAGAIN; + + switch (virtio_mem_send_request(vm, &req)) { + case VIRTIO_MEM_RESP_ACK: + vm->plugged_size += size; + return 0; + case VIRTIO_MEM_RESP_NACK: + return -EAGAIN; + case VIRTIO_MEM_RESP_BUSY: + return -EBUSY; + case VIRTIO_MEM_RESP_ERROR: + return -EINVAL; + default: + return -ENOMEM; + } +} + +static int virtio_mem_send_unplug_request(struct virtio_mem *vm, uint64_t addr, + uint64_t size) +{ + const uint64_t nb_vm_blocks = size / vm->device_block_size; + const struct virtio_mem_req req = { + .type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_UNPLUG), + .u.unplug.addr = cpu_to_virtio64(vm->vdev, addr), + .u.unplug.nb_blocks = cpu_to_virtio16(vm->vdev, nb_vm_blocks), + }; + + if (atomic_read(&vm->config_changed)) + return -EAGAIN; + + switch (virtio_mem_send_request(vm, &req)) { + case VIRTIO_MEM_RESP_ACK: + vm->plugged_size -= size; + return 0; + case VIRTIO_MEM_RESP_BUSY: + return -EBUSY; + case VIRTIO_MEM_RESP_ERROR: + return -EINVAL; + default: + return -ENOMEM; + } +} + +static int virtio_mem_send_unplug_all_request(struct virtio_mem *vm) +{ + const struct virtio_mem_req req = { + .type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_UNPLUG_ALL), + }; + + switch (virtio_mem_send_request(vm, &req)) { + case VIRTIO_MEM_RESP_ACK: + vm->unplug_all_required = false; + vm->plugged_size = 0; + /* usable region might have shrunk */ + atomic_set(&vm->config_changed, 1); + return 0; + case VIRTIO_MEM_RESP_BUSY: + return -EBUSY; + default: + return -ENOMEM; + } +} + +/* + * Plug selected subblocks. Updates the plugged state, but not the state + * of the memory block. + */ +static int virtio_mem_mb_plug_sb(struct virtio_mem *vm, unsigned long mb_id, + int sb_id, int count) +{ + const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id) + + sb_id * vm->subblock_size; + const uint64_t size = count * vm->subblock_size; + int rc; + + dev_dbg(&vm->vdev->dev, "plugging memory block: %lu : %i - %i\n", mb_id, + sb_id, sb_id + count - 1); + + rc = virtio_mem_send_plug_request(vm, addr, size); + if (!rc) + virtio_mem_mb_set_sb_plugged(vm, mb_id, sb_id, count); + return rc; +} + +/* + * Unplug selected subblocks. Updates the plugged state, but not the state + * of the memory block. + */ +static int virtio_mem_mb_unplug_sb(struct virtio_mem *vm, unsigned long mb_id, + int sb_id, int count) +{ + const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id) + + sb_id * vm->subblock_size; + const uint64_t size = count * vm->subblock_size; + int rc; + + dev_dbg(&vm->vdev->dev, "unplugging memory block: %lu : %i - %i\n", + mb_id, sb_id, sb_id + count - 1); + + rc = virtio_mem_send_unplug_request(vm, addr, size); + if (!rc) + virtio_mem_mb_set_sb_unplugged(vm, mb_id, sb_id, count); + return rc; +} + +/* + * Unplug the desired number of plugged subblocks of a offline or not-added + * memory block. Will fail if any subblock cannot get unplugged (instead of + * skipping it). + * + * Will not modify the state of the memory block. + * + * Note: can fail after some subblocks were unplugged. + */ +static int virtio_mem_mb_unplug_any_sb(struct virtio_mem *vm, + unsigned long mb_id, uint64_t *nb_sb) +{ + int sb_id, count; + int rc; + + while (*nb_sb) { + sb_id = virtio_mem_mb_first_plugged_sb(vm, mb_id); + if (sb_id >= vm->nb_sb_per_mb) + break; + count = 1; + while (count < *nb_sb && + sb_id + count < vm->nb_sb_per_mb && + virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id + count, + 1)) + count++; + + rc = virtio_mem_mb_unplug_sb(vm, mb_id, sb_id, count); + if (rc) + return rc; + *nb_sb -= count; + } + + return 0; +} + +/* + * Unplug all plugged subblocks of an offline or not-added memory block. + * + * Will not modify the state of the memory block. + * + * Note: can fail after some subblocks were unplugged. + */ +static int virtio_mem_mb_unplug(struct virtio_mem *vm, unsigned long mb_id) +{ + uint64_t nb_sb = vm->nb_sb_per_mb; + + return virtio_mem_mb_unplug_any_sb(vm, mb_id, &nb_sb); +} + +/* + * Prepare tracking data for the next memory block. + */ +static int virtio_mem_prepare_next_mb(struct virtio_mem *vm, + unsigned long *mb_id) +{ + int rc; + + if (vm->next_mb_id > vm->last_usable_mb_id) + return -ENOSPC; + + /* Resize the state array if required. */ + rc = virtio_mem_mb_state_prepare_next_mb(vm); + if (rc) + return rc; + + /* Resize the subblock bitmap if required. */ + rc = virtio_mem_sb_bitmap_prepare_next_mb(vm); + if (rc) + return rc; + + vm->nb_mb_state[VIRTIO_MEM_MB_STATE_UNUSED]++; + *mb_id = vm->next_mb_id++; + return 0; +} + +/* + * Don't add too many blocks that are not onlined yet to avoid running OOM. + */ +static bool virtio_mem_too_many_mb_offline(struct virtio_mem *vm) +{ + unsigned long nb_offline; + + nb_offline = vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE] + + vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL]; + return nb_offline >= VIRTIO_MEM_NB_OFFLINE_THRESHOLD; +} + +/* + * Try to plug the desired number of subblocks and add the memory block + * to Linux. + * + * Will modify the state of the memory block. + */ +static int virtio_mem_mb_plug_and_add(struct virtio_mem *vm, + unsigned long mb_id, + uint64_t *nb_sb) +{ + const int count = min_t(int, *nb_sb, vm->nb_sb_per_mb); + int rc, rc2; + + if (WARN_ON_ONCE(!count)) + return -EINVAL; + + /* + * Plug the requested number of subblocks before adding it to linux, + * so that onlining will directly online all plugged subblocks. + */ + rc = virtio_mem_mb_plug_sb(vm, mb_id, 0, count); + if (rc) + return rc; + + /* + * Mark the block properly offline before adding it to Linux, + * so the memory notifiers will find the block in the right state. + */ + if (count == vm->nb_sb_per_mb) + virtio_mem_mb_set_state(vm, mb_id, + VIRTIO_MEM_MB_STATE_OFFLINE); + else + virtio_mem_mb_set_state(vm, mb_id, + VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL); + + /* Add the memory block to linux - if that fails, try to unplug. */ + rc = virtio_mem_mb_add(vm, mb_id); + if (rc) { + enum virtio_mem_mb_state new_state = VIRTIO_MEM_MB_STATE_UNUSED; + + dev_err(&vm->vdev->dev, + "adding memory block %lu failed with %d\n", mb_id, rc); + rc2 = virtio_mem_mb_unplug_sb(vm, mb_id, 0, count); + + /* + * TODO: Linux MM does not properly clean up yet in all cases + * where adding of memory failed - especially on -ENOMEM. + */ + if (rc2) + new_state = VIRTIO_MEM_MB_STATE_PLUGGED; + virtio_mem_mb_set_state(vm, mb_id, new_state); + return rc; + } + + *nb_sb -= count; + return 0; +} + +/* + * Try to plug the desired number of subblocks of a memory block that + * is already added to Linux. + * + * Will modify the state of the memory block. + * + * Note: Can fail after some subblocks were successfully plugged. + */ +static int virtio_mem_mb_plug_any_sb(struct virtio_mem *vm, unsigned long mb_id, + uint64_t *nb_sb, bool online) +{ + unsigned long pfn, nr_pages; + int sb_id, count; + int rc; + + if (WARN_ON_ONCE(!*nb_sb)) + return -EINVAL; + + while (*nb_sb) { + sb_id = virtio_mem_mb_first_unplugged_sb(vm, mb_id); + if (sb_id >= vm->nb_sb_per_mb) + break; + count = 1; + while (count < *nb_sb && + sb_id + count < vm->nb_sb_per_mb && + !virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id + count, + 1)) + count++; + + rc = virtio_mem_mb_plug_sb(vm, mb_id, sb_id, count); + if (rc) + return rc; + *nb_sb -= count; + if (!online) + continue; + + /* fake-online the pages if the memory block is online */ + pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) + + sb_id * vm->subblock_size); + nr_pages = PFN_DOWN(count * vm->subblock_size); + virtio_mem_fake_online(pfn, nr_pages); + } + + if (virtio_mem_mb_test_sb_plugged(vm, mb_id, 0, vm->nb_sb_per_mb)) { + if (online) + virtio_mem_mb_set_state(vm, mb_id, + VIRTIO_MEM_MB_STATE_ONLINE); + else + virtio_mem_mb_set_state(vm, mb_id, + VIRTIO_MEM_MB_STATE_OFFLINE); + } + + return rc; +} + +/* + * Try to plug the requested amount of memory. + */ +static int virtio_mem_plug_request(struct virtio_mem *vm, uint64_t diff) +{ + uint64_t nb_sb = diff / vm->subblock_size; + unsigned long mb_id; + int rc; + + if (!nb_sb) + return 0; + + /* Don't race with onlining/offlining */ + mutex_lock(&vm->hotplug_mutex); + + /* Try to plug subblocks of partially plugged online blocks. */ + virtio_mem_for_each_mb_state(vm, mb_id, + VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL) { + rc = virtio_mem_mb_plug_any_sb(vm, mb_id, &nb_sb, true); + if (rc || !nb_sb) + goto out_unlock; + cond_resched(); + } + + /* Try to plug subblocks of partially plugged offline blocks. */ + virtio_mem_for_each_mb_state(vm, mb_id, + VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL) { + rc = virtio_mem_mb_plug_any_sb(vm, mb_id, &nb_sb, false); + if (rc || !nb_sb) + goto out_unlock; + cond_resched(); + } + + /* + * We won't be working on online/offline memory blocks from this point, + * so we can't race with memory onlining/offlining. Drop the mutex. + */ + mutex_unlock(&vm->hotplug_mutex); + + /* Try to plug and add unused blocks */ + virtio_mem_for_each_mb_state(vm, mb_id, VIRTIO_MEM_MB_STATE_UNUSED) { + if (virtio_mem_too_many_mb_offline(vm)) + return -ENOSPC; + + rc = virtio_mem_mb_plug_and_add(vm, mb_id, &nb_sb); + if (rc || !nb_sb) + return rc; + cond_resched(); + } + + /* Try to prepare, plug and add new blocks */ + while (nb_sb) { + if (virtio_mem_too_many_mb_offline(vm)) + return -ENOSPC; + + rc = virtio_mem_prepare_next_mb(vm, &mb_id); + if (rc) + return rc; + rc = virtio_mem_mb_plug_and_add(vm, mb_id, &nb_sb); + if (rc) + return rc; + cond_resched(); + } + + return 0; +out_unlock: + mutex_unlock(&vm->hotplug_mutex); + return rc; +} + +/* + * Try to unplug all blocks that couldn't be unplugged before, for example, + * because the hypervisor was busy. + */ +static int virtio_mem_unplug_pending_mb(struct virtio_mem *vm) +{ + unsigned long mb_id; + int rc; + + virtio_mem_for_each_mb_state(vm, mb_id, VIRTIO_MEM_MB_STATE_PLUGGED) { + rc = virtio_mem_mb_unplug(vm, mb_id); + if (rc) + return rc; + virtio_mem_mb_set_state(vm, mb_id, VIRTIO_MEM_MB_STATE_UNUSED); + } + + return 0; +} + +/* + * Update all parts of the config that could have changed. + */ +static void virtio_mem_refresh_config(struct virtio_mem *vm) +{ + const uint64_t phys_limit = 1UL << MAX_PHYSMEM_BITS; + uint64_t new_plugged_size, usable_region_size, end_addr; + + /* the plugged_size is just a reflection of what _we_ did previously */ + virtio_cread(vm->vdev, struct virtio_mem_config, plugged_size, + &new_plugged_size); + if (WARN_ON_ONCE(new_plugged_size != vm->plugged_size)) + vm->plugged_size = new_plugged_size; + + /* calculate the last usable memory block id */ + virtio_cread(vm->vdev, struct virtio_mem_config, + usable_region_size, &usable_region_size); + end_addr = vm->addr + usable_region_size; + end_addr = min(end_addr, phys_limit); + vm->last_usable_mb_id = virtio_mem_phys_to_mb_id(end_addr) - 1; + + /* see if there is a request to change the size */ + virtio_cread(vm->vdev, struct virtio_mem_config, requested_size, + &vm->requested_size); + + dev_info(&vm->vdev->dev, "plugged size: 0x%llx", vm->plugged_size); + dev_info(&vm->vdev->dev, "requested size: 0x%llx", vm->requested_size); +} + +/* + * Workqueue function for handling plug/unplug requests and config updates. + */ +static void virtio_mem_run_wq(struct work_struct *work) +{ + struct virtio_mem *vm = container_of(work, struct virtio_mem, wq); + uint64_t diff; + int rc; + + hrtimer_cancel(&vm->retry_timer); + + if (vm->broken) + return; + +retry: + rc = 0; + + /* Make sure we start with a clean state if there are leftovers. */ + if (unlikely(vm->unplug_all_required)) + rc = virtio_mem_send_unplug_all_request(vm); + + if (atomic_read(&vm->config_changed)) { + atomic_set(&vm->config_changed, 0); + virtio_mem_refresh_config(vm); + } + + /* Unplug any leftovers from previous runs */ + if (!rc) + rc = virtio_mem_unplug_pending_mb(vm); + + if (!rc && vm->requested_size != vm->plugged_size) { + if (vm->requested_size > vm->plugged_size) { + diff = vm->requested_size - vm->plugged_size; + rc = virtio_mem_plug_request(vm, diff); + } + /* TODO: try to unplug memory */ + } + + switch (rc) { + case 0: + break; + case -ENOSPC: + /* + * We cannot add any more memory (alignment, physical limit) + * or we have too many offline memory blocks. + */ + break; + case -EBUSY: + /* + * The hypervisor cannot process our request right now + * (e.g., out of memory, migrating). + */ + case -ENOMEM: + /* Out of memory, try again later. */ + hrtimer_start(&vm->retry_timer, + ms_to_ktime(VIRTIO_MEM_RETRY_TIMER_MS), + HRTIMER_MODE_REL); + break; + case -EAGAIN: + /* Retry immediately (e.g., the config changed). */ + goto retry; + default: + /* Unknown error, mark as broken */ + dev_err(&vm->vdev->dev, + "unknown error, marking device broken: %d\n", rc); + vm->broken = true; + } +} + +static enum hrtimer_restart virtio_mem_timer_expired(struct hrtimer *timer) +{ + struct virtio_mem *vm = container_of(timer, struct virtio_mem, + retry_timer); + + virtio_mem_retry(vm); + return HRTIMER_NORESTART; +} + +static void virtio_mem_handle_response(struct virtqueue *vq) +{ + struct virtio_mem *vm = vq->vdev->priv; + + wake_up(&vm->host_resp); +} + +static int virtio_mem_init_vq(struct virtio_mem *vm) +{ + struct virtqueue *vq; + + vq = virtio_find_single_vq(vm->vdev, virtio_mem_handle_response, + "guest-request"); + if (IS_ERR(vq)) + return PTR_ERR(vq); + vm->vq = vq; + + return 0; +} + +/* + * Test if any memory in the range is present in Linux. + */ +static bool virtio_mem_any_memory_present(unsigned long start, + unsigned long size) +{ + const unsigned long start_pfn = PFN_DOWN(start); + const unsigned long end_pfn = PFN_UP(start + size); + unsigned long pfn; + + for (pfn = start_pfn; pfn != end_pfn; pfn++) + if (present_section_nr(pfn_to_section_nr(pfn))) + return true; + + return false; +} + +static int virtio_mem_init(struct virtio_mem *vm) +{ + const uint64_t phys_limit = 1UL << MAX_PHYSMEM_BITS; + + if (!vm->vdev->config->get) { + dev_err(&vm->vdev->dev, "config access disabled\n"); + return -EINVAL; + } + + /* + * We don't want to (un)plug or reuse any memory when in kdump. The + * memory is still accessible (but not mapped). + */ + if (is_kdump_kernel()) { + dev_warn(&vm->vdev->dev, "disabled in kdump kernel\n"); + return -EBUSY; + } + + /* Fetch all properties that can't change. */ + virtio_cread(vm->vdev, struct virtio_mem_config, plugged_size, + &vm->plugged_size); + virtio_cread(vm->vdev, struct virtio_mem_config, block_size, + &vm->device_block_size); + virtio_cread(vm->vdev, struct virtio_mem_config, addr, &vm->addr); + virtio_cread(vm->vdev, struct virtio_mem_config, region_size, + &vm->region_size); + + /* + * If we still have memory plugged, we might have to unplug all + * memory first. However, if somebody simply unloaded the driver + * we would have to reinitialize the old state - something we don't + * support yet. Detect if we have any memory in the area present. + */ + if (vm->plugged_size) { + uint64_t usable_region_size; + + virtio_cread(vm->vdev, struct virtio_mem_config, + usable_region_size, &usable_region_size); + + if (virtio_mem_any_memory_present(vm->addr, + usable_region_size)) { + dev_err(&vm->vdev->dev, + "reloading the driver is not supported\n"); + return -EINVAL; + } + /* + * Note: it might happen that the device is busy and + * unplugging all memory might take some time. + */ + dev_info(&vm->vdev->dev, "unplugging all memory required\n"); + vm->unplug_all_required = 1; + } + + /* + * We always hotplug memory in memory block granularity. This way, + * we have to wait for exactly one memory block to online. + */ + if (vm->device_block_size > memory_block_size_bytes()) { + dev_err(&vm->vdev->dev, + "The block size is not supported (too big).\n"); + return -EINVAL; + } + + /* bad device setup - warn only */ + if (!IS_ALIGNED(vm->addr, memory_block_size_bytes())) + dev_warn(&vm->vdev->dev, + "The alignment of the physical start address can make some memory unusable.\n"); + if (!IS_ALIGNED(vm->addr + vm->region_size, memory_block_size_bytes())) + dev_warn(&vm->vdev->dev, + "The alignment of the physical end address can make some memory unusable.\n"); + if (vm->addr + vm->region_size > phys_limit) + dev_warn(&vm->vdev->dev, + "Some memory is not addressable. This can make some memory unusable.\n"); + + /* + * Calculate the subblock size: + * - At least MAX_ORDER - 1 / pageblock_order. + * - At least the device block size. + * In the worst case, a single subblock per memory block. + */ + vm->subblock_size = PAGE_SIZE * 1u << max_t(uint32_t, MAX_ORDER - 1, + pageblock_order); + vm->subblock_size = max_t(uint32_t, vm->device_block_size, + vm->subblock_size); + vm->nb_sb_per_mb = memory_block_size_bytes() / vm->subblock_size; + + /* Round up to the next full memory block */ + vm->first_mb_id = virtio_mem_phys_to_mb_id(vm->addr - 1 + + memory_block_size_bytes()); + vm->next_mb_id = vm->first_mb_id; + vm->last_mb_id = virtio_mem_phys_to_mb_id(vm->addr + + vm->region_size) - 1; + + dev_info(&vm->vdev->dev, "start address: 0x%llx", vm->addr); + dev_info(&vm->vdev->dev, "region size: 0x%llx", vm->region_size); + dev_info(&vm->vdev->dev, "device block size: 0x%x", + vm->device_block_size); + dev_info(&vm->vdev->dev, "memory block size: 0x%lx", + memory_block_size_bytes()); + dev_info(&vm->vdev->dev, "subblock size: 0x%x", + vm->subblock_size); + + return 0; +} + +static int virtio_mem_probe(struct virtio_device *vdev) +{ + struct virtio_mem *vm; + int rc = -EINVAL; + + vdev->priv = vm = kzalloc(sizeof(*vm), GFP_KERNEL); + if (!vm) + return -ENOMEM; + + init_waitqueue_head(&vm->host_resp); + vm->vdev = vdev; + INIT_WORK(&vm->wq, virtio_mem_run_wq); + mutex_init(&vm->hotplug_mutex); + INIT_LIST_HEAD(&vm->next); + spin_lock_init(&vm->removal_lock); + hrtimer_init(&vm->retry_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + vm->retry_timer.function = virtio_mem_timer_expired; + + /* register the virtqueue */ + rc = virtio_mem_init_vq(vm); + if (rc) + goto out_free_vm; + + /* initialize the device by querying the config */ + rc = virtio_mem_init(vm); + if (rc) + goto out_del_vq; + + /* register callbacks */ + vm->memory_notifier.notifier_call = virtio_mem_memory_notifier_cb; + rc = register_memory_notifier(&vm->memory_notifier); + if (rc) + goto out_del_vq; + rc = register_virtio_mem_device(vm); + if (rc) + goto out_unreg_mem; + + virtio_device_ready(vdev); + + /* trigger a config update to start processing the requested_size */ + atomic_set(&vm->config_changed, 1); + queue_work(system_freezable_wq, &vm->wq); + + return 0; +out_unreg_mem: + unregister_memory_notifier(&vm->memory_notifier); +out_del_vq: + vdev->config->del_vqs(vdev); +out_free_vm: + kfree(vm); + vdev->priv = NULL; + + return rc; +} + +static void virtio_mem_remove(struct virtio_device *vdev) +{ + struct virtio_mem *vm = vdev->priv; + unsigned long mb_id; + int rc; + + /* + * Make sure the workqueue won't be triggered anymore and no memory + * blocks can be onlined/offlined until we're finished here. + */ + mutex_lock(&vm->hotplug_mutex); + spin_lock_irq(&vm->removal_lock); + vm->removing = true; + spin_unlock_irq(&vm->removal_lock); + mutex_unlock(&vm->hotplug_mutex); + + /* wait until the workqueue stopped */ + cancel_work_sync(&vm->wq); + hrtimer_cancel(&vm->retry_timer); + + /* + * After we unregistered our callbacks, user space can online partially + * plugged offline blocks. Make sure to remove them. + */ + virtio_mem_for_each_mb_state(vm, mb_id, + VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL) { + rc = virtio_mem_mb_remove(vm, mb_id); + BUG_ON(rc); + virtio_mem_mb_set_state(vm, mb_id, VIRTIO_MEM_MB_STATE_UNUSED); + } + + /* unregister callbacks */ + unregister_virtio_mem_device(vm); + unregister_memory_notifier(&vm->memory_notifier); + + /* + * There is no way we could reliably remove all memory we have added to + * the system. And there is no way to stop the driver/device from going + * away. Warn at least. + */ + if (vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE] || + vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL] || + vm->nb_mb_state[VIRTIO_MEM_MB_STATE_ONLINE] || + vm->nb_mb_state[VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL] || + vm->nb_mb_state[VIRTIO_MEM_MB_STATE_ONLINE_MOVABLE]) + dev_warn(&vdev->dev, "device still has system memory added\n"); + + /* remove all tracking data - no locking needed */ + vfree(vm->mb_state); + vfree(vm->sb_bitmap); + + /* reset the device and cleanup the queues */ + vdev->config->reset(vdev); + vdev->config->del_vqs(vdev); + + kfree(vm); + vdev->priv = NULL; +} + +static void virtio_mem_config_changed(struct virtio_device *vdev) +{ + struct virtio_mem *vm = vdev->priv; + + atomic_set(&vm->config_changed, 1); + virtio_mem_retry(vm); +} + +#ifdef CONFIG_PM_SLEEP +static int virtio_mem_freeze(struct virtio_device *vdev) +{ + /* + * When restarting the VM, all memory is usually unplugged. Don't + * allow to suspend/hibernate. + */ + dev_err(&vdev->dev, "save/restore not supported.\n"); + return -EPERM; +} + +static int virtio_mem_restore(struct virtio_device *vdev) +{ + return -EPERM; +} +#endif + +static struct virtio_device_id virtio_mem_id_table[] = { + { VIRTIO_ID_MEM, VIRTIO_DEV_ANY_ID }, + { 0 }, +}; + +static struct virtio_driver virtio_mem_driver = { + .driver.name = KBUILD_MODNAME, + .driver.owner = THIS_MODULE, + .id_table = virtio_mem_id_table, + .probe = virtio_mem_probe, + .remove = virtio_mem_remove, + .config_changed = virtio_mem_config_changed, +#ifdef CONFIG_PM_SLEEP + .freeze = virtio_mem_freeze, + .restore = virtio_mem_restore, +#endif +}; + +module_virtio_driver(virtio_mem_driver); +MODULE_DEVICE_TABLE(virtio, virtio_mem_id_table); +MODULE_AUTHOR("David Hildenbrand "); +MODULE_DESCRIPTION("Virtio-mem driver"); +MODULE_LICENSE("GPL"); diff --git a/include/uapi/linux/virtio_ids.h b/include/uapi/linux/virtio_ids.h index ecc27a17401a..b052355ac7a3 100644 --- a/include/uapi/linux/virtio_ids.h +++ b/include/uapi/linux/virtio_ids.h @@ -44,6 +44,7 @@ #define VIRTIO_ID_VSOCK 19 /* virtio vsock transport */ #define VIRTIO_ID_CRYPTO 20 /* virtio crypto */ #define VIRTIO_ID_IOMMU 23 /* virtio IOMMU */ +#define VIRTIO_ID_MEM 24 /* virtio mem */ #define VIRTIO_ID_FS 26 /* virtio filesystem */ #define VIRTIO_ID_PMEM 27 /* virtio pmem */ #define VIRTIO_ID_MAC80211_HWSIM 29 /* virtio mac80211-hwsim */ diff --git a/include/uapi/linux/virtio_mem.h b/include/uapi/linux/virtio_mem.h new file mode 100644 index 000000000000..1bfade78bdfd --- /dev/null +++ b/include/uapi/linux/virtio_mem.h @@ -0,0 +1,200 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Virtio Mem Device + * + * Copyright Red Hat, Inc. 2020 + * + * Authors: + * David Hildenbrand + * + * This header is BSD licensed so anyone can use the definitions + * to implement compatible drivers/servers: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of IBM nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL IBM OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _LINUX_VIRTIO_MEM_H +#define _LINUX_VIRTIO_MEM_H + +#include +#include +#include +#include + +/* + * Each virtio-mem device manages a dedicated region in physical address + * space. Each device can belong to a single NUMA node, multiple devices + * for a single NUMA node are possible. A virtio-mem device is like a + * "resizable DIMM" consisting of small memory blocks that can be plugged + * or unplugged. The device driver is responsible for (un)plugging memory + * blocks on demand. + * + * Virtio-mem devices can only operate on their assigned memory region in + * order to (un)plug memory. A device cannot (un)plug memory belonging to + * other devices. + * + * The "region_size" corresponds to the maximum amount of memory that can + * be provided by a device. The "size" corresponds to the amount of memory + * that is currently plugged. "requested_size" corresponds to a request + * from the device to the device driver to (un)plug blocks. The + * device driver should try to (un)plug blocks in order to reach the + * "requested_size". It is impossible to plug more memory than requested. + * + * The "usable_region_size" represents the memory region that can actually + * be used to (un)plug memory. It is always at least as big as the + * "requested_size" and will grow dynamically. It will only shrink when + * explicitly triggered (VIRTIO_MEM_REQ_UNPLUG). + * + * There are no guarantees what will happen if unplugged memory is + * read/written. Such memory should, in general, not be touched. E.g., + * even writing might succeed, but the values will simply be discarded at + * random points in time. + * + * It can happen that the device cannot process a request, because it is + * busy. The device driver has to retry later. + * + * Usually, during system resets all memory will get unplugged, so the + * device driver can start with a clean state. However, in specific + * scenarios (if the device is busy) it can happen that the device still + * has memory plugged. The device driver can request to unplug all memory + * (VIRTIO_MEM_REQ_UNPLUG) - which might take a while to succeed if the + * device is busy. + */ + +/* --- virtio-mem: guest -> host requests --- */ + +/* request to plug memory blocks */ +#define VIRTIO_MEM_REQ_PLUG 0 +/* request to unplug memory blocks */ +#define VIRTIO_MEM_REQ_UNPLUG 1 +/* request to unplug all blocks and shrink the usable size */ +#define VIRTIO_MEM_REQ_UNPLUG_ALL 2 +/* request information about the plugged state of memory blocks */ +#define VIRTIO_MEM_REQ_STATE 3 + +struct virtio_mem_req_plug { + __virtio64 addr; + __virtio16 nb_blocks; +}; + +struct virtio_mem_req_unplug { + __virtio64 addr; + __virtio16 nb_blocks; +}; + +struct virtio_mem_req_state { + __virtio64 addr; + __virtio16 nb_blocks; +}; + +struct virtio_mem_req { + __virtio16 type; + __virtio16 padding[3]; + + union { + struct virtio_mem_req_plug plug; + struct virtio_mem_req_unplug unplug; + struct virtio_mem_req_state state; + } u; +}; + + +/* --- virtio-mem: host -> guest response --- */ + +/* + * Request processed successfully, applicable for + * - VIRTIO_MEM_REQ_PLUG + * - VIRTIO_MEM_REQ_UNPLUG + * - VIRTIO_MEM_REQ_UNPLUG_ALL + * - VIRTIO_MEM_REQ_STATE + */ +#define VIRTIO_MEM_RESP_ACK 0 +/* + * Request denied - e.g. trying to plug more than requested, applicable for + * - VIRTIO_MEM_REQ_PLUG + */ +#define VIRTIO_MEM_RESP_NACK 1 +/* + * Request cannot be processed right now, try again later, applicable for + * - VIRTIO_MEM_REQ_PLUG + * - VIRTIO_MEM_REQ_UNPLUG + * - VIRTIO_MEM_REQ_UNPLUG_ALL + */ +#define VIRTIO_MEM_RESP_BUSY 2 +/* + * Error in request (e.g. addresses/alignment), applicable for + * - VIRTIO_MEM_REQ_PLUG + * - VIRTIO_MEM_REQ_UNPLUG + * - VIRTIO_MEM_REQ_STATE + */ +#define VIRTIO_MEM_RESP_ERROR 3 + + +/* State of memory blocks is "plugged" */ +#define VIRTIO_MEM_STATE_PLUGGED 0 +/* State of memory blocks is "unplugged" */ +#define VIRTIO_MEM_STATE_UNPLUGGED 1 +/* State of memory blocks is "mixed" */ +#define VIRTIO_MEM_STATE_MIXED 2 + +struct virtio_mem_resp_state { + __virtio16 state; +}; + +struct virtio_mem_resp { + __virtio16 type; + __virtio16 padding[3]; + + union { + struct virtio_mem_resp_state state; + } u; +}; + +/* --- virtio-mem: configuration --- */ + +struct virtio_mem_config { + /* Block size and alignment. Cannot change. */ + __u32 block_size; + __u32 padding; + /* Start address of the memory region. Cannot change. */ + __u64 addr; + /* Region size (maximum). Cannot change. */ + __u64 region_size; + /* + * Currently usable region size. Can grow up to region_size. Can + * shrink due to VIRTIO_MEM_REQ_UNPLUG_ALL (in which case no config + * update will be sent). + */ + __u64 usable_region_size; + /* + * Currently used size. Changes due to plug/unplug requests, but no + * config updates will be sent. + */ + __u64 plugged_size; + /* Requested size. New plug requests cannot exceed it. Can change. */ + __u64 requested_size; +}; + +#endif /* _LINUX_VIRTIO_MEM_H */ From dfb0b2e46d04d0b58754426184571a959bf30dce Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Thu, 7 May 2020 16:01:26 +0200 Subject: [PATCH 19/40] MAINTAINERS: Add myself as virtio-mem maintainer Let's make sure patches/bug reports find the right person. Cc: "Michael S. Tsirkin" Signed-off-by: David Hildenbrand Link: https://lore.kernel.org/r/20200507140139.17083-3-david@redhat.com Signed-off-by: Michael S. Tsirkin --- MAINTAINERS | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index b5106b7d6d0c..88170f86ad4e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -17986,6 +17986,13 @@ S: Maintained F: drivers/iommu/virtio-iommu.c F: include/uapi/linux/virtio_iommu.h +VIRTIO MEM DRIVER +M: David Hildenbrand +L: virtualization@lists.linux-foundation.org +S: Maintained +F: drivers/virtio/virtio_mem.c +F: include/uapi/linux/virtio_mem.h + VIRTUAL BOX GUEST DEVICE DRIVER M: Hans de Goede M: Arnd Bergmann From f2af6d3978d74a7891d0f428537b4494498202cb Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Thu, 7 May 2020 16:01:27 +0200 Subject: [PATCH 20/40] virtio-mem: Allow to specify an ACPI PXM as nid We want to allow to specify (similar as for a DIMM), to which node a virtio-mem device (and, therefore, its memory) belongs. Add a new virtio-mem feature flag and export pxm_to_node, so it can be used in kernel module context. Acked-by: Michal Hocko # for the export Acked-by: "Rafael J. Wysocki" # for the export Acked-by: Pankaj Gupta Tested-by: Pankaj Gupta Cc: "Michael S. Tsirkin" Cc: Jason Wang Cc: Oscar Salvador Cc: Michal Hocko Cc: Igor Mammedov Cc: Dave Young Cc: Andrew Morton Cc: Dan Williams Cc: Pavel Tatashin Cc: Stefan Hajnoczi Cc: Vlastimil Babka Cc: Len Brown Cc: linux-acpi@vger.kernel.org Signed-off-by: David Hildenbrand Link: https://lore.kernel.org/r/20200507140139.17083-4-david@redhat.com Signed-off-by: Michael S. Tsirkin --- drivers/acpi/numa/srat.c | 1 + drivers/virtio/virtio_mem.c | 39 +++++++++++++++++++++++++++++++-- include/uapi/linux/virtio_mem.h | 10 ++++++++- 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/drivers/acpi/numa/srat.c b/drivers/acpi/numa/srat.c index 47b4969d9b93..5be5a977da1b 100644 --- a/drivers/acpi/numa/srat.c +++ b/drivers/acpi/numa/srat.c @@ -35,6 +35,7 @@ int pxm_to_node(int pxm) return NUMA_NO_NODE; return pxm_to_node_map[pxm]; } +EXPORT_SYMBOL(pxm_to_node); int node_to_pxm(int node) { diff --git a/drivers/virtio/virtio_mem.c b/drivers/virtio/virtio_mem.c index 5d1dcaa6fc42..270ddeaec059 100644 --- a/drivers/virtio/virtio_mem.c +++ b/drivers/virtio/virtio_mem.c @@ -21,6 +21,8 @@ #include #include +#include + enum virtio_mem_mb_state { /* Unplugged, not added to Linux. Can be reused later. */ VIRTIO_MEM_MB_STATE_UNUSED = 0, @@ -72,6 +74,8 @@ struct virtio_mem { /* The device block size (for communicating with the device). */ uint32_t device_block_size; + /* The translated node id. NUMA_NO_NODE in case not specified. */ + int nid; /* Physical start address of the memory region. */ uint64_t addr; /* Maximum region size in bytes. */ @@ -389,7 +393,10 @@ static int virtio_mem_sb_bitmap_prepare_next_mb(struct virtio_mem *vm) static int virtio_mem_mb_add(struct virtio_mem *vm, unsigned long mb_id) { const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id); - int nid = memory_add_physaddr_to_nid(addr); + int nid = vm->nid; + + if (nid == NUMA_NO_NODE) + nid = memory_add_physaddr_to_nid(addr); dev_dbg(&vm->vdev->dev, "adding memory block: %lu\n", mb_id); return add_memory(nid, addr, memory_block_size_bytes()); @@ -407,7 +414,10 @@ static int virtio_mem_mb_add(struct virtio_mem *vm, unsigned long mb_id) static int virtio_mem_mb_remove(struct virtio_mem *vm, unsigned long mb_id) { const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id); - int nid = memory_add_physaddr_to_nid(addr); + int nid = vm->nid; + + if (nid == NUMA_NO_NODE) + nid = memory_add_physaddr_to_nid(addr); dev_dbg(&vm->vdev->dev, "removing memory block: %lu\n", mb_id); return remove_memory(nid, addr, memory_block_size_bytes()); @@ -426,6 +436,17 @@ static void virtio_mem_retry(struct virtio_mem *vm) spin_unlock_irqrestore(&vm->removal_lock, flags); } +static int virtio_mem_translate_node_id(struct virtio_mem *vm, uint16_t node_id) +{ + int node = NUMA_NO_NODE; + +#if defined(CONFIG_ACPI_NUMA) + if (virtio_has_feature(vm->vdev, VIRTIO_MEM_F_ACPI_PXM)) + node = pxm_to_node(node_id); +#endif + return node; +} + /* * Test if a virtio-mem device overlaps with the given range. Can be called * from (notifier) callbacks lockless. @@ -1267,6 +1288,7 @@ static bool virtio_mem_any_memory_present(unsigned long start, static int virtio_mem_init(struct virtio_mem *vm) { const uint64_t phys_limit = 1UL << MAX_PHYSMEM_BITS; + uint16_t node_id; if (!vm->vdev->config->get) { dev_err(&vm->vdev->dev, "config access disabled\n"); @@ -1287,6 +1309,9 @@ static int virtio_mem_init(struct virtio_mem *vm) &vm->plugged_size); virtio_cread(vm->vdev, struct virtio_mem_config, block_size, &vm->device_block_size); + virtio_cread(vm->vdev, struct virtio_mem_config, node_id, + &node_id); + vm->nid = virtio_mem_translate_node_id(vm, node_id); virtio_cread(vm->vdev, struct virtio_mem_config, addr, &vm->addr); virtio_cread(vm->vdev, struct virtio_mem_config, region_size, &vm->region_size); @@ -1365,6 +1390,8 @@ static int virtio_mem_init(struct virtio_mem *vm) memory_block_size_bytes()); dev_info(&vm->vdev->dev, "subblock size: 0x%x", vm->subblock_size); + if (vm->nid != NUMA_NO_NODE) + dev_info(&vm->vdev->dev, "nid: %d", vm->nid); return 0; } @@ -1508,12 +1535,20 @@ static int virtio_mem_restore(struct virtio_device *vdev) } #endif +static unsigned int virtio_mem_features[] = { +#if defined(CONFIG_NUMA) && defined(CONFIG_ACPI_NUMA) + VIRTIO_MEM_F_ACPI_PXM, +#endif +}; + static struct virtio_device_id virtio_mem_id_table[] = { { VIRTIO_ID_MEM, VIRTIO_DEV_ANY_ID }, { 0 }, }; static struct virtio_driver virtio_mem_driver = { + .feature_table = virtio_mem_features, + .feature_table_size = ARRAY_SIZE(virtio_mem_features), .driver.name = KBUILD_MODNAME, .driver.owner = THIS_MODULE, .id_table = virtio_mem_id_table, diff --git a/include/uapi/linux/virtio_mem.h b/include/uapi/linux/virtio_mem.h index 1bfade78bdfd..e0a9dc7397c3 100644 --- a/include/uapi/linux/virtio_mem.h +++ b/include/uapi/linux/virtio_mem.h @@ -83,6 +83,12 @@ * device is busy. */ +/* --- virtio-mem: feature bits --- */ + +/* node_id is an ACPI PXM and is valid */ +#define VIRTIO_MEM_F_ACPI_PXM 0 + + /* --- virtio-mem: guest -> host requests --- */ /* request to plug memory blocks */ @@ -177,7 +183,9 @@ struct virtio_mem_resp { struct virtio_mem_config { /* Block size and alignment. Cannot change. */ __u32 block_size; - __u32 padding; + /* Valid with VIRTIO_MEM_F_ACPI_PXM. Cannot change. */ + __u16 node_id; + __u16 padding; /* Start address of the memory region. Cannot change. */ __u64 addr; /* Region size (maximum). Cannot change. */ From c627ff5d982276908188fae86dbe727ed49c9594 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Thu, 7 May 2020 16:01:28 +0200 Subject: [PATCH 21/40] virtio-mem: Paravirtualized memory hotunplug part 1 Unplugging subblocks of memory blocks that are offline is easy. All we have to do is watch out for concurrent onlining activity. Tested-by: Pankaj Gupta Cc: "Michael S. Tsirkin" Cc: Jason Wang Cc: Oscar Salvador Cc: Michal Hocko Cc: Igor Mammedov Cc: Dave Young Cc: Andrew Morton Cc: Dan Williams Cc: Pavel Tatashin Cc: Stefan Hajnoczi Cc: Vlastimil Babka Signed-off-by: David Hildenbrand Link: https://lore.kernel.org/r/20200507140139.17083-5-david@redhat.com Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_mem.c | 116 +++++++++++++++++++++++++++++++++++- 1 file changed, 114 insertions(+), 2 deletions(-) diff --git a/drivers/virtio/virtio_mem.c b/drivers/virtio/virtio_mem.c index 270ddeaec059..a3ec795be8be 100644 --- a/drivers/virtio/virtio_mem.c +++ b/drivers/virtio/virtio_mem.c @@ -123,7 +123,7 @@ struct virtio_mem { * * When this lock is held the pointers can't change, ONLINE and * OFFLINE blocks can't change the state and no subblocks will get - * plugged. + * plugged/unplugged. */ struct mutex hotplug_mutex; bool hotplug_active; @@ -280,6 +280,12 @@ static int virtio_mem_mb_state_prepare_next_mb(struct virtio_mem *vm) _mb_id++) \ if (virtio_mem_mb_get_state(_vm, _mb_id) == _state) +#define virtio_mem_for_each_mb_state_rev(_vm, _mb_id, _state) \ + for (_mb_id = _vm->next_mb_id - 1; \ + _mb_id >= _vm->first_mb_id && _vm->nb_mb_state[_state]; \ + _mb_id--) \ + if (virtio_mem_mb_get_state(_vm, _mb_id) == _state) + /* * Mark all selected subblocks plugged. * @@ -325,6 +331,19 @@ static bool virtio_mem_mb_test_sb_plugged(struct virtio_mem *vm, bit + count; } +/* + * Test if all selected subblocks are unplugged. + */ +static bool virtio_mem_mb_test_sb_unplugged(struct virtio_mem *vm, + unsigned long mb_id, int sb_id, + int count) +{ + const int bit = (mb_id - vm->first_mb_id) * vm->nb_sb_per_mb + sb_id; + + /* TODO: Helper similar to bitmap_set() */ + return find_next_bit(vm->sb_bitmap, bit + count, bit) >= bit + count; +} + /* * Find the first plugged subblock. Returns vm->nb_sb_per_mb in case there is * none. @@ -513,6 +532,9 @@ static void virtio_mem_notify_offline(struct virtio_mem *vm, BUG(); break; } + + /* trigger the workqueue, maybe we can now unplug memory. */ + virtio_mem_retry(vm); } static void virtio_mem_notify_online(struct virtio_mem *vm, unsigned long mb_id, @@ -1122,6 +1144,94 @@ out_unlock: return rc; } +/* + * Unplug the desired number of plugged subblocks of an offline memory block. + * Will fail if any subblock cannot get unplugged (instead of skipping it). + * + * Will modify the state of the memory block. Might temporarily drop the + * hotplug_mutex. + * + * Note: Can fail after some subblocks were successfully unplugged. + */ +static int virtio_mem_mb_unplug_any_sb_offline(struct virtio_mem *vm, + unsigned long mb_id, + uint64_t *nb_sb) +{ + int rc; + + rc = virtio_mem_mb_unplug_any_sb(vm, mb_id, nb_sb); + + /* some subblocks might have been unplugged even on failure */ + if (!virtio_mem_mb_test_sb_plugged(vm, mb_id, 0, vm->nb_sb_per_mb)) + virtio_mem_mb_set_state(vm, mb_id, + VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL); + if (rc) + return rc; + + if (virtio_mem_mb_test_sb_unplugged(vm, mb_id, 0, vm->nb_sb_per_mb)) { + /* + * Remove the block from Linux - this should never fail. + * Hinder the block from getting onlined by marking it + * unplugged. Temporarily drop the mutex, so + * any pending GOING_ONLINE requests can be serviced/rejected. + */ + virtio_mem_mb_set_state(vm, mb_id, + VIRTIO_MEM_MB_STATE_UNUSED); + + mutex_unlock(&vm->hotplug_mutex); + rc = virtio_mem_mb_remove(vm, mb_id); + BUG_ON(rc); + mutex_lock(&vm->hotplug_mutex); + } + return 0; +} + +/* + * Try to unplug the requested amount of memory. + */ +static int virtio_mem_unplug_request(struct virtio_mem *vm, uint64_t diff) +{ + uint64_t nb_sb = diff / vm->subblock_size; + unsigned long mb_id; + int rc; + + if (!nb_sb) + return 0; + + /* + * We'll drop the mutex a couple of times when it is safe to do so. + * This might result in some blocks switching the state (online/offline) + * and we could miss them in this run - we will retry again later. + */ + mutex_lock(&vm->hotplug_mutex); + + /* Try to unplug subblocks of partially plugged offline blocks. */ + virtio_mem_for_each_mb_state_rev(vm, mb_id, + VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL) { + rc = virtio_mem_mb_unplug_any_sb_offline(vm, mb_id, + &nb_sb); + if (rc || !nb_sb) + goto out_unlock; + cond_resched(); + } + + /* Try to unplug subblocks of plugged offline blocks. */ + virtio_mem_for_each_mb_state_rev(vm, mb_id, + VIRTIO_MEM_MB_STATE_OFFLINE) { + rc = virtio_mem_mb_unplug_any_sb_offline(vm, mb_id, + &nb_sb); + if (rc || !nb_sb) + goto out_unlock; + cond_resched(); + } + + mutex_unlock(&vm->hotplug_mutex); + return 0; +out_unlock: + mutex_unlock(&vm->hotplug_mutex); + return rc; +} + /* * Try to unplug all blocks that couldn't be unplugged before, for example, * because the hypervisor was busy. @@ -1204,8 +1314,10 @@ retry: if (vm->requested_size > vm->plugged_size) { diff = vm->requested_size - vm->plugged_size; rc = virtio_mem_plug_request(vm, diff); + } else { + diff = vm->plugged_size - vm->requested_size; + rc = virtio_mem_unplug_request(vm, diff); } - /* TODO: try to unplug memory */ } switch (rc) { From 255f598507083905995ecab96392770ae03aac7f Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Thu, 7 May 2020 16:01:29 +0200 Subject: [PATCH 22/40] virtio-mem: Paravirtualized memory hotunplug part 2 We also want to unplug online memory (contained in online memory blocks and, therefore, managed by the buddy), and eventually replug it later. When requested to unplug memory, we use alloc_contig_range() to allocate subblocks in online memory blocks (so we are the owner) and send them to our hypervisor. When requested to plug memory, we can replug such memory using free_contig_range() after asking our hypervisor. We also want to mark all allocated pages PG_offline, so nobody will touch them. To differentiate pages that were never onlined when onlining the memory block from pages allocated via alloc_contig_range(), we use PageDirty(). Based on this flag, virtio_mem_fake_online() can either online the pages for the first time or use free_contig_range(). It is worth noting that there are no guarantees on how much memory can actually get unplugged again. All device memory might completely be fragmented with unmovable data, such that no subblock can get unplugged. We are not touching the ZONE_MOVABLE. If memory is onlined to the ZONE_MOVABLE, it can only get unplugged after that memory was offlined manually by user space. In normal operation, virtio-mem memory is suggested to be onlined to ZONE_NORMAL. In the future, we will try to make unplug more likely to succeed. Add a module parameter to control if online memory shall be touched. As we want to access alloc_contig_range()/free_contig_range() from kernel module context, export the symbols. Note: Whenever virtio-mem uses alloc_contig_range(), all affected pages are on the same node, in the same zone, and contain no holes. Acked-by: Michal Hocko # to export contig range allocator API Tested-by: Pankaj Gupta Cc: "Michael S. Tsirkin" Cc: Jason Wang Cc: Oscar Salvador Cc: Michal Hocko Cc: Igor Mammedov Cc: Dave Young Cc: Andrew Morton Cc: Dan Williams Cc: Pavel Tatashin Cc: Stefan Hajnoczi Cc: Vlastimil Babka Cc: Mel Gorman Cc: Mike Rapoport Cc: Alexander Duyck Cc: Alexander Potapenko Signed-off-by: David Hildenbrand Link: https://lore.kernel.org/r/20200507140139.17083-6-david@redhat.com Signed-off-by: Michael S. Tsirkin --- drivers/virtio/Kconfig | 1 + drivers/virtio/virtio_mem.c | 157 ++++++++++++++++++++++++++++++++---- mm/page_alloc.c | 2 + 3 files changed, 146 insertions(+), 14 deletions(-) diff --git a/drivers/virtio/Kconfig b/drivers/virtio/Kconfig index d6dde7d2cf76..4c1e14615001 100644 --- a/drivers/virtio/Kconfig +++ b/drivers/virtio/Kconfig @@ -85,6 +85,7 @@ config VIRTIO_MEM depends on VIRTIO depends on MEMORY_HOTPLUG_SPARSE depends on MEMORY_HOTREMOVE + select CONTIG_ALLOC help This driver provides access to virtio-mem paravirtualized memory devices, allowing to hotplug and hotunplug memory. diff --git a/drivers/virtio/virtio_mem.c b/drivers/virtio/virtio_mem.c index a3ec795be8be..74f0d3cb1d22 100644 --- a/drivers/virtio/virtio_mem.c +++ b/drivers/virtio/virtio_mem.c @@ -23,6 +23,10 @@ #include +static bool unplug_online = true; +module_param(unplug_online, bool, 0644); +MODULE_PARM_DESC(unplug_online, "Try to unplug online memory"); + enum virtio_mem_mb_state { /* Unplugged, not added to Linux. Can be reused later. */ VIRTIO_MEM_MB_STATE_UNUSED = 0, @@ -654,23 +658,35 @@ static int virtio_mem_memory_notifier_cb(struct notifier_block *nb, } /* - * Set a range of pages PG_offline. + * Set a range of pages PG_offline. Remember pages that were never onlined + * (via generic_online_page()) using PageDirty(). */ static void virtio_mem_set_fake_offline(unsigned long pfn, - unsigned int nr_pages) + unsigned int nr_pages, bool onlined) { - for (; nr_pages--; pfn++) - __SetPageOffline(pfn_to_page(pfn)); + for (; nr_pages--; pfn++) { + struct page *page = pfn_to_page(pfn); + + __SetPageOffline(page); + if (!onlined) + SetPageDirty(page); + } } /* - * Clear PG_offline from a range of pages. + * Clear PG_offline from a range of pages. If the pages were never onlined, + * (via generic_online_page()), clear PageDirty(). */ static void virtio_mem_clear_fake_offline(unsigned long pfn, - unsigned int nr_pages) + unsigned int nr_pages, bool onlined) { - for (; nr_pages--; pfn++) - __ClearPageOffline(pfn_to_page(pfn)); + for (; nr_pages--; pfn++) { + struct page *page = pfn_to_page(pfn); + + __ClearPageOffline(page); + if (!onlined) + ClearPageDirty(page); + } } /* @@ -686,10 +702,26 @@ static void virtio_mem_fake_online(unsigned long pfn, unsigned int nr_pages) * We are always called with subblock granularity, which is at least * aligned to MAX_ORDER - 1. */ - virtio_mem_clear_fake_offline(pfn, nr_pages); + for (i = 0; i < nr_pages; i += 1 << order) { + struct page *page = pfn_to_page(pfn + i); - for (i = 0; i < nr_pages; i += 1 << order) - generic_online_page(pfn_to_page(pfn + i), order); + /* + * If the page is PageDirty(), it was kept fake-offline when + * onlining the memory block. Otherwise, it was allocated + * using alloc_contig_range(). All pages in a subblock are + * alike. + */ + if (PageDirty(page)) { + virtio_mem_clear_fake_offline(pfn + i, 1 << order, + false); + generic_online_page(page, order); + } else { + virtio_mem_clear_fake_offline(pfn + i, 1 << order, + true); + free_contig_range(pfn + i, 1 << order); + adjust_managed_page_count(page, 1 << order); + } + } } static void virtio_mem_online_page_cb(struct page *page, unsigned int order) @@ -718,7 +750,8 @@ static void virtio_mem_online_page_cb(struct page *page, unsigned int order) if (virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id, 1)) generic_online_page(page, order); else - virtio_mem_set_fake_offline(PFN_DOWN(addr), 1 << order); + virtio_mem_set_fake_offline(PFN_DOWN(addr), 1 << order, + false); rcu_read_unlock(); return; } @@ -1186,6 +1219,72 @@ static int virtio_mem_mb_unplug_any_sb_offline(struct virtio_mem *vm, return 0; } +/* + * Unplug the desired number of plugged subblocks of an online memory block. + * Will skip subblock that are busy. + * + * Will modify the state of the memory block. + * + * Note: Can fail after some subblocks were successfully unplugged. Can + * return 0 even if subblocks were busy and could not get unplugged. + */ +static int virtio_mem_mb_unplug_any_sb_online(struct virtio_mem *vm, + unsigned long mb_id, + uint64_t *nb_sb) +{ + const unsigned long nr_pages = PFN_DOWN(vm->subblock_size); + unsigned long start_pfn; + int rc, sb_id; + + /* + * TODO: To increase the performance we want to try bigger, consecutive + * subblocks first before falling back to single subblocks. Also, + * we should sense via something like is_mem_section_removable() + * first if it makes sense to go ahead any try to allocate. + */ + for (sb_id = 0; sb_id < vm->nb_sb_per_mb && *nb_sb; sb_id++) { + /* Find the next candidate subblock */ + while (sb_id < vm->nb_sb_per_mb && + !virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id, 1)) + sb_id++; + if (sb_id >= vm->nb_sb_per_mb) + break; + + start_pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) + + sb_id * vm->subblock_size); + rc = alloc_contig_range(start_pfn, start_pfn + nr_pages, + MIGRATE_MOVABLE, GFP_KERNEL); + if (rc == -ENOMEM) + /* whoops, out of memory */ + return rc; + if (rc) + /* memory busy, we can't unplug this chunk */ + continue; + + /* Mark it as fake-offline before unplugging it */ + virtio_mem_set_fake_offline(start_pfn, nr_pages, true); + adjust_managed_page_count(pfn_to_page(start_pfn), -nr_pages); + + /* Try to unplug the allocated memory */ + rc = virtio_mem_mb_unplug_sb(vm, mb_id, sb_id, 1); + if (rc) { + /* Return the memory to the buddy. */ + virtio_mem_fake_online(start_pfn, nr_pages); + return rc; + } + + virtio_mem_mb_set_state(vm, mb_id, + VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL); + *nb_sb -= 1; + } + + /* + * TODO: Once all subblocks of a memory block were unplugged, we want + * to offline the memory block and remove it. + */ + return 0; +} + /* * Try to unplug the requested amount of memory. */ @@ -1225,8 +1324,37 @@ static int virtio_mem_unplug_request(struct virtio_mem *vm, uint64_t diff) cond_resched(); } + if (!unplug_online) { + mutex_unlock(&vm->hotplug_mutex); + return 0; + } + + /* Try to unplug subblocks of partially plugged online blocks. */ + virtio_mem_for_each_mb_state_rev(vm, mb_id, + VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL) { + rc = virtio_mem_mb_unplug_any_sb_online(vm, mb_id, + &nb_sb); + if (rc || !nb_sb) + goto out_unlock; + mutex_unlock(&vm->hotplug_mutex); + cond_resched(); + mutex_lock(&vm->hotplug_mutex); + } + + /* Try to unplug subblocks of plugged online blocks. */ + virtio_mem_for_each_mb_state_rev(vm, mb_id, + VIRTIO_MEM_MB_STATE_ONLINE) { + rc = virtio_mem_mb_unplug_any_sb_online(vm, mb_id, + &nb_sb); + if (rc || !nb_sb) + goto out_unlock; + mutex_unlock(&vm->hotplug_mutex); + cond_resched(); + mutex_lock(&vm->hotplug_mutex); + } + mutex_unlock(&vm->hotplug_mutex); - return 0; + return nb_sb ? -EBUSY : 0; out_unlock: mutex_unlock(&vm->hotplug_mutex); return rc; @@ -1332,7 +1460,8 @@ retry: case -EBUSY: /* * The hypervisor cannot process our request right now - * (e.g., out of memory, migrating). + * (e.g., out of memory, migrating) or we cannot free up + * any memory to unplug it (all plugged memory is busy). */ case -ENOMEM: /* Out of memory, try again later. */ diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 13cc653122b7..ce1c9df54eac 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -8603,6 +8603,7 @@ done: pfn_max_align_up(end), migratetype); return ret; } +EXPORT_SYMBOL(alloc_contig_range); static int __alloc_contig_pages(unsigned long start_pfn, unsigned long nr_pages, gfp_t gfp_mask) @@ -8718,6 +8719,7 @@ void free_contig_range(unsigned long pfn, unsigned int nr_pages) } WARN(count != 0, "%d pages are still in use!\n", count); } +EXPORT_SYMBOL(free_contig_range); /* * The zone indicated has a new number of managed_pages; batch sizes and percpu From aa218795cb5fd583c94fc838dc76b7379dc4976a Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Thu, 7 May 2020 16:01:30 +0200 Subject: [PATCH 23/40] mm: Allow to offline unmovable PageOffline() pages via MEM_GOING_OFFLINE virtio-mem wants to allow to offline memory blocks of which some parts were unplugged (allocated via alloc_contig_range()), especially, to later offline and remove completely unplugged memory blocks. The important part is that PageOffline() has to remain set until the section is offline, so these pages will never get accessed (e.g., when dumping). The pages should not be handed back to the buddy (which would require clearing PageOffline() and result in issues if offlining fails and the pages are suddenly in the buddy). Let's allow to do that by allowing to isolate any PageOffline() page when offlining. This way, we can reach the memory hotplug notifier MEM_GOING_OFFLINE, where the driver can signal that he is fine with offlining this page by dropping its reference count. PageOffline() pages with a reference count of 0 can then be skipped when offlining the pages (like if they were free, however they are not in the buddy). Anybody who uses PageOffline() pages and does not agree to offline them (e.g., Hyper-V balloon, XEN balloon, VMWare balloon for 2MB pages) will not decrement the reference count and make offlining fail when trying to migrate such an unmovable page. So there should be no observable change. Same applies to balloon compaction users (movable PageOffline() pages), the pages will simply be migrated. Note 1: If offlining fails, a driver has to increment the reference count again in MEM_CANCEL_OFFLINE. Note 2: A driver that makes use of this has to be aware that re-onlining the memory block has to be handled by hooking into onlining code (online_page_callback_t), resetting the page PageOffline() and not giving them to the buddy. Reviewed-by: Alexander Duyck Acked-by: Michal Hocko Tested-by: Pankaj Gupta Acked-by: Andrew Morton Cc: Andrew Morton Cc: Juergen Gross Cc: Konrad Rzeszutek Wilk Cc: Pavel Tatashin Cc: Alexander Duyck Cc: Vlastimil Babka Cc: Johannes Weiner Cc: Anthony Yznaga Cc: Michal Hocko Cc: Oscar Salvador Cc: Mel Gorman Cc: Mike Rapoport Cc: Dan Williams Cc: Anshuman Khandual Cc: Qian Cai Cc: Pingfan Liu Signed-off-by: David Hildenbrand Link: https://lore.kernel.org/r/20200507140139.17083-7-david@redhat.com Signed-off-by: Michael S. Tsirkin --- include/linux/page-flags.h | 10 +++++++++ mm/memory_hotplug.c | 44 +++++++++++++++++++++++++++++--------- mm/page_alloc.c | 24 +++++++++++++++++++++ mm/page_isolation.c | 9 ++++++++ 4 files changed, 77 insertions(+), 10 deletions(-) diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 222f6f7b2bb3..6be1aa559b1e 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -777,6 +777,16 @@ PAGE_TYPE_OPS(Buddy, buddy) * not onlined when onlining the section). * The content of these pages is effectively stale. Such pages should not * be touched (read/write/dump/save) except by their owner. + * + * If a driver wants to allow to offline unmovable PageOffline() pages without + * putting them back to the buddy, it can do so via the memory notifier by + * decrementing the reference count in MEM_GOING_OFFLINE and incrementing the + * reference count in MEM_CANCEL_OFFLINE. When offlining, the PageOffline() + * pages (now with a reference count of zero) are treated like free pages, + * allowing the containing memory block to get offlined. A driver that + * relies on this feature is aware that re-onlining the memory block will + * require to re-set the pages PageOffline() and not giving them to the + * buddy via online_page_callback_t. */ PAGE_TYPE_OPS(Offline, offline) diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index fc0aad0bc1f5..008e4a7ed8bc 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -1224,11 +1224,17 @@ struct zone *test_pages_in_a_zone(unsigned long start_pfn, /* * Scan pfn range [start,end) to find movable/migratable pages (LRU pages, - * non-lru movable pages and hugepages). We scan pfn because it's much - * easier than scanning over linked list. This function returns the pfn - * of the first found movable page if it's found, otherwise 0. + * non-lru movable pages and hugepages). Will skip over most unmovable + * pages (esp., pages that can be skipped when offlining), but bail out on + * definitely unmovable pages. + * + * Returns: + * 0 in case a movable page is found and movable_pfn was updated. + * -ENOENT in case no movable page was found. + * -EBUSY in case a definitely unmovable page was found. */ -static unsigned long scan_movable_pages(unsigned long start, unsigned long end) +static int scan_movable_pages(unsigned long start, unsigned long end, + unsigned long *movable_pfn) { unsigned long pfn; @@ -1240,18 +1246,30 @@ static unsigned long scan_movable_pages(unsigned long start, unsigned long end) continue; page = pfn_to_page(pfn); if (PageLRU(page)) - return pfn; + goto found; if (__PageMovable(page)) - return pfn; + goto found; + + /* + * PageOffline() pages that are not marked __PageMovable() and + * have a reference count > 0 (after MEM_GOING_OFFLINE) are + * definitely unmovable. If their reference count would be 0, + * they could at least be skipped when offlining memory. + */ + if (PageOffline(page) && page_count(page)) + return -EBUSY; if (!PageHuge(page)) continue; head = compound_head(page); if (page_huge_active(head)) - return pfn; + goto found; skip = compound_nr(head) - (page - head); pfn += skip - 1; } + return -ENOENT; +found: + *movable_pfn = pfn; return 0; } @@ -1518,7 +1536,8 @@ static int __ref __offline_pages(unsigned long start_pfn, } do { - for (pfn = start_pfn; pfn;) { + pfn = start_pfn; + do { if (signal_pending(current)) { ret = -EINTR; reason = "signal backoff"; @@ -1528,14 +1547,19 @@ static int __ref __offline_pages(unsigned long start_pfn, cond_resched(); lru_add_drain_all(); - pfn = scan_movable_pages(pfn, end_pfn); - if (pfn) { + ret = scan_movable_pages(pfn, end_pfn, &pfn); + if (!ret) { /* * TODO: fatal migration failures should bail * out */ do_migrate_range(pfn, end_pfn); } + } while (!ret); + + if (ret != -ENOENT) { + reason = "unmovable page"; + goto failed_removal_isolated; } /* diff --git a/mm/page_alloc.c b/mm/page_alloc.c index ce1c9df54eac..9b1c49c3097d 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -8372,6 +8372,19 @@ struct page *has_unmovable_pages(struct zone *zone, struct page *page, if ((flags & MEMORY_OFFLINE) && PageHWPoison(page)) continue; + /* + * We treat all PageOffline() pages as movable when offlining + * to give drivers a chance to decrement their reference count + * in MEM_GOING_OFFLINE in order to indicate that these pages + * can be offlined as there are no direct references anymore. + * For actually unmovable PageOffline() where the driver does + * not support this, we will fail later when trying to actually + * move these pages that still have a reference count > 0. + * (false negatives in this function only) + */ + if ((flags & MEMORY_OFFLINE) && PageOffline(page)) + continue; + if (__PageMovable(page) || PageLRU(page)) continue; @@ -8792,6 +8805,17 @@ __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) offlined_pages++; continue; } + /* + * At this point all remaining PageOffline() pages have a + * reference count of 0 and can simply be skipped. + */ + if (PageOffline(page)) { + BUG_ON(page_count(page)); + BUG_ON(PageBuddy(page)); + pfn++; + offlined_pages++; + continue; + } BUG_ON(page_count(page)); BUG_ON(!PageBuddy(page)); diff --git a/mm/page_isolation.c b/mm/page_isolation.c index 2c11a38d6e87..f6d07c5f0d34 100644 --- a/mm/page_isolation.c +++ b/mm/page_isolation.c @@ -151,6 +151,7 @@ __first_valid_page(unsigned long pfn, unsigned long nr_pages) * a bit mask) * MEMORY_OFFLINE - isolate to offline (!allocate) memory * e.g., skip over PageHWPoison() pages + * and PageOffline() pages. * REPORT_FAILURE - report details about the failure to * isolate the range * @@ -259,6 +260,14 @@ __test_page_isolated_in_pageblock(unsigned long pfn, unsigned long end_pfn, else if ((flags & MEMORY_OFFLINE) && PageHWPoison(page)) /* A HWPoisoned page cannot be also PageBuddy */ pfn++; + else if ((flags & MEMORY_OFFLINE) && PageOffline(page) && + !page_count(page)) + /* + * The responsible driver agreed to skip PageOffline() + * pages when offlining memory by dropping its + * reference in MEM_GOING_OFFLINE. + */ + pfn++; else break; } From 8e5c921ca0cd9aa59386e6be4b86b32f0ba7296b Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Thu, 7 May 2020 16:01:31 +0200 Subject: [PATCH 24/40] virtio-mem: Allow to offline partially unplugged memory blocks Dropping the reference count of PageOffline() pages during MEM_GOING_ONLINE allows offlining code to skip them. However, we also have to clear PG_reserved, because PG_reserved pages get detected as unmovable right away. Take care of restoring the reference count when offlining is canceled. Clarify why we don't have to perform any action when unloading the driver. Also, let's add a warning if anybody is still holding a reference to unplugged pages when offlining. Tested-by: Pankaj Gupta Cc: "Michael S. Tsirkin" Cc: Jason Wang Cc: Oscar Salvador Cc: Michal Hocko Cc: Igor Mammedov Cc: Dave Young Cc: Andrew Morton Cc: Dan Williams Cc: Pavel Tatashin Cc: Stefan Hajnoczi Cc: Vlastimil Babka Signed-off-by: David Hildenbrand Link: https://lore.kernel.org/r/20200507140139.17083-8-david@redhat.com Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_mem.c | 68 ++++++++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/drivers/virtio/virtio_mem.c b/drivers/virtio/virtio_mem.c index 74f0d3cb1d22..b0b41c73ce89 100644 --- a/drivers/virtio/virtio_mem.c +++ b/drivers/virtio/virtio_mem.c @@ -572,6 +572,57 @@ static void virtio_mem_notify_online(struct virtio_mem *vm, unsigned long mb_id, virtio_mem_retry(vm); } +static void virtio_mem_notify_going_offline(struct virtio_mem *vm, + unsigned long mb_id) +{ + const unsigned long nr_pages = PFN_DOWN(vm->subblock_size); + struct page *page; + unsigned long pfn; + int sb_id, i; + + for (sb_id = 0; sb_id < vm->nb_sb_per_mb; sb_id++) { + if (virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id, 1)) + continue; + /* + * Drop our reference to the pages so the memory can get + * offlined and add the unplugged pages to the managed + * page counters (so offlining code can correctly subtract + * them again). + */ + pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) + + sb_id * vm->subblock_size); + adjust_managed_page_count(pfn_to_page(pfn), nr_pages); + for (i = 0; i < nr_pages; i++) { + page = pfn_to_page(pfn + i); + if (WARN_ON(!page_ref_dec_and_test(page))) + dump_page(page, "unplugged page referenced"); + } + } +} + +static void virtio_mem_notify_cancel_offline(struct virtio_mem *vm, + unsigned long mb_id) +{ + const unsigned long nr_pages = PFN_DOWN(vm->subblock_size); + unsigned long pfn; + int sb_id, i; + + for (sb_id = 0; sb_id < vm->nb_sb_per_mb; sb_id++) { + if (virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id, 1)) + continue; + /* + * Get the reference we dropped when going offline and + * subtract the unplugged pages from the managed page + * counters. + */ + pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) + + sb_id * vm->subblock_size); + adjust_managed_page_count(pfn_to_page(pfn), -nr_pages); + for (i = 0; i < nr_pages; i++) + page_ref_inc(pfn_to_page(pfn + i)); + } +} + /* * This callback will either be called synchronously from add_memory() or * asynchronously (e.g., triggered via user space). We have to be careful @@ -618,6 +669,7 @@ static int virtio_mem_memory_notifier_cb(struct notifier_block *nb, break; } vm->hotplug_active = true; + virtio_mem_notify_going_offline(vm, mb_id); break; case MEM_GOING_ONLINE: mutex_lock(&vm->hotplug_mutex); @@ -642,6 +694,12 @@ static int virtio_mem_memory_notifier_cb(struct notifier_block *nb, mutex_unlock(&vm->hotplug_mutex); break; case MEM_CANCEL_OFFLINE: + if (!vm->hotplug_active) + break; + virtio_mem_notify_cancel_offline(vm, mb_id); + vm->hotplug_active = false; + mutex_unlock(&vm->hotplug_mutex); + break; case MEM_CANCEL_ONLINE: if (!vm->hotplug_active) break; @@ -668,8 +726,11 @@ static void virtio_mem_set_fake_offline(unsigned long pfn, struct page *page = pfn_to_page(pfn); __SetPageOffline(page); - if (!onlined) + if (!onlined) { SetPageDirty(page); + /* FIXME: remove after cleanups */ + ClearPageReserved(page); + } } } @@ -1722,6 +1783,11 @@ static void virtio_mem_remove(struct virtio_device *vdev) BUG_ON(rc); virtio_mem_mb_set_state(vm, mb_id, VIRTIO_MEM_MB_STATE_UNUSED); } + /* + * After we unregistered our callbacks, user space can no longer + * offline partially plugged online memory blocks. No need to worry + * about them. + */ /* unregister callbacks */ unregister_virtio_mem_device(vm); From 08b3acd7a68fc17902e1cb6b146389322840deab Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Thu, 7 May 2020 16:01:32 +0200 Subject: [PATCH 25/40] mm/memory_hotplug: Introduce offline_and_remove_memory() virtio-mem wants to offline and remove a memory block once it unplugged all subblocks (e.g., using alloc_contig_range()). Let's provide an interface to do that from a driver. virtio-mem already supports to offline partially unplugged memory blocks. Offlining a fully unplugged memory block will not require to migrate any pages. All unplugged subblocks are PageOffline() and have a reference count of 0 - so offlining code will simply skip them. All we need is an interface to offline and remove the memory from kernel module context, where we don't have access to the memory block devices (esp. find_memory_block() and device_offline()) and the device hotplug lock. To keep things simple, allow to only work on a single memory block. Acked-by: Michal Hocko Tested-by: Pankaj Gupta Acked-by: Andrew Morton Cc: Andrew Morton Cc: David Hildenbrand Cc: Oscar Salvador Cc: Michal Hocko Cc: Pavel Tatashin Cc: Wei Yang Cc: Dan Williams Cc: Qian Cai Signed-off-by: David Hildenbrand Link: https://lore.kernel.org/r/20200507140139.17083-9-david@redhat.com Signed-off-by: Michael S. Tsirkin --- include/linux/memory_hotplug.h | 1 + mm/memory_hotplug.c | 37 ++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index 93d9ada74ddd..cb7499843f5c 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h @@ -319,6 +319,7 @@ extern void try_offline_node(int nid); extern int offline_pages(unsigned long start_pfn, unsigned long nr_pages); extern int remove_memory(int nid, u64 start, u64 size); extern void __remove_memory(int nid, u64 start, u64 size); +extern int offline_and_remove_memory(int nid, u64 start, u64 size); #else static inline bool is_mem_section_removable(unsigned long pfn, diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 008e4a7ed8bc..4acb99aa9bf4 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -1821,4 +1821,41 @@ int remove_memory(int nid, u64 start, u64 size) return rc; } EXPORT_SYMBOL_GPL(remove_memory); + +/* + * Try to offline and remove a memory block. Might take a long time to + * finish in case memory is still in use. Primarily useful for memory devices + * that logically unplugged all memory (so it's no longer in use) and want to + * offline + remove the memory block. + */ +int offline_and_remove_memory(int nid, u64 start, u64 size) +{ + struct memory_block *mem; + int rc = -EINVAL; + + if (!IS_ALIGNED(start, memory_block_size_bytes()) || + size != memory_block_size_bytes()) + return rc; + + lock_device_hotplug(); + mem = find_memory_block(__pfn_to_section(PFN_DOWN(start))); + if (mem) + rc = device_offline(&mem->dev); + /* Ignore if the device is already offline. */ + if (rc > 0) + rc = 0; + + /* + * In case we succeeded to offline the memory block, remove it. + * This cannot fail as it cannot get onlined in the meantime. + */ + if (!rc) { + rc = try_remove_memory(nid, start, size); + WARN_ON_ONCE(rc); + } + unlock_device_hotplug(); + + return rc; +} +EXPORT_SYMBOL_GPL(offline_and_remove_memory); #endif /* CONFIG_MEMORY_HOTREMOVE */ From a573238786f8f16aca6946fc7b804b965e3038e9 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Thu, 7 May 2020 16:01:33 +0200 Subject: [PATCH 26/40] virtio-mem: Offline and remove completely unplugged memory blocks Let's offline+remove memory blocks once all subblocks are unplugged. We can use the new Linux MM interface for that. As no memory is in use anymore, this shouldn't take a long time and shouldn't fail. There might be corner cases where the offlining could still fail (especially, if another notifier NACKs the offlining request). Acked-by: Pankaj Gupta Tested-by: Pankaj Gupta Cc: "Michael S. Tsirkin" Cc: Jason Wang Cc: Oscar Salvador Cc: Michal Hocko Cc: Igor Mammedov Cc: Dave Young Cc: Andrew Morton Cc: Dan Williams Cc: Pavel Tatashin Cc: Stefan Hajnoczi Cc: Vlastimil Babka Signed-off-by: David Hildenbrand Link: https://lore.kernel.org/r/20200507140139.17083-10-david@redhat.com Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_mem.c | 47 +++++++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/drivers/virtio/virtio_mem.c b/drivers/virtio/virtio_mem.c index b0b41c73ce89..a2edb87e5ed8 100644 --- a/drivers/virtio/virtio_mem.c +++ b/drivers/virtio/virtio_mem.c @@ -446,6 +446,28 @@ static int virtio_mem_mb_remove(struct virtio_mem *vm, unsigned long mb_id) return remove_memory(nid, addr, memory_block_size_bytes()); } +/* + * Try to offline and remove a memory block from Linux. + * + * Must not be called with the vm->hotplug_mutex held (possible deadlock with + * onlining code). + * + * Will not modify the state of the memory block. + */ +static int virtio_mem_mb_offline_and_remove(struct virtio_mem *vm, + unsigned long mb_id) +{ + const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id); + int nid = vm->nid; + + if (nid == NUMA_NO_NODE) + nid = memory_add_physaddr_to_nid(addr); + + dev_dbg(&vm->vdev->dev, "offlining and removing memory block: %lu\n", + mb_id); + return offline_and_remove_memory(nid, addr, memory_block_size_bytes()); +} + /* * Trigger the workqueue so the device can perform its magic. */ @@ -537,7 +559,13 @@ static void virtio_mem_notify_offline(struct virtio_mem *vm, break; } - /* trigger the workqueue, maybe we can now unplug memory. */ + /* + * Trigger the workqueue, maybe we can now unplug memory. Also, + * when we offline and remove a memory block, this will re-trigger + * us immediately - which is often nice because the removal of + * the memory block (e.g., memmap) might have freed up memory + * on other memory blocks we manage. + */ virtio_mem_retry(vm); } @@ -1284,7 +1312,8 @@ static int virtio_mem_mb_unplug_any_sb_offline(struct virtio_mem *vm, * Unplug the desired number of plugged subblocks of an online memory block. * Will skip subblock that are busy. * - * Will modify the state of the memory block. + * Will modify the state of the memory block. Might temporarily drop the + * hotplug_mutex. * * Note: Can fail after some subblocks were successfully unplugged. Can * return 0 even if subblocks were busy and could not get unplugged. @@ -1340,9 +1369,19 @@ static int virtio_mem_mb_unplug_any_sb_online(struct virtio_mem *vm, } /* - * TODO: Once all subblocks of a memory block were unplugged, we want - * to offline the memory block and remove it. + * Once all subblocks of a memory block were unplugged, offline and + * remove it. This will usually not fail, as no memory is in use + * anymore - however some other notifiers might NACK the request. */ + if (virtio_mem_mb_test_sb_unplugged(vm, mb_id, 0, vm->nb_sb_per_mb)) { + mutex_unlock(&vm->hotplug_mutex); + rc = virtio_mem_mb_offline_and_remove(vm, mb_id); + mutex_lock(&vm->hotplug_mutex); + if (!rc) + virtio_mem_mb_set_state(vm, mb_id, + VIRTIO_MEM_MB_STATE_UNUSED); + } + return 0; } From 23e77b5dc9cd88709c48ada936c07bdd72c49426 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Thu, 7 May 2020 16:01:34 +0200 Subject: [PATCH 27/40] virtio-mem: Better retry handling Let's start with a retry interval of 5 seconds and double the time until we reach 5 minutes, in case we keep getting errors. Reset the retry interval in case we succeeded. The two main reasons for having to retry are - The hypervisor is busy and cannot process our request - We cannot reach the desired requested_size (esp., not enough memory can get unplugged because we can't allocate any subblocks). Tested-by: Pankaj Gupta Cc: "Michael S. Tsirkin" Cc: Jason Wang Cc: Oscar Salvador Cc: Michal Hocko Cc: Igor Mammedov Cc: Dave Young Cc: Andrew Morton Cc: Dan Williams Cc: Pavel Tatashin Cc: Stefan Hajnoczi Cc: Vlastimil Babka Signed-off-by: David Hildenbrand Link: https://lore.kernel.org/r/20200507140139.17083-11-david@redhat.com Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_mem.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/drivers/virtio/virtio_mem.c b/drivers/virtio/virtio_mem.c index a2edb87e5ed8..eb4c16d634e0 100644 --- a/drivers/virtio/virtio_mem.c +++ b/drivers/virtio/virtio_mem.c @@ -141,7 +141,9 @@ struct virtio_mem { /* Timer for retrying to plug/unplug memory. */ struct hrtimer retry_timer; -#define VIRTIO_MEM_RETRY_TIMER_MS 30000 + unsigned int retry_timer_ms; +#define VIRTIO_MEM_RETRY_TIMER_MIN_MS 50000 +#define VIRTIO_MEM_RETRY_TIMER_MAX_MS 300000 /* Memory notifier (online/offline events). */ struct notifier_block memory_notifier; @@ -1550,6 +1552,7 @@ retry: switch (rc) { case 0: + vm->retry_timer_ms = VIRTIO_MEM_RETRY_TIMER_MIN_MS; break; case -ENOSPC: /* @@ -1565,8 +1568,7 @@ retry: */ case -ENOMEM: /* Out of memory, try again later. */ - hrtimer_start(&vm->retry_timer, - ms_to_ktime(VIRTIO_MEM_RETRY_TIMER_MS), + hrtimer_start(&vm->retry_timer, ms_to_ktime(vm->retry_timer_ms), HRTIMER_MODE_REL); break; case -EAGAIN: @@ -1586,6 +1588,8 @@ static enum hrtimer_restart virtio_mem_timer_expired(struct hrtimer *timer) retry_timer); virtio_mem_retry(vm); + vm->retry_timer_ms = min_t(unsigned int, vm->retry_timer_ms * 2, + VIRTIO_MEM_RETRY_TIMER_MAX_MS); return HRTIMER_NORESTART; } @@ -1754,6 +1758,7 @@ static int virtio_mem_probe(struct virtio_device *vdev) spin_lock_init(&vm->removal_lock); hrtimer_init(&vm->retry_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); vm->retry_timer.function = virtio_mem_timer_expired; + vm->retry_timer_ms = VIRTIO_MEM_RETRY_TIMER_MIN_MS; /* register the virtqueue */ rc = virtio_mem_init_vq(vm); From ebf71552bb0e690cad523ad175e8c4c89a33c333 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Thu, 7 May 2020 16:01:35 +0200 Subject: [PATCH 28/40] virtio-mem: Add parent resource for all added "System RAM" Let's add a parent resource, named after the virtio device (inspired by drivers/dax/kmem.c). This allows user space to identify which memory belongs to which virtio-mem device. With this change and two virtio-mem devices: :/# cat /proc/iomem 00000000-00000fff : Reserved 00001000-0009fbff : System RAM [...] 140000000-333ffffff : virtio0 140000000-147ffffff : System RAM 148000000-14fffffff : System RAM 150000000-157ffffff : System RAM [...] 334000000-3033ffffff : virtio1 338000000-33fffffff : System RAM 340000000-347ffffff : System RAM 348000000-34fffffff : System RAM [...] Cc: "Michael S. Tsirkin" Cc: Pankaj Gupta Signed-off-by: David Hildenbrand Link: https://lore.kernel.org/r/20200507140139.17083-12-david@redhat.com Signed-off-by: Michael S. Tsirkin Reviewed-by: Pankaj Gupta --- drivers/virtio/virtio_mem.c | 52 ++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/drivers/virtio/virtio_mem.c b/drivers/virtio/virtio_mem.c index eb4c16d634e0..80cdb9e6b3c4 100644 --- a/drivers/virtio/virtio_mem.c +++ b/drivers/virtio/virtio_mem.c @@ -99,6 +99,9 @@ struct virtio_mem { /* Id of the next memory bock to prepare when needed. */ unsigned long next_mb_id; + /* The parent resource for all memory added via this device. */ + struct resource *parent_resource; + /* Summary of all memory block states. */ unsigned long nb_mb_state[VIRTIO_MEM_MB_STATE_COUNT]; #define VIRTIO_MEM_NB_OFFLINE_THRESHOLD 10 @@ -1741,6 +1744,44 @@ static int virtio_mem_init(struct virtio_mem *vm) return 0; } +static int virtio_mem_create_resource(struct virtio_mem *vm) +{ + /* + * When force-unloading the driver and removing the device, we + * could have a garbage pointer. Duplicate the string. + */ + const char *name = kstrdup(dev_name(&vm->vdev->dev), GFP_KERNEL); + + if (!name) + return -ENOMEM; + + vm->parent_resource = __request_mem_region(vm->addr, vm->region_size, + name, IORESOURCE_SYSTEM_RAM); + if (!vm->parent_resource) { + kfree(name); + dev_warn(&vm->vdev->dev, "could not reserve device region\n"); + return -EBUSY; + } + + /* The memory is not actually busy - make add_memory() work. */ + vm->parent_resource->flags &= ~IORESOURCE_BUSY; + return 0; +} + +static void virtio_mem_delete_resource(struct virtio_mem *vm) +{ + const char *name; + + if (!vm->parent_resource) + return; + + name = vm->parent_resource->name; + release_resource(vm->parent_resource); + kfree(vm->parent_resource); + kfree(name); + vm->parent_resource = NULL; +} + static int virtio_mem_probe(struct virtio_device *vdev) { struct virtio_mem *vm; @@ -1770,11 +1811,16 @@ static int virtio_mem_probe(struct virtio_device *vdev) if (rc) goto out_del_vq; + /* create the parent resource for all memory */ + rc = virtio_mem_create_resource(vm); + if (rc) + goto out_del_vq; + /* register callbacks */ vm->memory_notifier.notifier_call = virtio_mem_memory_notifier_cb; rc = register_memory_notifier(&vm->memory_notifier); if (rc) - goto out_del_vq; + goto out_del_resource; rc = register_virtio_mem_device(vm); if (rc) goto out_unreg_mem; @@ -1788,6 +1834,8 @@ static int virtio_mem_probe(struct virtio_device *vdev) return 0; out_unreg_mem: unregister_memory_notifier(&vm->memory_notifier); +out_del_resource: + virtio_mem_delete_resource(vm); out_del_vq: vdev->config->del_vqs(vdev); out_free_vm: @@ -1848,6 +1896,8 @@ static void virtio_mem_remove(struct virtio_device *vdev) vm->nb_mb_state[VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL] || vm->nb_mb_state[VIRTIO_MEM_MB_STATE_ONLINE_MOVABLE]) dev_warn(&vdev->dev, "device still has system memory added\n"); + else + virtio_mem_delete_resource(vm); /* remove all tracking data - no locking needed */ vfree(vm->mb_state); From 3c42e198e668e4040ef5cf3ad60d57765abc08a4 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Thu, 7 May 2020 16:01:36 +0200 Subject: [PATCH 29/40] virtio-mem: Drop manual check for already present memory Registering our parent resource will fail if any memory is still present (e.g., because somebody unloaded the driver and tries to reload it). No need for the manual check. Move our "unplug all" handling to after registering the resource. Cc: "Michael S. Tsirkin" Cc: Pankaj Gupta Signed-off-by: David Hildenbrand Link: https://lore.kernel.org/r/20200507140139.17083-13-david@redhat.com Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_mem.c | 55 ++++++++----------------------------- 1 file changed, 12 insertions(+), 43 deletions(-) diff --git a/drivers/virtio/virtio_mem.c b/drivers/virtio/virtio_mem.c index 80cdb9e6b3c4..8dd57b61b09b 100644 --- a/drivers/virtio/virtio_mem.c +++ b/drivers/virtio/virtio_mem.c @@ -1616,23 +1616,6 @@ static int virtio_mem_init_vq(struct virtio_mem *vm) return 0; } -/* - * Test if any memory in the range is present in Linux. - */ -static bool virtio_mem_any_memory_present(unsigned long start, - unsigned long size) -{ - const unsigned long start_pfn = PFN_DOWN(start); - const unsigned long end_pfn = PFN_UP(start + size); - unsigned long pfn; - - for (pfn = start_pfn; pfn != end_pfn; pfn++) - if (present_section_nr(pfn_to_section_nr(pfn))) - return true; - - return false; -} - static int virtio_mem_init(struct virtio_mem *vm) { const uint64_t phys_limit = 1UL << MAX_PHYSMEM_BITS; @@ -1664,32 +1647,6 @@ static int virtio_mem_init(struct virtio_mem *vm) virtio_cread(vm->vdev, struct virtio_mem_config, region_size, &vm->region_size); - /* - * If we still have memory plugged, we might have to unplug all - * memory first. However, if somebody simply unloaded the driver - * we would have to reinitialize the old state - something we don't - * support yet. Detect if we have any memory in the area present. - */ - if (vm->plugged_size) { - uint64_t usable_region_size; - - virtio_cread(vm->vdev, struct virtio_mem_config, - usable_region_size, &usable_region_size); - - if (virtio_mem_any_memory_present(vm->addr, - usable_region_size)) { - dev_err(&vm->vdev->dev, - "reloading the driver is not supported\n"); - return -EINVAL; - } - /* - * Note: it might happen that the device is busy and - * unplugging all memory might take some time. - */ - dev_info(&vm->vdev->dev, "unplugging all memory required\n"); - vm->unplug_all_required = 1; - } - /* * We always hotplug memory in memory block granularity. This way, * we have to wait for exactly one memory block to online. @@ -1760,6 +1717,8 @@ static int virtio_mem_create_resource(struct virtio_mem *vm) if (!vm->parent_resource) { kfree(name); dev_warn(&vm->vdev->dev, "could not reserve device region\n"); + dev_info(&vm->vdev->dev, + "reloading the driver is not supported\n"); return -EBUSY; } @@ -1816,6 +1775,16 @@ static int virtio_mem_probe(struct virtio_device *vdev) if (rc) goto out_del_vq; + /* + * If we still have memory plugged, we have to unplug all memory first. + * Registering our parent resource makes sure that this memory isn't + * actually in use (e.g., trying to reload the driver). + */ + if (vm->plugged_size) { + vm->unplug_all_required = 1; + dev_info(&vm->vdev->dev, "unplugging all memory is required\n"); + } + /* register callbacks */ vm->memory_notifier.notifier_call = virtio_mem_memory_notifier_cb; rc = register_memory_notifier(&vm->memory_notifier); From 562e08cd249f98af3a3e0845998f3b27b56b0067 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Thu, 7 May 2020 16:01:37 +0200 Subject: [PATCH 30/40] virtio-mem: Unplug subblocks right-to-left We unplug blocks right-to-left, let's also unplug subblocks within a block right-to-left. Cc: "Michael S. Tsirkin" Cc: Pankaj Gupta Signed-off-by: David Hildenbrand Link: https://lore.kernel.org/r/20200507140139.17083-14-david@redhat.com Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_mem.c | 38 ++++++++++++++++--------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/drivers/virtio/virtio_mem.c b/drivers/virtio/virtio_mem.c index 8dd57b61b09b..a719e1a04ac7 100644 --- a/drivers/virtio/virtio_mem.c +++ b/drivers/virtio/virtio_mem.c @@ -353,18 +353,6 @@ static bool virtio_mem_mb_test_sb_unplugged(struct virtio_mem *vm, return find_next_bit(vm->sb_bitmap, bit + count, bit) >= bit + count; } -/* - * Find the first plugged subblock. Returns vm->nb_sb_per_mb in case there is - * none. - */ -static int virtio_mem_mb_first_plugged_sb(struct virtio_mem *vm, - unsigned long mb_id) -{ - const int bit = (mb_id - vm->first_mb_id) * vm->nb_sb_per_mb; - - return find_next_bit(vm->sb_bitmap, bit + vm->nb_sb_per_mb, bit) - bit; -} - /* * Find the first unplugged subblock. Returns vm->nb_sb_per_mb in case there is * none. @@ -1016,21 +1004,27 @@ static int virtio_mem_mb_unplug_any_sb(struct virtio_mem *vm, int sb_id, count; int rc; + sb_id = vm->nb_sb_per_mb - 1; while (*nb_sb) { - sb_id = virtio_mem_mb_first_plugged_sb(vm, mb_id); - if (sb_id >= vm->nb_sb_per_mb) + /* Find the next candidate subblock */ + while (sb_id >= 0 && + virtio_mem_mb_test_sb_unplugged(vm, mb_id, sb_id, 1)) + sb_id--; + if (sb_id < 0) break; + /* Try to unplug multiple subblocks at a time */ count = 1; - while (count < *nb_sb && - sb_id + count < vm->nb_sb_per_mb && - virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id + count, - 1)) + while (count < *nb_sb && sb_id > 0 && + virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id - 1, 1)) { count++; + sb_id--; + } rc = virtio_mem_mb_unplug_sb(vm, mb_id, sb_id, count); if (rc) return rc; *nb_sb -= count; + sb_id--; } return 0; @@ -1337,12 +1331,12 @@ static int virtio_mem_mb_unplug_any_sb_online(struct virtio_mem *vm, * we should sense via something like is_mem_section_removable() * first if it makes sense to go ahead any try to allocate. */ - for (sb_id = 0; sb_id < vm->nb_sb_per_mb && *nb_sb; sb_id++) { + for (sb_id = vm->nb_sb_per_mb - 1; sb_id >= 0 && *nb_sb; sb_id--) { /* Find the next candidate subblock */ - while (sb_id < vm->nb_sb_per_mb && + while (sb_id >= 0 && !virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id, 1)) - sb_id++; - if (sb_id >= vm->nb_sb_per_mb) + sb_id--; + if (sb_id < 0) break; start_pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) + From 8d4edcfe78c0008d95effc0c90455cee59e18d10 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Thu, 7 May 2020 16:01:38 +0200 Subject: [PATCH 31/40] virtio-mem: Use -ETXTBSY as error code if the device is busy Let's be able to distinguish if the device or if memory is busy. Cc: "Michael S. Tsirkin" Cc: Pankaj Gupta Signed-off-by: David Hildenbrand Link: https://lore.kernel.org/r/20200507140139.17083-15-david@redhat.com Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_mem.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/drivers/virtio/virtio_mem.c b/drivers/virtio/virtio_mem.c index a719e1a04ac7..abd93b778a26 100644 --- a/drivers/virtio/virtio_mem.c +++ b/drivers/virtio/virtio_mem.c @@ -893,7 +893,7 @@ static int virtio_mem_send_plug_request(struct virtio_mem *vm, uint64_t addr, case VIRTIO_MEM_RESP_NACK: return -EAGAIN; case VIRTIO_MEM_RESP_BUSY: - return -EBUSY; + return -ETXTBSY; case VIRTIO_MEM_RESP_ERROR: return -EINVAL; default: @@ -919,7 +919,7 @@ static int virtio_mem_send_unplug_request(struct virtio_mem *vm, uint64_t addr, vm->plugged_size -= size; return 0; case VIRTIO_MEM_RESP_BUSY: - return -EBUSY; + return -ETXTBSY; case VIRTIO_MEM_RESP_ERROR: return -EINVAL; default: @@ -941,7 +941,7 @@ static int virtio_mem_send_unplug_all_request(struct virtio_mem *vm) atomic_set(&vm->config_changed, 1); return 0; case VIRTIO_MEM_RESP_BUSY: - return -EBUSY; + return -ETXTBSY; default: return -ENOMEM; } @@ -1557,11 +1557,15 @@ retry: * or we have too many offline memory blocks. */ break; - case -EBUSY: + case -ETXTBSY: /* * The hypervisor cannot process our request right now - * (e.g., out of memory, migrating) or we cannot free up - * any memory to unplug it (all plugged memory is busy). + * (e.g., out of memory, migrating); + */ + case -EBUSY: + /* + * We cannot free up any memory to unplug it (all plugged memory + * is busy). */ case -ENOMEM: /* Out of memory, try again later. */ From 72f9525ad76b1ddfe663285805982e9d57c7b2c2 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Thu, 7 May 2020 16:01:39 +0200 Subject: [PATCH 32/40] virtio-mem: Try to unplug the complete online memory block first Right now, we always try to unplug single subblocks when processing an online memory block. Let's try to unplug the complete online memory block first, in case it is fully plugged and the unplug request is large enough. Fallback to single subblocks in case the memory block cannot get unplugged as a whole. Cc: "Michael S. Tsirkin" Cc: Pankaj Gupta Signed-off-by: David Hildenbrand Link: https://lore.kernel.org/r/20200507140139.17083-16-david@redhat.com Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_mem.c | 88 ++++++++++++++++++++++++------------- 1 file changed, 57 insertions(+), 31 deletions(-) diff --git a/drivers/virtio/virtio_mem.c b/drivers/virtio/virtio_mem.c index abd93b778a26..9e523db3bee1 100644 --- a/drivers/virtio/virtio_mem.c +++ b/drivers/virtio/virtio_mem.c @@ -1307,6 +1307,46 @@ static int virtio_mem_mb_unplug_any_sb_offline(struct virtio_mem *vm, return 0; } +/* + * Unplug the given plugged subblocks of an online memory block. + * + * Will modify the state of the memory block. + */ +static int virtio_mem_mb_unplug_sb_online(struct virtio_mem *vm, + unsigned long mb_id, int sb_id, + int count) +{ + const unsigned long nr_pages = PFN_DOWN(vm->subblock_size) * count; + unsigned long start_pfn; + int rc; + + start_pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) + + sb_id * vm->subblock_size); + rc = alloc_contig_range(start_pfn, start_pfn + nr_pages, + MIGRATE_MOVABLE, GFP_KERNEL); + if (rc == -ENOMEM) + /* whoops, out of memory */ + return rc; + if (rc) + return -EBUSY; + + /* Mark it as fake-offline before unplugging it */ + virtio_mem_set_fake_offline(start_pfn, nr_pages, true); + adjust_managed_page_count(pfn_to_page(start_pfn), -nr_pages); + + /* Try to unplug the allocated memory */ + rc = virtio_mem_mb_unplug_sb(vm, mb_id, sb_id, count); + if (rc) { + /* Return the memory to the buddy. */ + virtio_mem_fake_online(start_pfn, nr_pages); + return rc; + } + + virtio_mem_mb_set_state(vm, mb_id, + VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL); + return 0; +} + /* * Unplug the desired number of plugged subblocks of an online memory block. * Will skip subblock that are busy. @@ -1321,16 +1361,21 @@ static int virtio_mem_mb_unplug_any_sb_online(struct virtio_mem *vm, unsigned long mb_id, uint64_t *nb_sb) { - const unsigned long nr_pages = PFN_DOWN(vm->subblock_size); - unsigned long start_pfn; int rc, sb_id; - /* - * TODO: To increase the performance we want to try bigger, consecutive - * subblocks first before falling back to single subblocks. Also, - * we should sense via something like is_mem_section_removable() - * first if it makes sense to go ahead any try to allocate. - */ + /* If possible, try to unplug the complete block in one shot. */ + if (*nb_sb >= vm->nb_sb_per_mb && + virtio_mem_mb_test_sb_plugged(vm, mb_id, 0, vm->nb_sb_per_mb)) { + rc = virtio_mem_mb_unplug_sb_online(vm, mb_id, 0, + vm->nb_sb_per_mb); + if (!rc) { + *nb_sb -= vm->nb_sb_per_mb; + goto unplugged; + } else if (rc != -EBUSY) + return rc; + } + + /* Fallback to single subblocks. */ for (sb_id = vm->nb_sb_per_mb - 1; sb_id >= 0 && *nb_sb; sb_id--) { /* Find the next candidate subblock */ while (sb_id >= 0 && @@ -1339,34 +1384,15 @@ static int virtio_mem_mb_unplug_any_sb_online(struct virtio_mem *vm, if (sb_id < 0) break; - start_pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) + - sb_id * vm->subblock_size); - rc = alloc_contig_range(start_pfn, start_pfn + nr_pages, - MIGRATE_MOVABLE, GFP_KERNEL); - if (rc == -ENOMEM) - /* whoops, out of memory */ - return rc; - if (rc) - /* memory busy, we can't unplug this chunk */ + rc = virtio_mem_mb_unplug_sb_online(vm, mb_id, sb_id, 1); + if (rc == -EBUSY) continue; - - /* Mark it as fake-offline before unplugging it */ - virtio_mem_set_fake_offline(start_pfn, nr_pages, true); - adjust_managed_page_count(pfn_to_page(start_pfn), -nr_pages); - - /* Try to unplug the allocated memory */ - rc = virtio_mem_mb_unplug_sb(vm, mb_id, sb_id, 1); - if (rc) { - /* Return the memory to the buddy. */ - virtio_mem_fake_online(start_pfn, nr_pages); + else if (rc) return rc; - } - - virtio_mem_mb_set_state(vm, mb_id, - VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL); *nb_sb -= 1; } +unplugged: /* * Once all subblocks of a memory block were unplugged, offline and * remove it. This will usually not fail, as no memory is in use From fce8afd76e3a4d8c59c92f84f8027569fd7031d0 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Fri, 15 May 2020 12:14:02 +0200 Subject: [PATCH 33/40] virtio-mem: Don't rely on implicit compiler padding for requests The compiler will add padding after the last member, make that explicit. The size of a request is always 24 bytes. The size of a response always 10 bytes. Add compile-time checks. Cc: "Michael S. Tsirkin" Cc: Pankaj Gupta Cc: teawater Signed-off-by: David Hildenbrand Link: https://lore.kernel.org/r/20200515101402.16597-1-david@redhat.com Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_mem.c | 3 +++ include/uapi/linux/virtio_mem.h | 3 +++ 2 files changed, 6 insertions(+) diff --git a/drivers/virtio/virtio_mem.c b/drivers/virtio/virtio_mem.c index 9e523db3bee1..f658fe9149be 100644 --- a/drivers/virtio/virtio_mem.c +++ b/drivers/virtio/virtio_mem.c @@ -1770,6 +1770,9 @@ static int virtio_mem_probe(struct virtio_device *vdev) struct virtio_mem *vm; int rc = -EINVAL; + BUILD_BUG_ON(sizeof(struct virtio_mem_req) != 24); + BUILD_BUG_ON(sizeof(struct virtio_mem_resp) != 10); + vdev->priv = vm = kzalloc(sizeof(*vm), GFP_KERNEL); if (!vm) return -ENOMEM; diff --git a/include/uapi/linux/virtio_mem.h b/include/uapi/linux/virtio_mem.h index e0a9dc7397c3..a455c488a995 100644 --- a/include/uapi/linux/virtio_mem.h +++ b/include/uapi/linux/virtio_mem.h @@ -103,16 +103,19 @@ struct virtio_mem_req_plug { __virtio64 addr; __virtio16 nb_blocks; + __virtio16 padding[3]; }; struct virtio_mem_req_unplug { __virtio64 addr; __virtio16 nb_blocks; + __virtio16 padding[3]; }; struct virtio_mem_req_state { __virtio64 addr; __virtio16 nb_blocks; + __virtio16 padding[3]; }; struct virtio_mem_req { From bb02e6e63d0e71188bc5fe5f4732e66bc8b5dceb Mon Sep 17 00:00:00 2001 From: Zhu Lingshan Date: Fri, 5 Jun 2020 18:27:12 +0800 Subject: [PATCH 34/40] ifcvf: ignore continuous setting same status value User space may try to set status of same value multiple times, this patch handles this case more efficiently by ignoring the setting of the same status value. Signed-off-by: Zhu Lingshan Link: https://lore.kernel.org/r/1591352835-22441-3-git-send-email-lingshan.zhu@intel.com Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/ifcvf/ifcvf_main.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c index d529ed681fe6..63a6366b4355 100644 --- a/drivers/vdpa/ifcvf/ifcvf_main.c +++ b/drivers/vdpa/ifcvf/ifcvf_main.c @@ -179,6 +179,9 @@ static void ifcvf_vdpa_set_status(struct vdpa_device *vdpa_dev, u8 status) adapter = dev_get_drvdata(vdpa_dev->dev.parent); status_old = ifcvf_get_status(vf); + if (status_old == status) + return; + if ((status_old & VIRTIO_CONFIG_S_DRIVER_OK) && !(status & VIRTIO_CONFIG_S_DRIVER_OK)) { ifcvf_stop_datapath(adapter); From 776f395004d829bbbf18c159ed9beb517a208c71 Mon Sep 17 00:00:00 2001 From: Zhu Lingshan Date: Fri, 5 Jun 2020 18:27:13 +0800 Subject: [PATCH 35/40] vhost_vdpa: Support config interrupt in vdpa This commit implements config interrupt support in vhost_vdpa layer. Signed-off-by: Zhu Lingshan Acked-by: Jason Wang Link: https://lore.kernel.org/r/1591352835-22441-4-git-send-email-lingshan.zhu@intel.com Signed-off-by: Michael S. Tsirkin --- drivers/vhost/vdpa.c | 47 ++++++++++++++++++++++++++++++++++++++ include/uapi/linux/vhost.h | 4 ++++ 2 files changed, 51 insertions(+) diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c index 6ca7660ee6b5..77a0c9fb6cc3 100644 --- a/drivers/vhost/vdpa.c +++ b/drivers/vhost/vdpa.c @@ -22,6 +22,7 @@ #include #include #include +#include #include "vhost.h" @@ -71,6 +72,7 @@ struct vhost_vdpa { int nvqs; int virtio_id; int minor; + struct eventfd_ctx *config_ctx; }; static DEFINE_IDA(vhost_vdpa_ida); @@ -102,6 +104,17 @@ static irqreturn_t vhost_vdpa_virtqueue_cb(void *private) return IRQ_HANDLED; } +static irqreturn_t vhost_vdpa_config_cb(void *private) +{ + struct vhost_vdpa *v = private; + struct eventfd_ctx *config_ctx = v->config_ctx; + + if (config_ctx) + eventfd_signal(config_ctx, 1); + + return IRQ_HANDLED; +} + static void vhost_vdpa_reset(struct vhost_vdpa *v) { struct vdpa_device *vdpa = v->vdpa; @@ -289,6 +302,36 @@ static long vhost_vdpa_get_vring_num(struct vhost_vdpa *v, u16 __user *argp) return 0; } +static void vhost_vdpa_config_put(struct vhost_vdpa *v) +{ + if (v->config_ctx) + eventfd_ctx_put(v->config_ctx); +} + +static long vhost_vdpa_set_config_call(struct vhost_vdpa *v, u32 __user *argp) +{ + struct vdpa_callback cb; + int fd; + struct eventfd_ctx *ctx; + + cb.callback = vhost_vdpa_config_cb; + cb.private = v->vdpa; + if (copy_from_user(&fd, argp, sizeof(fd))) + return -EFAULT; + + ctx = fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(fd); + swap(ctx, v->config_ctx); + + if (!IS_ERR_OR_NULL(ctx)) + eventfd_ctx_put(ctx); + + if (IS_ERR(v->config_ctx)) + return PTR_ERR(v->config_ctx); + + v->vdpa->config->set_config_cb(v->vdpa, &cb); + + return 0; +} static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd, void __user *argp) { @@ -396,6 +439,9 @@ static long vhost_vdpa_unlocked_ioctl(struct file *filep, case VHOST_SET_LOG_FD: r = -ENOIOCTLCMD; break; + case VHOST_VDPA_SET_CONFIG_CALL: + r = vhost_vdpa_set_config_call(v, argp); + break; default: r = vhost_dev_ioctl(&v->vdev, cmd, argp); if (r == -ENOIOCTLCMD) @@ -730,6 +776,7 @@ static int vhost_vdpa_release(struct inode *inode, struct file *filep) vhost_dev_stop(&v->vdev); vhost_vdpa_iotlb_free(v); vhost_vdpa_free_domain(v); + vhost_vdpa_config_put(v); vhost_dev_cleanup(&v->vdev); kfree(v->vdev.vqs); mutex_unlock(&d->mutex); diff --git a/include/uapi/linux/vhost.h b/include/uapi/linux/vhost.h index 9fe72e4b1373..0c2349612e77 100644 --- a/include/uapi/linux/vhost.h +++ b/include/uapi/linux/vhost.h @@ -15,6 +15,8 @@ #include #include +#define VHOST_FILE_UNBIND -1 + /* ioctls */ #define VHOST_VIRTIO 0xAF @@ -140,4 +142,6 @@ /* Get the max ring size. */ #define VHOST_VDPA_GET_VRING_NUM _IOR(VHOST_VIRTIO, 0x76, __u16) +/* Set event fd for config interrupt*/ +#define VHOST_VDPA_SET_CONFIG_CALL _IOW(VHOST_VIRTIO, 0x77, int) #endif From e0136c16fae95dc3762f3912f6f9336f5f57fe81 Mon Sep 17 00:00:00 2001 From: Zhu Lingshan Date: Fri, 5 Jun 2020 18:27:14 +0800 Subject: [PATCH 36/40] vhost: replace -1 with VHOST_FILE_UNBIND in ioctls This commit replaces -1 with VHOST_FILE_UNBIND in ioctls since we have added such a macro in the uapi header for vdpa_host. Signed-off-by: Zhu Lingshan Acked-by: Jason Wang Link: https://lore.kernel.org/r/1591352835-22441-5-git-send-email-lingshan.zhu@intel.com Signed-off-by: Michael S. Tsirkin --- drivers/vhost/vhost.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index 534d1267b761..172da092107e 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -1612,7 +1612,7 @@ long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *arg r = -EFAULT; break; } - eventfp = f.fd == -1 ? NULL : eventfd_fget(f.fd); + eventfp = f.fd == VHOST_FILE_UNBIND ? NULL : eventfd_fget(f.fd); if (IS_ERR(eventfp)) { r = PTR_ERR(eventfp); break; @@ -1628,7 +1628,7 @@ long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *arg r = -EFAULT; break; } - ctx = f.fd == -1 ? NULL : eventfd_ctx_fdget(f.fd); + ctx = f.fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(f.fd); if (IS_ERR(ctx)) { r = PTR_ERR(ctx); break; @@ -1640,7 +1640,7 @@ long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *arg r = -EFAULT; break; } - ctx = f.fd == -1 ? NULL : eventfd_ctx_fdget(f.fd); + ctx = f.fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(f.fd); if (IS_ERR(ctx)) { r = PTR_ERR(ctx); break; @@ -1765,7 +1765,7 @@ long vhost_dev_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp) r = get_user(fd, (int __user *)argp); if (r < 0) break; - ctx = fd == -1 ? NULL : eventfd_ctx_fdget(fd); + ctx = fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(fd); if (IS_ERR(ctx)) { r = PTR_ERR(ctx); break; From e7991f376a4dd837f455d9e2112a280617854d12 Mon Sep 17 00:00:00 2001 From: Zhu Lingshan Date: Fri, 5 Jun 2020 18:27:15 +0800 Subject: [PATCH 37/40] ifcvf: implement config interrupt in IFCVF This commit implements config interrupt support in IFC VF Signed-off-by: Zhu Lingshan Acked-by: Jason Wang Link: https://lore.kernel.org/r/1591352835-22441-6-git-send-email-lingshan.zhu@intel.com Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/ifcvf/ifcvf_base.c | 3 +++ drivers/vdpa/ifcvf/ifcvf_base.h | 4 ++++ drivers/vdpa/ifcvf/ifcvf_main.c | 23 ++++++++++++++++++++++- 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/drivers/vdpa/ifcvf/ifcvf_base.c b/drivers/vdpa/ifcvf/ifcvf_base.c index e24371d644b5..94bf0328b68d 100644 --- a/drivers/vdpa/ifcvf/ifcvf_base.c +++ b/drivers/vdpa/ifcvf/ifcvf_base.c @@ -185,6 +185,9 @@ void ifcvf_set_status(struct ifcvf_hw *hw, u8 status) void ifcvf_reset(struct ifcvf_hw *hw) { + hw->config_cb.callback = NULL; + hw->config_cb.private = NULL; + ifcvf_set_status(hw, 0); /* flush set_status, make sure VF is stopped, reset */ ifcvf_get_status(hw); diff --git a/drivers/vdpa/ifcvf/ifcvf_base.h b/drivers/vdpa/ifcvf/ifcvf_base.h index e80307092351..f4554412e607 100644 --- a/drivers/vdpa/ifcvf/ifcvf_base.h +++ b/drivers/vdpa/ifcvf/ifcvf_base.h @@ -27,6 +27,7 @@ ((1ULL << VIRTIO_NET_F_MAC) | \ (1ULL << VIRTIO_F_ANY_LAYOUT) | \ (1ULL << VIRTIO_F_VERSION_1) | \ + (1ULL << VIRTIO_NET_F_STATUS) | \ (1ULL << VIRTIO_F_ORDER_PLATFORM) | \ (1ULL << VIRTIO_F_IOMMU_PLATFORM) | \ (1ULL << VIRTIO_NET_F_MRG_RXBUF)) @@ -81,6 +82,9 @@ struct ifcvf_hw { void __iomem *net_cfg; struct vring_info vring[IFCVF_MAX_QUEUE_PAIRS * 2]; void __iomem * const *base; + char config_msix_name[256]; + struct vdpa_callback config_cb; + }; struct ifcvf_adapter { diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c index 63a6366b4355..f5a60c14b979 100644 --- a/drivers/vdpa/ifcvf/ifcvf_main.c +++ b/drivers/vdpa/ifcvf/ifcvf_main.c @@ -18,6 +18,16 @@ #define DRIVER_AUTHOR "Intel Corporation" #define IFCVF_DRIVER_NAME "ifcvf" +static irqreturn_t ifcvf_config_changed(int irq, void *arg) +{ + struct ifcvf_hw *vf = arg; + + if (vf->config_cb.callback) + return vf->config_cb.callback(vf->config_cb.private); + + return IRQ_HANDLED; +} + static irqreturn_t ifcvf_intr_handler(int irq, void *arg) { struct vring_info *vring = arg; @@ -59,6 +69,14 @@ static int ifcvf_request_irq(struct ifcvf_adapter *adapter) return ret; } + snprintf(vf->config_msix_name, 256, "ifcvf[%s]-config\n", + pci_name(pdev)); + vector = 0; + irq = pci_irq_vector(pdev, vector); + ret = devm_request_irq(&pdev->dev, irq, + ifcvf_config_changed, 0, + vf->config_msix_name, vf); + for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) { snprintf(vf->vring[i].msix_name, 256, "ifcvf[%s]-%d\n", pci_name(pdev), i); @@ -328,7 +346,10 @@ static void ifcvf_vdpa_set_config(struct vdpa_device *vdpa_dev, static void ifcvf_vdpa_set_config_cb(struct vdpa_device *vdpa_dev, struct vdpa_callback *cb) { - /* We don't support config interrupt */ + struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); + + vf->config_cb.callback = cb->callback; + vf->config_cb.private = cb->private; } /* From b3fb6de7c6019c5d8495c3a115d42a0f118f631c Mon Sep 17 00:00:00 2001 From: "Michael S. Tsirkin" Date: Mon, 8 Jun 2020 01:43:22 -0400 Subject: [PATCH 38/40] virtio-mem: drop unnecessary initialization rc is initialized to -ENIVAL but that's never used. Drop it. Fixes: 5f1f79bbc9e2 ("virtio-mem: Paravirtualized memory hotplug") Reported-by: kernel test robot Signed-off-by: Michael S. Tsirkin Acked-by: David Hildenbrand --- drivers/virtio/virtio_mem.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/virtio/virtio_mem.c b/drivers/virtio/virtio_mem.c index f658fe9149be..2f357142ea5e 100644 --- a/drivers/virtio/virtio_mem.c +++ b/drivers/virtio/virtio_mem.c @@ -1768,7 +1768,7 @@ static void virtio_mem_delete_resource(struct virtio_mem *vm) static int virtio_mem_probe(struct virtio_device *vdev) { struct virtio_mem *vm; - int rc = -EINVAL; + int rc; BUILD_BUG_ON(sizeof(struct virtio_mem_req) != 24); BUILD_BUG_ON(sizeof(struct virtio_mem_resp) != 10); From 544fc7dbbf920a3e64d109c416ee229e8e1763c5 Mon Sep 17 00:00:00 2001 From: "Michael S. Tsirkin" Date: Mon, 8 Jun 2020 02:03:15 -0400 Subject: [PATCH 39/40] virtio_mem: convert device block size into 64bit If subblock size is large (e.g. 1G) 32 bit math involving it can overflow. Rather than try to catch all instances of that, let's tweak block size to 64 bit. It ripples through UAPI which is an ABI change, but it's not too late to make it, and it will allow supporting >4Gbyte blocks while might become necessary down the road. Fixes: 5f1f79bbc9e26 ("virtio-mem: Paravirtualized memory hotplug") Signed-off-by: Michael S. Tsirkin Acked-by: David Hildenbrand --- drivers/virtio/virtio_mem.c | 18 +++++++++--------- include/uapi/linux/virtio_mem.h | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/drivers/virtio/virtio_mem.c b/drivers/virtio/virtio_mem.c index 2f357142ea5e..50c689f25045 100644 --- a/drivers/virtio/virtio_mem.c +++ b/drivers/virtio/virtio_mem.c @@ -77,7 +77,7 @@ struct virtio_mem { uint64_t requested_size; /* The device block size (for communicating with the device). */ - uint32_t device_block_size; + uint64_t device_block_size; /* The translated node id. NUMA_NO_NODE in case not specified. */ int nid; /* Physical start address of the memory region. */ @@ -86,7 +86,7 @@ struct virtio_mem { uint64_t region_size; /* The subblock size. */ - uint32_t subblock_size; + uint64_t subblock_size; /* The number of subblocks per memory block. */ uint32_t nb_sb_per_mb; @@ -1698,9 +1698,9 @@ static int virtio_mem_init(struct virtio_mem *vm) * - At least the device block size. * In the worst case, a single subblock per memory block. */ - vm->subblock_size = PAGE_SIZE * 1u << max_t(uint32_t, MAX_ORDER - 1, - pageblock_order); - vm->subblock_size = max_t(uint32_t, vm->device_block_size, + vm->subblock_size = PAGE_SIZE * 1ul << max_t(uint32_t, MAX_ORDER - 1, + pageblock_order); + vm->subblock_size = max_t(uint64_t, vm->device_block_size, vm->subblock_size); vm->nb_sb_per_mb = memory_block_size_bytes() / vm->subblock_size; @@ -1713,12 +1713,12 @@ static int virtio_mem_init(struct virtio_mem *vm) dev_info(&vm->vdev->dev, "start address: 0x%llx", vm->addr); dev_info(&vm->vdev->dev, "region size: 0x%llx", vm->region_size); - dev_info(&vm->vdev->dev, "device block size: 0x%x", - vm->device_block_size); + dev_info(&vm->vdev->dev, "device block size: 0x%llx", + (unsigned long long)vm->device_block_size); dev_info(&vm->vdev->dev, "memory block size: 0x%lx", memory_block_size_bytes()); - dev_info(&vm->vdev->dev, "subblock size: 0x%x", - vm->subblock_size); + dev_info(&vm->vdev->dev, "subblock size: 0x%llx", + (unsigned long long)vm->subblock_size); if (vm->nid != NUMA_NO_NODE) dev_info(&vm->vdev->dev, "nid: %d", vm->nid); diff --git a/include/uapi/linux/virtio_mem.h b/include/uapi/linux/virtio_mem.h index a455c488a995..a9ffe041843c 100644 --- a/include/uapi/linux/virtio_mem.h +++ b/include/uapi/linux/virtio_mem.h @@ -185,10 +185,10 @@ struct virtio_mem_resp { struct virtio_mem_config { /* Block size and alignment. Cannot change. */ - __u32 block_size; + __u64 block_size; /* Valid with VIRTIO_MEM_F_ACPI_PXM. Cannot change. */ __u16 node_id; - __u16 padding; + __u8 padding[6]; /* Start address of the memory region. Cannot change. */ __u64 addr; /* Region size (maximum). Cannot change. */ From 044e4b09223039e571e6ec540e25552054208765 Mon Sep 17 00:00:00 2001 From: "Michael S. Tsirkin" Date: Mon, 8 Jun 2020 08:42:09 -0400 Subject: [PATCH 40/40] vhost/test: fix up after API change Pass a flag to request kernel thread use. Fixes: 01fcb1cbc88e ("vhost: allow device that does not depend on vhost worker") Signed-off-by: Michael S. Tsirkin --- drivers/vhost/test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/vhost/test.c b/drivers/vhost/test.c index 9a3a09005e03..0466921f4772 100644 --- a/drivers/vhost/test.c +++ b/drivers/vhost/test.c @@ -120,7 +120,7 @@ static int vhost_test_open(struct inode *inode, struct file *f) vqs[VHOST_TEST_VQ] = &n->vqs[VHOST_TEST_VQ]; n->vqs[VHOST_TEST_VQ].handle_kick = handle_vq_kick; vhost_dev_init(dev, vqs, VHOST_TEST_VQ_MAX, UIO_MAXIOV, - VHOST_TEST_PKT_WEIGHT, VHOST_TEST_WEIGHT, NULL); + VHOST_TEST_PKT_WEIGHT, VHOST_TEST_WEIGHT, true, NULL); f->private_data = n;