diff options
| -rw-r--r-- | Documentation/driver-api/cxl/memory-devices.rst | 14 | ||||
| -rw-r--r-- | drivers/cxl/Kconfig | 13 | ||||
| -rw-r--r-- | drivers/cxl/Makefile | 8 | ||||
| -rw-r--r-- | drivers/cxl/bus.c | 29 | ||||
| -rw-r--r-- | drivers/cxl/core.c | 261 | ||||
| -rw-r--r-- | drivers/cxl/cxl.h | 162 | ||||
| -rw-r--r-- | drivers/cxl/mem.h | 79 | ||||
| -rw-r--r-- | drivers/cxl/pci.c (renamed from drivers/cxl/mem.c) | 397 |
8 files changed, 698 insertions, 265 deletions
diff --git a/Documentation/driver-api/cxl/memory-devices.rst b/Documentation/driver-api/cxl/memory-devices.rst index 1bad466f9167..44c8ddbc8415 100644 --- a/Documentation/driver-api/cxl/memory-devices.rst +++ b/Documentation/driver-api/cxl/memory-devices.rst @@ -22,16 +22,16 @@ This section covers the driver infrastructure for a CXL memory device. CXL Memory Device ----------------- -.. kernel-doc:: drivers/cxl/mem.c - :doc: cxl mem +.. kernel-doc:: drivers/cxl/pci.c + :doc: cxl pci -.. kernel-doc:: drivers/cxl/mem.c +.. kernel-doc:: drivers/cxl/pci.c :internal: -CXL Bus -------- -.. kernel-doc:: drivers/cxl/bus.c - :doc: cxl bus +CXL Core +-------- +.. kernel-doc:: drivers/cxl/core.c + :doc: cxl core External Interfaces =================== diff --git a/drivers/cxl/Kconfig b/drivers/cxl/Kconfig index 97dc4d751651..5483ba92b6da 100644 --- a/drivers/cxl/Kconfig +++ b/drivers/cxl/Kconfig @@ -21,15 +21,10 @@ config CXL_MEM as if the memory was attached to the typical CPU memory controller. - Say 'y/m' to enable a driver (named "cxl_mem.ko" when built as - a module) that will attach to CXL.mem devices for - configuration, provisioning, and health monitoring. This - driver is required for dynamic provisioning of CXL.mem - attached memory which is a prerequisite for persistent memory - support. Typically volatile memory is mapped by platform - firmware and included in the platform memory map, but in some - cases the OS is responsible for mapping that memory. See - Chapter 2.3 Type 3 CXL Device in the CXL 2.0 specification. + Say 'y/m' to enable a driver that will attach to CXL.mem devices for + configuration and management primarily via the mailbox interface. See + Chapter 2.3 Type 3 CXL Device in the CXL 2.0 specification for more + details. If unsure say 'm'. diff --git a/drivers/cxl/Makefile b/drivers/cxl/Makefile index a314a1891f4d..d9d282dc15be 100644 --- a/drivers/cxl/Makefile +++ b/drivers/cxl/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_CXL_BUS) += cxl_bus.o -obj-$(CONFIG_CXL_MEM) += cxl_mem.o +obj-$(CONFIG_CXL_BUS) += cxl_core.o +obj-$(CONFIG_CXL_MEM) += cxl_pci.o ccflags-y += -DDEFAULT_SYMBOL_NAMESPACE=CXL -cxl_bus-y := bus.o -cxl_mem-y := mem.o +cxl_core-y := core.o +cxl_pci-y := pci.o diff --git a/drivers/cxl/bus.c b/drivers/cxl/bus.c deleted file mode 100644 index 58f74796d525..000000000000 --- a/drivers/cxl/bus.c +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* Copyright(c) 2020 Intel Corporation. All rights reserved. */ -#include <linux/device.h> -#include <linux/module.h> - -/** - * DOC: cxl bus - * - * The CXL bus provides namespace for control devices and a rendezvous - * point for cross-device interleave coordination. - */ -struct bus_type cxl_bus_type = { - .name = "cxl", -}; -EXPORT_SYMBOL_GPL(cxl_bus_type); - -static __init int cxl_bus_init(void) -{ - return bus_register(&cxl_bus_type); -} - -static void cxl_bus_exit(void) -{ - bus_unregister(&cxl_bus_type); -} - -module_init(cxl_bus_init); -module_exit(cxl_bus_exit); -MODULE_LICENSE("GPL v2"); diff --git a/drivers/cxl/core.c b/drivers/cxl/core.c new file mode 100644 index 000000000000..853666d8a9f5 --- /dev/null +++ b/drivers/cxl/core.c @@ -0,0 +1,261 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2020 Intel Corporation. All rights reserved. */ +#include <linux/io-64-nonatomic-lo-hi.h> +#include <linux/device.h> +#include <linux/module.h> +#include <linux/pci.h> +#include "cxl.h" + +/** + * DOC: cxl core + * + * The CXL core provides a sysfs hierarchy for control devices and a rendezvous + * point for cross-device interleave coordination through cxl ports. + */ + +/** + * cxl_probe_component_regs() - Detect CXL Component register blocks + * @dev: Host device of the @base mapping + * @base: Mapping containing the HDM Decoder Capability Header + * @map: Map object describing the register block information found + * + * See CXL 2.0 8.2.4 Component Register Layout and Definition + * See CXL 2.0 8.2.5.5 CXL Device Register Interface + * + * Probe for component register information and return it in map object. + */ +void cxl_probe_component_regs(struct device *dev, void __iomem *base, + struct cxl_component_reg_map *map) +{ + int cap, cap_count; + u64 cap_array; + + *map = (struct cxl_component_reg_map) { 0 }; + + /* + * CXL.cache and CXL.mem registers are at offset 0x1000 as defined in + * CXL 2.0 8.2.4 Table 141. + */ + base += CXL_CM_OFFSET; + + cap_array = readq(base + CXL_CM_CAP_HDR_OFFSET); + + if (FIELD_GET(CXL_CM_CAP_HDR_ID_MASK, cap_array) != CM_CAP_HDR_CAP_ID) { + dev_err(dev, + "Couldn't locate the CXL.cache and CXL.mem capability array header./n"); + return; + } + + /* It's assumed that future versions will be backward compatible */ + cap_count = FIELD_GET(CXL_CM_CAP_HDR_ARRAY_SIZE_MASK, cap_array); + + for (cap = 1; cap <= cap_count; cap++) { + void __iomem *register_block; + u32 hdr; + int decoder_cnt; + u16 cap_id, offset; + u32 length; + + hdr = readl(base + cap * 0x4); + + cap_id = FIELD_GET(CXL_CM_CAP_HDR_ID_MASK, hdr); + offset = FIELD_GET(CXL_CM_CAP_PTR_MASK, hdr); + register_block = base + offset; + + switch (cap_id) { + case CXL_CM_CAP_CAP_ID_HDM: + dev_dbg(dev, "found HDM decoder capability (0x%x)\n", + offset); + + hdr = readl(register_block); + + decoder_cnt = FIELD_GET(CXL_HDM_DECODER_COUNT_MASK, hdr); + length = 0x20 * decoder_cnt + 0x10; + + map->hdm_decoder.valid = true; + map->hdm_decoder.offset = offset; + map->hdm_decoder.size = length; + break; + default: + dev_dbg(dev, "Unknown CM cap ID: %d (0x%x)\n", cap_id, + offset); + break; + } + } +} +EXPORT_SYMBOL_GPL(cxl_probe_component_regs); + +/** + * cxl_probe_device_regs() - Detect CXL Device register blocks + * @dev: Host device of the @base mapping + * @base: Mapping of CXL 2.0 8.2.8 CXL Device Register Interface + * @map: Map object describing the register block information found + * + * Probe for device register information and return it in map object. + */ +void cxl_probe_device_regs(struct device *dev, void __iomem *base, + struct cxl_device_reg_map *map) +{ + int cap, cap_count; + u64 cap_array; + + *map = (struct cxl_device_reg_map){ 0 }; + + cap_array = readq(base + CXLDEV_CAP_ARRAY_OFFSET); + if (FIELD_GET(CXLDEV_CAP_ARRAY_ID_MASK, cap_array) != + CXLDEV_CAP_ARRAY_CAP_ID) + return; + + cap_count = FIELD_GET(CXLDEV_CAP_ARRAY_COUNT_MASK, cap_array); + + for (cap = 1; cap <= cap_count; cap++) { + u32 offset, length; + u16 cap_id; + + cap_id = FIELD_GET(CXLDEV_CAP_HDR_CAP_ID_MASK, + readl(base + cap * 0x10)); + offset = readl(base + cap * 0x10 + 0x4); + length = readl(base + cap * 0x10 + 0x8); + + switch (cap_id) { + case CXLDEV_CAP_CAP_ID_DEVICE_STATUS: + dev_dbg(dev, "found Status capability (0x%x)\n", offset); + + map->status.valid = true; + map->status.offset = offset; + map->status.size = length; + break; + case CXLDEV_CAP_CAP_ID_PRIMARY_MAILBOX: + dev_dbg(dev, "found Mailbox capability (0x%x)\n", offset); + map->mbox.valid = true; + map->mbox.offset = offset; + map->mbox.size = length; + break; + case CXLDEV_CAP_CAP_ID_SECONDARY_MAILBOX: + dev_dbg(dev, "found Secondary Mailbox capability (0x%x)\n", offset); + break; + case CXLDEV_CAP_CAP_ID_MEMDEV: + dev_dbg(dev, "found Memory Device capability (0x%x)\n", offset); + map->memdev.valid = true; + map->memdev.offset = offset; + map->memdev.size = length; + break; + default: + if (cap_id >= 0x8000) + dev_dbg(dev, "Vendor cap ID: %#x offset: %#x\n", cap_id, offset); + else + dev_dbg(dev, "Unknown cap ID: %#x offset: %#x\n", cap_id, offset); + break; + } + } +} +EXPORT_SYMBOL_GPL(cxl_probe_device_regs); + +static void __iomem *devm_cxl_iomap_block(struct device *dev, + resource_size_t addr, + resource_size_t length) +{ + void __iomem *ret_val; + struct resource *res; + + res = devm_request_mem_region(dev, addr, length, dev_name(dev)); + if (!res) { + resource_size_t end = addr + length - 1; + + dev_err(dev, "Failed to request region %pa-%pa\n", &addr, &end); + return NULL; + } + + ret_val = devm_ioremap(dev, addr, length); + if (!ret_val) + dev_err(dev, "Failed to map region %pr\n", res); + + return ret_val; +} + +int cxl_map_component_regs(struct pci_dev *pdev, + struct cxl_component_regs *regs, + struct cxl_register_map *map) +{ + struct device *dev = &pdev->dev; + resource_size_t phys_addr; + resource_size_t length; + + phys_addr = pci_resource_start(pdev, map->barno); + phys_addr += map->block_offset; + + phys_addr += map->component_map.hdm_decoder.offset; + length = map->component_map.hdm_decoder.size; + regs->hdm_decoder = devm_cxl_iomap_block(dev, phys_addr, length); + if (!regs->hdm_decoder) + return -ENOMEM; + + return 0; +} +EXPORT_SYMBOL_GPL(cxl_map_component_regs); + +int cxl_map_device_regs(struct pci_dev *pdev, + struct cxl_device_regs *regs, + struct cxl_register_map *map) +{ + struct device *dev = &pdev->dev; + resource_size_t phys_addr; + + phys_addr = pci_resource_start(pdev, map->barno); + phys_addr += map->block_offset; + + if (map->device_map.status.valid) { + resource_size_t addr; + resource_size_t length; + + addr = phys_addr + map->device_map.status.offset; + length = map->device_map.status.size; + regs->status = devm_cxl_iomap_block(dev, addr, length); + if (!regs->status) + return -ENOMEM; + } + + if (map->device_map.mbox.valid) { + resource_size_t addr; + resource_size_t length; + + addr = phys_addr + map->device_map.mbox.offset; + length = map->device_map.mbox.size; + regs->mbox = devm_cxl_iomap_block(dev, addr, length); + if (!regs->mbox) + return -ENOMEM; + } + + if (map->device_map.memdev.valid) { + resource_size_t addr; + resource_size_t length; + + addr = phys_addr + map->device_map.memdev.offset; + length = map->device_map.memdev.size; + regs->memdev = devm_cxl_iomap_block(dev, addr, length); + if (!regs->memdev) + return -ENOMEM; + } + + return 0; +} +EXPORT_SYMBOL_GPL(cxl_map_device_regs); + +struct bus_type cxl_bus_type = { + .name = "cxl", +}; +EXPORT_SYMBOL_GPL(cxl_bus_type); + +static __init int cxl_core_init(void) +{ + return bus_register(&cxl_bus_type); +} + +static void cxl_core_exit(void) +{ + bus_unregister(&cxl_bus_type); +} + +module_init(cxl_core_init); +module_exit(cxl_core_exit); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 6f14838c2d25..2c47e9cffd44 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -8,6 +8,31 @@ #include <linux/bitops.h> #include <linux/io.h> +/* CXL 2.0 8.2.5 CXL.cache and CXL.mem Registers*/ +#define CXL_CM_OFFSET 0x1000 +#define CXL_CM_CAP_HDR_OFFSET 0x0 +#define CXL_CM_CAP_HDR_ID_MASK GENMASK(15, 0) +#define CM_CAP_HDR_CAP_ID 1 +#define CXL_CM_CAP_HDR_VERSION_MASK GENMASK(19, 16) +#define CM_CAP_HDR_CAP_VERSION 1 +#define CXL_CM_CAP_HDR_CACHE_MEM_VERSION_MASK GENMASK(23, 20) +#define CM_CAP_HDR_CACHE_MEM_VERSION 1 +#define CXL_CM_CAP_HDR_ARRAY_SIZE_MASK GENMASK(31, 24) +#define CXL_CM_CAP_PTR_MASK GENMASK(31, 20) + +#define CXL_CM_CAP_CAP_ID_HDM 0x5 +#define CXL_CM_CAP_CAP_HDM_VERSION 1 + +/* HDM decoders CXL 2.0 8.2.5.12 CXL HDM Decoder Capability Structure */ +#define CXL_HDM_DECODER_CAP_OFFSET 0x0 +#define CXL_HDM_DECODER_COUNT_MASK GENMASK(3, 0) +#define CXL_HDM_DECODER_TARGET_COUNT_MASK GENMASK(7, 4) +#define CXL_HDM_DECODER0_BASE_LOW_OFFSET 0x10 +#define CXL_HDM_DECODER0_BASE_HIGH_OFFSET 0x14 +#define CXL_HDM_DECODER0_SIZE_LOW_OFFSET 0x18 +#define CXL_HDM_DECODER0_SIZE_HIGH_OFFSET 0x1c +#define CXL_HDM_DECODER0_CTRL_OFFSET 0x20 + /* CXL 2.0 8.2.8.1 Device Capabilities Array Register */ #define CXLDEV_CAP_ARRAY_OFFSET 0x0 #define CXLDEV_CAP_ARRAY_CAP_ID 0 @@ -34,62 +59,91 @@ #define CXLDEV_MBOX_BG_CMD_STATUS_OFFSET 0x18 #define CXLDEV_MBOX_PAYLOAD_OFFSET 0x20 -/* CXL 2.0 8.2.8.5.1.1 Memory Device Status Register */ -#define CXLMDEV_STATUS_OFFSET 0x0 -#define CXLMDEV_DEV_FATAL BIT(0) -#define CXLMDEV_FW_HALT BIT(1) -#define CXLMDEV_STATUS_MEDIA_STATUS_MASK GENMASK(3, 2) -#define CXLMDEV_MS_NOT_READY 0 -#define CXLMDEV_MS_READY 1 -#define CXLMDEV_MS_ERROR 2 -#define CXLMDEV_MS_DISABLED 3 -#define CXLMDEV_READY(status) \ - (FIELD_GET(CXLMDEV_STATUS_MEDIA_STATUS_MASK, status) == \ - CXLMDEV_MS_READY) -#define CXLMDEV_MBOX_IF_READY BIT(4) -#define CXLMDEV_RESET_NEEDED_MASK GENMASK(7, 5) -#define CXLMDEV_RESET_NEEDED_NOT 0 -#define CXLMDEV_RESET_NEEDED_COLD 1 -#define CXLMDEV_RESET_NEEDED_WARM 2 -#define CXLMDEV_RESET_NEEDED_HOT 3 -#define CXLMDEV_RESET_NEEDED_CXL 4 -#define CXLMDEV_RESET_NEEDED(status) \ - (FIELD_GET(CXLMDEV_RESET_NEEDED_MASK, status) != \ - CXLMDEV_RESET_NEEDED_NOT) - -struct cxl_memdev; -/** - * struct cxl_mem - A CXL memory device - * @pdev: The PCI device associated with this CXL device. - * @regs: IO mappings to the device's MMIO - * @status_regs: CXL 2.0 8.2.8.3 Device Status Registers - * @mbox_regs: CXL 2.0 8.2.8.4 Mailbox Registers - * @memdev_regs: CXL 2.0 8.2.8.5 Memory Device Registers - * @payload_size: Size of space for payload - * (CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register) - * @mbox_mutex: Mutex to synchronize mailbox access. - * @firmware_version: Firmware version for the memory device. - * @enabled_commands: Hardware commands found enabled in CEL. - * @pmem_range: Persistent memory capacity information. - * @ram_range: Volatile memory capacity information. +#define CXL_COMPONENT_REGS() \ + void __iomem *hdm_decoder + +#define CXL_DEVICE_REGS() \ + void __iomem *status; \ + void __iomem *mbox; \ + void __iomem *memdev + +/* See note for 'struct cxl_regs' for the rationale of this organization */ +/* + * CXL_COMPONENT_REGS - Common set of CXL Component register block base pointers + * @hdm_decoder: CXL 2.0 8.2.5.12 CXL HDM Decoder Capability Structure + */ +struct cxl_component_regs { + CXL_COMPONENT_REGS(); +}; + +/* See note for 'struct cxl_regs' for the rationale of this organization */ +/* + * CXL_DEVICE_REGS - Common set of CXL Device register block base pointers + * @status: CXL 2.0 8.2.8.3 Device Status Registers + * @mbox: CXL 2.0 8.2.8.4 Mailbox Registers + * @memdev: CXL 2.0 8.2.8.5 Memory Device Registers + */ +struct cxl_device_regs { + CXL_DEVICE_REGS(); +}; + +/* + * Note, the anonymous union organization allows for per + * register-block-type helper routines, without requiring block-type + * agnostic code to include the prefix. */ -struct cxl_mem { - struct pci_dev *pdev; - void __iomem *regs; - struct cxl_memdev *cxlmd; - - void __iomem *status_regs; - void __iomem *mbox_regs; - void __iomem *memdev_regs; - - size_t payload_size; - struct mutex mbox_mutex; /* Protects device mailbox and firmware */ - char firmware_version[0x10]; - unsigned long *enabled_cmds; - - struct range pmem_range; - struct range ram_range; +struct cxl_regs { + union { + struct { + CXL_COMPONENT_REGS(); + }; + struct cxl_component_regs component; + }; + union { + struct { + CXL_DEVICE_REGS(); + }; + struct cxl_device_regs device_regs; + }; +}; + +struct cxl_reg_map { + bool valid; + unsigned long offset; + unsigned long size; +}; + +struct cxl_component_reg_map { + struct cxl_reg_map hdm_decoder; }; +struct cxl_device_reg_map { + struct cxl_reg_map status; + struct cxl_reg_map mbox; + struct cxl_reg_map memdev; +}; + +struct cxl_register_map { + struct list_head list; + u64 block_offset; + u8 reg_type; + u8 barno; + union { + struct cxl_component_reg_map component_map; + struct cxl_device_reg_map device_map; + }; +}; + +void cxl_probe_component_regs(struct device *dev, void __iomem *base, + struct cxl_component_reg_map *map); +void cxl_probe_device_regs(struct device *dev, void __iomem *base, + struct cxl_device_reg_map *map); +int cxl_map_component_regs(struct pci_dev *pdev, + struct cxl_component_regs *regs, + struct cxl_register_map *map); +int cxl_map_device_regs(struct pci_dev *pdev, + struct cxl_device_regs *regs, + struct cxl_register_map *map); + extern struct bus_type cxl_bus_type; #endif /* __CXL_H__ */ diff --git a/drivers/cxl/mem.h b/drivers/cxl/mem.h new file mode 100644 index 000000000000..13868ff7cadf --- /dev/null +++ b/drivers/cxl/mem.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2020-2021 Intel Corporation. */ +#ifndef __CXL_MEM_H__ +#define __CXL_MEM_H__ + +/* CXL 2.0 8.2.8.5.1.1 Memory Device Status Register */ +#define CXLMDEV_STATUS_OFFSET 0x0 +#define CXLMDEV_DEV_FATAL BIT(0) +#define CXLMDEV_FW_HALT BIT(1) +#define CXLMDEV_STATUS_MEDIA_STATUS_MASK GENMASK(3, 2) +#define CXLMDEV_MS_NOT_READY 0 +#define CXLMDEV_MS_READY 1 +#define CXLMDEV_MS_ERROR 2 +#define CXLMDEV_MS_DISABLED 3 +#define CXLMDEV_READY(status) \ + (FIELD_GET(CXLMDEV_STATUS_MEDIA_STATUS_MASK, status) == \ + CXLMDEV_MS_READY) +#define CXLMDEV_MBOX_IF_READY BIT(4) +#define CXLMDEV_RESET_NEEDED_MASK GENMASK(7, 5) +#define CXLMDEV_RESET_NEEDED_NOT 0 +#define CXLMDEV_RESET_NEEDED_COLD 1 +#define CXLMDEV_RESET_NEEDED_WARM 2 +#define CXLMDEV_RESET_NEEDED_HOT 3 +#define CXLMDEV_RESET_NEEDED_CXL 4 +#define CXLMDEV_RESET_NEEDED(status) \ + (FIELD_GET(CXLMDEV_RESET_NEEDED_MASK, status) != \ + CXLMDEV_RESET_NEEDED_NOT) + +/* + * An entire PCI topology full of devices should be enough for any + * config + */ +#define CXL_MEM_MAX_DEVS 65536 + +/** + * struct cxl_memdev - CXL bus object representing a Type-3 Memory Device + * @dev: driver core device object + * @cdev: char dev core object for ioctl operations + * @cxlm: pointer to the parent device driver data + * @id: id number of this memdev instance. + */ +struct cxl_memdev { + struct device dev; + struct cdev cdev; + struct cxl_mem *cxlm; + int id; +}; + +/** + * struct cxl_mem - A CXL memory device + * @pdev: The PCI device associated with this CXL device. + * @cxlmd: Logical memory device chardev / interface + * @regs: Parsed register blocks + * @payload_size: Size of space for payload + * (CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register) + * @lsa_size: Size of Label Storage Area + * (CXL 2.0 8.2.9.5.1.1 Identify Memory Device) + * @mbox_mutex: Mutex to synchronize mailbox access. + * @firmware_version: Firmware version for the memory device. + * @enabled_cmds: Hardware commands found enabled in CEL. + * @pmem_range: Persistent memory capacity information. + * @ram_range: Volatile memory capacity information. + */ +struct cxl_mem { + struct pci_dev *pdev; + struct cxl_memdev *cxlmd; + + struct cxl_regs regs; + + size_t payload_size; + size_t lsa_size; + struct mutex mbox_mutex; /* Protects device mailbox and firmware */ + char firmware_version[0x10]; + unsigned long *enabled_cmds; + + struct range pmem_range; + struct range ram_range; +}; +#endif /* __CXL_MEM_H__ */ diff --git a/drivers/cxl/mem.c b/drivers/cxl/pci.c index 2acc6173da36..5a1705b52278 100644 --- a/drivers/cxl/mem.c +++ b/drivers/cxl/pci.c @@ -6,6 +6,7 @@ #include <linux/module.h> #include <linux/sizes.h> #include <linux/mutex.h> +#include <linux/list.h> #include <linux/cdev.h> #include <linux/idr.h> #include <linux/pci.h> @@ -13,12 +14,14 @@ #include <linux/io-64-nonatomic-lo-hi.h> #include "pci.h" #include "cxl.h" +#include "mem.h" /** - * DOC: cxl mem + * DOC: cxl pci * - * This implements a CXL memory device ("type-3") as it is defined by the - * Compute Express Link specification. + * This implements the PCI exclusive functionality for a CXL device as it is + * defined by the Compute Express Link specification. CXL devices may surface + * certain functionality even if it isn't CXL enabled. * * The driver has several responsibilities, mainly: * - Create the memX device and register on the CXL bus. @@ -26,18 +29,10 @@ * - Probe the device attributes to establish sysfs interface. * - Provide an IOCTL interface to userspace to communicate with the device for * things like firmware update. - * - Support management of interleave sets. - * - Handle and manage error conditions. */ -/* - * An entire PCI topology full of devices should be enough for any - * config - */ -#define CXL_MEM_MAX_DEVS 65536 - #define cxl_doorbell_busy(cxlm) \ - (readl((cxlm)->mbox_regs + CXLDEV_MBOX_CTRL_OFFSET) & \ + (readl((cxlm)->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET) & \ CXLDEV_MBOX_CTRL_DOORBELL) /* CXL 2.0 - 8.2.8.4 */ @@ -92,20 +87,6 @@ struct mbox_cmd { #define CXL_MBOX_SUCCESS 0 }; -/** - * struct cxl_memdev - CXL bus object representing a Type-3 Memory Device - * @dev: driver core device object - * @cdev: char dev core object for ioctl operations - * @cxlm: pointer to the parent device driver data - * @id: id number of this memdev instance. - */ -struct cxl_memdev { - struct device dev; - struct cdev cdev; - struct cxl_mem *cxlm; - int id; -}; - static int cxl_mem_major; static DEFINE_IDA(cxl_memdev_ida); static DECLARE_RWSEM(cxl_memdev_rwsem); @@ -292,7 +273,7 @@ static void cxl_mem_mbox_timeout(struct cxl_mem *cxlm, static int __cxl_mem_mbox_send_cmd(struct cxl_mem *cxlm, struct mbox_cmd *mbox_cmd) { - void __iomem *payload = cxlm->mbox_regs + CXLDEV_MBOX_PAYLOAD_OFFSET; + void __iomem *payload = cxlm->regs.mbox + CXLDEV_MBOX_PAYLOAD_OFFSET; u64 cmd_reg, status_reg; size_t out_len; int rc; @@ -335,12 +316,12 @@ static int __cxl_mem_mbox_send_cmd(struct cxl_mem *cxlm, } /* #2, #3 */ - writeq(cmd_reg, cxlm->mbox_regs + CXLDEV_MBOX_CMD_OFFSET); + writeq(cmd_reg, cxlm->regs.mbox + CXLDEV_MBOX_CMD_OFFSET); /* #4 */ dev_dbg(&cxlm->pdev->dev, "Sending command\n"); writel(CXLDEV_MBOX_CTRL_DOORBELL, - cxlm->mbox_regs + CXLDEV_MBOX_CTRL_OFFSET); + cxlm->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET); /* #5 */ rc = cxl_mem_wait_for_doorbell(cxlm); @@ -350,7 +331,7 @@ static int __cxl_mem_mbox_send_cmd(struct cxl_mem *cxlm, } /* #6 */ - status_reg = readq(cxlm->mbox_regs + CXLDEV_MBOX_STATUS_OFFSET); + status_reg = readq(cxlm->regs.mbox + CXLDEV_MBOX_STATUS_OFFSET); mbox_cmd->return_code = FIELD_GET(CXLDEV_MBOX_STATUS_RET_CODE_MASK, status_reg); @@ -360,7 +341,7 @@ static int __cxl_mem_mbox_send_cmd(struct cxl_mem *cxlm, } /* #7 */ - cmd_reg = readq(cxlm->mbox_regs + CXLDEV_MBOX_CMD_OFFSET); + cmd_reg = readq(cxlm->regs.mbox + CXLDEV_MBOX_CMD_OFFSET); out_len = FIELD_GET(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, cmd_reg); /* #8 */ @@ -421,7 +402,7 @@ static int cxl_mem_mbox_get(struct cxl_mem *cxlm) goto out; } - md_status = readq(cxlm->memdev_regs + CXLMDEV_STATUS_OFFSET); + md_status = readq(cxlm->regs.memdev + CXLMDEV_STATUS_OFFSET); if (!(md_status & CXLMDEV_MBOX_IF_READY && CXLMDEV_READY(md_status))) { dev_err(dev, "mbox: reported doorbell ready, but not mbox ready\n"); rc = -EBUSY; @@ -890,75 +871,9 @@ static int cxl_mem_mbox_send_cmd(struct cxl_mem *cxlm, u16 opcode, return 0; } -/** - * cxl_mem_setup_regs() - Setup necessary MMIO. - * @cxlm: The CXL memory device to communicate with. - * - * Return: 0 if all necessary registers mapped. - * - * A memory device is required by spec to implement a certain set of MMIO - * regions. The purpose of this function is to enumerate and map those - * registers. - */ -static int cxl_mem_setup_regs(struct cxl_mem *cxlm) -{ - struct device *dev = &cxlm->pdev->dev; - int cap, cap_count; - u64 cap_array; - - cap_array = readq(cxlm->regs + CXLDEV_CAP_ARRAY_OFFSET); - if (FIELD_GET(CXLDEV_CAP_ARRAY_ID_MASK, cap_array) != - CXLDEV_CAP_ARRAY_CAP_ID) - return -ENODEV; - - cap_count = FIELD_GET(CXLDEV_CAP_ARRAY_COUNT_MASK, cap_array); - - for (cap = 1; cap <= cap_count; cap++) { - void __iomem *register_block; - u32 offset; - u16 cap_id; - - cap_id = FIELD_GET(CXLDEV_CAP_HDR_CAP_ID_MASK, - readl(cxlm->regs + cap * 0x10)); - offset = readl(cxlm->regs + cap * 0x10 + 0x4); - register_block = cxlm->regs + offset; - - switch (cap_id) { - case CXLDEV_CAP_CAP_ID_DEVICE_STATUS: - dev_dbg(dev, "found Status capability (0x%x)\n", offset); - cxlm->status_regs = register_block; - break; - case CXLDEV_CAP_CAP_ID_PRIMARY_MAILBOX: - dev_dbg(dev, "found Mailbox capability (0x%x)\n", offset); - cxlm->mbox_regs = register_block; - break; - case CXLDEV_CAP_CAP_ID_SECONDARY_MAILBOX: - dev_dbg(dev, "found Secondary Mailbox capability (0x%x)\n", offset); - break; - case CXLDEV_CAP_CAP_ID_MEMDEV: - dev_dbg(dev, "found Memory Device capability (0x%x)\n", offset); - cxlm->memdev_regs = register_block; - break; - default: - dev_dbg(dev, "Unknown cap ID: %d (0x%x)\n", cap_id, offset); - break; - } - } - - if (!cxlm->status_regs || !cxlm->mbox_regs || !cxlm->memdev_regs) { - dev_err(dev, "registers not found: %s%s%s\n", - !cxlm->status_regs ? "status " : "", - !cxlm->mbox_regs ? "mbox " : "", - !cxlm->memdev_regs ? "memdev" : ""); - return -ENXIO; - } - - return 0; -} - static int cxl_mem_setup_mailbox(struct cxl_mem *cxlm) { - const int cap = readl(cxlm->mbox_regs + CXLDEV_MBOX_CAPS_OFFSET); + const int cap = readl(cxlm->regs.mbox + CXLDEV_MBOX_CAPS_OFFSET); cxlm->payload_size = 1 << FIELD_GET(CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK, cap); @@ -983,55 +898,62 @@ static int cxl_mem_setup_mailbox(struct cxl_mem *cxlm) return 0; } -static struct cxl_mem *cxl_mem_create(struct pci_dev *pdev, u32 reg_lo, - u32 reg_hi) +static struct cxl_mem *cxl_mem_create(struct pci_dev *pdev) { struct device *dev = &pdev->dev; struct cxl_mem *cxlm; - void __iomem *regs; - u64 offset; - u8 bar; - int rc; - cxlm = devm_kzalloc(&pdev->dev, sizeof(*cxlm), GFP_KERNEL); + cxlm = devm_kzalloc(dev, sizeof(*cxlm), GFP_KERNEL); if (!cxlm) { dev_err(dev, "No memory available\n"); - return NULL; - } - - offset = ((u64)reg_hi << 32) | (reg_lo & CXL_REGLOC_ADDR_MASK); - bar = FIELD_GET(CXL_REGLOC_BIR_MASK, reg_lo); - - /* Basic sanity check that BAR is big enough */ - if (pci_resource_len(pdev, bar) < offset) { - dev_err(dev, "BAR%d: %pr: too small (offset: %#llx)\n", bar, - &pdev->resource[bar], (unsigned long long)offset); - return NULL; - } - - rc = pcim_iomap_regions(pdev, BIT(bar), pci_name(pdev)); - if (rc) { - dev_err(dev, "failed to map registers\n"); - return NULL; + return ERR_PTR(-ENOMEM); } - regs = pcim_iomap_table(pdev)[bar]; mutex_init(&cxlm->mbox_mutex); cxlm->pdev = pdev; - cxlm->regs = regs + offset; cxlm->enabled_cmds = devm_kmalloc_array(dev, BITS_TO_LONGS(cxl_cmd_count), sizeof(unsigned long), GFP_KERNEL | __GFP_ZERO); if (!cxlm->enabled_cmds) { dev_err(dev, "No memory available for bitmap\n"); - return NULL; + return ERR_PTR(-ENOMEM); } - dev_dbg(dev, "Mapped CXL Memory Device resource\n"); return cxlm; } +static void __iomem *cxl_mem_map_regblock(struct cxl_mem *cxlm, + u8 bar, u64 offset) +{ + struct pci_dev *pdev = cxlm->pdev; + struct device *dev = &pdev->dev; + void __iomem *addr; + + /* Basic sanity check that BAR is big enough */ + if (pci_resource_len(pdev, bar) < offset) { + dev_err(dev, "BAR%d: %pr: too small (offset: %#llx)\n", bar, + &pdev->resource[bar], (unsigned long long)offset); + return IOMEM_ERR_PTR(-ENXIO); + } + + addr = pci_iomap(pdev, bar, 0); + if (!addr) { + dev_err(dev, "failed to map registers\n"); + return addr; + } + + dev_dbg(dev, "Mapped CXL Memory Device resource bar %u @ %#llx\n", + bar, offset); + + return addr; +} + +static void cxl_mem_unmap_regblock(struct cxl_mem *cxlm, void __iomem *base) +{ + pci_iounmap(cxlm->pdev, base); +} + static int cxl_mem_dvsec(struct pci_dev *pdev, int dvsec) { int pos; @@ -1055,6 +977,171 @@ static int cxl_mem_dvsec(struct pci_dev *pdev, int dvsec) return 0; } +static int cxl_probe_regs(struct cxl_mem *cxlm, void __iomem *base, + struct cxl_register_map *map) +{ + struct pci_dev *pdev = cxlm->pdev; + struct device *dev = &pdev->dev; + struct cxl_component_reg_map *comp_map; + struct cxl_device_reg_map *dev_map; + + switch (map->reg_type) { + case CXL_REGLOC_RBI_COMPONENT: + comp_map = &map->component_map; + cxl_probe_component_regs(dev, base, comp_map); + if (!comp_map->hdm_decoder.valid) { + dev_err(dev, "HDM decoder registers not found\n"); + return -ENXIO; + } + + dev_dbg(dev, "Set up component registers\n"); + break; + case CXL_REGLOC_RBI_MEMDEV: + dev_map = &map->device_map; + cxl_probe_device_regs(dev, base, dev_map); + if (!dev_map->status.valid || !dev_map->mbox.valid || + !dev_map->memdev.valid) { + dev_err(dev, "registers not found: %s%s%s\n", + !dev_map->status.valid ? "status " : "", + !dev_map->mbox.valid ? "status " : "", + !dev_map->memdev.valid ? "status " : ""); + return -ENXIO; + } + + dev_dbg(dev, "Probing device registers...\n"); + break; + default: + break; + } + + return 0; +} + +static int cxl_map_regs(struct cxl_mem *cxlm, struct cxl_register_map *map) +{ + struct pci_dev *pdev = cxlm->pdev; + struct device *dev = &pdev->dev; + + switch (map->reg_type) { + case CXL_REGLOC_RBI_COMPONENT: + cxl_map_component_regs(pdev, &cxlm->regs.component, map); + dev_dbg(dev, "Mapping component registers...\n"); + break; + case CXL_REGLOC_RBI_MEMDEV: + cxl_map_device_regs(pdev, &cxlm->regs.device_regs, map); + dev_dbg(dev, "Probing device registers...\n"); + break; + default: + break; + } + + return 0; +} + +static void cxl_decode_register_block(u32 reg_lo, u32 reg_hi, + u8 *bar, u64 *offset, u8 *reg_type) +{ + *offset = ((u64)reg_hi << 32) | (reg_lo & CXL_REGLOC_ADDR_MASK); + *bar = FIELD_GET(CXL_REGLOC_BIR_MASK, reg_lo); + *reg_type = FIELD_GET(CXL_REGLOC_RBI_MASK, reg_lo); +} + +/** + * cxl_mem_setup_regs() - Setup necessary MMIO. + * @cxlm: The CXL memory device to communicate with. + * + * Return: 0 if all necessary registers mapped. + * + * A memory device is required by spec to implement a certain set of MMIO + * regions. The purpose of this function is to enumerate and map those + * registers. + */ +static int cxl_mem_setup_regs(struct cxl_mem *cxlm) +{ + struct pci_dev *pdev = cxlm->pdev; + struct device *dev = &pdev->dev; + u32 regloc_size, regblocks; + void __iomem *base; + int regloc, i; + struct cxl_register_map *map, *n; + LIST_HEAD(register_maps); + int ret = 0; + + regloc = cxl_mem_dvsec(pdev, PCI_DVSEC_ID_CXL_REGLOC_OFFSET); + if (!regloc) { + dev_err(dev, "register location dvsec not found\n"); + return -ENXIO; + } + + if (pci_request_mem_regions(pdev, pci_name(pdev))) + return -ENODEV; + + /* Get the size of the Register Locator DVSEC */ + pci_read_config_dword(pdev, regloc + PCI_DVSEC_HEADER1, ®loc_size); + regloc_size = FIELD_GET(PCI_DVSEC_HEADER1_LENGTH_MASK, regloc_size); + + regloc += PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET; + regblocks = (regloc_size - PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET) / 8; + + for (i = 0; i < regblocks; i++, regloc += 8) { + u32 reg_lo, reg_hi; + u8 reg_type; + u64 offset; + u8 bar; + + map = kzalloc(sizeof(*map), GFP_KERNEL); + if (!map) { + ret = -ENOMEM; + goto free_maps; + } + + list_add(&map->list, ®ister_maps); + + pci_read_config_dword(pdev, regloc, ®_lo); + pci_read_config_dword(pdev, regloc + 4, ®_hi); + + cxl_decode_register_block(reg_lo, reg_hi, &bar, &offset, + ®_type); + + dev_dbg(dev, "Found register block in bar %u @ 0x%llx of type %u\n", + bar, offset, reg_type); + + base = cxl_mem_map_regblock(cxlm, bar, offset); + if (!base) { + ret = -ENOMEM; + goto free_maps; + } + + map->barno = bar; + map->block_offset = offset; + map->reg_type = reg_type; + + ret = cxl_probe_regs(cxlm, base + offset, map); + + /* Always unmap the regblock regardless of probe success */ + cxl_mem_unmap_regblock(cxlm, base); + + if (ret) + goto free_maps; + } + + pci_release_mem_regions(pdev); + + list_for_each_entry(map, ®ister_maps, list) { + ret = cxl_map_regs(cxlm, map); + if (ret) + goto free_maps; + } + +free_maps: + list_for_each_entry_safe(map, n, ®ister_maps, list) { + list_del(&map->list); + kfree(map); + } + + return ret; +} + static struct cxl_memdev *to_cxl_memdev(struct device *dev) { return container_of(dev, struct cxl_memdev, dev); @@ -1094,6 +1181,16 @@ static ssize_t payload_max_show(struct device *dev, } static DEVICE_ATTR_RO(payload_max); +static ssize_t label_storage_size_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + struct cxl_mem *cxlm = cxlmd->cxlm; + + return sysfs_emit(buf, "%zu\n", cxlm->lsa_size); +} +static DEVICE_ATTR_RO(label_storage_size); + static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -1123,6 +1220,7 @@ static struct device_attribute dev_attr_pmem_size = static struct attribute *cxl_memdev_attributes[] = { &dev_attr_firmware_version.attr, &dev_attr_payload_max.attr, + &dev_attr_label_storage_size.attr, NULL, }; @@ -1455,6 +1553,7 @@ static int cxl_mem_identify(struct cxl_mem *cxlm) cxlm->pmem_range.end = le64_to_cpu(id.persistent_capacity) * SZ_256M - 1; + cxlm->lsa_size = le32_to_cpu(id.lsa_size); memcpy(cxlm->firmware_version, id.fw_revision, sizeof(id.fw_revision)); return 0; @@ -1462,46 +1561,16 @@ static int cxl_mem_identify(struct cxl_mem *cxlm) static int cxl_mem_probe(struct pci_dev *pdev, const struct pci_device_id *id) { - struct device *dev = &pdev->dev; - struct cxl_mem *cxlm = NULL; - u32 regloc_size, regblocks; - int rc, regloc, i; + struct cxl_mem *cxlm; + int rc; rc = pcim_enable_device(pdev); if (rc) return rc; - regloc = cxl_mem_dvsec(pdev, PCI_DVSEC_ID_CXL_REGLOC_OFFSET); - if (!regloc) { - dev_err(dev, "register location dvsec not found\n"); - return -ENXIO; - } - - /* Get the size of the Register Locator DVSEC */ - pci_read_config_dword(pdev, regloc + PCI_DVSEC_HEADER1, ®loc_size); - regloc_size = FIELD_GET(PCI_DVSEC_HEADER1_LENGTH_MASK, regloc_size); - - regloc += PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET; - regblocks = (regloc_size - PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET) / 8; - - for (i = 0; i < regblocks; i++, regloc += 8) { - u32 reg_lo, reg_hi; - u8 reg_type; - - /* "register low and high" contain other bits */ - pci_read_config_dword(pdev, regloc, ®_lo); - pci_read_config_dword(pdev, regloc + 4, ®_hi); - - reg_type = FIELD_GET(CXL_REGLOC_RBI_MASK, reg_lo); - - if (reg_type == CXL_REGLOC_RBI_MEMDEV) { - cxlm = cxl_mem_create(pdev, reg_lo, reg_hi); - break; - } - } - - if (!cxlm) - return -ENODEV; + cxlm = cxl_mem_create(pdev); + if (IS_ERR(cxlm)) + return PTR_ERR(cxlm); rc = cxl_mem_setup_regs(cxlm); if (rc) @@ -1544,6 +1613,10 @@ static __init int cxl_mem_init(void) dev_t devt; int rc; + /* Double check the anonymous union trickery in struct cxl_regs */ + BUILD_BUG_ON(offsetof(struct cxl_regs, memdev) != + offsetof(struct cxl_regs, device_regs.memdev)); + rc = alloc_chrdev_region(&devt, 0, CXL_MEM_MAX_DEVS, "cxl"); if (rc) return rc; |