diff options
25 files changed, 246 insertions, 4081 deletions
diff --git a/Documentation/ABI/stable/sysfs-driver-dma-idxd b/Documentation/ABI/stable/sysfs-driver-dma-idxd index 8e2c2c405db2..3becc9a82bdf 100644 --- a/Documentation/ABI/stable/sysfs-driver-dma-idxd +++ b/Documentation/ABI/stable/sysfs-driver-dma-idxd @@ -22,6 +22,7 @@ Date: Oct 25, 2019 KernelVersion: 5.6.0 Contact: [email protected] Description: The largest number of work descriptors in a batch. + It's not visible when the device does not support batch. What: /sys/bus/dsa/devices/dsa<m>/max_work_queues_size Date: Oct 25, 2019 @@ -49,6 +50,8 @@ Description: The total number of read buffers supported by this device. The read buffers represent resources within the DSA implementation, and these resources are allocated by engines to support operations. See DSA spec v1.2 9.2.4 Total Read Buffers. + It's not visible when the device does not support Read Buffer + allocation control. What: /sys/bus/dsa/devices/dsa<m>/max_transfer_size Date: Oct 25, 2019 @@ -122,6 +125,8 @@ Contact: [email protected] Description: The maximum number of read buffers that may be in use at one time by operations that access low bandwidth memory in the device. See DSA spec v1.2 9.2.8 GENCFG on Global Read Buffer Limit. + It's not visible when the device does not support Read Buffer + allocation control. What: /sys/bus/dsa/devices/dsa<m>/cmd_status Date: Aug 28, 2020 @@ -205,6 +210,7 @@ KernelVersion: 5.10.0 Contact: [email protected] Description: The max batch size for this workqueue. Cannot exceed device max batch size. Configurable parameter. + It's not visible when the device does not support batch. What: /sys/bus/dsa/devices/wq<m>.<n>/ats_disable Date: Nov 13, 2020 @@ -250,6 +256,8 @@ KernelVersion: 5.17.0 Contact: [email protected] Description: Enable the use of global read buffer limit for the group. See DSA spec v1.2 9.2.18 GRPCFG Use Global Read Buffer Limit. + It's not visible when the device does not support Read Buffer + allocation control. What: /sys/bus/dsa/devices/group<m>.<n>/read_buffers_allowed Date: Dec 10, 2021 @@ -258,6 +266,8 @@ Contact: [email protected] Description: Indicates max number of read buffers that may be in use at one time by all engines in the group. See DSA spec v1.2 9.2.18 GRPCFG Read Buffers Allowed. + It's not visible when the device does not support Read Buffer + allocation control. What: /sys/bus/dsa/devices/group<m>.<n>/read_buffers_reserved Date: Dec 10, 2021 @@ -266,6 +276,8 @@ Contact: [email protected] Description: Indicates the number of Read Buffers reserved for the use of engines in the group. See DSA spec v1.2 9.2.18 GRPCFG Read Buffers Reserved. + It's not visible when the device does not support Read Buffer + allocation control. What: /sys/bus/dsa/devices/group<m>.<n>/desc_progress_limit Date: Sept 14, 2022 diff --git a/Documentation/devicetree/bindings/dma/ingenic,dma.yaml b/Documentation/devicetree/bindings/dma/ingenic,dma.yaml index 3b0b3b919af8..e42b8ce948db 100644 --- a/Documentation/devicetree/bindings/dma/ingenic,dma.yaml +++ b/Documentation/devicetree/bindings/dma/ingenic,dma.yaml @@ -18,6 +18,7 @@ properties: - enum: - ingenic,jz4740-dma - ingenic,jz4725b-dma + - ingenic,jz4755-dma - ingenic,jz4760-dma - ingenic,jz4760-bdma - ingenic,jz4760-mdma diff --git a/Documentation/devicetree/bindings/dma/qcom,gpi.yaml b/Documentation/devicetree/bindings/dma/qcom,gpi.yaml index eabf8a76d3a0..232895fa1d8d 100644 --- a/Documentation/devicetree/bindings/dma/qcom,gpi.yaml +++ b/Documentation/devicetree/bindings/dma/qcom,gpi.yaml @@ -18,14 +18,23 @@ allOf: properties: compatible: - enum: - - qcom,sc7280-gpi-dma - - qcom,sdm845-gpi-dma - - qcom,sm6350-gpi-dma - - qcom,sm8150-gpi-dma - - qcom,sm8250-gpi-dma - - qcom,sm8350-gpi-dma - - qcom,sm8450-gpi-dma + oneOf: + - enum: + - qcom,sdm845-gpi-dma + - qcom,sm6350-gpi-dma + - items: + - enum: + - qcom,sc7280-gpi-dma + - qcom,sm6115-gpi-dma + - qcom,sm8350-gpi-dma + - qcom,sm8450-gpi-dma + - const: qcom,sm6350-gpi-dma + - items: + - enum: + - qcom,sdm670-gpi-dma + - qcom,sm8150-gpi-dma + - qcom,sm8250-gpi-dma + - const: qcom,sdm845-gpi-dma reg: maxItems: 1 diff --git a/Documentation/driver-api/driver-model/devres.rst b/Documentation/driver-api/driver-model/devres.rst index 687adb58048e..99a989ebee87 100644 --- a/Documentation/driver-api/driver-model/devres.rst +++ b/Documentation/driver-api/driver-model/devres.rst @@ -437,6 +437,7 @@ SERDEV SLAVE DMA ENGINE devm_acpi_dma_controller_register() + devm_acpi_dma_controller_free() SPI devm_spi_alloc_master() diff --git a/MAINTAINERS b/MAINTAINERS index cf0f18502372..cd1264d24db8 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -10332,11 +10332,6 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu.git F: drivers/iommu/intel/ F: include/linux/intel-svm.h -INTEL IOP-ADMA DMA DRIVER -R: Dan Williams <[email protected]> -S: Odd fixes -F: drivers/dma/iop-adma.c - INTEL IPU3 CSI-2 CIO2 DRIVER M: Yong Zhi <[email protected]> M: Sakari Ailus <[email protected]> diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 7524b62a8870..ea81d825575f 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -357,14 +357,6 @@ config INTEL_IOATDMA If unsure, say N. -config INTEL_IOP_ADMA - tristate "Intel IOP32x ADMA support" - depends on ARCH_IOP32X || COMPILE_TEST - select DMA_ENGINE - select ASYNC_TX_ENABLE_CHANNEL_SWITCH - help - Enable support for the Intel(R) IOP Series RAID engines. - config K3_DMA tristate "Hisilicon K3 DMA support" depends on ARCH_HI3xxx || ARCH_HISI || COMPILE_TEST @@ -617,18 +609,6 @@ config SPRD_DMA help Enable support for the on-chip DMA controller on Spreadtrum platform. -config S3C24XX_DMAC - bool "Samsung S3C24XX DMA support" - depends on ARCH_S3C24XX || COMPILE_TEST - select DMA_ENGINE - select DMA_VIRTUAL_CHANNELS - help - Support for the Samsung S3C24XX DMA controller driver. The - DMA controller is having multiple DMA channels which can be - configured for different peripherals like audio, UART, SPI. - The DMA controller can transfer data from memory to peripheral, - periphal to memory, periphal to periphal and memory to memory. - config TXX9_DMAC tristate "Toshiba TXx9 SoC DMA support" depends on MACH_TX49XX diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index 10f7d4241001..a4fd1ce29510 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -44,7 +44,6 @@ obj-$(CONFIG_IMX_SDMA) += imx-sdma.o obj-$(CONFIG_INTEL_IDMA64) += idma64.o obj-$(CONFIG_INTEL_IOATDMA) += ioat/ obj-y += idxd/ -obj-$(CONFIG_INTEL_IOP_ADMA) += iop-adma.o obj-$(CONFIG_K3_DMA) += k3dma.o obj-$(CONFIG_LPC18XX_DMAMUX) += lpc18xx-dmamux.o obj-$(CONFIG_MILBEAUT_HDMAC) += milbeaut-hdmac.o @@ -71,7 +70,6 @@ obj-$(CONFIG_STM32_DMA) += stm32-dma.o obj-$(CONFIG_STM32_DMAMUX) += stm32-dmamux.o obj-$(CONFIG_STM32_MDMA) += stm32-mdma.o obj-$(CONFIG_SPRD_DMA) += sprd-dma.o -obj-$(CONFIG_S3C24XX_DMAC) += s3c24xx-dma.o obj-$(CONFIG_TXX9_DMAC) += txx9dmac.o obj-$(CONFIG_TEGRA186_GPC_DMA) += tegra186-gpc-dma.o obj-$(CONFIG_TEGRA20_APB_DMA) += tegra20-apb-dma.o diff --git a/drivers/dma/apple-admac.c b/drivers/dma/apple-admac.c index a2cc520225d3..90f28bda29c8 100644 --- a/drivers/dma/apple-admac.c +++ b/drivers/dma/apple-admac.c @@ -21,6 +21,12 @@ #define NCHANNELS_MAX 64 #define IRQ_NOUTPUTS 4 +/* + * For allocation purposes we split the cache + * memory into blocks of fixed size (given in bytes). + */ +#define SRAM_BLOCK 2048 + #define RING_WRITE_SLOT GENMASK(1, 0) #define RING_READ_SLOT GENMASK(5, 4) #define RING_FULL BIT(9) @@ -36,6 +42,9 @@ #define REG_TX_STOP 0x0004 #define REG_RX_START 0x0008 #define REG_RX_STOP 0x000c +#define REG_IMPRINT 0x0090 +#define REG_TX_SRAM_SIZE 0x0094 +#define REG_RX_SRAM_SIZE 0x0098 #define REG_CHAN_CTL(ch) (0x8000 + (ch) * 0x200) #define REG_CHAN_CTL_RST_RINGS BIT(0) @@ -53,7 +62,9 @@ #define BUS_WIDTH_FRAME_2_WORDS 0x10 #define BUS_WIDTH_FRAME_4_WORDS 0x20 -#define CHAN_BUFSIZE 0x8000 +#define REG_CHAN_SRAM_CARVEOUT(ch) (0x8050 + (ch) * 0x200) +#define CHAN_SRAM_CARVEOUT_SIZE GENMASK(31, 16) +#define CHAN_SRAM_CARVEOUT_BASE GENMASK(15, 0) #define REG_CHAN_FIFOCTL(ch) (0x8054 + (ch) * 0x200) #define CHAN_FIFOCTL_LIMIT GENMASK(31, 16) @@ -76,6 +87,8 @@ struct admac_chan { struct dma_chan chan; struct tasklet_struct tasklet; + u32 carveout; + spinlock_t lock; struct admac_tx *current_tx; int nperiod_acks; @@ -92,12 +105,24 @@ struct admac_chan { struct list_head to_free; }; +struct admac_sram { + u32 size; + /* + * SRAM_CARVEOUT has 16-bit fields, so the SRAM cannot be larger than + * 64K and a 32-bit bitfield over 2K blocks covers it. + */ + u32 allocated; +}; + struct admac_data { struct dma_device dma; struct device *dev; __iomem void *base; struct reset_control *rstc; + struct mutex cache_alloc_lock; + struct admac_sram txcache, rxcache; + int irq; int irq_index; int nchannels; @@ -118,6 +143,60 @@ struct admac_tx { struct list_head node; }; +static int admac_alloc_sram_carveout(struct admac_data *ad, + enum dma_transfer_direction dir, + u32 *out) +{ + struct admac_sram *sram; + int i, ret = 0, nblocks; + + if (dir == DMA_MEM_TO_DEV) + sram = &ad->txcache; + else + sram = &ad->rxcache; + + mutex_lock(&ad->cache_alloc_lock); + + nblocks = sram->size / SRAM_BLOCK; + for (i = 0; i < nblocks; i++) + if (!(sram->allocated & BIT(i))) + break; + + if (i < nblocks) { + *out = FIELD_PREP(CHAN_SRAM_CARVEOUT_BASE, i * SRAM_BLOCK) | + FIELD_PREP(CHAN_SRAM_CARVEOUT_SIZE, SRAM_BLOCK); + sram->allocated |= BIT(i); + } else { + ret = -EBUSY; + } + + mutex_unlock(&ad->cache_alloc_lock); + + return ret; +} + +static void admac_free_sram_carveout(struct admac_data *ad, + enum dma_transfer_direction dir, + u32 carveout) +{ + struct admac_sram *sram; + u32 base = FIELD_GET(CHAN_SRAM_CARVEOUT_BASE, carveout); + int i; + + if (dir == DMA_MEM_TO_DEV) + sram = &ad->txcache; + else + sram = &ad->rxcache; + + if (WARN_ON(base >= sram->size)) + return; + + mutex_lock(&ad->cache_alloc_lock); + i = base / SRAM_BLOCK; + sram->allocated &= ~BIT(i); + mutex_unlock(&ad->cache_alloc_lock); +} + static void admac_modify(struct admac_data *ad, int reg, u32 mask, u32 val) { void __iomem *addr = ad->base + reg; @@ -466,15 +545,28 @@ static void admac_synchronize(struct dma_chan *chan) static int admac_alloc_chan_resources(struct dma_chan *chan) { struct admac_chan *adchan = to_admac_chan(chan); + struct admac_data *ad = adchan->host; + int ret; dma_cookie_init(&adchan->chan); + ret = admac_alloc_sram_carveout(ad, admac_chan_direction(adchan->no), + &adchan->carveout); + if (ret < 0) + return ret; + + writel_relaxed(adchan->carveout, + ad->base + REG_CHAN_SRAM_CARVEOUT(adchan->no)); return 0; } static void admac_free_chan_resources(struct dma_chan *chan) { + struct admac_chan *adchan = to_admac_chan(chan); + admac_terminate_all(chan); admac_synchronize(chan); + admac_free_sram_carveout(adchan->host, admac_chan_direction(adchan->no), + adchan->carveout); } static struct dma_chan *admac_dma_of_xlate(struct of_phandle_args *dma_spec, @@ -712,6 +804,7 @@ static int admac_probe(struct platform_device *pdev) platform_set_drvdata(pdev, ad); ad->dev = &pdev->dev; ad->nchannels = nchannels; + mutex_init(&ad->cache_alloc_lock); /* * The controller has 4 IRQ outputs. Try them all until @@ -801,6 +894,13 @@ static int admac_probe(struct platform_device *pdev) goto free_irq; } + ad->txcache.size = readl_relaxed(ad->base + REG_TX_SRAM_SIZE); + ad->rxcache.size = readl_relaxed(ad->base + REG_RX_SRAM_SIZE); + + dev_info(&pdev->dev, "Audio DMA Controller\n"); + dev_info(&pdev->dev, "imprint %x TX cache %u RX cache %u\n", + readl_relaxed(ad->base + REG_IMPRINT), ad->txcache.size, ad->rxcache.size); + return 0; free_irq: diff --git a/drivers/dma/dma-jz4780.c b/drivers/dma/dma-jz4780.c index 2a483802d9ee..9c1a6e9a9c03 100644 --- a/drivers/dma/dma-jz4780.c +++ b/drivers/dma/dma-jz4780.c @@ -1038,6 +1038,13 @@ static const struct jz4780_dma_soc_data jz4725b_dma_soc_data = { JZ_SOC_DATA_BREAK_LINKS, }; +static const struct jz4780_dma_soc_data jz4755_dma_soc_data = { + .nb_channels = 4, + .transfer_ord_max = 5, + .flags = JZ_SOC_DATA_PER_CHAN_PM | JZ_SOC_DATA_NO_DCKES_DCKEC | + JZ_SOC_DATA_BREAK_LINKS, +}; + static const struct jz4780_dma_soc_data jz4760_dma_soc_data = { .nb_channels = 5, .transfer_ord_max = 6, @@ -1101,6 +1108,7 @@ static const struct jz4780_dma_soc_data x1830_dma_soc_data = { static const struct of_device_id jz4780_dma_dt_match[] = { { .compatible = "ingenic,jz4740-dma", .data = &jz4740_dma_soc_data }, { .compatible = "ingenic,jz4725b-dma", .data = &jz4725b_dma_soc_data }, + { .compatible = "ingenic,jz4755-dma", .data = &jz4755_dma_soc_data }, { .compatible = "ingenic,jz4760-dma", .data = &jz4760_dma_soc_data }, { .compatible = "ingenic,jz4760-mdma", .data = &jz4760_mdma_soc_data }, { .compatible = "ingenic,jz4760-bdma", .data = &jz4760_bdma_soc_data }, diff --git a/drivers/dma/idma64.c b/drivers/dma/idma64.c index f4c07ad3be15..c33087c5cd02 100644 --- a/drivers/dma/idma64.c +++ b/drivers/dma/idma64.c @@ -600,7 +600,7 @@ static int idma64_probe(struct idma64_chip *chip) return 0; } -static int idma64_remove(struct idma64_chip *chip) +static void idma64_remove(struct idma64_chip *chip) { struct idma64 *idma64 = chip->idma64; unsigned short i; @@ -618,8 +618,6 @@ static int idma64_remove(struct idma64_chip *chip) tasklet_kill(&idma64c->vchan.task); } - - return 0; } /* ---------------------------------------------------------------------- */ @@ -664,7 +662,9 @@ static int idma64_platform_remove(struct platform_device *pdev) { struct idma64_chip *chip = platform_get_drvdata(pdev); - return idma64_remove(chip); + idma64_remove(chip); + + return 0; } static int __maybe_unused idma64_pm_suspend(struct device *dev) diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index 7269bd54554f..3229dfc78650 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -528,6 +528,22 @@ static bool idxd_group_attr_progress_limit_invisible(struct attribute *attr, !idxd->hw.group_cap.progress_limit; } +static bool idxd_group_attr_read_buffers_invisible(struct attribute *attr, + struct idxd_device *idxd) +{ + /* + * Intel IAA does not support Read Buffer allocation control, + * make these attributes invisible. + */ + return (attr == &dev_attr_group_use_token_limit.attr || + attr == &dev_attr_group_use_read_buffer_limit.attr || + attr == &dev_attr_group_tokens_allowed.attr || + attr == &dev_attr_group_read_buffers_allowed.attr || + attr == &dev_attr_group_tokens_reserved.attr || + attr == &dev_attr_group_read_buffers_reserved.attr) && + idxd->data->type == IDXD_TYPE_IAX; +} + static umode_t idxd_group_attr_visible(struct kobject *kobj, struct attribute *attr, int n) { @@ -538,6 +554,9 @@ static umode_t idxd_group_attr_visible(struct kobject *kobj, if (idxd_group_attr_progress_limit_invisible(attr, idxd)) return 0; + if (idxd_group_attr_read_buffers_invisible(attr, idxd)) + return 0; + return attr->mode; } @@ -1233,6 +1252,14 @@ static bool idxd_wq_attr_op_config_invisible(struct attribute *attr, !idxd->hw.wq_cap.op_config; } +static bool idxd_wq_attr_max_batch_size_invisible(struct attribute *attr, + struct idxd_device *idxd) +{ + /* Intel IAA does not support batch processing, make it invisible */ + return attr == &dev_attr_wq_max_batch_size.attr && + idxd->data->type == IDXD_TYPE_IAX; +} + static umode_t idxd_wq_attr_visible(struct kobject *kobj, struct attribute *attr, int n) { @@ -1243,6 +1270,9 @@ static umode_t idxd_wq_attr_visible(struct kobject *kobj, if (idxd_wq_attr_op_config_invisible(attr, idxd)) return 0; + if (idxd_wq_attr_max_batch_size_invisible(attr, idxd)) + return 0; + return attr->mode; } @@ -1533,6 +1563,43 @@ static ssize_t cmd_status_store(struct device *dev, struct device_attribute *att } static DEVICE_ATTR_RW(cmd_status); +static bool idxd_device_attr_max_batch_size_invisible(struct attribute *attr, + struct idxd_device *idxd) +{ + /* Intel IAA does not support batch processing, make it invisible */ + return attr == &dev_attr_max_batch_size.attr && + idxd->data->type == IDXD_TYPE_IAX; +} + +static bool idxd_device_attr_read_buffers_invisible(struct attribute *attr, + struct idxd_device *idxd) +{ + /* + * Intel IAA does not support Read Buffer allocation control, + * make these attributes invisible. + */ + return (attr == &dev_attr_max_tokens.attr || + attr == &dev_attr_max_read_buffers.attr || + attr == &dev_attr_token_limit.attr || + attr == &dev_attr_read_buffer_limit.attr) && + idxd->data->type == IDXD_TYPE_IAX; +} + +static umode_t idxd_device_attr_visible(struct kobject *kobj, + struct attribute *attr, int n) +{ + struct device *dev = container_of(kobj, struct device, kobj); + struct idxd_device *idxd = confdev_to_idxd(dev); + + if (idxd_device_attr_max_batch_size_invisible(attr, idxd)) + return 0; + + if (idxd_device_attr_read_buffers_invisible(attr, idxd)) + return 0; + + return attr->mode; +} + static struct attribute *idxd_device_attributes[] = { &dev_attr_version.attr, &dev_attr_max_groups.attr, @@ -1560,6 +1627,7 @@ static struct attribute *idxd_device_attributes[] = { static const struct attribute_group idxd_device_attribute_group = { .attrs = idxd_device_attributes, + .is_visible = idxd_device_attr_visible, }; static const struct attribute_group *idxd_attribute_groups[] = { diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c index e2070df6cad2..79d244011093 100644 --- a/drivers/dma/ioat/dma.c +++ b/drivers/dma/ioat/dma.c @@ -33,7 +33,7 @@ MODULE_PARM_DESC(completion_timeout, static int idle_timeout = 2000; module_param(idle_timeout, int, 0644); MODULE_PARM_DESC(idle_timeout, - "set ioat idel timeout [msec] (default 2000 [msec])"); + "set ioat idle timeout [msec] (default 2000 [msec])"); #define IDLE_TIMEOUT msecs_to_jiffies(idle_timeout) #define COMPLETION_TIMEOUT msecs_to_jiffies(completion_timeout) diff --git a/drivers/dma/iop-adma.c b/drivers/dma/iop-adma.c deleted file mode 100644 index 310b899d581f..000000000000 --- a/drivers/dma/iop-adma.c +++ /dev/null @@ -1,1554 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * offload engine driver for the Intel Xscale series of i/o processors - * Copyright © 2006, Intel Corporation. - */ - -/* - * This driver supports the asynchrounous DMA copy and RAID engines available - * on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x) - */ - -#include <linux/init.h> -#include <linux/module.h> -#include <linux/delay.h> -#include <linux/dma-mapping.h> -#include <linux/spinlock.h> -#include <linux/interrupt.h> -#include <linux/platform_device.h> -#include <linux/prefetch.h> -#include <linux/memory.h> -#include <linux/ioport.h> -#include <linux/raid/pq.h> -#include <linux/slab.h> - -#include "iop-adma.h" -#include "dmaengine.h" - -#define to_iop_adma_chan(chan) container_of(chan, struct iop_adma_chan, common) -#define to_iop_adma_device(dev) \ - container_of(dev, struct iop_adma_device, common) -#define tx_to_iop_adma_slot(tx) \ - container_of(tx, struct iop_adma_desc_slot, async_tx) - -/** - * iop_adma_free_slots - flags descriptor slots for reuse - * @slot: Slot to free - * Caller must hold &iop_chan->lock while calling this function - */ -static void iop_adma_free_slots(struct iop_adma_desc_slot *slot) -{ - int stride = slot->slots_per_op; - - while (stride--) { - slot->slots_per_op = 0; - slot = list_entry(slot->slot_node.next, - struct iop_adma_desc_slot, - slot_node); - } -} - -static dma_cookie_t -iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc, - struct iop_adma_chan *iop_chan, dma_cookie_t cookie) -{ - struct dma_async_tx_descriptor *tx = &desc->async_tx; - - BUG_ON(tx->cookie < 0); - if (tx->cookie > 0) { - cookie = tx->cookie; - tx->cookie = 0; - - /* call the callback (must not sleep or submit new - * operations to this channel) - */ - dmaengine_desc_get_callback_invoke(tx, NULL); - - dma_descriptor_unmap(tx); - if (desc->group_head) - desc->group_head = NULL; - } - - /* run dependent operations */ - dma_run_dependencies(tx); - - return cookie; -} - -static int -iop_adma_clean_slot(struct iop_adma_desc_slot *desc, - struct iop_adma_chan *iop_chan) -{ - /* the client is allowed to attach dependent operations - * until 'ack' is set - */ - if (!async_tx_test_ack(&desc->async_tx)) - return 0; - - /* leave the last descriptor in the chain - * so we can append to it - */ - if (desc->chain_node.next == &iop_chan->chain) - return 1; - - dev_dbg(iop_chan->device->common.dev, - "\tfree slot: %d slots_per_op: %d\n", - desc->idx, desc->slots_per_op); - - list_del(&desc->chain_node); - iop_adma_free_slots(desc); - - return 0; -} - -static void __iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan) -{ - struct iop_adma_desc_slot *iter, *_iter, *grp_start = NULL; - dma_cookie_t cookie = 0; - u32 current_desc = iop_chan_get_current_descriptor(iop_chan); - int busy = iop_chan_is_busy(iop_chan); - int seen_current = 0, slot_cnt = 0, slots_per_op = 0; - - dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); - /* free completed slots from the chain starting with - * the oldest descriptor - */ - list_for_each_entry_safe(iter, _iter, &iop_chan->chain, - chain_node) { - pr_debug("\tcookie: %d slot: %d busy: %d " - "this_desc: %pad next_desc: %#llx ack: %d\n", - iter->async_tx.cookie, iter->idx, busy, - &iter->async_tx.phys, (u64)iop_desc_get_next_desc(iter), - async_tx_test_ack(&iter->async_tx)); - prefetch(_iter); - prefetch(&_iter->async_tx); - - /* do not advance past the current descriptor loaded into the - * hardware channel, subsequent descriptors are either in - * process or have not been submitted - */ - if (seen_current) - break; - - /* stop the search if we reach the current descriptor and the - * channel is busy, or if it appears that the current descriptor - * needs to be re-read (i.e. has been appended to) - */ - if (iter->async_tx.phys == current_desc) { - BUG_ON(seen_current++); - if (busy || iop_desc_get_next_desc(iter)) - break; - } - - /* detect the start of a group transaction */ - if (!slot_cnt && !slots_per_op) { - slot_cnt = iter->slot_cnt; - slots_per_op = iter->slots_per_op; - if (slot_cnt <= slots_per_op) { - slot_cnt = 0; - slots_per_op = 0; - } - } - - if (slot_cnt) { - pr_debug("\tgroup++\n"); - if (!grp_start) - grp_start = iter; - slot_cnt -= slots_per_op; - } - - /* all the members of a group are complete */ - if (slots_per_op != 0 && slot_cnt == 0) { - struct iop_adma_desc_slot *grp_iter, *_grp_iter; - int end_of_chain = 0; - pr_debug("\tgroup end\n"); - - /* collect the total results */ - if (grp_start->xor_check_result) { - u32 zero_sum_result = 0; - slot_cnt = grp_start->slot_cnt; - grp_iter = grp_start; - - list_for_each_entry_from(grp_iter, - &iop_chan->chain, chain_node) { - zero_sum_result |= - iop_desc_get_zero_result(grp_iter); - pr_debug("\titer%d result: %d\n", - grp_iter->idx, zero_sum_result); - slot_cnt -= slots_per_op; - if (slot_cnt == 0) - break; - } - pr_debug("\tgrp_start->xor_check_result: %p\n", - grp_start->xor_check_result); - *grp_start->xor_check_result = zero_sum_result; - } - - /* clean up the group */ - slot_cnt = grp_start->slot_cnt; - grp_iter = grp_start; - list_for_each_entry_safe_from(grp_iter, _grp_iter, - &iop_chan->chain, chain_node) { - cookie = iop_adma_run_tx_complete_actions( - grp_iter, iop_chan, cookie); - - slot_cnt -= slots_per_op; - end_of_chain = iop_adma_clean_slot(grp_iter, - iop_chan); - - if (slot_cnt == 0 || end_of_chain) - break; - } - - /* the group should be complete at this point */ - BUG_ON(slot_cnt); - - slots_per_op = 0; - grp_start = NULL; - if (end_of_chain) - break; - else - continue; - } else if (slots_per_op) /* wait for group completion */ - continue; - - /* write back zero sum results (single descriptor case) */ - if (iter->xor_check_result && iter->async_tx.cookie) - *iter->xor_check_result = - iop_desc_get_zero_result(iter); - - cookie = iop_adma_run_tx_complete_actions( - iter, iop_chan, cookie); - - if (iop_adma_clean_slot(iter, iop_chan)) - break; - } - - if (cookie > 0) { - iop_chan->common.completed_cookie = cookie; - pr_debug("\tcompleted cookie %d\n", cookie); - } -} - -static void -iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan) -{ - spin_lock_bh(&iop_chan->lock); - __iop_adma_slot_cleanup(iop_chan); - spin_unlock_bh(&iop_chan->lock); -} - -static void iop_adma_tasklet(struct tasklet_struct *t) -{ - struct iop_adma_chan *iop_chan = from_tasklet(iop_chan, t, - irq_tasklet); - - /* lockdep will flag depedency submissions as potentially - * recursive locking, this is not the case as a dependency - * submission will never recurse a channels submit routine. - * There are checks in async_tx.c to prevent this. - */ - spin_lock_nested(&iop_chan->lock, SINGLE_DEPTH_NESTING); - __iop_adma_slot_cleanup(iop_chan); - spin_unlock(&iop_chan->lock); -} - -static struct iop_adma_desc_slot * -iop_adma_alloc_slots(struct iop_adma_chan *iop_chan, int num_slots, - int slots_per_op) -{ - struct iop_adma_desc_slot *iter, *_iter, *alloc_start = NULL; - LIST_HEAD(chain); - int slots_found, retry = 0; - - /* start search from the last allocated descrtiptor - * if a contiguous allocation can not be found start searching - * from the beginning of the list - */ -retry: - slots_found = 0; - if (retry == 0) - iter = iop_chan->last_used; - else - iter = list_entry(&iop_chan->all_slots, - struct iop_adma_desc_slot, - slot_node); - - list_for_each_entry_safe_continue( - iter, _iter, &iop_chan->all_slots, slot_node) { - prefetch(_iter); - prefetch(&_iter->async_tx); - if (iter->slots_per_op) { - /* give up after finding the first busy slot - * on the second pass through the list - */ - if (retry) - break; - - slots_found = 0; - continue; - } - - /* start the allocation if the slot is correctly aligned */ - if (!slots_found++) { - if (iop_desc_is_aligned(iter, slots_per_op)) - alloc_start = iter; - else { - slots_found = 0; - continue; - } - } - - if (slots_found == num_slots) { - struct iop_adma_desc_slot *alloc_tail = NULL; - struct iop_adma_desc_slot *last_used = NULL; - iter = alloc_start; - while (num_slots) { - int i; - dev_dbg(iop_chan->device->common.dev, - "allocated slot: %d " - "(desc %p phys: %#llx) slots_per_op %d\n", - iter->idx, iter->hw_desc, - (u64)iter->async_tx.phys, slots_per_op); - - /* pre-ack all but the last descriptor */ - if (num_slots != slots_per_op) - async_tx_ack(&iter->async_tx); - - list_add_tail(&iter->chain_node, &chain); - alloc_tail = iter; - iter->async_tx.cookie = 0; - iter->slot_cnt = num_slots; - iter->xor_check_result = NULL; - for (i = 0; i < slots_per_op; i++) { - iter->slots_per_op = slots_per_op - i; - last_used = iter; - iter = list_entry(iter->slot_node.next, - struct iop_adma_desc_slot, - slot_node); - } - num_slots -= slots_per_op; - } - alloc_tail->group_head = alloc_start; - alloc_tail->async_tx.cookie = -EBUSY; - list_splice(&chain, &alloc_tail->tx_list); - iop_chan->last_used = last_used; - iop_desc_clear_next_desc(alloc_start); - iop_desc_clear_next_desc(alloc_tail); - return alloc_tail; - } - } - if (!retry++) - goto retry; - - /* perform direct reclaim if the allocation fails */ - __iop_adma_slot_cleanup(iop_chan); - - return NULL; -} - -static void iop_adma_check_threshold(struct iop_adma_chan *iop_chan) -{ - dev_dbg(iop_chan->device->common.dev, "pending: %d\n", - iop_chan->pending); - - if (iop_chan->pending >= IOP_ADMA_THRESHOLD) { - iop_chan->pending = 0; - iop_chan_append(iop_chan); - } -} - -static dma_cookie_t -iop_adma_tx_submit(struct dma_async_tx_descriptor *tx) -{ - struct iop_adma_desc_slot *sw_desc = tx_to_iop_adma_slot(tx); - struct iop_adma_chan *iop_chan = to_iop_adma_chan(tx->chan); - struct iop_adma_desc_slot *grp_start, *old_chain_tail; - int slot_cnt; - dma_cookie_t cookie; - dma_addr_t next_dma; - - grp_start = sw_desc->group_head; - slot_cnt = grp_start->slot_cnt; - - spin_lock_bh(&iop_chan->lock); - cookie = dma_cookie_assign(tx); - - old_chain_tail = list_entry(iop_chan->chain.prev, - struct iop_adma_desc_slot, chain_node); - list_splice_init(&sw_desc->tx_list, - &old_chain_tail->chain_node); - - /* fix up the hardware chain */ - next_dma = grp_start->async_tx.phys; - iop_desc_set_next_desc(old_chain_tail, next_dma); - BUG_ON(iop_desc_get_next_desc(old_chain_tail) != next_dma); /* flush */ - - /* check for pre-chained descriptors */ - iop_paranoia(iop_desc_get_next_desc(sw_desc)); - - /* increment the pending count by the number of slots - * memcpy operations have a 1:1 (slot:operation) relation - * other operations are heavier and will pop the threshold - * more often. - */ - iop_chan->pending += slot_cnt; - iop_adma_check_threshold(iop_chan); - spin_unlock_bh(&iop_chan->lock); - - dev_dbg(iop_chan->device->common.dev, "%s cookie: %d slot: %d\n", - __func__, sw_desc->async_tx.cookie, sw_desc->idx); - - return cookie; -} - -static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan); -static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan); - -/** - * iop_adma_alloc_chan_resources - returns the number of allocated descriptors - * @chan: allocate descriptor resources for this channel - * - * Note: We keep the slots for 1 operation on iop_chan->chain at all times. To - * avoid deadlock, via async_xor, num_descs_in_pool must at a minimum be - * greater than 2x the number slots needed to satisfy a device->max_xor - * request. - * */ -static int iop_adma_alloc_chan_resources(struct dma_chan *chan) -{ - char *hw_desc; - dma_addr_t dma_desc; - int idx; - struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); - struct iop_adma_desc_slot *slot = NULL; - int init = iop_chan->slots_allocated ? 0 : 1; - struct iop_adma_platform_data *plat_data = - dev_get_platdata(&iop_chan->device->pdev->dev); - int num_descs_in_pool = plat_data->pool_size/IOP_ADMA_SLOT_SIZE; - - /* Allocate descriptor slots */ - do { - idx = iop_chan->slots_allocated; - if (idx == num_descs_in_pool) - break; - - slot = kzalloc(sizeof(*slot), GFP_KERNEL); - if (!slot) { - printk(KERN_INFO "IOP ADMA Channel only initialized" - " %d descriptor slots", idx); - break; - } - hw_desc = (char *) iop_chan->device->dma_desc_pool_virt; - slot->hw_desc = (void *) &hw_desc[idx * IOP_ADMA_SLOT_SIZE]; - - dma_async_tx_descriptor_init(&slot->async_tx, chan); - slot->async_tx.tx_submit = iop_adma_tx_submit; - INIT_LIST_HEAD(&slot->tx_list); - INIT_LIST_HEAD(&slot->chain_node); - INIT_LIST_HEAD(&slot->slot_node); - dma_desc = iop_chan->device->dma_desc_pool; - slot->async_tx.phys = dma_desc + idx * IOP_ADMA_SLOT_SIZE; - slot->idx = idx; - - spin_lock_bh(&iop_chan->lock); - iop_chan->slots_allocated++; - list_add_tail(&slot->slot_node, &iop_chan->all_slots); - spin_unlock_bh(&iop_chan->lock); - } while (iop_chan->slots_allocated < num_descs_in_pool); - - if (idx && !iop_chan->last_used) - iop_chan->last_used = list_entry(iop_chan->all_slots.next, - struct iop_adma_desc_slot, - slot_node); - - dev_dbg(iop_chan->device->common.dev, - "allocated %d descriptor slots last_used: %p\n", - iop_chan->slots_allocated, iop_chan->last_used); - - /* initialize the channel and the chain with a null operation */ - if (init) { - if (dma_has_cap(DMA_MEMCPY, - iop_chan->device->common.cap_mask)) - iop_chan_start_null_memcpy(iop_chan); - else if (dma_has_cap(DMA_XOR, - iop_chan->device->common.cap_mask)) - iop_chan_start_null_xor(iop_chan); - else - BUG(); - } - - return (idx > 0) ? idx : -ENOMEM; -} - -static struct dma_async_tx_descriptor * -iop_adma_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags) -{ - struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); - struct iop_adma_desc_slot *sw_desc, *grp_start; - int slot_cnt, slots_per_op; - - dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); - - spin_lock_bh(&iop_chan->lock); - slot_cnt = iop_chan_interrupt_slot_count(&slots_per_op, iop_chan); - sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); - if (sw_desc) { - grp_start = sw_desc->group_head; - iop_desc_init_interrupt(grp_start, iop_chan); - sw_desc->async_tx.flags = flags; - } - spin_unlock_bh(&iop_chan->lock); - - return sw_desc ? &sw_desc->async_tx : NULL; -} - -static struct dma_async_tx_descriptor * -iop_adma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest, - dma_addr_t dma_src, size_t len, unsigned long flags) -{ - struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); - struct iop_adma_desc_slot *sw_desc, *grp_start; - int slot_cnt, slots_per_op; - - if (unlikely(!len)) - return NULL; - BUG_ON(len > IOP_ADMA_MAX_BYTE_COUNT); - - dev_dbg(iop_chan->device->common.dev, "%s len: %zu\n", - __func__, len); - - spin_lock_bh(&iop_chan->lock); - slot_cnt = iop_chan_memcpy_slot_count(len, &slots_per_op); - sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); - if (sw_desc) { - grp_start = sw_desc->group_head; - iop_desc_init_memcpy(grp_start, flags); - iop_desc_set_byte_count(grp_start, iop_chan, len); - iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest); - iop_desc_set_memcpy_src_addr(grp_start, dma_src); - sw_desc->async_tx.flags = flags; - } - spin_unlock_bh(&iop_chan->lock); - - return sw_desc ? &sw_desc->async_tx : NULL; -} - -static struct dma_async_tx_descriptor * -iop_adma_prep_dma_xor(struct dma_chan *chan, dma_addr_t dma_dest, - dma_addr_t *dma_src, unsigned int src_cnt, size_t len, - unsigned long flags) -{ - struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); - struct iop_adma_desc_slot *sw_desc, *grp_start; - int slot_cnt, slots_per_op; - - if (unlikely(!len)) - return NULL; - BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT); - - dev_dbg(iop_chan->device->common.dev, - "%s src_cnt: %d len: %zu flags: %lx\n", - __func__, src_cnt, len, flags); - - spin_lock_bh(&iop_chan->lock); - slot_cnt = iop_chan_xor_slot_count(len, src_cnt, &slots_per_op); - sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); - if (sw_desc) { - grp_start = sw_desc->group_head; - iop_desc_init_xor(grp_start, src_cnt, flags); - iop_desc_set_byte_count(grp_start, iop_chan, len); - iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest); - sw_desc->async_tx.flags = flags; - while (src_cnt--) - iop_desc_set_xor_src_addr(grp_start, src_cnt, - dma_src[src_cnt]); - } - spin_unlock_bh(&iop_chan->lock); - - return sw_desc ? &sw_desc->async_tx : NULL; -} - -static struct dma_async_tx_descriptor * -iop_adma_prep_dma_xor_val(struct dma_chan *chan, dma_addr_t *dma_src, - unsigned int src_cnt, size_t len, u32 *result, - unsigned long flags) -{ - struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); - struct iop_adma_desc_slot *sw_desc, *grp_start; - int slot_cnt, slots_per_op; - - if (unlikely(!len)) - return NULL; - - dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %zu\n", - __func__, src_cnt, len); - - spin_lock_bh(&iop_chan->lock); - slot_cnt = iop_chan_zero_sum_slot_count(len, src_cnt, &slots_per_op); - sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); - if (sw_desc) { - grp_start = sw_desc->group_head; - iop_desc_init_zero_sum(grp_start, src_cnt, flags); - iop_desc_set_zero_sum_byte_count(grp_start, len); - grp_start->xor_check_result = result; - pr_debug("\t%s: grp_start->xor_check_result: %p\n", - __func__, grp_start->xor_check_result); - sw_desc->async_tx.flags = flags; - while (src_cnt--) - iop_desc_set_zero_sum_src_addr(grp_start, src_cnt, - dma_src[src_cnt]); - } - spin_unlock_bh(&iop_chan->lock); - - return sw_desc ? &sw_desc->async_tx : NULL; -} - -static struct dma_async_tx_descriptor * -iop_adma_prep_dma_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, - unsigned int src_cnt, const unsigned char *scf, size_t len, - unsigned long flags) -{ - struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); - struct iop_adma_desc_slot *sw_desc, *g; - int slot_cnt, slots_per_op; - int continue_srcs; - - if (unlikely(!len)) - return NULL; - BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT); - - dev_dbg(iop_chan->device->common.dev, - "%s src_cnt: %d len: %zu flags: %lx\n", - __func__, src_cnt, len, flags); - - if (dmaf_p_disabled_continue(flags)) - continue_srcs = 1+src_cnt; - else if (dmaf_continue(flags)) - continue_srcs = 3+src_cnt; - else - continue_srcs = 0+src_cnt; - - spin_lock_bh(&iop_chan->lock); - slot_cnt = iop_chan_pq_slot_count(len, continue_srcs, &slots_per_op); - sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); - if (sw_desc) { - int i; - - g = sw_desc->group_head; - iop_desc_set_byte_count(g, iop_chan, len); - - /* even if P is disabled its destination address (bits - * [3:0]) must match Q. It is ok if P points to an - * invalid address, it won't be written. - */ - if (flags & DMA_PREP_PQ_DISABLE_P) - dst[0] = dst[1] & 0x7; - - iop_desc_set_pq_addr(g, dst); - sw_desc->async_tx.flags = flags; - for (i = 0; i < src_cnt; i++) - iop_desc_set_pq_src_addr(g, i, src[i], scf[i]); - - /* if we are continuing a previous operation factor in - * the old p and q values, see the comment for dma_maxpq - * in include/linux/dmaengine.h - */ - if (dmaf_p_disabled_continue(flags)) - iop_desc_set_pq_src_addr(g, i++, dst[1], 1); - else if (dmaf_continue(flags)) { - iop_desc_set_pq_src_addr(g, i++, dst[0], 0); - iop_desc_set_pq_src_addr(g, i++, dst[1], 1); - iop_desc_set_pq_src_addr(g, i++, dst[1], 0); - } - iop_desc_init_pq(g, i, flags); - } - spin_unlock_bh(&iop_chan->lock); - - return sw_desc ? &sw_desc->async_tx : NULL; -} - -static struct dma_async_tx_descriptor * -iop_adma_prep_dma_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src, - unsigned int src_cnt, const unsigned char *scf, - size_t len, enum sum_check_flags *pqres, - unsigned long flags) -{ - struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); - struct iop_adma_desc_slot *sw_desc, *g; - int slot_cnt, slots_per_op; - - if (unlikely(!len)) - return NULL; - BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT); - - dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %zu\n", - __func__, src_cnt, len); - - spin_lock_bh(&iop_chan->lock); - slot_cnt = iop_chan_pq_zero_sum_slot_count(len, src_cnt + 2, &slots_per_op); - sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); - if (sw_desc) { - /* for validate operations p and q are tagged onto the - * end of the source list - */ - int pq_idx = src_cnt; - - g = sw_desc->group_head; - iop_desc_init_pq_zero_sum(g, src_cnt+2, flags); - iop_desc_set_pq_zero_sum_byte_count(g, len); - g->pq_check_result = pqres; - pr_debug("\t%s: g->pq_check_result: %p\n", - __func__, g->pq_check_result); - sw_desc->async_tx.flags = flags; - while (src_cnt--) - iop_desc_set_pq_zero_sum_src_addr(g, src_cnt, - src[src_cnt], - scf[src_cnt]); - iop_desc_set_pq_zero_sum_addr(g, pq_idx, src); - } - spin_unlock_bh(&iop_chan->lock); - - return sw_desc ? &sw_desc->async_tx : NULL; -} - -static void iop_adma_free_chan_resources(struct dma_chan *chan) -{ - struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); - struct iop_adma_desc_slot *iter, *_iter; - int in_use_descs = 0; - - iop_adma_slot_cleanup(iop_chan); - - spin_lock_bh(&iop_chan->lock); - list_for_each_entry_safe(iter, _iter, &iop_chan->chain, - chain_node) { - in_use_descs++; - list_del(&iter->chain_node); - } - list_for_each_entry_safe_reverse( - iter, _iter, &iop_chan->all_slots, slot_node) { - list_del(&iter->slot_node); - kfree(iter); - iop_chan->slots_allocated--; - } - iop_chan->last_used = NULL; - - dev_dbg(iop_chan->device->common.dev, "%s slots_allocated %d\n", - __func__, iop_chan->slots_allocated); - spin_unlock_bh(&iop_chan->lock); - - /* one is ok since we left it on there on purpose */ - if (in_use_descs > 1) - printk(KERN_ERR "IOP: Freeing %d in use descriptors!\n", - in_use_descs - 1); -} - -/** - * iop_adma_status - poll the status of an ADMA transaction - * @chan: ADMA channel handle - * @cookie: ADMA transaction identifier - * @txstate: a holder for the current state of the channel or NULL - */ -static enum dma_status iop_adma_status(struct dma_chan *chan, - dma_cookie_t cookie, - struct dma_tx_state *txstate) -{ - struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); - int ret; - - ret = dma_cookie_status(chan, cookie, txstate); - if (ret == DMA_COMPLETE) - return ret; - - iop_adma_slot_cleanup(iop_chan); - - return dma_cookie_status(chan, cookie, txstate); -} - -static irqreturn_t iop_adma_eot_handler(int irq, void *data) -{ - struct iop_adma_chan *chan = data; - - dev_dbg(chan->device->common.dev, "%s\n", __func__); - - tasklet_schedule(&chan->irq_tasklet); - - iop_adma_device_clear_eot_status(chan); - - return IRQ_HANDLED; -} - -static irqreturn_t iop_adma_eoc_handler(int irq, void *data) -{ - struct iop_adma_chan *chan = data; - - dev_dbg(chan->device->common.dev, "%s\n", __func__); - - tasklet_schedule(&chan->irq_tasklet); - - iop_adma_device_clear_eoc_status(chan); - - return IRQ_HANDLED; -} - -static irqreturn_t iop_adma_err_handler(int irq, void *data) -{ - struct iop_adma_chan *chan = data; - unsigned long status = iop_chan_get_status(chan); - - dev_err(chan->device->common.dev, - "error ( %s%s%s%s%s%s%s)\n", - iop_is_err_int_parity(status, chan) ? "int_parity " : "", - iop_is_err_mcu_abort(status, chan) ? "mcu_abort " : "", - iop_is_err_int_tabort(status, chan) ? "int_tabort " : "", - iop_is_err_int_mabort(status, chan) ? "int_mabort " : "", - iop_is_err_pci_tabort(status, chan) ? "pci_tabort " : "", - iop_is_err_pci_mabort(status, chan) ? "pci_mabort " : "", - iop_is_err_split_tx(status, chan) ? "split_tx " : ""); - - iop_adma_device_clear_err_status(chan); - - BUG(); - - return IRQ_HANDLED; -} - -static void iop_adma_issue_pending(struct dma_chan *chan) -{ - struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); - - if (iop_chan->pending) { - iop_chan->pending = 0; - iop_chan_append(iop_chan); - } -} - -/* - * Perform a transaction to verify the HW works. - */ -#define IOP_ADMA_TEST_SIZE 2000 - -static int iop_adma_memcpy_self_test(struct iop_adma_device *device) -{ - int i; - void *src, *dest; - dma_addr_t src_dma, dest_dma; - struct dma_chan *dma_chan; - dma_cookie_t cookie; - struct dma_async_tx_descriptor *tx; - int err = 0; - struct iop_adma_chan *iop_chan; - - dev_dbg(device->common.dev, "%s\n", __func__); - - src = kmalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL); - if (!src) - return -ENOMEM; - dest = kzalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL); - if (!dest) { - kfree(src); - return -ENOMEM; - } - - /* Fill in src buffer */ - for (i = 0; i < IOP_ADMA_TEST_SIZE; i++) - ((u8 *) src)[i] = (u8)i; - - /* Start copy, using first DMA channel */ - dma_chan = container_of(device->common.channels.next, - struct dma_chan, - device_node); - if (iop_adma_alloc_chan_resources(dma_chan) < 1) { - err = -ENODEV; - goto out; - } - - dest_dma = dma_map_single(dma_chan->device->dev, dest, - IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE); - src_dma = dma_map_single(dma_chan->device->dev, src, - IOP_ADMA_TEST_SIZE, DMA_TO_DEVICE); - tx = iop_adma_prep_dma_memcpy(dma_chan, dest_dma, src_dma, - IOP_ADMA_TEST_SIZE, - DMA_PREP_INTERRUPT | DMA_CTRL_ACK); - - cookie = iop_adma_tx_submit(tx); - iop_adma_issue_pending(dma_chan); - msleep(1); - - if (iop_adma_status(dma_chan, cookie, NULL) != - DMA_COMPLETE) { - dev_err(dma_chan->device->dev, - "Self-test copy timed out, disabling\n"); - err = -ENODEV; - goto free_resources; - } - - iop_chan = to_iop_adma_chan(dma_chan); - dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma, - IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE); - if (memcmp(src, dest, IOP_ADMA_TEST_SIZE)) { - dev_err(dma_chan->device->dev, - "Self-test copy failed compare, disabling\n"); - err = -ENODEV; - goto free_resources; - } - -free_resources: - iop_adma_free_chan_resources(dma_chan); -out: - kfree(src); - kfree(dest); - return err; -} - -#define IOP_ADMA_NUM_SRC_TEST 4 /* must be <= 15 */ -static int -iop_adma_xor_val_self_test(struct iop_adma_device *device) -{ - int i, src_idx; - struct page *dest; - struct page *xor_srcs[IOP_ADMA_NUM_SRC_TEST]; - struct page *zero_sum_srcs[IOP_ADMA_NUM_SRC_TEST + 1]; - dma_addr_t dma_srcs[IOP_ADMA_NUM_SRC_TEST + 1]; - dma_addr_t dest_dma; - struct dma_async_tx_descriptor *tx; - struct dma_chan *dma_chan; - dma_cookie_t cookie; - u8 cmp_byte = 0; - u32 cmp_word; - u32 zero_sum_result; - int err = 0; - struct iop_adma_chan *iop_chan; - - dev_dbg(device->common.dev, "%s\n", __func__); - - for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) { - xor_srcs[src_idx] = alloc_page(GFP_KERNEL); - if (!xor_srcs[src_idx]) { - while (src_idx--) - __free_page(xor_srcs[src_idx]); - return -ENOMEM; - } - } - - dest = alloc_page(GFP_KERNEL); - if (!dest) { - while (src_idx--) - __free_page(xor_srcs[src_idx]); - return -ENOMEM; - } - - /* Fill in src buffers */ - for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) { - u8 *ptr = page_address(xor_srcs[src_idx]); - for (i = 0; i < PAGE_SIZE; i++) - ptr[i] = (1 << src_idx); - } - - for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) - cmp_byte ^= (u8) (1 << src_idx); - - cmp_word = (cmp_byte << 24) | (cmp_byte << 16) | - (cmp_byte << 8) | cmp_byte; - - memset(page_address(dest), 0, PAGE_SIZE); - - dma_chan = container_of(device->common.channels.next, - struct dma_chan, - device_node); - if (iop_adma_alloc_chan_resources(dma_chan) < 1) { - err = -ENODEV; - goto out; - } - - /* test xor */ - dest_dma = dma_map_page(dma_chan->device->dev, dest, 0, - PAGE_SIZE, DMA_FROM_DEVICE); - for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) - dma_srcs[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i], - 0, PAGE_SIZE, DMA_TO_DEVICE); - tx = iop_adma_prep_dma_xor(dma_chan, dest_dma, dma_srcs, - IOP_ADMA_NUM_SRC_TEST, PAGE_SIZE, - DMA_PREP_INTERRUPT | DMA_CTRL_ACK); - - cookie = iop_adma_tx_submit(tx); - iop_adma_issue_pending(dma_chan); - msleep(8); - - if (iop_adma_status(dma_chan, cookie, NULL) != - DMA_COMPLETE) { - dev_err(dma_chan->device->dev, - "Self-test xor timed out, disabling\n"); - err = -ENODEV; - goto free_resources; - } - - iop_chan = to_iop_adma_chan(dma_chan); - dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma, - PAGE_SIZE, DMA_FROM_DEVICE); - for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) { - u32 *ptr = page_address(dest); - if (ptr[i] != cmp_word) { - dev_err(dma_chan->device->dev, - "Self-test xor failed compare, disabling\n"); - err = -ENODEV; - goto free_resources; - } - } - dma_sync_single_for_device(&iop_chan->device->pdev->dev, dest_dma, - PAGE_SIZE, DMA_TO_DEVICE); - - /* skip zero sum if the capability is not present */ - if (!dma_has_cap(DMA_XOR_VAL, dma_chan->device->cap_mask)) - goto free_resources; - - /* zero sum the sources with the destintation page */ - for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) - zero_sum_srcs[i] = xor_srcs[i]; - zero_sum_srcs[i] = dest; - - zero_sum_result = 1; - - for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++) - dma_srcs[i] = dma_map_page(dma_chan->device->dev, - zero_sum_srcs[i], 0, PAGE_SIZE, - DMA_TO_DEVICE); - tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs, - IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE, - &zero_sum_result, - DMA_PREP_INTERRUPT | DMA_CTRL_ACK); - - cookie = iop_adma_tx_submit(tx); - iop_adma_issue_pending(dma_chan); - msleep(8); - - if (iop_adma_status(dma_chan, cookie, NULL) != DMA_COMPLETE) { - dev_err(dma_chan->device->dev, - "Self-test zero sum timed out, disabling\n"); - err = -ENODEV; - goto free_resources; - } - - if (zero_sum_result != 0) { - dev_err(dma_chan->device->dev, - "Self-test zero sum failed compare, disabling\n"); - err = -ENODEV; - goto free_resources; - } - - /* test for non-zero parity sum */ - zero_sum_result = 0; - for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++) - dma_srcs[i] = dma_map_page(dma_chan->device->dev, - zero_sum_srcs[i], 0, PAGE_SIZE, - DMA_TO_DEVICE); - tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs, - IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE, - &zero_sum_result, - DMA_PREP_INTERRUPT | DMA_CTRL_ACK); - - cookie = iop_adma_tx_submit(tx); - iop_adma_issue_pending(dma_chan); - msleep(8); - - if (iop_adma_status(dma_chan, cookie, NULL) != DMA_COMPLETE) { - dev_err(dma_chan->device->dev, - "Self-test non-zero sum timed out, disabling\n"); - err = -ENODEV; - goto free_resources; - } - - if (zero_sum_result != 1) { - dev_err(dma_chan->device->dev, - "Self-test non-zero sum failed compare, disabling\n"); - err = -ENODEV; - goto free_resources; - } - -free_resources: - iop_adma_free_chan_resources(dma_chan); -out: - src_idx = IOP_ADMA_NUM_SRC_TEST; - while (src_idx--) - __free_page(xor_srcs[src_idx]); - __free_page(dest); - return err; -} - -#ifdef CONFIG_RAID6_PQ -static int -iop_adma_pq_zero_sum_self_test(struct iop_adma_device *device) -{ - /* combined sources, software pq results, and extra hw pq results */ - struct page *pq[IOP_ADMA_NUM_SRC_TEST+2+2]; - /* ptr to the extra hw pq buffers defined above */ - struct page **pq_hw = &pq[IOP_ADMA_NUM_SRC_TEST+2]; - /* address conversion buffers (dma_map / page_address) */ - void *pq_sw[IOP_ADMA_NUM_SRC_TEST+2]; - dma_addr_t pq_src[IOP_ADMA_NUM_SRC_TEST+2]; - dma_addr_t *pq_dest = &pq_src[IOP_ADMA_NUM_SRC_TEST]; - - int i; - struct dma_async_tx_descriptor *tx; - struct dma_chan *dma_chan; - dma_cookie_t cookie; - u32 zero_sum_result; - int err = 0; - struct device *dev; - - dev_dbg(device->common.dev, "%s\n", __func__); - - for (i = 0; i < ARRAY_SIZE(pq); i++) { - pq[i] = alloc_page(GFP_KERNEL); - if (!pq[i]) { - while (i--) - __free_page(pq[i]); - return -ENOMEM; - } - } - - /* Fill in src buffers */ - for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) { - pq_sw[i] = page_address(pq[i]); - memset(pq_sw[i], 0x11111111 * (1<<i), PAGE_SIZE); - } - pq_sw[i] = page_address(pq[i]); - pq_sw[i+1] = page_address(pq[i+1]); - - dma_chan = container_of(device->common.channels.next, - struct dma_chan, - device_node); - if (iop_adma_alloc_chan_resources(dma_chan) < 1) { - err = -ENODEV; - goto out; - } - - dev = dma_chan->device->dev; - - /* initialize the dests */ - memset(page_address(pq_hw[0]), 0 , PAGE_SIZE); - memset(page_address(pq_hw[1]), 0 , PAGE_SIZE); - - /* test pq */ - pq_dest[0] = dma_map_page(dev, pq_hw[0], 0, PAGE_SIZE, DMA_FROM_DEVICE); - pq_dest[1] = dma_map_page(dev, pq_hw[1], 0, PAGE_SIZE, DMA_FROM_DEVICE); - for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) - pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE, - DMA_TO_DEVICE); - - tx = iop_adma_prep_dma_pq(dma_chan, pq_dest, pq_src, - IOP_ADMA_NUM_SRC_TEST, (u8 *)raid6_gfexp, - PAGE_SIZE, - DMA_PREP_INTERRUPT | - DMA_CTRL_ACK); - - cookie = iop_adma_tx_submit(tx); - iop_adma_issue_pending(dma_chan); - msleep(8); - - if (iop_adma_status(dma_chan, cookie, NULL) != - DMA_COMPLETE) { - dev_err(dev, "Self-test pq timed out, disabling\n"); - err = -ENODEV; - goto free_resources; - } - - raid6_call.gen_syndrome(IOP_ADMA_NUM_SRC_TEST+2, PAGE_SIZE, pq_sw); - - if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST], - page_address(pq_hw[0]), PAGE_SIZE) != 0) { - dev_err(dev, "Self-test p failed compare, disabling\n"); - err = -ENODEV; - goto free_resources; - } - if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST+1], - page_address(pq_hw[1]), PAGE_SIZE) != 0) { - dev_err(dev, "Self-test q failed compare, disabling\n"); - err = -ENODEV; - goto free_resources; - } - - /* test correct zero sum using the software generated pq values */ - for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++) - pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE, - DMA_TO_DEVICE); - - zero_sum_result = ~0; - tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST], - pq_src, IOP_ADMA_NUM_SRC_TEST, - raid6_gfexp, PAGE_SIZE, &zero_sum_result, - DMA_PREP_INTERRUPT|DMA_CTRL_ACK); - - cookie = iop_adma_tx_submit(tx); - iop_adma_issue_pending(dma_chan); - msleep(8); - - if (iop_adma_status(dma_chan, cookie, NULL) != - DMA_COMPLETE) { - dev_err(dev, "Self-test pq-zero-sum timed out, disabling\n"); - err = -ENODEV; - goto free_resources; - } - - if (zero_sum_result != 0) { - dev_err(dev, "Self-test pq-zero-sum failed to validate: %x\n", - zero_sum_result); - err = -ENODEV; - goto free_resources; - } - - /* test incorrect zero sum */ - i = IOP_ADMA_NUM_SRC_TEST; - memset(pq_sw[i] + 100, 0, 100); - memset(pq_sw[i+1] + 200, 0, 200); - for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++) - pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE, - DMA_TO_DEVICE); - - zero_sum_result = 0; - tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST], - pq_src, IOP_ADMA_NUM_SRC_TEST, - raid6_gfexp, PAGE_SIZE, &zero_sum_result, - DMA_PREP_INTERRUPT|DMA_CTRL_ACK); - - cookie = iop_adma_tx_submit(tx); - iop_adma_issue_pending(dma_chan); - msleep(8); - - if (iop_adma_status(dma_chan, cookie, NULL) != - DMA_COMPLETE) { - dev_err(dev, "Self-test !pq-zero-sum timed out, disabling\n"); - err = -ENODEV; - goto free_resources; - } - - if (zero_sum_result != (SUM_CHECK_P_RESULT | SUM_CHECK_Q_RESULT)) { - dev_err(dev, "Self-test !pq-zero-sum failed to validate: %x\n", - zero_sum_result); - err = -ENODEV; - goto free_resources; - } - -free_resources: - iop_adma_free_chan_resources(dma_chan); -out: - i = ARRAY_SIZE(pq); - while (i--) - __free_page(pq[i]); - return err; -} -#endif - -static int iop_adma_remove(struct platform_device *dev) -{ - struct iop_adma_device *device = platform_get_drvdata(dev); - struct dma_chan *chan, *_chan; - struct iop_adma_chan *iop_chan; - struct iop_adma_platform_data *plat_data = dev_get_platdata(&dev->dev); - - dma_async_device_unregister(&device->common); - - dma_free_coherent(&dev->dev, plat_data->pool_size, - device->dma_desc_pool_virt, device->dma_desc_pool); - - list_for_each_entry_safe(chan, _chan, &device->common.channels, - device_node) { - iop_chan = to_iop_adma_chan(chan); - list_del(&chan->device_node); - kfree(iop_chan); - } - kfree(device); - - return 0; -} - -static int iop_adma_probe(struct platform_device *pdev) -{ - struct resource *res; - int ret = 0, i; - struct iop_adma_device *adev; - struct iop_adma_chan *iop_chan; - struct dma_device *dma_dev; - struct iop_adma_platform_data *plat_data = dev_get_platdata(&pdev->dev); - - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!res) - return -ENODEV; - - if (!devm_request_mem_region(&pdev->dev, res->start, - resource_size(res), pdev->name)) - return -EBUSY; - - adev = kzalloc(sizeof(*adev), GFP_KERNEL); - if (!adev) - return -ENOMEM; - dma_dev = &adev->common; - - /* allocate coherent memory for hardware descriptors - * note: writecombine gives slightly better performance, but - * requires that we explicitly flush the writes - */ - adev->dma_desc_pool_virt = dma_alloc_wc(&pdev->dev, - plat_data->pool_size, - &adev->dma_desc_pool, - GFP_KERNEL); - if (!adev->dma_desc_pool_virt) { - ret = -ENOMEM; - goto err_free_adev; - } - - dev_dbg(&pdev->dev, "%s: allocated descriptor pool virt %p phys %pad\n", - __func__, adev->dma_desc_pool_virt, &adev->dma_desc_pool); - - adev->id = plat_data->hw_id; - - /* discover transaction capabilites from the platform data */ - dma_dev->cap_mask = plat_data->cap_mask; - - adev->pdev = pdev; - platform_set_drvdata(pdev, adev); - - INIT_LIST_HEAD(&dma_dev->channels); - - /* set base routines */ - dma_dev->device_alloc_chan_resources = iop_adma_alloc_chan_resources; - dma_dev->device_free_chan_resources = iop_adma_free_chan_resources; - dma_dev->device_tx_status = iop_adma_status; - dma_dev->device_issue_pending = iop_adma_issue_pending; - dma_dev->dev = &pdev->dev; - - /* set prep routines based on capability */ - if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) - dma_dev->device_prep_dma_memcpy = iop_adma_prep_dma_memcpy; - if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { - dma_dev->max_xor = iop_adma_get_max_xor(); - dma_dev->device_prep_dma_xor = iop_adma_prep_dma_xor; - } - if (dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask)) - dma_dev->device_prep_dma_xor_val = - iop_adma_prep_dma_xor_val; - if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) { - dma_set_maxpq(dma_dev, iop_adma_get_max_pq(), 0); - dma_dev->device_prep_dma_pq = iop_adma_prep_dma_pq; - } - if (dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask)) - dma_dev->device_prep_dma_pq_val = - iop_adma_prep_dma_pq_val; - if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask)) - dma_dev->device_prep_dma_interrupt = - iop_adma_prep_dma_interrupt; - - iop_chan = kzalloc(sizeof(*iop_chan), GFP_KERNEL); - if (!iop_chan) { - ret = -ENOMEM; - goto err_free_dma; - } - iop_chan->device = adev; - - iop_chan->mmr_base = devm_ioremap(&pdev->dev, res->start, - resource_size(res)); - if (!iop_chan->mmr_base) { - ret = -ENOMEM; - goto err_free_iop_chan; - } - tasklet_setup(&iop_chan->irq_tasklet, iop_adma_tasklet); - - /* clear errors before enabling interrupts */ - iop_adma_device_clear_err_status(iop_chan); - - for (i = 0; i < 3; i++) { - static const irq_handler_t handler[] = { - iop_adma_eot_handler, - iop_adma_eoc_handler, - iop_adma_err_handler - }; - int irq = platform_get_irq(pdev, i); - if (irq < 0) { - ret = -ENXIO; - goto err_free_iop_chan; - } else { - ret = devm_request_irq(&pdev->dev, irq, - handler[i], 0, pdev->name, iop_chan); - if (ret) - goto err_free_iop_chan; - } - } - - spin_lock_init(&iop_chan->lock); - INIT_LIST_HEAD(&iop_chan->chain); - INIT_LIST_HEAD(&iop_chan->all_slots); - iop_chan->common.device = dma_dev; - dma_cookie_init(&iop_chan->common); - list_add_tail(&iop_chan->common.device_node, &dma_dev->channels); - - if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) { - ret = iop_adma_memcpy_self_test(adev); - dev_dbg(&pdev->dev, "memcpy self test returned %d\n", ret); - if (ret) - goto err_free_iop_chan; - } - - if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { - ret = iop_adma_xor_val_self_test(adev); - dev_dbg(&pdev->dev, "xor self test returned %d\n", ret); - if (ret) - goto err_free_iop_chan; - } - - if (dma_has_cap(DMA_PQ, dma_dev->cap_mask) && - dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask)) { - #ifdef CONFIG_RAID6_PQ - ret = iop_adma_pq_zero_sum_self_test(adev); - dev_dbg(&pdev->dev, "pq self test returned %d\n", ret); - #else - /* can not test raid6, so do not publish capability */ - dma_cap_clear(DMA_PQ, dma_dev->cap_mask); - dma_cap_clear(DMA_PQ_VAL, dma_dev->cap_mask); - ret = 0; - #endif - if (ret) - goto err_free_iop_chan; - } - - dev_info(&pdev->dev, "Intel(R) IOP: ( %s%s%s%s%s%s)\n", - dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "pq " : "", - dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask) ? "pq_val " : "", - dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "", - dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask) ? "xor_val " : "", - dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "", - dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : ""); - - dma_async_device_register(dma_dev); - goto out; - - err_free_iop_chan: - kfree(iop_chan); - err_free_dma: - dma_free_coherent(&adev->pdev->dev, plat_data->pool_size, - adev->dma_desc_pool_virt, adev->dma_desc_pool); - err_free_adev: - kfree(adev); - out: - return ret; -} - -static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan) -{ - struct iop_adma_desc_slot *sw_desc, *grp_start; - dma_cookie_t cookie; - int slot_cnt, slots_per_op; - - dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); - - spin_lock_bh(&iop_chan->lock); - slot_cnt = iop_chan_memcpy_slot_count(0, &slots_per_op); - sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); - if (sw_desc) { - grp_start = sw_desc->group_head; - - list_splice_init(&sw_desc->tx_list, &iop_chan->chain); - async_tx_ack(&sw_desc->async_tx); - iop_desc_init_memcpy(grp_start, 0); - iop_desc_set_byte_count(grp_start, iop_chan, 0); - iop_desc_set_dest_addr(grp_start, iop_chan, 0); - iop_desc_set_memcpy_src_addr(grp_start, 0); - - cookie = dma_cookie_assign(&sw_desc->async_tx); - - /* initialize the completed cookie to be less than - * the most recently used cookie - */ - iop_chan->common.completed_cookie = cookie - 1; - - /* channel should not be busy */ - BUG_ON(iop_chan_is_busy(iop_chan)); - - /* clear any prior error-status bits */ - iop_adma_device_clear_err_status(iop_chan); - - /* disable operation */ - iop_chan_disable(iop_chan); - - /* set the descriptor address */ - iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys); - - /* 1/ don't add pre-chained descriptors - * 2/ dummy read to flush next_desc write - */ - BUG_ON(iop_desc_get_next_desc(sw_desc)); - - /* run the descriptor */ - iop_chan_enable(iop_chan); - } else - dev_err(iop_chan->device->common.dev, - "failed to allocate null descriptor\n"); - spin_unlock_bh(&iop_chan->lock); -} - -static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan) -{ - struct iop_adma_desc_slot *sw_desc, *grp_start; - dma_cookie_t cookie; - int slot_cnt, slots_per_op; - - dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); - - spin_lock_bh(&iop_chan->lock); - slot_cnt = iop_chan_xor_slot_count(0, 2, &slots_per_op); - sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); - if (sw_desc) { - grp_start = sw_desc->group_head; - list_splice_init(&sw_desc->tx_list, &iop_chan->chain); - async_tx_ack(&sw_desc->async_tx); - iop_desc_init_null_xor(grp_start, 2, 0); - iop_desc_set_byte_count(grp_start, iop_chan, 0); - iop_desc_set_dest_addr(grp_start, iop_chan, 0); - iop_desc_set_xor_src_addr(grp_start, 0, 0); - iop_desc_set_xor_src_addr(grp_start, 1, 0); - - cookie = dma_cookie_assign(&sw_desc->async_tx); - - /* initialize the completed cookie to be less than - * the most recently used cookie - */ - iop_chan->common.completed_cookie = cookie - 1; - - /* channel should not be busy */ - BUG_ON(iop_chan_is_busy(iop_chan)); - - /* clear any prior error-status bits */ - iop_adma_device_clear_err_status(iop_chan); - - /* disable operation */ - iop_chan_disable(iop_chan); - - /* set the descriptor address */ - iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys); - - /* 1/ don't add pre-chained descriptors - * 2/ dummy read to flush next_desc write - */ - BUG_ON(iop_desc_get_next_desc(sw_desc)); - - /* run the descriptor */ - iop_chan_enable(iop_chan); - } else - dev_err(iop_chan->device->common.dev, - "failed to allocate null descriptor\n"); - spin_unlock_bh(&iop_chan->lock); -} - -static struct platform_driver iop_adma_driver = { - .probe = iop_adma_probe, - .remove = iop_adma_remove, - .driver = { - .name = "iop-adma", - }, -}; - -module_platform_driver(iop_adma_driver); - -MODULE_AUTHOR("Intel Corporation"); -MODULE_DESCRIPTION("IOP ADMA Engine Driver"); -MODULE_LICENSE("GPL"); -MODULE_ALIAS("platform:iop-adma"); diff --git a/drivers/dma/iop-adma.h b/drivers/dma/iop-adma.h deleted file mode 100644 index d44eabb6f5eb..000000000000 --- a/drivers/dma/iop-adma.h +++ /dev/null @@ -1,914 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright © 2006, Intel Corporation. - */ -#ifndef _ADMA_H -#define _ADMA_H -#include <linux/types.h> -#include <linux/io.h> -#include <linux/platform_data/dma-iop32x.h> - -/* Memory copy units */ -#define DMA_CCR(chan) (chan->mmr_base + 0x0) -#define DMA_CSR(chan) (chan->mmr_base + 0x4) -#define DMA_DAR(chan) (chan->mmr_base + 0xc) -#define DMA_NDAR(chan) (chan->mmr_base + 0x10) -#define DMA_PADR(chan) (chan->mmr_base + 0x14) -#define DMA_PUADR(chan) (chan->mmr_base + 0x18) -#define DMA_LADR(chan) (chan->mmr_base + 0x1c) -#define DMA_BCR(chan) (chan->mmr_base + 0x20) -#define DMA_DCR(chan) (chan->mmr_base + 0x24) - -/* Application accelerator unit */ -#define AAU_ACR(chan) (chan->mmr_base + 0x0) -#define AAU_ASR(chan) (chan->mmr_base + 0x4) -#define AAU_ADAR(chan) (chan->mmr_base + 0x8) -#define AAU_ANDAR(chan) (chan->mmr_base + 0xc) -#define AAU_SAR(src, chan) (chan->mmr_base + (0x10 + ((src) << 2))) -#define AAU_DAR(chan) (chan->mmr_base + 0x20) -#define AAU_ABCR(chan) (chan->mmr_base + 0x24) -#define AAU_ADCR(chan) (chan->mmr_base + 0x28) -#define AAU_SAR_EDCR(src_edc) (chan->mmr_base + (0x02c + ((src_edc-4) << 2))) -#define AAU_EDCR0_IDX 8 -#define AAU_EDCR1_IDX 17 -#define AAU_EDCR2_IDX 26 - -struct iop3xx_aau_desc_ctrl { - unsigned int int_en:1; - unsigned int blk1_cmd_ctrl:3; - unsigned int blk2_cmd_ctrl:3; - unsigned int blk3_cmd_ctrl:3; - unsigned int blk4_cmd_ctrl:3; - unsigned int blk5_cmd_ctrl:3; - unsigned int blk6_cmd_ctrl:3; - unsigned int blk7_cmd_ctrl:3; - unsigned int blk8_cmd_ctrl:3; - unsigned int blk_ctrl:2; - unsigned int dual_xor_en:1; - unsigned int tx_complete:1; - unsigned int zero_result_err:1; - unsigned int zero_result_en:1; - unsigned int dest_write_en:1; -}; - -struct iop3xx_aau_e_desc_ctrl { - unsigned int reserved:1; - unsigned int blk1_cmd_ctrl:3; - unsigned int blk2_cmd_ctrl:3; - unsigned int blk3_cmd_ctrl:3; - unsigned int blk4_cmd_ctrl:3; - unsigned int blk5_cmd_ctrl:3; - unsigned int blk6_cmd_ctrl:3; - unsigned int blk7_cmd_ctrl:3; - unsigned int blk8_cmd_ctrl:3; - unsigned int reserved2:7; -}; - -struct iop3xx_dma_desc_ctrl { - unsigned int pci_transaction:4; - unsigned int int_en:1; - unsigned int dac_cycle_en:1; - unsigned int mem_to_mem_en:1; - unsigned int crc_data_tx_en:1; - unsigned int crc_gen_en:1; - unsigned int crc_seed_dis:1; - unsigned int reserved:21; - unsigned int crc_tx_complete:1; -}; - -struct iop3xx_desc_dma { - u32 next_desc; - union { - u32 pci_src_addr; - u32 pci_dest_addr; - u32 src_addr; - }; - union { - u32 upper_pci_src_addr; - u32 upper_pci_dest_addr; - }; - union { - u32 local_pci_src_addr; - u32 local_pci_dest_addr; - u32 dest_addr; - }; - u32 byte_count; - union { - u32 desc_ctrl; - struct iop3xx_dma_desc_ctrl desc_ctrl_field; - }; - u32 crc_addr; -}; - -struct iop3xx_desc_aau { - u32 next_desc; - u32 src[4]; - u32 dest_addr; - u32 byte_count; - union { - u32 desc_ctrl; - struct iop3xx_aau_desc_ctrl desc_ctrl_field; - }; - union { - u32 src_addr; - u32 e_desc_ctrl; - struct iop3xx_aau_e_desc_ctrl e_desc_ctrl_field; - } src_edc[31]; -}; - -struct iop3xx_aau_gfmr { - unsigned int gfmr1:8; - unsigned int gfmr2:8; - unsigned int gfmr3:8; - unsigned int gfmr4:8; -}; - -struct iop3xx_desc_pq_xor { - u32 next_desc; - u32 src[3]; - union { - u32 data_mult1; - struct iop3xx_aau_gfmr data_mult1_field; - }; - u32 dest_addr; - u32 byte_count; - union { - u32 desc_ctrl; - struct iop3xx_aau_desc_ctrl desc_ctrl_field; - }; - union { - u32 src_addr; - u32 e_desc_ctrl; - struct iop3xx_aau_e_desc_ctrl e_desc_ctrl_field; - u32 data_multiplier; - struct iop3xx_aau_gfmr data_mult_field; - u32 reserved; - } src_edc_gfmr[19]; -}; - -struct iop3xx_desc_dual_xor { - u32 next_desc; - u32 src0_addr; - u32 src1_addr; - u32 h_src_addr; - u32 d_src_addr; - u32 h_dest_addr; - u32 byte_count; - union { - u32 desc_ctrl; - struct iop3xx_aau_desc_ctrl desc_ctrl_field; - }; - u32 d_dest_addr; -}; - -union iop3xx_desc { - struct iop3xx_desc_aau *aau; - struct iop3xx_desc_dma *dma; - struct iop3xx_desc_pq_xor *pq_xor; - struct iop3xx_desc_dual_xor *dual_xor; - void *ptr; -}; - -/* No support for p+q operations */ -static inline int -iop_chan_pq_slot_count(size_t len, int src_cnt, int *slots_per_op) -{ - BUG(); - return 0; -} - -static inline void -iop_desc_init_pq(struct iop_adma_desc_slot *desc, int src_cnt, - unsigned long flags) -{ - BUG(); -} - -static inline void -iop_desc_set_pq_addr(struct iop_adma_desc_slot *desc, dma_addr_t *addr) -{ - BUG(); -} - -static inline void -iop_desc_set_pq_src_addr(struct iop_adma_desc_slot *desc, int src_idx, - dma_addr_t addr, unsigned char coef) -{ - BUG(); -} - -static inline int -iop_chan_pq_zero_sum_slot_count(size_t len, int src_cnt, int *slots_per_op) -{ - BUG(); - return 0; -} - -static inline void -iop_desc_init_pq_zero_sum(struct iop_adma_desc_slot *desc, int src_cnt, - unsigned long flags) -{ - BUG(); -} - -static inline void -iop_desc_set_pq_zero_sum_byte_count(struct iop_adma_desc_slot *desc, u32 len) -{ - BUG(); -} - -#define iop_desc_set_pq_zero_sum_src_addr iop_desc_set_pq_src_addr - -static inline void -iop_desc_set_pq_zero_sum_addr(struct iop_adma_desc_slot *desc, int pq_idx, - dma_addr_t *src) -{ - BUG(); -} - -static inline int iop_adma_get_max_xor(void) -{ - return 32; -} - -static inline int iop_adma_get_max_pq(void) -{ - BUG(); - return 0; -} - -static inline u32 iop_chan_get_current_descriptor(struct iop_adma_chan *chan) -{ - int id = chan->device->id; - - switch (id) { - case DMA0_ID: - case DMA1_ID: - return __raw_readl(DMA_DAR(chan)); - case AAU_ID: - return __raw_readl(AAU_ADAR(chan)); - default: - BUG(); - } - return 0; -} - -static inline void iop_chan_set_next_descriptor(struct iop_adma_chan *chan, - u32 next_desc_addr) -{ - int id = chan->device->id; - - switch (id) { - case DMA0_ID: - case DMA1_ID: - __raw_writel(next_desc_addr, DMA_NDAR(chan)); - break; - case AAU_ID: - __raw_writel(next_desc_addr, AAU_ANDAR(chan)); - break; - } - -} - -#define IOP_ADMA_STATUS_BUSY (1 << 10) -#define IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT (1024) -#define IOP_ADMA_XOR_MAX_BYTE_COUNT (16 * 1024 * 1024) -#define IOP_ADMA_MAX_BYTE_COUNT (16 * 1024 * 1024) - -static inline int iop_chan_is_busy(struct iop_adma_chan *chan) -{ - u32 status = __raw_readl(DMA_CSR(chan)); - return (status & IOP_ADMA_STATUS_BUSY) ? 1 : 0; -} - -static inline int iop_desc_is_aligned(struct iop_adma_desc_slot *desc, - int num_slots) -{ - /* num_slots will only ever be 1, 2, 4, or 8 */ - return (desc->idx & (num_slots - 1)) ? 0 : 1; -} - -/* to do: support large (i.e. > hw max) buffer sizes */ -static inline int iop_chan_memcpy_slot_count(size_t len, int *slots_per_op) -{ - *slots_per_op = 1; - return 1; -} - -/* to do: support large (i.e. > hw max) buffer sizes */ -static inline int iop_chan_memset_slot_count(size_t len, int *slots_per_op) -{ - *slots_per_op = 1; - return 1; -} - -static inline int iop3xx_aau_xor_slot_count(size_t len, int src_cnt, - int *slots_per_op) -{ - static const char slot_count_table[] = { - 1, 1, 1, 1, /* 01 - 04 */ - 2, 2, 2, 2, /* 05 - 08 */ - 4, 4, 4, 4, /* 09 - 12 */ - 4, 4, 4, 4, /* 13 - 16 */ - 8, 8, 8, 8, /* 17 - 20 */ - 8, 8, 8, 8, /* 21 - 24 */ - 8, 8, 8, 8, /* 25 - 28 */ - 8, 8, 8, 8, /* 29 - 32 */ - }; - *slots_per_op = slot_count_table[src_cnt - 1]; - return *slots_per_op; -} - -static inline int -iop_chan_interrupt_slot_count(int *slots_per_op, struct iop_adma_chan *chan) -{ - switch (chan->device->id) { - case DMA0_ID: - case DMA1_ID: - return iop_chan_memcpy_slot_count(0, slots_per_op); - case AAU_ID: - return iop3xx_aau_xor_slot_count(0, 2, slots_per_op); - default: - BUG(); - } - return 0; -} - -static inline int iop_chan_xor_slot_count(size_t len, int src_cnt, - int *slots_per_op) -{ - int slot_cnt = iop3xx_aau_xor_slot_count(len, src_cnt, slots_per_op); - - if (len <= IOP_ADMA_XOR_MAX_BYTE_COUNT) - return slot_cnt; - - len -= IOP_ADMA_XOR_MAX_BYTE_COUNT; - while (len > IOP_ADMA_XOR_MAX_BYTE_COUNT) { - len -= IOP_ADMA_XOR_MAX_BYTE_COUNT; - slot_cnt += *slots_per_op; - } - - slot_cnt += *slots_per_op; - - return slot_cnt; -} - -/* zero sum on iop3xx is limited to 1k at a time so it requires multiple - * descriptors - */ -static inline int iop_chan_zero_sum_slot_count(size_t len, int src_cnt, - int *slots_per_op) -{ - int slot_cnt = iop3xx_aau_xor_slot_count(len, src_cnt, slots_per_op); - - if (len <= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT) - return slot_cnt; - - len -= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT; - while (len > IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT) { - len -= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT; - slot_cnt += *slots_per_op; - } - - slot_cnt += *slots_per_op; - - return slot_cnt; -} - -static inline u32 iop_desc_get_byte_count(struct iop_adma_desc_slot *desc, - struct iop_adma_chan *chan) -{ - union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, }; - - switch (chan->device->id) { - case DMA0_ID: - case DMA1_ID: - return hw_desc.dma->byte_count; - case AAU_ID: - return hw_desc.aau->byte_count; - default: - BUG(); - } - return 0; -} - -/* translate the src_idx to a descriptor word index */ -static inline int __desc_idx(int src_idx) -{ - static const int desc_idx_table[] = { 0, 0, 0, 0, - 0, 1, 2, 3, - 5, 6, 7, 8, - 9, 10, 11, 12, - 14, 15, 16, 17, - 18, 19, 20, 21, - 23, 24, 25, 26, - 27, 28, 29, 30, - }; - - return desc_idx_table[src_idx]; -} - -static inline u32 iop_desc_get_src_addr(struct iop_adma_desc_slot *desc, - struct iop_adma_chan *chan, - int src_idx) -{ - union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, }; - - switch (chan->device->id) { - case DMA0_ID: - case DMA1_ID: - return hw_desc.dma->src_addr; - case AAU_ID: - break; - default: - BUG(); - } - - if (src_idx < 4) - return hw_desc.aau->src[src_idx]; - else - return hw_desc.aau->src_edc[__desc_idx(src_idx)].src_addr; -} - -static inline void iop3xx_aau_desc_set_src_addr(struct iop3xx_desc_aau *hw_desc, - int src_idx, dma_addr_t addr) -{ - if (src_idx < 4) - hw_desc->src[src_idx] = addr; - else - hw_desc->src_edc[__desc_idx(src_idx)].src_addr = addr; -} - -static inline void -iop_desc_init_memcpy(struct iop_adma_desc_slot *desc, unsigned long flags) -{ - struct iop3xx_desc_dma *hw_desc = desc->hw_desc; - union { - u32 value; - struct iop3xx_dma_desc_ctrl field; - } u_desc_ctrl; - - u_desc_ctrl.value = 0; - u_desc_ctrl.field.mem_to_mem_en = 1; - u_desc_ctrl.field.pci_transaction = 0xe; /* memory read block */ - u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT; - hw_desc->desc_ctrl = u_desc_ctrl.value; - hw_desc->upper_pci_src_addr = 0; - hw_desc->crc_addr = 0; -} - -static inline void -iop_desc_init_memset(struct iop_adma_desc_slot *desc, unsigned long flags) -{ - struct iop3xx_desc_aau *hw_desc = desc->hw_desc; - union { - u32 value; - struct iop3xx_aau_desc_ctrl field; - } u_desc_ctrl; - - u_desc_ctrl.value = 0; - u_desc_ctrl.field.blk1_cmd_ctrl = 0x2; /* memory block fill */ - u_desc_ctrl.field.dest_write_en = 1; - u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT; - hw_desc->desc_ctrl = u_desc_ctrl.value; -} - -static inline u32 -iop3xx_desc_init_xor(struct iop3xx_desc_aau *hw_desc, int src_cnt, - unsigned long flags) -{ - int i, shift; - u32 edcr; - union { - u32 value; - struct iop3xx_aau_desc_ctrl field; - } u_desc_ctrl; - - u_desc_ctrl.value = 0; - switch (src_cnt) { - case 25 ... 32: - u_desc_ctrl.field.blk_ctrl = 0x3; /* use EDCR[2:0] */ - edcr = 0; - shift = 1; - for (i = 24; i < src_cnt; i++) { - edcr |= (1 << shift); - shift += 3; - } - hw_desc->src_edc[AAU_EDCR2_IDX].e_desc_ctrl = edcr; - src_cnt = 24; - fallthrough; - case 17 ... 24: - if (!u_desc_ctrl.field.blk_ctrl) { - hw_desc->src_edc[AAU_EDCR2_IDX].e_desc_ctrl = 0; - u_desc_ctrl.field.blk_ctrl = 0x3; /* use EDCR[2:0] */ - } - edcr = 0; - shift = 1; - for (i = 16; i < src_cnt; i++) { - edcr |= (1 << shift); - shift += 3; - } - hw_desc->src_edc[AAU_EDCR1_IDX].e_desc_ctrl = edcr; - src_cnt = 16; - fallthrough; - case 9 ... 16: - if (!u_desc_ctrl.field.blk_ctrl) - u_desc_ctrl.field.blk_ctrl = 0x2; /* use EDCR0 */ - edcr = 0; - shift = 1; - for (i = 8; i < src_cnt; i++) { - edcr |= (1 << shift); - shift += 3; - } - hw_desc->src_edc[AAU_EDCR0_IDX].e_desc_ctrl = edcr; - src_cnt = 8; - fallthrough; - case 2 ... 8: - shift = 1; - for (i = 0; i < src_cnt; i++) { - u_desc_ctrl.value |= (1 << shift); - shift += 3; - } - - if (!u_desc_ctrl.field.blk_ctrl && src_cnt > 4) - u_desc_ctrl.field.blk_ctrl = 0x1; /* use mini-desc */ - } - - u_desc_ctrl.field.dest_write_en = 1; - u_desc_ctrl.field.blk1_cmd_ctrl = 0x7; /* direct fill */ - u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT; - hw_desc->desc_ctrl = u_desc_ctrl.value; - - return u_desc_ctrl.value; -} - -static inline void -iop_desc_init_xor(struct iop_adma_desc_slot *desc, int src_cnt, - unsigned long flags) -{ - iop3xx_desc_init_xor(desc->hw_desc, src_cnt, flags); -} - -/* return the number of operations */ -static inline int -iop_desc_init_zero_sum(struct iop_adma_desc_slot *desc, int src_cnt, - unsigned long flags) -{ - int slot_cnt = desc->slot_cnt, slots_per_op = desc->slots_per_op; - struct iop3xx_desc_aau *hw_desc, *prev_hw_desc, *iter; - union { - u32 value; - struct iop3xx_aau_desc_ctrl field; - } u_desc_ctrl; - int i, j; - - hw_desc = desc->hw_desc; - - for (i = 0, j = 0; (slot_cnt -= slots_per_op) >= 0; - i += slots_per_op, j++) { - iter = iop_hw_desc_slot_idx(hw_desc, i); - u_desc_ctrl.value = iop3xx_desc_init_xor(iter, src_cnt, flags); - u_desc_ctrl.field.dest_write_en = 0; - u_desc_ctrl.field.zero_result_en = 1; - u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT; - iter->desc_ctrl = u_desc_ctrl.value; - - /* for the subsequent descriptors preserve the store queue - * and chain them together - */ - if (i) { - prev_hw_desc = - iop_hw_desc_slot_idx(hw_desc, i - slots_per_op); - prev_hw_desc->next_desc = - (u32) (desc->async_tx.phys + (i << 5)); - } - } - - return j; -} - -static inline void -iop_desc_init_null_xor(struct iop_adma_desc_slot *desc, int src_cnt, - unsigned long flags) -{ - struct iop3xx_desc_aau *hw_desc = desc->hw_desc; - union { - u32 value; - struct iop3xx_aau_desc_ctrl field; - } u_desc_ctrl; - - u_desc_ctrl.value = 0; - switch (src_cnt) { - case 25 ... 32: - u_desc_ctrl.field.blk_ctrl = 0x3; /* use EDCR[2:0] */ - hw_desc->src_edc[AAU_EDCR2_IDX].e_desc_ctrl = 0; - fallthrough; - case 17 ... 24: - if (!u_desc_ctrl.field.blk_ctrl) { - hw_desc->src_edc[AAU_EDCR2_IDX].e_desc_ctrl = 0; - u_desc_ctrl.field.blk_ctrl = 0x3; /* use EDCR[2:0] */ - } - hw_desc->src_edc[AAU_EDCR1_IDX].e_desc_ctrl = 0; - fallthrough; - case 9 ... 16: - if (!u_desc_ctrl.field.blk_ctrl) - u_desc_ctrl.field.blk_ctrl = 0x2; /* use EDCR0 */ - hw_desc->src_edc[AAU_EDCR0_IDX].e_desc_ctrl = 0; - fallthrough; - case 1 ... 8: - if (!u_desc_ctrl.field.blk_ctrl && src_cnt > 4) - u_desc_ctrl.field.blk_ctrl = 0x1; /* use mini-desc */ - } - - u_desc_ctrl.field.dest_write_en = 0; - u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT; - hw_desc->desc_ctrl = u_desc_ctrl.value; -} - -static inline void iop_desc_set_byte_count(struct iop_adma_desc_slot *desc, - struct iop_adma_chan *chan, - u32 byte_count) -{ - union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, }; - - switch (chan->device->id) { - case DMA0_ID: - case DMA1_ID: - hw_desc.dma->byte_count = byte_count; - break; - case AAU_ID: - hw_desc.aau->byte_count = byte_count; - break; - default: - BUG(); - } -} - -static inline void -iop_desc_init_interrupt(struct iop_adma_desc_slot *desc, - struct iop_adma_chan *chan) -{ - union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, }; - - switch (chan->device->id) { - case DMA0_ID: - case DMA1_ID: - iop_desc_init_memcpy(desc, 1); - hw_desc.dma->byte_count = 0; - hw_desc.dma->dest_addr = 0; - hw_desc.dma->src_addr = 0; - break; - case AAU_ID: - iop_desc_init_null_xor(desc, 2, 1); - hw_desc.aau->byte_count = 0; - hw_desc.aau->dest_addr = 0; - hw_desc.aau->src[0] = 0; - hw_desc.aau->src[1] = 0; - break; - default: - BUG(); - } -} - -static inline void -iop_desc_set_zero_sum_byte_count(struct iop_adma_desc_slot *desc, u32 len) -{ - int slots_per_op = desc->slots_per_op; - struct iop3xx_desc_aau *hw_desc = desc->hw_desc, *iter; - int i = 0; - - if (len <= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT) { - hw_desc->byte_count = len; - } else { - do { - iter = iop_hw_desc_slot_idx(hw_desc, i); - iter->byte_count = IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT; - len -= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT; - i += slots_per_op; - } while (len > IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT); - - iter = iop_hw_desc_slot_idx(hw_desc, i); - iter->byte_count = len; - } -} - -static inline void iop_desc_set_dest_addr(struct iop_adma_desc_slot *desc, - struct iop_adma_chan *chan, - dma_addr_t addr) -{ - union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, }; - - switch (chan->device->id) { - case DMA0_ID: - case DMA1_ID: - hw_desc.dma->dest_addr = addr; - break; - case AAU_ID: - hw_desc.aau->dest_addr = addr; - break; - default: - BUG(); - } -} - -static inline void iop_desc_set_memcpy_src_addr(struct iop_adma_desc_slot *desc, - dma_addr_t addr) -{ - struct iop3xx_desc_dma *hw_desc = desc->hw_desc; - hw_desc->src_addr = addr; -} - -static inline void -iop_desc_set_zero_sum_src_addr(struct iop_adma_desc_slot *desc, int src_idx, - dma_addr_t addr) -{ - - struct iop3xx_desc_aau *hw_desc = desc->hw_desc, *iter; - int slot_cnt = desc->slot_cnt, slots_per_op = desc->slots_per_op; - int i; - - for (i = 0; (slot_cnt -= slots_per_op) >= 0; - i += slots_per_op, addr += IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT) { - iter = iop_hw_desc_slot_idx(hw_desc, i); - iop3xx_aau_desc_set_src_addr(iter, src_idx, addr); - } -} - -static inline void iop_desc_set_xor_src_addr(struct iop_adma_desc_slot *desc, - int src_idx, dma_addr_t addr) -{ - - struct iop3xx_desc_aau *hw_desc = desc->hw_desc, *iter; - int slot_cnt = desc->slot_cnt, slots_per_op = desc->slots_per_op; - int i; - - for (i = 0; (slot_cnt -= slots_per_op) >= 0; - i += slots_per_op, addr += IOP_ADMA_XOR_MAX_BYTE_COUNT) { - iter = iop_hw_desc_slot_idx(hw_desc, i); - iop3xx_aau_desc_set_src_addr(iter, src_idx, addr); - } -} - -static inline void iop_desc_set_next_desc(struct iop_adma_desc_slot *desc, - u32 next_desc_addr) -{ - /* hw_desc->next_desc is the same location for all channels */ - union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, }; - - iop_paranoia(hw_desc.dma->next_desc); - hw_desc.dma->next_desc = next_desc_addr; -} - -static inline u32 iop_desc_get_next_desc(struct iop_adma_desc_slot *desc) -{ - /* hw_desc->next_desc is the same location for all channels */ - union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, }; - return hw_desc.dma->next_desc; -} - -static inline void iop_desc_clear_next_desc(struct iop_adma_desc_slot *desc) -{ - /* hw_desc->next_desc is the same location for all channels */ - union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, }; - hw_desc.dma->next_desc = 0; -} - -static inline void iop_desc_set_block_fill_val(struct iop_adma_desc_slot *desc, - u32 val) -{ - struct iop3xx_desc_aau *hw_desc = desc->hw_desc; - hw_desc->src[0] = val; -} - -static inline enum sum_check_flags -iop_desc_get_zero_result(struct iop_adma_desc_slot *desc) -{ - struct iop3xx_desc_aau *hw_desc = desc->hw_desc; - struct iop3xx_aau_desc_ctrl desc_ctrl = hw_desc->desc_ctrl_field; - - iop_paranoia(!(desc_ctrl.tx_complete && desc_ctrl.zero_result_en)); - return desc_ctrl.zero_result_err << SUM_CHECK_P; -} - -static inline void iop_chan_append(struct iop_adma_chan *chan) -{ - u32 dma_chan_ctrl; - - dma_chan_ctrl = __raw_readl(DMA_CCR(chan)); - dma_chan_ctrl |= 0x2; - __raw_writel(dma_chan_ctrl, DMA_CCR(chan)); -} - -static inline u32 iop_chan_get_status(struct iop_adma_chan *chan) -{ - return __raw_readl(DMA_CSR(chan)); -} - -static inline void iop_chan_disable(struct iop_adma_chan *chan) -{ - u32 dma_chan_ctrl = __raw_readl(DMA_CCR(chan)); - dma_chan_ctrl &= ~1; - __raw_writel(dma_chan_ctrl, DMA_CCR(chan)); -} - -static inline void iop_chan_enable(struct iop_adma_chan *chan) -{ - u32 dma_chan_ctrl = __raw_readl(DMA_CCR(chan)); - - dma_chan_ctrl |= 1; - __raw_writel(dma_chan_ctrl, DMA_CCR(chan)); -} - -static inline void iop_adma_device_clear_eot_status(struct iop_adma_chan *chan) -{ - u32 status = __raw_readl(DMA_CSR(chan)); - status &= (1 << 9); - __raw_writel(status, DMA_CSR(chan)); -} - -static inline void iop_adma_device_clear_eoc_status(struct iop_adma_chan *chan) -{ - u32 status = __raw_readl(DMA_CSR(chan)); - status &= (1 << 8); - __raw_writel(status, DMA_CSR(chan)); -} - -static inline void iop_adma_device_clear_err_status(struct iop_adma_chan *chan) -{ - u32 status = __raw_readl(DMA_CSR(chan)); - - switch (chan->device->id) { - case DMA0_ID: - case DMA1_ID: - status &= (1 << 5) | (1 << 3) | (1 << 2) | (1 << 1); - break; - case AAU_ID: - status &= (1 << 5); - break; - default: - BUG(); - } - - __raw_writel(status, DMA_CSR(chan)); -} - -static inline int -iop_is_err_int_parity(unsigned long status, struct iop_adma_chan *chan) -{ - return 0; -} - -static inline int -iop_is_err_mcu_abort(unsigned long status, struct iop_adma_chan *chan) -{ - return 0; -} - -static inline int -iop_is_err_int_tabort(unsigned long status, struct iop_adma_chan *chan) -{ - return 0; -} - -static inline int -iop_is_err_int_mabort(unsigned long status, struct iop_adma_chan *chan) -{ - return test_bit(5, &status); -} - -static inline int -iop_is_err_pci_tabort(unsigned long status, struct iop_adma_chan *chan) -{ - switch (chan->device->id) { - case DMA0_ID: - case DMA1_ID: - return test_bit(2, &status); - default: - return 0; - } -} - -static inline int -iop_is_err_pci_mabort(unsigned long status, struct iop_adma_chan *chan) -{ - switch (chan->device->id) { - case DMA0_ID: - case DMA1_ID: - return test_bit(3, &status); - default: - return 0; - } -} - -static inline int -iop_is_err_split_tx(unsigned long status, struct iop_adma_chan *chan) -{ - switch (chan->device->id) { - case DMA0_ID: - case DMA1_ID: - return test_bit(1, &status); - default: - return 0; - } -} -#endif /* _ADMA_H */ diff --git a/drivers/dma/qcom/gpi.c b/drivers/dma/qcom/gpi.c index 3f56514bbef8..061add832295 100644 --- a/drivers/dma/qcom/gpi.c +++ b/drivers/dma/qcom/gpi.c @@ -2286,9 +2286,14 @@ static int gpi_probe(struct platform_device *pdev) } static const struct of_device_id gpi_of_match[] = { - { .compatible = "qcom,sc7280-gpi-dma", .data = (void *)0x10000 }, { .compatible = "qcom,sdm845-gpi-dma", .data = (void *)0x0 }, { .compatible = "qcom,sm6350-gpi-dma", .data = (void *)0x10000 }, + /* + * Do not grow the list for compatible devices. Instead use + * qcom,sdm845-gpi-dma (for ee_offset = 0x0) or qcom,sm6350-gpi-dma + * (for ee_offset = 0x10000). + */ + { .compatible = "qcom,sc7280-gpi-dma", .data = (void *)0x10000 }, { .compatible = "qcom,sm8150-gpi-dma", .data = (void *)0x0 }, { .compatible = "qcom,sm8250-gpi-dma", .data = (void *)0x0 }, { .compatible = "qcom,sm8350-gpi-dma", .data = (void *)0x10000 }, diff --git a/drivers/dma/s3c24xx-dma.c b/drivers/dma/s3c24xx-dma.c deleted file mode 100644 index a09eeb545f7d..000000000000 --- a/drivers/dma/s3c24xx-dma.c +++ /dev/null @@ -1,1428 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * S3C24XX DMA handling - * - * Copyright (c) 2013 Heiko Stuebner <[email protected]> - * - * based on amba-pl08x.c - * - * Copyright (c) 2006 ARM Ltd. - * Copyright (c) 2010 ST-Ericsson SA - * - * Author: Peter Pearse <[email protected]> - * Author: Linus Walleij <[email protected]> - * - * The DMA controllers in S3C24XX SoCs have a varying number of DMA signals - * that can be routed to any of the 4 to 8 hardware-channels. - * - * Therefore on these DMA controllers the number of channels - * and the number of incoming DMA signals are two totally different things. - * It is usually not possible to theoretically handle all physical signals, - * so a multiplexing scheme with possible denial of use is necessary. - * - * Open items: - * - bursts - */ - -#include <linux/platform_device.h> -#include <linux/types.h> -#include <linux/dmaengine.h> -#include <linux/dma-mapping.h> -#include <linux/interrupt.h> -#include <linux/clk.h> -#include <linux/module.h> -#include <linux/mod_devicetable.h> -#include <linux/slab.h> -#include <linux/platform_data/dma-s3c24xx.h> - -#include "dmaengine.h" -#include "virt-dma.h" - -#define MAX_DMA_CHANNELS 8 - -#define S3C24XX_DISRC 0x00 -#define S3C24XX_DISRCC 0x04 -#define S3C24XX_DISRCC_INC_INCREMENT 0 -#define S3C24XX_DISRCC_INC_FIXED BIT(0) -#define S3C24XX_DISRCC_LOC_AHB 0 -#define S3C24XX_DISRCC_LOC_APB BIT(1) - -#define S3C24XX_DIDST 0x08 -#define S3C24XX_DIDSTC 0x0c -#define S3C24XX_DIDSTC_INC_INCREMENT 0 -#define S3C24XX_DIDSTC_INC_FIXED BIT(0) -#define S3C24XX_DIDSTC_LOC_AHB 0 -#define S3C24XX_DIDSTC_LOC_APB BIT(1) -#define S3C24XX_DIDSTC_INT_TC0 0 -#define S3C24XX_DIDSTC_INT_RELOAD BIT(2) - -#define S3C24XX_DCON 0x10 - -#define S3C24XX_DCON_TC_MASK 0xfffff -#define S3C24XX_DCON_DSZ_BYTE (0 << 20) -#define S3C24XX_DCON_DSZ_HALFWORD (1 << 20) -#define S3C24XX_DCON_DSZ_WORD (2 << 20) -#define S3C24XX_DCON_DSZ_MASK (3 << 20) -#define S3C24XX_DCON_DSZ_SHIFT 20 -#define S3C24XX_DCON_AUTORELOAD 0 -#define S3C24XX_DCON_NORELOAD BIT(22) -#define S3C24XX_DCON_HWTRIG BIT(23) -#define S3C24XX_DCON_HWSRC_SHIFT 24 -#define S3C24XX_DCON_SERV_SINGLE 0 -#define S3C24XX_DCON_SERV_WHOLE BIT(27) -#define S3C24XX_DCON_TSZ_UNIT 0 -#define S3C24XX_DCON_TSZ_BURST4 BIT(28) -#define S3C24XX_DCON_INT BIT(29) -#define S3C24XX_DCON_SYNC_PCLK 0 -#define S3C24XX_DCON_SYNC_HCLK BIT(30) -#define S3C24XX_DCON_DEMAND 0 -#define S3C24XX_DCON_HANDSHAKE BIT(31) - -#define S3C24XX_DSTAT 0x14 -#define S3C24XX_DSTAT_STAT_BUSY BIT(20) -#define S3C24XX_DSTAT_CURRTC_MASK 0xfffff - -#define S3C24XX_DMASKTRIG 0x20 -#define S3C24XX_DMASKTRIG_SWTRIG BIT(0) -#define S3C24XX_DMASKTRIG_ON BIT(1) -#define S3C24XX_DMASKTRIG_STOP BIT(2) - -#define S3C24XX_DMAREQSEL 0x24 -#define S3C24XX_DMAREQSEL_HW BIT(0) - -/* - * S3C2410, S3C2440 and S3C2442 SoCs cannot select any physical channel - * for a DMA source. Instead only specific channels are valid. - * All of these SoCs have 4 physical channels and the number of request - * source bits is 3. Additionally we also need 1 bit to mark the channel - * as valid. - * Therefore we separate the chansel element of the channel data into 4 - * parts of 4 bits each, to hold the information if the channel is valid - * and the hw request source to use. - * - * Example: - * SDI is valid on channels 0, 2 and 3 - with varying hw request sources. - * For it the chansel field would look like - * - * ((BIT(3) | 1) << 3 * 4) | // channel 3, with request source 1 - * ((BIT(3) | 2) << 2 * 4) | // channel 2, with request source 2 - * ((BIT(3) | 2) << 0 * 4) // channel 0, with request source 2 - */ -#define S3C24XX_CHANSEL_WIDTH 4 -#define S3C24XX_CHANSEL_VALID BIT(3) -#define S3C24XX_CHANSEL_REQ_MASK 7 - -/* - * struct soc_data - vendor-specific config parameters for individual SoCs - * @stride: spacing between the registers of each channel - * @has_reqsel: does the controller use the newer requestselection mechanism - * @has_clocks: are controllable dma-clocks present - */ -struct soc_data { - int stride; - bool has_reqsel; - bool has_clocks; -}; - -/* - * enum s3c24xx_dma_chan_state - holds the virtual channel states - * @S3C24XX_DMA_CHAN_IDLE: the channel is idle - * @S3C24XX_DMA_CHAN_RUNNING: the channel has allocated a physical transport - * channel and is running a transfer on it - * @S3C24XX_DMA_CHAN_WAITING: the channel is waiting for a physical transport - * channel to become available (only pertains to memcpy channels) - */ -enum s3c24xx_dma_chan_state { - S3C24XX_DMA_CHAN_IDLE, - S3C24XX_DMA_CHAN_RUNNING, - S3C24XX_DMA_CHAN_WAITING, -}; - -/* - * struct s3c24xx_sg - structure containing data per sg - * @src_addr: src address of sg - * @dst_addr: dst address of sg - * @len: transfer len in bytes - * @node: node for txd's dsg_list - */ -struct s3c24xx_sg { - dma_addr_t src_addr; - dma_addr_t dst_addr; - size_t len; - struct list_head node; -}; - -/* - * struct s3c24xx_txd - wrapper for struct dma_async_tx_descriptor - * @vd: virtual DMA descriptor - * @dsg_list: list of children sg's - * @at: sg currently being transfered - * @width: transfer width - * @disrcc: value for source control register - * @didstc: value for destination control register - * @dcon: base value for dcon register - * @cyclic: indicate cyclic transfer - */ -struct s3c24xx_txd { - struct virt_dma_desc vd; - struct list_head dsg_list; - struct list_head *at; - u8 width; - u32 disrcc; - u32 didstc; - u32 dcon; - bool cyclic; -}; - -struct s3c24xx_dma_chan; - -/* - * struct s3c24xx_dma_phy - holder for the physical channels - * @id: physical index to this channel - * @valid: does the channel have all required elements - * @base: virtual memory base (remapped) for the this channel - * @irq: interrupt for this channel - * @clk: clock for this channel - * @lock: a lock to use when altering an instance of this struct - * @serving: virtual channel currently being served by this physicalchannel - * @host: a pointer to the host (internal use) - */ -struct s3c24xx_dma_phy { - unsigned int id; - bool valid; - void __iomem *base; - int irq; - struct clk *clk; - spinlock_t lock; - struct s3c24xx_dma_chan *serving; - struct s3c24xx_dma_engine *host; -}; - -/* - * struct s3c24xx_dma_chan - this structure wraps a DMA ENGINE channel - * @id: the id of the channel - * @name: name of the channel - * @vc: wrapped virtual channel - * @phy: the physical channel utilized by this channel, if there is one - * @runtime_addr: address for RX/TX according to the runtime config - * @at: active transaction on this channel - * @lock: a lock for this channel data - * @host: a pointer to the host (internal use) - * @state: whether the channel is idle, running etc - * @slave: whether this channel is a device (slave) or for memcpy - */ -struct s3c24xx_dma_chan { - int id; - const char *name; - struct virt_dma_chan vc; - struct s3c24xx_dma_phy *phy; - struct dma_slave_config cfg; - struct s3c24xx_txd *at; - struct s3c24xx_dma_engine *host; - enum s3c24xx_dma_chan_state state; - bool slave; -}; - -/* - * struct s3c24xx_dma_engine - the local state holder for the S3C24XX - * @pdev: the corresponding platform device - * @pdata: platform data passed in from the platform/machine - * @base: virtual memory base (remapped) - * @slave: slave engine for this instance - * @memcpy: memcpy engine for this instance - * @phy_chans: array of data for the physical channels - */ -struct s3c24xx_dma_engine { - struct platform_device *pdev; - const struct s3c24xx_dma_platdata *pdata; - struct soc_data *sdata; - void __iomem *base; - struct dma_device slave; - struct dma_device memcpy; - struct s3c24xx_dma_phy *phy_chans; -}; - -/* - * Physical channel handling - */ - -/* - * Check whether a certain channel is busy or not. - */ -static int s3c24xx_dma_phy_busy(struct s3c24xx_dma_phy *phy) -{ - unsigned int val = readl(phy->base + S3C24XX_DSTAT); - return val & S3C24XX_DSTAT_STAT_BUSY; -} - -static bool s3c24xx_dma_phy_valid(struct s3c24xx_dma_chan *s3cchan, - struct s3c24xx_dma_phy *phy) -{ - struct s3c24xx_dma_engine *s3cdma = s3cchan->host; - const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata; - struct s3c24xx_dma_channel *cdata = &pdata->channels[s3cchan->id]; - int phyvalid; - - /* every phy is valid for memcopy channels */ - if (!s3cchan->slave) - return true; - - /* On newer variants all phys can be used for all virtual channels */ - if (s3cdma->sdata->has_reqsel) - return true; - - phyvalid = (cdata->chansel >> (phy->id * S3C24XX_CHANSEL_WIDTH)); - return (phyvalid & S3C24XX_CHANSEL_VALID) ? true : false; -} - -/* - * Allocate a physical channel for a virtual channel - * - * Try to locate a physical channel to be used for this transfer. If all - * are taken return NULL and the requester will have to cope by using - * some fallback PIO mode or retrying later. - */ -static -struct s3c24xx_dma_phy *s3c24xx_dma_get_phy(struct s3c24xx_dma_chan *s3cchan) -{ - struct s3c24xx_dma_engine *s3cdma = s3cchan->host; - struct s3c24xx_dma_phy *phy = NULL; - unsigned long flags; - int i; - int ret; - - for (i = 0; i < s3cdma->pdata->num_phy_channels; i++) { - phy = &s3cdma->phy_chans[i]; - - if (!phy->valid) - continue; - - if (!s3c24xx_dma_phy_valid(s3cchan, phy)) - continue; - - spin_lock_irqsave(&phy->lock, flags); - - if (!phy->serving) { - phy->serving = s3cchan; - spin_unlock_irqrestore(&phy->lock, flags); - break; - } - - spin_unlock_irqrestore(&phy->lock, flags); - } - - /* No physical channel available, cope with it */ - if (i == s3cdma->pdata->num_phy_channels) { - dev_warn(&s3cdma->pdev->dev, "no phy channel available\n"); - return NULL; - } - - /* start the phy clock */ - if (s3cdma->sdata->has_clocks) { - ret = clk_enable(phy->clk); - if (ret) { - dev_err(&s3cdma->pdev->dev, "could not enable clock for channel %d, err %d\n", - phy->id, ret); - phy->serving = NULL; - return NULL; - } - } - - return phy; -} - -/* - * Mark the physical channel as free. - * - * This drops the link between the physical and virtual channel. - */ -static inline void s3c24xx_dma_put_phy(struct s3c24xx_dma_phy *phy) -{ - struct s3c24xx_dma_engine *s3cdma = phy->host; - - if (s3cdma->sdata->has_clocks) - clk_disable(phy->clk); - - phy->serving = NULL; -} - -/* - * Stops the channel by writing the stop bit. - * This should not be used for an on-going transfer, but as a method of - * shutting down a channel (eg, when it's no longer used) or terminating a - * transfer. - */ -static void s3c24xx_dma_terminate_phy(struct s3c24xx_dma_phy *phy) -{ - writel(S3C24XX_DMASKTRIG_STOP, phy->base + S3C24XX_DMASKTRIG); -} - -/* - * Virtual channel handling - */ - -static inline -struct s3c24xx_dma_chan *to_s3c24xx_dma_chan(struct dma_chan *chan) -{ - return container_of(chan, struct s3c24xx_dma_chan, vc.chan); -} - -static u32 s3c24xx_dma_getbytes_chan(struct s3c24xx_dma_chan *s3cchan) -{ - struct s3c24xx_dma_phy *phy = s3cchan->phy; - struct s3c24xx_txd *txd = s3cchan->at; - u32 tc = readl(phy->base + S3C24XX_DSTAT) & S3C24XX_DSTAT_CURRTC_MASK; - - return tc * txd->width; -} - -static int s3c24xx_dma_set_runtime_config(struct dma_chan *chan, - struct dma_slave_config *config) -{ - struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan); - unsigned long flags; - int ret = 0; - - /* Reject definitely invalid configurations */ - if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES || - config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES) - return -EINVAL; - - spin_lock_irqsave(&s3cchan->vc.lock, flags); - - if (!s3cchan->slave) { - ret = -EINVAL; - goto out; - } - - s3cchan->cfg = *config; - -out: - spin_unlock_irqrestore(&s3cchan->vc.lock, flags); - return ret; -} - -/* - * Transfer handling - */ - -static inline -struct s3c24xx_txd *to_s3c24xx_txd(struct dma_async_tx_descriptor *tx) -{ - return container_of(tx, struct s3c24xx_txd, vd.tx); -} - -static struct s3c24xx_txd *s3c24xx_dma_get_txd(void) -{ - struct s3c24xx_txd *txd = kzalloc(sizeof(*txd), GFP_NOWAIT); - - if (txd) { - INIT_LIST_HEAD(&txd->dsg_list); - txd->dcon = S3C24XX_DCON_INT | S3C24XX_DCON_NORELOAD; - } - - return txd; -} - -static void s3c24xx_dma_free_txd(struct s3c24xx_txd *txd) -{ - struct s3c24xx_sg *dsg, *_dsg; - - list_for_each_entry_safe(dsg, _dsg, &txd->dsg_list, node) { - list_del(&dsg->node); - kfree(dsg); - } - - kfree(txd); -} - -static void s3c24xx_dma_start_next_sg(struct s3c24xx_dma_chan *s3cchan, - struct s3c24xx_txd *txd) -{ - struct s3c24xx_dma_engine *s3cdma = s3cchan->host; - struct s3c24xx_dma_phy *phy = s3cchan->phy; - const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata; - struct s3c24xx_sg *dsg = list_entry(txd->at, struct s3c24xx_sg, node); - u32 dcon = txd->dcon; - u32 val; - - /* transfer-size and -count from len and width */ - switch (txd->width) { - case 1: - dcon |= S3C24XX_DCON_DSZ_BYTE | dsg->len; - break; - case 2: - dcon |= S3C24XX_DCON_DSZ_HALFWORD | (dsg->len / 2); - break; - case 4: - dcon |= S3C24XX_DCON_DSZ_WORD | (dsg->len / 4); - break; - } - - if (s3cchan->slave) { - struct s3c24xx_dma_channel *cdata = - &pdata->channels[s3cchan->id]; - - if (s3cdma->sdata->has_reqsel) { - writel_relaxed((cdata->chansel << 1) | - S3C24XX_DMAREQSEL_HW, - phy->base + S3C24XX_DMAREQSEL); - } else { - int csel = cdata->chansel >> (phy->id * - S3C24XX_CHANSEL_WIDTH); - - csel &= S3C24XX_CHANSEL_REQ_MASK; - dcon |= csel << S3C24XX_DCON_HWSRC_SHIFT; - dcon |= S3C24XX_DCON_HWTRIG; - } - } else { - if (s3cdma->sdata->has_reqsel) - writel_relaxed(0, phy->base + S3C24XX_DMAREQSEL); - } - - writel_relaxed(dsg->src_addr, phy->base + S3C24XX_DISRC); - writel_relaxed(txd->disrcc, phy->base + S3C24XX_DISRCC); - writel_relaxed(dsg->dst_addr, phy->base + S3C24XX_DIDST); - writel_relaxed(txd->didstc, phy->base + S3C24XX_DIDSTC); - writel_relaxed(dcon, phy->base + S3C24XX_DCON); - - val = readl_relaxed(phy->base + S3C24XX_DMASKTRIG); - val &= ~S3C24XX_DMASKTRIG_STOP; - val |= S3C24XX_DMASKTRIG_ON; - - /* trigger the dma operation for memcpy transfers */ - if (!s3cchan->slave) - val |= S3C24XX_DMASKTRIG_SWTRIG; - - writel(val, phy->base + S3C24XX_DMASKTRIG); -} - -/* - * Set the initial DMA register values and start first sg. - */ -static void s3c24xx_dma_start_next_txd(struct s3c24xx_dma_chan *s3cchan) -{ - struct s3c24xx_dma_phy *phy = s3cchan->phy; - struct virt_dma_desc *vd = vchan_next_desc(&s3cchan->vc); - struct s3c24xx_txd *txd = to_s3c24xx_txd(&vd->tx); - - list_del(&txd->vd.node); - - s3cchan->at = txd; - - /* Wait for channel inactive */ - while (s3c24xx_dma_phy_busy(phy)) - cpu_relax(); - - /* point to the first element of the sg list */ - txd->at = txd->dsg_list.next; - s3c24xx_dma_start_next_sg(s3cchan, txd); -} - -/* - * Try to allocate a physical channel. When successful, assign it to - * this virtual channel, and initiate the next descriptor. The - * virtual channel lock must be held at this point. - */ -static void s3c24xx_dma_phy_alloc_and_start(struct s3c24xx_dma_chan *s3cchan) -{ - struct s3c24xx_dma_engine *s3cdma = s3cchan->host; - struct s3c24xx_dma_phy *phy; - - phy = s3c24xx_dma_get_phy(s3cchan); - if (!phy) { - dev_dbg(&s3cdma->pdev->dev, "no physical channel available for xfer on %s\n", - s3cchan->name); - s3cchan->state = S3C24XX_DMA_CHAN_WAITING; - return; - } - - dev_dbg(&s3cdma->pdev->dev, "allocated physical channel %d for xfer on %s\n", - phy->id, s3cchan->name); - - s3cchan->phy = phy; - s3cchan->state = S3C24XX_DMA_CHAN_RUNNING; - - s3c24xx_dma_start_next_txd(s3cchan); -} - -static void s3c24xx_dma_phy_reassign_start(struct s3c24xx_dma_phy *phy, - struct s3c24xx_dma_chan *s3cchan) -{ - struct s3c24xx_dma_engine *s3cdma = s3cchan->host; - - dev_dbg(&s3cdma->pdev->dev, "reassigned physical channel %d for xfer on %s\n", - phy->id, s3cchan->name); - - /* - * We do this without taking the lock; we're really only concerned - * about whether this pointer is NULL or not, and we're guaranteed - * that this will only be called when it _already_ is non-NULL. - */ - phy->serving = s3cchan; - s3cchan->phy = phy; - s3cchan->state = S3C24XX_DMA_CHAN_RUNNING; - s3c24xx_dma_start_next_txd(s3cchan); -} - -/* - * Free a physical DMA channel, potentially reallocating it to another - * virtual channel if we have any pending. - */ -static void s3c24xx_dma_phy_free(struct s3c24xx_dma_chan *s3cchan) -{ - struct s3c24xx_dma_engine *s3cdma = s3cchan->host; - struct s3c24xx_dma_chan *p, *next; - -retry: - next = NULL; - - /* Find a waiting virtual channel for the next transfer. */ - list_for_each_entry(p, &s3cdma->memcpy.channels, vc.chan.device_node) - if (p->state == S3C24XX_DMA_CHAN_WAITING) { - next = p; - break; - } - - if (!next) { - list_for_each_entry(p, &s3cdma->slave.channels, - vc.chan.device_node) - if (p->state == S3C24XX_DMA_CHAN_WAITING && - s3c24xx_dma_phy_valid(p, s3cchan->phy)) { - next = p; - break; - } - } - - /* Ensure that the physical channel is stopped */ - s3c24xx_dma_terminate_phy(s3cchan->phy); - - if (next) { - bool success; - - /* - * Eww. We know this isn't going to deadlock - * but lockdep probably doesn't. - */ - spin_lock(&next->vc.lock); - /* Re-check the state now that we have the lock */ - success = next->state == S3C24XX_DMA_CHAN_WAITING; - if (success) - s3c24xx_dma_phy_reassign_start(s3cchan->phy, next); - spin_unlock(&next->vc.lock); - - /* If the state changed, try to find another channel */ - if (!success) - goto retry; - } else { - /* No more jobs, so free up the physical channel */ - s3c24xx_dma_put_phy(s3cchan->phy); - } - - s3cchan->phy = NULL; - s3cchan->state = S3C24XX_DMA_CHAN_IDLE; -} - -static void s3c24xx_dma_desc_free(struct virt_dma_desc *vd) -{ - struct s3c24xx_txd *txd = to_s3c24xx_txd(&vd->tx); - struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(vd->tx.chan); - - if (!s3cchan->slave) - dma_descriptor_unmap(&vd->tx); - - s3c24xx_dma_free_txd(txd); -} - -static irqreturn_t s3c24xx_dma_irq(int irq, void *data) -{ - struct s3c24xx_dma_phy *phy = data; - struct s3c24xx_dma_chan *s3cchan = phy->serving; - struct s3c24xx_txd *txd; - - dev_dbg(&phy->host->pdev->dev, "interrupt on channel %d\n", phy->id); - - /* - * Interrupts happen to notify the completion of a transfer and the - * channel should have moved into its stop state already on its own. - * Therefore interrupts on channels not bound to a virtual channel - * should never happen. Nevertheless send a terminate command to the - * channel if the unlikely case happens. - */ - if (unlikely(!s3cchan)) { - dev_err(&phy->host->pdev->dev, "interrupt on unused channel %d\n", - phy->id); - - s3c24xx_dma_terminate_phy(phy); - - return IRQ_HANDLED; - } - - spin_lock(&s3cchan->vc.lock); - txd = s3cchan->at; - if (txd) { - /* when more sg's are in this txd, start the next one */ - if (!list_is_last(txd->at, &txd->dsg_list)) { - txd->at = txd->at->next; - if (txd->cyclic) - vchan_cyclic_callback(&txd->vd); - s3c24xx_dma_start_next_sg(s3cchan, txd); - } else if (!txd->cyclic) { - s3cchan->at = NULL; - vchan_cookie_complete(&txd->vd); - - /* - * And start the next descriptor (if any), - * otherwise free this channel. - */ - if (vchan_next_desc(&s3cchan->vc)) - s3c24xx_dma_start_next_txd(s3cchan); - else - s3c24xx_dma_phy_free(s3cchan); - } else { - vchan_cyclic_callback(&txd->vd); - - /* Cyclic: reset at beginning */ - txd->at = txd->dsg_list.next; - s3c24xx_dma_start_next_sg(s3cchan, txd); - } - } - spin_unlock(&s3cchan->vc.lock); - - return IRQ_HANDLED; -} - -/* - * The DMA ENGINE API - */ - -static int s3c24xx_dma_terminate_all(struct dma_chan *chan) -{ - struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan); - struct s3c24xx_dma_engine *s3cdma = s3cchan->host; - LIST_HEAD(head); - unsigned long flags; - int ret; - - spin_lock_irqsave(&s3cchan->vc.lock, flags); - - if (!s3cchan->phy && !s3cchan->at) { - dev_err(&s3cdma->pdev->dev, "trying to terminate already stopped channel %d\n", - s3cchan->id); - ret = -EINVAL; - goto unlock; - } - - s3cchan->state = S3C24XX_DMA_CHAN_IDLE; - - /* Mark physical channel as free */ - if (s3cchan->phy) - s3c24xx_dma_phy_free(s3cchan); - - /* Dequeue current job */ - if (s3cchan->at) { - vchan_terminate_vdesc(&s3cchan->at->vd); - s3cchan->at = NULL; - } - - /* Dequeue jobs not yet fired as well */ - - vchan_get_all_descriptors(&s3cchan->vc, &head); - - spin_unlock_irqrestore(&s3cchan->vc.lock, flags); - - vchan_dma_desc_free_list(&s3cchan->vc, &head); - - return 0; - -unlock: - spin_unlock_irqrestore(&s3cchan->vc.lock, flags); - - return ret; -} - -static void s3c24xx_dma_synchronize(struct dma_chan *chan) -{ - struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan); - - vchan_synchronize(&s3cchan->vc); -} - -static void s3c24xx_dma_free_chan_resources(struct dma_chan *chan) -{ - /* Ensure all queued descriptors are freed */ - vchan_free_chan_resources(to_virt_chan(chan)); -} - -static enum dma_status s3c24xx_dma_tx_status(struct dma_chan *chan, - dma_cookie_t cookie, struct dma_tx_state *txstate) -{ - struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan); - struct s3c24xx_txd *txd; - struct s3c24xx_sg *dsg; - struct virt_dma_desc *vd; - unsigned long flags; - enum dma_status ret; - size_t bytes = 0; - - spin_lock_irqsave(&s3cchan->vc.lock, flags); - ret = dma_cookie_status(chan, cookie, txstate); - - /* - * There's no point calculating the residue if there's - * no txstate to store the value. - */ - if (ret == DMA_COMPLETE || !txstate) { - spin_unlock_irqrestore(&s3cchan->vc.lock, flags); - return ret; - } - - vd = vchan_find_desc(&s3cchan->vc, cookie); - if (vd) { - /* On the issued list, so hasn't been processed yet */ - txd = to_s3c24xx_txd(&vd->tx); - - list_for_each_entry(dsg, &txd->dsg_list, node) - bytes += dsg->len; - } else { - /* - * Currently running, so sum over the pending sg's and - * the currently active one. - */ - txd = s3cchan->at; - - dsg = list_entry(txd->at, struct s3c24xx_sg, node); - list_for_each_entry_from(dsg, &txd->dsg_list, node) - bytes += dsg->len; - - bytes += s3c24xx_dma_getbytes_chan(s3cchan); - } - spin_unlock_irqrestore(&s3cchan->vc.lock, flags); - - /* - * This cookie not complete yet - * Get number of bytes left in the active transactions and queue - */ - dma_set_residue(txstate, bytes); - - /* Whether waiting or running, we're in progress */ - return ret; -} - -/* - * Initialize a descriptor to be used by memcpy submit - */ -static struct dma_async_tx_descriptor *s3c24xx_dma_prep_memcpy( - struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, - size_t len, unsigned long flags) -{ - struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan); - struct s3c24xx_dma_engine *s3cdma = s3cchan->host; - struct s3c24xx_txd *txd; - struct s3c24xx_sg *dsg; - int src_mod, dest_mod; - - dev_dbg(&s3cdma->pdev->dev, "prepare memcpy of %zu bytes from %s\n", - len, s3cchan->name); - - if ((len & S3C24XX_DCON_TC_MASK) != len) { - dev_err(&s3cdma->pdev->dev, "memcpy size %zu to large\n", len); - return NULL; - } - - txd = s3c24xx_dma_get_txd(); - if (!txd) - return NULL; - - dsg = kzalloc(sizeof(*dsg), GFP_NOWAIT); - if (!dsg) { - s3c24xx_dma_free_txd(txd); - return NULL; - } - list_add_tail(&dsg->node, &txd->dsg_list); - - dsg->src_addr = src; - dsg->dst_addr = dest; - dsg->len = len; - - /* - * Determine a suitable transfer width. - * The DMA controller cannot fetch/store information which is not - * naturally aligned on the bus, i.e., a 4 byte fetch must start at - * an address divisible by 4 - more generally addr % width must be 0. - */ - src_mod = src % 4; - dest_mod = dest % 4; - switch (len % 4) { - case 0: - txd->width = (src_mod == 0 && dest_mod == 0) ? 4 : 1; - break; - case 2: - txd->width = ((src_mod == 2 || src_mod == 0) && - (dest_mod == 2 || dest_mod == 0)) ? 2 : 1; - break; - default: - txd->width = 1; - break; - } - - txd->disrcc = S3C24XX_DISRCC_LOC_AHB | S3C24XX_DISRCC_INC_INCREMENT; - txd->didstc = S3C24XX_DIDSTC_LOC_AHB | S3C24XX_DIDSTC_INC_INCREMENT; - txd->dcon |= S3C24XX_DCON_DEMAND | S3C24XX_DCON_SYNC_HCLK | - S3C24XX_DCON_SERV_WHOLE; - - return vchan_tx_prep(&s3cchan->vc, &txd->vd, flags); -} - -static struct dma_async_tx_descriptor *s3c24xx_dma_prep_dma_cyclic( - struct dma_chan *chan, dma_addr_t addr, size_t size, size_t period, - enum dma_transfer_direction direction, unsigned long flags) -{ - struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan); - struct s3c24xx_dma_engine *s3cdma = s3cchan->host; - const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata; - struct s3c24xx_dma_channel *cdata = &pdata->channels[s3cchan->id]; - struct s3c24xx_txd *txd; - struct s3c24xx_sg *dsg; - unsigned sg_len; - dma_addr_t slave_addr; - u32 hwcfg = 0; - int i; - - dev_dbg(&s3cdma->pdev->dev, - "prepare cyclic transaction of %zu bytes with period %zu from %s\n", - size, period, s3cchan->name); - - if (!is_slave_direction(direction)) { - dev_err(&s3cdma->pdev->dev, - "direction %d unsupported\n", direction); - return NULL; - } - - txd = s3c24xx_dma_get_txd(); - if (!txd) - return NULL; - - txd->cyclic = 1; - - if (cdata->handshake) - txd->dcon |= S3C24XX_DCON_HANDSHAKE; - - switch (cdata->bus) { - case S3C24XX_DMA_APB: - txd->dcon |= S3C24XX_DCON_SYNC_PCLK; - hwcfg |= S3C24XX_DISRCC_LOC_APB; - break; - case S3C24XX_DMA_AHB: - txd->dcon |= S3C24XX_DCON_SYNC_HCLK; - hwcfg |= S3C24XX_DISRCC_LOC_AHB; - break; - } - - /* - * Always assume our peripheral desintation is a fixed - * address in memory. - */ - hwcfg |= S3C24XX_DISRCC_INC_FIXED; - - /* - * Individual dma operations are requested by the slave, - * so serve only single atomic operations (S3C24XX_DCON_SERV_SINGLE). - */ - txd->dcon |= S3C24XX_DCON_SERV_SINGLE; - - if (direction == DMA_MEM_TO_DEV) { - txd->disrcc = S3C24XX_DISRCC_LOC_AHB | - S3C24XX_DISRCC_INC_INCREMENT; - txd->didstc = hwcfg; - slave_addr = s3cchan->cfg.dst_addr; - txd->width = s3cchan->cfg.dst_addr_width; - } else { - txd->disrcc = hwcfg; - txd->didstc = S3C24XX_DIDSTC_LOC_AHB | - S3C24XX_DIDSTC_INC_INCREMENT; - slave_addr = s3cchan->cfg.src_addr; - txd->width = s3cchan->cfg.src_addr_width; - } - - sg_len = size / period; - - for (i = 0; i < sg_len; i++) { - dsg = kzalloc(sizeof(*dsg), GFP_NOWAIT); - if (!dsg) { - s3c24xx_dma_free_txd(txd); - return NULL; - } - list_add_tail(&dsg->node, &txd->dsg_list); - - dsg->len = period; - /* Check last period length */ - if (i == sg_len - 1) - dsg->len = size - period * i; - if (direction == DMA_MEM_TO_DEV) { - dsg->src_addr = addr + period * i; - dsg->dst_addr = slave_addr; - } else { /* DMA_DEV_TO_MEM */ - dsg->src_addr = slave_addr; - dsg->dst_addr = addr + period * i; - } - } - - return vchan_tx_prep(&s3cchan->vc, &txd->vd, flags); -} - -static struct dma_async_tx_descriptor *s3c24xx_dma_prep_slave_sg( - struct dma_chan *chan, struct scatterlist *sgl, - unsigned int sg_len, enum dma_transfer_direction direction, - unsigned long flags, void *context) -{ - struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan); - struct s3c24xx_dma_engine *s3cdma = s3cchan->host; - const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata; - struct s3c24xx_dma_channel *cdata = &pdata->channels[s3cchan->id]; - struct s3c24xx_txd *txd; - struct s3c24xx_sg *dsg; - struct scatterlist *sg; - dma_addr_t slave_addr; - u32 hwcfg = 0; - int tmp; - - dev_dbg(&s3cdma->pdev->dev, "prepare transaction of %d bytes from %s\n", - sg_dma_len(sgl), s3cchan->name); - - txd = s3c24xx_dma_get_txd(); - if (!txd) - return NULL; - - if (cdata->handshake) - txd->dcon |= S3C24XX_DCON_HANDSHAKE; - - switch (cdata->bus) { - case S3C24XX_DMA_APB: - txd->dcon |= S3C24XX_DCON_SYNC_PCLK; - hwcfg |= S3C24XX_DISRCC_LOC_APB; - break; - case S3C24XX_DMA_AHB: - txd->dcon |= S3C24XX_DCON_SYNC_HCLK; - hwcfg |= S3C24XX_DISRCC_LOC_AHB; - break; - } - - /* - * Always assume our peripheral desintation is a fixed - * address in memory. - */ - hwcfg |= S3C24XX_DISRCC_INC_FIXED; - - /* - * Individual dma operations are requested by the slave, - * so serve only single atomic operations (S3C24XX_DCON_SERV_SINGLE). - */ - txd->dcon |= S3C24XX_DCON_SERV_SINGLE; - - if (direction == DMA_MEM_TO_DEV) { - txd->disrcc = S3C24XX_DISRCC_LOC_AHB | - S3C24XX_DISRCC_INC_INCREMENT; - txd->didstc = hwcfg; - slave_addr = s3cchan->cfg.dst_addr; - txd->width = s3cchan->cfg.dst_addr_width; - } else if (direction == DMA_DEV_TO_MEM) { - txd->disrcc = hwcfg; - txd->didstc = S3C24XX_DIDSTC_LOC_AHB | - S3C24XX_DIDSTC_INC_INCREMENT; - slave_addr = s3cchan->cfg.src_addr; - txd->width = s3cchan->cfg.src_addr_width; - } else { - s3c24xx_dma_free_txd(txd); - dev_err(&s3cdma->pdev->dev, - "direction %d unsupported\n", direction); - return NULL; - } - - for_each_sg(sgl, sg, sg_len, tmp) { - dsg = kzalloc(sizeof(*dsg), GFP_NOWAIT); - if (!dsg) { - s3c24xx_dma_free_txd(txd); - return NULL; - } - list_add_tail(&dsg->node, &txd->dsg_list); - - dsg->len = sg_dma_len(sg); - if (direction == DMA_MEM_TO_DEV) { - dsg->src_addr = sg_dma_address(sg); - dsg->dst_addr = slave_addr; - } else { /* DMA_DEV_TO_MEM */ - dsg->src_addr = slave_addr; - dsg->dst_addr = sg_dma_address(sg); - } - } - - return vchan_tx_prep(&s3cchan->vc, &txd->vd, flags); -} - -/* - * Slave transactions callback to the slave device to allow - * synchronization of slave DMA signals with the DMAC enable - */ -static void s3c24xx_dma_issue_pending(struct dma_chan *chan) -{ - struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan); - unsigned long flags; - - spin_lock_irqsave(&s3cchan->vc.lock, flags); - if (vchan_issue_pending(&s3cchan->vc)) { - if (!s3cchan->phy && s3cchan->state != S3C24XX_DMA_CHAN_WAITING) - s3c24xx_dma_phy_alloc_and_start(s3cchan); - } - spin_unlock_irqrestore(&s3cchan->vc.lock, flags); -} - -/* - * Bringup and teardown - */ - -/* - * Initialise the DMAC memcpy/slave channels. - * Make a local wrapper to hold required data - */ -static int s3c24xx_dma_init_virtual_channels(struct s3c24xx_dma_engine *s3cdma, - struct dma_device *dmadev, unsigned int channels, bool slave) -{ - struct s3c24xx_dma_chan *chan; - int i; - - INIT_LIST_HEAD(&dmadev->channels); - - /* - * Register as many memcpy as we have physical channels, - * we won't always be able to use all but the code will have - * to cope with that situation. - */ - for (i = 0; i < channels; i++) { - chan = devm_kzalloc(dmadev->dev, sizeof(*chan), GFP_KERNEL); - if (!chan) - return -ENOMEM; - - chan->id = i; - chan->host = s3cdma; - chan->state = S3C24XX_DMA_CHAN_IDLE; - - if (slave) { - chan->slave = true; - chan->name = kasprintf(GFP_KERNEL, "slave%d", i); - if (!chan->name) - return -ENOMEM; - } else { - chan->name = kasprintf(GFP_KERNEL, "memcpy%d", i); - if (!chan->name) - return -ENOMEM; - } - dev_dbg(dmadev->dev, - "initialize virtual channel \"%s\"\n", - chan->name); - - chan->vc.desc_free = s3c24xx_dma_desc_free; - vchan_init(&chan->vc, dmadev); - } - dev_info(dmadev->dev, "initialized %d virtual %s channels\n", - i, slave ? "slave" : "memcpy"); - return i; -} - -static void s3c24xx_dma_free_virtual_channels(struct dma_device *dmadev) -{ - struct s3c24xx_dma_chan *chan = NULL; - struct s3c24xx_dma_chan *next; - - list_for_each_entry_safe(chan, - next, &dmadev->channels, vc.chan.device_node) { - list_del(&chan->vc.chan.device_node); - tasklet_kill(&chan->vc.task); - } -} - -/* s3c2410, s3c2440 and s3c2442 have a 0x40 stride without separate clocks */ -static struct soc_data soc_s3c2410 = { - .stride = 0x40, - .has_reqsel = false, - .has_clocks = false, -}; - -/* s3c2412 and s3c2413 have a 0x40 stride and dmareqsel mechanism */ -static struct soc_data soc_s3c2412 = { - .stride = 0x40, - .has_reqsel = true, - .has_clocks = true, -}; - -/* s3c2443 and following have a 0x100 stride and dmareqsel mechanism */ -static struct soc_data soc_s3c2443 = { - .stride = 0x100, - .has_reqsel = true, - .has_clocks = true, -}; - -static const struct platform_device_id s3c24xx_dma_driver_ids[] = { - { - .name = "s3c2410-dma", - .driver_data = (kernel_ulong_t)&soc_s3c2410, - }, { - .name = "s3c2412-dma", - .driver_data = (kernel_ulong_t)&soc_s3c2412, - }, { - .name = "s3c2443-dma", - .driver_data = (kernel_ulong_t)&soc_s3c2443, - }, - { }, -}; - -static struct soc_data *s3c24xx_dma_get_soc_data(struct platform_device *pdev) -{ - return (struct soc_data *) - platform_get_device_id(pdev)->driver_data; -} - -static int s3c24xx_dma_probe(struct platform_device *pdev) -{ - const struct s3c24xx_dma_platdata *pdata = dev_get_platdata(&pdev->dev); - struct s3c24xx_dma_engine *s3cdma; - struct soc_data *sdata; - struct resource *res; - int ret; - int i; - - if (!pdata) { - dev_err(&pdev->dev, "platform data missing\n"); - return -ENODEV; - } - - /* Basic sanity check */ - if (pdata->num_phy_channels > MAX_DMA_CHANNELS) { - dev_err(&pdev->dev, "too many dma channels %d, max %d\n", - pdata->num_phy_channels, MAX_DMA_CHANNELS); - return -EINVAL; - } - - sdata = s3c24xx_dma_get_soc_data(pdev); - if (!sdata) - return -EINVAL; - - s3cdma = devm_kzalloc(&pdev->dev, sizeof(*s3cdma), GFP_KERNEL); - if (!s3cdma) - return -ENOMEM; - - s3cdma->pdev = pdev; - s3cdma->pdata = pdata; - s3cdma->sdata = sdata; - - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - s3cdma->base = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(s3cdma->base)) - return PTR_ERR(s3cdma->base); - - s3cdma->phy_chans = devm_kcalloc(&pdev->dev, - pdata->num_phy_channels, - sizeof(struct s3c24xx_dma_phy), - GFP_KERNEL); - if (!s3cdma->phy_chans) - return -ENOMEM; - - /* acquire irqs and clocks for all physical channels */ - for (i = 0; i < pdata->num_phy_channels; i++) { - struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i]; - char clk_name[6]; - - phy->id = i; - phy->base = s3cdma->base + (i * sdata->stride); - phy->host = s3cdma; - - phy->irq = platform_get_irq(pdev, i); - if (phy->irq < 0) - continue; - - ret = devm_request_irq(&pdev->dev, phy->irq, s3c24xx_dma_irq, - 0, pdev->name, phy); - if (ret) { - dev_err(&pdev->dev, "Unable to request irq for channel %d, error %d\n", - i, ret); - continue; - } - - if (sdata->has_clocks) { - sprintf(clk_name, "dma.%d", i); - phy->clk = devm_clk_get(&pdev->dev, clk_name); - if (IS_ERR(phy->clk) && sdata->has_clocks) { - dev_err(&pdev->dev, "unable to acquire clock for channel %d, error %lu\n", - i, PTR_ERR(phy->clk)); - continue; - } - - ret = clk_prepare(phy->clk); - if (ret) { - dev_err(&pdev->dev, "clock for phy %d failed, error %d\n", - i, ret); - continue; - } - } - - spin_lock_init(&phy->lock); - phy->valid = true; - - dev_dbg(&pdev->dev, "physical channel %d is %s\n", - i, s3c24xx_dma_phy_busy(phy) ? "BUSY" : "FREE"); - } - - /* Initialize memcpy engine */ - dma_cap_set(DMA_MEMCPY, s3cdma->memcpy.cap_mask); - dma_cap_set(DMA_PRIVATE, s3cdma->memcpy.cap_mask); - s3cdma->memcpy.dev = &pdev->dev; - s3cdma->memcpy.device_free_chan_resources = - s3c24xx_dma_free_chan_resources; - s3cdma->memcpy.device_prep_dma_memcpy = s3c24xx_dma_prep_memcpy; - s3cdma->memcpy.device_tx_status = s3c24xx_dma_tx_status; - s3cdma->memcpy.device_issue_pending = s3c24xx_dma_issue_pending; - s3cdma->memcpy.device_config = s3c24xx_dma_set_runtime_config; - s3cdma->memcpy.device_terminate_all = s3c24xx_dma_terminate_all; - s3cdma->memcpy.device_synchronize = s3c24xx_dma_synchronize; - - /* Initialize slave engine for SoC internal dedicated peripherals */ - dma_cap_set(DMA_SLAVE, s3cdma->slave.cap_mask); - dma_cap_set(DMA_CYCLIC, s3cdma->slave.cap_mask); - dma_cap_set(DMA_PRIVATE, s3cdma->slave.cap_mask); - s3cdma->slave.dev = &pdev->dev; - s3cdma->slave.device_free_chan_resources = - s3c24xx_dma_free_chan_resources; - s3cdma->slave.device_tx_status = s3c24xx_dma_tx_status; - s3cdma->slave.device_issue_pending = s3c24xx_dma_issue_pending; - s3cdma->slave.device_prep_slave_sg = s3c24xx_dma_prep_slave_sg; - s3cdma->slave.device_prep_dma_cyclic = s3c24xx_dma_prep_dma_cyclic; - s3cdma->slave.device_config = s3c24xx_dma_set_runtime_config; - s3cdma->slave.device_terminate_all = s3c24xx_dma_terminate_all; - s3cdma->slave.device_synchronize = s3c24xx_dma_synchronize; - s3cdma->slave.filter.map = pdata->slave_map; - s3cdma->slave.filter.mapcnt = pdata->slavecnt; - s3cdma->slave.filter.fn = s3c24xx_dma_filter; - - /* Register as many memcpy channels as there are physical channels */ - ret = s3c24xx_dma_init_virtual_channels(s3cdma, &s3cdma->memcpy, - pdata->num_phy_channels, false); - if (ret <= 0) { - dev_warn(&pdev->dev, - "%s failed to enumerate memcpy channels - %d\n", - __func__, ret); - goto err_memcpy; - } - - /* Register slave channels */ - ret = s3c24xx_dma_init_virtual_channels(s3cdma, &s3cdma->slave, - pdata->num_channels, true); - if (ret <= 0) { - dev_warn(&pdev->dev, - "%s failed to enumerate slave channels - %d\n", - __func__, ret); - goto err_slave; - } - - ret = dma_async_device_register(&s3cdma->memcpy); - if (ret) { - dev_warn(&pdev->dev, - "%s failed to register memcpy as an async device - %d\n", - __func__, ret); - goto err_memcpy_reg; - } - - ret = dma_async_device_register(&s3cdma->slave); - if (ret) { - dev_warn(&pdev->dev, - "%s failed to register slave as an async device - %d\n", - __func__, ret); - goto err_slave_reg; - } - - platform_set_drvdata(pdev, s3cdma); - dev_info(&pdev->dev, "Loaded dma driver with %d physical channels\n", - pdata->num_phy_channels); - - return 0; - -err_slave_reg: - dma_async_device_unregister(&s3cdma->memcpy); -err_memcpy_reg: - s3c24xx_dma_free_virtual_channels(&s3cdma->slave); -err_slave: - s3c24xx_dma_free_virtual_channels(&s3cdma->memcpy); -err_memcpy: - if (sdata->has_clocks) - for (i = 0; i < pdata->num_phy_channels; i++) { - struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i]; - if (phy->valid) - clk_unprepare(phy->clk); - } - - return ret; -} - -static void s3c24xx_dma_free_irq(struct platform_device *pdev, - struct s3c24xx_dma_engine *s3cdma) -{ - int i; - - for (i = 0; i < s3cdma->pdata->num_phy_channels; i++) { - struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i]; - - devm_free_irq(&pdev->dev, phy->irq, phy); - } -} - -static int s3c24xx_dma_remove(struct platform_device *pdev) -{ - const struct s3c24xx_dma_platdata *pdata = dev_get_platdata(&pdev->dev); - struct s3c24xx_dma_engine *s3cdma = platform_get_drvdata(pdev); - struct soc_data *sdata = s3c24xx_dma_get_soc_data(pdev); - int i; - - dma_async_device_unregister(&s3cdma->slave); - dma_async_device_unregister(&s3cdma->memcpy); - - s3c24xx_dma_free_irq(pdev, s3cdma); - - s3c24xx_dma_free_virtual_channels(&s3cdma->slave); - s3c24xx_dma_free_virtual_channels(&s3cdma->memcpy); - - if (sdata->has_clocks) - for (i = 0; i < pdata->num_phy_channels; i++) { - struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i]; - if (phy->valid) - clk_unprepare(phy->clk); - } - - return 0; -} - -static struct platform_driver s3c24xx_dma_driver = { - .driver = { - .name = "s3c24xx-dma", - }, - .id_table = s3c24xx_dma_driver_ids, - .probe = s3c24xx_dma_probe, - .remove = s3c24xx_dma_remove, -}; - -module_platform_driver(s3c24xx_dma_driver); - -bool s3c24xx_dma_filter(struct dma_chan *chan, void *param) -{ - struct s3c24xx_dma_chan *s3cchan; - - if (chan->device->dev->driver != &s3c24xx_dma_driver.driver) - return false; - - s3cchan = to_s3c24xx_dma_chan(chan); - - return s3cchan->id == (uintptr_t)param; -} -EXPORT_SYMBOL(s3c24xx_dma_filter); - -MODULE_DESCRIPTION("S3C24XX DMA Driver"); -MODULE_AUTHOR("Heiko Stuebner"); -MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/sh/shdma-arm.h b/drivers/dma/sh/shdma-arm.h deleted file mode 100644 index 7459f9a13b5b..000000000000 --- a/drivers/dma/sh/shdma-arm.h +++ /dev/null @@ -1,48 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Renesas SuperH DMA Engine support - * - * Copyright (C) 2013 Renesas Electronics, Inc. - */ - -#ifndef SHDMA_ARM_H -#define SHDMA_ARM_H - -#include "shdma.h" - -/* Transmit sizes and respective CHCR register values */ -enum { - XMIT_SZ_8BIT = 0, - XMIT_SZ_16BIT = 1, - XMIT_SZ_32BIT = 2, - XMIT_SZ_64BIT = 7, - XMIT_SZ_128BIT = 3, - XMIT_SZ_256BIT = 4, - XMIT_SZ_512BIT = 5, -}; - -/* log2(size / 8) - used to calculate number of transfers */ -#define SH_DMAE_TS_SHIFT { \ - [XMIT_SZ_8BIT] = 0, \ - [XMIT_SZ_16BIT] = 1, \ - [XMIT_SZ_32BIT] = 2, \ - [XMIT_SZ_64BIT] = 3, \ - [XMIT_SZ_128BIT] = 4, \ - [XMIT_SZ_256BIT] = 5, \ - [XMIT_SZ_512BIT] = 6, \ -} - -#define TS_LOW_BIT 0x3 /* --xx */ -#define TS_HI_BIT 0xc /* xx-- */ - -#define TS_LOW_SHIFT (3) -#define TS_HI_SHIFT (20 - 2) /* 2 bits for shifted low TS */ - -#define TS_INDEX2VAL(i) \ - ((((i) & TS_LOW_BIT) << TS_LOW_SHIFT) |\ - (((i) & TS_HI_BIT) << TS_HI_SHIFT)) - -#define CHCR_TX(xmit_sz) (DM_FIX | SM_INC | RS_ERS | TS_INDEX2VAL((xmit_sz))) -#define CHCR_RX(xmit_sz) (DM_INC | SM_FIX | RS_ERS | TS_INDEX2VAL((xmit_sz))) - -#endif diff --git a/drivers/dma/ti/Kconfig b/drivers/dma/ti/Kconfig index 79618fac119a..2adc2cca10e9 100644 --- a/drivers/dma/ti/Kconfig +++ b/drivers/dma/ti/Kconfig @@ -35,7 +35,7 @@ config DMA_OMAP DMA engine is found on OMAP and DRA7xx parts. config TI_K3_UDMA - bool "Texas Instruments UDMA support" + tristate "Texas Instruments UDMA support" depends on ARCH_K3 depends on TI_SCI_PROTOCOL depends on TI_SCI_INTA_IRQCHIP @@ -48,7 +48,7 @@ config TI_K3_UDMA DMA engine is used in AM65x and j721e. config TI_K3_UDMA_GLUE_LAYER - bool "Texas Instruments UDMA Glue layer for non DMAengine users" + tristate "Texas Instruments UDMA Glue layer for non DMAengine users" depends on ARCH_K3 depends on TI_K3_UDMA help @@ -56,7 +56,8 @@ config TI_K3_UDMA_GLUE_LAYER If unsure, say N. config TI_K3_PSIL - bool + tristate + default TI_K3_UDMA config TI_DMA_CROSSBAR bool diff --git a/drivers/dma/ti/Makefile b/drivers/dma/ti/Makefile index d3a303f0d7c6..b53d05b11ca5 100644 --- a/drivers/dma/ti/Makefile +++ b/drivers/dma/ti/Makefile @@ -4,11 +4,12 @@ obj-$(CONFIG_TI_EDMA) += edma.o obj-$(CONFIG_DMA_OMAP) += omap-dma.o obj-$(CONFIG_TI_K3_UDMA) += k3-udma.o obj-$(CONFIG_TI_K3_UDMA_GLUE_LAYER) += k3-udma-glue.o -obj-$(CONFIG_TI_K3_PSIL) += k3-psil.o \ - k3-psil-am654.o \ - k3-psil-j721e.o \ - k3-psil-j7200.o \ - k3-psil-am64.o \ - k3-psil-j721s2.o \ - k3-psil-am62.o +k3-psil-lib-objs := k3-psil.o \ + k3-psil-am654.o \ + k3-psil-j721e.o \ + k3-psil-j7200.o \ + k3-psil-am64.o \ + k3-psil-j721s2.o \ + k3-psil-am62.o +obj-$(CONFIG_TI_K3_PSIL) += k3-psil-lib.o obj-$(CONFIG_TI_DMA_CROSSBAR) += dma-crossbar.o diff --git a/drivers/dma/ti/k3-psil.c b/drivers/dma/ti/k3-psil.c index 761a384093d2..8b6533a1eeeb 100644 --- a/drivers/dma/ti/k3-psil.c +++ b/drivers/dma/ti/k3-psil.c @@ -5,6 +5,7 @@ */ #include <linux/kernel.h> +#include <linux/module.h> #include <linux/device.h> #include <linux/init.h> #include <linux/mutex.h> @@ -101,3 +102,4 @@ int psil_set_new_ep_config(struct device *dev, const char *name, return 0; } EXPORT_SYMBOL_GPL(psil_set_new_ep_config); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/ti/k3-udma-glue.c b/drivers/dma/ti/k3-udma-glue.c index 4f1aeb81e9c7..789193ed0386 100644 --- a/drivers/dma/ti/k3-udma-glue.c +++ b/drivers/dma/ti/k3-udma-glue.c @@ -6,6 +6,7 @@ * */ +#include <linux/module.h> #include <linux/atomic.h> #include <linux/delay.h> #include <linux/dma-mapping.h> @@ -1436,4 +1437,6 @@ static int __init k3_udma_glue_class_init(void) { return class_register(&k3_udma_glue_devclass); } -arch_initcall(k3_udma_glue_class_init); + +module_init(k3_udma_glue_class_init); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c index 7b5081989b3d..ce8b80bb34d7 100644 --- a/drivers/dma/ti/k3-udma.c +++ b/drivers/dma/ti/k3-udma.c @@ -5,6 +5,7 @@ */ #include <linux/kernel.h> +#include <linux/module.h> #include <linux/delay.h> #include <linux/dmaengine.h> #include <linux/dma-mapping.h> @@ -4335,18 +4336,10 @@ static const struct of_device_id udma_of_match[] = { .compatible = "ti,j721e-navss-mcu-udmap", .data = &j721e_mcu_data, }, - { /* Sentinel */ }, -}; - -static const struct of_device_id bcdma_of_match[] = { { .compatible = "ti,am64-dmss-bcdma", .data = &am64_bcdma_data, }, - { /* Sentinel */ }, -}; - -static const struct of_device_id pktdma_of_match[] = { { .compatible = "ti,am64-dmss-pktdma", .data = &am64_pktdma_data, @@ -5271,14 +5264,9 @@ static int udma_probe(struct platform_device *pdev) return -ENOMEM; match = of_match_node(udma_of_match, dev->of_node); - if (!match) - match = of_match_node(bcdma_of_match, dev->of_node); if (!match) { - match = of_match_node(pktdma_of_match, dev->of_node); - if (!match) { - dev_err(dev, "No compatible match found\n"); - return -ENODEV; - } + dev_err(dev, "No compatible match found\n"); + return -ENODEV; } ud->match_data = match->data; @@ -5511,27 +5499,9 @@ static struct platform_driver udma_driver = { }, .probe = udma_probe, }; -builtin_platform_driver(udma_driver); -static struct platform_driver bcdma_driver = { - .driver = { - .name = "ti-bcdma", - .of_match_table = bcdma_of_match, - .suppress_bind_attrs = true, - }, - .probe = udma_probe, -}; -builtin_platform_driver(bcdma_driver); - -static struct platform_driver pktdma_driver = { - .driver = { - .name = "ti-pktdma", - .of_match_table = pktdma_of_match, - .suppress_bind_attrs = true, - }, - .probe = udma_probe, -}; -builtin_platform_driver(pktdma_driver); +module_platform_driver(udma_driver); +MODULE_LICENSE("GPL v2"); /* Private interfaces to UDMA */ #include "k3-udma-private.c" diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c index 8cd4e69dc7b4..a8d23cdf883e 100644 --- a/drivers/dma/xilinx/xilinx_dma.c +++ b/drivers/dma/xilinx/xilinx_dma.c @@ -1659,6 +1659,8 @@ static void xilinx_dma_issue_pending(struct dma_chan *dchan) * xilinx_dma_device_config - Configure the DMA channel * @dchan: DMA channel * @config: channel configuration + * + * Return: 0 always. */ static int xilinx_dma_device_config(struct dma_chan *dchan, struct dma_slave_config *config) @@ -2924,7 +2926,7 @@ static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev, * @xdev: Driver specific device structure * @node: Device node * - * Return: 0 always. + * Return: '0' on success and failure value on error. */ static int xilinx_dma_child_probe(struct xilinx_dma_device *xdev, struct device_node *node) diff --git a/drivers/of/irq.c b/drivers/of/irq.c index 2bac44f09554..e9bf5236ed89 100644 --- a/drivers/of/irq.c +++ b/drivers/of/irq.c @@ -730,6 +730,7 @@ struct irq_domain *of_msi_get_domain(struct device *dev, return NULL; } +EXPORT_SYMBOL_GPL(of_msi_get_domain); /** * of_msi_configure - Set the msi_domain field of a device diff --git a/include/linux/platform_data/dma-s3c24xx.h b/include/linux/platform_data/dma-s3c24xx.h deleted file mode 100644 index 96d02dbeea67..000000000000 --- a/include/linux/platform_data/dma-s3c24xx.h +++ /dev/null @@ -1,48 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * S3C24XX DMA handling - * - * Copyright (c) 2013 Heiko Stuebner <[email protected]> - */ - -/* Helper to encode the source selection constraints for early s3c socs. */ -#define S3C24XX_DMA_CHANREQ(src, chan) ((BIT(3) | src) << chan * 4) - -enum s3c24xx_dma_bus { - S3C24XX_DMA_APB, - S3C24XX_DMA_AHB, -}; - -/** - * @bus: on which bus does the peripheral reside - AHB or APB. - * @handshake: is a handshake with the peripheral necessary - * @chansel: channel selection information, depending on variant; reqsel for - * s3c2443 and later and channel-selection map for earlier SoCs - * see CHANSEL doc in s3c2443-dma.c - */ -struct s3c24xx_dma_channel { - enum s3c24xx_dma_bus bus; - bool handshake; - u16 chansel; -}; - -struct dma_slave_map; - -/** - * struct s3c24xx_dma_platdata - platform specific settings - * @num_phy_channels: number of physical channels - * @channels: array of virtual channel descriptions - * @num_channels: number of virtual channels - * @slave_map: dma slave map matching table - * @slavecnt: number of elements in slave_map - */ -struct s3c24xx_dma_platdata { - int num_phy_channels; - struct s3c24xx_dma_channel *channels; - int num_channels; - const struct dma_slave_map *slave_map; - int slavecnt; -}; - -struct dma_chan; -bool s3c24xx_dma_filter(struct dma_chan *chan, void *param); |