diff options
Diffstat (limited to 'drivers/crypto')
135 files changed, 3946 insertions, 11108 deletions
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 52a9b7cf6576..37da0c070a88 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -71,10 +71,26 @@ config ZCRYPT help Select this option if you want to enable support for s390 cryptographic adapters like: - + PCI-X Cryptographic Coprocessor (PCIXCC) - + Crypto Express 2,3,4 or 5 Coprocessor (CEXxC) - + Crypto Express 2,3,4 or 5 Accelerator (CEXxA) - + Crypto Express 4 or 5 EP11 Coprocessor (CEXxP) + + Crypto Express 2 up to 7 Coprocessor (CEXxC) + + Crypto Express 2 up to 7 Accelerator (CEXxA) + + Crypto Express 4 up to 7 EP11 Coprocessor (CEXxP) + +config ZCRYPT_DEBUG + bool "Enable debug features for s390 cryptographic adapters" + default n + depends on DEBUG_KERNEL + depends on ZCRYPT + help + Say 'Y' here to enable some additional debug features on the + s390 cryptographic adapters driver. + + There will be some more sysfs attributes displayed for ap cards + and queues and some flags on crypto requests are interpreted as + debugging messages to force error injection. + + Do not enable on production level kernel build. + + If unsure, say N. config ZCRYPT_MULTIDEVNODES bool "Support for multiple zcrypt device nodes" @@ -873,6 +889,7 @@ config CRYPTO_DEV_SA2UL select CRYPTO_AES select CRYPTO_AES_ARM64 select CRYPTO_ALGAPI + select CRYPTO_AUTHENC select CRYPTO_SHA1 select CRYPTO_SHA256 select CRYPTO_SHA512 diff --git a/drivers/crypto/allwinner/Kconfig b/drivers/crypto/allwinner/Kconfig index 12e7c6a85a02..0cdfe0e8cc66 100644 --- a/drivers/crypto/allwinner/Kconfig +++ b/drivers/crypto/allwinner/Kconfig @@ -59,6 +59,32 @@ config CRYPTO_DEV_SUN8I_CE_DEBUG This will create /sys/kernel/debug/sun8i-ce/stats for displaying the number of requests per flow and per algorithm. +config CRYPTO_DEV_SUN8I_CE_HASH + bool "Enable support for hash on sun8i-ce" + depends on CRYPTO_DEV_SUN8I_CE + select MD5 + select SHA1 + select SHA256 + select SHA512 + help + Say y to enable support for hash algorithms. + +config CRYPTO_DEV_SUN8I_CE_PRNG + bool "Support for Allwinner Crypto Engine PRNG" + depends on CRYPTO_DEV_SUN8I_CE + select CRYPTO_RNG + help + Select this option if you want to provide kernel-side support for + the Pseudo-Random Number Generator found in the Crypto Engine. + +config CRYPTO_DEV_SUN8I_CE_TRNG + bool "Support for Allwinner Crypto Engine TRNG" + depends on CRYPTO_DEV_SUN8I_CE + select HW_RANDOM + help + Select this option if you want to provide kernel-side support for + the True Random Number Generator found in the Crypto Engine. + config CRYPTO_DEV_SUN8I_SS tristate "Support for Allwinner Security System cryptographic offloader" select CRYPTO_SKCIPHER @@ -85,3 +111,20 @@ config CRYPTO_DEV_SUN8I_SS_DEBUG Say y to enable sun8i-ss debug stats. This will create /sys/kernel/debug/sun8i-ss/stats for displaying the number of requests per flow and per algorithm. + +config CRYPTO_DEV_SUN8I_SS_PRNG + bool "Support for Allwinner Security System PRNG" + depends on CRYPTO_DEV_SUN8I_SS + select CRYPTO_RNG + help + Select this option if you want to provide kernel-side support for + the Pseudo-Random Number Generator found in the Security System. + +config CRYPTO_DEV_SUN8I_SS_HASH + bool "Enable support for hash on sun8i-ss" + depends on CRYPTO_DEV_SUN8I_SS + select MD5 + select SHA1 + select SHA256 + help + Say y to enable support for hash algorithms. diff --git a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-hash.c b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-hash.c index dc35edd90034..1dff48558f53 100644 --- a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-hash.c +++ b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-hash.c @@ -9,6 +9,7 @@ * You could find the datasheet in Documentation/arm/sunxi.rst */ #include "sun4i-ss.h" +#include <asm/unaligned.h> #include <linux/scatterlist.h> /* This is a totally arbitrary value */ @@ -196,7 +197,7 @@ static int sun4i_hash(struct ahash_request *areq) struct sg_mapping_iter mi; int in_r, err = 0; size_t copied = 0; - __le32 wb = 0; + u32 wb = 0; dev_dbg(ss->dev, "%s %s bc=%llu len=%u mode=%x wl=%u h0=%0x", __func__, crypto_tfm_alg_name(areq->base.tfm), @@ -408,7 +409,7 @@ hash_final: nbw = op->len - 4 * nwait; if (nbw) { - wb = cpu_to_le32(*(u32 *)(op->buf + nwait * 4)); + wb = le32_to_cpup((__le32 *)(op->buf + nwait * 4)); wb &= GENMASK((nbw * 8) - 1, 0); op->byte_count += nbw; @@ -417,7 +418,7 @@ hash_final: /* write the remaining bytes of the nbw buffer */ wb |= ((1 << 7) << (nbw * 8)); - bf[j++] = le32_to_cpu(wb); + ((__le32 *)bf)[j++] = cpu_to_le32(wb); /* * number of space to pad to obtain 64o minus 8(size) minus 4 (final 1) @@ -479,16 +480,16 @@ hash_final: /* Get the hash from the device */ if (op->mode == SS_OP_SHA1) { for (i = 0; i < 5; i++) { + v = readl(ss->base + SS_MD0 + i * 4); if (ss->variant->sha1_in_be) - v = cpu_to_le32(readl(ss->base + SS_MD0 + i * 4)); + put_unaligned_le32(v, areq->result + i * 4); else - v = cpu_to_be32(readl(ss->base + SS_MD0 + i * 4)); - memcpy(areq->result + i * 4, &v, 4); + put_unaligned_be32(v, areq->result + i * 4); } } else { for (i = 0; i < 4; i++) { - v = cpu_to_le32(readl(ss->base + SS_MD0 + i * 4)); - memcpy(areq->result + i * 4, &v, 4); + v = readl(ss->base + SS_MD0 + i * 4); + put_unaligned_le32(v, areq->result + i * 4); } } diff --git a/drivers/crypto/allwinner/sun8i-ce/Makefile b/drivers/crypto/allwinner/sun8i-ce/Makefile index 08b68c3c1ca9..0842eb2d9408 100644 --- a/drivers/crypto/allwinner/sun8i-ce/Makefile +++ b/drivers/crypto/allwinner/sun8i-ce/Makefile @@ -1,2 +1,5 @@ obj-$(CONFIG_CRYPTO_DEV_SUN8I_CE) += sun8i-ce.o sun8i-ce-y += sun8i-ce-core.o sun8i-ce-cipher.o +sun8i-ce-$(CONFIG_CRYPTO_DEV_SUN8I_CE_HASH) += sun8i-ce-hash.o +sun8i-ce-$(CONFIG_CRYPTO_DEV_SUN8I_CE_PRNG) += sun8i-ce-prng.o +sun8i-ce-$(CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG) += sun8i-ce-trng.o diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c index b4d5fea27d20..33707a2e55ff 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c @@ -75,8 +75,9 @@ static int sun8i_ce_cipher_fallback(struct skcipher_request *areq) return err; } -static int sun8i_ce_cipher(struct skcipher_request *areq) +static int sun8i_ce_cipher_prepare(struct crypto_engine *engine, void *async_req) { + struct skcipher_request *areq = container_of(async_req, struct skcipher_request, base); struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); struct sun8i_ce_dev *ce = op->ce; @@ -87,8 +88,6 @@ static int sun8i_ce_cipher(struct skcipher_request *areq) struct ce_task *cet; struct scatterlist *sg; unsigned int todo, len, offset, ivsize; - dma_addr_t addr_iv = 0, addr_key = 0; - void *backup_iv = NULL; u32 common, sym; int flow, i; int nr_sgs = 0; @@ -119,7 +118,7 @@ static int sun8i_ce_cipher(struct skcipher_request *areq) common |= rctx->op_dir | CE_COMM_INT; cet->t_common_ctl = cpu_to_le32(common); /* CTS and recent CE (H6) need length in bytes, in word otherwise */ - if (ce->variant->has_t_dlen_in_bytes) + if (ce->variant->cipher_t_dlen_in_bytes) cet->t_dlen = cpu_to_le32(areq->cryptlen); else cet->t_dlen = cpu_to_le32(areq->cryptlen / 4); @@ -141,41 +140,41 @@ static int sun8i_ce_cipher(struct skcipher_request *areq) cet->t_sym_ctl = cpu_to_le32(sym); cet->t_asym_ctl = 0; - addr_key = dma_map_single(ce->dev, op->key, op->keylen, DMA_TO_DEVICE); - cet->t_key = cpu_to_le32(addr_key); - if (dma_mapping_error(ce->dev, addr_key)) { + rctx->addr_key = dma_map_single(ce->dev, op->key, op->keylen, DMA_TO_DEVICE); + if (dma_mapping_error(ce->dev, rctx->addr_key)) { dev_err(ce->dev, "Cannot DMA MAP KEY\n"); err = -EFAULT; goto theend; } + cet->t_key = cpu_to_le32(rctx->addr_key); ivsize = crypto_skcipher_ivsize(tfm); if (areq->iv && crypto_skcipher_ivsize(tfm) > 0) { - chan->ivlen = ivsize; - chan->bounce_iv = kzalloc(ivsize, GFP_KERNEL | GFP_DMA); - if (!chan->bounce_iv) { + rctx->ivlen = ivsize; + rctx->bounce_iv = kzalloc(ivsize, GFP_KERNEL | GFP_DMA); + if (!rctx->bounce_iv) { err = -ENOMEM; goto theend_key; } if (rctx->op_dir & CE_DECRYPTION) { - backup_iv = kzalloc(ivsize, GFP_KERNEL); - if (!backup_iv) { + rctx->backup_iv = kzalloc(ivsize, GFP_KERNEL); + if (!rctx->backup_iv) { err = -ENOMEM; goto theend_key; } offset = areq->cryptlen - ivsize; - scatterwalk_map_and_copy(backup_iv, areq->src, offset, - ivsize, 0); + scatterwalk_map_and_copy(rctx->backup_iv, areq->src, + offset, ivsize, 0); } - memcpy(chan->bounce_iv, areq->iv, ivsize); - addr_iv = dma_map_single(ce->dev, chan->bounce_iv, chan->ivlen, - DMA_TO_DEVICE); - cet->t_iv = cpu_to_le32(addr_iv); - if (dma_mapping_error(ce->dev, addr_iv)) { + memcpy(rctx->bounce_iv, areq->iv, ivsize); + rctx->addr_iv = dma_map_single(ce->dev, rctx->bounce_iv, rctx->ivlen, + DMA_TO_DEVICE); + if (dma_mapping_error(ce->dev, rctx->addr_iv)) { dev_err(ce->dev, "Cannot DMA MAP IV\n"); err = -ENOMEM; goto theend_iv; } + cet->t_iv = cpu_to_le32(rctx->addr_iv); } if (areq->src == areq->dst) { @@ -235,7 +234,9 @@ static int sun8i_ce_cipher(struct skcipher_request *areq) } chan->timeout = areq->cryptlen; - err = sun8i_ce_run_task(ce, flow, crypto_tfm_alg_name(areq->base.tfm)); + rctx->nr_sgs = nr_sgs; + rctx->nr_sgd = nr_sgd; + return 0; theend_sgs: if (areq->src == areq->dst) { @@ -248,34 +249,83 @@ theend_sgs: theend_iv: if (areq->iv && ivsize > 0) { - if (addr_iv) - dma_unmap_single(ce->dev, addr_iv, chan->ivlen, - DMA_TO_DEVICE); + if (rctx->addr_iv) + dma_unmap_single(ce->dev, rctx->addr_iv, rctx->ivlen, DMA_TO_DEVICE); offset = areq->cryptlen - ivsize; if (rctx->op_dir & CE_DECRYPTION) { - memcpy(areq->iv, backup_iv, ivsize); - kfree_sensitive(backup_iv); + memcpy(areq->iv, rctx->backup_iv, ivsize); + kfree_sensitive(rctx->backup_iv); } else { scatterwalk_map_and_copy(areq->iv, areq->dst, offset, ivsize, 0); } - kfree(chan->bounce_iv); + kfree(rctx->bounce_iv); } theend_key: - dma_unmap_single(ce->dev, addr_key, op->keylen, DMA_TO_DEVICE); + dma_unmap_single(ce->dev, rctx->addr_key, op->keylen, DMA_TO_DEVICE); theend: return err; } -static int sun8i_ce_handle_cipher_request(struct crypto_engine *engine, void *areq) +static int sun8i_ce_cipher_run(struct crypto_engine *engine, void *areq) { - int err; struct skcipher_request *breq = container_of(areq, struct skcipher_request, base); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(breq); + struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun8i_ce_dev *ce = op->ce; + struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(breq); + int flow, err; - err = sun8i_ce_cipher(breq); + flow = rctx->flow; + err = sun8i_ce_run_task(ce, flow, crypto_tfm_alg_name(breq->base.tfm)); crypto_finalize_skcipher_request(engine, breq, err); + return 0; +} + +static int sun8i_ce_cipher_unprepare(struct crypto_engine *engine, void *async_req) +{ + struct skcipher_request *areq = container_of(async_req, struct skcipher_request, base); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun8i_ce_dev *ce = op->ce; + struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + struct sun8i_ce_flow *chan; + struct ce_task *cet; + unsigned int ivsize, offset; + int nr_sgs = rctx->nr_sgs; + int nr_sgd = rctx->nr_sgd; + int flow; + + flow = rctx->flow; + chan = &ce->chanlist[flow]; + cet = chan->tl; + ivsize = crypto_skcipher_ivsize(tfm); + + if (areq->src == areq->dst) { + dma_unmap_sg(ce->dev, areq->src, nr_sgs, DMA_BIDIRECTIONAL); + } else { + if (nr_sgs > 0) + dma_unmap_sg(ce->dev, areq->src, nr_sgs, DMA_TO_DEVICE); + dma_unmap_sg(ce->dev, areq->dst, nr_sgd, DMA_FROM_DEVICE); + } + + if (areq->iv && ivsize > 0) { + if (cet->t_iv) + dma_unmap_single(ce->dev, rctx->addr_iv, rctx->ivlen, DMA_TO_DEVICE); + offset = areq->cryptlen - ivsize; + if (rctx->op_dir & CE_DECRYPTION) { + memcpy(areq->iv, rctx->backup_iv, ivsize); + kfree_sensitive(rctx->backup_iv); + } else { + scatterwalk_map_and_copy(areq->iv, areq->dst, offset, + ivsize, 0); + } + kfree(rctx->bounce_iv); + } + + dma_unmap_single(ce->dev, rctx->addr_key, op->keylen, DMA_TO_DEVICE); return 0; } @@ -347,9 +397,9 @@ int sun8i_ce_cipher_init(struct crypto_tfm *tfm) crypto_tfm_alg_driver_name(&sktfm->base), crypto_tfm_alg_driver_name(crypto_skcipher_tfm(op->fallback_tfm))); - op->enginectx.op.do_one_request = sun8i_ce_handle_cipher_request; - op->enginectx.op.prepare_request = NULL; - op->enginectx.op.unprepare_request = NULL; + op->enginectx.op.do_one_request = sun8i_ce_cipher_run; + op->enginectx.op.prepare_request = sun8i_ce_cipher_prepare; + op->enginectx.op.unprepare_request = sun8i_ce_cipher_unprepare; err = pm_runtime_get_sync(op->ce->dev); if (err < 0) @@ -366,10 +416,7 @@ void sun8i_ce_cipher_exit(struct crypto_tfm *tfm) { struct sun8i_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm); - if (op->key) { - memzero_explicit(op->key, op->keylen); - kfree(op->key); - } + kfree_sensitive(op->key); crypto_free_skcipher(op->fallback_tfm); pm_runtime_put_sync_suspend(op->ce->dev); } @@ -391,10 +438,7 @@ int sun8i_ce_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, dev_dbg(ce->dev, "ERROR: Invalid keylen %u\n", keylen); return -EINVAL; } - if (op->key) { - memzero_explicit(op->key, op->keylen); - kfree(op->key); - } + kfree_sensitive(op->key); op->keylen = keylen; op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA); if (!op->key) @@ -416,10 +460,7 @@ int sun8i_ce_des3_setkey(struct crypto_skcipher *tfm, const u8 *key, if (err) return err; - if (op->key) { - memzero_explicit(op->key, op->keylen); - kfree(op->key); - } + kfree_sensitive(op->key); op->keylen = keylen; op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA); if (!op->key) diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c index 138759dc8190..158422ff5695 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c @@ -22,6 +22,7 @@ #include <linux/platform_device.h> #include <linux/pm_runtime.h> #include <linux/reset.h> +#include <crypto/internal/rng.h> #include <crypto/internal/skcipher.h> #include "sun8i-ce.h" @@ -35,73 +36,108 @@ static const struct ce_variant ce_h3_variant = { .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES, }, + .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256, + CE_ALG_SHA384, CE_ALG_SHA512 + }, .op_mode = { CE_OP_ECB, CE_OP_CBC }, .ce_clks = { { "bus", 0, 200000000 }, { "mod", 50000000, 0 }, - } + }, + .esr = ESR_H3, + .prng = CE_ALG_PRNG, + .trng = CE_ID_NOTSUPP, }; static const struct ce_variant ce_h5_variant = { .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES, }, + .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256, + CE_ID_NOTSUPP, CE_ID_NOTSUPP + }, .op_mode = { CE_OP_ECB, CE_OP_CBC }, .ce_clks = { { "bus", 0, 200000000 }, { "mod", 300000000, 0 }, - } + }, + .esr = ESR_H5, + .prng = CE_ALG_PRNG, + .trng = CE_ID_NOTSUPP, }; static const struct ce_variant ce_h6_variant = { .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES, }, + .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256, + CE_ALG_SHA384, CE_ALG_SHA512 + }, .op_mode = { CE_OP_ECB, CE_OP_CBC }, - .has_t_dlen_in_bytes = true, + .cipher_t_dlen_in_bytes = true, + .hash_t_dlen_in_bits = true, + .prng_t_dlen_in_bytes = true, + .trng_t_dlen_in_bytes = true, .ce_clks = { { "bus", 0, 200000000 }, { "mod", 300000000, 0 }, { "ram", 0, 400000000 }, - } + }, + .esr = ESR_H6, + .prng = CE_ALG_PRNG_V2, + .trng = CE_ALG_TRNG_V2, }; static const struct ce_variant ce_a64_variant = { .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES, }, + .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256, + CE_ID_NOTSUPP, CE_ID_NOTSUPP + }, .op_mode = { CE_OP_ECB, CE_OP_CBC }, .ce_clks = { { "bus", 0, 200000000 }, { "mod", 300000000, 0 }, - } + }, + .esr = ESR_A64, + .prng = CE_ALG_PRNG, + .trng = CE_ID_NOTSUPP, }; static const struct ce_variant ce_r40_variant = { .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES, }, + .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256, + CE_ID_NOTSUPP, CE_ID_NOTSUPP + }, .op_mode = { CE_OP_ECB, CE_OP_CBC }, .ce_clks = { { "bus", 0, 200000000 }, { "mod", 300000000, 0 }, - } + }, + .esr = ESR_R40, + .prng = CE_ALG_PRNG, + .trng = CE_ID_NOTSUPP, }; /* * sun8i_ce_get_engine_number() get the next channel slot * This is a simple round-robin way of getting the next channel + * The flow 3 is reserve for xRNG operations */ int sun8i_ce_get_engine_number(struct sun8i_ce_dev *ce) { - return atomic_inc_return(&ce->flow) % MAXFLOW; + return atomic_inc_return(&ce->flow) % (MAXFLOW - 1); } int sun8i_ce_run_task(struct sun8i_ce_dev *ce, int flow, const char *name) { u32 v; int err = 0; + struct ce_task *cet = ce->chanlist[flow].tl; #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG ce->chanlist[flow].stat_req++; @@ -120,7 +156,10 @@ int sun8i_ce_run_task(struct sun8i_ce_dev *ce, int flow, const char *name) /* Be sure all data is written before enabling the task */ wmb(); - v = 1 | (ce->chanlist[flow].tl->t_common_ctl & 0x7F) << 8; + /* Only H6 needs to write a part of t_common_ctl along with "1", but since it is ignored + * on older SoCs, we have no reason to complicate things. + */ + v = 1 | ((le32_to_cpu(ce->chanlist[flow].tl->t_common_ctl) & 0x7F) << 8); writel(v, ce->base + CE_TLR); mutex_unlock(&ce->mlock); @@ -128,19 +167,56 @@ int sun8i_ce_run_task(struct sun8i_ce_dev *ce, int flow, const char *name) msecs_to_jiffies(ce->chanlist[flow].timeout)); if (ce->chanlist[flow].status == 0) { - dev_err(ce->dev, "DMA timeout for %s\n", name); + dev_err(ce->dev, "DMA timeout for %s (tm=%d) on flow %d\n", name, + ce->chanlist[flow].timeout, flow); err = -EFAULT; } /* No need to lock for this read, the channel is locked so * nothing could modify the error value for this channel */ v = readl(ce->base + CE_ESR); - if (v) { + switch (ce->variant->esr) { + case ESR_H3: + /* Sadly, the error bit is not per flow */ + if (v) { + dev_err(ce->dev, "CE ERROR: %x for flow %x\n", v, flow); + err = -EFAULT; + print_hex_dump(KERN_INFO, "TASK: ", DUMP_PREFIX_NONE, 16, 4, + cet, sizeof(struct ce_task), false); + } + if (v & CE_ERR_ALGO_NOTSUP) + dev_err(ce->dev, "CE ERROR: algorithm not supported\n"); + if (v & CE_ERR_DATALEN) + dev_err(ce->dev, "CE ERROR: data length error\n"); + if (v & CE_ERR_KEYSRAM) + dev_err(ce->dev, "CE ERROR: keysram access error for AES\n"); + break; + case ESR_A64: + case ESR_H5: + case ESR_R40: v >>= (flow * 4); + v &= 0xF; + if (v) { + dev_err(ce->dev, "CE ERROR: %x for flow %x\n", v, flow); + err = -EFAULT; + print_hex_dump(KERN_INFO, "TASK: ", DUMP_PREFIX_NONE, 16, 4, + cet, sizeof(struct ce_task), false); + } + if (v & CE_ERR_ALGO_NOTSUP) + dev_err(ce->dev, "CE ERROR: algorithm not supported\n"); + if (v & CE_ERR_DATALEN) + dev_err(ce->dev, "CE ERROR: data length error\n"); + if (v & CE_ERR_KEYSRAM) + dev_err(ce->dev, "CE ERROR: keysram access error for AES\n"); + break; + case ESR_H6: + v >>= (flow * 8); v &= 0xFF; if (v) { dev_err(ce->dev, "CE ERROR: %x for flow %x\n", v, flow); err = -EFAULT; + print_hex_dump(KERN_INFO, "TASK: ", DUMP_PREFIX_NONE, 16, 4, + cet, sizeof(struct ce_task), false); } if (v & CE_ERR_ALGO_NOTSUP) dev_err(ce->dev, "CE ERROR: algorithm not supported\n"); @@ -150,7 +226,10 @@ int sun8i_ce_run_task(struct sun8i_ce_dev *ce, int flow, const char *name) dev_err(ce->dev, "CE ERROR: keysram access error for AES\n"); if (v & CE_ERR_ADDR_INVALID) dev_err(ce->dev, "CE ERROR: address invalid\n"); - } + if (v & CE_ERR_KEYLADDER) + dev_err(ce->dev, "CE ERROR: key ladder configuration error\n"); + break; + } return err; } @@ -280,13 +359,214 @@ static struct sun8i_ce_alg_template ce_algs[] = { .decrypt = sun8i_ce_skdecrypt, } }, +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_HASH +{ .type = CRYPTO_ALG_TYPE_AHASH, + .ce_algo_id = CE_ID_HASH_MD5, + .alg.hash = { + .init = sun8i_ce_hash_init, + .update = sun8i_ce_hash_update, + .final = sun8i_ce_hash_final, + .finup = sun8i_ce_hash_finup, + .digest = sun8i_ce_hash_digest, + .export = sun8i_ce_hash_export, + .import = sun8i_ce_hash_import, + .halg = { + .digestsize = MD5_DIGEST_SIZE, + .statesize = sizeof(struct md5_state), + .base = { + .cra_name = "md5", + .cra_driver_name = "md5-sun8i-ce", + .cra_priority = 300, + .cra_alignmask = 3, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = MD5_HMAC_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_init = sun8i_ce_hash_crainit, + .cra_exit = sun8i_ce_hash_craexit, + } + } + } +}, +{ .type = CRYPTO_ALG_TYPE_AHASH, + .ce_algo_id = CE_ID_HASH_SHA1, + .alg.hash = { + .init = sun8i_ce_hash_init, + .update = sun8i_ce_hash_update, + .final = sun8i_ce_hash_final, + .finup = sun8i_ce_hash_finup, + .digest = sun8i_ce_hash_digest, + .export = sun8i_ce_hash_export, + .import = sun8i_ce_hash_import, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct sha1_state), + .base = { + .cra_name = "sha1", + .cra_driver_name = "sha1-sun8i-ce", + .cra_priority = 300, + .cra_alignmask = 3, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_init = sun8i_ce_hash_crainit, + .cra_exit = sun8i_ce_hash_craexit, + } + } + } +}, +{ .type = CRYPTO_ALG_TYPE_AHASH, + .ce_algo_id = CE_ID_HASH_SHA224, + .alg.hash = { + .init = sun8i_ce_hash_init, + .update = sun8i_ce_hash_update, + .final = sun8i_ce_hash_final, + .finup = sun8i_ce_hash_finup, + .digest = sun8i_ce_hash_digest, + .export = sun8i_ce_hash_export, + .import = sun8i_ce_hash_import, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha224", + .cra_driver_name = "sha224-sun8i-ce", + .cra_priority = 300, + .cra_alignmask = 3, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_init = sun8i_ce_hash_crainit, + .cra_exit = sun8i_ce_hash_craexit, + } + } + } +}, +{ .type = CRYPTO_ALG_TYPE_AHASH, + .ce_algo_id = CE_ID_HASH_SHA256, + .alg.hash = { + .init = sun8i_ce_hash_init, + .update = sun8i_ce_hash_update, + .final = sun8i_ce_hash_final, + .finup = sun8i_ce_hash_finup, + .digest = sun8i_ce_hash_digest, + .export = sun8i_ce_hash_export, + .import = sun8i_ce_hash_import, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha256", + .cra_driver_name = "sha256-sun8i-ce", + .cra_priority = 300, + .cra_alignmask = 3, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_init = sun8i_ce_hash_crainit, + .cra_exit = sun8i_ce_hash_craexit, + } + } + } +}, +{ .type = CRYPTO_ALG_TYPE_AHASH, + .ce_algo_id = CE_ID_HASH_SHA384, + .alg.hash = { + .init = sun8i_ce_hash_init, + .update = sun8i_ce_hash_update, + .final = sun8i_ce_hash_final, + .finup = sun8i_ce_hash_finup, + .digest = sun8i_ce_hash_digest, + .export = sun8i_ce_hash_export, + .import = sun8i_ce_hash_import, + .halg = { + .digestsize = SHA384_DIGEST_SIZE, + .statesize = sizeof(struct sha512_state), + .base = { + .cra_name = "sha384", + .cra_driver_name = "sha384-sun8i-ce", + .cra_priority = 300, + .cra_alignmask = 3, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_init = sun8i_ce_hash_crainit, + .cra_exit = sun8i_ce_hash_craexit, + } + } + } +}, +{ .type = CRYPTO_ALG_TYPE_AHASH, + .ce_algo_id = CE_ID_HASH_SHA512, + .alg.hash = { + .init = sun8i_ce_hash_init, + .update = sun8i_ce_hash_update, + .final = sun8i_ce_hash_final, + .finup = sun8i_ce_hash_finup, + .digest = sun8i_ce_hash_digest, + .export = sun8i_ce_hash_export, + .import = sun8i_ce_hash_import, + .halg = { + .digestsize = SHA512_DIGEST_SIZE, + .statesize = sizeof(struct sha512_state), + .base = { + .cra_name = "sha512", + .cra_driver_name = "sha512-sun8i-ce", + .cra_priority = 300, + .cra_alignmask = 3, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_init = sun8i_ce_hash_crainit, + .cra_exit = sun8i_ce_hash_craexit, + } + } + } +}, +#endif +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_PRNG +{ + .type = CRYPTO_ALG_TYPE_RNG, + .alg.rng = { + .base = { + .cra_name = "stdrng", + .cra_driver_name = "sun8i-ce-prng", + .cra_priority = 300, + .cra_ctxsize = sizeof(struct sun8i_ce_rng_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_init = sun8i_ce_prng_init, + .cra_exit = sun8i_ce_prng_exit, + }, + .generate = sun8i_ce_prng_generate, + .seed = sun8i_ce_prng_seed, + .seedsize = PRNG_SEED_SIZE, + } +}, +#endif }; #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG -static int sun8i_ce_dbgfs_read(struct seq_file *seq, void *v) +static int sun8i_ce_debugfs_show(struct seq_file *seq, void *v) { struct sun8i_ce_dev *ce = seq->private; - int i; + unsigned int i; for (i = 0; i < MAXFLOW; i++) seq_printf(seq, "Channel %d: nreq %lu\n", i, ce->chanlist[i].stat_req); @@ -301,23 +581,28 @@ static int sun8i_ce_dbgfs_read(struct seq_file *seq, void *v) ce_algs[i].alg.skcipher.base.cra_name, ce_algs[i].stat_req, ce_algs[i].stat_fb); break; + case CRYPTO_ALG_TYPE_AHASH: + seq_printf(seq, "%s %s %lu %lu\n", + ce_algs[i].alg.hash.halg.base.cra_driver_name, + ce_algs[i].alg.hash.halg.base.cra_name, + ce_algs[i].stat_req, ce_algs[i].stat_fb); + break; + case CRYPTO_ALG_TYPE_RNG: + seq_printf(seq, "%s %s %lu %lu\n", + ce_algs[i].alg.rng.base.cra_driver_name, + ce_algs[i].alg.rng.base.cra_name, + ce_algs[i].stat_req, ce_algs[i].stat_bytes); + break; } } +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG + seq_printf(seq, "HWRNG %lu %lu\n", + ce->hwrng_stat_req, ce->hwrng_stat_bytes); +#endif return 0; } -static int sun8i_ce_dbgfs_open(struct inode *inode, struct file *file) -{ - return single_open(file, sun8i_ce_dbgfs_read, inode->i_private); -} - -static const struct file_operations sun8i_ce_debugfs_fops = { - .owner = THIS_MODULE, - .open = sun8i_ce_dbgfs_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(sun8i_ce_debugfs); #endif static void sun8i_ce_free_chanlist(struct sun8i_ce_dev *ce, int i) @@ -482,7 +767,8 @@ static int sun8i_ce_get_clks(struct sun8i_ce_dev *ce) static int sun8i_ce_register_algs(struct sun8i_ce_dev *ce) { - int ce_method, err, id, i; + int ce_method, err, id; + unsigned int i; for (i = 0; i < ARRAY_SIZE(ce_algs); i++) { ce_algs[i].ce = ce; @@ -515,6 +801,43 @@ static int sun8i_ce_register_algs(struct sun8i_ce_dev *ce) return err; } break; + case CRYPTO_ALG_TYPE_AHASH: + id = ce_algs[i].ce_algo_id; + ce_method = ce->variant->alg_hash[id]; + if (ce_method == CE_ID_NOTSUPP) { + dev_info(ce->dev, + "DEBUG: Algo of %s not supported\n", + ce_algs[i].alg.hash.halg.base.cra_name); + ce_algs[i].ce = NULL; + break; + } + dev_info(ce->dev, "Register %s\n", + ce_algs[i].alg.hash.halg.base.cra_name); + err = crypto_register_ahash(&ce_algs[i].alg.hash); + if (err) { + dev_err(ce->dev, "ERROR: Fail to register %s\n", + ce_algs[i].alg.hash.halg.base.cra_name); + ce_algs[i].ce = NULL; + return err; + } + break; + case CRYPTO_ALG_TYPE_RNG: + if (ce->variant->prng == CE_ID_NOTSUPP) { + dev_info(ce->dev, + "DEBUG: Algo of %s not supported\n", + ce_algs[i].alg.rng.base.cra_name); + ce_algs[i].ce = NULL; + break; + } + dev_info(ce->dev, "Register %s\n", + ce_algs[i].alg.rng.base.cra_name); + err = crypto_register_rng(&ce_algs[i].alg.rng); + if (err) { + dev_err(ce->dev, "Fail to register %s\n", + ce_algs[i].alg.rng.base.cra_name); + ce_algs[i].ce = NULL; + } + break; default: ce_algs[i].ce = NULL; dev_err(ce->dev, "ERROR: tried to register an unknown algo\n"); @@ -525,7 +848,7 @@ static int sun8i_ce_register_algs(struct sun8i_ce_dev *ce) static void sun8i_ce_unregister_algs(struct sun8i_ce_dev *ce) { - int i; + unsigned int i; for (i = 0; i < ARRAY_SIZE(ce_algs); i++) { if (!ce_algs[i].ce) @@ -536,6 +859,16 @@ static void sun8i_ce_unregister_algs(struct sun8i_ce_dev *ce) ce_algs[i].alg.skcipher.base.cra_name); crypto_unregister_skcipher(&ce_algs[i].alg.skcipher); break; + case CRYPTO_ALG_TYPE_AHASH: + dev_info(ce->dev, "Unregister %d %s\n", i, + ce_algs[i].alg.hash.halg.base.cra_name); + crypto_unregister_ahash(&ce_algs[i].alg.hash); + break; + case CRYPTO_ALG_TYPE_RNG: + dev_info(ce->dev, "Unregister %d %s\n", i, + ce_algs[i].alg.rng.base.cra_name); + crypto_unregister_rng(&ce_algs[i].alg.rng); + break; } } } @@ -573,14 +906,12 @@ static int sun8i_ce_probe(struct platform_device *pdev) return irq; ce->reset = devm_reset_control_get(&pdev->dev, NULL); - if (IS_ERR(ce->reset)) { - if (PTR_ERR(ce->reset) == -EPROBE_DEFER) - return PTR_ERR(ce->reset); - dev_err(&pdev->dev, "No reset control found\n"); - return PTR_ERR(ce->reset); - } + if (IS_ERR(ce->reset)) + return dev_err_probe(&pdev->dev, PTR_ERR(ce->reset), + "No reset control found\n"); mutex_init(&ce->mlock); + mutex_init(&ce->rnglock); err = sun8i_ce_allocate_chanlist(ce); if (err) @@ -605,6 +936,10 @@ static int sun8i_ce_probe(struct platform_device *pdev) if (err < 0) goto error_alg; +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG + sun8i_ce_hwrng_register(ce); +#endif + v = readl(ce->base + CE_CTR); v >>= CE_DIE_ID_SHIFT; v &= CE_DIE_ID_MASK; @@ -634,6 +969,10 @@ static int sun8i_ce_remove(struct platform_device *pdev) { struct sun8i_ce_dev *ce = platform_get_drvdata(pdev); +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG + sun8i_ce_hwrng_unregister(ce); +#endif + sun8i_ce_unregister_algs(ce); #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c new file mode 100644 index 000000000000..fa2f1b4fad7b --- /dev/null +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c @@ -0,0 +1,413 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * sun8i-ce-hash.c - hardware cryptographic offloader for + * Allwinner H3/A64/H5/H2+/H6/R40 SoC + * + * Copyright (C) 2015-2020 Corentin Labbe <[email protected]> + * + * This file add support for MD5 and SHA1/SHA224/SHA256/SHA384/SHA512. + * + * You could find the datasheet in Documentation/arm/sunxi/README + */ +#include <linux/dma-mapping.h> +#include <linux/pm_runtime.h> +#include <linux/scatterlist.h> +#include <crypto/internal/hash.h> +#include <crypto/sha.h> +#include <crypto/md5.h> +#include "sun8i-ce.h" + +int sun8i_ce_hash_crainit(struct crypto_tfm *tfm) +{ + struct sun8i_ce_hash_tfm_ctx *op = crypto_tfm_ctx(tfm); + struct ahash_alg *alg = __crypto_ahash_alg(tfm->__crt_alg); + struct sun8i_ce_alg_template *algt; + int err; + + memset(op, 0, sizeof(struct sun8i_ce_hash_tfm_ctx)); + + algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash); + op->ce = algt->ce; + + op->enginectx.op.do_one_request = sun8i_ce_hash_run; + op->enginectx.op.prepare_request = NULL; + op->enginectx.op.unprepare_request = NULL; + + /* FALLBACK */ + op->fallback_tfm = crypto_alloc_ahash(crypto_tfm_alg_name(tfm), 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(op->fallback_tfm)) { + dev_err(algt->ce->dev, "Fallback driver could no be loaded\n"); + return PTR_ERR(op->fallback_tfm); + } + + if (algt->alg.hash.halg.statesize < crypto_ahash_statesize(op->fallback_tfm)) + algt->alg.hash.halg.statesize = crypto_ahash_statesize(op->fallback_tfm); + + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct sun8i_ce_hash_reqctx) + + crypto_ahash_reqsize(op->fallback_tfm)); + + dev_info(op->ce->dev, "Fallback for %s is %s\n", + crypto_tfm_alg_driver_name(tfm), + crypto_tfm_alg_driver_name(&op->fallback_tfm->base)); + err = pm_runtime_get_sync(op->ce->dev); + if (err < 0) + goto error_pm; + return 0; +error_pm: + pm_runtime_put_noidle(op->ce->dev); + crypto_free_ahash(op->fallback_tfm); + return err; +} + +void sun8i_ce_hash_craexit(struct crypto_tfm *tfm) +{ + struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_tfm_ctx(tfm); + + crypto_free_ahash(tfmctx->fallback_tfm); + pm_runtime_put_sync_suspend(tfmctx->ce->dev); +} + +int sun8i_ce_hash_init(struct ahash_request *areq) +{ + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + + memset(rctx, 0, sizeof(struct sun8i_ce_hash_reqctx)); + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + + return crypto_ahash_init(&rctx->fallback_req); +} + +int sun8i_ce_hash_export(struct ahash_request *areq, void *out) +{ + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + + return crypto_ahash_export(&rctx->fallback_req, out); +} + +int sun8i_ce_hash_import(struct ahash_request *areq, const void *in) +{ + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + + return crypto_ahash_import(&rctx->fallback_req, in); +} + +int sun8i_ce_hash_final(struct ahash_request *areq) +{ + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG + struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg); + struct sun8i_ce_alg_template *algt; +#endif + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + rctx->fallback_req.base.flags = areq->base.flags & + CRYPTO_TFM_REQ_MAY_SLEEP; + rctx->fallback_req.result = areq->result; + +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG + algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash); + algt->stat_fb++; +#endif + + return crypto_ahash_final(&rctx->fallback_req); +} + +int sun8i_ce_hash_update(struct ahash_request *areq) +{ + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + rctx->fallback_req.base.flags = areq->base.flags & + CRYPTO_TFM_REQ_MAY_SLEEP; + rctx->fallback_req.nbytes = areq->nbytes; + rctx->fallback_req.src = areq->src; + + return crypto_ahash_update(&rctx->fallback_req); +} + +int sun8i_ce_hash_finup(struct ahash_request *areq) +{ + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG + struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg); + struct sun8i_ce_alg_template *algt; +#endif + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + rctx->fallback_req.base.flags = areq->base.flags & + CRYPTO_TFM_REQ_MAY_SLEEP; + + rctx->fallback_req.nbytes = areq->nbytes; + rctx->fallback_req.src = areq->src; + rctx->fallback_req.result = areq->result; +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG + algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash); + algt->stat_fb++; +#endif + + return crypto_ahash_finup(&rctx->fallback_req); +} + +static int sun8i_ce_hash_digest_fb(struct ahash_request *areq) +{ + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG + struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg); + struct sun8i_ce_alg_template *algt; +#endif + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + rctx->fallback_req.base.flags = areq->base.flags & + CRYPTO_TFM_REQ_MAY_SLEEP; + + rctx->fallback_req.nbytes = areq->nbytes; + rctx->fallback_req.src = areq->src; + rctx->fallback_req.result = areq->result; +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG + algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash); + algt->stat_fb++; +#endif + + return crypto_ahash_digest(&rctx->fallback_req); +} + +static bool sun8i_ce_hash_need_fallback(struct ahash_request *areq) +{ + struct scatterlist *sg; + + if (areq->nbytes == 0) + return true; + /* we need to reserve one SG for padding one */ + if (sg_nents(areq->src) > MAX_SG - 1) + return true; + sg = areq->src; + while (sg) { + if (sg->length % 4 || !IS_ALIGNED(sg->offset, sizeof(u32))) + return true; + sg = sg_next(sg); + } + return false; +} + +int sun8i_ce_hash_digest(struct ahash_request *areq) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg); + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); + struct sun8i_ce_alg_template *algt; + struct sun8i_ce_dev *ce; + struct crypto_engine *engine; + struct scatterlist *sg; + int nr_sgs, e, i; + + if (sun8i_ce_hash_need_fallback(areq)) + return sun8i_ce_hash_digest_fb(areq); + + nr_sgs = sg_nents(areq->src); + if (nr_sgs > MAX_SG - 1) + return sun8i_ce_hash_digest_fb(areq); + + for_each_sg(areq->src, sg, nr_sgs, i) { + if (sg->length % 4 || !IS_ALIGNED(sg->offset, sizeof(u32))) + return sun8i_ce_hash_digest_fb(areq); + } + + algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash); + ce = algt->ce; + + e = sun8i_ce_get_engine_number(ce); + rctx->flow = e; + engine = ce->chanlist[e].engine; + + return crypto_transfer_hash_request_to_engine(engine, areq); +} + +int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) +{ + struct ahash_request *areq = container_of(breq, struct ahash_request, base); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg); + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); + struct sun8i_ce_alg_template *algt; + struct sun8i_ce_dev *ce; + struct sun8i_ce_flow *chan; + struct ce_task *cet; + struct scatterlist *sg; + int nr_sgs, flow, err; + unsigned int len; + u32 common; + u64 byte_count; + __le32 *bf; + void *buf; + int j, i, todo; + int nbw = 0; + u64 fill, min_fill; + __be64 *bebits; + __le64 *lebits; + void *result; + u64 bs; + int digestsize; + dma_addr_t addr_res, addr_pad; + + algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash); + ce = algt->ce; + + bs = algt->alg.hash.halg.base.cra_blocksize; + digestsize = algt->alg.hash.halg.digestsize; + if (digestsize == SHA224_DIGEST_SIZE) + digestsize = SHA256_DIGEST_SIZE; + if (digestsize == SHA384_DIGEST_SIZE) + digestsize = SHA512_DIGEST_SIZE; + + /* the padding could be up to two block. */ + buf = kzalloc(bs * 2, GFP_KERNEL | GFP_DMA); + if (!buf) + return -ENOMEM; + bf = (__le32 *)buf; + + result = kzalloc(digestsize, GFP_KERNEL | GFP_DMA); + if (!result) + return -ENOMEM; + + flow = rctx->flow; + chan = &ce->chanlist[flow]; + +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG + algt->stat_req++; +#endif + dev_dbg(ce->dev, "%s %s len=%d\n", __func__, crypto_tfm_alg_name(areq->base.tfm), areq->nbytes); + + cet = chan->tl; + memset(cet, 0, sizeof(struct ce_task)); + + cet->t_id = cpu_to_le32(flow); + common = ce->variant->alg_hash[algt->ce_algo_id]; + common |= CE_COMM_INT; + cet->t_common_ctl = cpu_to_le32(common); + + cet->t_sym_ctl = 0; + cet->t_asym_ctl = 0; + + nr_sgs = dma_map_sg(ce->dev, areq->src, sg_nents(areq->src), DMA_TO_DEVICE); + if (nr_sgs <= 0 || nr_sgs > MAX_SG) { + dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs); + err = -EINVAL; + goto theend; + } + + len = areq->nbytes; + for_each_sg(areq->src, sg, nr_sgs, i) { + cet->t_src[i].addr = cpu_to_le32(sg_dma_address(sg)); + todo = min(len, sg_dma_len(sg)); + cet->t_src[i].len = cpu_to_le32(todo / 4); + len -= todo; + } + if (len > 0) { + dev_err(ce->dev, "remaining len %d\n", len); + err = -EINVAL; + goto theend; + } + addr_res = dma_map_single(ce->dev, result, digestsize, DMA_FROM_DEVICE); + cet->t_dst[0].addr = cpu_to_le32(addr_res); + cet->t_dst[0].len = cpu_to_le32(digestsize / 4); + if (dma_mapping_error(ce->dev, addr_res)) { + dev_err(ce->dev, "DMA map dest\n"); + err = -EINVAL; + goto theend; + } + + byte_count = areq->nbytes; + j = 0; + bf[j++] = cpu_to_le32(0x80); + + if (bs == 64) { + fill = 64 - (byte_count % 64); + min_fill = 2 * sizeof(u32) + (nbw ? 0 : sizeof(u32)); + } else { + fill = 128 - (byte_count % 128); + min_fill = 4 * sizeof(u32) + (nbw ? 0 : sizeof(u32)); + } + + if (fill < min_fill) + fill += bs; + + j += (fill - min_fill) / sizeof(u32); + + switch (algt->ce_algo_id) { + case CE_ID_HASH_MD5: + lebits = (__le64 *)&bf[j]; + *lebits = cpu_to_le64(byte_count << 3); + j += 2; + break; + case CE_ID_HASH_SHA1: + case CE_ID_HASH_SHA224: + case CE_ID_HASH_SHA256: + bebits = (__be64 *)&bf[j]; + *bebits = cpu_to_be64(byte_count << 3); + j += 2; + break; + case CE_ID_HASH_SHA384: + case CE_ID_HASH_SHA512: + bebits = (__be64 *)&bf[j]; + *bebits = cpu_to_be64(byte_count >> 61); + j += 2; + bebits = (__be64 *)&bf[j]; + *bebits = cpu_to_be64(byte_count << 3); + j += 2; + break; + } + + addr_pad = dma_map_single(ce->dev, buf, j * 4, DMA_TO_DEVICE); + cet->t_src[i].addr = cpu_to_le32(addr_pad); + cet->t_src[i].len = cpu_to_le32(j); + if (dma_mapping_error(ce->dev, addr_pad)) { + dev_err(ce->dev, "DMA error on padding SG\n"); + err = -EINVAL; + goto theend; + } + + if (ce->variant->hash_t_dlen_in_bits) + cet->t_dlen = cpu_to_le32((areq->nbytes + j * 4) * 8); + else + cet->t_dlen = cpu_to_le32(areq->nbytes / 4 + j); + + chan->timeout = areq->nbytes; + + err = sun8i_ce_run_task(ce, flow, crypto_tfm_alg_name(areq->base.tfm)); + + dma_unmap_single(ce->dev, addr_pad, j * 4, DMA_TO_DEVICE); + dma_unmap_sg(ce->dev, areq->src, nr_sgs, DMA_TO_DEVICE); + dma_unmap_single(ce->dev, addr_res, digestsize, DMA_FROM_DEVICE); + + kfree(buf); + + memcpy(areq->result, result, algt->alg.hash.halg.digestsize); + kfree(result); +theend: + crypto_finalize_hash_request(engine, breq, err); + return 0; +} diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-prng.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-prng.c new file mode 100644 index 000000000000..78503006949c --- /dev/null +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-prng.c @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * sun8i-ce-prng.c - hardware cryptographic offloader for + * Allwinner H3/A64/H5/H2+/H6/R40 SoC + * + * Copyright (C) 2015-2020 Corentin Labbe <[email protected]> + * + * This file handle the PRNG + * + * You could find a link for the datasheet in Documentation/arm/sunxi/README + */ +#include "sun8i-ce.h" +#include <linux/dma-mapping.h> +#include <linux/pm_runtime.h> +#include <crypto/internal/rng.h> + +int sun8i_ce_prng_init(struct crypto_tfm *tfm) +{ + struct sun8i_ce_rng_tfm_ctx *ctx = crypto_tfm_ctx(tfm); + + memset(ctx, 0, sizeof(struct sun8i_ce_rng_tfm_ctx)); + return 0; +} + +void sun8i_ce_prng_exit(struct crypto_tfm *tfm) +{ + struct sun8i_ce_rng_tfm_ctx *ctx = crypto_tfm_ctx(tfm); + + memzero_explicit(ctx->seed, ctx->slen); + kfree(ctx->seed); + ctx->seed = NULL; + ctx->slen = 0; +} + +int sun8i_ce_prng_seed(struct crypto_rng *tfm, const u8 *seed, + unsigned int slen) +{ + struct sun8i_ce_rng_tfm_ctx *ctx = crypto_rng_ctx(tfm); + + if (ctx->seed && ctx->slen != slen) { + memzero_explicit(ctx->seed, ctx->slen); + kfree(ctx->seed); + ctx->slen = 0; + ctx->seed = NULL; + } + if (!ctx->seed) + ctx->seed = kmalloc(slen, GFP_KERNEL | GFP_DMA); + if (!ctx->seed) + return -ENOMEM; + + memcpy(ctx->seed, seed, slen); + ctx->slen = slen; + + return 0; +} + +int sun8i_ce_prng_generate(struct crypto_rng *tfm, const u8 *src, + unsigned int slen, u8 *dst, unsigned int dlen) +{ + struct sun8i_ce_rng_tfm_ctx *ctx = crypto_rng_ctx(tfm); + struct rng_alg *alg = crypto_rng_alg(tfm); + struct sun8i_ce_alg_template *algt; + struct sun8i_ce_dev *ce; + dma_addr_t dma_iv, dma_dst; + int err = 0; + int flow = 3; + unsigned int todo; + struct sun8i_ce_flow *chan; + struct ce_task *cet; + u32 common, sym; + void *d; + + algt = container_of(alg, struct sun8i_ce_alg_template, alg.rng); + ce = algt->ce; + + if (ctx->slen == 0) { + dev_err(ce->dev, "not seeded\n"); + return -EINVAL; + } + + /* we want dlen + seedsize rounded up to a multiple of PRNG_DATA_SIZE */ + todo = dlen + ctx->slen + PRNG_DATA_SIZE * 2; + todo -= todo % PRNG_DATA_SIZE; + + d = kzalloc(todo, GFP_KERNEL | GFP_DMA); + if (!d) { + err = -ENOMEM; + goto err_mem; + } + + dev_dbg(ce->dev, "%s PRNG slen=%u dlen=%u todo=%u multi=%u\n", __func__, + slen, dlen, todo, todo / PRNG_DATA_SIZE); + +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG + algt->stat_req++; + algt->stat_bytes += todo; +#endif + + dma_iv = dma_map_single(ce->dev, ctx->seed, ctx->slen, DMA_TO_DEVICE); + if (dma_mapping_error(ce->dev, dma_iv)) { + dev_err(ce->dev, "Cannot DMA MAP IV\n"); + goto err_iv; + } + + dma_dst = dma_map_single(ce->dev, d, todo, DMA_FROM_DEVICE); + if (dma_mapping_error(ce->dev, dma_dst)) { + dev_err(ce->dev, "Cannot DMA MAP DST\n"); + err = -EFAULT; + goto err_dst; + } + + err = pm_runtime_get_sync(ce->dev); + if (err < 0) { + pm_runtime_put_noidle(ce->dev); + goto err_pm; + } + + mutex_lock(&ce->rnglock); + chan = &ce->chanlist[flow]; + + cet = &chan->tl[0]; + memset(cet, 0, sizeof(struct ce_task)); + + cet->t_id = cpu_to_le32(flow); + common = ce->variant->prng | CE_COMM_INT; + cet->t_common_ctl = cpu_to_le32(common); + + /* recent CE (H6) need length in bytes, in word otherwise */ + if (ce->variant->prng_t_dlen_in_bytes) + cet->t_dlen = cpu_to_le32(todo); + else + cet->t_dlen = cpu_to_le32(todo / 4); + + sym = PRNG_LD; + cet->t_sym_ctl = cpu_to_le32(sym); + cet->t_asym_ctl = 0; + + cet->t_key = cpu_to_le32(dma_iv); + cet->t_iv = cpu_to_le32(dma_iv); + + cet->t_dst[0].addr = cpu_to_le32(dma_dst); + cet->t_dst[0].len = cpu_to_le32(todo / 4); + ce->chanlist[flow].timeout = 2000; + + err = sun8i_ce_run_task(ce, 3, "PRNG"); + mutex_unlock(&ce->rnglock); + + pm_runtime_put(ce->dev); + +err_pm: + dma_unmap_single(ce->dev, dma_dst, todo, DMA_FROM_DEVICE); +err_dst: + dma_unmap_single(ce->dev, dma_iv, ctx->slen, DMA_TO_DEVICE); + + if (!err) { + memcpy(dst, d, dlen); + memcpy(ctx->seed, d + dlen, ctx->slen); + } + memzero_explicit(d, todo); +err_iv: + kfree(d); +err_mem: + return err; +} diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-trng.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-trng.c new file mode 100644 index 000000000000..654328160d19 --- /dev/null +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-trng.c @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * sun8i-ce-trng.c - hardware cryptographic offloader for + * Allwinner H3/A64/H5/H2+/H6/R40 SoC + * + * Copyright (C) 2015-2020 Corentin Labbe <[email protected]> + * + * This file handle the TRNG + * + * You could find a link for the datasheet in Documentation/arm/sunxi/README + */ +#include "sun8i-ce.h" +#include <linux/dma-mapping.h> +#include <linux/pm_runtime.h> +#include <linux/hw_random.h> +/* + * Note that according to the algorithm ID, 2 versions of the TRNG exists, + * The first present in H3/H5/R40/A64 and the second present in H6. + * This file adds support for both, but only the second is working + * reliabily according to rngtest. + **/ + +static int sun8i_ce_trng_read(struct hwrng *rng, void *data, size_t max, bool wait) +{ + struct sun8i_ce_dev *ce; + dma_addr_t dma_dst; + int err = 0; + int flow = 3; + unsigned int todo; + struct sun8i_ce_flow *chan; + struct ce_task *cet; + u32 common; + void *d; + + ce = container_of(rng, struct sun8i_ce_dev, trng); + + /* round the data length to a multiple of 32*/ + todo = max + 32; + todo -= todo % 32; + + d = kzalloc(todo, GFP_KERNEL | GFP_DMA); + if (!d) + return -ENOMEM; + +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG + ce->hwrng_stat_req++; + ce->hwrng_stat_bytes += todo; +#endif + + dma_dst = dma_map_single(ce->dev, d, todo, DMA_FROM_DEVICE); + if (dma_mapping_error(ce->dev, dma_dst)) { + dev_err(ce->dev, "Cannot DMA MAP DST\n"); + err = -EFAULT; + goto err_dst; + } + + err = pm_runtime_get_sync(ce->dev); + if (err < 0) { + pm_runtime_put_noidle(ce->dev); + goto err_pm; + } + + mutex_lock(&ce->rnglock); + chan = &ce->chanlist[flow]; + + cet = &chan->tl[0]; + memset(cet, 0, sizeof(struct ce_task)); + + cet->t_id = cpu_to_le32(flow); + common = ce->variant->trng | CE_COMM_INT; + cet->t_common_ctl = cpu_to_le32(common); + + /* recent CE (H6) need length in bytes, in word otherwise */ + if (ce->variant->trng_t_dlen_in_bytes) + cet->t_dlen = cpu_to_le32(todo); + else + cet->t_dlen = cpu_to_le32(todo / 4); + + cet->t_sym_ctl = 0; + cet->t_asym_ctl = 0; + + cet->t_dst[0].addr = cpu_to_le32(dma_dst); + cet->t_dst[0].len = cpu_to_le32(todo / 4); + ce->chanlist[flow].timeout = todo; + + err = sun8i_ce_run_task(ce, 3, "TRNG"); + mutex_unlock(&ce->rnglock); + + pm_runtime_put(ce->dev); + +err_pm: + dma_unmap_single(ce->dev, dma_dst, todo, DMA_FROM_DEVICE); + + if (!err) { + memcpy(data, d, max); + err = max; + } + memzero_explicit(d, todo); +err_dst: + kfree(d); + return err; +} + +int sun8i_ce_hwrng_register(struct sun8i_ce_dev *ce) +{ + int ret; + + if (ce->variant->trng == CE_ID_NOTSUPP) { + dev_info(ce->dev, "TRNG not supported\n"); + return 0; + } + ce->trng.name = "sun8i Crypto Engine TRNG"; + ce->trng.read = sun8i_ce_trng_read; + ce->trng.quality = 1000; + + ret = hwrng_register(&ce->trng); + if (ret) + dev_err(ce->dev, "Fail to register the TRNG\n"); + return ret; +} + +void sun8i_ce_hwrng_unregister(struct sun8i_ce_dev *ce) +{ + if (ce->variant->trng == CE_ID_NOTSUPP) + return; + hwrng_unregister(&ce->trng); +} diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h index 963645fe4adb..558027516aed 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h @@ -12,6 +12,11 @@ #include <linux/atomic.h> #include <linux/debugfs.h> #include <linux/crypto.h> +#include <linux/hw_random.h> +#include <crypto/internal/hash.h> +#include <crypto/md5.h> +#include <crypto/rng.h> +#include <crypto/sha.h> /* CE Registers */ #define CE_TDQ 0x00 @@ -45,6 +50,16 @@ #define CE_ALG_AES 0 #define CE_ALG_DES 1 #define CE_ALG_3DES 2 +#define CE_ALG_MD5 16 +#define CE_ALG_SHA1 17 +#define CE_ALG_SHA224 18 +#define CE_ALG_SHA256 19 +#define CE_ALG_SHA384 20 +#define CE_ALG_SHA512 21 +#define CE_ALG_TRNG 48 +#define CE_ALG_PRNG 49 +#define CE_ALG_TRNG_V2 0x1c +#define CE_ALG_PRNG_V2 0x1d /* Used in ce_variant */ #define CE_ID_NOTSUPP 0xFF @@ -54,6 +69,14 @@ #define CE_ID_CIPHER_DES3 2 #define CE_ID_CIPHER_MAX 3 +#define CE_ID_HASH_MD5 0 +#define CE_ID_HASH_SHA1 1 +#define CE_ID_HASH_SHA224 2 +#define CE_ID_HASH_SHA256 3 +#define CE_ID_HASH_SHA384 4 +#define CE_ID_HASH_SHA512 5 +#define CE_ID_HASH_MAX 6 + #define CE_ID_OP_ECB 0 #define CE_ID_OP_CBC 1 #define CE_ID_OP_MAX 2 @@ -65,6 +88,16 @@ #define CE_ERR_ADDR_INVALID BIT(5) #define CE_ERR_KEYLADDER BIT(6) +#define ESR_H3 0 +#define ESR_A64 1 +#define ESR_R40 2 +#define ESR_H5 3 +#define ESR_H6 4 + +#define PRNG_DATA_SIZE (160 / 8) +#define PRNG_SEED_SIZE DIV_ROUND_UP(175, 8) +#define PRNG_LD BIT(17) + #define CE_DIE_ID_SHIFT 16 #define CE_DIE_ID_MASK 0x07 @@ -90,16 +123,34 @@ struct ce_clock { * struct ce_variant - Describe CE capability for each variant hardware * @alg_cipher: list of supported ciphers. for each CE_ID_ this will give the * coresponding CE_ALG_XXX value + * @alg_hash: list of supported hashes. for each CE_ID_ this will give the + * corresponding CE_ALG_XXX value * @op_mode: list of supported block modes - * @has_t_dlen_in_bytes: Does the request size for cipher is in + * @cipher_t_dlen_in_bytes: Does the request size for cipher is in + * bytes or words + * @hash_t_dlen_in_bytes: Does the request size for hash is in + * bits or words + * @prng_t_dlen_in_bytes: Does the request size for PRNG is in + * bytes or words + * @trng_t_dlen_in_bytes: Does the request size for TRNG is in * bytes or words * @ce_clks: list of clocks needed by this variant + * @esr: The type of error register + * @prng: The CE_ALG_XXX value for the PRNG + * @trng: The CE_ALG_XXX value for the TRNG */ struct ce_variant { char alg_cipher[CE_ID_CIPHER_MAX]; + char alg_hash[CE_ID_HASH_MAX]; u32 op_mode[CE_ID_OP_MAX]; - bool has_t_dlen_in_bytes; + bool cipher_t_dlen_in_bytes; + bool hash_t_dlen_in_bits; + bool prng_t_dlen_in_bytes; + bool trng_t_dlen_in_bytes; struct ce_clock ce_clks[CE_MAX_CLOCKS]; + int esr; + unsigned char prng; + unsigned char trng; }; struct sginfo { @@ -129,8 +180,6 @@ struct ce_task { /* * struct sun8i_ce_flow - Information used by each flow * @engine: ptr to the crypto_engine for this flow - * @bounce_iv: buffer which contain the IV - * @ivlen: size of bounce_iv * @complete: completion for the current task on this flow * @status: set to 1 by interrupt if task is done * @t_phy: Physical address of task @@ -139,8 +188,6 @@ struct ce_task { */ struct sun8i_ce_flow { struct crypto_engine *engine; - void *bounce_iv; - unsigned int ivlen; struct completion complete; int status; dma_addr_t t_phy; @@ -158,6 +205,7 @@ struct sun8i_ce_flow { * @reset: pointer to reset controller * @dev: the platform device * @mlock: Control access to device registers + * @rnglock: Control access to the RNG (dedicated channel 3) * @chanlist: array of all flow * @flow: flow to use in next request * @variant: pointer to variant specific data @@ -170,6 +218,7 @@ struct sun8i_ce_dev { struct reset_control *reset; struct device *dev; struct mutex mlock; + struct mutex rnglock; struct sun8i_ce_flow *chanlist; atomic_t flow; const struct ce_variant *variant; @@ -177,17 +226,38 @@ struct sun8i_ce_dev { struct dentry *dbgfs_dir; struct dentry *dbgfs_stats; #endif +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG + struct hwrng trng; +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG + unsigned long hwrng_stat_req; + unsigned long hwrng_stat_bytes; +#endif +#endif }; /* * struct sun8i_cipher_req_ctx - context for a skcipher request * @op_dir: direction (encrypt vs decrypt) for this request * @flow: the flow to use for this request + * @backup_iv: buffer which contain the next IV to store + * @bounce_iv: buffer which contain the IV + * @ivlen: size of bounce_iv + * @nr_sgs: The number of source SG (as given by dma_map_sg()) + * @nr_sgd: The number of destination SG (as given by dma_map_sg()) + * @addr_iv: The IV addr returned by dma_map_single, need to unmap later + * @addr_key: The key addr returned by dma_map_single, need to unmap later * @fallback_req: request struct for invoking the fallback skcipher TFM */ struct sun8i_cipher_req_ctx { u32 op_dir; int flow; + void *backup_iv; + void *bounce_iv; + unsigned int ivlen; + int nr_sgs; + int nr_sgd; + dma_addr_t addr_iv; + dma_addr_t addr_key; struct skcipher_request fallback_req; // keep at the end }; @@ -208,6 +278,38 @@ struct sun8i_cipher_tfm_ctx { }; /* + * struct sun8i_ce_hash_tfm_ctx - context for an ahash TFM + * @enginectx: crypto_engine used by this TFM + * @ce: pointer to the private data of driver handling this TFM + * @fallback_tfm: pointer to the fallback TFM + */ +struct sun8i_ce_hash_tfm_ctx { + struct crypto_engine_ctx enginectx; + struct sun8i_ce_dev *ce; + struct crypto_ahash *fallback_tfm; +}; + +/* + * struct sun8i_ce_hash_reqctx - context for an ahash request + * @fallback_req: pre-allocated fallback request + * @flow: the flow to use for this request + */ +struct sun8i_ce_hash_reqctx { + struct ahash_request fallback_req; + int flow; +}; + +/* + * struct sun8i_ce_prng_ctx - context for PRNG TFM + * @seed: The seed to use + * @slen: The size of the seed + */ +struct sun8i_ce_rng_tfm_ctx { + void *seed; + unsigned int slen; +}; + +/* * struct sun8i_ce_alg_template - crypto_alg template * @type: the CRYPTO_ALG_TYPE for this template * @ce_algo_id: the CE_ID for this template @@ -217,6 +319,7 @@ struct sun8i_cipher_tfm_ctx { * @alg: one of sub struct must be used * @stat_req: number of request done on this template * @stat_fb: number of request which has fallbacked + * @stat_bytes: total data size done by this template */ struct sun8i_ce_alg_template { u32 type; @@ -225,10 +328,13 @@ struct sun8i_ce_alg_template { struct sun8i_ce_dev *ce; union { struct skcipher_alg skcipher; + struct ahash_alg hash; + struct rng_alg rng; } alg; #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG unsigned long stat_req; unsigned long stat_fb; + unsigned long stat_bytes; #endif }; @@ -246,3 +352,24 @@ int sun8i_ce_skencrypt(struct skcipher_request *areq); int sun8i_ce_get_engine_number(struct sun8i_ce_dev *ce); int sun8i_ce_run_task(struct sun8i_ce_dev *ce, int flow, const char *name); + +int sun8i_ce_hash_crainit(struct crypto_tfm *tfm); +void sun8i_ce_hash_craexit(struct crypto_tfm *tfm); +int sun8i_ce_hash_init(struct ahash_request *areq); +int sun8i_ce_hash_export(struct ahash_request *areq, void *out); +int sun8i_ce_hash_import(struct ahash_request *areq, const void *in); +int sun8i_ce_hash(struct ahash_request *areq); +int sun8i_ce_hash_final(struct ahash_request *areq); +int sun8i_ce_hash_update(struct ahash_request *areq); +int sun8i_ce_hash_finup(struct ahash_request *areq); +int sun8i_ce_hash_digest(struct ahash_request *areq); +int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq); + +int sun8i_ce_prng_generate(struct crypto_rng *tfm, const u8 *src, + unsigned int slen, u8 *dst, unsigned int dlen); +int sun8i_ce_prng_seed(struct crypto_rng *tfm, const u8 *seed, unsigned int slen); +void sun8i_ce_prng_exit(struct crypto_tfm *tfm); +int sun8i_ce_prng_init(struct crypto_tfm *tfm); + +int sun8i_ce_hwrng_register(struct sun8i_ce_dev *ce); +void sun8i_ce_hwrng_unregister(struct sun8i_ce_dev *ce); diff --git a/drivers/crypto/allwinner/sun8i-ss/Makefile b/drivers/crypto/allwinner/sun8i-ss/Makefile index add7b0543fd5..aabfd893c817 100644 --- a/drivers/crypto/allwinner/sun8i-ss/Makefile +++ b/drivers/crypto/allwinner/sun8i-ss/Makefile @@ -1,2 +1,4 @@ obj-$(CONFIG_CRYPTO_DEV_SUN8I_SS) += sun8i-ss.o sun8i-ss-y += sun8i-ss-core.o sun8i-ss-cipher.o +sun8i-ss-$(CONFIG_CRYPTO_DEV_SUN8I_SS_PRNG) += sun8i-ss-prng.o +sun8i-ss-$(CONFIG_CRYPTO_DEV_SUN8I_SS_HASH) += sun8i-ss-hash.o diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c index 7b39b4495571..ed2a69f82e1c 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c @@ -248,7 +248,6 @@ theend_iv: offset = areq->cryptlen - ivsize; if (rctx->op_dir & SS_DECRYPTION) { memcpy(areq->iv, backup_iv, ivsize); - memzero_explicit(backup_iv, ivsize); kfree_sensitive(backup_iv); } else { scatterwalk_map_and_copy(areq->iv, areq->dst, offset, @@ -368,10 +367,7 @@ void sun8i_ss_cipher_exit(struct crypto_tfm *tfm) { struct sun8i_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm); - if (op->key) { - memzero_explicit(op->key, op->keylen); - kfree(op->key); - } + kfree_sensitive(op->key); crypto_free_skcipher(op->fallback_tfm); pm_runtime_put_sync(op->ss->dev); } @@ -393,10 +389,7 @@ int sun8i_ss_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, dev_dbg(ss->dev, "ERROR: Invalid keylen %u\n", keylen); return -EINVAL; } - if (op->key) { - memzero_explicit(op->key, op->keylen); - kfree(op->key); - } + kfree_sensitive(op->key); op->keylen = keylen; op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA); if (!op->key) @@ -419,10 +412,7 @@ int sun8i_ss_des3_setkey(struct crypto_skcipher *tfm, const u8 *key, return -EINVAL; } - if (op->key) { - memzero_explicit(op->key, op->keylen); - kfree(op->key); - } + kfree_sensitive(op->key); op->keylen = keylen; op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA); if (!op->key) diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c index 9a23515783a6..e0ddc684798d 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c @@ -22,6 +22,7 @@ #include <linux/platform_device.h> #include <linux/pm_runtime.h> #include <linux/reset.h> +#include <crypto/internal/rng.h> #include <crypto/internal/skcipher.h> #include "sun8i-ss.h" @@ -40,6 +41,8 @@ static const struct ss_variant ss_a80_variant = { static const struct ss_variant ss_a83t_variant = { .alg_cipher = { SS_ALG_AES, SS_ALG_DES, SS_ALG_3DES, }, + .alg_hash = { SS_ALG_MD5, SS_ALG_SHA1, SS_ALG_SHA224, SS_ALG_SHA256, + }, .op_mode = { SS_OP_ECB, SS_OP_CBC, }, .ss_clks = { @@ -61,7 +64,7 @@ int sun8i_ss_run_task(struct sun8i_ss_dev *ss, struct sun8i_cipher_req_ctx *rctx const char *name) { int flow = rctx->flow; - u32 v = 1; + u32 v = SS_START; int i; #ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG @@ -264,13 +267,154 @@ static struct sun8i_ss_alg_template ss_algs[] = { .decrypt = sun8i_ss_skdecrypt, } }, +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_PRNG +{ + .type = CRYPTO_ALG_TYPE_RNG, + .alg.rng = { + .base = { + .cra_name = "stdrng", + .cra_driver_name = "sun8i-ss-prng", + .cra_priority = 300, + .cra_ctxsize = sizeof(struct sun8i_ss_rng_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_init = sun8i_ss_prng_init, + .cra_exit = sun8i_ss_prng_exit, + }, + .generate = sun8i_ss_prng_generate, + .seed = sun8i_ss_prng_seed, + .seedsize = PRNG_SEED_SIZE, + } +}, +#endif +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_HASH +{ .type = CRYPTO_ALG_TYPE_AHASH, + .ss_algo_id = SS_ID_HASH_MD5, + .alg.hash = { + .init = sun8i_ss_hash_init, + .update = sun8i_ss_hash_update, + .final = sun8i_ss_hash_final, + .finup = sun8i_ss_hash_finup, + .digest = sun8i_ss_hash_digest, + .export = sun8i_ss_hash_export, + .import = sun8i_ss_hash_import, + .halg = { + .digestsize = MD5_DIGEST_SIZE, + .statesize = sizeof(struct md5_state), + .base = { + .cra_name = "md5", + .cra_driver_name = "md5-sun8i-ss", + .cra_priority = 300, + .cra_alignmask = 3, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = MD5_HMAC_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun8i_ss_hash_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_init = sun8i_ss_hash_crainit, + .cra_exit = sun8i_ss_hash_craexit, + } + } + } +}, +{ .type = CRYPTO_ALG_TYPE_AHASH, + .ss_algo_id = SS_ID_HASH_SHA1, + .alg.hash = { + .init = sun8i_ss_hash_init, + .update = sun8i_ss_hash_update, + .final = sun8i_ss_hash_final, + .finup = sun8i_ss_hash_finup, + .digest = sun8i_ss_hash_digest, + .export = sun8i_ss_hash_export, + .import = sun8i_ss_hash_import, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct sha1_state), + .base = { + .cra_name = "sha1", + .cra_driver_name = "sha1-sun8i-ss", + .cra_priority = 300, + .cra_alignmask = 3, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun8i_ss_hash_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_init = sun8i_ss_hash_crainit, + .cra_exit = sun8i_ss_hash_craexit, + } + } + } +}, +{ .type = CRYPTO_ALG_TYPE_AHASH, + .ss_algo_id = SS_ID_HASH_SHA224, + .alg.hash = { + .init = sun8i_ss_hash_init, + .update = sun8i_ss_hash_update, + .final = sun8i_ss_hash_final, + .finup = sun8i_ss_hash_finup, + .digest = sun8i_ss_hash_digest, + .export = sun8i_ss_hash_export, + .import = sun8i_ss_hash_import, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha224", + .cra_driver_name = "sha224-sun8i-ss", + .cra_priority = 300, + .cra_alignmask = 3, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun8i_ss_hash_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_init = sun8i_ss_hash_crainit, + .cra_exit = sun8i_ss_hash_craexit, + } + } + } +}, +{ .type = CRYPTO_ALG_TYPE_AHASH, + .ss_algo_id = SS_ID_HASH_SHA256, + .alg.hash = { + .init = sun8i_ss_hash_init, + .update = sun8i_ss_hash_update, + .final = sun8i_ss_hash_final, + .finup = sun8i_ss_hash_finup, + .digest = sun8i_ss_hash_digest, + .export = sun8i_ss_hash_export, + .import = sun8i_ss_hash_import, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha256", + .cra_driver_name = "sha256-sun8i-ss", + .cra_priority = 300, + .cra_alignmask = 3, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun8i_ss_hash_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_init = sun8i_ss_hash_crainit, + .cra_exit = sun8i_ss_hash_craexit, + } + } + } +}, +#endif }; #ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG -static int sun8i_ss_dbgfs_read(struct seq_file *seq, void *v) +static int sun8i_ss_debugfs_show(struct seq_file *seq, void *v) { struct sun8i_ss_dev *ss = seq->private; - int i; + unsigned int i; for (i = 0; i < MAXFLOW; i++) seq_printf(seq, "Channel %d: nreq %lu\n", i, ss->flows[i].stat_req); @@ -280,28 +424,29 @@ static int sun8i_ss_dbgfs_read(struct seq_file *seq, void *v) continue; switch (ss_algs[i].type) { case CRYPTO_ALG_TYPE_SKCIPHER: - seq_printf(seq, "%s %s %lu %lu\n", + seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n", ss_algs[i].alg.skcipher.base.cra_driver_name, ss_algs[i].alg.skcipher.base.cra_name, ss_algs[i].stat_req, ss_algs[i].stat_fb); break; + case CRYPTO_ALG_TYPE_RNG: + seq_printf(seq, "%s %s reqs=%lu tsize=%lu\n", + ss_algs[i].alg.rng.base.cra_driver_name, + ss_algs[i].alg.rng.base.cra_name, + ss_algs[i].stat_req, ss_algs[i].stat_bytes); + break; + case CRYPTO_ALG_TYPE_AHASH: + seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n", + ss_algs[i].alg.hash.halg.base.cra_driver_name, + ss_algs[i].alg.hash.halg.base.cra_name, + ss_algs[i].stat_req, ss_algs[i].stat_fb); + break; } } return 0; } -static int sun8i_ss_dbgfs_open(struct inode *inode, struct file *file) -{ - return single_open(file, sun8i_ss_dbgfs_read, inode->i_private); -} - -static const struct file_operations sun8i_ss_debugfs_fops = { - .owner = THIS_MODULE, - .open = sun8i_ss_dbgfs_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(sun8i_ss_debugfs); #endif static void sun8i_ss_free_flows(struct sun8i_ss_dev *ss, int i) @@ -415,7 +560,8 @@ static void sun8i_ss_pm_exit(struct sun8i_ss_dev *ss) static int sun8i_ss_register_algs(struct sun8i_ss_dev *ss) { - int ss_method, err, id, i; + int ss_method, err, id; + unsigned int i; for (i = 0; i < ARRAY_SIZE(ss_algs); i++) { ss_algs[i].ss = ss; @@ -448,6 +594,34 @@ static int sun8i_ss_register_algs(struct sun8i_ss_dev *ss) return err; } break; + case CRYPTO_ALG_TYPE_RNG: + err = crypto_register_rng(&ss_algs[i].alg.rng); + if (err) { + dev_err(ss->dev, "Fail to register %s\n", + ss_algs[i].alg.rng.base.cra_name); + ss_algs[i].ss = NULL; + } + break; + case CRYPTO_ALG_TYPE_AHASH: + id = ss_algs[i].ss_algo_id; + ss_method = ss->variant->alg_hash[id]; + if (ss_method == SS_ID_NOTSUPP) { + dev_info(ss->dev, + "DEBUG: Algo of %s not supported\n", + ss_algs[i].alg.hash.halg.base.cra_name); + ss_algs[i].ss = NULL; + break; + } + dev_info(ss->dev, "Register %s\n", + ss_algs[i].alg.hash.halg.base.cra_name); + err = crypto_register_ahash(&ss_algs[i].alg.hash); + if (err) { + dev_err(ss->dev, "ERROR: Fail to register %s\n", + ss_algs[i].alg.hash.halg.base.cra_name); + ss_algs[i].ss = NULL; + return err; + } + break; default: ss_algs[i].ss = NULL; dev_err(ss->dev, "ERROR: tried to register an unknown algo\n"); @@ -458,7 +632,7 @@ static int sun8i_ss_register_algs(struct sun8i_ss_dev *ss) static void sun8i_ss_unregister_algs(struct sun8i_ss_dev *ss) { - int i; + unsigned int i; for (i = 0; i < ARRAY_SIZE(ss_algs); i++) { if (!ss_algs[i].ss) @@ -469,6 +643,16 @@ static void sun8i_ss_unregister_algs(struct sun8i_ss_dev *ss) ss_algs[i].alg.skcipher.base.cra_name); crypto_unregister_skcipher(&ss_algs[i].alg.skcipher); break; + case CRYPTO_ALG_TYPE_RNG: + dev_info(ss->dev, "Unregister %d %s\n", i, + ss_algs[i].alg.rng.base.cra_name); + crypto_unregister_rng(&ss_algs[i].alg.rng); + break; + case CRYPTO_ALG_TYPE_AHASH: + dev_info(ss->dev, "Unregister %d %s\n", i, + ss_algs[i].alg.hash.halg.base.cra_name); + crypto_unregister_ahash(&ss_algs[i].alg.hash); + break; } } } @@ -545,12 +729,9 @@ static int sun8i_ss_probe(struct platform_device *pdev) return irq; ss->reset = devm_reset_control_get(&pdev->dev, NULL); - if (IS_ERR(ss->reset)) { - if (PTR_ERR(ss->reset) == -EPROBE_DEFER) - return PTR_ERR(ss->reset); - dev_err(&pdev->dev, "No reset control found\n"); - return PTR_ERR(ss->reset); - } + if (IS_ERR(ss->reset)) + return dev_err_probe(&pdev->dev, PTR_ERR(ss->reset), + "No reset control found\n"); mutex_init(&ss->mlock); diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c new file mode 100644 index 000000000000..b6ab2054f217 --- /dev/null +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c @@ -0,0 +1,444 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * sun8i-ss-hash.c - hardware cryptographic offloader for + * Allwinner A80/A83T SoC + * + * Copyright (C) 2015-2020 Corentin Labbe <[email protected]> + * + * This file add support for MD5 and SHA1/SHA224/SHA256. + * + * You could find the datasheet in Documentation/arm/sunxi.rst + */ +#include <linux/dma-mapping.h> +#include <linux/pm_runtime.h> +#include <linux/scatterlist.h> +#include <crypto/internal/hash.h> +#include <crypto/sha.h> +#include <crypto/md5.h> +#include "sun8i-ss.h" + +int sun8i_ss_hash_crainit(struct crypto_tfm *tfm) +{ + struct sun8i_ss_hash_tfm_ctx *op = crypto_tfm_ctx(tfm); + struct ahash_alg *alg = __crypto_ahash_alg(tfm->__crt_alg); + struct sun8i_ss_alg_template *algt; + int err; + + memset(op, 0, sizeof(struct sun8i_ss_hash_tfm_ctx)); + + algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash); + op->ss = algt->ss; + + op->enginectx.op.do_one_request = sun8i_ss_hash_run; + op->enginectx.op.prepare_request = NULL; + op->enginectx.op.unprepare_request = NULL; + + /* FALLBACK */ + op->fallback_tfm = crypto_alloc_ahash(crypto_tfm_alg_name(tfm), 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(op->fallback_tfm)) { + dev_err(algt->ss->dev, "Fallback driver could no be loaded\n"); + return PTR_ERR(op->fallback_tfm); + } + + if (algt->alg.hash.halg.statesize < crypto_ahash_statesize(op->fallback_tfm)) + algt->alg.hash.halg.statesize = crypto_ahash_statesize(op->fallback_tfm); + + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct sun8i_ss_hash_reqctx) + + crypto_ahash_reqsize(op->fallback_tfm)); + + dev_info(op->ss->dev, "Fallback for %s is %s\n", + crypto_tfm_alg_driver_name(tfm), + crypto_tfm_alg_driver_name(&op->fallback_tfm->base)); + err = pm_runtime_get_sync(op->ss->dev); + if (err < 0) + goto error_pm; + return 0; +error_pm: + pm_runtime_put_noidle(op->ss->dev); + crypto_free_ahash(op->fallback_tfm); + return err; +} + +void sun8i_ss_hash_craexit(struct crypto_tfm *tfm) +{ + struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_tfm_ctx(tfm); + + crypto_free_ahash(tfmctx->fallback_tfm); + pm_runtime_put_sync_suspend(tfmctx->ss->dev); +} + +int sun8i_ss_hash_init(struct ahash_request *areq) +{ + struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + + memset(rctx, 0, sizeof(struct sun8i_ss_hash_reqctx)); + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + + return crypto_ahash_init(&rctx->fallback_req); +} + +int sun8i_ss_hash_export(struct ahash_request *areq, void *out) +{ + struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + + return crypto_ahash_export(&rctx->fallback_req, out); +} + +int sun8i_ss_hash_import(struct ahash_request *areq, const void *in) +{ + struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + + return crypto_ahash_import(&rctx->fallback_req, in); +} + +int sun8i_ss_hash_final(struct ahash_request *areq) +{ + struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG + struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg); + struct sun8i_ss_alg_template *algt; +#endif + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + rctx->fallback_req.base.flags = areq->base.flags & + CRYPTO_TFM_REQ_MAY_SLEEP; + rctx->fallback_req.result = areq->result; + +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG + algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash); + algt->stat_fb++; +#endif + + return crypto_ahash_final(&rctx->fallback_req); +} + +int sun8i_ss_hash_update(struct ahash_request *areq) +{ + struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + rctx->fallback_req.base.flags = areq->base.flags & + CRYPTO_TFM_REQ_MAY_SLEEP; + rctx->fallback_req.nbytes = areq->nbytes; + rctx->fallback_req.src = areq->src; + + return crypto_ahash_update(&rctx->fallback_req); +} + +int sun8i_ss_hash_finup(struct ahash_request *areq) +{ + struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG + struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg); + struct sun8i_ss_alg_template *algt; +#endif + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + rctx->fallback_req.base.flags = areq->base.flags & + CRYPTO_TFM_REQ_MAY_SLEEP; + + rctx->fallback_req.nbytes = areq->nbytes; + rctx->fallback_req.src = areq->src; + rctx->fallback_req.result = areq->result; +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG + algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash); + algt->stat_fb++; +#endif + + return crypto_ahash_finup(&rctx->fallback_req); +} + +static int sun8i_ss_hash_digest_fb(struct ahash_request *areq) +{ + struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG + struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg); + struct sun8i_ss_alg_template *algt; +#endif + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + rctx->fallback_req.base.flags = areq->base.flags & + CRYPTO_TFM_REQ_MAY_SLEEP; + + rctx->fallback_req.nbytes = areq->nbytes; + rctx->fallback_req.src = areq->src; + rctx->fallback_req.result = areq->result; +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG + algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash); + algt->stat_fb++; +#endif + + return crypto_ahash_digest(&rctx->fallback_req); +} + +static int sun8i_ss_run_hash_task(struct sun8i_ss_dev *ss, + struct sun8i_ss_hash_reqctx *rctx, + const char *name) +{ + int flow = rctx->flow; + u32 v = SS_START; + int i; + +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG + ss->flows[flow].stat_req++; +#endif + + /* choose between stream0/stream1 */ + if (flow) + v |= SS_FLOW1; + else + v |= SS_FLOW0; + + v |= rctx->method; + + for (i = 0; i < MAX_SG; i++) { + if (!rctx->t_dst[i].addr) + break; + + mutex_lock(&ss->mlock); + if (i > 0) { + v |= BIT(17); + writel(rctx->t_dst[i - 1].addr, ss->base + SS_KEY_ADR_REG); + writel(rctx->t_dst[i - 1].addr, ss->base + SS_IV_ADR_REG); + } + + dev_dbg(ss->dev, + "Processing SG %d on flow %d %s ctl=%x %d to %d method=%x src=%x dst=%x\n", + i, flow, name, v, + rctx->t_src[i].len, rctx->t_dst[i].len, + rctx->method, rctx->t_src[i].addr, rctx->t_dst[i].addr); + + writel(rctx->t_src[i].addr, ss->base + SS_SRC_ADR_REG); + writel(rctx->t_dst[i].addr, ss->base + SS_DST_ADR_REG); + writel(rctx->t_src[i].len, ss->base + SS_LEN_ADR_REG); + writel(BIT(0) | BIT(1), ss->base + SS_INT_CTL_REG); + + reinit_completion(&ss->flows[flow].complete); + ss->flows[flow].status = 0; + wmb(); + + writel(v, ss->base + SS_CTL_REG); + mutex_unlock(&ss->mlock); + wait_for_completion_interruptible_timeout(&ss->flows[flow].complete, + msecs_to_jiffies(2000)); + if (ss->flows[flow].status == 0) { + dev_err(ss->dev, "DMA timeout for %s\n", name); + return -EFAULT; + } + } + + return 0; +} + +static bool sun8i_ss_hash_need_fallback(struct ahash_request *areq) +{ + struct scatterlist *sg; + + if (areq->nbytes == 0) + return true; + /* we need to reserve one SG for the padding one */ + if (sg_nents(areq->src) > MAX_SG - 1) + return true; + sg = areq->src; + while (sg) { + /* SS can operate hash only on full block size + * since SS support only MD5,sha1,sha224 and sha256, blocksize + * is always 64 + * TODO: handle request if last SG is not len%64 + * but this will need to copy data on a new SG of size=64 + */ + if (sg->length % 64 || !IS_ALIGNED(sg->offset, sizeof(u32))) + return true; + sg = sg_next(sg); + } + return false; +} + +int sun8i_ss_hash_digest(struct ahash_request *areq) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg); + struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); + struct sun8i_ss_alg_template *algt; + struct sun8i_ss_dev *ss; + struct crypto_engine *engine; + struct scatterlist *sg; + int nr_sgs, e, i; + + if (sun8i_ss_hash_need_fallback(areq)) + return sun8i_ss_hash_digest_fb(areq); + + nr_sgs = sg_nents(areq->src); + if (nr_sgs > MAX_SG - 1) + return sun8i_ss_hash_digest_fb(areq); + + for_each_sg(areq->src, sg, nr_sgs, i) { + if (sg->length % 4 || !IS_ALIGNED(sg->offset, sizeof(u32))) + return sun8i_ss_hash_digest_fb(areq); + } + + algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash); + ss = algt->ss; + + e = sun8i_ss_get_engine_number(ss); + rctx->flow = e; + engine = ss->flows[e].engine; + + return crypto_transfer_hash_request_to_engine(engine, areq); +} + +/* sun8i_ss_hash_run - run an ahash request + * Send the data of the request to the SS along with an extra SG with padding + */ +int sun8i_ss_hash_run(struct crypto_engine *engine, void *breq) +{ + struct ahash_request *areq = container_of(breq, struct ahash_request, base); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg); + struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); + struct sun8i_ss_alg_template *algt; + struct sun8i_ss_dev *ss; + struct scatterlist *sg; + int nr_sgs, err, digestsize; + unsigned int len; + u64 fill, min_fill, byte_count; + void *pad, *result; + int j, i, todo; + __be64 *bebits; + __le64 *lebits; + dma_addr_t addr_res, addr_pad; + __le32 *bf; + + algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash); + ss = algt->ss; + + digestsize = algt->alg.hash.halg.digestsize; + if (digestsize == SHA224_DIGEST_SIZE) + digestsize = SHA256_DIGEST_SIZE; + + /* the padding could be up to two block. */ + pad = kzalloc(algt->alg.hash.halg.base.cra_blocksize * 2, GFP_KERNEL | GFP_DMA); + if (!pad) + return -ENOMEM; + bf = (__le32 *)pad; + + result = kzalloc(digestsize, GFP_KERNEL | GFP_DMA); + if (!result) + return -ENOMEM; + + for (i = 0; i < MAX_SG; i++) { + rctx->t_dst[i].addr = 0; + rctx->t_dst[i].len = 0; + } + +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG + algt->stat_req++; +#endif + + rctx->method = ss->variant->alg_hash[algt->ss_algo_id]; + + nr_sgs = dma_map_sg(ss->dev, areq->src, sg_nents(areq->src), DMA_TO_DEVICE); + if (nr_sgs <= 0 || nr_sgs > MAX_SG) { + dev_err(ss->dev, "Invalid sg number %d\n", nr_sgs); + err = -EINVAL; + goto theend; + } + + addr_res = dma_map_single(ss->dev, result, digestsize, DMA_FROM_DEVICE); + if (dma_mapping_error(ss->dev, addr_res)) { + dev_err(ss->dev, "DMA map dest\n"); + err = -EINVAL; + goto theend; + } + + len = areq->nbytes; + for_each_sg(areq->src, sg, nr_sgs, i) { + rctx->t_src[i].addr = sg_dma_address(sg); + todo = min(len, sg_dma_len(sg)); + rctx->t_src[i].len = todo / 4; + len -= todo; + rctx->t_dst[i].addr = addr_res; + rctx->t_dst[i].len = digestsize / 4; + } + if (len > 0) { + dev_err(ss->dev, "remaining len %d\n", len); + err = -EINVAL; + goto theend; + } + + byte_count = areq->nbytes; + j = 0; + bf[j++] = cpu_to_le32(0x80); + + fill = 64 - (byte_count % 64); + min_fill = 3 * sizeof(u32); + + if (fill < min_fill) + fill += 64; + + j += (fill - min_fill) / sizeof(u32); + + switch (algt->ss_algo_id) { + case SS_ID_HASH_MD5: + lebits = (__le64 *)&bf[j]; + *lebits = cpu_to_le64(byte_count << 3); + j += 2; + break; + case SS_ID_HASH_SHA1: + case SS_ID_HASH_SHA224: + case SS_ID_HASH_SHA256: + bebits = (__be64 *)&bf[j]; + *bebits = cpu_to_be64(byte_count << 3); + j += 2; + break; + } + + addr_pad = dma_map_single(ss->dev, pad, j * 4, DMA_TO_DEVICE); + rctx->t_src[i].addr = addr_pad; + rctx->t_src[i].len = j; + rctx->t_dst[i].addr = addr_res; + rctx->t_dst[i].len = digestsize / 4; + if (dma_mapping_error(ss->dev, addr_pad)) { + dev_err(ss->dev, "DMA error on padding SG\n"); + err = -EINVAL; + goto theend; + } + + err = sun8i_ss_run_hash_task(ss, rctx, crypto_tfm_alg_name(areq->base.tfm)); + + dma_unmap_single(ss->dev, addr_pad, j * 4, DMA_TO_DEVICE); + dma_unmap_sg(ss->dev, areq->src, nr_sgs, DMA_TO_DEVICE); + dma_unmap_single(ss->dev, addr_res, digestsize, DMA_FROM_DEVICE); + + kfree(pad); + + memcpy(areq->result, result, algt->alg.hash.halg.digestsize); + kfree(result); +theend: + crypto_finalize_hash_request(engine, breq, err); + return 0; +} diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-prng.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-prng.c new file mode 100644 index 000000000000..08a1473b2145 --- /dev/null +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-prng.c @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * sun8i-ss-prng.c - hardware cryptographic offloader for + * Allwinner A80/A83T SoC + * + * Copyright (C) 2015-2020 Corentin Labbe <[email protected]> + * + * This file handle the PRNG found in the SS + * + * You could find a link for the datasheet in Documentation/arm/sunxi.rst + */ +#include "sun8i-ss.h" +#include <linux/dma-mapping.h> +#include <linux/pm_runtime.h> +#include <crypto/internal/rng.h> + +int sun8i_ss_prng_seed(struct crypto_rng *tfm, const u8 *seed, + unsigned int slen) +{ + struct sun8i_ss_rng_tfm_ctx *ctx = crypto_rng_ctx(tfm); + + if (ctx->seed && ctx->slen != slen) { + memzero_explicit(ctx->seed, ctx->slen); + kfree(ctx->seed); + ctx->slen = 0; + ctx->seed = NULL; + } + if (!ctx->seed) + ctx->seed = kmalloc(slen, GFP_KERNEL | GFP_DMA); + if (!ctx->seed) + return -ENOMEM; + + memcpy(ctx->seed, seed, slen); + ctx->slen = slen; + + return 0; +} + +int sun8i_ss_prng_init(struct crypto_tfm *tfm) +{ + struct sun8i_ss_rng_tfm_ctx *ctx = crypto_tfm_ctx(tfm); + + memset(ctx, 0, sizeof(struct sun8i_ss_rng_tfm_ctx)); + return 0; +} + +void sun8i_ss_prng_exit(struct crypto_tfm *tfm) +{ + struct sun8i_ss_rng_tfm_ctx *ctx = crypto_tfm_ctx(tfm); + + memzero_explicit(ctx->seed, ctx->slen); + kfree(ctx->seed); + ctx->seed = NULL; + ctx->slen = 0; +} + +int sun8i_ss_prng_generate(struct crypto_rng *tfm, const u8 *src, + unsigned int slen, u8 *dst, unsigned int dlen) +{ + struct sun8i_ss_rng_tfm_ctx *ctx = crypto_rng_ctx(tfm); + struct rng_alg *alg = crypto_rng_alg(tfm); + struct sun8i_ss_alg_template *algt; + struct sun8i_ss_dev *ss; + dma_addr_t dma_iv, dma_dst; + unsigned int todo; + int err = 0; + int flow; + void *d; + u32 v; + + algt = container_of(alg, struct sun8i_ss_alg_template, alg.rng); + ss = algt->ss; + + if (ctx->slen == 0) { + dev_err(ss->dev, "The PRNG is not seeded\n"); + return -EINVAL; + } + + /* The SS does not give an updated seed, so we need to get a new one. + * So we will ask for an extra PRNG_SEED_SIZE data. + * We want dlen + seedsize rounded up to a multiple of PRNG_DATA_SIZE + */ + todo = dlen + PRNG_SEED_SIZE + PRNG_DATA_SIZE; + todo -= todo % PRNG_DATA_SIZE; + + d = kzalloc(todo, GFP_KERNEL | GFP_DMA); + if (!d) + return -ENOMEM; + + flow = sun8i_ss_get_engine_number(ss); + +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG + algt->stat_req++; + algt->stat_bytes += todo; +#endif + + v = SS_ALG_PRNG | SS_PRNG_CONTINUE | SS_START; + if (flow) + v |= SS_FLOW1; + else + v |= SS_FLOW0; + + dma_iv = dma_map_single(ss->dev, ctx->seed, ctx->slen, DMA_TO_DEVICE); + if (dma_mapping_error(ss->dev, dma_iv)) { + dev_err(ss->dev, "Cannot DMA MAP IV\n"); + return -EFAULT; + } + + dma_dst = dma_map_single(ss->dev, d, todo, DMA_FROM_DEVICE); + if (dma_mapping_error(ss->dev, dma_dst)) { + dev_err(ss->dev, "Cannot DMA MAP DST\n"); + err = -EFAULT; + goto err_iv; + } + + err = pm_runtime_get_sync(ss->dev); + if (err < 0) { + pm_runtime_put_noidle(ss->dev); + goto err_pm; + } + err = 0; + + mutex_lock(&ss->mlock); + writel(dma_iv, ss->base + SS_IV_ADR_REG); + /* the PRNG act badly (failing rngtest) without SS_KEY_ADR_REG set */ + writel(dma_iv, ss->base + SS_KEY_ADR_REG); + writel(dma_dst, ss->base + SS_DST_ADR_REG); + writel(todo / 4, ss->base + SS_LEN_ADR_REG); + + reinit_completion(&ss->flows[flow].complete); + ss->flows[flow].status = 0; + /* Be sure all data is written before enabling the task */ + wmb(); + + writel(v, ss->base + SS_CTL_REG); + + wait_for_completion_interruptible_timeout(&ss->flows[flow].complete, + msecs_to_jiffies(todo)); + if (ss->flows[flow].status == 0) { + dev_err(ss->dev, "DMA timeout for PRNG (size=%u)\n", todo); + err = -EFAULT; + } + /* Since cipher and hash use the linux/cryptoengine and that we have + * a cryptoengine per flow, we are sure that they will issue only one + * request per flow. + * Since the cryptoengine wait for completion before submitting a new + * one, the mlock could be left just after the final writel. + * But cryptoengine cannot handle crypto_rng, so we need to be sure + * nothing will use our flow. + * The easiest way is to grab mlock until the hardware end our requests. + * We could have used a per flow lock, but this would increase + * complexity. + * The drawback is that no request could be handled for the other flow. + */ + mutex_unlock(&ss->mlock); + + pm_runtime_put(ss->dev); + +err_pm: + dma_unmap_single(ss->dev, dma_dst, todo, DMA_FROM_DEVICE); +err_iv: + dma_unmap_single(ss->dev, dma_iv, ctx->slen, DMA_TO_DEVICE); + + if (!err) { + memcpy(dst, d, dlen); + /* Update seed */ + memcpy(ctx->seed, d + dlen, ctx->slen); + } + memzero_explicit(d, todo); + kfree(d); + + return err; +} diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h index 0405767f1f7e..1a66457f4a20 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h @@ -8,10 +8,16 @@ #include <crypto/aes.h> #include <crypto/des.h> #include <crypto/engine.h> +#include <crypto/rng.h> #include <crypto/skcipher.h> #include <linux/atomic.h> #include <linux/debugfs.h> #include <linux/crypto.h> +#include <crypto/internal/hash.h> +#include <crypto/md5.h> +#include <crypto/sha.h> + +#define SS_START 1 #define SS_ENCRYPTION 0 #define SS_DECRYPTION BIT(6) @@ -19,6 +25,11 @@ #define SS_ALG_AES 0 #define SS_ALG_DES (1 << 2) #define SS_ALG_3DES (2 << 2) +#define SS_ALG_MD5 (3 << 2) +#define SS_ALG_PRNG (4 << 2) +#define SS_ALG_SHA1 (6 << 2) +#define SS_ALG_SHA224 (7 << 2) +#define SS_ALG_SHA256 (8 << 2) #define SS_CTL_REG 0x00 #define SS_INT_CTL_REG 0x04 @@ -47,9 +58,17 @@ #define SS_OP_ECB 0 #define SS_OP_CBC (1 << 13) +#define SS_ID_HASH_MD5 0 +#define SS_ID_HASH_SHA1 1 +#define SS_ID_HASH_SHA224 2 +#define SS_ID_HASH_SHA256 3 +#define SS_ID_HASH_MAX 4 + #define SS_FLOW0 BIT(30) #define SS_FLOW1 BIT(31) +#define SS_PRNG_CONTINUE BIT(18) + #define MAX_SG 8 #define MAXFLOW 2 @@ -59,6 +78,9 @@ #define SS_DIE_ID_SHIFT 20 #define SS_DIE_ID_MASK 0x07 +#define PRNG_DATA_SIZE (160 / 8) +#define PRNG_SEED_SIZE DIV_ROUND_UP(175, 8) + /* * struct ss_clock - Describe clocks used by sun8i-ss * @name: Name of clock needed by this variant @@ -75,11 +97,14 @@ struct ss_clock { * struct ss_variant - Describe SS capability for each variant hardware * @alg_cipher: list of supported ciphers. for each SS_ID_ this will give the * coresponding SS_ALG_XXX value + * @alg_hash: list of supported hashes. for each SS_ID_ this will give the + * corresponding SS_ALG_XXX value * @op_mode: list of supported block modes - * @ss_clks! list of clock needed by this variant + * @ss_clks: list of clock needed by this variant */ struct ss_variant { char alg_cipher[SS_ID_CIPHER_MAX]; + char alg_hash[SS_ID_HASH_MAX]; u32 op_mode[SS_ID_OP_MAX]; struct ss_clock ss_clks[SS_MAX_CLOCKS]; }; @@ -170,6 +195,8 @@ struct sun8i_cipher_req_ctx { * @keylen: len of the key * @ss: pointer to the private data of driver handling this TFM * @fallback_tfm: pointer to the fallback TFM + * + * enginectx must be the first element */ struct sun8i_cipher_tfm_ctx { struct crypto_engine_ctx enginectx; @@ -180,6 +207,46 @@ struct sun8i_cipher_tfm_ctx { }; /* + * struct sun8i_ss_prng_ctx - context for PRNG TFM + * @seed: The seed to use + * @slen: The size of the seed + */ +struct sun8i_ss_rng_tfm_ctx { + void *seed; + unsigned int slen; +}; + +/* + * struct sun8i_ss_hash_tfm_ctx - context for an ahash TFM + * @enginectx: crypto_engine used by this TFM + * @fallback_tfm: pointer to the fallback TFM + * @ss: pointer to the private data of driver handling this TFM + * + * enginectx must be the first element + */ +struct sun8i_ss_hash_tfm_ctx { + struct crypto_engine_ctx enginectx; + struct crypto_ahash *fallback_tfm; + struct sun8i_ss_dev *ss; +}; + +/* + * struct sun8i_ss_hash_reqctx - context for an ahash request + * @t_src: list of DMA address and size for source SGs + * @t_dst: list of DMA address and size for destination SGs + * @fallback_req: pre-allocated fallback request + * @method: the register value for the algorithm used by this request + * @flow: the flow to use for this request + */ +struct sun8i_ss_hash_reqctx { + struct sginfo t_src[MAX_SG]; + struct sginfo t_dst[MAX_SG]; + struct ahash_request fallback_req; + u32 method; + int flow; +}; + +/* * struct sun8i_ss_alg_template - crypto_alg template * @type: the CRYPTO_ALG_TYPE for this template * @ss_algo_id: the SS_ID for this template @@ -189,6 +256,7 @@ struct sun8i_cipher_tfm_ctx { * @alg: one of sub struct must be used * @stat_req: number of request done on this template * @stat_fb: number of request which has fallbacked + * @stat_bytes: total data size done by this template */ struct sun8i_ss_alg_template { u32 type; @@ -197,10 +265,13 @@ struct sun8i_ss_alg_template { struct sun8i_ss_dev *ss; union { struct skcipher_alg skcipher; + struct rng_alg rng; + struct ahash_alg hash; } alg; #ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG unsigned long stat_req; unsigned long stat_fb; + unsigned long stat_bytes; #endif }; @@ -218,3 +289,19 @@ int sun8i_ss_skencrypt(struct skcipher_request *areq); int sun8i_ss_get_engine_number(struct sun8i_ss_dev *ss); int sun8i_ss_run_task(struct sun8i_ss_dev *ss, struct sun8i_cipher_req_ctx *rctx, const char *name); +int sun8i_ss_prng_generate(struct crypto_rng *tfm, const u8 *src, + unsigned int slen, u8 *dst, unsigned int dlen); +int sun8i_ss_prng_seed(struct crypto_rng *tfm, const u8 *seed, unsigned int slen); +int sun8i_ss_prng_init(struct crypto_tfm *tfm); +void sun8i_ss_prng_exit(struct crypto_tfm *tfm); + +int sun8i_ss_hash_crainit(struct crypto_tfm *tfm); +void sun8i_ss_hash_craexit(struct crypto_tfm *tfm); +int sun8i_ss_hash_init(struct ahash_request *areq); +int sun8i_ss_hash_export(struct ahash_request *areq, void *out); +int sun8i_ss_hash_import(struct ahash_request *areq, const void *in); +int sun8i_ss_hash_final(struct ahash_request *areq); +int sun8i_ss_hash_update(struct ahash_request *areq); +int sun8i_ss_hash_finup(struct ahash_request *areq); +int sun8i_ss_hash_digest(struct ahash_request *areq); +int sun8i_ss_hash_run(struct crypto_engine *engine, void *breq); diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c index f7fc0c464125..7729a637fb02 100644 --- a/drivers/crypto/amcc/crypto4xx_alg.c +++ b/drivers/crypto/amcc/crypto4xx_alg.c @@ -55,7 +55,7 @@ static void set_dynamic_sa_command_1(struct dynamic_sa_ctl *sa, u32 cm, sa->sa_command_1.w = 0; sa->sa_command_1.bf.crypto_mode31 = (cm & 4) >> 2; sa->sa_command_1.bf.crypto_mode9_8 = cm & 3; - sa->sa_command_1.bf.feedback_mode = cfb, + sa->sa_command_1.bf.feedback_mode = cfb; sa->sa_command_1.bf.sa_rev = 1; sa->sa_command_1.bf.hmac_muting = hmac_mc; sa->sa_command_1.bf.extended_seq_num = esn; diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h index 6b6841359190..a4e25b46cd0a 100644 --- a/drivers/crypto/amcc/crypto4xx_core.h +++ b/drivers/crypto/amcc/crypto4xx_core.h @@ -15,6 +15,7 @@ #include <linux/ratelimit.h> #include <linux/mutex.h> +#include <linux/scatterlist.h> #include <crypto/internal/hash.h> #include <crypto/internal/aead.h> #include <crypto/internal/rng.h> diff --git a/drivers/crypto/amlogic/amlogic-gxl-cipher.c b/drivers/crypto/amlogic/amlogic-gxl-cipher.c index d93210726697..8b5e07316352 100644 --- a/drivers/crypto/amlogic/amlogic-gxl-cipher.c +++ b/drivers/crypto/amlogic/amlogic-gxl-cipher.c @@ -99,7 +99,7 @@ static int meson_cipher(struct skcipher_request *areq) unsigned int keyivlen, ivsize, offset, tloffset; dma_addr_t phykeyiv; void *backup_iv = NULL, *bkeyiv; - __le32 v; + u32 v; algt = container_of(alg, struct meson_alg_template, alg.skcipher); @@ -340,10 +340,7 @@ void meson_cipher_exit(struct crypto_tfm *tfm) { struct meson_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm); - if (op->key) { - memzero_explicit(op->key, op->keylen); - kfree(op->key); - } + kfree_sensitive(op->key); crypto_free_skcipher(op->fallback_tfm); } @@ -367,10 +364,7 @@ int meson_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, dev_dbg(mc->dev, "ERROR: Invalid keylen %u\n", keylen); return -EINVAL; } - if (op->key) { - memzero_explicit(op->key, op->keylen); - kfree(op->key); - } + kfree_sensitive(op->key); op->keylen = keylen; op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA); if (!op->key) diff --git a/drivers/crypto/amlogic/amlogic-gxl-core.c b/drivers/crypto/amlogic/amlogic-gxl-core.c index 466552acbbbb..5bbeff433c8c 100644 --- a/drivers/crypto/amlogic/amlogic-gxl-core.c +++ b/drivers/crypto/amlogic/amlogic-gxl-core.c @@ -98,7 +98,7 @@ static struct meson_alg_template mc_algs[] = { }; #ifdef CONFIG_CRYPTO_DEV_AMLOGIC_GXL_DEBUG -static int meson_dbgfs_read(struct seq_file *seq, void *v) +static int meson_debugfs_show(struct seq_file *seq, void *v) { struct meson_dev *mc = seq->private; int i; @@ -118,19 +118,7 @@ static int meson_dbgfs_read(struct seq_file *seq, void *v) } return 0; } - -static int meson_dbgfs_open(struct inode *inode, struct file *file) -{ - return single_open(file, meson_dbgfs_read, inode->i_private); -} - -static const struct file_operations meson_debugfs_fops = { - .owner = THIS_MODULE, - .open = meson_dbgfs_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(meson_debugfs); #endif static void meson_free_chanlist(struct meson_dev *mc, int i) diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c index a6e14491e080..b1d286004295 100644 --- a/drivers/crypto/atmel-aes.c +++ b/drivers/crypto/atmel-aes.c @@ -1539,7 +1539,7 @@ static int atmel_aes_gcm_length(struct atmel_aes_dev *dd) /* Write incr32(J0) into IV. */ j0_lsw = j0[3]; - j0[3] = cpu_to_be32(be32_to_cpu(j0[3]) + 1); + be32_add_cpu(&j0[3], 1); atmel_aes_write_block(dd, AES_IVR(0), j0); j0[3] = j0_lsw; diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c index ed40dbb98c6b..4d63cb13a54f 100644 --- a/drivers/crypto/atmel-tdes.c +++ b/drivers/crypto/atmel-tdes.c @@ -912,7 +912,7 @@ static void atmel_tdes_skcipher_alg_init(struct skcipher_alg *alg) { alg->base.cra_priority = ATMEL_TDES_PRIORITY; alg->base.cra_flags = CRYPTO_ALG_ASYNC; - alg->base.cra_ctxsize = sizeof(struct atmel_tdes_ctx), + alg->base.cra_ctxsize = sizeof(struct atmel_tdes_ctx); alg->base.cra_module = THIS_MODULE; alg->init = atmel_tdes_init_tfm; diff --git a/drivers/crypto/bcm/cipher.c b/drivers/crypto/bcm/cipher.c index 8a7fa1ae1ade..50d169e61b41 100644 --- a/drivers/crypto/bcm/cipher.c +++ b/drivers/crypto/bcm/cipher.c @@ -165,10 +165,6 @@ spu_skcipher_rx_sg_create(struct brcm_message *mssg, return -EFAULT; } - if (ctx->cipher.alg == CIPHER_ALG_RC4) - /* Add buffer to catch 260-byte SUPDT field for RC4 */ - sg_set_buf(sg++, rctx->msg_buf.c.supdt_tweak, SPU_SUPDT_LEN); - if (stat_pad_len) sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len); @@ -317,7 +313,6 @@ static int handle_skcipher_req(struct iproc_reqctx_s *rctx) u8 local_iv_ctr[MAX_IV_SIZE]; u32 stat_pad_len; /* num bytes to align status field */ u32 pad_len; /* total length of all padding */ - bool update_key = false; struct brcm_message *mssg; /* mailbox message */ /* number of entries in src and dst sg in mailbox message. */ @@ -391,28 +386,6 @@ static int handle_skcipher_req(struct iproc_reqctx_s *rctx) } } - if (ctx->cipher.alg == CIPHER_ALG_RC4) { - rx_frag_num++; - if (chunk_start) { - /* - * for non-first RC4 chunks, use SUPDT from previous - * response as key for this chunk. - */ - cipher_parms.key_buf = rctx->msg_buf.c.supdt_tweak; - update_key = true; - cipher_parms.type = CIPHER_TYPE_UPDT; - } else if (!rctx->is_encrypt) { - /* - * First RC4 chunk. For decrypt, key in pre-built msg - * header may have been changed if encrypt required - * multiple chunks. So revert the key to the - * ctx->enckey value. - */ - update_key = true; - cipher_parms.type = CIPHER_TYPE_INIT; - } - } - if (ctx->max_payload == SPU_MAX_PAYLOAD_INF) flow_log("max_payload infinite\n"); else @@ -425,14 +398,9 @@ static int handle_skcipher_req(struct iproc_reqctx_s *rctx) memcpy(rctx->msg_buf.bcm_spu_req_hdr, ctx->bcm_spu_req_hdr, sizeof(rctx->msg_buf.bcm_spu_req_hdr)); - /* - * Pass SUPDT field as key. Key field in finish() call is only used - * when update_key has been set above for RC4. Will be ignored in - * all other cases. - */ spu->spu_cipher_req_finish(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN, ctx->spu_req_hdr_len, !(rctx->is_encrypt), - &cipher_parms, update_key, chunksize); + &cipher_parms, chunksize); atomic64_add(chunksize, &iproc_priv.bytes_out); @@ -527,9 +495,6 @@ static void handle_skcipher_resp(struct iproc_reqctx_s *rctx) __func__, rctx->total_received, payload_len); dump_sg(req->dst, rctx->total_received, payload_len); - if (ctx->cipher.alg == CIPHER_ALG_RC4) - packet_dump(" supdt ", rctx->msg_buf.c.supdt_tweak, - SPU_SUPDT_LEN); rctx->total_received += payload_len; if (rctx->total_received == rctx->total_todo) { @@ -1853,26 +1818,6 @@ static int aes_setkey(struct crypto_skcipher *cipher, const u8 *key, return 0; } -static int rc4_setkey(struct crypto_skcipher *cipher, const u8 *key, - unsigned int keylen) -{ - struct iproc_ctx_s *ctx = crypto_skcipher_ctx(cipher); - int i; - - ctx->enckeylen = ARC4_MAX_KEY_SIZE + ARC4_STATE_SIZE; - - ctx->enckey[0] = 0x00; /* 0x00 */ - ctx->enckey[1] = 0x00; /* i */ - ctx->enckey[2] = 0x00; /* 0x00 */ - ctx->enckey[3] = 0x00; /* j */ - for (i = 0; i < ARC4_MAX_KEY_SIZE; i++) - ctx->enckey[i + ARC4_STATE_SIZE] = key[i % keylen]; - - ctx->cipher_type = CIPHER_TYPE_INIT; - - return 0; -} - static int skcipher_setkey(struct crypto_skcipher *cipher, const u8 *key, unsigned int keylen) { @@ -1895,9 +1840,6 @@ static int skcipher_setkey(struct crypto_skcipher *cipher, const u8 *key, case CIPHER_ALG_AES: err = aes_setkey(cipher, key, keylen); break; - case CIPHER_ALG_RC4: - err = rc4_setkey(cipher, key, keylen); - break; default: pr_err("%s() Error: unknown cipher alg\n", __func__); err = -EINVAL; @@ -1905,11 +1847,9 @@ static int skcipher_setkey(struct crypto_skcipher *cipher, const u8 *key, if (err) return err; - /* RC4 already populated ctx->enkey */ - if (ctx->cipher.alg != CIPHER_ALG_RC4) { - memcpy(ctx->enckey, key, keylen); - ctx->enckeylen = keylen; - } + memcpy(ctx->enckey, key, keylen); + ctx->enckeylen = keylen; + /* SPU needs XTS keys in the reverse order the crypto API presents */ if ((ctx->cipher.alg == CIPHER_ALG_AES) && (ctx->cipher.mode == CIPHER_MODE_XTS)) { @@ -2872,9 +2812,6 @@ static int aead_authenc_setkey(struct crypto_aead *cipher, goto badkey; } break; - case CIPHER_ALG_RC4: - ctx->cipher_type = CIPHER_TYPE_INIT; - break; default: pr_err("%s() Error: Unknown cipher alg\n", __func__); return -EINVAL; @@ -2930,7 +2867,6 @@ static int aead_gcm_ccm_setkey(struct crypto_aead *cipher, ctx->enckeylen = keylen; ctx->authkeylen = 0; - memcpy(ctx->enckey, key, ctx->enckeylen); switch (ctx->enckeylen) { case AES_KEYSIZE_128: @@ -2946,6 +2882,8 @@ static int aead_gcm_ccm_setkey(struct crypto_aead *cipher, goto badkey; } + memcpy(ctx->enckey, key, ctx->enckeylen); + flow_log(" enckeylen:%u authkeylen:%u\n", ctx->enckeylen, ctx->authkeylen); flow_dump(" enc: ", ctx->enckey, ctx->enckeylen); @@ -3000,6 +2938,10 @@ static int aead_gcm_esp_setkey(struct crypto_aead *cipher, struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher); flow_log("%s\n", __func__); + + if (keylen < GCM_ESP_SALT_SIZE) + return -EINVAL; + ctx->salt_len = GCM_ESP_SALT_SIZE; ctx->salt_offset = GCM_ESP_SALT_OFFSET; memcpy(ctx->salt, key + keylen - GCM_ESP_SALT_SIZE, GCM_ESP_SALT_SIZE); @@ -3028,6 +2970,10 @@ static int rfc4543_gcm_esp_setkey(struct crypto_aead *cipher, struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher); flow_log("%s\n", __func__); + + if (keylen < GCM_ESP_SALT_SIZE) + return -EINVAL; + ctx->salt_len = GCM_ESP_SALT_SIZE; ctx->salt_offset = GCM_ESP_SALT_OFFSET; memcpy(ctx->salt, key + keylen - GCM_ESP_SALT_SIZE, GCM_ESP_SALT_SIZE); @@ -3057,6 +3003,10 @@ static int aead_ccm_esp_setkey(struct crypto_aead *cipher, struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher); flow_log("%s\n", __func__); + + if (keylen < CCM_ESP_SALT_SIZE) + return -EINVAL; + ctx->salt_len = CCM_ESP_SALT_SIZE; ctx->salt_offset = CCM_ESP_SALT_OFFSET; memcpy(ctx->salt, key + keylen - CCM_ESP_SALT_SIZE, CCM_ESP_SALT_SIZE); @@ -3606,25 +3556,6 @@ static struct iproc_alg_s driver_algs[] = { { .type = CRYPTO_ALG_TYPE_SKCIPHER, .alg.skcipher = { - .base.cra_name = "ecb(arc4)", - .base.cra_driver_name = "ecb-arc4-iproc", - .base.cra_blocksize = ARC4_BLOCK_SIZE, - .min_keysize = ARC4_MIN_KEY_SIZE, - .max_keysize = ARC4_MAX_KEY_SIZE, - .ivsize = 0, - }, - .cipher_info = { - .alg = CIPHER_ALG_RC4, - .mode = CIPHER_MODE_NONE, - }, - .auth_info = { - .alg = HASH_ALG_NONE, - .mode = HASH_MODE_NONE, - }, - }, - { - .type = CRYPTO_ALG_TYPE_SKCIPHER, - .alg.skcipher = { .base.cra_name = "ofb(des)", .base.cra_driver_name = "ofb-des-iproc", .base.cra_blocksize = DES_BLOCK_SIZE, @@ -4526,15 +4457,9 @@ static void spu_counters_init(void) static int spu_register_skcipher(struct iproc_alg_s *driver_alg) { - struct spu_hw *spu = &iproc_priv.spu; struct skcipher_alg *crypto = &driver_alg->alg.skcipher; int err; - /* SPU2 does not support RC4 */ - if ((driver_alg->cipher_info.alg == CIPHER_ALG_RC4) && - (spu->spu_type == SPU_TYPE_SPU2)) - return 0; - crypto->base.cra_module = THIS_MODULE; crypto->base.cra_priority = cipher_pri; crypto->base.cra_alignmask = 0; diff --git a/drivers/crypto/bcm/cipher.h b/drivers/crypto/bcm/cipher.h index b6d83e3aa46c..035c8389cb3d 100644 --- a/drivers/crypto/bcm/cipher.h +++ b/drivers/crypto/bcm/cipher.h @@ -388,7 +388,6 @@ struct spu_hw { u16 spu_req_hdr_len, unsigned int is_inbound, struct spu_cipher_parms *cipher_parms, - bool update_key, unsigned int data_size); void (*spu_request_pad)(u8 *pad_start, u32 gcm_padding, u32 hash_pad_len, enum hash_alg auth_alg, diff --git a/drivers/crypto/bcm/spu.c b/drivers/crypto/bcm/spu.c index e7562e9bf396..fe126f95c702 100644 --- a/drivers/crypto/bcm/spu.c +++ b/drivers/crypto/bcm/spu.c @@ -222,10 +222,6 @@ void spum_dump_msg_hdr(u8 *buf, unsigned int buf_len) cipher_key_len = 24; name = "3DES"; break; - case CIPHER_ALG_RC4: - cipher_key_len = 260; - name = "ARC4"; - break; case CIPHER_ALG_AES: switch (cipher_type) { case CIPHER_TYPE_AES128: @@ -919,21 +915,16 @@ u16 spum_cipher_req_init(u8 *spu_hdr, struct spu_cipher_parms *cipher_parms) * @spu_req_hdr_len: Length in bytes of the SPU request header * @isInbound: 0 encrypt, 1 decrypt * @cipher_parms: Parameters describing cipher operation to be performed - * @update_key: If true, rewrite the cipher key in SCTX * @data_size: Length of the data in the BD field * * Assumes much of the header was already filled in at setkey() time in * spum_cipher_req_init(). - * spum_cipher_req_init() fills in the encryption key. For RC4, when submitting - * a request for a non-first chunk, we use the 260-byte SUPDT field from the - * previous response as the key. update_key is true for this case. Unused in all - * other cases. + * spum_cipher_req_init() fills in the encryption key. */ void spum_cipher_req_finish(u8 *spu_hdr, u16 spu_req_hdr_len, unsigned int is_inbound, struct spu_cipher_parms *cipher_parms, - bool update_key, unsigned int data_size) { struct SPUHEADER *spuh; @@ -948,11 +939,6 @@ void spum_cipher_req_finish(u8 *spu_hdr, flow_log(" in: %u\n", is_inbound); flow_log(" cipher alg: %u, cipher_type: %u\n", cipher_parms->alg, cipher_parms->type); - if (update_key) { - flow_log(" cipher key len: %u\n", cipher_parms->key_len); - flow_dump(" key: ", cipher_parms->key_buf, - cipher_parms->key_len); - } /* * In XTS mode, API puts "i" parameter (block tweak) in IV. For @@ -981,13 +967,6 @@ void spum_cipher_req_finish(u8 *spu_hdr, else cipher_bits &= ~CIPHER_INBOUND; - /* update encryption key for RC4 on non-first chunk */ - if (update_key) { - spuh->sa.cipher_flags |= - cipher_parms->type << CIPHER_TYPE_SHIFT; - memcpy(spuh + 1, cipher_parms->key_buf, cipher_parms->key_len); - } - if (cipher_parms->alg && cipher_parms->iv_buf && cipher_parms->iv_len) /* cipher iv provided so put it in here */ memcpy(bdesc_ptr - cipher_parms->iv_len, cipher_parms->iv_buf, diff --git a/drivers/crypto/bcm/spu.h b/drivers/crypto/bcm/spu.h index b247bc5b9354..dd132389bcaa 100644 --- a/drivers/crypto/bcm/spu.h +++ b/drivers/crypto/bcm/spu.h @@ -251,7 +251,6 @@ void spum_cipher_req_finish(u8 *spu_hdr, u16 spu_req_hdr_len, unsigned int is_inbound, struct spu_cipher_parms *cipher_parms, - bool update_key, unsigned int data_size); void spum_request_pad(u8 *pad_start, diff --git a/drivers/crypto/bcm/spu2.c b/drivers/crypto/bcm/spu2.c index 59abb5ecefa4..c860ffb0b4c3 100644 --- a/drivers/crypto/bcm/spu2.c +++ b/drivers/crypto/bcm/spu2.c @@ -1170,21 +1170,16 @@ u16 spu2_cipher_req_init(u8 *spu_hdr, struct spu_cipher_parms *cipher_parms) * @spu_req_hdr_len: Length in bytes of the SPU request header * @isInbound: 0 encrypt, 1 decrypt * @cipher_parms: Parameters describing cipher operation to be performed - * @update_key: If true, rewrite the cipher key in SCTX * @data_size: Length of the data in the BD field * * Assumes much of the header was already filled in at setkey() time in * spu_cipher_req_init(). - * spu_cipher_req_init() fills in the encryption key. For RC4, when submitting a - * request for a non-first chunk, we use the 260-byte SUPDT field from the - * previous response as the key. update_key is true for this case. Unused in all - * other cases. + * spu_cipher_req_init() fills in the encryption key. */ void spu2_cipher_req_finish(u8 *spu_hdr, u16 spu_req_hdr_len, unsigned int is_inbound, struct spu_cipher_parms *cipher_parms, - bool update_key, unsigned int data_size) { struct SPU2_FMD *fmd; @@ -1196,11 +1191,6 @@ void spu2_cipher_req_finish(u8 *spu_hdr, flow_log(" in: %u\n", is_inbound); flow_log(" cipher alg: %u, cipher_type: %u\n", cipher_parms->alg, cipher_parms->type); - if (update_key) { - flow_log(" cipher key len: %u\n", cipher_parms->key_len); - flow_dump(" key: ", cipher_parms->key_buf, - cipher_parms->key_len); - } flow_log(" iv len: %d\n", cipher_parms->iv_len); flow_dump(" iv: ", cipher_parms->iv_buf, cipher_parms->iv_len); flow_log(" data_size: %u\n", data_size); diff --git a/drivers/crypto/bcm/spu2.h b/drivers/crypto/bcm/spu2.h index 03af6c38df7f..6e666bfb3cfc 100644 --- a/drivers/crypto/bcm/spu2.h +++ b/drivers/crypto/bcm/spu2.h @@ -200,7 +200,6 @@ void spu2_cipher_req_finish(u8 *spu_hdr, u16 spu_req_hdr_len, unsigned int is_inbound, struct spu_cipher_parms *cipher_parms, - bool update_key, unsigned int data_size); void spu2_request_pad(u8 *pad_start, u32 gcm_padding, u32 hash_pad_len, enum hash_alg auth_alg, enum hash_mode auth_mode, diff --git a/drivers/crypto/caam/Kconfig b/drivers/crypto/caam/Kconfig index bc35aa0ec07a..84ea7cba5ee5 100644 --- a/drivers/crypto/caam/Kconfig +++ b/drivers/crypto/caam/Kconfig @@ -101,6 +101,7 @@ config CRYPTO_DEV_FSL_CAAM_CRYPTO_API select CRYPTO_AUTHENC select CRYPTO_SKCIPHER select CRYPTO_LIB_DES + select CRYPTO_XTS help Selecting this will offload crypto for users of the scatterlist crypto API (such as the linux native IPSec @@ -114,6 +115,7 @@ config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI select CRYPTO_AUTHENC select CRYPTO_SKCIPHER select CRYPTO_DES + select CRYPTO_XTS help Selecting this will use CAAM Queue Interface (QI) for sending & receiving crypto jobs to/from CAAM. This gives better performance @@ -165,6 +167,7 @@ config CRYPTO_DEV_FSL_DPAA2_CAAM select CRYPTO_AEAD select CRYPTO_HASH select CRYPTO_DES + select CRYPTO_XTS help CAAM driver for QorIQ Data Path Acceleration Architecture 2. It handles DPSECI DPAA2 objects that sit on the Management Complex diff --git a/drivers/crypto/caam/Makefile b/drivers/crypto/caam/Makefile index 68d5cc0f28e2..3570286eb9ce 100644 --- a/drivers/crypto/caam/Makefile +++ b/drivers/crypto/caam/Makefile @@ -27,6 +27,8 @@ ifneq ($(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI),) ccflags-y += -DCONFIG_CAAM_QI endif +caam-$(CONFIG_DEBUG_FS) += debugfs.o + obj-$(CONFIG_CRYPTO_DEV_FSL_DPAA2_CAAM) += dpaa2_caam.o dpaa2_caam-y := caamalg_qi2.o dpseci.o diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 91feda5b63f6..cf5bd7666dfc 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -57,6 +57,8 @@ #include "key_gen.h" #include "caamalg_desc.h" #include <crypto/engine.h> +#include <crypto/xts.h> +#include <asm/unaligned.h> /* * crypto alg @@ -114,10 +116,13 @@ struct caam_ctx { struct alginfo adata; struct alginfo cdata; unsigned int authsize; + bool xts_key_fallback; + struct crypto_skcipher *fallback; }; struct caam_skcipher_req_ctx { struct skcipher_edesc *edesc; + struct skcipher_request fallback_req; }; struct caam_aead_req_ctx { @@ -829,11 +834,23 @@ static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, { struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); struct device *jrdev = ctx->jrdev; + struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); u32 *desc; + int err; - if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) { + err = xts_verify_key(skcipher, key, keylen); + if (err) { dev_dbg(jrdev, "key size mismatch\n"); - return -EINVAL; + return err; + } + + if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256) + ctx->xts_key_fallback = true; + + if (ctrlpriv->era <= 8 || ctx->xts_key_fallback) { + err = crypto_skcipher_setkey(ctx->fallback, key, keylen); + if (err) + return err; } ctx->cdata.keylen = keylen; @@ -1755,6 +1772,14 @@ static int skcipher_do_one_req(struct crypto_engine *engine, void *areq) return ret; } +static inline bool xts_skcipher_ivsize(struct skcipher_request *req) +{ + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + unsigned int ivsize = crypto_skcipher_ivsize(skcipher); + + return !!get_unaligned((u64 *)(req->iv + (ivsize / 2))); +} + static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt) { struct skcipher_edesc *edesc; @@ -1762,12 +1787,34 @@ static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt) struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); struct device *jrdev = ctx->jrdev; struct caam_drv_private_jr *jrpriv = dev_get_drvdata(jrdev); + struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); u32 *desc; int ret = 0; - if (!req->cryptlen) + /* + * XTS is expected to return an error even for input length = 0 + * Note that the case input length < block size will be caught during + * HW offloading and return an error. + */ + if (!req->cryptlen && !ctx->fallback) return 0; + if (ctx->fallback && ((ctrlpriv->era <= 8 && xts_skcipher_ivsize(req)) || + ctx->xts_key_fallback)) { + struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req); + + skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback); + skcipher_request_set_callback(&rctx->fallback_req, + req->base.flags, + req->base.complete, + req->base.data); + skcipher_request_set_crypt(&rctx->fallback_req, req->src, + req->dst, req->cryptlen, req->iv); + + return encrypt ? crypto_skcipher_encrypt(&rctx->fallback_req) : + crypto_skcipher_decrypt(&rctx->fallback_req); + } + /* allocate extended descriptor */ edesc = skcipher_edesc_alloc(req, DESC_JOB_IO_LEN * CAAM_CMD_SZ); if (IS_ERR(edesc)) @@ -1905,6 +1952,7 @@ static struct caam_skcipher_alg driver_algs[] = { .base = { .cra_name = "xts(aes)", .cra_driver_name = "xts-aes-caam", + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = AES_BLOCK_SIZE, }, .setkey = xts_skcipher_setkey, @@ -3344,13 +3392,35 @@ static int caam_cra_init(struct crypto_skcipher *tfm) struct caam_skcipher_alg *caam_alg = container_of(alg, typeof(*caam_alg), skcipher); struct caam_ctx *ctx = crypto_skcipher_ctx(tfm); - - crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_skcipher_req_ctx)); + u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; + int ret = 0; ctx->enginectx.op.do_one_request = skcipher_do_one_req; - return caam_init_common(crypto_skcipher_ctx(tfm), &caam_alg->caam, - false); + if (alg_aai == OP_ALG_AAI_XTS) { + const char *tfm_name = crypto_tfm_alg_name(&tfm->base); + struct crypto_skcipher *fallback; + + fallback = crypto_alloc_skcipher(tfm_name, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(fallback)) { + dev_err(ctx->jrdev, "Failed to allocate %s fallback: %ld\n", + tfm_name, PTR_ERR(fallback)); + return PTR_ERR(fallback); + } + + ctx->fallback = fallback; + crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_skcipher_req_ctx) + + crypto_skcipher_reqsize(fallback)); + } else { + crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_skcipher_req_ctx)); + } + + ret = caam_init_common(ctx, &caam_alg->caam, false); + if (ret && ctx->fallback) + crypto_free_skcipher(ctx->fallback); + + return ret; } static int caam_aead_init(struct crypto_aead *tfm) @@ -3378,7 +3448,11 @@ static void caam_exit_common(struct caam_ctx *ctx) static void caam_cra_exit(struct crypto_skcipher *tfm) { - caam_exit_common(crypto_skcipher_ctx(tfm)); + struct caam_ctx *ctx = crypto_skcipher_ctx(tfm); + + if (ctx->fallback) + crypto_free_skcipher(ctx->fallback); + caam_exit_common(ctx); } static void caam_aead_exit(struct crypto_aead *tfm) @@ -3412,8 +3486,8 @@ static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg) alg->base.cra_module = THIS_MODULE; alg->base.cra_priority = CAAM_CRA_PRIORITY; alg->base.cra_ctxsize = sizeof(struct caam_ctx); - alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_KERN_DRIVER_ONLY; + alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY); alg->init = caam_cra_init; alg->exit = caam_cra_exit; diff --git a/drivers/crypto/caam/caamalg_desc.c b/drivers/crypto/caam/caamalg_desc.c index d6c58184bb57..7571e1ac913b 100644 --- a/drivers/crypto/caam/caamalg_desc.c +++ b/drivers/crypto/caam/caamalg_desc.c @@ -373,6 +373,7 @@ EXPORT_SYMBOL(cnstr_shdsc_aead_encap); * with OP_ALG_AAI_HMAC_PRECOMP. * @ivsize: initialization vector size * @icvsize: integrity check value (ICV) size (truncated or full) + * @geniv: whether to generate Encrypted Chain IV * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template * @nonce: pointer to rfc3686 nonce * @ctx1_iv_off: IV offset in CONTEXT1 register @@ -1550,13 +1551,14 @@ void cnstr_shdsc_xts_skcipher_encap(u32 * const desc, struct alginfo *cdata) set_jump_tgt_here(desc, key_jump_cmd); /* - * create sequence for loading the sector index - * Upper 8B of IV - will be used as sector index - * Lower 8B of IV - will be discarded + * create sequence for loading the sector index / 16B tweak value + * Lower 8B of IV - sector index / tweak lower half + * Upper 8B of IV - upper half of 16B tweak */ append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | (0x20 << LDST_OFFSET_SHIFT)); - append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); + append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | + (0x30 << LDST_OFFSET_SHIFT)); /* Load operation */ append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | @@ -1565,9 +1567,11 @@ void cnstr_shdsc_xts_skcipher_encap(u32 * const desc, struct alginfo *cdata) /* Perform operation */ skcipher_append_src_dst(desc); - /* Store upper 8B of IV */ + /* Store lower 8B and upper 8B of IV */ append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | (0x20 << LDST_OFFSET_SHIFT)); + append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | + (0x30 << LDST_OFFSET_SHIFT)); print_hex_dump_debug("xts skcipher enc shdesc@" __stringify(__LINE__) ": ", DUMP_PREFIX_ADDRESS, 16, 4, @@ -1609,23 +1613,25 @@ void cnstr_shdsc_xts_skcipher_decap(u32 * const desc, struct alginfo *cdata) set_jump_tgt_here(desc, key_jump_cmd); /* - * create sequence for loading the sector index - * Upper 8B of IV - will be used as sector index - * Lower 8B of IV - will be discarded + * create sequence for loading the sector index / 16B tweak value + * Lower 8B of IV - sector index / tweak lower half + * Upper 8B of IV - upper half of 16B tweak */ append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | (0x20 << LDST_OFFSET_SHIFT)); - append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); - + append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | + (0x30 << LDST_OFFSET_SHIFT)); /* Load operation */ append_dec_op1(desc, cdata->algtype); /* Perform operation */ skcipher_append_src_dst(desc); - /* Store upper 8B of IV */ + /* Store lower 8B and upper 8B of IV */ append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | (0x20 << LDST_OFFSET_SHIFT)); + append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | + (0x30 << LDST_OFFSET_SHIFT)); print_hex_dump_debug("xts skcipher dec shdesc@" __stringify(__LINE__) ": ", DUMP_PREFIX_ADDRESS, 16, 4, desc, diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c index bb1c0106a95c..66f60d78bdc8 100644 --- a/drivers/crypto/caam/caamalg_qi.c +++ b/drivers/crypto/caam/caamalg_qi.c @@ -18,6 +18,8 @@ #include "qi.h" #include "jr.h" #include "caamalg_desc.h" +#include <crypto/xts.h> +#include <asm/unaligned.h> /* * crypto alg @@ -67,6 +69,12 @@ struct caam_ctx { struct device *qidev; spinlock_t lock; /* Protects multiple init of driver context */ struct caam_drv_ctx *drv_ctx[NUM_OP]; + bool xts_key_fallback; + struct crypto_skcipher *fallback; +}; + +struct caam_skcipher_req_ctx { + struct skcipher_request fallback_req; }; static int aead_set_sh_desc(struct crypto_aead *aead) @@ -725,11 +733,23 @@ static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, { struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); struct device *jrdev = ctx->jrdev; + struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); int ret = 0; + int err; - if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) { + err = xts_verify_key(skcipher, key, keylen); + if (err) { dev_dbg(jrdev, "key size mismatch\n"); - return -EINVAL; + return err; + } + + if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256) + ctx->xts_key_fallback = true; + + if (ctrlpriv->era <= 8 || ctx->xts_key_fallback) { + err = crypto_skcipher_setkey(ctx->fallback, key, keylen); + if (err) + return err; } ctx->cdata.keylen = keylen; @@ -1373,16 +1393,46 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, return edesc; } +static inline bool xts_skcipher_ivsize(struct skcipher_request *req) +{ + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + unsigned int ivsize = crypto_skcipher_ivsize(skcipher); + + return !!get_unaligned((u64 *)(req->iv + (ivsize / 2))); +} + static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt) { struct skcipher_edesc *edesc; struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); + struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); int ret; - if (!req->cryptlen) + /* + * XTS is expected to return an error even for input length = 0 + * Note that the case input length < block size will be caught during + * HW offloading and return an error. + */ + if (!req->cryptlen && !ctx->fallback) return 0; + if (ctx->fallback && ((ctrlpriv->era <= 8 && xts_skcipher_ivsize(req)) || + ctx->xts_key_fallback)) { + struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req); + + skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback); + skcipher_request_set_callback(&rctx->fallback_req, + req->base.flags, + req->base.complete, + req->base.data); + skcipher_request_set_crypt(&rctx->fallback_req, req->src, + req->dst, req->cryptlen, req->iv); + + return encrypt ? crypto_skcipher_encrypt(&rctx->fallback_req) : + crypto_skcipher_decrypt(&rctx->fallback_req); + } + if (unlikely(caam_congested)) return -EAGAIN; @@ -1507,6 +1557,7 @@ static struct caam_skcipher_alg driver_algs[] = { .base = { .cra_name = "xts(aes)", .cra_driver_name = "xts-aes-caam-qi", + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = AES_BLOCK_SIZE, }, .setkey = xts_skcipher_setkey, @@ -2440,9 +2491,32 @@ static int caam_cra_init(struct crypto_skcipher *tfm) struct skcipher_alg *alg = crypto_skcipher_alg(tfm); struct caam_skcipher_alg *caam_alg = container_of(alg, typeof(*caam_alg), skcipher); + struct caam_ctx *ctx = crypto_skcipher_ctx(tfm); + u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; + int ret = 0; + + if (alg_aai == OP_ALG_AAI_XTS) { + const char *tfm_name = crypto_tfm_alg_name(&tfm->base); + struct crypto_skcipher *fallback; + + fallback = crypto_alloc_skcipher(tfm_name, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(fallback)) { + dev_err(ctx->jrdev, "Failed to allocate %s fallback: %ld\n", + tfm_name, PTR_ERR(fallback)); + return PTR_ERR(fallback); + } + + ctx->fallback = fallback; + crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_skcipher_req_ctx) + + crypto_skcipher_reqsize(fallback)); + } + + ret = caam_init_common(ctx, &caam_alg->caam, false); + if (ret && ctx->fallback) + crypto_free_skcipher(ctx->fallback); - return caam_init_common(crypto_skcipher_ctx(tfm), &caam_alg->caam, - false); + return ret; } static int caam_aead_init(struct crypto_aead *tfm) @@ -2468,7 +2542,11 @@ static void caam_exit_common(struct caam_ctx *ctx) static void caam_cra_exit(struct crypto_skcipher *tfm) { - caam_exit_common(crypto_skcipher_ctx(tfm)); + struct caam_ctx *ctx = crypto_skcipher_ctx(tfm); + + if (ctx->fallback) + crypto_free_skcipher(ctx->fallback); + caam_exit_common(ctx); } static void caam_aead_exit(struct crypto_aead *tfm) @@ -2502,8 +2580,8 @@ static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg) alg->base.cra_module = THIS_MODULE; alg->base.cra_priority = CAAM_CRA_PRIORITY; alg->base.cra_ctxsize = sizeof(struct caam_ctx); - alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_KERN_DRIVER_ONLY; + alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY); alg->init = caam_cra_init; alg->exit = caam_cra_exit; diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c index 66ae1d581168..98c1ff1744bb 100644 --- a/drivers/crypto/caam/caamalg_qi2.c +++ b/drivers/crypto/caam/caamalg_qi2.c @@ -19,6 +19,8 @@ #include <linux/fsl/mc.h> #include <soc/fsl/dpaa2-io.h> #include <soc/fsl/dpaa2-fd.h> +#include <crypto/xts.h> +#include <asm/unaligned.h> #define CAAM_CRA_PRIORITY 2000 @@ -59,7 +61,7 @@ struct caam_skcipher_alg { }; /** - * caam_ctx - per-session context + * struct caam_ctx - per-session context * @flc: Flow Contexts array * @key: [authentication key], encryption key * @flc_dma: I/O virtual addresses of the Flow Contexts @@ -80,6 +82,8 @@ struct caam_ctx { struct alginfo adata; struct alginfo cdata; unsigned int authsize; + bool xts_key_fallback; + struct crypto_skcipher *fallback; }; static void *dpaa2_caam_iova_to_virt(struct dpaa2_caam_priv *priv, @@ -1054,12 +1058,24 @@ static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, { struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); struct device *dev = ctx->dev; + struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); struct caam_flc *flc; u32 *desc; + int err; - if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) { + err = xts_verify_key(skcipher, key, keylen); + if (err) { dev_dbg(dev, "key size mismatch\n"); - return -EINVAL; + return err; + } + + if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256) + ctx->xts_key_fallback = true; + + if (priv->sec_attr.era <= 8 || ctx->xts_key_fallback) { + err = crypto_skcipher_setkey(ctx->fallback, key, keylen); + if (err) + return err; } ctx->cdata.keylen = keylen; @@ -1443,17 +1459,44 @@ static void skcipher_decrypt_done(void *cbk_ctx, u32 status) skcipher_request_complete(req, ecode); } +static inline bool xts_skcipher_ivsize(struct skcipher_request *req) +{ + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + unsigned int ivsize = crypto_skcipher_ivsize(skcipher); + + return !!get_unaligned((u64 *)(req->iv + (ivsize / 2))); +} + static int skcipher_encrypt(struct skcipher_request *req) { struct skcipher_edesc *edesc; struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); struct caam_request *caam_req = skcipher_request_ctx(req); + struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev); int ret; - if (!req->cryptlen) + /* + * XTS is expected to return an error even for input length = 0 + * Note that the case input length < block size will be caught during + * HW offloading and return an error. + */ + if (!req->cryptlen && !ctx->fallback) return 0; + if (ctx->fallback && ((priv->sec_attr.era <= 8 && xts_skcipher_ivsize(req)) || + ctx->xts_key_fallback)) { + skcipher_request_set_tfm(&caam_req->fallback_req, ctx->fallback); + skcipher_request_set_callback(&caam_req->fallback_req, + req->base.flags, + req->base.complete, + req->base.data); + skcipher_request_set_crypt(&caam_req->fallback_req, req->src, + req->dst, req->cryptlen, req->iv); + + return crypto_skcipher_encrypt(&caam_req->fallback_req); + } + /* allocate extended descriptor */ edesc = skcipher_edesc_alloc(req); if (IS_ERR(edesc)) @@ -1480,10 +1523,30 @@ static int skcipher_decrypt(struct skcipher_request *req) struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); struct caam_request *caam_req = skcipher_request_ctx(req); + struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev); int ret; - if (!req->cryptlen) + /* + * XTS is expected to return an error even for input length = 0 + * Note that the case input length < block size will be caught during + * HW offloading and return an error. + */ + if (!req->cryptlen && !ctx->fallback) return 0; + + if (ctx->fallback && ((priv->sec_attr.era <= 8 && xts_skcipher_ivsize(req)) || + ctx->xts_key_fallback)) { + skcipher_request_set_tfm(&caam_req->fallback_req, ctx->fallback); + skcipher_request_set_callback(&caam_req->fallback_req, + req->base.flags, + req->base.complete, + req->base.data); + skcipher_request_set_crypt(&caam_req->fallback_req, req->src, + req->dst, req->cryptlen, req->iv); + + return crypto_skcipher_decrypt(&caam_req->fallback_req); + } + /* allocate extended descriptor */ edesc = skcipher_edesc_alloc(req); if (IS_ERR(edesc)) @@ -1537,9 +1600,34 @@ static int caam_cra_init_skcipher(struct crypto_skcipher *tfm) struct skcipher_alg *alg = crypto_skcipher_alg(tfm); struct caam_skcipher_alg *caam_alg = container_of(alg, typeof(*caam_alg), skcipher); + struct caam_ctx *ctx = crypto_skcipher_ctx(tfm); + u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; + int ret = 0; - crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_request)); - return caam_cra_init(crypto_skcipher_ctx(tfm), &caam_alg->caam, false); + if (alg_aai == OP_ALG_AAI_XTS) { + const char *tfm_name = crypto_tfm_alg_name(&tfm->base); + struct crypto_skcipher *fallback; + + fallback = crypto_alloc_skcipher(tfm_name, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(fallback)) { + dev_err(ctx->dev, "Failed to allocate %s fallback: %ld\n", + tfm_name, PTR_ERR(fallback)); + return PTR_ERR(fallback); + } + + ctx->fallback = fallback; + crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_request) + + crypto_skcipher_reqsize(fallback)); + } else { + crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_request)); + } + + ret = caam_cra_init(ctx, &caam_alg->caam, false); + if (ret && ctx->fallback) + crypto_free_skcipher(ctx->fallback); + + return ret; } static int caam_cra_init_aead(struct crypto_aead *tfm) @@ -1562,7 +1650,11 @@ static void caam_exit_common(struct caam_ctx *ctx) static void caam_cra_exit(struct crypto_skcipher *tfm) { - caam_exit_common(crypto_skcipher_ctx(tfm)); + struct caam_ctx *ctx = crypto_skcipher_ctx(tfm); + + if (ctx->fallback) + crypto_free_skcipher(ctx->fallback); + caam_exit_common(ctx); } static void caam_cra_exit_aead(struct crypto_aead *tfm) @@ -1665,6 +1757,7 @@ static struct caam_skcipher_alg driver_algs[] = { .base = { .cra_name = "xts(aes)", .cra_driver_name = "xts-aes-caam-qi2", + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = AES_BLOCK_SIZE, }, .setkey = xts_skcipher_setkey, @@ -2912,8 +3005,8 @@ static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg) alg->base.cra_module = THIS_MODULE; alg->base.cra_priority = CAAM_CRA_PRIORITY; alg->base.cra_ctxsize = sizeof(struct caam_ctx); - alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_KERN_DRIVER_ONLY; + alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY); alg->init = caam_cra_init_skcipher; alg->exit = caam_cra_exit; @@ -2951,7 +3044,7 @@ enum hash_optype { }; /** - * caam_hash_ctx - ahash per-session context + * struct caam_hash_ctx - ahash per-session context * @flc: Flow Contexts array * @key: authentication key * @flc_dma: I/O virtual addresses of the Flow Contexts @@ -5115,8 +5208,7 @@ static int dpaa2_caam_probe(struct fsl_mc_device *dpseci_dev) /* DPIO */ err = dpaa2_dpseci_dpio_setup(priv); if (err) { - if (err != -EPROBE_DEFER) - dev_err(dev, "dpaa2_dpseci_dpio_setup() failed\n"); + dev_err_probe(dev, err, "dpaa2_dpseci_dpio_setup() failed\n"); goto err_dpio_setup; } diff --git a/drivers/crypto/caam/caamalg_qi2.h b/drivers/crypto/caam/caamalg_qi2.h index f29cb7bd7dd3..d35253407ade 100644 --- a/drivers/crypto/caam/caamalg_qi2.h +++ b/drivers/crypto/caam/caamalg_qi2.h @@ -13,6 +13,7 @@ #include <linux/netdevice.h> #include "dpseci.h" #include "desc_constr.h" +#include <crypto/skcipher.h> #define DPAA2_CAAM_STORE_SIZE 16 /* NAPI weight *must* be a multiple of the store size. */ @@ -186,6 +187,7 @@ struct caam_request { void (*cbk)(void *ctx, u32 err); void *ctx; void *edesc; + struct skcipher_request fallback_req; }; /** diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c index 94502f1d4b48..ca0361b2dbb0 100644 --- a/drivers/crypto/caam/ctrl.c +++ b/drivers/crypto/caam/ctrl.c @@ -13,6 +13,7 @@ #include <linux/fsl/mc.h> #include "compat.h" +#include "debugfs.h" #include "regs.h" #include "intern.h" #include "jr.h" @@ -332,11 +333,10 @@ static int instantiate_rng(struct device *ctrldev, int state_handle_mask, kfree(desc); - if (!ret) - ret = devm_add_action_or_reset(ctrldev, devm_deinstantiate_rng, - ctrldev); + if (ret) + return ret; - return ret; + return devm_add_action_or_reset(ctrldev, devm_deinstantiate_rng, ctrldev); } /* @@ -443,7 +443,9 @@ static int caam_get_era_from_hw(struct caam_ctrl __iomem *ctrl) * by u-boot. * In case this property is not passed an attempt to retrieve the CAAM * era via register reads will be made. - **/ + * + * @ctrl: controller region + */ static int caam_get_era(struct caam_ctrl __iomem *ctrl) { struct device_node *caam_node; @@ -582,12 +584,10 @@ static int init_clocks(struct device *dev, const struct caam_imx_data *data) return devm_add_action_or_reset(dev, disable_clocks, ctrlpriv); } -#ifdef CONFIG_DEBUG_FS static void caam_remove_debugfs(void *root) { debugfs_remove_recursive(root); } -#endif #ifdef CONFIG_FSL_MC_BUS static bool check_version(struct fsl_mc_version *mc_version, u32 major, @@ -619,10 +619,7 @@ static int caam_probe(struct platform_device *pdev) struct device_node *nprop, *np; struct caam_ctrl __iomem *ctrl; struct caam_drv_private *ctrlpriv; -#ifdef CONFIG_DEBUG_FS - struct caam_perfmon *perfmon; struct dentry *dfs_root; -#endif u32 scfgr, comp_params; u8 rng_vid; int pg_size; @@ -777,21 +774,15 @@ static int caam_probe(struct platform_device *pdev) ctrlpriv->era = caam_get_era(ctrl); ctrlpriv->domain = iommu_get_domain_for_dev(dev); -#ifdef CONFIG_DEBUG_FS - /* - * FIXME: needs better naming distinction, as some amalgamation of - * "caam" and nprop->full_name. The OF name isn't distinctive, - * but does separate instances - */ - perfmon = (struct caam_perfmon __force *)&ctrl->perfmon; - dfs_root = debugfs_create_dir(dev_name(dev), NULL); - ret = devm_add_action_or_reset(dev, caam_remove_debugfs, dfs_root); - if (ret) - return ret; + if (IS_ENABLED(CONFIG_DEBUG_FS)) { + ret = devm_add_action_or_reset(dev, caam_remove_debugfs, + dfs_root); + if (ret) + return ret; + } - ctrlpriv->ctl = debugfs_create_dir("ctl", dfs_root); -#endif + caam_debugfs_init(ctrlpriv, dfs_root); /* Check to see if (DPAA 1.x) QI present. If so, enable */ if (ctrlpriv->qi_present && !caam_dpaa2) { @@ -912,57 +903,6 @@ static int caam_probe(struct platform_device *pdev) dev_info(dev, "job rings = %d, qi = %d\n", ctrlpriv->total_jobrs, ctrlpriv->qi_present); -#ifdef CONFIG_DEBUG_FS - debugfs_create_file("rq_dequeued", S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->req_dequeued, - &caam_fops_u64_ro); - debugfs_create_file("ob_rq_encrypted", S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ob_enc_req, - &caam_fops_u64_ro); - debugfs_create_file("ib_rq_decrypted", S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ib_dec_req, - &caam_fops_u64_ro); - debugfs_create_file("ob_bytes_encrypted", S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ob_enc_bytes, - &caam_fops_u64_ro); - debugfs_create_file("ob_bytes_protected", S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ob_prot_bytes, - &caam_fops_u64_ro); - debugfs_create_file("ib_bytes_decrypted", S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ib_dec_bytes, - &caam_fops_u64_ro); - debugfs_create_file("ib_bytes_validated", S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ib_valid_bytes, - &caam_fops_u64_ro); - - /* Controller level - global status values */ - debugfs_create_file("fault_addr", S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->faultaddr, - &caam_fops_u32_ro); - debugfs_create_file("fault_detail", S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->faultdetail, - &caam_fops_u32_ro); - debugfs_create_file("fault_status", S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->status, - &caam_fops_u32_ro); - - /* Internal covering keys (useful in non-secure mode only) */ - ctrlpriv->ctl_kek_wrap.data = (__force void *)&ctrlpriv->ctrl->kek[0]; - ctrlpriv->ctl_kek_wrap.size = KEK_KEY_SIZE * sizeof(u32); - debugfs_create_blob("kek", S_IRUSR | S_IRGRP | S_IROTH, ctrlpriv->ctl, - &ctrlpriv->ctl_kek_wrap); - - ctrlpriv->ctl_tkek_wrap.data = (__force void *)&ctrlpriv->ctrl->tkek[0]; - ctrlpriv->ctl_tkek_wrap.size = KEK_KEY_SIZE * sizeof(u32); - debugfs_create_blob("tkek", S_IRUSR | S_IRGRP | S_IROTH, ctrlpriv->ctl, - &ctrlpriv->ctl_tkek_wrap); - - ctrlpriv->ctl_tdsk_wrap.data = (__force void *)&ctrlpriv->ctrl->tdsk[0]; - ctrlpriv->ctl_tdsk_wrap.size = KEK_KEY_SIZE * sizeof(u32); - debugfs_create_blob("tdsk", S_IRUSR | S_IRGRP | S_IROTH, ctrlpriv->ctl, - &ctrlpriv->ctl_tdsk_wrap); -#endif - ret = devm_of_platform_populate(dev); if (ret) dev_err(dev, "JR platform devices creation error\n"); diff --git a/drivers/crypto/caam/debugfs.c b/drivers/crypto/caam/debugfs.c new file mode 100644 index 000000000000..8ebf18398166 --- /dev/null +++ b/drivers/crypto/caam/debugfs.c @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) +/* Copyright 2019 NXP */ + +#include <linux/debugfs.h> +#include "compat.h" +#include "debugfs.h" +#include "regs.h" +#include "intern.h" + +static int caam_debugfs_u64_get(void *data, u64 *val) +{ + *val = caam64_to_cpu(*(u64 *)data); + return 0; +} + +static int caam_debugfs_u32_get(void *data, u64 *val) +{ + *val = caam32_to_cpu(*(u32 *)data); + return 0; +} + +DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u32_ro, caam_debugfs_u32_get, NULL, "%llu\n"); +DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u64_ro, caam_debugfs_u64_get, NULL, "%llu\n"); + +#ifdef CONFIG_CAAM_QI +/* + * This is a counter for the number of times the congestion group (where all + * the request and response queueus are) reached congestion. Incremented + * each time the congestion callback is called with congested == true. + */ +static u64 times_congested; + +void caam_debugfs_qi_congested(void) +{ + times_congested++; +} + +void caam_debugfs_qi_init(struct caam_drv_private *ctrlpriv) +{ + debugfs_create_file("qi_congested", 0444, ctrlpriv->ctl, + ×_congested, &caam_fops_u64_ro); +} +#endif + +void caam_debugfs_init(struct caam_drv_private *ctrlpriv, struct dentry *root) +{ + struct caam_perfmon *perfmon; + + /* + * FIXME: needs better naming distinction, as some amalgamation of + * "caam" and nprop->full_name. The OF name isn't distinctive, + * but does separate instances + */ + perfmon = (struct caam_perfmon __force *)&ctrlpriv->ctrl->perfmon; + + ctrlpriv->ctl = debugfs_create_dir("ctl", root); + + debugfs_create_file("rq_dequeued", 0444, ctrlpriv->ctl, + &perfmon->req_dequeued, &caam_fops_u64_ro); + debugfs_create_file("ob_rq_encrypted", 0444, ctrlpriv->ctl, + &perfmon->ob_enc_req, &caam_fops_u64_ro); + debugfs_create_file("ib_rq_decrypted", 0444, ctrlpriv->ctl, + &perfmon->ib_dec_req, &caam_fops_u64_ro); + debugfs_create_file("ob_bytes_encrypted", 0444, ctrlpriv->ctl, + &perfmon->ob_enc_bytes, &caam_fops_u64_ro); + debugfs_create_file("ob_bytes_protected", 0444, ctrlpriv->ctl, + &perfmon->ob_prot_bytes, &caam_fops_u64_ro); + debugfs_create_file("ib_bytes_decrypted", 0444, ctrlpriv->ctl, + &perfmon->ib_dec_bytes, &caam_fops_u64_ro); + debugfs_create_file("ib_bytes_validated", 0444, ctrlpriv->ctl, + &perfmon->ib_valid_bytes, &caam_fops_u64_ro); + + /* Controller level - global status values */ + debugfs_create_file("fault_addr", 0444, ctrlpriv->ctl, + &perfmon->faultaddr, &caam_fops_u32_ro); + debugfs_create_file("fault_detail", 0444, ctrlpriv->ctl, + &perfmon->faultdetail, &caam_fops_u32_ro); + debugfs_create_file("fault_status", 0444, ctrlpriv->ctl, + &perfmon->status, &caam_fops_u32_ro); + + /* Internal covering keys (useful in non-secure mode only) */ + ctrlpriv->ctl_kek_wrap.data = (__force void *)&ctrlpriv->ctrl->kek[0]; + ctrlpriv->ctl_kek_wrap.size = KEK_KEY_SIZE * sizeof(u32); + debugfs_create_blob("kek", 0444, ctrlpriv->ctl, + &ctrlpriv->ctl_kek_wrap); + + ctrlpriv->ctl_tkek_wrap.data = (__force void *)&ctrlpriv->ctrl->tkek[0]; + ctrlpriv->ctl_tkek_wrap.size = KEK_KEY_SIZE * sizeof(u32); + debugfs_create_blob("tkek", 0444, ctrlpriv->ctl, + &ctrlpriv->ctl_tkek_wrap); + + ctrlpriv->ctl_tdsk_wrap.data = (__force void *)&ctrlpriv->ctrl->tdsk[0]; + ctrlpriv->ctl_tdsk_wrap.size = KEK_KEY_SIZE * sizeof(u32); + debugfs_create_blob("tdsk", 0444, ctrlpriv->ctl, + &ctrlpriv->ctl_tdsk_wrap); +} diff --git a/drivers/crypto/caam/debugfs.h b/drivers/crypto/caam/debugfs.h new file mode 100644 index 000000000000..661d768acdbf --- /dev/null +++ b/drivers/crypto/caam/debugfs.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */ +/* Copyright 2019 NXP */ + +#ifndef CAAM_DEBUGFS_H +#define CAAM_DEBUGFS_H + +struct dentry; +struct caam_drv_private; + +#ifdef CONFIG_DEBUG_FS +void caam_debugfs_init(struct caam_drv_private *ctrlpriv, struct dentry *root); +#else +static inline void caam_debugfs_init(struct caam_drv_private *ctrlpriv, + struct dentry *root) +{} +#endif + +#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_CAAM_QI) +void caam_debugfs_qi_congested(void); +void caam_debugfs_qi_init(struct caam_drv_private *ctrlpriv); +#else +static inline void caam_debugfs_qi_congested(void) {} +static inline void caam_debugfs_qi_init(struct caam_drv_private *ctrlpriv) {} +#endif + +#endif /* CAAM_DEBUGFS_H */ diff --git a/drivers/crypto/caam/dpseci-debugfs.c b/drivers/crypto/caam/dpseci-debugfs.c index c5bfc923abd8..0eca8c2fd916 100644 --- a/drivers/crypto/caam/dpseci-debugfs.c +++ b/drivers/crypto/caam/dpseci-debugfs.c @@ -44,33 +44,14 @@ static int dpseci_dbg_fqs_show(struct seq_file *file, void *offset) return 0; } -static int dpseci_dbg_fqs_open(struct inode *inode, struct file *file) -{ - int err; - struct dpaa2_caam_priv *priv; - - priv = (struct dpaa2_caam_priv *)inode->i_private; - - err = single_open(file, dpseci_dbg_fqs_show, priv); - if (err < 0) - dev_err(priv->dev, "single_open() failed\n"); - - return err; -} - -static const struct file_operations dpseci_dbg_fq_ops = { - .open = dpseci_dbg_fqs_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(dpseci_dbg_fqs); void dpaa2_dpseci_debugfs_init(struct dpaa2_caam_priv *priv) { priv->dfs_root = debugfs_create_dir(dev_name(priv->dev), NULL); debugfs_create_file("fq_stats", 0444, priv->dfs_root, priv, - &dpseci_dbg_fq_ops); + &dpseci_dbg_fqs_fops); } void dpaa2_dpseci_debugfs_exit(struct dpaa2_caam_priv *priv) diff --git a/drivers/crypto/caam/intern.h b/drivers/crypto/caam/intern.h index 402d6a362e8c..9112279a4de0 100644 --- a/drivers/crypto/caam/intern.h +++ b/drivers/crypto/caam/intern.h @@ -195,23 +195,6 @@ static inline void caam_qi_algapi_exit(void) #endif /* CONFIG_CAAM_QI */ -#ifdef CONFIG_DEBUG_FS -static int caam_debugfs_u64_get(void *data, u64 *val) -{ - *val = caam64_to_cpu(*(u64 *)data); - return 0; -} - -static int caam_debugfs_u32_get(void *data, u64 *val) -{ - *val = caam32_to_cpu(*(u32 *)data); - return 0; -} - -DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u32_ro, caam_debugfs_u32_get, NULL, "%llu\n"); -DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u64_ro, caam_debugfs_u64_get, NULL, "%llu\n"); -#endif - static inline u64 caam_get_dma_mask(struct device *dev) { struct device_node *nprop = dev->of_node; diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c index bf6b03b17251..6f669966ba2c 100644 --- a/drivers/crypto/caam/jr.c +++ b/drivers/crypto/caam/jr.c @@ -324,7 +324,7 @@ EXPORT_SYMBOL(caam_jr_alloc); /** * caam_jr_free() - Free the Job Ring - * @rdev - points to the dev that identifies the Job ring to + * @rdev: points to the dev that identifies the Job ring to * be released. **/ void caam_jr_free(struct device *rdev) @@ -349,15 +349,15 @@ EXPORT_SYMBOL(caam_jr_free); * of this request. This has the form: * callback(struct device *dev, u32 *desc, u32 stat, void *arg) * where: - * @dev: contains the job ring device that processed this + * dev: contains the job ring device that processed this * response. - * @desc: descriptor that initiated the request, same as + * desc: descriptor that initiated the request, same as * "desc" being argued to caam_jr_enqueue(). - * @status: untranslated status received from CAAM. See the + * status: untranslated status received from CAAM. See the * reference manual for a detailed description of * error meaning, or see the JRSTA definitions in the * register header file - * @areq: optional pointer to an argument passed with the + * areq: optional pointer to an argument passed with the * original request * @areq: optional pointer to a user argument for use at callback * time. diff --git a/drivers/crypto/caam/qi.c b/drivers/crypto/caam/qi.c index b390b935db6d..ec53528d8205 100644 --- a/drivers/crypto/caam/qi.c +++ b/drivers/crypto/caam/qi.c @@ -11,6 +11,7 @@ #include <linux/kthread.h> #include <soc/fsl/qman.h> +#include "debugfs.h" #include "regs.h" #include "qi.h" #include "desc.h" @@ -73,15 +74,6 @@ static struct caam_qi_priv qipriv ____cacheline_aligned; bool caam_congested __read_mostly; EXPORT_SYMBOL(caam_congested); -#ifdef CONFIG_DEBUG_FS -/* - * This is a counter for the number of times the congestion group (where all - * the request and response queueus are) reached congestion. Incremented - * each time the congestion callback is called with congested == true. - */ -static u64 times_congested; -#endif - /* * This is a a cache of buffers, from which the users of CAAM QI driver * can allocate short (CAAM_QI_MEMCACHE_SIZE) buffers. It's faster than @@ -544,9 +536,8 @@ static void cgr_cb(struct qman_portal *qm, struct qman_cgr *cgr, int congested) caam_congested = congested; if (congested) { -#ifdef CONFIG_DEBUG_FS - times_congested++; -#endif + caam_debugfs_qi_congested(); + pr_debug_ratelimited("CAAM entered congestion\n"); } else { @@ -775,10 +766,7 @@ int caam_qi_init(struct platform_device *caam_pdev) return -ENOMEM; } -#ifdef CONFIG_DEBUG_FS - debugfs_create_file("qi_congested", 0444, ctrlpriv->ctl, - ×_congested, &caam_fops_u64_ro); -#endif + caam_debugfs_qi_init(ctrlpriv); err = devm_add_action_or_reset(qidev, caam_qi_shutdown, ctrlpriv); if (err) diff --git a/drivers/crypto/cavium/cpt/cptvf_algs.c b/drivers/crypto/cavium/cpt/cptvf_algs.c index 5af0dc2a8909..ce3b91c612f0 100644 --- a/drivers/crypto/cavium/cpt/cptvf_algs.c +++ b/drivers/crypto/cavium/cpt/cptvf_algs.c @@ -451,13 +451,7 @@ static struct skcipher_alg algs[] = { { static inline int cav_register_algs(void) { - int err = 0; - - err = crypto_register_skciphers(algs, ARRAY_SIZE(algs)); - if (err) - return err; - - return 0; + return crypto_register_skciphers(algs, ARRAY_SIZE(algs)); } static inline void cav_unregister_algs(void) diff --git a/drivers/crypto/cavium/nitrox/nitrox_main.c b/drivers/crypto/cavium/nitrox/nitrox_main.c index cee2a2713038..9d14be97e381 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_main.c +++ b/drivers/crypto/cavium/nitrox/nitrox_main.c @@ -451,6 +451,7 @@ static int nitrox_probe(struct pci_dev *pdev, err = pci_request_mem_regions(pdev, nitrox_driver_name); if (err) { pci_disable_device(pdev); + dev_err(&pdev->dev, "Failed to request mem regions!\n"); return err; } pci_set_master(pdev); diff --git a/drivers/crypto/cavium/zip/zip_main.c b/drivers/crypto/cavium/zip/zip_main.c index 194624b4855b..d35216e2f6cd 100644 --- a/drivers/crypto/cavium/zip/zip_main.c +++ b/drivers/crypto/cavium/zip/zip_main.c @@ -460,7 +460,7 @@ static void zip_unregister_compression_device(void) #include <linux/debugfs.h> /* Displays ZIP device statistics */ -static int zip_show_stats(struct seq_file *s, void *unused) +static int zip_stats_show(struct seq_file *s, void *unused) { u64 val = 0ull; u64 avg_chunk = 0ull, avg_cr = 0ull; @@ -523,7 +523,7 @@ static int zip_show_stats(struct seq_file *s, void *unused) } /* Clears stats data */ -static int zip_clear_stats(struct seq_file *s, void *unused) +static int zip_clear_show(struct seq_file *s, void *unused) { int index = 0; @@ -558,7 +558,7 @@ static struct zip_registers zipregs[64] = { }; /* Prints registers' contents */ -static int zip_print_regs(struct seq_file *s, void *unused) +static int zip_regs_show(struct seq_file *s, void *unused) { u64 val = 0; int i = 0, index = 0; @@ -584,41 +584,9 @@ static int zip_print_regs(struct seq_file *s, void *unused) return 0; } -static int zip_stats_open(struct inode *inode, struct file *file) -{ - return single_open(file, zip_show_stats, NULL); -} - -static const struct file_operations zip_stats_fops = { - .owner = THIS_MODULE, - .open = zip_stats_open, - .read = seq_read, - .release = single_release, -}; - -static int zip_clear_open(struct inode *inode, struct file *file) -{ - return single_open(file, zip_clear_stats, NULL); -} - -static const struct file_operations zip_clear_fops = { - .owner = THIS_MODULE, - .open = zip_clear_open, - .read = seq_read, - .release = single_release, -}; - -static int zip_regs_open(struct inode *inode, struct file *file) -{ - return single_open(file, zip_print_regs, NULL); -} - -static const struct file_operations zip_regs_fops = { - .owner = THIS_MODULE, - .open = zip_regs_open, - .read = seq_read, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(zip_stats); +DEFINE_SHOW_ATTRIBUTE(zip_clear); +DEFINE_SHOW_ATTRIBUTE(zip_regs); /* Root directory for thunderx_zip debugfs entry */ static struct dentry *zip_debugfs_root; diff --git a/drivers/crypto/ccp/ccp-ops.c b/drivers/crypto/ccp/ccp-ops.c index bd270e66185e..d6a8f4e4b14a 100644 --- a/drivers/crypto/ccp/ccp-ops.c +++ b/drivers/crypto/ccp/ccp-ops.c @@ -8,6 +8,7 @@ * Author: Gary R Hook <[email protected]> */ +#include <linux/dma-mapping.h> #include <linux/module.h> #include <linux/kernel.h> #include <linux/interrupt.h> @@ -1744,7 +1745,7 @@ ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) break; default: ret = -EINVAL; - goto e_ctx; + goto e_data; } } else { /* Stash the context */ diff --git a/drivers/crypto/ccree/cc_cipher.c b/drivers/crypto/ccree/cc_cipher.c index d77ae981b64b..dafa6577a845 100644 --- a/drivers/crypto/ccree/cc_cipher.c +++ b/drivers/crypto/ccree/cc_cipher.c @@ -75,8 +75,7 @@ static int validate_keys_sizes(struct cc_cipher_ctx *ctx_p, u32 size) switch (size) { case CC_AES_128_BIT_KEY_SIZE: case CC_AES_192_BIT_KEY_SIZE: - if (ctx_p->cipher_mode != DRV_CIPHER_XTS && - ctx_p->cipher_mode != DRV_CIPHER_BITLOCKER) + if (ctx_p->cipher_mode != DRV_CIPHER_XTS) return 0; break; case CC_AES_256_BIT_KEY_SIZE: @@ -84,8 +83,7 @@ static int validate_keys_sizes(struct cc_cipher_ctx *ctx_p, u32 size) case (CC_AES_192_BIT_KEY_SIZE * 2): case (CC_AES_256_BIT_KEY_SIZE * 2): if (ctx_p->cipher_mode == DRV_CIPHER_XTS || - ctx_p->cipher_mode == DRV_CIPHER_ESSIV || - ctx_p->cipher_mode == DRV_CIPHER_BITLOCKER) + ctx_p->cipher_mode == DRV_CIPHER_ESSIV) return 0; break; default: @@ -122,7 +120,6 @@ static int validate_data_size(struct cc_cipher_ctx *ctx_p, case DRV_CIPHER_ECB: case DRV_CIPHER_CBC: case DRV_CIPHER_ESSIV: - case DRV_CIPHER_BITLOCKER: if (IS_ALIGNED(size, AES_BLOCK_SIZE)) return 0; break; @@ -348,8 +345,7 @@ static int cc_cipher_sethkey(struct crypto_skcipher *sktfm, const u8 *key, } if (ctx_p->cipher_mode == DRV_CIPHER_XTS || - ctx_p->cipher_mode == DRV_CIPHER_ESSIV || - ctx_p->cipher_mode == DRV_CIPHER_BITLOCKER) { + ctx_p->cipher_mode == DRV_CIPHER_ESSIV) { if (hki.hw_key1 == hki.hw_key2) { dev_err(dev, "Illegal hw key numbers (%d,%d)\n", hki.hw_key1, hki.hw_key2); @@ -547,7 +543,6 @@ static void cc_setup_readiv_desc(struct crypto_tfm *tfm, break; case DRV_CIPHER_XTS: case DRV_CIPHER_ESSIV: - case DRV_CIPHER_BITLOCKER: /* IV */ hw_desc_init(&desc[*seq_size]); set_setup_mode(&desc[*seq_size], SETUP_WRITE_STATE1); @@ -602,7 +597,6 @@ static void cc_setup_state_desc(struct crypto_tfm *tfm, break; case DRV_CIPHER_XTS: case DRV_CIPHER_ESSIV: - case DRV_CIPHER_BITLOCKER: break; default: dev_err(dev, "Unsupported cipher mode (%d)\n", cipher_mode); @@ -624,16 +618,8 @@ static void cc_setup_xex_state_desc(struct crypto_tfm *tfm, dma_addr_t key_dma_addr = ctx_p->user.key_dma_addr; unsigned int key_len = (ctx_p->keylen / 2); dma_addr_t iv_dma_addr = req_ctx->gen_ctx.iv_dma_addr; - unsigned int du_size = nbytes; unsigned int key_offset = key_len; - struct cc_crypto_alg *cc_alg = - container_of(tfm->__crt_alg, struct cc_crypto_alg, - skcipher_alg.base); - - if (cc_alg->data_unit) - du_size = cc_alg->data_unit; - switch (cipher_mode) { case DRV_CIPHER_ECB: break; @@ -644,7 +630,6 @@ static void cc_setup_xex_state_desc(struct crypto_tfm *tfm, break; case DRV_CIPHER_XTS: case DRV_CIPHER_ESSIV: - case DRV_CIPHER_BITLOCKER: if (cipher_mode == DRV_CIPHER_ESSIV) key_len = SHA256_DIGEST_SIZE; @@ -661,7 +646,7 @@ static void cc_setup_xex_state_desc(struct crypto_tfm *tfm, (key_dma_addr + key_offset), key_len, NS_BIT); } - set_xex_data_unit_size(&desc[*seq_size], du_size); + set_xex_data_unit_size(&desc[*seq_size], nbytes); set_flow_mode(&desc[*seq_size], S_DIN_to_AES2); set_key_size_aes(&desc[*seq_size], key_len); set_setup_mode(&desc[*seq_size], SETUP_LOAD_XEX_KEY); @@ -758,7 +743,6 @@ static void cc_setup_key_desc(struct crypto_tfm *tfm, break; case DRV_CIPHER_XTS: case DRV_CIPHER_ESSIV: - case DRV_CIPHER_BITLOCKER: /* Load AES key */ hw_desc_init(&desc[*seq_size]); set_cipher_mode(&desc[*seq_size], cipher_mode); @@ -1039,44 +1023,6 @@ static const struct cc_alg_template skcipher_algs[] = { .sec_func = true, }, { - .name = "xts512(paes)", - .driver_name = "xts-paes-du512-ccree", - .blocksize = 1, - .template_skcipher = { - .setkey = cc_cipher_sethkey, - .encrypt = cc_cipher_encrypt, - .decrypt = cc_cipher_decrypt, - .min_keysize = CC_HW_KEY_SIZE, - .max_keysize = CC_HW_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - }, - .cipher_mode = DRV_CIPHER_XTS, - .flow_mode = S_DIN_to_AES, - .data_unit = 512, - .min_hw_rev = CC_HW_REV_712, - .std_body = CC_STD_NIST, - .sec_func = true, - }, - { - .name = "xts4096(paes)", - .driver_name = "xts-paes-du4096-ccree", - .blocksize = 1, - .template_skcipher = { - .setkey = cc_cipher_sethkey, - .encrypt = cc_cipher_encrypt, - .decrypt = cc_cipher_decrypt, - .min_keysize = CC_HW_KEY_SIZE, - .max_keysize = CC_HW_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - }, - .cipher_mode = DRV_CIPHER_XTS, - .flow_mode = S_DIN_to_AES, - .data_unit = 4096, - .min_hw_rev = CC_HW_REV_712, - .std_body = CC_STD_NIST, - .sec_func = true, - }, - { .name = "essiv(cbc(paes),sha256)", .driver_name = "essiv-paes-ccree", .blocksize = AES_BLOCK_SIZE, @@ -1095,100 +1041,6 @@ static const struct cc_alg_template skcipher_algs[] = { .sec_func = true, }, { - .name = "essiv512(cbc(paes),sha256)", - .driver_name = "essiv-paes-du512-ccree", - .blocksize = AES_BLOCK_SIZE, - .template_skcipher = { - .setkey = cc_cipher_sethkey, - .encrypt = cc_cipher_encrypt, - .decrypt = cc_cipher_decrypt, - .min_keysize = CC_HW_KEY_SIZE, - .max_keysize = CC_HW_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - }, - .cipher_mode = DRV_CIPHER_ESSIV, - .flow_mode = S_DIN_to_AES, - .data_unit = 512, - .min_hw_rev = CC_HW_REV_712, - .std_body = CC_STD_NIST, - .sec_func = true, - }, - { - .name = "essiv4096(cbc(paes),sha256)", - .driver_name = "essiv-paes-du4096-ccree", - .blocksize = AES_BLOCK_SIZE, - .template_skcipher = { - .setkey = cc_cipher_sethkey, - .encrypt = cc_cipher_encrypt, - .decrypt = cc_cipher_decrypt, - .min_keysize = CC_HW_KEY_SIZE, - .max_keysize = CC_HW_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - }, - .cipher_mode = DRV_CIPHER_ESSIV, - .flow_mode = S_DIN_to_AES, - .data_unit = 4096, - .min_hw_rev = CC_HW_REV_712, - .std_body = CC_STD_NIST, - .sec_func = true, - }, - { - .name = "bitlocker(paes)", - .driver_name = "bitlocker-paes-ccree", - .blocksize = AES_BLOCK_SIZE, - .template_skcipher = { - .setkey = cc_cipher_sethkey, - .encrypt = cc_cipher_encrypt, - .decrypt = cc_cipher_decrypt, - .min_keysize = CC_HW_KEY_SIZE, - .max_keysize = CC_HW_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - }, - .cipher_mode = DRV_CIPHER_BITLOCKER, - .flow_mode = S_DIN_to_AES, - .min_hw_rev = CC_HW_REV_712, - .std_body = CC_STD_NIST, - .sec_func = true, - }, - { - .name = "bitlocker512(paes)", - .driver_name = "bitlocker-paes-du512-ccree", - .blocksize = AES_BLOCK_SIZE, - .template_skcipher = { - .setkey = cc_cipher_sethkey, - .encrypt = cc_cipher_encrypt, - .decrypt = cc_cipher_decrypt, - .min_keysize = CC_HW_KEY_SIZE, - .max_keysize = CC_HW_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - }, - .cipher_mode = DRV_CIPHER_BITLOCKER, - .flow_mode = S_DIN_to_AES, - .data_unit = 512, - .min_hw_rev = CC_HW_REV_712, - .std_body = CC_STD_NIST, - .sec_func = true, - }, - { - .name = "bitlocker4096(paes)", - .driver_name = "bitlocker-paes-du4096-ccree", - .blocksize = AES_BLOCK_SIZE, - .template_skcipher = { - .setkey = cc_cipher_sethkey, - .encrypt = cc_cipher_encrypt, - .decrypt = cc_cipher_decrypt, - .min_keysize = CC_HW_KEY_SIZE, - .max_keysize = CC_HW_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - }, - .cipher_mode = DRV_CIPHER_BITLOCKER, - .flow_mode = S_DIN_to_AES, - .data_unit = 4096, - .min_hw_rev = CC_HW_REV_712, - .std_body = CC_STD_NIST, - .sec_func = true, - }, - { .name = "ecb(paes)", .driver_name = "ecb-paes-ccree", .blocksize = AES_BLOCK_SIZE, @@ -1300,42 +1152,6 @@ static const struct cc_alg_template skcipher_algs[] = { .std_body = CC_STD_NIST, }, { - .name = "xts512(aes)", - .driver_name = "xts-aes-du512-ccree", - .blocksize = 1, - .template_skcipher = { - .setkey = cc_cipher_setkey, - .encrypt = cc_cipher_encrypt, - .decrypt = cc_cipher_decrypt, - .min_keysize = AES_MIN_KEY_SIZE * 2, - .max_keysize = AES_MAX_KEY_SIZE * 2, - .ivsize = AES_BLOCK_SIZE, - }, - .cipher_mode = DRV_CIPHER_XTS, - .flow_mode = S_DIN_to_AES, - .data_unit = 512, - .min_hw_rev = CC_HW_REV_712, - .std_body = CC_STD_NIST, - }, - { - .name = "xts4096(aes)", - .driver_name = "xts-aes-du4096-ccree", - .blocksize = 1, - .template_skcipher = { - .setkey = cc_cipher_setkey, - .encrypt = cc_cipher_encrypt, - .decrypt = cc_cipher_decrypt, - .min_keysize = AES_MIN_KEY_SIZE * 2, - .max_keysize = AES_MAX_KEY_SIZE * 2, - .ivsize = AES_BLOCK_SIZE, - }, - .cipher_mode = DRV_CIPHER_XTS, - .flow_mode = S_DIN_to_AES, - .data_unit = 4096, - .min_hw_rev = CC_HW_REV_712, - .std_body = CC_STD_NIST, - }, - { .name = "essiv(cbc(aes),sha256)", .driver_name = "essiv-aes-ccree", .blocksize = AES_BLOCK_SIZE, @@ -1353,95 +1169,6 @@ static const struct cc_alg_template skcipher_algs[] = { .std_body = CC_STD_NIST, }, { - .name = "essiv512(cbc(aes),sha256)", - .driver_name = "essiv-aes-du512-ccree", - .blocksize = AES_BLOCK_SIZE, - .template_skcipher = { - .setkey = cc_cipher_setkey, - .encrypt = cc_cipher_encrypt, - .decrypt = cc_cipher_decrypt, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - }, - .cipher_mode = DRV_CIPHER_ESSIV, - .flow_mode = S_DIN_to_AES, - .data_unit = 512, - .min_hw_rev = CC_HW_REV_712, - .std_body = CC_STD_NIST, - }, - { - .name = "essiv4096(cbc(aes),sha256)", - .driver_name = "essiv-aes-du4096-ccree", - .blocksize = AES_BLOCK_SIZE, - .template_skcipher = { - .setkey = cc_cipher_setkey, - .encrypt = cc_cipher_encrypt, - .decrypt = cc_cipher_decrypt, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - }, - .cipher_mode = DRV_CIPHER_ESSIV, - .flow_mode = S_DIN_to_AES, - .data_unit = 4096, - .min_hw_rev = CC_HW_REV_712, - .std_body = CC_STD_NIST, - }, - { - .name = "bitlocker(aes)", - .driver_name = "bitlocker-aes-ccree", - .blocksize = AES_BLOCK_SIZE, - .template_skcipher = { - .setkey = cc_cipher_setkey, - .encrypt = cc_cipher_encrypt, - .decrypt = cc_cipher_decrypt, - .min_keysize = AES_MIN_KEY_SIZE * 2, - .max_keysize = AES_MAX_KEY_SIZE * 2, - .ivsize = AES_BLOCK_SIZE, - }, - .cipher_mode = DRV_CIPHER_BITLOCKER, - .flow_mode = S_DIN_to_AES, - .min_hw_rev = CC_HW_REV_712, - .std_body = CC_STD_NIST, - }, - { - .name = "bitlocker512(aes)", - .driver_name = "bitlocker-aes-du512-ccree", - .blocksize = AES_BLOCK_SIZE, - .template_skcipher = { - .setkey = cc_cipher_setkey, - .encrypt = cc_cipher_encrypt, - .decrypt = cc_cipher_decrypt, - .min_keysize = AES_MIN_KEY_SIZE * 2, - .max_keysize = AES_MAX_KEY_SIZE * 2, - .ivsize = AES_BLOCK_SIZE, - }, - .cipher_mode = DRV_CIPHER_BITLOCKER, - .flow_mode = S_DIN_to_AES, - .data_unit = 512, - .min_hw_rev = CC_HW_REV_712, - .std_body = CC_STD_NIST, - }, - { - .name = "bitlocker4096(aes)", - .driver_name = "bitlocker-aes-du4096-ccree", - .blocksize = AES_BLOCK_SIZE, - .template_skcipher = { - .setkey = cc_cipher_setkey, - .encrypt = cc_cipher_encrypt, - .decrypt = cc_cipher_decrypt, - .min_keysize = AES_MIN_KEY_SIZE * 2, - .max_keysize = AES_MAX_KEY_SIZE * 2, - .ivsize = AES_BLOCK_SIZE, - }, - .cipher_mode = DRV_CIPHER_BITLOCKER, - .flow_mode = S_DIN_to_AES, - .data_unit = 4096, - .min_hw_rev = CC_HW_REV_712, - .std_body = CC_STD_NIST, - }, - { .name = "ecb(aes)", .driver_name = "ecb-aes-ccree", .blocksize = AES_BLOCK_SIZE, @@ -1712,7 +1439,6 @@ static struct cc_crypto_alg *cc_create_alg(const struct cc_alg_template *tmpl, t_alg->cipher_mode = tmpl->cipher_mode; t_alg->flow_mode = tmpl->flow_mode; - t_alg->data_unit = tmpl->data_unit; return t_alg; } diff --git a/drivers/crypto/ccree/cc_crypto_ctx.h b/drivers/crypto/ccree/cc_crypto_ctx.h index ccf960a0d989..bd9a1c0896b3 100644 --- a/drivers/crypto/ccree/cc_crypto_ctx.h +++ b/drivers/crypto/ccree/cc_crypto_ctx.h @@ -108,7 +108,6 @@ enum drv_cipher_mode { DRV_CIPHER_CBC_CTS = 11, DRV_CIPHER_GCTR = 12, DRV_CIPHER_ESSIV = 13, - DRV_CIPHER_BITLOCKER = 14, DRV_CIPHER_RESERVE32B = S32_MAX }; diff --git a/drivers/crypto/ccree/cc_driver.c b/drivers/crypto/ccree/cc_driver.c index 2d50991b9a17..6f519d3e896c 100644 --- a/drivers/crypto/ccree/cc_driver.c +++ b/drivers/crypto/ccree/cc_driver.c @@ -300,11 +300,8 @@ static int init_cc_resources(struct platform_device *plat_dev) new_drvdata->plat_dev = plat_dev; clk = devm_clk_get_optional(dev, NULL); - if (IS_ERR(clk)) { - if (PTR_ERR(clk) != -EPROBE_DEFER) - dev_err(dev, "Error getting clock: %pe\n", clk); - return PTR_ERR(clk); - } + if (IS_ERR(clk)) + return dev_err_probe(dev, PTR_ERR(clk), "Error getting clock\n"); new_drvdata->clk = clk; new_drvdata->coherent = of_dma_is_coherent(np); diff --git a/drivers/crypto/ccree/cc_driver.h b/drivers/crypto/ccree/cc_driver.h index d938886390d2..af77b2020350 100644 --- a/drivers/crypto/ccree/cc_driver.h +++ b/drivers/crypto/ccree/cc_driver.h @@ -162,7 +162,6 @@ struct cc_crypto_alg { int cipher_mode; int flow_mode; /* Note: currently, refers to the cipher mode only. */ int auth_mode; - unsigned int data_unit; struct cc_drvdata *drvdata; struct skcipher_alg skcipher_alg; struct aead_alg aead_alg; diff --git a/drivers/crypto/ccree/cc_pm.c b/drivers/crypto/ccree/cc_pm.c index d39e1664fc7e..3c65bf070c90 100644 --- a/drivers/crypto/ccree/cc_pm.c +++ b/drivers/crypto/ccree/cc_pm.c @@ -65,8 +65,12 @@ const struct dev_pm_ops ccree_pm = { int cc_pm_get(struct device *dev) { int rc = pm_runtime_get_sync(dev); + if (rc < 0) { + pm_runtime_put_noidle(dev); + return rc; + } - return (rc == 1 ? 0 : rc); + return 0; } void cc_pm_put_suspend(struct device *dev) diff --git a/drivers/crypto/chelsio/Kconfig b/drivers/crypto/chelsio/Kconfig index 2984fdf51e85..f886401af13e 100644 --- a/drivers/crypto/chelsio/Kconfig +++ b/drivers/crypto/chelsio/Kconfig @@ -21,35 +21,3 @@ config CRYPTO_DEV_CHELSIO To compile this driver as a module, choose M here: the module will be called chcr. - -config CHELSIO_IPSEC_INLINE - bool "Chelsio IPSec XFRM Tx crypto offload" - depends on CHELSIO_T4 - depends on CRYPTO_DEV_CHELSIO - depends on XFRM_OFFLOAD - depends on INET_ESP_OFFLOAD || INET6_ESP_OFFLOAD - default n - help - Enable support for IPSec Tx Inline. - -config CRYPTO_DEV_CHELSIO_TLS - tristate "Chelsio Crypto Inline TLS Driver" - depends on CHELSIO_T4 - depends on TLS_TOE - select CRYPTO_DEV_CHELSIO - help - Support Chelsio Inline TLS with Chelsio crypto accelerator. - - To compile this driver as a module, choose M here: the module - will be called chtls. - -config CHELSIO_TLS_DEVICE - bool "Chelsio Inline KTLS Offload" - depends on CHELSIO_T4 - depends on TLS_DEVICE - select CRYPTO_DEV_CHELSIO - default y - help - This flag enables support for kernel tls offload over Chelsio T6 - crypto accelerator. CONFIG_CHELSIO_TLS_DEVICE flag can be enabled - only if CONFIG_TLS and CONFIG_TLS_DEVICE flags are enabled. diff --git a/drivers/crypto/chelsio/Makefile b/drivers/crypto/chelsio/Makefile index 0e9d035927e9..2e5df484ab01 100644 --- a/drivers/crypto/chelsio/Makefile +++ b/drivers/crypto/chelsio/Makefile @@ -3,8 +3,3 @@ ccflags-y := -I $(srctree)/drivers/net/ethernet/chelsio/cxgb4 obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chcr.o chcr-objs := chcr_core.o chcr_algo.o -#ifdef CONFIG_CHELSIO_TLS_DEVICE -chcr-objs += chcr_ktls.o -#endif -chcr-$(CONFIG_CHELSIO_IPSEC_INLINE) += chcr_ipsec.o -obj-$(CONFIG_CRYPTO_DEV_CHELSIO_TLS) += chtls/ diff --git a/drivers/crypto/chelsio/chcr_algo.h b/drivers/crypto/chelsio/chcr_algo.h index d4f6e010dc79..507aafe93f21 100644 --- a/drivers/crypto/chelsio/chcr_algo.h +++ b/drivers/crypto/chelsio/chcr_algo.h @@ -86,39 +86,6 @@ KEY_CONTEXT_OPAD_PRESENT_M) #define KEY_CONTEXT_OPAD_PRESENT_F KEY_CONTEXT_OPAD_PRESENT_V(1U) -#define TLS_KEYCTX_RXFLIT_CNT_S 24 -#define TLS_KEYCTX_RXFLIT_CNT_V(x) ((x) << TLS_KEYCTX_RXFLIT_CNT_S) - -#define TLS_KEYCTX_RXPROT_VER_S 20 -#define TLS_KEYCTX_RXPROT_VER_M 0xf -#define TLS_KEYCTX_RXPROT_VER_V(x) ((x) << TLS_KEYCTX_RXPROT_VER_S) - -#define TLS_KEYCTX_RXCIPH_MODE_S 16 -#define TLS_KEYCTX_RXCIPH_MODE_M 0xf -#define TLS_KEYCTX_RXCIPH_MODE_V(x) ((x) << TLS_KEYCTX_RXCIPH_MODE_S) - -#define TLS_KEYCTX_RXAUTH_MODE_S 12 -#define TLS_KEYCTX_RXAUTH_MODE_M 0xf -#define TLS_KEYCTX_RXAUTH_MODE_V(x) ((x) << TLS_KEYCTX_RXAUTH_MODE_S) - -#define TLS_KEYCTX_RXCIAU_CTRL_S 11 -#define TLS_KEYCTX_RXCIAU_CTRL_V(x) ((x) << TLS_KEYCTX_RXCIAU_CTRL_S) - -#define TLS_KEYCTX_RX_SEQCTR_S 9 -#define TLS_KEYCTX_RX_SEQCTR_M 0x3 -#define TLS_KEYCTX_RX_SEQCTR_V(x) ((x) << TLS_KEYCTX_RX_SEQCTR_S) - -#define TLS_KEYCTX_RX_VALID_S 8 -#define TLS_KEYCTX_RX_VALID_V(x) ((x) << TLS_KEYCTX_RX_VALID_S) - -#define TLS_KEYCTX_RXCK_SIZE_S 3 -#define TLS_KEYCTX_RXCK_SIZE_M 0x7 -#define TLS_KEYCTX_RXCK_SIZE_V(x) ((x) << TLS_KEYCTX_RXCK_SIZE_S) - -#define TLS_KEYCTX_RXMK_SIZE_S 0 -#define TLS_KEYCTX_RXMK_SIZE_M 0x7 -#define TLS_KEYCTX_RXMK_SIZE_V(x) ((x) << TLS_KEYCTX_RXMK_SIZE_S) - #define CHCR_HASH_MAX_DIGEST_SIZE 64 #define CHCR_MAX_SHA_DIGEST_SIZE 64 diff --git a/drivers/crypto/chelsio/chcr_common.h b/drivers/crypto/chelsio/chcr_common.h deleted file mode 100644 index 33f589cbfba1..000000000000 --- a/drivers/crypto/chelsio/chcr_common.h +++ /dev/null @@ -1,135 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright (C) 2020 Chelsio Communications. All rights reserved. */ - -#ifndef __CHCR_COMMON_H__ -#define __CHCR_COMMON_H__ - -#include "cxgb4.h" - -#define CHCR_MAX_SALT 4 -#define CHCR_KEYCTX_MAC_KEY_SIZE_128 0 -#define CHCR_KEYCTX_CIPHER_KEY_SIZE_128 0 -#define CHCR_SCMD_CIPHER_MODE_AES_GCM 2 -#define CHCR_SCMD_CIPHER_MODE_AES_CTR 3 -#define CHCR_CPL_TX_SEC_PDU_LEN_64BIT 2 -#define CHCR_SCMD_SEQ_NO_CTRL_64BIT 3 -#define CHCR_SCMD_PROTO_VERSION_TLS 0 -#define CHCR_SCMD_PROTO_VERSION_GENERIC 4 -#define CHCR_SCMD_AUTH_MODE_GHASH 4 -#define AES_BLOCK_LEN 16 - -enum chcr_state { - CHCR_INIT = 0, - CHCR_ATTACH, - CHCR_DETACH, -}; - -struct chcr_dev { - spinlock_t lock_chcr_dev; /* chcr dev structure lock */ - enum chcr_state state; - atomic_t inflight; - int wqretry; - struct delayed_work detach_work; - struct completion detach_comp; - unsigned char tx_channel_id; -}; - -struct uld_ctx { - struct list_head entry; - struct cxgb4_lld_info lldi; - struct chcr_dev dev; -}; - -struct ktls_key_ctx { - __be32 ctx_hdr; - u8 salt[CHCR_MAX_SALT]; - __be64 iv_to_auth; - unsigned char key[TLS_CIPHER_AES_GCM_128_KEY_SIZE + - TLS_CIPHER_AES_GCM_256_TAG_SIZE]; -}; - -/* Crypto key context */ -#define KEY_CONTEXT_CTX_LEN_S 24 -#define KEY_CONTEXT_CTX_LEN_V(x) ((x) << KEY_CONTEXT_CTX_LEN_S) - -#define KEY_CONTEXT_SALT_PRESENT_S 10 -#define KEY_CONTEXT_SALT_PRESENT_V(x) ((x) << KEY_CONTEXT_SALT_PRESENT_S) -#define KEY_CONTEXT_SALT_PRESENT_F KEY_CONTEXT_SALT_PRESENT_V(1U) - -#define KEY_CONTEXT_VALID_S 0 -#define KEY_CONTEXT_VALID_V(x) ((x) << KEY_CONTEXT_VALID_S) -#define KEY_CONTEXT_VALID_F KEY_CONTEXT_VALID_V(1U) - -#define KEY_CONTEXT_CK_SIZE_S 6 -#define KEY_CONTEXT_CK_SIZE_V(x) ((x) << KEY_CONTEXT_CK_SIZE_S) - -#define KEY_CONTEXT_MK_SIZE_S 2 -#define KEY_CONTEXT_MK_SIZE_V(x) ((x) << KEY_CONTEXT_MK_SIZE_S) - -#define KEY_CONTEXT_OPAD_PRESENT_S 11 -#define KEY_CONTEXT_OPAD_PRESENT_V(x) ((x) << KEY_CONTEXT_OPAD_PRESENT_S) -#define KEY_CONTEXT_OPAD_PRESENT_F KEY_CONTEXT_OPAD_PRESENT_V(1U) - -#define FILL_KEY_CTX_HDR(ck_size, mk_size, ctx_len) \ - htonl(KEY_CONTEXT_MK_SIZE_V(mk_size) | \ - KEY_CONTEXT_CK_SIZE_V(ck_size) | \ - KEY_CONTEXT_VALID_F | \ - KEY_CONTEXT_SALT_PRESENT_F | \ - KEY_CONTEXT_CTX_LEN_V((ctx_len))) - -struct uld_ctx *assign_chcr_device(void); - -static inline void *chcr_copy_to_txd(const void *src, const struct sge_txq *q, - void *pos, int length) -{ - int left = (void *)q->stat - pos; - u64 *p; - - if (likely(length <= left)) { - memcpy(pos, src, length); - pos += length; - } else { - memcpy(pos, src, left); - memcpy(q->desc, src + left, length - left); - pos = (void *)q->desc + (length - left); - } - /* 0-pad to multiple of 16 */ - p = PTR_ALIGN(pos, 8); - if ((uintptr_t)p & 8) { - *p = 0; - return p + 1; - } - return p; -} - -static inline unsigned int chcr_txq_avail(const struct sge_txq *q) -{ - return q->size - 1 - q->in_use; -} - -static inline void chcr_txq_advance(struct sge_txq *q, unsigned int n) -{ - q->in_use += n; - q->pidx += n; - if (q->pidx >= q->size) - q->pidx -= q->size; -} - -static inline void chcr_eth_txq_stop(struct sge_eth_txq *q) -{ - netif_tx_stop_queue(q->txq); - q->q.stops++; -} - -static inline unsigned int chcr_sgl_len(unsigned int n) -{ - n--; - return (3 * n) / 2 + (n & 1) + 2; -} - -static inline unsigned int chcr_flits_to_desc(unsigned int n) -{ - WARN_ON(n > SGE_MAX_WR_LEN / 8); - return DIV_ROUND_UP(n, 8); -} -#endif /* __CHCR_COMMON_H__ */ diff --git a/drivers/crypto/chelsio/chcr_core.c b/drivers/crypto/chelsio/chcr_core.c index bd8dac806e7a..f91f9d762a45 100644 --- a/drivers/crypto/chelsio/chcr_core.c +++ b/drivers/crypto/chelsio/chcr_core.c @@ -33,23 +33,8 @@ static int cpl_fw6_pld_handler(struct adapter *adap, unsigned char *input); static void *chcr_uld_add(const struct cxgb4_lld_info *lld); static int chcr_uld_state_change(void *handle, enum cxgb4_state state); -#if defined(CONFIG_CHELSIO_TLS_DEVICE) -static const struct tlsdev_ops chcr_ktls_ops = { - .tls_dev_add = chcr_ktls_dev_add, - .tls_dev_del = chcr_ktls_dev_del, -}; -#endif - -#ifdef CONFIG_CHELSIO_IPSEC_INLINE -static void update_netdev_features(void); -#endif /* CONFIG_CHELSIO_IPSEC_INLINE */ - static chcr_handler_func work_handlers[NUM_CPL_CMDS] = { [CPL_FW6_PLD] = cpl_fw6_pld_handler, -#ifdef CONFIG_CHELSIO_TLS_DEVICE - [CPL_ACT_OPEN_RPL] = chcr_ktls_cpl_act_open_rpl, - [CPL_SET_TCB_RPL] = chcr_ktls_cpl_set_tcb_rpl, -#endif }; static struct cxgb4_uld_info chcr_uld_info = { @@ -60,12 +45,6 @@ static struct cxgb4_uld_info chcr_uld_info = { .add = chcr_uld_add, .state_change = chcr_uld_state_change, .rx_handler = chcr_uld_rx_handler, -#if defined(CONFIG_CHELSIO_IPSEC_INLINE) || defined(CONFIG_CHELSIO_TLS_DEVICE) - .tx_handler = chcr_uld_tx_handler, -#endif /* CONFIG_CHELSIO_IPSEC_INLINE || CONFIG_CHELSIO_TLS_DEVICE */ -#if defined(CONFIG_CHELSIO_TLS_DEVICE) - .tlsdev_ops = &chcr_ktls_ops, -#endif }; static void detach_work_fn(struct work_struct *work) @@ -148,7 +127,7 @@ static void chcr_dev_init(struct uld_ctx *u_ctx) static int chcr_dev_move(struct uld_ctx *u_ctx) { - mutex_lock(&drv_data.drv_mutex); + mutex_lock(&drv_data.drv_mutex); if (drv_data.last_dev == u_ctx) { if (list_is_last(&drv_data.last_dev->entry, &drv_data.act_dev)) drv_data.last_dev = list_first_entry(&drv_data.act_dev, @@ -241,23 +220,6 @@ int chcr_uld_rx_handler(void *handle, const __be64 *rsp, return 0; } -#if defined(CONFIG_CHELSIO_IPSEC_INLINE) || defined(CONFIG_CHELSIO_TLS_DEVICE) -int chcr_uld_tx_handler(struct sk_buff *skb, struct net_device *dev) -{ - /* In case if skb's decrypted bit is set, it's nic tls packet, else it's - * ipsec packet. - */ -#ifdef CONFIG_CHELSIO_TLS_DEVICE - if (skb->decrypted) - return chcr_ktls_xmit(skb, dev); -#endif -#ifdef CONFIG_CHELSIO_IPSEC_INLINE - return chcr_ipsec_xmit(skb, dev); -#endif - return 0; -} -#endif /* CONFIG_CHELSIO_IPSEC_INLINE || CONFIG_CHELSIO_TLS_DEVICE */ - static void chcr_detach_device(struct uld_ctx *u_ctx) { struct chcr_dev *dev = &u_ctx->dev; @@ -305,24 +267,6 @@ static int chcr_uld_state_change(void *handle, enum cxgb4_state state) return ret; } -#ifdef CONFIG_CHELSIO_IPSEC_INLINE -static void update_netdev_features(void) -{ - struct uld_ctx *u_ctx, *tmp; - - mutex_lock(&drv_data.drv_mutex); - list_for_each_entry_safe(u_ctx, tmp, &drv_data.inact_dev, entry) { - if (u_ctx->lldi.crypto & ULP_CRYPTO_IPSEC_INLINE) - chcr_add_xfrmops(&u_ctx->lldi); - } - list_for_each_entry_safe(u_ctx, tmp, &drv_data.act_dev, entry) { - if (u_ctx->lldi.crypto & ULP_CRYPTO_IPSEC_INLINE) - chcr_add_xfrmops(&u_ctx->lldi); - } - mutex_unlock(&drv_data.drv_mutex); -} -#endif /* CONFIG_CHELSIO_IPSEC_INLINE */ - static int __init chcr_crypto_init(void) { INIT_LIST_HEAD(&drv_data.act_dev); @@ -332,12 +276,6 @@ static int __init chcr_crypto_init(void) drv_data.last_dev = NULL; cxgb4_register_uld(CXGB4_ULD_CRYPTO, &chcr_uld_info); - #ifdef CONFIG_CHELSIO_IPSEC_INLINE - rtnl_lock(); - update_netdev_features(); - rtnl_unlock(); - #endif /* CONFIG_CHELSIO_IPSEC_INLINE */ - return 0; } diff --git a/drivers/crypto/chelsio/chcr_core.h b/drivers/crypto/chelsio/chcr_core.h index 67d77abd6775..b02f981e7c32 100644 --- a/drivers/crypto/chelsio/chcr_core.h +++ b/drivers/crypto/chelsio/chcr_core.h @@ -72,54 +72,6 @@ struct _key_ctx { unsigned char key[]; }; -#define KEYCTX_TX_WR_IV_S 55 -#define KEYCTX_TX_WR_IV_M 0x1ffULL -#define KEYCTX_TX_WR_IV_V(x) ((x) << KEYCTX_TX_WR_IV_S) -#define KEYCTX_TX_WR_IV_G(x) \ - (((x) >> KEYCTX_TX_WR_IV_S) & KEYCTX_TX_WR_IV_M) - -#define KEYCTX_TX_WR_AAD_S 47 -#define KEYCTX_TX_WR_AAD_M 0xffULL -#define KEYCTX_TX_WR_AAD_V(x) ((x) << KEYCTX_TX_WR_AAD_S) -#define KEYCTX_TX_WR_AAD_G(x) (((x) >> KEYCTX_TX_WR_AAD_S) & \ - KEYCTX_TX_WR_AAD_M) - -#define KEYCTX_TX_WR_AADST_S 39 -#define KEYCTX_TX_WR_AADST_M 0xffULL -#define KEYCTX_TX_WR_AADST_V(x) ((x) << KEYCTX_TX_WR_AADST_S) -#define KEYCTX_TX_WR_AADST_G(x) \ - (((x) >> KEYCTX_TX_WR_AADST_S) & KEYCTX_TX_WR_AADST_M) - -#define KEYCTX_TX_WR_CIPHER_S 30 -#define KEYCTX_TX_WR_CIPHER_M 0x1ffULL -#define KEYCTX_TX_WR_CIPHER_V(x) ((x) << KEYCTX_TX_WR_CIPHER_S) -#define KEYCTX_TX_WR_CIPHER_G(x) \ - (((x) >> KEYCTX_TX_WR_CIPHER_S) & KEYCTX_TX_WR_CIPHER_M) - -#define KEYCTX_TX_WR_CIPHERST_S 23 -#define KEYCTX_TX_WR_CIPHERST_M 0x7f -#define KEYCTX_TX_WR_CIPHERST_V(x) ((x) << KEYCTX_TX_WR_CIPHERST_S) -#define KEYCTX_TX_WR_CIPHERST_G(x) \ - (((x) >> KEYCTX_TX_WR_CIPHERST_S) & KEYCTX_TX_WR_CIPHERST_M) - -#define KEYCTX_TX_WR_AUTH_S 14 -#define KEYCTX_TX_WR_AUTH_M 0x1ff -#define KEYCTX_TX_WR_AUTH_V(x) ((x) << KEYCTX_TX_WR_AUTH_S) -#define KEYCTX_TX_WR_AUTH_G(x) \ - (((x) >> KEYCTX_TX_WR_AUTH_S) & KEYCTX_TX_WR_AUTH_M) - -#define KEYCTX_TX_WR_AUTHST_S 7 -#define KEYCTX_TX_WR_AUTHST_M 0x7f -#define KEYCTX_TX_WR_AUTHST_V(x) ((x) << KEYCTX_TX_WR_AUTHST_S) -#define KEYCTX_TX_WR_AUTHST_G(x) \ - (((x) >> KEYCTX_TX_WR_AUTHST_S) & KEYCTX_TX_WR_AUTHST_M) - -#define KEYCTX_TX_WR_AUTHIN_S 0 -#define KEYCTX_TX_WR_AUTHIN_M 0x7f -#define KEYCTX_TX_WR_AUTHIN_V(x) ((x) << KEYCTX_TX_WR_AUTHIN_S) -#define KEYCTX_TX_WR_AUTHIN_G(x) \ - (((x) >> KEYCTX_TX_WR_AUTHIN_S) & KEYCTX_TX_WR_AUTHIN_M) - #define WQ_RETRY 5 struct chcr_driver_data { struct list_head act_dev; @@ -157,42 +109,6 @@ struct uld_ctx { struct chcr_dev dev; }; -struct sge_opaque_hdr { - void *dev; - dma_addr_t addr[MAX_SKB_FRAGS + 1]; -}; - -struct chcr_ipsec_req { - struct ulp_txpkt ulptx; - struct ulptx_idata sc_imm; - struct cpl_tx_sec_pdu sec_cpl; - struct _key_ctx key_ctx; -}; - -struct chcr_ipsec_wr { - struct fw_ulptx_wr wreq; - struct chcr_ipsec_req req; -}; - -#define ESN_IV_INSERT_OFFSET 12 -struct chcr_ipsec_aadiv { - __be32 spi; - u8 seq_no[8]; - u8 iv[8]; -}; - -struct ipsec_sa_entry { - int hmac_ctrl; - u16 esn; - u16 resv; - unsigned int enckey_len; - unsigned int kctx_len; - unsigned int authsize; - __be32 key_ctx_hdr; - char salt[MAX_SALT]; - char key[2 * AES_MAX_KEY_SIZE]; -}; - /* * sgl_len - calculates the size of an SGL of the given capacity * @n: the number of SGL entries @@ -221,18 +137,4 @@ int chcr_uld_rx_handler(void *handle, const __be64 *rsp, int chcr_uld_tx_handler(struct sk_buff *skb, struct net_device *dev); int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input, int err); -int chcr_ipsec_xmit(struct sk_buff *skb, struct net_device *dev); -void chcr_add_xfrmops(const struct cxgb4_lld_info *lld); -#ifdef CONFIG_CHELSIO_TLS_DEVICE -int chcr_ktls_cpl_act_open_rpl(struct adapter *adap, unsigned char *input); -int chcr_ktls_cpl_set_tcb_rpl(struct adapter *adap, unsigned char *input); -int chcr_ktls_xmit(struct sk_buff *skb, struct net_device *dev); -extern int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk, - enum tls_offload_ctx_dir direction, - struct tls_crypto_info *crypto_info, - u32 start_offload_tcp_sn); -extern void chcr_ktls_dev_del(struct net_device *netdev, - struct tls_context *tls_ctx, - enum tls_offload_ctx_dir direction); -#endif #endif /* __CHCR_CORE_H__ */ diff --git a/drivers/crypto/chelsio/chcr_ipsec.c b/drivers/crypto/chelsio/chcr_ipsec.c deleted file mode 100644 index 967babd67a51..000000000000 --- a/drivers/crypto/chelsio/chcr_ipsec.c +++ /dev/null @@ -1,754 +0,0 @@ -/* - * This file is part of the Chelsio T6 Crypto driver for Linux. - * - * Copyright (c) 2003-2017 Chelsio Communications, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - * Written and Maintained by: - * Atul Gupta ([email protected]) - */ - -#define pr_fmt(fmt) "chcr:" fmt - -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/crypto.h> -#include <linux/skbuff.h> -#include <linux/rtnetlink.h> -#include <linux/highmem.h> -#include <linux/if_vlan.h> -#include <linux/ip.h> -#include <linux/netdevice.h> -#include <net/esp.h> -#include <net/xfrm.h> -#include <crypto/aes.h> -#include <crypto/algapi.h> -#include <crypto/hash.h> -#include <crypto/sha.h> -#include <crypto/authenc.h> -#include <crypto/internal/aead.h> -#include <crypto/null.h> -#include <crypto/internal/skcipher.h> -#include <crypto/aead.h> -#include <crypto/scatterwalk.h> -#include <crypto/internal/hash.h> - -#include "chcr_core.h" -#include "chcr_algo.h" -#include "chcr_crypto.h" - -/* - * Max Tx descriptor space we allow for an Ethernet packet to be inlined - * into a WR. - */ -#define MAX_IMM_TX_PKT_LEN 256 -#define GCM_ESP_IV_SIZE 8 - -static int chcr_xfrm_add_state(struct xfrm_state *x); -static void chcr_xfrm_del_state(struct xfrm_state *x); -static void chcr_xfrm_free_state(struct xfrm_state *x); -static bool chcr_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x); -static void chcr_advance_esn_state(struct xfrm_state *x); - -static const struct xfrmdev_ops chcr_xfrmdev_ops = { - .xdo_dev_state_add = chcr_xfrm_add_state, - .xdo_dev_state_delete = chcr_xfrm_del_state, - .xdo_dev_state_free = chcr_xfrm_free_state, - .xdo_dev_offload_ok = chcr_ipsec_offload_ok, - .xdo_dev_state_advance_esn = chcr_advance_esn_state, -}; - -/* Add offload xfrms to Chelsio Interface */ -void chcr_add_xfrmops(const struct cxgb4_lld_info *lld) -{ - struct net_device *netdev = NULL; - int i; - - for (i = 0; i < lld->nports; i++) { - netdev = lld->ports[i]; - if (!netdev) - continue; - netdev->xfrmdev_ops = &chcr_xfrmdev_ops; - netdev->hw_enc_features |= NETIF_F_HW_ESP; - netdev->features |= NETIF_F_HW_ESP; - netdev_change_features(netdev); - } -} - -static inline int chcr_ipsec_setauthsize(struct xfrm_state *x, - struct ipsec_sa_entry *sa_entry) -{ - int hmac_ctrl; - int authsize = x->aead->alg_icv_len / 8; - - sa_entry->authsize = authsize; - - switch (authsize) { - case ICV_8: - hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; - break; - case ICV_12: - hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; - break; - case ICV_16: - hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; - break; - default: - return -EINVAL; - } - return hmac_ctrl; -} - -static inline int chcr_ipsec_setkey(struct xfrm_state *x, - struct ipsec_sa_entry *sa_entry) -{ - int keylen = (x->aead->alg_key_len + 7) / 8; - unsigned char *key = x->aead->alg_key; - int ck_size, key_ctx_size = 0; - unsigned char ghash_h[AEAD_H_SIZE]; - struct crypto_aes_ctx aes; - int ret = 0; - - if (keylen > 3) { - keylen -= 4; /* nonce/salt is present in the last 4 bytes */ - memcpy(sa_entry->salt, key + keylen, 4); - } - - if (keylen == AES_KEYSIZE_128) { - ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; - } else if (keylen == AES_KEYSIZE_192) { - ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; - } else if (keylen == AES_KEYSIZE_256) { - ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; - } else { - pr_err("GCM: Invalid key length %d\n", keylen); - ret = -EINVAL; - goto out; - } - - memcpy(sa_entry->key, key, keylen); - sa_entry->enckey_len = keylen; - key_ctx_size = sizeof(struct _key_ctx) + - ((DIV_ROUND_UP(keylen, 16)) << 4) + - AEAD_H_SIZE; - - sa_entry->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, - CHCR_KEYCTX_MAC_KEY_SIZE_128, - 0, 0, - key_ctx_size >> 4); - - /* Calculate the H = CIPH(K, 0 repeated 16 times). - * It will go in key context - */ - ret = aes_expandkey(&aes, key, keylen); - if (ret) { - sa_entry->enckey_len = 0; - goto out; - } - memset(ghash_h, 0, AEAD_H_SIZE); - aes_encrypt(&aes, ghash_h, ghash_h); - memzero_explicit(&aes, sizeof(aes)); - - memcpy(sa_entry->key + (DIV_ROUND_UP(sa_entry->enckey_len, 16) * - 16), ghash_h, AEAD_H_SIZE); - sa_entry->kctx_len = ((DIV_ROUND_UP(sa_entry->enckey_len, 16)) << 4) + - AEAD_H_SIZE; -out: - return ret; -} - -/* - * chcr_xfrm_add_state - * returns 0 on success, negative error if failed to send message to FPGA - * positive error if FPGA returned a bad response - */ -static int chcr_xfrm_add_state(struct xfrm_state *x) -{ - struct ipsec_sa_entry *sa_entry; - int res = 0; - - if (x->props.aalgo != SADB_AALG_NONE) { - pr_debug("CHCR: Cannot offload authenticated xfrm states\n"); - return -EINVAL; - } - if (x->props.calgo != SADB_X_CALG_NONE) { - pr_debug("CHCR: Cannot offload compressed xfrm states\n"); - return -EINVAL; - } - if (x->props.family != AF_INET && - x->props.family != AF_INET6) { - pr_debug("CHCR: Only IPv4/6 xfrm state offloaded\n"); - return -EINVAL; - } - if (x->props.mode != XFRM_MODE_TRANSPORT && - x->props.mode != XFRM_MODE_TUNNEL) { - pr_debug("CHCR: Only transport and tunnel xfrm offload\n"); - return -EINVAL; - } - if (x->id.proto != IPPROTO_ESP) { - pr_debug("CHCR: Only ESP xfrm state offloaded\n"); - return -EINVAL; - } - if (x->encap) { - pr_debug("CHCR: Encapsulated xfrm state not offloaded\n"); - return -EINVAL; - } - if (!x->aead) { - pr_debug("CHCR: Cannot offload xfrm states without aead\n"); - return -EINVAL; - } - if (x->aead->alg_icv_len != 128 && - x->aead->alg_icv_len != 96) { - pr_debug("CHCR: Cannot offload xfrm states with AEAD ICV length other than 96b & 128b\n"); - return -EINVAL; - } - if ((x->aead->alg_key_len != 128 + 32) && - (x->aead->alg_key_len != 256 + 32)) { - pr_debug("CHCR: Cannot offload xfrm states with AEAD key length other than 128/256 bit\n"); - return -EINVAL; - } - if (x->tfcpad) { - pr_debug("CHCR: Cannot offload xfrm states with tfc padding\n"); - return -EINVAL; - } - if (!x->geniv) { - pr_debug("CHCR: Cannot offload xfrm states without geniv\n"); - return -EINVAL; - } - if (strcmp(x->geniv, "seqiv")) { - pr_debug("CHCR: Cannot offload xfrm states with geniv other than seqiv\n"); - return -EINVAL; - } - - sa_entry = kzalloc(sizeof(*sa_entry), GFP_KERNEL); - if (!sa_entry) { - res = -ENOMEM; - goto out; - } - - sa_entry->hmac_ctrl = chcr_ipsec_setauthsize(x, sa_entry); - if (x->props.flags & XFRM_STATE_ESN) - sa_entry->esn = 1; - chcr_ipsec_setkey(x, sa_entry); - x->xso.offload_handle = (unsigned long)sa_entry; - try_module_get(THIS_MODULE); -out: - return res; -} - -static void chcr_xfrm_del_state(struct xfrm_state *x) -{ - /* do nothing */ - if (!x->xso.offload_handle) - return; -} - -static void chcr_xfrm_free_state(struct xfrm_state *x) -{ - struct ipsec_sa_entry *sa_entry; - - if (!x->xso.offload_handle) - return; - - sa_entry = (struct ipsec_sa_entry *)x->xso.offload_handle; - kfree(sa_entry); - module_put(THIS_MODULE); -} - -static bool chcr_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x) -{ - if (x->props.family == AF_INET) { - /* Offload with IP options is not supported yet */ - if (ip_hdr(skb)->ihl > 5) - return false; - } else { - /* Offload with IPv6 extension headers is not support yet */ - if (ipv6_ext_hdr(ipv6_hdr(skb)->nexthdr)) - return false; - } - return true; -} - -static void chcr_advance_esn_state(struct xfrm_state *x) -{ - /* do nothing */ - if (!x->xso.offload_handle) - return; -} - -static inline int is_eth_imm(const struct sk_buff *skb, - struct ipsec_sa_entry *sa_entry) -{ - unsigned int kctx_len; - int hdrlen; - - kctx_len = sa_entry->kctx_len; - hdrlen = sizeof(struct fw_ulptx_wr) + - sizeof(struct chcr_ipsec_req) + kctx_len; - - hdrlen += sizeof(struct cpl_tx_pkt); - if (sa_entry->esn) - hdrlen += (DIV_ROUND_UP(sizeof(struct chcr_ipsec_aadiv), 16) - << 4); - if (skb->len <= MAX_IMM_TX_PKT_LEN - hdrlen) - return hdrlen; - return 0; -} - -static inline unsigned int calc_tx_sec_flits(const struct sk_buff *skb, - struct ipsec_sa_entry *sa_entry, - bool *immediate) -{ - unsigned int kctx_len; - unsigned int flits; - int aadivlen; - int hdrlen; - - kctx_len = sa_entry->kctx_len; - hdrlen = is_eth_imm(skb, sa_entry); - aadivlen = sa_entry->esn ? DIV_ROUND_UP(sizeof(struct chcr_ipsec_aadiv), - 16) : 0; - aadivlen <<= 4; - - /* If the skb is small enough, we can pump it out as a work request - * with only immediate data. In that case we just have to have the - * TX Packet header plus the skb data in the Work Request. - */ - - if (hdrlen) { - *immediate = true; - return DIV_ROUND_UP(skb->len + hdrlen, sizeof(__be64)); - } - - flits = sgl_len(skb_shinfo(skb)->nr_frags + 1); - - /* Otherwise, we're going to have to construct a Scatter gather list - * of the skb body and fragments. We also include the flits necessary - * for the TX Packet Work Request and CPL. We always have a firmware - * Write Header (incorporated as part of the cpl_tx_pkt_lso and - * cpl_tx_pkt structures), followed by either a TX Packet Write CPL - * message or, if we're doing a Large Send Offload, an LSO CPL message - * with an embedded TX Packet Write CPL message. - */ - flits += (sizeof(struct fw_ulptx_wr) + - sizeof(struct chcr_ipsec_req) + - kctx_len + - sizeof(struct cpl_tx_pkt_core) + - aadivlen) / sizeof(__be64); - return flits; -} - -inline void *copy_esn_pktxt(struct sk_buff *skb, - struct net_device *dev, - void *pos, - struct ipsec_sa_entry *sa_entry) -{ - struct chcr_ipsec_aadiv *aadiv; - struct ulptx_idata *sc_imm; - struct ip_esp_hdr *esphdr; - struct xfrm_offload *xo; - struct sge_eth_txq *q; - struct adapter *adap; - struct port_info *pi; - __be64 seqno; - u32 qidx; - u32 seqlo; - u8 *iv; - int eoq; - int len; - - pi = netdev_priv(dev); - adap = pi->adapter; - qidx = skb->queue_mapping; - q = &adap->sge.ethtxq[qidx + pi->first_qset]; - - /* end of queue, reset pos to start of queue */ - eoq = (void *)q->q.stat - pos; - if (!eoq) - pos = q->q.desc; - - len = DIV_ROUND_UP(sizeof(struct chcr_ipsec_aadiv), 16) << 4; - memset(pos, 0, len); - aadiv = (struct chcr_ipsec_aadiv *)pos; - esphdr = (struct ip_esp_hdr *)skb_transport_header(skb); - iv = skb_transport_header(skb) + sizeof(struct ip_esp_hdr); - xo = xfrm_offload(skb); - - aadiv->spi = (esphdr->spi); - seqlo = ntohl(esphdr->seq_no); - seqno = cpu_to_be64(seqlo + ((u64)xo->seq.hi << 32)); - memcpy(aadiv->seq_no, &seqno, 8); - iv = skb_transport_header(skb) + sizeof(struct ip_esp_hdr); - memcpy(aadiv->iv, iv, 8); - - if (is_eth_imm(skb, sa_entry) && !skb_is_nonlinear(skb)) { - sc_imm = (struct ulptx_idata *)(pos + - (DIV_ROUND_UP(sizeof(struct chcr_ipsec_aadiv), - sizeof(__be64)) << 3)); - sc_imm->cmd_more = FILL_CMD_MORE(0); - sc_imm->len = cpu_to_be32(skb->len); - } - pos += len; - return pos; -} - -inline void *copy_cpltx_pktxt(struct sk_buff *skb, - struct net_device *dev, - void *pos, - struct ipsec_sa_entry *sa_entry) -{ - struct cpl_tx_pkt_core *cpl; - struct sge_eth_txq *q; - struct adapter *adap; - struct port_info *pi; - u32 ctrl0, qidx; - u64 cntrl = 0; - int left; - - pi = netdev_priv(dev); - adap = pi->adapter; - qidx = skb->queue_mapping; - q = &adap->sge.ethtxq[qidx + pi->first_qset]; - - left = (void *)q->q.stat - pos; - if (!left) - pos = q->q.desc; - - cpl = (struct cpl_tx_pkt_core *)pos; - - cntrl = TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F; - ctrl0 = TXPKT_OPCODE_V(CPL_TX_PKT_XT) | TXPKT_INTF_V(pi->tx_chan) | - TXPKT_PF_V(adap->pf); - if (skb_vlan_tag_present(skb)) { - q->vlan_ins++; - cntrl |= TXPKT_VLAN_VLD_F | TXPKT_VLAN_V(skb_vlan_tag_get(skb)); - } - - cpl->ctrl0 = htonl(ctrl0); - cpl->pack = htons(0); - cpl->len = htons(skb->len); - cpl->ctrl1 = cpu_to_be64(cntrl); - - pos += sizeof(struct cpl_tx_pkt_core); - /* Copy ESN info for HW */ - if (sa_entry->esn) - pos = copy_esn_pktxt(skb, dev, pos, sa_entry); - return pos; -} - -inline void *copy_key_cpltx_pktxt(struct sk_buff *skb, - struct net_device *dev, - void *pos, - struct ipsec_sa_entry *sa_entry) -{ - struct _key_ctx *key_ctx; - int left, eoq, key_len; - struct sge_eth_txq *q; - struct adapter *adap; - struct port_info *pi; - unsigned int qidx; - - pi = netdev_priv(dev); - adap = pi->adapter; - qidx = skb->queue_mapping; - q = &adap->sge.ethtxq[qidx + pi->first_qset]; - key_len = sa_entry->kctx_len; - - /* end of queue, reset pos to start of queue */ - eoq = (void *)q->q.stat - pos; - left = eoq; - if (!eoq) { - pos = q->q.desc; - left = 64 * q->q.size; - } - - /* Copy the Key context header */ - key_ctx = (struct _key_ctx *)pos; - key_ctx->ctx_hdr = sa_entry->key_ctx_hdr; - memcpy(key_ctx->salt, sa_entry->salt, MAX_SALT); - pos += sizeof(struct _key_ctx); - left -= sizeof(struct _key_ctx); - - if (likely(key_len <= left)) { - memcpy(key_ctx->key, sa_entry->key, key_len); - pos += key_len; - } else { - memcpy(pos, sa_entry->key, left); - memcpy(q->q.desc, sa_entry->key + left, - key_len - left); - pos = (u8 *)q->q.desc + (key_len - left); - } - /* Copy CPL TX PKT XT */ - pos = copy_cpltx_pktxt(skb, dev, pos, sa_entry); - - return pos; -} - -inline void *chcr_crypto_wreq(struct sk_buff *skb, - struct net_device *dev, - void *pos, - int credits, - struct ipsec_sa_entry *sa_entry) -{ - struct port_info *pi = netdev_priv(dev); - struct adapter *adap = pi->adapter; - unsigned int ivsize = GCM_ESP_IV_SIZE; - struct chcr_ipsec_wr *wr; - bool immediate = false; - u16 immdatalen = 0; - unsigned int flits; - u32 ivinoffset; - u32 aadstart; - u32 aadstop; - u32 ciphstart; - u16 sc_more = 0; - u32 ivdrop = 0; - u32 esnlen = 0; - u32 wr_mid; - u16 ndesc; - int qidx = skb_get_queue_mapping(skb); - struct sge_eth_txq *q = &adap->sge.ethtxq[qidx + pi->first_qset]; - unsigned int kctx_len = sa_entry->kctx_len; - int qid = q->q.cntxt_id; - - atomic_inc(&adap->chcr_stats.ipsec_cnt); - - flits = calc_tx_sec_flits(skb, sa_entry, &immediate); - ndesc = DIV_ROUND_UP(flits, 2); - if (sa_entry->esn) - ivdrop = 1; - - if (immediate) - immdatalen = skb->len; - - if (sa_entry->esn) { - esnlen = sizeof(struct chcr_ipsec_aadiv); - if (!skb_is_nonlinear(skb)) - sc_more = 1; - } - - /* WR Header */ - wr = (struct chcr_ipsec_wr *)pos; - wr->wreq.op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR)); - wr_mid = FW_CRYPTO_LOOKASIDE_WR_LEN16_V(ndesc); - - if (unlikely(credits < ETHTXQ_STOP_THRES)) { - netif_tx_stop_queue(q->txq); - q->q.stops++; - if (!q->dbqt) - wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F; - } - wr_mid |= FW_ULPTX_WR_DATA_F; - wr->wreq.flowid_len16 = htonl(wr_mid); - - /* ULPTX */ - wr->req.ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(pi->port_id, qid); - wr->req.ulptx.len = htonl(ndesc - 1); - - /* Sub-command */ - wr->req.sc_imm.cmd_more = FILL_CMD_MORE(!immdatalen || sc_more); - wr->req.sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) + - sizeof(wr->req.key_ctx) + - kctx_len + - sizeof(struct cpl_tx_pkt_core) + - esnlen + - (esnlen ? 0 : immdatalen)); - - /* CPL_SEC_PDU */ - ivinoffset = sa_entry->esn ? (ESN_IV_INSERT_OFFSET + 1) : - (skb_transport_offset(skb) + - sizeof(struct ip_esp_hdr) + 1); - wr->req.sec_cpl.op_ivinsrtofst = htonl( - CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) | - CPL_TX_SEC_PDU_CPLLEN_V(2) | - CPL_TX_SEC_PDU_PLACEHOLDER_V(1) | - CPL_TX_SEC_PDU_IVINSRTOFST_V( - ivinoffset)); - - wr->req.sec_cpl.pldlen = htonl(skb->len + esnlen); - aadstart = sa_entry->esn ? 1 : (skb_transport_offset(skb) + 1); - aadstop = sa_entry->esn ? ESN_IV_INSERT_OFFSET : - (skb_transport_offset(skb) + - sizeof(struct ip_esp_hdr)); - ciphstart = skb_transport_offset(skb) + sizeof(struct ip_esp_hdr) + - GCM_ESP_IV_SIZE + 1; - ciphstart += sa_entry->esn ? esnlen : 0; - - wr->req.sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( - aadstart, - aadstop, - ciphstart, 0); - - wr->req.sec_cpl.cipherstop_lo_authinsert = - FILL_SEC_CPL_AUTHINSERT(0, ciphstart, - sa_entry->authsize, - sa_entry->authsize); - wr->req.sec_cpl.seqno_numivs = - FILL_SEC_CPL_SCMD0_SEQNO(CHCR_ENCRYPT_OP, 1, - CHCR_SCMD_CIPHER_MODE_AES_GCM, - CHCR_SCMD_AUTH_MODE_GHASH, - sa_entry->hmac_ctrl, - ivsize >> 1); - wr->req.sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, - 0, ivdrop, 0); - - pos += sizeof(struct fw_ulptx_wr) + - sizeof(struct ulp_txpkt) + - sizeof(struct ulptx_idata) + - sizeof(struct cpl_tx_sec_pdu); - - pos = copy_key_cpltx_pktxt(skb, dev, pos, sa_entry); - - return pos; -} - -/** - * flits_to_desc - returns the num of Tx descriptors for the given flits - * @n: the number of flits - * - * Returns the number of Tx descriptors needed for the supplied number - * of flits. - */ -static inline unsigned int flits_to_desc(unsigned int n) -{ - WARN_ON(n > SGE_MAX_WR_LEN / 8); - return DIV_ROUND_UP(n, 8); -} - -static inline unsigned int txq_avail(const struct sge_txq *q) -{ - return q->size - 1 - q->in_use; -} - -static void eth_txq_stop(struct sge_eth_txq *q) -{ - netif_tx_stop_queue(q->txq); - q->q.stops++; -} - -static inline void txq_advance(struct sge_txq *q, unsigned int n) -{ - q->in_use += n; - q->pidx += n; - if (q->pidx >= q->size) - q->pidx -= q->size; -} - -/* - * chcr_ipsec_xmit called from ULD Tx handler - */ -int chcr_ipsec_xmit(struct sk_buff *skb, struct net_device *dev) -{ - struct xfrm_state *x = xfrm_input_state(skb); - unsigned int last_desc, ndesc, flits = 0; - struct ipsec_sa_entry *sa_entry; - u64 *pos, *end, *before, *sgl; - struct tx_sw_desc *sgl_sdesc; - int qidx, left, credits; - bool immediate = false; - struct sge_eth_txq *q; - struct adapter *adap; - struct port_info *pi; - struct sec_path *sp; - - if (!x->xso.offload_handle) - return NETDEV_TX_BUSY; - - sa_entry = (struct ipsec_sa_entry *)x->xso.offload_handle; - - sp = skb_sec_path(skb); - if (sp->len != 1) { -out_free: dev_kfree_skb_any(skb); - return NETDEV_TX_OK; - } - - pi = netdev_priv(dev); - adap = pi->adapter; - qidx = skb->queue_mapping; - q = &adap->sge.ethtxq[qidx + pi->first_qset]; - - cxgb4_reclaim_completed_tx(adap, &q->q, true); - - flits = calc_tx_sec_flits(skb, sa_entry, &immediate); - ndesc = flits_to_desc(flits); - credits = txq_avail(&q->q) - ndesc; - - if (unlikely(credits < 0)) { - eth_txq_stop(q); - dev_err(adap->pdev_dev, - "%s: Tx ring %u full while queue awake! cred:%d %d %d flits:%d\n", - dev->name, qidx, credits, ndesc, txq_avail(&q->q), - flits); - return NETDEV_TX_BUSY; - } - - last_desc = q->q.pidx + ndesc - 1; - if (last_desc >= q->q.size) - last_desc -= q->q.size; - sgl_sdesc = &q->q.sdesc[last_desc]; - - if (!immediate && - unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) { - memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr)); - q->mapping_err++; - goto out_free; - } - - pos = (u64 *)&q->q.desc[q->q.pidx]; - before = (u64 *)pos; - end = (u64 *)pos + flits; - /* Setup IPSec CPL */ - pos = (void *)chcr_crypto_wreq(skb, dev, (void *)pos, - credits, sa_entry); - if (before > (u64 *)pos) { - left = (u8 *)end - (u8 *)q->q.stat; - end = (void *)q->q.desc + left; - } - if (pos == (u64 *)q->q.stat) { - left = (u8 *)end - (u8 *)q->q.stat; - end = (void *)q->q.desc + left; - pos = (void *)q->q.desc; - } - - sgl = (void *)pos; - if (immediate) { - cxgb4_inline_tx_skb(skb, &q->q, sgl); - dev_consume_skb_any(skb); - } else { - cxgb4_write_sgl(skb, &q->q, (void *)sgl, end, - 0, sgl_sdesc->addr); - skb_orphan(skb); - sgl_sdesc->skb = skb; - } - txq_advance(&q->q, ndesc); - - cxgb4_ring_tx_db(adap, &q->q, ndesc); - return NETDEV_TX_OK; -} diff --git a/drivers/crypto/chelsio/chcr_ktls.c b/drivers/crypto/chelsio/chcr_ktls.c deleted file mode 100644 index c5cce024886a..000000000000 --- a/drivers/crypto/chelsio/chcr_ktls.c +++ /dev/null @@ -1,2017 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* Copyright (C) 2020 Chelsio Communications. All rights reserved. */ - -#ifdef CONFIG_CHELSIO_TLS_DEVICE -#include <linux/highmem.h> -#include "chcr_ktls.h" -#include "clip_tbl.h" - -static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info); -/* - * chcr_ktls_save_keys: calculate and save crypto keys. - * @tx_info - driver specific tls info. - * @crypto_info - tls crypto information. - * @direction - TX/RX direction. - * return - SUCCESS/FAILURE. - */ -static int chcr_ktls_save_keys(struct chcr_ktls_info *tx_info, - struct tls_crypto_info *crypto_info, - enum tls_offload_ctx_dir direction) -{ - int ck_size, key_ctx_size, mac_key_size, keylen, ghash_size, ret; - unsigned char ghash_h[TLS_CIPHER_AES_GCM_256_TAG_SIZE]; - struct tls12_crypto_info_aes_gcm_128 *info_128_gcm; - struct ktls_key_ctx *kctx = &tx_info->key_ctx; - struct crypto_cipher *cipher; - unsigned char *key, *salt; - - switch (crypto_info->cipher_type) { - case TLS_CIPHER_AES_GCM_128: - info_128_gcm = - (struct tls12_crypto_info_aes_gcm_128 *)crypto_info; - keylen = TLS_CIPHER_AES_GCM_128_KEY_SIZE; - ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; - tx_info->salt_size = TLS_CIPHER_AES_GCM_128_SALT_SIZE; - mac_key_size = CHCR_KEYCTX_MAC_KEY_SIZE_128; - tx_info->iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE; - tx_info->iv = be64_to_cpu(*(__be64 *)info_128_gcm->iv); - - ghash_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE; - key = info_128_gcm->key; - salt = info_128_gcm->salt; - tx_info->record_no = *(u64 *)info_128_gcm->rec_seq; - - /* The SCMD fields used when encrypting a full TLS - * record. Its a one time calculation till the - * connection exists. - */ - tx_info->scmd0_seqno_numivs = - SCMD_SEQ_NO_CTRL_V(CHCR_SCMD_SEQ_NO_CTRL_64BIT) | - SCMD_CIPH_AUTH_SEQ_CTRL_F | - SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_TLS) | - SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_GCM) | - SCMD_AUTH_MODE_V(CHCR_SCMD_AUTH_MODE_GHASH) | - SCMD_IV_SIZE_V(TLS_CIPHER_AES_GCM_128_IV_SIZE >> 1) | - SCMD_NUM_IVS_V(1); - - /* keys will be sent inline. */ - tx_info->scmd0_ivgen_hdrlen = SCMD_KEY_CTX_INLINE_F; - - /* The SCMD fields used when encrypting a partial TLS - * record (no trailer and possibly a truncated payload). - */ - tx_info->scmd0_short_seqno_numivs = - SCMD_CIPH_AUTH_SEQ_CTRL_F | - SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_GENERIC) | - SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_CTR) | - SCMD_IV_SIZE_V(AES_BLOCK_LEN >> 1); - - tx_info->scmd0_short_ivgen_hdrlen = - tx_info->scmd0_ivgen_hdrlen | SCMD_AADIVDROP_F; - - break; - - default: - pr_err("GCM: cipher type 0x%x not supported\n", - crypto_info->cipher_type); - ret = -EINVAL; - goto out; - } - - key_ctx_size = CHCR_KTLS_KEY_CTX_LEN + - roundup(keylen, 16) + ghash_size; - /* Calculate the H = CIPH(K, 0 repeated 16 times). - * It will go in key context - */ - cipher = crypto_alloc_cipher("aes", 0, 0); - if (IS_ERR(cipher)) { - ret = -ENOMEM; - goto out; - } - - ret = crypto_cipher_setkey(cipher, key, keylen); - if (ret) - goto out1; - - memset(ghash_h, 0, ghash_size); - crypto_cipher_encrypt_one(cipher, ghash_h, ghash_h); - - /* fill the Key context */ - if (direction == TLS_OFFLOAD_CTX_DIR_TX) { - kctx->ctx_hdr = FILL_KEY_CTX_HDR(ck_size, - mac_key_size, - key_ctx_size >> 4); - } else { - ret = -EINVAL; - goto out1; - } - - memcpy(kctx->salt, salt, tx_info->salt_size); - memcpy(kctx->key, key, keylen); - memcpy(kctx->key + keylen, ghash_h, ghash_size); - tx_info->key_ctx_len = key_ctx_size; - -out1: - crypto_free_cipher(cipher); -out: - return ret; -} - -static int chcr_ktls_update_connection_state(struct chcr_ktls_info *tx_info, - int new_state) -{ - /* This function can be called from both rx (interrupt context) and tx - * queue contexts. - */ - spin_lock_bh(&tx_info->lock); - switch (tx_info->connection_state) { - case KTLS_CONN_CLOSED: - tx_info->connection_state = new_state; - break; - - case KTLS_CONN_ACT_OPEN_REQ: - /* only go forward if state is greater than current state. */ - if (new_state <= tx_info->connection_state) - break; - /* update to the next state and also initialize TCB */ - tx_info->connection_state = new_state; - fallthrough; - case KTLS_CONN_ACT_OPEN_RPL: - /* if we are stuck in this state, means tcb init might not - * received by HW, try sending it again. - */ - if (!chcr_init_tcb_fields(tx_info)) - tx_info->connection_state = KTLS_CONN_SET_TCB_REQ; - break; - - case KTLS_CONN_SET_TCB_REQ: - /* only go forward if state is greater than current state. */ - if (new_state <= tx_info->connection_state) - break; - /* update to the next state and check if l2t_state is valid */ - tx_info->connection_state = new_state; - fallthrough; - case KTLS_CONN_SET_TCB_RPL: - /* Check if l2t state is valid, then move to ready state. */ - if (cxgb4_check_l2t_valid(tx_info->l2te)) { - tx_info->connection_state = KTLS_CONN_TX_READY; - atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_ctx); - } - break; - - case KTLS_CONN_TX_READY: - /* nothing to be done here */ - break; - - default: - pr_err("unknown KTLS connection state\n"); - break; - } - spin_unlock_bh(&tx_info->lock); - - return tx_info->connection_state; -} -/* - * chcr_ktls_act_open_req: creates TCB entry for ipv4 connection. - * @sk - tcp socket. - * @tx_info - driver specific tls info. - * @atid - connection active tid. - * return - send success/failure. - */ -static int chcr_ktls_act_open_req(struct sock *sk, - struct chcr_ktls_info *tx_info, - int atid) -{ - struct inet_sock *inet = inet_sk(sk); - struct cpl_t6_act_open_req *cpl6; - struct cpl_act_open_req *cpl; - struct sk_buff *skb; - unsigned int len; - int qid_atid; - u64 options; - - len = sizeof(*cpl6); - skb = alloc_skb(len, GFP_KERNEL); - if (unlikely(!skb)) - return -ENOMEM; - /* mark it a control pkt */ - set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id); - - cpl6 = __skb_put_zero(skb, len); - cpl = (struct cpl_act_open_req *)cpl6; - INIT_TP_WR(cpl6, 0); - qid_atid = TID_QID_V(tx_info->rx_qid) | - TID_TID_V(atid); - OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, qid_atid)); - cpl->local_port = inet->inet_sport; - cpl->peer_port = inet->inet_dport; - cpl->local_ip = inet->inet_rcv_saddr; - cpl->peer_ip = inet->inet_daddr; - - /* fill first 64 bit option field. */ - options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F | - SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan); - cpl->opt0 = cpu_to_be64(options); - - /* next 64 bit option field. */ - options = - TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]); - cpl->opt2 = htonl(options); - - return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te); -} - -#if IS_ENABLED(CONFIG_IPV6) -/* - * chcr_ktls_act_open_req6: creates TCB entry for ipv6 connection. - * @sk - tcp socket. - * @tx_info - driver specific tls info. - * @atid - connection active tid. - * return - send success/failure. - */ -static int chcr_ktls_act_open_req6(struct sock *sk, - struct chcr_ktls_info *tx_info, - int atid) -{ - struct inet_sock *inet = inet_sk(sk); - struct cpl_t6_act_open_req6 *cpl6; - struct cpl_act_open_req6 *cpl; - struct sk_buff *skb; - unsigned int len; - int qid_atid; - u64 options; - - len = sizeof(*cpl6); - skb = alloc_skb(len, GFP_KERNEL); - if (unlikely(!skb)) - return -ENOMEM; - /* mark it a control pkt */ - set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id); - - cpl6 = __skb_put_zero(skb, len); - cpl = (struct cpl_act_open_req6 *)cpl6; - INIT_TP_WR(cpl6, 0); - qid_atid = TID_QID_V(tx_info->rx_qid) | TID_TID_V(atid); - OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, qid_atid)); - cpl->local_port = inet->inet_sport; - cpl->peer_port = inet->inet_dport; - cpl->local_ip_hi = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[0]; - cpl->local_ip_lo = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[8]; - cpl->peer_ip_hi = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[0]; - cpl->peer_ip_lo = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[8]; - - /* first 64 bit option field. */ - options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F | - SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan); - cpl->opt0 = cpu_to_be64(options); - /* next 64 bit option field. */ - options = - TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]); - cpl->opt2 = htonl(options); - - return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te); -} -#endif /* #if IS_ENABLED(CONFIG_IPV6) */ - -/* - * chcr_setup_connection: create a TCB entry so that TP will form tcp packets. - * @sk - tcp socket. - * @tx_info - driver specific tls info. - * return: NET_TX_OK/NET_XMIT_DROP - */ -static int chcr_setup_connection(struct sock *sk, - struct chcr_ktls_info *tx_info) -{ - struct tid_info *t = &tx_info->adap->tids; - int atid, ret = 0; - - atid = cxgb4_alloc_atid(t, tx_info); - if (atid == -1) - return -EINVAL; - - tx_info->atid = atid; - tx_info->ip_family = sk->sk_family; - - if (sk->sk_family == AF_INET) { - tx_info->ip_family = AF_INET; - ret = chcr_ktls_act_open_req(sk, tx_info, atid); -#if IS_ENABLED(CONFIG_IPV6) - } else { - if (!sk->sk_ipv6only && - ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED) { - tx_info->ip_family = AF_INET; - ret = chcr_ktls_act_open_req(sk, tx_info, atid); - } else { - tx_info->ip_family = AF_INET6; - ret = cxgb4_clip_get(tx_info->netdev, - (const u32 *) - &sk->sk_v6_rcv_saddr.s6_addr, - 1); - if (ret) - goto out; - ret = chcr_ktls_act_open_req6(sk, tx_info, atid); - } -#endif - } - - /* if return type is NET_XMIT_CN, msg will be sent but delayed, mark ret - * success, if any other return type clear atid and return that failure. - */ - if (ret) { - if (ret == NET_XMIT_CN) - ret = 0; - else - cxgb4_free_atid(t, atid); - goto out; - } - - /* update the connection state */ - chcr_ktls_update_connection_state(tx_info, KTLS_CONN_ACT_OPEN_REQ); -out: - return ret; -} - -/* - * chcr_set_tcb_field: update tcb fields. - * @tx_info - driver specific tls info. - * @word - TCB word. - * @mask - TCB word related mask. - * @val - TCB word related value. - * @no_reply - set 1 if not looking for TP response. - */ -static int chcr_set_tcb_field(struct chcr_ktls_info *tx_info, u16 word, - u64 mask, u64 val, int no_reply) -{ - struct cpl_set_tcb_field *req; - struct sk_buff *skb; - - skb = alloc_skb(sizeof(struct cpl_set_tcb_field), GFP_ATOMIC); - if (!skb) - return -ENOMEM; - - req = (struct cpl_set_tcb_field *)__skb_put_zero(skb, sizeof(*req)); - INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, tx_info->tid); - req->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) | - NO_REPLY_V(no_reply)); - req->word_cookie = htons(TCB_WORD_V(word)); - req->mask = cpu_to_be64(mask); - req->val = cpu_to_be64(val); - - set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id); - return cxgb4_ofld_send(tx_info->netdev, skb); -} - -/* - * chcr_ktls_mark_tcb_close: mark tcb state to CLOSE - * @tx_info - driver specific tls info. - * return: NET_TX_OK/NET_XMIT_DROP. - */ -static int chcr_ktls_mark_tcb_close(struct chcr_ktls_info *tx_info) -{ - return chcr_set_tcb_field(tx_info, TCB_T_STATE_W, - TCB_T_STATE_V(TCB_T_STATE_M), - CHCR_TCB_STATE_CLOSED, 1); -} - -/* - * chcr_ktls_dev_del: call back for tls_dev_del. - * Remove the tid and l2t entry and close the connection. - * it per connection basis. - * @netdev - net device. - * @tls_cts - tls context. - * @direction - TX/RX crypto direction - */ -void chcr_ktls_dev_del(struct net_device *netdev, - struct tls_context *tls_ctx, - enum tls_offload_ctx_dir direction) -{ - struct chcr_ktls_ofld_ctx_tx *tx_ctx = - chcr_get_ktls_tx_context(tls_ctx); - struct chcr_ktls_info *tx_info = tx_ctx->chcr_info; - struct sock *sk; - - if (!tx_info) - return; - sk = tx_info->sk; - - spin_lock(&tx_info->lock); - tx_info->connection_state = KTLS_CONN_CLOSED; - spin_unlock(&tx_info->lock); - - /* clear l2t entry */ - if (tx_info->l2te) - cxgb4_l2t_release(tx_info->l2te); - -#if IS_ENABLED(CONFIG_IPV6) - /* clear clip entry */ - if (tx_info->ip_family == AF_INET6) - cxgb4_clip_release(netdev, - (const u32 *)&sk->sk_v6_daddr.in6_u.u6_addr8, - 1); -#endif - - /* clear tid */ - if (tx_info->tid != -1) { - /* clear tcb state and then release tid */ - chcr_ktls_mark_tcb_close(tx_info); - cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan, - tx_info->tid, tx_info->ip_family); - } - - atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_connection_close); - kvfree(tx_info); - tx_ctx->chcr_info = NULL; - /* release module refcount */ - module_put(THIS_MODULE); -} - -/* - * chcr_ktls_dev_add: call back for tls_dev_add. - * Create a tcb entry for TP. Also add l2t entry for the connection. And - * generate keys & save those keys locally. - * @netdev - net device. - * @tls_cts - tls context. - * @direction - TX/RX crypto direction - * return: SUCCESS/FAILURE. - */ -int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk, - enum tls_offload_ctx_dir direction, - struct tls_crypto_info *crypto_info, - u32 start_offload_tcp_sn) -{ - struct tls_context *tls_ctx = tls_get_ctx(sk); - struct chcr_ktls_ofld_ctx_tx *tx_ctx; - struct chcr_ktls_info *tx_info; - struct dst_entry *dst; - struct adapter *adap; - struct port_info *pi; - struct neighbour *n; - u8 daaddr[16]; - int ret = -1; - - tx_ctx = chcr_get_ktls_tx_context(tls_ctx); - - pi = netdev_priv(netdev); - adap = pi->adapter; - if (direction == TLS_OFFLOAD_CTX_DIR_RX) { - pr_err("not expecting for RX direction\n"); - ret = -EINVAL; - goto out; - } - if (tx_ctx->chcr_info) { - ret = -EINVAL; - goto out; - } - - tx_info = kvzalloc(sizeof(*tx_info), GFP_KERNEL); - if (!tx_info) { - ret = -ENOMEM; - goto out; - } - - spin_lock_init(&tx_info->lock); - - /* clear connection state */ - spin_lock(&tx_info->lock); - tx_info->connection_state = KTLS_CONN_CLOSED; - spin_unlock(&tx_info->lock); - - tx_info->sk = sk; - /* initialize tid and atid to -1, 0 is a also a valid id. */ - tx_info->tid = -1; - tx_info->atid = -1; - - tx_info->adap = adap; - tx_info->netdev = netdev; - tx_info->first_qset = pi->first_qset; - tx_info->tx_chan = pi->tx_chan; - tx_info->smt_idx = pi->smt_idx; - tx_info->port_id = pi->port_id; - - tx_info->rx_qid = chcr_get_first_rx_qid(adap); - if (unlikely(tx_info->rx_qid < 0)) - goto out2; - - tx_info->prev_seq = start_offload_tcp_sn; - tx_info->tcp_start_seq_number = start_offload_tcp_sn; - - /* save crypto keys */ - ret = chcr_ktls_save_keys(tx_info, crypto_info, direction); - if (ret < 0) - goto out2; - - /* get peer ip */ - if (sk->sk_family == AF_INET) { - memcpy(daaddr, &sk->sk_daddr, 4); -#if IS_ENABLED(CONFIG_IPV6) - } else { - if (!sk->sk_ipv6only && - ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED) - memcpy(daaddr, &sk->sk_daddr, 4); - else - memcpy(daaddr, sk->sk_v6_daddr.in6_u.u6_addr8, 16); -#endif - } - - /* get the l2t index */ - dst = sk_dst_get(sk); - if (!dst) { - pr_err("DST entry not found\n"); - goto out2; - } - n = dst_neigh_lookup(dst, daaddr); - if (!n || !n->dev) { - pr_err("neighbour not found\n"); - dst_release(dst); - goto out2; - } - tx_info->l2te = cxgb4_l2t_get(adap->l2t, n, n->dev, 0); - - neigh_release(n); - dst_release(dst); - - if (!tx_info->l2te) { - pr_err("l2t entry not found\n"); - goto out2; - } - - tx_ctx->chcr_info = tx_info; - - /* create a filter and call cxgb4_l2t_send to send the packet out, which - * will take care of updating l2t entry in hw if not already done. - */ - ret = chcr_setup_connection(sk, tx_info); - if (ret) - goto out2; - - /* Driver shouldn't be removed until any single connection exists */ - if (!try_module_get(THIS_MODULE)) { - ret = -EINVAL; - goto out2; - } - - atomic64_inc(&adap->chcr_stats.ktls_tx_connection_open); - return 0; -out2: - kvfree(tx_info); -out: - atomic64_inc(&adap->chcr_stats.ktls_tx_connection_fail); - return ret; -} - -/* - * chcr_init_tcb_fields: Initialize tcb fields to handle TCP seq number - * handling. - * @tx_info - driver specific tls info. - * return: NET_TX_OK/NET_XMIT_DROP - */ -static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info) -{ - int ret = 0; - - /* set tcb in offload and bypass */ - ret = - chcr_set_tcb_field(tx_info, TCB_T_FLAGS_W, - TCB_T_FLAGS_V(TF_CORE_BYPASS_F | TF_NON_OFFLOAD_F), - TCB_T_FLAGS_V(TF_CORE_BYPASS_F), 1); - if (ret) - return ret; - /* reset snd_una and snd_next fields in tcb */ - ret = chcr_set_tcb_field(tx_info, TCB_SND_UNA_RAW_W, - TCB_SND_NXT_RAW_V(TCB_SND_NXT_RAW_M) | - TCB_SND_UNA_RAW_V(TCB_SND_UNA_RAW_M), - 0, 1); - if (ret) - return ret; - - /* reset send max */ - ret = chcr_set_tcb_field(tx_info, TCB_SND_MAX_RAW_W, - TCB_SND_MAX_RAW_V(TCB_SND_MAX_RAW_M), - 0, 1); - if (ret) - return ret; - - /* update l2t index and request for tp reply to confirm tcb is - * initialised to handle tx traffic. - */ - ret = chcr_set_tcb_field(tx_info, TCB_L2T_IX_W, - TCB_L2T_IX_V(TCB_L2T_IX_M), - TCB_L2T_IX_V(tx_info->l2te->idx), 0); - return ret; -} - -/* - * chcr_ktls_cpl_act_open_rpl: connection reply received from TP. - */ -int chcr_ktls_cpl_act_open_rpl(struct adapter *adap, unsigned char *input) -{ - const struct cpl_act_open_rpl *p = (void *)input; - struct chcr_ktls_info *tx_info = NULL; - unsigned int atid, tid, status; - struct tid_info *t; - - tid = GET_TID(p); - status = AOPEN_STATUS_G(ntohl(p->atid_status)); - atid = TID_TID_G(AOPEN_ATID_G(ntohl(p->atid_status))); - - t = &adap->tids; - tx_info = lookup_atid(t, atid); - - if (!tx_info || tx_info->atid != atid) { - pr_err("tx_info or atid is not correct\n"); - return -1; - } - - if (!status) { - tx_info->tid = tid; - cxgb4_insert_tid(t, tx_info, tx_info->tid, tx_info->ip_family); - - cxgb4_free_atid(t, atid); - tx_info->atid = -1; - /* update the connection state */ - chcr_ktls_update_connection_state(tx_info, - KTLS_CONN_ACT_OPEN_RPL); - } - return 0; -} - -/* - * chcr_ktls_cpl_set_tcb_rpl: TCB reply received from TP. - */ -int chcr_ktls_cpl_set_tcb_rpl(struct adapter *adap, unsigned char *input) -{ - const struct cpl_set_tcb_rpl *p = (void *)input; - struct chcr_ktls_info *tx_info = NULL; - struct tid_info *t; - u32 tid; - - tid = GET_TID(p); - - t = &adap->tids; - tx_info = lookup_tid(t, tid); - if (!tx_info || tx_info->tid != tid) { - pr_err("tx_info or atid is not correct\n"); - return -1; - } - /* update the connection state */ - chcr_ktls_update_connection_state(tx_info, KTLS_CONN_SET_TCB_RPL); - return 0; -} - -static void *__chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info, - u32 tid, void *pos, u16 word, u64 mask, - u64 val, u32 reply) -{ - struct cpl_set_tcb_field_core *cpl; - struct ulptx_idata *idata; - struct ulp_txpkt *txpkt; - - /* ULP_TXPKT */ - txpkt = pos; - txpkt->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) | ULP_TXPKT_DEST_V(0)); - txpkt->len = htonl(DIV_ROUND_UP(CHCR_SET_TCB_FIELD_LEN, 16)); - - /* ULPTX_IDATA sub-command */ - idata = (struct ulptx_idata *)(txpkt + 1); - idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM)); - idata->len = htonl(sizeof(*cpl)); - pos = idata + 1; - - cpl = pos; - /* CPL_SET_TCB_FIELD */ - OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); - cpl->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) | - NO_REPLY_V(!reply)); - cpl->word_cookie = htons(TCB_WORD_V(word)); - cpl->mask = cpu_to_be64(mask); - cpl->val = cpu_to_be64(val); - - /* ULPTX_NOOP */ - idata = (struct ulptx_idata *)(cpl + 1); - idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP)); - idata->len = htonl(0); - pos = idata + 1; - - return pos; -} - - -/* - * chcr_write_cpl_set_tcb_ulp: update tcb values. - * TCB is responsible to create tcp headers, so all the related values - * should be correctly updated. - * @tx_info - driver specific tls info. - * @q - tx queue on which packet is going out. - * @tid - TCB identifier. - * @pos - current index where should we start writing. - * @word - TCB word. - * @mask - TCB word related mask. - * @val - TCB word related value. - * @reply - set 1 if looking for TP response. - * return - next position to write. - */ -static void *chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info, - struct sge_eth_txq *q, u32 tid, - void *pos, u16 word, u64 mask, - u64 val, u32 reply) -{ - int left = (void *)q->q.stat - pos; - - if (unlikely(left < CHCR_SET_TCB_FIELD_LEN)) { - if (!left) { - pos = q->q.desc; - } else { - u8 buf[48] = {0}; - - __chcr_write_cpl_set_tcb_ulp(tx_info, tid, buf, word, - mask, val, reply); - - return chcr_copy_to_txd(buf, &q->q, pos, - CHCR_SET_TCB_FIELD_LEN); - } - } - - pos = __chcr_write_cpl_set_tcb_ulp(tx_info, tid, pos, word, - mask, val, reply); - - /* check again if we are at the end of the queue */ - if (left == CHCR_SET_TCB_FIELD_LEN) - pos = q->q.desc; - - return pos; -} - -/* - * chcr_ktls_xmit_tcb_cpls: update tcb entry so that TP will create the header - * with updated values like tcp seq, ack, window etc. - * @tx_info - driver specific tls info. - * @q - TX queue. - * @tcp_seq - * @tcp_ack - * @tcp_win - * return: NETDEV_TX_BUSY/NET_TX_OK. - */ -static int chcr_ktls_xmit_tcb_cpls(struct chcr_ktls_info *tx_info, - struct sge_eth_txq *q, u64 tcp_seq, - u64 tcp_ack, u64 tcp_win) -{ - bool first_wr = ((tx_info->prev_ack == 0) && (tx_info->prev_win == 0)); - u32 len, cpl = 0, ndesc, wr_len; - struct fw_ulptx_wr *wr; - int credits; - void *pos; - - wr_len = sizeof(*wr); - /* there can be max 4 cpls, check if we have enough credits */ - len = wr_len + 4 * roundup(CHCR_SET_TCB_FIELD_LEN, 16); - ndesc = DIV_ROUND_UP(len, 64); - - credits = chcr_txq_avail(&q->q) - ndesc; - if (unlikely(credits < 0)) { - chcr_eth_txq_stop(q); - return NETDEV_TX_BUSY; - } - - pos = &q->q.desc[q->q.pidx]; - /* make space for WR, we'll fill it later when we know all the cpls - * being sent out and have complete length. - */ - wr = pos; - pos += wr_len; - /* update tx_max if its a re-transmit or the first wr */ - if (first_wr || tcp_seq != tx_info->prev_seq) { - pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos, - TCB_TX_MAX_W, - TCB_TX_MAX_V(TCB_TX_MAX_M), - TCB_TX_MAX_V(tcp_seq), 0); - cpl++; - } - /* reset snd una if it's a re-transmit pkt */ - if (tcp_seq != tx_info->prev_seq) { - /* reset snd_una */ - pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos, - TCB_SND_UNA_RAW_W, - TCB_SND_UNA_RAW_V - (TCB_SND_UNA_RAW_M), - TCB_SND_UNA_RAW_V(0), 0); - atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_ooo); - cpl++; - } - /* update ack */ - if (first_wr || tx_info->prev_ack != tcp_ack) { - pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos, - TCB_RCV_NXT_W, - TCB_RCV_NXT_V(TCB_RCV_NXT_M), - TCB_RCV_NXT_V(tcp_ack), 0); - tx_info->prev_ack = tcp_ack; - cpl++; - } - /* update receive window */ - if (first_wr || tx_info->prev_win != tcp_win) { - pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos, - TCB_RCV_WND_W, - TCB_RCV_WND_V(TCB_RCV_WND_M), - TCB_RCV_WND_V(tcp_win), 0); - tx_info->prev_win = tcp_win; - cpl++; - } - - if (cpl) { - /* get the actual length */ - len = wr_len + cpl * roundup(CHCR_SET_TCB_FIELD_LEN, 16); - /* ULPTX wr */ - wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR)); - wr->cookie = 0; - /* fill len in wr field */ - wr->flowid_len16 = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(len, 16))); - - ndesc = DIV_ROUND_UP(len, 64); - chcr_txq_advance(&q->q, ndesc); - cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc); - } - return 0; -} - -/* - * chcr_ktls_skb_copy - * @nskb - new skb where the frags to be added. - * @skb - old skb from which frags will be copied. - */ -static void chcr_ktls_skb_copy(struct sk_buff *skb, struct sk_buff *nskb) -{ - int i; - - for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { - skb_shinfo(nskb)->frags[i] = skb_shinfo(skb)->frags[i]; - __skb_frag_ref(&skb_shinfo(nskb)->frags[i]); - } - - skb_shinfo(nskb)->nr_frags = skb_shinfo(skb)->nr_frags; - nskb->len += skb->data_len; - nskb->data_len = skb->data_len; - nskb->truesize += skb->data_len; -} - -/* - * chcr_ktls_get_tx_flits - * returns number of flits to be sent out, it includes key context length, WR - * size and skb fragments. - */ -static unsigned int -chcr_ktls_get_tx_flits(const struct sk_buff *skb, unsigned int key_ctx_len) -{ - return chcr_sgl_len(skb_shinfo(skb)->nr_frags) + - DIV_ROUND_UP(key_ctx_len + CHCR_KTLS_WR_SIZE, 8); -} - -/* - * chcr_ktls_check_tcp_options: To check if there is any TCP option availbale - * other than timestamp. - * @skb - skb contains partial record.. - * return: 1 / 0 - */ -static int -chcr_ktls_check_tcp_options(struct tcphdr *tcp) -{ - int cnt, opt, optlen; - u_char *cp; - - cp = (u_char *)(tcp + 1); - cnt = (tcp->doff << 2) - sizeof(struct tcphdr); - for (; cnt > 0; cnt -= optlen, cp += optlen) { - opt = cp[0]; - if (opt == TCPOPT_EOL) - break; - if (opt == TCPOPT_NOP) { - optlen = 1; - } else { - if (cnt < 2) - break; - optlen = cp[1]; - if (optlen < 2 || optlen > cnt) - break; - } - switch (opt) { - case TCPOPT_NOP: - break; - default: - return 1; - } - } - return 0; -} - -/* - * chcr_ktls_write_tcp_options : TP can't send out all the options, we need to - * send out separately. - * @tx_info - driver specific tls info. - * @skb - skb contains partial record.. - * @q - TX queue. - * @tx_chan - channel number. - * return: NETDEV_TX_OK/NETDEV_TX_BUSY. - */ -static int -chcr_ktls_write_tcp_options(struct chcr_ktls_info *tx_info, struct sk_buff *skb, - struct sge_eth_txq *q, uint32_t tx_chan) -{ - struct fw_eth_tx_pkt_wr *wr; - struct cpl_tx_pkt_core *cpl; - u32 ctrl, iplen, maclen; -#if IS_ENABLED(CONFIG_IPV6) - struct ipv6hdr *ip6; -#endif - unsigned int ndesc; - struct tcphdr *tcp; - int len16, pktlen; - struct iphdr *ip; - int credits; - u8 buf[150]; - void *pos; - - iplen = skb_network_header_len(skb); - maclen = skb_mac_header_len(skb); - - /* packet length = eth hdr len + ip hdr len + tcp hdr len - * (including options). - */ - pktlen = skb->len - skb->data_len; - - ctrl = sizeof(*cpl) + pktlen; - len16 = DIV_ROUND_UP(sizeof(*wr) + ctrl, 16); - /* check how many descriptors needed */ - ndesc = DIV_ROUND_UP(len16, 4); - - credits = chcr_txq_avail(&q->q) - ndesc; - if (unlikely(credits < 0)) { - chcr_eth_txq_stop(q); - return NETDEV_TX_BUSY; - } - - pos = &q->q.desc[q->q.pidx]; - wr = pos; - - /* Firmware work request header */ - wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) | - FW_WR_IMMDLEN_V(ctrl)); - - wr->equiq_to_len16 = htonl(FW_WR_LEN16_V(len16)); - wr->r3 = 0; - - cpl = (void *)(wr + 1); - - /* CPL header */ - cpl->ctrl0 = htonl(TXPKT_OPCODE_V(CPL_TX_PKT) | TXPKT_INTF_V(tx_chan) | - TXPKT_PF_V(tx_info->adap->pf)); - cpl->pack = 0; - cpl->len = htons(pktlen); - /* checksum offload */ - cpl->ctrl1 = 0; - - pos = cpl + 1; - - memcpy(buf, skb->data, pktlen); - if (tx_info->ip_family == AF_INET) { - /* we need to correct ip header len */ - ip = (struct iphdr *)(buf + maclen); - ip->tot_len = htons(pktlen - maclen); -#if IS_ENABLED(CONFIG_IPV6) - } else { - ip6 = (struct ipv6hdr *)(buf + maclen); - ip6->payload_len = htons(pktlen - maclen - iplen); -#endif - } - /* now take care of the tcp header, if fin is not set then clear push - * bit as well, and if fin is set, it will be sent at the last so we - * need to update the tcp sequence number as per the last packet. - */ - tcp = (struct tcphdr *)(buf + maclen + iplen); - - if (!tcp->fin) - tcp->psh = 0; - else - tcp->seq = htonl(tx_info->prev_seq); - - chcr_copy_to_txd(buf, &q->q, pos, pktlen); - - chcr_txq_advance(&q->q, ndesc); - cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc); - return 0; -} - -/* chcr_ktls_skb_shift - Shifts request length paged data from skb to another. - * @tgt- buffer into which tail data gets added - * @skb- buffer from which the paged data comes from - * @shiftlen- shift up to this many bytes - */ -static int chcr_ktls_skb_shift(struct sk_buff *tgt, struct sk_buff *skb, - int shiftlen) -{ - skb_frag_t *fragfrom, *fragto; - int from, to, todo; - - WARN_ON(shiftlen > skb->data_len); - - todo = shiftlen; - from = 0; - to = 0; - fragfrom = &skb_shinfo(skb)->frags[from]; - - while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) { - fragfrom = &skb_shinfo(skb)->frags[from]; - fragto = &skb_shinfo(tgt)->frags[to]; - - if (todo >= skb_frag_size(fragfrom)) { - *fragto = *fragfrom; - todo -= skb_frag_size(fragfrom); - from++; - to++; - - } else { - __skb_frag_ref(fragfrom); - skb_frag_page_copy(fragto, fragfrom); - skb_frag_off_copy(fragto, fragfrom); - skb_frag_size_set(fragto, todo); - - skb_frag_off_add(fragfrom, todo); - skb_frag_size_sub(fragfrom, todo); - todo = 0; - - to++; - break; - } - } - - /* Ready to "commit" this state change to tgt */ - skb_shinfo(tgt)->nr_frags = to; - - /* Reposition in the original skb */ - to = 0; - while (from < skb_shinfo(skb)->nr_frags) - skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++]; - - skb_shinfo(skb)->nr_frags = to; - - WARN_ON(todo > 0 && !skb_shinfo(skb)->nr_frags); - - skb->len -= shiftlen; - skb->data_len -= shiftlen; - skb->truesize -= shiftlen; - tgt->len += shiftlen; - tgt->data_len += shiftlen; - tgt->truesize += shiftlen; - - return shiftlen; -} - -/* - * chcr_ktls_xmit_wr_complete: This sends out the complete record. If an skb - * received has partial end part of the record, send out the complete record, so - * that crypto block will be able to generate TAG/HASH. - * @skb - segment which has complete or partial end part. - * @tx_info - driver specific tls info. - * @q - TX queue. - * @tcp_seq - * @tcp_push - tcp push bit. - * @mss - segment size. - * return: NETDEV_TX_BUSY/NET_TX_OK. - */ -static int chcr_ktls_xmit_wr_complete(struct sk_buff *skb, - struct chcr_ktls_info *tx_info, - struct sge_eth_txq *q, u32 tcp_seq, - bool tcp_push, u32 mss) -{ - u32 len16, wr_mid = 0, flits = 0, ndesc, cipher_start; - struct adapter *adap = tx_info->adap; - int credits, left, last_desc; - struct tx_sw_desc *sgl_sdesc; - struct cpl_tx_data *tx_data; - struct cpl_tx_sec_pdu *cpl; - struct ulptx_idata *idata; - struct ulp_txpkt *ulptx; - struct fw_ulptx_wr *wr; - void *pos; - u64 *end; - - /* get the number of flits required */ - flits = chcr_ktls_get_tx_flits(skb, tx_info->key_ctx_len); - /* number of descriptors */ - ndesc = chcr_flits_to_desc(flits); - /* check if enough credits available */ - credits = chcr_txq_avail(&q->q) - ndesc; - if (unlikely(credits < 0)) { - chcr_eth_txq_stop(q); - return NETDEV_TX_BUSY; - } - - if (unlikely(credits < ETHTXQ_STOP_THRES)) { - /* Credits are below the threshold vaues, stop the queue after - * injecting the Work Request for this packet. - */ - chcr_eth_txq_stop(q); - wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F; - } - - last_desc = q->q.pidx + ndesc - 1; - if (last_desc >= q->q.size) - last_desc -= q->q.size; - sgl_sdesc = &q->q.sdesc[last_desc]; - - if (unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) { - memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr)); - q->mapping_err++; - return NETDEV_TX_BUSY; - } - - pos = &q->q.desc[q->q.pidx]; - end = (u64 *)pos + flits; - /* FW_ULPTX_WR */ - wr = pos; - /* WR will need len16 */ - len16 = DIV_ROUND_UP(flits, 2); - wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR)); - wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16)); - wr->cookie = 0; - pos += sizeof(*wr); - /* ULP_TXPKT */ - ulptx = pos; - ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) | - ULP_TXPKT_CHANNELID_V(tx_info->port_id) | - ULP_TXPKT_FID_V(q->q.cntxt_id) | - ULP_TXPKT_RO_F); - ulptx->len = htonl(len16 - 1); - /* ULPTX_IDATA sub-command */ - idata = (struct ulptx_idata *)(ulptx + 1); - idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F); - /* idata length will include cpl_tx_sec_pdu + key context size + - * cpl_tx_data header. - */ - idata->len = htonl(sizeof(*cpl) + tx_info->key_ctx_len + - sizeof(*tx_data)); - /* SEC CPL */ - cpl = (struct cpl_tx_sec_pdu *)(idata + 1); - cpl->op_ivinsrtofst = - htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) | - CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT) | - CPL_TX_SEC_PDU_PLACEHOLDER_V(1) | - CPL_TX_SEC_PDU_IVINSRTOFST_V(TLS_HEADER_SIZE + 1)); - cpl->pldlen = htonl(skb->data_len); - - /* encryption should start after tls header size + iv size */ - cipher_start = TLS_HEADER_SIZE + tx_info->iv_size + 1; - - cpl->aadstart_cipherstop_hi = - htonl(CPL_TX_SEC_PDU_AADSTART_V(1) | - CPL_TX_SEC_PDU_AADSTOP_V(TLS_HEADER_SIZE) | - CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start)); - - /* authentication will also start after tls header + iv size */ - cpl->cipherstop_lo_authinsert = - htonl(CPL_TX_SEC_PDU_AUTHSTART_V(cipher_start) | - CPL_TX_SEC_PDU_AUTHSTOP_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE) | - CPL_TX_SEC_PDU_AUTHINSERT_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE)); - - /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */ - cpl->seqno_numivs = htonl(tx_info->scmd0_seqno_numivs); - cpl->ivgen_hdrlen = htonl(tx_info->scmd0_ivgen_hdrlen); - cpl->scmd1 = cpu_to_be64(tx_info->record_no); - - pos = cpl + 1; - /* check if space left to fill the keys */ - left = (void *)q->q.stat - pos; - if (!left) { - left = (void *)end - (void *)q->q.stat; - pos = q->q.desc; - end = pos + left; - } - - pos = chcr_copy_to_txd(&tx_info->key_ctx, &q->q, pos, - tx_info->key_ctx_len); - left = (void *)q->q.stat - pos; - - if (!left) { - left = (void *)end - (void *)q->q.stat; - pos = q->q.desc; - end = pos + left; - } - /* CPL_TX_DATA */ - tx_data = (void *)pos; - OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid)); - tx_data->len = htonl(TX_DATA_MSS_V(mss) | TX_LENGTH_V(skb->data_len)); - - tx_data->rsvd = htonl(tcp_seq); - - tx_data->flags = htonl(TX_BYPASS_F); - if (tcp_push) - tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F); - - /* check left again, it might go beyond queue limit */ - pos = tx_data + 1; - left = (void *)q->q.stat - pos; - - /* check the position again */ - if (!left) { - left = (void *)end - (void *)q->q.stat; - pos = q->q.desc; - end = pos + left; - } - - /* send the complete packet except the header */ - cxgb4_write_sgl(skb, &q->q, pos, end, skb->len - skb->data_len, - sgl_sdesc->addr); - sgl_sdesc->skb = skb; - - chcr_txq_advance(&q->q, ndesc); - cxgb4_ring_tx_db(adap, &q->q, ndesc); - atomic64_inc(&adap->chcr_stats.ktls_tx_send_records); - - return 0; -} - -/* - * chcr_ktls_xmit_wr_short: This is to send out partial records. If its - * a middle part of a record, fetch the prior data to make it 16 byte aligned - * and then only send it out. - * - * @skb - skb contains partial record.. - * @tx_info - driver specific tls info. - * @q - TX queue. - * @tcp_seq - * @tcp_push - tcp push bit. - * @mss - segment size. - * @tls_rec_offset - offset from start of the tls record. - * @perior_data - data before the current segment, required to make this record - * 16 byte aligned. - * @prior_data_len - prior_data length (less than 16) - * return: NETDEV_TX_BUSY/NET_TX_OK. - */ -static int chcr_ktls_xmit_wr_short(struct sk_buff *skb, - struct chcr_ktls_info *tx_info, - struct sge_eth_txq *q, - u32 tcp_seq, bool tcp_push, u32 mss, - u32 tls_rec_offset, u8 *prior_data, - u32 prior_data_len) -{ - struct adapter *adap = tx_info->adap; - u32 len16, wr_mid = 0, cipher_start; - unsigned int flits = 0, ndesc; - int credits, left, last_desc; - struct tx_sw_desc *sgl_sdesc; - struct cpl_tx_data *tx_data; - struct cpl_tx_sec_pdu *cpl; - struct ulptx_idata *idata; - struct ulp_txpkt *ulptx; - struct fw_ulptx_wr *wr; - __be64 iv_record; - void *pos; - u64 *end; - - /* get the number of flits required, it's a partial record so 2 flits - * (AES_BLOCK_SIZE) will be added. - */ - flits = chcr_ktls_get_tx_flits(skb, tx_info->key_ctx_len) + 2; - /* get the correct 8 byte IV of this record */ - iv_record = cpu_to_be64(tx_info->iv + tx_info->record_no); - /* If it's a middle record and not 16 byte aligned to run AES CTR, need - * to make it 16 byte aligned. So atleadt 2 extra flits of immediate - * data will be added. - */ - if (prior_data_len) - flits += 2; - /* number of descriptors */ - ndesc = chcr_flits_to_desc(flits); - /* check if enough credits available */ - credits = chcr_txq_avail(&q->q) - ndesc; - if (unlikely(credits < 0)) { - chcr_eth_txq_stop(q); - return NETDEV_TX_BUSY; - } - - if (unlikely(credits < ETHTXQ_STOP_THRES)) { - chcr_eth_txq_stop(q); - wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F; - } - - last_desc = q->q.pidx + ndesc - 1; - if (last_desc >= q->q.size) - last_desc -= q->q.size; - sgl_sdesc = &q->q.sdesc[last_desc]; - - if (unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) { - memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr)); - q->mapping_err++; - return NETDEV_TX_BUSY; - } - - pos = &q->q.desc[q->q.pidx]; - end = (u64 *)pos + flits; - /* FW_ULPTX_WR */ - wr = pos; - /* WR will need len16 */ - len16 = DIV_ROUND_UP(flits, 2); - wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR)); - wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16)); - wr->cookie = 0; - pos += sizeof(*wr); - /* ULP_TXPKT */ - ulptx = pos; - ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) | - ULP_TXPKT_CHANNELID_V(tx_info->port_id) | - ULP_TXPKT_FID_V(q->q.cntxt_id) | - ULP_TXPKT_RO_F); - ulptx->len = htonl(len16 - 1); - /* ULPTX_IDATA sub-command */ - idata = (struct ulptx_idata *)(ulptx + 1); - idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F); - /* idata length will include cpl_tx_sec_pdu + key context size + - * cpl_tx_data header. - */ - idata->len = htonl(sizeof(*cpl) + tx_info->key_ctx_len + - sizeof(*tx_data) + AES_BLOCK_LEN + prior_data_len); - /* SEC CPL */ - cpl = (struct cpl_tx_sec_pdu *)(idata + 1); - /* cipher start will have tls header + iv size extra if its a header - * part of tls record. else only 16 byte IV will be added. - */ - cipher_start = - AES_BLOCK_LEN + 1 + - (!tls_rec_offset ? TLS_HEADER_SIZE + tx_info->iv_size : 0); - - cpl->op_ivinsrtofst = - htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) | - CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT) | - CPL_TX_SEC_PDU_IVINSRTOFST_V(1)); - cpl->pldlen = htonl(skb->data_len + AES_BLOCK_LEN + prior_data_len); - cpl->aadstart_cipherstop_hi = - htonl(CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start)); - cpl->cipherstop_lo_authinsert = 0; - /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */ - cpl->seqno_numivs = htonl(tx_info->scmd0_short_seqno_numivs); - cpl->ivgen_hdrlen = htonl(tx_info->scmd0_short_ivgen_hdrlen); - cpl->scmd1 = 0; - - pos = cpl + 1; - /* check if space left to fill the keys */ - left = (void *)q->q.stat - pos; - if (!left) { - left = (void *)end - (void *)q->q.stat; - pos = q->q.desc; - end = pos + left; - } - - pos = chcr_copy_to_txd(&tx_info->key_ctx, &q->q, pos, - tx_info->key_ctx_len); - left = (void *)q->q.stat - pos; - - if (!left) { - left = (void *)end - (void *)q->q.stat; - pos = q->q.desc; - end = pos + left; - } - /* CPL_TX_DATA */ - tx_data = (void *)pos; - OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid)); - tx_data->len = htonl(TX_DATA_MSS_V(mss) | - TX_LENGTH_V(skb->data_len + prior_data_len)); - tx_data->rsvd = htonl(tcp_seq); - tx_data->flags = htonl(TX_BYPASS_F); - if (tcp_push) - tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F); - - /* check left again, it might go beyond queue limit */ - pos = tx_data + 1; - left = (void *)q->q.stat - pos; - - /* check the position again */ - if (!left) { - left = (void *)end - (void *)q->q.stat; - pos = q->q.desc; - end = pos + left; - } - /* copy the 16 byte IV for AES-CTR, which includes 4 bytes of salt, 8 - * bytes of actual IV and 4 bytes of 16 byte-sequence. - */ - memcpy(pos, tx_info->key_ctx.salt, tx_info->salt_size); - memcpy(pos + tx_info->salt_size, &iv_record, tx_info->iv_size); - *(__be32 *)(pos + tx_info->salt_size + tx_info->iv_size) = - htonl(2 + (tls_rec_offset ? ((tls_rec_offset - - (TLS_HEADER_SIZE + tx_info->iv_size)) / AES_BLOCK_LEN) : 0)); - - pos += 16; - /* Prior_data_len will always be less than 16 bytes, fill the - * prio_data_len after AES_CTRL_BLOCK and clear the remaining length - * to 0. - */ - if (prior_data_len) - pos = chcr_copy_to_txd(prior_data, &q->q, pos, 16); - /* send the complete packet except the header */ - cxgb4_write_sgl(skb, &q->q, pos, end, skb->len - skb->data_len, - sgl_sdesc->addr); - sgl_sdesc->skb = skb; - - chcr_txq_advance(&q->q, ndesc); - cxgb4_ring_tx_db(adap, &q->q, ndesc); - - return 0; -} - -/* - * chcr_ktls_tx_plaintxt: This handler will take care of the records which has - * only plain text (only tls header and iv) - * @tx_info - driver specific tls info. - * @skb - skb contains partial record.. - * @tcp_seq - * @mss - segment size. - * @tcp_push - tcp push bit. - * @q - TX queue. - * @port_id : port number - * @perior_data - data before the current segment, required to make this record - * 16 byte aligned. - * @prior_data_len - prior_data length (less than 16) - * return: NETDEV_TX_BUSY/NET_TX_OK. - */ -static int chcr_ktls_tx_plaintxt(struct chcr_ktls_info *tx_info, - struct sk_buff *skb, u32 tcp_seq, u32 mss, - bool tcp_push, struct sge_eth_txq *q, - u32 port_id, u8 *prior_data, - u32 prior_data_len) -{ - int credits, left, len16, last_desc; - unsigned int flits = 0, ndesc; - struct tx_sw_desc *sgl_sdesc; - struct cpl_tx_data *tx_data; - struct ulptx_idata *idata; - struct ulp_txpkt *ulptx; - struct fw_ulptx_wr *wr; - u32 wr_mid = 0; - void *pos; - u64 *end; - - flits = DIV_ROUND_UP(CHCR_PLAIN_TX_DATA_LEN, 8); - flits += chcr_sgl_len(skb_shinfo(skb)->nr_frags); - if (prior_data_len) - flits += 2; - /* WR will need len16 */ - len16 = DIV_ROUND_UP(flits, 2); - /* check how many descriptors needed */ - ndesc = DIV_ROUND_UP(flits, 8); - - credits = chcr_txq_avail(&q->q) - ndesc; - if (unlikely(credits < 0)) { - chcr_eth_txq_stop(q); - return NETDEV_TX_BUSY; - } - - if (unlikely(credits < ETHTXQ_STOP_THRES)) { - chcr_eth_txq_stop(q); - wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F; - } - - last_desc = q->q.pidx + ndesc - 1; - if (last_desc >= q->q.size) - last_desc -= q->q.size; - sgl_sdesc = &q->q.sdesc[last_desc]; - - if (unlikely(cxgb4_map_skb(tx_info->adap->pdev_dev, skb, - sgl_sdesc->addr) < 0)) { - memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr)); - q->mapping_err++; - return NETDEV_TX_BUSY; - } - - pos = &q->q.desc[q->q.pidx]; - end = (u64 *)pos + flits; - /* FW_ULPTX_WR */ - wr = pos; - wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR)); - wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16)); - wr->cookie = 0; - pos += sizeof(*wr); - /* ULP_TXPKT */ - ulptx = (struct ulp_txpkt *)(wr + 1); - ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) | - ULP_TXPKT_DATAMODIFY_V(0) | - ULP_TXPKT_CHANNELID_V(tx_info->port_id) | - ULP_TXPKT_DEST_V(0) | - ULP_TXPKT_FID_V(q->q.cntxt_id) | ULP_TXPKT_RO_V(1)); - ulptx->len = htonl(len16 - 1); - /* ULPTX_IDATA sub-command */ - idata = (struct ulptx_idata *)(ulptx + 1); - idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F); - idata->len = htonl(sizeof(*tx_data) + prior_data_len); - /* CPL_TX_DATA */ - tx_data = (struct cpl_tx_data *)(idata + 1); - OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid)); - tx_data->len = htonl(TX_DATA_MSS_V(mss) | - TX_LENGTH_V(skb->data_len + prior_data_len)); - /* set tcp seq number */ - tx_data->rsvd = htonl(tcp_seq); - tx_data->flags = htonl(TX_BYPASS_F); - if (tcp_push) - tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F); - - pos = tx_data + 1; - /* apart from prior_data_len, we should set remaining part of 16 bytes - * to be zero. - */ - if (prior_data_len) - pos = chcr_copy_to_txd(prior_data, &q->q, pos, 16); - - /* check left again, it might go beyond queue limit */ - left = (void *)q->q.stat - pos; - - /* check the position again */ - if (!left) { - left = (void *)end - (void *)q->q.stat; - pos = q->q.desc; - end = pos + left; - } - /* send the complete packet including the header */ - cxgb4_write_sgl(skb, &q->q, pos, end, skb->len - skb->data_len, - sgl_sdesc->addr); - sgl_sdesc->skb = skb; - - chcr_txq_advance(&q->q, ndesc); - cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc); - return 0; -} - -/* - * chcr_ktls_copy_record_in_skb - * @nskb - new skb where the frags to be added. - * @record - specific record which has complete 16k record in frags. - */ -static void chcr_ktls_copy_record_in_skb(struct sk_buff *nskb, - struct tls_record_info *record) -{ - int i = 0; - - for (i = 0; i < record->num_frags; i++) { - skb_shinfo(nskb)->frags[i] = record->frags[i]; - /* increase the frag ref count */ - __skb_frag_ref(&skb_shinfo(nskb)->frags[i]); - } - - skb_shinfo(nskb)->nr_frags = record->num_frags; - nskb->data_len = record->len; - nskb->len += record->len; - nskb->truesize += record->len; -} - -/* - * chcr_ktls_update_snd_una: Reset the SEND_UNA. It will be done to avoid - * sending the same segment again. It will discard the segment which is before - * the current tx max. - * @tx_info - driver specific tls info. - * @q - TX queue. - * return: NET_TX_OK/NET_XMIT_DROP. - */ -static int chcr_ktls_update_snd_una(struct chcr_ktls_info *tx_info, - struct sge_eth_txq *q) -{ - struct fw_ulptx_wr *wr; - unsigned int ndesc; - int credits; - void *pos; - u32 len; - - len = sizeof(*wr) + roundup(CHCR_SET_TCB_FIELD_LEN, 16); - ndesc = DIV_ROUND_UP(len, 64); - - credits = chcr_txq_avail(&q->q) - ndesc; - if (unlikely(credits < 0)) { - chcr_eth_txq_stop(q); - return NETDEV_TX_BUSY; - } - - pos = &q->q.desc[q->q.pidx]; - - wr = pos; - /* ULPTX wr */ - wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR)); - wr->cookie = 0; - /* fill len in wr field */ - wr->flowid_len16 = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(len, 16))); - - pos += sizeof(*wr); - - pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos, - TCB_SND_UNA_RAW_W, - TCB_SND_UNA_RAW_V(TCB_SND_UNA_RAW_M), - TCB_SND_UNA_RAW_V(0), 0); - - chcr_txq_advance(&q->q, ndesc); - cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc); - - return 0; -} - -/* - * chcr_end_part_handler: This handler will handle the record which - * is complete or if record's end part is received. T6 adapter has a issue that - * it can't send out TAG with partial record so if its an end part then we have - * to send TAG as well and for which we need to fetch the complete record and - * send it to crypto module. - * @tx_info - driver specific tls info. - * @skb - skb contains partial record. - * @record - complete record of 16K size. - * @tcp_seq - * @mss - segment size in which TP needs to chop a packet. - * @tcp_push_no_fin - tcp push if fin is not set. - * @q - TX queue. - * @tls_end_offset - offset from end of the record. - * @last wr : check if this is the last part of the skb going out. - * return: NETDEV_TX_OK/NETDEV_TX_BUSY. - */ -static int chcr_end_part_handler(struct chcr_ktls_info *tx_info, - struct sk_buff *skb, - struct tls_record_info *record, - u32 tcp_seq, int mss, bool tcp_push_no_fin, - struct sge_eth_txq *q, - u32 tls_end_offset, bool last_wr) -{ - struct sk_buff *nskb = NULL; - /* check if it is a complete record */ - if (tls_end_offset == record->len) { - nskb = skb; - atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_complete_pkts); - } else { - dev_kfree_skb_any(skb); - - nskb = alloc_skb(0, GFP_KERNEL); - if (!nskb) - return NETDEV_TX_BUSY; - /* copy complete record in skb */ - chcr_ktls_copy_record_in_skb(nskb, record); - /* packet is being sent from the beginning, update the tcp_seq - * accordingly. - */ - tcp_seq = tls_record_start_seq(record); - /* reset snd una, so the middle record won't send the already - * sent part. - */ - if (chcr_ktls_update_snd_una(tx_info, q)) - goto out; - atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_end_pkts); - } - - if (chcr_ktls_xmit_wr_complete(nskb, tx_info, q, tcp_seq, - (last_wr && tcp_push_no_fin), - mss)) { - goto out; - } - return 0; -out: - dev_kfree_skb_any(nskb); - return NETDEV_TX_BUSY; -} - -/* - * chcr_short_record_handler: This handler will take care of the records which - * doesn't have end part (1st part or the middle part(/s) of a record). In such - * cases, AES CTR will be used in place of AES GCM to send out partial packet. - * This partial record might be the first part of the record, or the middle - * part. In case of middle record we should fetch the prior data to make it 16 - * byte aligned. If it has a partial tls header or iv then get to the start of - * tls header. And if it has partial TAG, then remove the complete TAG and send - * only the payload. - * There is one more possibility that it gets a partial header, send that - * portion as a plaintext. - * @tx_info - driver specific tls info. - * @skb - skb contains partial record.. - * @record - complete record of 16K size. - * @tcp_seq - * @mss - segment size in which TP needs to chop a packet. - * @tcp_push_no_fin - tcp push if fin is not set. - * @q - TX queue. - * @tls_end_offset - offset from end of the record. - * return: NETDEV_TX_OK/NETDEV_TX_BUSY. - */ -static int chcr_short_record_handler(struct chcr_ktls_info *tx_info, - struct sk_buff *skb, - struct tls_record_info *record, - u32 tcp_seq, int mss, bool tcp_push_no_fin, - struct sge_eth_txq *q, u32 tls_end_offset) -{ - u32 tls_rec_offset = tcp_seq - tls_record_start_seq(record); - u8 prior_data[16] = {0}; - u32 prior_data_len = 0; - u32 data_len; - - /* check if the skb is ending in middle of tag/HASH, its a big - * trouble, send the packet before the HASH. - */ - int remaining_record = tls_end_offset - skb->data_len; - - if (remaining_record > 0 && - remaining_record < TLS_CIPHER_AES_GCM_128_TAG_SIZE) { - int trimmed_len = skb->data_len - - (TLS_CIPHER_AES_GCM_128_TAG_SIZE - remaining_record); - struct sk_buff *tmp_skb = NULL; - /* don't process the pkt if it is only a partial tag */ - if (skb->data_len < TLS_CIPHER_AES_GCM_128_TAG_SIZE) - goto out; - - WARN_ON(trimmed_len > skb->data_len); - - /* shift to those many bytes */ - tmp_skb = alloc_skb(0, GFP_KERNEL); - if (unlikely(!tmp_skb)) - goto out; - - chcr_ktls_skb_shift(tmp_skb, skb, trimmed_len); - /* free the last trimmed portion */ - dev_kfree_skb_any(skb); - skb = tmp_skb; - atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_trimmed_pkts); - } - data_len = skb->data_len; - /* check if the middle record's start point is 16 byte aligned. CTR - * needs 16 byte aligned start point to start encryption. - */ - if (tls_rec_offset) { - /* there is an offset from start, means its a middle record */ - int remaining = 0; - - if (tls_rec_offset < (TLS_HEADER_SIZE + tx_info->iv_size)) { - prior_data_len = tls_rec_offset; - tls_rec_offset = 0; - remaining = 0; - } else { - prior_data_len = - (tls_rec_offset - - (TLS_HEADER_SIZE + tx_info->iv_size)) - % AES_BLOCK_LEN; - remaining = tls_rec_offset - prior_data_len; - } - - /* if prior_data_len is not zero, means we need to fetch prior - * data to make this record 16 byte aligned, or we need to reach - * to start offset. - */ - if (prior_data_len) { - int i = 0; - u8 *data = NULL; - skb_frag_t *f; - u8 *vaddr; - int frag_size = 0, frag_delta = 0; - - while (remaining > 0) { - frag_size = skb_frag_size(&record->frags[i]); - if (remaining < frag_size) - break; - - remaining -= frag_size; - i++; - } - f = &record->frags[i]; - vaddr = kmap_atomic(skb_frag_page(f)); - - data = vaddr + skb_frag_off(f) + remaining; - frag_delta = skb_frag_size(f) - remaining; - - if (frag_delta >= prior_data_len) { - memcpy(prior_data, data, prior_data_len); - kunmap_atomic(vaddr); - } else { - memcpy(prior_data, data, frag_delta); - kunmap_atomic(vaddr); - /* get the next page */ - f = &record->frags[i + 1]; - vaddr = kmap_atomic(skb_frag_page(f)); - data = vaddr + skb_frag_off(f); - memcpy(prior_data + frag_delta, - data, (prior_data_len - frag_delta)); - kunmap_atomic(vaddr); - } - /* reset tcp_seq as per the prior_data_required len */ - tcp_seq -= prior_data_len; - /* include prio_data_len for further calculation. - */ - data_len += prior_data_len; - } - /* reset snd una, so the middle record won't send the already - * sent part. - */ - if (chcr_ktls_update_snd_una(tx_info, q)) - goto out; - atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_middle_pkts); - } else { - /* Else means, its a partial first part of the record. Check if - * its only the header, don't need to send for encryption then. - */ - if (data_len <= TLS_HEADER_SIZE + tx_info->iv_size) { - if (chcr_ktls_tx_plaintxt(tx_info, skb, tcp_seq, mss, - tcp_push_no_fin, q, - tx_info->port_id, - prior_data, - prior_data_len)) { - goto out; - } - return 0; - } - atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_start_pkts); - } - - if (chcr_ktls_xmit_wr_short(skb, tx_info, q, tcp_seq, tcp_push_no_fin, - mss, tls_rec_offset, prior_data, - prior_data_len)) { - goto out; - } - - return 0; -out: - dev_kfree_skb_any(skb); - return NETDEV_TX_BUSY; -} - -/* nic tls TX handler */ -int chcr_ktls_xmit(struct sk_buff *skb, struct net_device *dev) -{ - struct chcr_ktls_ofld_ctx_tx *tx_ctx; - struct tcphdr *th = tcp_hdr(skb); - int data_len, qidx, ret = 0, mss; - struct tls_record_info *record; - struct chcr_stats_debug *stats; - struct chcr_ktls_info *tx_info; - u32 tls_end_offset, tcp_seq; - struct tls_context *tls_ctx; - struct sk_buff *local_skb; - int new_connection_state; - struct sge_eth_txq *q; - struct adapter *adap; - unsigned long flags; - - tcp_seq = ntohl(th->seq); - - mss = skb_is_gso(skb) ? skb_shinfo(skb)->gso_size : skb->data_len; - - /* check if we haven't set it for ktls offload */ - if (!skb->sk || !tls_is_sk_tx_device_offloaded(skb->sk)) - goto out; - - tls_ctx = tls_get_ctx(skb->sk); - if (unlikely(tls_ctx->netdev != dev)) - goto out; - - tx_ctx = chcr_get_ktls_tx_context(tls_ctx); - tx_info = tx_ctx->chcr_info; - - if (unlikely(!tx_info)) - goto out; - - /* check the connection state, we don't need to pass new connection - * state, state machine will check and update the new state if it is - * stuck due to responses not received from HW. - * Start the tx handling only if state is KTLS_CONN_TX_READY. - */ - new_connection_state = chcr_ktls_update_connection_state(tx_info, 0); - if (new_connection_state != KTLS_CONN_TX_READY) - goto out; - - /* don't touch the original skb, make a new skb to extract each records - * and send them separately. - */ - local_skb = alloc_skb(0, GFP_KERNEL); - - if (unlikely(!local_skb)) - return NETDEV_TX_BUSY; - - adap = tx_info->adap; - stats = &adap->chcr_stats; - - qidx = skb->queue_mapping; - q = &adap->sge.ethtxq[qidx + tx_info->first_qset]; - cxgb4_reclaim_completed_tx(adap, &q->q, true); - /* if tcp options are set but finish is not send the options first */ - if (!th->fin && chcr_ktls_check_tcp_options(th)) { - ret = chcr_ktls_write_tcp_options(tx_info, skb, q, - tx_info->tx_chan); - if (ret) - return NETDEV_TX_BUSY; - } - /* update tcb */ - ret = chcr_ktls_xmit_tcb_cpls(tx_info, q, ntohl(th->seq), - ntohl(th->ack_seq), - ntohs(th->window)); - if (ret) { - dev_kfree_skb_any(local_skb); - return NETDEV_TX_BUSY; - } - - /* copy skb contents into local skb */ - chcr_ktls_skb_copy(skb, local_skb); - - /* go through the skb and send only one record at a time. */ - data_len = skb->data_len; - /* TCP segments can be in received either complete or partial. - * chcr_end_part_handler will handle cases if complete record or end - * part of the record is received. Incase of partial end part of record, - * we will send the complete record again. - */ - - do { - int i; - - cxgb4_reclaim_completed_tx(adap, &q->q, true); - /* lock taken */ - spin_lock_irqsave(&tx_ctx->base.lock, flags); - /* fetch the tls record */ - record = tls_get_record(&tx_ctx->base, tcp_seq, - &tx_info->record_no); - /* By the time packet reached to us, ACK is received, and record - * won't be found in that case, handle it gracefully. - */ - if (unlikely(!record)) { - spin_unlock_irqrestore(&tx_ctx->base.lock, flags); - atomic64_inc(&stats->ktls_tx_drop_no_sync_data); - goto out; - } - - if (unlikely(tls_record_is_start_marker(record))) { - spin_unlock_irqrestore(&tx_ctx->base.lock, flags); - atomic64_inc(&stats->ktls_tx_skip_no_sync_data); - goto out; - } - - /* increase page reference count of the record, so that there - * won't be any chance of page free in middle if in case stack - * receives ACK and try to delete the record. - */ - for (i = 0; i < record->num_frags; i++) - __skb_frag_ref(&record->frags[i]); - /* lock cleared */ - spin_unlock_irqrestore(&tx_ctx->base.lock, flags); - - tls_end_offset = record->end_seq - tcp_seq; - - pr_debug("seq 0x%x, end_seq 0x%x prev_seq 0x%x, datalen 0x%x\n", - tcp_seq, record->end_seq, tx_info->prev_seq, data_len); - /* if a tls record is finishing in this SKB */ - if (tls_end_offset <= data_len) { - struct sk_buff *nskb = NULL; - - if (tls_end_offset < data_len) { - nskb = alloc_skb(0, GFP_KERNEL); - if (unlikely(!nskb)) { - ret = -ENOMEM; - goto clear_ref; - } - - chcr_ktls_skb_shift(nskb, local_skb, - tls_end_offset); - } else { - /* its the only record in this skb, directly - * point it. - */ - nskb = local_skb; - } - ret = chcr_end_part_handler(tx_info, nskb, record, - tcp_seq, mss, - (!th->fin && th->psh), q, - tls_end_offset, - (nskb == local_skb)); - - if (ret && nskb != local_skb) - dev_kfree_skb_any(local_skb); - - data_len -= tls_end_offset; - /* tcp_seq increment is required to handle next record. - */ - tcp_seq += tls_end_offset; - } else { - ret = chcr_short_record_handler(tx_info, local_skb, - record, tcp_seq, mss, - (!th->fin && th->psh), - q, tls_end_offset); - data_len = 0; - } -clear_ref: - /* clear the frag ref count which increased locally before */ - for (i = 0; i < record->num_frags; i++) { - /* clear the frag ref count */ - __skb_frag_unref(&record->frags[i]); - } - /* if any failure, come out from the loop. */ - if (ret) - goto out; - /* length should never be less than 0 */ - WARN_ON(data_len < 0); - - } while (data_len > 0); - - tx_info->prev_seq = ntohl(th->seq) + skb->data_len; - - atomic64_inc(&stats->ktls_tx_encrypted_packets); - atomic64_add(skb->data_len, &stats->ktls_tx_encrypted_bytes); - - /* tcp finish is set, send a separate tcp msg including all the options - * as well. - */ - if (th->fin) - chcr_ktls_write_tcp_options(tx_info, skb, q, tx_info->tx_chan); - -out: - dev_kfree_skb_any(skb); - return NETDEV_TX_OK; -} -#endif /* CONFIG_CHELSIO_TLS_DEVICE */ diff --git a/drivers/crypto/chelsio/chcr_ktls.h b/drivers/crypto/chelsio/chcr_ktls.h deleted file mode 100644 index 5cbd84b1da05..000000000000 --- a/drivers/crypto/chelsio/chcr_ktls.h +++ /dev/null @@ -1,103 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright (C) 2020 Chelsio Communications. All rights reserved. */ - -#ifndef __CHCR_KTLS_H__ -#define __CHCR_KTLS_H__ - -#ifdef CONFIG_CHELSIO_TLS_DEVICE -#include <net/tls.h> -#include "cxgb4.h" -#include "t4_msg.h" -#include "t4_tcb.h" -#include "l2t.h" -#include "chcr_common.h" -#include "cxgb4_uld.h" - -#define CHCR_TCB_STATE_CLOSED 0 -#define CHCR_KTLS_KEY_CTX_LEN 16 -#define CHCR_SET_TCB_FIELD_LEN sizeof(struct cpl_set_tcb_field) -#define CHCR_PLAIN_TX_DATA_LEN (sizeof(struct fw_ulptx_wr) +\ - sizeof(struct ulp_txpkt) +\ - sizeof(struct ulptx_idata) +\ - sizeof(struct cpl_tx_data)) - -#define CHCR_KTLS_WR_SIZE (CHCR_PLAIN_TX_DATA_LEN +\ - sizeof(struct cpl_tx_sec_pdu)) - -enum chcr_ktls_conn_state { - KTLS_CONN_CLOSED, - KTLS_CONN_ACT_OPEN_REQ, - KTLS_CONN_ACT_OPEN_RPL, - KTLS_CONN_SET_TCB_REQ, - KTLS_CONN_SET_TCB_RPL, - KTLS_CONN_TX_READY, -}; - -struct chcr_ktls_info { - struct sock *sk; - spinlock_t lock; /* state machine lock */ - struct ktls_key_ctx key_ctx; - struct adapter *adap; - struct l2t_entry *l2te; - struct net_device *netdev; - u64 iv; - u64 record_no; - int tid; - int atid; - int rx_qid; - u32 iv_size; - u32 prev_seq; - u32 prev_ack; - u32 salt_size; - u32 key_ctx_len; - u32 scmd0_seqno_numivs; - u32 scmd0_ivgen_hdrlen; - u32 tcp_start_seq_number; - u32 scmd0_short_seqno_numivs; - u32 scmd0_short_ivgen_hdrlen; - enum chcr_ktls_conn_state connection_state; - u16 prev_win; - u8 tx_chan; - u8 smt_idx; - u8 port_id; - u8 ip_family; - u8 first_qset; -}; - -struct chcr_ktls_ofld_ctx_tx { - struct tls_offload_context_tx base; - struct chcr_ktls_info *chcr_info; -}; - -static inline struct chcr_ktls_ofld_ctx_tx * -chcr_get_ktls_tx_context(struct tls_context *tls_ctx) -{ - BUILD_BUG_ON(sizeof(struct chcr_ktls_ofld_ctx_tx) > - TLS_OFFLOAD_CONTEXT_SIZE_TX); - return container_of(tls_offload_ctx_tx(tls_ctx), - struct chcr_ktls_ofld_ctx_tx, - base); -} - -static inline int chcr_get_first_rx_qid(struct adapter *adap) -{ - /* u_ctx is saved in adap, fetch it */ - struct uld_ctx *u_ctx = adap->uld[CXGB4_ULD_CRYPTO].handle; - - if (!u_ctx) - return -1; - return u_ctx->lldi.rxq_ids[0]; -} - -int chcr_ktls_cpl_act_open_rpl(struct adapter *adap, unsigned char *input); -int chcr_ktls_cpl_set_tcb_rpl(struct adapter *adap, unsigned char *input); -int chcr_ktls_xmit(struct sk_buff *skb, struct net_device *dev); -int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk, - enum tls_offload_ctx_dir direction, - struct tls_crypto_info *crypto_info, - u32 start_offload_tcp_sn); -void chcr_ktls_dev_del(struct net_device *netdev, - struct tls_context *tls_ctx, - enum tls_offload_ctx_dir direction); -#endif /* CONFIG_CHELSIO_TLS_DEVICE */ -#endif /* __CHCR_KTLS_H__ */ diff --git a/drivers/crypto/chelsio/chtls/Makefile b/drivers/crypto/chelsio/chtls/Makefile deleted file mode 100644 index bc11495acdb3..000000000000 --- a/drivers/crypto/chelsio/chtls/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(srctree)/drivers/net/ethernet/chelsio/cxgb4 \ - -I $(srctree)/drivers/crypto/chelsio - -obj-$(CONFIG_CRYPTO_DEV_CHELSIO_TLS) += chtls.o -chtls-objs := chtls_main.o chtls_cm.o chtls_io.o chtls_hw.o diff --git a/drivers/crypto/chelsio/chtls/chtls.h b/drivers/crypto/chelsio/chtls/chtls.h deleted file mode 100644 index 459442704eb1..000000000000 --- a/drivers/crypto/chelsio/chtls/chtls.h +++ /dev/null @@ -1,492 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (c) 2018 Chelsio Communications, Inc. - */ - -#ifndef __CHTLS_H__ -#define __CHTLS_H__ - -#include <crypto/aes.h> -#include <crypto/algapi.h> -#include <crypto/hash.h> -#include <crypto/sha.h> -#include <crypto/authenc.h> -#include <crypto/ctr.h> -#include <crypto/gf128mul.h> -#include <crypto/internal/aead.h> -#include <crypto/null.h> -#include <crypto/internal/skcipher.h> -#include <crypto/aead.h> -#include <crypto/scatterwalk.h> -#include <crypto/internal/hash.h> -#include <linux/tls.h> -#include <net/tls.h> -#include <net/tls_toe.h> - -#include "t4fw_api.h" -#include "t4_msg.h" -#include "cxgb4.h" -#include "cxgb4_uld.h" -#include "l2t.h" -#include "chcr_algo.h" -#include "chcr_core.h" -#include "chcr_crypto.h" - -#define MAX_IVS_PAGE 256 -#define TLS_KEY_CONTEXT_SZ 64 -#define CIPHER_BLOCK_SIZE 16 -#define GCM_TAG_SIZE 16 -#define KEY_ON_MEM_SZ 16 -#define AEAD_EXPLICIT_DATA_SIZE 8 -#define TLS_HEADER_LENGTH 5 -#define SCMD_CIPH_MODE_AES_GCM 2 -/* Any MFS size should work and come from openssl */ -#define TLS_MFS 16384 - -#define RSS_HDR sizeof(struct rss_header) -#define TLS_WR_CPL_LEN \ - (sizeof(struct fw_tlstx_data_wr) + sizeof(struct cpl_tx_tls_sfo)) - -enum { - CHTLS_KEY_CONTEXT_DSGL, - CHTLS_KEY_CONTEXT_IMM, - CHTLS_KEY_CONTEXT_DDR, -}; - -enum { - CHTLS_LISTEN_START, - CHTLS_LISTEN_STOP, -}; - -/* Flags for return value of CPL message handlers */ -enum { - CPL_RET_BUF_DONE = 1, /* buffer processing done */ - CPL_RET_BAD_MSG = 2, /* bad CPL message */ - CPL_RET_UNKNOWN_TID = 4 /* unexpected unknown TID */ -}; - -#define LISTEN_INFO_HASH_SIZE 32 -#define RSPQ_HASH_BITS 5 -struct listen_info { - struct listen_info *next; /* Link to next entry */ - struct sock *sk; /* The listening socket */ - unsigned int stid; /* The server TID */ -}; - -enum { - T4_LISTEN_START_PENDING, - T4_LISTEN_STARTED -}; - -enum csk_flags { - CSK_CALLBACKS_CHKD, /* socket callbacks have been sanitized */ - CSK_ABORT_REQ_RCVD, /* received one ABORT_REQ_RSS message */ - CSK_TX_MORE_DATA, /* sending ULP data; don't set SHOVE bit */ - CSK_TX_WAIT_IDLE, /* suspend Tx until in-flight data is ACKed */ - CSK_ABORT_SHUTDOWN, /* shouldn't send more abort requests */ - CSK_ABORT_RPL_PENDING, /* expecting an abort reply */ - CSK_CLOSE_CON_REQUESTED,/* we've sent a close_conn_req */ - CSK_TX_DATA_SENT, /* sent a TX_DATA WR on this connection */ - CSK_TX_FAILOVER, /* Tx traffic failing over */ - CSK_UPDATE_RCV_WND, /* Need to update rcv window */ - CSK_RST_ABORTED, /* outgoing RST was aborted */ - CSK_TLS_HANDSHK, /* TLS Handshake */ - CSK_CONN_INLINE, /* Connection on HW */ -}; - -enum chtls_cdev_state { - CHTLS_CDEV_STATE_UP = 1 -}; - -struct listen_ctx { - struct sock *lsk; - struct chtls_dev *cdev; - struct sk_buff_head synq; - u32 state; -}; - -struct key_map { - unsigned long *addr; - unsigned int start; - unsigned int available; - unsigned int size; - spinlock_t lock; /* lock for key id request from map */ -} __packed; - -struct tls_scmd { - u32 seqno_numivs; - u32 ivgen_hdrlen; -}; - -struct chtls_dev { - struct tls_toe_device tlsdev; - struct list_head list; - struct cxgb4_lld_info *lldi; - struct pci_dev *pdev; - struct listen_info *listen_hash_tab[LISTEN_INFO_HASH_SIZE]; - spinlock_t listen_lock; /* lock for listen list */ - struct net_device **ports; - struct tid_info *tids; - unsigned int pfvf; - const unsigned short *mtus; - - struct idr hwtid_idr; - struct idr stid_idr; - - spinlock_t idr_lock ____cacheline_aligned_in_smp; - - struct net_device *egr_dev[NCHAN * 2]; - struct sk_buff *rspq_skb_cache[1 << RSPQ_HASH_BITS]; - struct sk_buff *askb; - - struct sk_buff_head deferq; - struct work_struct deferq_task; - - struct list_head list_node; - struct list_head rcu_node; - struct list_head na_node; - unsigned int send_page_order; - int max_host_sndbuf; - struct key_map kmap; - unsigned int cdev_state; -}; - -struct chtls_listen { - struct chtls_dev *cdev; - struct sock *sk; -}; - -struct chtls_hws { - struct sk_buff_head sk_recv_queue; - u8 txqid; - u8 ofld; - u16 type; - u16 rstate; - u16 keyrpl; - u16 pldlen; - u16 rcvpld; - u16 compute; - u16 expansion; - u16 keylen; - u16 pdus; - u16 adjustlen; - u16 ivsize; - u16 txleft; - u32 mfs; - s32 txkey; - s32 rxkey; - u32 fcplenmax; - u32 copied_seq; - u64 tx_seq_no; - struct tls_scmd scmd; - union { - struct tls12_crypto_info_aes_gcm_128 aes_gcm_128; - struct tls12_crypto_info_aes_gcm_256 aes_gcm_256; - } crypto_info; -}; - -struct chtls_sock { - struct sock *sk; - struct chtls_dev *cdev; - struct l2t_entry *l2t_entry; /* pointer to the L2T entry */ - struct net_device *egress_dev; /* TX_CHAN for act open retry */ - - struct sk_buff_head txq; - struct sk_buff *wr_skb_head; - struct sk_buff *wr_skb_tail; - struct sk_buff *ctrl_skb_cache; - struct sk_buff *txdata_skb_cache; /* abort path messages */ - struct kref kref; - unsigned long flags; - u32 opt2; - u32 wr_credits; - u32 wr_unacked; - u32 wr_max_credits; - u32 wr_nondata; - u32 hwtid; /* TCP Control Block ID */ - u32 txq_idx; - u32 rss_qid; - u32 tid; - u32 idr; - u32 mss; - u32 ulp_mode; - u32 tx_chan; - u32 rx_chan; - u32 sndbuf; - u32 txplen_max; - u32 mtu_idx; /* MTU table index */ - u32 smac_idx; - u8 port_id; - u8 tos; - u16 resv2; - u32 delack_mode; - u32 delack_seq; - u32 snd_win; - u32 rcv_win; - - void *passive_reap_next; /* placeholder for passive */ - struct chtls_hws tlshws; - struct synq { - struct sk_buff *next; - struct sk_buff *prev; - } synq; - struct listen_ctx *listen_ctx; -}; - -struct tls_hdr { - u8 type; - u16 version; - u16 length; -} __packed; - -struct tlsrx_cmp_hdr { - u8 type; - u16 version; - u16 length; - - u64 tls_seq; - u16 reserved1; - u8 res_to_mac_error; -} __packed; - -/* res_to_mac_error fields */ -#define TLSRX_HDR_PKT_INT_ERROR_S 4 -#define TLSRX_HDR_PKT_INT_ERROR_M 0x1 -#define TLSRX_HDR_PKT_INT_ERROR_V(x) \ - ((x) << TLSRX_HDR_PKT_INT_ERROR_S) -#define TLSRX_HDR_PKT_INT_ERROR_G(x) \ - (((x) >> TLSRX_HDR_PKT_INT_ERROR_S) & TLSRX_HDR_PKT_INT_ERROR_M) -#define TLSRX_HDR_PKT_INT_ERROR_F TLSRX_HDR_PKT_INT_ERROR_V(1U) - -#define TLSRX_HDR_PKT_SPP_ERROR_S 3 -#define TLSRX_HDR_PKT_SPP_ERROR_M 0x1 -#define TLSRX_HDR_PKT_SPP_ERROR_V(x) ((x) << TLSRX_HDR_PKT_SPP_ERROR) -#define TLSRX_HDR_PKT_SPP_ERROR_G(x) \ - (((x) >> TLSRX_HDR_PKT_SPP_ERROR_S) & TLSRX_HDR_PKT_SPP_ERROR_M) -#define TLSRX_HDR_PKT_SPP_ERROR_F TLSRX_HDR_PKT_SPP_ERROR_V(1U) - -#define TLSRX_HDR_PKT_CCDX_ERROR_S 2 -#define TLSRX_HDR_PKT_CCDX_ERROR_M 0x1 -#define TLSRX_HDR_PKT_CCDX_ERROR_V(x) ((x) << TLSRX_HDR_PKT_CCDX_ERROR_S) -#define TLSRX_HDR_PKT_CCDX_ERROR_G(x) \ - (((x) >> TLSRX_HDR_PKT_CCDX_ERROR_S) & TLSRX_HDR_PKT_CCDX_ERROR_M) -#define TLSRX_HDR_PKT_CCDX_ERROR_F TLSRX_HDR_PKT_CCDX_ERROR_V(1U) - -#define TLSRX_HDR_PKT_PAD_ERROR_S 1 -#define TLSRX_HDR_PKT_PAD_ERROR_M 0x1 -#define TLSRX_HDR_PKT_PAD_ERROR_V(x) ((x) << TLSRX_HDR_PKT_PAD_ERROR_S) -#define TLSRX_HDR_PKT_PAD_ERROR_G(x) \ - (((x) >> TLSRX_HDR_PKT_PAD_ERROR_S) & TLSRX_HDR_PKT_PAD_ERROR_M) -#define TLSRX_HDR_PKT_PAD_ERROR_F TLSRX_HDR_PKT_PAD_ERROR_V(1U) - -#define TLSRX_HDR_PKT_MAC_ERROR_S 0 -#define TLSRX_HDR_PKT_MAC_ERROR_M 0x1 -#define TLSRX_HDR_PKT_MAC_ERROR_V(x) ((x) << TLSRX_HDR_PKT_MAC_ERROR) -#define TLSRX_HDR_PKT_MAC_ERROR_G(x) \ - (((x) >> S_TLSRX_HDR_PKT_MAC_ERROR_S) & TLSRX_HDR_PKT_MAC_ERROR_M) -#define TLSRX_HDR_PKT_MAC_ERROR_F TLSRX_HDR_PKT_MAC_ERROR_V(1U) - -#define TLSRX_HDR_PKT_ERROR_M 0x1F -#define CONTENT_TYPE_ERROR 0x7F - -struct ulp_mem_rw { - __be32 cmd; - __be32 len16; /* command length */ - __be32 dlen; /* data length in 32-byte units */ - __be32 lock_addr; -}; - -struct tls_key_wr { - __be32 op_to_compl; - __be32 flowid_len16; - __be32 ftid; - u8 reneg_to_write_rx; - u8 protocol; - __be16 mfs; -}; - -struct tls_key_req { - struct tls_key_wr wr; - struct ulp_mem_rw req; - struct ulptx_idata sc_imm; -}; - -/* - * This lives in skb->cb and is used to chain WRs in a linked list. - */ -struct wr_skb_cb { - struct l2t_skb_cb l2t; /* reserve space for l2t CB */ - struct sk_buff *next_wr; /* next write request */ -}; - -/* Per-skb backlog handler. Run when a socket's backlog is processed. */ -struct blog_skb_cb { - void (*backlog_rcv)(struct sock *sk, struct sk_buff *skb); - struct chtls_dev *cdev; -}; - -/* - * Similar to tcp_skb_cb but with ULP elements added to support TLS, - * etc. - */ -struct ulp_skb_cb { - struct wr_skb_cb wr; /* reserve space for write request */ - u16 flags; /* TCP-like flags */ - u8 psh; - u8 ulp_mode; /* ULP mode/submode of sk_buff */ - u32 seq; /* TCP sequence number */ - union { /* ULP-specific fields */ - struct { - u8 type; - u8 ofld; - u8 iv; - } tls; - } ulp; -}; - -#define ULP_SKB_CB(skb) ((struct ulp_skb_cb *)&((skb)->cb[0])) -#define BLOG_SKB_CB(skb) ((struct blog_skb_cb *)(skb)->cb) - -/* - * Flags for ulp_skb_cb.flags. - */ -enum { - ULPCB_FLAG_NEED_HDR = 1 << 0, /* packet needs a TX_DATA_WR header */ - ULPCB_FLAG_NO_APPEND = 1 << 1, /* don't grow this skb */ - ULPCB_FLAG_BARRIER = 1 << 2, /* set TX_WAIT_IDLE after sending */ - ULPCB_FLAG_HOLD = 1 << 3, /* skb not ready for Tx yet */ - ULPCB_FLAG_COMPL = 1 << 4, /* request WR completion */ - ULPCB_FLAG_URG = 1 << 5, /* urgent data */ - ULPCB_FLAG_TLS_HDR = 1 << 6, /* payload with tls hdr */ - ULPCB_FLAG_NO_HDR = 1 << 7, /* not a ofld wr */ -}; - -/* The ULP mode/submode of an skbuff */ -#define skb_ulp_mode(skb) (ULP_SKB_CB(skb)->ulp_mode) -#define TCP_PAGE(sk) (sk->sk_frag.page) -#define TCP_OFF(sk) (sk->sk_frag.offset) - -static inline struct chtls_dev *to_chtls_dev(struct tls_toe_device *tlsdev) -{ - return container_of(tlsdev, struct chtls_dev, tlsdev); -} - -static inline void csk_set_flag(struct chtls_sock *csk, - enum csk_flags flag) -{ - __set_bit(flag, &csk->flags); -} - -static inline void csk_reset_flag(struct chtls_sock *csk, - enum csk_flags flag) -{ - __clear_bit(flag, &csk->flags); -} - -static inline bool csk_conn_inline(const struct chtls_sock *csk) -{ - return test_bit(CSK_CONN_INLINE, &csk->flags); -} - -static inline int csk_flag(const struct sock *sk, enum csk_flags flag) -{ - struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); - - if (!csk_conn_inline(csk)) - return 0; - return test_bit(flag, &csk->flags); -} - -static inline int csk_flag_nochk(const struct chtls_sock *csk, - enum csk_flags flag) -{ - return test_bit(flag, &csk->flags); -} - -static inline void *cplhdr(struct sk_buff *skb) -{ - return skb->data; -} - -static inline int is_neg_adv(unsigned int status) -{ - return status == CPL_ERR_RTX_NEG_ADVICE || - status == CPL_ERR_KEEPALV_NEG_ADVICE || - status == CPL_ERR_PERSIST_NEG_ADVICE; -} - -static inline void process_cpl_msg(void (*fn)(struct sock *, struct sk_buff *), - struct sock *sk, - struct sk_buff *skb) -{ - skb_reset_mac_header(skb); - skb_reset_network_header(skb); - skb_reset_transport_header(skb); - - bh_lock_sock(sk); - if (unlikely(sock_owned_by_user(sk))) { - BLOG_SKB_CB(skb)->backlog_rcv = fn; - __sk_add_backlog(sk, skb); - } else { - fn(sk, skb); - } - bh_unlock_sock(sk); -} - -static inline void chtls_sock_free(struct kref *ref) -{ - struct chtls_sock *csk = container_of(ref, struct chtls_sock, - kref); - kfree(csk); -} - -static inline void __chtls_sock_put(const char *fn, struct chtls_sock *csk) -{ - kref_put(&csk->kref, chtls_sock_free); -} - -static inline void __chtls_sock_get(const char *fn, - struct chtls_sock *csk) -{ - kref_get(&csk->kref); -} - -static inline void send_or_defer(struct sock *sk, struct tcp_sock *tp, - struct sk_buff *skb, int through_l2t) -{ - struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); - - if (through_l2t) { - /* send through L2T */ - cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry); - } else { - /* send directly */ - cxgb4_ofld_send(csk->egress_dev, skb); - } -} - -typedef int (*chtls_handler_func)(struct chtls_dev *, struct sk_buff *); -extern chtls_handler_func chtls_handlers[NUM_CPL_CMDS]; -void chtls_install_cpl_ops(struct sock *sk); -int chtls_init_kmap(struct chtls_dev *cdev, struct cxgb4_lld_info *lldi); -void chtls_listen_stop(struct chtls_dev *cdev, struct sock *sk); -int chtls_listen_start(struct chtls_dev *cdev, struct sock *sk); -void chtls_close(struct sock *sk, long timeout); -int chtls_disconnect(struct sock *sk, int flags); -void chtls_shutdown(struct sock *sk, int how); -void chtls_destroy_sock(struct sock *sk); -int chtls_sendmsg(struct sock *sk, struct msghdr *msg, size_t size); -int chtls_recvmsg(struct sock *sk, struct msghdr *msg, - size_t len, int nonblock, int flags, int *addr_len); -int chtls_sendpage(struct sock *sk, struct page *page, - int offset, size_t size, int flags); -int send_tx_flowc_wr(struct sock *sk, int compl, - u32 snd_nxt, u32 rcv_nxt); -void chtls_tcp_push(struct sock *sk, int flags); -int chtls_push_frames(struct chtls_sock *csk, int comp); -int chtls_set_tcb_tflag(struct sock *sk, unsigned int bit_pos, int val); -int chtls_setkey(struct chtls_sock *csk, u32 keylen, u32 mode, int cipher_type); -void skb_entail(struct sock *sk, struct sk_buff *skb, int flags); -unsigned int keyid_to_addr(int start_addr, int keyid); -void free_tls_keyid(struct sock *sk); -#endif diff --git a/drivers/crypto/chelsio/chtls/chtls_cm.c b/drivers/crypto/chelsio/chtls/chtls_cm.c deleted file mode 100644 index 05520dccd906..000000000000 --- a/drivers/crypto/chelsio/chtls/chtls_cm.c +++ /dev/null @@ -1,2327 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (c) 2018 Chelsio Communications, Inc. - * - * Written by: Atul Gupta ([email protected]) - */ - -#include <linux/module.h> -#include <linux/list.h> -#include <linux/workqueue.h> -#include <linux/skbuff.h> -#include <linux/timer.h> -#include <linux/notifier.h> -#include <linux/inetdevice.h> -#include <linux/ip.h> -#include <linux/tcp.h> -#include <linux/sched/signal.h> -#include <linux/kallsyms.h> -#include <linux/kprobes.h> -#include <linux/if_vlan.h> -#include <linux/ipv6.h> -#include <net/ipv6.h> -#include <net/transp_v6.h> -#include <net/ip6_route.h> -#include <net/inet_common.h> -#include <net/tcp.h> -#include <net/dst.h> -#include <net/tls.h> -#include <net/addrconf.h> -#include <net/secure_seq.h> - -#include "chtls.h" -#include "chtls_cm.h" -#include "clip_tbl.h" - -/* - * State transitions and actions for close. Note that if we are in SYN_SENT - * we remain in that state as we cannot control a connection while it's in - * SYN_SENT; such connections are allowed to establish and are then aborted. - */ -static unsigned char new_state[16] = { - /* current state: new state: action: */ - /* (Invalid) */ TCP_CLOSE, - /* TCP_ESTABLISHED */ TCP_FIN_WAIT1 | TCP_ACTION_FIN, - /* TCP_SYN_SENT */ TCP_SYN_SENT, - /* TCP_SYN_RECV */ TCP_FIN_WAIT1 | TCP_ACTION_FIN, - /* TCP_FIN_WAIT1 */ TCP_FIN_WAIT1, - /* TCP_FIN_WAIT2 */ TCP_FIN_WAIT2, - /* TCP_TIME_WAIT */ TCP_CLOSE, - /* TCP_CLOSE */ TCP_CLOSE, - /* TCP_CLOSE_WAIT */ TCP_LAST_ACK | TCP_ACTION_FIN, - /* TCP_LAST_ACK */ TCP_LAST_ACK, - /* TCP_LISTEN */ TCP_CLOSE, - /* TCP_CLOSING */ TCP_CLOSING, -}; - -static struct chtls_sock *chtls_sock_create(struct chtls_dev *cdev) -{ - struct chtls_sock *csk = kzalloc(sizeof(*csk), GFP_ATOMIC); - - if (!csk) - return NULL; - - csk->txdata_skb_cache = alloc_skb(TXDATA_SKB_LEN, GFP_ATOMIC); - if (!csk->txdata_skb_cache) { - kfree(csk); - return NULL; - } - - kref_init(&csk->kref); - csk->cdev = cdev; - skb_queue_head_init(&csk->txq); - csk->wr_skb_head = NULL; - csk->wr_skb_tail = NULL; - csk->mss = MAX_MSS; - csk->tlshws.ofld = 1; - csk->tlshws.txkey = -1; - csk->tlshws.rxkey = -1; - csk->tlshws.mfs = TLS_MFS; - skb_queue_head_init(&csk->tlshws.sk_recv_queue); - return csk; -} - -static void chtls_sock_release(struct kref *ref) -{ - struct chtls_sock *csk = - container_of(ref, struct chtls_sock, kref); - - kfree(csk); -} - -static struct net_device *chtls_find_netdev(struct chtls_dev *cdev, - struct sock *sk) -{ - struct net_device *ndev = cdev->ports[0]; -#if IS_ENABLED(CONFIG_IPV6) - struct net_device *temp; - int addr_type; -#endif - - switch (sk->sk_family) { - case PF_INET: - if (likely(!inet_sk(sk)->inet_rcv_saddr)) - return ndev; - ndev = __ip_dev_find(&init_net, inet_sk(sk)->inet_rcv_saddr, false); - break; -#if IS_ENABLED(CONFIG_IPV6) - case PF_INET6: - addr_type = ipv6_addr_type(&sk->sk_v6_rcv_saddr); - if (likely(addr_type == IPV6_ADDR_ANY)) - return ndev; - - for_each_netdev_rcu(&init_net, temp) { - if (ipv6_chk_addr(&init_net, (struct in6_addr *) - &sk->sk_v6_rcv_saddr, temp, 1)) { - ndev = temp; - break; - } - } - break; -#endif - default: - return NULL; - } - - if (!ndev) - return NULL; - - if (is_vlan_dev(ndev)) - return vlan_dev_real_dev(ndev); - return ndev; -} - -static void assign_rxopt(struct sock *sk, unsigned int opt) -{ - const struct chtls_dev *cdev; - struct chtls_sock *csk; - struct tcp_sock *tp; - - csk = rcu_dereference_sk_user_data(sk); - tp = tcp_sk(sk); - - cdev = csk->cdev; - tp->tcp_header_len = sizeof(struct tcphdr); - tp->rx_opt.mss_clamp = cdev->mtus[TCPOPT_MSS_G(opt)] - 40; - tp->mss_cache = tp->rx_opt.mss_clamp; - tp->rx_opt.tstamp_ok = TCPOPT_TSTAMP_G(opt); - tp->rx_opt.snd_wscale = TCPOPT_SACK_G(opt); - tp->rx_opt.wscale_ok = TCPOPT_WSCALE_OK_G(opt); - SND_WSCALE(tp) = TCPOPT_SND_WSCALE_G(opt); - if (!tp->rx_opt.wscale_ok) - tp->rx_opt.rcv_wscale = 0; - if (tp->rx_opt.tstamp_ok) { - tp->tcp_header_len += TCPOLEN_TSTAMP_ALIGNED; - tp->rx_opt.mss_clamp -= TCPOLEN_TSTAMP_ALIGNED; - } else if (csk->opt2 & TSTAMPS_EN_F) { - csk->opt2 &= ~TSTAMPS_EN_F; - csk->mtu_idx = TCPOPT_MSS_G(opt); - } -} - -static void chtls_purge_receive_queue(struct sock *sk) -{ - struct sk_buff *skb; - - while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) { - skb_dst_set(skb, (void *)NULL); - kfree_skb(skb); - } -} - -static void chtls_purge_write_queue(struct sock *sk) -{ - struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); - struct sk_buff *skb; - - while ((skb = __skb_dequeue(&csk->txq))) { - sk->sk_wmem_queued -= skb->truesize; - __kfree_skb(skb); - } -} - -static void chtls_purge_recv_queue(struct sock *sk) -{ - struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); - struct chtls_hws *tlsk = &csk->tlshws; - struct sk_buff *skb; - - while ((skb = __skb_dequeue(&tlsk->sk_recv_queue)) != NULL) { - skb_dst_set(skb, NULL); - kfree_skb(skb); - } -} - -static void abort_arp_failure(void *handle, struct sk_buff *skb) -{ - struct cpl_abort_req *req = cplhdr(skb); - struct chtls_dev *cdev; - - cdev = (struct chtls_dev *)handle; - req->cmd = CPL_ABORT_NO_RST; - cxgb4_ofld_send(cdev->lldi->ports[0], skb); -} - -static struct sk_buff *alloc_ctrl_skb(struct sk_buff *skb, int len) -{ - if (likely(skb && !skb_shared(skb) && !skb_cloned(skb))) { - __skb_trim(skb, 0); - refcount_add(2, &skb->users); - } else { - skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL); - } - return skb; -} - -static void chtls_send_abort(struct sock *sk, int mode, struct sk_buff *skb) -{ - struct cpl_abort_req *req; - struct chtls_sock *csk; - struct tcp_sock *tp; - - csk = rcu_dereference_sk_user_data(sk); - tp = tcp_sk(sk); - - if (!skb) - skb = alloc_ctrl_skb(csk->txdata_skb_cache, sizeof(*req)); - - req = (struct cpl_abort_req *)skb_put(skb, sizeof(*req)); - INIT_TP_WR_CPL(req, CPL_ABORT_REQ, csk->tid); - skb_set_queue_mapping(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA); - req->rsvd0 = htonl(tp->snd_nxt); - req->rsvd1 = !csk_flag_nochk(csk, CSK_TX_DATA_SENT); - req->cmd = mode; - t4_set_arp_err_handler(skb, csk->cdev, abort_arp_failure); - send_or_defer(sk, tp, skb, mode == CPL_ABORT_SEND_RST); -} - -static void chtls_send_reset(struct sock *sk, int mode, struct sk_buff *skb) -{ - struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); - - if (unlikely(csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN) || - !csk->cdev)) { - if (sk->sk_state == TCP_SYN_RECV) - csk_set_flag(csk, CSK_RST_ABORTED); - goto out; - } - - if (!csk_flag_nochk(csk, CSK_TX_DATA_SENT)) { - struct tcp_sock *tp = tcp_sk(sk); - - if (send_tx_flowc_wr(sk, 0, tp->snd_nxt, tp->rcv_nxt) < 0) - WARN_ONCE(1, "send tx flowc error"); - csk_set_flag(csk, CSK_TX_DATA_SENT); - } - - csk_set_flag(csk, CSK_ABORT_RPL_PENDING); - chtls_purge_write_queue(sk); - - csk_set_flag(csk, CSK_ABORT_SHUTDOWN); - if (sk->sk_state != TCP_SYN_RECV) - chtls_send_abort(sk, mode, skb); - else - goto out; - - return; -out: - kfree_skb(skb); -} - -static void release_tcp_port(struct sock *sk) -{ - if (inet_csk(sk)->icsk_bind_hash) - inet_put_port(sk); -} - -static void tcp_uncork(struct sock *sk) -{ - struct tcp_sock *tp = tcp_sk(sk); - - if (tp->nonagle & TCP_NAGLE_CORK) { - tp->nonagle &= ~TCP_NAGLE_CORK; - chtls_tcp_push(sk, 0); - } -} - -static void chtls_close_conn(struct sock *sk) -{ - struct cpl_close_con_req *req; - struct chtls_sock *csk; - struct sk_buff *skb; - unsigned int tid; - unsigned int len; - - len = roundup(sizeof(struct cpl_close_con_req), 16); - csk = rcu_dereference_sk_user_data(sk); - tid = csk->tid; - - skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL); - req = (struct cpl_close_con_req *)__skb_put(skb, len); - memset(req, 0, len); - req->wr.wr_hi = htonl(FW_WR_OP_V(FW_TP_WR) | - FW_WR_IMMDLEN_V(sizeof(*req) - - sizeof(req->wr))); - req->wr.wr_mid = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*req), 16)) | - FW_WR_FLOWID_V(tid)); - - OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid)); - - tcp_uncork(sk); - skb_entail(sk, skb, ULPCB_FLAG_NO_HDR | ULPCB_FLAG_NO_APPEND); - if (sk->sk_state != TCP_SYN_SENT) - chtls_push_frames(csk, 1); -} - -/* - * Perform a state transition during close and return the actions indicated - * for the transition. Do not make this function inline, the main reason - * it exists at all is to avoid multiple inlining of tcp_set_state. - */ -static int make_close_transition(struct sock *sk) -{ - int next = (int)new_state[sk->sk_state]; - - tcp_set_state(sk, next & TCP_STATE_MASK); - return next & TCP_ACTION_FIN; -} - -void chtls_close(struct sock *sk, long timeout) -{ - int data_lost, prev_state; - struct chtls_sock *csk; - - csk = rcu_dereference_sk_user_data(sk); - - lock_sock(sk); - sk->sk_shutdown |= SHUTDOWN_MASK; - - data_lost = skb_queue_len(&sk->sk_receive_queue); - data_lost |= skb_queue_len(&csk->tlshws.sk_recv_queue); - chtls_purge_recv_queue(sk); - chtls_purge_receive_queue(sk); - - if (sk->sk_state == TCP_CLOSE) { - goto wait; - } else if (data_lost || sk->sk_state == TCP_SYN_SENT) { - chtls_send_reset(sk, CPL_ABORT_SEND_RST, NULL); - release_tcp_port(sk); - goto unlock; - } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) { - sk->sk_prot->disconnect(sk, 0); - } else if (make_close_transition(sk)) { - chtls_close_conn(sk); - } -wait: - if (timeout) - sk_stream_wait_close(sk, timeout); - -unlock: - prev_state = sk->sk_state; - sock_hold(sk); - sock_orphan(sk); - - release_sock(sk); - - local_bh_disable(); - bh_lock_sock(sk); - - if (prev_state != TCP_CLOSE && sk->sk_state == TCP_CLOSE) - goto out; - - if (sk->sk_state == TCP_FIN_WAIT2 && tcp_sk(sk)->linger2 < 0 && - !csk_flag(sk, CSK_ABORT_SHUTDOWN)) { - struct sk_buff *skb; - - skb = alloc_skb(sizeof(struct cpl_abort_req), GFP_ATOMIC); - if (skb) - chtls_send_reset(sk, CPL_ABORT_SEND_RST, skb); - } - - if (sk->sk_state == TCP_CLOSE) - inet_csk_destroy_sock(sk); - -out: - bh_unlock_sock(sk); - local_bh_enable(); - sock_put(sk); -} - -/* - * Wait until a socket enters on of the given states. - */ -static int wait_for_states(struct sock *sk, unsigned int states) -{ - DECLARE_WAITQUEUE(wait, current); - struct socket_wq _sk_wq; - long current_timeo; - int err = 0; - - current_timeo = 200; - - /* - * We want this to work even when there's no associated struct socket. - * In that case we provide a temporary wait_queue_head_t. - */ - if (!sk->sk_wq) { - init_waitqueue_head(&_sk_wq.wait); - _sk_wq.fasync_list = NULL; - init_rcu_head_on_stack(&_sk_wq.rcu); - RCU_INIT_POINTER(sk->sk_wq, &_sk_wq); - } - - add_wait_queue(sk_sleep(sk), &wait); - while (!sk_in_state(sk, states)) { - if (!current_timeo) { - err = -EBUSY; - break; - } - if (signal_pending(current)) { - err = sock_intr_errno(current_timeo); - break; - } - set_current_state(TASK_UNINTERRUPTIBLE); - release_sock(sk); - if (!sk_in_state(sk, states)) - current_timeo = schedule_timeout(current_timeo); - __set_current_state(TASK_RUNNING); - lock_sock(sk); - } - remove_wait_queue(sk_sleep(sk), &wait); - - if (rcu_dereference(sk->sk_wq) == &_sk_wq) - sk->sk_wq = NULL; - return err; -} - -int chtls_disconnect(struct sock *sk, int flags) -{ - struct tcp_sock *tp; - int err; - - tp = tcp_sk(sk); - chtls_purge_recv_queue(sk); - chtls_purge_receive_queue(sk); - chtls_purge_write_queue(sk); - - if (sk->sk_state != TCP_CLOSE) { - sk->sk_err = ECONNRESET; - chtls_send_reset(sk, CPL_ABORT_SEND_RST, NULL); - err = wait_for_states(sk, TCPF_CLOSE); - if (err) - return err; - } - chtls_purge_recv_queue(sk); - chtls_purge_receive_queue(sk); - tp->max_window = 0xFFFF << (tp->rx_opt.snd_wscale); - return tcp_disconnect(sk, flags); -} - -#define SHUTDOWN_ELIGIBLE_STATE (TCPF_ESTABLISHED | \ - TCPF_SYN_RECV | TCPF_CLOSE_WAIT) -void chtls_shutdown(struct sock *sk, int how) -{ - if ((how & SEND_SHUTDOWN) && - sk_in_state(sk, SHUTDOWN_ELIGIBLE_STATE) && - make_close_transition(sk)) - chtls_close_conn(sk); -} - -void chtls_destroy_sock(struct sock *sk) -{ - struct chtls_sock *csk; - - csk = rcu_dereference_sk_user_data(sk); - chtls_purge_recv_queue(sk); - csk->ulp_mode = ULP_MODE_NONE; - chtls_purge_write_queue(sk); - free_tls_keyid(sk); - kref_put(&csk->kref, chtls_sock_release); - csk->cdev = NULL; - if (sk->sk_family == AF_INET) - sk->sk_prot = &tcp_prot; -#if IS_ENABLED(CONFIG_IPV6) - else - sk->sk_prot = &tcpv6_prot; -#endif - sk->sk_prot->destroy(sk); -} - -static void reset_listen_child(struct sock *child) -{ - struct chtls_sock *csk = rcu_dereference_sk_user_data(child); - struct sk_buff *skb; - - skb = alloc_ctrl_skb(csk->txdata_skb_cache, - sizeof(struct cpl_abort_req)); - - chtls_send_reset(child, CPL_ABORT_SEND_RST, skb); - sock_orphan(child); - INC_ORPHAN_COUNT(child); - if (child->sk_state == TCP_CLOSE) - inet_csk_destroy_sock(child); -} - -static void chtls_disconnect_acceptq(struct sock *listen_sk) -{ - struct request_sock **pprev; - - pprev = ACCEPT_QUEUE(listen_sk); - while (*pprev) { - struct request_sock *req = *pprev; - - if (req->rsk_ops == &chtls_rsk_ops || - req->rsk_ops == &chtls_rsk_opsv6) { - struct sock *child = req->sk; - - *pprev = req->dl_next; - sk_acceptq_removed(listen_sk); - reqsk_put(req); - sock_hold(child); - local_bh_disable(); - bh_lock_sock(child); - release_tcp_port(child); - reset_listen_child(child); - bh_unlock_sock(child); - local_bh_enable(); - sock_put(child); - } else { - pprev = &req->dl_next; - } - } -} - -static int listen_hashfn(const struct sock *sk) -{ - return ((unsigned long)sk >> 10) & (LISTEN_INFO_HASH_SIZE - 1); -} - -static struct listen_info *listen_hash_add(struct chtls_dev *cdev, - struct sock *sk, - unsigned int stid) -{ - struct listen_info *p = kmalloc(sizeof(*p), GFP_KERNEL); - - if (p) { - int key = listen_hashfn(sk); - - p->sk = sk; - p->stid = stid; - spin_lock(&cdev->listen_lock); - p->next = cdev->listen_hash_tab[key]; - cdev->listen_hash_tab[key] = p; - spin_unlock(&cdev->listen_lock); - } - return p; -} - -static int listen_hash_find(struct chtls_dev *cdev, - struct sock *sk) -{ - struct listen_info *p; - int stid = -1; - int key; - - key = listen_hashfn(sk); - - spin_lock(&cdev->listen_lock); - for (p = cdev->listen_hash_tab[key]; p; p = p->next) - if (p->sk == sk) { - stid = p->stid; - break; - } - spin_unlock(&cdev->listen_lock); - return stid; -} - -static int listen_hash_del(struct chtls_dev *cdev, - struct sock *sk) -{ - struct listen_info *p, **prev; - int stid = -1; - int key; - - key = listen_hashfn(sk); - prev = &cdev->listen_hash_tab[key]; - - spin_lock(&cdev->listen_lock); - for (p = *prev; p; prev = &p->next, p = p->next) - if (p->sk == sk) { - stid = p->stid; - *prev = p->next; - kfree(p); - break; - } - spin_unlock(&cdev->listen_lock); - return stid; -} - -static void cleanup_syn_rcv_conn(struct sock *child, struct sock *parent) -{ - struct request_sock *req; - struct chtls_sock *csk; - - csk = rcu_dereference_sk_user_data(child); - req = csk->passive_reap_next; - - reqsk_queue_removed(&inet_csk(parent)->icsk_accept_queue, req); - __skb_unlink((struct sk_buff *)&csk->synq, &csk->listen_ctx->synq); - chtls_reqsk_free(req); - csk->passive_reap_next = NULL; -} - -static void chtls_reset_synq(struct listen_ctx *listen_ctx) -{ - struct sock *listen_sk = listen_ctx->lsk; - - while (!skb_queue_empty(&listen_ctx->synq)) { - struct chtls_sock *csk = - container_of((struct synq *)__skb_dequeue - (&listen_ctx->synq), struct chtls_sock, synq); - struct sock *child = csk->sk; - - cleanup_syn_rcv_conn(child, listen_sk); - sock_hold(child); - local_bh_disable(); - bh_lock_sock(child); - release_tcp_port(child); - reset_listen_child(child); - bh_unlock_sock(child); - local_bh_enable(); - sock_put(child); - } -} - -int chtls_listen_start(struct chtls_dev *cdev, struct sock *sk) -{ - struct net_device *ndev; -#if IS_ENABLED(CONFIG_IPV6) - bool clip_valid = false; -#endif - struct listen_ctx *ctx; - struct adapter *adap; - struct port_info *pi; - int ret = 0; - int stid; - - rcu_read_lock(); - ndev = chtls_find_netdev(cdev, sk); - rcu_read_unlock(); - if (!ndev) - return -EBADF; - - pi = netdev_priv(ndev); - adap = pi->adapter; - if (!(adap->flags & CXGB4_FULL_INIT_DONE)) - return -EBADF; - - if (listen_hash_find(cdev, sk) >= 0) /* already have it */ - return -EADDRINUSE; - - ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); - if (!ctx) - return -ENOMEM; - - __module_get(THIS_MODULE); - ctx->lsk = sk; - ctx->cdev = cdev; - ctx->state = T4_LISTEN_START_PENDING; - skb_queue_head_init(&ctx->synq); - - stid = cxgb4_alloc_stid(cdev->tids, sk->sk_family, ctx); - if (stid < 0) - goto free_ctx; - - sock_hold(sk); - if (!listen_hash_add(cdev, sk, stid)) - goto free_stid; - - if (sk->sk_family == PF_INET) { - ret = cxgb4_create_server(ndev, stid, - inet_sk(sk)->inet_rcv_saddr, - inet_sk(sk)->inet_sport, 0, - cdev->lldi->rxq_ids[0]); -#if IS_ENABLED(CONFIG_IPV6) - } else { - int addr_type; - - addr_type = ipv6_addr_type(&sk->sk_v6_rcv_saddr); - if (addr_type != IPV6_ADDR_ANY) { - ret = cxgb4_clip_get(ndev, (const u32 *) - &sk->sk_v6_rcv_saddr, 1); - if (ret) - goto del_hash; - clip_valid = true; - } - ret = cxgb4_create_server6(ndev, stid, - &sk->sk_v6_rcv_saddr, - inet_sk(sk)->inet_sport, - cdev->lldi->rxq_ids[0]); -#endif - } - if (ret > 0) - ret = net_xmit_errno(ret); - if (ret) - goto del_hash; - return 0; -del_hash: -#if IS_ENABLED(CONFIG_IPV6) - if (clip_valid) - cxgb4_clip_release(ndev, (const u32 *)&sk->sk_v6_rcv_saddr, 1); -#endif - listen_hash_del(cdev, sk); -free_stid: - cxgb4_free_stid(cdev->tids, stid, sk->sk_family); - sock_put(sk); -free_ctx: - kfree(ctx); - module_put(THIS_MODULE); - return -EBADF; -} - -void chtls_listen_stop(struct chtls_dev *cdev, struct sock *sk) -{ - struct listen_ctx *listen_ctx; - int stid; - - stid = listen_hash_del(cdev, sk); - if (stid < 0) - return; - - listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid); - chtls_reset_synq(listen_ctx); - - cxgb4_remove_server(cdev->lldi->ports[0], stid, - cdev->lldi->rxq_ids[0], sk->sk_family == PF_INET6); - -#if IS_ENABLED(CONFIG_IPV6) - if (sk->sk_family == PF_INET6) { - struct chtls_sock *csk; - int addr_type = 0; - - csk = rcu_dereference_sk_user_data(sk); - addr_type = ipv6_addr_type((const struct in6_addr *) - &sk->sk_v6_rcv_saddr); - if (addr_type != IPV6_ADDR_ANY) - cxgb4_clip_release(csk->egress_dev, (const u32 *) - &sk->sk_v6_rcv_saddr, 1); - } -#endif - chtls_disconnect_acceptq(sk); -} - -static int chtls_pass_open_rpl(struct chtls_dev *cdev, struct sk_buff *skb) -{ - struct cpl_pass_open_rpl *rpl = cplhdr(skb) + RSS_HDR; - unsigned int stid = GET_TID(rpl); - struct listen_ctx *listen_ctx; - - listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid); - if (!listen_ctx) - return CPL_RET_BUF_DONE; - - if (listen_ctx->state == T4_LISTEN_START_PENDING) { - listen_ctx->state = T4_LISTEN_STARTED; - return CPL_RET_BUF_DONE; - } - - if (rpl->status != CPL_ERR_NONE) { - pr_info("Unexpected PASS_OPEN_RPL status %u for STID %u\n", - rpl->status, stid); - return CPL_RET_BUF_DONE; - } - cxgb4_free_stid(cdev->tids, stid, listen_ctx->lsk->sk_family); - sock_put(listen_ctx->lsk); - kfree(listen_ctx); - module_put(THIS_MODULE); - - return 0; -} - -static int chtls_close_listsrv_rpl(struct chtls_dev *cdev, struct sk_buff *skb) -{ - struct cpl_close_listsvr_rpl *rpl = cplhdr(skb) + RSS_HDR; - struct listen_ctx *listen_ctx; - unsigned int stid; - void *data; - - stid = GET_TID(rpl); - data = lookup_stid(cdev->tids, stid); - listen_ctx = (struct listen_ctx *)data; - - if (rpl->status != CPL_ERR_NONE) { - pr_info("Unexpected CLOSE_LISTSRV_RPL status %u for STID %u\n", - rpl->status, stid); - return CPL_RET_BUF_DONE; - } - - cxgb4_free_stid(cdev->tids, stid, listen_ctx->lsk->sk_family); - sock_put(listen_ctx->lsk); - kfree(listen_ctx); - module_put(THIS_MODULE); - - return 0; -} - -static void chtls_purge_wr_queue(struct sock *sk) -{ - struct sk_buff *skb; - - while ((skb = dequeue_wr(sk)) != NULL) - kfree_skb(skb); -} - -static void chtls_release_resources(struct sock *sk) -{ - struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); - struct chtls_dev *cdev = csk->cdev; - unsigned int tid = csk->tid; - struct tid_info *tids; - - if (!cdev) - return; - - tids = cdev->tids; - kfree_skb(csk->txdata_skb_cache); - csk->txdata_skb_cache = NULL; - - if (csk->wr_credits != csk->wr_max_credits) { - chtls_purge_wr_queue(sk); - chtls_reset_wr_list(csk); - } - - if (csk->l2t_entry) { - cxgb4_l2t_release(csk->l2t_entry); - csk->l2t_entry = NULL; - } - - if (sk->sk_state != TCP_SYN_SENT) { - cxgb4_remove_tid(tids, csk->port_id, tid, sk->sk_family); - sock_put(sk); - } -} - -static void chtls_conn_done(struct sock *sk) -{ - if (sock_flag(sk, SOCK_DEAD)) - chtls_purge_receive_queue(sk); - sk_wakeup_sleepers(sk, 0); - tcp_done(sk); -} - -static void do_abort_syn_rcv(struct sock *child, struct sock *parent) -{ - /* - * If the server is still open we clean up the child connection, - * otherwise the server already did the clean up as it was purging - * its SYN queue and the skb was just sitting in its backlog. - */ - if (likely(parent->sk_state == TCP_LISTEN)) { - cleanup_syn_rcv_conn(child, parent); - /* Without the below call to sock_orphan, - * we leak the socket resource with syn_flood test - * as inet_csk_destroy_sock will not be called - * in tcp_done since SOCK_DEAD flag is not set. - * Kernel handles this differently where new socket is - * created only after 3 way handshake is done. - */ - sock_orphan(child); - percpu_counter_inc((child)->sk_prot->orphan_count); - chtls_release_resources(child); - chtls_conn_done(child); - } else { - if (csk_flag(child, CSK_RST_ABORTED)) { - chtls_release_resources(child); - chtls_conn_done(child); - } - } -} - -static void pass_open_abort(struct sock *child, struct sock *parent, - struct sk_buff *skb) -{ - do_abort_syn_rcv(child, parent); - kfree_skb(skb); -} - -static void bl_pass_open_abort(struct sock *lsk, struct sk_buff *skb) -{ - pass_open_abort(skb->sk, lsk, skb); -} - -static void chtls_pass_open_arp_failure(struct sock *sk, - struct sk_buff *skb) -{ - const struct request_sock *oreq; - struct chtls_sock *csk; - struct chtls_dev *cdev; - struct sock *parent; - void *data; - - csk = rcu_dereference_sk_user_data(sk); - cdev = csk->cdev; - - /* - * If the connection is being aborted due to the parent listening - * socket going away there's nothing to do, the ABORT_REQ will close - * the connection. - */ - if (csk_flag(sk, CSK_ABORT_RPL_PENDING)) { - kfree_skb(skb); - return; - } - - oreq = csk->passive_reap_next; - data = lookup_stid(cdev->tids, oreq->ts_recent); - parent = ((struct listen_ctx *)data)->lsk; - - bh_lock_sock(parent); - if (!sock_owned_by_user(parent)) { - pass_open_abort(sk, parent, skb); - } else { - BLOG_SKB_CB(skb)->backlog_rcv = bl_pass_open_abort; - __sk_add_backlog(parent, skb); - } - bh_unlock_sock(parent); -} - -static void chtls_accept_rpl_arp_failure(void *handle, - struct sk_buff *skb) -{ - struct sock *sk = (struct sock *)handle; - - sock_hold(sk); - process_cpl_msg(chtls_pass_open_arp_failure, sk, skb); - sock_put(sk); -} - -static unsigned int chtls_select_mss(const struct chtls_sock *csk, - unsigned int pmtu, - struct cpl_pass_accept_req *req) -{ - struct chtls_dev *cdev; - struct dst_entry *dst; - unsigned int tcpoptsz; - unsigned int iphdrsz; - unsigned int mtu_idx; - struct tcp_sock *tp; - unsigned int mss; - struct sock *sk; - - mss = ntohs(req->tcpopt.mss); - sk = csk->sk; - dst = __sk_dst_get(sk); - cdev = csk->cdev; - tp = tcp_sk(sk); - tcpoptsz = 0; - -#if IS_ENABLED(CONFIG_IPV6) - if (sk->sk_family == AF_INET6) - iphdrsz = sizeof(struct ipv6hdr) + sizeof(struct tcphdr); - else -#endif - iphdrsz = sizeof(struct iphdr) + sizeof(struct tcphdr); - if (req->tcpopt.tstamp) - tcpoptsz += round_up(TCPOLEN_TIMESTAMP, 4); - - tp->advmss = dst_metric_advmss(dst); - if (USER_MSS(tp) && tp->advmss > USER_MSS(tp)) - tp->advmss = USER_MSS(tp); - if (tp->advmss > pmtu - iphdrsz) - tp->advmss = pmtu - iphdrsz; - if (mss && tp->advmss > mss) - tp->advmss = mss; - - tp->advmss = cxgb4_best_aligned_mtu(cdev->lldi->mtus, - iphdrsz + tcpoptsz, - tp->advmss - tcpoptsz, - 8, &mtu_idx); - tp->advmss -= iphdrsz; - - inet_csk(sk)->icsk_pmtu_cookie = pmtu; - return mtu_idx; -} - -static unsigned int select_rcv_wscale(int space, int wscale_ok, int win_clamp) -{ - int wscale = 0; - - if (space > MAX_RCV_WND) - space = MAX_RCV_WND; - if (win_clamp && win_clamp < space) - space = win_clamp; - - if (wscale_ok) { - while (wscale < 14 && (65535 << wscale) < space) - wscale++; - } - return wscale; -} - -static void chtls_pass_accept_rpl(struct sk_buff *skb, - struct cpl_pass_accept_req *req, - unsigned int tid) - -{ - struct cpl_t5_pass_accept_rpl *rpl5; - struct cxgb4_lld_info *lldi; - const struct tcphdr *tcph; - const struct tcp_sock *tp; - struct chtls_sock *csk; - unsigned int len; - struct sock *sk; - u32 opt2, hlen; - u64 opt0; - - sk = skb->sk; - tp = tcp_sk(sk); - csk = sk->sk_user_data; - csk->tid = tid; - lldi = csk->cdev->lldi; - len = roundup(sizeof(*rpl5), 16); - - rpl5 = __skb_put_zero(skb, len); - INIT_TP_WR(rpl5, tid); - - OPCODE_TID(rpl5) = cpu_to_be32(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL, - csk->tid)); - csk->mtu_idx = chtls_select_mss(csk, dst_mtu(__sk_dst_get(sk)), - req); - opt0 = TCAM_BYPASS_F | - WND_SCALE_V(RCV_WSCALE(tp)) | - MSS_IDX_V(csk->mtu_idx) | - L2T_IDX_V(csk->l2t_entry->idx) | - NAGLE_V(!(tp->nonagle & TCP_NAGLE_OFF)) | - TX_CHAN_V(csk->tx_chan) | - SMAC_SEL_V(csk->smac_idx) | - DSCP_V(csk->tos >> 2) | - ULP_MODE_V(ULP_MODE_TLS) | - RCV_BUFSIZ_V(min(tp->rcv_wnd >> 10, RCV_BUFSIZ_M)); - - opt2 = RX_CHANNEL_V(0) | - RSS_QUEUE_VALID_F | RSS_QUEUE_V(csk->rss_qid); - - if (!is_t5(lldi->adapter_type)) - opt2 |= RX_FC_DISABLE_F; - if (req->tcpopt.tstamp) - opt2 |= TSTAMPS_EN_F; - if (req->tcpopt.sack) - opt2 |= SACK_EN_F; - hlen = ntohl(req->hdr_len); - - tcph = (struct tcphdr *)((u8 *)(req + 1) + - T6_ETH_HDR_LEN_G(hlen) + T6_IP_HDR_LEN_G(hlen)); - if (tcph->ece && tcph->cwr) - opt2 |= CCTRL_ECN_V(1); - opt2 |= CONG_CNTRL_V(CONG_ALG_NEWRENO); - opt2 |= T5_ISS_F; - opt2 |= T5_OPT_2_VALID_F; - opt2 |= WND_SCALE_EN_V(WSCALE_OK(tp)); - rpl5->opt0 = cpu_to_be64(opt0); - rpl5->opt2 = cpu_to_be32(opt2); - rpl5->iss = cpu_to_be32((prandom_u32() & ~7UL) - 1); - set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->port_id); - t4_set_arp_err_handler(skb, sk, chtls_accept_rpl_arp_failure); - cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry); -} - -static void inet_inherit_port(struct inet_hashinfo *hash_info, - struct sock *lsk, struct sock *newsk) -{ - local_bh_disable(); - __inet_inherit_port(lsk, newsk); - local_bh_enable(); -} - -static int chtls_backlog_rcv(struct sock *sk, struct sk_buff *skb) -{ - if (skb->protocol) { - kfree_skb(skb); - return 0; - } - BLOG_SKB_CB(skb)->backlog_rcv(sk, skb); - return 0; -} - -static void chtls_set_tcp_window(struct chtls_sock *csk) -{ - struct net_device *ndev = csk->egress_dev; - struct port_info *pi = netdev_priv(ndev); - unsigned int linkspeed; - u8 scale; - - linkspeed = pi->link_cfg.speed; - scale = linkspeed / SPEED_10000; -#define CHTLS_10G_RCVWIN (256 * 1024) - csk->rcv_win = CHTLS_10G_RCVWIN; - if (scale) - csk->rcv_win *= scale; -#define CHTLS_10G_SNDWIN (256 * 1024) - csk->snd_win = CHTLS_10G_SNDWIN; - if (scale) - csk->snd_win *= scale; -} - -static struct sock *chtls_recv_sock(struct sock *lsk, - struct request_sock *oreq, - void *network_hdr, - const struct cpl_pass_accept_req *req, - struct chtls_dev *cdev) -{ - struct neighbour *n = NULL; - struct inet_sock *newinet; - const struct iphdr *iph; - struct tls_context *ctx; - struct net_device *ndev; - struct chtls_sock *csk; - struct dst_entry *dst; - struct tcp_sock *tp; - struct sock *newsk; - u16 port_id; - int rxq_idx; - int step; - - iph = (const struct iphdr *)network_hdr; - newsk = tcp_create_openreq_child(lsk, oreq, cdev->askb); - if (!newsk) - goto free_oreq; - - if (lsk->sk_family == AF_INET) { - dst = inet_csk_route_child_sock(lsk, newsk, oreq); - if (!dst) - goto free_sk; - - n = dst_neigh_lookup(dst, &iph->saddr); -#if IS_ENABLED(CONFIG_IPV6) - } else { - const struct ipv6hdr *ip6h; - struct flowi6 fl6; - - ip6h = (const struct ipv6hdr *)network_hdr; - memset(&fl6, 0, sizeof(fl6)); - fl6.flowi6_proto = IPPROTO_TCP; - fl6.saddr = ip6h->daddr; - fl6.daddr = ip6h->saddr; - fl6.fl6_dport = inet_rsk(oreq)->ir_rmt_port; - fl6.fl6_sport = htons(inet_rsk(oreq)->ir_num); - security_req_classify_flow(oreq, flowi6_to_flowi(&fl6)); - dst = ip6_dst_lookup_flow(sock_net(lsk), lsk, &fl6, NULL); - if (IS_ERR(dst)) - goto free_sk; - n = dst_neigh_lookup(dst, &ip6h->saddr); -#endif - } - if (!n) - goto free_sk; - - ndev = n->dev; - if (!ndev) - goto free_dst; - port_id = cxgb4_port_idx(ndev); - - csk = chtls_sock_create(cdev); - if (!csk) - goto free_dst; - - csk->l2t_entry = cxgb4_l2t_get(cdev->lldi->l2t, n, ndev, 0); - if (!csk->l2t_entry) - goto free_csk; - - newsk->sk_user_data = csk; - newsk->sk_backlog_rcv = chtls_backlog_rcv; - - tp = tcp_sk(newsk); - newinet = inet_sk(newsk); - - if (iph->version == 0x4) { - newinet->inet_daddr = iph->saddr; - newinet->inet_rcv_saddr = iph->daddr; - newinet->inet_saddr = iph->daddr; -#if IS_ENABLED(CONFIG_IPV6) - } else { - struct tcp6_sock *newtcp6sk = (struct tcp6_sock *)newsk; - struct inet_request_sock *treq = inet_rsk(oreq); - struct ipv6_pinfo *newnp = inet6_sk(newsk); - struct ipv6_pinfo *np = inet6_sk(lsk); - - inet_sk(newsk)->pinet6 = &newtcp6sk->inet6; - memcpy(newnp, np, sizeof(struct ipv6_pinfo)); - newsk->sk_v6_daddr = treq->ir_v6_rmt_addr; - newsk->sk_v6_rcv_saddr = treq->ir_v6_loc_addr; - inet6_sk(newsk)->saddr = treq->ir_v6_loc_addr; - newnp->ipv6_fl_list = NULL; - newnp->pktoptions = NULL; - newsk->sk_bound_dev_if = treq->ir_iif; - newinet->inet_opt = NULL; - newinet->inet_daddr = LOOPBACK4_IPV6; - newinet->inet_saddr = LOOPBACK4_IPV6; -#endif - } - - oreq->ts_recent = PASS_OPEN_TID_G(ntohl(req->tos_stid)); - sk_setup_caps(newsk, dst); - ctx = tls_get_ctx(lsk); - newsk->sk_destruct = ctx->sk_destruct; - csk->sk = newsk; - csk->passive_reap_next = oreq; - csk->tx_chan = cxgb4_port_chan(ndev); - csk->port_id = port_id; - csk->egress_dev = ndev; - csk->tos = PASS_OPEN_TOS_G(ntohl(req->tos_stid)); - chtls_set_tcp_window(csk); - tp->rcv_wnd = csk->rcv_win; - csk->sndbuf = csk->snd_win; - csk->ulp_mode = ULP_MODE_TLS; - step = cdev->lldi->nrxq / cdev->lldi->nchan; - csk->rss_qid = cdev->lldi->rxq_ids[port_id * step]; - rxq_idx = port_id * step; - csk->txq_idx = (rxq_idx < cdev->lldi->ntxq) ? rxq_idx : - port_id * step; - csk->sndbuf = newsk->sk_sndbuf; - csk->smac_idx = ((struct port_info *)netdev_priv(ndev))->smt_idx; - RCV_WSCALE(tp) = select_rcv_wscale(tcp_full_space(newsk), - sock_net(newsk)-> - ipv4.sysctl_tcp_window_scaling, - tp->window_clamp); - neigh_release(n); - inet_inherit_port(&tcp_hashinfo, lsk, newsk); - csk_set_flag(csk, CSK_CONN_INLINE); - bh_unlock_sock(newsk); /* tcp_create_openreq_child ->sk_clone_lock */ - - return newsk; -free_csk: - chtls_sock_release(&csk->kref); -free_dst: - dst_release(dst); -free_sk: - inet_csk_prepare_forced_close(newsk); - tcp_done(newsk); -free_oreq: - chtls_reqsk_free(oreq); - return NULL; -} - -/* - * Populate a TID_RELEASE WR. The skb must be already propely sized. - */ -static void mk_tid_release(struct sk_buff *skb, - unsigned int chan, unsigned int tid) -{ - struct cpl_tid_release *req; - unsigned int len; - - len = roundup(sizeof(struct cpl_tid_release), 16); - req = (struct cpl_tid_release *)__skb_put(skb, len); - memset(req, 0, len); - set_wr_txq(skb, CPL_PRIORITY_SETUP, chan); - INIT_TP_WR_CPL(req, CPL_TID_RELEASE, tid); -} - -static int chtls_get_module(struct sock *sk) -{ - struct inet_connection_sock *icsk = inet_csk(sk); - - if (!try_module_get(icsk->icsk_ulp_ops->owner)) - return -1; - - return 0; -} - -static void chtls_pass_accept_request(struct sock *sk, - struct sk_buff *skb) -{ - struct cpl_t5_pass_accept_rpl *rpl; - struct cpl_pass_accept_req *req; - struct listen_ctx *listen_ctx; - struct vlan_ethhdr *vlan_eh; - struct request_sock *oreq; - struct sk_buff *reply_skb; - struct chtls_sock *csk; - struct chtls_dev *cdev; - struct ipv6hdr *ip6h; - struct tcphdr *tcph; - struct sock *newsk; - struct ethhdr *eh; - struct iphdr *iph; - void *network_hdr; - unsigned int stid; - unsigned int len; - unsigned int tid; - bool th_ecn, ect; - __u8 ip_dsfield; /* IPv4 tos or IPv6 dsfield */ - u16 eth_hdr_len; - bool ecn_ok; - - req = cplhdr(skb) + RSS_HDR; - tid = GET_TID(req); - cdev = BLOG_SKB_CB(skb)->cdev; - newsk = lookup_tid(cdev->tids, tid); - stid = PASS_OPEN_TID_G(ntohl(req->tos_stid)); - if (newsk) { - pr_info("tid (%d) already in use\n", tid); - return; - } - - len = roundup(sizeof(*rpl), 16); - reply_skb = alloc_skb(len, GFP_ATOMIC); - if (!reply_skb) { - cxgb4_remove_tid(cdev->tids, 0, tid, sk->sk_family); - kfree_skb(skb); - return; - } - - if (sk->sk_state != TCP_LISTEN) - goto reject; - - if (inet_csk_reqsk_queue_is_full(sk)) - goto reject; - - if (sk_acceptq_is_full(sk)) - goto reject; - - - eth_hdr_len = T6_ETH_HDR_LEN_G(ntohl(req->hdr_len)); - if (eth_hdr_len == ETH_HLEN) { - eh = (struct ethhdr *)(req + 1); - iph = (struct iphdr *)(eh + 1); - ip6h = (struct ipv6hdr *)(eh + 1); - network_hdr = (void *)(eh + 1); - } else { - vlan_eh = (struct vlan_ethhdr *)(req + 1); - iph = (struct iphdr *)(vlan_eh + 1); - ip6h = (struct ipv6hdr *)(vlan_eh + 1); - network_hdr = (void *)(vlan_eh + 1); - } - - if (iph->version == 0x4) { - tcph = (struct tcphdr *)(iph + 1); - skb_set_network_header(skb, (void *)iph - (void *)req); - oreq = inet_reqsk_alloc(&chtls_rsk_ops, sk, true); - } else { - tcph = (struct tcphdr *)(ip6h + 1); - skb_set_network_header(skb, (void *)ip6h - (void *)req); - oreq = inet_reqsk_alloc(&chtls_rsk_opsv6, sk, false); - } - - if (!oreq) - goto reject; - - oreq->rsk_rcv_wnd = 0; - oreq->rsk_window_clamp = 0; - oreq->syncookie = 0; - oreq->mss = 0; - oreq->ts_recent = 0; - - tcp_rsk(oreq)->tfo_listener = false; - tcp_rsk(oreq)->rcv_isn = ntohl(tcph->seq); - chtls_set_req_port(oreq, tcph->source, tcph->dest); - if (iph->version == 0x4) { - chtls_set_req_addr(oreq, iph->daddr, iph->saddr); - ip_dsfield = ipv4_get_dsfield(iph); -#if IS_ENABLED(CONFIG_IPV6) - } else { - inet_rsk(oreq)->ir_v6_rmt_addr = ipv6_hdr(skb)->saddr; - inet_rsk(oreq)->ir_v6_loc_addr = ipv6_hdr(skb)->daddr; - ip_dsfield = ipv6_get_dsfield(ipv6_hdr(skb)); -#endif - } - if (req->tcpopt.wsf <= 14 && - sock_net(sk)->ipv4.sysctl_tcp_window_scaling) { - inet_rsk(oreq)->wscale_ok = 1; - inet_rsk(oreq)->snd_wscale = req->tcpopt.wsf; - } - inet_rsk(oreq)->ir_iif = sk->sk_bound_dev_if; - th_ecn = tcph->ece && tcph->cwr; - if (th_ecn) { - ect = !INET_ECN_is_not_ect(ip_dsfield); - ecn_ok = sock_net(sk)->ipv4.sysctl_tcp_ecn; - if ((!ect && ecn_ok) || tcp_ca_needs_ecn(sk)) - inet_rsk(oreq)->ecn_ok = 1; - } - - newsk = chtls_recv_sock(sk, oreq, network_hdr, req, cdev); - if (!newsk) - goto free_oreq; - - if (chtls_get_module(newsk)) - goto reject; - inet_csk_reqsk_queue_added(sk); - reply_skb->sk = newsk; - chtls_install_cpl_ops(newsk); - cxgb4_insert_tid(cdev->tids, newsk, tid, newsk->sk_family); - csk = rcu_dereference_sk_user_data(newsk); - listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid); - csk->listen_ctx = listen_ctx; - __skb_queue_tail(&listen_ctx->synq, (struct sk_buff *)&csk->synq); - chtls_pass_accept_rpl(reply_skb, req, tid); - kfree_skb(skb); - return; - -free_oreq: - chtls_reqsk_free(oreq); -reject: - mk_tid_release(reply_skb, 0, tid); - cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb); - kfree_skb(skb); -} - -/* - * Handle a CPL_PASS_ACCEPT_REQ message. - */ -static int chtls_pass_accept_req(struct chtls_dev *cdev, struct sk_buff *skb) -{ - struct cpl_pass_accept_req *req = cplhdr(skb) + RSS_HDR; - struct listen_ctx *ctx; - unsigned int stid; - unsigned int tid; - struct sock *lsk; - void *data; - - stid = PASS_OPEN_TID_G(ntohl(req->tos_stid)); - tid = GET_TID(req); - - data = lookup_stid(cdev->tids, stid); - if (!data) - return 1; - - ctx = (struct listen_ctx *)data; - lsk = ctx->lsk; - - if (unlikely(tid_out_of_range(cdev->tids, tid))) { - pr_info("passive open TID %u too large\n", tid); - return 1; - } - - BLOG_SKB_CB(skb)->cdev = cdev; - process_cpl_msg(chtls_pass_accept_request, lsk, skb); - return 0; -} - -/* - * Completes some final bits of initialization for just established connections - * and changes their state to TCP_ESTABLISHED. - * - * snd_isn here is the ISN after the SYN, i.e., the true ISN + 1. - */ -static void make_established(struct sock *sk, u32 snd_isn, unsigned int opt) -{ - struct tcp_sock *tp = tcp_sk(sk); - - tp->pushed_seq = snd_isn; - tp->write_seq = snd_isn; - tp->snd_nxt = snd_isn; - tp->snd_una = snd_isn; - inet_sk(sk)->inet_id = prandom_u32(); - assign_rxopt(sk, opt); - - if (tp->rcv_wnd > (RCV_BUFSIZ_M << 10)) - tp->rcv_wup -= tp->rcv_wnd - (RCV_BUFSIZ_M << 10); - - smp_mb(); - tcp_set_state(sk, TCP_ESTABLISHED); -} - -static void chtls_abort_conn(struct sock *sk, struct sk_buff *skb) -{ - struct sk_buff *abort_skb; - - abort_skb = alloc_skb(sizeof(struct cpl_abort_req), GFP_ATOMIC); - if (abort_skb) - chtls_send_reset(sk, CPL_ABORT_SEND_RST, abort_skb); -} - -static struct sock *reap_list; -static DEFINE_SPINLOCK(reap_list_lock); - -/* - * Process the reap list. - */ -DECLARE_TASK_FUNC(process_reap_list, task_param) -{ - spin_lock_bh(&reap_list_lock); - while (reap_list) { - struct sock *sk = reap_list; - struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); - - reap_list = csk->passive_reap_next; - csk->passive_reap_next = NULL; - spin_unlock(&reap_list_lock); - sock_hold(sk); - - bh_lock_sock(sk); - chtls_abort_conn(sk, NULL); - sock_orphan(sk); - if (sk->sk_state == TCP_CLOSE) - inet_csk_destroy_sock(sk); - bh_unlock_sock(sk); - sock_put(sk); - spin_lock(&reap_list_lock); - } - spin_unlock_bh(&reap_list_lock); -} - -static DECLARE_WORK(reap_task, process_reap_list); - -static void add_to_reap_list(struct sock *sk) -{ - struct chtls_sock *csk = sk->sk_user_data; - - local_bh_disable(); - bh_lock_sock(sk); - release_tcp_port(sk); /* release the port immediately */ - - spin_lock(&reap_list_lock); - csk->passive_reap_next = reap_list; - reap_list = sk; - if (!csk->passive_reap_next) - schedule_work(&reap_task); - spin_unlock(&reap_list_lock); - bh_unlock_sock(sk); - local_bh_enable(); -} - -static void add_pass_open_to_parent(struct sock *child, struct sock *lsk, - struct chtls_dev *cdev) -{ - struct request_sock *oreq; - struct chtls_sock *csk; - - if (lsk->sk_state != TCP_LISTEN) - return; - - csk = child->sk_user_data; - oreq = csk->passive_reap_next; - csk->passive_reap_next = NULL; - - reqsk_queue_removed(&inet_csk(lsk)->icsk_accept_queue, oreq); - __skb_unlink((struct sk_buff *)&csk->synq, &csk->listen_ctx->synq); - - if (sk_acceptq_is_full(lsk)) { - chtls_reqsk_free(oreq); - add_to_reap_list(child); - } else { - refcount_set(&oreq->rsk_refcnt, 1); - inet_csk_reqsk_queue_add(lsk, oreq, child); - lsk->sk_data_ready(lsk); - } -} - -static void bl_add_pass_open_to_parent(struct sock *lsk, struct sk_buff *skb) -{ - struct sock *child = skb->sk; - - skb->sk = NULL; - add_pass_open_to_parent(child, lsk, BLOG_SKB_CB(skb)->cdev); - kfree_skb(skb); -} - -static int chtls_pass_establish(struct chtls_dev *cdev, struct sk_buff *skb) -{ - struct cpl_pass_establish *req = cplhdr(skb) + RSS_HDR; - struct chtls_sock *csk; - struct sock *lsk, *sk; - unsigned int hwtid; - - hwtid = GET_TID(req); - sk = lookup_tid(cdev->tids, hwtid); - if (!sk) - return (CPL_RET_UNKNOWN_TID | CPL_RET_BUF_DONE); - - bh_lock_sock(sk); - if (unlikely(sock_owned_by_user(sk))) { - kfree_skb(skb); - } else { - unsigned int stid; - void *data; - - csk = sk->sk_user_data; - csk->wr_max_credits = 64; - csk->wr_credits = 64; - csk->wr_unacked = 0; - make_established(sk, ntohl(req->snd_isn), ntohs(req->tcp_opt)); - stid = PASS_OPEN_TID_G(ntohl(req->tos_stid)); - sk->sk_state_change(sk); - if (unlikely(sk->sk_socket)) - sk_wake_async(sk, 0, POLL_OUT); - - data = lookup_stid(cdev->tids, stid); - lsk = ((struct listen_ctx *)data)->lsk; - - bh_lock_sock(lsk); - if (unlikely(skb_queue_empty(&csk->listen_ctx->synq))) { - /* removed from synq */ - bh_unlock_sock(lsk); - kfree_skb(skb); - goto unlock; - } - - if (likely(!sock_owned_by_user(lsk))) { - kfree_skb(skb); - add_pass_open_to_parent(sk, lsk, cdev); - } else { - skb->sk = sk; - BLOG_SKB_CB(skb)->cdev = cdev; - BLOG_SKB_CB(skb)->backlog_rcv = - bl_add_pass_open_to_parent; - __sk_add_backlog(lsk, skb); - } - bh_unlock_sock(lsk); - } -unlock: - bh_unlock_sock(sk); - return 0; -} - -/* - * Handle receipt of an urgent pointer. - */ -static void handle_urg_ptr(struct sock *sk, u32 urg_seq) -{ - struct tcp_sock *tp = tcp_sk(sk); - - urg_seq--; - if (tp->urg_data && !after(urg_seq, tp->urg_seq)) - return; /* duplicate pointer */ - - sk_send_sigurg(sk); - if (tp->urg_seq == tp->copied_seq && tp->urg_data && - !sock_flag(sk, SOCK_URGINLINE) && - tp->copied_seq != tp->rcv_nxt) { - struct sk_buff *skb = skb_peek(&sk->sk_receive_queue); - - tp->copied_seq++; - if (skb && tp->copied_seq - ULP_SKB_CB(skb)->seq >= skb->len) - chtls_free_skb(sk, skb); - } - - tp->urg_data = TCP_URG_NOTYET; - tp->urg_seq = urg_seq; -} - -static void check_sk_callbacks(struct chtls_sock *csk) -{ - struct sock *sk = csk->sk; - - if (unlikely(sk->sk_user_data && - !csk_flag_nochk(csk, CSK_CALLBACKS_CHKD))) - csk_set_flag(csk, CSK_CALLBACKS_CHKD); -} - -/* - * Handles Rx data that arrives in a state where the socket isn't accepting - * new data. - */ -static void handle_excess_rx(struct sock *sk, struct sk_buff *skb) -{ - if (!csk_flag(sk, CSK_ABORT_SHUTDOWN)) - chtls_abort_conn(sk, skb); - - kfree_skb(skb); -} - -static void chtls_recv_data(struct sock *sk, struct sk_buff *skb) -{ - struct cpl_rx_data *hdr = cplhdr(skb) + RSS_HDR; - struct chtls_sock *csk; - struct tcp_sock *tp; - - csk = rcu_dereference_sk_user_data(sk); - tp = tcp_sk(sk); - - if (unlikely(sk->sk_shutdown & RCV_SHUTDOWN)) { - handle_excess_rx(sk, skb); - return; - } - - ULP_SKB_CB(skb)->seq = ntohl(hdr->seq); - ULP_SKB_CB(skb)->psh = hdr->psh; - skb_ulp_mode(skb) = ULP_MODE_NONE; - - skb_reset_transport_header(skb); - __skb_pull(skb, sizeof(*hdr) + RSS_HDR); - if (!skb->data_len) - __skb_trim(skb, ntohs(hdr->len)); - - if (unlikely(hdr->urg)) - handle_urg_ptr(sk, tp->rcv_nxt + ntohs(hdr->urg)); - if (unlikely(tp->urg_data == TCP_URG_NOTYET && - tp->urg_seq - tp->rcv_nxt < skb->len)) - tp->urg_data = TCP_URG_VALID | - skb->data[tp->urg_seq - tp->rcv_nxt]; - - if (unlikely(hdr->dack_mode != csk->delack_mode)) { - csk->delack_mode = hdr->dack_mode; - csk->delack_seq = tp->rcv_nxt; - } - - tcp_hdr(skb)->fin = 0; - tp->rcv_nxt += skb->len; - - __skb_queue_tail(&sk->sk_receive_queue, skb); - - if (!sock_flag(sk, SOCK_DEAD)) { - check_sk_callbacks(csk); - sk->sk_data_ready(sk); - } -} - -static int chtls_rx_data(struct chtls_dev *cdev, struct sk_buff *skb) -{ - struct cpl_rx_data *req = cplhdr(skb) + RSS_HDR; - unsigned int hwtid = GET_TID(req); - struct sock *sk; - - sk = lookup_tid(cdev->tids, hwtid); - if (unlikely(!sk)) { - pr_err("can't find conn. for hwtid %u.\n", hwtid); - return -EINVAL; - } - skb_dst_set(skb, NULL); - process_cpl_msg(chtls_recv_data, sk, skb); - return 0; -} - -static void chtls_recv_pdu(struct sock *sk, struct sk_buff *skb) -{ - struct cpl_tls_data *hdr = cplhdr(skb); - struct chtls_sock *csk; - struct chtls_hws *tlsk; - struct tcp_sock *tp; - - csk = rcu_dereference_sk_user_data(sk); - tlsk = &csk->tlshws; - tp = tcp_sk(sk); - - if (unlikely(sk->sk_shutdown & RCV_SHUTDOWN)) { - handle_excess_rx(sk, skb); - return; - } - - ULP_SKB_CB(skb)->seq = ntohl(hdr->seq); - ULP_SKB_CB(skb)->flags = 0; - skb_ulp_mode(skb) = ULP_MODE_TLS; - - skb_reset_transport_header(skb); - __skb_pull(skb, sizeof(*hdr)); - if (!skb->data_len) - __skb_trim(skb, - CPL_TLS_DATA_LENGTH_G(ntohl(hdr->length_pkd))); - - if (unlikely(tp->urg_data == TCP_URG_NOTYET && tp->urg_seq - - tp->rcv_nxt < skb->len)) - tp->urg_data = TCP_URG_VALID | - skb->data[tp->urg_seq - tp->rcv_nxt]; - - tcp_hdr(skb)->fin = 0; - tlsk->pldlen = CPL_TLS_DATA_LENGTH_G(ntohl(hdr->length_pkd)); - __skb_queue_tail(&tlsk->sk_recv_queue, skb); -} - -static int chtls_rx_pdu(struct chtls_dev *cdev, struct sk_buff *skb) -{ - struct cpl_tls_data *req = cplhdr(skb); - unsigned int hwtid = GET_TID(req); - struct sock *sk; - - sk = lookup_tid(cdev->tids, hwtid); - if (unlikely(!sk)) { - pr_err("can't find conn. for hwtid %u.\n", hwtid); - return -EINVAL; - } - skb_dst_set(skb, NULL); - process_cpl_msg(chtls_recv_pdu, sk, skb); - return 0; -} - -static void chtls_set_hdrlen(struct sk_buff *skb, unsigned int nlen) -{ - struct tlsrx_cmp_hdr *tls_cmp_hdr = cplhdr(skb); - - skb->hdr_len = ntohs((__force __be16)tls_cmp_hdr->length); - tls_cmp_hdr->length = ntohs((__force __be16)nlen); -} - -static void chtls_rx_hdr(struct sock *sk, struct sk_buff *skb) -{ - struct tlsrx_cmp_hdr *tls_hdr_pkt; - struct cpl_rx_tls_cmp *cmp_cpl; - struct sk_buff *skb_rec; - struct chtls_sock *csk; - struct chtls_hws *tlsk; - struct tcp_sock *tp; - - cmp_cpl = cplhdr(skb); - csk = rcu_dereference_sk_user_data(sk); - tlsk = &csk->tlshws; - tp = tcp_sk(sk); - - ULP_SKB_CB(skb)->seq = ntohl(cmp_cpl->seq); - ULP_SKB_CB(skb)->flags = 0; - - skb_reset_transport_header(skb); - __skb_pull(skb, sizeof(*cmp_cpl)); - tls_hdr_pkt = (struct tlsrx_cmp_hdr *)skb->data; - if (tls_hdr_pkt->res_to_mac_error & TLSRX_HDR_PKT_ERROR_M) - tls_hdr_pkt->type = CONTENT_TYPE_ERROR; - if (!skb->data_len) - __skb_trim(skb, TLS_HEADER_LENGTH); - - tp->rcv_nxt += - CPL_RX_TLS_CMP_PDULENGTH_G(ntohl(cmp_cpl->pdulength_length)); - - ULP_SKB_CB(skb)->flags |= ULPCB_FLAG_TLS_HDR; - skb_rec = __skb_dequeue(&tlsk->sk_recv_queue); - if (!skb_rec) { - __skb_queue_tail(&sk->sk_receive_queue, skb); - } else { - chtls_set_hdrlen(skb, tlsk->pldlen); - tlsk->pldlen = 0; - __skb_queue_tail(&sk->sk_receive_queue, skb); - __skb_queue_tail(&sk->sk_receive_queue, skb_rec); - } - - if (!sock_flag(sk, SOCK_DEAD)) { - check_sk_callbacks(csk); - sk->sk_data_ready(sk); - } -} - -static int chtls_rx_cmp(struct chtls_dev *cdev, struct sk_buff *skb) -{ - struct cpl_rx_tls_cmp *req = cplhdr(skb); - unsigned int hwtid = GET_TID(req); - struct sock *sk; - - sk = lookup_tid(cdev->tids, hwtid); - if (unlikely(!sk)) { - pr_err("can't find conn. for hwtid %u.\n", hwtid); - return -EINVAL; - } - skb_dst_set(skb, NULL); - process_cpl_msg(chtls_rx_hdr, sk, skb); - - return 0; -} - -static void chtls_timewait(struct sock *sk) -{ - struct tcp_sock *tp = tcp_sk(sk); - - tp->rcv_nxt++; - tp->rx_opt.ts_recent_stamp = ktime_get_seconds(); - tp->srtt_us = 0; - tcp_time_wait(sk, TCP_TIME_WAIT, 0); -} - -static void chtls_peer_close(struct sock *sk, struct sk_buff *skb) -{ - struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); - - if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) - goto out; - - sk->sk_shutdown |= RCV_SHUTDOWN; - sock_set_flag(sk, SOCK_DONE); - - switch (sk->sk_state) { - case TCP_SYN_RECV: - case TCP_ESTABLISHED: - tcp_set_state(sk, TCP_CLOSE_WAIT); - break; - case TCP_FIN_WAIT1: - tcp_set_state(sk, TCP_CLOSING); - break; - case TCP_FIN_WAIT2: - chtls_release_resources(sk); - if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) - chtls_conn_done(sk); - else - chtls_timewait(sk); - break; - default: - pr_info("cpl_peer_close in bad state %d\n", sk->sk_state); - } - - if (!sock_flag(sk, SOCK_DEAD)) { - sk->sk_state_change(sk); - /* Do not send POLL_HUP for half duplex close. */ - - if ((sk->sk_shutdown & SEND_SHUTDOWN) || - sk->sk_state == TCP_CLOSE) - sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP); - else - sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); - } -out: - kfree_skb(skb); -} - -static void chtls_close_con_rpl(struct sock *sk, struct sk_buff *skb) -{ - struct cpl_close_con_rpl *rpl = cplhdr(skb) + RSS_HDR; - struct chtls_sock *csk; - struct tcp_sock *tp; - - csk = rcu_dereference_sk_user_data(sk); - - if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) - goto out; - - tp = tcp_sk(sk); - - tp->snd_una = ntohl(rpl->snd_nxt) - 1; /* exclude FIN */ - - switch (sk->sk_state) { - case TCP_CLOSING: - chtls_release_resources(sk); - if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) - chtls_conn_done(sk); - else - chtls_timewait(sk); - break; - case TCP_LAST_ACK: - chtls_release_resources(sk); - chtls_conn_done(sk); - break; - case TCP_FIN_WAIT1: - tcp_set_state(sk, TCP_FIN_WAIT2); - sk->sk_shutdown |= SEND_SHUTDOWN; - - if (!sock_flag(sk, SOCK_DEAD)) - sk->sk_state_change(sk); - else if (tcp_sk(sk)->linger2 < 0 && - !csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN)) - chtls_abort_conn(sk, skb); - break; - default: - pr_info("close_con_rpl in bad state %d\n", sk->sk_state); - } -out: - kfree_skb(skb); -} - -static struct sk_buff *get_cpl_skb(struct sk_buff *skb, - size_t len, gfp_t gfp) -{ - if (likely(!skb_is_nonlinear(skb) && !skb_cloned(skb))) { - WARN_ONCE(skb->len < len, "skb alloc error"); - __skb_trim(skb, len); - skb_get(skb); - } else { - skb = alloc_skb(len, gfp); - if (skb) - __skb_put(skb, len); - } - return skb; -} - -static void set_abort_rpl_wr(struct sk_buff *skb, unsigned int tid, - int cmd) -{ - struct cpl_abort_rpl *rpl = cplhdr(skb); - - INIT_TP_WR_CPL(rpl, CPL_ABORT_RPL, tid); - rpl->cmd = cmd; -} - -static void send_defer_abort_rpl(struct chtls_dev *cdev, struct sk_buff *skb) -{ - struct cpl_abort_req_rss *req = cplhdr(skb); - struct sk_buff *reply_skb; - - reply_skb = alloc_skb(sizeof(struct cpl_abort_rpl), - GFP_KERNEL | __GFP_NOFAIL); - __skb_put(reply_skb, sizeof(struct cpl_abort_rpl)); - set_abort_rpl_wr(reply_skb, GET_TID(req), - (req->status & CPL_ABORT_NO_RST)); - set_wr_txq(reply_skb, CPL_PRIORITY_DATA, req->status >> 1); - cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb); - kfree_skb(skb); -} - -/* - * Add an skb to the deferred skb queue for processing from process context. - */ -static void t4_defer_reply(struct sk_buff *skb, struct chtls_dev *cdev, - defer_handler_t handler) -{ - DEFERRED_SKB_CB(skb)->handler = handler; - spin_lock_bh(&cdev->deferq.lock); - __skb_queue_tail(&cdev->deferq, skb); - if (skb_queue_len(&cdev->deferq) == 1) - schedule_work(&cdev->deferq_task); - spin_unlock_bh(&cdev->deferq.lock); -} - -static void send_abort_rpl(struct sock *sk, struct sk_buff *skb, - struct chtls_dev *cdev, int status, int queue) -{ - struct cpl_abort_req_rss *req = cplhdr(skb); - struct sk_buff *reply_skb; - struct chtls_sock *csk; - - csk = rcu_dereference_sk_user_data(sk); - - reply_skb = alloc_skb(sizeof(struct cpl_abort_rpl), - GFP_KERNEL); - - if (!reply_skb) { - req->status = (queue << 1); - t4_defer_reply(skb, cdev, send_defer_abort_rpl); - return; - } - - set_abort_rpl_wr(reply_skb, GET_TID(req), status); - kfree_skb(skb); - - set_wr_txq(reply_skb, CPL_PRIORITY_DATA, queue); - if (csk_conn_inline(csk)) { - struct l2t_entry *e = csk->l2t_entry; - - if (e && sk->sk_state != TCP_SYN_RECV) { - cxgb4_l2t_send(csk->egress_dev, reply_skb, e); - return; - } - } - cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb); -} - -static void chtls_send_abort_rpl(struct sock *sk, struct sk_buff *skb, - struct chtls_dev *cdev, - int status, int queue) -{ - struct cpl_abort_req_rss *req = cplhdr(skb) + RSS_HDR; - struct sk_buff *reply_skb; - struct chtls_sock *csk; - unsigned int tid; - - csk = rcu_dereference_sk_user_data(sk); - tid = GET_TID(req); - - reply_skb = get_cpl_skb(skb, sizeof(struct cpl_abort_rpl), gfp_any()); - if (!reply_skb) { - req->status = (queue << 1) | status; - t4_defer_reply(skb, cdev, send_defer_abort_rpl); - return; - } - - set_abort_rpl_wr(reply_skb, tid, status); - kfree_skb(skb); - set_wr_txq(reply_skb, CPL_PRIORITY_DATA, queue); - if (csk_conn_inline(csk)) { - struct l2t_entry *e = csk->l2t_entry; - - if (e && sk->sk_state != TCP_SYN_RECV) { - cxgb4_l2t_send(csk->egress_dev, reply_skb, e); - return; - } - } - cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb); -} - -/* - * This is run from a listener's backlog to abort a child connection in - * SYN_RCV state (i.e., one on the listener's SYN queue). - */ -static void bl_abort_syn_rcv(struct sock *lsk, struct sk_buff *skb) -{ - struct chtls_sock *csk; - struct sock *child; - int queue; - - child = skb->sk; - csk = rcu_dereference_sk_user_data(child); - queue = csk->txq_idx; - - skb->sk = NULL; - do_abort_syn_rcv(child, lsk); - send_abort_rpl(child, skb, BLOG_SKB_CB(skb)->cdev, - CPL_ABORT_NO_RST, queue); -} - -static int abort_syn_rcv(struct sock *sk, struct sk_buff *skb) -{ - const struct request_sock *oreq; - struct listen_ctx *listen_ctx; - struct chtls_sock *csk; - struct chtls_dev *cdev; - struct sock *psk; - void *ctx; - - csk = sk->sk_user_data; - oreq = csk->passive_reap_next; - cdev = csk->cdev; - - if (!oreq) - return -1; - - ctx = lookup_stid(cdev->tids, oreq->ts_recent); - if (!ctx) - return -1; - - listen_ctx = (struct listen_ctx *)ctx; - psk = listen_ctx->lsk; - - bh_lock_sock(psk); - if (!sock_owned_by_user(psk)) { - int queue = csk->txq_idx; - - do_abort_syn_rcv(sk, psk); - send_abort_rpl(sk, skb, cdev, CPL_ABORT_NO_RST, queue); - } else { - skb->sk = sk; - BLOG_SKB_CB(skb)->backlog_rcv = bl_abort_syn_rcv; - __sk_add_backlog(psk, skb); - } - bh_unlock_sock(psk); - return 0; -} - -static void chtls_abort_req_rss(struct sock *sk, struct sk_buff *skb) -{ - const struct cpl_abort_req_rss *req = cplhdr(skb) + RSS_HDR; - struct chtls_sock *csk = sk->sk_user_data; - int rst_status = CPL_ABORT_NO_RST; - int queue = csk->txq_idx; - - if (is_neg_adv(req->status)) { - if (sk->sk_state == TCP_SYN_RECV) - chtls_set_tcb_tflag(sk, 0, 0); - - kfree_skb(skb); - return; - } - - csk_reset_flag(csk, CSK_ABORT_REQ_RCVD); - - if (!csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN) && - !csk_flag_nochk(csk, CSK_TX_DATA_SENT)) { - struct tcp_sock *tp = tcp_sk(sk); - - if (send_tx_flowc_wr(sk, 0, tp->snd_nxt, tp->rcv_nxt) < 0) - WARN_ONCE(1, "send_tx_flowc error"); - csk_set_flag(csk, CSK_TX_DATA_SENT); - } - - csk_set_flag(csk, CSK_ABORT_SHUTDOWN); - - if (!csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) { - sk->sk_err = ETIMEDOUT; - - if (!sock_flag(sk, SOCK_DEAD)) - sk->sk_error_report(sk); - - if (sk->sk_state == TCP_SYN_RECV && !abort_syn_rcv(sk, skb)) - return; - - chtls_release_resources(sk); - chtls_conn_done(sk); - } - - chtls_send_abort_rpl(sk, skb, BLOG_SKB_CB(skb)->cdev, - rst_status, queue); -} - -static void chtls_abort_rpl_rss(struct sock *sk, struct sk_buff *skb) -{ - struct cpl_abort_rpl_rss *rpl = cplhdr(skb) + RSS_HDR; - struct chtls_sock *csk; - struct chtls_dev *cdev; - - csk = rcu_dereference_sk_user_data(sk); - cdev = csk->cdev; - - if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) { - csk_reset_flag(csk, CSK_ABORT_RPL_PENDING); - if (!csk_flag_nochk(csk, CSK_ABORT_REQ_RCVD)) { - if (sk->sk_state == TCP_SYN_SENT) { - cxgb4_remove_tid(cdev->tids, - csk->port_id, - GET_TID(rpl), - sk->sk_family); - sock_put(sk); - } - chtls_release_resources(sk); - chtls_conn_done(sk); - } - } - kfree_skb(skb); -} - -static int chtls_conn_cpl(struct chtls_dev *cdev, struct sk_buff *skb) -{ - struct cpl_peer_close *req = cplhdr(skb) + RSS_HDR; - void (*fn)(struct sock *sk, struct sk_buff *skb); - unsigned int hwtid = GET_TID(req); - struct chtls_sock *csk; - struct sock *sk; - u8 opcode; - - opcode = ((const struct rss_header *)cplhdr(skb))->opcode; - - sk = lookup_tid(cdev->tids, hwtid); - if (!sk) - goto rel_skb; - - csk = sk->sk_user_data; - - switch (opcode) { - case CPL_PEER_CLOSE: - fn = chtls_peer_close; - break; - case CPL_CLOSE_CON_RPL: - fn = chtls_close_con_rpl; - break; - case CPL_ABORT_REQ_RSS: - /* - * Save the offload device in the skb, we may process this - * message after the socket has closed. - */ - BLOG_SKB_CB(skb)->cdev = csk->cdev; - fn = chtls_abort_req_rss; - break; - case CPL_ABORT_RPL_RSS: - fn = chtls_abort_rpl_rss; - break; - default: - goto rel_skb; - } - - process_cpl_msg(fn, sk, skb); - return 0; - -rel_skb: - kfree_skb(skb); - return 0; -} - -static void chtls_rx_ack(struct sock *sk, struct sk_buff *skb) -{ - struct cpl_fw4_ack *hdr = cplhdr(skb) + RSS_HDR; - struct chtls_sock *csk = sk->sk_user_data; - struct tcp_sock *tp = tcp_sk(sk); - u32 credits = hdr->credits; - u32 snd_una; - - snd_una = ntohl(hdr->snd_una); - csk->wr_credits += credits; - - if (csk->wr_unacked > csk->wr_max_credits - csk->wr_credits) - csk->wr_unacked = csk->wr_max_credits - csk->wr_credits; - - while (credits) { - struct sk_buff *pskb = csk->wr_skb_head; - u32 csum; - - if (unlikely(!pskb)) { - if (csk->wr_nondata) - csk->wr_nondata -= credits; - break; - } - csum = (__force u32)pskb->csum; - if (unlikely(credits < csum)) { - pskb->csum = (__force __wsum)(csum - credits); - break; - } - dequeue_wr(sk); - credits -= csum; - kfree_skb(pskb); - } - if (hdr->seq_vld & CPL_FW4_ACK_FLAGS_SEQVAL) { - if (unlikely(before(snd_una, tp->snd_una))) { - kfree_skb(skb); - return; - } - - if (tp->snd_una != snd_una) { - tp->snd_una = snd_una; - tp->rcv_tstamp = tcp_time_stamp(tp); - if (tp->snd_una == tp->snd_nxt && - !csk_flag_nochk(csk, CSK_TX_FAILOVER)) - csk_reset_flag(csk, CSK_TX_WAIT_IDLE); - } - } - - if (hdr->seq_vld & CPL_FW4_ACK_FLAGS_CH) { - unsigned int fclen16 = roundup(failover_flowc_wr_len, 16); - - csk->wr_credits -= fclen16; - csk_reset_flag(csk, CSK_TX_WAIT_IDLE); - csk_reset_flag(csk, CSK_TX_FAILOVER); - } - if (skb_queue_len(&csk->txq) && chtls_push_frames(csk, 0)) - sk->sk_write_space(sk); - - kfree_skb(skb); -} - -static int chtls_wr_ack(struct chtls_dev *cdev, struct sk_buff *skb) -{ - struct cpl_fw4_ack *rpl = cplhdr(skb) + RSS_HDR; - unsigned int hwtid = GET_TID(rpl); - struct sock *sk; - - sk = lookup_tid(cdev->tids, hwtid); - if (unlikely(!sk)) { - pr_err("can't find conn. for hwtid %u.\n", hwtid); - return -EINVAL; - } - process_cpl_msg(chtls_rx_ack, sk, skb); - - return 0; -} - -chtls_handler_func chtls_handlers[NUM_CPL_CMDS] = { - [CPL_PASS_OPEN_RPL] = chtls_pass_open_rpl, - [CPL_CLOSE_LISTSRV_RPL] = chtls_close_listsrv_rpl, - [CPL_PASS_ACCEPT_REQ] = chtls_pass_accept_req, - [CPL_PASS_ESTABLISH] = chtls_pass_establish, - [CPL_RX_DATA] = chtls_rx_data, - [CPL_TLS_DATA] = chtls_rx_pdu, - [CPL_RX_TLS_CMP] = chtls_rx_cmp, - [CPL_PEER_CLOSE] = chtls_conn_cpl, - [CPL_CLOSE_CON_RPL] = chtls_conn_cpl, - [CPL_ABORT_REQ_RSS] = chtls_conn_cpl, - [CPL_ABORT_RPL_RSS] = chtls_conn_cpl, - [CPL_FW4_ACK] = chtls_wr_ack, -}; diff --git a/drivers/crypto/chelsio/chtls/chtls_cm.h b/drivers/crypto/chelsio/chtls/chtls_cm.h deleted file mode 100644 index 47ba81e42f5d..000000000000 --- a/drivers/crypto/chelsio/chtls/chtls_cm.h +++ /dev/null @@ -1,222 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (c) 2018 Chelsio Communications, Inc. - */ - -#ifndef __CHTLS_CM_H__ -#define __CHTLS_CM_H__ - -/* - * TCB settings - */ -/* 3:0 */ -#define TCB_ULP_TYPE_W 0 -#define TCB_ULP_TYPE_S 0 -#define TCB_ULP_TYPE_M 0xfULL -#define TCB_ULP_TYPE_V(x) ((x) << TCB_ULP_TYPE_S) - -/* 11:4 */ -#define TCB_ULP_RAW_W 0 -#define TCB_ULP_RAW_S 4 -#define TCB_ULP_RAW_M 0xffULL -#define TCB_ULP_RAW_V(x) ((x) << TCB_ULP_RAW_S) - -#define TF_TLS_KEY_SIZE_S 7 -#define TF_TLS_KEY_SIZE_V(x) ((x) << TF_TLS_KEY_SIZE_S) - -#define TF_TLS_CONTROL_S 2 -#define TF_TLS_CONTROL_V(x) ((x) << TF_TLS_CONTROL_S) - -#define TF_TLS_ACTIVE_S 1 -#define TF_TLS_ACTIVE_V(x) ((x) << TF_TLS_ACTIVE_S) - -#define TF_TLS_ENABLE_S 0 -#define TF_TLS_ENABLE_V(x) ((x) << TF_TLS_ENABLE_S) - -#define TF_RX_QUIESCE_S 15 -#define TF_RX_QUIESCE_V(x) ((x) << TF_RX_QUIESCE_S) - -/* - * Max receive window supported by HW in bytes. Only a small part of it can - * be set through option0, the rest needs to be set through RX_DATA_ACK. - */ -#define MAX_RCV_WND ((1U << 27) - 1) -#define MAX_MSS 65536 - -/* - * Min receive window. We want it to be large enough to accommodate receive - * coalescing, handle jumbo frames, and not trigger sender SWS avoidance. - */ -#define MIN_RCV_WND (24 * 1024U) -#define LOOPBACK(x) (((x) & htonl(0xff000000)) == htonl(0x7f000000)) - -/* ulp_mem_io + ulptx_idata + payload + padding */ -#define MAX_IMM_ULPTX_WR_LEN (32 + 8 + 256 + 8) - -/* for TX: a skb must have a headroom of at least TX_HEADER_LEN bytes */ -#define TX_HEADER_LEN \ - (sizeof(struct fw_ofld_tx_data_wr) + sizeof(struct sge_opaque_hdr)) -#define TX_TLSHDR_LEN \ - (sizeof(struct fw_tlstx_data_wr) + sizeof(struct cpl_tx_tls_sfo) + \ - sizeof(struct sge_opaque_hdr)) -#define TXDATA_SKB_LEN 128 - -enum { - CPL_TX_TLS_SFO_TYPE_CCS, - CPL_TX_TLS_SFO_TYPE_ALERT, - CPL_TX_TLS_SFO_TYPE_HANDSHAKE, - CPL_TX_TLS_SFO_TYPE_DATA, - CPL_TX_TLS_SFO_TYPE_HEARTBEAT, -}; - -enum { - TLS_HDR_TYPE_CCS = 20, - TLS_HDR_TYPE_ALERT, - TLS_HDR_TYPE_HANDSHAKE, - TLS_HDR_TYPE_RECORD, - TLS_HDR_TYPE_HEARTBEAT, -}; - -typedef void (*defer_handler_t)(struct chtls_dev *dev, struct sk_buff *skb); -extern struct request_sock_ops chtls_rsk_ops; -extern struct request_sock_ops chtls_rsk_opsv6; - -struct deferred_skb_cb { - defer_handler_t handler; - struct chtls_dev *dev; -}; - -#define DEFERRED_SKB_CB(skb) ((struct deferred_skb_cb *)(skb)->cb) -#define failover_flowc_wr_len offsetof(struct fw_flowc_wr, mnemval[3]) -#define WR_SKB_CB(skb) ((struct wr_skb_cb *)(skb)->cb) -#define ACCEPT_QUEUE(sk) (&inet_csk(sk)->icsk_accept_queue.rskq_accept_head) - -#define SND_WSCALE(tp) ((tp)->rx_opt.snd_wscale) -#define RCV_WSCALE(tp) ((tp)->rx_opt.rcv_wscale) -#define USER_MSS(tp) ((tp)->rx_opt.user_mss) -#define TS_RECENT_STAMP(tp) ((tp)->rx_opt.ts_recent_stamp) -#define WSCALE_OK(tp) ((tp)->rx_opt.wscale_ok) -#define TSTAMP_OK(tp) ((tp)->rx_opt.tstamp_ok) -#define SACK_OK(tp) ((tp)->rx_opt.sack_ok) -#define INC_ORPHAN_COUNT(sk) percpu_counter_inc((sk)->sk_prot->orphan_count) - -/* TLS SKB */ -#define skb_ulp_tls_inline(skb) (ULP_SKB_CB(skb)->ulp.tls.ofld) -#define skb_ulp_tls_iv_imm(skb) (ULP_SKB_CB(skb)->ulp.tls.iv) - -void chtls_defer_reply(struct sk_buff *skb, struct chtls_dev *dev, - defer_handler_t handler); - -/* - * Returns true if the socket is in one of the supplied states. - */ -static inline unsigned int sk_in_state(const struct sock *sk, - unsigned int states) -{ - return states & (1 << sk->sk_state); -} - -static void chtls_rsk_destructor(struct request_sock *req) -{ - /* do nothing */ -} - -static inline void chtls_init_rsk_ops(struct proto *chtls_tcp_prot, - struct request_sock_ops *chtls_tcp_ops, - struct proto *tcp_prot, int family) -{ - memset(chtls_tcp_ops, 0, sizeof(*chtls_tcp_ops)); - chtls_tcp_ops->family = family; - chtls_tcp_ops->obj_size = sizeof(struct tcp_request_sock); - chtls_tcp_ops->destructor = chtls_rsk_destructor; - chtls_tcp_ops->slab = tcp_prot->rsk_prot->slab; - chtls_tcp_prot->rsk_prot = chtls_tcp_ops; -} - -static inline void chtls_reqsk_free(struct request_sock *req) -{ - if (req->rsk_listener) - sock_put(req->rsk_listener); - kmem_cache_free(req->rsk_ops->slab, req); -} - -#define DECLARE_TASK_FUNC(task, task_param) \ - static void task(struct work_struct *task_param) - -static inline void sk_wakeup_sleepers(struct sock *sk, bool interruptable) -{ - struct socket_wq *wq; - - rcu_read_lock(); - wq = rcu_dereference(sk->sk_wq); - if (skwq_has_sleeper(wq)) { - if (interruptable) - wake_up_interruptible(sk_sleep(sk)); - else - wake_up_all(sk_sleep(sk)); - } - rcu_read_unlock(); -} - -static inline void chtls_set_req_port(struct request_sock *oreq, - __be16 source, __be16 dest) -{ - inet_rsk(oreq)->ir_rmt_port = source; - inet_rsk(oreq)->ir_num = ntohs(dest); -} - -static inline void chtls_set_req_addr(struct request_sock *oreq, - __be32 local_ip, __be32 peer_ip) -{ - inet_rsk(oreq)->ir_loc_addr = local_ip; - inet_rsk(oreq)->ir_rmt_addr = peer_ip; -} - -static inline void chtls_free_skb(struct sock *sk, struct sk_buff *skb) -{ - skb_dst_set(skb, NULL); - __skb_unlink(skb, &sk->sk_receive_queue); - __kfree_skb(skb); -} - -static inline void chtls_kfree_skb(struct sock *sk, struct sk_buff *skb) -{ - skb_dst_set(skb, NULL); - __skb_unlink(skb, &sk->sk_receive_queue); - kfree_skb(skb); -} - -static inline void chtls_reset_wr_list(struct chtls_sock *csk) -{ - csk->wr_skb_head = NULL; - csk->wr_skb_tail = NULL; -} - -static inline void enqueue_wr(struct chtls_sock *csk, struct sk_buff *skb) -{ - WR_SKB_CB(skb)->next_wr = NULL; - - skb_get(skb); - - if (!csk->wr_skb_head) - csk->wr_skb_head = skb; - else - WR_SKB_CB(csk->wr_skb_tail)->next_wr = skb; - csk->wr_skb_tail = skb; -} - -static inline struct sk_buff *dequeue_wr(struct sock *sk) -{ - struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); - struct sk_buff *skb = NULL; - - skb = csk->wr_skb_head; - - if (likely(skb)) { - /* Don't bother clearing the tail */ - csk->wr_skb_head = WR_SKB_CB(skb)->next_wr; - WR_SKB_CB(skb)->next_wr = NULL; - } - return skb; -} -#endif diff --git a/drivers/crypto/chelsio/chtls/chtls_hw.c b/drivers/crypto/chelsio/chtls/chtls_hw.c deleted file mode 100644 index f1820aca0d33..000000000000 --- a/drivers/crypto/chelsio/chtls/chtls_hw.c +++ /dev/null @@ -1,426 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (c) 2018 Chelsio Communications, Inc. - * - * Written by: Atul Gupta ([email protected]) - */ - -#include <linux/module.h> -#include <linux/list.h> -#include <linux/workqueue.h> -#include <linux/skbuff.h> -#include <linux/timer.h> -#include <linux/notifier.h> -#include <linux/inetdevice.h> -#include <linux/ip.h> -#include <linux/tcp.h> -#include <linux/tls.h> -#include <net/tls.h> - -#include "chtls.h" -#include "chtls_cm.h" - -static void __set_tcb_field_direct(struct chtls_sock *csk, - struct cpl_set_tcb_field *req, u16 word, - u64 mask, u64 val, u8 cookie, int no_reply) -{ - struct ulptx_idata *sc; - - INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, csk->tid); - req->wr.wr_mid |= htonl(FW_WR_FLOWID_V(csk->tid)); - req->reply_ctrl = htons(NO_REPLY_V(no_reply) | - QUEUENO_V(csk->rss_qid)); - req->word_cookie = htons(TCB_WORD_V(word) | TCB_COOKIE_V(cookie)); - req->mask = cpu_to_be64(mask); - req->val = cpu_to_be64(val); - sc = (struct ulptx_idata *)(req + 1); - sc->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP)); - sc->len = htonl(0); -} - -static void __set_tcb_field(struct sock *sk, struct sk_buff *skb, u16 word, - u64 mask, u64 val, u8 cookie, int no_reply) -{ - struct cpl_set_tcb_field *req; - struct chtls_sock *csk; - struct ulptx_idata *sc; - unsigned int wrlen; - - wrlen = roundup(sizeof(*req) + sizeof(*sc), 16); - csk = rcu_dereference_sk_user_data(sk); - - req = (struct cpl_set_tcb_field *)__skb_put(skb, wrlen); - __set_tcb_field_direct(csk, req, word, mask, val, cookie, no_reply); - set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id); -} - -/* - * Send control message to HW, message go as immediate data and packet - * is freed immediately. - */ -static int chtls_set_tcb_field(struct sock *sk, u16 word, u64 mask, u64 val) -{ - struct cpl_set_tcb_field *req; - unsigned int credits_needed; - struct chtls_sock *csk; - struct ulptx_idata *sc; - struct sk_buff *skb; - unsigned int wrlen; - int ret; - - wrlen = roundup(sizeof(*req) + sizeof(*sc), 16); - - skb = alloc_skb(wrlen, GFP_ATOMIC); - if (!skb) - return -ENOMEM; - - credits_needed = DIV_ROUND_UP(wrlen, 16); - csk = rcu_dereference_sk_user_data(sk); - - __set_tcb_field(sk, skb, word, mask, val, 0, 1); - skb_set_queue_mapping(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA); - csk->wr_credits -= credits_needed; - csk->wr_unacked += credits_needed; - enqueue_wr(csk, skb); - ret = cxgb4_ofld_send(csk->egress_dev, skb); - if (ret < 0) - kfree_skb(skb); - return ret < 0 ? ret : 0; -} - -/* - * Set one of the t_flags bits in the TCB. - */ -int chtls_set_tcb_tflag(struct sock *sk, unsigned int bit_pos, int val) -{ - return chtls_set_tcb_field(sk, 1, 1ULL << bit_pos, - (u64)val << bit_pos); -} - -static int chtls_set_tcb_keyid(struct sock *sk, int keyid) -{ - return chtls_set_tcb_field(sk, 31, 0xFFFFFFFFULL, keyid); -} - -static int chtls_set_tcb_seqno(struct sock *sk) -{ - return chtls_set_tcb_field(sk, 28, ~0ULL, 0); -} - -static int chtls_set_tcb_quiesce(struct sock *sk, int val) -{ - return chtls_set_tcb_field(sk, 1, (1ULL << TF_RX_QUIESCE_S), - TF_RX_QUIESCE_V(val)); -} - -/* TLS Key bitmap processing */ -int chtls_init_kmap(struct chtls_dev *cdev, struct cxgb4_lld_info *lldi) -{ - unsigned int num_key_ctx, bsize; - int ksize; - - num_key_ctx = (lldi->vr->key.size / TLS_KEY_CONTEXT_SZ); - bsize = BITS_TO_LONGS(num_key_ctx); - - cdev->kmap.size = num_key_ctx; - cdev->kmap.available = bsize; - ksize = sizeof(*cdev->kmap.addr) * bsize; - cdev->kmap.addr = kvzalloc(ksize, GFP_KERNEL); - if (!cdev->kmap.addr) - return -ENOMEM; - - cdev->kmap.start = lldi->vr->key.start; - spin_lock_init(&cdev->kmap.lock); - return 0; -} - -static int get_new_keyid(struct chtls_sock *csk, u32 optname) -{ - struct net_device *dev = csk->egress_dev; - struct chtls_dev *cdev = csk->cdev; - struct chtls_hws *hws; - struct adapter *adap; - int keyid; - - adap = netdev2adap(dev); - hws = &csk->tlshws; - - spin_lock_bh(&cdev->kmap.lock); - keyid = find_first_zero_bit(cdev->kmap.addr, cdev->kmap.size); - if (keyid < cdev->kmap.size) { - __set_bit(keyid, cdev->kmap.addr); - if (optname == TLS_RX) - hws->rxkey = keyid; - else - hws->txkey = keyid; - atomic_inc(&adap->chcr_stats.tls_key); - } else { - keyid = -1; - } - spin_unlock_bh(&cdev->kmap.lock); - return keyid; -} - -void free_tls_keyid(struct sock *sk) -{ - struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); - struct net_device *dev = csk->egress_dev; - struct chtls_dev *cdev = csk->cdev; - struct chtls_hws *hws; - struct adapter *adap; - - if (!cdev->kmap.addr) - return; - - adap = netdev2adap(dev); - hws = &csk->tlshws; - - spin_lock_bh(&cdev->kmap.lock); - if (hws->rxkey >= 0) { - __clear_bit(hws->rxkey, cdev->kmap.addr); - atomic_dec(&adap->chcr_stats.tls_key); - hws->rxkey = -1; - } - if (hws->txkey >= 0) { - __clear_bit(hws->txkey, cdev->kmap.addr); - atomic_dec(&adap->chcr_stats.tls_key); - hws->txkey = -1; - } - spin_unlock_bh(&cdev->kmap.lock); -} - -unsigned int keyid_to_addr(int start_addr, int keyid) -{ - return (start_addr + (keyid * TLS_KEY_CONTEXT_SZ)) >> 5; -} - -static void chtls_rxkey_ivauth(struct _key_ctx *kctx) -{ - kctx->iv_to_auth = cpu_to_be64(KEYCTX_TX_WR_IV_V(6ULL) | - KEYCTX_TX_WR_AAD_V(1ULL) | - KEYCTX_TX_WR_AADST_V(5ULL) | - KEYCTX_TX_WR_CIPHER_V(14ULL) | - KEYCTX_TX_WR_CIPHERST_V(0ULL) | - KEYCTX_TX_WR_AUTH_V(14ULL) | - KEYCTX_TX_WR_AUTHST_V(16ULL) | - KEYCTX_TX_WR_AUTHIN_V(16ULL)); -} - -static int chtls_key_info(struct chtls_sock *csk, - struct _key_ctx *kctx, - u32 keylen, u32 optname, - int cipher_type) -{ - unsigned char key[AES_MAX_KEY_SIZE]; - unsigned char *key_p, *salt; - unsigned char ghash_h[AEAD_H_SIZE]; - int ck_size, key_ctx_size, kctx_mackey_size, salt_size; - struct crypto_aes_ctx aes; - int ret; - - key_ctx_size = sizeof(struct _key_ctx) + - roundup(keylen, 16) + AEAD_H_SIZE; - - /* GCM mode of AES supports 128 and 256 bit encryption, so - * prepare key context base on GCM cipher type - */ - switch (cipher_type) { - case TLS_CIPHER_AES_GCM_128: { - struct tls12_crypto_info_aes_gcm_128 *gcm_ctx_128 = - (struct tls12_crypto_info_aes_gcm_128 *) - &csk->tlshws.crypto_info; - memcpy(key, gcm_ctx_128->key, keylen); - - key_p = gcm_ctx_128->key; - salt = gcm_ctx_128->salt; - ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; - salt_size = TLS_CIPHER_AES_GCM_128_SALT_SIZE; - kctx_mackey_size = CHCR_KEYCTX_MAC_KEY_SIZE_128; - break; - } - case TLS_CIPHER_AES_GCM_256: { - struct tls12_crypto_info_aes_gcm_256 *gcm_ctx_256 = - (struct tls12_crypto_info_aes_gcm_256 *) - &csk->tlshws.crypto_info; - memcpy(key, gcm_ctx_256->key, keylen); - - key_p = gcm_ctx_256->key; - salt = gcm_ctx_256->salt; - ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; - salt_size = TLS_CIPHER_AES_GCM_256_SALT_SIZE; - kctx_mackey_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; - break; - } - default: - pr_err("GCM: Invalid key length %d\n", keylen); - return -EINVAL; - } - - /* Calculate the H = CIPH(K, 0 repeated 16 times). - * It will go in key context - */ - ret = aes_expandkey(&aes, key, keylen); - if (ret) - return ret; - - memset(ghash_h, 0, AEAD_H_SIZE); - aes_encrypt(&aes, ghash_h, ghash_h); - memzero_explicit(&aes, sizeof(aes)); - csk->tlshws.keylen = key_ctx_size; - - /* Copy the Key context */ - if (optname == TLS_RX) { - int key_ctx; - - key_ctx = ((key_ctx_size >> 4) << 3); - kctx->ctx_hdr = FILL_KEY_CRX_HDR(ck_size, - kctx_mackey_size, - 0, 0, key_ctx); - chtls_rxkey_ivauth(kctx); - } else { - kctx->ctx_hdr = FILL_KEY_CTX_HDR(ck_size, - kctx_mackey_size, - 0, 0, key_ctx_size >> 4); - } - - memcpy(kctx->salt, salt, salt_size); - memcpy(kctx->key, key_p, keylen); - memcpy(kctx->key + keylen, ghash_h, AEAD_H_SIZE); - /* erase key info from driver */ - memset(key_p, 0, keylen); - - return 0; -} - -static void chtls_set_scmd(struct chtls_sock *csk) -{ - struct chtls_hws *hws = &csk->tlshws; - - hws->scmd.seqno_numivs = - SCMD_SEQ_NO_CTRL_V(3) | - SCMD_PROTO_VERSION_V(0) | - SCMD_ENC_DEC_CTRL_V(0) | - SCMD_CIPH_AUTH_SEQ_CTRL_V(1) | - SCMD_CIPH_MODE_V(2) | - SCMD_AUTH_MODE_V(4) | - SCMD_HMAC_CTRL_V(0) | - SCMD_IV_SIZE_V(4) | - SCMD_NUM_IVS_V(1); - - hws->scmd.ivgen_hdrlen = - SCMD_IV_GEN_CTRL_V(1) | - SCMD_KEY_CTX_INLINE_V(0) | - SCMD_TLS_FRAG_ENABLE_V(1); -} - -int chtls_setkey(struct chtls_sock *csk, u32 keylen, - u32 optname, int cipher_type) -{ - struct tls_key_req *kwr; - struct chtls_dev *cdev; - struct _key_ctx *kctx; - int wrlen, klen, len; - struct sk_buff *skb; - struct sock *sk; - int keyid; - int kaddr; - int ret; - - cdev = csk->cdev; - sk = csk->sk; - - klen = roundup((keylen + AEAD_H_SIZE) + sizeof(*kctx), 32); - wrlen = roundup(sizeof(*kwr), 16); - len = klen + wrlen; - - /* Flush out-standing data before new key takes effect */ - if (optname == TLS_TX) { - lock_sock(sk); - if (skb_queue_len(&csk->txq)) - chtls_push_frames(csk, 0); - release_sock(sk); - } - - skb = alloc_skb(len, GFP_KERNEL); - if (!skb) - return -ENOMEM; - - keyid = get_new_keyid(csk, optname); - if (keyid < 0) { - ret = -ENOSPC; - goto out_nokey; - } - - kaddr = keyid_to_addr(cdev->kmap.start, keyid); - kwr = (struct tls_key_req *)__skb_put_zero(skb, len); - kwr->wr.op_to_compl = - cpu_to_be32(FW_WR_OP_V(FW_ULPTX_WR) | FW_WR_COMPL_F | - FW_WR_ATOMIC_V(1U)); - kwr->wr.flowid_len16 = - cpu_to_be32(FW_WR_LEN16_V(DIV_ROUND_UP(len, 16) | - FW_WR_FLOWID_V(csk->tid))); - kwr->wr.protocol = 0; - kwr->wr.mfs = htons(TLS_MFS); - kwr->wr.reneg_to_write_rx = optname; - - /* ulptx command */ - kwr->req.cmd = cpu_to_be32(ULPTX_CMD_V(ULP_TX_MEM_WRITE) | - T5_ULP_MEMIO_ORDER_V(1) | - T5_ULP_MEMIO_IMM_V(1)); - kwr->req.len16 = cpu_to_be32((csk->tid << 8) | - DIV_ROUND_UP(len - sizeof(kwr->wr), 16)); - kwr->req.dlen = cpu_to_be32(ULP_MEMIO_DATA_LEN_V(klen >> 5)); - kwr->req.lock_addr = cpu_to_be32(ULP_MEMIO_ADDR_V(kaddr)); - - /* sub command */ - kwr->sc_imm.cmd_more = cpu_to_be32(ULPTX_CMD_V(ULP_TX_SC_IMM)); - kwr->sc_imm.len = cpu_to_be32(klen); - - lock_sock(sk); - /* key info */ - kctx = (struct _key_ctx *)(kwr + 1); - ret = chtls_key_info(csk, kctx, keylen, optname, cipher_type); - if (ret) - goto out_notcb; - - set_wr_txq(skb, CPL_PRIORITY_DATA, csk->tlshws.txqid); - csk->wr_credits -= DIV_ROUND_UP(len, 16); - csk->wr_unacked += DIV_ROUND_UP(len, 16); - enqueue_wr(csk, skb); - cxgb4_ofld_send(csk->egress_dev, skb); - - chtls_set_scmd(csk); - /* Clear quiesce for Rx key */ - if (optname == TLS_RX) { - ret = chtls_set_tcb_keyid(sk, keyid); - if (ret) - goto out_notcb; - ret = chtls_set_tcb_field(sk, 0, - TCB_ULP_RAW_V(TCB_ULP_RAW_M), - TCB_ULP_RAW_V((TF_TLS_KEY_SIZE_V(1) | - TF_TLS_CONTROL_V(1) | - TF_TLS_ACTIVE_V(1) | - TF_TLS_ENABLE_V(1)))); - if (ret) - goto out_notcb; - ret = chtls_set_tcb_seqno(sk); - if (ret) - goto out_notcb; - ret = chtls_set_tcb_quiesce(sk, 0); - if (ret) - goto out_notcb; - csk->tlshws.rxkey = keyid; - } else { - csk->tlshws.tx_seq_no = 0; - csk->tlshws.txkey = keyid; - } - - release_sock(sk); - return ret; -out_notcb: - release_sock(sk); - free_tls_keyid(sk); -out_nokey: - kfree_skb(skb); - return ret; -} diff --git a/drivers/crypto/chelsio/chtls/chtls_io.c b/drivers/crypto/chelsio/chtls/chtls_io.c deleted file mode 100644 index 2e9acae1cba3..000000000000 --- a/drivers/crypto/chelsio/chtls/chtls_io.c +++ /dev/null @@ -1,1907 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (c) 2018 Chelsio Communications, Inc. - * - * Written by: Atul Gupta ([email protected]) - */ - -#include <linux/module.h> -#include <linux/list.h> -#include <linux/workqueue.h> -#include <linux/skbuff.h> -#include <linux/timer.h> -#include <linux/notifier.h> -#include <linux/inetdevice.h> -#include <linux/ip.h> -#include <linux/tcp.h> -#include <linux/sched/signal.h> -#include <net/tcp.h> -#include <net/busy_poll.h> -#include <crypto/aes.h> - -#include "chtls.h" -#include "chtls_cm.h" - -static bool is_tls_tx(struct chtls_sock *csk) -{ - return csk->tlshws.txkey >= 0; -} - -static bool is_tls_rx(struct chtls_sock *csk) -{ - return csk->tlshws.rxkey >= 0; -} - -static int data_sgl_len(const struct sk_buff *skb) -{ - unsigned int cnt; - - cnt = skb_shinfo(skb)->nr_frags; - return sgl_len(cnt) * 8; -} - -static int nos_ivs(struct sock *sk, unsigned int size) -{ - struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); - - return DIV_ROUND_UP(size, csk->tlshws.mfs); -} - -static int set_ivs_imm(struct sock *sk, const struct sk_buff *skb) -{ - int ivs_size = nos_ivs(sk, skb->len) * CIPHER_BLOCK_SIZE; - int hlen = TLS_WR_CPL_LEN + data_sgl_len(skb); - - if ((hlen + KEY_ON_MEM_SZ + ivs_size) < - MAX_IMM_OFLD_TX_DATA_WR_LEN) { - ULP_SKB_CB(skb)->ulp.tls.iv = 1; - return 1; - } - ULP_SKB_CB(skb)->ulp.tls.iv = 0; - return 0; -} - -static int max_ivs_size(struct sock *sk, int size) -{ - return nos_ivs(sk, size) * CIPHER_BLOCK_SIZE; -} - -static int ivs_size(struct sock *sk, const struct sk_buff *skb) -{ - return set_ivs_imm(sk, skb) ? (nos_ivs(sk, skb->len) * - CIPHER_BLOCK_SIZE) : 0; -} - -static int flowc_wr_credits(int nparams, int *flowclenp) -{ - int flowclen16, flowclen; - - flowclen = offsetof(struct fw_flowc_wr, mnemval[nparams]); - flowclen16 = DIV_ROUND_UP(flowclen, 16); - flowclen = flowclen16 * 16; - - if (flowclenp) - *flowclenp = flowclen; - - return flowclen16; -} - -static struct sk_buff *create_flowc_wr_skb(struct sock *sk, - struct fw_flowc_wr *flowc, - int flowclen) -{ - struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); - struct sk_buff *skb; - - skb = alloc_skb(flowclen, GFP_ATOMIC); - if (!skb) - return NULL; - - __skb_put_data(skb, flowc, flowclen); - skb_set_queue_mapping(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA); - - return skb; -} - -static int send_flowc_wr(struct sock *sk, struct fw_flowc_wr *flowc, - int flowclen) -{ - struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); - struct tcp_sock *tp = tcp_sk(sk); - struct sk_buff *skb; - int flowclen16; - int ret; - - flowclen16 = flowclen / 16; - - if (csk_flag(sk, CSK_TX_DATA_SENT)) { - skb = create_flowc_wr_skb(sk, flowc, flowclen); - if (!skb) - return -ENOMEM; - - skb_entail(sk, skb, - ULPCB_FLAG_NO_HDR | ULPCB_FLAG_NO_APPEND); - return 0; - } - - ret = cxgb4_immdata_send(csk->egress_dev, - csk->txq_idx, - flowc, flowclen); - if (!ret) - return flowclen16; - skb = create_flowc_wr_skb(sk, flowc, flowclen); - if (!skb) - return -ENOMEM; - send_or_defer(sk, tp, skb, 0); - return flowclen16; -} - -static u8 tcp_state_to_flowc_state(u8 state) -{ - switch (state) { - case TCP_ESTABLISHED: - return FW_FLOWC_MNEM_TCPSTATE_ESTABLISHED; - case TCP_CLOSE_WAIT: - return FW_FLOWC_MNEM_TCPSTATE_CLOSEWAIT; - case TCP_FIN_WAIT1: - return FW_FLOWC_MNEM_TCPSTATE_FINWAIT1; - case TCP_CLOSING: - return FW_FLOWC_MNEM_TCPSTATE_CLOSING; - case TCP_LAST_ACK: - return FW_FLOWC_MNEM_TCPSTATE_LASTACK; - case TCP_FIN_WAIT2: - return FW_FLOWC_MNEM_TCPSTATE_FINWAIT2; - } - - return FW_FLOWC_MNEM_TCPSTATE_ESTABLISHED; -} - -int send_tx_flowc_wr(struct sock *sk, int compl, - u32 snd_nxt, u32 rcv_nxt) -{ - struct flowc_packed { - struct fw_flowc_wr fc; - struct fw_flowc_mnemval mnemval[FW_FLOWC_MNEM_MAX]; - } __packed sflowc; - int nparams, paramidx, flowclen16, flowclen; - struct fw_flowc_wr *flowc; - struct chtls_sock *csk; - struct tcp_sock *tp; - - csk = rcu_dereference_sk_user_data(sk); - tp = tcp_sk(sk); - memset(&sflowc, 0, sizeof(sflowc)); - flowc = &sflowc.fc; - -#define FLOWC_PARAM(__m, __v) \ - do { \ - flowc->mnemval[paramidx].mnemonic = FW_FLOWC_MNEM_##__m; \ - flowc->mnemval[paramidx].val = cpu_to_be32(__v); \ - paramidx++; \ - } while (0) - - paramidx = 0; - - FLOWC_PARAM(PFNVFN, FW_PFVF_CMD_PFN_V(csk->cdev->lldi->pf)); - FLOWC_PARAM(CH, csk->tx_chan); - FLOWC_PARAM(PORT, csk->tx_chan); - FLOWC_PARAM(IQID, csk->rss_qid); - FLOWC_PARAM(SNDNXT, tp->snd_nxt); - FLOWC_PARAM(RCVNXT, tp->rcv_nxt); - FLOWC_PARAM(SNDBUF, csk->sndbuf); - FLOWC_PARAM(MSS, tp->mss_cache); - FLOWC_PARAM(TCPSTATE, tcp_state_to_flowc_state(sk->sk_state)); - - if (SND_WSCALE(tp)) - FLOWC_PARAM(RCV_SCALE, SND_WSCALE(tp)); - - if (csk->ulp_mode == ULP_MODE_TLS) - FLOWC_PARAM(ULD_MODE, ULP_MODE_TLS); - - if (csk->tlshws.fcplenmax) - FLOWC_PARAM(TXDATAPLEN_MAX, csk->tlshws.fcplenmax); - - nparams = paramidx; -#undef FLOWC_PARAM - - flowclen16 = flowc_wr_credits(nparams, &flowclen); - flowc->op_to_nparams = - cpu_to_be32(FW_WR_OP_V(FW_FLOWC_WR) | - FW_WR_COMPL_V(compl) | - FW_FLOWC_WR_NPARAMS_V(nparams)); - flowc->flowid_len16 = cpu_to_be32(FW_WR_LEN16_V(flowclen16) | - FW_WR_FLOWID_V(csk->tid)); - - return send_flowc_wr(sk, flowc, flowclen); -} - -/* Copy IVs to WR */ -static int tls_copy_ivs(struct sock *sk, struct sk_buff *skb) - -{ - struct chtls_sock *csk; - unsigned char *iv_loc; - struct chtls_hws *hws; - unsigned char *ivs; - u16 number_of_ivs; - struct page *page; - int err = 0; - - csk = rcu_dereference_sk_user_data(sk); - hws = &csk->tlshws; - number_of_ivs = nos_ivs(sk, skb->len); - - if (number_of_ivs > MAX_IVS_PAGE) { - pr_warn("MAX IVs in PAGE exceeded %d\n", number_of_ivs); - return -ENOMEM; - } - - /* generate the IVs */ - ivs = kmalloc_array(CIPHER_BLOCK_SIZE, number_of_ivs, GFP_ATOMIC); - if (!ivs) - return -ENOMEM; - get_random_bytes(ivs, number_of_ivs * CIPHER_BLOCK_SIZE); - - if (skb_ulp_tls_iv_imm(skb)) { - /* send the IVs as immediate data in the WR */ - iv_loc = (unsigned char *)__skb_push(skb, number_of_ivs * - CIPHER_BLOCK_SIZE); - if (iv_loc) - memcpy(iv_loc, ivs, number_of_ivs * CIPHER_BLOCK_SIZE); - - hws->ivsize = number_of_ivs * CIPHER_BLOCK_SIZE; - } else { - /* Send the IVs as sgls */ - /* Already accounted IV DSGL for credits */ - skb_shinfo(skb)->nr_frags--; - page = alloc_pages(sk->sk_allocation | __GFP_COMP, 0); - if (!page) { - pr_info("%s : Page allocation for IVs failed\n", - __func__); - err = -ENOMEM; - goto out; - } - memcpy(page_address(page), ivs, number_of_ivs * - CIPHER_BLOCK_SIZE); - skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, page, 0, - number_of_ivs * CIPHER_BLOCK_SIZE); - hws->ivsize = 0; - } -out: - kfree(ivs); - return err; -} - -/* Copy Key to WR */ -static void tls_copy_tx_key(struct sock *sk, struct sk_buff *skb) -{ - struct ulptx_sc_memrd *sc_memrd; - struct chtls_sock *csk; - struct chtls_dev *cdev; - struct ulptx_idata *sc; - struct chtls_hws *hws; - u32 immdlen; - int kaddr; - - csk = rcu_dereference_sk_user_data(sk); - hws = &csk->tlshws; - cdev = csk->cdev; - - immdlen = sizeof(*sc) + sizeof(*sc_memrd); - kaddr = keyid_to_addr(cdev->kmap.start, hws->txkey); - sc = (struct ulptx_idata *)__skb_push(skb, immdlen); - if (sc) { - sc->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP)); - sc->len = htonl(0); - sc_memrd = (struct ulptx_sc_memrd *)(sc + 1); - sc_memrd->cmd_to_len = - htonl(ULPTX_CMD_V(ULP_TX_SC_MEMRD) | - ULP_TX_SC_MORE_V(1) | - ULPTX_LEN16_V(hws->keylen >> 4)); - sc_memrd->addr = htonl(kaddr); - } -} - -static u64 tlstx_incr_seqnum(struct chtls_hws *hws) -{ - return hws->tx_seq_no++; -} - -static bool is_sg_request(const struct sk_buff *skb) -{ - return skb->peeked || - (skb->len > MAX_IMM_ULPTX_WR_LEN); -} - -/* - * Returns true if an sk_buff carries urgent data. - */ -static bool skb_urgent(struct sk_buff *skb) -{ - return ULP_SKB_CB(skb)->flags & ULPCB_FLAG_URG; -} - -/* TLS content type for CPL SFO */ -static unsigned char tls_content_type(unsigned char content_type) -{ - switch (content_type) { - case TLS_HDR_TYPE_CCS: - return CPL_TX_TLS_SFO_TYPE_CCS; - case TLS_HDR_TYPE_ALERT: - return CPL_TX_TLS_SFO_TYPE_ALERT; - case TLS_HDR_TYPE_HANDSHAKE: - return CPL_TX_TLS_SFO_TYPE_HANDSHAKE; - case TLS_HDR_TYPE_HEARTBEAT: - return CPL_TX_TLS_SFO_TYPE_HEARTBEAT; - } - return CPL_TX_TLS_SFO_TYPE_DATA; -} - -static void tls_tx_data_wr(struct sock *sk, struct sk_buff *skb, - int dlen, int tls_immd, u32 credits, - int expn, int pdus) -{ - struct fw_tlstx_data_wr *req_wr; - struct cpl_tx_tls_sfo *req_cpl; - unsigned int wr_ulp_mode_force; - struct tls_scmd *updated_scmd; - unsigned char data_type; - struct chtls_sock *csk; - struct net_device *dev; - struct chtls_hws *hws; - struct tls_scmd *scmd; - struct adapter *adap; - unsigned char *req; - int immd_len; - int iv_imm; - int len; - - csk = rcu_dereference_sk_user_data(sk); - iv_imm = skb_ulp_tls_iv_imm(skb); - dev = csk->egress_dev; - adap = netdev2adap(dev); - hws = &csk->tlshws; - scmd = &hws->scmd; - len = dlen + expn; - - dlen = (dlen < hws->mfs) ? dlen : hws->mfs; - atomic_inc(&adap->chcr_stats.tls_pdu_tx); - - updated_scmd = scmd; - updated_scmd->seqno_numivs &= 0xffffff80; - updated_scmd->seqno_numivs |= SCMD_NUM_IVS_V(pdus); - hws->scmd = *updated_scmd; - - req = (unsigned char *)__skb_push(skb, sizeof(struct cpl_tx_tls_sfo)); - req_cpl = (struct cpl_tx_tls_sfo *)req; - req = (unsigned char *)__skb_push(skb, (sizeof(struct - fw_tlstx_data_wr))); - - req_wr = (struct fw_tlstx_data_wr *)req; - immd_len = (tls_immd ? dlen : 0); - req_wr->op_to_immdlen = - htonl(FW_WR_OP_V(FW_TLSTX_DATA_WR) | - FW_TLSTX_DATA_WR_COMPL_V(1) | - FW_TLSTX_DATA_WR_IMMDLEN_V(immd_len)); - req_wr->flowid_len16 = htonl(FW_TLSTX_DATA_WR_FLOWID_V(csk->tid) | - FW_TLSTX_DATA_WR_LEN16_V(credits)); - wr_ulp_mode_force = TX_ULP_MODE_V(ULP_MODE_TLS); - - if (is_sg_request(skb)) - wr_ulp_mode_force |= FW_OFLD_TX_DATA_WR_ALIGNPLD_F | - ((tcp_sk(sk)->nonagle & TCP_NAGLE_OFF) ? 0 : - FW_OFLD_TX_DATA_WR_SHOVE_F); - - req_wr->lsodisable_to_flags = - htonl(TX_ULP_MODE_V(ULP_MODE_TLS) | - TX_URG_V(skb_urgent(skb)) | - T6_TX_FORCE_F | wr_ulp_mode_force | - TX_SHOVE_V((!csk_flag(sk, CSK_TX_MORE_DATA)) && - skb_queue_empty(&csk->txq))); - - req_wr->ctxloc_to_exp = - htonl(FW_TLSTX_DATA_WR_NUMIVS_V(pdus) | - FW_TLSTX_DATA_WR_EXP_V(expn) | - FW_TLSTX_DATA_WR_CTXLOC_V(CHTLS_KEY_CONTEXT_DDR) | - FW_TLSTX_DATA_WR_IVDSGL_V(!iv_imm) | - FW_TLSTX_DATA_WR_KEYSIZE_V(hws->keylen >> 4)); - - /* Fill in the length */ - req_wr->plen = htonl(len); - req_wr->mfs = htons(hws->mfs); - req_wr->adjustedplen_pkd = - htons(FW_TLSTX_DATA_WR_ADJUSTEDPLEN_V(hws->adjustlen)); - req_wr->expinplenmax_pkd = - htons(FW_TLSTX_DATA_WR_EXPINPLENMAX_V(hws->expansion)); - req_wr->pdusinplenmax_pkd = - FW_TLSTX_DATA_WR_PDUSINPLENMAX_V(hws->pdus); - req_wr->r10 = 0; - - data_type = tls_content_type(ULP_SKB_CB(skb)->ulp.tls.type); - req_cpl->op_to_seg_len = htonl(CPL_TX_TLS_SFO_OPCODE_V(CPL_TX_TLS_SFO) | - CPL_TX_TLS_SFO_DATA_TYPE_V(data_type) | - CPL_TX_TLS_SFO_CPL_LEN_V(2) | - CPL_TX_TLS_SFO_SEG_LEN_V(dlen)); - req_cpl->pld_len = htonl(len - expn); - - req_cpl->type_protover = htonl(CPL_TX_TLS_SFO_TYPE_V - ((data_type == CPL_TX_TLS_SFO_TYPE_HEARTBEAT) ? - TLS_HDR_TYPE_HEARTBEAT : 0) | - CPL_TX_TLS_SFO_PROTOVER_V(0)); - - /* create the s-command */ - req_cpl->r1_lo = 0; - req_cpl->seqno_numivs = cpu_to_be32(hws->scmd.seqno_numivs); - req_cpl->ivgen_hdrlen = cpu_to_be32(hws->scmd.ivgen_hdrlen); - req_cpl->scmd1 = cpu_to_be64(tlstx_incr_seqnum(hws)); -} - -/* - * Calculate the TLS data expansion size - */ -static int chtls_expansion_size(struct sock *sk, int data_len, - int fullpdu, - unsigned short *pducnt) -{ - struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); - struct chtls_hws *hws = &csk->tlshws; - struct tls_scmd *scmd = &hws->scmd; - int fragsize = hws->mfs; - int expnsize = 0; - int fragleft; - int fragcnt; - int expppdu; - - if (SCMD_CIPH_MODE_G(scmd->seqno_numivs) == - SCMD_CIPH_MODE_AES_GCM) { - expppdu = GCM_TAG_SIZE + AEAD_EXPLICIT_DATA_SIZE + - TLS_HEADER_LENGTH; - - if (fullpdu) { - *pducnt = data_len / (expppdu + fragsize); - if (*pducnt > 32) - *pducnt = 32; - else if (!*pducnt) - *pducnt = 1; - expnsize = (*pducnt) * expppdu; - return expnsize; - } - fragcnt = (data_len / fragsize); - expnsize = fragcnt * expppdu; - fragleft = data_len % fragsize; - if (fragleft > 0) - expnsize += expppdu; - } - return expnsize; -} - -/* WR with IV, KEY and CPL SFO added */ -static void make_tlstx_data_wr(struct sock *sk, struct sk_buff *skb, - int tls_tx_imm, int tls_len, u32 credits) -{ - unsigned short pdus_per_ulp = 0; - struct chtls_sock *csk; - struct chtls_hws *hws; - int expn_sz; - int pdus; - - csk = rcu_dereference_sk_user_data(sk); - hws = &csk->tlshws; - pdus = DIV_ROUND_UP(tls_len, hws->mfs); - expn_sz = chtls_expansion_size(sk, tls_len, 0, NULL); - if (!hws->compute) { - hws->expansion = chtls_expansion_size(sk, - hws->fcplenmax, - 1, &pdus_per_ulp); - hws->pdus = pdus_per_ulp; - hws->adjustlen = hws->pdus * - ((hws->expansion / hws->pdus) + hws->mfs); - hws->compute = 1; - } - if (tls_copy_ivs(sk, skb)) - return; - tls_copy_tx_key(sk, skb); - tls_tx_data_wr(sk, skb, tls_len, tls_tx_imm, credits, expn_sz, pdus); - hws->tx_seq_no += (pdus - 1); -} - -static void make_tx_data_wr(struct sock *sk, struct sk_buff *skb, - unsigned int immdlen, int len, - u32 credits, u32 compl) -{ - struct fw_ofld_tx_data_wr *req; - unsigned int wr_ulp_mode_force; - struct chtls_sock *csk; - unsigned int opcode; - - csk = rcu_dereference_sk_user_data(sk); - opcode = FW_OFLD_TX_DATA_WR; - - req = (struct fw_ofld_tx_data_wr *)__skb_push(skb, sizeof(*req)); - req->op_to_immdlen = htonl(WR_OP_V(opcode) | - FW_WR_COMPL_V(compl) | - FW_WR_IMMDLEN_V(immdlen)); - req->flowid_len16 = htonl(FW_WR_FLOWID_V(csk->tid) | - FW_WR_LEN16_V(credits)); - - wr_ulp_mode_force = TX_ULP_MODE_V(csk->ulp_mode); - if (is_sg_request(skb)) - wr_ulp_mode_force |= FW_OFLD_TX_DATA_WR_ALIGNPLD_F | - ((tcp_sk(sk)->nonagle & TCP_NAGLE_OFF) ? 0 : - FW_OFLD_TX_DATA_WR_SHOVE_F); - - req->tunnel_to_proxy = htonl(wr_ulp_mode_force | - TX_URG_V(skb_urgent(skb)) | - TX_SHOVE_V((!csk_flag(sk, CSK_TX_MORE_DATA)) && - skb_queue_empty(&csk->txq))); - req->plen = htonl(len); -} - -static int chtls_wr_size(struct chtls_sock *csk, const struct sk_buff *skb, - bool size) -{ - int wr_size; - - wr_size = TLS_WR_CPL_LEN; - wr_size += KEY_ON_MEM_SZ; - wr_size += ivs_size(csk->sk, skb); - - if (size) - return wr_size; - - /* frags counted for IV dsgl */ - if (!skb_ulp_tls_iv_imm(skb)) - skb_shinfo(skb)->nr_frags++; - - return wr_size; -} - -static bool is_ofld_imm(struct chtls_sock *csk, const struct sk_buff *skb) -{ - int length = skb->len; - - if (skb->peeked || skb->len > MAX_IMM_ULPTX_WR_LEN) - return false; - - if (likely(ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NEED_HDR)) { - /* Check TLS header len for Immediate */ - if (csk->ulp_mode == ULP_MODE_TLS && - skb_ulp_tls_inline(skb)) - length += chtls_wr_size(csk, skb, true); - else - length += sizeof(struct fw_ofld_tx_data_wr); - - return length <= MAX_IMM_OFLD_TX_DATA_WR_LEN; - } - return true; -} - -static unsigned int calc_tx_flits(const struct sk_buff *skb, - unsigned int immdlen) -{ - unsigned int flits, cnt; - - flits = immdlen / 8; /* headers */ - cnt = skb_shinfo(skb)->nr_frags; - if (skb_tail_pointer(skb) != skb_transport_header(skb)) - cnt++; - return flits + sgl_len(cnt); -} - -static void arp_failure_discard(void *handle, struct sk_buff *skb) -{ - kfree_skb(skb); -} - -int chtls_push_frames(struct chtls_sock *csk, int comp) -{ - struct chtls_hws *hws = &csk->tlshws; - struct tcp_sock *tp; - struct sk_buff *skb; - int total_size = 0; - struct sock *sk; - int wr_size; - - wr_size = sizeof(struct fw_ofld_tx_data_wr); - sk = csk->sk; - tp = tcp_sk(sk); - - if (unlikely(sk_in_state(sk, TCPF_SYN_SENT | TCPF_CLOSE))) - return 0; - - if (unlikely(csk_flag(sk, CSK_ABORT_SHUTDOWN))) - return 0; - - while (csk->wr_credits && (skb = skb_peek(&csk->txq)) && - (!(ULP_SKB_CB(skb)->flags & ULPCB_FLAG_HOLD) || - skb_queue_len(&csk->txq) > 1)) { - unsigned int credit_len = skb->len; - unsigned int credits_needed; - unsigned int completion = 0; - int tls_len = skb->len;/* TLS data len before IV/key */ - unsigned int immdlen; - int len = skb->len; /* length [ulp bytes] inserted by hw */ - int flowclen16 = 0; - int tls_tx_imm = 0; - - immdlen = skb->len; - if (!is_ofld_imm(csk, skb)) { - immdlen = skb_transport_offset(skb); - if (skb_ulp_tls_inline(skb)) - wr_size = chtls_wr_size(csk, skb, false); - credit_len = 8 * calc_tx_flits(skb, immdlen); - } else { - if (skb_ulp_tls_inline(skb)) { - wr_size = chtls_wr_size(csk, skb, false); - tls_tx_imm = 1; - } - } - if (likely(ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NEED_HDR)) - credit_len += wr_size; - credits_needed = DIV_ROUND_UP(credit_len, 16); - if (!csk_flag_nochk(csk, CSK_TX_DATA_SENT)) { - flowclen16 = send_tx_flowc_wr(sk, 1, tp->snd_nxt, - tp->rcv_nxt); - if (flowclen16 <= 0) - break; - csk->wr_credits -= flowclen16; - csk->wr_unacked += flowclen16; - csk->wr_nondata += flowclen16; - csk_set_flag(csk, CSK_TX_DATA_SENT); - } - - if (csk->wr_credits < credits_needed) { - if (skb_ulp_tls_inline(skb) && - !skb_ulp_tls_iv_imm(skb)) - skb_shinfo(skb)->nr_frags--; - break; - } - - __skb_unlink(skb, &csk->txq); - skb_set_queue_mapping(skb, (csk->txq_idx << 1) | - CPL_PRIORITY_DATA); - if (hws->ofld) - hws->txqid = (skb->queue_mapping >> 1); - skb->csum = (__force __wsum)(credits_needed + csk->wr_nondata); - csk->wr_credits -= credits_needed; - csk->wr_unacked += credits_needed; - csk->wr_nondata = 0; - enqueue_wr(csk, skb); - - if (likely(ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NEED_HDR)) { - if ((comp && csk->wr_unacked == credits_needed) || - (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_COMPL) || - csk->wr_unacked >= csk->wr_max_credits / 2) { - completion = 1; - csk->wr_unacked = 0; - } - if (skb_ulp_tls_inline(skb)) - make_tlstx_data_wr(sk, skb, tls_tx_imm, - tls_len, credits_needed); - else - make_tx_data_wr(sk, skb, immdlen, len, - credits_needed, completion); - tp->snd_nxt += len; - tp->lsndtime = tcp_jiffies32; - if (completion) - ULP_SKB_CB(skb)->flags &= ~ULPCB_FLAG_NEED_HDR; - } else { - struct cpl_close_con_req *req = cplhdr(skb); - unsigned int cmd = CPL_OPCODE_G(ntohl - (OPCODE_TID(req))); - - if (cmd == CPL_CLOSE_CON_REQ) - csk_set_flag(csk, - CSK_CLOSE_CON_REQUESTED); - - if ((ULP_SKB_CB(skb)->flags & ULPCB_FLAG_COMPL) && - (csk->wr_unacked >= csk->wr_max_credits / 2)) { - req->wr.wr_hi |= htonl(FW_WR_COMPL_F); - csk->wr_unacked = 0; - } - } - total_size += skb->truesize; - if (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_BARRIER) - csk_set_flag(csk, CSK_TX_WAIT_IDLE); - t4_set_arp_err_handler(skb, NULL, arp_failure_discard); - cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry); - } - sk->sk_wmem_queued -= total_size; - return total_size; -} - -static void mark_urg(struct tcp_sock *tp, int flags, - struct sk_buff *skb) -{ - if (unlikely(flags & MSG_OOB)) { - tp->snd_up = tp->write_seq; - ULP_SKB_CB(skb)->flags = ULPCB_FLAG_URG | - ULPCB_FLAG_BARRIER | - ULPCB_FLAG_NO_APPEND | - ULPCB_FLAG_NEED_HDR; - } -} - -/* - * Returns true if a connection should send more data to TCP engine - */ -static bool should_push(struct sock *sk) -{ - struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); - struct chtls_dev *cdev = csk->cdev; - struct tcp_sock *tp = tcp_sk(sk); - - /* - * If we've released our offload resources there's nothing to do ... - */ - if (!cdev) - return false; - - /* - * If there aren't any work requests in flight, or there isn't enough - * data in flight, or Nagle is off then send the current TX_DATA - * otherwise hold it and wait to accumulate more data. - */ - return csk->wr_credits == csk->wr_max_credits || - (tp->nonagle & TCP_NAGLE_OFF); -} - -/* - * Returns true if a TCP socket is corked. - */ -static bool corked(const struct tcp_sock *tp, int flags) -{ - return (flags & MSG_MORE) || (tp->nonagle & TCP_NAGLE_CORK); -} - -/* - * Returns true if a send should try to push new data. - */ -static bool send_should_push(struct sock *sk, int flags) -{ - return should_push(sk) && !corked(tcp_sk(sk), flags); -} - -void chtls_tcp_push(struct sock *sk, int flags) -{ - struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); - int qlen = skb_queue_len(&csk->txq); - - if (likely(qlen)) { - struct sk_buff *skb = skb_peek_tail(&csk->txq); - struct tcp_sock *tp = tcp_sk(sk); - - mark_urg(tp, flags, skb); - - if (!(ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NO_APPEND) && - corked(tp, flags)) { - ULP_SKB_CB(skb)->flags |= ULPCB_FLAG_HOLD; - return; - } - - ULP_SKB_CB(skb)->flags &= ~ULPCB_FLAG_HOLD; - if (qlen == 1 && - ((ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NO_APPEND) || - should_push(sk))) - chtls_push_frames(csk, 1); - } -} - -/* - * Calculate the size for a new send sk_buff. It's maximum size so we can - * pack lots of data into it, unless we plan to send it immediately, in which - * case we size it more tightly. - * - * Note: we don't bother compensating for MSS < PAGE_SIZE because it doesn't - * arise in normal cases and when it does we are just wasting memory. - */ -static int select_size(struct sock *sk, int io_len, int flags, int len) -{ - const int pgbreak = SKB_MAX_HEAD(len); - - /* - * If the data wouldn't fit in the main body anyway, put only the - * header in the main body so it can use immediate data and place all - * the payload in page fragments. - */ - if (io_len > pgbreak) - return 0; - - /* - * If we will be accumulating payload get a large main body. - */ - if (!send_should_push(sk, flags)) - return pgbreak; - - return io_len; -} - -void skb_entail(struct sock *sk, struct sk_buff *skb, int flags) -{ - struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); - struct tcp_sock *tp = tcp_sk(sk); - - ULP_SKB_CB(skb)->seq = tp->write_seq; - ULP_SKB_CB(skb)->flags = flags; - __skb_queue_tail(&csk->txq, skb); - sk->sk_wmem_queued += skb->truesize; - - if (TCP_PAGE(sk) && TCP_OFF(sk)) { - put_page(TCP_PAGE(sk)); - TCP_PAGE(sk) = NULL; - TCP_OFF(sk) = 0; - } -} - -static struct sk_buff *get_tx_skb(struct sock *sk, int size) -{ - struct sk_buff *skb; - - skb = alloc_skb(size + TX_HEADER_LEN, sk->sk_allocation); - if (likely(skb)) { - skb_reserve(skb, TX_HEADER_LEN); - skb_entail(sk, skb, ULPCB_FLAG_NEED_HDR); - skb_reset_transport_header(skb); - } - return skb; -} - -static struct sk_buff *get_record_skb(struct sock *sk, int size, bool zcopy) -{ - struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); - struct sk_buff *skb; - - skb = alloc_skb(((zcopy ? 0 : size) + TX_TLSHDR_LEN + - KEY_ON_MEM_SZ + max_ivs_size(sk, size)), - sk->sk_allocation); - if (likely(skb)) { - skb_reserve(skb, (TX_TLSHDR_LEN + - KEY_ON_MEM_SZ + max_ivs_size(sk, size))); - skb_entail(sk, skb, ULPCB_FLAG_NEED_HDR); - skb_reset_transport_header(skb); - ULP_SKB_CB(skb)->ulp.tls.ofld = 1; - ULP_SKB_CB(skb)->ulp.tls.type = csk->tlshws.type; - } - return skb; -} - -static void tx_skb_finalize(struct sk_buff *skb) -{ - struct ulp_skb_cb *cb = ULP_SKB_CB(skb); - - if (!(cb->flags & ULPCB_FLAG_NO_HDR)) - cb->flags = ULPCB_FLAG_NEED_HDR; - cb->flags |= ULPCB_FLAG_NO_APPEND; -} - -static void push_frames_if_head(struct sock *sk) -{ - struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); - - if (skb_queue_len(&csk->txq) == 1) - chtls_push_frames(csk, 1); -} - -static int chtls_skb_copy_to_page_nocache(struct sock *sk, - struct iov_iter *from, - struct sk_buff *skb, - struct page *page, - int off, int copy) -{ - int err; - - err = skb_do_copy_data_nocache(sk, skb, from, page_address(page) + - off, copy, skb->len); - if (err) - return err; - - skb->len += copy; - skb->data_len += copy; - skb->truesize += copy; - sk->sk_wmem_queued += copy; - return 0; -} - -static int csk_mem_free(struct chtls_dev *cdev, struct sock *sk) -{ - return (cdev->max_host_sndbuf - sk->sk_wmem_queued); -} - -static int csk_wait_memory(struct chtls_dev *cdev, - struct sock *sk, long *timeo_p) -{ - DEFINE_WAIT_FUNC(wait, woken_wake_function); - int err = 0; - long current_timeo; - long vm_wait = 0; - bool noblock; - - current_timeo = *timeo_p; - noblock = (*timeo_p ? false : true); - if (csk_mem_free(cdev, sk)) { - current_timeo = (prandom_u32() % (HZ / 5)) + 2; - vm_wait = (prandom_u32() % (HZ / 5)) + 2; - } - - add_wait_queue(sk_sleep(sk), &wait); - while (1) { - sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); - - if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) - goto do_error; - if (!*timeo_p) { - if (noblock) - set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); - goto do_nonblock; - } - if (signal_pending(current)) - goto do_interrupted; - sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); - if (csk_mem_free(cdev, sk) && !vm_wait) - break; - - set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); - sk->sk_write_pending++; - sk_wait_event(sk, ¤t_timeo, sk->sk_err || - (sk->sk_shutdown & SEND_SHUTDOWN) || - (csk_mem_free(cdev, sk) && !vm_wait), &wait); - sk->sk_write_pending--; - - if (vm_wait) { - vm_wait -= current_timeo; - current_timeo = *timeo_p; - if (current_timeo != MAX_SCHEDULE_TIMEOUT) { - current_timeo -= vm_wait; - if (current_timeo < 0) - current_timeo = 0; - } - vm_wait = 0; - } - *timeo_p = current_timeo; - } -do_rm_wq: - remove_wait_queue(sk_sleep(sk), &wait); - return err; -do_error: - err = -EPIPE; - goto do_rm_wq; -do_nonblock: - err = -EAGAIN; - goto do_rm_wq; -do_interrupted: - err = sock_intr_errno(*timeo_p); - goto do_rm_wq; -} - -static int chtls_proccess_cmsg(struct sock *sk, struct msghdr *msg, - unsigned char *record_type) -{ - struct cmsghdr *cmsg; - int rc = -EINVAL; - - for_each_cmsghdr(cmsg, msg) { - if (!CMSG_OK(msg, cmsg)) - return -EINVAL; - if (cmsg->cmsg_level != SOL_TLS) - continue; - - switch (cmsg->cmsg_type) { - case TLS_SET_RECORD_TYPE: - if (cmsg->cmsg_len < CMSG_LEN(sizeof(*record_type))) - return -EINVAL; - - if (msg->msg_flags & MSG_MORE) - return -EINVAL; - - *record_type = *(unsigned char *)CMSG_DATA(cmsg); - rc = 0; - break; - default: - return -EINVAL; - } - } - - return rc; -} - -int chtls_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) -{ - struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); - struct chtls_dev *cdev = csk->cdev; - struct tcp_sock *tp = tcp_sk(sk); - struct sk_buff *skb; - int mss, flags, err; - int recordsz = 0; - int copied = 0; - long timeo; - - lock_sock(sk); - flags = msg->msg_flags; - timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT); - - if (!sk_in_state(sk, TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) { - err = sk_stream_wait_connect(sk, &timeo); - if (err) - goto out_err; - } - - sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); - err = -EPIPE; - if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) - goto out_err; - - mss = csk->mss; - csk_set_flag(csk, CSK_TX_MORE_DATA); - - while (msg_data_left(msg)) { - int copy = 0; - - skb = skb_peek_tail(&csk->txq); - if (skb) { - copy = mss - skb->len; - skb->ip_summed = CHECKSUM_UNNECESSARY; - } - if (!csk_mem_free(cdev, sk)) - goto wait_for_sndbuf; - - if (is_tls_tx(csk) && !csk->tlshws.txleft) { - unsigned char record_type = TLS_RECORD_TYPE_DATA; - - if (unlikely(msg->msg_controllen)) { - err = chtls_proccess_cmsg(sk, msg, - &record_type); - if (err) - goto out_err; - - /* Avoid appending tls handshake, alert to tls data */ - if (skb) - tx_skb_finalize(skb); - } - - recordsz = size; - csk->tlshws.txleft = recordsz; - csk->tlshws.type = record_type; - } - - if (!skb || (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NO_APPEND) || - copy <= 0) { -new_buf: - if (skb) { - tx_skb_finalize(skb); - push_frames_if_head(sk); - } - - if (is_tls_tx(csk)) { - skb = get_record_skb(sk, - select_size(sk, - recordsz, - flags, - TX_TLSHDR_LEN), - false); - } else { - skb = get_tx_skb(sk, - select_size(sk, size, flags, - TX_HEADER_LEN)); - } - if (unlikely(!skb)) - goto wait_for_memory; - - skb->ip_summed = CHECKSUM_UNNECESSARY; - copy = mss; - } - if (copy > size) - copy = size; - - if (skb_tailroom(skb) > 0) { - copy = min(copy, skb_tailroom(skb)); - if (is_tls_tx(csk)) - copy = min_t(int, copy, csk->tlshws.txleft); - err = skb_add_data_nocache(sk, skb, - &msg->msg_iter, copy); - if (err) - goto do_fault; - } else { - int i = skb_shinfo(skb)->nr_frags; - struct page *page = TCP_PAGE(sk); - int pg_size = PAGE_SIZE; - int off = TCP_OFF(sk); - bool merge; - - if (page) - pg_size = page_size(page); - if (off < pg_size && - skb_can_coalesce(skb, i, page, off)) { - merge = true; - goto copy; - } - merge = false; - if (i == (is_tls_tx(csk) ? (MAX_SKB_FRAGS - 1) : - MAX_SKB_FRAGS)) - goto new_buf; - - if (page && off == pg_size) { - put_page(page); - TCP_PAGE(sk) = page = NULL; - pg_size = PAGE_SIZE; - } - - if (!page) { - gfp_t gfp = sk->sk_allocation; - int order = cdev->send_page_order; - - if (order) { - page = alloc_pages(gfp | __GFP_COMP | - __GFP_NOWARN | - __GFP_NORETRY, - order); - if (page) - pg_size <<= order; - } - if (!page) { - page = alloc_page(gfp); - pg_size = PAGE_SIZE; - } - if (!page) - goto wait_for_memory; - off = 0; - } -copy: - if (copy > pg_size - off) - copy = pg_size - off; - if (is_tls_tx(csk)) - copy = min_t(int, copy, csk->tlshws.txleft); - - err = chtls_skb_copy_to_page_nocache(sk, &msg->msg_iter, - skb, page, - off, copy); - if (unlikely(err)) { - if (!TCP_PAGE(sk)) { - TCP_PAGE(sk) = page; - TCP_OFF(sk) = 0; - } - goto do_fault; - } - /* Update the skb. */ - if (merge) { - skb_frag_size_add( - &skb_shinfo(skb)->frags[i - 1], - copy); - } else { - skb_fill_page_desc(skb, i, page, off, copy); - if (off + copy < pg_size) { - /* space left keep page */ - get_page(page); - TCP_PAGE(sk) = page; - } else { - TCP_PAGE(sk) = NULL; - } - } - TCP_OFF(sk) = off + copy; - } - if (unlikely(skb->len == mss)) - tx_skb_finalize(skb); - tp->write_seq += copy; - copied += copy; - size -= copy; - - if (is_tls_tx(csk)) - csk->tlshws.txleft -= copy; - - if (corked(tp, flags) && - (sk_stream_wspace(sk) < sk_stream_min_wspace(sk))) - ULP_SKB_CB(skb)->flags |= ULPCB_FLAG_NO_APPEND; - - if (size == 0) - goto out; - - if (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NO_APPEND) - push_frames_if_head(sk); - continue; -wait_for_sndbuf: - set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); -wait_for_memory: - err = csk_wait_memory(cdev, sk, &timeo); - if (err) - goto do_error; - } -out: - csk_reset_flag(csk, CSK_TX_MORE_DATA); - if (copied) - chtls_tcp_push(sk, flags); -done: - release_sock(sk); - return copied; -do_fault: - if (!skb->len) { - __skb_unlink(skb, &csk->txq); - sk->sk_wmem_queued -= skb->truesize; - __kfree_skb(skb); - } -do_error: - if (copied) - goto out; -out_err: - if (csk_conn_inline(csk)) - csk_reset_flag(csk, CSK_TX_MORE_DATA); - copied = sk_stream_error(sk, flags, err); - goto done; -} - -int chtls_sendpage(struct sock *sk, struct page *page, - int offset, size_t size, int flags) -{ - struct chtls_sock *csk; - struct chtls_dev *cdev; - int mss, err, copied; - struct tcp_sock *tp; - long timeo; - - tp = tcp_sk(sk); - copied = 0; - csk = rcu_dereference_sk_user_data(sk); - cdev = csk->cdev; - timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT); - - err = sk_stream_wait_connect(sk, &timeo); - if (!sk_in_state(sk, TCPF_ESTABLISHED | TCPF_CLOSE_WAIT) && - err != 0) - goto out_err; - - mss = csk->mss; - csk_set_flag(csk, CSK_TX_MORE_DATA); - - while (size > 0) { - struct sk_buff *skb = skb_peek_tail(&csk->txq); - int copy, i; - - if (!skb || (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NO_APPEND) || - (copy = mss - skb->len) <= 0) { -new_buf: - if (!csk_mem_free(cdev, sk)) - goto wait_for_sndbuf; - - if (is_tls_tx(csk)) { - skb = get_record_skb(sk, - select_size(sk, size, - flags, - TX_TLSHDR_LEN), - true); - } else { - skb = get_tx_skb(sk, 0); - } - if (!skb) - goto wait_for_memory; - copy = mss; - } - if (copy > size) - copy = size; - - i = skb_shinfo(skb)->nr_frags; - if (skb_can_coalesce(skb, i, page, offset)) { - skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy); - } else if (i < MAX_SKB_FRAGS) { - get_page(page); - skb_fill_page_desc(skb, i, page, offset, copy); - } else { - tx_skb_finalize(skb); - push_frames_if_head(sk); - goto new_buf; - } - - skb->len += copy; - if (skb->len == mss) - tx_skb_finalize(skb); - skb->data_len += copy; - skb->truesize += copy; - sk->sk_wmem_queued += copy; - tp->write_seq += copy; - copied += copy; - offset += copy; - size -= copy; - - if (corked(tp, flags) && - (sk_stream_wspace(sk) < sk_stream_min_wspace(sk))) - ULP_SKB_CB(skb)->flags |= ULPCB_FLAG_NO_APPEND; - - if (!size) - break; - - if (unlikely(ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NO_APPEND)) - push_frames_if_head(sk); - continue; -wait_for_sndbuf: - set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); -wait_for_memory: - err = csk_wait_memory(cdev, sk, &timeo); - if (err) - goto do_error; - } -out: - csk_reset_flag(csk, CSK_TX_MORE_DATA); - if (copied) - chtls_tcp_push(sk, flags); -done: - release_sock(sk); - return copied; - -do_error: - if (copied) - goto out; - -out_err: - if (csk_conn_inline(csk)) - csk_reset_flag(csk, CSK_TX_MORE_DATA); - copied = sk_stream_error(sk, flags, err); - goto done; -} - -static void chtls_select_window(struct sock *sk) -{ - struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); - struct tcp_sock *tp = tcp_sk(sk); - unsigned int wnd = tp->rcv_wnd; - - wnd = max_t(unsigned int, wnd, tcp_full_space(sk)); - wnd = max_t(unsigned int, MIN_RCV_WND, wnd); - - if (wnd > MAX_RCV_WND) - wnd = MAX_RCV_WND; - -/* - * Check if we need to grow the receive window in response to an increase in - * the socket's receive buffer size. Some applications increase the buffer - * size dynamically and rely on the window to grow accordingly. - */ - - if (wnd > tp->rcv_wnd) { - tp->rcv_wup -= wnd - tp->rcv_wnd; - tp->rcv_wnd = wnd; - /* Mark the receive window as updated */ - csk_reset_flag(csk, CSK_UPDATE_RCV_WND); - } -} - -/* - * Send RX credits through an RX_DATA_ACK CPL message. We are permitted - * to return without sending the message in case we cannot allocate - * an sk_buff. Returns the number of credits sent. - */ -static u32 send_rx_credits(struct chtls_sock *csk, u32 credits) -{ - struct cpl_rx_data_ack *req; - struct sk_buff *skb; - - skb = alloc_skb(sizeof(*req), GFP_ATOMIC); - if (!skb) - return 0; - __skb_put(skb, sizeof(*req)); - req = (struct cpl_rx_data_ack *)skb->head; - - set_wr_txq(skb, CPL_PRIORITY_ACK, csk->port_id); - INIT_TP_WR(req, csk->tid); - OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_RX_DATA_ACK, - csk->tid)); - req->credit_dack = cpu_to_be32(RX_CREDITS_V(credits) | - RX_FORCE_ACK_F); - cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb); - return credits; -} - -#define CREDIT_RETURN_STATE (TCPF_ESTABLISHED | \ - TCPF_FIN_WAIT1 | \ - TCPF_FIN_WAIT2) - -/* - * Called after some received data has been read. It returns RX credits - * to the HW for the amount of data processed. - */ -static void chtls_cleanup_rbuf(struct sock *sk, int copied) -{ - struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); - struct tcp_sock *tp; - int must_send; - u32 credits; - u32 thres; - - thres = 15 * 1024; - - if (!sk_in_state(sk, CREDIT_RETURN_STATE)) - return; - - chtls_select_window(sk); - tp = tcp_sk(sk); - credits = tp->copied_seq - tp->rcv_wup; - if (unlikely(!credits)) - return; - -/* - * For coalescing to work effectively ensure the receive window has - * at least 16KB left. - */ - must_send = credits + 16384 >= tp->rcv_wnd; - - if (must_send || credits >= thres) - tp->rcv_wup += send_rx_credits(csk, credits); -} - -static int chtls_pt_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, - int nonblock, int flags, int *addr_len) -{ - struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); - struct chtls_hws *hws = &csk->tlshws; - struct net_device *dev = csk->egress_dev; - struct adapter *adap = netdev2adap(dev); - struct tcp_sock *tp = tcp_sk(sk); - unsigned long avail; - int buffers_freed; - int copied = 0; - int target; - long timeo; - - buffers_freed = 0; - - timeo = sock_rcvtimeo(sk, nonblock); - target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); - - if (unlikely(csk_flag(sk, CSK_UPDATE_RCV_WND))) - chtls_cleanup_rbuf(sk, copied); - - do { - struct sk_buff *skb; - u32 offset = 0; - - if (unlikely(tp->urg_data && - tp->urg_seq == tp->copied_seq)) { - if (copied) - break; - if (signal_pending(current)) { - copied = timeo ? sock_intr_errno(timeo) : - -EAGAIN; - break; - } - } - skb = skb_peek(&sk->sk_receive_queue); - if (skb) - goto found_ok_skb; - if (csk->wr_credits && - skb_queue_len(&csk->txq) && - chtls_push_frames(csk, csk->wr_credits == - csk->wr_max_credits)) - sk->sk_write_space(sk); - - if (copied >= target && !READ_ONCE(sk->sk_backlog.tail)) - break; - - if (copied) { - if (sk->sk_err || sk->sk_state == TCP_CLOSE || - (sk->sk_shutdown & RCV_SHUTDOWN) || - signal_pending(current)) - break; - - if (!timeo) - break; - } else { - if (sock_flag(sk, SOCK_DONE)) - break; - if (sk->sk_err) { - copied = sock_error(sk); - break; - } - if (sk->sk_shutdown & RCV_SHUTDOWN) - break; - if (sk->sk_state == TCP_CLOSE) { - copied = -ENOTCONN; - break; - } - if (!timeo) { - copied = -EAGAIN; - break; - } - if (signal_pending(current)) { - copied = sock_intr_errno(timeo); - break; - } - } - if (READ_ONCE(sk->sk_backlog.tail)) { - release_sock(sk); - lock_sock(sk); - chtls_cleanup_rbuf(sk, copied); - continue; - } - - if (copied >= target) - break; - chtls_cleanup_rbuf(sk, copied); - sk_wait_data(sk, &timeo, NULL); - continue; -found_ok_skb: - if (!skb->len) { - skb_dst_set(skb, NULL); - __skb_unlink(skb, &sk->sk_receive_queue); - kfree_skb(skb); - - if (!copied && !timeo) { - copied = -EAGAIN; - break; - } - - if (copied < target) { - release_sock(sk); - lock_sock(sk); - continue; - } - break; - } - offset = hws->copied_seq; - avail = skb->len - offset; - if (len < avail) - avail = len; - - if (unlikely(tp->urg_data)) { - u32 urg_offset = tp->urg_seq - tp->copied_seq; - - if (urg_offset < avail) { - if (urg_offset) { - avail = urg_offset; - } else if (!sock_flag(sk, SOCK_URGINLINE)) { - /* First byte is urgent, skip */ - tp->copied_seq++; - offset++; - avail--; - if (!avail) - goto skip_copy; - } - } - } - /* Set record type if not already done. For a non-data record, - * do not proceed if record type could not be copied. - */ - if (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_TLS_HDR) { - struct tls_hdr *thdr = (struct tls_hdr *)skb->data; - int cerr = 0; - - cerr = put_cmsg(msg, SOL_TLS, TLS_GET_RECORD_TYPE, - sizeof(thdr->type), &thdr->type); - - if (cerr && thdr->type != TLS_RECORD_TYPE_DATA) - return -EIO; - /* don't send tls header, skip copy */ - goto skip_copy; - } - - if (skb_copy_datagram_msg(skb, offset, msg, avail)) { - if (!copied) { - copied = -EFAULT; - break; - } - } - - copied += avail; - len -= avail; - hws->copied_seq += avail; -skip_copy: - if (tp->urg_data && after(tp->copied_seq, tp->urg_seq)) - tp->urg_data = 0; - - if ((avail + offset) >= skb->len) { - if (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_TLS_HDR) { - tp->copied_seq += skb->len; - hws->rcvpld = skb->hdr_len; - } else { - atomic_inc(&adap->chcr_stats.tls_pdu_rx); - tp->copied_seq += hws->rcvpld; - } - chtls_free_skb(sk, skb); - buffers_freed++; - hws->copied_seq = 0; - if (copied >= target && - !skb_peek(&sk->sk_receive_queue)) - break; - } - } while (len > 0); - - if (buffers_freed) - chtls_cleanup_rbuf(sk, copied); - release_sock(sk); - return copied; -} - -/* - * Peek at data in a socket's receive buffer. - */ -static int peekmsg(struct sock *sk, struct msghdr *msg, - size_t len, int nonblock, int flags) -{ - struct tcp_sock *tp = tcp_sk(sk); - u32 peek_seq, offset; - struct sk_buff *skb; - int copied = 0; - size_t avail; /* amount of available data in current skb */ - long timeo; - - lock_sock(sk); - timeo = sock_rcvtimeo(sk, nonblock); - peek_seq = tp->copied_seq; - - do { - if (unlikely(tp->urg_data && tp->urg_seq == peek_seq)) { - if (copied) - break; - if (signal_pending(current)) { - copied = timeo ? sock_intr_errno(timeo) : - -EAGAIN; - break; - } - } - - skb_queue_walk(&sk->sk_receive_queue, skb) { - offset = peek_seq - ULP_SKB_CB(skb)->seq; - if (offset < skb->len) - goto found_ok_skb; - } - - /* empty receive queue */ - if (copied) - break; - if (sock_flag(sk, SOCK_DONE)) - break; - if (sk->sk_err) { - copied = sock_error(sk); - break; - } - if (sk->sk_shutdown & RCV_SHUTDOWN) - break; - if (sk->sk_state == TCP_CLOSE) { - copied = -ENOTCONN; - break; - } - if (!timeo) { - copied = -EAGAIN; - break; - } - if (signal_pending(current)) { - copied = sock_intr_errno(timeo); - break; - } - - if (READ_ONCE(sk->sk_backlog.tail)) { - /* Do not sleep, just process backlog. */ - release_sock(sk); - lock_sock(sk); - } else { - sk_wait_data(sk, &timeo, NULL); - } - - if (unlikely(peek_seq != tp->copied_seq)) { - if (net_ratelimit()) - pr_info("TCP(%s:%d), race in MSG_PEEK.\n", - current->comm, current->pid); - peek_seq = tp->copied_seq; - } - continue; - -found_ok_skb: - avail = skb->len - offset; - if (len < avail) - avail = len; - /* - * Do we have urgent data here? We need to skip over the - * urgent byte. - */ - if (unlikely(tp->urg_data)) { - u32 urg_offset = tp->urg_seq - peek_seq; - - if (urg_offset < avail) { - /* - * The amount of data we are preparing to copy - * contains urgent data. - */ - if (!urg_offset) { /* First byte is urgent */ - if (!sock_flag(sk, SOCK_URGINLINE)) { - peek_seq++; - offset++; - avail--; - } - if (!avail) - continue; - } else { - /* stop short of the urgent data */ - avail = urg_offset; - } - } - } - - /* - * If MSG_TRUNC is specified the data is discarded. - */ - if (likely(!(flags & MSG_TRUNC))) - if (skb_copy_datagram_msg(skb, offset, msg, len)) { - if (!copied) { - copied = -EFAULT; - break; - } - } - peek_seq += avail; - copied += avail; - len -= avail; - } while (len > 0); - - release_sock(sk); - return copied; -} - -int chtls_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, - int nonblock, int flags, int *addr_len) -{ - struct tcp_sock *tp = tcp_sk(sk); - struct chtls_sock *csk; - unsigned long avail; /* amount of available data in current skb */ - int buffers_freed; - int copied = 0; - long timeo; - int target; /* Read at least this many bytes */ - - buffers_freed = 0; - - if (unlikely(flags & MSG_OOB)) - return tcp_prot.recvmsg(sk, msg, len, nonblock, flags, - addr_len); - - if (unlikely(flags & MSG_PEEK)) - return peekmsg(sk, msg, len, nonblock, flags); - - if (sk_can_busy_loop(sk) && - skb_queue_empty_lockless(&sk->sk_receive_queue) && - sk->sk_state == TCP_ESTABLISHED) - sk_busy_loop(sk, nonblock); - - lock_sock(sk); - csk = rcu_dereference_sk_user_data(sk); - - if (is_tls_rx(csk)) - return chtls_pt_recvmsg(sk, msg, len, nonblock, - flags, addr_len); - - timeo = sock_rcvtimeo(sk, nonblock); - target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); - - if (unlikely(csk_flag(sk, CSK_UPDATE_RCV_WND))) - chtls_cleanup_rbuf(sk, copied); - - do { - struct sk_buff *skb; - u32 offset; - - if (unlikely(tp->urg_data && tp->urg_seq == tp->copied_seq)) { - if (copied) - break; - if (signal_pending(current)) { - copied = timeo ? sock_intr_errno(timeo) : - -EAGAIN; - break; - } - } - - skb = skb_peek(&sk->sk_receive_queue); - if (skb) - goto found_ok_skb; - - if (csk->wr_credits && - skb_queue_len(&csk->txq) && - chtls_push_frames(csk, csk->wr_credits == - csk->wr_max_credits)) - sk->sk_write_space(sk); - - if (copied >= target && !READ_ONCE(sk->sk_backlog.tail)) - break; - - if (copied) { - if (sk->sk_err || sk->sk_state == TCP_CLOSE || - (sk->sk_shutdown & RCV_SHUTDOWN) || - signal_pending(current)) - break; - } else { - if (sock_flag(sk, SOCK_DONE)) - break; - if (sk->sk_err) { - copied = sock_error(sk); - break; - } - if (sk->sk_shutdown & RCV_SHUTDOWN) - break; - if (sk->sk_state == TCP_CLOSE) { - copied = -ENOTCONN; - break; - } - if (!timeo) { - copied = -EAGAIN; - break; - } - if (signal_pending(current)) { - copied = sock_intr_errno(timeo); - break; - } - } - - if (READ_ONCE(sk->sk_backlog.tail)) { - release_sock(sk); - lock_sock(sk); - chtls_cleanup_rbuf(sk, copied); - continue; - } - - if (copied >= target) - break; - chtls_cleanup_rbuf(sk, copied); - sk_wait_data(sk, &timeo, NULL); - continue; - -found_ok_skb: - if (!skb->len) { - chtls_kfree_skb(sk, skb); - if (!copied && !timeo) { - copied = -EAGAIN; - break; - } - - if (copied < target) - continue; - - break; - } - - offset = tp->copied_seq - ULP_SKB_CB(skb)->seq; - avail = skb->len - offset; - if (len < avail) - avail = len; - - if (unlikely(tp->urg_data)) { - u32 urg_offset = tp->urg_seq - tp->copied_seq; - - if (urg_offset < avail) { - if (urg_offset) { - avail = urg_offset; - } else if (!sock_flag(sk, SOCK_URGINLINE)) { - tp->copied_seq++; - offset++; - avail--; - if (!avail) - goto skip_copy; - } - } - } - - if (likely(!(flags & MSG_TRUNC))) { - if (skb_copy_datagram_msg(skb, offset, - msg, avail)) { - if (!copied) { - copied = -EFAULT; - break; - } - } - } - - tp->copied_seq += avail; - copied += avail; - len -= avail; - -skip_copy: - if (tp->urg_data && after(tp->copied_seq, tp->urg_seq)) - tp->urg_data = 0; - - if (avail + offset >= skb->len) { - chtls_free_skb(sk, skb); - buffers_freed++; - - if (copied >= target && - !skb_peek(&sk->sk_receive_queue)) - break; - } - } while (len > 0); - - if (buffers_freed) - chtls_cleanup_rbuf(sk, copied); - - release_sock(sk); - return copied; -} diff --git a/drivers/crypto/chelsio/chtls/chtls_main.c b/drivers/crypto/chelsio/chtls/chtls_main.c deleted file mode 100644 index 66d247efd561..000000000000 --- a/drivers/crypto/chelsio/chtls/chtls_main.c +++ /dev/null @@ -1,641 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (c) 2018 Chelsio Communications, Inc. - * - * Written by: Atul Gupta ([email protected]) - */ -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/skbuff.h> -#include <linux/socket.h> -#include <linux/hash.h> -#include <linux/in.h> -#include <linux/net.h> -#include <linux/ip.h> -#include <linux/tcp.h> -#include <net/ipv6.h> -#include <net/transp_v6.h> -#include <net/tcp.h> -#include <net/tls.h> - -#include "chtls.h" -#include "chtls_cm.h" - -#define DRV_NAME "chtls" - -/* - * chtls device management - * maintains a list of the chtls devices - */ -static LIST_HEAD(cdev_list); -static DEFINE_MUTEX(cdev_mutex); - -static DEFINE_MUTEX(notify_mutex); -static RAW_NOTIFIER_HEAD(listen_notify_list); -static struct proto chtls_cpl_prot, chtls_cpl_protv6; -struct request_sock_ops chtls_rsk_ops, chtls_rsk_opsv6; -static uint send_page_order = (14 - PAGE_SHIFT < 0) ? 0 : 14 - PAGE_SHIFT; - -static void register_listen_notifier(struct notifier_block *nb) -{ - mutex_lock(¬ify_mutex); - raw_notifier_chain_register(&listen_notify_list, nb); - mutex_unlock(¬ify_mutex); -} - -static void unregister_listen_notifier(struct notifier_block *nb) -{ - mutex_lock(¬ify_mutex); - raw_notifier_chain_unregister(&listen_notify_list, nb); - mutex_unlock(¬ify_mutex); -} - -static int listen_notify_handler(struct notifier_block *this, - unsigned long event, void *data) -{ - struct chtls_listen *clisten; - int ret = NOTIFY_DONE; - - clisten = (struct chtls_listen *)data; - - switch (event) { - case CHTLS_LISTEN_START: - ret = chtls_listen_start(clisten->cdev, clisten->sk); - kfree(clisten); - break; - case CHTLS_LISTEN_STOP: - chtls_listen_stop(clisten->cdev, clisten->sk); - kfree(clisten); - break; - } - return ret; -} - -static struct notifier_block listen_notifier = { - .notifier_call = listen_notify_handler -}; - -static int listen_backlog_rcv(struct sock *sk, struct sk_buff *skb) -{ - if (likely(skb_transport_header(skb) != skb_network_header(skb))) - return tcp_v4_do_rcv(sk, skb); - BLOG_SKB_CB(skb)->backlog_rcv(sk, skb); - return 0; -} - -static int chtls_start_listen(struct chtls_dev *cdev, struct sock *sk) -{ - struct chtls_listen *clisten; - - if (sk->sk_protocol != IPPROTO_TCP) - return -EPROTONOSUPPORT; - - if (sk->sk_family == PF_INET && - LOOPBACK(inet_sk(sk)->inet_rcv_saddr)) - return -EADDRNOTAVAIL; - - sk->sk_backlog_rcv = listen_backlog_rcv; - clisten = kmalloc(sizeof(*clisten), GFP_KERNEL); - if (!clisten) - return -ENOMEM; - clisten->cdev = cdev; - clisten->sk = sk; - mutex_lock(¬ify_mutex); - raw_notifier_call_chain(&listen_notify_list, - CHTLS_LISTEN_START, clisten); - mutex_unlock(¬ify_mutex); - return 0; -} - -static void chtls_stop_listen(struct chtls_dev *cdev, struct sock *sk) -{ - struct chtls_listen *clisten; - - if (sk->sk_protocol != IPPROTO_TCP) - return; - - clisten = kmalloc(sizeof(*clisten), GFP_KERNEL); - if (!clisten) - return; - clisten->cdev = cdev; - clisten->sk = sk; - mutex_lock(¬ify_mutex); - raw_notifier_call_chain(&listen_notify_list, - CHTLS_LISTEN_STOP, clisten); - mutex_unlock(¬ify_mutex); -} - -static int chtls_inline_feature(struct tls_toe_device *dev) -{ - struct net_device *netdev; - struct chtls_dev *cdev; - int i; - - cdev = to_chtls_dev(dev); - - for (i = 0; i < cdev->lldi->nports; i++) { - netdev = cdev->ports[i]; - if (netdev->features & NETIF_F_HW_TLS_RECORD) - return 1; - } - return 0; -} - -static int chtls_create_hash(struct tls_toe_device *dev, struct sock *sk) -{ - struct chtls_dev *cdev = to_chtls_dev(dev); - - if (sk->sk_state == TCP_LISTEN) - return chtls_start_listen(cdev, sk); - return 0; -} - -static void chtls_destroy_hash(struct tls_toe_device *dev, struct sock *sk) -{ - struct chtls_dev *cdev = to_chtls_dev(dev); - - if (sk->sk_state == TCP_LISTEN) - chtls_stop_listen(cdev, sk); -} - -static void chtls_free_uld(struct chtls_dev *cdev) -{ - int i; - - tls_toe_unregister_device(&cdev->tlsdev); - kvfree(cdev->kmap.addr); - idr_destroy(&cdev->hwtid_idr); - for (i = 0; i < (1 << RSPQ_HASH_BITS); i++) - kfree_skb(cdev->rspq_skb_cache[i]); - kfree(cdev->lldi); - kfree_skb(cdev->askb); - kfree(cdev); -} - -static inline void chtls_dev_release(struct kref *kref) -{ - struct tls_toe_device *dev; - struct chtls_dev *cdev; - struct adapter *adap; - - dev = container_of(kref, struct tls_toe_device, kref); - cdev = to_chtls_dev(dev); - - /* Reset tls rx/tx stats */ - adap = pci_get_drvdata(cdev->pdev); - atomic_set(&adap->chcr_stats.tls_pdu_tx, 0); - atomic_set(&adap->chcr_stats.tls_pdu_rx, 0); - - chtls_free_uld(cdev); -} - -static void chtls_register_dev(struct chtls_dev *cdev) -{ - struct tls_toe_device *tlsdev = &cdev->tlsdev; - - strlcpy(tlsdev->name, "chtls", TLS_TOE_DEVICE_NAME_MAX); - strlcat(tlsdev->name, cdev->lldi->ports[0]->name, - TLS_TOE_DEVICE_NAME_MAX); - tlsdev->feature = chtls_inline_feature; - tlsdev->hash = chtls_create_hash; - tlsdev->unhash = chtls_destroy_hash; - tlsdev->release = chtls_dev_release; - kref_init(&tlsdev->kref); - tls_toe_register_device(tlsdev); - cdev->cdev_state = CHTLS_CDEV_STATE_UP; -} - -static void process_deferq(struct work_struct *task_param) -{ - struct chtls_dev *cdev = container_of(task_param, - struct chtls_dev, deferq_task); - struct sk_buff *skb; - - spin_lock_bh(&cdev->deferq.lock); - while ((skb = __skb_dequeue(&cdev->deferq)) != NULL) { - spin_unlock_bh(&cdev->deferq.lock); - DEFERRED_SKB_CB(skb)->handler(cdev, skb); - spin_lock_bh(&cdev->deferq.lock); - } - spin_unlock_bh(&cdev->deferq.lock); -} - -static int chtls_get_skb(struct chtls_dev *cdev) -{ - cdev->askb = alloc_skb(sizeof(struct tcphdr), GFP_KERNEL); - if (!cdev->askb) - return -ENOMEM; - - skb_put(cdev->askb, sizeof(struct tcphdr)); - skb_reset_transport_header(cdev->askb); - memset(cdev->askb->data, 0, cdev->askb->len); - return 0; -} - -static void *chtls_uld_add(const struct cxgb4_lld_info *info) -{ - struct cxgb4_lld_info *lldi; - struct chtls_dev *cdev; - int i, j; - - cdev = kzalloc(sizeof(*cdev), GFP_KERNEL); - if (!cdev) - goto out; - - lldi = kzalloc(sizeof(*lldi), GFP_KERNEL); - if (!lldi) - goto out_lldi; - - if (chtls_get_skb(cdev)) - goto out_skb; - - *lldi = *info; - cdev->lldi = lldi; - cdev->pdev = lldi->pdev; - cdev->tids = lldi->tids; - cdev->ports = lldi->ports; - cdev->mtus = lldi->mtus; - cdev->tids = lldi->tids; - cdev->pfvf = FW_VIID_PFN_G(cxgb4_port_viid(lldi->ports[0])) - << FW_VIID_PFN_S; - - for (i = 0; i < (1 << RSPQ_HASH_BITS); i++) { - unsigned int size = 64 - sizeof(struct rsp_ctrl) - 8; - - cdev->rspq_skb_cache[i] = __alloc_skb(size, - gfp_any(), 0, - lldi->nodeid); - if (unlikely(!cdev->rspq_skb_cache[i])) - goto out_rspq_skb; - } - - idr_init(&cdev->hwtid_idr); - INIT_WORK(&cdev->deferq_task, process_deferq); - spin_lock_init(&cdev->listen_lock); - spin_lock_init(&cdev->idr_lock); - cdev->send_page_order = min_t(uint, get_order(32768), - send_page_order); - cdev->max_host_sndbuf = 48 * 1024; - - if (lldi->vr->key.size) - if (chtls_init_kmap(cdev, lldi)) - goto out_rspq_skb; - - mutex_lock(&cdev_mutex); - list_add_tail(&cdev->list, &cdev_list); - mutex_unlock(&cdev_mutex); - - return cdev; -out_rspq_skb: - for (j = 0; j < i; j++) - kfree_skb(cdev->rspq_skb_cache[j]); - kfree_skb(cdev->askb); -out_skb: - kfree(lldi); -out_lldi: - kfree(cdev); -out: - return NULL; -} - -static void chtls_free_all_uld(void) -{ - struct chtls_dev *cdev, *tmp; - - mutex_lock(&cdev_mutex); - list_for_each_entry_safe(cdev, tmp, &cdev_list, list) { - if (cdev->cdev_state == CHTLS_CDEV_STATE_UP) { - list_del(&cdev->list); - kref_put(&cdev->tlsdev.kref, cdev->tlsdev.release); - } - } - mutex_unlock(&cdev_mutex); -} - -static int chtls_uld_state_change(void *handle, enum cxgb4_state new_state) -{ - struct chtls_dev *cdev = handle; - - switch (new_state) { - case CXGB4_STATE_UP: - chtls_register_dev(cdev); - break; - case CXGB4_STATE_DOWN: - break; - case CXGB4_STATE_START_RECOVERY: - break; - case CXGB4_STATE_DETACH: - mutex_lock(&cdev_mutex); - list_del(&cdev->list); - mutex_unlock(&cdev_mutex); - kref_put(&cdev->tlsdev.kref, cdev->tlsdev.release); - break; - default: - break; - } - return 0; -} - -static struct sk_buff *copy_gl_to_skb_pkt(const struct pkt_gl *gl, - const __be64 *rsp, - u32 pktshift) -{ - struct sk_buff *skb; - - /* Allocate space for cpl_pass_accpet_req which will be synthesized by - * driver. Once driver synthesizes cpl_pass_accpet_req the skb will go - * through the regular cpl_pass_accept_req processing in TOM. - */ - skb = alloc_skb(gl->tot_len + sizeof(struct cpl_pass_accept_req) - - pktshift, GFP_ATOMIC); - if (unlikely(!skb)) - return NULL; - __skb_put(skb, gl->tot_len + sizeof(struct cpl_pass_accept_req) - - pktshift); - /* For now we will copy cpl_rx_pkt in the skb */ - skb_copy_to_linear_data(skb, rsp, sizeof(struct cpl_rx_pkt)); - skb_copy_to_linear_data_offset(skb, sizeof(struct cpl_pass_accept_req) - , gl->va + pktshift, - gl->tot_len - pktshift); - - return skb; -} - -static int chtls_recv_packet(struct chtls_dev *cdev, - const struct pkt_gl *gl, const __be64 *rsp) -{ - unsigned int opcode = *(u8 *)rsp; - struct sk_buff *skb; - int ret; - - skb = copy_gl_to_skb_pkt(gl, rsp, cdev->lldi->sge_pktshift); - if (!skb) - return -ENOMEM; - - ret = chtls_handlers[opcode](cdev, skb); - if (ret & CPL_RET_BUF_DONE) - kfree_skb(skb); - - return 0; -} - -static int chtls_recv_rsp(struct chtls_dev *cdev, const __be64 *rsp) -{ - unsigned long rspq_bin; - unsigned int opcode; - struct sk_buff *skb; - unsigned int len; - int ret; - - len = 64 - sizeof(struct rsp_ctrl) - 8; - opcode = *(u8 *)rsp; - - rspq_bin = hash_ptr((void *)rsp, RSPQ_HASH_BITS); - skb = cdev->rspq_skb_cache[rspq_bin]; - if (skb && !skb_is_nonlinear(skb) && - !skb_shared(skb) && !skb_cloned(skb)) { - refcount_inc(&skb->users); - if (refcount_read(&skb->users) == 2) { - __skb_trim(skb, 0); - if (skb_tailroom(skb) >= len) - goto copy_out; - } - refcount_dec(&skb->users); - } - skb = alloc_skb(len, GFP_ATOMIC); - if (unlikely(!skb)) - return -ENOMEM; - -copy_out: - __skb_put(skb, len); - skb_copy_to_linear_data(skb, rsp, len); - skb_reset_network_header(skb); - skb_reset_transport_header(skb); - ret = chtls_handlers[opcode](cdev, skb); - - if (ret & CPL_RET_BUF_DONE) - kfree_skb(skb); - return 0; -} - -static void chtls_recv(struct chtls_dev *cdev, - struct sk_buff **skbs, const __be64 *rsp) -{ - struct sk_buff *skb = *skbs; - unsigned int opcode; - int ret; - - opcode = *(u8 *)rsp; - - __skb_push(skb, sizeof(struct rss_header)); - skb_copy_to_linear_data(skb, rsp, sizeof(struct rss_header)); - - ret = chtls_handlers[opcode](cdev, skb); - if (ret & CPL_RET_BUF_DONE) - kfree_skb(skb); -} - -static int chtls_uld_rx_handler(void *handle, const __be64 *rsp, - const struct pkt_gl *gl) -{ - struct chtls_dev *cdev = handle; - unsigned int opcode; - struct sk_buff *skb; - - opcode = *(u8 *)rsp; - - if (unlikely(opcode == CPL_RX_PKT)) { - if (chtls_recv_packet(cdev, gl, rsp) < 0) - goto nomem; - return 0; - } - - if (!gl) - return chtls_recv_rsp(cdev, rsp); - -#define RX_PULL_LEN 128 - skb = cxgb4_pktgl_to_skb(gl, RX_PULL_LEN, RX_PULL_LEN); - if (unlikely(!skb)) - goto nomem; - chtls_recv(cdev, &skb, rsp); - return 0; - -nomem: - return -ENOMEM; -} - -static int do_chtls_getsockopt(struct sock *sk, char __user *optval, - int __user *optlen) -{ - struct tls_crypto_info crypto_info = { 0 }; - - crypto_info.version = TLS_1_2_VERSION; - if (copy_to_user(optval, &crypto_info, sizeof(struct tls_crypto_info))) - return -EFAULT; - return 0; -} - -static int chtls_getsockopt(struct sock *sk, int level, int optname, - char __user *optval, int __user *optlen) -{ - struct tls_context *ctx = tls_get_ctx(sk); - - if (level != SOL_TLS) - return ctx->sk_proto->getsockopt(sk, level, - optname, optval, optlen); - - return do_chtls_getsockopt(sk, optval, optlen); -} - -static int do_chtls_setsockopt(struct sock *sk, int optname, - sockptr_t optval, unsigned int optlen) -{ - struct tls_crypto_info *crypto_info, tmp_crypto_info; - struct chtls_sock *csk; - int keylen; - int cipher_type; - int rc = 0; - - csk = rcu_dereference_sk_user_data(sk); - - if (sockptr_is_null(optval) || optlen < sizeof(*crypto_info)) { - rc = -EINVAL; - goto out; - } - - rc = copy_from_sockptr(&tmp_crypto_info, optval, sizeof(*crypto_info)); - if (rc) { - rc = -EFAULT; - goto out; - } - - /* check version */ - if (tmp_crypto_info.version != TLS_1_2_VERSION) { - rc = -ENOTSUPP; - goto out; - } - - crypto_info = (struct tls_crypto_info *)&csk->tlshws.crypto_info; - - /* GCM mode of AES supports 128 and 256 bit encryption, so - * copy keys from user based on GCM cipher type. - */ - switch (tmp_crypto_info.cipher_type) { - case TLS_CIPHER_AES_GCM_128: { - /* Obtain version and type from previous copy */ - crypto_info[0] = tmp_crypto_info; - /* Now copy the following data */ - rc = copy_from_sockptr_offset((char *)crypto_info + - sizeof(*crypto_info), - optval, sizeof(*crypto_info), - sizeof(struct tls12_crypto_info_aes_gcm_128) - - sizeof(*crypto_info)); - - if (rc) { - rc = -EFAULT; - goto out; - } - - keylen = TLS_CIPHER_AES_GCM_128_KEY_SIZE; - cipher_type = TLS_CIPHER_AES_GCM_128; - break; - } - case TLS_CIPHER_AES_GCM_256: { - crypto_info[0] = tmp_crypto_info; - rc = copy_from_sockptr_offset((char *)crypto_info + - sizeof(*crypto_info), - optval, sizeof(*crypto_info), - sizeof(struct tls12_crypto_info_aes_gcm_256) - - sizeof(*crypto_info)); - - if (rc) { - rc = -EFAULT; - goto out; - } - - keylen = TLS_CIPHER_AES_GCM_256_KEY_SIZE; - cipher_type = TLS_CIPHER_AES_GCM_256; - break; - } - default: - rc = -EINVAL; - goto out; - } - rc = chtls_setkey(csk, keylen, optname, cipher_type); -out: - return rc; -} - -static int chtls_setsockopt(struct sock *sk, int level, int optname, - sockptr_t optval, unsigned int optlen) -{ - struct tls_context *ctx = tls_get_ctx(sk); - - if (level != SOL_TLS) - return ctx->sk_proto->setsockopt(sk, level, - optname, optval, optlen); - - return do_chtls_setsockopt(sk, optname, optval, optlen); -} - -static struct cxgb4_uld_info chtls_uld_info = { - .name = DRV_NAME, - .nrxq = MAX_ULD_QSETS, - .ntxq = MAX_ULD_QSETS, - .rxq_size = 1024, - .add = chtls_uld_add, - .state_change = chtls_uld_state_change, - .rx_handler = chtls_uld_rx_handler, -}; - -void chtls_install_cpl_ops(struct sock *sk) -{ - if (sk->sk_family == AF_INET) - sk->sk_prot = &chtls_cpl_prot; - else - sk->sk_prot = &chtls_cpl_protv6; -} - -static void __init chtls_init_ulp_ops(void) -{ - chtls_cpl_prot = tcp_prot; - chtls_init_rsk_ops(&chtls_cpl_prot, &chtls_rsk_ops, - &tcp_prot, PF_INET); - chtls_cpl_prot.close = chtls_close; - chtls_cpl_prot.disconnect = chtls_disconnect; - chtls_cpl_prot.destroy = chtls_destroy_sock; - chtls_cpl_prot.shutdown = chtls_shutdown; - chtls_cpl_prot.sendmsg = chtls_sendmsg; - chtls_cpl_prot.sendpage = chtls_sendpage; - chtls_cpl_prot.recvmsg = chtls_recvmsg; - chtls_cpl_prot.setsockopt = chtls_setsockopt; - chtls_cpl_prot.getsockopt = chtls_getsockopt; -#if IS_ENABLED(CONFIG_IPV6) - chtls_cpl_protv6 = chtls_cpl_prot; - chtls_init_rsk_ops(&chtls_cpl_protv6, &chtls_rsk_opsv6, - &tcpv6_prot, PF_INET6); -#endif -} - -static int __init chtls_register(void) -{ - chtls_init_ulp_ops(); - register_listen_notifier(&listen_notifier); - cxgb4_register_uld(CXGB4_ULD_TLS, &chtls_uld_info); - return 0; -} - -static void __exit chtls_unregister(void) -{ - unregister_listen_notifier(&listen_notifier); - chtls_free_all_uld(); - cxgb4_unregister_uld(CXGB4_ULD_TLS); -} - -module_init(chtls_register); -module_exit(chtls_unregister); - -MODULE_DESCRIPTION("Chelsio TLS Inline driver"); -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Chelsio Communications"); -MODULE_VERSION(DRV_VERSION); diff --git a/drivers/crypto/hifn_795x.c b/drivers/crypto/hifn_795x.c index 354836468c5d..7e7a8f01ea6b 100644 --- a/drivers/crypto/hifn_795x.c +++ b/drivers/crypto/hifn_795x.c @@ -780,8 +780,8 @@ static int hifn_register_rng(struct hifn_device *dev) dev->pk_clk_freq) * 256; dev->rng.name = dev->name; - dev->rng.data_present = hifn_rng_data_present, - dev->rng.data_read = hifn_rng_data_read, + dev->rng.data_present = hifn_rng_data_present; + dev->rng.data_read = hifn_rng_data_read; dev->rng.priv = (unsigned long)dev; return hwrng_register(&dev->rng); @@ -1235,7 +1235,8 @@ static int hifn_setup_src_desc(struct hifn_device *dev, struct page *page, int idx; dma_addr_t addr; - addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_TODEVICE); + addr = dma_map_page(&dev->pdev->dev, page, offset, size, + DMA_TO_DEVICE); idx = dma->srci; @@ -1293,7 +1294,8 @@ static void hifn_setup_dst_desc(struct hifn_device *dev, struct page *page, int idx; dma_addr_t addr; - addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_FROMDEVICE); + addr = dma_map_page(&dev->pdev->dev, page, offset, size, + DMA_FROM_DEVICE); idx = dma->dsti; dma->dstr[idx].p = __cpu_to_le32(addr); @@ -2470,7 +2472,7 @@ static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id) return err; pci_set_master(pdev); - err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); + err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); if (err) goto err_out_disable_pci_device; @@ -2514,8 +2516,9 @@ static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id) } } - dev->desc_virt = pci_zalloc_consistent(pdev, sizeof(struct hifn_dma), - &dev->desc_dma); + dev->desc_virt = dma_alloc_coherent(&pdev->dev, + sizeof(struct hifn_dma), + &dev->desc_dma, GFP_KERNEL); if (!dev->desc_virt) { dev_err(&pdev->dev, "Failed to allocate descriptor rings.\n"); err = -ENOMEM; @@ -2572,8 +2575,8 @@ err_out_free_irq: free_irq(dev->irq, dev); tasklet_kill(&dev->tasklet); err_out_free_desc: - pci_free_consistent(pdev, sizeof(struct hifn_dma), - dev->desc_virt, dev->desc_dma); + dma_free_coherent(&pdev->dev, sizeof(struct hifn_dma), dev->desc_virt, + dev->desc_dma); err_out_unmap_bars: for (i = 0; i < 3; ++i) @@ -2610,8 +2613,8 @@ static void hifn_remove(struct pci_dev *pdev) hifn_flush(dev); - pci_free_consistent(pdev, sizeof(struct hifn_dma), - dev->desc_virt, dev->desc_dma); + dma_free_coherent(&pdev->dev, sizeof(struct hifn_dma), + dev->desc_virt, dev->desc_dma); for (i = 0; i < 3; ++i) if (dev->bar[i]) iounmap(dev->bar[i]); @@ -2642,9 +2645,6 @@ static int __init hifn_init(void) unsigned int freq; int err; - /* HIFN supports only 32-bit addresses */ - BUILD_BUG_ON(sizeof(dma_addr_t) != 4); - if (strncmp(hifn_pll_ref, "ext", 3) && strncmp(hifn_pll_ref, "pci", 3)) { pr_err("hifn795x: invalid hifn_pll_ref clock, must be pci or ext"); diff --git a/drivers/crypto/hisilicon/hpre/hpre.h b/drivers/crypto/hisilicon/hpre/hpre.h index ed730d173e95..f69252b24671 100644 --- a/drivers/crypto/hisilicon/hpre/hpre.h +++ b/drivers/crypto/hisilicon/hpre/hpre.h @@ -56,7 +56,6 @@ struct hpre_dfx { * Just relevant for PF. */ struct hpre_debug { - struct dentry *debug_root; struct hpre_dfx dfx[HPRE_DFX_FILE_NUM]; struct hpre_debugfs_file files[HPRE_DEBUGFS_FILE_NUM]; }; diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index 7b5cb27d473d..a87f9904087a 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -98,9 +98,6 @@ struct hpre_asym_request { struct timespec64 req_time; }; -static DEFINE_MUTEX(hpre_alg_lock); -static unsigned int hpre_active_devs; - static int hpre_alloc_req_id(struct hpre_ctx *ctx) { unsigned long flags; @@ -191,8 +188,7 @@ static int hpre_get_data_dma_addr(struct hpre_asym_request *hpre_req, hpre_req->dst = NULL; dma_dir = DMA_FROM_DEVICE; } - *tmp = dma_map_single(dev, sg_virt(data), - len, dma_dir); + *tmp = dma_map_single(dev, sg_virt(data), len, dma_dir); if (unlikely(dma_mapping_error(dev, *tmp))) { dev_err(dev, "dma map data err!\n"); return -ENOMEM; @@ -242,8 +238,8 @@ static int hpre_hw_data_init(struct hpre_asym_request *hpre_req, ((is_dh && !is_src) || !is_dh)) ret = hpre_get_data_dma_addr(hpre_req, data, len, is_src, &tmp); else - ret = hpre_prepare_dma_buf(hpre_req, data, len, - is_src, &tmp); + ret = hpre_prepare_dma_buf(hpre_req, data, len, is_src, &tmp); + if (unlikely(ret)) return ret; @@ -270,11 +266,9 @@ static void hpre_hw_data_clr_all(struct hpre_ctx *ctx, if (src) { if (req->src) - dma_free_coherent(dev, ctx->key_sz, - req->src, tmp); + dma_free_coherent(dev, ctx->key_sz, req->src, tmp); else - dma_unmap_single(dev, tmp, - ctx->key_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, tmp, ctx->key_sz, DMA_TO_DEVICE); } tmp = le64_to_cpu(sqe->out); @@ -477,7 +471,7 @@ static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa) h_req->areq.dh = kreq; msg = &h_req->req; memset(msg, 0, sizeof(*msg)); - msg->key = cpu_to_le64((u64)ctx->dh.dma_xa_p); + msg->key = cpu_to_le64(ctx->dh.dma_xa_p); } msg->dw0 |= cpu_to_le32(0x1 << HPRE_SQE_DONE_SHIFT); @@ -534,6 +528,8 @@ static int hpre_dh_compute_value(struct kpp_request *req) ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 1); if (unlikely(ret)) goto clear_all; + } else { + msg->in = cpu_to_le64(ctx->dh.dma_g); } ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 1); @@ -743,7 +739,7 @@ static int hpre_rsa_enc(struct akcipher_request *req) return ret; msg->dw0 |= cpu_to_le32(HPRE_ALG_NC_NCRT); - msg->key = cpu_to_le64((u64)ctx->rsa.dma_pubkey); + msg->key = cpu_to_le64(ctx->rsa.dma_pubkey); ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 0); if (unlikely(ret)) @@ -791,11 +787,11 @@ static int hpre_rsa_dec(struct akcipher_request *req) return ret; if (ctx->crt_g2_mode) { - msg->key = cpu_to_le64((u64)ctx->rsa.dma_crt_prikey); + msg->key = cpu_to_le64(ctx->rsa.dma_crt_prikey); msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_NC_CRT); } else { - msg->key = cpu_to_le64((u64)ctx->rsa.dma_prikey); + msg->key = cpu_to_le64(ctx->rsa.dma_prikey); msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_NC_NCRT); } @@ -1160,36 +1156,25 @@ static struct kpp_alg dh = { int hpre_algs_register(void) { - int ret = 0; - - mutex_lock(&hpre_alg_lock); - if (++hpre_active_devs == 1) { - rsa.base.cra_flags = 0; - ret = crypto_register_akcipher(&rsa); - if (ret) - goto unlock; + int ret; + + rsa.base.cra_flags = 0; + ret = crypto_register_akcipher(&rsa); + if (ret) + return ret; #ifdef CONFIG_CRYPTO_DH - ret = crypto_register_kpp(&dh); - if (ret) { - crypto_unregister_akcipher(&rsa); - goto unlock; - } + ret = crypto_register_kpp(&dh); + if (ret) + crypto_unregister_akcipher(&rsa); #endif - } -unlock: - mutex_unlock(&hpre_alg_lock); return ret; } void hpre_algs_unregister(void) { - mutex_lock(&hpre_alg_lock); - if (--hpre_active_devs == 0) { - crypto_unregister_akcipher(&rsa); + crypto_unregister_akcipher(&rsa); #ifdef CONFIG_CRYPTO_DH - crypto_unregister_kpp(&dh); + crypto_unregister_kpp(&dh); #endif - } - mutex_unlock(&hpre_alg_lock); } diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index b135c74fb619..a33394d91bbf 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -90,7 +90,6 @@ #define HPRE_SQE_MASK_OFFSET 8 #define HPRE_SQE_MASK_LEN 24 -static struct hisi_qm_list hpre_devices; static const char hpre_name[] = "hisi_hpre"; static struct dentry *hpre_debugfs_root; static const struct pci_device_id hpre_dev_ids[] = { @@ -106,6 +105,11 @@ struct hpre_hw_error { const char *msg; }; +static struct hisi_qm_list hpre_devices = { + .register_to_crypto = hpre_algs_register, + .unregister_from_crypto = hpre_algs_unregister, +}; + static const char * const hpre_debug_file_name[] = { [HPRE_CURRENT_QM] = "current_qm", [HPRE_CLEAR_ENABLE] = "rdclr_en", @@ -186,7 +190,7 @@ static const struct kernel_param_ops hpre_pf_q_num_ops = { static u32 pf_q_num = HPRE_PF_DEF_Q_NUM; module_param_cb(pf_q_num, &hpre_pf_q_num_ops, &pf_q_num, 0444); -MODULE_PARM_DESC(pf_q_num, "Number of queues in PF of CS(1-1024)"); +MODULE_PARM_DESC(pf_q_num, "Number of queues in PF of CS(2-1024)"); static const struct kernel_param_ops vfs_num_ops = { .set = vfs_num_set, @@ -864,9 +868,7 @@ static int hpre_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) dev_warn(&pdev->dev, "init debugfs fail!\n"); - hisi_qm_add_to_list(qm, &hpre_devices); - - ret = hpre_algs_register(); + ret = hisi_qm_alg_register(qm, &hpre_devices); if (ret < 0) { pci_err(pdev, "fail to register algs to crypto!\n"); goto err_with_qm_start; @@ -875,18 +877,17 @@ static int hpre_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (qm->fun_type == QM_HW_PF && vfs_num) { ret = hisi_qm_sriov_enable(pdev, vfs_num); if (ret < 0) - goto err_with_crypto_register; + goto err_with_alg_register; } return 0; -err_with_crypto_register: - hpre_algs_unregister(); +err_with_alg_register: + hisi_qm_alg_unregister(qm, &hpre_devices); err_with_qm_start: - hisi_qm_del_from_list(qm, &hpre_devices); hpre_debugfs_exit(qm); - hisi_qm_stop(qm); + hisi_qm_stop(qm, QM_NORMAL); err_with_err_init: hisi_qm_dev_err_uninit(qm); @@ -899,14 +900,13 @@ err_with_qm_init: static void hpre_remove(struct pci_dev *pdev) { - struct hpre *hpre = pci_get_drvdata(pdev); - struct hisi_qm *qm = &hpre->qm; + struct hisi_qm *qm = pci_get_drvdata(pdev); int ret; - hpre_algs_unregister(); - hisi_qm_del_from_list(qm, &hpre_devices); + hisi_qm_wait_task_finish(qm, &hpre_devices); + hisi_qm_alg_unregister(qm, &hpre_devices); if (qm->fun_type == QM_HW_PF && qm->vfs_num) { - ret = hisi_qm_sriov_disable(pdev); + ret = hisi_qm_sriov_disable(pdev, qm->is_frozen); if (ret) { pci_err(pdev, "Disable SRIOV fail!\n"); return; @@ -918,7 +918,7 @@ static void hpre_remove(struct pci_dev *pdev) } hpre_debugfs_exit(qm); - hisi_qm_stop(qm); + hisi_qm_stop(qm, QM_NORMAL); hisi_qm_dev_err_uninit(qm); hisi_qm_uninit(qm); } @@ -939,6 +939,7 @@ static struct pci_driver hpre_pci_driver = { .sriov_configure = IS_ENABLED(CONFIG_PCI_IOV) ? hisi_qm_sriov_configure : NULL, .err_handler = &hpre_err_handler, + .shutdown = hisi_qm_dev_shutdown, }; static void hpre_register_debugfs(void) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 6527c53b073f..530f23116d7c 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -180,7 +180,10 @@ #define QM_DBG_TMP_BUF_LEN 22 #define QM_PCI_COMMAND_INVALID ~0 +#define WAIT_PERIOD 20 +#define REMOVE_WAIT_DELAY 10 #define QM_SQE_ADDR_MASK GENMASK(7, 0) +#define QM_EQ_DEPTH (1024 * 2) #define QM_MK_CQC_DW3_V1(hop_num, pg_sz, buf_sz, cqe_sz) \ (((hop_num) << QM_CQ_HOP_NUM_SHIFT) | \ @@ -652,7 +655,7 @@ static void qm_work_process(struct work_struct *work) qp = qm_to_hisi_qp(qm, eqe); qm_poll_qp(qp, qm); - if (qm->status.eq_head == QM_Q_DEPTH - 1) { + if (qm->status.eq_head == QM_EQ_DEPTH - 1) { qm->status.eqc_phase = !qm->status.eqc_phase; eqe = qm->eqe; qm->status.eq_head = 0; @@ -661,7 +664,7 @@ static void qm_work_process(struct work_struct *work) qm->status.eq_head++; } - if (eqe_num == QM_Q_DEPTH / 2 - 1) { + if (eqe_num == QM_EQ_DEPTH / 2 - 1) { eqe_num = 0; qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0); } @@ -754,7 +757,7 @@ static void qm_init_qp_status(struct hisi_qp *qp) qp_status->sq_tail = 0; qp_status->cq_head = 0; qp_status->cqc_phase = true; - atomic_set(&qp_status->flags, 0); + atomic_set(&qp_status->used, 0); } static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base, @@ -1046,17 +1049,7 @@ static int qm_regs_show(struct seq_file *s, void *unused) return 0; } -static int qm_regs_open(struct inode *inode, struct file *file) -{ - return single_open(file, qm_regs_show, inode->i_private); -} - -static const struct file_operations qm_regs_fops = { - .owner = THIS_MODULE, - .open = qm_regs_open, - .read = seq_read, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(qm_regs); static ssize_t qm_cmd_read(struct file *filp, char __user *buffer, size_t count, loff_t *pos) @@ -1370,7 +1363,13 @@ static int qm_eq_aeq_dump(struct hisi_qm *qm, const char *s, return -EINVAL; ret = kstrtou32(s, 0, &xeqe_id); - if (ret || xeqe_id >= QM_Q_DEPTH) { + if (ret) + return -EINVAL; + + if (!strcmp(name, "EQE") && xeqe_id >= QM_EQ_DEPTH) { + dev_err(dev, "Please input eqe num (0-%d)", QM_EQ_DEPTH - 1); + return -EINVAL; + } else if (!strcmp(name, "AEQE") && xeqe_id >= QM_Q_DEPTH) { dev_err(dev, "Please input aeqe num (0-%d)", QM_Q_DEPTH - 1); return -EINVAL; } @@ -1420,17 +1419,18 @@ static int qm_dbg_help(struct hisi_qm *qm, char *s) static int qm_cmd_write_dump(struct hisi_qm *qm, const char *cmd_buf) { struct device *dev = &qm->pdev->dev; - char *presult, *s; + char *presult, *s, *s_tmp; int ret; s = kstrdup(cmd_buf, GFP_KERNEL); if (!s) return -ENOMEM; + s_tmp = s; presult = strsep(&s, " "); if (!presult) { - kfree(s); - return -EINVAL; + ret = -EINVAL; + goto err_buffer_free; } if (!strcmp(presult, "sqc")) @@ -1459,7 +1459,8 @@ static int qm_cmd_write_dump(struct hisi_qm *qm, const char *cmd_buf) if (ret) dev_info(dev, "Please echo help\n"); - kfree(s); +err_buffer_free: + kfree(s_tmp); return ret; } @@ -1644,7 +1645,7 @@ static void *qm_get_avail_sqe(struct hisi_qp *qp) struct hisi_qp_status *qp_status = &qp->qp_status; u16 sq_tail = qp_status->sq_tail; - if (unlikely(atomic_read(&qp->qp_status.used) == QM_Q_DEPTH)) + if (unlikely(atomic_read(&qp->qp_status.used) == QM_Q_DEPTH - 1)) return NULL; return qp->sqe + sq_tail * qp->qm->sqe_size; @@ -1981,7 +1982,7 @@ int hisi_qp_send(struct hisi_qp *qp, const void *msg) if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP || atomic_read(&qp->qm->status.flags) == QM_STOP || qp->is_resetting)) { - dev_info(&qp->qm->pdev->dev, "QP is stopped or resetting\n"); + dev_info_ratelimited(&qp->qm->pdev->dev, "QP is stopped or resetting\n"); return -EAGAIN; } @@ -2215,6 +2216,82 @@ static int qm_alloc_uacce(struct hisi_qm *qm) } /** + * qm_frozen() - Try to froze QM to cut continuous queue request. If + * there is user on the QM, return failure without doing anything. + * @qm: The qm needed to be fronzen. + * + * This function frozes QM, then we can do SRIOV disabling. + */ +static int qm_frozen(struct hisi_qm *qm) +{ + down_write(&qm->qps_lock); + + if (qm->is_frozen) { + up_write(&qm->qps_lock); + return 0; + } + + if (!qm->qp_in_used) { + qm->qp_in_used = qm->qp_num; + qm->is_frozen = true; + up_write(&qm->qps_lock); + return 0; + } + + up_write(&qm->qps_lock); + + return -EBUSY; +} + +static int qm_try_frozen_vfs(struct pci_dev *pdev, + struct hisi_qm_list *qm_list) +{ + struct hisi_qm *qm, *vf_qm; + struct pci_dev *dev; + int ret = 0; + + if (!qm_list || !pdev) + return -EINVAL; + + /* Try to frozen all the VFs as disable SRIOV */ + mutex_lock(&qm_list->lock); + list_for_each_entry(qm, &qm_list->list, list) { + dev = qm->pdev; + if (dev == pdev) + continue; + if (pci_physfn(dev) == pdev) { + vf_qm = pci_get_drvdata(dev); + ret = qm_frozen(vf_qm); + if (ret) + goto frozen_fail; + } + } + +frozen_fail: + mutex_unlock(&qm_list->lock); + + return ret; +} + +/** + * hisi_qm_wait_task_finish() - Wait until the task is finished + * when removing the driver. + * @qm: The qm needed to wait for the task to finish. + * @qm_list: The list of all available devices. + */ +void hisi_qm_wait_task_finish(struct hisi_qm *qm, struct hisi_qm_list *qm_list) +{ + while (qm_frozen(qm) || + ((qm->fun_type == QM_HW_PF) && + qm_try_frozen_vfs(qm->pdev, qm_list))) { + msleep(WAIT_PERIOD); + } + + udelay(REMOVE_WAIT_DELAY); +} +EXPORT_SYMBOL_GPL(hisi_qm_wait_task_finish); + +/** * hisi_qm_get_free_qp_num() - Get free number of qp in qm. * @qm: The qm which want to get free qp. * @@ -2282,7 +2359,7 @@ static int hisi_qm_memory_init(struct hisi_qm *qm) } while (0) idr_init(&qm->qp_idr); - qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * QM_Q_DEPTH) + + qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * QM_EQ_DEPTH) + QMC_ALIGN(sizeof(struct qm_aeqe) * QM_Q_DEPTH) + QMC_ALIGN(sizeof(struct qm_sqc) * qm->qp_num) + QMC_ALIGN(sizeof(struct qm_cqc) * qm->qp_num); @@ -2292,7 +2369,7 @@ static int hisi_qm_memory_init(struct hisi_qm *qm) if (!qm->qdma.va) return -ENOMEM; - QM_INIT_BUF(qm, eqe, QM_Q_DEPTH); + QM_INIT_BUF(qm, eqe, QM_EQ_DEPTH); QM_INIT_BUF(qm, aeqe, QM_Q_DEPTH); QM_INIT_BUF(qm, sqc, qm->qp_num); QM_INIT_BUF(qm, cqc, qm->qp_num); @@ -2338,6 +2415,7 @@ static void hisi_qm_pre_init(struct hisi_qm *qm) mutex_init(&qm->mailbox_lock); init_rwsem(&qm->qps_lock); qm->qp_in_used = 0; + qm->is_frozen = false; } /** @@ -2462,7 +2540,7 @@ static int qm_eq_ctx_cfg(struct hisi_qm *qm) eqc->base_h = cpu_to_le32(upper_32_bits(qm->eqe_dma)); if (qm->ver == QM_HW_V1) eqc->dw3 = cpu_to_le32(QM_EQE_AEQE_SIZE); - eqc->dw6 = cpu_to_le32((QM_Q_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT)); + eqc->dw6 = cpu_to_le32((QM_EQ_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT)); ret = qm_mb(qm, QM_MB_CMD_EQC, eqc_dma, 0, 0); dma_unmap_single(dev, eqc_dma, sizeof(struct qm_eqc), DMA_TO_DEVICE); kfree(eqc); @@ -2633,18 +2711,20 @@ static void qm_clear_queues(struct hisi_qm *qm) /** * hisi_qm_stop() - Stop a qm. * @qm: The qm which will be stopped. + * @r: The reason to stop qm. * * This function stops qm and its qps, then qm can not accept request. * Related resources are not released at this state, we can use hisi_qm_start * to let qm start again. */ -int hisi_qm_stop(struct hisi_qm *qm) +int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r) { struct device *dev = &qm->pdev->dev; int ret = 0; down_write(&qm->qps_lock); + qm->status.stop_reason = r; if (!qm_avail_state(qm, QM_STOP)) { ret = -EPERM; goto err_unlock; @@ -3081,11 +3161,12 @@ EXPORT_SYMBOL_GPL(hisi_qm_sriov_enable); /** * hisi_qm_sriov_disable - disable virtual functions - * @pdev: the PCI device + * @pdev: the PCI device. + * @is_frozen: true when all the VFs are frozen. * - * Return failure if there are VFs assigned already. + * Return failure if there are VFs assigned already or VF is in used. */ -int hisi_qm_sriov_disable(struct pci_dev *pdev) +int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen) { struct hisi_qm *qm = pci_get_drvdata(pdev); @@ -3094,7 +3175,12 @@ int hisi_qm_sriov_disable(struct pci_dev *pdev) return -EPERM; } - /* remove in hpre_pci_driver will be called to free VF resources */ + /* While VF is in used, SRIOV cannot be disabled. */ + if (!is_frozen && qm_try_frozen_vfs(pdev, qm->qm_list)) { + pci_err(pdev, "Task is using its VF!\n"); + return -EBUSY; + } + pci_disable_sriov(pdev); return qm_clear_vft_config(qm); } @@ -3110,7 +3196,7 @@ EXPORT_SYMBOL_GPL(hisi_qm_sriov_disable); int hisi_qm_sriov_configure(struct pci_dev *pdev, int num_vfs) { if (num_vfs == 0) - return hisi_qm_sriov_disable(pdev); + return hisi_qm_sriov_disable(pdev, 0); else return hisi_qm_sriov_enable(pdev, num_vfs); } @@ -3290,10 +3376,10 @@ static int qm_set_msi(struct hisi_qm *qm, bool set) return 0; } -static int qm_vf_reset_prepare(struct hisi_qm *qm) +static int qm_vf_reset_prepare(struct hisi_qm *qm, + enum qm_stop_reason stop_reason) { struct hisi_qm_list *qm_list = qm->qm_list; - int stop_reason = qm->status.stop_reason; struct pci_dev *pdev = qm->pdev; struct pci_dev *virtfn; struct hisi_qm *vf_qm; @@ -3306,8 +3392,10 @@ static int qm_vf_reset_prepare(struct hisi_qm *qm) continue; if (pci_physfn(virtfn) == pdev) { - vf_qm->status.stop_reason = stop_reason; - ret = hisi_qm_stop(vf_qm); + /* save VFs PCIE BAR configuration */ + pci_save_state(virtfn); + + ret = hisi_qm_stop(vf_qm, stop_reason); if (ret) goto stop_fail; } @@ -3346,15 +3434,14 @@ static int qm_controller_reset_prepare(struct hisi_qm *qm) } if (qm->vfs_num) { - ret = qm_vf_reset_prepare(qm); + ret = qm_vf_reset_prepare(qm, QM_SOFT_RESET); if (ret) { pci_err(pdev, "Fails to stop VFs!\n"); return ret; } } - qm->status.stop_reason = QM_SOFT_RESET; - ret = hisi_qm_stop(qm); + ret = hisi_qm_stop(qm, QM_SOFT_RESET); if (ret) { pci_err(pdev, "Fails to stop QM!\n"); return ret; @@ -3471,6 +3558,9 @@ static int qm_vf_reset_done(struct hisi_qm *qm) continue; if (pci_physfn(virtfn) == pdev) { + /* enable VFs PCIE BAR configuration */ + pci_restore_state(virtfn); + ret = qm_restart(vf_qm); if (ret) goto restart_fail; @@ -3695,7 +3785,7 @@ void hisi_qm_reset_prepare(struct pci_dev *pdev) } if (qm->vfs_num) { - ret = qm_vf_reset_prepare(qm); + ret = qm_vf_reset_prepare(qm, QM_FLR); if (ret) { pci_err(pdev, "Failed to prepare reset, ret = %d.\n", ret); @@ -3703,7 +3793,7 @@ void hisi_qm_reset_prepare(struct pci_dev *pdev) } } - ret = hisi_qm_stop(qm); + ret = hisi_qm_stop(qm, QM_FLR); if (ret) { pci_err(pdev, "Failed to stop QM, ret = %d.\n", ret); return; @@ -3821,6 +3911,23 @@ err_aeq_irq: return ret; } +/** + * hisi_qm_dev_shutdown() - Shutdown device. + * @pdev: The device will be shutdown. + * + * This function will stop qm when OS shutdown or rebooting. + */ +void hisi_qm_dev_shutdown(struct pci_dev *pdev) +{ + struct hisi_qm *qm = pci_get_drvdata(pdev); + int ret; + + ret = hisi_qm_stop(qm, QM_NORMAL); + if (ret) + dev_err(&pdev->dev, "Fail to stop qm in shutdown!\n"); +} +EXPORT_SYMBOL_GPL(hisi_qm_dev_shutdown); + static void hisi_qm_controller_reset(struct work_struct *rst_work) { struct hisi_qm *qm = container_of(rst_work, struct hisi_qm, rst_work); @@ -3834,6 +3941,58 @@ static void hisi_qm_controller_reset(struct work_struct *rst_work) } /** + * hisi_qm_alg_register() - Register alg to crypto and add qm to qm_list. + * @qm: The qm needs add. + * @qm_list: The qm list. + * + * This function adds qm to qm list, and will register algorithm to + * crypto when the qm list is empty. + */ +int hisi_qm_alg_register(struct hisi_qm *qm, struct hisi_qm_list *qm_list) +{ + int flag = 0; + int ret = 0; + + mutex_lock(&qm_list->lock); + if (list_empty(&qm_list->list)) + flag = 1; + list_add_tail(&qm->list, &qm_list->list); + mutex_unlock(&qm_list->lock); + + if (flag) { + ret = qm_list->register_to_crypto(); + if (ret) { + mutex_lock(&qm_list->lock); + list_del(&qm->list); + mutex_unlock(&qm_list->lock); + } + } + + return ret; +} +EXPORT_SYMBOL_GPL(hisi_qm_alg_register); + +/** + * hisi_qm_alg_unregister() - Unregister alg from crypto and delete qm from + * qm list. + * @qm: The qm needs delete. + * @qm_list: The qm list. + * + * This function deletes qm from qm list, and will unregister algorithm + * from crypto when the qm list is empty. + */ +void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list) +{ + mutex_lock(&qm_list->lock); + list_del(&qm->list); + mutex_unlock(&qm_list->lock); + + if (list_empty(&qm_list->list)) + qm_list->unregister_from_crypto(); +} +EXPORT_SYMBOL_GPL(hisi_qm_alg_unregister); + +/** * hisi_qm_init() - Initialize configures about qm. * @qm: The qm needing init. * diff --git a/drivers/crypto/hisilicon/qm.h b/drivers/crypto/hisilicon/qm.h index 6c1d3c7d64ee..0420f4ce7197 100644 --- a/drivers/crypto/hisilicon/qm.h +++ b/drivers/crypto/hisilicon/qm.h @@ -79,7 +79,7 @@ #define QM_BASE_CE QM_ECC_1BIT #define QM_Q_DEPTH 1024 - +#define QM_MIN_QNUM 2 #define HISI_ACC_SGL_SGE_NR_MAX 255 /* page number for queue file region */ @@ -193,6 +193,8 @@ struct hisi_qm_err_ini { struct hisi_qm_list { struct mutex lock; struct list_head list; + int (*register_to_crypto)(void); + void (*unregister_from_crypto)(void); }; struct hisi_qm { @@ -243,6 +245,7 @@ struct hisi_qm { const char *algs; bool use_sva; + bool is_frozen; resource_size_t phys_base; resource_size_t phys_size; struct uacce_device *uacce; @@ -306,7 +309,7 @@ static inline int q_num_set(const char *val, const struct kernel_param *kp, } ret = kstrtou32(val, 10, &n); - if (ret || !n || n > q_num) + if (ret || n < QM_MIN_QNUM || n > q_num) return -EINVAL; return param_set_int(val, kp); @@ -336,26 +339,10 @@ static inline void hisi_qm_init_list(struct hisi_qm_list *qm_list) mutex_init(&qm_list->lock); } -static inline void hisi_qm_add_to_list(struct hisi_qm *qm, - struct hisi_qm_list *qm_list) -{ - mutex_lock(&qm_list->lock); - list_add_tail(&qm->list, &qm_list->list); - mutex_unlock(&qm_list->lock); -} - -static inline void hisi_qm_del_from_list(struct hisi_qm *qm, - struct hisi_qm_list *qm_list) -{ - mutex_lock(&qm_list->lock); - list_del(&qm->list); - mutex_unlock(&qm_list->lock); -} - int hisi_qm_init(struct hisi_qm *qm); void hisi_qm_uninit(struct hisi_qm *qm); int hisi_qm_start(struct hisi_qm *qm); -int hisi_qm_stop(struct hisi_qm *qm); +int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r); struct hisi_qp *hisi_qm_create_qp(struct hisi_qm *qm, u8 alg_type); int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg); int hisi_qm_stop_qp(struct hisi_qp *qp); @@ -367,7 +354,7 @@ int hisi_qm_debug_init(struct hisi_qm *qm); enum qm_hw_ver hisi_qm_get_hw_version(struct pci_dev *pdev); void hisi_qm_debug_regs_clear(struct hisi_qm *qm); int hisi_qm_sriov_enable(struct pci_dev *pdev, int max_vfs); -int hisi_qm_sriov_disable(struct pci_dev *pdev); +int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen); int hisi_qm_sriov_configure(struct pci_dev *pdev, int num_vfs); void hisi_qm_dev_err_init(struct hisi_qm *qm); void hisi_qm_dev_err_uninit(struct hisi_qm *qm); @@ -390,4 +377,8 @@ void hisi_acc_free_sgl_pool(struct device *dev, int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num, u8 alg_type, int node, struct hisi_qp **qps); void hisi_qm_free_qps(struct hisi_qp **qps, int qp_num); +void hisi_qm_dev_shutdown(struct pci_dev *pdev); +void hisi_qm_wait_task_finish(struct hisi_qm *qm, struct hisi_qm_list *qm_list); +int hisi_qm_alg_register(struct hisi_qm *qm, struct hisi_qm_list *qm_list); +void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list); #endif diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 497969ae8b23..bb493423668c 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -66,8 +66,6 @@ #define SEC_SQE_AEAD_FLAG 3 #define SEC_SQE_DONE 0x1 -static atomic_t sec_active_devs; - /* Get an en/de-cipher queue cyclically to balance load over queues of TFM */ static inline int sec_alloc_queue_id(struct sec_ctx *ctx, struct sec_req *req) { @@ -342,11 +340,14 @@ static int sec_alg_resource_alloc(struct sec_ctx *ctx, ret = sec_alloc_pbuf_resource(dev, res); if (ret) { dev_err(dev, "fail to alloc pbuf dma resource!\n"); - goto alloc_fail; + goto alloc_pbuf_fail; } } return 0; +alloc_pbuf_fail: + if (ctx->alg_type == SEC_AEAD) + sec_free_mac_resource(dev, qp_ctx->res); alloc_fail: sec_free_civ_resource(dev, res); @@ -457,8 +458,10 @@ static int sec_ctx_base_init(struct sec_ctx *ctx) ctx->fake_req_limit = QM_Q_DEPTH >> 1; ctx->qp_ctx = kcalloc(sec->ctx_q_num, sizeof(struct sec_qp_ctx), GFP_KERNEL); - if (!ctx->qp_ctx) - return -ENOMEM; + if (!ctx->qp_ctx) { + ret = -ENOMEM; + goto err_destroy_qps; + } for (i = 0; i < sec->ctx_q_num; i++) { ret = sec_create_qp_ctx(&sec->qm, ctx, i, 0); @@ -467,12 +470,15 @@ static int sec_ctx_base_init(struct sec_ctx *ctx) } return 0; + err_sec_release_qp_ctx: for (i = i - 1; i >= 0; i--) sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]); - sec_destroy_qps(ctx->qps, sec->ctx_q_num); kfree(ctx->qp_ctx); +err_destroy_qps: + sec_destroy_qps(ctx->qps, sec->ctx_q_num); + return ret; } @@ -1633,33 +1639,24 @@ static struct aead_alg sec_aeads[] = { int sec_register_to_crypto(void) { - int ret = 0; + int ret; /* To avoid repeat register */ - if (atomic_add_return(1, &sec_active_devs) == 1) { - ret = crypto_register_skciphers(sec_skciphers, - ARRAY_SIZE(sec_skciphers)); - if (ret) - return ret; - - ret = crypto_register_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); - if (ret) - goto reg_aead_fail; - } - - return ret; - -reg_aead_fail: - crypto_unregister_skciphers(sec_skciphers, ARRAY_SIZE(sec_skciphers)); + ret = crypto_register_skciphers(sec_skciphers, + ARRAY_SIZE(sec_skciphers)); + if (ret) + return ret; + ret = crypto_register_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); + if (ret) + crypto_unregister_skciphers(sec_skciphers, + ARRAY_SIZE(sec_skciphers)); return ret; } void sec_unregister_from_crypto(void) { - if (atomic_sub_return(1, &sec_active_devs) == 0) { - crypto_unregister_skciphers(sec_skciphers, - ARRAY_SIZE(sec_skciphers)); - crypto_unregister_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); - } + crypto_unregister_skciphers(sec_skciphers, + ARRAY_SIZE(sec_skciphers)); + crypto_unregister_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); } diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 2297425486cb..548896394c4b 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -99,7 +99,11 @@ struct sec_dfx_item { static const char sec_name[] = "hisi_sec2"; static struct dentry *sec_debugfs_root; -static struct hisi_qm_list sec_devices; + +static struct hisi_qm_list sec_devices = { + .register_to_crypto = sec_register_to_crypto, + .unregister_from_crypto = sec_unregister_from_crypto, +}; static const struct sec_hw_error sec_hw_errors[] = { {.int_msk = BIT(0), .msg = "sec_axi_rresp_err_rint"}, @@ -165,7 +169,7 @@ static const struct kernel_param_ops sec_pf_q_num_ops = { static u32 pf_q_num = SEC_PF_DEF_Q_NUM; module_param_cb(pf_q_num, &sec_pf_q_num_ops, &pf_q_num, 0444); -MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 0-4096, v2 0-1024)"); +MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 2-4096, v2 2-1024)"); static int sec_ctx_q_num_set(const char *val, const struct kernel_param *kp) { @@ -879,29 +883,26 @@ static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) pci_warn(pdev, "Failed to init debugfs!\n"); - hisi_qm_add_to_list(qm, &sec_devices); - - ret = sec_register_to_crypto(); + ret = hisi_qm_alg_register(qm, &sec_devices); if (ret < 0) { pr_err("Failed to register driver to crypto.\n"); - goto err_remove_from_list; + goto err_qm_stop; } if (qm->fun_type == QM_HW_PF && vfs_num) { ret = hisi_qm_sriov_enable(pdev, vfs_num); if (ret < 0) - goto err_crypto_unregister; + goto err_alg_unregister; } return 0; -err_crypto_unregister: - sec_unregister_from_crypto(); +err_alg_unregister: + hisi_qm_alg_unregister(qm, &sec_devices); -err_remove_from_list: - hisi_qm_del_from_list(qm, &sec_devices); +err_qm_stop: sec_debugfs_exit(qm); - hisi_qm_stop(qm); + hisi_qm_stop(qm, QM_NORMAL); err_probe_uninit: sec_probe_uninit(qm); @@ -914,19 +915,16 @@ err_qm_uninit: static void sec_remove(struct pci_dev *pdev) { - struct sec_dev *sec = pci_get_drvdata(pdev); - struct hisi_qm *qm = &sec->qm; - - sec_unregister_from_crypto(); - - hisi_qm_del_from_list(qm, &sec_devices); + struct hisi_qm *qm = pci_get_drvdata(pdev); + hisi_qm_wait_task_finish(qm, &sec_devices); + hisi_qm_alg_unregister(qm, &sec_devices); if (qm->fun_type == QM_HW_PF && qm->vfs_num) - hisi_qm_sriov_disable(pdev); + hisi_qm_sriov_disable(pdev, qm->is_frozen); sec_debugfs_exit(qm); - (void)hisi_qm_stop(qm); + (void)hisi_qm_stop(qm, QM_NORMAL); if (qm->fun_type == QM_HW_PF) sec_debug_regs_clear(qm); @@ -950,6 +948,7 @@ static struct pci_driver sec_pci_driver = { .remove = sec_remove, .err_handler = &sec_err_handler, .sriov_configure = hisi_qm_sriov_configure, + .shutdown = hisi_qm_dev_shutdown, }; static void sec_register_debugfs(void) diff --git a/drivers/crypto/hisilicon/zip/zip.h b/drivers/crypto/hisilicon/zip/zip.h index 4484be13812b..92397f993e23 100644 --- a/drivers/crypto/hisilicon/zip/zip.h +++ b/drivers/crypto/hisilicon/zip/zip.h @@ -9,20 +9,6 @@ #include <linux/list.h> #include "../qm.h" -/* hisi_zip_sqe dw3 */ -#define HZIP_BD_STATUS_M GENMASK(7, 0) -/* hisi_zip_sqe dw7 */ -#define HZIP_IN_SGE_DATA_OFFSET_M GENMASK(23, 0) -/* hisi_zip_sqe dw8 */ -#define HZIP_OUT_SGE_DATA_OFFSET_M GENMASK(23, 0) -/* hisi_zip_sqe dw9 */ -#define HZIP_REQ_TYPE_M GENMASK(7, 0) -#define HZIP_ALG_TYPE_ZLIB 0x02 -#define HZIP_ALG_TYPE_GZIP 0x03 -#define HZIP_BUF_TYPE_M GENMASK(11, 8) -#define HZIP_PBUFFER 0x0 -#define HZIP_SGL 0x1 - enum hisi_zip_error_type { /* negative compression */ HZIP_NC_ERR = 0x0d, @@ -39,7 +25,6 @@ struct hisi_zip_ctrl; struct hisi_zip { struct hisi_qm qm; - struct list_head list; struct hisi_zip_ctrl *ctrl; struct hisi_zip_dfx dfx; }; diff --git a/drivers/crypto/hisilicon/zip/zip_crypto.c b/drivers/crypto/hisilicon/zip/zip_crypto.c index 01fd6a78111d..08b4660b014c 100644 --- a/drivers/crypto/hisilicon/zip/zip_crypto.c +++ b/drivers/crypto/hisilicon/zip/zip_crypto.c @@ -6,6 +6,20 @@ #include <linux/scatterlist.h> #include "zip.h" +/* hisi_zip_sqe dw3 */ +#define HZIP_BD_STATUS_M GENMASK(7, 0) +/* hisi_zip_sqe dw7 */ +#define HZIP_IN_SGE_DATA_OFFSET_M GENMASK(23, 0) +/* hisi_zip_sqe dw8 */ +#define HZIP_OUT_SGE_DATA_OFFSET_M GENMASK(23, 0) +/* hisi_zip_sqe dw9 */ +#define HZIP_REQ_TYPE_M GENMASK(7, 0) +#define HZIP_ALG_TYPE_ZLIB 0x02 +#define HZIP_ALG_TYPE_GZIP 0x03 +#define HZIP_BUF_TYPE_M GENMASK(11, 8) +#define HZIP_PBUFFER 0x0 +#define HZIP_SGL 0x1 + #define HZIP_ZLIB_HEAD_SIZE 2 #define HZIP_GZIP_HEAD_SIZE 10 @@ -16,22 +30,29 @@ #define GZIP_HEAD_FLG_SHIFT 3 #define GZIP_HEAD_FEXTRA_SHIFT 10 -#define GZIP_HEAD_FEXTRA_XLEN 2 +#define GZIP_HEAD_FEXTRA_XLEN 2UL #define GZIP_HEAD_FHCRC_SIZE 2 -#define HZIP_CTX_Q_NUM 2 #define HZIP_GZIP_HEAD_BUF 256 #define HZIP_ALG_PRIORITY 300 #define HZIP_SGL_SGE_NR 10 static const u8 zlib_head[HZIP_ZLIB_HEAD_SIZE] = {0x78, 0x9c}; -static const u8 gzip_head[HZIP_GZIP_HEAD_SIZE] = {0x1f, 0x8b, 0x08, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x03}; +static const u8 gzip_head[HZIP_GZIP_HEAD_SIZE] = { + 0x1f, 0x8b, 0x08, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x03 +}; + enum hisi_zip_alg_type { HZIP_ALG_TYPE_COMP = 0, HZIP_ALG_TYPE_DECOMP = 1, }; +enum { + HZIP_QPC_COMP, + HZIP_QPC_DECOMP, + HZIP_CTX_Q_NUM +}; + #define COMP_NAME_TO_TYPE(alg_name) \ (!strcmp((alg_name), "zlib-deflate") ? HZIP_ALG_TYPE_ZLIB : \ !strcmp((alg_name), "gzip") ? HZIP_ALG_TYPE_GZIP : 0) \ @@ -46,13 +67,13 @@ enum hisi_zip_alg_type { struct hisi_zip_req { struct acomp_req *req; - int sskip; - int dskip; + u32 sskip; + u32 dskip; struct hisi_acc_hw_sgl *hw_src; struct hisi_acc_hw_sgl *hw_dst; dma_addr_t dma_src; dma_addr_t dma_dst; - int req_id; + u16 req_id; }; struct hisi_zip_req_q { @@ -71,8 +92,6 @@ struct hisi_zip_qp_ctx { }; struct hisi_zip_ctx { -#define QPC_COMP 0 -#define QPC_DECOMP 1 struct hisi_zip_qp_ctx qp_ctx[HZIP_CTX_Q_NUM]; }; @@ -116,7 +135,7 @@ static void hisi_zip_config_tag(struct hisi_zip_sqe *sqe, u32 tag) static void hisi_zip_fill_sqe(struct hisi_zip_sqe *sqe, u8 req_type, dma_addr_t s_addr, dma_addr_t d_addr, u32 slen, - u32 dlen, int sskip, int dskip) + u32 dlen, u32 sskip, u32 dskip) { memset(sqe, 0, sizeof(struct hisi_zip_sqe)); @@ -143,7 +162,7 @@ static int hisi_zip_start_qp(struct hisi_qp *qp, struct hisi_zip_qp_ctx *ctx, ret = hisi_qm_start_qp(qp, 0); if (ret < 0) { - dev_err(dev, "start qp failed!\n"); + dev_err(dev, "failed to start qp (%d)!\n", ret); return ret; } @@ -166,7 +185,7 @@ static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type, int ret = zip_create_qps(qps, HZIP_CTX_Q_NUM, node); if (ret) { - pr_err("Can not create zip qps!\n"); + pr_err("failed to create zip qps (%d)!\n", ret); return -ENODEV; } @@ -264,11 +283,11 @@ static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx) return 0; err_free_loop1: - kfree(ctx->qp_ctx[QPC_DECOMP].req_q.req_bitmap); + kfree(ctx->qp_ctx[HZIP_QPC_DECOMP].req_q.req_bitmap); err_free_loop0: - kfree(ctx->qp_ctx[QPC_COMP].req_q.q); + kfree(ctx->qp_ctx[HZIP_QPC_COMP].req_q.q); err_free_bitmap: - kfree(ctx->qp_ctx[QPC_COMP].req_q.req_bitmap); + kfree(ctx->qp_ctx[HZIP_QPC_COMP].req_q.req_bitmap); return ret; } @@ -303,8 +322,8 @@ static int hisi_zip_create_sgl_pool(struct hisi_zip_ctx *ctx) return 0; err_free_sgl_pool0: - hisi_acc_free_sgl_pool(&ctx->qp_ctx[QPC_COMP].qp->qm->pdev->dev, - ctx->qp_ctx[QPC_COMP].sgl_pool); + hisi_acc_free_sgl_pool(&ctx->qp_ctx[HZIP_QPC_COMP].qp->qm->pdev->dev, + ctx->qp_ctx[HZIP_QPC_COMP].sgl_pool); return -ENOMEM; } @@ -342,7 +361,6 @@ static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data) atomic64_inc(&dfx->recv_cnt); status = sqe->dw3 & HZIP_BD_STATUS_M; - if (status != 0 && status != HZIP_NC_ERR) { dev_err(dev, "%scompress fail in qp%u: %u, output: %u\n", (qp->alg_type == 0) ? "" : "de", qp->qp_id, status, @@ -377,19 +395,28 @@ static int hisi_zip_acomp_init(struct crypto_acomp *tfm) { const char *alg_name = crypto_tfm_alg_name(&tfm->base); struct hisi_zip_ctx *ctx = crypto_tfm_ctx(&tfm->base); + struct device *dev; int ret; ret = hisi_zip_ctx_init(ctx, COMP_NAME_TO_TYPE(alg_name), tfm->base.node); - if (ret) + if (ret) { + pr_err("failed to init ctx (%d)!\n", ret); return ret; + } + + dev = &ctx->qp_ctx[0].qp->qm->pdev->dev; ret = hisi_zip_create_req_q(ctx); - if (ret) + if (ret) { + dev_err(dev, "failed to create request queue (%d)!\n", ret); goto err_ctx_exit; + } ret = hisi_zip_create_sgl_pool(ctx); - if (ret) + if (ret) { + dev_err(dev, "failed to create sgl pool (%d)!\n", ret); goto err_release_req_q; + } hisi_zip_set_acomp_cb(ctx, hisi_zip_acomp_cb); @@ -419,13 +446,15 @@ static int add_comp_head(struct scatterlist *dst, u8 req_type) int ret; ret = sg_copy_from_buffer(dst, sg_nents(dst), head, head_size); - if (ret != head_size) + if (ret != head_size) { + pr_err("the head size of buffer is wrong (%d)!\n", ret); return -ENOMEM; + } return head_size; } -static size_t get_gzip_head_size(struct scatterlist *sgl) +static size_t __maybe_unused get_gzip_head_size(struct scatterlist *sgl) { char buf[HZIP_GZIP_HEAD_BUF]; @@ -434,13 +463,20 @@ static size_t get_gzip_head_size(struct scatterlist *sgl) return __get_gzip_head_size(buf); } -static size_t get_comp_head_size(struct scatterlist *src, u8 req_type) +static int get_comp_head_size(struct acomp_req *acomp_req, u8 req_type) { + if (!acomp_req->src || !acomp_req->slen) + return -EINVAL; + + if ((req_type == HZIP_ALG_TYPE_GZIP) && + (acomp_req->slen < GZIP_HEAD_FEXTRA_SHIFT)) + return -EINVAL; + switch (req_type) { case HZIP_ALG_TYPE_ZLIB: return TO_HEAD_SIZE(HZIP_ALG_TYPE_ZLIB); case HZIP_ALG_TYPE_GZIP: - return get_gzip_head_size(src); + return TO_HEAD_SIZE(HZIP_ALG_TYPE_GZIP); default: pr_err("request type does not support!\n"); return -EINVAL; @@ -462,7 +498,7 @@ static struct hisi_zip_req *hisi_zip_create_req(struct acomp_req *req, if (req_id >= req_q->size) { write_unlock(&req_q->req_lock); dev_dbg(&qp_ctx->qp->qm->pdev->dev, "req cache is full!\n"); - return ERR_PTR(-EBUSY); + return ERR_PTR(-EAGAIN); } set_bit(req_id, req_q->req_bitmap); @@ -492,8 +528,7 @@ static int hisi_zip_do_work(struct hisi_zip_req *req, struct hisi_acc_sgl_pool *pool = qp_ctx->sgl_pool; struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx; struct hisi_zip_sqe zip_sqe; - dma_addr_t input; - dma_addr_t output; + dma_addr_t input, output; int ret; if (!a_req->src || !a_req->slen || !a_req->dst || !a_req->dlen) @@ -501,8 +536,11 @@ static int hisi_zip_do_work(struct hisi_zip_req *req, req->hw_src = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->src, pool, req->req_id << 1, &input); - if (IS_ERR(req->hw_src)) + if (IS_ERR(req->hw_src)) { + dev_err(dev, "failed to map the src buffer to hw sgl (%ld)!\n", + PTR_ERR(req->hw_src)); return PTR_ERR(req->hw_src); + } req->dma_src = input; req->hw_dst = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->dst, pool, @@ -510,6 +548,8 @@ static int hisi_zip_do_work(struct hisi_zip_req *req, &output); if (IS_ERR(req->hw_dst)) { ret = PTR_ERR(req->hw_dst); + dev_err(dev, "failed to map the dst buffer to hw slg (%d)!\n", + ret); goto err_unmap_input; } req->dma_dst = output; @@ -524,6 +564,8 @@ static int hisi_zip_do_work(struct hisi_zip_req *req, ret = hisi_qp_send(qp, &zip_sqe); if (ret < 0) { atomic64_inc(&dfx->send_busy_cnt); + ret = -EAGAIN; + dev_dbg_ratelimited(dev, "failed to send request!\n"); goto err_unmap_output; } @@ -539,23 +581,29 @@ err_unmap_input: static int hisi_zip_acompress(struct acomp_req *acomp_req) { struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm); - struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[QPC_COMP]; + struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_COMP]; + struct device *dev = &qp_ctx->qp->qm->pdev->dev; struct hisi_zip_req *req; int head_size; int ret; /* let's output compression head now */ head_size = add_comp_head(acomp_req->dst, qp_ctx->qp->req_type); - if (head_size < 0) - return -ENOMEM; + if (head_size < 0) { + dev_err_ratelimited(dev, "failed to add comp head (%d)!\n", + head_size); + return head_size; + } - req = hisi_zip_create_req(acomp_req, qp_ctx, (size_t)head_size, true); + req = hisi_zip_create_req(acomp_req, qp_ctx, head_size, true); if (IS_ERR(req)) return PTR_ERR(req); ret = hisi_zip_do_work(req, qp_ctx); - if (ret != -EINPROGRESS) + if (ret != -EINPROGRESS) { + dev_info_ratelimited(dev, "failed to do compress (%d)!\n", ret); hisi_zip_remove_req(qp_ctx, req); + } return ret; } @@ -563,20 +611,28 @@ static int hisi_zip_acompress(struct acomp_req *acomp_req) static int hisi_zip_adecompress(struct acomp_req *acomp_req) { struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm); - struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[QPC_DECOMP]; + struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_DECOMP]; + struct device *dev = &qp_ctx->qp->qm->pdev->dev; struct hisi_zip_req *req; - size_t head_size; - int ret; + int head_size, ret; - head_size = get_comp_head_size(acomp_req->src, qp_ctx->qp->req_type); + head_size = get_comp_head_size(acomp_req, qp_ctx->qp->req_type); + if (head_size < 0) { + dev_err_ratelimited(dev, "failed to get comp head size (%d)!\n", + head_size); + return head_size; + } req = hisi_zip_create_req(acomp_req, qp_ctx, head_size, false); if (IS_ERR(req)) return PTR_ERR(req); ret = hisi_zip_do_work(req, qp_ctx); - if (ret != -EINPROGRESS) + if (ret != -EINPROGRESS) { + dev_info_ratelimited(dev, "failed to do decompress (%d)!\n", + ret); hisi_zip_remove_req(qp_ctx, req); + } return ret; } @@ -611,17 +667,17 @@ static struct acomp_alg hisi_zip_acomp_gzip = { int hisi_zip_register_to_crypto(void) { - int ret = 0; + int ret; ret = crypto_register_acomp(&hisi_zip_acomp_zlib); if (ret) { - pr_err("Zlib acomp algorithm registration failed\n"); + pr_err("failed to register to zlib (%d)!\n", ret); return ret; } ret = crypto_register_acomp(&hisi_zip_acomp_gzip); if (ret) { - pr_err("Gzip acomp algorithm registration failed\n"); + pr_err("failed to register to gzip (%d)!\n", ret); crypto_unregister_acomp(&hisi_zip_acomp_zlib); } diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index e2845b2c963d..4bd2c811abba 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -17,7 +17,6 @@ #define PCI_DEVICE_ID_ZIP_PF 0xa250 #define PCI_DEVICE_ID_ZIP_VF 0xa251 -#define HZIP_VF_NUM 63 #define HZIP_QUEUE_NUM_V1 4096 #define HZIP_QUEUE_NUM_V2 1024 @@ -30,18 +29,18 @@ #define DECOMP3_ENABLE BIT(5) #define DECOMP4_ENABLE BIT(6) #define DECOMP5_ENABLE BIT(7) -#define ALL_COMP_DECOMP_EN (COMP0_ENABLE | COMP1_ENABLE | \ +#define HZIP_ALL_COMP_DECOMP_EN (COMP0_ENABLE | COMP1_ENABLE | \ DECOMP0_ENABLE | DECOMP1_ENABLE | \ DECOMP2_ENABLE | DECOMP3_ENABLE | \ DECOMP4_ENABLE | DECOMP5_ENABLE) -#define DECOMP_CHECK_ENABLE BIT(16) +#define HZIP_DECOMP_CHECK_ENABLE BIT(16) #define HZIP_FSM_MAX_CNT 0x301008 #define HZIP_PORT_ARCA_CHE_0 0x301040 #define HZIP_PORT_ARCA_CHE_1 0x301044 #define HZIP_PORT_AWCA_CHE_0 0x301060 #define HZIP_PORT_AWCA_CHE_1 0x301064 -#define CACHE_ALL_EN 0xffffffff +#define HZIP_CACHE_ALL_EN 0xffffffff #define HZIP_BD_RUSER_32_63 0x301110 #define HZIP_SGL_RUSER_32_63 0x30111c @@ -83,7 +82,7 @@ #define HZIP_PF_DEF_Q_BASE 0 #define HZIP_SOFT_CTRL_CNT_CLR_CE 0x301000 -#define SOFT_CTRL_CNT_CLR_CE_BIT BIT(0) +#define HZIP_SOFT_CTRL_CNT_CLR_CE_BIT BIT(0) #define HZIP_SOFT_CTRL_ZIP_CONTROL 0x30100C #define HZIP_AXI_SHUTDOWN_ENABLE BIT(14) #define HZIP_WR_PORT BIT(11) @@ -92,9 +91,13 @@ #define HZIP_SQE_MASK_OFFSET 64 #define HZIP_SQE_MASK_LEN 48 +#define HZIP_CNT_CLR_CE_EN BIT(0) +#define HZIP_RO_CNT_CLR_CE_EN BIT(2) +#define HZIP_RD_CNT_CLR_CE_EN (HZIP_CNT_CLR_CE_EN | \ + HZIP_RO_CNT_CLR_CE_EN) + static const char hisi_zip_name[] = "hisi_zip"; static struct dentry *hzip_debugfs_root; -static struct hisi_qm_list zip_devices; struct hisi_zip_hw_error { u32 int_msk; @@ -106,6 +109,11 @@ struct zip_dfx_item { u32 offset; }; +static struct hisi_qm_list zip_devices = { + .register_to_crypto = hisi_zip_register_to_crypto, + .unregister_from_crypto = hisi_zip_unregister_from_crypto, +}; + static struct zip_dfx_item zip_dfx_files[] = { {"send_cnt", offsetof(struct hisi_zip_dfx, send_cnt)}, {"recv_cnt", offsetof(struct hisi_zip_dfx, recv_cnt)}, @@ -153,7 +161,6 @@ struct ctrl_debug_file { */ struct hisi_zip_ctrl { struct hisi_zip *hisi_zip; - struct dentry *debug_root; struct ctrl_debug_file files[HZIP_DEBUG_FILE_NUM]; }; @@ -216,7 +223,7 @@ static const struct kernel_param_ops pf_q_num_ops = { static u32 pf_q_num = HZIP_PF_DEF_Q_NUM; module_param_cb(pf_q_num, &pf_q_num_ops, &pf_q_num, 0444); -MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 1-4096, v2 1-1024)"); +MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 2-4096, v2 2-1024)"); static const struct kernel_param_ops vfs_num_ops = { .set = vfs_num_set, @@ -256,15 +263,16 @@ static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm) /* qm cache */ writel(AXI_M_CFG, base + QM_AXI_M_CFG); writel(AXI_M_CFG_ENABLE, base + QM_AXI_M_CFG_ENABLE); + /* disable FLR triggered by BME(bus master enable) */ writel(PEH_AXUSER_CFG, base + QM_PEH_AXUSER_CFG); writel(PEH_AXUSER_CFG_ENABLE, base + QM_PEH_AXUSER_CFG_ENABLE); /* cache */ - writel(CACHE_ALL_EN, base + HZIP_PORT_ARCA_CHE_0); - writel(CACHE_ALL_EN, base + HZIP_PORT_ARCA_CHE_1); - writel(CACHE_ALL_EN, base + HZIP_PORT_AWCA_CHE_0); - writel(CACHE_ALL_EN, base + HZIP_PORT_AWCA_CHE_1); + writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_ARCA_CHE_0); + writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_ARCA_CHE_1); + writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_AWCA_CHE_0); + writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_AWCA_CHE_1); /* user domain configurations */ writel(AXUSER_BASE, base + HZIP_BD_RUSER_32_63); @@ -280,10 +288,10 @@ static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm) } /* let's open all compression/decompression cores */ - writel(DECOMP_CHECK_ENABLE | ALL_COMP_DECOMP_EN, + writel(HZIP_DECOMP_CHECK_ENABLE | HZIP_ALL_COMP_DECOMP_EN, base + HZIP_CLOCK_GATE_CTRL); - /* enable sqc writeback */ + /* enable sqc,cqc writeback */ writel(SQC_CACHE_ENABLE | CQC_CACHE_ENABLE | SQC_CACHE_WB_ENABLE | CQC_CACHE_WB_ENABLE | FIELD_PREP(SQC_CACHE_WB_THRD, 1) | FIELD_PREP(CQC_CACHE_WB_THRD, 1), base + QM_CACHE_CTL); @@ -309,7 +317,7 @@ static void hisi_zip_hw_error_enable(struct hisi_qm *qm) writel(0x1, qm->io_base + HZIP_CORE_INT_RAS_CE_ENB); writel(0x0, qm->io_base + HZIP_CORE_INT_RAS_FE_ENB); writel(HZIP_CORE_INT_RAS_NFE_ENABLE, - qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB); + qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB); /* enable ZIP hw error interrupts */ writel(0, qm->io_base + HZIP_CORE_INT_MASK_REG); @@ -356,7 +364,7 @@ static int current_qm_write(struct ctrl_debug_file *file, u32 val) if (val > qm->vfs_num) return -EINVAL; - /* Calculate curr_qm_qp_num and store */ + /* According PF or VF Dev ID to calculation curr_qm_qp_num and store */ if (val == 0) { qm->debug.curr_qm_qp_num = qm->qp_num; } else { @@ -387,7 +395,7 @@ static u32 clear_enable_read(struct ctrl_debug_file *file) struct hisi_qm *qm = file_to_qm(file); return readl(qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE) & - SOFT_CTRL_CNT_CLR_CE_BIT; + HZIP_SOFT_CTRL_CNT_CLR_CE_BIT; } static int clear_enable_write(struct ctrl_debug_file *file, u32 val) @@ -399,14 +407,14 @@ static int clear_enable_write(struct ctrl_debug_file *file, u32 val) return -EINVAL; tmp = (readl(qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE) & - ~SOFT_CTRL_CNT_CLR_CE_BIT) | val; + ~HZIP_SOFT_CTRL_CNT_CLR_CE_BIT) | val; writel(tmp, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE); return 0; } -static ssize_t ctrl_debug_read(struct file *filp, char __user *buf, - size_t count, loff_t *pos) +static ssize_t hisi_zip_ctrl_debug_read(struct file *filp, char __user *buf, + size_t count, loff_t *pos) { struct ctrl_debug_file *file = filp->private_data; char tbuf[HZIP_BUF_SIZE]; @@ -426,12 +434,13 @@ static ssize_t ctrl_debug_read(struct file *filp, char __user *buf, return -EINVAL; } spin_unlock_irq(&file->lock); - ret = sprintf(tbuf, "%u\n", val); + ret = scnprintf(tbuf, sizeof(tbuf), "%u\n", val); return simple_read_from_buffer(buf, count, pos, tbuf, ret); } -static ssize_t ctrl_debug_write(struct file *filp, const char __user *buf, - size_t count, loff_t *pos) +static ssize_t hisi_zip_ctrl_debug_write(struct file *filp, + const char __user *buf, + size_t count, loff_t *pos) { struct ctrl_debug_file *file = filp->private_data; char tbuf[HZIP_BUF_SIZE]; @@ -480,11 +489,10 @@ err_input: static const struct file_operations ctrl_debug_fops = { .owner = THIS_MODULE, .open = simple_open, - .read = ctrl_debug_read, - .write = ctrl_debug_write, + .read = hisi_zip_ctrl_debug_read, + .write = hisi_zip_ctrl_debug_write, }; - static int zip_debugfs_atomic64_set(void *data, u64 val) { if (val) @@ -505,10 +513,8 @@ static int zip_debugfs_atomic64_get(void *data, u64 *val) DEFINE_DEBUGFS_ATTRIBUTE(zip_atomic64_ops, zip_debugfs_atomic64_get, zip_debugfs_atomic64_set, "%llu\n"); -static int hisi_zip_core_debug_init(struct hisi_zip_ctrl *ctrl) +static int hisi_zip_core_debug_init(struct hisi_qm *qm) { - struct hisi_zip *hisi_zip = ctrl->hisi_zip; - struct hisi_qm *qm = &hisi_zip->qm; struct device *dev = &qm->pdev->dev; struct debugfs_regset32 *regset; struct dentry *tmp_d; @@ -517,9 +523,10 @@ static int hisi_zip_core_debug_init(struct hisi_zip_ctrl *ctrl) for (i = 0; i < HZIP_CORE_NUM; i++) { if (i < HZIP_COMP_CORE_NUM) - sprintf(buf, "comp_core%d", i); + scnprintf(buf, sizeof(buf), "comp_core%d", i); else - sprintf(buf, "decomp_core%d", i - HZIP_COMP_CORE_NUM); + scnprintf(buf, sizeof(buf), "decomp_core%d", + i - HZIP_COMP_CORE_NUM); regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL); if (!regset) @@ -529,7 +536,7 @@ static int hisi_zip_core_debug_init(struct hisi_zip_ctrl *ctrl) regset->nregs = ARRAY_SIZE(hzip_dfx_regs); regset->base = qm->io_base + core_offsets[i]; - tmp_d = debugfs_create_dir(buf, ctrl->debug_root); + tmp_d = debugfs_create_dir(buf, qm->debug.debug_root); debugfs_create_regset32("regs", 0444, tmp_d, regset); } @@ -548,33 +555,32 @@ static void hisi_zip_dfx_debug_init(struct hisi_qm *qm) for (i = 0; i < ARRAY_SIZE(zip_dfx_files); i++) { data = (atomic64_t *)((uintptr_t)dfx + zip_dfx_files[i].offset); debugfs_create_file(zip_dfx_files[i].name, - 0644, - tmp_dir, - data, - &zip_atomic64_ops); + 0644, tmp_dir, data, + &zip_atomic64_ops); } } -static int hisi_zip_ctrl_debug_init(struct hisi_zip_ctrl *ctrl) +static int hisi_zip_ctrl_debug_init(struct hisi_qm *qm) { + struct hisi_zip *zip = container_of(qm, struct hisi_zip, qm); int i; for (i = HZIP_CURRENT_QM; i < HZIP_DEBUG_FILE_NUM; i++) { - spin_lock_init(&ctrl->files[i].lock); - ctrl->files[i].ctrl = ctrl; - ctrl->files[i].index = i; + spin_lock_init(&zip->ctrl->files[i].lock); + zip->ctrl->files[i].ctrl = zip->ctrl; + zip->ctrl->files[i].index = i; debugfs_create_file(ctrl_debug_file_name[i], 0600, - ctrl->debug_root, ctrl->files + i, + qm->debug.debug_root, + zip->ctrl->files + i, &ctrl_debug_fops); } - return hisi_zip_core_debug_init(ctrl); + return hisi_zip_core_debug_init(qm); } -static int hisi_zip_debugfs_init(struct hisi_zip *hisi_zip) +static int hisi_zip_debugfs_init(struct hisi_qm *qm) { - struct hisi_qm *qm = &hisi_zip->qm; struct device *dev = &qm->pdev->dev; struct dentry *dev_d; int ret; @@ -589,8 +595,7 @@ static int hisi_zip_debugfs_init(struct hisi_zip *hisi_zip) goto failed_to_create; if (qm->fun_type == QM_HW_PF) { - hisi_zip->ctrl->debug_root = dev_d; - ret = hisi_zip_ctrl_debug_init(hisi_zip->ctrl); + ret = hisi_zip_ctrl_debug_init(qm); if (ret) goto failed_to_create; } @@ -604,25 +609,36 @@ failed_to_create: return ret; } -static void hisi_zip_debug_regs_clear(struct hisi_zip *hisi_zip) +/* hisi_zip_debug_regs_clear() - clear the zip debug regs */ +static void hisi_zip_debug_regs_clear(struct hisi_qm *qm) { - struct hisi_qm *qm = &hisi_zip->qm; + int i, j; + /* clear current_qm */ writel(0x0, qm->io_base + QM_DFX_MB_CNT_VF); writel(0x0, qm->io_base + QM_DFX_DB_CNT_VF); + + /* enable register read_clear bit */ + writel(HZIP_RD_CNT_CLR_CE_EN, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE); + for (i = 0; i < ARRAY_SIZE(core_offsets); i++) + for (j = 0; j < ARRAY_SIZE(hzip_dfx_regs); j++) + readl(qm->io_base + core_offsets[i] + + hzip_dfx_regs[j].offset); + + /* disable register read_clear bit */ writel(0x0, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE); hisi_qm_debug_regs_clear(qm); } -static void hisi_zip_debugfs_exit(struct hisi_zip *hisi_zip) +static void hisi_zip_debugfs_exit(struct hisi_qm *qm) { - struct hisi_qm *qm = &hisi_zip->qm; - debugfs_remove_recursive(qm->debug.debug_root); - if (qm->fun_type == QM_HW_PF) - hisi_zip_debug_regs_clear(hisi_zip); + if (qm->fun_type == QM_HW_PF) { + hisi_zip_debug_regs_clear(qm); + qm->debug.curr_qm_qp_num = 0; + } } static void hisi_zip_log_hw_error(struct hisi_qm *qm, u32 err_sts) @@ -634,7 +650,7 @@ static void hisi_zip_log_hw_error(struct hisi_qm *qm, u32 err_sts) while (err->msg) { if (err->int_msk & err_sts) { dev_err(dev, "%s [error status=0x%x] found\n", - err->msg, err->int_msk); + err->msg, err->int_msk); if (err->int_msk & HZIP_CORE_INT_STATUS_M_ECC) { err_val = readl(qm->io_base + @@ -642,9 +658,6 @@ static void hisi_zip_log_hw_error(struct hisi_qm *qm, u32 err_sts) dev_err(dev, "hisi-zip multi ecc sram num=0x%x\n", ((err_val >> HZIP_SRAM_ECC_ERR_NUM_SHIFT) & 0xFF)); - dev_err(dev, "hisi-zip multi ecc sram addr=0x%x\n", - (err_val >> - HZIP_SRAM_ECC_ERR_ADDR_SHIFT)); } } err++; @@ -729,7 +742,7 @@ static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) hisi_zip_set_user_domain_and_cache(qm); hisi_qm_dev_err_init(qm); - hisi_zip_debug_regs_clear(hisi_zip); + hisi_zip_debug_regs_clear(qm); return 0; } @@ -747,6 +760,7 @@ static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) if (qm->fun_type == QM_HW_PF) { qm->qp_base = HZIP_PF_DEF_Q_BASE; qm->qp_num = pf_q_num; + qm->debug.curr_qm_qp_num = pf_q_num; qm->qm_list = &zip_devices; } else if (qm->fun_type == QM_HW_VF && qm->ver == QM_HW_V1) { /* @@ -803,32 +817,44 @@ static int hisi_zip_probe(struct pci_dev *pdev, const struct pci_device_id *id) ret = hisi_qm_start(qm); if (ret) - goto err_qm_uninit; + goto err_dev_err_uninit; - ret = hisi_zip_debugfs_init(hisi_zip); + ret = hisi_zip_debugfs_init(qm); if (ret) - dev_err(&pdev->dev, "Failed to init debugfs (%d)!\n", ret); + pci_err(pdev, "failed to init debugfs (%d)!\n", ret); - hisi_qm_add_to_list(qm, &zip_devices); + ret = hisi_qm_alg_register(qm, &zip_devices); + if (ret < 0) { + pci_err(pdev, "failed to register driver to crypto!\n"); + goto err_qm_stop; + } if (qm->uacce) { ret = uacce_register(qm->uacce); - if (ret) - goto err_qm_uninit; + if (ret) { + pci_err(pdev, "failed to register uacce (%d)!\n", ret); + goto err_qm_alg_unregister; + } } if (qm->fun_type == QM_HW_PF && vfs_num > 0) { ret = hisi_qm_sriov_enable(pdev, vfs_num); if (ret < 0) - goto err_remove_from_list; + goto err_qm_alg_unregister; } return 0; -err_remove_from_list: - hisi_qm_del_from_list(qm, &zip_devices); - hisi_zip_debugfs_exit(hisi_zip); - hisi_qm_stop(qm); +err_qm_alg_unregister: + hisi_qm_alg_unregister(qm, &zip_devices); + +err_qm_stop: + hisi_zip_debugfs_exit(qm); + hisi_qm_stop(qm, QM_NORMAL); + +err_dev_err_uninit: + hisi_qm_dev_err_uninit(qm); + err_qm_uninit: hisi_qm_uninit(qm); @@ -837,18 +863,18 @@ err_qm_uninit: static void hisi_zip_remove(struct pci_dev *pdev) { - struct hisi_zip *hisi_zip = pci_get_drvdata(pdev); - struct hisi_qm *qm = &hisi_zip->qm; + struct hisi_qm *qm = pci_get_drvdata(pdev); - if (qm->fun_type == QM_HW_PF && qm->vfs_num) - hisi_qm_sriov_disable(pdev); + hisi_qm_wait_task_finish(qm, &zip_devices); + hisi_qm_alg_unregister(qm, &zip_devices); - hisi_zip_debugfs_exit(hisi_zip); - hisi_qm_stop(qm); + if (qm->fun_type == QM_HW_PF && qm->vfs_num) + hisi_qm_sriov_disable(pdev, qm->is_frozen); + hisi_zip_debugfs_exit(qm); + hisi_qm_stop(qm, QM_NORMAL); hisi_qm_dev_err_uninit(qm); hisi_qm_uninit(qm); - hisi_qm_del_from_list(qm, &zip_devices); } static const struct pci_error_handlers hisi_zip_err_handler = { @@ -866,6 +892,7 @@ static struct pci_driver hisi_zip_pci_driver = { .sriov_configure = IS_ENABLED(CONFIG_PCI_IOV) ? hisi_qm_sriov_configure : NULL, .err_handler = &hisi_zip_err_handler, + .shutdown = hisi_qm_dev_shutdown, }; static void hisi_zip_register_debugfs(void) @@ -890,29 +917,15 @@ static int __init hisi_zip_init(void) ret = pci_register_driver(&hisi_zip_pci_driver); if (ret < 0) { + hisi_zip_unregister_debugfs(); pr_err("Failed to register pci driver.\n"); - goto err_pci; } - ret = hisi_zip_register_to_crypto(); - if (ret < 0) { - pr_err("Failed to register driver to crypto.\n"); - goto err_crypto; - } - - return 0; - -err_crypto: - pci_unregister_driver(&hisi_zip_pci_driver); -err_pci: - hisi_zip_unregister_debugfs(); - return ret; } static void __exit hisi_zip_exit(void) { - hisi_zip_unregister_from_crypto(); pci_unregister_driver(&hisi_zip_pci_driver); hisi_zip_unregister_debugfs(); } diff --git a/drivers/crypto/img-hash.c b/drivers/crypto/img-hash.c index 87226b7c2795..91f555ccbb31 100644 --- a/drivers/crypto/img-hash.c +++ b/drivers/crypto/img-hash.c @@ -7,6 +7,7 @@ */ #include <linux/clk.h> +#include <linux/dma-mapping.h> #include <linux/dmaengine.h> #include <linux/interrupt.h> #include <linux/io.h> diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index fa7398e68858..eb2418450f12 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -304,6 +304,11 @@ static void eip197_init_firmware(struct safexcel_crypto_priv *priv) /* Enable access to all IFPP program memories */ writel(EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN, EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe)); + + /* bypass the OCE, if present */ + if (priv->flags & EIP197_OCE) + writel(EIP197_DEBUG_OCE_BYPASS, EIP197_PE(priv) + + EIP197_PE_DEBUG(pe)); } } @@ -1495,6 +1500,9 @@ static int safexcel_probe_generic(void *pdev, hwopt = readl(EIP197_GLOBAL(priv) + EIP197_OPTIONS); hiaopt = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_OPTIONS); + priv->hwconfig.icever = 0; + priv->hwconfig.ocever = 0; + priv->hwconfig.psever = 0; if (priv->flags & SAFEXCEL_HW_EIP197) { /* EIP197 */ peopt = readl(EIP197_PE(priv) + EIP197_PE_OPTIONS(0)); @@ -1513,8 +1521,37 @@ static int safexcel_probe_generic(void *pdev, EIP197_N_RINGS_MASK; if (hiaopt & EIP197_HIA_OPT_HAS_PE_ARB) priv->flags |= EIP197_PE_ARB; - if (EIP206_OPT_ICE_TYPE(peopt) == 1) + if (EIP206_OPT_ICE_TYPE(peopt) == 1) { priv->flags |= EIP197_ICE; + /* Detect ICE EIP207 class. engine and version */ + version = readl(EIP197_PE(priv) + + EIP197_PE_ICE_VERSION(0)); + if (EIP197_REG_LO16(version) != EIP207_VERSION_LE) { + dev_err(dev, "EIP%d: ICE EIP207 not detected.\n", + peid); + return -ENODEV; + } + priv->hwconfig.icever = EIP197_VERSION_MASK(version); + } + if (EIP206_OPT_OCE_TYPE(peopt) == 1) { + priv->flags |= EIP197_OCE; + /* Detect EIP96PP packet stream editor and version */ + version = readl(EIP197_PE(priv) + EIP197_PE_PSE_VERSION(0)); + if (EIP197_REG_LO16(version) != EIP96_VERSION_LE) { + dev_err(dev, "EIP%d: EIP96PP not detected.\n", peid); + return -ENODEV; + } + priv->hwconfig.psever = EIP197_VERSION_MASK(version); + /* Detect OCE EIP207 class. engine and version */ + version = readl(EIP197_PE(priv) + + EIP197_PE_ICE_VERSION(0)); + if (EIP197_REG_LO16(version) != EIP207_VERSION_LE) { + dev_err(dev, "EIP%d: OCE EIP207 not detected.\n", + peid); + return -ENODEV; + } + priv->hwconfig.ocever = EIP197_VERSION_MASK(version); + } /* If not a full TRC, then assume simple TRC */ if (!(hwopt & EIP197_OPT_HAS_TRC)) priv->flags |= EIP197_SIMPLE_TRC; @@ -1552,13 +1589,14 @@ static int safexcel_probe_generic(void *pdev, EIP197_PE_EIP96_OPTIONS(0)); /* Print single info line describing what we just detected */ - dev_info(priv->dev, "EIP%d:%x(%d,%d,%d,%d)-HIA:%x(%d,%d,%d),PE:%x/%x,alg:%08x\n", + dev_info(priv->dev, "EIP%d:%x(%d,%d,%d,%d)-HIA:%x(%d,%d,%d),PE:%x/%x(alg:%08x)/%x/%x/%x\n", peid, priv->hwconfig.hwver, hwctg, priv->hwconfig.hwnumpes, priv->hwconfig.hwnumrings, priv->hwconfig.hwnumraic, priv->hwconfig.hiaver, priv->hwconfig.hwdataw, priv->hwconfig.hwcfsize, priv->hwconfig.hwrfsize, priv->hwconfig.ppver, priv->hwconfig.pever, - priv->hwconfig.algo_flags); + priv->hwconfig.algo_flags, priv->hwconfig.icever, + priv->hwconfig.ocever, priv->hwconfig.psever); safexcel_configure(priv); diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 7c5fe382d272..9045f2d7f4c6 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -12,7 +12,9 @@ #include <crypto/algapi.h> #include <crypto/internal/hash.h> #include <crypto/sha.h> +#include <crypto/sha3.h> #include <crypto/skcipher.h> +#include <linux/types.h> #define EIP197_HIA_VERSION_BE 0xca35 #define EIP197_HIA_VERSION_LE 0x35ca @@ -22,6 +24,7 @@ #define EIP96_VERSION_LE 0x9f60 #define EIP201_VERSION_LE 0x36c9 #define EIP206_VERSION_LE 0x31ce +#define EIP207_VERSION_LE 0x30cf #define EIP197_REG_LO16(reg) (reg & 0xffff) #define EIP197_REG_HI16(reg) ((reg >> 16) & 0xffff) #define EIP197_VERSION_MASK(reg) ((reg >> 16) & 0xfff) @@ -34,6 +37,7 @@ /* EIP206 OPTIONS ENCODING */ #define EIP206_OPT_ICE_TYPE(n) ((n>>8)&3) +#define EIP206_OPT_OCE_TYPE(n) ((n>>10)&3) /* EIP197 OPTIONS ENCODING */ #define EIP197_OPT_HAS_TRC BIT(31) @@ -168,6 +172,7 @@ #define EIP197_PE_ICE_FPP_CTRL(n) (0x0d80 + (0x2000 * (n))) #define EIP197_PE_ICE_PPTF_CTRL(n) (0x0e00 + (0x2000 * (n))) #define EIP197_PE_ICE_RAM_CTRL(n) (0x0ff0 + (0x2000 * (n))) +#define EIP197_PE_ICE_VERSION(n) (0x0ffc + (0x2000 * (n))) #define EIP197_PE_EIP96_TOKEN_CTRL(n) (0x1000 + (0x2000 * (n))) #define EIP197_PE_EIP96_FUNCTION_EN(n) (0x1004 + (0x2000 * (n))) #define EIP197_PE_EIP96_CONTEXT_CTRL(n) (0x1008 + (0x2000 * (n))) @@ -176,8 +181,11 @@ #define EIP197_PE_EIP96_FUNCTION2_EN(n) (0x1030 + (0x2000 * (n))) #define EIP197_PE_EIP96_OPTIONS(n) (0x13f8 + (0x2000 * (n))) #define EIP197_PE_EIP96_VERSION(n) (0x13fc + (0x2000 * (n))) +#define EIP197_PE_OCE_VERSION(n) (0x1bfc + (0x2000 * (n))) #define EIP197_PE_OUT_DBUF_THRES(n) (0x1c00 + (0x2000 * (n))) #define EIP197_PE_OUT_TBUF_THRES(n) (0x1d00 + (0x2000 * (n))) +#define EIP197_PE_PSE_VERSION(n) (0x1efc + (0x2000 * (n))) +#define EIP197_PE_DEBUG(n) (0x1ff4 + (0x2000 * (n))) #define EIP197_PE_OPTIONS(n) (0x1ff8 + (0x2000 * (n))) #define EIP197_PE_VERSION(n) (0x1ffc + (0x2000 * (n))) #define EIP197_MST_CTRL 0xfff4 @@ -352,6 +360,9 @@ /* EIP197_PE_EIP96_TOKEN_CTRL2 */ #define EIP197_PE_EIP96_TOKEN_CTRL2_CTX_DONE BIT(3) +/* EIP197_PE_DEBUG */ +#define EIP197_DEBUG_OCE_BYPASS BIT(1) + /* EIP197_STRC_CONFIG */ #define EIP197_STRC_CONFIG_INIT BIT(31) #define EIP197_STRC_CONFIG_LARGE_REC(s) (s<<8) @@ -776,6 +787,7 @@ enum safexcel_flags { EIP197_PE_ARB = BIT(2), EIP197_ICE = BIT(3), EIP197_SIMPLE_TRC = BIT(4), + EIP197_OCE = BIT(5), }; struct safexcel_hwconfig { @@ -783,7 +795,10 @@ struct safexcel_hwconfig { int hwver; int hiaver; int ppver; + int icever; int pever; + int ocever; + int psever; int hwdataw; int hwcfsize; int hwrfsize; @@ -819,8 +834,16 @@ struct safexcel_context { struct crypto_async_request *req, bool *complete, int *ret); struct safexcel_context_record *ctxr; + struct safexcel_crypto_priv *priv; dma_addr_t ctxr_dma; + union { + __le32 le[SHA3_512_BLOCK_SIZE / 4]; + __be32 be[SHA3_512_BLOCK_SIZE / 4]; + u32 word[SHA3_512_BLOCK_SIZE / 4]; + u8 byte[SHA3_512_BLOCK_SIZE]; + } ipad, opad; + int ring; bool needs_inv; bool exit_inv; @@ -898,8 +921,9 @@ void safexcel_rdr_req_set(struct safexcel_crypto_priv *priv, inline struct crypto_async_request * safexcel_rdr_req_get(struct safexcel_crypto_priv *priv, int ring); void safexcel_inv_complete(struct crypto_async_request *req, int error); -int safexcel_hmac_setkey(const char *alg, const u8 *key, unsigned int keylen, - void *istate, void *ostate); +int safexcel_hmac_setkey(struct safexcel_context *base, const u8 *key, + unsigned int keylen, const char *alg, + unsigned int state_sz); /* available algorithms */ extern struct safexcel_alg_template safexcel_alg_ecb_des; diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index 1ac3253b7903..9bcfb79a030f 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -61,8 +61,6 @@ struct safexcel_cipher_ctx { /* All the below is AEAD specific */ u32 hash_alg; u32 state_sz; - __be32 ipad[SHA512_DIGEST_SIZE / sizeof(u32)]; - __be32 opad[SHA512_DIGEST_SIZE / sizeof(u32)]; struct crypto_cipher *hkaes; struct crypto_aead *fback; @@ -375,7 +373,7 @@ static int safexcel_skcipher_aes_setkey(struct crypto_skcipher *ctfm, { struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; struct crypto_aes_ctx aes; int ret, i; @@ -406,11 +404,11 @@ static int safexcel_aead_setkey(struct crypto_aead *ctfm, const u8 *key, { struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); - struct safexcel_ahash_export_state istate, ostate; - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; struct crypto_authenc_keys keys; struct crypto_aes_ctx aes; int err = -EINVAL, i; + const char *alg; if (unlikely(crypto_authenc_extractkeys(&keys, key, len))) goto badkey; @@ -465,53 +463,37 @@ static int safexcel_aead_setkey(struct crypto_aead *ctfm, const u8 *key, /* Auth key */ switch (ctx->hash_alg) { case CONTEXT_CONTROL_CRYPTO_ALG_SHA1: - if (safexcel_hmac_setkey("safexcel-sha1", keys.authkey, - keys.authkeylen, &istate, &ostate)) - goto badkey; + alg = "safexcel-sha1"; break; case CONTEXT_CONTROL_CRYPTO_ALG_SHA224: - if (safexcel_hmac_setkey("safexcel-sha224", keys.authkey, - keys.authkeylen, &istate, &ostate)) - goto badkey; + alg = "safexcel-sha224"; break; case CONTEXT_CONTROL_CRYPTO_ALG_SHA256: - if (safexcel_hmac_setkey("safexcel-sha256", keys.authkey, - keys.authkeylen, &istate, &ostate)) - goto badkey; + alg = "safexcel-sha256"; break; case CONTEXT_CONTROL_CRYPTO_ALG_SHA384: - if (safexcel_hmac_setkey("safexcel-sha384", keys.authkey, - keys.authkeylen, &istate, &ostate)) - goto badkey; + alg = "safexcel-sha384"; break; case CONTEXT_CONTROL_CRYPTO_ALG_SHA512: - if (safexcel_hmac_setkey("safexcel-sha512", keys.authkey, - keys.authkeylen, &istate, &ostate)) - goto badkey; + alg = "safexcel-sha512"; break; case CONTEXT_CONTROL_CRYPTO_ALG_SM3: - if (safexcel_hmac_setkey("safexcel-sm3", keys.authkey, - keys.authkeylen, &istate, &ostate)) - goto badkey; + alg = "safexcel-sm3"; break; default: dev_err(priv->dev, "aead: unsupported hash algorithm\n"); goto badkey; } - if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma && - (memcmp(ctx->ipad, istate.state, ctx->state_sz) || - memcmp(ctx->opad, ostate.state, ctx->state_sz))) - ctx->base.needs_inv = true; + if (safexcel_hmac_setkey(&ctx->base, keys.authkey, keys.authkeylen, + alg, ctx->state_sz)) + goto badkey; /* Now copy the keys into the context */ for (i = 0; i < keys.enckeylen / sizeof(u32); i++) ctx->key[i] = cpu_to_le32(((u32 *)keys.enckey)[i]); ctx->key_len = keys.enckeylen; - memcpy(ctx->ipad, &istate.state, ctx->state_sz); - memcpy(ctx->opad, &ostate.state, ctx->state_sz); - memzero_explicit(&keys, sizeof(keys)); return 0; @@ -525,7 +507,7 @@ static int safexcel_context_control(struct safexcel_cipher_ctx *ctx, struct safexcel_cipher_req *sreq, struct safexcel_command_desc *cdesc) { - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; int ctrl_size = ctx->key_len / sizeof(u32); cdesc->control_data.control1 = ctx->mode; @@ -692,7 +674,7 @@ static int safexcel_send_req(struct crypto_async_request *base, int ring, struct skcipher_request *areq = skcipher_request_cast(base); struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(areq); struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; struct safexcel_command_desc *cdesc; struct safexcel_command_desc *first_cdesc = NULL; struct safexcel_result_desc *rdesc, *first_rdesc = NULL; @@ -718,10 +700,10 @@ static int safexcel_send_req(struct crypto_async_request *base, int ring, totlen_dst += digestsize; memcpy(ctx->base.ctxr->data + ctx->key_len / sizeof(u32), - ctx->ipad, ctx->state_sz); + &ctx->base.ipad, ctx->state_sz); if (!ctx->xcm) memcpy(ctx->base.ctxr->data + (ctx->key_len + - ctx->state_sz) / sizeof(u32), ctx->opad, + ctx->state_sz) / sizeof(u32), &ctx->base.opad, ctx->state_sz); } else if ((ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) && (sreq->direction == SAFEXCEL_DECRYPT)) { @@ -1020,7 +1002,7 @@ static int safexcel_cipher_send_inv(struct crypto_async_request *base, int ring, int *commands, int *results) { struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; int ret; ret = safexcel_invalidate_cache(base, priv, ctx->base.ctxr_dma, ring); @@ -1039,7 +1021,7 @@ static int safexcel_skcipher_send(struct crypto_async_request *async, int ring, struct skcipher_request *req = skcipher_request_cast(async); struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); struct safexcel_cipher_req *sreq = skcipher_request_ctx(req); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; int ret; BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv); @@ -1072,7 +1054,7 @@ static int safexcel_aead_send(struct crypto_async_request *async, int ring, struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); struct safexcel_cipher_req *sreq = aead_request_ctx(req); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; int ret; BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv); @@ -1094,7 +1076,7 @@ static int safexcel_cipher_exit_inv(struct crypto_tfm *tfm, struct safexcel_inv_result *result) { struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; int ring = ctx->base.ring; init_completion(&result->completion); @@ -1157,7 +1139,7 @@ static int safexcel_queue_req(struct crypto_async_request *base, enum safexcel_cipher_direction dir) { struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; int ret, ring; sreq->needs_inv = false; @@ -1211,7 +1193,7 @@ static int safexcel_skcipher_cra_init(struct crypto_tfm *tfm) crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm), sizeof(struct safexcel_cipher_req)); - ctx->priv = tmpl->priv; + ctx->base.priv = tmpl->priv; ctx->base.send = safexcel_skcipher_send; ctx->base.handle_result = safexcel_skcipher_handle_result; @@ -1237,7 +1219,7 @@ static int safexcel_cipher_cra_exit(struct crypto_tfm *tfm) static void safexcel_skcipher_cra_exit(struct crypto_tfm *tfm) { struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; int ret; if (safexcel_cipher_cra_exit(tfm)) @@ -1257,7 +1239,7 @@ static void safexcel_skcipher_cra_exit(struct crypto_tfm *tfm) static void safexcel_aead_cra_exit(struct crypto_tfm *tfm) { struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; int ret; if (safexcel_cipher_cra_exit(tfm)) @@ -1431,7 +1413,7 @@ static int safexcel_skcipher_aesctr_setkey(struct crypto_skcipher *ctfm, { struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; struct crypto_aes_ctx aes; int ret, i; unsigned int keylen; @@ -1505,7 +1487,7 @@ static int safexcel_des_setkey(struct crypto_skcipher *ctfm, const u8 *key, unsigned int len) { struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; int ret; ret = verify_skcipher_des_key(ctfm, key); @@ -1604,7 +1586,7 @@ static int safexcel_des3_ede_setkey(struct crypto_skcipher *ctfm, const u8 *key, unsigned int len) { struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; int err; err = verify_skcipher_des3_key(ctfm, key); @@ -1723,7 +1705,7 @@ static int safexcel_aead_cra_init(struct crypto_tfm *tfm) crypto_aead_set_reqsize(__crypto_aead_cast(tfm), sizeof(struct safexcel_cipher_req)); - ctx->priv = tmpl->priv; + ctx->base.priv = tmpl->priv; ctx->alg = SAFEXCEL_AES; /* default */ ctx->blocksz = AES_BLOCK_SIZE; @@ -2466,7 +2448,7 @@ static int safexcel_skcipher_aesxts_setkey(struct crypto_skcipher *ctfm, { struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; struct crypto_aes_ctx aes; int ret, i; unsigned int keylen; @@ -2580,7 +2562,7 @@ static int safexcel_aead_gcm_setkey(struct crypto_aead *ctfm, const u8 *key, { struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; struct crypto_aes_ctx aes; u32 hashkey[AES_BLOCK_SIZE >> 2]; int ret, i; @@ -2618,7 +2600,7 @@ static int safexcel_aead_gcm_setkey(struct crypto_aead *ctfm, const u8 *key, if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) { - if (be32_to_cpu(ctx->ipad[i]) != hashkey[i]) { + if (be32_to_cpu(ctx->base.ipad.be[i]) != hashkey[i]) { ctx->base.needs_inv = true; break; } @@ -2626,7 +2608,7 @@ static int safexcel_aead_gcm_setkey(struct crypto_aead *ctfm, const u8 *key, } for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) - ctx->ipad[i] = cpu_to_be32(hashkey[i]); + ctx->base.ipad.be[i] = cpu_to_be32(hashkey[i]); memzero_explicit(hashkey, AES_BLOCK_SIZE); memzero_explicit(&aes, sizeof(aes)); @@ -2693,7 +2675,7 @@ static int safexcel_aead_ccm_setkey(struct crypto_aead *ctfm, const u8 *key, { struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; struct crypto_aes_ctx aes; int ret, i; @@ -2714,7 +2696,7 @@ static int safexcel_aead_ccm_setkey(struct crypto_aead *ctfm, const u8 *key, for (i = 0; i < len / sizeof(u32); i++) { ctx->key[i] = cpu_to_le32(aes.key_enc[i]); - ctx->ipad[i + 2 * AES_BLOCK_SIZE / sizeof(u32)] = + ctx->base.ipad.be[i + 2 * AES_BLOCK_SIZE / sizeof(u32)] = cpu_to_be32(aes.key_enc[i]); } @@ -2815,7 +2797,7 @@ struct safexcel_alg_template safexcel_alg_ccm = { static void safexcel_chacha20_setkey(struct safexcel_cipher_ctx *ctx, const u8 *key) { - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) if (memcmp(ctx->key, key, CHACHA_KEY_SIZE)) @@ -3084,7 +3066,7 @@ static int safexcel_skcipher_sm4_setkey(struct crypto_skcipher *ctfm, { struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; if (len != SM4_KEY_SIZE) return -EINVAL; diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index 16a467969d8e..56d5ccb5cc00 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -20,7 +20,6 @@ struct safexcel_ahash_ctx { struct safexcel_context base; - struct safexcel_crypto_priv *priv; u32 alg; u8 key_sz; @@ -29,9 +28,6 @@ struct safexcel_ahash_ctx { bool fb_init_done; bool fb_do_setkey; - __le32 ipad[SHA3_512_BLOCK_SIZE / sizeof(__le32)]; - __le32 opad[SHA3_512_BLOCK_SIZE / sizeof(__le32)]; - struct crypto_cipher *kaes; struct crypto_ahash *fback; struct crypto_shash *shpre; @@ -111,7 +107,7 @@ static void safexcel_context_control(struct safexcel_ahash_ctx *ctx, struct safexcel_ahash_req *req, struct safexcel_command_desc *cdesc) { - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; u64 count = 0; cdesc->control_data.control0 = ctx->alg; @@ -124,7 +120,7 @@ static void safexcel_context_control(struct safexcel_ahash_ctx *ctx, */ if (unlikely(req->digest == CONTEXT_CONTROL_DIGEST_XCM)) { if (req->xcbcmac) - memcpy(ctx->base.ctxr->data, ctx->ipad, ctx->key_sz); + memcpy(ctx->base.ctxr->data, &ctx->base.ipad, ctx->key_sz); else memcpy(ctx->base.ctxr->data, req->state, req->state_sz); @@ -206,7 +202,7 @@ static void safexcel_context_control(struct safexcel_ahash_ctx *ctx, } else { /* HMAC */ /* Need outer digest for HMAC finalization */ memcpy(ctx->base.ctxr->data + (req->state_sz >> 2), - ctx->opad, req->state_sz); + &ctx->base.opad, req->state_sz); /* Single pass HMAC - no digest count */ cdesc->control_data.control0 |= @@ -275,7 +271,7 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, memcpy(sreq->cache, sreq->state, crypto_ahash_digestsize(ahash)); - memcpy(sreq->state, ctx->opad, sreq->digest_sz); + memcpy(sreq->state, &ctx->base.opad, sreq->digest_sz); sreq->len = sreq->block_sz + crypto_ahash_digestsize(ahash); @@ -316,7 +312,7 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring, struct ahash_request *areq = ahash_request_cast(async); struct safexcel_ahash_req *req = ahash_request_ctx(areq); struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; struct safexcel_command_desc *cdesc, *first_cdesc = NULL; struct safexcel_result_desc *rdesc; struct scatterlist *sg; @@ -379,10 +375,14 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring, // 10- padding for XCBCMAC & CMAC req->cache[cache_len + skip] = 0x80; // HW will use K2 iso K3 - compensate! - for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) - ((__be32 *)req->cache)[i] ^= - cpu_to_be32(le32_to_cpu( - ctx->ipad[i] ^ ctx->ipad[i + 4])); + for (i = 0; i < AES_BLOCK_SIZE / 4; i++) { + u32 *cache = (void *)req->cache; + u32 *ipad = ctx->base.ipad.word; + u32 x; + + x = ipad[i] ^ ipad[i + 4]; + cache[i] ^= swab(x); + } } cache_len = AES_BLOCK_SIZE; queued = queued + extra; @@ -591,7 +591,7 @@ static int safexcel_ahash_send_inv(struct crypto_async_request *async, struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); int ret; - ret = safexcel_invalidate_cache(async, ctx->priv, + ret = safexcel_invalidate_cache(async, ctx->base.priv, ctx->base.ctxr_dma, ring); if (unlikely(ret)) return ret; @@ -620,7 +620,7 @@ static int safexcel_ahash_send(struct crypto_async_request *async, static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm) { struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; EIP197_REQUEST_ON_STACK(req, ahash, EIP197_AHASH_REQ_SIZE); struct safexcel_ahash_req *rctx = ahash_request_ctx(req); struct safexcel_inv_result result = {}; @@ -688,7 +688,7 @@ static int safexcel_ahash_enqueue(struct ahash_request *areq) { struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); struct safexcel_ahash_req *req = ahash_request_ctx(areq); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; int ret, ring; req->needs_inv = false; @@ -702,7 +702,7 @@ static int safexcel_ahash_enqueue(struct ahash_request *areq) /* invalidate for HMAC finish with odigest changed */ (req->finish && req->hmac && memcmp(ctx->base.ctxr->data + (req->state_sz>>2), - ctx->opad, req->state_sz)))) + &ctx->base.opad, req->state_sz)))) /* * We're still setting needs_inv here, even though it is * cleared right away, because the needs_inv flag can be @@ -803,7 +803,7 @@ static int safexcel_ahash_final(struct ahash_request *areq) ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5 && req->len == sizeof(u32) && !areq->nbytes)) { /* Zero length CRC32 */ - memcpy(areq->result, ctx->ipad, sizeof(u32)); + memcpy(areq->result, &ctx->base.ipad, sizeof(u32)); return 0; } else if (unlikely(ctx->cbcmac && req->len == AES_BLOCK_SIZE && !areq->nbytes)) { @@ -815,9 +815,12 @@ static int safexcel_ahash_final(struct ahash_request *areq) /* Zero length (X)CBC/CMAC */ int i; - for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) - ((__be32 *)areq->result)[i] = - cpu_to_be32(le32_to_cpu(ctx->ipad[i + 4]));//K3 + for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) { + u32 *result = (void *)areq->result; + + /* K3 */ + result[i] = swab(ctx->base.ipad.word[i + 4]); + } areq->result[0] ^= 0x80; // 10- padding crypto_cipher_encrypt_one(ctx->kaes, areq->result, areq->result); return 0; @@ -917,7 +920,7 @@ static int safexcel_ahash_cra_init(struct crypto_tfm *tfm) container_of(__crypto_ahash_alg(tfm->__crt_alg), struct safexcel_alg_template, alg.ahash); - ctx->priv = tmpl->priv; + ctx->base.priv = tmpl->priv; ctx->base.send = safexcel_ahash_send; ctx->base.handle_result = safexcel_handle_result; ctx->fb_do_setkey = false; @@ -956,7 +959,7 @@ static int safexcel_sha1_digest(struct ahash_request *areq) static void safexcel_ahash_cra_exit(struct crypto_tfm *tfm) { struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; int ret; /* context not allocated, skip invalidation */ @@ -1012,7 +1015,7 @@ static int safexcel_hmac_sha1_init(struct ahash_request *areq) memset(req, 0, sizeof(*req)); /* Start from ipad precompute */ - memcpy(req->state, ctx->ipad, SHA1_DIGEST_SIZE); + memcpy(req->state, &ctx->base.ipad, SHA1_DIGEST_SIZE); /* Already processed the key^ipad part now! */ req->len = SHA1_BLOCK_SIZE; req->processed = SHA1_BLOCK_SIZE; @@ -1082,8 +1085,7 @@ static int safexcel_hmac_init_pad(struct ahash_request *areq, } /* Avoid leaking */ - memzero_explicit(keydup, keylen); - kfree(keydup); + kfree_sensitive(keydup); if (ret) return ret; @@ -1135,8 +1137,9 @@ static int safexcel_hmac_init_iv(struct ahash_request *areq, return crypto_ahash_export(areq, state); } -int safexcel_hmac_setkey(const char *alg, const u8 *key, unsigned int keylen, - void *istate, void *ostate) +static int __safexcel_hmac_setkey(const char *alg, const u8 *key, + unsigned int keylen, + void *istate, void *ostate) { struct ahash_request *areq; struct crypto_ahash *tfm; @@ -1185,30 +1188,38 @@ free_ahash: return ret; } -static int safexcel_hmac_alg_setkey(struct crypto_ahash *tfm, const u8 *key, - unsigned int keylen, const char *alg, - unsigned int state_sz) +int safexcel_hmac_setkey(struct safexcel_context *base, const u8 *key, + unsigned int keylen, const char *alg, + unsigned int state_sz) { - struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = base->priv; struct safexcel_ahash_export_state istate, ostate; int ret; - ret = safexcel_hmac_setkey(alg, key, keylen, &istate, &ostate); + ret = __safexcel_hmac_setkey(alg, key, keylen, &istate, &ostate); if (ret) return ret; - if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr && - (memcmp(ctx->ipad, istate.state, state_sz) || - memcmp(ctx->opad, ostate.state, state_sz))) - ctx->base.needs_inv = true; + if (priv->flags & EIP197_TRC_CACHE && base->ctxr && + (memcmp(&base->ipad, istate.state, state_sz) || + memcmp(&base->opad, ostate.state, state_sz))) + base->needs_inv = true; - memcpy(ctx->ipad, &istate.state, state_sz); - memcpy(ctx->opad, &ostate.state, state_sz); + memcpy(&base->ipad, &istate.state, state_sz); + memcpy(&base->opad, &ostate.state, state_sz); return 0; } +static int safexcel_hmac_alg_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen, const char *alg, + unsigned int state_sz) +{ + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + + return safexcel_hmac_setkey(&ctx->base, key, keylen, alg, state_sz); +} + static int safexcel_hmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen) { @@ -1377,7 +1388,7 @@ static int safexcel_hmac_sha224_init(struct ahash_request *areq) memset(req, 0, sizeof(*req)); /* Start from ipad precompute */ - memcpy(req->state, ctx->ipad, SHA256_DIGEST_SIZE); + memcpy(req->state, &ctx->base.ipad, SHA256_DIGEST_SIZE); /* Already processed the key^ipad part now! */ req->len = SHA256_BLOCK_SIZE; req->processed = SHA256_BLOCK_SIZE; @@ -1449,7 +1460,7 @@ static int safexcel_hmac_sha256_init(struct ahash_request *areq) memset(req, 0, sizeof(*req)); /* Start from ipad precompute */ - memcpy(req->state, ctx->ipad, SHA256_DIGEST_SIZE); + memcpy(req->state, &ctx->base.ipad, SHA256_DIGEST_SIZE); /* Already processed the key^ipad part now! */ req->len = SHA256_BLOCK_SIZE; req->processed = SHA256_BLOCK_SIZE; @@ -1635,7 +1646,7 @@ static int safexcel_hmac_sha512_init(struct ahash_request *areq) memset(req, 0, sizeof(*req)); /* Start from ipad precompute */ - memcpy(req->state, ctx->ipad, SHA512_DIGEST_SIZE); + memcpy(req->state, &ctx->base.ipad, SHA512_DIGEST_SIZE); /* Already processed the key^ipad part now! */ req->len = SHA512_BLOCK_SIZE; req->processed = SHA512_BLOCK_SIZE; @@ -1707,7 +1718,7 @@ static int safexcel_hmac_sha384_init(struct ahash_request *areq) memset(req, 0, sizeof(*req)); /* Start from ipad precompute */ - memcpy(req->state, ctx->ipad, SHA512_DIGEST_SIZE); + memcpy(req->state, &ctx->base.ipad, SHA512_DIGEST_SIZE); /* Already processed the key^ipad part now! */ req->len = SHA512_BLOCK_SIZE; req->processed = SHA512_BLOCK_SIZE; @@ -1829,7 +1840,7 @@ static int safexcel_hmac_md5_init(struct ahash_request *areq) memset(req, 0, sizeof(*req)); /* Start from ipad precompute */ - memcpy(req->state, ctx->ipad, MD5_DIGEST_SIZE); + memcpy(req->state, &ctx->base.ipad, MD5_DIGEST_SIZE); /* Already processed the key^ipad part now! */ req->len = MD5_HMAC_BLOCK_SIZE; req->processed = MD5_HMAC_BLOCK_SIZE; @@ -1900,7 +1911,7 @@ static int safexcel_crc32_cra_init(struct crypto_tfm *tfm) int ret = safexcel_ahash_cra_init(tfm); /* Default 'key' is all zeroes */ - memset(ctx->ipad, 0, sizeof(u32)); + memset(&ctx->base.ipad, 0, sizeof(u32)); return ret; } @@ -1912,7 +1923,7 @@ static int safexcel_crc32_init(struct ahash_request *areq) memset(req, 0, sizeof(*req)); /* Start from loaded key */ - req->state[0] = (__force __le32)le32_to_cpu(~ctx->ipad[0]); + req->state[0] = cpu_to_le32(~ctx->base.ipad.word[0]); /* Set processed to non-zero to enable invalidation detection */ req->len = sizeof(u32); req->processed = sizeof(u32); @@ -1934,7 +1945,7 @@ static int safexcel_crc32_setkey(struct crypto_ahash *tfm, const u8 *key, if (keylen != sizeof(u32)) return -EINVAL; - memcpy(ctx->ipad, key, sizeof(u32)); + memcpy(&ctx->base.ipad, key, sizeof(u32)); return 0; } @@ -1984,7 +1995,7 @@ static int safexcel_cbcmac_init(struct ahash_request *areq) memset(req, 0, sizeof(*req)); /* Start from loaded keys */ - memcpy(req->state, ctx->ipad, ctx->key_sz); + memcpy(req->state, &ctx->base.ipad, ctx->key_sz); /* Set processed to non-zero to enable invalidation detection */ req->len = AES_BLOCK_SIZE; req->processed = AES_BLOCK_SIZE; @@ -2009,9 +2020,9 @@ static int safexcel_cbcmac_setkey(struct crypto_ahash *tfm, const u8 *key, if (ret) return ret; - memset(ctx->ipad, 0, 2 * AES_BLOCK_SIZE); + memset(&ctx->base.ipad, 0, 2 * AES_BLOCK_SIZE); for (i = 0; i < len / sizeof(u32); i++) - ctx->ipad[i + 8] = (__force __le32)cpu_to_be32(aes.key_enc[i]); + ctx->base.ipad.be[i + 8] = cpu_to_be32(aes.key_enc[i]); if (len == AES_KEYSIZE_192) { ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192; @@ -2093,8 +2104,7 @@ static int safexcel_xcbcmac_setkey(struct crypto_ahash *tfm, const u8 *key, crypto_cipher_encrypt_one(ctx->kaes, (u8 *)key_tmp + AES_BLOCK_SIZE, "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3"); for (i = 0; i < 3 * AES_BLOCK_SIZE / sizeof(u32); i++) - ctx->ipad[i] = - cpu_to_le32((__force u32)cpu_to_be32(key_tmp[i])); + ctx->base.ipad.word[i] = swab(key_tmp[i]); crypto_cipher_clear_flags(ctx->kaes, CRYPTO_TFM_REQ_MASK); crypto_cipher_set_flags(ctx->kaes, crypto_ahash_get_flags(tfm) & @@ -2177,8 +2187,7 @@ static int safexcel_cmac_setkey(struct crypto_ahash *tfm, const u8 *key, return ret; for (i = 0; i < len / sizeof(u32); i++) - ctx->ipad[i + 8] = - cpu_to_le32((__force u32)cpu_to_be32(aes.key_enc[i])); + ctx->base.ipad.word[i + 8] = swab(aes.key_enc[i]); /* precompute the CMAC key material */ crypto_cipher_clear_flags(ctx->kaes, CRYPTO_TFM_REQ_MASK); @@ -2209,7 +2218,7 @@ static int safexcel_cmac_setkey(struct crypto_ahash *tfm, const u8 *key, /* end of code borrowed from crypto/cmac.c */ for (i = 0; i < 2 * AES_BLOCK_SIZE / sizeof(u32); i++) - ctx->ipad[i] = (__force __le32)cpu_to_be32(((u32 *)consts)[i]); + ctx->base.ipad.be[i] = cpu_to_be32(((u32 *)consts)[i]); if (len == AES_KEYSIZE_192) { ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192; @@ -2331,7 +2340,7 @@ static int safexcel_hmac_sm3_init(struct ahash_request *areq) memset(req, 0, sizeof(*req)); /* Start from ipad precompute */ - memcpy(req->state, ctx->ipad, SM3_DIGEST_SIZE); + memcpy(req->state, &ctx->base.ipad, SM3_DIGEST_SIZE); /* Already processed the key^ipad part now! */ req->len = SM3_BLOCK_SIZE; req->processed = SM3_BLOCK_SIZE; @@ -2424,11 +2433,11 @@ static int safexcel_sha3_fbcheck(struct ahash_request *req) /* Set fallback cipher HMAC key */ u8 key[SHA3_224_BLOCK_SIZE]; - memcpy(key, ctx->ipad, + memcpy(key, &ctx->base.ipad, crypto_ahash_blocksize(ctx->fback) / 2); memcpy(key + crypto_ahash_blocksize(ctx->fback) / 2, - ctx->opad, + &ctx->base.opad, crypto_ahash_blocksize(ctx->fback) / 2); ret = crypto_ahash_setkey(ctx->fback, key, crypto_ahash_blocksize(ctx->fback)); @@ -2801,7 +2810,7 @@ static int safexcel_hmac_sha3_setkey(struct crypto_ahash *tfm, const u8 *key, * first using our fallback cipher */ ret = crypto_shash_digest(ctx->shdesc, key, keylen, - (u8 *)ctx->ipad); + ctx->base.ipad.byte); keylen = crypto_shash_digestsize(ctx->shpre); /* @@ -2810,8 +2819,8 @@ static int safexcel_hmac_sha3_setkey(struct crypto_ahash *tfm, const u8 *key, */ if (keylen > crypto_ahash_blocksize(tfm) / 2) /* Buffers overlap, need to use memmove iso memcpy! */ - memmove(ctx->opad, - (u8 *)ctx->ipad + + memmove(&ctx->base.opad, + ctx->base.ipad.byte + crypto_ahash_blocksize(tfm) / 2, keylen - crypto_ahash_blocksize(tfm) / 2); } else { @@ -2821,11 +2830,11 @@ static int safexcel_hmac_sha3_setkey(struct crypto_ahash *tfm, const u8 *key, * to match the existing HMAC driver infrastructure. */ if (keylen <= crypto_ahash_blocksize(tfm) / 2) { - memcpy(ctx->ipad, key, keylen); + memcpy(&ctx->base.ipad, key, keylen); } else { - memcpy(ctx->ipad, key, + memcpy(&ctx->base.ipad, key, crypto_ahash_blocksize(tfm) / 2); - memcpy(ctx->opad, + memcpy(&ctx->base.opad, key + crypto_ahash_blocksize(tfm) / 2, keylen - crypto_ahash_blocksize(tfm) / 2); } @@ -2833,11 +2842,11 @@ static int safexcel_hmac_sha3_setkey(struct crypto_ahash *tfm, const u8 *key, /* Pad key with zeroes */ if (keylen <= crypto_ahash_blocksize(tfm) / 2) { - memset((u8 *)ctx->ipad + keylen, 0, + memset(ctx->base.ipad.byte + keylen, 0, crypto_ahash_blocksize(tfm) / 2 - keylen); - memset(ctx->opad, 0, crypto_ahash_blocksize(tfm) / 2); + memset(&ctx->base.opad, 0, crypto_ahash_blocksize(tfm) / 2); } else { - memset((u8 *)ctx->opad + keylen - + memset(ctx->base.opad.byte + keylen - crypto_ahash_blocksize(tfm) / 2, 0, crypto_ahash_blocksize(tfm) - keylen); } @@ -2856,7 +2865,7 @@ static int safexcel_hmac_sha3_224_init(struct ahash_request *areq) memset(req, 0, sizeof(*req)); /* Copy (half of) the key */ - memcpy(req->state, ctx->ipad, SHA3_224_BLOCK_SIZE / 2); + memcpy(req->state, &ctx->base.ipad, SHA3_224_BLOCK_SIZE / 2); /* Start of HMAC should have len == processed == blocksize */ req->len = SHA3_224_BLOCK_SIZE; req->processed = SHA3_224_BLOCK_SIZE; @@ -2927,7 +2936,7 @@ static int safexcel_hmac_sha3_256_init(struct ahash_request *areq) memset(req, 0, sizeof(*req)); /* Copy (half of) the key */ - memcpy(req->state, ctx->ipad, SHA3_256_BLOCK_SIZE / 2); + memcpy(req->state, &ctx->base.ipad, SHA3_256_BLOCK_SIZE / 2); /* Start of HMAC should have len == processed == blocksize */ req->len = SHA3_256_BLOCK_SIZE; req->processed = SHA3_256_BLOCK_SIZE; @@ -2998,7 +3007,7 @@ static int safexcel_hmac_sha3_384_init(struct ahash_request *areq) memset(req, 0, sizeof(*req)); /* Copy (half of) the key */ - memcpy(req->state, ctx->ipad, SHA3_384_BLOCK_SIZE / 2); + memcpy(req->state, &ctx->base.ipad, SHA3_384_BLOCK_SIZE / 2); /* Start of HMAC should have len == processed == blocksize */ req->len = SHA3_384_BLOCK_SIZE; req->processed = SHA3_384_BLOCK_SIZE; @@ -3069,7 +3078,7 @@ static int safexcel_hmac_sha3_512_init(struct ahash_request *areq) memset(req, 0, sizeof(*req)); /* Copy (half of) the key */ - memcpy(req->state, ctx->ipad, SHA3_512_BLOCK_SIZE / 2); + memcpy(req->state, &ctx->base.ipad, SHA3_512_BLOCK_SIZE / 2); /* Start of HMAC should have len == processed == blocksize */ req->len = SHA3_512_BLOCK_SIZE; req->processed = SHA3_512_BLOCK_SIZE; diff --git a/drivers/crypto/inside-secure/safexcel_ring.c b/drivers/crypto/inside-secure/safexcel_ring.c index e454c3d44f07..90f15032c8df 100644 --- a/drivers/crypto/inside-secure/safexcel_ring.c +++ b/drivers/crypto/inside-secure/safexcel_ring.c @@ -236,8 +236,8 @@ struct safexcel_result_desc *safexcel_add_rdesc(struct safexcel_crypto_priv *pri rdesc->particle_size = len; rdesc->rsvd0 = 0; - rdesc->descriptor_overflow = 0; - rdesc->buffer_overflow = 0; + rdesc->descriptor_overflow = 1; /* assume error */ + rdesc->buffer_overflow = 1; /* assume error */ rdesc->last_seg = last; rdesc->first_seg = first; rdesc->result_size = EIP197_RD64_RESULT_SIZE; @@ -245,9 +245,10 @@ struct safexcel_result_desc *safexcel_add_rdesc(struct safexcel_crypto_priv *pri rdesc->data_lo = lower_32_bits(data); rdesc->data_hi = upper_32_bits(data); - /* Clear length & error code in result token */ + /* Clear length in result token */ rtoken->packet_length = 0; - rtoken->error_code = 0; + /* Assume errors - HW will clear if not the case */ + rtoken->error_code = 0x7fff; return rdesc; } diff --git a/drivers/crypto/ixp4xx_crypto.c b/drivers/crypto/ixp4xx_crypto.c index f478bb0a566a..276012e7c482 100644 --- a/drivers/crypto/ixp4xx_crypto.c +++ b/drivers/crypto/ixp4xx_crypto.c @@ -528,7 +528,7 @@ static void release_ixp_crypto(struct device *dev) if (crypt_virt) { dma_free_coherent(dev, - NPE_QLEN_TOTAL * sizeof( struct crypt_ctl), + NPE_QLEN * sizeof(struct crypt_ctl), crypt_virt, crypt_phys); } } diff --git a/drivers/crypto/marvell/cesa/cesa.c b/drivers/crypto/marvell/cesa/cesa.c index d63bca9718dc..06211858bf2e 100644 --- a/drivers/crypto/marvell/cesa/cesa.c +++ b/drivers/crypto/marvell/cesa/cesa.c @@ -437,7 +437,6 @@ static int mv_cesa_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct mv_cesa_dev *cesa; struct mv_cesa_engine *engines; - struct resource *res; int irq, ret, i, cpu; u32 sram_size; @@ -475,8 +474,7 @@ static int mv_cesa_probe(struct platform_device *pdev) spin_lock_init(&cesa->lock); - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "regs"); - cesa->regs = devm_ioremap_resource(dev, res); + cesa->regs = devm_platform_ioremap_resource_byname(pdev, "regs"); if (IS_ERR(cesa->regs)) return PTR_ERR(cesa->regs); diff --git a/drivers/crypto/marvell/cesa/cesa.h b/drivers/crypto/marvell/cesa/cesa.h index 0c9cbb681e49..fabfaaccca87 100644 --- a/drivers/crypto/marvell/cesa/cesa.h +++ b/drivers/crypto/marvell/cesa/cesa.h @@ -2,12 +2,10 @@ #ifndef __MARVELL_CESA_H__ #define __MARVELL_CESA_H__ -#include <crypto/algapi.h> -#include <crypto/hash.h> #include <crypto/internal/hash.h> #include <crypto/internal/skcipher.h> -#include <linux/crypto.h> +#include <linux/dma-direction.h> #include <linux/dmapool.h> #define CESA_ENGINE_OFF(i) (((i) * 0x2000)) @@ -239,7 +237,7 @@ struct mv_cesa_sec_accel_desc { * Context associated to a cipher operation. */ struct mv_cesa_skcipher_op_ctx { - u32 key[8]; + __le32 key[8]; u32 iv[4]; }; @@ -252,7 +250,7 @@ struct mv_cesa_skcipher_op_ctx { */ struct mv_cesa_hash_op_ctx { u32 iv[16]; - u32 hash[8]; + __le32 hash[8]; }; /** @@ -300,8 +298,14 @@ struct mv_cesa_op_ctx { */ struct mv_cesa_tdma_desc { __le32 byte_cnt; - __le32 src; - __le32 dst; + union { + __le32 src; + dma_addr_t src_dma; + }; + union { + __le32 dst; + dma_addr_t dst_dma; + }; __le32 next_dma; /* Software state */ @@ -506,7 +510,7 @@ struct mv_cesa_hash_ctx { */ struct mv_cesa_hmac_ctx { struct mv_cesa_ctx base; - u32 iv[16]; + __be32 iv[16]; }; /** diff --git a/drivers/crypto/marvell/cesa/cipher.c b/drivers/crypto/marvell/cesa/cipher.c index 45b4d7a29833..b4a6ff9dd6d5 100644 --- a/drivers/crypto/marvell/cesa/cipher.c +++ b/drivers/crypto/marvell/cesa/cipher.c @@ -11,6 +11,8 @@ #include <crypto/aes.h> #include <crypto/internal/des.h> +#include <linux/device.h> +#include <linux/dma-mapping.h> #include "cesa.h" @@ -262,8 +264,7 @@ static int mv_cesa_aes_setkey(struct crypto_skcipher *cipher, const u8 *key, remaining = (ctx->aes.key_length - 16) / 4; offset = ctx->aes.key_length + 24 - remaining; for (i = 0; i < remaining; i++) - ctx->aes.key_dec[4 + i] = - cpu_to_le32(ctx->aes.key_enc[offset + i]); + ctx->aes.key_dec[4 + i] = ctx->aes.key_enc[offset + i]; return 0; } diff --git a/drivers/crypto/marvell/cesa/hash.c b/drivers/crypto/marvell/cesa/hash.c index f2a2fc111164..add7ea011c98 100644 --- a/drivers/crypto/marvell/cesa/hash.c +++ b/drivers/crypto/marvell/cesa/hash.c @@ -12,6 +12,8 @@ #include <crypto/hmac.h> #include <crypto/md5.h> #include <crypto/sha.h> +#include <linux/device.h> +#include <linux/dma-mapping.h> #include "cesa.h" @@ -222,9 +224,11 @@ static void mv_cesa_ahash_std_step(struct ahash_request *req) CESA_SA_DATA_SRAM_OFFSET + len, new_cache_ptr); } else { - len += mv_cesa_ahash_pad_req(creq, - engine->sram + len + - CESA_SA_DATA_SRAM_OFFSET); + i = mv_cesa_ahash_pad_req(creq, creq->cache); + len += i; + memcpy_toio(engine->sram + len + + CESA_SA_DATA_SRAM_OFFSET, + creq->cache, i); } if (frag_mode == CESA_SA_DESC_CFG_LAST_FRAG) @@ -342,7 +346,7 @@ static void mv_cesa_ahash_complete(struct crypto_async_request *req) */ data = creq->base.chain.last->op->ctx.hash.hash; for (i = 0; i < digsize / 4; i++) - creq->state[i] = cpu_to_le32(data[i]); + creq->state[i] = le32_to_cpu(data[i]); memcpy(ahashreq->result, data, digsize); } else { @@ -1265,10 +1269,10 @@ static int mv_cesa_ahmac_md5_setkey(struct crypto_ahash *tfm, const u8 *key, return ret; for (i = 0; i < ARRAY_SIZE(istate.hash); i++) - ctx->iv[i] = be32_to_cpu(istate.hash[i]); + ctx->iv[i] = cpu_to_be32(istate.hash[i]); for (i = 0; i < ARRAY_SIZE(ostate.hash); i++) - ctx->iv[i + 8] = be32_to_cpu(ostate.hash[i]); + ctx->iv[i + 8] = cpu_to_be32(ostate.hash[i]); return 0; } @@ -1336,10 +1340,10 @@ static int mv_cesa_ahmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key, return ret; for (i = 0; i < ARRAY_SIZE(istate.state); i++) - ctx->iv[i] = be32_to_cpu(istate.state[i]); + ctx->iv[i] = cpu_to_be32(istate.state[i]); for (i = 0; i < ARRAY_SIZE(ostate.state); i++) - ctx->iv[i + 8] = be32_to_cpu(ostate.state[i]); + ctx->iv[i + 8] = cpu_to_be32(ostate.state[i]); return 0; } @@ -1394,10 +1398,10 @@ static int mv_cesa_ahmac_sha256_setkey(struct crypto_ahash *tfm, const u8 *key, return ret; for (i = 0; i < ARRAY_SIZE(istate.state); i++) - ctx->iv[i] = be32_to_cpu(istate.state[i]); + ctx->iv[i] = cpu_to_be32(istate.state[i]); for (i = 0; i < ARRAY_SIZE(ostate.state); i++) - ctx->iv[i + 8] = be32_to_cpu(ostate.state[i]); + ctx->iv[i + 8] = cpu_to_be32(ostate.state[i]); return 0; } diff --git a/drivers/crypto/marvell/cesa/tdma.c b/drivers/crypto/marvell/cesa/tdma.c index b81ee276fe0e..5d9c48fb72b2 100644 --- a/drivers/crypto/marvell/cesa/tdma.c +++ b/drivers/crypto/marvell/cesa/tdma.c @@ -83,10 +83,10 @@ void mv_cesa_dma_prepare(struct mv_cesa_req *dreq, for (tdma = dreq->chain.first; tdma; tdma = tdma->next) { if (tdma->flags & CESA_TDMA_DST_IN_SRAM) - tdma->dst = cpu_to_le32(tdma->dst + engine->sram_dma); + tdma->dst = cpu_to_le32(tdma->dst_dma + engine->sram_dma); if (tdma->flags & CESA_TDMA_SRC_IN_SRAM) - tdma->src = cpu_to_le32(tdma->src + engine->sram_dma); + tdma->src = cpu_to_le32(tdma->src_dma + engine->sram_dma); if ((tdma->flags & CESA_TDMA_TYPE_MSK) == CESA_TDMA_OP) mv_cesa_adjust_op(engine, tdma->op); @@ -114,7 +114,7 @@ void mv_cesa_tdma_chain(struct mv_cesa_engine *engine, */ if (!(last->flags & CESA_TDMA_BREAK_CHAIN) && !(dreq->chain.first->flags & CESA_TDMA_SET_STATE)) - last->next_dma = dreq->chain.first->cur_dma; + last->next_dma = cpu_to_le32(dreq->chain.first->cur_dma); } } @@ -237,8 +237,8 @@ int mv_cesa_dma_add_result_op(struct mv_cesa_tdma_chain *chain, dma_addr_t src, return -EIO; tdma->byte_cnt = cpu_to_le32(size | BIT(31)); - tdma->src = src; - tdma->dst = op_desc->src; + tdma->src_dma = src; + tdma->dst_dma = op_desc->src_dma; tdma->op = op_desc->op; flags &= (CESA_TDMA_DST_IN_SRAM | CESA_TDMA_SRC_IN_SRAM); @@ -272,7 +272,7 @@ struct mv_cesa_op_ctx *mv_cesa_dma_add_op(struct mv_cesa_tdma_chain *chain, tdma->op = op; tdma->byte_cnt = cpu_to_le32(size | BIT(31)); tdma->src = cpu_to_le32(dma_handle); - tdma->dst = CESA_SA_CFG_SRAM_OFFSET; + tdma->dst_dma = CESA_SA_CFG_SRAM_OFFSET; tdma->flags = CESA_TDMA_DST_IN_SRAM | CESA_TDMA_OP; return op; @@ -289,8 +289,8 @@ int mv_cesa_dma_add_data_transfer(struct mv_cesa_tdma_chain *chain, return PTR_ERR(tdma); tdma->byte_cnt = cpu_to_le32(size | BIT(31)); - tdma->src = src; - tdma->dst = dst; + tdma->src_dma = src; + tdma->dst_dma = dst; flags &= (CESA_TDMA_DST_IN_SRAM | CESA_TDMA_SRC_IN_SRAM); tdma->flags = flags | CESA_TDMA_DATA; diff --git a/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c b/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c index cc103b1bc224..40b482198ebc 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c +++ b/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c @@ -824,18 +824,12 @@ static ssize_t eng_grp_info_show(struct device *dev, static int create_sysfs_eng_grps_info(struct device *dev, struct otx_cpt_eng_grp_info *eng_grp) { - int ret; - eng_grp->info_attr.show = eng_grp_info_show; eng_grp->info_attr.store = NULL; eng_grp->info_attr.attr.name = eng_grp->sysfs_info_name; eng_grp->info_attr.attr.mode = 0440; sysfs_attr_init(&eng_grp->info_attr.attr); - ret = device_create_file(dev, &eng_grp->info_attr); - if (ret) - return ret; - - return 0; + return device_create_file(dev, &eng_grp->info_attr); } static void ucode_unload(struct device *dev, struct otx_cpt_ucode *ucode) diff --git a/drivers/crypto/mediatek/mtk-aes.c b/drivers/crypto/mediatek/mtk-aes.c index 4ad3571ab6af..7323066724c3 100644 --- a/drivers/crypto/mediatek/mtk-aes.c +++ b/drivers/crypto/mediatek/mtk-aes.c @@ -126,7 +126,7 @@ struct mtk_aes_ctx { struct mtk_aes_ctr_ctx { struct mtk_aes_base_ctx base; - u32 iv[AES_BLOCK_SIZE / sizeof(u32)]; + __be32 iv[AES_BLOCK_SIZE / sizeof(u32)]; size_t offset; struct scatterlist src[2]; struct scatterlist dst[2]; @@ -242,22 +242,6 @@ static inline void mtk_aes_restore_sg(const struct mtk_aes_dma *dma) sg->length += dma->remainder; } -static inline void mtk_aes_write_state_le(__le32 *dst, const u32 *src, u32 size) -{ - int i; - - for (i = 0; i < SIZE_IN_WORDS(size); i++) - dst[i] = cpu_to_le32(src[i]); -} - -static inline void mtk_aes_write_state_be(__be32 *dst, const u32 *src, u32 size) -{ - int i; - - for (i = 0; i < SIZE_IN_WORDS(size); i++) - dst[i] = cpu_to_be32(src[i]); -} - static inline int mtk_aes_complete(struct mtk_cryp *cryp, struct mtk_aes_rec *aes, int err) @@ -321,7 +305,7 @@ static int mtk_aes_xmit(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) /* Prepare enough space for authenticated tag */ if (aes->flags & AES_FLAGS_GCM) - res->hdr += AES_BLOCK_SIZE; + le32_add_cpu(&res->hdr, AES_BLOCK_SIZE); /* * Make sure that all changes to the DMA ring are done before we @@ -449,10 +433,10 @@ static void mtk_aes_info_init(struct mtk_cryp *cryp, struct mtk_aes_rec *aes, return; } - mtk_aes_write_state_le(info->state + ctx->keylen, (void *)req->iv, - AES_BLOCK_SIZE); + memcpy(info->state + ctx->keylen, req->iv, AES_BLOCK_SIZE); ctr: - info->tfm[0] += AES_TFM_SIZE(SIZE_IN_WORDS(AES_BLOCK_SIZE)); + le32_add_cpu(&info->tfm[0], + le32_to_cpu(AES_TFM_SIZE(SIZE_IN_WORDS(AES_BLOCK_SIZE)))); info->tfm[1] |= AES_TFM_FULL_IV; info->cmd[cnt++] = AES_CMD2; ecb: @@ -601,8 +585,7 @@ static int mtk_aes_ctr_transfer(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) scatterwalk_ffwd(cctx->dst, req->dst, cctx->offset)); /* Write IVs into transform state buffer. */ - mtk_aes_write_state_le(ctx->info.state + ctx->keylen, cctx->iv, - AES_BLOCK_SIZE); + memcpy(ctx->info.state + ctx->keylen, cctx->iv, AES_BLOCK_SIZE); if (unlikely(fragmented)) { /* @@ -654,7 +637,7 @@ static int mtk_aes_setkey(struct crypto_skcipher *tfm, } ctx->keylen = SIZE_IN_WORDS(keylen); - mtk_aes_write_state_le(ctx->key, (const u32 *)key, keylen); + memcpy(ctx->key, key, keylen); return 0; } @@ -848,7 +831,7 @@ mtk_aes_gcm_ctx_cast(struct mtk_aes_base_ctx *ctx) static int mtk_aes_gcm_tag_verify(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) { - u32 status = cryp->ring[aes->id]->res_prev->ct; + __le32 status = cryp->ring[aes->id]->res_prev->ct; return mtk_aes_complete(cryp, aes, (status & AES_AUTH_TAG_ERR) ? -EBADMSG : 0); @@ -866,7 +849,7 @@ static void mtk_aes_gcm_info_init(struct mtk_cryp *cryp, u32 ivsize = crypto_aead_ivsize(crypto_aead_reqtfm(req)); u32 cnt = 0; - ctx->ct_hdr = AES_CT_CTRL_HDR | len; + ctx->ct_hdr = AES_CT_CTRL_HDR | cpu_to_le32(len); info->cmd[cnt++] = AES_GCM_CMD0 | cpu_to_le32(req->assoclen); info->cmd[cnt++] = AES_GCM_CMD1 | cpu_to_le32(req->assoclen); @@ -889,8 +872,8 @@ static void mtk_aes_gcm_info_init(struct mtk_cryp *cryp, info->tfm[1] = AES_TFM_CTR_INIT | AES_TFM_IV_CTR_MODE | AES_TFM_3IV | AES_TFM_ENC_HASH; - mtk_aes_write_state_le(info->state + ctx->keylen + SIZE_IN_WORDS( - AES_BLOCK_SIZE), (const u32 *)req->iv, ivsize); + memcpy(info->state + ctx->keylen + SIZE_IN_WORDS(AES_BLOCK_SIZE), + req->iv, ivsize); } static int mtk_aes_gcm_dma(struct mtk_cryp *cryp, struct mtk_aes_rec *aes, @@ -994,9 +977,13 @@ static int mtk_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key, u32 keylen) { struct mtk_aes_base_ctx *ctx = crypto_aead_ctx(aead); - u8 hash[AES_BLOCK_SIZE] __aligned(4) = {}; + union { + u32 x32[SIZE_IN_WORDS(AES_BLOCK_SIZE)]; + u8 x8[AES_BLOCK_SIZE]; + } hash = {}; struct crypto_aes_ctx aes_ctx; int err; + int i; switch (keylen) { case AES_KEYSIZE_128: @@ -1019,12 +1006,16 @@ static int mtk_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key, if (err) return err; - aes_encrypt(&aes_ctx, hash, hash); + aes_encrypt(&aes_ctx, hash.x8, hash.x8); memzero_explicit(&aes_ctx, sizeof(aes_ctx)); - mtk_aes_write_state_le(ctx->key, (const u32 *)key, keylen); - mtk_aes_write_state_be(ctx->key + ctx->keylen, (const u32 *)hash, - AES_BLOCK_SIZE); + memcpy(ctx->key, key, keylen); + + /* Why do we need to do this? */ + for (i = 0; i < SIZE_IN_WORDS(AES_BLOCK_SIZE); i++) + hash.x32[i] = swab32(hash.x32[i]); + + memcpy(ctx->key + ctx->keylen, &hash, AES_BLOCK_SIZE); return 0; } diff --git a/drivers/crypto/mediatek/mtk-platform.c b/drivers/crypto/mediatek/mtk-platform.c index 7e3ad085b5bd..9d878620e5c9 100644 --- a/drivers/crypto/mediatek/mtk-platform.c +++ b/drivers/crypto/mediatek/mtk-platform.c @@ -185,8 +185,6 @@ static int mtk_dfe_dse_state_check(struct mtk_cryp *cryp) static int mtk_dfe_dse_reset(struct mtk_cryp *cryp) { - int err; - /* Reset DSE/DFE and correct system priorities for all rings. */ writel(MTK_DFSE_THR_CTRL_RESET, cryp->base + DFE_THR_CTRL); writel(0, cryp->base + DFE_PRIO_0); @@ -200,11 +198,7 @@ static int mtk_dfe_dse_reset(struct mtk_cryp *cryp) writel(0, cryp->base + DSE_PRIO_2); writel(0, cryp->base + DSE_PRIO_3); - err = mtk_dfe_dse_state_check(cryp); - if (err) - return err; - - return 0; + return mtk_dfe_dse_state_check(cryp); } static void mtk_cmd_desc_ring_setup(struct mtk_cryp *cryp, @@ -442,7 +436,7 @@ static void mtk_desc_dma_free(struct mtk_cryp *cryp) static int mtk_desc_ring_alloc(struct mtk_cryp *cryp) { struct mtk_ring **ring = cryp->ring; - int i, err = ENOMEM; + int i; for (i = 0; i < MTK_RING_MAX; i++) { ring[i] = kzalloc(sizeof(**ring), GFP_KERNEL); @@ -469,14 +463,14 @@ static int mtk_desc_ring_alloc(struct mtk_cryp *cryp) return 0; err_cleanup: - for (; i--; ) { + do { dma_free_coherent(cryp->dev, MTK_DESC_RING_SZ, ring[i]->res_base, ring[i]->res_dma); dma_free_coherent(cryp->dev, MTK_DESC_RING_SZ, ring[i]->cmd_base, ring[i]->cmd_dma); kfree(ring[i]); - } - return err; + } while (i--); + return -ENOMEM; } static int mtk_crypto_probe(struct platform_device *pdev) diff --git a/drivers/crypto/mediatek/mtk-sha.c b/drivers/crypto/mediatek/mtk-sha.c index da3f0b8814aa..3d5d7d68b03b 100644 --- a/drivers/crypto/mediatek/mtk-sha.c +++ b/drivers/crypto/mediatek/mtk-sha.c @@ -239,7 +239,7 @@ static int mtk_sha_append_sg(struct mtk_sha_reqctx *ctx) static void mtk_sha_fill_padding(struct mtk_sha_reqctx *ctx, u32 len) { u32 index, padlen; - u64 bits[2]; + __be64 bits[2]; u64 size = ctx->digcnt; size += ctx->bufcnt; diff --git a/drivers/crypto/n2_core.c b/drivers/crypto/n2_core.c index d8aec5153b21..3642bf83d809 100644 --- a/drivers/crypto/n2_core.c +++ b/drivers/crypto/n2_core.c @@ -249,7 +249,7 @@ static inline bool n2_should_run_async(struct spu_queue *qp, int this_len) struct n2_ahash_alg { struct list_head entry; const u8 *hash_zero; - const u32 *hash_init; + const u8 *hash_init; u8 hw_op_hashsz; u8 digest_size; u8 auth_type; @@ -662,7 +662,6 @@ struct n2_skcipher_context { u8 aes[AES_MAX_KEY_SIZE]; u8 des[DES_KEY_SIZE]; u8 des3[3 * DES_KEY_SIZE]; - u8 arc4[258]; /* S-box, X, Y */ } key; }; @@ -789,36 +788,6 @@ static int n2_3des_setkey(struct crypto_skcipher *skcipher, const u8 *key, return 0; } -static int n2_arc4_setkey(struct crypto_skcipher *skcipher, const u8 *key, - unsigned int keylen) -{ - struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher); - struct n2_skcipher_context *ctx = crypto_tfm_ctx(tfm); - struct n2_skcipher_alg *n2alg = n2_skcipher_alg(skcipher); - u8 *s = ctx->key.arc4; - u8 *x = s + 256; - u8 *y = x + 1; - int i, j, k; - - ctx->enc_type = n2alg->enc_type; - - j = k = 0; - *x = 0; - *y = 0; - for (i = 0; i < 256; i++) - s[i] = i; - for (i = 0; i < 256; i++) { - u8 a = s[i]; - j = (j + key[k] + a) & 0xff; - s[i] = s[j]; - s[j] = a; - if (++k >= keylen) - k = 0; - } - - return 0; -} - static inline int skcipher_descriptor_len(int nbytes, unsigned int block_size) { int this_len = nbytes; @@ -1122,21 +1091,6 @@ struct n2_skcipher_tmpl { }; static const struct n2_skcipher_tmpl skcipher_tmpls[] = { - /* ARC4: only ECB is supported (chaining bits ignored) */ - { .name = "ecb(arc4)", - .drv_name = "ecb-arc4", - .block_size = 1, - .enc_type = (ENC_TYPE_ALG_RC4_STREAM | - ENC_TYPE_CHAINING_ECB), - .skcipher = { - .min_keysize = 1, - .max_keysize = 256, - .setkey = n2_arc4_setkey, - .encrypt = n2_encrypt_ecb, - .decrypt = n2_decrypt_ecb, - }, - }, - /* DES: ECB CBC and CFB are supported */ { .name = "ecb(des)", .drv_name = "ecb-des", @@ -1271,7 +1225,7 @@ static LIST_HEAD(skcipher_algs); struct n2_hash_tmpl { const char *name; const u8 *hash_zero; - const u32 *hash_init; + const u8 *hash_init; u8 hw_op_hashsz; u8 digest_size; u8 block_size; @@ -1279,7 +1233,7 @@ struct n2_hash_tmpl { u8 hmac_type; }; -static const u32 n2_md5_init[MD5_HASH_WORDS] = { +static const __le32 n2_md5_init[MD5_HASH_WORDS] = { cpu_to_le32(MD5_H0), cpu_to_le32(MD5_H1), cpu_to_le32(MD5_H2), @@ -1300,7 +1254,7 @@ static const u32 n2_sha224_init[SHA256_DIGEST_SIZE / 4] = { static const struct n2_hash_tmpl hash_tmpls[] = { { .name = "md5", .hash_zero = md5_zero_message_hash, - .hash_init = n2_md5_init, + .hash_init = (u8 *)n2_md5_init, .auth_type = AUTH_TYPE_MD5, .hmac_type = AUTH_TYPE_HMAC_MD5, .hw_op_hashsz = MD5_DIGEST_SIZE, @@ -1308,7 +1262,7 @@ static const struct n2_hash_tmpl hash_tmpls[] = { .block_size = MD5_HMAC_BLOCK_SIZE }, { .name = "sha1", .hash_zero = sha1_zero_message_hash, - .hash_init = n2_sha1_init, + .hash_init = (u8 *)n2_sha1_init, .auth_type = AUTH_TYPE_SHA1, .hmac_type = AUTH_TYPE_HMAC_SHA1, .hw_op_hashsz = SHA1_DIGEST_SIZE, @@ -1316,7 +1270,7 @@ static const struct n2_hash_tmpl hash_tmpls[] = { .block_size = SHA1_BLOCK_SIZE }, { .name = "sha256", .hash_zero = sha256_zero_message_hash, - .hash_init = n2_sha256_init, + .hash_init = (u8 *)n2_sha256_init, .auth_type = AUTH_TYPE_SHA256, .hmac_type = AUTH_TYPE_HMAC_SHA256, .hw_op_hashsz = SHA256_DIGEST_SIZE, @@ -1324,7 +1278,7 @@ static const struct n2_hash_tmpl hash_tmpls[] = { .block_size = SHA256_BLOCK_SIZE }, { .name = "sha224", .hash_zero = sha224_zero_message_hash, - .hash_init = n2_sha224_init, + .hash_init = (u8 *)n2_sha224_init, .auth_type = AUTH_TYPE_SHA256, .hmac_type = AUTH_TYPE_RESERVED, .hw_op_hashsz = SHA256_DIGEST_SIZE, diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index 954d703f2981..a3b38d2c92e7 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -39,6 +39,7 @@ #include <crypto/hash.h> #include <crypto/hmac.h> #include <crypto/internal/hash.h> +#include <crypto/engine.h> #define MD5_DIGEST_SIZE 16 @@ -100,7 +101,6 @@ #define DEFAULT_AUTOSUSPEND_DELAY 1000 /* mostly device flags */ -#define FLAGS_BUSY 0 #define FLAGS_FINAL 1 #define FLAGS_DMA_ACTIVE 2 #define FLAGS_OUTPUT_READY 3 @@ -144,7 +144,7 @@ struct omap_sham_dev; struct omap_sham_reqctx { struct omap_sham_dev *dd; unsigned long flags; - unsigned long op; + u8 op; u8 digest[SHA512_DIGEST_SIZE] OMAP_ALIGNED; size_t digcnt; @@ -168,6 +168,7 @@ struct omap_sham_hmac_ctx { }; struct omap_sham_ctx { + struct crypto_engine_ctx enginectx; unsigned long flags; /* fallback stuff */ @@ -219,7 +220,6 @@ struct omap_sham_dev { struct device *dev; void __iomem *io_base; int irq; - spinlock_t lock; int err; struct dma_chan *dma_lch; struct tasklet_struct done_task; @@ -230,6 +230,7 @@ struct omap_sham_dev { int fallback_sz; struct crypto_queue queue; struct ahash_request *req; + struct crypto_engine *engine; const struct omap_sham_pdata *pdata; }; @@ -245,6 +246,9 @@ static struct omap_sham_drv sham = { .lock = __SPIN_LOCK_UNLOCKED(sham.lock), }; +static int omap_sham_enqueue(struct ahash_request *req, unsigned int op); +static void omap_sham_finish_req(struct ahash_request *req, int err); + static inline u32 omap_sham_read(struct omap_sham_dev *dd, u32 offset) { return __raw_readl(dd->io_base + offset); @@ -456,6 +460,9 @@ static void omap_sham_write_ctrl_omap4(struct omap_sham_dev *dd, size_t length, struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req); u32 val, mask; + if (likely(ctx->digcnt)) + omap_sham_write(dd, SHA_REG_DIGCNT(dd), ctx->digcnt); + /* * Setting ALGO_CONST only for the first iteration and * CLOSE_HASH only for the last one. Note that flags mode bits @@ -854,13 +861,16 @@ static int omap_sham_align_sgs(struct scatterlist *sg, return 0; } -static int omap_sham_prepare_request(struct ahash_request *req, bool update) +static int omap_sham_prepare_request(struct crypto_engine *engine, void *areq) { + struct ahash_request *req = container_of(areq, struct ahash_request, + base); struct omap_sham_reqctx *rctx = ahash_request_ctx(req); int bs; int ret; unsigned int nbytes; bool final = rctx->flags & BIT(FLAGS_FINUP); + bool update = rctx->op == OP_UPDATE; int hash_later; bs = get_block_size(rctx); @@ -1021,7 +1031,7 @@ static int omap_sham_update_req(struct omap_sham_dev *dd) struct omap_sham_reqctx *ctx = ahash_request_ctx(req); int err; bool final = (ctx->flags & BIT(FLAGS_FINUP)) && - !(dd->flags & BIT(FLAGS_HUGE)); + !(dd->flags & BIT(FLAGS_HUGE)); dev_dbg(dd->dev, "update_req: total: %u, digcnt: %zd, final: %d", ctx->total, ctx->digcnt, final); @@ -1069,6 +1079,39 @@ static int omap_sham_final_req(struct omap_sham_dev *dd) return err; } +static int omap_sham_hash_one_req(struct crypto_engine *engine, void *areq) +{ + struct ahash_request *req = container_of(areq, struct ahash_request, + base); + struct omap_sham_reqctx *ctx = ahash_request_ctx(req); + struct omap_sham_dev *dd = ctx->dd; + int err; + bool final = (ctx->flags & BIT(FLAGS_FINUP)) && + !(dd->flags & BIT(FLAGS_HUGE)); + + dev_dbg(dd->dev, "hash-one: op: %u, total: %u, digcnt: %zd, final: %d", + ctx->op, ctx->total, ctx->digcnt, final); + + dd->req = req; + + err = omap_sham_hw_init(dd); + if (err) + return err; + + if (ctx->digcnt) + dd->pdata->copy_hash(req, 0); + + if (ctx->op == OP_UPDATE) + err = omap_sham_update_req(dd); + else if (ctx->op == OP_FINAL) + err = omap_sham_final_req(dd); + + if (err != -EINPROGRESS) + omap_sham_finish_req(req, err); + + return 0; +} + static int omap_sham_finish_hmac(struct ahash_request *req) { struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm); @@ -1116,25 +1159,20 @@ static void omap_sham_finish_req(struct ahash_request *req, int err) ctx->sg = NULL; - dd->flags &= ~(BIT(FLAGS_SGS_ALLOCED) | BIT(FLAGS_SGS_COPIED)); + dd->flags &= ~(BIT(FLAGS_SGS_ALLOCED) | BIT(FLAGS_SGS_COPIED) | + BIT(FLAGS_CPU) | BIT(FLAGS_DMA_READY) | + BIT(FLAGS_OUTPUT_READY)); + + if (!err) + dd->pdata->copy_hash(req, 1); if (dd->flags & BIT(FLAGS_HUGE)) { - dd->flags &= ~(BIT(FLAGS_CPU) | BIT(FLAGS_DMA_READY) | - BIT(FLAGS_OUTPUT_READY) | BIT(FLAGS_HUGE)); - omap_sham_prepare_request(req, ctx->op == OP_UPDATE); - if (ctx->op == OP_UPDATE || (dd->flags & BIT(FLAGS_HUGE))) { - err = omap_sham_update_req(dd); - if (err != -EINPROGRESS && - (ctx->flags & BIT(FLAGS_FINUP))) - err = omap_sham_final_req(dd); - } else if (ctx->op == OP_FINAL) { - omap_sham_final_req(dd); - } + /* Re-enqueue the request */ + omap_sham_enqueue(req, ctx->op); return; } if (!err) { - dd->pdata->copy_hash(req, 1); if (test_bit(FLAGS_FINAL, &dd->flags)) err = omap_sham_finish(req); } else { @@ -1142,7 +1180,7 @@ static void omap_sham_finish_req(struct ahash_request *req, int err) } /* atomic operation is not needed here */ - dd->flags &= ~(BIT(FLAGS_BUSY) | BIT(FLAGS_FINAL) | BIT(FLAGS_CPU) | + dd->flags &= ~(BIT(FLAGS_FINAL) | BIT(FLAGS_CPU) | BIT(FLAGS_DMA_READY) | BIT(FLAGS_OUTPUT_READY)); pm_runtime_mark_last_busy(dd->dev); @@ -1150,81 +1188,13 @@ static void omap_sham_finish_req(struct ahash_request *req, int err) ctx->offset = 0; - if (req->base.complete) - req->base.complete(&req->base, err); + crypto_finalize_hash_request(dd->engine, req, err); } static int omap_sham_handle_queue(struct omap_sham_dev *dd, struct ahash_request *req) { - struct crypto_async_request *async_req, *backlog; - struct omap_sham_reqctx *ctx; - unsigned long flags; - int err = 0, ret = 0; - -retry: - spin_lock_irqsave(&dd->lock, flags); - if (req) - ret = ahash_enqueue_request(&dd->queue, req); - if (test_bit(FLAGS_BUSY, &dd->flags)) { - spin_unlock_irqrestore(&dd->lock, flags); - return ret; - } - backlog = crypto_get_backlog(&dd->queue); - async_req = crypto_dequeue_request(&dd->queue); - if (async_req) - set_bit(FLAGS_BUSY, &dd->flags); - spin_unlock_irqrestore(&dd->lock, flags); - - if (!async_req) - return ret; - - if (backlog) - backlog->complete(backlog, -EINPROGRESS); - - req = ahash_request_cast(async_req); - dd->req = req; - ctx = ahash_request_ctx(req); - - err = omap_sham_prepare_request(req, ctx->op == OP_UPDATE); - if (err || !ctx->total) - goto err1; - - dev_dbg(dd->dev, "handling new req, op: %lu, nbytes: %d\n", - ctx->op, req->nbytes); - - err = omap_sham_hw_init(dd); - if (err) - goto err1; - - if (ctx->digcnt) - /* request has changed - restore hash */ - dd->pdata->copy_hash(req, 0); - - if (ctx->op == OP_UPDATE || (dd->flags & BIT(FLAGS_HUGE))) { - err = omap_sham_update_req(dd); - if (err != -EINPROGRESS && (ctx->flags & BIT(FLAGS_FINUP))) - /* no final() after finup() */ - err = omap_sham_final_req(dd); - } else if (ctx->op == OP_FINAL) { - err = omap_sham_final_req(dd); - } -err1: - dev_dbg(dd->dev, "exit, err: %d\n", err); - - if (err != -EINPROGRESS) { - /* done_task will not finish it, so do it here */ - omap_sham_finish_req(req, err); - req = NULL; - - /* - * Execute next request immediately if there is anything - * in queue. - */ - goto retry; - } - - return ret; + return crypto_transfer_hash_request_to_engine(dd->engine, req); } static int omap_sham_enqueue(struct ahash_request *req, unsigned int op) @@ -1394,6 +1364,10 @@ static int omap_sham_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base) } + tctx->enginectx.op.do_one_request = omap_sham_hash_one_req; + tctx->enginectx.op.prepare_request = omap_sham_prepare_request; + tctx->enginectx.op.unprepare_request = NULL; + return 0; } @@ -1757,11 +1731,6 @@ static void omap_sham_done_task(unsigned long data) dev_dbg(dd->dev, "%s: flags=%lx\n", __func__, dd->flags); - if (!test_bit(FLAGS_BUSY, &dd->flags)) { - omap_sham_handle_queue(dd, NULL); - return; - } - if (test_bit(FLAGS_CPU, &dd->flags)) { if (test_and_clear_bit(FLAGS_OUTPUT_READY, &dd->flags)) goto finish; @@ -1786,20 +1755,12 @@ finish: dev_dbg(dd->dev, "update done: err: %d\n", err); /* finish curent request */ omap_sham_finish_req(dd->req, err); - - /* If we are not busy, process next req */ - if (!test_bit(FLAGS_BUSY, &dd->flags)) - omap_sham_handle_queue(dd, NULL); } static irqreturn_t omap_sham_irq_common(struct omap_sham_dev *dd) { - if (!test_bit(FLAGS_BUSY, &dd->flags)) { - dev_warn(dd->dev, "Interrupt when no active requests.\n"); - } else { - set_bit(FLAGS_OUTPUT_READY, &dd->flags); - tasklet_schedule(&dd->done_task); - } + set_bit(FLAGS_OUTPUT_READY, &dd->flags); + tasklet_schedule(&dd->done_task); return IRQ_HANDLED; } @@ -2072,7 +2033,6 @@ static ssize_t queue_len_store(struct device *dev, struct omap_sham_dev *dd = dev_get_drvdata(dev); ssize_t status; long value; - unsigned long flags; status = kstrtol(buf, 0, &value); if (status) @@ -2086,9 +2046,7 @@ static ssize_t queue_len_store(struct device *dev, * than current size, it will just not accept new entries until * it has shrank enough. */ - spin_lock_irqsave(&dd->lock, flags); dd->queue.max_qlen = value; - spin_unlock_irqrestore(&dd->lock, flags); return size; } @@ -2125,7 +2083,6 @@ static int omap_sham_probe(struct platform_device *pdev) platform_set_drvdata(pdev, dd); INIT_LIST_HEAD(&dd->list); - spin_lock_init(&dd->lock); tasklet_init(&dd->done_task, omap_sham_done_task, (unsigned long)dd); crypto_init_queue(&dd->queue, OMAP_SHAM_QUEUE_LENGTH); @@ -2190,6 +2147,16 @@ static int omap_sham_probe(struct platform_device *pdev) list_add_tail(&dd->list, &sham.dev_list); spin_unlock(&sham.lock); + dd->engine = crypto_engine_alloc_init(dev, 1); + if (!dd->engine) { + err = -ENOMEM; + goto err_engine; + } + + err = crypto_engine_start(dd->engine); + if (err) + goto err_engine_start; + for (i = 0; i < dd->pdata->algs_info_size; i++) { if (dd->pdata->algs_info[i].registered) break; @@ -2223,6 +2190,12 @@ err_algs: for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) crypto_unregister_ahash( &dd->pdata->algs_info[i].algs_list[j]); +err_engine_start: + crypto_engine_exit(dd->engine); +err_engine: + spin_lock(&sham.lock); + list_del(&dd->list); + spin_unlock(&sham.lock); err_pm: pm_runtime_disable(dev); if (!dd->polling_mode) diff --git a/drivers/crypto/padlock-aes.c b/drivers/crypto/padlock-aes.c index 62c6fe88b212..1be549a07a21 100644 --- a/drivers/crypto/padlock-aes.c +++ b/drivers/crypto/padlock-aes.c @@ -18,6 +18,7 @@ #include <linux/errno.h> #include <linux/interrupt.h> #include <linux/kernel.h> +#include <linux/mm.h> #include <linux/percpu.h> #include <linux/smp.h> #include <linux/slab.h> diff --git a/drivers/crypto/picoxcell_crypto.c b/drivers/crypto/picoxcell_crypto.c index dac6eb37fff9..fb34bf92861d 100644 --- a/drivers/crypto/picoxcell_crypto.c +++ b/drivers/crypto/picoxcell_crypto.c @@ -1685,11 +1685,6 @@ static int spacc_probe(struct platform_device *pdev) goto err_clk_put; } - ret = device_create_file(&pdev->dev, &dev_attr_stat_irq_thresh); - if (ret) - goto err_clk_disable; - - /* * Use an IRQ threshold of 50% as a default. This seems to be a * reasonable trade off of latency against throughput but can be @@ -1697,6 +1692,10 @@ static int spacc_probe(struct platform_device *pdev) */ engine->stat_irq_thresh = (engine->fifo_sz / 2); + ret = device_create_file(&pdev->dev, &dev_attr_stat_irq_thresh); + if (ret) + goto err_clk_disable; + /* * Configure the interrupts. We only use the STAT_CNT interrupt as we * only submit a new packet for processing when we complete another in diff --git a/drivers/crypto/qat/qat_c3xxx/adf_drv.c b/drivers/crypto/qat/qat_c3xxx/adf_drv.c index 020d099409e5..ed0e8e33fe4b 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_drv.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_drv.c @@ -18,12 +18,9 @@ #include <adf_cfg.h> #include "adf_c3xxx_hw_data.h" -#define ADF_SYSTEM_DEVICE(device_id) \ - {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)} - static const struct pci_device_id adf_pci_tbl[] = { - ADF_SYSTEM_DEVICE(ADF_C3XXX_PCI_DEVICE_ID), - {0,} + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C3XXX), }, + { } }; MODULE_DEVICE_TABLE(pci, adf_pci_tbl); @@ -58,7 +55,7 @@ static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) if (accel_dev->hw_device) { switch (accel_pci_dev->pci_dev->device) { - case ADF_C3XXX_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_C3XXX: adf_clean_hw_data_c3xxx(accel_dev->hw_device); break; default: @@ -83,7 +80,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) int ret; switch (ent->device) { - case ADF_C3XXX_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_C3XXX: break; default: dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device); @@ -143,10 +140,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } /* Create dev top level debugfs entry */ - snprintf(name, sizeof(name), "%s%s_%02x:%02d.%d", - ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name, - pdev->bus->number, PCI_SLOT(pdev->devfn), - PCI_FUNC(pdev->devfn)); + snprintf(name, sizeof(name), "%s%s_%s", ADF_DEVICE_NAME_PREFIX, + hw_data->dev_class->name, pci_name(pdev)); accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); @@ -203,7 +198,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } pci_set_master(pdev); - if (adf_enable_aer(accel_dev, &adf_driver)) { + if (adf_enable_aer(accel_dev)) { dev_err(&pdev->dev, "Failed to enable aer\n"); ret = -EFAULT; goto out_err_free_reg; diff --git a/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c b/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c index 11039fe55f61..456979b136a2 100644 --- a/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c +++ b/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c @@ -18,12 +18,9 @@ #include <adf_cfg.h> #include "adf_c3xxxvf_hw_data.h" -#define ADF_SYSTEM_DEVICE(device_id) \ - {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)} - static const struct pci_device_id adf_pci_tbl[] = { - ADF_SYSTEM_DEVICE(ADF_C3XXXIOV_PCI_DEVICE_ID), - {0,} + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C3XXX_VF), }, + { } }; MODULE_DEVICE_TABLE(pci, adf_pci_tbl); @@ -58,7 +55,7 @@ static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) if (accel_dev->hw_device) { switch (accel_pci_dev->pci_dev->device) { - case ADF_C3XXXIOV_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_C3XXX_VF: adf_clean_hw_data_c3xxxiov(accel_dev->hw_device); break; default: @@ -85,7 +82,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) int ret; switch (ent->device) { - case ADF_C3XXXIOV_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_C3XXX_VF: break; default: dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device); @@ -127,10 +124,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) accel_pci_dev->sku = hw_data->get_sku(hw_data); /* Create dev top level debugfs entry */ - snprintf(name, sizeof(name), "%s%s_%02x:%02d.%d", - ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name, - pdev->bus->number, PCI_SLOT(pdev->devfn), - PCI_FUNC(pdev->devfn)); + snprintf(name, sizeof(name), "%s%s_%s", ADF_DEVICE_NAME_PREFIX, + hw_data->dev_class->name, pci_name(pdev)); accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); diff --git a/drivers/crypto/qat/qat_c62x/adf_drv.c b/drivers/crypto/qat/qat_c62x/adf_drv.c index 4ba9c14383af..d8e7c9c25590 100644 --- a/drivers/crypto/qat/qat_c62x/adf_drv.c +++ b/drivers/crypto/qat/qat_c62x/adf_drv.c @@ -18,12 +18,9 @@ #include <adf_cfg.h> #include "adf_c62x_hw_data.h" -#define ADF_SYSTEM_DEVICE(device_id) \ - {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)} - static const struct pci_device_id adf_pci_tbl[] = { - ADF_SYSTEM_DEVICE(ADF_C62X_PCI_DEVICE_ID), - {0,} + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C62X), }, + { } }; MODULE_DEVICE_TABLE(pci, adf_pci_tbl); @@ -58,7 +55,7 @@ static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) if (accel_dev->hw_device) { switch (accel_pci_dev->pci_dev->device) { - case ADF_C62X_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_C62X: adf_clean_hw_data_c62x(accel_dev->hw_device); break; default: @@ -83,7 +80,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) int ret; switch (ent->device) { - case ADF_C62X_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_C62X: break; default: dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device); @@ -143,10 +140,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } /* Create dev top level debugfs entry */ - snprintf(name, sizeof(name), "%s%s_%02x:%02d.%d", - ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name, - pdev->bus->number, PCI_SLOT(pdev->devfn), - PCI_FUNC(pdev->devfn)); + snprintf(name, sizeof(name), "%s%s_%s", ADF_DEVICE_NAME_PREFIX, + hw_data->dev_class->name, pci_name(pdev)); accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); @@ -203,7 +198,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } pci_set_master(pdev); - if (adf_enable_aer(accel_dev, &adf_driver)) { + if (adf_enable_aer(accel_dev)) { dev_err(&pdev->dev, "Failed to enable aer\n"); ret = -EFAULT; goto out_err_free_reg; diff --git a/drivers/crypto/qat/qat_c62xvf/adf_drv.c b/drivers/crypto/qat/qat_c62xvf/adf_drv.c index b8b021d54bb5..b9810f79eb84 100644 --- a/drivers/crypto/qat/qat_c62xvf/adf_drv.c +++ b/drivers/crypto/qat/qat_c62xvf/adf_drv.c @@ -18,12 +18,9 @@ #include <adf_cfg.h> #include "adf_c62xvf_hw_data.h" -#define ADF_SYSTEM_DEVICE(device_id) \ - {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)} - static const struct pci_device_id adf_pci_tbl[] = { - ADF_SYSTEM_DEVICE(ADF_C62XIOV_PCI_DEVICE_ID), - {0,} + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C62X_VF), }, + { } }; MODULE_DEVICE_TABLE(pci, adf_pci_tbl); @@ -58,7 +55,7 @@ static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) if (accel_dev->hw_device) { switch (accel_pci_dev->pci_dev->device) { - case ADF_C62XIOV_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_C62X_VF: adf_clean_hw_data_c62xiov(accel_dev->hw_device); break; default: @@ -85,7 +82,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) int ret; switch (ent->device) { - case ADF_C62XIOV_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_C62X_VF: break; default: dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device); @@ -127,10 +124,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) accel_pci_dev->sku = hw_data->get_sku(hw_data); /* Create dev top level debugfs entry */ - snprintf(name, sizeof(name), "%s%s_%02x:%02d.%d", - ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name, - pdev->bus->number, PCI_SLOT(pdev->devfn), - PCI_FUNC(pdev->devfn)); + snprintf(name, sizeof(name), "%s%s_%s", ADF_DEVICE_NAME_PREFIX, + hw_data->dev_class->name, pci_name(pdev)); accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index c1db8c26afb6..06952ece53d9 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -15,12 +15,6 @@ #define ADF_C62XVF_DEVICE_NAME "c6xxvf" #define ADF_C3XXX_DEVICE_NAME "c3xxx" #define ADF_C3XXXVF_DEVICE_NAME "c3xxxvf" -#define ADF_DH895XCC_PCI_DEVICE_ID 0x435 -#define ADF_DH895XCCIOV_PCI_DEVICE_ID 0x443 -#define ADF_C62X_PCI_DEVICE_ID 0x37c8 -#define ADF_C62XIOV_PCI_DEVICE_ID 0x37c9 -#define ADF_C3XXX_PCI_DEVICE_ID 0x19e2 -#define ADF_C3XXXIOV_PCI_DEVICE_ID 0x19e3 #define ADF_ERRSOU3 (0x3A000 + 0x0C) #define ADF_ERRSOU5 (0x3A000 + 0xD8) #define ADF_DEVICE_FUSECTL_OFFSET 0x40 diff --git a/drivers/crypto/qat/qat_common/adf_aer.c b/drivers/crypto/qat/qat_common/adf_aer.c index 32102e27e559..d2ae293d0df6 100644 --- a/drivers/crypto/qat/qat_common/adf_aer.c +++ b/drivers/crypto/qat/qat_common/adf_aer.c @@ -175,7 +175,6 @@ static const struct pci_error_handlers adf_err_handler = { /** * adf_enable_aer() - Enable Advance Error Reporting for acceleration device * @accel_dev: Pointer to acceleration device. - * @adf: PCI device driver owning the given acceleration device. * * Function enables PCI Advance Error Reporting for the * QAT acceleration device accel_dev. @@ -183,11 +182,12 @@ static const struct pci_error_handlers adf_err_handler = { * * Return: 0 on success, error code otherwise. */ -int adf_enable_aer(struct adf_accel_dev *accel_dev, struct pci_driver *adf) +int adf_enable_aer(struct adf_accel_dev *accel_dev) { struct pci_dev *pdev = accel_to_pci_dev(accel_dev); + struct pci_driver *pdrv = pdev->driver; - adf->err_handler = &adf_err_handler; + pdrv->err_handler = &adf_err_handler; pci_enable_pcie_error_reporting(pdev); return 0; } diff --git a/drivers/crypto/qat/qat_common/adf_cfg.c b/drivers/crypto/qat/qat_common/adf_cfg.c index ac462796cefc..22ae32838113 100644 --- a/drivers/crypto/qat/qat_common/adf_cfg.c +++ b/drivers/crypto/qat/qat_common/adf_cfg.c @@ -52,24 +52,7 @@ static const struct seq_operations qat_dev_cfg_sops = { .show = qat_dev_cfg_show }; -static int qat_dev_cfg_open(struct inode *inode, struct file *file) -{ - int ret = seq_open(file, &qat_dev_cfg_sops); - - if (!ret) { - struct seq_file *seq_f = file->private_data; - - seq_f->private = inode->i_private; - } - return ret; -} - -static const struct file_operations qat_dev_cfg_fops = { - .open = qat_dev_cfg_open, - .read = seq_read, - .llseek = seq_lseek, - .release = seq_release -}; +DEFINE_SEQ_ATTRIBUTE(qat_dev_cfg); /** * adf_cfg_dev_add() - Create an acceleration device configuration table. diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h index ebfcb4ea618d..f22342f612c1 100644 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -95,7 +95,7 @@ void adf_ae_fw_release(struct adf_accel_dev *accel_dev); int adf_ae_start(struct adf_accel_dev *accel_dev); int adf_ae_stop(struct adf_accel_dev *accel_dev); -int adf_enable_aer(struct adf_accel_dev *accel_dev, struct pci_driver *adf); +int adf_enable_aer(struct adf_accel_dev *accel_dev); void adf_disable_aer(struct adf_accel_dev *accel_dev); void adf_reset_sbr(struct adf_accel_dev *accel_dev); void adf_reset_flr(struct adf_accel_dev *accel_dev); diff --git a/drivers/crypto/qat/qat_common/adf_ctl_drv.c b/drivers/crypto/qat/qat_common/adf_ctl_drv.c index 71d0c44aacca..eb9b3be9d8eb 100644 --- a/drivers/crypto/qat/qat_common/adf_ctl_drv.c +++ b/drivers/crypto/qat/qat_common/adf_ctl_drv.c @@ -416,8 +416,6 @@ static long adf_ctl_ioctl(struct file *fp, unsigned int cmd, unsigned long arg) static int __init adf_register_ctl_device_driver(void) { - mutex_init(&adf_ctl_lock); - if (adf_chr_drv_create()) goto err_chr_dev; diff --git a/drivers/crypto/qat/qat_common/adf_dev_mgr.c b/drivers/crypto/qat/qat_common/adf_dev_mgr.c index 72753af056b3..92ec035576df 100644 --- a/drivers/crypto/qat/qat_common/adf_dev_mgr.c +++ b/drivers/crypto/qat/qat_common/adf_dev_mgr.c @@ -285,7 +285,7 @@ struct adf_accel_dev *adf_devmgr_get_first(void) /** * adf_devmgr_pci_to_accel_dev() - Get accel_dev associated with the pci_dev. - * @accel_dev: Pointer to pci device. + * @pci_dev: Pointer to pci device. * * Function returns acceleration device associated with the given pci device. * To be used by QAT device specific drivers. diff --git a/drivers/crypto/qat/qat_common/adf_sriov.c b/drivers/crypto/qat/qat_common/adf_sriov.c index 8827aa139f96..963b2bea78f2 100644 --- a/drivers/crypto/qat/qat_common/adf_sriov.c +++ b/drivers/crypto/qat/qat_common/adf_sriov.c @@ -173,10 +173,14 @@ EXPORT_SYMBOL_GPL(adf_disable_sriov); /** * adf_sriov_configure() - Enable SRIOV for the device * @pdev: Pointer to pci device. + * @numvfs: Number of virtual functions (VFs) to enable. + * + * Note that the @numvfs parameter is ignored and all VFs supported by the + * device are enabled due to the design of the hardware. * * Function enables SRIOV for the pci device. * - * Return: 0 on success, error code otherwise. + * Return: number of VFs enabled on success, error code otherwise. */ int adf_sriov_configure(struct pci_dev *pdev, int numvfs) { diff --git a/drivers/crypto/qat/qat_common/adf_transport_debug.c b/drivers/crypto/qat/qat_common/adf_transport_debug.c index 2a2eccbf56ec..dac25ba47260 100644 --- a/drivers/crypto/qat/qat_common/adf_transport_debug.c +++ b/drivers/crypto/qat/qat_common/adf_transport_debug.c @@ -77,31 +77,14 @@ static void adf_ring_stop(struct seq_file *sfile, void *v) mutex_unlock(&ring_read_lock); } -static const struct seq_operations adf_ring_sops = { +static const struct seq_operations adf_ring_debug_sops = { .start = adf_ring_start, .next = adf_ring_next, .stop = adf_ring_stop, .show = adf_ring_show }; -static int adf_ring_open(struct inode *inode, struct file *file) -{ - int ret = seq_open(file, &adf_ring_sops); - - if (!ret) { - struct seq_file *seq_f = file->private_data; - - seq_f->private = inode->i_private; - } - return ret; -} - -static const struct file_operations adf_ring_debug_fops = { - .open = adf_ring_open, - .read = seq_read, - .llseek = seq_lseek, - .release = seq_release -}; +DEFINE_SEQ_ATTRIBUTE(adf_ring_debug); int adf_ring_debugfs_add(struct adf_etr_ring_data *ring, const char *name) { @@ -188,31 +171,14 @@ static void adf_bank_stop(struct seq_file *sfile, void *v) mutex_unlock(&bank_read_lock); } -static const struct seq_operations adf_bank_sops = { +static const struct seq_operations adf_bank_debug_sops = { .start = adf_bank_start, .next = adf_bank_next, .stop = adf_bank_stop, .show = adf_bank_show }; -static int adf_bank_open(struct inode *inode, struct file *file) -{ - int ret = seq_open(file, &adf_bank_sops); - - if (!ret) { - struct seq_file *seq_f = file->private_data; - - seq_f->private = inode->i_private; - } - return ret; -} - -static const struct file_operations adf_bank_debug_fops = { - .open = adf_bank_open, - .read = seq_read, - .llseek = seq_lseek, - .release = seq_release -}; +DEFINE_SEQ_ATTRIBUTE(adf_bank_debug); int adf_bank_debugfs_add(struct adf_etr_bank_data *bank) { diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index 72753b84dc95..d552dbcfe0a0 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -828,6 +828,11 @@ static int qat_alg_aead_dec(struct aead_request *areq) struct icp_qat_fw_la_bulk_req *msg; int digst_size = crypto_aead_authsize(aead_tfm); int ret, ctr = 0; + u32 cipher_len; + + cipher_len = areq->cryptlen - digst_size; + if (cipher_len % AES_BLOCK_SIZE != 0) + return -EINVAL; ret = qat_alg_sgl_to_bufl(ctx->inst, areq->src, areq->dst, qat_req); if (unlikely(ret)) @@ -842,7 +847,7 @@ static int qat_alg_aead_dec(struct aead_request *areq) qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp; qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp; cipher_param = (void *)&qat_req->req.serv_specif_rqpars; - cipher_param->cipher_length = areq->cryptlen - digst_size; + cipher_param->cipher_length = cipher_len; cipher_param->cipher_offset = areq->assoclen; memcpy(cipher_param->u.cipher_IV_array, areq->iv, AES_BLOCK_SIZE); auth_param = (void *)((u8 *)cipher_param + sizeof(*cipher_param)); @@ -871,6 +876,9 @@ static int qat_alg_aead_enc(struct aead_request *areq) u8 *iv = areq->iv; int ret, ctr = 0; + if (areq->cryptlen % AES_BLOCK_SIZE != 0) + return -EINVAL; + ret = qat_alg_sgl_to_bufl(ctx->inst, areq->src, areq->dst, qat_req); if (unlikely(ret)) return ret; diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index fa467e0f8285..6b9d47682d04 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -2,6 +2,7 @@ /* Copyright(c) 2014 - 2020 Intel Corporation */ #include <linux/slab.h> #include <linux/delay.h> +#include <linux/pci_ids.h> #include "adf_accel_devices.h" #include "adf_common_drv.h" @@ -412,7 +413,7 @@ static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle) unsigned int csr_val; int times = 30; - if (handle->pci_dev->device != ADF_DH895XCC_PCI_DEVICE_ID) + if (handle->pci_dev->device != PCI_DEVICE_ID_INTEL_QAT_DH895XCC) return 0; csr_val = ADF_CSR_RD(csr_addr, 0); @@ -672,13 +673,13 @@ int qat_hal_init(struct adf_accel_dev *accel_dev) (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v + LOCAL_TO_XFER_REG_OFFSET); handle->pci_dev = pci_info->pci_dev; - if (handle->pci_dev->device == ADF_DH895XCC_PCI_DEVICE_ID) { + if (handle->pci_dev->device == PCI_DEVICE_ID_INTEL_QAT_DH895XCC) { sram_bar = &pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)]; handle->hal_sram_addr_v = sram_bar->virt_addr; } handle->fw_auth = (handle->pci_dev->device == - ADF_DH895XCC_PCI_DEVICE_ID) ? false : true; + PCI_DEVICE_ID_INTEL_QAT_DH895XCC) ? false : true; handle->hal_handle = kzalloc(sizeof(*handle->hal_handle), GFP_KERNEL); if (!handle->hal_handle) goto out_hal_handle; diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 00c615f9f9a8..5d1f28cd6680 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -4,6 +4,7 @@ #include <linux/ctype.h> #include <linux/kernel.h> #include <linux/delay.h> +#include <linux/pci_ids.h> #include "adf_accel_devices.h" #include "adf_common_drv.h" #include "icp_qat_uclo.h" @@ -711,11 +712,11 @@ static unsigned int qat_uclo_get_dev_type(struct icp_qat_fw_loader_handle *handle) { switch (handle->pci_dev->device) { - case ADF_DH895XCC_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: return ICP_QAT_AC_895XCC_DEV_TYPE; - case ADF_C62X_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_C62X: return ICP_QAT_AC_C62X_DEV_TYPE; - case ADF_C3XXX_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_C3XXX: return ICP_QAT_AC_C3XXX_DEV_TYPE; default: pr_err("QAT: unsupported device 0x%x\n", @@ -1391,7 +1392,7 @@ int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, status = qat_uclo_auth_fw(handle, desc); qat_uclo_ummap_auth_fw(handle, &desc); } else { - if (handle->pci_dev->device == ADF_C3XXX_PCI_DEVICE_ID) { + if (handle->pci_dev->device == PCI_DEVICE_ID_INTEL_QAT_C3XXX) { pr_err("QAT: C3XXX doesn't support unsigned MMP\n"); return -EINVAL; } diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c index 4e877b75822b..ecb4f6f20e22 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c @@ -18,12 +18,9 @@ #include <adf_cfg.h> #include "adf_dh895xcc_hw_data.h" -#define ADF_SYSTEM_DEVICE(device_id) \ - {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)} - static const struct pci_device_id adf_pci_tbl[] = { - ADF_SYSTEM_DEVICE(ADF_DH895XCC_PCI_DEVICE_ID), - {0,} + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_DH895XCC), }, + { } }; MODULE_DEVICE_TABLE(pci, adf_pci_tbl); @@ -58,7 +55,7 @@ static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) if (accel_dev->hw_device) { switch (accel_pci_dev->pci_dev->device) { - case ADF_DH895XCC_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: adf_clean_hw_data_dh895xcc(accel_dev->hw_device); break; default: @@ -83,7 +80,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) int ret; switch (ent->device) { - case ADF_DH895XCC_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: break; default: dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device); @@ -143,10 +140,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } /* Create dev top level debugfs entry */ - snprintf(name, sizeof(name), "%s%s_%02x:%02d.%d", - ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name, - pdev->bus->number, PCI_SLOT(pdev->devfn), - PCI_FUNC(pdev->devfn)); + snprintf(name, sizeof(name), "%s%s_%s", ADF_DEVICE_NAME_PREFIX, + hw_data->dev_class->name, pci_name(pdev)); accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); @@ -205,7 +200,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } pci_set_master(pdev); - if (adf_enable_aer(accel_dev, &adf_driver)) { + if (adf_enable_aer(accel_dev)) { dev_err(&pdev->dev, "Failed to enable aer\n"); ret = -EFAULT; goto out_err_free_reg; diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c b/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c index 7d6e1db272c2..404cf9df6922 100644 --- a/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c +++ b/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c @@ -18,12 +18,9 @@ #include <adf_cfg.h> #include "adf_dh895xccvf_hw_data.h" -#define ADF_SYSTEM_DEVICE(device_id) \ - {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)} - static const struct pci_device_id adf_pci_tbl[] = { - ADF_SYSTEM_DEVICE(ADF_DH895XCCIOV_PCI_DEVICE_ID), - {0,} + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_DH895XCC_VF), }, + { } }; MODULE_DEVICE_TABLE(pci, adf_pci_tbl); @@ -58,7 +55,7 @@ static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) if (accel_dev->hw_device) { switch (accel_pci_dev->pci_dev->device) { - case ADF_DH895XCCIOV_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_DH895XCC_VF: adf_clean_hw_data_dh895xcciov(accel_dev->hw_device); break; default: @@ -85,7 +82,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) int ret; switch (ent->device) { - case ADF_DH895XCCIOV_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_DH895XCC_VF: break; default: dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device); @@ -127,10 +124,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) accel_pci_dev->sku = hw_data->get_sku(hw_data); /* Create dev top level debugfs entry */ - snprintf(name, sizeof(name), "%s%s_%02x:%02d.%d", - ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name, - pdev->bus->number, PCI_SLOT(pdev->devfn), - PCI_FUNC(pdev->devfn)); + snprintf(name, sizeof(name), "%s%s_%s", ADF_DEVICE_NAME_PREFIX, + hw_data->dev_class->name, pci_name(pdev)); accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); diff --git a/drivers/crypto/qce/core.c b/drivers/crypto/qce/core.c index cb6d61eb7302..ea616b7259ae 100644 --- a/drivers/crypto/qce/core.c +++ b/drivers/crypto/qce/core.c @@ -4,6 +4,7 @@ */ #include <linux/clk.h> +#include <linux/dma-mapping.h> #include <linux/interrupt.h> #include <linux/module.h> #include <linux/mod_devicetable.h> diff --git a/drivers/crypto/qce/sha.c b/drivers/crypto/qce/sha.c index c230843e2ffb..87be96a0b0bb 100644 --- a/drivers/crypto/qce/sha.c +++ b/drivers/crypto/qce/sha.c @@ -4,6 +4,7 @@ */ #include <linux/device.h> +#include <linux/dma-mapping.h> #include <linux/interrupt.h> #include <crypto/internal/hash.h> diff --git a/drivers/crypto/qce/skcipher.c b/drivers/crypto/qce/skcipher.c index 5630c5addd28..a2d3da0ad95f 100644 --- a/drivers/crypto/qce/skcipher.c +++ b/drivers/crypto/qce/skcipher.c @@ -4,6 +4,7 @@ */ #include <linux/device.h> +#include <linux/dma-mapping.h> #include <linux/interrupt.h> #include <linux/moduleparam.h> #include <linux/types.h> diff --git a/drivers/crypto/qcom-rng.c b/drivers/crypto/qcom-rng.c index 4730f84b646d..99ba8d51d102 100644 --- a/drivers/crypto/qcom-rng.c +++ b/drivers/crypto/qcom-rng.c @@ -7,6 +7,7 @@ #include <linux/acpi.h> #include <linux/clk.h> #include <linux/crypto.h> +#include <linux/io.h> #include <linux/module.h> #include <linux/of.h> #include <linux/platform_device.h> diff --git a/drivers/crypto/rockchip/rk3288_crypto.c b/drivers/crypto/rockchip/rk3288_crypto.c index f385587f99af..35d73061d156 100644 --- a/drivers/crypto/rockchip/rk3288_crypto.c +++ b/drivers/crypto/rockchip/rk3288_crypto.c @@ -10,6 +10,7 @@ */ #include "rk3288_crypto.h" +#include <linux/dma-mapping.h> #include <linux/module.h> #include <linux/platform_device.h> #include <linux/of.h> diff --git a/drivers/crypto/rockchip/rk3288_crypto.h b/drivers/crypto/rockchip/rk3288_crypto.h index 2b49c677afdb..3db595570c9c 100644 --- a/drivers/crypto/rockchip/rk3288_crypto.h +++ b/drivers/crypto/rockchip/rk3288_crypto.h @@ -7,6 +7,7 @@ #include <crypto/algapi.h> #include <linux/interrupt.h> #include <linux/delay.h> +#include <linux/scatterlist.h> #include <crypto/internal/hash.h> #include <crypto/internal/skcipher.h> diff --git a/drivers/crypto/rockchip/rk3288_crypto_ahash.c b/drivers/crypto/rockchip/rk3288_crypto_ahash.c index 6b7ecbec092e..81befe7febaa 100644 --- a/drivers/crypto/rockchip/rk3288_crypto_ahash.c +++ b/drivers/crypto/rockchip/rk3288_crypto_ahash.c @@ -8,6 +8,7 @@ * * Some ideas are from marvell/cesa.c and s5p-sss.c driver. */ +#include <linux/device.h> #include "rk3288_crypto.h" /* diff --git a/drivers/crypto/rockchip/rk3288_crypto_skcipher.c b/drivers/crypto/rockchip/rk3288_crypto_skcipher.c index 4a75c8e1fa6c..1cece1a7d3f0 100644 --- a/drivers/crypto/rockchip/rk3288_crypto_skcipher.c +++ b/drivers/crypto/rockchip/rk3288_crypto_skcipher.c @@ -8,6 +8,7 @@ * * Some ideas are from marvell-cesa.c and s5p-sss.c driver. */ +#include <linux/device.h> #include "rk3288_crypto.h" #define RK_CRYPTO_DEC BIT(0) diff --git a/drivers/crypto/s5p-sss.c b/drivers/crypto/s5p-sss.c index 341433fbcc4a..88a6c853ffd7 100644 --- a/drivers/crypto/s5p-sss.c +++ b/drivers/crypto/s5p-sss.c @@ -260,6 +260,7 @@ struct s5p_aes_ctx { * struct s5p_aes_dev - Crypto device state container * @dev: Associated device * @clk: Clock for accessing hardware + * @pclk: APB bus clock necessary to access the hardware * @ioaddr: Mapped IO memory region * @aes_ioaddr: Per-varian offset for AES block IO memory * @irq_fc: Feed control interrupt line @@ -342,13 +343,13 @@ struct s5p_aes_dev { * @engine: Bits for selecting type of HASH in SSS block * @sg: sg for DMA transfer * @sg_len: Length of sg for DMA transfer - * @sgl[]: sg for joining buffer and req->src scatterlist + * @sgl: sg for joining buffer and req->src scatterlist * @skip: Skip offset in req->src for current op * @total: Total number of bytes for current request * @finup: Keep state for finup or final. * @error: Keep track of error. * @bufcnt: Number of bytes holded in buffer[] - * @buffer[]: For byte(s) from end of req->src in UPDATE op + * @buffer: For byte(s) from end of req->src in UPDATE op */ struct s5p_hash_reqctx { struct s5p_aes_dev *dd; @@ -1125,7 +1126,7 @@ static int s5p_hash_copy_sg_lists(struct s5p_hash_reqctx *ctx, * s5p_hash_prepare_sgs() - prepare sg for processing * @ctx: request context * @sg: source scatterlist request - * @nbytes: number of bytes to process from sg + * @new_len: number of bytes to process from sg * @final: final flag * * Check two conditions: (1) if buffers in sg have len aligned data, and (2) @@ -2200,11 +2201,10 @@ static int s5p_aes_probe(struct platform_device *pdev) } pdata->clk = devm_clk_get(dev, variant->clk_names[0]); - if (IS_ERR(pdata->clk)) { - dev_err(dev, "failed to find secss clock %s\n", - variant->clk_names[0]); - return -ENOENT; - } + if (IS_ERR(pdata->clk)) + return dev_err_probe(dev, PTR_ERR(pdata->clk), + "failed to find secss clock %s\n", + variant->clk_names[0]); err = clk_prepare_enable(pdata->clk); if (err < 0) { @@ -2216,9 +2216,9 @@ static int s5p_aes_probe(struct platform_device *pdev) if (variant->clk_names[1]) { pdata->pclk = devm_clk_get(dev, variant->clk_names[1]); if (IS_ERR(pdata->pclk)) { - dev_err(dev, "failed to find clock %s\n", - variant->clk_names[1]); - err = -ENOENT; + err = dev_err_probe(dev, PTR_ERR(pdata->pclk), + "failed to find clock %s\n", + variant->clk_names[1]); goto err_clk; } @@ -2307,8 +2307,7 @@ err_algs: tasklet_kill(&pdata->tasklet); err_irq: - if (pdata->pclk) - clk_disable_unprepare(pdata->pclk); + clk_disable_unprepare(pdata->pclk); err_clk: clk_disable_unprepare(pdata->clk); @@ -2338,8 +2337,7 @@ static int s5p_aes_remove(struct platform_device *pdev) pdata->use_hash = false; } - if (pdata->pclk) - clk_disable_unprepare(pdata->pclk); + clk_disable_unprepare(pdata->pclk); clk_disable_unprepare(pdata->clk); s5p_dev = NULL; diff --git a/drivers/crypto/sa2ul.c b/drivers/crypto/sa2ul.c index 5bc099052bd2..eda93fab95fe 100644 --- a/drivers/crypto/sa2ul.c +++ b/drivers/crypto/sa2ul.c @@ -143,33 +143,38 @@ struct sa_alg_tmpl { }; /** + * struct sa_mapped_sg: scatterlist information for tx and rx + * @mapped: Set to true if the @sgt is mapped + * @dir: mapping direction used for @sgt + * @split_sg: Set if the sg is split and needs to be freed up + * @static_sg: Static scatterlist entry for overriding data + * @sgt: scatterlist table for DMA API use + */ +struct sa_mapped_sg { + bool mapped; + enum dma_data_direction dir; + struct scatterlist static_sg; + struct scatterlist *split_sg; + struct sg_table sgt; +}; +/** * struct sa_rx_data: RX Packet miscellaneous data place holder * @req: crypto request data pointer * @ddev: pointer to the DMA device * @tx_in: dma_async_tx_descriptor pointer for rx channel - * @split_src_sg: Set if the src sg is split and needs to be freed up - * @split_dst_sg: Set if the dst sg is split and needs to be freed up + * @mapped_sg: Information on tx (0) and rx (1) scatterlist DMA mapping * @enc: Flag indicating either encryption or decryption * @enc_iv_size: Initialisation vector size * @iv_idx: Initialisation vector index - * @rx_sg: Static scatterlist entry for overriding RX data - * @tx_sg: Static scatterlist entry for overriding TX data - * @src: Source data pointer - * @dst: Destination data pointer */ struct sa_rx_data { void *req; struct device *ddev; struct dma_async_tx_descriptor *tx_in; - struct scatterlist *split_src_sg; - struct scatterlist *split_dst_sg; + struct sa_mapped_sg mapped_sg[2]; u8 enc; u8 enc_iv_size; u8 iv_idx; - struct scatterlist rx_sg; - struct scatterlist tx_sg; - struct scatterlist *src; - struct scatterlist *dst; }; /** @@ -976,23 +981,46 @@ static int sa_3des_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key, return sa_cipher_setkey(tfm, key, keylen, &ad); } +static void sa_sync_from_device(struct sa_rx_data *rxd) +{ + struct sg_table *sgt; + + if (rxd->mapped_sg[0].dir == DMA_BIDIRECTIONAL) + sgt = &rxd->mapped_sg[0].sgt; + else + sgt = &rxd->mapped_sg[1].sgt; + + dma_sync_sgtable_for_cpu(rxd->ddev, sgt, DMA_FROM_DEVICE); +} + +static void sa_free_sa_rx_data(struct sa_rx_data *rxd) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(rxd->mapped_sg); i++) { + struct sa_mapped_sg *mapped_sg = &rxd->mapped_sg[i]; + + if (mapped_sg->mapped) { + dma_unmap_sgtable(rxd->ddev, &mapped_sg->sgt, + mapped_sg->dir, 0); + kfree(mapped_sg->split_sg); + } + } + + kfree(rxd); +} + static void sa_aes_dma_in_callback(void *data) { struct sa_rx_data *rxd = (struct sa_rx_data *)data; struct skcipher_request *req; - int sglen; u32 *result; __be32 *mdptr; size_t ml, pl; int i; - enum dma_data_direction dir_src; - bool diff_dst; + sa_sync_from_device(rxd); req = container_of(rxd->req, struct skcipher_request, base); - sglen = sg_nents_for_len(req->src, req->cryptlen); - - diff_dst = (req->src != req->dst) ? true : false; - dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL; if (req->iv) { mdptr = (__be32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, @@ -1003,18 +1031,7 @@ static void sa_aes_dma_in_callback(void *data) result[i] = be32_to_cpu(mdptr[i + rxd->iv_idx]); } - dma_unmap_sg(rxd->ddev, req->src, sglen, dir_src); - kfree(rxd->split_src_sg); - - if (diff_dst) { - sglen = sg_nents_for_len(req->dst, req->cryptlen); - - dma_unmap_sg(rxd->ddev, req->dst, sglen, - DMA_FROM_DEVICE); - kfree(rxd->split_dst_sg); - } - - kfree(rxd); + sa_free_sa_rx_data(rxd); skcipher_request_complete(req, 0); } @@ -1043,7 +1060,6 @@ static int sa_run(struct sa_req *req) struct device *ddev; struct dma_chan *dma_rx; int sg_nents, src_nents, dst_nents; - int mapped_src_nents, mapped_dst_nents; struct scatterlist *src, *dst; size_t pl, ml, split_size; struct sa_ctx_info *sa_ctx = req->enc ? &req->ctx->enc : &req->ctx->dec; @@ -1052,6 +1068,7 @@ static int sa_run(struct sa_req *req) u32 *mdptr; bool diff_dst; enum dma_data_direction dir_src; + struct sa_mapped_sg *mapped_sg; gfp_flags = req->base->flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; @@ -1082,6 +1099,7 @@ static int sa_run(struct sa_req *req) dma_rx = pdata->dma_rx1; ddev = dma_rx->device->dev; + rxd->ddev = ddev; memcpy(cmdl, sa_ctx->cmdl, sa_ctx->cmdl_size); @@ -1109,60 +1127,90 @@ static int sa_run(struct sa_req *req) split_size = req->size; + mapped_sg = &rxd->mapped_sg[0]; if (sg_nents == 1 && split_size <= req->src->length) { - src = &rxd->rx_sg; + src = &mapped_sg->static_sg; + src_nents = 1; sg_init_table(src, 1); sg_set_page(src, sg_page(req->src), split_size, req->src->offset); - src_nents = 1; - dma_map_sg(ddev, src, sg_nents, dir_src); + + mapped_sg->sgt.sgl = src; + mapped_sg->sgt.orig_nents = src_nents; + ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0); + if (ret) + return ret; + + mapped_sg->dir = dir_src; + mapped_sg->mapped = true; } else { - mapped_src_nents = dma_map_sg(ddev, req->src, sg_nents, - dir_src); - ret = sg_split(req->src, mapped_src_nents, 0, 1, &split_size, - &src, &src_nents, gfp_flags); + mapped_sg->sgt.sgl = req->src; + mapped_sg->sgt.orig_nents = sg_nents; + ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0); + if (ret) + return ret; + + mapped_sg->dir = dir_src; + mapped_sg->mapped = true; + + ret = sg_split(mapped_sg->sgt.sgl, mapped_sg->sgt.nents, 0, 1, + &split_size, &src, &src_nents, gfp_flags); if (ret) { - src_nents = sg_nents; - src = req->src; + src_nents = mapped_sg->sgt.nents; + src = mapped_sg->sgt.sgl; } else { - rxd->split_src_sg = src; + mapped_sg->split_sg = src; } } + dma_sync_sgtable_for_device(ddev, &mapped_sg->sgt, DMA_TO_DEVICE); + if (!diff_dst) { dst_nents = src_nents; dst = src; } else { dst_nents = sg_nents_for_len(req->dst, req->size); + mapped_sg = &rxd->mapped_sg[1]; if (dst_nents == 1 && split_size <= req->dst->length) { - dst = &rxd->tx_sg; + dst = &mapped_sg->static_sg; + dst_nents = 1; sg_init_table(dst, 1); sg_set_page(dst, sg_page(req->dst), split_size, req->dst->offset); - dst_nents = 1; - dma_map_sg(ddev, dst, dst_nents, DMA_FROM_DEVICE); + + mapped_sg->sgt.sgl = dst; + mapped_sg->sgt.orig_nents = dst_nents; + ret = dma_map_sgtable(ddev, &mapped_sg->sgt, + DMA_FROM_DEVICE, 0); + if (ret) + goto err_cleanup; + + mapped_sg->dir = DMA_FROM_DEVICE; + mapped_sg->mapped = true; } else { - mapped_dst_nents = dma_map_sg(ddev, req->dst, dst_nents, - DMA_FROM_DEVICE); - ret = sg_split(req->dst, mapped_dst_nents, 0, 1, - &split_size, &dst, &dst_nents, + mapped_sg->sgt.sgl = req->dst; + mapped_sg->sgt.orig_nents = dst_nents; + ret = dma_map_sgtable(ddev, &mapped_sg->sgt, + DMA_FROM_DEVICE, 0); + if (ret) + goto err_cleanup; + + mapped_sg->dir = DMA_FROM_DEVICE; + mapped_sg->mapped = true; + + ret = sg_split(mapped_sg->sgt.sgl, mapped_sg->sgt.nents, + 0, 1, &split_size, &dst, &dst_nents, gfp_flags); if (ret) { - dst_nents = dst_nents; - dst = req->dst; + dst_nents = mapped_sg->sgt.nents; + dst = mapped_sg->sgt.sgl; } else { - rxd->split_dst_sg = dst; + mapped_sg->split_sg = dst; } } } - if (unlikely(src_nents != sg_nents)) { - dev_warn_ratelimited(sa_k3_dev, "failed to map tx pkt\n"); - ret = -EIO; - goto err_cleanup; - } - rxd->tx_in = dmaengine_prep_slave_sg(dma_rx, dst, dst_nents, DMA_DEV_TO_MEM, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); @@ -1174,9 +1222,6 @@ static int sa_run(struct sa_req *req) rxd->req = (void *)req->base; rxd->enc = req->enc; - rxd->ddev = ddev; - rxd->src = src; - rxd->dst = dst; rxd->iv_idx = req->ctx->iv_idx; rxd->enc_iv_size = sa_ctx->cmdl_upd_info.enc_iv.size; rxd->tx_in->callback = req->callback; @@ -1214,16 +1259,7 @@ static int sa_run(struct sa_req *req) return -EINPROGRESS; err_cleanup: - dma_unmap_sg(ddev, req->src, sg_nents, DMA_TO_DEVICE); - kfree(rxd->split_src_sg); - - if (req->src != req->dst) { - dst_nents = sg_nents_for_len(req->dst, req->size); - dma_unmap_sg(ddev, req->dst, dst_nents, DMA_FROM_DEVICE); - kfree(rxd->split_dst_sg); - } - - kfree(rxd); + sa_free_sa_rx_data(rxd); return ret; } @@ -1293,11 +1329,12 @@ static void sa_sha_dma_in_callback(void *data) struct ahash_request *req; struct crypto_ahash *tfm; unsigned int authsize; - int i, sg_nents; + int i; size_t ml, pl; u32 *result; __be32 *mdptr; + sa_sync_from_device(rxd); req = container_of(rxd->req, struct ahash_request, base); tfm = crypto_ahash_reqtfm(req); authsize = crypto_ahash_digestsize(tfm); @@ -1308,12 +1345,7 @@ static void sa_sha_dma_in_callback(void *data) for (i = 0; i < (authsize / 4); i++) result[i] = be32_to_cpu(mdptr[i + 4]); - sg_nents = sg_nents_for_len(req->src, req->nbytes); - dma_unmap_sg(rxd->ddev, req->src, sg_nents, DMA_FROM_DEVICE); - - kfree(rxd->split_src_sg); - - kfree(rxd); + sa_free_sa_rx_data(rxd); ahash_request_complete(req, 0); } @@ -1482,8 +1514,8 @@ static int sa_sha_init(struct ahash_request *req) struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); - dev_dbg(sa_k3_dev, "init: digest size: %d, rctx=%llx\n", - crypto_ahash_digestsize(tfm), (u64)rctx); + dev_dbg(sa_k3_dev, "init: digest size: %u, rctx=%p\n", + crypto_ahash_digestsize(tfm), rctx); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); rctx->fallback_req.base.flags = @@ -1637,43 +1669,28 @@ static void sa_aead_dma_in_callback(void *data) unsigned int authsize; u8 auth_tag[SA_MAX_AUTH_TAG_SZ]; size_t pl, ml; - int i, sglen; + int i; int err = 0; u16 auth_len; u32 *mdptr; - bool diff_dst; - enum dma_data_direction dir_src; + sa_sync_from_device(rxd); req = container_of(rxd->req, struct aead_request, base); tfm = crypto_aead_reqtfm(req); start = req->assoclen + req->cryptlen; authsize = crypto_aead_authsize(tfm); - diff_dst = (req->src != req->dst) ? true : false; - dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL; - mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml); for (i = 0; i < (authsize / 4); i++) mdptr[i + 4] = swab32(mdptr[i + 4]); auth_len = req->assoclen + req->cryptlen; - if (!rxd->enc) - auth_len -= authsize; - - sglen = sg_nents_for_len(rxd->src, auth_len); - dma_unmap_sg(rxd->ddev, rxd->src, sglen, dir_src); - kfree(rxd->split_src_sg); - - if (diff_dst) { - sglen = sg_nents_for_len(rxd->dst, auth_len); - dma_unmap_sg(rxd->ddev, rxd->dst, sglen, DMA_FROM_DEVICE); - kfree(rxd->split_dst_sg); - } if (rxd->enc) { scatterwalk_map_and_copy(&mdptr[4], req->dst, start, authsize, 1); } else { + auth_len -= authsize; start -= authsize; scatterwalk_map_and_copy(auth_tag, req->src, start, authsize, 0); @@ -1681,7 +1698,7 @@ static void sa_aead_dma_in_callback(void *data) err = memcmp(&mdptr[4], auth_tag, authsize) ? -EBADMSG : 0; } - kfree(rxd); + sa_free_sa_rx_data(rxd); aead_request_complete(req, err); } @@ -2243,25 +2260,21 @@ static int sa_dma_init(struct sa_crypto_data *dd) return ret; dd->dma_rx1 = dma_request_chan(dd->dev, "rx1"); - if (IS_ERR(dd->dma_rx1)) { - if (PTR_ERR(dd->dma_rx1) != -EPROBE_DEFER) - dev_err(dd->dev, "Unable to request rx1 DMA channel\n"); - return PTR_ERR(dd->dma_rx1); - } + if (IS_ERR(dd->dma_rx1)) + return dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx1), + "Unable to request rx1 DMA channel\n"); dd->dma_rx2 = dma_request_chan(dd->dev, "rx2"); if (IS_ERR(dd->dma_rx2)) { dma_release_channel(dd->dma_rx1); - if (PTR_ERR(dd->dma_rx2) != -EPROBE_DEFER) - dev_err(dd->dev, "Unable to request rx2 DMA channel\n"); - return PTR_ERR(dd->dma_rx2); + return dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx2), + "Unable to request rx2 DMA channel\n"); } dd->dma_tx = dma_request_chan(dd->dev, "tx"); if (IS_ERR(dd->dma_tx)) { - if (PTR_ERR(dd->dma_tx) != -EPROBE_DEFER) - dev_err(dd->dev, "Unable to request tx DMA channel\n"); - ret = PTR_ERR(dd->dma_tx); + ret = dev_err_probe(dd->dev, PTR_ERR(dd->dma_tx), + "Unable to request tx DMA channel\n"); goto err_dma_tx; } @@ -2333,7 +2346,7 @@ static int sa_ul_probe(struct platform_device *pdev) pm_runtime_enable(dev); ret = pm_runtime_get_sync(dev); - if (ret) { + if (ret < 0) { dev_err(&pdev->dev, "%s: failed to get sync: %d\n", __func__, ret); return ret; diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c index 0c8cb23ae708..d60679c79822 100644 --- a/drivers/crypto/sahara.c +++ b/drivers/crypto/sahara.c @@ -18,7 +18,7 @@ #include <crypto/sha.h> #include <linux/clk.h> -#include <linux/crypto.h> +#include <linux/dma-mapping.h> #include <linux/interrupt.h> #include <linux/io.h> #include <linux/irq.h> diff --git a/drivers/crypto/stm32/Kconfig b/drivers/crypto/stm32/Kconfig index 4ef3eb11361c..4a4c3284ae1f 100644 --- a/drivers/crypto/stm32/Kconfig +++ b/drivers/crypto/stm32/Kconfig @@ -3,6 +3,7 @@ config CRYPTO_DEV_STM32_CRC tristate "Support for STM32 crc accelerators" depends on ARCH_STM32 select CRYPTO_HASH + select CRC32 help This enables support for the CRC32 hw accelerator which can be found on STMicroelectronics STM32 SOC. diff --git a/drivers/crypto/stm32/stm32-crc32.c b/drivers/crypto/stm32/stm32-crc32.c index 3ba41148c2a4..75867c0b0017 100644 --- a/drivers/crypto/stm32/stm32-crc32.c +++ b/drivers/crypto/stm32/stm32-crc32.c @@ -6,7 +6,10 @@ #include <linux/bitrev.h> #include <linux/clk.h> +#include <linux/crc32.h> #include <linux/crc32poly.h> +#include <linux/io.h> +#include <linux/kernel.h> #include <linux/module.h> #include <linux/mod_devicetable.h> #include <linux/platform_device.h> @@ -147,7 +150,6 @@ static int burst_update(struct shash_desc *desc, const u8 *d8, struct stm32_crc_desc_ctx *ctx = shash_desc_ctx(desc); struct stm32_crc_ctx *mctx = crypto_shash_ctx(desc->tfm); struct stm32_crc *crc; - unsigned long flags; crc = stm32_crc_get_next_crc(); if (!crc) @@ -155,7 +157,15 @@ static int burst_update(struct shash_desc *desc, const u8 *d8, pm_runtime_get_sync(crc->dev); - spin_lock_irqsave(&crc->lock, flags); + if (!spin_trylock(&crc->lock)) { + /* Hardware is busy, calculate crc32 by software */ + if (mctx->poly == CRC32_POLY_LE) + ctx->partial = crc32_le(ctx->partial, d8, length); + else + ctx->partial = __crc32c_le(ctx->partial, d8, length); + + goto pm_out; + } /* * Restore previously calculated CRC for this context as init value @@ -195,8 +205,9 @@ static int burst_update(struct shash_desc *desc, const u8 *d8, /* Store partial result */ ctx->partial = readl_relaxed(crc->regs + CRC_DR); - spin_unlock_irqrestore(&crc->lock, flags); + spin_unlock(&crc->lock); +pm_out: pm_runtime_mark_last_busy(crc->dev); pm_runtime_put_autosuspend(crc->dev); @@ -216,9 +227,8 @@ static int stm32_crc_update(struct shash_desc *desc, const u8 *d8, return burst_update(desc, d8, length); /* Digest first bytes not 32bit aligned at first pass in the loop */ - size = min(length, - burst_sz + (unsigned int)d8 - ALIGN_DOWN((unsigned int)d8, - sizeof(u32))); + size = min_t(size_t, length, burst_sz + (size_t)d8 - + ALIGN_DOWN((size_t)d8, sizeof(u32))); for (rem_sz = length, cur = d8; rem_sz; rem_sz -= size, cur += size, size = min(rem_sz, burst_sz)) { ret = burst_update(desc, cur, size); diff --git a/drivers/crypto/stm32/stm32-cryp.c b/drivers/crypto/stm32/stm32-cryp.c index d347a1d6e351..2670c30332fa 100644 --- a/drivers/crypto/stm32/stm32-cryp.c +++ b/drivers/crypto/stm32/stm32-cryp.c @@ -118,7 +118,7 @@ struct stm32_cryp_ctx { struct crypto_engine_ctx enginectx; struct stm32_cryp *cryp; int keylen; - u32 key[AES_KEYSIZE_256 / sizeof(u32)]; + __be32 key[AES_KEYSIZE_256 / sizeof(u32)]; unsigned long flags; }; @@ -380,24 +380,24 @@ static int stm32_cryp_copy_sgs(struct stm32_cryp *cryp) return 0; } -static void stm32_cryp_hw_write_iv(struct stm32_cryp *cryp, u32 *iv) +static void stm32_cryp_hw_write_iv(struct stm32_cryp *cryp, __be32 *iv) { if (!iv) return; - stm32_cryp_write(cryp, CRYP_IV0LR, cpu_to_be32(*iv++)); - stm32_cryp_write(cryp, CRYP_IV0RR, cpu_to_be32(*iv++)); + stm32_cryp_write(cryp, CRYP_IV0LR, be32_to_cpu(*iv++)); + stm32_cryp_write(cryp, CRYP_IV0RR, be32_to_cpu(*iv++)); if (is_aes(cryp)) { - stm32_cryp_write(cryp, CRYP_IV1LR, cpu_to_be32(*iv++)); - stm32_cryp_write(cryp, CRYP_IV1RR, cpu_to_be32(*iv++)); + stm32_cryp_write(cryp, CRYP_IV1LR, be32_to_cpu(*iv++)); + stm32_cryp_write(cryp, CRYP_IV1RR, be32_to_cpu(*iv++)); } } static void stm32_cryp_get_iv(struct stm32_cryp *cryp) { struct skcipher_request *req = cryp->req; - u32 *tmp = (void *)req->iv; + __be32 *tmp = (void *)req->iv; if (!tmp) return; @@ -417,13 +417,13 @@ static void stm32_cryp_hw_write_key(struct stm32_cryp *c) int r_id; if (is_des(c)) { - stm32_cryp_write(c, CRYP_K1LR, cpu_to_be32(c->ctx->key[0])); - stm32_cryp_write(c, CRYP_K1RR, cpu_to_be32(c->ctx->key[1])); + stm32_cryp_write(c, CRYP_K1LR, be32_to_cpu(c->ctx->key[0])); + stm32_cryp_write(c, CRYP_K1RR, be32_to_cpu(c->ctx->key[1])); } else { r_id = CRYP_K3RR; for (i = c->ctx->keylen / sizeof(u32); i > 0; i--, r_id -= 4) stm32_cryp_write(c, r_id, - cpu_to_be32(c->ctx->key[i - 1])); + be32_to_cpu(c->ctx->key[i - 1])); } } @@ -469,7 +469,7 @@ static unsigned int stm32_cryp_get_input_text_len(struct stm32_cryp *cryp) static int stm32_cryp_gcm_init(struct stm32_cryp *cryp, u32 cfg) { int ret; - u32 iv[4]; + __be32 iv[4]; /* Phase 1 : init */ memcpy(iv, cryp->areq->iv, 12); @@ -491,6 +491,7 @@ static int stm32_cryp_ccm_init(struct stm32_cryp *cryp, u32 cfg) { int ret; u8 iv[AES_BLOCK_SIZE], b0[AES_BLOCK_SIZE]; + __be32 *bd; u32 *d; unsigned int i, textlen; @@ -498,7 +499,7 @@ static int stm32_cryp_ccm_init(struct stm32_cryp *cryp, u32 cfg) memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE); memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1); iv[AES_BLOCK_SIZE - 1] = 1; - stm32_cryp_hw_write_iv(cryp, (u32 *)iv); + stm32_cryp_hw_write_iv(cryp, (__be32 *)iv); /* Build B0 */ memcpy(b0, iv, AES_BLOCK_SIZE); @@ -518,11 +519,14 @@ static int stm32_cryp_ccm_init(struct stm32_cryp *cryp, u32 cfg) /* Write B0 */ d = (u32 *)b0; + bd = (__be32 *)b0; for (i = 0; i < AES_BLOCK_32; i++) { + u32 xd = d[i]; + if (!cryp->caps->padding_wa) - *d = cpu_to_be32(*d); - stm32_cryp_write(cryp, CRYP_DIN, *d++); + xd = be32_to_cpu(bd[i]); + stm32_cryp_write(cryp, CRYP_DIN, xd); } /* Wait for end of processing */ @@ -617,7 +621,7 @@ static int stm32_cryp_hw_init(struct stm32_cryp *cryp) case CR_TDES_CBC: case CR_AES_CBC: case CR_AES_CTR: - stm32_cryp_hw_write_iv(cryp, (u32 *)cryp->req->iv); + stm32_cryp_hw_write_iv(cryp, (__be32 *)cryp->req->iv); break; default: @@ -1120,7 +1124,7 @@ static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp) /* GCM: write aad and payload size (in bits) */ size_bit = cryp->areq->assoclen * 8; if (cryp->caps->swap_final) - size_bit = cpu_to_be32(size_bit); + size_bit = (__force u32)cpu_to_be32(size_bit); stm32_cryp_write(cryp, CRYP_DIN, 0); stm32_cryp_write(cryp, CRYP_DIN, size_bit); @@ -1129,7 +1133,7 @@ static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp) cryp->areq->cryptlen - AES_BLOCK_SIZE; size_bit *= 8; if (cryp->caps->swap_final) - size_bit = cpu_to_be32(size_bit); + size_bit = (__force u32)cpu_to_be32(size_bit); stm32_cryp_write(cryp, CRYP_DIN, 0); stm32_cryp_write(cryp, CRYP_DIN, size_bit); @@ -1137,14 +1141,19 @@ static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp) /* CCM: write CTR0 */ u8 iv[AES_BLOCK_SIZE]; u32 *iv32 = (u32 *)iv; + __be32 *biv; + + biv = (void *)iv; memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE); memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1); for (i = 0; i < AES_BLOCK_32; i++) { + u32 xiv = iv32[i]; + if (!cryp->caps->padding_wa) - *iv32 = cpu_to_be32(*iv32); - stm32_cryp_write(cryp, CRYP_DIN, *iv32++); + xiv = be32_to_cpu(biv[i]); + stm32_cryp_write(cryp, CRYP_DIN, xiv); } } diff --git a/drivers/crypto/stm32/stm32-hash.c b/drivers/crypto/stm32/stm32-hash.c index 03c5e6683805..e3e25278a970 100644 --- a/drivers/crypto/stm32/stm32-hash.c +++ b/drivers/crypto/stm32/stm32-hash.c @@ -9,6 +9,7 @@ #include <linux/clk.h> #include <linux/crypto.h> #include <linux/delay.h> +#include <linux/dma-mapping.h> #include <linux/dmaengine.h> #include <linux/interrupt.h> #include <linux/io.h> @@ -748,7 +749,7 @@ static int stm32_hash_final_req(struct stm32_hash_dev *hdev) static void stm32_hash_copy_hash(struct ahash_request *req) { struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); - u32 *hash = (u32 *)rctx->digest; + __be32 *hash = (void *)rctx->digest; unsigned int i, hashsize; switch (rctx->flags & HASH_FLAGS_ALGO_MASK) { @@ -769,7 +770,7 @@ static void stm32_hash_copy_hash(struct ahash_request *req) } for (i = 0; i < hashsize / sizeof(u32); i++) - hash[i] = be32_to_cpu(stm32_hash_read(rctx->hdev, + hash[i] = cpu_to_be32(stm32_hash_read(rctx->hdev, HASH_HREG(i))); } @@ -1463,14 +1464,9 @@ static int stm32_hash_probe(struct platform_device *pdev) } hdev->clk = devm_clk_get(&pdev->dev, NULL); - if (IS_ERR(hdev->clk)) { - if (PTR_ERR(hdev->clk) != -EPROBE_DEFER) { - dev_err(dev, "failed to get clock for hash (%lu)\n", - PTR_ERR(hdev->clk)); - } - - return PTR_ERR(hdev->clk); - } + if (IS_ERR(hdev->clk)) + return dev_err_probe(dev, PTR_ERR(hdev->clk), + "failed to get clock for hash\n"); ret = clk_prepare_enable(hdev->clk); if (ret) { diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 7c547352a862..66773892f665 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -806,10 +806,10 @@ static int talitos_register_rng(struct device *dev) struct talitos_private *priv = dev_get_drvdata(dev); int err; - priv->rng.name = dev_driver_string(dev), - priv->rng.init = talitos_rng_init, - priv->rng.data_present = talitos_rng_data_present, - priv->rng.data_read = talitos_rng_data_read, + priv->rng.name = dev_driver_string(dev); + priv->rng.init = talitos_rng_init; + priv->rng.data_present = talitos_rng_data_present; + priv->rng.data_read = talitos_rng_data_read; priv->rng.priv = (unsigned long)dev; err = hwrng_register(&priv->rng); diff --git a/drivers/crypto/ux500/cryp/cryp_core.c b/drivers/crypto/ux500/cryp/cryp_core.c index 800dfc4d16c4..c3adeb2e5823 100644 --- a/drivers/crypto/ux500/cryp/cryp_core.c +++ b/drivers/crypto/ux500/cryp/cryp_core.c @@ -11,13 +11,15 @@ #include <linux/clk.h> #include <linux/completion.h> -#include <linux/crypto.h> +#include <linux/device.h> +#include <linux/dma-mapping.h> #include <linux/dmaengine.h> #include <linux/err.h> #include <linux/errno.h> #include <linux/interrupt.h> #include <linux/io.h> #include <linux/irqreturn.h> +#include <linux/kernel.h> #include <linux/klist.h> #include <linux/module.h> #include <linux/mod_devicetable.h> @@ -27,7 +29,6 @@ #include <linux/platform_data/dma-ste-dma40.h> #include <crypto/aes.h> -#include <crypto/algapi.h> #include <crypto/ctr.h> #include <crypto/internal/des.h> #include <crypto/internal/skcipher.h> @@ -92,17 +93,6 @@ struct cryp_ctx { static struct cryp_driver_data driver_data; /** - * uint8p_to_uint32_be - 4*uint8 to uint32 big endian - * @in: Data to convert. - */ -static inline u32 uint8p_to_uint32_be(u8 *in) -{ - u32 *data = (u32 *)in; - - return cpu_to_be32p(data); -} - -/** * swap_bits_in_byte - mirror the bits in a byte * @b: the byte to be mirrored * @@ -284,6 +274,7 @@ static int cfg_ivs(struct cryp_device_data *device_data, struct cryp_ctx *ctx) int i; int status = 0; int num_of_regs = ctx->blocksize / 8; + __be32 *civ = (__be32 *)ctx->iv; u32 iv[AES_BLOCK_SIZE / 4]; dev_dbg(device_data->dev, "[%s]", __func__); @@ -300,7 +291,7 @@ static int cfg_ivs(struct cryp_device_data *device_data, struct cryp_ctx *ctx) } for (i = 0; i < ctx->blocksize / 4; i++) - iv[i] = uint8p_to_uint32_be(ctx->iv + i*4); + iv[i] = be32_to_cpup(civ + i); for (i = 0; i < num_of_regs; i++) { status = cfg_iv(device_data, iv[i*2], iv[i*2+1], @@ -339,23 +330,24 @@ static int cfg_keys(struct cryp_ctx *ctx) int i; int num_of_regs = ctx->keylen / 8; u32 swapped_key[CRYP_MAX_KEY_SIZE / 4]; + __be32 *ckey = (__be32 *)ctx->key; int cryp_error = 0; dev_dbg(ctx->device->dev, "[%s]", __func__); if (mode_is_aes(ctx->config.algomode)) { - swap_words_in_key_and_bits_in_byte((u8 *)ctx->key, + swap_words_in_key_and_bits_in_byte((u8 *)ckey, (u8 *)swapped_key, ctx->keylen); } else { for (i = 0; i < ctx->keylen / 4; i++) - swapped_key[i] = uint8p_to_uint32_be(ctx->key + i*4); + swapped_key[i] = be32_to_cpup(ckey + i); } for (i = 0; i < num_of_regs; i++) { cryp_error = set_key(ctx->device, - *(((u32 *)swapped_key)+i*2), - *(((u32 *)swapped_key)+i*2+1), + swapped_key[i * 2], + swapped_key[i * 2 + 1], (enum cryp_key_reg_index) i); if (cryp_error != 0) { diff --git a/drivers/crypto/ux500/hash/hash_core.c b/drivers/crypto/ux500/hash/hash_core.c index a5ee8c2fb4e0..3d407eebb2ba 100644 --- a/drivers/crypto/ux500/hash/hash_core.c +++ b/drivers/crypto/ux500/hash/hash_core.c @@ -15,6 +15,7 @@ #include <linux/clk.h> #include <linux/device.h> +#include <linux/dma-mapping.h> #include <linux/err.h> #include <linux/init.h> #include <linux/io.h> @@ -1071,27 +1072,32 @@ int hash_hw_update(struct ahash_request *req) struct hash_ctx *ctx = crypto_ahash_ctx(tfm); struct hash_req_ctx *req_ctx = ahash_request_ctx(req); struct crypto_hash_walk walk; - int msg_length = crypto_hash_walk_first(req, &walk); - - /* Empty message ("") is correct indata */ - if (msg_length == 0) - return ret; + int msg_length; index = req_ctx->state.index; buffer = (u8 *)req_ctx->state.buffer; + ret = hash_get_device_data(ctx, &device_data); + if (ret) + return ret; + + msg_length = crypto_hash_walk_first(req, &walk); + + /* Empty message ("") is correct indata */ + if (msg_length == 0) { + ret = 0; + goto release_dev; + } + /* Check if ctx->state.length + msg_length overflows */ if (msg_length > (req_ctx->state.length.low_word + msg_length) && HASH_HIGH_WORD_MAX_VAL == req_ctx->state.length.high_word) { pr_err("%s: HASH_MSG_LENGTH_OVERFLOW!\n", __func__); - return -EPERM; + ret = crypto_hash_walk_done(&walk, -EPERM); + goto release_dev; } - ret = hash_get_device_data(ctx, &device_data); - if (ret) - return ret; - /* Main loop */ while (0 != msg_length) { data_buffer = walk.data; @@ -1101,7 +1107,8 @@ int hash_hw_update(struct ahash_request *req) if (ret) { dev_err(device_data->dev, "%s: hash_internal_hw_update() failed!\n", __func__); - goto out; + crypto_hash_walk_done(&walk, ret); + goto release_dev; } msg_length = crypto_hash_walk_done(&walk, 0); @@ -1111,7 +1118,7 @@ int hash_hw_update(struct ahash_request *req) dev_dbg(device_data->dev, "%s: indata length=%d, bin=%d\n", __func__, req_ctx->state.index, req_ctx->state.bit_index); -out: +release_dev: release_hash_device(device_data); return ret; diff --git a/drivers/crypto/virtio/Kconfig b/drivers/crypto/virtio/Kconfig index fb294174e408..b894e3a8be4f 100644 --- a/drivers/crypto/virtio/Kconfig +++ b/drivers/crypto/virtio/Kconfig @@ -5,7 +5,6 @@ config CRYPTO_DEV_VIRTIO select CRYPTO_AEAD select CRYPTO_SKCIPHER select CRYPTO_ENGINE - default m help This driver provides support for virtio crypto device. If you choose 'M' here, this module will be called virtio_crypto. diff --git a/drivers/crypto/xilinx/zynqmp-aes-gcm.c b/drivers/crypto/xilinx/zynqmp-aes-gcm.c index 27079354dbe9..bf1f421e05f2 100644 --- a/drivers/crypto/xilinx/zynqmp-aes-gcm.c +++ b/drivers/crypto/xilinx/zynqmp-aes-gcm.c @@ -10,6 +10,7 @@ #include <crypto/internal/aead.h> #include <crypto/scatterwalk.h> +#include <linux/dma-mapping.h> #include <linux/module.h> #include <linux/of_device.h> #include <linux/platform_device.h> |