From a7cf8658460edafb03438eeeb0a7d3c91572d4db Mon Sep 17 00:00:00 2001 From: Pascal van Leeuwen Date: Mon, 9 Sep 2019 13:10:29 +0200 Subject: crypto: inside-secure - Added support for CRC32 This patch adds support for the CRC32 "hash" algorithm Signed-off-by: Pascal van Leeuwen Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers/crypto/inside-secure/safexcel.c') diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 4ab1bde8dd9b..44cd806047c4 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1169,6 +1169,7 @@ static struct safexcel_alg_template *safexcel_algs[] = { &safexcel_alg_xts_aes, &safexcel_alg_gcm, &safexcel_alg_ccm, + &safexcel_alg_crc32, }; static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv) -- cgit From b98687bb3b10a0e261c445aac227476bf11dab08 Mon Sep 17 00:00:00 2001 From: Pascal van Leeuwen Date: Mon, 9 Sep 2019 13:13:06 +0200 Subject: crypto: inside-secure - Added support for the AES CBCMAC ahash This patch adds support for the AES-CBCMAC authentication algorithm. Signed-off-by: Pascal van Leeuwen Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 1 + drivers/crypto/inside-secure/safexcel.h | 1 + drivers/crypto/inside-secure/safexcel_hash.c | 235 ++++++++++++++++++++++----- 3 files changed, 196 insertions(+), 41 deletions(-) (limited to 'drivers/crypto/inside-secure/safexcel.c') diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 44cd806047c4..8860b2d557b1 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1170,6 +1170,7 @@ static struct safexcel_alg_template *safexcel_algs[] = { &safexcel_alg_gcm, &safexcel_alg_ccm, &safexcel_alg_crc32, + &safexcel_alg_cbcmac, }; static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv) diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 601557614964..d73283da2bf2 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -855,5 +855,6 @@ extern struct safexcel_alg_template safexcel_alg_xts_aes; extern struct safexcel_alg_template safexcel_alg_gcm; extern struct safexcel_alg_template safexcel_alg_ccm; extern struct safexcel_alg_template safexcel_alg_crc32; +extern struct safexcel_alg_template safexcel_alg_cbcmac; #endif diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index 9d1e8cf071eb..8df4fdc652bd 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -5,6 +5,7 @@ * Antoine Tenart */ +#include #include #include #include @@ -19,6 +20,7 @@ struct safexcel_ahash_ctx { struct safexcel_crypto_priv *priv; u32 alg; + u8 key_sz; u32 ipad[SHA512_DIGEST_SIZE / sizeof(u32)]; u32 opad[SHA512_DIGEST_SIZE / sizeof(u32)]; @@ -31,6 +33,8 @@ struct safexcel_ahash_req { bool needs_inv; bool hmac_zlen; bool len_is_le; + bool not_first; + bool xcbcmac; int nents; dma_addr_t result_dma; @@ -57,21 +61,31 @@ static inline u64 safexcel_queued_len(struct safexcel_ahash_req *req) } static void safexcel_hash_token(struct safexcel_command_desc *cdesc, - u32 input_length, u32 result_length) + u32 input_length, u32 result_length, + bool xcbcmac) { struct safexcel_token *token = (struct safexcel_token *)cdesc->control_data.token; token[0].opcode = EIP197_TOKEN_OPCODE_DIRECTION; token[0].packet_length = input_length; - token[0].stat = EIP197_TOKEN_STAT_LAST_HASH; token[0].instructions = EIP197_TOKEN_INS_TYPE_HASH; - token[1].opcode = EIP197_TOKEN_OPCODE_INSERT; - token[1].packet_length = result_length; - token[1].stat = EIP197_TOKEN_STAT_LAST_HASH | + input_length &= 15; + if (unlikely(xcbcmac && input_length)) { + token[1].opcode = EIP197_TOKEN_OPCODE_INSERT; + token[1].packet_length = 16 - input_length; + token[1].stat = EIP197_TOKEN_STAT_LAST_HASH; + token[1].instructions = EIP197_TOKEN_INS_TYPE_HASH; + } else { + token[0].stat = EIP197_TOKEN_STAT_LAST_HASH; + } + + token[2].opcode = EIP197_TOKEN_OPCODE_INSERT; + token[2].stat = EIP197_TOKEN_STAT_LAST_HASH | EIP197_TOKEN_STAT_LAST_PACKET; - token[1].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT | + token[2].packet_length = result_length; + token[2].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT | EIP197_TOKEN_INS_INSERT_HASH_DIGEST; } @@ -90,29 +104,40 @@ static void safexcel_context_control(struct safexcel_ahash_ctx *ctx, * descriptor. */ if (unlikely(req->digest == CONTEXT_CONTROL_DIGEST_XCM)) { - ctx->base.ctxr->data[0] = req->state[0]; - - cdesc->control_data.control0 |= req->digest | - CONTEXT_CONTROL_TYPE_HASH_OUT | - CONTEXT_CONTROL_SIZE(4); + if (req->xcbcmac) + memcpy(ctx->base.ctxr->data, ctx->ipad, ctx->key_sz); + else + memcpy(ctx->base.ctxr->data, req->state, req->state_sz); + if (!req->finish && req->xcbcmac) + cdesc->control_data.control0 |= + CONTEXT_CONTROL_DIGEST_XCM | + CONTEXT_CONTROL_TYPE_HASH_OUT | + CONTEXT_CONTROL_NO_FINISH_HASH | + CONTEXT_CONTROL_SIZE(req->state_sz / + sizeof(u32)); + else + cdesc->control_data.control0 |= + CONTEXT_CONTROL_DIGEST_XCM | + CONTEXT_CONTROL_TYPE_HASH_OUT | + CONTEXT_CONTROL_SIZE(req->state_sz / + sizeof(u32)); return; } else if (!req->processed) { /* First - and possibly only - block of basic hash only */ - if (req->finish) { + if (req->finish) cdesc->control_data.control0 |= req->digest | CONTEXT_CONTROL_TYPE_HASH_OUT | CONTEXT_CONTROL_RESTART_HASH | /* ensure its not 0! */ CONTEXT_CONTROL_SIZE(1); - } else { + else cdesc->control_data.control0 |= req->digest | CONTEXT_CONTROL_TYPE_HASH_OUT | CONTEXT_CONTROL_RESTART_HASH | CONTEXT_CONTROL_NO_FINISH_HASH | /* ensure its not 0! */ CONTEXT_CONTROL_SIZE(1); - } return; } @@ -246,7 +271,8 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, return 1; } - if (unlikely(sreq->digest == CONTEXT_CONTROL_DIGEST_XCM)) { + if (unlikely(sreq->digest == CONTEXT_CONTROL_DIGEST_XCM && + ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_CRC32)) { /* Undo final XOR with 0xffffffff ...*/ *(u32 *)areq->result = ~sreq->state[0]; } else { @@ -274,10 +300,10 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring, struct safexcel_command_desc *cdesc, *first_cdesc = NULL; struct safexcel_result_desc *rdesc; struct scatterlist *sg; - int i, extra = 0, n_cdesc = 0, ret = 0; - u64 queued, len, cache_len; + int i, extra = 0, n_cdesc = 0, ret = 0, cache_len, skip = 0, res_sz; + u64 queued, len; - queued = len = safexcel_queued_len(req); + queued = safexcel_queued_len(req); if (queued <= HASH_CACHE_SIZE) cache_len = queued; else @@ -300,15 +326,43 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring, areq->nbytes - extra); queued -= extra; - len -= extra; if (!queued) { *commands = 0; *results = 0; return 0; } + + extra = 0; + } + + if (unlikely(req->xcbcmac && req->processed > AES_BLOCK_SIZE)) { + if (unlikely(cache_len < AES_BLOCK_SIZE)) { + /* + * Cache contains less than 1 full block, complete. + */ + extra = AES_BLOCK_SIZE - cache_len; + if (queued > cache_len) { + /* More data follows: borrow bytes */ + u64 tmp = queued - cache_len; + + skip = min_t(u64, tmp, extra); + sg_pcopy_to_buffer(areq->src, + sg_nents(areq->src), + req->cache + cache_len, + skip, 0); + } + extra -= skip; + memset(req->cache + cache_len + skip, 0, extra); + cache_len = AES_BLOCK_SIZE; + queued = queued + extra; + } + + /* XCBC continue: XOR previous result into 1st word */ + crypto_xor(req->cache, (const u8 *)req->state, AES_BLOCK_SIZE); } + len = queued; /* Add a command descriptor for the cached data, if any */ if (cache_len) { req->cache_dma = dma_map_single(priv->dev, req->cache, @@ -319,8 +373,8 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring, req->cache_sz = cache_len; first_cdesc = safexcel_add_cdesc(priv, ring, 1, (cache_len == len), - req->cache_dma, cache_len, len, - ctx->base.ctxr_dma); + req->cache_dma, cache_len, + len, ctx->base.ctxr_dma); if (IS_ERR(first_cdesc)) { ret = PTR_ERR(first_cdesc); goto unmap_cache; @@ -332,10 +386,6 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring, goto send_command; } - /* Skip descriptor generation for zero-length requests */ - if (!areq->nbytes) - goto send_command; - /* Now handle the current ahash request buffer(s) */ req->nents = dma_map_sg(priv->dev, areq->src, sg_nents_for_len(areq->src, @@ -349,34 +399,43 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring, for_each_sg(areq->src, sg, req->nents, i) { int sglen = sg_dma_len(sg); + if (unlikely(sglen <= skip)) { + skip -= sglen; + continue; + } + /* Do not overflow the request */ - if (queued < sglen) + if ((queued + skip) <= sglen) sglen = queued; + else + sglen -= skip; cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc, !(queued - sglen), - sg_dma_address(sg), - sglen, len, ctx->base.ctxr_dma); + sg_dma_address(sg) + skip, sglen, + len, ctx->base.ctxr_dma); if (IS_ERR(cdesc)) { ret = PTR_ERR(cdesc); goto unmap_sg; } - n_cdesc++; - if (n_cdesc == 1) + if (!n_cdesc) first_cdesc = cdesc; + n_cdesc++; queued -= sglen; if (!queued) break; + skip = 0; } send_command: /* Setup the context options */ safexcel_context_control(ctx, req, first_cdesc); - /* Add the token */ - safexcel_hash_token(first_cdesc, len, req->state_sz); + /* Add the token. Note that the XCBC result is only 1 AES block. */ + res_sz = req->xcbcmac ? AES_BLOCK_SIZE : req->state_sz; + safexcel_hash_token(first_cdesc, len, res_sz, req->xcbcmac); req->result_dma = dma_map_single(priv->dev, req->state, req->state_sz, DMA_FROM_DEVICE); @@ -387,7 +446,7 @@ send_command: /* Add a result descriptor */ rdesc = safexcel_add_rdesc(priv, ring, 1, 1, req->result_dma, - req->state_sz); + res_sz); if (IS_ERR(rdesc)) { ret = PTR_ERR(rdesc); goto unmap_result; @@ -395,7 +454,7 @@ send_command: safexcel_rdr_req_set(priv, ring, rdesc, &areq->base); - req->processed += len; + req->processed += len - extra; *commands = n_cdesc; *results = 1; @@ -405,7 +464,10 @@ unmap_result: dma_unmap_single(priv->dev, req->result_dma, req->state_sz, DMA_FROM_DEVICE); unmap_sg: - dma_unmap_sg(priv->dev, areq->src, req->nents, DMA_TO_DEVICE); + if (req->nents) { + dma_unmap_sg(priv->dev, areq->src, req->nents, DMA_TO_DEVICE); + req->nents = 0; + } cdesc_rollback: for (i = 0; i < n_cdesc; i++) safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr); @@ -603,14 +665,10 @@ static int safexcel_ahash_enqueue(struct ahash_request *areq) if (ctx->base.ctxr) { if (priv->flags & EIP197_TRC_CACHE && !ctx->base.needs_inv && - req->processed && - (/* invalidate for basic hash continuation finish */ - (req->finish && - (req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED)) || + /* invalidate for *any* non-XCBC continuation */ + ((req->not_first && !req->xcbcmac) || /* invalidate if (i)digest changed */ memcmp(ctx->base.ctxr->data, req->state, req->state_sz) || - /* invalidate for HMAC continuation finish */ - (req->finish && (req->processed != req->block_sz)) || /* invalidate for HMAC finish with odigest changed */ (req->finish && req->hmac && memcmp(ctx->base.ctxr->data + (req->state_sz>>2), @@ -635,6 +693,7 @@ static int safexcel_ahash_enqueue(struct ahash_request *areq) if (!ctx->base.ctxr) return -ENOMEM; } + req->not_first = true; ring = ctx->base.ring; @@ -712,6 +771,11 @@ static int safexcel_ahash_final(struct ahash_request *areq) /* Zero length CRC32 */ memcpy(areq->result, ctx->ipad, sizeof(u32)); return 0; + } else if (unlikely(req->xcbcmac && req->len == AES_BLOCK_SIZE && + !areq->nbytes)) { + /* Zero length CBC MAC */ + memset(areq->result, 0, AES_BLOCK_SIZE); + return 0; } else if (unlikely(req->hmac && (req->len == req->block_sz) && !areq->nbytes)) { @@ -1841,3 +1905,92 @@ struct safexcel_alg_template safexcel_alg_crc32 = { }, }, }; + +static int safexcel_cbcmac_init(struct ahash_request *areq) +{ + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + struct safexcel_ahash_req *req = ahash_request_ctx(areq); + + memset(req, 0, sizeof(*req)); + + /* Start from loaded keys */ + memcpy(req->state, ctx->ipad, ctx->key_sz); + /* Set processed to non-zero to enable invalidation detection */ + req->len = AES_BLOCK_SIZE; + req->processed = AES_BLOCK_SIZE; + + req->digest = CONTEXT_CONTROL_DIGEST_XCM; + req->state_sz = ctx->key_sz; + req->block_sz = AES_BLOCK_SIZE; + req->xcbcmac = true; + + return 0; +} + +static int safexcel_cbcmac_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int len) +{ + struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); + struct crypto_aes_ctx aes; + int ret, i; + + ret = aes_expandkey(&aes, key, len); + if (ret) { + crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return ret; + } + + memset(ctx->ipad, 0, 2 * AES_BLOCK_SIZE); + for (i = 0; i < len / sizeof(u32); i++) + ctx->ipad[i + 8] = cpu_to_be32(aes.key_enc[i]); + + if (len == AES_KEYSIZE_192) { + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192; + ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE; + } else if (len == AES_KEYSIZE_256) { + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256; + ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE; + } else { + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128; + ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE; + } + + memzero_explicit(&aes, sizeof(aes)); + return 0; +} + +static int safexcel_cbcmac_digest(struct ahash_request *areq) +{ + return safexcel_cbcmac_init(areq) ?: safexcel_ahash_finup(areq); +} + +struct safexcel_alg_template safexcel_alg_cbcmac = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = 0, + .alg.ahash = { + .init = safexcel_cbcmac_init, + .update = safexcel_ahash_update, + .final = safexcel_ahash_final, + .finup = safexcel_ahash_finup, + .digest = safexcel_cbcmac_digest, + .setkey = safexcel_cbcmac_setkey, + .export = safexcel_ahash_export, + .import = safexcel_ahash_import, + .halg = { + .digestsize = AES_BLOCK_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "cbcmac(aes)", + .cra_driver_name = "safexcel-cbcmac-aes", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_ahash_cra_init, + .cra_exit = safexcel_ahash_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; -- cgit From 38f21b4bab11fc877ff18dd02f77f2c34f1105b9 Mon Sep 17 00:00:00 2001 From: Pascal van Leeuwen Date: Mon, 9 Sep 2019 13:13:07 +0200 Subject: crypto: inside-secure - Added support for the AES XCBC ahash This patch adds support for the AES XCBC authentication algorithm Signed-off-by: Pascal van Leeuwen Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 1 + drivers/crypto/inside-secure/safexcel.h | 1 + drivers/crypto/inside-secure/safexcel_hash.c | 136 ++++++++++++++++++++++++++- 3 files changed, 134 insertions(+), 4 deletions(-) (limited to 'drivers/crypto/inside-secure/safexcel.c') diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 8860b2d557b1..593ae66062ba 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1171,6 +1171,7 @@ static struct safexcel_alg_template *safexcel_algs[] = { &safexcel_alg_ccm, &safexcel_alg_crc32, &safexcel_alg_cbcmac, + &safexcel_alg_xcbcmac, }; static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv) diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index d73283da2bf2..6f08b133df2d 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -856,5 +856,6 @@ extern struct safexcel_alg_template safexcel_alg_gcm; extern struct safexcel_alg_template safexcel_alg_ccm; extern struct safexcel_alg_template safexcel_alg_crc32; extern struct safexcel_alg_template safexcel_alg_cbcmac; +extern struct safexcel_alg_template safexcel_alg_xcbcmac; #endif diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index 8df4fdc652bd..6576430c59ff 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -21,9 +22,12 @@ struct safexcel_ahash_ctx { u32 alg; u8 key_sz; + bool cbcmac; u32 ipad[SHA512_DIGEST_SIZE / sizeof(u32)]; u32 opad[SHA512_DIGEST_SIZE / sizeof(u32)]; + + struct crypto_cipher *kaes; }; struct safexcel_ahash_req { @@ -62,7 +66,7 @@ static inline u64 safexcel_queued_len(struct safexcel_ahash_req *req) static void safexcel_hash_token(struct safexcel_command_desc *cdesc, u32 input_length, u32 result_length, - bool xcbcmac) + bool cbcmac) { struct safexcel_token *token = (struct safexcel_token *)cdesc->control_data.token; @@ -72,7 +76,7 @@ static void safexcel_hash_token(struct safexcel_command_desc *cdesc, token[0].instructions = EIP197_TOKEN_INS_TYPE_HASH; input_length &= 15; - if (unlikely(xcbcmac && input_length)) { + if (unlikely(cbcmac && input_length)) { token[1].opcode = EIP197_TOKEN_OPCODE_INSERT; token[1].packet_length = 16 - input_length; token[1].stat = EIP197_TOKEN_STAT_LAST_HASH; @@ -354,6 +358,15 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring, } extra -= skip; memset(req->cache + cache_len + skip, 0, extra); + if (!ctx->cbcmac && extra) { + // 10- padding for XCBCMAC & CMAC + req->cache[cache_len + skip] = 0x80; + // HW will use K2 iso K3 - compensate! + for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) + ((u32 *)req->cache)[i] ^= + cpu_to_be32(ctx->ipad[i]) ^ + cpu_to_be32(ctx->ipad[i + 4]); + } cache_len = AES_BLOCK_SIZE; queued = queued + extra; } @@ -435,7 +448,7 @@ send_command: /* Add the token. Note that the XCBC result is only 1 AES block. */ res_sz = req->xcbcmac ? AES_BLOCK_SIZE : req->state_sz; - safexcel_hash_token(first_cdesc, len, res_sz, req->xcbcmac); + safexcel_hash_token(first_cdesc, len, res_sz, ctx->cbcmac); req->result_dma = dma_map_single(priv->dev, req->state, req->state_sz, DMA_FROM_DEVICE); @@ -771,11 +784,22 @@ static int safexcel_ahash_final(struct ahash_request *areq) /* Zero length CRC32 */ memcpy(areq->result, ctx->ipad, sizeof(u32)); return 0; - } else if (unlikely(req->xcbcmac && req->len == AES_BLOCK_SIZE && + } else if (unlikely(ctx->cbcmac && req->len == AES_BLOCK_SIZE && !areq->nbytes)) { /* Zero length CBC MAC */ memset(areq->result, 0, AES_BLOCK_SIZE); return 0; + } else if (unlikely(req->xcbcmac && req->len == AES_BLOCK_SIZE && + !areq->nbytes)) { + /* Zero length (X)CBC/CMAC */ + int i; + + for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) + ((u32 *)areq->result)[i] = + cpu_to_be32(ctx->ipad[i + 4]); // K3 + areq->result[0] ^= 0x80; // 10- padding + crypto_cipher_encrypt_one(ctx->kaes, areq->result, areq->result); + return 0; } else if (unlikely(req->hmac && (req->len == req->block_sz) && !areq->nbytes)) { @@ -1954,6 +1978,7 @@ static int safexcel_cbcmac_setkey(struct crypto_ahash *tfm, const u8 *key, ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128; ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE; } + ctx->cbcmac = true; memzero_explicit(&aes, sizeof(aes)); return 0; @@ -1994,3 +2019,106 @@ struct safexcel_alg_template safexcel_alg_cbcmac = { }, }, }; + +static int safexcel_xcbcmac_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int len) +{ + struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); + struct crypto_aes_ctx aes; + u32 key_tmp[3 * AES_BLOCK_SIZE / sizeof(u32)]; + int ret, i; + + ret = aes_expandkey(&aes, key, len); + if (ret) { + crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return ret; + } + + /* precompute the XCBC key material */ + crypto_cipher_clear_flags(ctx->kaes, CRYPTO_TFM_REQ_MASK); + crypto_cipher_set_flags(ctx->kaes, crypto_ahash_get_flags(tfm) & + CRYPTO_TFM_REQ_MASK); + ret = crypto_cipher_setkey(ctx->kaes, key, len); + crypto_ahash_set_flags(tfm, crypto_cipher_get_flags(ctx->kaes) & + CRYPTO_TFM_RES_MASK); + if (ret) + return ret; + + crypto_cipher_encrypt_one(ctx->kaes, (u8 *)key_tmp + 2 * AES_BLOCK_SIZE, + "\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1"); + crypto_cipher_encrypt_one(ctx->kaes, (u8 *)key_tmp, + "\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2"); + crypto_cipher_encrypt_one(ctx->kaes, (u8 *)key_tmp + AES_BLOCK_SIZE, + "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3"); + for (i = 0; i < 3 * AES_BLOCK_SIZE / sizeof(u32); i++) + ctx->ipad[i] = cpu_to_be32(key_tmp[i]); + + crypto_cipher_clear_flags(ctx->kaes, CRYPTO_TFM_REQ_MASK); + crypto_cipher_set_flags(ctx->kaes, crypto_ahash_get_flags(tfm) & + CRYPTO_TFM_REQ_MASK); + ret = crypto_cipher_setkey(ctx->kaes, + (u8 *)key_tmp + 2 * AES_BLOCK_SIZE, + AES_MIN_KEY_SIZE); + crypto_ahash_set_flags(tfm, crypto_cipher_get_flags(ctx->kaes) & + CRYPTO_TFM_RES_MASK); + if (ret) + return ret; + + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128; + ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE; + ctx->cbcmac = false; + + memzero_explicit(&aes, sizeof(aes)); + return 0; +} + +static int safexcel_xcbcmac_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_ahash_cra_init(tfm); + ctx->kaes = crypto_alloc_cipher("aes", 0, 0); + if (IS_ERR(ctx->kaes)) + return PTR_ERR(ctx->kaes); + + return 0; +} + +static void safexcel_xcbcmac_cra_exit(struct crypto_tfm *tfm) +{ + struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); + + crypto_free_cipher(ctx->kaes); + safexcel_ahash_cra_exit(tfm); +} + +struct safexcel_alg_template safexcel_alg_xcbcmac = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = 0, + .alg.ahash = { + .init = safexcel_cbcmac_init, + .update = safexcel_ahash_update, + .final = safexcel_ahash_final, + .finup = safexcel_ahash_finup, + .digest = safexcel_cbcmac_digest, + .setkey = safexcel_xcbcmac_setkey, + .export = safexcel_ahash_export, + .import = safexcel_ahash_import, + .halg = { + .digestsize = AES_BLOCK_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "xcbc(aes)", + .cra_driver_name = "safexcel-xcbc-aes", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_xcbcmac_cra_init, + .cra_exit = safexcel_xcbcmac_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; -- cgit From 7a627db9cafdb173e255ec8bf67fd5ba91ae0664 Mon Sep 17 00:00:00 2001 From: Pascal van Leeuwen Date: Mon, 9 Sep 2019 13:13:08 +0200 Subject: crypto: inside-secure - Added support for the AES-CMAC ahash This patch adds support for the AES-CMAC authentication algorithm. Signed-off-by: Pascal van Leeuwen Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 1 + drivers/crypto/inside-secure/safexcel.h | 1 + drivers/crypto/inside-secure/safexcel_hash.c | 99 ++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+) (limited to 'drivers/crypto/inside-secure/safexcel.c') diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 593ae66062ba..d0f49a56f531 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1172,6 +1172,7 @@ static struct safexcel_alg_template *safexcel_algs[] = { &safexcel_alg_crc32, &safexcel_alg_cbcmac, &safexcel_alg_xcbcmac, + &safexcel_alg_cmac, }; static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv) diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 6f08b133df2d..6ddc6d1bebfb 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -857,5 +857,6 @@ extern struct safexcel_alg_template safexcel_alg_ccm; extern struct safexcel_alg_template safexcel_alg_crc32; extern struct safexcel_alg_template safexcel_alg_cbcmac; extern struct safexcel_alg_template safexcel_alg_xcbcmac; +extern struct safexcel_alg_template safexcel_alg_cmac; #endif diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index 6576430c59ff..0224779f2984 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -2122,3 +2122,102 @@ struct safexcel_alg_template safexcel_alg_xcbcmac = { }, }, }; + +static int safexcel_cmac_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int len) +{ + struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); + struct crypto_aes_ctx aes; + __be64 consts[4]; + u64 _const[2]; + u8 msb_mask, gfmask; + int ret, i; + + ret = aes_expandkey(&aes, key, len); + if (ret) { + crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return ret; + } + + for (i = 0; i < len / sizeof(u32); i++) + ctx->ipad[i + 8] = cpu_to_be32(aes.key_enc[i]); + + /* precompute the CMAC key material */ + crypto_cipher_clear_flags(ctx->kaes, CRYPTO_TFM_REQ_MASK); + crypto_cipher_set_flags(ctx->kaes, crypto_ahash_get_flags(tfm) & + CRYPTO_TFM_REQ_MASK); + ret = crypto_cipher_setkey(ctx->kaes, key, len); + crypto_ahash_set_flags(tfm, crypto_cipher_get_flags(ctx->kaes) & + CRYPTO_TFM_RES_MASK); + if (ret) + return ret; + + /* code below borrowed from crypto/cmac.c */ + /* encrypt the zero block */ + memset(consts, 0, AES_BLOCK_SIZE); + crypto_cipher_encrypt_one(ctx->kaes, (u8 *)consts, (u8 *)consts); + + gfmask = 0x87; + _const[0] = be64_to_cpu(consts[1]); + _const[1] = be64_to_cpu(consts[0]); + + /* gf(2^128) multiply zero-ciphertext with u and u^2 */ + for (i = 0; i < 4; i += 2) { + msb_mask = ((s64)_const[1] >> 63) & gfmask; + _const[1] = (_const[1] << 1) | (_const[0] >> 63); + _const[0] = (_const[0] << 1) ^ msb_mask; + + consts[i + 0] = cpu_to_be64(_const[1]); + consts[i + 1] = cpu_to_be64(_const[0]); + } + /* end of code borrowed from crypto/cmac.c */ + + for (i = 0; i < 2 * AES_BLOCK_SIZE / sizeof(u32); i++) + ctx->ipad[i] = cpu_to_be32(((u32 *)consts)[i]); + + if (len == AES_KEYSIZE_192) { + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192; + ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE; + } else if (len == AES_KEYSIZE_256) { + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256; + ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE; + } else { + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128; + ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE; + } + ctx->cbcmac = false; + + memzero_explicit(&aes, sizeof(aes)); + return 0; +} + +struct safexcel_alg_template safexcel_alg_cmac = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = 0, + .alg.ahash = { + .init = safexcel_cbcmac_init, + .update = safexcel_ahash_update, + .final = safexcel_ahash_final, + .finup = safexcel_ahash_finup, + .digest = safexcel_cbcmac_digest, + .setkey = safexcel_cmac_setkey, + .export = safexcel_ahash_export, + .import = safexcel_ahash_import, + .halg = { + .digestsize = AES_BLOCK_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "cmac(aes)", + .cra_driver_name = "safexcel-cmac-aes", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_xcbcmac_cra_init, + .cra_exit = safexcel_xcbcmac_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; -- cgit From 4a593fb35d5ccf3ddd41c68ac1cc88d06ec74341 Mon Sep 17 00:00:00 2001 From: Pascal van Leeuwen Date: Wed, 18 Sep 2019 23:25:56 +0200 Subject: crypto: inside-secure - Added support for the CHACHA20 skcipher Added support for the CHACHA20 skcipher algorithm. Tested on an eip197c-iesb configuration in the Xilinx VCU118 devboard, passes all testmgr vectors plus the extra fuzzing tests. changes since v1: - rebased on top of DES library changes done on cryptodev/master - fixed crypto/Kconfig so that generic fallback is compiled as well changes since v2: - made switch entry SAFEXCEL_AES explit and added empty default, as requested by Antoine Tenart. Also needed to make SM4 patches apply. changes since v3: - nothing Signed-off-by: Pascal van Leeuwen Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 1 + drivers/crypto/inside-secure/safexcel.h | 3 + drivers/crypto/inside-secure/safexcel_cipher.c | 83 +++++++++++++++++++++++++- 3 files changed, 86 insertions(+), 1 deletion(-) (limited to 'drivers/crypto/inside-secure/safexcel.c') diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index d0f49a56f531..f958c927f95e 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1173,6 +1173,7 @@ static struct safexcel_alg_template *safexcel_algs[] = { &safexcel_alg_cbcmac, &safexcel_alg_xcbcmac, &safexcel_alg_cmac, + &safexcel_alg_chacha20, }; static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv) diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 6ddc6d1bebfb..c7f1a201354a 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -358,6 +358,7 @@ struct safexcel_context_record { #define CONTEXT_CONTROL_CRYPTO_ALG_AES128 (0x5 << 17) #define CONTEXT_CONTROL_CRYPTO_ALG_AES192 (0x6 << 17) #define CONTEXT_CONTROL_CRYPTO_ALG_AES256 (0x7 << 17) +#define CONTEXT_CONTROL_CRYPTO_ALG_CHACHA20 (0x8 << 17) #define CONTEXT_CONTROL_DIGEST_PRECOMPUTED (0x1 << 21) #define CONTEXT_CONTROL_DIGEST_XCM (0x2 << 21) #define CONTEXT_CONTROL_DIGEST_HMAC (0x3 << 21) @@ -378,6 +379,7 @@ struct safexcel_context_record { /* control1 */ #define CONTEXT_CONTROL_CRYPTO_MODE_ECB (0 << 0) #define CONTEXT_CONTROL_CRYPTO_MODE_CBC (1 << 0) +#define CONTEXT_CONTROL_CHACHA20_MODE_256_32 (2 << 0) #define CONTEXT_CONTROL_CRYPTO_MODE_OFB (4 << 0) #define CONTEXT_CONTROL_CRYPTO_MODE_CFB (5 << 0) #define CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD (6 << 0) @@ -858,5 +860,6 @@ extern struct safexcel_alg_template safexcel_alg_crc32; extern struct safexcel_alg_template safexcel_alg_cbcmac; extern struct safexcel_alg_template safexcel_alg_xcbcmac; extern struct safexcel_alg_template safexcel_alg_cmac; +extern struct safexcel_alg_template safexcel_alg_chacha20; #endif diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index ef51f8c2b473..15d98a913da0 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -5,13 +5,14 @@ * Antoine Tenart */ +#include #include #include #include - #include #include #include +#include #include #include #include @@ -33,6 +34,7 @@ enum safexcel_cipher_alg { SAFEXCEL_DES, SAFEXCEL_3DES, SAFEXCEL_AES, + SAFEXCEL_CHACHA20, }; struct safexcel_cipher_ctx { @@ -80,6 +82,15 @@ static void safexcel_cipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv, /* 32 bit counter, start at 1 (big endian!) */ cdesc->control_data.token[3] = cpu_to_be32(1); + return; + } else if (ctx->alg == SAFEXCEL_CHACHA20) { + cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; + + /* 96 bit nonce part */ + memcpy(&cdesc->control_data.token[0], &iv[4], 12); + /* 32 bit counter */ + cdesc->control_data.token[3] = *(u32 *)iv; + return; } else if (ctx->xcm == EIP197_XCM_MODE_GCM) { cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; @@ -116,6 +127,8 @@ static void safexcel_cipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv, block_sz = AES_BLOCK_SIZE; cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; break; + default: + break; } memcpy(cdesc->control_data.token, iv, block_sz); } @@ -480,6 +493,9 @@ static int safexcel_context_control(struct safexcel_cipher_ctx *ctx, ctx->key_len >> ctx->xts); return -EINVAL; } + } else if (ctx->alg == SAFEXCEL_CHACHA20) { + cdesc->control_data.control0 |= + CONTEXT_CONTROL_CRYPTO_ALG_CHACHA20; } return 0; @@ -2303,3 +2319,68 @@ struct safexcel_alg_template safexcel_alg_ccm = { }, }, }; + +static int safexcel_skcipher_chacha20_setkey(struct crypto_skcipher *ctfm, + const u8 *key, unsigned int len) +{ + struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm); + struct safexcel_crypto_priv *priv = ctx->priv; + int i; + + if (len != CHACHA_KEY_SIZE) { + crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { + for (i = 0; i < CHACHA_KEY_SIZE / sizeof(u32); i++) { + if (ctx->key[i] != + get_unaligned_le32(key + i * sizeof(u32))) { + ctx->base.needs_inv = true; + break; + } + } + } + + for (i = 0; i < CHACHA_KEY_SIZE / sizeof(u32); i++) + ctx->key[i] = get_unaligned_le32(key + i * sizeof(u32)); + ctx->key_len = CHACHA_KEY_SIZE; + + return 0; +} + +static int safexcel_skcipher_chacha20_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_skcipher_cra_init(tfm); + ctx->alg = SAFEXCEL_CHACHA20; + ctx->mode = CONTEXT_CONTROL_CHACHA20_MODE_256_32; + return 0; +} + +struct safexcel_alg_template safexcel_alg_chacha20 = { + .type = SAFEXCEL_ALG_TYPE_SKCIPHER, + .algo_mask = SAFEXCEL_ALG_CHACHA20, + .alg.skcipher = { + .setkey = safexcel_skcipher_chacha20_setkey, + .encrypt = safexcel_encrypt, + .decrypt = safexcel_decrypt, + .min_keysize = CHACHA_KEY_SIZE, + .max_keysize = CHACHA_KEY_SIZE, + .ivsize = CHACHA_IV_SIZE, + .base = { + .cra_name = "chacha20", + .cra_driver_name = "safexcel-chacha20", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_skcipher_chacha20_cra_init, + .cra_exit = safexcel_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; -- cgit From a60619211dd188a5dfa18761b82d096cda76fc9f Mon Sep 17 00:00:00 2001 From: Pascal van Leeuwen Date: Wed, 18 Sep 2019 23:25:57 +0200 Subject: crypto: inside-secure - Add support for the Chacha20-Poly1305 AEAD This patch adds support for the Chacha20-Poly1305 cipher suite. It adds both the basic rfc7539(chacha20,poly1305) as well as the rfc7539esp(chacha20,poly1305) variant for IPsec ESP acceleration. changes since v1: - rebased on top of DES library changes done on cryptodev/master - fixed crypto/Kconfig so that generic fallback is compiled as well changes since v2: - nothing changes since v3: - Fixed a problem where the tcrypt performance test would run fully on the fallback cipher instead of the HW due to using an AAD length of 8 for rfc7539esp. While this is not actually legal ESP (which includes SPI and sequence number in the AAD as well), it is both inconvenient and not necessary to run these vectors on the fallback cipher. - Due to above, also realised that for plain (non-ESP) rfc7539, you probably want to be able to run vectors with less than 8 bytes of AAD on the HW, and this is actually possible as long as cryptlen is large enough, so made that possible as well. Signed-off-by: Pascal van Leeuwen Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 2 + drivers/crypto/inside-secure/safexcel.h | 8 + drivers/crypto/inside-secure/safexcel_cipher.c | 277 ++++++++++++++++++++++--- 3 files changed, 263 insertions(+), 24 deletions(-) (limited to 'drivers/crypto/inside-secure/safexcel.c') diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index f958c927f95e..b81f0bc5cd50 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1174,6 +1174,8 @@ static struct safexcel_alg_template *safexcel_algs[] = { &safexcel_alg_xcbcmac, &safexcel_alg_cmac, &safexcel_alg_chacha20, + &safexcel_alg_chachapoly, + &safexcel_alg_chachapoly_esp, }; static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv) diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index c7f1a201354a..282d59e523e9 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -373,6 +373,7 @@ struct safexcel_context_record { #define CONTEXT_CONTROL_CRYPTO_ALG_XCBC128 (0x1 << 23) #define CONTEXT_CONTROL_CRYPTO_ALG_XCBC192 (0x2 << 23) #define CONTEXT_CONTROL_CRYPTO_ALG_XCBC256 (0x3 << 23) +#define CONTEXT_CONTROL_CRYPTO_ALG_POLY1305 (0xf << 23) #define CONTEXT_CONTROL_INV_FR (0x5 << 24) #define CONTEXT_CONTROL_INV_TR (0x6 << 24) @@ -385,6 +386,7 @@ struct safexcel_context_record { #define CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD (6 << 0) #define CONTEXT_CONTROL_CRYPTO_MODE_XTS (7 << 0) #define CONTEXT_CONTROL_CRYPTO_MODE_XCM ((6 << 0) | BIT(17)) +#define CONTEXT_CONTROL_CHACHA20_MODE_CALC_OTK (12 << 0) #define CONTEXT_CONTROL_IV0 BIT(5) #define CONTEXT_CONTROL_IV1 BIT(6) #define CONTEXT_CONTROL_IV2 BIT(7) @@ -397,6 +399,10 @@ struct safexcel_context_record { #define EIP197_XCM_MODE_GCM 1 #define EIP197_XCM_MODE_CCM 2 +#define EIP197_AEAD_TYPE_IPSEC_ESP 2 +#define EIP197_AEAD_IPSEC_IV_SIZE 8 +#define EIP197_AEAD_IPSEC_NONCE_SIZE 4 + /* The hash counter given to the engine in the context has a granularity of * 64 bits. */ @@ -861,5 +867,7 @@ extern struct safexcel_alg_template safexcel_alg_cbcmac; extern struct safexcel_alg_template safexcel_alg_xcbcmac; extern struct safexcel_alg_template safexcel_alg_cmac; extern struct safexcel_alg_template safexcel_alg_chacha20; +extern struct safexcel_alg_template safexcel_alg_chachapoly; +extern struct safexcel_alg_template safexcel_alg_chachapoly_esp; #endif diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index 15d98a913da0..f1e6cae9e1c8 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -43,8 +44,8 @@ struct safexcel_cipher_ctx { u32 mode; enum safexcel_cipher_alg alg; - bool aead; - int xcm; /* 0=authenc, 1=GCM, 2 reserved for CCM */ + char aead; /* !=0=AEAD, 2=IPSec ESP AEAD */ + char xcm; /* 0=authenc, 1=GCM, 2 reserved for CCM */ __le32 key[16]; u32 nonce; @@ -57,6 +58,7 @@ struct safexcel_cipher_ctx { u32 opad[SHA512_DIGEST_SIZE / sizeof(u32)]; struct crypto_cipher *hkaes; + struct crypto_aead *fback; }; struct safexcel_cipher_req { @@ -86,10 +88,24 @@ static void safexcel_cipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv, } else if (ctx->alg == SAFEXCEL_CHACHA20) { cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; - /* 96 bit nonce part */ - memcpy(&cdesc->control_data.token[0], &iv[4], 12); - /* 32 bit counter */ - cdesc->control_data.token[3] = *(u32 *)iv; + if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) { + /* 32 bit nonce part */ + cdesc->control_data.token[0] = ctx->nonce; + /* 64 bit IV part */ + memcpy(&cdesc->control_data.token[1], iv, 8); + /* 32 bit counter, starting at 0 */ + cdesc->control_data.token[3] = 0; + } else if (ctx->aead) { + /* 96 bit nonce part */ + memcpy(&cdesc->control_data.token[0], iv, 12); + /* 32 bit counter, starting at 0 */ + cdesc->control_data.token[3] = 0; + } else { + /* 96 bit nonce part */ + memcpy(&cdesc->control_data.token[0], &iv[4], 12); + /* 32 bit counter */ + cdesc->control_data.token[3] = *(u32 *)iv; + } return; } else if (ctx->xcm == EIP197_XCM_MODE_GCM) { @@ -195,12 +211,20 @@ static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv, token[13].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT; } + if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) { + /* For ESP mode, skip over the IV */ + token[7].opcode = EIP197_TOKEN_OPCODE_DIRECTION; + token[7].packet_length = EIP197_AEAD_IPSEC_IV_SIZE; + + assoclen -= EIP197_AEAD_IPSEC_IV_SIZE; + } + token[6].opcode = EIP197_TOKEN_OPCODE_DIRECTION; token[6].packet_length = assoclen; + token[6].instructions = EIP197_TOKEN_INS_LAST | + EIP197_TOKEN_INS_TYPE_HASH; - if (likely(cryptlen)) { - token[6].instructions = EIP197_TOKEN_INS_TYPE_HASH; - + if (likely(cryptlen || ctx->alg == SAFEXCEL_CHACHA20)) { token[10].opcode = EIP197_TOKEN_OPCODE_DIRECTION; token[10].packet_length = cryptlen; token[10].stat = EIP197_TOKEN_STAT_LAST_HASH; @@ -210,8 +234,6 @@ static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv, EIP197_TOKEN_INS_TYPE_OUTPUT; } else if (ctx->xcm != EIP197_XCM_MODE_CCM) { token[6].stat = EIP197_TOKEN_STAT_LAST_HASH; - token[6].instructions = EIP197_TOKEN_INS_LAST | - EIP197_TOKEN_INS_TYPE_HASH; } if (!ctx->xcm) @@ -226,10 +248,7 @@ static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv, token[9].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT | EIP197_TOKEN_INS_TYPE_CRYPTO; - if (ctx->xcm == EIP197_XCM_MODE_GCM) { - token[6].instructions = EIP197_TOKEN_INS_LAST | - EIP197_TOKEN_INS_TYPE_HASH; - } else { + if (ctx->xcm != EIP197_XCM_MODE_GCM) { u8 *cbcmaciv = (u8 *)&token[1]; u32 *aadlen = (u32 *)&token[5]; @@ -388,7 +407,7 @@ static int safexcel_aead_setkey(struct crypto_aead *ctfm, const u8 *key, goto badkey; break; default: - dev_err(priv->dev, "aead: unsupported hash algorithm\n"); + dev_err(priv->dev, "aead: unsupported hash algorithmn"); goto badkey; } @@ -436,6 +455,17 @@ static int safexcel_context_control(struct safexcel_cipher_ctx *ctx, CONTEXT_CONTROL_DIGEST_XCM | ctx->hash_alg | CONTEXT_CONTROL_SIZE(ctrl_size); + } else if (ctx->alg == SAFEXCEL_CHACHA20) { + /* Chacha20-Poly1305 */ + cdesc->control_data.control0 = + CONTEXT_CONTROL_KEY_EN | + CONTEXT_CONTROL_CRYPTO_ALG_CHACHA20 | + (sreq->direction == SAFEXCEL_ENCRYPT ? + CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT : + CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN) | + ctx->hash_alg | + CONTEXT_CONTROL_SIZE(ctrl_size); + return 0; } else { ctrl_size += ctx->state_sz / sizeof(u32) * 2; cdesc->control_data.control0 = @@ -2320,18 +2350,12 @@ struct safexcel_alg_template safexcel_alg_ccm = { }, }; -static int safexcel_skcipher_chacha20_setkey(struct crypto_skcipher *ctfm, - const u8 *key, unsigned int len) +static void safexcel_chacha20_setkey(struct safexcel_cipher_ctx *ctx, + const u8 *key) { - struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm); struct safexcel_crypto_priv *priv = ctx->priv; int i; - if (len != CHACHA_KEY_SIZE) { - crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN); - return -EINVAL; - } - if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { for (i = 0; i < CHACHA_KEY_SIZE / sizeof(u32); i++) { if (ctx->key[i] != @@ -2345,6 +2369,18 @@ static int safexcel_skcipher_chacha20_setkey(struct crypto_skcipher *ctfm, for (i = 0; i < CHACHA_KEY_SIZE / sizeof(u32); i++) ctx->key[i] = get_unaligned_le32(key + i * sizeof(u32)); ctx->key_len = CHACHA_KEY_SIZE; +} + +static int safexcel_skcipher_chacha20_setkey(struct crypto_skcipher *ctfm, + const u8 *key, unsigned int len) +{ + struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm); + + if (len != CHACHA_KEY_SIZE) { + crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + safexcel_chacha20_setkey(ctx, key); return 0; } @@ -2384,3 +2420,196 @@ struct safexcel_alg_template safexcel_alg_chacha20 = { }, }, }; + +static int safexcel_aead_chachapoly_setkey(struct crypto_aead *ctfm, + const u8 *key, unsigned int len) +{ + struct safexcel_cipher_ctx *ctx = crypto_aead_ctx(ctfm); + + if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP && + len > EIP197_AEAD_IPSEC_NONCE_SIZE) { + /* ESP variant has nonce appended to key */ + len -= EIP197_AEAD_IPSEC_NONCE_SIZE; + ctx->nonce = *(u32 *)(key + len); + } + if (len != CHACHA_KEY_SIZE) { + crypto_aead_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + safexcel_chacha20_setkey(ctx, key); + + return 0; +} + +static int safexcel_aead_chachapoly_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + if (authsize != POLY1305_DIGEST_SIZE) + return -EINVAL; + return 0; +} + +static int safexcel_aead_chachapoly_crypt(struct aead_request *req, + enum safexcel_cipher_direction dir) +{ + struct safexcel_cipher_req *creq = aead_request_ctx(req); + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct crypto_tfm *tfm = crypto_aead_tfm(aead); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct aead_request *subreq = aead_request_ctx(req); + u32 key[CHACHA_KEY_SIZE / sizeof(u32) + 1]; + int i, ret = 0; + + /* + * Instead of wasting time detecting umpteen silly corner cases, + * just dump all "small" requests to the fallback implementation. + * HW would not be faster on such small requests anyway. + */ + if (likely((ctx->aead != EIP197_AEAD_TYPE_IPSEC_ESP || + req->assoclen >= EIP197_AEAD_IPSEC_IV_SIZE) && + req->cryptlen > POLY1305_DIGEST_SIZE)) { + return safexcel_queue_req(&req->base, creq, dir); + } + + /* HW cannot do full (AAD+payload) zero length, use fallback */ + for (i = 0; i < CHACHA_KEY_SIZE / sizeof(u32); i++) + key[i] = cpu_to_le32(ctx->key[i]); + if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) { + /* ESP variant has nonce appended to the key */ + key[CHACHA_KEY_SIZE / sizeof(u32)] = ctx->nonce; + ret = crypto_aead_setkey(ctx->fback, (u8 *)key, + CHACHA_KEY_SIZE + + EIP197_AEAD_IPSEC_NONCE_SIZE); + } else { + ret = crypto_aead_setkey(ctx->fback, (u8 *)key, + CHACHA_KEY_SIZE); + } + if (ret) { + crypto_aead_clear_flags(aead, CRYPTO_TFM_REQ_MASK); + crypto_aead_set_flags(aead, crypto_aead_get_flags(ctx->fback) & + CRYPTO_TFM_REQ_MASK); + return ret; + } + + aead_request_set_tfm(subreq, ctx->fback); + aead_request_set_callback(subreq, req->base.flags, req->base.complete, + req->base.data); + aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, + req->iv); + aead_request_set_ad(subreq, req->assoclen); + + return (dir == SAFEXCEL_ENCRYPT) ? + crypto_aead_encrypt(subreq) : + crypto_aead_decrypt(subreq); +} + +static int safexcel_aead_chachapoly_encrypt(struct aead_request *req) +{ + return safexcel_aead_chachapoly_crypt(req, SAFEXCEL_ENCRYPT); +} + +static int safexcel_aead_chachapoly_decrypt(struct aead_request *req) +{ + return safexcel_aead_chachapoly_crypt(req, SAFEXCEL_DECRYPT); +} + +static int safexcel_aead_chachapoly_cra_init(struct crypto_tfm *tfm) +{ + struct crypto_aead *aead = __crypto_aead_cast(tfm); + struct aead_alg *alg = crypto_aead_alg(aead); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_cra_init(tfm); + ctx->alg = SAFEXCEL_CHACHA20; + ctx->mode = CONTEXT_CONTROL_CHACHA20_MODE_256_32 | + CONTEXT_CONTROL_CHACHA20_MODE_CALC_OTK; + ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_POLY1305; + ctx->state_sz = 0; /* Precomputed by HW */ + + /* Allocate fallback implementation */ + ctx->fback = crypto_alloc_aead(alg->base.cra_name, 0, + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->fback)) + return PTR_ERR(ctx->fback); + + crypto_aead_set_reqsize(aead, max(sizeof(struct safexcel_cipher_req), + sizeof(struct aead_request) + + crypto_aead_reqsize(ctx->fback))); + + return 0; +} + +static void safexcel_aead_chachapoly_cra_exit(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + crypto_free_aead(ctx->fback); + safexcel_aead_cra_exit(tfm); +} + +struct safexcel_alg_template safexcel_alg_chachapoly = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_CHACHA20 | SAFEXCEL_ALG_POLY1305, + .alg.aead = { + .setkey = safexcel_aead_chachapoly_setkey, + .setauthsize = safexcel_aead_chachapoly_setauthsize, + .encrypt = safexcel_aead_chachapoly_encrypt, + .decrypt = safexcel_aead_chachapoly_decrypt, + .ivsize = CHACHAPOLY_IV_SIZE, + .maxauthsize = POLY1305_DIGEST_SIZE, + .base = { + .cra_name = "rfc7539(chacha20,poly1305)", + .cra_driver_name = "safexcel-chacha20-poly1305", + /* +1 to put it above HW chacha + SW poly */ + .cra_priority = SAFEXCEL_CRA_PRIORITY + 1, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_chachapoly_cra_init, + .cra_exit = safexcel_aead_chachapoly_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_chachapolyesp_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + int ret; + + ret = safexcel_aead_chachapoly_cra_init(tfm); + ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP; + return ret; +} + +struct safexcel_alg_template safexcel_alg_chachapoly_esp = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_CHACHA20 | SAFEXCEL_ALG_POLY1305, + .alg.aead = { + .setkey = safexcel_aead_chachapoly_setkey, + .setauthsize = safexcel_aead_chachapoly_setauthsize, + .encrypt = safexcel_aead_chachapoly_encrypt, + .decrypt = safexcel_aead_chachapoly_decrypt, + .ivsize = CHACHAPOLY_IV_SIZE - EIP197_AEAD_IPSEC_NONCE_SIZE, + .maxauthsize = POLY1305_DIGEST_SIZE, + .base = { + .cra_name = "rfc7539esp(chacha20,poly1305)", + .cra_driver_name = "safexcel-chacha20-poly1305-esp", + /* +1 to put it above HW chacha + SW poly */ + .cra_priority = SAFEXCEL_CRA_PRIORITY + 1, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_chachapolyesp_cra_init, + .cra_exit = safexcel_aead_chachapoly_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; -- cgit From 0f2bc13181ce22d365929176a3d3d47fb857c202 Mon Sep 17 00:00:00 2001 From: Pascal van Leeuwen Date: Fri, 13 Sep 2019 17:20:36 +0200 Subject: crypto: inside-secure - Added support for basic SM3 ahash Added support for the SM3 ahash algorithm changes since v1: - moved definition of CONTEXT_CONTROL_CRYPTO_ALG_SM3 (0x7) up above 0xf changes since v2: - allow compilation if CONFIG_CRYPTO_SM3 is not set Acked-by: Antoine Tenart Signed-off-by: Pascal van Leeuwen Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 1 + drivers/crypto/inside-secure/safexcel.h | 8 ++++ drivers/crypto/inside-secure/safexcel_hash.c | 64 ++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+) (limited to 'drivers/crypto/inside-secure/safexcel.c') diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index b81f0bc5cd50..32682e0cf84e 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1176,6 +1176,7 @@ static struct safexcel_alg_template *safexcel_algs[] = { &safexcel_alg_chacha20, &safexcel_alg_chachapoly, &safexcel_alg_chachapoly_esp, + &safexcel_alg_sm3, }; static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv) diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 282d59e523e9..e2993b58e596 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -373,6 +373,7 @@ struct safexcel_context_record { #define CONTEXT_CONTROL_CRYPTO_ALG_XCBC128 (0x1 << 23) #define CONTEXT_CONTROL_CRYPTO_ALG_XCBC192 (0x2 << 23) #define CONTEXT_CONTROL_CRYPTO_ALG_XCBC256 (0x3 << 23) +#define CONTEXT_CONTROL_CRYPTO_ALG_SM3 (0x7 << 23) #define CONTEXT_CONTROL_CRYPTO_ALG_POLY1305 (0xf << 23) #define CONTEXT_CONTROL_INV_FR (0x5 << 24) #define CONTEXT_CONTROL_INV_TR (0x6 << 24) @@ -663,6 +664,12 @@ enum safexcel_eip_version { /* Priority we use for advertising our algorithms */ #define SAFEXCEL_CRA_PRIORITY 300 +/* SM3 digest result for zero length message */ +#define EIP197_SM3_ZEROM_HASH "\x1A\xB2\x1D\x83\x55\xCF\xA1\x7F" \ + "\x8E\x61\x19\x48\x31\xE8\x1A\x8F" \ + "\x22\xBE\xC8\xC7\x28\xFE\xFB\x74" \ + "\x7E\xD0\x35\xEB\x50\x82\xAA\x2B" + /* EIP algorithm presence flags */ enum safexcel_eip_algorithms { SAFEXCEL_ALG_BC0 = BIT(5), @@ -869,5 +876,6 @@ extern struct safexcel_alg_template safexcel_alg_cmac; extern struct safexcel_alg_template safexcel_alg_chacha20; extern struct safexcel_alg_template safexcel_alg_chachapoly; extern struct safexcel_alg_template safexcel_alg_chachapoly_esp; +extern struct safexcel_alg_template safexcel_alg_sm3; #endif diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index 0224779f2984..873b77449b90 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -776,6 +777,14 @@ static int safexcel_ahash_final(struct ahash_request *areq) else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA512) memcpy(areq->result, sha512_zero_message_hash, SHA512_DIGEST_SIZE); + else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SM3) { + if (IS_ENABLED(CONFIG_CRYPTO_SM3)) + memcpy(areq->result, sm3_zero_message_hash, + SM3_DIGEST_SIZE); + else + memcpy(areq->result, + EIP197_SM3_ZEROM_HASH, SM3_DIGEST_SIZE); + } return 0; } else if (unlikely(req->digest == CONTEXT_CONTROL_DIGEST_XCM && @@ -2221,3 +2230,58 @@ struct safexcel_alg_template safexcel_alg_cmac = { }, }, }; + +static int safexcel_sm3_init(struct ahash_request *areq) +{ + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + struct safexcel_ahash_req *req = ahash_request_ctx(areq); + + memset(req, 0, sizeof(*req)); + + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3; + req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; + req->state_sz = SM3_DIGEST_SIZE; + req->block_sz = SM3_BLOCK_SIZE; + + return 0; +} + +static int safexcel_sm3_digest(struct ahash_request *areq) +{ + int ret = safexcel_sm3_init(areq); + + if (ret) + return ret; + + return safexcel_ahash_finup(areq); +} + +struct safexcel_alg_template safexcel_alg_sm3 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = SAFEXCEL_ALG_SM3, + .alg.ahash = { + .init = safexcel_sm3_init, + .update = safexcel_ahash_update, + .final = safexcel_ahash_final, + .finup = safexcel_ahash_finup, + .digest = safexcel_sm3_digest, + .export = safexcel_ahash_export, + .import = safexcel_ahash_import, + .halg = { + .digestsize = SM3_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "sm3", + .cra_driver_name = "safexcel-sm3", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SM3_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_ahash_cra_init, + .cra_exit = safexcel_ahash_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; -- cgit From aa3a43e6e41901a82e47aee172df1eb406470f94 Mon Sep 17 00:00:00 2001 From: Pascal van Leeuwen Date: Fri, 13 Sep 2019 17:20:37 +0200 Subject: crypto: inside-secure - Added support for HMAC-SM3 ahash Added support for the hmac(sm3) ahash authentication algorithm changes since v1: - added Acked-by tag below, no changes to the source changes since v2: - nothing Acked-by: Antoine Tenart Signed-off-by: Pascal van Leeuwen Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 1 + drivers/crypto/inside-secure/safexcel.h | 1 + drivers/crypto/inside-secure/safexcel_hash.c | 70 ++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+) (limited to 'drivers/crypto/inside-secure/safexcel.c') diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 32682e0cf84e..87d431afd44d 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1177,6 +1177,7 @@ static struct safexcel_alg_template *safexcel_algs[] = { &safexcel_alg_chachapoly, &safexcel_alg_chachapoly_esp, &safexcel_alg_sm3, + &safexcel_alg_hmac_sm3, }; static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv) diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index e2993b58e596..1b2d709dff39 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -877,5 +877,6 @@ extern struct safexcel_alg_template safexcel_alg_chacha20; extern struct safexcel_alg_template safexcel_alg_chachapoly; extern struct safexcel_alg_template safexcel_alg_chachapoly_esp; extern struct safexcel_alg_template safexcel_alg_sm3; +extern struct safexcel_alg_template safexcel_alg_hmac_sm3; #endif diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index 873b77449b90..272e5fd0de7e 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -2285,3 +2285,73 @@ struct safexcel_alg_template safexcel_alg_sm3 = { }, }, }; + +static int safexcel_hmac_sm3_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sm3", + SM3_DIGEST_SIZE); +} + +static int safexcel_hmac_sm3_init(struct ahash_request *areq) +{ + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + struct safexcel_ahash_req *req = ahash_request_ctx(areq); + + memset(req, 0, sizeof(*req)); + + /* Start from ipad precompute */ + memcpy(req->state, ctx->ipad, SM3_DIGEST_SIZE); + /* Already processed the key^ipad part now! */ + req->len = SM3_BLOCK_SIZE; + req->processed = SM3_BLOCK_SIZE; + + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3; + req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; + req->state_sz = SM3_DIGEST_SIZE; + req->block_sz = SM3_BLOCK_SIZE; + req->hmac = true; + + return 0; +} + +static int safexcel_hmac_sm3_digest(struct ahash_request *areq) +{ + int ret = safexcel_hmac_sm3_init(areq); + + if (ret) + return ret; + + return safexcel_ahash_finup(areq); +} + +struct safexcel_alg_template safexcel_alg_hmac_sm3 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = SAFEXCEL_ALG_SM3, + .alg.ahash = { + .init = safexcel_hmac_sm3_init, + .update = safexcel_ahash_update, + .final = safexcel_ahash_final, + .finup = safexcel_ahash_finup, + .digest = safexcel_hmac_sm3_digest, + .setkey = safexcel_hmac_sm3_setkey, + .export = safexcel_ahash_export, + .import = safexcel_ahash_import, + .halg = { + .digestsize = SM3_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "hmac(sm3)", + .cra_driver_name = "safexcel-hmac-sm3", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SM3_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_ahash_cra_init, + .cra_exit = safexcel_ahash_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; -- cgit From fcca797d2adb83be198db8530f1fd4f6c60e81fe Mon Sep 17 00:00:00 2001 From: Pascal van Leeuwen Date: Fri, 13 Sep 2019 11:10:36 +0200 Subject: crypto: inside-secure - Add support for the ecb(sm4) skcipher This patch adds support for SM4 in ECB mode, i.e. skcipher ecb(sm4). changes since v1: - make SAFEXCEL_SM4 case entry explit, using the proper SM4_BLOCK_SIZE instead of "borrowing" the AES code which "coincidentally" works Signed-off-by: Pascal van Leeuwen Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 1 + drivers/crypto/inside-secure/safexcel.h | 2 + drivers/crypto/inside-secure/safexcel_cipher.c | 94 ++++++++++++++++++++++++++ 3 files changed, 97 insertions(+) (limited to 'drivers/crypto/inside-secure/safexcel.c') diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 87d431afd44d..0ac3b45b7b69 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1178,6 +1178,7 @@ static struct safexcel_alg_template *safexcel_algs[] = { &safexcel_alg_chachapoly_esp, &safexcel_alg_sm3, &safexcel_alg_hmac_sm3, + &safexcel_alg_ecb_sm4, }; static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv) diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 1b2d709dff39..3a91cc26ed19 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -359,6 +359,7 @@ struct safexcel_context_record { #define CONTEXT_CONTROL_CRYPTO_ALG_AES192 (0x6 << 17) #define CONTEXT_CONTROL_CRYPTO_ALG_AES256 (0x7 << 17) #define CONTEXT_CONTROL_CRYPTO_ALG_CHACHA20 (0x8 << 17) +#define CONTEXT_CONTROL_CRYPTO_ALG_SM4 (0xd << 17) #define CONTEXT_CONTROL_DIGEST_PRECOMPUTED (0x1 << 21) #define CONTEXT_CONTROL_DIGEST_XCM (0x2 << 21) #define CONTEXT_CONTROL_DIGEST_HMAC (0x3 << 21) @@ -878,5 +879,6 @@ extern struct safexcel_alg_template safexcel_alg_chachapoly; extern struct safexcel_alg_template safexcel_alg_chachapoly_esp; extern struct safexcel_alg_template safexcel_alg_sm3; extern struct safexcel_alg_template safexcel_alg_hmac_sm3; +extern struct safexcel_alg_template safexcel_alg_ecb_sm4; #endif diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index f1e6cae9e1c8..4d7492455c7c 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -36,6 +37,7 @@ enum safexcel_cipher_alg { SAFEXCEL_3DES, SAFEXCEL_AES, SAFEXCEL_CHACHA20, + SAFEXCEL_SM4, }; struct safexcel_cipher_ctx { @@ -139,6 +141,10 @@ static void safexcel_cipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv, block_sz = DES3_EDE_BLOCK_SIZE; cdesc->control_data.options |= EIP197_OPTION_2_TOKEN_IV_CMD; break; + case SAFEXCEL_SM4: + block_sz = SM4_BLOCK_SIZE; + cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; + break; case SAFEXCEL_AES: block_sz = AES_BLOCK_SIZE; cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; @@ -526,6 +532,9 @@ static int safexcel_context_control(struct safexcel_cipher_ctx *ctx, } else if (ctx->alg == SAFEXCEL_CHACHA20) { cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_CHACHA20; + } else if (ctx->alg == SAFEXCEL_SM4) { + cdesc->control_data.control0 |= + CONTEXT_CONTROL_CRYPTO_ALG_SM4; } return 0; @@ -2613,3 +2622,88 @@ struct safexcel_alg_template safexcel_alg_chachapoly_esp = { }, }, }; + +static int safexcel_skcipher_sm4_setkey(struct crypto_skcipher *ctfm, + const u8 *key, unsigned int len) +{ + struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct safexcel_crypto_priv *priv = ctx->priv; + int i; + + if (len != SM4_KEY_SIZE) { + crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { + for (i = 0; i < SM4_KEY_SIZE / sizeof(u32); i++) { + if (ctx->key[i] != + get_unaligned_le32(key + i * sizeof(u32))) { + ctx->base.needs_inv = true; + break; + } + } + } + + for (i = 0; i < SM4_KEY_SIZE / sizeof(u32); i++) + ctx->key[i] = get_unaligned_le32(key + i * sizeof(u32)); + ctx->key_len = SM4_KEY_SIZE; + + return 0; +} + +static int safexcel_sm4_blk_encrypt(struct skcipher_request *req) +{ + /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ + if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) + return -EINVAL; + else + return safexcel_queue_req(&req->base, skcipher_request_ctx(req), + SAFEXCEL_ENCRYPT); +} + +static int safexcel_sm4_blk_decrypt(struct skcipher_request *req) +{ + /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ + if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) + return -EINVAL; + else + return safexcel_queue_req(&req->base, skcipher_request_ctx(req), + SAFEXCEL_DECRYPT); +} + +static int safexcel_skcipher_sm4_ecb_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_skcipher_cra_init(tfm); + ctx->alg = SAFEXCEL_SM4; + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB; + return 0; +} + +struct safexcel_alg_template safexcel_alg_ecb_sm4 = { + .type = SAFEXCEL_ALG_TYPE_SKCIPHER, + .algo_mask = SAFEXCEL_ALG_SM4, + .alg.skcipher = { + .setkey = safexcel_skcipher_sm4_setkey, + .encrypt = safexcel_sm4_blk_encrypt, + .decrypt = safexcel_sm4_blk_decrypt, + .min_keysize = SM4_KEY_SIZE, + .max_keysize = SM4_KEY_SIZE, + .base = { + .cra_name = "ecb(sm4)", + .cra_driver_name = "safexcel-ecb-sm4", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SM4_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_skcipher_sm4_ecb_cra_init, + .cra_exit = safexcel_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; -- cgit From 6f2d14288ec1d659503c37e965de825131462a1a Mon Sep 17 00:00:00 2001 From: Pascal van Leeuwen Date: Fri, 13 Sep 2019 11:10:37 +0200 Subject: crypto: inside-secure - Add support for the cbc(sm4) skcipher This patch adds support for SM4 in CBC mode, i.e. skcipher cbc(sm4). changes since v1: - nothing Signed-off-by: Pascal van Leeuwen Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 1 + drivers/crypto/inside-secure/safexcel.h | 1 + drivers/crypto/inside-secure/safexcel_cipher.c | 36 ++++++++++++++++++++++++++ 3 files changed, 38 insertions(+) (limited to 'drivers/crypto/inside-secure/safexcel.c') diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 0ac3b45b7b69..ac1d5b89cd8d 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1179,6 +1179,7 @@ static struct safexcel_alg_template *safexcel_algs[] = { &safexcel_alg_sm3, &safexcel_alg_hmac_sm3, &safexcel_alg_ecb_sm4, + &safexcel_alg_cbc_sm4, }; static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv) diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 3a91cc26ed19..7c63590f4e5d 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -880,5 +880,6 @@ extern struct safexcel_alg_template safexcel_alg_chachapoly_esp; extern struct safexcel_alg_template safexcel_alg_sm3; extern struct safexcel_alg_template safexcel_alg_hmac_sm3; extern struct safexcel_alg_template safexcel_alg_ecb_sm4; +extern struct safexcel_alg_template safexcel_alg_cbc_sm4; #endif diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index 4d7492455c7c..f5d39e1a74e7 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -2707,3 +2707,39 @@ struct safexcel_alg_template safexcel_alg_ecb_sm4 = { }, }, }; + +static int safexcel_skcipher_sm4_cbc_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_skcipher_cra_init(tfm); + ctx->alg = SAFEXCEL_SM4; + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; + return 0; +} + +struct safexcel_alg_template safexcel_alg_cbc_sm4 = { + .type = SAFEXCEL_ALG_TYPE_SKCIPHER, + .algo_mask = SAFEXCEL_ALG_SM4, + .alg.skcipher = { + .setkey = safexcel_skcipher_sm4_setkey, + .encrypt = safexcel_sm4_blk_encrypt, + .decrypt = safexcel_sm4_blk_decrypt, + .min_keysize = SM4_KEY_SIZE, + .max_keysize = SM4_KEY_SIZE, + .ivsize = SM4_BLOCK_SIZE, + .base = { + .cra_name = "cbc(sm4)", + .cra_driver_name = "safexcel-cbc-sm4", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SM4_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_skcipher_sm4_cbc_cra_init, + .cra_exit = safexcel_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; -- cgit From 03a6cfb97f8f28373a6fed68deefccb5e98e1072 Mon Sep 17 00:00:00 2001 From: Pascal van Leeuwen Date: Fri, 13 Sep 2019 11:10:38 +0200 Subject: crypto: inside-secure - Add support for the ofb(sm4) skcipher This patch adds support for SM4 in OFB mode, i.e. skcipher ofb(sm4). changes since v1: - nothing Signed-off-by: Pascal van Leeuwen Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 1 + drivers/crypto/inside-secure/safexcel.h | 1 + drivers/crypto/inside-secure/safexcel_cipher.c | 36 ++++++++++++++++++++++++++ 3 files changed, 38 insertions(+) (limited to 'drivers/crypto/inside-secure/safexcel.c') diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index ac1d5b89cd8d..e3873f2db7d3 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1180,6 +1180,7 @@ static struct safexcel_alg_template *safexcel_algs[] = { &safexcel_alg_hmac_sm3, &safexcel_alg_ecb_sm4, &safexcel_alg_cbc_sm4, + &safexcel_alg_ofb_sm4, }; static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv) diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 7c63590f4e5d..765f5e8706bd 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -881,5 +881,6 @@ extern struct safexcel_alg_template safexcel_alg_sm3; extern struct safexcel_alg_template safexcel_alg_hmac_sm3; extern struct safexcel_alg_template safexcel_alg_ecb_sm4; extern struct safexcel_alg_template safexcel_alg_cbc_sm4; +extern struct safexcel_alg_template safexcel_alg_ofb_sm4; #endif diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index f5d39e1a74e7..b0e4bfb2e518 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -2743,3 +2743,39 @@ struct safexcel_alg_template safexcel_alg_cbc_sm4 = { }, }, }; + +static int safexcel_skcipher_sm4_ofb_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_skcipher_cra_init(tfm); + ctx->alg = SAFEXCEL_SM4; + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_OFB; + return 0; +} + +struct safexcel_alg_template safexcel_alg_ofb_sm4 = { + .type = SAFEXCEL_ALG_TYPE_SKCIPHER, + .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_AES_XFB, + .alg.skcipher = { + .setkey = safexcel_skcipher_sm4_setkey, + .encrypt = safexcel_encrypt, + .decrypt = safexcel_decrypt, + .min_keysize = SM4_KEY_SIZE, + .max_keysize = SM4_KEY_SIZE, + .ivsize = SM4_BLOCK_SIZE, + .base = { + .cra_name = "ofb(sm4)", + .cra_driver_name = "safexcel-ofb-sm4", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_skcipher_sm4_ofb_cra_init, + .cra_exit = safexcel_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; -- cgit From 7468ab22d7ab968b99c75552c879cb5d96e50523 Mon Sep 17 00:00:00 2001 From: Pascal van Leeuwen Date: Fri, 13 Sep 2019 11:10:40 +0200 Subject: crypto: inside-secure - Add support for the cfb(sm4) skcipher This patch adds support for SM4 in CFB mode, i.e. skcipher cfb(sm4). changes since v1: - nothing Signed-off-by: Pascal van Leeuwen Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 1 + drivers/crypto/inside-secure/safexcel.h | 1 + drivers/crypto/inside-secure/safexcel_cipher.c | 36 ++++++++++++++++++++++++++ 3 files changed, 38 insertions(+) (limited to 'drivers/crypto/inside-secure/safexcel.c') diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index e3873f2db7d3..be39f0bbe909 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1181,6 +1181,7 @@ static struct safexcel_alg_template *safexcel_algs[] = { &safexcel_alg_ecb_sm4, &safexcel_alg_cbc_sm4, &safexcel_alg_ofb_sm4, + &safexcel_alg_cfb_sm4, }; static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv) diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 765f5e8706bd..3f729a604136 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -882,5 +882,6 @@ extern struct safexcel_alg_template safexcel_alg_hmac_sm3; extern struct safexcel_alg_template safexcel_alg_ecb_sm4; extern struct safexcel_alg_template safexcel_alg_cbc_sm4; extern struct safexcel_alg_template safexcel_alg_ofb_sm4; +extern struct safexcel_alg_template safexcel_alg_cfb_sm4; #endif diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index b0e4bfb2e518..583cc8331706 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -2779,3 +2779,39 @@ struct safexcel_alg_template safexcel_alg_ofb_sm4 = { }, }, }; + +static int safexcel_skcipher_sm4_cfb_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_skcipher_cra_init(tfm); + ctx->alg = SAFEXCEL_SM4; + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CFB; + return 0; +} + +struct safexcel_alg_template safexcel_alg_cfb_sm4 = { + .type = SAFEXCEL_ALG_TYPE_SKCIPHER, + .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_AES_XFB, + .alg.skcipher = { + .setkey = safexcel_skcipher_sm4_setkey, + .encrypt = safexcel_encrypt, + .decrypt = safexcel_decrypt, + .min_keysize = SM4_KEY_SIZE, + .max_keysize = SM4_KEY_SIZE, + .ivsize = SM4_BLOCK_SIZE, + .base = { + .cra_name = "cfb(sm4)", + .cra_driver_name = "safexcel-cfb-sm4", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_skcipher_sm4_cfb_cra_init, + .cra_exit = safexcel_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; -- cgit From f77e5dc08ad4eaf2b3428276cfd18ce55dd6b251 Mon Sep 17 00:00:00 2001 From: Pascal van Leeuwen Date: Fri, 13 Sep 2019 11:10:41 +0200 Subject: crypto: inside-secure - Add support for the rfc3685(ctr(sm4)) skcipher This patch adds support for SM4 in (32 bit) CTR mode, i.e. skcipher rfc3686(ctr(sm4)). changes since v1: - nothing Signed-off-by: Pascal van Leeuwen Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 1 + drivers/crypto/inside-secure/safexcel.h | 1 + drivers/crypto/inside-secure/safexcel_cipher.c | 51 ++++++++++++++++++++++++++ 3 files changed, 53 insertions(+) (limited to 'drivers/crypto/inside-secure/safexcel.c') diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index be39f0bbe909..3c140d8c3ae4 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1182,6 +1182,7 @@ static struct safexcel_alg_template *safexcel_algs[] = { &safexcel_alg_cbc_sm4, &safexcel_alg_ofb_sm4, &safexcel_alg_cfb_sm4, + &safexcel_alg_ctr_sm4, }; static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv) diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 3f729a604136..62965fb63dc5 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -883,5 +883,6 @@ extern struct safexcel_alg_template safexcel_alg_ecb_sm4; extern struct safexcel_alg_template safexcel_alg_cbc_sm4; extern struct safexcel_alg_template safexcel_alg_ofb_sm4; extern struct safexcel_alg_template safexcel_alg_cfb_sm4; +extern struct safexcel_alg_template safexcel_alg_ctr_sm4; #endif diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index 583cc8331706..ed1b186a5535 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -2815,3 +2815,54 @@ struct safexcel_alg_template safexcel_alg_cfb_sm4 = { }, }, }; + +static int safexcel_skcipher_sm4ctr_setkey(struct crypto_skcipher *ctfm, + const u8 *key, unsigned int len) +{ + struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + /* last 4 bytes of key are the nonce! */ + ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE); + /* exclude the nonce here */ + len -= CTR_RFC3686_NONCE_SIZE; + + return safexcel_skcipher_sm4_setkey(ctfm, key, len); +} + +static int safexcel_skcipher_sm4_ctr_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_skcipher_cra_init(tfm); + ctx->alg = SAFEXCEL_SM4; + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; + return 0; +} + +struct safexcel_alg_template safexcel_alg_ctr_sm4 = { + .type = SAFEXCEL_ALG_TYPE_SKCIPHER, + .algo_mask = SAFEXCEL_ALG_SM4, + .alg.skcipher = { + .setkey = safexcel_skcipher_sm4ctr_setkey, + .encrypt = safexcel_encrypt, + .decrypt = safexcel_decrypt, + /* Add nonce size */ + .min_keysize = SM4_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, + .max_keysize = SM4_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, + .ivsize = CTR_RFC3686_IV_SIZE, + .base = { + .cra_name = "rfc3686(ctr(sm4))", + .cra_driver_name = "safexcel-ctr-sm4", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_skcipher_sm4_ctr_cra_init, + .cra_exit = safexcel_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; -- cgit From 1769f704e55b11e97ff4e52226e6fbddaa1e654e Mon Sep 17 00:00:00 2001 From: Pascal van Leeuwen Date: Fri, 13 Sep 2019 20:44:50 +0200 Subject: crypto: inside-secure - Add SM4 based authenc AEAD ciphersuites This patch adds support for the authenc(hmac(sha1),cbc(sm4)), authenc(hmac(sm3),cbc(sm4)), authenc(hmac(sha1),rfc3686(ctr(sm4))), and authenc(hmac(sm3),rfc3686(ctr(sm4))) aead ciphersuites. These are necessary to support IPsec according to the Chinese standard GM/T 022-1014 - IPsec VPN specification. Note that there are no testvectors present in testmgr for these ciphersuites. However, considering all building blocks have already been verified elsewhere, it is fair to assume the generic implementation to be correct-by-construction. The hardware implementation has been fuzzed against this generic implementation by means of a locally modified testmgr. The intention is to upstream these testmgr changes but this is pending other testmgr changes being made by Eric Biggers. The patch has been tested with the eip197c_iewxkbc configuration on the Xilinx VCU118 development board, using the abovementioned modified testmgr This patch applies on top of "Add support for SM4 ciphers" and needs to be applied before "Add (HMAC) SHA3 support". Signed-off-by: Pascal van Leeuwen Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 4 + drivers/crypto/inside-secure/safexcel.h | 4 + drivers/crypto/inside-secure/safexcel_cipher.c | 280 +++++++++++++++++++++++-- 3 files changed, 274 insertions(+), 14 deletions(-) (limited to 'drivers/crypto/inside-secure/safexcel.c') diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 3c140d8c3ae4..8f7fdd023262 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1183,6 +1183,10 @@ static struct safexcel_alg_template *safexcel_algs[] = { &safexcel_alg_ofb_sm4, &safexcel_alg_cfb_sm4, &safexcel_alg_ctr_sm4, + &safexcel_alg_authenc_hmac_sha1_cbc_sm4, + &safexcel_alg_authenc_hmac_sm3_cbc_sm4, + &safexcel_alg_authenc_hmac_sha1_ctr_sm4, + &safexcel_alg_authenc_hmac_sm3_ctr_sm4, }; static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv) diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 62965fb63dc5..1d75044c3e97 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -884,5 +884,9 @@ extern struct safexcel_alg_template safexcel_alg_cbc_sm4; extern struct safexcel_alg_template safexcel_alg_ofb_sm4; extern struct safexcel_alg_template safexcel_alg_cfb_sm4; extern struct safexcel_alg_template safexcel_alg_ctr_sm4; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_sm4; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sm3_cbc_sm4; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_ctr_sm4; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sm3_ctr_sm4; #endif diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index ed1b186a5535..226b76180b90 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -349,19 +350,18 @@ static int safexcel_aead_setkey(struct crypto_aead *ctfm, const u8 *key, struct crypto_aes_ctx aes; int err = -EINVAL; - if (crypto_authenc_extractkeys(&keys, key, len) != 0) + if (unlikely(crypto_authenc_extractkeys(&keys, key, len))) goto badkey; if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD) { - /* Minimum keysize is minimum AES key size + nonce size */ - if (keys.enckeylen < (AES_MIN_KEY_SIZE + - CTR_RFC3686_NONCE_SIZE)) + /* Must have at least space for the nonce here */ + if (unlikely(keys.enckeylen < CTR_RFC3686_NONCE_SIZE)) goto badkey; /* last 4 bytes of key are the nonce! */ ctx->nonce = *(u32 *)(keys.enckey + keys.enckeylen - CTR_RFC3686_NONCE_SIZE); /* exclude the nonce here */ - keys.enckeylen -= CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; + keys.enckeylen -= CTR_RFC3686_NONCE_SIZE; } /* Encryption key */ @@ -376,6 +376,10 @@ static int safexcel_aead_setkey(struct crypto_aead *ctfm, const u8 *key, if (unlikely(err)) goto badkey; break; + case SAFEXCEL_SM4: + if (unlikely(keys.enckeylen != SM4_KEY_SIZE)) + goto badkey; + break; default: dev_err(priv->dev, "aead: unsupported cipher algorithm\n"); goto badkey; @@ -412,6 +416,11 @@ static int safexcel_aead_setkey(struct crypto_aead *ctfm, const u8 *key, keys.authkeylen, &istate, &ostate)) goto badkey; break; + case CONTEXT_CONTROL_CRYPTO_ALG_SM3: + if (safexcel_hmac_setkey("safexcel-sm3", keys.authkey, + keys.authkeylen, &istate, &ostate)) + goto badkey; + break; default: dev_err(priv->dev, "aead: unsupported hash algorithmn"); goto badkey; @@ -2522,18 +2531,13 @@ static int safexcel_aead_chachapoly_decrypt(struct aead_request *req) return safexcel_aead_chachapoly_crypt(req, SAFEXCEL_DECRYPT); } -static int safexcel_aead_chachapoly_cra_init(struct crypto_tfm *tfm) +static int safexcel_aead_fallback_cra_init(struct crypto_tfm *tfm) { struct crypto_aead *aead = __crypto_aead_cast(tfm); struct aead_alg *alg = crypto_aead_alg(aead); struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); safexcel_aead_cra_init(tfm); - ctx->alg = SAFEXCEL_CHACHA20; - ctx->mode = CONTEXT_CONTROL_CHACHA20_MODE_256_32 | - CONTEXT_CONTROL_CHACHA20_MODE_CALC_OTK; - ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_POLY1305; - ctx->state_sz = 0; /* Precomputed by HW */ /* Allocate fallback implementation */ ctx->fback = crypto_alloc_aead(alg->base.cra_name, 0, @@ -2549,7 +2553,20 @@ static int safexcel_aead_chachapoly_cra_init(struct crypto_tfm *tfm) return 0; } -static void safexcel_aead_chachapoly_cra_exit(struct crypto_tfm *tfm) +static int safexcel_aead_chachapoly_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_fallback_cra_init(tfm); + ctx->alg = SAFEXCEL_CHACHA20; + ctx->mode = CONTEXT_CONTROL_CHACHA20_MODE_256_32 | + CONTEXT_CONTROL_CHACHA20_MODE_CALC_OTK; + ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_POLY1305; + ctx->state_sz = 0; /* Precomputed by HW */ + return 0; +} + +static void safexcel_aead_fallback_cra_exit(struct crypto_tfm *tfm) { struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); @@ -2579,7 +2596,7 @@ struct safexcel_alg_template safexcel_alg_chachapoly = { .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), .cra_alignmask = 0, .cra_init = safexcel_aead_chachapoly_cra_init, - .cra_exit = safexcel_aead_chachapoly_cra_exit, + .cra_exit = safexcel_aead_fallback_cra_exit, .cra_module = THIS_MODULE, }, }, @@ -2617,7 +2634,7 @@ struct safexcel_alg_template safexcel_alg_chachapoly_esp = { .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), .cra_alignmask = 0, .cra_init = safexcel_aead_chachapolyesp_cra_init, - .cra_exit = safexcel_aead_chachapoly_cra_exit, + .cra_exit = safexcel_aead_fallback_cra_exit, .cra_module = THIS_MODULE, }, }, @@ -2866,3 +2883,238 @@ struct safexcel_alg_template safexcel_alg_ctr_sm4 = { }, }, }; + +static int safexcel_aead_sm4_blk_encrypt(struct aead_request *req) +{ + /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ + if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) + return -EINVAL; + + return safexcel_queue_req(&req->base, aead_request_ctx(req), + SAFEXCEL_ENCRYPT); +} + +static int safexcel_aead_sm4_blk_decrypt(struct aead_request *req) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + + /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ + if ((req->cryptlen - crypto_aead_authsize(tfm)) & (SM4_BLOCK_SIZE - 1)) + return -EINVAL; + + return safexcel_queue_req(&req->base, aead_request_ctx(req), + SAFEXCEL_DECRYPT); +} + +static int safexcel_aead_sm4cbc_sha1_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_cra_init(tfm); + ctx->alg = SAFEXCEL_SM4; + ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1; + ctx->state_sz = SHA1_DIGEST_SIZE; + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_sm4 = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SHA1, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_sm4_blk_encrypt, + .decrypt = safexcel_aead_sm4_blk_decrypt, + .ivsize = SM4_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha1),cbc(sm4))", + .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-sm4", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SM4_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sm4cbc_sha1_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_fallback_setkey(struct crypto_aead *ctfm, + const u8 *key, unsigned int len) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + /* Keep fallback cipher synchronized */ + return crypto_aead_setkey(ctx->fback, (u8 *)key, len) ?: + safexcel_aead_setkey(ctfm, key, len); +} + +static int safexcel_aead_fallback_setauthsize(struct crypto_aead *ctfm, + unsigned int authsize) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + /* Keep fallback cipher synchronized */ + return crypto_aead_setauthsize(ctx->fback, authsize); +} + +static int safexcel_aead_fallback_crypt(struct aead_request *req, + enum safexcel_cipher_direction dir) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct crypto_tfm *tfm = crypto_aead_tfm(aead); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct aead_request *subreq = aead_request_ctx(req); + + aead_request_set_tfm(subreq, ctx->fback); + aead_request_set_callback(subreq, req->base.flags, req->base.complete, + req->base.data); + aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, + req->iv); + aead_request_set_ad(subreq, req->assoclen); + + return (dir == SAFEXCEL_ENCRYPT) ? + crypto_aead_encrypt(subreq) : + crypto_aead_decrypt(subreq); +} + +static int safexcel_aead_sm4cbc_sm3_encrypt(struct aead_request *req) +{ + struct safexcel_cipher_req *creq = aead_request_ctx(req); + + /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ + if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) + return -EINVAL; + else if (req->cryptlen || req->assoclen) /* If input length > 0 only */ + return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); + + /* HW cannot do full (AAD+payload) zero length, use fallback */ + return safexcel_aead_fallback_crypt(req, SAFEXCEL_ENCRYPT); +} + +static int safexcel_aead_sm4cbc_sm3_decrypt(struct aead_request *req) +{ + struct safexcel_cipher_req *creq = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + + /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ + if ((req->cryptlen - crypto_aead_authsize(tfm)) & (SM4_BLOCK_SIZE - 1)) + return -EINVAL; + else if (req->cryptlen > crypto_aead_authsize(tfm) || req->assoclen) + /* If input length > 0 only */ + return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); + + /* HW cannot do full (AAD+payload) zero length, use fallback */ + return safexcel_aead_fallback_crypt(req, SAFEXCEL_DECRYPT); +} + +static int safexcel_aead_sm4cbc_sm3_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_fallback_cra_init(tfm); + ctx->alg = SAFEXCEL_SM4; + ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3; + ctx->state_sz = SM3_DIGEST_SIZE; + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sm3_cbc_sm4 = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SM3, + .alg.aead = { + .setkey = safexcel_aead_fallback_setkey, + .setauthsize = safexcel_aead_fallback_setauthsize, + .encrypt = safexcel_aead_sm4cbc_sm3_encrypt, + .decrypt = safexcel_aead_sm4cbc_sm3_decrypt, + .ivsize = SM4_BLOCK_SIZE, + .maxauthsize = SM3_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sm3),cbc(sm4))", + .cra_driver_name = "safexcel-authenc-hmac-sm3-cbc-sm4", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SM4_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sm4cbc_sm3_cra_init, + .cra_exit = safexcel_aead_fallback_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sm4ctr_sha1_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sm4cbc_sha1_cra_init(tfm); + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_ctr_sm4 = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SHA1, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha1),rfc3686(ctr(sm4)))", + .cra_driver_name = "safexcel-authenc-hmac-sha1-ctr-sm4", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sm4ctr_sha1_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sm4ctr_sm3_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sm4cbc_sm3_cra_init(tfm); + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sm3_ctr_sm4 = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SM3, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SM3_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sm3),rfc3686(ctr(sm4)))", + .cra_driver_name = "safexcel-authenc-hmac-sm3-ctr-sm4", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sm4ctr_sm3_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; -- cgit From aaf5a3839a3fcba06ca978e85cb6f66117ad44dd Mon Sep 17 00:00:00 2001 From: Pascal van Leeuwen Date: Fri, 13 Sep 2019 20:56:47 +0200 Subject: crypto: inside-secure - Add SHA3 family of basic hash algorithms This patch adds support for sha3-224, sha3-256, sha3-384 and sha3-512 basic hashes. The patch has been tested with the eip197c_iewxkbc configuration on the Xilinx VCU118 development board, including the testmgr extra tests. changes since v1: - nothing Signed-off-by: Pascal van Leeuwen Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 4 + drivers/crypto/inside-secure/safexcel.h | 9 + drivers/crypto/inside-secure/safexcel_hash.c | 351 +++++++++++++++++++++++++++ 3 files changed, 364 insertions(+) (limited to 'drivers/crypto/inside-secure/safexcel.c') diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 8f7fdd023262..4c91bbfd7a50 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1187,6 +1187,10 @@ static struct safexcel_alg_template *safexcel_algs[] = { &safexcel_alg_authenc_hmac_sm3_cbc_sm4, &safexcel_alg_authenc_hmac_sha1_ctr_sm4, &safexcel_alg_authenc_hmac_sm3_ctr_sm4, + &safexcel_alg_sha3_224, + &safexcel_alg_sha3_256, + &safexcel_alg_sha3_384, + &safexcel_alg_sha3_512, }; static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv) diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 1d75044c3e97..f41982d29469 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -360,6 +360,7 @@ struct safexcel_context_record { #define CONTEXT_CONTROL_CRYPTO_ALG_AES256 (0x7 << 17) #define CONTEXT_CONTROL_CRYPTO_ALG_CHACHA20 (0x8 << 17) #define CONTEXT_CONTROL_CRYPTO_ALG_SM4 (0xd << 17) +#define CONTEXT_CONTROL_DIGEST_INITIAL (0x0 << 21) #define CONTEXT_CONTROL_DIGEST_PRECOMPUTED (0x1 << 21) #define CONTEXT_CONTROL_DIGEST_XCM (0x2 << 21) #define CONTEXT_CONTROL_DIGEST_HMAC (0x3 << 21) @@ -375,6 +376,10 @@ struct safexcel_context_record { #define CONTEXT_CONTROL_CRYPTO_ALG_XCBC192 (0x2 << 23) #define CONTEXT_CONTROL_CRYPTO_ALG_XCBC256 (0x3 << 23) #define CONTEXT_CONTROL_CRYPTO_ALG_SM3 (0x7 << 23) +#define CONTEXT_CONTROL_CRYPTO_ALG_SHA3_256 (0xb << 23) +#define CONTEXT_CONTROL_CRYPTO_ALG_SHA3_224 (0xc << 23) +#define CONTEXT_CONTROL_CRYPTO_ALG_SHA3_512 (0xd << 23) +#define CONTEXT_CONTROL_CRYPTO_ALG_SHA3_384 (0xe << 23) #define CONTEXT_CONTROL_CRYPTO_ALG_POLY1305 (0xf << 23) #define CONTEXT_CONTROL_INV_FR (0x5 << 24) #define CONTEXT_CONTROL_INV_TR (0x6 << 24) @@ -888,5 +893,9 @@ extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_sm4; extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sm3_cbc_sm4; extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_ctr_sm4; extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sm3_ctr_sm4; +extern struct safexcel_alg_template safexcel_alg_sha3_224; +extern struct safexcel_alg_template safexcel_alg_sha3_256; +extern struct safexcel_alg_template safexcel_alg_sha3_384; +extern struct safexcel_alg_template safexcel_alg_sha3_512; #endif diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index 272e5fd0de7e..1d67cf2984fe 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -24,11 +25,14 @@ struct safexcel_ahash_ctx { u32 alg; u8 key_sz; bool cbcmac; + bool do_fallback; + bool fb_init_done; u32 ipad[SHA512_DIGEST_SIZE / sizeof(u32)]; u32 opad[SHA512_DIGEST_SIZE / sizeof(u32)]; struct crypto_cipher *kaes; + struct crypto_ahash *fback; }; struct safexcel_ahash_req { @@ -2355,3 +2359,350 @@ struct safexcel_alg_template safexcel_alg_hmac_sm3 = { }, }, }; + +static int safexcel_sha3_224_init(struct ahash_request *areq) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct safexcel_ahash_req *req = ahash_request_ctx(areq); + + memset(req, 0, sizeof(*req)); + + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_224; + req->digest = CONTEXT_CONTROL_DIGEST_INITIAL; + req->state_sz = SHA3_224_DIGEST_SIZE; + req->block_sz = SHA3_224_BLOCK_SIZE; + ctx->do_fallback = false; + ctx->fb_init_done = false; + return 0; +} + +static int safexcel_sha3_fbcheck(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct ahash_request *subreq = ahash_request_ctx(req); + int ret = 0; + + if (ctx->do_fallback) { + ahash_request_set_tfm(subreq, ctx->fback); + ahash_request_set_callback(subreq, req->base.flags, + req->base.complete, req->base.data); + ahash_request_set_crypt(subreq, req->src, req->result, + req->nbytes); + if (!ctx->fb_init_done) { + ret = crypto_ahash_init(subreq); + ctx->fb_init_done = true; + } + } + return ret; +} + +static int safexcel_sha3_update(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct ahash_request *subreq = ahash_request_ctx(req); + + ctx->do_fallback = true; + return safexcel_sha3_fbcheck(req) ?: crypto_ahash_update(subreq); +} + +static int safexcel_sha3_final(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct ahash_request *subreq = ahash_request_ctx(req); + + ctx->do_fallback = true; + return safexcel_sha3_fbcheck(req) ?: crypto_ahash_final(subreq); +} + +static int safexcel_sha3_finup(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct ahash_request *subreq = ahash_request_ctx(req); + + ctx->do_fallback |= !req->nbytes; + if (ctx->do_fallback) + /* Update or ex/import happened or len 0, cannot use the HW */ + return safexcel_sha3_fbcheck(req) ?: + crypto_ahash_finup(subreq); + else + return safexcel_ahash_finup(req); +} + +static int safexcel_sha3_digest_fallback(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct ahash_request *subreq = ahash_request_ctx(req); + + ctx->do_fallback = true; + ctx->fb_init_done = false; + return safexcel_sha3_fbcheck(req) ?: crypto_ahash_finup(subreq); +} + +static int safexcel_sha3_224_digest(struct ahash_request *req) +{ + if (req->nbytes) + return safexcel_sha3_224_init(req) ?: safexcel_ahash_finup(req); + + /* HW cannot do zero length hash, use fallback instead */ + return safexcel_sha3_digest_fallback(req); +} + +static int safexcel_sha3_export(struct ahash_request *req, void *out) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct ahash_request *subreq = ahash_request_ctx(req); + + ctx->do_fallback = true; + return safexcel_sha3_fbcheck(req) ?: crypto_ahash_export(subreq, out); +} + +static int safexcel_sha3_import(struct ahash_request *req, const void *in) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct ahash_request *subreq = ahash_request_ctx(req); + + ctx->do_fallback = true; + return safexcel_sha3_fbcheck(req) ?: crypto_ahash_import(subreq, in); + // return safexcel_ahash_import(req, in); +} + +static int safexcel_sha3_cra_init(struct crypto_tfm *tfm) +{ + struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); + struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_ahash_cra_init(tfm); + + /* Allocate fallback implementation */ + ctx->fback = crypto_alloc_ahash(crypto_tfm_alg_name(tfm), 0, + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->fback)) + return PTR_ERR(ctx->fback); + + /* Update statesize from fallback algorithm! */ + crypto_hash_alg_common(ahash)->statesize = + crypto_ahash_statesize(ctx->fback); + crypto_ahash_set_reqsize(ahash, max(sizeof(struct safexcel_ahash_req), + sizeof(struct ahash_request) + + crypto_ahash_reqsize(ctx->fback))); + return 0; +} + +static void safexcel_sha3_cra_exit(struct crypto_tfm *tfm) +{ + struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); + + crypto_free_ahash(ctx->fback); + safexcel_ahash_cra_exit(tfm); +} + +struct safexcel_alg_template safexcel_alg_sha3_224 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = SAFEXCEL_ALG_SHA3, + .alg.ahash = { + .init = safexcel_sha3_224_init, + .update = safexcel_sha3_update, + .final = safexcel_sha3_final, + .finup = safexcel_sha3_finup, + .digest = safexcel_sha3_224_digest, + .export = safexcel_sha3_export, + .import = safexcel_sha3_import, + .halg = { + .digestsize = SHA3_224_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "sha3-224", + .cra_driver_name = "safexcel-sha3-224", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA3_224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_sha3_cra_init, + .cra_exit = safexcel_sha3_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +static int safexcel_sha3_256_init(struct ahash_request *areq) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct safexcel_ahash_req *req = ahash_request_ctx(areq); + + memset(req, 0, sizeof(*req)); + + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_256; + req->digest = CONTEXT_CONTROL_DIGEST_INITIAL; + req->state_sz = SHA3_256_DIGEST_SIZE; + req->block_sz = SHA3_256_BLOCK_SIZE; + ctx->do_fallback = false; + ctx->fb_init_done = false; + return 0; +} + +static int safexcel_sha3_256_digest(struct ahash_request *req) +{ + if (req->nbytes) + return safexcel_sha3_256_init(req) ?: safexcel_ahash_finup(req); + + /* HW cannot do zero length hash, use fallback instead */ + return safexcel_sha3_digest_fallback(req); +} + +struct safexcel_alg_template safexcel_alg_sha3_256 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = SAFEXCEL_ALG_SHA3, + .alg.ahash = { + .init = safexcel_sha3_256_init, + .update = safexcel_sha3_update, + .final = safexcel_sha3_final, + .finup = safexcel_sha3_finup, + .digest = safexcel_sha3_256_digest, + .export = safexcel_sha3_export, + .import = safexcel_sha3_import, + .halg = { + .digestsize = SHA3_256_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "sha3-256", + .cra_driver_name = "safexcel-sha3-256", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA3_256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_sha3_cra_init, + .cra_exit = safexcel_sha3_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +static int safexcel_sha3_384_init(struct ahash_request *areq) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct safexcel_ahash_req *req = ahash_request_ctx(areq); + + memset(req, 0, sizeof(*req)); + + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_384; + req->digest = CONTEXT_CONTROL_DIGEST_INITIAL; + req->state_sz = SHA3_384_DIGEST_SIZE; + req->block_sz = SHA3_384_BLOCK_SIZE; + ctx->do_fallback = false; + ctx->fb_init_done = false; + return 0; +} + +static int safexcel_sha3_384_digest(struct ahash_request *req) +{ + if (req->nbytes) + return safexcel_sha3_384_init(req) ?: safexcel_ahash_finup(req); + + /* HW cannot do zero length hash, use fallback instead */ + return safexcel_sha3_digest_fallback(req); +} + +struct safexcel_alg_template safexcel_alg_sha3_384 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = SAFEXCEL_ALG_SHA3, + .alg.ahash = { + .init = safexcel_sha3_384_init, + .update = safexcel_sha3_update, + .final = safexcel_sha3_final, + .finup = safexcel_sha3_finup, + .digest = safexcel_sha3_384_digest, + .export = safexcel_sha3_export, + .import = safexcel_sha3_import, + .halg = { + .digestsize = SHA3_384_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "sha3-384", + .cra_driver_name = "safexcel-sha3-384", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA3_384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_sha3_cra_init, + .cra_exit = safexcel_sha3_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +static int safexcel_sha3_512_init(struct ahash_request *areq) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct safexcel_ahash_req *req = ahash_request_ctx(areq); + + memset(req, 0, sizeof(*req)); + + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_512; + req->digest = CONTEXT_CONTROL_DIGEST_INITIAL; + req->state_sz = SHA3_512_DIGEST_SIZE; + req->block_sz = SHA3_512_BLOCK_SIZE; + ctx->do_fallback = false; + ctx->fb_init_done = false; + return 0; +} + +static int safexcel_sha3_512_digest(struct ahash_request *req) +{ + if (req->nbytes) + return safexcel_sha3_512_init(req) ?: safexcel_ahash_finup(req); + + /* HW cannot do zero length hash, use fallback instead */ + return safexcel_sha3_digest_fallback(req); +} + +struct safexcel_alg_template safexcel_alg_sha3_512 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = SAFEXCEL_ALG_SHA3, + .alg.ahash = { + .init = safexcel_sha3_512_init, + .update = safexcel_sha3_update, + .final = safexcel_sha3_final, + .finup = safexcel_sha3_finup, + .digest = safexcel_sha3_512_digest, + .export = safexcel_sha3_export, + .import = safexcel_sha3_import, + .halg = { + .digestsize = SHA3_512_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "sha3-512", + .cra_driver_name = "safexcel-sha3-512", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA3_512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_sha3_cra_init, + .cra_exit = safexcel_sha3_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; -- cgit From 6c1c09b3dd0c217eeab3f123e24e58cf6ba30713 Mon Sep 17 00:00:00 2001 From: Pascal van Leeuwen Date: Fri, 13 Sep 2019 20:56:48 +0200 Subject: crypto: inside-secure - Add HMAC-SHA3 family of authentication algorithms This patch adds support for hmac(sha3-224), hmac(sha3-256), hmac(sha3-384) and hmac(sha3-512) authentication algorithms. The patch has been tested with the eip197c_iewxkbc configuration on the Xilinx VCU118 development board, including the testmgr extra tests. changes since v1: - nothing Signed-off-by: Pascal van Leeuwen Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 4 + drivers/crypto/inside-secure/safexcel.h | 4 + drivers/crypto/inside-secure/safexcel_hash.c | 441 ++++++++++++++++++++++++++- 3 files changed, 436 insertions(+), 13 deletions(-) (limited to 'drivers/crypto/inside-secure/safexcel.c') diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 4c91bbfd7a50..c8157f22abf4 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1191,6 +1191,10 @@ static struct safexcel_alg_template *safexcel_algs[] = { &safexcel_alg_sha3_256, &safexcel_alg_sha3_384, &safexcel_alg_sha3_512, + &safexcel_alg_hmac_sha3_224, + &safexcel_alg_hmac_sha3_256, + &safexcel_alg_hmac_sha3_384, + &safexcel_alg_hmac_sha3_512, }; static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv) diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index f41982d29469..275f8b5356f7 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -897,5 +897,9 @@ extern struct safexcel_alg_template safexcel_alg_sha3_224; extern struct safexcel_alg_template safexcel_alg_sha3_256; extern struct safexcel_alg_template safexcel_alg_sha3_384; extern struct safexcel_alg_template safexcel_alg_sha3_512; +extern struct safexcel_alg_template safexcel_alg_hmac_sha3_224; +extern struct safexcel_alg_template safexcel_alg_hmac_sha3_256; +extern struct safexcel_alg_template safexcel_alg_hmac_sha3_384; +extern struct safexcel_alg_template safexcel_alg_hmac_sha3_512; #endif diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index 1d67cf2984fe..85c3a075f283 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -27,12 +27,15 @@ struct safexcel_ahash_ctx { bool cbcmac; bool do_fallback; bool fb_init_done; + bool fb_do_setkey; - u32 ipad[SHA512_DIGEST_SIZE / sizeof(u32)]; - u32 opad[SHA512_DIGEST_SIZE / sizeof(u32)]; + u32 ipad[SHA3_512_BLOCK_SIZE / sizeof(u32)]; + u32 opad[SHA3_512_BLOCK_SIZE / sizeof(u32)]; struct crypto_cipher *kaes; struct crypto_ahash *fback; + struct crypto_shash *shpre; + struct shash_desc *shdesc; }; struct safexcel_ahash_req { @@ -52,7 +55,8 @@ struct safexcel_ahash_req { u8 state_sz; /* expected state size, only set once */ u8 block_sz; /* block size, only set once */ - u32 state[SHA512_DIGEST_SIZE / sizeof(u32)] __aligned(sizeof(u32)); + u8 digest_sz; /* output digest size, only set once */ + u32 state[SHA3_512_BLOCK_SIZE / sizeof(u32)] __aligned(sizeof(u32)); u64 len; u64 processed; @@ -246,7 +250,7 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, } if (sreq->result_dma) { - dma_unmap_single(priv->dev, sreq->result_dma, sreq->state_sz, + dma_unmap_single(priv->dev, sreq->result_dma, sreq->digest_sz, DMA_FROM_DEVICE); sreq->result_dma = 0; } @@ -265,7 +269,7 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, memcpy(sreq->cache, sreq->state, crypto_ahash_digestsize(ahash)); - memcpy(sreq->state, ctx->opad, sreq->state_sz); + memcpy(sreq->state, ctx->opad, sreq->digest_sz); sreq->len = sreq->block_sz + crypto_ahash_digestsize(ahash); @@ -309,7 +313,7 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring, struct safexcel_command_desc *cdesc, *first_cdesc = NULL; struct safexcel_result_desc *rdesc; struct scatterlist *sg; - int i, extra = 0, n_cdesc = 0, ret = 0, cache_len, skip = 0, res_sz; + int i, extra = 0, n_cdesc = 0, ret = 0, cache_len, skip = 0; u64 queued, len; queued = safexcel_queued_len(req); @@ -451,11 +455,10 @@ send_command: /* Setup the context options */ safexcel_context_control(ctx, req, first_cdesc); - /* Add the token. Note that the XCBC result is only 1 AES block. */ - res_sz = req->xcbcmac ? AES_BLOCK_SIZE : req->state_sz; - safexcel_hash_token(first_cdesc, len, res_sz, ctx->cbcmac); + /* Add the token */ + safexcel_hash_token(first_cdesc, len, req->digest_sz, ctx->cbcmac); - req->result_dma = dma_map_single(priv->dev, req->state, req->state_sz, + req->result_dma = dma_map_single(priv->dev, req->state, req->digest_sz, DMA_FROM_DEVICE); if (dma_mapping_error(priv->dev, req->result_dma)) { ret = -EINVAL; @@ -464,7 +467,7 @@ send_command: /* Add a result descriptor */ rdesc = safexcel_add_rdesc(priv, ring, 1, 1, req->result_dma, - res_sz); + req->digest_sz); if (IS_ERR(rdesc)) { ret = PTR_ERR(rdesc); goto unmap_result; @@ -479,7 +482,7 @@ send_command: return 0; unmap_result: - dma_unmap_single(priv->dev, req->result_dma, req->state_sz, + dma_unmap_single(priv->dev, req->result_dma, req->digest_sz, DMA_FROM_DEVICE); unmap_sg: if (req->nents) { @@ -912,6 +915,7 @@ static int safexcel_ahash_cra_init(struct crypto_tfm *tfm) ctx->priv = tmpl->priv; ctx->base.send = safexcel_ahash_send; ctx->base.handle_result = safexcel_handle_result; + ctx->fb_do_setkey = false; crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), sizeof(struct safexcel_ahash_req)); @@ -928,6 +932,7 @@ static int safexcel_sha1_init(struct ahash_request *areq) ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1; req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; req->state_sz = SHA1_DIGEST_SIZE; + req->digest_sz = SHA1_DIGEST_SIZE; req->block_sz = SHA1_BLOCK_SIZE; return 0; @@ -1009,6 +1014,7 @@ static int safexcel_hmac_sha1_init(struct ahash_request *areq) ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1; req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; req->state_sz = SHA1_DIGEST_SIZE; + req->digest_sz = SHA1_DIGEST_SIZE; req->block_sz = SHA1_BLOCK_SIZE; req->hmac = true; @@ -1245,6 +1251,7 @@ static int safexcel_sha256_init(struct ahash_request *areq) ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256; req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; req->state_sz = SHA256_DIGEST_SIZE; + req->digest_sz = SHA256_DIGEST_SIZE; req->block_sz = SHA256_BLOCK_SIZE; return 0; @@ -1300,6 +1307,7 @@ static int safexcel_sha224_init(struct ahash_request *areq) ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224; req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; req->state_sz = SHA256_DIGEST_SIZE; + req->digest_sz = SHA256_DIGEST_SIZE; req->block_sz = SHA256_BLOCK_SIZE; return 0; @@ -1368,6 +1376,7 @@ static int safexcel_hmac_sha224_init(struct ahash_request *areq) ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224; req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; req->state_sz = SHA256_DIGEST_SIZE; + req->digest_sz = SHA256_DIGEST_SIZE; req->block_sz = SHA256_BLOCK_SIZE; req->hmac = true; @@ -1438,6 +1447,7 @@ static int safexcel_hmac_sha256_init(struct ahash_request *areq) ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256; req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; req->state_sz = SHA256_DIGEST_SIZE; + req->digest_sz = SHA256_DIGEST_SIZE; req->block_sz = SHA256_BLOCK_SIZE; req->hmac = true; @@ -1495,6 +1505,7 @@ static int safexcel_sha512_init(struct ahash_request *areq) ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512; req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; req->state_sz = SHA512_DIGEST_SIZE; + req->digest_sz = SHA512_DIGEST_SIZE; req->block_sz = SHA512_BLOCK_SIZE; return 0; @@ -1550,6 +1561,7 @@ static int safexcel_sha384_init(struct ahash_request *areq) ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384; req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; req->state_sz = SHA512_DIGEST_SIZE; + req->digest_sz = SHA512_DIGEST_SIZE; req->block_sz = SHA512_BLOCK_SIZE; return 0; @@ -1618,6 +1630,7 @@ static int safexcel_hmac_sha512_init(struct ahash_request *areq) ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512; req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; req->state_sz = SHA512_DIGEST_SIZE; + req->digest_sz = SHA512_DIGEST_SIZE; req->block_sz = SHA512_BLOCK_SIZE; req->hmac = true; @@ -1688,6 +1701,7 @@ static int safexcel_hmac_sha384_init(struct ahash_request *areq) ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384; req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; req->state_sz = SHA512_DIGEST_SIZE; + req->digest_sz = SHA512_DIGEST_SIZE; req->block_sz = SHA512_BLOCK_SIZE; req->hmac = true; @@ -1745,6 +1759,7 @@ static int safexcel_md5_init(struct ahash_request *areq) ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_MD5; req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; req->state_sz = MD5_DIGEST_SIZE; + req->digest_sz = MD5_DIGEST_SIZE; req->block_sz = MD5_HMAC_BLOCK_SIZE; return 0; @@ -1806,6 +1821,7 @@ static int safexcel_hmac_md5_init(struct ahash_request *areq) ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_MD5; req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; req->state_sz = MD5_DIGEST_SIZE; + req->digest_sz = MD5_DIGEST_SIZE; req->block_sz = MD5_HMAC_BLOCK_SIZE; req->len_is_le = true; /* MD5 is little endian! ... */ req->hmac = true; @@ -1887,6 +1903,7 @@ static int safexcel_crc32_init(struct ahash_request *areq) ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_CRC32; req->digest = CONTEXT_CONTROL_DIGEST_XCM; req->state_sz = sizeof(u32); + req->digest_sz = sizeof(u32); req->block_sz = sizeof(u32); return 0; @@ -1958,6 +1975,7 @@ static int safexcel_cbcmac_init(struct ahash_request *areq) req->digest = CONTEXT_CONTROL_DIGEST_XCM; req->state_sz = ctx->key_sz; + req->digest_sz = AES_BLOCK_SIZE; req->block_sz = AES_BLOCK_SIZE; req->xcbcmac = true; @@ -2245,6 +2263,7 @@ static int safexcel_sm3_init(struct ahash_request *areq) ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3; req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; req->state_sz = SM3_DIGEST_SIZE; + req->digest_sz = SM3_DIGEST_SIZE; req->block_sz = SM3_BLOCK_SIZE; return 0; @@ -2313,6 +2332,7 @@ static int safexcel_hmac_sm3_init(struct ahash_request *areq) ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3; req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; req->state_sz = SM3_DIGEST_SIZE; + req->digest_sz = SM3_DIGEST_SIZE; req->block_sz = SM3_BLOCK_SIZE; req->hmac = true; @@ -2371,6 +2391,7 @@ static int safexcel_sha3_224_init(struct ahash_request *areq) ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_224; req->digest = CONTEXT_CONTROL_DIGEST_INITIAL; req->state_sz = SHA3_224_DIGEST_SIZE; + req->digest_sz = SHA3_224_DIGEST_SIZE; req->block_sz = SHA3_224_BLOCK_SIZE; ctx->do_fallback = false; ctx->fb_init_done = false; @@ -2391,7 +2412,23 @@ static int safexcel_sha3_fbcheck(struct ahash_request *req) ahash_request_set_crypt(subreq, req->src, req->result, req->nbytes); if (!ctx->fb_init_done) { - ret = crypto_ahash_init(subreq); + if (ctx->fb_do_setkey) { + /* Set fallback cipher HMAC key */ + u8 key[SHA3_224_BLOCK_SIZE]; + + memcpy(key, ctx->ipad, + crypto_ahash_blocksize(ctx->fback) / 2); + memcpy(key + + crypto_ahash_blocksize(ctx->fback) / 2, + ctx->opad, + crypto_ahash_blocksize(ctx->fback) / 2); + ret = crypto_ahash_setkey(ctx->fback, key, + crypto_ahash_blocksize(ctx->fback)); + memzero_explicit(key, + crypto_ahash_blocksize(ctx->fback)); + ctx->fb_do_setkey = false; + } + ret = ret ?: crypto_ahash_init(subreq); ctx->fb_init_done = true; } } @@ -2547,6 +2584,7 @@ static int safexcel_sha3_256_init(struct ahash_request *areq) ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_256; req->digest = CONTEXT_CONTROL_DIGEST_INITIAL; req->state_sz = SHA3_256_DIGEST_SIZE; + req->digest_sz = SHA3_256_DIGEST_SIZE; req->block_sz = SHA3_256_BLOCK_SIZE; ctx->do_fallback = false; ctx->fb_init_done = false; @@ -2604,6 +2642,7 @@ static int safexcel_sha3_384_init(struct ahash_request *areq) ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_384; req->digest = CONTEXT_CONTROL_DIGEST_INITIAL; req->state_sz = SHA3_384_DIGEST_SIZE; + req->digest_sz = SHA3_384_DIGEST_SIZE; req->block_sz = SHA3_384_BLOCK_SIZE; ctx->do_fallback = false; ctx->fb_init_done = false; @@ -2661,6 +2700,7 @@ static int safexcel_sha3_512_init(struct ahash_request *areq) ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_512; req->digest = CONTEXT_CONTROL_DIGEST_INITIAL; req->state_sz = SHA3_512_DIGEST_SIZE; + req->digest_sz = SHA3_512_DIGEST_SIZE; req->block_sz = SHA3_512_BLOCK_SIZE; ctx->do_fallback = false; ctx->fb_init_done = false; @@ -2706,3 +2746,378 @@ struct safexcel_alg_template safexcel_alg_sha3_512 = { }, }, }; + +static int safexcel_hmac_sha3_cra_init(struct crypto_tfm *tfm, const char *alg) +{ + struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); + int ret; + + ret = safexcel_sha3_cra_init(tfm); + if (ret) + return ret; + + /* Allocate precalc basic digest implementation */ + ctx->shpre = crypto_alloc_shash(alg, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->shpre)) + return PTR_ERR(ctx->shpre); + + ctx->shdesc = kmalloc(sizeof(*ctx->shdesc) + + crypto_shash_descsize(ctx->shpre), GFP_KERNEL); + if (!ctx->shdesc) { + crypto_free_shash(ctx->shpre); + return -ENOMEM; + } + ctx->shdesc->tfm = ctx->shpre; + return 0; +} + +static void safexcel_hmac_sha3_cra_exit(struct crypto_tfm *tfm) +{ + struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); + + crypto_free_ahash(ctx->fback); + crypto_free_shash(ctx->shpre); + kfree(ctx->shdesc); + safexcel_ahash_cra_exit(tfm); +} + +static int safexcel_hmac_sha3_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + int ret = 0; + + if (keylen > crypto_ahash_blocksize(tfm)) { + /* + * If the key is larger than the blocksize, then hash it + * first using our fallback cipher + */ + ret = crypto_shash_digest(ctx->shdesc, key, keylen, + (u8 *)ctx->ipad); + keylen = crypto_shash_digestsize(ctx->shpre); + + /* + * If the digest is larger than half the blocksize, we need to + * move the rest to opad due to the way our HMAC infra works. + */ + if (keylen > crypto_ahash_blocksize(tfm) / 2) + /* Buffers overlap, need to use memmove iso memcpy! */ + memmove(ctx->opad, + (u8 *)ctx->ipad + + crypto_ahash_blocksize(tfm) / 2, + keylen - crypto_ahash_blocksize(tfm) / 2); + } else { + /* + * Copy the key to our ipad & opad buffers + * Note that ipad and opad each contain one half of the key, + * to match the existing HMAC driver infrastructure. + */ + if (keylen <= crypto_ahash_blocksize(tfm) / 2) { + memcpy(ctx->ipad, key, keylen); + } else { + memcpy(ctx->ipad, key, + crypto_ahash_blocksize(tfm) / 2); + memcpy(ctx->opad, + key + crypto_ahash_blocksize(tfm) / 2, + keylen - crypto_ahash_blocksize(tfm) / 2); + } + } + + /* Pad key with zeroes */ + if (keylen <= crypto_ahash_blocksize(tfm) / 2) { + memset((u8 *)ctx->ipad + keylen, 0, + crypto_ahash_blocksize(tfm) / 2 - keylen); + memset(ctx->opad, 0, crypto_ahash_blocksize(tfm) / 2); + } else { + memset((u8 *)ctx->opad + keylen - + crypto_ahash_blocksize(tfm) / 2, 0, + crypto_ahash_blocksize(tfm) - keylen); + } + + /* If doing fallback, still need to set the new key! */ + ctx->fb_do_setkey = true; + return ret; +} + +static int safexcel_hmac_sha3_224_init(struct ahash_request *areq) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct safexcel_ahash_req *req = ahash_request_ctx(areq); + + memset(req, 0, sizeof(*req)); + + /* Copy (half of) the key */ + memcpy(req->state, ctx->ipad, SHA3_224_BLOCK_SIZE / 2); + /* Start of HMAC should have len == processed == blocksize */ + req->len = SHA3_224_BLOCK_SIZE; + req->processed = SHA3_224_BLOCK_SIZE; + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_224; + req->digest = CONTEXT_CONTROL_DIGEST_HMAC; + req->state_sz = SHA3_224_BLOCK_SIZE / 2; + req->digest_sz = SHA3_224_DIGEST_SIZE; + req->block_sz = SHA3_224_BLOCK_SIZE; + req->hmac = true; + ctx->do_fallback = false; + ctx->fb_init_done = false; + return 0; +} + +static int safexcel_hmac_sha3_224_digest(struct ahash_request *req) +{ + if (req->nbytes) + return safexcel_hmac_sha3_224_init(req) ?: + safexcel_ahash_finup(req); + + /* HW cannot do zero length HMAC, use fallback instead */ + return safexcel_sha3_digest_fallback(req); +} + +static int safexcel_hmac_sha3_224_cra_init(struct crypto_tfm *tfm) +{ + return safexcel_hmac_sha3_cra_init(tfm, "sha3-224"); +} + +struct safexcel_alg_template safexcel_alg_hmac_sha3_224 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = SAFEXCEL_ALG_SHA3, + .alg.ahash = { + .init = safexcel_hmac_sha3_224_init, + .update = safexcel_sha3_update, + .final = safexcel_sha3_final, + .finup = safexcel_sha3_finup, + .digest = safexcel_hmac_sha3_224_digest, + .setkey = safexcel_hmac_sha3_setkey, + .export = safexcel_sha3_export, + .import = safexcel_sha3_import, + .halg = { + .digestsize = SHA3_224_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "hmac(sha3-224)", + .cra_driver_name = "safexcel-hmac-sha3-224", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA3_224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_hmac_sha3_224_cra_init, + .cra_exit = safexcel_hmac_sha3_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +static int safexcel_hmac_sha3_256_init(struct ahash_request *areq) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct safexcel_ahash_req *req = ahash_request_ctx(areq); + + memset(req, 0, sizeof(*req)); + + /* Copy (half of) the key */ + memcpy(req->state, ctx->ipad, SHA3_256_BLOCK_SIZE / 2); + /* Start of HMAC should have len == processed == blocksize */ + req->len = SHA3_256_BLOCK_SIZE; + req->processed = SHA3_256_BLOCK_SIZE; + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_256; + req->digest = CONTEXT_CONTROL_DIGEST_HMAC; + req->state_sz = SHA3_256_BLOCK_SIZE / 2; + req->digest_sz = SHA3_256_DIGEST_SIZE; + req->block_sz = SHA3_256_BLOCK_SIZE; + req->hmac = true; + ctx->do_fallback = false; + ctx->fb_init_done = false; + return 0; +} + +static int safexcel_hmac_sha3_256_digest(struct ahash_request *req) +{ + if (req->nbytes) + return safexcel_hmac_sha3_256_init(req) ?: + safexcel_ahash_finup(req); + + /* HW cannot do zero length HMAC, use fallback instead */ + return safexcel_sha3_digest_fallback(req); +} + +static int safexcel_hmac_sha3_256_cra_init(struct crypto_tfm *tfm) +{ + return safexcel_hmac_sha3_cra_init(tfm, "sha3-256"); +} + +struct safexcel_alg_template safexcel_alg_hmac_sha3_256 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = SAFEXCEL_ALG_SHA3, + .alg.ahash = { + .init = safexcel_hmac_sha3_256_init, + .update = safexcel_sha3_update, + .final = safexcel_sha3_final, + .finup = safexcel_sha3_finup, + .digest = safexcel_hmac_sha3_256_digest, + .setkey = safexcel_hmac_sha3_setkey, + .export = safexcel_sha3_export, + .import = safexcel_sha3_import, + .halg = { + .digestsize = SHA3_256_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "hmac(sha3-256)", + .cra_driver_name = "safexcel-hmac-sha3-256", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA3_256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_hmac_sha3_256_cra_init, + .cra_exit = safexcel_hmac_sha3_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +static int safexcel_hmac_sha3_384_init(struct ahash_request *areq) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct safexcel_ahash_req *req = ahash_request_ctx(areq); + + memset(req, 0, sizeof(*req)); + + /* Copy (half of) the key */ + memcpy(req->state, ctx->ipad, SHA3_384_BLOCK_SIZE / 2); + /* Start of HMAC should have len == processed == blocksize */ + req->len = SHA3_384_BLOCK_SIZE; + req->processed = SHA3_384_BLOCK_SIZE; + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_384; + req->digest = CONTEXT_CONTROL_DIGEST_HMAC; + req->state_sz = SHA3_384_BLOCK_SIZE / 2; + req->digest_sz = SHA3_384_DIGEST_SIZE; + req->block_sz = SHA3_384_BLOCK_SIZE; + req->hmac = true; + ctx->do_fallback = false; + ctx->fb_init_done = false; + return 0; +} + +static int safexcel_hmac_sha3_384_digest(struct ahash_request *req) +{ + if (req->nbytes) + return safexcel_hmac_sha3_384_init(req) ?: + safexcel_ahash_finup(req); + + /* HW cannot do zero length HMAC, use fallback instead */ + return safexcel_sha3_digest_fallback(req); +} + +static int safexcel_hmac_sha3_384_cra_init(struct crypto_tfm *tfm) +{ + return safexcel_hmac_sha3_cra_init(tfm, "sha3-384"); +} + +struct safexcel_alg_template safexcel_alg_hmac_sha3_384 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = SAFEXCEL_ALG_SHA3, + .alg.ahash = { + .init = safexcel_hmac_sha3_384_init, + .update = safexcel_sha3_update, + .final = safexcel_sha3_final, + .finup = safexcel_sha3_finup, + .digest = safexcel_hmac_sha3_384_digest, + .setkey = safexcel_hmac_sha3_setkey, + .export = safexcel_sha3_export, + .import = safexcel_sha3_import, + .halg = { + .digestsize = SHA3_384_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "hmac(sha3-384)", + .cra_driver_name = "safexcel-hmac-sha3-384", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA3_384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_hmac_sha3_384_cra_init, + .cra_exit = safexcel_hmac_sha3_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +static int safexcel_hmac_sha3_512_init(struct ahash_request *areq) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct safexcel_ahash_req *req = ahash_request_ctx(areq); + + memset(req, 0, sizeof(*req)); + + /* Copy (half of) the key */ + memcpy(req->state, ctx->ipad, SHA3_512_BLOCK_SIZE / 2); + /* Start of HMAC should have len == processed == blocksize */ + req->len = SHA3_512_BLOCK_SIZE; + req->processed = SHA3_512_BLOCK_SIZE; + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_512; + req->digest = CONTEXT_CONTROL_DIGEST_HMAC; + req->state_sz = SHA3_512_BLOCK_SIZE / 2; + req->digest_sz = SHA3_512_DIGEST_SIZE; + req->block_sz = SHA3_512_BLOCK_SIZE; + req->hmac = true; + ctx->do_fallback = false; + ctx->fb_init_done = false; + return 0; +} + +static int safexcel_hmac_sha3_512_digest(struct ahash_request *req) +{ + if (req->nbytes) + return safexcel_hmac_sha3_512_init(req) ?: + safexcel_ahash_finup(req); + + /* HW cannot do zero length HMAC, use fallback instead */ + return safexcel_sha3_digest_fallback(req); +} + +static int safexcel_hmac_sha3_512_cra_init(struct crypto_tfm *tfm) +{ + return safexcel_hmac_sha3_cra_init(tfm, "sha3-512"); +} +struct safexcel_alg_template safexcel_alg_hmac_sha3_512 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = SAFEXCEL_ALG_SHA3, + .alg.ahash = { + .init = safexcel_hmac_sha3_512_init, + .update = safexcel_sha3_update, + .final = safexcel_sha3_final, + .finup = safexcel_sha3_finup, + .digest = safexcel_hmac_sha3_512_digest, + .setkey = safexcel_hmac_sha3_setkey, + .export = safexcel_sha3_export, + .import = safexcel_sha3_import, + .halg = { + .digestsize = SHA3_512_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "hmac(sha3-512)", + .cra_driver_name = "safexcel-hmac-sha3-512", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA3_512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_hmac_sha3_512_cra_init, + .cra_exit = safexcel_hmac_sha3_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; -- cgit From bb7679b840cc7cf23868e05c5ef7a044e7fafd97 Mon Sep 17 00:00:00 2001 From: Pascal van Leeuwen Date: Fri, 13 Sep 2019 22:04:44 +0200 Subject: crypto: inside-secure - Added support for authenc HMAC-SHA1/DES-CBC This patch adds support for the authenc(hmac(sha1),cbc(des)) aead changes since v1: - rebased on top of DES changes made to cryptodev/master Signed-off-by: Pascal van Leeuwen Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 1 + drivers/crypto/inside-secure/safexcel.h | 1 + drivers/crypto/inside-secure/safexcel_cipher.c | 39 ++++++++++++++++++++++++++ 3 files changed, 41 insertions(+) (limited to 'drivers/crypto/inside-secure/safexcel.c') diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index c8157f22abf4..462dbf60e818 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1195,6 +1195,7 @@ static struct safexcel_alg_template *safexcel_algs[] = { &safexcel_alg_hmac_sha3_256, &safexcel_alg_hmac_sha3_384, &safexcel_alg_hmac_sha3_512, + &safexcel_alg_authenc_hmac_sha1_cbc_des, }; static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv) diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 275f8b5356f7..fe00b8725b8b 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -901,5 +901,6 @@ extern struct safexcel_alg_template safexcel_alg_hmac_sha3_224; extern struct safexcel_alg_template safexcel_alg_hmac_sha3_256; extern struct safexcel_alg_template safexcel_alg_hmac_sha3_384; extern struct safexcel_alg_template safexcel_alg_hmac_sha3_512; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des; #endif diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index 226b76180b90..68aad4ab92f9 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -366,6 +366,11 @@ static int safexcel_aead_setkey(struct crypto_aead *ctfm, const u8 *key, /* Encryption key */ switch (ctx->alg) { + case SAFEXCEL_DES: + err = verify_aead_des_key(ctfm, keys.enckey, keys.enckeylen); + if (unlikely(err)) + goto badkey_expflags; + break; case SAFEXCEL_3DES: err = verify_aead_des3_key(ctfm, keys.enckey, keys.enckeylen); if (unlikely(err)) @@ -1841,6 +1846,40 @@ struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des3_ede = { }, }; +static int safexcel_aead_sha1_des_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sha1_cra_init(tfm); + ctx->alg = SAFEXCEL_DES; /* override default */ + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA1, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha1),cbc(des))", + .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-des", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha1_des_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + static int safexcel_aead_sha1_ctr_cra_init(struct crypto_tfm *tfm) { struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); -- cgit From f0a8bdf0b113b51d023c94468628ec90d4375acc Mon Sep 17 00:00:00 2001 From: Pascal van Leeuwen Date: Fri, 13 Sep 2019 22:04:45 +0200 Subject: crypto: inside-secure - Added support for authenc HMAC-SHA2/3DES-CBC This patch adds support for the authenc(hmac(sha224),cbc(des3_ede)), authenc(hmac(sha256),cbc(des3_ede)), authenc(hmac(sha384),cbc(des3_ede)) and authenc(hmac(sha512),cbc(des3_ede)) aead's changes since v1: - nothing Signed-off-by: Pascal van Leeuwen Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 4 + drivers/crypto/inside-secure/safexcel.h | 4 + drivers/crypto/inside-secure/safexcel_cipher.c | 136 +++++++++++++++++++++++++ 3 files changed, 144 insertions(+) (limited to 'drivers/crypto/inside-secure/safexcel.c') diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 462dbf60e818..36523dd74599 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1196,6 +1196,10 @@ static struct safexcel_alg_template *safexcel_algs[] = { &safexcel_alg_hmac_sha3_384, &safexcel_alg_hmac_sha3_512, &safexcel_alg_authenc_hmac_sha1_cbc_des, + &safexcel_alg_authenc_hmac_sha256_cbc_des3_ede, + &safexcel_alg_authenc_hmac_sha224_cbc_des3_ede, + &safexcel_alg_authenc_hmac_sha512_cbc_des3_ede, + &safexcel_alg_authenc_hmac_sha384_cbc_des3_ede, }; static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv) diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index fe00b8725b8b..b1703e322ab8 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -902,5 +902,9 @@ extern struct safexcel_alg_template safexcel_alg_hmac_sha3_256; extern struct safexcel_alg_template safexcel_alg_hmac_sha3_384; extern struct safexcel_alg_template safexcel_alg_hmac_sha3_512; extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_des3_ede; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des3_ede; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des3_ede; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des3_ede; #endif diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index 68aad4ab92f9..0803b0ba233c 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -1846,6 +1846,142 @@ struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des3_ede = { }, }; +static int safexcel_aead_sha256_des3_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sha256_cra_init(tfm); + ctx->alg = SAFEXCEL_3DES; /* override default */ + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_des3_ede = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha256),cbc(des3_ede))", + .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-des3_ede", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha256_des3_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sha224_des3_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sha224_cra_init(tfm); + ctx->alg = SAFEXCEL_3DES; /* override default */ + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des3_ede = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha224),cbc(des3_ede))", + .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-des3_ede", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha224_des3_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sha512_des3_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sha512_cra_init(tfm); + ctx->alg = SAFEXCEL_3DES; /* override default */ + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des3_ede = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha512),cbc(des3_ede))", + .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-des3_ede", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha512_des3_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sha384_des3_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sha384_cra_init(tfm); + ctx->alg = SAFEXCEL_3DES; /* override default */ + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des3_ede = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha384),cbc(des3_ede))", + .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-des3_ede", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha384_des3_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + static int safexcel_aead_sha1_des_cra_init(struct crypto_tfm *tfm) { struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); -- cgit From 457a6fdf4caaac5d5006bfd609f33b8b2a383750 Mon Sep 17 00:00:00 2001 From: Pascal van Leeuwen Date: Fri, 13 Sep 2019 22:04:46 +0200 Subject: crypto: inside-secure - Added support for authenc HMAC-SHA2/DES-CBC This patch adds support for the authenc(hmac(sha224),cbc(des)), authenc(hmac(sha256),cbc(des)), authenc(hmac(sha384),cbc(des)) and authenc(hmac(sha512),cbc(des)) aead's changes since v1: - nothing Signed-off-by: Pascal van Leeuwen Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 4 + drivers/crypto/inside-secure/safexcel.h | 4 + drivers/crypto/inside-secure/safexcel_cipher.c | 136 +++++++++++++++++++++++++ 3 files changed, 144 insertions(+) (limited to 'drivers/crypto/inside-secure/safexcel.c') diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 36523dd74599..094b581c194e 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1200,6 +1200,10 @@ static struct safexcel_alg_template *safexcel_algs[] = { &safexcel_alg_authenc_hmac_sha224_cbc_des3_ede, &safexcel_alg_authenc_hmac_sha512_cbc_des3_ede, &safexcel_alg_authenc_hmac_sha384_cbc_des3_ede, + &safexcel_alg_authenc_hmac_sha256_cbc_des, + &safexcel_alg_authenc_hmac_sha224_cbc_des, + &safexcel_alg_authenc_hmac_sha512_cbc_des, + &safexcel_alg_authenc_hmac_sha384_cbc_des, }; static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv) diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index b1703e322ab8..9522594cd3a4 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -906,5 +906,9 @@ extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_des3_ed extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des3_ede; extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des3_ede; extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des3_ede; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_des; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des; #endif diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index 0803b0ba233c..6c3fdd650ecc 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -2016,6 +2016,142 @@ struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des = { }, }; +static int safexcel_aead_sha256_des_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sha256_cra_init(tfm); + ctx->alg = SAFEXCEL_DES; /* override default */ + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_des = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha256),cbc(des))", + .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-des", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha256_des_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sha224_des_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sha224_cra_init(tfm); + ctx->alg = SAFEXCEL_DES; /* override default */ + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha224),cbc(des))", + .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-des", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha224_des_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sha512_des_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sha512_cra_init(tfm); + ctx->alg = SAFEXCEL_DES; /* override default */ + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha512),cbc(des))", + .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-des", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha512_des_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sha384_des_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sha384_cra_init(tfm); + ctx->alg = SAFEXCEL_DES; /* override default */ + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha384),cbc(des))", + .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-des", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha384_des_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + static int safexcel_aead_sha1_ctr_cra_init(struct crypto_tfm *tfm) { struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); -- cgit From b8c5d882c8334d05754b69dcdf1cfd6bc48a9e12 Mon Sep 17 00:00:00 2001 From: Pascal van Leeuwen Date: Tue, 17 Sep 2019 11:55:18 +0200 Subject: crypto: inside-secure - Fix stability issue with Macchiatobin This patch corrects an error in the Transform Record Cache initialization code that was causing intermittent stability problems on the Macchiatobin board. Unfortunately, due to HW platform specifics, the problem could not happen on the main development platform, being the VCU118 Xilinx development board. And since it was a problem with hash table access, it was very dependent on the actual physical context record DMA buffers being used, i.e. with some (bad) luck it could seemingly work quit stable for a while. Signed-off-by: Pascal van Leeuwen Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/crypto/inside-secure/safexcel.c') diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 094b581c194e..93227ba69c78 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -221,9 +221,9 @@ static void eip197_trc_cache_init(struct safexcel_crypto_priv *priv) /* Step #3: Determine log2 of hash table size */ cs_ht_sz = __fls(asize - cs_rc_max) - 2; /* Step #4: determine current size of hash table in dwords */ - cs_ht_wc = 16<> 4)); + cs_rc_max = min_t(uint, cs_rc_abs_max, asize - (cs_ht_wc >> 2)); /* Clear the cache RAMs */ eip197_trc_cache_clear(priv, cs_rc_max, cs_ht_wc); -- cgit From d96cf190e5dc1838b8529bad271f2261d07870e8 Mon Sep 17 00:00:00 2001 From: Pascal van Leeuwen Date: Tue, 17 Sep 2019 11:55:19 +0200 Subject: crypto: inside-secure - Fixed corner case TRC admin RAM probing issue This patch fixed a corner case admin RAM probing issue witnessed on the Xilinx VCU118 FPGA development board with an EIP197 configuration with 4096 words of admin RAM, of which only 2050 were recognised. Signed-off-by: Pascal van Leeuwen Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 48 ++++++++++++++++++++++----------- drivers/crypto/inside-secure/safexcel.h | 2 ++ 2 files changed, 34 insertions(+), 16 deletions(-) (limited to 'drivers/crypto/inside-secure/safexcel.c') diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 93227ba69c78..1878b79336cb 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -75,9 +75,9 @@ static void eip197_trc_cache_banksel(struct safexcel_crypto_priv *priv, } static u32 eip197_trc_cache_probe(struct safexcel_crypto_priv *priv, - int maxbanks, u32 probemask) + int maxbanks, u32 probemask, u32 stride) { - u32 val, addrhi, addrlo, addrmid; + u32 val, addrhi, addrlo, addrmid, addralias, delta, marker; int actbank; /* @@ -87,32 +87,37 @@ static u32 eip197_trc_cache_probe(struct safexcel_crypto_priv *priv, addrhi = 1 << (16 + maxbanks); addrlo = 0; actbank = min(maxbanks - 1, 0); - while ((addrhi - addrlo) > 32) { + while ((addrhi - addrlo) > stride) { /* write marker to lowest address in top half */ addrmid = (addrhi + addrlo) >> 1; + marker = (addrmid ^ 0xabadbabe) & probemask; /* Unique */ eip197_trc_cache_banksel(priv, addrmid, &actbank); - writel((addrmid | (addrlo << 16)) & probemask, + writel(marker, priv->base + EIP197_CLASSIFICATION_RAMS + (addrmid & 0xffff)); - /* write marker to lowest address in bottom half */ - eip197_trc_cache_banksel(priv, addrlo, &actbank); - writel((addrlo | (addrhi << 16)) & probemask, - priv->base + EIP197_CLASSIFICATION_RAMS + - (addrlo & 0xffff)); + /* write invalid markers to possible aliases */ + delta = 1 << __fls(addrmid); + while (delta >= stride) { + addralias = addrmid - delta; + eip197_trc_cache_banksel(priv, addralias, &actbank); + writel(~marker, + priv->base + EIP197_CLASSIFICATION_RAMS + + (addralias & 0xffff)); + delta >>= 1; + } /* read back marker from top half */ eip197_trc_cache_banksel(priv, addrmid, &actbank); val = readl(priv->base + EIP197_CLASSIFICATION_RAMS + (addrmid & 0xffff)); - if (val == ((addrmid | (addrlo << 16)) & probemask)) { + if ((val & probemask) == marker) /* read back correct, continue with top half */ addrlo = addrmid; - } else { + else /* not read back correct, continue with bottom half */ addrhi = addrmid; - } } return addrhi; } @@ -150,7 +155,7 @@ static void eip197_trc_cache_clear(struct safexcel_crypto_priv *priv, htable_offset + i * sizeof(u32)); } -static void eip197_trc_cache_init(struct safexcel_crypto_priv *priv) +static int eip197_trc_cache_init(struct safexcel_crypto_priv *priv) { u32 val, dsize, asize; int cs_rc_max, cs_ht_wc, cs_trc_rec_wc, cs_trc_lg_rec_wc; @@ -183,7 +188,7 @@ static void eip197_trc_cache_init(struct safexcel_crypto_priv *priv) writel(val, priv->base + EIP197_TRC_PARAMS); /* Probed data RAM size in bytes */ - dsize = eip197_trc_cache_probe(priv, maxbanks, 0xffffffff); + dsize = eip197_trc_cache_probe(priv, maxbanks, 0xffffffff, 32); /* * Now probe the administration RAM size pretty much the same way @@ -196,11 +201,18 @@ static void eip197_trc_cache_init(struct safexcel_crypto_priv *priv) writel(val, priv->base + EIP197_TRC_PARAMS); /* Probed admin RAM size in admin words */ - asize = eip197_trc_cache_probe(priv, 0, 0xbfffffff) >> 4; + asize = eip197_trc_cache_probe(priv, 0, 0x3fffffff, 16) >> 4; /* Clear any ECC errors detected while probing! */ writel(0, priv->base + EIP197_TRC_ECCCTRL); + /* Sanity check probing results */ + if (dsize < EIP197_MIN_DSIZE || asize < EIP197_MIN_ASIZE) { + dev_err(priv->dev, "Record cache probing failed (%d,%d).", + dsize, asize); + return -ENODEV; + } + /* * Determine optimal configuration from RAM sizes * Note that we assume that the physical RAM configuration is sane @@ -251,6 +263,7 @@ static void eip197_trc_cache_init(struct safexcel_crypto_priv *priv) dev_info(priv->dev, "TRC init: %dd,%da (%dr,%dh)\n", dsize, asize, cs_rc_max, cs_ht_wc + cs_ht_wc); + return 0; } static void eip197_init_firmware(struct safexcel_crypto_priv *priv) @@ -737,7 +750,10 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv) writel(GENMASK(30, 20), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK); if (priv->flags & SAFEXCEL_HW_EIP197) { - eip197_trc_cache_init(priv); + ret = eip197_trc_cache_init(priv); + if (ret) + return ret; + priv->flags |= EIP197_TRC_CACHE; ret = eip197_load_firmwares(priv); diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 9522594cd3a4..548eb81c1004 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -439,6 +439,8 @@ struct safexcel_context_record { #define EIP197_TRC_PARAMS2_RC_SZ_SMALL(n) ((n) << 18) /* Cache helpers */ +#define EIP197_MIN_DSIZE 1024 +#define EIP197_MIN_ASIZE 8 #define EIP197_CS_TRC_REC_WC 64 #define EIP197_CS_RC_SIZE (4 * sizeof(u32)) #define EIP197_CS_RC_NEXT(x) (x) -- cgit From a19052d4228783052c0ebcb83f74ccac3986acdf Mon Sep 17 00:00:00 2001 From: Pascal van Leeuwen Date: Tue, 17 Sep 2019 12:07:59 +0200 Subject: crypto: inside-secure - Added support for the rfc4106(gcm(aes)) AEAD This patch adds support for rfc4106(gcm(aes)) for use with IPsec ESP Signed-off-by: Pascal van Leeuwen Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 1 + drivers/crypto/inside-secure/safexcel.h | 1 + drivers/crypto/inside-secure/safexcel_cipher.c | 112 ++++++++++++++++++++----- 3 files changed, 91 insertions(+), 23 deletions(-) (limited to 'drivers/crypto/inside-secure/safexcel.c') diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 1878b79336cb..a06a744a9bb8 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1220,6 +1220,7 @@ static struct safexcel_alg_template *safexcel_algs[] = { &safexcel_alg_authenc_hmac_sha224_cbc_des, &safexcel_alg_authenc_hmac_sha512_cbc_des, &safexcel_alg_authenc_hmac_sha384_cbc_des, + &safexcel_alg_rfc4106_gcm, }; static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv) diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 548eb81c1004..79ee8927ad86 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -912,5 +912,6 @@ extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_des; extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des; extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des; extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des; +extern struct safexcel_alg_template safexcel_alg_rfc4106_gcm; #endif diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index 6c3fdd650ecc..97d1fa14a717 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -77,47 +77,47 @@ static void safexcel_cipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv, { u32 block_sz = 0; - if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD) { + if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD || + ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) { cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; /* 32 bit nonce */ cdesc->control_data.token[0] = ctx->nonce; /* 64 bit IV part */ memcpy(&cdesc->control_data.token[1], iv, 8); - /* 32 bit counter, start at 1 (big endian!) */ - cdesc->control_data.token[3] = cpu_to_be32(1); + + if (ctx->alg == SAFEXCEL_CHACHA20) { + /* 32 bit counter, starting at 0 */ + cdesc->control_data.token[3] = 0; + } else { + /* 32 bit counter, start at 1 (big endian!) */ + cdesc->control_data.token[3] = cpu_to_be32(1); + } return; - } else if (ctx->alg == SAFEXCEL_CHACHA20) { + } else if (ctx->xcm == EIP197_XCM_MODE_GCM || + (ctx->aead && ctx->alg == SAFEXCEL_CHACHA20)) { cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; - if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) { - /* 32 bit nonce part */ - cdesc->control_data.token[0] = ctx->nonce; - /* 64 bit IV part */ - memcpy(&cdesc->control_data.token[1], iv, 8); - /* 32 bit counter, starting at 0 */ - cdesc->control_data.token[3] = 0; - } else if (ctx->aead) { - /* 96 bit nonce part */ - memcpy(&cdesc->control_data.token[0], iv, 12); + /* 96 bit IV part */ + memcpy(&cdesc->control_data.token[0], iv, 12); + + if (ctx->alg == SAFEXCEL_CHACHA20) { /* 32 bit counter, starting at 0 */ cdesc->control_data.token[3] = 0; } else { - /* 96 bit nonce part */ - memcpy(&cdesc->control_data.token[0], &iv[4], 12); - /* 32 bit counter */ - cdesc->control_data.token[3] = *(u32 *)iv; + /* 32 bit counter, start at 1 (big endian!) */ + cdesc->control_data.token[3] = cpu_to_be32(1); } return; - } else if (ctx->xcm == EIP197_XCM_MODE_GCM) { + } else if (ctx->alg == SAFEXCEL_CHACHA20) { cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; - /* 96 bit IV part */ - memcpy(&cdesc->control_data.token[0], iv, 12); - /* 32 bit counter, start at 1 (big endian!) */ - cdesc->control_data.token[3] = cpu_to_be32(1); + /* 96 bit nonce part */ + memcpy(&cdesc->control_data.token[0], &iv[4], 12); + /* 32 bit counter */ + cdesc->control_data.token[3] = *(u32 *)iv; return; } else if (ctx->xcm == EIP197_XCM_MODE_CCM) { @@ -3429,3 +3429,69 @@ struct safexcel_alg_template safexcel_alg_authenc_hmac_sm3_ctr_sm4 = { }, }, }; + +static int safexcel_rfc4106_gcm_setkey(struct crypto_aead *ctfm, const u8 *key, + unsigned int len) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + /* last 4 bytes of key are the nonce! */ + ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE); + + len -= CTR_RFC3686_NONCE_SIZE; + return safexcel_aead_gcm_setkey(ctfm, key, len); +} + +static int safexcel_rfc4106_gcm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + return crypto_rfc4106_check_authsize(authsize); +} + +static int safexcel_rfc4106_encrypt(struct aead_request *req) +{ + return crypto_ipsec_check_assoclen(req->assoclen) ?: + safexcel_aead_encrypt(req); +} + +static int safexcel_rfc4106_decrypt(struct aead_request *req) +{ + return crypto_ipsec_check_assoclen(req->assoclen) ?: + safexcel_aead_decrypt(req); +} + +static int safexcel_rfc4106_gcm_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + int ret; + + ret = safexcel_aead_gcm_cra_init(tfm); + ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP; + return ret; +} + +struct safexcel_alg_template safexcel_alg_rfc4106_gcm = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH, + .alg.aead = { + .setkey = safexcel_rfc4106_gcm_setkey, + .setauthsize = safexcel_rfc4106_gcm_setauthsize, + .encrypt = safexcel_rfc4106_encrypt, + .decrypt = safexcel_rfc4106_decrypt, + .ivsize = GCM_RFC4106_IV_SIZE, + .maxauthsize = GHASH_DIGEST_SIZE, + .base = { + .cra_name = "rfc4106(gcm(aes))", + .cra_driver_name = "safexcel-rfc4106-gcm-aes", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_rfc4106_gcm_cra_init, + .cra_exit = safexcel_aead_gcm_cra_exit, + }, + }, +}; -- cgit From 92c60cefa465035e8ce4949c62a3c38269f3d1eb Mon Sep 17 00:00:00 2001 From: Pascal van Leeuwen Date: Tue, 17 Sep 2019 12:08:00 +0200 Subject: crypto: inside-secure - Added support for the rfc4543(gcm(aes)) "AEAD" This patch adds support for rfc4543(gcm(aes)) - i.e. AES-GMAC - for use with IPsec ESP Signed-off-by: Pascal van Leeuwen Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 1 + drivers/crypto/inside-secure/safexcel.h | 2 + drivers/crypto/inside-secure/safexcel_cipher.c | 86 +++++++++++++++++++++----- 3 files changed, 74 insertions(+), 15 deletions(-) (limited to 'drivers/crypto/inside-secure/safexcel.c') diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index a06a744a9bb8..56557210a39f 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1221,6 +1221,7 @@ static struct safexcel_alg_template *safexcel_algs[] = { &safexcel_alg_authenc_hmac_sha512_cbc_des, &safexcel_alg_authenc_hmac_sha384_cbc_des, &safexcel_alg_rfc4106_gcm, + &safexcel_alg_rfc4543_gcm, }; static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv) diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 79ee8927ad86..6cb4197ee09e 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -407,6 +407,7 @@ struct safexcel_context_record { #define EIP197_XCM_MODE_CCM 2 #define EIP197_AEAD_TYPE_IPSEC_ESP 2 +#define EIP197_AEAD_TYPE_IPSEC_ESP_GMAC 3 #define EIP197_AEAD_IPSEC_IV_SIZE 8 #define EIP197_AEAD_IPSEC_NONCE_SIZE 4 @@ -913,5 +914,6 @@ extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des; extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des; extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des; extern struct safexcel_alg_template safexcel_alg_rfc4106_gcm; +extern struct safexcel_alg_template safexcel_alg_rfc4543_gcm; #endif diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index 97d1fa14a717..5cccf3c48b60 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -47,7 +47,7 @@ struct safexcel_cipher_ctx { u32 mode; enum safexcel_cipher_alg alg; - char aead; /* !=0=AEAD, 2=IPSec ESP AEAD */ + char aead; /* !=0=AEAD, 2=IPSec ESP AEAD, 3=IPsec ESP GMAC */ char xcm; /* 0=authenc, 1=GCM, 2 reserved for CCM */ __le32 key[16]; @@ -78,7 +78,7 @@ static void safexcel_cipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv, u32 block_sz = 0; if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD || - ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) { + ctx->aead & EIP197_AEAD_TYPE_IPSEC_ESP) { /* _ESP and _ESP_GMAC */ cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; /* 32 bit nonce */ @@ -219,7 +219,7 @@ static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv, } if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) { - /* For ESP mode, skip over the IV */ + /* For ESP mode (and not GMAC), skip over the IV */ token[7].opcode = EIP197_TOKEN_OPCODE_DIRECTION; token[7].packet_length = EIP197_AEAD_IPSEC_IV_SIZE; @@ -235,10 +235,18 @@ static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv, token[10].opcode = EIP197_TOKEN_OPCODE_DIRECTION; token[10].packet_length = cryptlen; token[10].stat = EIP197_TOKEN_STAT_LAST_HASH; - token[10].instructions = EIP197_TOKEN_INS_LAST | - EIP197_TOKEN_INS_TYPE_CRYPTO | - EIP197_TOKEN_INS_TYPE_HASH | - EIP197_TOKEN_INS_TYPE_OUTPUT; + if (unlikely(ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP_GMAC)) { + token[6].instructions = EIP197_TOKEN_INS_TYPE_HASH; + /* Do not send to crypt engine in case of GMAC */ + token[10].instructions = EIP197_TOKEN_INS_LAST | + EIP197_TOKEN_INS_TYPE_HASH | + EIP197_TOKEN_INS_TYPE_OUTPUT; + } else { + token[10].instructions = EIP197_TOKEN_INS_LAST | + EIP197_TOKEN_INS_TYPE_CRYPTO | + EIP197_TOKEN_INS_TYPE_HASH | + EIP197_TOKEN_INS_TYPE_OUTPUT; + } } else if (ctx->xcm != EIP197_XCM_MODE_CCM) { token[6].stat = EIP197_TOKEN_STAT_LAST_HASH; } @@ -494,17 +502,21 @@ static int safexcel_context_control(struct safexcel_cipher_ctx *ctx, ctx->hash_alg | CONTEXT_CONTROL_SIZE(ctrl_size); } - if (sreq->direction == SAFEXCEL_ENCRYPT) - cdesc->control_data.control0 |= - (ctx->xcm == EIP197_XCM_MODE_CCM) ? - CONTEXT_CONTROL_TYPE_HASH_ENCRYPT_OUT : - CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT; + if (sreq->direction == SAFEXCEL_ENCRYPT && + (ctx->xcm == EIP197_XCM_MODE_CCM || + ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP_GMAC)) + cdesc->control_data.control0 |= + CONTEXT_CONTROL_TYPE_HASH_ENCRYPT_OUT; + else if (sreq->direction == SAFEXCEL_ENCRYPT) + cdesc->control_data.control0 |= + CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT; + else if (ctx->xcm == EIP197_XCM_MODE_CCM) + cdesc->control_data.control0 |= + CONTEXT_CONTROL_TYPE_DECRYPT_HASH_IN; else cdesc->control_data.control0 |= - (ctx->xcm == EIP197_XCM_MODE_CCM) ? - CONTEXT_CONTROL_TYPE_DECRYPT_HASH_IN : - CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN; + CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN; } else { if (sreq->direction == SAFEXCEL_ENCRYPT) cdesc->control_data.control0 = @@ -3495,3 +3507,47 @@ struct safexcel_alg_template safexcel_alg_rfc4106_gcm = { }, }, }; + +static int safexcel_rfc4543_gcm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + if (authsize != GHASH_DIGEST_SIZE) + return -EINVAL; + + return 0; +} + +static int safexcel_rfc4543_gcm_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + int ret; + + ret = safexcel_aead_gcm_cra_init(tfm); + ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP_GMAC; + return ret; +} + +struct safexcel_alg_template safexcel_alg_rfc4543_gcm = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH, + .alg.aead = { + .setkey = safexcel_rfc4106_gcm_setkey, + .setauthsize = safexcel_rfc4543_gcm_setauthsize, + .encrypt = safexcel_rfc4106_encrypt, + .decrypt = safexcel_rfc4106_decrypt, + .ivsize = GCM_RFC4543_IV_SIZE, + .maxauthsize = GHASH_DIGEST_SIZE, + .base = { + .cra_name = "rfc4543(gcm(aes))", + .cra_driver_name = "safexcel-rfc4543-gcm-aes", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_rfc4543_gcm_cra_init, + .cra_exit = safexcel_aead_gcm_cra_exit, + }, + }, +}; -- cgit From a9a89624f086d82483b5e26bb98e8a04a6d9ab0a Mon Sep 17 00:00:00 2001 From: Pascal van Leeuwen Date: Tue, 17 Sep 2019 12:08:01 +0200 Subject: crypto: inside-secure - Added support for the rfc4309(ccm(aes)) AEAD This patch adds support for rfc4309(ccm(aes)) for use with IPsec ESP Signed-off-by: Pascal van Leeuwen Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 1 + drivers/crypto/inside-secure/safexcel.h | 5 +- drivers/crypto/inside-secure/safexcel_cipher.c | 165 +++++++++++++++++++------ 3 files changed, 134 insertions(+), 37 deletions(-) (limited to 'drivers/crypto/inside-secure/safexcel.c') diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 56557210a39f..ed34118e747a 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1222,6 +1222,7 @@ static struct safexcel_alg_template *safexcel_algs[] = { &safexcel_alg_authenc_hmac_sha384_cbc_des, &safexcel_alg_rfc4106_gcm, &safexcel_alg_rfc4543_gcm, + &safexcel_alg_rfc4309_ccm, }; static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv) diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 6cb4197ee09e..0b95389e3caf 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -28,7 +28,7 @@ /* Static configuration */ #define EIP197_DEFAULT_RING_SIZE 400 -#define EIP197_MAX_TOKENS 18 +#define EIP197_MAX_TOKENS 19 #define EIP197_MAX_RINGS 4 #define EIP197_FETCH_DEPTH 2 #define EIP197_MAX_BATCH_SZ 64 @@ -410,6 +410,8 @@ struct safexcel_context_record { #define EIP197_AEAD_TYPE_IPSEC_ESP_GMAC 3 #define EIP197_AEAD_IPSEC_IV_SIZE 8 #define EIP197_AEAD_IPSEC_NONCE_SIZE 4 +#define EIP197_AEAD_IPSEC_COUNTER_SIZE 4 +#define EIP197_AEAD_IPSEC_CCM_NONCE_SIZE 3 /* The hash counter given to the engine in the context has a granularity of * 64 bits. @@ -915,5 +917,6 @@ extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des; extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des; extern struct safexcel_alg_template safexcel_alg_rfc4106_gcm; extern struct safexcel_alg_template safexcel_alg_rfc4543_gcm; +extern struct safexcel_alg_template safexcel_alg_rfc4309_ccm; #endif diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index 5cccf3c48b60..cecc56073337 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -86,7 +86,8 @@ static void safexcel_cipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv, /* 64 bit IV part */ memcpy(&cdesc->control_data.token[1], iv, 8); - if (ctx->alg == SAFEXCEL_CHACHA20) { + if (ctx->alg == SAFEXCEL_CHACHA20 || + ctx->xcm == EIP197_XCM_MODE_CCM) { /* 32 bit counter, starting at 0 */ cdesc->control_data.token[3] = 0; } else { @@ -189,39 +190,39 @@ static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv, if (direction == SAFEXCEL_ENCRYPT) { /* align end of instruction sequence to end of token */ token = (struct safexcel_token *)(cdesc->control_data.token + - EIP197_MAX_TOKENS - 13); + EIP197_MAX_TOKENS - 14); - token[12].opcode = EIP197_TOKEN_OPCODE_INSERT; - token[12].packet_length = digestsize; - token[12].stat = EIP197_TOKEN_STAT_LAST_HASH | + token[13].opcode = EIP197_TOKEN_OPCODE_INSERT; + token[13].packet_length = digestsize; + token[13].stat = EIP197_TOKEN_STAT_LAST_HASH | EIP197_TOKEN_STAT_LAST_PACKET; - token[12].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT | + token[13].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT | EIP197_TOKEN_INS_INSERT_HASH_DIGEST; } else { cryptlen -= digestsize; /* align end of instruction sequence to end of token */ token = (struct safexcel_token *)(cdesc->control_data.token + - EIP197_MAX_TOKENS - 14); + EIP197_MAX_TOKENS - 15); - token[12].opcode = EIP197_TOKEN_OPCODE_RETRIEVE; - token[12].packet_length = digestsize; - token[12].stat = EIP197_TOKEN_STAT_LAST_HASH | + token[13].opcode = EIP197_TOKEN_OPCODE_RETRIEVE; + token[13].packet_length = digestsize; + token[13].stat = EIP197_TOKEN_STAT_LAST_HASH | EIP197_TOKEN_STAT_LAST_PACKET; - token[12].instructions = EIP197_TOKEN_INS_INSERT_HASH_DIGEST; + token[13].instructions = EIP197_TOKEN_INS_INSERT_HASH_DIGEST; - token[13].opcode = EIP197_TOKEN_OPCODE_VERIFY; - token[13].packet_length = digestsize | + token[14].opcode = EIP197_TOKEN_OPCODE_VERIFY; + token[14].packet_length = digestsize | EIP197_TOKEN_HASH_RESULT_VERIFY; - token[13].stat = EIP197_TOKEN_STAT_LAST_HASH | + token[14].stat = EIP197_TOKEN_STAT_LAST_HASH | EIP197_TOKEN_STAT_LAST_PACKET; - token[13].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT; + token[14].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT; } if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) { /* For ESP mode (and not GMAC), skip over the IV */ - token[7].opcode = EIP197_TOKEN_OPCODE_DIRECTION; - token[7].packet_length = EIP197_AEAD_IPSEC_IV_SIZE; + token[8].opcode = EIP197_TOKEN_OPCODE_DIRECTION; + token[8].packet_length = EIP197_AEAD_IPSEC_IV_SIZE; assoclen -= EIP197_AEAD_IPSEC_IV_SIZE; } @@ -232,17 +233,17 @@ static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv, EIP197_TOKEN_INS_TYPE_HASH; if (likely(cryptlen || ctx->alg == SAFEXCEL_CHACHA20)) { - token[10].opcode = EIP197_TOKEN_OPCODE_DIRECTION; - token[10].packet_length = cryptlen; - token[10].stat = EIP197_TOKEN_STAT_LAST_HASH; + token[11].opcode = EIP197_TOKEN_OPCODE_DIRECTION; + token[11].packet_length = cryptlen; + token[11].stat = EIP197_TOKEN_STAT_LAST_HASH; if (unlikely(ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP_GMAC)) { token[6].instructions = EIP197_TOKEN_INS_TYPE_HASH; /* Do not send to crypt engine in case of GMAC */ - token[10].instructions = EIP197_TOKEN_INS_LAST | + token[11].instructions = EIP197_TOKEN_INS_LAST | EIP197_TOKEN_INS_TYPE_HASH | EIP197_TOKEN_INS_TYPE_OUTPUT; } else { - token[10].instructions = EIP197_TOKEN_INS_LAST | + token[11].instructions = EIP197_TOKEN_INS_LAST | EIP197_TOKEN_INS_TYPE_CRYPTO | EIP197_TOKEN_INS_TYPE_HASH | EIP197_TOKEN_INS_TYPE_OUTPUT; @@ -254,16 +255,17 @@ static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv, if (!ctx->xcm) return; - token[8].opcode = EIP197_TOKEN_OPCODE_INSERT_REMRES; - token[8].packet_length = 0; - token[8].instructions = AES_BLOCK_SIZE; + token[9].opcode = EIP197_TOKEN_OPCODE_INSERT_REMRES; + token[9].packet_length = 0; + token[9].instructions = AES_BLOCK_SIZE; - token[9].opcode = EIP197_TOKEN_OPCODE_INSERT; - token[9].packet_length = AES_BLOCK_SIZE; - token[9].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT | - EIP197_TOKEN_INS_TYPE_CRYPTO; + token[10].opcode = EIP197_TOKEN_OPCODE_INSERT; + token[10].packet_length = AES_BLOCK_SIZE; + token[10].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT | + EIP197_TOKEN_INS_TYPE_CRYPTO; if (ctx->xcm != EIP197_XCM_MODE_GCM) { + u8 *final_iv = (u8 *)cdesc->control_data.token; u8 *cbcmaciv = (u8 *)&token[1]; u32 *aadlen = (u32 *)&token[5]; @@ -274,11 +276,11 @@ static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv, token[0].instructions = EIP197_TOKEN_INS_ORIGIN_TOKEN | EIP197_TOKEN_INS_TYPE_HASH; /* Variable length IV part */ - memcpy(cbcmaciv, iv, 15 - iv[0]); + memcpy(cbcmaciv, final_iv, 15 - final_iv[0]); /* fixup flags byte */ cbcmaciv[0] |= ((assoclen > 0) << 6) | ((digestsize - 2) << 2); /* Clear upper bytes of variable message length to 0 */ - memset(cbcmaciv + 15 - iv[0], 0, iv[0] - 1); + memset(cbcmaciv + 15 - final_iv[0], 0, final_iv[0] - 1); /* insert lower 2 bytes of message length */ cbcmaciv[14] = cryptlen >> 8; cbcmaciv[15] = cryptlen & 255; @@ -299,13 +301,13 @@ static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv, token[7].instructions = EIP197_TOKEN_INS_TYPE_HASH; /* Align crypto data towards hash engine */ - token[10].stat = 0; + token[11].stat = 0; - token[11].opcode = EIP197_TOKEN_OPCODE_INSERT; + token[12].opcode = EIP197_TOKEN_OPCODE_INSERT; cryptlen &= 15; - token[11].packet_length = cryptlen ? 16 - cryptlen : 0; - token[11].stat = EIP197_TOKEN_STAT_LAST_HASH; - token[11].instructions = EIP197_TOKEN_INS_TYPE_HASH; + token[12].packet_length = cryptlen ? 16 - cryptlen : 0; + token[12].stat = EIP197_TOKEN_STAT_LAST_HASH; + token[12].instructions = EIP197_TOKEN_INS_TYPE_HASH; } else { token[7].stat = EIP197_TOKEN_STAT_LAST_HASH; token[7].instructions = EIP197_TOKEN_INS_LAST | @@ -3551,3 +3553,94 @@ struct safexcel_alg_template safexcel_alg_rfc4543_gcm = { }, }, }; + +static int safexcel_rfc4309_ccm_setkey(struct crypto_aead *ctfm, const u8 *key, + unsigned int len) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + /* First byte of the nonce = L = always 3 for RFC4309 (4 byte ctr) */ + *(u8 *)&ctx->nonce = EIP197_AEAD_IPSEC_COUNTER_SIZE - 1; + /* last 3 bytes of key are the nonce! */ + memcpy((u8 *)&ctx->nonce + 1, key + len - + EIP197_AEAD_IPSEC_CCM_NONCE_SIZE, + EIP197_AEAD_IPSEC_CCM_NONCE_SIZE); + + len -= EIP197_AEAD_IPSEC_CCM_NONCE_SIZE; + return safexcel_aead_ccm_setkey(ctfm, key, len); +} + +static int safexcel_rfc4309_ccm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + /* Borrowed from crypto/ccm.c */ + switch (authsize) { + case 8: + case 12: + case 16: + break; + default: + return -EINVAL; + } + + return 0; +} + +static int safexcel_rfc4309_ccm_encrypt(struct aead_request *req) +{ + struct safexcel_cipher_req *creq = aead_request_ctx(req); + + /* Borrowed from crypto/ccm.c */ + if (req->assoclen != 16 && req->assoclen != 20) + return -EINVAL; + + return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); +} + +static int safexcel_rfc4309_ccm_decrypt(struct aead_request *req) +{ + struct safexcel_cipher_req *creq = aead_request_ctx(req); + + /* Borrowed from crypto/ccm.c */ + if (req->assoclen != 16 && req->assoclen != 20) + return -EINVAL; + + return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); +} + +static int safexcel_rfc4309_ccm_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + int ret; + + ret = safexcel_aead_ccm_cra_init(tfm); + ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP; + return ret; +} + +struct safexcel_alg_template safexcel_alg_rfc4309_ccm = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_CBC_MAC_ALL, + .alg.aead = { + .setkey = safexcel_rfc4309_ccm_setkey, + .setauthsize = safexcel_rfc4309_ccm_setauthsize, + .encrypt = safexcel_rfc4309_ccm_encrypt, + .decrypt = safexcel_rfc4309_ccm_decrypt, + .ivsize = EIP197_AEAD_IPSEC_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + .base = { + .cra_name = "rfc4309(ccm(aes))", + .cra_driver_name = "safexcel-rfc4309-ccm-aes", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_rfc4309_ccm_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; -- cgit From 84ca4e54ab792b550b802950639ab31c57ebe989 Mon Sep 17 00:00:00 2001 From: Pascal van Leeuwen Date: Wed, 18 Sep 2019 08:42:39 +0200 Subject: crypto: inside-secure - Add support for 256 bit wide internal bus This patch adds support for large EIP197's with a 256 bit wide internal bus, which affects the format of the result descriptor due to internal alignment requirements. Signed-off-by: Pascal van Leeuwen Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 101 +++++++++++++++------------ drivers/crypto/inside-secure/safexcel.h | 18 +++-- drivers/crypto/inside-secure/safexcel_ring.c | 4 +- 3 files changed, 72 insertions(+), 51 deletions(-) (limited to 'drivers/crypto/inside-secure/safexcel.c') diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index ed34118e747a..0bcf36c3810b 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -492,12 +492,12 @@ static int safexcel_hw_setup_cdesc_rings(struct safexcel_crypto_priv *priv) writel(upper_32_bits(priv->ring[i].cdr.base_dma), EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI); - writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.cd_offset << 16) | + writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.cd_offset << 14) | priv->config.cd_size, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE); writel(((cd_fetch_cnt * (cd_size_rnd << priv->hwconfig.hwdataw)) << 16) | - (cd_fetch_cnt * priv->config.cd_offset), + (cd_fetch_cnt * (priv->config.cd_offset / sizeof(u32))), EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG); /* Configure DMA tx control */ @@ -540,13 +540,13 @@ static int safexcel_hw_setup_rdesc_rings(struct safexcel_crypto_priv *priv) writel(upper_32_bits(priv->ring[i].rdr.base_dma), EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI); - writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.rd_offset << 16) | + writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.rd_offset << 14) | priv->config.rd_size, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE); writel(((rd_fetch_cnt * (rd_size_rnd << priv->hwconfig.hwdataw)) << 16) | - (rd_fetch_cnt * priv->config.rd_offset), + (rd_fetch_cnt * (priv->config.rd_offset / sizeof(u32))), EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG); /* Configure DMA tx control */ @@ -572,7 +572,7 @@ static int safexcel_hw_setup_rdesc_rings(struct safexcel_crypto_priv *priv) static int safexcel_hw_init(struct safexcel_crypto_priv *priv) { u32 val; - int i, ret, pe; + int i, ret, pe, opbuflo, opbufhi; dev_dbg(priv->dev, "HW init: using %d pipe(s) and %d ring(s)\n", priv->config.pes, priv->config.rings); @@ -652,9 +652,16 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv) ; /* DMA transfer size to use */ + if (priv->hwconfig.hwnumpes > 4) { + opbuflo = 9; + opbufhi = 10; + } else { + opbuflo = 7; + opbufhi = 8; + } val = EIP197_HIA_DSE_CFG_DIS_DEBUG; - val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(7) | - EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(8); + val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(opbuflo) | + EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(opbufhi); val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS); val |= EIP197_HIA_DSE_CFG_ALWAYS_BUFFERABLE; /* FIXME: instability issues can occur for EIP97 but disabling @@ -668,8 +675,8 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv) writel(0, EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe)); /* Configure the procesing engine thresholds */ - writel(EIP197_PE_OUT_DBUF_THRES_MIN(7) | - EIP197_PE_OUT_DBUF_THRES_MAX(8), + writel(EIP197_PE_OUT_DBUF_THRES_MIN(opbuflo) | + EIP197_PE_OUT_DBUF_THRES_MAX(opbufhi), EIP197_PE(priv) + EIP197_PE_OUT_DBUF_THRES(pe)); /* Processing Engine configuration */ @@ -709,7 +716,7 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv) writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR); - writel((EIP197_DEFAULT_RING_SIZE * priv->config.cd_offset) << 2, + writel((EIP197_DEFAULT_RING_SIZE * priv->config.cd_offset), EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_SIZE); } @@ -732,7 +739,7 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv) EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR); /* Ring size */ - writel((EIP197_DEFAULT_RING_SIZE * priv->config.rd_offset) << 2, + writel((EIP197_DEFAULT_RING_SIZE * priv->config.rd_offset), EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_SIZE); } @@ -852,20 +859,24 @@ finalize: spin_unlock_bh(&priv->ring[ring].lock); /* let the RDR know we have pending descriptors */ - writel((rdesc * priv->config.rd_offset) << 2, + writel((rdesc * priv->config.rd_offset), EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT); /* let the CDR know we have pending descriptors */ - writel((cdesc * priv->config.cd_offset) << 2, + writel((cdesc * priv->config.cd_offset), EIP197_HIA_CDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT); } inline int safexcel_rdesc_check_errors(struct safexcel_crypto_priv *priv, - struct safexcel_result_desc *rdesc) + void *rdp) { - if (likely((!rdesc->descriptor_overflow) && - (!rdesc->buffer_overflow) && - (!rdesc->result_data.error_code))) + struct safexcel_result_desc *rdesc = rdp; + struct result_data_desc *result_data = rdp + priv->config.res_offset; + + if (likely((!rdesc->last_seg) || /* Rest only valid if last seg! */ + ((!rdesc->descriptor_overflow) && + (!rdesc->buffer_overflow) && + (!result_data->error_code)))) return 0; if (rdesc->descriptor_overflow) @@ -874,13 +885,14 @@ inline int safexcel_rdesc_check_errors(struct safexcel_crypto_priv *priv, if (rdesc->buffer_overflow) dev_err(priv->dev, "Buffer overflow detected"); - if (rdesc->result_data.error_code & 0x4066) { + if (result_data->error_code & 0x4066) { /* Fatal error (bits 1,2,5,6 & 14) */ dev_err(priv->dev, "result descriptor error (%x)", - rdesc->result_data.error_code); + result_data->error_code); + return -EIO; - } else if (rdesc->result_data.error_code & + } else if (result_data->error_code & (BIT(7) | BIT(4) | BIT(3) | BIT(0))) { /* * Give priority over authentication fails: @@ -888,7 +900,7 @@ inline int safexcel_rdesc_check_errors(struct safexcel_crypto_priv *priv, * something wrong with the input! */ return -EINVAL; - } else if (rdesc->result_data.error_code & BIT(9)) { + } else if (result_data->error_code & BIT(9)) { /* Authentication failed */ return -EBADMSG; } @@ -1019,7 +1031,7 @@ handle_results: acknowledge: if (i) writel(EIP197_xDR_PROC_xD_PKT(i) | - EIP197_xDR_PROC_xD_COUNT(tot_descs * priv->config.rd_offset), + (tot_descs * priv->config.rd_offset), EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT); /* If the number of requests overflowed the counter, try to proceed more @@ -1292,30 +1304,25 @@ static void safexcel_unregister_algorithms(struct safexcel_crypto_priv *priv) static void safexcel_configure(struct safexcel_crypto_priv *priv) { - u32 val, mask = 0; - - val = readl(EIP197_HIA_AIC_G(priv) + EIP197_HIA_OPTIONS); + u32 mask = BIT(priv->hwconfig.hwdataw) - 1; - /* Read number of PEs from the engine */ - if (priv->flags & SAFEXCEL_HW_EIP197) - /* Wider field width for all EIP197 type engines */ - mask = EIP197_N_PES_MASK; - else - /* Narrow field width for EIP97 type engine */ - mask = EIP97_N_PES_MASK; - - priv->config.pes = (val >> EIP197_N_PES_OFFSET) & mask; - - priv->config.rings = min_t(u32, val & GENMASK(3, 0), max_rings); + priv->config.pes = priv->hwconfig.hwnumpes; + priv->config.rings = min_t(u32, priv->hwconfig.hwnumrings, max_rings); - val = (val & GENMASK(27, 25)) >> 25; - mask = BIT(val) - 1; - - priv->config.cd_size = (sizeof(struct safexcel_command_desc) / sizeof(u32)); + priv->config.cd_size = EIP197_CD64_FETCH_SIZE; priv->config.cd_offset = (priv->config.cd_size + mask) & ~mask; - priv->config.rd_size = (sizeof(struct safexcel_result_desc) / sizeof(u32)); + /* res token is behind the descr, but ofs must be rounded to buswdth */ + priv->config.res_offset = (EIP197_RD64_FETCH_SIZE + mask) & ~mask; + /* now the size of the descr is this 1st part plus the result struct */ + priv->config.rd_size = priv->config.res_offset + + EIP197_RD64_RESULT_SIZE; priv->config.rd_offset = (priv->config.rd_size + mask) & ~mask; + + /* convert dwords to bytes */ + priv->config.cd_offset *= sizeof(u32); + priv->config.rd_offset *= sizeof(u32); + priv->config.res_offset *= sizeof(u32); } static void safexcel_init_register_offsets(struct safexcel_crypto_priv *priv) @@ -1457,6 +1464,10 @@ static int safexcel_probe_generic(void *pdev, priv->hwconfig.hwrfsize = ((hiaopt >> EIP197_RFSIZE_OFFSET) & EIP197_RFSIZE_MASK) + EIP197_RFSIZE_ADJUST; + priv->hwconfig.hwnumpes = (hiaopt >> EIP197_N_PES_OFFSET) & + EIP197_N_PES_MASK; + priv->hwconfig.hwnumrings = (hiaopt >> EIP197_N_RINGS_OFFSET) & + EIP197_N_RINGS_MASK; } else { /* EIP97 */ priv->hwconfig.hwdataw = (hiaopt >> EIP197_HWDATAW_OFFSET) & @@ -1465,6 +1476,9 @@ static int safexcel_probe_generic(void *pdev, EIP97_CFSIZE_MASK; priv->hwconfig.hwrfsize = (hiaopt >> EIP97_RFSIZE_OFFSET) & EIP97_RFSIZE_MASK; + priv->hwconfig.hwnumpes = 1; /* by definition */ + priv->hwconfig.hwnumrings = (hiaopt >> EIP197_N_RINGS_OFFSET) & + EIP197_N_RINGS_MASK; } /* Get supported algorithms from EIP96 transform engine */ @@ -1472,8 +1486,9 @@ static int safexcel_probe_generic(void *pdev, EIP197_PE_EIP96_OPTIONS(0)); /* Print single info line describing what we just detected */ - dev_info(priv->dev, "EIP%d:%x(%d)-HIA:%x(%d,%d,%d),PE:%x,alg:%08x\n", - peid, priv->hwconfig.hwver, hwctg, priv->hwconfig.hiaver, + dev_info(priv->dev, "EIP%d:%x(%d,%d,%d,%d)-HIA:%x(%d,%d,%d),PE:%x,alg:%08x\n", + peid, priv->hwconfig.hwver, hwctg, priv->hwconfig.hwnumpes, + priv->hwconfig.hwnumrings, priv->hwconfig.hiaver, priv->hwconfig.hwdataw, priv->hwconfig.hwcfsize, priv->hwconfig.hwrfsize, priv->hwconfig.pever, priv->hwconfig.algo_flags); diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 0b95389e3caf..2400814e8c14 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -213,7 +213,6 @@ /* EIP197_HIA_xDR_PROC_COUNT */ #define EIP197_xDR_PROC_xD_PKT_OFFSET 24 #define EIP197_xDR_PROC_xD_PKT_MASK GENMASK(6, 0) -#define EIP197_xDR_PROC_xD_COUNT(n) ((n) << 2) #define EIP197_xDR_PROC_xD_PKT(n) ((n) << 24) #define EIP197_xDR_PROC_CLR_COUNT BIT(31) @@ -228,6 +227,8 @@ #define EIP197_HIA_RA_PE_CTRL_EN BIT(30) /* EIP197_HIA_OPTIONS */ +#define EIP197_N_RINGS_OFFSET 0 +#define EIP197_N_RINGS_MASK GENMASK(3, 0) #define EIP197_N_PES_OFFSET 4 #define EIP197_N_PES_MASK GENMASK(4, 0) #define EIP97_N_PES_MASK GENMASK(2, 0) @@ -486,16 +487,15 @@ struct safexcel_result_desc { u32 data_lo; u32 data_hi; - - struct result_data_desc result_data; } __packed; /* * The EIP(1)97 only needs to fetch the descriptor part of * the result descriptor, not the result token part! */ -#define EIP197_RD64_FETCH_SIZE ((sizeof(struct safexcel_result_desc) -\ - sizeof(struct result_data_desc)) /\ +#define EIP197_RD64_FETCH_SIZE (sizeof(struct safexcel_result_desc) /\ + sizeof(u32)) +#define EIP197_RD64_RESULT_SIZE (sizeof(struct result_data_desc) /\ sizeof(u32)) struct safexcel_token { @@ -582,6 +582,9 @@ struct safexcel_command_desc { struct safexcel_control_data_desc control_data; } __packed; +#define EIP197_CD64_FETCH_SIZE (sizeof(struct safexcel_command_desc) /\ + sizeof(u32)) + /* * Internal structures & functions */ @@ -625,6 +628,7 @@ struct safexcel_config { u32 rd_size; u32 rd_offset; + u32 res_offset; }; struct safexcel_work_data { @@ -734,6 +738,8 @@ struct safexcel_hwconfig { int hwdataw; int hwcfsize; int hwrfsize; + int hwnumpes; + int hwnumrings; }; struct safexcel_crypto_priv { @@ -805,7 +811,7 @@ struct safexcel_inv_result { void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring); int safexcel_rdesc_check_errors(struct safexcel_crypto_priv *priv, - struct safexcel_result_desc *rdesc); + void *rdp); void safexcel_complete(struct safexcel_crypto_priv *priv, int ring); int safexcel_invalidate_cache(struct crypto_async_request *async, struct safexcel_crypto_priv *priv, diff --git a/drivers/crypto/inside-secure/safexcel_ring.c b/drivers/crypto/inside-secure/safexcel_ring.c index 0f269b89cfd4..5323e9179b4f 100644 --- a/drivers/crypto/inside-secure/safexcel_ring.c +++ b/drivers/crypto/inside-secure/safexcel_ring.c @@ -14,7 +14,7 @@ int safexcel_init_ring_descriptors(struct safexcel_crypto_priv *priv, struct safexcel_desc_ring *cdr, struct safexcel_desc_ring *rdr) { - cdr->offset = sizeof(u32) * priv->config.cd_offset; + cdr->offset = priv->config.cd_offset; cdr->base = dmam_alloc_coherent(priv->dev, cdr->offset * EIP197_DEFAULT_RING_SIZE, &cdr->base_dma, GFP_KERNEL); @@ -24,7 +24,7 @@ int safexcel_init_ring_descriptors(struct safexcel_crypto_priv *priv, cdr->base_end = cdr->base + cdr->offset * (EIP197_DEFAULT_RING_SIZE - 1); cdr->read = cdr->base; - rdr->offset = sizeof(u32) * priv->config.rd_offset; + rdr->offset = priv->config.rd_offset; rdr->base = dmam_alloc_coherent(priv->dev, rdr->offset * EIP197_DEFAULT_RING_SIZE, &rdr->base_dma, GFP_KERNEL); -- cgit From 946a4a2a49195e2d45c71fcb0d887fc1b3fad567 Mon Sep 17 00:00:00 2001 From: Pascal van Leeuwen Date: Wed, 18 Sep 2019 08:42:40 +0200 Subject: crypto: inside-secure - Add support for HW with less ring AIC's than rings The current driver assumes one dedicated ring interrupt controller per ring. However, some existing EIP(1)97 HW has less ring AIC's than rings. This patch allows the driver to work with such HW by detecting how many ring AIC's are present and restricting the number of rings it *uses* by the number of ring AIC's present. This allows it to at least function. (optimization for the future: add ring dispatch functionality in the interrupt service routine such that multiple rings can be supported from one ring AIC, allowing all rings to be used) Signed-off-by: Pascal van Leeuwen Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 20 ++++++++++++++++---- drivers/crypto/inside-secure/safexcel.h | 4 ++++ 2 files changed, 20 insertions(+), 4 deletions(-) (limited to 'drivers/crypto/inside-secure/safexcel.c') diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 0bcf36c3810b..c40eb1befc51 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1308,6 +1308,9 @@ static void safexcel_configure(struct safexcel_crypto_priv *priv) priv->config.pes = priv->hwconfig.hwnumpes; priv->config.rings = min_t(u32, priv->hwconfig.hwnumrings, max_rings); + /* Cannot currently support more rings than we have ring AICs! */ + priv->config.rings = min_t(u32, priv->config.rings, + priv->hwconfig.hwnumraic); priv->config.cd_size = EIP197_CD64_FETCH_SIZE; priv->config.cd_offset = (priv->config.cd_size + mask) & ~mask; @@ -1481,6 +1484,15 @@ static int safexcel_probe_generic(void *pdev, EIP197_N_RINGS_MASK; } + /* Scan for ring AIC's */ + for (i = 0; i < EIP197_MAX_RING_AIC; i++) { + version = readl(EIP197_HIA_AIC_R(priv) + + EIP197_HIA_AIC_R_VERSION(i)); + if (EIP197_REG_LO16(version) != EIP201_VERSION_LE) + break; + } + priv->hwconfig.hwnumraic = i; + /* Get supported algorithms from EIP96 transform engine */ priv->hwconfig.algo_flags = readl(EIP197_PE(priv) + EIP197_PE_EIP96_OPTIONS(0)); @@ -1488,10 +1500,10 @@ static int safexcel_probe_generic(void *pdev, /* Print single info line describing what we just detected */ dev_info(priv->dev, "EIP%d:%x(%d,%d,%d,%d)-HIA:%x(%d,%d,%d),PE:%x,alg:%08x\n", peid, priv->hwconfig.hwver, hwctg, priv->hwconfig.hwnumpes, - priv->hwconfig.hwnumrings, priv->hwconfig.hiaver, - priv->hwconfig.hwdataw, priv->hwconfig.hwcfsize, - priv->hwconfig.hwrfsize, priv->hwconfig.pever, - priv->hwconfig.algo_flags); + priv->hwconfig.hwnumrings, priv->hwconfig.hwnumraic, + priv->hwconfig.hiaver, priv->hwconfig.hwdataw, + priv->hwconfig.hwcfsize, priv->hwconfig.hwrfsize, + priv->hwconfig.pever, priv->hwconfig.algo_flags); safexcel_configure(priv); diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 2400814e8c14..227771363359 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -19,6 +19,7 @@ #define EIP97_VERSION_LE 0x9e61 #define EIP197_VERSION_LE 0x3ac5 #define EIP96_VERSION_LE 0x9f60 +#define EIP201_VERSION_LE 0x36c9 #define EIP197_REG_LO16(reg) (reg & 0xffff) #define EIP197_REG_HI16(reg) ((reg >> 16) & 0xffff) #define EIP197_VERSION_MASK(reg) ((reg >> 16) & 0xfff) @@ -32,6 +33,7 @@ #define EIP197_MAX_RINGS 4 #define EIP197_FETCH_DEPTH 2 #define EIP197_MAX_BATCH_SZ 64 +#define EIP197_MAX_RING_AIC 14 #define EIP197_GFP_FLAGS(base) ((base).flags & CRYPTO_TFM_REQ_MAY_SLEEP ? \ GFP_KERNEL : GFP_ATOMIC) @@ -138,6 +140,7 @@ #define EIP197_HIA_AIC_R_ENABLED_STAT(r) (0xe010 - EIP197_HIA_AIC_R_OFF(r)) #define EIP197_HIA_AIC_R_ACK(r) (0xe010 - EIP197_HIA_AIC_R_OFF(r)) #define EIP197_HIA_AIC_R_ENABLE_CLR(r) (0xe014 - EIP197_HIA_AIC_R_OFF(r)) +#define EIP197_HIA_AIC_R_VERSION(r) (0xe01c - EIP197_HIA_AIC_R_OFF(r)) #define EIP197_HIA_AIC_G_ENABLE_CTRL 0xf808 #define EIP197_HIA_AIC_G_ENABLED_STAT 0xf810 #define EIP197_HIA_AIC_G_ACK 0xf810 @@ -740,6 +743,7 @@ struct safexcel_hwconfig { int hwrfsize; int hwnumpes; int hwnumrings; + int hwnumraic; }; struct safexcel_crypto_priv { -- cgit From 5fd39c4d96c9bcbda46960772a01d90f5f72037f Mon Sep 17 00:00:00 2001 From: Pascal van Leeuwen Date: Wed, 18 Sep 2019 12:41:26 +0200 Subject: crypto: inside-secure - Add support for the EIP196 This patch adds support for the EIP196, which is an EIP197 derivative that has no classification hardware and a simplified record cache. The patch has been tested with the eip196b-ie and eip197c-iewxkbc configurations on the Xilinx VCU118 development board as well as on the Macchiatobin board (Marvell A8K - EIP197b-ieswx), including the crypto extra tests. Note that this patchset applies on top of the earlier submitted "Add support for eip197f_iewc" series. Signed-off-by: Pascal van Leeuwen Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 69 ++++++++++++++++++++++------ drivers/crypto/inside-secure/safexcel.h | 30 +++++++++++- drivers/crypto/inside-secure/safexcel_ring.c | 1 + 3 files changed, 86 insertions(+), 14 deletions(-) (limited to 'drivers/crypto/inside-secure/safexcel.c') diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index c40eb1befc51..9fb4947389e3 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -484,6 +484,14 @@ static int safexcel_hw_setup_cdesc_rings(struct safexcel_crypto_priv *priv) cd_fetch_cnt = ((1 << priv->hwconfig.hwcfsize) / cd_size_rnd) - 1; } + /* + * Since we're using command desc's way larger than formally specified, + * we need to check whether we can fit even 1 for low-end EIP196's! + */ + if (!cd_fetch_cnt) { + dev_err(priv->dev, "Unable to fit even 1 command desc!\n"); + return -ENODEV; + } for (i = 0; i < priv->config.rings; i++) { /* ring base address */ @@ -608,8 +616,8 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv) writel(EIP197_DxE_THR_CTRL_RESET_PE, EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe)); - if (priv->flags & SAFEXCEL_HW_EIP197) - /* Reset HIA input interface arbiter (EIP197 only) */ + if (priv->flags & EIP197_PE_ARB) + /* Reset HIA input interface arbiter (if present) */ writel(EIP197_HIA_RA_PE_CTRL_RESET, EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL(pe)); @@ -756,22 +764,28 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv) /* Clear any HIA interrupt */ writel(GENMASK(30, 20), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK); - if (priv->flags & SAFEXCEL_HW_EIP197) { + if (priv->flags & EIP197_SIMPLE_TRC) { + writel(EIP197_STRC_CONFIG_INIT | + EIP197_STRC_CONFIG_LARGE_REC(EIP197_CS_TRC_REC_WC) | + EIP197_STRC_CONFIG_SMALL_REC(EIP197_CS_TRC_REC_WC), + priv->base + EIP197_STRC_CONFIG); + writel(EIP197_PE_EIP96_TOKEN_CTRL2_CTX_DONE, + EIP197_PE(priv) + EIP197_PE_EIP96_TOKEN_CTRL2(0)); + } else if (priv->flags & SAFEXCEL_HW_EIP197) { ret = eip197_trc_cache_init(priv); if (ret) return ret; + } - priv->flags |= EIP197_TRC_CACHE; - + if (priv->flags & EIP197_ICE) { ret = eip197_load_firmwares(priv); if (ret) return ret; } - safexcel_hw_setup_cdesc_rings(priv); - safexcel_hw_setup_rdesc_rings(priv); - - return 0; + return safexcel_hw_setup_cdesc_rings(priv) ?: + safexcel_hw_setup_rdesc_rings(priv) ?: + 0; } /* Called with ring's lock taken */ @@ -1371,7 +1385,7 @@ static int safexcel_probe_generic(void *pdev, int is_pci_dev) { struct device *dev = priv->dev; - u32 peid, version, mask, val, hiaopt; + u32 peid, version, mask, val, hiaopt, hwopt, peopt; int i, ret, hwctg; priv->context_pool = dmam_pool_create("safexcel-context", dev, @@ -1433,13 +1447,16 @@ static int safexcel_probe_generic(void *pdev, */ version = readl(EIP197_GLOBAL(priv) + EIP197_VERSION); if (((priv->flags & SAFEXCEL_HW_EIP197) && - (EIP197_REG_LO16(version) != EIP197_VERSION_LE)) || + (EIP197_REG_LO16(version) != EIP197_VERSION_LE) && + (EIP197_REG_LO16(version) != EIP196_VERSION_LE)) || ((!(priv->flags & SAFEXCEL_HW_EIP197) && (EIP197_REG_LO16(version) != EIP97_VERSION_LE)))) { /* * We did not find the device that matched our initial probing * (or our initial probing failed) Report appropriate error. */ + dev_err(priv->dev, "Probing for EIP97/EIP19x failed - no such device (read %08x)\n", + version); return -ENODEV; } @@ -1447,6 +1464,14 @@ static int safexcel_probe_generic(void *pdev, hwctg = version >> 28; peid = version & 255; + /* Detect EIP206 processing pipe */ + version = readl(EIP197_PE(priv) + + EIP197_PE_VERSION(0)); + if (EIP197_REG_LO16(version) != EIP206_VERSION_LE) { + dev_err(priv->dev, "EIP%d: EIP206 not detected\n", peid); + return -ENODEV; + } + priv->hwconfig.ppver = EIP197_VERSION_MASK(version); + /* Detect EIP96 packet engine and version */ version = readl(EIP197_PE(priv) + EIP197_PE_EIP96_VERSION(0)); if (EIP197_REG_LO16(version) != EIP96_VERSION_LE) { @@ -1455,10 +1480,13 @@ static int safexcel_probe_generic(void *pdev, } priv->hwconfig.pever = EIP197_VERSION_MASK(version); + hwopt = readl(EIP197_GLOBAL(priv) + EIP197_OPTIONS); hiaopt = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_OPTIONS); if (priv->flags & SAFEXCEL_HW_EIP197) { /* EIP197 */ + peopt = readl(EIP197_PE(priv) + EIP197_PE_OPTIONS(0)); + priv->hwconfig.hwdataw = (hiaopt >> EIP197_HWDATAW_OFFSET) & EIP197_HWDATAW_MASK; priv->hwconfig.hwcfsize = ((hiaopt >> EIP197_CFSIZE_OFFSET) & @@ -1471,6 +1499,15 @@ static int safexcel_probe_generic(void *pdev, EIP197_N_PES_MASK; priv->hwconfig.hwnumrings = (hiaopt >> EIP197_N_RINGS_OFFSET) & EIP197_N_RINGS_MASK; + if (hiaopt & EIP197_HIA_OPT_HAS_PE_ARB) + priv->flags |= EIP197_PE_ARB; + if (EIP206_OPT_ICE_TYPE(peopt) == 1) + priv->flags |= EIP197_ICE; + /* If not a full TRC, then assume simple TRC */ + if (!(hwopt & EIP197_OPT_HAS_TRC)) + priv->flags |= EIP197_SIMPLE_TRC; + /* EIP197 always has SOME form of TRC */ + priv->flags |= EIP197_TRC_CACHE; } else { /* EIP97 */ priv->hwconfig.hwdataw = (hiaopt >> EIP197_HWDATAW_OFFSET) & @@ -1492,18 +1529,24 @@ static int safexcel_probe_generic(void *pdev, break; } priv->hwconfig.hwnumraic = i; + /* Low-end EIP196 may not have any ring AIC's ... */ + if (!priv->hwconfig.hwnumraic) { + dev_err(priv->dev, "No ring interrupt controller present!\n"); + return -ENODEV; + } /* Get supported algorithms from EIP96 transform engine */ priv->hwconfig.algo_flags = readl(EIP197_PE(priv) + EIP197_PE_EIP96_OPTIONS(0)); /* Print single info line describing what we just detected */ - dev_info(priv->dev, "EIP%d:%x(%d,%d,%d,%d)-HIA:%x(%d,%d,%d),PE:%x,alg:%08x\n", + dev_info(priv->dev, "EIP%d:%x(%d,%d,%d,%d)-HIA:%x(%d,%d,%d),PE:%x/%x,alg:%08x\n", peid, priv->hwconfig.hwver, hwctg, priv->hwconfig.hwnumpes, priv->hwconfig.hwnumrings, priv->hwconfig.hwnumraic, priv->hwconfig.hiaver, priv->hwconfig.hwdataw, priv->hwconfig.hwcfsize, priv->hwconfig.hwrfsize, - priv->hwconfig.pever, priv->hwconfig.algo_flags); + priv->hwconfig.ppver, priv->hwconfig.pever, + priv->hwconfig.algo_flags); safexcel_configure(priv); diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 227771363359..30103d5ff6eb 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -17,9 +17,11 @@ #define EIP197_HIA_VERSION_BE 0xca35 #define EIP197_HIA_VERSION_LE 0x35ca #define EIP97_VERSION_LE 0x9e61 +#define EIP196_VERSION_LE 0x3bc4 #define EIP197_VERSION_LE 0x3ac5 #define EIP96_VERSION_LE 0x9f60 #define EIP201_VERSION_LE 0x36c9 +#define EIP206_VERSION_LE 0x31ce #define EIP197_REG_LO16(reg) (reg & 0xffff) #define EIP197_REG_HI16(reg) ((reg >> 16) & 0xffff) #define EIP197_VERSION_MASK(reg) ((reg >> 16) & 0xfff) @@ -27,6 +29,15 @@ ((reg >> 4) & 0xf0) | \ ((reg >> 12) & 0xf)) +/* EIP197 HIA OPTIONS ENCODING */ +#define EIP197_HIA_OPT_HAS_PE_ARB BIT(29) + +/* EIP206 OPTIONS ENCODING */ +#define EIP206_OPT_ICE_TYPE(n) ((n>>8)&3) + +/* EIP197 OPTIONS ENCODING */ +#define EIP197_OPT_HAS_TRC BIT(31) + /* Static configuration */ #define EIP197_DEFAULT_RING_SIZE 400 #define EIP197_MAX_TOKENS 19 @@ -160,12 +171,16 @@ #define EIP197_PE_EIP96_FUNCTION_EN(n) (0x1004 + (0x2000 * (n))) #define EIP197_PE_EIP96_CONTEXT_CTRL(n) (0x1008 + (0x2000 * (n))) #define EIP197_PE_EIP96_CONTEXT_STAT(n) (0x100c + (0x2000 * (n))) +#define EIP197_PE_EIP96_TOKEN_CTRL2(n) (0x102c + (0x2000 * (n))) #define EIP197_PE_EIP96_FUNCTION2_EN(n) (0x1030 + (0x2000 * (n))) #define EIP197_PE_EIP96_OPTIONS(n) (0x13f8 + (0x2000 * (n))) #define EIP197_PE_EIP96_VERSION(n) (0x13fc + (0x2000 * (n))) #define EIP197_PE_OUT_DBUF_THRES(n) (0x1c00 + (0x2000 * (n))) #define EIP197_PE_OUT_TBUF_THRES(n) (0x1d00 + (0x2000 * (n))) +#define EIP197_PE_OPTIONS(n) (0x1ff8 + (0x2000 * (n))) +#define EIP197_PE_VERSION(n) (0x1ffc + (0x2000 * (n))) #define EIP197_MST_CTRL 0xfff4 +#define EIP197_OPTIONS 0xfff8 #define EIP197_VERSION 0xfffc /* EIP197-specific registers, no indirection */ @@ -181,6 +196,7 @@ #define EIP197_TRC_ECCADMINSTAT 0xf0838 #define EIP197_TRC_ECCDATASTAT 0xf083c #define EIP197_TRC_ECCDATA 0xf0840 +#define EIP197_STRC_CONFIG 0xf43f0 #define EIP197_FLUE_CACHEBASE_LO(n) (0xf6000 + (32 * (n))) #define EIP197_FLUE_CACHEBASE_HI(n) (0xf6004 + (32 * (n))) #define EIP197_FLUE_CONFIG(n) (0xf6010 + (32 * (n))) @@ -331,6 +347,14 @@ #define EIP197_ADDRESS_MODE BIT(8) #define EIP197_CONTROL_MODE BIT(9) +/* EIP197_PE_EIP96_TOKEN_CTRL2 */ +#define EIP197_PE_EIP96_TOKEN_CTRL2_CTX_DONE BIT(3) + +/* EIP197_STRC_CONFIG */ +#define EIP197_STRC_CONFIG_INIT BIT(31) +#define EIP197_STRC_CONFIG_LARGE_REC(s) (s<<8) +#define EIP197_STRC_CONFIG_SMALL_REC(s) (s<<0) + /* EIP197_FLUE_CONFIG */ #define EIP197_FLUE_CONFIG_MAGIC 0xc7000004 @@ -472,7 +496,7 @@ struct result_data_desc { u16 application_id; u16 rsvd1; - u32 rsvd2; + u32 rsvd2[5]; } __packed; @@ -731,12 +755,16 @@ struct safexcel_register_offsets { enum safexcel_flags { EIP197_TRC_CACHE = BIT(0), SAFEXCEL_HW_EIP197 = BIT(1), + EIP197_PE_ARB = BIT(2), + EIP197_ICE = BIT(3), + EIP197_SIMPLE_TRC = BIT(4), }; struct safexcel_hwconfig { enum safexcel_eip_algorithms algo_flags; int hwver; int hiaver; + int ppver; int pever; int hwdataw; int hwcfsize; diff --git a/drivers/crypto/inside-secure/safexcel_ring.c b/drivers/crypto/inside-secure/safexcel_ring.c index 5323e9179b4f..9237ba745c2f 100644 --- a/drivers/crypto/inside-secure/safexcel_ring.c +++ b/drivers/crypto/inside-secure/safexcel_ring.c @@ -180,6 +180,7 @@ struct safexcel_result_desc *safexcel_add_rdesc(struct safexcel_crypto_priv *pri rdesc->first_seg = first; rdesc->last_seg = last; + rdesc->result_size = EIP197_RD64_RESULT_SIZE; rdesc->particle_size = len; rdesc->data_lo = lower_32_bits(data); rdesc->data_hi = upper_32_bits(data); -- cgit From 74e6bd472b6d9e80ec9972989d8991736fe46c51 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Mon, 30 Sep 2019 14:14:33 +0200 Subject: crypto: inside-secure - Fix a maybe-uninitialized warning A previous fixup avoided an unused variable warning but replaced it with a slightly scarier warning: drivers/crypto/inside-secure/safexcel.c:1100:6: error: variable 'irq' is used uninitialized whenever 'if' condition is false [-Werror,-Wsometimes-uninitialized] This is harmless as it is impossible to get into this case, but the compiler has no way of knowing that. Add an explicit error handling case to make it obvious to both compilers and humans reading the source. Fixes: 212ef6f29e5b ("crypto: inside-secure - Fix unused variable warning when CONFIG_PCI=n") Signed-off-by: Arnd Bergmann Acked-by: Pascal van Leeuwen Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'drivers/crypto/inside-secure/safexcel.c') diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 9fb4947389e3..cd7dcd8986f5 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1162,6 +1162,8 @@ static int safexcel_request_ring_irq(void *pdev, int irqid, irq_name, irq); return irq; } + } else { + return -ENXIO; } ret = devm_request_threaded_irq(dev, irq, handler, -- cgit From 977da0738f3ba3569b883ed6209c300bfcb695d4 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Mon, 30 Sep 2019 14:14:35 +0200 Subject: crypto: inside-secure - Remove #ifdef checks When both PCI and OF are disabled, no drivers are registered, and we get some unused-function warnings: drivers/crypto/inside-secure/safexcel.c:1221:13: error: unused function 'safexcel_unregister_algorithms' [-Werror,-Wunused-function] static void safexcel_unregister_algorithms(struct safexcel_crypto_priv *priv) drivers/crypto/inside-secure/safexcel.c:1307:12: error: unused function 'safexcel_probe_generic' [-Werror,-Wunused-function] static int safexcel_probe_generic(void *pdev, drivers/crypto/inside-secure/safexcel.c:1531:13: error: unused function 'safexcel_hw_reset_rings' [-Werror,-Wunused-function] static void safexcel_hw_reset_rings(struct safexcel_crypto_priv *priv) It's better to make the compiler see what is going on and remove such ifdef checks completely. In case of PCI, this is trivial since pci_register_driver() is defined to an empty function that makes the compiler subsequently drop all unused code silently. The global pcireg_rc/ofreg_rc variables are not actually needed here since the driver registration does not fail in ways that would make it helpful. For CONFIG_OF, an IS_ENABLED() check is still required, since platform drivers can exist both with and without it. A little change to linux/pci.h is needed to ensure that pcim_enable_device() is visible to the driver. Moving the declaration outside of ifdef would be sufficient here, but for consistency with the rest of the file, adding an inline helper is probably best. Fixes: 212ef6f29e5b ("crypto: inside-secure - Fix unused variable warning when CONFIG_PCI=n") Signed-off-by: Arnd Bergmann Acked-by: Bjorn Helgaas # pci.h Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 49 ++++++++------------------------- include/linux/pci.h | 1 + 2 files changed, 13 insertions(+), 37 deletions(-) (limited to 'drivers/crypto/inside-secure/safexcel.c') diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index cd7dcd8986f5..5342ba61b02e 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1671,7 +1671,6 @@ static void safexcel_hw_reset_rings(struct safexcel_crypto_priv *priv) } } -#if IS_ENABLED(CONFIG_OF) /* for Device Tree platform driver */ static int safexcel_probe(struct platform_device *pdev) @@ -1790,9 +1789,7 @@ static struct platform_driver crypto_safexcel = { .of_match_table = safexcel_of_match_table, }, }; -#endif -#if IS_ENABLED(CONFIG_PCI) /* PCIE devices - i.e. Inside Secure development boards */ static int safexcel_pci_probe(struct pci_dev *pdev, @@ -1913,54 +1910,32 @@ static struct pci_driver safexcel_pci_driver = { .probe = safexcel_pci_probe, .remove = safexcel_pci_remove, }; -#endif - -/* Unfortunately, we have to resort to global variables here */ -#if IS_ENABLED(CONFIG_PCI) -int pcireg_rc = -EINVAL; /* Default safe value */ -#endif -#if IS_ENABLED(CONFIG_OF) -int ofreg_rc = -EINVAL; /* Default safe value */ -#endif static int __init safexcel_init(void) { -#if IS_ENABLED(CONFIG_PCI) + int ret; + /* Register PCI driver */ - pcireg_rc = pci_register_driver(&safexcel_pci_driver); -#endif + ret = pci_register_driver(&safexcel_pci_driver); -#if IS_ENABLED(CONFIG_OF) /* Register platform driver */ - ofreg_rc = platform_driver_register(&crypto_safexcel); - #if IS_ENABLED(CONFIG_PCI) - /* Return success if either PCI or OF registered OK */ - return pcireg_rc ? ofreg_rc : 0; - #else - return ofreg_rc; - #endif -#else - #if IS_ENABLED(CONFIG_PCI) - return pcireg_rc; - #else - return -EINVAL; - #endif -#endif + if (IS_ENABLED(CONFIG_OF) && !ret) { + ret = platform_driver_register(&crypto_safexcel); + if (ret) + pci_unregister_driver(&safexcel_pci_driver); + } + + return ret; } static void __exit safexcel_exit(void) { -#if IS_ENABLED(CONFIG_OF) /* Unregister platform driver */ - if (!ofreg_rc) + if (IS_ENABLED(CONFIG_OF)) platform_driver_unregister(&crypto_safexcel); -#endif -#if IS_ENABLED(CONFIG_PCI) /* Unregister PCI driver if successfully registered before */ - if (!pcireg_rc) - pci_unregister_driver(&safexcel_pci_driver); -#endif + pci_unregister_driver(&safexcel_pci_driver); } module_init(safexcel_init); diff --git a/include/linux/pci.h b/include/linux/pci.h index f9088c89a534..1a6cf19eac2d 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -1686,6 +1686,7 @@ static inline struct pci_dev *pci_get_class(unsigned int class, static inline void pci_set_master(struct pci_dev *dev) { } static inline int pci_enable_device(struct pci_dev *dev) { return -EIO; } static inline void pci_disable_device(struct pci_dev *dev) { } +static inline int pcim_enable_device(struct pci_dev *pdev) { return -EIO; } static inline int pci_assign_resource(struct pci_dev *dev, int i) { return -EBUSY; } static inline int __pci_register_driver(struct pci_driver *drv, -- cgit From e9fc36719b529b89614e0238cf32d32732593551 Mon Sep 17 00:00:00 2001 From: Pascal van Leeuwen Date: Thu, 17 Oct 2019 17:49:09 +0200 Subject: crypto: inside-secure - Made locally used safexcel_pci_remove() static safexcel_pci_remove() is only used locally in the module and not exported, so added a static function specifier. This fixes a sparse issue reported by Ben Dooks. Fixes: 625f269a5a7a ("crypto: inside-secure - add support for...") Signed-off-by: Pascal van Leeuwen Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/crypto/inside-secure/safexcel.c') diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 5342ba61b02e..e5f2bd703c2f 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1880,7 +1880,7 @@ static int safexcel_pci_probe(struct pci_dev *pdev, return rc; } -void safexcel_pci_remove(struct pci_dev *pdev) +static void safexcel_pci_remove(struct pci_dev *pdev) { struct safexcel_crypto_priv *priv = pci_get_drvdata(pdev); int i; -- cgit From 13a1bb93f7b1c90099c557696951884ea7dcda34 Mon Sep 17 00:00:00 2001 From: Pascal van Leeuwen Date: Tue, 22 Oct 2019 11:01:43 +0200 Subject: crypto: inside-secure - Fixed warnings on inconsistent byte order handling This fixes a bunch of endianness related sparse warnings reported by the kbuild test robot as well as Ben Dooks. Credits for the fix to safexcel.c go to Ben Dooks. Reported-by: kbuild test robot Reported-by: Ben Dooks Signed-off-by: Pascal van Leeuwen Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 5 +- drivers/crypto/inside-secure/safexcel.h | 4 +- drivers/crypto/inside-secure/safexcel_cipher.c | 88 ++++++++++++-------------- drivers/crypto/inside-secure/safexcel_hash.c | 31 +++++---- 4 files changed, 61 insertions(+), 67 deletions(-) (limited to 'drivers/crypto/inside-secure/safexcel.c') diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index e5f2bd703c2f..fdd99ef07432 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -311,13 +311,14 @@ static void eip197_init_firmware(struct safexcel_crypto_priv *priv) static int eip197_write_firmware(struct safexcel_crypto_priv *priv, const struct firmware *fw) { - const u32 *data = (const u32 *)fw->data; + const __be32 *data = (const __be32 *)fw->data; int i; /* Write the firmware */ for (i = 0; i < fw->size / sizeof(u32); i++) writel(be32_to_cpu(data[i]), - priv->base + EIP197_CLASSIFICATION_RAMS + i * sizeof(u32)); + priv->base + EIP197_CLASSIFICATION_RAMS + + i * sizeof(__be32)); /* Exclude final 2 NOPs from size */ return i - EIP197_FW_TERMINAL_NOPS; diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 30103d5ff6eb..ba03e4daf64c 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -360,8 +360,8 @@ /* Context Control */ struct safexcel_context_record { - u32 control0; - u32 control1; + __le32 control0; + __le32 control1; __le32 data[40]; } __packed; diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index 407ebcd8d71f..98f9fc60c268 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -57,8 +57,8 @@ struct safexcel_cipher_ctx { /* All the below is AEAD specific */ u32 hash_alg; u32 state_sz; - u32 ipad[SHA512_DIGEST_SIZE / sizeof(u32)]; - u32 opad[SHA512_DIGEST_SIZE / sizeof(u32)]; + __be32 ipad[SHA512_DIGEST_SIZE / sizeof(u32)]; + __be32 opad[SHA512_DIGEST_SIZE / sizeof(u32)]; struct crypto_cipher *hkaes; struct crypto_aead *fback; @@ -92,7 +92,8 @@ static void safexcel_cipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv, cdesc->control_data.token[3] = 0; } else { /* 32 bit counter, start at 1 (big endian!) */ - cdesc->control_data.token[3] = cpu_to_be32(1); + cdesc->control_data.token[3] = + (__force u32)cpu_to_be32(1); } return; @@ -108,7 +109,8 @@ static void safexcel_cipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv, cdesc->control_data.token[3] = 0; } else { /* 32 bit counter, start at 1 (big endian!) */ - cdesc->control_data.token[3] = cpu_to_be32(1); + *(__be32 *)&cdesc->control_data.token[3] = + cpu_to_be32(1); } return; @@ -267,7 +269,7 @@ static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv, if (ctx->xcm != EIP197_XCM_MODE_GCM) { u8 *final_iv = (u8 *)cdesc->control_data.token; u8 *cbcmaciv = (u8 *)&token[1]; - u32 *aadlen = (u32 *)&token[5]; + __le32 *aadlen = (__le32 *)&token[5]; /* Construct IV block B0 for the CBC-MAC */ token[0].opcode = EIP197_TOKEN_OPCODE_INSERT; @@ -286,7 +288,8 @@ static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv, cbcmaciv[15] = cryptlen & 255; if (assoclen) { - *aadlen = cpu_to_le32(cpu_to_be16(assoclen)); + *aadlen = cpu_to_le32((assoclen >> 8) | + ((assoclen & 0xff) << 8)); assoclen += 2; } @@ -333,7 +336,7 @@ static int safexcel_skcipher_aes_setkey(struct crypto_skcipher *ctfm, if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { for (i = 0; i < len / sizeof(u32); i++) { - if (ctx->key[i] != cpu_to_le32(aes.key_enc[i])) { + if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { ctx->base.needs_inv = true; break; } @@ -358,7 +361,7 @@ static int safexcel_aead_setkey(struct crypto_aead *ctfm, const u8 *key, struct safexcel_crypto_priv *priv = ctx->priv; struct crypto_authenc_keys keys; struct crypto_aes_ctx aes; - int err = -EINVAL; + int err = -EINVAL, i; if (unlikely(crypto_authenc_extractkeys(&keys, key, len))) goto badkey; @@ -400,9 +403,14 @@ static int safexcel_aead_setkey(struct crypto_aead *ctfm, const u8 *key, goto badkey; } - if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma && - memcmp(ctx->key, keys.enckey, keys.enckeylen)) - ctx->base.needs_inv = true; + if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { + for (i = 0; i < keys.enckeylen / sizeof(u32); i++) { + if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { + ctx->base.needs_inv = true; + break; + } + } + } /* Auth key */ switch (ctx->hash_alg) { @@ -450,7 +458,8 @@ static int safexcel_aead_setkey(struct crypto_aead *ctfm, const u8 *key, ctx->base.needs_inv = true; /* Now copy the keys into the context */ - memcpy(ctx->key, keys.enckey, keys.enckeylen); + for (i = 0; i < keys.enckeylen / sizeof(u32); i++) + ctx->key[i] = cpu_to_le32(aes.key_enc[i]); ctx->key_len = keys.enckeylen; memcpy(ctx->ipad, &istate.state, ctx->state_sz); @@ -1378,7 +1387,7 @@ static int safexcel_skcipher_aesctr_setkey(struct crypto_skcipher *ctfm, if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { for (i = 0; i < keylen / sizeof(u32); i++) { - if (ctx->key[i] != cpu_to_le32(aes.key_enc[i])) { + if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { ctx->base.needs_inv = true; break; } @@ -1534,13 +1543,11 @@ static int safexcel_des3_ede_setkey(struct crypto_skcipher *ctfm, return err; /* if context exits and key changed, need to invalidate it */ - if (ctx->base.ctxr_dma) { + if (ctx->base.ctxr_dma) if (memcmp(ctx->key, key, len)) ctx->base.needs_inv = true; - } memcpy(ctx->key, key, len); - ctx->key_len = len; return 0; @@ -2361,7 +2368,7 @@ static int safexcel_skcipher_aesxts_setkey(struct crypto_skcipher *ctfm, if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { for (i = 0; i < keylen / sizeof(u32); i++) { - if (ctx->key[i] != cpu_to_le32(aes.key_enc[i])) { + if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { ctx->base.needs_inv = true; break; } @@ -2380,8 +2387,8 @@ static int safexcel_skcipher_aesxts_setkey(struct crypto_skcipher *ctfm, if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { for (i = 0; i < keylen / sizeof(u32); i++) { - if (ctx->key[i + keylen / sizeof(u32)] != - cpu_to_le32(aes.key_enc[i])) { + if (le32_to_cpu(ctx->key[i + keylen / sizeof(u32)]) != + aes.key_enc[i]) { ctx->base.needs_inv = true; break; } @@ -2471,7 +2478,7 @@ static int safexcel_aead_gcm_setkey(struct crypto_aead *ctfm, const u8 *key, if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { for (i = 0; i < len / sizeof(u32); i++) { - if (ctx->key[i] != cpu_to_le32(aes.key_enc[i])) { + if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { ctx->base.needs_inv = true; break; } @@ -2498,7 +2505,7 @@ static int safexcel_aead_gcm_setkey(struct crypto_aead *ctfm, const u8 *key, if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) { - if (ctx->ipad[i] != cpu_to_be32(hashkey[i])) { + if (be32_to_cpu(ctx->ipad[i]) != hashkey[i]) { ctx->base.needs_inv = true; break; } @@ -2588,7 +2595,7 @@ static int safexcel_aead_ccm_setkey(struct crypto_aead *ctfm, const u8 *key, if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { for (i = 0; i < len / sizeof(u32); i++) { - if (ctx->key[i] != cpu_to_le32(aes.key_enc[i])) { + if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { ctx->base.needs_inv = true; break; } @@ -2697,20 +2704,12 @@ static void safexcel_chacha20_setkey(struct safexcel_cipher_ctx *ctx, const u8 *key) { struct safexcel_crypto_priv *priv = ctx->priv; - int i; - if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { - for (i = 0; i < CHACHA_KEY_SIZE / sizeof(u32); i++) { - if (ctx->key[i] != - get_unaligned_le32(key + i * sizeof(u32))) { - ctx->base.needs_inv = true; - break; - } - } - } + if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) + if (memcmp(ctx->key, key, CHACHA_KEY_SIZE)) + ctx->base.needs_inv = true; - for (i = 0; i < CHACHA_KEY_SIZE / sizeof(u32); i++) - ctx->key[i] = get_unaligned_le32(key + i * sizeof(u32)); + memcpy(ctx->key, key, CHACHA_KEY_SIZE); ctx->key_len = CHACHA_KEY_SIZE; } @@ -2801,7 +2800,7 @@ static int safexcel_aead_chachapoly_crypt(struct aead_request *req, struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); struct aead_request *subreq = aead_request_ctx(req); u32 key[CHACHA_KEY_SIZE / sizeof(u32) + 1]; - int i, ret = 0; + int ret = 0; /* * Instead of wasting time detecting umpteen silly corner cases, @@ -2815,8 +2814,7 @@ static int safexcel_aead_chachapoly_crypt(struct aead_request *req, } /* HW cannot do full (AAD+payload) zero length, use fallback */ - for (i = 0; i < CHACHA_KEY_SIZE / sizeof(u32); i++) - key[i] = cpu_to_le32(ctx->key[i]); + memcpy(key, ctx->key, CHACHA_KEY_SIZE); if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) { /* ESP variant has nonce appended to the key */ key[CHACHA_KEY_SIZE / sizeof(u32)] = ctx->nonce; @@ -2971,25 +2969,17 @@ static int safexcel_skcipher_sm4_setkey(struct crypto_skcipher *ctfm, struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); struct safexcel_crypto_priv *priv = ctx->priv; - int i; if (len != SM4_KEY_SIZE) { crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN); return -EINVAL; } - if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { - for (i = 0; i < SM4_KEY_SIZE / sizeof(u32); i++) { - if (ctx->key[i] != - get_unaligned_le32(key + i * sizeof(u32))) { - ctx->base.needs_inv = true; - break; - } - } - } + if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) + if (memcmp(ctx->key, key, SM4_KEY_SIZE)) + ctx->base.needs_inv = true; - for (i = 0; i < SM4_KEY_SIZE / sizeof(u32); i++) - ctx->key[i] = get_unaligned_le32(key + i * sizeof(u32)); + memcpy(ctx->key, key, SM4_KEY_SIZE); ctx->key_len = SM4_KEY_SIZE; return 0; diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index b60f9fbd4017..2134daef24f6 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -29,8 +29,8 @@ struct safexcel_ahash_ctx { bool fb_init_done; bool fb_do_setkey; - u32 ipad[SHA3_512_BLOCK_SIZE / sizeof(u32)]; - u32 opad[SHA3_512_BLOCK_SIZE / sizeof(u32)]; + __le32 ipad[SHA3_512_BLOCK_SIZE / sizeof(__le32)]; + __le32 opad[SHA3_512_BLOCK_SIZE / sizeof(__le32)]; struct crypto_cipher *kaes; struct crypto_ahash *fback; @@ -56,7 +56,8 @@ struct safexcel_ahash_req { u8 state_sz; /* expected state size, only set once */ u8 block_sz; /* block size, only set once */ u8 digest_sz; /* output digest size, only set once */ - u32 state[SHA3_512_BLOCK_SIZE / sizeof(u32)] __aligned(sizeof(u32)); + __le32 state[SHA3_512_BLOCK_SIZE / + sizeof(__le32)] __aligned(sizeof(__le32)); u64 len; u64 processed; @@ -287,7 +288,7 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, if (unlikely(sreq->digest == CONTEXT_CONTROL_DIGEST_XCM && ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_CRC32)) { /* Undo final XOR with 0xffffffff ...*/ - *(u32 *)areq->result = ~sreq->state[0]; + *(__le32 *)areq->result = ~sreq->state[0]; } else { memcpy(areq->result, sreq->state, crypto_ahash_digestsize(ahash)); @@ -372,9 +373,9 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring, req->cache[cache_len + skip] = 0x80; // HW will use K2 iso K3 - compensate! for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) - ((u32 *)req->cache)[i] ^= - cpu_to_be32(ctx->ipad[i]) ^ - cpu_to_be32(ctx->ipad[i + 4]); + ((__be32 *)req->cache)[i] ^= + cpu_to_be32(le32_to_cpu( + ctx->ipad[i] ^ ctx->ipad[i + 4])); } cache_len = AES_BLOCK_SIZE; queued = queued + extra; @@ -807,8 +808,8 @@ static int safexcel_ahash_final(struct ahash_request *areq) int i; for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) - ((u32 *)areq->result)[i] = - cpu_to_be32(ctx->ipad[i + 4]); // K3 + ((__be32 *)areq->result)[i] = + cpu_to_be32(le32_to_cpu(ctx->ipad[i + 4]));//K3 areq->result[0] ^= 0x80; // 10- padding crypto_cipher_encrypt_one(ctx->kaes, areq->result, areq->result); return 0; @@ -1891,7 +1892,7 @@ static int safexcel_crc32_init(struct ahash_request *areq) memset(req, 0, sizeof(*req)); /* Start from loaded key */ - req->state[0] = cpu_to_le32(~ctx->ipad[0]); + req->state[0] = (__force __le32)le32_to_cpu(~ctx->ipad[0]); /* Set processed to non-zero to enable invalidation detection */ req->len = sizeof(u32); req->processed = sizeof(u32); @@ -1993,7 +1994,7 @@ static int safexcel_cbcmac_setkey(struct crypto_ahash *tfm, const u8 *key, memset(ctx->ipad, 0, 2 * AES_BLOCK_SIZE); for (i = 0; i < len / sizeof(u32); i++) - ctx->ipad[i + 8] = cpu_to_be32(aes.key_enc[i]); + ctx->ipad[i + 8] = (__force __le32)cpu_to_be32(aes.key_enc[i]); if (len == AES_KEYSIZE_192) { ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192; @@ -2078,7 +2079,8 @@ static int safexcel_xcbcmac_setkey(struct crypto_ahash *tfm, const u8 *key, crypto_cipher_encrypt_one(ctx->kaes, (u8 *)key_tmp + AES_BLOCK_SIZE, "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3"); for (i = 0; i < 3 * AES_BLOCK_SIZE / sizeof(u32); i++) - ctx->ipad[i] = cpu_to_be32(key_tmp[i]); + ctx->ipad[i] = + cpu_to_le32((__force u32)cpu_to_be32(key_tmp[i])); crypto_cipher_clear_flags(ctx->kaes, CRYPTO_TFM_REQ_MASK); crypto_cipher_set_flags(ctx->kaes, crypto_ahash_get_flags(tfm) & @@ -2164,7 +2166,8 @@ static int safexcel_cmac_setkey(struct crypto_ahash *tfm, const u8 *key, } for (i = 0; i < len / sizeof(u32); i++) - ctx->ipad[i + 8] = cpu_to_be32(aes.key_enc[i]); + ctx->ipad[i + 8] = + cpu_to_le32((__force u32)cpu_to_be32(aes.key_enc[i])); /* precompute the CMAC key material */ crypto_cipher_clear_flags(ctx->kaes, CRYPTO_TFM_REQ_MASK); @@ -2197,7 +2200,7 @@ static int safexcel_cmac_setkey(struct crypto_ahash *tfm, const u8 *key, /* end of code borrowed from crypto/cmac.c */ for (i = 0; i < 2 * AES_BLOCK_SIZE / sizeof(u32); i++) - ctx->ipad[i] = cpu_to_be32(((u32 *)consts)[i]); + ctx->ipad[i] = (__force __le32)cpu_to_be32(((u32 *)consts)[i]); if (len == AES_KEYSIZE_192) { ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192; -- cgit From df40c4e6cdaaacbf26d1206adb2419733de3bbf1 Mon Sep 17 00:00:00 2001 From: Chuhong Yuan Date: Fri, 1 Nov 2019 22:37:15 +0800 Subject: crypto: inside-secure - Add missed clk_disable_unprepare safexcel_remove misses disabling priv->reg_clk like what is done when probe fails. Add the missed call to fix it. Signed-off-by: Chuhong Yuan Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers/crypto/inside-secure/safexcel.c') diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index fdd99ef07432..64894d8b442a 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1749,6 +1749,7 @@ static int safexcel_remove(struct platform_device *pdev) safexcel_unregister_algorithms(priv); safexcel_hw_reset_rings(priv); + clk_disable_unprepare(priv->reg_clk); clk_disable_unprepare(priv->clk); for (i = 0; i < priv->config.rings; i++) -- cgit