From ccd3129aca286c41616afe357e3494c5b43350a0 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Thu, 5 Oct 2023 09:14:12 -0700 Subject: nvmet-fc: Annotate struct nvmet_fc_tgt_queue with __counted_by Prepare for the coming implementation by GCC and Clang of the __counted_by attribute. Flexible array members annotated with __counted_by can have their accesses bounds-checked at run-time via CONFIG_UBSAN_BOUNDS (for array indexing) and CONFIG_FORTIFY_SOURCE (for strcpy/memcpy-family functions). As found with Coccinelle[1], add __counted_by for struct nvmet_fc_tgt_queue. Additionally, since the element count member must be set before accessing the annotated flexible array member, move its initialization earlier. Cc: James Smart Cc: Christoph Hellwig Cc: Sagi Grimberg Cc: linux-nvme@lists.infradead.org Reviewed-by: Gustavo A. R. Silva Reviewed-by: Chaitanya Kulkarni Link: https://github.com/kees/kernel-tools/blob/trunk/coccinelle/examples/counted_by.cocci [1] Signed-off-by: Kees Cook Signed-off-by: Keith Busch --- drivers/nvme/target/fc.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/nvme/target/fc.c b/drivers/nvme/target/fc.c index 1ab6601fdd5c..bd59990b5250 100644 --- a/drivers/nvme/target/fc.c +++ b/drivers/nvme/target/fc.c @@ -146,7 +146,8 @@ struct nvmet_fc_tgt_queue { struct workqueue_struct *work_q; struct kref ref; struct rcu_head rcu; - struct nvmet_fc_fcp_iod fod[]; /* array of fcp_iods */ + /* array of fcp_iods */ + struct nvmet_fc_fcp_iod fod[] __counted_by(sqsize); } __aligned(sizeof(unsigned long long)); struct nvmet_fc_hostport { -- cgit From 9d77eb52778499a97cab662aa96de4e2e4fa72d3 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Thu, 24 Aug 2023 16:39:08 +0200 Subject: nvme-keyring: register '.nvme' keyring Register a '.nvme' keyring to hold keys for TLS and DH-HMAC-CHAP and add a new config option NVME_KEYRING. We need a separate keyring for NVMe as the configuration is done via individual commands (eg for configfs), and the usual per-session or per-process keyrings can't be used. Signed-off-by: Hannes Reinecke Reviewed-by: Sagi Grimberg Signed-off-by: Keith Busch --- drivers/nvme/common/Kconfig | 4 ++++ drivers/nvme/common/Makefile | 3 ++- drivers/nvme/common/keyring.c | 40 ++++++++++++++++++++++++++++++++++++++++ drivers/nvme/host/core.c | 10 ++++++++-- include/linux/nvme-keyring.h | 28 ++++++++++++++++++++++++++++ 5 files changed, 82 insertions(+), 3 deletions(-) create mode 100644 drivers/nvme/common/keyring.c create mode 100644 include/linux/nvme-keyring.h diff --git a/drivers/nvme/common/Kconfig b/drivers/nvme/common/Kconfig index 4514f44362dd..641b27adb047 100644 --- a/drivers/nvme/common/Kconfig +++ b/drivers/nvme/common/Kconfig @@ -2,3 +2,7 @@ config NVME_COMMON tristate + +config NVME_KEYRING + bool + select KEYS diff --git a/drivers/nvme/common/Makefile b/drivers/nvme/common/Makefile index 720c625b8a52..0cbd0b0b8d49 100644 --- a/drivers/nvme/common/Makefile +++ b/drivers/nvme/common/Makefile @@ -4,4 +4,5 @@ ccflags-y += -I$(src) obj-$(CONFIG_NVME_COMMON) += nvme-common.o -nvme-common-y += auth.o +nvme-common-$(CONFIG_NVME_AUTH) += auth.o +nvme-common-$(CONFIG_NVME_KEYRING) += keyring.o diff --git a/drivers/nvme/common/keyring.c b/drivers/nvme/common/keyring.c new file mode 100644 index 000000000000..5cf64b278119 --- /dev/null +++ b/drivers/nvme/common/keyring.c @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2023 Hannes Reinecke, SUSE Labs + */ + +#include +#include +#include +#include +#include + +static struct key *nvme_keyring; + +key_serial_t nvme_keyring_id(void) +{ + return nvme_keyring->serial; +} +EXPORT_SYMBOL_GPL(nvme_keyring_id); + +int nvme_keyring_init(void) +{ + nvme_keyring = keyring_alloc(".nvme", + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, + current_cred(), + (KEY_POS_ALL & ~KEY_POS_SETATTR) | + (KEY_USR_ALL & ~KEY_USR_SETATTR), + KEY_ALLOC_NOT_IN_QUOTA, NULL, NULL); + if (IS_ERR(nvme_keyring)) + return PTR_ERR(nvme_keyring); + + return 0; +} +EXPORT_SYMBOL_GPL(nvme_keyring_init); + +void nvme_keyring_exit(void) +{ + key_revoke(nvme_keyring); + key_put(nvme_keyring); +} +EXPORT_SYMBOL_GPL(nvme_keyring_exit); diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 21783aa2ee8e..a49b65d34cda 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -25,6 +25,7 @@ #include "nvme.h" #include "fabrics.h" #include +#include #define CREATE_TRACE_POINTS #include "trace.h" @@ -4723,12 +4724,16 @@ static int __init nvme_core_init(void) result = PTR_ERR(nvme_ns_chr_class); goto unregister_generic_ns; } - - result = nvme_init_auth(); + result = nvme_keyring_init(); if (result) goto destroy_ns_chr; + result = nvme_init_auth(); + if (result) + goto keyring_exit; return 0; +keyring_exit: + nvme_keyring_exit(); destroy_ns_chr: class_destroy(nvme_ns_chr_class); unregister_generic_ns: @@ -4752,6 +4757,7 @@ out: static void __exit nvme_core_exit(void) { nvme_exit_auth(); + nvme_keyring_exit(); class_destroy(nvme_ns_chr_class); class_destroy(nvme_subsys_class); class_destroy(nvme_class); diff --git a/include/linux/nvme-keyring.h b/include/linux/nvme-keyring.h new file mode 100644 index 000000000000..32bd264a71e6 --- /dev/null +++ b/include/linux/nvme-keyring.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2023 Hannes Reinecke, SUSE Labs + */ + +#ifndef _NVME_KEYRING_H +#define _NVME_KEYRING_H + +#ifdef CONFIG_NVME_KEYRING + +key_serial_t nvme_keyring_id(void); +int nvme_keyring_init(void); +void nvme_keyring_exit(void); + +#else + +static inline key_serial_t nvme_keyring_id(void) +{ + return 0; +} +static inline int nvme_keyring_init(void) +{ + return 0; +} +static inline void nvme_keyring_exit(void) {} + +#endif /* !CONFIG_NVME_KEYRING */ +#endif /* _NVME_KEYRING_H */ -- cgit From 3bac969b16b7bc304ba56d030847920fc7073a91 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Thu, 24 Aug 2023 16:39:09 +0200 Subject: nvme-keyring: define a 'psk' keytype Define a 'psk' keytype to hold the NVMe TLS PSKs. Signed-off-by: Hannes Reinecke Reviewed-by: Sagi Grimberg Signed-off-by: Keith Busch --- drivers/nvme/common/keyring.c | 94 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/drivers/nvme/common/keyring.c b/drivers/nvme/common/keyring.c index 5cf64b278119..494dd365052e 100644 --- a/drivers/nvme/common/keyring.c +++ b/drivers/nvme/common/keyring.c @@ -8,6 +8,8 @@ #include #include #include +#include +#include static struct key *nvme_keyring; @@ -17,8 +19,94 @@ key_serial_t nvme_keyring_id(void) } EXPORT_SYMBOL_GPL(nvme_keyring_id); +static void nvme_tls_psk_describe(const struct key *key, struct seq_file *m) +{ + seq_puts(m, key->description); + seq_printf(m, ": %u", key->datalen); +} + +static bool nvme_tls_psk_match(const struct key *key, + const struct key_match_data *match_data) +{ + const char *match_id; + size_t match_len; + + if (!key->description) { + pr_debug("%s: no key description\n", __func__); + return false; + } + match_len = strlen(key->description); + pr_debug("%s: id %s len %zd\n", __func__, key->description, match_len); + + if (!match_data->raw_data) { + pr_debug("%s: no match data\n", __func__); + return false; + } + match_id = match_data->raw_data; + pr_debug("%s: match '%s' '%s' len %zd\n", + __func__, match_id, key->description, match_len); + return !memcmp(key->description, match_id, match_len); +} + +static int nvme_tls_psk_match_preparse(struct key_match_data *match_data) +{ + match_data->lookup_type = KEYRING_SEARCH_LOOKUP_ITERATE; + match_data->cmp = nvme_tls_psk_match; + return 0; +} + +static struct key_type nvme_tls_psk_key_type = { + .name = "psk", + .flags = KEY_TYPE_NET_DOMAIN, + .preparse = user_preparse, + .free_preparse = user_free_preparse, + .match_preparse = nvme_tls_psk_match_preparse, + .instantiate = generic_key_instantiate, + .revoke = user_revoke, + .destroy = user_destroy, + .describe = nvme_tls_psk_describe, + .read = user_read, +}; + +static struct key *nvme_tls_psk_lookup(struct key *keyring, + const char *hostnqn, const char *subnqn, + int hmac, bool generated) +{ + char *identity; + size_t identity_len = (NVMF_NQN_SIZE) * 2 + 11; + key_ref_t keyref; + key_serial_t keyring_id; + + identity = kzalloc(identity_len, GFP_KERNEL); + if (!identity) + return ERR_PTR(-ENOMEM); + + snprintf(identity, identity_len, "NVMe0%c%02d %s %s", + generated ? 'G' : 'R', hmac, hostnqn, subnqn); + + if (!keyring) + keyring = nvme_keyring; + keyring_id = key_serial(keyring); + pr_debug("keyring %x lookup tls psk '%s'\n", + keyring_id, identity); + keyref = keyring_search(make_key_ref(keyring, true), + &nvme_tls_psk_key_type, + identity, false); + if (IS_ERR(keyref)) { + pr_debug("lookup tls psk '%s' failed, error %ld\n", + identity, PTR_ERR(keyref)); + kfree(identity); + return ERR_PTR(-ENOKEY); + } + kfree(identity); + + return key_ref_to_ptr(keyref); +} + int nvme_keyring_init(void) { + int err; + nvme_keyring = keyring_alloc(".nvme", GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, current_cred(), @@ -28,12 +116,18 @@ int nvme_keyring_init(void) if (IS_ERR(nvme_keyring)) return PTR_ERR(nvme_keyring); + err = register_key_type(&nvme_tls_psk_key_type); + if (err) { + key_put(nvme_keyring); + return err; + } return 0; } EXPORT_SYMBOL_GPL(nvme_keyring_init); void nvme_keyring_exit(void) { + unregister_key_type(&nvme_tls_psk_key_type); key_revoke(nvme_keyring); key_put(nvme_keyring); } -- cgit From 646f45b23218c7a97a84259d8eeb22dad5711fc8 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Thu, 24 Aug 2023 16:39:10 +0200 Subject: nvme: add TCP TSAS definitions Signed-off-by: Hannes Reinecke Reviewed-by: Sagi Grimberg Signed-off-by: Keith Busch --- include/linux/nvme.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/include/linux/nvme.h b/include/linux/nvme.h index 26dd3f859d9d..a7ba74babad7 100644 --- a/include/linux/nvme.h +++ b/include/linux/nvme.h @@ -108,6 +108,13 @@ enum { NVMF_RDMA_CMS_RDMA_CM = 1, /* Sockets based endpoint addressing */ }; +/* TSAS SECTYPE for TCP transport */ +enum { + NVMF_TCP_SECTYPE_NONE = 0, /* No Security */ + NVMF_TCP_SECTYPE_TLS12 = 1, /* TLSv1.2, NVMe-oF 1.1 and NVMe-TCP 3.6.1.1 */ + NVMF_TCP_SECTYPE_TLS13 = 2, /* TLSv1.3, NVMe-oF 1.1 and NVMe-TCP 3.6.1.1 */ +}; + #define NVME_AQ_DEPTH 32 #define NVME_NR_AEN_COMMANDS 1 #define NVME_AQ_BLK_MQ_DEPTH (NVME_AQ_DEPTH - NVME_NR_AEN_COMMANDS) @@ -1493,6 +1500,9 @@ struct nvmf_disc_rsp_page_entry { __u16 pkey; __u8 resv10[246]; } rdma; + struct tcp { + __u8 sectype; + } tcp; } tsas; }; -- cgit From a86062aac34d100a3117c0fff91ee1892ebfb460 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Thu, 24 Aug 2023 16:39:11 +0200 Subject: nvme-tcp: add definitions for TLS cipher suites Signed-off-by: Hannes Reinecke Reviewed-by: Sagi Grimberg Signed-off-by: Keith Busch --- include/linux/nvme-tcp.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/include/linux/nvme-tcp.h b/include/linux/nvme-tcp.h index 57ebe1267f7f..e07e8978d691 100644 --- a/include/linux/nvme-tcp.h +++ b/include/linux/nvme-tcp.h @@ -18,6 +18,12 @@ enum nvme_tcp_pfv { NVME_TCP_PFV_1_0 = 0x0, }; +enum nvme_tcp_tls_cipher { + NVME_TCP_TLS_CIPHER_INVALID = 0, + NVME_TCP_TLS_CIPHER_SHA256 = 1, + NVME_TCP_TLS_CIPHER_SHA384 = 2, +}; + enum nvme_tcp_fatal_error_status { NVME_TCP_FES_INVALID_PDU_HDR = 0x01, NVME_TCP_FES_PDU_SEQ_ERR = 0x02, -- cgit From 501cc6f4aca8dc0958c4d9716f0233ba7cff4830 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Thu, 24 Aug 2023 16:39:12 +0200 Subject: nvme-keyring: implement nvme_tls_psk_default() Implement a function to select the preferred PSK for TLS. A 'retained' PSK should be preferred over a 'generated' PSK, and SHA-384 should be preferred to SHA-256. Signed-off-by: Hannes Reinecke Reviewed-by: Sagi Grimberg Signed-off-by: Keith Busch --- drivers/nvme/common/keyring.c | 48 +++++++++++++++++++++++++++++++++++++++++++ include/linux/nvme-keyring.h | 8 ++++++++ 2 files changed, 56 insertions(+) diff --git a/drivers/nvme/common/keyring.c b/drivers/nvme/common/keyring.c index 494dd365052e..f8d9a208397b 100644 --- a/drivers/nvme/common/keyring.c +++ b/drivers/nvme/common/keyring.c @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -103,6 +104,53 @@ static struct key *nvme_tls_psk_lookup(struct key *keyring, return key_ref_to_ptr(keyref); } +/* + * NVMe PSK priority list + * + * 'Retained' PSKs (ie 'generated == false') + * should be preferred to 'generated' PSKs, + * and SHA-384 should be preferred to SHA-256. + */ +struct nvme_tls_psk_priority_list { + bool generated; + enum nvme_tcp_tls_cipher cipher; +} nvme_tls_psk_prio[] = { + { .generated = false, + .cipher = NVME_TCP_TLS_CIPHER_SHA384, }, + { .generated = false, + .cipher = NVME_TCP_TLS_CIPHER_SHA256, }, + { .generated = true, + .cipher = NVME_TCP_TLS_CIPHER_SHA384, }, + { .generated = true, + .cipher = NVME_TCP_TLS_CIPHER_SHA256, }, +}; + +/* + * nvme_tls_psk_default - Return the preferred PSK to use for TLS ClientHello + */ +key_serial_t nvme_tls_psk_default(struct key *keyring, + const char *hostnqn, const char *subnqn) +{ + struct key *tls_key; + key_serial_t tls_key_id; + int prio; + + for (prio = 0; prio < ARRAY_SIZE(nvme_tls_psk_prio); prio++) { + bool generated = nvme_tls_psk_prio[prio].generated; + enum nvme_tcp_tls_cipher cipher = nvme_tls_psk_prio[prio].cipher; + + tls_key = nvme_tls_psk_lookup(keyring, hostnqn, subnqn, + cipher, generated); + if (!IS_ERR(tls_key)) { + tls_key_id = tls_key->serial; + key_put(tls_key); + return tls_key_id; + } + } + return 0; +} +EXPORT_SYMBOL_GPL(nvme_tls_psk_default); + int nvme_keyring_init(void) { int err; diff --git a/include/linux/nvme-keyring.h b/include/linux/nvme-keyring.h index 32bd264a71e6..4efea9dd967c 100644 --- a/include/linux/nvme-keyring.h +++ b/include/linux/nvme-keyring.h @@ -8,12 +8,20 @@ #ifdef CONFIG_NVME_KEYRING +key_serial_t nvme_tls_psk_default(struct key *keyring, + const char *hostnqn, const char *subnqn); + key_serial_t nvme_keyring_id(void); int nvme_keyring_init(void); void nvme_keyring_exit(void); #else +static inline key_serial_t nvme_tls_psk_default(struct key *keyring, + const char *hostnqn, const char *subnqn) +{ + return 0; +} static inline key_serial_t nvme_keyring_id(void) { return 0; -- cgit From 037c34318a479294cdb98dc8018edd5d191b68c0 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Thu, 24 Aug 2023 16:39:13 +0200 Subject: security/keys: export key_lookup() For in-kernel consumers one cannot readily assign a user (eg when running from a workqueue), so the normal key search permissions cannot be applied. This patch exports the 'key_lookup()' function for a simple lookup of keys without checking for permissions. Signed-off-by: Hannes Reinecke Reviewed-by: Sagi Grimberg Acked-by: David Howells Signed-off-by: Keith Busch --- include/linux/key.h | 1 + security/keys/key.c | 1 + 2 files changed, 2 insertions(+) diff --git a/include/linux/key.h b/include/linux/key.h index 938d7ecfb495..943a432da3ae 100644 --- a/include/linux/key.h +++ b/include/linux/key.h @@ -515,6 +515,7 @@ extern void key_init(void); #define key_init() do { } while(0) #define key_free_user_ns(ns) do { } while(0) #define key_remove_domain(d) do { } while(0) +#define key_lookup(k) NULL #endif /* CONFIG_KEYS */ #endif /* __KERNEL__ */ diff --git a/security/keys/key.c b/security/keys/key.c index 5c0c7df833f8..0260a1902922 100644 --- a/security/keys/key.c +++ b/security/keys/key.c @@ -693,6 +693,7 @@ error: spin_unlock(&key_serial_lock); return key; } +EXPORT_SYMBOL(key_lookup); /* * Find and lock the specified key type against removal. -- cgit From e40d4eb84089eae14a3396ba8b0db7b1f24ef2f8 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Thu, 24 Aug 2023 16:39:14 +0200 Subject: nvme-tcp: allocate socket file When using the TLS upcall we need to allocate a socket file such that the userspace daemon is able to use the socket. Signed-off-by: Hannes Reinecke Reviewed-by: Sagi Grimberg Signed-off-by: Keith Busch --- drivers/nvme/host/tcp.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index 5b332d9f87fc..5af6c7df44a6 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -1338,7 +1338,9 @@ static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid) } noreclaim_flag = memalloc_noreclaim_save(); - sock_release(queue->sock); + /* ->sock will be released by fput() */ + fput(queue->sock->file); + queue->sock = NULL; memalloc_noreclaim_restore(noreclaim_flag); kfree(queue->pdu); @@ -1512,6 +1514,7 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid) struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); struct nvme_tcp_queue *queue = &ctrl->queues[qid]; int ret, rcv_pdu_size; + struct file *sock_file; mutex_init(&queue->queue_lock); queue->ctrl = ctrl; @@ -1534,6 +1537,11 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid) goto err_destroy_mutex; } + sock_file = sock_alloc_file(queue->sock, O_CLOEXEC, NULL); + if (IS_ERR(sock_file)) { + ret = PTR_ERR(sock_file); + goto err_destroy_mutex; + } nvme_tcp_reclassify_socket(queue->sock); /* Single syn retry */ @@ -1640,7 +1648,8 @@ err_crypto: if (queue->hdr_digest || queue->data_digest) nvme_tcp_free_crypto(queue); err_sock: - sock_release(queue->sock); + /* ->sock will be released by fput() */ + fput(queue->sock->file); queue->sock = NULL; err_destroy_mutex: mutex_destroy(&queue->send_mutex); -- cgit From be8e82caa685997b524dc7e4932853fd2fbe6199 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Thu, 24 Aug 2023 16:39:15 +0200 Subject: nvme-tcp: enable TLS handshake upcall Add a fabrics option 'tls' and start the TLS handshake upcall with the default PSK. When TLS is started the PSK key serial number is displayed in the sysfs attribute 'tls_key' Signed-off-by: Hannes Reinecke Reviewed-by: Sagi Grimberg Signed-off-by: Keith Busch --- drivers/nvme/host/Kconfig | 15 +++++ drivers/nvme/host/core.c | 2 +- drivers/nvme/host/fabrics.c | 12 ++++ drivers/nvme/host/fabrics.h | 3 + drivers/nvme/host/nvme.h | 1 + drivers/nvme/host/sysfs.c | 21 +++++++ drivers/nvme/host/tcp.c | 138 ++++++++++++++++++++++++++++++++++++++++++-- 7 files changed, 185 insertions(+), 7 deletions(-) diff --git a/drivers/nvme/host/Kconfig b/drivers/nvme/host/Kconfig index 2f6a7f8c94e8..a517030d7d50 100644 --- a/drivers/nvme/host/Kconfig +++ b/drivers/nvme/host/Kconfig @@ -92,6 +92,21 @@ config NVME_TCP If unsure, say N. +config NVME_TCP_TLS + bool "NVMe over Fabrics TCP TLS encryption support" + depends on NVME_TCP + select NVME_COMMON + select NVME_KEYRING + select NET_HANDSHAKE + select KEYS + help + Enables TLS encryption for NVMe TCP using the netlink handshake API. + + The TLS handshake daemon is availble at + https://github.com/oracle/ktls-utils. + + If unsure, say N. + config NVME_AUTH bool "NVM Express over Fabrics In-Band Authentication" depends on NVME_CORE diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index a49b65d34cda..55e54ffb1003 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -4400,7 +4400,7 @@ static void nvme_free_ctrl(struct device *dev) if (!subsys || ctrl->instance != subsys->instance) ida_free(&nvme_instance_ida, ctrl->instance); - + key_put(ctrl->tls_key); nvme_free_cels(ctrl); nvme_mpath_uninit(ctrl); nvme_auth_stop(ctrl); diff --git a/drivers/nvme/host/fabrics.c b/drivers/nvme/host/fabrics.c index 8175d49f2909..ddad482c3537 100644 --- a/drivers/nvme/host/fabrics.c +++ b/drivers/nvme/host/fabrics.c @@ -647,6 +647,9 @@ static const match_table_t opt_tokens = { { NVMF_OPT_DISCOVERY, "discovery" }, { NVMF_OPT_DHCHAP_SECRET, "dhchap_secret=%s" }, { NVMF_OPT_DHCHAP_CTRL_SECRET, "dhchap_ctrl_secret=%s" }, +#ifdef CONFIG_NVME_TCP_TLS + { NVMF_OPT_TLS, "tls" }, +#endif { NVMF_OPT_ERR, NULL } }; @@ -671,6 +674,7 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts, opts->hdr_digest = false; opts->data_digest = false; opts->tos = -1; /* < 0 == use transport default */ + opts->tls = false; options = o = kstrdup(buf, GFP_KERNEL); if (!options) @@ -955,6 +959,14 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts, kfree(opts->dhchap_ctrl_secret); opts->dhchap_ctrl_secret = p; break; + case NVMF_OPT_TLS: + if (!IS_ENABLED(CONFIG_NVME_TCP_TLS)) { + pr_err("TLS is not supported\n"); + ret = -EINVAL; + goto out; + } + opts->tls = true; + break; default: pr_warn("unknown parameter or missing value '%s' in ctrl creation request\n", p); diff --git a/drivers/nvme/host/fabrics.h b/drivers/nvme/host/fabrics.h index 82e7a27ffbde..dac17c3fee26 100644 --- a/drivers/nvme/host/fabrics.h +++ b/drivers/nvme/host/fabrics.h @@ -70,6 +70,7 @@ enum { NVMF_OPT_DISCOVERY = 1 << 22, NVMF_OPT_DHCHAP_SECRET = 1 << 23, NVMF_OPT_DHCHAP_CTRL_SECRET = 1 << 24, + NVMF_OPT_TLS = 1 << 25, }; /** @@ -102,6 +103,7 @@ enum { * @dhchap_secret: DH-HMAC-CHAP secret * @dhchap_ctrl_secret: DH-HMAC-CHAP controller secret for bi-directional * authentication + * @tls: Start TLS encrypted connections (TCP) * @disable_sqflow: disable controller sq flow control * @hdr_digest: generate/verify header digest (TCP) * @data_digest: generate/verify data digest (TCP) @@ -128,6 +130,7 @@ struct nvmf_ctrl_options { struct nvmf_host *host; char *dhchap_secret; char *dhchap_ctrl_secret; + bool tls; bool disable_sqflow; bool hdr_digest; bool data_digest; diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index f35647c470af..6fe7966f720b 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -357,6 +357,7 @@ struct nvme_ctrl { struct nvme_dhchap_key *ctrl_key; u16 transaction; #endif + struct key *tls_key; /* Power saving configuration */ u64 ps_max_latency_us; diff --git a/drivers/nvme/host/sysfs.c b/drivers/nvme/host/sysfs.c index 212e1b05d298..d0966159981c 100644 --- a/drivers/nvme/host/sysfs.c +++ b/drivers/nvme/host/sysfs.c @@ -527,6 +527,19 @@ static DEVICE_ATTR(dhchap_ctrl_secret, S_IRUGO | S_IWUSR, nvme_ctrl_dhchap_ctrl_secret_show, nvme_ctrl_dhchap_ctrl_secret_store); #endif +#ifdef CONFIG_NVME_TCP_TLS +static ssize_t tls_key_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct nvme_ctrl *ctrl = dev_get_drvdata(dev); + + if (!ctrl->tls_key) + return 0; + return sysfs_emit(buf, "%08x", key_serial(ctrl->tls_key)); +} +static DEVICE_ATTR_RO(tls_key); +#endif + static struct attribute *nvme_dev_attrs[] = { &dev_attr_reset_controller.attr, &dev_attr_rescan_controller.attr, @@ -553,6 +566,9 @@ static struct attribute *nvme_dev_attrs[] = { #ifdef CONFIG_NVME_AUTH &dev_attr_dhchap_secret.attr, &dev_attr_dhchap_ctrl_secret.attr, +#endif +#ifdef CONFIG_NVME_TCP_TLS + &dev_attr_tls_key.attr, #endif NULL }; @@ -583,6 +599,11 @@ static umode_t nvme_dev_attrs_are_visible(struct kobject *kobj, if (a == &dev_attr_dhchap_ctrl_secret.attr && !ctrl->opts) return 0; #endif +#ifdef CONFIG_NVME_TCP_TLS + if (a == &dev_attr_tls_key.attr && + (!ctrl->opts || strcmp(ctrl->opts->transport, "tcp"))) + return 0; +#endif return a->mode; } diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index 5af6c7df44a6..696fc2a7da52 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -8,9 +8,13 @@ #include #include #include +#include #include +#include #include #include +#include +#include #include #include #include @@ -31,6 +35,16 @@ static int so_priority; module_param(so_priority, int, 0644); MODULE_PARM_DESC(so_priority, "nvme tcp socket optimize priority"); +#ifdef CONFIG_NVME_TCP_TLS +/* + * TLS handshake timeout + */ +static int tls_handshake_timeout = 10; +module_param(tls_handshake_timeout, int, 0644); +MODULE_PARM_DESC(tls_handshake_timeout, + "nvme TLS handshake timeout in seconds (default 10)"); +#endif + #ifdef CONFIG_DEBUG_LOCK_ALLOC /* lockdep can detect a circular dependency of the form * sk_lock -> mmap_lock (page fault) -> fs locks -> sk_lock @@ -146,7 +160,10 @@ struct nvme_tcp_queue { struct ahash_request *snd_hash; __le32 exp_ddgst; __le32 recv_ddgst; - +#ifdef CONFIG_NVME_TCP_TLS + struct completion tls_complete; + int tls_err; +#endif struct page_frag_cache pf_cache; void (*state_change)(struct sock *); @@ -1509,7 +1526,92 @@ static void nvme_tcp_set_queue_io_cpu(struct nvme_tcp_queue *queue) queue->io_cpu = cpumask_next_wrap(n - 1, cpu_online_mask, -1, false); } -static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid) +#ifdef CONFIG_NVME_TCP_TLS +static void nvme_tcp_tls_done(void *data, int status, key_serial_t pskid) +{ + struct nvme_tcp_queue *queue = data; + struct nvme_tcp_ctrl *ctrl = queue->ctrl; + int qid = nvme_tcp_queue_id(queue); + struct key *tls_key; + + dev_dbg(ctrl->ctrl.device, "queue %d: TLS handshake done, key %x, status %d\n", + qid, pskid, status); + + if (status) { + queue->tls_err = -status; + goto out_complete; + } + + tls_key = key_lookup(pskid); + if (IS_ERR(tls_key)) { + dev_warn(ctrl->ctrl.device, "queue %d: Invalid key %x\n", + qid, pskid); + queue->tls_err = -ENOKEY; + } else { + ctrl->ctrl.tls_key = tls_key; + queue->tls_err = 0; + } + +out_complete: + complete(&queue->tls_complete); +} + +static int nvme_tcp_start_tls(struct nvme_ctrl *nctrl, + struct nvme_tcp_queue *queue, + key_serial_t pskid) +{ + int qid = nvme_tcp_queue_id(queue); + int ret; + struct tls_handshake_args args; + unsigned long tmo = tls_handshake_timeout * HZ; + key_serial_t keyring = nvme_keyring_id(); + + dev_dbg(nctrl->device, "queue %d: start TLS with key %x\n", + qid, pskid); + memset(&args, 0, sizeof(args)); + args.ta_sock = queue->sock; + args.ta_done = nvme_tcp_tls_done; + args.ta_data = queue; + args.ta_my_peerids[0] = pskid; + args.ta_num_peerids = 1; + args.ta_keyring = keyring; + args.ta_timeout_ms = tls_handshake_timeout * 1000; + queue->tls_err = -EOPNOTSUPP; + init_completion(&queue->tls_complete); + ret = tls_client_hello_psk(&args, GFP_KERNEL); + if (ret) { + dev_err(nctrl->device, "queue %d: failed to start TLS: %d\n", + qid, ret); + return ret; + } + ret = wait_for_completion_interruptible_timeout(&queue->tls_complete, tmo); + if (ret <= 0) { + if (ret == 0) + ret = -ETIMEDOUT; + + dev_err(nctrl->device, + "queue %d: TLS handshake failed, error %d\n", + qid, ret); + tls_handshake_cancel(queue->sock->sk); + } else { + dev_dbg(nctrl->device, + "queue %d: TLS handshake complete, error %d\n", + qid, queue->tls_err); + ret = queue->tls_err; + } + return ret; +} +#else +static int nvme_tcp_start_tls(struct nvme_ctrl *nctrl, + struct nvme_tcp_queue *queue, + key_serial_t pskid) +{ + return -EPROTONOSUPPORT; +} +#endif + +static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid, + key_serial_t pskid) { struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); struct nvme_tcp_queue *queue = &ctrl->queues[qid]; @@ -1632,6 +1734,13 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid) goto err_rcv_pdu; } + /* If PSKs are configured try to start TLS */ + if (pskid) { + ret = nvme_tcp_start_tls(nctrl, queue, pskid); + if (ret) + goto err_init_connect; + } + ret = nvme_tcp_init_connection(queue); if (ret) goto err_init_connect; @@ -1781,10 +1890,22 @@ out_stop_queues: static int nvme_tcp_alloc_admin_queue(struct nvme_ctrl *ctrl) { int ret; + key_serial_t pskid = 0; + + if (ctrl->opts->tls) { + pskid = nvme_tls_psk_default(NULL, + ctrl->opts->host->nqn, + ctrl->opts->subsysnqn); + if (!pskid) { + dev_err(ctrl->device, "no valid PSK found\n"); + ret = -ENOKEY; + goto out_free_queue; + } + } - ret = nvme_tcp_alloc_queue(ctrl, 0); + ret = nvme_tcp_alloc_queue(ctrl, 0, pskid); if (ret) - return ret; + goto out_free_queue; ret = nvme_tcp_alloc_async_req(to_tcp_ctrl(ctrl)); if (ret) @@ -1801,8 +1922,13 @@ static int __nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl) { int i, ret; + if (ctrl->opts->tls && !ctrl->tls_key) { + dev_err(ctrl->device, "no PSK negotiated\n"); + return -ENOKEY; + } for (i = 1; i < ctrl->queue_count; i++) { - ret = nvme_tcp_alloc_queue(ctrl, i); + ret = nvme_tcp_alloc_queue(ctrl, i, + key_serial(ctrl->tls_key)); if (ret) goto out_free_queues; } @@ -2630,7 +2756,7 @@ static struct nvmf_transport_ops nvme_tcp_transport = { NVMF_OPT_HOST_TRADDR | NVMF_OPT_CTRL_LOSS_TMO | NVMF_OPT_HDR_DIGEST | NVMF_OPT_DATA_DIGEST | NVMF_OPT_NR_WRITE_QUEUES | NVMF_OPT_NR_POLL_QUEUES | - NVMF_OPT_TOS | NVMF_OPT_HOST_IFACE, + NVMF_OPT_TOS | NVMF_OPT_HOST_IFACE | NVMF_OPT_TLS, .create_ctrl = nvme_tcp_create_ctrl, }; -- cgit From 2837966ab2a8117b002ee9c02fa62a01db787844 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Thu, 24 Aug 2023 16:39:16 +0200 Subject: nvme-tcp: control message handling for recvmsg() kTLS is sending TLS ALERT messages as control messages for recvmsg(). As we can't do anything sensible with it just abort the connection and let the userspace agent to a re-negotiation. Signed-off-by: Hannes Reinecke Reviewed-by: Sagi Grimberg Signed-off-by: Keith Busch --- drivers/nvme/host/tcp.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index 696fc2a7da52..69975b655179 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -1369,6 +1370,8 @@ static int nvme_tcp_init_connection(struct nvme_tcp_queue *queue) { struct nvme_tcp_icreq_pdu *icreq; struct nvme_tcp_icresp_pdu *icresp; + char cbuf[CMSG_LEN(sizeof(char))] = {}; + u8 ctype; struct msghdr msg = {}; struct kvec iov; bool ctrl_hdgst, ctrl_ddgst; @@ -1406,11 +1409,23 @@ static int nvme_tcp_init_connection(struct nvme_tcp_queue *queue) memset(&msg, 0, sizeof(msg)); iov.iov_base = icresp; iov.iov_len = sizeof(*icresp); + if (queue->ctrl->ctrl.opts->tls) { + msg.msg_control = cbuf; + msg.msg_controllen = sizeof(cbuf); + } ret = kernel_recvmsg(queue->sock, &msg, &iov, 1, iov.iov_len, msg.msg_flags); if (ret < 0) goto free_icresp; - + if (queue->ctrl->ctrl.opts->tls) { + ctype = tls_get_record_type(queue->sock->sk, + (struct cmsghdr *)cbuf); + if (ctype != TLS_RECORD_TYPE_DATA) { + pr_err("queue %d: unhandled TLS record %d\n", + nvme_tcp_queue_id(queue), ctype); + return -ENOTCONN; + } + } ret = -EINVAL; if (icresp->hdr.type != nvme_tcp_icresp) { pr_err("queue %d: bad type returned %d\n", -- cgit From e4f4aabb6f66177410e573bdf821f29f9746765f Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Thu, 24 Aug 2023 16:39:17 +0200 Subject: nvme-tcp: improve icreq/icresp logging When icreq/icresp fails we should be printing out a warning to inform the user that the connection could not be established; without it there won't be anything in the kernel message log, just an error code returned to nvme-cli. Signed-off-by: Hannes Reinecke Reviewed-by: Sagi Grimberg Signed-off-by: Keith Busch --- drivers/nvme/host/tcp.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index 69975b655179..4656ad20e157 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -1403,8 +1403,11 @@ static int nvme_tcp_init_connection(struct nvme_tcp_queue *queue) iov.iov_base = icreq; iov.iov_len = sizeof(*icreq); ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len); - if (ret < 0) + if (ret < 0) { + pr_warn("queue %d: failed to send icreq, error %d\n", + nvme_tcp_queue_id(queue), ret); goto free_icresp; + } memset(&msg, 0, sizeof(msg)); iov.iov_base = icresp; @@ -1415,8 +1418,11 @@ static int nvme_tcp_init_connection(struct nvme_tcp_queue *queue) } ret = kernel_recvmsg(queue->sock, &msg, &iov, 1, iov.iov_len, msg.msg_flags); - if (ret < 0) + if (ret < 0) { + pr_warn("queue %d: failed to receive icresp, error %d\n", + nvme_tcp_queue_id(queue), ret); goto free_icresp; + } if (queue->ctrl->ctrl.opts->tls) { ctype = tls_get_record_type(queue->sock->sk, (struct cmsghdr *)cbuf); -- cgit From adf22c520b5ec45a6f05a19ed26d61fe463b6626 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Thu, 24 Aug 2023 16:39:18 +0200 Subject: nvme-fabrics: parse options 'keyring' and 'tls_key' Parse the fabrics options 'keyring' and 'tls_key' and store the referenced keys in the options structure. Signed-off-by: Hannes Reinecke Reviewed-by: Sagi Grimberg Signed-off-by: Keith Busch --- drivers/nvme/host/fabrics.c | 55 ++++++++++++++++++++++++++++++++++++++++++++- drivers/nvme/host/fabrics.h | 6 +++++ drivers/nvme/host/tcp.c | 14 ++++++++---- 3 files changed, 70 insertions(+), 5 deletions(-) diff --git a/drivers/nvme/host/fabrics.c b/drivers/nvme/host/fabrics.c index ddad482c3537..4673ead69c5f 100644 --- a/drivers/nvme/host/fabrics.c +++ b/drivers/nvme/host/fabrics.c @@ -12,6 +12,7 @@ #include #include "nvme.h" #include "fabrics.h" +#include static LIST_HEAD(nvmf_transports); static DECLARE_RWSEM(nvmf_transports_rwsem); @@ -622,6 +623,23 @@ static struct nvmf_transport_ops *nvmf_lookup_transport( return NULL; } +static struct key *nvmf_parse_key(int key_id) +{ + struct key *key; + + if (!IS_ENABLED(CONFIG_NVME_TCP_TLS)) { + pr_err("TLS is not supported\n"); + return ERR_PTR(-EINVAL); + } + + key = key_lookup(key_id); + if (!IS_ERR(key)) + pr_err("key id %08x not found\n", key_id); + else + pr_debug("Using key id %08x\n", key_id); + return key; +} + static const match_table_t opt_tokens = { { NVMF_OPT_TRANSPORT, "transport=%s" }, { NVMF_OPT_TRADDR, "traddr=%s" }, @@ -643,6 +661,10 @@ static const match_table_t opt_tokens = { { NVMF_OPT_NR_WRITE_QUEUES, "nr_write_queues=%d" }, { NVMF_OPT_NR_POLL_QUEUES, "nr_poll_queues=%d" }, { NVMF_OPT_TOS, "tos=%d" }, +#ifdef CONFIG_NVME_TCP_TLS + { NVMF_OPT_KEYRING, "keyring=%d" }, + { NVMF_OPT_TLS_KEY, "tls_key=%d" }, +#endif { NVMF_OPT_FAIL_FAST_TMO, "fast_io_fail_tmo=%d" }, { NVMF_OPT_DISCOVERY, "discovery" }, { NVMF_OPT_DHCHAP_SECRET, "dhchap_secret=%s" }, @@ -660,9 +682,10 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts, char *options, *o, *p; int token, ret = 0; size_t nqnlen = 0; - int ctrl_loss_tmo = NVMF_DEF_CTRL_LOSS_TMO; + int ctrl_loss_tmo = NVMF_DEF_CTRL_LOSS_TMO, key_id; uuid_t hostid; char hostnqn[NVMF_NQN_SIZE]; + struct key *key; /* Set defaults */ opts->queue_size = NVMF_DEF_QUEUE_SIZE; @@ -675,6 +698,8 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts, opts->data_digest = false; opts->tos = -1; /* < 0 == use transport default */ opts->tls = false; + opts->tls_key = NULL; + opts->keyring = NULL; options = o = kstrdup(buf, GFP_KERNEL); if (!options) @@ -928,6 +953,32 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts, } opts->tos = token; break; + case NVMF_OPT_KEYRING: + if (match_int(args, &key_id) || key_id <= 0) { + ret = -EINVAL; + goto out; + } + key = nvmf_parse_key(key_id); + if (IS_ERR(key)) { + ret = PTR_ERR(key); + goto out; + } + key_put(opts->keyring); + opts->keyring = key; + break; + case NVMF_OPT_TLS_KEY: + if (match_int(args, &key_id) || key_id <= 0) { + ret = -EINVAL; + goto out; + } + key = nvmf_parse_key(key_id); + if (IS_ERR(key)) { + ret = PTR_ERR(key); + goto out; + } + key_put(opts->tls_key); + opts->tls_key = key; + break; case NVMF_OPT_DISCOVERY: opts->discovery_nqn = true; break; @@ -1168,6 +1219,8 @@ static int nvmf_check_allowed_opts(struct nvmf_ctrl_options *opts, void nvmf_free_options(struct nvmf_ctrl_options *opts) { nvmf_host_put(opts->host); + key_put(opts->keyring); + key_put(opts->tls_key); kfree(opts->transport); kfree(opts->traddr); kfree(opts->trsvcid); diff --git a/drivers/nvme/host/fabrics.h b/drivers/nvme/host/fabrics.h index dac17c3fee26..fbaee5a7be19 100644 --- a/drivers/nvme/host/fabrics.h +++ b/drivers/nvme/host/fabrics.h @@ -71,6 +71,8 @@ enum { NVMF_OPT_DHCHAP_SECRET = 1 << 23, NVMF_OPT_DHCHAP_CTRL_SECRET = 1 << 24, NVMF_OPT_TLS = 1 << 25, + NVMF_OPT_KEYRING = 1 << 26, + NVMF_OPT_TLS_KEY = 1 << 27, }; /** @@ -103,6 +105,8 @@ enum { * @dhchap_secret: DH-HMAC-CHAP secret * @dhchap_ctrl_secret: DH-HMAC-CHAP controller secret for bi-directional * authentication + * @keyring: Keyring to use for key lookups + * @tls_key: TLS key for encrypted connections (TCP) * @tls: Start TLS encrypted connections (TCP) * @disable_sqflow: disable controller sq flow control * @hdr_digest: generate/verify header digest (TCP) @@ -130,6 +134,8 @@ struct nvmf_ctrl_options { struct nvmf_host *host; char *dhchap_secret; char *dhchap_ctrl_secret; + struct key *keyring; + struct key *tls_key; bool tls; bool disable_sqflow; bool hdr_digest; diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index 4656ad20e157..4714a902f4ca 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -1595,6 +1595,8 @@ static int nvme_tcp_start_tls(struct nvme_ctrl *nctrl, args.ta_data = queue; args.ta_my_peerids[0] = pskid; args.ta_num_peerids = 1; + if (nctrl->opts->keyring) + keyring = key_serial(nctrl->opts->keyring); args.ta_keyring = keyring; args.ta_timeout_ms = tls_handshake_timeout * 1000; queue->tls_err = -EOPNOTSUPP; @@ -1914,9 +1916,12 @@ static int nvme_tcp_alloc_admin_queue(struct nvme_ctrl *ctrl) key_serial_t pskid = 0; if (ctrl->opts->tls) { - pskid = nvme_tls_psk_default(NULL, - ctrl->opts->host->nqn, - ctrl->opts->subsysnqn); + if (ctrl->opts->tls_key) + pskid = key_serial(ctrl->opts->tls_key); + else + pskid = nvme_tls_psk_default(ctrl->opts->keyring, + ctrl->opts->host->nqn, + ctrl->opts->subsysnqn); if (!pskid) { dev_err(ctrl->device, "no valid PSK found\n"); ret = -ENOKEY; @@ -2777,7 +2782,8 @@ static struct nvmf_transport_ops nvme_tcp_transport = { NVMF_OPT_HOST_TRADDR | NVMF_OPT_CTRL_LOSS_TMO | NVMF_OPT_HDR_DIGEST | NVMF_OPT_DATA_DIGEST | NVMF_OPT_NR_WRITE_QUEUES | NVMF_OPT_NR_POLL_QUEUES | - NVMF_OPT_TOS | NVMF_OPT_HOST_IFACE | NVMF_OPT_TLS, + NVMF_OPT_TOS | NVMF_OPT_HOST_IFACE | NVMF_OPT_TLS | + NVMF_OPT_KEYRING | NVMF_OPT_TLS_KEY, .create_ctrl = nvme_tcp_create_ctrl, }; -- cgit From 3f123494db725d899fdf990d894648bcc33cf7a3 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Thu, 24 Aug 2023 16:39:19 +0200 Subject: nvmet: make TCP sectype settable via configfs Add a new configfs attribute 'addr_tsas' to make the TCP sectype settable via configfs. Signed-off-by: Hannes Reinecke Reviewed-by: Sagi Grimberg Signed-off-by: Keith Busch --- drivers/nvme/target/configfs.c | 75 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c index 907143870da5..27a0f9c768e8 100644 --- a/drivers/nvme/target/configfs.c +++ b/drivers/nvme/target/configfs.c @@ -159,6 +159,11 @@ static const struct nvmet_type_name_map nvmet_addr_treq[] = { { NVMF_TREQ_NOT_REQUIRED, "not required" }, }; +static inline u8 nvmet_port_disc_addr_treq_mask(struct nvmet_port *port) +{ + return (port->disc_addr.treq & ~NVME_TREQ_SECURE_CHANNEL_MASK); +} + static ssize_t nvmet_addr_treq_show(struct config_item *item, char *page) { u8 treq = to_nvmet_port(item)->disc_addr.treq & @@ -178,7 +183,7 @@ static ssize_t nvmet_addr_treq_store(struct config_item *item, const char *page, size_t count) { struct nvmet_port *port = to_nvmet_port(item); - u8 treq = port->disc_addr.treq & ~NVME_TREQ_SECURE_CHANNEL_MASK; + u8 treq = nvmet_port_disc_addr_treq_mask(port); int i; if (nvmet_is_port_enabled(port, __func__)) @@ -303,6 +308,11 @@ static void nvmet_port_init_tsas_rdma(struct nvmet_port *port) port->disc_addr.tsas.rdma.cms = NVMF_RDMA_CMS_RDMA_CM; } +static void nvmet_port_init_tsas_tcp(struct nvmet_port *port, int sectype) +{ + port->disc_addr.tsas.tcp.sectype = sectype; +} + static ssize_t nvmet_addr_trtype_store(struct config_item *item, const char *page, size_t count) { @@ -325,11 +335,73 @@ found: port->disc_addr.trtype = nvmet_transport[i].type; if (port->disc_addr.trtype == NVMF_TRTYPE_RDMA) nvmet_port_init_tsas_rdma(port); + else if (port->disc_addr.trtype == NVMF_TRTYPE_TCP) + nvmet_port_init_tsas_tcp(port, NVMF_TCP_SECTYPE_NONE); return count; } CONFIGFS_ATTR(nvmet_, addr_trtype); +static const struct nvmet_type_name_map nvmet_addr_tsas_tcp[] = { + { NVMF_TCP_SECTYPE_NONE, "none" }, + { NVMF_TCP_SECTYPE_TLS13, "tls1.3" }, +}; + +static const struct nvmet_type_name_map nvmet_addr_tsas_rdma[] = { + { NVMF_RDMA_QPTYPE_CONNECTED, "connected" }, + { NVMF_RDMA_QPTYPE_DATAGRAM, "datagram" }, +}; + +static ssize_t nvmet_addr_tsas_show(struct config_item *item, + char *page) +{ + struct nvmet_port *port = to_nvmet_port(item); + int i; + + if (port->disc_addr.trtype == NVMF_TRTYPE_TCP) { + for (i = 0; i < ARRAY_SIZE(nvmet_addr_tsas_tcp); i++) { + if (port->disc_addr.tsas.tcp.sectype == nvmet_addr_tsas_tcp[i].type) + return sprintf(page, "%s\n", nvmet_addr_tsas_tcp[i].name); + } + } else if (port->disc_addr.trtype == NVMF_TRTYPE_RDMA) { + for (i = 0; i < ARRAY_SIZE(nvmet_addr_tsas_rdma); i++) { + if (port->disc_addr.tsas.rdma.qptype == nvmet_addr_tsas_rdma[i].type) + return sprintf(page, "%s\n", nvmet_addr_tsas_rdma[i].name); + } + } + return sprintf(page, "reserved\n"); +} + +static ssize_t nvmet_addr_tsas_store(struct config_item *item, + const char *page, size_t count) +{ + struct nvmet_port *port = to_nvmet_port(item); + u8 sectype; + int i; + + if (nvmet_is_port_enabled(port, __func__)) + return -EACCES; + + if (port->disc_addr.trtype != NVMF_TRTYPE_TCP) + return -EINVAL; + + for (i = 0; i < ARRAY_SIZE(nvmet_addr_tsas_tcp); i++) { + if (sysfs_streq(page, nvmet_addr_tsas_tcp[i].name)) { + sectype = nvmet_addr_tsas_tcp[i].type; + goto found; + } + } + + pr_err("Invalid value '%s' for tsas\n", page); + return -EINVAL; + +found: + nvmet_port_init_tsas_tcp(port, sectype); + return count; +} + +CONFIGFS_ATTR(nvmet_, addr_tsas); + /* * Namespace structures & file operation functions below */ @@ -1741,6 +1813,7 @@ static struct configfs_attribute *nvmet_port_attrs[] = { &nvmet_attr_addr_traddr, &nvmet_attr_addr_trsvcid, &nvmet_attr_addr_trtype, + &nvmet_attr_addr_tsas, &nvmet_attr_param_inline_data_size, #ifdef CONFIG_BLK_DEV_INTEGRITY &nvmet_attr_param_pi_enable, -- cgit From 4f8cce2d32a3011bff8c8a4bbc69bca5a7bdd030 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Thu, 24 Aug 2023 16:39:20 +0200 Subject: nvmet-tcp: make nvmet_tcp_alloc_queue() a void function The return value from nvmet_tcp_alloc_queue() are just used to figure out if sock_release() need to be called. So this patch moves sock_release() into nvmet_tcp_alloc_queue() and make it a void function. Signed-off-by: Hannes Reinecke Reviewed-by: Sagi Grimberg Reviewed-by: Nitesh Shetty Signed-off-by: Keith Busch --- drivers/nvme/target/tcp.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c index cd92d7ddf5ed..0dd90770a2e1 100644 --- a/drivers/nvme/target/tcp.c +++ b/drivers/nvme/target/tcp.c @@ -1621,15 +1621,17 @@ static int nvmet_tcp_set_queue_sock(struct nvmet_tcp_queue *queue) return ret; } -static int nvmet_tcp_alloc_queue(struct nvmet_tcp_port *port, +static void nvmet_tcp_alloc_queue(struct nvmet_tcp_port *port, struct socket *newsock) { struct nvmet_tcp_queue *queue; int ret; queue = kzalloc(sizeof(*queue), GFP_KERNEL); - if (!queue) - return -ENOMEM; + if (!queue) { + ret = -ENOMEM; + goto out_release; + } INIT_WORK(&queue->release_work, nvmet_tcp_release_queue_work); INIT_WORK(&queue->io_work, nvmet_tcp_io_work); @@ -1666,7 +1668,7 @@ static int nvmet_tcp_alloc_queue(struct nvmet_tcp_port *port, if (ret) goto out_destroy_sq; - return 0; + return; out_destroy_sq: mutex_lock(&nvmet_tcp_queue_mutex); list_del_init(&queue->queue_list); @@ -1678,7 +1680,9 @@ out_ida_remove: ida_free(&nvmet_tcp_queue_ida, queue->idx); out_free_queue: kfree(queue); - return ret; +out_release: + pr_err("failed to allocate queue, error %d\n", ret); + sock_release(newsock); } static void nvmet_tcp_accept_work(struct work_struct *w) @@ -1695,11 +1699,7 @@ static void nvmet_tcp_accept_work(struct work_struct *w) pr_warn("failed to accept err=%d\n", ret); return; } - ret = nvmet_tcp_alloc_queue(port, newsock); - if (ret) { - pr_err("failed to allocate queue\n"); - sock_release(newsock); - } + nvmet_tcp_alloc_queue(port, newsock); } } -- cgit From 79a4f186ad37423996e7835e9d3d6055ff1a181d Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Thu, 24 Aug 2023 16:39:21 +0200 Subject: nvmet-tcp: allocate socket file For the TLS upcall we need to allocate a socket file such that the userspace daemon is able to use the socket. Signed-off-by: Hannes Reinecke Reviewed-by: Sagi Grimberg Signed-off-by: Keith Busch --- drivers/nvme/target/tcp.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c index 0dd90770a2e1..e72786b70f68 100644 --- a/drivers/nvme/target/tcp.c +++ b/drivers/nvme/target/tcp.c @@ -1493,12 +1493,12 @@ static void nvmet_tcp_release_queue_work(struct work_struct *w) nvmet_sq_destroy(&queue->nvme_sq); cancel_work_sync(&queue->io_work); nvmet_tcp_free_cmd_data_in_buffers(queue); - sock_release(queue->sock); + /* ->sock will be released by fput() */ + fput(queue->sock->file); nvmet_tcp_free_cmds(queue); if (queue->hdr_digest || queue->data_digest) nvmet_tcp_free_crypto(queue); ida_free(&nvmet_tcp_queue_ida, queue->idx); - page = virt_to_head_page(queue->pf_cache.va); __page_frag_cache_drain(page, queue->pf_cache.pagecnt_bias); kfree(queue); @@ -1625,6 +1625,7 @@ static void nvmet_tcp_alloc_queue(struct nvmet_tcp_port *port, struct socket *newsock) { struct nvmet_tcp_queue *queue; + struct file *sock_file = NULL; int ret; queue = kzalloc(sizeof(*queue), GFP_KERNEL); @@ -1644,10 +1645,16 @@ static void nvmet_tcp_alloc_queue(struct nvmet_tcp_port *port, init_llist_head(&queue->resp_list); INIT_LIST_HEAD(&queue->resp_send_list); + sock_file = sock_alloc_file(queue->sock, O_CLOEXEC, NULL); + if (IS_ERR(sock_file)) { + ret = PTR_ERR(sock_file); + goto out_free_queue; + } + queue->idx = ida_alloc(&nvmet_tcp_queue_ida, GFP_KERNEL); if (queue->idx < 0) { ret = queue->idx; - goto out_free_queue; + goto out_sock; } ret = nvmet_tcp_alloc_cmd(queue, &queue->connect); @@ -1678,11 +1685,14 @@ out_free_connect: nvmet_tcp_free_cmd(&queue->connect); out_ida_remove: ida_free(&nvmet_tcp_queue_ida, queue->idx); +out_sock: + fput(queue->sock->file); out_free_queue: kfree(queue); out_release: pr_err("failed to allocate queue, error %d\n", ret); - sock_release(newsock); + if (!sock_file) + sock_release(newsock); } static void nvmet_tcp_accept_work(struct work_struct *w) -- cgit From eb398812bd046e86a332229d8a17525a05351b20 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Thu, 24 Aug 2023 16:39:22 +0200 Subject: nvmet: Set 'TREQ' to 'required' when TLS is enabled The current implementation does not support secure concatenation, so 'TREQ' is always set to 'required' when TLS is enabled. Signed-off-by: Hannes Reinecke Reviewed-by: Sagi Grimberg Signed-off-by: Keith Busch --- drivers/nvme/target/configfs.c | 15 +++++++++++++-- drivers/nvme/target/nvmet.h | 5 +++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c index 27a0f9c768e8..483569c3f622 100644 --- a/drivers/nvme/target/configfs.c +++ b/drivers/nvme/target/configfs.c @@ -166,8 +166,7 @@ static inline u8 nvmet_port_disc_addr_treq_mask(struct nvmet_port *port) static ssize_t nvmet_addr_treq_show(struct config_item *item, char *page) { - u8 treq = to_nvmet_port(item)->disc_addr.treq & - NVME_TREQ_SECURE_CHANNEL_MASK; + u8 treq = nvmet_port_disc_addr_treq_secure_channel(to_nvmet_port(item)); int i; for (i = 0; i < ARRAY_SIZE(nvmet_addr_treq); i++) { @@ -376,6 +375,7 @@ static ssize_t nvmet_addr_tsas_store(struct config_item *item, const char *page, size_t count) { struct nvmet_port *port = to_nvmet_port(item); + u8 treq = nvmet_port_disc_addr_treq_mask(port); u8 sectype; int i; @@ -397,6 +397,17 @@ static ssize_t nvmet_addr_tsas_store(struct config_item *item, found: nvmet_port_init_tsas_tcp(port, sectype); + /* + * The TLS implementation currently does not support + * secure concatenation, so TREQ is always set to 'required' + * if TLS is enabled. + */ + if (sectype == NVMF_TCP_SECTYPE_TLS13) { + treq |= NVMF_TREQ_REQUIRED; + } else { + treq |= NVMF_TREQ_NOT_SPECIFIED; + } + port->disc_addr.treq = treq; return count; } diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h index 8cfd60f3b564..87da62e4b743 100644 --- a/drivers/nvme/target/nvmet.h +++ b/drivers/nvme/target/nvmet.h @@ -178,6 +178,11 @@ static inline struct nvmet_port *ana_groups_to_port( ana_groups_group); } +static inline u8 nvmet_port_disc_addr_treq_secure_channel(struct nvmet_port *port) +{ + return (port->disc_addr.treq & NVME_TREQ_SECURE_CHANNEL_MASK); +} + struct nvmet_ctrl { struct nvmet_subsys *subsys; struct nvmet_sq **sqs; -- cgit From 675b453e024154dd547921c6e6d5b58747ba7e0e Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Thu, 24 Aug 2023 16:39:23 +0200 Subject: nvmet-tcp: enable TLS handshake upcall TLS handshake is handled in userspace with the netlink tls handshake protocol. The patch adds a function to start the TLS handshake upcall for any incoming network connections if the TCP TSAS sectype is set to 'tls1.3'. A config option NVME_TARGET_TCP_TLS selects whether the TLS handshake upcall should be compiled in. The patch also adds reference counting to struct nvmet_tcp_queue to ensure the queue is always valid when the the TLS handshake completes. Signed-off-by: Hannes Reinecke Reviewed-by: Sagi Grimberg Signed-off-by: Keith Busch --- drivers/nvme/target/Kconfig | 15 ++++ drivers/nvme/target/configfs.c | 21 ++++++ drivers/nvme/target/nvmet.h | 1 + drivers/nvme/target/tcp.c | 154 +++++++++++++++++++++++++++++++++++++++-- 4 files changed, 187 insertions(+), 4 deletions(-) diff --git a/drivers/nvme/target/Kconfig b/drivers/nvme/target/Kconfig index 79fc64035ee3..c56cb1005327 100644 --- a/drivers/nvme/target/Kconfig +++ b/drivers/nvme/target/Kconfig @@ -84,6 +84,21 @@ config NVME_TARGET_TCP If unsure, say N. +config NVME_TARGET_TCP_TLS + bool "NVMe over Fabrics TCP target TLS encryption support" + depends on NVME_TARGET_TCP + select NVME_COMMON + select NVME_KEYRING + select NET_HANDSHAKE + select KEYS + help + Enables TLS encryption for the NVMe TCP target using the netlink handshake API. + + The TLS handshake daemon is available at + https://github.com/oracle/ktls-utils. + + If unsure, say N. + config NVME_TARGET_AUTH bool "NVMe over Fabrics In-band Authentication support" depends on NVME_TARGET diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c index 483569c3f622..b780ce049163 100644 --- a/drivers/nvme/target/configfs.c +++ b/drivers/nvme/target/configfs.c @@ -15,6 +15,7 @@ #ifdef CONFIG_NVME_TARGET_AUTH #include #endif +#include #include #include @@ -396,6 +397,17 @@ static ssize_t nvmet_addr_tsas_store(struct config_item *item, return -EINVAL; found: + if (sectype == NVMF_TCP_SECTYPE_TLS13) { + if (!IS_ENABLED(CONFIG_NVME_TARGET_TCP_TLS)) { + pr_err("TLS is not supported\n"); + return -EINVAL; + } + if (!port->keyring) { + pr_err("TLS keyring not configured\n"); + return -EINVAL; + } + } + nvmet_port_init_tsas_tcp(port, sectype); /* * The TLS implementation currently does not support @@ -1814,6 +1826,7 @@ static void nvmet_port_release(struct config_item *item) flush_workqueue(nvmet_wq); list_del(&port->global_entry); + key_put(port->keyring); kfree(port->ana_state); kfree(port); } @@ -1863,6 +1876,14 @@ static struct config_group *nvmet_ports_make(struct config_group *group, return ERR_PTR(-ENOMEM); } + if (nvme_keyring_id()) { + port->keyring = key_lookup(nvme_keyring_id()); + if (IS_ERR(port->keyring)) { + pr_warn("NVMe keyring not available, disabling TLS\n"); + port->keyring = NULL; + } + } + for (i = 1; i <= NVMET_MAX_ANAGRPS; i++) { if (i == NVMET_DEFAULT_ANA_GRPID) port->ana_state[1] = NVME_ANA_OPTIMIZED; diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h index 87da62e4b743..e35a03260f45 100644 --- a/drivers/nvme/target/nvmet.h +++ b/drivers/nvme/target/nvmet.h @@ -158,6 +158,7 @@ struct nvmet_port { struct config_group ana_groups_group; struct nvmet_ana_group ana_default_group; enum nvme_ana_state *ana_state; + struct key *keyring; void *priv; bool enabled; int inline_data_size; diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c index e72786b70f68..58a4a737be77 100644 --- a/drivers/nvme/target/tcp.c +++ b/drivers/nvme/target/tcp.c @@ -8,9 +8,13 @@ #include #include #include +#include #include +#include #include #include +#include +#include #include #include #include @@ -66,6 +70,16 @@ device_param_cb(idle_poll_period_usecs, &set_param_ops, MODULE_PARM_DESC(idle_poll_period_usecs, "nvmet tcp io_work poll till idle time period in usecs: Default 0"); +#ifdef CONFIG_NVME_TARGET_TCP_TLS +/* + * TLS handshake timeout + */ +static int tls_handshake_timeout = 10; +module_param(tls_handshake_timeout, int, 0644); +MODULE_PARM_DESC(tls_handshake_timeout, + "nvme TLS handshake timeout in seconds (default 10)"); +#endif + #define NVMET_TCP_RECV_BUDGET 8 #define NVMET_TCP_SEND_BUDGET 8 #define NVMET_TCP_IO_WORK_BUDGET 64 @@ -122,8 +136,10 @@ struct nvmet_tcp_cmd { enum nvmet_tcp_queue_state { NVMET_TCP_Q_CONNECTING, + NVMET_TCP_Q_TLS_HANDSHAKE, NVMET_TCP_Q_LIVE, NVMET_TCP_Q_DISCONNECTING, + NVMET_TCP_Q_FAILED, }; struct nvmet_tcp_queue { @@ -132,6 +148,7 @@ struct nvmet_tcp_queue { struct work_struct io_work; struct nvmet_cq nvme_cq; struct nvmet_sq nvme_sq; + struct kref kref; /* send state */ struct nvmet_tcp_cmd *cmds; @@ -155,6 +172,10 @@ struct nvmet_tcp_queue { struct ahash_request *snd_hash; struct ahash_request *rcv_hash; + /* TLS state */ + key_serial_t tls_pskid; + struct delayed_work tls_handshake_tmo_work; + unsigned long poll_end; spinlock_t state_lock; @@ -918,6 +939,7 @@ static int nvmet_tcp_handle_icreq(struct nvmet_tcp_queue *queue) free_crypto: if (queue->hdr_digest || queue->data_digest) nvmet_tcp_free_crypto(queue); + queue->state = NVMET_TCP_Q_FAILED; return ret; } @@ -1283,12 +1305,25 @@ done: return ret; } +static void nvmet_tcp_release_queue(struct kref *kref) +{ + struct nvmet_tcp_queue *queue = + container_of(kref, struct nvmet_tcp_queue, kref); + + WARN_ON(queue->state != NVMET_TCP_Q_DISCONNECTING); + queue_work(nvmet_wq, &queue->release_work); +} + static void nvmet_tcp_schedule_release_queue(struct nvmet_tcp_queue *queue) { spin_lock(&queue->state_lock); + if (queue->state == NVMET_TCP_Q_TLS_HANDSHAKE) { + /* Socket closed during handshake */ + tls_handshake_cancel(queue->sock->sk); + } if (queue->state != NVMET_TCP_Q_DISCONNECTING) { queue->state = NVMET_TCP_Q_DISCONNECTING; - queue_work(nvmet_wq, &queue->release_work); + kref_put(&queue->kref, nvmet_tcp_release_queue); } spin_unlock(&queue->state_lock); } @@ -1485,6 +1520,7 @@ static void nvmet_tcp_release_queue_work(struct work_struct *w) mutex_unlock(&nvmet_tcp_queue_mutex); nvmet_tcp_restore_socket_callbacks(queue); + cancel_delayed_work_sync(&queue->tls_handshake_tmo_work); cancel_work_sync(&queue->io_work); /* stop accepting incoming data */ queue->rcv_state = NVMET_TCP_RECV_ERR; @@ -1512,8 +1548,13 @@ static void nvmet_tcp_data_ready(struct sock *sk) read_lock_bh(&sk->sk_callback_lock); queue = sk->sk_user_data; - if (likely(queue)) - queue_work_on(queue_cpu(queue), nvmet_tcp_wq, &queue->io_work); + if (likely(queue)) { + if (queue->data_ready) + queue->data_ready(sk); + if (queue->state != NVMET_TCP_Q_TLS_HANDSHAKE) + queue_work_on(queue_cpu(queue), nvmet_tcp_wq, + &queue->io_work); + } read_unlock_bh(&sk->sk_callback_lock); } @@ -1621,6 +1662,87 @@ static int nvmet_tcp_set_queue_sock(struct nvmet_tcp_queue *queue) return ret; } +#ifdef CONFIG_NVME_TARGET_TCP_TLS +static void nvmet_tcp_tls_handshake_done(void *data, int status, + key_serial_t peerid) +{ + struct nvmet_tcp_queue *queue = data; + + pr_debug("queue %d: TLS handshake done, key %x, status %d\n", + queue->idx, peerid, status); + spin_lock_bh(&queue->state_lock); + if (WARN_ON(queue->state != NVMET_TCP_Q_TLS_HANDSHAKE)) { + spin_unlock_bh(&queue->state_lock); + return; + } + if (!status) { + queue->tls_pskid = peerid; + queue->state = NVMET_TCP_Q_CONNECTING; + } else + queue->state = NVMET_TCP_Q_FAILED; + spin_unlock_bh(&queue->state_lock); + + cancel_delayed_work_sync(&queue->tls_handshake_tmo_work); + if (status) + nvmet_tcp_schedule_release_queue(queue); + else + nvmet_tcp_set_queue_sock(queue); + kref_put(&queue->kref, nvmet_tcp_release_queue); +} + +static void nvmet_tcp_tls_handshake_timeout(struct work_struct *w) +{ + struct nvmet_tcp_queue *queue = container_of(to_delayed_work(w), + struct nvmet_tcp_queue, tls_handshake_tmo_work); + + pr_warn("queue %d: TLS handshake timeout\n", queue->idx); + /* + * If tls_handshake_cancel() fails we've lost the race with + * nvmet_tcp_tls_handshake_done() */ + if (!tls_handshake_cancel(queue->sock->sk)) + return; + spin_lock_bh(&queue->state_lock); + if (WARN_ON(queue->state != NVMET_TCP_Q_TLS_HANDSHAKE)) { + spin_unlock_bh(&queue->state_lock); + return; + } + queue->state = NVMET_TCP_Q_FAILED; + spin_unlock_bh(&queue->state_lock); + nvmet_tcp_schedule_release_queue(queue); + kref_put(&queue->kref, nvmet_tcp_release_queue); +} + +static int nvmet_tcp_tls_handshake(struct nvmet_tcp_queue *queue) +{ + int ret = -EOPNOTSUPP; + struct tls_handshake_args args; + + if (queue->state != NVMET_TCP_Q_TLS_HANDSHAKE) { + pr_warn("cannot start TLS in state %d\n", queue->state); + return -EINVAL; + } + + kref_get(&queue->kref); + pr_debug("queue %d: TLS ServerHello\n", queue->idx); + memset(&args, 0, sizeof(args)); + args.ta_sock = queue->sock; + args.ta_done = nvmet_tcp_tls_handshake_done; + args.ta_data = queue; + args.ta_keyring = key_serial(queue->port->nport->keyring); + args.ta_timeout_ms = tls_handshake_timeout * 1000; + + ret = tls_server_hello_psk(&args, GFP_KERNEL); + if (ret) { + kref_put(&queue->kref, nvmet_tcp_release_queue); + pr_err("failed to start TLS, err=%d\n", ret); + } else { + queue_delayed_work(nvmet_wq, &queue->tls_handshake_tmo_work, + tls_handshake_timeout * HZ); + } + return ret; +} +#endif + static void nvmet_tcp_alloc_queue(struct nvmet_tcp_port *port, struct socket *newsock) { @@ -1636,11 +1758,16 @@ static void nvmet_tcp_alloc_queue(struct nvmet_tcp_port *port, INIT_WORK(&queue->release_work, nvmet_tcp_release_queue_work); INIT_WORK(&queue->io_work, nvmet_tcp_io_work); + kref_init(&queue->kref); queue->sock = newsock; queue->port = port; queue->nr_cmds = 0; spin_lock_init(&queue->state_lock); - queue->state = NVMET_TCP_Q_CONNECTING; + if (queue->port->nport->disc_addr.tsas.tcp.sectype == + NVMF_TCP_SECTYPE_TLS13) + queue->state = NVMET_TCP_Q_TLS_HANDSHAKE; + else + queue->state = NVMET_TCP_Q_CONNECTING; INIT_LIST_HEAD(&queue->free_list); init_llist_head(&queue->resp_list); INIT_LIST_HEAD(&queue->resp_send_list); @@ -1671,6 +1798,25 @@ static void nvmet_tcp_alloc_queue(struct nvmet_tcp_port *port, list_add_tail(&queue->queue_list, &nvmet_tcp_queue_list); mutex_unlock(&nvmet_tcp_queue_mutex); +#ifdef CONFIG_NVME_TARGET_TCP_TLS + INIT_DELAYED_WORK(&queue->tls_handshake_tmo_work, + nvmet_tcp_tls_handshake_timeout); + if (queue->state == NVMET_TCP_Q_TLS_HANDSHAKE) { + struct sock *sk = queue->sock->sk; + + /* Restore the default callbacks before starting upcall */ + read_lock_bh(&sk->sk_callback_lock); + sk->sk_user_data = NULL; + sk->sk_data_ready = port->data_ready; + read_unlock_bh(&sk->sk_callback_lock); + if (!nvmet_tcp_tls_handshake(queue)) + return; + + /* TLS handshake failed, terminate the connection */ + goto out_destroy_sq; + } +#endif + ret = nvmet_tcp_set_queue_sock(queue); if (ret) goto out_destroy_sq; -- cgit From a1c5dd8355b172b44a4ff8248fb487006236a918 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Thu, 24 Aug 2023 16:39:24 +0200 Subject: nvmet-tcp: control messages for recvmsg() kTLS requires control messages for recvmsg() to relay any out-of-band TLS messages (eg TLS alerts) to the caller. Signed-off-by: Hannes Reinecke Reviewed-by: Sagi Grimberg Signed-off-by: Keith Busch --- drivers/nvme/target/tcp.c | 93 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 80 insertions(+), 13 deletions(-) diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c index 58a4a737be77..c9bca7d61605 100644 --- a/drivers/nvme/target/tcp.c +++ b/drivers/nvme/target/tcp.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -118,6 +119,7 @@ struct nvmet_tcp_cmd { u32 pdu_len; u32 pdu_recv; int sg_idx; + char recv_cbuf[CMSG_LEN(sizeof(char))]; struct msghdr recv_msg; struct bio_vec *iov; u32 flags; @@ -1121,20 +1123,65 @@ static inline bool nvmet_tcp_pdu_valid(u8 type) return false; } +static int nvmet_tcp_tls_record_ok(struct nvmet_tcp_queue *queue, + struct msghdr *msg, char *cbuf) +{ + struct cmsghdr *cmsg = (struct cmsghdr *)cbuf; + u8 ctype, level, description; + int ret = 0; + + ctype = tls_get_record_type(queue->sock->sk, cmsg); + switch (ctype) { + case 0: + break; + case TLS_RECORD_TYPE_DATA: + break; + case TLS_RECORD_TYPE_ALERT: + tls_alert_recv(queue->sock->sk, msg, &level, &description); + if (level == TLS_ALERT_LEVEL_FATAL) { + pr_err("queue %d: TLS Alert desc %u\n", + queue->idx, description); + ret = -ENOTCONN; + } else { + pr_warn("queue %d: TLS Alert desc %u\n", + queue->idx, description); + ret = -EAGAIN; + } + break; + default: + /* discard this record type */ + pr_err("queue %d: TLS record %d unhandled\n", + queue->idx, ctype); + ret = -EAGAIN; + break; + } + return ret; +} + static int nvmet_tcp_try_recv_pdu(struct nvmet_tcp_queue *queue) { struct nvme_tcp_hdr *hdr = &queue->pdu.cmd.hdr; - int len; + int len, ret; struct kvec iov; + char cbuf[CMSG_LEN(sizeof(char))] = {}; struct msghdr msg = { .msg_flags = MSG_DONTWAIT }; recv: iov.iov_base = (void *)&queue->pdu + queue->offset; iov.iov_len = queue->left; + if (queue->tls_pskid) { + msg.msg_control = cbuf; + msg.msg_controllen = sizeof(cbuf); + } len = kernel_recvmsg(queue->sock, &msg, &iov, 1, iov.iov_len, msg.msg_flags); if (unlikely(len < 0)) return len; + if (queue->tls_pskid) { + ret = nvmet_tcp_tls_record_ok(queue, &msg, cbuf); + if (ret < 0) + return ret; + } queue->offset += len; queue->left -= len; @@ -1187,16 +1234,22 @@ static void nvmet_tcp_prep_recv_ddgst(struct nvmet_tcp_cmd *cmd) static int nvmet_tcp_try_recv_data(struct nvmet_tcp_queue *queue) { struct nvmet_tcp_cmd *cmd = queue->cmd; - int ret; + int len, ret; while (msg_data_left(&cmd->recv_msg)) { - ret = sock_recvmsg(cmd->queue->sock, &cmd->recv_msg, + len = sock_recvmsg(cmd->queue->sock, &cmd->recv_msg, cmd->recv_msg.msg_flags); - if (ret <= 0) - return ret; + if (len <= 0) + return len; + if (queue->tls_pskid) { + ret = nvmet_tcp_tls_record_ok(cmd->queue, + &cmd->recv_msg, cmd->recv_cbuf); + if (ret < 0) + return ret; + } - cmd->pdu_recv += ret; - cmd->rbytes_done += ret; + cmd->pdu_recv += len; + cmd->rbytes_done += len; } if (queue->data_digest) { @@ -1214,20 +1267,30 @@ static int nvmet_tcp_try_recv_data(struct nvmet_tcp_queue *queue) static int nvmet_tcp_try_recv_ddgst(struct nvmet_tcp_queue *queue) { struct nvmet_tcp_cmd *cmd = queue->cmd; - int ret; + int ret, len; + char cbuf[CMSG_LEN(sizeof(char))] = {}; struct msghdr msg = { .msg_flags = MSG_DONTWAIT }; struct kvec iov = { .iov_base = (void *)&cmd->recv_ddgst + queue->offset, .iov_len = queue->left }; - ret = kernel_recvmsg(queue->sock, &msg, &iov, 1, + if (queue->tls_pskid) { + msg.msg_control = cbuf; + msg.msg_controllen = sizeof(cbuf); + } + len = kernel_recvmsg(queue->sock, &msg, &iov, 1, iov.iov_len, msg.msg_flags); - if (unlikely(ret < 0)) - return ret; + if (unlikely(len < 0)) + return len; + if (queue->tls_pskid) { + ret = nvmet_tcp_tls_record_ok(queue, &msg, cbuf); + if (ret < 0) + return ret; + } - queue->offset += ret; - queue->left -= ret; + queue->offset += len; + queue->left -= len; if (queue->left) return -EAGAIN; @@ -1407,6 +1470,10 @@ static int nvmet_tcp_alloc_cmd(struct nvmet_tcp_queue *queue, if (!c->r2t_pdu) goto out_free_data; + if (queue->state == NVMET_TCP_Q_TLS_HANDSHAKE) { + c->recv_msg.msg_control = c->recv_cbuf; + c->recv_msg.msg_controllen = sizeof(c->recv_cbuf); + } c->recv_msg.msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL; list_add_tail(&c->entry, &queue->free_list); -- cgit From 70525e5d82f62407d8436aa2a6c3385829ab5a3e Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Thu, 24 Aug 2023 16:39:25 +0200 Subject: nvmet-tcp: peek icreq before starting TLS Incoming connection might be either 'normal' NVMe-TCP connections starting with icreq or TLS handshakes. To ensure that 'normal' connections can still be handled we need to peek the first packet and only start TLS handshake if it's not an icreq. With that we can lift the restriction to always set TREQ to 'required' when TLS1.3 is enabled. Signed-off-by: Hannes Reinecke Signed-off-by: Keith Busch --- drivers/nvme/target/configfs.c | 25 ++++++++++++++--- drivers/nvme/target/nvmet.h | 5 ++++ drivers/nvme/target/tcp.c | 61 ++++++++++++++++++++++++++++++++++++++---- 3 files changed, 82 insertions(+), 9 deletions(-) diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c index b780ce049163..9eed6e6765ea 100644 --- a/drivers/nvme/target/configfs.c +++ b/drivers/nvme/target/configfs.c @@ -198,6 +198,20 @@ static ssize_t nvmet_addr_treq_store(struct config_item *item, return -EINVAL; found: + if (port->disc_addr.trtype == NVMF_TRTYPE_TCP && + port->disc_addr.tsas.tcp.sectype == NVMF_TCP_SECTYPE_TLS13) { + switch (nvmet_addr_treq[i].type) { + case NVMF_TREQ_NOT_SPECIFIED: + pr_debug("treq '%s' not allowed for TLS1.3\n", + nvmet_addr_treq[i].name); + return -EINVAL; + case NVMF_TREQ_NOT_REQUIRED: + pr_warn("Allow non-TLS connections while TLS1.3 is enabled\n"); + break; + default: + break; + } + } treq |= nvmet_addr_treq[i].type; port->disc_addr.treq = treq; return count; @@ -410,12 +424,15 @@ found: nvmet_port_init_tsas_tcp(port, sectype); /* - * The TLS implementation currently does not support - * secure concatenation, so TREQ is always set to 'required' - * if TLS is enabled. + * If TLS is enabled TREQ should be set to 'required' per default */ if (sectype == NVMF_TCP_SECTYPE_TLS13) { - treq |= NVMF_TREQ_REQUIRED; + u8 sc = nvmet_port_disc_addr_treq_secure_channel(port); + + if (sc == NVMF_TREQ_NOT_SPECIFIED) + treq |= NVMF_TREQ_REQUIRED; + else + treq |= sc; } else { treq |= NVMF_TREQ_NOT_SPECIFIED; } diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h index e35a03260f45..3e179019ca7c 100644 --- a/drivers/nvme/target/nvmet.h +++ b/drivers/nvme/target/nvmet.h @@ -184,6 +184,11 @@ static inline u8 nvmet_port_disc_addr_treq_secure_channel(struct nvmet_port *por return (port->disc_addr.treq & NVME_TREQ_SECURE_CHANNEL_MASK); } +static inline bool nvmet_port_secure_channel_required(struct nvmet_port *port) +{ + return nvmet_port_disc_addr_treq_secure_channel(port) == NVMF_TREQ_REQUIRED; +} + struct nvmet_ctrl { struct nvmet_subsys *subsys; struct nvmet_sq **sqs; diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c index c9bca7d61605..0dc3b75de7b6 100644 --- a/drivers/nvme/target/tcp.c +++ b/drivers/nvme/target/tcp.c @@ -1730,6 +1730,54 @@ static int nvmet_tcp_set_queue_sock(struct nvmet_tcp_queue *queue) } #ifdef CONFIG_NVME_TARGET_TCP_TLS +static int nvmet_tcp_try_peek_pdu(struct nvmet_tcp_queue *queue) +{ + struct nvme_tcp_hdr *hdr = &queue->pdu.cmd.hdr; + int len, ret; + struct kvec iov = { + .iov_base = (u8 *)&queue->pdu + queue->offset, + .iov_len = sizeof(struct nvme_tcp_hdr), + }; + char cbuf[CMSG_LEN(sizeof(char))] = {}; + struct msghdr msg = { + .msg_control = cbuf, + .msg_controllen = sizeof(cbuf), + .msg_flags = MSG_PEEK, + }; + + if (nvmet_port_secure_channel_required(queue->port->nport)) + return 0; + + len = kernel_recvmsg(queue->sock, &msg, &iov, 1, + iov.iov_len, msg.msg_flags); + if (unlikely(len < 0)) { + pr_debug("queue %d: peek error %d\n", + queue->idx, len); + return len; + } + + ret = nvmet_tcp_tls_record_ok(queue, &msg, cbuf); + if (ret < 0) + return ret; + + if (len < sizeof(struct nvme_tcp_hdr)) { + pr_debug("queue %d: short read, %d bytes missing\n", + queue->idx, (int)iov.iov_len - len); + return -EAGAIN; + } + pr_debug("queue %d: hdr type %d hlen %d plen %d size %d\n", + queue->idx, hdr->type, hdr->hlen, hdr->plen, + (int)sizeof(struct nvme_tcp_icreq_pdu)); + if (hdr->type == nvme_tcp_icreq && + hdr->hlen == sizeof(struct nvme_tcp_icreq_pdu) && + hdr->plen == (__le32)sizeof(struct nvme_tcp_icreq_pdu)) { + pr_debug("queue %d: icreq detected\n", + queue->idx); + return len; + } + return 0; +} + static void nvmet_tcp_tls_handshake_done(void *data, int status, key_serial_t peerid) { @@ -1876,11 +1924,14 @@ static void nvmet_tcp_alloc_queue(struct nvmet_tcp_port *port, sk->sk_user_data = NULL; sk->sk_data_ready = port->data_ready; read_unlock_bh(&sk->sk_callback_lock); - if (!nvmet_tcp_tls_handshake(queue)) - return; - - /* TLS handshake failed, terminate the connection */ - goto out_destroy_sq; + if (!nvmet_tcp_try_peek_pdu(queue)) { + if (!nvmet_tcp_tls_handshake(queue)) + return; + /* TLS handshake failed, terminate the connection */ + goto out_destroy_sq; + } + /* Not a TLS connection, continue with normal processing */ + queue->state = NVMET_TCP_Q_CONNECTING; } #endif -- cgit From d680063482885c15d68e958212c3d6ad40a510dd Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Thu, 12 Oct 2023 14:22:48 +0200 Subject: nvme: rework NVME_AUTH Kconfig selection Having a single Kconfig symbol NVME_AUTH conflates the selection of the authentication functions from nvme/common and nvme/host, causing kbuild robot to complain when building the nvme target only. So introduce a Kconfig symbol NVME_HOST_AUTH for the nvme host bits and use NVME_AUTH for the common functions only. And move the CRYPTO selection into nvme/common to make it easier to read. Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202310120733.TlPOVeJm-lkp@intel.com/ Signed-off-by: Hannes Reinecke Signed-off-by: Keith Busch --- drivers/nvme/common/Kconfig | 9 +++++++++ drivers/nvme/host/Kconfig | 9 ++------- drivers/nvme/host/Makefile | 2 +- drivers/nvme/host/core.c | 2 +- drivers/nvme/host/nvme.h | 4 ++-- drivers/nvme/host/sysfs.c | 6 +++--- drivers/nvme/target/Kconfig | 7 +------ 7 files changed, 19 insertions(+), 20 deletions(-) diff --git a/drivers/nvme/common/Kconfig b/drivers/nvme/common/Kconfig index 641b27adb047..06c8df00d1e2 100644 --- a/drivers/nvme/common/Kconfig +++ b/drivers/nvme/common/Kconfig @@ -6,3 +6,12 @@ config NVME_COMMON config NVME_KEYRING bool select KEYS + +config NVME_AUTH + bool + select CRYPTO + select CRYPTO_HMAC + select CRYPTO_SHA256 + select CRYPTO_SHA512 + select CRYPTO_DH + select CRYPTO_DH_RFC7919_GROUPS diff --git a/drivers/nvme/host/Kconfig b/drivers/nvme/host/Kconfig index a517030d7d50..48f7d72de5e9 100644 --- a/drivers/nvme/host/Kconfig +++ b/drivers/nvme/host/Kconfig @@ -107,16 +107,11 @@ config NVME_TCP_TLS If unsure, say N. -config NVME_AUTH +config NVME_HOST_AUTH bool "NVM Express over Fabrics In-Band Authentication" depends on NVME_CORE select NVME_COMMON - select CRYPTO - select CRYPTO_HMAC - select CRYPTO_SHA256 - select CRYPTO_SHA512 - select CRYPTO_DH - select CRYPTO_DH_RFC7919_GROUPS + select NVME_AUTH help This provides support for NVMe over Fabrics In-Band Authentication. diff --git a/drivers/nvme/host/Makefile b/drivers/nvme/host/Makefile index c7c3cf202d12..6414ec968f99 100644 --- a/drivers/nvme/host/Makefile +++ b/drivers/nvme/host/Makefile @@ -17,7 +17,7 @@ nvme-core-$(CONFIG_NVME_MULTIPATH) += multipath.o nvme-core-$(CONFIG_BLK_DEV_ZONED) += zns.o nvme-core-$(CONFIG_FAULT_INJECTION_DEBUG_FS) += fault_inject.o nvme-core-$(CONFIG_NVME_HWMON) += hwmon.o -nvme-core-$(CONFIG_NVME_AUTH) += auth.o +nvme-core-$(CONFIG_NVME_HOST_AUTH) += auth.o nvme-y += pci.o diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 55e54ffb1003..62612f87aafa 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -421,7 +421,7 @@ void nvme_complete_rq(struct request *req) nvme_failover_req(req); return; case AUTHENTICATE: -#ifdef CONFIG_NVME_AUTH +#ifdef CONFIG_NVME_HOST_AUTH queue_work(nvme_wq, &ctrl->dhchap_auth_work); nvme_retry_req(req); #else diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index 6fe7966f720b..39a90b7cb125 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -349,7 +349,7 @@ struct nvme_ctrl { struct work_struct ana_work; #endif -#ifdef CONFIG_NVME_AUTH +#ifdef CONFIG_NVME_HOST_AUTH struct work_struct dhchap_auth_work; struct mutex dhchap_auth_mutex; struct nvme_dhchap_queue_context *dhchap_ctxs; @@ -1049,7 +1049,7 @@ static inline bool nvme_ctrl_sgl_supported(struct nvme_ctrl *ctrl) return ctrl->sgls & ((1 << 0) | (1 << 1)); } -#ifdef CONFIG_NVME_AUTH +#ifdef CONFIG_NVME_HOST_AUTH int __init nvme_init_auth(void); void __exit nvme_exit_auth(void); int nvme_auth_init_ctrl(struct nvme_ctrl *ctrl); diff --git a/drivers/nvme/host/sysfs.c b/drivers/nvme/host/sysfs.c index d0966159981c..c6b7fbd4d34d 100644 --- a/drivers/nvme/host/sysfs.c +++ b/drivers/nvme/host/sysfs.c @@ -409,7 +409,7 @@ static ssize_t dctype_show(struct device *dev, } static DEVICE_ATTR_RO(dctype); -#ifdef CONFIG_NVME_AUTH +#ifdef CONFIG_NVME_HOST_AUTH static ssize_t nvme_ctrl_dhchap_secret_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -563,7 +563,7 @@ static struct attribute *nvme_dev_attrs[] = { &dev_attr_kato.attr, &dev_attr_cntrltype.attr, &dev_attr_dctype.attr, -#ifdef CONFIG_NVME_AUTH +#ifdef CONFIG_NVME_HOST_AUTH &dev_attr_dhchap_secret.attr, &dev_attr_dhchap_ctrl_secret.attr, #endif @@ -593,7 +593,7 @@ static umode_t nvme_dev_attrs_are_visible(struct kobject *kobj, return 0; if (a == &dev_attr_fast_io_fail_tmo.attr && !ctrl->opts) return 0; -#ifdef CONFIG_NVME_AUTH +#ifdef CONFIG_NVME_HOST_AUTH if (a == &dev_attr_dhchap_secret.attr && !ctrl->opts) return 0; if (a == &dev_attr_dhchap_ctrl_secret.attr && !ctrl->opts) diff --git a/drivers/nvme/target/Kconfig b/drivers/nvme/target/Kconfig index c56cb1005327..fa479c9f5c3d 100644 --- a/drivers/nvme/target/Kconfig +++ b/drivers/nvme/target/Kconfig @@ -103,12 +103,7 @@ config NVME_TARGET_AUTH bool "NVMe over Fabrics In-band Authentication support" depends on NVME_TARGET select NVME_COMMON - select CRYPTO - select CRYPTO_HMAC - select CRYPTO_SHA256 - select CRYPTO_SHA512 - select CRYPTO_DH - select CRYPTO_DH_RFC7919_GROUPS + select NVME_AUTH help This enables support for NVMe over Fabrics In-band Authentication -- cgit From bbacf79201a1664f174e341f4bc0205b25b309d7 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Thu, 12 Oct 2023 14:59:54 +0200 Subject: nvmet-tcp: use 'spin_lock_bh' for state_lock() nvmet_tcp_schedule_release_queue() is called from socket state change callbacks, which may be called from an softirq context. So use 'spin_lock_bh' to avoid a spin lock warning. Signed-off-by: Hannes Reinecke Signed-off-by: Keith Busch --- drivers/nvme/target/tcp.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c index 0dc3b75de7b6..4336fe048e43 100644 --- a/drivers/nvme/target/tcp.c +++ b/drivers/nvme/target/tcp.c @@ -1379,7 +1379,7 @@ static void nvmet_tcp_release_queue(struct kref *kref) static void nvmet_tcp_schedule_release_queue(struct nvmet_tcp_queue *queue) { - spin_lock(&queue->state_lock); + spin_lock_bh(&queue->state_lock); if (queue->state == NVMET_TCP_Q_TLS_HANDSHAKE) { /* Socket closed during handshake */ tls_handshake_cancel(queue->sock->sk); @@ -1388,7 +1388,7 @@ static void nvmet_tcp_schedule_release_queue(struct nvmet_tcp_queue *queue) queue->state = NVMET_TCP_Q_DISCONNECTING; kref_put(&queue->kref, nvmet_tcp_release_queue); } - spin_unlock(&queue->state_lock); + spin_unlock_bh(&queue->state_lock); } static inline void nvmet_tcp_arm_queue_deadline(struct nvmet_tcp_queue *queue) -- cgit From 3ebed3749f1767927229d568eea29daaf9f272d5 Mon Sep 17 00:00:00 2001 From: Mark O'Donovan Date: Tue, 17 Oct 2023 17:09:17 +0000 Subject: nvme-auth: alloc nvme_dhchap_key as single buffer Co-developed-by: Akash Appaiah Signed-off-by: Akash Appaiah Signed-off-by: Mark O'Donovan Reviewed-by: Hannes Reinecke Signed-off-by: Keith Busch --- drivers/nvme/common/auth.c | 37 +++++++++++++++++++++++++------------ include/linux/nvme-auth.h | 4 +++- 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/drivers/nvme/common/auth.c b/drivers/nvme/common/auth.c index d90e4f0c08b7..16a071448d54 100644 --- a/drivers/nvme/common/auth.c +++ b/drivers/nvme/common/auth.c @@ -150,6 +150,14 @@ size_t nvme_auth_hmac_hash_len(u8 hmac_id) } EXPORT_SYMBOL_GPL(nvme_auth_hmac_hash_len); +u32 nvme_auth_key_struct_size(u32 key_len) +{ + struct nvme_dhchap_key key; + + return struct_size(&key, key, key_len); +} +EXPORT_SYMBOL_GPL(nvme_auth_key_struct_size); + struct nvme_dhchap_key *nvme_auth_extract_key(unsigned char *secret, u8 key_hash) { @@ -163,14 +171,9 @@ struct nvme_dhchap_key *nvme_auth_extract_key(unsigned char *secret, p = strrchr(secret, ':'); if (p) allocated_len = p - secret; - key = kzalloc(sizeof(*key), GFP_KERNEL); + key = nvme_auth_alloc_key(allocated_len, 0); if (!key) return ERR_PTR(-ENOMEM); - key->key = kzalloc(allocated_len, GFP_KERNEL); - if (!key->key) { - ret = -ENOMEM; - goto out_free_key; - } key_len = base64_decode(secret, allocated_len, key->key); if (key_len < 0) { @@ -213,19 +216,29 @@ struct nvme_dhchap_key *nvme_auth_extract_key(unsigned char *secret, key->hash = key_hash; return key; out_free_secret: - kfree_sensitive(key->key); -out_free_key: - kfree(key); + nvme_auth_free_key(key); return ERR_PTR(ret); } EXPORT_SYMBOL_GPL(nvme_auth_extract_key); +struct nvme_dhchap_key *nvme_auth_alloc_key(u32 len, u8 hash) +{ + u32 num_bytes = nvme_auth_key_struct_size(len); + struct nvme_dhchap_key *key = kzalloc(num_bytes, GFP_KERNEL); + + if (key) { + key->len = len; + key->hash = hash; + } + return key; +} +EXPORT_SYMBOL_GPL(nvme_auth_alloc_key); + void nvme_auth_free_key(struct nvme_dhchap_key *key) { if (!key) return; - kfree_sensitive(key->key); - kfree(key); + kfree_sensitive(key); } EXPORT_SYMBOL_GPL(nvme_auth_free_key); @@ -237,7 +250,7 @@ u8 *nvme_auth_transform_key(struct nvme_dhchap_key *key, char *nqn) u8 *transformed_key; int ret; - if (!key || !key->key) { + if (!key) { pr_warn("No key specified\n"); return ERR_PTR(-ENOKEY); } diff --git a/include/linux/nvme-auth.h b/include/linux/nvme-auth.h index dcb8030062dd..a5ae9abe1ef6 100644 --- a/include/linux/nvme-auth.h +++ b/include/linux/nvme-auth.h @@ -9,9 +9,9 @@ #include struct nvme_dhchap_key { - u8 *key; size_t len; u8 hash; + u8 key[]; }; u32 nvme_auth_get_seqnum(void); @@ -24,9 +24,11 @@ const char *nvme_auth_digest_name(u8 hmac_id); size_t nvme_auth_hmac_hash_len(u8 hmac_id); u8 nvme_auth_hmac_id(const char *hmac_name); +u32 nvme_auth_key_struct_size(u32 key_len); struct nvme_dhchap_key *nvme_auth_extract_key(unsigned char *secret, u8 key_hash); void nvme_auth_free_key(struct nvme_dhchap_key *key); +struct nvme_dhchap_key *nvme_auth_alloc_key(u32 len, u8 hash); u8 *nvme_auth_transform_key(struct nvme_dhchap_key *key, char *nqn); int nvme_auth_generate_key(u8 *secret, struct nvme_dhchap_key **ret_key); int nvme_auth_augmented_challenge(u8 hmac_id, u8 *skey, size_t skey_len, -- cgit From f047daed179a451657d1e66b5fe4030a593a000c Mon Sep 17 00:00:00 2001 From: Mark O'Donovan Date: Tue, 17 Oct 2023 17:09:18 +0000 Subject: nvme-auth: use transformed key size to create resp This does not change current behaviour as the driver currently verifies that the secret size is the same size as the length of the transformation hash. Co-developed-by: Akash Appaiah Signed-off-by: Akash Appaiah Signed-off-by: Mark O'Donovan Reviewed-by: Hannes Reinecke Signed-off-by: Keith Busch --- drivers/nvme/common/auth.c | 23 ++++++++++++++--------- drivers/nvme/host/auth.c | 30 +++++++++++++++--------------- drivers/nvme/target/auth.c | 31 +++++++++++++++++-------------- include/linux/nvme-auth.h | 3 ++- 4 files changed, 48 insertions(+), 39 deletions(-) diff --git a/drivers/nvme/common/auth.c b/drivers/nvme/common/auth.c index 16a071448d54..f954aeb647a5 100644 --- a/drivers/nvme/common/auth.c +++ b/drivers/nvme/common/auth.c @@ -242,21 +242,25 @@ void nvme_auth_free_key(struct nvme_dhchap_key *key) } EXPORT_SYMBOL_GPL(nvme_auth_free_key); -u8 *nvme_auth_transform_key(struct nvme_dhchap_key *key, char *nqn) +struct nvme_dhchap_key *nvme_auth_transform_key( + struct nvme_dhchap_key *key, char *nqn) { const char *hmac_name; struct crypto_shash *key_tfm; struct shash_desc *shash; - u8 *transformed_key; - int ret; + struct nvme_dhchap_key *transformed_key; + int ret, key_len; if (!key) { pr_warn("No key specified\n"); return ERR_PTR(-ENOKEY); } if (key->hash == 0) { - transformed_key = kmemdup(key->key, key->len, GFP_KERNEL); - return transformed_key ? transformed_key : ERR_PTR(-ENOMEM); + key_len = nvme_auth_key_struct_size(key->len); + transformed_key = kmemdup(key, key_len, GFP_KERNEL); + if (!transformed_key) + return ERR_PTR(-ENOMEM); + return transformed_key; } hmac_name = nvme_auth_hmac_name(key->hash); if (!hmac_name) { @@ -266,7 +270,7 @@ u8 *nvme_auth_transform_key(struct nvme_dhchap_key *key, char *nqn) key_tfm = crypto_alloc_shash(hmac_name, 0, 0); if (IS_ERR(key_tfm)) - return (u8 *)key_tfm; + return ERR_CAST(key_tfm); shash = kmalloc(sizeof(struct shash_desc) + crypto_shash_descsize(key_tfm), @@ -276,7 +280,8 @@ u8 *nvme_auth_transform_key(struct nvme_dhchap_key *key, char *nqn) goto out_free_key; } - transformed_key = kzalloc(crypto_shash_digestsize(key_tfm), GFP_KERNEL); + key_len = crypto_shash_digestsize(key_tfm); + transformed_key = nvme_auth_alloc_key(key_len, key->hash); if (!transformed_key) { ret = -ENOMEM; goto out_free_shash; @@ -295,7 +300,7 @@ u8 *nvme_auth_transform_key(struct nvme_dhchap_key *key, char *nqn) ret = crypto_shash_update(shash, "NVMe-over-Fabrics", 17); if (ret < 0) goto out_free_transformed_key; - ret = crypto_shash_final(shash, transformed_key); + ret = crypto_shash_final(shash, transformed_key->key); if (ret < 0) goto out_free_transformed_key; @@ -305,7 +310,7 @@ u8 *nvme_auth_transform_key(struct nvme_dhchap_key *key, char *nqn) return transformed_key; out_free_transformed_key: - kfree_sensitive(transformed_key); + nvme_auth_free_key(transformed_key); out_free_shash: kfree(shash); out_free_key: diff --git a/drivers/nvme/host/auth.c b/drivers/nvme/host/auth.c index daf5d144a8ea..de1390d705dc 100644 --- a/drivers/nvme/host/auth.c +++ b/drivers/nvme/host/auth.c @@ -23,6 +23,7 @@ struct nvme_dhchap_queue_context { struct nvme_ctrl *ctrl; struct crypto_shash *shash_tfm; struct crypto_kpp *dh_tfm; + struct nvme_dhchap_key *transformed_key; void *buf; int qid; int error; @@ -36,7 +37,6 @@ struct nvme_dhchap_queue_context { u8 c1[64]; u8 c2[64]; u8 response[64]; - u8 *host_response; u8 *ctrl_key; u8 *host_key; u8 *sess_key; @@ -428,12 +428,12 @@ static int nvme_auth_dhchap_setup_host_response(struct nvme_ctrl *ctrl, dev_dbg(ctrl->device, "%s: qid %d host response seq %u transaction %d\n", __func__, chap->qid, chap->s1, chap->transaction); - if (!chap->host_response) { - chap->host_response = nvme_auth_transform_key(ctrl->host_key, + if (!chap->transformed_key) { + chap->transformed_key = nvme_auth_transform_key(ctrl->host_key, ctrl->opts->host->nqn); - if (IS_ERR(chap->host_response)) { - ret = PTR_ERR(chap->host_response); - chap->host_response = NULL; + if (IS_ERR(chap->transformed_key)) { + ret = PTR_ERR(chap->transformed_key); + chap->transformed_key = NULL; return ret; } } else { @@ -442,7 +442,7 @@ static int nvme_auth_dhchap_setup_host_response(struct nvme_ctrl *ctrl, } ret = crypto_shash_setkey(chap->shash_tfm, - chap->host_response, ctrl->host_key->len); + chap->transformed_key->key, chap->transformed_key->len); if (ret) { dev_warn(ctrl->device, "qid %d: failed to set key, error %d\n", chap->qid, ret); @@ -508,19 +508,19 @@ static int nvme_auth_dhchap_setup_ctrl_response(struct nvme_ctrl *ctrl, struct nvme_dhchap_queue_context *chap) { SHASH_DESC_ON_STACK(shash, chap->shash_tfm); - u8 *ctrl_response; + struct nvme_dhchap_key *transformed_key; u8 buf[4], *challenge = chap->c2; int ret; - ctrl_response = nvme_auth_transform_key(ctrl->ctrl_key, + transformed_key = nvme_auth_transform_key(ctrl->ctrl_key, ctrl->opts->subsysnqn); - if (IS_ERR(ctrl_response)) { - ret = PTR_ERR(ctrl_response); + if (IS_ERR(transformed_key)) { + ret = PTR_ERR(transformed_key); return ret; } ret = crypto_shash_setkey(chap->shash_tfm, - ctrl_response, ctrl->ctrl_key->len); + transformed_key->key, transformed_key->len); if (ret) { dev_warn(ctrl->device, "qid %d: failed to set key, error %d\n", chap->qid, ret); @@ -586,7 +586,7 @@ static int nvme_auth_dhchap_setup_ctrl_response(struct nvme_ctrl *ctrl, out: if (challenge != chap->c2) kfree(challenge); - kfree(ctrl_response); + nvme_auth_free_key(transformed_key); return ret; } @@ -648,8 +648,8 @@ gen_sesskey: static void nvme_auth_reset_dhchap(struct nvme_dhchap_queue_context *chap) { - kfree_sensitive(chap->host_response); - chap->host_response = NULL; + nvme_auth_free_key(chap->transformed_key); + chap->transformed_key = NULL; kfree_sensitive(chap->host_key); chap->host_key = NULL; chap->host_key_len = 0; diff --git a/drivers/nvme/target/auth.c b/drivers/nvme/target/auth.c index 4dcddcf95279..3ddbc3880cac 100644 --- a/drivers/nvme/target/auth.c +++ b/drivers/nvme/target/auth.c @@ -267,7 +267,8 @@ int nvmet_auth_host_hash(struct nvmet_req *req, u8 *response, struct shash_desc *shash; struct nvmet_ctrl *ctrl = req->sq->ctrl; const char *hash_name; - u8 *challenge = req->sq->dhchap_c1, *host_response; + u8 *challenge = req->sq->dhchap_c1; + struct nvme_dhchap_key *transformed_key; u8 buf[4]; int ret; @@ -291,14 +292,15 @@ int nvmet_auth_host_hash(struct nvmet_req *req, u8 *response, goto out_free_tfm; } - host_response = nvme_auth_transform_key(ctrl->host_key, ctrl->hostnqn); - if (IS_ERR(host_response)) { - ret = PTR_ERR(host_response); + transformed_key = nvme_auth_transform_key(ctrl->host_key, + ctrl->hostnqn); + if (IS_ERR(transformed_key)) { + ret = PTR_ERR(transformed_key); goto out_free_tfm; } - ret = crypto_shash_setkey(shash_tfm, host_response, - ctrl->host_key->len); + ret = crypto_shash_setkey(shash_tfm, transformed_key->key, + transformed_key->len); if (ret) goto out_free_response; @@ -365,7 +367,7 @@ out: kfree(challenge); kfree(shash); out_free_response: - kfree_sensitive(host_response); + nvme_auth_free_key(transformed_key); out_free_tfm: crypto_free_shash(shash_tfm); return 0; @@ -378,7 +380,8 @@ int nvmet_auth_ctrl_hash(struct nvmet_req *req, u8 *response, struct shash_desc *shash; struct nvmet_ctrl *ctrl = req->sq->ctrl; const char *hash_name; - u8 *challenge = req->sq->dhchap_c2, *ctrl_response; + u8 *challenge = req->sq->dhchap_c2; + struct nvme_dhchap_key *transformed_key; u8 buf[4]; int ret; @@ -402,15 +405,15 @@ int nvmet_auth_ctrl_hash(struct nvmet_req *req, u8 *response, goto out_free_tfm; } - ctrl_response = nvme_auth_transform_key(ctrl->ctrl_key, + transformed_key = nvme_auth_transform_key(ctrl->ctrl_key, ctrl->subsysnqn); - if (IS_ERR(ctrl_response)) { - ret = PTR_ERR(ctrl_response); + if (IS_ERR(transformed_key)) { + ret = PTR_ERR(transformed_key); goto out_free_tfm; } - ret = crypto_shash_setkey(shash_tfm, ctrl_response, - ctrl->ctrl_key->len); + ret = crypto_shash_setkey(shash_tfm, transformed_key->key, + transformed_key->len); if (ret) goto out_free_response; @@ -474,7 +477,7 @@ out: kfree(challenge); kfree(shash); out_free_response: - kfree_sensitive(ctrl_response); + nvme_auth_free_key(transformed_key); out_free_tfm: crypto_free_shash(shash_tfm); return 0; diff --git a/include/linux/nvme-auth.h b/include/linux/nvme-auth.h index a5ae9abe1ef6..c1d0bc5d9624 100644 --- a/include/linux/nvme-auth.h +++ b/include/linux/nvme-auth.h @@ -29,7 +29,8 @@ struct nvme_dhchap_key *nvme_auth_extract_key(unsigned char *secret, u8 key_hash); void nvme_auth_free_key(struct nvme_dhchap_key *key); struct nvme_dhchap_key *nvme_auth_alloc_key(u32 len, u8 hash); -u8 *nvme_auth_transform_key(struct nvme_dhchap_key *key, char *nqn); +struct nvme_dhchap_key *nvme_auth_transform_key( + struct nvme_dhchap_key *key, char *nqn); int nvme_auth_generate_key(u8 *secret, struct nvme_dhchap_key **ret_key); int nvme_auth_augmented_challenge(u8 hmac_id, u8 *skey, size_t skey_len, u8 *challenge, u8 *aug, size_t hlen); -- cgit From 32445526d836f827ea7e74ef933610b531d239df Mon Sep 17 00:00:00 2001 From: Mark O'Donovan Date: Tue, 17 Oct 2023 17:09:19 +0000 Subject: nvme-auth: allow mixing of secret and hash lengths We can now use any of the secret transformation hashes with a secret, regardless of the secret size. e.g. a 32 byte key with the SHA-512(64 byte) hash. The example secret from the spec should now be permitted with any of the following: DHHC-1:00:ia6zGodOr4SEG0Zzaw398rpY0wqipUWj4jWjUh4HWUz6aQ2n: DHHC-1:01:ia6zGodOr4SEG0Zzaw398rpY0wqipUWj4jWjUh4HWUz6aQ2n: DHHC-1:02:ia6zGodOr4SEG0Zzaw398rpY0wqipUWj4jWjUh4HWUz6aQ2n: DHHC-1:03:ia6zGodOr4SEG0Zzaw398rpY0wqipUWj4jWjUh4HWUz6aQ2n: Note: Secrets are still restricted to 32,48 or 64 bits. Co-developed-by: Akash Appaiah Signed-off-by: Akash Appaiah Signed-off-by: Mark O'Donovan Reviewed-by: Hannes Reinecke Signed-off-by: Keith Busch --- drivers/nvme/common/auth.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/drivers/nvme/common/auth.c b/drivers/nvme/common/auth.c index f954aeb647a5..a8e87dfbeab2 100644 --- a/drivers/nvme/common/auth.c +++ b/drivers/nvme/common/auth.c @@ -190,14 +190,6 @@ struct nvme_dhchap_key *nvme_auth_extract_key(unsigned char *secret, goto out_free_secret; } - if (key_hash > 0 && - (key_len - 4) != nvme_auth_hmac_hash_len(key_hash)) { - pr_err("Mismatched key len %d for %s\n", key_len, - nvme_auth_hmac_name(key_hash)); - ret = -EINVAL; - goto out_free_secret; - } - /* The last four bytes is the CRC in little-endian format */ key_len -= 4; /* -- cgit