From c2a90eee29f41630225c9a64d26c425e1d50b401 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 8 Jul 2024 16:53:25 -0700 Subject: scsi: ufs: core: Add UFSHCD_QUIRK_CUSTOM_CRYPTO_PROFILE Add UFSHCD_QUIRK_CUSTOM_CRYPTO_PROFILE which lets UFS host drivers initialize the blk_crypto_profile themselves rather than have it be initialized by ufshcd-core according to the UFSHCI standard. This is needed to support inline encryption on the "Exynos" UFS controller which has a nonstandard interface. Reviewed-by: Bart Van Assche Reviewed-by: Peter Griffin Signed-off-by: Eric Biggers Link: https://lore.kernel.org/r/20240708235330.103590-2-ebiggers@kernel.org Reviewed-by: Alim Akhtar Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufshcd-crypto.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'drivers/ufs/core') diff --git a/drivers/ufs/core/ufshcd-crypto.c b/drivers/ufs/core/ufshcd-crypto.c index f2c4422cab86..debc925ae439 100644 --- a/drivers/ufs/core/ufshcd-crypto.c +++ b/drivers/ufs/core/ufshcd-crypto.c @@ -159,6 +159,9 @@ int ufshcd_hba_init_crypto_capabilities(struct ufs_hba *hba) int err = 0; enum blk_crypto_mode_num blk_mode_num; + if (hba->quirks & UFSHCD_QUIRK_CUSTOM_CRYPTO_PROFILE) + return 0; + /* * Don't use crypto if either the hardware doesn't advertise the * standard crypto capability bit *or* if the vendor specific driver @@ -228,9 +231,10 @@ void ufshcd_init_crypto(struct ufs_hba *hba) if (!(hba->caps & UFSHCD_CAP_CRYPTO)) return; - /* Clear all keyslots - the number of keyslots is (CFGC + 1) */ - for (slot = 0; slot < hba->crypto_capabilities.config_count + 1; slot++) - ufshcd_clear_keyslot(hba, slot); + /* Clear all keyslots. */ + for (slot = 0; slot < hba->crypto_profile.num_slots; slot++) + hba->crypto_profile.ll_ops.keyslot_evict(&hba->crypto_profile, + NULL, slot); } void ufshcd_crypto_register(struct ufs_hba *hba, struct request_queue *q) -- cgit From ec99818afb03b1ebeb0b6ed0d5fd42143be79586 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 8 Jul 2024 16:53:26 -0700 Subject: scsi: ufs: core: fold ufshcd_clear_keyslot() into its caller Fold ufshcd_clear_keyslot() into its only remaining caller. Reviewed-by: Bart Van Assche Reviewed-by: Peter Griffin Signed-off-by: Eric Biggers Link: https://lore.kernel.org/r/20240708235330.103590-3-ebiggers@kernel.org Reviewed-by: Alim Akhtar Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufshcd-crypto.c | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) (limited to 'drivers/ufs/core') diff --git a/drivers/ufs/core/ufshcd-crypto.c b/drivers/ufs/core/ufshcd-crypto.c index debc925ae439..b4980fd91cee 100644 --- a/drivers/ufs/core/ufshcd-crypto.c +++ b/drivers/ufs/core/ufshcd-crypto.c @@ -95,8 +95,12 @@ static int ufshcd_crypto_keyslot_program(struct blk_crypto_profile *profile, return err; } -static int ufshcd_clear_keyslot(struct ufs_hba *hba, int slot) +static int ufshcd_crypto_keyslot_evict(struct blk_crypto_profile *profile, + const struct blk_crypto_key *key, + unsigned int slot) { + struct ufs_hba *hba = + container_of(profile, struct ufs_hba, crypto_profile); /* * Clear the crypto cfg on the device. Clearing CFGE * might not be sufficient, so just clear the entire cfg. @@ -106,16 +110,6 @@ static int ufshcd_clear_keyslot(struct ufs_hba *hba, int slot) return ufshcd_program_key(hba, &cfg, slot); } -static int ufshcd_crypto_keyslot_evict(struct blk_crypto_profile *profile, - const struct blk_crypto_key *key, - unsigned int slot) -{ - struct ufs_hba *hba = - container_of(profile, struct ufs_hba, crypto_profile); - - return ufshcd_clear_keyslot(hba, slot); -} - bool ufshcd_crypto_enable(struct ufs_hba *hba) { if (!(hba->caps & UFSHCD_CAP_CRYPTO)) -- cgit From e95881e0081a30e132b5ca087f1e07fc08608a7e Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 8 Jul 2024 16:53:27 -0700 Subject: scsi: ufs: core: Add UFSHCD_QUIRK_BROKEN_CRYPTO_ENABLE Add UFSHCD_QUIRK_BROKEN_CRYPTO_ENABLE which tells the UFS core to not use the crypto enable bit defined by the UFS specification. This is needed to support inline encryption on the "Exynos" UFS controller. Reviewed-by: Bart Van Assche Reviewed-by: Peter Griffin Signed-off-by: Eric Biggers Link: https://lore.kernel.org/r/20240708235330.103590-4-ebiggers@kernel.org Reviewed-by: Alim Akhtar Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufshcd-crypto.c | 8 ++++++++ include/ufs/ufshcd.h | 7 +++++++ 2 files changed, 15 insertions(+) (limited to 'drivers/ufs/core') diff --git a/drivers/ufs/core/ufshcd-crypto.c b/drivers/ufs/core/ufshcd-crypto.c index b4980fd91cee..a714dad82cd1 100644 --- a/drivers/ufs/core/ufshcd-crypto.c +++ b/drivers/ufs/core/ufshcd-crypto.c @@ -110,6 +110,10 @@ static int ufshcd_crypto_keyslot_evict(struct blk_crypto_profile *profile, return ufshcd_program_key(hba, &cfg, slot); } +/* + * Reprogram the keyslots if needed, and return true if CRYPTO_GENERAL_ENABLE + * should be used in the host controller initialization sequence. + */ bool ufshcd_crypto_enable(struct ufs_hba *hba) { if (!(hba->caps & UFSHCD_CAP_CRYPTO)) @@ -117,6 +121,10 @@ bool ufshcd_crypto_enable(struct ufs_hba *hba) /* Reset might clear all keys, so reprogram all the keys. */ blk_crypto_reprogram_all_keys(&hba->crypto_profile); + + if (hba->quirks & UFSHCD_QUIRK_BROKEN_CRYPTO_ENABLE) + return false; + return true; } diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index 4f8982e52ac4..e07104f355c0 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -654,6 +654,13 @@ enum ufshcd_quirks { * ufs_hba_variant_ops::init() must do it instead. */ UFSHCD_QUIRK_CUSTOM_CRYPTO_PROFILE = 1 << 22, + + /* + * This quirk needs to be enabled if the host controller supports inline + * encryption but does not support the CRYPTO_GENERAL_ENABLE bit, i.e. + * host controller initialization fails if that bit is set. + */ + UFSHCD_QUIRK_BROKEN_CRYPTO_ENABLE = 1 << 23, }; enum ufshcd_caps { -- cgit From 8ecea3da1567e0648b5d37a6faec73fc9c8571ba Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 8 Jul 2024 16:53:28 -0700 Subject: scsi: ufs: core: Add fill_crypto_prdt variant op Add a variant op to allow host drivers to initialize nonstandard crypto-related fields in the PRDT. This is needed to support inline encryption on the "Exynos" UFS controller. Note that this will be used together with the support for overriding the PRDT entry size that was already added by commit ada1e653a5ea ("scsi: ufs: core: Allow UFS host drivers to override the sg entry size"). Reviewed-by: Bart Van Assche Reviewed-by: Peter Griffin Signed-off-by: Eric Biggers Link: https://lore.kernel.org/r/20240708235330.103590-5-ebiggers@kernel.org Reviewed-by: Alim Akhtar Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufshcd-crypto.h | 19 +++++++++++++++++++ drivers/ufs/core/ufshcd.c | 2 +- include/ufs/ufshcd.h | 4 ++++ 3 files changed, 24 insertions(+), 1 deletion(-) (limited to 'drivers/ufs/core') diff --git a/drivers/ufs/core/ufshcd-crypto.h b/drivers/ufs/core/ufshcd-crypto.h index be8596f20ba2..3eb8df42e194 100644 --- a/drivers/ufs/core/ufshcd-crypto.h +++ b/drivers/ufs/core/ufshcd-crypto.h @@ -37,6 +37,19 @@ ufshcd_prepare_req_desc_hdr_crypto(struct ufshcd_lrb *lrbp, h->dunu = cpu_to_le32(upper_32_bits(lrbp->data_unit_num)); } +static inline int ufshcd_crypto_fill_prdt(struct ufs_hba *hba, + struct ufshcd_lrb *lrbp) +{ + struct scsi_cmnd *cmd = lrbp->cmd; + const struct bio_crypt_ctx *crypt_ctx = scsi_cmd_to_rq(cmd)->crypt_ctx; + + if (crypt_ctx && hba->vops && hba->vops->fill_crypto_prdt) + return hba->vops->fill_crypto_prdt(hba, crypt_ctx, + lrbp->ucd_prdt_ptr, + scsi_sg_count(cmd)); + return 0; +} + bool ufshcd_crypto_enable(struct ufs_hba *hba); int ufshcd_hba_init_crypto_capabilities(struct ufs_hba *hba); @@ -54,6 +67,12 @@ static inline void ufshcd_prepare_req_desc_hdr_crypto(struct ufshcd_lrb *lrbp, struct request_desc_header *h) { } +static inline int ufshcd_crypto_fill_prdt(struct ufs_hba *hba, + struct ufshcd_lrb *lrbp) +{ + return 0; +} + static inline bool ufshcd_crypto_enable(struct ufs_hba *hba) { return false; diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index 41bf2e249c83..06619682087c 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -2641,7 +2641,7 @@ static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) ufshcd_sgl_to_prdt(hba, lrbp, sg_segments, scsi_sglist(cmd)); - return 0; + return ufshcd_crypto_fill_prdt(hba, lrbp); } /** diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index e07104f355c0..fb791d1a6d00 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -322,6 +322,7 @@ struct ufs_pwr_mode_info { * @device_reset: called to issue a reset pulse on the UFS device * @config_scaling_param: called to configure clock scaling parameters * @program_key: program or evict an inline encryption key + * @fill_crypto_prdt: initialize crypto-related fields in the PRDT * @event_notify: called to notify important events * @reinit_notify: called to notify reinit of UFSHCD during max gear switch * @mcq_config_resource: called to configure MCQ platform resources @@ -367,6 +368,9 @@ struct ufs_hba_variant_ops { struct devfreq_simple_ondemand_data *data); int (*program_key)(struct ufs_hba *hba, const union ufs_crypto_cfg_entry *cfg, int slot); + int (*fill_crypto_prdt)(struct ufs_hba *hba, + const struct bio_crypt_ctx *crypt_ctx, + void *prdt, unsigned int num_segments); void (*event_notify)(struct ufs_hba *hba, enum ufs_event_type evt, void *data); void (*reinit_notify)(struct ufs_hba *); -- cgit From 4c45dba50a3750a0834353c4187e7896b158bc0c Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 8 Jul 2024 16:53:29 -0700 Subject: scsi: ufs: core: Add UFSHCD_QUIRK_KEYS_IN_PRDT Since the nonstandard inline encryption support on Exynos SoCs requires that raw cryptographic keys be copied into the PRDT, it is desirable to zeroize those keys after each request to keep them from being left in memory. Therefore, add a quirk bit that enables the zeroization. We could instead do the zeroization unconditionally. However, using a quirk bit avoids adding the zeroization overhead to standard devices. Reviewed-by: Bart Van Assche Reviewed-by: Peter Griffin Signed-off-by: Eric Biggers Link: https://lore.kernel.org/r/20240708235330.103590-6-ebiggers@kernel.org Reviewed-by: Alim Akhtar Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufshcd-crypto.h | 17 +++++++++++++++++ drivers/ufs/core/ufshcd.c | 1 + include/ufs/ufshcd.h | 8 ++++++++ 3 files changed, 26 insertions(+) (limited to 'drivers/ufs/core') diff --git a/drivers/ufs/core/ufshcd-crypto.h b/drivers/ufs/core/ufshcd-crypto.h index 3eb8df42e194..89bb97c14c15 100644 --- a/drivers/ufs/core/ufshcd-crypto.h +++ b/drivers/ufs/core/ufshcd-crypto.h @@ -50,6 +50,20 @@ static inline int ufshcd_crypto_fill_prdt(struct ufs_hba *hba, return 0; } +static inline void ufshcd_crypto_clear_prdt(struct ufs_hba *hba, + struct ufshcd_lrb *lrbp) +{ + if (!(hba->quirks & UFSHCD_QUIRK_KEYS_IN_PRDT)) + return; + + if (!(scsi_cmd_to_rq(lrbp->cmd)->crypt_ctx)) + return; + + /* Zeroize the PRDT because it can contain cryptographic keys. */ + memzero_explicit(lrbp->ucd_prdt_ptr, + ufshcd_sg_entry_size(hba) * scsi_sg_count(lrbp->cmd)); +} + bool ufshcd_crypto_enable(struct ufs_hba *hba); int ufshcd_hba_init_crypto_capabilities(struct ufs_hba *hba); @@ -73,6 +87,9 @@ static inline int ufshcd_crypto_fill_prdt(struct ufs_hba *hba, return 0; } +static inline void ufshcd_crypto_clear_prdt(struct ufs_hba *hba, + struct ufshcd_lrb *lrbp) { } + static inline bool ufshcd_crypto_enable(struct ufs_hba *hba) { return false; diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index 06619682087c..7c5ad68e49f2 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -5479,6 +5479,7 @@ void ufshcd_release_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd = lrbp->cmd; scsi_dma_unmap(cmd); + ufshcd_crypto_clear_prdt(hba, lrbp); ufshcd_release(hba); ufshcd_clk_scaling_update_busy(hba); } diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index fb791d1a6d00..80accfbe48aa 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -665,6 +665,14 @@ enum ufshcd_quirks { * host controller initialization fails if that bit is set. */ UFSHCD_QUIRK_BROKEN_CRYPTO_ENABLE = 1 << 23, + + /* + * This quirk needs to be enabled if the host controller driver copies + * cryptographic keys into the PRDT in order to send them to hardware, + * and therefore the PRDT should be zeroized after each request (as per + * the standard best practice for managing keys). + */ + UFSHCD_QUIRK_KEYS_IN_PRDT = 1 << 24, }; enum ufshcd_caps { -- cgit