From 9ec54934ce857065e38523a2010e20182e76f515 Mon Sep 17 00:00:00 2001 From: Avri Altman Date: Thu, 30 May 2024 17:25:07 +0300 Subject: scsi: ufs: core: Allow RTT negotiation The rtt-upiu packets precede any data-out upiu packets, thus synchronizing the data input to the device: this mostly applies to write operations, but there are other operations that requires rtt as well. There are several rules binding this rtt - data-out dialog, specifically There can be at most outstanding bMaxNumOfRTT such packets. This might have an effect on write performance (sequential write in particular), as each data-out upiu must wait for its rtt sibling. UFSHCI expects bMaxNumOfRTT to be min(bDeviceRTTCap, NORTT). However, as of today, there does not appears to be no-one who sets it: not the host controller nor the driver. It wasn't an issue up to now: bMaxNumOfRTT is set to 2 after manufacturing, and wasn't limiting the write performance. UFS4.0, and specifically gear 5 changes this, and requires the device to be more attentive. This doesn't come free - the device has to allocate more resources to that end, but the sequential write performance improvement is significant. Early measurements shows 25% gain when moving from rtt 2 to 9. Therefore, set bMaxNumOfRTT to be min(bDeviceRTTCap, NORTT) as UFSHCI expects. Signed-off-by: Avri Altman Link: https://lore.kernel.org/r/20240530142510.734-2-avri.altman@wdc.com Reviewed-by: Bean Huo Reviewed-by: Bart Van Assche Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufshcd.c | 38 ++++++++++++++++++++++++++++++++++++++ include/ufs/ufs.h | 2 ++ include/ufs/ufshcd.h | 2 ++ include/ufs/ufshci.h | 1 + 4 files changed, 43 insertions(+) diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index 0cf07194bbe8..dda6d7e44436 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -102,6 +102,9 @@ /* Default RTC update every 10 seconds */ #define UFS_RTC_UPDATE_INTERVAL_MS (10 * MSEC_PER_SEC) +/* bMaxNumOfRTT is equal to two after device manufacturing */ +#define DEFAULT_MAX_NUM_RTT 2 + /* UFSHC 4.0 compliant HC support this mode. */ static bool use_mcq_mode = true; @@ -2405,6 +2408,8 @@ static inline int ufshcd_hba_capabilities(struct ufs_hba *hba) ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1; hba->reserved_slot = hba->nutrs - 1; + hba->nortt = FIELD_GET(MASK_NUMBER_OUTSTANDING_RTT, hba->capabilities) + 1; + /* Read crypto capabilities */ err = ufshcd_hba_init_crypto_capabilities(hba); if (err) { @@ -8121,6 +8126,35 @@ out: dev_info->b_ext_iid_en = ext_iid_en; } +static void ufshcd_set_rtt(struct ufs_hba *hba) +{ + struct ufs_dev_info *dev_info = &hba->dev_info; + u32 rtt = 0; + u32 dev_rtt = 0; + + /* RTT override makes sense only for UFS-4.0 and above */ + if (dev_info->wspecversion < 0x400) + return; + + if (ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, + QUERY_ATTR_IDN_MAX_NUM_OF_RTT, 0, 0, &dev_rtt)) { + dev_err(hba->dev, "failed reading bMaxNumOfRTT\n"); + return; + } + + /* do not override if it was already written */ + if (dev_rtt != DEFAULT_MAX_NUM_RTT) + return; + + rtt = min_t(int, dev_info->rtt_cap, hba->nortt); + if (rtt == dev_rtt) + return; + + if (ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, + QUERY_ATTR_IDN_MAX_NUM_OF_RTT, 0, 0, &rtt)) + dev_err(hba->dev, "failed writing bMaxNumOfRTT\n"); +} + void ufshcd_fixup_dev_quirks(struct ufs_hba *hba, const struct ufs_dev_quirk *fixups) { @@ -8256,6 +8290,8 @@ static int ufs_get_device_desc(struct ufs_hba *hba) desc_buf[DEVICE_DESC_PARAM_SPEC_VER + 1]; dev_info->bqueuedepth = desc_buf[DEVICE_DESC_PARAM_Q_DPTH]; + dev_info->rtt_cap = desc_buf[DEVICE_DESC_PARAM_RTT_CAP]; + model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME]; err = ufshcd_read_string_desc(hba, model_index, @@ -8508,6 +8544,8 @@ static int ufshcd_device_params_init(struct ufs_hba *hba) goto out; } + ufshcd_set_rtt(hba); + ufshcd_get_ref_clk_gating_wait(hba); if (!ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG, diff --git a/include/ufs/ufs.h b/include/ufs/ufs.h index b6003749bc83..853e95957c31 100644 --- a/include/ufs/ufs.h +++ b/include/ufs/ufs.h @@ -592,6 +592,8 @@ struct ufs_dev_info { enum ufs_rtc_time rtc_type; time64_t rtc_time_baseline; u32 rtc_update_period; + + u8 rtt_cap; /* bDeviceRTTCap */ }; /* diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index bad88bd91995..d74bd2d67b06 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -819,6 +819,7 @@ enum ufshcd_mcq_opr { * @capabilities: UFS Controller Capabilities * @mcq_capabilities: UFS Multi Circular Queue capabilities * @nutrs: Transfer Request Queue depth supported by controller + * @nortt - Max outstanding RTTs supported by controller * @nutmrs: Task Management Queue depth supported by controller * @reserved_slot: Used to submit device commands. Protected by @dev_cmd.lock. * @ufs_version: UFS Version to which controller complies @@ -957,6 +958,7 @@ struct ufs_hba { u32 capabilities; int nutrs; + int nortt; u32 mcq_capabilities; int nutmrs; u32 reserved_slot; diff --git a/include/ufs/ufshci.h b/include/ufs/ufshci.h index 385e1c6b8d60..c50f92bf2e1d 100644 --- a/include/ufs/ufshci.h +++ b/include/ufs/ufshci.h @@ -68,6 +68,7 @@ enum { /* Controller capability masks */ enum { MASK_TRANSFER_REQUESTS_SLOTS = 0x0000001F, + MASK_NUMBER_OUTSTANDING_RTT = 0x0000FF00, MASK_TASK_MANAGEMENT_REQUEST_SLOTS = 0x00070000, MASK_EHSLUTRD_SUPPORTED = 0x00400000, MASK_AUTO_HIBERN8_SUPPORT = 0x00800000, -- cgit From e75ff63300c5e8fac31649f438ebad6af88e0032 Mon Sep 17 00:00:00 2001 From: Avri Altman Date: Thu, 30 May 2024 17:25:08 +0300 Subject: scsi: ufs: core: Maximum RTT supported by the host driver Allow platform vendors to take precedence having their own max rtt support. This makes sense because the host controller's nortt characteristic may vary among vendors. while at it, set this value for Mediatek, as requested by Peter - https://lore.kernel.org/all/0a57d6bab739d6a10584f2baba115d00dfc9c94c.camel@mediatek.com/ Signed-off-by: Avri Altman Link: https://lore.kernel.org/r/20240530142510.734-3-avri.altman@wdc.com Reviewed-by: Peter Wang Reviewed-by: Bart Van Assche Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufshcd.c | 5 ++++- drivers/ufs/host/ufs-mediatek.c | 1 + drivers/ufs/host/ufs-mediatek.h | 3 +++ include/ufs/ufshcd.h | 2 ++ 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index dda6d7e44436..41bf2e249c83 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -8131,6 +8131,8 @@ static void ufshcd_set_rtt(struct ufs_hba *hba) struct ufs_dev_info *dev_info = &hba->dev_info; u32 rtt = 0; u32 dev_rtt = 0; + int host_rtt_cap = hba->vops && hba->vops->max_num_rtt ? + hba->vops->max_num_rtt : hba->nortt; /* RTT override makes sense only for UFS-4.0 and above */ if (dev_info->wspecversion < 0x400) @@ -8146,7 +8148,8 @@ static void ufshcd_set_rtt(struct ufs_hba *hba) if (dev_rtt != DEFAULT_MAX_NUM_RTT) return; - rtt = min_t(int, dev_info->rtt_cap, hba->nortt); + rtt = min_t(int, dev_info->rtt_cap, host_rtt_cap); + if (rtt == dev_rtt) return; diff --git a/drivers/ufs/host/ufs-mediatek.c b/drivers/ufs/host/ufs-mediatek.c index c4f997196c57..c7a0ab9b1f59 100644 --- a/drivers/ufs/host/ufs-mediatek.c +++ b/drivers/ufs/host/ufs-mediatek.c @@ -1785,6 +1785,7 @@ static int ufs_mtk_config_esi(struct ufs_hba *hba) */ static const struct ufs_hba_variant_ops ufs_hba_mtk_vops = { .name = "mediatek.ufshci", + .max_num_rtt = MTK_MAX_NUM_RTT, .init = ufs_mtk_init, .get_ufs_hci_version = ufs_mtk_get_ufs_hci_version, .setup_clocks = ufs_mtk_setup_clocks, diff --git a/drivers/ufs/host/ufs-mediatek.h b/drivers/ufs/host/ufs-mediatek.h index 3ff17e95afab..05d76a6bd772 100644 --- a/drivers/ufs/host/ufs-mediatek.h +++ b/drivers/ufs/host/ufs-mediatek.h @@ -189,4 +189,7 @@ struct ufs_mtk_host { /* MTK delay of autosuspend: 500 ms */ #define MTK_RPM_AUTOSUSPEND_DELAY_MS 500 +/* MTK RTT support number */ +#define MTK_MAX_NUM_RTT 2 + #endif /* !_UFS_MEDIATEK_H */ diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index d74bd2d67b06..ef04ec8aad69 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -295,6 +295,7 @@ struct ufs_pwr_mode_info { /** * struct ufs_hba_variant_ops - variant specific callbacks * @name: variant name + * @max_num_rtt: maximum RTT supported by the host * @init: called when the driver is initialized * @exit: called to cleanup everything done in init * @get_ufs_hci_version: called to get UFS HCI version @@ -332,6 +333,7 @@ struct ufs_pwr_mode_info { */ struct ufs_hba_variant_ops { const char *name; + int max_num_rtt; int (*init)(struct ufs_hba *); void (*exit)(struct ufs_hba *); u32 (*get_ufs_hci_version)(struct ufs_hba *); -- cgit From 600edc6620a4380b9f6027f293dac09eb0f22048 Mon Sep 17 00:00:00 2001 From: Avri Altman Date: Thu, 30 May 2024 17:25:09 +0300 Subject: scsi: ufs: sysfs: Make max_number_of_rtt read-write Given the importance of the RTT parameter, we want to be able to configure it via sysfs. This is because UFS users should be discouraged from change UFS device parameters without the UFSHCI driver being aware of these changes. Signed-off-by: Avri Altman Link: https://lore.kernel.org/r/20240530142510.734-4-avri.altman@wdc.com Reviewed-by: Bart Van Assche Signed-off-by: Martin K. Petersen --- Documentation/ABI/testing/sysfs-driver-ufs | 14 +++--- drivers/ufs/core/ufs-sysfs.c | 73 +++++++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 7 deletions(-) diff --git a/Documentation/ABI/testing/sysfs-driver-ufs b/Documentation/ABI/testing/sysfs-driver-ufs index 5bf7073b4f75..fe943ce76c60 100644 --- a/Documentation/ABI/testing/sysfs-driver-ufs +++ b/Documentation/ABI/testing/sysfs-driver-ufs @@ -920,14 +920,16 @@ Description: This file shows whether the configuration descriptor is locked. What: /sys/bus/platform/drivers/ufshcd/*/attributes/max_number_of_rtt What: /sys/bus/platform/devices/*.ufs/attributes/max_number_of_rtt -Date: February 2018 -Contact: Stanislav Nijnikov +Date: May 2024 +Contact: Avri Altman Description: This file provides the maximum current number of - outstanding RTTs in device that is allowed. The full - information about the attribute could be found at - UFS specifications 2.1. + outstanding RTTs in device that is allowed. bMaxNumOfRTT is a + read-write persistent attribute and is equal to two after device + manufacturing. It shall not be set to a value greater than + bDeviceRTTCap value, and it may be set only when the hw queues are + empty. - The file is read only. + The file is read write. What: /sys/bus/platform/drivers/ufshcd/*/attributes/exception_event_control What: /sys/bus/platform/devices/*.ufs/attributes/exception_event_control diff --git a/drivers/ufs/core/ufs-sysfs.c b/drivers/ufs/core/ufs-sysfs.c index 3d049967f6bc..e80a32421a8c 100644 --- a/drivers/ufs/core/ufs-sysfs.c +++ b/drivers/ufs/core/ufs-sysfs.c @@ -1340,6 +1340,78 @@ static const struct attribute_group ufs_sysfs_flags_group = { .attrs = ufs_sysfs_device_flags, }; +static ssize_t max_number_of_rtt_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + u32 rtt; + int ret; + + down(&hba->host_sem); + if (!ufshcd_is_user_access_allowed(hba)) { + up(&hba->host_sem); + return -EBUSY; + } + + ufshcd_rpm_get_sync(hba); + ret = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_READ_ATTR, + QUERY_ATTR_IDN_MAX_NUM_OF_RTT, 0, 0, &rtt); + ufshcd_rpm_put_sync(hba); + + if (ret) + goto out; + + ret = sysfs_emit(buf, "0x%08X\n", rtt); + +out: + up(&hba->host_sem); + return ret; +} + +static ssize_t max_number_of_rtt_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + struct ufs_dev_info *dev_info = &hba->dev_info; + struct scsi_device *sdev; + unsigned int rtt; + int ret; + + if (kstrtouint(buf, 0, &rtt)) + return -EINVAL; + + if (rtt > dev_info->rtt_cap) { + dev_err(dev, "rtt can be at most bDeviceRTTCap\n"); + return -EINVAL; + } + + down(&hba->host_sem); + if (!ufshcd_is_user_access_allowed(hba)) { + ret = -EBUSY; + goto out; + } + + ufshcd_rpm_get_sync(hba); + + shost_for_each_device(sdev, hba->host) + blk_mq_freeze_queue(sdev->request_queue); + + ret = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, + QUERY_ATTR_IDN_MAX_NUM_OF_RTT, 0, 0, &rtt); + + shost_for_each_device(sdev, hba->host) + blk_mq_unfreeze_queue(sdev->request_queue); + + ufshcd_rpm_put_sync(hba); + +out: + up(&hba->host_sem); + return ret < 0 ? ret : count; +} + +static DEVICE_ATTR_RW(max_number_of_rtt); + static inline bool ufshcd_is_wb_attrs(enum attr_idn idn) { return idn >= QUERY_ATTR_IDN_WB_FLUSH_STATUS && @@ -1387,7 +1459,6 @@ UFS_ATTRIBUTE(max_data_in_size, _MAX_DATA_IN); UFS_ATTRIBUTE(max_data_out_size, _MAX_DATA_OUT); UFS_ATTRIBUTE(reference_clock_frequency, _REF_CLK_FREQ); UFS_ATTRIBUTE(configuration_descriptor_lock, _CONF_DESC_LOCK); -UFS_ATTRIBUTE(max_number_of_rtt, _MAX_NUM_OF_RTT); UFS_ATTRIBUTE(exception_event_control, _EE_CONTROL); UFS_ATTRIBUTE(exception_event_status, _EE_STATUS); UFS_ATTRIBUTE(ffu_status, _FFU_STATUS); -- cgit