diff options
Diffstat (limited to 'drivers/cpufreq')
32 files changed, 602 insertions, 312 deletions
diff --git a/drivers/cpufreq/Kconfig b/drivers/cpufreq/Kconfig index 2561b215432a..92a83a9bb2e1 100644 --- a/drivers/cpufreq/Kconfig +++ b/drivers/cpufreq/Kconfig @@ -217,6 +217,20 @@ config CPUFREQ_DT If in doubt, say N. +config CPUFREQ_VIRT + tristate "Virtual cpufreq driver" + depends on GENERIC_ARCH_TOPOLOGY + help + This adds a virtualized cpufreq driver for guest kernels that + read/writes to a MMIO region for a virtualized cpufreq device to + communicate with the host. It sends performance requests to the host + which gets used as a hint to schedule vCPU threads and select CPU + frequency. If a VM does not support a virtualized FIE such as AMUs, + it updates the frequency scaling factor by polling host CPU frequency + to enable accurate Per-Entity Load Tracking for tasks running in the guest. + + If in doubt, say N. + config CPUFREQ_DT_PLATDEV tristate "Generic DT based cpufreq platdev driver" depends on OF diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile index 0f184031dd12..10d7d6e55da8 100644 --- a/drivers/cpufreq/Makefile +++ b/drivers/cpufreq/Makefile @@ -16,6 +16,7 @@ obj-$(CONFIG_CPU_FREQ_GOV_ATTR_SET) += cpufreq_governor_attr_set.o obj-$(CONFIG_CPUFREQ_DT) += cpufreq-dt.o obj-$(CONFIG_CPUFREQ_DT_PLATDEV) += cpufreq-dt-platdev.o +obj-$(CONFIG_CPUFREQ_VIRT) += virtual-cpufreq.o # Traces CFLAGS_amd-pstate-trace.o := -I$(src) diff --git a/drivers/cpufreq/acpi-cpufreq.c b/drivers/cpufreq/acpi-cpufreq.c index 0f04feb6cafa..c9ebacf5c88e 100644 --- a/drivers/cpufreq/acpi-cpufreq.c +++ b/drivers/cpufreq/acpi-cpufreq.c @@ -73,20 +73,17 @@ static unsigned int acpi_pstate_strict; static bool boost_state(unsigned int cpu) { - u32 lo, hi; u64 msr; switch (boot_cpu_data.x86_vendor) { case X86_VENDOR_INTEL: case X86_VENDOR_CENTAUR: case X86_VENDOR_ZHAOXIN: - rdmsr_on_cpu(cpu, MSR_IA32_MISC_ENABLE, &lo, &hi); - msr = lo | ((u64)hi << 32); + rdmsrl_on_cpu(cpu, MSR_IA32_MISC_ENABLE, &msr); return !(msr & MSR_IA32_MISC_ENABLE_TURBO_DISABLE); case X86_VENDOR_HYGON: case X86_VENDOR_AMD: - rdmsr_on_cpu(cpu, MSR_K7_HWCR, &lo, &hi); - msr = lo | ((u64)hi << 32); + rdmsrl_on_cpu(cpu, MSR_K7_HWCR, &msr); return !(msr & MSR_K7_HWCR_CPB_DIS); } return false; @@ -1028,7 +1025,7 @@ static struct platform_driver acpi_cpufreq_platdrv = { .driver = { .name = "acpi-cpufreq", }, - .remove_new = acpi_cpufreq_remove, + .remove = acpi_cpufreq_remove, }; static int __init acpi_cpufreq_init(void) diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c index f66701514d90..a261d7300951 100644 --- a/drivers/cpufreq/amd-pstate-ut.c +++ b/drivers/cpufreq/amd-pstate-ut.c @@ -227,10 +227,10 @@ static void amd_pstate_ut_check_freq(u32 index) goto skip_test; } - if (cpudata->min_freq != policy->min) { + if (cpudata->lowest_nonlinear_freq != policy->min) { amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; - pr_err("%s cpu%d cpudata_min_freq=%d policy_min=%d, they should be equal!\n", - __func__, cpu, cpudata->min_freq, policy->min); + pr_err("%s cpu%d cpudata_lowest_nonlinear_freq=%d policy_min=%d, they should be equal!\n", + __func__, cpu, cpudata->lowest_nonlinear_freq, policy->min); goto skip_test; } diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index b63863f77c67..d7630bab2516 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -233,7 +233,7 @@ static int amd_pstate_get_energy_pref_index(struct amd_cpudata *cpudata) return index; } -static void pstate_update_perf(struct amd_cpudata *cpudata, u32 min_perf, +static void msr_update_perf(struct amd_cpudata *cpudata, u32 min_perf, u32 des_perf, u32 max_perf, bool fast_switch) { if (fast_switch) @@ -243,7 +243,7 @@ static void pstate_update_perf(struct amd_cpudata *cpudata, u32 min_perf, READ_ONCE(cpudata->cppc_req_cached)); } -DEFINE_STATIC_CALL(amd_pstate_update_perf, pstate_update_perf); +DEFINE_STATIC_CALL(amd_pstate_update_perf, msr_update_perf); static inline void amd_pstate_update_perf(struct amd_cpudata *cpudata, u32 min_perf, u32 des_perf, @@ -306,11 +306,17 @@ static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata, return ret; } -static inline int pstate_enable(bool enable) +static inline int msr_cppc_enable(bool enable) { int ret, cpu; unsigned long logical_proc_id_mask = 0; + /* + * MSR_AMD_CPPC_ENABLE is write-once, once set it cannot be cleared. + */ + if (!enable) + return 0; + if (enable == cppc_enabled) return 0; @@ -332,7 +338,7 @@ static inline int pstate_enable(bool enable) return 0; } -static int cppc_enable(bool enable) +static int shmem_cppc_enable(bool enable) { int cpu, ret = 0; struct cppc_perf_ctrls perf_ctrls; @@ -359,14 +365,14 @@ static int cppc_enable(bool enable) return ret; } -DEFINE_STATIC_CALL(amd_pstate_enable, pstate_enable); +DEFINE_STATIC_CALL(amd_pstate_cppc_enable, msr_cppc_enable); -static inline int amd_pstate_enable(bool enable) +static inline int amd_pstate_cppc_enable(bool enable) { - return static_call(amd_pstate_enable)(enable); + return static_call(amd_pstate_cppc_enable)(enable); } -static int pstate_init_perf(struct amd_cpudata *cpudata) +static int msr_init_perf(struct amd_cpudata *cpudata) { u64 cap1; @@ -385,7 +391,7 @@ static int pstate_init_perf(struct amd_cpudata *cpudata) return 0; } -static int cppc_init_perf(struct amd_cpudata *cpudata) +static int shmem_init_perf(struct amd_cpudata *cpudata) { struct cppc_perf_caps cppc_perf; @@ -420,14 +426,14 @@ static int cppc_init_perf(struct amd_cpudata *cpudata) return ret; } -DEFINE_STATIC_CALL(amd_pstate_init_perf, pstate_init_perf); +DEFINE_STATIC_CALL(amd_pstate_init_perf, msr_init_perf); static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata) { return static_call(amd_pstate_init_perf)(cpudata); } -static void cppc_update_perf(struct amd_cpudata *cpudata, +static void shmem_update_perf(struct amd_cpudata *cpudata, u32 min_perf, u32 des_perf, u32 max_perf, bool fast_switch) { @@ -527,9 +533,28 @@ cpufreq_policy_put: cpufreq_cpu_put(policy); } -static int amd_pstate_verify(struct cpufreq_policy_data *policy) +static int amd_pstate_verify(struct cpufreq_policy_data *policy_data) { - cpufreq_verify_within_cpu_limits(policy); + /* + * Initialize lower frequency limit (i.e.policy->min) with + * lowest_nonlinear_frequency which is the most energy efficient + * frequency. Override the initial value set by cpufreq core and + * amd-pstate qos_requests. + */ + if (policy_data->min == FREQ_QOS_MIN_DEFAULT_VALUE) { + struct cpufreq_policy *policy = cpufreq_cpu_get(policy_data->cpu); + struct amd_cpudata *cpudata; + + if (!policy) + return -EINVAL; + + cpudata = policy->driver_data; + policy_data->min = cpudata->lowest_nonlinear_freq; + cpufreq_cpu_put(policy); + } + + cpufreq_verify_within_cpu_limits(policy_data); + pr_debug("policy_max =%d, policy_min=%d\n", policy_data->max, policy_data->min); return 0; } @@ -665,34 +690,12 @@ static void amd_pstate_adjust_perf(unsigned int cpu, static int amd_pstate_cpu_boost_update(struct cpufreq_policy *policy, bool on) { struct amd_cpudata *cpudata = policy->driver_data; - struct cppc_perf_ctrls perf_ctrls; - u32 highest_perf, nominal_perf, nominal_freq, max_freq; + u32 nominal_freq, max_freq; int ret = 0; - highest_perf = READ_ONCE(cpudata->highest_perf); - nominal_perf = READ_ONCE(cpudata->nominal_perf); nominal_freq = READ_ONCE(cpudata->nominal_freq); max_freq = READ_ONCE(cpudata->max_freq); - if (boot_cpu_has(X86_FEATURE_CPPC)) { - u64 value = READ_ONCE(cpudata->cppc_req_cached); - - value &= ~GENMASK_ULL(7, 0); - value |= on ? highest_perf : nominal_perf; - WRITE_ONCE(cpudata->cppc_req_cached, value); - - wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); - } else { - perf_ctrls.max_perf = on ? highest_perf : nominal_perf; - ret = cppc_set_perf(cpudata->cpu, &perf_ctrls); - if (ret) { - cpufreq_cpu_release(policy); - pr_debug("Failed to set max perf on CPU:%d. ret:%d\n", - cpudata->cpu, ret); - return ret; - } - } - if (on) policy->cpuinfo.max_freq = max_freq; else if (policy->cpuinfo.max_freq > nominal_freq * 1000) @@ -847,7 +850,7 @@ static u32 amd_pstate_get_transition_delay_us(unsigned int cpu) transition_delay_ns = cppc_get_transition_latency(cpu); if (transition_delay_ns == CPUFREQ_ETERNAL) { - if (cpu_feature_enabled(X86_FEATURE_FAST_CPPC)) + if (cpu_feature_enabled(X86_FEATURE_AMD_FAST_CPPC)) return AMD_PSTATE_FAST_CPPC_TRANSITION_DELAY; else return AMD_PSTATE_TRANSITION_DELAY; @@ -1001,7 +1004,7 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) policy->fast_switch_possible = true; ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0], - FREQ_QOS_MIN, policy->cpuinfo.min_freq); + FREQ_QOS_MIN, FREQ_QOS_MIN_DEFAULT_VALUE); if (ret < 0) { dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret); goto free_cpudata1; @@ -1045,7 +1048,7 @@ static int amd_pstate_cpu_resume(struct cpufreq_policy *policy) { int ret; - ret = amd_pstate_enable(true); + ret = amd_pstate_cppc_enable(true); if (ret) pr_err("failed to enable amd-pstate during resume, return %d\n", ret); @@ -1056,7 +1059,7 @@ static int amd_pstate_cpu_suspend(struct cpufreq_policy *policy) { int ret; - ret = amd_pstate_enable(false); + ret = amd_pstate_cppc_enable(false); if (ret) pr_err("failed to disable amd-pstate during suspend, return %d\n", ret); @@ -1189,25 +1192,41 @@ static ssize_t show_energy_performance_preference( static void amd_pstate_driver_cleanup(void) { - amd_pstate_enable(false); + amd_pstate_cppc_enable(false); cppc_state = AMD_PSTATE_DISABLE; current_pstate_driver = NULL; } +static int amd_pstate_set_driver(int mode_idx) +{ + if (mode_idx >= AMD_PSTATE_DISABLE && mode_idx < AMD_PSTATE_MAX) { + cppc_state = mode_idx; + if (cppc_state == AMD_PSTATE_DISABLE) + pr_info("driver is explicitly disabled\n"); + + if (cppc_state == AMD_PSTATE_ACTIVE) + current_pstate_driver = &amd_pstate_epp_driver; + + if (cppc_state == AMD_PSTATE_PASSIVE || cppc_state == AMD_PSTATE_GUIDED) + current_pstate_driver = &amd_pstate_driver; + + return 0; + } + + return -EINVAL; +} + static int amd_pstate_register_driver(int mode) { int ret; - if (mode == AMD_PSTATE_PASSIVE || mode == AMD_PSTATE_GUIDED) - current_pstate_driver = &amd_pstate_driver; - else if (mode == AMD_PSTATE_ACTIVE) - current_pstate_driver = &amd_pstate_epp_driver; - else - return -EINVAL; + ret = amd_pstate_set_driver(mode); + if (ret) + return ret; cppc_state = mode; - ret = amd_pstate_enable(true); + ret = amd_pstate_cppc_enable(true); if (ret) { pr_err("failed to enable cppc during amd-pstate driver registration, return %d\n", ret); @@ -1485,6 +1504,8 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) WRITE_ONCE(cpudata->cppc_cap1_cached, value); } + current_pstate_driver->adjust_perf = NULL; + return 0; free_cpudata1: @@ -1507,26 +1528,13 @@ static void amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy) { struct amd_cpudata *cpudata = policy->driver_data; - u32 max_perf, min_perf, min_limit_perf, max_limit_perf; + u32 max_perf, min_perf; u64 value; s16 epp; - if (cpudata->boost_supported && !policy->boost_enabled) - max_perf = READ_ONCE(cpudata->nominal_perf); - else - max_perf = READ_ONCE(cpudata->highest_perf); + max_perf = READ_ONCE(cpudata->highest_perf); min_perf = READ_ONCE(cpudata->lowest_perf); - max_limit_perf = div_u64(policy->max * max_perf, policy->cpuinfo.max_freq); - min_limit_perf = div_u64(policy->min * max_perf, policy->cpuinfo.max_freq); - - if (min_limit_perf < min_perf) - min_limit_perf = min_perf; - - if (max_limit_perf < min_limit_perf) - max_limit_perf = min_limit_perf; - - WRITE_ONCE(cpudata->max_limit_perf, max_limit_perf); - WRITE_ONCE(cpudata->min_limit_perf, min_limit_perf); + amd_pstate_update_min_max_limit(policy); max_perf = clamp_t(unsigned long, max_perf, cpudata->min_limit_perf, cpudata->max_limit_perf); @@ -1535,7 +1543,7 @@ static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy) value = READ_ONCE(cpudata->cppc_req_cached); if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) - min_perf = max_perf; + min_perf = min(cpudata->nominal_perf, max_perf); /* Initial min/max values for CPPC Performance Controls Register */ value &= ~AMD_CPPC_MIN_PERF(~0L); @@ -1563,12 +1571,6 @@ static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy) if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) epp = 0; - /* Set initial EPP value */ - if (cpu_feature_enabled(X86_FEATURE_CPPC)) { - value &= ~GENMASK_ULL(31, 24); - value |= (u64)epp << 24; - } - WRITE_ONCE(cpudata->cppc_req_cached, value); return amd_pstate_set_epp(cpudata, epp); } @@ -1605,7 +1607,7 @@ static void amd_pstate_epp_reenable(struct amd_cpudata *cpudata) u64 value, max_perf; int ret; - ret = amd_pstate_enable(true); + ret = amd_pstate_cppc_enable(true); if (ret) pr_err("failed to enable amd pstate during resume, return %d\n", ret); @@ -1616,8 +1618,9 @@ static void amd_pstate_epp_reenable(struct amd_cpudata *cpudata) wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); } else { perf_ctrls.max_perf = max_perf; - perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(cpudata->epp_cached); cppc_set_perf(cpudata->cpu, &perf_ctrls); + perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(cpudata->epp_cached); + cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); } } @@ -1657,9 +1660,11 @@ static void amd_pstate_epp_offline(struct cpufreq_policy *policy) wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); } else { perf_ctrls.desired_perf = 0; + perf_ctrls.min_perf = min_perf; perf_ctrls.max_perf = min_perf; - perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(HWP_EPP_BALANCE_POWERSAVE); cppc_set_perf(cpudata->cpu, &perf_ctrls); + perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(HWP_EPP_BALANCE_POWERSAVE); + cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); } mutex_unlock(&amd_pstate_limits_lock); } @@ -1679,13 +1684,6 @@ static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) return 0; } -static int amd_pstate_epp_verify_policy(struct cpufreq_policy_data *policy) -{ - cpufreq_verify_within_cpu_limits(policy); - pr_debug("policy_max =%d, policy_min=%d\n", policy->max, policy->min); - return 0; -} - static int amd_pstate_epp_suspend(struct cpufreq_policy *policy) { struct amd_cpudata *cpudata = policy->driver_data; @@ -1699,7 +1697,7 @@ static int amd_pstate_epp_suspend(struct cpufreq_policy *policy) cpudata->suspended = true; /* disable CPPC in lowlevel firmware */ - ret = amd_pstate_enable(false); + ret = amd_pstate_cppc_enable(false); if (ret) pr_err("failed to suspend, return %d\n", ret); @@ -1741,7 +1739,7 @@ static struct cpufreq_driver amd_pstate_driver = { static struct cpufreq_driver amd_pstate_epp_driver = { .flags = CPUFREQ_CONST_LOOPS, - .verify = amd_pstate_epp_verify_policy, + .verify = amd_pstate_verify, .setpolicy = amd_pstate_epp_set_policy, .init = amd_pstate_epp_cpu_init, .exit = amd_pstate_epp_cpu_exit, @@ -1755,26 +1753,7 @@ static struct cpufreq_driver amd_pstate_epp_driver = { .attr = amd_pstate_epp_attr, }; -static int __init amd_pstate_set_driver(int mode_idx) -{ - if (mode_idx >= AMD_PSTATE_DISABLE && mode_idx < AMD_PSTATE_MAX) { - cppc_state = mode_idx; - if (cppc_state == AMD_PSTATE_DISABLE) - pr_info("driver is explicitly disabled\n"); - - if (cppc_state == AMD_PSTATE_ACTIVE) - current_pstate_driver = &amd_pstate_epp_driver; - - if (cppc_state == AMD_PSTATE_PASSIVE || cppc_state == AMD_PSTATE_GUIDED) - current_pstate_driver = &amd_pstate_driver; - - return 0; - } - - return -EINVAL; -} - -/** +/* * CPPC function is not supported for family ID 17H with model_ID ranging from 0x10 to 0x2F. * show the debug message that helps to check if the CPU has CPPC support for loading issue. */ @@ -1864,10 +1843,10 @@ static int __init amd_pstate_init(void) if (cppc_state == AMD_PSTATE_UNDEFINED) { /* Disable on the following configs by default: * 1. Undefined platforms - * 2. Server platforms + * 2. Server platforms with CPUs older than Family 0x1A. */ if (amd_pstate_acpi_pm_profile_undefined() || - amd_pstate_acpi_pm_profile_server()) { + (amd_pstate_acpi_pm_profile_server() && boot_cpu_data.x86 < 0x1A)) { pr_info("driver load is disabled, boot with specific mode to enable this\n"); return -ENODEV; } @@ -1875,50 +1854,31 @@ static int __init amd_pstate_init(void) cppc_state = CONFIG_X86_AMD_PSTATE_DEFAULT_MODE; } - switch (cppc_state) { - case AMD_PSTATE_DISABLE: + if (cppc_state == AMD_PSTATE_DISABLE) { pr_info("driver load is disabled, boot with specific mode to enable this\n"); return -ENODEV; - case AMD_PSTATE_PASSIVE: - case AMD_PSTATE_ACTIVE: - case AMD_PSTATE_GUIDED: - ret = amd_pstate_set_driver(cppc_state); - if (ret) - return ret; - break; - default: - return -EINVAL; } /* capability check */ if (cpu_feature_enabled(X86_FEATURE_CPPC)) { pr_debug("AMD CPPC MSR based functionality is supported\n"); - if (cppc_state != AMD_PSTATE_ACTIVE) - current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; } else { pr_debug("AMD CPPC shared memory based functionality is supported\n"); - static_call_update(amd_pstate_enable, cppc_enable); - static_call_update(amd_pstate_init_perf, cppc_init_perf); - static_call_update(amd_pstate_update_perf, cppc_update_perf); + static_call_update(amd_pstate_cppc_enable, shmem_cppc_enable); + static_call_update(amd_pstate_init_perf, shmem_init_perf); + static_call_update(amd_pstate_update_perf, shmem_update_perf); } - if (amd_pstate_prefcore) { - ret = amd_detect_prefcore(&amd_pstate_prefcore); - if (ret) - return ret; - } - - /* enable amd pstate feature */ - ret = amd_pstate_enable(true); + ret = amd_pstate_register_driver(cppc_state); if (ret) { - pr_err("failed to enable driver mode(%d)\n", cppc_state); + pr_err("failed to register with return %d\n", ret); return ret; } - ret = cpufreq_register_driver(current_pstate_driver); - if (ret) { - pr_err("failed to register with return %d\n", ret); - goto disable_driver; + if (amd_pstate_prefcore) { + ret = amd_detect_prefcore(&amd_pstate_prefcore); + if (ret) + return ret; } dev_root = bus_get_dev_root(&cpu_subsys); @@ -1935,8 +1895,7 @@ static int __init amd_pstate_init(void) global_attr_free: cpufreq_unregister_driver(current_pstate_driver); -disable_driver: - amd_pstate_enable(false); + amd_pstate_cppc_enable(false); return ret; } device_initcall(amd_pstate_init); diff --git a/drivers/cpufreq/brcmstb-avs-cpufreq.c b/drivers/cpufreq/brcmstb-avs-cpufreq.c index ea8438550b49..2fd0f6be6fa3 100644 --- a/drivers/cpufreq/brcmstb-avs-cpufreq.c +++ b/drivers/cpufreq/brcmstb-avs-cpufreq.c @@ -474,8 +474,8 @@ static bool brcm_avs_is_firmware_loaded(struct private_data *priv) rc = brcm_avs_get_pmap(priv, NULL); magic = readl(priv->base + AVS_MBOX_MAGIC); - return (magic == AVS_FIRMWARE_MAGIC) && ((rc != -ENOTSUPP) || - (rc != -EINVAL)); + return (magic == AVS_FIRMWARE_MAGIC) && (rc != -ENOTSUPP) && + (rc != -EINVAL); } static unsigned int brcm_avs_cpufreq_get(unsigned int cpu) @@ -777,7 +777,7 @@ static struct platform_driver brcm_avs_cpufreq_platdrv = { .of_match_table = brcm_avs_cpufreq_match, }, .probe = brcm_avs_cpufreq_probe, - .remove_new = brcm_avs_cpufreq_remove, + .remove = brcm_avs_cpufreq_remove, }; module_platform_driver(brcm_avs_cpufreq_platdrv); diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c index 2b8708475ac7..bd8f75accfa0 100644 --- a/drivers/cpufreq/cppc_cpufreq.c +++ b/drivers/cpufreq/cppc_cpufreq.c @@ -36,33 +36,15 @@ static LIST_HEAD(cpu_data_list); static bool boost_supported; -struct cppc_workaround_oem_info { - char oem_id[ACPI_OEM_ID_SIZE + 1]; - char oem_table_id[ACPI_OEM_TABLE_ID_SIZE + 1]; - u32 oem_revision; -}; - -static struct cppc_workaround_oem_info wa_info[] = { - { - .oem_id = "HISI ", - .oem_table_id = "HIP07 ", - .oem_revision = 0, - }, { - .oem_id = "HISI ", - .oem_table_id = "HIP08 ", - .oem_revision = 0, - } -}; - static struct cpufreq_driver cppc_cpufreq_driver; +#ifdef CONFIG_ACPI_CPPC_CPUFREQ_FIE static enum { FIE_UNSET = -1, FIE_ENABLED, FIE_DISABLED } fie_disabled = FIE_UNSET; -#ifdef CONFIG_ACPI_CPPC_CPUFREQ_FIE module_param(fie_disabled, int, 0444); MODULE_PARM_DESC(fie_disabled, "Disable Frequency Invariance Engine (FIE)"); @@ -78,7 +60,6 @@ struct cppc_freq_invariance { static DEFINE_PER_CPU(struct cppc_freq_invariance, cppc_freq_inv); static struct kthread_worker *kworker_fie; -static unsigned int hisi_cppc_cpufreq_get_rate(unsigned int cpu); static int cppc_perf_from_fbctrs(struct cppc_cpudata *cpu_data, struct cppc_perf_fb_ctrs *fb_ctrs_t0, struct cppc_perf_fb_ctrs *fb_ctrs_t1); @@ -118,6 +99,9 @@ static void cppc_scale_freq_workfn(struct kthread_work *work) perf = cppc_perf_from_fbctrs(cpu_data, &cppc_fi->prev_perf_fb_ctrs, &fb_ctrs); + if (!perf) + return; + cppc_fi->prev_perf_fb_ctrs = fb_ctrs; perf <<= SCHED_CAPACITY_SHIFT; @@ -420,6 +404,9 @@ static int cppc_get_cpu_power(struct device *cpu_dev, struct cppc_cpudata *cpu_data; policy = cpufreq_cpu_get_raw(cpu_dev->id); + if (!policy) + return -EINVAL; + cpu_data = policy->driver_data; perf_caps = &cpu_data->perf_caps; max_cap = arch_scale_cpu_capacity(cpu_dev->id); @@ -487,6 +474,9 @@ static int cppc_get_cpu_cost(struct device *cpu_dev, unsigned long KHz, int step; policy = cpufreq_cpu_get_raw(cpu_dev->id); + if (!policy) + return -EINVAL; + cpu_data = policy->driver_data; perf_caps = &cpu_data->perf_caps; max_cap = arch_scale_cpu_capacity(cpu_dev->id); @@ -724,13 +714,31 @@ static int cppc_perf_from_fbctrs(struct cppc_cpudata *cpu_data, delta_delivered = get_delta(fb_ctrs_t1->delivered, fb_ctrs_t0->delivered); - /* Check to avoid divide-by zero and invalid delivered_perf */ + /* + * Avoid divide-by zero and unchanged feedback counters. + * Leave it for callers to handle. + */ if (!delta_reference || !delta_delivered) - return cpu_data->perf_ctrls.desired_perf; + return 0; return (reference_perf * delta_delivered) / delta_reference; } +static int cppc_get_perf_ctrs_sample(int cpu, + struct cppc_perf_fb_ctrs *fb_ctrs_t0, + struct cppc_perf_fb_ctrs *fb_ctrs_t1) +{ + int ret; + + ret = cppc_get_perf_ctrs(cpu, fb_ctrs_t0); + if (ret) + return ret; + + udelay(2); /* 2usec delay between sampling */ + + return cppc_get_perf_ctrs(cpu, fb_ctrs_t1); +} + static unsigned int cppc_cpufreq_get_rate(unsigned int cpu) { struct cppc_perf_fb_ctrs fb_ctrs_t0 = {0}, fb_ctrs_t1 = {0}; @@ -746,18 +754,32 @@ static unsigned int cppc_cpufreq_get_rate(unsigned int cpu) cpufreq_cpu_put(policy); - ret = cppc_get_perf_ctrs(cpu, &fb_ctrs_t0); - if (ret) - return 0; - - udelay(2); /* 2usec delay between sampling */ - - ret = cppc_get_perf_ctrs(cpu, &fb_ctrs_t1); - if (ret) - return 0; + ret = cppc_get_perf_ctrs_sample(cpu, &fb_ctrs_t0, &fb_ctrs_t1); + if (ret) { + if (ret == -EFAULT) + /* Any of the associated CPPC regs is 0. */ + goto out_invalid_counters; + else + return 0; + } delivered_perf = cppc_perf_from_fbctrs(cpu_data, &fb_ctrs_t0, &fb_ctrs_t1); + if (!delivered_perf) + goto out_invalid_counters; + + return cppc_perf_to_khz(&cpu_data->perf_caps, delivered_perf); + +out_invalid_counters: + /* + * Feedback counters could be unchanged or 0 when a cpu enters a + * low-power idle state, e.g. clock-gated or power-gated. + * Use desired perf for reflecting frequency. Get the latest register + * value first as some platforms may update the actual delivered perf + * there; if failed, resort to the cached desired perf. + */ + if (cppc_get_desired_perf(cpu, &delivered_perf)) + delivered_perf = cpu_data->perf_ctrls.desired_perf; return cppc_perf_to_khz(&cpu_data->perf_caps, delivered_perf); } @@ -812,57 +834,6 @@ static struct cpufreq_driver cppc_cpufreq_driver = { .name = "cppc_cpufreq", }; -/* - * HISI platform does not support delivered performance counter and - * reference performance counter. It can calculate the performance using the - * platform specific mechanism. We reuse the desired performance register to - * store the real performance calculated by the platform. - */ -static unsigned int hisi_cppc_cpufreq_get_rate(unsigned int cpu) -{ - struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); - struct cppc_cpudata *cpu_data; - u64 desired_perf; - int ret; - - if (!policy) - return -ENODEV; - - cpu_data = policy->driver_data; - - cpufreq_cpu_put(policy); - - ret = cppc_get_desired_perf(cpu, &desired_perf); - if (ret < 0) - return -EIO; - - return cppc_perf_to_khz(&cpu_data->perf_caps, desired_perf); -} - -static void cppc_check_hisi_workaround(void) -{ - struct acpi_table_header *tbl; - acpi_status status = AE_OK; - int i; - - status = acpi_get_table(ACPI_SIG_PCCT, 0, &tbl); - if (ACPI_FAILURE(status) || !tbl) - return; - - for (i = 0; i < ARRAY_SIZE(wa_info); i++) { - if (!memcmp(wa_info[i].oem_id, tbl->oem_id, ACPI_OEM_ID_SIZE) && - !memcmp(wa_info[i].oem_table_id, tbl->oem_table_id, ACPI_OEM_TABLE_ID_SIZE) && - wa_info[i].oem_revision == tbl->oem_revision) { - /* Overwrite the get() callback */ - cppc_cpufreq_driver.get = hisi_cppc_cpufreq_get_rate; - fie_disabled = FIE_DISABLED; - break; - } - } - - acpi_put_table(tbl); -} - static int __init cppc_cpufreq_init(void) { int ret; @@ -870,7 +841,6 @@ static int __init cppc_cpufreq_init(void) if (!acpi_cpc_valid()) return -ENODEV; - cppc_check_hisi_workaround(); cppc_freq_invariance_init(); populate_efficiency_class(); diff --git a/drivers/cpufreq/cpufreq-dt-platdev.c b/drivers/cpufreq/cpufreq-dt-platdev.c index 18942bfe9c95..2a3e8bd317c9 100644 --- a/drivers/cpufreq/cpufreq-dt-platdev.c +++ b/drivers/cpufreq/cpufreq-dt-platdev.c @@ -103,6 +103,7 @@ static const struct of_device_id allowlist[] __initconst = { * platforms using "operating-points-v2" property. */ static const struct of_device_id blocklist[] __initconst = { + { .compatible = "allwinner,sun50i-a100" }, { .compatible = "allwinner,sun50i-h6", }, { .compatible = "allwinner,sun50i-h616", }, { .compatible = "allwinner,sun50i-h618", }, diff --git a/drivers/cpufreq/cpufreq-dt.c b/drivers/cpufreq/cpufreq-dt.c index 983443396f8f..3a7c3372bda7 100644 --- a/drivers/cpufreq/cpufreq-dt.c +++ b/drivers/cpufreq/cpufreq-dt.c @@ -345,7 +345,7 @@ static struct platform_driver dt_cpufreq_platdrv = { .name = "cpufreq-dt", }, .probe = dt_cpufreq_probe, - .remove_new = dt_cpufreq_remove, + .remove = dt_cpufreq_remove, }; module_platform_driver(dt_cpufreq_platdrv); diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index f98c9438760c..1a4cae54a01b 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -1520,7 +1520,7 @@ static int cpufreq_online(unsigned int cpu) * frequency for longer duration. Hence, a BUG_ON(). */ BUG_ON(ret); - pr_info("%s: CPU%d: Running at unlisted initial frequency: %u KHz, changing to: %u KHz\n", + pr_info("%s: CPU%d: Running at unlisted initial frequency: %u kHz, changing to: %u kHz\n", __func__, policy->cpu, old_freq, policy->cur); } } diff --git a/drivers/cpufreq/davinci-cpufreq.c b/drivers/cpufreq/davinci-cpufreq.c index 7d2754411d8c..8736be3a06ce 100644 --- a/drivers/cpufreq/davinci-cpufreq.c +++ b/drivers/cpufreq/davinci-cpufreq.c @@ -145,7 +145,7 @@ static struct platform_driver davinci_cpufreq_driver = { .driver = { .name = "cpufreq-davinci", }, - .remove_new = __exit_p(davinci_cpufreq_remove), + .remove = __exit_p(davinci_cpufreq_remove), }; int __init davinci_cpufreq_init(void) diff --git a/drivers/cpufreq/imx-cpufreq-dt.c b/drivers/cpufreq/imx-cpufreq-dt.c index 577bb9e2f112..1492c92ffc1a 100644 --- a/drivers/cpufreq/imx-cpufreq-dt.c +++ b/drivers/cpufreq/imx-cpufreq-dt.c @@ -183,7 +183,7 @@ static void imx_cpufreq_dt_remove(struct platform_device *pdev) static struct platform_driver imx_cpufreq_dt_driver = { .probe = imx_cpufreq_dt_probe, - .remove_new = imx_cpufreq_dt_remove, + .remove = imx_cpufreq_dt_remove, .driver = { .name = "imx-cpufreq-dt", }, diff --git a/drivers/cpufreq/imx6q-cpufreq.c b/drivers/cpufreq/imx6q-cpufreq.c index c20d3ecc5a81..f3c99f378ad6 100644 --- a/drivers/cpufreq/imx6q-cpufreq.c +++ b/drivers/cpufreq/imx6q-cpufreq.c @@ -522,7 +522,7 @@ static struct platform_driver imx6q_cpufreq_platdrv = { .name = "imx6q-cpufreq", }, .probe = imx6q_cpufreq_probe, - .remove_new = imx6q_cpufreq_remove, + .remove = imx6q_cpufreq_remove, }; module_platform_driver(imx6q_cpufreq_platdrv); diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 400337f3b572..b8e2396a708a 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -3655,6 +3655,8 @@ static const struct x86_cpu_id intel_epp_default[] = { X86_MATCH_VFM(INTEL_ALDERLAKE_L, HWP_SET_DEF_BALANCE_PERF_EPP(102)), X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, HWP_SET_DEF_BALANCE_PERF_EPP(32)), X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, HWP_SET_DEF_BALANCE_PERF_EPP(32)), + X86_MATCH_VFM(INTEL_GRANITERAPIDS_X, HWP_SET_DEF_BALANCE_PERF_EPP(32)), + X86_MATCH_VFM(INTEL_GRANITERAPIDS_D, HWP_SET_DEF_BALANCE_PERF_EPP(32)), X86_MATCH_VFM(INTEL_METEORLAKE_L, HWP_SET_EPP_VALUES(HWP_EPP_POWERSAVE, 179, 64, 16)), X86_MATCH_VFM(INTEL_ARROWLAKE, HWP_SET_EPP_VALUES(HWP_EPP_POWERSAVE, diff --git a/drivers/cpufreq/kirkwood-cpufreq.c b/drivers/cpufreq/kirkwood-cpufreq.c index fd20b986d1f2..312f2654d1d5 100644 --- a/drivers/cpufreq/kirkwood-cpufreq.c +++ b/drivers/cpufreq/kirkwood-cpufreq.c @@ -189,7 +189,7 @@ static void kirkwood_cpufreq_remove(struct platform_device *pdev) static struct platform_driver kirkwood_cpufreq_platform_driver = { .probe = kirkwood_cpufreq_probe, - .remove_new = kirkwood_cpufreq_remove, + .remove = kirkwood_cpufreq_remove, .driver = { .name = "kirkwood-cpufreq", }, diff --git a/drivers/cpufreq/loongson2_cpufreq.c b/drivers/cpufreq/loongson2_cpufreq.c index 6a8e97896d38..ed1a6dbad638 100644 --- a/drivers/cpufreq/loongson2_cpufreq.c +++ b/drivers/cpufreq/loongson2_cpufreq.c @@ -148,7 +148,9 @@ static int __init cpufreq_init(void) ret = cpufreq_register_driver(&loongson2_cpufreq_driver); - if (!ret && !nowait) { + if (ret) { + platform_driver_unregister(&platform_driver); + } else if (!nowait) { saved_cpu_wait = cpu_wait; cpu_wait = loongson2_cpu_wait; } diff --git a/drivers/cpufreq/loongson3_cpufreq.c b/drivers/cpufreq/loongson3_cpufreq.c index 6b5e6798d9a2..bd34bf0fafa5 100644 --- a/drivers/cpufreq/loongson3_cpufreq.c +++ b/drivers/cpufreq/loongson3_cpufreq.c @@ -346,8 +346,11 @@ static int loongson3_cpufreq_probe(struct platform_device *pdev) { int i, ret; - for (i = 0; i < MAX_PACKAGES; i++) - devm_mutex_init(&pdev->dev, &cpufreq_mutex[i]); + for (i = 0; i < MAX_PACKAGES; i++) { + ret = devm_mutex_init(&pdev->dev, &cpufreq_mutex[i]); + if (ret) + return ret; + } ret = do_service_request(0, 0, CMD_GET_VERSION, 0, 0); if (ret <= 0) @@ -386,7 +389,7 @@ static struct platform_driver loongson3_platform_driver = { }, .id_table = cpufreq_id_table, .probe = loongson3_cpufreq_probe, - .remove_new = loongson3_cpufreq_remove, + .remove = loongson3_cpufreq_remove, }; module_platform_driver(loongson3_platform_driver); diff --git a/drivers/cpufreq/mediatek-cpufreq-hw.c b/drivers/cpufreq/mediatek-cpufreq-hw.c index 8925e096d5b9..9252ebd60373 100644 --- a/drivers/cpufreq/mediatek-cpufreq-hw.c +++ b/drivers/cpufreq/mediatek-cpufreq-hw.c @@ -62,7 +62,7 @@ mtk_cpufreq_get_cpu_power(struct device *cpu_dev, unsigned long *uW, policy = cpufreq_cpu_get_raw(cpu_dev->id); if (!policy) - return 0; + return -EINVAL; data = policy->driver_data; @@ -344,7 +344,7 @@ MODULE_DEVICE_TABLE(of, mtk_cpufreq_hw_match); static struct platform_driver mtk_cpufreq_hw_driver = { .probe = mtk_cpufreq_hw_driver_probe, - .remove_new = mtk_cpufreq_hw_driver_remove, + .remove = mtk_cpufreq_hw_driver_remove, .driver = { .name = "mtk-cpufreq-hw", .of_match_table = mtk_cpufreq_hw_match, diff --git a/drivers/cpufreq/omap-cpufreq.c b/drivers/cpufreq/omap-cpufreq.c index de8be0a8932d..106220c0fd11 100644 --- a/drivers/cpufreq/omap-cpufreq.c +++ b/drivers/cpufreq/omap-cpufreq.c @@ -188,7 +188,7 @@ static struct platform_driver omap_cpufreq_platdrv = { .name = "omap-cpufreq", }, .probe = omap_cpufreq_probe, - .remove_new = omap_cpufreq_remove, + .remove = omap_cpufreq_remove, }; module_platform_driver(omap_cpufreq_platdrv); diff --git a/drivers/cpufreq/pcc-cpufreq.c b/drivers/cpufreq/pcc-cpufreq.c index 771efbf51a48..ac2e90a65f0c 100644 --- a/drivers/cpufreq/pcc-cpufreq.c +++ b/drivers/cpufreq/pcc-cpufreq.c @@ -615,7 +615,7 @@ static struct platform_driver pcc_cpufreq_platdrv = { .driver = { .name = "pcc-cpufreq", }, - .remove_new = pcc_cpufreq_remove, + .remove = pcc_cpufreq_remove, }; static int __init pcc_cpufreq_init(void) diff --git a/drivers/cpufreq/qcom-cpufreq-hw.c b/drivers/cpufreq/qcom-cpufreq-hw.c index 900d6844c43d..98129565acb8 100644 --- a/drivers/cpufreq/qcom-cpufreq-hw.c +++ b/drivers/cpufreq/qcom-cpufreq-hw.c @@ -736,7 +736,7 @@ static void qcom_cpufreq_hw_driver_remove(struct platform_device *pdev) static struct platform_driver qcom_cpufreq_hw_driver = { .probe = qcom_cpufreq_hw_driver_probe, - .remove_new = qcom_cpufreq_hw_driver_remove, + .remove = qcom_cpufreq_hw_driver_remove, .driver = { .name = "qcom-cpufreq-hw", .of_match_table = qcom_cpufreq_hw_match, diff --git a/drivers/cpufreq/qcom-cpufreq-nvmem.c b/drivers/cpufreq/qcom-cpufreq-nvmem.c index 703308fb891a..3a8ed723a23e 100644 --- a/drivers/cpufreq/qcom-cpufreq-nvmem.c +++ b/drivers/cpufreq/qcom-cpufreq-nvmem.c @@ -52,12 +52,13 @@ struct qcom_cpufreq_match_data { struct nvmem_cell *speedbin_nvmem, char **pvs_name, struct qcom_cpufreq_drv *drv); - const char **genpd_names; + const char **pd_names; + unsigned int num_pd_names; }; struct qcom_cpufreq_drv_cpu { int opp_token; - struct device **virt_devs; + struct dev_pm_domain_list *pd_list; }; struct qcom_cpufreq_drv { @@ -395,8 +396,6 @@ static int qcom_cpufreq_ipq8074_name_version(struct device *cpu_dev, return 0; } -static const char *generic_genpd_names[] = { "perf", NULL }; - static const struct qcom_cpufreq_match_data match_data_kryo = { .get_version = qcom_cpufreq_kryo_name_version, }; @@ -407,13 +406,13 @@ static const struct qcom_cpufreq_match_data match_data_krait = { static const struct qcom_cpufreq_match_data match_data_msm8909 = { .get_version = qcom_cpufreq_simple_get_version, - .genpd_names = generic_genpd_names, + .pd_names = (const char *[]) { "perf" }, + .num_pd_names = 1, }; -static const char *qcs404_genpd_names[] = { "cpr", NULL }; - static const struct qcom_cpufreq_match_data match_data_qcs404 = { - .genpd_names = qcs404_genpd_names, + .pd_names = (const char *[]) { "cpr" }, + .num_pd_names = 1, }; static const struct qcom_cpufreq_match_data match_data_ipq6018 = { @@ -428,28 +427,16 @@ static const struct qcom_cpufreq_match_data match_data_ipq8074 = { .get_version = qcom_cpufreq_ipq8074_name_version, }; -static void qcom_cpufreq_suspend_virt_devs(struct qcom_cpufreq_drv *drv, unsigned int cpu) -{ - const char * const *name = drv->data->genpd_names; - int i; - - if (!drv->cpus[cpu].virt_devs) - return; - - for (i = 0; *name; i++, name++) - device_set_awake_path(drv->cpus[cpu].virt_devs[i]); -} - -static void qcom_cpufreq_put_virt_devs(struct qcom_cpufreq_drv *drv, unsigned int cpu) +static void qcom_cpufreq_suspend_pd_devs(struct qcom_cpufreq_drv *drv, unsigned int cpu) { - const char * const *name = drv->data->genpd_names; + struct dev_pm_domain_list *pd_list = drv->cpus[cpu].pd_list; int i; - if (!drv->cpus[cpu].virt_devs) + if (!pd_list) return; - for (i = 0; *name; i++, name++) - pm_runtime_put(drv->cpus[cpu].virt_devs[i]); + for (i = 0; i < pd_list->num_pds; i++) + device_set_awake_path(pd_list->pd_devs[i]); } static int qcom_cpufreq_probe(struct platform_device *pdev) @@ -503,7 +490,6 @@ static int qcom_cpufreq_probe(struct platform_device *pdev) } for_each_possible_cpu(cpu) { - struct device **virt_devs = NULL; struct dev_pm_opp_config config = { .supported_hw = NULL, }; @@ -522,12 +508,7 @@ static int qcom_cpufreq_probe(struct platform_device *pdev) config.prop_name = pvs_name; } - if (drv->data->genpd_names) { - config.genpd_names = drv->data->genpd_names; - config.virt_devs = &virt_devs; - } - - if (config.supported_hw || config.genpd_names) { + if (config.supported_hw) { drv->cpus[cpu].opp_token = dev_pm_opp_set_config(cpu_dev, &config); if (drv->cpus[cpu].opp_token < 0) { ret = drv->cpus[cpu].opp_token; @@ -536,25 +517,18 @@ static int qcom_cpufreq_probe(struct platform_device *pdev) } } - if (virt_devs) { - const char * const *name = config.genpd_names; - int i, j; - - for (i = 0; *name; i++, name++) { - ret = pm_runtime_resume_and_get(virt_devs[i]); - if (ret) { - dev_err(cpu_dev, "failed to resume %s: %d\n", - *name, ret); - - /* Rollback previous PM runtime calls */ - name = config.genpd_names; - for (j = 0; *name && j < i; j++, name++) - pm_runtime_put(virt_devs[j]); - - goto free_opp; - } - } - drv->cpus[cpu].virt_devs = virt_devs; + if (drv->data->pd_names) { + struct dev_pm_domain_attach_data attach_data = { + .pd_names = drv->data->pd_names, + .num_pd_names = drv->data->num_pd_names, + .pd_flags = PD_FLAG_DEV_LINK_ON | + PD_FLAG_REQUIRED_OPP, + }; + + ret = dev_pm_domain_attach_list(cpu_dev, &attach_data, + &drv->cpus[cpu].pd_list); + if (ret < 0) + goto free_opp; } } @@ -570,7 +544,7 @@ static int qcom_cpufreq_probe(struct platform_device *pdev) free_opp: for_each_possible_cpu(cpu) { - qcom_cpufreq_put_virt_devs(drv, cpu); + dev_pm_domain_detach_list(drv->cpus[cpu].pd_list); dev_pm_opp_clear_config(drv->cpus[cpu].opp_token); } return ret; @@ -584,7 +558,7 @@ static void qcom_cpufreq_remove(struct platform_device *pdev) platform_device_unregister(cpufreq_dt_pdev); for_each_possible_cpu(cpu) { - qcom_cpufreq_put_virt_devs(drv, cpu); + dev_pm_domain_detach_list(drv->cpus[cpu].pd_list); dev_pm_opp_clear_config(drv->cpus[cpu].opp_token); } } @@ -595,7 +569,7 @@ static int qcom_cpufreq_suspend(struct device *dev) unsigned int cpu; for_each_possible_cpu(cpu) - qcom_cpufreq_suspend_virt_devs(drv, cpu); + qcom_cpufreq_suspend_pd_devs(drv, cpu); return 0; } @@ -604,7 +578,7 @@ static DEFINE_SIMPLE_DEV_PM_OPS(qcom_cpufreq_pm_ops, qcom_cpufreq_suspend, NULL) static struct platform_driver qcom_cpufreq_driver = { .probe = qcom_cpufreq_probe, - .remove_new = qcom_cpufreq_remove, + .remove = qcom_cpufreq_remove, .driver = { .name = "qcom-cpufreq-nvmem", .pm = pm_sleep_ptr(&qcom_cpufreq_pm_ops), diff --git a/drivers/cpufreq/qoriq-cpufreq.c b/drivers/cpufreq/qoriq-cpufreq.c index 3519bf34d397..a37ce051236c 100644 --- a/drivers/cpufreq/qoriq-cpufreq.c +++ b/drivers/cpufreq/qoriq-cpufreq.c @@ -296,7 +296,7 @@ static struct platform_driver qoriq_cpufreq_platform_driver = { .name = "qoriq-cpufreq", }, .probe = qoriq_cpufreq_probe, - .remove_new = qoriq_cpufreq_remove, + .remove = qoriq_cpufreq_remove, }; module_platform_driver(qoriq_cpufreq_platform_driver); diff --git a/drivers/cpufreq/raspberrypi-cpufreq.c b/drivers/cpufreq/raspberrypi-cpufreq.c index e0705cc9a57d..5050932954e3 100644 --- a/drivers/cpufreq/raspberrypi-cpufreq.c +++ b/drivers/cpufreq/raspberrypi-cpufreq.c @@ -85,7 +85,7 @@ static struct platform_driver raspberrypi_cpufreq_driver = { .name = "raspberrypi-cpufreq", }, .probe = raspberrypi_cpufreq_probe, - .remove_new = raspberrypi_cpufreq_remove, + .remove = raspberrypi_cpufreq_remove, }; module_platform_driver(raspberrypi_cpufreq_driver); diff --git a/drivers/cpufreq/scmi-cpufreq.c b/drivers/cpufreq/scmi-cpufreq.c index 5892c73e129d..07d6f9a9b7c8 100644 --- a/drivers/cpufreq/scmi-cpufreq.c +++ b/drivers/cpufreq/scmi-cpufreq.c @@ -287,7 +287,7 @@ static int scmi_cpufreq_init(struct cpufreq_policy *policy) ret = cpufreq_enable_boost_support(); if (ret) { dev_warn(cpu_dev, "failed to enable boost: %d\n", ret); - goto out_free_opp; + goto out_free_table; } else { scmi_cpufreq_hw_attr[1] = &cpufreq_freq_attr_scaling_boost_freqs; scmi_cpufreq_driver.boost_enabled = true; @@ -296,6 +296,8 @@ static int scmi_cpufreq_init(struct cpufreq_policy *policy) return 0; +out_free_table: + dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table); out_free_opp: dev_pm_opp_remove_all_dynamic(cpu_dev); diff --git a/drivers/cpufreq/scpi-cpufreq.c b/drivers/cpufreq/scpi-cpufreq.c index 8d73e6e8be2a..cd89c1b9832c 100644 --- a/drivers/cpufreq/scpi-cpufreq.c +++ b/drivers/cpufreq/scpi-cpufreq.c @@ -217,7 +217,7 @@ static struct platform_driver scpi_cpufreq_platdrv = { .name = "scpi-cpufreq", }, .probe = scpi_cpufreq_probe, - .remove_new = scpi_cpufreq_remove, + .remove = scpi_cpufreq_remove, }; module_platform_driver(scpi_cpufreq_platdrv); diff --git a/drivers/cpufreq/sun50i-cpufreq-nvmem.c b/drivers/cpufreq/sun50i-cpufreq-nvmem.c index 293921acec93..17d6a149f580 100644 --- a/drivers/cpufreq/sun50i-cpufreq-nvmem.c +++ b/drivers/cpufreq/sun50i-cpufreq-nvmem.c @@ -22,6 +22,9 @@ #define NVMEM_MASK 0x7 #define NVMEM_SHIFT 5 +#define SUN50I_A100_NVMEM_MASK 0xf +#define SUN50I_A100_NVMEM_SHIFT 12 + static struct platform_device *cpufreq_dt_pdev, *sun50i_cpufreq_pdev; struct sunxi_cpufreq_data { @@ -45,6 +48,23 @@ static u32 sun50i_h6_efuse_xlate(u32 speedbin) return 0; } +static u32 sun50i_a100_efuse_xlate(u32 speedbin) +{ + u32 efuse_value; + + efuse_value = (speedbin >> SUN50I_A100_NVMEM_SHIFT) & + SUN50I_A100_NVMEM_MASK; + + switch (efuse_value) { + case 0b100: + return 2; + case 0b010: + return 1; + default: + return 0; + } +} + static int get_soc_id_revision(void) { #ifdef CONFIG_HAVE_ARM_SMCCC_DISCOVERY @@ -108,6 +128,10 @@ static struct sunxi_cpufreq_data sun50i_h6_cpufreq_data = { .efuse_xlate = sun50i_h6_efuse_xlate, }; +static struct sunxi_cpufreq_data sun50i_a100_cpufreq_data = { + .efuse_xlate = sun50i_a100_efuse_xlate, +}; + static struct sunxi_cpufreq_data sun50i_h616_cpufreq_data = { .efuse_xlate = sun50i_h616_efuse_xlate, }; @@ -116,6 +140,9 @@ static const struct of_device_id cpu_opp_match_list[] = { { .compatible = "allwinner,sun50i-h6-operating-points", .data = &sun50i_h6_cpufreq_data, }, + { .compatible = "allwinner,sun50i-a100-operating-points", + .data = &sun50i_a100_cpufreq_data, + }, { .compatible = "allwinner,sun50i-h616-operating-points", .data = &sun50i_h616_cpufreq_data, }, @@ -283,7 +310,7 @@ static void sun50i_cpufreq_nvmem_remove(struct platform_device *pdev) static struct platform_driver sun50i_cpufreq_driver = { .probe = sun50i_cpufreq_nvmem_probe, - .remove_new = sun50i_cpufreq_nvmem_remove, + .remove = sun50i_cpufreq_nvmem_remove, .driver = { .name = "sun50i-cpufreq-nvmem", }, @@ -291,6 +318,7 @@ static struct platform_driver sun50i_cpufreq_driver = { static const struct of_device_id sun50i_cpufreq_match_list[] = { { .compatible = "allwinner,sun50i-h6" }, + { .compatible = "allwinner,sun50i-a100" }, { .compatible = "allwinner,sun50i-h616" }, { .compatible = "allwinner,sun50i-h618" }, { .compatible = "allwinner,sun50i-h700" }, diff --git a/drivers/cpufreq/tegra186-cpufreq.c b/drivers/cpufreq/tegra186-cpufreq.c index 7b8fcfa55038..c7761eb99f3c 100644 --- a/drivers/cpufreq/tegra186-cpufreq.c +++ b/drivers/cpufreq/tegra186-cpufreq.c @@ -276,7 +276,7 @@ static struct platform_driver tegra186_cpufreq_platform_driver = { .of_match_table = tegra186_cpufreq_of_match, }, .probe = tegra186_cpufreq_probe, - .remove_new = tegra186_cpufreq_remove, + .remove = tegra186_cpufreq_remove, }; module_platform_driver(tegra186_cpufreq_platform_driver); diff --git a/drivers/cpufreq/tegra194-cpufreq.c b/drivers/cpufreq/tegra194-cpufreq.c index 07ea7ed61b68..9055dd398e7f 100644 --- a/drivers/cpufreq/tegra194-cpufreq.c +++ b/drivers/cpufreq/tegra194-cpufreq.c @@ -818,7 +818,7 @@ static struct platform_driver tegra194_ccplex_driver = { .of_match_table = tegra194_cpufreq_of_match, }, .probe = tegra194_cpufreq_probe, - .remove_new = tegra194_cpufreq_remove, + .remove = tegra194_cpufreq_remove, }; module_platform_driver(tegra194_ccplex_driver); diff --git a/drivers/cpufreq/ti-cpufreq.c b/drivers/cpufreq/ti-cpufreq.c index ba621ce1cdda..5a5147277cd0 100644 --- a/drivers/cpufreq/ti-cpufreq.c +++ b/drivers/cpufreq/ti-cpufreq.c @@ -93,6 +93,8 @@ struct ti_cpufreq_soc_data { bool multi_regulator; /* Backward compatibility hack: Might have missing syscon */ #define TI_QUIRK_SYSCON_MAY_BE_MISSING 0x1 +/* Backward compatibility hack: new syscon size is 1 register wide */ +#define TI_QUIRK_SYSCON_IS_SINGLE_REG 0x2 u8 quirks; }; @@ -316,8 +318,8 @@ static struct ti_cpufreq_soc_data am625_soc_data = { .efuse_offset = 0x0018, .efuse_mask = 0x07c0, .efuse_shift = 0x6, - .rev_offset = 0x0014, .multi_regulator = false, + .quirks = TI_QUIRK_SYSCON_IS_SINGLE_REG, }; static struct ti_cpufreq_soc_data am62a7_soc_data = { @@ -325,7 +327,6 @@ static struct ti_cpufreq_soc_data am62a7_soc_data = { .efuse_offset = 0x0, .efuse_mask = 0x07c0, .efuse_shift = 0x6, - .rev_offset = 0x0014, .multi_regulator = false, }; @@ -334,7 +335,6 @@ static struct ti_cpufreq_soc_data am62p5_soc_data = { .efuse_offset = 0x0, .efuse_mask = 0x07c0, .efuse_shift = 0x6, - .rev_offset = 0x0014, .multi_regulator = false, }; @@ -354,6 +354,10 @@ static int ti_cpufreq_get_efuse(struct ti_cpufreq_data *opp_data, ret = regmap_read(opp_data->syscon, opp_data->soc_data->efuse_offset, &efuse); + + if (opp_data->soc_data->quirks & TI_QUIRK_SYSCON_IS_SINGLE_REG && ret == -EIO) + ret = regmap_read(opp_data->syscon, 0x0, &efuse); + if (opp_data->soc_data->quirks & TI_QUIRK_SYSCON_MAY_BE_MISSING && ret == -EIO) { /* not a syscon register! */ void __iomem *regs = ioremap(OMAP3_SYSCON_BASE + diff --git a/drivers/cpufreq/vexpress-spc-cpufreq.c b/drivers/cpufreq/vexpress-spc-cpufreq.c index 3fadf536c429..0f86cdb7ec8a 100644 --- a/drivers/cpufreq/vexpress-spc-cpufreq.c +++ b/drivers/cpufreq/vexpress-spc-cpufreq.c @@ -565,7 +565,7 @@ static struct platform_driver ve_spc_cpufreq_platdrv = { .name = "vexpress-spc-cpufreq", }, .probe = ve_spc_cpufreq_probe, - .remove_new = ve_spc_cpufreq_remove, + .remove = ve_spc_cpufreq_remove, }; module_platform_driver(ve_spc_cpufreq_platdrv); diff --git a/drivers/cpufreq/virtual-cpufreq.c b/drivers/cpufreq/virtual-cpufreq.c new file mode 100644 index 000000000000..a050b3a6737f --- /dev/null +++ b/drivers/cpufreq/virtual-cpufreq.c @@ -0,0 +1,333 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2024 Google LLC + */ + +#include <linux/arch_topology.h> +#include <linux/cpufreq.h> +#include <linux/init.h> +#include <linux/sched.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of_address.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> +#include <linux/slab.h> + +/* + * CPU0..CPUn + * +-------------+-------------------------------+--------+-------+ + * | Register | Description | Offset | Len | + * +-------------+-------------------------------+--------+-------+ + * | cur_perf | read this register to get | 0x0 | 0x4 | + * | | the current perf (integer val | | | + * | | representing perf relative to | | | + * | | max performance) | | | + * | | that vCPU is running at | | | + * +-------------+-------------------------------+--------+-------+ + * | set_perf | write to this register to set | 0x4 | 0x4 | + * | | perf value of the vCPU | | | + * +-------------+-------------------------------+--------+-------+ + * | perftbl_len | number of entries in perf | 0x8 | 0x4 | + * | | table. A single entry in the | | | + * | | perf table denotes no table | | | + * | | and the entry contains | | | + * | | the maximum perf value | | | + * | | that this vCPU supports. | | | + * | | The guest can request any | | | + * | | value between 1 and max perf | | | + * | | when perftbls are not used. | | | + * +---------------------------------------------+--------+-------+ + * | perftbl_sel | write to this register to | 0xc | 0x4 | + * | | select perf table entry to | | | + * | | read from | | | + * +---------------------------------------------+--------+-------+ + * | perftbl_rd | read this register to get | 0x10 | 0x4 | + * | | perf value of the selected | | | + * | | entry based on perftbl_sel | | | + * +---------------------------------------------+--------+-------+ + * | perf_domain | performance domain number | 0x14 | 0x4 | + * | | that this vCPU belongs to. | | | + * | | vCPUs sharing the same perf | | | + * | | domain number are part of the | | | + * | | same performance domain. | | | + * +-------------+-------------------------------+--------+-------+ + */ + +#define REG_CUR_PERF_STATE_OFFSET 0x0 +#define REG_SET_PERF_STATE_OFFSET 0x4 +#define REG_PERFTBL_LEN_OFFSET 0x8 +#define REG_PERFTBL_SEL_OFFSET 0xc +#define REG_PERFTBL_RD_OFFSET 0x10 +#define REG_PERF_DOMAIN_OFFSET 0x14 +#define PER_CPU_OFFSET 0x1000 + +#define PERFTBL_MAX_ENTRIES 64U + +static void __iomem *base; +static DEFINE_PER_CPU(u32, perftbl_num_entries); + +static void virt_scale_freq_tick(void) +{ + int cpu = smp_processor_id(); + u32 max_freq = (u32)cpufreq_get_hw_max_freq(cpu); + u64 cur_freq; + unsigned long scale; + + cur_freq = (u64)readl_relaxed(base + cpu * PER_CPU_OFFSET + + REG_CUR_PERF_STATE_OFFSET); + + cur_freq <<= SCHED_CAPACITY_SHIFT; + scale = (unsigned long)div_u64(cur_freq, max_freq); + scale = min(scale, SCHED_CAPACITY_SCALE); + + this_cpu_write(arch_freq_scale, scale); +} + +static struct scale_freq_data virt_sfd = { + .source = SCALE_FREQ_SOURCE_VIRT, + .set_freq_scale = virt_scale_freq_tick, +}; + +static unsigned int virt_cpufreq_set_perf(struct cpufreq_policy *policy, + unsigned int target_freq) +{ + writel_relaxed(target_freq, + base + policy->cpu * PER_CPU_OFFSET + REG_SET_PERF_STATE_OFFSET); + return 0; +} + +static unsigned int virt_cpufreq_fast_switch(struct cpufreq_policy *policy, + unsigned int target_freq) +{ + virt_cpufreq_set_perf(policy, target_freq); + return target_freq; +} + +static u32 virt_cpufreq_get_perftbl_entry(int cpu, u32 idx) +{ + writel_relaxed(idx, base + cpu * PER_CPU_OFFSET + + REG_PERFTBL_SEL_OFFSET); + return readl_relaxed(base + cpu * PER_CPU_OFFSET + + REG_PERFTBL_RD_OFFSET); +} + +static int virt_cpufreq_target(struct cpufreq_policy *policy, + unsigned int target_freq, + unsigned int relation) +{ + struct cpufreq_freqs freqs; + int ret = 0; + + freqs.old = policy->cur; + freqs.new = target_freq; + + cpufreq_freq_transition_begin(policy, &freqs); + ret = virt_cpufreq_set_perf(policy, target_freq); + cpufreq_freq_transition_end(policy, &freqs, ret != 0); + + return ret; +} + +static int virt_cpufreq_get_sharing_cpus(struct cpufreq_policy *policy) +{ + u32 cur_perf_domain, perf_domain; + struct device *cpu_dev; + int cpu; + + cur_perf_domain = readl_relaxed(base + policy->cpu * + PER_CPU_OFFSET + REG_PERF_DOMAIN_OFFSET); + + for_each_possible_cpu(cpu) { + cpu_dev = get_cpu_device(cpu); + if (!cpu_dev) + continue; + + perf_domain = readl_relaxed(base + cpu * + PER_CPU_OFFSET + REG_PERF_DOMAIN_OFFSET); + + if (perf_domain == cur_perf_domain) + cpumask_set_cpu(cpu, policy->cpus); + } + + return 0; +} + +static int virt_cpufreq_get_freq_info(struct cpufreq_policy *policy) +{ + struct cpufreq_frequency_table *table; + u32 num_perftbl_entries, idx; + + num_perftbl_entries = per_cpu(perftbl_num_entries, policy->cpu); + + if (num_perftbl_entries == 1) { + policy->cpuinfo.min_freq = 1; + policy->cpuinfo.max_freq = virt_cpufreq_get_perftbl_entry(policy->cpu, 0); + + policy->min = policy->cpuinfo.min_freq; + policy->max = policy->cpuinfo.max_freq; + + policy->cur = policy->max; + return 0; + } + + table = kcalloc(num_perftbl_entries + 1, sizeof(*table), GFP_KERNEL); + if (!table) + return -ENOMEM; + + for (idx = 0; idx < num_perftbl_entries; idx++) + table[idx].frequency = virt_cpufreq_get_perftbl_entry(policy->cpu, idx); + + table[idx].frequency = CPUFREQ_TABLE_END; + policy->freq_table = table; + + return 0; +} + +static int virt_cpufreq_cpu_init(struct cpufreq_policy *policy) +{ + struct device *cpu_dev; + int ret; + + cpu_dev = get_cpu_device(policy->cpu); + if (!cpu_dev) + return -ENODEV; + + ret = virt_cpufreq_get_freq_info(policy); + if (ret) { + dev_warn(cpu_dev, "failed to get cpufreq info\n"); + return ret; + } + + ret = virt_cpufreq_get_sharing_cpus(policy); + if (ret) { + dev_warn(cpu_dev, "failed to get sharing cpumask\n"); + return ret; + } + + /* + * To simplify and improve latency of handling frequency requests on + * the host side, this ensures that the vCPU thread triggering the MMIO + * abort is the same thread whose performance constraints (Ex. uclamp + * settings) need to be updated. This simplifies the VMM (Virtual + * Machine Manager) having to find the correct vCPU thread and/or + * facing permission issues when configuring other threads. + */ + policy->dvfs_possible_from_any_cpu = false; + policy->fast_switch_possible = true; + + /* + * Using the default SCALE_FREQ_SOURCE_CPUFREQ is insufficient since + * the actual physical CPU frequency may not match requested frequency + * from the vCPU thread due to frequency update latencies or other + * inputs to the physical CPU frequency selection. This additional FIE + * source allows for more accurate freq_scale updates and only takes + * effect if another FIE source such as AMUs have not been registered. + */ + topology_set_scale_freq_source(&virt_sfd, policy->cpus); + + return 0; +} + +static void virt_cpufreq_cpu_exit(struct cpufreq_policy *policy) +{ + topology_clear_scale_freq_source(SCALE_FREQ_SOURCE_VIRT, policy->related_cpus); + kfree(policy->freq_table); +} + +static int virt_cpufreq_online(struct cpufreq_policy *policy) +{ + /* Nothing to restore. */ + return 0; +} + +static int virt_cpufreq_offline(struct cpufreq_policy *policy) +{ + /* Dummy offline() to avoid exit() being called and freeing resources. */ + return 0; +} + +static int virt_cpufreq_verify_policy(struct cpufreq_policy_data *policy) +{ + if (policy->freq_table) + return cpufreq_frequency_table_verify(policy, policy->freq_table); + + cpufreq_verify_within_cpu_limits(policy); + return 0; +} + +static struct cpufreq_driver cpufreq_virt_driver = { + .name = "virt-cpufreq", + .init = virt_cpufreq_cpu_init, + .exit = virt_cpufreq_cpu_exit, + .online = virt_cpufreq_online, + .offline = virt_cpufreq_offline, + .verify = virt_cpufreq_verify_policy, + .target = virt_cpufreq_target, + .fast_switch = virt_cpufreq_fast_switch, + .attr = cpufreq_generic_attr, +}; + +static int virt_cpufreq_driver_probe(struct platform_device *pdev) +{ + u32 num_perftbl_entries; + int ret, cpu; + + base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(base)) + return PTR_ERR(base); + + for_each_possible_cpu(cpu) { + num_perftbl_entries = readl_relaxed(base + cpu * PER_CPU_OFFSET + + REG_PERFTBL_LEN_OFFSET); + + if (!num_perftbl_entries || num_perftbl_entries > PERFTBL_MAX_ENTRIES) + return -ENODEV; + + per_cpu(perftbl_num_entries, cpu) = num_perftbl_entries; + } + + ret = cpufreq_register_driver(&cpufreq_virt_driver); + if (ret) { + dev_err(&pdev->dev, "Virtual CPUFreq driver failed to register: %d\n", ret); + return ret; + } + + dev_dbg(&pdev->dev, "Virtual CPUFreq driver initialized\n"); + return 0; +} + +static void virt_cpufreq_driver_remove(struct platform_device *pdev) +{ + cpufreq_unregister_driver(&cpufreq_virt_driver); +} + +static const struct of_device_id virt_cpufreq_match[] = { + { .compatible = "qemu,virtual-cpufreq", .data = NULL}, + {} +}; +MODULE_DEVICE_TABLE(of, virt_cpufreq_match); + +static struct platform_driver virt_cpufreq_driver = { + .probe = virt_cpufreq_driver_probe, + .remove = virt_cpufreq_driver_remove, + .driver = { + .name = "virt-cpufreq", + .of_match_table = virt_cpufreq_match, + }, +}; + +static int __init virt_cpufreq_init(void) +{ + return platform_driver_register(&virt_cpufreq_driver); +} +postcore_initcall(virt_cpufreq_init); + +static void __exit virt_cpufreq_exit(void) +{ + platform_driver_unregister(&virt_cpufreq_driver); +} +module_exit(virt_cpufreq_exit); + +MODULE_DESCRIPTION("Virtual cpufreq driver"); +MODULE_LICENSE("GPL"); |