diff options
80 files changed, 3076 insertions, 456 deletions
diff --git a/Documentation/admin-guide/perf/hisi-pcie-pmu.rst b/Documentation/admin-guide/perf/hisi-pcie-pmu.rst index 7e863662e2d4..5541ff40e06a 100644 --- a/Documentation/admin-guide/perf/hisi-pcie-pmu.rst +++ b/Documentation/admin-guide/perf/hisi-pcie-pmu.rst @@ -37,9 +37,21 @@ Example usage of perf:: hisi_pcie0_core0/rx_mwr_cnt/ [kernel PMU event] ------------------------------------------ - $# perf stat -e hisi_pcie0_core0/rx_mwr_latency/ - $# perf stat -e hisi_pcie0_core0/rx_mwr_cnt/ - $# perf stat -g -e hisi_pcie0_core0/rx_mwr_latency/ -e hisi_pcie0_core0/rx_mwr_cnt/ + $# perf stat -e hisi_pcie0_core0/rx_mwr_latency,port=0xffff/ + $# perf stat -e hisi_pcie0_core0/rx_mwr_cnt,port=0xffff/ + +The related events usually used to calculate the bandwidth, latency or others. +They need to start and end counting at the same time, therefore related events +are best used in the same event group to get the expected value. There are two +ways to know if they are related events: + +a) By event name, such as the latency events "xxx_latency, xxx_cnt" or + bandwidth events "xxx_flux, xxx_time". +b) By event type, such as "event=0xXXXX, event=0x1XXXX". + +Example usage of perf group:: + + $# perf stat -e "{hisi_pcie0_core0/rx_mwr_latency,port=0xffff/,hisi_pcie0_core0/rx_mwr_cnt,port=0xffff/}" The current driver does not support sampling. So "perf record" is unsupported. Also attach to a task is unsupported for PCIe PMU. @@ -51,8 +63,12 @@ Filter options PMU could only monitor the performance of traffic downstream target Root Ports or downstream target Endpoint. PCIe PMU driver support "port" and - "bdf" interfaces for users, and these two interfaces aren't supported at the - same time. + "bdf" interfaces for users. + Please notice that, one of these two interfaces must be set, and these two + interfaces aren't supported at the same time. If they are both set, only + "port" filter is valid. + If "port" filter not being set or is set explicitly to zero (default), the + "bdf" filter will be in effect, because "bdf=0" meaning 0000:000:00.0. - port @@ -95,7 +111,7 @@ Filter options Example usage of perf:: - $# perf stat -e hisi_pcie0_core0/rx_mrd_flux,trig_len=0x4,trig_mode=1/ sleep 5 + $# perf stat -e hisi_pcie0_core0/rx_mrd_flux,port=0xffff,trig_len=0x4,trig_mode=1/ sleep 5 3. Threshold filter @@ -109,7 +125,7 @@ Filter options Example usage of perf:: - $# perf stat -e hisi_pcie0_core0/rx_mrd_flux,thr_len=0x4,thr_mode=1/ sleep 5 + $# perf stat -e hisi_pcie0_core0/rx_mrd_flux,port=0xffff,thr_len=0x4,thr_mode=1/ sleep 5 4. TLP Length filter @@ -127,4 +143,4 @@ Filter options Example usage of perf:: - $# perf stat -e hisi_pcie0_core0/rx_mrd_flux,len_mode=0x1/ sleep 5 + $# perf stat -e hisi_pcie0_core0/rx_mrd_flux,port=0xffff,len_mode=0x1/ sleep 5 diff --git a/Documentation/admin-guide/perf/index.rst b/Documentation/admin-guide/perf/index.rst index f4a4513c526f..7eb3dcd6f4da 100644 --- a/Documentation/admin-guide/perf/index.rst +++ b/Documentation/admin-guide/perf/index.rst @@ -13,6 +13,7 @@ Performance monitor support imx-ddr qcom_l2_pmu qcom_l3_pmu + starfive_starlink_pmu arm-ccn arm-cmn xgene-pmu diff --git a/Documentation/admin-guide/perf/starfive_starlink_pmu.rst b/Documentation/admin-guide/perf/starfive_starlink_pmu.rst new file mode 100644 index 000000000000..2932ddb4eb76 --- /dev/null +++ b/Documentation/admin-guide/perf/starfive_starlink_pmu.rst @@ -0,0 +1,46 @@ +================================================ +StarFive StarLink Performance Monitor Unit (PMU) +================================================ + +StarFive StarLink Performance Monitor Unit (PMU) exists within the +StarLink Coherent Network on Chip (CNoC) that connects multiple CPU +clusters with an L3 memory system. + +The uncore PMU supports overflow interrupt, up to 16 programmable 64bit +event counters, and an independent 64bit cycle counter. +The PMU can only be accessed via Memory Mapped I/O and are common to the +cores connected to the same PMU. + +Driver exposes supported PMU events in sysfs "events" directory under:: + + /sys/bus/event_source/devices/starfive_starlink_pmu/events/ + +Driver exposes cpu used to handle PMU events in sysfs "cpumask" directory +under:: + + /sys/bus/event_source/devices/starfive_starlink_pmu/cpumask/ + +Driver describes the format of config (event ID) in sysfs "format" directory +under:: + + /sys/bus/event_source/devices/starfive_starlink_pmu/format/ + +Example of perf usage:: + + $ perf list + + starfive_starlink_pmu/cycles/ [Kernel PMU event] + starfive_starlink_pmu/read_hit/ [Kernel PMU event] + starfive_starlink_pmu/read_miss/ [Kernel PMU event] + starfive_starlink_pmu/read_request/ [Kernel PMU event] + starfive_starlink_pmu/release_request/ [Kernel PMU event] + starfive_starlink_pmu/write_hit/ [Kernel PMU event] + starfive_starlink_pmu/write_miss/ [Kernel PMU event] + starfive_starlink_pmu/write_request/ [Kernel PMU event] + starfive_starlink_pmu/writeback/ [Kernel PMU event] + + + $ perf stat -a -e /starfive_starlink_pmu/cycles/ sleep 1 + +Sampling is not supported. As a result, "perf record" is not supported. +Attaching to a task is not supported, only system-wide counting is supported. diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst index e8c2ce1f9df6..fa16b895c997 100644 --- a/Documentation/arch/arm64/silicon-errata.rst +++ b/Documentation/arch/arm64/silicon-errata.rst @@ -35,8 +35,9 @@ can be triggered by Linux). For software workarounds that may adversely impact systems unaffected by the erratum in question, a Kconfig entry is added under "Kernel Features" -> "ARM errata workarounds via the alternatives framework". -These are enabled by default and patched in at runtime when an affected -CPU is detected. For less-intrusive workarounds, a Kconfig option is not +With the exception of workarounds for errata deemed "rare" by Arm, these +are enabled by default and patched in at runtime when an affected CPU is +detected. For less-intrusive workarounds, a Kconfig option is not available and the code is structured (preferably with a comment) in such a way that the erratum will not be hit. diff --git a/Documentation/arch/arm64/sme.rst b/Documentation/arch/arm64/sme.rst index 3d0e53ecac4f..be317d457417 100644 --- a/Documentation/arch/arm64/sme.rst +++ b/Documentation/arch/arm64/sme.rst @@ -75,7 +75,7 @@ model features for SME is included in Appendix A. 2. Vector lengths ------------------ -SME defines a second vector length similar to the SVE vector length which is +SME defines a second vector length similar to the SVE vector length which controls the size of the streaming mode SVE vectors and the ZA matrix array. The ZA matrix is square with each side having as many bytes as a streaming mode SVE vector. @@ -238,12 +238,12 @@ prctl(PR_SME_SET_VL, unsigned long arg) bits of Z0..Z31 except for Z0 bits [127:0] .. Z31 bits [127:0] to become unspecified, including both streaming and non-streaming SVE state. Calling PR_SME_SET_VL with vl equal to the thread's current vector - length, or calling PR_SME_SET_VL with the PR_SVE_SET_VL_ONEXEC flag, + length, or calling PR_SME_SET_VL with the PR_SME_SET_VL_ONEXEC flag, does not constitute a change to the vector length for this purpose. * Changing the vector length causes PSTATE.ZA and PSTATE.SM to be cleared. Calling PR_SME_SET_VL with vl equal to the thread's current vector - length, or calling PR_SME_SET_VL with the PR_SVE_SET_VL_ONEXEC flag, + length, or calling PR_SME_SET_VL with the PR_SME_SET_VL_ONEXEC flag, does not constitute a change to the vector length for this purpose. @@ -379,9 +379,8 @@ The regset data starts with struct user_za_header, containing: /proc/sys/abi/sme_default_vector_length Writing the text representation of an integer to this file sets the system - default vector length to the specified value, unless the value is greater - than the maximum vector length supported by the system in which case the - default vector length is set to that maximum. + default vector length to the specified value rounded to a supported value + using the same rules as for setting vector length via PR_SME_SET_VL. The result can be determined by reopening the file and reading its contents. diff --git a/Documentation/arch/arm64/sve.rst b/Documentation/arch/arm64/sve.rst index 0d9a426e9f85..8d8837fc39ec 100644 --- a/Documentation/arch/arm64/sve.rst +++ b/Documentation/arch/arm64/sve.rst @@ -117,11 +117,6 @@ the SVE instruction set architecture. * The SVE registers are not used to pass arguments to or receive results from any syscall. -* In practice the affected registers/bits will be preserved or will be replaced - with zeros on return from a syscall, but userspace should not make - assumptions about this. The kernel behaviour may vary on a case-by-case - basis. - * All other SVE state of a thread, including the currently configured vector length, the state of the PR_SVE_VL_INHERIT flag, and the deferred vector length (if any), is preserved across all syscalls, subject to the specific @@ -428,9 +423,8 @@ The regset data starts with struct user_sve_header, containing: /proc/sys/abi/sve_default_vector_length Writing the text representation of an integer to this file sets the system - default vector length to the specified value, unless the value is greater - than the maximum vector length supported by the system in which case the - default vector length is set to that maximum. + default vector length to the specified value rounded to a supported value + using the same rules as for setting vector length via PR_SVE_SET_VL. The result can be determined by reopening the file and reading its contents. diff --git a/Documentation/devicetree/bindings/perf/arm,coresight-pmu.yaml b/Documentation/devicetree/bindings/perf/arm,coresight-pmu.yaml new file mode 100644 index 000000000000..985b62990f80 --- /dev/null +++ b/Documentation/devicetree/bindings/perf/arm,coresight-pmu.yaml @@ -0,0 +1,39 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/perf/arm,coresight-pmu.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Arm Coresight Performance Monitoring Unit Architecture + +maintainers: + - Robin Murphy <[email protected]> + +properties: + compatible: + const: arm,coresight-pmu + + reg: + items: + - description: Register page 0 + - description: Register page 1, if the PMU implements the dual-page extension + minItems: 1 + + interrupts: + items: + - description: Overflow interrupt + + cpus: + description: If the PMU is associated with a particular CPU or subset of CPUs, + array of phandles to the appropriate CPU node(s) + + reg-io-width: + description: Granularity at which PMU register accesses are single-copy atomic + default: 4 + enum: [4, 8] + +required: + - compatible + - reg + +additionalProperties: false diff --git a/Documentation/devicetree/bindings/perf/starfive,jh8100-starlink-pmu.yaml b/Documentation/devicetree/bindings/perf/starfive,jh8100-starlink-pmu.yaml new file mode 100644 index 000000000000..915c6b814026 --- /dev/null +++ b/Documentation/devicetree/bindings/perf/starfive,jh8100-starlink-pmu.yaml @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/perf/starfive,jh8100-starlink-pmu.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: StarFive JH8100 StarLink PMU + +maintainers: + - Ji Sheng Teoh <[email protected]> + +description: + StarFive's JH8100 StarLink PMU integrates one or more CPU cores with a + shared L3 memory system. The PMU support overflow interrupt, up to + 16 programmable 64bit event counters, and an independent 64bit cycle + counter. StarFive's JH8100 StarLink PMU is accessed via MMIO. + +properties: + compatible: + const: starfive,jh8100-starlink-pmu + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + +required: + - compatible + - reg + - interrupts + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + pmu@12900000 { + compatible = "starfive,jh8100-starlink-pmu"; + reg = <0x0 0x12900000 0x0 0x10000>; + interrupts = <34>; + }; + }; diff --git a/Documentation/rust/arch-support.rst b/Documentation/rust/arch-support.rst index 73203ba1e901..5c4fa9f5d1cd 100644 --- a/Documentation/rust/arch-support.rst +++ b/Documentation/rust/arch-support.rst @@ -15,6 +15,7 @@ support corresponds to ``S`` values in the ``MAINTAINERS`` file. ============= ================ ============================================== Architecture Level of support Constraints ============= ================ ============================================== +``arm64`` Maintained Little Endian only. ``loongarch`` Maintained - ``um`` Maintained ``x86_64`` only. ``x86`` Maintained ``x86_64`` only. diff --git a/MAINTAINERS b/MAINTAINERS index 960512bec428..b7306d64d58b 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -20940,6 +20940,13 @@ S: Maintained T: git https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git/ F: Documentation/devicetree/bindings/soc/starfive/ +STARFIVE STARLINK PMU DRIVER +M: Ji Sheng Teoh <[email protected]> +S: Maintained +F: Documentation/admin-guide/perf/starfive_starlink_pmu.rst +F: Documentation/devicetree/bindings/perf/starfive,jh8100-starlink-pmu.yaml +F: drivers/perf/starfive_starlink_pmu.c + STARFIVE TRNG DRIVER M: Jia Jie Ho <[email protected]> S: Supported @@ -561,7 +561,6 @@ KBUILD_CFLAGS += -fno-strict-aliasing KBUILD_CPPFLAGS := -D__KERNEL__ KBUILD_RUSTFLAGS := $(rust_common_flags) \ - --target=$(objtree)/scripts/target.json \ -Cpanic=abort -Cembed-bitcode=n -Clto=n \ -Cforce-unwind-tables=n -Ccodegen-units=1 \ -Csymbol-mangling-version=v0 \ diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index aa7c1d435139..e14e92eb5ba5 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -120,6 +120,7 @@ config ARM64 select CLONE_BACKWARDS select COMMON_CLK select CPU_PM if (SUSPEND || CPU_IDLE) + select CPUMASK_OFFSTACK if NR_CPUS > 256 select CRC32 select DCACHE_WORD_ACCESS select DYNAMIC_FTRACE if FUNCTION_TRACER @@ -198,7 +199,7 @@ config ARM64 if DYNAMIC_FTRACE_WITH_ARGS && DYNAMIC_FTRACE_WITH_CALL_OPS select HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS \ if (DYNAMIC_FTRACE_WITH_ARGS && !CFI_CLANG && \ - !CC_OPTIMIZE_FOR_SIZE) + (CC_IS_CLANG || !CC_OPTIMIZE_FOR_SIZE)) select FTRACE_MCOUNT_USE_PATCHABLE_FUNCTION_ENTRY \ if DYNAMIC_FTRACE_WITH_ARGS select HAVE_SAMPLE_FTRACE_DIRECT @@ -229,6 +230,7 @@ config ARM64 select HAVE_FUNCTION_ARG_ACCESS_API select MMU_GATHER_RCU_TABLE_FREE select HAVE_RSEQ + select HAVE_RUST if CPU_LITTLE_ENDIAN select HAVE_STACKPROTECTOR select HAVE_SYSCALL_TRACEPOINTS select HAVE_KPROBES @@ -547,9 +549,8 @@ config ARM64_ERRATUM_832075 If unsure, say Y. config ARM64_ERRATUM_834220 - bool "Cortex-A57: 834220: Stage 2 translation fault might be incorrectly reported in presence of a Stage 1 fault" + bool "Cortex-A57: 834220: Stage 2 translation fault might be incorrectly reported in presence of a Stage 1 fault (rare)" depends on KVM - default y help This option adds an alternative code sequence to work around ARM erratum 834220 on Cortex-A57 parts up to r1p2. @@ -565,7 +566,7 @@ config ARM64_ERRATUM_834220 as it depends on the alternative framework, which will only patch the kernel if an affected CPU is detected. - If unsure, say Y. + If unsure, say N. config ARM64_ERRATUM_1742098 bool "Cortex-A57/A72: 1742098: ELR recorded incorrectly on interrupt taken between cryptographic instructions in a sequence" @@ -692,8 +693,7 @@ config ARM64_WORKAROUND_REPEAT_TLBI bool config ARM64_ERRATUM_2441007 - bool "Cortex-A55: Completion of affected memory accesses might not be guaranteed by completion of a TLBI" - default y + bool "Cortex-A55: Completion of affected memory accesses might not be guaranteed by completion of a TLBI (rare)" select ARM64_WORKAROUND_REPEAT_TLBI help This option adds a workaround for ARM Cortex-A55 erratum #2441007. @@ -706,11 +706,10 @@ config ARM64_ERRATUM_2441007 Work around this by adding the affected CPUs to the list that needs TLB sequences to be done twice. - If unsure, say Y. + If unsure, say N. config ARM64_ERRATUM_1286807 - bool "Cortex-A76: Modification of the translation table for a virtual address might lead to read-after-read ordering violation" - default y + bool "Cortex-A76: Modification of the translation table for a virtual address might lead to read-after-read ordering violation (rare)" select ARM64_WORKAROUND_REPEAT_TLBI help This option adds a workaround for ARM Cortex-A76 erratum 1286807. @@ -724,6 +723,8 @@ config ARM64_ERRATUM_1286807 invalidated has been observed by other observers. The workaround repeats the TLBI+DSB operation. + If unsure, say N. + config ARM64_ERRATUM_1463225 bool "Cortex-A76: Software Step might prevent interrupt recognition" default y @@ -743,8 +744,7 @@ config ARM64_ERRATUM_1463225 If unsure, say Y. config ARM64_ERRATUM_1542419 - bool "Neoverse-N1: workaround mis-ordering of instruction fetches" - default y + bool "Neoverse-N1: workaround mis-ordering of instruction fetches (rare)" help This option adds a workaround for ARM Neoverse-N1 erratum 1542419. @@ -756,7 +756,7 @@ config ARM64_ERRATUM_1542419 Workaround the issue by hiding the DIC feature from EL0. This forces user-space to perform cache maintenance. - If unsure, say Y. + If unsure, say N. config ARM64_ERRATUM_1508412 bool "Cortex-A77: 1508412: workaround deadlock on sequence of NC/Device load and store exclusive or PAR read" @@ -931,8 +931,7 @@ config ARM64_ERRATUM_2224489 If unsure, say Y. config ARM64_ERRATUM_2441009 - bool "Cortex-A510: Completion of affected memory accesses might not be guaranteed by completion of a TLBI" - default y + bool "Cortex-A510: Completion of affected memory accesses might not be guaranteed by completion of a TLBI (rare)" select ARM64_WORKAROUND_REPEAT_TLBI help This option adds a workaround for ARM Cortex-A510 erratum #2441009. @@ -945,7 +944,7 @@ config ARM64_ERRATUM_2441009 Work around this by adding the affected CPUs to the list that needs TLB sequences to be done twice. - If unsure, say Y. + If unsure, say N. config ARM64_ERRATUM_2064142 bool "Cortex-A510: 2064142: workaround TRBE register writes while disabled" @@ -1427,7 +1426,7 @@ config SCHED_SMT config NR_CPUS int "Maximum number of CPUs (2-4096)" range 2 4096 - default "256" + default "512" config HOTPLUG_CPU bool "Support for hot-pluggable CPUs" diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile index a88cdf910687..0e075d3c546b 100644 --- a/arch/arm64/Makefile +++ b/arch/arm64/Makefile @@ -41,6 +41,8 @@ KBUILD_CFLAGS += -mgeneral-regs-only \ KBUILD_CFLAGS += $(call cc-disable-warning, psabi) KBUILD_AFLAGS += $(compat_vdso) +KBUILD_RUSTFLAGS += --target=aarch64-unknown-none -Ctarget-feature="-neon" + KBUILD_CFLAGS += $(call cc-option,-mabi=lp64) KBUILD_AFLAGS += $(call cc-option,-mabi=lp64) @@ -65,7 +67,9 @@ endif ifeq ($(CONFIG_ARM64_BTI_KERNEL),y) KBUILD_CFLAGS += -mbranch-protection=pac-ret+bti + KBUILD_RUSTFLAGS += -Zbranch-protection=bti,pac-ret else ifeq ($(CONFIG_ARM64_PTR_AUTH_KERNEL),y) + KBUILD_RUSTFLAGS += -Zbranch-protection=pac-ret ifeq ($(CONFIG_CC_HAS_BRANCH_PROT_PAC_RET),y) KBUILD_CFLAGS += -mbranch-protection=pac-ret else diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h index 513787e43329..96b18a707507 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -38,10 +38,6 @@ msr daifset, #0xf .endm - .macro enable_daif - msr daifclr, #0xf - .endm - /* * Save/restore interrupts. */ diff --git a/arch/arm64/include/asm/brk-imm.h b/arch/arm64/include/asm/brk-imm.h index 1abdcd508a11..beb42c62b6ac 100644 --- a/arch/arm64/include/asm/brk-imm.h +++ b/arch/arm64/include/asm/brk-imm.h @@ -11,6 +11,7 @@ * 0x004: for installing kprobes * 0x005: for installing uprobes * 0x006: for kprobe software single-step + * 0x007: for kretprobe return * Allowed values for kgdb are 0x400 - 0x7ff * 0x100: for triggering a fault on purpose (reserved) * 0x400: for dynamic BRK instruction @@ -23,6 +24,7 @@ #define KPROBES_BRK_IMM 0x004 #define UPROBES_BRK_IMM 0x005 #define KPROBES_BRK_SS_IMM 0x006 +#define KRETPROBES_BRK_IMM 0x007 #define FAULT_BRK_IMM 0x100 #define KGDB_DYN_DBG_BRK_IMM 0x400 #define KGDB_COMPILED_DBG_BRK_IMM 0x401 diff --git a/arch/arm64/include/asm/elf.h b/arch/arm64/include/asm/elf.h index 97932fbf973d..3f93f4eef953 100644 --- a/arch/arm64/include/asm/elf.h +++ b/arch/arm64/include/asm/elf.h @@ -201,16 +201,16 @@ extern int arch_setup_additional_pages(struct linux_binprm *bprm, #define COMPAT_ELF_PLATFORM ("v8l") #endif -#ifdef CONFIG_COMPAT - -/* PIE load location for compat arm. Must match ARM ELF_ET_DYN_BASE. */ -#define COMPAT_ELF_ET_DYN_BASE 0x000400000UL - /* AArch32 registers. */ #define COMPAT_ELF_NGREG 18 typedef unsigned int compat_elf_greg_t; typedef compat_elf_greg_t compat_elf_gregset_t[COMPAT_ELF_NGREG]; +#ifdef CONFIG_COMPAT + +/* PIE load location for compat arm. Must match ARM ELF_ET_DYN_BASE. */ +#define COMPAT_ELF_ET_DYN_BASE 0x000400000UL + /* AArch32 EABI. */ #define EF_ARM_EABI_MASK 0xff000000 int compat_elf_check_arch(const struct elf32_hdr *); diff --git a/arch/arm64/include/asm/exception.h b/arch/arm64/include/asm/exception.h index ad688e157c9b..f296662590c7 100644 --- a/arch/arm64/include/asm/exception.h +++ b/arch/arm64/include/asm/exception.h @@ -74,7 +74,7 @@ void do_el0_fpac(struct pt_regs *regs, unsigned long esr); void do_el1_fpac(struct pt_regs *regs, unsigned long esr); void do_el0_mops(struct pt_regs *regs, unsigned long esr); void do_serror(struct pt_regs *regs, unsigned long esr); -void do_notify_resume(struct pt_regs *regs, unsigned long thread_flags); +void do_signal(struct pt_regs *regs); void __noreturn panic_bad_stack(struct pt_regs *regs, unsigned long esr, unsigned long far); #endif /* __ASM_EXCEPTION_H */ diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h index 74afca3bd312..47cbd1da40b4 100644 --- a/arch/arm64/include/asm/fpsimd.h +++ b/arch/arm64/include/asm/fpsimd.h @@ -21,7 +21,6 @@ #include <linux/stddef.h> #include <linux/types.h> -#ifdef CONFIG_COMPAT /* Masks for extracting the FPSR and FPCR from the FPSCR */ #define VFP_FPSCR_STAT_MASK 0xf800009f #define VFP_FPSCR_CTRL_MASK 0x07f79f00 @@ -30,7 +29,6 @@ * control/status register. */ #define VFP_STATE_SIZE ((32 * 8) + 4) -#endif static inline unsigned long cpacr_save_enable_kernel_sve(void) { diff --git a/arch/arm64/include/asm/hw_breakpoint.h b/arch/arm64/include/asm/hw_breakpoint.h index 84055329cd8b..bd81cf17744a 100644 --- a/arch/arm64/include/asm/hw_breakpoint.h +++ b/arch/arm64/include/asm/hw_breakpoint.h @@ -59,7 +59,6 @@ static inline void decode_ctrl_reg(u32 reg, /* Watchpoints */ #define ARM_BREAKPOINT_LOAD 1 #define ARM_BREAKPOINT_STORE 2 -#define AARCH64_ESR_ACCESS_MASK (1 << 6) /* Lengths */ #define ARM_BREAKPOINT_LEN_1 0x1 diff --git a/arch/arm64/include/asm/io.h b/arch/arm64/include/asm/io.h index 3b694511b98f..8d825522c55c 100644 --- a/arch/arm64/include/asm/io.h +++ b/arch/arm64/include/asm/io.h @@ -24,25 +24,29 @@ #define __raw_writeb __raw_writeb static __always_inline void __raw_writeb(u8 val, volatile void __iomem *addr) { - asm volatile("strb %w0, [%1]" : : "rZ" (val), "r" (addr)); + volatile u8 __iomem *ptr = addr; + asm volatile("strb %w0, %1" : : "rZ" (val), "Qo" (*ptr)); } #define __raw_writew __raw_writew static __always_inline void __raw_writew(u16 val, volatile void __iomem *addr) { - asm volatile("strh %w0, [%1]" : : "rZ" (val), "r" (addr)); + volatile u16 __iomem *ptr = addr; + asm volatile("strh %w0, %1" : : "rZ" (val), "Qo" (*ptr)); } #define __raw_writel __raw_writel static __always_inline void __raw_writel(u32 val, volatile void __iomem *addr) { - asm volatile("str %w0, [%1]" : : "rZ" (val), "r" (addr)); + volatile u32 __iomem *ptr = addr; + asm volatile("str %w0, %1" : : "rZ" (val), "Qo" (*ptr)); } #define __raw_writeq __raw_writeq static __always_inline void __raw_writeq(u64 val, volatile void __iomem *addr) { - asm volatile("str %x0, [%1]" : : "rZ" (val), "r" (addr)); + volatile u64 __iomem *ptr = addr; + asm volatile("str %x0, %1" : : "rZ" (val), "Qo" (*ptr)); } #define __raw_readb __raw_readb diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h index d82305ab420f..60904a6c4b42 100644 --- a/arch/arm64/include/asm/memory.h +++ b/arch/arm64/include/asm/memory.h @@ -30,8 +30,8 @@ * keep a constant PAGE_OFFSET and "fallback" to using the higher end * of the VMEMMAP where 52-bit support is not available in hardware. */ -#define VMEMMAP_SHIFT (PAGE_SHIFT - STRUCT_PAGE_MAX_SHIFT) -#define VMEMMAP_SIZE ((_PAGE_END(VA_BITS_MIN) - PAGE_OFFSET) >> VMEMMAP_SHIFT) +#define VMEMMAP_RANGE (_PAGE_END(VA_BITS_MIN) - PAGE_OFFSET) +#define VMEMMAP_SIZE ((VMEMMAP_RANGE >> PAGE_SHIFT) * sizeof(struct page)) /* * PAGE_OFFSET - the virtual address of the start of the linear map, at the @@ -47,11 +47,11 @@ #define MODULES_END (MODULES_VADDR + MODULES_VSIZE) #define MODULES_VADDR (_PAGE_END(VA_BITS_MIN)) #define MODULES_VSIZE (SZ_2G) -#define VMEMMAP_START (-(UL(1) << (VA_BITS - VMEMMAP_SHIFT))) -#define VMEMMAP_END (VMEMMAP_START + VMEMMAP_SIZE) -#define PCI_IO_END (VMEMMAP_START - SZ_8M) -#define PCI_IO_START (PCI_IO_END - PCI_IO_SIZE) -#define FIXADDR_TOP (VMEMMAP_START - SZ_32M) +#define VMEMMAP_START (VMEMMAP_END - VMEMMAP_SIZE) +#define VMEMMAP_END (-UL(SZ_1G)) +#define PCI_IO_START (VMEMMAP_END + SZ_8M) +#define PCI_IO_END (PCI_IO_START + PCI_IO_SIZE) +#define FIXADDR_TOP (-UL(SZ_8M)) #if VA_BITS > 48 #define VA_BITS_MIN (48) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 79ce70fbb751..522c21348ae8 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -18,11 +18,15 @@ * VMALLOC range. * * VMALLOC_START: beginning of the kernel vmalloc space - * VMALLOC_END: extends to the available space below vmemmap, PCI I/O space - * and fixed mappings + * VMALLOC_END: extends to the available space below vmemmap */ #define VMALLOC_START (MODULES_END) -#define VMALLOC_END (VMEMMAP_START - SZ_256M) +#if VA_BITS == VA_BITS_MIN +#define VMALLOC_END (VMEMMAP_START - SZ_8M) +#else +#define VMEMMAP_UNUSED_NPAGES ((_PAGE_OFFSET(vabits_actual) - PAGE_OFFSET) >> PAGE_SHIFT) +#define VMALLOC_END (VMEMMAP_START + VMEMMAP_UNUSED_NPAGES * sizeof(struct page) - SZ_8M) +#endif #define vmemmap ((struct page *)VMEMMAP_START - (memstart_addr >> PAGE_SHIFT)) diff --git a/arch/arm64/include/uapi/asm/sve_context.h b/arch/arm64/include/uapi/asm/sve_context.h index 754ab751b523..72aefc081061 100644 --- a/arch/arm64/include/uapi/asm/sve_context.h +++ b/arch/arm64/include/uapi/asm/sve_context.h @@ -13,6 +13,17 @@ #define __SVE_VQ_BYTES 16 /* number of bytes per quadword */ +/* + * Yes, __SVE_VQ_MAX is 512 QUADWORDS. + * + * To help ensure forward portability, this is much larger than the + * current maximum value defined by the SVE architecture. While arrays + * or static allocations can be sized based on this value, watch out! + * It will waste a surprisingly large amount of memory. + * + * Dynamic sizing based on the actual runtime vector length is likely to + * be preferable for most purposes. + */ #define __SVE_VQ_MIN 1 #define __SVE_VQ_MAX 512 diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c index 5a7dbbe0ce63..81496083c041 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -75,8 +75,8 @@ int main(void) DEFINE(S_FP, offsetof(struct pt_regs, regs[29])); DEFINE(S_LR, offsetof(struct pt_regs, regs[30])); DEFINE(S_SP, offsetof(struct pt_regs, sp)); - DEFINE(S_PSTATE, offsetof(struct pt_regs, pstate)); DEFINE(S_PC, offsetof(struct pt_regs, pc)); + DEFINE(S_PSTATE, offsetof(struct pt_regs, pstate)); DEFINE(S_SYSCALLNO, offsetof(struct pt_regs, syscallno)); DEFINE(S_SDEI_TTBR1, offsetof(struct pt_regs, sdei_ttbr1)); DEFINE(S_PMR_SAVE, offsetof(struct pt_regs, pmr_save)); diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index aefda789f510..6e1cca7b2098 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -3124,13 +3124,9 @@ static void __init enable_cpu_capabilities(u16 scope_mask) boot_scope = !!(scope_mask & SCOPE_BOOT_CPU); for (i = 0; i < ARM64_NCAPS; i++) { - unsigned int num; - caps = cpucap_ptrs[i]; - if (!caps || !(caps->type & scope_mask)) - continue; - num = caps->capability; - if (!cpus_have_cap(num)) + if (!caps || !(caps->type & scope_mask) || + !cpus_have_cap(caps->capability)) continue; if (boot_scope && caps->cpu_enable) diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index 0fc94207e69a..b77a15955f28 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -10,6 +10,7 @@ #include <linux/linkage.h> #include <linux/lockdep.h> #include <linux/ptrace.h> +#include <linux/resume_user_mode.h> #include <linux/sched.h> #include <linux/sched/debug.h> #include <linux/thread_info.h> @@ -126,16 +127,49 @@ static __always_inline void __exit_to_user_mode(void) lockdep_hardirqs_on(CALLER_ADDR0); } +static void do_notify_resume(struct pt_regs *regs, unsigned long thread_flags) +{ + do { + local_irq_enable(); + + if (thread_flags & _TIF_NEED_RESCHED) + schedule(); + + if (thread_flags & _TIF_UPROBE) + uprobe_notify_resume(regs); + + if (thread_flags & _TIF_MTE_ASYNC_FAULT) { + clear_thread_flag(TIF_MTE_ASYNC_FAULT); + send_sig_fault(SIGSEGV, SEGV_MTEAERR, + (void __user *)NULL, current); + } + + if (thread_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) + do_signal(regs); + + if (thread_flags & _TIF_NOTIFY_RESUME) + resume_user_mode_work(regs); + + if (thread_flags & _TIF_FOREIGN_FPSTATE) + fpsimd_restore_current_state(); + + local_irq_disable(); + thread_flags = read_thread_flags(); + } while (thread_flags & _TIF_WORK_MASK); +} + static __always_inline void exit_to_user_mode_prepare(struct pt_regs *regs) { unsigned long flags; - local_daif_mask(); + local_irq_disable(); flags = read_thread_flags(); if (unlikely(flags & _TIF_WORK_MASK)) do_notify_resume(regs, flags); + local_daif_mask(); + lockdep_sys_exit(); } diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index 8e24b5e5e192..0cd2bfb38bb0 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -1146,6 +1146,8 @@ void cpu_enable_sve(const struct arm64_cpu_capabilities *__always_unused p) { write_sysreg(read_sysreg(CPACR_EL1) | CPACR_EL1_ZEN_EL1EN, CPACR_EL1); isb(); + + write_sysreg_s(0, SYS_ZCR_EL1); } void __init sve_setup(void) @@ -1257,6 +1259,9 @@ void cpu_enable_sme(const struct arm64_cpu_capabilities *__always_unused p) write_sysreg(read_sysreg(CPACR_EL1) | CPACR_EL1_SMEN_EL1EN, CPACR_EL1); isb(); + /* Ensure all bits in SMCR are set to known values */ + write_sysreg_s(0, SYS_SMCR_EL1); + /* Allow EL0 to access TPIDR2 */ write_sysreg(read_sysreg(SCTLR_EL1) | SCTLR_ELx_ENTP2, SCTLR_EL1); isb(); diff --git a/arch/arm64/kernel/hw_breakpoint.c b/arch/arm64/kernel/hw_breakpoint.c index 35225632d70a..2f5755192c2b 100644 --- a/arch/arm64/kernel/hw_breakpoint.c +++ b/arch/arm64/kernel/hw_breakpoint.c @@ -21,6 +21,7 @@ #include <asm/current.h> #include <asm/debug-monitors.h> +#include <asm/esr.h> #include <asm/hw_breakpoint.h> #include <asm/traps.h> #include <asm/cputype.h> @@ -779,7 +780,7 @@ static int watchpoint_handler(unsigned long addr, unsigned long esr, * Check that the access type matches. * 0 => load, otherwise => store */ - access = (esr & AARCH64_ESR_ACCESS_MASK) ? HW_BREAKPOINT_W : + access = (esr & ESR_ELx_WNR) ? HW_BREAKPOINT_W : HW_BREAKPOINT_R; if (!(access & hw_breakpoint_type(wp))) continue; diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h index 5e4dc72ab1bd..e931ce078a00 100644 --- a/arch/arm64/kernel/image-vars.h +++ b/arch/arm64/kernel/image-vars.h @@ -36,6 +36,8 @@ PROVIDE(__pi___memcpy = __pi_memcpy); PROVIDE(__pi___memmove = __pi_memmove); PROVIDE(__pi___memset = __pi_memset); +PROVIDE(__pi_vabits_actual = vabits_actual); + #ifdef CONFIG_KVM /* diff --git a/arch/arm64/kernel/pi/kaslr_early.c b/arch/arm64/kernel/pi/kaslr_early.c index 17bff6e399e4..b9e0bb4bc6a9 100644 --- a/arch/arm64/kernel/pi/kaslr_early.c +++ b/arch/arm64/kernel/pi/kaslr_early.c @@ -14,6 +14,7 @@ #include <asm/archrandom.h> #include <asm/memory.h> +#include <asm/pgtable.h> /* taken from lib/string.c */ static char *__strstr(const char *s1, const char *s2) @@ -87,7 +88,7 @@ static u64 get_kaslr_seed(void *fdt) asmlinkage u64 kaslr_early_init(void *fdt) { - u64 seed; + u64 seed, range; if (is_kaslr_disabled_cmdline(fdt)) return 0; @@ -102,9 +103,9 @@ asmlinkage u64 kaslr_early_init(void *fdt) /* * OK, so we are proceeding with KASLR enabled. Calculate a suitable * kernel image offset from the seed. Let's place the kernel in the - * middle half of the VMALLOC area (VA_BITS_MIN - 2), and stay clear of - * the lower and upper quarters to avoid colliding with other - * allocations. + * 'middle' half of the VMALLOC area, and stay clear of the lower and + * upper quarters to avoid colliding with other allocations. */ - return BIT(VA_BITS_MIN - 3) + (seed & GENMASK(VA_BITS_MIN - 3, 0)); + range = (VMALLOC_END - KIMAGE_VADDR) / 2; + return range / 2 + (((__uint128_t)range * seed) >> 64); } diff --git a/arch/arm64/kernel/probes/kprobes.c b/arch/arm64/kernel/probes/kprobes.c index 70b91a8c6bb3..327855a11df2 100644 --- a/arch/arm64/kernel/probes/kprobes.c +++ b/arch/arm64/kernel/probes/kprobes.c @@ -371,6 +371,21 @@ static struct break_hook kprobes_break_ss_hook = { .fn = kprobe_breakpoint_ss_handler, }; +static int __kprobes +kretprobe_breakpoint_handler(struct pt_regs *regs, unsigned long esr) +{ + if (regs->pc != (unsigned long)__kretprobe_trampoline) + return DBG_HOOK_ERROR; + + regs->pc = kretprobe_trampoline_handler(regs, (void *)regs->regs[29]); + return DBG_HOOK_HANDLED; +} + +static struct break_hook kretprobes_break_hook = { + .imm = KRETPROBES_BRK_IMM, + .fn = kretprobe_breakpoint_handler, +}; + /* * Provide a blacklist of symbols identifying ranges which cannot be kprobed. * This blacklist is exposed to userspace via debugfs (kprobes/blacklist). @@ -396,11 +411,6 @@ int __init arch_populate_kprobe_blacklist(void) return ret; } -void __kprobes __used *trampoline_probe_handler(struct pt_regs *regs) -{ - return (void *)kretprobe_trampoline_handler(regs, (void *)regs->regs[29]); -} - void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs) { @@ -420,6 +430,7 @@ int __init arch_init_kprobes(void) { register_kernel_break_hook(&kprobes_break_hook); register_kernel_break_hook(&kprobes_break_ss_hook); + register_kernel_break_hook(&kretprobes_break_hook); return 0; } diff --git a/arch/arm64/kernel/probes/kprobes_trampoline.S b/arch/arm64/kernel/probes/kprobes_trampoline.S index 9a6499bed58b..a362f3dbb3d1 100644 --- a/arch/arm64/kernel/probes/kprobes_trampoline.S +++ b/arch/arm64/kernel/probes/kprobes_trampoline.S @@ -4,83 +4,17 @@ */ #include <linux/linkage.h> -#include <asm/asm-offsets.h> +#include <asm/asm-bug.h> #include <asm/assembler.h> .text - .macro save_all_base_regs - stp x0, x1, [sp, #S_X0] - stp x2, x3, [sp, #S_X2] - stp x4, x5, [sp, #S_X4] - stp x6, x7, [sp, #S_X6] - stp x8, x9, [sp, #S_X8] - stp x10, x11, [sp, #S_X10] - stp x12, x13, [sp, #S_X12] - stp x14, x15, [sp, #S_X14] - stp x16, x17, [sp, #S_X16] - stp x18, x19, [sp, #S_X18] - stp x20, x21, [sp, #S_X20] - stp x22, x23, [sp, #S_X22] - stp x24, x25, [sp, #S_X24] - stp x26, x27, [sp, #S_X26] - stp x28, x29, [sp, #S_X28] - add x0, sp, #PT_REGS_SIZE - stp lr, x0, [sp, #S_LR] - /* - * Construct a useful saved PSTATE - */ - mrs x0, nzcv - mrs x1, daif - orr x0, x0, x1 - mrs x1, CurrentEL - orr x0, x0, x1 - mrs x1, SPSel - orr x0, x0, x1 - stp xzr, x0, [sp, #S_PC] - .endm - - .macro restore_all_base_regs - ldr x0, [sp, #S_PSTATE] - and x0, x0, #(PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | PSR_V_BIT) - msr nzcv, x0 - ldp x0, x1, [sp, #S_X0] - ldp x2, x3, [sp, #S_X2] - ldp x4, x5, [sp, #S_X4] - ldp x6, x7, [sp, #S_X6] - ldp x8, x9, [sp, #S_X8] - ldp x10, x11, [sp, #S_X10] - ldp x12, x13, [sp, #S_X12] - ldp x14, x15, [sp, #S_X14] - ldp x16, x17, [sp, #S_X16] - ldp x18, x19, [sp, #S_X18] - ldp x20, x21, [sp, #S_X20] - ldp x22, x23, [sp, #S_X22] - ldp x24, x25, [sp, #S_X24] - ldp x26, x27, [sp, #S_X26] - ldp x28, x29, [sp, #S_X28] - .endm - SYM_CODE_START(__kretprobe_trampoline) - sub sp, sp, #PT_REGS_SIZE - - save_all_base_regs - - /* Setup a frame pointer. */ - add x29, sp, #S_FP - - mov x0, sp - bl trampoline_probe_handler /* - * Replace trampoline address in lr with actual orig_ret_addr return - * address. + * Trigger a breakpoint exception. The PC will be adjusted by + * kretprobe_breakpoint_handler(), and no subsequent instructions will + * be executed from the trampoline. */ - mov lr, x0 - - /* The frame pointer (x29) is restored with other registers. */ - restore_all_base_regs - - add sp, sp, #PT_REGS_SIZE - ret - + brk #KRETPROBES_BRK_IMM + ASM_BUG() SYM_CODE_END(__kretprobe_trampoline) diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 7387b68c745b..4ae31b7af6c3 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -290,9 +290,6 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) fpsimd_preserve_current_state(); *dst = *src; - /* We rely on the above assignment to initialize dst's thread_flags: */ - BUILD_BUG_ON(!IS_ENABLED(CONFIG_THREAD_INFO_IN_TASK)); - /* * Detach src's sve_state (if any) from dst so that it does not * get erroneously used or freed prematurely. dst's copies diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index aacb45bd36e6..b096c8be3bcf 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -174,7 +174,6 @@ static void ptrace_hbptriggered(struct perf_event *bp, struct arch_hw_breakpoint *bkpt = counter_arch_bp(bp); const char *desc = "Hardware breakpoint trap (ptrace)"; -#ifdef CONFIG_COMPAT if (is_compat_task()) { int si_errno = 0; int i; @@ -196,7 +195,7 @@ static void ptrace_hbptriggered(struct perf_event *bp, desc); return; } -#endif + arm64_force_sig_fault(SIGTRAP, TRAP_HWBKPT, bkpt->trigger, desc); } @@ -1638,7 +1637,6 @@ static const struct user_regset_view user_aarch64_view = { .regsets = aarch64_regsets, .n = ARRAY_SIZE(aarch64_regsets) }; -#ifdef CONFIG_COMPAT enum compat_regset { REGSET_COMPAT_GPR, REGSET_COMPAT_VFP, @@ -1895,6 +1893,7 @@ static const struct user_regset_view user_aarch32_ptrace_view = { .regsets = aarch32_ptrace_regsets, .n = ARRAY_SIZE(aarch32_ptrace_regsets) }; +#ifdef CONFIG_COMPAT static int compat_ptrace_read_user(struct task_struct *tsk, compat_ulong_t off, compat_ulong_t __user *ret) { @@ -2156,7 +2155,6 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request, const struct user_regset_view *task_user_regset_view(struct task_struct *task) { -#ifdef CONFIG_COMPAT /* * Core dumping of 32-bit tasks or compat ptrace requests must use the * user_aarch32_view compatible with arm32. Native ptrace requests on @@ -2167,7 +2165,7 @@ const struct user_regset_view *task_user_regset_view(struct task_struct *task) return &user_aarch32_view; else if (is_compat_thread(task_thread_info(task))) return &user_aarch32_ptrace_view; -#endif + return &user_aarch64_view; } diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index 42c690bb2d60..ab43bfa85368 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -320,9 +320,8 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) dynamic_scs_init(); /* - * Unmask asynchronous aborts and fiq after bringing up possible - * earlycon. (Report possible System Errors once we can report this - * occurred). + * Unmask SError as soon as possible after initializing earlycon so + * that we can report any SErrors immediately. */ local_daif_restore(DAIF_PROCCTX_NOIRQ); diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c index 460823baa603..ac69b604cac9 100644 --- a/arch/arm64/kernel/signal.c +++ b/arch/arm64/kernel/signal.c @@ -16,8 +16,8 @@ #include <linux/uaccess.h> #include <linux/sizes.h> #include <linux/string.h> -#include <linux/resume_user_mode.h> #include <linux/ratelimit.h> +#include <linux/rseq.h> #include <linux/syscalls.h> #include <asm/daifflags.h> @@ -1266,7 +1266,7 @@ static void handle_signal(struct ksignal *ksig, struct pt_regs *regs) * the kernel can handle, and then we build all the user-level signal handling * stack-frames in one go after that. */ -static void do_signal(struct pt_regs *regs) +void do_signal(struct pt_regs *regs) { unsigned long continue_addr = 0, restart_addr = 0; int retval = 0; @@ -1337,41 +1337,6 @@ static void do_signal(struct pt_regs *regs) restore_saved_sigmask(); } -void do_notify_resume(struct pt_regs *regs, unsigned long thread_flags) -{ - do { - if (thread_flags & _TIF_NEED_RESCHED) { - /* Unmask Debug and SError for the next task */ - local_daif_restore(DAIF_PROCCTX_NOIRQ); - - schedule(); - } else { - local_daif_restore(DAIF_PROCCTX); - - if (thread_flags & _TIF_UPROBE) - uprobe_notify_resume(regs); - - if (thread_flags & _TIF_MTE_ASYNC_FAULT) { - clear_thread_flag(TIF_MTE_ASYNC_FAULT); - send_sig_fault(SIGSEGV, SEGV_MTEAERR, - (void __user *)NULL, current); - } - - if (thread_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) - do_signal(regs); - - if (thread_flags & _TIF_NOTIFY_RESUME) - resume_user_mode_work(regs); - - if (thread_flags & _TIF_FOREIGN_FPSTATE) - fpsimd_restore_current_state(); - } - - local_daif_mask(); - thread_flags = read_thread_flags(); - } while (thread_flags & _TIF_WORK_MASK); -} - unsigned long __ro_after_init signal_minsigstksz; /* diff --git a/arch/arm64/kernel/syscall.c b/arch/arm64/kernel/syscall.c index 9a70d9746b66..ad198262b981 100644 --- a/arch/arm64/kernel/syscall.c +++ b/arch/arm64/kernel/syscall.c @@ -20,14 +20,11 @@ long sys_ni_syscall(void); static long do_ni_syscall(struct pt_regs *regs, int scno) { -#ifdef CONFIG_COMPAT - long ret; if (is_compat_task()) { - ret = compat_arm_syscall(regs, scno); + long ret = compat_arm_syscall(regs, scno); if (ret != -ENOSYS) return ret; } -#endif return sys_ni_syscall(); } diff --git a/arch/arm64/mm/fixmap.c b/arch/arm64/mm/fixmap.c index c0a3301203bd..6fc17b2e1714 100644 --- a/arch/arm64/mm/fixmap.c +++ b/arch/arm64/mm/fixmap.c @@ -16,6 +16,9 @@ #include <asm/pgalloc.h> #include <asm/tlbflush.h> +/* ensure that the fixmap region does not grow down into the PCI I/O region */ +static_assert(FIXADDR_TOT_START > PCI_IO_END); + #define NR_BM_PTE_TABLES \ SPAN_NR_ENTRIES(FIXADDR_TOT_START, FIXADDR_TOP, PMD_SHIFT) #define NR_BM_PMD_TABLES \ diff --git a/arch/arm64/mm/ptdump.c b/arch/arm64/mm/ptdump.c index e305b6593c4e..5f0849528ccf 100644 --- a/arch/arm64/mm/ptdump.c +++ b/arch/arm64/mm/ptdump.c @@ -26,34 +26,6 @@ #include <asm/ptdump.h> -enum address_markers_idx { - PAGE_OFFSET_NR = 0, - PAGE_END_NR, -#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS) - KASAN_START_NR, -#endif -}; - -static struct addr_marker address_markers[] = { - { PAGE_OFFSET, "Linear Mapping start" }, - { 0 /* PAGE_END */, "Linear Mapping end" }, -#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS) - { 0 /* KASAN_SHADOW_START */, "Kasan shadow start" }, - { KASAN_SHADOW_END, "Kasan shadow end" }, -#endif - { MODULES_VADDR, "Modules start" }, - { MODULES_END, "Modules end" }, - { VMALLOC_START, "vmalloc() area" }, - { VMALLOC_END, "vmalloc() end" }, - { FIXADDR_TOT_START, "Fixmap start" }, - { FIXADDR_TOP, "Fixmap end" }, - { PCI_IO_START, "PCI I/O start" }, - { PCI_IO_END, "PCI I/O end" }, - { VMEMMAP_START, "vmemmap start" }, - { VMEMMAP_START + VMEMMAP_SIZE, "vmemmap end" }, - { -1, NULL }, -}; - #define pt_dump_seq_printf(m, fmt, args...) \ ({ \ if (m) \ @@ -339,9 +311,8 @@ static void __init ptdump_initialize(void) pg_level[i].mask |= pg_level[i].bits[j].mask; } -static struct ptdump_info kernel_ptdump_info = { +static struct ptdump_info kernel_ptdump_info __ro_after_init = { .mm = &init_mm, - .markers = address_markers, .base_addr = PAGE_OFFSET, }; @@ -375,10 +346,31 @@ void ptdump_check_wx(void) static int __init ptdump_init(void) { - address_markers[PAGE_END_NR].start_address = PAGE_END; + u64 page_offset = _PAGE_OFFSET(vabits_actual); + u64 vmemmap_start = (u64)virt_to_page((void *)page_offset); + struct addr_marker m[] = { + { PAGE_OFFSET, "Linear Mapping start" }, + { PAGE_END, "Linear Mapping end" }, #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS) - address_markers[KASAN_START_NR].start_address = KASAN_SHADOW_START; + { KASAN_SHADOW_START, "Kasan shadow start" }, + { KASAN_SHADOW_END, "Kasan shadow end" }, #endif + { MODULES_VADDR, "Modules start" }, + { MODULES_END, "Modules end" }, + { VMALLOC_START, "vmalloc() area" }, + { VMALLOC_END, "vmalloc() end" }, + { vmemmap_start, "vmemmap start" }, + { VMEMMAP_END, "vmemmap end" }, + { PCI_IO_START, "PCI I/O start" }, + { PCI_IO_END, "PCI I/O end" }, + { FIXADDR_TOT_START, "Fixmap start" }, + { FIXADDR_TOP, "Fixmap end" }, + { -1, NULL }, + }; + static struct addr_marker address_markers[ARRAY_SIZE(m)] __ro_after_init; + + kernel_ptdump_info.markers = memcpy(address_markers, m, sizeof(m)); + ptdump_initialize(); ptdump_debugfs_register(&kernel_ptdump_info, "kernel_page_tables"); return 0; diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 4c9b67934367..508224a0e078 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -200,6 +200,7 @@ UnsignedEnum 27:24 PerfMon 0b0110 PMUv3p5 0b0111 PMUv3p7 0b1000 PMUv3p8 + 0b1001 PMUv3p9 0b1111 IMPDEF EndEnum Enum 23:20 MProfDbg @@ -231,6 +232,7 @@ Enum 3:0 CopDbg 0b1000 Debugv8p2 0b1001 Debugv8p4 0b1010 Debugv8p8 + 0b1011 Debugv8p9 EndEnum EndSysreg @@ -1221,6 +1223,7 @@ UnsignedEnum 35:32 PMSVer 0b0010 V1P1 0b0011 V1P2 0b0100 V1P3 + 0b0101 V1P4 EndEnum Field 31:28 CTX_CMPs Res0 27:24 @@ -1247,11 +1250,41 @@ UnsignedEnum 3:0 DebugVer 0b1000 V8P2 0b1001 V8P4 0b1010 V8P8 + 0b1011 V8P9 EndEnum EndSysreg Sysreg ID_AA64DFR1_EL1 3 0 0 5 1 -Res0 63:0 +Field 63:56 ABL_CMPs +UnsignedEnum 55:52 DPFZS + 0b0000 IGNR + 0b0001 FRZN +EndEnum +UnsignedEnum 51:48 EBEP + 0b0000 NI + 0b0001 IMP +EndEnum +UnsignedEnum 47:44 ITE + 0b0000 NI + 0b0001 IMP +EndEnum +UnsignedEnum 43:40 ABLE + 0b0000 NI + 0b0001 IMP +EndEnum +UnsignedEnum 39:36 PMICNTR + 0b0000 NI + 0b0001 IMP +EndEnum +UnsignedEnum 35:32 SPMU + 0b0000 NI + 0b0001 IMP + 0b0010 IMP_SPMZR +EndEnum +Field 31:24 CTX_CMPs +Field 23:16 WRPs +Field 15:8 BRPs +Field 7:0 SYSPMUID EndSysreg Sysreg ID_AA64AFR0_EL1 3 0 0 5 4 diff --git a/arch/loongarch/Makefile b/arch/loongarch/Makefile index 983aa2b1629a..fa4fb09909ae 100644 --- a/arch/loongarch/Makefile +++ b/arch/loongarch/Makefile @@ -82,6 +82,7 @@ KBUILD_AFLAGS_MODULE += -Wa,-mla-global-with-abs KBUILD_CFLAGS_MODULE += -fplt -Wa,-mla-global-with-abs,-mla-local-with-abs endif +KBUILD_RUSTFLAGS += --target=$(objtree)/scripts/target.json KBUILD_RUSTFLAGS_MODULE += -Crelocation-model=pic ifeq ($(CONFIG_RELOCATABLE),y) diff --git a/arch/x86/Makefile b/arch/x86/Makefile index 2264db14a25d..18cf8f0cf7cd 100644 --- a/arch/x86/Makefile +++ b/arch/x86/Makefile @@ -68,6 +68,7 @@ export BITS # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53383 # KBUILD_CFLAGS += -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -mno-avx +KBUILD_RUSTFLAGS += --target=$(objtree)/scripts/target.json KBUILD_RUSTFLAGS += -Ctarget-feature=-sse,-sse2,-sse3,-ssse3,-sse4.1,-sse4.2,-avx,-avx2 ifeq ($(CONFIG_X86_KERNEL_IBT),y) diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig index ec6e0d9194a1..004d86230aa6 100644 --- a/drivers/perf/Kconfig +++ b/drivers/perf/Kconfig @@ -86,6 +86,15 @@ config RISCV_PMU_SBI full perf feature support i.e. counter overflow, privilege mode filtering, counter configuration. +config STARFIVE_STARLINK_PMU + depends on ARCH_STARFIVE || (COMPILE_TEST && 64BIT) + bool "StarFive StarLink PMU" + help + Provide support for StarLink Performance Monitor Unit. + StarLink Performance Monitor Unit integrates one or more cores with + an L3 memory system. The L3 cache events are added into perf event + subsystem, allowing monitoring of various L3 cache perf events. + config ARM_PMU_ACPI depends on ARM_PMU && ACPI def_bool y diff --git a/drivers/perf/Makefile b/drivers/perf/Makefile index a06338e3401c..29b1c28203ef 100644 --- a/drivers/perf/Makefile +++ b/drivers/perf/Makefile @@ -15,6 +15,7 @@ obj-$(CONFIG_QCOM_L3_PMU) += qcom_l3_pmu.o obj-$(CONFIG_RISCV_PMU) += riscv_pmu.o obj-$(CONFIG_RISCV_PMU_LEGACY) += riscv_pmu_legacy.o obj-$(CONFIG_RISCV_PMU_SBI) += riscv_pmu_sbi.o +obj-$(CONFIG_STARFIVE_STARLINK_PMU) += starfive_starlink_pmu.o obj-$(CONFIG_THUNDERX2_PMU) += thunderx2_pmu.o obj-$(CONFIG_XGENE_PMU) += xgene_pmu.o obj-$(CONFIG_ARM_SPE_PMU) += arm_spe_pmu.o diff --git a/drivers/perf/alibaba_uncore_drw_pmu.c b/drivers/perf/alibaba_uncore_drw_pmu.c index 19d459a36be5..a9277dcf90ce 100644 --- a/drivers/perf/alibaba_uncore_drw_pmu.c +++ b/drivers/perf/alibaba_uncore_drw_pmu.c @@ -729,7 +729,7 @@ static int ali_drw_pmu_probe(struct platform_device *pdev) return ret; } -static int ali_drw_pmu_remove(struct platform_device *pdev) +static void ali_drw_pmu_remove(struct platform_device *pdev) { struct ali_drw_pmu *drw_pmu = platform_get_drvdata(pdev); @@ -739,8 +739,6 @@ static int ali_drw_pmu_remove(struct platform_device *pdev) ali_drw_pmu_uninit_irq(drw_pmu); perf_pmu_unregister(&drw_pmu->pmu); - - return 0; } static int ali_drw_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) @@ -795,7 +793,7 @@ static struct platform_driver ali_drw_pmu_driver = { .acpi_match_table = ali_drw_acpi_match, }, .probe = ali_drw_pmu_probe, - .remove = ali_drw_pmu_remove, + .remove_new = ali_drw_pmu_remove, }; static int __init ali_drw_pmu_init(void) diff --git a/drivers/perf/amlogic/meson_g12_ddr_pmu.c b/drivers/perf/amlogic/meson_g12_ddr_pmu.c index 15d52ab3276a..99cc791892bc 100644 --- a/drivers/perf/amlogic/meson_g12_ddr_pmu.c +++ b/drivers/perf/amlogic/meson_g12_ddr_pmu.c @@ -355,11 +355,9 @@ static int g12_ddr_pmu_probe(struct platform_device *pdev) return meson_ddr_pmu_create(pdev); } -static int g12_ddr_pmu_remove(struct platform_device *pdev) +static void g12_ddr_pmu_remove(struct platform_device *pdev) { meson_ddr_pmu_remove(pdev); - - return 0; } static const struct of_device_id meson_ddr_pmu_dt_match[] = { @@ -381,7 +379,7 @@ MODULE_DEVICE_TABLE(of, meson_ddr_pmu_dt_match); static struct platform_driver g12_ddr_pmu_driver = { .probe = g12_ddr_pmu_probe, - .remove = g12_ddr_pmu_remove, + .remove_new = g12_ddr_pmu_remove, .driver = { .name = "meson-g12-ddr-pmu", diff --git a/drivers/perf/arm-cci.c b/drivers/perf/arm-cci.c index 61de861eaf91..6be03f81ae5d 100644 --- a/drivers/perf/arm-cci.c +++ b/drivers/perf/arm-cci.c @@ -1697,16 +1697,14 @@ error_pmu_init: return ret; } -static int cci_pmu_remove(struct platform_device *pdev) +static void cci_pmu_remove(struct platform_device *pdev) { if (!g_cci_pmu) - return 0; + return; cpuhp_remove_state(CPUHP_AP_PERF_ARM_CCI_ONLINE); perf_pmu_unregister(&g_cci_pmu->pmu); g_cci_pmu = NULL; - - return 0; } static struct platform_driver cci_pmu_driver = { @@ -1716,7 +1714,7 @@ static struct platform_driver cci_pmu_driver = { .suppress_bind_attrs = true, }, .probe = cci_pmu_probe, - .remove = cci_pmu_remove, + .remove_new = cci_pmu_remove, }; module_platform_driver(cci_pmu_driver); diff --git a/drivers/perf/arm-ccn.c b/drivers/perf/arm-ccn.c index 728d13d8e98a..641471bd5eff 100644 --- a/drivers/perf/arm-ccn.c +++ b/drivers/perf/arm-ccn.c @@ -1515,13 +1515,11 @@ static int arm_ccn_probe(struct platform_device *pdev) return arm_ccn_pmu_init(ccn); } -static int arm_ccn_remove(struct platform_device *pdev) +static void arm_ccn_remove(struct platform_device *pdev) { struct arm_ccn *ccn = platform_get_drvdata(pdev); arm_ccn_pmu_cleanup(ccn); - - return 0; } static const struct of_device_id arm_ccn_match[] = { @@ -1539,7 +1537,7 @@ static struct platform_driver arm_ccn_driver = { .suppress_bind_attrs = true, }, .probe = arm_ccn_probe, - .remove = arm_ccn_remove, + .remove_new = arm_ccn_remove, }; static int __init arm_ccn_init(void) diff --git a/drivers/perf/arm-cmn.c b/drivers/perf/arm-cmn.c index c584165b13ba..564da632b8d1 100644 --- a/drivers/perf/arm-cmn.c +++ b/drivers/perf/arm-cmn.c @@ -493,6 +493,7 @@ static void arm_cmn_show_logid(struct seq_file *s, int x, int y, int p, int d) for (dn = cmn->dns; dn->type; dn++) { struct arm_cmn_nodeid nid = arm_cmn_nid(cmn, dn->id); + int pad = dn->logid < 10; if (dn->type == CMN_TYPE_XP) continue; @@ -503,7 +504,7 @@ static void arm_cmn_show_logid(struct seq_file *s, int x, int y, int p, int d) if (nid.x != x || nid.y != y || nid.port != p || nid.dev != d) continue; - seq_printf(s, " #%-2d |", dn->logid); + seq_printf(s, " %*c#%-*d |", pad + 1, ' ', 3 - pad, dn->logid); return; } seq_puts(s, " |"); @@ -516,7 +517,7 @@ static int arm_cmn_map_show(struct seq_file *s, void *data) seq_puts(s, " X"); for (x = 0; x < cmn->mesh_x; x++) - seq_printf(s, " %d ", x); + seq_printf(s, " %-2d ", x); seq_puts(s, "\nY P D+"); y = cmn->mesh_y; while (y--) { @@ -526,13 +527,13 @@ static int arm_cmn_map_show(struct seq_file *s, void *data) for (x = 0; x < cmn->mesh_x; x++) seq_puts(s, "--------+"); - seq_printf(s, "\n%d |", y); + seq_printf(s, "\n%-2d |", y); for (x = 0; x < cmn->mesh_x; x++) { struct arm_cmn_node *xp = cmn->xps + xp_base + x; for (p = 0; p < CMN_MAX_PORTS; p++) port[p][x] = arm_cmn_device_connect_info(cmn, xp, p); - seq_printf(s, " XP #%-2d |", xp_base + x); + seq_printf(s, " XP #%-3d|", xp_base + x); } seq_puts(s, "\n |"); @@ -2504,7 +2505,7 @@ static int arm_cmn_probe(struct platform_device *pdev) return err; } -static int arm_cmn_remove(struct platform_device *pdev) +static void arm_cmn_remove(struct platform_device *pdev) { struct arm_cmn *cmn = platform_get_drvdata(pdev); @@ -2513,7 +2514,6 @@ static int arm_cmn_remove(struct platform_device *pdev) perf_pmu_unregister(&cmn->pmu); cpuhp_state_remove_instance_nocalls(arm_cmn_hp_state, &cmn->cpuhp_node); debugfs_remove(cmn->debug); - return 0; } #ifdef CONFIG_OF @@ -2544,7 +2544,7 @@ static struct platform_driver arm_cmn_driver = { .acpi_match_table = ACPI_PTR(arm_cmn_acpi_match), }, .probe = arm_cmn_probe, - .remove = arm_cmn_remove, + .remove_new = arm_cmn_remove, }; static int __init arm_cmn_init(void) diff --git a/drivers/perf/arm_cspmu/arm_cspmu.c b/drivers/perf/arm_cspmu/arm_cspmu.c index 50b89b989ce7..b9a252272f1e 100644 --- a/drivers/perf/arm_cspmu/arm_cspmu.c +++ b/drivers/perf/arm_cspmu/arm_cspmu.c @@ -27,6 +27,7 @@ #include <linux/io-64-nonatomic-lo-hi.h> #include <linux/module.h> #include <linux/mutex.h> +#include <linux/of.h> #include <linux/perf_event.h> #include <linux/platform_device.h> @@ -100,13 +101,6 @@ #define ARM_CSPMU_ACTIVE_CPU_MASK 0x0 #define ARM_CSPMU_ASSOCIATED_CPU_MASK 0x1 -/* Check and use default if implementer doesn't provide attribute callback */ -#define CHECK_DEFAULT_IMPL_OPS(ops, callback) \ - do { \ - if (!ops->callback) \ - ops->callback = arm_cspmu_ ## callback; \ - } while (0) - /* * Maximum poll count for reading counter value using high-low-high sequence. */ @@ -121,7 +115,9 @@ static void arm_cspmu_set_ev_filter(struct arm_cspmu *cspmu, static struct acpi_apmt_node *arm_cspmu_apmt_node(struct device *dev) { - return *(struct acpi_apmt_node **)dev_get_platdata(dev); + struct acpi_apmt_node **ptr = dev_get_platdata(dev); + + return ptr ? *ptr : NULL; } /* @@ -317,6 +313,10 @@ static const char *arm_cspmu_get_name(const struct arm_cspmu *cspmu) dev = cspmu->dev; apmt_node = arm_cspmu_apmt_node(dev); + if (!apmt_node) + return devm_kasprintf(dev, GFP_KERNEL, PMUNAME "_%u", + atomic_fetch_inc(&pmu_idx[0])); + pmu_type = apmt_node->type; if (pmu_type >= ACPI_APMT_NODE_TYPE_COUNT) { @@ -408,21 +408,32 @@ static struct arm_cspmu_impl_match *arm_cspmu_impl_match_get(u32 pmiidr) return NULL; } +#define DEFAULT_IMPL_OP(name) .name = arm_cspmu_##name + static int arm_cspmu_init_impl_ops(struct arm_cspmu *cspmu) { int ret = 0; - struct arm_cspmu_impl_ops *impl_ops = &cspmu->impl.ops; struct acpi_apmt_node *apmt_node = arm_cspmu_apmt_node(cspmu->dev); struct arm_cspmu_impl_match *match; - /* - * Get PMU implementer and product id from APMT node. - * If APMT node doesn't have implementer/product id, try get it - * from PMIIDR. - */ - cspmu->impl.pmiidr = - (apmt_node->impl_id) ? apmt_node->impl_id : - readl(cspmu->base0 + PMIIDR); + /* Start with a default PMU implementation */ + cspmu->impl.module = THIS_MODULE; + cspmu->impl.pmiidr = readl(cspmu->base0 + PMIIDR); + cspmu->impl.ops = (struct arm_cspmu_impl_ops) { + DEFAULT_IMPL_OP(get_event_attrs), + DEFAULT_IMPL_OP(get_format_attrs), + DEFAULT_IMPL_OP(get_identifier), + DEFAULT_IMPL_OP(get_name), + DEFAULT_IMPL_OP(is_cycle_counter_event), + DEFAULT_IMPL_OP(event_type), + DEFAULT_IMPL_OP(event_filter), + DEFAULT_IMPL_OP(set_ev_filter), + DEFAULT_IMPL_OP(event_attr_is_visible), + }; + + /* Firmware may override implementer/product ID from PMIIDR */ + if (apmt_node && apmt_node->impl_id) + cspmu->impl.pmiidr = apmt_node->impl_id; /* Find implementer specific attribute ops. */ match = arm_cspmu_impl_match_get(cspmu->impl.pmiidr); @@ -450,24 +461,9 @@ static int arm_cspmu_init_impl_ops(struct arm_cspmu *cspmu) } mutex_unlock(&arm_cspmu_lock); + } - if (ret) - return ret; - } else - cspmu->impl.module = THIS_MODULE; - - /* Use default callbacks if implementer doesn't provide one. */ - CHECK_DEFAULT_IMPL_OPS(impl_ops, get_event_attrs); - CHECK_DEFAULT_IMPL_OPS(impl_ops, get_format_attrs); - CHECK_DEFAULT_IMPL_OPS(impl_ops, get_identifier); - CHECK_DEFAULT_IMPL_OPS(impl_ops, get_name); - CHECK_DEFAULT_IMPL_OPS(impl_ops, is_cycle_counter_event); - CHECK_DEFAULT_IMPL_OPS(impl_ops, event_type); - CHECK_DEFAULT_IMPL_OPS(impl_ops, event_filter); - CHECK_DEFAULT_IMPL_OPS(impl_ops, event_attr_is_visible); - CHECK_DEFAULT_IMPL_OPS(impl_ops, set_ev_filter); - - return 0; + return ret; } static struct attribute_group * @@ -512,23 +508,16 @@ arm_cspmu_alloc_format_attr_group(struct arm_cspmu *cspmu) return format_group; } -static struct attribute_group ** -arm_cspmu_alloc_attr_group(struct arm_cspmu *cspmu) +static int arm_cspmu_alloc_attr_groups(struct arm_cspmu *cspmu) { - struct attribute_group **attr_groups = NULL; - struct device *dev = cspmu->dev; + const struct attribute_group **attr_groups = cspmu->attr_groups; const struct arm_cspmu_impl_ops *impl_ops = &cspmu->impl.ops; cspmu->identifier = impl_ops->get_identifier(cspmu); cspmu->name = impl_ops->get_name(cspmu); if (!cspmu->identifier || !cspmu->name) - return NULL; - - attr_groups = devm_kcalloc(dev, 5, sizeof(struct attribute_group *), - GFP_KERNEL); - if (!attr_groups) - return NULL; + return -ENOMEM; attr_groups[0] = arm_cspmu_alloc_event_attr_group(cspmu); attr_groups[1] = arm_cspmu_alloc_format_attr_group(cspmu); @@ -536,18 +525,14 @@ arm_cspmu_alloc_attr_group(struct arm_cspmu *cspmu) attr_groups[3] = &arm_cspmu_cpumask_attr_group; if (!attr_groups[0] || !attr_groups[1]) - return NULL; + return -ENOMEM; - return attr_groups; + return 0; } static inline void arm_cspmu_reset_counters(struct arm_cspmu *cspmu) { - u32 pmcr = 0; - - pmcr |= PMCR_P; - pmcr |= PMCR_C; - writel(pmcr, cspmu->base0 + PMCR); + writel(PMCR_C | PMCR_P, cspmu->base0 + PMCR); } static inline void arm_cspmu_start_counters(struct arm_cspmu *cspmu) @@ -962,7 +947,14 @@ static struct arm_cspmu *arm_cspmu_alloc(struct platform_device *pdev) platform_set_drvdata(pdev, cspmu); apmt_node = arm_cspmu_apmt_node(dev); - cspmu->has_atomic_dword = apmt_node->flags & ACPI_APMT_FLAGS_ATOMIC; + if (apmt_node) { + cspmu->has_atomic_dword = apmt_node->flags & ACPI_APMT_FLAGS_ATOMIC; + } else { + u32 width = 0; + + device_property_read_u32(dev, "reg-io-width", &width); + cspmu->has_atomic_dword = (width == 8); + } return cspmu; } @@ -1153,11 +1145,6 @@ static int arm_cspmu_acpi_get_cpus(struct arm_cspmu *cspmu) } } - if (cpumask_empty(&cspmu->associated_cpus)) { - dev_dbg(cspmu->dev, "No cpu associated with the PMU\n"); - return -ENODEV; - } - return 0; } #else @@ -1167,19 +1154,45 @@ static int arm_cspmu_acpi_get_cpus(struct arm_cspmu *cspmu) } #endif +static int arm_cspmu_of_get_cpus(struct arm_cspmu *cspmu) +{ + struct of_phandle_iterator it; + int ret, cpu; + + of_for_each_phandle(&it, ret, dev_of_node(cspmu->dev), "cpus", NULL, 0) { + cpu = of_cpu_node_to_id(it.node); + if (cpu < 0) + continue; + cpumask_set_cpu(cpu, &cspmu->associated_cpus); + } + return ret == -ENOENT ? 0 : ret; +} + static int arm_cspmu_get_cpus(struct arm_cspmu *cspmu) { - return arm_cspmu_acpi_get_cpus(cspmu); + int ret = 0; + + if (arm_cspmu_apmt_node(cspmu->dev)) + ret = arm_cspmu_acpi_get_cpus(cspmu); + else if (device_property_present(cspmu->dev, "cpus")) + ret = arm_cspmu_of_get_cpus(cspmu); + else + cpumask_copy(&cspmu->associated_cpus, cpu_possible_mask); + + if (!ret && cpumask_empty(&cspmu->associated_cpus)) { + dev_dbg(cspmu->dev, "No cpu associated with the PMU\n"); + ret = -ENODEV; + } + return ret; } static int arm_cspmu_register_pmu(struct arm_cspmu *cspmu) { int ret, capabilities; - struct attribute_group **attr_groups; - attr_groups = arm_cspmu_alloc_attr_group(cspmu); - if (!attr_groups) - return -ENOMEM; + ret = arm_cspmu_alloc_attr_groups(cspmu); + if (ret) + return ret; ret = cpuhp_state_add_instance(arm_cspmu_cpuhp_state, &cspmu->cpuhp_node); @@ -1201,12 +1214,11 @@ static int arm_cspmu_register_pmu(struct arm_cspmu *cspmu) .start = arm_cspmu_start, .stop = arm_cspmu_stop, .read = arm_cspmu_read, - .attr_groups = (const struct attribute_group **)attr_groups, + .attr_groups = cspmu->attr_groups, .capabilities = capabilities, }; /* Hardware counter init */ - arm_cspmu_stop_counters(cspmu); arm_cspmu_reset_counters(cspmu); ret = perf_pmu_register(&cspmu->pmu, cspmu->name, -1); @@ -1252,14 +1264,12 @@ static int arm_cspmu_device_probe(struct platform_device *pdev) return ret; } -static int arm_cspmu_device_remove(struct platform_device *pdev) +static void arm_cspmu_device_remove(struct platform_device *pdev) { struct arm_cspmu *cspmu = platform_get_drvdata(pdev); perf_pmu_unregister(&cspmu->pmu); cpuhp_state_remove_instance(arm_cspmu_cpuhp_state, &cspmu->cpuhp_node); - - return 0; } static const struct platform_device_id arm_cspmu_id[] = { @@ -1268,13 +1278,20 @@ static const struct platform_device_id arm_cspmu_id[] = { }; MODULE_DEVICE_TABLE(platform, arm_cspmu_id); +static const struct of_device_id arm_cspmu_of_match[] = { + { .compatible = "arm,coresight-pmu" }, + {} +}; +MODULE_DEVICE_TABLE(of, arm_cspmu_of_match); + static struct platform_driver arm_cspmu_driver = { .driver = { - .name = DRVNAME, - .suppress_bind_attrs = true, - }, + .name = DRVNAME, + .of_match_table = arm_cspmu_of_match, + .suppress_bind_attrs = true, + }, .probe = arm_cspmu_device_probe, - .remove = arm_cspmu_device_remove, + .remove_new = arm_cspmu_device_remove, .id_table = arm_cspmu_id, }; diff --git a/drivers/perf/arm_cspmu/arm_cspmu.h b/drivers/perf/arm_cspmu/arm_cspmu.h index 2fe723555a6b..c9163acfe810 100644 --- a/drivers/perf/arm_cspmu/arm_cspmu.h +++ b/drivers/perf/arm_cspmu/arm_cspmu.h @@ -157,6 +157,7 @@ struct arm_cspmu { int cycle_counter_logical_idx; struct arm_cspmu_hw_events hw_events; + const struct attribute_group *attr_groups[5]; struct arm_cspmu_impl impl; }; diff --git a/drivers/perf/arm_cspmu/nvidia_cspmu.c b/drivers/perf/arm_cspmu/nvidia_cspmu.c index 0382b702f092..5b84b701ad62 100644 --- a/drivers/perf/arm_cspmu/nvidia_cspmu.c +++ b/drivers/perf/arm_cspmu/nvidia_cspmu.c @@ -388,12 +388,6 @@ static int nv_cspmu_init_ops(struct arm_cspmu *cspmu) impl_ops->get_format_attrs = nv_cspmu_get_format_attrs; impl_ops->get_name = nv_cspmu_get_name; - /* Set others to NULL to use default callback. */ - impl_ops->event_type = NULL; - impl_ops->event_attr_is_visible = NULL; - impl_ops->get_identifier = NULL; - impl_ops->is_cycle_counter_event = NULL; - return 0; } diff --git a/drivers/perf/arm_dmc620_pmu.c b/drivers/perf/arm_dmc620_pmu.c index 30cea6859574..8a81be2dd5ec 100644 --- a/drivers/perf/arm_dmc620_pmu.c +++ b/drivers/perf/arm_dmc620_pmu.c @@ -724,7 +724,7 @@ out_teardown_dev: return ret; } -static int dmc620_pmu_device_remove(struct platform_device *pdev) +static void dmc620_pmu_device_remove(struct platform_device *pdev) { struct dmc620_pmu *dmc620_pmu = platform_get_drvdata(pdev); @@ -732,8 +732,6 @@ static int dmc620_pmu_device_remove(struct platform_device *pdev) /* perf will synchronise RCU before devres can free dmc620_pmu */ perf_pmu_unregister(&dmc620_pmu->pmu); - - return 0; } static const struct acpi_device_id dmc620_acpi_match[] = { @@ -748,7 +746,7 @@ static struct platform_driver dmc620_pmu_driver = { .suppress_bind_attrs = true, }, .probe = dmc620_pmu_device_probe, - .remove = dmc620_pmu_device_remove, + .remove_new = dmc620_pmu_device_remove, }; static int __init dmc620_pmu_init(void) diff --git a/drivers/perf/arm_dsu_pmu.c b/drivers/perf/arm_dsu_pmu.c index 7ec4498e312f..bae3ca37f846 100644 --- a/drivers/perf/arm_dsu_pmu.c +++ b/drivers/perf/arm_dsu_pmu.c @@ -774,14 +774,12 @@ static int dsu_pmu_device_probe(struct platform_device *pdev) return rc; } -static int dsu_pmu_device_remove(struct platform_device *pdev) +static void dsu_pmu_device_remove(struct platform_device *pdev) { struct dsu_pmu *dsu_pmu = platform_get_drvdata(pdev); perf_pmu_unregister(&dsu_pmu->pmu); cpuhp_state_remove_instance(dsu_pmu_cpuhp_state, &dsu_pmu->cpuhp_node); - - return 0; } static const struct of_device_id dsu_pmu_of_match[] = { @@ -806,7 +804,7 @@ static struct platform_driver dsu_pmu_driver = { .suppress_bind_attrs = true, }, .probe = dsu_pmu_device_probe, - .remove = dsu_pmu_device_remove, + .remove_new = dsu_pmu_device_remove, }; static int dsu_pmu_cpu_online(unsigned int cpu, struct hlist_node *node) diff --git a/drivers/perf/arm_smmuv3_pmu.c b/drivers/perf/arm_smmuv3_pmu.c index 6303b82566f9..af3ab3eb944e 100644 --- a/drivers/perf/arm_smmuv3_pmu.c +++ b/drivers/perf/arm_smmuv3_pmu.c @@ -965,14 +965,12 @@ out_unregister: return err; } -static int smmu_pmu_remove(struct platform_device *pdev) +static void smmu_pmu_remove(struct platform_device *pdev) { struct smmu_pmu *smmu_pmu = platform_get_drvdata(pdev); perf_pmu_unregister(&smmu_pmu->pmu); cpuhp_state_remove_instance_nocalls(cpuhp_state_num, &smmu_pmu->node); - - return 0; } static void smmu_pmu_shutdown(struct platform_device *pdev) @@ -997,7 +995,7 @@ static struct platform_driver smmu_pmu_driver = { .suppress_bind_attrs = true, }, .probe = smmu_pmu_probe, - .remove = smmu_pmu_remove, + .remove_new = smmu_pmu_remove, .shutdown = smmu_pmu_shutdown, }; diff --git a/drivers/perf/arm_spe_pmu.c b/drivers/perf/arm_spe_pmu.c index b622d75d8c9e..35f0de03416f 100644 --- a/drivers/perf/arm_spe_pmu.c +++ b/drivers/perf/arm_spe_pmu.c @@ -1263,14 +1263,13 @@ out_free_handle: return ret; } -static int arm_spe_pmu_device_remove(struct platform_device *pdev) +static void arm_spe_pmu_device_remove(struct platform_device *pdev) { struct arm_spe_pmu *spe_pmu = platform_get_drvdata(pdev); arm_spe_pmu_perf_destroy(spe_pmu); arm_spe_pmu_dev_teardown(spe_pmu); free_percpu(spe_pmu->handle); - return 0; } static struct platform_driver arm_spe_pmu_driver = { @@ -1281,7 +1280,7 @@ static struct platform_driver arm_spe_pmu_driver = { .suppress_bind_attrs = true, }, .probe = arm_spe_pmu_device_probe, - .remove = arm_spe_pmu_device_remove, + .remove_new = arm_spe_pmu_device_remove, }; static int __init arm_spe_pmu_init(void) diff --git a/drivers/perf/fsl_imx8_ddr_perf.c b/drivers/perf/fsl_imx8_ddr_perf.c index 7dbfaee372c7..4e8fa5a48fcf 100644 --- a/drivers/perf/fsl_imx8_ddr_perf.c +++ b/drivers/perf/fsl_imx8_ddr_perf.c @@ -826,7 +826,7 @@ cpuhp_state_err: return ret; } -static int ddr_perf_remove(struct platform_device *pdev) +static void ddr_perf_remove(struct platform_device *pdev) { struct ddr_pmu *pmu = platform_get_drvdata(pdev); @@ -836,7 +836,6 @@ static int ddr_perf_remove(struct platform_device *pdev) perf_pmu_unregister(&pmu->pmu); ida_free(&ddr_ida, pmu->id); - return 0; } static struct platform_driver imx_ddr_pmu_driver = { @@ -846,7 +845,7 @@ static struct platform_driver imx_ddr_pmu_driver = { .suppress_bind_attrs = true, }, .probe = ddr_perf_probe, - .remove = ddr_perf_remove, + .remove_new = ddr_perf_remove, }; module_platform_driver(imx_ddr_pmu_driver); diff --git a/drivers/perf/fsl_imx9_ddr_perf.c b/drivers/perf/fsl_imx9_ddr_perf.c index 9685645bfe04..72c2d3074cde 100644 --- a/drivers/perf/fsl_imx9_ddr_perf.c +++ b/drivers/perf/fsl_imx9_ddr_perf.c @@ -679,7 +679,7 @@ format_string_err: return ret; } -static int ddr_perf_remove(struct platform_device *pdev) +static void ddr_perf_remove(struct platform_device *pdev) { struct ddr_pmu *pmu = platform_get_drvdata(pdev); @@ -689,8 +689,6 @@ static int ddr_perf_remove(struct platform_device *pdev) perf_pmu_unregister(&pmu->pmu); ida_free(&ddr_ida, pmu->id); - - return 0; } static struct platform_driver imx_ddr_pmu_driver = { @@ -700,7 +698,7 @@ static struct platform_driver imx_ddr_pmu_driver = { .suppress_bind_attrs = true, }, .probe = ddr_perf_probe, - .remove = ddr_perf_remove, + .remove_new = ddr_perf_remove, }; module_platform_driver(imx_ddr_pmu_driver); diff --git a/drivers/perf/hisilicon/hisi_pcie_pmu.c b/drivers/perf/hisilicon/hisi_pcie_pmu.c index b90ba8aca3fa..5d1f0e9fdb08 100644 --- a/drivers/perf/hisilicon/hisi_pcie_pmu.c +++ b/drivers/perf/hisilicon/hisi_pcie_pmu.c @@ -216,10 +216,8 @@ static void hisi_pcie_pmu_writeq(struct hisi_pcie_pmu *pcie_pmu, u32 reg_offset, writeq_relaxed(val, pcie_pmu->base + offset); } -static void hisi_pcie_pmu_config_filter(struct perf_event *event) +static u64 hisi_pcie_pmu_get_event_ctrl_val(struct perf_event *event) { - struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); - struct hw_perf_event *hwc = &event->hw; u64 port, trig_len, thr_len, len_mode; u64 reg = HISI_PCIE_INIT_SET; @@ -256,10 +254,19 @@ static void hisi_pcie_pmu_config_filter(struct perf_event *event) else reg |= FIELD_PREP(HISI_PCIE_LEN_M, HISI_PCIE_LEN_M_DEFAULT); + return reg; +} + +static void hisi_pcie_pmu_config_event_ctrl(struct perf_event *event) +{ + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); + struct hw_perf_event *hwc = &event->hw; + u64 reg = hisi_pcie_pmu_get_event_ctrl_val(event); + hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, hwc->idx, reg); } -static void hisi_pcie_pmu_clear_filter(struct perf_event *event) +static void hisi_pcie_pmu_clear_event_ctrl(struct perf_event *event) { struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); struct hw_perf_event *hwc = &event->hw; @@ -299,18 +306,24 @@ static bool hisi_pcie_pmu_valid_filter(struct perf_event *event, if (hisi_pcie_get_trig_len(event) > HISI_PCIE_TRIG_MAX_VAL) return false; - if (requester_id) { - if (!hisi_pcie_pmu_valid_requester_id(pcie_pmu, requester_id)) - return false; - } + /* Need to explicitly set filter of "port" or "bdf" */ + if (!hisi_pcie_get_port(event) && + !hisi_pcie_pmu_valid_requester_id(pcie_pmu, requester_id)) + return false; return true; } +/* + * Check Whether two events share the same config. The same config means not + * only the event code, but also the filter settings of the two events are + * the same. + */ static bool hisi_pcie_pmu_cmp_event(struct perf_event *target, struct perf_event *event) { - return hisi_pcie_get_real_event(target) == hisi_pcie_get_real_event(event); + return hisi_pcie_pmu_get_event_ctrl_val(target) == + hisi_pcie_pmu_get_event_ctrl_val(event); } static bool hisi_pcie_pmu_validate_event_group(struct perf_event *event) @@ -385,40 +398,32 @@ static u64 hisi_pcie_pmu_read_counter(struct perf_event *event) return hisi_pcie_pmu_readq(pcie_pmu, event->hw.event_base, idx); } -static int hisi_pcie_pmu_find_related_event(struct hisi_pcie_pmu *pcie_pmu, - struct perf_event *event) +/* + * Check all work events, if a relevant event is found then we return it + * first, otherwise return the first idle counter (need to reset). + */ +static int hisi_pcie_pmu_get_event_idx(struct hisi_pcie_pmu *pcie_pmu, + struct perf_event *event) { + int first_idle = -EAGAIN; struct perf_event *sibling; int idx; for (idx = 0; idx < HISI_PCIE_MAX_COUNTERS; idx++) { sibling = pcie_pmu->hw_events[idx]; - if (!sibling) - continue; - - if (!hisi_pcie_pmu_cmp_event(sibling, event)) + if (!sibling) { + if (first_idle == -EAGAIN) + first_idle = idx; continue; + } /* Related events must be used in group */ - if (sibling->group_leader == event->group_leader) + if (hisi_pcie_pmu_cmp_event(sibling, event) && + sibling->group_leader == event->group_leader) return idx; - else - return -EINVAL; } - return idx; -} - -static int hisi_pcie_pmu_get_event_idx(struct hisi_pcie_pmu *pcie_pmu) -{ - int idx; - - for (idx = 0; idx < HISI_PCIE_MAX_COUNTERS; idx++) { - if (!pcie_pmu->hw_events[idx]) - return idx; - } - - return -EINVAL; + return first_idle; } static void hisi_pcie_pmu_event_update(struct perf_event *event) @@ -505,7 +510,7 @@ static void hisi_pcie_pmu_start(struct perf_event *event, int flags) WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); hwc->state = 0; - hisi_pcie_pmu_config_filter(event); + hisi_pcie_pmu_config_event_ctrl(event); hisi_pcie_pmu_enable_counter(pcie_pmu, hwc); hisi_pcie_pmu_enable_int(pcie_pmu, hwc); hisi_pcie_pmu_set_period(event); @@ -526,7 +531,7 @@ static void hisi_pcie_pmu_stop(struct perf_event *event, int flags) hisi_pcie_pmu_event_update(event); hisi_pcie_pmu_disable_int(pcie_pmu, hwc); hisi_pcie_pmu_disable_counter(pcie_pmu, hwc); - hisi_pcie_pmu_clear_filter(event); + hisi_pcie_pmu_clear_event_ctrl(event); WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); hwc->state |= PERF_HES_STOPPED; @@ -544,27 +549,18 @@ static int hisi_pcie_pmu_add(struct perf_event *event, int flags) hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; - /* Check all working events to find a related event. */ - idx = hisi_pcie_pmu_find_related_event(pcie_pmu, event); - if (idx < 0) - return idx; - - /* Current event shares an enabled counter with the related event */ - if (idx < HISI_PCIE_MAX_COUNTERS) { - hwc->idx = idx; - goto start_count; - } - - idx = hisi_pcie_pmu_get_event_idx(pcie_pmu); + idx = hisi_pcie_pmu_get_event_idx(pcie_pmu, event); if (idx < 0) return idx; hwc->idx = idx; - pcie_pmu->hw_events[idx] = event; - /* Reset Counter to avoid previous statistic interference. */ - hisi_pcie_pmu_reset_counter(pcie_pmu, idx); -start_count: + /* No enabled counter found with related event, reset it */ + if (!pcie_pmu->hw_events[idx]) { + hisi_pcie_pmu_reset_counter(pcie_pmu, idx); + pcie_pmu->hw_events[idx] = event; + } + if (flags & PERF_EF_START) hisi_pcie_pmu_start(event, PERF_EF_RELOAD); @@ -714,10 +710,18 @@ static struct attribute *hisi_pcie_pmu_events_attr[] = { HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_cnt, 0x10210), HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_latency, 0x0011), HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_cnt, 0x10011), + HISI_PCIE_PMU_EVENT_ATTR(rx_mwr_flux, 0x0104), + HISI_PCIE_PMU_EVENT_ATTR(rx_mwr_time, 0x10104), HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_flux, 0x0804), HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_time, 0x10804), + HISI_PCIE_PMU_EVENT_ATTR(rx_cpl_flux, 0x2004), + HISI_PCIE_PMU_EVENT_ATTR(rx_cpl_time, 0x12004), + HISI_PCIE_PMU_EVENT_ATTR(tx_mwr_flux, 0x0105), + HISI_PCIE_PMU_EVENT_ATTR(tx_mwr_time, 0x10105), HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_flux, 0x0405), HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_time, 0x10405), + HISI_PCIE_PMU_EVENT_ATTR(tx_cpl_flux, 0x1005), + HISI_PCIE_PMU_EVENT_ATTR(tx_cpl_time, 0x11005), NULL }; diff --git a/drivers/perf/hisilicon/hisi_uncore_cpa_pmu.c b/drivers/perf/hisilicon/hisi_uncore_cpa_pmu.c index 40f1bc9f9b91..0e923f94fa5b 100644 --- a/drivers/perf/hisilicon/hisi_uncore_cpa_pmu.c +++ b/drivers/perf/hisilicon/hisi_uncore_cpa_pmu.c @@ -341,7 +341,7 @@ static int hisi_cpa_pmu_probe(struct platform_device *pdev) return ret; } -static int hisi_cpa_pmu_remove(struct platform_device *pdev) +static void hisi_cpa_pmu_remove(struct platform_device *pdev) { struct hisi_pmu *cpa_pmu = platform_get_drvdata(pdev); @@ -349,7 +349,6 @@ static int hisi_cpa_pmu_remove(struct platform_device *pdev) cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HISI_CPA_ONLINE, &cpa_pmu->node); hisi_cpa_pmu_enable_pm(cpa_pmu); - return 0; } static struct platform_driver hisi_cpa_pmu_driver = { @@ -359,7 +358,7 @@ static struct platform_driver hisi_cpa_pmu_driver = { .suppress_bind_attrs = true, }, .probe = hisi_cpa_pmu_probe, - .remove = hisi_cpa_pmu_remove, + .remove_new = hisi_cpa_pmu_remove, }; static int __init hisi_cpa_pmu_module_init(void) diff --git a/drivers/perf/hisilicon/hisi_uncore_ddrc_pmu.c b/drivers/perf/hisilicon/hisi_uncore_ddrc_pmu.c index ffb039d05d07..b804e3738113 100644 --- a/drivers/perf/hisilicon/hisi_uncore_ddrc_pmu.c +++ b/drivers/perf/hisilicon/hisi_uncore_ddrc_pmu.c @@ -531,14 +531,13 @@ static int hisi_ddrc_pmu_probe(struct platform_device *pdev) return ret; } -static int hisi_ddrc_pmu_remove(struct platform_device *pdev) +static void hisi_ddrc_pmu_remove(struct platform_device *pdev) { struct hisi_pmu *ddrc_pmu = platform_get_drvdata(pdev); perf_pmu_unregister(&ddrc_pmu->pmu); cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HISI_DDRC_ONLINE, &ddrc_pmu->node); - return 0; } static struct platform_driver hisi_ddrc_pmu_driver = { @@ -548,7 +547,7 @@ static struct platform_driver hisi_ddrc_pmu_driver = { .suppress_bind_attrs = true, }, .probe = hisi_ddrc_pmu_probe, - .remove = hisi_ddrc_pmu_remove, + .remove_new = hisi_ddrc_pmu_remove, }; static int __init hisi_ddrc_pmu_module_init(void) diff --git a/drivers/perf/hisilicon/hisi_uncore_hha_pmu.c b/drivers/perf/hisilicon/hisi_uncore_hha_pmu.c index 15caf99e1eef..21e69b1cdd4d 100644 --- a/drivers/perf/hisilicon/hisi_uncore_hha_pmu.c +++ b/drivers/perf/hisilicon/hisi_uncore_hha_pmu.c @@ -534,14 +534,13 @@ static int hisi_hha_pmu_probe(struct platform_device *pdev) return ret; } -static int hisi_hha_pmu_remove(struct platform_device *pdev) +static void hisi_hha_pmu_remove(struct platform_device *pdev) { struct hisi_pmu *hha_pmu = platform_get_drvdata(pdev); perf_pmu_unregister(&hha_pmu->pmu); cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HISI_HHA_ONLINE, &hha_pmu->node); - return 0; } static struct platform_driver hisi_hha_pmu_driver = { @@ -551,7 +550,7 @@ static struct platform_driver hisi_hha_pmu_driver = { .suppress_bind_attrs = true, }, .probe = hisi_hha_pmu_probe, - .remove = hisi_hha_pmu_remove, + .remove_new = hisi_hha_pmu_remove, }; static int __init hisi_hha_pmu_module_init(void) diff --git a/drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c b/drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c index 794dbcd19b7a..51ba76871097 100644 --- a/drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c +++ b/drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c @@ -568,14 +568,13 @@ static int hisi_l3c_pmu_probe(struct platform_device *pdev) return ret; } -static int hisi_l3c_pmu_remove(struct platform_device *pdev) +static void hisi_l3c_pmu_remove(struct platform_device *pdev) { struct hisi_pmu *l3c_pmu = platform_get_drvdata(pdev); perf_pmu_unregister(&l3c_pmu->pmu); cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HISI_L3_ONLINE, &l3c_pmu->node); - return 0; } static struct platform_driver hisi_l3c_pmu_driver = { @@ -585,7 +584,7 @@ static struct platform_driver hisi_l3c_pmu_driver = { .suppress_bind_attrs = true, }, .probe = hisi_l3c_pmu_probe, - .remove = hisi_l3c_pmu_remove, + .remove_new = hisi_l3c_pmu_remove, }; static int __init hisi_l3c_pmu_module_init(void) diff --git a/drivers/perf/hisilicon/hisi_uncore_pa_pmu.c b/drivers/perf/hisilicon/hisi_uncore_pa_pmu.c index 797cf201996a..3cdb35c741f9 100644 --- a/drivers/perf/hisilicon/hisi_uncore_pa_pmu.c +++ b/drivers/perf/hisilicon/hisi_uncore_pa_pmu.c @@ -514,14 +514,13 @@ static int hisi_pa_pmu_probe(struct platform_device *pdev) return ret; } -static int hisi_pa_pmu_remove(struct platform_device *pdev) +static void hisi_pa_pmu_remove(struct platform_device *pdev) { struct hisi_pmu *pa_pmu = platform_get_drvdata(pdev); perf_pmu_unregister(&pa_pmu->pmu); cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HISI_PA_ONLINE, &pa_pmu->node); - return 0; } static const struct acpi_device_id hisi_pa_pmu_acpi_match[] = { @@ -539,7 +538,7 @@ static struct platform_driver hisi_pa_pmu_driver = { .suppress_bind_attrs = true, }, .probe = hisi_pa_pmu_probe, - .remove = hisi_pa_pmu_remove, + .remove_new = hisi_pa_pmu_remove, }; static int __init hisi_pa_pmu_module_init(void) diff --git a/drivers/perf/hisilicon/hisi_uncore_sllc_pmu.c b/drivers/perf/hisilicon/hisi_uncore_sllc_pmu.c index e706ca567676..765bbd61db26 100644 --- a/drivers/perf/hisilicon/hisi_uncore_sllc_pmu.c +++ b/drivers/perf/hisilicon/hisi_uncore_sllc_pmu.c @@ -460,14 +460,13 @@ static int hisi_sllc_pmu_probe(struct platform_device *pdev) return ret; } -static int hisi_sllc_pmu_remove(struct platform_device *pdev) +static void hisi_sllc_pmu_remove(struct platform_device *pdev) { struct hisi_pmu *sllc_pmu = platform_get_drvdata(pdev); perf_pmu_unregister(&sllc_pmu->pmu); cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HISI_SLLC_ONLINE, &sllc_pmu->node); - return 0; } static struct platform_driver hisi_sllc_pmu_driver = { @@ -477,7 +476,7 @@ static struct platform_driver hisi_sllc_pmu_driver = { .suppress_bind_attrs = true, }, .probe = hisi_sllc_pmu_probe, - .remove = hisi_sllc_pmu_remove, + .remove_new = hisi_sllc_pmu_remove, }; static int __init hisi_sllc_pmu_module_init(void) diff --git a/drivers/perf/hisilicon/hisi_uncore_uc_pmu.c b/drivers/perf/hisilicon/hisi_uncore_uc_pmu.c index 636fb79647c8..481dcc9e8fbf 100644 --- a/drivers/perf/hisilicon/hisi_uncore_uc_pmu.c +++ b/drivers/perf/hisilicon/hisi_uncore_uc_pmu.c @@ -287,12 +287,52 @@ static u64 hisi_uc_pmu_read_counter(struct hisi_pmu *uc_pmu, return readq(uc_pmu->base + HISI_UC_CNTR_REGn(hwc->idx)); } -static void hisi_uc_pmu_write_counter(struct hisi_pmu *uc_pmu, +static bool hisi_uc_pmu_get_glb_en_state(struct hisi_pmu *uc_pmu) +{ + u32 val; + + val = readl(uc_pmu->base + HISI_UC_EVENT_CTRL_REG); + return !!FIELD_GET(HISI_UC_EVENT_GLB_EN, val); +} + +static void hisi_uc_pmu_write_counter_normal(struct hisi_pmu *uc_pmu, struct hw_perf_event *hwc, u64 val) { writeq(val, uc_pmu->base + HISI_UC_CNTR_REGn(hwc->idx)); } +static void hisi_uc_pmu_write_counter_quirk_v2(struct hisi_pmu *uc_pmu, + struct hw_perf_event *hwc, u64 val) +{ + hisi_uc_pmu_start_counters(uc_pmu); + hisi_uc_pmu_write_counter_normal(uc_pmu, hwc, val); + hisi_uc_pmu_stop_counters(uc_pmu); +} + +static void hisi_uc_pmu_write_counter(struct hisi_pmu *uc_pmu, + struct hw_perf_event *hwc, u64 val) +{ + bool enable = hisi_uc_pmu_get_glb_en_state(uc_pmu); + bool erratum = uc_pmu->identifier == HISI_PMU_V2; + + /* + * HiSilicon UC PMU v2 suffers the erratum 162700402 that the + * PMU counter cannot be set due to the lack of clock under power + * saving mode. This will lead to error or inaccurate counts. + * The clock can be enabled by the PMU global enabling control. + * The irq handler and pmu_start() will call the function to set + * period. If the function under irq context, the PMU has been + * enabled therefore we set counter directly. Other situations + * the PMU is disabled, we need to enable it to turn on the + * counter clock to set period, and then restore PMU enable + * status, the counter can hold its value without a clock. + */ + if (enable || !erratum) + hisi_uc_pmu_write_counter_normal(uc_pmu, hwc, val); + else + hisi_uc_pmu_write_counter_quirk_v2(uc_pmu, hwc, val); +} + static void hisi_uc_pmu_enable_counter_int(struct hisi_pmu *uc_pmu, struct hw_perf_event *hwc) { diff --git a/drivers/perf/marvell_cn10k_ddr_pmu.c b/drivers/perf/marvell_cn10k_ddr_pmu.c index 524ba82bfce2..e2abca188dbe 100644 --- a/drivers/perf/marvell_cn10k_ddr_pmu.c +++ b/drivers/perf/marvell_cn10k_ddr_pmu.c @@ -697,7 +697,7 @@ error: return ret; } -static int cn10k_ddr_perf_remove(struct platform_device *pdev) +static void cn10k_ddr_perf_remove(struct platform_device *pdev) { struct cn10k_ddr_pmu *ddr_pmu = platform_get_drvdata(pdev); @@ -706,7 +706,6 @@ static int cn10k_ddr_perf_remove(struct platform_device *pdev) &ddr_pmu->node); perf_pmu_unregister(&ddr_pmu->pmu); - return 0; } #ifdef CONFIG_OF @@ -733,7 +732,7 @@ static struct platform_driver cn10k_ddr_pmu_driver = { .suppress_bind_attrs = true, }, .probe = cn10k_ddr_perf_probe, - .remove = cn10k_ddr_perf_remove, + .remove_new = cn10k_ddr_perf_remove, }; static int __init cn10k_ddr_pmu_init(void) diff --git a/drivers/perf/marvell_cn10k_tad_pmu.c b/drivers/perf/marvell_cn10k_tad_pmu.c index fec8e82edb95..9e635f355470 100644 --- a/drivers/perf/marvell_cn10k_tad_pmu.c +++ b/drivers/perf/marvell_cn10k_tad_pmu.c @@ -351,15 +351,13 @@ static int tad_pmu_probe(struct platform_device *pdev) return ret; } -static int tad_pmu_remove(struct platform_device *pdev) +static void tad_pmu_remove(struct platform_device *pdev) { struct tad_pmu *pmu = platform_get_drvdata(pdev); cpuhp_state_remove_instance_nocalls(tad_pmu_cpuhp_state, &pmu->node); perf_pmu_unregister(&pmu->pmu); - - return 0; } #ifdef CONFIG_OF @@ -385,7 +383,7 @@ static struct platform_driver tad_pmu_driver = { .suppress_bind_attrs = true, }, .probe = tad_pmu_probe, - .remove = tad_pmu_remove, + .remove_new = tad_pmu_remove, }; static int tad_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) diff --git a/drivers/perf/qcom_l2_pmu.c b/drivers/perf/qcom_l2_pmu.c index 3f9a98c17a89..148df5ae8ef8 100644 --- a/drivers/perf/qcom_l2_pmu.c +++ b/drivers/perf/qcom_l2_pmu.c @@ -965,7 +965,7 @@ out_unregister: return err; } -static int l2_cache_pmu_remove(struct platform_device *pdev) +static void l2_cache_pmu_remove(struct platform_device *pdev) { struct l2cache_pmu *l2cache_pmu = to_l2cache_pmu(platform_get_drvdata(pdev)); @@ -973,7 +973,6 @@ static int l2_cache_pmu_remove(struct platform_device *pdev) perf_pmu_unregister(&l2cache_pmu->pmu); cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE, &l2cache_pmu->node); - return 0; } static struct platform_driver l2_cache_pmu_driver = { @@ -983,7 +982,7 @@ static struct platform_driver l2_cache_pmu_driver = { .suppress_bind_attrs = true, }, .probe = l2_cache_pmu_probe, - .remove = l2_cache_pmu_remove, + .remove_new = l2_cache_pmu_remove, }; static int __init register_l2_cache_pmu_driver(void) diff --git a/drivers/perf/starfive_starlink_pmu.c b/drivers/perf/starfive_starlink_pmu.c new file mode 100644 index 000000000000..5e5a672b4229 --- /dev/null +++ b/drivers/perf/starfive_starlink_pmu.c @@ -0,0 +1,642 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * StarFive's StarLink PMU driver + * + * Copyright (C) 2023 StarFive Technology Co., Ltd. + * + * Author: Ji Sheng Teoh <[email protected]> + * + */ + +#define STARLINK_PMU_PDEV_NAME "starfive_starlink_pmu" +#define pr_fmt(fmt) STARLINK_PMU_PDEV_NAME ": " fmt + +#include <linux/bitmap.h> +#include <linux/cpu_pm.h> +#include <linux/io.h> +#include <linux/irq.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/mod_devicetable.h> +#include <linux/perf_event.h> +#include <linux/platform_device.h> +#include <linux/sysfs.h> + +#define STARLINK_PMU_MAX_COUNTERS 64 +#define STARLINK_PMU_NUM_COUNTERS 16 +#define STARLINK_PMU_IDX_CYCLE_COUNTER 63 + +#define STARLINK_PMU_EVENT_SELECT 0x060 +#define STARLINK_PMU_EVENT_COUNTER 0x160 +#define STARLINK_PMU_COUNTER_MASK GENMASK_ULL(63, 0) +#define STARLINK_PMU_CYCLE_COUNTER 0x058 + +#define STARLINK_PMU_CONTROL 0x040 +#define STARLINK_PMU_GLOBAL_ENABLE BIT_ULL(0) + +#define STARLINK_PMU_INTERRUPT_ENABLE 0x050 +#define STARLINK_PMU_COUNTER_OVERFLOW_STATUS 0x048 +#define STARLINK_PMU_CYCLE_OVERFLOW_MASK BIT_ULL(63) + +#define STARLINK_CYCLES 0x058 +#define CACHE_READ_REQUEST 0x04000701 +#define CACHE_WRITE_REQUEST 0x03000001 +#define CACHE_RELEASE_REQUEST 0x0003e001 +#define CACHE_READ_HIT 0x00901202 +#define CACHE_READ_MISS 0x04008002 +#define CACHE_WRITE_HIT 0x006c0002 +#define CACHE_WRITE_MISS 0x03000002 +#define CACHE_WRITEBACK 0x00000403 + +#define to_starlink_pmu(p) (container_of(p, struct starlink_pmu, pmu)) + +#define STARLINK_FORMAT_ATTR(_name, _config) \ + (&((struct dev_ext_attribute[]) { \ + { .attr = __ATTR(_name, 0444, starlink_pmu_sysfs_format_show, NULL), \ + .var = (void *)_config, } \ + })[0].attr.attr) + +#define STARLINK_EVENT_ATTR(_name, _id) \ + PMU_EVENT_ATTR_ID(_name, starlink_pmu_sysfs_event_show, _id) + +static int starlink_pmu_cpuhp_state; + +struct starlink_hw_events { + struct perf_event *events[STARLINK_PMU_MAX_COUNTERS]; + DECLARE_BITMAP(used_mask, STARLINK_PMU_MAX_COUNTERS); +}; + +struct starlink_pmu { + struct pmu pmu; + struct starlink_hw_events __percpu *hw_events; + struct hlist_node node; + struct notifier_block starlink_pmu_pm_nb; + void __iomem *pmu_base; + cpumask_t cpumask; + int irq; +}; + +static ssize_t +starlink_pmu_sysfs_format_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct dev_ext_attribute *eattr = container_of(attr, + struct dev_ext_attribute, attr); + + return sysfs_emit(buf, "%s\n", (char *)eattr->var); +} + +static struct attribute *starlink_pmu_format_attrs[] = { + STARLINK_FORMAT_ATTR(event, "config:0-31"), + NULL +}; + +static const struct attribute_group starlink_pmu_format_attr_group = { + .name = "format", + .attrs = starlink_pmu_format_attrs, +}; + +static ssize_t +starlink_pmu_sysfs_event_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct perf_pmu_events_attr *eattr = container_of(attr, + struct perf_pmu_events_attr, attr); + + return sysfs_emit(buf, "event=0x%02llx\n", eattr->id); +} + +static struct attribute *starlink_pmu_event_attrs[] = { + STARLINK_EVENT_ATTR(cycles, STARLINK_CYCLES), + STARLINK_EVENT_ATTR(read_request, CACHE_READ_REQUEST), + STARLINK_EVENT_ATTR(write_request, CACHE_WRITE_REQUEST), + STARLINK_EVENT_ATTR(release_request, CACHE_RELEASE_REQUEST), + STARLINK_EVENT_ATTR(read_hit, CACHE_READ_HIT), + STARLINK_EVENT_ATTR(read_miss, CACHE_READ_MISS), + STARLINK_EVENT_ATTR(write_hit, CACHE_WRITE_HIT), + STARLINK_EVENT_ATTR(write_miss, CACHE_WRITE_MISS), + STARLINK_EVENT_ATTR(writeback, CACHE_WRITEBACK), + NULL +}; + +static const struct attribute_group starlink_pmu_events_attr_group = { + .name = "events", + .attrs = starlink_pmu_event_attrs, +}; + +static ssize_t +cpumask_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct starlink_pmu *starlink_pmu = to_starlink_pmu(dev_get_drvdata(dev)); + + return cpumap_print_to_pagebuf(true, buf, &starlink_pmu->cpumask); +} + +static DEVICE_ATTR_RO(cpumask); + +static struct attribute *starlink_pmu_cpumask_attrs[] = { + &dev_attr_cpumask.attr, + NULL +}; + +static const struct attribute_group starlink_pmu_cpumask_attr_group = { + .attrs = starlink_pmu_cpumask_attrs, +}; + +static const struct attribute_group *starlink_pmu_attr_groups[] = { + &starlink_pmu_format_attr_group, + &starlink_pmu_events_attr_group, + &starlink_pmu_cpumask_attr_group, + NULL +}; + +static void starlink_pmu_set_event_period(struct perf_event *event) +{ + struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu); + struct hw_perf_event *hwc = &event->hw; + int idx = event->hw.idx; + + /* + * Program counter to half of it's max count to handle + * cases of extreme interrupt latency. + */ + u64 val = STARLINK_PMU_COUNTER_MASK >> 1; + + local64_set(&hwc->prev_count, val); + if (hwc->config == STARLINK_CYCLES) + writeq(val, starlink_pmu->pmu_base + STARLINK_PMU_CYCLE_COUNTER); + else + writeq(val, starlink_pmu->pmu_base + STARLINK_PMU_EVENT_COUNTER + + idx * sizeof(u64)); +} + +static void starlink_pmu_counter_start(struct perf_event *event, + struct starlink_pmu *starlink_pmu) +{ + struct hw_perf_event *hwc = &event->hw; + int idx = event->hw.idx; + u64 val; + + /* + * Enable counter overflow interrupt[63:0], + * which is mapped as follow: + * + * event counter 0 - Bit [0] + * event counter 1 - Bit [1] + * ... + * cycle counter - Bit [63] + */ + val = readq(starlink_pmu->pmu_base + STARLINK_PMU_INTERRUPT_ENABLE); + + if (hwc->config == STARLINK_CYCLES) { + /* + * Cycle count has its dedicated register, and it starts + * counting as soon as STARLINK_PMU_GLOBAL_ENABLE is set. + */ + val |= STARLINK_PMU_CYCLE_OVERFLOW_MASK; + } else { + writeq(event->hw.config, starlink_pmu->pmu_base + + STARLINK_PMU_EVENT_SELECT + idx * sizeof(u64)); + + val |= BIT_ULL(idx); + } + + writeq(val, starlink_pmu->pmu_base + STARLINK_PMU_INTERRUPT_ENABLE); + + writeq(STARLINK_PMU_GLOBAL_ENABLE, starlink_pmu->pmu_base + + STARLINK_PMU_CONTROL); +} + +static void starlink_pmu_counter_stop(struct perf_event *event, + struct starlink_pmu *starlink_pmu) +{ + struct hw_perf_event *hwc = &event->hw; + int idx = event->hw.idx; + u64 val; + + val = readq(starlink_pmu->pmu_base + STARLINK_PMU_CONTROL); + val &= ~STARLINK_PMU_GLOBAL_ENABLE; + writeq(val, starlink_pmu->pmu_base + STARLINK_PMU_CONTROL); + + val = readq(starlink_pmu->pmu_base + STARLINK_PMU_INTERRUPT_ENABLE); + if (hwc->config == STARLINK_CYCLES) + val &= ~STARLINK_PMU_CYCLE_OVERFLOW_MASK; + else + val &= ~BIT_ULL(idx); + + writeq(val, starlink_pmu->pmu_base + STARLINK_PMU_INTERRUPT_ENABLE); +} + +static void starlink_pmu_update(struct perf_event *event) +{ + struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu); + struct hw_perf_event *hwc = &event->hw; + int idx = hwc->idx; + u64 prev_raw_count, new_raw_count; + u64 oldval; + u64 delta; + + do { + prev_raw_count = local64_read(&hwc->prev_count); + if (hwc->config == STARLINK_CYCLES) + new_raw_count = readq(starlink_pmu->pmu_base + + STARLINK_PMU_CYCLE_COUNTER); + else + new_raw_count = readq(starlink_pmu->pmu_base + + STARLINK_PMU_EVENT_COUNTER + + idx * sizeof(u64)); + oldval = local64_cmpxchg(&hwc->prev_count, prev_raw_count, + new_raw_count); + } while (oldval != prev_raw_count); + + delta = (new_raw_count - prev_raw_count) & STARLINK_PMU_COUNTER_MASK; + local64_add(delta, &event->count); +} + +static void starlink_pmu_start(struct perf_event *event, int flags) +{ + struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu); + struct hw_perf_event *hwc = &event->hw; + + if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED))) + return; + + if (flags & PERF_EF_RELOAD) + WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); + + hwc->state = 0; + + starlink_pmu_set_event_period(event); + starlink_pmu_counter_start(event, starlink_pmu); + + perf_event_update_userpage(event); +} + +static void starlink_pmu_stop(struct perf_event *event, int flags) +{ + struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu); + struct hw_perf_event *hwc = &event->hw; + + if (hwc->state & PERF_HES_STOPPED) + return; + + starlink_pmu_counter_stop(event, starlink_pmu); + starlink_pmu_update(event); + hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; +} + +static int starlink_pmu_add(struct perf_event *event, int flags) +{ + struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu); + struct starlink_hw_events *hw_events = + this_cpu_ptr(starlink_pmu->hw_events); + struct hw_perf_event *hwc = &event->hw; + unsigned long *used_mask = hw_events->used_mask; + u32 n_events = STARLINK_PMU_NUM_COUNTERS; + int idx; + + /* + * Cycle counter has dedicated register to hold counter value. + * Event other than cycle count has to be enabled through + * event select register, and assigned with independent counter + * as they appear. + */ + + if (hwc->config == STARLINK_CYCLES) { + idx = STARLINK_PMU_IDX_CYCLE_COUNTER; + } else { + idx = find_first_zero_bit(used_mask, n_events); + /* All counter are in use */ + if (idx < 0) + return idx; + + set_bit(idx, used_mask); + } + + hwc->idx = idx; + hw_events->events[idx] = event; + hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; + + if (flags & PERF_EF_START) + starlink_pmu_start(event, PERF_EF_RELOAD); + + perf_event_update_userpage(event); + + return 0; +} + +static void starlink_pmu_del(struct perf_event *event, int flags) +{ + struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu); + struct starlink_hw_events *hw_events = + this_cpu_ptr(starlink_pmu->hw_events); + struct hw_perf_event *hwc = &event->hw; + + starlink_pmu_stop(event, PERF_EF_UPDATE); + hw_events->events[hwc->idx] = NULL; + clear_bit(hwc->idx, hw_events->used_mask); + + perf_event_update_userpage(event); +} + +static bool starlink_pmu_validate_event_group(struct perf_event *event) +{ + struct perf_event *leader = event->group_leader; + struct perf_event *sibling; + int counter = 1; + + /* + * Ensure hardware events in the group are on the same PMU, + * software events are acceptable. + */ + if (event->group_leader->pmu != event->pmu && + !is_software_event(event->group_leader)) + return false; + + for_each_sibling_event(sibling, leader) { + if (sibling->pmu != event->pmu && !is_software_event(sibling)) + return false; + + counter++; + } + + return counter <= STARLINK_PMU_NUM_COUNTERS; +} + +static int starlink_pmu_event_init(struct perf_event *event) +{ + struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu); + struct hw_perf_event *hwc = &event->hw; + + /* + * Sampling is not supported, as counters are shared + * by all CPU. + */ + if (hwc->sample_period) + return -EOPNOTSUPP; + + /* + * Per-task and attach to a task are not supported, + * as uncore events are not specific to any CPU. + */ + if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK) + return -EOPNOTSUPP; + + if (!starlink_pmu_validate_event_group(event)) + return -EINVAL; + + hwc->idx = -1; + hwc->config = event->attr.config; + event->cpu = cpumask_first(&starlink_pmu->cpumask); + + return 0; +} + +static irqreturn_t starlink_pmu_handle_irq(int irq_num, void *data) +{ + struct starlink_pmu *starlink_pmu = data; + struct starlink_hw_events *hw_events = + this_cpu_ptr(starlink_pmu->hw_events); + bool handled = false; + int idx; + u64 overflow_status; + + for (idx = 0; idx < STARLINK_PMU_MAX_COUNTERS; idx++) { + struct perf_event *event = hw_events->events[idx]; + + if (!event) + continue; + + overflow_status = readq(starlink_pmu->pmu_base + + STARLINK_PMU_COUNTER_OVERFLOW_STATUS); + if (!(overflow_status & BIT_ULL(idx))) + continue; + + writeq(BIT_ULL(idx), starlink_pmu->pmu_base + + STARLINK_PMU_COUNTER_OVERFLOW_STATUS); + + starlink_pmu_update(event); + starlink_pmu_set_event_period(event); + handled = true; + } + return IRQ_RETVAL(handled); +} + +static int starlink_setup_irqs(struct starlink_pmu *starlink_pmu, + struct platform_device *pdev) +{ + int ret, irq; + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return -EINVAL; + + ret = devm_request_irq(&pdev->dev, irq, starlink_pmu_handle_irq, + 0, STARLINK_PMU_PDEV_NAME, starlink_pmu); + if (ret) + return dev_err_probe(&pdev->dev, ret, "Failed to request IRQ\n"); + + starlink_pmu->irq = irq; + + return 0; +} + +static int starlink_pmu_pm_notify(struct notifier_block *b, + unsigned long cmd, void *v) +{ + struct starlink_pmu *starlink_pmu = container_of(b, struct starlink_pmu, + starlink_pmu_pm_nb); + struct starlink_hw_events *hw_events = + this_cpu_ptr(starlink_pmu->hw_events); + int enabled = bitmap_weight(hw_events->used_mask, + STARLINK_PMU_MAX_COUNTERS); + struct perf_event *event; + int idx; + + if (!enabled) + return NOTIFY_OK; + + for (idx = 0; idx < STARLINK_PMU_MAX_COUNTERS; idx++) { + event = hw_events->events[idx]; + if (!event) + continue; + + switch (cmd) { + case CPU_PM_ENTER: + /* Stop and update the counter */ + starlink_pmu_stop(event, PERF_EF_UPDATE); + break; + case CPU_PM_EXIT: + case CPU_PM_ENTER_FAILED: + /* Restore and enable the counter */ + starlink_pmu_start(event, PERF_EF_RELOAD); + break; + default: + break; + } + } + + return NOTIFY_OK; +} + +static int starlink_pmu_pm_register(struct starlink_pmu *starlink_pmu) +{ + if (!IS_ENABLED(CONFIG_CPU_PM)) + return 0; + + starlink_pmu->starlink_pmu_pm_nb.notifier_call = starlink_pmu_pm_notify; + return cpu_pm_register_notifier(&starlink_pmu->starlink_pmu_pm_nb); +} + +static void starlink_pmu_pm_unregister(struct starlink_pmu *starlink_pmu) +{ + if (!IS_ENABLED(CONFIG_CPU_PM)) + return; + + cpu_pm_unregister_notifier(&starlink_pmu->starlink_pmu_pm_nb); +} + +static void starlink_pmu_destroy(struct starlink_pmu *starlink_pmu) +{ + starlink_pmu_pm_unregister(starlink_pmu); + cpuhp_state_remove_instance(starlink_pmu_cpuhp_state, + &starlink_pmu->node); +} + +static int starlink_pmu_probe(struct platform_device *pdev) +{ + struct starlink_pmu *starlink_pmu; + struct starlink_hw_events *hw_events; + struct resource *res; + int cpuid, i, ret; + + starlink_pmu = devm_kzalloc(&pdev->dev, sizeof(*starlink_pmu), GFP_KERNEL); + if (!starlink_pmu) + return -ENOMEM; + + starlink_pmu->pmu_base = + devm_platform_get_and_ioremap_resource(pdev, 0, &res); + if (IS_ERR(starlink_pmu->pmu_base)) + return PTR_ERR(starlink_pmu->pmu_base); + + starlink_pmu->hw_events = alloc_percpu_gfp(struct starlink_hw_events, + GFP_KERNEL); + if (!starlink_pmu->hw_events) { + dev_err(&pdev->dev, "Failed to allocate per-cpu PMU data\n"); + return -ENOMEM; + } + + for_each_possible_cpu(cpuid) { + hw_events = per_cpu_ptr(starlink_pmu->hw_events, cpuid); + for (i = 0; i < STARLINK_PMU_MAX_COUNTERS; i++) + hw_events->events[i] = NULL; + } + + ret = starlink_setup_irqs(starlink_pmu, pdev); + if (ret) + return ret; + + ret = cpuhp_state_add_instance(starlink_pmu_cpuhp_state, + &starlink_pmu->node); + if (ret) { + dev_err(&pdev->dev, "Failed to register hotplug\n"); + return ret; + } + + ret = starlink_pmu_pm_register(starlink_pmu); + if (ret) { + cpuhp_state_remove_instance(starlink_pmu_cpuhp_state, + &starlink_pmu->node); + return ret; + } + + starlink_pmu->pmu = (struct pmu) { + .task_ctx_nr = perf_invalid_context, + .event_init = starlink_pmu_event_init, + .add = starlink_pmu_add, + .del = starlink_pmu_del, + .start = starlink_pmu_start, + .stop = starlink_pmu_stop, + .read = starlink_pmu_update, + .attr_groups = starlink_pmu_attr_groups, + }; + + ret = perf_pmu_register(&starlink_pmu->pmu, STARLINK_PMU_PDEV_NAME, -1); + if (ret) + starlink_pmu_destroy(starlink_pmu); + + return ret; +} + +static const struct of_device_id starlink_pmu_of_match[] = { + { .compatible = "starfive,jh8100-starlink-pmu" }, + {} +}; +MODULE_DEVICE_TABLE(of, starlink_pmu_of_match); + +static struct platform_driver starlink_pmu_driver = { + .driver = { + .name = STARLINK_PMU_PDEV_NAME, + .of_match_table = starlink_pmu_of_match, + .suppress_bind_attrs = true, + }, + .probe = starlink_pmu_probe, +}; + +static int +starlink_pmu_online_cpu(unsigned int cpu, struct hlist_node *node) +{ + struct starlink_pmu *starlink_pmu = hlist_entry_safe(node, + struct starlink_pmu, + node); + + if (cpumask_empty(&starlink_pmu->cpumask)) + cpumask_set_cpu(cpu, &starlink_pmu->cpumask); + + WARN_ON(irq_set_affinity(starlink_pmu->irq, cpumask_of(cpu))); + + return 0; +} + +static int +starlink_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) +{ + struct starlink_pmu *starlink_pmu = hlist_entry_safe(node, + struct starlink_pmu, + node); + unsigned int target; + + if (!cpumask_test_and_clear_cpu(cpu, &starlink_pmu->cpumask)) + return 0; + + target = cpumask_any_but(cpu_online_mask, cpu); + if (target >= nr_cpu_ids) + return 0; + + perf_pmu_migrate_context(&starlink_pmu->pmu, cpu, target); + + cpumask_set_cpu(target, &starlink_pmu->cpumask); + WARN_ON(irq_set_affinity(starlink_pmu->irq, cpumask_of(target))); + + return 0; +} + +static int __init starlink_pmu_init(void) +{ + int ret; + + ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, + "soc/starfive/starlink_pmu:online", + starlink_pmu_online_cpu, + starlink_pmu_offline_cpu); + if (ret < 0) + return ret; + + starlink_pmu_cpuhp_state = ret; + + return platform_driver_register(&starlink_pmu_driver); +} + +device_initcall(starlink_pmu_init); diff --git a/drivers/perf/thunderx2_pmu.c b/drivers/perf/thunderx2_pmu.c index 1edb9c03704f..e16d10c763de 100644 --- a/drivers/perf/thunderx2_pmu.c +++ b/drivers/perf/thunderx2_pmu.c @@ -993,7 +993,7 @@ static int tx2_uncore_probe(struct platform_device *pdev) return 0; } -static int tx2_uncore_remove(struct platform_device *pdev) +static void tx2_uncore_remove(struct platform_device *pdev) { struct tx2_uncore_pmu *tx2_pmu, *temp; struct device *dev = &pdev->dev; @@ -1009,7 +1009,6 @@ static int tx2_uncore_remove(struct platform_device *pdev) } } } - return 0; } static struct platform_driver tx2_uncore_driver = { @@ -1019,7 +1018,7 @@ static struct platform_driver tx2_uncore_driver = { .suppress_bind_attrs = true, }, .probe = tx2_uncore_probe, - .remove = tx2_uncore_remove, + .remove_new = tx2_uncore_remove, }; static int __init tx2_uncore_driver_init(void) diff --git a/drivers/perf/xgene_pmu.c b/drivers/perf/xgene_pmu.c index 7ce344248dda..0d49343d704b 100644 --- a/drivers/perf/xgene_pmu.c +++ b/drivers/perf/xgene_pmu.c @@ -1937,7 +1937,7 @@ xgene_pmu_dev_cleanup(struct xgene_pmu *xgene_pmu, struct list_head *pmus) } } -static int xgene_pmu_remove(struct platform_device *pdev) +static void xgene_pmu_remove(struct platform_device *pdev) { struct xgene_pmu *xgene_pmu = dev_get_drvdata(&pdev->dev); @@ -1947,13 +1947,11 @@ static int xgene_pmu_remove(struct platform_device *pdev) xgene_pmu_dev_cleanup(xgene_pmu, &xgene_pmu->mcpmus); cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_APM_XGENE_ONLINE, &xgene_pmu->node); - - return 0; } static struct platform_driver xgene_pmu_driver = { .probe = xgene_pmu_probe, - .remove = xgene_pmu_remove, + .remove_new = xgene_pmu_remove, .driver = { .name = "xgene-pmu", .of_match_table = xgene_pmu_of_match, diff --git a/rust/Makefile b/rust/Makefile index 9d2a16cc91cb..fe045dbc701e 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -297,6 +297,7 @@ bindgen_skip_c_flags := -mno-fp-ret-in-387 -mpreferred-stack-boundary=% \ # Derived from `scripts/Makefile.clang`. BINDGEN_TARGET_x86 := x86_64-linux-gnu +BINDGEN_TARGET_arm64 := aarch64-linux-gnu BINDGEN_TARGET := $(BINDGEN_TARGET_$(SRCARCH)) # All warnings are inhibited since GCC builds are very experimental, @@ -434,8 +435,11 @@ $(obj)/core.o: private skip_clippy = 1 $(obj)/core.o: private skip_flags = -Dunreachable_pub $(obj)/core.o: private rustc_objcopy = $(foreach sym,$(redirect-intrinsics),--redefine-sym $(sym)=__rust$(sym)) $(obj)/core.o: private rustc_target_flags = $(core-cfgs) -$(obj)/core.o: $(RUST_LIB_SRC)/core/src/lib.rs scripts/target.json FORCE +$(obj)/core.o: $(RUST_LIB_SRC)/core/src/lib.rs FORCE $(call if_changed_dep,rustc_library) +ifneq ($(or $(CONFIG_X86_64),$(CONFIG_LOONGARCH)),) +$(obj)/core.o: scripts/target.json +endif $(obj)/compiler_builtins.o: private rustc_objcopy = -w -W '__*' $(obj)/compiler_builtins.o: $(src)/compiler_builtins.rs $(obj)/core.o FORCE diff --git a/scripts/Makefile b/scripts/Makefile index 576cf64be667..6673cbb6194f 100644 --- a/scripts/Makefile +++ b/scripts/Makefile @@ -11,12 +11,14 @@ hostprogs-always-$(CONFIG_MODULE_SIG_FORMAT) += sign-file hostprogs-always-$(CONFIG_SYSTEM_EXTRA_CERTIFICATE) += insert-sys-cert hostprogs-always-$(CONFIG_RUST_KERNEL_DOCTESTS) += rustdoc_test_builder hostprogs-always-$(CONFIG_RUST_KERNEL_DOCTESTS) += rustdoc_test_gen -always-$(CONFIG_RUST) += target.json +ifneq ($(or $(CONFIG_X86_64),$(CONFIG_LOONGARCH)),) +always-$(CONFIG_RUST) += target.json filechk_rust_target = $< < include/config/auto.conf $(obj)/target.json: scripts/generate_rust_target include/config/auto.conf FORCE $(call filechk,rust_target) +endif hostprogs += generate_rust_target generate_rust_target-rust := y diff --git a/scripts/generate_rust_target.rs b/scripts/generate_rust_target.rs index 0da52b548ba5..9e117957ea7b 100644 --- a/scripts/generate_rust_target.rs +++ b/scripts/generate_rust_target.rs @@ -148,7 +148,9 @@ fn main() { let mut ts = TargetSpec::new(); // `llvm-target`s are taken from `scripts/Makefile.clang`. - if cfg.has("X86_64") { + if cfg.has("ARM64") { + panic!("arm64 uses the builtin rustc aarch64-unknown-none target"); + } else if cfg.has("X86_64") { ts.push("arch", "x86_64"); ts.push( "data-layout", diff --git a/tools/testing/selftests/arm64/fp/.gitignore b/tools/testing/selftests/arm64/fp/.gitignore index ebc86757bdd8..00e52c966281 100644 --- a/tools/testing/selftests/arm64/fp/.gitignore +++ b/tools/testing/selftests/arm64/fp/.gitignore @@ -1,4 +1,5 @@ fp-pidbench +fp-ptrace fp-stress fpsimd-test rdvl-sme diff --git a/tools/testing/selftests/arm64/fp/Makefile b/tools/testing/selftests/arm64/fp/Makefile index b413b0af07f9..55d4f00d9e8e 100644 --- a/tools/testing/selftests/arm64/fp/Makefile +++ b/tools/testing/selftests/arm64/fp/Makefile @@ -5,7 +5,9 @@ top_srcdir = $(realpath ../../../../../) CFLAGS += $(KHDR_INCLUDES) -TEST_GEN_PROGS := fp-stress \ +TEST_GEN_PROGS := \ + fp-ptrace \ + fp-stress \ sve-ptrace sve-probe-vls \ vec-syscfg \ za-fork za-ptrace @@ -24,6 +26,7 @@ EXTRA_CLEAN += $(OUTPUT)/asm-utils.o $(OUTPUT)/rdvl.o $(OUTPUT)/za-fork-asm.o # Build with nolibc to avoid effects due to libc's clone() support $(OUTPUT)/fp-pidbench: fp-pidbench.S $(OUTPUT)/asm-utils.o $(CC) -nostdlib $^ -o $@ +$(OUTPUT)/fp-ptrace: fp-ptrace.c fp-ptrace-asm.S $(OUTPUT)/fpsimd-test: fpsimd-test.S $(OUTPUT)/asm-utils.o $(CC) -nostdlib $^ -o $@ $(OUTPUT)/rdvl-sve: rdvl-sve.c $(OUTPUT)/rdvl.o diff --git a/tools/testing/selftests/arm64/fp/fp-ptrace-asm.S b/tools/testing/selftests/arm64/fp/fp-ptrace-asm.S new file mode 100644 index 000000000000..7ad59d92d02b --- /dev/null +++ b/tools/testing/selftests/arm64/fp/fp-ptrace-asm.S @@ -0,0 +1,279 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2021-3 ARM Limited. +// +// Assembly portion of the FP ptrace test + +// +// Load values from memory into registers, break on a breakpoint, then +// break on a further breakpoint +// + +#include "fp-ptrace.h" +#include "sme-inst.h" + +.arch_extension sve + +// Load and save register values with pauses for ptrace +// +// x0 - SVE in use +// x1 - SME in use +// x2 - SME2 in use +// x3 - FA64 supported + +.globl load_and_save +load_and_save: + stp x11, x12, [sp, #-0x10]! + + // This should be redundant in the SVE case + ldr x7, =v_in + ldp q0, q1, [x7] + ldp q2, q3, [x7, #16 * 2] + ldp q4, q5, [x7, #16 * 4] + ldp q6, q7, [x7, #16 * 6] + ldp q8, q9, [x7, #16 * 8] + ldp q10, q11, [x7, #16 * 10] + ldp q12, q13, [x7, #16 * 12] + ldp q14, q15, [x7, #16 * 14] + ldp q16, q17, [x7, #16 * 16] + ldp q18, q19, [x7, #16 * 18] + ldp q20, q21, [x7, #16 * 20] + ldp q22, q23, [x7, #16 * 22] + ldp q24, q25, [x7, #16 * 24] + ldp q26, q27, [x7, #16 * 26] + ldp q28, q29, [x7, #16 * 28] + ldp q30, q31, [x7, #16 * 30] + + // SME? + cbz x1, check_sve_in + + adrp x7, svcr_in + ldr x7, [x7, :lo12:svcr_in] + // SVCR is 0 by default, avoid triggering SME if not in use + cbz x7, check_sve_in + msr S3_3_C4_C2_2, x7 + + // ZA? + tbz x7, #SVCR_ZA_SHIFT, check_sm_in + rdsvl 11, 1 + mov w12, #0 + ldr x6, =za_in +1: _ldr_za 12, 6 + add x6, x6, x11 + add x12, x12, #1 + cmp x11, x12 + bne 1b + + // ZT? + cbz x2, check_sm_in + adrp x6, zt_in + add x6, x6, :lo12:zt_in + _ldr_zt 6 + + // In streaming mode? +check_sm_in: + tbz x7, #SVCR_SM_SHIFT, check_sve_in + mov x4, x3 // Load FFR if we have FA64 + b load_sve + + // SVE? +check_sve_in: + cbz x0, wait_for_writes + mov x4, #1 + +load_sve: + ldr x7, =z_in + ldr z0, [x7, #0, MUL VL] + ldr z1, [x7, #1, MUL VL] + ldr z2, [x7, #2, MUL VL] + ldr z3, [x7, #3, MUL VL] + ldr z4, [x7, #4, MUL VL] + ldr z5, [x7, #5, MUL VL] + ldr z6, [x7, #6, MUL VL] + ldr z7, [x7, #7, MUL VL] + ldr z8, [x7, #8, MUL VL] + ldr z9, [x7, #9, MUL VL] + ldr z10, [x7, #10, MUL VL] + ldr z11, [x7, #11, MUL VL] + ldr z12, [x7, #12, MUL VL] + ldr z13, [x7, #13, MUL VL] + ldr z14, [x7, #14, MUL VL] + ldr z15, [x7, #15, MUL VL] + ldr z16, [x7, #16, MUL VL] + ldr z17, [x7, #17, MUL VL] + ldr z18, [x7, #18, MUL VL] + ldr z19, [x7, #19, MUL VL] + ldr z20, [x7, #20, MUL VL] + ldr z21, [x7, #21, MUL VL] + ldr z22, [x7, #22, MUL VL] + ldr z23, [x7, #23, MUL VL] + ldr z24, [x7, #24, MUL VL] + ldr z25, [x7, #25, MUL VL] + ldr z26, [x7, #26, MUL VL] + ldr z27, [x7, #27, MUL VL] + ldr z28, [x7, #28, MUL VL] + ldr z29, [x7, #29, MUL VL] + ldr z30, [x7, #30, MUL VL] + ldr z31, [x7, #31, MUL VL] + + // FFR is not present in base SME + cbz x4, 1f + ldr x7, =ffr_in + ldr p0, [x7] + ldr x7, [x7, #0] + cbz x7, 1f + wrffr p0.b +1: + + ldr x7, =p_in + ldr p0, [x7, #0, MUL VL] + ldr p1, [x7, #1, MUL VL] + ldr p2, [x7, #2, MUL VL] + ldr p3, [x7, #3, MUL VL] + ldr p4, [x7, #4, MUL VL] + ldr p5, [x7, #5, MUL VL] + ldr p6, [x7, #6, MUL VL] + ldr p7, [x7, #7, MUL VL] + ldr p8, [x7, #8, MUL VL] + ldr p9, [x7, #9, MUL VL] + ldr p10, [x7, #10, MUL VL] + ldr p11, [x7, #11, MUL VL] + ldr p12, [x7, #12, MUL VL] + ldr p13, [x7, #13, MUL VL] + ldr p14, [x7, #14, MUL VL] + ldr p15, [x7, #15, MUL VL] + +wait_for_writes: + // Wait for the parent + brk #0 + + // Save values + ldr x7, =v_out + stp q0, q1, [x7] + stp q2, q3, [x7, #16 * 2] + stp q4, q5, [x7, #16 * 4] + stp q6, q7, [x7, #16 * 6] + stp q8, q9, [x7, #16 * 8] + stp q10, q11, [x7, #16 * 10] + stp q12, q13, [x7, #16 * 12] + stp q14, q15, [x7, #16 * 14] + stp q16, q17, [x7, #16 * 16] + stp q18, q19, [x7, #16 * 18] + stp q20, q21, [x7, #16 * 20] + stp q22, q23, [x7, #16 * 22] + stp q24, q25, [x7, #16 * 24] + stp q26, q27, [x7, #16 * 26] + stp q28, q29, [x7, #16 * 28] + stp q30, q31, [x7, #16 * 30] + + // SME? + cbz x1, check_sve_out + + rdsvl 11, 1 + adrp x6, sme_vl_out + str x11, [x6, :lo12:sme_vl_out] + + mrs x7, S3_3_C4_C2_2 + adrp x6, svcr_out + str x7, [x6, :lo12:svcr_out] + + // ZA? + tbz x7, #SVCR_ZA_SHIFT, check_sm_out + mov w12, #0 + ldr x6, =za_out +1: _str_za 12, 6 + add x6, x6, x11 + add x12, x12, #1 + cmp x11, x12 + bne 1b + + // ZT? + cbz x2, check_sm_out + adrp x6, zt_out + add x6, x6, :lo12:zt_out + _str_zt 6 + + // In streaming mode? +check_sm_out: + tbz x7, #SVCR_SM_SHIFT, check_sve_out + mov x4, x3 // FFR? + b read_sve + + // SVE? +check_sve_out: + cbz x0, wait_for_reads + mov x4, #1 + + rdvl x7, #1 + adrp x6, sve_vl_out + str x7, [x6, :lo12:sve_vl_out] + +read_sve: + ldr x7, =z_out + str z0, [x7, #0, MUL VL] + str z1, [x7, #1, MUL VL] + str z2, [x7, #2, MUL VL] + str z3, [x7, #3, MUL VL] + str z4, [x7, #4, MUL VL] + str z5, [x7, #5, MUL VL] + str z6, [x7, #6, MUL VL] + str z7, [x7, #7, MUL VL] + str z8, [x7, #8, MUL VL] + str z9, [x7, #9, MUL VL] + str z10, [x7, #10, MUL VL] + str z11, [x7, #11, MUL VL] + str z12, [x7, #12, MUL VL] + str z13, [x7, #13, MUL VL] + str z14, [x7, #14, MUL VL] + str z15, [x7, #15, MUL VL] + str z16, [x7, #16, MUL VL] + str z17, [x7, #17, MUL VL] + str z18, [x7, #18, MUL VL] + str z19, [x7, #19, MUL VL] + str z20, [x7, #20, MUL VL] + str z21, [x7, #21, MUL VL] + str z22, [x7, #22, MUL VL] + str z23, [x7, #23, MUL VL] + str z24, [x7, #24, MUL VL] + str z25, [x7, #25, MUL VL] + str z26, [x7, #26, MUL VL] + str z27, [x7, #27, MUL VL] + str z28, [x7, #28, MUL VL] + str z29, [x7, #29, MUL VL] + str z30, [x7, #30, MUL VL] + str z31, [x7, #31, MUL VL] + + ldr x7, =p_out + str p0, [x7, #0, MUL VL] + str p1, [x7, #1, MUL VL] + str p2, [x7, #2, MUL VL] + str p3, [x7, #3, MUL VL] + str p4, [x7, #4, MUL VL] + str p5, [x7, #5, MUL VL] + str p6, [x7, #6, MUL VL] + str p7, [x7, #7, MUL VL] + str p8, [x7, #8, MUL VL] + str p9, [x7, #9, MUL VL] + str p10, [x7, #10, MUL VL] + str p11, [x7, #11, MUL VL] + str p12, [x7, #12, MUL VL] + str p13, [x7, #13, MUL VL] + str p14, [x7, #14, MUL VL] + str p15, [x7, #15, MUL VL] + + // Only save FFR if it exists + cbz x4, wait_for_reads + ldr x7, =ffr_out + rdffr p0.b + str p0, [x7] + +wait_for_reads: + // Wait for the parent + brk #0 + + // Ensure we don't leave ourselves in streaming mode + cbz x1, out + msr S3_3_C4_C2_2, xzr + +out: + ldp x11, x12, [sp, #-0x10] + ret diff --git a/tools/testing/selftests/arm64/fp/fp-ptrace.c b/tools/testing/selftests/arm64/fp/fp-ptrace.c new file mode 100644 index 000000000000..c7ceafe5f471 --- /dev/null +++ b/tools/testing/selftests/arm64/fp/fp-ptrace.c @@ -0,0 +1,1503 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2023 ARM Limited. + * Original author: Mark Brown <[email protected]> + */ + +#define _GNU_SOURCE + +#include <errno.h> +#include <stdbool.h> +#include <stddef.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +#include <sys/auxv.h> +#include <sys/prctl.h> +#include <sys/ptrace.h> +#include <sys/types.h> +#include <sys/uio.h> +#include <sys/wait.h> + +#include <linux/kernel.h> + +#include <asm/sigcontext.h> +#include <asm/sve_context.h> +#include <asm/ptrace.h> + +#include "../../kselftest.h" + +#include "fp-ptrace.h" + +/* <linux/elf.h> and <sys/auxv.h> don't like each other, so: */ +#ifndef NT_ARM_SVE +#define NT_ARM_SVE 0x405 +#endif + +#ifndef NT_ARM_SSVE +#define NT_ARM_SSVE 0x40b +#endif + +#ifndef NT_ARM_ZA +#define NT_ARM_ZA 0x40c +#endif + +#ifndef NT_ARM_ZT +#define NT_ARM_ZT 0x40d +#endif + +#define ARCH_VQ_MAX 256 + +/* VL 128..2048 in powers of 2 */ +#define MAX_NUM_VLS 5 + +#define NUM_FPR 32 +__uint128_t v_in[NUM_FPR]; +__uint128_t v_expected[NUM_FPR]; +__uint128_t v_out[NUM_FPR]; + +char z_in[__SVE_ZREGS_SIZE(ARCH_VQ_MAX)]; +char z_expected[__SVE_ZREGS_SIZE(ARCH_VQ_MAX)]; +char z_out[__SVE_ZREGS_SIZE(ARCH_VQ_MAX)]; + +char p_in[__SVE_PREGS_SIZE(ARCH_VQ_MAX)]; +char p_expected[__SVE_PREGS_SIZE(ARCH_VQ_MAX)]; +char p_out[__SVE_PREGS_SIZE(ARCH_VQ_MAX)]; + +char ffr_in[__SVE_PREG_SIZE(ARCH_VQ_MAX)]; +char ffr_expected[__SVE_PREG_SIZE(ARCH_VQ_MAX)]; +char ffr_out[__SVE_PREG_SIZE(ARCH_VQ_MAX)]; + +char za_in[ZA_SIG_REGS_SIZE(ARCH_VQ_MAX)]; +char za_expected[ZA_SIG_REGS_SIZE(ARCH_VQ_MAX)]; +char za_out[ZA_SIG_REGS_SIZE(ARCH_VQ_MAX)]; + +char zt_in[ZT_SIG_REG_BYTES]; +char zt_expected[ZT_SIG_REG_BYTES]; +char zt_out[ZT_SIG_REG_BYTES]; + +uint64_t sve_vl_out; +uint64_t sme_vl_out; +uint64_t svcr_in, svcr_expected, svcr_out; + +void load_and_save(int sve, int sme, int sme2, int fa64); + +static bool got_alarm; + +static void handle_alarm(int sig, siginfo_t *info, void *context) +{ + got_alarm = true; +} + +#ifdef CONFIG_CPU_BIG_ENDIAN +static __uint128_t arm64_cpu_to_le128(__uint128_t x) +{ + u64 a = swab64(x); + u64 b = swab64(x >> 64); + + return ((__uint128_t)a << 64) | b; +} +#else +static __uint128_t arm64_cpu_to_le128(__uint128_t x) +{ + return x; +} +#endif + +#define arm64_le128_to_cpu(x) arm64_cpu_to_le128(x) + +static bool sve_supported(void) +{ + return getauxval(AT_HWCAP) & HWCAP_SVE; +} + +static bool sme_supported(void) +{ + return getauxval(AT_HWCAP2) & HWCAP2_SME; +} + +static bool sme2_supported(void) +{ + return getauxval(AT_HWCAP2) & HWCAP2_SME2; +} + +static bool fa64_supported(void) +{ + return getauxval(AT_HWCAP2) & HWCAP2_SME_FA64; +} + +static bool compare_buffer(const char *name, void *out, + void *expected, size_t size) +{ + void *tmp; + + if (memcmp(out, expected, size) == 0) + return true; + + ksft_print_msg("Mismatch in %s\n", name); + + /* Did we just get zeros back? */ + tmp = malloc(size); + if (!tmp) { + ksft_print_msg("OOM allocating %lu bytes for %s\n", + size, name); + ksft_exit_fail(); + } + memset(tmp, 0, size); + + if (memcmp(out, tmp, size) == 0) + ksft_print_msg("%s is zero\n", name); + + free(tmp); + + return false; +} + +struct test_config { + int sve_vl_in; + int sve_vl_expected; + int sme_vl_in; + int sme_vl_expected; + int svcr_in; + int svcr_expected; +}; + +struct test_definition { + const char *name; + bool sve_vl_change; + bool (*supported)(struct test_config *config); + void (*set_expected_values)(struct test_config *config); + void (*modify_values)(pid_t child, struct test_config *test_config); +}; + +static int vl_in(struct test_config *config) +{ + int vl; + + if (config->svcr_in & SVCR_SM) + vl = config->sme_vl_in; + else + vl = config->sve_vl_in; + + return vl; +} + +static int vl_expected(struct test_config *config) +{ + int vl; + + if (config->svcr_expected & SVCR_SM) + vl = config->sme_vl_expected; + else + vl = config->sve_vl_expected; + + return vl; +} + +static void run_child(struct test_config *config) +{ + int ret; + + /* Let the parent attach to us */ + ret = ptrace(PTRACE_TRACEME, 0, 0, 0); + if (ret < 0) + ksft_exit_fail_msg("PTRACE_TRACEME failed: %s (%d)\n", + strerror(errno), errno); + + /* VL setup */ + if (sve_supported()) { + ret = prctl(PR_SVE_SET_VL, config->sve_vl_in); + if (ret != config->sve_vl_in) { + ksft_print_msg("Failed to set SVE VL %d: %d\n", + config->sve_vl_in, ret); + } + } + + if (sme_supported()) { + ret = prctl(PR_SME_SET_VL, config->sme_vl_in); + if (ret != config->sme_vl_in) { + ksft_print_msg("Failed to set SME VL %d: %d\n", + config->sme_vl_in, ret); + } + } + + /* Load values and wait for the parent */ + load_and_save(sve_supported(), sme_supported(), + sme2_supported(), fa64_supported()); + + exit(0); +} + +static void read_one_child_regs(pid_t child, char *name, + struct iovec *iov_parent, + struct iovec *iov_child) +{ + int len = iov_parent->iov_len; + int ret; + + ret = process_vm_readv(child, iov_parent, 1, iov_child, 1, 0); + if (ret == -1) + ksft_print_msg("%s read failed: %s (%d)\n", + name, strerror(errno), errno); + else if (ret != len) + ksft_print_msg("Short read of %s: %d\n", name, ret); +} + +static void read_child_regs(pid_t child) +{ + struct iovec iov_parent, iov_child; + + /* + * Since the child fork()ed from us the buffer addresses are + * the same in parent and child. + */ + iov_parent.iov_base = &v_out; + iov_parent.iov_len = sizeof(v_out); + iov_child.iov_base = &v_out; + iov_child.iov_len = sizeof(v_out); + read_one_child_regs(child, "FPSIMD", &iov_parent, &iov_child); + + if (sve_supported() || sme_supported()) { + iov_parent.iov_base = &sve_vl_out; + iov_parent.iov_len = sizeof(sve_vl_out); + iov_child.iov_base = &sve_vl_out; + iov_child.iov_len = sizeof(sve_vl_out); + read_one_child_regs(child, "SVE VL", &iov_parent, &iov_child); + + iov_parent.iov_base = &z_out; + iov_parent.iov_len = sizeof(z_out); + iov_child.iov_base = &z_out; + iov_child.iov_len = sizeof(z_out); + read_one_child_regs(child, "Z", &iov_parent, &iov_child); + + iov_parent.iov_base = &p_out; + iov_parent.iov_len = sizeof(p_out); + iov_child.iov_base = &p_out; + iov_child.iov_len = sizeof(p_out); + read_one_child_regs(child, "P", &iov_parent, &iov_child); + + iov_parent.iov_base = &ffr_out; + iov_parent.iov_len = sizeof(ffr_out); + iov_child.iov_base = &ffr_out; + iov_child.iov_len = sizeof(ffr_out); + read_one_child_regs(child, "FFR", &iov_parent, &iov_child); + } + + if (sme_supported()) { + iov_parent.iov_base = &sme_vl_out; + iov_parent.iov_len = sizeof(sme_vl_out); + iov_child.iov_base = &sme_vl_out; + iov_child.iov_len = sizeof(sme_vl_out); + read_one_child_regs(child, "SME VL", &iov_parent, &iov_child); + + iov_parent.iov_base = &svcr_out; + iov_parent.iov_len = sizeof(svcr_out); + iov_child.iov_base = &svcr_out; + iov_child.iov_len = sizeof(svcr_out); + read_one_child_regs(child, "SVCR", &iov_parent, &iov_child); + + iov_parent.iov_base = &za_out; + iov_parent.iov_len = sizeof(za_out); + iov_child.iov_base = &za_out; + iov_child.iov_len = sizeof(za_out); + read_one_child_regs(child, "ZA", &iov_parent, &iov_child); + } + + if (sme2_supported()) { + iov_parent.iov_base = &zt_out; + iov_parent.iov_len = sizeof(zt_out); + iov_child.iov_base = &zt_out; + iov_child.iov_len = sizeof(zt_out); + read_one_child_regs(child, "ZT", &iov_parent, &iov_child); + } +} + +static bool continue_breakpoint(pid_t child, + enum __ptrace_request restart_type) +{ + struct user_pt_regs pt_regs; + struct iovec iov; + int ret; + + /* Get PC */ + iov.iov_base = &pt_regs; + iov.iov_len = sizeof(pt_regs); + ret = ptrace(PTRACE_GETREGSET, child, NT_PRSTATUS, &iov); + if (ret < 0) { + ksft_print_msg("Failed to get PC: %s (%d)\n", + strerror(errno), errno); + return false; + } + + /* Skip over the BRK */ + pt_regs.pc += 4; + ret = ptrace(PTRACE_SETREGSET, child, NT_PRSTATUS, &iov); + if (ret < 0) { + ksft_print_msg("Failed to skip BRK: %s (%d)\n", + strerror(errno), errno); + return false; + } + + /* Restart */ + ret = ptrace(restart_type, child, 0, 0); + if (ret < 0) { + ksft_print_msg("Failed to restart child: %s (%d)\n", + strerror(errno), errno); + return false; + } + + return true; +} + +static bool check_ptrace_values_sve(pid_t child, struct test_config *config) +{ + struct user_sve_header *sve; + struct user_fpsimd_state *fpsimd; + struct iovec iov; + int ret, vq; + bool pass = true; + + if (!sve_supported()) + return true; + + vq = __sve_vq_from_vl(config->sve_vl_in); + + iov.iov_len = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE); + iov.iov_base = malloc(iov.iov_len); + if (!iov.iov_base) { + ksft_print_msg("OOM allocating %lu byte SVE buffer\n", + iov.iov_len); + return false; + } + + ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_SVE, &iov); + if (ret != 0) { + ksft_print_msg("Failed to read initial SVE: %s (%d)\n", + strerror(errno), errno); + pass = false; + goto out; + } + + sve = iov.iov_base; + + if (sve->vl != config->sve_vl_in) { + ksft_print_msg("Mismatch in initial SVE VL: %d != %d\n", + sve->vl, config->sve_vl_in); + pass = false; + } + + /* If we are in streaming mode we should just read FPSIMD */ + if ((config->svcr_in & SVCR_SM) && (sve->flags & SVE_PT_REGS_SVE)) { + ksft_print_msg("NT_ARM_SVE reports SVE with PSTATE.SM\n"); + pass = false; + } + + if (sve->size != SVE_PT_SIZE(vq, sve->flags)) { + ksft_print_msg("Mismatch in SVE header size: %d != %lu\n", + sve->size, SVE_PT_SIZE(vq, sve->flags)); + pass = false; + } + + /* The registers might be in completely different formats! */ + if (sve->flags & SVE_PT_REGS_SVE) { + if (!compare_buffer("initial SVE Z", + iov.iov_base + SVE_PT_SVE_ZREG_OFFSET(vq, 0), + z_in, SVE_PT_SVE_ZREGS_SIZE(vq))) + pass = false; + + if (!compare_buffer("initial SVE P", + iov.iov_base + SVE_PT_SVE_PREG_OFFSET(vq, 0), + p_in, SVE_PT_SVE_PREGS_SIZE(vq))) + pass = false; + + if (!compare_buffer("initial SVE FFR", + iov.iov_base + SVE_PT_SVE_FFR_OFFSET(vq), + ffr_in, SVE_PT_SVE_PREG_SIZE(vq))) + pass = false; + } else { + fpsimd = iov.iov_base + SVE_PT_FPSIMD_OFFSET; + if (!compare_buffer("initial V via SVE", &fpsimd->vregs[0], + v_in, sizeof(v_in))) + pass = false; + } + +out: + free(iov.iov_base); + return pass; +} + +static bool check_ptrace_values_ssve(pid_t child, struct test_config *config) +{ + struct user_sve_header *sve; + struct user_fpsimd_state *fpsimd; + struct iovec iov; + int ret, vq; + bool pass = true; + + if (!sme_supported()) + return true; + + vq = __sve_vq_from_vl(config->sme_vl_in); + + iov.iov_len = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE); + iov.iov_base = malloc(iov.iov_len); + if (!iov.iov_base) { + ksft_print_msg("OOM allocating %lu byte SSVE buffer\n", + iov.iov_len); + return false; + } + + ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_SSVE, &iov); + if (ret != 0) { + ksft_print_msg("Failed to read initial SSVE: %s (%d)\n", + strerror(errno), errno); + pass = false; + goto out; + } + + sve = iov.iov_base; + + if (sve->vl != config->sme_vl_in) { + ksft_print_msg("Mismatch in initial SSVE VL: %d != %d\n", + sve->vl, config->sme_vl_in); + pass = false; + } + + if ((config->svcr_in & SVCR_SM) && !(sve->flags & SVE_PT_REGS_SVE)) { + ksft_print_msg("NT_ARM_SSVE reports FPSIMD with PSTATE.SM\n"); + pass = false; + } + + if (sve->size != SVE_PT_SIZE(vq, sve->flags)) { + ksft_print_msg("Mismatch in SSVE header size: %d != %lu\n", + sve->size, SVE_PT_SIZE(vq, sve->flags)); + pass = false; + } + + /* The registers might be in completely different formats! */ + if (sve->flags & SVE_PT_REGS_SVE) { + if (!compare_buffer("initial SSVE Z", + iov.iov_base + SVE_PT_SVE_ZREG_OFFSET(vq, 0), + z_in, SVE_PT_SVE_ZREGS_SIZE(vq))) + pass = false; + + if (!compare_buffer("initial SSVE P", + iov.iov_base + SVE_PT_SVE_PREG_OFFSET(vq, 0), + p_in, SVE_PT_SVE_PREGS_SIZE(vq))) + pass = false; + + if (!compare_buffer("initial SSVE FFR", + iov.iov_base + SVE_PT_SVE_FFR_OFFSET(vq), + ffr_in, SVE_PT_SVE_PREG_SIZE(vq))) + pass = false; + } else { + fpsimd = iov.iov_base + SVE_PT_FPSIMD_OFFSET; + if (!compare_buffer("initial V via SSVE", + &fpsimd->vregs[0], v_in, sizeof(v_in))) + pass = false; + } + +out: + free(iov.iov_base); + return pass; +} + +static bool check_ptrace_values_za(pid_t child, struct test_config *config) +{ + struct user_za_header *za; + struct iovec iov; + int ret, vq; + bool pass = true; + + if (!sme_supported()) + return true; + + vq = __sve_vq_from_vl(config->sme_vl_in); + + iov.iov_len = ZA_SIG_CONTEXT_SIZE(vq); + iov.iov_base = malloc(iov.iov_len); + if (!iov.iov_base) { + ksft_print_msg("OOM allocating %lu byte ZA buffer\n", + iov.iov_len); + return false; + } + + ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_ZA, &iov); + if (ret != 0) { + ksft_print_msg("Failed to read initial ZA: %s (%d)\n", + strerror(errno), errno); + pass = false; + goto out; + } + + za = iov.iov_base; + + if (za->vl != config->sme_vl_in) { + ksft_print_msg("Mismatch in initial SME VL: %d != %d\n", + za->vl, config->sme_vl_in); + pass = false; + } + + /* If PSTATE.ZA is not set we should just read the header */ + if (config->svcr_in & SVCR_ZA) { + if (za->size != ZA_PT_SIZE(vq)) { + ksft_print_msg("Unexpected ZA ptrace read size: %d != %lu\n", + za->size, ZA_PT_SIZE(vq)); + pass = false; + } + + if (!compare_buffer("initial ZA", + iov.iov_base + ZA_PT_ZA_OFFSET, + za_in, ZA_PT_ZA_SIZE(vq))) + pass = false; + } else { + if (za->size != sizeof(*za)) { + ksft_print_msg("Unexpected ZA ptrace read size: %d != %lu\n", + za->size, sizeof(*za)); + pass = false; + } + } + +out: + free(iov.iov_base); + return pass; +} + +static bool check_ptrace_values_zt(pid_t child, struct test_config *config) +{ + uint8_t buf[512]; + struct iovec iov; + int ret; + + if (!sme2_supported()) + return true; + + iov.iov_base = &buf; + iov.iov_len = ZT_SIG_REG_BYTES; + ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_ZT, &iov); + if (ret != 0) { + ksft_print_msg("Failed to read initial ZT: %s (%d)\n", + strerror(errno), errno); + return false; + } + + return compare_buffer("initial ZT", buf, zt_in, ZT_SIG_REG_BYTES); +} + + +static bool check_ptrace_values(pid_t child, struct test_config *config) +{ + bool pass = true; + struct user_fpsimd_state fpsimd; + struct iovec iov; + int ret; + + iov.iov_base = &fpsimd; + iov.iov_len = sizeof(fpsimd); + ret = ptrace(PTRACE_GETREGSET, child, NT_PRFPREG, &iov); + if (ret == 0) { + if (!compare_buffer("initial V", &fpsimd.vregs, v_in, + sizeof(v_in))) { + pass = false; + } + } else { + ksft_print_msg("Failed to read initial V: %s (%d)\n", + strerror(errno), errno); + pass = false; + } + + if (!check_ptrace_values_sve(child, config)) + pass = false; + + if (!check_ptrace_values_ssve(child, config)) + pass = false; + + if (!check_ptrace_values_za(child, config)) + pass = false; + + if (!check_ptrace_values_zt(child, config)) + pass = false; + + return pass; +} + +static bool run_parent(pid_t child, struct test_definition *test, + struct test_config *config) +{ + int wait_status, ret; + pid_t pid; + bool pass; + + /* Initial attach */ + while (1) { + pid = waitpid(child, &wait_status, 0); + if (pid < 0) { + if (errno == EINTR) + continue; + ksft_exit_fail_msg("waitpid() failed: %s (%d)\n", + strerror(errno), errno); + } + + if (pid == child) + break; + } + + if (WIFEXITED(wait_status)) { + ksft_print_msg("Child exited loading values with status %d\n", + WEXITSTATUS(wait_status)); + pass = false; + goto out; + } + + if (WIFSIGNALED(wait_status)) { + ksft_print_msg("Child died from signal %d loading values\n", + WTERMSIG(wait_status)); + pass = false; + goto out; + } + + /* Read initial values via ptrace */ + pass = check_ptrace_values(child, config); + + /* Do whatever writes we want to do */ + if (test->modify_values) + test->modify_values(child, config); + + if (!continue_breakpoint(child, PTRACE_CONT)) + goto cleanup; + + while (1) { + pid = waitpid(child, &wait_status, 0); + if (pid < 0) { + if (errno == EINTR) + continue; + ksft_exit_fail_msg("waitpid() failed: %s (%d)\n", + strerror(errno), errno); + } + + if (pid == child) + break; + } + + if (WIFEXITED(wait_status)) { + ksft_print_msg("Child exited saving values with status %d\n", + WEXITSTATUS(wait_status)); + pass = false; + goto out; + } + + if (WIFSIGNALED(wait_status)) { + ksft_print_msg("Child died from signal %d saving values\n", + WTERMSIG(wait_status)); + pass = false; + goto out; + } + + /* See what happened as a result */ + read_child_regs(child); + + if (!continue_breakpoint(child, PTRACE_DETACH)) + goto cleanup; + + /* The child should exit cleanly */ + got_alarm = false; + alarm(1); + while (1) { + if (got_alarm) { + ksft_print_msg("Wait for child timed out\n"); + goto cleanup; + } + + pid = waitpid(child, &wait_status, 0); + if (pid < 0) { + if (errno == EINTR) + continue; + ksft_exit_fail_msg("waitpid() failed: %s (%d)\n", + strerror(errno), errno); + } + + if (pid == child) + break; + } + alarm(0); + + if (got_alarm) { + ksft_print_msg("Timed out waiting for child\n"); + pass = false; + goto cleanup; + } + + if (pid == child && WIFSIGNALED(wait_status)) { + ksft_print_msg("Child died from signal %d cleaning up\n", + WTERMSIG(wait_status)); + pass = false; + goto out; + } + + if (pid == child && WIFEXITED(wait_status)) { + if (WEXITSTATUS(wait_status) != 0) { + ksft_print_msg("Child exited with error %d\n", + WEXITSTATUS(wait_status)); + pass = false; + } + } else { + ksft_print_msg("Child did not exit cleanly\n"); + pass = false; + goto cleanup; + } + + goto out; + +cleanup: + ret = kill(child, SIGKILL); + if (ret != 0) { + ksft_print_msg("kill() failed: %s (%d)\n", + strerror(errno), errno); + return false; + } + + while (1) { + pid = waitpid(child, &wait_status, 0); + if (pid < 0) { + if (errno == EINTR) + continue; + ksft_exit_fail_msg("waitpid() failed: %s (%d)\n", + strerror(errno), errno); + } + + if (pid == child) + break; + } + +out: + return pass; +} + +static void fill_random(void *buf, size_t size) +{ + int i; + uint32_t *lbuf = buf; + + /* random() returns a 32 bit number regardless of the size of long */ + for (i = 0; i < size / sizeof(uint32_t); i++) + lbuf[i] = random(); +} + +static void fill_random_ffr(void *buf, size_t vq) +{ + uint8_t *lbuf = buf; + int bits, i; + + /* + * Only values with a continuous set of 0..n bits set are + * valid for FFR, set all bits then clear a random number of + * high bits. + */ + memset(buf, 0, __SVE_FFR_SIZE(vq)); + + bits = random() % (__SVE_FFR_SIZE(vq) * 8); + for (i = 0; i < bits / 8; i++) + lbuf[i] = 0xff; + if (bits / 8 != __SVE_FFR_SIZE(vq)) + lbuf[i] = (1 << (bits % 8)) - 1; +} + +static void fpsimd_to_sve(__uint128_t *v, char *z, int vl) +{ + int vq = __sve_vq_from_vl(vl); + int i; + __uint128_t *p; + + if (!vl) + return; + + for (i = 0; i < __SVE_NUM_ZREGS; i++) { + p = (__uint128_t *)&z[__SVE_ZREG_OFFSET(vq, i)]; + *p = arm64_cpu_to_le128(v[i]); + } +} + +static void set_initial_values(struct test_config *config) +{ + int vq = __sve_vq_from_vl(vl_in(config)); + int sme_vq = __sve_vq_from_vl(config->sme_vl_in); + + svcr_in = config->svcr_in; + svcr_expected = config->svcr_expected; + svcr_out = 0; + + fill_random(&v_in, sizeof(v_in)); + memcpy(v_expected, v_in, sizeof(v_in)); + memset(v_out, 0, sizeof(v_out)); + + /* Changes will be handled in the test case */ + if (sve_supported() || (config->svcr_in & SVCR_SM)) { + /* The low 128 bits of Z are shared with the V registers */ + fill_random(&z_in, __SVE_ZREGS_SIZE(vq)); + fpsimd_to_sve(v_in, z_in, vl_in(config)); + memcpy(z_expected, z_in, __SVE_ZREGS_SIZE(vq)); + memset(z_out, 0, sizeof(z_out)); + + fill_random(&p_in, __SVE_PREGS_SIZE(vq)); + memcpy(p_expected, p_in, __SVE_PREGS_SIZE(vq)); + memset(p_out, 0, sizeof(p_out)); + + if ((config->svcr_in & SVCR_SM) && !fa64_supported()) + memset(ffr_in, 0, __SVE_PREG_SIZE(vq)); + else + fill_random_ffr(&ffr_in, vq); + memcpy(ffr_expected, ffr_in, __SVE_PREG_SIZE(vq)); + memset(ffr_out, 0, __SVE_PREG_SIZE(vq)); + } + + if (config->svcr_in & SVCR_ZA) + fill_random(za_in, ZA_SIG_REGS_SIZE(sme_vq)); + else + memset(za_in, 0, ZA_SIG_REGS_SIZE(sme_vq)); + if (config->svcr_expected & SVCR_ZA) + memcpy(za_expected, za_in, ZA_SIG_REGS_SIZE(sme_vq)); + else + memset(za_expected, 0, ZA_SIG_REGS_SIZE(sme_vq)); + if (sme_supported()) + memset(za_out, 0, sizeof(za_out)); + + if (sme2_supported()) { + if (config->svcr_in & SVCR_ZA) + fill_random(zt_in, ZT_SIG_REG_BYTES); + else + memset(zt_in, 0, ZT_SIG_REG_BYTES); + if (config->svcr_expected & SVCR_ZA) + memcpy(zt_expected, zt_in, ZT_SIG_REG_BYTES); + else + memset(zt_expected, 0, ZT_SIG_REG_BYTES); + memset(zt_out, 0, sizeof(zt_out)); + } +} + +static bool check_memory_values(struct test_config *config) +{ + bool pass = true; + int vq, sme_vq; + + if (!compare_buffer("saved V", v_out, v_expected, sizeof(v_out))) + pass = false; + + vq = __sve_vq_from_vl(vl_expected(config)); + sme_vq = __sve_vq_from_vl(config->sme_vl_expected); + + if (svcr_out != svcr_expected) { + ksft_print_msg("Mismatch in saved SVCR %lx != %lx\n", + svcr_out, svcr_expected); + pass = false; + } + + if (sve_vl_out != config->sve_vl_expected) { + ksft_print_msg("Mismatch in SVE VL: %ld != %d\n", + sve_vl_out, config->sve_vl_expected); + pass = false; + } + + if (sme_vl_out != config->sme_vl_expected) { + ksft_print_msg("Mismatch in SME VL: %ld != %d\n", + sme_vl_out, config->sme_vl_expected); + pass = false; + } + + if (!compare_buffer("saved Z", z_out, z_expected, + __SVE_ZREGS_SIZE(vq))) + pass = false; + + if (!compare_buffer("saved P", p_out, p_expected, + __SVE_PREGS_SIZE(vq))) + pass = false; + + if (!compare_buffer("saved FFR", ffr_out, ffr_expected, + __SVE_PREG_SIZE(vq))) + pass = false; + + if (!compare_buffer("saved ZA", za_out, za_expected, + ZA_PT_ZA_SIZE(sme_vq))) + pass = false; + + if (!compare_buffer("saved ZT", zt_out, zt_expected, ZT_SIG_REG_BYTES)) + pass = false; + + return pass; +} + +static bool sve_sme_same(struct test_config *config) +{ + if (config->sve_vl_in != config->sve_vl_expected) + return false; + + if (config->sme_vl_in != config->sme_vl_expected) + return false; + + if (config->svcr_in != config->svcr_expected) + return false; + + return true; +} + +static bool sve_write_supported(struct test_config *config) +{ + if (!sve_supported() && !sme_supported()) + return false; + + if ((config->svcr_in & SVCR_ZA) != (config->svcr_expected & SVCR_ZA)) + return false; + + if (config->svcr_expected & SVCR_SM) { + if (config->sve_vl_in != config->sve_vl_expected) { + return false; + } + + /* Changing the SME VL disables ZA */ + if ((config->svcr_expected & SVCR_ZA) && + (config->sme_vl_in != config->sme_vl_expected)) { + return false; + } + } else { + if (config->sme_vl_in != config->sme_vl_expected) { + return false; + } + } + + return true; +} + +static void fpsimd_write_expected(struct test_config *config) +{ + int vl; + + fill_random(&v_expected, sizeof(v_expected)); + + /* The SVE registers are flushed by a FPSIMD write */ + vl = vl_expected(config); + + memset(z_expected, 0, __SVE_ZREGS_SIZE(__sve_vq_from_vl(vl))); + memset(p_expected, 0, __SVE_PREGS_SIZE(__sve_vq_from_vl(vl))); + memset(ffr_expected, 0, __SVE_PREG_SIZE(__sve_vq_from_vl(vl))); + + fpsimd_to_sve(v_expected, z_expected, vl); +} + +static void fpsimd_write(pid_t child, struct test_config *test_config) +{ + struct user_fpsimd_state fpsimd; + struct iovec iov; + int ret; + + memset(&fpsimd, 0, sizeof(fpsimd)); + memcpy(&fpsimd.vregs, v_expected, sizeof(v_expected)); + + iov.iov_base = &fpsimd; + iov.iov_len = sizeof(fpsimd); + ret = ptrace(PTRACE_SETREGSET, child, NT_PRFPREG, &iov); + if (ret == -1) + ksft_print_msg("FPSIMD set failed: (%s) %d\n", + strerror(errno), errno); +} + +static void sve_write_expected(struct test_config *config) +{ + int vl = vl_expected(config); + int sme_vq = __sve_vq_from_vl(config->sme_vl_expected); + + fill_random(z_expected, __SVE_ZREGS_SIZE(__sve_vq_from_vl(vl))); + fill_random(p_expected, __SVE_PREGS_SIZE(__sve_vq_from_vl(vl))); + + if ((svcr_expected & SVCR_SM) && !fa64_supported()) + memset(ffr_expected, 0, __SVE_PREG_SIZE(sme_vq)); + else + fill_random_ffr(ffr_expected, __sve_vq_from_vl(vl)); + + /* Share the low bits of Z with V */ + fill_random(&v_expected, sizeof(v_expected)); + fpsimd_to_sve(v_expected, z_expected, vl); + + if (config->sme_vl_in != config->sme_vl_expected) { + memset(za_expected, 0, ZA_PT_ZA_SIZE(sme_vq)); + memset(zt_expected, 0, sizeof(zt_expected)); + } +} + +static void sve_write(pid_t child, struct test_config *config) +{ + struct user_sve_header *sve; + struct iovec iov; + int ret, vl, vq, regset; + + vl = vl_expected(config); + vq = __sve_vq_from_vl(vl); + + iov.iov_len = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE); + iov.iov_base = malloc(iov.iov_len); + if (!iov.iov_base) { + ksft_print_msg("Failed allocating %lu byte SVE write buffer\n", + iov.iov_len); + return; + } + memset(iov.iov_base, 0, iov.iov_len); + + sve = iov.iov_base; + sve->size = iov.iov_len; + sve->flags = SVE_PT_REGS_SVE; + sve->vl = vl; + + memcpy(iov.iov_base + SVE_PT_SVE_ZREG_OFFSET(vq, 0), + z_expected, SVE_PT_SVE_ZREGS_SIZE(vq)); + memcpy(iov.iov_base + SVE_PT_SVE_PREG_OFFSET(vq, 0), + p_expected, SVE_PT_SVE_PREGS_SIZE(vq)); + memcpy(iov.iov_base + SVE_PT_SVE_FFR_OFFSET(vq), + ffr_expected, SVE_PT_SVE_PREG_SIZE(vq)); + + if (svcr_expected & SVCR_SM) + regset = NT_ARM_SSVE; + else + regset = NT_ARM_SVE; + + ret = ptrace(PTRACE_SETREGSET, child, regset, &iov); + if (ret != 0) + ksft_print_msg("Failed to write SVE: %s (%d)\n", + strerror(errno), errno); + + free(iov.iov_base); +} + +static bool za_write_supported(struct test_config *config) +{ + if (config->svcr_expected & SVCR_SM) { + if (!(config->svcr_in & SVCR_SM)) + return false; + + /* Changing the SME VL exits streaming mode */ + if (config->sme_vl_in != config->sme_vl_expected) { + return false; + } + } + + /* Can't disable SM outside a VL change */ + if ((config->svcr_in & SVCR_SM) && + !(config->svcr_expected & SVCR_SM)) + return false; + + return true; +} + +static void za_write_expected(struct test_config *config) +{ + int sme_vq, sve_vq; + + sme_vq = __sve_vq_from_vl(config->sme_vl_expected); + + if (config->svcr_expected & SVCR_ZA) { + fill_random(za_expected, ZA_PT_ZA_SIZE(sme_vq)); + } else { + memset(za_expected, 0, ZA_PT_ZA_SIZE(sme_vq)); + memset(zt_expected, 0, sizeof(zt_expected)); + } + + /* Changing the SME VL flushes ZT, SVE state and exits SM */ + if (config->sme_vl_in != config->sme_vl_expected) { + svcr_expected &= ~SVCR_SM; + + sve_vq = __sve_vq_from_vl(vl_expected(config)); + memset(z_expected, 0, __SVE_ZREGS_SIZE(sve_vq)); + memset(p_expected, 0, __SVE_PREGS_SIZE(sve_vq)); + memset(ffr_expected, 0, __SVE_PREG_SIZE(sve_vq)); + memset(zt_expected, 0, sizeof(zt_expected)); + + fpsimd_to_sve(v_expected, z_expected, vl_expected(config)); + } +} + +static void za_write(pid_t child, struct test_config *config) +{ + struct user_za_header *za; + struct iovec iov; + int ret, vq; + + vq = __sve_vq_from_vl(config->sme_vl_expected); + + if (config->svcr_expected & SVCR_ZA) + iov.iov_len = ZA_PT_SIZE(vq); + else + iov.iov_len = sizeof(*za); + iov.iov_base = malloc(iov.iov_len); + if (!iov.iov_base) { + ksft_print_msg("Failed allocating %lu byte ZA write buffer\n", + iov.iov_len); + return; + } + memset(iov.iov_base, 0, iov.iov_len); + + za = iov.iov_base; + za->size = iov.iov_len; + za->vl = config->sme_vl_expected; + if (config->svcr_expected & SVCR_ZA) + memcpy(iov.iov_base + ZA_PT_ZA_OFFSET, za_expected, + ZA_PT_ZA_SIZE(vq)); + + ret = ptrace(PTRACE_SETREGSET, child, NT_ARM_ZA, &iov); + if (ret != 0) + ksft_print_msg("Failed to write ZA: %s (%d)\n", + strerror(errno), errno); + + free(iov.iov_base); +} + +static bool zt_write_supported(struct test_config *config) +{ + if (!sme2_supported()) + return false; + if (config->sme_vl_in != config->sme_vl_expected) + return false; + if (!(config->svcr_expected & SVCR_ZA)) + return false; + if ((config->svcr_in & SVCR_SM) != (config->svcr_expected & SVCR_SM)) + return false; + + return true; +} + +static void zt_write_expected(struct test_config *config) +{ + int sme_vq; + + sme_vq = __sve_vq_from_vl(config->sme_vl_expected); + + if (config->svcr_expected & SVCR_ZA) { + fill_random(zt_expected, sizeof(zt_expected)); + } else { + memset(za_expected, 0, ZA_PT_ZA_SIZE(sme_vq)); + memset(zt_expected, 0, sizeof(zt_expected)); + } +} + +static void zt_write(pid_t child, struct test_config *config) +{ + struct iovec iov; + int ret; + + iov.iov_len = ZT_SIG_REG_BYTES; + iov.iov_base = zt_expected; + ret = ptrace(PTRACE_SETREGSET, child, NT_ARM_ZT, &iov); + if (ret != 0) + ksft_print_msg("Failed to write ZT: %s (%d)\n", + strerror(errno), errno); +} + +/* Actually run a test */ +static void run_test(struct test_definition *test, struct test_config *config) +{ + pid_t child; + char name[1024]; + bool pass; + + if (sve_supported() && sme_supported()) + snprintf(name, sizeof(name), "%s, SVE %d->%d, SME %d/%x->%d/%x", + test->name, + config->sve_vl_in, config->sve_vl_expected, + config->sme_vl_in, config->svcr_in, + config->sme_vl_expected, config->svcr_expected); + else if (sve_supported()) + snprintf(name, sizeof(name), "%s, SVE %d->%d", test->name, + config->sve_vl_in, config->sve_vl_expected); + else if (sme_supported()) + snprintf(name, sizeof(name), "%s, SME %d/%x->%d/%x", + test->name, + config->sme_vl_in, config->svcr_in, + config->sme_vl_expected, config->svcr_expected); + else + snprintf(name, sizeof(name), "%s", test->name); + + if (test->supported && !test->supported(config)) { + ksft_test_result_skip("%s\n", name); + return; + } + + set_initial_values(config); + + if (test->set_expected_values) + test->set_expected_values(config); + + child = fork(); + if (child < 0) + ksft_exit_fail_msg("fork() failed: %s (%d)\n", + strerror(errno), errno); + /* run_child() never returns */ + if (child == 0) + run_child(config); + + pass = run_parent(child, test, config); + if (!check_memory_values(config)) + pass = false; + + ksft_test_result(pass, "%s\n", name); +} + +static void run_tests(struct test_definition defs[], int count, + struct test_config *config) +{ + int i; + + for (i = 0; i < count; i++) + run_test(&defs[i], config); +} + +static struct test_definition base_test_defs[] = { + { + .name = "No writes", + .supported = sve_sme_same, + }, + { + .name = "FPSIMD write", + .supported = sve_sme_same, + .set_expected_values = fpsimd_write_expected, + .modify_values = fpsimd_write, + }, +}; + +static struct test_definition sve_test_defs[] = { + { + .name = "SVE write", + .supported = sve_write_supported, + .set_expected_values = sve_write_expected, + .modify_values = sve_write, + }, +}; + +static struct test_definition za_test_defs[] = { + { + .name = "ZA write", + .supported = za_write_supported, + .set_expected_values = za_write_expected, + .modify_values = za_write, + }, +}; + +static struct test_definition zt_test_defs[] = { + { + .name = "ZT write", + .supported = zt_write_supported, + .set_expected_values = zt_write_expected, + .modify_values = zt_write, + }, +}; + +static int sve_vls[MAX_NUM_VLS], sme_vls[MAX_NUM_VLS]; +static int sve_vl_count, sme_vl_count; + +static void probe_vls(const char *name, int vls[], int *vl_count, int set_vl) +{ + unsigned int vq; + int vl; + + *vl_count = 0; + + for (vq = ARCH_VQ_MAX; vq > 0; vq /= 2) { + vl = prctl(set_vl, vq * 16); + if (vl == -1) + ksft_exit_fail_msg("SET_VL failed: %s (%d)\n", + strerror(errno), errno); + + vl &= PR_SVE_VL_LEN_MASK; + + if (*vl_count && (vl == vls[*vl_count - 1])) + break; + + vq = sve_vq_from_vl(vl); + + vls[*vl_count] = vl; + *vl_count += 1; + } + + if (*vl_count > 2) { + /* Just use the minimum and maximum */ + vls[1] = vls[*vl_count - 1]; + ksft_print_msg("%d %s VLs, using %d and %d\n", + *vl_count, name, vls[0], vls[1]); + *vl_count = 2; + } else { + ksft_print_msg("%d %s VLs\n", *vl_count, name); + } +} + +static struct { + int svcr_in, svcr_expected; +} svcr_combinations[] = { + { .svcr_in = 0, .svcr_expected = 0, }, + { .svcr_in = 0, .svcr_expected = SVCR_SM, }, + { .svcr_in = 0, .svcr_expected = SVCR_ZA, }, + /* Can't enable both SM and ZA with a single ptrace write */ + + { .svcr_in = SVCR_SM, .svcr_expected = 0, }, + { .svcr_in = SVCR_SM, .svcr_expected = SVCR_SM, }, + { .svcr_in = SVCR_SM, .svcr_expected = SVCR_ZA, }, + { .svcr_in = SVCR_SM, .svcr_expected = SVCR_SM | SVCR_ZA, }, + + { .svcr_in = SVCR_ZA, .svcr_expected = 0, }, + { .svcr_in = SVCR_ZA, .svcr_expected = SVCR_SM, }, + { .svcr_in = SVCR_ZA, .svcr_expected = SVCR_ZA, }, + { .svcr_in = SVCR_ZA, .svcr_expected = SVCR_SM | SVCR_ZA, }, + + { .svcr_in = SVCR_SM | SVCR_ZA, .svcr_expected = 0, }, + { .svcr_in = SVCR_SM | SVCR_ZA, .svcr_expected = SVCR_SM, }, + { .svcr_in = SVCR_SM | SVCR_ZA, .svcr_expected = SVCR_ZA, }, + { .svcr_in = SVCR_SM | SVCR_ZA, .svcr_expected = SVCR_SM | SVCR_ZA, }, +}; + +static void run_sve_tests(void) +{ + struct test_config test_config; + int i, j; + + if (!sve_supported()) + return; + + test_config.sme_vl_in = sme_vls[0]; + test_config.sme_vl_expected = sme_vls[0]; + test_config.svcr_in = 0; + test_config.svcr_expected = 0; + + for (i = 0; i < sve_vl_count; i++) { + test_config.sve_vl_in = sve_vls[i]; + + for (j = 0; j < sve_vl_count; j++) { + test_config.sve_vl_expected = sve_vls[j]; + + run_tests(base_test_defs, + ARRAY_SIZE(base_test_defs), + &test_config); + if (sve_supported()) + run_tests(sve_test_defs, + ARRAY_SIZE(sve_test_defs), + &test_config); + } + } + +} + +static void run_sme_tests(void) +{ + struct test_config test_config; + int i, j, k; + + if (!sme_supported()) + return; + + test_config.sve_vl_in = sve_vls[0]; + test_config.sve_vl_expected = sve_vls[0]; + + /* + * Every SME VL/SVCR combination + */ + for (i = 0; i < sme_vl_count; i++) { + test_config.sme_vl_in = sme_vls[i]; + + for (j = 0; j < sme_vl_count; j++) { + test_config.sme_vl_expected = sme_vls[j]; + + for (k = 0; k < ARRAY_SIZE(svcr_combinations); k++) { + test_config.svcr_in = svcr_combinations[k].svcr_in; + test_config.svcr_expected = svcr_combinations[k].svcr_expected; + + run_tests(base_test_defs, + ARRAY_SIZE(base_test_defs), + &test_config); + run_tests(sve_test_defs, + ARRAY_SIZE(sve_test_defs), + &test_config); + run_tests(za_test_defs, + ARRAY_SIZE(za_test_defs), + &test_config); + + if (sme2_supported()) + run_tests(zt_test_defs, + ARRAY_SIZE(zt_test_defs), + &test_config); + } + } + } +} + +int main(void) +{ + struct test_config test_config; + struct sigaction sa; + int tests, ret, tmp; + + srandom(getpid()); + + ksft_print_header(); + + if (sve_supported()) { + probe_vls("SVE", sve_vls, &sve_vl_count, PR_SVE_SET_VL); + + tests = ARRAY_SIZE(base_test_defs) + + ARRAY_SIZE(sve_test_defs); + tests *= sve_vl_count * sve_vl_count; + } else { + /* Only run the FPSIMD tests */ + sve_vl_count = 1; + tests = ARRAY_SIZE(base_test_defs); + } + + if (sme_supported()) { + probe_vls("SME", sme_vls, &sme_vl_count, PR_SME_SET_VL); + + tmp = ARRAY_SIZE(base_test_defs) + ARRAY_SIZE(sve_test_defs) + + ARRAY_SIZE(za_test_defs); + + if (sme2_supported()) + tmp += ARRAY_SIZE(zt_test_defs); + + tmp *= sme_vl_count * sme_vl_count; + tmp *= ARRAY_SIZE(svcr_combinations); + tests += tmp; + } else { + sme_vl_count = 1; + } + + if (sme2_supported()) + ksft_print_msg("SME2 supported\n"); + + if (fa64_supported()) + ksft_print_msg("FA64 supported\n"); + + ksft_set_plan(tests); + + /* Get signal handers ready before we start any children */ + memset(&sa, 0, sizeof(sa)); + sa.sa_sigaction = handle_alarm; + sa.sa_flags = SA_RESTART | SA_SIGINFO; + sigemptyset(&sa.sa_mask); + ret = sigaction(SIGALRM, &sa, NULL); + if (ret < 0) + ksft_print_msg("Failed to install SIGALRM handler: %s (%d)\n", + strerror(errno), errno); + + /* + * Run the test set if there is no SVE or SME, with those we + * have to pick a VL for each run. + */ + if (!sve_supported()) { + test_config.sve_vl_in = 0; + test_config.sve_vl_expected = 0; + test_config.sme_vl_in = 0; + test_config.sme_vl_expected = 0; + test_config.svcr_in = 0; + test_config.svcr_expected = 0; + + run_tests(base_test_defs, ARRAY_SIZE(base_test_defs), + &test_config); + } + + run_sve_tests(); + run_sme_tests(); + + ksft_finished(); +} diff --git a/tools/testing/selftests/arm64/fp/fp-ptrace.h b/tools/testing/selftests/arm64/fp/fp-ptrace.h new file mode 100644 index 000000000000..db4f2c4d750c --- /dev/null +++ b/tools/testing/selftests/arm64/fp/fp-ptrace.h @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2021-3 ARM Limited. + +#ifndef FP_PTRACE_H +#define FP_PTRACE_H + +#define SVCR_SM_SHIFT 0 +#define SVCR_ZA_SHIFT 1 + +#define SVCR_SM (1 << SVCR_SM_SHIFT) +#define SVCR_ZA (1 << SVCR_ZA_SHIFT) + +#endif |