diff options
Diffstat (limited to 'arch/riscv')
83 files changed, 2672 insertions, 484 deletions
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 48b7faf62d0b..3ceec2ca84fa 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -16,6 +16,8 @@ config RISCV select ACPI_REDUCED_HARDWARE_ONLY if ACPI select ARCH_DMA_DEFAULT_COHERENT select ARCH_ENABLE_HUGEPAGE_MIGRATION if HUGETLB_PAGE && MIGRATION + select ARCH_ENABLE_MEMORY_HOTPLUG if SPARSEMEM_VMEMMAP + select ARCH_ENABLE_MEMORY_HOTREMOVE if MEMORY_HOTPLUG select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE select ARCH_HAS_BINFMT_FLAT @@ -35,6 +37,7 @@ config RISCV select ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE select ARCH_HAS_PMEM_API select ARCH_HAS_PREPARE_SYNC_CORE_CMD + select ARCH_HAS_PTE_DEVMAP if 64BIT && MMU select ARCH_HAS_PTE_SPECIAL select ARCH_HAS_SET_DIRECT_MAP if MMU select ARCH_HAS_SET_MEMORY if MMU @@ -46,6 +49,7 @@ config RISCV select ARCH_HAS_UBSAN select ARCH_HAS_VDSO_DATA select ARCH_KEEP_MEMBLOCK if ACPI + select ARCH_MHP_MEMMAP_ON_MEMORY_ENABLE if 64BIT && MMU select ARCH_OPTIONAL_KERNEL_RWX if ARCH_HAS_STRICT_KERNEL_RWX select ARCH_OPTIONAL_KERNEL_RWX_DEFAULT select ARCH_STACKWALK @@ -69,6 +73,7 @@ config RISCV select ARCH_WANT_GENERAL_HUGETLB if !RISCV_ISA_SVNAPOT select ARCH_WANT_HUGE_PMD_SHARE if 64BIT select ARCH_WANT_LD_ORPHAN_WARN if !XIP_KERNEL + select ARCH_WANT_OPTIMIZE_DAX_VMEMMAP select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP select ARCH_WANTS_NO_INSTR select ARCH_WANTS_THP_SWAP if HAVE_ARCH_TRANSPARENT_HUGEPAGE @@ -595,6 +600,19 @@ config RISCV_ISA_V_PREEMPTIVE preemption. Enabling this config will result in higher memory consumption due to the allocation of per-task's kernel Vector context. +config RISCV_ISA_ZAWRS + bool "Zawrs extension support for more efficient busy waiting" + depends on RISCV_ALTERNATIVE + default y + help + The Zawrs extension defines instructions to be used in polling loops + which allow a hart to enter a low-power state or to trap to the + hypervisor while waiting on a store to a memory location. Enable the + use of these instructions in the kernel when the Zawrs extension is + detected at boot. + + If you don't know what to do here, say Y. + config TOOLCHAIN_HAS_ZBB bool default y @@ -610,6 +628,18 @@ config TOOLCHAIN_HAS_VECTOR_CRYPTO def_bool $(as-instr, .option arch$(comma) +v$(comma) +zvkb) depends on AS_HAS_OPTION_ARCH +config RISCV_ISA_ZBA + bool "Zba extension support for bit manipulation instructions" + default y + help + Add support for enabling optimisations in the kernel when the Zba + extension is detected at boot. + + The Zba extension provides instructions to accelerate the generation + of addresses that index into arrays of basic data types. + + If you don't know what to do here, say Y. + config RISCV_ISA_ZBB bool "Zbb extension support for bit manipulation instructions" depends on TOOLCHAIN_HAS_ZBB @@ -625,6 +655,29 @@ config RISCV_ISA_ZBB If you don't know what to do here, say Y. +config TOOLCHAIN_HAS_ZBC + bool + default y + depends on !64BIT || $(cc-option,-mabi=lp64 -march=rv64ima_zbc) + depends on !32BIT || $(cc-option,-mabi=ilp32 -march=rv32ima_zbc) + depends on LLD_VERSION >= 150000 || LD_VERSION >= 23900 + depends on AS_HAS_OPTION_ARCH + +config RISCV_ISA_ZBC + bool "Zbc extension support for carry-less multiplication instructions" + depends on TOOLCHAIN_HAS_ZBC + depends on MMU + depends on RISCV_ALTERNATIVE + default y + help + Adds support to dynamically detect the presence of the Zbc + extension (carry-less multiplication) and enable its usage. + + The Zbc extension could accelerate CRC (cyclic redundancy check) + calculations. + + If you don't know what to do here, say Y. + config RISCV_ISA_ZICBOM bool "Zicbom extension support for non-coherent DMA operation" depends on MMU @@ -654,13 +707,6 @@ config RISCV_ISA_ZICBOZ If you don't know what to do here, say Y. -config TOOLCHAIN_HAS_ZIHINTPAUSE - bool - default y - depends on !64BIT || $(cc-option,-mabi=lp64 -march=rv64ima_zihintpause) - depends on !32BIT || $(cc-option,-mabi=ilp32 -march=rv32ima_zihintpause) - depends on LLD_VERSION >= 150000 || LD_VERSION >= 23600 - config TOOLCHAIN_NEEDS_EXPLICIT_ZICSR_ZIFENCEI def_bool y # https://sourceware.org/git/?p=binutils-gdb.git;a=commit;h=aed44286efa8ae8717a77d94b51ac3614e2ca6dc @@ -968,6 +1014,17 @@ config EFI allow the kernel to be booted as an EFI application. This is only useful on systems that have UEFI firmware. +config DMI + bool "Enable support for SMBIOS (DMI) tables" + depends on EFI + default y + help + This enables SMBIOS/DMI feature for systems. + + This option is only useful on systems that have UEFI firmware. + However, even with this option, the resultant kernel should + continue to boot on existing non-UEFI platforms. + config CC_HAVE_STACKPROTECTOR_TLS def_bool $(cc-option,-mstack-protector-guard=tls -mstack-protector-guard-reg=tp -mstack-protector-guard-offset=0) diff --git a/arch/riscv/Makefile b/arch/riscv/Makefile index 06de9d365088..6fe682139d2e 100644 --- a/arch/riscv/Makefile +++ b/arch/riscv/Makefile @@ -82,9 +82,6 @@ else riscv-march-$(CONFIG_TOOLCHAIN_NEEDS_EXPLICIT_ZICSR_ZIFENCEI) := $(riscv-march-y)_zicsr_zifencei endif -# Check if the toolchain supports Zihintpause extension -riscv-march-$(CONFIG_TOOLCHAIN_HAS_ZIHINTPAUSE) := $(riscv-march-y)_zihintpause - # Remove F,D,V from isa string for all. Keep extensions between "fd" and "v" by # matching non-v and non-multi-letter extensions out with the filter ([^v_]*) KBUILD_CFLAGS += -march=$(shell echo $(riscv-march-y) | sed -E 's/(rv32ima|rv64ima)fd([^v_]*)v?/\1\2/') diff --git a/arch/riscv/boot/dts/allwinner/Makefile b/arch/riscv/boot/dts/allwinner/Makefile index 87f70b1af6b4..1c91be38ea16 100644 --- a/arch/riscv/boot/dts/allwinner/Makefile +++ b/arch/riscv/boot/dts/allwinner/Makefile @@ -1,4 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 +dtb-$(CONFIG_ARCH_SUNXI) += sun20i-d1-clockworkpi-v3.14.dtb +dtb-$(CONFIG_ARCH_SUNXI) += sun20i-d1-devterm-v3.14.dtb dtb-$(CONFIG_ARCH_SUNXI) += sun20i-d1-dongshan-nezha-stu.dtb dtb-$(CONFIG_ARCH_SUNXI) += sun20i-d1-lichee-rv-86-panel-480p.dtb dtb-$(CONFIG_ARCH_SUNXI) += sun20i-d1-lichee-rv-86-panel-720p.dtb diff --git a/arch/riscv/boot/dts/allwinner/sun20i-d1-clockworkpi-v3.14.dts b/arch/riscv/boot/dts/allwinner/sun20i-d1-clockworkpi-v3.14.dts new file mode 100644 index 000000000000..750aec6cf2f2 --- /dev/null +++ b/arch/riscv/boot/dts/allwinner/sun20i-d1-clockworkpi-v3.14.dts @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: (GPL-2.0+ or MIT) +// Copyright (C) 2022 Samuel Holland <samuel@sholland.org> + +#include <dt-bindings/gpio/gpio.h> + +/dts-v1/; + +#include "sun20i-d1.dtsi" +#include "sun20i-common-regulators.dtsi" + +/ { + model = "ClockworkPi v3.14 (R-01)"; + compatible = "clockwork,r-01-clockworkpi-v3.14", "allwinner,sun20i-d1"; + + aliases { + ethernet0 = &ap6256; + serial0 = &uart0; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + /* + * This regulator is PWM-controlled, but the PWM controller is not + * yet supported, so fix the regulator to its default voltage. + */ + reg_vdd_cpu: vdd-cpu { + compatible = "regulator-fixed"; + regulator-name = "vdd-cpu"; + regulator-min-microvolt = <1100000>; + regulator-max-microvolt = <1100000>; + vin-supply = <®_vcc>; + }; + + wifi_pwrseq: wifi-pwrseq { + compatible = "mmc-pwrseq-simple"; + reset-gpios = <&pio 6 11 GPIO_ACTIVE_LOW>; /* PG11/GPIO3 */ + }; +}; + +&cpu0 { + cpu-supply = <®_vdd_cpu>; +}; + +&dcxo { + clock-frequency = <24000000>; +}; + +&ehci1 { + status = "okay"; +}; + +&i2c0 { + pinctrl-0 = <&i2c0_pb10_pins>; + pinctrl-names = "default"; + status = "okay"; + + axp221: pmic@34 { + compatible = "x-powers,axp228", "x-powers,axp221"; + reg = <0x34>; + interrupt-parent = <&pio>; + interrupts = <4 9 IRQ_TYPE_LEVEL_LOW>; /* PE9/GPIO2 */ + interrupt-controller; + #interrupt-cells = <1>; + + ac_power_supply: ac-power { + compatible = "x-powers,axp221-ac-power-supply"; + }; + + axp_adc: adc { + compatible = "x-powers,axp221-adc"; + #io-channel-cells = <1>; + }; + + battery_power_supply: battery-power { + compatible = "x-powers,axp221-battery-power-supply"; + }; + + axp_gpio: gpio { + compatible = "x-powers,axp221-gpio"; + gpio-controller; + #gpio-cells = <2>; + }; + + regulators { + x-powers,dcdc-freq = <3000>; + + reg_dcdc1: dcdc1 { + regulator-name = "sys-3v3"; + regulator-always-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + + reg_dcdc3: dcdc3 { + regulator-name = "sys-1v8"; + regulator-always-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + + reg_aldo1: aldo1 { + regulator-name = "aud-3v3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + + reg_aldo2: aldo2 { + regulator-name = "disp-3v3"; + regulator-always-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + + reg_aldo3: aldo3 { + regulator-name = "vdd-wifi"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + + /* DLDO1 and ELDO1-3 are connected in parallel. */ + reg_dldo1: dldo1 { + regulator-name = "vbat-wifi-a"; + regulator-always-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + + /* DLDO2-DLDO4 are connected in parallel. */ + reg_dldo2: dldo2 { + regulator-name = "vcc-3v3-ext-a"; + regulator-always-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + + reg_dldo3: dldo3 { + regulator-name = "vcc-3v3-ext-b"; + regulator-always-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + + reg_dldo4: dldo4 { + regulator-name = "vcc-3v3-ext-c"; + regulator-always-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + + reg_eldo1: eldo1 { + regulator-name = "vbat-wifi-b"; + regulator-always-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + + reg_eldo2: eldo2 { + regulator-name = "vbat-wifi-c"; + regulator-always-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + + reg_eldo3: eldo3 { + regulator-name = "vbat-wifi-d"; + regulator-always-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + }; + + usb_power_supply: usb-power { + compatible = "x-powers,axp221-usb-power-supply"; + status = "disabled"; + }; + }; +}; + +&mmc0 { + broken-cd; + bus-width = <4>; + disable-wp; + vmmc-supply = <®_dcdc1>; + vqmmc-supply = <®_vcc_3v3>; + pinctrl-0 = <&mmc0_pins>; + pinctrl-names = "default"; + status = "okay"; +}; + +&mmc1 { + bus-width = <4>; + mmc-pwrseq = <&wifi_pwrseq>; + non-removable; + vmmc-supply = <®_dldo1>; + vqmmc-supply = <®_aldo3>; + pinctrl-0 = <&mmc1_pins>; + pinctrl-names = "default"; + status = "okay"; + + ap6256: wifi@1 { + compatible = "brcm,bcm43456-fmac", "brcm,bcm4329-fmac"; + reg = <1>; + interrupt-parent = <&pio>; + interrupts = <6 10 IRQ_TYPE_LEVEL_LOW>; /* PG10/GPIO4 */ + interrupt-names = "host-wake"; + }; +}; + +&ohci1 { + status = "okay"; +}; + +&pio { + vcc-pg-supply = <®_ldoa>; +}; + +&uart0 { + pinctrl-0 = <&uart0_pb8_pins>; + pinctrl-names = "default"; + status = "okay"; +}; + +&uart1 { + uart-has-rtscts; + pinctrl-0 = <&uart1_pg6_pins>, <&uart1_pg8_rts_cts_pins>; + pinctrl-names = "default"; + status = "okay"; + + bluetooth { + compatible = "brcm,bcm4345c5"; + interrupt-parent = <&pio>; + interrupts = <6 17 IRQ_TYPE_LEVEL_HIGH>; /* PG17/GPIO6 */ + device-wakeup-gpios = <&pio 6 16 GPIO_ACTIVE_HIGH>; /* PG16/GPIO7 */ + shutdown-gpios = <&pio 6 18 GPIO_ACTIVE_HIGH>; /* PG18/GPIO5 */ + max-speed = <1500000>; + vbat-supply = <®_dldo1>; + vddio-supply = <®_aldo3>; + }; +}; + +&usb_otg { + dr_mode = "peripheral"; + status = "okay"; +}; + +&usbphy { + usb0_vbus_power-supply = <&ac_power_supply>; + usb1_vbus-supply = <®_vcc>; + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/allwinner/sun20i-d1-devterm-v3.14.dts b/arch/riscv/boot/dts/allwinner/sun20i-d1-devterm-v3.14.dts new file mode 100644 index 000000000000..bc5c84f22762 --- /dev/null +++ b/arch/riscv/boot/dts/allwinner/sun20i-d1-devterm-v3.14.dts @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: (GPL-2.0+ or MIT) +// Copyright (C) 2022 Samuel Holland <samuel@sholland.org> + +#include "sun20i-d1-clockworkpi-v3.14.dts" + +/ { + model = "Clockwork DevTerm (R-01)"; + compatible = "clockwork,r-01-devterm-v3.14", + "clockwork,r-01-clockworkpi-v3.14", + "allwinner,sun20i-d1"; + + fan { + compatible = "gpio-fan"; + gpios = <&pio 3 10 GPIO_ACTIVE_HIGH>; /* PD10/GPIO41 */ + gpio-fan,speed-map = <0 0>, + <6000 1>; + #cooling-cells = <2>; + }; + + i2c-gpio-0 { + compatible = "i2c-gpio"; + sda-gpios = <&pio 3 14 (GPIO_ACTIVE_HIGH|GPIO_OPEN_DRAIN)>; /* PD14/GPIO44 */ + scl-gpios = <&pio 3 15 (GPIO_ACTIVE_HIGH|GPIO_OPEN_DRAIN)>; /* PD15/GPIO45 */ + #address-cells = <1>; + #size-cells = <0>; + + adc@54 { + compatible = "ti,adc101c"; + reg = <0x54>; + interrupt-parent = <&pio>; + interrupts = <4 12 IRQ_TYPE_LEVEL_LOW>; /* PE12/GPIO35 */ + vref-supply = <®_dldo2>; + #io-channel-cells = <1>; + }; + }; +}; diff --git a/arch/riscv/boot/dts/allwinner/sunxi-d1s-t113.dtsi b/arch/riscv/boot/dts/allwinner/sunxi-d1s-t113.dtsi index 5a9d7f5a75b4..e4175adb028d 100644 --- a/arch/riscv/boot/dts/allwinner/sunxi-d1s-t113.dtsi +++ b/arch/riscv/boot/dts/allwinner/sunxi-d1s-t113.dtsi @@ -396,6 +396,17 @@ ranges; #address-cells = <1>; #size-cells = <1>; + + regulators@3000150 { + compatible = "allwinner,sun20i-d1-system-ldos"; + reg = <0x3000150 0x4>; + + reg_ldoa: ldoa { + }; + + reg_ldob: ldob { + }; + }; }; dma: dma-controller@3002000 { diff --git a/arch/riscv/boot/dts/microchip/Makefile b/arch/riscv/boot/dts/microchip/Makefile index e177815bf1a2..f51aeeb9fd3b 100644 --- a/arch/riscv/boot/dts/microchip/Makefile +++ b/arch/riscv/boot/dts/microchip/Makefile @@ -1,4 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 +dtb-$(CONFIG_ARCH_MICROCHIP_POLARFIRE) += mpfs-beaglev-fire.dtb dtb-$(CONFIG_ARCH_MICROCHIP_POLARFIRE) += mpfs-icicle-kit.dtb dtb-$(CONFIG_ARCH_MICROCHIP_POLARFIRE) += mpfs-m100pfsevp.dtb dtb-$(CONFIG_ARCH_MICROCHIP_POLARFIRE) += mpfs-polarberry.dtb diff --git a/arch/riscv/boot/dts/microchip/mpfs-beaglev-fire-fabric.dtsi b/arch/riscv/boot/dts/microchip/mpfs-beaglev-fire-fabric.dtsi new file mode 100644 index 000000000000..e153eaf9b90e --- /dev/null +++ b/arch/riscv/boot/dts/microchip/mpfs-beaglev-fire-fabric.dtsi @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) + +/ { + fabric_clk3: fabric-clk3 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <50000000>; + }; + + fabric_clk1: fabric-clk1 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <125000000>; + }; + + fabric-bus@40000000 { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <2>; + ranges = <0x0 0x40000000 0x0 0x40000000 0x0 0x20000000>, /* FIC3-FAB */ + <0x0 0x60000000 0x0 0x60000000 0x0 0x20000000>, /* FIC0, LO */ + <0x0 0xe0000000 0x0 0xe0000000 0x0 0x20000000>, /* FIC1, LO */ + <0x20 0x0 0x20 0x0 0x10 0x0>, /* FIC0,HI */ + <0x30 0x0 0x30 0x0 0x10 0x0>; /* FIC1,HI */ + + cape_gpios_p8: gpio@41100000 { + compatible = "microchip,coregpio-rtl-v3"; + reg = <0x0 0x41100000 0x0 0x1000>; + clocks = <&fabric_clk3>; + gpio-controller; + #gpio-cells = <2>; + ngpios = <16>; + gpio-line-names = "P8_PIN31", "P8_PIN32", "P8_PIN33", "P8_PIN34", + "P8_PIN35", "P8_PIN36", "P8_PIN37", "P8_PIN38", + "P8_PIN39", "P8_PIN40", "P8_PIN41", "P8_PIN42", + "P8_PIN43", "P8_PIN44", "P8_PIN45", "P8_PIN46"; + }; + + cape_gpios_p9: gpio@41200000 { + compatible = "microchip,coregpio-rtl-v3"; + reg = <0x0 0x41200000 0x0 0x1000>; + clocks = <&fabric_clk3>; + gpio-controller; + #gpio-cells = <2>; + ngpios = <20>; + gpio-line-names = "P9_PIN11", "P9_PIN12", "P9_PIN13", "P9_PIN14", + "P9_PIN15", "P9_PIN16", "P9_PIN17", "P9_PIN18", + "P9_PIN21", "P9_PIN22", "P9_PIN23", "P9_PIN24", + "P9_PIN25", "P9_PIN26", "P9_PIN27", "P9_PIN28", + "P9_PIN29", "P9_PIN31", "P9_PIN41", "P9_PIN42"; + }; + + hsi_gpios: gpio@44000000 { + compatible = "microchip,coregpio-rtl-v3"; + reg = <0x0 0x44000000 0x0 0x1000>; + clocks = <&fabric_clk3>; + gpio-controller; + #gpio-cells = <2>; + ngpios = <20>; + gpio-line-names = "B0_HSIO70N", "B0_HSIO71N", "B0_HSIO83N", + "B0_HSIO73N_C2P_CLKN", "B0_HSIO70P", "B0_HSIO71P", + "B0_HSIO83P", "B0_HSIO73N_C2P_CLKP", "XCVR1_RX_VALID", + "XCVR1_LOCK", "XCVR1_ERROR", "XCVR2_RX_VALID", + "XCVR2_LOCK", "XCVR2_ERROR", "XCVR3_RX_VALID", + "XCVR3_LOCK", "XCVR3_ERROR", "XCVR_0B_REF_CLK_PLL_LOCK", + "XCVR_0C_REF_CLK_PLL_LOCK", "B0_HSIO81N"; + }; + }; + + refclk_ccc: cccrefclk { + compatible = "fixed-clock"; + #clock-cells = <0>; + }; +}; + +&ccc_nw { + clocks = <&refclk_ccc>, <&refclk_ccc>, <&refclk_ccc>, <&refclk_ccc>, + <&refclk_ccc>, <&refclk_ccc>; + clock-names = "pll0_ref0", "pll0_ref1", "pll1_ref0", "pll1_ref1", + "dll0_ref", "dll1_ref"; + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/microchip/mpfs-beaglev-fire.dts b/arch/riscv/boot/dts/microchip/mpfs-beaglev-fire.dts new file mode 100644 index 000000000000..47cf693beb68 --- /dev/null +++ b/arch/riscv/boot/dts/microchip/mpfs-beaglev-fire.dts @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* Copyright (c) 2020-2021 Microchip Technology Inc */ + +/dts-v1/; + +#include <dt-bindings/gpio/gpio.h> +#include "mpfs.dtsi" +#include "mpfs-beaglev-fire-fabric.dtsi" + +/* Clock frequency (in Hz) of MTIMER */ +#define MTIMER_FREQ 1000000 + +/ { + #address-cells = <2>; + #size-cells = <2>; + model = "BeagleBoard BeagleV-Fire"; + compatible = "beagle,beaglev-fire", "microchip,mpfs"; + + aliases { + serial0 = &mmuart0; + serial1 = &mmuart1; + serial2 = &mmuart2; + serial3 = &mmuart3; + serial4 = &mmuart4; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + cpus { + timebase-frequency = <MTIMER_FREQ>; + }; + + ddrc_cache_lo: memory@80000000 { + device_type = "memory"; + reg = <0x0 0x80000000 0x0 0x40000000>; + status = "okay"; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + hss: hss-buffer@103fc00000 { + compatible = "shared-dma-pool"; + reg = <0x10 0x3fc00000 0x0 0x400000>; + no-map; + }; + }; + + imx219_clk: camera-clk { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <24000000>; + }; + + imx219_vana: fixedregulator-0 { + compatible = "regulator-fixed"; + regulator-name = "imx219_vana"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + }; + + imx219_vdig: fixedregulator-1 { + compatible = "regulator-fixed"; + regulator-name = "imx219_vdig"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + + imx219_vddl: fixedregulator-2 { + compatible = "regulator-fixed"; + regulator-name = "imx219_vddl"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + }; + +}; + +&gpio2 { + interrupts = <53>, <53>, <53>, <53>, + <53>, <53>, <53>, <53>, + <53>, <53>, <53>, <53>, + <53>, <53>, <53>, <53>, + <53>, <53>, <53>, <53>, + <53>, <53>, <53>, <53>, + <53>, <53>, <53>, <53>, + <53>, <53>, <53>, <53>; + ngpios=<32>; + gpio-line-names = "P8_PIN3_USER_LED_0", "P8_PIN4_USER_LED_1", "P8_PIN5_USER_LED_2", + "P8_PIN6_USER_LED_3", "P8_PIN7_USER_LED_4", "P8_PIN8_USER_LED_5", + "P8_PIN9_USER_LED_6", "P8_PIN10_USER_LED_7", "P8_PIN11_USER_LED_8", + "P8_PIN12_USER_LED_9", "P8_PIN13_USER_LED_10", "P8_PIN14_USER_LED_11", + "P8_PIN15", "P8_PIN16", "P8_PIN17", "P8_PIN18", "P8_PIN19", "P8_PIN20", + "P8_PIN21", "P8_PIN22", "P8_PIN23", "P8_PIN24", "P8_PIN25", "P8_PIN26", + "P8_PIN27", "P8_PIN28", "P8_PIN29", "P8_PIN30", "M2_W_DISABLE1", + "M2_W_DISABLE2", "VIO_ENABLE", "SD_DET"; + status = "okay"; + + vio-enable-hog { + gpio-hog; + gpios = <30 30>; + output-high; + line-name = "VIO_ENABLE"; + }; + + sd-det-hog { + gpio-hog; + gpios = <31 31>; + input; + line-name = "SD_DET"; + }; +}; + +&i2c0 { + status = "okay"; +}; + +&i2c1 { + status = "okay"; + + eeprom: eeprom@50 { + compatible = "atmel,24c32"; + reg = <0x50>; + }; + + imx219: sensor@10 { + compatible = "sony,imx219"; + reg = <0x10>; + clocks = <&imx219_clk>; + VANA-supply = <&imx219_vana>; /* 2.8v */ + VDIG-supply = <&imx219_vdig>; /* 1.8v */ + VDDL-supply = <&imx219_vddl>; /* 1.2v */ + + port { + imx219_0: endpoint { + data-lanes = <1 2>; + clock-noncontinuous; + link-frequencies = /bits/ 64 <456000000>; + }; + }; + }; +}; + +&mac0 { + status = "okay"; + phy-mode = "sgmii"; + phy-handle = <&phy0>; + phy0: ethernet-phy@0 { + reg = <0>; + }; +}; + +&mbox { + status = "okay"; +}; + +&mmc { + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-mmc-highspeed; + mmc-ddr-1_8v; + mmc-hs200-1_8v; + sd-uhs-sdr12; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; + status = "okay"; +}; + +&mmuart0 { + status = "okay"; +}; + +&mmuart1 { + status = "okay"; +}; + +&refclk { + clock-frequency = <125000000>; +}; + +&refclk_ccc { + clock-frequency = <50000000>; +}; + +&rtc { + status = "okay"; +}; + +&spi0 { + status = "okay"; +}; + +&spi1 { + status = "okay"; +}; + +&syscontroller { + microchip,bitstream-flash = <&sys_ctrl_flash>; + status = "okay"; +}; + +&syscontroller_qspi { + status = "okay"; + + sys_ctrl_flash: flash@0 { // MT25QL01GBBB8ESF-0SIT + compatible = "jedec,spi-nor"; + #address-cells = <1>; + #size-cells = <1>; + spi-max-frequency = <20000000>; + spi-rx-bus-width = <1>; + reg = <0>; + }; +}; + +&usb { + status = "okay"; + dr_mode = "otg"; +}; diff --git a/arch/riscv/boot/dts/sophgo/sg2042-milkv-pioneer.dts b/arch/riscv/boot/dts/sophgo/sg2042-milkv-pioneer.dts index 49b4b9c2c101..80cb017974d8 100644 --- a/arch/riscv/boot/dts/sophgo/sg2042-milkv-pioneer.dts +++ b/arch/riscv/boot/dts/sophgo/sg2042-milkv-pioneer.dts @@ -14,6 +14,18 @@ }; }; +&cgi_main { + clock-frequency = <25000000>; +}; + +&cgi_dpll0 { + clock-frequency = <25000000>; +}; + +&cgi_dpll1 { + clock-frequency = <25000000>; +}; + &uart0 { status = "okay"; }; diff --git a/arch/riscv/boot/dts/sophgo/sg2042.dtsi b/arch/riscv/boot/dts/sophgo/sg2042.dtsi index 81fda312f988..34c802bd3f9b 100644 --- a/arch/riscv/boot/dts/sophgo/sg2042.dtsi +++ b/arch/riscv/boot/dts/sophgo/sg2042.dtsi @@ -4,8 +4,10 @@ */ /dts-v1/; +#include <dt-bindings/clock/sophgo,sg2042-clkgen.h> +#include <dt-bindings/clock/sophgo,sg2042-pll.h> +#include <dt-bindings/clock/sophgo,sg2042-rpgate.h> #include <dt-bindings/interrupt-controller/irq.h> - #include <dt-bindings/reset/sophgo,sg2042-reset.h> #include "sg2042-cpus.dtsi" @@ -20,12 +22,60 @@ serial0 = &uart0; }; + cgi_main: oscillator0 { + compatible = "fixed-clock"; + clock-output-names = "cgi_main"; + #clock-cells = <0>; + }; + + cgi_dpll0: oscillator1 { + compatible = "fixed-clock"; + clock-output-names = "cgi_dpll0"; + #clock-cells = <0>; + }; + + cgi_dpll1: oscillator2 { + compatible = "fixed-clock"; + clock-output-names = "cgi_dpll1"; + #clock-cells = <0>; + }; + soc: soc { compatible = "simple-bus"; #address-cells = <2>; #size-cells = <2>; ranges; + pllclk: clock-controller@70300100c0 { + compatible = "sophgo,sg2042-pll"; + reg = <0x70 0x300100c0 0x0 0x40>; + clocks = <&cgi_main>, <&cgi_dpll0>, <&cgi_dpll1>; + clock-names = "cgi_main", "cgi_dpll0", "cgi_dpll1"; + #clock-cells = <1>; + }; + + rpgate: clock-controller@7030010368 { + compatible = "sophgo,sg2042-rpgate"; + reg = <0x70 0x30010368 0x0 0x98>; + clocks = <&clkgen GATE_CLK_RP_CPU_NORMAL>; + clock-names = "rpgate"; + #clock-cells = <1>; + }; + + clkgen: clock-controller@7030012000 { + compatible = "sophgo,sg2042-clkgen"; + reg = <0x70 0x30012000 0x0 0x1000>; + clocks = <&pllclk MPLL_CLK>, + <&pllclk FPLL_CLK>, + <&pllclk DPLL0_CLK>, + <&pllclk DPLL1_CLK>; + clock-names = "mpll", + "fpll", + "dpll0", + "dpll1"; + #clock-cells = <1>; + }; + clint_mswi: interrupt-controller@7094000000 { compatible = "sophgo,sg2042-aclint-mswi", "thead,c900-aclint-mswi"; reg = <0x00000070 0x94000000 0x00000000 0x00004000>; @@ -341,6 +391,9 @@ interrupt-parent = <&intc>; interrupts = <112 IRQ_TYPE_LEVEL_HIGH>; clock-frequency = <500000000>; + clocks = <&clkgen GATE_CLK_UART_500M>, + <&clkgen GATE_CLK_APB_UART>; + clock-names = "baudclk", "apb_pclk"; reg-shift = <2>; reg-io-width = <4>; resets = <&rstgen RST_UART0>; diff --git a/arch/riscv/boot/dts/starfive/Makefile b/arch/riscv/boot/dts/starfive/Makefile index 2fa0cd7f31c3..7a163a7d6ba3 100644 --- a/arch/riscv/boot/dts/starfive/Makefile +++ b/arch/riscv/boot/dts/starfive/Makefile @@ -9,5 +9,6 @@ dtb-$(CONFIG_ARCH_STARFIVE) += jh7100-beaglev-starlight.dtb dtb-$(CONFIG_ARCH_STARFIVE) += jh7100-starfive-visionfive-v1.dtb dtb-$(CONFIG_ARCH_STARFIVE) += jh7110-milkv-mars.dtb +dtb-$(CONFIG_ARCH_STARFIVE) += jh7110-pine64-star64.dtb dtb-$(CONFIG_ARCH_STARFIVE) += jh7110-starfive-visionfive-2-v1.2a.dtb dtb-$(CONFIG_ARCH_STARFIVE) += jh7110-starfive-visionfive-2-v1.3b.dtb diff --git a/arch/riscv/boot/dts/starfive/jh7110-common.dtsi b/arch/riscv/boot/dts/starfive/jh7110-common.dtsi index 68d16717db8c..ca2d44d59d48 100644 --- a/arch/riscv/boot/dts/starfive/jh7110-common.dtsi +++ b/arch/riscv/boot/dts/starfive/jh7110-common.dtsi @@ -294,6 +294,20 @@ status = "okay"; }; +&pcie0 { + perst-gpios = <&sysgpio 26 GPIO_ACTIVE_LOW>; + phys = <&pciephy0>; + pinctrl-names = "default"; + pinctrl-0 = <&pcie0_pins>; +}; + +&pcie1 { + perst-gpios = <&sysgpio 28 GPIO_ACTIVE_LOW>; + phys = <&pciephy1>; + pinctrl-names = "default"; + pinctrl-0 = <&pcie1_pins>; +}; + &pwmdac { pinctrl-names = "default"; pinctrl-0 = <&pwmdac_pins>; @@ -321,16 +335,13 @@ #size-cells = <1>; spl@0 { - reg = <0x0 0x80000>; + reg = <0x0 0xf0000>; }; uboot-env@f0000 { reg = <0xf0000 0x10000>; }; uboot@100000 { - reg = <0x100000 0x400000>; - }; - reserved-data@600000 { - reg = <0x600000 0xa00000>; + reg = <0x100000 0xf00000>; }; }; }; @@ -476,6 +487,54 @@ }; }; + pcie0_pins: pcie0-0 { + clkreq-pins { + pinmux = <GPIOMUX(27, GPOUT_LOW, + GPOEN_DISABLE, + GPI_NONE)>; + bias-pull-down; + drive-strength = <2>; + input-enable; + input-schmitt-disable; + slew-rate = <0>; + }; + + wake-pins { + pinmux = <GPIOMUX(32, GPOUT_LOW, + GPOEN_DISABLE, + GPI_NONE)>; + bias-pull-up; + drive-strength = <2>; + input-enable; + input-schmitt-disable; + slew-rate = <0>; + }; + }; + + pcie1_pins: pcie1-0 { + clkreq-pins { + pinmux = <GPIOMUX(29, GPOUT_LOW, + GPOEN_DISABLE, + GPI_NONE)>; + bias-pull-down; + drive-strength = <2>; + input-enable; + input-schmitt-disable; + slew-rate = <0>; + }; + + wake-pins { + pinmux = <GPIOMUX(21, GPOUT_LOW, + GPOEN_DISABLE, + GPI_NONE)>; + bias-pull-up; + drive-strength = <2>; + input-enable; + input-schmitt-disable; + slew-rate = <0>; + }; + }; + pwmdac_pins: pwmdac-0 { pwmdac-pins { pinmux = <GPIOMUX(33, GPOUT_SYS_PWMDAC_LEFT, diff --git a/arch/riscv/boot/dts/starfive/jh7110-milkv-mars.dts b/arch/riscv/boot/dts/starfive/jh7110-milkv-mars.dts index fa0eac78e0ba..5cb9e99e1dac 100644 --- a/arch/riscv/boot/dts/starfive/jh7110-milkv-mars.dts +++ b/arch/riscv/boot/dts/starfive/jh7110-milkv-mars.dts @@ -17,6 +17,13 @@ assigned-clock-parents = <&aoncrg JH7110_AONCLK_GMAC0_RMII_RTX>; }; +&pcie0 { + status = "okay"; +}; + +&pcie1 { + status = "okay"; +}; &phy0 { motorcomm,tx-clk-adj-enabled; diff --git a/arch/riscv/boot/dts/starfive/jh7110-pine64-star64.dts b/arch/riscv/boot/dts/starfive/jh7110-pine64-star64.dts new file mode 100644 index 000000000000..b720cdd15ed6 --- /dev/null +++ b/arch/riscv/boot/dts/starfive/jh7110-pine64-star64.dts @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2024 Henry Bell <dmoo_dv@protonmail.com> + */ + +/dts-v1/; +#include "jh7110-common.dtsi" + +/ { + model = "Pine64 Star64"; + compatible = "pine64,star64", "starfive,jh7110"; + aliases { + ethernet1 = &gmac1; + }; +}; + +&gmac0 { + starfive,tx-use-rgmii-clk; + assigned-clocks = <&aoncrg JH7110_AONCLK_GMAC0_TX>; + assigned-clock-parents = <&aoncrg JH7110_AONCLK_GMAC0_RMII_RTX>; +}; + +&gmac1 { + phy-handle = <&phy1>; + phy-mode = "rgmii-id"; + starfive,tx-use-rgmii-clk; + assigned-clocks = <&syscrg JH7110_SYSCLK_GMAC1_TX>; + assigned-clock-parents = <&syscrg JH7110_SYSCLK_GMAC1_RMII_RTX>; + status = "okay"; + + mdio { + #address-cells = <1>; + #size-cells = <0>; + compatible = "snps,dwmac-mdio"; + + phy1: ethernet-phy@1 { + reg = <1>; + }; + }; +}; + +&pcie1 { + status = "okay"; +}; + +&phy0 { + rx-internal-delay-ps = <1900>; + tx-internal-delay-ps = <1500>; + motorcomm,rx-clk-drv-microamp = <2910>; + motorcomm,rx-data-drv-microamp = <2910>; + motorcomm,tx-clk-adj-enabled; + motorcomm,tx-clk-10-inverted; + motorcomm,tx-clk-100-inverted; + motorcomm,tx-clk-1000-inverted; +}; + +&phy1 { + rx-internal-delay-ps = <0>; + tx-internal-delay-ps = <300>; + motorcomm,rx-clk-drv-microamp = <2910>; + motorcomm,rx-data-drv-microamp = <2910>; + motorcomm,tx-clk-adj-enabled; + motorcomm,tx-clk-10-inverted; + motorcomm,tx-clk-100-inverted; +}; diff --git a/arch/riscv/boot/dts/starfive/jh7110-starfive-visionfive-2.dtsi b/arch/riscv/boot/dts/starfive/jh7110-starfive-visionfive-2.dtsi index 9d70f21c86fc..18f38fc790a4 100644 --- a/arch/riscv/boot/dts/starfive/jh7110-starfive-visionfive-2.dtsi +++ b/arch/riscv/boot/dts/starfive/jh7110-starfive-visionfive-2.dtsi @@ -32,3 +32,11 @@ &mmc0 { non-removable; }; + +&pcie0 { + status = "okay"; +}; + +&pcie1 { + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/starfive/jh7110.dtsi b/arch/riscv/boot/dts/starfive/jh7110.dtsi index 18047195c600..0d8339357bad 100644 --- a/arch/riscv/boot/dts/starfive/jh7110.dtsi +++ b/arch/riscv/boot/dts/starfive/jh7110.dtsi @@ -387,12 +387,13 @@ }; uart0: serial@10000000 { - compatible = "snps,dw-apb-uart"; + compatible = "starfive,jh7110-uart", "snps,dw-apb-uart"; reg = <0x0 0x10000000 0x0 0x10000>; clocks = <&syscrg JH7110_SYSCLK_UART0_CORE>, <&syscrg JH7110_SYSCLK_UART0_APB>; clock-names = "baudclk", "apb_pclk"; - resets = <&syscrg JH7110_SYSRST_UART0_APB>; + resets = <&syscrg JH7110_SYSRST_UART0_APB>, + <&syscrg JH7110_SYSRST_UART0_CORE>; interrupts = <32>; reg-io-width = <4>; reg-shift = <2>; @@ -400,12 +401,13 @@ }; uart1: serial@10010000 { - compatible = "snps,dw-apb-uart"; + compatible = "starfive,jh7110-uart", "snps,dw-apb-uart"; reg = <0x0 0x10010000 0x0 0x10000>; clocks = <&syscrg JH7110_SYSCLK_UART1_CORE>, <&syscrg JH7110_SYSCLK_UART1_APB>; clock-names = "baudclk", "apb_pclk"; - resets = <&syscrg JH7110_SYSRST_UART1_APB>; + resets = <&syscrg JH7110_SYSRST_UART1_APB>, + <&syscrg JH7110_SYSRST_UART1_CORE>; interrupts = <33>; reg-io-width = <4>; reg-shift = <2>; @@ -413,12 +415,13 @@ }; uart2: serial@10020000 { - compatible = "snps,dw-apb-uart"; + compatible = "starfive,jh7110-uart", "snps,dw-apb-uart"; reg = <0x0 0x10020000 0x0 0x10000>; clocks = <&syscrg JH7110_SYSCLK_UART2_CORE>, <&syscrg JH7110_SYSCLK_UART2_APB>; clock-names = "baudclk", "apb_pclk"; - resets = <&syscrg JH7110_SYSRST_UART2_APB>; + resets = <&syscrg JH7110_SYSRST_UART2_APB>, + <&syscrg JH7110_SYSRST_UART2_CORE>; interrupts = <34>; reg-io-width = <4>; reg-shift = <2>; @@ -642,12 +645,13 @@ }; uart3: serial@12000000 { - compatible = "snps,dw-apb-uart"; + compatible = "starfive,jh7110-uart", "snps,dw-apb-uart"; reg = <0x0 0x12000000 0x0 0x10000>; clocks = <&syscrg JH7110_SYSCLK_UART3_CORE>, <&syscrg JH7110_SYSCLK_UART3_APB>; clock-names = "baudclk", "apb_pclk"; - resets = <&syscrg JH7110_SYSRST_UART3_APB>; + resets = <&syscrg JH7110_SYSRST_UART3_APB>, + <&syscrg JH7110_SYSRST_UART3_CORE>; interrupts = <45>; reg-io-width = <4>; reg-shift = <2>; @@ -655,12 +659,13 @@ }; uart4: serial@12010000 { - compatible = "snps,dw-apb-uart"; + compatible = "starfive,jh7110-uart", "snps,dw-apb-uart"; reg = <0x0 0x12010000 0x0 0x10000>; clocks = <&syscrg JH7110_SYSCLK_UART4_CORE>, <&syscrg JH7110_SYSCLK_UART4_APB>; clock-names = "baudclk", "apb_pclk"; - resets = <&syscrg JH7110_SYSRST_UART4_APB>; + resets = <&syscrg JH7110_SYSRST_UART4_APB>, + <&syscrg JH7110_SYSRST_UART4_CORE>; interrupts = <46>; reg-io-width = <4>; reg-shift = <2>; @@ -668,12 +673,13 @@ }; uart5: serial@12020000 { - compatible = "snps,dw-apb-uart"; + compatible = "starfive,jh7110-uart", "snps,dw-apb-uart"; reg = <0x0 0x12020000 0x0 0x10000>; clocks = <&syscrg JH7110_SYSCLK_UART5_CORE>, <&syscrg JH7110_SYSCLK_UART5_APB>; clock-names = "baudclk", "apb_pclk"; - resets = <&syscrg JH7110_SYSRST_UART5_APB>; + resets = <&syscrg JH7110_SYSRST_UART5_APB>, + <&syscrg JH7110_SYSRST_UART5_CORE>; interrupts = <47>; reg-io-width = <4>; reg-shift = <2>; @@ -1214,5 +1220,91 @@ #reset-cells = <1>; power-domains = <&pwrc JH7110_PD_VOUT>; }; + + pcie0: pcie@940000000 { + compatible = "starfive,jh7110-pcie"; + reg = <0x9 0x40000000 0x0 0x1000000>, + <0x0 0x2b000000 0x0 0x100000>; + reg-names = "cfg", "apb"; + linux,pci-domain = <0>; + #address-cells = <3>; + #size-cells = <2>; + #interrupt-cells = <1>; + ranges = <0x82000000 0x0 0x30000000 0x0 0x30000000 0x0 0x08000000>, + <0xc3000000 0x9 0x00000000 0x9 0x00000000 0x0 0x40000000>; + interrupts = <56>; + interrupt-map-mask = <0x0 0x0 0x0 0x7>; + interrupt-map = <0x0 0x0 0x0 0x1 &pcie_intc0 0x1>, + <0x0 0x0 0x0 0x2 &pcie_intc0 0x2>, + <0x0 0x0 0x0 0x3 &pcie_intc0 0x3>, + <0x0 0x0 0x0 0x4 &pcie_intc0 0x4>; + msi-controller; + device_type = "pci"; + starfive,stg-syscon = <&stg_syscon>; + bus-range = <0x0 0xff>; + clocks = <&syscrg JH7110_SYSCLK_NOC_BUS_STG_AXI>, + <&stgcrg JH7110_STGCLK_PCIE0_TL>, + <&stgcrg JH7110_STGCLK_PCIE0_AXI_MST0>, + <&stgcrg JH7110_STGCLK_PCIE0_APB>; + clock-names = "noc", "tl", "axi_mst0", "apb"; + resets = <&stgcrg JH7110_STGRST_PCIE0_AXI_MST0>, + <&stgcrg JH7110_STGRST_PCIE0_AXI_SLV0>, + <&stgcrg JH7110_STGRST_PCIE0_AXI_SLV>, + <&stgcrg JH7110_STGRST_PCIE0_BRG>, + <&stgcrg JH7110_STGRST_PCIE0_CORE>, + <&stgcrg JH7110_STGRST_PCIE0_APB>; + reset-names = "mst0", "slv0", "slv", "brg", + "core", "apb"; + status = "disabled"; + + pcie_intc0: interrupt-controller { + #address-cells = <0>; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; + + pcie1: pcie@9c0000000 { + compatible = "starfive,jh7110-pcie"; + reg = <0x9 0xc0000000 0x0 0x1000000>, + <0x0 0x2c000000 0x0 0x100000>; + reg-names = "cfg", "apb"; + linux,pci-domain = <1>; + #address-cells = <3>; + #size-cells = <2>; + #interrupt-cells = <1>; + ranges = <0x82000000 0x0 0x38000000 0x0 0x38000000 0x0 0x08000000>, + <0xc3000000 0x9 0x80000000 0x9 0x80000000 0x0 0x40000000>; + interrupts = <57>; + interrupt-map-mask = <0x0 0x0 0x0 0x7>; + interrupt-map = <0x0 0x0 0x0 0x1 &pcie_intc1 0x1>, + <0x0 0x0 0x0 0x2 &pcie_intc1 0x2>, + <0x0 0x0 0x0 0x3 &pcie_intc1 0x3>, + <0x0 0x0 0x0 0x4 &pcie_intc1 0x4>; + msi-controller; + device_type = "pci"; + starfive,stg-syscon = <&stg_syscon>; + bus-range = <0x0 0xff>; + clocks = <&syscrg JH7110_SYSCLK_NOC_BUS_STG_AXI>, + <&stgcrg JH7110_STGCLK_PCIE1_TL>, + <&stgcrg JH7110_STGCLK_PCIE1_AXI_MST0>, + <&stgcrg JH7110_STGCLK_PCIE1_APB>; + clock-names = "noc", "tl", "axi_mst0", "apb"; + resets = <&stgcrg JH7110_STGRST_PCIE1_AXI_MST0>, + <&stgcrg JH7110_STGRST_PCIE1_AXI_SLV0>, + <&stgcrg JH7110_STGRST_PCIE1_AXI_SLV>, + <&stgcrg JH7110_STGRST_PCIE1_BRG>, + <&stgcrg JH7110_STGRST_PCIE1_CORE>, + <&stgcrg JH7110_STGRST_PCIE1_APB>; + reset-names = "mst0", "slv0", "slv", "brg", + "core", "apb"; + status = "disabled"; + + pcie_intc1: interrupt-controller { + #address-cells = <0>; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; }; }; diff --git a/arch/riscv/boot/dts/thead/th1520.dtsi b/arch/riscv/boot/dts/thead/th1520.dtsi index d2fa25839012..3c9974062c20 100644 --- a/arch/riscv/boot/dts/thead/th1520.dtsi +++ b/arch/riscv/boot/dts/thead/th1520.dtsi @@ -122,6 +122,87 @@ }; }; + pmu { + compatible = "riscv,pmu"; + riscv,event-to-mhpmcounters = + <0x00003 0x00003 0x0007fff8>, + <0x00004 0x00004 0x0007fff8>, + <0x00005 0x00005 0x0007fff8>, + <0x00006 0x00006 0x0007fff8>, + <0x00007 0x00007 0x0007fff8>, + <0x00008 0x00008 0x0007fff8>, + <0x00009 0x00009 0x0007fff8>, + <0x0000a 0x0000a 0x0007fff8>, + <0x10000 0x10000 0x0007fff8>, + <0x10001 0x10001 0x0007fff8>, + <0x10002 0x10002 0x0007fff8>, + <0x10003 0x10003 0x0007fff8>, + <0x10010 0x10010 0x0007fff8>, + <0x10011 0x10011 0x0007fff8>, + <0x10012 0x10012 0x0007fff8>, + <0x10013 0x10013 0x0007fff8>; + riscv,event-to-mhpmevent = + <0x00003 0x00000000 0x00000001>, + <0x00004 0x00000000 0x00000002>, + <0x00006 0x00000000 0x00000006>, + <0x00005 0x00000000 0x00000007>, + <0x00007 0x00000000 0x00000008>, + <0x00008 0x00000000 0x00000009>, + <0x00009 0x00000000 0x0000000a>, + <0x0000a 0x00000000 0x0000000b>, + <0x10000 0x00000000 0x0000000c>, + <0x10001 0x00000000 0x0000000d>, + <0x10002 0x00000000 0x0000000e>, + <0x10003 0x00000000 0x0000000f>, + <0x10010 0x00000000 0x00000010>, + <0x10011 0x00000000 0x00000011>, + <0x10012 0x00000000 0x00000012>, + <0x10013 0x00000000 0x00000013>; + riscv,raw-event-to-mhpmcounters = + <0x00000000 0x00000001 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x00000002 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x00000003 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x00000004 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x00000005 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x00000006 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x00000007 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x00000008 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x00000009 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x0000000a 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x0000000b 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x0000000c 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x0000000d 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x0000000e 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x0000000f 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x00000010 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x00000011 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x00000012 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x00000013 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x00000014 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x00000015 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x00000016 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x00000017 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x00000018 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x00000019 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x0000001a 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x0000001b 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x0000001c 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x0000001d 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x0000001e 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x0000001f 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x00000020 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x00000021 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x00000022 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x00000023 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x00000024 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x00000025 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x00000026 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x00000027 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x00000028 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x00000029 0xffffffff 0xffffffff 0x0007fff8>, + <0x00000000 0x0000002a 0xffffffff 0xffffffff 0x0007fff8>; + }; + osc: oscillator { compatible = "fixed-clock"; clock-output-names = "osc_24m"; diff --git a/arch/riscv/configs/defconfig b/arch/riscv/configs/defconfig index 12dc8c73a8ac..3f1f055866af 100644 --- a/arch/riscv/configs/defconfig +++ b/arch/riscv/configs/defconfig @@ -110,8 +110,10 @@ CONFIG_PCIEPORTBUS=y CONFIG_PCI_HOST_GENERIC=y CONFIG_PCIE_XILINX=y CONFIG_PCIE_FU740=y +CONFIG_PCIE_STARFIVE_HOST=m CONFIG_DEVTMPFS=y CONFIG_DEVTMPFS_MOUNT=y +CONFIG_SIFIVE_CCACHE=y CONFIG_MTD=y CONFIG_MTD_BLOCK=y CONFIG_MTD_CFI=y @@ -143,6 +145,7 @@ CONFIG_RAVB=y CONFIG_STMMAC_ETH=m CONFIG_MICREL_PHY=y CONFIG_MICROSEMI_PHY=y +CONFIG_MOTORCOMM_PHY=y CONFIG_CAN_RCAR_CANFD=m CONFIG_INPUT_MOUSEDEV=y CONFIG_KEYBOARD_SUN4I_LRADC=m @@ -155,24 +158,35 @@ CONFIG_SERIAL_EARLYCON_RISCV_SBI=y CONFIG_VIRTIO_CONSOLE=y CONFIG_HW_RANDOM=y CONFIG_HW_RANDOM_VIRTIO=y +CONFIG_HW_RANDOM_JH7110=m +CONFIG_I2C=y CONFIG_I2C_CHARDEV=m +CONFIG_I2C_DESIGNWARE_PLATFORM=y CONFIG_I2C_MV64XXX=m CONFIG_I2C_RIIC=y CONFIG_SPI=y +CONFIG_SPI_CADENCE_QUADSPI=m +CONFIG_SPI_PL022=m CONFIG_SPI_RSPI=m CONFIG_SPI_SIFIVE=y CONFIG_SPI_SUN6I=y # CONFIG_PTP_1588_CLOCK is not set CONFIG_GPIO_SIFIVE=y +CONFIG_POWER_RESET_GPIO_RESTART=y +CONFIG_SENSORS_SFCTEMP=m CONFIG_CPU_THERMAL=y CONFIG_DEVFREQ_THERMAL=y CONFIG_RZG2L_THERMAL=y CONFIG_WATCHDOG=y CONFIG_SUNXI_WATCHDOG=y CONFIG_RENESAS_RZG2LWDT=y +CONFIG_MFD_AXP20X_I2C=y CONFIG_REGULATOR=y CONFIG_REGULATOR_FIXED_VOLTAGE=y +CONFIG_REGULATOR_AXP20X=y CONFIG_REGULATOR_GPIO=y +CONFIG_MEDIA_SUPPORT=m +CONFIG_VIDEO_CADENCE_CSI2RX=m CONFIG_DRM=m CONFIG_DRM_RADEON=m CONFIG_DRM_NOUVEAU=m @@ -184,6 +198,10 @@ CONFIG_SOUND=y CONFIG_SND=y CONFIG_SND_SOC=y CONFIG_SND_SOC_RZ=m +CONFIG_SND_DESIGNWARE_I2S=m +CONFIG_SND_SOC_STARFIVE=m +CONFIG_SND_SOC_JH7110_PWMDAC=m +CONFIG_SND_SOC_JH7110_TDM=m CONFIG_SND_SOC_WM8978=m CONFIG_SND_SIMPLE_CARD=m CONFIG_USB=y @@ -197,6 +215,11 @@ CONFIG_USB_OHCI_HCD_PLATFORM=y CONFIG_USB_RENESAS_USBHS=m CONFIG_USB_STORAGE=y CONFIG_USB_UAS=y +CONFIG_USB_CDNS_SUPPORT=m +CONFIG_USB_CDNS3=m +CONFIG_USB_CDNS3_GADGET=y +CONFIG_USB_CDNS3_HOST=y +CONFIG_USB_CDNS3_STARFIVE=m CONFIG_USB_MUSB_HDRC=m CONFIG_USB_MUSB_SUNXI=m CONFIG_NOP_USB_XCEIV=m @@ -246,6 +269,9 @@ CONFIG_RZG2L_ADC=m CONFIG_RESET_RZG2L_USBPHY_CTRL=y CONFIG_PHY_SUN4I_USB=m CONFIG_PHY_RCAR_GEN3_USB2=y +CONFIG_PHY_STARFIVE_JH7110_DPHY_RX=m +CONFIG_PHY_STARFIVE_JH7110_PCIE=m +CONFIG_PHY_STARFIVE_JH7110_USB=m CONFIG_LIBNVDIMM=y CONFIG_NVMEM_SUNXI_SID=y CONFIG_EXT4_FS=y diff --git a/arch/riscv/include/asm/Kbuild b/arch/riscv/include/asm/Kbuild index 504f8b7e72d4..5c589770f2a8 100644 --- a/arch/riscv/include/asm/Kbuild +++ b/arch/riscv/include/asm/Kbuild @@ -1,4 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 +syscall-y += syscall_table_32.h +syscall-y += syscall_table_64.h + generic-y += early_ioremap.h generic-y += flat.h generic-y += kvm_para.h diff --git a/arch/riscv/include/asm/arch_hweight.h b/arch/riscv/include/asm/arch_hweight.h index 85b2c443823e..613769b9cdc9 100644 --- a/arch/riscv/include/asm/arch_hweight.h +++ b/arch/riscv/include/asm/arch_hweight.h @@ -26,9 +26,9 @@ static __always_inline unsigned int __arch_hweight32(unsigned int w) asm (".option push\n" ".option arch,+zbb\n" - CPOPW "%0, %0\n" + CPOPW "%0, %1\n" ".option pop\n" - : "+r" (w) : :); + : "=r" (w) : "r" (w) :); return w; @@ -57,9 +57,9 @@ static __always_inline unsigned long __arch_hweight64(__u64 w) asm (".option push\n" ".option arch,+zbb\n" - "cpop %0, %0\n" + "cpop %0, %1\n" ".option pop\n" - : "+r" (w) : :); + : "=r" (w) : "r" (w) :); return w; diff --git a/arch/riscv/include/asm/barrier.h b/arch/riscv/include/asm/barrier.h index 880b56d8480d..e1d9bf1deca6 100644 --- a/arch/riscv/include/asm/barrier.h +++ b/arch/riscv/include/asm/barrier.h @@ -11,6 +11,7 @@ #define _ASM_RISCV_BARRIER_H #ifndef __ASSEMBLY__ +#include <asm/cmpxchg.h> #include <asm/fence.h> #define nop() __asm__ __volatile__ ("nop") @@ -28,21 +29,6 @@ #define __smp_rmb() RISCV_FENCE(r, r) #define __smp_wmb() RISCV_FENCE(w, w) -#define __smp_store_release(p, v) \ -do { \ - compiletime_assert_atomic_type(*p); \ - RISCV_FENCE(rw, w); \ - WRITE_ONCE(*p, v); \ -} while (0) - -#define __smp_load_acquire(p) \ -({ \ - typeof(*p) ___p1 = READ_ONCE(*p); \ - compiletime_assert_atomic_type(*p); \ - RISCV_FENCE(r, rw); \ - ___p1; \ -}) - /* * This is a very specific barrier: it's currently only used in two places in * the kernel, both in the scheduler. See include/linux/spinlock.h for the two @@ -70,6 +56,35 @@ do { \ */ #define smp_mb__after_spinlock() RISCV_FENCE(iorw, iorw) +#define __smp_store_release(p, v) \ +do { \ + compiletime_assert_atomic_type(*p); \ + RISCV_FENCE(rw, w); \ + WRITE_ONCE(*p, v); \ +} while (0) + +#define __smp_load_acquire(p) \ +({ \ + typeof(*p) ___p1 = READ_ONCE(*p); \ + compiletime_assert_atomic_type(*p); \ + RISCV_FENCE(r, rw); \ + ___p1; \ +}) + +#ifdef CONFIG_RISCV_ISA_ZAWRS +#define smp_cond_load_relaxed(ptr, cond_expr) ({ \ + typeof(ptr) __PTR = (ptr); \ + __unqual_scalar_typeof(*ptr) VAL; \ + for (;;) { \ + VAL = READ_ONCE(*__PTR); \ + if (cond_expr) \ + break; \ + __cmpwait_relaxed(ptr, VAL); \ + } \ + (typeof(*ptr))VAL; \ +}) +#endif + #include <asm-generic/barrier.h> #endif /* __ASSEMBLY__ */ diff --git a/arch/riscv/include/asm/cmpxchg.h b/arch/riscv/include/asm/cmpxchg.h index 808b4c78462e..ebbce134917c 100644 --- a/arch/riscv/include/asm/cmpxchg.h +++ b/arch/riscv/include/asm/cmpxchg.h @@ -8,7 +8,10 @@ #include <linux/bug.h> +#include <asm/alternative-macros.h> #include <asm/fence.h> +#include <asm/hwcap.h> +#include <asm/insn-def.h> #define __arch_xchg_masked(sc_sfx, prepend, append, r, p, n) \ ({ \ @@ -223,4 +226,59 @@ arch_cmpxchg_release((ptr), (o), (n)); \ }) +#ifdef CONFIG_RISCV_ISA_ZAWRS +/* + * Despite wrs.nto being "WRS-with-no-timeout", in the absence of changes to + * @val we expect it to still terminate within a "reasonable" amount of time + * for an implementation-specific other reason, a pending, locally-enabled + * interrupt, or because it has been configured to raise an illegal + * instruction exception. + */ +static __always_inline void __cmpwait(volatile void *ptr, + unsigned long val, + int size) +{ + unsigned long tmp; + + asm goto(ALTERNATIVE("j %l[no_zawrs]", "nop", + 0, RISCV_ISA_EXT_ZAWRS, 1) + : : : : no_zawrs); + + switch (size) { + case 4: + asm volatile( + " lr.w %0, %1\n" + " xor %0, %0, %2\n" + " bnez %0, 1f\n" + ZAWRS_WRS_NTO "\n" + "1:" + : "=&r" (tmp), "+A" (*(u32 *)ptr) + : "r" (val)); + break; +#if __riscv_xlen == 64 + case 8: + asm volatile( + " lr.d %0, %1\n" + " xor %0, %0, %2\n" + " bnez %0, 1f\n" + ZAWRS_WRS_NTO "\n" + "1:" + : "=&r" (tmp), "+A" (*(u64 *)ptr) + : "r" (val)); + break; +#endif + default: + BUILD_BUG(); + } + + return; + +no_zawrs: + asm volatile(RISCV_PAUSE : : : "memory"); +} + +#define __cmpwait_relaxed(ptr, val) \ + __cmpwait((ptr), (unsigned long)(val), sizeof(*(ptr))) +#endif + #endif /* _ASM_RISCV_CMPXCHG_H */ diff --git a/arch/riscv/include/asm/cpufeature.h b/arch/riscv/include/asm/cpufeature.h index 347805446151..000796c2d0b1 100644 --- a/arch/riscv/include/asm/cpufeature.h +++ b/arch/riscv/include/asm/cpufeature.h @@ -70,6 +70,7 @@ struct riscv_isa_ext_data { const char *property; const unsigned int *subset_ext_ids; const unsigned int subset_ext_size; + int (*validate)(const struct riscv_isa_ext_data *data, const unsigned long *isa_bitmap); }; extern const struct riscv_isa_ext_data riscv_isa_ext[]; diff --git a/arch/riscv/include/asm/dmi.h b/arch/riscv/include/asm/dmi.h new file mode 100644 index 000000000000..ca7cce557ef7 --- /dev/null +++ b/arch/riscv/include/asm/dmi.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2024 Intel Corporation + * + * based on arch/arm64/include/asm/dmi.h + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + */ + +#ifndef __ASM_DMI_H +#define __ASM_DMI_H + +#include <linux/io.h> +#include <linux/slab.h> + +#define dmi_early_remap(x, l) memremap(x, l, MEMREMAP_WB) +#define dmi_early_unmap(x, l) memunmap(x) +#define dmi_remap(x, l) memremap(x, l, MEMREMAP_WB) +#define dmi_unmap(x) memunmap(x) +#define dmi_alloc(l) kzalloc(l, GFP_KERNEL) + +#endif diff --git a/arch/riscv/include/asm/ftrace.h b/arch/riscv/include/asm/ftrace.h index 9eb31a7ea0aa..2cddd79ff21b 100644 --- a/arch/riscv/include/asm/ftrace.h +++ b/arch/riscv/include/asm/ftrace.h @@ -11,7 +11,6 @@ #if defined(CONFIG_FUNCTION_GRAPH_TRACER) && defined(CONFIG_FRAME_POINTER) #define HAVE_FUNCTION_GRAPH_FP_TEST #endif -#define HAVE_FUNCTION_GRAPH_RET_ADDR_PTR #define ARCH_SUPPORTS_FTRACE_OPS 1 #ifndef __ASSEMBLY__ diff --git a/arch/riscv/include/asm/hugetlb.h b/arch/riscv/include/asm/hugetlb.h index b1ce97a9dbfc..faf3624d8057 100644 --- a/arch/riscv/include/asm/hugetlb.h +++ b/arch/riscv/include/asm/hugetlb.h @@ -44,7 +44,7 @@ int huge_ptep_set_access_flags(struct vm_area_struct *vma, pte_t pte, int dirty); #define __HAVE_ARCH_HUGE_PTEP_GET -pte_t huge_ptep_get(pte_t *ptep); +pte_t huge_ptep_get(struct mm_struct *mm, unsigned long addr, pte_t *ptep); pte_t arch_make_huge_pte(pte_t entry, unsigned int shift, vm_flags_t flags); #define arch_make_huge_pte arch_make_huge_pte diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index e17d0078a651..b18b202ca141 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -81,6 +81,18 @@ #define RISCV_ISA_EXT_ZTSO 72 #define RISCV_ISA_EXT_ZACAS 73 #define RISCV_ISA_EXT_XANDESPMU 74 +#define RISCV_ISA_EXT_ZVE32X 75 +#define RISCV_ISA_EXT_ZVE32F 76 +#define RISCV_ISA_EXT_ZVE64X 77 +#define RISCV_ISA_EXT_ZVE64F 78 +#define RISCV_ISA_EXT_ZVE64D 79 +#define RISCV_ISA_EXT_ZIMOP 80 +#define RISCV_ISA_EXT_ZCA 81 +#define RISCV_ISA_EXT_ZCB 82 +#define RISCV_ISA_EXT_ZCD 83 +#define RISCV_ISA_EXT_ZCF 84 +#define RISCV_ISA_EXT_ZCMOP 85 +#define RISCV_ISA_EXT_ZAWRS 86 #define RISCV_ISA_EXT_XLINUXENVCFG 127 diff --git a/arch/riscv/include/asm/hwprobe.h b/arch/riscv/include/asm/hwprobe.h index 630507dff5ea..150a9877b0af 100644 --- a/arch/riscv/include/asm/hwprobe.h +++ b/arch/riscv/include/asm/hwprobe.h @@ -8,7 +8,7 @@ #include <uapi/asm/hwprobe.h> -#define RISCV_HWPROBE_MAX_KEY 6 +#define RISCV_HWPROBE_MAX_KEY 7 static inline bool riscv_hwprobe_key_is_valid(__s64 key) { diff --git a/arch/riscv/include/asm/insn-def.h b/arch/riscv/include/asm/insn-def.h index e27179b26086..9a913010cdd9 100644 --- a/arch/riscv/include/asm/insn-def.h +++ b/arch/riscv/include/asm/insn-def.h @@ -196,4 +196,8 @@ INSN_I(OPCODE_MISC_MEM, FUNC3(2), __RD(0), \ RS1(base), SIMM12(4)) +#define RISCV_PAUSE ".4byte 0x100000f" +#define ZAWRS_WRS_NTO ".4byte 0x00d00073" +#define ZAWRS_WRS_STO ".4byte 0x01d00073" + #endif /* __ASM_INSN_DEF_H */ diff --git a/arch/riscv/include/asm/jump_label.h b/arch/riscv/include/asm/jump_label.h index 4a35d787c019..1c768d02bd0c 100644 --- a/arch/riscv/include/asm/jump_label.h +++ b/arch/riscv/include/asm/jump_label.h @@ -12,6 +12,8 @@ #include <linux/types.h> #include <asm/asm.h> +#define HAVE_JUMP_LABEL_BATCH + #define JUMP_LABEL_NOP_SIZE 4 static __always_inline bool arch_static_branch(struct static_key * const key, @@ -44,7 +46,7 @@ static __always_inline bool arch_static_branch_jump(struct static_key * const ke " .option push \n\t" " .option norelax \n\t" " .option norvc \n\t" - "1: jal zero, %l[label] \n\t" + "1: j %l[label] \n\t" " .option pop \n\t" " .pushsection __jump_table, \"aw\" \n\t" " .align " RISCV_LGPTR " \n\t" diff --git a/arch/riscv/include/asm/kasan.h b/arch/riscv/include/asm/kasan.h index 0b85e363e778..e6a0071bdb56 100644 --- a/arch/riscv/include/asm/kasan.h +++ b/arch/riscv/include/asm/kasan.h @@ -6,8 +6,6 @@ #ifndef __ASSEMBLY__ -#ifdef CONFIG_KASAN - /* * The following comment was copied from arm64: * KASAN_SHADOW_START: beginning of the kernel virtual addresses. @@ -34,6 +32,8 @@ */ #define KASAN_SHADOW_START ((KASAN_SHADOW_END - KASAN_SHADOW_SIZE) & PGDIR_MASK) #define KASAN_SHADOW_END MODULES_LOWEST_VADDR + +#ifdef CONFIG_KASAN #define KASAN_SHADOW_OFFSET _AC(CONFIG_KASAN_SHADOW_OFFSET, UL) void kasan_init(void); diff --git a/arch/riscv/include/asm/kvm_aia_aplic.h b/arch/riscv/include/asm/kvm_aia_aplic.h deleted file mode 100644 index 6dd1a4809ec1..000000000000 --- a/arch/riscv/include/asm/kvm_aia_aplic.h +++ /dev/null @@ -1,58 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2021 Western Digital Corporation or its affiliates. - * Copyright (C) 2022 Ventana Micro Systems Inc. - */ -#ifndef __KVM_RISCV_AIA_IMSIC_H -#define __KVM_RISCV_AIA_IMSIC_H - -#include <linux/bitops.h> - -#define APLIC_MAX_IDC BIT(14) -#define APLIC_MAX_SOURCE 1024 - -#define APLIC_DOMAINCFG 0x0000 -#define APLIC_DOMAINCFG_RDONLY 0x80000000 -#define APLIC_DOMAINCFG_IE BIT(8) -#define APLIC_DOMAINCFG_DM BIT(2) -#define APLIC_DOMAINCFG_BE BIT(0) - -#define APLIC_SOURCECFG_BASE 0x0004 -#define APLIC_SOURCECFG_D BIT(10) -#define APLIC_SOURCECFG_CHILDIDX_MASK 0x000003ff -#define APLIC_SOURCECFG_SM_MASK 0x00000007 -#define APLIC_SOURCECFG_SM_INACTIVE 0x0 -#define APLIC_SOURCECFG_SM_DETACH 0x1 -#define APLIC_SOURCECFG_SM_EDGE_RISE 0x4 -#define APLIC_SOURCECFG_SM_EDGE_FALL 0x5 -#define APLIC_SOURCECFG_SM_LEVEL_HIGH 0x6 -#define APLIC_SOURCECFG_SM_LEVEL_LOW 0x7 - -#define APLIC_IRQBITS_PER_REG 32 - -#define APLIC_SETIP_BASE 0x1c00 -#define APLIC_SETIPNUM 0x1cdc - -#define APLIC_CLRIP_BASE 0x1d00 -#define APLIC_CLRIPNUM 0x1ddc - -#define APLIC_SETIE_BASE 0x1e00 -#define APLIC_SETIENUM 0x1edc - -#define APLIC_CLRIE_BASE 0x1f00 -#define APLIC_CLRIENUM 0x1fdc - -#define APLIC_SETIPNUM_LE 0x2000 -#define APLIC_SETIPNUM_BE 0x2004 - -#define APLIC_GENMSI 0x3000 - -#define APLIC_TARGET_BASE 0x3004 -#define APLIC_TARGET_HART_IDX_SHIFT 18 -#define APLIC_TARGET_HART_IDX_MASK 0x3fff -#define APLIC_TARGET_GUEST_IDX_SHIFT 12 -#define APLIC_TARGET_GUEST_IDX_MASK 0x3f -#define APLIC_TARGET_IPRIO_MASK 0xff -#define APLIC_TARGET_EIID_MASK 0x7ff - -#endif diff --git a/arch/riscv/include/asm/kvm_aia_imsic.h b/arch/riscv/include/asm/kvm_aia_imsic.h deleted file mode 100644 index da5881d2bde0..000000000000 --- a/arch/riscv/include/asm/kvm_aia_imsic.h +++ /dev/null @@ -1,38 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2021 Western Digital Corporation or its affiliates. - * Copyright (C) 2022 Ventana Micro Systems Inc. - */ -#ifndef __KVM_RISCV_AIA_IMSIC_H -#define __KVM_RISCV_AIA_IMSIC_H - -#include <linux/types.h> -#include <asm/csr.h> - -#define IMSIC_MMIO_PAGE_SHIFT 12 -#define IMSIC_MMIO_PAGE_SZ (1UL << IMSIC_MMIO_PAGE_SHIFT) -#define IMSIC_MMIO_PAGE_LE 0x00 -#define IMSIC_MMIO_PAGE_BE 0x04 - -#define IMSIC_MIN_ID 63 -#define IMSIC_MAX_ID 2048 - -#define IMSIC_EIDELIVERY 0x70 - -#define IMSIC_EITHRESHOLD 0x72 - -#define IMSIC_EIP0 0x80 -#define IMSIC_EIP63 0xbf -#define IMSIC_EIPx_BITS 32 - -#define IMSIC_EIE0 0xc0 -#define IMSIC_EIE63 0xff -#define IMSIC_EIEx_BITS 32 - -#define IMSIC_FIRST IMSIC_EIDELIVERY -#define IMSIC_LAST IMSIC_EIE63 - -#define IMSIC_MMIO_SETIPNUM_LE 0x00 -#define IMSIC_MMIO_SETIPNUM_BE 0x04 - -#endif diff --git a/arch/riscv/include/asm/kvm_host.h b/arch/riscv/include/asm/kvm_host.h index d96281278586..2e2254fd2a2a 100644 --- a/arch/riscv/include/asm/kvm_host.h +++ b/arch/riscv/include/asm/kvm_host.h @@ -80,6 +80,7 @@ struct kvm_vcpu_stat { struct kvm_vcpu_stat_generic generic; u64 ecall_exit_stat; u64 wfi_exit_stat; + u64 wrs_exit_stat; u64 mmio_exit_user; u64 mmio_exit_kernel; u64 csr_exit_user; @@ -286,7 +287,6 @@ struct kvm_vcpu_arch { }; static inline void kvm_arch_sync_events(struct kvm *kvm) {} -static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {} #define KVM_RISCV_GSTAGE_TLB_MIN_ORDER 12 diff --git a/arch/riscv/include/asm/mmu.h b/arch/riscv/include/asm/mmu.h index 947fd60f9051..c9e03e9da3dc 100644 --- a/arch/riscv/include/asm/mmu.h +++ b/arch/riscv/include/asm/mmu.h @@ -31,8 +31,8 @@ typedef struct { #define cntx2asid(cntx) ((cntx) & SATP_ASID_MASK) #define cntx2version(cntx) ((cntx) & ~SATP_ASID_MASK) -void __init create_pgd_mapping(pgd_t *pgdp, uintptr_t va, phys_addr_t pa, - phys_addr_t sz, pgprot_t prot); +void __meminit create_pgd_mapping(pgd_t *pgdp, uintptr_t va, phys_addr_t pa, phys_addr_t sz, + pgprot_t prot); #endif /* __ASSEMBLY__ */ #endif /* _ASM_RISCV_MMU_H */ diff --git a/arch/riscv/include/asm/page.h b/arch/riscv/include/asm/page.h index 115ac98b8d72..235fd45d998d 100644 --- a/arch/riscv/include/asm/page.h +++ b/arch/riscv/include/asm/page.h @@ -188,6 +188,11 @@ extern phys_addr_t __phys_addr_symbol(unsigned long x); unsigned long kaslr_offset(void); +static __always_inline void *pfn_to_kaddr(unsigned long pfn) +{ + return __va(pfn << PAGE_SHIFT); +} + #endif /* __ASSEMBLY__ */ #define virt_addr_valid(vaddr) ({ \ diff --git a/arch/riscv/include/asm/patch.h b/arch/riscv/include/asm/patch.h index 9f5d6e14c405..7228e266b9a1 100644 --- a/arch/riscv/include/asm/patch.h +++ b/arch/riscv/include/asm/patch.h @@ -9,7 +9,7 @@ int patch_insn_write(void *addr, const void *insn, size_t len); int patch_text_nosync(void *addr, const void *insns, size_t len); int patch_text_set_nosync(void *addr, u8 c, size_t len); -int patch_text(void *addr, u32 *insns, int ninsns); +int patch_text(void *addr, u32 *insns, size_t len); extern int riscv_patch_in_stop_machine; diff --git a/arch/riscv/include/asm/pgtable-64.h b/arch/riscv/include/asm/pgtable-64.h index 8c36a8818432..0897dd99ab8d 100644 --- a/arch/riscv/include/asm/pgtable-64.h +++ b/arch/riscv/include/asm/pgtable-64.h @@ -398,4 +398,24 @@ static inline struct page *pgd_page(pgd_t pgd) #define p4d_offset p4d_offset p4d_t *p4d_offset(pgd_t *pgd, unsigned long address); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static inline int pte_devmap(pte_t pte); +static inline pte_t pmd_pte(pmd_t pmd); + +static inline int pmd_devmap(pmd_t pmd) +{ + return pte_devmap(pmd_pte(pmd)); +} + +static inline int pud_devmap(pud_t pud) +{ + return 0; +} + +static inline int pgd_devmap(pgd_t pgd) +{ + return 0; +} +#endif + #endif /* _ASM_RISCV_PGTABLE_64_H */ diff --git a/arch/riscv/include/asm/pgtable-bits.h b/arch/riscv/include/asm/pgtable-bits.h index 179bd4afece4..a8f5205cea54 100644 --- a/arch/riscv/include/asm/pgtable-bits.h +++ b/arch/riscv/include/asm/pgtable-bits.h @@ -19,6 +19,7 @@ #define _PAGE_SOFT (3 << 8) /* Reserved for software */ #define _PAGE_SPECIAL (1 << 8) /* RSW: 0x1 */ +#define _PAGE_DEVMAP (1 << 9) /* RSW, devmap */ #define _PAGE_TABLE _PAGE_PRESENT /* diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index aad8b8ca51f1..089f3c9f56a3 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -165,7 +165,7 @@ struct pt_alloc_ops { #endif }; -extern struct pt_alloc_ops pt_ops __initdata; +extern struct pt_alloc_ops pt_ops __meminitdata; #ifdef CONFIG_MMU /* Number of PGD entries that a user-mode program can use */ @@ -350,6 +350,19 @@ static inline int pte_present(pte_t pte) return (pte_val(pte) & (_PAGE_PRESENT | _PAGE_PROT_NONE)); } +#define pte_accessible pte_accessible +static inline unsigned long pte_accessible(struct mm_struct *mm, pte_t a) +{ + if (pte_val(a) & _PAGE_PRESENT) + return true; + + if ((pte_val(a) & _PAGE_PROT_NONE) && + atomic_read(&mm->tlb_flush_pending)) + return true; + + return false; +} + static inline int pte_none(pte_t pte) { return (pte_val(pte) == 0); @@ -390,6 +403,13 @@ static inline int pte_special(pte_t pte) return pte_val(pte) & _PAGE_SPECIAL; } +#ifdef CONFIG_ARCH_HAS_PTE_DEVMAP +static inline int pte_devmap(pte_t pte) +{ + return pte_val(pte) & _PAGE_DEVMAP; +} +#endif + /* static inline pte_t pte_rdprotect(pte_t pte) */ static inline pte_t pte_wrprotect(pte_t pte) @@ -431,6 +451,11 @@ static inline pte_t pte_mkspecial(pte_t pte) return __pte(pte_val(pte) | _PAGE_SPECIAL); } +static inline pte_t pte_mkdevmap(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_DEVMAP); +} + static inline pte_t pte_mkhuge(pte_t pte) { return pte; @@ -489,8 +514,8 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, #define update_mmu_cache(vma, addr, ptep) \ update_mmu_cache_range(NULL, vma, addr, ptep, 1) -#define __HAVE_ARCH_UPDATE_MMU_TLB -#define update_mmu_tlb update_mmu_cache +#define update_mmu_tlb_range(vma, addr, ptep, nr) \ + update_mmu_cache_range(NULL, vma, addr, ptep, nr) static inline void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp) @@ -721,6 +746,11 @@ static inline pmd_t pmd_mkdirty(pmd_t pmd) return pte_pmd(pte_mkdirty(pmd_pte(pmd))); } +static inline pmd_t pmd_mkdevmap(pmd_t pmd) +{ + return pte_pmd(pte_mkdevmap(pmd_pte(pmd))); +} + static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, pmd_t pmd) { diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h index 68c3432dc6ea..8702b8721a27 100644 --- a/arch/riscv/include/asm/processor.h +++ b/arch/riscv/include/asm/processor.h @@ -57,6 +57,12 @@ #define STACK_TOP DEFAULT_MAP_WINDOW +#ifdef CONFIG_MMU +#define user_max_virt_addr() arch_get_mmap_end(ULONG_MAX, 0, 0) +#else +#define user_max_virt_addr() 0 +#endif /* CONFIG_MMU */ + /* * This decides where the kernel will search for a free chunk of vm * space during mmap's. diff --git a/arch/riscv/include/asm/sbi.h b/arch/riscv/include/asm/sbi.h index 1079e214fe85..7cffd4ffecd0 100644 --- a/arch/riscv/include/asm/sbi.h +++ b/arch/riscv/include/asm/sbi.h @@ -304,10 +304,12 @@ struct sbiret { }; void sbi_init(void); -struct sbiret sbi_ecall(int ext, int fid, unsigned long arg0, - unsigned long arg1, unsigned long arg2, - unsigned long arg3, unsigned long arg4, - unsigned long arg5); +struct sbiret __sbi_ecall(unsigned long arg0, unsigned long arg1, + unsigned long arg2, unsigned long arg3, + unsigned long arg4, unsigned long arg5, + int fid, int ext); +#define sbi_ecall(e, f, a0, a1, a2, a3, a4, a5) \ + __sbi_ecall(a0, a1, a2, a3, a4, a5, f, e) #ifdef CONFIG_RISCV_SBI_V01 void sbi_console_putchar(int ch); diff --git a/arch/riscv/include/asm/syscall_table.h b/arch/riscv/include/asm/syscall_table.h new file mode 100644 index 000000000000..0c2d61782813 --- /dev/null +++ b/arch/riscv/include/asm/syscall_table.h @@ -0,0 +1,7 @@ +#include <asm/bitsperlong.h> + +#if __BITS_PER_LONG == 64 +#include <asm/syscall_table_64.h> +#else +#include <asm/syscall_table_32.h> +#endif diff --git a/arch/riscv/include/asm/trace.h b/arch/riscv/include/asm/trace.h new file mode 100644 index 000000000000..6151cee5450c --- /dev/null +++ b/arch/riscv/include/asm/trace.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM riscv + +#if !defined(_TRACE_RISCV_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_RISCV_H + +#include <linux/tracepoint.h> + +TRACE_EVENT_CONDITION(sbi_call, + TP_PROTO(int ext, int fid), + TP_ARGS(ext, fid), + TP_CONDITION(ext != SBI_EXT_HSM), + + TP_STRUCT__entry( + __field(int, ext) + __field(int, fid) + ), + + TP_fast_assign( + __entry->ext = ext; + __entry->fid = fid; + ), + + TP_printk("ext=0x%x fid=%d", __entry->ext, __entry->fid) +); + +TRACE_EVENT_CONDITION(sbi_return, + TP_PROTO(int ext, long error, long value), + TP_ARGS(ext, error, value), + TP_CONDITION(ext != SBI_EXT_HSM), + + TP_STRUCT__entry( + __field(long, error) + __field(long, value) + ), + + TP_fast_assign( + __entry->error = error; + __entry->value = value; + ), + + TP_printk("error=%ld value=0x%lx", __entry->error, __entry->value) +); + +#endif /* _TRACE_RISCV_H */ + +#undef TRACE_INCLUDE_PATH +#undef TRACE_INCLUDE_FILE + +#define TRACE_INCLUDE_PATH asm +#define TRACE_INCLUDE_FILE trace + +#include <trace/define_trace.h> diff --git a/arch/riscv/include/asm/unistd.h b/arch/riscv/include/asm/unistd.h index 221630bdbd07..e6d904fa67c5 100644 --- a/arch/riscv/include/asm/unistd.h +++ b/arch/riscv/include/asm/unistd.h @@ -3,11 +3,6 @@ * Copyright (C) 2012 Regents of the University of California */ -/* - * There is explicitly no include guard here because this file is expected to - * be included multiple times. - */ - #define __ARCH_WANT_SYS_CLONE #ifdef CONFIG_COMPAT @@ -21,6 +16,14 @@ #define __ARCH_WANT_COMPAT_FADVISE64_64 #endif +#if defined(__LP64__) && !defined(__SYSCALL_COMPAT) +#define __ARCH_WANT_NEW_STAT +#define __ARCH_WANT_SET_GET_RLIMIT +#endif /* __LP64__ */ + +#define __ARCH_WANT_MEMFD_SECRET + + #include <uapi/asm/unistd.h> #define NR_syscalls (__NR_syscalls) diff --git a/arch/riscv/include/asm/vdso/processor.h b/arch/riscv/include/asm/vdso/processor.h index 96b65a5396df..8f383f05a290 100644 --- a/arch/riscv/include/asm/vdso/processor.h +++ b/arch/riscv/include/asm/vdso/processor.h @@ -5,6 +5,7 @@ #ifndef __ASSEMBLY__ #include <asm/barrier.h> +#include <asm/insn-def.h> static inline void cpu_relax(void) { @@ -14,16 +15,11 @@ static inline void cpu_relax(void) __asm__ __volatile__ ("div %0, %0, zero" : "=r" (dummy)); #endif -#ifdef CONFIG_TOOLCHAIN_HAS_ZIHINTPAUSE /* * Reduce instruction retirement. * This assumes the PC changes. */ - __asm__ __volatile__ ("pause"); -#else - /* Encoding of the pause instruction */ - __asm__ __volatile__ (".4byte 0x100000F"); -#endif + __asm__ __volatile__ (RISCV_PAUSE); barrier(); } diff --git a/arch/riscv/include/asm/vector.h b/arch/riscv/include/asm/vector.h index 731dcd0ed4de..be7d309cca8a 100644 --- a/arch/riscv/include/asm/vector.h +++ b/arch/riscv/include/asm/vector.h @@ -37,7 +37,7 @@ static inline u32 riscv_v_flags(void) static __always_inline bool has_vector(void) { - return riscv_has_extension_unlikely(RISCV_ISA_EXT_v); + return riscv_has_extension_unlikely(RISCV_ISA_EXT_ZVE32X); } static inline void __riscv_v_vstate_clean(struct pt_regs *regs) @@ -91,7 +91,7 @@ static __always_inline void __vstate_csr_restore(struct __riscv_v_ext_state *src { asm volatile ( ".option push\n\t" - ".option arch, +v\n\t" + ".option arch, +zve32x\n\t" "vsetvl x0, %2, %1\n\t" ".option pop\n\t" "csrw " __stringify(CSR_VSTART) ", %0\n\t" @@ -109,7 +109,7 @@ static inline void __riscv_v_vstate_save(struct __riscv_v_ext_state *save_to, __vstate_csr_save(save_to); asm volatile ( ".option push\n\t" - ".option arch, +v\n\t" + ".option arch, +zve32x\n\t" "vsetvli %0, x0, e8, m8, ta, ma\n\t" "vse8.v v0, (%1)\n\t" "add %1, %1, %0\n\t" @@ -131,7 +131,7 @@ static inline void __riscv_v_vstate_restore(struct __riscv_v_ext_state *restore_ riscv_v_enable(); asm volatile ( ".option push\n\t" - ".option arch, +v\n\t" + ".option arch, +zve32x\n\t" "vsetvli %0, x0, e8, m8, ta, ma\n\t" "vle8.v v0, (%1)\n\t" "add %1, %1, %0\n\t" @@ -153,7 +153,7 @@ static inline void __riscv_v_vstate_discard(void) riscv_v_enable(); asm volatile ( ".option push\n\t" - ".option arch, +v\n\t" + ".option arch, +zve32x\n\t" "vsetvli %0, x0, e8, m8, ta, ma\n\t" "vmv.v.i v0, -1\n\t" "vmv.v.i v8, -1\n\t" diff --git a/arch/riscv/include/uapi/asm/Kbuild b/arch/riscv/include/uapi/asm/Kbuild index f66554cd5c45..89ac01faa5ae 100644 --- a/arch/riscv/include/uapi/asm/Kbuild +++ b/arch/riscv/include/uapi/asm/Kbuild @@ -1 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 +syscall-y += unistd_32.h +syscall-y += unistd_64.h diff --git a/arch/riscv/include/uapi/asm/hwprobe.h b/arch/riscv/include/uapi/asm/hwprobe.h index dda76a05420b..8b8f6ac0eae2 100644 --- a/arch/riscv/include/uapi/asm/hwprobe.h +++ b/arch/riscv/include/uapi/asm/hwprobe.h @@ -60,6 +60,18 @@ struct riscv_hwprobe { #define RISCV_HWPROBE_EXT_ZACAS (1ULL << 34) #define RISCV_HWPROBE_EXT_ZICOND (1ULL << 35) #define RISCV_HWPROBE_EXT_ZIHINTPAUSE (1ULL << 36) +#define RISCV_HWPROBE_EXT_ZVE32X (1ULL << 37) +#define RISCV_HWPROBE_EXT_ZVE32F (1ULL << 38) +#define RISCV_HWPROBE_EXT_ZVE64X (1ULL << 39) +#define RISCV_HWPROBE_EXT_ZVE64F (1ULL << 40) +#define RISCV_HWPROBE_EXT_ZVE64D (1ULL << 41) +#define RISCV_HWPROBE_EXT_ZIMOP (1ULL << 42) +#define RISCV_HWPROBE_EXT_ZCA (1ULL << 43) +#define RISCV_HWPROBE_EXT_ZCB (1ULL << 44) +#define RISCV_HWPROBE_EXT_ZCD (1ULL << 45) +#define RISCV_HWPROBE_EXT_ZCF (1ULL << 46) +#define RISCV_HWPROBE_EXT_ZCMOP (1ULL << 47) +#define RISCV_HWPROBE_EXT_ZAWRS (1ULL << 48) #define RISCV_HWPROBE_KEY_CPUPERF_0 5 #define RISCV_HWPROBE_MISALIGNED_UNKNOWN (0 << 0) #define RISCV_HWPROBE_MISALIGNED_EMULATED (1 << 0) @@ -68,6 +80,7 @@ struct riscv_hwprobe { #define RISCV_HWPROBE_MISALIGNED_UNSUPPORTED (4 << 0) #define RISCV_HWPROBE_MISALIGNED_MASK (7 << 0) #define RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE 6 +#define RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS 7 /* Increase RISCV_HWPROBE_MAX_KEY when adding items. */ /* Flags */ diff --git a/arch/riscv/include/uapi/asm/kvm.h b/arch/riscv/include/uapi/asm/kvm.h index e878e7cc3978..e97db3296456 100644 --- a/arch/riscv/include/uapi/asm/kvm.h +++ b/arch/riscv/include/uapi/asm/kvm.h @@ -168,6 +168,13 @@ enum KVM_RISCV_ISA_EXT_ID { KVM_RISCV_ISA_EXT_ZTSO, KVM_RISCV_ISA_EXT_ZACAS, KVM_RISCV_ISA_EXT_SSCOFPMF, + KVM_RISCV_ISA_EXT_ZIMOP, + KVM_RISCV_ISA_EXT_ZCA, + KVM_RISCV_ISA_EXT_ZCB, + KVM_RISCV_ISA_EXT_ZCD, + KVM_RISCV_ISA_EXT_ZCF, + KVM_RISCV_ISA_EXT_ZCMOP, + KVM_RISCV_ISA_EXT_ZAWRS, KVM_RISCV_ISA_EXT_MAX, }; diff --git a/arch/riscv/include/uapi/asm/unistd.h b/arch/riscv/include/uapi/asm/unistd.h index 950ab3fd4409..81896bbbf727 100644 --- a/arch/riscv/include/uapi/asm/unistd.h +++ b/arch/riscv/include/uapi/asm/unistd.h @@ -14,41 +14,10 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see <https://www.gnu.org/licenses/>. */ +#include <asm/bitsperlong.h> -#if defined(__LP64__) && !defined(__SYSCALL_COMPAT) -#define __ARCH_WANT_NEW_STAT -#define __ARCH_WANT_SET_GET_RLIMIT -#endif /* __LP64__ */ - -#define __ARCH_WANT_SYS_CLONE3 -#define __ARCH_WANT_MEMFD_SECRET - -#include <asm-generic/unistd.h> - -/* - * Allows the instruction cache to be flushed from userspace. Despite RISC-V - * having a direct 'fence.i' instruction available to userspace (which we - * can't trap!), that's not actually viable when running on Linux because the - * kernel might schedule a process on another hart. There is no way for - * userspace to handle this without invoking the kernel (as it doesn't know the - * thread->hart mappings), so we've defined a RISC-V specific system call to - * flush the instruction cache. - * - * __NR_riscv_flush_icache is defined to flush the instruction cache over an - * address range, with the flush applying to either all threads or just the - * caller. We don't currently do anything with the address range, that's just - * in there for forwards compatibility. - */ -#ifndef __NR_riscv_flush_icache -#define __NR_riscv_flush_icache (__NR_arch_specific_syscall + 15) -#endif -__SYSCALL(__NR_riscv_flush_icache, sys_riscv_flush_icache) - -/* - * Allows userspace to query the kernel for CPU architecture and - * microarchitecture details across a given set of CPUs. - */ -#ifndef __NR_riscv_hwprobe -#define __NR_riscv_hwprobe (__NR_arch_specific_syscall + 14) +#if __BITS_PER_LONG == 64 +#include <asm/unistd_64.h> +#else +#include <asm/unistd_32.h> #endif -__SYSCALL(__NR_riscv_hwprobe, sys_riscv_hwprobe) diff --git a/arch/riscv/kernel/Makefile.syscalls b/arch/riscv/kernel/Makefile.syscalls new file mode 100644 index 000000000000..52087a023b3d --- /dev/null +++ b/arch/riscv/kernel/Makefile.syscalls @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 + +syscall_abis_32 += riscv memfd_secret +syscall_abis_64 += riscv newstat rlimit memfd_secret diff --git a/arch/riscv/kernel/compat_syscall_table.c b/arch/riscv/kernel/compat_syscall_table.c index ad7f2d712f5f..e884c069e88f 100644 --- a/arch/riscv/kernel/compat_syscall_table.c +++ b/arch/riscv/kernel/compat_syscall_table.c @@ -8,9 +8,11 @@ #include <asm-generic/syscalls.h> #include <asm/syscall.h> +#define __SYSCALL_WITH_COMPAT(nr, native, compat) __SYSCALL(nr, compat) + #undef __SYSCALL #define __SYSCALL(nr, call) asmlinkage long __riscv_##call(const struct pt_regs *); -#include <asm/unistd.h> +#include <asm/syscall_table_32.h> #undef __SYSCALL #define __SYSCALL(nr, call) [nr] = __riscv_##call, @@ -19,5 +21,5 @@ asmlinkage long compat_sys_rt_sigreturn(void); void * const compat_sys_call_table[__NR_syscalls] = { [0 ... __NR_syscalls - 1] = __riscv_sys_ni_syscall, -#include <asm/unistd.h> +#include <asm/syscall_table_32.h> }; diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index 5ef48cb20ee1..0366dc3baf33 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -72,51 +72,89 @@ bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, unsigned i } EXPORT_SYMBOL_GPL(__riscv_isa_extension_available); -static bool riscv_isa_extension_check(int id) +static int riscv_ext_zicbom_validate(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) { - switch (id) { - case RISCV_ISA_EXT_ZICBOM: - if (!riscv_cbom_block_size) { - pr_err("Zicbom detected in ISA string, disabling as no cbom-block-size found\n"); - return false; - } else if (!is_power_of_2(riscv_cbom_block_size)) { - pr_err("Zicbom disabled as cbom-block-size present, but is not a power-of-2\n"); - return false; - } - return true; - case RISCV_ISA_EXT_ZICBOZ: - if (!riscv_cboz_block_size) { - pr_err("Zicboz detected in ISA string, disabling as no cboz-block-size found\n"); - return false; - } else if (!is_power_of_2(riscv_cboz_block_size)) { - pr_err("Zicboz disabled as cboz-block-size present, but is not a power-of-2\n"); - return false; - } - return true; - case RISCV_ISA_EXT_INVALID: - return false; + if (!riscv_cbom_block_size) { + pr_err("Zicbom detected in ISA string, disabling as no cbom-block-size found\n"); + return -EINVAL; + } + if (!is_power_of_2(riscv_cbom_block_size)) { + pr_err("Zicbom disabled as cbom-block-size present, but is not a power-of-2\n"); + return -EINVAL; } + return 0; +} - return true; +static int riscv_ext_zicboz_validate(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) +{ + if (!riscv_cboz_block_size) { + pr_err("Zicboz detected in ISA string, disabling as no cboz-block-size found\n"); + return -EINVAL; + } + if (!is_power_of_2(riscv_cboz_block_size)) { + pr_err("Zicboz disabled as cboz-block-size present, but is not a power-of-2\n"); + return -EINVAL; + } + return 0; } -#define _RISCV_ISA_EXT_DATA(_name, _id, _subset_exts, _subset_exts_size) { \ - .name = #_name, \ - .property = #_name, \ - .id = _id, \ - .subset_ext_ids = _subset_exts, \ - .subset_ext_size = _subset_exts_size \ +#define _RISCV_ISA_EXT_DATA(_name, _id, _subset_exts, _subset_exts_size, _validate) { \ + .name = #_name, \ + .property = #_name, \ + .id = _id, \ + .subset_ext_ids = _subset_exts, \ + .subset_ext_size = _subset_exts_size, \ + .validate = _validate \ } -#define __RISCV_ISA_EXT_DATA(_name, _id) _RISCV_ISA_EXT_DATA(_name, _id, NULL, 0) +#define __RISCV_ISA_EXT_DATA(_name, _id) _RISCV_ISA_EXT_DATA(_name, _id, NULL, 0, NULL) + +#define __RISCV_ISA_EXT_DATA_VALIDATE(_name, _id, _validate) \ + _RISCV_ISA_EXT_DATA(_name, _id, NULL, 0, _validate) /* Used to declare pure "lasso" extension (Zk for instance) */ #define __RISCV_ISA_EXT_BUNDLE(_name, _bundled_exts) \ - _RISCV_ISA_EXT_DATA(_name, RISCV_ISA_EXT_INVALID, _bundled_exts, ARRAY_SIZE(_bundled_exts)) + _RISCV_ISA_EXT_DATA(_name, RISCV_ISA_EXT_INVALID, _bundled_exts, \ + ARRAY_SIZE(_bundled_exts), NULL) /* Used to declare extensions that are a superset of other extensions (Zvbb for instance) */ #define __RISCV_ISA_EXT_SUPERSET(_name, _id, _sub_exts) \ - _RISCV_ISA_EXT_DATA(_name, _id, _sub_exts, ARRAY_SIZE(_sub_exts)) + _RISCV_ISA_EXT_DATA(_name, _id, _sub_exts, ARRAY_SIZE(_sub_exts), NULL) +#define __RISCV_ISA_EXT_SUPERSET_VALIDATE(_name, _id, _sub_exts, _validate) \ + _RISCV_ISA_EXT_DATA(_name, _id, _sub_exts, ARRAY_SIZE(_sub_exts), _validate) + +static int riscv_ext_zca_depends(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) +{ + if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZCA)) + return 0; + + return -EPROBE_DEFER; +} +static int riscv_ext_zcd_validate(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) +{ + if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZCA) && + __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_d)) + return 0; + + return -EPROBE_DEFER; +} + +static int riscv_ext_zcf_validate(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) +{ + if (IS_ENABLED(CONFIG_64BIT)) + return -EINVAL; + + if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZCA) && + __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_f)) + return 0; + + return -EPROBE_DEFER; +} static const unsigned int riscv_zk_bundled_exts[] = { RISCV_ISA_EXT_ZBKB, @@ -188,6 +226,40 @@ static const unsigned int riscv_zvbb_exts[] = { RISCV_ISA_EXT_ZVKB }; +#define RISCV_ISA_EXT_ZVE64F_IMPLY_LIST \ + RISCV_ISA_EXT_ZVE64X, \ + RISCV_ISA_EXT_ZVE32F, \ + RISCV_ISA_EXT_ZVE32X + +#define RISCV_ISA_EXT_ZVE64D_IMPLY_LIST \ + RISCV_ISA_EXT_ZVE64F, \ + RISCV_ISA_EXT_ZVE64F_IMPLY_LIST + +#define RISCV_ISA_EXT_V_IMPLY_LIST \ + RISCV_ISA_EXT_ZVE64D, \ + RISCV_ISA_EXT_ZVE64D_IMPLY_LIST + +static const unsigned int riscv_zve32f_exts[] = { + RISCV_ISA_EXT_ZVE32X +}; + +static const unsigned int riscv_zve64f_exts[] = { + RISCV_ISA_EXT_ZVE64F_IMPLY_LIST +}; + +static const unsigned int riscv_zve64d_exts[] = { + RISCV_ISA_EXT_ZVE64D_IMPLY_LIST +}; + +static const unsigned int riscv_v_exts[] = { + RISCV_ISA_EXT_V_IMPLY_LIST +}; + +static const unsigned int riscv_zve64x_exts[] = { + RISCV_ISA_EXT_ZVE32X, + RISCV_ISA_EXT_ZVE64X +}; + /* * While the [ms]envcfg CSRs were not defined until version 1.12 of the RISC-V * privileged ISA, the existence of the CSRs is implied by any extension which @@ -199,6 +271,21 @@ static const unsigned int riscv_xlinuxenvcfg_exts[] = { }; /* + * Zc* spec states that: + * - C always implies Zca + * - C+F implies Zcf (RV32 only) + * - C+D implies Zcd + * + * These extensions will be enabled and then validated depending on the + * availability of F/D RV32. + */ +static const unsigned int riscv_c_exts[] = { + RISCV_ISA_EXT_ZCA, + RISCV_ISA_EXT_ZCF, + RISCV_ISA_EXT_ZCD, +}; + +/* * The canonical order of ISA extension names in the ISA string is defined in * chapter 27 of the unprivileged specification. * @@ -244,11 +331,13 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(f, RISCV_ISA_EXT_f), __RISCV_ISA_EXT_DATA(d, RISCV_ISA_EXT_d), __RISCV_ISA_EXT_DATA(q, RISCV_ISA_EXT_q), - __RISCV_ISA_EXT_DATA(c, RISCV_ISA_EXT_c), - __RISCV_ISA_EXT_DATA(v, RISCV_ISA_EXT_v), + __RISCV_ISA_EXT_SUPERSET(c, RISCV_ISA_EXT_c, riscv_c_exts), + __RISCV_ISA_EXT_SUPERSET(v, RISCV_ISA_EXT_v, riscv_v_exts), __RISCV_ISA_EXT_DATA(h, RISCV_ISA_EXT_h), - __RISCV_ISA_EXT_SUPERSET(zicbom, RISCV_ISA_EXT_ZICBOM, riscv_xlinuxenvcfg_exts), - __RISCV_ISA_EXT_SUPERSET(zicboz, RISCV_ISA_EXT_ZICBOZ, riscv_xlinuxenvcfg_exts), + __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicbom, RISCV_ISA_EXT_ZICBOM, riscv_xlinuxenvcfg_exts, + riscv_ext_zicbom_validate), + __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicboz, RISCV_ISA_EXT_ZICBOZ, riscv_xlinuxenvcfg_exts, + riscv_ext_zicboz_validate), __RISCV_ISA_EXT_DATA(zicntr, RISCV_ISA_EXT_ZICNTR), __RISCV_ISA_EXT_DATA(zicond, RISCV_ISA_EXT_ZICOND), __RISCV_ISA_EXT_DATA(zicsr, RISCV_ISA_EXT_ZICSR), @@ -256,10 +345,17 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(zihintntl, RISCV_ISA_EXT_ZIHINTNTL), __RISCV_ISA_EXT_DATA(zihintpause, RISCV_ISA_EXT_ZIHINTPAUSE), __RISCV_ISA_EXT_DATA(zihpm, RISCV_ISA_EXT_ZIHPM), + __RISCV_ISA_EXT_DATA(zimop, RISCV_ISA_EXT_ZIMOP), __RISCV_ISA_EXT_DATA(zacas, RISCV_ISA_EXT_ZACAS), + __RISCV_ISA_EXT_DATA(zawrs, RISCV_ISA_EXT_ZAWRS), __RISCV_ISA_EXT_DATA(zfa, RISCV_ISA_EXT_ZFA), __RISCV_ISA_EXT_DATA(zfh, RISCV_ISA_EXT_ZFH), __RISCV_ISA_EXT_DATA(zfhmin, RISCV_ISA_EXT_ZFHMIN), + __RISCV_ISA_EXT_DATA(zca, RISCV_ISA_EXT_ZCA), + __RISCV_ISA_EXT_DATA_VALIDATE(zcb, RISCV_ISA_EXT_ZCB, riscv_ext_zca_depends), + __RISCV_ISA_EXT_DATA_VALIDATE(zcd, RISCV_ISA_EXT_ZCD, riscv_ext_zcd_validate), + __RISCV_ISA_EXT_DATA_VALIDATE(zcf, RISCV_ISA_EXT_ZCF, riscv_ext_zcf_validate), + __RISCV_ISA_EXT_DATA_VALIDATE(zcmop, RISCV_ISA_EXT_ZCMOP, riscv_ext_zca_depends), __RISCV_ISA_EXT_DATA(zba, RISCV_ISA_EXT_ZBA), __RISCV_ISA_EXT_DATA(zbb, RISCV_ISA_EXT_ZBB), __RISCV_ISA_EXT_DATA(zbc, RISCV_ISA_EXT_ZBC), @@ -280,6 +376,11 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(ztso, RISCV_ISA_EXT_ZTSO), __RISCV_ISA_EXT_SUPERSET(zvbb, RISCV_ISA_EXT_ZVBB, riscv_zvbb_exts), __RISCV_ISA_EXT_DATA(zvbc, RISCV_ISA_EXT_ZVBC), + __RISCV_ISA_EXT_SUPERSET(zve32f, RISCV_ISA_EXT_ZVE32F, riscv_zve32f_exts), + __RISCV_ISA_EXT_DATA(zve32x, RISCV_ISA_EXT_ZVE32X), + __RISCV_ISA_EXT_SUPERSET(zve64d, RISCV_ISA_EXT_ZVE64D, riscv_zve64d_exts), + __RISCV_ISA_EXT_SUPERSET(zve64f, RISCV_ISA_EXT_ZVE64F, riscv_zve64f_exts), + __RISCV_ISA_EXT_SUPERSET(zve64x, RISCV_ISA_EXT_ZVE64X, riscv_zve64x_exts), __RISCV_ISA_EXT_DATA(zvfh, RISCV_ISA_EXT_ZVFH), __RISCV_ISA_EXT_DATA(zvfhmin, RISCV_ISA_EXT_ZVFHMIN), __RISCV_ISA_EXT_DATA(zvkb, RISCV_ISA_EXT_ZVKB), @@ -309,33 +410,93 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { const size_t riscv_isa_ext_count = ARRAY_SIZE(riscv_isa_ext); -static void __init match_isa_ext(const struct riscv_isa_ext_data *ext, const char *name, - const char *name_end, struct riscv_isainfo *isainfo) +static void riscv_isa_set_ext(const struct riscv_isa_ext_data *ext, unsigned long *bitmap) { - if ((name_end - name == strlen(ext->name)) && - !strncasecmp(name, ext->name, name_end - name)) { - /* - * If this is a bundle, enable all the ISA extensions that - * comprise the bundle. - */ - if (ext->subset_ext_size) { - for (int i = 0; i < ext->subset_ext_size; i++) { - if (riscv_isa_extension_check(ext->subset_ext_ids[i])) - set_bit(ext->subset_ext_ids[i], isainfo->isa); + if (ext->id != RISCV_ISA_EXT_INVALID) + set_bit(ext->id, bitmap); + + for (int i = 0; i < ext->subset_ext_size; i++) { + if (ext->subset_ext_ids[i] != RISCV_ISA_EXT_INVALID) + set_bit(ext->subset_ext_ids[i], bitmap); + } +} + +static const struct riscv_isa_ext_data *riscv_get_isa_ext_data(unsigned int ext_id) +{ + for (int i = 0; i < riscv_isa_ext_count; i++) { + if (riscv_isa_ext[i].id == ext_id) + return &riscv_isa_ext[i]; + } + + return NULL; +} + +/* + * "Resolve" a source ISA bitmap into one that matches kernel configuration as + * well as correct extension dependencies. Some extensions depends on specific + * kernel configuration to be usable (V needs CONFIG_RISCV_ISA_V for instance) + * and this function will actually validate all the extensions provided in + * source_isa into the resolved_isa based on extensions validate() callbacks. + */ +static void __init riscv_resolve_isa(unsigned long *source_isa, + unsigned long *resolved_isa, unsigned long *this_hwcap, + unsigned long *isa2hwcap) +{ + bool loop; + const struct riscv_isa_ext_data *ext; + DECLARE_BITMAP(prev_resolved_isa, RISCV_ISA_EXT_MAX); + int max_loop_count = riscv_isa_ext_count, ret; + unsigned int bit; + + do { + loop = false; + if (max_loop_count-- < 0) { + pr_err("Failed to reach a stable ISA state\n"); + return; + } + bitmap_copy(prev_resolved_isa, resolved_isa, RISCV_ISA_EXT_MAX); + for_each_set_bit(bit, source_isa, RISCV_ISA_EXT_MAX) { + ext = riscv_get_isa_ext_data(bit); + if (!ext) + continue; + + if (ext->validate) { + ret = ext->validate(ext, resolved_isa); + if (ret == -EPROBE_DEFER) { + loop = true; + continue; + } else if (ret) { + /* Disable the extension entirely */ + clear_bit(ext->id, source_isa); + continue; + } } + + set_bit(ext->id, resolved_isa); + /* No need to keep it in source isa now that it is enabled */ + clear_bit(ext->id, source_isa); + + /* Single letter extensions get set in hwcap */ + if (ext->id < RISCV_ISA_EXT_BASE) + *this_hwcap |= isa2hwcap[ext->id]; } + } while (loop && memcmp(prev_resolved_isa, resolved_isa, sizeof(prev_resolved_isa))); +} - /* - * This is valid even for bundle extensions which uses the RISCV_ISA_EXT_INVALID id - * (rejected by riscv_isa_extension_check()). - */ - if (riscv_isa_extension_check(ext->id)) - set_bit(ext->id, isainfo->isa); +static void __init match_isa_ext(const char *name, const char *name_end, unsigned long *bitmap) +{ + for (int i = 0; i < riscv_isa_ext_count; i++) { + const struct riscv_isa_ext_data *ext = &riscv_isa_ext[i]; + + if ((name_end - name == strlen(ext->name)) && + !strncasecmp(name, ext->name, name_end - name)) { + riscv_isa_set_ext(ext, bitmap); + break; + } } } -static void __init riscv_parse_isa_string(unsigned long *this_hwcap, struct riscv_isainfo *isainfo, - unsigned long *isa2hwcap, const char *isa) +static void __init riscv_parse_isa_string(const char *isa, unsigned long *bitmap) { /* * For all possible cpus, we have already validated in @@ -348,7 +509,7 @@ static void __init riscv_parse_isa_string(unsigned long *this_hwcap, struct risc while (*isa) { const char *ext = isa++; const char *ext_end = isa; - bool ext_long = false, ext_err = false; + bool ext_err = false; switch (*ext) { case 's': @@ -388,7 +549,6 @@ static void __init riscv_parse_isa_string(unsigned long *this_hwcap, struct risc * character itself while eliminating the extensions version number. * A simple re-increment solves this problem. */ - ext_long = true; for (; *isa && *isa != '_'; ++isa) if (unlikely(!isalnum(*isa))) ext_err = true; @@ -468,17 +628,8 @@ static void __init riscv_parse_isa_string(unsigned long *this_hwcap, struct risc if (unlikely(ext_err)) continue; - if (!ext_long) { - int nr = tolower(*ext) - 'a'; - if (riscv_isa_extension_check(nr)) { - *this_hwcap |= isa2hwcap[nr]; - set_bit(nr, isainfo->isa); - } - } else { - for (int i = 0; i < riscv_isa_ext_count; i++) - match_isa_ext(&riscv_isa_ext[i], ext, ext_end, isainfo); - } + match_isa_ext(ext, ext_end, bitmap); } } @@ -505,6 +656,7 @@ static void __init riscv_fill_hwcap_from_isa_string(unsigned long *isa2hwcap) for_each_possible_cpu(cpu) { struct riscv_isainfo *isainfo = &hart_isa[cpu]; unsigned long this_hwcap = 0; + DECLARE_BITMAP(source_isa, RISCV_ISA_EXT_MAX) = { 0 }; if (acpi_disabled) { node = of_cpu_device_node_get(cpu); @@ -527,7 +679,7 @@ static void __init riscv_fill_hwcap_from_isa_string(unsigned long *isa2hwcap) } } - riscv_parse_isa_string(&this_hwcap, isainfo, isa2hwcap, isa); + riscv_parse_isa_string(isa, source_isa); /* * These ones were as they were part of the base ISA when the @@ -535,10 +687,10 @@ static void __init riscv_fill_hwcap_from_isa_string(unsigned long *isa2hwcap) * unconditionally where `i` is in riscv,isa on DT systems. */ if (acpi_disabled) { - set_bit(RISCV_ISA_EXT_ZICSR, isainfo->isa); - set_bit(RISCV_ISA_EXT_ZIFENCEI, isainfo->isa); - set_bit(RISCV_ISA_EXT_ZICNTR, isainfo->isa); - set_bit(RISCV_ISA_EXT_ZIHPM, isainfo->isa); + set_bit(RISCV_ISA_EXT_ZICSR, source_isa); + set_bit(RISCV_ISA_EXT_ZIFENCEI, source_isa); + set_bit(RISCV_ISA_EXT_ZICNTR, source_isa); + set_bit(RISCV_ISA_EXT_ZIHPM, source_isa); } /* @@ -551,9 +703,11 @@ static void __init riscv_fill_hwcap_from_isa_string(unsigned long *isa2hwcap) */ if (acpi_disabled && boot_vendorid == THEAD_VENDOR_ID && boot_archid == 0x0) { this_hwcap &= ~isa2hwcap[RISCV_ISA_EXT_v]; - clear_bit(RISCV_ISA_EXT_v, isainfo->isa); + clear_bit(RISCV_ISA_EXT_v, source_isa); } + riscv_resolve_isa(source_isa, isainfo->isa, &this_hwcap, isa2hwcap); + /* * All "okay" hart should have same isa. Set HWCAP based on * common capabilities of every "okay" hart, in case they don't @@ -582,6 +736,7 @@ static int __init riscv_fill_hwcap_from_ext_list(unsigned long *isa2hwcap) unsigned long this_hwcap = 0; struct device_node *cpu_node; struct riscv_isainfo *isainfo = &hart_isa[cpu]; + DECLARE_BITMAP(source_isa, RISCV_ISA_EXT_MAX) = { 0 }; cpu_node = of_cpu_device_node_get(cpu); if (!cpu_node) { @@ -601,22 +756,11 @@ static int __init riscv_fill_hwcap_from_ext_list(unsigned long *isa2hwcap) ext->property) < 0) continue; - if (ext->subset_ext_size) { - for (int j = 0; j < ext->subset_ext_size; j++) { - if (riscv_isa_extension_check(ext->subset_ext_ids[j])) - set_bit(ext->subset_ext_ids[j], isainfo->isa); - } - } - - if (riscv_isa_extension_check(ext->id)) { - set_bit(ext->id, isainfo->isa); - - /* Only single letter extensions get set in hwcap */ - if (strnlen(riscv_isa_ext[i].name, 2) == 1) - this_hwcap |= isa2hwcap[riscv_isa_ext[i].id]; - } + riscv_isa_set_ext(ext, source_isa); } + riscv_resolve_isa(source_isa, isainfo->isa, &this_hwcap, isa2hwcap); + of_node_put(cpu_node); /* @@ -686,8 +830,14 @@ void __init riscv_fill_hwcap(void) elf_hwcap &= ~COMPAT_HWCAP_ISA_F; } - if (elf_hwcap & COMPAT_HWCAP_ISA_V) { + if (__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_ZVE32X)) { + /* + * This cannot fail when called on the boot hart + */ riscv_v_setup_vsize(); + } + + if (elf_hwcap & COMPAT_HWCAP_ISA_V) { /* * ISA string in device tree might have 'v' flag, but * CONFIG_RISCV_ISA_V is disabled in kernel. diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S index 4236a69c35cb..356d5397b2a2 100644 --- a/arch/riscv/kernel/head.S +++ b/arch/riscv/kernel/head.S @@ -165,10 +165,21 @@ secondary_start_sbi: #endif call .Lsetup_trap_vector scs_load_current - tail smp_callin + call smp_callin #endif /* CONFIG_SMP */ .align 2 +.Lsecondary_park: + /* + * Park this hart if we: + * - have too many harts on CONFIG_RISCV_BOOT_SPINWAIT + * - receive an early trap, before setup_trap_vector finished + * - fail in smp_callin(), as a successful one wouldn't return + */ + wfi + j .Lsecondary_park + +.align 2 .Lsetup_trap_vector: /* Set trap vector to exception handler */ la a0, handle_exception @@ -181,12 +192,6 @@ secondary_start_sbi: csrw CSR_SCRATCH, zero ret -.align 2 -.Lsecondary_park: - /* We lack SMP support or have too many harts, so park this hart */ - wfi - j .Lsecondary_park - SYM_CODE_END(_start) SYM_CODE_START(_start_kernel) @@ -300,6 +305,9 @@ SYM_CODE_START(_start_kernel) #else mv a0, a1 #endif /* CONFIG_BUILTIN_DTB */ + /* Set trap vector to spin forever to help debug */ + la a3, .Lsecondary_park + csrw CSR_TVEC, a3 call setup_vm #ifdef CONFIG_MMU la a0, early_pg_dir diff --git a/arch/riscv/kernel/jump_label.c b/arch/riscv/kernel/jump_label.c index e6694759dbd0..11ad789c60c6 100644 --- a/arch/riscv/kernel/jump_label.c +++ b/arch/riscv/kernel/jump_label.c @@ -9,13 +9,14 @@ #include <linux/memory.h> #include <linux/mutex.h> #include <asm/bug.h> +#include <asm/cacheflush.h> #include <asm/patch.h> #define RISCV_INSN_NOP 0x00000013U #define RISCV_INSN_JAL 0x0000006fU -void arch_jump_label_transform(struct jump_entry *entry, - enum jump_label_type type) +bool arch_jump_label_transform_queue(struct jump_entry *entry, + enum jump_label_type type) { void *addr = (void *)jump_entry_code(entry); u32 insn; @@ -24,7 +25,7 @@ void arch_jump_label_transform(struct jump_entry *entry, long offset = jump_entry_target(entry) - jump_entry_code(entry); if (WARN_ON(offset & 1 || offset < -524288 || offset >= 524288)) - return; + return true; insn = RISCV_INSN_JAL | (((u32)offset & GENMASK(19, 12)) << (12 - 12)) | @@ -36,6 +37,13 @@ void arch_jump_label_transform(struct jump_entry *entry, } mutex_lock(&text_mutex); - patch_text_nosync(addr, &insn, sizeof(insn)); + patch_insn_write(addr, &insn, sizeof(insn)); mutex_unlock(&text_mutex); + + return true; +} + +void arch_jump_label_transform_apply(void) +{ + flush_icache_all(); } diff --git a/arch/riscv/kernel/patch.c b/arch/riscv/kernel/patch.c index ab03732d06c4..69e5796fc51f 100644 --- a/arch/riscv/kernel/patch.c +++ b/arch/riscv/kernel/patch.c @@ -19,7 +19,7 @@ struct patch_insn { void *addr; u32 *insns; - int ninsns; + size_t len; atomic_t cpu_count; }; @@ -54,7 +54,7 @@ static __always_inline void *patch_map(void *addr, const unsigned int fixmap) BUG_ON(!page); return (void *)set_fixmap_offset(fixmap, page_to_phys(page) + - (uintaddr & ~PAGE_MASK)); + offset_in_page(addr)); } static void patch_unmap(int fixmap) @@ -65,8 +65,8 @@ NOKPROBE_SYMBOL(patch_unmap); static int __patch_insn_set(void *addr, u8 c, size_t len) { + bool across_pages = (offset_in_page(addr) + len) > PAGE_SIZE; void *waddr = addr; - bool across_pages = (((uintptr_t)addr & ~PAGE_MASK) + len) > PAGE_SIZE; /* * Only two pages can be mapped at a time for writing. @@ -110,8 +110,8 @@ NOKPROBE_SYMBOL(__patch_insn_set); static int __patch_insn_write(void *addr, const void *insn, size_t len) { + bool across_pages = (offset_in_page(addr) + len) > PAGE_SIZE; void *waddr = addr; - bool across_pages = (((uintptr_t) addr & ~PAGE_MASK) + len) > PAGE_SIZE; int ret; /* @@ -179,31 +179,32 @@ NOKPROBE_SYMBOL(__patch_insn_write); static int patch_insn_set(void *addr, u8 c, size_t len) { - size_t patched = 0; size_t size; - int ret = 0; + int ret; /* * __patch_insn_set() can only work on 2 pages at a time so call it in a * loop with len <= 2 * PAGE_SIZE. */ - while (patched < len && !ret) { - size = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(addr + patched), len - patched); - ret = __patch_insn_set(addr + patched, c, size); - - patched += size; + while (len) { + size = min(len, PAGE_SIZE * 2 - offset_in_page(addr)); + ret = __patch_insn_set(addr, c, size); + if (ret) + return ret; + + addr += size; + len -= size; } - return ret; + return 0; } NOKPROBE_SYMBOL(patch_insn_set); int patch_text_set_nosync(void *addr, u8 c, size_t len) { - u32 *tp = addr; int ret; - ret = patch_insn_set(tp, c, len); + ret = patch_insn_set(addr, c, len); return ret; } @@ -211,31 +212,33 @@ NOKPROBE_SYMBOL(patch_text_set_nosync); int patch_insn_write(void *addr, const void *insn, size_t len) { - size_t patched = 0; size_t size; - int ret = 0; + int ret; /* * Copy the instructions to the destination address, two pages at a time * because __patch_insn_write() can only handle len <= 2 * PAGE_SIZE. */ - while (patched < len && !ret) { - size = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(addr + patched), len - patched); - ret = __patch_insn_write(addr + patched, insn + patched, size); - - patched += size; + while (len) { + size = min(len, PAGE_SIZE * 2 - offset_in_page(addr)); + ret = __patch_insn_write(addr, insn, size); + if (ret) + return ret; + + addr += size; + insn += size; + len -= size; } - return ret; + return 0; } NOKPROBE_SYMBOL(patch_insn_write); int patch_text_nosync(void *addr, const void *insns, size_t len) { - u32 *tp = addr; int ret; - ret = patch_insn_write(tp, insns, len); + ret = patch_insn_write(addr, insns, len); return ret; } @@ -244,14 +247,10 @@ NOKPROBE_SYMBOL(patch_text_nosync); static int patch_text_cb(void *data) { struct patch_insn *patch = data; - unsigned long len; - int i, ret = 0; + int ret = 0; if (atomic_inc_return(&patch->cpu_count) == num_online_cpus()) { - for (i = 0; ret == 0 && i < patch->ninsns; i++) { - len = GET_INSN_LENGTH(patch->insns[i]); - ret = patch_insn_write(patch->addr + i * len, &patch->insns[i], len); - } + ret = patch_insn_write(patch->addr, patch->insns, patch->len); /* * Make sure the patching store is effective *before* we * increment the counter which releases all waiting CPUs @@ -271,13 +270,13 @@ static int patch_text_cb(void *data) } NOKPROBE_SYMBOL(patch_text_cb); -int patch_text(void *addr, u32 *insns, int ninsns) +int patch_text(void *addr, u32 *insns, size_t len) { int ret; struct patch_insn patch = { .addr = addr, .insns = insns, - .ninsns = ninsns, + .len = len, .cpu_count = ATOMIC_INIT(0), }; diff --git a/arch/riscv/kernel/probes/kprobes.c b/arch/riscv/kernel/probes/kprobes.c index dfb28e57d900..474a65213657 100644 --- a/arch/riscv/kernel/probes/kprobes.c +++ b/arch/riscv/kernel/probes/kprobes.c @@ -24,14 +24,13 @@ post_kprobe_handler(struct kprobe *, struct kprobe_ctlblk *, struct pt_regs *); static void __kprobes arch_prepare_ss_slot(struct kprobe *p) { + size_t len = GET_INSN_LENGTH(p->opcode); u32 insn = __BUG_INSN_32; - unsigned long offset = GET_INSN_LENGTH(p->opcode); - p->ainsn.api.restore = (unsigned long)p->addr + offset; + p->ainsn.api.restore = (unsigned long)p->addr + len; - patch_text(p->ainsn.api.insn, &p->opcode, 1); - patch_text((void *)((unsigned long)(p->ainsn.api.insn) + offset), - &insn, 1); + patch_text_nosync(p->ainsn.api.insn, &p->opcode, len); + patch_text_nosync(p->ainsn.api.insn + len, &insn, GET_INSN_LENGTH(insn)); } static void __kprobes arch_prepare_simulate(struct kprobe *p) @@ -108,16 +107,18 @@ int __kprobes arch_prepare_kprobe(struct kprobe *p) /* install breakpoint in text */ void __kprobes arch_arm_kprobe(struct kprobe *p) { - u32 insn = (p->opcode & __INSN_LENGTH_MASK) == __INSN_LENGTH_32 ? - __BUG_INSN_32 : __BUG_INSN_16; + size_t len = GET_INSN_LENGTH(p->opcode); + u32 insn = len == 4 ? __BUG_INSN_32 : __BUG_INSN_16; - patch_text(p->addr, &insn, 1); + patch_text(p->addr, &insn, len); } /* remove breakpoint from text */ void __kprobes arch_disarm_kprobe(struct kprobe *p) { - patch_text(p->addr, &p->opcode, 1); + size_t len = GET_INSN_LENGTH(p->opcode); + + patch_text(p->addr, &p->opcode, len); } void __kprobes arch_remove_kprobe(struct kprobe *p) diff --git a/arch/riscv/kernel/sbi.c b/arch/riscv/kernel/sbi.c index e66e0999a800..837bdab2601b 100644 --- a/arch/riscv/kernel/sbi.c +++ b/arch/riscv/kernel/sbi.c @@ -14,6 +14,9 @@ #include <asm/smp.h> #include <asm/tlbflush.h> +#define CREATE_TRACE_POINTS +#include <asm/trace.h> + /* default SBI version is 0.1 */ unsigned long sbi_spec_version __ro_after_init = SBI_SPEC_VERSION_DEFAULT; EXPORT_SYMBOL(sbi_spec_version); @@ -24,13 +27,15 @@ static int (*__sbi_rfence)(int fid, const struct cpumask *cpu_mask, unsigned long start, unsigned long size, unsigned long arg4, unsigned long arg5) __ro_after_init; -struct sbiret sbi_ecall(int ext, int fid, unsigned long arg0, - unsigned long arg1, unsigned long arg2, - unsigned long arg3, unsigned long arg4, - unsigned long arg5) +struct sbiret __sbi_ecall(unsigned long arg0, unsigned long arg1, + unsigned long arg2, unsigned long arg3, + unsigned long arg4, unsigned long arg5, + int fid, int ext) { struct sbiret ret; + trace_sbi_call(ext, fid); + register uintptr_t a0 asm ("a0") = (uintptr_t)(arg0); register uintptr_t a1 asm ("a1") = (uintptr_t)(arg1); register uintptr_t a2 asm ("a2") = (uintptr_t)(arg2); @@ -46,9 +51,11 @@ struct sbiret sbi_ecall(int ext, int fid, unsigned long arg0, ret.error = a0; ret.value = a1; + trace_sbi_return(ext, ret.error, ret.value); + return ret; } -EXPORT_SYMBOL(sbi_ecall); +EXPORT_SYMBOL(__sbi_ecall); int sbi_err_map_linux_errno(int err) { diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c index 1319b29ce3b5..19baf0d574d3 100644 --- a/arch/riscv/kernel/smpboot.c +++ b/arch/riscv/kernel/smpboot.c @@ -214,6 +214,15 @@ asmlinkage __visible void smp_callin(void) struct mm_struct *mm = &init_mm; unsigned int curr_cpuid = smp_processor_id(); + if (has_vector()) { + /* + * Return as early as possible so the hart with a mismatching + * vlen won't boot. + */ + if (riscv_v_setup_vsize()) + return; + } + /* All kernel threads share the same mm context. */ mmgrab(mm); current->active_mm = mm; @@ -226,11 +235,6 @@ asmlinkage __visible void smp_callin(void) numa_add_cpu(curr_cpuid); set_cpu_online(curr_cpuid, true); - if (has_vector()) { - if (riscv_v_setup_vsize()) - elf_hwcap &= ~COMPAT_HWCAP_ISA_V; - } - riscv_user_isa_enable(); /* diff --git a/arch/riscv/kernel/sys_hwprobe.c b/arch/riscv/kernel/sys_hwprobe.c index 969ef3d59dbe..685594769535 100644 --- a/arch/riscv/kernel/sys_hwprobe.c +++ b/arch/riscv/kernel/sys_hwprobe.c @@ -8,6 +8,7 @@ #include <asm/cacheflush.h> #include <asm/cpufeature.h> #include <asm/hwprobe.h> +#include <asm/processor.h> #include <asm/sbi.h> #include <asm/switch_to.h> #include <asm/uaccess.h> @@ -69,7 +70,7 @@ static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, if (riscv_isa_extension_available(NULL, c)) pair->value |= RISCV_HWPROBE_IMA_C; - if (has_vector()) + if (has_vector() && riscv_isa_extension_available(NULL, v)) pair->value |= RISCV_HWPROBE_IMA_V; /* @@ -112,8 +113,22 @@ static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, EXT_KEY(ZACAS); EXT_KEY(ZICOND); EXT_KEY(ZIHINTPAUSE); + EXT_KEY(ZIMOP); + EXT_KEY(ZCA); + EXT_KEY(ZCB); + EXT_KEY(ZCMOP); + EXT_KEY(ZAWRS); + /* + * All the following extensions must depend on the kernel + * support of V. + */ if (has_vector()) { + EXT_KEY(ZVE32X); + EXT_KEY(ZVE32F); + EXT_KEY(ZVE64X); + EXT_KEY(ZVE64F); + EXT_KEY(ZVE64D); EXT_KEY(ZVBB); EXT_KEY(ZVBC); EXT_KEY(ZVKB); @@ -132,6 +147,8 @@ static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, EXT_KEY(ZFH); EXT_KEY(ZFHMIN); EXT_KEY(ZFA); + EXT_KEY(ZCD); + EXT_KEY(ZCF); } #undef EXT_KEY } @@ -216,6 +233,9 @@ static void hwprobe_one_pair(struct riscv_hwprobe *pair, if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOZ)) pair->value = riscv_cboz_block_size; break; + case RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS: + pair->value = user_max_virt_addr(); + break; /* * For forward compatibility, unknown keys don't fail the whole diff --git a/arch/riscv/kernel/syscall_table.c b/arch/riscv/kernel/syscall_table.c index dda913764903..6f1a36cb0f3f 100644 --- a/arch/riscv/kernel/syscall_table.c +++ b/arch/riscv/kernel/syscall_table.c @@ -9,14 +9,16 @@ #include <asm-generic/syscalls.h> #include <asm/syscall.h> +#define __SYSCALL_WITH_COMPAT(nr, native, compat) __SYSCALL(nr, native) + #undef __SYSCALL #define __SYSCALL(nr, call) asmlinkage long __riscv_##call(const struct pt_regs *); -#include <asm/unistd.h> +#include <asm/syscall_table.h> #undef __SYSCALL #define __SYSCALL(nr, call) [nr] = __riscv_##call, void * const sys_call_table[__NR_syscalls] = { [0 ... __NR_syscalls - 1] = __riscv_sys_ni_syscall, -#include <asm/unistd.h> +#include <asm/syscall_table.h> }; diff --git a/arch/riscv/kernel/vector.c b/arch/riscv/kernel/vector.c index 6727d1d3b8f2..682b3feee451 100644 --- a/arch/riscv/kernel/vector.c +++ b/arch/riscv/kernel/vector.c @@ -173,8 +173,11 @@ bool riscv_v_first_use_handler(struct pt_regs *regs) u32 __user *epc = (u32 __user *)regs->epc; u32 insn = (u32)regs->badaddr; + if (!has_vector()) + return false; + /* Do not handle if V is not supported, or disabled */ - if (!(ELF_HWCAP & COMPAT_HWCAP_ISA_V)) + if (!riscv_v_vstate_ctrl_user_allowed()) return false; /* If V has been enabled then it is not the first-use trap */ diff --git a/arch/riscv/kvm/aia.c b/arch/riscv/kvm/aia.c index 0f0a9d11bb5f..2967d305c442 100644 --- a/arch/riscv/kvm/aia.c +++ b/arch/riscv/kvm/aia.c @@ -10,12 +10,12 @@ #include <linux/kernel.h> #include <linux/bitops.h> #include <linux/irq.h> +#include <linux/irqchip/riscv-imsic.h> #include <linux/irqdomain.h> #include <linux/kvm_host.h> #include <linux/percpu.h> #include <linux/spinlock.h> #include <asm/cpufeature.h> -#include <asm/kvm_aia_imsic.h> struct aia_hgei_control { raw_spinlock_t lock; @@ -394,6 +394,8 @@ int kvm_riscv_aia_alloc_hgei(int cpu, struct kvm_vcpu *owner, { int ret = -ENOENT; unsigned long flags; + const struct imsic_global_config *gc; + const struct imsic_local_config *lc; struct aia_hgei_control *hgctrl = per_cpu_ptr(&aia_hgei, cpu); if (!kvm_riscv_aia_available() || !hgctrl) @@ -409,11 +411,14 @@ int kvm_riscv_aia_alloc_hgei(int cpu, struct kvm_vcpu *owner, raw_spin_unlock_irqrestore(&hgctrl->lock, flags); - /* TODO: To be updated later by AIA IMSIC HW guest file support */ - if (hgei_va) - *hgei_va = NULL; - if (hgei_pa) - *hgei_pa = 0; + gc = imsic_get_global_config(); + lc = (gc) ? per_cpu_ptr(gc->local, cpu) : NULL; + if (lc && ret > 0) { + if (hgei_va) + *hgei_va = lc->msi_va + (ret * IMSIC_MMIO_PAGE_SZ); + if (hgei_pa) + *hgei_pa = lc->msi_pa + (ret * IMSIC_MMIO_PAGE_SZ); + } return ret; } @@ -605,9 +610,11 @@ void kvm_riscv_aia_disable(void) int kvm_riscv_aia_init(void) { int rc; + const struct imsic_global_config *gc; if (!riscv_isa_extension_available(NULL, SxAIA)) return -ENODEV; + gc = imsic_get_global_config(); /* Figure-out number of bits in HGEIE */ csr_write(CSR_HGEIE, -1UL); @@ -619,17 +626,17 @@ int kvm_riscv_aia_init(void) /* * Number of usable HGEI lines should be minimum of per-HART * IMSIC guest files and number of bits in HGEIE - * - * TODO: To be updated later by AIA IMSIC HW guest file support */ - kvm_riscv_aia_nr_hgei = 0; + if (gc) + kvm_riscv_aia_nr_hgei = min((ulong)kvm_riscv_aia_nr_hgei, + BIT(gc->guest_index_bits) - 1); + else + kvm_riscv_aia_nr_hgei = 0; - /* - * Find number of guest MSI IDs - * - * TODO: To be updated later by AIA IMSIC HW guest file support - */ + /* Find number of guest MSI IDs */ kvm_riscv_aia_max_ids = IMSIC_MAX_ID; + if (gc && kvm_riscv_aia_nr_hgei) + kvm_riscv_aia_max_ids = gc->nr_guest_ids + 1; /* Initialize guest external interrupt line management */ rc = aia_hgei_init(); diff --git a/arch/riscv/kvm/aia_aplic.c b/arch/riscv/kvm/aia_aplic.c index b467ba5ed910..da6ff1bade0d 100644 --- a/arch/riscv/kvm/aia_aplic.c +++ b/arch/riscv/kvm/aia_aplic.c @@ -7,12 +7,12 @@ * Anup Patel <apatel@ventanamicro.com> */ +#include <linux/irqchip/riscv-aplic.h> #include <linux/kvm_host.h> #include <linux/math.h> #include <linux/spinlock.h> #include <linux/swab.h> #include <kvm/iodev.h> -#include <asm/kvm_aia_aplic.h> struct aplic_irq { raw_spinlock_t lock; diff --git a/arch/riscv/kvm/aia_device.c b/arch/riscv/kvm/aia_device.c index 5cd407c6a8e4..39cd26af5a69 100644 --- a/arch/riscv/kvm/aia_device.c +++ b/arch/riscv/kvm/aia_device.c @@ -8,9 +8,9 @@ */ #include <linux/bits.h> +#include <linux/irqchip/riscv-imsic.h> #include <linux/kvm_host.h> #include <linux/uaccess.h> -#include <asm/kvm_aia_imsic.h> static void unlock_vcpus(struct kvm *kvm, int vcpu_lock_idx) { diff --git a/arch/riscv/kvm/aia_imsic.c b/arch/riscv/kvm/aia_imsic.c index e808723a85f1..0a1e859323b4 100644 --- a/arch/riscv/kvm/aia_imsic.c +++ b/arch/riscv/kvm/aia_imsic.c @@ -9,13 +9,13 @@ #include <linux/atomic.h> #include <linux/bitmap.h> +#include <linux/irqchip/riscv-imsic.h> #include <linux/kvm_host.h> #include <linux/math.h> #include <linux/spinlock.h> #include <linux/swab.h> #include <kvm/iodev.h> #include <asm/csr.h> -#include <asm/kvm_aia_imsic.h> #define IMSIC_MAX_EIX (IMSIC_MAX_ID / BITS_PER_TYPE(u64)) diff --git a/arch/riscv/kvm/trace.h b/arch/riscv/kvm/trace.h new file mode 100644 index 000000000000..3d54175d805c --- /dev/null +++ b/arch/riscv/kvm/trace.h @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Tracepoints for RISC-V KVM + * + * Copyright 2024 Beijing ESWIN Computing Technology Co., Ltd. + * + */ +#if !defined(_TRACE_KVM_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_KVM_H + +#include <linux/tracepoint.h> + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM kvm + +TRACE_EVENT(kvm_entry, + TP_PROTO(struct kvm_vcpu *vcpu), + TP_ARGS(vcpu), + + TP_STRUCT__entry( + __field(unsigned long, pc) + ), + + TP_fast_assign( + __entry->pc = vcpu->arch.guest_context.sepc; + ), + + TP_printk("PC: 0x016%lx", __entry->pc) +); + +TRACE_EVENT(kvm_exit, + TP_PROTO(struct kvm_cpu_trap *trap), + TP_ARGS(trap), + + TP_STRUCT__entry( + __field(unsigned long, sepc) + __field(unsigned long, scause) + __field(unsigned long, stval) + __field(unsigned long, htval) + __field(unsigned long, htinst) + ), + + TP_fast_assign( + __entry->sepc = trap->sepc; + __entry->scause = trap->scause; + __entry->stval = trap->stval; + __entry->htval = trap->htval; + __entry->htinst = trap->htinst; + ), + + TP_printk("SEPC:0x%lx, SCAUSE:0x%lx, STVAL:0x%lx, HTVAL:0x%lx, HTINST:0x%lx", + __entry->sepc, + __entry->scause, + __entry->stval, + __entry->htval, + __entry->htinst) +); + +#endif /* _TRACE_RSICV_KVM_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE trace + +/* This part must be outside protection */ +#include <trace/define_trace.h> diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c index 17e21df36cc1..8d7d381737ee 100644 --- a/arch/riscv/kvm/vcpu.c +++ b/arch/riscv/kvm/vcpu.c @@ -21,10 +21,14 @@ #include <asm/cacheflush.h> #include <asm/kvm_vcpu_vector.h> +#define CREATE_TRACE_POINTS +#include "trace.h" + const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { KVM_GENERIC_VCPU_STATS(), STATS_DESC_COUNTER(VCPU, ecall_exit_stat), STATS_DESC_COUNTER(VCPU, wfi_exit_stat), + STATS_DESC_COUNTER(VCPU, wrs_exit_stat), STATS_DESC_COUNTER(VCPU, mmio_exit_user), STATS_DESC_COUNTER(VCPU, mmio_exit_kernel), STATS_DESC_COUNTER(VCPU, csr_exit_user), @@ -760,7 +764,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) return ret; } - if (run->immediate_exit) { + if (!vcpu->wants_to_run) { kvm_vcpu_srcu_read_unlock(vcpu); return -EINTR; } @@ -831,6 +835,8 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) */ kvm_riscv_local_tlb_sanitize(vcpu); + trace_kvm_entry(vcpu); + guest_timing_enter_irqoff(); kvm_riscv_vcpu_enter_exit(vcpu); @@ -869,6 +875,8 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) local_irq_enable(); + trace_kvm_exit(&trap); + preempt_enable(); kvm_vcpu_srcu_read_lock(vcpu); diff --git a/arch/riscv/kvm/vcpu_exit.c b/arch/riscv/kvm/vcpu_exit.c index 5761f95abb60..fa98e5c024b2 100644 --- a/arch/riscv/kvm/vcpu_exit.c +++ b/arch/riscv/kvm/vcpu_exit.c @@ -185,6 +185,8 @@ int kvm_riscv_vcpu_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, case EXC_INST_ILLEGAL: case EXC_LOAD_MISALIGNED: case EXC_STORE_MISALIGNED: + case EXC_LOAD_ACCESS: + case EXC_STORE_ACCESS: if (vcpu->arch.guest_context.hstatus & HSTATUS_SPV) { kvm_riscv_vcpu_trap_redirect(vcpu, trap); ret = 1; diff --git a/arch/riscv/kvm/vcpu_insn.c b/arch/riscv/kvm/vcpu_insn.c index ee7215f4071f..97dec18e6989 100644 --- a/arch/riscv/kvm/vcpu_insn.c +++ b/arch/riscv/kvm/vcpu_insn.c @@ -16,6 +16,9 @@ #define INSN_MASK_WFI 0xffffffff #define INSN_MATCH_WFI 0x10500073 +#define INSN_MASK_WRS 0xffffffff +#define INSN_MATCH_WRS 0x00d00073 + #define INSN_MATCH_CSRRW 0x1073 #define INSN_MASK_CSRRW 0x707f #define INSN_MATCH_CSRRS 0x2073 @@ -203,6 +206,13 @@ static int wfi_insn(struct kvm_vcpu *vcpu, struct kvm_run *run, ulong insn) return KVM_INSN_CONTINUE_NEXT_SEPC; } +static int wrs_insn(struct kvm_vcpu *vcpu, struct kvm_run *run, ulong insn) +{ + vcpu->stat.wrs_exit_stat++; + kvm_vcpu_on_spin(vcpu, vcpu->arch.guest_context.sstatus & SR_SPP); + return KVM_INSN_CONTINUE_NEXT_SEPC; +} + struct csr_func { unsigned int base; unsigned int count; @@ -378,6 +388,11 @@ static const struct insn_func system_opcode_funcs[] = { .match = INSN_MATCH_WFI, .func = wfi_insn, }, + { + .mask = INSN_MASK_WRS, + .match = INSN_MATCH_WRS, + .func = wrs_insn, + }, }; static int system_opcode_insn(struct kvm_vcpu *vcpu, struct kvm_run *run, diff --git a/arch/riscv/kvm/vcpu_onereg.c b/arch/riscv/kvm/vcpu_onereg.c index 62874fbca29f..b319c4c13c54 100644 --- a/arch/riscv/kvm/vcpu_onereg.c +++ b/arch/riscv/kvm/vcpu_onereg.c @@ -42,6 +42,7 @@ static const unsigned long kvm_isa_ext_arr[] = { KVM_ISA_EXT_ARR(SVNAPOT), KVM_ISA_EXT_ARR(SVPBMT), KVM_ISA_EXT_ARR(ZACAS), + KVM_ISA_EXT_ARR(ZAWRS), KVM_ISA_EXT_ARR(ZBA), KVM_ISA_EXT_ARR(ZBB), KVM_ISA_EXT_ARR(ZBC), @@ -49,6 +50,11 @@ static const unsigned long kvm_isa_ext_arr[] = { KVM_ISA_EXT_ARR(ZBKC), KVM_ISA_EXT_ARR(ZBKX), KVM_ISA_EXT_ARR(ZBS), + KVM_ISA_EXT_ARR(ZCA), + KVM_ISA_EXT_ARR(ZCB), + KVM_ISA_EXT_ARR(ZCD), + KVM_ISA_EXT_ARR(ZCF), + KVM_ISA_EXT_ARR(ZCMOP), KVM_ISA_EXT_ARR(ZFA), KVM_ISA_EXT_ARR(ZFH), KVM_ISA_EXT_ARR(ZFHMIN), @@ -61,6 +67,7 @@ static const unsigned long kvm_isa_ext_arr[] = { KVM_ISA_EXT_ARR(ZIHINTNTL), KVM_ISA_EXT_ARR(ZIHINTPAUSE), KVM_ISA_EXT_ARR(ZIHPM), + KVM_ISA_EXT_ARR(ZIMOP), KVM_ISA_EXT_ARR(ZKND), KVM_ISA_EXT_ARR(ZKNE), KVM_ISA_EXT_ARR(ZKNH), @@ -126,6 +133,7 @@ static bool kvm_riscv_vcpu_isa_disable_allowed(unsigned long ext) case KVM_RISCV_ISA_EXT_SVINVAL: case KVM_RISCV_ISA_EXT_SVNAPOT: case KVM_RISCV_ISA_EXT_ZACAS: + case KVM_RISCV_ISA_EXT_ZAWRS: case KVM_RISCV_ISA_EXT_ZBA: case KVM_RISCV_ISA_EXT_ZBB: case KVM_RISCV_ISA_EXT_ZBC: @@ -133,6 +141,11 @@ static bool kvm_riscv_vcpu_isa_disable_allowed(unsigned long ext) case KVM_RISCV_ISA_EXT_ZBKC: case KVM_RISCV_ISA_EXT_ZBKX: case KVM_RISCV_ISA_EXT_ZBS: + case KVM_RISCV_ISA_EXT_ZCA: + case KVM_RISCV_ISA_EXT_ZCB: + case KVM_RISCV_ISA_EXT_ZCD: + case KVM_RISCV_ISA_EXT_ZCF: + case KVM_RISCV_ISA_EXT_ZCMOP: case KVM_RISCV_ISA_EXT_ZFA: case KVM_RISCV_ISA_EXT_ZFH: case KVM_RISCV_ISA_EXT_ZFHMIN: @@ -143,6 +156,7 @@ static bool kvm_riscv_vcpu_isa_disable_allowed(unsigned long ext) case KVM_RISCV_ISA_EXT_ZIHINTNTL: case KVM_RISCV_ISA_EXT_ZIHINTPAUSE: case KVM_RISCV_ISA_EXT_ZIHPM: + case KVM_RISCV_ISA_EXT_ZIMOP: case KVM_RISCV_ISA_EXT_ZKND: case KVM_RISCV_ISA_EXT_ZKNE: case KVM_RISCV_ISA_EXT_ZKNH: diff --git a/arch/riscv/lib/Makefile b/arch/riscv/lib/Makefile index bd6e6c1b0497..2b369f51b0a5 100644 --- a/arch/riscv/lib/Makefile +++ b/arch/riscv/lib/Makefile @@ -13,6 +13,7 @@ endif lib-$(CONFIG_MMU) += uaccess.o lib-$(CONFIG_64BIT) += tishift.o lib-$(CONFIG_RISCV_ISA_ZICBOZ) += clear_page.o +lib-$(CONFIG_RISCV_ISA_ZBC) += crc32.o obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o lib-$(CONFIG_RISCV_ISA_V) += xor.o diff --git a/arch/riscv/lib/crc32.c b/arch/riscv/lib/crc32.c new file mode 100644 index 000000000000..d7dc599af3ef --- /dev/null +++ b/arch/riscv/lib/crc32.c @@ -0,0 +1,294 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Accelerated CRC32 implementation with Zbc extension. + * + * Copyright (C) 2024 Intel Corporation + */ + +#include <asm/hwcap.h> +#include <asm/alternative-macros.h> +#include <asm/byteorder.h> + +#include <linux/types.h> +#include <linux/minmax.h> +#include <linux/crc32poly.h> +#include <linux/crc32.h> +#include <linux/byteorder/generic.h> + +/* + * Refer to https://www.corsix.org/content/barrett-reduction-polynomials for + * better understanding of how this math works. + * + * let "+" denotes polynomial add (XOR) + * let "-" denotes polynomial sub (XOR) + * let "*" denotes polynomial multiplication + * let "/" denotes polynomial floor division + * let "S" denotes source data, XLEN bit wide + * let "P" denotes CRC32 polynomial + * let "T" denotes 2^(XLEN+32) + * let "QT" denotes quotient of T/P, with the bit for 2^XLEN being implicit + * + * crc32(S, P) + * => S * (2^32) - S * (2^32) / P * P + * => lowest 32 bits of: S * (2^32) / P * P + * => lowest 32 bits of: S * (2^32) * (T / P) / T * P + * => lowest 32 bits of: S * (2^32) * quotient / T * P + * => lowest 32 bits of: S * quotient / 2^XLEN * P + * => lowest 32 bits of: (clmul_high_part(S, QT) + S) * P + * => clmul_low_part(clmul_high_part(S, QT) + S, P) + * + * In terms of below implementations, the BE case is more intuitive, since the + * higher order bit sits at more significant position. + */ + +#if __riscv_xlen == 64 +/* Slide by XLEN bits per iteration */ +# define STEP_ORDER 3 + +/* Each below polynomial quotient has an implicit bit for 2^XLEN */ + +/* Polynomial quotient of (2^(XLEN+32))/CRC32_POLY, in LE format */ +# define CRC32_POLY_QT_LE 0x5a72d812fb808b20 + +/* Polynomial quotient of (2^(XLEN+32))/CRC32C_POLY, in LE format */ +# define CRC32C_POLY_QT_LE 0xa434f61c6f5389f8 + +/* Polynomial quotient of (2^(XLEN+32))/CRC32_POLY, in BE format, it should be + * the same as the bit-reversed version of CRC32_POLY_QT_LE + */ +# define CRC32_POLY_QT_BE 0x04d101df481b4e5a + +static inline u64 crc32_le_prep(u32 crc, unsigned long const *ptr) +{ + return (u64)crc ^ (__force u64)__cpu_to_le64(*ptr); +} + +static inline u32 crc32_le_zbc(unsigned long s, u32 poly, unsigned long poly_qt) +{ + u32 crc; + + /* We don't have a "clmulrh" insn, so use clmul + slli instead. */ + asm volatile (".option push\n" + ".option arch,+zbc\n" + "clmul %0, %1, %2\n" + "slli %0, %0, 1\n" + "xor %0, %0, %1\n" + "clmulr %0, %0, %3\n" + "srli %0, %0, 32\n" + ".option pop\n" + : "=&r" (crc) + : "r" (s), + "r" (poly_qt), + "r" ((u64)poly << 32) + :); + return crc; +} + +static inline u64 crc32_be_prep(u32 crc, unsigned long const *ptr) +{ + return ((u64)crc << 32) ^ (__force u64)__cpu_to_be64(*ptr); +} + +#elif __riscv_xlen == 32 +# define STEP_ORDER 2 +/* Each quotient should match the upper half of its analog in RV64 */ +# define CRC32_POLY_QT_LE 0xfb808b20 +# define CRC32C_POLY_QT_LE 0x6f5389f8 +# define CRC32_POLY_QT_BE 0x04d101df + +static inline u32 crc32_le_prep(u32 crc, unsigned long const *ptr) +{ + return crc ^ (__force u32)__cpu_to_le32(*ptr); +} + +static inline u32 crc32_le_zbc(unsigned long s, u32 poly, unsigned long poly_qt) +{ + u32 crc; + + /* We don't have a "clmulrh" insn, so use clmul + slli instead. */ + asm volatile (".option push\n" + ".option arch,+zbc\n" + "clmul %0, %1, %2\n" + "slli %0, %0, 1\n" + "xor %0, %0, %1\n" + "clmulr %0, %0, %3\n" + ".option pop\n" + : "=&r" (crc) + : "r" (s), + "r" (poly_qt), + "r" (poly) + :); + return crc; +} + +static inline u32 crc32_be_prep(u32 crc, unsigned long const *ptr) +{ + return crc ^ (__force u32)__cpu_to_be32(*ptr); +} + +#else +# error "Unexpected __riscv_xlen" +#endif + +static inline u32 crc32_be_zbc(unsigned long s) +{ + u32 crc; + + asm volatile (".option push\n" + ".option arch,+zbc\n" + "clmulh %0, %1, %2\n" + "xor %0, %0, %1\n" + "clmul %0, %0, %3\n" + ".option pop\n" + : "=&r" (crc) + : "r" (s), + "r" (CRC32_POLY_QT_BE), + "r" (CRC32_POLY_BE) + :); + return crc; +} + +#define STEP (1 << STEP_ORDER) +#define OFFSET_MASK (STEP - 1) + +typedef u32 (*fallback)(u32 crc, unsigned char const *p, size_t len); + +static inline u32 crc32_le_unaligned(u32 crc, unsigned char const *p, + size_t len, u32 poly, + unsigned long poly_qt) +{ + size_t bits = len * 8; + unsigned long s = 0; + u32 crc_low = 0; + + for (int i = 0; i < len; i++) + s = ((unsigned long)*p++ << (__riscv_xlen - 8)) | (s >> 8); + + s ^= (unsigned long)crc << (__riscv_xlen - bits); + if (__riscv_xlen == 32 || len < sizeof(u32)) + crc_low = crc >> bits; + + crc = crc32_le_zbc(s, poly, poly_qt); + crc ^= crc_low; + + return crc; +} + +static inline u32 __pure crc32_le_generic(u32 crc, unsigned char const *p, + size_t len, u32 poly, + unsigned long poly_qt, + fallback crc_fb) +{ + size_t offset, head_len, tail_len; + unsigned long const *p_ul; + unsigned long s; + + asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, + RISCV_ISA_EXT_ZBC, 1) + : : : : legacy); + + /* Handle the unaligned head. */ + offset = (unsigned long)p & OFFSET_MASK; + if (offset && len) { + head_len = min(STEP - offset, len); + crc = crc32_le_unaligned(crc, p, head_len, poly, poly_qt); + p += head_len; + len -= head_len; + } + + tail_len = len & OFFSET_MASK; + len = len >> STEP_ORDER; + p_ul = (unsigned long const *)p; + + for (int i = 0; i < len; i++) { + s = crc32_le_prep(crc, p_ul); + crc = crc32_le_zbc(s, poly, poly_qt); + p_ul++; + } + + /* Handle the tail bytes. */ + p = (unsigned char const *)p_ul; + if (tail_len) + crc = crc32_le_unaligned(crc, p, tail_len, poly, poly_qt); + + return crc; + +legacy: + return crc_fb(crc, p, len); +} + +u32 __pure crc32_le(u32 crc, unsigned char const *p, size_t len) +{ + return crc32_le_generic(crc, p, len, CRC32_POLY_LE, CRC32_POLY_QT_LE, + crc32_le_base); +} + +u32 __pure __crc32c_le(u32 crc, unsigned char const *p, size_t len) +{ + return crc32_le_generic(crc, p, len, CRC32C_POLY_LE, + CRC32C_POLY_QT_LE, __crc32c_le_base); +} + +static inline u32 crc32_be_unaligned(u32 crc, unsigned char const *p, + size_t len) +{ + size_t bits = len * 8; + unsigned long s = 0; + u32 crc_low = 0; + + s = 0; + for (int i = 0; i < len; i++) + s = *p++ | (s << 8); + + if (__riscv_xlen == 32 || len < sizeof(u32)) { + s ^= crc >> (32 - bits); + crc_low = crc << bits; + } else { + s ^= (unsigned long)crc << (bits - 32); + } + + crc = crc32_be_zbc(s); + crc ^= crc_low; + + return crc; +} + +u32 __pure crc32_be(u32 crc, unsigned char const *p, size_t len) +{ + size_t offset, head_len, tail_len; + unsigned long const *p_ul; + unsigned long s; + + asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, + RISCV_ISA_EXT_ZBC, 1) + : : : : legacy); + + /* Handle the unaligned head. */ + offset = (unsigned long)p & OFFSET_MASK; + if (offset && len) { + head_len = min(STEP - offset, len); + crc = crc32_be_unaligned(crc, p, head_len); + p += head_len; + len -= head_len; + } + + tail_len = len & OFFSET_MASK; + len = len >> STEP_ORDER; + p_ul = (unsigned long const *)p; + + for (int i = 0; i < len; i++) { + s = crc32_be_prep(crc, p_ul); + crc = crc32_be_zbc(s); + p_ul++; + } + + /* Handle the tail bytes. */ + p = (unsigned char const *)p_ul; + if (tail_len) + crc = crc32_be_unaligned(crc, p, tail_len); + + return crc; + +legacy: + return crc32_be_base(crc, p, len); +} diff --git a/arch/riscv/lib/uaccess.S b/arch/riscv/lib/uaccess.S index 1399d797d81b..6a9f116bb545 100644 --- a/arch/riscv/lib/uaccess.S +++ b/arch/riscv/lib/uaccess.S @@ -14,7 +14,7 @@ SYM_FUNC_START(__asm_copy_to_user) #ifdef CONFIG_RISCV_ISA_V - ALTERNATIVE("j fallback_scalar_usercopy", "nop", 0, RISCV_ISA_EXT_v, CONFIG_RISCV_ISA_V) + ALTERNATIVE("j fallback_scalar_usercopy", "nop", 0, RISCV_ISA_EXT_ZVE32X, CONFIG_RISCV_ISA_V) REG_L t0, riscv_v_usercopy_threshold bltu a2, t0, fallback_scalar_usercopy tail enter_vector_usercopy diff --git a/arch/riscv/mm/hugetlbpage.c b/arch/riscv/mm/hugetlbpage.c index 0ebd968b33c9..42314f093922 100644 --- a/arch/riscv/mm/hugetlbpage.c +++ b/arch/riscv/mm/hugetlbpage.c @@ -3,7 +3,7 @@ #include <linux/err.h> #ifdef CONFIG_RISCV_ISA_SVNAPOT -pte_t huge_ptep_get(pte_t *ptep) +pte_t huge_ptep_get(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { unsigned long pte_num; int i; diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index e3405e4b99af..bfa2dea95354 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -28,6 +28,7 @@ #include <asm/fixmap.h> #include <asm/io.h> +#include <asm/kasan.h> #include <asm/numa.h> #include <asm/pgtable.h> #include <asm/sections.h> @@ -296,7 +297,7 @@ static void __init setup_bootmem(void) } #ifdef CONFIG_MMU -struct pt_alloc_ops pt_ops __initdata; +struct pt_alloc_ops pt_ops __meminitdata; pgd_t swapper_pg_dir[PTRS_PER_PGD] __page_aligned_bss; pgd_t trampoline_pg_dir[PTRS_PER_PGD] __page_aligned_bss; @@ -358,7 +359,7 @@ static inline pte_t *__init get_pte_virt_fixmap(phys_addr_t pa) return (pte_t *)set_fixmap_offset(FIX_PTE, pa); } -static inline pte_t *__init get_pte_virt_late(phys_addr_t pa) +static inline pte_t *__meminit get_pte_virt_late(phys_addr_t pa) { return (pte_t *) __va(pa); } @@ -377,7 +378,7 @@ static inline phys_addr_t __init alloc_pte_fixmap(uintptr_t va) return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); } -static phys_addr_t __init alloc_pte_late(uintptr_t va) +static phys_addr_t __meminit alloc_pte_late(uintptr_t va) { struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL & ~__GFP_HIGHMEM, 0); @@ -385,9 +386,8 @@ static phys_addr_t __init alloc_pte_late(uintptr_t va) return __pa((pte_t *)ptdesc_address(ptdesc)); } -static void __init create_pte_mapping(pte_t *ptep, - uintptr_t va, phys_addr_t pa, - phys_addr_t sz, pgprot_t prot) +static void __meminit create_pte_mapping(pte_t *ptep, uintptr_t va, phys_addr_t pa, phys_addr_t sz, + pgprot_t prot) { uintptr_t pte_idx = pte_index(va); @@ -441,7 +441,7 @@ static pmd_t *__init get_pmd_virt_fixmap(phys_addr_t pa) return (pmd_t *)set_fixmap_offset(FIX_PMD, pa); } -static pmd_t *__init get_pmd_virt_late(phys_addr_t pa) +static pmd_t *__meminit get_pmd_virt_late(phys_addr_t pa) { return (pmd_t *) __va(pa); } @@ -458,7 +458,7 @@ static phys_addr_t __init alloc_pmd_fixmap(uintptr_t va) return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); } -static phys_addr_t __init alloc_pmd_late(uintptr_t va) +static phys_addr_t __meminit alloc_pmd_late(uintptr_t va) { struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL & ~__GFP_HIGHMEM, 0); @@ -466,9 +466,9 @@ static phys_addr_t __init alloc_pmd_late(uintptr_t va) return __pa((pmd_t *)ptdesc_address(ptdesc)); } -static void __init create_pmd_mapping(pmd_t *pmdp, - uintptr_t va, phys_addr_t pa, - phys_addr_t sz, pgprot_t prot) +static void __meminit create_pmd_mapping(pmd_t *pmdp, + uintptr_t va, phys_addr_t pa, + phys_addr_t sz, pgprot_t prot) { pte_t *ptep; phys_addr_t pte_phys; @@ -504,7 +504,7 @@ static pud_t *__init get_pud_virt_fixmap(phys_addr_t pa) return (pud_t *)set_fixmap_offset(FIX_PUD, pa); } -static pud_t *__init get_pud_virt_late(phys_addr_t pa) +static pud_t *__meminit get_pud_virt_late(phys_addr_t pa) { return (pud_t *)__va(pa); } @@ -522,7 +522,7 @@ static phys_addr_t __init alloc_pud_fixmap(uintptr_t va) return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); } -static phys_addr_t alloc_pud_late(uintptr_t va) +static phys_addr_t __meminit alloc_pud_late(uintptr_t va) { unsigned long vaddr; @@ -542,7 +542,7 @@ static p4d_t *__init get_p4d_virt_fixmap(phys_addr_t pa) return (p4d_t *)set_fixmap_offset(FIX_P4D, pa); } -static p4d_t *__init get_p4d_virt_late(phys_addr_t pa) +static p4d_t *__meminit get_p4d_virt_late(phys_addr_t pa) { return (p4d_t *)__va(pa); } @@ -560,7 +560,7 @@ static phys_addr_t __init alloc_p4d_fixmap(uintptr_t va) return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); } -static phys_addr_t alloc_p4d_late(uintptr_t va) +static phys_addr_t __meminit alloc_p4d_late(uintptr_t va) { unsigned long vaddr; @@ -569,9 +569,8 @@ static phys_addr_t alloc_p4d_late(uintptr_t va) return __pa(vaddr); } -static void __init create_pud_mapping(pud_t *pudp, - uintptr_t va, phys_addr_t pa, - phys_addr_t sz, pgprot_t prot) +static void __meminit create_pud_mapping(pud_t *pudp, uintptr_t va, phys_addr_t pa, phys_addr_t sz, + pgprot_t prot) { pmd_t *nextp; phys_addr_t next_phys; @@ -596,9 +595,8 @@ static void __init create_pud_mapping(pud_t *pudp, create_pmd_mapping(nextp, va, pa, sz, prot); } -static void __init create_p4d_mapping(p4d_t *p4dp, - uintptr_t va, phys_addr_t pa, - phys_addr_t sz, pgprot_t prot) +static void __meminit create_p4d_mapping(p4d_t *p4dp, uintptr_t va, phys_addr_t pa, phys_addr_t sz, + pgprot_t prot) { pud_t *nextp; phys_addr_t next_phys; @@ -654,9 +652,8 @@ static void __init create_p4d_mapping(p4d_t *p4dp, #define create_pmd_mapping(__pmdp, __va, __pa, __sz, __prot) do {} while(0) #endif /* __PAGETABLE_PMD_FOLDED */ -void __init create_pgd_mapping(pgd_t *pgdp, - uintptr_t va, phys_addr_t pa, - phys_addr_t sz, pgprot_t prot) +void __meminit create_pgd_mapping(pgd_t *pgdp, uintptr_t va, phys_addr_t pa, phys_addr_t sz, + pgprot_t prot) { pgd_next_t *nextp; phys_addr_t next_phys; @@ -681,8 +678,7 @@ void __init create_pgd_mapping(pgd_t *pgdp, create_pgd_next_mapping(nextp, va, pa, sz, prot); } -static uintptr_t __init best_map_size(phys_addr_t pa, uintptr_t va, - phys_addr_t size) +static uintptr_t __meminit best_map_size(phys_addr_t pa, uintptr_t va, phys_addr_t size) { if (debug_pagealloc_enabled()) return PAGE_SIZE; @@ -718,7 +714,7 @@ asmlinkage void __init __copy_data(void) #endif #ifdef CONFIG_STRICT_KERNEL_RWX -static __init pgprot_t pgprot_from_va(uintptr_t va) +static __meminit pgprot_t pgprot_from_va(uintptr_t va) { if (is_va_kernel_text(va)) return PAGE_KERNEL_READ_EXEC; @@ -743,7 +739,7 @@ void mark_rodata_ro(void) set_memory_ro); } #else -static __init pgprot_t pgprot_from_va(uintptr_t va) +static __meminit pgprot_t pgprot_from_va(uintptr_t va) { if (IS_ENABLED(CONFIG_64BIT) && !is_kernel_mapping(va)) return PAGE_KERNEL; @@ -1235,9 +1231,8 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) pt_ops_set_fixmap(); } -static void __init create_linear_mapping_range(phys_addr_t start, - phys_addr_t end, - uintptr_t fixed_map_size) +static void __meminit create_linear_mapping_range(phys_addr_t start, phys_addr_t end, + uintptr_t fixed_map_size, const pgprot_t *pgprot) { phys_addr_t pa; uintptr_t va, map_size; @@ -1248,7 +1243,7 @@ static void __init create_linear_mapping_range(phys_addr_t start, best_map_size(pa, va, end - pa); create_pgd_mapping(swapper_pg_dir, va, pa, map_size, - pgprot_from_va(va)); + pgprot ? *pgprot : pgprot_from_va(va)); } } @@ -1292,22 +1287,19 @@ static void __init create_linear_mapping_page_table(void) if (end >= __pa(PAGE_OFFSET) + memory_limit) end = __pa(PAGE_OFFSET) + memory_limit; - create_linear_mapping_range(start, end, 0); + create_linear_mapping_range(start, end, 0, NULL); } #ifdef CONFIG_STRICT_KERNEL_RWX - create_linear_mapping_range(ktext_start, ktext_start + ktext_size, 0); - create_linear_mapping_range(krodata_start, - krodata_start + krodata_size, 0); + create_linear_mapping_range(ktext_start, ktext_start + ktext_size, 0, NULL); + create_linear_mapping_range(krodata_start, krodata_start + krodata_size, 0, NULL); memblock_clear_nomap(ktext_start, ktext_size); memblock_clear_nomap(krodata_start, krodata_size); #endif #ifdef CONFIG_KFENCE - create_linear_mapping_range(kfence_pool, - kfence_pool + KFENCE_POOL_SIZE, - PAGE_SIZE); + create_linear_mapping_range(kfence_pool, kfence_pool + KFENCE_POOL_SIZE, PAGE_SIZE, NULL); memblock_clear_nomap(kfence_pool, KFENCE_POOL_SIZE); #endif @@ -1439,7 +1431,7 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, * memory hotplug, we are not able to update all the page tables with * the new PMDs. */ - return vmemmap_populate_hugepages(start, end, node, NULL); + return vmemmap_populate_hugepages(start, end, node, altmap); } #endif @@ -1493,11 +1485,19 @@ failed: panic("Failed to pre-allocate %s pages for %s area\n", lvl, area); } +#define PAGE_END KASAN_SHADOW_START + void __init pgtable_cache_init(void) { preallocate_pgd_pages_range(VMALLOC_START, VMALLOC_END, "vmalloc"); if (IS_ENABLED(CONFIG_MODULES)) preallocate_pgd_pages_range(MODULES_VADDR, MODULES_END, "bpf/modules"); + if (IS_ENABLED(CONFIG_MEMORY_HOTPLUG)) { + preallocate_pgd_pages_range(VMEMMAP_START, VMEMMAP_END, "vmemmap"); + preallocate_pgd_pages_range(PAGE_OFFSET, PAGE_END, "direct map"); + if (IS_ENABLED(CONFIG_KASAN)) + preallocate_pgd_pages_range(KASAN_SHADOW_START, KASAN_SHADOW_END, "kasan"); + } } #endif @@ -1534,3 +1534,270 @@ struct execmem_info __init *execmem_arch_setup(void) } #endif /* CONFIG_MMU */ #endif /* CONFIG_EXECMEM */ + +#ifdef CONFIG_MEMORY_HOTPLUG +static void __meminit free_pte_table(pte_t *pte_start, pmd_t *pmd) +{ + struct page *page = pmd_page(*pmd); + struct ptdesc *ptdesc = page_ptdesc(page); + pte_t *pte; + int i; + + for (i = 0; i < PTRS_PER_PTE; i++) { + pte = pte_start + i; + if (!pte_none(*pte)) + return; + } + + pagetable_pte_dtor(ptdesc); + if (PageReserved(page)) + free_reserved_page(page); + else + pagetable_free(ptdesc); + pmd_clear(pmd); +} + +static void __meminit free_pmd_table(pmd_t *pmd_start, pud_t *pud) +{ + struct page *page = pud_page(*pud); + struct ptdesc *ptdesc = page_ptdesc(page); + pmd_t *pmd; + int i; + + for (i = 0; i < PTRS_PER_PMD; i++) { + pmd = pmd_start + i; + if (!pmd_none(*pmd)) + return; + } + + pagetable_pmd_dtor(ptdesc); + if (PageReserved(page)) + free_reserved_page(page); + else + pagetable_free(ptdesc); + pud_clear(pud); +} + +static void __meminit free_pud_table(pud_t *pud_start, p4d_t *p4d) +{ + struct page *page = p4d_page(*p4d); + pud_t *pud; + int i; + + for (i = 0; i < PTRS_PER_PUD; i++) { + pud = pud_start + i; + if (!pud_none(*pud)) + return; + } + + if (PageReserved(page)) + free_reserved_page(page); + else + free_pages((unsigned long)page_address(page), 0); + p4d_clear(p4d); +} + +static void __meminit free_vmemmap_storage(struct page *page, size_t size, + struct vmem_altmap *altmap) +{ + int order = get_order(size); + + if (altmap) { + vmem_altmap_free(altmap, size >> PAGE_SHIFT); + return; + } + + if (PageReserved(page)) { + unsigned int nr_pages = 1 << order; + + while (nr_pages--) + free_reserved_page(page++); + return; + } + + free_pages((unsigned long)page_address(page), order); +} + +static void __meminit remove_pte_mapping(pte_t *pte_base, unsigned long addr, unsigned long end, + bool is_vmemmap, struct vmem_altmap *altmap) +{ + unsigned long next; + pte_t *ptep, pte; + + for (; addr < end; addr = next) { + next = (addr + PAGE_SIZE) & PAGE_MASK; + if (next > end) + next = end; + + ptep = pte_base + pte_index(addr); + pte = ptep_get(ptep); + if (!pte_present(*ptep)) + continue; + + pte_clear(&init_mm, addr, ptep); + if (is_vmemmap) + free_vmemmap_storage(pte_page(pte), PAGE_SIZE, altmap); + } +} + +static void __meminit remove_pmd_mapping(pmd_t *pmd_base, unsigned long addr, unsigned long end, + bool is_vmemmap, struct vmem_altmap *altmap) +{ + unsigned long next; + pte_t *pte_base; + pmd_t *pmdp, pmd; + + for (; addr < end; addr = next) { + next = pmd_addr_end(addr, end); + pmdp = pmd_base + pmd_index(addr); + pmd = pmdp_get(pmdp); + if (!pmd_present(pmd)) + continue; + + if (pmd_leaf(pmd)) { + pmd_clear(pmdp); + if (is_vmemmap) + free_vmemmap_storage(pmd_page(pmd), PMD_SIZE, altmap); + continue; + } + + pte_base = (pte_t *)pmd_page_vaddr(*pmdp); + remove_pte_mapping(pte_base, addr, next, is_vmemmap, altmap); + free_pte_table(pte_base, pmdp); + } +} + +static void __meminit remove_pud_mapping(pud_t *pud_base, unsigned long addr, unsigned long end, + bool is_vmemmap, struct vmem_altmap *altmap) +{ + unsigned long next; + pud_t *pudp, pud; + pmd_t *pmd_base; + + for (; addr < end; addr = next) { + next = pud_addr_end(addr, end); + pudp = pud_base + pud_index(addr); + pud = pudp_get(pudp); + if (!pud_present(pud)) + continue; + + if (pud_leaf(pud)) { + if (pgtable_l4_enabled) { + pud_clear(pudp); + if (is_vmemmap) + free_vmemmap_storage(pud_page(pud), PUD_SIZE, altmap); + } + continue; + } + + pmd_base = pmd_offset(pudp, 0); + remove_pmd_mapping(pmd_base, addr, next, is_vmemmap, altmap); + + if (pgtable_l4_enabled) + free_pmd_table(pmd_base, pudp); + } +} + +static void __meminit remove_p4d_mapping(p4d_t *p4d_base, unsigned long addr, unsigned long end, + bool is_vmemmap, struct vmem_altmap *altmap) +{ + unsigned long next; + p4d_t *p4dp, p4d; + pud_t *pud_base; + + for (; addr < end; addr = next) { + next = p4d_addr_end(addr, end); + p4dp = p4d_base + p4d_index(addr); + p4d = p4dp_get(p4dp); + if (!p4d_present(p4d)) + continue; + + if (p4d_leaf(p4d)) { + if (pgtable_l5_enabled) { + p4d_clear(p4dp); + if (is_vmemmap) + free_vmemmap_storage(p4d_page(p4d), P4D_SIZE, altmap); + } + continue; + } + + pud_base = pud_offset(p4dp, 0); + remove_pud_mapping(pud_base, addr, next, is_vmemmap, altmap); + + if (pgtable_l5_enabled) + free_pud_table(pud_base, p4dp); + } +} + +static void __meminit remove_pgd_mapping(unsigned long va, unsigned long end, bool is_vmemmap, + struct vmem_altmap *altmap) +{ + unsigned long addr, next; + p4d_t *p4d_base; + pgd_t *pgd; + + for (addr = va; addr < end; addr = next) { + next = pgd_addr_end(addr, end); + pgd = pgd_offset_k(addr); + + if (!pgd_present(*pgd)) + continue; + + if (pgd_leaf(*pgd)) + continue; + + p4d_base = p4d_offset(pgd, 0); + remove_p4d_mapping(p4d_base, addr, next, is_vmemmap, altmap); + } + + flush_tlb_all(); +} + +static void __meminit remove_linear_mapping(phys_addr_t start, u64 size) +{ + unsigned long va = (unsigned long)__va(start); + unsigned long end = (unsigned long)__va(start + size); + + remove_pgd_mapping(va, end, false, NULL); +} + +struct range arch_get_mappable_range(void) +{ + struct range mhp_range; + + mhp_range.start = __pa(PAGE_OFFSET); + mhp_range.end = __pa(PAGE_END - 1); + return mhp_range; +} + +int __ref arch_add_memory(int nid, u64 start, u64 size, struct mhp_params *params) +{ + int ret = 0; + + create_linear_mapping_range(start, start + size, 0, ¶ms->pgprot); + ret = __add_pages(nid, start >> PAGE_SHIFT, size >> PAGE_SHIFT, params); + if (ret) { + remove_linear_mapping(start, size); + goto out; + } + + max_pfn = PFN_UP(start + size); + max_low_pfn = max_pfn; + + out: + flush_tlb_all(); + return ret; +} + +void __ref arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap) +{ + __remove_pages(start >> PAGE_SHIFT, size >> PAGE_SHIFT, altmap); + remove_linear_mapping(start, size); + flush_tlb_all(); +} + +void __ref vmemmap_free(unsigned long start, unsigned long end, struct vmem_altmap *altmap) +{ + remove_pgd_mapping(start, end, true, altmap); +} +#endif /* CONFIG_MEMORY_HOTPLUG */ diff --git a/arch/riscv/mm/ptdump.c b/arch/riscv/mm/ptdump.c index 1289cc6d3700..9d5f657a251b 100644 --- a/arch/riscv/mm/ptdump.c +++ b/arch/riscv/mm/ptdump.c @@ -6,6 +6,7 @@ #include <linux/efi.h> #include <linux/init.h> #include <linux/debugfs.h> +#include <linux/memory_hotplug.h> #include <linux/seq_file.h> #include <linux/ptdump.h> @@ -370,7 +371,9 @@ bool ptdump_check_wx(void) static int ptdump_show(struct seq_file *m, void *v) { + get_online_mems(); ptdump_walk(m, m->private); + put_online_mems(); return 0; } diff --git a/arch/riscv/net/bpf_jit.h b/arch/riscv/net/bpf_jit.h index fdbf88ca8b70..1d1c78d4cff1 100644 --- a/arch/riscv/net/bpf_jit.h +++ b/arch/riscv/net/bpf_jit.h @@ -18,6 +18,11 @@ static inline bool rvc_enabled(void) return IS_ENABLED(CONFIG_RISCV_ISA_C); } +static inline bool rvzba_enabled(void) +{ + return IS_ENABLED(CONFIG_RISCV_ISA_ZBA) && riscv_has_extension_likely(RISCV_ISA_EXT_ZBA); +} + static inline bool rvzbb_enabled(void) { return IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && riscv_has_extension_likely(RISCV_ISA_EXT_ZBB); @@ -737,6 +742,17 @@ static inline u16 rvc_swsp(u32 imm8, u8 rs2) return rv_css_insn(0x6, imm, rs2, 0x2); } +/* RVZBA instructions. */ +static inline u32 rvzba_sh2add(u8 rd, u8 rs1, u8 rs2) +{ + return rv_r_insn(0x10, rs2, rs1, 0x4, rd, 0x33); +} + +static inline u32 rvzba_sh3add(u8 rd, u8 rs1, u8 rs2) +{ + return rv_r_insn(0x10, rs2, rs1, 0x6, rd, 0x33); +} + /* RVZBB instructions. */ static inline u32 rvzbb_sextb(u8 rd, u8 rs1) { @@ -939,6 +955,14 @@ static inline u16 rvc_sdsp(u32 imm9, u8 rs2) return rv_css_insn(0x7, imm, rs2, 0x2); } +/* RV64-only ZBA instructions. */ + +static inline u32 rvzba_zextw(u8 rd, u8 rs1) +{ + /* add.uw rd, rs1, ZERO */ + return rv_r_insn(0x04, RV_REG_ZERO, rs1, 0, rd, 0x3b); +} + #endif /* __riscv_xlen == 64 */ /* Helper functions that emit RVC instructions when possible. */ @@ -1082,6 +1106,28 @@ static inline void emit_sw(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx) emit(rv_sw(rs1, off, rs2), ctx); } +static inline void emit_sh2add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) +{ + if (rvzba_enabled()) { + emit(rvzba_sh2add(rd, rs1, rs2), ctx); + return; + } + + emit_slli(rd, rs1, 2, ctx); + emit_add(rd, rd, rs2, ctx); +} + +static inline void emit_sh3add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) +{ + if (rvzba_enabled()) { + emit(rvzba_sh3add(rd, rs1, rs2), ctx); + return; + } + + emit_slli(rd, rs1, 3, ctx); + emit_add(rd, rd, rs2, ctx); +} + /* RV64-only helper functions. */ #if __riscv_xlen == 64 @@ -1161,6 +1207,11 @@ static inline void emit_zexth(u8 rd, u8 rs, struct rv_jit_context *ctx) static inline void emit_zextw(u8 rd, u8 rs, struct rv_jit_context *ctx) { + if (rvzba_enabled()) { + emit(rvzba_zextw(rd, rs), ctx); + return; + } + emit_slli(rd, rs, 32, ctx); emit_srli(rd, rd, 32, ctx); } diff --git a/arch/riscv/net/bpf_jit_comp32.c b/arch/riscv/net/bpf_jit_comp32.c index f5ba73bb153d..592dd86fbf81 100644 --- a/arch/riscv/net/bpf_jit_comp32.c +++ b/arch/riscv/net/bpf_jit_comp32.c @@ -811,8 +811,7 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) * if (!prog) * goto out; */ - emit(rv_slli(RV_REG_T0, lo(idx_reg), 2), ctx); - emit(rv_add(RV_REG_T0, RV_REG_T0, lo(arr_reg)), ctx); + emit_sh2add(RV_REG_T0, lo(idx_reg), lo(arr_reg), ctx); off = offsetof(struct bpf_array, ptrs); if (is_12b_check(off, insn)) return -1; diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index 79a001d5533e..99f34409fb60 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -15,7 +15,11 @@ #include <asm/percpu.h> #include "bpf_jit.h" +#define RV_MAX_REG_ARGS 8 #define RV_FENTRY_NINSNS 2 +#define RV_FENTRY_NBYTES (RV_FENTRY_NINSNS * 4) +/* imm that allows emit_imm to emit max count insns */ +#define RV_MAX_COUNT_IMM 0x7FFF7FF7FF7FF7FF #define RV_REG_TCC RV_REG_A6 #define RV_REG_TCC_SAVED RV_REG_S6 /* Store A6 in S6 if program do calls */ @@ -380,8 +384,7 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) * if (!prog) * goto out; */ - emit_slli(RV_REG_T2, RV_REG_A2, 3, ctx); - emit_add(RV_REG_T2, RV_REG_T2, RV_REG_A1, ctx); + emit_sh3add(RV_REG_T2, RV_REG_A2, RV_REG_A1, ctx); off = offsetof(struct bpf_array, ptrs); if (is_12b_check(off, insn)) return -1; @@ -537,8 +540,10 @@ static void emit_atomic(u8 rd, u8 rs, s16 off, s32 imm, bool is64, /* r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg); */ case BPF_CMPXCHG: r0 = bpf_to_rv_reg(BPF_REG_0, ctx); - emit(is64 ? rv_addi(RV_REG_T2, r0, 0) : - rv_addiw(RV_REG_T2, r0, 0), ctx); + if (is64) + emit_mv(RV_REG_T2, r0, ctx); + else + emit_addiw(RV_REG_T2, r0, 0, ctx); emit(is64 ? rv_lr_d(r0, 0, rd, 0, 0) : rv_lr_w(r0, 0, rd, 0, 0), ctx); jmp_offset = ninsns_rvoff(8); @@ -672,7 +677,7 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type, if (ret) return ret; - if (memcmp(ip, old_insns, RV_FENTRY_NINSNS * 4)) + if (memcmp(ip, old_insns, RV_FENTRY_NBYTES)) return -EFAULT; ret = gen_jump_or_nops(new_addr, ip, new_insns, is_call); @@ -681,34 +686,53 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type, cpus_read_lock(); mutex_lock(&text_mutex); - if (memcmp(ip, new_insns, RV_FENTRY_NINSNS * 4)) - ret = patch_text(ip, new_insns, RV_FENTRY_NINSNS); + if (memcmp(ip, new_insns, RV_FENTRY_NBYTES)) + ret = patch_text(ip, new_insns, RV_FENTRY_NBYTES); mutex_unlock(&text_mutex); cpus_read_unlock(); return ret; } -static void store_args(int nregs, int args_off, struct rv_jit_context *ctx) +static void store_args(int nr_arg_slots, int args_off, struct rv_jit_context *ctx) { int i; - for (i = 0; i < nregs; i++) { - emit_sd(RV_REG_FP, -args_off, RV_REG_A0 + i, ctx); + for (i = 0; i < nr_arg_slots; i++) { + if (i < RV_MAX_REG_ARGS) { + emit_sd(RV_REG_FP, -args_off, RV_REG_A0 + i, ctx); + } else { + /* skip slots for T0 and FP of traced function */ + emit_ld(RV_REG_T1, 16 + (i - RV_MAX_REG_ARGS) * 8, RV_REG_FP, ctx); + emit_sd(RV_REG_FP, -args_off, RV_REG_T1, ctx); + } args_off -= 8; } } -static void restore_args(int nregs, int args_off, struct rv_jit_context *ctx) +static void restore_args(int nr_reg_args, int args_off, struct rv_jit_context *ctx) { int i; - for (i = 0; i < nregs; i++) { + for (i = 0; i < nr_reg_args; i++) { emit_ld(RV_REG_A0 + i, -args_off, RV_REG_FP, ctx); args_off -= 8; } } +static void restore_stack_args(int nr_stack_args, int args_off, int stk_arg_off, + struct rv_jit_context *ctx) +{ + int i; + + for (i = 0; i < nr_stack_args; i++) { + emit_ld(RV_REG_T1, -(args_off - RV_MAX_REG_ARGS * 8), RV_REG_FP, ctx); + emit_sd(RV_REG_FP, -stk_arg_off, RV_REG_T1, ctx); + args_off -= 8; + stk_arg_off -= 8; + } +} + static int invoke_bpf_prog(struct bpf_tramp_link *l, int args_off, int retval_off, int run_ctx_off, bool save_ret, struct rv_jit_context *ctx) { @@ -781,8 +805,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, { int i, ret, offset; int *branches_off = NULL; - int stack_size = 0, nregs = m->nr_args; - int retval_off, args_off, nregs_off, ip_off, run_ctx_off, sreg_off; + int stack_size = 0, nr_arg_slots = 0; + int retval_off, args_off, nregs_off, ip_off, run_ctx_off, sreg_off, stk_arg_off; struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY]; struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT]; struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN]; @@ -828,20 +852,21 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, * FP - sreg_off [ callee saved reg ] * * [ pads ] pads for 16 bytes alignment + * + * [ stack_argN ] + * [ ... ] + * FP - stk_arg_off [ stack_arg1 ] BPF_TRAMP_F_CALL_ORIG */ if (flags & (BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SHARE_IPMODIFY)) return -ENOTSUPP; - /* extra regiters for struct arguments */ - for (i = 0; i < m->nr_args; i++) - if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG) - nregs += round_up(m->arg_size[i], 8) / 8 - 1; - - /* 8 arguments passed by registers */ - if (nregs > 8) + if (m->nr_args > MAX_BPF_FUNC_ARGS) return -ENOTSUPP; + for (i = 0; i < m->nr_args; i++) + nr_arg_slots += round_up(m->arg_size[i], 8) / 8; + /* room of trampoline frame to store return address and frame pointer */ stack_size += 16; @@ -851,7 +876,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, retval_off = stack_size; } - stack_size += nregs * 8; + stack_size += nr_arg_slots * 8; args_off = stack_size; stack_size += 8; @@ -868,7 +893,13 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, stack_size += 8; sreg_off = stack_size; - stack_size = round_up(stack_size, 16); + if ((flags & BPF_TRAMP_F_CALL_ORIG) && (nr_arg_slots - RV_MAX_REG_ARGS > 0)) + stack_size += (nr_arg_slots - RV_MAX_REG_ARGS) * 8; + + stack_size = round_up(stack_size, STACK_ALIGN); + + /* room for args on stack must be at the top of stack */ + stk_arg_off = stack_size; if (!is_struct_ops) { /* For the trampoline called from function entry, @@ -905,17 +936,17 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, emit_sd(RV_REG_FP, -ip_off, RV_REG_T1, ctx); } - emit_li(RV_REG_T1, nregs, ctx); + emit_li(RV_REG_T1, nr_arg_slots, ctx); emit_sd(RV_REG_FP, -nregs_off, RV_REG_T1, ctx); - store_args(nregs, args_off, ctx); + store_args(nr_arg_slots, args_off, ctx); /* skip to actual body of traced function */ if (flags & BPF_TRAMP_F_SKIP_FRAME) orig_call += RV_FENTRY_NINSNS * 4; if (flags & BPF_TRAMP_F_CALL_ORIG) { - emit_imm(RV_REG_A0, (const s64)im, ctx); + emit_imm(RV_REG_A0, ctx->insns ? (const s64)im : RV_MAX_COUNT_IMM, ctx); ret = emit_call((const u64)__bpf_tramp_enter, true, ctx); if (ret) return ret; @@ -948,13 +979,14 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, } if (flags & BPF_TRAMP_F_CALL_ORIG) { - restore_args(nregs, args_off, ctx); + restore_args(min_t(int, nr_arg_slots, RV_MAX_REG_ARGS), args_off, ctx); + restore_stack_args(nr_arg_slots - RV_MAX_REG_ARGS, args_off, stk_arg_off, ctx); ret = emit_call((const u64)orig_call, true, ctx); if (ret) goto out; emit_sd(RV_REG_FP, -retval_off, RV_REG_A0, ctx); emit_sd(RV_REG_FP, -(retval_off - 8), regmap[BPF_REG_0], ctx); - im->ip_after_call = ctx->insns + ctx->ninsns; + im->ip_after_call = ctx->ro_insns + ctx->ninsns; /* 2 nops reserved for auipc+jalr pair */ emit(rv_nop(), ctx); emit(rv_nop(), ctx); @@ -975,15 +1007,15 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, } if (flags & BPF_TRAMP_F_CALL_ORIG) { - im->ip_epilogue = ctx->insns + ctx->ninsns; - emit_imm(RV_REG_A0, (const s64)im, ctx); + im->ip_epilogue = ctx->ro_insns + ctx->ninsns; + emit_imm(RV_REG_A0, ctx->insns ? (const s64)im : RV_MAX_COUNT_IMM, ctx); ret = emit_call((const u64)__bpf_tramp_exit, true, ctx); if (ret) goto out; } if (flags & BPF_TRAMP_F_RESTORE_REGS) - restore_args(nregs, args_off, ctx); + restore_args(min_t(int, nr_arg_slots, RV_MAX_REG_ARGS), args_off, ctx); if (save_ret) { emit_ld(RV_REG_A0, -retval_off, RV_REG_FP, ctx); @@ -1038,31 +1070,52 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, return ret < 0 ? ret : ninsns_rvoff(ctx.ninsns); } -int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, - void *image_end, const struct btf_func_model *m, +void *arch_alloc_bpf_trampoline(unsigned int size) +{ + return bpf_prog_pack_alloc(size, bpf_fill_ill_insns); +} + +void arch_free_bpf_trampoline(void *image, unsigned int size) +{ + bpf_prog_pack_free(image, size); +} + +int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *ro_image, + void *ro_image_end, const struct btf_func_model *m, u32 flags, struct bpf_tramp_links *tlinks, void *func_addr) { int ret; + void *image, *res; struct rv_jit_context ctx; + u32 size = ro_image_end - ro_image; + + image = kvmalloc(size, GFP_KERNEL); + if (!image) + return -ENOMEM; ctx.ninsns = 0; - /* - * The bpf_int_jit_compile() uses a RW buffer (ctx.insns) to write the - * JITed instructions and later copies it to a RX region (ctx.ro_insns). - * It also uses ctx.ro_insns to calculate offsets for jumps etc. As the - * trampoline image uses the same memory area for writing and execution, - * both ctx.insns and ctx.ro_insns can be set to image. - */ ctx.insns = image; - ctx.ro_insns = image; + ctx.ro_insns = ro_image; ret = __arch_prepare_bpf_trampoline(im, m, tlinks, func_addr, flags, &ctx); if (ret < 0) - return ret; + goto out; - bpf_flush_icache(ctx.insns, ctx.insns + ctx.ninsns); + if (WARN_ON(size < ninsns_rvoff(ctx.ninsns))) { + ret = -E2BIG; + goto out; + } + + res = bpf_arch_text_copy(ro_image, image, size); + if (IS_ERR(res)) { + ret = PTR_ERR(res); + goto out; + } - return ninsns_rvoff(ret); + bpf_flush_icache(ro_image, ro_image_end); +out: + kvfree(image); + return ret < 0 ? ret : size; } int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, @@ -1097,12 +1150,10 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, /* Load current CPU number in T1 */ emit_ld(RV_REG_T1, offsetof(struct thread_info, cpu), RV_REG_TP, ctx); - /* << 3 because offsets are 8 bytes */ - emit_slli(RV_REG_T1, RV_REG_T1, 3, ctx); /* Load address of __per_cpu_offset array in T2 */ emit_addr(RV_REG_T2, (u64)&__per_cpu_offset, extra_pass, ctx); - /* Add offset of current CPU to __per_cpu_offset */ - emit_add(RV_REG_T1, RV_REG_T2, RV_REG_T1, ctx); + /* Get address of __per_cpu_offset[cpu] in T1 */ + emit_sh3add(RV_REG_T1, RV_REG_T1, RV_REG_T2, ctx); /* Load __per_cpu_offset[cpu] in T1 */ emit_ld(RV_REG_T1, 0, RV_REG_T1, ctx); /* Add the offset to Rd */ @@ -1960,7 +2011,7 @@ void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog) { int i, stack_adjust = 0, store_offset, bpf_stack_adjust; - bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); + bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, STACK_ALIGN); if (bpf_stack_adjust) mark_fp(ctx); @@ -1982,7 +2033,7 @@ void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog) if (ctx->arena_vm_start) stack_adjust += 8; - stack_adjust = round_up(stack_adjust, 16); + stack_adjust = round_up(stack_adjust, STACK_ALIGN); stack_adjust += bpf_stack_adjust; store_offset = stack_adjust - 8; diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c index 0a96abdaca65..6de753c667f4 100644 --- a/arch/riscv/net/bpf_jit_core.c +++ b/arch/riscv/net/bpf_jit_core.c @@ -178,8 +178,7 @@ skip_init_ctx: prog->jited_len = prog_size - cfi_get_offset(); if (!prog->is_func || extra_pass) { - if (WARN_ON(bpf_jit_binary_pack_finalize(prog, jit_data->ro_header, - jit_data->header))) { + if (WARN_ON(bpf_jit_binary_pack_finalize(jit_data->ro_header, jit_data->header))) { /* ro_header has been freed */ jit_data->ro_header = NULL; prog = orig_prog; @@ -258,7 +257,7 @@ void bpf_jit_free(struct bpf_prog *prog) * before freeing it. */ if (jit_data) { - bpf_jit_binary_pack_finalize(prog, jit_data->ro_header, jit_data->header); + bpf_jit_binary_pack_finalize(jit_data->ro_header, jit_data->header); kfree(jit_data); } hdr = bpf_jit_binary_pack_hdr(prog); |