aboutsummaryrefslogtreecommitdiff
path: root/arch/riscv/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'arch/riscv/kernel')
-rw-r--r--arch/riscv/kernel/Makefile2
-rw-r--r--arch/riscv/kernel/acpi.c251
-rw-r--r--arch/riscv/kernel/cpu-hotplug.c14
-rw-r--r--arch/riscv/kernel/cpu.c67
-rw-r--r--arch/riscv/kernel/cpufeature.c222
-rw-r--r--arch/riscv/kernel/entry.S8
-rw-r--r--arch/riscv/kernel/head.S41
-rw-r--r--arch/riscv/kernel/hibernate-asm.S5
-rw-r--r--arch/riscv/kernel/hibernate.c1
-rw-r--r--arch/riscv/kernel/irq.c68
-rw-r--r--arch/riscv/kernel/probes/uprobes.c2
-rw-r--r--arch/riscv/kernel/process.c20
-rw-r--r--arch/riscv/kernel/ptrace.c70
-rw-r--r--arch/riscv/kernel/setup.c14
-rw-r--r--arch/riscv/kernel/signal.c220
-rw-r--r--arch/riscv/kernel/smpboot.c86
-rw-r--r--arch/riscv/kernel/sys_riscv.c52
-rw-r--r--arch/riscv/kernel/time.c25
-rw-r--r--arch/riscv/kernel/traps.c61
-rw-r--r--arch/riscv/kernel/vdso/rt_sigreturn.S2
-rw-r--r--arch/riscv/kernel/vector.c276
21 files changed, 1370 insertions, 137 deletions
diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile
index 153864e4f399..506cc4a9a45a 100644
--- a/arch/riscv/kernel/Makefile
+++ b/arch/riscv/kernel/Makefile
@@ -60,6 +60,7 @@ obj-$(CONFIG_MMU) += vdso.o vdso/
obj-$(CONFIG_RISCV_M_MODE) += traps_misaligned.o
obj-$(CONFIG_FPU) += fpu.o
+obj-$(CONFIG_RISCV_ISA_V) += vector.o
obj-$(CONFIG_SMP) += smpboot.o
obj-$(CONFIG_SMP) += smp.o
obj-$(CONFIG_SMP) += cpu_ops.o
@@ -96,3 +97,4 @@ obj-$(CONFIG_COMPAT) += compat_signal.o
obj-$(CONFIG_COMPAT) += compat_vdso/
obj-$(CONFIG_64BIT) += pi/
+obj-$(CONFIG_ACPI) += acpi.o
diff --git a/arch/riscv/kernel/acpi.c b/arch/riscv/kernel/acpi.c
new file mode 100644
index 000000000000..5ee03ebab80e
--- /dev/null
+++ b/arch/riscv/kernel/acpi.c
@@ -0,0 +1,251 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * RISC-V Specific Low-Level ACPI Boot Support
+ *
+ * Copyright (C) 2013-2014, Linaro Ltd.
+ * Author: Al Stone <[email protected]>
+ * Author: Graeme Gregory <[email protected]>
+ * Author: Hanjun Guo <[email protected]>
+ * Author: Tomasz Nowicki <[email protected]>
+ * Author: Naresh Bhat <[email protected]>
+ *
+ * Copyright (C) 2021-2023, Ventana Micro Systems Inc.
+ * Author: Sunil V L <[email protected]>
+ */
+
+#include <linux/acpi.h>
+#include <linux/io.h>
+#include <linux/pci.h>
+#include <linux/efi.h>
+
+int acpi_noirq = 1; /* skip ACPI IRQ initialization */
+int acpi_disabled = 1;
+EXPORT_SYMBOL(acpi_disabled);
+
+int acpi_pci_disabled = 1; /* skip ACPI PCI scan and IRQ initialization */
+EXPORT_SYMBOL(acpi_pci_disabled);
+
+static bool param_acpi_off __initdata;
+static bool param_acpi_on __initdata;
+static bool param_acpi_force __initdata;
+
+static struct acpi_madt_rintc cpu_madt_rintc[NR_CPUS];
+
+static int __init parse_acpi(char *arg)
+{
+ if (!arg)
+ return -EINVAL;
+
+ /* "acpi=off" disables both ACPI table parsing and interpreter */
+ if (strcmp(arg, "off") == 0)
+ param_acpi_off = true;
+ else if (strcmp(arg, "on") == 0) /* prefer ACPI over DT */
+ param_acpi_on = true;
+ else if (strcmp(arg, "force") == 0) /* force ACPI to be enabled */
+ param_acpi_force = true;
+ else
+ return -EINVAL; /* Core will print when we return error */
+
+ return 0;
+}
+early_param("acpi", parse_acpi);
+
+/*
+ * acpi_fadt_sanity_check() - Check FADT presence and carry out sanity
+ * checks on it
+ *
+ * Return 0 on success, <0 on failure
+ */
+static int __init acpi_fadt_sanity_check(void)
+{
+ struct acpi_table_header *table;
+ struct acpi_table_fadt *fadt;
+ acpi_status status;
+ int ret = 0;
+
+ /*
+ * FADT is required on riscv; retrieve it to check its presence
+ * and carry out revision and ACPI HW reduced compliancy tests
+ */
+ status = acpi_get_table(ACPI_SIG_FADT, 0, &table);
+ if (ACPI_FAILURE(status)) {
+ const char *msg = acpi_format_exception(status);
+
+ pr_err("Failed to get FADT table, %s\n", msg);
+ return -ENODEV;
+ }
+
+ fadt = (struct acpi_table_fadt *)table;
+
+ /*
+ * The revision in the table header is the FADT's Major revision. The
+ * FADT also has a minor revision, which is stored in the FADT itself.
+ *
+ * TODO: Currently, we check for 6.5 as the minimum version to check
+ * for HW_REDUCED flag. However, once RISC-V updates are released in
+ * the ACPI spec, we need to update this check for exact minor revision
+ */
+ if (table->revision < 6 || (table->revision == 6 && fadt->minor_revision < 5))
+ pr_err(FW_BUG "Unsupported FADT revision %d.%d, should be 6.5+\n",
+ table->revision, fadt->minor_revision);
+
+ if (!(fadt->flags & ACPI_FADT_HW_REDUCED)) {
+ pr_err("FADT not ACPI hardware reduced compliant\n");
+ ret = -EINVAL;
+ }
+
+ /*
+ * acpi_get_table() creates FADT table mapping that
+ * should be released after parsing and before resuming boot
+ */
+ acpi_put_table(table);
+ return ret;
+}
+
+/*
+ * acpi_boot_table_init() called from setup_arch(), always.
+ * 1. find RSDP and get its address, and then find XSDT
+ * 2. extract all tables and checksums them all
+ * 3. check ACPI FADT HW reduced flag
+ *
+ * We can parse ACPI boot-time tables such as MADT after
+ * this function is called.
+ *
+ * On return ACPI is enabled if either:
+ *
+ * - ACPI tables are initialized and sanity checks passed
+ * - acpi=force was passed in the command line and ACPI was not disabled
+ * explicitly through acpi=off command line parameter
+ *
+ * ACPI is disabled on function return otherwise
+ */
+void __init acpi_boot_table_init(void)
+{
+ /*
+ * Enable ACPI instead of device tree unless
+ * - ACPI has been disabled explicitly (acpi=off), or
+ * - firmware has not populated ACPI ptr in EFI system table
+ * and ACPI has not been [force] enabled (acpi=on|force)
+ */
+ if (param_acpi_off ||
+ (!param_acpi_on && !param_acpi_force &&
+ efi.acpi20 == EFI_INVALID_TABLE_ADDR))
+ return;
+
+ /*
+ * ACPI is disabled at this point. Enable it in order to parse
+ * the ACPI tables and carry out sanity checks
+ */
+ enable_acpi();
+
+ /*
+ * If ACPI tables are initialized and FADT sanity checks passed,
+ * leave ACPI enabled and carry on booting; otherwise disable ACPI
+ * on initialization error.
+ * If acpi=force was passed on the command line it forces ACPI
+ * to be enabled even if its initialization failed.
+ */
+ if (acpi_table_init() || acpi_fadt_sanity_check()) {
+ pr_err("Failed to init ACPI tables\n");
+ if (!param_acpi_force)
+ disable_acpi();
+ }
+}
+
+static int acpi_parse_madt_rintc(union acpi_subtable_headers *header, const unsigned long end)
+{
+ struct acpi_madt_rintc *rintc = (struct acpi_madt_rintc *)header;
+ int cpuid;
+
+ if (!(rintc->flags & ACPI_MADT_ENABLED))
+ return 0;
+
+ cpuid = riscv_hartid_to_cpuid(rintc->hart_id);
+ /*
+ * When CONFIG_SMP is disabled, mapping won't be created for
+ * all cpus.
+ * CPUs more than num_possible_cpus, will be ignored.
+ */
+ if (cpuid >= 0 && cpuid < num_possible_cpus())
+ cpu_madt_rintc[cpuid] = *rintc;
+
+ return 0;
+}
+
+/*
+ * Instead of parsing (and freeing) the ACPI table, cache
+ * the RINTC structures since they are frequently used
+ * like in cpuinfo.
+ */
+void __init acpi_init_rintc_map(void)
+{
+ if (acpi_table_parse_madt(ACPI_MADT_TYPE_RINTC, acpi_parse_madt_rintc, 0) <= 0) {
+ pr_err("No valid RINTC entries exist\n");
+ BUG();
+ }
+}
+
+struct acpi_madt_rintc *acpi_cpu_get_madt_rintc(int cpu)
+{
+ return &cpu_madt_rintc[cpu];
+}
+
+u32 get_acpi_id_for_cpu(int cpu)
+{
+ return acpi_cpu_get_madt_rintc(cpu)->uid;
+}
+
+/*
+ * __acpi_map_table() will be called before paging_init(), so early_ioremap()
+ * or early_memremap() should be called here to for ACPI table mapping.
+ */
+void __init __iomem *__acpi_map_table(unsigned long phys, unsigned long size)
+{
+ if (!size)
+ return NULL;
+
+ return early_ioremap(phys, size);
+}
+
+void __init __acpi_unmap_table(void __iomem *map, unsigned long size)
+{
+ if (!map || !size)
+ return;
+
+ early_iounmap(map, size);
+}
+
+void *acpi_os_ioremap(acpi_physical_address phys, acpi_size size)
+{
+ return memremap(phys, size, MEMREMAP_WB);
+}
+
+#ifdef CONFIG_PCI
+
+/*
+ * These interfaces are defined just to enable building ACPI core.
+ * TODO: Update it with actual implementation when external interrupt
+ * controller support is added in RISC-V ACPI.
+ */
+int raw_pci_read(unsigned int domain, unsigned int bus, unsigned int devfn,
+ int reg, int len, u32 *val)
+{
+ return PCIBIOS_DEVICE_NOT_FOUND;
+}
+
+int raw_pci_write(unsigned int domain, unsigned int bus, unsigned int devfn,
+ int reg, int len, u32 val)
+{
+ return PCIBIOS_DEVICE_NOT_FOUND;
+}
+
+int acpi_pci_bus_find_domain_nr(struct pci_bus *bus)
+{
+ return -1;
+}
+
+struct pci_bus *pci_acpi_scan_root(struct acpi_pci_root *root)
+{
+ return NULL;
+}
+#endif /* CONFIG_PCI */
diff --git a/arch/riscv/kernel/cpu-hotplug.c b/arch/riscv/kernel/cpu-hotplug.c
index a941adc7cbf2..457a18efcb11 100644
--- a/arch/riscv/kernel/cpu-hotplug.c
+++ b/arch/riscv/kernel/cpu-hotplug.c
@@ -8,6 +8,7 @@
#include <linux/sched.h>
#include <linux/err.h>
#include <linux/irq.h>
+#include <linux/cpuhotplug.h>
#include <linux/cpu.h>
#include <linux/sched/hotplug.h>
#include <asm/irq.h>
@@ -49,17 +50,15 @@ int __cpu_disable(void)
return ret;
}
+#ifdef CONFIG_HOTPLUG_CPU
/*
- * Called on the thread which is asking for a CPU to be shutdown.
+ * Called on the thread which is asking for a CPU to be shutdown, if the
+ * CPU reported dead to the hotplug core.
*/
-void __cpu_die(unsigned int cpu)
+void arch_cpuhp_cleanup_dead_cpu(unsigned int cpu)
{
int ret = 0;
- if (!cpu_wait_death(cpu, 5)) {
- pr_err("CPU %u: didn't die\n", cpu);
- return;
- }
pr_notice("CPU%u: off\n", cpu);
/* Verify from the firmware if the cpu is really stopped*/
@@ -76,9 +75,10 @@ void __noreturn arch_cpu_idle_dead(void)
{
idle_task_exit();
- (void)cpu_report_death();
+ cpuhp_ap_report_dead();
cpu_ops[smp_processor_id()]->cpu_stop();
/* It should never reach here */
BUG();
}
+#endif
diff --git a/arch/riscv/kernel/cpu.c b/arch/riscv/kernel/cpu.c
index c96aa56cf1c7..a2fc952318e9 100644
--- a/arch/riscv/kernel/cpu.c
+++ b/arch/riscv/kernel/cpu.c
@@ -3,10 +3,13 @@
* Copyright (C) 2012 Regents of the University of California
*/
+#include <linux/acpi.h>
#include <linux/cpu.h>
+#include <linux/ctype.h>
#include <linux/init.h>
#include <linux/seq_file.h>
#include <linux/of.h>
+#include <asm/acpi.h>
#include <asm/cpufeature.h>
#include <asm/csr.h>
#include <asm/hwcap.h>
@@ -20,6 +23,26 @@
*/
int riscv_of_processor_hartid(struct device_node *node, unsigned long *hart)
{
+ int cpu;
+
+ *hart = (unsigned long)of_get_cpu_hwid(node, 0);
+ if (*hart == ~0UL) {
+ pr_warn("Found CPU without hart ID\n");
+ return -ENODEV;
+ }
+
+ cpu = riscv_hartid_to_cpuid(*hart);
+ if (cpu < 0)
+ return cpu;
+
+ if (!cpu_possible(cpu))
+ return -ENODEV;
+
+ return 0;
+}
+
+int riscv_early_of_processor_hartid(struct device_node *node, unsigned long *hart)
+{
const char *isa;
if (!of_device_is_compatible(node, "riscv")) {
@@ -27,7 +50,7 @@ int riscv_of_processor_hartid(struct device_node *node, unsigned long *hart)
return -ENODEV;
}
- *hart = (unsigned long) of_get_cpu_hwid(node, 0);
+ *hart = (unsigned long)of_get_cpu_hwid(node, 0);
if (*hart == ~0UL) {
pr_warn("Found CPU without hart ID\n");
return -ENODEV;
@@ -42,10 +65,12 @@ int riscv_of_processor_hartid(struct device_node *node, unsigned long *hart)
pr_warn("CPU with hartid=%lu has no \"riscv,isa\" property\n", *hart);
return -ENODEV;
}
- if (isa[0] != 'r' || isa[1] != 'v') {
- pr_warn("CPU with hartid=%lu has an invalid ISA of \"%s\"\n", *hart, isa);
+
+ if (IS_ENABLED(CONFIG_32BIT) && strncasecmp(isa, "rv32ima", 7))
+ return -ENODEV;
+
+ if (IS_ENABLED(CONFIG_64BIT) && strncasecmp(isa, "rv64ima", 7))
return -ENODEV;
- }
return 0;
}
@@ -183,8 +208,14 @@ arch_initcall(riscv_cpuinfo_init);
static struct riscv_isa_ext_data isa_ext_arr[] = {
__RISCV_ISA_EXT_DATA(zicbom, RISCV_ISA_EXT_ZICBOM),
__RISCV_ISA_EXT_DATA(zicboz, RISCV_ISA_EXT_ZICBOZ),
+ __RISCV_ISA_EXT_DATA(zicntr, RISCV_ISA_EXT_ZICNTR),
+ __RISCV_ISA_EXT_DATA(zicsr, RISCV_ISA_EXT_ZICSR),
+ __RISCV_ISA_EXT_DATA(zifencei, RISCV_ISA_EXT_ZIFENCEI),
__RISCV_ISA_EXT_DATA(zihintpause, RISCV_ISA_EXT_ZIHINTPAUSE),
+ __RISCV_ISA_EXT_DATA(zihpm, RISCV_ISA_EXT_ZIHPM),
+ __RISCV_ISA_EXT_DATA(zba, RISCV_ISA_EXT_ZBA),
__RISCV_ISA_EXT_DATA(zbb, RISCV_ISA_EXT_ZBB),
+ __RISCV_ISA_EXT_DATA(zbs, RISCV_ISA_EXT_ZBS),
__RISCV_ISA_EXT_DATA(smaia, RISCV_ISA_EXT_SMAIA),
__RISCV_ISA_EXT_DATA(ssaia, RISCV_ISA_EXT_SSAIA),
__RISCV_ISA_EXT_DATA(sscofpmf, RISCV_ISA_EXT_SSCOFPMF),
@@ -283,23 +314,35 @@ static void c_stop(struct seq_file *m, void *v)
static int c_show(struct seq_file *m, void *v)
{
unsigned long cpu_id = (unsigned long)v - 1;
- struct device_node *node = of_get_cpu_node(cpu_id, NULL);
struct riscv_cpuinfo *ci = per_cpu_ptr(&riscv_cpuinfo, cpu_id);
+ struct device_node *node;
const char *compat, *isa;
seq_printf(m, "processor\t: %lu\n", cpu_id);
seq_printf(m, "hart\t\t: %lu\n", cpuid_to_hartid_map(cpu_id));
- if (!of_property_read_string(node, "riscv,isa", &isa))
- print_isa(m, isa);
- print_mmu(m);
- if (!of_property_read_string(node, "compatible", &compat)
- && strcmp(compat, "riscv"))
- seq_printf(m, "uarch\t\t: %s\n", compat);
+
+ if (acpi_disabled) {
+ node = of_get_cpu_node(cpu_id, NULL);
+ if (!of_property_read_string(node, "riscv,isa", &isa))
+ print_isa(m, isa);
+
+ print_mmu(m);
+ if (!of_property_read_string(node, "compatible", &compat) &&
+ strcmp(compat, "riscv"))
+ seq_printf(m, "uarch\t\t: %s\n", compat);
+
+ of_node_put(node);
+ } else {
+ if (!acpi_get_riscv_isa(NULL, cpu_id, &isa))
+ print_isa(m, isa);
+
+ print_mmu(m);
+ }
+
seq_printf(m, "mvendorid\t: 0x%lx\n", ci->mvendorid);
seq_printf(m, "marchid\t\t: 0x%lx\n", ci->marchid);
seq_printf(m, "mimpid\t\t: 0x%lx\n", ci->mimpid);
seq_puts(m, "\n");
- of_node_put(node);
return 0;
}
diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c
index b1d6b7e4b829..bdcf460ea53d 100644
--- a/arch/riscv/kernel/cpufeature.c
+++ b/arch/riscv/kernel/cpufeature.c
@@ -6,18 +6,22 @@
* Copyright (C) 2017 SiFive
*/
+#include <linux/acpi.h>
#include <linux/bitmap.h>
#include <linux/ctype.h>
#include <linux/log2.h>
#include <linux/memory.h>
#include <linux/module.h>
#include <linux/of.h>
+#include <linux/of_device.h>
+#include <asm/acpi.h>
#include <asm/alternative.h>
#include <asm/cacheflush.h>
#include <asm/cpufeature.h>
#include <asm/hwcap.h>
#include <asm/patch.h>
#include <asm/processor.h>
+#include <asm/vector.h>
#define NUM_ALPHA_EXTS ('z' - 'a' + 1)
@@ -26,6 +30,9 @@ unsigned long elf_hwcap __read_mostly;
/* Host ISA bitmap */
static DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX) __read_mostly;
+/* Per-cpu ISA extensions. */
+struct riscv_isainfo hart_isa[NR_CPUS];
+
/* Performance information */
DEFINE_PER_CPU(long, misaligned_access_speed);
@@ -71,10 +78,10 @@ static bool riscv_isa_extension_check(int id)
switch (id) {
case RISCV_ISA_EXT_ZICBOM:
if (!riscv_cbom_block_size) {
- pr_err("Zicbom detected in ISA string, but no cbom-block-size found\n");
+ pr_err("Zicbom detected in ISA string, disabling as no cbom-block-size found\n");
return false;
} else if (!is_power_of_2(riscv_cbom_block_size)) {
- pr_err("cbom-block-size present, but is not a power-of-2\n");
+ pr_err("Zicbom disabled as cbom-block-size present, but is not a power-of-2\n");
return false;
}
return true;
@@ -99,7 +106,9 @@ void __init riscv_fill_hwcap(void)
char print_str[NUM_ALPHA_EXTS + 1];
int i, j, rc;
unsigned long isa2hwcap[26] = {0};
- unsigned long hartid;
+ struct acpi_table_header *rhct;
+ acpi_status status;
+ unsigned int cpu;
isa2hwcap['i' - 'a'] = COMPAT_HWCAP_ISA_I;
isa2hwcap['m' - 'a'] = COMPAT_HWCAP_ISA_M;
@@ -107,45 +116,59 @@ void __init riscv_fill_hwcap(void)
isa2hwcap['f' - 'a'] = COMPAT_HWCAP_ISA_F;
isa2hwcap['d' - 'a'] = COMPAT_HWCAP_ISA_D;
isa2hwcap['c' - 'a'] = COMPAT_HWCAP_ISA_C;
+ isa2hwcap['v' - 'a'] = COMPAT_HWCAP_ISA_V;
elf_hwcap = 0;
bitmap_zero(riscv_isa, RISCV_ISA_EXT_MAX);
- for_each_of_cpu_node(node) {
+ if (!acpi_disabled) {
+ status = acpi_get_table(ACPI_SIG_RHCT, 0, &rhct);
+ if (ACPI_FAILURE(status))
+ return;
+ }
+
+ for_each_possible_cpu(cpu) {
+ struct riscv_isainfo *isainfo = &hart_isa[cpu];
unsigned long this_hwcap = 0;
- DECLARE_BITMAP(this_isa, RISCV_ISA_EXT_MAX);
- const char *temp;
- rc = riscv_of_processor_hartid(node, &hartid);
- if (rc < 0)
- continue;
+ if (acpi_disabled) {
+ node = of_cpu_device_node_get(cpu);
+ if (!node) {
+ pr_warn("Unable to find cpu node\n");
+ continue;
+ }
- if (of_property_read_string(node, "riscv,isa", &isa)) {
- pr_warn("Unable to find \"riscv,isa\" devicetree entry\n");
- continue;
+ rc = of_property_read_string(node, "riscv,isa", &isa);
+ of_node_put(node);
+ if (rc) {
+ pr_warn("Unable to find \"riscv,isa\" devicetree entry\n");
+ continue;
+ }
+ } else {
+ rc = acpi_get_riscv_isa(rhct, cpu, &isa);
+ if (rc < 0) {
+ pr_warn("Unable to get ISA for the hart - %d\n", cpu);
+ continue;
+ }
}
- temp = isa;
-#if IS_ENABLED(CONFIG_32BIT)
- if (!strncmp(isa, "rv32", 4))
- isa += 4;
-#elif IS_ENABLED(CONFIG_64BIT)
- if (!strncmp(isa, "rv64", 4))
- isa += 4;
-#endif
- /* The riscv,isa DT property must start with rv64 or rv32 */
- if (temp == isa)
- continue;
- bitmap_zero(this_isa, RISCV_ISA_EXT_MAX);
- for (; *isa; ++isa) {
+ /*
+ * For all possible cpus, we have already validated in
+ * the boot process that they at least contain "rv" and
+ * whichever of "32"/"64" this kernel supports, and so this
+ * section can be skipped.
+ */
+ isa += 4;
+
+ while (*isa) {
const char *ext = isa++;
const char *ext_end = isa;
bool ext_long = false, ext_err = false;
switch (*ext) {
case 's':
- /**
+ /*
* Workaround for invalid single-letter 's' & 'u'(QEMU).
* No need to set the bit in riscv_isa as 's' & 'u' are
* not valid ISA extensions. It works until multi-letter
@@ -157,75 +180,123 @@ void __init riscv_fill_hwcap(void)
break;
}
fallthrough;
+ case 'S':
case 'x':
+ case 'X':
case 'z':
+ case 'Z':
+ /*
+ * Before attempting to parse the extension itself, we find its end.
+ * As multi-letter extensions must be split from other multi-letter
+ * extensions with an "_", the end of a multi-letter extension will
+ * either be the null character or the "_" at the start of the next
+ * multi-letter extension.
+ *
+ * Next, as the extensions version is currently ignored, we
+ * eliminate that portion. This is done by parsing backwards from
+ * the end of the extension, removing any numbers. This may be a
+ * major or minor number however, so the process is repeated if a
+ * minor number was found.
+ *
+ * ext_end is intended to represent the first character *after* the
+ * name portion of an extension, but will be decremented to the last
+ * character itself while eliminating the extensions version number.
+ * A simple re-increment solves this problem.
+ */
ext_long = true;
- /* Multi-letter extension must be delimited */
for (; *isa && *isa != '_'; ++isa)
- if (unlikely(!islower(*isa)
- && !isdigit(*isa)))
+ if (unlikely(!isalnum(*isa)))
ext_err = true;
- /* Parse backwards */
+
ext_end = isa;
if (unlikely(ext_err))
break;
+
if (!isdigit(ext_end[-1]))
break;
- /* Skip the minor version */
+
while (isdigit(*--ext_end))
;
- if (ext_end[0] != 'p'
- || !isdigit(ext_end[-1])) {
- /* Advance it to offset the pre-decrement */
+
+ if (tolower(ext_end[0]) != 'p' || !isdigit(ext_end[-1])) {
++ext_end;
break;
}
- /* Skip the major version */
+
while (isdigit(*--ext_end))
;
+
++ext_end;
break;
default:
- if (unlikely(!islower(*ext))) {
+ /*
+ * Things are a little easier for single-letter extensions, as they
+ * are parsed forwards.
+ *
+ * After checking that our starting position is valid, we need to
+ * ensure that, when isa was incremented at the start of the loop,
+ * that it arrived at the start of the next extension.
+ *
+ * If we are already on a non-digit, there is nothing to do. Either
+ * we have a multi-letter extension's _, or the start of an
+ * extension.
+ *
+ * Otherwise we have found the current extension's major version
+ * number. Parse past it, and a subsequent p/minor version number
+ * if present. The `p` extension must not appear immediately after
+ * a number, so there is no fear of missing it.
+ *
+ */
+ if (unlikely(!isalpha(*ext))) {
ext_err = true;
break;
}
- /* Find next extension */
+
if (!isdigit(*isa))
break;
- /* Skip the minor version */
+
while (isdigit(*++isa))
;
- if (*isa != 'p')
+
+ if (tolower(*isa) != 'p')
break;
+
if (!isdigit(*++isa)) {
--isa;
break;
}
- /* Skip the major version */
+
while (isdigit(*++isa))
;
+
break;
}
- if (*isa != '_')
- --isa;
-#define SET_ISA_EXT_MAP(name, bit) \
- do { \
- if ((ext_end - ext == sizeof(name) - 1) && \
- !memcmp(ext, name, sizeof(name) - 1) && \
- riscv_isa_extension_check(bit)) \
- set_bit(bit, this_isa); \
- } while (false) \
+ /*
+ * The parser expects that at the start of an iteration isa points to the
+ * first character of the next extension. As we stop parsing an extension
+ * on meeting a non-alphanumeric character, an extra increment is needed
+ * where the succeeding extension is a multi-letter prefixed with an "_".
+ */
+ if (*isa == '_')
+ ++isa;
+
+#define SET_ISA_EXT_MAP(name, bit) \
+ do { \
+ if ((ext_end - ext == sizeof(name) - 1) && \
+ !strncasecmp(ext, name, sizeof(name) - 1) && \
+ riscv_isa_extension_check(bit)) \
+ set_bit(bit, isainfo->isa); \
+ } while (false) \
if (unlikely(ext_err))
continue;
if (!ext_long) {
- int nr = *ext - 'a';
+ int nr = tolower(*ext) - 'a';
if (riscv_isa_extension_check(nr)) {
this_hwcap |= isa2hwcap[nr];
- set_bit(nr, this_isa);
+ set_bit(nr, isainfo->isa);
}
} else {
/* sorted alphabetically */
@@ -236,7 +307,9 @@ void __init riscv_fill_hwcap(void)
SET_ISA_EXT_MAP("svinval", RISCV_ISA_EXT_SVINVAL);
SET_ISA_EXT_MAP("svnapot", RISCV_ISA_EXT_SVNAPOT);
SET_ISA_EXT_MAP("svpbmt", RISCV_ISA_EXT_SVPBMT);
+ SET_ISA_EXT_MAP("zba", RISCV_ISA_EXT_ZBA);
SET_ISA_EXT_MAP("zbb", RISCV_ISA_EXT_ZBB);
+ SET_ISA_EXT_MAP("zbs", RISCV_ISA_EXT_ZBS);
SET_ISA_EXT_MAP("zicbom", RISCV_ISA_EXT_ZICBOM);
SET_ISA_EXT_MAP("zicboz", RISCV_ISA_EXT_ZICBOZ);
SET_ISA_EXT_MAP("zihintpause", RISCV_ISA_EXT_ZIHINTPAUSE);
@@ -245,6 +318,23 @@ void __init riscv_fill_hwcap(void)
}
/*
+ * Linux requires the following extensions, so we may as well
+ * always set them.
+ */
+ set_bit(RISCV_ISA_EXT_ZICSR, isainfo->isa);
+ set_bit(RISCV_ISA_EXT_ZIFENCEI, isainfo->isa);
+
+ /*
+ * These ones were as they were part of the base ISA when the
+ * port & dt-bindings were upstreamed, and so can be set
+ * unconditionally where `i` is in riscv,isa on DT systems.
+ */
+ if (acpi_disabled) {
+ set_bit(RISCV_ISA_EXT_ZICNTR, isainfo->isa);
+ set_bit(RISCV_ISA_EXT_ZIHPM, isainfo->isa);
+ }
+
+ /*
* All "okay" hart should have same isa. Set HWCAP based on
* common capabilities of every "okay" hart, in case they don't
* have.
@@ -255,11 +345,14 @@ void __init riscv_fill_hwcap(void)
elf_hwcap = this_hwcap;
if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX))
- bitmap_copy(riscv_isa, this_isa, RISCV_ISA_EXT_MAX);
+ bitmap_copy(riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX);
else
- bitmap_and(riscv_isa, riscv_isa, this_isa, RISCV_ISA_EXT_MAX);
+ bitmap_and(riscv_isa, riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX);
}
+ if (!acpi_disabled && rhct)
+ acpi_put_table((struct acpi_table_header *)rhct);
+
/* We don't support systems with F but without D, so mask those out
* here. */
if ((elf_hwcap & COMPAT_HWCAP_ISA_F) && !(elf_hwcap & COMPAT_HWCAP_ISA_D)) {
@@ -267,6 +360,17 @@ void __init riscv_fill_hwcap(void)
elf_hwcap &= ~COMPAT_HWCAP_ISA_F;
}
+ if (elf_hwcap & COMPAT_HWCAP_ISA_V) {
+ riscv_v_setup_vsize();
+ /*
+ * ISA string in device tree might have 'v' flag, but
+ * CONFIG_RISCV_ISA_V is disabled in kernel.
+ * Clear V flag in elf_hwcap if CONFIG_RISCV_ISA_V is disabled.
+ */
+ if (!IS_ENABLED(CONFIG_RISCV_ISA_V))
+ elf_hwcap &= ~COMPAT_HWCAP_ISA_V;
+ }
+
memset(print_str, 0, sizeof(print_str));
for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++)
if (riscv_isa[0] & BIT_MASK(i))
@@ -280,6 +384,18 @@ void __init riscv_fill_hwcap(void)
pr_info("riscv: ELF capabilities %s\n", print_str);
}
+unsigned long riscv_get_elf_hwcap(void)
+{
+ unsigned long hwcap;
+
+ hwcap = (elf_hwcap & ((1UL << RISCV_ISA_EXT_BASE) - 1));
+
+ if (!riscv_v_vstate_ctrl_user_allowed())
+ hwcap &= ~COMPAT_HWCAP_ISA_V;
+
+ return hwcap;
+}
+
#ifdef CONFIG_RISCV_ALTERNATIVE
/*
* Alternative patch sites consider 48 bits when determining when to patch
diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S
index 3fbb100bc9e4..143a2bb3e697 100644
--- a/arch/riscv/kernel/entry.S
+++ b/arch/riscv/kernel/entry.S
@@ -48,10 +48,10 @@ _save_context:
* Disable user-mode memory access as it should only be set in the
* actual user copy routines.
*
- * Disable the FPU to detect illegal usage of floating point in kernel
- * space.
+ * Disable the FPU/Vector to detect illegal usage of floating point
+ * or vector in kernel space.
*/
- li t0, SR_SUM | SR_FS
+ li t0, SR_SUM | SR_FS_VS
REG_L s0, TASK_TI_USER_SP(tp)
csrrc s1, CSR_STATUS, t0
@@ -348,6 +348,6 @@ SYM_CODE_END(excp_vect_table)
#ifndef CONFIG_MMU
SYM_CODE_START(__user_rt_sigreturn)
li a7, __NR_rt_sigreturn
- scall
+ ecall
SYM_CODE_END(__user_rt_sigreturn)
#endif
diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S
index 4bf6c449d78b..11c3b94c4534 100644
--- a/arch/riscv/kernel/head.S
+++ b/arch/riscv/kernel/head.S
@@ -140,10 +140,10 @@ secondary_start_sbi:
.option pop
/*
- * Disable FPU to detect illegal usage of
- * floating point in kernel space
+ * Disable FPU & VECTOR to detect illegal usage of
+ * floating point or vector in kernel space
*/
- li t0, SR_FS
+ li t0, SR_FS_VS
csrc CSR_STATUS, t0
/* Set trap vector to spin forever to help debug */
@@ -234,10 +234,10 @@ pmp_done:
.option pop
/*
- * Disable FPU to detect illegal usage of
- * floating point in kernel space
+ * Disable FPU & VECTOR to detect illegal usage of
+ * floating point or vector in kernel space
*/
- li t0, SR_FS
+ li t0, SR_FS_VS
csrc CSR_STATUS, t0
#ifdef CONFIG_RISCV_BOOT_SPINWAIT
@@ -301,6 +301,7 @@ clear_bss_done:
la tp, init_task
la sp, init_thread_union + THREAD_SIZE
XIP_FIXUP_OFFSET sp
+ addi sp, sp, -PT_SIZE_ON_STACK
#ifdef CONFIG_BUILTIN_DTB
la a0, __dtb_start
XIP_FIXUP_OFFSET a0
@@ -318,6 +319,7 @@ clear_bss_done:
/* Restore C environment */
la tp, init_task
la sp, init_thread_union + THREAD_SIZE
+ addi sp, sp, -PT_SIZE_ON_STACK
#ifdef CONFIG_KASAN
call kasan_early_init
@@ -392,7 +394,7 @@ ENTRY(reset_regs)
#ifdef CONFIG_FPU
csrr t0, CSR_MISA
andi t0, t0, (COMPAT_HWCAP_ISA_F | COMPAT_HWCAP_ISA_D)
- beqz t0, .Lreset_regs_done
+ beqz t0, .Lreset_regs_done_fpu
li t1, SR_FS
csrs CSR_STATUS, t1
@@ -430,8 +432,31 @@ ENTRY(reset_regs)
fmv.s.x f31, zero
csrw fcsr, 0
/* note that the caller must clear SR_FS */
+.Lreset_regs_done_fpu:
#endif /* CONFIG_FPU */
-.Lreset_regs_done:
+
+#ifdef CONFIG_RISCV_ISA_V
+ csrr t0, CSR_MISA
+ li t1, COMPAT_HWCAP_ISA_V
+ and t0, t0, t1
+ beqz t0, .Lreset_regs_done_vector
+
+ /*
+ * Clear vector registers and reset vcsr
+ * VLMAX has a defined value, VLEN is a constant,
+ * and this form of vsetvli is defined to set vl to VLMAX.
+ */
+ li t1, SR_VS
+ csrs CSR_STATUS, t1
+ csrs CSR_VCSR, x0
+ vsetvli t1, x0, e8, m8, ta, ma
+ vmv.v.i v0, 0
+ vmv.v.i v8, 0
+ vmv.v.i v16, 0
+ vmv.v.i v24, 0
+ /* note that the caller must clear SR_VS */
+.Lreset_regs_done_vector:
+#endif /* CONFIG_RISCV_ISA_V */
ret
END(reset_regs)
#endif /* CONFIG_RISCV_M_MODE */
diff --git a/arch/riscv/kernel/hibernate-asm.S b/arch/riscv/kernel/hibernate-asm.S
index effaf5ca5da0..d698dd7df637 100644
--- a/arch/riscv/kernel/hibernate-asm.S
+++ b/arch/riscv/kernel/hibernate-asm.S
@@ -28,7 +28,6 @@ ENTRY(__hibernate_cpu_resume)
REG_L a0, hibernate_cpu_context
- suspend_restore_csrs
suspend_restore_regs
/* Return zero value. */
@@ -50,7 +49,7 @@ ENTRY(hibernate_restore_image)
REG_L s4, restore_pblist
REG_L a1, relocated_restore_code
- jalr a1
+ jr a1
END(hibernate_restore_image)
/*
@@ -73,5 +72,5 @@ ENTRY(hibernate_core_restore_code)
REG_L s4, HIBERN_PBE_NEXT(s4)
bnez s4, .Lcopy
- jalr s2
+ jr s2
END(hibernate_core_restore_code)
diff --git a/arch/riscv/kernel/hibernate.c b/arch/riscv/kernel/hibernate.c
index 264b2dcdd67e..671b686c0158 100644
--- a/arch/riscv/kernel/hibernate.c
+++ b/arch/riscv/kernel/hibernate.c
@@ -80,7 +80,6 @@ int pfn_is_nosave(unsigned long pfn)
void notrace save_processor_state(void)
{
- WARN_ON(num_online_cpus() != 1);
}
void notrace restore_processor_state(void)
diff --git a/arch/riscv/kernel/irq.c b/arch/riscv/kernel/irq.c
index eb9a68a539e6..d0577cc6a081 100644
--- a/arch/riscv/kernel/irq.c
+++ b/arch/riscv/kernel/irq.c
@@ -11,6 +11,9 @@
#include <linux/module.h>
#include <linux/seq_file.h>
#include <asm/sbi.h>
+#include <asm/smp.h>
+#include <asm/softirq_stack.h>
+#include <asm/stacktrace.h>
static struct fwnode_handle *(*__get_intc_node)(void);
@@ -28,6 +31,70 @@ struct fwnode_handle *riscv_get_intc_hwnode(void)
}
EXPORT_SYMBOL_GPL(riscv_get_intc_hwnode);
+#ifdef CONFIG_IRQ_STACKS
+#include <asm/irq_stack.h>
+
+DEFINE_PER_CPU(ulong *, irq_stack_ptr);
+
+#ifdef CONFIG_VMAP_STACK
+static void init_irq_stacks(void)
+{
+ int cpu;
+ ulong *p;
+
+ for_each_possible_cpu(cpu) {
+ p = arch_alloc_vmap_stack(IRQ_STACK_SIZE, cpu_to_node(cpu));
+ per_cpu(irq_stack_ptr, cpu) = p;
+ }
+}
+#else
+/* irq stack only needs to be 16 byte aligned - not IRQ_STACK_SIZE aligned. */
+DEFINE_PER_CPU_ALIGNED(ulong [IRQ_STACK_SIZE/sizeof(ulong)], irq_stack);
+
+static void init_irq_stacks(void)
+{
+ int cpu;
+
+ for_each_possible_cpu(cpu)
+ per_cpu(irq_stack_ptr, cpu) = per_cpu(irq_stack, cpu);
+}
+#endif /* CONFIG_VMAP_STACK */
+
+#ifdef CONFIG_HAVE_SOFTIRQ_ON_OWN_STACK
+void do_softirq_own_stack(void)
+{
+#ifdef CONFIG_IRQ_STACKS
+ if (on_thread_stack()) {
+ ulong *sp = per_cpu(irq_stack_ptr, smp_processor_id())
+ + IRQ_STACK_SIZE/sizeof(ulong);
+ __asm__ __volatile(
+ "addi sp, sp, -"RISCV_SZPTR "\n"
+ REG_S" ra, (sp) \n"
+ "addi sp, sp, -"RISCV_SZPTR "\n"
+ REG_S" s0, (sp) \n"
+ "addi s0, sp, 2*"RISCV_SZPTR "\n"
+ "move sp, %[sp] \n"
+ "call __do_softirq \n"
+ "addi sp, s0, -2*"RISCV_SZPTR"\n"
+ REG_L" s0, (sp) \n"
+ "addi sp, sp, "RISCV_SZPTR "\n"
+ REG_L" ra, (sp) \n"
+ "addi sp, sp, "RISCV_SZPTR "\n"
+ :
+ : [sp] "r" (sp)
+ : "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7",
+ "t0", "t1", "t2", "t3", "t4", "t5", "t6",
+ "memory");
+ } else
+#endif
+ __do_softirq();
+}
+#endif /* CONFIG_HAVE_SOFTIRQ_ON_OWN_STACK */
+
+#else
+static void init_irq_stacks(void) {}
+#endif /* CONFIG_IRQ_STACKS */
+
int arch_show_interrupts(struct seq_file *p, int prec)
{
show_ipi_stats(p, prec);
@@ -36,6 +103,7 @@ int arch_show_interrupts(struct seq_file *p, int prec)
void __init init_IRQ(void)
{
+ init_irq_stacks();
irqchip_init();
if (!handle_arch_irq)
panic("No interrupt controller found.");
diff --git a/arch/riscv/kernel/probes/uprobes.c b/arch/riscv/kernel/probes/uprobes.c
index c976a21cd4bd..194f166b2cc4 100644
--- a/arch/riscv/kernel/probes/uprobes.c
+++ b/arch/riscv/kernel/probes/uprobes.c
@@ -67,6 +67,7 @@ int arch_uprobe_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
struct uprobe_task *utask = current->utask;
WARN_ON_ONCE(current->thread.bad_cause != UPROBE_TRAP_NR);
+ current->thread.bad_cause = utask->autask.saved_cause;
instruction_pointer_set(regs, utask->vaddr + auprobe->insn_size);
@@ -102,6 +103,7 @@ void arch_uprobe_abort_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
{
struct uprobe_task *utask = current->utask;
+ current->thread.bad_cause = utask->autask.saved_cause;
/*
* Task has received a fatal signal, so reset back to probbed
* address.
diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c
index e2a060066730..e32d737e039f 100644
--- a/arch/riscv/kernel/process.c
+++ b/arch/riscv/kernel/process.c
@@ -24,6 +24,7 @@
#include <asm/switch_to.h>
#include <asm/thread_info.h>
#include <asm/cpuidle.h>
+#include <asm/vector.h>
register unsigned long gp_in_global __asm__("gp");
@@ -146,12 +147,29 @@ void flush_thread(void)
fstate_off(current, task_pt_regs(current));
memset(&current->thread.fstate, 0, sizeof(current->thread.fstate));
#endif
+#ifdef CONFIG_RISCV_ISA_V
+ /* Reset vector state */
+ riscv_v_vstate_ctrl_init(current);
+ riscv_v_vstate_off(task_pt_regs(current));
+ kfree(current->thread.vstate.datap);
+ memset(&current->thread.vstate, 0, sizeof(struct __riscv_v_ext_state));
+#endif
+}
+
+void arch_release_task_struct(struct task_struct *tsk)
+{
+ /* Free the vector context of datap. */
+ if (has_vector())
+ kfree(tsk->thread.vstate.datap);
}
int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
{
fstate_save(src, task_pt_regs(src));
*dst = *src;
+ /* clear entire V context, including datap for a new task */
+ memset(&dst->thread.vstate, 0, sizeof(struct __riscv_v_ext_state));
+
return 0;
}
@@ -176,6 +194,8 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args)
p->thread.s[1] = (unsigned long)args->fn_arg;
} else {
*childregs = *(current_pt_regs());
+ /* Turn off status.VS */
+ riscv_v_vstate_off(childregs);
if (usp) /* User fork */
childregs->sp = usp;
if (clone_flags & CLONE_SETTLS)
diff --git a/arch/riscv/kernel/ptrace.c b/arch/riscv/kernel/ptrace.c
index 23c48b14a0e7..1d572cf3140f 100644
--- a/arch/riscv/kernel/ptrace.c
+++ b/arch/riscv/kernel/ptrace.c
@@ -7,6 +7,7 @@
* Copied from arch/tile/kernel/ptrace.c
*/
+#include <asm/vector.h>
#include <asm/ptrace.h>
#include <asm/syscall.h>
#include <asm/thread_info.h>
@@ -24,6 +25,9 @@ enum riscv_regset {
#ifdef CONFIG_FPU
REGSET_F,
#endif
+#ifdef CONFIG_RISCV_ISA_V
+ REGSET_V,
+#endif
};
static int riscv_gpr_get(struct task_struct *target,
@@ -80,6 +84,61 @@ static int riscv_fpr_set(struct task_struct *target,
}
#endif
+#ifdef CONFIG_RISCV_ISA_V
+static int riscv_vr_get(struct task_struct *target,
+ const struct user_regset *regset,
+ struct membuf to)
+{
+ struct __riscv_v_ext_state *vstate = &target->thread.vstate;
+
+ if (!riscv_v_vstate_query(task_pt_regs(target)))
+ return -EINVAL;
+
+ /*
+ * Ensure the vector registers have been saved to the memory before
+ * copying them to membuf.
+ */
+ if (target == current)
+ riscv_v_vstate_save(current, task_pt_regs(current));
+
+ /* Copy vector header from vstate. */
+ membuf_write(&to, vstate, offsetof(struct __riscv_v_ext_state, datap));
+ membuf_zero(&to, sizeof(vstate->datap));
+
+ /* Copy all the vector registers from vstate. */
+ return membuf_write(&to, vstate->datap, riscv_v_vsize);
+}
+
+static int riscv_vr_set(struct task_struct *target,
+ const struct user_regset *regset,
+ unsigned int pos, unsigned int count,
+ const void *kbuf, const void __user *ubuf)
+{
+ int ret, size;
+ struct __riscv_v_ext_state *vstate = &target->thread.vstate;
+
+ if (!riscv_v_vstate_query(task_pt_regs(target)))
+ return -EINVAL;
+
+ /* Copy rest of the vstate except datap */
+ ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, vstate, 0,
+ offsetof(struct __riscv_v_ext_state, datap));
+ if (unlikely(ret))
+ return ret;
+
+ /* Skip copy datap. */
+ size = sizeof(vstate->datap);
+ count -= size;
+ ubuf += size;
+
+ /* Copy all the vector registers. */
+ pos = 0;
+ ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, vstate->datap,
+ 0, riscv_v_vsize);
+ return ret;
+}
+#endif
+
static const struct user_regset riscv_user_regset[] = {
[REGSET_X] = {
.core_note_type = NT_PRSTATUS,
@@ -99,6 +158,17 @@ static const struct user_regset riscv_user_regset[] = {
.set = riscv_fpr_set,
},
#endif
+#ifdef CONFIG_RISCV_ISA_V
+ [REGSET_V] = {
+ .core_note_type = NT_RISCV_VECTOR,
+ .align = 16,
+ .n = ((32 * RISCV_MAX_VLENB) +
+ sizeof(struct __riscv_v_ext_state)) / sizeof(__u32),
+ .size = sizeof(__u32),
+ .regset_get = riscv_vr_get,
+ .set = riscv_vr_set,
+ },
+#endif
};
static const struct user_regset_view riscv_user_native_view = {
diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c
index 36b026057503..971fe776e2f8 100644
--- a/arch/riscv/kernel/setup.c
+++ b/arch/riscv/kernel/setup.c
@@ -8,6 +8,7 @@
* Nick Kossifidis <[email protected]>
*/
+#include <linux/acpi.h>
#include <linux/cpu.h>
#include <linux/init.h>
#include <linux/mm.h>
@@ -21,6 +22,7 @@
#include <linux/efi.h>
#include <linux/crash_dump.h>
+#include <asm/acpi.h>
#include <asm/alternative.h>
#include <asm/cacheflush.h>
#include <asm/cpu_ops.h>
@@ -262,6 +264,8 @@ static void __init parse_dtb(void)
#endif
}
+extern void __init init_rt_signal_env(void);
+
void __init setup_arch(char **cmdline_p)
{
parse_dtb();
@@ -270,11 +274,16 @@ void __init setup_arch(char **cmdline_p)
*cmdline_p = boot_command_line;
early_ioremap_setup();
+ sbi_init();
jump_label_init();
parse_early_param();
efi_init();
paging_init();
+
+ /* Parse the ACPI tables for possible boot-time configuration */
+ acpi_boot_table_init();
+
#if IS_ENABLED(CONFIG_BUILTIN_DTB)
unflatten_and_copy_device_tree();
#else
@@ -283,7 +292,6 @@ void __init setup_arch(char **cmdline_p)
misc_mem_init();
init_resources();
- sbi_init();
#ifdef CONFIG_KASAN
kasan_init();
@@ -293,8 +301,12 @@ void __init setup_arch(char **cmdline_p)
setup_smp();
#endif
+ if (!acpi_disabled)
+ acpi_init_rintc_map();
+
riscv_init_cbo_blocksizes();
riscv_fill_hwcap();
+ init_rt_signal_env();
apply_boot_alternatives();
if (IS_ENABLED(CONFIG_RISCV_ISA_ZICBOM) &&
riscv_isa_extension_available(NULL, ZICBOM))
diff --git a/arch/riscv/kernel/signal.c b/arch/riscv/kernel/signal.c
index 9aff9d720590..180d951d3624 100644
--- a/arch/riscv/kernel/signal.c
+++ b/arch/riscv/kernel/signal.c
@@ -19,10 +19,14 @@
#include <asm/signal.h>
#include <asm/signal32.h>
#include <asm/switch_to.h>
+#include <asm/vector.h>
#include <asm/csr.h>
#include <asm/cacheflush.h>
+unsigned long signal_minsigstksz __ro_after_init;
+
extern u32 __user_rt_sigreturn[2];
+static size_t riscv_v_sc_size __ro_after_init;
#define DEBUG_SIG 0
@@ -40,26 +44,13 @@ static long restore_fp_state(struct pt_regs *regs,
{
long err;
struct __riscv_d_ext_state __user *state = &sc_fpregs->d;
- size_t i;
err = __copy_from_user(&current->thread.fstate, state, sizeof(*state));
if (unlikely(err))
return err;
fstate_restore(current, regs);
-
- /* We support no other extension state at this time. */
- for (i = 0; i < ARRAY_SIZE(sc_fpregs->q.reserved); i++) {
- u32 value;
-
- err = __get_user(value, &sc_fpregs->q.reserved[i]);
- if (unlikely(err))
- break;
- if (value != 0)
- return -EINVAL;
- }
-
- return err;
+ return 0;
}
static long save_fp_state(struct pt_regs *regs,
@@ -67,52 +58,186 @@ static long save_fp_state(struct pt_regs *regs,
{
long err;
struct __riscv_d_ext_state __user *state = &sc_fpregs->d;
- size_t i;
fstate_save(current, regs);
err = __copy_to_user(state, &current->thread.fstate, sizeof(*state));
+ return err;
+}
+#else
+#define save_fp_state(task, regs) (0)
+#define restore_fp_state(task, regs) (0)
+#endif
+
+#ifdef CONFIG_RISCV_ISA_V
+
+static long save_v_state(struct pt_regs *regs, void __user **sc_vec)
+{
+ struct __riscv_ctx_hdr __user *hdr;
+ struct __sc_riscv_v_state __user *state;
+ void __user *datap;
+ long err;
+
+ hdr = *sc_vec;
+ /* Place state to the user's signal context space after the hdr */
+ state = (struct __sc_riscv_v_state __user *)(hdr + 1);
+ /* Point datap right after the end of __sc_riscv_v_state */
+ datap = state + 1;
+
+ /* datap is designed to be 16 byte aligned for better performance */
+ WARN_ON(unlikely(!IS_ALIGNED((unsigned long)datap, 16)));
+
+ riscv_v_vstate_save(current, regs);
+ /* Copy everything of vstate but datap. */
+ err = __copy_to_user(&state->v_state, &current->thread.vstate,
+ offsetof(struct __riscv_v_ext_state, datap));
+ /* Copy the pointer datap itself. */
+ err |= __put_user(datap, &state->v_state.datap);
+ /* Copy the whole vector content to user space datap. */
+ err |= __copy_to_user(datap, current->thread.vstate.datap, riscv_v_vsize);
+ /* Copy magic to the user space after saving all vector conetext */
+ err |= __put_user(RISCV_V_MAGIC, &hdr->magic);
+ err |= __put_user(riscv_v_sc_size, &hdr->size);
if (unlikely(err))
return err;
- /* We support no other extension state at this time. */
- for (i = 0; i < ARRAY_SIZE(sc_fpregs->q.reserved); i++) {
- err = __put_user(0, &sc_fpregs->q.reserved[i]);
- if (unlikely(err))
- break;
- }
+ /* Only progress the sv_vec if everything has done successfully */
+ *sc_vec += riscv_v_sc_size;
+ return 0;
+}
+
+/*
+ * Restore Vector extension context from the user's signal frame. This function
+ * assumes a valid extension header. So magic and size checking must be done by
+ * the caller.
+ */
+static long __restore_v_state(struct pt_regs *regs, void __user *sc_vec)
+{
+ long err;
+ struct __sc_riscv_v_state __user *state = sc_vec;
+ void __user *datap;
+
+ /* Copy everything of __sc_riscv_v_state except datap. */
+ err = __copy_from_user(&current->thread.vstate, &state->v_state,
+ offsetof(struct __riscv_v_ext_state, datap));
+ if (unlikely(err))
+ return err;
+
+ /* Copy the pointer datap itself. */
+ err = __get_user(datap, &state->v_state.datap);
+ if (unlikely(err))
+ return err;
+ /*
+ * Copy the whole vector content from user space datap. Use
+ * copy_from_user to prevent information leak.
+ */
+ err = copy_from_user(current->thread.vstate.datap, datap, riscv_v_vsize);
+ if (unlikely(err))
+ return err;
+
+ riscv_v_vstate_restore(current, regs);
return err;
}
#else
-#define save_fp_state(task, regs) (0)
-#define restore_fp_state(task, regs) (0)
+#define save_v_state(task, regs) (0)
+#define __restore_v_state(task, regs) (0)
#endif
static long restore_sigcontext(struct pt_regs *regs,
struct sigcontext __user *sc)
{
+ void __user *sc_ext_ptr = &sc->sc_extdesc.hdr;
+ __u32 rsvd;
long err;
/* sc_regs is structured the same as the start of pt_regs */
err = __copy_from_user(regs, &sc->sc_regs, sizeof(sc->sc_regs));
+ if (unlikely(err))
+ return err;
+
/* Restore the floating-point state. */
- if (has_fpu())
- err |= restore_fp_state(regs, &sc->sc_fpregs);
+ if (has_fpu()) {
+ err = restore_fp_state(regs, &sc->sc_fpregs);
+ if (unlikely(err))
+ return err;
+ }
+
+ /* Check the reserved word before extensions parsing */
+ err = __get_user(rsvd, &sc->sc_extdesc.reserved);
+ if (unlikely(err))
+ return err;
+ if (unlikely(rsvd))
+ return -EINVAL;
+
+ while (!err) {
+ __u32 magic, size;
+ struct __riscv_ctx_hdr __user *head = sc_ext_ptr;
+
+ err |= __get_user(magic, &head->magic);
+ err |= __get_user(size, &head->size);
+ if (unlikely(err))
+ return err;
+
+ sc_ext_ptr += sizeof(*head);
+ switch (magic) {
+ case END_MAGIC:
+ if (size != END_HDR_SIZE)
+ return -EINVAL;
+
+ return 0;
+ case RISCV_V_MAGIC:
+ if (!has_vector() || !riscv_v_vstate_query(regs) ||
+ size != riscv_v_sc_size)
+ return -EINVAL;
+
+ err = __restore_v_state(regs, sc_ext_ptr);
+ break;
+ default:
+ return -EINVAL;
+ }
+ sc_ext_ptr = (void __user *)head + size;
+ }
return err;
}
+static size_t get_rt_frame_size(bool cal_all)
+{
+ struct rt_sigframe __user *frame;
+ size_t frame_size;
+ size_t total_context_size = 0;
+
+ frame_size = sizeof(*frame);
+
+ if (has_vector()) {
+ if (cal_all || riscv_v_vstate_query(task_pt_regs(current)))
+ total_context_size += riscv_v_sc_size;
+ }
+ /*
+ * Preserved a __riscv_ctx_hdr for END signal context header if an
+ * extension uses __riscv_extra_ext_header
+ */
+ if (total_context_size)
+ total_context_size += sizeof(struct __riscv_ctx_hdr);
+
+ frame_size += total_context_size;
+
+ frame_size = round_up(frame_size, 16);
+ return frame_size;
+}
+
SYSCALL_DEFINE0(rt_sigreturn)
{
struct pt_regs *regs = current_pt_regs();
struct rt_sigframe __user *frame;
struct task_struct *task;
sigset_t set;
+ size_t frame_size = get_rt_frame_size(false);
/* Always make any pending restarted system calls return -EINTR */
current->restart_block.fn = do_no_restart_syscall;
frame = (struct rt_sigframe __user *)regs->sp;
- if (!access_ok(frame, sizeof(*frame)))
+ if (!access_ok(frame, frame_size))
goto badframe;
if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set)))
@@ -146,12 +271,23 @@ static long setup_sigcontext(struct rt_sigframe __user *frame,
struct pt_regs *regs)
{
struct sigcontext __user *sc = &frame->uc.uc_mcontext;
+ struct __riscv_ctx_hdr __user *sc_ext_ptr = &sc->sc_extdesc.hdr;
long err;
+
/* sc_regs is structured the same as the start of pt_regs */
err = __copy_to_user(&sc->sc_regs, regs, sizeof(sc->sc_regs));
/* Save the floating-point state. */
if (has_fpu())
err |= save_fp_state(regs, &sc->sc_fpregs);
+ /* Save the vector state. */
+ if (has_vector() && riscv_v_vstate_query(regs))
+ err |= save_v_state(regs, (void __user **)&sc_ext_ptr);
+ /* Write zero to fp-reserved space and check it on restore_sigcontext */
+ err |= __put_user(0, &sc->sc_extdesc.reserved);
+ /* And put END __riscv_ctx_hdr at the end. */
+ err |= __put_user(END_MAGIC, &sc_ext_ptr->magic);
+ err |= __put_user(END_HDR_SIZE, &sc_ext_ptr->size);
+
return err;
}
@@ -175,6 +311,13 @@ static inline void __user *get_sigframe(struct ksignal *ksig,
/* Align the stack frame. */
sp &= ~0xfUL;
+ /*
+ * Fail if the size of the altstack is not large enough for the
+ * sigframe construction.
+ */
+ if (current->sas_ss_size && sp < current->sas_ss_sp)
+ return (void __user __force *)-1UL;
+
return (void __user *)sp;
}
@@ -184,9 +327,10 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set,
struct rt_sigframe __user *frame;
long err = 0;
unsigned long __maybe_unused addr;
+ size_t frame_size = get_rt_frame_size(false);
- frame = get_sigframe(ksig, regs, sizeof(*frame));
- if (!access_ok(frame, sizeof(*frame)))
+ frame = get_sigframe(ksig, regs, frame_size);
+ if (!access_ok(frame, frame_size))
return -EFAULT;
err |= copy_siginfo_to_user(&frame->info, &ksig->info);
@@ -319,3 +463,23 @@ void arch_do_signal_or_restart(struct pt_regs *regs)
*/
restore_saved_sigmask();
}
+
+void init_rt_signal_env(void);
+void __init init_rt_signal_env(void)
+{
+ riscv_v_sc_size = sizeof(struct __riscv_ctx_hdr) +
+ sizeof(struct __sc_riscv_v_state) + riscv_v_vsize;
+ /*
+ * Determine the stack space required for guaranteed signal delivery.
+ * The signal_minsigstksz will be populated into the AT_MINSIGSTKSZ entry
+ * in the auxiliary array at process startup.
+ */
+ signal_minsigstksz = get_rt_frame_size(true);
+}
+
+#ifdef CONFIG_DYNAMIC_SIGFRAME
+bool sigaltstack_size_valid(size_t ss_size)
+{
+ return ss_size > get_rt_frame_size(false);
+}
+#endif /* CONFIG_DYNAMIC_SIGFRAME */
diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c
index 445a4efee267..bb0b76e1a6d4 100644
--- a/arch/riscv/kernel/smpboot.c
+++ b/arch/riscv/kernel/smpboot.c
@@ -8,6 +8,7 @@
* Copyright (C) 2017 SiFive
*/
+#include <linux/acpi.h>
#include <linux/arch_topology.h>
#include <linux/module.h>
#include <linux/init.h>
@@ -31,6 +32,8 @@
#include <asm/tlbflush.h>
#include <asm/sections.h>
#include <asm/smp.h>
+#include <uapi/asm/hwcap.h>
+#include <asm/vector.h>
#include "head.h"
@@ -70,7 +73,73 @@ void __init smp_prepare_cpus(unsigned int max_cpus)
}
}
-void __init setup_smp(void)
+#ifdef CONFIG_ACPI
+static unsigned int cpu_count = 1;
+
+static int __init acpi_parse_rintc(union acpi_subtable_headers *header, const unsigned long end)
+{
+ unsigned long hart;
+ static bool found_boot_cpu;
+ struct acpi_madt_rintc *processor = (struct acpi_madt_rintc *)header;
+
+ /*
+ * Each RINTC structure in MADT will have a flag. If ACPI_MADT_ENABLED
+ * bit in the flag is not enabled, it means OS should not try to enable
+ * the cpu to which RINTC belongs.
+ */
+ if (!(processor->flags & ACPI_MADT_ENABLED))
+ return 0;
+
+ if (BAD_MADT_ENTRY(processor, end))
+ return -EINVAL;
+
+ acpi_table_print_madt_entry(&header->common);
+
+ hart = processor->hart_id;
+ if (hart == INVALID_HARTID) {
+ pr_warn("Invalid hartid\n");
+ return 0;
+ }
+
+ if (hart == cpuid_to_hartid_map(0)) {
+ BUG_ON(found_boot_cpu);
+ found_boot_cpu = true;
+ early_map_cpu_to_node(0, acpi_numa_get_nid(cpu_count));
+ return 0;
+ }
+
+ if (cpu_count >= NR_CPUS) {
+ pr_warn("NR_CPUS is too small for the number of ACPI tables.\n");
+ return 0;
+ }
+
+ cpuid_to_hartid_map(cpu_count) = hart;
+ early_map_cpu_to_node(cpu_count, acpi_numa_get_nid(cpu_count));
+ cpu_count++;
+
+ return 0;
+}
+
+static void __init acpi_parse_and_init_cpus(void)
+{
+ int cpuid;
+
+ cpu_set_ops(0);
+
+ acpi_table_parse_madt(ACPI_MADT_TYPE_RINTC, acpi_parse_rintc, 0);
+
+ for (cpuid = 1; cpuid < nr_cpu_ids; cpuid++) {
+ if (cpuid_to_hartid_map(cpuid) != INVALID_HARTID) {
+ cpu_set_ops(cpuid);
+ set_cpu_possible(cpuid, true);
+ }
+ }
+}
+#else
+#define acpi_parse_and_init_cpus(...) do { } while (0)
+#endif
+
+static void __init of_parse_and_init_cpus(void)
{
struct device_node *dn;
unsigned long hart;
@@ -81,7 +150,7 @@ void __init setup_smp(void)
cpu_set_ops(0);
for_each_of_cpu_node(dn) {
- rc = riscv_of_processor_hartid(dn, &hart);
+ rc = riscv_early_of_processor_hartid(dn, &hart);
if (rc < 0)
continue;
@@ -116,6 +185,14 @@ void __init setup_smp(void)
}
}
+void __init setup_smp(void)
+{
+ if (acpi_disabled)
+ of_parse_and_init_cpus();
+ else
+ acpi_parse_and_init_cpus();
+}
+
static int start_secondary_cpu(int cpu, struct task_struct *tidle)
{
if (cpu_ops[cpu]->cpu_start)
@@ -169,6 +246,11 @@ asmlinkage __visible void smp_callin(void)
set_cpu_online(curr_cpuid, 1);
probe_vendor_features(curr_cpuid);
+ if (has_vector()) {
+ if (riscv_v_setup_vsize())
+ elf_hwcap &= ~COMPAT_HWCAP_ISA_V;
+ }
+
/*
* Remote TLB flushes are ignored while the CPU is offline, so emit
* a local TLB flush right now just in case.
diff --git a/arch/riscv/kernel/sys_riscv.c b/arch/riscv/kernel/sys_riscv.c
index 5db29683ebee..26ef5526bfb4 100644
--- a/arch/riscv/kernel/sys_riscv.c
+++ b/arch/riscv/kernel/sys_riscv.c
@@ -10,6 +10,7 @@
#include <asm/cpufeature.h>
#include <asm/hwprobe.h>
#include <asm/sbi.h>
+#include <asm/vector.h>
#include <asm/switch_to.h>
#include <asm/uaccess.h>
#include <asm/unistd.h>
@@ -121,6 +122,49 @@ static void hwprobe_arch_id(struct riscv_hwprobe *pair,
pair->value = id;
}
+static void hwprobe_isa_ext0(struct riscv_hwprobe *pair,
+ const struct cpumask *cpus)
+{
+ int cpu;
+ u64 missing = 0;
+
+ pair->value = 0;
+ if (has_fpu())
+ pair->value |= RISCV_HWPROBE_IMA_FD;
+
+ if (riscv_isa_extension_available(NULL, c))
+ pair->value |= RISCV_HWPROBE_IMA_C;
+
+ if (has_vector())
+ pair->value |= RISCV_HWPROBE_IMA_V;
+
+ /*
+ * Loop through and record extensions that 1) anyone has, and 2) anyone
+ * doesn't have.
+ */
+ for_each_cpu(cpu, cpus) {
+ struct riscv_isainfo *isainfo = &hart_isa[cpu];
+
+ if (riscv_isa_extension_available(isainfo->isa, ZBA))
+ pair->value |= RISCV_HWPROBE_EXT_ZBA;
+ else
+ missing |= RISCV_HWPROBE_EXT_ZBA;
+
+ if (riscv_isa_extension_available(isainfo->isa, ZBB))
+ pair->value |= RISCV_HWPROBE_EXT_ZBB;
+ else
+ missing |= RISCV_HWPROBE_EXT_ZBB;
+
+ if (riscv_isa_extension_available(isainfo->isa, ZBS))
+ pair->value |= RISCV_HWPROBE_EXT_ZBS;
+ else
+ missing |= RISCV_HWPROBE_EXT_ZBS;
+ }
+
+ /* Now turn off reporting features if any CPU is missing it. */
+ pair->value &= ~missing;
+}
+
static u64 hwprobe_misaligned(const struct cpumask *cpus)
{
int cpu;
@@ -164,13 +208,7 @@ static void hwprobe_one_pair(struct riscv_hwprobe *pair,
break;
case RISCV_HWPROBE_KEY_IMA_EXT_0:
- pair->value = 0;
- if (has_fpu())
- pair->value |= RISCV_HWPROBE_IMA_FD;
-
- if (riscv_isa_extension_available(NULL, c))
- pair->value |= RISCV_HWPROBE_IMA_C;
-
+ hwprobe_isa_ext0(pair, cpus);
break;
case RISCV_HWPROBE_KEY_CPUPERF_0:
diff --git a/arch/riscv/kernel/time.c b/arch/riscv/kernel/time.c
index babaf3b48ba8..23641e82a9df 100644
--- a/arch/riscv/kernel/time.c
+++ b/arch/riscv/kernel/time.c
@@ -4,6 +4,7 @@
* Copyright (C) 2017 SiFive
*/
+#include <linux/acpi.h>
#include <linux/of_clk.h>
#include <linux/clockchips.h>
#include <linux/clocksource.h>
@@ -18,17 +19,29 @@ EXPORT_SYMBOL_GPL(riscv_timebase);
void __init time_init(void)
{
struct device_node *cpu;
+ struct acpi_table_rhct *rhct;
+ acpi_status status;
u32 prop;
- cpu = of_find_node_by_path("/cpus");
- if (!cpu || of_property_read_u32(cpu, "timebase-frequency", &prop))
- panic(KERN_WARNING "RISC-V system with no 'timebase-frequency' in DTS\n");
- of_node_put(cpu);
- riscv_timebase = prop;
+ if (acpi_disabled) {
+ cpu = of_find_node_by_path("/cpus");
+ if (!cpu || of_property_read_u32(cpu, "timebase-frequency", &prop))
+ panic("RISC-V system with no 'timebase-frequency' in DTS\n");
+
+ of_node_put(cpu);
+ riscv_timebase = prop;
+ of_clk_init(NULL);
+ } else {
+ status = acpi_get_table(ACPI_SIG_RHCT, 0, (struct acpi_table_header **)&rhct);
+ if (ACPI_FAILURE(status))
+ panic("RISC-V ACPI system with no RHCT table\n");
+
+ riscv_timebase = rhct->time_base_freq;
+ acpi_put_table((struct acpi_table_header *)rhct);
+ }
lpj_fine = riscv_timebase / HZ;
- of_clk_init(NULL);
timer_probe();
tick_setup_hrtimer_broadcast();
diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c
index 8c258b78c925..5158961ea977 100644
--- a/arch/riscv/kernel/traps.c
+++ b/arch/riscv/kernel/traps.c
@@ -26,6 +26,8 @@
#include <asm/ptrace.h>
#include <asm/syscall.h>
#include <asm/thread_info.h>
+#include <asm/vector.h>
+#include <asm/irq_stack.h>
int show_unhandled_signals = 1;
@@ -145,8 +147,29 @@ DO_ERROR_INFO(do_trap_insn_misaligned,
SIGBUS, BUS_ADRALN, "instruction address misaligned");
DO_ERROR_INFO(do_trap_insn_fault,
SIGSEGV, SEGV_ACCERR, "instruction access fault");
-DO_ERROR_INFO(do_trap_insn_illegal,
- SIGILL, ILL_ILLOPC, "illegal instruction");
+
+asmlinkage __visible __trap_section void do_trap_insn_illegal(struct pt_regs *regs)
+{
+ if (user_mode(regs)) {
+ irqentry_enter_from_user_mode(regs);
+
+ local_irq_enable();
+
+ if (!riscv_v_first_use_handler(regs))
+ do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc,
+ "Oops - illegal instruction");
+
+ irqentry_exit_to_user_mode(regs);
+ } else {
+ irqentry_state_t state = irqentry_nmi_enter(regs);
+
+ do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc,
+ "Oops - illegal instruction");
+
+ irqentry_nmi_exit(regs, state);
+ }
+}
+
DO_ERROR_INFO(do_trap_load_fault,
SIGSEGV, SEGV_ACCERR, "load access fault");
#ifndef CONFIG_RISCV_M_MODE
@@ -305,16 +328,46 @@ asmlinkage __visible noinstr void do_page_fault(struct pt_regs *regs)
}
#endif
-asmlinkage __visible noinstr void do_irq(struct pt_regs *regs)
+static void noinstr handle_riscv_irq(struct pt_regs *regs)
{
struct pt_regs *old_regs;
- irqentry_state_t state = irqentry_enter(regs);
irq_enter_rcu();
old_regs = set_irq_regs(regs);
handle_arch_irq(regs);
set_irq_regs(old_regs);
irq_exit_rcu();
+}
+
+asmlinkage void noinstr do_irq(struct pt_regs *regs)
+{
+ irqentry_state_t state = irqentry_enter(regs);
+#ifdef CONFIG_IRQ_STACKS
+ if (on_thread_stack()) {
+ ulong *sp = per_cpu(irq_stack_ptr, smp_processor_id())
+ + IRQ_STACK_SIZE/sizeof(ulong);
+ __asm__ __volatile(
+ "addi sp, sp, -"RISCV_SZPTR "\n"
+ REG_S" ra, (sp) \n"
+ "addi sp, sp, -"RISCV_SZPTR "\n"
+ REG_S" s0, (sp) \n"
+ "addi s0, sp, 2*"RISCV_SZPTR "\n"
+ "move sp, %[sp] \n"
+ "move a0, %[regs] \n"
+ "call handle_riscv_irq \n"
+ "addi sp, s0, -2*"RISCV_SZPTR"\n"
+ REG_L" s0, (sp) \n"
+ "addi sp, sp, "RISCV_SZPTR "\n"
+ REG_L" ra, (sp) \n"
+ "addi sp, sp, "RISCV_SZPTR "\n"
+ :
+ : [sp] "r" (sp), [regs] "r" (regs)
+ : "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7",
+ "t0", "t1", "t2", "t3", "t4", "t5", "t6",
+ "memory");
+ } else
+#endif
+ handle_riscv_irq(regs);
irqentry_exit(regs, state);
}
diff --git a/arch/riscv/kernel/vdso/rt_sigreturn.S b/arch/riscv/kernel/vdso/rt_sigreturn.S
index 0573705eac76..10438c7c626a 100644
--- a/arch/riscv/kernel/vdso/rt_sigreturn.S
+++ b/arch/riscv/kernel/vdso/rt_sigreturn.S
@@ -11,6 +11,6 @@ ENTRY(__vdso_rt_sigreturn)
.cfi_startproc
.cfi_signal_frame
li a7, __NR_rt_sigreturn
- scall
+ ecall
.cfi_endproc
ENDPROC(__vdso_rt_sigreturn)
diff --git a/arch/riscv/kernel/vector.c b/arch/riscv/kernel/vector.c
new file mode 100644
index 000000000000..f9c8e19ab301
--- /dev/null
+++ b/arch/riscv/kernel/vector.c
@@ -0,0 +1,276 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) 2023 SiFive
+ * Author: Andy Chiu <[email protected]>
+ */
+#include <linux/export.h>
+#include <linux/sched/signal.h>
+#include <linux/types.h>
+#include <linux/slab.h>
+#include <linux/sched.h>
+#include <linux/uaccess.h>
+#include <linux/prctl.h>
+
+#include <asm/thread_info.h>
+#include <asm/processor.h>
+#include <asm/insn.h>
+#include <asm/vector.h>
+#include <asm/csr.h>
+#include <asm/elf.h>
+#include <asm/ptrace.h>
+#include <asm/bug.h>
+
+static bool riscv_v_implicit_uacc = IS_ENABLED(CONFIG_RISCV_ISA_V_DEFAULT_ENABLE);
+
+unsigned long riscv_v_vsize __read_mostly;
+EXPORT_SYMBOL_GPL(riscv_v_vsize);
+
+int riscv_v_setup_vsize(void)
+{
+ unsigned long this_vsize;
+
+ /* There are 32 vector registers with vlenb length. */
+ riscv_v_enable();
+ this_vsize = csr_read(CSR_VLENB) * 32;
+ riscv_v_disable();
+
+ if (!riscv_v_vsize) {
+ riscv_v_vsize = this_vsize;
+ return 0;
+ }
+
+ if (riscv_v_vsize != this_vsize) {
+ WARN(1, "RISCV_ISA_V only supports one vlenb on SMP systems");
+ return -EOPNOTSUPP;
+ }
+
+ return 0;
+}
+
+static bool insn_is_vector(u32 insn_buf)
+{
+ u32 opcode = insn_buf & __INSN_OPCODE_MASK;
+ u32 width, csr;
+
+ /*
+ * All V-related instructions, including CSR operations are 4-Byte. So,
+ * do not handle if the instruction length is not 4-Byte.
+ */
+ if (unlikely(GET_INSN_LENGTH(insn_buf) != 4))
+ return false;
+
+ switch (opcode) {
+ case RVV_OPCODE_VECTOR:
+ return true;
+ case RVV_OPCODE_VL:
+ case RVV_OPCODE_VS:
+ width = RVV_EXRACT_VL_VS_WIDTH(insn_buf);
+ if (width == RVV_VL_VS_WIDTH_8 || width == RVV_VL_VS_WIDTH_16 ||
+ width == RVV_VL_VS_WIDTH_32 || width == RVV_VL_VS_WIDTH_64)
+ return true;
+
+ break;
+ case RVG_OPCODE_SYSTEM:
+ csr = RVG_EXTRACT_SYSTEM_CSR(insn_buf);
+ if ((csr >= CSR_VSTART && csr <= CSR_VCSR) ||
+ (csr >= CSR_VL && csr <= CSR_VLENB))
+ return true;
+ }
+
+ return false;
+}
+
+static int riscv_v_thread_zalloc(void)
+{
+ void *datap;
+
+ datap = kzalloc(riscv_v_vsize, GFP_KERNEL);
+ if (!datap)
+ return -ENOMEM;
+
+ current->thread.vstate.datap = datap;
+ memset(&current->thread.vstate, 0, offsetof(struct __riscv_v_ext_state,
+ datap));
+ return 0;
+}
+
+#define VSTATE_CTRL_GET_CUR(x) ((x) & PR_RISCV_V_VSTATE_CTRL_CUR_MASK)
+#define VSTATE_CTRL_GET_NEXT(x) (((x) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK) >> 2)
+#define VSTATE_CTRL_MAKE_NEXT(x) (((x) << 2) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK)
+#define VSTATE_CTRL_GET_INHERIT(x) (!!((x) & PR_RISCV_V_VSTATE_CTRL_INHERIT))
+static inline int riscv_v_ctrl_get_cur(struct task_struct *tsk)
+{
+ return VSTATE_CTRL_GET_CUR(tsk->thread.vstate_ctrl);
+}
+
+static inline int riscv_v_ctrl_get_next(struct task_struct *tsk)
+{
+ return VSTATE_CTRL_GET_NEXT(tsk->thread.vstate_ctrl);
+}
+
+static inline bool riscv_v_ctrl_test_inherit(struct task_struct *tsk)
+{
+ return VSTATE_CTRL_GET_INHERIT(tsk->thread.vstate_ctrl);
+}
+
+static inline void riscv_v_ctrl_set(struct task_struct *tsk, int cur, int nxt,
+ bool inherit)
+{
+ unsigned long ctrl;
+
+ ctrl = cur & PR_RISCV_V_VSTATE_CTRL_CUR_MASK;
+ ctrl |= VSTATE_CTRL_MAKE_NEXT(nxt);
+ if (inherit)
+ ctrl |= PR_RISCV_V_VSTATE_CTRL_INHERIT;
+ tsk->thread.vstate_ctrl = ctrl;
+}
+
+bool riscv_v_vstate_ctrl_user_allowed(void)
+{
+ return riscv_v_ctrl_get_cur(current) == PR_RISCV_V_VSTATE_CTRL_ON;
+}
+EXPORT_SYMBOL_GPL(riscv_v_vstate_ctrl_user_allowed);
+
+bool riscv_v_first_use_handler(struct pt_regs *regs)
+{
+ u32 __user *epc = (u32 __user *)regs->epc;
+ u32 insn = (u32)regs->badaddr;
+
+ /* Do not handle if V is not supported, or disabled */
+ if (!(ELF_HWCAP & COMPAT_HWCAP_ISA_V))
+ return false;
+
+ /* If V has been enabled then it is not the first-use trap */
+ if (riscv_v_vstate_query(regs))
+ return false;
+
+ /* Get the instruction */
+ if (!insn) {
+ if (__get_user(insn, epc))
+ return false;
+ }
+
+ /* Filter out non-V instructions */
+ if (!insn_is_vector(insn))
+ return false;
+
+ /* Sanity check. datap should be null by the time of the first-use trap */
+ WARN_ON(current->thread.vstate.datap);
+
+ /*
+ * Now we sure that this is a V instruction. And it executes in the
+ * context where VS has been off. So, try to allocate the user's V
+ * context and resume execution.
+ */
+ if (riscv_v_thread_zalloc()) {
+ force_sig(SIGBUS);
+ return true;
+ }
+ riscv_v_vstate_on(regs);
+ return true;
+}
+
+void riscv_v_vstate_ctrl_init(struct task_struct *tsk)
+{
+ bool inherit;
+ int cur, next;
+
+ if (!has_vector())
+ return;
+
+ next = riscv_v_ctrl_get_next(tsk);
+ if (!next) {
+ if (READ_ONCE(riscv_v_implicit_uacc))
+ cur = PR_RISCV_V_VSTATE_CTRL_ON;
+ else
+ cur = PR_RISCV_V_VSTATE_CTRL_OFF;
+ } else {
+ cur = next;
+ }
+ /* Clear next mask if inherit-bit is not set */
+ inherit = riscv_v_ctrl_test_inherit(tsk);
+ if (!inherit)
+ next = PR_RISCV_V_VSTATE_CTRL_DEFAULT;
+
+ riscv_v_ctrl_set(tsk, cur, next, inherit);
+}
+
+long riscv_v_vstate_ctrl_get_current(void)
+{
+ if (!has_vector())
+ return -EINVAL;
+
+ return current->thread.vstate_ctrl & PR_RISCV_V_VSTATE_CTRL_MASK;
+}
+
+long riscv_v_vstate_ctrl_set_current(unsigned long arg)
+{
+ bool inherit;
+ int cur, next;
+
+ if (!has_vector())
+ return -EINVAL;
+
+ if (arg & ~PR_RISCV_V_VSTATE_CTRL_MASK)
+ return -EINVAL;
+
+ cur = VSTATE_CTRL_GET_CUR(arg);
+ switch (cur) {
+ case PR_RISCV_V_VSTATE_CTRL_OFF:
+ /* Do not allow user to turn off V if current is not off */
+ if (riscv_v_ctrl_get_cur(current) != PR_RISCV_V_VSTATE_CTRL_OFF)
+ return -EPERM;
+
+ break;
+ case PR_RISCV_V_VSTATE_CTRL_ON:
+ break;
+ case PR_RISCV_V_VSTATE_CTRL_DEFAULT:
+ cur = riscv_v_ctrl_get_cur(current);
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ next = VSTATE_CTRL_GET_NEXT(arg);
+ inherit = VSTATE_CTRL_GET_INHERIT(arg);
+ switch (next) {
+ case PR_RISCV_V_VSTATE_CTRL_DEFAULT:
+ case PR_RISCV_V_VSTATE_CTRL_OFF:
+ case PR_RISCV_V_VSTATE_CTRL_ON:
+ riscv_v_ctrl_set(current, cur, next, inherit);
+ return 0;
+ }
+
+ return -EINVAL;
+}
+
+#ifdef CONFIG_SYSCTL
+
+static struct ctl_table riscv_v_default_vstate_table[] = {
+ {
+ .procname = "riscv_v_default_allow",
+ .data = &riscv_v_implicit_uacc,
+ .maxlen = sizeof(riscv_v_implicit_uacc),
+ .mode = 0644,
+ .proc_handler = proc_dobool,
+ },
+ { }
+};
+
+static int __init riscv_v_sysctl_init(void)
+{
+ if (has_vector())
+ if (!register_sysctl("abi", riscv_v_default_vstate_table))
+ return -EINVAL;
+ return 0;
+}
+
+#else /* ! CONFIG_SYSCTL */
+static int __init riscv_v_sysctl_init(void) { return 0; }
+#endif /* ! CONFIG_SYSCTL */
+
+static int riscv_v_init(void)
+{
+ return riscv_v_sysctl_init();
+}
+core_initcall(riscv_v_init);
Powered by cgit, Gruvboxed by Blaster4385.