diff options
Diffstat (limited to 'arch/s390/kernel')
50 files changed, 791 insertions, 717 deletions
diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile index 1f0fe98f6db9..060ce548fe8b 100644 --- a/arch/s390/kernel/Makefile +++ b/arch/s390/kernel/Makefile @@ -2,20 +2,41 @@ # Makefile for the linux kernel. # -KCOV_INSTRUMENT_early.o := n -KCOV_INSTRUMENT_sclp.o := n -KCOV_INSTRUMENT_als.o := n - ifdef CONFIG_FUNCTION_TRACER -# Don't trace early setup code and tracing code -CFLAGS_REMOVE_early.o = $(CC_FLAGS_FTRACE) -CFLAGS_REMOVE_ftrace.o = $(CC_FLAGS_FTRACE) + +# Do not trace tracer code +CFLAGS_REMOVE_ftrace.o = $(CC_FLAGS_FTRACE) + +# Do not trace early setup code +CFLAGS_REMOVE_als.o = $(CC_FLAGS_FTRACE) +CFLAGS_REMOVE_early.o = $(CC_FLAGS_FTRACE) + +endif + +GCOV_PROFILE_als.o := n +GCOV_PROFILE_early.o := n + +KCOV_INSTRUMENT_als.o := n +KCOV_INSTRUMENT_early.o := n + +UBSAN_SANITIZE_als.o := n +UBSAN_SANITIZE_early.o := n + +# +# Use -march=z900 for als.c to be able to print an error +# message if the kernel is started on a machine which is too old +# +ifneq ($(CC_FLAGS_MARCH),-march=z900) +CFLAGS_REMOVE_als.o += $(CC_FLAGS_MARCH) +CFLAGS_als.o += -march=z900 +AFLAGS_REMOVE_head.o += $(CC_FLAGS_MARCH) +AFLAGS_head.o += -march=z900 endif # # Passing null pointers is ok for smp code, since we access the lowcore here. # -CFLAGS_smp.o := -Wno-nonnull +CFLAGS_smp.o := -Wno-nonnull # # Disable tailcall optimizations for stack / callchain walking functions @@ -30,31 +51,11 @@ CFLAGS_dumpstack.o += -fno-optimize-sibling-calls # CFLAGS_ptrace.o += -DUTS_MACHINE='"$(UTS_MACHINE)"' -CFLAGS_sysinfo.o += -w - -# -# Use -march=z900 for sclp.c and als.c to be able to print an error -# message if the kernel is started on a machine which is too old -# -CFLAGS_REMOVE_sclp.o = $(CC_FLAGS_FTRACE) -CFLAGS_REMOVE_als.o = $(CC_FLAGS_FTRACE) -ifneq ($(CC_FLAGS_MARCH),-march=z900) -CFLAGS_REMOVE_sclp.o += $(CC_FLAGS_MARCH) -CFLAGS_sclp.o += -march=z900 -CFLAGS_REMOVE_als.o += $(CC_FLAGS_MARCH) -CFLAGS_als.o += -march=z900 -AFLAGS_REMOVE_head.o += $(CC_FLAGS_MARCH) -AFLAGS_head.o += -march=z900 -endif -GCOV_PROFILE_sclp.o := n -GCOV_PROFILE_als.o := n -UBSAN_SANITIZE_als.o := n -UBSAN_SANITIZE_early.o := n -UBSAN_SANITIZE_sclp.o := n +CFLAGS_sysinfo.o += -w obj-y := traps.o time.o process.o base.o early.o setup.o idle.o vtime.o obj-y += processor.o sys_s390.o ptrace.o signal.o cpcmd.o ebcdic.o nmi.o -obj-y += debug.o irq.o ipl.o dis.o diag.o sclp.o vdso.o als.o +obj-y += debug.o irq.o ipl.o dis.o diag.o vdso.o als.o obj-y += sysinfo.o jump_label.o lgr.o os_info.o machine_kexec.o pgm_check.o obj-y += runtime_instr.o cache.o fpu.o dumpstack.o obj-y += entry.o reipl.o relocate_kernel.o @@ -69,7 +70,7 @@ obj-$(CONFIG_AUDIT) += audit.o compat-obj-$(CONFIG_AUDIT) += compat_audit.o obj-$(CONFIG_COMPAT) += compat_linux.o compat_signal.o obj-$(CONFIG_COMPAT) += compat_wrapper.o $(compat-obj-y) - +obj-$(CONFIG_EARLY_PRINTK) += early_printk.o obj-$(CONFIG_STACKTRACE) += stacktrace.o obj-$(CONFIG_KPROBES) += kprobes.o obj-$(CONFIG_FUNCTION_TRACER) += mcount.o ftrace.o diff --git a/arch/s390/kernel/als.c b/arch/s390/kernel/als.c index a16e9d1bf9e3..14769eb52a33 100644 --- a/arch/s390/kernel/als.c +++ b/arch/s390/kernel/als.c @@ -41,7 +41,8 @@ static void __init print_machine_type(void) get_cpu_id(&id); u16_to_hex(type_str, id.machine); strcat(mach_str, type_str); - _sclp_print_early(mach_str); + strcat(mach_str, "\n"); + sclp_early_printk(mach_str); } static void __init u16_to_decimal(char *str, u16 val) @@ -79,7 +80,8 @@ static void __init print_missing_facilities(void) * z/VM adds a four character prefix. */ if (strlen(als_str) > 70) { - _sclp_print_early(als_str); + strcat(als_str, "\n"); + sclp_early_printk(als_str); *als_str = '\0'; } u16_to_decimal(val_str, i * BITS_PER_LONG + j); @@ -87,13 +89,14 @@ static void __init print_missing_facilities(void) first = 0; } } - _sclp_print_early(als_str); - _sclp_print_early("See Principles of Operations for facility bits"); + strcat(als_str, "\n"); + sclp_early_printk(als_str); + sclp_early_printk("See Principles of Operations for facility bits\n"); } static void __init facility_mismatch(void) { - _sclp_print_early("The Linux kernel requires more recent processor hardware"); + sclp_early_printk("The Linux kernel requires more recent processor hardware\n"); print_machine_type(); print_missing_facilities(); disabled_wait(0x8badcccc); diff --git a/arch/s390/kernel/asm-offsets.c b/arch/s390/kernel/asm-offsets.c index f3df9e0a5dec..c4b3570ded5b 100644 --- a/arch/s390/kernel/asm-offsets.c +++ b/arch/s390/kernel/asm-offsets.c @@ -25,12 +25,14 @@ int main(void) { /* task struct offsets */ - OFFSET(__TASK_thread_info, task_struct, stack); + OFFSET(__TASK_stack, task_struct, stack); OFFSET(__TASK_thread, task_struct, thread); OFFSET(__TASK_pid, task_struct, pid); BLANK(); /* thread struct offsets */ OFFSET(__THREAD_ksp, thread_struct, ksp); + OFFSET(__THREAD_sysc_table, thread_struct, sys_call_table); + OFFSET(__THREAD_last_break, thread_struct, last_break); OFFSET(__THREAD_FPU_fpc, thread_struct, fpu.fpc); OFFSET(__THREAD_FPU_regs, thread_struct, fpu.regs); OFFSET(__THREAD_per_cause, thread_struct, per_event.cause); @@ -39,14 +41,7 @@ int main(void) OFFSET(__THREAD_trap_tdb, thread_struct, trap_tdb); BLANK(); /* thread info offsets */ - OFFSET(__TI_task, thread_info, task); - OFFSET(__TI_flags, thread_info, flags); - OFFSET(__TI_sysc_table, thread_info, sys_call_table); - OFFSET(__TI_cpu, thread_info, cpu); - OFFSET(__TI_precount, thread_info, preempt_count); - OFFSET(__TI_user_timer, thread_info, user_timer); - OFFSET(__TI_system_timer, thread_info, system_timer); - OFFSET(__TI_last_break, thread_info, last_break); + OFFSET(__TI_flags, task_struct, thread_info.flags); BLANK(); /* pt_regs offsets */ OFFSET(__PT_ARGS, pt_regs, args); @@ -79,6 +74,8 @@ int main(void) OFFSET(__VDSO_ECTG_OK, vdso_data, ectg_available); OFFSET(__VDSO_TK_MULT, vdso_data, tk_mult); OFFSET(__VDSO_TK_SHIFT, vdso_data, tk_shift); + OFFSET(__VDSO_TS_DIR, vdso_data, ts_dir); + OFFSET(__VDSO_TS_END, vdso_data, ts_end); OFFSET(__VDSO_ECTG_BASE, vdso_per_cpu_data, ectg_timer_base); OFFSET(__VDSO_ECTG_USER, vdso_per_cpu_data, ectg_user_time); OFFSET(__VDSO_CPU_NR, vdso_per_cpu_data, cpu_nr); @@ -159,7 +156,6 @@ int main(void) OFFSET(__LC_INT_CLOCK, lowcore, int_clock); OFFSET(__LC_MCCK_CLOCK, lowcore, mcck_clock); OFFSET(__LC_CURRENT, lowcore, current_task); - OFFSET(__LC_THREAD_INFO, lowcore, thread_info); OFFSET(__LC_KERNEL_STACK, lowcore, kernel_stack); OFFSET(__LC_ASYNC_STACK, lowcore, async_stack); OFFSET(__LC_PANIC_STACK, lowcore, panic_stack); @@ -173,6 +169,7 @@ int main(void) OFFSET(__LC_PERCPU_OFFSET, lowcore, percpu_offset); OFFSET(__LC_VDSO_PER_CPU, lowcore, vdso_per_cpu_data); OFFSET(__LC_MACHINE_FLAGS, lowcore, machine_flags); + OFFSET(__LC_PREEMPT_COUNT, lowcore, preempt_count); OFFSET(__LC_GMAP, lowcore, gmap); OFFSET(__LC_PASTE, lowcore, paste); /* software defined ABI-relevant lowcore locations 0xe00 - 0xe20 */ diff --git a/arch/s390/kernel/compat_linux.c b/arch/s390/kernel/compat_linux.c index 0f9cd90c11af..a3d14161abcb 100644 --- a/arch/s390/kernel/compat_linux.c +++ b/arch/s390/kernel/compat_linux.c @@ -28,7 +28,6 @@ #include <linux/shm.h> #include <linux/uio.h> #include <linux/quota.h> -#include <linux/module.h> #include <linux/poll.h> #include <linux/personality.h> #include <linux/stat.h> @@ -51,7 +50,7 @@ #include <linux/slab.h> #include <asm/types.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include <net/scm.h> #include <net/sock.h> diff --git a/arch/s390/kernel/compat_signal.c b/arch/s390/kernel/compat_signal.c index 4af60374eba0..362350cc485c 100644 --- a/arch/s390/kernel/compat_signal.c +++ b/arch/s390/kernel/compat_signal.c @@ -23,7 +23,7 @@ #include <linux/personality.h> #include <linux/binfmts.h> #include <asm/ucontext.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include <asm/lowcore.h> #include <asm/switch_to.h> #include "compat_linux.h" @@ -446,7 +446,7 @@ static int setup_frame32(struct ksignal *ksig, sigset_t *set, /* set extra registers only for synchronous signals */ regs->gprs[4] = regs->int_code & 127; regs->gprs[5] = regs->int_parm_long; - regs->gprs[6] = task_thread_info(current)->last_break; + regs->gprs[6] = current->thread.last_break; } return 0; @@ -523,7 +523,7 @@ static int setup_rt_frame32(struct ksignal *ksig, sigset_t *set, regs->gprs[2] = ksig->sig; regs->gprs[3] = (__force __u64) &frame->info; regs->gprs[4] = (__force __u64) &frame->uc; - regs->gprs[5] = task_thread_info(current)->last_break; + regs->gprs[5] = current->thread.last_break; return 0; } diff --git a/arch/s390/kernel/cpcmd.c b/arch/s390/kernel/cpcmd.c index 7f48e568ac64..9f0e4a2785f7 100644 --- a/arch/s390/kernel/cpcmd.c +++ b/arch/s390/kernel/cpcmd.c @@ -9,7 +9,7 @@ #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt #include <linux/kernel.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/slab.h> #include <linux/spinlock.h> #include <linux/stddef.h> diff --git a/arch/s390/kernel/crash_dump.c b/arch/s390/kernel/crash_dump.c index f9293bfefb7f..dd1d5c62c374 100644 --- a/arch/s390/kernel/crash_dump.c +++ b/arch/s390/kernel/crash_dump.c @@ -8,7 +8,8 @@ #include <linux/crash_dump.h> #include <asm/lowcore.h> #include <linux/kernel.h> -#include <linux/module.h> +#include <linux/init.h> +#include <linux/mm.h> #include <linux/gfp.h> #include <linux/slab.h> #include <linux/bootmem.h> @@ -31,6 +32,7 @@ static struct memblock_type oldmem_type = { .max = 1, .total_size = 0, .regions = &oldmem_region, + .name = "oldmem", }; struct save_area { @@ -329,7 +331,11 @@ static void *nt_init_name(void *buf, Elf64_Word type, void *desc, int d_len, static inline void *nt_init(void *buf, Elf64_Word type, void *desc, int d_len) { - return nt_init_name(buf, type, desc, d_len, KEXEC_CORE_NOTE_NAME); + const char *note_name = "LINUX"; + + if (type == NT_PRPSINFO || type == NT_PRSTATUS || type == NT_PRFPREG) + note_name = KEXEC_CORE_NOTE_NAME; + return nt_init_name(buf, type, desc, d_len, note_name); } /* diff --git a/arch/s390/kernel/debug.c b/arch/s390/kernel/debug.c index aa12de72fd47..530226b6cb19 100644 --- a/arch/s390/kernel/debug.c +++ b/arch/s390/kernel/debug.c @@ -19,8 +19,8 @@ #include <linux/ctype.h> #include <linux/string.h> #include <linux/sysctl.h> -#include <asm/uaccess.h> -#include <linux/module.h> +#include <linux/uaccess.h> +#include <linux/export.h> #include <linux/init.h> #include <linux/fs.h> #include <linux/debugfs.h> @@ -866,7 +866,7 @@ static inline void debug_finish_entry(debug_info_t * id, debug_entry_t* active, int level, int exception) { - active->id.stck = get_tod_clock_fast(); + active->id.stck = get_tod_clock_fast() - sched_clock_base_cc; active->id.fields.cpuid = smp_processor_id(); active->caller = __builtin_return_address(0); active->id.fields.exception = exception; @@ -1455,23 +1455,24 @@ int debug_dflt_header_fn(debug_info_t * id, struct debug_view *view, int area, debug_entry_t * entry, char *out_buf) { - struct timespec64 time_spec; + unsigned long sec, usec; char *except_str; unsigned long caller; int rc = 0; unsigned int level; level = entry->id.fields.level; - stck_to_timespec64(entry->id.stck, &time_spec); + sec = (entry->id.stck >> 12) + (sched_clock_base_cc >> 12); + sec = sec - (TOD_UNIX_EPOCH >> 12); + usec = do_div(sec, USEC_PER_SEC); if (entry->id.fields.exception) except_str = "*"; else except_str = "-"; caller = (unsigned long) entry->caller; - rc += sprintf(out_buf, "%02i %011lld:%06lu %1u %1s %02i %p ", - area, (long long)time_spec.tv_sec, - time_spec.tv_nsec / 1000, level, except_str, + rc += sprintf(out_buf, "%02i %011ld:%06lu %1u %1s %02i %p ", + area, sec, usec, level, except_str, entry->id.fields.cpuid, (void *)caller); return rc; } diff --git a/arch/s390/kernel/diag.c b/arch/s390/kernel/diag.c index a97354c8c667..ac6abcd3fe6a 100644 --- a/arch/s390/kernel/diag.c +++ b/arch/s390/kernel/diag.c @@ -5,7 +5,8 @@ * Author(s): Michael Holzheu <holzheu@de.ibm.com> */ -#include <linux/module.h> +#include <linux/export.h> +#include <linux/init.h> #include <linux/cpu.h> #include <linux/seq_file.h> #include <linux/debugfs.h> diff --git a/arch/s390/kernel/dis.c b/arch/s390/kernel/dis.c index c74c59236f44..f7e82302a71e 100644 --- a/arch/s390/kernel/dis.c +++ b/arch/s390/kernel/dis.c @@ -16,13 +16,13 @@ #include <linux/init.h> #include <linux/interrupt.h> #include <linux/delay.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/kallsyms.h> #include <linux/reboot.h> #include <linux/kprobes.h> #include <linux/kdebug.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include <asm/dis.h> #include <asm/io.h> #include <linux/atomic.h> diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index 2374c5b46bbc..4e65c79cc5f2 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -293,6 +293,7 @@ static noinline __init void setup_lowcore_early(void) psw.addr = (unsigned long) s390_base_pgm_handler; S390_lowcore.program_new_psw = psw; s390_base_pgm_handler_fn = early_pgm_check_handler; + S390_lowcore.preempt_count = INIT_PREEMPT_COUNT; } static noinline __init void setup_facility_list(void) @@ -353,6 +354,10 @@ static __init void detect_machine_facilities(void) S390_lowcore.machine_flags |= MACHINE_FLAG_VX; __ctl_set_bit(0, 17); } + if (test_facility(130)) { + S390_lowcore.machine_flags |= MACHINE_FLAG_NX; + __ctl_set_bit(0, 20); + } } static inline void save_vector_registers(void) @@ -363,6 +368,18 @@ static inline void save_vector_registers(void) #endif } +static int __init topology_setup(char *str) +{ + bool enabled; + int rc; + + rc = kstrtobool(str, &enabled); + if (!rc && !enabled) + S390_lowcore.machine_flags &= ~MACHINE_HAS_TOPOLOGY; + return rc; +} +early_param("topology", topology_setup); + static int __init disable_vector_extension(char *str) { S390_lowcore.machine_flags &= ~MACHINE_FLAG_VX; @@ -371,6 +388,21 @@ static int __init disable_vector_extension(char *str) } early_param("novx", disable_vector_extension); +static int __init noexec_setup(char *str) +{ + bool enabled; + int rc; + + rc = kstrtobool(str, &enabled); + if (!rc && !enabled) { + /* Disable no-execute support */ + S390_lowcore.machine_flags &= ~MACHINE_FLAG_NX; + __ctl_clear_bit(0, 20); + } + return rc; +} +early_param("noexec", noexec_setup); + static int __init cad_setup(char *str) { int val; @@ -391,7 +423,49 @@ static int __init cad_init(void) } early_initcall(cad_init); -static __init void rescue_initrd(void) +static __init void memmove_early(void *dst, const void *src, size_t n) +{ + unsigned long addr; + long incr; + psw_t old; + + if (!n) + return; + incr = 1; + if (dst > src) { + incr = -incr; + dst += n - 1; + src += n - 1; + } + old = S390_lowcore.program_new_psw; + S390_lowcore.program_new_psw.mask = __extract_psw(); + asm volatile( + " larl %[addr],1f\n" + " stg %[addr],%[psw_pgm_addr]\n" + "0: mvc 0(1,%[dst]),0(%[src])\n" + " agr %[dst],%[incr]\n" + " agr %[src],%[incr]\n" + " brctg %[n],0b\n" + "1:\n" + : [addr] "=&d" (addr), + [psw_pgm_addr] "=Q" (S390_lowcore.program_new_psw.addr), + [dst] "+&a" (dst), [src] "+&a" (src), [n] "+d" (n) + : [incr] "d" (incr) + : "cc", "memory"); + S390_lowcore.program_new_psw = old; +} + +static __init noinline void ipl_save_parameters(void) +{ + void *src, *dst; + + src = (void *)(unsigned long) S390_lowcore.ipl_parmblock_ptr; + dst = (void *) IPL_PARMBLOCK_ORIGIN; + memmove_early(dst, src, PAGE_SIZE); + S390_lowcore.ipl_parmblock_ptr = IPL_PARMBLOCK_ORIGIN; +} + +static __init noinline void rescue_initrd(void) { #ifdef CONFIG_BLK_DEV_INITRD unsigned long min_initrd_addr = (unsigned long) _end + (4UL << 20); @@ -405,7 +479,7 @@ static __init void rescue_initrd(void) return; if (INITRD_START >= min_initrd_addr) return; - memmove((void *) min_initrd_addr, (void *) INITRD_START, INITRD_SIZE); + memmove_early((void *) min_initrd_addr, (void *) INITRD_START, INITRD_SIZE); INITRD_START = min_initrd_addr; #endif } @@ -467,7 +541,8 @@ void __init startup_init(void) ipl_save_parameters(); rescue_initrd(); clear_bss_section(); - ptff_init(); + ipl_verify_parameters(); + time_early_init(); init_kernel_storage_key(); lockdep_off(); setup_lowcore_early(); diff --git a/arch/s390/kernel/early_printk.c b/arch/s390/kernel/early_printk.c new file mode 100644 index 000000000000..819cb15c67e8 --- /dev/null +++ b/arch/s390/kernel/early_printk.c @@ -0,0 +1,35 @@ +/* + * Copyright IBM Corp. 2017 + */ + +#include <linux/console.h> +#include <linux/kernel.h> +#include <linux/init.h> +#include <asm/sclp.h> + +static void sclp_early_write(struct console *con, const char *s, unsigned int len) +{ + __sclp_early_printk(s, len); +} + +static struct console sclp_early_console = { + .name = "earlysclp", + .write = sclp_early_write, + .flags = CON_PRINTBUFFER | CON_BOOT, + .index = -1, +}; + +static int __init setup_early_printk(char *buf) +{ + if (early_console) + return 0; + /* Accept only "earlyprintk" and "earlyprintk=sclp" */ + if (buf && strncmp(buf, "sclp", 4)) + return 0; + if (!sclp.has_linemode && !sclp.has_vt220) + return 0; + early_console = &sclp_early_console; + register_console(early_console); + return 0; +} +early_param("earlyprintk", setup_early_printk); diff --git a/arch/s390/kernel/ebcdic.c b/arch/s390/kernel/ebcdic.c index b971c6be6298..1d5392b36ad8 100644 --- a/arch/s390/kernel/ebcdic.c +++ b/arch/s390/kernel/ebcdic.c @@ -8,8 +8,8 @@ * Martin Peschke <peschke@fh-brandenburg.de> */ -#include <linux/module.h> -#include <asm/types.h> +#include <linux/types.h> +#include <linux/export.h> #include <asm/ebcdic.h> /* diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index 49a30737adde..dff2152350a7 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -42,7 +42,7 @@ __PT_R13 = __PT_GPRS + 104 __PT_R14 = __PT_GPRS + 112 __PT_R15 = __PT_GPRS + 120 -STACK_SHIFT = PAGE_SHIFT + THREAD_ORDER +STACK_SHIFT = PAGE_SHIFT + THREAD_SIZE_ORDER STACK_SIZE = 1 << STACK_SHIFT STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE @@ -50,7 +50,8 @@ _TIF_WORK = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \ _TIF_UPROBE) _TIF_TRACE = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \ _TIF_SYSCALL_TRACEPOINT) -_CIF_WORK = (_CIF_MCCK_PENDING | _CIF_ASCE | _CIF_FPU) +_CIF_WORK = (_CIF_MCCK_PENDING | _CIF_ASCE_PRIMARY | \ + _CIF_ASCE_SECONDARY | _CIF_FPU) _PIF_WORK = (_PIF_PER_TRAP) #define BASED(name) name-cleanup_critical(%r13) @@ -103,8 +104,7 @@ _PIF_WORK = (_PIF_PER_TRAP) CHECK_STACK 1<<STACK_SHIFT,\savearea aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) j 3f -1: LAST_BREAK %r14 - UPDATE_VTIME %r14,%r15,\timer +1: UPDATE_VTIME %r14,%r15,\timer 2: lg %r15,__LC_ASYNC_STACK # load async stack 3: la %r11,STACK_FRAME_OVERHEAD(%r15) .endm @@ -121,12 +121,6 @@ _PIF_WORK = (_PIF_PER_TRAP) mvc __LC_LAST_UPDATE_TIMER(8),\enter_timer .endm - .macro LAST_BREAK scratch - srag \scratch,%r10,23 - jz .+10 - stg %r10,__TI_last_break(%r12) - .endm - .macro REENABLE_IRQS stg %r8,__LC_RETURN_PSW ni __LC_RETURN_PSW,0xbf @@ -186,14 +180,13 @@ ENTRY(__switch_to) stmg %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task lgr %r1,%r2 aghi %r1,__TASK_thread # thread_struct of prev task - lg %r5,__TASK_thread_info(%r3) # get thread_info of next + lg %r5,__TASK_stack(%r3) # start of kernel stack of next stg %r15,__THREAD_ksp(%r1) # store kernel stack of prev lgr %r1,%r3 aghi %r1,__TASK_thread # thread_struct of next task lgr %r15,%r5 aghi %r15,STACK_INIT # end of kernel stack of next stg %r3,__LC_CURRENT # store task struct of next - stg %r5,__LC_THREAD_INFO # store thread info of next stg %r15,__LC_KERNEL_STACK # store end of kernel stack lg %r15,__THREAD_ksp(%r1) # load kernel stack of next /* c4 is used in guest detection: arch/s390/kernel/perf_cpum_sf.c */ @@ -273,22 +266,22 @@ ENTRY(system_call) stpt __LC_SYNC_ENTER_TIMER .Lsysc_stmg: stmg %r8,%r15,__LC_SAVE_AREA_SYNC - lg %r10,__LC_LAST_BREAK - lg %r12,__LC_THREAD_INFO + lg %r12,__LC_CURRENT + lghi %r13,__TASK_thread lghi %r14,_PIF_SYSCALL .Lsysc_per: lg %r15,__LC_KERNEL_STACK la %r11,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs - LAST_BREAK %r13 .Lsysc_vtime: - UPDATE_VTIME %r10,%r13,__LC_SYNC_ENTER_TIMER + UPDATE_VTIME %r8,%r9,__LC_SYNC_ENTER_TIMER stmg %r0,%r7,__PT_R0(%r11) mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC mvc __PT_PSW(16,%r11),__LC_SVC_OLD_PSW mvc __PT_INT_CODE(4,%r11),__LC_SVC_ILC stg %r14,__PT_FLAGS(%r11) .Lsysc_do_svc: - lg %r10,__TI_sysc_table(%r12) # address of system call table + # load address of system call table + lg %r10,__THREAD_sysc_table(%r13,%r12) llgh %r8,__PT_INT_CODE+2(%r11) slag %r8,%r8,2 # shift and test for svc 0 jnz .Lsysc_nr_ok @@ -347,8 +340,8 @@ ENTRY(system_call) jo .Lsysc_notify_resume TSTMSK __LC_CPU_FLAGS,_CIF_FPU jo .Lsysc_vxrs - TSTMSK __LC_CPU_FLAGS,_CIF_ASCE - jo .Lsysc_uaccess + TSTMSK __LC_CPU_FLAGS,(_CIF_ASCE_PRIMARY|_CIF_ASCE_SECONDARY) + jnz .Lsysc_asce j .Lsysc_return # beware of critical section cleanup # @@ -366,12 +359,15 @@ ENTRY(system_call) jg s390_handle_mcck # TIF bit will be cleared by handler # -# _CIF_ASCE is set, load user space asce +# _CIF_ASCE_PRIMARY and/or CIF_ASCE_SECONDARY set, load user space asce # -.Lsysc_uaccess: - ni __LC_CPU_FLAGS+7,255-_CIF_ASCE +.Lsysc_asce: + ni __LC_CPU_FLAGS+7,255-_CIF_ASCE_PRIMARY lctlg %c1,%c1,__LC_USER_ASCE # load primary asce - j .Lsysc_return + TSTMSK __LC_CPU_FLAGS,_CIF_ASCE_SECONDARY + jz .Lsysc_return + larl %r14,.Lsysc_return + jg set_fs_fixup # # CIF_FPU is set, restore floating-point controls and floating-point registers. @@ -388,14 +384,11 @@ ENTRY(system_call) brasl %r14,do_signal TSTMSK __PT_FLAGS(%r11),_PIF_SYSCALL jno .Lsysc_return +.Lsysc_do_syscall: + lghi %r13,__TASK_thread lmg %r2,%r7,__PT_R2(%r11) # load svc arguments - lg %r10,__TI_sysc_table(%r12) # address of system call table - lghi %r8,0 # svc 0 returns -ENOSYS - llgh %r1,__PT_INT_CODE+2(%r11) # load new svc number - cghi %r1,NR_syscalls - jnl .Lsysc_nr_ok # invalid svc number -> do svc 0 - slag %r8,%r1,2 - j .Lsysc_nr_ok # restart svc + lghi %r1,0 # svc 0 returns -ENOSYS + j .Lsysc_do_svc # # _TIF_NOTIFY_RESUME is set, call do_notify_resume @@ -457,7 +450,7 @@ ENTRY(system_call) # ENTRY(ret_from_fork) la %r11,STACK_FRAME_OVERHEAD(%r15) - lg %r12,__LC_THREAD_INFO + lg %r12,__LC_CURRENT brasl %r14,schedule_tail TRACE_IRQS_ON ssm __LC_SVC_NEW_PSW # reenable interrupts @@ -478,7 +471,7 @@ ENTRY(pgm_check_handler) stpt __LC_SYNC_ENTER_TIMER stmg %r8,%r15,__LC_SAVE_AREA_SYNC lg %r10,__LC_LAST_BREAK - lg %r12,__LC_THREAD_INFO + lg %r12,__LC_CURRENT larl %r13,cleanup_critical lmg %r8,%r9,__LC_PGM_OLD_PSW tmhh %r8,0x0001 # test problem state bit @@ -498,16 +491,16 @@ ENTRY(pgm_check_handler) 1: CHECK_STACK STACK_SIZE,__LC_SAVE_AREA_SYNC aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) j 3f -2: LAST_BREAK %r14 - UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER +2: UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER lg %r15,__LC_KERNEL_STACK - lg %r14,__TI_task(%r12) + lgr %r14,%r12 aghi %r14,__TASK_thread # pointer to thread_struct lghi %r13,__LC_PGM_TDB tm __LC_PGM_ILC+2,0x02 # check for transaction abort jz 3f mvc __THREAD_trap_tdb(256,%r14),0(%r13) 3: la %r11,STACK_FRAME_OVERHEAD(%r15) + stg %r10,__THREAD_last_break(%r14) stmg %r0,%r7,__PT_R0(%r11) mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC stmg %r8,%r9,__PT_PSW(%r11) @@ -537,6 +530,8 @@ ENTRY(pgm_check_handler) LOCKDEP_SYS_EXIT tm __PT_PSW+1(%r11),0x01 # returning to user ? jno .Lsysc_restore + TSTMSK __PT_FLAGS(%r11),_PIF_SYSCALL + jo .Lsysc_do_syscall j .Lsysc_tif # @@ -554,6 +549,7 @@ ENTRY(pgm_check_handler) # .Lpgm_svcper: mvc __LC_RETURN_PSW(8),__LC_SVC_NEW_PSW + lghi %r13,__TASK_thread larl %r14,.Lsysc_per stg %r14,__LC_RETURN_PSW+8 lghi %r14,_PIF_SYSCALL | _PIF_PER_TRAP @@ -566,8 +562,7 @@ ENTRY(io_int_handler) STCK __LC_INT_CLOCK stpt __LC_ASYNC_ENTER_TIMER stmg %r8,%r15,__LC_SAVE_AREA_ASYNC - lg %r10,__LC_LAST_BREAK - lg %r12,__LC_THREAD_INFO + lg %r12,__LC_CURRENT larl %r13,cleanup_critical lmg %r8,%r9,__LC_IO_OLD_PSW SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER @@ -626,7 +621,7 @@ ENTRY(io_int_handler) jo .Lio_work_user # yes -> do resched & signal #ifdef CONFIG_PREEMPT # check for preemptive scheduling - icm %r0,15,__TI_precount(%r12) + icm %r0,15,__LC_PREEMPT_COUNT jnz .Lio_restore # preemption is disabled TSTMSK __TI_flags(%r12),_TIF_NEED_RESCHED jno .Lio_restore @@ -670,8 +665,8 @@ ENTRY(io_int_handler) jo .Lio_notify_resume TSTMSK __LC_CPU_FLAGS,_CIF_FPU jo .Lio_vxrs - TSTMSK __LC_CPU_FLAGS,_CIF_ASCE - jo .Lio_uaccess + TSTMSK __LC_CPU_FLAGS,(_CIF_ASCE_PRIMARY|_CIF_ASCE_SECONDARY) + jnz .Lio_asce j .Lio_return # beware of critical section cleanup # @@ -684,12 +679,15 @@ ENTRY(io_int_handler) j .Lio_return # -# _CIF_ASCE is set, load user space asce +# _CIF_ASCE_PRIMARY and/or CIF_ASCE_SECONDARY set, load user space asce # -.Lio_uaccess: - ni __LC_CPU_FLAGS+7,255-_CIF_ASCE +.Lio_asce: + ni __LC_CPU_FLAGS+7,255-_CIF_ASCE_PRIMARY lctlg %c1,%c1,__LC_USER_ASCE # load primary asce - j .Lio_return + TSTMSK __LC_CPU_FLAGS,_CIF_ASCE_SECONDARY + jz .Lio_return + larl %r14,.Lio_return + jg set_fs_fixup # # CIF_FPU is set, restore floating-point controls and floating-point registers. @@ -740,8 +738,7 @@ ENTRY(ext_int_handler) STCK __LC_INT_CLOCK stpt __LC_ASYNC_ENTER_TIMER stmg %r8,%r15,__LC_SAVE_AREA_ASYNC - lg %r10,__LC_LAST_BREAK - lg %r12,__LC_THREAD_INFO + lg %r12,__LC_CURRENT larl %r13,cleanup_critical lmg %r8,%r9,__LC_EXT_OLD_PSW SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER @@ -798,13 +795,10 @@ ENTRY(save_fpu_regs) TSTMSK __LC_CPU_FLAGS,_CIF_FPU bor %r14 stfpc __THREAD_FPU_fpc(%r2) -.Lsave_fpu_regs_fpc_end: lg %r3,__THREAD_FPU_regs(%r2) TSTMSK __LC_MACHINE_FLAGS,MACHINE_FLAG_VX jz .Lsave_fpu_regs_fp # no -> store FP regs -.Lsave_fpu_regs_vx_low: VSTM %v0,%v15,0,%r3 # vstm 0,15,0(3) -.Lsave_fpu_regs_vx_high: VSTM %v16,%v31,256,%r3 # vstm 16,31,256(3) j .Lsave_fpu_regs_done # -> set CIF_FPU flag .Lsave_fpu_regs_fp: @@ -851,9 +845,7 @@ load_fpu_regs: TSTMSK __LC_MACHINE_FLAGS,MACHINE_FLAG_VX lg %r4,__THREAD_FPU_regs(%r4) # %r4 <- reg save area jz .Lload_fpu_regs_fp # -> no VX, load FP regs -.Lload_fpu_regs_vx: VLM %v0,%v15,0,%r4 -.Lload_fpu_regs_vx_high: VLM %v16,%v31,256,%r4 j .Lload_fpu_regs_done .Lload_fpu_regs_fp: @@ -888,8 +880,7 @@ ENTRY(mcck_int_handler) la %r1,4095 # revalidate r1 spt __LC_CPU_TIMER_SAVE_AREA-4095(%r1) # revalidate cpu timer lmg %r0,%r15,__LC_GPREGS_SAVE_AREA-4095(%r1)# revalidate gprs - lg %r10,__LC_LAST_BREAK - lg %r12,__LC_THREAD_INFO + lg %r12,__LC_CURRENT larl %r13,cleanup_critical lmg %r8,%r9,__LC_MCK_OLD_PSW TSTMSK __LC_MCCK_CODE,MCCK_CODE_SYSTEM_DAMAGE @@ -948,7 +939,7 @@ ENTRY(mcck_int_handler) .Lmcck_panic: lg %r15,__LC_PANIC_STACK - aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) + la %r11,STACK_FRAME_OVERHEAD(%r15) j .Lmcck_skip # @@ -1083,9 +1074,10 @@ cleanup_critical: 0: # check if base register setup + TIF bit load has been done clg %r9,BASED(.Lcleanup_system_call_insn+16) jhe 0f - # set up saved registers r10 and r12 - stg %r10,16(%r11) # r10 last break - stg %r12,32(%r11) # r12 thread-info pointer + # set up saved register r12 task struct pointer + stg %r12,32(%r11) + # set up saved register r13 __TASK_thread offset + mvc 40(8,%r11),BASED(.Lcleanup_system_call_const) 0: # check if the user time update has been done clg %r9,BASED(.Lcleanup_system_call_insn+24) jh 0f @@ -1102,12 +1094,7 @@ cleanup_critical: stg %r15,__LC_SYSTEM_TIMER 0: # update accounting time stamp mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER - # do LAST_BREAK - lg %r9,16(%r11) - srag %r9,%r9,23 - jz 0f - mvc __TI_last_break(8,%r12),16(%r11) -0: # set up saved register r11 + # set up saved register r11 lg %r15,__LC_KERNEL_STACK la %r9,STACK_FRAME_OVERHEAD(%r15) stg %r9,24(%r11) # r11 pt_regs pointer @@ -1129,6 +1116,8 @@ cleanup_critical: .quad .Lsysc_per .quad .Lsysc_vtime+36 .quad .Lsysc_vtime+42 +.Lcleanup_system_call_const: + .quad __TASK_thread .Lcleanup_sysc_tif: larl %r9,.Lsysc_tif diff --git a/arch/s390/kernel/entry.h b/arch/s390/kernel/entry.h index e79f030dd276..33f901865326 100644 --- a/arch/s390/kernel/entry.h +++ b/arch/s390/kernel/entry.h @@ -80,5 +80,6 @@ long sys_s390_pci_mmio_read(unsigned long, void __user *, size_t); DECLARE_PER_CPU(u64, mt_cycles[8]); void verify_facilities(void); +void set_fs_fixup(void); #endif /* _ENTRY_H */ diff --git a/arch/s390/kernel/head.S b/arch/s390/kernel/head.S index 4431905f8cfa..0b5ebf8a3d30 100644 --- a/arch/s390/kernel/head.S +++ b/arch/s390/kernel/head.S @@ -315,7 +315,7 @@ ENTRY(startup_kdump) jg startup_continue .Lstack: - .long 0x8000 + (1<<(PAGE_SHIFT+THREAD_ORDER)) + .long 0x8000 + (1<<(PAGE_SHIFT+THREAD_SIZE_ORDER)) .align 8 6: .long 0x7fffffff,0xffffffff diff --git a/arch/s390/kernel/head64.S b/arch/s390/kernel/head64.S index 03c2b469c472..482d3526e32b 100644 --- a/arch/s390/kernel/head64.S +++ b/arch/s390/kernel/head64.S @@ -32,11 +32,10 @@ ENTRY(startup_continue) # # Setup stack # - larl %r15,init_thread_union - stg %r15,__LC_THREAD_INFO # cache thread info in lowcore - lg %r14,__TI_task(%r15) # cache current in lowcore + larl %r14,init_task stg %r14,__LC_CURRENT - aghi %r15,1<<(PAGE_SHIFT+THREAD_ORDER) # init_task_union + THREAD_SIZE + larl %r15,init_thread_union + aghi %r15,1<<(PAGE_SHIFT+THREAD_SIZE_ORDER) # init_task_union + THREAD_SIZE stg %r15,__LC_KERNEL_STACK # set end of kernel stack aghi %r15,-160 # diff --git a/arch/s390/kernel/idle.c b/arch/s390/kernel/idle.c index 7a55c29b0b33..fb07a70820af 100644 --- a/arch/s390/kernel/idle.c +++ b/arch/s390/kernel/idle.c @@ -12,7 +12,7 @@ #include <linux/notifier.h> #include <linux/init.h> #include <linux/cpu.h> -#include <asm/cputime.h> +#include <linux/cputime.h> #include <asm/nmi.h> #include <asm/smp.h> #include "entry.h" @@ -43,7 +43,7 @@ void enabled_wait(void) idle->clock_idle_enter = idle->clock_idle_exit = 0ULL; idle->idle_time += idle_time; idle->idle_count++; - account_idle_time(idle_time); + account_idle_time(cputime_to_nsecs(idle_time)); write_seqcount_end(&idle->seqcount); } NOKPROBE_SYMBOL(enabled_wait); @@ -57,8 +57,8 @@ static ssize_t show_idle_count(struct device *dev, do { seq = read_seqcount_begin(&idle->seqcount); - idle_count = ACCESS_ONCE(idle->idle_count); - if (ACCESS_ONCE(idle->clock_idle_enter)) + idle_count = READ_ONCE(idle->idle_count); + if (READ_ONCE(idle->clock_idle_enter)) idle_count++; } while (read_seqcount_retry(&idle->seqcount, seq)); return sprintf(buf, "%llu\n", idle_count); @@ -75,16 +75,16 @@ static ssize_t show_idle_time(struct device *dev, do { now = get_tod_clock(); seq = read_seqcount_begin(&idle->seqcount); - idle_time = ACCESS_ONCE(idle->idle_time); - idle_enter = ACCESS_ONCE(idle->clock_idle_enter); - idle_exit = ACCESS_ONCE(idle->clock_idle_exit); + idle_time = READ_ONCE(idle->idle_time); + idle_enter = READ_ONCE(idle->clock_idle_enter); + idle_exit = READ_ONCE(idle->clock_idle_exit); } while (read_seqcount_retry(&idle->seqcount, seq)); idle_time += idle_enter ? ((idle_exit ? : now) - idle_enter) : 0; return sprintf(buf, "%llu\n", idle_time >> 12); } DEVICE_ATTR(idle_time_us, 0444, show_idle_time, NULL); -cputime64_t arch_cpu_idle_time(int cpu) +u64 arch_cpu_idle_time(int cpu) { struct s390_idle_data *idle = &per_cpu(s390_idle, cpu); unsigned long long now, idle_enter, idle_exit; @@ -93,10 +93,11 @@ cputime64_t arch_cpu_idle_time(int cpu) do { now = get_tod_clock(); seq = read_seqcount_begin(&idle->seqcount); - idle_enter = ACCESS_ONCE(idle->clock_idle_enter); - idle_exit = ACCESS_ONCE(idle->clock_idle_exit); + idle_enter = READ_ONCE(idle->clock_idle_enter); + idle_exit = READ_ONCE(idle->clock_idle_exit); } while (read_seqcount_retry(&idle->seqcount, seq)); - return idle_enter ? ((idle_exit ?: now) - idle_enter) : 0; + + return cputime_to_nsecs(idle_enter ? ((idle_exit ?: now) - idle_enter) : 0); } void arch_cpu_idle_enter(void) diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c index 295bfb7124bc..b67dafb7b7cf 100644 --- a/arch/s390/kernel/ipl.c +++ b/arch/s390/kernel/ipl.c @@ -8,7 +8,8 @@ */ #include <linux/types.h> -#include <linux/module.h> +#include <linux/export.h> +#include <linux/init.h> #include <linux/device.h> #include <linux/delay.h> #include <linux/reboot.h> @@ -1546,7 +1547,8 @@ static void dump_reipl_run(struct shutdown_trigger *trigger) unsigned long ipib = (unsigned long) reipl_block_actual; unsigned int csum; - csum = csum_partial(reipl_block_actual, reipl_block_actual->hdr.len, 0); + csum = (__force unsigned int) + csum_partial(reipl_block_actual, reipl_block_actual->hdr.len, 0); mem_assign_absolute(S390_lowcore.ipib, ipib); mem_assign_absolute(S390_lowcore.ipib_checksum, csum); dump_run(trigger); @@ -1863,7 +1865,7 @@ static int __init s390_ipl_init(void) { char str[8] = {0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40}; - sclp_get_ipl_info(&sclp_ipl_info); + sclp_early_get_ipl_info(&sclp_ipl_info); /* * Fix loadparm: There are systems where the (SCSI) LOADPARM * returned by read SCP info is invalid (contains EBCDIC blanks) @@ -1991,10 +1993,9 @@ void __init ipl_update_parameters(void) diag308_set_works = 1; } -void __init ipl_save_parameters(void) +void __init ipl_verify_parameters(void) { struct cio_iplinfo iplinfo; - void *src, *dst; if (cio_get_iplinfo(&iplinfo)) return; @@ -2005,10 +2006,6 @@ void __init ipl_save_parameters(void) if (!iplinfo.is_qdio) return; ipl_flags |= IPL_PARMBLOCK_VALID; - src = (void *)(unsigned long)S390_lowcore.ipl_parmblock_ptr; - dst = (void *)IPL_PARMBLOCK_ORIGIN; - memmove(dst, src, PAGE_SIZE); - S390_lowcore.ipl_parmblock_ptr = IPL_PARMBLOCK_ORIGIN; } static LIST_HEAD(rcall); diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c index 285d6561076d..6dca93b29bed 100644 --- a/arch/s390/kernel/irq.c +++ b/arch/s390/kernel/irq.c @@ -12,11 +12,12 @@ #include <linux/seq_file.h> #include <linux/proc_fs.h> #include <linux/profile.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/kernel.h> #include <linux/ftrace.h> #include <linux/errno.h> #include <linux/slab.h> +#include <linux/init.h> #include <linux/cpu.h> #include <linux/irq.h> #include <asm/irq_regs.h> @@ -168,7 +169,7 @@ void do_softirq_own_stack(void) old = current_stack_pointer(); /* Check against async. stack address range. */ new = S390_lowcore.async_stack; - if (((new - old) >> (PAGE_SHIFT + THREAD_ORDER)) != 0) { + if (((new - old) >> (PAGE_SHIFT + THREAD_SIZE_ORDER)) != 0) { /* Need to switch to the async. stack. */ new -= STACK_FRAME_OVERHEAD; ((struct stack_frame *) new)->back_chain = old; diff --git a/arch/s390/kernel/jump_label.c b/arch/s390/kernel/jump_label.c index 083b05f5f5ab..6aa630a8d24f 100644 --- a/arch/s390/kernel/jump_label.c +++ b/arch/s390/kernel/jump_label.c @@ -4,7 +4,6 @@ * Copyright IBM Corp. 2011 * Author(s): Jan Glauber <jang@linux.vnet.ibm.com> */ -#include <linux/module.h> #include <linux/uaccess.h> #include <linux/stop_machine.h> #include <linux/jump_label.h> diff --git a/arch/s390/kernel/kprobes.c b/arch/s390/kernel/kprobes.c index fdb40424acfe..76f9eda1d7c0 100644 --- a/arch/s390/kernel/kprobes.c +++ b/arch/s390/kernel/kprobes.c @@ -33,7 +33,7 @@ #include <linux/ftrace.h> #include <asm/cacheflush.h> #include <asm/sections.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include <asm/dis.h> DEFINE_PER_CPU(struct kprobe *, current_kprobe); @@ -45,11 +45,17 @@ DEFINE_INSN_CACHE_OPS(dmainsn); static void *alloc_dmainsn_page(void) { - return (void *)__get_free_page(GFP_KERNEL | GFP_DMA); + void *page; + + page = (void *) __get_free_page(GFP_KERNEL | GFP_DMA); + if (page) + set_memory_x((unsigned long) page, 1); + return page; } static void free_dmainsn_page(void *page) { + set_memory_nx((unsigned long) page, 1); free_page((unsigned long)page); } diff --git a/arch/s390/kernel/lgr.c b/arch/s390/kernel/lgr.c index 6ea6d69339b5..ae7dff110054 100644 --- a/arch/s390/kernel/lgr.c +++ b/arch/s390/kernel/lgr.c @@ -5,7 +5,8 @@ * Author(s): Michael Holzheu <holzheu@linux.vnet.ibm.com> */ -#include <linux/module.h> +#include <linux/init.h> +#include <linux/export.h> #include <linux/timer.h> #include <linux/slab.h> #include <asm/facility.h> @@ -183,4 +184,4 @@ static int __init lgr_init(void) lgr_timer_set(); return 0; } -module_init(lgr_init); +device_initcall(lgr_init); diff --git a/arch/s390/kernel/module.c b/arch/s390/kernel/module.c index fbc07891f9e7..1a27f307a920 100644 --- a/arch/s390/kernel/module.c +++ b/arch/s390/kernel/module.c @@ -45,7 +45,8 @@ void *module_alloc(unsigned long size) if (PAGE_ALIGN(size) > MODULES_LEN) return NULL; return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, - GFP_KERNEL, PAGE_KERNEL, 0, NUMA_NO_NODE, + GFP_KERNEL, PAGE_KERNEL_EXEC, + 0, NUMA_NO_NODE, __builtin_return_address(0)); } diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index 9a32f7419d78..80c093e0c6f1 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -13,7 +13,7 @@ #include <linux/errno.h> #include <linux/hardirq.h> #include <linux/time.h> -#include <linux/module.h> +#include <linux/export.h> #include <asm/lowcore.h> #include <asm/smp.h> #include <asm/stp.h> @@ -102,7 +102,7 @@ static int notrace s390_validate_registers(union mci mci, int umode) { int kill_task; u64 zero; - void *fpt_save_area, *fpt_creg_save_area; + void *fpt_save_area; kill_task = 0; zero = 0; @@ -116,6 +116,19 @@ static int notrace s390_validate_registers(union mci mci, int umode) s390_handle_damage(); kill_task = 1; } + /* Validate control registers */ + if (!mci.cr) { + /* + * Control registers have unknown contents. + * Can't recover and therefore stopping machine. + */ + s390_handle_damage(); + } else { + asm volatile( + " lctlg 0,15,0(%0)\n" + " ptlb\n" + : : "a" (&S390_lowcore.cregs_save_area) : "memory"); + } if (!mci.fp) { /* * Floating point registers can't be restored. If the @@ -130,7 +143,6 @@ static int notrace s390_validate_registers(union mci mci, int umode) kill_task = 1; } fpt_save_area = &S390_lowcore.floating_pt_save_area; - fpt_creg_save_area = &S390_lowcore.fpt_creg_save_area; if (!mci.fc) { /* * Floating point control register can't be restored. @@ -142,11 +154,13 @@ static int notrace s390_validate_registers(union mci mci, int umode) */ if (S390_lowcore.fpu_flags & KERNEL_FPC) s390_handle_damage(); - asm volatile("lfpc 0(%0)" : : "a" (&zero), "m" (zero)); + asm volatile("lfpc %0" : : "Q" (zero)); if (!test_cpu_flag(CIF_FPU)) kill_task = 1; - } else - asm volatile("lfpc 0(%0)" : : "a" (fpt_creg_save_area)); + } else { + asm volatile("lfpc %0" + : : "Q" (S390_lowcore.fpt_creg_save_area)); + } if (!MACHINE_HAS_VX) { /* Validate floating point registers */ @@ -167,7 +181,7 @@ static int notrace s390_validate_registers(union mci mci, int umode) " ld 13,104(%0)\n" " ld 14,112(%0)\n" " ld 15,120(%0)\n" - : : "a" (fpt_save_area)); + : : "a" (fpt_save_area) : "memory"); } else { /* Validate vector registers */ union ctlreg0 cr0; @@ -207,18 +221,6 @@ static int notrace s390_validate_registers(union mci mci, int umode) */ kill_task = 1; } - /* Validate control registers */ - if (!mci.cr) { - /* - * Control registers have unknown contents. - * Can't recover and therefore stopping machine. - */ - s390_handle_damage(); - } else { - asm volatile( - " lctlg 0,15,0(%0)" - : : "a" (&S390_lowcore.cregs_save_area)); - } /* * We don't even try to validate the TOD register, since we simply * can't write something sensible into that register. @@ -234,9 +236,9 @@ static int notrace s390_validate_registers(union mci mci, int umode) : : : "0", "cc"); else asm volatile( - " l 0,0(%0)\n" + " l 0,%0\n" " sckpf" - : : "a" (&S390_lowcore.tod_progreg_save_area) + : : "Q" (S390_lowcore.tod_progreg_save_area) : "0", "cc"); /* Validate clock comparator register */ set_clock_comparator(S390_lowcore.clock_comparator); diff --git a/arch/s390/kernel/os_info.c b/arch/s390/kernel/os_info.c index 87f05e475ae8..753ba63182b9 100644 --- a/arch/s390/kernel/os_info.c +++ b/arch/s390/kernel/os_info.c @@ -26,7 +26,7 @@ static struct os_info os_info __page_aligned_data; u32 os_info_csum(struct os_info *os_info) { int size = sizeof(*os_info) - offsetof(struct os_info, version_major); - return csum_partial(&os_info->version_major, size, 0); + return (__force u32)csum_partial(&os_info->version_major, size, 0); } /* @@ -46,7 +46,7 @@ void os_info_entry_add(int nr, void *ptr, u64 size) { os_info.entry[nr].addr = (u64)(unsigned long)ptr; os_info.entry[nr].size = size; - os_info.entry[nr].csum = csum_partial(ptr, size, 0); + os_info.entry[nr].csum = (__force u32)csum_partial(ptr, size, 0); os_info.csum = os_info_csum(&os_info); } @@ -93,7 +93,7 @@ static void os_info_old_alloc(int nr, int align) msg = "copy failed"; goto fail_free; } - csum = csum_partial(buf_align, size, 0); + csum = (__force u32)csum_partial(buf_align, size, 0); if (csum != os_info_old->entry[nr].csum) { msg = "checksum failed"; goto fail_free; diff --git a/arch/s390/kernel/perf_cpum_cf.c b/arch/s390/kernel/perf_cpum_cf.c index 037c2a253ae4..1aba10e90906 100644 --- a/arch/s390/kernel/perf_cpum_cf.c +++ b/arch/s390/kernel/perf_cpum_cf.c @@ -711,7 +711,7 @@ static int __init cpumf_pmu_init(void) return rc; } return cpuhp_setup_state(CPUHP_AP_PERF_S390_CF_ONLINE, - "AP_PERF_S390_CF_ONLINE", + "perf/s390/cf:online", s390_pmu_online_cpu, s390_pmu_offline_cpu); } early_initcall(cpumf_pmu_init); diff --git a/arch/s390/kernel/perf_cpum_cf_events.c b/arch/s390/kernel/perf_cpum_cf_events.c index 4554a4bae39e..c343ac2cf6c5 100644 --- a/arch/s390/kernel/perf_cpum_cf_events.c +++ b/arch/s390/kernel/perf_cpum_cf_events.c @@ -309,7 +309,7 @@ __init const struct attribute_group **cpumf_cf_event_group(void) default: model = NULL; break; - }; + } if (!model) goto out; diff --git a/arch/s390/kernel/perf_cpum_sf.c b/arch/s390/kernel/perf_cpum_sf.c index fcc634c1479a..1c0b58545c04 100644 --- a/arch/s390/kernel/perf_cpum_sf.c +++ b/arch/s390/kernel/perf_cpum_sf.c @@ -995,39 +995,36 @@ static int perf_push_sample(struct perf_event *event, struct sf_raw_sample *sfr) regs.int_parm = CPU_MF_INT_SF_PRA; sde_regs = (struct perf_sf_sde_regs *) ®s.int_parm_long; - regs.psw.addr = sfr->basic.ia; - if (sfr->basic.T) - regs.psw.mask |= PSW_MASK_DAT; - if (sfr->basic.W) - regs.psw.mask |= PSW_MASK_WAIT; - if (sfr->basic.P) - regs.psw.mask |= PSW_MASK_PSTATE; - switch (sfr->basic.AS) { - case 0x0: - regs.psw.mask |= PSW_ASC_PRIMARY; - break; - case 0x1: - regs.psw.mask |= PSW_ASC_ACCREG; - break; - case 0x2: - regs.psw.mask |= PSW_ASC_SECONDARY; - break; - case 0x3: - regs.psw.mask |= PSW_ASC_HOME; - break; - } + psw_bits(regs.psw).ia = sfr->basic.ia; + psw_bits(regs.psw).t = sfr->basic.T; + psw_bits(regs.psw).w = sfr->basic.W; + psw_bits(regs.psw).p = sfr->basic.P; + psw_bits(regs.psw).as = sfr->basic.AS; /* - * A non-zero guest program parameter indicates a guest - * sample. - * Note that some early samples or samples from guests without + * Use the hardware provided configuration level to decide if the + * sample belongs to a guest or host. If that is not available, + * fall back to the following heuristics: + * A non-zero guest program parameter always indicates a guest + * sample. Some early samples or samples from guests without * lpp usage would be misaccounted to the host. We use the asn - * value as a heuristic to detect most of these guest samples. - * If the value differs from the host hpp value, we assume - * it to be a KVM guest. + * value as an addon heuristic to detect most of these guest samples. + * If the value differs from the host hpp value, we assume to be a + * KVM guest. */ - if (sfr->basic.gpp || sfr->basic.prim_asn != (u16) sfr->basic.hpp) + switch (sfr->basic.CL) { + case 1: /* logical partition */ + sde_regs->in_guest = 0; + break; + case 2: /* virtual machine */ sde_regs->in_guest = 1; + break; + default: /* old machine, use heuristics */ + if (sfr->basic.gpp || + sfr->basic.prim_asn != (u16)sfr->basic.hpp) + sde_regs->in_guest = 1; + break; + } overflow = 0; if (perf_exclude_event(event, ®s, sde_regs)) @@ -1626,7 +1623,7 @@ static int __init init_cpum_sampling_pmu(void) goto out; } - cpuhp_setup_state(CPUHP_AP_PERF_S390_SF_ONLINE, "AP_PERF_S390_SF_ONLINE", + cpuhp_setup_state(CPUHP_AP_PERF_S390_SF_ONLINE, "perf/s390/sf:online", s390_pmu_sf_online_cpu, s390_pmu_sf_offline_cpu); out: return err; diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index bba4fa74b321..54281660582c 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -23,7 +23,7 @@ #include <linux/compat.h> #include <linux/kprobes.h> #include <linux/random.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/init_task.h> #include <asm/io.h> #include <asm/processor.h> @@ -100,10 +100,9 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) return 0; } -int copy_thread(unsigned long clone_flags, unsigned long new_stackp, - unsigned long arg, struct task_struct *p) +int copy_thread_tls(unsigned long clone_flags, unsigned long new_stackp, + unsigned long arg, struct task_struct *p, unsigned long tls) { - struct thread_info *ti; struct fake_frame { struct stack_frame sf; @@ -121,9 +120,8 @@ int copy_thread(unsigned long clone_flags, unsigned long new_stackp, memset(&p->thread.per_event, 0, sizeof(p->thread.per_event)); clear_tsk_thread_flag(p, TIF_SINGLE_STEP); /* Initialize per thread user and system timer values */ - ti = task_thread_info(p); - ti->user_timer = 0; - ti->system_timer = 0; + p->thread.user_timer = 0; + p->thread.system_timer = 0; frame->sf.back_chain = 0; /* new return point is ret_from_fork */ @@ -158,7 +156,6 @@ int copy_thread(unsigned long clone_flags, unsigned long new_stackp, /* Set a new TLS ? */ if (clone_flags & CLONE_SETTLS) { - unsigned long tls = frame->childregs.gprs[6]; if (is_compat_task()) { p->thread.acrs[0] = (unsigned int)tls; } else { @@ -236,3 +233,16 @@ unsigned long arch_randomize_brk(struct mm_struct *mm) ret = PAGE_ALIGN(mm->brk + brk_rnd()); return (ret > mm->brk) ? ret : mm->brk; } + +void set_fs_fixup(void) +{ + struct pt_regs *regs = current_pt_regs(); + static bool warned; + + set_fs(USER_DS); + if (warned) + return; + WARN(1, "Unbalanced set_fs - int code: 0x%x\n", regs->int_code); + show_registers(regs); + warned = true; +} diff --git a/arch/s390/kernel/processor.c b/arch/s390/kernel/processor.c index 81d0808085e6..21004aaac69b 100644 --- a/arch/s390/kernel/processor.c +++ b/arch/s390/kernel/processor.c @@ -32,7 +32,7 @@ static bool machine_has_cpu_mhz; void __init cpu_detect_mhz_feature(void) { if (test_facility(34) && __ecag(ECAG_CPU_ATTRIBUTE, 0) != -1UL) - machine_has_cpu_mhz = 1; + machine_has_cpu_mhz = true; } static void update_cpu_mhz(void *arg) @@ -53,7 +53,7 @@ void s390_update_cpu_mhz(void) on_each_cpu(update_cpu_mhz, NULL, 0); } -void notrace cpu_relax(void) +void notrace cpu_relax_yield(void) { if (!smp_cpu_mtid && MACHINE_HAS_DIAG44) { diag_stat_inc(DIAG_STAT_X044); @@ -61,7 +61,7 @@ void notrace cpu_relax(void) } barrier(); } -EXPORT_SYMBOL(cpu_relax); +EXPORT_SYMBOL(cpu_relax_yield); /* * cpu_init - initializes state that is per-CPU. @@ -92,7 +92,7 @@ static void show_cpu_summary(struct seq_file *m, void *v) { static const char *hwcap_str[] = { "esan3", "zarch", "stfle", "msa", "ldisp", "eimm", "dfp", - "edat", "etf3eh", "highgprs", "te", "vx" + "edat", "etf3eh", "highgprs", "te", "vx", "vxd", "vxe" }; static const char * const int_hwcap_str[] = { "sie" diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c index 9336e824e2db..12020b55887b 100644 --- a/arch/s390/kernel/ptrace.c +++ b/arch/s390/kernel/ptrace.c @@ -26,7 +26,7 @@ #include <asm/page.h> #include <asm/pgtable.h> #include <asm/pgalloc.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include <asm/unistd.h> #include <asm/switch_to.h> #include "entry.h" @@ -461,7 +461,7 @@ long arch_ptrace(struct task_struct *child, long request, } return 0; case PTRACE_GET_LAST_BREAK: - put_user(task_thread_info(child)->last_break, + put_user(child->thread.last_break, (unsigned long __user *) data); return 0; case PTRACE_ENABLE_TE: @@ -811,7 +811,7 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request, } return 0; case PTRACE_GET_LAST_BREAK: - put_user(task_thread_info(child)->last_break, + put_user(child->thread.last_break, (unsigned int __user *) data); return 0; } @@ -963,6 +963,11 @@ static int s390_fpregs_set(struct task_struct *target, if (target == current) save_fpu_regs(); + if (MACHINE_HAS_VX) + convert_vx_to_fp(fprs, target->thread.fpu.vxrs); + else + memcpy(&fprs, target->thread.fpu.fprs, sizeof(fprs)); + /* If setting FPC, must validate it first. */ if (count > 0 && pos < offsetof(s390_fp_regs, fprs)) { u32 ufpc[2] = { target->thread.fpu.fpc, 0 }; @@ -997,10 +1002,10 @@ static int s390_last_break_get(struct task_struct *target, if (count > 0) { if (kbuf) { unsigned long *k = kbuf; - *k = task_thread_info(target)->last_break; + *k = target->thread.last_break; } else { unsigned long __user *u = ubuf; - if (__put_user(task_thread_info(target)->last_break, u)) + if (__put_user(target->thread.last_break, u)) return -EFAULT; } } @@ -1067,6 +1072,9 @@ static int s390_vxrs_low_set(struct task_struct *target, if (target == current) save_fpu_regs(); + for (i = 0; i < __NUM_VXRS_LOW; i++) + vxrs[i] = *((__u64 *)(target->thread.fpu.vxrs + i) + 1); + rc = user_regset_copyin(&pos, &count, &kbuf, &ubuf, vxrs, 0, -1); if (rc == 0) for (i = 0; i < __NUM_VXRS_LOW; i++) @@ -1113,7 +1121,7 @@ static int s390_system_call_get(struct task_struct *target, unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf) { - unsigned int *data = &task_thread_info(target)->system_call; + unsigned int *data = &target->thread.system_call; return user_regset_copyout(&pos, &count, &kbuf, &ubuf, data, 0, sizeof(unsigned int)); } @@ -1123,7 +1131,7 @@ static int s390_system_call_set(struct task_struct *target, unsigned int pos, unsigned int count, const void *kbuf, const void __user *ubuf) { - unsigned int *data = &task_thread_info(target)->system_call; + unsigned int *data = &target->thread.system_call; return user_regset_copyin(&pos, &count, &kbuf, &ubuf, data, 0, sizeof(unsigned int)); } @@ -1327,7 +1335,7 @@ static int s390_compat_last_break_get(struct task_struct *target, compat_ulong_t last_break; if (count > 0) { - last_break = task_thread_info(target)->last_break; + last_break = target->thread.last_break; if (kbuf) { unsigned long *k = kbuf; *k = last_break; diff --git a/arch/s390/kernel/sclp.c b/arch/s390/kernel/sclp.c deleted file mode 100644 index f08af675f36f..000000000000 --- a/arch/s390/kernel/sclp.c +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Copyright IBM Corp. 2015 - * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> - */ -#include <linux/kernel.h> -#include <asm/ebcdic.h> -#include <asm/irq.h> -#include <asm/lowcore.h> -#include <asm/processor.h> -#include <asm/sclp.h> - -#define EVTYP_VT220MSG_MASK 0x00000040 -#define EVTYP_MSG_MASK 0x40000000 - -static char _sclp_work_area[4096] __aligned(PAGE_SIZE) __section(data); -static bool have_vt220 __section(data); -static bool have_linemode __section(data); - -static void _sclp_wait_int(void) -{ - unsigned long cr0, cr0_new, psw_mask, addr; - psw_t psw_ext_save, psw_wait; - - __ctl_store(cr0, 0, 0); - cr0_new = cr0 | 0x200; - __ctl_load(cr0_new, 0, 0); - - psw_ext_save = S390_lowcore.external_new_psw; - psw_mask = __extract_psw(); - S390_lowcore.external_new_psw.mask = psw_mask; - psw_wait.mask = psw_mask | PSW_MASK_EXT | PSW_MASK_WAIT; - S390_lowcore.ext_int_code = 0; - - do { - asm volatile( - " larl %[addr],0f\n" - " stg %[addr],%[psw_wait_addr]\n" - " stg %[addr],%[psw_ext_addr]\n" - " lpswe %[psw_wait]\n" - "0:\n" - : [addr] "=&d" (addr), - [psw_wait_addr] "=Q" (psw_wait.addr), - [psw_ext_addr] "=Q" (S390_lowcore.external_new_psw.addr) - : [psw_wait] "Q" (psw_wait) - : "cc", "memory"); - } while (S390_lowcore.ext_int_code != EXT_IRQ_SERVICE_SIG); - - __ctl_load(cr0, 0, 0); - S390_lowcore.external_new_psw = psw_ext_save; -} - -static int _sclp_servc(unsigned int cmd, char *sccb) -{ - unsigned int cc; - - do { - asm volatile( - " .insn rre,0xb2200000,%1,%2\n" - " ipm %0\n" - : "=d" (cc) : "d" (cmd), "a" (sccb) - : "cc", "memory"); - cc >>= 28; - if (cc == 3) - return -EINVAL; - _sclp_wait_int(); - } while (cc != 0); - return (*(unsigned short *)(sccb + 6) == 0x20) ? 0 : -EIO; -} - -static int _sclp_setup(int disable) -{ - static unsigned char init_sccb[] = { - 0x00, 0x1c, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x04, - 0x80, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 - }; - unsigned int *masks; - int rc; - - memcpy(_sclp_work_area, init_sccb, 28); - masks = (unsigned int *)(_sclp_work_area + 12); - if (disable) - memset(masks, 0, 16); - /* SCLP write mask */ - rc = _sclp_servc(0x00780005, _sclp_work_area); - if (rc) - return rc; - have_vt220 = masks[2] & EVTYP_VT220MSG_MASK; - have_linemode = masks[2] & EVTYP_MSG_MASK; - return 0; -} - -/* Output multi-line text using SCLP Message interface. */ -static void _sclp_print_lm(const char *str) -{ - static unsigned char write_head[] = { - /* sccb header */ - 0x00, 0x52, /* 0 */ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 2 */ - /* evbuf */ - 0x00, 0x4a, /* 8 */ - 0x02, 0x00, 0x00, 0x00, /* 10 */ - /* mdb */ - 0x00, 0x44, /* 14 */ - 0x00, 0x01, /* 16 */ - 0xd4, 0xc4, 0xc2, 0x40, /* 18 */ - 0x00, 0x00, 0x00, 0x01, /* 22 */ - /* go */ - 0x00, 0x38, /* 26 */ - 0x00, 0x01, /* 28 */ - 0x00, 0x00, 0x00, 0x00, /* 30 */ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 34 */ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 42 */ - 0x00, 0x00, 0x00, 0x00, /* 50 */ - 0x00, 0x00, /* 54 */ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 56 */ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 64 */ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 72 */ - 0x00, 0x00, /* 80 */ - }; - static unsigned char write_mto[] = { - /* mto */ - 0x00, 0x0a, /* 0 */ - 0x00, 0x04, /* 2 */ - 0x10, 0x00, /* 4 */ - 0x00, 0x00, 0x00, 0x00 /* 6 */ - }; - unsigned char *ptr, ch; - unsigned int count; - - memcpy(_sclp_work_area, write_head, sizeof(write_head)); - ptr = _sclp_work_area + sizeof(write_head); - do { - memcpy(ptr, write_mto, sizeof(write_mto)); - for (count = sizeof(write_mto); (ch = *str++) != 0; count++) { - if (ch == 0x0a) - break; - ptr[count] = _ascebc[ch]; - } - /* Update length fields in mto, mdb, evbuf and sccb */ - *(unsigned short *) ptr = count; - *(unsigned short *)(_sclp_work_area + 14) += count; - *(unsigned short *)(_sclp_work_area + 8) += count; - *(unsigned short *)(_sclp_work_area + 0) += count; - ptr += count; - } while (ch != 0); - - /* SCLP write data */ - _sclp_servc(0x00760005, _sclp_work_area); -} - -/* Output multi-line text (plus a newline) using SCLP VT220 - * interface. - */ -static void _sclp_print_vt220(const char *str) -{ - static unsigned char const write_head[] = { - /* sccb header */ - 0x00, 0x0e, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - /* evbuf header */ - 0x00, 0x06, - 0x1a, 0x00, 0x00, 0x00, - }; - size_t len = strlen(str); - - if (sizeof(write_head) + len >= sizeof(_sclp_work_area)) - len = sizeof(_sclp_work_area) - sizeof(write_head) - 1; - - memcpy(_sclp_work_area, write_head, sizeof(write_head)); - memcpy(_sclp_work_area + sizeof(write_head), str, len); - _sclp_work_area[sizeof(write_head) + len] = '\n'; - - /* Update length fields in evbuf and sccb headers */ - *(unsigned short *)(_sclp_work_area + 8) += len + 1; - *(unsigned short *)(_sclp_work_area + 0) += len + 1; - - /* SCLP write data */ - (void)_sclp_servc(0x00760005, _sclp_work_area); -} - -/* Output one or more lines of text on the SCLP console (VT220 and / - * or line-mode). All lines get terminated; no need for a trailing LF. - */ -void _sclp_print_early(const char *str) -{ - if (_sclp_setup(0) != 0) - return; - if (have_linemode) - _sclp_print_lm(str); - if (have_vt220) - _sclp_print_vt220(str); - _sclp_setup(1); -} diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 7f7ba5f23f13..e4d811f17971 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -35,6 +35,7 @@ #include <linux/root_dev.h> #include <linux/console.h> #include <linux/kernel_stat.h> +#include <linux/dma-contiguous.h> #include <linux/device.h> #include <linux/notifier.h> #include <linux/pfn.h> @@ -303,7 +304,7 @@ static void __init setup_lowcore(void) * Setup lowcore for boot cpu */ BUILD_BUG_ON(sizeof(struct lowcore) != LC_PAGES * 4096); - lc = __alloc_bootmem_low(LC_PAGES * PAGE_SIZE, LC_PAGES * PAGE_SIZE, 0); + lc = memblock_virt_alloc_low(sizeof(*lc), sizeof(*lc)); lc->restart_psw.mask = PSW_KERNEL_BITS; lc->restart_psw.addr = (unsigned long) restart_int_handler; lc->external_new_psw.mask = PSW_KERNEL_BITS | @@ -324,15 +325,15 @@ static void __init setup_lowcore(void) lc->kernel_stack = ((unsigned long) &init_thread_union) + THREAD_SIZE - STACK_FRAME_OVERHEAD - sizeof(struct pt_regs); lc->async_stack = (unsigned long) - __alloc_bootmem(ASYNC_SIZE, ASYNC_SIZE, 0) + memblock_virt_alloc(ASYNC_SIZE, ASYNC_SIZE) + ASYNC_SIZE - STACK_FRAME_OVERHEAD - sizeof(struct pt_regs); lc->panic_stack = (unsigned long) - __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, 0) + memblock_virt_alloc(PAGE_SIZE, PAGE_SIZE) + PAGE_SIZE - STACK_FRAME_OVERHEAD - sizeof(struct pt_regs); - lc->current_task = (unsigned long) init_thread_union.thread_info.task; - lc->thread_info = (unsigned long) &init_thread_union; + lc->current_task = (unsigned long)&init_task; lc->lpp = LPP_MAGIC; lc->machine_flags = S390_lowcore.machine_flags; + lc->preempt_count = S390_lowcore.preempt_count; lc->stfl_fac_list = S390_lowcore.stfl_fac_list; memcpy(lc->stfle_fac_list, S390_lowcore.stfle_fac_list, MAX_FACILITY_BIT/8); @@ -349,7 +350,7 @@ static void __init setup_lowcore(void) lc->last_update_timer = S390_lowcore.last_update_timer; lc->last_update_clock = S390_lowcore.last_update_clock; - restart_stack = __alloc_bootmem(ASYNC_SIZE, ASYNC_SIZE, 0); + restart_stack = memblock_virt_alloc(ASYNC_SIZE, ASYNC_SIZE); restart_stack += ASYNC_SIZE; /* @@ -412,7 +413,7 @@ static void __init setup_resources(void) bss_resource.end = (unsigned long) &__bss_stop - 1; for_each_memblock(memory, reg) { - res = alloc_bootmem_low(sizeof(*res)); + res = memblock_virt_alloc(sizeof(*res), 8); res->flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM; res->name = "System RAM"; @@ -426,7 +427,7 @@ static void __init setup_resources(void) std_res->start > res->end) continue; if (std_res->end > res->end) { - sub_res = alloc_bootmem_low(sizeof(*sub_res)); + sub_res = memblock_virt_alloc(sizeof(*sub_res), 8); *sub_res = *std_res; sub_res->end = res->end; std_res->start = res->end + 1; @@ -445,7 +446,7 @@ static void __init setup_resources(void) * part of the System RAM resource. */ if (crashk_res.end) { - memblock_add(crashk_res.start, resource_size(&crashk_res)); + memblock_add_node(crashk_res.start, resource_size(&crashk_res), 0); memblock_reserve(crashk_res.start, resource_size(&crashk_res)); insert_resource(&iomem_resource, &crashk_res); } @@ -484,7 +485,7 @@ static void __init setup_memory_end(void) max_pfn = max_low_pfn = PFN_DOWN(memory_end); memblock_remove(memory_end, ULONG_MAX); - pr_notice("Max memory size: %luMB\n", memory_end >> 20); + pr_notice("The maximum memory size is %luMB\n", memory_end >> 20); } static void __init setup_vmcoreinfo(void) @@ -635,6 +636,8 @@ static void __init reserve_crashkernel(void) static void __init reserve_initrd(void) { #ifdef CONFIG_BLK_DEV_INITRD + if (!INITRD_START || !INITRD_SIZE) + return; initrd_start = INITRD_START; initrd_end = initrd_start + INITRD_SIZE; memblock_reserve(INITRD_START, INITRD_SIZE); @@ -649,7 +652,7 @@ static void __init check_initrd(void) #ifdef CONFIG_BLK_DEV_INITRD if (INITRD_START && INITRD_SIZE && !memblock_is_region_memory(INITRD_START, INITRD_SIZE)) { - pr_err("initrd does not fit memory.\n"); + pr_err("The initial RAM disk does not fit into the memory\n"); memblock_free(INITRD_START, INITRD_SIZE); initrd_start = initrd_end = 0; } @@ -746,7 +749,7 @@ static int __init setup_hwcaps(void) /* * Huge page support HWCAP_S390_HPAGE is bit 7. */ - if (MACHINE_HAS_HPAGE) + if (MACHINE_HAS_EDAT1) elf_hwcap |= HWCAP_S390_HPAGE; /* @@ -766,8 +769,14 @@ static int __init setup_hwcaps(void) * can be disabled with the "novx" parameter. Use MACHINE_HAS_VX * instead of facility bit 129. */ - if (MACHINE_HAS_VX) + if (MACHINE_HAS_VX) { elf_hwcap |= HWCAP_S390_VXRS; + if (test_facility(134)) + elf_hwcap |= HWCAP_S390_VXRS_EXT; + if (test_facility(135)) + elf_hwcap |= HWCAP_S390_VXRS_BCD; + } + get_cpu_id(&cpu_id); add_device_randomness(&cpu_id, sizeof(cpu_id)); switch (cpu_id.machine) { @@ -819,10 +828,10 @@ static void __init setup_randomness(void) { struct sysinfo_3_2_2 *vmms; - vmms = (struct sysinfo_3_2_2 *) alloc_page(GFP_KERNEL); - if (vmms && stsi(vmms, 3, 2, 2) == 0 && vmms->count) - add_device_randomness(&vmms, vmms->count); - free_page((unsigned long) vmms); + vmms = (struct sysinfo_3_2_2 *) memblock_alloc(PAGE_SIZE, PAGE_SIZE); + if (stsi(vmms, 3, 2, 2) == 0 && vmms->count) + add_device_randomness(&vmms->vm, sizeof(vmms->vm[0]) * vmms->count); + memblock_free((unsigned long) vmms, PAGE_SIZE); } /* @@ -903,6 +912,7 @@ void __init setup_arch(char **cmdline_p) setup_memory_end(); setup_memory(); + dma_contiguous_reserve(memory_end); check_initrd(); reserve_crashkernel(); @@ -921,6 +931,8 @@ void __init setup_arch(char **cmdline_p) cpu_detect_mhz_feature(); cpu_init(); numa_setup(); + smp_detect_cpus(); + topology_init_early(); /* * Create kernel page tables and switch to virtual addressing. diff --git a/arch/s390/kernel/signal.c b/arch/s390/kernel/signal.c index d82562cf0a0e..62a4c263e887 100644 --- a/arch/s390/kernel/signal.c +++ b/arch/s390/kernel/signal.c @@ -26,7 +26,7 @@ #include <linux/syscalls.h> #include <linux/compat.h> #include <asm/ucontext.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include <asm/lowcore.h> #include <asm/switch_to.h> #include "entry.h" @@ -359,7 +359,7 @@ static int setup_frame(int sig, struct k_sigaction *ka, /* set extra registers only for synchronous signals */ regs->gprs[4] = regs->int_code & 127; regs->gprs[5] = regs->int_parm_long; - regs->gprs[6] = task_thread_info(current)->last_break; + regs->gprs[6] = current->thread.last_break; } return 0; } @@ -430,7 +430,7 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, regs->gprs[2] = ksig->sig; regs->gprs[3] = (unsigned long) &frame->info; regs->gprs[4] = (unsigned long) &frame->uc; - regs->gprs[5] = task_thread_info(current)->last_break; + regs->gprs[5] = current->thread.last_break; return 0; } @@ -467,13 +467,13 @@ void do_signal(struct pt_regs *regs) * the debugger may change all our registers, including the system * call information. */ - current_thread_info()->system_call = + current->thread.system_call = test_pt_regs_flag(regs, PIF_SYSCALL) ? regs->int_code : 0; if (get_signal(&ksig)) { /* Whee! Actually deliver the signal. */ - if (current_thread_info()->system_call) { - regs->int_code = current_thread_info()->system_call; + if (current->thread.system_call) { + regs->int_code = current->thread.system_call; /* Check for system call restarting. */ switch (regs->gprs[2]) { case -ERESTART_RESTARTBLOCK: @@ -506,8 +506,8 @@ void do_signal(struct pt_regs *regs) /* No handlers present - check for system call restart */ clear_pt_regs_flag(regs, PIF_SYSCALL); - if (current_thread_info()->system_call) { - regs->int_code = current_thread_info()->system_call; + if (current->thread.system_call) { + regs->int_code = current->thread.system_call; switch (regs->gprs[2]) { case -ERESTART_RESTARTBLOCK: /* Restart with sys_restart_syscall */ diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 35531fe1c5ea..d0a74d7ce433 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -19,7 +19,8 @@ #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt #include <linux/workqueue.h> -#include <linux/module.h> +#include <linux/bootmem.h> +#include <linux/export.h> #include <linux/init.h> #include <linux/mm.h> #include <linux/err.h> @@ -259,16 +260,14 @@ static void pcpu_prepare_secondary(struct pcpu *pcpu, int cpu) static void pcpu_attach_task(struct pcpu *pcpu, struct task_struct *tsk) { struct lowcore *lc = pcpu->lowcore; - struct thread_info *ti = task_thread_info(tsk); lc->kernel_stack = (unsigned long) task_stack_page(tsk) + THREAD_SIZE - STACK_FRAME_OVERHEAD - sizeof(struct pt_regs); - lc->thread_info = (unsigned long) task_thread_info(tsk); lc->current_task = (unsigned long) tsk; lc->lpp = LPP_MAGIC; lc->current_pid = tsk->pid; - lc->user_timer = ti->user_timer; - lc->system_timer = ti->system_timer; + lc->user_timer = tsk->thread.user_timer; + lc->system_timer = tsk->thread.system_timer; lc->steal_timer = 0; } @@ -368,10 +367,15 @@ int smp_find_processor_id(u16 address) return -1; } -int smp_vcpu_scheduled(int cpu) +bool arch_vcpu_is_preempted(int cpu) { - return pcpu_running(pcpu_devices + cpu); + if (test_cpu_flag_of(CIF_ENABLED_WAIT, cpu)) + return false; + if (pcpu_running(pcpu_devices + cpu)) + return false; + return true; } +EXPORT_SYMBOL(arch_vcpu_is_preempted); void smp_yield_cpu(int cpu) { @@ -657,14 +661,12 @@ int smp_cpu_get_polarization(int cpu) return pcpu_devices[cpu].polarization; } -static struct sclp_core_info *smp_get_core_info(void) +static void __ref smp_get_core_info(struct sclp_core_info *info, int early) { static int use_sigp_detection; - struct sclp_core_info *info; int address; - info = kzalloc(sizeof(*info), GFP_KERNEL); - if (info && (use_sigp_detection || sclp_get_core_info(info))) { + if (use_sigp_detection || sclp_get_core_info(info, early)) { use_sigp_detection = 1; for (address = 0; address < (SCLP_MAX_CORES << smp_cpu_mt_shift); @@ -678,7 +680,6 @@ static struct sclp_core_info *smp_get_core_info(void) } info->combined = info->configured; } - return info; } static int smp_add_present_cpu(int cpu); @@ -719,17 +720,15 @@ static int __smp_rescan_cpus(struct sclp_core_info *info, int sysfs_add) return nr; } -static void __init smp_detect_cpus(void) +void __init smp_detect_cpus(void) { unsigned int cpu, mtid, c_cpus, s_cpus; struct sclp_core_info *info; u16 address; /* Get CPU information */ - info = smp_get_core_info(); - if (!info) - panic("smp_detect_cpus failed to allocate memory\n"); - + info = memblock_virt_alloc(sizeof(*info), 8); + smp_get_core_info(info, 1); /* Find boot CPU type */ if (sclp.has_core_type) { address = stap(); @@ -765,7 +764,7 @@ static void __init smp_detect_cpus(void) get_online_cpus(); __smp_rescan_cpus(info, 0); put_online_cpus(); - kfree(info); + memblock_free_early((unsigned long)info, sizeof(*info)); } /* @@ -802,7 +801,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle) pcpu = pcpu_devices + cpu; if (pcpu->state != CPU_STATE_CONFIGURED) return -EIO; - base = cpu - (cpu % (smp_cpu_mtid + 1)); + base = smp_get_base_cpu(cpu); for (i = 0; i <= smp_cpu_mtid; i++) { if (base + i < nr_cpu_ids) if (cpu_online(base + i)) @@ -902,7 +901,6 @@ void __init smp_prepare_cpus(unsigned int max_cpus) /* request the 0x1202 external call external interrupt */ if (register_external_irq(EXT_IRQ_EXTERNAL_CALL, do_ext_call_interrupt)) panic("Couldn't request external interrupt 0x1202"); - smp_detect_cpus(); } void __init smp_prepare_boot_cpu(void) @@ -968,7 +966,7 @@ static ssize_t cpu_configure_store(struct device *dev, rc = -EBUSY; /* disallow configuration changes of online cpus and cpu 0 */ cpu = dev->id; - cpu -= cpu % (smp_cpu_mtid + 1); + cpu = smp_get_base_cpu(cpu); if (cpu == 0) goto out; for (i = 0; i <= smp_cpu_mtid; i++) @@ -1047,22 +1045,18 @@ static struct attribute_group cpu_online_attr_group = { .attrs = cpu_online_attrs, }; -static int smp_cpu_notify(struct notifier_block *self, unsigned long action, - void *hcpu) +static int smp_cpu_online(unsigned int cpu) { - unsigned int cpu = (unsigned int)(long)hcpu; struct device *s = &per_cpu(cpu_device, cpu)->dev; - int err = 0; - switch (action & ~CPU_TASKS_FROZEN) { - case CPU_ONLINE: - err = sysfs_create_group(&s->kobj, &cpu_online_attr_group); - break; - case CPU_DEAD: - sysfs_remove_group(&s->kobj, &cpu_online_attr_group); - break; - } - return notifier_from_errno(err); + return sysfs_create_group(&s->kobj, &cpu_online_attr_group); +} +static int smp_cpu_pre_down(unsigned int cpu) +{ + struct device *s = &per_cpu(cpu_device, cpu)->dev; + + sysfs_remove_group(&s->kobj, &cpu_online_attr_group); + return 0; } static int smp_add_present_cpu(int cpu) @@ -1083,20 +1077,12 @@ static int smp_add_present_cpu(int cpu) rc = sysfs_create_group(&s->kobj, &cpu_common_attr_group); if (rc) goto out_cpu; - if (cpu_online(cpu)) { - rc = sysfs_create_group(&s->kobj, &cpu_online_attr_group); - if (rc) - goto out_online; - } rc = topology_cpu_init(c); if (rc) goto out_topology; return 0; out_topology: - if (cpu_online(cpu)) - sysfs_remove_group(&s->kobj, &cpu_online_attr_group); -out_online: sysfs_remove_group(&s->kobj, &cpu_common_attr_group); out_cpu: #ifdef CONFIG_HOTPLUG_CPU @@ -1113,9 +1099,10 @@ int __ref smp_rescan_cpus(void) struct sclp_core_info *info; int nr; - info = smp_get_core_info(); + info = kzalloc(sizeof(*info), GFP_KERNEL); if (!info) return -ENOMEM; + smp_get_core_info(info, 0); get_online_cpus(); mutex_lock(&smp_cpu_state_mutex); nr = __smp_rescan_cpus(info, 1); @@ -1149,17 +1136,15 @@ static int __init s390_smp_init(void) if (rc) return rc; #endif - cpu_notifier_register_begin(); for_each_present_cpu(cpu) { rc = smp_add_present_cpu(cpu); if (rc) goto out; } - __hotcpu_notifier(smp_cpu_notify, 0); - + rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "s390/smp:online", + smp_cpu_online, smp_cpu_pre_down); out: - cpu_notifier_register_done(); return rc; } subsys_initcall(s390_smp_init); diff --git a/arch/s390/kernel/stacktrace.c b/arch/s390/kernel/stacktrace.c index 355db9db8210..0085b2d8ed7d 100644 --- a/arch/s390/kernel/stacktrace.c +++ b/arch/s390/kernel/stacktrace.c @@ -8,7 +8,7 @@ #include <linux/sched.h> #include <linux/stacktrace.h> #include <linux/kallsyms.h> -#include <linux/module.h> +#include <linux/export.h> static int __save_address(void *data, unsigned long address, int nosched) { diff --git a/arch/s390/kernel/swsusp.S b/arch/s390/kernel/swsusp.S index 2d6b6e81f812..6bebc935e9c2 100644 --- a/arch/s390/kernel/swsusp.S +++ b/arch/s390/kernel/swsusp.S @@ -194,9 +194,9 @@ pgm_check_entry: /* Suspend CPU not available -> panic */ larl %r15,init_thread_union - ahi %r15,1<<(PAGE_SHIFT+THREAD_ORDER) + ahi %r15,1<<(PAGE_SHIFT+THREAD_SIZE_ORDER) larl %r2,.Lpanic_string - larl %r3,_sclp_print_early + larl %r3,sclp_early_printk lghi %r1,0 sam31 sigp %r1,%r0,SIGP_SET_ARCHITECTURE @@ -273,7 +273,7 @@ restore_registers: .Ldisabled_wait_31: .long 0x000a0000,0x00000000 .Lpanic_string: - .asciz "Resume not possible because suspend CPU is no longer available" + .asciz "Resume not possible because suspend CPU is no longer available\n" .align 8 .Lrestart_diag308_psw: .long 0x00080000,0x80000000 diff --git a/arch/s390/kernel/sys_s390.c b/arch/s390/kernel/sys_s390.c index f145490cce54..b7af452978ca 100644 --- a/arch/s390/kernel/sys_s390.c +++ b/arch/s390/kernel/sys_s390.c @@ -27,7 +27,7 @@ #include <linux/personality.h> #include <linux/unistd.h> #include <linux/ipc.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include "entry.h" /* diff --git a/arch/s390/kernel/sysinfo.c b/arch/s390/kernel/sysinfo.c index bfda6aa40280..12b6b138e354 100644 --- a/arch/s390/kernel/sysinfo.c +++ b/arch/s390/kernel/sysinfo.c @@ -10,7 +10,7 @@ #include <linux/seq_file.h> #include <linux/init.h> #include <linux/delay.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/slab.h> #include <asm/ebcdic.h> #include <asm/sysinfo.h> @@ -56,6 +56,20 @@ int stsi(void *sysinfo, int fc, int sel1, int sel2) } EXPORT_SYMBOL(stsi); +static bool convert_ext_name(unsigned char encoding, char *name, size_t len) +{ + switch (encoding) { + case 1: /* EBCDIC */ + EBCASC(name, len); + break; + case 2: /* UTF-8 */ + break; + default: + return false; + } + return true; +} + static void stsi_1_1_1(struct seq_file *m, struct sysinfo_1_1_1 *info) { int i; @@ -207,24 +221,19 @@ static void stsi_2_2_2(struct seq_file *m, struct sysinfo_2_2_2 *info) seq_printf(m, "LPAR CPUs S-MTID: %d\n", info->mt_stid); seq_printf(m, "LPAR CPUs PS-MTID: %d\n", info->mt_psmtid); } + if (convert_ext_name(info->vsne, info->ext_name, sizeof(info->ext_name))) { + seq_printf(m, "LPAR Extended Name: %-.256s\n", info->ext_name); + seq_printf(m, "LPAR UUID: %pUb\n", &info->uuid); + } } static void print_ext_name(struct seq_file *m, int lvl, struct sysinfo_3_2_2 *info) { - if (info->vm[lvl].ext_name_encoding == 0) - return; - if (info->ext_names[lvl][0] == 0) - return; - switch (info->vm[lvl].ext_name_encoding) { - case 1: /* EBCDIC */ - EBCASC(info->ext_names[lvl], sizeof(info->ext_names[lvl])); - break; - case 2: /* UTF-8 */ - break; - default: + size_t len = sizeof(info->ext_names[lvl]); + + if (!convert_ext_name(info->vm[lvl].evmne, info->ext_names[lvl], len)) return; - } seq_printf(m, "VM%02d Extended Name: %-.256s\n", lvl, info->ext_names[lvl]); } diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c index 0bfcc492987e..de66abb479c9 100644 --- a/arch/s390/kernel/time.c +++ b/arch/s390/kernel/time.c @@ -16,7 +16,7 @@ #include <linux/kernel_stat.h> #include <linux/errno.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/sched.h> #include <linux/kernel.h> #include <linux/param.h> @@ -38,7 +38,7 @@ #include <linux/clockchips.h> #include <linux/gfp.h> #include <linux/kprobes.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include <asm/facility.h> #include <asm/delay.h> #include <asm/div64.h> @@ -59,19 +59,27 @@ ATOMIC_NOTIFIER_HEAD(s390_epoch_delta_notifier); EXPORT_SYMBOL(s390_epoch_delta_notifier); unsigned char ptff_function_mask[16]; -unsigned long lpar_offset; -unsigned long initial_leap_seconds; + +static unsigned long long lpar_offset; +static unsigned long long initial_leap_seconds; +static unsigned long long tod_steering_end; +static long long tod_steering_delta; /* * Get time offsets with PTFF */ -void __init ptff_init(void) +void __init time_early_init(void) { struct ptff_qto qto; struct ptff_qui qui; + /* Initialize TOD steering parameters */ + tod_steering_end = sched_clock_base_cc; + vdso_data->ts_end = tod_steering_end; + if (!test_facility(28)) return; + ptff(&ptff_function_mask, sizeof(ptff_function_mask), PTFF_QAF); /* get LPAR offset */ @@ -80,7 +88,7 @@ void __init ptff_init(void) /* get initial leap seconds */ if (ptff_query(PTFF_QUI) && ptff(&qui, sizeof(qui), PTFF_QUI) == 0) - initial_leap_seconds = (unsigned long) + initial_leap_seconds = (unsigned long long) ((long) qui.old_leap * 4096000000L); } @@ -102,7 +110,7 @@ unsigned long long monotonic_clock(void) } EXPORT_SYMBOL(monotonic_clock); -void tod_to_timeval(__u64 todval, struct timespec64 *xt) +static void tod_to_timeval(__u64 todval, struct timespec64 *xt) { unsigned long long sec; @@ -112,7 +120,6 @@ void tod_to_timeval(__u64 todval, struct timespec64 *xt) todval -= (sec * 1000000) << 12; xt->tv_nsec = ((todval * 1000) >> 12); } -EXPORT_SYMBOL(tod_to_timeval); void clock_comparator_work(void) { @@ -123,18 +130,6 @@ void clock_comparator_work(void) cd->event_handler(cd); } -/* - * Fixup the clock comparator. - */ -static void fixup_clock_comparator(unsigned long long delta) -{ - /* If nobody is waiting there's nothing to fix. */ - if (S390_lowcore.clock_comparator == -1ULL) - return; - S390_lowcore.clock_comparator += delta; - set_clock_comparator(S390_lowcore.clock_comparator); -} - static int s390_next_event(unsigned long delta, struct clock_event_device *evt) { @@ -213,9 +208,23 @@ void read_boot_clock64(struct timespec64 *ts) tod_to_timeval(clock - TOD_UNIX_EPOCH, ts); } -static cycle_t read_tod_clock(struct clocksource *cs) +static u64 read_tod_clock(struct clocksource *cs) { - return get_tod_clock(); + unsigned long long now, adj; + + preempt_disable(); /* protect from changes to steering parameters */ + now = get_tod_clock(); + adj = tod_steering_end - now; + if (unlikely((s64) adj >= 0)) + /* + * manually steer by 1 cycle every 2^16 cycles. This + * corresponds to shifting the tod delta by 15. 1s is + * therefore steered in ~9h. The adjust will decrease + * over time, until it finally reaches 0. + */ + now += (tod_steering_delta < 0) ? (adj >> 15) : -(adj >> 15); + preempt_enable(); + return now; } static struct clocksource clocksource_tod = { @@ -384,6 +393,55 @@ static inline int check_sync_clock(void) return rc; } +/* + * Apply clock delta to the global data structures. + * This is called once on the CPU that performed the clock sync. + */ +static void clock_sync_global(unsigned long long delta) +{ + unsigned long now, adj; + struct ptff_qto qto; + + /* Fixup the monotonic sched clock. */ + sched_clock_base_cc += delta; + /* Adjust TOD steering parameters. */ + vdso_data->tb_update_count++; + now = get_tod_clock(); + adj = tod_steering_end - now; + if (unlikely((s64) adj >= 0)) + /* Calculate how much of the old adjustment is left. */ + tod_steering_delta = (tod_steering_delta < 0) ? + -(adj >> 15) : (adj >> 15); + tod_steering_delta += delta; + if ((abs(tod_steering_delta) >> 48) != 0) + panic("TOD clock sync offset %lli is too large to drift\n", + tod_steering_delta); + tod_steering_end = now + (abs(tod_steering_delta) << 15); + vdso_data->ts_dir = (tod_steering_delta < 0) ? 0 : 1; + vdso_data->ts_end = tod_steering_end; + vdso_data->tb_update_count++; + /* Update LPAR offset. */ + if (ptff_query(PTFF_QTO) && ptff(&qto, sizeof(qto), PTFF_QTO) == 0) + lpar_offset = qto.tod_epoch_difference; + /* Call the TOD clock change notifier. */ + atomic_notifier_call_chain(&s390_epoch_delta_notifier, 0, &delta); +} + +/* + * Apply clock delta to the per-CPU data structures of this CPU. + * This is called for each online CPU after the call to clock_sync_global. + */ +static void clock_sync_local(unsigned long long delta) +{ + /* Add the delta to the clock comparator. */ + if (S390_lowcore.clock_comparator != -1ULL) { + S390_lowcore.clock_comparator += delta; + set_clock_comparator(S390_lowcore.clock_comparator); + } + /* Adjust the last_update_clock time-stamp. */ + S390_lowcore.last_update_clock += delta; +} + /* Single threaded workqueue used for stp sync events */ static struct workqueue_struct *time_sync_wq; @@ -397,31 +455,9 @@ static void __init time_init_wq(void) struct clock_sync_data { atomic_t cpus; int in_sync; - unsigned long long fixup_cc; + unsigned long long clock_delta; }; -static void clock_sync_cpu(struct clock_sync_data *sync) -{ - atomic_dec(&sync->cpus); - enable_sync_clock(); - while (sync->in_sync == 0) { - __udelay(1); - /* - * A different cpu changes *in_sync. Therefore use - * barrier() to force memory access. - */ - barrier(); - } - if (sync->in_sync != 1) - /* Didn't work. Clear per-cpu in sync bit again. */ - disable_sync_clock(NULL); - /* - * This round of TOD syncing is done. Set the clock comparator - * to the next tick and let the processor continue. - */ - fixup_clock_comparator(sync->fixup_cc); -} - /* * Server Time Protocol (STP) code. */ @@ -455,7 +491,7 @@ static void __init stp_reset(void) pr_warn("The real or virtual hardware system does not provide an STP interface\n"); free_page((unsigned long) stp_page); stp_page = NULL; - stp_online = 0; + stp_online = false; } } @@ -523,54 +559,46 @@ void stp_queue_work(void) static int stp_sync_clock(void *data) { - static int first; + struct clock_sync_data *sync = data; unsigned long long clock_delta; - struct clock_sync_data *stp_sync; - struct ptff_qto qto; + static int first; int rc; - stp_sync = data; - - if (xchg(&first, 1) == 1) { - /* Slave */ - clock_sync_cpu(stp_sync); - return 0; - } - - /* Wait until all other cpus entered the sync function. */ - while (atomic_read(&stp_sync->cpus) != 0) - cpu_relax(); - enable_sync_clock(); - - rc = 0; - if (stp_info.todoff[0] || stp_info.todoff[1] || - stp_info.todoff[2] || stp_info.todoff[3] || - stp_info.tmd != 2) { - rc = chsc_sstpc(stp_page, STP_OP_SYNC, 0, &clock_delta); - if (rc == 0) { - /* fixup the monotonic sched clock */ - sched_clock_base_cc += clock_delta; - if (ptff_query(PTFF_QTO) && - ptff(&qto, sizeof(qto), PTFF_QTO) == 0) - /* Update LPAR offset */ - lpar_offset = qto.tod_epoch_difference; - atomic_notifier_call_chain(&s390_epoch_delta_notifier, - 0, &clock_delta); - stp_sync->fixup_cc = clock_delta; - fixup_clock_comparator(clock_delta); - rc = chsc_sstpi(stp_page, &stp_info, - sizeof(struct stp_sstpi)); - if (rc == 0 && stp_info.tmd != 2) - rc = -EAGAIN; + if (xchg(&first, 1) == 0) { + /* Wait until all other cpus entered the sync function. */ + while (atomic_read(&sync->cpus) != 0) + cpu_relax(); + rc = 0; + if (stp_info.todoff[0] || stp_info.todoff[1] || + stp_info.todoff[2] || stp_info.todoff[3] || + stp_info.tmd != 2) { + rc = chsc_sstpc(stp_page, STP_OP_SYNC, 0, + &clock_delta); + if (rc == 0) { + sync->clock_delta = clock_delta; + clock_sync_global(clock_delta); + rc = chsc_sstpi(stp_page, &stp_info, + sizeof(struct stp_sstpi)); + if (rc == 0 && stp_info.tmd != 2) + rc = -EAGAIN; + } } + sync->in_sync = rc ? -EAGAIN : 1; + xchg(&first, 0); + } else { + /* Slave */ + atomic_dec(&sync->cpus); + /* Wait for in_sync to be set. */ + while (READ_ONCE(sync->in_sync) == 0) + __udelay(1); } - if (rc) { + if (sync->in_sync != 1) + /* Didn't work. Clear per-cpu in sync bit again. */ disable_sync_clock(NULL); - stp_sync->in_sync = -EAGAIN; - } else - stp_sync->in_sync = 1; - xchg(&first, 0); + /* Apply clock delta to per-CPU fields of this CPU. */ + clock_sync_local(sync->clock_delta); + return 0; } diff --git a/arch/s390/kernel/topology.c b/arch/s390/kernel/topology.c index e959c02e0cac..2cd5f4f1013c 100644 --- a/arch/s390/kernel/topology.c +++ b/arch/s390/kernel/topology.c @@ -7,6 +7,7 @@ #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt #include <linux/workqueue.h> +#include <linux/bootmem.h> #include <linux/cpuset.h> #include <linux/device.h> #include <linux/export.h> @@ -37,26 +38,27 @@ static void set_topology_timer(void); static void topology_work_fn(struct work_struct *work); static struct sysinfo_15_1_x *tl_info; -static bool topology_enabled = true; static DECLARE_WORK(topology_work, topology_work_fn); /* - * Socket/Book linked lists and per_cpu(cpu_topology) updates are + * Socket/Book linked lists and cpu_topology updates are * protected by "sched_domains_mutex". */ static struct mask_info socket_info; static struct mask_info book_info; static struct mask_info drawer_info; -DEFINE_PER_CPU(struct cpu_topology_s390, cpu_topology); -EXPORT_PER_CPU_SYMBOL_GPL(cpu_topology); +struct cpu_topology_s390 cpu_topology[NR_CPUS]; +EXPORT_SYMBOL_GPL(cpu_topology); + +cpumask_t cpus_with_topology; static cpumask_t cpu_group_map(struct mask_info *info, unsigned int cpu) { cpumask_t mask; cpumask_copy(&mask, cpumask_of(cpu)); - if (!topology_enabled || !MACHINE_HAS_TOPOLOGY) + if (!MACHINE_HAS_TOPOLOGY) return mask; for (; info; info = info->next) { if (cpumask_test_cpu(cpu, &info->mask)) @@ -71,7 +73,7 @@ static cpumask_t cpu_thread_map(unsigned int cpu) int i; cpumask_copy(&mask, cpumask_of(cpu)); - if (!topology_enabled || !MACHINE_HAS_TOPOLOGY) + if (!MACHINE_HAS_TOPOLOGY) return mask; cpu -= cpu % (smp_cpu_mtid + 1); for (i = 0; i <= smp_cpu_mtid; i++) @@ -97,7 +99,7 @@ static void add_cpus_to_mask(struct topology_core *tl_core, if (lcpu < 0) continue; for (i = 0; i <= smp_cpu_mtid; i++) { - topo = &per_cpu(cpu_topology, lcpu + i); + topo = &cpu_topology[lcpu + i]; topo->drawer_id = drawer->id; topo->book_id = book->id; topo->socket_id = socket->id; @@ -106,6 +108,7 @@ static void add_cpus_to_mask(struct topology_core *tl_core, cpumask_set_cpu(lcpu + i, &drawer->mask); cpumask_set_cpu(lcpu + i, &book->mask); cpumask_set_cpu(lcpu + i, &socket->mask); + cpumask_set_cpu(lcpu + i, &cpus_with_topology); smp_cpu_set_polarization(lcpu + i, tl_core->pp); } } @@ -220,7 +223,7 @@ static void update_cpu_masks(void) int cpu; for_each_possible_cpu(cpu) { - topo = &per_cpu(cpu_topology, cpu); + topo = &cpu_topology[cpu]; topo->thread_mask = cpu_thread_map(cpu); topo->core_mask = cpu_group_map(&socket_info, cpu); topo->book_mask = cpu_group_map(&book_info, cpu); @@ -231,6 +234,8 @@ static void update_cpu_masks(void) topo->socket_id = cpu; topo->book_id = cpu; topo->drawer_id = cpu; + if (cpu_present(cpu)) + cpumask_set_cpu(cpu, &cpus_with_topology); } } numa_update_cpu_topology(); @@ -241,12 +246,12 @@ void store_topology(struct sysinfo_15_1_x *info) stsi(info, 15, 1, min(topology_max_mnest, 4)); } -int arch_update_cpu_topology(void) +static int __arch_update_cpu_topology(void) { struct sysinfo_15_1_x *info = tl_info; - struct device *dev; - int cpu, rc = 0; + int rc = 0; + cpumask_clear(&cpus_with_topology); if (MACHINE_HAS_TOPOLOGY) { rc = 1; store_topology(info); @@ -255,6 +260,15 @@ int arch_update_cpu_topology(void) update_cpu_masks(); if (!MACHINE_HAS_TOPOLOGY) topology_update_polarization_simple(); + return rc; +} + +int arch_update_cpu_topology(void) +{ + struct device *dev; + int cpu, rc; + + rc = __arch_update_cpu_topology(); for_each_online_cpu(cpu) { dev = get_cpu_device(cpu); kobject_uevent(&dev->kobj, KOBJ_CHANGE); @@ -394,30 +408,24 @@ int topology_cpu_init(struct cpu *cpu) static const struct cpumask *cpu_thread_mask(int cpu) { - return &per_cpu(cpu_topology, cpu).thread_mask; + return &cpu_topology[cpu].thread_mask; } const struct cpumask *cpu_coregroup_mask(int cpu) { - return &per_cpu(cpu_topology, cpu).core_mask; + return &cpu_topology[cpu].core_mask; } static const struct cpumask *cpu_book_mask(int cpu) { - return &per_cpu(cpu_topology, cpu).book_mask; + return &cpu_topology[cpu].book_mask; } static const struct cpumask *cpu_drawer_mask(int cpu) { - return &per_cpu(cpu_topology, cpu).drawer_mask; -} - -static int __init early_parse_topology(char *p) -{ - return kstrtobool(p, &topology_enabled); + return &cpu_topology[cpu].drawer_mask; } -early_param("topology", early_parse_topology); static struct sched_domain_topology_level s390_topology[] = { { cpu_thread_mask, cpu_smt_flags, SD_INIT_NAME(SMT) }, @@ -438,32 +446,30 @@ static void __init alloc_masks(struct sysinfo_15_1_x *info, nr_masks *= info->mag[TOPOLOGY_NR_MAG - offset - 1 - i]; nr_masks = max(nr_masks, 1); for (i = 0; i < nr_masks; i++) { - mask->next = kzalloc(sizeof(*mask->next), GFP_KERNEL); + mask->next = memblock_virt_alloc(sizeof(*mask->next), 8); mask = mask->next; } } -static int __init s390_topology_init(void) +void __init topology_init_early(void) { struct sysinfo_15_1_x *info; - int i; + set_sched_topology(s390_topology); if (!MACHINE_HAS_TOPOLOGY) - return 0; - tl_info = (struct sysinfo_15_1_x *)__get_free_page(GFP_KERNEL); + goto out; + tl_info = memblock_virt_alloc(PAGE_SIZE, PAGE_SIZE); info = tl_info; store_topology(info); - pr_info("The CPU configuration topology of the machine is:"); - for (i = 0; i < TOPOLOGY_NR_MAG; i++) - printk(KERN_CONT " %d", info->mag[i]); - printk(KERN_CONT " / %d\n", info->mnest); + pr_info("The CPU configuration topology of the machine is: %d %d %d %d %d %d / %d\n", + info->mag[0], info->mag[1], info->mag[2], info->mag[3], + info->mag[4], info->mag[5], info->mnest); alloc_masks(info, &socket_info, 1); alloc_masks(info, &book_info, 2); alloc_masks(info, &drawer_info, 3); - set_sched_topology(s390_topology); - return 0; +out: + __arch_update_cpu_topology(); } -early_initcall(s390_topology_init); static int __init topology_init(void) { diff --git a/arch/s390/kernel/traps.c b/arch/s390/kernel/traps.c index d0539f76fd24..283ad7840335 100644 --- a/arch/s390/kernel/traps.c +++ b/arch/s390/kernel/traps.c @@ -19,7 +19,7 @@ #include <linux/sched.h> #include <linux/mm.h> #include <linux/slab.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include <asm/fpu/api.h> #include "entry.h" diff --git a/arch/s390/kernel/vdso.c b/arch/s390/kernel/vdso.c index 5904abf6b1ae..10516ae3b55e 100644 --- a/arch/s390/kernel/vdso.c +++ b/arch/s390/kernel/vdso.c @@ -9,7 +9,7 @@ * as published by the Free Software Foundation. */ -#include <linux/module.h> +#include <linux/init.h> #include <linux/errno.h> #include <linux/sched.h> #include <linux/kernel.h> diff --git a/arch/s390/kernel/vdso32/clock_gettime.S b/arch/s390/kernel/vdso32/clock_gettime.S index 5eec9afbb5b5..a5769b83d90e 100644 --- a/arch/s390/kernel/vdso32/clock_gettime.S +++ b/arch/s390/kernel/vdso32/clock_gettime.S @@ -99,8 +99,27 @@ __kernel_clock_gettime: tml %r4,0x0001 /* pending update ? loop */ jnz 11b stcke 0(%r15) /* Store TOD clock */ - lm %r0,%r1,1(%r15) - s %r0,__VDSO_XTIME_STAMP(%r5) /* TOD - cycle_last */ + lm %r0,%r1,__VDSO_TS_END(%r5) /* TOD steering end time */ + s %r0,1(%r15) /* no - ts_steering_end */ + sl %r1,5(%r15) + brc 3,22f + ahi %r0,-1 +22: ltr %r0,%r0 /* past end of steering? */ + jm 24f + srdl %r0,15 /* 1 per 2^16 */ + tm __VDSO_TS_DIR+3(%r5),0x01 /* steering direction? */ + jz 23f + lcr %r0,%r0 /* negative TOD offset */ + lcr %r1,%r1 + je 23f + ahi %r0,-1 +23: a %r0,1(%r15) /* add TOD timestamp */ + al %r1,5(%r15) + brc 12,25f + ahi %r0,1 + j 25f +24: lm %r0,%r1,1(%r15) /* load TOD timestamp */ +25: s %r0,__VDSO_XTIME_STAMP(%r5) /* TOD - cycle_last */ sl %r1,__VDSO_XTIME_STAMP+4(%r5) brc 3,12f ahi %r0,-1 diff --git a/arch/s390/kernel/vdso32/gettimeofday.S b/arch/s390/kernel/vdso32/gettimeofday.S index 719de6186b20..63b86dceb0bf 100644 --- a/arch/s390/kernel/vdso32/gettimeofday.S +++ b/arch/s390/kernel/vdso32/gettimeofday.S @@ -31,8 +31,27 @@ __kernel_gettimeofday: tml %r4,0x0001 /* pending update ? loop */ jnz 1b stcke 0(%r15) /* Store TOD clock */ - lm %r0,%r1,1(%r15) - s %r0,__VDSO_XTIME_STAMP(%r5) /* TOD - cycle_last */ + lm %r0,%r1,__VDSO_TS_END(%r5) /* TOD steering end time */ + s %r0,1(%r15) + sl %r1,5(%r15) + brc 3,14f + ahi %r0,-1 +14: ltr %r0,%r0 /* past end of steering? */ + jm 16f + srdl %r0,15 /* 1 per 2^16 */ + tm __VDSO_TS_DIR+3(%r5),0x01 /* steering direction? */ + jz 15f + lcr %r0,%r0 /* negative TOD offset */ + lcr %r1,%r1 + je 15f + ahi %r0,-1 +15: a %r0,1(%r15) /* add TOD timestamp */ + al %r1,5(%r15) + brc 12,17f + ahi %r0,1 + j 17f +16: lm %r0,%r1,1(%r15) /* load TOD timestamp */ +17: s %r0,__VDSO_XTIME_STAMP(%r5) /* TOD - cycle_last */ sl %r1,__VDSO_XTIME_STAMP+4(%r5) brc 3,3f ahi %r0,-1 diff --git a/arch/s390/kernel/vdso64/clock_gettime.S b/arch/s390/kernel/vdso64/clock_gettime.S index 61541fb93dc6..9c3b12626dba 100644 --- a/arch/s390/kernel/vdso64/clock_gettime.S +++ b/arch/s390/kernel/vdso64/clock_gettime.S @@ -83,8 +83,17 @@ __kernel_clock_gettime: tmll %r4,0x0001 /* pending update ? loop */ jnz 5b stcke 0(%r15) /* Store TOD clock */ - lgf %r2,__VDSO_TK_SHIFT(%r5) /* Timekeeper shift */ lg %r1,1(%r15) + lg %r0,__VDSO_TS_END(%r5) /* TOD steering end time */ + slgr %r0,%r1 /* now - ts_steering_end */ + ltgr %r0,%r0 /* past end of steering ? */ + jm 17f + srlg %r0,%r0,15 /* 1 per 2^16 */ + tm __VDSO_TS_DIR+3(%r5),0x01 /* steering direction? */ + jz 18f + lcgr %r0,%r0 /* negative TOD offset */ +18: algr %r1,%r0 /* add steering offset */ +17: lgf %r2,__VDSO_TK_SHIFT(%r5) /* Timekeeper shift */ sg %r1,__VDSO_XTIME_STAMP(%r5) /* TOD - cycle_last */ msgf %r1,__VDSO_TK_MULT(%r5) /* * tk->mult */ alg %r1,__VDSO_XTIME_NSEC(%r5) /* + tk->xtime_nsec */ diff --git a/arch/s390/kernel/vdso64/gettimeofday.S b/arch/s390/kernel/vdso64/gettimeofday.S index 6ce46707663c..b02e62f3bc12 100644 --- a/arch/s390/kernel/vdso64/gettimeofday.S +++ b/arch/s390/kernel/vdso64/gettimeofday.S @@ -31,7 +31,16 @@ __kernel_gettimeofday: jnz 0b stcke 0(%r15) /* Store TOD clock */ lg %r1,1(%r15) - sg %r1,__VDSO_XTIME_STAMP(%r5) /* TOD - cycle_last */ + lg %r0,__VDSO_TS_END(%r5) /* TOD steering end time */ + slgr %r0,%r1 /* now - ts_steering_end */ + ltgr %r0,%r0 /* past end of steering ? */ + jm 6f + srlg %r0,%r0,15 /* 1 per 2^16 */ + tm __VDSO_TS_DIR+3(%r5),0x01 /* steering direction? */ + jz 7f + lcgr %r0,%r0 /* negative TOD offset */ +7: algr %r1,%r0 /* add steering offset */ +6: sg %r1,__VDSO_XTIME_STAMP(%r5) /* TOD - cycle_last */ msgf %r1,__VDSO_TK_MULT(%r5) /* * tk->mult */ alg %r1,__VDSO_XTIME_NSEC(%r5) /* + tk->xtime_nsec */ lg %r0,__VDSO_XTIME_SEC(%r5) /* tk->xtime_sec */ diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S index 3667d20e997f..5ccf95396251 100644 --- a/arch/s390/kernel/vmlinux.lds.S +++ b/arch/s390/kernel/vmlinux.lds.S @@ -44,6 +44,7 @@ SECTIONS *(.gnu.warning) } :text = 0x0700 + . = ALIGN(PAGE_SIZE); _etext = .; /* End of text section */ NOTES :text :note @@ -79,7 +80,13 @@ SECTIONS . = ALIGN(PAGE_SIZE); /* Init code and data */ __init_begin = .; - INIT_TEXT_SECTION(PAGE_SIZE) + . = ALIGN(PAGE_SIZE); + .init.text : AT(ADDR(.init.text) - LOAD_OFFSET) { + VMLINUX_SYMBOL(_sinittext) = . ; + INIT_TEXT + . = ALIGN(PAGE_SIZE); + VMLINUX_SYMBOL(_einittext) = . ; + } /* * .exit.text is discarded at runtime, not link time, diff --git a/arch/s390/kernel/vtime.c b/arch/s390/kernel/vtime.c index 856e30d8463f..b4a3e9e06ef2 100644 --- a/arch/s390/kernel/vtime.c +++ b/arch/s390/kernel/vtime.c @@ -6,13 +6,13 @@ */ #include <linux/kernel_stat.h> +#include <linux/cputime.h> #include <linux/export.h> #include <linux/kernel.h> #include <linux/timex.h> #include <linux/types.h> #include <linux/time.h> -#include <asm/cputime.h> #include <asm/vtimer.h> #include <asm/vtime.h> #include <asm/cpu_mf.h> @@ -90,15 +90,41 @@ static void update_mt_scaling(void) __this_cpu_write(mt_scaling_jiffies, jiffies_64); } +static inline u64 update_tsk_timer(unsigned long *tsk_vtime, u64 new) +{ + u64 delta; + + delta = new - *tsk_vtime; + *tsk_vtime = new; + return delta; +} + + +static inline u64 scale_vtime(u64 vtime) +{ + u64 mult = __this_cpu_read(mt_scaling_mult); + u64 div = __this_cpu_read(mt_scaling_div); + + if (smp_cpu_mtid) + return vtime * mult / div; + return vtime; +} + +static void account_system_index_scaled(struct task_struct *p, + cputime_t cputime, cputime_t scaled, + enum cpu_usage_stat index) +{ + p->stimescaled += cputime_to_nsecs(scaled); + account_system_index_time(p, cputime_to_nsecs(cputime), index); +} + /* * Update process times based on virtual cpu times stored by entry.S * to the lowcore fields user_timer, system_timer & steal_clock. */ -static int do_account_vtime(struct task_struct *tsk, int hardirq_offset) +static int do_account_vtime(struct task_struct *tsk) { - struct thread_info *ti = task_thread_info(tsk); - u64 timer, clock, user, system, steal; - u64 user_scaled, system_scaled; + u64 timer, clock, user, guest, system, hardirq, softirq, steal; timer = S390_lowcore.last_update_timer; clock = S390_lowcore.last_update_clock; @@ -111,55 +137,76 @@ static int do_account_vtime(struct task_struct *tsk, int hardirq_offset) #endif : "=m" (S390_lowcore.last_update_timer), "=m" (S390_lowcore.last_update_clock)); - S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer; - S390_lowcore.steal_timer += S390_lowcore.last_update_clock - clock; + clock = S390_lowcore.last_update_clock - clock; + timer -= S390_lowcore.last_update_timer; + + if (hardirq_count()) + S390_lowcore.hardirq_timer += timer; + else + S390_lowcore.system_timer += timer; /* Update MT utilization calculation */ if (smp_cpu_mtid && time_after64(jiffies_64, this_cpu_read(mt_scaling_jiffies))) update_mt_scaling(); - user = S390_lowcore.user_timer - ti->user_timer; - S390_lowcore.steal_timer -= user; - ti->user_timer = S390_lowcore.user_timer; - - system = S390_lowcore.system_timer - ti->system_timer; - S390_lowcore.steal_timer -= system; - ti->system_timer = S390_lowcore.system_timer; - - user_scaled = user; - system_scaled = system; - /* Do MT utilization scaling */ - if (smp_cpu_mtid) { - u64 mult = __this_cpu_read(mt_scaling_mult); - u64 div = __this_cpu_read(mt_scaling_div); + /* Calculate cputime delta */ + user = update_tsk_timer(&tsk->thread.user_timer, + READ_ONCE(S390_lowcore.user_timer)); + guest = update_tsk_timer(&tsk->thread.guest_timer, + READ_ONCE(S390_lowcore.guest_timer)); + system = update_tsk_timer(&tsk->thread.system_timer, + READ_ONCE(S390_lowcore.system_timer)); + hardirq = update_tsk_timer(&tsk->thread.hardirq_timer, + READ_ONCE(S390_lowcore.hardirq_timer)); + softirq = update_tsk_timer(&tsk->thread.softirq_timer, + READ_ONCE(S390_lowcore.softirq_timer)); + S390_lowcore.steal_timer += + clock - user - guest - system - hardirq - softirq; + + /* Push account value */ + if (user) { + account_user_time(tsk, cputime_to_nsecs(user)); + tsk->utimescaled += cputime_to_nsecs(scale_vtime(user)); + } - user_scaled = (user_scaled * mult) / div; - system_scaled = (system_scaled * mult) / div; + if (guest) { + account_guest_time(tsk, cputime_to_nsecs(guest)); + tsk->utimescaled += cputime_to_nsecs(scale_vtime(guest)); } - account_user_time(tsk, user, user_scaled); - account_system_time(tsk, hardirq_offset, system, system_scaled); + + if (system) + account_system_index_scaled(tsk, system, scale_vtime(system), + CPUTIME_SYSTEM); + if (hardirq) + account_system_index_scaled(tsk, hardirq, scale_vtime(hardirq), + CPUTIME_IRQ); + if (softirq) + account_system_index_scaled(tsk, softirq, scale_vtime(softirq), + CPUTIME_SOFTIRQ); steal = S390_lowcore.steal_timer; if ((s64) steal > 0) { S390_lowcore.steal_timer = 0; - account_steal_time(steal); + account_steal_time(cputime_to_nsecs(steal)); } - return virt_timer_forward(user + system); + return virt_timer_forward(user + guest + system + hardirq + softirq); } void vtime_task_switch(struct task_struct *prev) { - struct thread_info *ti; - - do_account_vtime(prev, 0); - ti = task_thread_info(prev); - ti->user_timer = S390_lowcore.user_timer; - ti->system_timer = S390_lowcore.system_timer; - ti = task_thread_info(current); - S390_lowcore.user_timer = ti->user_timer; - S390_lowcore.system_timer = ti->system_timer; + do_account_vtime(prev); + prev->thread.user_timer = S390_lowcore.user_timer; + prev->thread.guest_timer = S390_lowcore.guest_timer; + prev->thread.system_timer = S390_lowcore.system_timer; + prev->thread.hardirq_timer = S390_lowcore.hardirq_timer; + prev->thread.softirq_timer = S390_lowcore.softirq_timer; + S390_lowcore.user_timer = current->thread.user_timer; + S390_lowcore.guest_timer = current->thread.guest_timer; + S390_lowcore.system_timer = current->thread.system_timer; + S390_lowcore.hardirq_timer = current->thread.hardirq_timer; + S390_lowcore.softirq_timer = current->thread.softirq_timer; } /* @@ -167,9 +214,9 @@ void vtime_task_switch(struct task_struct *prev) * accounting system time in order to correctly compute * the stolen time accounting. */ -void vtime_account_user(struct task_struct *tsk) +void vtime_flush(struct task_struct *tsk) { - if (do_account_vtime(tsk, HARDIRQ_OFFSET)) + if (do_account_vtime(tsk)) virt_timer_expire(); } @@ -179,32 +226,22 @@ void vtime_account_user(struct task_struct *tsk) */ void vtime_account_irq_enter(struct task_struct *tsk) { - struct thread_info *ti = task_thread_info(tsk); - u64 timer, system, system_scaled; + u64 timer; timer = S390_lowcore.last_update_timer; S390_lowcore.last_update_timer = get_vtimer(); - S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer; - - /* Update MT utilization calculation */ - if (smp_cpu_mtid && - time_after64(jiffies_64, this_cpu_read(mt_scaling_jiffies))) - update_mt_scaling(); - - system = S390_lowcore.system_timer - ti->system_timer; - S390_lowcore.steal_timer -= system; - ti->system_timer = S390_lowcore.system_timer; - system_scaled = system; - /* Do MT utilization scaling */ - if (smp_cpu_mtid) { - u64 mult = __this_cpu_read(mt_scaling_mult); - u64 div = __this_cpu_read(mt_scaling_div); - - system_scaled = (system_scaled * mult) / div; - } - account_system_time(tsk, 0, system, system_scaled); - - virt_timer_forward(system); + timer -= S390_lowcore.last_update_timer; + + if ((tsk->flags & PF_VCPU) && (irq_count() == 0)) + S390_lowcore.guest_timer += timer; + else if (hardirq_count()) + S390_lowcore.hardirq_timer += timer; + else if (in_serving_softirq()) + S390_lowcore.softirq_timer += timer; + else + S390_lowcore.system_timer += timer; + + virt_timer_forward(timer); } EXPORT_SYMBOL_GPL(vtime_account_irq_enter); |