From 1091670637be8bd34a39dd1ddcc0a10a7c88d4e2 Mon Sep 17 00:00:00 2001 From: Shile Zhang Date: Wed, 4 Dec 2019 08:46:31 +0800 Subject: scripts/sorttable: Rename 'sortextable' to 'sorttable' Use a more generic name for additional table sorting usecases, such as the upcoming ORC table sorting feature. This tool is not tied to exception table sorting anymore. No functional changes intended. [ mingo: Rewrote the changelog. ] Signed-off-by: Shile Zhang Acked-by: Peter Zijlstra (Intel) Cc: Josh Poimboeuf Cc: Masahiro Yamada Cc: Michal Marek Cc: linux-kbuild@vger.kernel.org Link: https://lkml.kernel.org/r/20191204004633.88660-6-shile.zhang@linux.alibaba.com Signed-off-by: Ingo Molnar --- scripts/sorttable.c | 373 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 373 insertions(+) create mode 100644 scripts/sorttable.c (limited to 'scripts/sorttable.c') diff --git a/scripts/sorttable.c b/scripts/sorttable.c new file mode 100644 index 000000000000..ff98b7db20c6 --- /dev/null +++ b/scripts/sorttable.c @@ -0,0 +1,373 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * sorttable.c: Sort the kernel's table + * + * Copyright 2011 - 2012 Cavium, Inc. + * + * Based on code taken from recortmcount.c which is: + * + * Copyright 2009 John F. Reiser . All rights reserved. + * + * Restructured to fit Linux format, as well as other updates: + * Copyright 2010 Steven Rostedt , Red Hat Inc. + */ + +/* + * Strategy: alter the vmlinux file in-place. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#ifndef EM_ARCOMPACT +#define EM_ARCOMPACT 93 +#endif + +#ifndef EM_XTENSA +#define EM_XTENSA 94 +#endif + +#ifndef EM_AARCH64 +#define EM_AARCH64 183 +#endif + +#ifndef EM_MICROBLAZE +#define EM_MICROBLAZE 189 +#endif + +#ifndef EM_ARCV2 +#define EM_ARCV2 195 +#endif + +static uint32_t (*r)(const uint32_t *); +static uint16_t (*r2)(const uint16_t *); +static uint64_t (*r8)(const uint64_t *); +static void (*w)(uint32_t, uint32_t *); +static void (*w2)(uint16_t, uint16_t *); +static void (*w8)(uint64_t, uint64_t *); +typedef void (*table_sort_t)(char *, int); + +/* + * Get the whole file as a programming convenience in order to avoid + * malloc+lseek+read+free of many pieces. If successful, then mmap + * avoids copying unused pieces; else just read the whole file. + * Open for both read and write. + */ +static void *mmap_file(char const *fname, size_t *size) +{ + int fd; + struct stat sb; + void *addr = NULL; + + fd = open(fname, O_RDWR); + if (fd < 0) { + perror(fname); + return NULL; + } + if (fstat(fd, &sb) < 0) { + perror(fname); + goto out; + } + if (!S_ISREG(sb.st_mode)) { + fprintf(stderr, "not a regular file: %s\n", fname); + goto out; + } + + addr = mmap(0, sb.st_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); + if (addr == MAP_FAILED) { + fprintf(stderr, "Could not mmap file: %s\n", fname); + goto out; + } + + *size = sb.st_size; + +out: + close(fd); + return addr; +} + +static uint32_t rbe(const uint32_t *x) +{ + return get_unaligned_be32(x); +} + +static uint16_t r2be(const uint16_t *x) +{ + return get_unaligned_be16(x); +} + +static uint64_t r8be(const uint64_t *x) +{ + return get_unaligned_be64(x); +} + +static uint32_t rle(const uint32_t *x) +{ + return get_unaligned_le32(x); +} + +static uint16_t r2le(const uint16_t *x) +{ + return get_unaligned_le16(x); +} + +static uint64_t r8le(const uint64_t *x) +{ + return get_unaligned_le64(x); +} + +static void wbe(uint32_t val, uint32_t *x) +{ + put_unaligned_be32(val, x); +} + +static void w2be(uint16_t val, uint16_t *x) +{ + put_unaligned_be16(val, x); +} + +static void w8be(uint64_t val, uint64_t *x) +{ + put_unaligned_be64(val, x); +} + +static void wle(uint32_t val, uint32_t *x) +{ + put_unaligned_le32(val, x); +} + +static void w2le(uint16_t val, uint16_t *x) +{ + put_unaligned_le16(val, x); +} + +static void w8le(uint64_t val, uint64_t *x) +{ + put_unaligned_le64(val, x); +} + +/* + * Move reserved section indices SHN_LORESERVE..SHN_HIRESERVE out of + * the way to -256..-1, to avoid conflicting with real section + * indices. + */ +#define SPECIAL(i) ((i) - (SHN_HIRESERVE + 1)) + +static inline int is_shndx_special(unsigned int i) +{ + return i != SHN_XINDEX && i >= SHN_LORESERVE && i <= SHN_HIRESERVE; +} + +/* Accessor for sym->st_shndx, hides ugliness of "64k sections" */ +static inline unsigned int get_secindex(unsigned int shndx, + unsigned int sym_offs, + const Elf32_Word *symtab_shndx_start) +{ + if (is_shndx_special(shndx)) + return SPECIAL(shndx); + if (shndx != SHN_XINDEX) + return shndx; + return r(&symtab_shndx_start[sym_offs]); +} + +/* 32 bit and 64 bit are very similar */ +#include "sorttable.h" +#define SORTTABLE_64 +#include "sorttable.h" + +static int compare_relative_table(const void *a, const void *b) +{ + int32_t av = (int32_t)r(a); + int32_t bv = (int32_t)r(b); + + if (av < bv) + return -1; + if (av > bv) + return 1; + return 0; +} + +static void sort_relative_table(char *extab_image, int image_size) +{ + int i = 0; + + /* + * Do the same thing the runtime sort does, first normalize to + * being relative to the start of the section. + */ + while (i < image_size) { + uint32_t *loc = (uint32_t *)(extab_image + i); + w(r(loc) + i, loc); + i += 4; + } + + qsort(extab_image, image_size / 8, 8, compare_relative_table); + + /* Now denormalize. */ + i = 0; + while (i < image_size) { + uint32_t *loc = (uint32_t *)(extab_image + i); + w(r(loc) - i, loc); + i += 4; + } +} + +static void x86_sort_relative_table(char *extab_image, int image_size) +{ + int i = 0; + + while (i < image_size) { + uint32_t *loc = (uint32_t *)(extab_image + i); + + w(r(loc) + i, loc); + w(r(loc + 1) + i + 4, loc + 1); + w(r(loc + 2) + i + 8, loc + 2); + + i += sizeof(uint32_t) * 3; + } + + qsort(extab_image, image_size / 12, 12, compare_relative_table); + + i = 0; + while (i < image_size) { + uint32_t *loc = (uint32_t *)(extab_image + i); + + w(r(loc) - i, loc); + w(r(loc + 1) - (i + 4), loc + 1); + w(r(loc + 2) - (i + 8), loc + 2); + + i += sizeof(uint32_t) * 3; + } +} + +static int do_file(char const *const fname, void *addr) +{ + int rc = -1; + Elf32_Ehdr *ehdr = addr; + table_sort_t custom_sort = NULL; + + switch (ehdr->e_ident[EI_DATA]) { + case ELFDATA2LSB: + r = rle; + r2 = r2le; + r8 = r8le; + w = wle; + w2 = w2le; + w8 = w8le; + break; + case ELFDATA2MSB: + r = rbe; + r2 = r2be; + r8 = r8be; + w = wbe; + w2 = w2be; + w8 = w8be; + break; + default: + fprintf(stderr, "unrecognized ELF data encoding %d: %s\n", + ehdr->e_ident[EI_DATA], fname); + return -1; + } + + if (memcmp(ELFMAG, ehdr->e_ident, SELFMAG) != 0 || + (r2(&ehdr->e_type) != ET_EXEC && r2(&ehdr->e_type) != ET_DYN) || + ehdr->e_ident[EI_VERSION] != EV_CURRENT) { + fprintf(stderr, "unrecognized ET_EXEC/ET_DYN file %s\n", fname); + return -1; + } + + switch (r2(&ehdr->e_machine)) { + case EM_386: + case EM_X86_64: + custom_sort = x86_sort_relative_table; + break; + case EM_S390: + case EM_AARCH64: + case EM_PARISC: + case EM_PPC: + case EM_PPC64: + custom_sort = sort_relative_table; + break; + case EM_ARCOMPACT: + case EM_ARCV2: + case EM_ARM: + case EM_MICROBLAZE: + case EM_MIPS: + case EM_XTENSA: + break; + default: + fprintf(stderr, "unrecognized e_machine %d %s\n", + r2(&ehdr->e_machine), fname); + return -1; + } + + switch (ehdr->e_ident[EI_CLASS]) { + case ELFCLASS32: + if (r2(&ehdr->e_ehsize) != sizeof(Elf32_Ehdr) || + r2(&ehdr->e_shentsize) != sizeof(Elf32_Shdr)) { + fprintf(stderr, + "unrecognized ET_EXEC/ET_DYN file: %s\n", fname); + break; + } + rc = do_sort_32(ehdr, fname, custom_sort); + break; + case ELFCLASS64: + { + Elf64_Ehdr *const ghdr = (Elf64_Ehdr *)ehdr; + if (r2(&ghdr->e_ehsize) != sizeof(Elf64_Ehdr) || + r2(&ghdr->e_shentsize) != sizeof(Elf64_Shdr)) { + fprintf(stderr, + "unrecognized ET_EXEC/ET_DYN file: %s\n", + fname); + break; + } + rc = do_sort_64(ghdr, fname, custom_sort); + } + break; + default: + fprintf(stderr, "unrecognized ELF class %d %s\n", + ehdr->e_ident[EI_CLASS], fname); + break; + } + + return rc; +} + +int main(int argc, char *argv[]) +{ + int i, n_error = 0; /* gcc-4.3.0 false positive complaint */ + size_t size = 0; + void *addr = NULL; + + if (argc < 2) { + fprintf(stderr, "usage: sorttable vmlinux...\n"); + return 0; + } + + /* Process each file in turn, allowing deep failure. */ + for (i = 1; i < argc; i++) { + addr = mmap_file(argv[i], &size); + if (!addr) { + ++n_error; + continue; + } + + if (do_file(argv[i], addr)) + ++n_error; + + munmap(addr, size); + } + + return !!n_error; +} -- cgit From 57fa1899428538e314a7e0d52a5b617af082389a Mon Sep 17 00:00:00 2001 From: Shile Zhang Date: Wed, 4 Dec 2019 08:46:32 +0800 Subject: scripts/sorttable: Implement build-time ORC unwind table sorting The ORC unwinder has two tables: .orc_unwind_ip and .orc_unwind, which need to be sorted for binary search. Previously this sorting was done during bootup. Sort them at build time to speed up booting. Add the ORC tables sorting in a parallel build process to speed up the build. [ mingo: Rewrote the changelog and fixed some comments. ] Suggested-by: Andy Lutomirski Suggested-by: Peter Zijlstra Reported-by: kbuild test robot Signed-off-by: Shile Zhang Acked-by: Peter Zijlstra (Intel) Cc: Josh Poimboeuf Cc: Masahiro Yamada Cc: Michal Marek Cc: linux-kbuild@vger.kernel.org Link: https://lkml.kernel.org/r/20191204004633.88660-7-shile.zhang@linux.alibaba.com Signed-off-by: Ingo Molnar --- scripts/Makefile | 9 +++ scripts/sorttable.c | 6 +- scripts/sorttable.h | 180 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 189 insertions(+), 6 deletions(-) (limited to 'scripts/sorttable.c') diff --git a/scripts/Makefile b/scripts/Makefile index 7491241e3a0d..b0e962611d50 100644 --- a/scripts/Makefile +++ b/scripts/Makefile @@ -24,6 +24,15 @@ HOSTCFLAGS_asn1_compiler.o = -I$(srctree)/include HOSTLDLIBS_sign-file = -lcrypto HOSTLDLIBS_extract-cert = -lcrypto +ifdef CONFIG_UNWINDER_ORC +ifeq ($(ARCH),x86_64) +ARCH := x86 +endif +HOSTCFLAGS_sorttable.o += -I$(srctree)/tools/arch/x86/include +HOSTCFLAGS_sorttable.o += -DUNWINDER_ORC_ENABLED +HOSTLDLIBS_sorttable = -lpthread +endif + always := $(hostprogs-y) $(hostprogs-m) # The following hostprogs-y programs are only build on demand diff --git a/scripts/sorttable.c b/scripts/sorttable.c index ff98b7db20c6..ec6b5e81eba1 100644 --- a/scripts/sorttable.c +++ b/scripts/sorttable.c @@ -2,6 +2,10 @@ /* * sorttable.c: Sort the kernel's table * + * Added ORC unwind tables sort support and other updates: + * Copyright (C) 1999-2019 Alibaba Group Holding Limited. by: + * Shile Zhang + * * Copyright 2011 - 2012 Cavium, Inc. * * Based on code taken from recortmcount.c which is: @@ -9,7 +13,7 @@ * Copyright 2009 John F. Reiser . All rights reserved. * * Restructured to fit Linux format, as well as other updates: - * Copyright 2010 Steven Rostedt , Red Hat Inc. + * Copyright 2010 Steven Rostedt , Red Hat Inc. */ /* diff --git a/scripts/sorttable.h b/scripts/sorttable.h index 82589ff90e25..a2baa2fefb13 100644 --- a/scripts/sorttable.h +++ b/scripts/sorttable.h @@ -2,8 +2,15 @@ /* * sorttable.h * + * Added ORC unwind tables sort support and other updates: + * Copyright (C) 1999-2019 Alibaba Group Holding Limited. by: + * Shile Zhang + * * Copyright 2011 - 2012 Cavium, Inc. * + * Some of code was taken out of arch/x86/kernel/unwind_orc.c, written by: + * Copyright (C) 2017 Josh Poimboeuf + * * Some of this code was taken out of recordmcount.h written by: * * Copyright 2009 John F. Reiser . All rights reserved. @@ -75,6 +82,104 @@ # define _w w #endif +#if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) +/* ORC unwinder only support X86_64 */ +#include +#include +#include + +#define ERRSTR_MAXSZ 256 + +char g_err[ERRSTR_MAXSZ]; +int *g_orc_ip_table; +struct orc_entry *g_orc_table; + +pthread_t orc_sort_thread; + +static inline unsigned long orc_ip(const int *ip) +{ + return (unsigned long)ip + *ip; +} + +static int orc_sort_cmp(const void *_a, const void *_b) +{ + struct orc_entry *orc_a; + const int *a = g_orc_ip_table + *(int *)_a; + const int *b = g_orc_ip_table + *(int *)_b; + unsigned long a_val = orc_ip(a); + unsigned long b_val = orc_ip(b); + + if (a_val > b_val) + return 1; + if (a_val < b_val) + return -1; + + /* + * The "weak" section terminator entries need to always be on the left + * to ensure the lookup code skips them in favor of real entries. + * These terminator entries exist to handle any gaps created by + * whitelisted .o files which didn't get objtool generation. + */ + orc_a = g_orc_table + (a - g_orc_ip_table); + return orc_a->sp_reg == ORC_REG_UNDEFINED && !orc_a->end ? -1 : 1; +} + +static void *sort_orctable(void *arg) +{ + int i; + int *idxs = NULL; + int *tmp_orc_ip_table = NULL; + struct orc_entry *tmp_orc_table = NULL; + unsigned int *orc_ip_size = (unsigned int *)arg; + unsigned int num_entries = *orc_ip_size / sizeof(int); + unsigned int orc_size = num_entries * sizeof(struct orc_entry); + + idxs = (int *)malloc(*orc_ip_size); + if (!idxs) { + snprintf(g_err, ERRSTR_MAXSZ, "malloc idxs: %s", + strerror(errno)); + pthread_exit(g_err); + } + + tmp_orc_ip_table = (int *)malloc(*orc_ip_size); + if (!tmp_orc_ip_table) { + snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_ip_table: %s", + strerror(errno)); + pthread_exit(g_err); + } + + tmp_orc_table = (struct orc_entry *)malloc(orc_size); + if (!tmp_orc_table) { + snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_table: %s", + strerror(errno)); + pthread_exit(g_err); + } + + /* initialize indices array, convert ip_table to absolute address */ + for (i = 0; i < num_entries; i++) { + idxs[i] = i; + tmp_orc_ip_table[i] = g_orc_ip_table[i] + i * sizeof(int); + } + memcpy(tmp_orc_table, g_orc_table, orc_size); + + qsort(idxs, num_entries, sizeof(int), orc_sort_cmp); + + for (i = 0; i < num_entries; i++) { + if (idxs[i] == i) + continue; + + /* convert back to relative address */ + g_orc_ip_table[i] = tmp_orc_ip_table[idxs[i]] - i * sizeof(int); + g_orc_table[i] = tmp_orc_table[idxs[i]]; + } + + free(idxs); + free(tmp_orc_ip_table); + free(tmp_orc_table); + pthread_exit(NULL); +} +#endif + static int compare_extable(const void *a, const void *b) { Elf_Addr av = _r(a); @@ -91,6 +196,7 @@ static int do_sort(Elf_Ehdr *ehdr, char const *const fname, table_sort_t custom_sort) { + int rc = -1; Elf_Shdr *s, *shdr = (Elf_Shdr *)((char *)ehdr + _r(&ehdr->e_shoff)); Elf_Shdr *strtab_sec = NULL; Elf_Shdr *symtab_sec = NULL; @@ -111,6 +217,11 @@ static int do_sort(Elf_Ehdr *ehdr, int idx; unsigned int shnum; unsigned int shstrndx; +#if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) + unsigned int orc_ip_size = 0; + unsigned int orc_size = 0; + unsigned int orc_num_entries = 0; +#endif shstrndx = r2(&ehdr->e_shstrndx); if (shstrndx == SHN_XINDEX) @@ -141,21 +252,61 @@ static int do_sort(Elf_Ehdr *ehdr, if (r(&s->sh_type) == SHT_SYMTAB_SHNDX) symtab_shndx = (Elf32_Word *)((const char *)ehdr + _r(&s->sh_offset)); + +#if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) + /* locate the ORC unwind tables */ + if (!strcmp(secstrings + idx, ".orc_unwind_ip")) { + orc_ip_size = s->sh_size; + g_orc_ip_table = (int *)((void *)ehdr + + s->sh_offset); + } + if (!strcmp(secstrings + idx, ".orc_unwind")) { + orc_size = s->sh_size; + g_orc_table = (struct orc_entry *)((void *)ehdr + + s->sh_offset); + } +#endif + } /* for loop */ + +#if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) + if (!g_orc_ip_table || !g_orc_table) { + fprintf(stderr, + "incomplete ORC unwind tables in file: %s\n", fname); + goto out; + } + + orc_num_entries = orc_ip_size / sizeof(int); + if (orc_ip_size % sizeof(int) != 0 || + orc_size % sizeof(struct orc_entry) != 0 || + orc_num_entries != orc_size / sizeof(struct orc_entry)) { + fprintf(stderr, + "inconsistent ORC unwind table entries in file: %s\n", + fname); + goto out; } + /* create thread to sort ORC unwind tables concurrently */ + if (pthread_create(&orc_sort_thread, NULL, + sort_orctable, &orc_ip_size)) { + fprintf(stderr, + "pthread_create orc_sort_thread failed '%s': %s\n", + strerror(errno), fname); + goto out; + } +#endif if (!extab_sec) { fprintf(stderr, "no __ex_table in file: %s\n", fname); - return -1; + goto out; } if (!symtab_sec) { fprintf(stderr, "no .symtab in file: %s\n", fname); - return -1; + goto out; } if (!strtab_sec) { fprintf(stderr, "no .strtab in file: %s\n", fname); - return -1; + goto out; } extab_image = (void *)ehdr + _r(&extab_sec->sh_offset); @@ -192,7 +343,7 @@ static int do_sort(Elf_Ehdr *ehdr, fprintf(stderr, "no main_extable_sort_needed symbol in file: %s\n", fname); - return -1; + goto out; } sort_needed_sec = &shdr[get_secindex(r2(&sym->st_shndx), @@ -205,6 +356,25 @@ static int do_sort(Elf_Ehdr *ehdr, /* extable has been sorted, clear the flag */ w(0, sort_needed_loc); + rc = 0; - return 0; +out: +#if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) + if (orc_sort_thread) { + void *retval = NULL; + /* wait for ORC tables sort done */ + rc = pthread_join(orc_sort_thread, &retval); + if (rc) + fprintf(stderr, + "pthread_join failed '%s': %s\n", + strerror(errno), fname); + else if (retval) { + rc = -1; + fprintf(stderr, + "failed to sort ORC tables '%s': %s\n", + (char *)retval, fname); + } + } +#endif + return rc; } -- cgit