diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/Kconfig | 12 | ||||
| -rw-r--r-- | lib/Kconfig.debug | 88 | ||||
| -rw-r--r-- | lib/Kconfig.ubsan | 10 | ||||
| -rw-r--r-- | lib/Makefile | 16 | ||||
| -rw-r--r-- | lib/argv_split.c | 4 | ||||
| -rw-r--r-- | lib/bch.c | 38 | ||||
| -rw-r--r-- | lib/bitmap-str.c | 510 | ||||
| -rw-r--r-- | lib/bitmap.c | 680 | ||||
| -rw-r--r-- | lib/checksum_kunit.c | 54 | ||||
| -rw-r--r-- | lib/closure.c | 211 | ||||
| -rw-r--r-- | lib/cpumask.c | 21 | ||||
| -rw-r--r-- | lib/crypto/Makefile | 2 | ||||
| -rw-r--r-- | lib/crypto/mpi/Makefile (renamed from lib/mpi/Makefile) | 0 | ||||
| -rw-r--r-- | lib/crypto/mpi/ec.c (renamed from lib/mpi/ec.c) | 0 | ||||
| -rw-r--r-- | lib/crypto/mpi/generic_mpih-add1.c (renamed from lib/mpi/generic_mpih-add1.c) | 0 | ||||
| -rw-r--r-- | lib/crypto/mpi/generic_mpih-lshift.c (renamed from lib/mpi/generic_mpih-lshift.c) | 0 | ||||
| -rw-r--r-- | lib/crypto/mpi/generic_mpih-mul1.c (renamed from lib/mpi/generic_mpih-mul1.c) | 0 | ||||
| -rw-r--r-- | lib/crypto/mpi/generic_mpih-mul2.c (renamed from lib/mpi/generic_mpih-mul2.c) | 0 | ||||
| -rw-r--r-- | lib/crypto/mpi/generic_mpih-mul3.c (renamed from lib/mpi/generic_mpih-mul3.c) | 0 | ||||
| -rw-r--r-- | lib/crypto/mpi/generic_mpih-rshift.c (renamed from lib/mpi/generic_mpih-rshift.c) | 0 | ||||
| -rw-r--r-- | lib/crypto/mpi/generic_mpih-sub1.c (renamed from lib/mpi/generic_mpih-sub1.c) | 0 | ||||
| -rw-r--r-- | lib/crypto/mpi/longlong.h (renamed from lib/mpi/longlong.h) | 0 | ||||
| -rw-r--r-- | lib/crypto/mpi/mpi-add.c (renamed from lib/mpi/mpi-add.c) | 0 | ||||
| -rw-r--r-- | lib/crypto/mpi/mpi-bit.c (renamed from lib/mpi/mpi-bit.c) | 0 | ||||
| -rw-r--r-- | lib/crypto/mpi/mpi-cmp.c (renamed from lib/mpi/mpi-cmp.c) | 8 | ||||
| -rw-r--r-- | lib/crypto/mpi/mpi-div.c (renamed from lib/mpi/mpi-div.c) | 0 | ||||
| -rw-r--r-- | lib/crypto/mpi/mpi-inline.h (renamed from lib/mpi/mpi-inline.h) | 0 | ||||
| -rw-r--r-- | lib/crypto/mpi/mpi-internal.h (renamed from lib/mpi/mpi-internal.h) | 0 | ||||
| -rw-r--r-- | lib/crypto/mpi/mpi-inv.c (renamed from lib/mpi/mpi-inv.c) | 0 | ||||
| -rw-r--r-- | lib/crypto/mpi/mpi-mod.c (renamed from lib/mpi/mpi-mod.c) | 0 | ||||
| -rw-r--r-- | lib/crypto/mpi/mpi-mul.c (renamed from lib/mpi/mpi-mul.c) | 0 | ||||
| -rw-r--r-- | lib/crypto/mpi/mpi-pow.c (renamed from lib/mpi/mpi-pow.c) | 0 | ||||
| -rw-r--r-- | lib/crypto/mpi/mpi-sub-ui.c (renamed from lib/mpi/mpi-sub-ui.c) | 0 | ||||
| -rw-r--r-- | lib/crypto/mpi/mpicoder.c (renamed from lib/mpi/mpicoder.c) | 0 | ||||
| -rw-r--r-- | lib/crypto/mpi/mpih-cmp.c (renamed from lib/mpi/mpih-cmp.c) | 0 | ||||
| -rw-r--r-- | lib/crypto/mpi/mpih-div.c (renamed from lib/mpi/mpih-div.c) | 0 | ||||
| -rw-r--r-- | lib/crypto/mpi/mpih-mul.c (renamed from lib/mpi/mpih-mul.c) | 0 | ||||
| -rw-r--r-- | lib/crypto/mpi/mpiutil.c (renamed from lib/mpi/mpiutil.c) | 0 | ||||
| -rw-r--r-- | lib/debugobjects.c | 2 | ||||
| -rw-r--r-- | lib/decompress_unxz.c | 3 | ||||
| -rw-r--r-- | lib/dynamic_debug.c | 8 | ||||
| -rw-r--r-- | lib/errname.c | 7 | ||||
| -rw-r--r-- | lib/error-inject.c | 2 | ||||
| -rw-r--r-- | lib/fonts/Kconfig | 2 | ||||
| -rw-r--r-- | lib/fw_table.c | 188 | ||||
| -rw-r--r-- | lib/genalloc.c | 4 | ||||
| -rw-r--r-- | lib/generic-radix-tree.c | 76 | ||||
| -rw-r--r-- | lib/group_cpus.c | 22 | ||||
| -rw-r--r-- | lib/idr.c | 4 | ||||
| -rw-r--r-- | lib/iov_iter.c | 503 | ||||
| -rw-r--r-- | lib/kobject.c | 62 | ||||
| -rw-r--r-- | lib/kobject_uevent.c | 8 | ||||
| -rw-r--r-- | lib/kstrtox.c | 2 | ||||
| -rw-r--r-- | lib/kunit/Kconfig | 2 | ||||
| -rw-r--r-- | lib/kunit/Makefile | 3 | ||||
| -rw-r--r-- | lib/kunit/assert.c | 14 | ||||
| -rw-r--r-- | lib/kunit/attributes.c | 414 | ||||
| -rw-r--r-- | lib/kunit/debugfs.c | 36 | ||||
| -rw-r--r-- | lib/kunit/executor.c | 262 | ||||
| -rw-r--r-- | lib/kunit/executor_test.c | 185 | ||||
| -rw-r--r-- | lib/kunit/kunit-example-test.c | 14 | ||||
| -rw-r--r-- | lib/kunit/kunit-test.c | 56 | ||||
| -rw-r--r-- | lib/kunit/string-stream-test.c | 525 | ||||
| -rw-r--r-- | lib/kunit/string-stream.c | 100 | ||||
| -rw-r--r-- | lib/kunit/string-stream.h | 16 | ||||
| -rw-r--r-- | lib/kunit/test.c | 165 | ||||
| -rw-r--r-- | lib/kunit_iov_iter.c | 777 | ||||
| -rw-r--r-- | lib/list_debug.c | 16 | ||||
| -rw-r--r-- | lib/llist.c | 28 | ||||
| -rw-r--r-- | lib/locking-selftest.c | 135 | ||||
| -rw-r--r-- | lib/logic_pio.c | 3 | ||||
| -rw-r--r-- | lib/lwq.c | 158 | ||||
| -rw-r--r-- | lib/maple_tree.c | 1319 | ||||
| -rw-r--r-- | lib/math/Makefile | 2 | ||||
| -rw-r--r-- | lib/math/int_log.c | 133 | ||||
| -rw-r--r-- | lib/memcpy_kunit.c | 8 | ||||
| -rw-r--r-- | lib/nlattr.c | 28 | ||||
| -rw-r--r-- | lib/nmi_backtrace.c | 6 | ||||
| -rw-r--r-- | lib/notifier-error-inject.c | 3 | ||||
| -rw-r--r-- | lib/objpool.c | 297 | ||||
| -rw-r--r-- | lib/percpu_counter.c | 141 | ||||
| -rw-r--r-- | lib/raid6/Makefile | 5 | ||||
| -rw-r--r-- | lib/raid6/algos.c | 18 | ||||
| -rw-r--r-- | lib/raid6/int.uc | 9 | ||||
| -rw-r--r-- | lib/raid6/loongarch.h | 38 | ||||
| -rw-r--r-- | lib/raid6/loongarch_simd.c | 422 | ||||
| -rw-r--r-- | lib/raid6/mktables.c | 2 | ||||
| -rw-r--r-- | lib/raid6/recov.c | 1 | ||||
| -rw-r--r-- | lib/raid6/recov_loongarch_simd.c | 513 | ||||
| -rw-r--r-- | lib/raid6/test/.gitignore | 3 | ||||
| -rw-r--r-- | lib/raid6/test/Makefile | 62 | ||||
| -rw-r--r-- | lib/rcuref.c | 2 | ||||
| -rw-r--r-- | lib/scatterlist.c | 4 | ||||
| -rw-r--r-- | lib/seq_buf.c | 28 | ||||
| -rw-r--r-- | lib/string_helpers.c | 25 | ||||
| -rw-r--r-- | lib/test_bitmap.c | 24 | ||||
| -rw-r--r-- | lib/test_bpf.c | 395 | ||||
| -rw-r--r-- | lib/test_hmm.c | 10 | ||||
| -rw-r--r-- | lib/test_ida.c | 40 | ||||
| -rw-r--r-- | lib/test_maple_tree.c | 263 | ||||
| -rw-r--r-- | lib/test_meminit.c | 2 | ||||
| -rw-r--r-- | lib/test_objpool.c | 690 | ||||
| -rw-r--r-- | lib/test_printf.c | 3 | ||||
| -rw-r--r-- | lib/test_scanf.c | 2 | ||||
| -rw-r--r-- | lib/ts_bm.c | 43 | ||||
| -rw-r--r-- | lib/ucs2_string.c | 52 | ||||
| -rw-r--r-- | lib/vsprintf.c | 37 | ||||
| -rw-r--r-- | lib/xarray.c | 8 | ||||
| -rw-r--r-- | lib/xz/Kconfig | 5 | ||||
| -rw-r--r-- | lib/zstd/common/fse_decompress.c | 2 |
110 files changed, 7753 insertions, 2348 deletions
diff --git a/lib/Kconfig b/lib/Kconfig index 5c2da561c516..3ea1c830efab 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -415,6 +415,7 @@ config REED_SOLOMON_DEC16 # config BCH tristate + select BITREVERSE config BCH_CONST_PARAMS bool @@ -505,6 +506,9 @@ config ASSOCIATIVE_ARRAY for more information. +config CLOSURES + bool + config HAS_IOMEM bool depends on !NO_IOMEM @@ -728,6 +732,11 @@ config PARMAN config OBJAGG tristate "objagg" if COMPILE_TEST +config LWQ_TEST + bool "Boot-time test for lwq queuing" + help + Run boot-time test of light-weight queuing. + endmenu config GENERIC_IOREMAP @@ -763,3 +772,6 @@ config ASN1_ENCODER config POLYNOMIAL tristate + +config FIRMWARE_TABLE + bool diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index d6798513a8c2..4405f81248fb 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -355,6 +355,11 @@ endchoice # "Compressed Debug information" config DEBUG_INFO_SPLIT bool "Produce split debuginfo in .dwo files" depends on $(cc-option,-gsplit-dwarf) + # RISC-V linker relaxation + -gsplit-dwarf has issues with LLVM and GCC + # prior to 12.x: + # https://github.com/llvm/llvm-project/issues/56642 + # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=99090 + depends on !RISCV || GCC_VERSION >= 120000 help Generate debug info into separate .dwo files. This significantly reduces the build directory size for builds with DEBUG_INFO, @@ -502,7 +507,7 @@ config SECTION_MISMATCH_WARN_ONLY config DEBUG_FORCE_FUNCTION_ALIGN_64B bool "Force all function address 64B aligned" - depends on EXPERT && (X86_64 || ARM64 || PPC32 || PPC64 || ARC || S390) + depends on EXPERT && (X86_64 || ARM64 || PPC32 || PPC64 || ARC || RISCV || S390) select FUNCTION_ALIGNMENT_64B help There are cases that a commit from one domain changes the function @@ -754,7 +759,7 @@ config SHRINKER_DEBUG config DEBUG_STACK_USAGE bool "Stack utilization instrumentation" - depends on DEBUG_KERNEL && !IA64 + depends on DEBUG_KERNEL help Enables the display of the minimum amount of free stack which each task has ever had available in the sysrq-T and sysrq-P debug output. @@ -1673,10 +1678,15 @@ menu "Debug kernel data structures" config DEBUG_LIST bool "Debug linked list manipulation" - depends on DEBUG_KERNEL || BUG_ON_DATA_CORRUPTION + depends on DEBUG_KERNEL + select LIST_HARDENED help - Enable this to turn on extended checks in the linked-list - walking routines. + Enable this to turn on extended checks in the linked-list walking + routines. + + This option trades better quality error reports for performance, and + is more suitable for kernel debugging. If you care about performance, + you should only enable CONFIG_LIST_HARDENED instead. If unsure, say N. @@ -1710,15 +1720,14 @@ config DEBUG_NOTIFIERS This is a relatively cheap check but if you care about maximum performance, say N. -config BUG_ON_DATA_CORRUPTION - bool "Trigger a BUG when data corruption is detected" - select DEBUG_LIST +config DEBUG_CLOSURES + bool "Debug closures (bcache async widgits)" + depends on CLOSURES + select DEBUG_FS help - Select this option if the kernel should BUG when it encounters - data corruption in kernel memory structures when they get checked - for validity. - - If unsure, say N. + Keeps all active closures in a linked list and provides a debugfs + interface to list them, which makes it possible to see asynchronous + operations that get stuck. config DEBUG_MAPLE_TREE bool "Debug maple trees" @@ -1730,21 +1739,6 @@ config DEBUG_MAPLE_TREE endmenu -config DEBUG_CREDENTIALS - bool "Debug credential management" - depends on DEBUG_KERNEL - help - Enable this to turn on some debug checking for credential - management. The additional code keeps track of the number of - pointers from task_structs to any given cred struct, and checks to - see that this number never exceeds the usage count of the cred - struct. - - Furthermore, if SELinux is enabled, this also checks that the - security pointer in the cred struct is never seen to be invalid. - - If unsure, say N. - source "kernel/rcu/Kconfig.debug" config DEBUG_WQ_FORCE_RR_CPU @@ -2237,6 +2231,17 @@ config TEST_DIV64 If unsure, say N. +config TEST_IOV_ITER + tristate "Test iov_iter operation" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + Enable this to turn on testing of the operation of the I/O iterator + (iov_iter). This test is executed only once during system boot (so + affects only boot time), or at module load time. + + If unsure, say N. + config KPROBES_SANITY_TEST tristate "Kprobes sanity tests" if !KUNIT_ALL_TESTS depends on DEBUG_KERNEL @@ -2701,6 +2706,9 @@ config MEMCPY_SLOW_KUNIT_TEST and bit ranges. These can be very slow, so they are split out as a separate config, in case they need to be disabled. + Note this config option will be replaced by the use of KUnit test + attributes. + config IS_SIGNED_TYPE_KUNIT_TEST tristate "Test is_signed_type() macro" if !KUNIT_ALL_TESTS depends on KUNIT @@ -2931,6 +2939,17 @@ config TEST_CLOCKSOURCE_WATCHDOG If unsure, say N. +config TEST_OBJPOOL + tristate "Test module for correctness and stress of objpool" + default n + depends on m && DEBUG_KERNEL + help + This builds the "test_objpool" module that should be used for + correctness verification and concurrent testings of objects + allocation and reclamation. + + If unsure, say N. + endif # RUNTIME_TESTING_MENU config ARCH_USE_MEMTEST @@ -3010,6 +3029,19 @@ config RUST_BUILD_ASSERT_ALLOW If unsure, say N. +config RUST_KERNEL_DOCTESTS + bool "Doctests for the `kernel` crate" if !KUNIT_ALL_TESTS + depends on RUST && KUNIT=y + default KUNIT_ALL_TESTS + help + This builds the documentation tests of the `kernel` crate + as KUnit tests. + + For more information on KUnit and unit tests in general, + please refer to the KUnit documentation in Documentation/dev-tools/kunit/. + + If unsure, say N. + endmenu # "Rust" endmenu # Kernel hacking diff --git a/lib/Kconfig.ubsan b/lib/Kconfig.ubsan index efae7e011956..59e21bfec188 100644 --- a/lib/Kconfig.ubsan +++ b/lib/Kconfig.ubsan @@ -13,7 +13,7 @@ menuconfig UBSAN if UBSAN config UBSAN_TRAP - bool "On Sanitizer warnings, abort the running kernel code" + bool "Abort on Sanitizer warnings (smaller kernel but less verbose)" depends on !COMPILE_TEST help Building kernels with Sanitizer features enabled tends to grow @@ -26,6 +26,14 @@ config UBSAN_TRAP the system. For some system builders this is an acceptable trade-off. + Also note that selecting Y will cause your kernel to Oops + with an "illegal instruction" error with no further details + when a UBSAN violation occurs. (Except on arm64, which will + report which Sanitizer failed.) This may make it hard to + determine whether an Oops was caused by UBSAN or to figure + out the details of a UBSAN violation. It makes the kernel log + output less useful for bug reports. + config CC_HAS_UBSAN_BOUNDS_STRICT def_bool $(cc-option,-fsanitize=bounds-strict) help diff --git a/lib/Makefile b/lib/Makefile index 1ffae65bb7ee..6b09731d8e61 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -34,7 +34,7 @@ lib-y := ctype.o string.o vsprintf.o cmdline.o \ is_single_threaded.o plist.o decompress.o kobject_uevent.o \ earlycpio.o seq_buf.o siphash.o dec_and_lock.o \ nmi_backtrace.o win_minmax.o memcat_p.o \ - buildid.o + buildid.o objpool.o lib-$(CONFIG_PRINTK) += dump_stack.o lib-$(CONFIG_SMP) += cpumask.o @@ -45,10 +45,10 @@ obj-y += lockref.o obj-y += bcd.o sort.o parser.o debug_locks.o random32.o \ bust_spinlocks.o kasprintf.o bitmap.o scatterlist.o \ list_sort.o uuid.o iov_iter.o clz_ctz.o \ - bsearch.o find_bit.o llist.o memweight.o kfifo.o \ + bsearch.o find_bit.o llist.o lwq.o memweight.o kfifo.o \ percpu-refcount.o rhashtable.o base64.o \ once.o refcount.o rcuref.o usercopy.o errseq.o bucket_locks.o \ - generic-radix-tree.o + generic-radix-tree.o bitmap-str.o obj-$(CONFIG_STRING_SELFTEST) += test_string.o obj-y += string_helpers.o obj-$(CONFIG_TEST_STRING_HELPERS) += test-string_helpers.o @@ -64,6 +64,7 @@ obj-$(CONFIG_TEST_BITOPS) += test_bitops.o CFLAGS_test_bitops.o += -Werror obj-$(CONFIG_CPUMASK_KUNIT_TEST) += cpumask_kunit.o obj-$(CONFIG_TEST_SYSCTL) += test_sysctl.o +obj-$(CONFIG_TEST_IOV_ITER) += kunit_iov_iter.o obj-$(CONFIG_HASH_KUNIT_TEST) += test_hash.o obj-$(CONFIG_TEST_IDA) += test_ida.o obj-$(CONFIG_TEST_UBSAN) += test_ubsan.o @@ -106,6 +107,8 @@ obj-$(CONFIG_KPROBES_SANITY_TEST) += test_kprobes.o obj-$(CONFIG_TEST_REF_TRACKER) += test_ref_tracker.o CFLAGS_test_fprobe.o += $(CC_FLAGS_FTRACE) obj-$(CONFIG_FPROBE_SANITY_TEST) += test_fprobe.o +obj-$(CONFIG_TEST_OBJPOOL) += test_objpool.o + # # CFLAGS for compiling floating point code inside the kernel. x86/Makefile turns # off the generation of FPU/SSE* instructions for kernel proper but FPU_FLAGS @@ -167,7 +170,7 @@ obj-$(CONFIG_BTREE) += btree.o obj-$(CONFIG_INTERVAL_TREE) += interval_tree.o obj-$(CONFIG_ASSOCIATIVE_ARRAY) += assoc_array.o obj-$(CONFIG_DEBUG_PREEMPT) += smp_processor_id.o -obj-$(CONFIG_DEBUG_LIST) += list_debug.o +obj-$(CONFIG_LIST_HARDENED) += list_debug.o obj-$(CONFIG_DEBUG_OBJECTS) += debugobjects.o obj-$(CONFIG_BITREVERSE) += bitrev.o @@ -254,12 +257,13 @@ obj-$(CONFIG_ATOMIC64_SELFTEST) += atomic64_test.o obj-$(CONFIG_CPU_RMAP) += cpu_rmap.o +obj-$(CONFIG_CLOSURES) += closure.o + obj-$(CONFIG_DQL) += dynamic_queue_limits.o obj-$(CONFIG_GLOB) += glob.o obj-$(CONFIG_GLOB_SELFTEST) += globtest.o -obj-$(CONFIG_MPILIB) += mpi/ obj-$(CONFIG_DIMLIB) += dim/ obj-$(CONFIG_SIGNATURE) += digsig.o @@ -405,6 +409,8 @@ obj-$(CONFIG_SIPHASH_KUNIT_TEST) += siphash_kunit.o obj-$(CONFIG_GENERIC_LIB_DEVMEM_IS_ALLOWED) += devmem_is_allowed.o +obj-$(CONFIG_FIRMWARE_TABLE) += fw_table.o + # FORTIFY_SOURCE compile-time behavior tests TEST_FORTIFY_SRCS = $(wildcard $(srctree)/$(src)/test_fortify/*-*.c) TEST_FORTIFY_LOGS = $(patsubst $(srctree)/$(src)/%.c, %.log, $(TEST_FORTIFY_SRCS)) diff --git a/lib/argv_split.c b/lib/argv_split.c index 1a19a0a93dc1..e28db8e3b58c 100644 --- a/lib/argv_split.c +++ b/lib/argv_split.c @@ -28,7 +28,7 @@ static int count_argc(const char *str) /** * argv_free - free an argv - * @argv - the argument vector to be freed + * @argv: the argument vector to be freed * * Frees an argv and the strings it points to. */ @@ -46,7 +46,7 @@ EXPORT_SYMBOL(argv_free); * @str: the string to be split * @argcp: returned argument count * - * Returns an array of pointers to strings which are split out from + * Returns: an array of pointers to strings which are split out from * @str. This is performed by strictly splitting on white-space; no * quote processing is performed. Multiple whitespace characters are * considered to be a single argument separator. The returned array diff --git a/lib/bch.c b/lib/bch.c index c8095f30f254..5f71fd76eca8 100644 --- a/lib/bch.c +++ b/lib/bch.c @@ -71,6 +71,7 @@ #include <linux/module.h> #include <linux/slab.h> #include <linux/bitops.h> +#include <linux/bitrev.h> #include <asm/byteorder.h> #include <linux/bch.h> @@ -114,47 +115,12 @@ struct gf_poly_deg1 { unsigned int c[2]; }; -static u8 swap_bits_table[] = { - 0x00, 0x80, 0x40, 0xc0, 0x20, 0xa0, 0x60, 0xe0, - 0x10, 0x90, 0x50, 0xd0, 0x30, 0xb0, 0x70, 0xf0, - 0x08, 0x88, 0x48, 0xc8, 0x28, 0xa8, 0x68, 0xe8, - 0x18, 0x98, 0x58, 0xd8, 0x38, 0xb8, 0x78, 0xf8, - 0x04, 0x84, 0x44, 0xc4, 0x24, 0xa4, 0x64, 0xe4, - 0x14, 0x94, 0x54, 0xd4, 0x34, 0xb4, 0x74, 0xf4, - 0x0c, 0x8c, 0x4c, 0xcc, 0x2c, 0xac, 0x6c, 0xec, - 0x1c, 0x9c, 0x5c, 0xdc, 0x3c, 0xbc, 0x7c, 0xfc, - 0x02, 0x82, 0x42, 0xc2, 0x22, 0xa2, 0x62, 0xe2, - 0x12, 0x92, 0x52, 0xd2, 0x32, 0xb2, 0x72, 0xf2, - 0x0a, 0x8a, 0x4a, 0xca, 0x2a, 0xaa, 0x6a, 0xea, - 0x1a, 0x9a, 0x5a, 0xda, 0x3a, 0xba, 0x7a, 0xfa, - 0x06, 0x86, 0x46, 0xc6, 0x26, 0xa6, 0x66, 0xe6, - 0x16, 0x96, 0x56, 0xd6, 0x36, 0xb6, 0x76, 0xf6, - 0x0e, 0x8e, 0x4e, 0xce, 0x2e, 0xae, 0x6e, 0xee, - 0x1e, 0x9e, 0x5e, 0xde, 0x3e, 0xbe, 0x7e, 0xfe, - 0x01, 0x81, 0x41, 0xc1, 0x21, 0xa1, 0x61, 0xe1, - 0x11, 0x91, 0x51, 0xd1, 0x31, 0xb1, 0x71, 0xf1, - 0x09, 0x89, 0x49, 0xc9, 0x29, 0xa9, 0x69, 0xe9, - 0x19, 0x99, 0x59, 0xd9, 0x39, 0xb9, 0x79, 0xf9, - 0x05, 0x85, 0x45, 0xc5, 0x25, 0xa5, 0x65, 0xe5, - 0x15, 0x95, 0x55, 0xd5, 0x35, 0xb5, 0x75, 0xf5, - 0x0d, 0x8d, 0x4d, 0xcd, 0x2d, 0xad, 0x6d, 0xed, - 0x1d, 0x9d, 0x5d, 0xdd, 0x3d, 0xbd, 0x7d, 0xfd, - 0x03, 0x83, 0x43, 0xc3, 0x23, 0xa3, 0x63, 0xe3, - 0x13, 0x93, 0x53, 0xd3, 0x33, 0xb3, 0x73, 0xf3, - 0x0b, 0x8b, 0x4b, 0xcb, 0x2b, 0xab, 0x6b, 0xeb, - 0x1b, 0x9b, 0x5b, 0xdb, 0x3b, 0xbb, 0x7b, 0xfb, - 0x07, 0x87, 0x47, 0xc7, 0x27, 0xa7, 0x67, 0xe7, - 0x17, 0x97, 0x57, 0xd7, 0x37, 0xb7, 0x77, 0xf7, - 0x0f, 0x8f, 0x4f, 0xcf, 0x2f, 0xaf, 0x6f, 0xef, - 0x1f, 0x9f, 0x5f, 0xdf, 0x3f, 0xbf, 0x7f, 0xff, -}; - static u8 swap_bits(struct bch_control *bch, u8 in) { if (!bch->swap_bits) return in; - return swap_bits_table[in]; + return bitrev8(in); } /* diff --git a/lib/bitmap-str.c b/lib/bitmap-str.c new file mode 100644 index 000000000000..be745209507a --- /dev/null +++ b/lib/bitmap-str.c @@ -0,0 +1,510 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include <linux/bitmap.h> +#include <linux/ctype.h> +#include <linux/errno.h> +#include <linux/err.h> +#include <linux/export.h> +#include <linux/hex.h> +#include <linux/kernel.h> +#include <linux/mm.h> +#include <linux/string.h> + +#include "kstrtox.h" + +/** + * bitmap_parse_user - convert an ASCII hex string in a user buffer into a bitmap + * + * @ubuf: pointer to user buffer containing string. + * @ulen: buffer size in bytes. If string is smaller than this + * then it must be terminated with a \0. + * @maskp: pointer to bitmap array that will contain result. + * @nmaskbits: size of bitmap, in bits. + */ +int bitmap_parse_user(const char __user *ubuf, + unsigned int ulen, unsigned long *maskp, + int nmaskbits) +{ + char *buf; + int ret; + + buf = memdup_user_nul(ubuf, ulen); + if (IS_ERR(buf)) + return PTR_ERR(buf); + + ret = bitmap_parse(buf, UINT_MAX, maskp, nmaskbits); + + kfree(buf); + return ret; +} +EXPORT_SYMBOL(bitmap_parse_user); + +/** + * bitmap_print_to_pagebuf - convert bitmap to list or hex format ASCII string + * @list: indicates whether the bitmap must be list + * @buf: page aligned buffer into which string is placed + * @maskp: pointer to bitmap to convert + * @nmaskbits: size of bitmap, in bits + * + * Output format is a comma-separated list of decimal numbers and + * ranges if list is specified or hex digits grouped into comma-separated + * sets of 8 digits/set. Returns the number of characters written to buf. + * + * It is assumed that @buf is a pointer into a PAGE_SIZE, page-aligned + * area and that sufficient storage remains at @buf to accommodate the + * bitmap_print_to_pagebuf() output. Returns the number of characters + * actually printed to @buf, excluding terminating '\0'. + */ +int bitmap_print_to_pagebuf(bool list, char *buf, const unsigned long *maskp, + int nmaskbits) +{ + ptrdiff_t len = PAGE_SIZE - offset_in_page(buf); + + return list ? scnprintf(buf, len, "%*pbl\n", nmaskbits, maskp) : + scnprintf(buf, len, "%*pb\n", nmaskbits, maskp); +} +EXPORT_SYMBOL(bitmap_print_to_pagebuf); + +/** + * bitmap_print_to_buf - convert bitmap to list or hex format ASCII string + * @list: indicates whether the bitmap must be list + * true: print in decimal list format + * false: print in hexadecimal bitmask format + * @buf: buffer into which string is placed + * @maskp: pointer to bitmap to convert + * @nmaskbits: size of bitmap, in bits + * @off: in the string from which we are copying, We copy to @buf + * @count: the maximum number of bytes to print + */ +static int bitmap_print_to_buf(bool list, char *buf, const unsigned long *maskp, + int nmaskbits, loff_t off, size_t count) +{ + const char *fmt = list ? "%*pbl\n" : "%*pb\n"; + ssize_t size; + void *data; + + data = kasprintf(GFP_KERNEL, fmt, nmaskbits, maskp); + if (!data) + return -ENOMEM; + + size = memory_read_from_buffer(buf, count, &off, data, strlen(data) + 1); + kfree(data); + + return size; +} + +/** + * bitmap_print_bitmask_to_buf - convert bitmap to hex bitmask format ASCII string + * @buf: buffer into which string is placed + * @maskp: pointer to bitmap to convert + * @nmaskbits: size of bitmap, in bits + * @off: in the string from which we are copying, We copy to @buf + * @count: the maximum number of bytes to print + * + * The bitmap_print_to_pagebuf() is used indirectly via its cpumap wrapper + * cpumap_print_to_pagebuf() or directly by drivers to export hexadecimal + * bitmask and decimal list to userspace by sysfs ABI. + * Drivers might be using a normal attribute for this kind of ABIs. A + * normal attribute typically has show entry as below:: + * + * static ssize_t example_attribute_show(struct device *dev, + * struct device_attribute *attr, char *buf) + * { + * ... + * return bitmap_print_to_pagebuf(true, buf, &mask, nr_trig_max); + * } + * + * show entry of attribute has no offset and count parameters and this + * means the file is limited to one page only. + * bitmap_print_to_pagebuf() API works terribly well for this kind of + * normal attribute with buf parameter and without offset, count:: + * + * bitmap_print_to_pagebuf(bool list, char *buf, const unsigned long *maskp, + * int nmaskbits) + * { + * } + * + * The problem is once we have a large bitmap, we have a chance to get a + * bitmask or list more than one page. Especially for list, it could be + * as complex as 0,3,5,7,9,... We have no simple way to know it exact size. + * It turns out bin_attribute is a way to break this limit. bin_attribute + * has show entry as below:: + * + * static ssize_t + * example_bin_attribute_show(struct file *filp, struct kobject *kobj, + * struct bin_attribute *attr, char *buf, + * loff_t offset, size_t count) + * { + * ... + * } + * + * With the new offset and count parameters, this makes sysfs ABI be able + * to support file size more than one page. For example, offset could be + * >= 4096. + * bitmap_print_bitmask_to_buf(), bitmap_print_list_to_buf() wit their + * cpumap wrapper cpumap_print_bitmask_to_buf(), cpumap_print_list_to_buf() + * make those drivers be able to support large bitmask and list after they + * move to use bin_attribute. In result, we have to pass the corresponding + * parameters such as off, count from bin_attribute show entry to this API. + * + * The role of cpumap_print_bitmask_to_buf() and cpumap_print_list_to_buf() + * is similar with cpumap_print_to_pagebuf(), the difference is that + * bitmap_print_to_pagebuf() mainly serves sysfs attribute with the assumption + * the destination buffer is exactly one page and won't be more than one page. + * cpumap_print_bitmask_to_buf() and cpumap_print_list_to_buf(), on the other + * hand, mainly serves bin_attribute which doesn't work with exact one page, + * and it can break the size limit of converted decimal list and hexadecimal + * bitmask. + * + * WARNING! + * + * This function is not a replacement for sprintf() or bitmap_print_to_pagebuf(). + * It is intended to workaround sysfs limitations discussed above and should be + * used carefully in general case for the following reasons: + * + * - Time complexity is O(nbits^2/count), comparing to O(nbits) for snprintf(). + * - Memory complexity is O(nbits), comparing to O(1) for snprintf(). + * - @off and @count are NOT offset and number of bits to print. + * - If printing part of bitmap as list, the resulting string is not a correct + * list representation of bitmap. Particularly, some bits within or out of + * related interval may be erroneously set or unset. The format of the string + * may be broken, so bitmap_parselist-like parser may fail parsing it. + * - If printing the whole bitmap as list by parts, user must ensure the order + * of calls of the function such that the offset is incremented linearly. + * - If printing the whole bitmap as list by parts, user must keep bitmap + * unchanged between the very first and very last call. Otherwise concatenated + * result may be incorrect, and format may be broken. + * + * Returns the number of characters actually printed to @buf + */ +int bitmap_print_bitmask_to_buf(char *buf, const unsigned long *maskp, + int nmaskbits, loff_t off, size_t count) +{ + return bitmap_print_to_buf(false, buf, maskp, nmaskbits, off, count); +} +EXPORT_SYMBOL(bitmap_print_bitmask_to_buf); + +/** + * bitmap_print_list_to_buf - convert bitmap to decimal list format ASCII string + * @buf: buffer into which string is placed + * @maskp: pointer to bitmap to convert + * @nmaskbits: size of bitmap, in bits + * @off: in the string from which we are copying, We copy to @buf + * @count: the maximum number of bytes to print + * + * Everything is same with the above bitmap_print_bitmask_to_buf() except + * the print format. + */ +int bitmap_print_list_to_buf(char *buf, const unsigned long *maskp, + int nmaskbits, loff_t off, size_t count) +{ + return bitmap_print_to_buf(true, buf, maskp, nmaskbits, off, count); +} +EXPORT_SYMBOL(bitmap_print_list_to_buf); + +/* + * Region 9-38:4/10 describes the following bitmap structure: + * 0 9 12 18 38 N + * .........****......****......****.................. + * ^ ^ ^ ^ ^ + * start off group_len end nbits + */ +struct region { + unsigned int start; + unsigned int off; + unsigned int group_len; + unsigned int end; + unsigned int nbits; +}; + +static void bitmap_set_region(const struct region *r, unsigned long *bitmap) +{ + unsigned int start; + + for (start = r->start; start <= r->end; start += r->group_len) + bitmap_set(bitmap, start, min(r->end - start + 1, r->off)); +} + +static int bitmap_check_region(const struct region *r) +{ + if (r->start > r->end || r->group_len == 0 || r->off > r->group_len) + return -EINVAL; + + if (r->end >= r->nbits) + return -ERANGE; + + return 0; +} + +static const char *bitmap_getnum(const char *str, unsigned int *num, + unsigned int lastbit) +{ + unsigned long long n; + unsigned int len; + + if (str[0] == 'N') { + *num = lastbit; + return str + 1; + } + + len = _parse_integer(str, 10, &n); + if (!len) + return ERR_PTR(-EINVAL); + if (len & KSTRTOX_OVERFLOW || n != (unsigned int)n) + return ERR_PTR(-EOVERFLOW); + + *num = n; + return str + len; +} + +static inline bool end_of_str(char c) +{ + return c == '\0' || c == '\n'; +} + +static inline bool __end_of_region(char c) +{ + return isspace(c) || c == ','; +} + +static inline bool end_of_region(char c) +{ + return __end_of_region(c) || end_of_str(c); +} + +/* + * The format allows commas and whitespaces at the beginning + * of the region. + */ +static const char *bitmap_find_region(const char *str) +{ + while (__end_of_region(*str)) + str++; + + return end_of_str(*str) ? NULL : str; +} + +static const char *bitmap_find_region_reverse(const char *start, const char *end) +{ + while (start <= end && __end_of_region(*end)) + end--; + + return end; +} + +static const char *bitmap_parse_region(const char *str, struct region *r) +{ + unsigned int lastbit = r->nbits - 1; + + if (!strncasecmp(str, "all", 3)) { + r->start = 0; + r->end = lastbit; + str += 3; + + goto check_pattern; + } + + str = bitmap_getnum(str, &r->start, lastbit); + if (IS_ERR(str)) + return str; + + if (end_of_region(*str)) + goto no_end; + + if (*str != '-') + return ERR_PTR(-EINVAL); + + str = bitmap_getnum(str + 1, &r->end, lastbit); + if (IS_ERR(str)) + return str; + +check_pattern: + if (end_of_region(*str)) + goto no_pattern; + + if (*str != ':') + return ERR_PTR(-EINVAL); + + str = bitmap_getnum(str + 1, &r->off, lastbit); + if (IS_ERR(str)) + return str; + + if (*str != '/') + return ERR_PTR(-EINVAL); + + return bitmap_getnum(str + 1, &r->group_len, lastbit); + +no_end: + r->end = r->start; +no_pattern: + r->off = r->end + 1; + r->group_len = r->end + 1; + + return end_of_str(*str) ? NULL : str; +} + +/** + * bitmap_parselist - convert list format ASCII string to bitmap + * @buf: read user string from this buffer; must be terminated + * with a \0 or \n. + * @maskp: write resulting mask here + * @nmaskbits: number of bits in mask to be written + * + * Input format is a comma-separated list of decimal numbers and + * ranges. Consecutively set bits are shown as two hyphen-separated + * decimal numbers, the smallest and largest bit numbers set in + * the range. + * Optionally each range can be postfixed to denote that only parts of it + * should be set. The range will divided to groups of specific size. + * From each group will be used only defined amount of bits. + * Syntax: range:used_size/group_size + * Example: 0-1023:2/256 ==> 0,1,256,257,512,513,768,769 + * The value 'N' can be used as a dynamically substituted token for the + * maximum allowed value; i.e (nmaskbits - 1). Keep in mind that it is + * dynamic, so if system changes cause the bitmap width to change, such + * as more cores in a CPU list, then any ranges using N will also change. + * + * Returns: 0 on success, -errno on invalid input strings. Error values: + * + * - ``-EINVAL``: wrong region format + * - ``-EINVAL``: invalid character in string + * - ``-ERANGE``: bit number specified too large for mask + * - ``-EOVERFLOW``: integer overflow in the input parameters + */ +int bitmap_parselist(const char *buf, unsigned long *maskp, int nmaskbits) +{ + struct region r; + long ret; + + r.nbits = nmaskbits; + bitmap_zero(maskp, r.nbits); + + while (buf) { + buf = bitmap_find_region(buf); + if (buf == NULL) + return 0; + + buf = bitmap_parse_region(buf, &r); + if (IS_ERR(buf)) + return PTR_ERR(buf); + + ret = bitmap_check_region(&r); + if (ret) + return ret; + + bitmap_set_region(&r, maskp); + } + + return 0; +} +EXPORT_SYMBOL(bitmap_parselist); + + +/** + * bitmap_parselist_user() - convert user buffer's list format ASCII + * string to bitmap + * + * @ubuf: pointer to user buffer containing string. + * @ulen: buffer size in bytes. If string is smaller than this + * then it must be terminated with a \0. + * @maskp: pointer to bitmap array that will contain result. + * @nmaskbits: size of bitmap, in bits. + * + * Wrapper for bitmap_parselist(), providing it with user buffer. + */ +int bitmap_parselist_user(const char __user *ubuf, + unsigned int ulen, unsigned long *maskp, + int nmaskbits) +{ + char *buf; + int ret; + + buf = memdup_user_nul(ubuf, ulen); + if (IS_ERR(buf)) + return PTR_ERR(buf); + + ret = bitmap_parselist(buf, maskp, nmaskbits); + + kfree(buf); + return ret; +} +EXPORT_SYMBOL(bitmap_parselist_user); + +static const char *bitmap_get_x32_reverse(const char *start, + const char *end, u32 *num) +{ + u32 ret = 0; + int c, i; + + for (i = 0; i < 32; i += 4) { + c = hex_to_bin(*end--); + if (c < 0) + return ERR_PTR(-EINVAL); + + ret |= c << i; + + if (start > end || __end_of_region(*end)) + goto out; + } + + if (hex_to_bin(*end--) >= 0) + return ERR_PTR(-EOVERFLOW); +out: + *num = ret; + return end; +} + +/** + * bitmap_parse - convert an ASCII hex string into a bitmap. + * @start: pointer to buffer containing string. + * @buflen: buffer size in bytes. If string is smaller than this + * then it must be terminated with a \0 or \n. In that case, + * UINT_MAX may be provided instead of string length. + * @maskp: pointer to bitmap array that will contain result. + * @nmaskbits: size of bitmap, in bits. + * + * Commas group hex digits into chunks. Each chunk defines exactly 32 + * bits of the resultant bitmask. No chunk may specify a value larger + * than 32 bits (%-EOVERFLOW), and if a chunk specifies a smaller value + * then leading 0-bits are prepended. %-EINVAL is returned for illegal + * characters. Grouping such as "1,,5", ",44", "," or "" is allowed. + * Leading, embedded and trailing whitespace accepted. + */ +int bitmap_parse(const char *start, unsigned int buflen, + unsigned long *maskp, int nmaskbits) +{ + const char *end = strnchrnul(start, buflen, '\n') - 1; + int chunks = BITS_TO_U32(nmaskbits); + u32 *bitmap = (u32 *)maskp; + int unset_bit; + int chunk; + + for (chunk = 0; ; chunk++) { + end = bitmap_find_region_reverse(start, end); + if (start > end) + break; + + if (!chunks--) + return -EOVERFLOW; + +#if defined(CONFIG_64BIT) && defined(__BIG_ENDIAN) + end = bitmap_get_x32_reverse(start, end, &bitmap[chunk ^ 1]); +#else + end = bitmap_get_x32_reverse(start, end, &bitmap[chunk]); +#endif + if (IS_ERR(end)) + return PTR_ERR(end); + } + + unset_bit = (BITS_TO_U32(nmaskbits) - chunks) * 32; + if (unset_bit < nmaskbits) { + bitmap_clear(maskp, unset_bit, nmaskbits - unset_bit); + return 0; + } + + if (find_next_bit(maskp, unset_bit, nmaskbits) != unset_bit) + return -EOVERFLOW; + + return 0; +} +EXPORT_SYMBOL(bitmap_parse); diff --git a/lib/bitmap.c b/lib/bitmap.c index ddb31015e38a..09522af227f1 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -6,21 +6,10 @@ #include <linux/bitmap.h> #include <linux/bitops.h> -#include <linux/bug.h> #include <linux/ctype.h> #include <linux/device.h> -#include <linux/errno.h> #include <linux/export.h> -#include <linux/kernel.h> -#include <linux/mm.h> #include <linux/slab.h> -#include <linux/string.h> -#include <linux/thread_info.h> -#include <linux/uaccess.h> - -#include <asm/page.h> - -#include "kstrtox.h" /** * DOC: bitmap introduction @@ -440,508 +429,6 @@ again: } EXPORT_SYMBOL(bitmap_find_next_zero_area_off); -/* - * Bitmap printing & parsing functions: first version by Nadia Yvette Chambers, - * second version by Paul Jackson, third by Joe Korty. - */ - -/** - * bitmap_parse_user - convert an ASCII hex string in a user buffer into a bitmap - * - * @ubuf: pointer to user buffer containing string. - * @ulen: buffer size in bytes. If string is smaller than this - * then it must be terminated with a \0. - * @maskp: pointer to bitmap array that will contain result. - * @nmaskbits: size of bitmap, in bits. - */ -int bitmap_parse_user(const char __user *ubuf, - unsigned int ulen, unsigned long *maskp, - int nmaskbits) -{ - char *buf; - int ret; - - buf = memdup_user_nul(ubuf, ulen); - if (IS_ERR(buf)) - return PTR_ERR(buf); - - ret = bitmap_parse(buf, UINT_MAX, maskp, nmaskbits); - - kfree(buf); - return ret; -} -EXPORT_SYMBOL(bitmap_parse_user); - -/** - * bitmap_print_to_pagebuf - convert bitmap to list or hex format ASCII string - * @list: indicates whether the bitmap must be list - * @buf: page aligned buffer into which string is placed - * @maskp: pointer to bitmap to convert - * @nmaskbits: size of bitmap, in bits - * - * Output format is a comma-separated list of decimal numbers and - * ranges if list is specified or hex digits grouped into comma-separated - * sets of 8 digits/set. Returns the number of characters written to buf. - * - * It is assumed that @buf is a pointer into a PAGE_SIZE, page-aligned - * area and that sufficient storage remains at @buf to accommodate the - * bitmap_print_to_pagebuf() output. Returns the number of characters - * actually printed to @buf, excluding terminating '\0'. - */ -int bitmap_print_to_pagebuf(bool list, char *buf, const unsigned long *maskp, - int nmaskbits) -{ - ptrdiff_t len = PAGE_SIZE - offset_in_page(buf); - - return list ? scnprintf(buf, len, "%*pbl\n", nmaskbits, maskp) : - scnprintf(buf, len, "%*pb\n", nmaskbits, maskp); -} -EXPORT_SYMBOL(bitmap_print_to_pagebuf); - -/** - * bitmap_print_to_buf - convert bitmap to list or hex format ASCII string - * @list: indicates whether the bitmap must be list - * true: print in decimal list format - * false: print in hexadecimal bitmask format - * @buf: buffer into which string is placed - * @maskp: pointer to bitmap to convert - * @nmaskbits: size of bitmap, in bits - * @off: in the string from which we are copying, We copy to @buf - * @count: the maximum number of bytes to print - */ -static int bitmap_print_to_buf(bool list, char *buf, const unsigned long *maskp, - int nmaskbits, loff_t off, size_t count) -{ - const char *fmt = list ? "%*pbl\n" : "%*pb\n"; - ssize_t size; - void *data; - - data = kasprintf(GFP_KERNEL, fmt, nmaskbits, maskp); - if (!data) - return -ENOMEM; - - size = memory_read_from_buffer(buf, count, &off, data, strlen(data) + 1); - kfree(data); - - return size; -} - -/** - * bitmap_print_bitmask_to_buf - convert bitmap to hex bitmask format ASCII string - * @buf: buffer into which string is placed - * @maskp: pointer to bitmap to convert - * @nmaskbits: size of bitmap, in bits - * @off: in the string from which we are copying, We copy to @buf - * @count: the maximum number of bytes to print - * - * The bitmap_print_to_pagebuf() is used indirectly via its cpumap wrapper - * cpumap_print_to_pagebuf() or directly by drivers to export hexadecimal - * bitmask and decimal list to userspace by sysfs ABI. - * Drivers might be using a normal attribute for this kind of ABIs. A - * normal attribute typically has show entry as below:: - * - * static ssize_t example_attribute_show(struct device *dev, - * struct device_attribute *attr, char *buf) - * { - * ... - * return bitmap_print_to_pagebuf(true, buf, &mask, nr_trig_max); - * } - * - * show entry of attribute has no offset and count parameters and this - * means the file is limited to one page only. - * bitmap_print_to_pagebuf() API works terribly well for this kind of - * normal attribute with buf parameter and without offset, count:: - * - * bitmap_print_to_pagebuf(bool list, char *buf, const unsigned long *maskp, - * int nmaskbits) - * { - * } - * - * The problem is once we have a large bitmap, we have a chance to get a - * bitmask or list more than one page. Especially for list, it could be - * as complex as 0,3,5,7,9,... We have no simple way to know it exact size. - * It turns out bin_attribute is a way to break this limit. bin_attribute - * has show entry as below:: - * - * static ssize_t - * example_bin_attribute_show(struct file *filp, struct kobject *kobj, - * struct bin_attribute *attr, char *buf, - * loff_t offset, size_t count) - * { - * ... - * } - * - * With the new offset and count parameters, this makes sysfs ABI be able - * to support file size more than one page. For example, offset could be - * >= 4096. - * bitmap_print_bitmask_to_buf(), bitmap_print_list_to_buf() wit their - * cpumap wrapper cpumap_print_bitmask_to_buf(), cpumap_print_list_to_buf() - * make those drivers be able to support large bitmask and list after they - * move to use bin_attribute. In result, we have to pass the corresponding - * parameters such as off, count from bin_attribute show entry to this API. - * - * The role of cpumap_print_bitmask_to_buf() and cpumap_print_list_to_buf() - * is similar with cpumap_print_to_pagebuf(), the difference is that - * bitmap_print_to_pagebuf() mainly serves sysfs attribute with the assumption - * the destination buffer is exactly one page and won't be more than one page. - * cpumap_print_bitmask_to_buf() and cpumap_print_list_to_buf(), on the other - * hand, mainly serves bin_attribute which doesn't work with exact one page, - * and it can break the size limit of converted decimal list and hexadecimal - * bitmask. - * - * WARNING! - * - * This function is not a replacement for sprintf() or bitmap_print_to_pagebuf(). - * It is intended to workaround sysfs limitations discussed above and should be - * used carefully in general case for the following reasons: - * - * - Time complexity is O(nbits^2/count), comparing to O(nbits) for snprintf(). - * - Memory complexity is O(nbits), comparing to O(1) for snprintf(). - * - @off and @count are NOT offset and number of bits to print. - * - If printing part of bitmap as list, the resulting string is not a correct - * list representation of bitmap. Particularly, some bits within or out of - * related interval may be erroneously set or unset. The format of the string - * may be broken, so bitmap_parselist-like parser may fail parsing it. - * - If printing the whole bitmap as list by parts, user must ensure the order - * of calls of the function such that the offset is incremented linearly. - * - If printing the whole bitmap as list by parts, user must keep bitmap - * unchanged between the very first and very last call. Otherwise concatenated - * result may be incorrect, and format may be broken. - * - * Returns the number of characters actually printed to @buf - */ -int bitmap_print_bitmask_to_buf(char *buf, const unsigned long *maskp, - int nmaskbits, loff_t off, size_t count) -{ - return bitmap_print_to_buf(false, buf, maskp, nmaskbits, off, count); -} -EXPORT_SYMBOL(bitmap_print_bitmask_to_buf); - -/** - * bitmap_print_list_to_buf - convert bitmap to decimal list format ASCII string - * @buf: buffer into which string is placed - * @maskp: pointer to bitmap to convert - * @nmaskbits: size of bitmap, in bits - * @off: in the string from which we are copying, We copy to @buf - * @count: the maximum number of bytes to print - * - * Everything is same with the above bitmap_print_bitmask_to_buf() except - * the print format. - */ -int bitmap_print_list_to_buf(char *buf, const unsigned long *maskp, - int nmaskbits, loff_t off, size_t count) -{ - return bitmap_print_to_buf(true, buf, maskp, nmaskbits, off, count); -} -EXPORT_SYMBOL(bitmap_print_list_to_buf); - -/* - * Region 9-38:4/10 describes the following bitmap structure: - * 0 9 12 18 38 N - * .........****......****......****.................. - * ^ ^ ^ ^ ^ - * start off group_len end nbits - */ -struct region { - unsigned int start; - unsigned int off; - unsigned int group_len; - unsigned int end; - unsigned int nbits; -}; - -static void bitmap_set_region(const struct region *r, unsigned long *bitmap) -{ - unsigned int start; - - for (start = r->start; start <= r->end; start += r->group_len) - bitmap_set(bitmap, start, min(r->end - start + 1, r->off)); -} - -static int bitmap_check_region(const struct region *r) -{ - if (r->start > r->end || r->group_len == 0 || r->off > r->group_len) - return -EINVAL; - - if (r->end >= r->nbits) - return -ERANGE; - - return 0; -} - -static const char *bitmap_getnum(const char *str, unsigned int *num, - unsigned int lastbit) -{ - unsigned long long n; - unsigned int len; - - if (str[0] == 'N') { - *num = lastbit; - return str + 1; - } - - len = _parse_integer(str, 10, &n); - if (!len) - return ERR_PTR(-EINVAL); - if (len & KSTRTOX_OVERFLOW || n != (unsigned int)n) - return ERR_PTR(-EOVERFLOW); - - *num = n; - return str + len; -} - -static inline bool end_of_str(char c) -{ - return c == '\0' || c == '\n'; -} - -static inline bool __end_of_region(char c) -{ - return isspace(c) || c == ','; -} - -static inline bool end_of_region(char c) -{ - return __end_of_region(c) || end_of_str(c); -} - -/* - * The format allows commas and whitespaces at the beginning - * of the region. - */ -static const char *bitmap_find_region(const char *str) -{ - while (__end_of_region(*str)) - str++; - - return end_of_str(*str) ? NULL : str; -} - -static const char *bitmap_find_region_reverse(const char *start, const char *end) -{ - while (start <= end && __end_of_region(*end)) - end--; - - return end; -} - -static const char *bitmap_parse_region(const char *str, struct region *r) -{ - unsigned int lastbit = r->nbits - 1; - - if (!strncasecmp(str, "all", 3)) { - r->start = 0; - r->end = lastbit; - str += 3; - - goto check_pattern; - } - - str = bitmap_getnum(str, &r->start, lastbit); - if (IS_ERR(str)) - return str; - - if (end_of_region(*str)) - goto no_end; - - if (*str != '-') - return ERR_PTR(-EINVAL); - - str = bitmap_getnum(str + 1, &r->end, lastbit); - if (IS_ERR(str)) - return str; - -check_pattern: - if (end_of_region(*str)) - goto no_pattern; - - if (*str != ':') - return ERR_PTR(-EINVAL); - - str = bitmap_getnum(str + 1, &r->off, lastbit); - if (IS_ERR(str)) - return str; - - if (*str != '/') - return ERR_PTR(-EINVAL); - - return bitmap_getnum(str + 1, &r->group_len, lastbit); - -no_end: - r->end = r->start; -no_pattern: - r->off = r->end + 1; - r->group_len = r->end + 1; - - return end_of_str(*str) ? NULL : str; -} - -/** - * bitmap_parselist - convert list format ASCII string to bitmap - * @buf: read user string from this buffer; must be terminated - * with a \0 or \n. - * @maskp: write resulting mask here - * @nmaskbits: number of bits in mask to be written - * - * Input format is a comma-separated list of decimal numbers and - * ranges. Consecutively set bits are shown as two hyphen-separated - * decimal numbers, the smallest and largest bit numbers set in - * the range. - * Optionally each range can be postfixed to denote that only parts of it - * should be set. The range will divided to groups of specific size. - * From each group will be used only defined amount of bits. - * Syntax: range:used_size/group_size - * Example: 0-1023:2/256 ==> 0,1,256,257,512,513,768,769 - * The value 'N' can be used as a dynamically substituted token for the - * maximum allowed value; i.e (nmaskbits - 1). Keep in mind that it is - * dynamic, so if system changes cause the bitmap width to change, such - * as more cores in a CPU list, then any ranges using N will also change. - * - * Returns: 0 on success, -errno on invalid input strings. Error values: - * - * - ``-EINVAL``: wrong region format - * - ``-EINVAL``: invalid character in string - * - ``-ERANGE``: bit number specified too large for mask - * - ``-EOVERFLOW``: integer overflow in the input parameters - */ -int bitmap_parselist(const char *buf, unsigned long *maskp, int nmaskbits) -{ - struct region r; - long ret; - - r.nbits = nmaskbits; - bitmap_zero(maskp, r.nbits); - - while (buf) { - buf = bitmap_find_region(buf); - if (buf == NULL) - return 0; - - buf = bitmap_parse_region(buf, &r); - if (IS_ERR(buf)) - return PTR_ERR(buf); - - ret = bitmap_check_region(&r); - if (ret) - return ret; - - bitmap_set_region(&r, maskp); - } - - return 0; -} -EXPORT_SYMBOL(bitmap_parselist); - - -/** - * bitmap_parselist_user() - convert user buffer's list format ASCII - * string to bitmap - * - * @ubuf: pointer to user buffer containing string. - * @ulen: buffer size in bytes. If string is smaller than this - * then it must be terminated with a \0. - * @maskp: pointer to bitmap array that will contain result. - * @nmaskbits: size of bitmap, in bits. - * - * Wrapper for bitmap_parselist(), providing it with user buffer. - */ -int bitmap_parselist_user(const char __user *ubuf, - unsigned int ulen, unsigned long *maskp, - int nmaskbits) -{ - char *buf; - int ret; - - buf = memdup_user_nul(ubuf, ulen); - if (IS_ERR(buf)) - return PTR_ERR(buf); - - ret = bitmap_parselist(buf, maskp, nmaskbits); - - kfree(buf); - return ret; -} -EXPORT_SYMBOL(bitmap_parselist_user); - -static const char *bitmap_get_x32_reverse(const char *start, - const char *end, u32 *num) -{ - u32 ret = 0; - int c, i; - - for (i = 0; i < 32; i += 4) { - c = hex_to_bin(*end--); - if (c < 0) - return ERR_PTR(-EINVAL); - - ret |= c << i; - - if (start > end || __end_of_region(*end)) - goto out; - } - - if (hex_to_bin(*end--) >= 0) - return ERR_PTR(-EOVERFLOW); -out: - *num = ret; - return end; -} - -/** - * bitmap_parse - convert an ASCII hex string into a bitmap. - * @start: pointer to buffer containing string. - * @buflen: buffer size in bytes. If string is smaller than this - * then it must be terminated with a \0 or \n. In that case, - * UINT_MAX may be provided instead of string length. - * @maskp: pointer to bitmap array that will contain result. - * @nmaskbits: size of bitmap, in bits. - * - * Commas group hex digits into chunks. Each chunk defines exactly 32 - * bits of the resultant bitmask. No chunk may specify a value larger - * than 32 bits (%-EOVERFLOW), and if a chunk specifies a smaller value - * then leading 0-bits are prepended. %-EINVAL is returned for illegal - * characters. Grouping such as "1,,5", ",44", "," or "" is allowed. - * Leading, embedded and trailing whitespace accepted. - */ -int bitmap_parse(const char *start, unsigned int buflen, - unsigned long *maskp, int nmaskbits) -{ - const char *end = strnchrnul(start, buflen, '\n') - 1; - int chunks = BITS_TO_U32(nmaskbits); - u32 *bitmap = (u32 *)maskp; - int unset_bit; - int chunk; - - for (chunk = 0; ; chunk++) { - end = bitmap_find_region_reverse(start, end); - if (start > end) - break; - - if (!chunks--) - return -EOVERFLOW; - -#if defined(CONFIG_64BIT) && defined(__BIG_ENDIAN) - end = bitmap_get_x32_reverse(start, end, &bitmap[chunk ^ 1]); -#else - end = bitmap_get_x32_reverse(start, end, &bitmap[chunk]); -#endif - if (IS_ERR(end)) - return PTR_ERR(end); - } - - unset_bit = (BITS_TO_U32(nmaskbits) - chunks) * 32; - if (unset_bit < nmaskbits) { - bitmap_clear(maskp, unset_bit, nmaskbits - unset_bit); - return 0; - } - - if (find_next_bit(maskp, unset_bit, nmaskbits) != unset_bit) - return -EOVERFLOW; - - return 0; -} -EXPORT_SYMBOL(bitmap_parse); - /** * bitmap_pos_to_ord - find ordinal of set bit at given position in bitmap * @buf: pointer to a bitmap @@ -988,7 +475,7 @@ static int bitmap_pos_to_ord(const unsigned long *buf, unsigned int pos, unsigne * to @dst. * * The positions of unset bits in @old are mapped to themselves - * (the identify map). + * (the identity map). * * Apply the above specified mapping to @src, placing the result in * @dst, clearing any bits previously set in @dst. @@ -1037,7 +524,7 @@ EXPORT_SYMBOL(bitmap_remap); * the position of the m-th set bit in @new, where m == n % w. * * The positions of unset bits in @old are mapped to themselves - * (the identify map). + * (the identity map). * * Apply the above specified mapping to bit position @oldbit, returning * the new bit position. @@ -1220,169 +707,6 @@ void bitmap_fold(unsigned long *dst, const unsigned long *orig, } #endif /* CONFIG_NUMA */ -/* - * Common code for bitmap_*_region() routines. - * bitmap: array of unsigned longs corresponding to the bitmap - * pos: the beginning of the region - * order: region size (log base 2 of number of bits) - * reg_op: operation(s) to perform on that region of bitmap - * - * Can set, verify and/or release a region of bits in a bitmap, - * depending on which combination of REG_OP_* flag bits is set. - * - * A region of a bitmap is a sequence of bits in the bitmap, of - * some size '1 << order' (a power of two), aligned to that same - * '1 << order' power of two. - * - * Returns 1 if REG_OP_ISFREE succeeds (region is all zero bits). - * Returns 0 in all other cases and reg_ops. - */ - -enum { - REG_OP_ISFREE, /* true if region is all zero bits */ - REG_OP_ALLOC, /* set all bits in region */ - REG_OP_RELEASE, /* clear all bits in region */ -}; - -static int __reg_op(unsigned long *bitmap, unsigned int pos, int order, int reg_op) -{ - int nbits_reg; /* number of bits in region */ - int index; /* index first long of region in bitmap */ - int offset; /* bit offset region in bitmap[index] */ - int nlongs_reg; /* num longs spanned by region in bitmap */ - int nbitsinlong; /* num bits of region in each spanned long */ - unsigned long mask; /* bitmask for one long of region */ - int i; /* scans bitmap by longs */ - int ret = 0; /* return value */ - - /* - * Either nlongs_reg == 1 (for small orders that fit in one long) - * or (offset == 0 && mask == ~0UL) (for larger multiword orders.) - */ - nbits_reg = 1 << order; - index = pos / BITS_PER_LONG; - offset = pos - (index * BITS_PER_LONG); - nlongs_reg = BITS_TO_LONGS(nbits_reg); - nbitsinlong = min(nbits_reg, BITS_PER_LONG); - - /* - * Can't do "mask = (1UL << nbitsinlong) - 1", as that - * overflows if nbitsinlong == BITS_PER_LONG. - */ - mask = (1UL << (nbitsinlong - 1)); - mask += mask - 1; - mask <<= offset; - - switch (reg_op) { - case REG_OP_ISFREE: - for (i = 0; i < nlongs_reg; i++) { - if (bitmap[index + i] & mask) - goto done; - } - ret = 1; /* all bits in region free (zero) */ - break; - - case REG_OP_ALLOC: - for (i = 0; i < nlongs_reg; i++) - bitmap[index + i] |= mask; - break; - - case REG_OP_RELEASE: - for (i = 0; i < nlongs_reg; i++) - bitmap[index + i] &= ~mask; - break; - } -done: - return ret; -} - -/** - * bitmap_find_free_region - find a contiguous aligned mem region - * @bitmap: array of unsigned longs corresponding to the bitmap - * @bits: number of bits in the bitmap - * @order: region size (log base 2 of number of bits) to find - * - * Find a region of free (zero) bits in a @bitmap of @bits bits and - * allocate them (set them to one). Only consider regions of length - * a power (@order) of two, aligned to that power of two, which - * makes the search algorithm much faster. - * - * Return the bit offset in bitmap of the allocated region, - * or -errno on failure. - */ -int bitmap_find_free_region(unsigned long *bitmap, unsigned int bits, int order) -{ - unsigned int pos, end; /* scans bitmap by regions of size order */ - - for (pos = 0 ; (end = pos + (1U << order)) <= bits; pos = end) { - if (!__reg_op(bitmap, pos, order, REG_OP_ISFREE)) - continue; - __reg_op(bitmap, pos, order, REG_OP_ALLOC); - return pos; - } - return -ENOMEM; -} -EXPORT_SYMBOL(bitmap_find_free_region); - -/** - * bitmap_release_region - release allocated bitmap region - * @bitmap: array of unsigned longs corresponding to the bitmap - * @pos: beginning of bit region to release - * @order: region size (log base 2 of number of bits) to release - * - * This is the complement to __bitmap_find_free_region() and releases - * the found region (by clearing it in the bitmap). - * - * No return value. - */ -void bitmap_release_region(unsigned long *bitmap, unsigned int pos, int order) -{ - __reg_op(bitmap, pos, order, REG_OP_RELEASE); -} -EXPORT_SYMBOL(bitmap_release_region); - -/** - * bitmap_allocate_region - allocate bitmap region - * @bitmap: array of unsigned longs corresponding to the bitmap - * @pos: beginning of bit region to allocate - * @order: region size (log base 2 of number of bits) to allocate - * - * Allocate (set bits in) a specified region of a bitmap. - * - * Return 0 on success, or %-EBUSY if specified region wasn't - * free (not all bits were zero). - */ -int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos, int order) -{ - if (!__reg_op(bitmap, pos, order, REG_OP_ISFREE)) - return -EBUSY; - return __reg_op(bitmap, pos, order, REG_OP_ALLOC); -} -EXPORT_SYMBOL(bitmap_allocate_region); - -/** - * bitmap_copy_le - copy a bitmap, putting the bits into little-endian order. - * @dst: destination buffer - * @src: bitmap to copy - * @nbits: number of bits in the bitmap - * - * Require nbits % BITS_PER_LONG == 0. - */ -#ifdef __BIG_ENDIAN -void bitmap_copy_le(unsigned long *dst, const unsigned long *src, unsigned int nbits) -{ - unsigned int i; - - for (i = 0; i < nbits/BITS_PER_LONG; i++) { - if (BITS_PER_LONG == 64) - dst[i] = cpu_to_le64(src[i]); - else - dst[i] = cpu_to_le32(src[i]); - } -} -EXPORT_SYMBOL(bitmap_copy_le); -#endif - unsigned long *bitmap_alloc(unsigned int nbits, gfp_t flags) { return kmalloc_array(BITS_TO_LONGS(nbits), sizeof(unsigned long), diff --git a/lib/checksum_kunit.c b/lib/checksum_kunit.c index ace3c4799fe1..0eed92b77ba3 100644 --- a/lib/checksum_kunit.c +++ b/lib/checksum_kunit.c @@ -10,7 +10,8 @@ #define MAX_ALIGN 64 #define TEST_BUFLEN (MAX_LEN + MAX_ALIGN) -static const __wsum random_init_sum = 0x2847aab; +/* Values for a little endian CPU. Byte swap each half on big endian CPU. */ +static const u32 random_init_sum = 0x2847aab; static const u8 random_buf[] = { 0xac, 0xd7, 0x76, 0x69, 0x6e, 0xf2, 0x93, 0x2c, 0x1f, 0xe0, 0xde, 0x86, 0x8f, 0x54, 0x33, 0x90, 0x95, 0xbf, 0xff, 0xb9, 0xea, 0x62, 0x6e, 0xb5, @@ -56,7 +57,9 @@ static const u8 random_buf[] = { 0xe1, 0xdf, 0x4b, 0xe1, 0x81, 0xe2, 0x17, 0x02, 0x7b, 0x58, 0x8b, 0x92, 0x1a, 0xac, 0x46, 0xdd, 0x2e, 0xce, 0x40, 0x09 }; -static const __sum16 expected_results[] = { + +/* Values for a little endian CPU. Byte swap on big endian CPU. */ +static const u16 expected_results[] = { 0x82d0, 0x8224, 0xab23, 0xaaad, 0x41ad, 0x413f, 0x4f3e, 0x4eab, 0x22ab, 0x228c, 0x428b, 0x41ad, 0xbbac, 0xbb1d, 0x671d, 0x66ea, 0xd6e9, 0xd654, 0x1754, 0x1655, 0x5d54, 0x5c6a, 0xfa69, 0xf9fb, 0x44fb, 0x4428, 0xf527, @@ -115,7 +118,9 @@ static const __sum16 expected_results[] = { 0x1d47, 0x3c46, 0x3bc5, 0x59c4, 0x59ad, 0x57ad, 0x5732, 0xff31, 0xfea6, 0x6ca6, 0x6c8c, 0xc08b, 0xc045, 0xe344, 0xe316, 0x1516, 0x14d6, }; -static const __wsum init_sums_no_overflow[] = { + +/* Values for a little endian CPU. Byte swap each half on big endian CPU. */ +static const u32 init_sums_no_overflow[] = { 0xffffffff, 0xfffffffb, 0xfffffbfb, 0xfffffbf7, 0xfffff7f7, 0xfffff7f3, 0xfffff3f3, 0xfffff3ef, 0xffffefef, 0xffffefeb, 0xffffebeb, 0xffffebe7, 0xffffe7e7, 0xffffe7e3, 0xffffe3e3, 0xffffe3df, 0xffffdfdf, 0xffffdfdb, @@ -208,7 +213,21 @@ static u8 tmp_buf[TEST_BUFLEN]; #define full_csum(buff, len, sum) csum_fold(csum_partial(buff, len, sum)) -#define CHECK_EQ(lhs, rhs) KUNIT_ASSERT_EQ(test, lhs, rhs) +#define CHECK_EQ(lhs, rhs) KUNIT_ASSERT_EQ(test, (__force u64)lhs, (__force u64)rhs) + +static __sum16 to_sum16(u16 x) +{ + return (__force __sum16)le16_to_cpu((__force __le16)x); +} + +/* This function swaps the bytes inside each half of a __wsum */ +static __wsum to_wsum(u32 x) +{ + u16 hi = le16_to_cpu((__force __le16)(x >> 16)); + u16 lo = le16_to_cpu((__force __le16)x); + + return (__force __wsum)((hi << 16) | lo); +} static void assert_setup_correct(struct kunit *test) { @@ -226,7 +245,8 @@ static void assert_setup_correct(struct kunit *test) static void test_csum_fixed_random_inputs(struct kunit *test) { int len, align; - __wsum result, expec, sum; + __wsum sum; + __sum16 result, expec; assert_setup_correct(test); for (align = 0; align < TEST_BUFLEN; ++align) { @@ -237,9 +257,9 @@ static void test_csum_fixed_random_inputs(struct kunit *test) /* * Test the precomputed random input. */ - sum = random_init_sum; + sum = to_wsum(random_init_sum); result = full_csum(&tmp_buf[align], len, sum); - expec = expected_results[len]; + expec = to_sum16(expected_results[len]); CHECK_EQ(result, expec); } } @@ -251,7 +271,8 @@ static void test_csum_fixed_random_inputs(struct kunit *test) static void test_csum_all_carry_inputs(struct kunit *test) { int len, align; - __wsum result, expec, sum; + __wsum sum; + __sum16 result, expec; assert_setup_correct(test); memset(tmp_buf, 0xff, TEST_BUFLEN); @@ -261,9 +282,9 @@ static void test_csum_all_carry_inputs(struct kunit *test) /* * All carries from input and initial sum. */ - sum = 0xffffffff; + sum = to_wsum(0xffffffff); result = full_csum(&tmp_buf[align], len, sum); - expec = (len & 1) ? 0xff00 : 0; + expec = to_sum16((len & 1) ? 0xff00 : 0); CHECK_EQ(result, expec); /* @@ -272,11 +293,11 @@ static void test_csum_all_carry_inputs(struct kunit *test) sum = 0; result = full_csum(&tmp_buf[align], len, sum); if (len & 1) - expec = 0xff00; + expec = to_sum16(0xff00); else if (len) expec = 0; else - expec = 0xffff; + expec = to_sum16(0xffff); CHECK_EQ(result, expec); } } @@ -290,7 +311,8 @@ static void test_csum_all_carry_inputs(struct kunit *test) static void test_csum_no_carry_inputs(struct kunit *test) { int len, align; - __wsum result, expec, sum; + __wsum sum; + __sum16 result, expec; assert_setup_correct(test); memset(tmp_buf, 0x4, TEST_BUFLEN); @@ -300,7 +322,7 @@ static void test_csum_no_carry_inputs(struct kunit *test) /* * Expect no carries. */ - sum = init_sums_no_overflow[len]; + sum = to_wsum(init_sums_no_overflow[len]); result = full_csum(&tmp_buf[align], len, sum); expec = 0; CHECK_EQ(result, expec); @@ -308,9 +330,9 @@ static void test_csum_no_carry_inputs(struct kunit *test) /* * Expect one carry. */ - sum = init_sums_no_overflow[len] + 1; + sum = to_wsum(init_sums_no_overflow[len] + 1); result = full_csum(&tmp_buf[align], len, sum); - expec = len ? 0xfffe : 0xffff; + expec = to_sum16(len ? 0xfffe : 0xffff); CHECK_EQ(result, expec); } } diff --git a/lib/closure.c b/lib/closure.c new file mode 100644 index 000000000000..c16540552d61 --- /dev/null +++ b/lib/closure.c @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Asynchronous refcounty things + * + * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com> + * Copyright 2012 Google, Inc. + */ + +#include <linux/closure.h> +#include <linux/debugfs.h> +#include <linux/export.h> +#include <linux/rcupdate.h> +#include <linux/seq_file.h> +#include <linux/sched/debug.h> + +static inline void closure_put_after_sub(struct closure *cl, int flags) +{ + int r = flags & CLOSURE_REMAINING_MASK; + + BUG_ON(flags & CLOSURE_GUARD_MASK); + BUG_ON(!r && (flags & ~CLOSURE_DESTRUCTOR)); + + if (!r) { + smp_acquire__after_ctrl_dep(); + + cl->closure_get_happened = false; + + if (cl->fn && !(flags & CLOSURE_DESTRUCTOR)) { + atomic_set(&cl->remaining, + CLOSURE_REMAINING_INITIALIZER); + closure_queue(cl); + } else { + struct closure *parent = cl->parent; + closure_fn *destructor = cl->fn; + + closure_debug_destroy(cl); + + if (destructor) + destructor(&cl->work); + + if (parent) + closure_put(parent); + } + } +} + +/* For clearing flags with the same atomic op as a put */ +void closure_sub(struct closure *cl, int v) +{ + closure_put_after_sub(cl, atomic_sub_return_release(v, &cl->remaining)); +} +EXPORT_SYMBOL(closure_sub); + +/* + * closure_put - decrement a closure's refcount + */ +void closure_put(struct closure *cl) +{ + closure_put_after_sub(cl, atomic_dec_return_release(&cl->remaining)); +} +EXPORT_SYMBOL(closure_put); + +/* + * closure_wake_up - wake up all closures on a wait list, without memory barrier + */ +void __closure_wake_up(struct closure_waitlist *wait_list) +{ + struct llist_node *list; + struct closure *cl, *t; + struct llist_node *reverse = NULL; + + list = llist_del_all(&wait_list->list); + + /* We first reverse the list to preserve FIFO ordering and fairness */ + reverse = llist_reverse_order(list); + + /* Then do the wakeups */ + llist_for_each_entry_safe(cl, t, reverse, list) { + closure_set_waiting(cl, 0); + closure_sub(cl, CLOSURE_WAITING + 1); + } +} +EXPORT_SYMBOL(__closure_wake_up); + +/** + * closure_wait - add a closure to a waitlist + * @waitlist: will own a ref on @cl, which will be released when + * closure_wake_up() is called on @waitlist. + * @cl: closure pointer. + * + */ +bool closure_wait(struct closure_waitlist *waitlist, struct closure *cl) +{ + if (atomic_read(&cl->remaining) & CLOSURE_WAITING) + return false; + + cl->closure_get_happened = true; + closure_set_waiting(cl, _RET_IP_); + atomic_add(CLOSURE_WAITING + 1, &cl->remaining); + llist_add(&cl->list, &waitlist->list); + + return true; +} +EXPORT_SYMBOL(closure_wait); + +struct closure_syncer { + struct task_struct *task; + int done; +}; + +static CLOSURE_CALLBACK(closure_sync_fn) +{ + struct closure *cl = container_of(ws, struct closure, work); + struct closure_syncer *s = cl->s; + struct task_struct *p; + + rcu_read_lock(); + p = READ_ONCE(s->task); + s->done = 1; + wake_up_process(p); + rcu_read_unlock(); +} + +void __sched __closure_sync(struct closure *cl) +{ + struct closure_syncer s = { .task = current }; + + cl->s = &s; + continue_at(cl, closure_sync_fn, NULL); + + while (1) { + set_current_state(TASK_UNINTERRUPTIBLE); + if (s.done) + break; + schedule(); + } + + __set_current_state(TASK_RUNNING); +} +EXPORT_SYMBOL(__closure_sync); + +#ifdef CONFIG_DEBUG_CLOSURES + +static LIST_HEAD(closure_list); +static DEFINE_SPINLOCK(closure_list_lock); + +void closure_debug_create(struct closure *cl) +{ + unsigned long flags; + + BUG_ON(cl->magic == CLOSURE_MAGIC_ALIVE); + cl->magic = CLOSURE_MAGIC_ALIVE; + + spin_lock_irqsave(&closure_list_lock, flags); + list_add(&cl->all, &closure_list); + spin_unlock_irqrestore(&closure_list_lock, flags); +} +EXPORT_SYMBOL(closure_debug_create); + +void closure_debug_destroy(struct closure *cl) +{ + unsigned long flags; + + BUG_ON(cl->magic != CLOSURE_MAGIC_ALIVE); + cl->magic = CLOSURE_MAGIC_DEAD; + + spin_lock_irqsave(&closure_list_lock, flags); + list_del(&cl->all); + spin_unlock_irqrestore(&closure_list_lock, flags); +} +EXPORT_SYMBOL(closure_debug_destroy); + +static int debug_show(struct seq_file *f, void *data) +{ + struct closure *cl; + + spin_lock_irq(&closure_list_lock); + + list_for_each_entry(cl, &closure_list, all) { + int r = atomic_read(&cl->remaining); + + seq_printf(f, "%p: %pS -> %pS p %p r %i ", + cl, (void *) cl->ip, cl->fn, cl->parent, + r & CLOSURE_REMAINING_MASK); + + seq_printf(f, "%s%s\n", + test_bit(WORK_STRUCT_PENDING_BIT, + work_data_bits(&cl->work)) ? "Q" : "", + r & CLOSURE_RUNNING ? "R" : ""); + + if (r & CLOSURE_WAITING) + seq_printf(f, " W %pS\n", + (void *) cl->waiting_on); + + seq_puts(f, "\n"); + } + + spin_unlock_irq(&closure_list_lock); + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(debug); + +static int __init closure_debug_init(void) +{ + debugfs_create_file("closures", 0400, NULL, NULL, &debug_fops); + return 0; +} +late_initcall(closure_debug_init) + +#endif diff --git a/lib/cpumask.c b/lib/cpumask.c index a7fd02b5ae26..e77ee9d46f71 100644 --- a/lib/cpumask.c +++ b/lib/cpumask.c @@ -14,7 +14,7 @@ * @start: the start point of the iteration * @wrap: assume @n crossing @start terminates the iteration * - * Returns >= nr_cpu_ids on completion + * Return: >= nr_cpu_ids on completion * * Note: the @wrap argument is required for the start condition when * we cannot assume @start is set in @mask. @@ -48,8 +48,9 @@ EXPORT_SYMBOL(cpumask_next_wrap); * @node: memory node from which to allocate or %NUMA_NO_NODE * * Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is - * a nop returning a constant 1 (in <linux/cpumask.h>) - * Returns TRUE if memory allocation succeeded, FALSE otherwise. + * a nop returning a constant 1 (in <linux/cpumask.h>). + * + * Return: TRUE if memory allocation succeeded, FALSE otherwise. * * In addition, mask will be NULL if this fails. Note that gcc is * usually smart enough to know that mask can never be NULL if @@ -115,7 +116,7 @@ void __init free_bootmem_cpumask_var(cpumask_var_t mask) * @i: index number * @node: local numa_node * - * Returns online CPU according to a numa aware policy; local cpus are returned + * Return: online CPU according to a numa aware policy; local cpus are returned * first, followed by non-local ones, then it wraps around. * * For those who wants to enumerate all CPUs based on their NUMA distances, @@ -146,9 +147,7 @@ unsigned int cpumask_local_spread(unsigned int i, int node) /* Wrap: we always want a cpu. */ i %= num_online_cpus(); - cpu = (node == NUMA_NO_NODE) ? - cpumask_nth(i, cpu_online_mask) : - sched_numa_find_nth_cpu(cpu_online_mask, i, node); + cpu = sched_numa_find_nth_cpu(cpu_online_mask, i, node); WARN_ON(cpu >= nr_cpu_ids); return cpu; @@ -165,7 +164,7 @@ static DEFINE_PER_CPU(int, distribute_cpu_mask_prev); * Iterated calls using the same srcp1 and srcp2 will be distributed within * their intersection. * - * Returns >= nr_cpu_ids if the intersection is empty. + * Return: >= nr_cpu_ids if the intersection is empty. */ unsigned int cpumask_any_and_distribute(const struct cpumask *src1p, const struct cpumask *src2p) @@ -184,6 +183,12 @@ unsigned int cpumask_any_and_distribute(const struct cpumask *src1p, } EXPORT_SYMBOL(cpumask_any_and_distribute); +/** + * cpumask_any_distribute - Return an arbitrary cpu from srcp + * @srcp: &cpumask for selection + * + * Return: >= nr_cpu_ids if the intersection is empty. + */ unsigned int cpumask_any_distribute(const struct cpumask *srcp) { unsigned int next, prev; diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 6ec2d4543d9c..8d1446c2be71 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -53,3 +53,5 @@ libblake2s-y += blake2s-selftest.o libchacha20poly1305-y += chacha20poly1305-selftest.o libcurve25519-y += curve25519-selftest.o endif + +obj-$(CONFIG_MPILIB) += mpi/ diff --git a/lib/mpi/Makefile b/lib/crypto/mpi/Makefile index 6e6ef9a34fe1..6e6ef9a34fe1 100644 --- a/lib/mpi/Makefile +++ b/lib/crypto/mpi/Makefile diff --git a/lib/mpi/ec.c b/lib/crypto/mpi/ec.c index 40f5908e57a4..40f5908e57a4 100644 --- a/lib/mpi/ec.c +++ b/lib/crypto/mpi/ec.c diff --git a/lib/mpi/generic_mpih-add1.c b/lib/crypto/mpi/generic_mpih-add1.c index 299308b5461c..299308b5461c 100644 --- a/lib/mpi/generic_mpih-add1.c +++ b/lib/crypto/mpi/generic_mpih-add1.c diff --git a/lib/mpi/generic_mpih-lshift.c b/lib/crypto/mpi/generic_mpih-lshift.c index 7b21f5938a50..7b21f5938a50 100644 --- a/lib/mpi/generic_mpih-lshift.c +++ b/lib/crypto/mpi/generic_mpih-lshift.c diff --git a/lib/mpi/generic_mpih-mul1.c b/lib/crypto/mpi/generic_mpih-mul1.c index e020e61d47b9..e020e61d47b9 100644 --- a/lib/mpi/generic_mpih-mul1.c +++ b/lib/crypto/mpi/generic_mpih-mul1.c diff --git a/lib/mpi/generic_mpih-mul2.c b/lib/crypto/mpi/generic_mpih-mul2.c index 9484d8528243..9484d8528243 100644 --- a/lib/mpi/generic_mpih-mul2.c +++ b/lib/crypto/mpi/generic_mpih-mul2.c diff --git a/lib/mpi/generic_mpih-mul3.c b/lib/crypto/mpi/generic_mpih-mul3.c index ccdbab4121e0..ccdbab4121e0 100644 --- a/lib/mpi/generic_mpih-mul3.c +++ b/lib/crypto/mpi/generic_mpih-mul3.c diff --git a/lib/mpi/generic_mpih-rshift.c b/lib/crypto/mpi/generic_mpih-rshift.c index e07bc69aa898..e07bc69aa898 100644 --- a/lib/mpi/generic_mpih-rshift.c +++ b/lib/crypto/mpi/generic_mpih-rshift.c diff --git a/lib/mpi/generic_mpih-sub1.c b/lib/crypto/mpi/generic_mpih-sub1.c index eea4382aad5f..eea4382aad5f 100644 --- a/lib/mpi/generic_mpih-sub1.c +++ b/lib/crypto/mpi/generic_mpih-sub1.c diff --git a/lib/mpi/longlong.h b/lib/crypto/mpi/longlong.h index b6fa1d08fb55..b6fa1d08fb55 100644 --- a/lib/mpi/longlong.h +++ b/lib/crypto/mpi/longlong.h diff --git a/lib/mpi/mpi-add.c b/lib/crypto/mpi/mpi-add.c index 9056fc5167fc..9056fc5167fc 100644 --- a/lib/mpi/mpi-add.c +++ b/lib/crypto/mpi/mpi-add.c diff --git a/lib/mpi/mpi-bit.c b/lib/crypto/mpi/mpi-bit.c index 070ba784c9f1..070ba784c9f1 100644 --- a/lib/mpi/mpi-bit.c +++ b/lib/crypto/mpi/mpi-bit.c diff --git a/lib/mpi/mpi-cmp.c b/lib/crypto/mpi/mpi-cmp.c index c4cfa3ff0581..0835b6213235 100644 --- a/lib/mpi/mpi-cmp.c +++ b/lib/crypto/mpi/mpi-cmp.c @@ -25,8 +25,12 @@ int mpi_cmp_ui(MPI u, unsigned long v) mpi_limb_t limb = v; mpi_normalize(u); - if (!u->nlimbs && !limb) - return 0; + if (u->nlimbs == 0) { + if (v == 0) + return 0; + else + return -1; + } if (u->sign) return -1; if (u->nlimbs > 1) diff --git a/lib/mpi/mpi-div.c b/lib/crypto/mpi/mpi-div.c index 45beab8b9e9e..45beab8b9e9e 100644 --- a/lib/mpi/mpi-div.c +++ b/lib/crypto/mpi/mpi-div.c diff --git a/lib/mpi/mpi-inline.h b/lib/crypto/mpi/mpi-inline.h index 980b6b940953..980b6b940953 100644 --- a/lib/mpi/mpi-inline.h +++ b/lib/crypto/mpi/mpi-inline.h diff --git a/lib/mpi/mpi-internal.h b/lib/crypto/mpi/mpi-internal.h index 554002182db1..554002182db1 100644 --- a/lib/mpi/mpi-internal.h +++ b/lib/crypto/mpi/mpi-internal.h diff --git a/lib/mpi/mpi-inv.c b/lib/crypto/mpi/mpi-inv.c index 61e37d18f793..61e37d18f793 100644 --- a/lib/mpi/mpi-inv.c +++ b/lib/crypto/mpi/mpi-inv.c diff --git a/lib/mpi/mpi-mod.c b/lib/crypto/mpi/mpi-mod.c index 54fcc01564d9..54fcc01564d9 100644 --- a/lib/mpi/mpi-mod.c +++ b/lib/crypto/mpi/mpi-mod.c diff --git a/lib/mpi/mpi-mul.c b/lib/crypto/mpi/mpi-mul.c index 7f4eda8560dc..7f4eda8560dc 100644 --- a/lib/mpi/mpi-mul.c +++ b/lib/crypto/mpi/mpi-mul.c diff --git a/lib/mpi/mpi-pow.c b/lib/crypto/mpi/mpi-pow.c index 2fd7a46d55ec..2fd7a46d55ec 100644 --- a/lib/mpi/mpi-pow.c +++ b/lib/crypto/mpi/mpi-pow.c diff --git a/lib/mpi/mpi-sub-ui.c b/lib/crypto/mpi/mpi-sub-ui.c index b41b082b5f3e..b41b082b5f3e 100644 --- a/lib/mpi/mpi-sub-ui.c +++ b/lib/crypto/mpi/mpi-sub-ui.c diff --git a/lib/mpi/mpicoder.c b/lib/crypto/mpi/mpicoder.c index 3cb6bd148fa9..3cb6bd148fa9 100644 --- a/lib/mpi/mpicoder.c +++ b/lib/crypto/mpi/mpicoder.c diff --git a/lib/mpi/mpih-cmp.c b/lib/crypto/mpi/mpih-cmp.c index f23709114a65..f23709114a65 100644 --- a/lib/mpi/mpih-cmp.c +++ b/lib/crypto/mpi/mpih-cmp.c diff --git a/lib/mpi/mpih-div.c b/lib/crypto/mpi/mpih-div.c index be70ee2e42d3..be70ee2e42d3 100644 --- a/lib/mpi/mpih-div.c +++ b/lib/crypto/mpi/mpih-div.c diff --git a/lib/mpi/mpih-mul.c b/lib/crypto/mpi/mpih-mul.c index e5f1c84e3c48..e5f1c84e3c48 100644 --- a/lib/mpi/mpih-mul.c +++ b/lib/crypto/mpi/mpih-mul.c diff --git a/lib/mpi/mpiutil.c b/lib/crypto/mpi/mpiutil.c index aa8c46544af8..aa8c46544af8 100644 --- a/lib/mpi/mpiutil.c +++ b/lib/crypto/mpi/mpiutil.c diff --git a/lib/debugobjects.c b/lib/debugobjects.c index a517256a270b..2a8e9d63fbe3 100644 --- a/lib/debugobjects.c +++ b/lib/debugobjects.c @@ -89,7 +89,7 @@ static int debug_objects_pool_size __read_mostly static int debug_objects_pool_min_level __read_mostly = ODEBUG_POOL_MIN_LEVEL; static const struct debug_obj_descr *descr_test __read_mostly; -static struct kmem_cache *obj_cache __read_mostly; +static struct kmem_cache *obj_cache __ro_after_init; /* * Track numbers of kmem_cache_alloc()/free() calls done. diff --git a/lib/decompress_unxz.c b/lib/decompress_unxz.c index 353268b9f129..842894158944 100644 --- a/lib/decompress_unxz.c +++ b/lib/decompress_unxz.c @@ -133,9 +133,6 @@ #ifdef CONFIG_ARM # define XZ_DEC_ARM #endif -#ifdef CONFIG_IA64 -# define XZ_DEC_IA64 -#endif #ifdef CONFIG_SPARC # define XZ_DEC_SPARC #endif diff --git a/lib/dynamic_debug.c b/lib/dynamic_debug.c index fdd6d9800a70..6fba6423cc10 100644 --- a/lib/dynamic_debug.c +++ b/lib/dynamic_debug.c @@ -88,10 +88,11 @@ static inline const char *trim_prefix(const char *path) return path + skip; } -static struct { unsigned flag:8; char opt_char; } opt_array[] = { +static const struct { unsigned flag:8; char opt_char; } opt_array[] = { { _DPRINTK_FLAGS_PRINT, 'p' }, { _DPRINTK_FLAGS_INCL_MODNAME, 'm' }, { _DPRINTK_FLAGS_INCL_FUNCNAME, 'f' }, + { _DPRINTK_FLAGS_INCL_SOURCENAME, 's' }, { _DPRINTK_FLAGS_INCL_LINENO, 'l' }, { _DPRINTK_FLAGS_INCL_TID, 't' }, { _DPRINTK_FLAGS_NONE, '_' }, @@ -808,7 +809,7 @@ const struct kernel_param_ops param_ops_dyndbg_classes = { }; EXPORT_SYMBOL(param_ops_dyndbg_classes); -#define PREFIX_SIZE 64 +#define PREFIX_SIZE 128 static int remaining(int wrote) { @@ -836,6 +837,9 @@ static char *__dynamic_emit_prefix(const struct _ddebug *desc, char *buf) if (desc->flags & _DPRINTK_FLAGS_INCL_FUNCNAME) pos += snprintf(buf + pos, remaining(pos), "%s:", desc->function); + if (desc->flags & _DPRINTK_FLAGS_INCL_SOURCENAME) + pos += snprintf(buf + pos, remaining(pos), "%s:", + trim_prefix(desc->filename)); if (desc->flags & _DPRINTK_FLAGS_INCL_LINENO) pos += snprintf(buf + pos, remaining(pos), "%d:", desc->lineno); diff --git a/lib/errname.c b/lib/errname.c index 67739b174a8c..4f9112b38f3a 100644 --- a/lib/errname.c +++ b/lib/errname.c @@ -111,9 +111,6 @@ static const char *names_0[] = { E(ENOSPC), E(ENOSR), E(ENOSTR), -#ifdef ENOSYM - E(ENOSYM), -#endif E(ENOSYS), E(ENOTBLK), E(ENOTCONN), @@ -144,9 +141,6 @@ static const char *names_0[] = { #endif E(EREMOTE), E(EREMOTEIO), -#ifdef EREMOTERELEASE - E(EREMOTERELEASE), -#endif E(ERESTART), E(ERFKILL), E(EROFS), @@ -228,3 +222,4 @@ const char *errname(int err) return err > 0 ? name + 1 : name; } +EXPORT_SYMBOL(errname); diff --git a/lib/error-inject.c b/lib/error-inject.c index 32c14770508e..887acd9a6ea6 100644 --- a/lib/error-inject.c +++ b/lib/error-inject.c @@ -217,8 +217,6 @@ static int __init ei_debugfs_init(void) struct dentry *dir, *file; dir = debugfs_create_dir("error_injection", NULL); - if (!dir) - return -ENOMEM; file = debugfs_create_file("list", 0444, dir, NULL, &ei_fops); if (!file) { diff --git a/lib/fonts/Kconfig b/lib/fonts/Kconfig index c035fde66aeb..7ee468ef21ec 100644 --- a/lib/fonts/Kconfig +++ b/lib/fonts/Kconfig @@ -98,7 +98,7 @@ config FONT_10x18 config FONT_SUN8x16 bool "Sparc console 8x16 font" - depends on FRAMEBUFFER_CONSOLE && (!SPARC && FONTS || SPARC) + depends on (FRAMEBUFFER_CONSOLE && (FONTS || SPARC)) || BOOTX_TEXT help This is the high resolution console font for Sun machines. Say Y. diff --git a/lib/fw_table.c b/lib/fw_table.c new file mode 100644 index 000000000000..294df54e33b6 --- /dev/null +++ b/lib/fw_table.c @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * fw_tables.c - Parsing support for ACPI and ACPI-like tables provided by + * platform or device firmware + * + * Copyright (C) 2001 Paul Diefenbaugh <paul.s.diefenbaugh@intel.com> + * Copyright (C) 2023 Intel Corp. + */ +#include <linux/errno.h> +#include <linux/acpi.h> +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/string.h> +#include <linux/types.h> + +enum acpi_subtable_type { + ACPI_SUBTABLE_COMMON, + ACPI_SUBTABLE_HMAT, + ACPI_SUBTABLE_PRMT, + ACPI_SUBTABLE_CEDT, +}; + +struct acpi_subtable_entry { + union acpi_subtable_headers *hdr; + enum acpi_subtable_type type; +}; + +static unsigned long __init_or_acpilib +acpi_get_entry_type(struct acpi_subtable_entry *entry) +{ + switch (entry->type) { + case ACPI_SUBTABLE_COMMON: + return entry->hdr->common.type; + case ACPI_SUBTABLE_HMAT: + return entry->hdr->hmat.type; + case ACPI_SUBTABLE_PRMT: + return 0; + case ACPI_SUBTABLE_CEDT: + return entry->hdr->cedt.type; + } + return 0; +} + +static unsigned long __init_or_acpilib +acpi_get_entry_length(struct acpi_subtable_entry *entry) +{ + switch (entry->type) { + case ACPI_SUBTABLE_COMMON: + return entry->hdr->common.length; + case ACPI_SUBTABLE_HMAT: + return entry->hdr->hmat.length; + case ACPI_SUBTABLE_PRMT: + return entry->hdr->prmt.length; + case ACPI_SUBTABLE_CEDT: + return entry->hdr->cedt.length; + } + return 0; +} + +static unsigned long __init_or_acpilib +acpi_get_subtable_header_length(struct acpi_subtable_entry *entry) +{ + switch (entry->type) { + case ACPI_SUBTABLE_COMMON: + return sizeof(entry->hdr->common); + case ACPI_SUBTABLE_HMAT: + return sizeof(entry->hdr->hmat); + case ACPI_SUBTABLE_PRMT: + return sizeof(entry->hdr->prmt); + case ACPI_SUBTABLE_CEDT: + return sizeof(entry->hdr->cedt); + } + return 0; +} + +static enum acpi_subtable_type __init_or_acpilib +acpi_get_subtable_type(char *id) +{ + if (strncmp(id, ACPI_SIG_HMAT, 4) == 0) + return ACPI_SUBTABLE_HMAT; + if (strncmp(id, ACPI_SIG_PRMT, 4) == 0) + return ACPI_SUBTABLE_PRMT; + if (strncmp(id, ACPI_SIG_CEDT, 4) == 0) + return ACPI_SUBTABLE_CEDT; + return ACPI_SUBTABLE_COMMON; +} + +static __init_or_acpilib bool has_handler(struct acpi_subtable_proc *proc) +{ + return proc->handler || proc->handler_arg; +} + +static __init_or_acpilib int call_handler(struct acpi_subtable_proc *proc, + union acpi_subtable_headers *hdr, + unsigned long end) +{ + if (proc->handler) + return proc->handler(hdr, end); + if (proc->handler_arg) + return proc->handler_arg(hdr, proc->arg, end); + return -EINVAL; +} + +/** + * acpi_parse_entries_array - for each proc_num find a suitable subtable + * + * @id: table id (for debugging purposes) + * @table_size: size of the root table + * @table_header: where does the table start? + * @proc: array of acpi_subtable_proc struct containing entry id + * and associated handler with it + * @proc_num: how big proc is? + * @max_entries: how many entries can we process? + * + * For each proc_num find a subtable with proc->id and run proc->handler + * on it. Assumption is that there's only single handler for particular + * entry id. + * + * The table_size is not the size of the complete ACPI table (the length + * field in the header struct), but only the size of the root table; i.e., + * the offset from the very first byte of the complete ACPI table, to the + * first byte of the very first subtable. + * + * On success returns sum of all matching entries for all proc handlers. + * Otherwise, -ENODEV or -EINVAL is returned. + */ +int __init_or_acpilib +acpi_parse_entries_array(char *id, unsigned long table_size, + struct acpi_table_header *table_header, + struct acpi_subtable_proc *proc, + int proc_num, unsigned int max_entries) +{ + unsigned long table_end, subtable_len, entry_len; + struct acpi_subtable_entry entry; + int count = 0; + int errs = 0; + int i; + + table_end = (unsigned long)table_header + table_header->length; + + /* Parse all entries looking for a match. */ + + entry.type = acpi_get_subtable_type(id); + entry.hdr = (union acpi_subtable_headers *) + ((unsigned long)table_header + table_size); + subtable_len = acpi_get_subtable_header_length(&entry); + + while (((unsigned long)entry.hdr) + subtable_len < table_end) { + if (max_entries && count >= max_entries) + break; + + for (i = 0; i < proc_num; i++) { + if (acpi_get_entry_type(&entry) != proc[i].id) + continue; + if (!has_handler(&proc[i]) || + (!errs && + call_handler(&proc[i], entry.hdr, table_end))) { + errs++; + continue; + } + + proc[i].count++; + break; + } + if (i != proc_num) + count++; + + /* + * If entry->length is 0, break from this loop to avoid + * infinite loop. + */ + entry_len = acpi_get_entry_length(&entry); + if (entry_len == 0) { + pr_err("[%4.4s:0x%02x] Invalid zero length\n", id, proc->id); + return -EINVAL; + } + + entry.hdr = (union acpi_subtable_headers *) + ((unsigned long)entry.hdr + entry_len); + } + + if (max_entries && count > max_entries) { + pr_warn("[%4.4s:0x%02x] found the maximum %i entries\n", + id, proc->id, count); + } + + return errs ? -EINVAL : count; +} diff --git a/lib/genalloc.c b/lib/genalloc.c index 6c644f954bc5..4fa5635bf81b 100644 --- a/lib/genalloc.c +++ b/lib/genalloc.c @@ -32,7 +32,9 @@ #include <linux/rculist.h> #include <linux/interrupt.h> #include <linux/genalloc.h> -#include <linux/of_device.h> +#include <linux/of.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> #include <linux/vmalloc.h> static inline size_t chunk_size(const struct gen_pool_chunk *chunk) diff --git a/lib/generic-radix-tree.c b/lib/generic-radix-tree.c index f25eb111c051..41f1bcdc4488 100644 --- a/lib/generic-radix-tree.c +++ b/lib/generic-radix-tree.c @@ -1,4 +1,5 @@ +#include <linux/atomic.h> #include <linux/export.h> #include <linux/generic-radix-tree.h> #include <linux/gfp.h> @@ -166,6 +167,10 @@ void *__genradix_iter_peek(struct genradix_iter *iter, struct genradix_root *r; struct genradix_node *n; unsigned level, i; + + if (iter->offset == SIZE_MAX) + return NULL; + restart: r = READ_ONCE(radix->root); if (!r) @@ -184,10 +189,17 @@ restart: (GENRADIX_ARY - 1); while (!n->children[i]) { + size_t objs_per_ptr = genradix_depth_size(level); + + if (iter->offset + objs_per_ptr < iter->offset) { + iter->offset = SIZE_MAX; + iter->pos = SIZE_MAX; + return NULL; + } + i++; - iter->offset = round_down(iter->offset + - genradix_depth_size(level), - genradix_depth_size(level)); + iter->offset = round_down(iter->offset + objs_per_ptr, + objs_per_ptr); iter->pos = (iter->offset >> PAGE_SHIFT) * objs_per_page; if (i == GENRADIX_ARY) @@ -201,6 +213,64 @@ restart: } EXPORT_SYMBOL(__genradix_iter_peek); +void *__genradix_iter_peek_prev(struct genradix_iter *iter, + struct __genradix *radix, + size_t objs_per_page, + size_t obj_size_plus_page_remainder) +{ + struct genradix_root *r; + struct genradix_node *n; + unsigned level, i; + + if (iter->offset == SIZE_MAX) + return NULL; + +restart: + r = READ_ONCE(radix->root); + if (!r) + return NULL; + + n = genradix_root_to_node(r); + level = genradix_root_to_depth(r); + + if (ilog2(iter->offset) >= genradix_depth_shift(level)) { + iter->offset = genradix_depth_size(level); + iter->pos = (iter->offset >> PAGE_SHIFT) * objs_per_page; + + iter->offset -= obj_size_plus_page_remainder; + iter->pos--; + } + + while (level) { + level--; + + i = (iter->offset >> genradix_depth_shift(level)) & + (GENRADIX_ARY - 1); + + while (!n->children[i]) { + size_t objs_per_ptr = genradix_depth_size(level); + + iter->offset = round_down(iter->offset, objs_per_ptr); + iter->pos = (iter->offset >> PAGE_SHIFT) * objs_per_page; + + if (!iter->offset) + return NULL; + + iter->offset -= obj_size_plus_page_remainder; + iter->pos--; + + if (!i) + goto restart; + --i; + } + + n = n->children[i]; + } + + return &n->data[iter->offset & (PAGE_SIZE - 1)]; +} +EXPORT_SYMBOL(__genradix_iter_peek_prev); + static void genradix_free_recurse(struct genradix_node *n, unsigned level) { if (level) { diff --git a/lib/group_cpus.c b/lib/group_cpus.c index aa3f6815bb12..ee272c4cefcc 100644 --- a/lib/group_cpus.c +++ b/lib/group_cpus.c @@ -366,13 +366,25 @@ struct cpumask *group_cpus_evenly(unsigned int numgrps) if (!masks) goto fail_node_to_cpumask; - /* Stabilize the cpumasks */ - cpus_read_lock(); build_node_to_cpumask(node_to_cpumask); + /* + * Make a local cache of 'cpu_present_mask', so the two stages + * spread can observe consistent 'cpu_present_mask' without holding + * cpu hotplug lock, then we can reduce deadlock risk with cpu + * hotplug code. + * + * Here CPU hotplug may happen when reading `cpu_present_mask`, and + * we can live with the case because it only affects that hotplug + * CPU is handled in the 1st or 2nd stage, and either way is correct + * from API user viewpoint since 2-stage spread is sort of + * optimization. + */ + cpumask_copy(npresmsk, data_race(cpu_present_mask)); + /* grouping present CPUs first */ ret = __group_cpus_evenly(curgrp, numgrps, node_to_cpumask, - cpu_present_mask, nmsk, masks); + npresmsk, nmsk, masks); if (ret < 0) goto fail_build_affinity; nr_present = ret; @@ -387,15 +399,13 @@ struct cpumask *group_cpus_evenly(unsigned int numgrps) curgrp = 0; else curgrp = nr_present; - cpumask_andnot(npresmsk, cpu_possible_mask, cpu_present_mask); + cpumask_andnot(npresmsk, cpu_possible_mask, npresmsk); ret = __group_cpus_evenly(curgrp, numgrps, node_to_cpumask, npresmsk, nmsk, masks); if (ret >= 0) nr_others = ret; fail_build_affinity: - cpus_read_unlock(); - if (ret >= 0) WARN_ON(nr_present + nr_others < numgrps); diff --git a/lib/idr.c b/lib/idr.c index 7ecdfdb5309e..da36054c3ca0 100644 --- a/lib/idr.c +++ b/lib/idr.c @@ -100,7 +100,7 @@ EXPORT_SYMBOL_GPL(idr_alloc); * @end: The maximum ID (exclusive). * @gfp: Memory allocation flags. * - * Allocates an unused ID in the range specified by @nextid and @end. If + * Allocates an unused ID in the range specified by @start and @end. If * @end is <= 0, it is treated as one larger than %INT_MAX. This allows * callers to use @start + N as @end as long as N is within integer range. * The search for an unused ID will start at the last ID allocated and will @@ -508,7 +508,7 @@ void ida_free(struct ida *ida, unsigned int id) goto delete; xas_store(&xas, xa_mk_value(v)); } else { - if (!test_bit(bit, bitmap->bitmap)) + if (!bitmap || !test_bit(bit, bitmap->bitmap)) goto err; __clear_bit(bit, bitmap->bitmap); xas_set_mark(&xas, XA_FREE_MARK); diff --git a/lib/iov_iter.c b/lib/iov_iter.c index e4dc809d1075..8ff6824a1005 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -1,5 +1,4 @@ // SPDX-License-Identifier: GPL-2.0-only -#include <crypto/hash.h> #include <linux/export.h> #include <linux/bvec.h> #include <linux/fault-inject-usercopy.h> @@ -10,192 +9,71 @@ #include <linux/vmalloc.h> #include <linux/splice.h> #include <linux/compat.h> -#include <net/checksum.h> #include <linux/scatterlist.h> #include <linux/instrumented.h> +#include <linux/iov_iter.h> -/* covers ubuf and kbuf alike */ -#define iterate_buf(i, n, base, len, off, __p, STEP) { \ - size_t __maybe_unused off = 0; \ - len = n; \ - base = __p + i->iov_offset; \ - len -= (STEP); \ - i->iov_offset += len; \ - n = len; \ -} - -/* covers iovec and kvec alike */ -#define iterate_iovec(i, n, base, len, off, __p, STEP) { \ - size_t off = 0; \ - size_t skip = i->iov_offset; \ - do { \ - len = min(n, __p->iov_len - skip); \ - if (likely(len)) { \ - base = __p->iov_base + skip; \ - len -= (STEP); \ - off += len; \ - skip += len; \ - n -= len; \ - if (skip < __p->iov_len) \ - break; \ - } \ - __p++; \ - skip = 0; \ - } while (n); \ - i->iov_offset = skip; \ - n = off; \ -} - -#define iterate_bvec(i, n, base, len, off, p, STEP) { \ - size_t off = 0; \ - unsigned skip = i->iov_offset; \ - while (n) { \ - unsigned offset = p->bv_offset + skip; \ - unsigned left; \ - void *kaddr = kmap_local_page(p->bv_page + \ - offset / PAGE_SIZE); \ - base = kaddr + offset % PAGE_SIZE; \ - len = min(min(n, (size_t)(p->bv_len - skip)), \ - (size_t)(PAGE_SIZE - offset % PAGE_SIZE)); \ - left = (STEP); \ - kunmap_local(kaddr); \ - len -= left; \ - off += len; \ - skip += len; \ - if (skip == p->bv_len) { \ - skip = 0; \ - p++; \ - } \ - n -= len; \ - if (left) \ - break; \ - } \ - i->iov_offset = skip; \ - n = off; \ -} - -#define iterate_xarray(i, n, base, len, __off, STEP) { \ - __label__ __out; \ - size_t __off = 0; \ - struct folio *folio; \ - loff_t start = i->xarray_start + i->iov_offset; \ - pgoff_t index = start / PAGE_SIZE; \ - XA_STATE(xas, i->xarray, index); \ - \ - len = PAGE_SIZE - offset_in_page(start); \ - rcu_read_lock(); \ - xas_for_each(&xas, folio, ULONG_MAX) { \ - unsigned left; \ - size_t offset; \ - if (xas_retry(&xas, folio)) \ - continue; \ - if (WARN_ON(xa_is_value(folio))) \ - break; \ - if (WARN_ON(folio_test_hugetlb(folio))) \ - break; \ - offset = offset_in_folio(folio, start + __off); \ - while (offset < folio_size(folio)) { \ - base = kmap_local_folio(folio, offset); \ - len = min(n, len); \ - left = (STEP); \ - kunmap_local(base); \ - len -= left; \ - __off += len; \ - n -= len; \ - if (left || n == 0) \ - goto __out; \ - offset += len; \ - len = PAGE_SIZE; \ - } \ - } \ -__out: \ - rcu_read_unlock(); \ - i->iov_offset += __off; \ - n = __off; \ -} - -#define __iterate_and_advance(i, n, base, len, off, I, K) { \ - if (unlikely(i->count < n)) \ - n = i->count; \ - if (likely(n)) { \ - if (likely(iter_is_ubuf(i))) { \ - void __user *base; \ - size_t len; \ - iterate_buf(i, n, base, len, off, \ - i->ubuf, (I)) \ - } else if (likely(iter_is_iovec(i))) { \ - const struct iovec *iov = iter_iov(i); \ - void __user *base; \ - size_t len; \ - iterate_iovec(i, n, base, len, off, \ - iov, (I)) \ - i->nr_segs -= iov - iter_iov(i); \ - i->__iov = iov; \ - } else if (iov_iter_is_bvec(i)) { \ - const struct bio_vec *bvec = i->bvec; \ - void *base; \ - size_t len; \ - iterate_bvec(i, n, base, len, off, \ - bvec, (K)) \ - i->nr_segs -= bvec - i->bvec; \ - i->bvec = bvec; \ - } else if (iov_iter_is_kvec(i)) { \ - const struct kvec *kvec = i->kvec; \ - void *base; \ - size_t len; \ - iterate_iovec(i, n, base, len, off, \ - kvec, (K)) \ - i->nr_segs -= kvec - i->kvec; \ - i->kvec = kvec; \ - } else if (iov_iter_is_xarray(i)) { \ - void *base; \ - size_t len; \ - iterate_xarray(i, n, base, len, off, \ - (K)) \ - } \ - i->count -= n; \ - } \ -} -#define iterate_and_advance(i, n, base, len, off, I, K) \ - __iterate_and_advance(i, n, base, len, off, I, ((void)(K),0)) - -static int copyout(void __user *to, const void *from, size_t n) +static __always_inline +size_t copy_to_user_iter(void __user *iter_to, size_t progress, + size_t len, void *from, void *priv2) { if (should_fail_usercopy()) - return n; - if (access_ok(to, n)) { - instrument_copy_to_user(to, from, n); - n = raw_copy_to_user(to, from, n); + return len; + if (access_ok(iter_to, len)) { + from += progress; + instrument_copy_to_user(iter_to, from, len); + len = raw_copy_to_user(iter_to, from, len); } - return n; + return len; } -static int copyout_nofault(void __user *to, const void *from, size_t n) +static __always_inline +size_t copy_to_user_iter_nofault(void __user *iter_to, size_t progress, + size_t len, void *from, void *priv2) { - long res; + ssize_t res; if (should_fail_usercopy()) - return n; - - res = copy_to_user_nofault(to, from, n); + return len; - return res < 0 ? n : res; + from += progress; + res = copy_to_user_nofault(iter_to, from, len); + return res < 0 ? len : res; } -static int copyin(void *to, const void __user *from, size_t n) +static __always_inline +size_t copy_from_user_iter(void __user *iter_from, size_t progress, + size_t len, void *to, void *priv2) { - size_t res = n; + size_t res = len; if (should_fail_usercopy()) - return n; - if (access_ok(from, n)) { - instrument_copy_from_user_before(to, from, n); - res = raw_copy_from_user(to, from, n); - instrument_copy_from_user_after(to, from, n, res); + return len; + if (access_ok(iter_from, len)) { + to += progress; + instrument_copy_from_user_before(to, iter_from, len); + res = raw_copy_from_user(to, iter_from, len); + instrument_copy_from_user_after(to, iter_from, len, res); } return res; } +static __always_inline +size_t memcpy_to_iter(void *iter_to, size_t progress, + size_t len, void *from, void *priv2) +{ + memcpy(iter_to, from + progress, len); + return 0; +} + +static __always_inline +size_t memcpy_from_iter(void *iter_from, size_t progress, + size_t len, void *to, void *priv2) +{ + memcpy(to + progress, iter_from, len); + return 0; +} + /* * fault_in_iov_iter_readable - fault in iov iterator for reading * @i: iterator @@ -290,7 +168,6 @@ void iov_iter_init(struct iov_iter *i, unsigned int direction, .iter_type = ITER_IOVEC, .copy_mc = false, .nofault = false, - .user_backed = true, .data_source = direction, .__iov = iov, .nr_segs = nr_segs, @@ -300,36 +177,35 @@ void iov_iter_init(struct iov_iter *i, unsigned int direction, } EXPORT_SYMBOL(iov_iter_init); -static __wsum csum_and_memcpy(void *to, const void *from, size_t len, - __wsum sum, size_t off) -{ - __wsum next = csum_partial_copy_nocheck(from, to, len); - return csum_block_add(sum, next, off); -} - size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i) { if (WARN_ON_ONCE(i->data_source)) return 0; if (user_backed_iter(i)) might_fault(); - iterate_and_advance(i, bytes, base, len, off, - copyout(base, addr + off, len), - memcpy(base, addr + off, len) - ) - - return bytes; + return iterate_and_advance(i, bytes, (void *)addr, + copy_to_user_iter, memcpy_to_iter); } EXPORT_SYMBOL(_copy_to_iter); #ifdef CONFIG_ARCH_HAS_COPY_MC -static int copyout_mc(void __user *to, const void *from, size_t n) -{ - if (access_ok(to, n)) { - instrument_copy_to_user(to, from, n); - n = copy_mc_to_user((__force void *) to, from, n); +static __always_inline +size_t copy_to_user_iter_mc(void __user *iter_to, size_t progress, + size_t len, void *from, void *priv2) +{ + if (access_ok(iter_to, len)) { + from += progress; + instrument_copy_to_user(iter_to, from, len); + len = copy_mc_to_user(iter_to, from, len); } - return n; + return len; +} + +static __always_inline +size_t memcpy_to_iter_mc(void *iter_to, size_t progress, + size_t len, void *from, void *priv2) +{ + return copy_mc_to_kernel(iter_to, from + progress, len); } /** @@ -362,22 +238,35 @@ size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i) return 0; if (user_backed_iter(i)) might_fault(); - __iterate_and_advance(i, bytes, base, len, off, - copyout_mc(base, addr + off, len), - copy_mc_to_kernel(base, addr + off, len) - ) - - return bytes; + return iterate_and_advance(i, bytes, (void *)addr, + copy_to_user_iter_mc, memcpy_to_iter_mc); } EXPORT_SYMBOL_GPL(_copy_mc_to_iter); #endif /* CONFIG_ARCH_HAS_COPY_MC */ -static void *memcpy_from_iter(struct iov_iter *i, void *to, const void *from, - size_t size) +static __always_inline +size_t memcpy_from_iter_mc(void *iter_from, size_t progress, + size_t len, void *to, void *priv2) { - if (iov_iter_is_copy_mc(i)) - return (void *)copy_mc_to_kernel(to, from, size); - return memcpy(to, from, size); + return copy_mc_to_kernel(to + progress, iter_from, len); +} + +static size_t __copy_from_iter_mc(void *addr, size_t bytes, struct iov_iter *i) +{ + if (unlikely(i->count < bytes)) + bytes = i->count; + if (unlikely(!bytes)) + return 0; + return iterate_bvec(i, bytes, addr, NULL, memcpy_from_iter_mc); +} + +static __always_inline +size_t __copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) +{ + if (unlikely(iov_iter_is_copy_mc(i))) + return __copy_from_iter_mc(addr, bytes, i); + return iterate_and_advance(i, bytes, addr, + copy_from_user_iter, memcpy_from_iter); } size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) @@ -387,30 +276,44 @@ size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) if (user_backed_iter(i)) might_fault(); - iterate_and_advance(i, bytes, base, len, off, - copyin(addr + off, base, len), - memcpy_from_iter(i, addr + off, base, len) - ) - - return bytes; + return __copy_from_iter(addr, bytes, i); } EXPORT_SYMBOL(_copy_from_iter); +static __always_inline +size_t copy_from_user_iter_nocache(void __user *iter_from, size_t progress, + size_t len, void *to, void *priv2) +{ + return __copy_from_user_inatomic_nocache(to + progress, iter_from, len); +} + size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i) { if (WARN_ON_ONCE(!i->data_source)) return 0; - iterate_and_advance(i, bytes, base, len, off, - __copy_from_user_inatomic_nocache(addr + off, base, len), - memcpy(addr + off, base, len) - ) - - return bytes; + return iterate_and_advance(i, bytes, addr, + copy_from_user_iter_nocache, + memcpy_from_iter); } EXPORT_SYMBOL(_copy_from_iter_nocache); #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE +static __always_inline +size_t copy_from_user_iter_flushcache(void __user *iter_from, size_t progress, + size_t len, void *to, void *priv2) +{ + return __copy_from_user_flushcache(to + progress, iter_from, len); +} + +static __always_inline +size_t memcpy_from_iter_flushcache(void *iter_from, size_t progress, + size_t len, void *to, void *priv2) +{ + memcpy_flushcache(to + progress, iter_from, len); + return 0; +} + /** * _copy_from_iter_flushcache - write destination through cpu cache * @addr: destination kernel address @@ -432,12 +335,9 @@ size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i) if (WARN_ON_ONCE(!i->data_source)) return 0; - iterate_and_advance(i, bytes, base, len, off, - __copy_from_user_flushcache(addr + off, base, len), - memcpy_flushcache(addr + off, base, len) - ) - - return bytes; + return iterate_and_advance(i, bytes, addr, + copy_from_user_iter_flushcache, + memcpy_from_iter_flushcache); } EXPORT_SYMBOL_GPL(_copy_from_iter_flushcache); #endif @@ -509,10 +409,9 @@ size_t copy_page_to_iter_nofault(struct page *page, unsigned offset, size_t byte void *kaddr = kmap_local_page(page); size_t n = min(bytes, (size_t)PAGE_SIZE - offset); - iterate_and_advance(i, n, base, len, off, - copyout_nofault(base, kaddr + offset + off, len), - memcpy(base, kaddr + offset + off, len) - ) + n = iterate_and_advance(i, n, kaddr + offset, + copy_to_user_iter_nofault, + memcpy_to_iter); kunmap_local(kaddr); res += n; bytes -= n; @@ -555,35 +454,56 @@ size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes, } EXPORT_SYMBOL(copy_page_from_iter); -size_t iov_iter_zero(size_t bytes, struct iov_iter *i) +static __always_inline +size_t zero_to_user_iter(void __user *iter_to, size_t progress, + size_t len, void *priv, void *priv2) +{ + return clear_user(iter_to, len); +} + +static __always_inline +size_t zero_to_iter(void *iter_to, size_t progress, + size_t len, void *priv, void *priv2) { - iterate_and_advance(i, bytes, base, len, count, - clear_user(base, len), - memset(base, 0, len) - ) + memset(iter_to, 0, len); + return 0; +} - return bytes; +size_t iov_iter_zero(size_t bytes, struct iov_iter *i) +{ + return iterate_and_advance(i, bytes, NULL, + zero_to_user_iter, zero_to_iter); } EXPORT_SYMBOL(iov_iter_zero); -size_t copy_page_from_iter_atomic(struct page *page, unsigned offset, size_t bytes, - struct iov_iter *i) +size_t copy_page_from_iter_atomic(struct page *page, size_t offset, + size_t bytes, struct iov_iter *i) { - char *kaddr = kmap_atomic(page), *p = kaddr + offset; - if (!page_copy_sane(page, offset, bytes)) { - kunmap_atomic(kaddr); + size_t n, copied = 0; + + if (!page_copy_sane(page, offset, bytes)) return 0; - } - if (WARN_ON_ONCE(!i->data_source)) { - kunmap_atomic(kaddr); + if (WARN_ON_ONCE(!i->data_source)) return 0; - } - iterate_and_advance(i, bytes, base, len, off, - copyin(p + off, base, len), - memcpy_from_iter(i, p + off, base, len) - ) - kunmap_atomic(kaddr); - return bytes; + + do { + char *p; + + n = bytes - copied; + if (PageHighMem(page)) { + page += offset / PAGE_SIZE; + offset %= PAGE_SIZE; + n = min_t(size_t, n, PAGE_SIZE - offset); + } + + p = kmap_atomic(page) + offset; + n = __copy_from_iter(p, n, i); + kunmap_atomic(p); + copied += n; + offset += n; + } while (PageHighMem(page) && copied != bytes && n > 0); + + return copied; } EXPORT_SYMBOL(copy_page_from_iter_atomic); @@ -1168,78 +1088,6 @@ ssize_t iov_iter_get_pages_alloc2(struct iov_iter *i, } EXPORT_SYMBOL(iov_iter_get_pages_alloc2); -size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum, - struct iov_iter *i) -{ - __wsum sum, next; - sum = *csum; - if (WARN_ON_ONCE(!i->data_source)) - return 0; - - iterate_and_advance(i, bytes, base, len, off, ({ - next = csum_and_copy_from_user(base, addr + off, len); - sum = csum_block_add(sum, next, off); - next ? 0 : len; - }), ({ - sum = csum_and_memcpy(addr + off, base, len, sum, off); - }) - ) - *csum = sum; - return bytes; -} -EXPORT_SYMBOL(csum_and_copy_from_iter); - -size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *_csstate, - struct iov_iter *i) -{ - struct csum_state *csstate = _csstate; - __wsum sum, next; - - if (WARN_ON_ONCE(i->data_source)) - return 0; - if (unlikely(iov_iter_is_discard(i))) { - // can't use csum_memcpy() for that one - data is not copied - csstate->csum = csum_block_add(csstate->csum, - csum_partial(addr, bytes, 0), - csstate->off); - csstate->off += bytes; - return bytes; - } - - sum = csum_shift(csstate->csum, csstate->off); - iterate_and_advance(i, bytes, base, len, off, ({ - next = csum_and_copy_to_user(addr + off, base, len); - sum = csum_block_add(sum, next, off); - next ? 0 : len; - }), ({ - sum = csum_and_memcpy(base, addr + off, len, sum, off); - }) - ) - csstate->csum = csum_shift(sum, csstate->off); - csstate->off += bytes; - return bytes; -} -EXPORT_SYMBOL(csum_and_copy_to_iter); - -size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp, - struct iov_iter *i) -{ -#ifdef CONFIG_CRYPTO_HASH - struct ahash_request *hash = hashp; - struct scatterlist sg; - size_t copied; - - copied = copy_to_iter(addr, bytes, i); - sg_init_one(&sg, addr, copied); - ahash_request_set_crypt(hash, &sg, NULL, copied); - crypto_ahash_update(hash); - return copied; -#else - return 0; -#endif -} -EXPORT_SYMBOL(hash_and_copy_to_iter); - static int iov_npages(const struct iov_iter *i, int maxpages) { size_t skip = i->iov_offset, size = i->count; @@ -1544,6 +1392,7 @@ int import_ubuf(int rw, void __user *buf, size_t len, struct iov_iter *i) iov_iter_ubuf(i, rw, buf, len); return 0; } +EXPORT_SYMBOL_GPL(import_ubuf); /** * iov_iter_restore() - Restore a &struct iov_iter to the same state as when @@ -1640,14 +1489,14 @@ static ssize_t iov_iter_extract_bvec_pages(struct iov_iter *i, size_t *offset0) { struct page **p, *page; - size_t skip = i->iov_offset, offset; + size_t skip = i->iov_offset, offset, size; int k; for (;;) { if (i->nr_segs == 0) return 0; - maxsize = min(maxsize, i->bvec->bv_len - skip); - if (maxsize) + size = min(maxsize, i->bvec->bv_len - skip); + if (size) break; i->iov_offset = 0; i->nr_segs--; @@ -1660,16 +1509,16 @@ static ssize_t iov_iter_extract_bvec_pages(struct iov_iter *i, offset = skip % PAGE_SIZE; *offset0 = offset; - maxpages = want_pages_array(pages, maxsize, offset, maxpages); + maxpages = want_pages_array(pages, size, offset, maxpages); if (!maxpages) return -ENOMEM; p = *pages; for (k = 0; k < maxpages; k++) p[k] = page + k; - maxsize = min_t(size_t, maxsize, maxpages * PAGE_SIZE - offset); - iov_iter_advance(i, maxsize); - return maxsize; + size = min_t(size_t, size, maxpages * PAGE_SIZE - offset); + iov_iter_advance(i, size); + return size; } /* @@ -1684,14 +1533,14 @@ static ssize_t iov_iter_extract_kvec_pages(struct iov_iter *i, { struct page **p, *page; const void *kaddr; - size_t skip = i->iov_offset, offset, len; + size_t skip = i->iov_offset, offset, len, size; int k; for (;;) { if (i->nr_segs == 0) return 0; - maxsize = min(maxsize, i->kvec->iov_len - skip); - if (maxsize) + size = min(maxsize, i->kvec->iov_len - skip); + if (size) break; i->iov_offset = 0; i->nr_segs--; @@ -1703,13 +1552,13 @@ static ssize_t iov_iter_extract_kvec_pages(struct iov_iter *i, offset = (unsigned long)kaddr & ~PAGE_MASK; *offset0 = offset; - maxpages = want_pages_array(pages, maxsize, offset, maxpages); + maxpages = want_pages_array(pages, size, offset, maxpages); if (!maxpages) return -ENOMEM; p = *pages; kaddr -= offset; - len = offset + maxsize; + len = offset + size; for (k = 0; k < maxpages; k++) { size_t seg = min_t(size_t, len, PAGE_SIZE); @@ -1723,9 +1572,9 @@ static ssize_t iov_iter_extract_kvec_pages(struct iov_iter *i, kaddr += PAGE_SIZE; } - maxsize = min_t(size_t, maxsize, maxpages * PAGE_SIZE - offset); - iov_iter_advance(i, maxsize); - return maxsize; + size = min_t(size_t, size, maxpages * PAGE_SIZE - offset); + iov_iter_advance(i, size); + return size; } /* diff --git a/lib/kobject.c b/lib/kobject.c index 16d530f9c174..59dbcbdb1c91 100644 --- a/lib/kobject.c +++ b/lib/kobject.c @@ -56,6 +56,14 @@ void kobject_get_ownership(const struct kobject *kobj, kuid_t *uid, kgid_t *gid) kobj->ktype->get_ownership(kobj, uid, gid); } +static bool kobj_ns_type_is_valid(enum kobj_ns_type type) +{ + if ((type <= KOBJ_NS_TYPE_NONE) || (type >= KOBJ_NS_TYPES)) + return false; + + return true; +} + static int create_dir(struct kobject *kobj) { const struct kobj_type *ktype = get_ktype(kobj); @@ -66,12 +74,10 @@ static int create_dir(struct kobject *kobj) if (error) return error; - if (ktype) { - error = sysfs_create_groups(kobj, ktype->default_groups); - if (error) { - sysfs_remove_dir(kobj); - return error; - } + error = sysfs_create_groups(kobj, ktype->default_groups); + if (error) { + sysfs_remove_dir(kobj); + return error; } /* @@ -86,8 +92,7 @@ static int create_dir(struct kobject *kobj) */ ops = kobj_child_ns_ops(kobj); if (ops) { - BUG_ON(ops->type <= KOBJ_NS_TYPE_NONE); - BUG_ON(ops->type >= KOBJ_NS_TYPES); + BUG_ON(!kobj_ns_type_is_valid(ops->type)); BUG_ON(!kobj_ns_type_registered(ops->type)); sysfs_enable_ns(kobj->sd); @@ -584,8 +589,7 @@ static void __kobject_del(struct kobject *kobj) sd = kobj->sd; ktype = get_ktype(kobj); - if (ktype) - sysfs_remove_groups(kobj, ktype->default_groups); + sysfs_remove_groups(kobj, ktype->default_groups); /* send "remove" if the caller did not do it but sent "add" */ if (kobj->state_add_uevent_sent && !kobj->state_remove_uevent_sent) { @@ -662,10 +666,6 @@ static void kobject_cleanup(struct kobject *kobj) pr_debug("'%s' (%p): %s, parent %p\n", kobject_name(kobj), kobj, __func__, kobj->parent); - if (t && !t->release) - pr_debug("'%s' (%p): does not have a release() function, it is broken and must be fixed. See Documentation/core-api/kobject.rst.\n", - kobject_name(kobj), kobj); - /* remove from sysfs if the caller did not do it */ if (kobj->state_in_sysfs) { pr_debug("'%s' (%p): auto cleanup kobject_del\n", @@ -676,10 +676,13 @@ static void kobject_cleanup(struct kobject *kobj) parent = NULL; } - if (t && t->release) { + if (t->release) { pr_debug("'%s' (%p): calling ktype release\n", kobject_name(kobj), kobj); t->release(kobj); + } else { + pr_debug("'%s' (%p): does not have a release() function, it is broken and must be fixed. See Documentation/core-api/kobject.rst.\n", + kobject_name(kobj), kobj); } /* free name if we allocated it */ @@ -854,6 +857,11 @@ int kset_register(struct kset *k) if (!k) return -EINVAL; + if (!k->kobj.ktype) { + pr_err("must have a ktype to be initialized properly!\n"); + return -EINVAL; + } + kset_init(k); err = kobject_add_internal(&k->kobj); if (err) { @@ -1017,11 +1025,7 @@ int kobj_ns_type_register(const struct kobj_ns_type_operations *ops) spin_lock(&kobj_ns_type_lock); error = -EINVAL; - if (type >= KOBJ_NS_TYPES) - goto out; - - error = -EINVAL; - if (type <= KOBJ_NS_TYPE_NONE) + if (!kobj_ns_type_is_valid(type)) goto out; error = -EBUSY; @@ -1041,7 +1045,7 @@ int kobj_ns_type_registered(enum kobj_ns_type type) int registered = 0; spin_lock(&kobj_ns_type_lock); - if ((type > KOBJ_NS_TYPE_NONE) && (type < KOBJ_NS_TYPES)) + if (kobj_ns_type_is_valid(type)) registered = kobj_ns_ops_tbl[type] != NULL; spin_unlock(&kobj_ns_type_lock); @@ -1052,7 +1056,7 @@ const struct kobj_ns_type_operations *kobj_child_ns_ops(const struct kobject *pa { const struct kobj_ns_type_operations *ops = NULL; - if (parent && parent->ktype && parent->ktype->child_ns_type) + if (parent && parent->ktype->child_ns_type) ops = parent->ktype->child_ns_type(parent); return ops; @@ -1068,8 +1072,7 @@ bool kobj_ns_current_may_mount(enum kobj_ns_type type) bool may_mount = true; spin_lock(&kobj_ns_type_lock); - if ((type > KOBJ_NS_TYPE_NONE) && (type < KOBJ_NS_TYPES) && - kobj_ns_ops_tbl[type]) + if (kobj_ns_type_is_valid(type) && kobj_ns_ops_tbl[type]) may_mount = kobj_ns_ops_tbl[type]->current_may_mount(); spin_unlock(&kobj_ns_type_lock); @@ -1081,8 +1084,7 @@ void *kobj_ns_grab_current(enum kobj_ns_type type) void *ns = NULL; spin_lock(&kobj_ns_type_lock); - if ((type > KOBJ_NS_TYPE_NONE) && (type < KOBJ_NS_TYPES) && - kobj_ns_ops_tbl[type]) + if (kobj_ns_type_is_valid(type) && kobj_ns_ops_tbl[type]) ns = kobj_ns_ops_tbl[type]->grab_current_ns(); spin_unlock(&kobj_ns_type_lock); @@ -1095,8 +1097,7 @@ const void *kobj_ns_netlink(enum kobj_ns_type type, struct sock *sk) const void *ns = NULL; spin_lock(&kobj_ns_type_lock); - if ((type > KOBJ_NS_TYPE_NONE) && (type < KOBJ_NS_TYPES) && - kobj_ns_ops_tbl[type]) + if (kobj_ns_type_is_valid(type) && kobj_ns_ops_tbl[type]) ns = kobj_ns_ops_tbl[type]->netlink_ns(sk); spin_unlock(&kobj_ns_type_lock); @@ -1108,8 +1109,7 @@ const void *kobj_ns_initial(enum kobj_ns_type type) const void *ns = NULL; spin_lock(&kobj_ns_type_lock); - if ((type > KOBJ_NS_TYPE_NONE) && (type < KOBJ_NS_TYPES) && - kobj_ns_ops_tbl[type]) + if (kobj_ns_type_is_valid(type) && kobj_ns_ops_tbl[type]) ns = kobj_ns_ops_tbl[type]->initial_ns(); spin_unlock(&kobj_ns_type_lock); @@ -1119,7 +1119,7 @@ const void *kobj_ns_initial(enum kobj_ns_type type) void kobj_ns_drop(enum kobj_ns_type type, void *ns) { spin_lock(&kobj_ns_type_lock); - if ((type > KOBJ_NS_TYPE_NONE) && (type < KOBJ_NS_TYPES) && + if (kobj_ns_type_is_valid(type) && kobj_ns_ops_tbl[type] && kobj_ns_ops_tbl[type]->drop_ns) kobj_ns_ops_tbl[type]->drop_ns(ns); spin_unlock(&kobj_ns_type_lock); diff --git a/lib/kobject_uevent.c b/lib/kobject_uevent.c index 7c44b7ae4c5c..fb9a2f06dd1e 100644 --- a/lib/kobject_uevent.c +++ b/lib/kobject_uevent.c @@ -254,10 +254,10 @@ static int init_uevent_argv(struct kobj_uevent_env *env, const char *subsystem) int buffer_size = sizeof(env->buf) - env->buflen; int len; - len = strlcpy(&env->buf[env->buflen], subsystem, buffer_size); - if (len >= buffer_size) { - pr_warn("init_uevent_argv: buffer size of %d too small, needed %d\n", - buffer_size, len); + len = strscpy(&env->buf[env->buflen], subsystem, buffer_size); + if (len < 0) { + pr_warn("%s: insufficient buffer space (%u left) for %s\n", + __func__, buffer_size, subsystem); return -ENOMEM; } diff --git a/lib/kstrtox.c b/lib/kstrtox.c index 08c14019841a..d586e6af5e5a 100644 --- a/lib/kstrtox.c +++ b/lib/kstrtox.c @@ -59,7 +59,7 @@ unsigned int _parse_integer_limit(const char *s, unsigned int base, unsigned lon rv = 0; while (max_chars--) { unsigned int c = *s; - unsigned int lc = c | 0x20; /* don't tolower() this line */ + unsigned int lc = _tolower(c); unsigned int val; if ('0' <= c && c <= '9') diff --git a/lib/kunit/Kconfig b/lib/kunit/Kconfig index 626719b95bad..68a6daec0aef 100644 --- a/lib/kunit/Kconfig +++ b/lib/kunit/Kconfig @@ -4,7 +4,7 @@ menuconfig KUNIT tristate "KUnit - Enable support for unit tests" - select GLOB if KUNIT=y + select GLOB help Enables support for kernel unit tests (KUnit), a lightweight unit testing and mocking framework for the Linux kernel. These tests are diff --git a/lib/kunit/Makefile b/lib/kunit/Makefile index cb417f504996..46f75f23dfe4 100644 --- a/lib/kunit/Makefile +++ b/lib/kunit/Makefile @@ -6,7 +6,8 @@ kunit-objs += test.o \ string-stream.o \ assert.o \ try-catch.o \ - executor.o + executor.o \ + attributes.o ifeq ($(CONFIG_KUNIT_DEBUGFS),y) kunit-objs += debugfs.o diff --git a/lib/kunit/assert.c b/lib/kunit/assert.c index 05a09652f5a1..dd1d633d0fe2 100644 --- a/lib/kunit/assert.c +++ b/lib/kunit/assert.c @@ -89,8 +89,7 @@ void kunit_ptr_not_err_assert_format(const struct kunit_assert *assert, EXPORT_SYMBOL_GPL(kunit_ptr_not_err_assert_format); /* Checks if `text` is a literal representing `value`, e.g. "5" and 5 */ -static bool is_literal(struct kunit *test, const char *text, long long value, - gfp_t gfp) +static bool is_literal(const char *text, long long value) { char *buffer; int len; @@ -100,14 +99,15 @@ static bool is_literal(struct kunit *test, const char *text, long long value, if (strlen(text) != len) return false; - buffer = kunit_kmalloc(test, len+1, gfp); + buffer = kmalloc(len+1, GFP_KERNEL); if (!buffer) return false; snprintf(buffer, len+1, "%lld", value); ret = strncmp(buffer, text, len) == 0; - kunit_kfree(test, buffer); + kfree(buffer); + return ret; } @@ -125,14 +125,12 @@ void kunit_binary_assert_format(const struct kunit_assert *assert, binary_assert->text->left_text, binary_assert->text->operation, binary_assert->text->right_text); - if (!is_literal(stream->test, binary_assert->text->left_text, - binary_assert->left_value, stream->gfp)) + if (!is_literal(binary_assert->text->left_text, binary_assert->left_value)) string_stream_add(stream, KUNIT_SUBSUBTEST_INDENT "%s == %lld (0x%llx)\n", binary_assert->text->left_text, binary_assert->left_value, binary_assert->left_value); - if (!is_literal(stream->test, binary_assert->text->right_text, - binary_assert->right_value, stream->gfp)) + if (!is_literal(binary_assert->text->right_text, binary_assert->right_value)) string_stream_add(stream, KUNIT_SUBSUBTEST_INDENT "%s == %lld (0x%llx)", binary_assert->text->right_text, binary_assert->right_value, diff --git a/lib/kunit/attributes.c b/lib/kunit/attributes.c new file mode 100644 index 000000000000..1b512f7e1838 --- /dev/null +++ b/lib/kunit/attributes.c @@ -0,0 +1,414 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit API to save and access test attributes + * + * Copyright (C) 2023, Google LLC. + * Author: Rae Moar <rmoar@google.com> + */ + +#include <kunit/test.h> +#include <kunit/attributes.h> + +/* Options for printing attributes: + * PRINT_ALWAYS - attribute is printed for every test case and suite if set + * PRINT_SUITE - attribute is printed for every suite if set but not for test cases + * PRINT_NEVER - attribute is never printed + */ +enum print_ops { + PRINT_ALWAYS, + PRINT_SUITE, + PRINT_NEVER, +}; + +/** + * struct kunit_attr - represents a test attribute and holds flexible + * helper functions to interact with attribute. + * + * @name: name of test attribute, eg. speed + * @get_attr: function to return attribute value given a test + * @to_string: function to return string representation of given + * attribute value + * @filter: function to indicate whether a given attribute value passes a + * filter + * @attr_default: default attribute value used during filtering + * @print: value of enum print_ops to indicate when to print attribute + */ +struct kunit_attr { + const char *name; + void *(*get_attr)(void *test_or_suite, bool is_test); + const char *(*to_string)(void *attr, bool *to_free); + int (*filter)(void *attr, const char *input, int *err); + void *attr_default; + enum print_ops print; +}; + +/* String Lists for enum Attributes */ + +static const char * const speed_str_list[] = {"unset", "very_slow", "slow", "normal"}; + +/* To String Methods */ + +static const char *attr_enum_to_string(void *attr, const char * const str_list[], bool *to_free) +{ + long val = (long)attr; + + *to_free = false; + if (!val) + return NULL; + return str_list[val]; +} + +static const char *attr_speed_to_string(void *attr, bool *to_free) +{ + return attr_enum_to_string(attr, speed_str_list, to_free); +} + +static const char *attr_string_to_string(void *attr, bool *to_free) +{ + *to_free = false; + return (char *) attr; +} + +/* Filter Methods */ + +static const char op_list[] = "<>!="; + +/* + * Returns whether the inputted integer value matches the filter given + * by the operation string and inputted integer. + */ +static int int_filter(long val, const char *op, int input, int *err) +{ + if (!strncmp(op, "<=", 2)) + return (val <= input); + else if (!strncmp(op, ">=", 2)) + return (val >= input); + else if (!strncmp(op, "!=", 2)) + return (val != input); + else if (!strncmp(op, ">", 1)) + return (val > input); + else if (!strncmp(op, "<", 1)) + return (val < input); + else if (!strncmp(op, "=", 1)) + return (val == input); + *err = -EINVAL; + pr_err("kunit executor: invalid filter operation: %s\n", op); + return false; +} + +/* + * Returns whether the inputted enum value "attr" matches the filter given + * by the input string. Note: the str_list includes the corresponding string + * list to the enum values. + */ +static int attr_enum_filter(void *attr, const char *input, int *err, + const char * const str_list[], int max) +{ + int i, j, input_int = -1; + long test_val = (long)attr; + const char *input_val = NULL; + + for (i = 0; input[i]; i++) { + if (!strchr(op_list, input[i])) { + input_val = input + i; + break; + } + } + + if (!input_val) { + *err = -EINVAL; + pr_err("kunit executor: filter value not found: %s\n", input); + return false; + } + + for (j = 0; j <= max; j++) { + if (!strcmp(input_val, str_list[j])) + input_int = j; + } + + if (input_int < 0) { + *err = -EINVAL; + pr_err("kunit executor: invalid filter input: %s\n", input); + return false; + } + + return int_filter(test_val, input, input_int, err); +} + +static int attr_speed_filter(void *attr, const char *input, int *err) +{ + return attr_enum_filter(attr, input, err, speed_str_list, KUNIT_SPEED_MAX); +} + +/* + * Returns whether the inputted string value (attr) matches the filter given + * by the input string. + */ +static int attr_string_filter(void *attr, const char *input, int *err) +{ + char *str = attr; + + if (!strncmp(input, "<", 1)) { + *err = -EINVAL; + pr_err("kunit executor: invalid filter input: %s\n", input); + return false; + } else if (!strncmp(input, ">", 1)) { + *err = -EINVAL; + pr_err("kunit executor: invalid filter input: %s\n", input); + return false; + } else if (!strncmp(input, "!=", 2)) { + return (strcmp(input + 2, str) != 0); + } else if (!strncmp(input, "=", 1)) { + return (strcmp(input + 1, str) == 0); + } + *err = -EINVAL; + pr_err("kunit executor: invalid filter operation: %s\n", input); + return false; +} + + +/* Get Attribute Methods */ + +static void *attr_speed_get(void *test_or_suite, bool is_test) +{ + struct kunit_suite *suite = is_test ? NULL : test_or_suite; + struct kunit_case *test = is_test ? test_or_suite : NULL; + + if (test) + return ((void *) test->attr.speed); + else + return ((void *) suite->attr.speed); +} + +static void *attr_module_get(void *test_or_suite, bool is_test) +{ + struct kunit_suite *suite = is_test ? NULL : test_or_suite; + struct kunit_case *test = is_test ? test_or_suite : NULL; + + // Suites get their module attribute from their first test_case + if (test) + return ((void *) test->module_name); + else if (kunit_suite_num_test_cases(suite) > 0) + return ((void *) suite->test_cases[0].module_name); + else + return (void *) ""; +} + +/* List of all Test Attributes */ + +static struct kunit_attr kunit_attr_list[] = { + { + .name = "speed", + .get_attr = attr_speed_get, + .to_string = attr_speed_to_string, + .filter = attr_speed_filter, + .attr_default = (void *)KUNIT_SPEED_NORMAL, + .print = PRINT_ALWAYS, + }, + { + .name = "module", + .get_attr = attr_module_get, + .to_string = attr_string_to_string, + .filter = attr_string_filter, + .attr_default = (void *)"", + .print = PRINT_SUITE, + } +}; + +/* Helper Functions to Access Attributes */ + +const char *kunit_attr_filter_name(struct kunit_attr_filter filter) +{ + return filter.attr->name; +} + +void kunit_print_attr(void *test_or_suite, bool is_test, unsigned int test_level) +{ + int i; + bool to_free = false; + void *attr; + const char *attr_name, *attr_str; + struct kunit_suite *suite = is_test ? NULL : test_or_suite; + struct kunit_case *test = is_test ? test_or_suite : NULL; + + for (i = 0; i < ARRAY_SIZE(kunit_attr_list); i++) { + if (kunit_attr_list[i].print == PRINT_NEVER || + (test && kunit_attr_list[i].print == PRINT_SUITE)) + continue; + attr = kunit_attr_list[i].get_attr(test_or_suite, is_test); + if (attr) { + attr_name = kunit_attr_list[i].name; + attr_str = kunit_attr_list[i].to_string(attr, &to_free); + if (test) { + kunit_log(KERN_INFO, test, "%*s# %s.%s: %s", + KUNIT_INDENT_LEN * test_level, "", test->name, + attr_name, attr_str); + } else { + kunit_log(KERN_INFO, suite, "%*s# %s: %s", + KUNIT_INDENT_LEN * test_level, "", attr_name, attr_str); + } + + /* Free to_string of attribute if needed */ + if (to_free) + kfree(attr_str); + } + } +} + +/* Helper Functions to Filter Attributes */ + +int kunit_get_filter_count(char *input) +{ + int i, comma_index = 0, count = 0; + + for (i = 0; input[i]; i++) { + if (input[i] == ',') { + if ((i - comma_index) > 1) + count++; + comma_index = i; + } + } + if ((i - comma_index) > 0) + count++; + return count; +} + +struct kunit_attr_filter kunit_next_attr_filter(char **filters, int *err) +{ + struct kunit_attr_filter filter = {}; + int i, j, comma_index = 0, new_start_index = 0; + int op_index = -1, attr_index = -1; + char op; + char *input = *filters; + + /* Parse input until operation */ + for (i = 0; input[i]; i++) { + if (op_index < 0 && strchr(op_list, input[i])) { + op_index = i; + } else if (!comma_index && input[i] == ',') { + comma_index = i; + } else if (comma_index && input[i] != ' ') { + new_start_index = i; + break; + } + } + + if (op_index <= 0) { + *err = -EINVAL; + pr_err("kunit executor: filter operation not found: %s\n", input); + return filter; + } + + /* Temporarily set operator to \0 character. */ + op = input[op_index]; + input[op_index] = '\0'; + + /* Find associated kunit_attr object */ + for (j = 0; j < ARRAY_SIZE(kunit_attr_list); j++) { + if (!strcmp(input, kunit_attr_list[j].name)) { + attr_index = j; + break; + } + } + + input[op_index] = op; + + if (attr_index < 0) { + *err = -EINVAL; + pr_err("kunit executor: attribute not found: %s\n", input); + } else { + filter.attr = &kunit_attr_list[attr_index]; + } + + if (comma_index > 0) { + input[comma_index] = '\0'; + filter.input = input + op_index; + input = input + new_start_index; + } else { + filter.input = input + op_index; + input = NULL; + } + + *filters = input; + + return filter; +} + +struct kunit_suite *kunit_filter_attr_tests(const struct kunit_suite *const suite, + struct kunit_attr_filter filter, char *action, int *err) +{ + int n = 0; + struct kunit_case *filtered, *test_case; + struct kunit_suite *copy; + void *suite_val, *test_val; + bool suite_result, test_result, default_result, result; + + /* Allocate memory for new copy of suite and list of test cases */ + copy = kmemdup(suite, sizeof(*copy), GFP_KERNEL); + if (!copy) + return ERR_PTR(-ENOMEM); + + kunit_suite_for_each_test_case(suite, test_case) { n++; } + + filtered = kcalloc(n + 1, sizeof(*filtered), GFP_KERNEL); + if (!filtered) { + kfree(copy); + return ERR_PTR(-ENOMEM); + } + + n = 0; + + /* Save filtering result on default value */ + default_result = filter.attr->filter(filter.attr->attr_default, filter.input, err); + if (*err) + goto err; + + /* Save suite attribute value and filtering result on that value */ + suite_val = filter.attr->get_attr((void *)suite, false); + suite_result = filter.attr->filter(suite_val, filter.input, err); + if (*err) + goto err; + + /* For each test case, save test case if passes filtering. */ + kunit_suite_for_each_test_case(suite, test_case) { + test_val = filter.attr->get_attr((void *) test_case, true); + test_result = filter.attr->filter(filter.attr->get_attr(test_case, true), + filter.input, err); + if (*err) + goto err; + + /* + * If attribute value of test case is set, filter on that value. + * If not, filter on suite value if set. If not, filter on + * default value. + */ + result = false; + if (test_val) { + if (test_result) + result = true; + } else if (suite_val) { + if (suite_result) + result = true; + } else if (default_result) { + result = true; + } + + if (result) { + filtered[n++] = *test_case; + } else if (action && strcmp(action, "skip") == 0) { + test_case->status = KUNIT_SKIPPED; + filtered[n++] = *test_case; + } + } + +err: + if (n == 0 || *err) { + kfree(copy); + kfree(filtered); + return NULL; + } + + copy->test_cases = filtered; + + return copy; +} diff --git a/lib/kunit/debugfs.c b/lib/kunit/debugfs.c index 22c5c496a68f..270d185737e6 100644 --- a/lib/kunit/debugfs.c +++ b/lib/kunit/debugfs.c @@ -37,14 +37,21 @@ void kunit_debugfs_init(void) debugfs_rootdir = debugfs_create_dir(KUNIT_DEBUGFS_ROOT, NULL); } -static void debugfs_print_result(struct seq_file *seq, - struct kunit_suite *suite, - struct kunit_case *test_case) +static void debugfs_print_result(struct seq_file *seq, struct string_stream *log) { - if (!test_case || !test_case->log) + struct string_stream_fragment *frag_container; + + if (!log) return; - seq_printf(seq, "%s", test_case->log); + /* + * Walk the fragments so we don't need to allocate a temporary + * buffer to hold the entire string. + */ + spin_lock(&log->lock); + list_for_each_entry(frag_container, &log->fragments, node) + seq_printf(seq, "%s", frag_container->fragment); + spin_unlock(&log->lock); } /* @@ -69,10 +76,9 @@ static int debugfs_print_results(struct seq_file *seq, void *v) seq_printf(seq, KUNIT_SUBTEST_INDENT "1..%zd\n", kunit_suite_num_test_cases(suite)); kunit_suite_for_each_test_case(suite, test_case) - debugfs_print_result(seq, suite, test_case); + debugfs_print_result(seq, test_case->log); - if (suite->log) - seq_printf(seq, "%s", suite->log); + debugfs_print_result(seq, suite->log); seq_printf(seq, "%s %d %s\n", kunit_status_to_ok_not_ok(success), 1, suite->name); @@ -105,9 +111,13 @@ void kunit_debugfs_create_suite(struct kunit_suite *suite) struct kunit_case *test_case; /* Allocate logs before creating debugfs representation. */ - suite->log = kzalloc(KUNIT_LOG_SIZE, GFP_KERNEL); - kunit_suite_for_each_test_case(suite, test_case) - test_case->log = kzalloc(KUNIT_LOG_SIZE, GFP_KERNEL); + suite->log = alloc_string_stream(GFP_KERNEL); + string_stream_set_append_newlines(suite->log, true); + + kunit_suite_for_each_test_case(suite, test_case) { + test_case->log = alloc_string_stream(GFP_KERNEL); + string_stream_set_append_newlines(test_case->log, true); + } suite->debugfs = debugfs_create_dir(suite->name, debugfs_rootdir); @@ -121,7 +131,7 @@ void kunit_debugfs_destroy_suite(struct kunit_suite *suite) struct kunit_case *test_case; debugfs_remove_recursive(suite->debugfs); - kfree(suite->log); + string_stream_destroy(suite->log); kunit_suite_for_each_test_case(suite, test_case) - kfree(test_case->log); + string_stream_destroy(test_case->log); } diff --git a/lib/kunit/executor.c b/lib/kunit/executor.c index 74982b83707c..1236b3cd2fbb 100644 --- a/lib/kunit/executor.c +++ b/lib/kunit/executor.c @@ -2,6 +2,7 @@ #include <linux/reboot.h> #include <kunit/test.h> +#include <kunit/attributes.h> #include <linux/glob.h> #include <linux/moduleparam.h> @@ -12,28 +13,59 @@ extern struct kunit_suite * const __kunit_suites_start[]; extern struct kunit_suite * const __kunit_suites_end[]; -#if IS_BUILTIN(CONFIG_KUNIT) - -static char *filter_glob_param; static char *action_param; -module_param_named(filter_glob, filter_glob_param, charp, 0); -MODULE_PARM_DESC(filter_glob, - "Filter which KUnit test suites/tests run at boot-time, e.g. list* or list*.*del_test"); -module_param_named(action, action_param, charp, 0); +module_param_named(action, action_param, charp, 0400); MODULE_PARM_DESC(action, "Changes KUnit executor behavior, valid values are:\n" "<none>: run the tests like normal\n" - "'list' to list test names instead of running them.\n"); + "'list' to list test names instead of running them.\n" + "'list_attr' to list test names and attributes instead of running them.\n"); + +const char *kunit_action(void) +{ + return action_param; +} + +static char *filter_glob_param; +static char *filter_param; +static char *filter_action_param; + +module_param_named(filter_glob, filter_glob_param, charp, 0400); +MODULE_PARM_DESC(filter_glob, + "Filter which KUnit test suites/tests run at boot-time, e.g. list* or list*.*del_test"); +module_param_named(filter, filter_param, charp, 0400); +MODULE_PARM_DESC(filter, + "Filter which KUnit test suites/tests run at boot-time using attributes, e.g. speed>slow"); +module_param_named(filter_action, filter_action_param, charp, 0400); +MODULE_PARM_DESC(filter_action, + "Changes behavior of filtered tests using attributes, valid values are:\n" + "<none>: do not run filtered tests as normal\n" + "'skip': skip all filtered tests instead so tests will appear in output\n"); + +const char *kunit_filter_glob(void) +{ + return filter_glob_param; +} + +char *kunit_filter(void) +{ + return filter_param; +} + +char *kunit_filter_action(void) +{ + return filter_action_param; +} /* glob_match() needs NULL terminated strings, so we need a copy of filter_glob_param. */ -struct kunit_test_filter { +struct kunit_glob_filter { char *suite_glob; char *test_glob; }; /* Split "suite_glob.test_glob" into two. Assumes filter_glob is not empty. */ -static void kunit_parse_filter_glob(struct kunit_test_filter *parsed, +static int kunit_parse_glob_filter(struct kunit_glob_filter *parsed, const char *filter_glob) { const int len = strlen(filter_glob); @@ -41,21 +73,33 @@ static void kunit_parse_filter_glob(struct kunit_test_filter *parsed, if (!period) { parsed->suite_glob = kzalloc(len + 1, GFP_KERNEL); + if (!parsed->suite_glob) + return -ENOMEM; + parsed->test_glob = NULL; strcpy(parsed->suite_glob, filter_glob); - return; + return 0; } parsed->suite_glob = kzalloc(period - filter_glob + 1, GFP_KERNEL); + if (!parsed->suite_glob) + return -ENOMEM; + parsed->test_glob = kzalloc(len - (period - filter_glob) + 1, GFP_KERNEL); + if (!parsed->test_glob) { + kfree(parsed->suite_glob); + return -ENOMEM; + } strncpy(parsed->suite_glob, filter_glob, period - filter_glob); strncpy(parsed->test_glob, period + 1, len - (period - filter_glob)); + + return 0; } /* Create a copy of suite with only tests that match test_glob. */ static struct kunit_suite * -kunit_filter_tests(const struct kunit_suite *const suite, const char *test_glob) +kunit_filter_glob_tests(const struct kunit_suite *const suite, const char *test_glob) { int n = 0; struct kunit_case *filtered, *test_case; @@ -89,90 +133,143 @@ kunit_filter_tests(const struct kunit_suite *const suite, const char *test_glob) return copy; } -static char *kunit_shutdown; -core_param(kunit_shutdown, kunit_shutdown, charp, 0644); - -/* Stores an array of suites, end points one past the end */ -struct suite_set { - struct kunit_suite * const *start; - struct kunit_suite * const *end; -}; - -static void kunit_free_suite_set(struct suite_set suite_set) +void kunit_free_suite_set(struct kunit_suite_set suite_set) { struct kunit_suite * const *suites; - for (suites = suite_set.start; suites < suite_set.end; suites++) + for (suites = suite_set.start; suites < suite_set.end; suites++) { + kfree((*suites)->test_cases); kfree(*suites); + } kfree(suite_set.start); } -static struct suite_set kunit_filter_suites(const struct suite_set *suite_set, - const char *filter_glob, - int *err) +struct kunit_suite_set +kunit_filter_suites(const struct kunit_suite_set *suite_set, + const char *filter_glob, + char *filters, + char *filter_action, + int *err) { - int i; - struct kunit_suite **copy, *filtered_suite; - struct suite_set filtered; - struct kunit_test_filter filter; + int i, j, k; + int filter_count = 0; + struct kunit_suite **copy, **copy_start, *filtered_suite, *new_filtered_suite; + struct kunit_suite_set filtered = {NULL, NULL}; + struct kunit_glob_filter parsed_glob; + struct kunit_attr_filter *parsed_filters = NULL; + struct kunit_suite * const *suites; const size_t max = suite_set->end - suite_set->start; - copy = kmalloc_array(max, sizeof(*filtered.start), GFP_KERNEL); - filtered.start = copy; + copy = kcalloc(max, sizeof(*filtered.start), GFP_KERNEL); if (!copy) { /* won't be able to run anything, return an empty set */ - filtered.end = copy; return filtered; } + copy_start = copy; - kunit_parse_filter_glob(&filter, filter_glob); + if (filter_glob) { + *err = kunit_parse_glob_filter(&parsed_glob, filter_glob); + if (*err) + goto free_copy; + } - for (i = 0; &suite_set->start[i] != suite_set->end; i++) { - if (!glob_match(filter.suite_glob, suite_set->start[i]->name)) - continue; + /* Parse attribute filters */ + if (filters) { + filter_count = kunit_get_filter_count(filters); + parsed_filters = kcalloc(filter_count, sizeof(*parsed_filters), GFP_KERNEL); + if (!parsed_filters) { + *err = -ENOMEM; + goto free_parsed_glob; + } + for (j = 0; j < filter_count; j++) + parsed_filters[j] = kunit_next_attr_filter(&filters, err); + if (*err) + goto free_parsed_filters; + } - filtered_suite = kunit_filter_tests(suite_set->start[i], filter.test_glob); - if (IS_ERR(filtered_suite)) { - *err = PTR_ERR(filtered_suite); - return filtered; + for (i = 0; &suite_set->start[i] != suite_set->end; i++) { + filtered_suite = suite_set->start[i]; + if (filter_glob) { + if (!glob_match(parsed_glob.suite_glob, filtered_suite->name)) + continue; + filtered_suite = kunit_filter_glob_tests(filtered_suite, + parsed_glob.test_glob); + if (IS_ERR(filtered_suite)) { + *err = PTR_ERR(filtered_suite); + goto free_filtered_suite; + } } + if (filter_count > 0 && parsed_filters != NULL) { + for (k = 0; k < filter_count; k++) { + new_filtered_suite = kunit_filter_attr_tests(filtered_suite, + parsed_filters[k], filter_action, err); + + /* Free previous copy of suite */ + if (k > 0 || filter_glob) { + kfree(filtered_suite->test_cases); + kfree(filtered_suite); + } + + filtered_suite = new_filtered_suite; + + if (*err) + goto free_filtered_suite; + + if (IS_ERR(filtered_suite)) { + *err = PTR_ERR(filtered_suite); + goto free_filtered_suite; + } + if (!filtered_suite) + break; + } + } + if (!filtered_suite) continue; *copy++ = filtered_suite; } + filtered.start = copy_start; filtered.end = copy; - kfree(filter.suite_glob); - kfree(filter.test_glob); - return filtered; -} +free_filtered_suite: + if (*err) { + for (suites = copy_start; suites < copy; suites++) { + kfree((*suites)->test_cases); + kfree(*suites); + } + } -static void kunit_handle_shutdown(void) -{ - if (!kunit_shutdown) - return; +free_parsed_filters: + if (filter_count) + kfree(parsed_filters); - if (!strcmp(kunit_shutdown, "poweroff")) - kernel_power_off(); - else if (!strcmp(kunit_shutdown, "halt")) - kernel_halt(); - else if (!strcmp(kunit_shutdown, "reboot")) - kernel_restart(NULL); +free_parsed_glob: + if (filter_glob) { + kfree(parsed_glob.suite_glob); + kfree(parsed_glob.test_glob); + } +free_copy: + if (*err) + kfree(copy_start); + + return filtered; } -static void kunit_exec_run_tests(struct suite_set *suite_set) +void kunit_exec_run_tests(struct kunit_suite_set *suite_set, bool builtin) { size_t num_suites = suite_set->end - suite_set->start; - pr_info("KTAP version 1\n"); - pr_info("1..%zu\n", num_suites); + if (builtin || num_suites) { + pr_info("KTAP version 1\n"); + pr_info("1..%zu\n", num_suites); + } __kunit_test_suites_init(suite_set->start, num_suites); } -static void kunit_exec_list_tests(struct suite_set *suite_set) +void kunit_exec_list_tests(struct kunit_suite_set *suite_set, bool include_attr) { struct kunit_suite * const *suites; struct kunit_case *test_case; @@ -180,23 +277,54 @@ static void kunit_exec_list_tests(struct suite_set *suite_set) /* Hack: print a ktap header so kunit.py can find the start of KUnit output. */ pr_info("KTAP version 1\n"); - for (suites = suite_set->start; suites < suite_set->end; suites++) + for (suites = suite_set->start; suites < suite_set->end; suites++) { + /* Print suite name and suite attributes */ + pr_info("%s\n", (*suites)->name); + if (include_attr) + kunit_print_attr((void *)(*suites), false, 0); + + /* Print test case name and attributes in suite */ kunit_suite_for_each_test_case((*suites), test_case) { pr_info("%s.%s\n", (*suites)->name, test_case->name); + if (include_attr) + kunit_print_attr((void *)test_case, true, 0); } + } +} + +#if IS_BUILTIN(CONFIG_KUNIT) + +static char *kunit_shutdown; +core_param(kunit_shutdown, kunit_shutdown, charp, 0644); + +static void kunit_handle_shutdown(void) +{ + if (!kunit_shutdown) + return; + + if (!strcmp(kunit_shutdown, "poweroff")) + kernel_power_off(); + else if (!strcmp(kunit_shutdown, "halt")) + kernel_halt(); + else if (!strcmp(kunit_shutdown, "reboot")) + kernel_restart(NULL); + } int kunit_run_all_tests(void) { - struct suite_set suite_set = {__kunit_suites_start, __kunit_suites_end}; + struct kunit_suite_set suite_set = { + __kunit_suites_start, __kunit_suites_end, + }; int err = 0; if (!kunit_enabled()) { pr_info("kunit: disabled\n"); goto out; } - if (filter_glob_param) { - suite_set = kunit_filter_suites(&suite_set, filter_glob_param, &err); + if (filter_glob_param || filter_param) { + suite_set = kunit_filter_suites(&suite_set, filter_glob_param, + filter_param, filter_action_param, &err); if (err) { pr_err("kunit executor: error filtering suites: %d\n", err); goto out; @@ -204,13 +332,15 @@ int kunit_run_all_tests(void) } if (!action_param) - kunit_exec_run_tests(&suite_set); + kunit_exec_run_tests(&suite_set, true); else if (strcmp(action_param, "list") == 0) - kunit_exec_list_tests(&suite_set); + kunit_exec_list_tests(&suite_set, false); + else if (strcmp(action_param, "list_attr") == 0) + kunit_exec_list_tests(&suite_set, true); else pr_err("kunit executor: unknown action '%s'\n", action_param); - if (filter_glob_param) { /* a copy was made of each suite */ + if (filter_glob_param || filter_param) { /* a copy was made of each suite */ kunit_free_suite_set(suite_set); } diff --git a/lib/kunit/executor_test.c b/lib/kunit/executor_test.c index ce6749af374d..22d4ee86dbed 100644 --- a/lib/kunit/executor_test.c +++ b/lib/kunit/executor_test.c @@ -7,8 +7,9 @@ */ #include <kunit/test.h> +#include <kunit/attributes.h> -static void kfree_at_end(struct kunit *test, const void *to_free); +static void free_suite_set_at_end(struct kunit *test, const void *to_free); static struct kunit_suite *alloc_fake_suite(struct kunit *test, const char *suite_name, struct kunit_case *test_cases); @@ -24,15 +25,15 @@ static struct kunit_case dummy_test_cases[] = { static void parse_filter_test(struct kunit *test) { - struct kunit_test_filter filter = {NULL, NULL}; + struct kunit_glob_filter filter = {NULL, NULL}; - kunit_parse_filter_glob(&filter, "suite"); + kunit_parse_glob_filter(&filter, "suite"); KUNIT_EXPECT_STREQ(test, filter.suite_glob, "suite"); KUNIT_EXPECT_FALSE(test, filter.test_glob); kfree(filter.suite_glob); kfree(filter.test_glob); - kunit_parse_filter_glob(&filter, "suite.test"); + kunit_parse_glob_filter(&filter, "suite.test"); KUNIT_EXPECT_STREQ(test, filter.suite_glob, "suite"); KUNIT_EXPECT_STREQ(test, filter.test_glob, "test"); kfree(filter.suite_glob); @@ -42,18 +43,20 @@ static void parse_filter_test(struct kunit *test) static void filter_suites_test(struct kunit *test) { struct kunit_suite *subsuite[3] = {NULL, NULL}; - struct suite_set suite_set = {.start = subsuite, .end = &subsuite[2]}; - struct suite_set got; + struct kunit_suite_set suite_set = { + .start = subsuite, .end = &subsuite[2], + }; + struct kunit_suite_set got; int err = 0; subsuite[0] = alloc_fake_suite(test, "suite1", dummy_test_cases); subsuite[1] = alloc_fake_suite(test, "suite2", dummy_test_cases); /* Want: suite1, suite2, NULL -> suite2, NULL */ - got = kunit_filter_suites(&suite_set, "suite2", &err); + got = kunit_filter_suites(&suite_set, "suite2", NULL, NULL, &err); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, got.start); KUNIT_ASSERT_EQ(test, err, 0); - kfree_at_end(test, got.start); + free_suite_set_at_end(test, &got); /* Validate we just have suite2 */ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, got.start[0]); @@ -66,18 +69,20 @@ static void filter_suites_test(struct kunit *test) static void filter_suites_test_glob_test(struct kunit *test) { struct kunit_suite *subsuite[3] = {NULL, NULL}; - struct suite_set suite_set = {.start = subsuite, .end = &subsuite[2]}; - struct suite_set got; + struct kunit_suite_set suite_set = { + .start = subsuite, .end = &subsuite[2], + }; + struct kunit_suite_set got; int err = 0; subsuite[0] = alloc_fake_suite(test, "suite1", dummy_test_cases); subsuite[1] = alloc_fake_suite(test, "suite2", dummy_test_cases); /* Want: suite1, suite2, NULL -> suite2 (just test1), NULL */ - got = kunit_filter_suites(&suite_set, "suite2.test2", &err); + got = kunit_filter_suites(&suite_set, "suite2.test2", NULL, NULL, &err); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, got.start); KUNIT_ASSERT_EQ(test, err, 0); - kfree_at_end(test, got.start); + free_suite_set_at_end(test, &got); /* Validate we just have suite2 */ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, got.start[0]); @@ -93,26 +98,152 @@ static void filter_suites_test_glob_test(struct kunit *test) static void filter_suites_to_empty_test(struct kunit *test) { struct kunit_suite *subsuite[3] = {NULL, NULL}; - struct suite_set suite_set = {.start = subsuite, .end = &subsuite[2]}; - struct suite_set got; + struct kunit_suite_set suite_set = { + .start = subsuite, .end = &subsuite[2], + }; + struct kunit_suite_set got; int err = 0; subsuite[0] = alloc_fake_suite(test, "suite1", dummy_test_cases); subsuite[1] = alloc_fake_suite(test, "suite2", dummy_test_cases); - got = kunit_filter_suites(&suite_set, "not_found", &err); + got = kunit_filter_suites(&suite_set, "not_found", NULL, NULL, &err); KUNIT_ASSERT_EQ(test, err, 0); - kfree_at_end(test, got.start); /* just in case */ + free_suite_set_at_end(test, &got); /* just in case */ KUNIT_EXPECT_PTR_EQ_MSG(test, got.start, got.end, "should be empty to indicate no match"); } +static void parse_filter_attr_test(struct kunit *test) +{ + int j, filter_count; + struct kunit_attr_filter *parsed_filters; + char filters[] = "speed>slow, module!=example", *filter = filters; + int err = 0; + + filter_count = kunit_get_filter_count(filters); + KUNIT_EXPECT_EQ(test, filter_count, 2); + + parsed_filters = kunit_kcalloc(test, filter_count, sizeof(*parsed_filters), + GFP_KERNEL); + for (j = 0; j < filter_count; j++) { + parsed_filters[j] = kunit_next_attr_filter(&filter, &err); + KUNIT_ASSERT_EQ_MSG(test, err, 0, "failed to parse filter '%s'", filters[j]); + } + + KUNIT_EXPECT_STREQ(test, kunit_attr_filter_name(parsed_filters[0]), "speed"); + KUNIT_EXPECT_STREQ(test, parsed_filters[0].input, ">slow"); + + KUNIT_EXPECT_STREQ(test, kunit_attr_filter_name(parsed_filters[1]), "module"); + KUNIT_EXPECT_STREQ(test, parsed_filters[1].input, "!=example"); +} + +static struct kunit_case dummy_attr_test_cases[] = { + /* .run_case is not important, just needs to be non-NULL */ + { .name = "slow", .run_case = dummy_test, .module_name = "dummy", + .attr.speed = KUNIT_SPEED_SLOW }, + { .name = "normal", .run_case = dummy_test, .module_name = "dummy" }, + {}, +}; + +static void filter_attr_test(struct kunit *test) +{ + struct kunit_suite *subsuite[3] = {NULL, NULL}; + struct kunit_suite_set suite_set = { + .start = subsuite, .end = &subsuite[2], + }; + struct kunit_suite_set got; + char filter[] = "speed>slow"; + int err = 0; + + subsuite[0] = alloc_fake_suite(test, "normal_suite", dummy_attr_test_cases); + subsuite[1] = alloc_fake_suite(test, "slow_suite", dummy_attr_test_cases); + subsuite[1]->attr.speed = KUNIT_SPEED_SLOW; // Set suite attribute + + /* + * Want: normal_suite(slow, normal), slow_suite(slow, normal), + * NULL -> normal_suite(normal), NULL + * + * The normal test in slow_suite is filtered out because the speed + * attribute is unset and thus, the filtering is based on the parent attribute + * of slow. + */ + got = kunit_filter_suites(&suite_set, NULL, filter, NULL, &err); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, got.start); + KUNIT_ASSERT_EQ(test, err, 0); + free_suite_set_at_end(test, &got); + + /* Validate we just have normal_suite */ + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, got.start[0]); + KUNIT_EXPECT_STREQ(test, got.start[0]->name, "normal_suite"); + KUNIT_ASSERT_EQ(test, got.end - got.start, 1); + + /* Now validate we just have normal test case */ + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, got.start[0]->test_cases); + KUNIT_EXPECT_STREQ(test, got.start[0]->test_cases[0].name, "normal"); + KUNIT_EXPECT_FALSE(test, got.start[0]->test_cases[1].name); +} + +static void filter_attr_empty_test(struct kunit *test) +{ + struct kunit_suite *subsuite[3] = {NULL, NULL}; + struct kunit_suite_set suite_set = { + .start = subsuite, .end = &subsuite[2], + }; + struct kunit_suite_set got; + char filter[] = "module!=dummy"; + int err = 0; + + subsuite[0] = alloc_fake_suite(test, "suite1", dummy_attr_test_cases); + subsuite[1] = alloc_fake_suite(test, "suite2", dummy_attr_test_cases); + + got = kunit_filter_suites(&suite_set, NULL, filter, NULL, &err); + KUNIT_ASSERT_EQ(test, err, 0); + free_suite_set_at_end(test, &got); /* just in case */ + + KUNIT_EXPECT_PTR_EQ_MSG(test, got.start, got.end, + "should be empty to indicate no match"); +} + +static void filter_attr_skip_test(struct kunit *test) +{ + struct kunit_suite *subsuite[2] = {NULL}; + struct kunit_suite_set suite_set = { + .start = subsuite, .end = &subsuite[1], + }; + struct kunit_suite_set got; + char filter[] = "speed>slow"; + int err = 0; + + subsuite[0] = alloc_fake_suite(test, "suite", dummy_attr_test_cases); + + /* Want: suite(slow, normal), NULL -> suite(slow with SKIP, normal), NULL */ + got = kunit_filter_suites(&suite_set, NULL, filter, "skip", &err); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, got.start); + KUNIT_ASSERT_EQ(test, err, 0); + free_suite_set_at_end(test, &got); + + /* Validate we have both the slow and normal test */ + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, got.start[0]->test_cases); + KUNIT_ASSERT_EQ(test, kunit_suite_num_test_cases(got.start[0]), 2); + KUNIT_EXPECT_STREQ(test, got.start[0]->test_cases[0].name, "slow"); + KUNIT_EXPECT_STREQ(test, got.start[0]->test_cases[1].name, "normal"); + + /* Now ensure slow is skipped and normal is not */ + KUNIT_EXPECT_EQ(test, got.start[0]->test_cases[0].status, KUNIT_SKIPPED); + KUNIT_EXPECT_FALSE(test, got.start[0]->test_cases[1].status); +} + static struct kunit_case executor_test_cases[] = { KUNIT_CASE(parse_filter_test), KUNIT_CASE(filter_suites_test), KUNIT_CASE(filter_suites_test_glob_test), KUNIT_CASE(filter_suites_to_empty_test), + KUNIT_CASE(parse_filter_attr_test), + KUNIT_CASE(filter_attr_test), + KUNIT_CASE(filter_attr_empty_test), + KUNIT_CASE(filter_attr_skip_test), {} }; @@ -125,18 +256,26 @@ kunit_test_suites(&executor_test_suite); /* Test helpers */ -/* Use the resource API to register a call to kfree(to_free). +static void free_suite_set(void *suite_set) +{ + kunit_free_suite_set(*(struct kunit_suite_set *)suite_set); + kfree(suite_set); +} + +/* Use the resource API to register a call to free_suite_set. * Since we never actually use the resource, it's safe to use on const data. */ -static void kfree_at_end(struct kunit *test, const void *to_free) +static void free_suite_set_at_end(struct kunit *test, const void *to_free) { - /* kfree() handles NULL already, but avoid allocating a no-op cleanup. */ - if (IS_ERR_OR_NULL(to_free)) + struct kunit_suite_set *free; + + if (!((struct kunit_suite_set *)to_free)->start) return; - kunit_add_action(test, - (kunit_action_t *)kfree, - (void *)to_free); + free = kzalloc(sizeof(struct kunit_suite_set), GFP_KERNEL); + *free = *(struct kunit_suite_set *)to_free; + + kunit_add_action(test, free_suite_set, (void *)free); } static struct kunit_suite *alloc_fake_suite(struct kunit *test, diff --git a/lib/kunit/kunit-example-test.c b/lib/kunit/kunit-example-test.c index b69b689ea850..6bb5c2ef6696 100644 --- a/lib/kunit/kunit-example-test.c +++ b/lib/kunit/kunit-example-test.c @@ -190,6 +190,7 @@ static void example_static_stub_test(struct kunit *test) static const struct example_param { int value; } example_params_array[] = { + { .value = 3, }, { .value = 2, }, { .value = 1, }, { .value = 0, }, @@ -213,14 +214,22 @@ static void example_params_test(struct kunit *test) KUNIT_ASSERT_NOT_NULL(test, param); /* Test can be skipped on unsupported param values */ - if (!param->value) - kunit_skip(test, "unsupported param value"); + if (!is_power_of_2(param->value)) + kunit_skip(test, "unsupported param value %d", param->value); /* You can use param values for parameterized testing */ KUNIT_EXPECT_EQ(test, param->value % param->value, 0); } /* + * This test should always pass. Can be used to practice filtering attributes. + */ +static void example_slow_test(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, 1 + 1, 2); +} + +/* * Here we make a list of all the test cases we want to add to the test suite * below. */ @@ -237,6 +246,7 @@ static struct kunit_case example_test_cases[] = { KUNIT_CASE(example_all_expect_macros_test), KUNIT_CASE(example_static_stub_test), KUNIT_CASE_PARAM(example_params_test, example_gen_params), + KUNIT_CASE_SLOW(example_slow_test), {} }; diff --git a/lib/kunit/kunit-test.c b/lib/kunit/kunit-test.c index 83d8e90ca7a2..de2113a58fa0 100644 --- a/lib/kunit/kunit-test.c +++ b/lib/kunit/kunit-test.c @@ -8,6 +8,7 @@ #include <kunit/test.h> #include <kunit/test-bug.h> +#include "string-stream.h" #include "try-catch-impl.h" struct kunit_try_catch_test_context { @@ -530,12 +531,27 @@ static struct kunit_suite kunit_resource_test_suite = { .test_cases = kunit_resource_test_cases, }; +/* + * Log tests call string_stream functions, which aren't exported. So only + * build this code if this test is built-in. + */ +#if IS_BUILTIN(CONFIG_KUNIT_TEST) + +/* This avoids a cast warning if kfree() is passed direct to kunit_add_action(). */ +static void kfree_wrapper(void *p) +{ + kfree(p); +} + static void kunit_log_test(struct kunit *test) { struct kunit_suite suite; - - suite.log = kunit_kzalloc(test, KUNIT_LOG_SIZE, GFP_KERNEL); +#ifdef CONFIG_KUNIT_DEBUGFS + char *full_log; +#endif + suite.log = kunit_alloc_string_stream(test, GFP_KERNEL); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, suite.log); + string_stream_set_append_newlines(suite.log, true); kunit_log(KERN_INFO, test, "put this in log."); kunit_log(KERN_INFO, test, "this too."); @@ -543,14 +559,21 @@ static void kunit_log_test(struct kunit *test) kunit_log(KERN_INFO, &suite, "along with this."); #ifdef CONFIG_KUNIT_DEBUGFS + KUNIT_EXPECT_TRUE(test, test->log->append_newlines); + + full_log = string_stream_get_string(test->log); + kunit_add_action(test, kfree_wrapper, full_log); KUNIT_EXPECT_NOT_ERR_OR_NULL(test, - strstr(test->log, "put this in log.")); + strstr(full_log, "put this in log.")); KUNIT_EXPECT_NOT_ERR_OR_NULL(test, - strstr(test->log, "this too.")); + strstr(full_log, "this too.")); + + full_log = string_stream_get_string(suite.log); + kunit_add_action(test, kfree_wrapper, full_log); KUNIT_EXPECT_NOT_ERR_OR_NULL(test, - strstr(suite.log, "add to suite log.")); + strstr(full_log, "add to suite log.")); KUNIT_EXPECT_NOT_ERR_OR_NULL(test, - strstr(suite.log, "along with this.")); + strstr(full_log, "along with this.")); #else KUNIT_EXPECT_NULL(test, test->log); #endif @@ -558,15 +581,30 @@ static void kunit_log_test(struct kunit *test) static void kunit_log_newline_test(struct kunit *test) { + char *full_log; + kunit_info(test, "Add newline\n"); if (test->log) { - KUNIT_ASSERT_NOT_NULL_MSG(test, strstr(test->log, "Add newline\n"), - "Missing log line, full log:\n%s", test->log); - KUNIT_EXPECT_NULL(test, strstr(test->log, "Add newline\n\n")); + full_log = string_stream_get_string(test->log); + kunit_add_action(test, kfree_wrapper, full_log); + KUNIT_ASSERT_NOT_NULL_MSG(test, strstr(full_log, "Add newline\n"), + "Missing log line, full log:\n%s", full_log); + KUNIT_EXPECT_NULL(test, strstr(full_log, "Add newline\n\n")); } else { kunit_skip(test, "only useful when debugfs is enabled"); } } +#else +static void kunit_log_test(struct kunit *test) +{ + kunit_skip(test, "Log tests only run when built-in"); +} + +static void kunit_log_newline_test(struct kunit *test) +{ + kunit_skip(test, "Log tests only run when built-in"); +} +#endif /* IS_BUILTIN(CONFIG_KUNIT_TEST) */ static struct kunit_case kunit_log_test_cases[] = { KUNIT_CASE(kunit_log_test), diff --git a/lib/kunit/string-stream-test.c b/lib/kunit/string-stream-test.c index 110f3a993250..06822766f29a 100644 --- a/lib/kunit/string-stream-test.c +++ b/lib/kunit/string-stream-test.c @@ -6,48 +6,539 @@ * Author: Brendan Higgins <brendanhiggins@google.com> */ +#include <kunit/static_stub.h> #include <kunit/test.h> +#include <linux/ktime.h> #include <linux/slab.h> +#include <linux/timekeeping.h> #include "string-stream.h" -static void string_stream_test_empty_on_creation(struct kunit *test) +struct string_stream_test_priv { + /* For testing resource-managed free. */ + struct string_stream *expected_free_stream; + bool stream_was_freed; + bool stream_free_again; +}; + +/* Avoids a cast warning if kfree() is passed direct to kunit_add_action(). */ +static void kfree_wrapper(void *p) +{ + kfree(p); +} + +/* Avoids a cast warning if string_stream_destroy() is passed direct to kunit_add_action(). */ +static void cleanup_raw_stream(void *p) +{ + struct string_stream *stream = p; + + string_stream_destroy(stream); +} + +static char *get_concatenated_string(struct kunit *test, struct string_stream *stream) +{ + char *str = string_stream_get_string(stream); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, str); + kunit_add_action(test, kfree_wrapper, (void *)str); + + return str; +} + +/* Managed string_stream object is initialized correctly. */ +static void string_stream_managed_init_test(struct kunit *test) +{ + struct string_stream *stream; + + /* Resource-managed initialization. */ + stream = kunit_alloc_string_stream(test, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream); + + KUNIT_EXPECT_EQ(test, stream->length, 0); + KUNIT_EXPECT_TRUE(test, list_empty(&stream->fragments)); + KUNIT_EXPECT_TRUE(test, (stream->gfp == GFP_KERNEL)); + KUNIT_EXPECT_FALSE(test, stream->append_newlines); + KUNIT_EXPECT_TRUE(test, string_stream_is_empty(stream)); +} + +/* Unmanaged string_stream object is initialized correctly. */ +static void string_stream_unmanaged_init_test(struct kunit *test) +{ + struct string_stream *stream; + + stream = alloc_string_stream(GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream); + kunit_add_action(test, cleanup_raw_stream, stream); + + KUNIT_EXPECT_EQ(test, stream->length, 0); + KUNIT_EXPECT_TRUE(test, list_empty(&stream->fragments)); + KUNIT_EXPECT_EQ(test, stream->gfp, GFP_KERNEL); + KUNIT_EXPECT_FALSE(test, stream->append_newlines); + + KUNIT_EXPECT_TRUE(test, string_stream_is_empty(stream)); +} + +static void string_stream_destroy_stub(struct string_stream *stream) +{ + struct kunit *fake_test = kunit_get_current_test(); + struct string_stream_test_priv *priv = fake_test->priv; + + /* The kunit could own string_streams other than the one we are testing. */ + if (stream == priv->expected_free_stream) { + if (priv->stream_was_freed) + priv->stream_free_again = true; + else + priv->stream_was_freed = true; + } + + /* + * Calling string_stream_destroy() will only call this function again + * because the redirection stub is still active. + * Avoid calling deactivate_static_stub() or changing current->kunit_test + * during cleanup. + */ + string_stream_clear(stream); + kfree(stream); +} + +/* kunit_free_string_stream() calls string_stream_desrtoy() */ +static void string_stream_managed_free_test(struct kunit *test) +{ + struct string_stream_test_priv *priv = test->priv; + + priv->expected_free_stream = NULL; + priv->stream_was_freed = false; + priv->stream_free_again = false; + + kunit_activate_static_stub(test, + string_stream_destroy, + string_stream_destroy_stub); + + priv->expected_free_stream = kunit_alloc_string_stream(test, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, priv->expected_free_stream); + + /* This should call the stub function. */ + kunit_free_string_stream(test, priv->expected_free_stream); + + KUNIT_EXPECT_TRUE(test, priv->stream_was_freed); + KUNIT_EXPECT_FALSE(test, priv->stream_free_again); +} + +/* string_stream object is freed when test is cleaned up. */ +static void string_stream_resource_free_test(struct kunit *test) +{ + struct string_stream_test_priv *priv = test->priv; + struct kunit *fake_test; + + fake_test = kunit_kzalloc(test, sizeof(*fake_test), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, fake_test); + + kunit_init_test(fake_test, "string_stream_fake_test", NULL); + fake_test->priv = priv; + + /* + * Activate stub before creating string_stream so the + * string_stream will be cleaned up first. + */ + priv->expected_free_stream = NULL; + priv->stream_was_freed = false; + priv->stream_free_again = false; + + kunit_activate_static_stub(fake_test, + string_stream_destroy, + string_stream_destroy_stub); + + priv->expected_free_stream = kunit_alloc_string_stream(fake_test, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, priv->expected_free_stream); + + /* Set current->kunit_test to fake_test so the static stub will be called. */ + current->kunit_test = fake_test; + + /* Cleanup test - the stub function should be called */ + kunit_cleanup(fake_test); + + /* Set current->kunit_test back to current test. */ + current->kunit_test = test; + + KUNIT_EXPECT_TRUE(test, priv->stream_was_freed); + KUNIT_EXPECT_FALSE(test, priv->stream_free_again); +} + +/* + * Add a series of lines to a string_stream. Check that all lines + * appear in the correct order and no characters are dropped. + */ +static void string_stream_line_add_test(struct kunit *test) +{ + struct string_stream *stream; + char line[60]; + char *concat_string, *pos, *string_end; + size_t len, total_len; + int num_lines, i; + + stream = kunit_alloc_string_stream(test, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream); + + /* Add series of sequence numbered lines */ + total_len = 0; + for (i = 0; i < 100; ++i) { + len = snprintf(line, sizeof(line), + "The quick brown fox jumps over the lazy penguin %d\n", i); + + /* Sanity-check that our test string isn't truncated */ + KUNIT_ASSERT_LT(test, len, sizeof(line)); + + string_stream_add(stream, line); + total_len += len; + } + num_lines = i; + + concat_string = get_concatenated_string(test, stream); + KUNIT_EXPECT_NOT_ERR_OR_NULL(test, concat_string); + KUNIT_EXPECT_EQ(test, strlen(concat_string), total_len); + + /* + * Split the concatenated string at the newlines and check that + * all the original added strings are present. + */ + pos = concat_string; + for (i = 0; i < num_lines; ++i) { + string_end = strchr(pos, '\n'); + KUNIT_EXPECT_NOT_NULL(test, string_end); + + /* Convert to NULL-terminated string */ + *string_end = '\0'; + + snprintf(line, sizeof(line), + "The quick brown fox jumps over the lazy penguin %d", i); + KUNIT_EXPECT_STREQ(test, pos, line); + + pos = string_end + 1; + } + + /* There shouldn't be any more data after this */ + KUNIT_EXPECT_EQ(test, strlen(pos), 0); +} + +/* Add a series of lines of variable length to a string_stream. */ +static void string_stream_variable_length_line_test(struct kunit *test) +{ + static const char line[] = + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + " 0123456789!$%^&*()_-+={}[]:;@'~#<>,.?/|"; + struct string_stream *stream; + struct rnd_state rnd; + char *concat_string, *pos, *string_end; + size_t offset, total_len; + int num_lines, i; + + stream = kunit_alloc_string_stream(test, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream); + + /* + * Log many lines of varying lengths until we have created + * many fragments. + * The "randomness" must be repeatable. + */ + prandom_seed_state(&rnd, 3141592653589793238ULL); + total_len = 0; + for (i = 0; i < 100; ++i) { + offset = prandom_u32_state(&rnd) % (sizeof(line) - 1); + string_stream_add(stream, "%s\n", &line[offset]); + total_len += sizeof(line) - offset; + } + num_lines = i; + + concat_string = get_concatenated_string(test, stream); + KUNIT_EXPECT_NOT_ERR_OR_NULL(test, concat_string); + KUNIT_EXPECT_EQ(test, strlen(concat_string), total_len); + + /* + * Split the concatenated string at the newlines and check that + * all the original added strings are present. + */ + prandom_seed_state(&rnd, 3141592653589793238ULL); + pos = concat_string; + for (i = 0; i < num_lines; ++i) { + string_end = strchr(pos, '\n'); + KUNIT_EXPECT_NOT_NULL(test, string_end); + + /* Convert to NULL-terminated string */ + *string_end = '\0'; + + offset = prandom_u32_state(&rnd) % (sizeof(line) - 1); + KUNIT_EXPECT_STREQ(test, pos, &line[offset]); + + pos = string_end + 1; + } + + /* There shouldn't be any more data after this */ + KUNIT_EXPECT_EQ(test, strlen(pos), 0); +} + +/* Appending the content of one string stream to another. */ +static void string_stream_append_test(struct kunit *test) +{ + static const char * const strings_1[] = { + "one", "two", "three", "four", "five", "six", + "seven", "eight", "nine", "ten", + }; + static const char * const strings_2[] = { + "Apple", "Pear", "Orange", "Banana", "Grape", "Apricot", + }; + struct string_stream *stream_1, *stream_2; + const char *stream1_content_before_append, *stream_2_content; + char *combined_content; + size_t combined_length; + int i; + + stream_1 = kunit_alloc_string_stream(test, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream_1); + + stream_2 = kunit_alloc_string_stream(test, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream_2); + + /* Append content of empty stream to empty stream */ + string_stream_append(stream_1, stream_2); + KUNIT_EXPECT_EQ(test, strlen(get_concatenated_string(test, stream_1)), 0); + + /* Add some data to stream_1 */ + for (i = 0; i < ARRAY_SIZE(strings_1); ++i) + string_stream_add(stream_1, "%s\n", strings_1[i]); + + stream1_content_before_append = get_concatenated_string(test, stream_1); + + /* Append content of empty stream to non-empty stream */ + string_stream_append(stream_1, stream_2); + KUNIT_EXPECT_STREQ(test, get_concatenated_string(test, stream_1), + stream1_content_before_append); + + /* Add some data to stream_2 */ + for (i = 0; i < ARRAY_SIZE(strings_2); ++i) + string_stream_add(stream_2, "%s\n", strings_2[i]); + + /* Append content of non-empty stream to non-empty stream */ + string_stream_append(stream_1, stream_2); + + /* + * End result should be the original content of stream_1 plus + * the content of stream_2. + */ + stream_2_content = get_concatenated_string(test, stream_2); + combined_length = strlen(stream1_content_before_append) + strlen(stream_2_content); + combined_length++; /* for terminating \0 */ + combined_content = kunit_kmalloc(test, combined_length, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, combined_content); + snprintf(combined_content, combined_length, "%s%s", + stream1_content_before_append, stream_2_content); + + KUNIT_EXPECT_STREQ(test, get_concatenated_string(test, stream_1), combined_content); + + /* Append content of non-empty stream to empty stream */ + kunit_free_string_stream(test, stream_1); + + stream_1 = kunit_alloc_string_stream(test, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream_1); + + string_stream_append(stream_1, stream_2); + KUNIT_EXPECT_STREQ(test, get_concatenated_string(test, stream_1), stream_2_content); +} + +/* Appending the content of one string stream to one with auto-newlining. */ +static void string_stream_append_auto_newline_test(struct kunit *test) +{ + struct string_stream *stream_1, *stream_2; + + /* Stream 1 has newline appending enabled */ + stream_1 = kunit_alloc_string_stream(test, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream_1); + string_stream_set_append_newlines(stream_1, true); + KUNIT_EXPECT_TRUE(test, stream_1->append_newlines); + + /* Stream 2 does not append newlines */ + stream_2 = kunit_alloc_string_stream(test, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream_2); + + /* Appending a stream with a newline should not add another newline */ + string_stream_add(stream_1, "Original string\n"); + string_stream_add(stream_2, "Appended content\n"); + string_stream_add(stream_2, "More stuff\n"); + string_stream_append(stream_1, stream_2); + KUNIT_EXPECT_STREQ(test, get_concatenated_string(test, stream_1), + "Original string\nAppended content\nMore stuff\n"); + + kunit_free_string_stream(test, stream_2); + stream_2 = kunit_alloc_string_stream(test, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream_2); + + /* + * Appending a stream without newline should add a final newline. + * The appended string_stream is treated as a single string so newlines + * should not be inserted between fragments. + */ + string_stream_add(stream_2, "Another"); + string_stream_add(stream_2, "And again"); + string_stream_append(stream_1, stream_2); + KUNIT_EXPECT_STREQ(test, get_concatenated_string(test, stream_1), + "Original string\nAppended content\nMore stuff\nAnotherAnd again\n"); +} + +/* Adding an empty string should not create a fragment. */ +static void string_stream_append_empty_string_test(struct kunit *test) { - struct string_stream *stream = alloc_string_stream(test, GFP_KERNEL); + struct string_stream *stream; + int original_frag_count; + + stream = kunit_alloc_string_stream(test, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream); + /* Formatted empty string */ + string_stream_add(stream, "%s", ""); KUNIT_EXPECT_TRUE(test, string_stream_is_empty(stream)); + KUNIT_EXPECT_TRUE(test, list_empty(&stream->fragments)); + + /* Adding an empty string to a non-empty stream */ + string_stream_add(stream, "Add this line"); + original_frag_count = list_count_nodes(&stream->fragments); + + string_stream_add(stream, "%s", ""); + KUNIT_EXPECT_EQ(test, list_count_nodes(&stream->fragments), original_frag_count); + KUNIT_EXPECT_STREQ(test, get_concatenated_string(test, stream), "Add this line"); } -static void string_stream_test_not_empty_after_add(struct kunit *test) +/* Adding strings without automatic newline appending */ +static void string_stream_no_auto_newline_test(struct kunit *test) { - struct string_stream *stream = alloc_string_stream(test, GFP_KERNEL); + struct string_stream *stream; - string_stream_add(stream, "Foo"); + stream = kunit_alloc_string_stream(test, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream); - KUNIT_EXPECT_FALSE(test, string_stream_is_empty(stream)); + /* + * Add some strings with and without newlines. All formatted newlines + * should be preserved. It should not add any extra newlines. + */ + string_stream_add(stream, "One"); + string_stream_add(stream, "Two\n"); + string_stream_add(stream, "%s\n", "Three"); + string_stream_add(stream, "%s", "Four\n"); + string_stream_add(stream, "Five\n%s", "Six"); + string_stream_add(stream, "Seven\n\n"); + string_stream_add(stream, "Eight"); + KUNIT_EXPECT_STREQ(test, get_concatenated_string(test, stream), + "OneTwo\nThree\nFour\nFive\nSixSeven\n\nEight"); } -static void string_stream_test_get_string(struct kunit *test) +/* Adding strings with automatic newline appending */ +static void string_stream_auto_newline_test(struct kunit *test) { - struct string_stream *stream = alloc_string_stream(test, GFP_KERNEL); - char *output; + struct string_stream *stream; + + stream = kunit_alloc_string_stream(test, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream); + + string_stream_set_append_newlines(stream, true); + KUNIT_EXPECT_TRUE(test, stream->append_newlines); + + /* + * Add some strings with and without newlines. Newlines should + * be appended to lines that do not end with \n, but newlines + * resulting from the formatting should not be changed. + */ + string_stream_add(stream, "One"); + string_stream_add(stream, "Two\n"); + string_stream_add(stream, "%s\n", "Three"); + string_stream_add(stream, "%s", "Four\n"); + string_stream_add(stream, "Five\n%s", "Six"); + string_stream_add(stream, "Seven\n\n"); + string_stream_add(stream, "Eight"); + KUNIT_EXPECT_STREQ(test, get_concatenated_string(test, stream), + "One\nTwo\nThree\nFour\nFive\nSix\nSeven\n\nEight\n"); +} + +/* + * This doesn't actually "test" anything. It reports time taken + * and memory used for logging a large number of lines. + */ +static void string_stream_performance_test(struct kunit *test) +{ + struct string_stream_fragment *frag_container; + struct string_stream *stream; + char test_line[101]; + ktime_t start_time, end_time; + size_t len, bytes_requested, actual_bytes_used, total_string_length; + int offset, i; + + stream = kunit_alloc_string_stream(test, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream); + + memset(test_line, 'x', sizeof(test_line) - 1); + test_line[sizeof(test_line) - 1] = '\0'; + + start_time = ktime_get(); + for (i = 0; i < 10000; i++) { + offset = i % (sizeof(test_line) - 1); + string_stream_add(stream, "%s: %d\n", &test_line[offset], i); + } + end_time = ktime_get(); + + /* + * Calculate memory used. This doesn't include invisible + * overhead due to kernel allocator fragment size rounding. + */ + bytes_requested = sizeof(*stream); + actual_bytes_used = ksize(stream); + total_string_length = 0; + + list_for_each_entry(frag_container, &stream->fragments, node) { + bytes_requested += sizeof(*frag_container); + actual_bytes_used += ksize(frag_container); + + len = strlen(frag_container->fragment); + total_string_length += len; + bytes_requested += len + 1; /* +1 for '\0' */ + actual_bytes_used += ksize(frag_container->fragment); + } + + kunit_info(test, "Time elapsed: %lld us\n", + ktime_us_delta(end_time, start_time)); + kunit_info(test, "Total string length: %zu\n", total_string_length); + kunit_info(test, "Bytes requested: %zu\n", bytes_requested); + kunit_info(test, "Actual bytes allocated: %zu\n", actual_bytes_used); +} + +static int string_stream_test_init(struct kunit *test) +{ + struct string_stream_test_priv *priv; + + priv = kunit_kzalloc(test, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; - string_stream_add(stream, "Foo"); - string_stream_add(stream, " %s", "bar"); + test->priv = priv; - output = string_stream_get_string(stream); - KUNIT_ASSERT_STREQ(test, output, "Foo bar"); + return 0; } static struct kunit_case string_stream_test_cases[] = { - KUNIT_CASE(string_stream_test_empty_on_creation), - KUNIT_CASE(string_stream_test_not_empty_after_add), - KUNIT_CASE(string_stream_test_get_string), + KUNIT_CASE(string_stream_managed_init_test), + KUNIT_CASE(string_stream_unmanaged_init_test), + KUNIT_CASE(string_stream_managed_free_test), + KUNIT_CASE(string_stream_resource_free_test), + KUNIT_CASE(string_stream_line_add_test), + KUNIT_CASE(string_stream_variable_length_line_test), + KUNIT_CASE(string_stream_append_test), + KUNIT_CASE(string_stream_append_auto_newline_test), + KUNIT_CASE(string_stream_append_empty_string_test), + KUNIT_CASE(string_stream_no_auto_newline_test), + KUNIT_CASE(string_stream_auto_newline_test), + KUNIT_CASE(string_stream_performance_test), {} }; static struct kunit_suite string_stream_test_suite = { .name = "string-stream-test", - .test_cases = string_stream_test_cases + .test_cases = string_stream_test_cases, + .init = string_stream_test_init, }; kunit_test_suites(&string_stream_test_suite); diff --git a/lib/kunit/string-stream.c b/lib/kunit/string-stream.c index cc32743c1171..a6f3616c2048 100644 --- a/lib/kunit/string-stream.c +++ b/lib/kunit/string-stream.c @@ -6,6 +6,7 @@ * Author: Brendan Higgins <brendanhiggins@google.com> */ +#include <kunit/static_stub.h> #include <kunit/test.h> #include <linux/list.h> #include <linux/slab.h> @@ -13,30 +14,28 @@ #include "string-stream.h" -static struct string_stream_fragment *alloc_string_stream_fragment( - struct kunit *test, int len, gfp_t gfp) +static struct string_stream_fragment *alloc_string_stream_fragment(int len, gfp_t gfp) { struct string_stream_fragment *frag; - frag = kunit_kzalloc(test, sizeof(*frag), gfp); + frag = kzalloc(sizeof(*frag), gfp); if (!frag) return ERR_PTR(-ENOMEM); - frag->fragment = kunit_kmalloc(test, len, gfp); + frag->fragment = kmalloc(len, gfp); if (!frag->fragment) { - kunit_kfree(test, frag); + kfree(frag); return ERR_PTR(-ENOMEM); } return frag; } -static void string_stream_fragment_destroy(struct kunit *test, - struct string_stream_fragment *frag) +static void string_stream_fragment_destroy(struct string_stream_fragment *frag) { list_del(&frag->node); - kunit_kfree(test, frag->fragment); - kunit_kfree(test, frag); + kfree(frag->fragment); + kfree(frag); } int string_stream_vadd(struct string_stream *stream, @@ -44,26 +43,44 @@ int string_stream_vadd(struct string_stream *stream, va_list args) { struct string_stream_fragment *frag_container; - int len; + int buf_len, result_len; va_list args_for_counting; /* Make a copy because `vsnprintf` could change it */ va_copy(args_for_counting, args); - /* Need space for null byte. */ - len = vsnprintf(NULL, 0, fmt, args_for_counting) + 1; + /* Evaluate length of formatted string */ + buf_len = vsnprintf(NULL, 0, fmt, args_for_counting); va_end(args_for_counting); - frag_container = alloc_string_stream_fragment(stream->test, - len, - stream->gfp); + if (buf_len == 0) + return 0; + + /* Reserve one extra for possible appended newline. */ + if (stream->append_newlines) + buf_len++; + + /* Need space for null byte. */ + buf_len++; + + frag_container = alloc_string_stream_fragment(buf_len, stream->gfp); if (IS_ERR(frag_container)) return PTR_ERR(frag_container); - len = vsnprintf(frag_container->fragment, len, fmt, args); + if (stream->append_newlines) { + /* Don't include reserved newline byte in writeable length. */ + result_len = vsnprintf(frag_container->fragment, buf_len - 1, fmt, args); + + /* Append newline if necessary. */ + if (frag_container->fragment[result_len - 1] != '\n') + result_len = strlcat(frag_container->fragment, "\n", buf_len); + } else { + result_len = vsnprintf(frag_container->fragment, buf_len, fmt, args); + } + spin_lock(&stream->lock); - stream->length += len; + stream->length += result_len; list_add_tail(&frag_container->node, &stream->fragments); spin_unlock(&stream->lock); @@ -82,7 +99,7 @@ int string_stream_add(struct string_stream *stream, const char *fmt, ...) return result; } -static void string_stream_clear(struct string_stream *stream) +void string_stream_clear(struct string_stream *stream) { struct string_stream_fragment *frag_container, *frag_container_safe; @@ -91,7 +108,7 @@ static void string_stream_clear(struct string_stream *stream) frag_container_safe, &stream->fragments, node) { - string_stream_fragment_destroy(stream->test, frag_container); + string_stream_fragment_destroy(frag_container); } stream->length = 0; spin_unlock(&stream->lock); @@ -103,7 +120,7 @@ char *string_stream_get_string(struct string_stream *stream) size_t buf_len = stream->length + 1; /* +1 for null byte. */ char *buf; - buf = kunit_kzalloc(stream->test, buf_len, stream->gfp); + buf = kzalloc(buf_len, stream->gfp); if (!buf) return NULL; @@ -119,13 +136,17 @@ int string_stream_append(struct string_stream *stream, struct string_stream *other) { const char *other_content; + int ret; other_content = string_stream_get_string(other); if (!other_content) return -ENOMEM; - return string_stream_add(stream, other_content); + ret = string_stream_add(stream, other_content); + kfree(other_content); + + return ret; } bool string_stream_is_empty(struct string_stream *stream) @@ -133,16 +154,15 @@ bool string_stream_is_empty(struct string_stream *stream) return list_empty(&stream->fragments); } -struct string_stream *alloc_string_stream(struct kunit *test, gfp_t gfp) +struct string_stream *alloc_string_stream(gfp_t gfp) { struct string_stream *stream; - stream = kunit_kzalloc(test, sizeof(*stream), gfp); + stream = kzalloc(sizeof(*stream), gfp); if (!stream) return ERR_PTR(-ENOMEM); stream->gfp = gfp; - stream->test = test; INIT_LIST_HEAD(&stream->fragments); spin_lock_init(&stream->lock); @@ -151,5 +171,37 @@ struct string_stream *alloc_string_stream(struct kunit *test, gfp_t gfp) void string_stream_destroy(struct string_stream *stream) { + KUNIT_STATIC_STUB_REDIRECT(string_stream_destroy, stream); + + if (!stream) + return; + string_stream_clear(stream); + kfree(stream); +} + +static void resource_free_string_stream(void *p) +{ + struct string_stream *stream = p; + + string_stream_destroy(stream); +} + +struct string_stream *kunit_alloc_string_stream(struct kunit *test, gfp_t gfp) +{ + struct string_stream *stream; + + stream = alloc_string_stream(gfp); + if (IS_ERR(stream)) + return stream; + + if (kunit_add_action_or_reset(test, resource_free_string_stream, stream) != 0) + return ERR_PTR(-ENOMEM); + + return stream; +} + +void kunit_free_string_stream(struct kunit *test, struct string_stream *stream) +{ + kunit_release_action(test, resource_free_string_stream, (void *)stream); } diff --git a/lib/kunit/string-stream.h b/lib/kunit/string-stream.h index b669f9a75a94..7be2450c7079 100644 --- a/lib/kunit/string-stream.h +++ b/lib/kunit/string-stream.h @@ -23,13 +23,17 @@ struct string_stream { struct list_head fragments; /* length and fragments are protected by this lock */ spinlock_t lock; - struct kunit *test; gfp_t gfp; + bool append_newlines; }; struct kunit; -struct string_stream *alloc_string_stream(struct kunit *test, gfp_t gfp); +struct string_stream *kunit_alloc_string_stream(struct kunit *test, gfp_t gfp); +void kunit_free_string_stream(struct kunit *test, struct string_stream *stream); + +struct string_stream *alloc_string_stream(gfp_t gfp); +void free_string_stream(struct string_stream *stream); int __printf(2, 3) string_stream_add(struct string_stream *stream, const char *fmt, ...); @@ -38,6 +42,8 @@ int __printf(2, 0) string_stream_vadd(struct string_stream *stream, const char *fmt, va_list args); +void string_stream_clear(struct string_stream *stream); + char *string_stream_get_string(struct string_stream *stream); int string_stream_append(struct string_stream *stream, @@ -47,4 +53,10 @@ bool string_stream_is_empty(struct string_stream *stream); void string_stream_destroy(struct string_stream *stream); +static inline void string_stream_set_append_newlines(struct string_stream *stream, + bool append_newlines) +{ + stream->append_newlines = append_newlines; +} + #endif /* _KUNIT_STRING_STREAM_H */ diff --git a/lib/kunit/test.c b/lib/kunit/test.c index 84e4666555c9..7aceb07a1af9 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -9,6 +9,7 @@ #include <kunit/resource.h> #include <kunit/test.h> #include <kunit/test-bug.h> +#include <kunit/attributes.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/moduleparam.h> @@ -108,51 +109,17 @@ static void kunit_print_test_stats(struct kunit *test, stats.total); } -/** - * kunit_log_newline() - Add newline to the end of log if one is not - * already present. - * @log: The log to add the newline to. - */ -static void kunit_log_newline(char *log) -{ - int log_len, len_left; - - log_len = strlen(log); - len_left = KUNIT_LOG_SIZE - log_len - 1; - - if (log_len > 0 && log[log_len - 1] != '\n') - strncat(log, "\n", len_left); -} - -/* - * Append formatted message to log, size of which is limited to - * KUNIT_LOG_SIZE bytes (including null terminating byte). - */ -void kunit_log_append(char *log, const char *fmt, ...) +/* Append formatted message to log. */ +void kunit_log_append(struct string_stream *log, const char *fmt, ...) { va_list args; - int len, log_len, len_left; if (!log) return; - log_len = strlen(log); - len_left = KUNIT_LOG_SIZE - log_len - 1; - if (len_left <= 0) - return; - - /* Evaluate length of line to add to log */ - va_start(args, fmt); - len = vsnprintf(NULL, 0, fmt, args) + 1; - va_end(args); - - /* Print formatted line to the log */ va_start(args, fmt); - vsnprintf(log + log_len, min(len, len_left), fmt, args); + string_stream_vadd(log, fmt, args); va_end(args); - - /* Add newline to end of log if not already present. */ - kunit_log_newline(log); } EXPORT_SYMBOL_GPL(kunit_log_append); @@ -168,6 +135,13 @@ size_t kunit_suite_num_test_cases(struct kunit_suite *suite) } EXPORT_SYMBOL_GPL(kunit_suite_num_test_cases); +/* Currently supported test levels */ +enum { + KUNIT_LEVEL_SUITE = 0, + KUNIT_LEVEL_CASE, + KUNIT_LEVEL_CASE_PARAM, +}; + static void kunit_print_suite_start(struct kunit_suite *suite) { /* @@ -181,17 +155,11 @@ static void kunit_print_suite_start(struct kunit_suite *suite) pr_info(KUNIT_SUBTEST_INDENT "KTAP version 1\n"); pr_info(KUNIT_SUBTEST_INDENT "# Subtest: %s\n", suite->name); + kunit_print_attr((void *)suite, false, KUNIT_LEVEL_CASE); pr_info(KUNIT_SUBTEST_INDENT "1..%zd\n", kunit_suite_num_test_cases(suite)); } -/* Currently supported test levels */ -enum { - KUNIT_LEVEL_SUITE = 0, - KUNIT_LEVEL_CASE, - KUNIT_LEVEL_CASE_PARAM, -}; - static void kunit_print_ok_not_ok(struct kunit *test, unsigned int test_level, enum kunit_status status, @@ -294,7 +262,7 @@ static void kunit_print_string_stream(struct kunit *test, kunit_err(test, "\n"); } else { kunit_err(test, "%s", buf); - kunit_kfree(test, buf); + kfree(buf); } } @@ -306,7 +274,7 @@ static void kunit_fail(struct kunit *test, const struct kunit_loc *loc, kunit_set_failure(test); - stream = alloc_string_stream(test, GFP_KERNEL); + stream = kunit_alloc_string_stream(test, GFP_KERNEL); if (IS_ERR(stream)) { WARN(true, "Could not allocate stream to print failed assertion in %s:%d\n", @@ -320,7 +288,7 @@ static void kunit_fail(struct kunit *test, const struct kunit_loc *loc, kunit_print_string_stream(test, stream); - string_stream_destroy(stream); + kunit_free_string_stream(test, stream); } void __noreturn __kunit_abort(struct kunit *test) @@ -357,19 +325,49 @@ void __kunit_do_failed_assertion(struct kunit *test, } EXPORT_SYMBOL_GPL(__kunit_do_failed_assertion); -void kunit_init_test(struct kunit *test, const char *name, char *log) +void kunit_init_test(struct kunit *test, const char *name, struct string_stream *log) { spin_lock_init(&test->lock); INIT_LIST_HEAD(&test->resources); test->name = name; test->log = log; if (test->log) - test->log[0] = '\0'; + string_stream_clear(log); test->status = KUNIT_SUCCESS; test->status_comment[0] = '\0'; } EXPORT_SYMBOL_GPL(kunit_init_test); +/* Only warn when a test takes more than twice the threshold */ +#define KUNIT_SPEED_WARNING_MULTIPLIER 2 + +/* Slow tests are defined as taking more than 1s */ +#define KUNIT_SPEED_SLOW_THRESHOLD_S 1 + +#define KUNIT_SPEED_SLOW_WARNING_THRESHOLD_S \ + (KUNIT_SPEED_WARNING_MULTIPLIER * KUNIT_SPEED_SLOW_THRESHOLD_S) + +#define s_to_timespec64(s) ns_to_timespec64((s) * NSEC_PER_SEC) + +static void kunit_run_case_check_speed(struct kunit *test, + struct kunit_case *test_case, + struct timespec64 duration) +{ + struct timespec64 slow_thr = + s_to_timespec64(KUNIT_SPEED_SLOW_WARNING_THRESHOLD_S); + enum kunit_speed speed = test_case->attr.speed; + + if (timespec64_compare(&duration, &slow_thr) < 0) + return; + + if (speed == KUNIT_SPEED_VERY_SLOW || speed == KUNIT_SPEED_SLOW) + return; + + kunit_warn(test, + "Test should be marked slow (runtime: %lld.%09lds)", + duration.tv_sec, duration.tv_nsec); +} + /* * Initializes and runs test case. Does not clean up or do post validations. */ @@ -377,6 +375,8 @@ static void kunit_run_case_internal(struct kunit *test, struct kunit_suite *suite, struct kunit_case *test_case) { + struct timespec64 start, end; + if (suite->init) { int ret; @@ -388,7 +388,13 @@ static void kunit_run_case_internal(struct kunit *test, } } + ktime_get_ts64(&start); + test_case->run_case(test); + + ktime_get_ts64(&end); + + kunit_run_case_check_speed(test, test_case, timespec64_sub(end, start)); } static void kunit_case_internal_cleanup(struct kunit *test) @@ -611,18 +617,22 @@ int kunit_run_tests(struct kunit_suite *suite) kunit_suite_for_each_test_case(suite, test_case) { struct kunit test = { .param_value = NULL, .param_index = 0 }; struct kunit_result_stats param_stats = { 0 }; - test_case->status = KUNIT_SKIPPED; kunit_init_test(&test, test_case->name, test_case->log); - - if (!test_case->generate_params) { + if (test_case->status == KUNIT_SKIPPED) { + /* Test marked as skip */ + test.status = KUNIT_SKIPPED; + kunit_update_stats(¶m_stats, test.status); + } else if (!test_case->generate_params) { /* Non-parameterised test. */ + test_case->status = KUNIT_SKIPPED; kunit_run_case_catch_errors(suite, test_case, &test); kunit_update_stats(¶m_stats, test.status); } else { /* Get initial param. */ param_desc[0] = '\0'; test.param_value = test_case->generate_params(NULL, param_desc); + test_case->status = KUNIT_SKIPPED; kunit_log(KERN_INFO, &test, KUNIT_SUBTEST_INDENT KUNIT_SUBTEST_INDENT "KTAP version 1\n"); kunit_log(KERN_INFO, &test, KUNIT_SUBTEST_INDENT KUNIT_SUBTEST_INDENT @@ -642,15 +652,18 @@ int kunit_run_tests(struct kunit_suite *suite) param_desc, test.status_comment); + kunit_update_stats(¶m_stats, test.status); + /* Get next param. */ param_desc[0] = '\0'; test.param_value = test_case->generate_params(test.param_value, param_desc); test.param_index++; - - kunit_update_stats(¶m_stats, test.status); + test.status = KUNIT_SUCCESS; + test.status_comment[0] = '\0'; } } + kunit_print_attr((void *)test_case, true, KUNIT_LEVEL_CASE); kunit_print_test_stats(&test, param_stats); @@ -695,6 +708,8 @@ int __kunit_test_suites_init(struct kunit_suite * const * const suites, int num_ return 0; } + kunit_suite_counter = 1; + static_branch_inc(&kunit_running); for (i = 0; i < num_suites; i++) { @@ -721,20 +736,51 @@ void __kunit_test_suites_exit(struct kunit_suite **suites, int num_suites) for (i = 0; i < num_suites; i++) kunit_exit_suite(suites[i]); - - kunit_suite_counter = 1; } EXPORT_SYMBOL_GPL(__kunit_test_suites_exit); #ifdef CONFIG_MODULES static void kunit_module_init(struct module *mod) { - __kunit_test_suites_init(mod->kunit_suites, mod->num_kunit_suites); + struct kunit_suite_set suite_set = { + mod->kunit_suites, mod->kunit_suites + mod->num_kunit_suites, + }; + const char *action = kunit_action(); + int err = 0; + + suite_set = kunit_filter_suites(&suite_set, + kunit_filter_glob() ?: "*.*", + kunit_filter(), kunit_filter_action(), + &err); + if (err) + pr_err("kunit module: error filtering suites: %d\n", err); + + mod->kunit_suites = (struct kunit_suite **)suite_set.start; + mod->num_kunit_suites = suite_set.end - suite_set.start; + + if (!action) + kunit_exec_run_tests(&suite_set, false); + else if (!strcmp(action, "list")) + kunit_exec_list_tests(&suite_set, false); + else if (!strcmp(action, "list_attr")) + kunit_exec_list_tests(&suite_set, true); + else + pr_err("kunit: unknown action '%s'\n", action); } static void kunit_module_exit(struct module *mod) { - __kunit_test_suites_exit(mod->kunit_suites, mod->num_kunit_suites); + struct kunit_suite_set suite_set = { + mod->kunit_suites, mod->kunit_suites + mod->num_kunit_suites, + }; + const char *action = kunit_action(); + + if (!action) + __kunit_test_suites_exit(mod->kunit_suites, + mod->num_kunit_suites); + + if (suite_set.start) + kunit_free_suite_set(suite_set); } static int kunit_module_notify(struct notifier_block *nb, unsigned long val, @@ -744,12 +790,13 @@ static int kunit_module_notify(struct notifier_block *nb, unsigned long val, switch (val) { case MODULE_STATE_LIVE: - kunit_module_init(mod); break; case MODULE_STATE_GOING: kunit_module_exit(mod); break; case MODULE_STATE_COMING: + kunit_module_init(mod); + break; case MODULE_STATE_UNFORMED: break; } diff --git a/lib/kunit_iov_iter.c b/lib/kunit_iov_iter.c new file mode 100644 index 000000000000..859b67c4d697 --- /dev/null +++ b/lib/kunit_iov_iter.c @@ -0,0 +1,777 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* I/O iterator tests. This can only test kernel-backed iterator types. + * + * Copyright (C) 2023 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/module.h> +#include <linux/vmalloc.h> +#include <linux/mm.h> +#include <linux/uio.h> +#include <linux/bvec.h> +#include <kunit/test.h> + +MODULE_DESCRIPTION("iov_iter testing"); +MODULE_AUTHOR("David Howells <dhowells@redhat.com>"); +MODULE_LICENSE("GPL"); + +struct kvec_test_range { + int from, to; +}; + +static const struct kvec_test_range kvec_test_ranges[] = { + { 0x00002, 0x00002 }, + { 0x00027, 0x03000 }, + { 0x05193, 0x18794 }, + { 0x20000, 0x20000 }, + { 0x20000, 0x24000 }, + { 0x24000, 0x27001 }, + { 0x29000, 0xffffb }, + { 0xffffd, 0xffffe }, + { -1 } +}; + +static inline u8 pattern(unsigned long x) +{ + return x & 0xff; +} + +static void iov_kunit_unmap(void *data) +{ + vunmap(data); +} + +static void *__init iov_kunit_create_buffer(struct kunit *test, + struct page ***ppages, + size_t npages) +{ + struct page **pages; + unsigned long got; + void *buffer; + + pages = kunit_kcalloc(test, npages, sizeof(struct page *), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pages); + *ppages = pages; + + got = alloc_pages_bulk_array(GFP_KERNEL, npages, pages); + if (got != npages) { + release_pages(pages, got); + KUNIT_ASSERT_EQ(test, got, npages); + } + + buffer = vmap(pages, npages, VM_MAP | VM_MAP_PUT_PAGES, PAGE_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, buffer); + + kunit_add_action_or_reset(test, iov_kunit_unmap, buffer); + return buffer; +} + +static void __init iov_kunit_load_kvec(struct kunit *test, + struct iov_iter *iter, int dir, + struct kvec *kvec, unsigned int kvmax, + void *buffer, size_t bufsize, + const struct kvec_test_range *pr) +{ + size_t size = 0; + int i; + + for (i = 0; i < kvmax; i++, pr++) { + if (pr->from < 0) + break; + KUNIT_ASSERT_GE(test, pr->to, pr->from); + KUNIT_ASSERT_LE(test, pr->to, bufsize); + kvec[i].iov_base = buffer + pr->from; + kvec[i].iov_len = pr->to - pr->from; + size += pr->to - pr->from; + } + KUNIT_ASSERT_LE(test, size, bufsize); + + iov_iter_kvec(iter, dir, kvec, i, size); +} + +/* + * Test copying to a ITER_KVEC-type iterator. + */ +static void __init iov_kunit_copy_to_kvec(struct kunit *test) +{ + const struct kvec_test_range *pr; + struct iov_iter iter; + struct page **spages, **bpages; + struct kvec kvec[8]; + u8 *scratch, *buffer; + size_t bufsize, npages, size, copied; + int i, patt; + + bufsize = 0x100000; + npages = bufsize / PAGE_SIZE; + + scratch = iov_kunit_create_buffer(test, &spages, npages); + for (i = 0; i < bufsize; i++) + scratch[i] = pattern(i); + + buffer = iov_kunit_create_buffer(test, &bpages, npages); + memset(buffer, 0, bufsize); + + iov_kunit_load_kvec(test, &iter, READ, kvec, ARRAY_SIZE(kvec), + buffer, bufsize, kvec_test_ranges); + size = iter.count; + + copied = copy_to_iter(scratch, size, &iter); + + KUNIT_EXPECT_EQ(test, copied, size); + KUNIT_EXPECT_EQ(test, iter.count, 0); + KUNIT_EXPECT_EQ(test, iter.nr_segs, 0); + + /* Build the expected image in the scratch buffer. */ + patt = 0; + memset(scratch, 0, bufsize); + for (pr = kvec_test_ranges; pr->from >= 0; pr++) + for (i = pr->from; i < pr->to; i++) + scratch[i] = pattern(patt++); + + /* Compare the images */ + for (i = 0; i < bufsize; i++) { + KUNIT_EXPECT_EQ_MSG(test, buffer[i], scratch[i], "at i=%x", i); + if (buffer[i] != scratch[i]) + return; + } + + KUNIT_SUCCEED(); +} + +/* + * Test copying from a ITER_KVEC-type iterator. + */ +static void __init iov_kunit_copy_from_kvec(struct kunit *test) +{ + const struct kvec_test_range *pr; + struct iov_iter iter; + struct page **spages, **bpages; + struct kvec kvec[8]; + u8 *scratch, *buffer; + size_t bufsize, npages, size, copied; + int i, j; + + bufsize = 0x100000; + npages = bufsize / PAGE_SIZE; + + buffer = iov_kunit_create_buffer(test, &bpages, npages); + for (i = 0; i < bufsize; i++) + buffer[i] = pattern(i); + + scratch = iov_kunit_create_buffer(test, &spages, npages); + memset(scratch, 0, bufsize); + + iov_kunit_load_kvec(test, &iter, WRITE, kvec, ARRAY_SIZE(kvec), + buffer, bufsize, kvec_test_ranges); + size = min(iter.count, bufsize); + + copied = copy_from_iter(scratch, size, &iter); + + KUNIT_EXPECT_EQ(test, copied, size); + KUNIT_EXPECT_EQ(test, iter.count, 0); + KUNIT_EXPECT_EQ(test, iter.nr_segs, 0); + + /* Build the expected image in the main buffer. */ + i = 0; + memset(buffer, 0, bufsize); + for (pr = kvec_test_ranges; pr->from >= 0; pr++) { + for (j = pr->from; j < pr->to; j++) { + buffer[i++] = pattern(j); + if (i >= bufsize) + goto stop; + } + } +stop: + + /* Compare the images */ + for (i = 0; i < bufsize; i++) { + KUNIT_EXPECT_EQ_MSG(test, scratch[i], buffer[i], "at i=%x", i); + if (scratch[i] != buffer[i]) + return; + } + + KUNIT_SUCCEED(); +} + +struct bvec_test_range { + int page, from, to; +}; + +static const struct bvec_test_range bvec_test_ranges[] = { + { 0, 0x0002, 0x0002 }, + { 1, 0x0027, 0x0893 }, + { 2, 0x0193, 0x0794 }, + { 3, 0x0000, 0x1000 }, + { 4, 0x0000, 0x1000 }, + { 5, 0x0000, 0x1000 }, + { 6, 0x0000, 0x0ffb }, + { 6, 0x0ffd, 0x0ffe }, + { -1, -1, -1 } +}; + +static void __init iov_kunit_load_bvec(struct kunit *test, + struct iov_iter *iter, int dir, + struct bio_vec *bvec, unsigned int bvmax, + struct page **pages, size_t npages, + size_t bufsize, + const struct bvec_test_range *pr) +{ + struct page *can_merge = NULL, *page; + size_t size = 0; + int i; + + for (i = 0; i < bvmax; i++, pr++) { + if (pr->from < 0) + break; + KUNIT_ASSERT_LT(test, pr->page, npages); + KUNIT_ASSERT_LT(test, pr->page * PAGE_SIZE, bufsize); + KUNIT_ASSERT_GE(test, pr->from, 0); + KUNIT_ASSERT_GE(test, pr->to, pr->from); + KUNIT_ASSERT_LE(test, pr->to, PAGE_SIZE); + + page = pages[pr->page]; + if (pr->from == 0 && pr->from != pr->to && page == can_merge) { + i--; + bvec[i].bv_len += pr->to; + } else { + bvec_set_page(&bvec[i], page, pr->to - pr->from, pr->from); + } + + size += pr->to - pr->from; + if ((pr->to & ~PAGE_MASK) == 0) + can_merge = page + pr->to / PAGE_SIZE; + else + can_merge = NULL; + } + + iov_iter_bvec(iter, dir, bvec, i, size); +} + +/* + * Test copying to a ITER_BVEC-type iterator. + */ +static void __init iov_kunit_copy_to_bvec(struct kunit *test) +{ + const struct bvec_test_range *pr; + struct iov_iter iter; + struct bio_vec bvec[8]; + struct page **spages, **bpages; + u8 *scratch, *buffer; + size_t bufsize, npages, size, copied; + int i, b, patt; + + bufsize = 0x100000; + npages = bufsize / PAGE_SIZE; + + scratch = iov_kunit_create_buffer(test, &spages, npages); + for (i = 0; i < bufsize; i++) + scratch[i] = pattern(i); + + buffer = iov_kunit_create_buffer(test, &bpages, npages); + memset(buffer, 0, bufsize); + + iov_kunit_load_bvec(test, &iter, READ, bvec, ARRAY_SIZE(bvec), + bpages, npages, bufsize, bvec_test_ranges); + size = iter.count; + + copied = copy_to_iter(scratch, size, &iter); + + KUNIT_EXPECT_EQ(test, copied, size); + KUNIT_EXPECT_EQ(test, iter.count, 0); + KUNIT_EXPECT_EQ(test, iter.nr_segs, 0); + + /* Build the expected image in the scratch buffer. */ + b = 0; + patt = 0; + memset(scratch, 0, bufsize); + for (pr = bvec_test_ranges; pr->from >= 0; pr++, b++) { + u8 *p = scratch + pr->page * PAGE_SIZE; + + for (i = pr->from; i < pr->to; i++) + p[i] = pattern(patt++); + } + + /* Compare the images */ + for (i = 0; i < bufsize; i++) { + KUNIT_EXPECT_EQ_MSG(test, buffer[i], scratch[i], "at i=%x", i); + if (buffer[i] != scratch[i]) + return; + } + + KUNIT_SUCCEED(); +} + +/* + * Test copying from a ITER_BVEC-type iterator. + */ +static void __init iov_kunit_copy_from_bvec(struct kunit *test) +{ + const struct bvec_test_range *pr; + struct iov_iter iter; + struct bio_vec bvec[8]; + struct page **spages, **bpages; + u8 *scratch, *buffer; + size_t bufsize, npages, size, copied; + int i, j; + + bufsize = 0x100000; + npages = bufsize / PAGE_SIZE; + + buffer = iov_kunit_create_buffer(test, &bpages, npages); + for (i = 0; i < bufsize; i++) + buffer[i] = pattern(i); + + scratch = iov_kunit_create_buffer(test, &spages, npages); + memset(scratch, 0, bufsize); + + iov_kunit_load_bvec(test, &iter, WRITE, bvec, ARRAY_SIZE(bvec), + bpages, npages, bufsize, bvec_test_ranges); + size = iter.count; + + copied = copy_from_iter(scratch, size, &iter); + + KUNIT_EXPECT_EQ(test, copied, size); + KUNIT_EXPECT_EQ(test, iter.count, 0); + KUNIT_EXPECT_EQ(test, iter.nr_segs, 0); + + /* Build the expected image in the main buffer. */ + i = 0; + memset(buffer, 0, bufsize); + for (pr = bvec_test_ranges; pr->from >= 0; pr++) { + size_t patt = pr->page * PAGE_SIZE; + + for (j = pr->from; j < pr->to; j++) { + buffer[i++] = pattern(patt + j); + if (i >= bufsize) + goto stop; + } + } +stop: + + /* Compare the images */ + for (i = 0; i < bufsize; i++) { + KUNIT_EXPECT_EQ_MSG(test, scratch[i], buffer[i], "at i=%x", i); + if (scratch[i] != buffer[i]) + return; + } + + KUNIT_SUCCEED(); +} + +static void iov_kunit_destroy_xarray(void *data) +{ + struct xarray *xarray = data; + + xa_destroy(xarray); + kfree(xarray); +} + +static void __init iov_kunit_load_xarray(struct kunit *test, + struct iov_iter *iter, int dir, + struct xarray *xarray, + struct page **pages, size_t npages) +{ + size_t size = 0; + int i; + + for (i = 0; i < npages; i++) { + void *x = xa_store(xarray, i, pages[i], GFP_KERNEL); + + KUNIT_ASSERT_FALSE(test, xa_is_err(x)); + size += PAGE_SIZE; + } + iov_iter_xarray(iter, dir, xarray, 0, size); +} + +static struct xarray *iov_kunit_create_xarray(struct kunit *test) +{ + struct xarray *xarray; + + xarray = kzalloc(sizeof(struct xarray), GFP_KERNEL); + xa_init(xarray); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, xarray); + kunit_add_action_or_reset(test, iov_kunit_destroy_xarray, xarray); + return xarray; +} + +/* + * Test copying to a ITER_XARRAY-type iterator. + */ +static void __init iov_kunit_copy_to_xarray(struct kunit *test) +{ + const struct kvec_test_range *pr; + struct iov_iter iter; + struct xarray *xarray; + struct page **spages, **bpages; + u8 *scratch, *buffer; + size_t bufsize, npages, size, copied; + int i, patt; + + bufsize = 0x100000; + npages = bufsize / PAGE_SIZE; + + xarray = iov_kunit_create_xarray(test); + + scratch = iov_kunit_create_buffer(test, &spages, npages); + for (i = 0; i < bufsize; i++) + scratch[i] = pattern(i); + + buffer = iov_kunit_create_buffer(test, &bpages, npages); + memset(buffer, 0, bufsize); + + iov_kunit_load_xarray(test, &iter, READ, xarray, bpages, npages); + + i = 0; + for (pr = kvec_test_ranges; pr->from >= 0; pr++) { + size = pr->to - pr->from; + KUNIT_ASSERT_LE(test, pr->to, bufsize); + + iov_iter_xarray(&iter, READ, xarray, pr->from, size); + copied = copy_to_iter(scratch + i, size, &iter); + + KUNIT_EXPECT_EQ(test, copied, size); + KUNIT_EXPECT_EQ(test, iter.count, 0); + KUNIT_EXPECT_EQ(test, iter.iov_offset, size); + i += size; + } + + /* Build the expected image in the scratch buffer. */ + patt = 0; + memset(scratch, 0, bufsize); + for (pr = kvec_test_ranges; pr->from >= 0; pr++) + for (i = pr->from; i < pr->to; i++) + scratch[i] = pattern(patt++); + + /* Compare the images */ + for (i = 0; i < bufsize; i++) { + KUNIT_EXPECT_EQ_MSG(test, buffer[i], scratch[i], "at i=%x", i); + if (buffer[i] != scratch[i]) + return; + } + + KUNIT_SUCCEED(); +} + +/* + * Test copying from a ITER_XARRAY-type iterator. + */ +static void __init iov_kunit_copy_from_xarray(struct kunit *test) +{ + const struct kvec_test_range *pr; + struct iov_iter iter; + struct xarray *xarray; + struct page **spages, **bpages; + u8 *scratch, *buffer; + size_t bufsize, npages, size, copied; + int i, j; + + bufsize = 0x100000; + npages = bufsize / PAGE_SIZE; + + xarray = iov_kunit_create_xarray(test); + + buffer = iov_kunit_create_buffer(test, &bpages, npages); + for (i = 0; i < bufsize; i++) + buffer[i] = pattern(i); + + scratch = iov_kunit_create_buffer(test, &spages, npages); + memset(scratch, 0, bufsize); + + iov_kunit_load_xarray(test, &iter, READ, xarray, bpages, npages); + + i = 0; + for (pr = kvec_test_ranges; pr->from >= 0; pr++) { + size = pr->to - pr->from; + KUNIT_ASSERT_LE(test, pr->to, bufsize); + + iov_iter_xarray(&iter, WRITE, xarray, pr->from, size); + copied = copy_from_iter(scratch + i, size, &iter); + + KUNIT_EXPECT_EQ(test, copied, size); + KUNIT_EXPECT_EQ(test, iter.count, 0); + KUNIT_EXPECT_EQ(test, iter.iov_offset, size); + i += size; + } + + /* Build the expected image in the main buffer. */ + i = 0; + memset(buffer, 0, bufsize); + for (pr = kvec_test_ranges; pr->from >= 0; pr++) { + for (j = pr->from; j < pr->to; j++) { + buffer[i++] = pattern(j); + if (i >= bufsize) + goto stop; + } + } +stop: + + /* Compare the images */ + for (i = 0; i < bufsize; i++) { + KUNIT_EXPECT_EQ_MSG(test, scratch[i], buffer[i], "at i=%x", i); + if (scratch[i] != buffer[i]) + return; + } + + KUNIT_SUCCEED(); +} + +/* + * Test the extraction of ITER_KVEC-type iterators. + */ +static void __init iov_kunit_extract_pages_kvec(struct kunit *test) +{ + const struct kvec_test_range *pr; + struct iov_iter iter; + struct page **bpages, *pagelist[8], **pages = pagelist; + struct kvec kvec[8]; + u8 *buffer; + ssize_t len; + size_t bufsize, size = 0, npages; + int i, from; + + bufsize = 0x100000; + npages = bufsize / PAGE_SIZE; + + buffer = iov_kunit_create_buffer(test, &bpages, npages); + + iov_kunit_load_kvec(test, &iter, READ, kvec, ARRAY_SIZE(kvec), + buffer, bufsize, kvec_test_ranges); + size = iter.count; + + pr = kvec_test_ranges; + from = pr->from; + do { + size_t offset0 = LONG_MAX; + + for (i = 0; i < ARRAY_SIZE(pagelist); i++) + pagelist[i] = (void *)(unsigned long)0xaa55aa55aa55aa55ULL; + + len = iov_iter_extract_pages(&iter, &pages, 100 * 1024, + ARRAY_SIZE(pagelist), 0, &offset0); + KUNIT_EXPECT_GE(test, len, 0); + if (len < 0) + break; + KUNIT_EXPECT_GE(test, (ssize_t)offset0, 0); + KUNIT_EXPECT_LT(test, offset0, PAGE_SIZE); + KUNIT_EXPECT_LE(test, len, size); + KUNIT_EXPECT_EQ(test, iter.count, size - len); + size -= len; + + if (len == 0) + break; + + for (i = 0; i < ARRAY_SIZE(pagelist); i++) { + struct page *p; + ssize_t part = min_t(ssize_t, len, PAGE_SIZE - offset0); + int ix; + + KUNIT_ASSERT_GE(test, part, 0); + while (from == pr->to) { + pr++; + from = pr->from; + if (from < 0) + goto stop; + } + ix = from / PAGE_SIZE; + KUNIT_ASSERT_LT(test, ix, npages); + p = bpages[ix]; + KUNIT_EXPECT_PTR_EQ(test, pagelist[i], p); + KUNIT_EXPECT_EQ(test, offset0, from % PAGE_SIZE); + from += part; + len -= part; + KUNIT_ASSERT_GE(test, len, 0); + if (len == 0) + break; + offset0 = 0; + } + + if (test->status == KUNIT_FAILURE) + break; + } while (iov_iter_count(&iter) > 0); + +stop: + KUNIT_EXPECT_EQ(test, size, 0); + KUNIT_EXPECT_EQ(test, iter.count, 0); + KUNIT_SUCCEED(); +} + +/* + * Test the extraction of ITER_BVEC-type iterators. + */ +static void __init iov_kunit_extract_pages_bvec(struct kunit *test) +{ + const struct bvec_test_range *pr; + struct iov_iter iter; + struct page **bpages, *pagelist[8], **pages = pagelist; + struct bio_vec bvec[8]; + ssize_t len; + size_t bufsize, size = 0, npages; + int i, from; + + bufsize = 0x100000; + npages = bufsize / PAGE_SIZE; + + iov_kunit_create_buffer(test, &bpages, npages); + iov_kunit_load_bvec(test, &iter, READ, bvec, ARRAY_SIZE(bvec), + bpages, npages, bufsize, bvec_test_ranges); + size = iter.count; + + pr = bvec_test_ranges; + from = pr->from; + do { + size_t offset0 = LONG_MAX; + + for (i = 0; i < ARRAY_SIZE(pagelist); i++) + pagelist[i] = (void *)(unsigned long)0xaa55aa55aa55aa55ULL; + + len = iov_iter_extract_pages(&iter, &pages, 100 * 1024, + ARRAY_SIZE(pagelist), 0, &offset0); + KUNIT_EXPECT_GE(test, len, 0); + if (len < 0) + break; + KUNIT_EXPECT_GE(test, (ssize_t)offset0, 0); + KUNIT_EXPECT_LT(test, offset0, PAGE_SIZE); + KUNIT_EXPECT_LE(test, len, size); + KUNIT_EXPECT_EQ(test, iter.count, size - len); + size -= len; + + if (len == 0) + break; + + for (i = 0; i < ARRAY_SIZE(pagelist); i++) { + struct page *p; + ssize_t part = min_t(ssize_t, len, PAGE_SIZE - offset0); + int ix; + + KUNIT_ASSERT_GE(test, part, 0); + while (from == pr->to) { + pr++; + from = pr->from; + if (from < 0) + goto stop; + } + ix = pr->page + from / PAGE_SIZE; + KUNIT_ASSERT_LT(test, ix, npages); + p = bpages[ix]; + KUNIT_EXPECT_PTR_EQ(test, pagelist[i], p); + KUNIT_EXPECT_EQ(test, offset0, from % PAGE_SIZE); + from += part; + len -= part; + KUNIT_ASSERT_GE(test, len, 0); + if (len == 0) + break; + offset0 = 0; + } + + if (test->status == KUNIT_FAILURE) + break; + } while (iov_iter_count(&iter) > 0); + +stop: + KUNIT_EXPECT_EQ(test, size, 0); + KUNIT_EXPECT_EQ(test, iter.count, 0); + KUNIT_SUCCEED(); +} + +/* + * Test the extraction of ITER_XARRAY-type iterators. + */ +static void __init iov_kunit_extract_pages_xarray(struct kunit *test) +{ + const struct kvec_test_range *pr; + struct iov_iter iter; + struct xarray *xarray; + struct page **bpages, *pagelist[8], **pages = pagelist; + ssize_t len; + size_t bufsize, size = 0, npages; + int i, from; + + bufsize = 0x100000; + npages = bufsize / PAGE_SIZE; + + xarray = iov_kunit_create_xarray(test); + + iov_kunit_create_buffer(test, &bpages, npages); + iov_kunit_load_xarray(test, &iter, READ, xarray, bpages, npages); + + for (pr = kvec_test_ranges; pr->from >= 0; pr++) { + from = pr->from; + size = pr->to - from; + KUNIT_ASSERT_LE(test, pr->to, bufsize); + + iov_iter_xarray(&iter, WRITE, xarray, from, size); + + do { + size_t offset0 = LONG_MAX; + + for (i = 0; i < ARRAY_SIZE(pagelist); i++) + pagelist[i] = (void *)(unsigned long)0xaa55aa55aa55aa55ULL; + + len = iov_iter_extract_pages(&iter, &pages, 100 * 1024, + ARRAY_SIZE(pagelist), 0, &offset0); + KUNIT_EXPECT_GE(test, len, 0); + if (len < 0) + break; + KUNIT_EXPECT_LE(test, len, size); + KUNIT_EXPECT_EQ(test, iter.count, size - len); + if (len == 0) + break; + size -= len; + KUNIT_EXPECT_GE(test, (ssize_t)offset0, 0); + KUNIT_EXPECT_LT(test, offset0, PAGE_SIZE); + + for (i = 0; i < ARRAY_SIZE(pagelist); i++) { + struct page *p; + ssize_t part = min_t(ssize_t, len, PAGE_SIZE - offset0); + int ix; + + KUNIT_ASSERT_GE(test, part, 0); + ix = from / PAGE_SIZE; + KUNIT_ASSERT_LT(test, ix, npages); + p = bpages[ix]; + KUNIT_EXPECT_PTR_EQ(test, pagelist[i], p); + KUNIT_EXPECT_EQ(test, offset0, from % PAGE_SIZE); + from += part; + len -= part; + KUNIT_ASSERT_GE(test, len, 0); + if (len == 0) + break; + offset0 = 0; + } + + if (test->status == KUNIT_FAILURE) + goto stop; + } while (iov_iter_count(&iter) > 0); + + KUNIT_EXPECT_EQ(test, size, 0); + KUNIT_EXPECT_EQ(test, iter.count, 0); + KUNIT_EXPECT_EQ(test, iter.iov_offset, pr->to - pr->from); + } + +stop: + KUNIT_SUCCEED(); +} + +static struct kunit_case __refdata iov_kunit_cases[] = { + KUNIT_CASE(iov_kunit_copy_to_kvec), + KUNIT_CASE(iov_kunit_copy_from_kvec), + KUNIT_CASE(iov_kunit_copy_to_bvec), + KUNIT_CASE(iov_kunit_copy_from_bvec), + KUNIT_CASE(iov_kunit_copy_to_xarray), + KUNIT_CASE(iov_kunit_copy_from_xarray), + KUNIT_CASE(iov_kunit_extract_pages_kvec), + KUNIT_CASE(iov_kunit_extract_pages_bvec), + KUNIT_CASE(iov_kunit_extract_pages_xarray), + {} +}; + +static struct kunit_suite iov_kunit_suite = { + .name = "iov_iter", + .test_cases = iov_kunit_cases, +}; + +kunit_test_suites(&iov_kunit_suite); diff --git a/lib/list_debug.c b/lib/list_debug.c index d98d43f80958..db602417febf 100644 --- a/lib/list_debug.c +++ b/lib/list_debug.c @@ -2,7 +2,8 @@ * Copyright 2006, Red Hat, Inc., Dave Jones * Released under the General Public License (GPL). * - * This file contains the linked list validation for DEBUG_LIST. + * This file contains the linked list validation and error reporting for + * LIST_HARDENED and DEBUG_LIST. */ #include <linux/export.h> @@ -17,8 +18,9 @@ * attempt). */ -bool __list_add_valid(struct list_head *new, struct list_head *prev, - struct list_head *next) +__list_valid_slowpath +bool __list_add_valid_or_report(struct list_head *new, struct list_head *prev, + struct list_head *next) { if (CHECK_DATA_CORRUPTION(prev == NULL, "list_add corruption. prev is NULL.\n") || @@ -37,9 +39,10 @@ bool __list_add_valid(struct list_head *new, struct list_head *prev, return true; } -EXPORT_SYMBOL(__list_add_valid); +EXPORT_SYMBOL(__list_add_valid_or_report); -bool __list_del_entry_valid(struct list_head *entry) +__list_valid_slowpath +bool __list_del_entry_valid_or_report(struct list_head *entry) { struct list_head *prev, *next; @@ -65,6 +68,5 @@ bool __list_del_entry_valid(struct list_head *entry) return false; return true; - } -EXPORT_SYMBOL(__list_del_entry_valid); +EXPORT_SYMBOL(__list_del_entry_valid_or_report); diff --git a/lib/llist.c b/lib/llist.c index 6e668fa5a2c6..f21d0cfbbaaa 100644 --- a/lib/llist.c +++ b/lib/llist.c @@ -66,6 +66,34 @@ struct llist_node *llist_del_first(struct llist_head *head) EXPORT_SYMBOL_GPL(llist_del_first); /** + * llist_del_first_this - delete given entry of lock-less list if it is first + * @head: the head for your lock-less list + * @this: a list entry. + * + * If head of the list is given entry, delete and return %true else + * return %false. + * + * Multiple callers can safely call this concurrently with multiple + * llist_add() callers, providing all the callers offer a different @this. + */ +bool llist_del_first_this(struct llist_head *head, + struct llist_node *this) +{ + struct llist_node *entry, *next; + + /* acquire ensures orderig wrt try_cmpxchg() is llist_del_first() */ + entry = smp_load_acquire(&head->first); + do { + if (entry != this) + return false; + next = READ_ONCE(entry->next); + } while (!try_cmpxchg(&head->first, &entry, next)); + + return true; +} +EXPORT_SYMBOL_GPL(llist_del_first_this); + +/** * llist_reverse_order - reverse order of a llist chain * @head: first item of the list to be reversed * diff --git a/lib/locking-selftest.c b/lib/locking-selftest.c index 8d24279fad05..6f6a5fc85b42 100644 --- a/lib/locking-selftest.c +++ b/lib/locking-selftest.c @@ -2506,94 +2506,29 @@ static void fs_reclaim_tests(void) pr_cont("\n"); } -#define __guard(cleanup) __maybe_unused __attribute__((__cleanup__(cleanup))) +/* Defines guard classes to create contexts */ +DEFINE_LOCK_GUARD_0(HARDIRQ, HARDIRQ_ENTER(), HARDIRQ_EXIT()) +DEFINE_LOCK_GUARD_0(NOTTHREADED_HARDIRQ, + do { + local_irq_disable(); + __irq_enter(); + WARN_ON(!in_irq()); + } while(0), HARDIRQ_EXIT()) +DEFINE_LOCK_GUARD_0(SOFTIRQ, SOFTIRQ_ENTER(), SOFTIRQ_EXIT()) + +/* Define RCU guards, should go away when RCU has its own guard definitions */ +DEFINE_LOCK_GUARD_0(RCU, rcu_read_lock(), rcu_read_unlock()) +DEFINE_LOCK_GUARD_0(RCU_BH, rcu_read_lock_bh(), rcu_read_unlock_bh()) +DEFINE_LOCK_GUARD_0(RCU_SCHED, rcu_read_lock_sched(), rcu_read_unlock_sched()) -static void hardirq_exit(int *_) -{ - HARDIRQ_EXIT(); -} - -#define HARDIRQ_CONTEXT(name, ...) \ - int hardirq_guard_##name __guard(hardirq_exit); \ - HARDIRQ_ENTER(); - -#define NOTTHREADED_HARDIRQ_CONTEXT(name, ...) \ - int notthreaded_hardirq_guard_##name __guard(hardirq_exit); \ - local_irq_disable(); \ - __irq_enter(); \ - WARN_ON(!in_irq()); - -static void softirq_exit(int *_) -{ - SOFTIRQ_EXIT(); -} - -#define SOFTIRQ_CONTEXT(name, ...) \ - int softirq_guard_##name __guard(softirq_exit); \ - SOFTIRQ_ENTER(); - -static void rcu_exit(int *_) -{ - rcu_read_unlock(); -} - -#define RCU_CONTEXT(name, ...) \ - int rcu_guard_##name __guard(rcu_exit); \ - rcu_read_lock(); - -static void rcu_bh_exit(int *_) -{ - rcu_read_unlock_bh(); -} - -#define RCU_BH_CONTEXT(name, ...) \ - int rcu_bh_guard_##name __guard(rcu_bh_exit); \ - rcu_read_lock_bh(); - -static void rcu_sched_exit(int *_) -{ - rcu_read_unlock_sched(); -} - -#define RCU_SCHED_CONTEXT(name, ...) \ - int rcu_sched_guard_##name __guard(rcu_sched_exit); \ - rcu_read_lock_sched(); - -static void raw_spinlock_exit(raw_spinlock_t **lock) -{ - raw_spin_unlock(*lock); -} - -#define RAW_SPINLOCK_CONTEXT(name, lock) \ - raw_spinlock_t *raw_spinlock_guard_##name __guard(raw_spinlock_exit) = &(lock); \ - raw_spin_lock(&(lock)); - -static void spinlock_exit(spinlock_t **lock) -{ - spin_unlock(*lock); -} - -#define SPINLOCK_CONTEXT(name, lock) \ - spinlock_t *spinlock_guard_##name __guard(spinlock_exit) = &(lock); \ - spin_lock(&(lock)); - -static void mutex_exit(struct mutex **lock) -{ - mutex_unlock(*lock); -} - -#define MUTEX_CONTEXT(name, lock) \ - struct mutex *mutex_guard_##name __guard(mutex_exit) = &(lock); \ - mutex_lock(&(lock)); #define GENERATE_2_CONTEXT_TESTCASE(outer, outer_lock, inner, inner_lock) \ \ static void __maybe_unused inner##_in_##outer(void) \ { \ - outer##_CONTEXT(_, outer_lock); \ - { \ - inner##_CONTEXT(_, inner_lock); \ - } \ + /* Relies the reversed clean-up ordering: inner first */ \ + guard(outer)(outer_lock); \ + guard(inner)(inner_lock); \ } /* @@ -2632,21 +2567,21 @@ GENERATE_2_CONTEXT_TESTCASE(SOFTIRQ, , inner, inner_lock) \ GENERATE_2_CONTEXT_TESTCASE(RCU, , inner, inner_lock) \ GENERATE_2_CONTEXT_TESTCASE(RCU_BH, , inner, inner_lock) \ GENERATE_2_CONTEXT_TESTCASE(RCU_SCHED, , inner, inner_lock) \ -GENERATE_2_CONTEXT_TESTCASE(RAW_SPINLOCK, raw_lock_A, inner, inner_lock) \ -GENERATE_2_CONTEXT_TESTCASE(SPINLOCK, lock_A, inner, inner_lock) \ -GENERATE_2_CONTEXT_TESTCASE(MUTEX, mutex_A, inner, inner_lock) +GENERATE_2_CONTEXT_TESTCASE(raw_spinlock, &raw_lock_A, inner, inner_lock) \ +GENERATE_2_CONTEXT_TESTCASE(spinlock, &lock_A, inner, inner_lock) \ +GENERATE_2_CONTEXT_TESTCASE(mutex, &mutex_A, inner, inner_lock) GENERATE_2_CONTEXT_TESTCASE_FOR_ALL_OUTER(RCU, ) -GENERATE_2_CONTEXT_TESTCASE_FOR_ALL_OUTER(RAW_SPINLOCK, raw_lock_B) -GENERATE_2_CONTEXT_TESTCASE_FOR_ALL_OUTER(SPINLOCK, lock_B) -GENERATE_2_CONTEXT_TESTCASE_FOR_ALL_OUTER(MUTEX, mutex_B) +GENERATE_2_CONTEXT_TESTCASE_FOR_ALL_OUTER(raw_spinlock, &raw_lock_B) +GENERATE_2_CONTEXT_TESTCASE_FOR_ALL_OUTER(spinlock, &lock_B) +GENERATE_2_CONTEXT_TESTCASE_FOR_ALL_OUTER(mutex, &mutex_B) /* the outer context allows all kinds of preemption */ #define DO_CONTEXT_TESTCASE_OUTER_PREEMPTIBLE(outer) \ dotest(RCU_in_##outer, SUCCESS, LOCKTYPE_RWLOCK); \ - dotest(RAW_SPINLOCK_in_##outer, SUCCESS, LOCKTYPE_SPIN); \ - dotest(SPINLOCK_in_##outer, SUCCESS, LOCKTYPE_SPIN); \ - dotest(MUTEX_in_##outer, SUCCESS, LOCKTYPE_MUTEX); \ + dotest(raw_spinlock_in_##outer, SUCCESS, LOCKTYPE_SPIN); \ + dotest(spinlock_in_##outer, SUCCESS, LOCKTYPE_SPIN); \ + dotest(mutex_in_##outer, SUCCESS, LOCKTYPE_MUTEX); \ /* * the outer context only allows the preemption introduced by spinlock_t (which @@ -2654,16 +2589,16 @@ GENERATE_2_CONTEXT_TESTCASE_FOR_ALL_OUTER(MUTEX, mutex_B) */ #define DO_CONTEXT_TESTCASE_OUTER_LIMITED_PREEMPTIBLE(outer) \ dotest(RCU_in_##outer, SUCCESS, LOCKTYPE_RWLOCK); \ - dotest(RAW_SPINLOCK_in_##outer, SUCCESS, LOCKTYPE_SPIN); \ - dotest(SPINLOCK_in_##outer, SUCCESS, LOCKTYPE_SPIN); \ - dotest(MUTEX_in_##outer, FAILURE, LOCKTYPE_MUTEX); \ + dotest(raw_spinlock_in_##outer, SUCCESS, LOCKTYPE_SPIN); \ + dotest(spinlock_in_##outer, SUCCESS, LOCKTYPE_SPIN); \ + dotest(mutex_in_##outer, FAILURE, LOCKTYPE_MUTEX); \ /* the outer doesn't allows any kind of preemption */ #define DO_CONTEXT_TESTCASE_OUTER_NOT_PREEMPTIBLE(outer) \ dotest(RCU_in_##outer, SUCCESS, LOCKTYPE_RWLOCK); \ - dotest(RAW_SPINLOCK_in_##outer, SUCCESS, LOCKTYPE_SPIN); \ - dotest(SPINLOCK_in_##outer, FAILURE, LOCKTYPE_SPIN); \ - dotest(MUTEX_in_##outer, FAILURE, LOCKTYPE_MUTEX); \ + dotest(raw_spinlock_in_##outer, SUCCESS, LOCKTYPE_SPIN); \ + dotest(spinlock_in_##outer, FAILURE, LOCKTYPE_SPIN); \ + dotest(mutex_in_##outer, FAILURE, LOCKTYPE_MUTEX); \ static void wait_context_tests(void) { @@ -2697,15 +2632,15 @@ static void wait_context_tests(void) pr_cont("\n"); print_testname("in RAW_SPINLOCK context"); - DO_CONTEXT_TESTCASE_OUTER_NOT_PREEMPTIBLE(RAW_SPINLOCK); + DO_CONTEXT_TESTCASE_OUTER_NOT_PREEMPTIBLE(raw_spinlock); pr_cont("\n"); print_testname("in SPINLOCK context"); - DO_CONTEXT_TESTCASE_OUTER_LIMITED_PREEMPTIBLE(SPINLOCK); + DO_CONTEXT_TESTCASE_OUTER_LIMITED_PREEMPTIBLE(spinlock); pr_cont("\n"); print_testname("in MUTEX context"); - DO_CONTEXT_TESTCASE_OUTER_PREEMPTIBLE(MUTEX); + DO_CONTEXT_TESTCASE_OUTER_PREEMPTIBLE(mutex); pr_cont("\n"); } diff --git a/lib/logic_pio.c b/lib/logic_pio.c index 07b4b9a1f54b..2ea564a40064 100644 --- a/lib/logic_pio.c +++ b/lib/logic_pio.c @@ -20,9 +20,6 @@ static LIST_HEAD(io_range_list); static DEFINE_MUTEX(io_range_mutex); -/* Consider a kernel general helper for this */ -#define in_range(b, first, len) ((b) >= (first) && (b) < (first) + (len)) - /** * logic_pio_register_range - register logical PIO range for a host * @new_range: pointer to the IO range to be registered. diff --git a/lib/lwq.c b/lib/lwq.c new file mode 100644 index 000000000000..57d080a4d53d --- /dev/null +++ b/lib/lwq.c @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Light-weight single-linked queue. + * + * Entries are enqueued to the head of an llist, with no blocking. + * This can happen in any context. + * + * Entries are dequeued using a spinlock to protect against multiple + * access. The llist is staged in reverse order, and refreshed + * from the llist when it exhausts. + * + * This is particularly suitable when work items are queued in BH or + * IRQ context, and where work items are handled one at a time by + * dedicated threads. + */ +#include <linux/rcupdate.h> +#include <linux/lwq.h> + +struct llist_node *__lwq_dequeue(struct lwq *q) +{ + struct llist_node *this; + + if (lwq_empty(q)) + return NULL; + spin_lock(&q->lock); + this = q->ready; + if (!this && !llist_empty(&q->new)) { + /* ensure queue doesn't appear transiently lwq_empty */ + smp_store_release(&q->ready, (void *)1); + this = llist_reverse_order(llist_del_all(&q->new)); + if (!this) + q->ready = NULL; + } + if (this) + q->ready = llist_next(this); + spin_unlock(&q->lock); + return this; +} +EXPORT_SYMBOL_GPL(__lwq_dequeue); + +/** + * lwq_dequeue_all - dequeue all currently enqueued objects + * @q: the queue to dequeue from + * + * Remove and return a linked list of llist_nodes of all the objects that were + * in the queue. The first on the list will be the object that was least + * recently enqueued. + */ +struct llist_node *lwq_dequeue_all(struct lwq *q) +{ + struct llist_node *r, *t, **ep; + + if (lwq_empty(q)) + return NULL; + + spin_lock(&q->lock); + r = q->ready; + q->ready = NULL; + t = llist_del_all(&q->new); + spin_unlock(&q->lock); + ep = &r; + while (*ep) + ep = &(*ep)->next; + *ep = llist_reverse_order(t); + return r; +} +EXPORT_SYMBOL_GPL(lwq_dequeue_all); + +#if IS_ENABLED(CONFIG_LWQ_TEST) + +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/wait_bit.h> +#include <linux/kthread.h> +#include <linux/delay.h> +struct tnode { + struct lwq_node n; + int i; + int c; +}; + +static int lwq_exercise(void *qv) +{ + struct lwq *q = qv; + int cnt; + struct tnode *t; + + for (cnt = 0; cnt < 10000; cnt++) { + wait_var_event(q, (t = lwq_dequeue(q, struct tnode, n)) != NULL); + t->c++; + if (lwq_enqueue(&t->n, q)) + wake_up_var(q); + } + while (!kthread_should_stop()) + schedule_timeout_idle(1); + return 0; +} + +static int lwq_test(void) +{ + int i; + struct lwq q; + struct llist_node *l, **t1, *t2; + struct tnode *t; + struct task_struct *threads[8]; + + printk(KERN_INFO "testing lwq....\n"); + lwq_init(&q); + printk(KERN_INFO " lwq: run some threads\n"); + for (i = 0; i < ARRAY_SIZE(threads); i++) + threads[i] = kthread_run(lwq_exercise, &q, "lwq-test-%d", i); + for (i = 0; i < 100; i++) { + t = kmalloc(sizeof(*t), GFP_KERNEL); + if (!t) + break; + t->i = i; + t->c = 0; + if (lwq_enqueue(&t->n, &q)) + wake_up_var(&q); + } + /* wait for threads to exit */ + for (i = 0; i < ARRAY_SIZE(threads); i++) + if (!IS_ERR_OR_NULL(threads[i])) + kthread_stop(threads[i]); + printk(KERN_INFO " lwq: dequeue first 50:"); + for (i = 0; i < 50 ; i++) { + if (i && (i % 10) == 0) { + printk(KERN_CONT "\n"); + printk(KERN_INFO " lwq: ... "); + } + t = lwq_dequeue(&q, struct tnode, n); + if (t) + printk(KERN_CONT " %d(%d)", t->i, t->c); + kfree(t); + } + printk(KERN_CONT "\n"); + l = lwq_dequeue_all(&q); + printk(KERN_INFO " lwq: delete the multiples of 3 (test lwq_for_each_safe())\n"); + lwq_for_each_safe(t, t1, t2, &l, n) { + if ((t->i % 3) == 0) { + t->i = -1; + kfree(t); + t = NULL; + } + } + if (l) + lwq_enqueue_batch(l, &q); + printk(KERN_INFO " lwq: dequeue remaining:"); + while ((t = lwq_dequeue(&q, struct tnode, n)) != NULL) { + printk(KERN_CONT " %d", t->i); + kfree(t); + } + printk(KERN_CONT "\n"); + return 0; +} + +module_init(lwq_test); +#endif /* CONFIG_LWQ_TEST*/ diff --git a/lib/maple_tree.c b/lib/maple_tree.c index f723024e1426..bb24d84a4922 100644 --- a/lib/maple_tree.c +++ b/lib/maple_tree.c @@ -75,6 +75,7 @@ #define MA_STATE_PREALLOC 4 #define ma_parent_ptr(x) ((struct maple_pnode *)(x)) +#define mas_tree_parent(x) ((unsigned long)(x->tree) | MA_ROOT_PARENT) #define ma_mnode_ptr(x) ((struct maple_node *)(x)) #define ma_enode_ptr(x) ((struct maple_enode *)(x)) static struct kmem_cache *maple_node_cache; @@ -255,6 +256,22 @@ bool mas_is_err(struct ma_state *mas) return xa_is_err(mas->node); } +static __always_inline bool mas_is_overflow(struct ma_state *mas) +{ + if (unlikely(mas->node == MAS_OVERFLOW)) + return true; + + return false; +} + +static __always_inline bool mas_is_underflow(struct ma_state *mas) +{ + if (unlikely(mas->node == MAS_UNDERFLOW)) + return true; + + return false; +} + static inline bool mas_searchable(struct ma_state *mas) { if (mas_is_none(mas)) @@ -729,33 +746,6 @@ mas_safe_min(struct ma_state *mas, unsigned long *pivots, unsigned char offset) } /* - * mas_logical_pivot() - Get the logical pivot of a given offset. - * @mas: The maple state - * @pivots: The pointer to the maple node pivots - * @offset: The offset into the pivot array - * @type: The maple node type - * - * When there is no value at a pivot (beyond the end of the data), then the - * pivot is actually @mas->max. - * - * Return: the logical pivot of a given @offset. - */ -static inline unsigned long -mas_logical_pivot(struct ma_state *mas, unsigned long *pivots, - unsigned char offset, enum maple_type type) -{ - unsigned long lpiv = mas_safe_pivot(mas, pivots, offset, type); - - if (likely(lpiv)) - return lpiv; - - if (likely(offset)) - return mas->max; - - return lpiv; -} - -/* * mte_set_pivot() - Set a pivot to a value in an encoded maple node. * @mn: The encoded maple node * @piv: The pivot offset @@ -804,6 +794,12 @@ static inline void __rcu **ma_slots(struct maple_node *mn, enum maple_type mt) } } +static inline bool mt_write_locked(const struct maple_tree *mt) +{ + return mt_external_lock(mt) ? mt_write_lock_is_held(mt) : + lockdep_is_held(&mt->ma_lock); +} + static inline bool mt_locked(const struct maple_tree *mt) { return mt_external_lock(mt) ? mt_lock_is_held(mt) : @@ -819,7 +815,7 @@ static inline void *mt_slot(const struct maple_tree *mt, static inline void *mt_slot_locked(struct maple_tree *mt, void __rcu **slots, unsigned char offset) { - return rcu_dereference_protected(slots[offset], mt_locked(mt)); + return rcu_dereference_protected(slots[offset], mt_write_locked(mt)); } /* * mas_slot_locked() - Get the slot value when holding the maple tree lock. @@ -862,7 +858,7 @@ static inline void *mas_root(struct ma_state *mas) static inline void *mt_root_locked(struct maple_tree *mt) { - return rcu_dereference_protected(mt->ma_root, mt_locked(mt)); + return rcu_dereference_protected(mt->ma_root, mt_write_locked(mt)); } /* @@ -1002,27 +998,9 @@ static inline void mat_add(struct ma_topiary *mat, mat->tail = dead_enode; } -static void mte_destroy_walk(struct maple_enode *, struct maple_tree *); -static inline void mas_free(struct ma_state *mas, struct maple_enode *used); - -/* - * mas_mat_free() - Free all nodes in a dead list. - * @mas - the maple state - * @mat - the ma_topiary linked list of dead nodes to free. - * - * Free walk a dead list. - */ -static void mas_mat_free(struct ma_state *mas, struct ma_topiary *mat) -{ - struct maple_enode *next; - - while (mat->head) { - next = mte_to_mat(mat->head)->next; - mas_free(mas, mat->head); - mat->head = next; - } -} - +static void mt_free_walk(struct rcu_head *head); +static void mt_destroy_walk(struct maple_enode *enode, struct maple_tree *mt, + bool free); /* * mas_mat_destroy() - Free all nodes and subtrees in a dead list. * @mas - the maple state @@ -1033,10 +1011,15 @@ static void mas_mat_free(struct ma_state *mas, struct ma_topiary *mat) static void mas_mat_destroy(struct ma_state *mas, struct ma_topiary *mat) { struct maple_enode *next; + struct maple_node *node; + bool in_rcu = mt_in_rcu(mas->tree); while (mat->head) { next = mte_to_mat(mat->head)->next; - mte_destroy_walk(mat->head, mat->mtree); + node = mte_to_node(mat->head); + mt_destroy_walk(mat->head, mas->tree, !in_rcu); + if (in_rcu) + call_rcu(&node->rcu, mt_free_walk); mat->head = next; } } @@ -1610,8 +1593,6 @@ ma_max_gap(struct maple_node *node, unsigned long *gaps, enum maple_type mt, * mas_max_gap() - find the largest gap in a non-leaf node and set the slot. * @mas: The maple state. * - * If the metadata gap is set to MAPLE_ARANGE64_META_MAX, there is no gap. - * * Return: The gap value. */ static inline unsigned long mas_max_gap(struct ma_state *mas) @@ -1628,9 +1609,6 @@ static inline unsigned long mas_max_gap(struct ma_state *mas) node = mas_mn(mas); MAS_BUG_ON(mas, mt != maple_arange_64); offset = ma_meta_gap(node, mt); - if (offset == MAPLE_ARANGE64_META_MAX) - return 0; - gaps = ma_gaps(node, mt); return gaps[offset]; } @@ -1662,10 +1640,7 @@ static inline void mas_parent_gap(struct ma_state *mas, unsigned char offset, ascend: MAS_BUG_ON(mas, pmt != maple_arange_64); meta_offset = ma_meta_gap(pnode, pmt); - if (meta_offset == MAPLE_ARANGE64_META_MAX) - meta_gap = 0; - else - meta_gap = pgaps[meta_offset]; + meta_gap = pgaps[meta_offset]; pgaps[offset] = new; @@ -1678,7 +1653,6 @@ ascend: ma_set_meta_gap(pnode, pmt, offset); } else if (new < meta_gap) { - meta_offset = 15; new = ma_max_gap(pnode, pgaps, pmt, &meta_offset); ma_set_meta_gap(pnode, pmt, meta_offset); } @@ -1731,7 +1705,7 @@ static inline void mas_adopt_children(struct ma_state *mas, struct maple_enode *parent) { enum maple_type type = mte_node_type(parent); - struct maple_node *node = mas_mn(mas); + struct maple_node *node = mte_to_node(parent); void __rcu **slots = ma_slots(node, type); unsigned long *pivots = ma_pivots(node, type); struct maple_enode *child; @@ -1745,53 +1719,54 @@ static inline void mas_adopt_children(struct ma_state *mas, } /* - * mas_replace() - Replace a maple node in the tree with mas->node. Uses the - * parent encoding to locate the maple node in the tree. - * @mas - the ma_state to use for operations. - * @advanced - boolean to adopt the child nodes and free the old node (false) or - * leave the node (true) and handle the adoption and free elsewhere. + * mas_put_in_tree() - Put a new node in the tree, smp_wmb(), and mark the old + * node as dead. + * @mas - the maple state with the new node + * @old_enode - The old maple encoded node to replace. */ -static inline void mas_replace(struct ma_state *mas, bool advanced) +static inline void mas_put_in_tree(struct ma_state *mas, + struct maple_enode *old_enode) __must_hold(mas->tree->ma_lock) { - struct maple_node *mn = mas_mn(mas); - struct maple_enode *old_enode; - unsigned char offset = 0; - void __rcu **slots = NULL; - - if (ma_is_root(mn)) { - old_enode = mas_root_locked(mas); - } else { - offset = mte_parent_slot(mas->node); - slots = ma_slots(mte_parent(mas->node), - mas_parent_type(mas, mas->node)); - old_enode = mas_slot_locked(mas, slots, offset); - } - - if (!advanced && !mte_is_leaf(mas->node)) - mas_adopt_children(mas, mas->node); + unsigned char offset; + void __rcu **slots; if (mte_is_root(mas->node)) { - mn->parent = ma_parent_ptr( - ((unsigned long)mas->tree | MA_ROOT_PARENT)); + mas_mn(mas)->parent = ma_parent_ptr(mas_tree_parent(mas)); rcu_assign_pointer(mas->tree->ma_root, mte_mk_root(mas->node)); mas_set_height(mas); } else { + + offset = mte_parent_slot(mas->node); + slots = ma_slots(mte_parent(mas->node), + mas_parent_type(mas, mas->node)); rcu_assign_pointer(slots[offset], mas->node); } - if (!advanced) { - mte_set_node_dead(old_enode); - mas_free(mas, old_enode); - } + mte_set_node_dead(old_enode); } /* - * mas_new_child() - Find the new child of a node. - * @mas: the maple state + * mas_replace_node() - Replace a node by putting it in the tree, marking it + * dead, and freeing it. + * the parent encoding to locate the maple node in the tree. + * @mas - the ma_state with @mas->node pointing to the new node. + * @old_enode - The old maple encoded node. + */ +static inline void mas_replace_node(struct ma_state *mas, + struct maple_enode *old_enode) + __must_hold(mas->tree->ma_lock) +{ + mas_put_in_tree(mas, old_enode); + mas_free(mas, old_enode); +} + +/* + * mas_find_child() - Find a child who has the parent @mas->node. + * @mas: the maple state with the parent. * @child: the maple state to store the child. */ -static inline bool mas_new_child(struct ma_state *mas, struct ma_state *child) +static inline bool mas_find_child(struct ma_state *mas, struct ma_state *child) __must_hold(mas->tree->ma_lock) { enum maple_type mt; @@ -2076,7 +2051,7 @@ static inline void mab_mas_cp(struct maple_big_node *b_node, end = j - 1; if (likely(!ma_is_leaf(mt) && mt_is_alloc(mas->tree))) { unsigned long max_gap = 0; - unsigned char offset = 15; + unsigned char offset = 0; gaps = ma_gaps(node, mt); do { @@ -2094,56 +2069,6 @@ static inline void mab_mas_cp(struct maple_big_node *b_node, } /* - * mas_descend_adopt() - Descend through a sub-tree and adopt children. - * @mas: the maple state with the maple encoded node of the sub-tree. - * - * Descend through a sub-tree and adopt children who do not have the correct - * parents set. Follow the parents which have the correct parents as they are - * the new entries which need to be followed to find other incorrectly set - * parents. - */ -static inline void mas_descend_adopt(struct ma_state *mas) -{ - struct ma_state list[3], next[3]; - int i, n; - - /* - * At each level there may be up to 3 correct parent pointers which indicates - * the new nodes which need to be walked to find any new nodes at a lower level. - */ - - for (i = 0; i < 3; i++) { - list[i] = *mas; - list[i].offset = 0; - next[i].offset = 0; - } - next[0] = *mas; - - while (!mte_is_leaf(list[0].node)) { - n = 0; - for (i = 0; i < 3; i++) { - if (mas_is_none(&list[i])) - continue; - - if (i && list[i-1].node == list[i].node) - continue; - - while ((n < 3) && (mas_new_child(&list[i], &next[n]))) - n++; - - mas_adopt_children(&list[i], list[i].node); - } - - while (n < 3) - next[n++].node = MAS_NONE; - - /* descend by setting the list to the children */ - for (i = 0; i < 3; i++) - list[i] = next[i]; - } -} - -/* * mas_bulk_rebalance() - Rebalance the end of a tree after a bulk insert. * @mas: The maple state * @end: The maple node end @@ -2211,7 +2136,7 @@ static noinline_for_kasan void mas_store_b_node(struct ma_wr_state *wr_mas, goto b_end; /* Handle new range ending before old range ends */ - piv = mas_logical_pivot(mas, wr_mas->pivots, offset_end, wr_mas->type); + piv = mas_safe_pivot(mas, wr_mas->pivots, offset_end, wr_mas->type); if (piv > mas->last) { if (piv == ULONG_MAX) mas_bulk_rebalance(mas, b_node->b_end, wr_mas->type); @@ -2333,98 +2258,6 @@ static inline void mas_wr_node_walk(struct ma_wr_state *wr_mas) } /* - * mas_topiary_range() - Add a range of slots to the topiary. - * @mas: The maple state - * @destroy: The topiary to add the slots (usually destroy) - * @start: The starting slot inclusively - * @end: The end slot inclusively - */ -static inline void mas_topiary_range(struct ma_state *mas, - struct ma_topiary *destroy, unsigned char start, unsigned char end) -{ - void __rcu **slots; - unsigned char offset; - - MAS_BUG_ON(mas, mte_is_leaf(mas->node)); - - slots = ma_slots(mas_mn(mas), mte_node_type(mas->node)); - for (offset = start; offset <= end; offset++) { - struct maple_enode *enode = mas_slot_locked(mas, slots, offset); - - if (mte_dead_node(enode)) - continue; - - mat_add(destroy, enode); - } -} - -/* - * mast_topiary() - Add the portions of the tree to the removal list; either to - * be freed or discarded (destroy walk). - * @mast: The maple_subtree_state. - */ -static inline void mast_topiary(struct maple_subtree_state *mast) -{ - MA_WR_STATE(wr_mas, mast->orig_l, NULL); - unsigned char r_start, r_end; - unsigned char l_start, l_end; - void __rcu **l_slots, **r_slots; - - wr_mas.type = mte_node_type(mast->orig_l->node); - mast->orig_l->index = mast->orig_l->last; - mas_wr_node_walk(&wr_mas); - l_start = mast->orig_l->offset + 1; - l_end = mas_data_end(mast->orig_l); - r_start = 0; - r_end = mast->orig_r->offset; - - if (r_end) - r_end--; - - l_slots = ma_slots(mas_mn(mast->orig_l), - mte_node_type(mast->orig_l->node)); - - r_slots = ma_slots(mas_mn(mast->orig_r), - mte_node_type(mast->orig_r->node)); - - if ((l_start < l_end) && - mte_dead_node(mas_slot_locked(mast->orig_l, l_slots, l_start))) { - l_start++; - } - - if (mte_dead_node(mas_slot_locked(mast->orig_r, r_slots, r_end))) { - if (r_end) - r_end--; - } - - if ((l_start > r_end) && (mast->orig_l->node == mast->orig_r->node)) - return; - - /* At the node where left and right sides meet, add the parts between */ - if (mast->orig_l->node == mast->orig_r->node) { - return mas_topiary_range(mast->orig_l, mast->destroy, - l_start, r_end); - } - - /* mast->orig_r is different and consumed. */ - if (mte_is_leaf(mast->orig_r->node)) - return; - - if (mte_dead_node(mas_slot_locked(mast->orig_l, l_slots, l_end))) - l_end--; - - - if (l_start <= l_end) - mas_topiary_range(mast->orig_l, mast->destroy, l_start, l_end); - - if (mte_dead_node(mas_slot_locked(mast->orig_r, r_slots, r_start))) - r_start++; - - if (r_start <= r_end) - mas_topiary_range(mast->orig_r, mast->destroy, 0, r_end); -} - -/* * mast_rebalance_next() - Rebalance against the next node * @mast: The maple subtree state * @old_r: The encoded maple node to the right (next node). @@ -2459,7 +2292,7 @@ static inline void mast_rebalance_prev(struct maple_subtree_state *mast) /* * mast_spanning_rebalance() - Rebalance nodes with nearest neighbour favouring * the node to the right. Checking the nodes to the right then the left at each - * level upwards until root is reached. Free and destroy as needed. + * level upwards until root is reached. * Data is copied into the @mast->bn. * @mast: The maple_subtree_state. */ @@ -2468,8 +2301,6 @@ bool mast_spanning_rebalance(struct maple_subtree_state *mast) { struct ma_state r_tmp = *mast->orig_r; struct ma_state l_tmp = *mast->orig_l; - struct maple_enode *ancestor = NULL; - unsigned char start, end; unsigned char depth = 0; r_tmp = *mast->orig_r; @@ -2478,87 +2309,25 @@ bool mast_spanning_rebalance(struct maple_subtree_state *mast) mas_ascend(mast->orig_r); mas_ascend(mast->orig_l); depth++; - if (!ancestor && - (mast->orig_r->node == mast->orig_l->node)) { - ancestor = mast->orig_r->node; - end = mast->orig_r->offset - 1; - start = mast->orig_l->offset + 1; - } - if (mast->orig_r->offset < mas_data_end(mast->orig_r)) { - if (!ancestor) { - ancestor = mast->orig_r->node; - start = 0; - } - mast->orig_r->offset++; do { mas_descend(mast->orig_r); mast->orig_r->offset = 0; - depth--; - } while (depth); + } while (--depth); mast_rebalance_next(mast); - do { - unsigned char l_off = 0; - struct maple_enode *child = r_tmp.node; - - mas_ascend(&r_tmp); - if (ancestor == r_tmp.node) - l_off = start; - - if (r_tmp.offset) - r_tmp.offset--; - - if (l_off < r_tmp.offset) - mas_topiary_range(&r_tmp, mast->destroy, - l_off, r_tmp.offset); - - if (l_tmp.node != child) - mat_add(mast->free, child); - - } while (r_tmp.node != ancestor); - *mast->orig_l = l_tmp; return true; - } else if (mast->orig_l->offset != 0) { - if (!ancestor) { - ancestor = mast->orig_l->node; - end = mas_data_end(mast->orig_l); - } - mast->orig_l->offset--; do { mas_descend(mast->orig_l); mast->orig_l->offset = mas_data_end(mast->orig_l); - depth--; - } while (depth); + } while (--depth); mast_rebalance_prev(mast); - do { - unsigned char r_off; - struct maple_enode *child = l_tmp.node; - - mas_ascend(&l_tmp); - if (ancestor == l_tmp.node) - r_off = end; - else - r_off = mas_data_end(&l_tmp); - - if (l_tmp.offset < r_off) - l_tmp.offset++; - - if (l_tmp.offset < r_off) - mas_topiary_range(&l_tmp, mast->destroy, - l_tmp.offset, r_off); - - if (r_tmp.node != child) - mat_add(mast->free, child); - - } while (l_tmp.node != ancestor); - *mast->orig_r = r_tmp; return true; } @@ -2570,36 +2339,24 @@ bool mast_spanning_rebalance(struct maple_subtree_state *mast) } /* - * mast_ascend_free() - Add current original maple state nodes to the free list - * and ascend. + * mast_ascend() - Ascend the original left and right maple states. * @mast: the maple subtree state. * - * Ascend the original left and right sides and add the previous nodes to the - * free list. Set the slots to point to the correct location in the new nodes. + * Ascend the original left and right sides. Set the offsets to point to the + * data already in the new tree (@mast->l and @mast->r). */ -static inline void -mast_ascend_free(struct maple_subtree_state *mast) +static inline void mast_ascend(struct maple_subtree_state *mast) { MA_WR_STATE(wr_mas, mast->orig_r, NULL); - struct maple_enode *left = mast->orig_l->node; - struct maple_enode *right = mast->orig_r->node; - mas_ascend(mast->orig_l); mas_ascend(mast->orig_r); - mat_add(mast->free, left); - - if (left != right) - mat_add(mast->free, right); mast->orig_r->offset = 0; mast->orig_r->index = mast->r->max; /* last should be larger than or equal to index */ if (mast->orig_r->last < mast->orig_r->index) mast->orig_r->last = mast->orig_r->index; - /* - * The node may not contain the value so set slot to ensure all - * of the nodes contents are freed or destroyed. - */ + wr_mas.type = mte_node_type(mast->orig_r->node); mas_wr_node_walk(&wr_mas); /* Set up the left side of things */ @@ -2778,58 +2535,152 @@ static inline void mast_set_split_parents(struct maple_subtree_state *mast, } /* - * mas_wmb_replace() - Write memory barrier and replace - * @mas: The maple state - * @free: the maple topiary list of nodes to free - * @destroy: The maple topiary list of nodes to destroy (walk and free) + * mas_topiary_node() - Dispose of a singe node + * @mas: The maple state for pushing nodes + * @enode: The encoded maple node + * @in_rcu: If the tree is in rcu mode * - * Updates gap as necessary. + * The node will either be RCU freed or pushed back on the maple state. */ -static inline void mas_wmb_replace(struct ma_state *mas, - struct ma_topiary *free, - struct ma_topiary *destroy) +static inline void mas_topiary_node(struct ma_state *mas, + struct maple_enode *enode, bool in_rcu) { - /* All nodes must see old data as dead prior to replacing that data */ - smp_wmb(); /* Needed for RCU */ + struct maple_node *tmp; - /* Insert the new data in the tree */ - mas_replace(mas, true); + if (enode == MAS_NONE) + return; - if (!mte_is_leaf(mas->node)) - mas_descend_adopt(mas); + tmp = mte_to_node(enode); + mte_set_node_dead(enode); + if (in_rcu) + ma_free_rcu(tmp); + else + mas_push_node(mas, tmp); +} - mas_mat_free(mas, free); +/* + * mas_topiary_replace() - Replace the data with new data, then repair the + * parent links within the new tree. Iterate over the dead sub-tree and collect + * the dead subtrees and topiary the nodes that are no longer of use. + * + * The new tree will have up to three children with the correct parent. Keep + * track of the new entries as they need to be followed to find the next level + * of new entries. + * + * The old tree will have up to three children with the old parent. Keep track + * of the old entries as they may have more nodes below replaced. Nodes within + * [index, last] are dead subtrees, others need to be freed and followed. + * + * @mas: The maple state pointing at the new data + * @old_enode: The maple encoded node being replaced + * + */ +static inline void mas_topiary_replace(struct ma_state *mas, + struct maple_enode *old_enode) +{ + struct ma_state tmp[3], tmp_next[3]; + MA_TOPIARY(subtrees, mas->tree); + bool in_rcu; + int i, n; - if (destroy) - mas_mat_destroy(mas, destroy); + /* Place data in tree & then mark node as old */ + mas_put_in_tree(mas, old_enode); - if (mte_is_leaf(mas->node)) - return; + /* Update the parent pointers in the tree */ + tmp[0] = *mas; + tmp[0].offset = 0; + tmp[1].node = MAS_NONE; + tmp[2].node = MAS_NONE; + while (!mte_is_leaf(tmp[0].node)) { + n = 0; + for (i = 0; i < 3; i++) { + if (mas_is_none(&tmp[i])) + continue; - mas_update_gap(mas); + while (n < 3) { + if (!mas_find_child(&tmp[i], &tmp_next[n])) + break; + n++; + } + + mas_adopt_children(&tmp[i], tmp[i].node); + } + + if (MAS_WARN_ON(mas, n == 0)) + break; + + while (n < 3) + tmp_next[n++].node = MAS_NONE; + + for (i = 0; i < 3; i++) + tmp[i] = tmp_next[i]; + } + + /* Collect the old nodes that need to be discarded */ + if (mte_is_leaf(old_enode)) + return mas_free(mas, old_enode); + + tmp[0] = *mas; + tmp[0].offset = 0; + tmp[0].node = old_enode; + tmp[1].node = MAS_NONE; + tmp[2].node = MAS_NONE; + in_rcu = mt_in_rcu(mas->tree); + do { + n = 0; + for (i = 0; i < 3; i++) { + if (mas_is_none(&tmp[i])) + continue; + + while (n < 3) { + if (!mas_find_child(&tmp[i], &tmp_next[n])) + break; + + if ((tmp_next[n].min >= tmp_next->index) && + (tmp_next[n].max <= tmp_next->last)) { + mat_add(&subtrees, tmp_next[n].node); + tmp_next[n].node = MAS_NONE; + } else { + n++; + } + } + } + + if (MAS_WARN_ON(mas, n == 0)) + break; + + while (n < 3) + tmp_next[n++].node = MAS_NONE; + + for (i = 0; i < 3; i++) { + mas_topiary_node(mas, tmp[i].node, in_rcu); + tmp[i] = tmp_next[i]; + } + } while (!mte_is_leaf(tmp[0].node)); + + for (i = 0; i < 3; i++) + mas_topiary_node(mas, tmp[i].node, in_rcu); + + mas_mat_destroy(mas, &subtrees); } /* - * mast_new_root() - Set a new tree root during subtree creation - * @mast: The maple subtree state + * mas_wmb_replace() - Write memory barrier and replace * @mas: The maple state + * @old: The old maple encoded node that is being replaced. + * + * Updates gap as necessary. */ -static inline void mast_new_root(struct maple_subtree_state *mast, - struct ma_state *mas) +static inline void mas_wmb_replace(struct ma_state *mas, + struct maple_enode *old_enode) { - mas_mn(mast->l)->parent = - ma_parent_ptr(((unsigned long)mas->tree | MA_ROOT_PARENT)); - if (!mte_dead_node(mast->orig_l->node) && - !mte_is_root(mast->orig_l->node)) { - do { - mast_ascend_free(mast); - mast_topiary(mast); - } while (!mte_is_root(mast->orig_l->node)); - } - if ((mast->orig_l->node != mas->node) && - (mast->l->depth > mas_mt_height(mas))) { - mat_add(mast->free, mas->node); - } + /* Insert the new data in the tree */ + mas_topiary_replace(mas, old_enode); + + if (mte_is_leaf(mas->node)) + return; + + mas_update_gap(mas); } /* @@ -3015,12 +2866,11 @@ static int mas_spanning_rebalance(struct ma_state *mas, unsigned char split, mid_split; unsigned char slot = 0; struct maple_enode *left = NULL, *middle = NULL, *right = NULL; + struct maple_enode *old_enode; MA_STATE(l_mas, mas->tree, mas->index, mas->index); MA_STATE(r_mas, mas->tree, mas->index, mas->last); MA_STATE(m_mas, mas->tree, mas->index, mas->index); - MA_TOPIARY(free, mas->tree); - MA_TOPIARY(destroy, mas->tree); /* * The tree needs to be rebalanced and leaves need to be kept at the same level. @@ -3029,8 +2879,6 @@ static int mas_spanning_rebalance(struct ma_state *mas, mast->l = &l_mas; mast->m = &m_mas; mast->r = &r_mas; - mast->free = &free; - mast->destroy = &destroy; l_mas.node = r_mas.node = m_mas.node = MAS_NONE; /* Check if this is not root and has sufficient data. */ @@ -3038,7 +2886,7 @@ static int mas_spanning_rebalance(struct ma_state *mas, unlikely(mast->bn->b_end <= mt_min_slots[mast->bn->type])) mast_spanning_rebalance(mast); - mast->orig_l->depth = 0; + l_mas.depth = 0; /* * Each level of the tree is examined and balanced, pushing data to the left or @@ -3049,7 +2897,7 @@ static int mas_spanning_rebalance(struct ma_state *mas, * original tree and the partially new tree. To remedy the parent pointers in * the old tree, the new data is swapped into the active tree and a walk down * the tree is performed and the parent pointers are updated. - * See mas_descend_adopt() for more information.. + * See mas_topiary_replace() for more information. */ while (count--) { mast->bn->b_end--; @@ -3066,13 +2914,13 @@ static int mas_spanning_rebalance(struct ma_state *mas, */ memset(mast->bn, 0, sizeof(struct maple_big_node)); mast->bn->type = mte_node_type(left); - mast->orig_l->depth++; + l_mas.depth++; /* Root already stored in l->node. */ if (mas_is_root_limits(mast->l)) goto new_root; - mast_ascend_free(mast); + mast_ascend(mast); mast_combine_cp_left(mast); l_mas.offset = mast->bn->b_end; mab_set_b_end(mast->bn, &l_mas, left); @@ -3081,7 +2929,6 @@ static int mas_spanning_rebalance(struct ma_state *mas, /* Copy anything necessary out of the right node. */ mast_combine_cp_right(mast); - mast_topiary(mast); mast->orig_l->last = mast->orig_l->max; if (mast_sufficient(mast)) @@ -3103,7 +2950,7 @@ static int mas_spanning_rebalance(struct ma_state *mas, l_mas.node = mt_mk_node(ma_mnode_ptr(mas_pop_node(mas)), mte_node_type(mast->orig_l->node)); - mast->orig_l->depth++; + l_mas.depth++; mab_mas_cp(mast->bn, 0, mt_slots[mast->bn->type] - 1, &l_mas, true); mas_set_parent(mas, left, l_mas.node, slot); if (middle) @@ -3114,23 +2961,20 @@ static int mas_spanning_rebalance(struct ma_state *mas, if (mas_is_root_limits(mast->l)) { new_root: - mast_new_root(mast, mas); + mas_mn(mast->l)->parent = ma_parent_ptr(mas_tree_parent(mas)); + while (!mte_is_root(mast->orig_l->node)) + mast_ascend(mast); } else { mas_mn(&l_mas)->parent = mas_mn(mast->orig_l)->parent; } - if (!mte_dead_node(mast->orig_l->node)) - mat_add(&free, mast->orig_l->node); - - mas->depth = mast->orig_l->depth; - *mast->orig_l = l_mas; - mte_set_node_dead(mas->node); - - /* Set up mas for insertion. */ - mast->orig_l->depth = mas->depth; - mast->orig_l->alloc = mas->alloc; - *mas = *mast->orig_l; - mas_wmb_replace(mas, &free, &destroy); + old_enode = mast->orig_l->node; + mas->depth = l_mas.depth; + mas->node = l_mas.node; + mas->min = l_mas.min; + mas->max = l_mas.max; + mas->offset = l_mas.offset; + mas_wmb_replace(mas, old_enode); mtree_range_walk(mas); return mast->bn->b_end; } @@ -3166,7 +3010,7 @@ static inline int mas_rebalance(struct ma_state *mas, * tries to combine the data in the same way. If one node contains the * entire range of the tree, then that node is used as a new root node. */ - mas_node_count(mas, 1 + empty_count * 3); + mas_node_count(mas, empty_count * 2 - 1); if (mas_is_err(mas)) return 0; @@ -3206,7 +3050,7 @@ static inline void mas_destroy_rebalance(struct ma_state *mas, unsigned char end { enum maple_type mt = mte_node_type(mas->node); struct maple_node reuse, *newnode, *parent, *new_left, *left, *node; - struct maple_enode *eparent; + struct maple_enode *eparent, *old_eparent; unsigned char offset, tmp, split = mt_slots[mt] / 2; void __rcu **l_slots, **slots; unsigned long *l_pivs, *pivs, gap; @@ -3248,7 +3092,7 @@ static inline void mas_destroy_rebalance(struct ma_state *mas, unsigned char end l_mas.max = l_pivs[split]; mas->min = l_mas.max + 1; - eparent = mt_mk_node(mte_parent(l_mas.node), + old_eparent = mt_mk_node(mte_parent(l_mas.node), mas_parent_type(&l_mas, l_mas.node)); tmp += end; if (!in_rcu) { @@ -3264,7 +3108,7 @@ static inline void mas_destroy_rebalance(struct ma_state *mas, unsigned char end memcpy(node, newnode, sizeof(struct maple_node)); ma_set_meta(node, mt, 0, tmp - 1); - mte_set_pivot(eparent, mte_parent_slot(l_mas.node), + mte_set_pivot(old_eparent, mte_parent_slot(l_mas.node), l_pivs[split]); /* Remove data from l_pivs. */ @@ -3272,6 +3116,7 @@ static inline void mas_destroy_rebalance(struct ma_state *mas, unsigned char end memset(l_pivs + tmp, 0, sizeof(unsigned long) * (max_p - tmp)); memset(l_slots + tmp, 0, sizeof(void *) * (max_s - tmp)); ma_set_meta(left, mt, 0, split); + eparent = old_eparent; goto done; } @@ -3296,7 +3141,7 @@ static inline void mas_destroy_rebalance(struct ma_state *mas, unsigned char end parent = mas_pop_node(mas); slots = ma_slots(parent, mt); pivs = ma_pivots(parent, mt); - memcpy(parent, mte_to_node(eparent), sizeof(struct maple_node)); + memcpy(parent, mte_to_node(old_eparent), sizeof(struct maple_node)); rcu_assign_pointer(slots[offset], mas->node); rcu_assign_pointer(slots[offset - 1], l_mas.node); pivs[offset - 1] = l_mas.max; @@ -3308,8 +3153,10 @@ done: mte_set_gap(eparent, mte_parent_slot(l_mas.node), gap); mas_ascend(mas); - if (in_rcu) - mas_replace(mas, false); + if (in_rcu) { + mas_replace_node(mas, old_eparent); + mas_adopt_children(mas, mas->node); + } mas_update_gap(mas); } @@ -3358,7 +3205,6 @@ static inline void mast_fill_bnode(struct maple_subtree_state *mast, unsigned char skip) { bool cp = true; - struct maple_enode *old = mas->node; unsigned char split; memset(mast->bn->gap, 0, sizeof(unsigned long) * ARRAY_SIZE(mast->bn->gap)); @@ -3370,7 +3216,6 @@ static inline void mast_fill_bnode(struct maple_subtree_state *mast, cp = false; } else { mas_ascend(mas); - mat_add(mast->free, old); mas->offset = mte_parent_slot(mas->node); } @@ -3474,13 +3319,11 @@ static inline bool mas_push_data(struct ma_state *mas, int height, split = mt_slots[mast->bn->type] - 2; if (left) { /* Switch mas to prev node */ - mat_add(mast->free, mas->node); *mas = tmp_mas; /* Start using mast->l for the left side. */ tmp_mas.node = mast->l->node; *mast->l = tmp_mas; } else { - mat_add(mast->free, tmp_mas.node); tmp_mas.node = mast->r->node; *mast->r = tmp_mas; split = slot_total - split; @@ -3507,6 +3350,7 @@ static int mas_split(struct ma_state *mas, struct maple_big_node *b_node) struct maple_subtree_state mast; int height = 0; unsigned char mid_split, split = 0; + struct maple_enode *old; /* * Splitting is handled differently from any other B-tree; the Maple @@ -3529,7 +3373,6 @@ static int mas_split(struct ma_state *mas, struct maple_big_node *b_node) MA_STATE(r_mas, mas->tree, mas->index, mas->last); MA_STATE(prev_l_mas, mas->tree, mas->index, mas->last); MA_STATE(prev_r_mas, mas->tree, mas->index, mas->last); - MA_TOPIARY(mat, mas->tree); trace_ma_op(__func__, mas); mas->depth = mas_mt_height(mas); @@ -3542,7 +3385,6 @@ static int mas_split(struct ma_state *mas, struct maple_big_node *b_node) mast.r = &r_mas; mast.orig_l = &prev_l_mas; mast.orig_r = &prev_r_mas; - mast.free = &mat; mast.bn = b_node; while (height++ <= mas->depth) { @@ -3582,9 +3424,9 @@ static int mas_split(struct ma_state *mas, struct maple_big_node *b_node) } /* Set the original node as dead */ - mat_add(mast.free, mas->node); + old = mas->node; mas->node = l_mas.node; - mas_wmb_replace(mas, mast.free, NULL); + mas_wmb_replace(mas, old); mtree_range_walk(mas); return 1; } @@ -3626,11 +3468,13 @@ static noinline_for_kasan int mas_commit_b_node(struct ma_wr_state *wr_mas, struct maple_big_node *b_node, unsigned char end) { struct maple_node *node; + struct maple_enode *old_enode; unsigned char b_end = b_node->b_end; enum maple_type b_type = b_node->type; + old_enode = wr_mas->mas->node; if ((b_end < mt_min_slots[b_type]) && - (!mte_is_root(wr_mas->mas->node)) && + (!mte_is_root(old_enode)) && (mas_mt_height(wr_mas->mas) > 1)) return mas_rebalance(wr_mas->mas, b_node); @@ -3648,7 +3492,7 @@ static noinline_for_kasan int mas_commit_b_node(struct ma_wr_state *wr_mas, node->parent = mas_mn(wr_mas->mas)->parent; wr_mas->mas->node = mt_mk_node(node, b_type); mab_mas_cp(b_node, 0, b_end, wr_mas->mas, false); - mas_replace(wr_mas->mas, false); + mas_replace_node(wr_mas->mas, old_enode); reuse_node: mas_update_gap(wr_mas->mas); return 1; @@ -3675,8 +3519,7 @@ static inline int mas_root_expand(struct ma_state *mas, void *entry) node = mas_pop_node(mas); pivots = ma_pivots(node, type); slots = ma_slots(node, type); - node->parent = ma_parent_ptr( - ((unsigned long)mas->tree | MA_ROOT_PARENT)); + node->parent = ma_parent_ptr(mas_tree_parent(mas)); mas->node = mt_mk_node(node, type); if (mas->index) { @@ -3919,6 +3762,7 @@ dead_node: return NULL; } +static void mte_destroy_walk(struct maple_enode *, struct maple_tree *); /* * mas_new_root() - Create a new root node that only contains the entry passed * in. @@ -3952,8 +3796,7 @@ static inline int mas_new_root(struct ma_state *mas, void *entry) node = mas_pop_node(mas); pivots = ma_pivots(node, type); slots = ma_slots(node, type); - node->parent = ma_parent_ptr( - ((unsigned long)mas->tree | MA_ROOT_PARENT)); + node->parent = ma_parent_ptr(mas_tree_parent(mas)); mas->node = mt_mk_node(node, type); rcu_assign_pointer(slots[0], entry); pivots[0] = mas->last; @@ -3986,7 +3829,6 @@ static inline int mas_wr_spanning_store(struct ma_wr_state *wr_mas) /* Left and Right side of spanning store */ MA_STATE(l_mas, NULL, 0, 0); MA_STATE(r_mas, NULL, 0, 0); - MA_WR_STATE(r_wr_mas, &r_mas, wr_mas->entry); MA_WR_STATE(l_wr_mas, &l_mas, wr_mas->entry); @@ -4147,9 +3989,10 @@ static inline bool mas_wr_node_store(struct ma_wr_state *wr_mas, done: mas_leaf_set_meta(mas, newnode, dst_pivots, maple_leaf_64, new_end); if (in_rcu) { - mte_set_node_dead(mas->node); + struct maple_enode *old_enode = mas->node; + mas->node = mt_mk_node(newnode, wr_mas->type); - mas_replace(mas, false); + mas_replace_node(mas, old_enode); } else { memcpy(wr_mas->node, newnode, sizeof(struct maple_node)); } @@ -4168,23 +4011,35 @@ static inline bool mas_wr_slot_store(struct ma_wr_state *wr_mas) { struct ma_state *mas = wr_mas->mas; unsigned char offset = mas->offset; + void __rcu **slots = wr_mas->slots; bool gap = false; - if (wr_mas->offset_end - offset != 1) - return false; - - gap |= !mt_slot_locked(mas->tree, wr_mas->slots, offset); - gap |= !mt_slot_locked(mas->tree, wr_mas->slots, offset + 1); + gap |= !mt_slot_locked(mas->tree, slots, offset); + gap |= !mt_slot_locked(mas->tree, slots, offset + 1); - if (mas->index == wr_mas->r_min) { - /* Overwriting the range and over a part of the next range. */ - rcu_assign_pointer(wr_mas->slots[offset], wr_mas->entry); - wr_mas->pivots[offset] = mas->last; - } else { - /* Overwriting a part of the range and over the next range */ - rcu_assign_pointer(wr_mas->slots[offset + 1], wr_mas->entry); + if (wr_mas->offset_end - offset == 1) { + if (mas->index == wr_mas->r_min) { + /* Overwriting the range and a part of the next one */ + rcu_assign_pointer(slots[offset], wr_mas->entry); + wr_mas->pivots[offset] = mas->last; + } else { + /* Overwriting a part of the range and the next one */ + rcu_assign_pointer(slots[offset + 1], wr_mas->entry); + wr_mas->pivots[offset] = mas->index - 1; + mas->offset++; /* Keep mas accurate. */ + } + } else if (!mt_in_rcu(mas->tree)) { + /* + * Expand the range, only partially overwriting the previous and + * next ranges + */ + gap |= !mt_slot_locked(mas->tree, slots, offset + 2); + rcu_assign_pointer(slots[offset + 1], wr_mas->entry); wr_mas->pivots[offset] = mas->index - 1; + wr_mas->pivots[offset + 1] = mas->last; mas->offset++; /* Keep mas accurate. */ + } else { + return false; } trace_ma_write(__func__, mas, 0, wr_mas->entry); @@ -4198,18 +4053,6 @@ static inline bool mas_wr_slot_store(struct ma_wr_state *wr_mas) return true; } -static inline void mas_wr_end_piv(struct ma_wr_state *wr_mas) -{ - while ((wr_mas->offset_end < wr_mas->node_end) && - (wr_mas->mas->last > wr_mas->pivots[wr_mas->offset_end])) - wr_mas->offset_end++; - - if (wr_mas->offset_end < wr_mas->node_end) - wr_mas->end_piv = wr_mas->pivots[wr_mas->offset_end]; - else - wr_mas->end_piv = wr_mas->mas->max; -} - static inline void mas_wr_extend_null(struct ma_wr_state *wr_mas) { struct ma_state *mas = wr_mas->mas; @@ -4246,6 +4089,21 @@ static inline void mas_wr_extend_null(struct ma_wr_state *wr_mas) } } +static inline void mas_wr_end_piv(struct ma_wr_state *wr_mas) +{ + while ((wr_mas->offset_end < wr_mas->node_end) && + (wr_mas->mas->last > wr_mas->pivots[wr_mas->offset_end])) + wr_mas->offset_end++; + + if (wr_mas->offset_end < wr_mas->node_end) + wr_mas->end_piv = wr_mas->pivots[wr_mas->offset_end]; + else + wr_mas->end_piv = wr_mas->mas->max; + + if (!wr_mas->entry) + mas_wr_extend_null(wr_mas); +} + static inline unsigned char mas_wr_new_end(struct ma_wr_state *wr_mas) { struct ma_state *mas = wr_mas->mas; @@ -4264,6 +4122,7 @@ static inline unsigned char mas_wr_new_end(struct ma_wr_state *wr_mas) /* * mas_wr_append: Attempt to append * @wr_mas: the maple write state + * @new_end: The end of the node after the modification * * This is currently unsafe in rcu mode since the end of the node may be cached * by readers while the node contents may be updated which could result in @@ -4271,39 +4130,55 @@ static inline unsigned char mas_wr_new_end(struct ma_wr_state *wr_mas) * * Return: True if appended, false otherwise */ -static inline bool mas_wr_append(struct ma_wr_state *wr_mas) +static inline bool mas_wr_append(struct ma_wr_state *wr_mas, + unsigned char new_end) { - unsigned char end = wr_mas->node_end; - unsigned char new_end = end + 1; - struct ma_state *mas = wr_mas->mas; - unsigned char node_pivots = mt_pivots[wr_mas->type]; + struct ma_state *mas; + void __rcu **slots; + unsigned char end; + mas = wr_mas->mas; if (mt_in_rcu(mas->tree)) return false; if (mas->offset != wr_mas->node_end) return false; - if (new_end < node_pivots) { + end = wr_mas->node_end; + if (mas->offset != end) + return false; + + if (new_end < mt_pivots[wr_mas->type]) { wr_mas->pivots[new_end] = wr_mas->pivots[end]; - ma_set_meta(wr_mas->node, maple_leaf_64, 0, new_end); + ma_set_meta(wr_mas->node, wr_mas->type, 0, new_end); } - if (mas->last == wr_mas->r_max) { - /* Append to end of range */ - rcu_assign_pointer(wr_mas->slots[new_end], wr_mas->entry); - wr_mas->pivots[end] = mas->index - 1; - mas->offset = new_end; + slots = wr_mas->slots; + if (new_end == end + 1) { + if (mas->last == wr_mas->r_max) { + /* Append to end of range */ + rcu_assign_pointer(slots[new_end], wr_mas->entry); + wr_mas->pivots[end] = mas->index - 1; + mas->offset = new_end; + } else { + /* Append to start of range */ + rcu_assign_pointer(slots[new_end], wr_mas->content); + wr_mas->pivots[end] = mas->last; + rcu_assign_pointer(slots[end], wr_mas->entry); + } } else { - /* Append to start of range */ - rcu_assign_pointer(wr_mas->slots[new_end], wr_mas->content); - wr_mas->pivots[end] = mas->last; - rcu_assign_pointer(wr_mas->slots[end], wr_mas->entry); + /* Append to the range without touching any boundaries. */ + rcu_assign_pointer(slots[new_end], wr_mas->content); + wr_mas->pivots[end + 1] = mas->last; + rcu_assign_pointer(slots[end + 1], wr_mas->entry); + wr_mas->pivots[end] = mas->index - 1; + mas->offset = end + 1; } if (!wr_mas->content || !wr_mas->entry) mas_update_gap(mas); + trace_ma_write(__func__, mas, new_end, wr_mas->entry); return true; } @@ -4345,7 +4220,7 @@ static inline void mas_wr_modify(struct ma_wr_state *wr_mas) goto slow_path; /* Attempt to append */ - if (new_end == wr_mas->node_end + 1 && mas_wr_append(wr_mas)) + if (mas_wr_append(wr_mas, new_end)) return; if (new_end == wr_mas->node_end && mas_wr_slot_store(wr_mas)) @@ -4385,10 +4260,6 @@ static inline void *mas_wr_store_entry(struct ma_wr_state *wr_mas) /* At this point, we are at the leaf node that needs to be altered. */ mas_wr_end_piv(wr_mas); - - if (!wr_mas->entry) - mas_wr_extend_null(wr_mas); - /* New root for a single pointer */ if (unlikely(!mas->index && mas->last == ULONG_MAX)) { mas_new_root(mas, wr_mas->entry); @@ -4560,10 +4431,13 @@ no_entry: * * @mas: The maple state * @max: The minimum starting range + * @empty: Can be empty + * @set_underflow: Set the @mas->node to underflow state on limit. * * Return: The entry in the previous slot which is possibly NULL */ -static void *mas_prev_slot(struct ma_state *mas, unsigned long min, bool empty) +static void *mas_prev_slot(struct ma_state *mas, unsigned long min, bool empty, + bool set_underflow) { void *entry; void __rcu **slots; @@ -4580,7 +4454,6 @@ retry: if (unlikely(mas_rewalk_if_dead(mas, node, save_point))) goto retry; -again: if (mas->min <= min) { pivot = mas_safe_min(mas, pivots, mas->offset); @@ -4588,9 +4461,10 @@ again: goto retry; if (pivot <= min) - return NULL; + goto underflow; } +again: if (likely(mas->offset)) { mas->offset--; mas->last = mas->index - 1; @@ -4602,7 +4476,7 @@ again: } if (mas_is_none(mas)) - return NULL; + goto underflow; mas->last = mas->max; node = mas_mn(mas); @@ -4619,10 +4493,19 @@ again: if (likely(entry)) return entry; - if (!empty) + if (!empty) { + if (mas->index <= min) + goto underflow; + goto again; + } return entry; + +underflow: + if (set_underflow) + mas->node = MAS_UNDERFLOW; + return NULL; } /* @@ -4712,10 +4595,13 @@ no_entry: * @mas: The maple state * @max: The maximum starting range * @empty: Can be empty + * @set_overflow: Should @mas->node be set to overflow when the limit is + * reached. * * Return: The entry in the next slot which is possibly NULL */ -static void *mas_next_slot(struct ma_state *mas, unsigned long max, bool empty) +static void *mas_next_slot(struct ma_state *mas, unsigned long max, bool empty, + bool set_overflow) { void __rcu **slots; unsigned long *pivots; @@ -4734,22 +4620,22 @@ retry: if (unlikely(mas_rewalk_if_dead(mas, node, save_point))) goto retry; -again: if (mas->max >= max) { if (likely(mas->offset < data_end)) pivot = pivots[mas->offset]; else - return NULL; /* must be mas->max */ + goto overflow; if (unlikely(mas_rewalk_if_dead(mas, node, save_point))) goto retry; if (pivot >= max) - return NULL; + goto overflow; } if (likely(mas->offset < data_end)) { mas->index = pivots[mas->offset] + 1; +again: mas->offset++; if (likely(mas->offset < data_end)) mas->last = pivots[mas->offset]; @@ -4761,8 +4647,11 @@ again: goto retry; } - if (mas_is_none(mas)) + if (WARN_ON_ONCE(mas_is_none(mas))) { + mas->node = MAS_OVERFLOW; return NULL; + goto overflow; + } mas->offset = 0; mas->index = mas->min; @@ -4781,12 +4670,20 @@ again: return entry; if (!empty) { - if (!mas->offset) - data_end = 2; + if (mas->last >= max) + goto overflow; + + mas->index = mas->last + 1; + /* Node cannot end on NULL, so it's safe to short-cut here */ goto again; } return entry; + +overflow: + if (set_overflow) + mas->node = MAS_OVERFLOW; + return NULL; } /* @@ -4796,17 +4693,20 @@ again: * * Set the @mas->node to the next entry and the range_start to * the beginning value for the entry. Does not check beyond @limit. - * Sets @mas->index and @mas->last to the limit if it is hit. + * Sets @mas->index and @mas->last to the range, Does not update @mas->index and + * @mas->last on overflow. * Restarts on dead nodes. * * Return: the next entry or %NULL. */ static inline void *mas_next_entry(struct ma_state *mas, unsigned long limit) { - if (mas->last >= limit) + if (mas->last >= limit) { + mas->node = MAS_OVERFLOW; return NULL; + } - return mas_next_slot(mas, limit, false); + return mas_next_slot(mas, limit, false, true); } /* @@ -4928,7 +4828,7 @@ static inline bool mas_anode_descend(struct ma_state *mas, unsigned long size) min = mas_safe_min(mas, pivots, offset); data_end = ma_data_end(node, type, pivots, mas->max); for (; offset <= data_end; offset++) { - pivot = mas_logical_pivot(mas, pivots, offset, type); + pivot = mas_safe_pivot(mas, pivots, offset, type); /* Not within lower bounds */ if (mas->index > pivot) @@ -4982,7 +4882,7 @@ void *mas_walk(struct ma_state *mas) { void *entry; - if (mas_is_none(mas) || mas_is_paused(mas) || mas_is_ptr(mas)) + if (!mas_is_active(mas) || !mas_is_start(mas)) mas->node = MAS_START; retry: entry = mas_state_walk(mas); @@ -5439,19 +5339,42 @@ static inline void mte_destroy_walk(struct maple_enode *enode, static void mas_wr_store_setup(struct ma_wr_state *wr_mas) { - if (unlikely(mas_is_paused(wr_mas->mas))) - mas_reset(wr_mas->mas); + if (!mas_is_active(wr_mas->mas)) { + if (mas_is_start(wr_mas->mas)) + return; - if (!mas_is_start(wr_mas->mas)) { - if (mas_is_none(wr_mas->mas)) { - mas_reset(wr_mas->mas); - } else { - wr_mas->r_max = wr_mas->mas->max; - wr_mas->type = mte_node_type(wr_mas->mas->node); - if (mas_is_span_wr(wr_mas)) - mas_reset(wr_mas->mas); - } + if (unlikely(mas_is_paused(wr_mas->mas))) + goto reset; + + if (unlikely(mas_is_none(wr_mas->mas))) + goto reset; + + if (unlikely(mas_is_overflow(wr_mas->mas))) + goto reset; + + if (unlikely(mas_is_underflow(wr_mas->mas))) + goto reset; } + + /* + * A less strict version of mas_is_span_wr() where we allow spanning + * writes within this node. This is to stop partial walks in + * mas_prealloc() from being reset. + */ + if (wr_mas->mas->last > wr_mas->mas->max) + goto reset; + + if (wr_mas->entry) + return; + + if (mte_is_leaf(wr_mas->mas->node) && + wr_mas->mas->last == wr_mas->mas->max) + goto reset; + + return; + +reset: + mas_reset(wr_mas->mas); } /* Interface */ @@ -5543,15 +5466,58 @@ EXPORT_SYMBOL_GPL(mas_store_prealloc); /** * mas_preallocate() - Preallocate enough nodes for a store operation * @mas: The maple state + * @entry: The entry that will be stored * @gfp: The GFP_FLAGS to use for allocations. * * Return: 0 on success, -ENOMEM if memory could not be allocated. */ -int mas_preallocate(struct ma_state *mas, gfp_t gfp) +int mas_preallocate(struct ma_state *mas, void *entry, gfp_t gfp) { + MA_WR_STATE(wr_mas, mas, entry); + unsigned char node_size; + int request = 1; int ret; - mas_node_count_gfp(mas, 1 + mas_mt_height(mas) * 3, gfp); + + if (unlikely(!mas->index && mas->last == ULONG_MAX)) + goto ask_now; + + mas_wr_store_setup(&wr_mas); + wr_mas.content = mas_start(mas); + /* Root expand */ + if (unlikely(mas_is_none(mas) || mas_is_ptr(mas))) + goto ask_now; + + if (unlikely(!mas_wr_walk(&wr_mas))) { + /* Spanning store, use worst case for now */ + request = 1 + mas_mt_height(mas) * 3; + goto ask_now; + } + + /* At this point, we are at the leaf node that needs to be altered. */ + /* Exact fit, no nodes needed. */ + if (wr_mas.r_min == mas->index && wr_mas.r_max == mas->last) + return 0; + + mas_wr_end_piv(&wr_mas); + node_size = mas_wr_new_end(&wr_mas); + if (node_size >= mt_slots[wr_mas.type]) { + /* Split, worst case for now. */ + request = 1 + mas_mt_height(mas) * 2; + goto ask_now; + } + + /* New root needs a singe node */ + if (unlikely(mte_is_root(mas->node))) + goto ask_now; + + /* Potential spanning rebalance collapsing a node, use worst-case */ + if (node_size - 1 <= mt_min_slots[wr_mas.type]) + request = mas_mt_height(mas) * 2 - 1; + + /* node store, slot store needs one node */ +ask_now: + mas_node_count_gfp(mas, request, gfp); mas->mas_flags |= MA_STATE_PREALLOC; if (likely(!mas_is_err(mas))) return 0; @@ -5661,7 +5627,7 @@ int mas_expected_entries(struct ma_state *mas, unsigned long nr_entries) /* Internal nodes */ nr_nodes += DIV_ROUND_UP(nr_nodes, nonleaf_cap); /* Add working room for split (2 nodes) + new parents */ - mas_node_count(mas, nr_nodes + 3); + mas_node_count_gfp(mas, nr_nodes + 3, GFP_KERNEL); /* Detect if allocations run out */ mas->mas_flags |= MA_STATE_PREALLOC; @@ -5682,8 +5648,25 @@ static inline bool mas_next_setup(struct ma_state *mas, unsigned long max, { bool was_none = mas_is_none(mas); - if (mas_is_none(mas) || mas_is_paused(mas)) + if (unlikely(mas->last >= max)) { + mas->node = MAS_OVERFLOW; + return true; + } + + if (mas_is_active(mas)) + return false; + + if (mas_is_none(mas) || mas_is_paused(mas)) { + mas->node = MAS_START; + } else if (mas_is_overflow(mas)) { + /* Overflowed before, but the max changed */ mas->node = MAS_START; + } else if (mas_is_underflow(mas)) { + mas->node = MAS_START; + *entry = mas_walk(mas); + if (*entry) + return true; + } if (mas_is_start(mas)) *entry = mas_walk(mas); /* Retries on dead nodes handled by mas_walk */ @@ -5702,6 +5685,7 @@ static inline bool mas_next_setup(struct ma_state *mas, unsigned long max, if (mas_is_none(mas)) return true; + return false; } @@ -5724,7 +5708,7 @@ void *mas_next(struct ma_state *mas, unsigned long max) return entry; /* Retries on dead nodes handled by mas_next_slot */ - return mas_next_slot(mas, max, false); + return mas_next_slot(mas, max, false, true); } EXPORT_SYMBOL_GPL(mas_next); @@ -5747,7 +5731,7 @@ void *mas_next_range(struct ma_state *mas, unsigned long max) return entry; /* Retries on dead nodes handled by mas_next_slot */ - return mas_next_slot(mas, max, true); + return mas_next_slot(mas, max, true, true); } EXPORT_SYMBOL_GPL(mas_next_range); @@ -5757,7 +5741,11 @@ EXPORT_SYMBOL_GPL(mas_next_range); * @index: The start index * @max: The maximum index to check * - * Return: The entry at @index or higher, or %NULL if nothing is found. + * Takes RCU read lock internally to protect the search, which does not + * protect the returned pointer after dropping RCU read lock. + * See also: Documentation/core-api/maple_tree.rst + * + * Return: The entry higher than @index or %NULL if nothing is found. */ void *mt_next(struct maple_tree *mt, unsigned long index, unsigned long max) { @@ -5774,18 +5762,31 @@ EXPORT_SYMBOL_GPL(mt_next); static inline bool mas_prev_setup(struct ma_state *mas, unsigned long min, void **entry) { - if (mas->index <= min) - goto none; + if (unlikely(mas->index <= min)) { + mas->node = MAS_UNDERFLOW; + return true; + } - if (mas_is_none(mas) || mas_is_paused(mas)) + if (mas_is_active(mas)) + return false; + + if (mas_is_overflow(mas)) { mas->node = MAS_START; + *entry = mas_walk(mas); + if (*entry) + return true; + } - if (mas_is_start(mas)) { - mas_walk(mas); - if (!mas->index) - goto none; + if (mas_is_none(mas) || mas_is_paused(mas)) { + mas->node = MAS_START; + } else if (mas_is_underflow(mas)) { + /* underflowed before but the min changed */ + mas->node = MAS_START; } + if (mas_is_start(mas)) + mas_walk(mas); + if (unlikely(mas_is_ptr(mas))) { if (!mas->index) goto none; @@ -5830,7 +5831,7 @@ void *mas_prev(struct ma_state *mas, unsigned long min) if (mas_prev_setup(mas, min, &entry)) return entry; - return mas_prev_slot(mas, min, false); + return mas_prev_slot(mas, min, false, true); } EXPORT_SYMBOL_GPL(mas_prev); @@ -5853,7 +5854,7 @@ void *mas_prev_range(struct ma_state *mas, unsigned long min) if (mas_prev_setup(mas, min, &entry)) return entry; - return mas_prev_slot(mas, min, true); + return mas_prev_slot(mas, min, true, true); } EXPORT_SYMBOL_GPL(mas_prev_range); @@ -5863,7 +5864,11 @@ EXPORT_SYMBOL_GPL(mas_prev_range); * @index: The start index * @min: The minimum index to check * - * Return: The entry at @index or lower, or %NULL if nothing is found. + * Takes RCU read lock internally to protect the search, which does not + * protect the returned pointer after dropping RCU read lock. + * See also: Documentation/core-api/maple_tree.rst + * + * Return: The entry before @index or %NULL if nothing is found. */ void *mt_prev(struct maple_tree *mt, unsigned long index, unsigned long min) { @@ -5907,24 +5912,35 @@ EXPORT_SYMBOL_GPL(mas_pause); static inline bool mas_find_setup(struct ma_state *mas, unsigned long max, void **entry) { - *entry = NULL; + if (mas_is_active(mas)) { + if (mas->last < max) + return false; + + return true; + } - if (unlikely(mas_is_none(mas))) { + if (mas_is_paused(mas)) { if (unlikely(mas->last >= max)) return true; - mas->index = mas->last; + mas->index = ++mas->last; mas->node = MAS_START; - } else if (unlikely(mas_is_paused(mas))) { + } else if (mas_is_none(mas)) { if (unlikely(mas->last >= max)) return true; + mas->index = mas->last; mas->node = MAS_START; - mas->index = ++mas->last; - } else if (unlikely(mas_is_ptr(mas))) - goto ptr_out_of_range; + } else if (mas_is_overflow(mas) || mas_is_underflow(mas)) { + if (mas->index > max) { + mas->node = MAS_OVERFLOW; + return true; + } - if (unlikely(mas_is_start(mas))) { + mas->node = MAS_START; + } + + if (mas_is_start(mas)) { /* First run or continue */ if (mas->index > max) return true; @@ -5974,7 +5990,7 @@ void *mas_find(struct ma_state *mas, unsigned long max) return entry; /* Retries on dead nodes handled by mas_next_slot */ - return mas_next_slot(mas, max, false); + return mas_next_slot(mas, max, false, false); } EXPORT_SYMBOL_GPL(mas_find); @@ -5992,13 +6008,13 @@ EXPORT_SYMBOL_GPL(mas_find); */ void *mas_find_range(struct ma_state *mas, unsigned long max) { - void *entry; + void *entry = NULL; if (mas_find_setup(mas, max, &entry)) return entry; /* Retries on dead nodes handled by mas_next_slot */ - return mas_next_slot(mas, max, true); + return mas_next_slot(mas, max, true, false); } EXPORT_SYMBOL_GPL(mas_find_range); @@ -6013,26 +6029,36 @@ EXPORT_SYMBOL_GPL(mas_find_range); static inline bool mas_find_rev_setup(struct ma_state *mas, unsigned long min, void **entry) { - *entry = NULL; - - if (unlikely(mas_is_none(mas))) { - if (mas->index <= min) - goto none; + if (mas_is_active(mas)) { + if (mas->index > min) + return false; - mas->last = mas->index; - mas->node = MAS_START; + return true; } - if (unlikely(mas_is_paused(mas))) { + if (mas_is_paused(mas)) { if (unlikely(mas->index <= min)) { mas->node = MAS_NONE; return true; } mas->node = MAS_START; mas->last = --mas->index; + } else if (mas_is_none(mas)) { + if (mas->index <= min) + goto none; + + mas->last = mas->index; + mas->node = MAS_START; + } else if (mas_is_underflow(mas) || mas_is_overflow(mas)) { + if (mas->last <= min) { + mas->node = MAS_UNDERFLOW; + return true; + } + + mas->node = MAS_START; } - if (unlikely(mas_is_start(mas))) { + if (mas_is_start(mas)) { /* First run or continue */ if (mas->index < min) return true; @@ -6083,13 +6109,13 @@ none: */ void *mas_find_rev(struct ma_state *mas, unsigned long min) { - void *entry; + void *entry = NULL; if (mas_find_rev_setup(mas, min, &entry)) return entry; /* Retries on dead nodes handled by mas_prev_slot */ - return mas_prev_slot(mas, min, false); + return mas_prev_slot(mas, min, false, false); } EXPORT_SYMBOL_GPL(mas_find_rev); @@ -6109,13 +6135,13 @@ EXPORT_SYMBOL_GPL(mas_find_rev); */ void *mas_find_range_rev(struct ma_state *mas, unsigned long min) { - void *entry; + void *entry = NULL; if (mas_find_rev_setup(mas, min, &entry)) return entry; /* Retries on dead nodes handled by mas_prev_slot */ - return mas_prev_slot(mas, min, true); + return mas_prev_slot(mas, min, true, false); } EXPORT_SYMBOL_GPL(mas_find_range_rev); @@ -6286,7 +6312,7 @@ int mtree_store(struct maple_tree *mt, unsigned long index, void *entry, EXPORT_SYMBOL(mtree_store); /** - * mtree_insert_range() - Insert an entry at a give range if there is no value. + * mtree_insert_range() - Insert an entry at a given range if there is no value. * @mt: The maple tree * @first: The start of the range * @last: The end of the range @@ -6322,11 +6348,11 @@ retry: EXPORT_SYMBOL(mtree_insert_range); /** - * mtree_insert() - Insert an entry at a give index if there is no value. + * mtree_insert() - Insert an entry at a given index if there is no value. * @mt: The maple tree * @index : The index to store the value * @entry: The entry to store - * @gfp: The FGP_FLAGS to use for allocations. + * @gfp: The GFP_FLAGS to use for allocations. * * Return: 0 on success, -EEXISTS if the range is occupied, -EINVAL on invalid * request, -ENOMEM if memory could not be allocated. @@ -6475,9 +6501,15 @@ EXPORT_SYMBOL(mtree_destroy); * mt_find() - Search from the start up until an entry is found. * @mt: The maple tree * @index: Pointer which contains the start location of the search - * @max: The maximum value to check + * @max: The maximum value of the search range * - * Handles locking. @index will be incremented to one beyond the range. + * Takes RCU read lock internally to protect the search, which does not + * protect the returned pointer after dropping RCU read lock. + * See also: Documentation/core-api/maple_tree.rst + * + * In case that an entry is found @index is updated to point to the next + * possible entry independent whether the found entry is occupying a + * single index or a range if indices. * * Return: The entry at or after the @index or %NULL */ @@ -6535,7 +6567,9 @@ EXPORT_SYMBOL(mt_find); * @index: Pointer which contains the start location of the search * @max: The maximum value to check * - * Handles locking, detects wrapping on index == 0 + * Same as mt_find() except that it checks @index for 0 before + * searching. If @index == 0, the search is aborted. This covers a wrap + * around of @index to 0 in an iterator loop. * * Return: The entry at or after the @index or %NULL */ @@ -6640,78 +6674,6 @@ static inline struct maple_enode *mas_get_slot(struct ma_state *mas, offset); } - -/* - * mas_first_entry() - Go the first leaf and find the first entry. - * @mas: the maple state. - * @limit: the maximum index to check. - * @*r_start: Pointer to set to the range start. - * - * Sets mas->offset to the offset of the entry, r_start to the range minimum. - * - * Return: The first entry or MAS_NONE. - */ -static inline void *mas_first_entry(struct ma_state *mas, struct maple_node *mn, - unsigned long limit, enum maple_type mt) - -{ - unsigned long max; - unsigned long *pivots; - void __rcu **slots; - void *entry = NULL; - - mas->index = mas->min; - if (mas->index > limit) - goto none; - - max = mas->max; - mas->offset = 0; - while (likely(!ma_is_leaf(mt))) { - MAS_WARN_ON(mas, mte_dead_node(mas->node)); - slots = ma_slots(mn, mt); - entry = mas_slot(mas, slots, 0); - pivots = ma_pivots(mn, mt); - if (unlikely(ma_dead_node(mn))) - return NULL; - max = pivots[0]; - mas->node = entry; - mn = mas_mn(mas); - mt = mte_node_type(mas->node); - } - MAS_WARN_ON(mas, mte_dead_node(mas->node)); - - mas->max = max; - slots = ma_slots(mn, mt); - entry = mas_slot(mas, slots, 0); - if (unlikely(ma_dead_node(mn))) - return NULL; - - /* Slot 0 or 1 must be set */ - if (mas->index > limit) - goto none; - - if (likely(entry)) - return entry; - - mas->offset = 1; - entry = mas_slot(mas, slots, 1); - pivots = ma_pivots(mn, mt); - if (unlikely(ma_dead_node(mn))) - return NULL; - - mas->index = pivots[0] + 1; - if (mas->index > limit) - goto none; - - if (likely(entry)) - return entry; - -none: - if (likely(!ma_dead_node(mn))) - mas->node = MAS_NONE; - return NULL; -} - /* Depth first search, post-order */ static void mas_dfs_postorder(struct ma_state *mas, unsigned long max) { @@ -6846,11 +6808,27 @@ static void mt_dump_arange64(const struct maple_tree *mt, void *entry, int i; pr_cont(" contents: "); - for (i = 0; i < MAPLE_ARANGE64_SLOTS; i++) - pr_cont("%lu ", node->gap[i]); + for (i = 0; i < MAPLE_ARANGE64_SLOTS; i++) { + switch (format) { + case mt_dump_hex: + pr_cont("%lx ", node->gap[i]); + break; + default: + case mt_dump_dec: + pr_cont("%lu ", node->gap[i]); + } + } pr_cont("| %02X %02X| ", node->meta.end, node->meta.gap); - for (i = 0; i < MAPLE_ARANGE64_SLOTS - 1; i++) - pr_cont("%p %lu ", node->slot[i], node->pivot[i]); + for (i = 0; i < MAPLE_ARANGE64_SLOTS - 1; i++) { + switch (format) { + case mt_dump_hex: + pr_cont("%p %lX ", node->slot[i], node->pivot[i]); + break; + default: + case mt_dump_dec: + pr_cont("%p %lu ", node->slot[i], node->pivot[i]); + } + } pr_cont("%p\n", node->slot[i]); for (i = 0; i < MAPLE_ARANGE64_SLOTS; i++) { unsigned long last = max; @@ -6934,15 +6912,16 @@ EXPORT_SYMBOL_GPL(mt_dump); static void mas_validate_gaps(struct ma_state *mas) { struct maple_enode *mte = mas->node; - struct maple_node *p_mn; + struct maple_node *p_mn, *node = mte_to_node(mte); + enum maple_type mt = mte_node_type(mas->node); unsigned long gap = 0, max_gap = 0; unsigned long p_end, p_start = mas->min; - unsigned char p_slot; + unsigned char p_slot, offset; unsigned long *gaps = NULL; - unsigned long *pivots = ma_pivots(mte_to_node(mte), mte_node_type(mte)); - int i; + unsigned long *pivots = ma_pivots(node, mt); + unsigned int i; - if (ma_is_dense(mte_node_type(mte))) { + if (ma_is_dense(mt)) { for (i = 0; i < mt_slot_count(mte); i++) { if (mas_get_slot(mas, i)) { if (gap > max_gap) @@ -6955,52 +6934,59 @@ static void mas_validate_gaps(struct ma_state *mas) goto counted; } - gaps = ma_gaps(mte_to_node(mte), mte_node_type(mte)); + gaps = ma_gaps(node, mt); for (i = 0; i < mt_slot_count(mte); i++) { - p_end = mas_logical_pivot(mas, pivots, i, mte_node_type(mte)); + p_end = mas_safe_pivot(mas, pivots, i, mt); if (!gaps) { - if (mas_get_slot(mas, i)) { - gap = 0; - goto not_empty; - } - - gap += p_end - p_start + 1; + if (!mas_get_slot(mas, i)) + gap = p_end - p_start + 1; } else { void *entry = mas_get_slot(mas, i); gap = gaps[i]; - if (!entry) { - if (gap != p_end - p_start + 1) { - pr_err("%p[%u] -> %p %lu != %lu - %lu + 1\n", - mas_mn(mas), i, - mas_get_slot(mas, i), gap, - p_end, p_start); - mt_dump(mas->tree, mt_dump_hex); - - MT_BUG_ON(mas->tree, - gap != p_end - p_start + 1); - } - } else { - if (gap > p_end - p_start + 1) { - pr_err("%p[%u] %lu >= %lu - %lu + 1 (%lu)\n", - mas_mn(mas), i, gap, p_end, p_start, - p_end - p_start + 1); - MT_BUG_ON(mas->tree, - gap > p_end - p_start + 1); - } + MT_BUG_ON(mas->tree, !entry); + + if (gap > p_end - p_start + 1) { + pr_err("%p[%u] %lu >= %lu - %lu + 1 (%lu)\n", + mas_mn(mas), i, gap, p_end, p_start, + p_end - p_start + 1); + MT_BUG_ON(mas->tree, gap > p_end - p_start + 1); } } if (gap > max_gap) max_gap = gap; -not_empty: + p_start = p_end + 1; if (p_end >= mas->max) break; } counted: + if (mt == maple_arange_64) { + offset = ma_meta_gap(node, mt); + if (offset > i) { + pr_err("gap offset %p[%u] is invalid\n", node, offset); + MT_BUG_ON(mas->tree, 1); + } + + if (gaps[offset] != max_gap) { + pr_err("gap %p[%u] is not the largest gap %lu\n", + node, offset, max_gap); + MT_BUG_ON(mas->tree, 1); + } + + MT_BUG_ON(mas->tree, !gaps); + for (i++ ; i < mt_slot_count(mte); i++) { + if (gaps[i] != 0) { + pr_err("gap %p[%u] beyond node limit != 0\n", + node, i); + MT_BUG_ON(mas->tree, 1); + } + } + } + if (mte_is_root(mte)) return; @@ -7010,10 +6996,8 @@ counted: if (ma_gaps(p_mn, mas_parent_type(mas, mte))[p_slot] != max_gap) { pr_err("gap %p[%u] != %lu\n", p_mn, p_slot, max_gap); mt_dump(mas->tree, mt_dump_hex); + MT_BUG_ON(mas->tree, 1); } - - MT_BUG_ON(mas->tree, - ma_gaps(p_mn, mas_parent_type(mas, mte))[p_slot] != max_gap); } static void mas_validate_parent_slot(struct ma_state *mas) @@ -7064,11 +7048,12 @@ static void mas_validate_child_slot(struct ma_state *mas) for (i = 0; i < mt_slots[type]; i++) { child = mas_slot(mas, slots, i); - if (!pivots[i] || pivots[i] == mas->max) - break; - if (!child) - break; + if (!child) { + pr_err("Non-leaf node lacks child at %p[%u]\n", + mas_mn(mas), i); + MT_BUG_ON(mas->tree, 1); + } if (mte_parent_slot(child) != i) { pr_err("Slot error at %p[%u]: child %p has pslot %u\n", @@ -7083,11 +7068,16 @@ static void mas_validate_child_slot(struct ma_state *mas) mte_to_node(mas->node)); MT_BUG_ON(mas->tree, 1); } + + if (i < mt_pivots[type] && pivots[i] == mas->max) + break; } } /* - * Validate all pivots are within mas->min and mas->max. + * Validate all pivots are within mas->min and mas->max, check metadata ends + * where the maximum ends and ensure there is no slots or pivots set outside of + * the end of the data. */ static void mas_validate_limits(struct ma_state *mas) { @@ -7097,26 +7087,15 @@ static void mas_validate_limits(struct ma_state *mas) void __rcu **slots = ma_slots(mte_to_node(mas->node), type); unsigned long *pivots = ma_pivots(mas_mn(mas), type); - /* all limits are fine here. */ - if (mte_is_root(mas->node)) - return; - for (i = 0; i < mt_slots[type]; i++) { unsigned long piv; piv = mas_safe_pivot(mas, pivots, i, type); - if (!piv && (i != 0)) - break; - - if (!mte_is_leaf(mas->node)) { - void *entry = mas_slot(mas, slots, i); - - if (!entry) - pr_err("%p[%u] cannot be null\n", - mas_mn(mas), i); - - MT_BUG_ON(mas->tree, !entry); + if (!piv && (i != 0)) { + pr_err("Missing node limit pivot at %p[%u]", + mas_mn(mas), i); + MAS_WARN_ON(mas, 1); } if (prev_piv > piv) { @@ -7139,6 +7118,13 @@ static void mas_validate_limits(struct ma_state *mas) if (piv == mas->max) break; } + + if (mas_data_end(mas) != i) { + pr_err("node%p: data_end %u != the last slot offset %u\n", + mas_mn(mas), mas_data_end(mas), i); + MT_BUG_ON(mas->tree, 1); + } + for (i += 1; i < mt_slots[type]; i++) { void *entry = mas_slot(mas, slots, i); @@ -7213,21 +7199,20 @@ void mt_validate(struct maple_tree *mt) if (!mas_searchable(&mas)) goto done; - mas_first_entry(&mas, mas_mn(&mas), ULONG_MAX, mte_node_type(mas.node)); + while (!mte_is_leaf(mas.node)) + mas_descend(&mas); + while (!mas_is_none(&mas)) { MAS_WARN_ON(&mas, mte_dead_node(mas.node)); - if (!mte_is_root(mas.node)) { - end = mas_data_end(&mas); - if (MAS_WARN_ON(&mas, - (end < mt_min_slot_count(mas.node)) && - (mas.max != ULONG_MAX))) { - pr_err("Invalid size %u of %p\n", end, - mas_mn(&mas)); - } + end = mas_data_end(&mas); + if (MAS_WARN_ON(&mas, (end < mt_min_slot_count(mas.node)) && + (mas.max != ULONG_MAX))) { + pr_err("Invalid size %u of %p\n", end, mas_mn(&mas)); } + mas_validate_parent_slot(&mas); - mas_validate_child_slot(&mas); mas_validate_limits(&mas); + mas_validate_child_slot(&mas); if (mt_is_alloc(mt)) mas_validate_gaps(&mas); mas_dfs_postorder(&mas, ULONG_MAX); diff --git a/lib/math/Makefile b/lib/math/Makefile index bfac26ddfc22..91fcdb0c9efe 100644 --- a/lib/math/Makefile +++ b/lib/math/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-y += div64.o gcd.o lcm.o int_pow.o int_sqrt.o reciprocal_div.o +obj-y += div64.o gcd.o lcm.o int_log.o int_pow.o int_sqrt.o reciprocal_div.o obj-$(CONFIG_CORDIC) += cordic.o obj-$(CONFIG_PRIME_NUMBERS) += prime_numbers.o diff --git a/lib/math/int_log.c b/lib/math/int_log.c new file mode 100644 index 000000000000..8f9da3a2ad39 --- /dev/null +++ b/lib/math/int_log.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: LGPL-2.1-or-later +/* + * Provides fixed-point logarithm operations. + * + * Copyright (C) 2006 Christoph Pfister (christophpfister@gmail.com) + */ + +#include <linux/bitops.h> +#include <linux/export.h> +#include <linux/int_log.h> +#include <linux/kernel.h> +#include <linux/types.h> + +#include <asm/bug.h> + +static const unsigned short logtable[256] = { + 0x0000, 0x0171, 0x02e0, 0x044e, 0x05ba, 0x0725, 0x088e, 0x09f7, + 0x0b5d, 0x0cc3, 0x0e27, 0x0f8a, 0x10eb, 0x124b, 0x13aa, 0x1508, + 0x1664, 0x17bf, 0x1919, 0x1a71, 0x1bc8, 0x1d1e, 0x1e73, 0x1fc6, + 0x2119, 0x226a, 0x23ba, 0x2508, 0x2656, 0x27a2, 0x28ed, 0x2a37, + 0x2b80, 0x2cc8, 0x2e0f, 0x2f54, 0x3098, 0x31dc, 0x331e, 0x345f, + 0x359f, 0x36de, 0x381b, 0x3958, 0x3a94, 0x3bce, 0x3d08, 0x3e41, + 0x3f78, 0x40af, 0x41e4, 0x4319, 0x444c, 0x457f, 0x46b0, 0x47e1, + 0x4910, 0x4a3f, 0x4b6c, 0x4c99, 0x4dc5, 0x4eef, 0x5019, 0x5142, + 0x526a, 0x5391, 0x54b7, 0x55dc, 0x5700, 0x5824, 0x5946, 0x5a68, + 0x5b89, 0x5ca8, 0x5dc7, 0x5ee5, 0x6003, 0x611f, 0x623a, 0x6355, + 0x646f, 0x6588, 0x66a0, 0x67b7, 0x68ce, 0x69e4, 0x6af8, 0x6c0c, + 0x6d20, 0x6e32, 0x6f44, 0x7055, 0x7165, 0x7274, 0x7383, 0x7490, + 0x759d, 0x76aa, 0x77b5, 0x78c0, 0x79ca, 0x7ad3, 0x7bdb, 0x7ce3, + 0x7dea, 0x7ef0, 0x7ff6, 0x80fb, 0x81ff, 0x8302, 0x8405, 0x8507, + 0x8608, 0x8709, 0x8809, 0x8908, 0x8a06, 0x8b04, 0x8c01, 0x8cfe, + 0x8dfa, 0x8ef5, 0x8fef, 0x90e9, 0x91e2, 0x92db, 0x93d2, 0x94ca, + 0x95c0, 0x96b6, 0x97ab, 0x98a0, 0x9994, 0x9a87, 0x9b7a, 0x9c6c, + 0x9d5e, 0x9e4f, 0x9f3f, 0xa02e, 0xa11e, 0xa20c, 0xa2fa, 0xa3e7, + 0xa4d4, 0xa5c0, 0xa6ab, 0xa796, 0xa881, 0xa96a, 0xaa53, 0xab3c, + 0xac24, 0xad0c, 0xadf2, 0xaed9, 0xafbe, 0xb0a4, 0xb188, 0xb26c, + 0xb350, 0xb433, 0xb515, 0xb5f7, 0xb6d9, 0xb7ba, 0xb89a, 0xb97a, + 0xba59, 0xbb38, 0xbc16, 0xbcf4, 0xbdd1, 0xbead, 0xbf8a, 0xc065, + 0xc140, 0xc21b, 0xc2f5, 0xc3cf, 0xc4a8, 0xc580, 0xc658, 0xc730, + 0xc807, 0xc8de, 0xc9b4, 0xca8a, 0xcb5f, 0xcc34, 0xcd08, 0xcddc, + 0xceaf, 0xcf82, 0xd054, 0xd126, 0xd1f7, 0xd2c8, 0xd399, 0xd469, + 0xd538, 0xd607, 0xd6d6, 0xd7a4, 0xd872, 0xd93f, 0xda0c, 0xdad9, + 0xdba5, 0xdc70, 0xdd3b, 0xde06, 0xded0, 0xdf9a, 0xe063, 0xe12c, + 0xe1f5, 0xe2bd, 0xe385, 0xe44c, 0xe513, 0xe5d9, 0xe69f, 0xe765, + 0xe82a, 0xe8ef, 0xe9b3, 0xea77, 0xeb3b, 0xebfe, 0xecc1, 0xed83, + 0xee45, 0xef06, 0xefc8, 0xf088, 0xf149, 0xf209, 0xf2c8, 0xf387, + 0xf446, 0xf505, 0xf5c3, 0xf680, 0xf73e, 0xf7fb, 0xf8b7, 0xf973, + 0xfa2f, 0xfaea, 0xfba5, 0xfc60, 0xfd1a, 0xfdd4, 0xfe8e, 0xff47, +}; + +unsigned int intlog2(u32 value) +{ + /** + * returns: log2(value) * 2^24 + * wrong result if value = 0 (log2(0) is undefined) + */ + unsigned int msb; + unsigned int logentry; + unsigned int significand; + unsigned int interpolation; + + if (unlikely(value == 0)) { + WARN_ON(1); + return 0; + } + + /* first detect the msb (count begins at 0) */ + msb = fls(value) - 1; + + /** + * now we use a logtable after the following method: + * + * log2(2^x * y) * 2^24 = x * 2^24 + log2(y) * 2^24 + * where x = msb and therefore 1 <= y < 2 + * first y is determined by shifting the value left + * so that msb is bit 31 + * 0x00231f56 -> 0x8C7D5800 + * the result is y * 2^31 -> "significand" + * then the highest 9 bits are used for a table lookup + * the highest bit is discarded because it's always set + * the highest nine bits in our example are 100011000 + * so we would use the entry 0x18 + */ + significand = value << (31 - msb); + logentry = (significand >> 23) % ARRAY_SIZE(logtable); + + /** + * last step we do is interpolation because of the + * limitations of the log table the error is that part of + * the significand which isn't used for lookup then we + * compute the ratio between the error and the next table entry + * and interpolate it between the log table entry used and the + * next one the biggest error possible is 0x7fffff + * (in our example it's 0x7D5800) + * needed value for next table entry is 0x800000 + * so the interpolation is + * (error / 0x800000) * (logtable_next - logtable_current) + * in the implementation the division is moved to the end for + * better accuracy there is also an overflow correction if + * logtable_next is 256 + */ + interpolation = ((significand & 0x7fffff) * + ((logtable[(logentry + 1) % ARRAY_SIZE(logtable)] - + logtable[logentry]) & 0xffff)) >> 15; + + /* now we return the result */ + return ((msb << 24) + (logtable[logentry] << 8) + interpolation); +} +EXPORT_SYMBOL(intlog2); + +unsigned int intlog10(u32 value) +{ + /** + * returns: log10(value) * 2^24 + * wrong result if value = 0 (log10(0) is undefined) + */ + u64 log; + + if (unlikely(value == 0)) { + WARN_ON(1); + return 0; + } + + log = intlog2(value); + + /** + * we use the following method: + * log10(x) = log2(x) * log10(2) + */ + + return (log * 646456993) >> 31; +} +EXPORT_SYMBOL(intlog10); diff --git a/lib/memcpy_kunit.c b/lib/memcpy_kunit.c index 887926f04731..440aee705ccc 100644 --- a/lib/memcpy_kunit.c +++ b/lib/memcpy_kunit.c @@ -551,10 +551,10 @@ static void strtomem_test(struct kunit *test) static struct kunit_case memcpy_test_cases[] = { KUNIT_CASE(memset_test), KUNIT_CASE(memcpy_test), - KUNIT_CASE(memcpy_large_test), - KUNIT_CASE(memmove_test), - KUNIT_CASE(memmove_large_test), - KUNIT_CASE(memmove_overlap_test), + KUNIT_CASE_SLOW(memcpy_large_test), + KUNIT_CASE_SLOW(memmove_test), + KUNIT_CASE_SLOW(memmove_large_test), + KUNIT_CASE_SLOW(memmove_overlap_test), KUNIT_CASE(strtomem_test), {} }; diff --git a/lib/nlattr.c b/lib/nlattr.c index 489e15bde5c1..dc15e7888fc1 100644 --- a/lib/nlattr.c +++ b/lib/nlattr.c @@ -134,6 +134,7 @@ void nla_get_range_unsigned(const struct nla_policy *pt, range->max = U32_MAX; break; case NLA_U64: + case NLA_UINT: case NLA_MSECS: range->max = U64_MAX; break; @@ -183,6 +184,9 @@ static int nla_validate_range_unsigned(const struct nla_policy *pt, case NLA_U64: value = nla_get_u64(nla); break; + case NLA_UINT: + value = nla_get_uint(nla); + break; case NLA_MSECS: value = nla_get_u64(nla); break; @@ -248,6 +252,7 @@ void nla_get_range_signed(const struct nla_policy *pt, range->max = S32_MAX; break; case NLA_S64: + case NLA_SINT: range->min = S64_MIN; range->max = S64_MAX; break; @@ -295,6 +300,9 @@ static int nla_validate_int_range_signed(const struct nla_policy *pt, case NLA_S64: value = nla_get_s64(nla); break; + case NLA_SINT: + value = nla_get_sint(nla); + break; default: return -EINVAL; } @@ -320,6 +328,7 @@ static int nla_validate_int_range(const struct nla_policy *pt, case NLA_U16: case NLA_U32: case NLA_U64: + case NLA_UINT: case NLA_MSECS: case NLA_BINARY: case NLA_BE16: @@ -329,6 +338,7 @@ static int nla_validate_int_range(const struct nla_policy *pt, case NLA_S16: case NLA_S32: case NLA_S64: + case NLA_SINT: return nla_validate_int_range_signed(pt, nla, extack); default: WARN_ON(1); @@ -355,6 +365,15 @@ static int nla_validate_mask(const struct nla_policy *pt, case NLA_U64: value = nla_get_u64(nla); break; + case NLA_UINT: + value = nla_get_uint(nla); + break; + case NLA_BE16: + value = ntohs(nla_get_be16(nla)); + break; + case NLA_BE32: + value = ntohl(nla_get_be32(nla)); + break; default: return -EINVAL; } @@ -427,6 +446,15 @@ static int validate_nla(const struct nlattr *nla, int maxtype, goto out_err; break; + case NLA_SINT: + case NLA_UINT: + if (attrlen != sizeof(u32) && attrlen != sizeof(u64)) { + NL_SET_ERR_MSG_ATTR_POL(extack, nla, pt, + "invalid attribute length"); + return -EINVAL; + } + break; + case NLA_BITFIELD32: if (attrlen != sizeof(struct nla_bitfield32)) goto out_err; diff --git a/lib/nmi_backtrace.c b/lib/nmi_backtrace.c index 5274bbb026d7..33c154264bfe 100644 --- a/lib/nmi_backtrace.c +++ b/lib/nmi_backtrace.c @@ -34,7 +34,7 @@ static unsigned long backtrace_flag; * they are passed being updated as a side effect of this call. */ void nmi_trigger_cpumask_backtrace(const cpumask_t *mask, - bool exclude_self, + int exclude_cpu, void (*raise)(cpumask_t *mask)) { int i, this_cpu = get_cpu(); @@ -49,8 +49,8 @@ void nmi_trigger_cpumask_backtrace(const cpumask_t *mask, } cpumask_copy(to_cpumask(backtrace_mask), mask); - if (exclude_self) - cpumask_clear_cpu(this_cpu, to_cpumask(backtrace_mask)); + if (exclude_cpu != -1) + cpumask_clear_cpu(exclude_cpu, to_cpumask(backtrace_mask)); /* * Don't try to send an NMI to this cpu; it may work on some diff --git a/lib/notifier-error-inject.c b/lib/notifier-error-inject.c index 2b24ea6c9497..954c3412d22d 100644 --- a/lib/notifier-error-inject.c +++ b/lib/notifier-error-inject.c @@ -83,9 +83,6 @@ static int __init err_inject_init(void) notifier_err_inject_dir = debugfs_create_dir("notifier-error-inject", NULL); - if (!notifier_err_inject_dir) - return -ENOMEM; - return 0; } diff --git a/lib/objpool.c b/lib/objpool.c new file mode 100644 index 000000000000..cfdc02420884 --- /dev/null +++ b/lib/objpool.c @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/objpool.h> +#include <linux/slab.h> +#include <linux/vmalloc.h> +#include <linux/atomic.h> +#include <linux/irqflags.h> +#include <linux/cpumask.h> +#include <linux/log2.h> + +/* + * objpool: ring-array based lockless MPMC/FIFO queues + * + * Copyright: wuqiang.matt@bytedance.com,mhiramat@kernel.org + */ + +/* initialize percpu objpool_slot */ +static int +objpool_init_percpu_slot(struct objpool_head *pool, + struct objpool_slot *slot, + int nodes, void *context, + objpool_init_obj_cb objinit) +{ + void *obj = (void *)&slot->entries[pool->capacity]; + int i; + + /* initialize elements of percpu objpool_slot */ + slot->mask = pool->capacity - 1; + + for (i = 0; i < nodes; i++) { + if (objinit) { + int rc = objinit(obj, context); + if (rc) + return rc; + } + slot->entries[slot->tail & slot->mask] = obj; + obj = obj + pool->obj_size; + slot->tail++; + slot->last = slot->tail; + pool->nr_objs++; + } + + return 0; +} + +/* allocate and initialize percpu slots */ +static int +objpool_init_percpu_slots(struct objpool_head *pool, int nr_objs, + void *context, objpool_init_obj_cb objinit) +{ + int i, cpu_count = 0; + + for (i = 0; i < pool->nr_cpus; i++) { + + struct objpool_slot *slot; + int nodes, size, rc; + + /* skip the cpu node which could never be present */ + if (!cpu_possible(i)) + continue; + + /* compute how many objects to be allocated with this slot */ + nodes = nr_objs / num_possible_cpus(); + if (cpu_count < (nr_objs % num_possible_cpus())) + nodes++; + cpu_count++; + + size = struct_size(slot, entries, pool->capacity) + + pool->obj_size * nodes; + + /* + * here we allocate percpu-slot & objs together in a single + * allocation to make it more compact, taking advantage of + * warm caches and TLB hits. in default vmalloc is used to + * reduce the pressure of kernel slab system. as we know, + * mimimal size of vmalloc is one page since vmalloc would + * always align the requested size to page size + */ + if (pool->gfp & GFP_ATOMIC) + slot = kmalloc_node(size, pool->gfp, cpu_to_node(i)); + else + slot = __vmalloc_node(size, sizeof(void *), pool->gfp, + cpu_to_node(i), __builtin_return_address(0)); + if (!slot) + return -ENOMEM; + memset(slot, 0, size); + pool->cpu_slots[i] = slot; + + /* initialize the objpool_slot of cpu node i */ + rc = objpool_init_percpu_slot(pool, slot, nodes, context, objinit); + if (rc) + return rc; + } + + return 0; +} + +/* cleanup all percpu slots of the object pool */ +static void objpool_fini_percpu_slots(struct objpool_head *pool) +{ + int i; + + if (!pool->cpu_slots) + return; + + for (i = 0; i < pool->nr_cpus; i++) + kvfree(pool->cpu_slots[i]); + kfree(pool->cpu_slots); +} + +/* initialize object pool and pre-allocate objects */ +int objpool_init(struct objpool_head *pool, int nr_objs, int object_size, + gfp_t gfp, void *context, objpool_init_obj_cb objinit, + objpool_fini_cb release) +{ + int rc, capacity, slot_size; + + /* check input parameters */ + if (nr_objs <= 0 || nr_objs > OBJPOOL_NR_OBJECT_MAX || + object_size <= 0 || object_size > OBJPOOL_OBJECT_SIZE_MAX) + return -EINVAL; + + /* align up to unsigned long size */ + object_size = ALIGN(object_size, sizeof(long)); + + /* calculate capacity of percpu objpool_slot */ + capacity = roundup_pow_of_two(nr_objs); + if (!capacity) + return -EINVAL; + + /* initialize objpool pool */ + memset(pool, 0, sizeof(struct objpool_head)); + pool->nr_cpus = nr_cpu_ids; + pool->obj_size = object_size; + pool->capacity = capacity; + pool->gfp = gfp & ~__GFP_ZERO; + pool->context = context; + pool->release = release; + slot_size = pool->nr_cpus * sizeof(struct objpool_slot); + pool->cpu_slots = kzalloc(slot_size, pool->gfp); + if (!pool->cpu_slots) + return -ENOMEM; + + /* initialize per-cpu slots */ + rc = objpool_init_percpu_slots(pool, nr_objs, context, objinit); + if (rc) + objpool_fini_percpu_slots(pool); + else + refcount_set(&pool->ref, pool->nr_objs + 1); + + return rc; +} +EXPORT_SYMBOL_GPL(objpool_init); + +/* adding object to slot, abort if the slot was already full */ +static inline int +objpool_try_add_slot(void *obj, struct objpool_head *pool, int cpu) +{ + struct objpool_slot *slot = pool->cpu_slots[cpu]; + uint32_t head, tail; + + /* loading tail and head as a local snapshot, tail first */ + tail = READ_ONCE(slot->tail); + + do { + head = READ_ONCE(slot->head); + /* fault caught: something must be wrong */ + WARN_ON_ONCE(tail - head > pool->nr_objs); + } while (!try_cmpxchg_acquire(&slot->tail, &tail, tail + 1)); + + /* now the tail position is reserved for the given obj */ + WRITE_ONCE(slot->entries[tail & slot->mask], obj); + /* update sequence to make this obj available for pop() */ + smp_store_release(&slot->last, tail + 1); + + return 0; +} + +/* reclaim an object to object pool */ +int objpool_push(void *obj, struct objpool_head *pool) +{ + unsigned long flags; + int rc; + + /* disable local irq to avoid preemption & interruption */ + raw_local_irq_save(flags); + rc = objpool_try_add_slot(obj, pool, raw_smp_processor_id()); + raw_local_irq_restore(flags); + + return rc; +} +EXPORT_SYMBOL_GPL(objpool_push); + +/* try to retrieve object from slot */ +static inline void *objpool_try_get_slot(struct objpool_head *pool, int cpu) +{ + struct objpool_slot *slot = pool->cpu_slots[cpu]; + /* load head snapshot, other cpus may change it */ + uint32_t head = smp_load_acquire(&slot->head); + + while (head != READ_ONCE(slot->last)) { + void *obj; + + /* + * data visibility of 'last' and 'head' could be out of + * order since memory updating of 'last' and 'head' are + * performed in push() and pop() independently + * + * before any retrieving attempts, pop() must guarantee + * 'last' is behind 'head', that is to say, there must + * be available objects in slot, which could be ensured + * by condition 'last != head && last - head <= nr_objs' + * that is equivalent to 'last - head - 1 < nr_objs' as + * 'last' and 'head' are both unsigned int32 + */ + if (READ_ONCE(slot->last) - head - 1 >= pool->nr_objs) { + head = READ_ONCE(slot->head); + continue; + } + + /* obj must be retrieved before moving forward head */ + obj = READ_ONCE(slot->entries[head & slot->mask]); + + /* move head forward to mark it's consumption */ + if (try_cmpxchg_release(&slot->head, &head, head + 1)) + return obj; + } + + return NULL; +} + +/* allocate an object from object pool */ +void *objpool_pop(struct objpool_head *pool) +{ + void *obj = NULL; + unsigned long flags; + int i, cpu; + + /* disable local irq to avoid preemption & interruption */ + raw_local_irq_save(flags); + + cpu = raw_smp_processor_id(); + for (i = 0; i < num_possible_cpus(); i++) { + obj = objpool_try_get_slot(pool, cpu); + if (obj) + break; + cpu = cpumask_next_wrap(cpu, cpu_possible_mask, -1, 1); + } + raw_local_irq_restore(flags); + + return obj; +} +EXPORT_SYMBOL_GPL(objpool_pop); + +/* release whole objpool forcely */ +void objpool_free(struct objpool_head *pool) +{ + if (!pool->cpu_slots) + return; + + /* release percpu slots */ + objpool_fini_percpu_slots(pool); + + /* call user's cleanup callback if provided */ + if (pool->release) + pool->release(pool, pool->context); +} +EXPORT_SYMBOL_GPL(objpool_free); + +/* drop the allocated object, rather reclaim it to objpool */ +int objpool_drop(void *obj, struct objpool_head *pool) +{ + if (!obj || !pool) + return -EINVAL; + + if (refcount_dec_and_test(&pool->ref)) { + objpool_free(pool); + return 0; + } + + return -EAGAIN; +} +EXPORT_SYMBOL_GPL(objpool_drop); + +/* drop unused objects and defref objpool for releasing */ +void objpool_fini(struct objpool_head *pool) +{ + int count = 1; /* extra ref for objpool itself */ + + /* drop all remained objects from objpool */ + while (objpool_pop(pool)) + count++; + + if (refcount_sub_and_test(count, &pool->ref)) + objpool_free(pool); +} +EXPORT_SYMBOL_GPL(objpool_fini); diff --git a/lib/percpu_counter.c b/lib/percpu_counter.c index 5004463c4f9f..44dd133594d4 100644 --- a/lib/percpu_counter.c +++ b/lib/percpu_counter.c @@ -151,48 +151,72 @@ s64 __percpu_counter_sum(struct percpu_counter *fbc) } EXPORT_SYMBOL(__percpu_counter_sum); -int __percpu_counter_init(struct percpu_counter *fbc, s64 amount, gfp_t gfp, - struct lock_class_key *key) +int __percpu_counter_init_many(struct percpu_counter *fbc, s64 amount, + gfp_t gfp, u32 nr_counters, + struct lock_class_key *key) { unsigned long flags __maybe_unused; - - raw_spin_lock_init(&fbc->lock); - lockdep_set_class(&fbc->lock, key); - fbc->count = amount; - fbc->counters = alloc_percpu_gfp(s32, gfp); - if (!fbc->counters) + size_t counter_size; + s32 __percpu *counters; + u32 i; + + counter_size = ALIGN(sizeof(*counters), __alignof__(*counters)); + counters = __alloc_percpu_gfp(nr_counters * counter_size, + __alignof__(*counters), gfp); + if (!counters) { + fbc[0].counters = NULL; return -ENOMEM; + } + + for (i = 0; i < nr_counters; i++) { + raw_spin_lock_init(&fbc[i].lock); + lockdep_set_class(&fbc[i].lock, key); +#ifdef CONFIG_HOTPLUG_CPU + INIT_LIST_HEAD(&fbc[i].list); +#endif + fbc[i].count = amount; + fbc[i].counters = (void *)counters + (i * counter_size); - debug_percpu_counter_activate(fbc); + debug_percpu_counter_activate(&fbc[i]); + } #ifdef CONFIG_HOTPLUG_CPU - INIT_LIST_HEAD(&fbc->list); spin_lock_irqsave(&percpu_counters_lock, flags); - list_add(&fbc->list, &percpu_counters); + for (i = 0; i < nr_counters; i++) + list_add(&fbc[i].list, &percpu_counters); spin_unlock_irqrestore(&percpu_counters_lock, flags); #endif return 0; } -EXPORT_SYMBOL(__percpu_counter_init); +EXPORT_SYMBOL(__percpu_counter_init_many); -void percpu_counter_destroy(struct percpu_counter *fbc) +void percpu_counter_destroy_many(struct percpu_counter *fbc, u32 nr_counters) { unsigned long flags __maybe_unused; + u32 i; - if (!fbc->counters) + if (WARN_ON_ONCE(!fbc)) return; - debug_percpu_counter_deactivate(fbc); + if (!fbc[0].counters) + return; + + for (i = 0; i < nr_counters; i++) + debug_percpu_counter_deactivate(&fbc[i]); #ifdef CONFIG_HOTPLUG_CPU spin_lock_irqsave(&percpu_counters_lock, flags); - list_del(&fbc->list); + for (i = 0; i < nr_counters; i++) + list_del(&fbc[i].list); spin_unlock_irqrestore(&percpu_counters_lock, flags); #endif - free_percpu(fbc->counters); - fbc->counters = NULL; + + free_percpu(fbc[0].counters); + + for (i = 0; i < nr_counters; i++) + fbc[i].counters = NULL; } -EXPORT_SYMBOL(percpu_counter_destroy); +EXPORT_SYMBOL(percpu_counter_destroy_many); int percpu_counter_batch __read_mostly = 32; EXPORT_SYMBOL(percpu_counter_batch); @@ -254,6 +278,85 @@ int __percpu_counter_compare(struct percpu_counter *fbc, s64 rhs, s32 batch) } EXPORT_SYMBOL(__percpu_counter_compare); +/* + * Compare counter, and add amount if total is: less than or equal to limit if + * amount is positive, or greater than or equal to limit if amount is negative. + * Return true if amount is added, or false if total would be beyond the limit. + * + * Negative limit is allowed, but unusual. + * When negative amounts (subs) are given to percpu_counter_limited_add(), + * the limit would most naturally be 0 - but other limits are also allowed. + * + * Overflow beyond S64_MAX is not allowed for: counter, limit and amount + * are all assumed to be sane (far from S64_MIN and S64_MAX). + */ +bool __percpu_counter_limited_add(struct percpu_counter *fbc, + s64 limit, s64 amount, s32 batch) +{ + s64 count; + s64 unknown; + unsigned long flags; + bool good = false; + + if (amount == 0) + return true; + + local_irq_save(flags); + unknown = batch * num_online_cpus(); + count = __this_cpu_read(*fbc->counters); + + /* Skip taking the lock when safe */ + if (abs(count + amount) <= batch && + ((amount > 0 && fbc->count + unknown <= limit) || + (amount < 0 && fbc->count - unknown >= limit))) { + this_cpu_add(*fbc->counters, amount); + local_irq_restore(flags); + return true; + } + + raw_spin_lock(&fbc->lock); + count = fbc->count + amount; + + /* Skip percpu_counter_sum() when safe */ + if (amount > 0) { + if (count - unknown > limit) + goto out; + if (count + unknown <= limit) + good = true; + } else { + if (count + unknown < limit) + goto out; + if (count - unknown >= limit) + good = true; + } + + if (!good) { + s32 *pcount; + int cpu; + + for_each_cpu_or(cpu, cpu_online_mask, cpu_dying_mask) { + pcount = per_cpu_ptr(fbc->counters, cpu); + count += *pcount; + } + if (amount > 0) { + if (count > limit) + goto out; + } else { + if (count < limit) + goto out; + } + good = true; + } + + count = __this_cpu_read(*fbc->counters); + fbc->count += count + amount; + __this_cpu_sub(*fbc->counters, count); +out: + raw_spin_unlock(&fbc->lock); + local_irq_restore(flags); + return good; +} + static int __init percpu_counter_startup(void) { int ret; diff --git a/lib/raid6/Makefile b/lib/raid6/Makefile index 45e17619422b..1c5420ff254e 100644 --- a/lib/raid6/Makefile +++ b/lib/raid6/Makefile @@ -2,13 +2,14 @@ obj-$(CONFIG_RAID6_PQ) += raid6_pq.o raid6_pq-y += algos.o recov.o tables.o int1.o int2.o int4.o \ - int8.o int16.o int32.o + int8.o raid6_pq-$(CONFIG_X86) += recov_ssse3.o recov_avx2.o mmx.o sse1.o sse2.o avx2.o avx512.o recov_avx512.o raid6_pq-$(CONFIG_ALTIVEC) += altivec1.o altivec2.o altivec4.o altivec8.o \ vpermxor1.o vpermxor2.o vpermxor4.o vpermxor8.o raid6_pq-$(CONFIG_KERNEL_MODE_NEON) += neon.o neon1.o neon2.o neon4.o neon8.o recov_neon.o recov_neon_inner.o raid6_pq-$(CONFIG_S390) += s390vx8.o recov_s390xc.o +raid6_pq-$(CONFIG_LOONGARCH) += loongarch_simd.o recov_loongarch_simd.o hostprogs += mktables @@ -54,7 +55,7 @@ endif quiet_cmd_unroll = UNROLL $@ cmd_unroll = $(AWK) -v N=$* -f $(srctree)/$(src)/unroll.awk < $< > $@ -targets += int1.c int2.c int4.c int8.c int16.c int32.c +targets += int1.c int2.c int4.c int8.c $(obj)/int%.c: $(src)/int.uc $(src)/unroll.awk FORCE $(call if_changed,unroll) diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c index a22a05c9af8a..cd2e88ee1f14 100644 --- a/lib/raid6/algos.c +++ b/lib/raid6/algos.c @@ -73,9 +73,13 @@ const struct raid6_calls * const raid6_algos[] = { &raid6_neonx2, &raid6_neonx1, #endif -#if defined(__ia64__) - &raid6_intx32, - &raid6_intx16, +#ifdef CONFIG_LOONGARCH +#ifdef CONFIG_CPU_HAS_LASX + &raid6_lasx, +#endif +#ifdef CONFIG_CPU_HAS_LSX + &raid6_lsx, +#endif #endif &raid6_intx8, &raid6_intx4, @@ -104,6 +108,14 @@ const struct raid6_recov_calls *const raid6_recov_algos[] = { #if defined(CONFIG_KERNEL_MODE_NEON) &raid6_recov_neon, #endif +#ifdef CONFIG_LOONGARCH +#ifdef CONFIG_CPU_HAS_LASX + &raid6_recov_lasx, +#endif +#ifdef CONFIG_CPU_HAS_LSX + &raid6_recov_lsx, +#endif +#endif &raid6_recov_intx1, NULL }; diff --git a/lib/raid6/int.uc b/lib/raid6/int.uc index 558aeac9342a..1ba56c3fa482 100644 --- a/lib/raid6/int.uc +++ b/lib/raid6/int.uc @@ -42,13 +42,6 @@ typedef u32 unative_t; /* - * IA-64 wants insane amounts of unrolling. On other architectures that - * is just a waste of space. - */ -#if ($# <= 8) || defined(__ia64__) - - -/* * These sub-operations are separate inlines since they can sometimes be * specially optimized using architecture-specific hacks. */ @@ -152,5 +145,3 @@ const struct raid6_calls raid6_intx$# = { "int" NSTRING "x$#", 0 }; - -#endif diff --git a/lib/raid6/loongarch.h b/lib/raid6/loongarch.h new file mode 100644 index 000000000000..acfc33ce7056 --- /dev/null +++ b/lib/raid6/loongarch.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2023 WANG Xuerui <git@xen0n.name> + * + * raid6/loongarch.h + * + * Definitions common to LoongArch RAID-6 code only + */ + +#ifndef _LIB_RAID6_LOONGARCH_H +#define _LIB_RAID6_LOONGARCH_H + +#ifdef __KERNEL__ + +#include <asm/cpu-features.h> +#include <asm/fpu.h> + +#else /* for user-space testing */ + +#include <sys/auxv.h> + +/* have to supply these defines for glibc 2.37- and musl */ +#ifndef HWCAP_LOONGARCH_LSX +#define HWCAP_LOONGARCH_LSX (1 << 4) +#endif +#ifndef HWCAP_LOONGARCH_LASX +#define HWCAP_LOONGARCH_LASX (1 << 5) +#endif + +#define kernel_fpu_begin() +#define kernel_fpu_end() + +#define cpu_has_lsx (getauxval(AT_HWCAP) & HWCAP_LOONGARCH_LSX) +#define cpu_has_lasx (getauxval(AT_HWCAP) & HWCAP_LOONGARCH_LASX) + +#endif /* __KERNEL__ */ + +#endif /* _LIB_RAID6_LOONGARCH_H */ diff --git a/lib/raid6/loongarch_simd.c b/lib/raid6/loongarch_simd.c new file mode 100644 index 000000000000..aa5d9f924ca3 --- /dev/null +++ b/lib/raid6/loongarch_simd.c @@ -0,0 +1,422 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * RAID6 syndrome calculations in LoongArch SIMD (LSX & LASX) + * + * Copyright 2023 WANG Xuerui <git@xen0n.name> + * + * Based on the generic RAID-6 code (int.uc): + * + * Copyright 2002-2004 H. Peter Anvin + */ + +#include <linux/raid/pq.h> +#include "loongarch.h" + +/* + * The vector algorithms are currently priority 0, which means the generic + * scalar algorithms are not being disabled if vector support is present. + * This is like the similar LoongArch RAID5 XOR code, with the main reason + * repeated here: it cannot be ruled out at this point of time, that some + * future (maybe reduced) models could run the vector algorithms slower than + * the scalar ones, maybe for errata or micro-op reasons. It may be + * appropriate to revisit this after one or two more uarch generations. + */ + +#ifdef CONFIG_CPU_HAS_LSX +#define NSIZE 16 + +static int raid6_has_lsx(void) +{ + return cpu_has_lsx; +} + +static void raid6_lsx_gen_syndrome(int disks, size_t bytes, void **ptrs) +{ + u8 **dptr = (u8 **)ptrs; + u8 *p, *q; + int d, z, z0; + + z0 = disks - 3; /* Highest data disk */ + p = dptr[z0+1]; /* XOR parity */ + q = dptr[z0+2]; /* RS syndrome */ + + kernel_fpu_begin(); + + /* + * $vr0, $vr1, $vr2, $vr3: wp + * $vr4, $vr5, $vr6, $vr7: wq + * $vr8, $vr9, $vr10, $vr11: wd + * $vr12, $vr13, $vr14, $vr15: w2 + * $vr16, $vr17, $vr18, $vr19: w1 + */ + for (d = 0; d < bytes; d += NSIZE*4) { + /* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */ + asm volatile("vld $vr0, %0" : : "m"(dptr[z0][d+0*NSIZE])); + asm volatile("vld $vr1, %0" : : "m"(dptr[z0][d+1*NSIZE])); + asm volatile("vld $vr2, %0" : : "m"(dptr[z0][d+2*NSIZE])); + asm volatile("vld $vr3, %0" : : "m"(dptr[z0][d+3*NSIZE])); + asm volatile("vori.b $vr4, $vr0, 0"); + asm volatile("vori.b $vr5, $vr1, 0"); + asm volatile("vori.b $vr6, $vr2, 0"); + asm volatile("vori.b $vr7, $vr3, 0"); + for (z = z0-1; z >= 0; z--) { + /* wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE]; */ + asm volatile("vld $vr8, %0" : : "m"(dptr[z][d+0*NSIZE])); + asm volatile("vld $vr9, %0" : : "m"(dptr[z][d+1*NSIZE])); + asm volatile("vld $vr10, %0" : : "m"(dptr[z][d+2*NSIZE])); + asm volatile("vld $vr11, %0" : : "m"(dptr[z][d+3*NSIZE])); + /* wp$$ ^= wd$$; */ + asm volatile("vxor.v $vr0, $vr0, $vr8"); + asm volatile("vxor.v $vr1, $vr1, $vr9"); + asm volatile("vxor.v $vr2, $vr2, $vr10"); + asm volatile("vxor.v $vr3, $vr3, $vr11"); + /* w2$$ = MASK(wq$$); */ + asm volatile("vslti.b $vr12, $vr4, 0"); + asm volatile("vslti.b $vr13, $vr5, 0"); + asm volatile("vslti.b $vr14, $vr6, 0"); + asm volatile("vslti.b $vr15, $vr7, 0"); + /* w1$$ = SHLBYTE(wq$$); */ + asm volatile("vslli.b $vr16, $vr4, 1"); + asm volatile("vslli.b $vr17, $vr5, 1"); + asm volatile("vslli.b $vr18, $vr6, 1"); + asm volatile("vslli.b $vr19, $vr7, 1"); + /* w2$$ &= NBYTES(0x1d); */ + asm volatile("vandi.b $vr12, $vr12, 0x1d"); + asm volatile("vandi.b $vr13, $vr13, 0x1d"); + asm volatile("vandi.b $vr14, $vr14, 0x1d"); + asm volatile("vandi.b $vr15, $vr15, 0x1d"); + /* w1$$ ^= w2$$; */ + asm volatile("vxor.v $vr16, $vr16, $vr12"); + asm volatile("vxor.v $vr17, $vr17, $vr13"); + asm volatile("vxor.v $vr18, $vr18, $vr14"); + asm volatile("vxor.v $vr19, $vr19, $vr15"); + /* wq$$ = w1$$ ^ wd$$; */ + asm volatile("vxor.v $vr4, $vr16, $vr8"); + asm volatile("vxor.v $vr5, $vr17, $vr9"); + asm volatile("vxor.v $vr6, $vr18, $vr10"); + asm volatile("vxor.v $vr7, $vr19, $vr11"); + } + /* *(unative_t *)&p[d+NSIZE*$$] = wp$$; */ + asm volatile("vst $vr0, %0" : "=m"(p[d+NSIZE*0])); + asm volatile("vst $vr1, %0" : "=m"(p[d+NSIZE*1])); + asm volatile("vst $vr2, %0" : "=m"(p[d+NSIZE*2])); + asm volatile("vst $vr3, %0" : "=m"(p[d+NSIZE*3])); + /* *(unative_t *)&q[d+NSIZE*$$] = wq$$; */ + asm volatile("vst $vr4, %0" : "=m"(q[d+NSIZE*0])); + asm volatile("vst $vr5, %0" : "=m"(q[d+NSIZE*1])); + asm volatile("vst $vr6, %0" : "=m"(q[d+NSIZE*2])); + asm volatile("vst $vr7, %0" : "=m"(q[d+NSIZE*3])); + } + + kernel_fpu_end(); +} + +static void raid6_lsx_xor_syndrome(int disks, int start, int stop, + size_t bytes, void **ptrs) +{ + u8 **dptr = (u8 **)ptrs; + u8 *p, *q; + int d, z, z0; + + z0 = stop; /* P/Q right side optimization */ + p = dptr[disks-2]; /* XOR parity */ + q = dptr[disks-1]; /* RS syndrome */ + + kernel_fpu_begin(); + + /* + * $vr0, $vr1, $vr2, $vr3: wp + * $vr4, $vr5, $vr6, $vr7: wq + * $vr8, $vr9, $vr10, $vr11: wd + * $vr12, $vr13, $vr14, $vr15: w2 + * $vr16, $vr17, $vr18, $vr19: w1 + */ + for (d = 0; d < bytes; d += NSIZE*4) { + /* P/Q data pages */ + /* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */ + asm volatile("vld $vr0, %0" : : "m"(dptr[z0][d+0*NSIZE])); + asm volatile("vld $vr1, %0" : : "m"(dptr[z0][d+1*NSIZE])); + asm volatile("vld $vr2, %0" : : "m"(dptr[z0][d+2*NSIZE])); + asm volatile("vld $vr3, %0" : : "m"(dptr[z0][d+3*NSIZE])); + asm volatile("vori.b $vr4, $vr0, 0"); + asm volatile("vori.b $vr5, $vr1, 0"); + asm volatile("vori.b $vr6, $vr2, 0"); + asm volatile("vori.b $vr7, $vr3, 0"); + for (z = z0-1; z >= start; z--) { + /* wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE]; */ + asm volatile("vld $vr8, %0" : : "m"(dptr[z][d+0*NSIZE])); + asm volatile("vld $vr9, %0" : : "m"(dptr[z][d+1*NSIZE])); + asm volatile("vld $vr10, %0" : : "m"(dptr[z][d+2*NSIZE])); + asm volatile("vld $vr11, %0" : : "m"(dptr[z][d+3*NSIZE])); + /* wp$$ ^= wd$$; */ + asm volatile("vxor.v $vr0, $vr0, $vr8"); + asm volatile("vxor.v $vr1, $vr1, $vr9"); + asm volatile("vxor.v $vr2, $vr2, $vr10"); + asm volatile("vxor.v $vr3, $vr3, $vr11"); + /* w2$$ = MASK(wq$$); */ + asm volatile("vslti.b $vr12, $vr4, 0"); + asm volatile("vslti.b $vr13, $vr5, 0"); + asm volatile("vslti.b $vr14, $vr6, 0"); + asm volatile("vslti.b $vr15, $vr7, 0"); + /* w1$$ = SHLBYTE(wq$$); */ + asm volatile("vslli.b $vr16, $vr4, 1"); + asm volatile("vslli.b $vr17, $vr5, 1"); + asm volatile("vslli.b $vr18, $vr6, 1"); + asm volatile("vslli.b $vr19, $vr7, 1"); + /* w2$$ &= NBYTES(0x1d); */ + asm volatile("vandi.b $vr12, $vr12, 0x1d"); + asm volatile("vandi.b $vr13, $vr13, 0x1d"); + asm volatile("vandi.b $vr14, $vr14, 0x1d"); + asm volatile("vandi.b $vr15, $vr15, 0x1d"); + /* w1$$ ^= w2$$; */ + asm volatile("vxor.v $vr16, $vr16, $vr12"); + asm volatile("vxor.v $vr17, $vr17, $vr13"); + asm volatile("vxor.v $vr18, $vr18, $vr14"); + asm volatile("vxor.v $vr19, $vr19, $vr15"); + /* wq$$ = w1$$ ^ wd$$; */ + asm volatile("vxor.v $vr4, $vr16, $vr8"); + asm volatile("vxor.v $vr5, $vr17, $vr9"); + asm volatile("vxor.v $vr6, $vr18, $vr10"); + asm volatile("vxor.v $vr7, $vr19, $vr11"); + } + + /* P/Q left side optimization */ + for (z = start-1; z >= 0; z--) { + /* w2$$ = MASK(wq$$); */ + asm volatile("vslti.b $vr12, $vr4, 0"); + asm volatile("vslti.b $vr13, $vr5, 0"); + asm volatile("vslti.b $vr14, $vr6, 0"); + asm volatile("vslti.b $vr15, $vr7, 0"); + /* w1$$ = SHLBYTE(wq$$); */ + asm volatile("vslli.b $vr16, $vr4, 1"); + asm volatile("vslli.b $vr17, $vr5, 1"); + asm volatile("vslli.b $vr18, $vr6, 1"); + asm volatile("vslli.b $vr19, $vr7, 1"); + /* w2$$ &= NBYTES(0x1d); */ + asm volatile("vandi.b $vr12, $vr12, 0x1d"); + asm volatile("vandi.b $vr13, $vr13, 0x1d"); + asm volatile("vandi.b $vr14, $vr14, 0x1d"); + asm volatile("vandi.b $vr15, $vr15, 0x1d"); + /* wq$$ = w1$$ ^ w2$$; */ + asm volatile("vxor.v $vr4, $vr16, $vr12"); + asm volatile("vxor.v $vr5, $vr17, $vr13"); + asm volatile("vxor.v $vr6, $vr18, $vr14"); + asm volatile("vxor.v $vr7, $vr19, $vr15"); + } + /* + * *(unative_t *)&p[d+NSIZE*$$] ^= wp$$; + * *(unative_t *)&q[d+NSIZE*$$] ^= wq$$; + */ + asm volatile( + "vld $vr20, %0\n\t" + "vld $vr21, %1\n\t" + "vld $vr22, %2\n\t" + "vld $vr23, %3\n\t" + "vld $vr24, %4\n\t" + "vld $vr25, %5\n\t" + "vld $vr26, %6\n\t" + "vld $vr27, %7\n\t" + "vxor.v $vr20, $vr20, $vr0\n\t" + "vxor.v $vr21, $vr21, $vr1\n\t" + "vxor.v $vr22, $vr22, $vr2\n\t" + "vxor.v $vr23, $vr23, $vr3\n\t" + "vxor.v $vr24, $vr24, $vr4\n\t" + "vxor.v $vr25, $vr25, $vr5\n\t" + "vxor.v $vr26, $vr26, $vr6\n\t" + "vxor.v $vr27, $vr27, $vr7\n\t" + "vst $vr20, %0\n\t" + "vst $vr21, %1\n\t" + "vst $vr22, %2\n\t" + "vst $vr23, %3\n\t" + "vst $vr24, %4\n\t" + "vst $vr25, %5\n\t" + "vst $vr26, %6\n\t" + "vst $vr27, %7\n\t" + : "+m"(p[d+NSIZE*0]), "+m"(p[d+NSIZE*1]), + "+m"(p[d+NSIZE*2]), "+m"(p[d+NSIZE*3]), + "+m"(q[d+NSIZE*0]), "+m"(q[d+NSIZE*1]), + "+m"(q[d+NSIZE*2]), "+m"(q[d+NSIZE*3]) + ); + } + + kernel_fpu_end(); +} + +const struct raid6_calls raid6_lsx = { + raid6_lsx_gen_syndrome, + raid6_lsx_xor_syndrome, + raid6_has_lsx, + "lsx", + .priority = 0 /* see the comment near the top of the file for reason */ +}; + +#undef NSIZE +#endif /* CONFIG_CPU_HAS_LSX */ + +#ifdef CONFIG_CPU_HAS_LASX +#define NSIZE 32 + +static int raid6_has_lasx(void) +{ + return cpu_has_lasx; +} + +static void raid6_lasx_gen_syndrome(int disks, size_t bytes, void **ptrs) +{ + u8 **dptr = (u8 **)ptrs; + u8 *p, *q; + int d, z, z0; + + z0 = disks - 3; /* Highest data disk */ + p = dptr[z0+1]; /* XOR parity */ + q = dptr[z0+2]; /* RS syndrome */ + + kernel_fpu_begin(); + + /* + * $xr0, $xr1: wp + * $xr2, $xr3: wq + * $xr4, $xr5: wd + * $xr6, $xr7: w2 + * $xr8, $xr9: w1 + */ + for (d = 0; d < bytes; d += NSIZE*2) { + /* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */ + asm volatile("xvld $xr0, %0" : : "m"(dptr[z0][d+0*NSIZE])); + asm volatile("xvld $xr1, %0" : : "m"(dptr[z0][d+1*NSIZE])); + asm volatile("xvori.b $xr2, $xr0, 0"); + asm volatile("xvori.b $xr3, $xr1, 0"); + for (z = z0-1; z >= 0; z--) { + /* wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE]; */ + asm volatile("xvld $xr4, %0" : : "m"(dptr[z][d+0*NSIZE])); + asm volatile("xvld $xr5, %0" : : "m"(dptr[z][d+1*NSIZE])); + /* wp$$ ^= wd$$; */ + asm volatile("xvxor.v $xr0, $xr0, $xr4"); + asm volatile("xvxor.v $xr1, $xr1, $xr5"); + /* w2$$ = MASK(wq$$); */ + asm volatile("xvslti.b $xr6, $xr2, 0"); + asm volatile("xvslti.b $xr7, $xr3, 0"); + /* w1$$ = SHLBYTE(wq$$); */ + asm volatile("xvslli.b $xr8, $xr2, 1"); + asm volatile("xvslli.b $xr9, $xr3, 1"); + /* w2$$ &= NBYTES(0x1d); */ + asm volatile("xvandi.b $xr6, $xr6, 0x1d"); + asm volatile("xvandi.b $xr7, $xr7, 0x1d"); + /* w1$$ ^= w2$$; */ + asm volatile("xvxor.v $xr8, $xr8, $xr6"); + asm volatile("xvxor.v $xr9, $xr9, $xr7"); + /* wq$$ = w1$$ ^ wd$$; */ + asm volatile("xvxor.v $xr2, $xr8, $xr4"); + asm volatile("xvxor.v $xr3, $xr9, $xr5"); + } + /* *(unative_t *)&p[d+NSIZE*$$] = wp$$; */ + asm volatile("xvst $xr0, %0" : "=m"(p[d+NSIZE*0])); + asm volatile("xvst $xr1, %0" : "=m"(p[d+NSIZE*1])); + /* *(unative_t *)&q[d+NSIZE*$$] = wq$$; */ + asm volatile("xvst $xr2, %0" : "=m"(q[d+NSIZE*0])); + asm volatile("xvst $xr3, %0" : "=m"(q[d+NSIZE*1])); + } + + kernel_fpu_end(); +} + +static void raid6_lasx_xor_syndrome(int disks, int start, int stop, + size_t bytes, void **ptrs) +{ + u8 **dptr = (u8 **)ptrs; + u8 *p, *q; + int d, z, z0; + + z0 = stop; /* P/Q right side optimization */ + p = dptr[disks-2]; /* XOR parity */ + q = dptr[disks-1]; /* RS syndrome */ + + kernel_fpu_begin(); + + /* + * $xr0, $xr1: wp + * $xr2, $xr3: wq + * $xr4, $xr5: wd + * $xr6, $xr7: w2 + * $xr8, $xr9: w1 + */ + for (d = 0; d < bytes; d += NSIZE*2) { + /* P/Q data pages */ + /* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */ + asm volatile("xvld $xr0, %0" : : "m"(dptr[z0][d+0*NSIZE])); + asm volatile("xvld $xr1, %0" : : "m"(dptr[z0][d+1*NSIZE])); + asm volatile("xvori.b $xr2, $xr0, 0"); + asm volatile("xvori.b $xr3, $xr1, 0"); + for (z = z0-1; z >= start; z--) { + /* wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE]; */ + asm volatile("xvld $xr4, %0" : : "m"(dptr[z][d+0*NSIZE])); + asm volatile("xvld $xr5, %0" : : "m"(dptr[z][d+1*NSIZE])); + /* wp$$ ^= wd$$; */ + asm volatile("xvxor.v $xr0, $xr0, $xr4"); + asm volatile("xvxor.v $xr1, $xr1, $xr5"); + /* w2$$ = MASK(wq$$); */ + asm volatile("xvslti.b $xr6, $xr2, 0"); + asm volatile("xvslti.b $xr7, $xr3, 0"); + /* w1$$ = SHLBYTE(wq$$); */ + asm volatile("xvslli.b $xr8, $xr2, 1"); + asm volatile("xvslli.b $xr9, $xr3, 1"); + /* w2$$ &= NBYTES(0x1d); */ + asm volatile("xvandi.b $xr6, $xr6, 0x1d"); + asm volatile("xvandi.b $xr7, $xr7, 0x1d"); + /* w1$$ ^= w2$$; */ + asm volatile("xvxor.v $xr8, $xr8, $xr6"); + asm volatile("xvxor.v $xr9, $xr9, $xr7"); + /* wq$$ = w1$$ ^ wd$$; */ + asm volatile("xvxor.v $xr2, $xr8, $xr4"); + asm volatile("xvxor.v $xr3, $xr9, $xr5"); + } + + /* P/Q left side optimization */ + for (z = start-1; z >= 0; z--) { + /* w2$$ = MASK(wq$$); */ + asm volatile("xvslti.b $xr6, $xr2, 0"); + asm volatile("xvslti.b $xr7, $xr3, 0"); + /* w1$$ = SHLBYTE(wq$$); */ + asm volatile("xvslli.b $xr8, $xr2, 1"); + asm volatile("xvslli.b $xr9, $xr3, 1"); + /* w2$$ &= NBYTES(0x1d); */ + asm volatile("xvandi.b $xr6, $xr6, 0x1d"); + asm volatile("xvandi.b $xr7, $xr7, 0x1d"); + /* wq$$ = w1$$ ^ w2$$; */ + asm volatile("xvxor.v $xr2, $xr8, $xr6"); + asm volatile("xvxor.v $xr3, $xr9, $xr7"); + } + /* + * *(unative_t *)&p[d+NSIZE*$$] ^= wp$$; + * *(unative_t *)&q[d+NSIZE*$$] ^= wq$$; + */ + asm volatile( + "xvld $xr10, %0\n\t" + "xvld $xr11, %1\n\t" + "xvld $xr12, %2\n\t" + "xvld $xr13, %3\n\t" + "xvxor.v $xr10, $xr10, $xr0\n\t" + "xvxor.v $xr11, $xr11, $xr1\n\t" + "xvxor.v $xr12, $xr12, $xr2\n\t" + "xvxor.v $xr13, $xr13, $xr3\n\t" + "xvst $xr10, %0\n\t" + "xvst $xr11, %1\n\t" + "xvst $xr12, %2\n\t" + "xvst $xr13, %3\n\t" + : "+m"(p[d+NSIZE*0]), "+m"(p[d+NSIZE*1]), + "+m"(q[d+NSIZE*0]), "+m"(q[d+NSIZE*1]) + ); + } + + kernel_fpu_end(); +} + +const struct raid6_calls raid6_lasx = { + raid6_lasx_gen_syndrome, + raid6_lasx_xor_syndrome, + raid6_has_lasx, + "lasx", + .priority = 0 /* see the comment near the top of the file for reason */ +}; +#undef NSIZE +#endif /* CONFIG_CPU_HAS_LASX */ diff --git a/lib/raid6/mktables.c b/lib/raid6/mktables.c index f02e10fa6238..3be03793237c 100644 --- a/lib/raid6/mktables.c +++ b/lib/raid6/mktables.c @@ -56,7 +56,9 @@ int main(int argc, char *argv[]) uint8_t v; uint8_t exptbl[256], invtbl[256]; + printf("#ifdef __KERNEL__\n"); printf("#include <linux/export.h>\n"); + printf("#endif\n"); printf("#include <linux/raid/pq.h>\n"); /* Compute multiplication table */ diff --git a/lib/raid6/recov.c b/lib/raid6/recov.c index e49d519de6cb..a7c1b2bbe40d 100644 --- a/lib/raid6/recov.c +++ b/lib/raid6/recov.c @@ -13,7 +13,6 @@ * the syndrome.) */ -#include <linux/export.h> #include <linux/raid/pq.h> /* Recover two failed data blocks. */ diff --git a/lib/raid6/recov_loongarch_simd.c b/lib/raid6/recov_loongarch_simd.c new file mode 100644 index 000000000000..94aeac85e6f7 --- /dev/null +++ b/lib/raid6/recov_loongarch_simd.c @@ -0,0 +1,513 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * RAID6 recovery algorithms in LoongArch SIMD (LSX & LASX) + * + * Copyright (C) 2023 WANG Xuerui <git@xen0n.name> + * + * Originally based on recov_avx2.c and recov_ssse3.c: + * + * Copyright (C) 2012 Intel Corporation + * Author: Jim Kukunas <james.t.kukunas@linux.intel.com> + */ + +#include <linux/raid/pq.h> +#include "loongarch.h" + +/* + * Unlike with the syndrome calculation algorithms, there's no boot-time + * selection of recovery algorithms by benchmarking, so we have to specify + * the priorities and hope the future cores will all have decent vector + * support (i.e. no LASX slower than LSX, or even scalar code). + */ + +#ifdef CONFIG_CPU_HAS_LSX +static int raid6_has_lsx(void) +{ + return cpu_has_lsx; +} + +static void raid6_2data_recov_lsx(int disks, size_t bytes, int faila, + int failb, void **ptrs) +{ + u8 *p, *q, *dp, *dq; + const u8 *pbmul; /* P multiplier table for B data */ + const u8 *qmul; /* Q multiplier table (for both) */ + + p = (u8 *)ptrs[disks - 2]; + q = (u8 *)ptrs[disks - 1]; + + /* + * Compute syndrome with zero for the missing data pages + * Use the dead data pages as temporary storage for + * delta p and delta q + */ + dp = (u8 *)ptrs[faila]; + ptrs[faila] = (void *)raid6_empty_zero_page; + ptrs[disks - 2] = dp; + dq = (u8 *)ptrs[failb]; + ptrs[failb] = (void *)raid6_empty_zero_page; + ptrs[disks - 1] = dq; + + raid6_call.gen_syndrome(disks, bytes, ptrs); + + /* Restore pointer table */ + ptrs[faila] = dp; + ptrs[failb] = dq; + ptrs[disks - 2] = p; + ptrs[disks - 1] = q; + + /* Now, pick the proper data tables */ + pbmul = raid6_vgfmul[raid6_gfexi[failb - faila]]; + qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^ raid6_gfexp[failb]]]; + + kernel_fpu_begin(); + + /* + * vr20, vr21: qmul + * vr22, vr23: pbmul + */ + asm volatile("vld $vr20, %0" : : "m" (qmul[0])); + asm volatile("vld $vr21, %0" : : "m" (qmul[16])); + asm volatile("vld $vr22, %0" : : "m" (pbmul[0])); + asm volatile("vld $vr23, %0" : : "m" (pbmul[16])); + + while (bytes) { + /* vr4 - vr7: Q */ + asm volatile("vld $vr4, %0" : : "m" (q[0])); + asm volatile("vld $vr5, %0" : : "m" (q[16])); + asm volatile("vld $vr6, %0" : : "m" (q[32])); + asm volatile("vld $vr7, %0" : : "m" (q[48])); + /* vr4 - vr7: Q + Qxy */ + asm volatile("vld $vr8, %0" : : "m" (dq[0])); + asm volatile("vld $vr9, %0" : : "m" (dq[16])); + asm volatile("vld $vr10, %0" : : "m" (dq[32])); + asm volatile("vld $vr11, %0" : : "m" (dq[48])); + asm volatile("vxor.v $vr4, $vr4, $vr8"); + asm volatile("vxor.v $vr5, $vr5, $vr9"); + asm volatile("vxor.v $vr6, $vr6, $vr10"); + asm volatile("vxor.v $vr7, $vr7, $vr11"); + /* vr0 - vr3: P */ + asm volatile("vld $vr0, %0" : : "m" (p[0])); + asm volatile("vld $vr1, %0" : : "m" (p[16])); + asm volatile("vld $vr2, %0" : : "m" (p[32])); + asm volatile("vld $vr3, %0" : : "m" (p[48])); + /* vr0 - vr3: P + Pxy */ + asm volatile("vld $vr8, %0" : : "m" (dp[0])); + asm volatile("vld $vr9, %0" : : "m" (dp[16])); + asm volatile("vld $vr10, %0" : : "m" (dp[32])); + asm volatile("vld $vr11, %0" : : "m" (dp[48])); + asm volatile("vxor.v $vr0, $vr0, $vr8"); + asm volatile("vxor.v $vr1, $vr1, $vr9"); + asm volatile("vxor.v $vr2, $vr2, $vr10"); + asm volatile("vxor.v $vr3, $vr3, $vr11"); + + /* vr8 - vr11: higher 4 bits of each byte of (Q + Qxy) */ + asm volatile("vsrli.b $vr8, $vr4, 4"); + asm volatile("vsrli.b $vr9, $vr5, 4"); + asm volatile("vsrli.b $vr10, $vr6, 4"); + asm volatile("vsrli.b $vr11, $vr7, 4"); + /* vr4 - vr7: lower 4 bits of each byte of (Q + Qxy) */ + asm volatile("vandi.b $vr4, $vr4, 0x0f"); + asm volatile("vandi.b $vr5, $vr5, 0x0f"); + asm volatile("vandi.b $vr6, $vr6, 0x0f"); + asm volatile("vandi.b $vr7, $vr7, 0x0f"); + /* lookup from qmul[0] */ + asm volatile("vshuf.b $vr4, $vr20, $vr20, $vr4"); + asm volatile("vshuf.b $vr5, $vr20, $vr20, $vr5"); + asm volatile("vshuf.b $vr6, $vr20, $vr20, $vr6"); + asm volatile("vshuf.b $vr7, $vr20, $vr20, $vr7"); + /* lookup from qmul[16] */ + asm volatile("vshuf.b $vr8, $vr21, $vr21, $vr8"); + asm volatile("vshuf.b $vr9, $vr21, $vr21, $vr9"); + asm volatile("vshuf.b $vr10, $vr21, $vr21, $vr10"); + asm volatile("vshuf.b $vr11, $vr21, $vr21, $vr11"); + /* vr16 - vr19: B(Q + Qxy) */ + asm volatile("vxor.v $vr16, $vr8, $vr4"); + asm volatile("vxor.v $vr17, $vr9, $vr5"); + asm volatile("vxor.v $vr18, $vr10, $vr6"); + asm volatile("vxor.v $vr19, $vr11, $vr7"); + + /* vr4 - vr7: higher 4 bits of each byte of (P + Pxy) */ + asm volatile("vsrli.b $vr4, $vr0, 4"); + asm volatile("vsrli.b $vr5, $vr1, 4"); + asm volatile("vsrli.b $vr6, $vr2, 4"); + asm volatile("vsrli.b $vr7, $vr3, 4"); + /* vr12 - vr15: lower 4 bits of each byte of (P + Pxy) */ + asm volatile("vandi.b $vr12, $vr0, 0x0f"); + asm volatile("vandi.b $vr13, $vr1, 0x0f"); + asm volatile("vandi.b $vr14, $vr2, 0x0f"); + asm volatile("vandi.b $vr15, $vr3, 0x0f"); + /* lookup from pbmul[0] */ + asm volatile("vshuf.b $vr12, $vr22, $vr22, $vr12"); + asm volatile("vshuf.b $vr13, $vr22, $vr22, $vr13"); + asm volatile("vshuf.b $vr14, $vr22, $vr22, $vr14"); + asm volatile("vshuf.b $vr15, $vr22, $vr22, $vr15"); + /* lookup from pbmul[16] */ + asm volatile("vshuf.b $vr4, $vr23, $vr23, $vr4"); + asm volatile("vshuf.b $vr5, $vr23, $vr23, $vr5"); + asm volatile("vshuf.b $vr6, $vr23, $vr23, $vr6"); + asm volatile("vshuf.b $vr7, $vr23, $vr23, $vr7"); + /* vr4 - vr7: A(P + Pxy) */ + asm volatile("vxor.v $vr4, $vr4, $vr12"); + asm volatile("vxor.v $vr5, $vr5, $vr13"); + asm volatile("vxor.v $vr6, $vr6, $vr14"); + asm volatile("vxor.v $vr7, $vr7, $vr15"); + + /* vr4 - vr7: A(P + Pxy) + B(Q + Qxy) = Dx */ + asm volatile("vxor.v $vr4, $vr4, $vr16"); + asm volatile("vxor.v $vr5, $vr5, $vr17"); + asm volatile("vxor.v $vr6, $vr6, $vr18"); + asm volatile("vxor.v $vr7, $vr7, $vr19"); + asm volatile("vst $vr4, %0" : "=m" (dq[0])); + asm volatile("vst $vr5, %0" : "=m" (dq[16])); + asm volatile("vst $vr6, %0" : "=m" (dq[32])); + asm volatile("vst $vr7, %0" : "=m" (dq[48])); + + /* vr0 - vr3: P + Pxy + Dx = Dy */ + asm volatile("vxor.v $vr0, $vr0, $vr4"); + asm volatile("vxor.v $vr1, $vr1, $vr5"); + asm volatile("vxor.v $vr2, $vr2, $vr6"); + asm volatile("vxor.v $vr3, $vr3, $vr7"); + asm volatile("vst $vr0, %0" : "=m" (dp[0])); + asm volatile("vst $vr1, %0" : "=m" (dp[16])); + asm volatile("vst $vr2, %0" : "=m" (dp[32])); + asm volatile("vst $vr3, %0" : "=m" (dp[48])); + + bytes -= 64; + p += 64; + q += 64; + dp += 64; + dq += 64; + } + + kernel_fpu_end(); +} + +static void raid6_datap_recov_lsx(int disks, size_t bytes, int faila, + void **ptrs) +{ + u8 *p, *q, *dq; + const u8 *qmul; /* Q multiplier table */ + + p = (u8 *)ptrs[disks - 2]; + q = (u8 *)ptrs[disks - 1]; + + /* + * Compute syndrome with zero for the missing data page + * Use the dead data page as temporary storage for delta q + */ + dq = (u8 *)ptrs[faila]; + ptrs[faila] = (void *)raid6_empty_zero_page; + ptrs[disks - 1] = dq; + + raid6_call.gen_syndrome(disks, bytes, ptrs); + + /* Restore pointer table */ + ptrs[faila] = dq; + ptrs[disks - 1] = q; + + /* Now, pick the proper data tables */ + qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]]; + + kernel_fpu_begin(); + + /* vr22, vr23: qmul */ + asm volatile("vld $vr22, %0" : : "m" (qmul[0])); + asm volatile("vld $vr23, %0" : : "m" (qmul[16])); + + while (bytes) { + /* vr0 - vr3: P + Dx */ + asm volatile("vld $vr0, %0" : : "m" (p[0])); + asm volatile("vld $vr1, %0" : : "m" (p[16])); + asm volatile("vld $vr2, %0" : : "m" (p[32])); + asm volatile("vld $vr3, %0" : : "m" (p[48])); + /* vr4 - vr7: Qx */ + asm volatile("vld $vr4, %0" : : "m" (dq[0])); + asm volatile("vld $vr5, %0" : : "m" (dq[16])); + asm volatile("vld $vr6, %0" : : "m" (dq[32])); + asm volatile("vld $vr7, %0" : : "m" (dq[48])); + /* vr4 - vr7: Q + Qx */ + asm volatile("vld $vr8, %0" : : "m" (q[0])); + asm volatile("vld $vr9, %0" : : "m" (q[16])); + asm volatile("vld $vr10, %0" : : "m" (q[32])); + asm volatile("vld $vr11, %0" : : "m" (q[48])); + asm volatile("vxor.v $vr4, $vr4, $vr8"); + asm volatile("vxor.v $vr5, $vr5, $vr9"); + asm volatile("vxor.v $vr6, $vr6, $vr10"); + asm volatile("vxor.v $vr7, $vr7, $vr11"); + + /* vr8 - vr11: higher 4 bits of each byte of (Q + Qx) */ + asm volatile("vsrli.b $vr8, $vr4, 4"); + asm volatile("vsrli.b $vr9, $vr5, 4"); + asm volatile("vsrli.b $vr10, $vr6, 4"); + asm volatile("vsrli.b $vr11, $vr7, 4"); + /* vr4 - vr7: lower 4 bits of each byte of (Q + Qx) */ + asm volatile("vandi.b $vr4, $vr4, 0x0f"); + asm volatile("vandi.b $vr5, $vr5, 0x0f"); + asm volatile("vandi.b $vr6, $vr6, 0x0f"); + asm volatile("vandi.b $vr7, $vr7, 0x0f"); + /* lookup from qmul[0] */ + asm volatile("vshuf.b $vr4, $vr22, $vr22, $vr4"); + asm volatile("vshuf.b $vr5, $vr22, $vr22, $vr5"); + asm volatile("vshuf.b $vr6, $vr22, $vr22, $vr6"); + asm volatile("vshuf.b $vr7, $vr22, $vr22, $vr7"); + /* lookup from qmul[16] */ + asm volatile("vshuf.b $vr8, $vr23, $vr23, $vr8"); + asm volatile("vshuf.b $vr9, $vr23, $vr23, $vr9"); + asm volatile("vshuf.b $vr10, $vr23, $vr23, $vr10"); + asm volatile("vshuf.b $vr11, $vr23, $vr23, $vr11"); + /* vr4 - vr7: qmul(Q + Qx) = Dx */ + asm volatile("vxor.v $vr4, $vr4, $vr8"); + asm volatile("vxor.v $vr5, $vr5, $vr9"); + asm volatile("vxor.v $vr6, $vr6, $vr10"); + asm volatile("vxor.v $vr7, $vr7, $vr11"); + asm volatile("vst $vr4, %0" : "=m" (dq[0])); + asm volatile("vst $vr5, %0" : "=m" (dq[16])); + asm volatile("vst $vr6, %0" : "=m" (dq[32])); + asm volatile("vst $vr7, %0" : "=m" (dq[48])); + + /* vr0 - vr3: P + Dx + Dx = P */ + asm volatile("vxor.v $vr0, $vr0, $vr4"); + asm volatile("vxor.v $vr1, $vr1, $vr5"); + asm volatile("vxor.v $vr2, $vr2, $vr6"); + asm volatile("vxor.v $vr3, $vr3, $vr7"); + asm volatile("vst $vr0, %0" : "=m" (p[0])); + asm volatile("vst $vr1, %0" : "=m" (p[16])); + asm volatile("vst $vr2, %0" : "=m" (p[32])); + asm volatile("vst $vr3, %0" : "=m" (p[48])); + + bytes -= 64; + p += 64; + q += 64; + dq += 64; + } + + kernel_fpu_end(); +} + +const struct raid6_recov_calls raid6_recov_lsx = { + .data2 = raid6_2data_recov_lsx, + .datap = raid6_datap_recov_lsx, + .valid = raid6_has_lsx, + .name = "lsx", + .priority = 1, +}; +#endif /* CONFIG_CPU_HAS_LSX */ + +#ifdef CONFIG_CPU_HAS_LASX +static int raid6_has_lasx(void) +{ + return cpu_has_lasx; +} + +static void raid6_2data_recov_lasx(int disks, size_t bytes, int faila, + int failb, void **ptrs) +{ + u8 *p, *q, *dp, *dq; + const u8 *pbmul; /* P multiplier table for B data */ + const u8 *qmul; /* Q multiplier table (for both) */ + + p = (u8 *)ptrs[disks - 2]; + q = (u8 *)ptrs[disks - 1]; + + /* + * Compute syndrome with zero for the missing data pages + * Use the dead data pages as temporary storage for + * delta p and delta q + */ + dp = (u8 *)ptrs[faila]; + ptrs[faila] = (void *)raid6_empty_zero_page; + ptrs[disks - 2] = dp; + dq = (u8 *)ptrs[failb]; + ptrs[failb] = (void *)raid6_empty_zero_page; + ptrs[disks - 1] = dq; + + raid6_call.gen_syndrome(disks, bytes, ptrs); + + /* Restore pointer table */ + ptrs[faila] = dp; + ptrs[failb] = dq; + ptrs[disks - 2] = p; + ptrs[disks - 1] = q; + + /* Now, pick the proper data tables */ + pbmul = raid6_vgfmul[raid6_gfexi[failb - faila]]; + qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^ raid6_gfexp[failb]]]; + + kernel_fpu_begin(); + + /* + * xr20, xr21: qmul + * xr22, xr23: pbmul + */ + asm volatile("vld $vr20, %0" : : "m" (qmul[0])); + asm volatile("vld $vr21, %0" : : "m" (qmul[16])); + asm volatile("vld $vr22, %0" : : "m" (pbmul[0])); + asm volatile("vld $vr23, %0" : : "m" (pbmul[16])); + asm volatile("xvreplve0.q $xr20, $xr20"); + asm volatile("xvreplve0.q $xr21, $xr21"); + asm volatile("xvreplve0.q $xr22, $xr22"); + asm volatile("xvreplve0.q $xr23, $xr23"); + + while (bytes) { + /* xr0, xr1: Q */ + asm volatile("xvld $xr0, %0" : : "m" (q[0])); + asm volatile("xvld $xr1, %0" : : "m" (q[32])); + /* xr0, xr1: Q + Qxy */ + asm volatile("xvld $xr4, %0" : : "m" (dq[0])); + asm volatile("xvld $xr5, %0" : : "m" (dq[32])); + asm volatile("xvxor.v $xr0, $xr0, $xr4"); + asm volatile("xvxor.v $xr1, $xr1, $xr5"); + /* xr2, xr3: P */ + asm volatile("xvld $xr2, %0" : : "m" (p[0])); + asm volatile("xvld $xr3, %0" : : "m" (p[32])); + /* xr2, xr3: P + Pxy */ + asm volatile("xvld $xr4, %0" : : "m" (dp[0])); + asm volatile("xvld $xr5, %0" : : "m" (dp[32])); + asm volatile("xvxor.v $xr2, $xr2, $xr4"); + asm volatile("xvxor.v $xr3, $xr3, $xr5"); + + /* xr4, xr5: higher 4 bits of each byte of (Q + Qxy) */ + asm volatile("xvsrli.b $xr4, $xr0, 4"); + asm volatile("xvsrli.b $xr5, $xr1, 4"); + /* xr0, xr1: lower 4 bits of each byte of (Q + Qxy) */ + asm volatile("xvandi.b $xr0, $xr0, 0x0f"); + asm volatile("xvandi.b $xr1, $xr1, 0x0f"); + /* lookup from qmul[0] */ + asm volatile("xvshuf.b $xr0, $xr20, $xr20, $xr0"); + asm volatile("xvshuf.b $xr1, $xr20, $xr20, $xr1"); + /* lookup from qmul[16] */ + asm volatile("xvshuf.b $xr4, $xr21, $xr21, $xr4"); + asm volatile("xvshuf.b $xr5, $xr21, $xr21, $xr5"); + /* xr6, xr7: B(Q + Qxy) */ + asm volatile("xvxor.v $xr6, $xr4, $xr0"); + asm volatile("xvxor.v $xr7, $xr5, $xr1"); + + /* xr4, xr5: higher 4 bits of each byte of (P + Pxy) */ + asm volatile("xvsrli.b $xr4, $xr2, 4"); + asm volatile("xvsrli.b $xr5, $xr3, 4"); + /* xr0, xr1: lower 4 bits of each byte of (P + Pxy) */ + asm volatile("xvandi.b $xr0, $xr2, 0x0f"); + asm volatile("xvandi.b $xr1, $xr3, 0x0f"); + /* lookup from pbmul[0] */ + asm volatile("xvshuf.b $xr0, $xr22, $xr22, $xr0"); + asm volatile("xvshuf.b $xr1, $xr22, $xr22, $xr1"); + /* lookup from pbmul[16] */ + asm volatile("xvshuf.b $xr4, $xr23, $xr23, $xr4"); + asm volatile("xvshuf.b $xr5, $xr23, $xr23, $xr5"); + /* xr0, xr1: A(P + Pxy) */ + asm volatile("xvxor.v $xr0, $xr0, $xr4"); + asm volatile("xvxor.v $xr1, $xr1, $xr5"); + + /* xr0, xr1: A(P + Pxy) + B(Q + Qxy) = Dx */ + asm volatile("xvxor.v $xr0, $xr0, $xr6"); + asm volatile("xvxor.v $xr1, $xr1, $xr7"); + + /* xr2, xr3: P + Pxy + Dx = Dy */ + asm volatile("xvxor.v $xr2, $xr2, $xr0"); + asm volatile("xvxor.v $xr3, $xr3, $xr1"); + + asm volatile("xvst $xr0, %0" : "=m" (dq[0])); + asm volatile("xvst $xr1, %0" : "=m" (dq[32])); + asm volatile("xvst $xr2, %0" : "=m" (dp[0])); + asm volatile("xvst $xr3, %0" : "=m" (dp[32])); + + bytes -= 64; + p += 64; + q += 64; + dp += 64; + dq += 64; + } + + kernel_fpu_end(); +} + +static void raid6_datap_recov_lasx(int disks, size_t bytes, int faila, + void **ptrs) +{ + u8 *p, *q, *dq; + const u8 *qmul; /* Q multiplier table */ + + p = (u8 *)ptrs[disks - 2]; + q = (u8 *)ptrs[disks - 1]; + + /* + * Compute syndrome with zero for the missing data page + * Use the dead data page as temporary storage for delta q + */ + dq = (u8 *)ptrs[faila]; + ptrs[faila] = (void *)raid6_empty_zero_page; + ptrs[disks - 1] = dq; + + raid6_call.gen_syndrome(disks, bytes, ptrs); + + /* Restore pointer table */ + ptrs[faila] = dq; + ptrs[disks - 1] = q; + + /* Now, pick the proper data tables */ + qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]]; + + kernel_fpu_begin(); + + /* xr22, xr23: qmul */ + asm volatile("vld $vr22, %0" : : "m" (qmul[0])); + asm volatile("xvreplve0.q $xr22, $xr22"); + asm volatile("vld $vr23, %0" : : "m" (qmul[16])); + asm volatile("xvreplve0.q $xr23, $xr23"); + + while (bytes) { + /* xr0, xr1: P + Dx */ + asm volatile("xvld $xr0, %0" : : "m" (p[0])); + asm volatile("xvld $xr1, %0" : : "m" (p[32])); + /* xr2, xr3: Qx */ + asm volatile("xvld $xr2, %0" : : "m" (dq[0])); + asm volatile("xvld $xr3, %0" : : "m" (dq[32])); + /* xr2, xr3: Q + Qx */ + asm volatile("xvld $xr4, %0" : : "m" (q[0])); + asm volatile("xvld $xr5, %0" : : "m" (q[32])); + asm volatile("xvxor.v $xr2, $xr2, $xr4"); + asm volatile("xvxor.v $xr3, $xr3, $xr5"); + + /* xr4, xr5: higher 4 bits of each byte of (Q + Qx) */ + asm volatile("xvsrli.b $xr4, $xr2, 4"); + asm volatile("xvsrli.b $xr5, $xr3, 4"); + /* xr2, xr3: lower 4 bits of each byte of (Q + Qx) */ + asm volatile("xvandi.b $xr2, $xr2, 0x0f"); + asm volatile("xvandi.b $xr3, $xr3, 0x0f"); + /* lookup from qmul[0] */ + asm volatile("xvshuf.b $xr2, $xr22, $xr22, $xr2"); + asm volatile("xvshuf.b $xr3, $xr22, $xr22, $xr3"); + /* lookup from qmul[16] */ + asm volatile("xvshuf.b $xr4, $xr23, $xr23, $xr4"); + asm volatile("xvshuf.b $xr5, $xr23, $xr23, $xr5"); + /* xr2, xr3: qmul(Q + Qx) = Dx */ + asm volatile("xvxor.v $xr2, $xr2, $xr4"); + asm volatile("xvxor.v $xr3, $xr3, $xr5"); + + /* xr0, xr1: P + Dx + Dx = P */ + asm volatile("xvxor.v $xr0, $xr0, $xr2"); + asm volatile("xvxor.v $xr1, $xr1, $xr3"); + + asm volatile("xvst $xr2, %0" : "=m" (dq[0])); + asm volatile("xvst $xr3, %0" : "=m" (dq[32])); + asm volatile("xvst $xr0, %0" : "=m" (p[0])); + asm volatile("xvst $xr1, %0" : "=m" (p[32])); + + bytes -= 64; + p += 64; + q += 64; + dq += 64; + } + + kernel_fpu_end(); +} + +const struct raid6_recov_calls raid6_recov_lasx = { + .data2 = raid6_2data_recov_lasx, + .datap = raid6_datap_recov_lasx, + .valid = raid6_has_lasx, + .name = "lasx", + .priority = 2, +}; +#endif /* CONFIG_CPU_HAS_LASX */ diff --git a/lib/raid6/test/.gitignore b/lib/raid6/test/.gitignore new file mode 100644 index 000000000000..1b68a77f348f --- /dev/null +++ b/lib/raid6/test/.gitignore @@ -0,0 +1,3 @@ +/int.uc +/neon.uc +/raid6test diff --git a/lib/raid6/test/Makefile b/lib/raid6/test/Makefile index 4fb7700a741b..2abe0076a636 100644 --- a/lib/raid6/test/Makefile +++ b/lib/raid6/test/Makefile @@ -6,14 +6,15 @@ pound := \# -CC = gcc -OPTFLAGS = -O2 # Adjust as desired -CFLAGS = -I.. -I ../../../include -g $(OPTFLAGS) -LD = ld -AWK = awk -f -AR = ar -RANLIB = ranlib -OBJS = int1.o int2.o int4.o int8.o int16.o int32.o recov.o algos.o tables.o +# Adjust as desired +CC = gcc +OPTFLAGS = -O2 +CFLAGS = -I.. -I ../../../include -g $(OPTFLAGS) +LD = ld +AWK = awk -f +AR = ar +RANLIB = ranlib +OBJS = int1.o int2.o int4.o int8.o int16.o int32.o recov.o algos.o tables.o ARCH := $(shell uname -m 2>/dev/null | sed -e /s/i.86/i386/) ifeq ($(ARCH),i386) @@ -34,24 +35,37 @@ ifeq ($(ARCH),aarch64) HAS_NEON = yes endif +ifeq ($(findstring ppc,$(ARCH)),ppc) + CFLAGS += -I../../../arch/powerpc/include + HAS_ALTIVEC := $(shell printf '$(pound)include <altivec.h>\nvector int a;\n' |\ + gcc -c -x c - >/dev/null && rm ./-.o && echo yes) +endif + +ifeq ($(ARCH),loongarch64) + CFLAGS += -I../../../arch/loongarch/include -DCONFIG_LOONGARCH=1 + CFLAGS += $(shell echo 'vld $$vr0, $$zero, 0' | \ + gcc -c -x assembler - >/dev/null 2>&1 && \ + rm ./-.o && echo -DCONFIG_CPU_HAS_LSX=1) + CFLAGS += $(shell echo 'xvld $$xr0, $$zero, 0' | \ + gcc -c -x assembler - >/dev/null 2>&1 && \ + rm ./-.o && echo -DCONFIG_CPU_HAS_LASX=1) +endif + ifeq ($(IS_X86),yes) OBJS += mmx.o sse1.o sse2.o avx2.o recov_ssse3.o recov_avx2.o avx512.o recov_avx512.o CFLAGS += -DCONFIG_X86 - CFLAGS += $(shell echo "vpmovm2b %k1, %zmm5" | \ - gcc -c -x assembler - >/dev/null 2>&1 && \ - rm ./-.o && echo -DCONFIG_AS_AVX512=1) + CFLAGS += $(shell echo "vpmovm2b %k1, %zmm5" | \ + gcc -c -x assembler - >/dev/null 2>&1 && \ + rm ./-.o && echo -DCONFIG_AS_AVX512=1) else ifeq ($(HAS_NEON),yes) OBJS += neon.o neon1.o neon2.o neon4.o neon8.o recov_neon.o recov_neon_inner.o CFLAGS += -DCONFIG_KERNEL_MODE_NEON=1 -else - HAS_ALTIVEC := $(shell printf '$(pound)include <altivec.h>\nvector int a;\n' |\ - gcc -c -x c - >/dev/null && rm ./-.o && echo yes) - ifeq ($(HAS_ALTIVEC),yes) - CFLAGS += -I../../../arch/powerpc/include - CFLAGS += -DCONFIG_ALTIVEC - OBJS += altivec1.o altivec2.o altivec4.o altivec8.o \ - vpermxor1.o vpermxor2.o vpermxor4.o vpermxor8.o - endif +else ifeq ($(HAS_ALTIVEC),yes) + CFLAGS += -DCONFIG_ALTIVEC + OBJS += altivec1.o altivec2.o altivec4.o altivec8.o \ + vpermxor1.o vpermxor2.o vpermxor4.o vpermxor8.o +else ifeq ($(ARCH),loongarch64) + OBJS += loongarch_simd.o recov_loongarch_simd.o endif .c.o: @@ -63,12 +77,12 @@ endif %.uc: ../%.uc cp -f $< $@ -all: raid6.a raid6test +all: raid6.a raid6test raid6.a: $(OBJS) - rm -f $@ - $(AR) cq $@ $^ - $(RANLIB) $@ + rm -f $@ + $(AR) cq $@ $^ + $(RANLIB) $@ raid6test: test.c raid6.a $(CC) $(CFLAGS) -o raid6test $^ diff --git a/lib/rcuref.c b/lib/rcuref.c index 5ec00a4a64d1..97f300eca927 100644 --- a/lib/rcuref.c +++ b/lib/rcuref.c @@ -248,7 +248,7 @@ bool rcuref_put_slowpath(rcuref_t *ref) * require a retry. If this fails the caller is not * allowed to deconstruct the object. */ - if (atomic_cmpxchg_release(&ref->refcnt, RCUREF_NOREF, RCUREF_DEAD) != RCUREF_NOREF) + if (!atomic_try_cmpxchg_release(&ref->refcnt, &cnt, RCUREF_DEAD)) return false; /* diff --git a/lib/scatterlist.c b/lib/scatterlist.c index c65566b4dc66..68b45c82c37a 100644 --- a/lib/scatterlist.c +++ b/lib/scatterlist.c @@ -265,7 +265,8 @@ EXPORT_SYMBOL(sg_free_table); * @table: The sg table header to use * @nents: Number of entries in sg list * @max_ents: The maximum number of entries the allocator returns per call - * @nents_first_chunk: Number of entries int the (preallocated) first + * @first_chunk: first SGL if preallocated (may be %NULL) + * @nents_first_chunk: Number of entries in the (preallocated) first * scatterlist chunk, 0 means no such preallocated chunk provided by user * @gfp_mask: GFP allocation mask * @alloc_fn: Allocator to use @@ -788,6 +789,7 @@ EXPORT_SYMBOL(__sg_page_iter_dma_next); * @miter: sg mapping iter to be started * @sgl: sg list to iterate over * @nents: number of sg entries + * @flags: sg iterator flags * * Description: * Starts mapping iterator @miter. diff --git a/lib/seq_buf.c b/lib/seq_buf.c index 45c450f423fa..010c730ca7fc 100644 --- a/lib/seq_buf.c +++ b/lib/seq_buf.c @@ -109,9 +109,7 @@ void seq_buf_do_printk(struct seq_buf *s, const char *lvl) if (s->size == 0 || s->len == 0) return; - seq_buf_terminate(s); - - start = s->buffer; + start = seq_buf_str(s); while ((lf = strchr(start, '\n'))) { int len = lf - start + 1; @@ -189,6 +187,7 @@ int seq_buf_puts(struct seq_buf *s, const char *str) seq_buf_set_overflow(s); return -1; } +EXPORT_SYMBOL_GPL(seq_buf_puts); /** * seq_buf_putc - sequence printing of simple character @@ -210,6 +209,7 @@ int seq_buf_putc(struct seq_buf *s, unsigned char c) seq_buf_set_overflow(s); return -1; } +EXPORT_SYMBOL_GPL(seq_buf_putc); /** * seq_buf_putmem - write raw data into the sequenc buffer @@ -324,23 +324,24 @@ int seq_buf_path(struct seq_buf *s, const struct path *path, const char *esc) * seq_buf_to_user - copy the sequence buffer to user space * @s: seq_buf descriptor * @ubuf: The userspace memory location to copy to + * @start: The first byte in the buffer to copy * @cnt: The amount to copy * * Copies the sequence buffer into the userspace memory pointed to - * by @ubuf. It starts from the last read position (@s->readpos) - * and writes up to @cnt characters or till it reaches the end of - * the content in the buffer (@s->len), which ever comes first. + * by @ubuf. It starts from @start and writes up to @cnt characters + * or until it reaches the end of the content in the buffer (@s->len), + * whichever comes first. * * On success, it returns a positive number of the number of bytes * it copied. * * On failure it returns -EBUSY if all of the content in the * sequence has been already read, which includes nothing in the - * sequence (@s->len == @s->readpos). + * sequence (@s->len == @start). * * Returns -EFAULT if the copy to userspace fails. */ -int seq_buf_to_user(struct seq_buf *s, char __user *ubuf, int cnt) +int seq_buf_to_user(struct seq_buf *s, char __user *ubuf, size_t start, int cnt) { int len; int ret; @@ -350,20 +351,17 @@ int seq_buf_to_user(struct seq_buf *s, char __user *ubuf, int cnt) len = seq_buf_used(s); - if (len <= s->readpos) + if (len <= start) return -EBUSY; - len -= s->readpos; + len -= start; if (cnt > len) cnt = len; - ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt); + ret = copy_to_user(ubuf, s->buffer + start, cnt); if (ret == cnt) return -EFAULT; - cnt -= ret; - - s->readpos += cnt; - return cnt; + return cnt - ret; } /** diff --git a/lib/string_helpers.c b/lib/string_helpers.c index d3b1dd718daf..7713f73e66b0 100644 --- a/lib/string_helpers.c +++ b/lib/string_helpers.c @@ -31,9 +31,11 @@ * giving the size in the required units. @buf should have room for * at least 9 bytes and will always be zero terminated. * + * Return value: number of characters of output that would have been written + * (which may be greater than len, if output was truncated). */ -void string_get_size(u64 size, u64 blk_size, const enum string_size_units units, - char *buf, int len) +int string_get_size(u64 size, u64 blk_size, const enum string_size_units units, + char *buf, int len) { static const char *const units_10[] = { "B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB" @@ -126,8 +128,8 @@ void string_get_size(u64 size, u64 blk_size, const enum string_size_units units, else unit = units_str[units][i]; - snprintf(buf, len, "%u%s %s", (u32)size, - tmp, unit); + return snprintf(buf, len, "%u%s %s", (u32)size, + tmp, unit); } EXPORT_SYMBOL(string_get_size); @@ -719,6 +721,21 @@ char *kstrdup_quotable_file(struct file *file, gfp_t gfp) } EXPORT_SYMBOL_GPL(kstrdup_quotable_file); +/* + * Returns duplicate string in which the @old characters are replaced by @new. + */ +char *kstrdup_and_replace(const char *src, char old, char new, gfp_t gfp) +{ + char *dst; + + dst = kstrdup(src, gfp); + if (!dst) + return NULL; + + return strreplace(dst, old, new); +} +EXPORT_SYMBOL_GPL(kstrdup_and_replace); + /** * kasprintf_strarray - allocate and fill array of sequential strings * @gfp: flags for the slab allocator diff --git a/lib/test_bitmap.c b/lib/test_bitmap.c index f2ea9f30c7c5..65f22c2578b0 100644 --- a/lib/test_bitmap.c +++ b/lib/test_bitmap.c @@ -330,6 +330,29 @@ static void __init test_copy(void) expect_eq_pbl("0-108,128-1023", bmap2, 1024); } +static void __init test_bitmap_region(void) +{ + int pos, order; + + DECLARE_BITMAP(bmap, 1000); + + bitmap_zero(bmap, 1000); + + for (order = 0; order < 10; order++) { + pos = bitmap_find_free_region(bmap, 1000, order); + if (order == 0) + expect_eq_uint(pos, 0); + else + expect_eq_uint(pos, order < 9 ? BIT(order) : -ENOMEM); + } + + bitmap_release_region(bmap, 0, 0); + for (order = 1; order < 9; order++) + bitmap_release_region(bmap, BIT(order), order); + + expect_eq_uint(bitmap_weight(bmap, 1000), 0); +} + #define EXP2_IN_BITS (sizeof(exp2) * 8) static void __init test_replace(void) @@ -1227,6 +1250,7 @@ static void __init selftest(void) test_zero_clear(); test_fill_set(); test_copy(); + test_bitmap_region(); test_replace(); test_bitmap_arr32(); test_bitmap_arr64(); diff --git a/lib/test_bpf.c b/lib/test_bpf.c index fa0833410ac1..7916503e6a6a 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -596,8 +596,8 @@ static int __bpf_fill_alu_shift(struct bpf_test *self, u8 op, { static const s64 regs[] = { 0x0123456789abcdefLL, /* dword > 0, word < 0 */ - 0xfedcba9876543210LL, /* dowrd < 0, word > 0 */ - 0xfedcba0198765432LL, /* dowrd < 0, word < 0 */ + 0xfedcba9876543210LL, /* dword < 0, word > 0 */ + 0xfedcba0198765432LL, /* dword < 0, word < 0 */ 0x0123458967abcdefLL, /* dword > 0, word > 0 */ }; int bits = alu32 ? 32 : 64; @@ -5111,6 +5111,104 @@ static struct bpf_test tests[] = { { }, { { 0, 0xffffffff } } }, + /* MOVSX32 */ + { + "ALU_MOVSX | BPF_B", + .u.insns_int = { + BPF_LD_IMM64(R2, 0x00000000ffffffefLL), + BPF_LD_IMM64(R3, 0xdeadbeefdeadbeefLL), + BPF_MOVSX32_REG(R1, R3, 8), + BPF_JMP_REG(BPF_JEQ, R2, R1, 2), + BPF_MOV32_IMM(R0, 2), + BPF_EXIT_INSN(), + BPF_MOV32_IMM(R0, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x1 } }, + }, + { + "ALU_MOVSX | BPF_H", + .u.insns_int = { + BPF_LD_IMM64(R2, 0x00000000ffffbeefLL), + BPF_LD_IMM64(R3, 0xdeadbeefdeadbeefLL), + BPF_MOVSX32_REG(R1, R3, 16), + BPF_JMP_REG(BPF_JEQ, R2, R1, 2), + BPF_MOV32_IMM(R0, 2), + BPF_EXIT_INSN(), + BPF_MOV32_IMM(R0, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x1 } }, + }, + { + "ALU_MOVSX | BPF_W", + .u.insns_int = { + BPF_LD_IMM64(R2, 0x00000000deadbeefLL), + BPF_LD_IMM64(R3, 0xdeadbeefdeadbeefLL), + BPF_MOVSX32_REG(R1, R3, 32), + BPF_JMP_REG(BPF_JEQ, R2, R1, 2), + BPF_MOV32_IMM(R0, 2), + BPF_EXIT_INSN(), + BPF_MOV32_IMM(R0, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x1 } }, + }, + /* MOVSX64 REG */ + { + "ALU64_MOVSX | BPF_B", + .u.insns_int = { + BPF_LD_IMM64(R2, 0xffffffffffffffefLL), + BPF_LD_IMM64(R3, 0xdeadbeefdeadbeefLL), + BPF_MOVSX64_REG(R1, R3, 8), + BPF_JMP_REG(BPF_JEQ, R2, R1, 2), + BPF_MOV32_IMM(R0, 2), + BPF_EXIT_INSN(), + BPF_MOV32_IMM(R0, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x1 } }, + }, + { + "ALU64_MOVSX | BPF_H", + .u.insns_int = { + BPF_LD_IMM64(R2, 0xffffffffffffbeefLL), + BPF_LD_IMM64(R3, 0xdeadbeefdeadbeefLL), + BPF_MOVSX64_REG(R1, R3, 16), + BPF_JMP_REG(BPF_JEQ, R2, R1, 2), + BPF_MOV32_IMM(R0, 2), + BPF_EXIT_INSN(), + BPF_MOV32_IMM(R0, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x1 } }, + }, + { + "ALU64_MOVSX | BPF_W", + .u.insns_int = { + BPF_LD_IMM64(R2, 0xffffffffdeadbeefLL), + BPF_LD_IMM64(R3, 0xdeadbeefdeadbeefLL), + BPF_MOVSX64_REG(R1, R3, 32), + BPF_JMP_REG(BPF_JEQ, R2, R1, 2), + BPF_MOV32_IMM(R0, 2), + BPF_EXIT_INSN(), + BPF_MOV32_IMM(R0, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x1 } }, + }, /* BPF_ALU | BPF_ADD | BPF_X */ { "ALU_ADD_X: 1 + 2 = 3", @@ -6105,6 +6203,106 @@ static struct bpf_test tests[] = { { }, { { 0, 2 } }, }, + /* BPF_ALU | BPF_DIV | BPF_X off=1 (SDIV) */ + { + "ALU_SDIV_X: -6 / 2 = -3", + .u.insns_int = { + BPF_LD_IMM64(R0, -6), + BPF_ALU32_IMM(BPF_MOV, R1, 2), + BPF_ALU32_REG_OFF(BPF_DIV, R0, R1, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -3 } }, + }, + /* BPF_ALU | BPF_DIV | BPF_K off=1 (SDIV) */ + { + "ALU_SDIV_K: -6 / 2 = -3", + .u.insns_int = { + BPF_LD_IMM64(R0, -6), + BPF_ALU32_IMM_OFF(BPF_DIV, R0, 2, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -3 } }, + }, + /* BPF_ALU64 | BPF_DIV | BPF_X off=1 (SDIV64) */ + { + "ALU64_SDIV_X: -6 / 2 = -3", + .u.insns_int = { + BPF_LD_IMM64(R0, -6), + BPF_ALU32_IMM(BPF_MOV, R1, 2), + BPF_ALU64_REG_OFF(BPF_DIV, R0, R1, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -3 } }, + }, + /* BPF_ALU64 | BPF_DIV | BPF_K off=1 (SDIV64) */ + { + "ALU64_SDIV_K: -6 / 2 = -3", + .u.insns_int = { + BPF_LD_IMM64(R0, -6), + BPF_ALU64_IMM_OFF(BPF_DIV, R0, 2, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -3 } }, + }, + /* BPF_ALU | BPF_MOD | BPF_X off=1 (SMOD) */ + { + "ALU_SMOD_X: -7 % 2 = -1", + .u.insns_int = { + BPF_LD_IMM64(R0, -7), + BPF_ALU32_IMM(BPF_MOV, R1, 2), + BPF_ALU32_REG_OFF(BPF_MOD, R0, R1, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -1 } }, + }, + /* BPF_ALU | BPF_MOD | BPF_K off=1 (SMOD) */ + { + "ALU_SMOD_K: -7 % 2 = -1", + .u.insns_int = { + BPF_LD_IMM64(R0, -7), + BPF_ALU32_IMM_OFF(BPF_MOD, R0, 2, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -1 } }, + }, + /* BPF_ALU64 | BPF_MOD | BPF_X off=1 (SMOD64) */ + { + "ALU64_SMOD_X: -7 % 2 = -1", + .u.insns_int = { + BPF_LD_IMM64(R0, -7), + BPF_ALU32_IMM(BPF_MOV, R1, 2), + BPF_ALU64_REG_OFF(BPF_MOD, R0, R1, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -1 } }, + }, + /* BPF_ALU64 | BPF_MOD | BPF_K off=1 (SMOD64) */ + { + "ALU64_SMOD_X: -7 % 2 = -1", + .u.insns_int = { + BPF_LD_IMM64(R0, -7), + BPF_ALU64_IMM_OFF(BPF_MOD, R0, 2, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -1 } }, + }, /* BPF_ALU | BPF_AND | BPF_X */ { "ALU_AND_X: 3 & 2 = 2", @@ -7837,6 +8035,104 @@ static struct bpf_test tests[] = { { }, { { 0, (u32) (cpu_to_le64(0xfedcba9876543210ULL) >> 32) } }, }, + /* BSWAP */ + { + "BSWAP 16: 0x0123456789abcdef -> 0xefcd", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_BSWAP(R0, 16), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xefcd } }, + }, + { + "BSWAP 32: 0x0123456789abcdef -> 0xefcdab89", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_BSWAP(R0, 32), + BPF_ALU64_REG(BPF_MOV, R1, R0), + BPF_ALU64_IMM(BPF_RSH, R1, 32), + BPF_ALU32_REG(BPF_ADD, R0, R1), /* R1 = 0 */ + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xefcdab89 } }, + }, + { + "BSWAP 64: 0x0123456789abcdef -> 0x67452301", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_BSWAP(R0, 64), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x67452301 } }, + }, + { + "BSWAP 64: 0x0123456789abcdef >> 32 -> 0xefcdab89", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_BSWAP(R0, 64), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xefcdab89 } }, + }, + /* BSWAP, reversed */ + { + "BSWAP 16: 0xfedcba9876543210 -> 0x1032", + .u.insns_int = { + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), + BPF_BSWAP(R0, 16), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x1032 } }, + }, + { + "BSWAP 32: 0xfedcba9876543210 -> 0x10325476", + .u.insns_int = { + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), + BPF_BSWAP(R0, 32), + BPF_ALU64_REG(BPF_MOV, R1, R0), + BPF_ALU64_IMM(BPF_RSH, R1, 32), + BPF_ALU32_REG(BPF_ADD, R0, R1), /* R1 = 0 */ + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x10325476 } }, + }, + { + "BSWAP 64: 0xfedcba9876543210 -> 0x98badcfe", + .u.insns_int = { + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), + BPF_BSWAP(R0, 64), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x98badcfe } }, + }, + { + "BSWAP 64: 0xfedcba9876543210 >> 32 -> 0x10325476", + .u.insns_int = { + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), + BPF_BSWAP(R0, 64), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x10325476 } }, + }, /* BPF_LDX_MEM B/H/W/DW */ { "BPF_LDX_MEM | BPF_B, base", @@ -8228,6 +8524,67 @@ static struct bpf_test tests[] = { { { 32, 0 } }, .stack_depth = 0, }, + /* BPF_LDX_MEMSX B/H/W */ + { + "BPF_LDX_MEMSX | BPF_B", + .u.insns_int = { + BPF_LD_IMM64(R1, 0xdead0000000000f0ULL), + BPF_LD_IMM64(R2, 0xfffffffffffffff0ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_LDX_MEMSX(BPF_B, R0, R10, -1), +#else + BPF_LDX_MEMSX(BPF_B, R0, R10, -8), +#endif + BPF_JMP_REG(BPF_JNE, R0, R2, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + { + "BPF_LDX_MEMSX | BPF_H", + .u.insns_int = { + BPF_LD_IMM64(R1, 0xdead00000000f123ULL), + BPF_LD_IMM64(R2, 0xfffffffffffff123ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_LDX_MEMSX(BPF_H, R0, R10, -2), +#else + BPF_LDX_MEMSX(BPF_H, R0, R10, -8), +#endif + BPF_JMP_REG(BPF_JNE, R0, R2, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + { + "BPF_LDX_MEMSX | BPF_W", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x00000000deadbeefULL), + BPF_LD_IMM64(R2, 0xffffffffdeadbeefULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_LDX_MEMSX(BPF_W, R0, R10, -4), +#else + BPF_LDX_MEMSX(BPF_W, R0, R10, -8), +#endif + BPF_JMP_REG(BPF_JNE, R0, R2, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, /* BPF_STX_MEM B/H/W/DW */ { "BPF_STX_MEM | BPF_B", @@ -9474,6 +9831,20 @@ static struct bpf_test tests[] = { { }, { { 0, 1 } }, }, + /* BPF_JMP32 | BPF_JA */ + { + "JMP32_JA: Unconditional jump: if (true) return 1", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_JMP32_IMM(BPF_JA, 0, 1, 0), + BPF_EXIT_INSN(), + BPF_ALU32_IMM(BPF_MOV, R0, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 1 } }, + }, /* BPF_JMP | BPF_JSLT | BPF_K */ { "JMP_JSLT_K: Signed jump: if (-2 < -1) return 1", @@ -14381,25 +14752,15 @@ static void *generate_test_data(struct bpf_test *test, int sub) * single fragment to the skb, filled with * test->frag_data. */ - void *ptr; - page = alloc_page(GFP_KERNEL); - if (!page) goto err_kfree_skb; - ptr = kmap(page); - if (!ptr) - goto err_free_page; - memcpy(ptr, test->frag_data, MAX_DATA); - kunmap(page); + memcpy(page_address(page), test->frag_data, MAX_DATA); skb_add_rx_frag(skb, 0, page, 0, MAX_DATA, MAX_DATA); } return skb; - -err_free_page: - __free_page(page); err_kfree_skb: kfree_skb(skb); return NULL; @@ -14577,8 +14938,10 @@ static int run_one(const struct bpf_prog *fp, struct bpf_test *test) if (ret == test->test[i].result) { pr_cont("%lld ", duration); } else { - pr_cont("ret %d != %d ", ret, - test->test[i].result); + s32 res = test->test[i].result; + + pr_cont("ret %d != %d (%#x != %#x)", + ret, res, ret, res); err_cnt++; } } @@ -15055,7 +15418,7 @@ static __init int prepare_tail_call_tests(struct bpf_array **pprogs) struct bpf_array *progs; int which, err; - /* Allocate the table of programs to be used for tall calls */ + /* Allocate the table of programs to be used for tail calls */ progs = kzalloc(struct_size(progs, ptrs, ntests + 1), GFP_KERNEL); if (!progs) goto out_nomem; diff --git a/lib/test_hmm.c b/lib/test_hmm.c index 67e6f83fe0f8..717dcb830127 100644 --- a/lib/test_hmm.c +++ b/lib/test_hmm.c @@ -368,16 +368,13 @@ static int dmirror_do_read(struct dmirror *dmirror, unsigned long start, for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { void *entry; struct page *page; - void *tmp; entry = xa_load(&dmirror->pt, pfn); page = xa_untag_pointer(entry); if (!page) return -ENOENT; - tmp = kmap(page); - memcpy(ptr, tmp, PAGE_SIZE); - kunmap(page); + memcpy_from_page(ptr, page, 0, PAGE_SIZE); ptr += PAGE_SIZE; bounce->cpages++; @@ -437,16 +434,13 @@ static int dmirror_do_write(struct dmirror *dmirror, unsigned long start, for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { void *entry; struct page *page; - void *tmp; entry = xa_load(&dmirror->pt, pfn); page = xa_untag_pointer(entry); if (!page || xa_pointer_tag(entry) != DPT_XA_TAG_WRITE) return -ENOENT; - tmp = kmap(page); - memcpy(tmp, ptr, PAGE_SIZE); - kunmap(page); + memcpy_to_page(page, 0, ptr, PAGE_SIZE); ptr += PAGE_SIZE; bounce->cpages++; diff --git a/lib/test_ida.c b/lib/test_ida.c index b06880625961..55105baa19da 100644 --- a/lib/test_ida.c +++ b/lib/test_ida.c @@ -150,6 +150,45 @@ static void ida_check_conv(struct ida *ida) IDA_BUG_ON(ida, !ida_is_empty(ida)); } +/* + * Check various situations where we attempt to free an ID we don't own. + */ +static void ida_check_bad_free(struct ida *ida) +{ + unsigned long i; + + printk("vvv Ignore \"not allocated\" warnings\n"); + /* IDA is empty; all of these will fail */ + ida_free(ida, 0); + for (i = 0; i < 31; i++) + ida_free(ida, 1 << i); + + /* IDA contains a single value entry */ + IDA_BUG_ON(ida, ida_alloc_min(ida, 3, GFP_KERNEL) != 3); + ida_free(ida, 0); + for (i = 0; i < 31; i++) + ida_free(ida, 1 << i); + + /* IDA contains a single bitmap */ + IDA_BUG_ON(ida, ida_alloc_min(ida, 1023, GFP_KERNEL) != 1023); + ida_free(ida, 0); + for (i = 0; i < 31; i++) + ida_free(ida, 1 << i); + + /* IDA contains a tree */ + IDA_BUG_ON(ida, ida_alloc_min(ida, (1 << 20) - 1, GFP_KERNEL) != (1 << 20) - 1); + ida_free(ida, 0); + for (i = 0; i < 31; i++) + ida_free(ida, 1 << i); + printk("^^^ \"not allocated\" warnings over\n"); + + ida_free(ida, 3); + ida_free(ida, 1023); + ida_free(ida, (1 << 20) - 1); + + IDA_BUG_ON(ida, !ida_is_empty(ida)); +} + static DEFINE_IDA(ida); static int ida_checks(void) @@ -162,6 +201,7 @@ static int ida_checks(void) ida_check_leaf(&ida, 1024 * 64); ida_check_max(&ida); ida_check_conv(&ida); + ida_check_bad_free(&ida); printk("IDA: %u of %u tests passed\n", tests_passed, tests_run); return (tests_run != tests_passed) ? 0 : -EINVAL; diff --git a/lib/test_maple_tree.c b/lib/test_maple_tree.c index 8d4c92cbdd0c..464eeb90d5ad 100644 --- a/lib/test_maple_tree.c +++ b/lib/test_maple_tree.c @@ -9,6 +9,7 @@ #include <linux/maple_tree.h> #include <linux/module.h> +#include <linux/rwsem.h> #define MTREE_ALLOC_MAX 0x2000000000000Ul #define CONFIG_MAPLE_SEARCH @@ -44,6 +45,8 @@ atomic_t maple_tree_tests_passed; /* #define BENCH_WALK */ /* #define BENCH_MT_FOR_EACH */ /* #define BENCH_FORK */ +/* #define BENCH_MAS_FOR_EACH */ +/* #define BENCH_MAS_PREV */ #ifdef __KERNEL__ #define mt_set_non_kernel(x) do {} while (0) @@ -1157,6 +1160,71 @@ static noinline void __init check_ranges(struct maple_tree *mt) MT_BUG_ON(mt, !mt_height(mt)); mtree_destroy(mt); + /* Check in-place modifications */ + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + /* Append to the start of last range */ + mt_set_non_kernel(50); + for (i = 0; i <= 500; i++) { + val = i * 5 + 1; + val2 = val + 4; + check_store_range(mt, val, val2, xa_mk_value(val), 0); + } + + /* Append to the last range without touching any boundaries */ + for (i = 0; i < 10; i++) { + val = val2 + 5; + val2 = val + 4; + check_store_range(mt, val, val2, xa_mk_value(val), 0); + } + + /* Append to the end of last range */ + val = val2; + for (i = 0; i < 10; i++) { + val += 5; + MT_BUG_ON(mt, mtree_test_store_range(mt, val, ULONG_MAX, + xa_mk_value(val)) != 0); + } + + /* Overwriting the range and over a part of the next range */ + for (i = 10; i < 30; i += 2) { + val = i * 5 + 1; + val2 = val + 5; + check_store_range(mt, val, val2, xa_mk_value(val), 0); + } + + /* Overwriting a part of the range and over the next range */ + for (i = 50; i < 70; i += 2) { + val2 = i * 5; + val = val2 - 5; + check_store_range(mt, val, val2, xa_mk_value(val), 0); + } + + /* + * Expand the range, only partially overwriting the previous and + * next ranges + */ + for (i = 100; i < 130; i += 3) { + val = i * 5 - 5; + val2 = i * 5 + 1; + check_store_range(mt, val, val2, xa_mk_value(val), 0); + } + + /* + * Expand the range, only partially overwriting the previous and + * next ranges, in RCU mode + */ + mt_set_in_rcu(mt); + for (i = 150; i < 180; i += 3) { + val = i * 5 - 5; + val2 = i * 5 + 1; + check_store_range(mt, val, val2, xa_mk_value(val), 0); + } + + MT_BUG_ON(mt, !mt_height(mt)); + mt_validate(mt); + mt_set_non_kernel(0); + mtree_destroy(mt); + /* Test rebalance gaps */ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); mt_set_non_kernel(50); @@ -1705,6 +1773,66 @@ static noinline void __init bench_mt_for_each(struct maple_tree *mt) } #endif +#if defined(BENCH_MAS_FOR_EACH) +static noinline void __init bench_mas_for_each(struct maple_tree *mt) +{ + int i, count = 1000000; + unsigned long max = 2500; + void *entry; + MA_STATE(mas, mt, 0, 0); + + for (i = 0; i < max; i += 5) { + int gap = 4; + + if (i % 30 == 0) + gap = 3; + mtree_store_range(mt, i, i + gap, xa_mk_value(i), GFP_KERNEL); + } + + rcu_read_lock(); + for (i = 0; i < count; i++) { + unsigned long j = 0; + + mas_for_each(&mas, entry, max) { + MT_BUG_ON(mt, entry != xa_mk_value(j)); + j += 5; + } + mas_set(&mas, 0); + } + rcu_read_unlock(); + +} +#endif +#if defined(BENCH_MAS_PREV) +static noinline void __init bench_mas_prev(struct maple_tree *mt) +{ + int i, count = 1000000; + unsigned long max = 2500; + void *entry; + MA_STATE(mas, mt, 0, 0); + + for (i = 0; i < max; i += 5) { + int gap = 4; + + if (i % 30 == 0) + gap = 3; + mtree_store_range(mt, i, i + gap, xa_mk_value(i), GFP_KERNEL); + } + + rcu_read_lock(); + for (i = 0; i < count; i++) { + unsigned long j = 2495; + + mas_set(&mas, ULONG_MAX); + while ((entry = mas_prev(&mas, 0)) != NULL) { + MT_BUG_ON(mt, entry != xa_mk_value(j)); + j -= 5; + } + } + rcu_read_unlock(); + +} +#endif /* check_forking - simulate the kernel forking sequence with the tree. */ static noinline void __init check_forking(struct maple_tree *mt) { @@ -1714,17 +1842,21 @@ static noinline void __init check_forking(struct maple_tree *mt) void *val; MA_STATE(mas, mt, 0, 0); MA_STATE(newmas, mt, 0, 0); + struct rw_semaphore newmt_lock; + + init_rwsem(&newmt_lock); for (i = 0; i <= nr_entries; i++) mtree_store_range(mt, i*10, i*10 + 5, xa_mk_value(i), GFP_KERNEL); mt_set_non_kernel(99999); - mt_init_flags(&newmt, MT_FLAGS_ALLOC_RANGE); + mt_init_flags(&newmt, MT_FLAGS_ALLOC_RANGE | MT_FLAGS_LOCK_EXTERN); + mt_set_external_lock(&newmt, &newmt_lock); newmas.tree = &newmt; mas_reset(&newmas); mas_reset(&mas); - mas_lock(&newmas); + down_write(&newmt_lock); mas.index = 0; mas.last = 0; if (mas_expected_entries(&newmas, nr_entries)) { @@ -1739,10 +1871,10 @@ static noinline void __init check_forking(struct maple_tree *mt) } rcu_read_unlock(); mas_destroy(&newmas); - mas_unlock(&newmas); mt_validate(&newmt); mt_set_non_kernel(0); - mtree_destroy(&newmt); + __mt_destroy(&newmt); + up_write(&newmt_lock); } static noinline void __init check_iteration(struct maple_tree *mt) @@ -1853,6 +1985,10 @@ static noinline void __init bench_forking(struct maple_tree *mt) void *val; MA_STATE(mas, mt, 0, 0); MA_STATE(newmas, mt, 0, 0); + struct rw_semaphore newmt_lock; + + init_rwsem(&newmt_lock); + mt_set_external_lock(&newmt, &newmt_lock); for (i = 0; i <= nr_entries; i++) mtree_store_range(mt, i*10, i*10 + 5, @@ -1867,7 +2003,7 @@ static noinline void __init bench_forking(struct maple_tree *mt) mas.index = 0; mas.last = 0; rcu_read_lock(); - mas_lock(&newmas); + down_write(&newmt_lock); if (mas_expected_entries(&newmas, nr_entries)) { printk("OOM!"); BUG_ON(1); @@ -1878,11 +2014,11 @@ static noinline void __init bench_forking(struct maple_tree *mt) mas_store(&newmas, val); } mas_destroy(&newmas); - mas_unlock(&newmas); rcu_read_unlock(); mt_validate(&newmt); mt_set_non_kernel(0); - mtree_destroy(&newmt); + __mt_destroy(&newmt); + up_write(&newmt_lock); } } #endif @@ -2039,7 +2175,7 @@ static noinline void __init next_prev_test(struct maple_tree *mt) MT_BUG_ON(mt, val != NULL); MT_BUG_ON(mt, mas.index != 0); MT_BUG_ON(mt, mas.last != 5); - MT_BUG_ON(mt, mas.node != MAS_NONE); + MT_BUG_ON(mt, mas.node != MAS_UNDERFLOW); mas.index = 0; mas.last = 5; @@ -2489,6 +2625,10 @@ static noinline void __init check_dup_gaps(struct maple_tree *mt, void *tmp; MA_STATE(mas, mt, 0, 0); MA_STATE(newmas, &newmt, 0, 0); + struct rw_semaphore newmt_lock; + + init_rwsem(&newmt_lock); + mt_set_external_lock(&newmt, &newmt_lock); if (!zero_start) i = 1; @@ -2498,9 +2638,9 @@ static noinline void __init check_dup_gaps(struct maple_tree *mt, mtree_store_range(mt, i*10, (i+1)*10 - gap, xa_mk_value(i), GFP_KERNEL); - mt_init_flags(&newmt, MT_FLAGS_ALLOC_RANGE); + mt_init_flags(&newmt, MT_FLAGS_ALLOC_RANGE | MT_FLAGS_LOCK_EXTERN); mt_set_non_kernel(99999); - mas_lock(&newmas); + down_write(&newmt_lock); ret = mas_expected_entries(&newmas, nr_entries); mt_set_non_kernel(0); MT_BUG_ON(mt, ret != 0); @@ -2513,9 +2653,9 @@ static noinline void __init check_dup_gaps(struct maple_tree *mt, } rcu_read_unlock(); mas_destroy(&newmas); - mas_unlock(&newmas); - mtree_destroy(&newmt); + __mt_destroy(&newmt); + up_write(&newmt_lock); } /* Duplicate many sizes of trees. Mainly to test expected entry values */ @@ -2790,6 +2930,7 @@ static noinline void __init check_empty_area_fill(struct maple_tree *mt) * exists MAS_NONE active range * exists active active range * DNE active active set to last range + * ERANGE active MAS_OVERFLOW last range * * Function ENTRY Start Result index & last * mas_prev() @@ -2818,6 +2959,7 @@ static noinline void __init check_empty_area_fill(struct maple_tree *mt) * any MAS_ROOT MAS_NONE 0 * exists active active range * DNE active active last range + * ERANGE active MAS_UNDERFLOW last range * * Function ENTRY Start Result index & last * mas_find() @@ -2828,7 +2970,7 @@ static noinline void __init check_empty_area_fill(struct maple_tree *mt) * DNE MAS_START MAS_NONE 0 * DNE MAS_PAUSE MAS_NONE 0 * DNE MAS_ROOT MAS_NONE 0 - * DNE MAS_NONE MAS_NONE 0 + * DNE MAS_NONE MAS_NONE 1 * if index == 0 * exists MAS_START MAS_ROOT 0 * exists MAS_PAUSE MAS_ROOT 0 @@ -2840,7 +2982,7 @@ static noinline void __init check_empty_area_fill(struct maple_tree *mt) * DNE MAS_START active set to max * exists MAS_PAUSE active range * DNE MAS_PAUSE active set to max - * exists MAS_NONE active range + * exists MAS_NONE active range (start at last) * exists active active range * DNE active active last range (max < last) * @@ -2865,7 +3007,7 @@ static noinline void __init check_empty_area_fill(struct maple_tree *mt) * DNE MAS_START active set to min * exists MAS_PAUSE active range * DNE MAS_PAUSE active set to min - * exists MAS_NONE active range + * exists MAS_NONE active range (start at index) * exists active active range * DNE active active last range (min > index) * @@ -2912,10 +3054,10 @@ static noinline void __init check_state_handling(struct maple_tree *mt) mtree_store_range(mt, 0, 0, ptr, GFP_KERNEL); mas_lock(&mas); - /* prev: Start -> none */ + /* prev: Start -> underflow*/ entry = mas_prev(&mas, 0); MT_BUG_ON(mt, entry != NULL); - MT_BUG_ON(mt, mas.node != MAS_NONE); + MT_BUG_ON(mt, mas.node != MAS_UNDERFLOW); /* prev: Start -> root */ mas_set(&mas, 10); @@ -2942,7 +3084,7 @@ static noinline void __init check_state_handling(struct maple_tree *mt) MT_BUG_ON(mt, entry != NULL); MT_BUG_ON(mt, mas.node != MAS_NONE); - /* next: start -> none */ + /* next: start -> none*/ mas_set(&mas, 10); entry = mas_next(&mas, ULONG_MAX); MT_BUG_ON(mt, mas.index != 1); @@ -3141,27 +3283,48 @@ static noinline void __init check_state_handling(struct maple_tree *mt) MT_BUG_ON(mt, mas.last != 0x2500); MT_BUG_ON(mt, !mas_active(mas)); - /* next:active -> active out of range*/ + /* next:active -> active beyond data */ entry = mas_next(&mas, 0x2999); MT_BUG_ON(mt, entry != NULL); MT_BUG_ON(mt, mas.index != 0x2501); MT_BUG_ON(mt, mas.last != 0x2fff); MT_BUG_ON(mt, !mas_active(mas)); - /* Continue after out of range*/ + /* Continue after last range ends after max */ entry = mas_next(&mas, ULONG_MAX); MT_BUG_ON(mt, entry != ptr3); MT_BUG_ON(mt, mas.index != 0x3000); MT_BUG_ON(mt, mas.last != 0x3500); MT_BUG_ON(mt, !mas_active(mas)); - /* next:active -> active out of range*/ + /* next:active -> active continued */ entry = mas_next(&mas, ULONG_MAX); MT_BUG_ON(mt, entry != NULL); MT_BUG_ON(mt, mas.index != 0x3501); MT_BUG_ON(mt, mas.last != ULONG_MAX); MT_BUG_ON(mt, !mas_active(mas)); + /* next:active -> overflow */ + entry = mas_next(&mas, ULONG_MAX); + MT_BUG_ON(mt, entry != NULL); + MT_BUG_ON(mt, mas.index != 0x3501); + MT_BUG_ON(mt, mas.last != ULONG_MAX); + MT_BUG_ON(mt, mas.node != MAS_OVERFLOW); + + /* next:overflow -> overflow */ + entry = mas_next(&mas, ULONG_MAX); + MT_BUG_ON(mt, entry != NULL); + MT_BUG_ON(mt, mas.index != 0x3501); + MT_BUG_ON(mt, mas.last != ULONG_MAX); + MT_BUG_ON(mt, mas.node != MAS_OVERFLOW); + + /* prev:overflow -> active */ + entry = mas_prev(&mas, 0); + MT_BUG_ON(mt, entry != ptr3); + MT_BUG_ON(mt, mas.index != 0x3000); + MT_BUG_ON(mt, mas.last != 0x3500); + MT_BUG_ON(mt, !mas_active(mas)); + /* next: none -> active, skip value at location */ mas_set(&mas, 0); entry = mas_next(&mas, ULONG_MAX); @@ -3180,11 +3343,46 @@ static noinline void __init check_state_handling(struct maple_tree *mt) MT_BUG_ON(mt, mas.last != 0x1500); MT_BUG_ON(mt, !mas_active(mas)); - /* prev:active -> active out of range*/ + /* prev:active -> active spanning end range */ + entry = mas_prev(&mas, 0x0100); + MT_BUG_ON(mt, entry != NULL); + MT_BUG_ON(mt, mas.index != 0); + MT_BUG_ON(mt, mas.last != 0x0FFF); + MT_BUG_ON(mt, !mas_active(mas)); + + /* prev:active -> underflow */ + entry = mas_prev(&mas, 0); + MT_BUG_ON(mt, entry != NULL); + MT_BUG_ON(mt, mas.index != 0); + MT_BUG_ON(mt, mas.last != 0x0FFF); + MT_BUG_ON(mt, mas.node != MAS_UNDERFLOW); + + /* prev:underflow -> underflow */ entry = mas_prev(&mas, 0); MT_BUG_ON(mt, entry != NULL); MT_BUG_ON(mt, mas.index != 0); MT_BUG_ON(mt, mas.last != 0x0FFF); + MT_BUG_ON(mt, mas.node != MAS_UNDERFLOW); + + /* next:underflow -> active */ + entry = mas_next(&mas, ULONG_MAX); + MT_BUG_ON(mt, entry != ptr); + MT_BUG_ON(mt, mas.index != 0x1000); + MT_BUG_ON(mt, mas.last != 0x1500); + MT_BUG_ON(mt, !mas_active(mas)); + + /* prev:first value -> underflow */ + entry = mas_prev(&mas, 0x1000); + MT_BUG_ON(mt, entry != NULL); + MT_BUG_ON(mt, mas.index != 0x1000); + MT_BUG_ON(mt, mas.last != 0x1500); + MT_BUG_ON(mt, mas.node != MAS_UNDERFLOW); + + /* find:underflow -> first value */ + entry = mas_find(&mas, ULONG_MAX); + MT_BUG_ON(mt, entry != ptr); + MT_BUG_ON(mt, mas.index != 0x1000); + MT_BUG_ON(mt, mas.last != 0x1500); MT_BUG_ON(mt, !mas_active(mas)); /* prev: pause ->active */ @@ -3198,14 +3396,14 @@ static noinline void __init check_state_handling(struct maple_tree *mt) MT_BUG_ON(mt, mas.last != 0x2500); MT_BUG_ON(mt, !mas_active(mas)); - /* prev:active -> active out of range*/ + /* prev:active -> active spanning min */ entry = mas_prev(&mas, 0x1600); MT_BUG_ON(mt, entry != NULL); MT_BUG_ON(mt, mas.index != 0x1501); MT_BUG_ON(mt, mas.last != 0x1FFF); MT_BUG_ON(mt, !mas_active(mas)); - /* prev: active ->active, continue*/ + /* prev: active ->active, continue */ entry = mas_prev(&mas, 0); MT_BUG_ON(mt, entry != ptr); MT_BUG_ON(mt, mas.index != 0x1000); @@ -3252,7 +3450,7 @@ static noinline void __init check_state_handling(struct maple_tree *mt) MT_BUG_ON(mt, mas.last != 0x2FFF); MT_BUG_ON(mt, !mas_active(mas)); - /* find: none ->active */ + /* find: overflow ->active */ entry = mas_find(&mas, 0x5000); MT_BUG_ON(mt, entry != ptr3); MT_BUG_ON(mt, mas.index != 0x3000); @@ -3433,6 +3631,20 @@ static int __init maple_tree_seed(void) mtree_destroy(&tree); goto skip; #endif +#if defined(BENCH_MAS_FOR_EACH) +#define BENCH + mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); + bench_mas_for_each(&tree); + mtree_destroy(&tree); + goto skip; +#endif +#if defined(BENCH_MAS_PREV) +#define BENCH + mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); + bench_mas_prev(&tree); + mtree_destroy(&tree); + goto skip; +#endif mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); check_iteration(&tree); @@ -3637,7 +3849,6 @@ static int __init maple_tree_seed(void) check_empty_area_fill(&tree); mtree_destroy(&tree); - mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); check_state_handling(&tree); mtree_destroy(&tree); diff --git a/lib/test_meminit.c b/lib/test_meminit.c index 60e1984c060f..0ae35223d773 100644 --- a/lib/test_meminit.c +++ b/lib/test_meminit.c @@ -93,7 +93,7 @@ static int __init test_pages(int *total_failures) int failures = 0, num_tests = 0; int i; - for (i = 0; i < 10; i++) + for (i = 0; i <= MAX_ORDER; i++) num_tests += do_alloc_pages_order(i, &failures); REPORT_FAILURES_IN_FN(); diff --git a/lib/test_objpool.c b/lib/test_objpool.c new file mode 100644 index 000000000000..bfdb81599832 --- /dev/null +++ b/lib/test_objpool.c @@ -0,0 +1,690 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Test module for lockless object pool + * + * Copyright: wuqiang.matt@bytedance.com + */ + +#include <linux/errno.h> +#include <linux/module.h> +#include <linux/moduleparam.h> +#include <linux/completion.h> +#include <linux/kthread.h> +#include <linux/slab.h> +#include <linux/vmalloc.h> +#include <linux/delay.h> +#include <linux/hrtimer.h> +#include <linux/objpool.h> + +#define OT_NR_MAX_BULK (16) + +/* memory usage */ +struct ot_mem_stat { + atomic_long_t alloc; + atomic_long_t free; +}; + +/* object allocation results */ +struct ot_obj_stat { + unsigned long nhits; + unsigned long nmiss; +}; + +/* control & results per testcase */ +struct ot_data { + struct rw_semaphore start; + struct completion wait; + struct completion rcu; + atomic_t nthreads ____cacheline_aligned_in_smp; + atomic_t stop ____cacheline_aligned_in_smp; + struct ot_mem_stat kmalloc; + struct ot_mem_stat vmalloc; + struct ot_obj_stat objects; + u64 duration; +}; + +/* testcase */ +struct ot_test { + int async; /* synchronous or asynchronous */ + int mode; /* only mode 0 supported */ + int objsz; /* object size */ + int duration; /* ms */ + int delay; /* ms */ + int bulk_normal; + int bulk_irq; + unsigned long hrtimer; /* ms */ + const char *name; + struct ot_data data; +}; + +/* per-cpu worker */ +struct ot_item { + struct objpool_head *pool; /* pool head */ + struct ot_test *test; /* test parameters */ + + void (*worker)(struct ot_item *item, int irq); + + /* hrtimer control */ + ktime_t hrtcycle; + struct hrtimer hrtimer; + + int bulk[2]; /* for thread and irq */ + int delay; + u32 niters; + + /* summary per thread */ + struct ot_obj_stat stat[2]; /* thread and irq */ + u64 duration; +}; + +/* + * memory leakage checking + */ + +static void *ot_kzalloc(struct ot_test *test, long size) +{ + void *ptr = kzalloc(size, GFP_KERNEL); + + if (ptr) + atomic_long_add(size, &test->data.kmalloc.alloc); + return ptr; +} + +static void ot_kfree(struct ot_test *test, void *ptr, long size) +{ + if (!ptr) + return; + atomic_long_add(size, &test->data.kmalloc.free); + kfree(ptr); +} + +static void ot_mem_report(struct ot_test *test) +{ + long alloc, free; + + pr_info("memory allocation summary for %s\n", test->name); + + alloc = atomic_long_read(&test->data.kmalloc.alloc); + free = atomic_long_read(&test->data.kmalloc.free); + pr_info(" kmalloc: %lu - %lu = %lu\n", alloc, free, alloc - free); + + alloc = atomic_long_read(&test->data.vmalloc.alloc); + free = atomic_long_read(&test->data.vmalloc.free); + pr_info(" vmalloc: %lu - %lu = %lu\n", alloc, free, alloc - free); +} + +/* user object instance */ +struct ot_node { + void *owner; + unsigned long data; + unsigned long refs; + unsigned long payload[32]; +}; + +/* user objpool manager */ +struct ot_context { + struct objpool_head pool; /* objpool head */ + struct ot_test *test; /* test parameters */ + void *ptr; /* user pool buffer */ + unsigned long size; /* buffer size */ + struct rcu_head rcu; +}; + +static DEFINE_PER_CPU(struct ot_item, ot_pcup_items); + +static int ot_init_data(struct ot_data *data) +{ + memset(data, 0, sizeof(*data)); + init_rwsem(&data->start); + init_completion(&data->wait); + init_completion(&data->rcu); + atomic_set(&data->nthreads, 1); + + return 0; +} + +static int ot_init_node(void *nod, void *context) +{ + struct ot_context *sop = context; + struct ot_node *on = nod; + + on->owner = &sop->pool; + return 0; +} + +static enum hrtimer_restart ot_hrtimer_handler(struct hrtimer *hrt) +{ + struct ot_item *item = container_of(hrt, struct ot_item, hrtimer); + struct ot_test *test = item->test; + + if (atomic_read_acquire(&test->data.stop)) + return HRTIMER_NORESTART; + + /* do bulk-testings for objects pop/push */ + item->worker(item, 1); + + hrtimer_forward(hrt, hrt->base->get_time(), item->hrtcycle); + return HRTIMER_RESTART; +} + +static void ot_start_hrtimer(struct ot_item *item) +{ + if (!item->test->hrtimer) + return; + hrtimer_start(&item->hrtimer, item->hrtcycle, HRTIMER_MODE_REL); +} + +static void ot_stop_hrtimer(struct ot_item *item) +{ + if (!item->test->hrtimer) + return; + hrtimer_cancel(&item->hrtimer); +} + +static int ot_init_hrtimer(struct ot_item *item, unsigned long hrtimer) +{ + struct hrtimer *hrt = &item->hrtimer; + + if (!hrtimer) + return -ENOENT; + + item->hrtcycle = ktime_set(0, hrtimer * 1000000UL); + hrtimer_init(hrt, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + hrt->function = ot_hrtimer_handler; + return 0; +} + +static int ot_init_cpu_item(struct ot_item *item, + struct ot_test *test, + struct objpool_head *pool, + void (*worker)(struct ot_item *, int)) +{ + memset(item, 0, sizeof(*item)); + item->pool = pool; + item->test = test; + item->worker = worker; + + item->bulk[0] = test->bulk_normal; + item->bulk[1] = test->bulk_irq; + item->delay = test->delay; + + /* initialize hrtimer */ + ot_init_hrtimer(item, item->test->hrtimer); + return 0; +} + +static int ot_thread_worker(void *arg) +{ + struct ot_item *item = arg; + struct ot_test *test = item->test; + ktime_t start; + + atomic_inc(&test->data.nthreads); + down_read(&test->data.start); + up_read(&test->data.start); + start = ktime_get(); + ot_start_hrtimer(item); + do { + if (atomic_read_acquire(&test->data.stop)) + break; + /* do bulk-testings for objects pop/push */ + item->worker(item, 0); + } while (!kthread_should_stop()); + ot_stop_hrtimer(item); + item->duration = (u64) ktime_us_delta(ktime_get(), start); + if (atomic_dec_and_test(&test->data.nthreads)) + complete(&test->data.wait); + + return 0; +} + +static void ot_perf_report(struct ot_test *test, u64 duration) +{ + struct ot_obj_stat total, normal = {0}, irq = {0}; + int cpu, nthreads = 0; + + pr_info("\n"); + pr_info("Testing summary for %s\n", test->name); + + for_each_possible_cpu(cpu) { + struct ot_item *item = per_cpu_ptr(&ot_pcup_items, cpu); + if (!item->duration) + continue; + normal.nhits += item->stat[0].nhits; + normal.nmiss += item->stat[0].nmiss; + irq.nhits += item->stat[1].nhits; + irq.nmiss += item->stat[1].nmiss; + pr_info("CPU: %d duration: %lluus\n", cpu, item->duration); + pr_info("\tthread:\t%16lu hits \t%16lu miss\n", + item->stat[0].nhits, item->stat[0].nmiss); + pr_info("\tirq: \t%16lu hits \t%16lu miss\n", + item->stat[1].nhits, item->stat[1].nmiss); + pr_info("\ttotal: \t%16lu hits \t%16lu miss\n", + item->stat[0].nhits + item->stat[1].nhits, + item->stat[0].nmiss + item->stat[1].nmiss); + nthreads++; + } + + total.nhits = normal.nhits + irq.nhits; + total.nmiss = normal.nmiss + irq.nmiss; + + pr_info("ALL: \tnthreads: %d duration: %lluus\n", nthreads, duration); + pr_info("SUM: \t%16lu hits \t%16lu miss\n", + total.nhits, total.nmiss); + + test->data.objects = total; + test->data.duration = duration; +} + +/* + * synchronous test cases for objpool manipulation + */ + +/* objpool manipulation for synchronous mode (percpu objpool) */ +static struct ot_context *ot_init_sync_m0(struct ot_test *test) +{ + struct ot_context *sop = NULL; + int max = num_possible_cpus() << 3; + gfp_t gfp = GFP_KERNEL; + + sop = (struct ot_context *)ot_kzalloc(test, sizeof(*sop)); + if (!sop) + return NULL; + sop->test = test; + if (test->objsz < 512) + gfp = GFP_ATOMIC; + + if (objpool_init(&sop->pool, max, test->objsz, + gfp, sop, ot_init_node, NULL)) { + ot_kfree(test, sop, sizeof(*sop)); + return NULL; + } + WARN_ON(max != sop->pool.nr_objs); + + return sop; +} + +static void ot_fini_sync(struct ot_context *sop) +{ + objpool_fini(&sop->pool); + ot_kfree(sop->test, sop, sizeof(*sop)); +} + +static struct { + struct ot_context * (*init)(struct ot_test *oc); + void (*fini)(struct ot_context *sop); +} g_ot_sync_ops[] = { + {.init = ot_init_sync_m0, .fini = ot_fini_sync}, +}; + +/* + * synchronous test cases: performance mode + */ + +static void ot_bulk_sync(struct ot_item *item, int irq) +{ + struct ot_node *nods[OT_NR_MAX_BULK]; + int i; + + for (i = 0; i < item->bulk[irq]; i++) + nods[i] = objpool_pop(item->pool); + + if (!irq && (item->delay || !(++(item->niters) & 0x7FFF))) + msleep(item->delay); + + while (i-- > 0) { + struct ot_node *on = nods[i]; + if (on) { + on->refs++; + objpool_push(on, item->pool); + item->stat[irq].nhits++; + } else { + item->stat[irq].nmiss++; + } + } +} + +static int ot_start_sync(struct ot_test *test) +{ + struct ot_context *sop; + ktime_t start; + u64 duration; + unsigned long timeout; + int cpu; + + /* initialize objpool for syncrhonous testcase */ + sop = g_ot_sync_ops[test->mode].init(test); + if (!sop) + return -ENOMEM; + + /* grab rwsem to block testing threads */ + down_write(&test->data.start); + + for_each_possible_cpu(cpu) { + struct ot_item *item = per_cpu_ptr(&ot_pcup_items, cpu); + struct task_struct *work; + + ot_init_cpu_item(item, test, &sop->pool, ot_bulk_sync); + + /* skip offline cpus */ + if (!cpu_online(cpu)) + continue; + + work = kthread_create_on_node(ot_thread_worker, item, + cpu_to_node(cpu), "ot_worker_%d", cpu); + if (IS_ERR(work)) { + pr_err("failed to create thread for cpu %d\n", cpu); + } else { + kthread_bind(work, cpu); + wake_up_process(work); + } + } + + /* wait a while to make sure all threads waiting at start line */ + msleep(20); + + /* in case no threads were created: memory insufficient ? */ + if (atomic_dec_and_test(&test->data.nthreads)) + complete(&test->data.wait); + + // sched_set_fifo_low(current); + + /* start objpool testing threads */ + start = ktime_get(); + up_write(&test->data.start); + + /* yeild cpu to worker threads for duration ms */ + timeout = msecs_to_jiffies(test->duration); + schedule_timeout_interruptible(timeout); + + /* tell workers threads to quit */ + atomic_set_release(&test->data.stop, 1); + + /* wait all workers threads finish and quit */ + wait_for_completion(&test->data.wait); + duration = (u64) ktime_us_delta(ktime_get(), start); + + /* cleanup objpool */ + g_ot_sync_ops[test->mode].fini(sop); + + /* report testing summary and performance results */ + ot_perf_report(test, duration); + + /* report memory allocation summary */ + ot_mem_report(test); + + return 0; +} + +/* + * asynchronous test cases: pool lifecycle controlled by refcount + */ + +static void ot_fini_async_rcu(struct rcu_head *rcu) +{ + struct ot_context *sop = container_of(rcu, struct ot_context, rcu); + struct ot_test *test = sop->test; + + /* here all cpus are aware of the stop event: test->data.stop = 1 */ + WARN_ON(!atomic_read_acquire(&test->data.stop)); + + objpool_fini(&sop->pool); + complete(&test->data.rcu); +} + +static void ot_fini_async(struct ot_context *sop) +{ + /* make sure the stop event is acknowledged by all cores */ + call_rcu(&sop->rcu, ot_fini_async_rcu); +} + +static int ot_objpool_release(struct objpool_head *head, void *context) +{ + struct ot_context *sop = context; + + WARN_ON(!head || !sop || head != &sop->pool); + + /* do context cleaning if needed */ + if (sop) + ot_kfree(sop->test, sop, sizeof(*sop)); + + return 0; +} + +static struct ot_context *ot_init_async_m0(struct ot_test *test) +{ + struct ot_context *sop = NULL; + int max = num_possible_cpus() << 3; + gfp_t gfp = GFP_KERNEL; + + sop = (struct ot_context *)ot_kzalloc(test, sizeof(*sop)); + if (!sop) + return NULL; + sop->test = test; + if (test->objsz < 512) + gfp = GFP_ATOMIC; + + if (objpool_init(&sop->pool, max, test->objsz, gfp, sop, + ot_init_node, ot_objpool_release)) { + ot_kfree(test, sop, sizeof(*sop)); + return NULL; + } + WARN_ON(max != sop->pool.nr_objs); + + return sop; +} + +static struct { + struct ot_context * (*init)(struct ot_test *oc); + void (*fini)(struct ot_context *sop); +} g_ot_async_ops[] = { + {.init = ot_init_async_m0, .fini = ot_fini_async}, +}; + +static void ot_nod_recycle(struct ot_node *on, struct objpool_head *pool, + int release) +{ + struct ot_context *sop; + + on->refs++; + + if (!release) { + /* push object back to opjpool for reuse */ + objpool_push(on, pool); + return; + } + + sop = container_of(pool, struct ot_context, pool); + WARN_ON(sop != pool->context); + + /* unref objpool with nod removed forever */ + objpool_drop(on, pool); +} + +static void ot_bulk_async(struct ot_item *item, int irq) +{ + struct ot_test *test = item->test; + struct ot_node *nods[OT_NR_MAX_BULK]; + int i, stop; + + for (i = 0; i < item->bulk[irq]; i++) + nods[i] = objpool_pop(item->pool); + + if (!irq) { + if (item->delay || !(++(item->niters) & 0x7FFF)) + msleep(item->delay); + get_cpu(); + } + + stop = atomic_read_acquire(&test->data.stop); + + /* drop all objects and deref objpool */ + while (i-- > 0) { + struct ot_node *on = nods[i]; + + if (on) { + on->refs++; + ot_nod_recycle(on, item->pool, stop); + item->stat[irq].nhits++; + } else { + item->stat[irq].nmiss++; + } + } + + if (!irq) + put_cpu(); +} + +static int ot_start_async(struct ot_test *test) +{ + struct ot_context *sop; + ktime_t start; + u64 duration; + unsigned long timeout; + int cpu; + + /* initialize objpool for syncrhonous testcase */ + sop = g_ot_async_ops[test->mode].init(test); + if (!sop) + return -ENOMEM; + + /* grab rwsem to block testing threads */ + down_write(&test->data.start); + + for_each_possible_cpu(cpu) { + struct ot_item *item = per_cpu_ptr(&ot_pcup_items, cpu); + struct task_struct *work; + + ot_init_cpu_item(item, test, &sop->pool, ot_bulk_async); + + /* skip offline cpus */ + if (!cpu_online(cpu)) + continue; + + work = kthread_create_on_node(ot_thread_worker, item, + cpu_to_node(cpu), "ot_worker_%d", cpu); + if (IS_ERR(work)) { + pr_err("failed to create thread for cpu %d\n", cpu); + } else { + kthread_bind(work, cpu); + wake_up_process(work); + } + } + + /* wait a while to make sure all threads waiting at start line */ + msleep(20); + + /* in case no threads were created: memory insufficient ? */ + if (atomic_dec_and_test(&test->data.nthreads)) + complete(&test->data.wait); + + /* start objpool testing threads */ + start = ktime_get(); + up_write(&test->data.start); + + /* yeild cpu to worker threads for duration ms */ + timeout = msecs_to_jiffies(test->duration); + schedule_timeout_interruptible(timeout); + + /* tell workers threads to quit */ + atomic_set_release(&test->data.stop, 1); + + /* do async-finalization */ + g_ot_async_ops[test->mode].fini(sop); + + /* wait all workers threads finish and quit */ + wait_for_completion(&test->data.wait); + duration = (u64) ktime_us_delta(ktime_get(), start); + + /* assure rcu callback is triggered */ + wait_for_completion(&test->data.rcu); + + /* + * now we are sure that objpool is finalized either + * by rcu callback or by worker threads + */ + + /* report testing summary and performance results */ + ot_perf_report(test, duration); + + /* report memory allocation summary */ + ot_mem_report(test); + + return 0; +} + +/* + * predefined testing cases: + * synchronous case / overrun case / async case + * + * async: synchronous or asynchronous testing + * mode: only mode 0 supported + * objsz: object size + * duration: int, total test time in ms + * delay: int, delay (in ms) between each iteration + * bulk_normal: int, repeat times for thread worker + * bulk_irq: int, repeat times for irq consumer + * hrtimer: unsigned long, hrtimer intervnal in ms + * name: char *, tag for current test ot_item + */ + +#define NODE_COMPACT sizeof(struct ot_node) +#define NODE_VMALLOC (512) + +static struct ot_test g_testcases[] = { + + /* sync & normal */ + {0, 0, NODE_COMPACT, 1000, 0, 1, 0, 0, "sync: percpu objpool"}, + {0, 0, NODE_VMALLOC, 1000, 0, 1, 0, 0, "sync: percpu objpool from vmalloc"}, + + /* sync & hrtimer */ + {0, 0, NODE_COMPACT, 1000, 0, 1, 1, 4, "sync & hrtimer: percpu objpool"}, + {0, 0, NODE_VMALLOC, 1000, 0, 1, 1, 4, "sync & hrtimer: percpu objpool from vmalloc"}, + + /* sync & overrun */ + {0, 0, NODE_COMPACT, 1000, 0, 16, 0, 0, "sync overrun: percpu objpool"}, + {0, 0, NODE_VMALLOC, 1000, 0, 16, 0, 0, "sync overrun: percpu objpool from vmalloc"}, + + /* async mode */ + {1, 0, NODE_COMPACT, 1000, 100, 1, 0, 0, "async: percpu objpool"}, + {1, 0, NODE_VMALLOC, 1000, 100, 1, 0, 0, "async: percpu objpool from vmalloc"}, + + /* async + hrtimer mode */ + {1, 0, NODE_COMPACT, 1000, 0, 4, 4, 4, "async & hrtimer: percpu objpool"}, + {1, 0, NODE_VMALLOC, 1000, 0, 4, 4, 4, "async & hrtimer: percpu objpool from vmalloc"}, +}; + +static int __init ot_mod_init(void) +{ + int i; + + /* perform testings */ + for (i = 0; i < ARRAY_SIZE(g_testcases); i++) { + ot_init_data(&g_testcases[i].data); + if (g_testcases[i].async) + ot_start_async(&g_testcases[i]); + else + ot_start_sync(&g_testcases[i]); + } + + /* show tests summary */ + pr_info("\n"); + pr_info("Summary of testcases:\n"); + for (i = 0; i < ARRAY_SIZE(g_testcases); i++) { + pr_info(" duration: %lluus \thits: %10lu \tmiss: %10lu \t%s\n", + g_testcases[i].data.duration, g_testcases[i].data.objects.nhits, + g_testcases[i].data.objects.nmiss, g_testcases[i].name); + } + + return -EAGAIN; +} + +static void __exit ot_mod_exit(void) +{ +} + +module_init(ot_mod_init); +module_exit(ot_mod_exit); + +MODULE_LICENSE("GPL");
\ No newline at end of file diff --git a/lib/test_printf.c b/lib/test_printf.c index 7677ebccf3c3..69b6a5e177f2 100644 --- a/lib/test_printf.c +++ b/lib/test_printf.c @@ -12,6 +12,7 @@ #include <linux/random.h> #include <linux/rtc.h> #include <linux/slab.h> +#include <linux/sprintf.h> #include <linux/string.h> #include <linux/bitmap.h> @@ -41,8 +42,6 @@ KSTM_MODULE_GLOBALS(); static char *test_buffer __initdata; static char *alloced_buffer __initdata; -extern bool no_hash_pointers; - static int __printf(4, 0) __init do_test(int bufsize, const char *expect, int elen, const char *fmt, va_list ap) diff --git a/lib/test_scanf.c b/lib/test_scanf.c index b620cf7de503..a2707af2951a 100644 --- a/lib/test_scanf.c +++ b/lib/test_scanf.c @@ -606,7 +606,7 @@ static void __init numbers_slice(void) #define test_number_prefix(T, str, scan_fmt, expect0, expect1, n_args, fn) \ do { \ const T expect[2] = { expect0, expect1 }; \ - T result[2] = {~expect[0], ~expect[1]}; \ + T result[2] = { (T)~expect[0], (T)~expect[1] }; \ \ _test(fn, &expect, str, scan_fmt, n_args, &result[0], &result[1]); \ } while (0) diff --git a/lib/ts_bm.c b/lib/ts_bm.c index c8ecbf74ef29..e5f30f9177df 100644 --- a/lib/ts_bm.c +++ b/lib/ts_bm.c @@ -55,6 +55,24 @@ struct ts_bm unsigned int good_shift[]; }; +static unsigned int matchpat(const u8 *pattern, unsigned int patlen, + const u8 *text, bool icase) +{ + unsigned int i; + + for (i = 0; i < patlen; i++) { + u8 t = *(text-i); + + if (icase) + t = toupper(t); + + if (t != *(pattern-i)) + break; + } + + return i; +} + static unsigned int bm_find(struct ts_config *conf, struct ts_state *state) { struct ts_bm *bm = ts_config_priv(conf); @@ -72,19 +90,18 @@ static unsigned int bm_find(struct ts_config *conf, struct ts_state *state) break; while (shift < text_len) { - DEBUGP("Searching in position %d (%c)\n", - shift, text[shift]); - for (i = 0; i < bm->patlen; i++) - if ((icase ? toupper(text[shift-i]) - : text[shift-i]) - != bm->pattern[bm->patlen-1-i]) - goto next; - - /* London calling... */ - DEBUGP("found!\n"); - return consumed + (shift-(bm->patlen-1)); - -next: bs = bm->bad_shift[text[shift-i]]; + DEBUGP("Searching in position %d (%c)\n", + shift, text[shift]); + + i = matchpat(&bm->pattern[bm->patlen-1], bm->patlen, + &text[shift], icase); + if (i == bm->patlen) { + /* London calling... */ + DEBUGP("found!\n"); + return consumed + (shift-(bm->patlen-1)); + } + + bs = bm->bad_shift[text[shift-i]]; /* Now jumping to... */ shift = max_t(int, shift-i+bs, shift+bm->good_shift[i]); diff --git a/lib/ucs2_string.c b/lib/ucs2_string.c index 0a559a42359b..9308bcfb2ad5 100644 --- a/lib/ucs2_string.c +++ b/lib/ucs2_string.c @@ -32,6 +32,58 @@ ucs2_strsize(const ucs2_char_t *data, unsigned long maxlength) } EXPORT_SYMBOL(ucs2_strsize); +/** + * ucs2_strscpy() - Copy a UCS2 string into a sized buffer. + * + * @dst: Pointer to the destination buffer where to copy the string to. + * @src: Pointer to the source buffer where to copy the string from. + * @count: Size of the destination buffer, in UCS2 (16-bit) characters. + * + * Like strscpy(), only for UCS2 strings. + * + * Copy the source string @src, or as much of it as fits, into the destination + * buffer @dst. The behavior is undefined if the string buffers overlap. The + * destination buffer @dst is always NUL-terminated, unless it's zero-sized. + * + * Return: The number of characters copied into @dst (excluding the trailing + * %NUL terminator) or -E2BIG if @count is 0 or @src was truncated due to the + * destination buffer being too small. + */ +ssize_t ucs2_strscpy(ucs2_char_t *dst, const ucs2_char_t *src, size_t count) +{ + long res; + + /* + * Ensure that we have a valid amount of space. We need to store at + * least one NUL-character. + */ + if (count == 0 || WARN_ON_ONCE(count > INT_MAX / sizeof(*dst))) + return -E2BIG; + + /* + * Copy at most 'count' characters, return early if we find a + * NUL-terminator. + */ + for (res = 0; res < count; res++) { + ucs2_char_t c; + + c = src[res]; + dst[res] = c; + + if (!c) + return res; + } + + /* + * The loop above terminated without finding a NUL-terminator, + * exceeding the 'count': Enforce proper NUL-termination and return + * error. + */ + dst[count - 1] = 0; + return -E2BIG; +} +EXPORT_SYMBOL(ucs2_strscpy); + int ucs2_strncmp(const ucs2_char_t *a, const ucs2_char_t *b, size_t len) { diff --git a/lib/vsprintf.c b/lib/vsprintf.c index 40f560959b16..552738f14275 100644 --- a/lib/vsprintf.c +++ b/lib/vsprintf.c @@ -34,6 +34,7 @@ #include <linux/dcache.h> #include <linux/cred.h> #include <linux/rtc.h> +#include <linux/sprintf.h> #include <linux/time.h> #include <linux/uuid.h> #include <linux/of.h> @@ -59,7 +60,8 @@ bool no_hash_pointers __ro_after_init; EXPORT_SYMBOL_GPL(no_hash_pointers); -static noinline unsigned long long simple_strntoull(const char *startp, size_t max_chars, char **endp, unsigned int base) +noinline +static unsigned long long simple_strntoull(const char *startp, char **endp, unsigned int base, size_t max_chars) { const char *cp; unsigned long long result = 0ULL; @@ -94,7 +96,7 @@ static noinline unsigned long long simple_strntoull(const char *startp, size_t m noinline unsigned long long simple_strtoull(const char *cp, char **endp, unsigned int base) { - return simple_strntoull(cp, INT_MAX, endp, base); + return simple_strntoull(cp, endp, base, INT_MAX); } EXPORT_SYMBOL(simple_strtoull); @@ -129,8 +131,8 @@ long simple_strtol(const char *cp, char **endp, unsigned int base) } EXPORT_SYMBOL(simple_strtol); -static long long simple_strntoll(const char *cp, size_t max_chars, char **endp, - unsigned int base) +noinline +static long long simple_strntoll(const char *cp, char **endp, unsigned int base, size_t max_chars) { /* * simple_strntoull() safely handles receiving max_chars==0 in the @@ -139,9 +141,9 @@ static long long simple_strntoll(const char *cp, size_t max_chars, char **endp, * and the content of *cp is irrelevant. */ if (*cp == '-' && max_chars > 0) - return -simple_strntoull(cp + 1, max_chars - 1, endp, base); + return -simple_strntoull(cp + 1, endp, base, max_chars - 1); - return simple_strntoull(cp, max_chars, endp, base); + return simple_strntoull(cp, endp, base, max_chars); } /** @@ -154,7 +156,7 @@ static long long simple_strntoll(const char *cp, size_t max_chars, char **endp, */ long long simple_strtoll(const char *cp, char **endp, unsigned int base) { - return simple_strntoll(cp, INT_MAX, endp, base); + return simple_strntoll(cp, endp, base, INT_MAX); } EXPORT_SYMBOL(simple_strtoll); @@ -2109,15 +2111,20 @@ char *fwnode_full_name_string(struct fwnode_handle *fwnode, char *buf, /* Loop starting from the root node to the current node. */ for (depth = fwnode_count_parents(fwnode); depth >= 0; depth--) { - struct fwnode_handle *__fwnode = - fwnode_get_nth_parent(fwnode, depth); + /* + * Only get a reference for other nodes (i.e. parent nodes). + * fwnode refcount may be 0 here. + */ + struct fwnode_handle *__fwnode = depth ? + fwnode_get_nth_parent(fwnode, depth) : fwnode; buf = string(buf, end, fwnode_get_name_prefix(__fwnode), default_str_spec); buf = string(buf, end, fwnode_get_name(__fwnode), default_str_spec); - fwnode_handle_put(__fwnode); + if (depth) + fwnode_handle_put(__fwnode); } return buf; @@ -3647,13 +3654,11 @@ int vsscanf(const char *buf, const char *fmt, va_list args) break; if (is_sign) - val.s = simple_strntoll(str, - field_width >= 0 ? field_width : INT_MAX, - &next, base); + val.s = simple_strntoll(str, &next, base, + field_width >= 0 ? field_width : INT_MAX); else - val.u = simple_strntoull(str, - field_width >= 0 ? field_width : INT_MAX, - &next, base); + val.u = simple_strntoull(str, &next, base, + field_width >= 0 ? field_width : INT_MAX); switch (qualifier) { case 'H': /* that's 'hh' in format */ diff --git a/lib/xarray.c b/lib/xarray.c index 2071a3718f4e..39f07bfc4dcc 100644 --- a/lib/xarray.c +++ b/lib/xarray.c @@ -206,7 +206,7 @@ static void *xas_descend(struct xa_state *xas, struct xa_node *node) void *entry = xa_entry(xas->xa, node, offset); xas->xa_node = node; - if (xa_is_sibling(entry)) { + while (xa_is_sibling(entry)) { offset = xa_to_sibling(entry); entry = xa_entry(xas->xa, node, offset); if (node->shift && xa_is_node(entry)) @@ -1802,6 +1802,9 @@ EXPORT_SYMBOL(xa_get_order); * stores the index into the @id pointer, then stores the entry at * that index. A concurrent lookup will not see an uninitialised @id. * + * Must only be operated on an xarray initialized with flag XA_FLAGS_ALLOC set + * in xa_init_flags(). + * * Context: Any context. Expects xa_lock to be held on entry. May * release and reacquire xa_lock if @gfp flags permit. * Return: 0 on success, -ENOMEM if memory could not be allocated or @@ -1850,6 +1853,9 @@ EXPORT_SYMBOL(__xa_alloc); * The search for an empty entry will start at @next and will wrap * around if necessary. * + * Must only be operated on an xarray initialized with flag XA_FLAGS_ALLOC set + * in xa_init_flags(). + * * Context: Any context. Expects xa_lock to be held on entry. May * release and reacquire xa_lock if @gfp flags permit. * Return: 0 if the allocation succeeded without wrapping. 1 if the diff --git a/lib/xz/Kconfig b/lib/xz/Kconfig index adce22ac18d6..aef086a6bf2f 100644 --- a/lib/xz/Kconfig +++ b/lib/xz/Kconfig @@ -19,11 +19,6 @@ config XZ_DEC_POWERPC default y select XZ_DEC_BCJ -config XZ_DEC_IA64 - bool "IA-64 BCJ filter decoder" if EXPERT - default y - select XZ_DEC_BCJ - config XZ_DEC_ARM bool "ARM BCJ filter decoder" if EXPERT default y diff --git a/lib/zstd/common/fse_decompress.c b/lib/zstd/common/fse_decompress.c index a0d06095be83..8dcb8ca39767 100644 --- a/lib/zstd/common/fse_decompress.c +++ b/lib/zstd/common/fse_decompress.c @@ -312,7 +312,7 @@ size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size typedef struct { short ncount[FSE_MAX_SYMBOL_VALUE + 1]; - FSE_DTable dtable[1]; /* Dynamically sized */ + FSE_DTable dtable[]; /* Dynamically sized */ } FSE_DecompressWksp; |