kbuild: use git-archive for source package creation

Commit 5c3d1d0abb ("kbuild: add a tool to list files ignored by git")
added a new tool, scripts/list-gitignored. My intention was to create
source packages without cleaning the source tree, without relying on git.

Linus strongly objected to it, and suggested using 'git archive' instead.
[1] [2] [3]

This commit goes in that direction - Remove scripts/list-gitignored.c
and rewrites Makefiles and scripts to use 'git archive' for building
Debian and RPM source packages. It also makes 'make perf-tar*-src-pkg'
use 'git archive' again.

Going forward, building source packages is only possible in a git-managed
tree. Building binary packages does not require git.

[1]: https://lore.kernel.org/lkml/CAHk-=wi49sMaC7vY1yMagk7eqLK=1jHeHQ=yZ_k45P=xBccnmA@mail.gmail.com/
[2]: https://lore.kernel.org/lkml/CAHk-=wh5AixGsLeT0qH2oZHKq0FLUTbyTw4qY921L=PwYgoGVw@mail.gmail.com/
[3]: https://lore.kernel.org/lkml/CAHk-=wgM-W6Fu==EoAVCabxyX8eYBz9kNC88-tm9ExRQwA79UQ@mail.gmail.com/

Fixes: 5c3d1d0abb ("kbuild: add a tool to list files ignored by git")
Fixes: e0ca16749a ("kbuild: make perf-tar*-src-pkg work without relying on git")
Suggested-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Masahiro Yamada <masahiroy@kernel.org>
This commit is contained in:
Masahiro Yamada 2023-03-16 00:50:18 +09:00
parent 81f59a26f3
commit 05e96e96a3
10 changed files with 181 additions and 1155 deletions

View file

@ -274,8 +274,7 @@ no-dot-config-targets := $(clean-targets) \
cscope gtags TAGS tags help% %docs check% coccicheck \
$(version_h) headers headers_% archheaders archscripts \
%asm-generic kernelversion %src-pkg dt_binding_check \
outputmakefile rustavailable rustfmt rustfmtcheck \
scripts_package
outputmakefile rustavailable rustfmt rustfmtcheck
# Installation targets should not require compiler. Unfortunately, vdso_install
# is an exception where build artifacts may be updated. This must be fixed.
no-compiler-targets := $(no-dot-config-targets) install dtbs_install \
@ -1656,10 +1655,6 @@ distclean: mrproper
%pkg: include/config/kernel.release FORCE
$(Q)$(MAKE) -f $(srctree)/scripts/Makefile.package $@
PHONY += scripts_package
scripts_package: scripts_basic
$(Q)$(MAKE) $(build)=scripts scripts/list-gitignored
# Brief documentation of the typical targets used
# ---------------------------------------------------------------------------

1
scripts/.gitignore vendored
View file

@ -3,7 +3,6 @@
/generate_rust_target
/insert-sys-cert
/kallsyms
/list-gitignored
/module.lds
/recordmcount
/sign-file

View file

@ -38,7 +38,7 @@ HOSTCFLAGS_sorttable.o += -DMCOUNT_SORT_ENABLED
endif
# The following programs are only built on demand
hostprogs += list-gitignored unifdef
hostprogs += unifdef
# The module linker script is preprocessed on demand
targets += module.lds

View file

@ -2,6 +2,7 @@
# Makefile for the different targets used to generate full packages of a kernel
include $(srctree)/scripts/Kbuild.include
include $(srctree)/scripts/Makefile.lib
KERNELPATH := kernel-$(subst -,_,$(KERNELRELEASE))
KBUILD_PKG_ROOTCMD ?="fakeroot -u"
@ -26,54 +27,46 @@ fi ; \
tar -I $(KGZIP) -c $(RCS_TAR_IGNORE) -f $(2).tar.gz \
--transform 's:^:$(2)/:S' $(TAR_CONTENT) $(3)
# .tmp_filelist .tmp_filelist_exclude
# tarball compression
# ---------------------------------------------------------------------------
scripts/list-gitignored: FORCE
$(Q)$(MAKE) -f $(srctree)/Makefile scripts_package
%.tar.gz: %.tar
$(call cmd,gzip)
# 1f5d3a6b6532e25a5cdf1f311956b2b03d343a48 removed '*.rej' from .gitignore,
# but it is definitely a generated file.
filechk_filelist = \
$< --exclude='*.rej' --output=$@_exclude --prefix=./ --rootdir=$(srctree) --stat=-
%.tar.bz2: %.tar
$(call cmd,bzip2)
.tmp_filelist: scripts/list-gitignored FORCE
$(call filechk,filelist)
%.tar.xz: %.tar
$(call cmd,xzmisc)
# tarball
%.tar.zst: %.tar
$(call cmd,zstd)
# Git
# ---------------------------------------------------------------------------
quiet_cmd_tar = TAR $@
cmd_tar = tar -c -f $@ $(tar-compress-opt) $(tar-exclude-opt) \
--owner=0 --group=0 --sort=name \
--transform 's:^\.:$*:S' -C $(tar-rootdir) .
filechk_HEAD = git -C $(srctree) rev-parse --verify HEAD 2>/dev/null
tar-rootdir := $(srctree)
.tmp_HEAD: check-git FORCE
$(call filechk,HEAD)
%.tar:
$(call cmd,tar)
%.tar.gz: private tar-compress-opt := -I $(KGZIP)
%.tar.gz:
$(call cmd,tar)
%.tar.bz2: private tar-compress-opt := -I $(KBZIP2)
%.tar.bz2:
$(call cmd,tar)
%.tar.xz: private tar-compress-opt := -I $(XZ)
%.tar.xz:
$(call cmd,tar)
%.tar.zst: private tar-compress-opt := -I $(ZSTD)
%.tar.zst:
$(call cmd,tar)
PHONY += check-git
check-git:
@if ! $(srctree)/scripts/check-git; then \
echo >&2 "error: creating source package requires git repository"; \
false; \
fi
# Linux source tarball
# ---------------------------------------------------------------------------
linux.tar.gz: tar-exclude-opt = --exclude=./$@ --exclude-from=$<_exclude
linux.tar.gz: .tmp_filelist
quiet_cmd_archive_linux = ARCHIVE $@
cmd_archive_linux = \
git -C $(srctree) archive --output=$$(realpath $@) --prefix=$(basename $@)/ $$(cat $<)
targets += linux.tar
linux.tar: .tmp_HEAD FORCE
$(call if_changed,archive_linux)
# rpm-pkg
# ---------------------------------------------------------------------------
@ -148,74 +141,62 @@ snap-pkg:
# dir-pkg tar*-pkg - tarball targets
# ---------------------------------------------------------------------------
tar-pkg-tarball = linux-$(KERNELRELEASE)-$(ARCH).$(1)
tar-pkg-phony = $(subst .,,$(1))-pkg
tar-install: FORCE
$(Q)$(MAKE) -f $(srctree)/Makefile
+$(Q)$(srctree)/scripts/package/buildtar $@
quiet_cmd_tar = TAR $@
cmd_tar = cd $<; tar cf ../$@ --owner=root --group=root --sort=name *
linux-$(KERNELRELEASE)-$(ARCH).tar: tar-install
$(call cmd,tar)
PHONY += dir-pkg
dir-pkg: tar-install
@echo "Kernel tree successfully created in $<"
define tar-pkg-rule
PHONY += $(tar-pkg-phony)
$(tar-pkg-phony): $(tar-pkg-tarball)
PHONY += tar-pkg
tar-pkg: linux-$(KERNELRELEASE)-$(ARCH).tar
@:
$(tar-pkg-tarball): private tar-rootdir := tar-install
$(tar-pkg-tarball): tar-install
endef
$(foreach x, tar tar.gz tar.bz2 tar.xz tar.zst, $(eval $(call tar-pkg-rule,$(x))))
tar%-pkg: linux-$(KERNELRELEASE)-$(ARCH).tar.% FORCE
@:
# perf-tar*-src-pkg - generate a source tarball with perf source
# ---------------------------------------------------------------------------
perf-tar-src-pkg-tarball = perf-$(KERNELVERSION).$(1)
perf-tar-src-pkg-phony = perf-$(subst .,,$(1))-src-pkg
.tmp_perf:
$(Q)mkdir .tmp_perf
quiet_cmd_stage_perf_src = STAGE $@
cmd_stage_perf_src = \
rm -rf $@; \
mkdir -p $@; \
tar -c -f - --exclude-from=$<_exclude -C $(srctree) --files-from=$(srctree)/tools/perf/MANIFEST | \
tar -x -f - -C $@
.tmp_perf: .tmp_filelist
$(call cmd,stage_perf_src)
filechk_perf_head = \
if test -z "$(git -C $(srctree) rev-parse --show-cdup 2>/dev/null)" && \
head=$$(git -C $(srctree) rev-parse --verify HEAD 2>/dev/null); then \
echo $$head; \
else \
echo "not a git tree"; \
fi
.tmp_perf/HEAD: .tmp_perf FORCE
$(call filechk,perf_head)
.tmp_perf/HEAD: .tmp_HEAD | .tmp_perf
$(call cmd,copy)
quiet_cmd_perf_version_file = GEN $@
cmd_perf_version_file = cd $(srctree)/tools/perf; util/PERF-VERSION-GEN $(dir $(abspath $@))
# PERF-VERSION-FILE and HEAD are independent, but this avoids updating the
# PERF-VERSION-FILE and .tmp_HEAD are independent, but this avoids updating the
# timestamp of PERF-VERSION-FILE.
# The best is to fix tools/perf/util/PERF-VERSION-GEN.
.tmp_perf/PERF-VERSION-FILE: .tmp_perf/HEAD $(srctree)/tools/perf/util/PERF-VERSION-GEN
.tmp_perf/PERF-VERSION-FILE: .tmp_HEAD $(srctree)/tools/perf/util/PERF-VERSION-GEN | .tmp_perf
$(call cmd,perf_version_file)
define perf-tar-src-pkg-rule
PHONY += $(perf-tar-src-pkg-phony)
$(perf-tar-src-pkg-phony): $(perf-tar-src-pkg-tarball)
quiet_cmd_archive_perf = ARCHIVE $@
cmd_archive_perf = \
git -C $(srctree) archive --output=$$(realpath $@) --prefix=$(basename $@)/ \
--add-file=$$(realpath $(word 2, $^)) \
--add-file=$$(realpath $(word 3, $^)) \
$$(cat $(word 2, $^))^{tree} $$(cat $<)
targets += perf-$(KERNELVERSION).tar
perf-$(KERNELVERSION).tar: tools/perf/MANIFEST .tmp_perf/HEAD .tmp_perf/PERF-VERSION-FILE FORCE
$(call if_changed,archive_perf)
PHONY += perf-tar-src-pkg
perf-tar-src-pkg: perf-$(KERNELVERSION).tar
@:
$(perf-tar-src-pkg-tarball): private tar-rootdir := .tmp_perf
$(perf-tar-src-pkg-tarball): .tmp_filelist .tmp_perf/HEAD .tmp_perf/PERF-VERSION-FILE
endef
$(foreach x, tar tar.gz tar.bz2 tar.xz tar.zst, $(eval $(call perf-tar-src-pkg-rule,$(x))))
perf-tar%-src-pkg: perf-$(KERNELVERSION).tar.% FORCE
@:
# Help text displayed when executing 'make help'
# ---------------------------------------------------------------------------
@ -243,4 +224,13 @@ help:
PHONY += FORCE
FORCE:
# Read all saved command lines and dependencies for the $(targets) we
# may be building above, using $(if_changed{,_dep}). As an
# optimization, we don't need to read them if the target does not
# exist, we will rebuild anyway in that case.
existing-targets := $(wildcard $(sort $(targets)))
-include $(foreach f,$(existing-targets),$(dir $(f)).$(notdir $(f)).cmd)
.PHONY: $(PHONY)

14
scripts/check-git Executable file
View file

@ -0,0 +1,14 @@
#!/bin/sh
# SPDX-License-Identifier: GPL-2.0-only
#
# succeed if we are in a git repository
srctree="$(dirname $0)/.."
if ! git -C "${srctree}" rev-parse --verify HEAD >/dev/null 2>/dev/null; then
exit 1
fi
if ! test -z $(git -C "${srctree}" rev-parse --show-cdup 2>/dev/null); then
exit 1
fi

File diff suppressed because it is too large Load diff

44
scripts/package/gen-diff-patch Executable file
View file

@ -0,0 +1,44 @@
#!/bin/sh
# SPDX-License-Identifier: GPL-2.0-only
diff_patch="${1}"
untracked_patch="${2}"
srctree=$(dirname $0)/../..
rm -f ${diff_patch} ${untracked_patch}
if ! ${srctree}/scripts/check-git; then
exit
fi
mkdir -p "$(dirname ${diff_patch})" "$(dirname ${untracked_patch})"
git -C "${srctree}" diff HEAD > "${diff_patch}"
if [ ! -s "${diff_patch}" ]; then
rm -f "${diff_patch}"
exit
fi
git -C ${srctree} status --porcelain --untracked-files=all |
while read stat path
do
if [ "${stat}" = '??' ]; then
if ! diff -u /dev/null "${srctree}/${path}" > .tmp_diff &&
! head -n1 .tmp_diff | grep -q "Binary files"; then
{
echo "--- /dev/null"
echo "+++ linux/$path"
cat .tmp_diff | tail -n +3
} >> ${untracked_patch}
fi
fi
done
rm -f .tmp_diff
if [ ! -s "${diff_patch}" ]; then
rm -f "${diff_patch}"
exit
fi

View file

@ -91,7 +91,7 @@ version=$KERNELRELEASE
if [ -n "$KDEB_PKGVERSION" ]; then
packageversion=$KDEB_PKGVERSION
else
packageversion=$version-$($srctree/init/build-version)
packageversion=$(${srctree}/scripts/setlocalversion --no-local ${srctree})-$($srctree/init/build-version)
fi
sourcename=${KDEB_SOURCENAME:-linux-upstream}
@ -152,6 +152,14 @@ mkdir -p debian/patches
} > debian/patches/config
echo config > debian/patches/series
$(dirname $0)/gen-diff-patch debian/patches/diff.patch debian/patches/untracked.patch
if [ -f debian/patches/diff.patch ]; then
echo diff.patch >> debian/patches/series
fi
if [ -f debian/patches/untracked.patch ]; then
echo untracked.patch >> debian/patches/series
fi
echo $debarch > debian/arch
extra_build_depends=", $(if_enabled_echo CONFIG_UNWINDER_ORC libelf-dev:native)"
extra_build_depends="$extra_build_depends, $(if_enabled_echo CONFIG_SYSTEM_TRUSTED_KEYRING libssl-dev:native)"

View file

@ -19,6 +19,8 @@ else
mkdir -p rpmbuild/SOURCES
cp linux.tar.gz rpmbuild/SOURCES
cp "${KCONFIG_CONFIG}" rpmbuild/SOURCES/config
$(dirname $0)/gen-diff-patch rpmbuild/SOURCES/diff.patch rpmbuild/SOURCES/untracked.patch
touch rpmbuild/SOURCES/diff.patch rpmbuild/SOURCES/untracked.patch
fi
if grep -q CONFIG_MODULES=y include/config/auto.conf; then
@ -53,6 +55,8 @@ sed -e '/^DEL/d' -e 's/^\t*//' <<EOF
URL: https://www.kernel.org
$S Source0: linux.tar.gz
$S Source1: config
$S Source2: diff.patch
$S Source3: untracked.patch
Provides: $PROVIDES
$S BuildRequires: bc binutils bison dwarves
$S BuildRequires: (elfutils-libelf-devel or libelf-devel) flex
@ -90,6 +94,12 @@ $S$M
$S %prep
$S %setup -q -n linux
$S cp %{SOURCE1} .config
$S if [ -s %{SOURCE2} ]; then
$S patch -p1 < %{SOURCE2}
$S fi
$S if [ -s %{SOURCE3} ]; then
$S patch -p1 < %{SOURCE3}
$S fi
$S
$S %build
$S $MAKE %{?_smp_mflags} KERNELRELEASE=$KERNELRELEASE KBUILD_BUILD_VERSION=%{release}

View file

@ -11,10 +11,16 @@
#
usage() {
echo "Usage: $0 [srctree]" >&2
echo "Usage: $0 [--no-local] [srctree]" >&2
exit 1
}
no_local=false
if test "$1" = "--no-local"; then
no_local=true
shift
fi
srctree=.
if test $# -gt 0; then
srctree=$1
@ -26,14 +32,22 @@ fi
scm_version()
{
local short
local short=false
local no_dirty=false
local tag
short=false
while [ $# -gt 0 ];
do
case "$1" in
--short)
short=true;;
--no-dirty)
no_dirty=true;;
esac
shift
done
cd "$srctree"
if test "$1" = "--short"; then
short=true
fi
if test -n "$(git rev-parse --show-cdup 2>/dev/null)"; then
return
@ -75,6 +89,10 @@ scm_version()
printf '%s%s' -g "$(echo $head | cut -c1-12)"
fi
if ${no_dirty}; then
return
fi
# Check for uncommitted changes.
# This script must avoid any write attempt to the source tree, which
# might be read-only.
@ -110,11 +128,6 @@ collect_files()
echo "$res"
}
if ! test -e include/config/auto.conf; then
echo "Error: kernelrelease not valid - run 'make prepare' to update it" >&2
exit 1
fi
if [ -z "${KERNELVERSION}" ]; then
echo "KERNELVERSION is not set" >&2
exit 1
@ -126,6 +139,16 @@ if test ! "$srctree" -ef .; then
file_localversion="${file_localversion}$(collect_files "$srctree"/localversion*)"
fi
if ${no_local}; then
echo "${KERNELVERSION}$(scm_version --no-dirty)"
exit 0
fi
if ! test -e include/config/auto.conf; then
echo "Error: kernelrelease not valid - run 'make prepare' to update it" >&2
exit 1
fi
# version string from CONFIG_LOCALVERSION
config_localversion=$(sed -n 's/^CONFIG_LOCALVERSION=\(.*\)$/\1/p' include/config/auto.conf)