From d419136ad5bfa0449083b1a07a9a265cea20dcd5 Mon Sep 17 00:00:00 2001 From: Christian Marangi Date: Tue, 6 Sep 2022 16:48:37 +0200 Subject: [PATCH] build: make find_md5 reproducible with AUTOREMOVE While experimenting with the AUTOREMOVE option in search of a way to use prebuilt host tools in different buildroot, it was discovered that the md5 generated by find_md5 in depends.mk is not reproducible. Currently the hash is generated by the path of the file in addition to the file mod time. Out of confusion, probably, there was an idea that such command was used on the package build_dir. Reality is that this command is run on the package files. (Makefile, patches, src) This is problematic because the package Makefile (for example) change at each git clone and base the hash on the Makefile mtime doesn't really reflect if the Makefile actually changes across a buildroot or not. A better approach is to generate an hash of each file and then generate an hash on the sort hash list. This way we remove the problem of git clone setting a wrong mtime while keeping the integrity of checking if a file changed for the package as any change will result in a different hash. Introduce a new kind of find_md5 function, find_md5_reproducible that apply this new logic and limit it only with AUTOREMOVE option set to prevent any kind of slowdown due to additional hash generation. Signed-off-by: Christian Marangi (cherry picked from commit 53a08e37437972ba0a8fbf953a93a70a6b784ef4) --- include/depends.mk | 1 + include/host-build.mk | 2 +- include/kernel-build.mk | 2 +- include/package.mk | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/include/depends.mk b/include/depends.mk index 465e4e952a..8ab80daf39 100644 --- a/include/depends.mk +++ b/include/depends.mk @@ -12,6 +12,7 @@ DEP_FINDPARAMS := -x "*/.svn*" -x ".*" -x "*:*" -x "*\!*" -x "* *" -x "*\\\#*" -x "*/.*_check" -x "*/.*.swp" -x "*/.pkgdir*" find_md5=find $(wildcard $(1)) -type f $(patsubst -x,-and -not -path,$(DEP_FINDPARAMS) $(2)) -printf "%p%T@\n" | sort | $(MKHASH) md5 +find_md5_reproducible=find $(wildcard $(1)) -type f $(patsubst -x,-and -not -path,$(DEP_FINDPARAMS) $(2)) -print0 | xargs -0 $(MKHASH) md5 | sort | $(MKHASH) md5 define rdep .PRECIOUS: $(2) diff --git a/include/host-build.mk b/include/host-build.mk index 7f76eb1a2f..c6f594be35 100644 --- a/include/host-build.mk +++ b/include/host-build.mk @@ -21,7 +21,7 @@ include $(INCLUDE_DIR)/depends.mk include $(INCLUDE_DIR)/quilt.mk BUILD_TYPES += host -HOST_STAMP_PREPARED=$(HOST_BUILD_DIR)/.prepared$(if $(HOST_QUILT)$(DUMP),,$(shell $(call find_md5,${CURDIR} $(PKG_FILE_DEPENDS),))_$(call confvar,CONFIG_AUTOREMOVE $(HOST_PREPARED_DEPENDS))) +HOST_STAMP_PREPARED=$(HOST_BUILD_DIR)/.prepared$(if $(HOST_QUILT)$(DUMP),,$(shell $(call $(if $(CONFIG_AUTOREMOVE),find_md5_reproducible,find_md5),${CURDIR} $(PKG_FILE_DEPENDS),))_$(call confvar,CONFIG_AUTOREMOVE $(HOST_PREPARED_DEPENDS))) HOST_STAMP_CONFIGURED:=$(HOST_BUILD_DIR)/.configured HOST_STAMP_BUILT:=$(HOST_BUILD_DIR)/.built HOST_BUILD_PREFIX?=$(if $(IS_PACKAGE_BUILD),$(STAGING_DIR_HOSTPKG),$(STAGING_DIR_HOST)) diff --git a/include/kernel-build.mk b/include/kernel-build.mk index 80da4455bc..5dd033cda6 100644 --- a/include/kernel-build.mk +++ b/include/kernel-build.mk @@ -10,7 +10,7 @@ ifneq ($(DUMP),1) endif KERNEL_FILE_DEPENDS=$(GENERIC_BACKPORT_DIR) $(GENERIC_PATCH_DIR) $(GENERIC_HACK_DIR) $(PATCH_DIR) $(GENERIC_FILES_DIR) $(FILES_DIR) -STAMP_PREPARED=$(LINUX_DIR)/.prepared$(if $(QUILT)$(DUMP),,_$(shell $(call find_md5,$(KERNEL_FILE_DEPENDS),))) +STAMP_PREPARED=$(LINUX_DIR)/.prepared$(if $(QUILT)$(DUMP),,_$(shell $(call $(if $(CONFIG_AUTOREMOVE),find_md5_reproducible,find_md5),$(KERNEL_FILE_DEPENDS),))) STAMP_CONFIGURED:=$(LINUX_DIR)/.configured include $(INCLUDE_DIR)/download.mk include $(INCLUDE_DIR)/quilt.mk diff --git a/include/package.mk b/include/package.mk index 509ef61e08..4220f624ff 100644 --- a/include/package.mk +++ b/include/package.mk @@ -85,7 +85,7 @@ ifneq ($(PREV_STAMP_PREPARED),) STAMP_PREPARED:=$(PREV_STAMP_PREPARED) CONFIG_AUTOREBUILD:= else - STAMP_PREPARED=$(PKG_BUILD_DIR)/.prepared$(if $(QUILT)$(DUMP),,_$(shell $(call find_md5,${CURDIR} $(PKG_FILE_DEPENDS),))_$(call confvar,CONFIG_AUTOREMOVE $(PKG_PREPARED_DEPENDS))) + STAMP_PREPARED=$(PKG_BUILD_DIR)/.prepared$(if $(QUILT)$(DUMP),,_$(shell $(call $(if $(CONFIG_AUTOREMOVE),find_md5_reproducible,find_md5),${CURDIR} $(PKG_FILE_DEPENDS),))_$(call confvar,CONFIG_AUTOREMOVE $(PKG_PREPARED_DEPENDS))) endif STAMP_CONFIGURED=$(PKG_BUILD_DIR)/.configured$(if $(DUMP),,_$(call confvar,$(PKG_CONFIG_DEPENDS))) STAMP_CONFIGURED_WILDCARD=$(PKG_BUILD_DIR)/.configured_* -- 2.30.2