Parallel build

Makefile

@@ -1,6 +1,12 @@
 all:
 -include .config
 
+ifneq "$(TOOLCHAIN)" ""
+$(info Using $(TOOLCHAIN) for cross compiler and packages)
+packages	:= $(TOOLCHAIN)/packages
+CROSS		:= $(TOOLCHAIN)/crossgcc/x86_64-linux-musl/bin/x86_64-linux-musl-
+endif
+
 modules-y 	:=
 pwd 		:= $(shell pwd)
 packages 	?= $(pwd)/packages
@@ -17,10 +23,11 @@ CPUS		:= $(shell nproc)
 BUILD_LOG := $(shell mkdir -p "$(log_dir)" )
 
 # Check that we have a correct version of make
+# that matches at least the major version
 LOCAL_MAKE_VERSION := $(shell $(MAKE) --version | head -1 | cut -d' ' -f3)
 include modules/make
 
-ifeq "$(LOCAL_MAKE_VERSION)" "$(make_version)"
+ifneq "" "$(filter $(make_version)%,$(LOCAL_MAKE_VERSION))"
 
 # This is the correct version of Make
 
@@ -65,9 +72,11 @@ endif
 
 
 # Create a temporary directory for the initrd
-initrd_dir	:= $(shell mktemp -d)
-initrd_lib_dir	:= $(initrd_dir)/lib
-initrd_bin_dir	:= $(initrd_dir)/bin
+initrd_dir	:= $(BOARD)
+initrd_tmp_dir	:= $(shell mktemp -d)
+initrd_lib_dir	:= $(initrd_tmp_dir)/lib
+initrd_bin_dir	:= $(initrd_tmp_dir)/bin
+modules-y += initrd
 
 $(shell mkdir -p "$(initrd_lib_dir)" "$(initrd_bin_dir)")
 
@@ -112,11 +121,8 @@ else
 $(error "$(BOARD): neither CONFIG_COREBOOT nor CONFIG_LINUXBOOT is set?")
 endif
 
-# helpful targets for common uses
-linux: $(build)/$(BOARD)/bzImage
-cpio: $(build)/$(BOARD)/initrd.cpio.xz
-
 # Disable all built in rules
+.INTERMEDIATE:
 .SUFFIXES:
 FORCE:
 
@@ -133,10 +139,6 @@ endef
 # as part of creating the Heads firmware image.
 include modules/*
 
-# These will be built via their intermediate targets
-# This increases the build time, so it is commented out for now
-#all: $(foreach m,$(modules-y),$m.intermediate)
-
 define bins =
 $(foreach m,$1,$(call prefix,$(build)/$($m_dir)/,$($m_output)))
 endef
@@ -155,7 +157,7 @@ endef
 # Build a cpio from a directory
 #
 define do-cpio =
-	$(call do,CPIO,$1,\
+	$(call do,CPIO     ,$1,\
 		( cd "$2"; \
 		find . \
 		| cpio \
@@ -169,17 +171,20 @@ define do-cpio =
 	@if ! cmp --quiet "$1.tmp" "$1" ; then \
 		mv "$1.tmp" "$1" ; \
 	else \
+		echo "$(DATE) UNCHANGED $(1:$(pwd)/%=%)" ; \
 		rm "$1.tmp" ; \
 	fi
 endef
 
 define do-copy =
-	$(call do,COPY,$1 => $2',\
-		sha256sum "$(1:$(pwd)/%=%)" ; \
+	$(call do,INSTALL  ,$1 => $2',\
 		if ! cmp --quiet "$1" "$2" ; then \
 			cp -a "$1" "$2"; \
+		else  \
+			echo "$(DATE) UNCHANGED $(1:$(pwd)/%=%)" ; \
 		fi
 	)
+	@sha256sum "$(2:$(pwd)/%=%)"
 endef
 
 
@@ -190,21 +195,25 @@ endef
 # expansion during the first evaluation.
 #
 define define_module =
+  # if they have not defined a separate base dir, define it
+  # as the same as their build dir.
+  $(eval $1_base_dir = $(or $($1_base_dir),$($1_dir)))
+
   ifneq ("$($1_repo)","")
     # Checkout the tree instead and touch the canary file so that we know
     # that the files are all present. No signature hashes are checked in
     # this case, since we don't have a stable version to compare against.
-    $(build)/$($1_dir)/.canary:
-	git clone $($1_repo) "$(build)/$($1_dir)"
+    $(build)/$($1_base_dir)/.canary:
+	git clone $($1_repo) "$(build)/$($1_base_dir)"
 	if [ -r patches/$1.patch ]; then \
-		( cd $(build)/$($1_dir) ; patch -p1 ) \
+		( cd $(build)/$($1_base_dir) ; patch -p1 ) \
 			< patches/$1.patch; \
 	fi
 	if [ -d patches/$1 ] && \
 	   [ -r patches/$1 ] ; then \
 		for patch in patches/$1/*.patch ; do \
 			echo "Applying patch file : $$$$patch " ;  \
-			( cd $(build)/$($1_dir) ; patch -p1 ) \
+			( cd $(build)/$($1_base_dir) ; patch -p1 ) \
 				< $$$$patch ; \
 		done ; \
 	fi
@@ -219,17 +228,17 @@ define define_module =
 
     # Unpack the tar file and touch the canary so that we know
     # that the files are all present
-    $(build)/$($1_dir)/.canary: $(packages)/.$1-$($1_version)_verify
+    $(build)/$($1_base_dir)/.canary: $(packages)/.$1-$($1_version)_verify
 	tar -xf "$(packages)/$($1_tar)" -C "$(build)"
 	if [ -r patches/$1-$($1_version).patch ]; then \
-		( cd $(build)/$($1_dir) ; patch -p1 ) \
+		( cd $(build)/$($1_base_dir) ; patch -p1 ) \
 			< patches/$1-$($1_version).patch; \
 	fi
 	if [ -d patches/$1-$($1_version) ] && \
 	   [ -r patches/$1-$($1_version) ] ; then \
 		for patch in patches/$1-$($1_version)/*.patch ; do \
 			echo "Applying patch file : $$$$patch " ;  \
-			( cd $(build)/$($1_dir) ; patch -p1 ) \
+			( cd $(build)/$($1_base_dir) ; patch -p1 ) \
 				< $$$$patch ; \
 		done ; \
 	fi
@@ -242,21 +251,29 @@ define define_module =
 
   ifeq "$($1_config)" ""
     # There is no official .config file
-    $($1_config_file_path): $(build)/$($1_dir)/.canary
+    $($1_config_file_path): $(build)/$($1_base_dir)/.canary
 	@mkdir -p $$(dir $$@)
 	@touch "$$@"
   else
     # Copy the stored config file into the unpacked directory
-    $($1_config_file_path): $($1_config) $(build)/$($1_dir)/.canary
+    $($1_config_file_path): $($1_config) $(build)/$($1_base_dir)/.canary
 	@mkdir -p $$(dir $$@)
 	$(call do-copy,$($1_config),$$@)
   endif
 
+  # The first time we have to wait for all the dependencies to be built
+  # before we can configure the target. Once the dep has been built,
+  # we only depend on it for a rebuild.
+  $(eval $1_config_wait := $(foreach d,$($1_depends),\
+	$(shell [ -r $(build)/$($d_dir)/.build ] || echo $d)))
+
   # Use the module's configure variable to build itself
+  # this has to wait for the dependencies to be built since
+  # cross compilers and libraries might be messed up
   $(dir $($1_config_file_path)).configured: \
-		$(build)/$($1_dir)/.canary \
+		$(build)/$($1_base_dir)/.canary \
+		$(foreach d,$($1_config_wait),$(build)/$($d_dir)/.build) \
 		$($1_config_file_path) \
-		$(foreach d,$($1_depends),$(call outputs,$d)) \
 		modules/$1
 	@echo "$(DATE) CONFIG $1"
 	@( \
@@ -270,19 +287,26 @@ define define_module =
 		$(VERBOSE_REDIRECT)
 	@touch "$$@"
 
-  # All of the outputs should result from building the intermediate target
-  $(call outputs,$1): $1.intermediate
-
-  # Short hand target for the module
-  #$1: $(call outputs,$1)
+  # Short hand for our module build target
+  $1: \
+	$(build)/$($1_dir)/.build \
+	$(call outputs,$1) \
 
   # Target for all of the outputs, which depend on their dependent modules
-  $1.intermediate: \
-		$(foreach d,$($1_depends),$d.intermediate) \
-		$(foreach d,$($1_depends),$(call outputs,$d)) \
+  # being built, as well as this module being configured
+  $(call outputs,$1): $(build)/$($1_dir)/.build
+
+  # If any of the outputs are missing, we should force a rebuild
+  # of the entire module
+  $(eval $1.force = $(shell \
+	stat $(call outputs,$1) >/dev/null 2>/dev/null || echo FORCE \
+  ))
+
+  $(build)/$($1_dir)/.build: $($1.force) \
+		$(foreach d,$($1_depends),$(build)/$($d_dir)/.build) \
 		$(dir $($1_config_file_path)).configured \
 
-	@echo "$(DATE) MAKE $1"
+	@echo "$(DATE) MAKE $1 --- $$@"
 	+@( \
 		echo "$(MAKE) \
 			-C \"$(build)/$($1_dir)\" \
@@ -301,7 +325,11 @@ define define_module =
 		tail -20 "$(log_dir)/$1.log"; \
 		exit 1; \
 	)
-	@echo "$(DATE) DONE $1"
+	$(call do,DONE,$1,\
+		touch $(build)/$($1_dir)/.build \
+	)
+
+
 
   $1.clean:
 	-$(RM) "$(build)/$($1_dir)/.configured"
@@ -312,7 +340,7 @@ endef
 $(call map, define_module, $(modules-y))
 
 # hack to force musl-cross to be built before musl
-#$(build)/$(musl_dir)/.configured: $(build)/$(musl-cross_dir)/../../crossgcc/x86_64-linux-musl/bin/x86_64-linux-musl-gcc
+#$(build)/$(musl_dir)/.configured: $(build)/$(musl-cross_dir)/../../crossgcc/x86_64-linux-musl/bin/x86_64-musl-linux-gcc
 
 #
 # Install a file into the initrd, if it changed from
@@ -366,27 +394,12 @@ $(foreach m, $(modules-y), \
 	$(call map,initrd_lib_add,$(call libs,$m)) \
 )
 
-# hack to install busybox into the initrd if busybox is configured
-ifeq "$(CONFIG_BUSYBOX)" "y"
-$(build)/$(BOARD)/heads.cpio: busybox.intermediate
-initrd_bins += $(initrd_bin_dir)/busybox
-endif
-
-$(initrd_bin_dir)/busybox: $(build)/$(busybox_dir)/busybox
-	$(do,SYMLINK,$@,$(MAKE) \
-		-C $(build)/$(busybox_dir) \
-		CC="$(heads_cc)" \
-		CONFIG_PREFIX="$(pwd)/initrd" \
-		$(MAKE_JOBS) \
-		install \
-	)
-
 #
 # hack to build cbmem from coreboot
 # this must be built *AFTER* musl, but since coreboot depends on other things
 # that depend on musl it should be ok.
 #
-COREBOOT_UTIL_DIR=$(build)/$(coreboot_dir)/util
+COREBOOT_UTIL_DIR=$(build)/$(coreboot_base_dir)/util
 ifeq ($(CONFIG_COREBOOT),y)
 $(eval $(call initrd_bin_add,$(COREBOOT_UTIL_DIR)/cbmem/cbmem))
 #$(eval $(call initrd_bin_add,$(COREBOOT_UTIL_DIR)/superiotool/superiotool))
@@ -396,14 +409,16 @@ endif
 $(COREBOOT_UTIL_DIR)/cbmem/cbmem \
 $(COREBOOT_UTIL_DIR)/superiotool/superiotool \
 $(COREBOOT_UTIL_DIR)/inteltool/inteltool \
-: $(build)/$(coreboot_dir)/.canary \
-	musl.intermediate
+: $(build)/$(coreboot_base_dir)/.canary \
+	$(build)/$(musl_dir)/.build
 	+$(call do,MAKE,$(notdir $@),\
 		$(MAKE) -C "$(dir $@)" $(CROSS_TOOLS) \
 	)
 
 # superio depends on zlib and pciutils
-$(COREBOOT_UTIL_DIR)/superiotool/superiotool: zlib.intermediate pciutils.intermediate
+$(COREBOOT_UTIL_DIR)/superiotool/superiotool: \
+	$(build)/$(zlib_dir)/.build \
+	$(build)/$(pciutils_dir)/.build \
 
 #
 # initrd image creation
@@ -423,57 +438,61 @@ $(COREBOOT_UTIL_DIR)/superiotool/superiotool: zlib.intermediate pciutils.interme
 #
 
 initrd-y += $(pwd)/blobs/dev.cpio
-initrd-y += $(build)/$(BOARD)/modules.cpio
-initrd-y += $(build)/$(BOARD)/tools.cpio
-initrd-$(CONFIG_HEADS) += $(build)/$(BOARD)/heads.cpio
+initrd-y += $(build)/$(initrd_dir)/modules.cpio
+initrd-y += $(build)/$(initrd_dir)/tools.cpio
+initrd-$(CONFIG_HEADS) += $(build)/$(initrd_dir)/heads.cpio
+
+#$(build)/$(initrd_dir)/.build: $(build)/$(initrd_dir)/initrd.cpio.xz
 
-initrd.intermediate: $(build)/$(BOARD)/initrd.cpio.xz
-$(build)/$(BOARD)/initrd.cpio.xz: $(initrd-y)
-	$(call do,CPIO-CLEAN,$@,\
+$(build)/$(initrd_dir)/initrd.cpio.xz: $(initrd-y)
+	$(call do,CPIO-XZ  ,$@,\
 	$(pwd)/bin/cpio-clean \
 		$^ \
 	| xz \
 		--check=crc32 \
 		--lzma2=dict=1MiB \
 		-9 \
-	| dd bs=512 conv=sync > "$@" \
+	| dd bs=512 conv=sync status=none > "$@.tmp" \
 	)
-	@sha256sum "$(@:$(pwd)/%=%)"
+	@if ! cmp --quiet "$@.tmp" "$@" ; then \
+		mv "$@.tmp" "$@" ; \
+		sha256sum "$(@:$(pwd)/%=%)" ; \
+	else \
+		echo "$(DATE) UNCHANGED $(@:$(pwd)/%=%)" ; \
+		rm "$@.tmp" ; \
+	fi
 
 #
 # The heads.cpio is built from the initrd directory in the
 # Heads tree.
 #
-$(build)/$(BOARD)/heads.cpio: FORCE
+$(build)/$(initrd_dir)/heads.cpio: FORCE
 	$(call do-cpio,$@,$(pwd)/initrd)
 
 
 #
 # The tools initrd is made from all of the things that we've
 # created during the submodule build.
 #
-$(build)/$(BOARD)/tools.cpio: \
+$(build)/$(initrd_dir)/tools.cpio: \
 	$(initrd_bins) \
 	$(initrd_libs) \
 
 	$(call do,INSTALL,$(CONFIG), \
-		mkdir -p "$(initrd_dir)/etc" ; \
+		mkdir -p "$(initrd_tmp_dir)/etc" ; \
 		export \
 			| grep ' CONFIG_' \
 			| sed -e 's/^declare -x /export /' \
 			-e 's/\\\"//g' \
-			> "$(initrd_dir)/etc/config" \
+			> "$(initrd_tmp_dir)/etc/config" \
 	)
-	$(call do-cpio,$@,$(initrd_dir))
-	@$(RM) -rf "$(initrd_dir)"
-
+	$(call do-cpio,$@,$(initrd_tmp_dir))
+	@$(RM) -rf "$(initrd_tmp_dir)"
 
+# Ensure that the initrd depends on all of the modules that produce
+# binaries for it
+$(build)/$(initrd_dir)/tools.cpio: $(foreach d,$(bin_modules-y),$(build)/$($d_dir)/.build)
 
-# This produces a ROM image that is written with the flashrom program
-$(build)/$(BOARD)/coreboot.rom: $(build)/$(coreboot_dir)/$(BOARD)/coreboot.rom
-	"$(build)/$(coreboot_dir)/$(BOARD)/cbfstool" "$<" print
-	$(call do,EXTRACT,$@,mv "$<" "$@")
-	@sha256sum "$(@:$(pwd)/%=%)"
 
 # List of all modules, excluding the slow to-build modules
 modules-slow := musl musl-cross kernel_headers
@@ -501,10 +520,6 @@ real.clean:
 	done
 	rm -rf ./install
 
-bootstrap:
-	+$(MAKE) \
-		musl-cross.intermediate \
-		$(build)/$(coreboot_dir)/util/crossgcc/xgcc/bin/i386-elf-gcc \
 
 else
 # Wrong make version detected -- build our local version
@@ -515,7 +530,7 @@ HEADS_MAKE := $(build)/$(make_dir)/make
 # Once we have a proper Make, we can just pass arguments into it
 all bootstrap linux cpio: $(HEADS_MAKE)
 	LANG=C MAKE=$(HEADS_MAKE) $(HEADS_MAKE) $(MAKE_JOBS) $@
-%.clean %.intermediate %.vol: $(HEADS_MAKE)
+%.clean %.vol: $(HEADS_MAKE)
 	LANG=C MAKE=$(HEADS_MAKE) $(HEADS_MAKE) $@
 
 # How to download and build the correct version of make

boards/qemu-linuxboot/qemu-linuxboot.config

@@ -16,7 +16,7 @@ CONFIG_QRENCODE=y
 CONFIG_TPMTOTP=y
 endif
 
-#CONFIG_FLASHROM=y
+CONFIG_FLASHROM=y
 CONFIG_FLASHTOOLS=y
 CONFIG_GPG=y
 CONFIG_KEXEC=y