build: refactor JSON info files to `profiles.json`

JSON info files contain machine readable information of built profiles
and resulting images. These files were added in commit 881ed09ee6e2
("build: create JSON files containing image info").

They are useful for firmware wizards and script checking for
reproducibility.

Currently all JSON files are stored next to the built images, resulting
in up to 168 individual files for the ath79/generic target.

This patch refactors the JSON creation to store individual per image
(not per profile) files in $(BUILD_DIR)/json_info_files and create an
single overview file called `profiles.json` in the target directory.

Storing per image files and not per profile solves the problem of
parallel file writes. If a profiles sysupgrade and factory image are
finished at the same time both processes would write to the same JSON
file, resulting in randomly broken outputs.

Some target like x86/64 do not use the image code yet, resulting in
missing JSON files. If no JSON info files were created, no
`profiles.json` files is created as it would be empty anyway.

As before, this creation is enabled by default only if `BUILDBOT` is set.

Tested via buildroot & ImageBuilder on ath79/generic, imx6 and x86/64.

Signed-off-by: Paul Spooren <mail@aparcar.org>
[json_info_files dir handling in Make, if case refactoring]
Signed-off-by: Petr Štetiar <ynezz@true.cz>
(backported from commit 07449f692ce4c4525e946401f4c3ed0cbbc8c4df)
Signed-off-by: Adrian Schmutzler <freifunk@adrianschmutzler.de>

diff --git a/Makefile b/Makefile
index 181c33b180f2016f6e1ce33c96b53791980ce940..73644a8cb999ec529740477791830de50f9f6bfb 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -87,6 +87,14 @@ prereq: $(target/stamp-prereq) tmp/.prereq_packages
                exit 1; \
        fi
 
+$(BIN_DIR)/profiles.json: FORCE
+       $(if $(CONFIG_JSON_OVERVIEW_IMAGE_INFO), \
+               WORK_DIR=$(BUILD_DIR)/json_info_files \
+                       $(SCRIPT_DIR)/json_overview_image_info.py $@ \
+       )
+
+json_overview_image_info: $(BIN_DIR)/profiles.json
+
 checksum: FORCE
        $(call sha256sums,$(BIN_DIR),$(CONFIG_BUILDBOT))
 
@@ -108,6 +116,7 @@ prepare: .config $(tools/stamp-compile) $(toolchain/stamp-compile)
 
 world: prepare $(target/stamp-compile) $(package/stamp-compile) $(package/stamp-install) $(target/stamp-install) FORCE
        $(_SINGLE)$(SUBMAKE) -r package/index
+       $(_SINGLE)$(SUBMAKE) -r json_overview_image_info
        $(_SINGLE)$(SUBMAKE) -r checksum
 
 .PHONY: clean dirclean prereq prepare world package/symlinks package/symlinks-install package/symlinks-clean

diff --git a/config/Config-build.in b/config/Config-build.in
index 9669fc86c781d070de6ccb724dd9e0253aa37027..e5d38578a22ee4550b3f4fa3ec107258a2f58040 100644 (file)
--- a/config/Config-build.in
+++ b/config/Config-build.in
@@ -7,12 +7,13 @@
 
 menu "Global build settings"
 
-       config JSON_ADD_IMAGE_INFO
-               bool "Create JSON info files per build image"
+       config JSON_OVERVIEW_IMAGE_INFO
+               bool "Create JSON info file overview per target"
                default BUILDBOT
                help
-                 The JSON info files contain information about the device and
-                 build images, stored next to the firmware images.
+                 Create a JSON info file called profiles.json in the target
+                 directory containing machine readable list of built profiles
+                 and resulting images.
 
        config ALL_NONSHARED
                bool "Select all target specific packages by default"

diff --git a/include/image.mk b/include/image.mk
index 1e0177c439860179a6a3088dc3c868a5f0bdff13..15f4fe9d3b9473ca6a9a53a4bbff9d729a8fe809 100644 (file)
--- a/include/image.mk
+++ b/include/image.mk
@@ -501,8 +501,11 @@ endef
 
 define Device/Build/image
   GZ_SUFFIX := $(if $(filter %dtb %gz,$(2)),,$(if $(and $(findstring ext4,$(1)),$(CONFIG_TARGET_IMAGES_GZIP)),.gz))
-  $$(_TARGET): $(BIN_DIR)/$(call IMAGE_NAME,$(1),$(2))$$(GZ_SUFFIX)
+  $$(_TARGET): $(if $(CONFIG_JSON_OVERVIEW_IMAGE_INFO), \
+         $(BUILD_DIR)/json_info_files/$(call IMAGE_NAME,$(1),$(2)).json, \
+         $(BIN_DIR)/$(call IMAGE_NAME,$(1),$(2))$$(GZ_SUFFIX))
   $(eval $(call Device/Export,$(KDIR)/tmp/$(call IMAGE_NAME,$(1),$(2)),$(1)))
+
   ROOTFS/$(1)/$(3) := \
        $(KDIR)/root.$(1)$$(strip \
                $$(if $$(FS_OPTIONS/$(1)),+fs=$$(call param_mangle,$$(FS_OPTIONS/$(1)))) \
@@ -524,20 +527,21 @@ define Device/Build/image
 
   $(BIN_DIR)/$(call IMAGE_NAME,$(1),$(2)): $(KDIR)/tmp/$(call IMAGE_NAME,$(1),$(2))
        cp $$^ $$@
-       $(if $(CONFIG_JSON_ADD_IMAGE_INFO), \
-               DEVICE_ID="$(DEVICE_NAME)" \
-               BIN_DIR="$(BIN_DIR)" \
-               IMAGE_NAME="$(IMAGE_NAME)" \
-               IMAGE_TYPE=$(word 1,$(subst ., ,$(2))) \
-               IMAGE_PREFIX="$(IMAGE_PREFIX)" \
-               DEVICE_TITLE="$(DEVICE_TITLE)" \
-               TARGET="$(BOARD)" \
-               SUBTARGET="$(SUBTARGET)" \
-               VERSION_NUMBER="$(VERSION_NUMBER)" \
-               VERSION_CODE="$(VERSION_CODE)" \
-               SUPPORTED_DEVICES="$(SUPPORTED_DEVICES)" \
-               $(TOPDIR)/scripts/json_add_image_info.py \
-       )
+
+  $(BUILD_DIR)/json_info_files/$(call IMAGE_NAME,$(1),$(2)).json: $(BIN_DIR)/$(call IMAGE_NAME,$(1),$(2))$$(GZ_SUFFIX)
+       @mkdir -p $$(shell dirname $$@)
+       DEVICE_ID="$(DEVICE_NAME)" \
+       BIN_DIR="$(BIN_DIR)" \
+       IMAGE_NAME="$(IMAGE_NAME)" \
+       IMAGE_TYPE=$(word 1,$(subst ., ,$(2))) \
+       IMAGE_PREFIX="$(IMAGE_PREFIX)" \
+       DEVICE_TITLE="$(DEVICE_TITLE)" \
+       TARGET="$(BOARD)" \
+       SUBTARGET="$(if $(SUBTARGET),$(SUBTARGET),generic)" \
+       VERSION_NUMBER="$(VERSION_NUMBER)" \
+       VERSION_CODE="$(VERSION_CODE)" \
+       SUPPORTED_DEVICES="$(SUPPORTED_DEVICES)" \
+       $(TOPDIR)/scripts/json_add_image_info.py $$@
 
 endef
 
@@ -556,8 +560,6 @@ define Device/Build/artifact
 endef
 
 define Device/Build
-  $(shell rm -f $(BIN_DIR)/$(IMG_PREFIX)-$(1).json)
-
   $(if $(CONFIG_TARGET_ROOTFS_INITRAMFS),$(call Device/Build/initramfs,$(1)))
   $(call Device/Build/kernel,$(1))
 
@@ -626,6 +628,7 @@ define BuildImage
 
     image_prepare: compile
                mkdir -p $(BIN_DIR) $(KDIR)/tmp
+               rm -rf $(BUILD_DIR)/json_info_files
                $(call Image/Prepare)
 
     legacy-images-prepare-make: image_prepare

diff --git a/scripts/json_add_image_info.py b/scripts/json_add_image_info.py
index 4a90eb95cf8d4f8ff0061249bd972ab6a76839bb..b4d2dd8d71097ad0c92a7b1bd415bc100caa0daa 100755 (executable)
--- a/scripts/json_add_image_info.py
+++ b/scripts/json_add_image_info.py
@@ -1,42 +1,50 @@
 #!/usr/bin/env python3
 
-import json
-import os
+from os import getenv
+from pathlib import Path
+from sys import argv
 import hashlib
+import json
 
+if len(argv) != 2:
+    print("ERROR: JSON info script requires output arg")
+    exit(1)
 
-def e(variable, default=None):
-    return os.environ.get(variable, default)
-
-
-json_path = "{}{}{}.json".format(e("BIN_DIR"), os.sep, e("IMAGE_PREFIX"))
+json_path = Path(argv[1])
+bin_dir = Path(getenv("BIN_DIR"))
+image_file = bin_dir / getenv("IMAGE_NAME")
 
-with open(os.path.join(e("BIN_DIR"), e("IMAGE_NAME")), "rb") as image_file:
-    image_hash = hashlib.sha256(image_file.read()).hexdigest()
+if not image_file.is_file():
+    print("Skip JSON creation for non existing image ", image_file)
+    exit(0)
 
 
 def get_titles():
-    return [{"title": e("DEVICE_TITLE")}]
-
-
-if not os.path.exists(json_path):
-    device_info = {
-        "id": e("DEVICE_ID"),
-        "image_prefix": e("IMAGE_PREFIX"),
-        "images": [],
-        "metadata_version": 1,
-        "supported_devices": e("SUPPORTED_DEVICES").split(),
-        "target": "{}/{}".format(e("TARGET"), e("SUBTARGET", "generic")),
-        "titles": get_titles(),
-        "version_commit": e("VERSION_CODE"),
-        "version_number": e("VERSION_NUMBER"),
-    }
-else:
-    with open(json_path, "r") as json_file:
-        device_info = json.load(json_file)
-
-image_info = {"type": e("IMAGE_TYPE"), "name": e("IMAGE_NAME"), "sha256": image_hash}
-device_info["images"].append(image_info)
-
-with open(json_path, "w") as json_file:
-    json.dump(device_info, json_file, sort_keys=True, indent="  ")
+    return [{"title": getenv("DEVICE_TITLE")}]
+
+
+device_id = getenv("DEVICE_ID")
+image_hash = hashlib.sha256(image_file.read_bytes()).hexdigest()
+
+image_info = {
+    "metadata_version": 1,
+    "target": "{}/{}".format(getenv("TARGET"), getenv("SUBTARGET")),
+    "version_code": getenv("VERSION_CODE"),
+    "version_number": getenv("VERSION_NUMBER"),
+    "profiles": {
+        device_id: {
+            "image_prefix": getenv("IMAGE_PREFIX"),
+            "images": [
+                {
+                    "type": getenv("IMAGE_TYPE"),
+                    "name": getenv("IMAGE_NAME"),
+                    "sha256": image_hash,
+                }
+            ],
+            "supported_devices": getenv("SUPPORTED_DEVICES").split(),
+            "titles": get_titles(),
+        }
+    },
+}
+
+json_path.write_text(json.dumps(image_info, separators=(",", ":")))

diff --git a/scripts/json_overview_image_info.py b/scripts/json_overview_image_info.py
new file mode 100755 (executable)
index 0000000..5ed8292
--- /dev/null
+++ b/scripts/json_overview_image_info.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+
+import json
+from pathlib import Path
+from os import getenv
+from sys import argv
+
+if len(argv) != 2:
+    print("JSON info files script requires ouput file as argument")
+    exit(1)
+
+output_path = Path(argv[1])
+
+assert getenv("WORK_DIR"), "$WORK_DIR required"
+
+work_dir = Path(getenv("WORK_DIR"))
+
+assert work_dir.is_dir(), "$WORK_DIR not a directory"
+
+output = {}
+
+for json_file in work_dir.glob("*.json"):
+    image_info = json.loads(json_file.read_text())
+    if not output:
+        output.update(image_info)
+    else:
+        # get first (and only) profile in json file
+        device_id = next(iter(image_info["profiles"].keys()))
+        if device_id not in output["profiles"]:
+            output["profiles"].update(image_info["profiles"])
+        else:
+            output["profiles"][device_id]["images"].append(
+                image_info["profiles"][device_id]["images"][0]
+            )
+
+if output:
+    output_path.write_text(json.dumps(output, sort_keys=True, separators=(",", ":")))
+else:
+    print("JSON info file script could not find any JSON files for target")

diff --git a/target/imagebuilder/files/Makefile b/target/imagebuilder/files/Makefile
index 15b3d5c35c0665dff4c92673ca00307fa546871d..835dd98ca5fe9f876247d3bf43c02459e640dcd1 100644 (file)
--- a/target/imagebuilder/files/Makefile
+++ b/target/imagebuilder/files/Makefile
@@ -118,6 +118,7 @@ _call_image: staging_dir/host/.prereq-build
        $(MAKE) package_install
        $(MAKE) -s prepare_rootfs
        $(MAKE) -s build_image
+       $(MAKE) -s json_overview_image_info
        $(MAKE) -s checksum
 
 _call_manifest: FORCE
@@ -163,12 +164,21 @@ prepare_rootfs: FORCE
        $(CP) $(TARGET_DIR) $(TARGET_DIR_ORIG)
        $(call prepare_rootfs,$(TARGET_DIR),$(USER_FILES),$(DISABLED_SERVICES))
 
+
 build_image: FORCE
        @echo
        @echo Building images...
        $(NO_TRACE_MAKE) -C target/linux/$(BOARD)/image install TARGET_BUILD=1 IB=1 EXTRA_IMAGE_NAME="$(EXTRA_IMAGE_NAME)" \
                $(if $(USER_PROFILE),PROFILE="$(USER_PROFILE)")
 
+$(BIN_DIR)/profiles.json: FORCE
+       $(if $(CONFIG_JSON_OVERVIEW_IMAGE_INFO), \
+               WORK_DIR=$(BUILD_DIR)/json_info_files \
+                       $(SCRIPT_DIR)/json_overview_image_info.py $@ \
+       )
+
+json_overview_image_info: $(BIN_DIR)/profiles.json
+
 checksum: FORCE
        @echo
        @echo Calculating checksums...