package: rework contents of package index.json

Rework the generation of the index.json version of the package
indexes to match the original intent (i.e., for use by the ASU
server and other downstream projects).  The current file contains
package names that have ABI versioning, making them unusable by ASU,
so we now remove the ABI suffixes.

Also adds a 'version' field to the json, so downstream utilities
can detect the new semantics of the package name fields.

Links: https://github.com/openwrt/openwrt/commit/218ce40cd738f3373438aab82467807a8707fb9c
Signed-off-by: Eric Fahlgren <ericfahlgren@gmail.com>
Link: https://github.com/openwrt/openwrt/pull/19051
Signed-off-by: Robert Marko <robimarko@gmail.com>

2 files changed, 87 insertions, 9 deletions

diff --git a/package/Makefile b/package/Makefile
index 701ed3b13b..fef68c5a1d 100644
--- a/package/Makefile
+++ b/package/Makefile
@@ -136,11 +136,8 @@ ifneq ($(CONFIG_USE_APK),)
 			--sign $(BUILD_KEY_APK_SEC) \
 			--output packages.adb \
 			*.apk; \
-		echo -n '{"architecture": "$(ARCH_PACKAGES)", "packages":{' > index.json; \
-		$(STAGING_DIR_HOST)/bin/apk adbdump packages.adb | \
-			awk '/- name: / {pkg = $$NF} ; / version: / {printf "\"%s\": \"%s\", ", pkg, $$NF}' | \
-			sed 's/, $$//' >> index.json; \
-		echo '}}' >> index.json; \
+		$(STAGING_DIR_HOST)/bin/apk adbdump --format json packages.adb | \
+			$(SCRIPT_DIR)/make-index-json.py -f apk -a "$(ARCH_PACKAGES)" - > index.json; \
 	done
 else
 	@for d in $(PACKAGE_SUBDIRS); do ( \
@@ -152,10 +149,7 @@ else
 			$(call ERROR_MESSAGE,WARNING: Applying padding in $$d/Packages to workaround usign SHA-512 bug!); \
 			{ echo ""; echo ""; } >> Packages;; \
 		esac; \
-		echo -n '{"architecture": "$(ARCH_PACKAGES)", "packages":{' > index.json; \
-		sed -n -e 's/^Package: \(.*\)$$/"\1":/p' -e 's/^Version: \(.*\)$$/"\1",/p' Packages | tr '\n' ' ' >> index.json; \
-		echo '}}' >> index.json; \
-		sed -i 's/, }}/}}/' index.json; \
+		$(SCRIPT_DIR)/make-index-json.py -f opkg -a "$(ARCH_PACKAGES)" Packages > index.json; \
 		gzip -9nc Packages > Packages.gz; \
 	); done
 ifdef CONFIG_SIGNED_PACKAGES
diff --git a/scripts/make-index-json.py b/scripts/make-index-json.py
new file mode 100755
index 0000000000..af5c6cf073
--- /dev/null
+++ b/scripts/make-index-json.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+"""
+Parse the native package index files into a json file for use by downstream
+tools.  See:
+
+    https://github.com/openwrt/openwrt/commit/218ce40cd738f3373438aab82467807a8707fb9c
+
+The "version 1" index.json contained ABI-versioned package names, making the
+unusable by the ASU server.  The version 2 format contains package names that
+have been stripped of their ABI version.
+"""
+
+import email.parser
+import json
+
+
+def parse_args():
+    from argparse import ArgumentParser
+
+    source_format = "apk", "opkg"
+
+    parser = ArgumentParser()
+    # fmt: off
+    parser.add_argument("-a", "--architecture", required=True,
+                        help="Required device architecture: like 'x86_64' or 'aarch64_generic'")
+    parser.add_argument("-f", "--source-format", required=True, choices=source_format,
+                        help="Required source format of input: 'apk' or 'opkg'")
+    parser.add_argument(dest="source",
+                        help="File name for input, '-' for stdin")
+    # fmt: on
+    args = parser.parse_args()
+    return args
+
+
+def parse_apk(text: str) -> dict:
+    packages: dict = {}
+
+    data = json.loads(text)
+    for package in data.get("packages", []):
+        package_name: str = package["name"]
+
+        for tag in package.get("tags", []):
+            if tag.startswith("openwrt:abiversion="):
+                package_abi: str = tag.split("=")[-1]
+                package_name = package_name.removesuffix(package_abi)
+                break
+
+        packages[package_name] = package["version"]
+
+    return packages
+
+
+def parse_opkg(text: str) -> dict:
+    packages: dict = {}
+
+    parser: email.parser.Parser = email.parser.Parser()
+    chunks: list[str] = text.strip().split("\n\n")
+    for chunk in chunks:
+        package: dict = parser.parsestr(chunk, headersonly=True)
+        package_name: str = package["Package"]
+        if package_abi := package.get("ABIVersion"):
+            package_name = package_name.removesuffix(package_abi)
+
+        packages[package_name] = package["Version"]
+
+    return packages
+
+
+if __name__ == "__main__":
+    import sys
+
+    args = parse_args()
+
+    input = sys.stdin if args.source == "-" else open(args.source, "r")
+    with input:
+        text: str = input.read()
+
+    packages = parse_apk(text) if args.source_format == "apk" else parse_opkg(text)
+    index = {
+        "version": 2,
+        "architecture": args.architecture,
+        "packages": packages,
+    }
+    print(json.dumps(index, indent=2))