package: rework contents of package index.json

Rework the generation of the index.json version of the package
indexes to match the original intent (i.e., for use by the ASU
server and other downstream projects).  The current file contains
package names that have ABI versioning, making them unusable by ASU,
so we now remove the ABI suffixes.

Also adds a 'version' field to the json, so downstream utilities
can detect the new semantics of the package name fields.

Links: https://github.com/openwrt/openwrt/commit/218ce40cd738f3373438aab82467807a8707fb9c
Signed-off-by: Eric Fahlgren <ericfahlgren@gmail.com>
Link: https://github.com/openwrt/openwrt/pull/19051
(cherry picked from commit 2b0b16f1d1571b23425b2d7ab5dc3816e2ceec12)
Link: https://github.com/openwrt/openwrt/pull/19097
Signed-off-by: Robert Marko <robimarko@gmail.com>

2 files changed, 85 insertions, 4 deletions

diff --git a/package/Makefile b/package/Makefile
index abe76a619f..2343777195 100644
--- a/package/Makefile
+++ b/package/Makefile
@@ -142,10 +142,7 @@ else
 			$(call ERROR_MESSAGE,WARNING: Applying padding in $$d/Packages to workaround usign SHA-512 bug!); \
 			{ echo ""; echo ""; } >> Packages;; \
 		esac; \
-		echo -n '{"architecture": "$(ARCH_PACKAGES)", "packages":{' > index.json; \
-		sed -n -e 's/^Package: \(.*\)$$/"\1":/p' -e 's/^Version: \(.*\)$$/"\1",/p' Packages | tr '\n' ' ' >> index.json; \
-		echo '}}' >> index.json; \
-		sed -i 's/, }}/}}/' index.json; \
+		$(SCRIPT_DIR)/make-index-json.py -f opkg -a "$(ARCH_PACKAGES)" Packages > index.json; \
 		gzip -9nc Packages > Packages.gz; \
 	); done
 ifdef CONFIG_SIGNED_PACKAGES
diff --git a/scripts/make-index-json.py b/scripts/make-index-json.py
new file mode 100755
index 0000000000..af5c6cf073
--- /dev/null
+++ b/scripts/make-index-json.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+"""
+Parse the native package index files into a json file for use by downstream
+tools.  See:
+
+    https://github.com/openwrt/openwrt/commit/218ce40cd738f3373438aab82467807a8707fb9c
+
+The "version 1" index.json contained ABI-versioned package names, making the
+unusable by the ASU server.  The version 2 format contains package names that
+have been stripped of their ABI version.
+"""
+
+import email.parser
+import json
+
+
+def parse_args():
+    from argparse import ArgumentParser
+
+    source_format = "apk", "opkg"
+
+    parser = ArgumentParser()
+    # fmt: off
+    parser.add_argument("-a", "--architecture", required=True,
+                        help="Required device architecture: like 'x86_64' or 'aarch64_generic'")
+    parser.add_argument("-f", "--source-format", required=True, choices=source_format,
+                        help="Required source format of input: 'apk' or 'opkg'")
+    parser.add_argument(dest="source",
+                        help="File name for input, '-' for stdin")
+    # fmt: on
+    args = parser.parse_args()
+    return args
+
+
+def parse_apk(text: str) -> dict:
+    packages: dict = {}
+
+    data = json.loads(text)
+    for package in data.get("packages", []):
+        package_name: str = package["name"]
+
+        for tag in package.get("tags", []):
+            if tag.startswith("openwrt:abiversion="):
+                package_abi: str = tag.split("=")[-1]
+                package_name = package_name.removesuffix(package_abi)
+                break
+
+        packages[package_name] = package["version"]
+
+    return packages
+
+
+def parse_opkg(text: str) -> dict:
+    packages: dict = {}
+
+    parser: email.parser.Parser = email.parser.Parser()
+    chunks: list[str] = text.strip().split("\n\n")
+    for chunk in chunks:
+        package: dict = parser.parsestr(chunk, headersonly=True)
+        package_name: str = package["Package"]
+        if package_abi := package.get("ABIVersion"):
+            package_name = package_name.removesuffix(package_abi)
+
+        packages[package_name] = package["Version"]
+
+    return packages
+
+
+if __name__ == "__main__":
+    import sys
+
+    args = parse_args()
+
+    input = sys.stdin if args.source == "-" else open(args.source, "r")
+    with input:
+        text: str = input.read()
+
+    packages = parse_apk(text) if args.source_format == "apk" else parse_opkg(text)
+    index = {
+        "version": 2,
+        "architecture": args.architecture,
+        "packages": packages,
+    }
+    print(json.dumps(index, indent=2))