3 Tool to create overview.json files and update the config.js.
6 from pathlib
import Path
16 SUPPORTED_METADATA_VERSION
= 1
19 # accepts {<file-path>: <file-content>}
20 def merge_profiles(profiles
, download_url
):
28 return "{} {} {}".format(
29 title
.get("vendor", ""), title
["model"], title
.get("variant", "")
32 def add_profile(id, target
, profile
, code
=None):
34 for image
in profile
["images"]:
35 images
.append({"name": image
["name"], "type": image
["type"]})
38 target
= profile
["target"]
40 for entry
in profile
["titles"]:
41 title
= get_title(entry
)
44 sys
.stderr
.write("Empty title. Skip title for {}".format(id))
47 output
["models"][title
] = {"id": id, "target": target
, "images": images
}
50 output
["models"][title
]["code"] = code
52 for path
, content
in profiles
.items():
53 obj
= json
.loads(content
)
55 if obj
["metadata_version"] != SUPPORTED_METADATA_VERSION
:
57 "{} has unsupported metadata version: {} => skip".format(
58 path
, obj
["metadata_version"]
63 code
= obj
.get("version_code", obj
.get("version_commit"))
65 if "version_code" not in output
:
66 output
= {"version_code": code
, "download_url": download_url
, "models": {}}
68 # if we have mixed codes/commits, store in device object
69 if output
["version_code"] == code
:
74 for id in obj
["profiles"]:
75 add_profile(id, obj
.get("target"), obj
["profiles"][id], code
)
77 add_profile(obj
["id"], obj
["target"], obj
, code
)
78 except json
.decoder
.JSONDecodeError
as e
:
79 sys
.stderr
.write("Skip {}\n {}\n".format(path
, e
))
81 sys
.stderr
.write("Abort on {}\n Missing key {}\n".format(path
, e
))
87 def update_config(config_path
, versions
):
89 with
open(str(config_path
), "r") as file:
92 content
= re
.sub("versions:[\\s]*{[^}]*}", "versions: {}".format(versions
), content
)
93 with
open(str(config_path
), "w+") as file:
98 Scrape profiles.json using links like https://downloads.openwrt.org/releases/19.07.3/targets/?json
99 Merge into overview.json files.
106 selector_path
= args
.selector
107 config_path
= "{}/config.js".format(selector_path
)
108 data_path
= "{}/data".format(selector_path
)
111 def handle_release(target
):
113 with urllib
.request
.urlopen("{}/?json".format(target
)) as file:
114 array
= json
.loads(file.read().decode("utf-8"))
115 for profile
in filter(lambda x
: x
.endswith("/profiles.json"), array
):
116 with urllib
.request
.urlopen("{}/{}".format(target
, profile
)) as file:
117 profiles
["{}/{}".format(target
, profile
)] = file.read().decode(
122 if not os
.path
.isfile(config_path
):
123 print("file not found: {}".format(config_path
))
127 with urllib
.request
.urlopen(url
) as infile
:
128 for path
in re
.findall(r
"href=[\"']?([^'\" >]+)", str(infile.read())):
129 if not path.startswith("/") and path.endswith("targets
/"):
130 release = path.strip("/").split("/")[-2]
131 download_url = "{}/{}/{{target}
}".format(url, path)
133 profiles = handle_release("{}/{}".format(url, path))
134 output = merge_profiles(profiles, download_url)
136 os.makedirs("{}/{}".format(data_path, release), exist_ok=True)
137 # write overview.json
139 "{}/{}/overview
.json
".format(data_path, release), "w
"
142 json.dump(output, outfile, indent=" ", sort_keys=True)
144 json.dump(output, outfile, sort_keys=True)
146 versions[release.upper()] = "data
/{}/overview
.json
".format(release)
148 update_config(config_path, versions)
152 Scrape profiles.json using wget (slower but more generic).
153 Merge into overview.json files.
158 def scrape_wget(args):
160 selector_path = args.selector
161 config_path = "{}/config
.js
".format(selector_path)
162 data_path = "{}/data
".format(selector_path)
165 with tempfile.TemporaryDirectory() as tmp_dir:
166 # download all profiles.json files
168 "wget
-c
-r
-P
{} -A
'profiles.json' --reject
-regex
'kmods|packages' --no
-parent
{}".format(
173 # delete empty folders
174 os.system("find
{}/* -type d
-empty
-delete
".format(tmp_dir))
176 # create overview.json files
177 for path in glob.glob("{}/*/snapshots
".format(tmp_dir)) + glob.glob(
178 "{}/*/releases
/*".format(tmp_dir)
180 release = os.path.basename(path)
181 base = path[len(tmp_dir) + 1 :]
184 for ppath in Path(path).rglob("profiles
.json
"):
185 with open(str(ppath), "r
") as file:
186 profiles[ppath] = file.read()
188 if len(profiles) == 0:
191 versions[release.upper()] = "data
/{}/overview
.json
".format(release)
193 output = merge_profiles(
194 profiles, "https
://{}/targets
/{{target}
}".format(base)
196 os.makedirs("{}/{}".format(data_path, release), exist_ok=True)
198 # write overview.json
199 with open("{}/{}/overview
.json
".format(data_path, release), "w
") as outfile:
201 json.dump(output, outfile, indent=" ", sort_keys=True)
203 json.dump(output, outfile, sort_keys=True)
205 update_config(config_path, versions)
209 Find and merge json files for a single release.
214 input_paths = args.input_path
215 # OpenWrt JSON device files
219 with open(str(path), "r
") as file:
220 profiles[path] = file.read()
222 for path in input_paths:
223 if os.path.isdir(path):
224 for filepath in Path(path).rglob("*.json
"):
227 if not path.endswith(".json
"):
228 sys.stderr.write("Folder does
not exists
: {}\n".format(path))
232 output = merge_profiles(profiles, args.download_url)
235 json.dump(output, sys.stdout, indent=" ", sort_keys=True)
237 json.dump(output, sys.stdout, sort_keys=True)
241 Scan local directory for releases with profiles.json.
242 Merge into overview.json files.
248 selector_path = args.selector
249 config_path = "{}/config
.js
".format(selector_path)
250 data_path = "{}/data
".format(selector_path)
253 # create overview.json files
254 for path in glob.glob("{}/snapshots
".format(args.directory)) + glob.glob(
255 "{}/releases
/*".format(args.directory)
257 release = os.path.basename(path)
258 base_dir = path[len(args.directory) + 1 :]
261 for ppath in Path(path).rglob("profiles
.json
"):
262 with open(str(ppath), "r
", encoding="utf
-8") as file:
263 profiles[ppath] = file.read()
265 if len(profiles) == 0:
268 versions[release.upper()] = "data
/{}/overview
.json
".format(release)
270 output = merge_profiles(
271 profiles, "https
://{}/{}/targets
/{{target}
}".format(args.domain, base_dir)
273 os.makedirs("{}/{}".format(data_path, release), exist_ok=True)
275 # write overview.json
276 with open("{}/{}/overview
.json
".format(data_path, release), "w
") as outfile:
278 json.dump(output, outfile, indent=" ", sort_keys=True)
280 json.dump(output, outfile, sort_keys=True)
282 update_config(config_path, versions)
286 parser = argparse.ArgumentParser()
288 "--formatted
", action="store_true
", help="Output formatted JSON data
."
290 subparsers = parser.add_subparsers(dest="action
")
291 subparsers.required = True
293 parser_merge = subparsers.add_parser(
295 help="Create a grid structure with horizontal
and vertical connections
.",
297 parser_merge.add_argument(
300 help="Input folder that
is traversed
for OpenWrt JSON device files
.",
302 parser_merge.add_argument(
306 help="Link to get the image
from. May contain {target}
, {version}
and {commit}
",
309 parser_scrape = subparsers.add_parser("scrape
", help="Scrape webpage
for releases
.")
310 parser_scrape.add_argument(
311 "domain
", help="Domain to scrape
. E
.g
. https
://downloads
.openwrt
.org
"
313 parser_scrape.add_argument("selector
", help="Path the config
.js
file is in.")
314 parser_scrape.add_argument(
315 "--use
-wget
", action="store_true
", help="Use wget to scrape the site
."
318 parser_scan = subparsers.add_parser("scan
", help="Scan directory
for releases
.")
319 parser_scan.add_argument(
321 help="Domain
for download_url attribute
in overview
.json
. E
.g
. https
://downloads
.openwrt
.org
",
323 parser_scan.add_argument("directory
", help="Directory to scan
for releases
.")
324 parser_scan.add_argument("selector
", help="Path the config
.js
file is in.")
326 args = parser.parse_args()
328 if args.action == "merge
":
331 if args.action == "scan
":
334 if args.action == "scrape
":
341 if __name__ == "__main__
":