3 from pathlib
import Path
14 Tool to create overview.json files and update the config.js.
17 parser
= argparse
.ArgumentParser()
18 parser
.add_argument("--formatted", action
="store_true",
19 help="Output formatted JSON data.")
20 subparsers
= parser
.add_subparsers(dest
="action", required
=True)
22 parser_merge
= subparsers
.add_parser("merge",
23 help="Create a grid structure with horizontal and vertical connections.")
24 parser_merge
.add_argument("input_path", nargs
="+",
25 help="Input folder that is traversed for OpenWrt JSON device files.")
26 parser_merge
.add_argument("--download-url", action
="store", default
="",
27 help="Link to get the image from. May contain {target}, {version} and {commit}")
28 #parser_merge.add_argument("--change-prefix",
29 # help="Change the openwrt- file name prefix.")
31 parser_scrape
= subparsers
.add_parser("scrape",
32 help="Create a grid structure of horizontal, vertical and vertical connections.")
33 parser_scrape
.add_argument("domain",
34 help="Domain to scrape. E.g. https://downloads.openwrt.org")
35 parser_scrape
.add_argument("selector",
36 help="Path the config.js file is in.")
37 parser_scrape
.add_argument("--use-wget", action
="store_true",
38 help="Use wget to scrape the site.")
40 args
= parser
.parse_args()
42 SUPPORTED_METADATA_VERSION
= 1
44 # accepts {<file-path>: <file-content>}
45 def merge_profiles(profiles
, download_url
):
53 return "{} {} {}".format(title
.get("vendor", ""), title
["model"], title
.get("variant", "")).strip()
55 def add_profile(id, target
, profile
, code
=None):
57 for image
in profile
["images"]:
58 images
.append({"name": image
["name"], "type": image
["type"]})
61 target
= profile
["target"]
63 #if args.change_prefix:
64 # change_prefix(images, "openwrt-", args.change_prefix)
66 for entry
in profile
["titles"]:
67 title
= get_title(entry
)
70 sys
.stderr
.write(f
"Empty title. Skip title in {path}\n")
73 output
["models"][title
] = {"id": id, "target": target
, "images": images
}
76 output
["models"][title
]["code"] = code
78 for path
, content
in profiles
.items():
79 obj
= json
.loads(content
)
81 if obj
["metadata_version"] != SUPPORTED_METADATA_VERSION
:
82 sys
.stderr
.write(f
"{path} has unsupported metadata version: {obj['metadata_version']} => skip\n")
85 code
= obj
.get("version_code", obj
.get("version_commit"))
87 if not "version_code" in output
:
90 "download_url": download_url
,
94 # if we have mixed codes/commits, store in device object
95 if output
["version_code"] == code
:
100 for id in obj
["profiles"]:
101 add_profile(id, obj
.get("target"), obj
["profiles"][id], code
)
103 add_profile(obj
["id"], obj
["target"], obj
, code
)
104 except json
.decoder
.JSONDecodeError
as e
:
105 sys
.stderr
.write(f
"Skip {path}\n {e}\n")
106 except KeyError as e
:
107 sys
.stderr
.write(f
"Abort on {path}\n Missing key {e}\n")
112 def update_config(config_path
, versions
):
114 with
open(config_path
, "r") as file:
115 content
= file.read()
117 content
= re
.sub("versions:[\\s]*{[^}]*}", f
"versions: {versions}" , content
)
118 with
open(config_path
, "w+") as file:
119 # save updated config
123 Scrape profiles.json using links like https://downloads.openwrt.org/releases/19.07.3/targets/?json
124 Merge into overview.json files.
127 def scrape(url
, selector_path
):
128 config_path
= f
"{selector_path}/config.js"
129 data_path
= f
"{selector_path}/data"
132 def handle_release(target
):
134 with urllib
.request
.urlopen(f
"{target}/?json") as file:
135 array
= json
.loads(file.read().decode("utf-8"))
136 for profile
in filter(lambda x
: x
.endswith("/profiles.json"), array
):
138 with urllib
.request
.urlopen(f
"{target}/{profile}") as file:
139 profiles
[f
"{target}/{profile}"] = file.read()
142 if not os
.path
.isfile(config_path
):
143 print(f
"file not found: {config_path}")
147 with urllib
.request
.urlopen(url
) as infile
:
148 for path
in re
.findall(r
"href=[\"']?([^'\" >]+)", str(infile.read())):
149 if not path.startswith("/") and path.endswith("targets
/"):
150 release = path.strip("/").split("/")[-2]
151 download_url = f"{url}
/{path}
/{{target}
}"
153 profiles = handle_release(f"{url}
/{path}
")
154 output = merge_profiles(profiles, download_url)
156 Path(f"{data_path}
/{release}
").mkdir(parents=True, exist_ok=True)
157 # write overview.json
158 with open(f"{data_path}
/{release}
/overview
.json
", "w
") as outfile:
160 json.dump(output, outfile, indent=" ", sort_keys=True)
162 json.dump(output, outfile, sort_keys=True)
164 versions[release.upper()] = f"data
/{release}
/overview
.json
"
166 update_config(config_path, versions)
169 Scrape profiles.json using wget (slower but more generic).
170 Merge into overview.json files.
173 def scrape_wget(url, selector_path):
174 config_path = f"{selector_path}
/config
.js
"
175 data_path = f"{selector_path}
/data
"
178 with tempfile.TemporaryDirectory() as tmp_dir:
179 # download all profiles.json files
180 os.system(f"wget
-c
-r
-P {tmp_dir}
-A
'profiles.json' --reject
-regex
'kmods|packages' --no
-parent {url}
")
182 # delete empty folders
183 os.system(f"find {tmp_dir}
/* -type d
-empty
-delete
")
185 # create overview.json files
186 for path in glob.glob(f"{tmp_dir}
/*/snapshots
") + glob.glob(f"{tmp_dir}
/*/releases
/*"):
187 release = os.path.basename(path)
188 base = path[len(tmp_dir)+1:]
190 versions[release.upper()] = f"data
/{release}
/overview
.json
"
191 os.system(f"mkdir
-p {selector_path}
/data
/{release}
/")
194 for ppath in Path(path).rglob('profiles.json'):
195 with open(ppath, "r
") as file:
196 profiles[ppath] = file.read()
198 output = merge_profiles(profiles, f"https
://{base}
/targets
/{{target}
}")
199 Path(f"{data_path}
/{release}
").mkdir(parents=True, exist_ok=True)
201 # write overview.json
202 with open(f"{data_path}
/{release}
/overview
.json
", "w
") as outfile:
204 json.dump(output, outfile, indent=" ", sort_keys=True)
206 json.dump(output, outfile, sort_keys=True)
208 update_config(config_path, versions)
211 def change_prefix(images, old_prefix, new_prefix):
213 if image["name
"].startswith(old_prefix):
214 image["name
"] = new_prefix + image["name
"][len(old_prefix):]
218 Find and merge json files for a single release.
220 def merge(input_paths):
221 # OpenWrt JSON device files
226 with open(path, "r
") as file:
227 profiles[path] = file.read()
229 for path in input_paths:
230 if os.path.isdir(path):
231 for filepath in Path(path).rglob("*.json
"):
234 if not path.endswith(".json
"):
235 sys.stderr.write(f"Folder does
not exists
: {path}
\n")
239 output = merge_profiles(profiles, args.download_url)
242 json.dump(output, sys.stdout, indent=" ", sort_keys=True)
244 json.dump(output, sys.stdout, sort_keys=True)
246 if args.action == "merge
":
247 merge(args.input_path)
249 if args.action == "scrape
":
251 scrape_wget(args.domain, args.selector)
253 scrape(args.domain, args.selector)