ramips: drop m25p,chunked-io from dts
[openwrt/openwrt.git] / scripts / dl_cleanup.py
1 #!/usr/bin/env python3
2 """
3 # OpenWrt download directory cleanup utility.
4 # Delete all but the very last version of the program tarballs.
5 #
6 # Copyright (C) 2010-2015 Michael Buesch <m@bues.ch>
7 # Copyright (C) 2013-2015 OpenWrt.org
8 """
9
10 from __future__ import print_function
11
12 import sys
13 import os
14 import re
15 import getopt
16
17 # Commandline options
18 opt_dryrun = False
19
20
21 def parseVer_1234(match, filepath):
22 progname = match.group(1)
23 progversion = (int(match.group(2)) << 64) |\
24 (int(match.group(3)) << 48) |\
25 (int(match.group(4)) << 32) |\
26 (int(match.group(5)) << 16)
27 return (progname, progversion)
28
29 def parseVer_123(match, filepath):
30 progname = match.group(1)
31 try:
32 patchlevel = match.group(5)
33 except IndexError as e:
34 patchlevel = None
35 if patchlevel:
36 patchlevel = ord(patchlevel[0])
37 else:
38 patchlevel = 0
39 progversion = (int(match.group(2)) << 64) |\
40 (int(match.group(3)) << 48) |\
41 (int(match.group(4)) << 32) |\
42 patchlevel
43 return (progname, progversion)
44
45 def parseVer_12(match, filepath):
46 progname = match.group(1)
47 try:
48 patchlevel = match.group(4)
49 except IndexError as e:
50 patchlevel = None
51 if patchlevel:
52 patchlevel = ord(patchlevel[0])
53 else:
54 patchlevel = 0
55 progversion = (int(match.group(2)) << 64) |\
56 (int(match.group(3)) << 48) |\
57 patchlevel
58 return (progname, progversion)
59
60 def parseVer_r(match, filepath):
61 progname = match.group(1)
62 progversion = (int(match.group(2)) << 64)
63 return (progname, progversion)
64
65 def parseVer_ymd(match, filepath):
66 progname = match.group(1)
67 progversion = (int(match.group(2)) << 64) |\
68 (int(match.group(3)) << 48) |\
69 (int(match.group(4)) << 32)
70 return (progname, progversion)
71
72 def parseVer_GIT(match, filepath):
73 progname = match.group(1)
74 st = os.stat(filepath)
75 progversion = int(st.st_mtime) << 64
76 return (progname, progversion)
77
78 extensions = (
79 ".tar.gz",
80 ".tar.bz2",
81 ".tar.xz",
82 ".orig.tar.gz",
83 ".orig.tar.bz2",
84 ".orig.tar.xz",
85 ".zip",
86 ".tgz",
87 ".tbz",
88 ".txz",
89 )
90
91 versionRegex = (
92 (re.compile(r"(.+)[-_](\d+)\.(\d+)\.(\d+)\.(\d+)"), parseVer_1234), # xxx-1.2.3.4
93 (re.compile(r"(.+)[-_](\d\d\d\d)-?(\d\d)-?(\d\d)"), parseVer_ymd), # xxx-YYYY-MM-DD
94 (re.compile(r"(.+)[-_]([0-9a-fA-F]{40,40})"), parseVer_GIT), # xxx-GIT_SHASUM
95 (re.compile(r"(.+)[-_](\d+)\.(\d+)\.(\d+)(\w?)"), parseVer_123), # xxx-1.2.3a
96 (re.compile(r"(.+)[-_](\d+)_(\d+)_(\d+)"), parseVer_123), # xxx-1_2_3
97 (re.compile(r"(.+)[-_](\d+)\.(\d+)(\w?)"), parseVer_12), # xxx-1.2a
98 (re.compile(r"(.+)[-_]r?(\d+)"), parseVer_r), # xxx-r1111
99 )
100
101 blacklist = [
102 ("linux", re.compile(r"linux-\d.*")),
103 ("gcc", re.compile(r"gcc-.*")),
104 ("wl_apsta", re.compile(r"wl_apsta.*")),
105 (".fw", re.compile(r".*\.fw")),
106 (".arm", re.compile(r".*\.arm")),
107 (".bin", re.compile(r".*\.bin")),
108 ("rt-firmware", re.compile(r"RT[\d\w]+_Firmware.*")),
109 ]
110
111 class EntryParseError(Exception): pass
112
113 class Entry:
114 def __init__(self, directory, filename):
115 self.directory = directory
116 self.filename = filename
117 self.progname = ""
118 self.fileext = ""
119
120 for ext in extensions:
121 if filename.endswith(ext):
122 filename = filename[0:0-len(ext)]
123 self.fileext = ext
124 break
125 else:
126 print(self.filename, "has an unknown file-extension")
127 raise EntryParseError("ext")
128 for (regex, parseVersion) in versionRegex:
129 match = regex.match(filename)
130 if match:
131 (self.progname, self.version) = parseVersion(
132 match, directory + "/" + filename + self.fileext)
133 break
134 else:
135 print(self.filename, "has an unknown version pattern")
136 raise EntryParseError("ver")
137
138 def getPath(self):
139 return (self.directory + "/" + self.filename).replace("//", "/")
140
141 def deleteFile(self):
142 path = self.getPath()
143 print("Deleting", path)
144 if not opt_dryrun:
145 os.unlink(path)
146
147 def __ge__(self, y):
148 return self.version >= y.version
149
150 def usage():
151 print("OpenWrt download directory cleanup utility")
152 print("Usage: " + sys.argv[0] + " [OPTIONS] <path/to/dl>")
153 print("")
154 print(" -d|--dry-run Do a dry-run. Don't delete any files")
155 print(" -B|--show-blacklist Show the blacklist and exit")
156 print(" -w|--whitelist ITEM Remove ITEM from blacklist")
157
158 def main(argv):
159 global opt_dryrun
160
161 try:
162 (opts, args) = getopt.getopt(argv[1:],
163 "hdBw:",
164 [ "help", "dry-run", "show-blacklist", "whitelist=", ])
165 if len(args) != 1:
166 usage()
167 return 1
168 except getopt.GetoptError as e:
169 usage()
170 return 1
171 directory = args[0]
172 for (o, v) in opts:
173 if o in ("-h", "--help"):
174 usage()
175 return 0
176 if o in ("-d", "--dry-run"):
177 opt_dryrun = True
178 if o in ("-w", "--whitelist"):
179 for i in range(0, len(blacklist)):
180 (name, regex) = blacklist[i]
181 if name == v:
182 del blacklist[i]
183 break
184 else:
185 print("Whitelist error: Item", v,\
186 "is not in blacklist")
187 return 1
188 if o in ("-B", "--show-blacklist"):
189 for (name, regex) in blacklist:
190 sep = "\t\t"
191 if len(name) >= 8:
192 sep = "\t"
193 print("%s%s(%s)" % (name, sep, regex.pattern))
194 return 0
195
196 # Create a directory listing and parse the file names.
197 entries = []
198 for filename in os.listdir(directory):
199 if filename == "." or filename == "..":
200 continue
201 for (name, regex) in blacklist:
202 if regex.match(filename):
203 if opt_dryrun:
204 print(filename, "is blacklisted")
205 break
206 else:
207 try:
208 entries.append(Entry(directory, filename))
209 except EntryParseError as e:
210 pass
211
212 # Create a map of programs
213 progmap = {}
214 for entry in entries:
215 if entry.progname in progmap.keys():
216 progmap[entry.progname].append(entry)
217 else:
218 progmap[entry.progname] = [entry,]
219
220 # Traverse the program map and delete everything but the last version
221 for prog in progmap:
222 lastVersion = None
223 versions = progmap[prog]
224 for version in versions:
225 if lastVersion is None or version >= lastVersion:
226 lastVersion = version
227 if lastVersion:
228 for version in versions:
229 if version is not lastVersion:
230 version.deleteFile()
231 if opt_dryrun:
232 print("Keeping", lastVersion.getPath())
233
234 return 0
235
236 if __name__ == "__main__":
237 sys.exit(main(sys.argv))