3 # Copyright (c) 2018 Yousong Zhou <yszhou4tech@gmail.com>
5 # This is free software, licensed under the GNU General Public License v2.
6 # See /LICENSE for more information.
24 TMPDIR
= os
.environ
.get('TMP_DIR') or '/tmp'
25 TMPDIR_DL
= os
.path
.join(TMPDIR
, 'dl')
28 class PathException(Exception): pass
29 class DownloadException(Exception): pass
33 """Context class for preparing and cleaning up directories.
35 If ``path`` ``isdir``, then it will be created on context enter.
37 If ``keep`` is True, then ``path`` will NOT be removed on context exit
40 def __init__(self
, path
, isdir
=True, keep
=False):
47 self
.mkdir_all(self
.path
)
50 def __exit__(self
, exc_type
, exc_value
, traceback
):
52 self
.rm_all(self
.path
)
56 """Same as mkdir -p."""
57 names
= os
.path
.split(path
)
60 p
= os
.path
.join(p
, name
)
65 names
= Path
._listdir
(dir_
)
67 p
= os
.path
.join(dir_
, name
)
76 Path
._os
_func
(os
.mkdir
, path
, errno
.EEXIST
)
80 Path
._os
_func
(os
.rmdir
, path
, errno
.ENOENT
)
84 Path
._os
_func
(os
.remove
, path
, errno
.ENOENT
)
88 return Path
._os
_func
(os
.listdir
, path
, errno
.ENOENT
, default
=[])
91 def _os_func(func
, path
, errno
, default
=None):
92 """Call func(path) in an idempotent way.
94 On exception ``ex``, if the type is OSError and ``ex.errno == errno``,
95 return ``default``, otherwise, re-raise
108 if os
.path
.isdir(path
):
109 Path
._rmdir
_all
(path
)
114 def untar(path
, into
=None):
115 """Extract tarball at ``path`` into subdir ``into``.
117 return subdir name if and only if there exists one, otherwise raise PathException
119 args
= ('tar', '-C', into
, '-xzf', path
, '--no-same-permissions')
120 subprocess
.check_call(args
, preexec_fn
=lambda: os
.umask(0o22))
121 dirs
= os
.listdir(into
)
125 raise PathException('untar %s: expecting a single subdir, got %s' % (path
, dirs
))
128 def tar(path
, subdir
, into
=None, ts
=None):
129 """Pack ``path`` into tarball ``into``."""
130 # --sort=name requires a recent build of GNU tar
131 args
= ['tar', '--numeric-owner', '--owner=0', '--group=0', '--sort=name']
132 args
+= ['-C', path
, '-cf', into
, subdir
]
133 envs
= os
.environ
.copy()
135 args
.append('--mtime=@%d' % ts
)
136 if into
.endswith('.xz'):
137 envs
['XZ_OPT'] = '-7e'
139 elif into
.endswith('.bz2'):
141 elif into
.endswith('.gz'):
145 raise PathException('unknown compression type %s' % into
)
146 subprocess
.check_call(args
, env
=envs
)
149 class GitHubCommitTsCache(object):
150 __cachef
= 'github.commit.ts.cache'
154 Path
.mkdir_all(TMPDIR_DL
)
155 self
.cachef
= os
.path
.join(TMPDIR_DL
, self
.__cachef
)
159 """Get timestamp with key ``k``."""
160 fileno
= os
.open(self
.cachef
, os
.O_RDONLY | os
.O_CREAT
)
161 with os
.fdopen(fileno
) as fin
:
163 fcntl
.lockf(fileno
, fcntl
.LOCK_SH
)
164 self
._cache
_init
(fin
)
166 ts
= self
.cache
[k
][0]
169 fcntl
.lockf(fileno
, fcntl
.LOCK_UN
)
173 """Update timestamp with ``k``."""
174 fileno
= os
.open(self
.cachef
, os
.O_RDWR | os
.O_CREAT
)
175 with os
.fdopen(fileno
, 'wb+') as f
:
177 fcntl
.lockf(fileno
, fcntl
.LOCK_EX
)
179 self
.cache
[k
] = (v
, int(time
.time()))
182 fcntl
.lockf(fileno
, fcntl
.LOCK_UN
)
184 def _cache_init(self
, fin
):
186 k
, ts
, updated
= line
.split()
188 updated
= int(updated
)
189 self
.cache
[k
] = (ts
, updated
)
191 def _cache_flush(self
, fout
):
192 cache
= sorted(self
.cache
.iteritems(), cmp=lambda a
, b
: b
[1][1] - a
[1][1])
193 cache
= cache
[:self
.__cachen
]
195 os
.ftruncate(fout
.fileno(), 0)
196 fout
.seek(0, os
.SEEK_SET
)
200 line
= '{0} {1} {2}\n'.format(k
, ts
, updated
)
204 class DownloadMethod(object):
205 """Base class of all download method."""
207 def __init__(self
, args
):
209 self
.urls
= args
.urls
210 self
.url
= self
.urls
[0]
211 self
.dl_dir
= args
.dl_dir
214 def resolve(cls
, args
):
215 """Resolve download method to use.
217 return instance of subclass of DownloadMethod
219 for c
in DOWNLOAD_METHODS
:
225 """Return True if it can do the download."""
226 return NotImplemented
229 """Do the download and put it into the download dir."""
230 return NotImplemented
233 class DownloadMethodGitHubTarball(DownloadMethod
):
234 """Download and repack archive tarabll from GitHub."""
236 __repo_url_regex
= re
.compile(r
'^(?:https|git)://github.com/(?P<owner>[^/]+)/(?P<repo>[^/]+)')
238 def __init__(self
, args
):
239 super(DownloadMethodGitHubTarball
, self
).__init
__(args
)
240 self
._init
_owner
_repo
()
241 self
.version
= args
.version
242 self
.subdir
= args
.subdir
243 self
.source
= args
.source
244 self
.commit_ts
= None # lazy load commit timestamp
245 self
.commit_ts_cache
= GitHubCommitTsCache()
246 self
.name
= 'github-tarball'
250 """Match if it's a GitHub clone url."""
253 if proto
== 'git' and isinstance(url
, basestring
) \
254 and (url
.startswith('https://github.com/') or url
.startswith('git://github.com/')):
259 """Download and repack GitHub archive tarball."""
260 self
._init
_commit
_ts
()
261 with
Path(TMPDIR_DL
, keep
=True) as dir_dl
:
262 # fetch tarball from GitHub
263 tarball_path
= os
.path
.join(dir_dl
.path
, self
.subdir
+ '.tar.gz.dl')
264 with
Path(tarball_path
, isdir
=False):
265 self
._fetch
(tarball_path
)
267 d
= os
.path
.join(dir_dl
.path
, self
.subdir
+ '.untar')
268 with
Path(d
) as dir_untar
:
269 tarball_prefix
= Path
.untar(tarball_path
, into
=dir_untar
.path
)
270 dir0
= os
.path
.join(dir_untar
.path
, tarball_prefix
)
271 dir1
= os
.path
.join(dir_untar
.path
, self
.subdir
)
273 if self
._has
_submodule
(dir0
):
274 raise DownloadException('unable to fetch submodules\' source code')
276 os
.rename(dir0
, dir1
)
278 into
=os
.path
.join(TMPDIR_DL
, self
.source
)
279 Path
.tar(dir_untar
.path
, self
.subdir
, into
=into
, ts
=self
.commit_ts
)
280 # move to target location
281 file1
= os
.path
.join(self
.dl_dir
, self
.source
)
283 shutil
.move(into
, file1
)
285 def _has_submodule(self
, dir_
):
286 m
= os
.path
.join(dir_
, '.gitmodules')
289 return st
.st_size
> 0
291 return e
.errno
!= errno
.ENOENT
293 def _init_owner_repo(self
):
295 m
= self
.__repo
_url
_regex
.search(url
)
297 raise DownloadException('invalid github url: %s' % url
)
298 owner
= m
.group('owner')
299 repo
= m
.group('repo')
300 if repo
.endswith('.git'):
305 def _init_commit_ts(self
):
306 if self
.commit_ts
is not None:
308 url
= self
._make
_repo
_url
_path
('git', 'commits', self
.version
)
309 ct
= self
.commit_ts_cache
.get(url
)
313 resp
= self
._make
_request
(url
)
315 data
= json
.loads(data
)
316 date
= data
['committer']['date']
317 date
= datetime
.datetime
.strptime(date
, '%Y-%m-%dT%H:%M:%SZ')
318 date
= date
.timetuple()
319 ct
= calendar
.timegm(date
)
321 self
.commit_ts_cache
.set(url
, ct
)
323 def _fetch(self
, path
):
324 """Fetch tarball of the specified version ref."""
326 url
= self
._make
_repo
_url
_path
('tarball', ref
)
327 resp
= self
._make
_request
(url
)
328 with
open(path
, 'wb') as fout
:
335 def _make_repo_url_path(self
, *args
):
336 url
= '/repos/{0}/{1}'.format(self
.owner
, self
.repo
)
338 url
+= '/' + '/'.join(args
)
341 def _make_request(self
, path
):
342 """Request GitHub API endpoint on ``path``."""
343 url
= 'https://api.github.com' + path
345 'Accept': 'application/vnd.github.v3+json',
346 'User-Agent': 'OpenWrt',
348 req
= urllib2
.Request(url
, headers
=headers
)
349 sslcontext
= ssl
._create
_unverified
_context
()
350 fileobj
= urllib2
.urlopen(req
, context
=sslcontext
)
354 class DownloadMethodCatchall(DownloadMethod
):
355 """Dummy method that knows names but not ways of download."""
357 def __init__(self
, args
):
358 super(DownloadMethodCatchall
, self
).__init
__(args
)
360 self
.proto
= args
.proto
361 self
.name
= self
._resolve
_name
()
363 def _resolve_name(self
):
367 ('default', ('@APACHE/', '@GITHUB/', '@GNOME/', '@GNU/',
368 '@KERNEL/', '@SF/', '@SAVANNAH/', 'ftp://', 'http://',
369 'https://', 'file://')),
370 ('git', ('git://', )),
371 ('svn', ('svn://', )),
372 ('cvs', ('cvs://', )),
373 ('bzr', ('sftp://', )),
374 ('bzr', ('sftp://', )),
377 for name
, prefixes
in methods_map
:
378 if any(url
.startswith(prefix
) for prefix
in prefixes
for url
in self
.urls
):
389 raise DownloadException
391 raise DownloadException('download method for %s is not yet implemented' % self
.name
)
395 DownloadMethodGitHubTarball
,
396 DownloadMethodCatchall
,
401 parser
= argparse
.ArgumentParser()
402 parser
.add_argument('action', choices
=('dl_method', 'dl'), help='Action to take')
403 parser
.add_argument('--urls', nargs
='+', metavar
='URL', help='Download URLs')
404 parser
.add_argument('--proto', help='Download proto')
405 parser
.add_argument('--subdir', help='Source code subdir name')
406 parser
.add_argument('--version', help='Source code version')
407 parser
.add_argument('--source', help='Source tarball filename')
408 parser
.add_argument('--dl-dir', default
=os
.getcwd(), help='Download dir')
409 args
= parser
.parse_args()
410 if args
.action
== 'dl_method':
411 method
= DownloadMethod
.resolve(args
)
412 sys
.stdout
.write(method
.name
+ '\n')
413 elif args
.action
== 'dl':
414 method
= DownloadMethod
.resolve(args
)
420 if __name__
== '__main__':