-#!/usr/bin/env python
+#!/usr/bin/env python3
#
# Copyright (c) 2018 Yousong Zhou <yszhou4tech@gmail.com>
#
import subprocess
import sys
import time
-import urllib2
+import urllib.request
TMPDIR = os.environ.get('TMP_DIR') or '/tmp'
TMPDIR_DL = os.path.join(TMPDIR, 'dl')
def tar(path, subdir, into=None, ts=None):
"""Pack ``path`` into tarball ``into``."""
# --sort=name requires a recent build of GNU tar
- args = ['tar', '--numeric-owner', '--owner=0', '--group=0', '--sort=name']
+ args = ['tar', '--numeric-owner', '--owner=0', '--group=0', '--sort=name', '--mode=a-s']
args += ['-C', path, '-cf', into, subdir]
envs = os.environ.copy()
if ts is not None:
def set(self, k, v):
"""Update timestamp with ``k``."""
fileno = os.open(self.cachef, os.O_RDWR | os.O_CREAT)
- with os.fdopen(fileno, 'wb+') as f:
+ with os.fdopen(fileno, 'w+') as f:
try:
fcntl.lockf(fileno, fcntl.LOCK_EX)
self._cache_init(f)
self.cache[k] = (ts, updated)
def _cache_flush(self, fout):
- cache = sorted(self.cache.iteritems(), cmp=lambda a, b: b[1][1] - a[1][1])
+ cache = sorted(self.cache.items(), key=lambda a: a[1][1])
cache = cache[:self.__cachen]
self.cache = {}
os.ftruncate(fout.fileno(), 0)
class DownloadGitHubTarball(object):
- """Download and repack archive tarabll from GitHub.
+ """Download and repack archive tarball from GitHub.
Compared with the method of packing after cloning the whole repo, this
method is more friendly to users with fragile internet connection.
- GitHub archives do not contain source codes for submodules.
- - GitHub archives seem to respect .gitattributes and ignore pathes with
+ - GitHub archives seem to respect .gitattributes and ignore paths with
export-ignore attributes.
For the first two issues, the method will fail loudly to allow fallback to
self.version = args.version
self.subdir = args.subdir
self.source = args.source
+ self.submodules = args.submodules
self.url = args.url
self._init_owner_repo()
self.xhash = args.hash
def download(self):
"""Download and repack GitHub archive tarball."""
+ if self.submodules and self.submodules != ['skip']:
+ raise self._error('Fetching submodules is not yet supported')
self._init_commit_ts()
with Path(TMPDIR_DL, keep=True) as dir_dl:
# fetch tarball from GitHub
dir0 = os.path.join(dir_untar.path, tarball_prefix)
dir1 = os.path.join(dir_untar.path, self.subdir)
# submodules check
- if self._has_submodule(dir0):
+ if self.submodules != ['skip'] and self._has_submodule(dir0):
raise self._error('Fetching submodules is not yet supported')
# rename subdir
os.rename(dir0, dir1)
version_is_sha1sum = len(self.version) == 40
if not version_is_sha1sum:
apis.insert(0, apis.pop())
+ reasons = ''
for api in apis:
url = api['url']
attr_path = api['attr_path']
self.commit_ts = ct
self.commit_ts_cache.set(url, ct)
return
- except Exception:
- pass
- raise self._error('Cannot fetch commit ts: {}'.format(url))
+ except Exception as e:
+ reasons += '\n' + (" {}: {}".format(url, e))
+ raise self._error('Cannot fetch commit ts:{}'.format(reasons))
def _init_commit_ts_remote_get(self, url, attrpath):
resp = self._make_request(url)
'Accept': 'application/vnd.github.v3+json',
'User-Agent': 'OpenWrt',
}
- req = urllib2.Request(url, headers=headers)
+ req = urllib.request.Request(url, headers=headers)
sslcontext = ssl._create_unverified_context()
- fileobj = urllib2.urlopen(req, context=sslcontext)
+ fileobj = urllib.request.urlopen(req, context=sslcontext)
return fileobj
def _error(self, msg):
parser.add_argument('--version', help='Source code version')
parser.add_argument('--source', help='Source tarball filename')
parser.add_argument('--hash', help='Source tarball\'s expected sha256sum')
+ parser.add_argument('--submodules', nargs='*', help='List of submodules, or "skip"')
args = parser.parse_args()
- method = DownloadGitHubTarball(args)
try:
+ method = DownloadGitHubTarball(args)
method.download()
- except Exception:
- sys.stderr.write('download {} from {} failed\n'.format(args.source, args.url))
- raise
+ except Exception as ex:
+ sys.stderr.write('{}: Download from {} failed\n'.format(args.source, args.url))
+ sys.stderr.write('{}\n'.format(ex))
+ sys.exit(1)
if __name__ == '__main__':
main()