gentree.py

   1 #!/usr/bin/env python
   2 #
   3 # Generate the output tree into a specified directory.
   4 #
   5
   6 import argparse, sys, os, errno, shutil, re, subprocess
   7
   8 # find self
   9 source_dir = os.path.abspath(os.path.dirname(__file__))
  10 sys.path.append(source_dir)
  11 # and import libraries we have
  12 from lib import kconfig, git, patch, make
  13
  14
  15 def read_copy_list(copyfile):
  16     """
  17     Read a copy-list file and return a list of (source, target)
  18     tuples. The source and target are usually the same, but in
  19     the copy-list file there may be a rename included.
  20     """
  21     ret = []
  22     for item in copyfile:
  23         # remove leading/trailing whitespace
  24         item = item.strip()
  25         # comments
  26         if not item or item[0] == '#':
  27             continue
  28         if item[0] == '/':
  29             raise Exception("Input path '%s' is absolute path, this isn't allowed" % (item, ))
  30         if ' -> ' in item:
  31             srcitem, dstitem = item.split(' -> ')
  32             if (srcitem[-1] == '/') != (dstitem[-1] == '/'):
  33                 raise Exception("Cannot copy file/dir to dir/file")
  34         else:
  35             srcitem = dstitem = item
  36         ret.append((srcitem, dstitem))
  37     return ret
  38
  39
  40 def read_dependencies(depfilename):
  41     """
  42     Read a (the) dependency file and return the list of
  43     dependencies as a dictionary, mapping a Kconfig symbol
  44     to a list of kernel version dependencies. While reading
  45     ignore blank/commented lines.
  46     """
  47     ret = {}
  48     depfile = open(depfilename, 'r')
  49     for item in depfile:
  50         item = item.strip()
  51         if not item or item[0] == '#':
  52             continue
  53         sym, dep = item.split()
  54         if not sym in ret:
  55             ret[sym] = [dep, ]
  56         else:
  57             ret[sym].append(dep)
  58     return ret
  59
  60
  61 def check_output_dir(d, clean):
  62     """
  63     Check that the output directory doesn't exist or is empty,
  64     unless clean is True in which case it's nuked. This helps
  65     sanity check the output when generating a tree, so usually
  66     running with --clean isn't suggested.
  67     """
  68     if clean:
  69         shutil.rmtree(d, ignore_errors=True)
  70     try:
  71         os.rmdir(d)
  72     except OSError, e:
  73         if e.errno != errno.ENOENT:
  74             raise
  75
  76
  77 def copytree(src, dst, symlinks=False, ignore=None):
  78     """
  79     Copy a directory tree. This differs from shutil.copytree()
  80     in that it allows destination directories to already exist.
  81     """
  82     names = os.listdir(src)
  83     if ignore is not None:
  84         ignored_names = ignore(src, names)
  85     else:
  86         ignored_names = set()
  87
  88     if not os.path.isdir(dst):
  89         os.makedirs(dst)
  90     errors = []
  91     for name in names:
  92         if name in ignored_names:
  93             continue
  94         srcname = os.path.join(src, name)
  95         dstname = os.path.join(dst, name)
  96         try:
  97             if symlinks and os.path.islink(srcname):
  98                 linkto = os.readlink(srcname)
  99                 os.symlink(linkto, dstname)
 100             elif os.path.isdir(srcname):
 101                 copytree(srcname, dstname, symlinks, ignore)
 102             else:
 103                 shutil.copy2(srcname, dstname)
 104         except (IOError, os.error) as why:
 105             errors.append((srcname, dstname, str(why)))
 106         # catch the Error from the recursive copytree so that we can
 107         # continue with other files
 108         except shutil.Error as err:
 109             errors.extend(err.args[0])
 110     try:
 111         shutil.copystat(src, dst)
 112     except WindowsError:
 113         # can't copy file access times on Windows
 114         pass
 115     except OSError as why:
 116         errors.extend((src, dst, str(why)))
 117     if errors:
 118         raise shutil.Error(errors)
 119
 120
 121 def copy_files(srcpath, copy_list, outdir):
 122     """
 123     Copy the copy_list files and directories from the srcpath
 124     to the outdir. The copy_list contains source and target
 125     names.
 126
 127     For now, it also ignores any *~ editor backup files, though
 128     this should probably be generalized (maybe using .gitignore?)
 129     Similarly the code that only copies some files (*.c, *.h,
 130     *.awk, Kconfig, Makefile) to avoid any build remnants in the
 131     kernel if they should exist.
 132     """
 133     for srcitem, tgtitem in copy_list:
 134         if tgtitem == '':
 135             copytree(srcpath, outdir, ignore=shutil.ignore_patterns('*~'))
 136         elif tgtitem[-1] == '/':
 137             def copy_ignore(dir, entries):
 138                 r = []
 139                 for i in entries:
 140                     if i[-1] in ('o', '~'):
 141                         r.append(i)
 142                 return r
 143             copytree(os.path.join(srcpath, srcitem),
 144                      os.path.join(outdir, tgtitem),
 145                      ignore=copy_ignore)
 146         else:
 147             try:
 148                 os.makedirs(os.path.join(outdir, os.path.dirname(tgtitem)))
 149             except OSError, e:
 150                 # ignore dirs we might have created just now
 151                 if e.errno != errno.EEXIST:
 152                     raise
 153             shutil.copy(os.path.join(srcpath, srcitem),
 154                         os.path.join(outdir, tgtitem))
 155
 156
 157 def copy_git_files(srcpath, copy_list, rev, outdir):
 158     """
 159     "Copy" files from a git repository. This really means listing them with
 160     ls-tree and then using git show to obtain all the blobs.
 161     """
 162     for srcitem, tgtitem in copy_list:
 163         for m, t, h, f in git.ls_tree(rev=rev, files=(srcitem,), tree=srcpath):
 164             assert t == 'blob'
 165             f = os.path.join(outdir, f)
 166             d = os.path.dirname(f)
 167             if not os.path.exists(d):
 168                 os.makedirs(d)
 169             outf = open(f, 'w')
 170             git.get_blob(h, outf, tree=srcpath)
 171             outf.close()
 172             os.chmod(f, int(m, 8))
 173
 174
 175 def git_debug_init(args):
 176     """
 177     Initialize a git repository in the output directory and commit the current
 178     code in it. This is only used for debugging the transformations this code
 179     will do to the output later.
 180     """
 181     if not args.gitdebug:
 182         return
 183     git.init(tree=args.outdir)
 184     git.commit_all("Copied code", tree=args.outdir)
 185
 186
 187 def git_debug_snapshot(args, name):
 188     """
 189     Take a git snapshot for the debugging.
 190     """
 191     if not args.gitdebug:
 192         return
 193     git.commit_all(name, tree=args.outdir)
 194
 195
 196 def _main():
 197     # set up and parse arguments
 198     parser = argparse.ArgumentParser(description='generate backport tree')
 199     parser.add_argument('kerneldir', metavar='<kernel tree>', type=str,
 200                         help='Kernel tree to copy drivers from')
 201     parser.add_argument('outdir', metavar='<output directory>', type=str,
 202                         help='Directory to write the generated tree to')
 203     parser.add_argument('--copy-list', metavar='<listfile>', type=argparse.FileType('r'),
 204                         default='copy-list',
 205                         help='File containing list of files/directories to copy, default "copy-list"')
 206     parser.add_argument('--git-revision', metavar='<revision>', type=str,
 207                         help='git commit revision (see gitrevisions(7)) to take objects from.' +
 208                              'If this is specified, the kernel tree is used as git object storage ' +
 209                              'and we use git ls-tree to get the files.')
 210     parser.add_argument('--clean', const=True, default=False, action="store_const",
 211                         help='Clean output directory instead of erroring if it isn\'t empty')
 212     parser.add_argument('--refresh', const=True, default=False, action="store_const",
 213                         help='Refresh patches as they are applied, the source dir will be modified!')
 214     parser.add_argument('--base-name', metavar='<name>', type=str, default='Linux',
 215                         help='name of base tree, default just "Linux"')
 216     parser.add_argument('--gitdebug', const=True, default=False, action="store_const",
 217                         help='Use git, in the output tree, to debug the various transformation steps ' +
 218                              'that the tree generation makes (apply patches, ...)')
 219     parser.add_argument('--verbose', const=True, default=False, action="store_const",
 220                         help='Print more verbose information')
 221     parser.add_argument('--extra-driver', nargs=2, metavar=('<source dir>', '<copy-list>'), type=str,
 222                         action='append', default=[], help='Extra driver directory/copy-list.')
 223     args = parser.parse_args()
 224
 225     def logwrite(msg):
 226         sys.stdout.write(msg)
 227         sys.stdout.write('\n')
 228         sys.stdout.flush()
 229
 230     return process(args.kerneldir, args.outdir, args.copy_list,
 231                    git_revision=args.git_revision, clean=args.clean,
 232                    refresh=args.refresh, base_name=args.base_name,
 233                    gitdebug=args.gitdebug, verbose=args.verbose,
 234                    extra_driver=args.extra_driver, logwrite=logwrite)
 235
 236 def process(kerneldir, outdir, copy_list_file, git_revision=None,
 237             clean=False, refresh=False, base_name="Linux", gitdebug=False,
 238             verbose=False, extra_driver=[], logwrite=lambda x:None,
 239             kernel_version_name=None, backport_version_name=None):
 240     class Args(object):
 241         def __init__(self, kerneldir, outdir, copy_list_file,
 242                      git_revision, clean, refresh, base_name,
 243                      gitdebug, verbose, extra_driver):
 244             self.kerneldir = kerneldir
 245             self.outdir = outdir
 246             self.copy_list = copy_list_file
 247             self.git_revision = git_revision
 248             self.clean = clean
 249             self.refresh = refresh
 250             self.base_name = base_name
 251             self.gitdebug = gitdebug
 252             self.verbose = verbose
 253             self.extra_driver = extra_driver
 254     args = Args(kerneldir, outdir, copy_list_file,
 255                 git_revision, clean, refresh, base_name,
 256                 gitdebug, verbose, extra_driver)
 257     # start processing ...
 258
 259     copy_list = read_copy_list(args.copy_list)
 260     deplist = read_dependencies(os.path.join(source_dir, 'dependencies'))
 261
 262     # validate output directory
 263     check_output_dir(args.outdir, args.clean)
 264
 265     # do the copy
 266     backport_files = [(x, x) for x in [
 267         'Kconfig', 'Makefile', 'Makefile.build', 'Makefile.kernel', '.gitignore',
 268         'Makefile.real', 'compat/', 'include/', 'kconfig/', 'defconfigs/',
 269     ]]
 270     if not args.git_revision:
 271         logwrite('Copy original source files ...')
 272         copy_files(os.path.join(source_dir, 'backport'), backport_files, args.outdir)
 273         copy_files(args.kerneldir, copy_list, args.outdir)
 274     else:
 275         logwrite('Get original source files from git ...')
 276         copy_files(os.path.join(source_dir, 'backport'), backport_files, args.outdir)
 277         copy_git_files(args.kerneldir, copy_list, args.git_revision, args.outdir)
 278
 279     # FIXME: should we add a git version of this (e.g. --git-extra-driver)?
 280     for src, copy_list in args.extra_driver:
 281         copy_files(src, read_copy_list(open(copy_list, 'r')), args.outdir)
 282
 283     git_debug_init(args)
 284
 285     logwrite('Apply patches ...')
 286     patchdirs = []
 287     for root, dirs, files in os.walk(os.path.join(source_dir, 'patches')):
 288         if not dirs:
 289             patchdirs.append(root)
 290     patchdirs.sort()
 291     for pdir in patchdirs:
 292         l = os.listdir(pdir)
 293         printed = False
 294         for pfile in l:
 295             # FIXME: again, use .gitignore?
 296             if pfile[-1] == '~':
 297                 continue
 298             pfile = os.path.join(pdir, pfile)
 299             # read the patch file
 300             p = patch.fromfile(pfile)
 301             # if it is one ...
 302             if not p:
 303                 continue
 304             # check if the first file the patch touches exists, if so
 305             # assume the patch needs to be applied -- otherwise continue
 306             patched_file = '/'.join(p.items[0].source.split('/')[1:])
 307             fullfn = os.path.join(args.outdir, patched_file)
 308             if not os.path.exists(fullfn):
 309                 continue
 310             if not printed:
 311                 if args.verbose:
 312                     logwrite("Applying changes from %s" % os.path.basename(pdir))
 313                 printed = True
 314             if args.refresh:
 315                 # but for refresh, of course look at all files the patch touches
 316                 for patchitem in p.items:
 317                     patched_file = '/'.join(patchitem.source.split('/')[1:])
 318                     fullfn = os.path.join(args.outdir, patched_file)
 319                     shutil.copyfile(fullfn, fullfn + '.orig_file')
 320
 321             process = subprocess.Popen(['patch', '-p1'], stdout=subprocess.PIPE,
 322                                        stderr=subprocess.STDOUT, stdin=subprocess.PIPE,
 323                                        close_fds=True, universal_newlines=True,
 324                                        cwd=args.outdir)
 325             output = process.communicate(input=open(pfile, 'r').read())[0]
 326             output = output.split('\n')
 327             if output[-1] == '':
 328                 output = output[:-1]
 329             if args.verbose:
 330                 for line in output:
 331                     logwrite('> %s' % line)
 332             if process.returncode != 0:
 333                 if not args.verbose:
 334                     logwrite("Failed to apply changes from %s" % os.path.basename(pdir))
 335                     for line in output:
 336                         logwrite('> %s' % line)
 337                 return 2
 338
 339             if args.refresh:
 340                 pfilef = open(pfile + '.tmp', 'w')
 341                 for patchitem in p.items:
 342                     patched_file = '/'.join(patchitem.source.split('/')[1:])
 343                     fullfn = os.path.join(args.outdir, patched_file)
 344                     process = subprocess.Popen(['diff', '-u', patched_file + '.orig_file', patched_file,
 345                                                 '--label', 'a/' + patched_file,
 346                                                 '--label', 'b/' + patched_file],
 347                                                stdout=pfilef, close_fds=True,
 348                                                universal_newlines=True, cwd=args.outdir)
 349                     process.wait()
 350                     os.unlink(fullfn + '.orig_file')
 351                     if not process.returncode in (0, 1):
 352                         logwrite("Diffing for refresh failed!")
 353                         pfilef.close()
 354                         os.unlink(pfile + '.tmp')
 355                         return 3
 356                 pfilef.close()
 357                 os.rename(pfile + '.tmp', pfile)
 358
 359         # remove orig/rej files that patch sometimes creates
 360         for root, dirs, files in os.walk(args.outdir):
 361             for f in files:
 362                 if f[-5:] == '.orig' or f[-4:] == '.rej':
 363                     os.unlink(os.path.join(root, f))
 364         if not printed:
 365             if args.verbose:
 366                 logwrite("Not applying changes from %s, not needed" % (os.path.basename(pdir),))
 367         else:
 368             git_debug_snapshot(args, "apply backport patches from %s" % (os.path.basename(pdir),))
 369
 370     # some post-processing is required
 371     configtree = kconfig.ConfigTree(os.path.join(args.outdir, 'Kconfig'))
 372     logwrite('Modify Kconfig tree ...')
 373     configtree.prune_sources(ignore=['Kconfig.kernel', 'Kconfig.versions'])
 374     git_debug_snapshot(args, "prune Kconfig tree")
 375     configtree.force_tristate_modular()
 376     git_debug_snapshot(args, "force tristate options modular")
 377     configtree.modify_selects()
 378     git_debug_snapshot(args, "convert select to depends on")
 379
 380     # write the versioning file
 381     backports_version = backport_version_name or git.describe(tree=source_dir)
 382     kernel_version = kernel_version_name or git.describe(tree=args.kerneldir)
 383     f = open(os.path.join(args.outdir, 'versions'), 'w')
 384     f.write('BACKPORTS_VERSION="%s"\n' % backports_version)
 385     f.write('BACKPORTED_KERNEL_VERSION="%s"\n' % kernel_version)
 386     f.write('BACKPORTED_KERNEL_NAME="%s"\n' % args.base_name)
 387     f.close()
 388
 389     symbols = configtree.symbols()
 390
 391     # write local symbol list -- needed during build
 392     f = open(os.path.join(args.outdir, '.local-symbols'), 'w')
 393     for sym in symbols:
 394         f.write('%s=\n' % sym)
 395     f.close()
 396
 397     git_debug_snapshot(args, "add versions/symbols files")
 398
 399     logwrite('Rewrite Makefiles and Kconfig files ...')
 400
 401     # rewrite Makefile and source symbols
 402     regexes = []
 403     for some_symbols in [symbols[i:i + 50] for i in range(0, len(symbols), 50)]:
 404         r = 'CONFIG_((' + '|'.join([s + '(_MODULE)?' for s in some_symbols]) + ')([^A-Za-z0-9_]|$))'
 405         regexes.append(re.compile(r, re.MULTILINE))
 406     for root, dirs, files in os.walk(args.outdir):
 407         # don't go into .git dir (possible debug thing)
 408         if '.git' in dirs:
 409             dirs.remove('.git')
 410         for f in files:
 411             data = open(os.path.join(root, f), 'r').read()
 412             for r in regexes:
 413                 data = r.sub(r'CPTCFG_\1', data)
 414             fo = open(os.path.join(root, f), 'w')
 415             fo.write(data)
 416             fo.close()
 417
 418     git_debug_snapshot(args, "rename config symbol usage")
 419
 420     # disable unbuildable Kconfig symbols and stuff Makefiles that doesn't exist
 421     maketree = make.MakeTree(os.path.join(args.outdir, 'Makefile.kernel'))
 422     disable_kconfig = []
 423     disable_makefile = []
 424     for sym in maketree.get_impossible_symbols():
 425         disable_kconfig.append(sym[7:])
 426         disable_makefile.append(sym[7:])
 427
 428     configtree.disable_symbols(disable_kconfig)
 429     git_debug_snapshot(args, "disable impossible kconfig symbols")
 430
 431     # add kernel version dependencies to Kconfig, from the dependency list
 432     # we read previously
 433     for sym in tuple(deplist.keys()):
 434         new = []
 435         for dep in deplist[sym]:
 436             new.append('!BACKPORT_KERNEL_%s' % dep.replace('.', '_'))
 437         deplist[sym] = new
 438     configtree.add_dependencies(deplist)
 439     git_debug_snapshot(args, "add kernel version dependencies")
 440
 441     # disable things in makefiles that can't be selected and that the
 442     # build shouldn't recurse into because they don't exist -- if we
 443     # don't do that then a symbol from the kernel could cause the build
 444     # to attempt to recurse and fail
 445     #
 446     # Note that we split the regex after 50 symbols, this is because of a
 447     # limitation in the regex implementation (it only supports 100 nested
 448     # groups -- 50 seemed safer and is still fast)
 449     regexes = []
 450     for some_symbols in [disable_makefile[i:i + 50] for i in range(0, len(disable_makefile), 50)]:
 451         r = '((CPTCFG|CONFIG)_(' + '|'.join([s for s in some_symbols]) + '))'
 452         regexes.append(re.compile(r, re.MULTILINE))
 453     for f in maketree.get_makefiles():
 454         data = open(f, 'r').read()
 455         for r in regexes:
 456             data = r.sub(r'IMPOSSIBLE_\3', data)
 457         fo = open(f, 'w')
 458         fo.write(data)
 459         fo.close()
 460     git_debug_snapshot(args, "disable unsatisfied Makefile parts")
 461
 462     logwrite('Done!')
 463     return 0
 464
 465 if __name__ == '__main__':
 466     ret = _main()
 467     if ret:
 468         sys.exit(ret)