gentree.py

   1 #!/usr/bin/env python
   2 #
   3 # Generate the output tree into a specified directory.
   4 #
   5
   6 import argparse, sys, os, errno, shutil, re, subprocess
   7
   8 # find self
   9 source_dir = os.path.abspath(os.path.dirname(__file__))
  10 sys.path.append(source_dir)
  11 # and import libraries we have
  12 from lib import kconfig, git, patch, make
  13
  14
  15 def read_copy_list(copyfile):
  16     """
  17     Read a copy-list file and return a list of (source, target)
  18     tuples. The source and target are usually the same, but in
  19     the copy-list file there may be a rename included.
  20     """
  21     ret = []
  22     for item in copyfile:
  23         # remove leading/trailing whitespace
  24         item = item.strip()
  25         # comments
  26         if not item or item[0] == '#':
  27             continue
  28         if item[0] == '/':
  29             raise Exception("Input path '%s' is absolute path, this isn't allowed" % (item, ))
  30         if ' -> ' in item:
  31             srcitem, dstitem = item.split(' -> ')
  32             if (srcitem[-1] == '/') != (dstitem[-1] == '/'):
  33                 raise Exception("Cannot copy file/dir to dir/file")
  34         else:
  35             srcitem = dstitem = item
  36         ret.append((srcitem, dstitem))
  37     return ret
  38
  39
  40 def read_dependencies(depfilename):
  41     """
  42     Read a (the) dependency file and return the list of
  43     dependencies as a dictionary, mapping a Kconfig symbol
  44     to a list of kernel version dependencies. While reading
  45     ignore blank/commented lines.
  46     """
  47     ret = {}
  48     depfile = open(depfilename, 'r')
  49     for item in depfile:
  50         item = item.strip()
  51         if not item or item[0] == '#':
  52             continue
  53         sym, dep = item.split()
  54         if not sym in ret:
  55             ret[sym] = [dep, ]
  56         else:
  57             ret[sym].append(dep)
  58     return ret
  59
  60
  61 def check_output_dir(d, clean):
  62     """
  63     Check that the output directory doesn't exist or is empty,
  64     unless clean is True in which case it's nuked. This helps
  65     sanity check the output when generating a tree, so usually
  66     running with --clean isn't suggested.
  67     """
  68     if clean:
  69         shutil.rmtree(d, ignore_errors=True)
  70     try:
  71         os.rmdir(d)
  72     except OSError, e:
  73         if e.errno != errno.ENOENT:
  74             raise
  75
  76
  77 def copytree(src, dst, symlinks=False, ignore=None):
  78     """
  79     Copy a directory tree. This differs from shutil.copytree()
  80     in that it allows destination directories to already exist.
  81     """
  82     names = os.listdir(src)
  83     if ignore is not None:
  84         ignored_names = ignore(src, names)
  85     else:
  86         ignored_names = set()
  87
  88     if not os.path.isdir(dst):
  89         os.makedirs(dst)
  90     errors = []
  91     for name in names:
  92         if name in ignored_names:
  93             continue
  94         srcname = os.path.join(src, name)
  95         dstname = os.path.join(dst, name)
  96         try:
  97             if symlinks and os.path.islink(srcname):
  98                 linkto = os.readlink(srcname)
  99                 os.symlink(linkto, dstname)
 100             elif os.path.isdir(srcname):
 101                 copytree(srcname, dstname, symlinks, ignore)
 102             else:
 103                 shutil.copy2(srcname, dstname)
 104         except (IOError, os.error) as why:
 105             errors.append((srcname, dstname, str(why)))
 106         # catch the Error from the recursive copytree so that we can
 107         # continue with other files
 108         except shutil.Error as err:
 109             errors.extend(err.args[0])
 110     try:
 111         shutil.copystat(src, dst)
 112     except WindowsError:
 113         # can't copy file access times on Windows
 114         pass
 115     except OSError as why:
 116         errors.extend((src, dst, str(why)))
 117     if errors:
 118         raise shutil.Error(errors)
 119
 120
 121 def copy_files(srcpath, copy_list, outdir):
 122     """
 123     Copy the copy_list files and directories from the srcpath
 124     to the outdir. The copy_list contains source and target
 125     names.
 126
 127     For now, it also ignores any *~ editor backup files, though
 128     this should probably be generalized (maybe using .gitignore?)
 129     Similarly the code that only copies some files (*.c, *.h,
 130     *.awk, Kconfig, Makefile) to avoid any build remnants in the
 131     kernel if they should exist.
 132     """
 133     for srcitem, tgtitem in copy_list:
 134         if tgtitem == '':
 135             copytree(srcpath, outdir, ignore=shutil.ignore_patterns('*~'))
 136         elif tgtitem[-1] == '/':
 137             def copy_ignore(dir, entries):
 138                 r = []
 139                 for i in entries:
 140                     if i[-2] == '.o' or i[-1] == '~':
 141                         r.append(i)
 142                 return r
 143             copytree(os.path.join(srcpath, srcitem),
 144                      os.path.join(outdir, tgtitem),
 145                      ignore=copy_ignore)
 146         else:
 147             try:
 148                 os.makedirs(os.path.join(outdir, os.path.dirname(tgtitem)))
 149             except OSError, e:
 150                 # ignore dirs we might have created just now
 151                 if e.errno != errno.EEXIST:
 152                     raise
 153             shutil.copy(os.path.join(srcpath, srcitem),
 154                         os.path.join(outdir, tgtitem))
 155
 156
 157 def copy_git_files(srcpath, copy_list, rev, outdir):
 158     """
 159     "Copy" files from a git repository. This really means listing them with
 160     ls-tree and then using git show to obtain all the blobs.
 161     """
 162     for srcitem, tgtitem in copy_list:
 163         for m, t, h, f in git.ls_tree(rev=rev, files=(srcitem,), tree=srcpath):
 164             assert t == 'blob'
 165             f = os.path.join(outdir, f.replace(srcitem, tgtitem))
 166             d = os.path.dirname(f)
 167             if not os.path.exists(d):
 168                 os.makedirs(d)
 169             outf = open(f, 'w')
 170             git.get_blob(h, outf, tree=srcpath)
 171             outf.close()
 172             os.chmod(f, int(m, 8))
 173
 174
 175 def add_automatic_backports(args):
 176     export = re.compile(r'^EXPORT_SYMBOL(_GPL)?\((?P<sym>[^\)]*)\)')
 177     bpi = kconfig.get_backport_info(os.path.join(args.outdir, 'compat', 'Kconfig'))
 178     for sym, vals in bpi.iteritems():
 179         symtype, c_files, h_files = vals
 180
 181         # first copy files
 182         files = []
 183         for f in c_files:
 184             files.append((f, os.path.join('compat', os.path.basename(f))))
 185         for f in h_files:
 186             files.append((os.path.join('include', f),
 187                           os.path.join('include', os.path.dirname(f), 'backport-' + os.path.basename(f))))
 188         if args.git_revision:
 189             copy_git_files(args.kerneldir, files, args.git_revision, args.outdir)
 190         else:
 191             copy_files(args.kerneldir, files, args.outdir)
 192
 193         # now add the Makefile line
 194         mf = open(os.path.join(args.outdir, 'compat', 'Makefile'), 'a+')
 195         o_files = [os.path.basename(f)[:-1] + 'o' for f in c_files]
 196         if symtype == 'tristate':
 197             mf.write('obj-$(CPTCFG_%s) += %s\n' % (sym, ' '.join(o_files)))
 198         elif symtype == 'bool':
 199             mf.write('compat-$(CPTCFG_%s) += %s\n' % (sym, ' '.join(o_files)))
 200
 201         # finally create the include file
 202         syms = []
 203         for f in c_files:
 204             for l in open(os.path.join(args.outdir, 'compat', os.path.basename(f)), 'r'):
 205                 m = export.match(l)
 206                 if m:
 207                     syms.append(m.group('sym'))
 208         for f in h_files:
 209             outf = open(os.path.join(args.outdir, 'include', f), 'w')
 210             outf.write('/* Automatically created during backport process */\n')
 211             outf.write('#ifndef CPTCFG_%s\n' % sym)
 212             outf.write('#include_next <%s>\n' % f)
 213             outf.write('#else\n');
 214             for s in syms:
 215                 outf.write('#undef %s\n' % s)
 216                 outf.write('#define %s LINUX_BACKPORT(%s)\n' % (s, s))
 217             outf.write('#include <%s>\n' % (os.path.dirname(f) + '/backport-' + os.path.basename(f), ))
 218             outf.write('#endif /* CPTCFG_%s */\n' % sym)
 219
 220 def git_debug_init(args):
 221     """
 222     Initialize a git repository in the output directory and commit the current
 223     code in it. This is only used for debugging the transformations this code
 224     will do to the output later.
 225     """
 226     if not args.gitdebug:
 227         return
 228     git.init(tree=args.outdir)
 229     git.commit_all("Copied backport", tree=args.outdir)
 230
 231
 232 def git_debug_snapshot(args, name):
 233     """
 234     Take a git snapshot for the debugging.
 235     """
 236     if not args.gitdebug:
 237         return
 238     git.commit_all(name, tree=args.outdir)
 239
 240
 241 def _main():
 242     # set up and parse arguments
 243     parser = argparse.ArgumentParser(description='generate backport tree')
 244     parser.add_argument('kerneldir', metavar='<kernel tree>', type=str,
 245                         help='Kernel tree to copy drivers from')
 246     parser.add_argument('outdir', metavar='<output directory>', type=str,
 247                         help='Directory to write the generated tree to')
 248     parser.add_argument('--copy-list', metavar='<listfile>', type=argparse.FileType('r'),
 249                         default='copy-list',
 250                         help='File containing list of files/directories to copy, default "copy-list"')
 251     parser.add_argument('--git-revision', metavar='<revision>', type=str,
 252                         help='git commit revision (see gitrevisions(7)) to take objects from.' +
 253                              'If this is specified, the kernel tree is used as git object storage ' +
 254                              'and we use git ls-tree to get the files.')
 255     parser.add_argument('--clean', const=True, default=False, action="store_const",
 256                         help='Clean output directory instead of erroring if it isn\'t empty')
 257     parser.add_argument('--refresh', const=True, default=False, action="store_const",
 258                         help='Refresh patches as they are applied, the source dir will be modified!')
 259     parser.add_argument('--base-name', metavar='<name>', type=str, default='Linux',
 260                         help='name of base tree, default just "Linux"')
 261     parser.add_argument('--gitdebug', const=True, default=False, action="store_const",
 262                         help='Use git, in the output tree, to debug the various transformation steps ' +
 263                              'that the tree generation makes (apply patches, ...)')
 264     parser.add_argument('--verbose', const=True, default=False, action="store_const",
 265                         help='Print more verbose information')
 266     parser.add_argument('--extra-driver', nargs=2, metavar=('<source dir>', '<copy-list>'), type=str,
 267                         action='append', default=[], help='Extra driver directory/copy-list.')
 268     args = parser.parse_args()
 269
 270     def logwrite(msg):
 271         sys.stdout.write(msg)
 272         sys.stdout.write('\n')
 273         sys.stdout.flush()
 274
 275     return process(args.kerneldir, args.outdir, args.copy_list,
 276                    git_revision=args.git_revision, clean=args.clean,
 277                    refresh=args.refresh, base_name=args.base_name,
 278                    gitdebug=args.gitdebug, verbose=args.verbose,
 279                    extra_driver=args.extra_driver, logwrite=logwrite)
 280
 281 def process(kerneldir, outdir, copy_list_file, git_revision=None,
 282             clean=False, refresh=False, base_name="Linux", gitdebug=False,
 283             verbose=False, extra_driver=[], logwrite=lambda x:None,
 284             kernel_version_name=None, backport_version_name=None):
 285     class Args(object):
 286         def __init__(self, kerneldir, outdir, copy_list_file,
 287                      git_revision, clean, refresh, base_name,
 288                      gitdebug, verbose, extra_driver):
 289             self.kerneldir = kerneldir
 290             self.outdir = outdir
 291             self.copy_list = copy_list_file
 292             self.git_revision = git_revision
 293             self.clean = clean
 294             self.refresh = refresh
 295             self.base_name = base_name
 296             self.gitdebug = gitdebug
 297             self.verbose = verbose
 298             self.extra_driver = extra_driver
 299     args = Args(kerneldir, outdir, copy_list_file,
 300                 git_revision, clean, refresh, base_name,
 301                 gitdebug, verbose, extra_driver)
 302     # start processing ...
 303
 304     copy_list = read_copy_list(args.copy_list)
 305     deplist = read_dependencies(os.path.join(source_dir, 'dependencies'))
 306
 307     # validate output directory
 308     check_output_dir(args.outdir, args.clean)
 309
 310     # do the copy
 311     backport_files = [(x, x) for x in [
 312         'Kconfig', 'Makefile', 'Makefile.build', 'Makefile.kernel', '.gitignore',
 313         'Makefile.real', 'compat/', 'include/', 'kconfig/', 'defconfigs/',
 314     ]]
 315     if not args.git_revision:
 316         logwrite('Copy original source files ...')
 317     else:
 318         logwrite('Get original source files from git ...')
 319
 320     copy_files(os.path.join(source_dir, 'backport'), backport_files, args.outdir)
 321
 322     git_debug_init(args)
 323
 324     add_automatic_backports(args)
 325     git_debug_snapshot(args, 'Add automatic backports')
 326
 327     if not args.git_revision:
 328         copy_files(args.kerneldir, copy_list, args.outdir)
 329     else:
 330         copy_git_files(args.kerneldir, copy_list, args.git_revision, args.outdir)
 331
 332     # FIXME: should we add a git version of this (e.g. --git-extra-driver)?
 333     for src, copy_list in args.extra_driver:
 334         copy_files(src, read_copy_list(open(copy_list, 'r')), args.outdir)
 335
 336     git_debug_snapshot(args, 'Add driver sources')
 337
 338     logwrite('Apply patches ...')
 339     patchdirs = []
 340     for root, dirs, files in os.walk(os.path.join(source_dir, 'patches')):
 341         if not dirs:
 342             patchdirs.append(root)
 343     patchdirs.sort()
 344     for pdir in patchdirs:
 345         l = os.listdir(pdir)
 346         printed = False
 347         for pfile in l:
 348             # FIXME: again, use .gitignore?
 349             if pfile[-1] == '~':
 350                 continue
 351             pfile = os.path.join(pdir, pfile)
 352             # read the patch file
 353             p = patch.fromfile(pfile)
 354             # if it is one ...
 355             if not p:
 356                 continue
 357             # check if the first file the patch touches exists, if so
 358             # assume the patch needs to be applied -- otherwise continue
 359             patched_file = '/'.join(p.items[0].source.split('/')[1:])
 360             fullfn = os.path.join(args.outdir, patched_file)
 361             if not os.path.exists(fullfn):
 362                 continue
 363             if not printed:
 364                 if args.verbose:
 365                     logwrite("Applying changes from %s" % os.path.basename(pdir))
 366                 printed = True
 367             if args.refresh:
 368                 # but for refresh, of course look at all files the patch touches
 369                 for patchitem in p.items:
 370                     patched_file = '/'.join(patchitem.source.split('/')[1:])
 371                     fullfn = os.path.join(args.outdir, patched_file)
 372                     shutil.copyfile(fullfn, fullfn + '.orig_file')
 373
 374             process = subprocess.Popen(['patch', '-p1'], stdout=subprocess.PIPE,
 375                                        stderr=subprocess.STDOUT, stdin=subprocess.PIPE,
 376                                        close_fds=True, universal_newlines=True,
 377                                        cwd=args.outdir)
 378             output = process.communicate(input=open(pfile, 'r').read())[0]
 379             output = output.split('\n')
 380             if output[-1] == '':
 381                 output = output[:-1]
 382             if args.verbose:
 383                 for line in output:
 384                     logwrite('> %s' % line)
 385             if process.returncode != 0:
 386                 if not args.verbose:
 387                     logwrite("Failed to apply changes from %s" % os.path.basename(pdir))
 388                     for line in output:
 389                         logwrite('> %s' % line)
 390                 return 2
 391
 392             if args.refresh:
 393                 pfilef = open(pfile + '.tmp', 'w')
 394                 for patchitem in p.items:
 395                     patched_file = '/'.join(patchitem.source.split('/')[1:])
 396                     fullfn = os.path.join(args.outdir, patched_file)
 397                     process = subprocess.Popen(['diff', '-p', '-u', patched_file + '.orig_file', patched_file,
 398                                                 '--label', 'a/' + patched_file,
 399                                                 '--label', 'b/' + patched_file],
 400                                                stdout=pfilef, close_fds=True,
 401                                                universal_newlines=True, cwd=args.outdir)
 402                     process.wait()
 403                     os.unlink(fullfn + '.orig_file')
 404                     if not process.returncode in (0, 1):
 405                         logwrite("Diffing for refresh failed!")
 406                         pfilef.close()
 407                         os.unlink(pfile + '.tmp')
 408                         return 3
 409                 pfilef.close()
 410                 os.rename(pfile + '.tmp', pfile)
 411
 412         # remove orig/rej files that patch sometimes creates
 413         for root, dirs, files in os.walk(args.outdir):
 414             for f in files:
 415                 if f[-5:] == '.orig' or f[-4:] == '.rej':
 416                     os.unlink(os.path.join(root, f))
 417         if not printed:
 418             if args.verbose:
 419                 logwrite("Not applying changes from %s, not needed" % (os.path.basename(pdir),))
 420         else:
 421             git_debug_snapshot(args, "apply backport patches from %s" % (os.path.basename(pdir),))
 422
 423     # some post-processing is required
 424     configtree = kconfig.ConfigTree(os.path.join(args.outdir, 'Kconfig'))
 425     logwrite('Modify Kconfig tree ...')
 426     configtree.prune_sources(ignore=['Kconfig.kernel', 'Kconfig.versions'])
 427     git_debug_snapshot(args, "prune Kconfig tree")
 428     configtree.force_tristate_modular()
 429     git_debug_snapshot(args, "force tristate options modular")
 430     configtree.modify_selects()
 431     git_debug_snapshot(args, "convert select to depends on")
 432
 433     # write the versioning file
 434     backports_version = backport_version_name or git.describe(tree=source_dir)
 435     kernel_version = kernel_version_name or git.describe(rev=args.git_revision or 'HEAD',
 436                                                          tree=args.kerneldir)
 437     f = open(os.path.join(args.outdir, 'versions'), 'w')
 438     f.write('BACKPORTS_VERSION="%s"\n' % backports_version)
 439     f.write('BACKPORTED_KERNEL_VERSION="%s"\n' % kernel_version)
 440     f.write('BACKPORTED_KERNEL_NAME="%s"\n' % args.base_name)
 441     f.close()
 442
 443     symbols = configtree.symbols()
 444
 445     # write local symbol list -- needed during build
 446     f = open(os.path.join(args.outdir, '.local-symbols'), 'w')
 447     for sym in symbols:
 448         f.write('%s=\n' % sym)
 449     f.close()
 450
 451     git_debug_snapshot(args, "add versions/symbols files")
 452
 453     logwrite('Rewrite Makefiles and Kconfig files ...')
 454
 455     # rewrite Makefile and source symbols
 456     regexes = []
 457     for some_symbols in [symbols[i:i + 50] for i in range(0, len(symbols), 50)]:
 458         r = 'CONFIG_((' + '|'.join([s + '(_MODULE)?' for s in some_symbols]) + ')([^A-Za-z0-9_]|$))'
 459         regexes.append(re.compile(r, re.MULTILINE))
 460     for root, dirs, files in os.walk(args.outdir):
 461         # don't go into .git dir (possible debug thing)
 462         if '.git' in dirs:
 463             dirs.remove('.git')
 464         for f in files:
 465             data = open(os.path.join(root, f), 'r').read()
 466             for r in regexes:
 467                 data = r.sub(r'CPTCFG_\1', data)
 468             fo = open(os.path.join(root, f), 'w')
 469             fo.write(data)
 470             fo.close()
 471
 472     git_debug_snapshot(args, "rename config symbol usage")
 473
 474     # disable unbuildable Kconfig symbols and stuff Makefiles that doesn't exist
 475     maketree = make.MakeTree(os.path.join(args.outdir, 'Makefile.kernel'))
 476     disable_kconfig = []
 477     disable_makefile = []
 478     for sym in maketree.get_impossible_symbols():
 479         disable_kconfig.append(sym[7:])
 480         disable_makefile.append(sym[7:])
 481
 482     configtree.disable_symbols(disable_kconfig)
 483     git_debug_snapshot(args, "disable impossible kconfig symbols")
 484
 485     # add kernel version dependencies to Kconfig, from the dependency list
 486     # we read previously
 487     for sym in tuple(deplist.keys()):
 488         new = []
 489         for dep in deplist[sym]:
 490             new.append('!BACKPORT_KERNEL_%s' % dep.replace('.', '_'))
 491         deplist[sym] = new
 492     configtree.add_dependencies(deplist)
 493     git_debug_snapshot(args, "add kernel version dependencies")
 494
 495     # disable things in makefiles that can't be selected and that the
 496     # build shouldn't recurse into because they don't exist -- if we
 497     # don't do that then a symbol from the kernel could cause the build
 498     # to attempt to recurse and fail
 499     #
 500     # Note that we split the regex after 50 symbols, this is because of a
 501     # limitation in the regex implementation (it only supports 100 nested
 502     # groups -- 50 seemed safer and is still fast)
 503     regexes = []
 504     for some_symbols in [disable_makefile[i:i + 50] for i in range(0, len(disable_makefile), 50)]:
 505         r = '((CPTCFG|CONFIG)_(' + '|'.join([s for s in some_symbols]) + '))'
 506         regexes.append(re.compile(r, re.MULTILINE))
 507     for f in maketree.get_makefiles():
 508         data = open(f, 'r').read()
 509         for r in regexes:
 510             data = r.sub(r'IMPOSSIBLE_\3', data)
 511         fo = open(f, 'w')
 512         fo.write(data)
 513         fo.close()
 514     git_debug_snapshot(args, "disable unsatisfied Makefile parts")
 515
 516     logwrite('Done!')
 517     return 0
 518
 519 if __name__ == '__main__':
 520     ret = _main()
 521     if ret:
 522         sys.exit(ret)