gentree.py

   1 #!/usr/bin/env python
   2 #
   3 # Generate the output tree into a specified directory.
   4 #
   5
   6 import argparse, sys, os, errno, shutil, re, subprocess
   7
   8 # find self
   9 source_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
  10 sys.path.append(os.path.join(source_dir, 'lib'))
  11 # and import libraries we have
  12 import kconfig, git, patch, make
  13
  14
  15 def read_copy_list(copyfile):
  16     """
  17     Read a copy-list file and return a list of (source, target)
  18     tuples. The source and target are usually the same, but in
  19     the copy-list file there may be a rename included.
  20     """
  21     ret = []
  22     for item in copyfile:
  23         # remove leading/trailing whitespace
  24         item = item.strip()
  25         # comments
  26         if not item or item[0] == '#':
  27             continue
  28         if item[0] == '/':
  29             raise Exception("Input path '%s' is absolute path, this isn't allowed" % (item, ))
  30         if ' -> ' in item:
  31             srcitem, dstitem = item.split(' -> ')
  32             if (srcitem[-1] == '/') != (dstitem[-1] == '/'):
  33                 raise Exception("Cannot copy file/dir to dir/file")
  34         else:
  35             srcitem = dstitem = item
  36         ret.append((srcitem, dstitem))
  37     return ret
  38
  39
  40 def read_dependencies(depfilename):
  41     """
  42     Read a (the) dependency file and return the list of
  43     dependencies as a dictionary, mapping a Kconfig symbol
  44     to a list of kernel version dependencies. While reading
  45     ignore blank/commented lines.
  46     """
  47     ret = {}
  48     depfile = open(depfilename, 'r')
  49     for item in depfile:
  50         item = item.strip()
  51         if not item or item[0] == '#':
  52             continue
  53         sym, dep = item.split()
  54         if not sym in ret:
  55             ret[sym] = [dep, ]
  56         else:
  57             ret[sym].append(dep)
  58     return ret
  59
  60
  61 def check_output_dir(d, clean):
  62     """
  63     Check that the output directory doesn't exist or is empty,
  64     unless clean is True in which case it's nuked. This helps
  65     sanity check the output when generating a tree, so usually
  66     running with --clean isn't suggested.
  67     """
  68     if clean:
  69         shutil.rmtree(d, ignore_errors=True)
  70     try:
  71         os.rmdir(d)
  72     except OSError, e:
  73         if e.errno != errno.ENOENT:
  74             raise
  75
  76
  77 def copytree(src, dst, symlinks=False, ignore=None):
  78     """
  79     Copy a directory tree. This differs from shutil.copytree()
  80     in that it allows destination directories to already exist.
  81     """
  82     names = os.listdir(src)
  83     if ignore is not None:
  84         ignored_names = ignore(src, names)
  85     else:
  86         ignored_names = set()
  87
  88     if not os.path.isdir(dst):
  89         os.makedirs(dst)
  90     errors = []
  91     for name in names:
  92         if name in ignored_names:
  93             continue
  94         srcname = os.path.join(src, name)
  95         dstname = os.path.join(dst, name)
  96         try:
  97             if symlinks and os.path.islink(srcname):
  98                 linkto = os.readlink(srcname)
  99                 os.symlink(linkto, dstname)
 100             elif os.path.isdir(srcname):
 101                 copytree(srcname, dstname, symlinks, ignore)
 102             else:
 103                 shutil.copy2(srcname, dstname)
 104         except (IOError, os.error) as why:
 105             errors.append((srcname, dstname, str(why)))
 106         # catch the Error from the recursive copytree so that we can
 107         # continue with other files
 108         except shutil.Error as err:
 109             errors.extend(err.args[0])
 110     try:
 111         shutil.copystat(src, dst)
 112     except WindowsError:
 113         # can't copy file access times on Windows
 114         pass
 115     except OSError as why:
 116         errors.extend((src, dst, str(why)))
 117     if errors:
 118         raise shutil.Error(errors)
 119
 120
 121 def copy_files(srcpath, copy_list, outdir):
 122     """
 123     Copy the copy_list files and directories from the srcpath
 124     to the outdir. The copy_list contains source and target
 125     names.
 126
 127     For now, it also ignores any *~ editor backup files, though
 128     this should probably be generalized (maybe using .gitignore?)
 129     Similarly the code that only copies some files (*.c, *.h,
 130     *.awk, Kconfig, Makefile) to avoid any build remnants in the
 131     kernel if they should exist.
 132     """
 133     for srcitem, tgtitem in copy_list:
 134         if tgtitem == '':
 135             copytree(srcpath, outdir, ignore=shutil.ignore_patterns('*~'))
 136         elif tgtitem[-1] == '/':
 137             def copy_ignore(dir, entries):
 138                 r = []
 139                 for i in entries:
 140                     if (not i[-1] in ('c', 'h') and
 141                         i[-4:] != '.awk' and
 142                         not i in ('Kconfig', 'Makefile') and
 143                         not os.path.isdir(os.path.join(dir, i))):
 144                         r.append(i)
 145                 return r
 146             copytree(os.path.join(srcpath, srcitem),
 147                      os.path.join(outdir, tgtitem),
 148                      ignore=copy_ignore)
 149         else:
 150             try:
 151                 os.makedirs(os.path.join(outdir, os.path.dirname(tgtitem)))
 152             except OSError, e:
 153                 # ignore dirs we might have created just now
 154                 if e.errno != errno.EEXIST:
 155                     raise
 156             shutil.copy(os.path.join(srcpath, srcitem),
 157                         os.path.join(outdir, tgtitem))
 158
 159
 160 def copy_git_files(srcpath, copy_list, rev, outdir):
 161     """
 162     "Copy" files from a git repository. This really means listing them with
 163     ls-tree and then using git show to obtain all the blobs.
 164     """
 165     for srcitem, tgtitem in copy_list:
 166         for m, t, h, f in git.ls_tree(rev=rev, files=(srcitem,), tree=srcpath):
 167             assert t == 'blob'
 168             f = os.path.join(outdir, f)
 169             d = os.path.dirname(f)
 170             if not os.path.exists(d):
 171                 os.makedirs(d)
 172             outf = open(f, 'w')
 173             git.get_blob(h, outf, tree=srcpath)
 174             outf.close()
 175             os.chmod(f, int(m, 8))
 176
 177
 178 def git_debug_init(args):
 179     """
 180     Initialize a git repository in the output directory and commit the current
 181     code in it. This is only used for debugging the transformations this code
 182     will do to the output later.
 183     """
 184     if not args.gitdebug:
 185         return
 186     git.init(tree=args.outdir)
 187     git.commit_all("Copied code", tree=args.outdir)
 188
 189
 190 def git_debug_snapshot(args, name):
 191     """
 192     Take a git snapshot for the debugging.
 193     """
 194     if not args.gitdebug:
 195         return
 196     git.commit_all(name, tree=args.outdir)
 197
 198
 199 def main():
 200     # set up and parse arguments
 201     parser = argparse.ArgumentParser(description='generate backport tree')
 202     parser.add_argument('kerneldir', metavar='<kernel tree>', type=str,
 203                         help='Kernel tree to copy drivers from')
 204     parser.add_argument('outdir', metavar='<output directory>', type=str,
 205                         help='Directory to write the generated tree to')
 206     parser.add_argument('--copy-list', metavar='<listfile>', type=argparse.FileType('r'),
 207                         default='copy-list',
 208                         help='File containing list of files/directories to copy, default "copy-list"')
 209     parser.add_argument('--git-revision', metavar='<revision>', type=str,
 210                         help='git commit revision (see gitrevisions(7)) to take objects from.' +
 211                              'If this is specified, the kernel tree is used as git object storage ' +
 212                              'and we use git ls-tree to get the files.')
 213     parser.add_argument('--clean', const=True, default=False, action="store_const",
 214                         help='Clean output directory instead of erroring if it isn\'t empty')
 215     parser.add_argument('--refresh', const=True, default=False, action="store_const",
 216                         help='Refresh patches as they are applied, the source dir will be modified!')
 217     parser.add_argument('--base-name', metavar='<name>', type=str, default='Linux',
 218                         help='name of base tree, default just "Linux"')
 219     parser.add_argument('--gitdebug', const=True, default=False, action="store_const",
 220                         help='Use git, in the output tree, to debug the various transformation steps ' +
 221                              'that the tree generation makes (apply patches, ...)')
 222     parser.add_argument('--verbose', const=True, default=False, action="store_const",
 223                         help='Print more verbose information')
 224     parser.add_argument('--extra-driver', nargs=2, metavar=('<source dir>', '<copy-list>'), type=str,
 225                         action='append', default=[], help='Extra driver directory/copy-list.')
 226     args = parser.parse_args()
 227
 228     # start processing ...
 229
 230     copy_list = read_copy_list(args.copy_list)
 231     deplist = read_dependencies(os.path.join(source_dir, 'dependencies'))
 232
 233     # validate output directory
 234     check_output_dir(args.outdir, args.clean)
 235
 236     # do the copy
 237     backport_files = [(x, x) for x in [
 238         'Kconfig', 'Makefile', 'Makefile.build', 'Makefile.kernel',
 239         'Makefile.real', 'compat/', 'include/', 'kconfig/',
 240     ]]
 241     if not args.git_revision:
 242         print 'Copy original source files ...'
 243         copy_files(os.path.join(source_dir, 'backport'), backport_files, args.outdir)
 244         copy_files(args.kerneldir, copy_list, args.outdir)
 245     else:
 246         print 'Get original source files from git ...'
 247         copy_files(os.path.join(source_dir, 'backport'), backport_files, args.outdir)
 248         copy_git_files(args.kerneldir, copy_list, args.git_revision, args.outdir)
 249
 250     # FIXME: should we add a git version of this (e.g. --git-extra-driver)?
 251     for src, copy_list in args.extra_driver:
 252         copy_files(src, read_copy_list(open(copy_list, 'r')), args.outdir)
 253
 254     git_debug_init(args)
 255
 256     print 'Apply patches ...'
 257     patchdirs = []
 258     for root, dirs, files in os.walk(os.path.join(source_dir, 'patches')):
 259         if not dirs:
 260             patchdirs.append(root)
 261     patchdirs.sort()
 262     for pdir in patchdirs:
 263         l = os.listdir(pdir)
 264         printed = False
 265         for pfile in l:
 266             # FIXME: again, use .gitignore?
 267             if pfile[-1] == '~':
 268                 continue
 269             pfile = os.path.join(pdir, pfile)
 270             # read the patch file
 271             p = patch.fromfile(pfile)
 272             # if it is one ...
 273             if not p:
 274                 continue
 275             # check if the first file the patch touches exists, if so
 276             # assume the patch needs to be applied -- otherwise continue
 277             patched_file = '/'.join(p.items[0].source.split('/')[1:])
 278             fullfn = os.path.join(args.outdir, patched_file)
 279             if not os.path.exists(fullfn):
 280                 continue
 281             if not printed:
 282                 if args.verbose:
 283                     print "Applying changes from", os.path.basename(pdir)
 284                 printed = True
 285             if args.refresh:
 286                 # but for refresh, of course look at all files the patch touches
 287                 for patchitem in p.items:
 288                     patched_file = '/'.join(patchitem.source.split('/')[1:])
 289                     fullfn = os.path.join(args.outdir, patched_file)
 290                     shutil.copyfile(fullfn, fullfn + '.orig_file')
 291
 292             process = subprocess.Popen(['patch', '-p1'], stdout=subprocess.PIPE,
 293                                        stderr=subprocess.STDOUT, stdin=subprocess.PIPE,
 294                                        close_fds=True, universal_newlines=True,
 295                                        cwd=args.outdir)
 296             output = process.communicate(input=open(pfile, 'r').read())[0]
 297             output = output.split('\n')
 298             if output[-1] == '':
 299                 output = output[:-1]
 300             if args.verbose:
 301                 for line in output:
 302                     print '>', line
 303             if process.returncode != 0:
 304                 if not args.verbose:
 305                     print "Failed to apply changes from", os.path.basename(pdir)
 306                     for line in output:
 307                         print '>', line
 308                 return 2
 309
 310             if args.refresh:
 311                 pfilef = open(pfile + '.tmp', 'w')
 312                 for patchitem in p.items:
 313                     patched_file = '/'.join(patchitem.source.split('/')[1:])
 314                     fullfn = os.path.join(args.outdir, patched_file)
 315                     process = subprocess.Popen(['diff', '-u', patched_file + '.orig_file', patched_file,
 316                                                 '--label', 'a/' + patched_file,
 317                                                 '--label', 'b/' + patched_file],
 318                                                stdout=pfilef, close_fds=True,
 319                                                universal_newlines=True, cwd=args.outdir)
 320                     process.wait()
 321                     os.unlink(fullfn + '.orig_file')
 322                     if not process.returncode in (0, 1):
 323                         print "Diffing for refresh failed!"
 324                         pfilef.close()
 325                         os.unlink(pfile + '.tmp')
 326                         return 3
 327                 pfilef.close()
 328                 os.rename(pfile + '.tmp', pfile)
 329
 330         # remove orig/rej files that patch sometimes creates
 331         for root, dirs, files in os.walk(args.outdir):
 332             for f in files:
 333                 if f[-5:] == '.orig' or f[-4:] == '.rej':
 334                     os.unlink(os.path.join(root, f))
 335         if not printed:
 336             if args.verbose:
 337                 print "Not applying changes from %s, not needed" % (os.path.basename(pdir),)
 338         else:
 339             git_debug_snapshot(args, "apply backport patches from %s" % (os.path.basename(pdir),))
 340
 341     # some post-processing is required
 342     configtree = kconfig.ConfigTree(os.path.join(args.outdir, 'Kconfig'))
 343     print 'Modify Kconfig tree ...'
 344     configtree.prune_sources(ignore=['Kconfig.kernel', 'Kconfig.versions'])
 345     git_debug_snapshot(args, "prune Kconfig tree")
 346     configtree.force_tristate_modular()
 347     git_debug_snapshot(args, "force tristate options modular")
 348     configtree.modify_selects()
 349     git_debug_snapshot(args, "convert select to depends on")
 350
 351     # write the versioning file
 352     backports_version = git.describe(tree=source_dir)
 353     kernel_version = git.describe(tree=args.kerneldir)
 354     f = open(os.path.join(args.outdir, 'versions'), 'w')
 355     f.write('BACKPORTS_VERSION="%s"\n' % backports_version)
 356     f.write('KERNEL_VERSION="%s"\n' % kernel_version)
 357     f.write('KERNEL_NAME="%s"\n' % args.base_name)
 358     f.close()
 359
 360     symbols = configtree.symbols()
 361
 362     # write local symbol list -- needed during build
 363     f = open(os.path.join(args.outdir, '.local-symbols'), 'w')
 364     for sym in symbols:
 365         f.write('%s=\n' % sym)
 366     f.close()
 367
 368     git_debug_snapshot(args, "add versions/symbols files")
 369
 370     print 'Rewrite Makefiles and Kconfig files ...'
 371
 372     # rewrite Makefile and source symbols
 373     regexes = []
 374     for some_symbols in [symbols[i:i + 50] for i in range(0, len(symbols), 50)]:
 375         r = 'CONFIG_((' + '|'.join([s + '(_MODULE)?' for s in some_symbols]) + ')([^A-Za-z0-9_]|$))'
 376         regexes.append(re.compile(r, re.MULTILINE))
 377     for root, dirs, files in os.walk(args.outdir):
 378         # don't go into .git dir (possible debug thing)
 379         if '.git' in dirs:
 380             dirs.remove('.git')
 381         for f in files:
 382             data = open(os.path.join(root, f), 'r').read()
 383             for r in regexes:
 384                 data = r.sub(r'CPTCFG_\1', data)
 385             fo = open(os.path.join(root, f), 'w')
 386             fo.write(data)
 387             fo.close()
 388
 389     git_debug_snapshot(args, "rename config symbol usage")
 390
 391     # disable unbuildable Kconfig symbols and stuff Makefiles that doesn't exist
 392     maketree = make.MakeTree(os.path.join(args.outdir, 'Makefile.kernel'))
 393     disable_kconfig = []
 394     disable_makefile = []
 395     for sym in maketree.get_impossible_symbols():
 396         if sym[:7] == 'CPTCFG_':
 397             disable_kconfig.append(sym[7:])
 398         else:
 399             disable_makefile.append(sym[7:])
 400
 401     configtree.disable_symbols(disable_kconfig)
 402     git_debug_snapshot(args, "disable impossible kconfig symbols")
 403
 404     # add kernel version dependencies to Kconfig, from the dependency list
 405     # we read previously
 406     for sym in tuple(deplist.keys()):
 407         new = []
 408         for dep in deplist[sym]:
 409             new.append('!BACKPORT_KERNEL_%s' % dep.replace('.', '_'))
 410         deplist[sym] = new
 411     configtree.add_dependencies(deplist)
 412     git_debug_snapshot(args, "add kernel version dependencies")
 413
 414     # disable things in makefiles that can't be selected and that the
 415     # build shouldn't recurse into because they don't exist -- if we
 416     # don't do that then a symbol from the kernel could cause the build
 417     # to attempt to recurse and fail
 418     #
 419     # Note that we split the regex after 50 symbols, this is because of a
 420     # limitation in the regex implementation (it only supports 100 nested
 421     # groups -- 50 seemed safer and is still fast)
 422     regexes = []
 423     for some_symbols in [disable_makefile[i:i + 50] for i in range(0, len(disable_makefile), 50)]:
 424         r = '(CONFIG_(' + '|'.join([s for s in some_symbols]) + '))'
 425         regexes.append(re.compile(r, re.MULTILINE))
 426     for f in maketree.get_makefiles():
 427         data = open(f, 'r').read()
 428         for r in regexes:
 429             data = r.sub(r'IMPOSSIBLE_\2', data)
 430         fo = open(f, 'w')
 431         fo.write(data)
 432         fo.close()
 433     git_debug_snapshot(args, "disable unsatisfied Makefile parts")
 434
 435     print 'Done!'
 436
 437 ret = main()
 438 if ret:
 439     sys.exit(ret)