gentree.py

   1 #!/usr/bin/env python
   2 #
   3 # Generate the output tree into a specified directory.
   4 #
   5
   6 import argparse, sys, os, errno, shutil, re, subprocess
   7
   8 # find self
   9 source_dir = os.path.abspath(os.path.dirname(__file__))
  10 sys.path.append(source_dir)
  11 # and import libraries we have
  12 from lib import kconfig, patch, make
  13 from lib import bpgit as git
  14
  15 def read_copy_list(copyfile):
  16     """
  17     Read a copy-list file and return a list of (source, target)
  18     tuples. The source and target are usually the same, but in
  19     the copy-list file there may be a rename included.
  20     """
  21     ret = []
  22     for item in copyfile:
  23         # remove leading/trailing whitespace
  24         item = item.strip()
  25         # comments
  26         if not item or item[0] == '#':
  27             continue
  28         if item[0] == '/':
  29             raise Exception("Input path '%s' is absolute path, this isn't allowed" % (item, ))
  30         if ' -> ' in item:
  31             srcitem, dstitem = item.split(' -> ')
  32             if (srcitem[-1] == '/') != (dstitem[-1] == '/'):
  33                 raise Exception("Cannot copy file/dir to dir/file")
  34         else:
  35             srcitem = dstitem = item
  36         ret.append((srcitem, dstitem))
  37     return ret
  38
  39
  40 def read_dependencies(depfilename):
  41     """
  42     Read a (the) dependency file and return the list of
  43     dependencies as a dictionary, mapping a Kconfig symbol
  44     to a list of kernel version dependencies. While reading
  45     ignore blank/commented lines.
  46     """
  47     ret = {}
  48     depfile = open(depfilename, 'r')
  49     for item in depfile:
  50         item = item.strip()
  51         if not item or item[0] == '#':
  52             continue
  53         sym, dep = item.split()
  54         if not sym in ret:
  55             ret[sym] = [dep, ]
  56         else:
  57             ret[sym].append(dep)
  58     return ret
  59
  60
  61 def check_output_dir(d, clean):
  62     """
  63     Check that the output directory doesn't exist or is empty,
  64     unless clean is True in which case it's nuked. This helps
  65     sanity check the output when generating a tree, so usually
  66     running with --clean isn't suggested.
  67     """
  68     if clean:
  69         shutil.rmtree(d, ignore_errors=True)
  70     try:
  71         os.rmdir(d)
  72     except OSError, e:
  73         if e.errno != errno.ENOENT:
  74             raise
  75
  76
  77 def copytree(src, dst, symlinks=False, ignore=None):
  78     """
  79     Copy a directory tree. This differs from shutil.copytree()
  80     in that it allows destination directories to already exist.
  81     """
  82     names = os.listdir(src)
  83     if ignore is not None:
  84         ignored_names = ignore(src, names)
  85     else:
  86         ignored_names = set()
  87
  88     if not os.path.isdir(dst):
  89         os.makedirs(dst)
  90     errors = []
  91     for name in names:
  92         if name in ignored_names:
  93             continue
  94         srcname = os.path.join(src, name)
  95         dstname = os.path.join(dst, name)
  96         try:
  97             if symlinks and os.path.islink(srcname):
  98                 linkto = os.readlink(srcname)
  99                 os.symlink(linkto, dstname)
 100             elif os.path.isdir(srcname):
 101                 copytree(srcname, dstname, symlinks, ignore)
 102             else:
 103                 shutil.copy2(srcname, dstname)
 104         except (IOError, os.error) as why:
 105             errors.append((srcname, dstname, str(why)))
 106         # catch the Error from the recursive copytree so that we can
 107         # continue with other files
 108         except shutil.Error as err:
 109             errors.extend(err.args[0])
 110     try:
 111         shutil.copystat(src, dst)
 112     except WindowsError:
 113         # can't copy file access times on Windows
 114         pass
 115     except OSError as why:
 116         errors.extend((src, dst, str(why)))
 117     if errors:
 118         raise shutil.Error(errors)
 119
 120
 121 def copy_files(srcpath, copy_list, outdir):
 122     """
 123     Copy the copy_list files and directories from the srcpath
 124     to the outdir. The copy_list contains source and target
 125     names.
 126
 127     For now, it also ignores any *~ editor backup files, though
 128     this should probably be generalized (maybe using .gitignore?)
 129     Similarly the code that only copies some files (*.c, *.h,
 130     *.awk, Kconfig, Makefile) to avoid any build remnants in the
 131     kernel if they should exist.
 132     """
 133     for srcitem, tgtitem in copy_list:
 134         if tgtitem == '':
 135             copytree(srcpath, outdir, ignore=shutil.ignore_patterns('*~'))
 136         elif tgtitem[-1] == '/':
 137             def copy_ignore(dir, entries):
 138                 r = []
 139                 for i in entries:
 140                     if i[-2:] == '.o' or i[-1] == '~':
 141                         r.append(i)
 142                 return r
 143             copytree(os.path.join(srcpath, srcitem),
 144                      os.path.join(outdir, tgtitem),
 145                      ignore=copy_ignore)
 146         else:
 147             try:
 148                 os.makedirs(os.path.join(outdir, os.path.dirname(tgtitem)))
 149             except OSError, e:
 150                 # ignore dirs we might have created just now
 151                 if e.errno != errno.EEXIST:
 152                     raise
 153             shutil.copy(os.path.join(srcpath, srcitem),
 154                         os.path.join(outdir, tgtitem))
 155
 156
 157 def copy_git_files(srcpath, copy_list, rev, outdir):
 158     """
 159     "Copy" files from a git repository. This really means listing them with
 160     ls-tree and then using git show to obtain all the blobs.
 161     """
 162     for srcitem, tgtitem in copy_list:
 163         for m, t, h, f in git.ls_tree(rev=rev, files=(srcitem,), tree=srcpath):
 164             assert t == 'blob'
 165             f = os.path.join(outdir, f.replace(srcitem, tgtitem))
 166             d = os.path.dirname(f)
 167             if not os.path.exists(d):
 168                 os.makedirs(d)
 169             outf = open(f, 'w')
 170             git.get_blob(h, outf, tree=srcpath)
 171             outf.close()
 172             os.chmod(f, int(m, 8))
 173
 174 def automatic_backport_mangle_c_file(name):
 175     return name.replace('/', '-')
 176
 177
 178 def add_automatic_backports(args):
 179     export = re.compile(r'^EXPORT_SYMBOL(_GPL)?\((?P<sym>[^\)]*)\)')
 180     bpi = kconfig.get_backport_info(os.path.join(args.outdir, 'compat', 'Kconfig'))
 181     configtree = kconfig.ConfigTree(os.path.join(args.outdir, 'Kconfig'))
 182     all_selects = configtree.all_selects()
 183     for sym, vals in bpi.iteritems():
 184         if sym.startswith('BACKPORT_BUILD_'):
 185             if not sym[15:] in all_selects:
 186                 continue
 187         symtype, module_name, c_files, h_files = vals
 188
 189         # first copy files
 190         files = []
 191         for f in c_files:
 192             files.append((f, os.path.join('compat', automatic_backport_mangle_c_file(f))))
 193         for f in h_files:
 194             files.append((os.path.join('include', f),
 195                           os.path.join('include', os.path.dirname(f), 'backport-' + os.path.basename(f))))
 196         if args.git_revision:
 197             copy_git_files(args.kerneldir, files, args.git_revision, args.outdir)
 198         else:
 199             copy_files(args.kerneldir, files, args.outdir)
 200
 201         # now add the Makefile line
 202         mf = open(os.path.join(args.outdir, 'compat', 'Makefile'), 'a+')
 203         o_files = [automatic_backport_mangle_c_file(f)[:-1] + 'o' for f in c_files]
 204         if symtype == 'tristate':
 205             if not module_name:
 206                 raise Exception('backporting a module requires a #module-name')
 207             for of in o_files:
 208                 mf.write('%s-objs += %s\n' % (module_name, of))
 209             mf.write('obj-$(CPTCFG_%s) += %s.o\n' % (sym, module_name))
 210         elif symtype == 'bool':
 211             mf.write('compat-$(CPTCFG_%s) += %s\n' % (sym, ' '.join(o_files)))
 212
 213         # finally create the include file
 214         syms = []
 215         for f in c_files:
 216             for l in open(os.path.join(args.outdir, 'compat',
 217                                        automatic_backport_mangle_c_file(f)), 'r'):
 218                 m = export.match(l)
 219                 if m:
 220                     syms.append(m.group('sym'))
 221         for f in h_files:
 222             outf = open(os.path.join(args.outdir, 'include', f), 'w')
 223             outf.write('/* Automatically created during backport process */\n')
 224             outf.write('#ifndef CPTCFG_%s\n' % sym)
 225             outf.write('#include_next <%s>\n' % f)
 226             outf.write('#else\n');
 227             for s in syms:
 228                 outf.write('#undef %s\n' % s)
 229                 outf.write('#define %s LINUX_BACKPORT(%s)\n' % (s, s))
 230             outf.write('#include <%s>\n' % (os.path.dirname(f) + '/backport-' + os.path.basename(f), ))
 231             outf.write('#endif /* CPTCFG_%s */\n' % sym)
 232
 233 def git_debug_init(args):
 234     """
 235     Initialize a git repository in the output directory and commit the current
 236     code in it. This is only used for debugging the transformations this code
 237     will do to the output later.
 238     """
 239     if not args.gitdebug:
 240         return
 241     git.init(tree=args.outdir)
 242     git.commit_all("Copied backport", tree=args.outdir)
 243
 244
 245 def git_debug_snapshot(args, name):
 246     """
 247     Take a git snapshot for the debugging.
 248     """
 249     if not args.gitdebug:
 250         return
 251     git.commit_all(name, tree=args.outdir)
 252
 253
 254 def _main():
 255     # set up and parse arguments
 256     parser = argparse.ArgumentParser(description='generate backport tree')
 257     parser.add_argument('kerneldir', metavar='<kernel tree>', type=str,
 258                         help='Kernel tree to copy drivers from')
 259     parser.add_argument('outdir', metavar='<output directory>', type=str,
 260                         help='Directory to write the generated tree to')
 261     parser.add_argument('--copy-list', metavar='<listfile>', type=argparse.FileType('r'),
 262                         default='copy-list',
 263                         help='File containing list of files/directories to copy, default "copy-list"')
 264     parser.add_argument('--git-revision', metavar='<revision>', type=str,
 265                         help='git commit revision (see gitrevisions(7)) to take objects from.' +
 266                              'If this is specified, the kernel tree is used as git object storage ' +
 267                              'and we use git ls-tree to get the files.')
 268     parser.add_argument('--clean', const=True, default=False, action="store_const",
 269                         help='Clean output directory instead of erroring if it isn\'t empty')
 270     parser.add_argument('--refresh', const=True, default=False, action="store_const",
 271                         help='Refresh patches as they are applied, the source dir will be modified!')
 272     parser.add_argument('--base-name', metavar='<name>', type=str, default='Linux',
 273                         help='name of base tree, default just "Linux"')
 274     parser.add_argument('--gitdebug', const=True, default=False, action="store_const",
 275                         help='Use git, in the output tree, to debug the various transformation steps ' +
 276                              'that the tree generation makes (apply patches, ...)')
 277     parser.add_argument('--verbose', const=True, default=False, action="store_const",
 278                         help='Print more verbose information')
 279     parser.add_argument('--extra-driver', nargs=2, metavar=('<source dir>', '<copy-list>'), type=str,
 280                         action='append', default=[], help='Extra driver directory/copy-list.')
 281     args = parser.parse_args()
 282
 283     def logwrite(msg):
 284         sys.stdout.write(msg)
 285         sys.stdout.write('\n')
 286         sys.stdout.flush()
 287
 288     return process(args.kerneldir, args.outdir, args.copy_list,
 289                    git_revision=args.git_revision, clean=args.clean,
 290                    refresh=args.refresh, base_name=args.base_name,
 291                    gitdebug=args.gitdebug, verbose=args.verbose,
 292                    extra_driver=args.extra_driver, logwrite=logwrite)
 293
 294 def process(kerneldir, outdir, copy_list_file, git_revision=None,
 295             clean=False, refresh=False, base_name="Linux", gitdebug=False,
 296             verbose=False, extra_driver=[], logwrite=lambda x:None,
 297             git_tracked_version=False):
 298     class Args(object):
 299         def __init__(self, kerneldir, outdir, copy_list_file,
 300                      git_revision, clean, refresh, base_name,
 301                      gitdebug, verbose, extra_driver):
 302             self.kerneldir = kerneldir
 303             self.outdir = outdir
 304             self.copy_list = copy_list_file
 305             self.git_revision = git_revision
 306             self.clean = clean
 307             self.refresh = refresh
 308             self.base_name = base_name
 309             self.gitdebug = gitdebug
 310             self.verbose = verbose
 311             self.extra_driver = extra_driver
 312     args = Args(kerneldir, outdir, copy_list_file,
 313                 git_revision, clean, refresh, base_name,
 314                 gitdebug, verbose, extra_driver)
 315     # start processing ...
 316
 317     copy_list = read_copy_list(args.copy_list)
 318     deplist = read_dependencies(os.path.join(source_dir, 'dependencies'))
 319
 320     # validate output directory
 321     check_output_dir(args.outdir, args.clean)
 322
 323     # do the copy
 324     backport_files = [(x, x) for x in [
 325         'Kconfig', 'Makefile', 'Makefile.build', 'Makefile.kernel', '.gitignore',
 326         'Makefile.real', 'compat/', 'backport-include/', 'kconf/', 'defconfigs/',
 327         'scripts/', '.blacklist.map', 'udev/',
 328     ]]
 329     if not args.git_revision:
 330         logwrite('Copy original source files ...')
 331     else:
 332         logwrite('Get original source files from git ...')
 333
 334     copy_files(os.path.join(source_dir, 'backport'), backport_files, args.outdir)
 335
 336     git_debug_init(args)
 337
 338     if not args.git_revision:
 339         copy_files(args.kerneldir, copy_list, args.outdir)
 340     else:
 341         copy_git_files(args.kerneldir, copy_list, args.git_revision, args.outdir)
 342
 343     # FIXME: should we add a git version of this (e.g. --git-extra-driver)?
 344     for src, copy_list in args.extra_driver:
 345         copy_files(src, read_copy_list(open(copy_list, 'r')), args.outdir)
 346
 347     git_debug_snapshot(args, 'Add driver sources')
 348
 349     add_automatic_backports(args)
 350     git_debug_snapshot(args, 'Add automatic backports')
 351
 352     logwrite('Apply patches ...')
 353     patches = []
 354     for root, dirs, files in os.walk(os.path.join(source_dir, 'patches')):
 355         for f in files:
 356             if f.endswith('.patch'):
 357                 patches.append(os.path.join(root, f))
 358     patches.sort()
 359     prefix_len = len(os.path.join(source_dir, 'patches')) + 1
 360     for pfile in patches:
 361         print_name = pfile[prefix_len:]
 362         # read the patch file
 363         p = patch.fromfile(pfile)
 364         # complain if it's not a patch
 365         if not p:
 366             raise Exception('No patch content found in %s' % print_name)
 367         # leading / seems to be stripped?
 368         if 'dev/null' in p.items[0].source:
 369             raise Exception('Patches creating files are not supported (in %s)' % print_name)
 370         # check if the first file the patch touches exists, if so
 371         # assume the patch needs to be applied -- otherwise continue
 372         patched_file = '/'.join(p.items[0].source.split('/')[1:])
 373         fullfn = os.path.join(args.outdir, patched_file)
 374         if not os.path.exists(fullfn):
 375             if args.verbose:
 376                 logwrite("Not applying %s, not needed" % print_name)
 377             continue
 378         if args.verbose:
 379             logwrite("Applying patch %s" % print_name)
 380
 381         if args.refresh:
 382             # but for refresh, of course look at all files the patch touches
 383             for patchitem in p.items:
 384                 patched_file = '/'.join(patchitem.source.split('/')[1:])
 385                 fullfn = os.path.join(args.outdir, patched_file)
 386                 shutil.copyfile(fullfn, fullfn + '.orig_file')
 387
 388         process = subprocess.Popen(['patch', '-p1'], stdout=subprocess.PIPE,
 389                                    stderr=subprocess.STDOUT, stdin=subprocess.PIPE,
 390                                    close_fds=True, universal_newlines=True,
 391                                    cwd=args.outdir)
 392         output = process.communicate(input=open(pfile, 'r').read())[0]
 393         output = output.split('\n')
 394         if output[-1] == '':
 395             output = output[:-1]
 396         if args.verbose:
 397             for line in output:
 398                 logwrite('> %s' % line)
 399         if process.returncode != 0:
 400             if not args.verbose:
 401                 logwrite("Failed to apply changes from %s" % print_name)
 402                 for line in output:
 403                     logwrite('> %s' % line)
 404             return 2
 405
 406         if args.refresh:
 407             pfilef = open(pfile + '.tmp', 'a')
 408             pfilef.write(p.top_header)
 409             pfilef.flush()
 410             for patchitem in p.items:
 411                 patched_file = '/'.join(patchitem.source.split('/')[1:])
 412                 fullfn = os.path.join(args.outdir, patched_file)
 413                 process = subprocess.Popen(['diff', '-p', '-u', patched_file + '.orig_file', patched_file,
 414                                             '--label', 'a/' + patched_file,
 415                                             '--label', 'b/' + patched_file],
 416                                            stdout=pfilef, close_fds=True,
 417                                            universal_newlines=True, cwd=args.outdir)
 418                 process.wait()
 419                 os.unlink(fullfn + '.orig_file')
 420                 if not process.returncode in (0, 1):
 421                     logwrite("Failed to diff to refresh %s" % print_name)
 422                     pfilef.close()
 423                     os.unlink(pfile + '.tmp')
 424                     return 3
 425             pfilef.close()
 426             os.rename(pfile + '.tmp', pfile)
 427
 428         # remove orig/rej files that patch sometimes creates
 429         for root, dirs, files in os.walk(args.outdir):
 430             for f in files:
 431                 if f[-5:] == '.orig' or f[-4:] == '.rej':
 432                     os.unlink(os.path.join(root, f))
 433         git_debug_snapshot(args, "apply backport patch %s" % print_name)
 434
 435     # some post-processing is required
 436     configtree = kconfig.ConfigTree(os.path.join(args.outdir, 'Kconfig'))
 437     logwrite('Modify Kconfig tree ...')
 438     configtree.prune_sources(ignore=['Kconfig.kernel', 'Kconfig.versions'])
 439     git_debug_snapshot(args, "prune Kconfig tree")
 440     configtree.force_tristate_modular()
 441     git_debug_snapshot(args, "force tristate options modular")
 442     configtree.modify_selects()
 443     git_debug_snapshot(args, "convert select to depends on")
 444
 445     # write the versioning file
 446     if git_tracked_version:
 447         backports_version = "(see git)"
 448         kernel_version = "(see git)"
 449     else:
 450         backports_version = git.describe(tree=source_dir)
 451         kernel_version = git.describe(rev=args.git_revision or 'HEAD',
 452                                       tree=args.kerneldir)
 453     f = open(os.path.join(args.outdir, 'versions'), 'w')
 454     f.write('BACKPORTS_VERSION="%s"\n' % backports_version)
 455     f.write('BACKPORTED_KERNEL_VERSION="%s"\n' % kernel_version)
 456     f.write('BACKPORTED_KERNEL_NAME="%s"\n' % args.base_name)
 457     if git_tracked_version:
 458         f.write('BACKPORTS_GIT_TRACKED="backport tracker ID: $(shell git rev-parse HEAD 2>/dev/null || echo \'not built in git tree\')"\n')
 459     f.close()
 460
 461     symbols = configtree.symbols()
 462
 463     # write local symbol list -- needed during build
 464     f = open(os.path.join(args.outdir, '.local-symbols'), 'w')
 465     for sym in symbols:
 466         f.write('%s=\n' % sym)
 467     f.close()
 468
 469     git_debug_snapshot(args, "add versions/symbols files")
 470
 471     logwrite('Rewrite Makefiles and Kconfig files ...')
 472
 473     # rewrite Makefile and source symbols
 474     regexes = []
 475     for some_symbols in [symbols[i:i + 50] for i in range(0, len(symbols), 50)]:
 476         r = 'CONFIG_((' + '|'.join([s + '(_MODULE)?' for s in some_symbols]) + ')([^A-Za-z0-9_]|$))'
 477         regexes.append(re.compile(r, re.MULTILINE))
 478     for root, dirs, files in os.walk(args.outdir):
 479         # don't go into .git dir (possible debug thing)
 480         if '.git' in dirs:
 481             dirs.remove('.git')
 482         for f in files:
 483             data = open(os.path.join(root, f), 'r').read()
 484             for r in regexes:
 485                 data = r.sub(r'CPTCFG_\1', data)
 486             data = re.sub(r'\$\(srctree\)', '$(backport_srctree)', data)
 487             data = re.sub(r'-Idrivers', '-I$(backport_srctree)/drivers', data)
 488             fo = open(os.path.join(root, f), 'w')
 489             fo.write(data)
 490             fo.close()
 491
 492     git_debug_snapshot(args, "rename config symbol / srctree usage")
 493
 494     # disable unbuildable Kconfig symbols and stuff Makefiles that doesn't exist
 495     maketree = make.MakeTree(os.path.join(args.outdir, 'Makefile.kernel'))
 496     disable_kconfig = []
 497     disable_makefile = []
 498     for sym in maketree.get_impossible_symbols():
 499         disable_kconfig.append(sym[7:])
 500         disable_makefile.append(sym[7:])
 501
 502     configtree.disable_symbols(disable_kconfig)
 503     git_debug_snapshot(args, "disable impossible kconfig symbols")
 504
 505     # add kernel version dependencies to Kconfig, from the dependency list
 506     # we read previously
 507     for sym in tuple(deplist.keys()):
 508         new = []
 509         for dep in deplist[sym]:
 510             if dep == "DISABLE":
 511                     new.append('BACKPORT_DISABLED_KCONFIG_OPTION')
 512             else:
 513                     new.append('!BACKPORT_KERNEL_%s' % dep.replace('.', '_'))
 514         deplist[sym] = new
 515     configtree.add_dependencies(deplist)
 516     git_debug_snapshot(args, "add kernel version dependencies")
 517
 518     # disable things in makefiles that can't be selected and that the
 519     # build shouldn't recurse into because they don't exist -- if we
 520     # don't do that then a symbol from the kernel could cause the build
 521     # to attempt to recurse and fail
 522     #
 523     # Note that we split the regex after 50 symbols, this is because of a
 524     # limitation in the regex implementation (it only supports 100 nested
 525     # groups -- 50 seemed safer and is still fast)
 526     regexes = []
 527     for some_symbols in [disable_makefile[i:i + 50] for i in range(0, len(disable_makefile), 50)]:
 528         r = '^([^#].*((CPTCFG|CONFIG)_(' + '|'.join([s for s in some_symbols]) + ')))'
 529         regexes.append(re.compile(r, re.MULTILINE))
 530     for f in maketree.get_makefiles():
 531         data = open(f, 'r').read()
 532         for r in regexes:
 533             data = r.sub(r'#\1', data)
 534         fo = open(f, 'w')
 535         fo.write(data)
 536         fo.close()
 537     git_debug_snapshot(args, "disable unsatisfied Makefile parts")
 538
 539     logwrite('Done!')
 540     return 0
 541
 542 if __name__ == '__main__':
 543     ret = _main()
 544     if ret:
 545         sys.exit(ret)