gentree.py

   1 #!/usr/bin/env python
   2 #
   3 # Generate the output tree into a specified directory.
   4 #
   5
   6 import argparse, sys, os, errno, shutil, re, subprocess
   7
   8 # find self
   9 source_dir = os.path.abspath(os.path.dirname(__file__))
  10 sys.path.append(source_dir)
  11 # and import libraries we have
  12 from lib import kconfig, patch, make
  13 from lib import bpgit as git
  14
  15 def read_copy_list(copyfile):
  16     """
  17     Read a copy-list file and return a list of (source, target)
  18     tuples. The source and target are usually the same, but in
  19     the copy-list file there may be a rename included.
  20     """
  21     ret = []
  22     for item in copyfile:
  23         # remove leading/trailing whitespace
  24         item = item.strip()
  25         # comments
  26         if not item or item[0] == '#':
  27             continue
  28         if item[0] == '/':
  29             raise Exception("Input path '%s' is absolute path, this isn't allowed" % (item, ))
  30         if ' -> ' in item:
  31             srcitem, dstitem = item.split(' -> ')
  32             if (srcitem[-1] == '/') != (dstitem[-1] == '/'):
  33                 raise Exception("Cannot copy file/dir to dir/file")
  34         else:
  35             srcitem = dstitem = item
  36         ret.append((srcitem, dstitem))
  37     return ret
  38
  39
  40 def read_dependencies(depfilename):
  41     """
  42     Read a (the) dependency file and return the list of
  43     dependencies as a dictionary, mapping a Kconfig symbol
  44     to a list of kernel version dependencies. While reading
  45     ignore blank/commented lines.
  46     """
  47     ret = {}
  48     depfile = open(depfilename, 'r')
  49     for item in depfile:
  50         item = item.strip()
  51         if not item or item[0] == '#':
  52             continue
  53         sym, dep = item.split()
  54         if not sym in ret:
  55             ret[sym] = [dep, ]
  56         else:
  57             ret[sym].append(dep)
  58     return ret
  59
  60
  61 def check_output_dir(d, clean):
  62     """
  63     Check that the output directory doesn't exist or is empty,
  64     unless clean is True in which case it's nuked. This helps
  65     sanity check the output when generating a tree, so usually
  66     running with --clean isn't suggested.
  67     """
  68     if clean:
  69         shutil.rmtree(d, ignore_errors=True)
  70     try:
  71         os.rmdir(d)
  72     except OSError as e:
  73         if e.errno != errno.ENOENT:
  74             raise
  75
  76
  77 def copytree(src, dst, symlinks=False, ignore=None):
  78     """
  79     Copy a directory tree. This differs from shutil.copytree()
  80     in that it allows destination directories to already exist.
  81     """
  82     names = os.listdir(src)
  83     if ignore is not None:
  84         ignored_names = ignore(src, names)
  85     else:
  86         ignored_names = set()
  87
  88     if not os.path.isdir(dst):
  89         os.makedirs(dst)
  90     errors = []
  91     for name in names:
  92         if name in ignored_names:
  93             continue
  94         srcname = os.path.join(src, name)
  95         dstname = os.path.join(dst, name)
  96         try:
  97             if symlinks and os.path.islink(srcname):
  98                 linkto = os.readlink(srcname)
  99                 os.symlink(linkto, dstname)
 100             elif os.path.isdir(srcname):
 101                 copytree(srcname, dstname, symlinks, ignore)
 102             else:
 103                 shutil.copy2(srcname, dstname)
 104         except (IOError, os.error) as why:
 105             errors.append((srcname, dstname, str(why)))
 106         # catch the Error from the recursive copytree so that we can
 107         # continue with other files
 108         except shutil.Error as err:
 109             errors.extend(err.args[0])
 110     try:
 111         shutil.copystat(src, dst)
 112     except WindowsError:
 113         # can't copy file access times on Windows
 114         pass
 115     except OSError as why:
 116         errors.extend((src, dst, str(why)))
 117     if errors:
 118         raise shutil.Error(errors)
 119
 120
 121 def copy_files(srcpath, copy_list, outdir):
 122     """
 123     Copy the copy_list files and directories from the srcpath
 124     to the outdir. The copy_list contains source and target
 125     names.
 126
 127     For now, it also ignores any *~ editor backup files, though
 128     this should probably be generalized (maybe using .gitignore?)
 129     Similarly the code that only copies some files (*.c, *.h,
 130     *.awk, Kconfig, Makefile) to avoid any build remnants in the
 131     kernel if they should exist.
 132     """
 133     for srcitem, tgtitem in copy_list:
 134         if tgtitem == '':
 135             copytree(srcpath, outdir, ignore=shutil.ignore_patterns('*~'))
 136         elif tgtitem[-1] == '/':
 137             def copy_ignore(dir, entries):
 138                 r = []
 139                 for i in entries:
 140                     if i[-2:] == '.o' or i[-1] == '~':
 141                         r.append(i)
 142                 return r
 143             copytree(os.path.join(srcpath, srcitem),
 144                      os.path.join(outdir, tgtitem),
 145                      ignore=copy_ignore)
 146         else:
 147             try:
 148                 os.makedirs(os.path.join(outdir, os.path.dirname(tgtitem)))
 149             except OSError as e:
 150                 # ignore dirs we might have created just now
 151                 if e.errno != errno.EEXIST:
 152                     raise
 153             shutil.copy(os.path.join(srcpath, srcitem),
 154                         os.path.join(outdir, tgtitem))
 155
 156
 157 def copy_git_files(srcpath, copy_list, rev, outdir):
 158     """
 159     "Copy" files from a git repository. This really means listing them with
 160     ls-tree and then using git show to obtain all the blobs.
 161     """
 162     for srcitem, tgtitem in copy_list:
 163         for m, t, h, f in git.ls_tree(rev=rev, files=(srcitem,), tree=srcpath):
 164             assert t == 'blob'
 165             f = os.path.join(outdir, f.replace(srcitem, tgtitem))
 166             d = os.path.dirname(f)
 167             if not os.path.exists(d):
 168                 os.makedirs(d)
 169             outf = open(f, 'w')
 170             git.get_blob(h, outf, tree=srcpath)
 171             outf.close()
 172             os.chmod(f, int(m, 8))
 173
 174 def automatic_backport_mangle_c_file(name):
 175     return name.replace('/', '-')
 176
 177
 178 def add_automatic_backports(args):
 179     disable_list = []
 180     export = re.compile(r'^EXPORT_SYMBOL(_GPL)?\((?P<sym>[^\)]*)\)')
 181     bpi = kconfig.get_backport_info(os.path.join(args.outdir, 'compat', 'Kconfig'))
 182     configtree = kconfig.ConfigTree(os.path.join(args.outdir, 'Kconfig'))
 183     all_selects = configtree.all_selects()
 184     for sym, vals in bpi.items():
 185         if sym.startswith('BACKPORT_BUILD_'):
 186             if not sym[15:] in all_selects:
 187                 disable_list.append(sym)
 188                 continue
 189         symtype, module_name, c_files, h_files = vals
 190
 191         # first copy files
 192         files = []
 193         for f in c_files:
 194             files.append((f, os.path.join('compat', automatic_backport_mangle_c_file(f))))
 195         for f in h_files:
 196             files.append((os.path.join('include', f),
 197                           os.path.join('include', os.path.dirname(f), 'backport-' + os.path.basename(f))))
 198         if args.git_revision:
 199             copy_git_files(args.kerneldir, files, args.git_revision, args.outdir)
 200         else:
 201             copy_files(args.kerneldir, files, args.outdir)
 202
 203         # now add the Makefile line
 204         mf = open(os.path.join(args.outdir, 'compat', 'Makefile'), 'a+')
 205         o_files = [automatic_backport_mangle_c_file(f)[:-1] + 'o' for f in c_files]
 206         if symtype == 'tristate':
 207             if not module_name:
 208                 raise Exception('backporting a module requires a #module-name')
 209             for of in o_files:
 210                 mf.write('%s-objs += %s\n' % (module_name, of))
 211             mf.write('obj-$(CPTCFG_%s) += %s.o\n' % (sym, module_name))
 212         elif symtype == 'bool':
 213             mf.write('compat-$(CPTCFG_%s) += %s\n' % (sym, ' '.join(o_files)))
 214
 215         # finally create the include file
 216         syms = []
 217         for f in c_files:
 218             for l in open(os.path.join(args.outdir, 'compat',
 219                                        automatic_backport_mangle_c_file(f)), 'r'):
 220                 m = export.match(l)
 221                 if m:
 222                     syms.append(m.group('sym'))
 223         for f in h_files:
 224             outf = open(os.path.join(args.outdir, 'include', f), 'w')
 225             outf.write('/* Automatically created during backport process */\n')
 226             outf.write('#ifndef CPTCFG_%s\n' % sym)
 227             outf.write('#include_next <%s>\n' % f)
 228             outf.write('#else\n');
 229             for s in syms:
 230                 outf.write('#undef %s\n' % s)
 231                 outf.write('#define %s LINUX_BACKPORT(%s)\n' % (s, s))
 232             outf.write('#include <%s>\n' % (os.path.dirname(f) + '/backport-' + os.path.basename(f), ))
 233             outf.write('#endif /* CPTCFG_%s */\n' % sym)
 234     return disable_list
 235
 236 def git_debug_init(args):
 237     """
 238     Initialize a git repository in the output directory and commit the current
 239     code in it. This is only used for debugging the transformations this code
 240     will do to the output later.
 241     """
 242     if not args.gitdebug:
 243         return
 244     git.init(tree=args.outdir)
 245     git.commit_all("Copied backport", tree=args.outdir)
 246
 247
 248 def git_debug_snapshot(args, name):
 249     """
 250     Take a git snapshot for the debugging.
 251     """
 252     if not args.gitdebug:
 253         return
 254     git.commit_all(name, tree=args.outdir)
 255
 256
 257 def _main():
 258     # set up and parse arguments
 259     parser = argparse.ArgumentParser(description='generate backport tree')
 260     parser.add_argument('kerneldir', metavar='<kernel tree>', type=str,
 261                         help='Kernel tree to copy drivers from')
 262     parser.add_argument('outdir', metavar='<output directory>', type=str,
 263                         help='Directory to write the generated tree to')
 264     parser.add_argument('--copy-list', metavar='<listfile>', type=argparse.FileType('r'),
 265                         default='copy-list',
 266                         help='File containing list of files/directories to copy, default "copy-list"')
 267     parser.add_argument('--git-revision', metavar='<revision>', type=str,
 268                         help='git commit revision (see gitrevisions(7)) to take objects from.' +
 269                              'If this is specified, the kernel tree is used as git object storage ' +
 270                              'and we use git ls-tree to get the files.')
 271     parser.add_argument('--clean', const=True, default=False, action="store_const",
 272                         help='Clean output directory instead of erroring if it isn\'t empty')
 273     parser.add_argument('--refresh', const=True, default=False, action="store_const",
 274                         help='Refresh patches as they are applied, the source dir will be modified!')
 275     parser.add_argument('--base-name', metavar='<name>', type=str, default='Linux',
 276                         help='name of base tree, default just "Linux"')
 277     parser.add_argument('--gitdebug', const=True, default=False, action="store_const",
 278                         help='Use git, in the output tree, to debug the various transformation steps ' +
 279                              'that the tree generation makes (apply patches, ...)')
 280     parser.add_argument('--verbose', const=True, default=False, action="store_const",
 281                         help='Print more verbose information')
 282     parser.add_argument('--extra-driver', nargs=2, metavar=('<source dir>', '<copy-list>'), type=str,
 283                         action='append', default=[], help='Extra driver directory/copy-list.')
 284     args = parser.parse_args()
 285
 286     def logwrite(msg):
 287         sys.stdout.write(msg)
 288         sys.stdout.write('\n')
 289         sys.stdout.flush()
 290
 291     return process(args.kerneldir, args.outdir, args.copy_list,
 292                    git_revision=args.git_revision, clean=args.clean,
 293                    refresh=args.refresh, base_name=args.base_name,
 294                    gitdebug=args.gitdebug, verbose=args.verbose,
 295                    extra_driver=args.extra_driver, logwrite=logwrite)
 296
 297 def process(kerneldir, outdir, copy_list_file, git_revision=None,
 298             clean=False, refresh=False, base_name="Linux", gitdebug=False,
 299             verbose=False, extra_driver=[], logwrite=lambda x:None,
 300             git_tracked_version=False):
 301     class Args(object):
 302         def __init__(self, kerneldir, outdir, copy_list_file,
 303                      git_revision, clean, refresh, base_name,
 304                      gitdebug, verbose, extra_driver):
 305             self.kerneldir = kerneldir
 306             self.outdir = outdir
 307             self.copy_list = copy_list_file
 308             self.git_revision = git_revision
 309             self.clean = clean
 310             self.refresh = refresh
 311             self.base_name = base_name
 312             self.gitdebug = gitdebug
 313             self.verbose = verbose
 314             self.extra_driver = extra_driver
 315     args = Args(kerneldir, outdir, copy_list_file,
 316                 git_revision, clean, refresh, base_name,
 317                 gitdebug, verbose, extra_driver)
 318     # start processing ...
 319
 320     copy_list = read_copy_list(args.copy_list)
 321     deplist = read_dependencies(os.path.join(source_dir, 'dependencies'))
 322
 323     # validate output directory
 324     check_output_dir(args.outdir, args.clean)
 325
 326     # do the copy
 327     backport_files = [(x, x) for x in [
 328         'Kconfig', 'Makefile', 'Makefile.build', 'Makefile.kernel', '.gitignore',
 329         'Makefile.real', 'compat/', 'backport-include/', 'kconf/', 'defconfigs/',
 330         'scripts/', '.blacklist.map', 'udev/',
 331     ]]
 332     if not args.git_revision:
 333         logwrite('Copy original source files ...')
 334     else:
 335         logwrite('Get original source files from git ...')
 336
 337     copy_files(os.path.join(source_dir, 'backport'), backport_files, args.outdir)
 338
 339     git_debug_init(args)
 340
 341     if not args.git_revision:
 342         copy_files(args.kerneldir, copy_list, args.outdir)
 343     else:
 344         copy_git_files(args.kerneldir, copy_list, args.git_revision, args.outdir)
 345
 346     # FIXME: should we add a git version of this (e.g. --git-extra-driver)?
 347     for src, copy_list in args.extra_driver:
 348         copy_files(src, read_copy_list(open(copy_list, 'r')), args.outdir)
 349
 350     git_debug_snapshot(args, 'Add driver sources')
 351
 352     disable_list = add_automatic_backports(args)
 353     if disable_list:
 354         bpcfg = kconfig.ConfigTree(os.path.join(args.outdir, 'compat', 'Kconfig'))
 355         bpcfg.disable_symbols(disable_list)
 356     git_debug_snapshot(args, 'Add automatic backports')
 357
 358     logwrite('Apply patches ...')
 359     patches = []
 360     for root, dirs, files in os.walk(os.path.join(source_dir, 'patches')):
 361         for f in files:
 362             if f.endswith('.patch'):
 363                 patches.append(os.path.join(root, f))
 364     patches.sort()
 365     prefix_len = len(os.path.join(source_dir, 'patches')) + 1
 366     for pfile in patches:
 367         print_name = pfile[prefix_len:]
 368         # read the patch file
 369         p = patch.fromfile(pfile)
 370         # complain if it's not a patch
 371         if not p:
 372             raise Exception('No patch content found in %s' % print_name)
 373         # leading / seems to be stripped?
 374         if 'dev/null' in p.items[0].source:
 375             raise Exception('Patches creating files are not supported (in %s)' % print_name)
 376         # check if the first file the patch touches exists, if so
 377         # assume the patch needs to be applied -- otherwise continue
 378         patched_file = '/'.join(p.items[0].source.split('/')[1:])
 379         fullfn = os.path.join(args.outdir, patched_file)
 380         if not os.path.exists(fullfn):
 381             if args.verbose:
 382                 logwrite("Not applying %s, not needed" % print_name)
 383             continue
 384         if args.verbose:
 385             logwrite("Applying patch %s" % print_name)
 386
 387         if args.refresh:
 388             # but for refresh, of course look at all files the patch touches
 389             for patchitem in p.items:
 390                 patched_file = '/'.join(patchitem.source.split('/')[1:])
 391                 fullfn = os.path.join(args.outdir, patched_file)
 392                 shutil.copyfile(fullfn, fullfn + '.orig_file')
 393
 394         process = subprocess.Popen(['patch', '-p1'], stdout=subprocess.PIPE,
 395                                    stderr=subprocess.STDOUT, stdin=subprocess.PIPE,
 396                                    close_fds=True, universal_newlines=True,
 397                                    cwd=args.outdir)
 398         output = process.communicate(input=open(pfile, 'r').read())[0]
 399         output = output.split('\n')
 400         if output[-1] == '':
 401             output = output[:-1]
 402         if args.verbose:
 403             for line in output:
 404                 logwrite('> %s' % line)
 405         if process.returncode != 0:
 406             if not args.verbose:
 407                 logwrite("Failed to apply changes from %s" % print_name)
 408                 for line in output:
 409                     logwrite('> %s' % line)
 410             return 2
 411
 412         if args.refresh:
 413             pfilef = open(pfile + '.tmp', 'a')
 414             pfilef.write(p.top_header)
 415             pfilef.flush()
 416             for patchitem in p.items:
 417                 patched_file = '/'.join(patchitem.source.split('/')[1:])
 418                 fullfn = os.path.join(args.outdir, patched_file)
 419                 process = subprocess.Popen(['diff', '-p', '-u', patched_file + '.orig_file', patched_file,
 420                                             '--label', 'a/' + patched_file,
 421                                             '--label', 'b/' + patched_file],
 422                                            stdout=pfilef, close_fds=True,
 423                                            universal_newlines=True, cwd=args.outdir)
 424                 process.wait()
 425                 os.unlink(fullfn + '.orig_file')
 426                 if not process.returncode in (0, 1):
 427                     logwrite("Failed to diff to refresh %s" % print_name)
 428                     pfilef.close()
 429                     os.unlink(pfile + '.tmp')
 430                     return 3
 431             pfilef.close()
 432             os.rename(pfile + '.tmp', pfile)
 433
 434         # remove orig/rej files that patch sometimes creates
 435         for root, dirs, files in os.walk(args.outdir):
 436             for f in files:
 437                 if f[-5:] == '.orig' or f[-4:] == '.rej':
 438                     os.unlink(os.path.join(root, f))
 439         git_debug_snapshot(args, "apply backport patch %s" % print_name)
 440
 441     # some post-processing is required
 442     configtree = kconfig.ConfigTree(os.path.join(args.outdir, 'Kconfig'))
 443     logwrite('Modify Kconfig tree ...')
 444     configtree.prune_sources(ignore=['Kconfig.kernel', 'Kconfig.versions'])
 445     git_debug_snapshot(args, "prune Kconfig tree")
 446     configtree.force_tristate_modular()
 447     git_debug_snapshot(args, "force tristate options modular")
 448     configtree.modify_selects()
 449     git_debug_snapshot(args, "convert select to depends on")
 450
 451     # write the versioning file
 452     if git_tracked_version:
 453         backports_version = "(see git)"
 454         kernel_version = "(see git)"
 455     else:
 456         backports_version = git.describe(tree=source_dir)
 457         kernel_version = git.describe(rev=args.git_revision or 'HEAD',
 458                                       tree=args.kerneldir)
 459     f = open(os.path.join(args.outdir, 'versions'), 'w')
 460     f.write('BACKPORTS_VERSION="%s"\n' % backports_version)
 461     f.write('BACKPORTED_KERNEL_VERSION="%s"\n' % kernel_version)
 462     f.write('BACKPORTED_KERNEL_NAME="%s"\n' % args.base_name)
 463     if git_tracked_version:
 464         f.write('BACKPORTS_GIT_TRACKED="backport tracker ID: $(shell git rev-parse HEAD 2>/dev/null || echo \'not built in git tree\')"\n')
 465     f.close()
 466
 467     symbols = configtree.symbols()
 468
 469     # write local symbol list -- needed during build
 470     f = open(os.path.join(args.outdir, '.local-symbols'), 'w')
 471     for sym in symbols:
 472         f.write('%s=\n' % sym)
 473     f.close()
 474
 475     git_debug_snapshot(args, "add versions/symbols files")
 476
 477     logwrite('Rewrite Makefiles and Kconfig files ...')
 478
 479     # rewrite Makefile and source symbols
 480     regexes = []
 481     for some_symbols in [symbols[i:i + 50] for i in range(0, len(symbols), 50)]:
 482         r = 'CONFIG_((' + '|'.join([s + '(_MODULE)?' for s in some_symbols]) + ')([^A-Za-z0-9_]|$))'
 483         regexes.append(re.compile(r, re.MULTILINE))
 484     for root, dirs, files in os.walk(args.outdir):
 485         # don't go into .git dir (possible debug thing)
 486         if '.git' in dirs:
 487             dirs.remove('.git')
 488         for f in files:
 489             data = open(os.path.join(root, f), 'r').read()
 490             for r in regexes:
 491                 data = r.sub(r'CPTCFG_\1', data)
 492             data = re.sub(r'\$\(srctree\)', '$(backport_srctree)', data)
 493             data = re.sub(r'-Idrivers', '-I$(backport_srctree)/drivers', data)
 494             fo = open(os.path.join(root, f), 'w')
 495             fo.write(data)
 496             fo.close()
 497
 498     git_debug_snapshot(args, "rename config symbol / srctree usage")
 499
 500     # disable unbuildable Kconfig symbols and stuff Makefiles that doesn't exist
 501     maketree = make.MakeTree(os.path.join(args.outdir, 'Makefile.kernel'))
 502     disable_kconfig = []
 503     disable_makefile = []
 504     for sym in maketree.get_impossible_symbols():
 505         disable_kconfig.append(sym[7:])
 506         disable_makefile.append(sym[7:])
 507
 508     configtree.disable_symbols(disable_kconfig)
 509     git_debug_snapshot(args, "disable impossible kconfig symbols")
 510
 511     # add kernel version dependencies to Kconfig, from the dependency list
 512     # we read previously
 513     for sym in tuple(deplist.keys()):
 514         new = []
 515         for dep in deplist[sym]:
 516             if dep == "DISABLE":
 517                     new.append('BACKPORT_DISABLED_KCONFIG_OPTION')
 518             else:
 519                     new.append('!BACKPORT_KERNEL_%s' % dep.replace('.', '_'))
 520         deplist[sym] = new
 521     configtree.add_dependencies(deplist)
 522     git_debug_snapshot(args, "add kernel version dependencies")
 523
 524     # disable things in makefiles that can't be selected and that the
 525     # build shouldn't recurse into because they don't exist -- if we
 526     # don't do that then a symbol from the kernel could cause the build
 527     # to attempt to recurse and fail
 528     #
 529     # Note that we split the regex after 50 symbols, this is because of a
 530     # limitation in the regex implementation (it only supports 100 nested
 531     # groups -- 50 seemed safer and is still fast)
 532     regexes = []
 533     for some_symbols in [disable_makefile[i:i + 50] for i in range(0, len(disable_makefile), 50)]:
 534         r = '^([^#].*((CPTCFG|CONFIG)_(' + '|'.join([s for s in some_symbols]) + ')))'
 535         regexes.append(re.compile(r, re.MULTILINE))
 536     for f in maketree.get_makefiles():
 537         data = open(f, 'r').read()
 538         for r in regexes:
 539             data = r.sub(r'#\1', data)
 540         fo = open(f, 'w')
 541         fo.write(data)
 542         fo.close()
 543     git_debug_snapshot(args, "disable unsatisfied Makefile parts")
 544
 545     logwrite('Done!')
 546     return 0
 547
 548 if __name__ == '__main__':
 549     ret = _main()
 550     if ret:
 551         sys.exit(ret)