gentree.py

   1 #!/usr/bin/env python
   2 #
   3 # Generate the output tree into a specified directory.
   4 #
   5
   6 import argparse, sys, os, errno, shutil, re, subprocess
   7
   8 # find self
   9 source_dir = os.path.abspath(os.path.dirname(__file__))
  10 sys.path.append(source_dir)
  11 # and import libraries we have
  12 from lib import kconfig, patch, make
  13 from lib import bpgit as git
  14
  15 def read_copy_list(copyfile):
  16     """
  17     Read a copy-list file and return a list of (source, target)
  18     tuples. The source and target are usually the same, but in
  19     the copy-list file there may be a rename included.
  20     """
  21     ret = []
  22     for item in copyfile:
  23         # remove leading/trailing whitespace
  24         item = item.strip()
  25         # comments
  26         if not item or item[0] == '#':
  27             continue
  28         if item[0] == '/':
  29             raise Exception("Input path '%s' is absolute path, this isn't allowed" % (item, ))
  30         if ' -> ' in item:
  31             srcitem, dstitem = item.split(' -> ')
  32             if (srcitem[-1] == '/') != (dstitem[-1] == '/'):
  33                 raise Exception("Cannot copy file/dir to dir/file")
  34         else:
  35             srcitem = dstitem = item
  36         ret.append((srcitem, dstitem))
  37     return ret
  38
  39
  40 def read_dependencies(depfilename):
  41     """
  42     Read a (the) dependency file and return the list of
  43     dependencies as a dictionary, mapping a Kconfig symbol
  44     to a list of kernel version dependencies.
  45
  46     If a backported feature that an upstream backported driver
  47     depends on had kconfig limitations (ie, debugging feature not
  48     available) a built constaint restriction can be expressed
  49     by using a kconfig expression. The kconfig expressions can
  50     be specified by using the "kconfig: " prefix.
  51
  52     While reading ignore blank or commented lines.
  53     """
  54     ret = {}
  55     depfile = open(depfilename, 'r')
  56     for item in depfile:
  57         kconfig_exp = ""
  58         item = item.strip()
  59         if not item or item[0] == '#':
  60             continue
  61         if "kconfig:" in item:
  62             sym, kconfig_exp = item.split(" ", 1)
  63             if not sym in ret:
  64                 ret[sym] = [kconfig_exp, ]
  65             else:
  66                 ret[sym].append(kconfig_exp)
  67         else:
  68             sym, dep = item.split()
  69             if not sym in ret:
  70                 ret[sym] = [dep, ]
  71             else:
  72                 ret[sym].append(dep)
  73     return ret
  74
  75
  76 def check_output_dir(d, clean):
  77     """
  78     Check that the output directory doesn't exist or is empty,
  79     unless clean is True in which case it's nuked. This helps
  80     sanity check the output when generating a tree, so usually
  81     running with --clean isn't suggested.
  82     """
  83     if clean:
  84         shutil.rmtree(d, ignore_errors=True)
  85     try:
  86         os.rmdir(d)
  87     except OSError as e:
  88         if e.errno != errno.ENOENT:
  89             raise
  90
  91
  92 def copytree(src, dst, symlinks=False, ignore=None):
  93     """
  94     Copy a directory tree. This differs from shutil.copytree()
  95     in that it allows destination directories to already exist.
  96     """
  97     names = os.listdir(src)
  98     if ignore is not None:
  99         ignored_names = ignore(src, names)
 100     else:
 101         ignored_names = set()
 102
 103     if not os.path.isdir(dst):
 104         os.makedirs(dst)
 105     errors = []
 106     for name in names:
 107         if name in ignored_names:
 108             continue
 109         srcname = os.path.join(src, name)
 110         dstname = os.path.join(dst, name)
 111         try:
 112             if symlinks and os.path.islink(srcname):
 113                 linkto = os.readlink(srcname)
 114                 os.symlink(linkto, dstname)
 115             elif os.path.isdir(srcname):
 116                 copytree(srcname, dstname, symlinks, ignore)
 117             else:
 118                 shutil.copy2(srcname, dstname)
 119         except (IOError, os.error) as why:
 120             errors.append((srcname, dstname, str(why)))
 121         # catch the Error from the recursive copytree so that we can
 122         # continue with other files
 123         except shutil.Error as err:
 124             errors.extend(err.args[0])
 125     try:
 126         shutil.copystat(src, dst)
 127     except WindowsError:
 128         # can't copy file access times on Windows
 129         pass
 130     except OSError as why:
 131         errors.extend((src, dst, str(why)))
 132     if errors:
 133         raise shutil.Error(errors)
 134
 135
 136 def copy_files(srcpath, copy_list, outdir):
 137     """
 138     Copy the copy_list files and directories from the srcpath
 139     to the outdir. The copy_list contains source and target
 140     names.
 141
 142     For now, it also ignores any *~ editor backup files, though
 143     this should probably be generalized (maybe using .gitignore?)
 144     Similarly the code that only copies some files (*.c, *.h,
 145     *.awk, Kconfig, Makefile) to avoid any build remnants in the
 146     kernel if they should exist.
 147     """
 148     for srcitem, tgtitem in copy_list:
 149         if tgtitem == '':
 150             copytree(srcpath, outdir, ignore=shutil.ignore_patterns('*~'))
 151         elif tgtitem[-1] == '/':
 152             def copy_ignore(dir, entries):
 153                 r = []
 154                 for i in entries:
 155                     if i[-2:] == '.o' or i[-1] == '~':
 156                         r.append(i)
 157                 return r
 158             copytree(os.path.join(srcpath, srcitem),
 159                      os.path.join(outdir, tgtitem),
 160                      ignore=copy_ignore)
 161         else:
 162             try:
 163                 os.makedirs(os.path.join(outdir, os.path.dirname(tgtitem)))
 164             except OSError as e:
 165                 # ignore dirs we might have created just now
 166                 if e.errno != errno.EEXIST:
 167                     raise
 168             shutil.copy(os.path.join(srcpath, srcitem),
 169                         os.path.join(outdir, tgtitem))
 170
 171
 172 def copy_git_files(srcpath, copy_list, rev, outdir):
 173     """
 174     "Copy" files from a git repository. This really means listing them with
 175     ls-tree and then using git show to obtain all the blobs.
 176     """
 177     for srcitem, tgtitem in copy_list:
 178         for m, t, h, f in git.ls_tree(rev=rev, files=(srcitem,), tree=srcpath):
 179             assert t == 'blob'
 180             f = os.path.join(outdir, f.replace(srcitem, tgtitem))
 181             d = os.path.dirname(f)
 182             if not os.path.exists(d):
 183                 os.makedirs(d)
 184             outf = open(f, 'w')
 185             git.get_blob(h, outf, tree=srcpath)
 186             outf.close()
 187             os.chmod(f, int(m, 8))
 188
 189 def automatic_backport_mangle_c_file(name):
 190     return name.replace('/', '-')
 191
 192
 193 def add_automatic_backports(args):
 194     disable_list = []
 195     export = re.compile(r'^EXPORT_SYMBOL(_GPL)?\((?P<sym>[^\)]*)\)')
 196     bpi = kconfig.get_backport_info(os.path.join(args.outdir, 'compat', 'Kconfig'))
 197     configtree = kconfig.ConfigTree(os.path.join(args.outdir, 'Kconfig'))
 198     all_selects = configtree.all_selects()
 199     for sym, vals in bpi.items():
 200         if sym.startswith('BACKPORT_BUILD_'):
 201             if not sym[15:] in all_selects:
 202                 disable_list.append(sym)
 203                 continue
 204         symtype, module_name, c_files, h_files = vals
 205
 206         # first copy files
 207         files = []
 208         for f in c_files:
 209             files.append((f, os.path.join('compat', automatic_backport_mangle_c_file(f))))
 210         for f in h_files:
 211             files.append((os.path.join('include', f),
 212                           os.path.join('include', os.path.dirname(f), 'backport-' + os.path.basename(f))))
 213         if args.git_revision:
 214             copy_git_files(args.kerneldir, files, args.git_revision, args.outdir)
 215         else:
 216             copy_files(args.kerneldir, files, args.outdir)
 217
 218         # now add the Makefile line
 219         mf = open(os.path.join(args.outdir, 'compat', 'Makefile'), 'a+')
 220         o_files = [automatic_backport_mangle_c_file(f)[:-1] + 'o' for f in c_files]
 221         if symtype == 'tristate':
 222             if not module_name:
 223                 raise Exception('backporting a module requires a #module-name')
 224             for of in o_files:
 225                 mf.write('%s-objs += %s\n' % (module_name, of))
 226             mf.write('obj-$(CPTCFG_%s) += %s.o\n' % (sym, module_name))
 227         elif symtype == 'bool':
 228             mf.write('compat-$(CPTCFG_%s) += %s\n' % (sym, ' '.join(o_files)))
 229
 230         # finally create the include file
 231         syms = []
 232         for f in c_files:
 233             for l in open(os.path.join(args.outdir, 'compat',
 234                                        automatic_backport_mangle_c_file(f)), 'r'):
 235                 m = export.match(l)
 236                 if m:
 237                     syms.append(m.group('sym'))
 238         for f in h_files:
 239             outf = open(os.path.join(args.outdir, 'include', f), 'w')
 240             outf.write('/* Automatically created during backport process */\n')
 241             outf.write('#ifndef CPTCFG_%s\n' % sym)
 242             outf.write('#include_next <%s>\n' % f)
 243             outf.write('#else\n');
 244             for s in syms:
 245                 outf.write('#undef %s\n' % s)
 246                 outf.write('#define %s LINUX_BACKPORT(%s)\n' % (s, s))
 247             outf.write('#include <%s>\n' % (os.path.dirname(f) + '/backport-' + os.path.basename(f), ))
 248             outf.write('#endif /* CPTCFG_%s */\n' % sym)
 249     return disable_list
 250
 251 def git_debug_init(args):
 252     """
 253     Initialize a git repository in the output directory and commit the current
 254     code in it. This is only used for debugging the transformations this code
 255     will do to the output later.
 256     """
 257     if not args.gitdebug:
 258         return
 259     git.init(tree=args.outdir)
 260     git.commit_all("Copied backport", tree=args.outdir)
 261
 262
 263 def git_debug_snapshot(args, name):
 264     """
 265     Take a git snapshot for the debugging.
 266     """
 267     if not args.gitdebug:
 268         return
 269     git.commit_all(name, tree=args.outdir)
 270
 271
 272 def _main():
 273     # set up and parse arguments
 274     parser = argparse.ArgumentParser(description='generate backport tree')
 275     parser.add_argument('kerneldir', metavar='<kernel tree>', type=str,
 276                         help='Kernel tree to copy drivers from')
 277     parser.add_argument('outdir', metavar='<output directory>', type=str,
 278                         help='Directory to write the generated tree to')
 279     parser.add_argument('--copy-list', metavar='<listfile>', type=argparse.FileType('r'),
 280                         default='copy-list',
 281                         help='File containing list of files/directories to copy, default "copy-list"')
 282     parser.add_argument('--git-revision', metavar='<revision>', type=str,
 283                         help='git commit revision (see gitrevisions(7)) to take objects from.' +
 284                              'If this is specified, the kernel tree is used as git object storage ' +
 285                              'and we use git ls-tree to get the files.')
 286     parser.add_argument('--clean', const=True, default=False, action="store_const",
 287                         help='Clean output directory instead of erroring if it isn\'t empty')
 288     parser.add_argument('--refresh', const=True, default=False, action="store_const",
 289                         help='Refresh patches as they are applied, the source dir will be modified!')
 290     parser.add_argument('--base-name', metavar='<name>', type=str, default='Linux',
 291                         help='name of base tree, default just "Linux"')
 292     parser.add_argument('--gitdebug', const=True, default=False, action="store_const",
 293                         help='Use git, in the output tree, to debug the various transformation steps ' +
 294                              'that the tree generation makes (apply patches, ...)')
 295     parser.add_argument('--verbose', const=True, default=False, action="store_const",
 296                         help='Print more verbose information')
 297     parser.add_argument('--extra-driver', nargs=2, metavar=('<source dir>', '<copy-list>'), type=str,
 298                         action='append', default=[], help='Extra driver directory/copy-list.')
 299     args = parser.parse_args()
 300
 301     def logwrite(msg):
 302         sys.stdout.write(msg)
 303         sys.stdout.write('\n')
 304         sys.stdout.flush()
 305
 306     return process(args.kerneldir, args.outdir, args.copy_list,
 307                    git_revision=args.git_revision, clean=args.clean,
 308                    refresh=args.refresh, base_name=args.base_name,
 309                    gitdebug=args.gitdebug, verbose=args.verbose,
 310                    extra_driver=args.extra_driver, logwrite=logwrite)
 311
 312 def process(kerneldir, outdir, copy_list_file, git_revision=None,
 313             clean=False, refresh=False, base_name="Linux", gitdebug=False,
 314             verbose=False, extra_driver=[], logwrite=lambda x:None,
 315             git_tracked_version=False):
 316     class Args(object):
 317         def __init__(self, kerneldir, outdir, copy_list_file,
 318                      git_revision, clean, refresh, base_name,
 319                      gitdebug, verbose, extra_driver):
 320             self.kerneldir = kerneldir
 321             self.outdir = outdir
 322             self.copy_list = copy_list_file
 323             self.git_revision = git_revision
 324             self.clean = clean
 325             self.refresh = refresh
 326             self.base_name = base_name
 327             self.gitdebug = gitdebug
 328             self.verbose = verbose
 329             self.extra_driver = extra_driver
 330     args = Args(kerneldir, outdir, copy_list_file,
 331                 git_revision, clean, refresh, base_name,
 332                 gitdebug, verbose, extra_driver)
 333     # start processing ...
 334
 335     copy_list = read_copy_list(args.copy_list)
 336     deplist = read_dependencies(os.path.join(source_dir, 'dependencies'))
 337
 338     # validate output directory
 339     check_output_dir(args.outdir, args.clean)
 340
 341     # do the copy
 342     backport_files = [(x, x) for x in [
 343         'Kconfig', 'Makefile', 'Makefile.build', 'Makefile.kernel', '.gitignore',
 344         'Makefile.real', 'compat/', 'backport-include/', 'kconf/', 'defconfigs/',
 345         'scripts/', '.blacklist.map', 'udev/',
 346     ]]
 347     if not args.git_revision:
 348         logwrite('Copy original source files ...')
 349     else:
 350         logwrite('Get original source files from git ...')
 351
 352     copy_files(os.path.join(source_dir, 'backport'), backport_files, args.outdir)
 353
 354     git_debug_init(args)
 355
 356     if not args.git_revision:
 357         copy_files(args.kerneldir, copy_list, args.outdir)
 358     else:
 359         copy_git_files(args.kerneldir, copy_list, args.git_revision, args.outdir)
 360
 361     # FIXME: should we add a git version of this (e.g. --git-extra-driver)?
 362     for src, copy_list in args.extra_driver:
 363         copy_files(src, read_copy_list(open(copy_list, 'r')), args.outdir)
 364
 365     git_debug_snapshot(args, 'Add driver sources')
 366
 367     disable_list = add_automatic_backports(args)
 368     if disable_list:
 369         bpcfg = kconfig.ConfigTree(os.path.join(args.outdir, 'compat', 'Kconfig'))
 370         bpcfg.disable_symbols(disable_list)
 371     git_debug_snapshot(args, 'Add automatic backports')
 372
 373     logwrite('Apply patches ...')
 374     patches = []
 375     for root, dirs, files in os.walk(os.path.join(source_dir, 'patches')):
 376         for f in files:
 377             if f.endswith('.patch'):
 378                 patches.append(os.path.join(root, f))
 379     patches.sort()
 380     prefix_len = len(os.path.join(source_dir, 'patches')) + 1
 381     for pfile in patches:
 382         print_name = pfile[prefix_len:]
 383         # read the patch file
 384         p = patch.fromfile(pfile)
 385         # complain if it's not a patch
 386         if not p:
 387             raise Exception('No patch content found in %s' % print_name)
 388         # leading / seems to be stripped?
 389         if 'dev/null' in p.items[0].source:
 390             raise Exception('Patches creating files are not supported (in %s)' % print_name)
 391         # check if the first file the patch touches exists, if so
 392         # assume the patch needs to be applied -- otherwise continue
 393         patched_file = '/'.join(p.items[0].source.split('/')[1:])
 394         fullfn = os.path.join(args.outdir, patched_file)
 395         if not os.path.exists(fullfn):
 396             if args.verbose:
 397                 logwrite("Not applying %s, not needed" % print_name)
 398             continue
 399         if args.verbose:
 400             logwrite("Applying patch %s" % print_name)
 401
 402         if args.refresh:
 403             # but for refresh, of course look at all files the patch touches
 404             for patchitem in p.items:
 405                 patched_file = '/'.join(patchitem.source.split('/')[1:])
 406                 fullfn = os.path.join(args.outdir, patched_file)
 407                 shutil.copyfile(fullfn, fullfn + '.orig_file')
 408
 409         process = subprocess.Popen(['patch', '-p1'], stdout=subprocess.PIPE,
 410                                    stderr=subprocess.STDOUT, stdin=subprocess.PIPE,
 411                                    close_fds=True, universal_newlines=True,
 412                                    cwd=args.outdir)
 413         output = process.communicate(input=open(pfile, 'r').read())[0]
 414         output = output.split('\n')
 415         if output[-1] == '':
 416             output = output[:-1]
 417         if args.verbose:
 418             for line in output:
 419                 logwrite('> %s' % line)
 420         if process.returncode != 0:
 421             if not args.verbose:
 422                 logwrite("Failed to apply changes from %s" % print_name)
 423                 for line in output:
 424                     logwrite('> %s' % line)
 425             return 2
 426
 427         if args.refresh:
 428             pfilef = open(pfile + '.tmp', 'a')
 429             pfilef.write(p.top_header)
 430             pfilef.flush()
 431             for patchitem in p.items:
 432                 patched_file = '/'.join(patchitem.source.split('/')[1:])
 433                 fullfn = os.path.join(args.outdir, patched_file)
 434                 process = subprocess.Popen(['diff', '-p', '-u', patched_file + '.orig_file', patched_file,
 435                                             '--label', 'a/' + patched_file,
 436                                             '--label', 'b/' + patched_file],
 437                                            stdout=pfilef, close_fds=True,
 438                                            universal_newlines=True, cwd=args.outdir)
 439                 process.wait()
 440                 os.unlink(fullfn + '.orig_file')
 441                 if not process.returncode in (0, 1):
 442                     logwrite("Failed to diff to refresh %s" % print_name)
 443                     pfilef.close()
 444                     os.unlink(pfile + '.tmp')
 445                     return 3
 446             pfilef.close()
 447             os.rename(pfile + '.tmp', pfile)
 448
 449         # remove orig/rej files that patch sometimes creates
 450         for root, dirs, files in os.walk(args.outdir):
 451             for f in files:
 452                 if f[-5:] == '.orig' or f[-4:] == '.rej':
 453                     os.unlink(os.path.join(root, f))
 454         git_debug_snapshot(args, "apply backport patch %s" % print_name)
 455
 456     # some post-processing is required
 457     configtree = kconfig.ConfigTree(os.path.join(args.outdir, 'Kconfig'))
 458     logwrite('Modify Kconfig tree ...')
 459     configtree.prune_sources(ignore=['Kconfig.kernel', 'Kconfig.versions'])
 460     git_debug_snapshot(args, "prune Kconfig tree")
 461     configtree.force_tristate_modular()
 462     git_debug_snapshot(args, "force tristate options modular")
 463     configtree.modify_selects()
 464     git_debug_snapshot(args, "convert select to depends on")
 465
 466     # write the versioning file
 467     if git_tracked_version:
 468         backports_version = "(see git)"
 469         kernel_version = "(see git)"
 470     else:
 471         backports_version = git.describe(tree=source_dir)
 472         kernel_version = git.describe(rev=args.git_revision or 'HEAD',
 473                                       tree=args.kerneldir)
 474     f = open(os.path.join(args.outdir, 'versions'), 'w')
 475     f.write('BACKPORTS_VERSION="%s"\n' % backports_version)
 476     f.write('BACKPORTED_KERNEL_VERSION="%s"\n' % kernel_version)
 477     f.write('BACKPORTED_KERNEL_NAME="%s"\n' % args.base_name)
 478     if git_tracked_version:
 479         f.write('BACKPORTS_GIT_TRACKED="backport tracker ID: $(shell git rev-parse HEAD 2>/dev/null || echo \'not built in git tree\')"\n')
 480     f.close()
 481
 482     symbols = configtree.symbols()
 483
 484     # write local symbol list -- needed during build
 485     f = open(os.path.join(args.outdir, '.local-symbols'), 'w')
 486     for sym in symbols:
 487         f.write('%s=\n' % sym)
 488     f.close()
 489
 490     git_debug_snapshot(args, "add versions/symbols files")
 491
 492     logwrite('Rewrite Makefiles and Kconfig files ...')
 493
 494     # rewrite Makefile and source symbols
 495     regexes = []
 496     for some_symbols in [symbols[i:i + 50] for i in range(0, len(symbols), 50)]:
 497         r = 'CONFIG_((' + '|'.join([s + '(_MODULE)?' for s in some_symbols]) + ')([^A-Za-z0-9_]|$))'
 498         regexes.append(re.compile(r, re.MULTILINE))
 499     for root, dirs, files in os.walk(args.outdir):
 500         # don't go into .git dir (possible debug thing)
 501         if '.git' in dirs:
 502             dirs.remove('.git')
 503         for f in files:
 504             data = open(os.path.join(root, f), 'r').read()
 505             for r in regexes:
 506                 data = r.sub(r'CPTCFG_\1', data)
 507             data = re.sub(r'\$\(srctree\)', '$(backport_srctree)', data)
 508             data = re.sub(r'-Idrivers', '-I$(backport_srctree)/drivers', data)
 509             fo = open(os.path.join(root, f), 'w')
 510             fo.write(data)
 511             fo.close()
 512
 513     git_debug_snapshot(args, "rename config symbol / srctree usage")
 514
 515     # disable unbuildable Kconfig symbols and stuff Makefiles that doesn't exist
 516     maketree = make.MakeTree(os.path.join(args.outdir, 'Makefile.kernel'))
 517     disable_kconfig = []
 518     disable_makefile = []
 519     for sym in maketree.get_impossible_symbols():
 520         disable_kconfig.append(sym[7:])
 521         disable_makefile.append(sym[7:])
 522
 523     configtree.disable_symbols(disable_kconfig)
 524     git_debug_snapshot(args, "disable impossible kconfig symbols")
 525
 526     # add kernel version dependencies to Kconfig, from the dependency list
 527     # we read previously
 528     for sym in tuple(deplist.keys()):
 529         new = []
 530         for dep in deplist[sym]:
 531             if "kconfig:" in dep:
 532                     kconfig_expr = dep.replace('kconfig: ', '')
 533                     new.append(kconfig_expr)
 534             elif (dep == "DISABLE"):
 535                     new.append('BACKPORT_DISABLED_KCONFIG_OPTION')
 536             else:
 537                     new.append('!BACKPORT_KERNEL_%s' % dep.replace('.', '_'))
 538         deplist[sym] = new
 539     configtree.add_dependencies(deplist)
 540     git_debug_snapshot(args, "add kernel version dependencies")
 541
 542     # disable things in makefiles that can't be selected and that the
 543     # build shouldn't recurse into because they don't exist -- if we
 544     # don't do that then a symbol from the kernel could cause the build
 545     # to attempt to recurse and fail
 546     #
 547     # Note that we split the regex after 50 symbols, this is because of a
 548     # limitation in the regex implementation (it only supports 100 nested
 549     # groups -- 50 seemed safer and is still fast)
 550     regexes = []
 551     for some_symbols in [disable_makefile[i:i + 50] for i in range(0, len(disable_makefile), 50)]:
 552         r = '^([^#].*((CPTCFG|CONFIG)_(' + '|'.join([s for s in some_symbols]) + ')))'
 553         regexes.append(re.compile(r, re.MULTILINE))
 554     for f in maketree.get_makefiles():
 555         data = open(f, 'r').read()
 556         for r in regexes:
 557             data = r.sub(r'#\1', data)
 558         fo = open(f, 'w')
 559         fo.write(data)
 560         fo.close()
 561     git_debug_snapshot(args, "disable unsatisfied Makefile parts")
 562
 563     logwrite('Done!')
 564     return 0
 565
 566 if __name__ == '__main__':
 567     ret = _main()
 568     if ret:
 569         sys.exit(ret)