gentree.py

   1 #!/usr/bin/env python
   2 #
   3 # Generate the output tree into a specified directory.
   4 #
   5
   6 import argparse, sys, os, errno, shutil, re, subprocess
   7 import tarfile, bz2
   8
   9 # find self
  10 source_dir = os.path.abspath(os.path.dirname(__file__))
  11 sys.path.append(source_dir)
  12 # and import libraries we have
  13 from lib import kconfig, patch, make
  14 from lib import bpgit as git
  15 from lib import bpgpg as gpg
  16 from lib import bpkup as kup
  17
  18 def read_copy_list(copyfile):
  19     """
  20     Read a copy-list file and return a list of (source, target)
  21     tuples. The source and target are usually the same, but in
  22     the copy-list file there may be a rename included.
  23     """
  24     ret = []
  25     for item in copyfile:
  26         # remove leading/trailing whitespace
  27         item = item.strip()
  28         # comments
  29         if not item or item[0] == '#':
  30             continue
  31         if item[0] == '/':
  32             raise Exception("Input path '%s' is absolute path, this isn't allowed" % (item, ))
  33         if ' -> ' in item:
  34             srcitem, dstitem = item.split(' -> ')
  35             if (srcitem[-1] == '/') != (dstitem[-1] == '/'):
  36                 raise Exception("Cannot copy file/dir to dir/file")
  37         else:
  38             srcitem = dstitem = item
  39         ret.append((srcitem, dstitem))
  40     return ret
  41
  42
  43 def read_dependencies(depfilename):
  44     """
  45     Read a (the) dependency file and return the list of
  46     dependencies as a dictionary, mapping a Kconfig symbol
  47     to a list of kernel version dependencies.
  48
  49     If a backported feature that an upstream backported driver
  50     depends on had kconfig limitations (ie, debugging feature not
  51     available) a built constaint restriction can be expressed
  52     by using a kconfig expression. The kconfig expressions can
  53     be specified by using the "kconfig: " prefix.
  54
  55     While reading ignore blank or commented lines.
  56     """
  57     ret = {}
  58     depfile = open(depfilename, 'r')
  59     for item in depfile:
  60         kconfig_exp = ""
  61         item = item.strip()
  62         if not item or item[0] == '#':
  63             continue
  64         if "kconfig:" in item:
  65             sym, kconfig_exp = item.split(" ", 1)
  66             if not sym in ret:
  67                 ret[sym] = [kconfig_exp, ]
  68             else:
  69                 ret[sym].append(kconfig_exp)
  70         else:
  71             sym, dep = item.split()
  72             if not sym in ret:
  73                 ret[sym] = [dep, ]
  74             else:
  75                 ret[sym].append(dep)
  76     return ret
  77
  78
  79 def check_output_dir(d, clean):
  80     """
  81     Check that the output directory doesn't exist or is empty,
  82     unless clean is True in which case it's nuked. This helps
  83     sanity check the output when generating a tree, so usually
  84     running with --clean isn't suggested.
  85     """
  86     if clean:
  87         shutil.rmtree(d, ignore_errors=True)
  88     try:
  89         os.rmdir(d)
  90     except OSError as e:
  91         if e.errno != errno.ENOENT:
  92             raise
  93
  94
  95 def copytree(src, dst, symlinks=False, ignore=None):
  96     """
  97     Copy a directory tree. This differs from shutil.copytree()
  98     in that it allows destination directories to already exist.
  99     """
 100     names = os.listdir(src)
 101     if ignore is not None:
 102         ignored_names = ignore(src, names)
 103     else:
 104         ignored_names = set()
 105
 106     if not os.path.isdir(dst):
 107         os.makedirs(dst)
 108     errors = []
 109     for name in names:
 110         if name in ignored_names:
 111             continue
 112         srcname = os.path.join(src, name)
 113         dstname = os.path.join(dst, name)
 114         try:
 115             if symlinks and os.path.islink(srcname):
 116                 linkto = os.readlink(srcname)
 117                 os.symlink(linkto, dstname)
 118             elif os.path.isdir(srcname):
 119                 copytree(srcname, dstname, symlinks, ignore)
 120             else:
 121                 shutil.copy2(srcname, dstname)
 122         except (IOError, os.error) as why:
 123             errors.append((srcname, dstname, str(why)))
 124         # catch the Error from the recursive copytree so that we can
 125         # continue with other files
 126         except shutil.Error as err:
 127             errors.extend(err.args[0])
 128     try:
 129         shutil.copystat(src, dst)
 130     except WindowsError:
 131         # can't copy file access times on Windows
 132         pass
 133     except OSError as why:
 134         errors.extend((src, dst, str(why)))
 135     if errors:
 136         raise shutil.Error(errors)
 137
 138
 139 def copy_files(srcpath, copy_list, outdir):
 140     """
 141     Copy the copy_list files and directories from the srcpath
 142     to the outdir. The copy_list contains source and target
 143     names.
 144
 145     For now, it also ignores any *~ editor backup files, though
 146     this should probably be generalized (maybe using .gitignore?)
 147     Similarly the code that only copies some files (*.c, *.h,
 148     *.awk, Kconfig, Makefile) to avoid any build remnants in the
 149     kernel if they should exist.
 150     """
 151     for srcitem, tgtitem in copy_list:
 152         if tgtitem == '':
 153             copytree(srcpath, outdir, ignore=shutil.ignore_patterns('*~'))
 154         elif tgtitem[-1] == '/':
 155             def copy_ignore(dir, entries):
 156                 r = []
 157                 for i in entries:
 158                     if i[-2:] == '.o' or i[-1] == '~':
 159                         r.append(i)
 160                 return r
 161             copytree(os.path.join(srcpath, srcitem),
 162                      os.path.join(outdir, tgtitem),
 163                      ignore=copy_ignore)
 164         else:
 165             try:
 166                 os.makedirs(os.path.join(outdir, os.path.dirname(tgtitem)))
 167             except OSError as e:
 168                 # ignore dirs we might have created just now
 169                 if e.errno != errno.EEXIST:
 170                     raise
 171             shutil.copy(os.path.join(srcpath, srcitem),
 172                         os.path.join(outdir, tgtitem))
 173
 174
 175 def copy_git_files(srcpath, copy_list, rev, outdir):
 176     """
 177     "Copy" files from a git repository. This really means listing them with
 178     ls-tree and then using git show to obtain all the blobs.
 179     """
 180     for srcitem, tgtitem in copy_list:
 181         for m, t, h, f in git.ls_tree(rev=rev, files=(srcitem,), tree=srcpath):
 182             assert t == 'blob'
 183             f = os.path.join(outdir, f.replace(srcitem, tgtitem))
 184             d = os.path.dirname(f)
 185             if not os.path.exists(d):
 186                 os.makedirs(d)
 187             outf = open(f, 'w')
 188             git.get_blob(h, outf, tree=srcpath)
 189             outf.close()
 190             os.chmod(f, int(m, 8))
 191
 192 def automatic_backport_mangle_c_file(name):
 193     return name.replace('/', '-')
 194
 195
 196 def add_automatic_backports(args):
 197     disable_list = []
 198     export = re.compile(r'^EXPORT_SYMBOL(_GPL)?\((?P<sym>[^\)]*)\)')
 199     bpi = kconfig.get_backport_info(os.path.join(args.outdir, 'compat', 'Kconfig'))
 200     configtree = kconfig.ConfigTree(os.path.join(args.outdir, 'Kconfig'))
 201     all_selects = configtree.all_selects()
 202     for sym, vals in bpi.items():
 203         if sym.startswith('BACKPORT_BUILD_'):
 204             if not sym[15:] in all_selects:
 205                 disable_list.append(sym)
 206                 continue
 207         symtype, module_name, c_files, h_files = vals
 208
 209         # first copy files
 210         files = []
 211         for f in c_files:
 212             files.append((f, os.path.join('compat', automatic_backport_mangle_c_file(f))))
 213         for f in h_files:
 214             files.append((os.path.join('include', f),
 215                           os.path.join('include', os.path.dirname(f), 'backport-' + os.path.basename(f))))
 216         if args.git_revision:
 217             copy_git_files(args.kerneldir, files, args.git_revision, args.outdir)
 218         else:
 219             copy_files(args.kerneldir, files, args.outdir)
 220
 221         # now add the Makefile line
 222         mf = open(os.path.join(args.outdir, 'compat', 'Makefile'), 'a+')
 223         o_files = [automatic_backport_mangle_c_file(f)[:-1] + 'o' for f in c_files]
 224         if symtype == 'tristate':
 225             if not module_name:
 226                 raise Exception('backporting a module requires a #module-name')
 227             for of in o_files:
 228                 mf.write('%s-objs += %s\n' % (module_name, of))
 229             mf.write('obj-$(CPTCFG_%s) += %s.o\n' % (sym, module_name))
 230         elif symtype == 'bool':
 231             mf.write('compat-$(CPTCFG_%s) += %s\n' % (sym, ' '.join(o_files)))
 232
 233         # finally create the include file
 234         syms = []
 235         for f in c_files:
 236             for l in open(os.path.join(args.outdir, 'compat',
 237                                        automatic_backport_mangle_c_file(f)), 'r'):
 238                 m = export.match(l)
 239                 if m:
 240                     syms.append(m.group('sym'))
 241         for f in h_files:
 242             outf = open(os.path.join(args.outdir, 'include', f), 'w')
 243             outf.write('/* Automatically created during backport process */\n')
 244             outf.write('#ifndef CPTCFG_%s\n' % sym)
 245             outf.write('#include_next <%s>\n' % f)
 246             outf.write('#else\n');
 247             for s in syms:
 248                 outf.write('#undef %s\n' % s)
 249                 outf.write('#define %s LINUX_BACKPORT(%s)\n' % (s, s))
 250             outf.write('#include <%s>\n' % (os.path.dirname(f) + '/backport-' + os.path.basename(f), ))
 251             outf.write('#endif /* CPTCFG_%s */\n' % sym)
 252     return disable_list
 253
 254 def git_debug_init(args):
 255     """
 256     Initialize a git repository in the output directory and commit the current
 257     code in it. This is only used for debugging the transformations this code
 258     will do to the output later.
 259     """
 260     if not args.gitdebug:
 261         return
 262     git.init(tree=args.outdir)
 263     git.commit_all("Copied backport", tree=args.outdir)
 264
 265
 266 def git_debug_snapshot(args, name):
 267     """
 268     Take a git snapshot for the debugging.
 269     """
 270     if not args.gitdebug:
 271         return
 272     git.commit_all(name, tree=args.outdir)
 273
 274 def get_rel_spec_stable(rel):
 275     """
 276     Returns release specs for a linux-stable backports based release.
 277     """
 278     if ("rc" in rel):
 279         m = re.match(r"(?P<VERSION>\d+)\.+" \
 280                      "(?P<PATCHLEVEL>\d+)[.]*" \
 281                      "(?P<SUBLEVEL>\d*)" \
 282                      "[-rc]+(?P<RC_VERSION>\d+)\-+" \
 283                      "(?P<RELMOD_UPDATE>\d+)[-]*" \
 284                      "(?P<RELMOD_TYPE>[usnpc]*)", \
 285                      rel)
 286     else:
 287         m = re.match(r"(?P<VERSION>\d+)\.+" \
 288                      "(?P<PATCHLEVEL>\d+)[.]*" \
 289                      "(?P<SUBLEVEL>\d*)\-+" \
 290                      "(?P<RELMOD_UPDATE>\d+)[-]*" \
 291                      "(?P<RELMOD_TYPE>[usnpc]*)", \
 292                      rel)
 293     if (not m):
 294         return m
 295     return m.groupdict()
 296
 297 def get_rel_spec_next(rel):
 298     """
 299     Returns release specs for a linux-next backports based release.
 300     """
 301     m = re.match(r"(?P<DATE_VERSION>\d+)[-]*" \
 302                  "(?P<RELMOD_UPDATE>\d*)[-]*" \
 303                  "(?P<RELMOD_TYPE>[usnpc]*)", \
 304                  rel)
 305     if (not m):
 306         return m
 307     return m.groupdict()
 308
 309 def get_rel_prep(rel):
 310     """
 311     Returns a dict with prep work details we need prior to
 312     uploading a backports release to kernel.org
 313     """
 314     rel_specs = get_rel_spec_stable(rel)
 315     is_stable = True
 316     rel_tag = ""
 317     paths = list()
 318     if (not rel_specs):
 319         rel_specs = get_rel_spec_next(rel)
 320         if (not rel_specs):
 321             sys.stdout.write("rel: %s\n" % rel)
 322             return None
 323         if (rel_specs['RELMOD_UPDATE'] == '0' or
 324             rel_specs['RELMOD_UPDATE'] == '1'):
 325             return None
 326         is_stable = False
 327         date = rel_specs['DATE_VERSION']
 328         year = date[0:4]
 329         if (len(year) != 4):
 330             return None
 331         month = date[4:6]
 332         if (len(month) != 2):
 333             return None
 334         day = date[6:8]
 335         if (len(day) != 2):
 336             return None
 337         paths.append(year)
 338         paths.append(month)
 339         paths.append(day)
 340         rel_tag = "backports-" + rel.replace(rel_specs['RELMOD_TYPE'], "")
 341     else:
 342         ignore = "-"
 343         if (not rel_specs['RELMOD_UPDATE']):
 344             return None
 345         if (rel_specs['RELMOD_UPDATE'] == '0'):
 346             return None
 347         ignore += rel_specs['RELMOD_UPDATE']
 348         if (rel_specs['RELMOD_TYPE'] != ''):
 349             ignore += rel_specs['RELMOD_TYPE']
 350         base_rel = rel.replace(ignore, "")
 351         paths.append("v" + base_rel)
 352         rel_tag = "v" + rel.replace(rel_specs['RELMOD_TYPE'], "")
 353
 354     rel_prep = dict(stable = is_stable,
 355                     expected_tag = rel_tag,
 356                     paths_to_create = paths)
 357     return rel_prep
 358
 359 def create_tar_and_bz2(tar_name, dir_to_tar):
 360     """
 361     We need both a tar file and bzip2 for kernel.org, the tar file
 362     gets signed, then we upload the compressed version, kup-server
 363     in the backend decompresses and verifies the tarball against
 364     our signature.
 365     """
 366     basename = os.path.basename(dir_to_tar)
 367     tar = tarfile.open(tar_name, "w")
 368     tar.add(dir_to_tar, basename)
 369     tar.close()
 370
 371     tar_file = open(tar_name, "r")
 372
 373     bz2_file = bz2.BZ2File(tar_name + ".bz2", 'wb', compresslevel=9)
 374     bz2_file.write(tar_file.read())
 375     bz2_file.close()
 376
 377 def upload_release(args, rel_prep, logwrite=lambda x:None):
 378     """
 379     Given a path of a relase make tarball out of it, PGP sign it, and
 380     then upload it to kernel.org using kup.
 381
 382     The linux-next based release do not require a RELMOD_UPDATE
 383     given that typically only one release is made per day. Using
 384     RELMOD_UPDATE for these releases is allowed though and if
 385     present it must be > 1.
 386
 387     The linux-stable based releases require a RELMOD_UPDATE.
 388
 389     RELMOD_UPDATE must be numeric and > 0 just as the RC releases
 390     of the Linux kernel.
 391
 392     The tree must also be tagged with the respective release, without
 393     the RELMOD_TYPE. For linux-next based releases this consists of
 394     backports- followed by DATE_VERSION and if RELMOD_TYPE is present.
 395     For linux-stable releases this consists of v followed by the
 396     full release version except the RELMOD_TYPE.
 397
 398     Uploads will not be allowed if these rules are not followed.
 399     """
 400     korg_path = "/pub/linux/kernel/projects/backports"
 401
 402     if (rel_prep['stable']):
 403         korg_path += "/stable"
 404
 405     parent = os.path.dirname(args.outdir)
 406     release = os.path.basename(args.outdir)
 407     tar_name = parent + '/' + release + ".tar"
 408     bzip2_name = tar_name + ".bz2"
 409
 410     create_tar_and_bz2(tar_name, args.outdir)
 411
 412     logwrite(gpg.sign(tar_name, extra_args=['--armor', '--detach-sign']))
 413
 414     logwrite("------------------------------------------------------")
 415
 416     if (not args.kup_test):
 417         logwrite("About to upload, current target path contents:")
 418     else:
 419         logwrite("kup-test: current target path contents:")
 420
 421     logwrite(kup.ls(path=korg_path))
 422
 423     for path in rel_prep['paths_to_create']:
 424         korg_path += '/' + path
 425         if (not args.kup_test):
 426             logwrite("create directory: %s" % korg_path)
 427             logwrite(kup.mkdir(korg_path))
 428     korg_path += '/'
 429     if (not args.kup_test):
 430         logwrite("upload file %s to %s" % (bzip2_name, korg_path))
 431         logwrite(kup.put(bzip2_name, tar_name + '.asc', korg_path))
 432         logwrite("\nFinished upload!\n")
 433         logwrite("Target path contents:")
 434         logwrite(kup.ls(path=korg_path))
 435     else:
 436         kup_cmd = "kup put /\n\t\t%s /\n\t\t%s /\n\t\t%s" % (bzip2_name, tar_name + '.asc', korg_path)
 437         logwrite("kup-test: skipping cmd: %s" % kup_cmd)
 438
 439 def _main():
 440     # set up and parse arguments
 441     parser = argparse.ArgumentParser(description='generate backport tree')
 442     parser.add_argument('kerneldir', metavar='<kernel tree>', type=str,
 443                         help='Kernel tree to copy drivers from')
 444     parser.add_argument('outdir', metavar='<output directory>', type=str,
 445                         help='Directory to write the generated tree to')
 446     parser.add_argument('--copy-list', metavar='<listfile>', type=argparse.FileType('r'),
 447                         default='copy-list',
 448                         help='File containing list of files/directories to copy, default "copy-list"')
 449     parser.add_argument('--git-revision', metavar='<revision>', type=str,
 450                         help='git commit revision (see gitrevisions(7)) to take objects from.' +
 451                              'If this is specified, the kernel tree is used as git object storage ' +
 452                              'and we use git ls-tree to get the files.')
 453     parser.add_argument('--clean', const=True, default=False, action="store_const",
 454                         help='Clean output directory instead of erroring if it isn\'t empty')
 455     parser.add_argument('--refresh', const=True, default=False, action="store_const",
 456                         help='Refresh patches as they are applied, the source dir will be modified!')
 457     parser.add_argument('--base-name', metavar='<name>', type=str, default='Linux',
 458                         help='name of base tree, default just "Linux"')
 459     parser.add_argument('--gitdebug', const=True, default=False, action="store_const",
 460                         help='Use git, in the output tree, to debug the various transformation steps ' +
 461                              'that the tree generation makes (apply patches, ...)')
 462     parser.add_argument('--verbose', const=True, default=False, action="store_const",
 463                         help='Print more verbose information')
 464     parser.add_argument('--extra-driver', nargs=2, metavar=('<source dir>', '<copy-list>'), type=str,
 465                         action='append', default=[], help='Extra driver directory/copy-list.')
 466     parser.add_argument('--kup', const=True, default=False, action="store_const",
 467                         help='For maintainers: upload a release to kernel.org')
 468     parser.add_argument('--kup-test', const=True, default=False, action="store_const",
 469                         help='For maintainers: do all the work as if you were about to ' +
 470                              'upload to kernel.org but do not do the final `kup put` ' +
 471                              'and also do not run any `kup mkdir` commands. This will ' +
 472                              'however run `kup ls` on the target paths so ' +
 473                              'at the very least we test your kup configuration. ' +
 474                              'If this is your first time uploading use this first!')
 475     args = parser.parse_args()
 476
 477     def logwrite(msg):
 478         sys.stdout.write(msg)
 479         sys.stdout.write('\n')
 480         sys.stdout.flush()
 481
 482     return process(args.kerneldir, args.outdir, args.copy_list,
 483                    git_revision=args.git_revision, clean=args.clean,
 484                    refresh=args.refresh, base_name=args.base_name,
 485                    gitdebug=args.gitdebug, verbose=args.verbose,
 486                    extra_driver=args.extra_driver,
 487                    kup=args.kup,
 488                    kup_test=args.kup_test,
 489                    logwrite=logwrite)
 490
 491 def process(kerneldir, outdir, copy_list_file, git_revision=None,
 492             clean=False, refresh=False, base_name="Linux", gitdebug=False,
 493             verbose=False, extra_driver=[], kup=False,
 494             kup_test=False,
 495             logwrite=lambda x:None,
 496             git_tracked_version=False):
 497     class Args(object):
 498         def __init__(self, kerneldir, outdir, copy_list_file,
 499                      git_revision, clean, refresh, base_name,
 500                      gitdebug, verbose, extra_driver, kup,
 501                      kup_test):
 502             self.kerneldir = kerneldir
 503             self.outdir = outdir
 504             self.copy_list = copy_list_file
 505             self.git_revision = git_revision
 506             self.clean = clean
 507             self.refresh = refresh
 508             self.base_name = base_name
 509             self.gitdebug = gitdebug
 510             self.verbose = verbose
 511             self.extra_driver = extra_driver
 512             self.kup = kup
 513             self.kup_test = kup_test
 514     def git_paranoia(tree=None, logwrite=lambda x:None):
 515         data = git.paranoia(tree)
 516         if (data['r'] != 0):
 517             logwrite('Cannot use %s' % tree)
 518             logwrite('%s' % data['output'])
 519             sys.exit(data['r'])
 520         else:
 521             logwrite('Validated tree: %s' % tree)
 522
 523     args = Args(kerneldir, outdir, copy_list_file,
 524                 git_revision, clean, refresh, base_name,
 525                 gitdebug, verbose, extra_driver, kup, kup_test)
 526     rel_prep = None
 527
 528     # start processing ...
 529     if (args.kup or args.kup_test):
 530         git_paranoia(source_dir, logwrite)
 531         git_paranoia(kerneldir, logwrite)
 532
 533         rel_describe = git.describe(rev=None, tree=source_dir, extra_args=['--dirty'])
 534         release = os.path.basename(args.outdir)
 535         version = release.replace("backports-", "")
 536
 537         rel_prep = get_rel_prep(version)
 538         if (not rel_prep):
 539             logwrite('Invalid backports release name: %s' % release)
 540             logwrite('For rules on the release name see upload_release()')
 541             sys.exit(1)
 542         rel_type = "linux-stable"
 543         if (not rel_prep['stable']):
 544             rel_type = "linux-next"
 545         if (rel_prep['expected_tag'] != rel_describe):
 546             logwrite('Unexpected %s based backports release tag on' % rel_type)
 547             logwrite('the backports tree tree: %s\n' % rel_describe)
 548             logwrite('You asked to make a release with this ')
 549             logwrite('directory name: %s' % release)
 550             logwrite('The actual expected tag we should find on')
 551             logwrite('the backports tree then is: %s\n' % rel_prep['expected_tag'])
 552             logwrite('For rules on the release name see upload_release()')
 553             sys.exit(1)
 554
 555     copy_list = read_copy_list(args.copy_list)
 556     deplist = read_dependencies(os.path.join(source_dir, 'dependencies'))
 557
 558     # validate output directory
 559     check_output_dir(args.outdir, args.clean)
 560
 561     # do the copy
 562     backport_files = [(x, x) for x in [
 563         'Kconfig', 'Makefile', 'Makefile.build', 'Makefile.kernel', '.gitignore',
 564         'Makefile.real', 'compat/', 'backport-include/', 'kconf/', 'defconfigs/',
 565         'scripts/', '.blacklist.map', 'udev/',
 566     ]]
 567     if not args.git_revision:
 568         logwrite('Copy original source files ...')
 569     else:
 570         logwrite('Get original source files from git ...')
 571
 572     copy_files(os.path.join(source_dir, 'backport'), backport_files, args.outdir)
 573
 574     git_debug_init(args)
 575
 576     if not args.git_revision:
 577         copy_files(args.kerneldir, copy_list, args.outdir)
 578     else:
 579         copy_git_files(args.kerneldir, copy_list, args.git_revision, args.outdir)
 580
 581     # FIXME: should we add a git version of this (e.g. --git-extra-driver)?
 582     for src, copy_list in args.extra_driver:
 583         if (args.kup or args.kup_test):
 584             git_paranoia(src)
 585         copy_files(src, read_copy_list(open(copy_list, 'r')), args.outdir)
 586
 587     git_debug_snapshot(args, 'Add driver sources')
 588
 589     disable_list = add_automatic_backports(args)
 590     if disable_list:
 591         bpcfg = kconfig.ConfigTree(os.path.join(args.outdir, 'compat', 'Kconfig'))
 592         bpcfg.disable_symbols(disable_list)
 593     git_debug_snapshot(args, 'Add automatic backports')
 594
 595     logwrite('Apply patches ...')
 596     patches = []
 597     sempatches = []
 598     for root, dirs, files in os.walk(os.path.join(source_dir, 'patches')):
 599         for f in files:
 600             if f.endswith('.patch'):
 601                 patches.append(os.path.join(root, f))
 602             if f.endswith('.cocci'):
 603                 sempatches.append(os.path.join(root, f))
 604     patches.sort()
 605     prefix_len = len(os.path.join(source_dir, 'patches')) + 1
 606     for pfile in patches:
 607         print_name = pfile[prefix_len:]
 608         # read the patch file
 609         p = patch.fromfile(pfile)
 610         # complain if it's not a patch
 611         if not p:
 612             raise Exception('No patch content found in %s' % print_name)
 613         # leading / seems to be stripped?
 614         if 'dev/null' in p.items[0].source:
 615             raise Exception('Patches creating files are not supported (in %s)' % print_name)
 616         # check if the first file the patch touches exists, if so
 617         # assume the patch needs to be applied -- otherwise continue
 618         patched_file = '/'.join(p.items[0].source.split('/')[1:])
 619         fullfn = os.path.join(args.outdir, patched_file)
 620         if not os.path.exists(fullfn):
 621             if args.verbose:
 622                 logwrite("Not applying %s, not needed" % print_name)
 623             continue
 624         if args.verbose:
 625             logwrite("Applying patch %s" % print_name)
 626
 627         if args.refresh:
 628             # but for refresh, of course look at all files the patch touches
 629             for patchitem in p.items:
 630                 patched_file = '/'.join(patchitem.source.split('/')[1:])
 631                 fullfn = os.path.join(args.outdir, patched_file)
 632                 shutil.copyfile(fullfn, fullfn + '.orig_file')
 633
 634         process = subprocess.Popen(['patch', '-p1'], stdout=subprocess.PIPE,
 635                                    stderr=subprocess.STDOUT, stdin=subprocess.PIPE,
 636                                    close_fds=True, universal_newlines=True,
 637                                    cwd=args.outdir)
 638         output = process.communicate(input=open(pfile, 'r').read())[0]
 639         output = output.split('\n')
 640         if output[-1] == '':
 641             output = output[:-1]
 642         if args.verbose:
 643             for line in output:
 644                 logwrite('> %s' % line)
 645         if process.returncode != 0:
 646             if not args.verbose:
 647                 logwrite("Failed to apply changes from %s" % print_name)
 648                 for line in output:
 649                     logwrite('> %s' % line)
 650             return 2
 651
 652         if args.refresh:
 653             pfilef = open(pfile + '.tmp', 'a')
 654             pfilef.write(p.top_header)
 655             pfilef.flush()
 656             for patchitem in p.items:
 657                 patched_file = '/'.join(patchitem.source.split('/')[1:])
 658                 fullfn = os.path.join(args.outdir, patched_file)
 659                 process = subprocess.Popen(['diff', '-p', '-u', patched_file + '.orig_file', patched_file,
 660                                             '--label', 'a/' + patched_file,
 661                                             '--label', 'b/' + patched_file],
 662                                            stdout=pfilef, close_fds=True,
 663                                            universal_newlines=True, cwd=args.outdir)
 664                 process.wait()
 665                 os.unlink(fullfn + '.orig_file')
 666                 if not process.returncode in (0, 1):
 667                     logwrite("Failed to diff to refresh %s" % print_name)
 668                     pfilef.close()
 669                     os.unlink(pfile + '.tmp')
 670                     return 3
 671             pfilef.close()
 672             os.rename(pfile + '.tmp', pfile)
 673
 674         # remove orig/rej files that patch sometimes creates
 675         for root, dirs, files in os.walk(args.outdir):
 676             for f in files:
 677                 if f[-5:] == '.orig' or f[-4:] == '.rej':
 678                     os.unlink(os.path.join(root, f))
 679         git_debug_snapshot(args, "apply backport patch %s" % print_name)
 680
 681     sempatches.sort()
 682     prefix_len = len(os.path.join(source_dir, 'patches')) + 1
 683     for cocci_file in sempatches:
 684         print_name = cocci_file[prefix_len:]
 685         if args.verbose:
 686             logwrite("Applying patch %s" % print_name)
 687
 688         process = subprocess.Popen(['spatch', '--sp-file', cocci_file, '--in-place',
 689                                     '--backup-suffix', '.cocci_backup', '--dir', '.'],
 690                                    stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
 691                                    close_fds=True, universal_newlines=True,
 692                                    cwd=args.outdir)
 693         output = process.communicate()[0]
 694         output = output.split('\n')
 695         if output[-1] == '':
 696             output = output[:-1]
 697         if args.verbose:
 698             for line in output:
 699                 logwrite('> %s' % line)
 700         if process.returncode != 0:
 701             if not args.verbose:
 702                 logwrite("Failed to apply changes from %s" % print_name)
 703                 for line in output:
 704                     logwrite('> %s' % line)
 705             return 2
 706
 707         # remove cocci_backup files
 708         for root, dirs, files in os.walk(args.outdir):
 709             for f in files:
 710                 if f.endswith('.cocci_backup'):
 711                     os.unlink(os.path.join(root, f))
 712         git_debug_snapshot(args, "apply backport patch %s" % print_name)
 713
 714     # some post-processing is required
 715     configtree = kconfig.ConfigTree(os.path.join(args.outdir, 'Kconfig'))
 716     logwrite('Modify Kconfig tree ...')
 717     configtree.prune_sources(ignore=['Kconfig.kernel', 'Kconfig.versions'])
 718     git_debug_snapshot(args, "prune Kconfig tree")
 719     configtree.force_tristate_modular()
 720     git_debug_snapshot(args, "force tristate options modular")
 721     configtree.modify_selects()
 722     git_debug_snapshot(args, "convert select to depends on")
 723
 724     # write the versioning file
 725     if git_tracked_version:
 726         backports_version = "(see git)"
 727         kernel_version = "(see git)"
 728     else:
 729         backports_version = git.describe(tree=source_dir, extra_args=['--long'])
 730         kernel_version = git.describe(rev=args.git_revision or 'HEAD',
 731                                       tree=args.kerneldir,
 732                                       extra_args=['--long'])
 733     f = open(os.path.join(args.outdir, 'versions'), 'w')
 734     f.write('BACKPORTS_VERSION="%s"\n' % backports_version)
 735     f.write('BACKPORTED_KERNEL_VERSION="%s"\n' % kernel_version)
 736     f.write('BACKPORTED_KERNEL_NAME="%s"\n' % args.base_name)
 737     if git_tracked_version:
 738         f.write('BACKPORTS_GIT_TRACKED="backport tracker ID: $(shell git rev-parse HEAD 2>/dev/null || echo \'not built in git tree\')"\n')
 739     f.close()
 740
 741     symbols = configtree.symbols()
 742
 743     # write local symbol list -- needed during build
 744     f = open(os.path.join(args.outdir, '.local-symbols'), 'w')
 745     for sym in symbols:
 746         f.write('%s=\n' % sym)
 747     f.close()
 748
 749     git_debug_snapshot(args, "add versions/symbols files")
 750
 751     logwrite('Rewrite Makefiles and Kconfig files ...')
 752
 753     # rewrite Makefile and source symbols
 754     regexes = []
 755     for some_symbols in [symbols[i:i + 50] for i in range(0, len(symbols), 50)]:
 756         r = 'CONFIG_((' + '|'.join([s + '(_MODULE)?' for s in some_symbols]) + ')([^A-Za-z0-9_]|$))'
 757         regexes.append(re.compile(r, re.MULTILINE))
 758     for root, dirs, files in os.walk(args.outdir):
 759         # don't go into .git dir (possible debug thing)
 760         if '.git' in dirs:
 761             dirs.remove('.git')
 762         for f in files:
 763             data = open(os.path.join(root, f), 'r').read()
 764             for r in regexes:
 765                 data = r.sub(r'CPTCFG_\1', data)
 766             data = re.sub(r'\$\(srctree\)', '$(backport_srctree)', data)
 767             data = re.sub(r'-Idrivers', '-I$(backport_srctree)/drivers', data)
 768             fo = open(os.path.join(root, f), 'w')
 769             fo.write(data)
 770             fo.close()
 771
 772     git_debug_snapshot(args, "rename config symbol / srctree usage")
 773
 774     # disable unbuildable Kconfig symbols and stuff Makefiles that doesn't exist
 775     maketree = make.MakeTree(os.path.join(args.outdir, 'Makefile.kernel'))
 776     disable_kconfig = []
 777     disable_makefile = []
 778     for sym in maketree.get_impossible_symbols():
 779         disable_kconfig.append(sym[7:])
 780         disable_makefile.append(sym[7:])
 781
 782     configtree.disable_symbols(disable_kconfig)
 783     git_debug_snapshot(args, "disable impossible kconfig symbols")
 784
 785     # add kernel version dependencies to Kconfig, from the dependency list
 786     # we read previously
 787     for sym in tuple(deplist.keys()):
 788         new = []
 789         for dep in deplist[sym]:
 790             if "kconfig:" in dep:
 791                     kconfig_expr = dep.replace('kconfig: ', '')
 792                     new.append(kconfig_expr)
 793             elif (dep == "DISABLE"):
 794                     new.append('BACKPORT_DISABLED_KCONFIG_OPTION')
 795             else:
 796                     new.append('!BACKPORT_KERNEL_%s' % dep.replace('.', '_'))
 797         deplist[sym] = new
 798     configtree.add_dependencies(deplist)
 799     git_debug_snapshot(args, "add kernel version dependencies")
 800
 801     # disable things in makefiles that can't be selected and that the
 802     # build shouldn't recurse into because they don't exist -- if we
 803     # don't do that then a symbol from the kernel could cause the build
 804     # to attempt to recurse and fail
 805     #
 806     # Note that we split the regex after 50 symbols, this is because of a
 807     # limitation in the regex implementation (it only supports 100 nested
 808     # groups -- 50 seemed safer and is still fast)
 809     regexes = []
 810     for some_symbols in [disable_makefile[i:i + 50] for i in range(0, len(disable_makefile), 50)]:
 811         r = '^([^#].*((CPTCFG|CONFIG)_(' + '|'.join([s for s in some_symbols]) + ')))'
 812         regexes.append(re.compile(r, re.MULTILINE))
 813     for f in maketree.get_makefiles():
 814         data = open(f, 'r').read()
 815         for r in regexes:
 816             data = r.sub(r'#\1', data)
 817         fo = open(f, 'w')
 818         fo.write(data)
 819         fo.close()
 820     git_debug_snapshot(args, "disable unsatisfied Makefile parts")
 821
 822     if (args.kup or args.kup_test):
 823         upload_release(args, rel_prep, logwrite=logwrite)
 824
 825     logwrite('Done!')
 826     return 0
 827
 828 if __name__ == '__main__':
 829     ret = _main()
 830     if ret:
 831         sys.exit(ret)