gentree.py

   1 #!/usr/bin/env python
   2 #
   3 # Generate the output tree into a specified directory.
   4 #
   5
   6 import argparse, sys, os, errno, shutil, re, subprocess
   7 import tarfile, gzip
   8
   9 # find self
  10 source_dir = os.path.abspath(os.path.dirname(__file__))
  11 sys.path.append(source_dir)
  12 # and import libraries we have
  13 from lib import kconfig, patch, make
  14 from lib import bpgit as git
  15 from lib import bpgpg as gpg
  16 from lib import bpkup as kup
  17 from lib.tempdir import tempdir
  18
  19 def read_copy_list(copyfile):
  20     """
  21     Read a copy-list file and return a list of (source, target)
  22     tuples. The source and target are usually the same, but in
  23     the copy-list file there may be a rename included.
  24     """
  25     ret = []
  26     for item in copyfile:
  27         # remove leading/trailing whitespace
  28         item = item.strip()
  29         # comments
  30         if not item or item[0] == '#':
  31             continue
  32         if item[0] == '/':
  33             raise Exception("Input path '%s' is absolute path, this isn't allowed" % (item, ))
  34         if ' -> ' in item:
  35             srcitem, dstitem = item.split(' -> ')
  36             if (srcitem[-1] == '/') != (dstitem[-1] == '/'):
  37                 raise Exception("Cannot copy file/dir to dir/file")
  38         else:
  39             srcitem = dstitem = item
  40         ret.append((srcitem, dstitem))
  41     return ret
  42
  43
  44 def read_dependencies(depfilename):
  45     """
  46     Read a (the) dependency file and return the list of
  47     dependencies as a dictionary, mapping a Kconfig symbol
  48     to a list of kernel version dependencies.
  49
  50     If a backported feature that an upstream backported driver
  51     depends on had kconfig limitations (ie, debugging feature not
  52     available) a built constaint restriction can be expressed
  53     by using a kconfig expression. The kconfig expressions can
  54     be specified by using the "kconfig: " prefix.
  55
  56     While reading ignore blank or commented lines.
  57     """
  58     ret = {}
  59     depfile = open(depfilename, 'r')
  60     for item in depfile:
  61         kconfig_exp = ""
  62         item = item.strip()
  63         if not item or item[0] == '#':
  64             continue
  65         if "kconfig:" in item:
  66             sym, kconfig_exp = item.split(" ", 1)
  67             if not sym in ret:
  68                 ret[sym] = [kconfig_exp, ]
  69             else:
  70                 ret[sym].append(kconfig_exp)
  71         else:
  72             sym, dep = item.split()
  73             if not sym in ret:
  74                 ret[sym] = [dep, ]
  75             else:
  76                 ret[sym].append(dep)
  77     return ret
  78
  79
  80 def check_output_dir(d, clean):
  81     """
  82     Check that the output directory doesn't exist or is empty,
  83     unless clean is True in which case it's nuked. This helps
  84     sanity check the output when generating a tree, so usually
  85     running with --clean isn't suggested.
  86     """
  87     if clean:
  88         shutil.rmtree(d, ignore_errors=True)
  89     try:
  90         os.rmdir(d)
  91     except OSError as e:
  92         if e.errno != errno.ENOENT:
  93             raise
  94
  95
  96 def copytree(src, dst, symlinks=False, ignore=None):
  97     """
  98     Copy a directory tree. This differs from shutil.copytree()
  99     in that it allows destination directories to already exist.
 100     """
 101     names = os.listdir(src)
 102     if ignore is not None:
 103         ignored_names = ignore(src, names)
 104     else:
 105         ignored_names = set()
 106
 107     if not os.path.isdir(dst):
 108         os.makedirs(dst)
 109     errors = []
 110     for name in names:
 111         if name in ignored_names:
 112             continue
 113         srcname = os.path.join(src, name)
 114         dstname = os.path.join(dst, name)
 115         try:
 116             if symlinks and os.path.islink(srcname):
 117                 linkto = os.readlink(srcname)
 118                 os.symlink(linkto, dstname)
 119             elif os.path.isdir(srcname):
 120                 copytree(srcname, dstname, symlinks, ignore)
 121             else:
 122                 shutil.copy2(srcname, dstname)
 123         except (IOError, os.error) as why:
 124             errors.append((srcname, dstname, str(why)))
 125         # catch the Error from the recursive copytree so that we can
 126         # continue with other files
 127         except shutil.Error as err:
 128             errors.extend(err.args[0])
 129     try:
 130         shutil.copystat(src, dst)
 131     except WindowsError:
 132         # can't copy file access times on Windows
 133         pass
 134     except OSError as why:
 135         errors.extend((src, dst, str(why)))
 136     if errors:
 137         raise shutil.Error(errors)
 138
 139
 140 def copy_files(srcpath, copy_list, outdir):
 141     """
 142     Copy the copy_list files and directories from the srcpath
 143     to the outdir. The copy_list contains source and target
 144     names.
 145
 146     For now, it also ignores any *~ editor backup files, though
 147     this should probably be generalized (maybe using .gitignore?)
 148     Similarly the code that only copies some files (*.c, *.h,
 149     *.awk, Kconfig, Makefile) to avoid any build remnants in the
 150     kernel if they should exist.
 151     """
 152     for srcitem, tgtitem in copy_list:
 153         if tgtitem == '':
 154             copytree(srcpath, outdir, ignore=shutil.ignore_patterns('*~'))
 155         elif tgtitem[-1] == '/':
 156             def copy_ignore(dir, entries):
 157                 r = []
 158                 for i in entries:
 159                     if i[-2:] == '.o' or i[-1] == '~':
 160                         r.append(i)
 161                 return r
 162             copytree(os.path.join(srcpath, srcitem),
 163                      os.path.join(outdir, tgtitem),
 164                      ignore=copy_ignore)
 165         else:
 166             try:
 167                 os.makedirs(os.path.join(outdir, os.path.dirname(tgtitem)))
 168             except OSError as e:
 169                 # ignore dirs we might have created just now
 170                 if e.errno != errno.EEXIST:
 171                     raise
 172             shutil.copy(os.path.join(srcpath, srcitem),
 173                         os.path.join(outdir, tgtitem))
 174
 175
 176 def copy_git_files(srcpath, copy_list, rev, outdir):
 177     """
 178     "Copy" files from a git repository. This really means listing them with
 179     ls-tree and then using git show to obtain all the blobs.
 180     """
 181     for srcitem, tgtitem in copy_list:
 182         for m, t, h, f in git.ls_tree(rev=rev, files=(srcitem,), tree=srcpath):
 183             assert t == 'blob'
 184             f = os.path.join(outdir, f.replace(srcitem, tgtitem))
 185             d = os.path.dirname(f)
 186             if not os.path.exists(d):
 187                 os.makedirs(d)
 188             outf = open(f, 'w')
 189             git.get_blob(h, outf, tree=srcpath)
 190             outf.close()
 191             os.chmod(f, int(m, 8))
 192
 193 def automatic_backport_mangle_c_file(name):
 194     return name.replace('/', '-')
 195
 196
 197 def add_automatic_backports(args):
 198     disable_list = []
 199     export = re.compile(r'^EXPORT_SYMBOL(_GPL)?\((?P<sym>[^\)]*)\)')
 200     bpi = kconfig.get_backport_info(os.path.join(args.outdir, 'compat', 'Kconfig'))
 201     configtree = kconfig.ConfigTree(os.path.join(args.outdir, 'Kconfig'))
 202     all_selects = configtree.all_selects()
 203     for sym, vals in bpi.items():
 204         if sym.startswith('BACKPORT_BUILD_'):
 205             if not sym[15:] in all_selects:
 206                 disable_list.append(sym)
 207                 continue
 208         symtype, module_name, c_files, h_files = vals
 209
 210         # first copy files
 211         files = []
 212         for f in c_files:
 213             files.append((f, os.path.join('compat', automatic_backport_mangle_c_file(f))))
 214         for f in h_files:
 215             files.append((os.path.join('include', f),
 216                           os.path.join('include', os.path.dirname(f), 'backport-' + os.path.basename(f))))
 217         if args.git_revision:
 218             copy_git_files(args.kerneldir, files, args.git_revision, args.outdir)
 219         else:
 220             copy_files(args.kerneldir, files, args.outdir)
 221
 222         # now add the Makefile line
 223         mf = open(os.path.join(args.outdir, 'compat', 'Makefile'), 'a+')
 224         o_files = [automatic_backport_mangle_c_file(f)[:-1] + 'o' for f in c_files]
 225         if symtype == 'tristate':
 226             if not module_name:
 227                 raise Exception('backporting a module requires a #module-name')
 228             for of in o_files:
 229                 mf.write('%s-objs += %s\n' % (module_name, of))
 230             mf.write('obj-$(CPTCFG_%s) += %s.o\n' % (sym, module_name))
 231         elif symtype == 'bool':
 232             mf.write('compat-$(CPTCFG_%s) += %s\n' % (sym, ' '.join(o_files)))
 233
 234         # finally create the include file
 235         syms = []
 236         for f in c_files:
 237             for l in open(os.path.join(args.outdir, 'compat',
 238                                        automatic_backport_mangle_c_file(f)), 'r'):
 239                 m = export.match(l)
 240                 if m:
 241                     syms.append(m.group('sym'))
 242         for f in h_files:
 243             outf = open(os.path.join(args.outdir, 'include', f), 'w')
 244             outf.write('/* Automatically created during backport process */\n')
 245             outf.write('#ifndef CPTCFG_%s\n' % sym)
 246             outf.write('#include_next <%s>\n' % f)
 247             outf.write('#else\n');
 248             for s in syms:
 249                 outf.write('#undef %s\n' % s)
 250                 outf.write('#define %s LINUX_BACKPORT(%s)\n' % (s, s))
 251             outf.write('#include <%s>\n' % (os.path.dirname(f) + '/backport-' + os.path.basename(f), ))
 252             outf.write('#endif /* CPTCFG_%s */\n' % sym)
 253     return disable_list
 254
 255 def git_debug_init(args):
 256     """
 257     Initialize a git repository in the output directory and commit the current
 258     code in it. This is only used for debugging the transformations this code
 259     will do to the output later.
 260     """
 261     if not args.gitdebug:
 262         return
 263     git.init(tree=args.outdir)
 264     git.commit_all("Copied backport", tree=args.outdir)
 265
 266
 267 def git_debug_snapshot(args, name):
 268     """
 269     Take a git snapshot for the debugging.
 270     """
 271     if not args.gitdebug:
 272         return
 273     git.commit_all(name, tree=args.outdir)
 274
 275 def get_rel_spec_stable(rel):
 276     """
 277     Returns release specs for a linux-stable backports based release.
 278     """
 279     if ("rc" in rel):
 280         m = re.match(r"(?P<VERSION>\d+)\.+" \
 281                      "(?P<PATCHLEVEL>\d+)[.]*" \
 282                      "(?P<SUBLEVEL>\d*)" \
 283                      "[-rc]+(?P<RC_VERSION>\d+)\-+" \
 284                      "(?P<RELMOD_UPDATE>\d+)[-]*" \
 285                      "(?P<RELMOD_TYPE>[usnpc]*)", \
 286                      rel)
 287     else:
 288         m = re.match(r"(?P<VERSION>\d+)\.+" \
 289                      "(?P<PATCHLEVEL>\d+)[.]*" \
 290                      "(?P<SUBLEVEL>\d*)\-+" \
 291                      "(?P<RELMOD_UPDATE>\d+)[-]*" \
 292                      "(?P<RELMOD_TYPE>[usnpc]*)", \
 293                      rel)
 294     if (not m):
 295         return m
 296     return m.groupdict()
 297
 298 def get_rel_spec_next(rel):
 299     """
 300     Returns release specs for a linux-next backports based release.
 301     """
 302     m = re.match(r"(?P<DATE_VERSION>\d+)[-]*" \
 303                  "(?P<RELMOD_UPDATE>\d*)[-]*" \
 304                  "(?P<RELMOD_TYPE>[usnpc]*)", \
 305                  rel)
 306     if (not m):
 307         return m
 308     return m.groupdict()
 309
 310 def get_rel_prep(rel):
 311     """
 312     Returns a dict with prep work details we need prior to
 313     uploading a backports release to kernel.org
 314     """
 315     rel_specs = get_rel_spec_stable(rel)
 316     is_stable = True
 317     rel_tag = ""
 318     paths = list()
 319     if (not rel_specs):
 320         rel_specs = get_rel_spec_next(rel)
 321         if (not rel_specs):
 322             sys.stdout.write("rel: %s\n" % rel)
 323             return None
 324         if (rel_specs['RELMOD_UPDATE'] == '0' or
 325             rel_specs['RELMOD_UPDATE'] == '1'):
 326             return None
 327         is_stable = False
 328         date = rel_specs['DATE_VERSION']
 329         year = date[0:4]
 330         if (len(year) != 4):
 331             return None
 332         month = date[4:6]
 333         if (len(month) != 2):
 334             return None
 335         day = date[6:8]
 336         if (len(day) != 2):
 337             return None
 338         paths.append(year)
 339         paths.append(month)
 340         paths.append(day)
 341         rel_tag = "backports-" + rel.replace(rel_specs['RELMOD_TYPE'], "")
 342     else:
 343         ignore = "-"
 344         if (not rel_specs['RELMOD_UPDATE']):
 345             return None
 346         if (rel_specs['RELMOD_UPDATE'] == '0'):
 347             return None
 348         ignore += rel_specs['RELMOD_UPDATE']
 349         if (rel_specs['RELMOD_TYPE'] != ''):
 350             ignore += rel_specs['RELMOD_TYPE']
 351         base_rel = rel.replace(ignore, "")
 352         paths.append("v" + base_rel)
 353         rel_tag = "v" + rel.replace(rel_specs['RELMOD_TYPE'], "")
 354
 355     rel_prep = dict(stable = is_stable,
 356                     expected_tag = rel_tag,
 357                     paths_to_create = paths)
 358     return rel_prep
 359
 360 def create_tar_and_gz(tar_name, dir_to_tar):
 361     """
 362     We need both a tar file and gzip for kernel.org, the tar file
 363     gets signed, then we upload the compressed version, kup-server
 364     in the backend decompresses and verifies the tarball against
 365     our signature.
 366     """
 367     basename = os.path.basename(dir_to_tar)
 368     tar = tarfile.open(tar_name, "w")
 369     tar.add(dir_to_tar, basename)
 370     tar.close()
 371
 372     tar_file = open(tar_name, "r")
 373
 374     gz_file = gzip.GzipFile(tar_name + ".gz", 'wb')
 375     gz_file.write(tar_file.read())
 376     gz_file.close()
 377
 378 def upload_release(args, rel_prep, logwrite=lambda x:None):
 379     """
 380     Given a path of a relase make tarball out of it, PGP sign it, and
 381     then upload it to kernel.org using kup.
 382
 383     The linux-next based release do not require a RELMOD_UPDATE
 384     given that typically only one release is made per day. Using
 385     RELMOD_UPDATE for these releases is allowed though and if
 386     present it must be > 1.
 387
 388     The linux-stable based releases require a RELMOD_UPDATE.
 389
 390     RELMOD_UPDATE must be numeric and > 0 just as the RC releases
 391     of the Linux kernel.
 392
 393     The tree must also be tagged with the respective release, without
 394     the RELMOD_TYPE. For linux-next based releases this consists of
 395     backports- followed by DATE_VERSION and if RELMOD_TYPE is present.
 396     For linux-stable releases this consists of v followed by the
 397     full release version except the RELMOD_TYPE.
 398
 399     Uploads will not be allowed if these rules are not followed.
 400     """
 401     korg_path = "/pub/linux/kernel/projects/backports"
 402
 403     if (rel_prep['stable']):
 404         korg_path += "/stable"
 405
 406     parent = os.path.dirname(args.outdir)
 407     release = os.path.basename(args.outdir)
 408     tar_name = parent + '/' + release + ".tar"
 409     gzip_name = tar_name + ".gz"
 410
 411     create_tar_and_gz(tar_name, args.outdir)
 412
 413     logwrite(gpg.sign(tar_name, extra_args=['--armor', '--detach-sign']))
 414
 415     logwrite("------------------------------------------------------")
 416
 417     if (not args.kup_test):
 418         logwrite("About to upload, current target path contents:")
 419     else:
 420         logwrite("kup-test: current target path contents:")
 421
 422     logwrite(kup.ls(path=korg_path))
 423
 424     for path in rel_prep['paths_to_create']:
 425         korg_path += '/' + path
 426         if (not args.kup_test):
 427             logwrite("create directory: %s" % korg_path)
 428             logwrite(kup.mkdir(korg_path))
 429     korg_path += '/'
 430     if (not args.kup_test):
 431         logwrite("upload file %s to %s" % (gzip_name, korg_path))
 432         logwrite(kup.put(gzip_name, tar_name + '.asc', korg_path))
 433         logwrite("\nFinished upload!\n")
 434         logwrite("Target path contents:")
 435         logwrite(kup.ls(path=korg_path))
 436     else:
 437         kup_cmd = "kup put /\n\t\t%s /\n\t\t%s /\n\t\t%s" % (gzip_name, tar_name + '.asc', korg_path)
 438         logwrite("kup-test: skipping cmd: %s" % kup_cmd)
 439
 440 def _main():
 441     # set up and parse arguments
 442     parser = argparse.ArgumentParser(description='generate backport tree')
 443     parser.add_argument('kerneldir', metavar='<kernel tree>', type=str,
 444                         help='Kernel tree to copy drivers from')
 445     parser.add_argument('outdir', metavar='<output directory>', type=str,
 446                         help='Directory to write the generated tree to')
 447     parser.add_argument('--copy-list', metavar='<listfile>', type=argparse.FileType('r'),
 448                         default='copy-list',
 449                         help='File containing list of files/directories to copy, default "copy-list"')
 450     parser.add_argument('--git-revision', metavar='<revision>', type=str,
 451                         help='git commit revision (see gitrevisions(7)) to take objects from.' +
 452                              'If this is specified, the kernel tree is used as git object storage ' +
 453                              'and we use git ls-tree to get the files.')
 454     parser.add_argument('--clean', const=True, default=False, action="store_const",
 455                         help='Clean output directory instead of erroring if it isn\'t empty')
 456     parser.add_argument('--refresh', const=True, default=False, action="store_const",
 457                         help='Refresh patches as they are applied, the source dir will be modified!')
 458     parser.add_argument('--base-name', metavar='<name>', type=str, default='Linux',
 459                         help='name of base tree, default just "Linux"')
 460     parser.add_argument('--gitdebug', const=True, default=False, action="store_const",
 461                         help='Use git, in the output tree, to debug the various transformation steps ' +
 462                              'that the tree generation makes (apply patches, ...)')
 463     parser.add_argument('--verbose', const=True, default=False, action="store_const",
 464                         help='Print more verbose information')
 465     parser.add_argument('--extra-driver', nargs=2, metavar=('<source dir>', '<copy-list>'), type=str,
 466                         action='append', default=[], help='Extra driver directory/copy-list.')
 467     parser.add_argument('--kup', const=True, default=False, action="store_const",
 468                         help='For maintainers: upload a release to kernel.org')
 469     parser.add_argument('--kup-test', const=True, default=False, action="store_const",
 470                         help='For maintainers: do all the work as if you were about to ' +
 471                              'upload to kernel.org but do not do the final `kup put` ' +
 472                              'and also do not run any `kup mkdir` commands. This will ' +
 473                              'however run `kup ls` on the target paths so ' +
 474                              'at the very least we test your kup configuration. ' +
 475                              'If this is your first time uploading use this first!')
 476     args = parser.parse_args()
 477
 478     def logwrite(msg):
 479         sys.stdout.write(msg)
 480         sys.stdout.write('\n')
 481         sys.stdout.flush()
 482
 483     return process(args.kerneldir, args.outdir, args.copy_list,
 484                    git_revision=args.git_revision, clean=args.clean,
 485                    refresh=args.refresh, base_name=args.base_name,
 486                    gitdebug=args.gitdebug, verbose=args.verbose,
 487                    extra_driver=args.extra_driver,
 488                    kup=args.kup,
 489                    kup_test=args.kup_test,
 490                    logwrite=logwrite)
 491
 492 def process(kerneldir, outdir, copy_list_file, git_revision=None,
 493             clean=False, refresh=False, base_name="Linux", gitdebug=False,
 494             verbose=False, extra_driver=[], kup=False,
 495             kup_test=False,
 496             logwrite=lambda x:None,
 497             git_tracked_version=False):
 498     class Args(object):
 499         def __init__(self, kerneldir, outdir, copy_list_file,
 500                      git_revision, clean, refresh, base_name,
 501                      gitdebug, verbose, extra_driver, kup,
 502                      kup_test):
 503             self.kerneldir = kerneldir
 504             self.outdir = outdir
 505             self.copy_list = copy_list_file
 506             self.git_revision = git_revision
 507             self.clean = clean
 508             self.refresh = refresh
 509             self.base_name = base_name
 510             self.gitdebug = gitdebug
 511             self.verbose = verbose
 512             self.extra_driver = extra_driver
 513             self.kup = kup
 514             self.kup_test = kup_test
 515     def git_paranoia(tree=None, logwrite=lambda x:None):
 516         data = git.paranoia(tree)
 517         if (data['r'] != 0):
 518             logwrite('Cannot use %s' % tree)
 519             logwrite('%s' % data['output'])
 520             sys.exit(data['r'])
 521         else:
 522             logwrite('Validated tree: %s' % tree)
 523
 524     args = Args(kerneldir, outdir, copy_list_file,
 525                 git_revision, clean, refresh, base_name,
 526                 gitdebug, verbose, extra_driver, kup, kup_test)
 527     rel_prep = None
 528
 529     # start processing ...
 530     if (args.kup or args.kup_test):
 531         git_paranoia(source_dir, logwrite)
 532         git_paranoia(kerneldir, logwrite)
 533
 534         rel_describe = git.describe(rev=None, tree=source_dir, extra_args=['--dirty'])
 535         release = os.path.basename(args.outdir)
 536         version = release.replace("backports-", "")
 537
 538         rel_prep = get_rel_prep(version)
 539         if (not rel_prep):
 540             logwrite('Invalid backports release name: %s' % release)
 541             logwrite('For rules on the release name see upload_release()')
 542             sys.exit(1)
 543         rel_type = "linux-stable"
 544         if (not rel_prep['stable']):
 545             rel_type = "linux-next"
 546         if (rel_prep['expected_tag'] != rel_describe):
 547             logwrite('Unexpected %s based backports release tag on' % rel_type)
 548             logwrite('the backports tree tree: %s\n' % rel_describe)
 549             logwrite('You asked to make a release with this ')
 550             logwrite('directory name: %s' % release)
 551             logwrite('The actual expected tag we should find on')
 552             logwrite('the backports tree then is: %s\n' % rel_prep['expected_tag'])
 553             logwrite('For rules on the release name see upload_release()')
 554             sys.exit(1)
 555
 556     copy_list = read_copy_list(args.copy_list)
 557     deplist = read_dependencies(os.path.join(source_dir, 'dependencies'))
 558
 559     # validate output directory
 560     check_output_dir(args.outdir, args.clean)
 561
 562     # do the copy
 563     backport_files = [(x, x) for x in [
 564         'Kconfig', 'Makefile', 'Makefile.build', 'Makefile.kernel', '.gitignore',
 565         'Makefile.real', 'compat/', 'backport-include/', 'kconf/', 'defconfigs/',
 566         'scripts/', '.blacklist.map', 'udev/',
 567     ]]
 568     if not args.git_revision:
 569         logwrite('Copy original source files ...')
 570     else:
 571         logwrite('Get original source files from git ...')
 572
 573     copy_files(os.path.join(source_dir, 'backport'), backport_files, args.outdir)
 574
 575     git_debug_init(args)
 576
 577     if not args.git_revision:
 578         copy_files(args.kerneldir, copy_list, args.outdir)
 579     else:
 580         copy_git_files(args.kerneldir, copy_list, args.git_revision, args.outdir)
 581
 582     # FIXME: should we add a git version of this (e.g. --git-extra-driver)?
 583     for src, copy_list in args.extra_driver:
 584         if (args.kup or args.kup_test):
 585             git_paranoia(src)
 586         copy_files(src, read_copy_list(open(copy_list, 'r')), args.outdir)
 587
 588     git_debug_snapshot(args, 'Add driver sources')
 589
 590     disable_list = add_automatic_backports(args)
 591     if disable_list:
 592         bpcfg = kconfig.ConfigTree(os.path.join(args.outdir, 'compat', 'Kconfig'))
 593         bpcfg.disable_symbols(disable_list)
 594     git_debug_snapshot(args, 'Add automatic backports')
 595
 596     logwrite('Apply patches ...')
 597     patches = []
 598     sempatches = []
 599     for root, dirs, files in os.walk(os.path.join(source_dir, 'patches')):
 600         for f in files:
 601             if f.endswith('.patch'):
 602                 patches.append(os.path.join(root, f))
 603             if f.endswith('.cocci'):
 604                 sempatches.append(os.path.join(root, f))
 605     patches.sort()
 606     prefix_len = len(os.path.join(source_dir, 'patches')) + 1
 607     for pfile in patches:
 608         print_name = pfile[prefix_len:]
 609         # read the patch file
 610         p = patch.fromfile(pfile)
 611         # complain if it's not a patch
 612         if not p:
 613             raise Exception('No patch content found in %s' % print_name)
 614         # leading / seems to be stripped?
 615         if 'dev/null' in p.items[0].source:
 616             raise Exception('Patches creating files are not supported (in %s)' % print_name)
 617         # check if the first file the patch touches exists, if so
 618         # assume the patch needs to be applied -- otherwise continue
 619         patched_file = '/'.join(p.items[0].source.split('/')[1:])
 620         fullfn = os.path.join(args.outdir, patched_file)
 621         if not os.path.exists(fullfn):
 622             if args.verbose:
 623                 logwrite("Not applying %s, not needed" % print_name)
 624             continue
 625         if args.verbose:
 626             logwrite("Applying patch %s" % print_name)
 627
 628         if args.refresh:
 629             # but for refresh, of course look at all files the patch touches
 630             for patchitem in p.items:
 631                 patched_file = '/'.join(patchitem.source.split('/')[1:])
 632                 fullfn = os.path.join(args.outdir, patched_file)
 633                 shutil.copyfile(fullfn, fullfn + '.orig_file')
 634
 635         process = subprocess.Popen(['patch', '-p1'], stdout=subprocess.PIPE,
 636                                    stderr=subprocess.STDOUT, stdin=subprocess.PIPE,
 637                                    close_fds=True, universal_newlines=True,
 638                                    cwd=args.outdir)
 639         output = process.communicate(input=open(pfile, 'r').read())[0]
 640         output = output.split('\n')
 641         if output[-1] == '':
 642             output = output[:-1]
 643         if args.verbose:
 644             for line in output:
 645                 logwrite('> %s' % line)
 646         if process.returncode != 0:
 647             if not args.verbose:
 648                 logwrite("Failed to apply changes from %s" % print_name)
 649                 for line in output:
 650                     logwrite('> %s' % line)
 651             return 2
 652
 653         if args.refresh:
 654             pfilef = open(pfile + '.tmp', 'a')
 655             pfilef.write(p.top_header)
 656             pfilef.flush()
 657             for patchitem in p.items:
 658                 patched_file = '/'.join(patchitem.source.split('/')[1:])
 659                 fullfn = os.path.join(args.outdir, patched_file)
 660                 process = subprocess.Popen(['diff', '-p', '-u', patched_file + '.orig_file', patched_file,
 661                                             '--label', 'a/' + patched_file,
 662                                             '--label', 'b/' + patched_file],
 663                                            stdout=pfilef, close_fds=True,
 664                                            universal_newlines=True, cwd=args.outdir)
 665                 process.wait()
 666                 os.unlink(fullfn + '.orig_file')
 667                 if not process.returncode in (0, 1):
 668                     logwrite("Failed to diff to refresh %s" % print_name)
 669                     pfilef.close()
 670                     os.unlink(pfile + '.tmp')
 671                     return 3
 672             pfilef.close()
 673             os.rename(pfile + '.tmp', pfile)
 674
 675         # remove orig/rej files that patch sometimes creates
 676         for root, dirs, files in os.walk(args.outdir):
 677             for f in files:
 678                 if f[-5:] == '.orig' or f[-4:] == '.rej':
 679                     os.unlink(os.path.join(root, f))
 680         git_debug_snapshot(args, "apply backport patch %s" % print_name)
 681
 682     sempatches.sort()
 683     with tempdir() as t:
 684         if not args.gitdebug:
 685             # combine all spatches
 686             fn = os.path.join(t, 'combined.cocci')
 687             f = open(fn, 'w')
 688             for cocci_file in sempatches:
 689                 for l in open(cocci_file, 'r'):
 690                     f.write(l)
 691                 f.write('\n')
 692             f.close()
 693             sempatches = [fn]
 694             prefix_len = 0
 695         else:
 696             prefix_len = len(os.path.join(source_dir, 'patches')) + 1
 697         for cocci_file in sempatches:
 698             print_name = cocci_file[prefix_len:]
 699             if args.verbose:
 700                 logwrite("Applying patch %s" % print_name)
 701
 702             process = subprocess.Popen(['spatch', '--sp-file', cocci_file, '--in-place',
 703                                         '--backup-suffix', '.cocci_backup', '--dir', '.'],
 704                                        stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
 705                                        close_fds=True, universal_newlines=True,
 706                                        cwd=args.outdir)
 707             output = process.communicate()[0]
 708             output = output.split('\n')
 709             if output[-1] == '':
 710                 output = output[:-1]
 711             if args.verbose:
 712                 for line in output:
 713                     logwrite('> %s' % line)
 714             if process.returncode != 0:
 715                 if not args.verbose:
 716                     logwrite("Failed to apply changes from %s" % print_name)
 717                     for line in output:
 718                         logwrite('> %s' % line)
 719                 return 2
 720
 721             # remove cocci_backup files
 722             for root, dirs, files in os.walk(args.outdir):
 723                 for f in files:
 724                     if f.endswith('.cocci_backup'):
 725                         os.unlink(os.path.join(root, f))
 726             git_debug_snapshot(args, "apply backport patch %s" % print_name)
 727
 728     # some post-processing is required
 729     configtree = kconfig.ConfigTree(os.path.join(args.outdir, 'Kconfig'))
 730     logwrite('Modify Kconfig tree ...')
 731     configtree.prune_sources(ignore=['Kconfig.kernel', 'Kconfig.versions'])
 732     git_debug_snapshot(args, "prune Kconfig tree")
 733     configtree.force_tristate_modular()
 734     git_debug_snapshot(args, "force tristate options modular")
 735     configtree.modify_selects()
 736     git_debug_snapshot(args, "convert select to depends on")
 737
 738     # write the versioning file
 739     if git_tracked_version:
 740         backports_version = "(see git)"
 741         kernel_version = "(see git)"
 742     else:
 743         backports_version = git.describe(tree=source_dir, extra_args=['--long'])
 744         kernel_version = git.describe(rev=args.git_revision or 'HEAD',
 745                                       tree=args.kerneldir,
 746                                       extra_args=['--long'])
 747     f = open(os.path.join(args.outdir, 'versions'), 'w')
 748     f.write('BACKPORTS_VERSION="%s"\n' % backports_version)
 749     f.write('BACKPORTED_KERNEL_VERSION="%s"\n' % kernel_version)
 750     f.write('BACKPORTED_KERNEL_NAME="%s"\n' % args.base_name)
 751     if git_tracked_version:
 752         f.write('BACKPORTS_GIT_TRACKED="backport tracker ID: $(shell git rev-parse HEAD 2>/dev/null || echo \'not built in git tree\')"\n')
 753     f.close()
 754
 755     symbols = configtree.symbols()
 756
 757     # write local symbol list -- needed during build
 758     f = open(os.path.join(args.outdir, '.local-symbols'), 'w')
 759     for sym in symbols:
 760         f.write('%s=\n' % sym)
 761     f.close()
 762
 763     git_debug_snapshot(args, "add versions/symbols files")
 764
 765     logwrite('Rewrite Makefiles and Kconfig files ...')
 766
 767     # rewrite Makefile and source symbols
 768     regexes = []
 769     for some_symbols in [symbols[i:i + 50] for i in range(0, len(symbols), 50)]:
 770         r = 'CONFIG_((' + '|'.join([s + '(_MODULE)?' for s in some_symbols]) + ')([^A-Za-z0-9_]|$))'
 771         regexes.append(re.compile(r, re.MULTILINE))
 772     for root, dirs, files in os.walk(args.outdir):
 773         # don't go into .git dir (possible debug thing)
 774         if '.git' in dirs:
 775             dirs.remove('.git')
 776         for f in files:
 777             data = open(os.path.join(root, f), 'r').read()
 778             for r in regexes:
 779                 data = r.sub(r'CPTCFG_\1', data)
 780             data = re.sub(r'\$\(srctree\)', '$(backport_srctree)', data)
 781             data = re.sub(r'-Idrivers', '-I$(backport_srctree)/drivers', data)
 782             fo = open(os.path.join(root, f), 'w')
 783             fo.write(data)
 784             fo.close()
 785
 786     git_debug_snapshot(args, "rename config symbol / srctree usage")
 787
 788     # disable unbuildable Kconfig symbols and stuff Makefiles that doesn't exist
 789     maketree = make.MakeTree(os.path.join(args.outdir, 'Makefile.kernel'))
 790     disable_kconfig = []
 791     disable_makefile = []
 792     for sym in maketree.get_impossible_symbols():
 793         disable_kconfig.append(sym[7:])
 794         disable_makefile.append(sym[7:])
 795
 796     configtree.disable_symbols(disable_kconfig)
 797     git_debug_snapshot(args, "disable impossible kconfig symbols")
 798
 799     # add kernel version dependencies to Kconfig, from the dependency list
 800     # we read previously
 801     for sym in tuple(deplist.keys()):
 802         new = []
 803         for dep in deplist[sym]:
 804             if "kconfig:" in dep:
 805                     kconfig_expr = dep.replace('kconfig: ', '')
 806                     new.append(kconfig_expr)
 807             elif (dep == "DISABLE"):
 808                     new.append('BACKPORT_DISABLED_KCONFIG_OPTION')
 809             else:
 810                     new.append('!BACKPORT_KERNEL_%s' % dep.replace('.', '_'))
 811         deplist[sym] = new
 812     configtree.add_dependencies(deplist)
 813     git_debug_snapshot(args, "add kernel version dependencies")
 814
 815     # disable things in makefiles that can't be selected and that the
 816     # build shouldn't recurse into because they don't exist -- if we
 817     # don't do that then a symbol from the kernel could cause the build
 818     # to attempt to recurse and fail
 819     #
 820     # Note that we split the regex after 50 symbols, this is because of a
 821     # limitation in the regex implementation (it only supports 100 nested
 822     # groups -- 50 seemed safer and is still fast)
 823     regexes = []
 824     for some_symbols in [disable_makefile[i:i + 50] for i in range(0, len(disable_makefile), 50)]:
 825         r = '^([^#].*((CPTCFG|CONFIG)_(' + '|'.join([s for s in some_symbols]) + ')))'
 826         regexes.append(re.compile(r, re.MULTILINE))
 827     for f in maketree.get_makefiles():
 828         data = open(f, 'r').read()
 829         for r in regexes:
 830             data = r.sub(r'#\1', data)
 831         fo = open(f, 'w')
 832         fo.write(data)
 833         fo.close()
 834     git_debug_snapshot(args, "disable unsatisfied Makefile parts")
 835
 836     if (args.kup or args.kup_test):
 837         upload_release(args, rel_prep, logwrite=logwrite)
 838
 839     logwrite('Done!')
 840     return 0
 841
 842 if __name__ == '__main__':
 843     ret = _main()
 844     if ret:
 845         sys.exit(ret)