gentree.py

   1 #!/usr/bin/env python
   2 #
   3 # Generate the output tree into a specified directory.
   4 #
   5
   6 import argparse, sys, os, errno, shutil, re, subprocess
   7 import tarfile, gzip
   8
   9 # find self
  10 source_dir = os.path.abspath(os.path.dirname(__file__))
  11 sys.path.append(source_dir)
  12 # and import libraries we have
  13 from lib import kconfig, patch, make
  14 from lib import bpgit as git
  15 from lib import bpgpg as gpg
  16 from lib import bpkup as kup
  17 from lib.tempdir import tempdir
  18 from lib import bpreqs as reqs
  19
  20 def read_copy_list(copyfile):
  21     """
  22     Read a copy-list file and return a list of (source, target)
  23     tuples. The source and target are usually the same, but in
  24     the copy-list file there may be a rename included.
  25     """
  26     ret = []
  27     for item in copyfile:
  28         # remove leading/trailing whitespace
  29         item = item.strip()
  30         # comments
  31         if not item or item[0] == '#':
  32             continue
  33         if item[0] == '/':
  34             raise Exception("Input path '%s' is absolute path, this isn't allowed" % (item, ))
  35         if ' -> ' in item:
  36             srcitem, dstitem = item.split(' -> ')
  37             if (srcitem[-1] == '/') != (dstitem[-1] == '/'):
  38                 raise Exception("Cannot copy file/dir to dir/file")
  39         else:
  40             srcitem = dstitem = item
  41         ret.append((srcitem, dstitem))
  42     return ret
  43
  44
  45 def read_dependencies(depfilename):
  46     """
  47     Read a (the) dependency file and return the list of
  48     dependencies as a dictionary, mapping a Kconfig symbol
  49     to a list of kernel version dependencies.
  50
  51     If a backported feature that an upstream backported driver
  52     depends on had kconfig limitations (ie, debugging feature not
  53     available) a built constaint restriction can be expressed
  54     by using a kconfig expression. The kconfig expressions can
  55     be specified by using the "kconfig: " prefix.
  56
  57     While reading ignore blank or commented lines.
  58     """
  59     ret = {}
  60     depfile = open(depfilename, 'r')
  61     for item in depfile:
  62         kconfig_exp = ""
  63         item = item.strip()
  64         if not item or item[0] == '#':
  65             continue
  66         if "kconfig:" in item:
  67             sym, kconfig_exp = item.split(" ", 1)
  68             if not sym in ret:
  69                 ret[sym] = [kconfig_exp, ]
  70             else:
  71                 ret[sym].append(kconfig_exp)
  72         else:
  73             sym, dep = item.split()
  74             if not sym in ret:
  75                 ret[sym] = [dep, ]
  76             else:
  77                 ret[sym].append(dep)
  78     return ret
  79
  80
  81 def check_output_dir(d, clean):
  82     """
  83     Check that the output directory doesn't exist or is empty,
  84     unless clean is True in which case it's nuked. This helps
  85     sanity check the output when generating a tree, so usually
  86     running with --clean isn't suggested.
  87     """
  88     if clean:
  89         shutil.rmtree(d, ignore_errors=True)
  90     try:
  91         os.rmdir(d)
  92     except OSError as e:
  93         if e.errno != errno.ENOENT:
  94             raise
  95
  96
  97 def copytree(src, dst, symlinks=False, ignore=None):
  98     """
  99     Copy a directory tree. This differs from shutil.copytree()
 100     in that it allows destination directories to already exist.
 101     """
 102     names = os.listdir(src)
 103     if ignore is not None:
 104         ignored_names = ignore(src, names)
 105     else:
 106         ignored_names = set()
 107
 108     if not os.path.isdir(dst):
 109         os.makedirs(dst)
 110     errors = []
 111     for name in names:
 112         if name in ignored_names:
 113             continue
 114         srcname = os.path.join(src, name)
 115         dstname = os.path.join(dst, name)
 116         try:
 117             if symlinks and os.path.islink(srcname):
 118                 linkto = os.readlink(srcname)
 119                 os.symlink(linkto, dstname)
 120             elif os.path.isdir(srcname):
 121                 copytree(srcname, dstname, symlinks, ignore)
 122             else:
 123                 shutil.copy2(srcname, dstname)
 124         except (IOError, os.error) as why:
 125             errors.append((srcname, dstname, str(why)))
 126         # catch the Error from the recursive copytree so that we can
 127         # continue with other files
 128         except shutil.Error as err:
 129             errors.extend(err.args[0])
 130     try:
 131         shutil.copystat(src, dst)
 132     except WindowsError:
 133         # can't copy file access times on Windows
 134         pass
 135     except OSError as why:
 136         errors.extend((src, dst, str(why)))
 137     if errors:
 138         raise shutil.Error(errors)
 139
 140
 141 def copy_files(srcpath, copy_list, outdir):
 142     """
 143     Copy the copy_list files and directories from the srcpath
 144     to the outdir. The copy_list contains source and target
 145     names.
 146
 147     For now, it also ignores any *~ editor backup files, though
 148     this should probably be generalized (maybe using .gitignore?)
 149     Similarly the code that only copies some files (*.c, *.h,
 150     *.awk, Kconfig, Makefile) to avoid any build remnants in the
 151     kernel if they should exist.
 152     """
 153     for srcitem, tgtitem in copy_list:
 154         if tgtitem == '':
 155             copytree(srcpath, outdir, ignore=shutil.ignore_patterns('*~'))
 156         elif tgtitem[-1] == '/':
 157             def copy_ignore(dir, entries):
 158                 r = []
 159                 for i in entries:
 160                     if i[-2:] == '.o' or i[-1] == '~':
 161                         r.append(i)
 162                 return r
 163             copytree(os.path.join(srcpath, srcitem),
 164                      os.path.join(outdir, tgtitem),
 165                      ignore=copy_ignore)
 166         else:
 167             try:
 168                 os.makedirs(os.path.join(outdir, os.path.dirname(tgtitem)))
 169             except OSError as e:
 170                 # ignore dirs we might have created just now
 171                 if e.errno != errno.EEXIST:
 172                     raise
 173             shutil.copy(os.path.join(srcpath, srcitem),
 174                         os.path.join(outdir, tgtitem))
 175
 176
 177 def copy_git_files(srcpath, copy_list, rev, outdir):
 178     """
 179     "Copy" files from a git repository. This really means listing them with
 180     ls-tree and then using git show to obtain all the blobs.
 181     """
 182     for srcitem, tgtitem in copy_list:
 183         for m, t, h, f in git.ls_tree(rev=rev, files=(srcitem,), tree=srcpath):
 184             assert t == 'blob'
 185             f = os.path.join(outdir, f.replace(srcitem, tgtitem))
 186             d = os.path.dirname(f)
 187             if not os.path.exists(d):
 188                 os.makedirs(d)
 189             outf = open(f, 'w')
 190             git.get_blob(h, outf, tree=srcpath)
 191             outf.close()
 192             os.chmod(f, int(m, 8))
 193
 194 def automatic_backport_mangle_c_file(name):
 195     return name.replace('/', '-')
 196
 197
 198 def add_automatic_backports(args):
 199     disable_list = []
 200     export = re.compile(r'^EXPORT_SYMBOL(_GPL)?\((?P<sym>[^\)]*)\)')
 201     bpi = kconfig.get_backport_info(os.path.join(args.outdir, 'compat', 'Kconfig'))
 202     configtree = kconfig.ConfigTree(os.path.join(args.outdir, 'Kconfig'))
 203     all_selects = configtree.all_selects()
 204     for sym, vals in bpi.items():
 205         if sym.startswith('BACKPORT_BUILD_'):
 206             if not sym[15:] in all_selects:
 207                 disable_list.append(sym)
 208                 continue
 209         symtype, module_name, c_files, h_files = vals
 210
 211         # first copy files
 212         files = []
 213         for f in c_files:
 214             files.append((f, os.path.join('compat', automatic_backport_mangle_c_file(f))))
 215         for f in h_files:
 216             files.append((os.path.join('include', f),
 217                           os.path.join('include', os.path.dirname(f), 'backport-' + os.path.basename(f))))
 218         if args.git_revision:
 219             copy_git_files(args.kerneldir, files, args.git_revision, args.outdir)
 220         else:
 221             copy_files(args.kerneldir, files, args.outdir)
 222
 223         # now add the Makefile line
 224         mf = open(os.path.join(args.outdir, 'compat', 'Makefile'), 'a+')
 225         o_files = [automatic_backport_mangle_c_file(f)[:-1] + 'o' for f in c_files]
 226         if symtype == 'tristate':
 227             if not module_name:
 228                 raise Exception('backporting a module requires a #module-name')
 229             for of in o_files:
 230                 mf.write('%s-objs += %s\n' % (module_name, of))
 231             mf.write('obj-$(CPTCFG_%s) += %s.o\n' % (sym, module_name))
 232         elif symtype == 'bool':
 233             mf.write('compat-$(CPTCFG_%s) += %s\n' % (sym, ' '.join(o_files)))
 234
 235         # finally create the include file
 236         syms = []
 237         for f in c_files:
 238             for l in open(os.path.join(args.outdir, 'compat',
 239                                        automatic_backport_mangle_c_file(f)), 'r'):
 240                 m = export.match(l)
 241                 if m:
 242                     syms.append(m.group('sym'))
 243         for f in h_files:
 244             outf = open(os.path.join(args.outdir, 'include', f), 'w')
 245             outf.write('/* Automatically created during backport process */\n')
 246             outf.write('#ifndef CPTCFG_%s\n' % sym)
 247             outf.write('#include_next <%s>\n' % f)
 248             outf.write('#else\n');
 249             for s in syms:
 250                 outf.write('#undef %s\n' % s)
 251                 outf.write('#define %s LINUX_BACKPORT(%s)\n' % (s, s))
 252             outf.write('#include <%s>\n' % (os.path.dirname(f) + '/backport-' + os.path.basename(f), ))
 253             outf.write('#endif /* CPTCFG_%s */\n' % sym)
 254     return disable_list
 255
 256 def git_debug_init(args):
 257     """
 258     Initialize a git repository in the output directory and commit the current
 259     code in it. This is only used for debugging the transformations this code
 260     will do to the output later.
 261     """
 262     if not args.gitdebug:
 263         return
 264     git.init(tree=args.outdir)
 265     git.commit_all("Copied backport", tree=args.outdir)
 266
 267
 268 def git_debug_snapshot(args, name):
 269     """
 270     Take a git snapshot for the debugging.
 271     """
 272     if not args.gitdebug:
 273         return
 274     git.commit_all(name, tree=args.outdir)
 275
 276 def get_rel_spec_stable(rel):
 277     """
 278     Returns release specs for a linux-stable backports based release.
 279     """
 280     if ("rc" in rel):
 281         m = re.match(r"(?P<VERSION>\d+)\.+" \
 282                      "(?P<PATCHLEVEL>\d+)[.]*" \
 283                      "(?P<SUBLEVEL>\d*)" \
 284                      "[-rc]+(?P<RC_VERSION>\d+)\-+" \
 285                      "(?P<RELMOD_UPDATE>\d+)[-]*" \
 286                      "(?P<RELMOD_TYPE>[usnpc]*)", \
 287                      rel)
 288     else:
 289         m = re.match(r"(?P<VERSION>\d+)\.+" \
 290                      "(?P<PATCHLEVEL>\d+)[.]*" \
 291                      "(?P<SUBLEVEL>\d*)\-+" \
 292                      "(?P<RELMOD_UPDATE>\d+)[-]*" \
 293                      "(?P<RELMOD_TYPE>[usnpc]*)", \
 294                      rel)
 295     if (not m):
 296         return m
 297     return m.groupdict()
 298
 299 def get_rel_spec_next(rel):
 300     """
 301     Returns release specs for a linux-next backports based release.
 302     """
 303     m = re.match(r"(?P<DATE_VERSION>\d+)[-]*" \
 304                  "(?P<RELMOD_UPDATE>\d*)[-]*" \
 305                  "(?P<RELMOD_TYPE>[usnpc]*)", \
 306                  rel)
 307     if (not m):
 308         return m
 309     return m.groupdict()
 310
 311 def get_rel_prep(rel):
 312     """
 313     Returns a dict with prep work details we need prior to
 314     uploading a backports release to kernel.org
 315     """
 316     rel_specs = get_rel_spec_stable(rel)
 317     is_stable = True
 318     rel_tag = ""
 319     paths = list()
 320     if (not rel_specs):
 321         rel_specs = get_rel_spec_next(rel)
 322         if (not rel_specs):
 323             sys.stdout.write("rel: %s\n" % rel)
 324             return None
 325         if (rel_specs['RELMOD_UPDATE'] == '0' or
 326             rel_specs['RELMOD_UPDATE'] == '1'):
 327             return None
 328         is_stable = False
 329         date = rel_specs['DATE_VERSION']
 330         year = date[0:4]
 331         if (len(year) != 4):
 332             return None
 333         month = date[4:6]
 334         if (len(month) != 2):
 335             return None
 336         day = date[6:8]
 337         if (len(day) != 2):
 338             return None
 339         paths.append(year)
 340         paths.append(month)
 341         paths.append(day)
 342         rel_tag = "backports-" + rel.replace(rel_specs['RELMOD_TYPE'], "")
 343     else:
 344         ignore = "-"
 345         if (not rel_specs['RELMOD_UPDATE']):
 346             return None
 347         if (rel_specs['RELMOD_UPDATE'] == '0'):
 348             return None
 349         ignore += rel_specs['RELMOD_UPDATE']
 350         if (rel_specs['RELMOD_TYPE'] != ''):
 351             ignore += rel_specs['RELMOD_TYPE']
 352         base_rel = rel.replace(ignore, "")
 353         paths.append("v" + base_rel)
 354         rel_tag = "v" + rel.replace(rel_specs['RELMOD_TYPE'], "")
 355
 356     rel_prep = dict(stable = is_stable,
 357                     expected_tag = rel_tag,
 358                     paths_to_create = paths)
 359     return rel_prep
 360
 361 def create_tar_and_gz(tar_name, dir_to_tar):
 362     """
 363     We need both a tar file and gzip for kernel.org, the tar file
 364     gets signed, then we upload the compressed version, kup-server
 365     in the backend decompresses and verifies the tarball against
 366     our signature.
 367     """
 368     basename = os.path.basename(dir_to_tar)
 369     tar = tarfile.open(tar_name, "w")
 370     tar.add(dir_to_tar, basename)
 371     tar.close()
 372
 373     tar_file = open(tar_name, "r")
 374
 375     gz_file = gzip.GzipFile(tar_name + ".gz", 'wb')
 376     gz_file.write(tar_file.read())
 377     gz_file.close()
 378
 379 def upload_release(args, rel_prep, logwrite=lambda x:None):
 380     """
 381     Given a path of a relase make tarball out of it, PGP sign it, and
 382     then upload it to kernel.org using kup.
 383
 384     The linux-next based release do not require a RELMOD_UPDATE
 385     given that typically only one release is made per day. Using
 386     RELMOD_UPDATE for these releases is allowed though and if
 387     present it must be > 1.
 388
 389     The linux-stable based releases require a RELMOD_UPDATE.
 390
 391     RELMOD_UPDATE must be numeric and > 0 just as the RC releases
 392     of the Linux kernel.
 393
 394     The tree must also be tagged with the respective release, without
 395     the RELMOD_TYPE. For linux-next based releases this consists of
 396     backports- followed by DATE_VERSION and if RELMOD_TYPE is present.
 397     For linux-stable releases this consists of v followed by the
 398     full release version except the RELMOD_TYPE.
 399
 400     Uploads will not be allowed if these rules are not followed.
 401     """
 402     korg_path = "/pub/linux/kernel/projects/backports"
 403
 404     if (rel_prep['stable']):
 405         korg_path += "/stable"
 406
 407     parent = os.path.dirname(args.outdir)
 408     release = os.path.basename(args.outdir)
 409     tar_name = parent + '/' + release + ".tar"
 410     gzip_name = tar_name + ".gz"
 411
 412     create_tar_and_gz(tar_name, args.outdir)
 413
 414     logwrite(gpg.sign(tar_name, extra_args=['--armor', '--detach-sign']))
 415
 416     logwrite("------------------------------------------------------")
 417
 418     if (not args.kup_test):
 419         logwrite("About to upload, current target path contents:")
 420     else:
 421         logwrite("kup-test: current target path contents:")
 422
 423     logwrite(kup.ls(path=korg_path))
 424
 425     for path in rel_prep['paths_to_create']:
 426         korg_path += '/' + path
 427         if (not args.kup_test):
 428             logwrite("create directory: %s" % korg_path)
 429             logwrite(kup.mkdir(korg_path))
 430     korg_path += '/'
 431     if (not args.kup_test):
 432         logwrite("upload file %s to %s" % (gzip_name, korg_path))
 433         logwrite(kup.put(gzip_name, tar_name + '.asc', korg_path))
 434         logwrite("\nFinished upload!\n")
 435         logwrite("Target path contents:")
 436         logwrite(kup.ls(path=korg_path))
 437     else:
 438         kup_cmd = "kup put /\n\t\t%s /\n\t\t%s /\n\t\t%s" % (gzip_name, tar_name + '.asc', korg_path)
 439         logwrite("kup-test: skipping cmd: %s" % kup_cmd)
 440
 441 def _main():
 442     # Our binary requirements go here
 443     req = reqs.Req()
 444     req.require('git')
 445     req.coccinelle('1.0.0-rc21')
 446     if not req.reqs_match():
 447         sys.exit(1)
 448
 449     # set up and parse arguments
 450     parser = argparse.ArgumentParser(description='generate backport tree')
 451     parser.add_argument('kerneldir', metavar='<kernel tree>', type=str,
 452                         help='Kernel tree to copy drivers from')
 453     parser.add_argument('outdir', metavar='<output directory>', type=str,
 454                         help='Directory to write the generated tree to')
 455     parser.add_argument('--copy-list', metavar='<listfile>', type=argparse.FileType('r'),
 456                         default='copy-list',
 457                         help='File containing list of files/directories to copy, default "copy-list"')
 458     parser.add_argument('--git-revision', metavar='<revision>', type=str,
 459                         help='git commit revision (see gitrevisions(7)) to take objects from.' +
 460                              'If this is specified, the kernel tree is used as git object storage ' +
 461                              'and we use git ls-tree to get the files.')
 462     parser.add_argument('--clean', const=True, default=False, action="store_const",
 463                         help='Clean output directory instead of erroring if it isn\'t empty')
 464     parser.add_argument('--refresh', const=True, default=False, action="store_const",
 465                         help='Refresh patches as they are applied, the source dir will be modified!')
 466     parser.add_argument('--base-name', metavar='<name>', type=str, default='Linux',
 467                         help='name of base tree, default just "Linux"')
 468     parser.add_argument('--gitdebug', const=True, default=False, action="store_const",
 469                         help='Use git, in the output tree, to debug the various transformation steps ' +
 470                              'that the tree generation makes (apply patches, ...)')
 471     parser.add_argument('--verbose', const=True, default=False, action="store_const",
 472                         help='Print more verbose information')
 473     parser.add_argument('--extra-driver', nargs=2, metavar=('<source dir>', '<copy-list>'), type=str,
 474                         action='append', default=[], help='Extra driver directory/copy-list.')
 475     parser.add_argument('--kup', const=True, default=False, action="store_const",
 476                         help='For maintainers: upload a release to kernel.org')
 477     parser.add_argument('--kup-test', const=True, default=False, action="store_const",
 478                         help='For maintainers: do all the work as if you were about to ' +
 479                              'upload to kernel.org but do not do the final `kup put` ' +
 480                              'and also do not run any `kup mkdir` commands. This will ' +
 481                              'however run `kup ls` on the target paths so ' +
 482                              'at the very least we test your kup configuration. ' +
 483                              'If this is your first time uploading use this first!')
 484     parser.add_argument('--test-cocci', metavar='<sp_file>', type=str, default=None,
 485                         help='Only use the cocci file passed for Coccinelle, don\'t do anything else, ' +
 486                              'also creates a git repo on the target directory for easy inspection ' +
 487                              'of changes done by Coccinelle.')
 488     parser.add_argument('--profile-cocci', metavar='<sp_file>', type=str, default=None,
 489                         help='Only use the cocci file passed and pass --profile  to Coccinelle, ' +
 490                              'also creates a git repo on the target directory for easy inspection ' +
 491                              'of changes done by Coccinelle.')
 492     args = parser.parse_args()
 493
 494     def logwrite(msg):
 495         sys.stdout.write(msg)
 496         sys.stdout.write('\n')
 497         sys.stdout.flush()
 498
 499     return process(args.kerneldir, args.outdir, args.copy_list,
 500                    git_revision=args.git_revision, clean=args.clean,
 501                    refresh=args.refresh, base_name=args.base_name,
 502                    gitdebug=args.gitdebug, verbose=args.verbose,
 503                    extra_driver=args.extra_driver,
 504                    kup=args.kup,
 505                    kup_test=args.kup_test,
 506                    test_cocci=args.test_cocci,
 507                    profile_cocci=args.profile_cocci,
 508                    logwrite=logwrite)
 509
 510 def process(kerneldir, outdir, copy_list_file, git_revision=None,
 511             clean=False, refresh=False, base_name="Linux", gitdebug=False,
 512             verbose=False, extra_driver=[], kup=False,
 513             kup_test=False,
 514             test_cocci=None,
 515             profile_cocci=None,
 516             logwrite=lambda x:None,
 517             git_tracked_version=False):
 518     class Args(object):
 519         def __init__(self, kerneldir, outdir, copy_list_file,
 520                      git_revision, clean, refresh, base_name,
 521                      gitdebug, verbose, extra_driver, kup,
 522                      kup_test,
 523                      test_cocci,
 524                      profile_cocci):
 525             self.kerneldir = kerneldir
 526             self.outdir = outdir
 527             self.copy_list = copy_list_file
 528             self.git_revision = git_revision
 529             self.clean = clean
 530             self.refresh = refresh
 531             self.base_name = base_name
 532             self.gitdebug = gitdebug
 533             self.verbose = verbose
 534             self.extra_driver = extra_driver
 535             self.kup = kup
 536             self.kup_test = kup_test
 537             self.test_cocci = test_cocci
 538             self.profile_cocci = profile_cocci
 539             if self.test_cocci or self.profile_cocci:
 540                 self.gitdebug = True
 541     def git_paranoia(tree=None, logwrite=lambda x:None):
 542         data = git.paranoia(tree)
 543         if (data['r'] != 0):
 544             logwrite('Cannot use %s' % tree)
 545             logwrite('%s' % data['output'])
 546             sys.exit(data['r'])
 547         else:
 548             logwrite('Validated tree: %s' % tree)
 549
 550     args = Args(kerneldir, outdir, copy_list_file,
 551                 git_revision, clean, refresh, base_name,
 552                 gitdebug, verbose, extra_driver, kup, kup_test,
 553                 test_cocci, profile_cocci)
 554     rel_prep = None
 555
 556     # start processing ...
 557     if (args.kup or args.kup_test):
 558         git_paranoia(source_dir, logwrite)
 559         git_paranoia(kerneldir, logwrite)
 560
 561         rel_describe = git.describe(rev=None, tree=source_dir, extra_args=['--dirty'])
 562         release = os.path.basename(args.outdir)
 563         version = release.replace("backports-", "")
 564
 565         rel_prep = get_rel_prep(version)
 566         if (not rel_prep):
 567             logwrite('Invalid backports release name: %s' % release)
 568             logwrite('For rules on the release name see upload_release()')
 569             sys.exit(1)
 570         rel_type = "linux-stable"
 571         if (not rel_prep['stable']):
 572             rel_type = "linux-next"
 573         if (rel_prep['expected_tag'] != rel_describe):
 574             logwrite('Unexpected %s based backports release tag on' % rel_type)
 575             logwrite('the backports tree tree: %s\n' % rel_describe)
 576             logwrite('You asked to make a release with this ')
 577             logwrite('directory name: %s' % release)
 578             logwrite('The actual expected tag we should find on')
 579             logwrite('the backports tree then is: %s\n' % rel_prep['expected_tag'])
 580             logwrite('For rules on the release name see upload_release()')
 581             sys.exit(1)
 582
 583     copy_list = read_copy_list(args.copy_list)
 584     deplist = read_dependencies(os.path.join(source_dir, 'dependencies'))
 585
 586     # validate output directory
 587     check_output_dir(args.outdir, args.clean)
 588
 589     # do the copy
 590     backport_files = [(x, x) for x in [
 591         'Kconfig', 'Makefile', 'Makefile.build', 'Makefile.kernel', '.gitignore',
 592         'Makefile.real', 'compat/', 'backport-include/', 'kconf/',
 593         'scripts/', '.blacklist.map',
 594     ]]
 595     if not args.git_revision:
 596         logwrite('Copy original source files ...')
 597     else:
 598         logwrite('Get original source files from git ...')
 599
 600     copy_files(os.path.join(source_dir, 'backport'), backport_files, args.outdir)
 601
 602     git_debug_init(args)
 603
 604     if not args.git_revision:
 605         copy_files(args.kerneldir, copy_list, args.outdir)
 606     else:
 607         copy_git_files(args.kerneldir, copy_list, args.git_revision, args.outdir)
 608
 609     # FIXME: should we add a git version of this (e.g. --git-extra-driver)?
 610     for src, copy_list in args.extra_driver:
 611         if (args.kup or args.kup_test):
 612             git_paranoia(src)
 613         copy_files(src, read_copy_list(open(copy_list, 'r')), args.outdir)
 614
 615     git_debug_snapshot(args, 'Add driver sources')
 616
 617     disable_list = add_automatic_backports(args)
 618     if disable_list:
 619         bpcfg = kconfig.ConfigTree(os.path.join(args.outdir, 'compat', 'Kconfig'))
 620         bpcfg.disable_symbols(disable_list)
 621     git_debug_snapshot(args, 'Add automatic backports')
 622
 623     test_cocci = args.test_cocci or args.profile_cocci
 624
 625     logwrite('Apply patches ...')
 626     patches = []
 627     sempatches = []
 628     for root, dirs, files in os.walk(os.path.join(source_dir, 'patches')):
 629         for f in files:
 630             if not test_cocci and f.endswith('.patch'):
 631                 patches.append(os.path.join(root, f))
 632             if f.endswith('.cocci'):
 633                 if test_cocci:
 634                     if f not in test_cocci:
 635                         continue
 636                     if args.test_cocci:
 637                         logwrite("Testing Coccinelle SmPL patch: %s" % test_cocci)
 638                     elif args.profile_cocci:
 639                         logwrite("Profiling Coccinelle SmPL patch: %s" % test_cocci)
 640                 sempatches.append(os.path.join(root, f))
 641     patches.sort()
 642     prefix_len = len(os.path.join(source_dir, 'patches')) + 1
 643     for pfile in patches:
 644         print_name = pfile[prefix_len:]
 645         # read the patch file
 646         p = patch.fromfile(pfile)
 647         # complain if it's not a patch
 648         if not p:
 649             raise Exception('No patch content found in %s' % print_name)
 650         # leading / seems to be stripped?
 651         if 'dev/null' in p.items[0].source:
 652             raise Exception('Patches creating files are not supported (in %s)' % print_name)
 653         # check if the first file the patch touches exists, if so
 654         # assume the patch needs to be applied -- otherwise continue
 655         patched_file = '/'.join(p.items[0].source.split('/')[1:])
 656         fullfn = os.path.join(args.outdir, patched_file)
 657         if not os.path.exists(fullfn):
 658             if args.verbose:
 659                 logwrite("Not applying %s, not needed" % print_name)
 660             continue
 661         if args.verbose:
 662             logwrite("Applying patch %s" % print_name)
 663
 664         if args.refresh:
 665             # but for refresh, of course look at all files the patch touches
 666             for patchitem in p.items:
 667                 patched_file = '/'.join(patchitem.source.split('/')[1:])
 668                 fullfn = os.path.join(args.outdir, patched_file)
 669                 shutil.copyfile(fullfn, fullfn + '.orig_file')
 670
 671         process = subprocess.Popen(['patch', '-p1'], stdout=subprocess.PIPE,
 672                                    stderr=subprocess.STDOUT, stdin=subprocess.PIPE,
 673                                    close_fds=True, universal_newlines=True,
 674                                    cwd=args.outdir)
 675         output = process.communicate(input=open(pfile, 'r').read())[0]
 676         output = output.split('\n')
 677         if output[-1] == '':
 678             output = output[:-1]
 679         if args.verbose:
 680             for line in output:
 681                 logwrite('> %s' % line)
 682         if process.returncode != 0:
 683             if not args.verbose:
 684                 logwrite("Failed to apply changes from %s" % print_name)
 685                 for line in output:
 686                     logwrite('> %s' % line)
 687             return 2
 688
 689         if args.refresh:
 690             pfilef = open(pfile + '.tmp', 'a')
 691             pfilef.write(p.top_header)
 692             pfilef.flush()
 693             for patchitem in p.items:
 694                 patched_file = '/'.join(patchitem.source.split('/')[1:])
 695                 fullfn = os.path.join(args.outdir, patched_file)
 696                 process = subprocess.Popen(['diff', '-p', '-u', patched_file + '.orig_file', patched_file,
 697                                             '--label', 'a/' + patched_file,
 698                                             '--label', 'b/' + patched_file],
 699                                            stdout=pfilef, close_fds=True,
 700                                            universal_newlines=True, cwd=args.outdir)
 701                 process.wait()
 702                 os.unlink(fullfn + '.orig_file')
 703                 if not process.returncode in (0, 1):
 704                     logwrite("Failed to diff to refresh %s" % print_name)
 705                     pfilef.close()
 706                     os.unlink(pfile + '.tmp')
 707                     return 3
 708             pfilef.close()
 709             os.rename(pfile + '.tmp', pfile)
 710
 711         # remove orig/rej files that patch sometimes creates
 712         for root, dirs, files in os.walk(args.outdir):
 713             for f in files:
 714                 if f[-5:] == '.orig' or f[-4:] == '.rej':
 715                     os.unlink(os.path.join(root, f))
 716         git_debug_snapshot(args, "apply backport patch %s" % print_name)
 717
 718     sempatches.sort()
 719     prefix_len = len(os.path.join(source_dir, 'patches')) + 1
 720
 721     for cocci_file in sempatches:
 722         # Until Coccinelle picks this up
 723         pycocci = os.path.join(source_dir, 'devel/pycocci')
 724         cmd = [pycocci, cocci_file]
 725         extra_spatch_args = []
 726         if args.profile_cocci:
 727             cmd.append('--profile-cocci')
 728         cmd.append(args.outdir)
 729         print_name = cocci_file[prefix_len:]
 730         if args.verbose:
 731             logwrite("Applying SmPL patch %s" % print_name)
 732         sprocess = subprocess.Popen(cmd,
 733                                     stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
 734                                     close_fds=True, universal_newlines=True,
 735                                     cwd=args.outdir)
 736         output = sprocess.communicate()[0]
 737         sprocess.wait()
 738         if sprocess.returncode != 0:
 739             logwrite("Failed to process SmPL patch %s" % print_name)
 740             return 3
 741         output = output.split('\n')
 742         if output[-1] == '':
 743             output = output[:-1]
 744         if args.verbose:
 745             for line in output:
 746                 logwrite('> %s' % line)
 747
 748         # remove cocci_backup files
 749         for root, dirs, files in os.walk(args.outdir):
 750             for f in files:
 751                 if f.endswith('.cocci_backup'):
 752                     os.unlink(os.path.join(root, f))
 753         git_debug_snapshot(args, "apply backport SmPL patch %s" % print_name)
 754
 755     if test_cocci:
 756         logwrite('Done!')
 757         return 0
 758
 759     # some post-processing is required
 760     configtree = kconfig.ConfigTree(os.path.join(args.outdir, 'Kconfig'))
 761     logwrite('Modify Kconfig tree ...')
 762     configtree.prune_sources(ignore=['Kconfig.kernel', 'Kconfig.versions'])
 763     git_debug_snapshot(args, "prune Kconfig tree")
 764     configtree.force_tristate_modular()
 765     git_debug_snapshot(args, "force tristate options modular")
 766     configtree.modify_selects()
 767     git_debug_snapshot(args, "convert select to depends on")
 768
 769     # write the versioning file
 770     if git_tracked_version:
 771         backports_version = "(see git)"
 772         kernel_version = "(see git)"
 773     else:
 774         backports_version = git.describe(tree=source_dir, extra_args=['--long'])
 775         kernel_version = git.describe(rev=args.git_revision or 'HEAD',
 776                                       tree=args.kerneldir,
 777                                       extra_args=['--long'])
 778     f = open(os.path.join(args.outdir, 'versions'), 'w')
 779     f.write('BACKPORTS_VERSION="%s"\n' % backports_version)
 780     f.write('BACKPORTED_KERNEL_VERSION="%s"\n' % kernel_version)
 781     f.write('BACKPORTED_KERNEL_NAME="%s"\n' % args.base_name)
 782     if git_tracked_version:
 783         f.write('BACKPORTS_GIT_TRACKED="backport tracker ID: $(shell git rev-parse HEAD 2>/dev/null || echo \'not built in git tree\')"\n')
 784     f.close()
 785
 786     symbols = configtree.symbols()
 787
 788     # write local symbol list -- needed during build
 789     f = open(os.path.join(args.outdir, '.local-symbols'), 'w')
 790     for sym in symbols:
 791         f.write('%s=\n' % sym)
 792     f.close()
 793
 794     git_debug_snapshot(args, "add versions/symbols files")
 795
 796     # add defconfigs that we want
 797     defconfigs_dir = os.path.join(source_dir, 'backport', 'defconfigs')
 798     os.mkdir(os.path.join(args.outdir, 'defconfigs'))
 799     for dfbase in os.listdir(defconfigs_dir):
 800         copy_defconfig = True
 801         dfsrc = os.path.join(defconfigs_dir, dfbase)
 802         for line in open(dfsrc, 'r'):
 803             if not '=' in line:
 804                 continue
 805             line_ok = False
 806             for sym in symbols:
 807                 if sym + '=' in line:
 808                     line_ok = True
 809                     break
 810             if not line_ok:
 811                 copy_defconfig = False
 812                 break
 813         if copy_defconfig:
 814             shutil.copy(dfsrc, os.path.join(args.outdir, 'defconfigs', dfbase))
 815
 816     git_debug_snapshot(args, "add (useful) defconfig files")
 817
 818     logwrite('Rewrite Makefiles and Kconfig files ...')
 819
 820     # rewrite Makefile and source symbols
 821     regexes = []
 822     for some_symbols in [symbols[i:i + 50] for i in range(0, len(symbols), 50)]:
 823         r = 'CONFIG_((' + '|'.join([s + '(_MODULE)?' for s in some_symbols]) + ')([^A-Za-z0-9_]|$))'
 824         regexes.append(re.compile(r, re.MULTILINE))
 825     for root, dirs, files in os.walk(args.outdir):
 826         # don't go into .git dir (possible debug thing)
 827         if '.git' in dirs:
 828             dirs.remove('.git')
 829         for f in files:
 830             data = open(os.path.join(root, f), 'r').read()
 831             for r in regexes:
 832                 data = r.sub(r'CPTCFG_\1', data)
 833             data = re.sub(r'\$\(srctree\)', '$(backport_srctree)', data)
 834             data = re.sub(r'-Idrivers', '-I$(backport_srctree)/drivers', data)
 835             fo = open(os.path.join(root, f), 'w')
 836             fo.write(data)
 837             fo.close()
 838
 839     git_debug_snapshot(args, "rename config symbol / srctree usage")
 840
 841     # disable unbuildable Kconfig symbols and stuff Makefiles that doesn't exist
 842     maketree = make.MakeTree(os.path.join(args.outdir, 'Makefile.kernel'))
 843     disable_kconfig = []
 844     disable_makefile = []
 845     for sym in maketree.get_impossible_symbols():
 846         disable_kconfig.append(sym[7:])
 847         disable_makefile.append(sym[7:])
 848
 849     configtree.disable_symbols(disable_kconfig)
 850     git_debug_snapshot(args, "disable impossible kconfig symbols")
 851
 852     # add kernel version dependencies to Kconfig, from the dependency list
 853     # we read previously
 854     for sym in tuple(deplist.keys()):
 855         new = []
 856         for dep in deplist[sym]:
 857             if "kconfig:" in dep:
 858                     kconfig_expr = dep.replace('kconfig: ', '')
 859                     new.append(kconfig_expr)
 860             elif (dep == "DISABLE"):
 861                     new.append('BACKPORT_DISABLED_KCONFIG_OPTION')
 862             else:
 863                     new.append('!BACKPORT_KERNEL_%s' % dep.replace('.', '_'))
 864         deplist[sym] = new
 865     configtree.add_dependencies(deplist)
 866     git_debug_snapshot(args, "add kernel version dependencies")
 867
 868     # disable things in makefiles that can't be selected and that the
 869     # build shouldn't recurse into because they don't exist -- if we
 870     # don't do that then a symbol from the kernel could cause the build
 871     # to attempt to recurse and fail
 872     #
 873     # Note that we split the regex after 50 symbols, this is because of a
 874     # limitation in the regex implementation (it only supports 100 nested
 875     # groups -- 50 seemed safer and is still fast)
 876     regexes = []
 877     for some_symbols in [disable_makefile[i:i + 50] for i in range(0, len(disable_makefile), 50)]:
 878         r = '^([^#].*((CPTCFG|CONFIG)_(' + '|'.join([s for s in some_symbols]) + ')))'
 879         regexes.append(re.compile(r, re.MULTILINE))
 880     for f in maketree.get_makefiles():
 881         data = open(f, 'r').read()
 882         for r in regexes:
 883             data = r.sub(r'#\1', data)
 884         fo = open(f, 'w')
 885         fo.write(data)
 886         fo.close()
 887     git_debug_snapshot(args, "disable unsatisfied Makefile parts")
 888
 889     if (args.kup or args.kup_test):
 890         req = reqs.Req()
 891         req.kup()
 892         if not req.reqs_match():
 893             sys.exit(1)
 894         upload_release(args, rel_prep, logwrite=logwrite)
 895
 896     logwrite('Done!')
 897     return 0
 898
 899 if __name__ == '__main__':
 900     ret = _main()
 901     if ret:
 902         sys.exit(ret)