gentree.py

   1 #!/usr/bin/env python
   2 #
   3 # Generate the output tree into a specified directory.
   4 #
   5
   6 import argparse, sys, os, errno, shutil, re, subprocess
   7 import tarfile, gzip
   8 from multiprocessing import cpu_count
   9
  10 # find self
  11 source_dir = os.path.abspath(os.path.dirname(__file__))
  12 sys.path.append(source_dir)
  13 # and import libraries we have
  14 from lib import kconfig, patch, make
  15 from lib import bpgit as git
  16 from lib import bpgpg as gpg
  17 from lib import bpkup as kup
  18 from lib.tempdir import tempdir
  19 from lib import bpreqs as reqs
  20 from lib import bpversion as gen_version
  21
  22 class Bp_Identity(object):
  23     """
  24     folks considering multiple integrations may want to
  25     consider stuffing versioning info here as well but
  26     that will need thought/design on sharing compat and
  27     module namespaces.
  28
  29     Use the *_resafe when combining on regexps, although we currently
  30     don't support regexps there perhaps later we will and this will
  31     just make things safer for the output regardless. Once and if those
  32     are added, how we actually use the others for regular printing will
  33     need to be considered.
  34     """
  35     def __init__(self, integrate=False, kconfig_prefix='CPTCFG_',
  36                  project_prefix='', project_dir='',
  37                  target_dir='', target_dir_name='',
  38                  kconfig_source_var=None):
  39         self.integrate = integrate
  40         self.kconfig_prefix = kconfig_prefix
  41         self.kconfig_prefix_resafe = re.escape(kconfig_prefix)
  42         self.project_prefix = project_prefix
  43         self.project_prefix_resafe = re.escape(project_prefix)
  44         self.full_prefix = kconfig_prefix + project_prefix
  45         self.full_prefix_resafe = re.escape(self.full_prefix)
  46         self.project_dir = project_dir
  47         self.target_dir = target_dir
  48         self.target_dir_name = target_dir_name
  49         self.kconfig_source_var = kconfig_source_var
  50         if self.kconfig_source_var:
  51             self.kconfig_source_var_resafe = re.escape(self.kconfig_source_var)
  52         else:
  53             self.kconfig_source_var_resafe = None
  54
  55 def read_copy_list(copyfile):
  56     """
  57     Read a copy-list file and return a list of (source, target)
  58     tuples. The source and target are usually the same, but in
  59     the copy-list file there may be a rename included.
  60     """
  61     ret = []
  62     for item in copyfile:
  63         # remove leading/trailing whitespace
  64         item = item.strip()
  65         # comments
  66         if not item or item[0] == '#':
  67             continue
  68         if item[0] == '/':
  69             raise Exception("Input path '%s' is absolute path, this isn't allowed" % (item, ))
  70         if ' -> ' in item:
  71             srcitem, dstitem = item.split(' -> ')
  72             if (srcitem[-1] == '/') != (dstitem[-1] == '/'):
  73                 raise Exception("Cannot copy file/dir to dir/file")
  74         else:
  75             srcitem = dstitem = item
  76         ret.append((srcitem, dstitem))
  77     return ret
  78
  79
  80 def read_dependencies(depfilename):
  81     """
  82     Read a (the) dependency file and return the list of
  83     dependencies as a dictionary, mapping a Kconfig symbol
  84     to a list of kernel version dependencies.
  85
  86     If a backported feature that an upstream backported driver
  87     depends on had kconfig limitations (ie, debugging feature not
  88     available) a built constaint restriction can be expressed
  89     by using a kconfig expression. The kconfig expressions can
  90     be specified by using the "kconfig: " prefix.
  91
  92     While reading ignore blank or commented lines.
  93     """
  94     ret = {}
  95     depfile = open(depfilename, 'r')
  96     for item in depfile:
  97         kconfig_exp = ""
  98         item = item.strip()
  99         if not item or item[0] == '#':
 100             continue
 101         if "kconfig:" in item:
 102             sym, kconfig_exp = item.split(" ", 1)
 103             if not sym in ret:
 104                 ret[sym] = [kconfig_exp, ]
 105             else:
 106                 ret[sym].append(kconfig_exp)
 107         else:
 108             sym, dep = item.split()
 109             if not sym in ret:
 110                 ret[sym] = [dep, ]
 111             else:
 112                 ret[sym].append(dep)
 113     return ret
 114
 115
 116 def check_output_dir(d, clean):
 117     """
 118     Check that the output directory doesn't exist or is empty,
 119     unless clean is True in which case it's nuked. This helps
 120     sanity check the output when generating a tree, so usually
 121     running with --clean isn't suggested.
 122     """
 123     if clean:
 124         shutil.rmtree(d, ignore_errors=True)
 125     try:
 126         os.rmdir(d)
 127     except OSError as e:
 128         if e.errno != errno.ENOENT:
 129             raise
 130
 131
 132 def copytree(src, dst, symlinks=False, ignore=None):
 133     """
 134     Copy a directory tree. This differs from shutil.copytree()
 135     in that it allows destination directories to already exist.
 136     """
 137     names = os.listdir(src)
 138     if ignore is not None:
 139         ignored_names = ignore(src, names)
 140     else:
 141         ignored_names = set()
 142
 143     if not os.path.isdir(dst):
 144         os.makedirs(dst)
 145     errors = []
 146     for name in names:
 147         if name in ignored_names:
 148             continue
 149         srcname = os.path.join(src, name)
 150         dstname = os.path.join(dst, name)
 151         try:
 152             if symlinks and os.path.islink(srcname):
 153                 linkto = os.readlink(srcname)
 154                 os.symlink(linkto, dstname)
 155             elif os.path.isdir(srcname):
 156                 copytree(srcname, dstname, symlinks, ignore)
 157             else:
 158                 shutil.copy2(srcname, dstname)
 159         except (IOError, os.error) as why:
 160             errors.append((srcname, dstname, str(why)))
 161         # catch the Error from the recursive copytree so that we can
 162         # continue with other files
 163         except shutil.Error as err:
 164             errors.extend(err.args[0])
 165     try:
 166         shutil.copystat(src, dst)
 167     except WindowsError:
 168         # can't copy file access times on Windows
 169         pass
 170     except OSError as why:
 171         errors.extend((src, dst, str(why)))
 172     if errors:
 173         raise shutil.Error(errors)
 174
 175
 176 def copy_files(srcpath, copy_list, outdir):
 177     """
 178     Copy the copy_list files and directories from the srcpath
 179     to the outdir. The copy_list contains source and target
 180     names.
 181
 182     For now, it also ignores any *~ editor backup files, though
 183     this should probably be generalized (maybe using .gitignore?)
 184     Similarly the code that only copies some files (*.c, *.h,
 185     *.awk, Kconfig, Makefile) to avoid any build remnants in the
 186     kernel if they should exist.
 187     """
 188     for srcitem, tgtitem in copy_list:
 189         if tgtitem == '':
 190             copytree(srcpath, outdir, ignore=shutil.ignore_patterns('*~'))
 191         elif tgtitem[-1] == '/':
 192             def copy_ignore(dir, entries):
 193                 r = []
 194                 for i in entries:
 195                     if i[-2:] == '.o' or i[-1] == '~':
 196                         r.append(i)
 197                 return r
 198             copytree(os.path.join(srcpath, srcitem),
 199                      os.path.join(outdir, tgtitem),
 200                      ignore=copy_ignore)
 201         else:
 202             try:
 203                 os.makedirs(os.path.join(outdir, os.path.dirname(tgtitem)))
 204             except OSError as e:
 205                 # ignore dirs we might have created just now
 206                 if e.errno != errno.EEXIST:
 207                     raise
 208             shutil.copy(os.path.join(srcpath, srcitem),
 209                         os.path.join(outdir, tgtitem))
 210
 211
 212 def copy_git_files(srcpath, copy_list, rev, outdir):
 213     """
 214     "Copy" files from a git repository. This really means listing them with
 215     ls-tree and then using git show to obtain all the blobs.
 216     """
 217     for srcitem, tgtitem in copy_list:
 218         for m, t, h, f in git.ls_tree(rev=rev, files=(srcitem,), tree=srcpath):
 219             assert t == 'blob'
 220             f = os.path.join(outdir, f.replace(srcitem, tgtitem))
 221             d = os.path.dirname(f)
 222             if not os.path.exists(d):
 223                 os.makedirs(d)
 224             outf = open(f, 'w')
 225             git.get_blob(h, outf, tree=srcpath)
 226             outf.close()
 227             os.chmod(f, int(m, 8))
 228
 229 def automatic_backport_mangle_c_file(name):
 230     return name.replace('/', '-')
 231
 232
 233 def add_automatic_backports(args):
 234     disable_list = []
 235     export = re.compile(r'^EXPORT_SYMBOL(_GPL)?\((?P<sym>[^\)]*)\)')
 236     bpi = kconfig.get_backport_info(os.path.join(args.bpid.target_dir, 'compat', 'Kconfig'))
 237     configtree = kconfig.ConfigTree(os.path.join(args.bpid.target_dir, 'Kconfig'), args.bpid)
 238     ignore=['Kconfig.kernel', 'Kconfig.versions']
 239     configtree.verify_sources(ignore=ignore)
 240     git_debug_snapshot(args, "verify sources for automatic backports")
 241     all_selects = configtree.all_selects()
 242     for sym, vals in bpi.items():
 243         if sym.startswith('BPAUTO_BUILD_'):
 244             if not sym[13:] in all_selects:
 245                 disable_list.append(sym)
 246                 continue
 247         symtype, module_name, c_files, h_files = vals
 248
 249         # first copy files
 250         files = []
 251         for f in c_files:
 252             files.append((f, os.path.join('compat', automatic_backport_mangle_c_file(f))))
 253         for f in h_files:
 254             files.append((os.path.join('include', f),
 255                           os.path.join('include', os.path.dirname(f), 'backport-' + os.path.basename(f))))
 256         if args.git_revision:
 257             copy_git_files(args.kerneldir, files, args.git_revision, args.bpid.target_dir)
 258         else:
 259             copy_files(args.kerneldir, files, args.bpid.target_dir)
 260
 261         # now add the Makefile line
 262         mf = open(os.path.join(args.bpid.target_dir, 'compat', 'Makefile'), 'a+')
 263         o_files = [automatic_backport_mangle_c_file(f)[:-1] + 'o' for f in c_files]
 264         if symtype == 'tristate':
 265             if not module_name:
 266                 raise Exception('backporting a module requires a #module-name')
 267             for of in o_files:
 268                 mf.write('%s-objs += %s\n' % (module_name, of))
 269             mf.write('obj-$(%s%s) += %s.o\n' % (args.bpid.full_prefix, sym, module_name))
 270         elif symtype == 'bool':
 271             mf.write('compat-$(%s%s) += %s\n' % (args.bpid.full_prefix, sym, ' '.join(o_files)))
 272
 273         # finally create the include file
 274         syms = []
 275         for f in c_files:
 276             for l in open(os.path.join(args.bpid.target_dir, 'compat',
 277                                        automatic_backport_mangle_c_file(f)), 'r'):
 278                 m = export.match(l)
 279                 if m:
 280                     syms.append(m.group('sym'))
 281         for f in h_files:
 282             outf = open(os.path.join(args.bpid.target_dir, 'include', f), 'w')
 283             outf.write('/* Automatically created during backport process */\n')
 284             outf.write('#ifndef %s%s\n' % (args.bpid.full_prefix, sym))
 285             outf.write('#include_next <%s>\n' % f)
 286             outf.write('#else\n');
 287             for s in syms:
 288                 outf.write('#undef %s\n' % s)
 289                 outf.write('#define %s LINUX_BACKPORT(%s)\n' % (s, s))
 290             outf.write('#include <%s>\n' % (os.path.dirname(f) + '/backport-' + os.path.basename(f), ))
 291             outf.write('#endif /* %s%s */\n' % (args.bpid.full_prefix, sym))
 292     return disable_list
 293
 294 def git_debug_init(args):
 295     """
 296     Initialize a git repository in the output directory and commit the current
 297     code in it. This is only used for debugging the transformations this code
 298     will do to the output later.
 299     """
 300     if not args.gitdebug:
 301         return
 302     # Git supports re-initialization, although not well documented it can
 303     # reset config stuff, lets avoid that if the tree already exists.
 304     if not os.path.exists(os.path.join(args.bpid.project_dir, '.git')):
 305         git.init(tree=args.bpid.project_dir)
 306     git.commit_all("Copied backport", tree=args.bpid.project_dir)
 307
 308
 309 def git_debug_snapshot(args, name):
 310     """
 311     Take a git snapshot for the debugging.
 312     """
 313     if not args.gitdebug:
 314         return
 315     git.commit_all(name, tree=args.bpid.project_dir)
 316
 317 def get_rel_spec_stable(rel):
 318     """
 319     Returns release specs for a linux-stable backports based release.
 320     """
 321     if ("rc" in rel):
 322         m = re.match(r"(?P<VERSION>\d+)\.+" \
 323                      "(?P<PATCHLEVEL>\d+)[.]*" \
 324                      "(?P<SUBLEVEL>\d*)" \
 325                      "[-rc]+(?P<RC_VERSION>\d+)\-+" \
 326                      "(?P<RELMOD_UPDATE>\d+)[-]*" \
 327                      "(?P<RELMOD_TYPE>[usnpc]*)", \
 328                      rel)
 329     else:
 330         m = re.match(r"(?P<VERSION>\d+)\.+" \
 331                      "(?P<PATCHLEVEL>\d+)[.]*" \
 332                      "(?P<SUBLEVEL>\d*)\-+" \
 333                      "(?P<RELMOD_UPDATE>\d+)[-]*" \
 334                      "(?P<RELMOD_TYPE>[usnpc]*)", \
 335                      rel)
 336     if (not m):
 337         return m
 338     return m.groupdict()
 339
 340 def get_rel_spec_next(rel):
 341     """
 342     Returns release specs for a linux-next backports based release.
 343     """
 344     m = re.match(r"(?P<DATE_VERSION>\d+)[-]*" \
 345                  "(?P<RELMOD_UPDATE>\d*)[-]*" \
 346                  "(?P<RELMOD_TYPE>[usnpc]*)", \
 347                  rel)
 348     if (not m):
 349         return m
 350     return m.groupdict()
 351
 352 def get_rel_prep(rel):
 353     """
 354     Returns a dict with prep work details we need prior to
 355     uploading a backports release to kernel.org
 356     """
 357     rel_specs = get_rel_spec_stable(rel)
 358     is_stable = True
 359     rel_tag = ""
 360     paths = list()
 361     if (not rel_specs):
 362         rel_specs = get_rel_spec_next(rel)
 363         if (not rel_specs):
 364             sys.stdout.write("rel: %s\n" % rel)
 365             return None
 366         if (rel_specs['RELMOD_UPDATE'] == '0' or
 367             rel_specs['RELMOD_UPDATE'] == '1'):
 368             return None
 369         is_stable = False
 370         date = rel_specs['DATE_VERSION']
 371         year = date[0:4]
 372         if (len(year) != 4):
 373             return None
 374         month = date[4:6]
 375         if (len(month) != 2):
 376             return None
 377         day = date[6:8]
 378         if (len(day) != 2):
 379             return None
 380         paths.append(year)
 381         paths.append(month)
 382         paths.append(day)
 383         rel_tag = "backports-" + rel.replace(rel_specs['RELMOD_TYPE'], "")
 384     else:
 385         ignore = "-"
 386         if (not rel_specs['RELMOD_UPDATE']):
 387             return None
 388         if (rel_specs['RELMOD_UPDATE'] == '0'):
 389             return None
 390         ignore += rel_specs['RELMOD_UPDATE']
 391         if (rel_specs['RELMOD_TYPE'] != ''):
 392             ignore += rel_specs['RELMOD_TYPE']
 393         base_rel = rel.replace(ignore, "")
 394         paths.append("v" + base_rel)
 395         rel_tag = "v" + rel.replace(rel_specs['RELMOD_TYPE'], "")
 396
 397     rel_prep = dict(stable = is_stable,
 398                     expected_tag = rel_tag,
 399                     paths_to_create = paths)
 400     return rel_prep
 401
 402 def create_tar_and_gz(tar_name, dir_to_tar):
 403     """
 404     We need both a tar file and gzip for kernel.org, the tar file
 405     gets signed, then we upload the compressed version, kup-server
 406     in the backend decompresses and verifies the tarball against
 407     our signature.
 408     """
 409     basename = os.path.basename(dir_to_tar)
 410     tar = tarfile.open(tar_name, "w")
 411     tar.add(dir_to_tar, basename)
 412     tar.close()
 413
 414     tar_file = open(tar_name, "r")
 415
 416     gz_file = gzip.GzipFile(tar_name + ".gz", 'wb')
 417     gz_file.write(tar_file.read())
 418     gz_file.close()
 419
 420 def upload_release(args, rel_prep, logwrite=lambda x:None):
 421     """
 422     Given a path of a relase make tarball out of it, PGP sign it, and
 423     then upload it to kernel.org using kup.
 424
 425     The linux-next based release do not require a RELMOD_UPDATE
 426     given that typically only one release is made per day. Using
 427     RELMOD_UPDATE for these releases is allowed though and if
 428     present it must be > 1.
 429
 430     The linux-stable based releases require a RELMOD_UPDATE.
 431
 432     RELMOD_UPDATE must be numeric and > 0 just as the RC releases
 433     of the Linux kernel.
 434
 435     The tree must also be tagged with the respective release, without
 436     the RELMOD_TYPE. For linux-next based releases this consists of
 437     backports- followed by DATE_VERSION and if RELMOD_TYPE is present.
 438     For linux-stable releases this consists of v followed by the
 439     full release version except the RELMOD_TYPE.
 440
 441     Uploads will not be allowed if these rules are not followed.
 442     """
 443     korg_path = "/pub/linux/kernel/projects/backports"
 444
 445     if (rel_prep['stable']):
 446         korg_path += "/stable"
 447
 448     parent = os.path.dirname(args.bpid.project_dir)
 449     release = os.path.basename(args.bpid.project_dir)
 450     tar_name = parent + '/' + release + ".tar"
 451     gzip_name = tar_name + ".gz"
 452
 453     create_tar_and_gz(tar_name, args.bpid.project_dir)
 454
 455     logwrite(gpg.sign(tar_name, extra_args=['--armor', '--detach-sign']))
 456
 457     logwrite("------------------------------------------------------")
 458
 459     if (not args.kup_test):
 460         logwrite("About to upload, current target path contents:")
 461     else:
 462         logwrite("kup-test: current target path contents:")
 463
 464     logwrite(kup.ls(path=korg_path))
 465
 466     for path in rel_prep['paths_to_create']:
 467         korg_path += '/' + path
 468         if (not args.kup_test):
 469             logwrite("create directory: %s" % korg_path)
 470             logwrite(kup.mkdir(korg_path))
 471     korg_path += '/'
 472     if (not args.kup_test):
 473         logwrite("upload file %s to %s" % (gzip_name, korg_path))
 474         logwrite(kup.put(gzip_name, tar_name + '.asc', korg_path))
 475         logwrite("\nFinished upload!\n")
 476         logwrite("Target path contents:")
 477         logwrite(kup.ls(path=korg_path))
 478     else:
 479         kup_cmd = "kup put /\n\t\t%s /\n\t\t%s /\n\t\t%s" % (gzip_name, tar_name + '.asc', korg_path)
 480         logwrite("kup-test: skipping cmd: %s" % kup_cmd)
 481
 482 def apply_patches(args, desc, source_dir, patch_src, target_dir, logwrite=lambda x:None):
 483     """
 484     Given a path of a directories of patches and SmPL patches apply
 485     them on the target directory. If requested refresh patches, or test
 486     a specific SmPL patch.
 487     """
 488     logwrite('Applying patches from %s to %s ...' % (patch_src, target_dir))
 489     test_cocci = args.test_cocci or args.profile_cocci
 490     test_cocci_found = False
 491     patches = []
 492     sempatches = []
 493     for root, dirs, files in os.walk(os.path.join(source_dir, patch_src)):
 494         for f in files:
 495             if not test_cocci and f.endswith('.patch'):
 496                 patches.append(os.path.join(root, f))
 497             if f.endswith('.cocci'):
 498                 if test_cocci:
 499                     if f not in test_cocci:
 500                         continue
 501                     test_cocci_found = True
 502                     if args.test_cocci:
 503                         logwrite("Testing Coccinelle SmPL patch: %s" % test_cocci)
 504                     elif args.profile_cocci:
 505                         logwrite("Profiling Coccinelle SmPL patch: %s" % test_cocci)
 506                 sempatches.append(os.path.join(root, f))
 507     patches.sort()
 508     prefix_len = len(os.path.join(source_dir, patch_src)) + 1
 509     for pfile in patches:
 510         print_name = pfile[prefix_len:]
 511         # read the patch file
 512         p = patch.fromfile(pfile)
 513         # complain if it's not a patch
 514         if not p:
 515             raise Exception('No patch content found in %s' % print_name)
 516         # leading / seems to be stripped?
 517         if 'dev/null' in p.items[0].source:
 518             raise Exception('Patches creating files are not supported (in %s)' % print_name)
 519         # check if the first file the patch touches exists, if so
 520         # assume the patch needs to be applied -- otherwise continue
 521         patched_file = '/'.join(p.items[0].source.split('/')[1:])
 522         fullfn = os.path.join(target_dir, patched_file)
 523         if not os.path.exists(fullfn):
 524             if args.verbose:
 525                 logwrite("Not applying %s, not needed" % print_name)
 526             continue
 527         if args.verbose:
 528             logwrite("Applying patch %s" % print_name)
 529
 530         if args.refresh:
 531             # but for refresh, of course look at all files the patch touches
 532             for patchitem in p.items:
 533                 patched_file = '/'.join(patchitem.source.split('/')[1:])
 534                 fullfn = os.path.join(target_dir, patched_file)
 535                 shutil.copyfile(fullfn, fullfn + '.orig_file')
 536
 537         process = subprocess.Popen(['patch', '-p1'], stdout=subprocess.PIPE,
 538                                    stderr=subprocess.STDOUT, stdin=subprocess.PIPE,
 539                                    close_fds=True, universal_newlines=True,
 540                                    cwd=target_dir)
 541         output = process.communicate(input=open(pfile, 'r').read())[0]
 542         output = output.split('\n')
 543         if output[-1] == '':
 544             output = output[:-1]
 545         if args.verbose:
 546             for line in output:
 547                 logwrite('> %s' % line)
 548         if process.returncode != 0:
 549             if not args.verbose:
 550                 logwrite("Failed to apply changes from %s" % print_name)
 551                 for line in output:
 552                     logwrite('> %s' % line)
 553             return 2
 554
 555         if args.refresh:
 556             pfilef = open(pfile + '.tmp', 'a')
 557             pfilef.write(p.top_header)
 558             pfilef.flush()
 559             for patchitem in p.items:
 560                 patched_file = '/'.join(patchitem.source.split('/')[1:])
 561                 fullfn = os.path.join(target_dir, patched_file)
 562                 process = subprocess.Popen(['diff', '-p', '-u', patched_file + '.orig_file', patched_file,
 563                                             '--label', 'a/' + patched_file,
 564                                             '--label', 'b/' + patched_file],
 565                                            stdout=pfilef, close_fds=True,
 566                                            universal_newlines=True, cwd=target_dir)
 567                 process.wait()
 568                 os.unlink(fullfn + '.orig_file')
 569                 if not process.returncode in (0, 1):
 570                     logwrite("Failed to diff to refresh %s" % print_name)
 571                     pfilef.close()
 572                     os.unlink(pfile + '.tmp')
 573                     return 2
 574             pfilef.close()
 575             os.rename(pfile + '.tmp', pfile)
 576
 577         # remove orig/rej files that patch sometimes creates
 578         for root, dirs, files in os.walk(target_dir):
 579             for f in files:
 580                 if f[-5:] == '.orig' or f[-4:] == '.rej':
 581                     os.unlink(os.path.join(root, f))
 582         git_debug_snapshot(args, "apply %s patch %s" % (desc, print_name))
 583
 584     sempatches.sort()
 585     prefix_len = len(os.path.join(source_dir, patch_src)) + 1
 586
 587     for cocci_file in sempatches:
 588         cmd = ['spatch',
 589             '--sp-file', cocci_file,
 590             '--in-place',
 591             '--recursive-includes',
 592             '--relax-include-path',
 593             '--use-coccigrep',
 594             '--timeout', '120',
 595             '-j', '%d' % cpu_count(),
 596             '--dir', os.path.abspath(target_dir) ]
 597         extra_spatch_args = []
 598         if args.profile_cocci:
 599             cmd.append('--profile')
 600         print_name = cocci_file[prefix_len:]
 601         if args.verbose:
 602             logwrite("Applying SmPL patch %s" % print_name)
 603         sprocess = subprocess.Popen(cmd,
 604                                     stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
 605                                     close_fds=True, universal_newlines=True,
 606                                     cwd=target_dir)
 607         output = sprocess.communicate()[0]
 608         sprocess.wait()
 609         if sprocess.returncode != 0:
 610             logwrite("Failed to process SmPL patch %s with %i" % (print_name, sprocess.returncode))
 611             return 2
 612         output = output.split('\n')
 613         if output[-1] == '':
 614             output = output[:-1]
 615         if args.verbose:
 616             for line in output:
 617                 logwrite('> %s' % line)
 618
 619         # remove cocci_backup files
 620         for root, dirs, files in os.walk(target_dir):
 621             for f in files:
 622                 if f.endswith('.cocci_backup'):
 623                     os.unlink(os.path.join(root, f))
 624         git_debug_snapshot(args, "apply %s SmPL patch %s" % (desc, print_name))
 625
 626     if test_cocci and test_cocci_found:
 627         logwrite('Done!')
 628         sys.exit(0)
 629
 630 def _main():
 631     # Our binary requirements go here
 632     req = reqs.Req()
 633     req.require('git')
 634     req.coccinelle('1.0.0-rc24')
 635     if not req.reqs_match():
 636         sys.exit(1)
 637
 638     # set up and parse arguments
 639     parser = argparse.ArgumentParser(description='generate backport tree')
 640     parser.add_argument('kerneldir', metavar='<kernel tree>', type=str,
 641                         help='Kernel tree to copy drivers from')
 642     parser.add_argument('outdir', metavar='<output directory>', type=str,
 643                         help='Directory to write the generated tree to')
 644     parser.add_argument('--copy-list', metavar='<listfile>', type=argparse.FileType('r'),
 645                         default='copy-list',
 646                         help='File containing list of files/directories to copy, default "copy-list"')
 647     parser.add_argument('--git-revision', metavar='<revision>', type=str,
 648                         help='git commit revision (see gitrevisions(7)) to take objects from.' +
 649                              'If this is specified, the kernel tree is used as git object storage ' +
 650                              'and we use git ls-tree to get the files.')
 651     parser.add_argument('--clean', const=True, default=False, action="store_const",
 652                         help='Clean output directory instead of erroring if it isn\'t empty')
 653     parser.add_argument('--integrate', const=True, default=False, action="store_const",
 654                         help='Integrate a future backported kernel solution into ' +
 655                              'an older kernel tree source directory.')
 656     parser.add_argument('--refresh', const=True, default=False, action="store_const",
 657                         help='Refresh patches as they are applied, the source dir will be modified!')
 658     parser.add_argument('--base-name', metavar='<name>', type=str, default='Linux',
 659                         help='name of base tree, default just "Linux"')
 660     parser.add_argument('--gitdebug', const=True, default=False, action="store_const",
 661                         help='Use git, in the output tree, to debug the various transformation steps ' +
 662                              'that the tree generation makes (apply patches, ...)')
 663     parser.add_argument('--verbose', const=True, default=False, action="store_const",
 664                         help='Print more verbose information')
 665     parser.add_argument('--extra-driver', nargs=2, metavar=('<source dir>', '<copy-list>'), type=str,
 666                         action='append', default=[], help='Extra driver directory/copy-list.')
 667     parser.add_argument('--kup', const=True, default=False, action="store_const",
 668                         help='For maintainers: upload a release to kernel.org')
 669     parser.add_argument('--kup-test', const=True, default=False, action="store_const",
 670                         help='For maintainers: do all the work as if you were about to ' +
 671                              'upload to kernel.org but do not do the final `kup put` ' +
 672                              'and also do not run any `kup mkdir` commands. This will ' +
 673                              'however run `kup ls` on the target paths so ' +
 674                              'at the very least we test your kup configuration. ' +
 675                              'If this is your first time uploading use this first!')
 676     parser.add_argument('--test-cocci', metavar='<sp_file>', type=str, default=None,
 677                         help='Only use the cocci file passed for Coccinelle, don\'t do anything else, ' +
 678                              'also creates a git repo on the target directory for easy inspection ' +
 679                              'of changes done by Coccinelle.')
 680     parser.add_argument('--profile-cocci', metavar='<sp_file>', type=str, default=None,
 681                         help='Only use the cocci file passed and pass --profile  to Coccinelle, ' +
 682                              'also creates a git repo on the target directory for easy inspection ' +
 683                              'of changes done by Coccinelle.')
 684     args = parser.parse_args()
 685
 686     # When building a package we use CPTCFG as we can rely on the
 687     # fact that kconfig treats CONFIG_ as an environment variable
 688     # requring less changes on code. For kernel integration we use
 689     # the longer CONFIG_BACKPORT given that we'll be sticking to
 690     # the kernel symbol namespace, to address that we do a final
 691     # search / replace. Technically its possible to rely on the
 692     # same prefix for packaging as with kernel integration but
 693     # there are already some users of the CPTCFG prefix.
 694     bpid = None
 695     if args.integrate:
 696         bpid = Bp_Identity(integrate = args.integrate,
 697                            kconfig_prefix = 'CONFIG_',
 698                            project_prefix = 'BACKPORT_',
 699                            project_dir = args.outdir,
 700                            target_dir = os.path.join(args.outdir, 'backports/'),
 701                            target_dir_name = 'backports/',
 702                            kconfig_source_var = '$BACKPORT_DIR',
 703                            )
 704     else:
 705         bpid = Bp_Identity(integrate = args.integrate,
 706                            kconfig_prefix = 'CPTCFG_',
 707                            project_prefix = '',
 708                            project_dir = args.outdir,
 709                            target_dir = args.outdir,
 710                            target_dir_name = '',
 711                            kconfig_source_var = '$BACKPORT_DIR',
 712                            )
 713
 714     def logwrite(msg):
 715         sys.stdout.write(msg)
 716         sys.stdout.write('\n')
 717         sys.stdout.flush()
 718
 719     return process(args.kerneldir, args.copy_list,
 720                    git_revision=args.git_revision,
 721                    bpid=bpid,
 722                    clean=args.clean,
 723                    refresh=args.refresh, base_name=args.base_name,
 724                    gitdebug=args.gitdebug, verbose=args.verbose,
 725                    extra_driver=args.extra_driver,
 726                    kup=args.kup,
 727                    kup_test=args.kup_test,
 728                    test_cocci=args.test_cocci,
 729                    profile_cocci=args.profile_cocci,
 730                    logwrite=logwrite)
 731
 732 def process(kerneldir, copy_list_file, git_revision=None,
 733             bpid=None,
 734             clean=False, refresh=False, base_name="Linux", gitdebug=False,
 735             verbose=False, extra_driver=[], kup=False,
 736             kup_test=False,
 737             test_cocci=None,
 738             profile_cocci=None,
 739             logwrite=lambda x:None,
 740             git_tracked_version=False):
 741     class Args(object):
 742         def __init__(self, kerneldir, copy_list_file,
 743                      git_revision, bpid, clean, refresh, base_name,
 744                      gitdebug, verbose, extra_driver, kup,
 745                      kup_test,
 746                      test_cocci,
 747                      profile_cocci):
 748             self.kerneldir = kerneldir
 749             self.copy_list = copy_list_file
 750             self.git_revision = git_revision
 751             self.bpid = bpid
 752             self.clean = clean
 753             self.refresh = refresh
 754             self.base_name = base_name
 755             self.gitdebug = gitdebug
 756             self.verbose = verbose
 757             self.extra_driver = extra_driver
 758             self.kup = kup
 759             self.kup_test = kup_test
 760             self.test_cocci = test_cocci
 761             self.profile_cocci = profile_cocci
 762             if self.test_cocci or self.profile_cocci:
 763                 self.gitdebug = True
 764     def git_paranoia(tree=None, logwrite=lambda x:None):
 765         data = git.paranoia(tree)
 766         if (data['r'] != 0):
 767             logwrite('Cannot use %s' % tree)
 768             logwrite('%s' % data['output'])
 769             sys.exit(data['r'])
 770         else:
 771             logwrite('Validated tree: %s' % tree)
 772
 773     args = Args(kerneldir, copy_list_file,
 774                 git_revision, bpid, clean, refresh, base_name,
 775                 gitdebug, verbose, extra_driver, kup, kup_test,
 776                 test_cocci, profile_cocci)
 777     rel_prep = None
 778
 779     if bpid.integrate:
 780         if args.kup_test or args.test_cocci or args.profile_cocci or args.refresh:
 781             logwrite('Cannot use integration with:\n\tkup_test\n\ttest_cocci\n\tprofile_cocci\n\trefresh\n');
 782             sys.exit(1)
 783
 784     # start processing ...
 785     if (args.kup or args.kup_test):
 786         git_paranoia(source_dir, logwrite)
 787         git_paranoia(kerneldir, logwrite)
 788
 789         rel_describe = git.describe(rev=None, tree=source_dir, extra_args=['--dirty'])
 790         release = os.path.basename(bpid.target_dir)
 791         version = release.replace("backports-", "")
 792
 793         rel_prep = get_rel_prep(version)
 794         if (not rel_prep):
 795             logwrite('Invalid backports release name: %s' % release)
 796             logwrite('For rules on the release name see upload_release()')
 797             sys.exit(1)
 798         rel_type = "linux-stable"
 799         if (not rel_prep['stable']):
 800             rel_type = "linux-next"
 801         if (rel_prep['expected_tag'] != rel_describe):
 802             logwrite('Unexpected %s based backports release tag on' % rel_type)
 803             logwrite('the backports tree tree: %s\n' % rel_describe)
 804             logwrite('You asked to make a release with this ')
 805             logwrite('directory name: %s' % release)
 806             logwrite('The actual expected tag we should find on')
 807             logwrite('the backports tree then is: %s\n' % rel_prep['expected_tag'])
 808             logwrite('For rules on the release name see upload_release()')
 809             sys.exit(1)
 810
 811     copy_list = read_copy_list(args.copy_list)
 812     deplist = read_dependencies(os.path.join(source_dir, 'dependencies'))
 813
 814     # validate output directory
 815     check_output_dir(bpid.target_dir, args.clean)
 816
 817     # do the copy
 818     backport_integrate_files = [
 819             ('Makefile.kernel', 'Makefile'),
 820             ('Kconfig.integrate', 'Kconfig'),
 821             ]
 822     backport_package_files = [(x, x) for x in [
 823         'Makefile',
 824         'kconf/',
 825         'Makefile.real',
 826         'Makefile.kernel',
 827         'Kconfig.package.hacks',
 828         'scripts/',
 829         '.blacklist.map',
 830         '.gitignore',
 831         'Makefile.build'] ]
 832     backport_package_files += [
 833             ('Kconfig.package', 'Kconfig'),
 834             ]
 835     backport_files = [(x, x) for x in [
 836         'Kconfig.sources',
 837         'compat/',
 838         'backport-include/',
 839     ]]
 840
 841     if not bpid.integrate:
 842         backport_files += backport_package_files
 843     else:
 844         backport_files += backport_integrate_files
 845
 846     if not args.git_revision:
 847         logwrite('Copy original source files ...')
 848     else:
 849         logwrite('Get original source files from git ...')
 850
 851     copy_files(os.path.join(source_dir, 'backport'), backport_files, bpid.target_dir)
 852
 853     git_debug_init(args)
 854
 855     if not args.git_revision:
 856         copy_files(args.kerneldir, copy_list, bpid.target_dir)
 857     else:
 858         copy_git_files(args.kerneldir, copy_list, args.git_revision, bpid.target_dir)
 859
 860     # FIXME: should we add a git version of this (e.g. --git-extra-driver)?
 861     for src, copy_list in args.extra_driver:
 862         if (args.kup or args.kup_test):
 863             git_paranoia(src)
 864         copy_files(src, read_copy_list(open(copy_list, 'r')), bpid.target_dir)
 865
 866     git_debug_snapshot(args, 'Add driver sources')
 867
 868     disable_list = add_automatic_backports(args)
 869     if git_tracked_version:
 870         backports_version = "(see git)"
 871         kernel_version = "(see git)"
 872     else:
 873         backports_version = git.describe(tree=source_dir, extra_args=['--long'])
 874         kernel_version = git.describe(rev=args.git_revision or 'HEAD',
 875                                       tree=args.kerneldir,
 876                                       extra_args=['--long'])
 877
 878     if not bpid.integrate:
 879         f = open(os.path.join(bpid.target_dir, 'versions'), 'w')
 880         f.write('BACKPORTS_VERSION="%s"\n' % backports_version)
 881         f.write('BACKPORTED_KERNEL_VERSION="%s"\n' % kernel_version)
 882         f.write('BACKPORTED_KERNEL_NAME="%s"\n' % args.base_name)
 883         if git_tracked_version:
 884             f.write('BACKPORTS_GIT_TRACKED="backport tracker ID: $(shell git rev-parse HEAD 2>/dev/null || echo \'not built in git tree\')"\n')
 885         f.close()
 886         git_debug_snapshot(args, "add versions files")
 887     else:
 888         kconf_regexes = [
 889                 (re.compile(r'.*(?P<key>%%BACKPORT_DIR%%)'), '%%BACKPORT_DIR%%', 'backports/'),
 890                 (re.compile(r'.*(?P<key>%%BACKPORTS_VERSION%%).*'), '%%BACKPORTS_VERSION%%', backports_version),
 891                 (re.compile(r'.*(?P<key>%%BACKPORTED_KERNEL_VERSION%%).*'), '%%BACKPORTED_KERNEL_VERSION%%', kernel_version),
 892                 (re.compile(r'.*(?P<key>%%BACKPORTED_KERNEL_NAME%%).*'), '%%BACKPORTED_KERNEL_NAME%%', args.base_name),
 893         ]
 894         out = ''
 895         for l in open(os.path.join(bpid.target_dir, 'Kconfig'), 'r'):
 896             for r in kconf_regexes:
 897                 m = r[0].match(l)
 898                 if m:
 899                     l = re.sub(r'(' + r[1] + ')', r'' + r[2] + '', l)
 900             out += l
 901         outf = open(os.path.join(bpid.target_dir, 'Kconfig'), 'w')
 902         outf.write(out)
 903         outf.close()
 904         git_debug_snapshot(args, "modify top level backports/Kconfig with backports identity")
 905
 906     if disable_list:
 907         # No need to verify_sources() as compat's Kconfig has no 'source' call
 908         bpcfg = kconfig.ConfigTree(os.path.join(bpid.target_dir, 'compat', 'Kconfig'), bpid)
 909         bpcfg.disable_symbols(disable_list)
 910     git_debug_snapshot(args, 'Add automatic backports')
 911
 912     failure = apply_patches(args, "backport", source_dir, 'patches', bpid.target_dir, logwrite)
 913     if failure:
 914         return failure
 915
 916     # Kernel integration requires Kconfig.versions already generated for you,
 917     # we cannot do this for a package as we have no idea what kernel folks
 918     # will be using.
 919     if bpid.integrate:
 920         kver = gen_version.kernelversion(bpid.project_dir)
 921         rel_specs = gen_version.get_rel_spec_stable(kver)
 922         if not rel_specs:
 923             logwrite('Cannot parse source kernel version, update parser')
 924             sys.exit(1)
 925         data = gen_version.genkconfig_versions(rel_specs)
 926         fo = open(os.path.join(bpid.target_dir, 'Kconfig.versions'), 'w')
 927         fo.write(data)
 928         fo.close()
 929         git_debug_snapshot(args, "generate kernel version requirement Kconfig file")
 930
 931     # some post-processing is required
 932     configtree = kconfig.ConfigTree(os.path.join(bpid.target_dir, 'Kconfig'), bpid)
 933     ignore=['Kconfig.kernel', 'Kconfig.versions', 'Kconfig.local']
 934
 935     configtree.verify_sources(ignore=ignore)
 936     git_debug_snapshot(args, "verify sources on top level backports Kconfig")
 937
 938     orig_symbols = configtree.symbols()
 939
 940     logwrite('Modify Kconfig tree ...')
 941     configtree.prune_sources(ignore=ignore)
 942     git_debug_snapshot(args, "prune Kconfig tree")
 943
 944     if not bpid.integrate:
 945         configtree.force_tristate_modular()
 946         git_debug_snapshot(args, "force tristate options modular")
 947
 948     ignore = [os.path.join(bpid.target_dir, x) for x in [
 949                 'Kconfig.package.hacks',
 950                 'Kconfig.versions',
 951                 'Kconfig.local',
 952                 'Kconfig',
 953                 ]
 954             ]
 955     configtree.adjust_backported_configs(ignore=ignore, orig_symbols=orig_symbols)
 956     git_debug_snapshot(args, "adjust backports config symbols we port")
 957
 958     configtree.modify_selects()
 959     git_debug_snapshot(args, "convert select to depends on")
 960
 961     symbols = configtree.symbols()
 962
 963     # write local symbol list -- needed during packaging build
 964     if not bpid.integrate:
 965         f = open(os.path.join(bpid.target_dir, 'local-symbols'), 'w')
 966         for sym in symbols:
 967             f.write('%s=\n' % sym)
 968         f.close()
 969         git_debug_snapshot(args, "add symbols files")
 970     # also write Kconfig.local, representing all local symbols
 971     # with a BACKPORTED_ prefix
 972     f = open(os.path.join(bpid.target_dir, 'Kconfig.local'), 'w')
 973     for sym in symbols:
 974         f.write('config BACKPORTED_%s\n' % sym)
 975         f.write('\ttristate\n')
 976         f.write('\tdefault %s\n' % sym)
 977     f.close()
 978     git_debug_snapshot(args, "add Kconfig.local")
 979
 980     # add defconfigs that we want
 981     defconfigs_dir = os.path.join(source_dir, 'backport', 'defconfigs')
 982     os.mkdir(os.path.join(bpid.target_dir, 'defconfigs'))
 983     for dfbase in os.listdir(defconfigs_dir):
 984         copy_defconfig = True
 985         dfsrc = os.path.join(defconfigs_dir, dfbase)
 986         for line in open(dfsrc, 'r'):
 987             if not '=' in line:
 988                 continue
 989             line_ok = False
 990             for sym in symbols:
 991                 if sym + '=' in line:
 992                     line_ok = True
 993                     break
 994             if not line_ok:
 995                 copy_defconfig = False
 996                 break
 997         if copy_defconfig:
 998             shutil.copy(dfsrc, os.path.join(bpid.target_dir, 'defconfigs', dfbase))
 999
1000     git_debug_snapshot(args, "add (useful) defconfig files")
1001
1002     logwrite('Rewrite Makefiles and Kconfig files ...')
1003
1004     # rewrite Makefile and source symbols
1005
1006     # symbols we know only we can provide under the backport project prefix
1007     # for which we need an exception.
1008     skip_orig_syms = [ bpid.project_prefix + x for x in [
1009             'INTEGRATE',
1010             ]
1011     ]
1012     parse_orig_syms = [x for x in orig_symbols if x not in skip_orig_syms ]
1013     regexes = []
1014     for some_symbols in [parse_orig_syms[i:i + 50] for i in range(0, len(parse_orig_syms), 50)]:
1015         r = 'CONFIG_((' + '|'.join([s + '(_MODULE)?' for s in some_symbols]) + ')([^A-Za-z0-9_]|$))'
1016         regexes.append(re.compile(r, re.MULTILINE))
1017     for root, dirs, files in os.walk(bpid.target_dir):
1018         # don't go into .git dir (possible debug thing)
1019         if '.git' in dirs:
1020             dirs.remove('.git')
1021         for f in files:
1022             data = open(os.path.join(root, f), 'r').read()
1023             for r in regexes:
1024                 data = r.sub(r'' + bpid.full_prefix + '\\1', data)
1025             data = re.sub(r'\$\(srctree\)', '$(backport_srctree)', data)
1026             data = re.sub(r'-Idrivers', '-I$(backport_srctree)/drivers', data)
1027             if bpid.integrate:
1028                 data = re.sub(r'CPTCFG_', bpid.full_prefix, data)
1029             fo = open(os.path.join(root, f), 'w')
1030             fo.write(data)
1031             fo.close()
1032
1033     git_debug_snapshot(args, "rename config symbol / srctree usage")
1034
1035     # disable unbuildable Kconfig symbols and stuff Makefiles that doesn't exist
1036     if bpid.integrate:
1037         maketree = make.MakeTree(os.path.join(bpid.target_dir, 'Makefile'))
1038     else:
1039         maketree = make.MakeTree(os.path.join(bpid.target_dir, 'Makefile.kernel'))
1040     disable_kconfig = []
1041     disable_makefile = []
1042     for sym in maketree.get_impossible_symbols():
1043         disable_kconfig.append(sym[7:])
1044         disable_makefile.append(sym[7:])
1045
1046     configtree.disable_symbols(disable_kconfig)
1047     git_debug_snapshot(args, "disable impossible kconfig symbols")
1048
1049     # add kernel version dependencies to Kconfig, from the dependency list
1050     # we read previously
1051     for sym in tuple(deplist.keys()):
1052         new = []
1053         for dep in deplist[sym]:
1054             if "kconfig:" in dep:
1055                     kconfig_expr = dep.replace('kconfig: ', '')
1056                     new.append(kconfig_expr)
1057             elif (dep == "DISABLE"):
1058                     new.append('BACKPORT_DISABLED_KCONFIG_OPTION')
1059             else:
1060                     new.append('!KERNEL_%s' % dep.replace('.', '_'))
1061         if bpid.integrate:
1062             deplist[sym] = ["BACKPORT_" + x for x in new]
1063         else:
1064             deplist[sym] = new
1065     configtree.add_dependencies(deplist)
1066     git_debug_snapshot(args, "add kernel version dependencies")
1067
1068     # disable things in makefiles that can't be selected and that the
1069     # build shouldn't recurse into because they don't exist -- if we
1070     # don't do that then a symbol from the kernel could cause the build
1071     # to attempt to recurse and fail
1072     #
1073     # Note that we split the regex after 50 symbols, this is because of a
1074     # limitation in the regex implementation (it only supports 100 nested
1075     # groups -- 50 seemed safer and is still fast)
1076     regexes = []
1077     for some_symbols in [disable_makefile[i:i + 50] for i in range(0, len(disable_makefile), 50)]:
1078         r = '^(([^#].*((' + bpid.full_prefix_resafe + '|CONFIG_)(' + '|'.join([s for s in some_symbols]) + ')))\W)'
1079         regexes.append(re.compile(r, re.MULTILINE))
1080     for f in maketree.get_makefiles():
1081         data = open(f, 'r').read()
1082         for r in regexes:
1083             data = r.sub(r'#\1', data)
1084         fo = open(f, 'w')
1085         fo.write(data)
1086         fo.close()
1087     git_debug_snapshot(args, "disable unsatisfied Makefile parts")
1088
1089     if bpid.integrate:
1090         f = open(os.path.join(bpid.project_dir, 'Kconfig'), 'a')
1091         f.write('source "backports/Kconfig"\n')
1092         f.close()
1093         git_debug_snapshot(args, "hooked backport to top level Kconfig")
1094
1095         failure = apply_patches(args, "integration", source_dir, 'integration-patches/',
1096                                 bpid.project_dir, logwrite)
1097         if failure:
1098             return failure
1099
1100     if (args.kup or args.kup_test):
1101         req = reqs.Req()
1102         req.kup()
1103         if not req.reqs_match():
1104             sys.exit(1)
1105         upload_release(args, rel_prep, logwrite=logwrite)
1106
1107     logwrite('Done!')
1108     return 0
1109
1110 if __name__ == '__main__':
1111     ret = _main()
1112     if ret:
1113         sys.exit(ret)