gentree.py

   1 #!/usr/bin/env python
   2 #
   3 # Generate the output tree into a specified directory.
   4 #
   5
   6 import argparse, sys, os, errno, shutil, re, subprocess
   7 import tarfile, gzip
   8
   9 # find self
  10 source_dir = os.path.abspath(os.path.dirname(__file__))
  11 sys.path.append(source_dir)
  12 # and import libraries we have
  13 from lib import kconfig, patch, make
  14 from lib import bpgit as git
  15 from lib import bpgpg as gpg
  16 from lib import bpkup as kup
  17 from lib.tempdir import tempdir
  18 from lib import bpreqs as reqs
  19
  20 class Bp_Identity(object):
  21     """
  22     folks considering multiple integrations may want to
  23     consider stuffing versioning info here as well but
  24     that will need thought/design on sharing compat and
  25     module namespaces.
  26
  27     Use the *_resafe when combining on regexps, although we currently
  28     don't support regexps there perhaps later we will and this will
  29     just make things safer for the output regardless. Once and if those
  30     are added, how we actually use the others for regular printing will
  31     need to be considered.
  32     """
  33     def __init__(self, integrate=False, kconfig_prefix='CPTCFG_',
  34                  project_prefix='', project_dir='',
  35                  target_dir='', target_dir_name='',
  36                  kconfig_source_var=None):
  37         self.integrate = integrate
  38         self.kconfig_prefix = kconfig_prefix
  39         self.kconfig_prefix_resafe = re.escape(kconfig_prefix)
  40         self.project_prefix = project_prefix
  41         self.project_prefix_resafe = re.escape(project_prefix)
  42         self.full_prefix = kconfig_prefix + project_prefix
  43         self.full_prefix_resafe = re.escape(self.full_prefix)
  44         self.project_dir = project_dir
  45         self.target_dir = target_dir
  46         self.target_dir_name = target_dir_name
  47         self.kconfig_source_var = kconfig_source_var
  48         if self.kconfig_source_var:
  49             self.kconfig_source_var_resafe = re.escape(self.kconfig_source_var)
  50         else:
  51             self.kconfig_source_var_resafe = None
  52
  53 def read_copy_list(copyfile):
  54     """
  55     Read a copy-list file and return a list of (source, target)
  56     tuples. The source and target are usually the same, but in
  57     the copy-list file there may be a rename included.
  58     """
  59     ret = []
  60     for item in copyfile:
  61         # remove leading/trailing whitespace
  62         item = item.strip()
  63         # comments
  64         if not item or item[0] == '#':
  65             continue
  66         if item[0] == '/':
  67             raise Exception("Input path '%s' is absolute path, this isn't allowed" % (item, ))
  68         if ' -> ' in item:
  69             srcitem, dstitem = item.split(' -> ')
  70             if (srcitem[-1] == '/') != (dstitem[-1] == '/'):
  71                 raise Exception("Cannot copy file/dir to dir/file")
  72         else:
  73             srcitem = dstitem = item
  74         ret.append((srcitem, dstitem))
  75     return ret
  76
  77
  78 def read_dependencies(depfilename):
  79     """
  80     Read a (the) dependency file and return the list of
  81     dependencies as a dictionary, mapping a Kconfig symbol
  82     to a list of kernel version dependencies.
  83
  84     If a backported feature that an upstream backported driver
  85     depends on had kconfig limitations (ie, debugging feature not
  86     available) a built constaint restriction can be expressed
  87     by using a kconfig expression. The kconfig expressions can
  88     be specified by using the "kconfig: " prefix.
  89
  90     While reading ignore blank or commented lines.
  91     """
  92     ret = {}
  93     depfile = open(depfilename, 'r')
  94     for item in depfile:
  95         kconfig_exp = ""
  96         item = item.strip()
  97         if not item or item[0] == '#':
  98             continue
  99         if "kconfig:" in item:
 100             sym, kconfig_exp = item.split(" ", 1)
 101             if not sym in ret:
 102                 ret[sym] = [kconfig_exp, ]
 103             else:
 104                 ret[sym].append(kconfig_exp)
 105         else:
 106             sym, dep = item.split()
 107             if not sym in ret:
 108                 ret[sym] = [dep, ]
 109             else:
 110                 ret[sym].append(dep)
 111     return ret
 112
 113
 114 def check_output_dir(d, clean):
 115     """
 116     Check that the output directory doesn't exist or is empty,
 117     unless clean is True in which case it's nuked. This helps
 118     sanity check the output when generating a tree, so usually
 119     running with --clean isn't suggested.
 120     """
 121     if clean:
 122         shutil.rmtree(d, ignore_errors=True)
 123     try:
 124         os.rmdir(d)
 125     except OSError as e:
 126         if e.errno != errno.ENOENT:
 127             raise
 128
 129
 130 def copytree(src, dst, symlinks=False, ignore=None):
 131     """
 132     Copy a directory tree. This differs from shutil.copytree()
 133     in that it allows destination directories to already exist.
 134     """
 135     names = os.listdir(src)
 136     if ignore is not None:
 137         ignored_names = ignore(src, names)
 138     else:
 139         ignored_names = set()
 140
 141     if not os.path.isdir(dst):
 142         os.makedirs(dst)
 143     errors = []
 144     for name in names:
 145         if name in ignored_names:
 146             continue
 147         srcname = os.path.join(src, name)
 148         dstname = os.path.join(dst, name)
 149         try:
 150             if symlinks and os.path.islink(srcname):
 151                 linkto = os.readlink(srcname)
 152                 os.symlink(linkto, dstname)
 153             elif os.path.isdir(srcname):
 154                 copytree(srcname, dstname, symlinks, ignore)
 155             else:
 156                 shutil.copy2(srcname, dstname)
 157         except (IOError, os.error) as why:
 158             errors.append((srcname, dstname, str(why)))
 159         # catch the Error from the recursive copytree so that we can
 160         # continue with other files
 161         except shutil.Error as err:
 162             errors.extend(err.args[0])
 163     try:
 164         shutil.copystat(src, dst)
 165     except WindowsError:
 166         # can't copy file access times on Windows
 167         pass
 168     except OSError as why:
 169         errors.extend((src, dst, str(why)))
 170     if errors:
 171         raise shutil.Error(errors)
 172
 173
 174 def copy_files(srcpath, copy_list, outdir):
 175     """
 176     Copy the copy_list files and directories from the srcpath
 177     to the outdir. The copy_list contains source and target
 178     names.
 179
 180     For now, it also ignores any *~ editor backup files, though
 181     this should probably be generalized (maybe using .gitignore?)
 182     Similarly the code that only copies some files (*.c, *.h,
 183     *.awk, Kconfig, Makefile) to avoid any build remnants in the
 184     kernel if they should exist.
 185     """
 186     for srcitem, tgtitem in copy_list:
 187         if tgtitem == '':
 188             copytree(srcpath, outdir, ignore=shutil.ignore_patterns('*~'))
 189         elif tgtitem[-1] == '/':
 190             def copy_ignore(dir, entries):
 191                 r = []
 192                 for i in entries:
 193                     if i[-2:] == '.o' or i[-1] == '~':
 194                         r.append(i)
 195                 return r
 196             copytree(os.path.join(srcpath, srcitem),
 197                      os.path.join(outdir, tgtitem),
 198                      ignore=copy_ignore)
 199         else:
 200             try:
 201                 os.makedirs(os.path.join(outdir, os.path.dirname(tgtitem)))
 202             except OSError as e:
 203                 # ignore dirs we might have created just now
 204                 if e.errno != errno.EEXIST:
 205                     raise
 206             shutil.copy(os.path.join(srcpath, srcitem),
 207                         os.path.join(outdir, tgtitem))
 208
 209
 210 def copy_git_files(srcpath, copy_list, rev, outdir):
 211     """
 212     "Copy" files from a git repository. This really means listing them with
 213     ls-tree and then using git show to obtain all the blobs.
 214     """
 215     for srcitem, tgtitem in copy_list:
 216         for m, t, h, f in git.ls_tree(rev=rev, files=(srcitem,), tree=srcpath):
 217             assert t == 'blob'
 218             f = os.path.join(outdir, f.replace(srcitem, tgtitem))
 219             d = os.path.dirname(f)
 220             if not os.path.exists(d):
 221                 os.makedirs(d)
 222             outf = open(f, 'w')
 223             git.get_blob(h, outf, tree=srcpath)
 224             outf.close()
 225             os.chmod(f, int(m, 8))
 226
 227 def automatic_backport_mangle_c_file(name):
 228     return name.replace('/', '-')
 229
 230
 231 def add_automatic_backports(args):
 232     disable_list = []
 233     export = re.compile(r'^EXPORT_SYMBOL(_GPL)?\((?P<sym>[^\)]*)\)')
 234     bpi = kconfig.get_backport_info(os.path.join(args.bpid.target_dir, 'compat', 'Kconfig'))
 235     configtree = kconfig.ConfigTree(os.path.join(args.bpid.target_dir, 'Kconfig'), args.bpid)
 236     all_selects = configtree.all_selects()
 237     for sym, vals in bpi.items():
 238         if sym.startswith('BPAUTO_BUILD_'):
 239             if not sym[15:] in all_selects:
 240                 disable_list.append(sym)
 241                 continue
 242         symtype, module_name, c_files, h_files = vals
 243
 244         # first copy files
 245         files = []
 246         for f in c_files:
 247             files.append((f, os.path.join('compat', automatic_backport_mangle_c_file(f))))
 248         for f in h_files:
 249             files.append((os.path.join('include', f),
 250                           os.path.join('include', os.path.dirname(f), 'backport-' + os.path.basename(f))))
 251         if args.git_revision:
 252             copy_git_files(args.kerneldir, files, args.git_revision, args.bpid.target_dir)
 253         else:
 254             copy_files(args.kerneldir, files, args.bpid.target_dir)
 255
 256         # now add the Makefile line
 257         mf = open(os.path.join(args.bpid.target_dir, 'compat', 'Makefile'), 'a+')
 258         o_files = [automatic_backport_mangle_c_file(f)[:-1] + 'o' for f in c_files]
 259         if symtype == 'tristate':
 260             if not module_name:
 261                 raise Exception('backporting a module requires a #module-name')
 262             for of in o_files:
 263                 mf.write('%s-objs += %s\n' % (module_name, of))
 264             mf.write('obj-$(%s%s) += %s.o\n' % (args.bpid.full_prefix, sym, module_name))
 265         elif symtype == 'bool':
 266             mf.write('compat-$(%s%s) += %s\n' % (args.bpid.full_prefix, sym, ' '.join(o_files)))
 267
 268         # finally create the include file
 269         syms = []
 270         for f in c_files:
 271             for l in open(os.path.join(args.bpid.target_dir, 'compat',
 272                                        automatic_backport_mangle_c_file(f)), 'r'):
 273                 m = export.match(l)
 274                 if m:
 275                     syms.append(m.group('sym'))
 276         for f in h_files:
 277             outf = open(os.path.join(args.bpid.target_dir, 'include', f), 'w')
 278             outf.write('/* Automatically created during backport process */\n')
 279             outf.write('#ifndef %s%s\n' % (args.bpid.full_prefix, sym))
 280             outf.write('#include_next <%s>\n' % f)
 281             outf.write('#else\n');
 282             for s in syms:
 283                 outf.write('#undef %s\n' % s)
 284                 outf.write('#define %s LINUX_BACKPORT(%s)\n' % (s, s))
 285             outf.write('#include <%s>\n' % (os.path.dirname(f) + '/backport-' + os.path.basename(f), ))
 286             outf.write('#endif /* %s%s */\n' % (args.bpid.full_prefix, sym))
 287     return disable_list
 288
 289 def git_debug_init(args):
 290     """
 291     Initialize a git repository in the output directory and commit the current
 292     code in it. This is only used for debugging the transformations this code
 293     will do to the output later.
 294     """
 295     if not args.gitdebug:
 296         return
 297     # Git supports re-initialization, although not well documented it can
 298     # reset config stuff, lets avoid that if the tree already exists.
 299     if not os.path.exists(os.path.join(args.bpid.project_dir, '.git')):
 300         git.init(tree=args.bpid.project_dir)
 301     git.commit_all("Copied backport", tree=args.bpid.project_dir)
 302
 303
 304 def git_debug_snapshot(args, name):
 305     """
 306     Take a git snapshot for the debugging.
 307     """
 308     if not args.gitdebug:
 309         return
 310     git.commit_all(name, tree=args.bpid.project_dir)
 311
 312 def get_rel_spec_stable(rel):
 313     """
 314     Returns release specs for a linux-stable backports based release.
 315     """
 316     if ("rc" in rel):
 317         m = re.match(r"(?P<VERSION>\d+)\.+" \
 318                      "(?P<PATCHLEVEL>\d+)[.]*" \
 319                      "(?P<SUBLEVEL>\d*)" \
 320                      "[-rc]+(?P<RC_VERSION>\d+)\-+" \
 321                      "(?P<RELMOD_UPDATE>\d+)[-]*" \
 322                      "(?P<RELMOD_TYPE>[usnpc]*)", \
 323                      rel)
 324     else:
 325         m = re.match(r"(?P<VERSION>\d+)\.+" \
 326                      "(?P<PATCHLEVEL>\d+)[.]*" \
 327                      "(?P<SUBLEVEL>\d*)\-+" \
 328                      "(?P<RELMOD_UPDATE>\d+)[-]*" \
 329                      "(?P<RELMOD_TYPE>[usnpc]*)", \
 330                      rel)
 331     if (not m):
 332         return m
 333     return m.groupdict()
 334
 335 def get_rel_spec_next(rel):
 336     """
 337     Returns release specs for a linux-next backports based release.
 338     """
 339     m = re.match(r"(?P<DATE_VERSION>\d+)[-]*" \
 340                  "(?P<RELMOD_UPDATE>\d*)[-]*" \
 341                  "(?P<RELMOD_TYPE>[usnpc]*)", \
 342                  rel)
 343     if (not m):
 344         return m
 345     return m.groupdict()
 346
 347 def get_rel_prep(rel):
 348     """
 349     Returns a dict with prep work details we need prior to
 350     uploading a backports release to kernel.org
 351     """
 352     rel_specs = get_rel_spec_stable(rel)
 353     is_stable = True
 354     rel_tag = ""
 355     paths = list()
 356     if (not rel_specs):
 357         rel_specs = get_rel_spec_next(rel)
 358         if (not rel_specs):
 359             sys.stdout.write("rel: %s\n" % rel)
 360             return None
 361         if (rel_specs['RELMOD_UPDATE'] == '0' or
 362             rel_specs['RELMOD_UPDATE'] == '1'):
 363             return None
 364         is_stable = False
 365         date = rel_specs['DATE_VERSION']
 366         year = date[0:4]
 367         if (len(year) != 4):
 368             return None
 369         month = date[4:6]
 370         if (len(month) != 2):
 371             return None
 372         day = date[6:8]
 373         if (len(day) != 2):
 374             return None
 375         paths.append(year)
 376         paths.append(month)
 377         paths.append(day)
 378         rel_tag = "backports-" + rel.replace(rel_specs['RELMOD_TYPE'], "")
 379     else:
 380         ignore = "-"
 381         if (not rel_specs['RELMOD_UPDATE']):
 382             return None
 383         if (rel_specs['RELMOD_UPDATE'] == '0'):
 384             return None
 385         ignore += rel_specs['RELMOD_UPDATE']
 386         if (rel_specs['RELMOD_TYPE'] != ''):
 387             ignore += rel_specs['RELMOD_TYPE']
 388         base_rel = rel.replace(ignore, "")
 389         paths.append("v" + base_rel)
 390         rel_tag = "v" + rel.replace(rel_specs['RELMOD_TYPE'], "")
 391
 392     rel_prep = dict(stable = is_stable,
 393                     expected_tag = rel_tag,
 394                     paths_to_create = paths)
 395     return rel_prep
 396
 397 def create_tar_and_gz(tar_name, dir_to_tar):
 398     """
 399     We need both a tar file and gzip for kernel.org, the tar file
 400     gets signed, then we upload the compressed version, kup-server
 401     in the backend decompresses and verifies the tarball against
 402     our signature.
 403     """
 404     basename = os.path.basename(dir_to_tar)
 405     tar = tarfile.open(tar_name, "w")
 406     tar.add(dir_to_tar, basename)
 407     tar.close()
 408
 409     tar_file = open(tar_name, "r")
 410
 411     gz_file = gzip.GzipFile(tar_name + ".gz", 'wb')
 412     gz_file.write(tar_file.read())
 413     gz_file.close()
 414
 415 def upload_release(args, rel_prep, logwrite=lambda x:None):
 416     """
 417     Given a path of a relase make tarball out of it, PGP sign it, and
 418     then upload it to kernel.org using kup.
 419
 420     The linux-next based release do not require a RELMOD_UPDATE
 421     given that typically only one release is made per day. Using
 422     RELMOD_UPDATE for these releases is allowed though and if
 423     present it must be > 1.
 424
 425     The linux-stable based releases require a RELMOD_UPDATE.
 426
 427     RELMOD_UPDATE must be numeric and > 0 just as the RC releases
 428     of the Linux kernel.
 429
 430     The tree must also be tagged with the respective release, without
 431     the RELMOD_TYPE. For linux-next based releases this consists of
 432     backports- followed by DATE_VERSION and if RELMOD_TYPE is present.
 433     For linux-stable releases this consists of v followed by the
 434     full release version except the RELMOD_TYPE.
 435
 436     Uploads will not be allowed if these rules are not followed.
 437     """
 438     korg_path = "/pub/linux/kernel/projects/backports"
 439
 440     if (rel_prep['stable']):
 441         korg_path += "/stable"
 442
 443     parent = os.path.dirname(args.bpid.project_dir)
 444     release = os.path.basename(args.bpid.project_dir)
 445     tar_name = parent + '/' + release + ".tar"
 446     gzip_name = tar_name + ".gz"
 447
 448     create_tar_and_gz(tar_name, args.bpid.project_dir)
 449
 450     logwrite(gpg.sign(tar_name, extra_args=['--armor', '--detach-sign']))
 451
 452     logwrite("------------------------------------------------------")
 453
 454     if (not args.kup_test):
 455         logwrite("About to upload, current target path contents:")
 456     else:
 457         logwrite("kup-test: current target path contents:")
 458
 459     logwrite(kup.ls(path=korg_path))
 460
 461     for path in rel_prep['paths_to_create']:
 462         korg_path += '/' + path
 463         if (not args.kup_test):
 464             logwrite("create directory: %s" % korg_path)
 465             logwrite(kup.mkdir(korg_path))
 466     korg_path += '/'
 467     if (not args.kup_test):
 468         logwrite("upload file %s to %s" % (gzip_name, korg_path))
 469         logwrite(kup.put(gzip_name, tar_name + '.asc', korg_path))
 470         logwrite("\nFinished upload!\n")
 471         logwrite("Target path contents:")
 472         logwrite(kup.ls(path=korg_path))
 473     else:
 474         kup_cmd = "kup put /\n\t\t%s /\n\t\t%s /\n\t\t%s" % (gzip_name, tar_name + '.asc', korg_path)
 475         logwrite("kup-test: skipping cmd: %s" % kup_cmd)
 476
 477 def apply_patches(args, desc, source_dir, patch_src, target_dir, logwrite=lambda x:None):
 478     """
 479     Given a path of a directories of patches and SmPL patches apply
 480     them on the target directory. If requested refresh patches, or test
 481     a specific SmPL patch.
 482     """
 483     logwrite('Applying patches from %s to %s ...' % (patch_src, target_dir))
 484     test_cocci = args.test_cocci or args.profile_cocci
 485     test_cocci_found = False
 486     patches = []
 487     sempatches = []
 488     for root, dirs, files in os.walk(os.path.join(source_dir, patch_src)):
 489         for f in files:
 490             if not test_cocci and f.endswith('.patch'):
 491                 patches.append(os.path.join(root, f))
 492             if f.endswith('.cocci'):
 493                 if test_cocci:
 494                     if f not in test_cocci:
 495                         continue
 496                     test_cocci_found = True
 497                     if args.test_cocci:
 498                         logwrite("Testing Coccinelle SmPL patch: %s" % test_cocci)
 499                     elif args.profile_cocci:
 500                         logwrite("Profiling Coccinelle SmPL patch: %s" % test_cocci)
 501                 sempatches.append(os.path.join(root, f))
 502     patches.sort()
 503     prefix_len = len(os.path.join(source_dir, patch_src)) + 1
 504     for pfile in patches:
 505         print_name = pfile[prefix_len:]
 506         # read the patch file
 507         p = patch.fromfile(pfile)
 508         # complain if it's not a patch
 509         if not p:
 510             raise Exception('No patch content found in %s' % print_name)
 511         # leading / seems to be stripped?
 512         if 'dev/null' in p.items[0].source:
 513             raise Exception('Patches creating files are not supported (in %s)' % print_name)
 514         # check if the first file the patch touches exists, if so
 515         # assume the patch needs to be applied -- otherwise continue
 516         patched_file = '/'.join(p.items[0].source.split('/')[1:])
 517         fullfn = os.path.join(target_dir, patched_file)
 518         if not os.path.exists(fullfn):
 519             if args.verbose:
 520                 logwrite("Not applying %s, not needed" % print_name)
 521             continue
 522         if args.verbose:
 523             logwrite("Applying patch %s" % print_name)
 524
 525         if args.refresh:
 526             # but for refresh, of course look at all files the patch touches
 527             for patchitem in p.items:
 528                 patched_file = '/'.join(patchitem.source.split('/')[1:])
 529                 fullfn = os.path.join(target_dir, patched_file)
 530                 shutil.copyfile(fullfn, fullfn + '.orig_file')
 531
 532         process = subprocess.Popen(['patch', '-p1'], stdout=subprocess.PIPE,
 533                                    stderr=subprocess.STDOUT, stdin=subprocess.PIPE,
 534                                    close_fds=True, universal_newlines=True,
 535                                    cwd=target_dir)
 536         output = process.communicate(input=open(pfile, 'r').read())[0]
 537         output = output.split('\n')
 538         if output[-1] == '':
 539             output = output[:-1]
 540         if args.verbose:
 541             for line in output:
 542                 logwrite('> %s' % line)
 543         if process.returncode != 0:
 544             if not args.verbose:
 545                 logwrite("Failed to apply changes from %s" % print_name)
 546                 for line in output:
 547                     logwrite('> %s' % line)
 548             raise Exception('Patch failed')
 549
 550         if args.refresh:
 551             pfilef = open(pfile + '.tmp', 'a')
 552             pfilef.write(p.top_header)
 553             pfilef.flush()
 554             for patchitem in p.items:
 555                 patched_file = '/'.join(patchitem.source.split('/')[1:])
 556                 fullfn = os.path.join(target_dir, patched_file)
 557                 process = subprocess.Popen(['diff', '-p', '-u', patched_file + '.orig_file', patched_file,
 558                                             '--label', 'a/' + patched_file,
 559                                             '--label', 'b/' + patched_file],
 560                                            stdout=pfilef, close_fds=True,
 561                                            universal_newlines=True, cwd=target_dir)
 562                 process.wait()
 563                 os.unlink(fullfn + '.orig_file')
 564                 if not process.returncode in (0, 1):
 565                     logwrite("Failed to diff to refresh %s" % print_name)
 566                     pfilef.close()
 567                     os.unlink(pfile + '.tmp')
 568                     raise Exception('Refresh failed')
 569             pfilef.close()
 570             os.rename(pfile + '.tmp', pfile)
 571
 572         # remove orig/rej files that patch sometimes creates
 573         for root, dirs, files in os.walk(target_dir):
 574             for f in files:
 575                 if f[-5:] == '.orig' or f[-4:] == '.rej':
 576                     os.unlink(os.path.join(root, f))
 577         git_debug_snapshot(args, "apply %s patch %s" % (desc, print_name))
 578
 579     sempatches.sort()
 580     prefix_len = len(os.path.join(source_dir, patch_src)) + 1
 581
 582     for cocci_file in sempatches:
 583         # Until Coccinelle picks this up
 584         pycocci = os.path.join(source_dir, 'devel/pycocci')
 585         cmd = [pycocci, cocci_file]
 586         extra_spatch_args = []
 587         if args.profile_cocci:
 588             cmd.append('--profile-cocci')
 589         cmd.append(os.path.abspath(target_dir))
 590         print_name = cocci_file[prefix_len:]
 591         if args.verbose:
 592             logwrite("Applying SmPL patch %s" % print_name)
 593         sprocess = subprocess.Popen(cmd,
 594                                     stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
 595                                     close_fds=True, universal_newlines=True,
 596                                     cwd=target_dir)
 597         output = sprocess.communicate()[0]
 598         sprocess.wait()
 599         if sprocess.returncode != 0:
 600             logwrite("Failed to process SmPL patch %s" % print_name)
 601             raise Exception('SmPL patch failed')
 602         output = output.split('\n')
 603         if output[-1] == '':
 604             output = output[:-1]
 605         if args.verbose:
 606             for line in output:
 607                 logwrite('> %s' % line)
 608
 609         # remove cocci_backup files
 610         for root, dirs, files in os.walk(target_dir):
 611             for f in files:
 612                 if f.endswith('.cocci_backup'):
 613                     os.unlink(os.path.join(root, f))
 614         git_debug_snapshot(args, "apply %s SmPL patch %s" % (desc, print_name))
 615
 616     if test_cocci and test_cocci_found:
 617         logwrite('Done!')
 618         sys.exit(0)
 619
 620 def _main():
 621     # Our binary requirements go here
 622     req = reqs.Req()
 623     req.require('git')
 624     req.coccinelle('1.0.0-rc21')
 625     if not req.reqs_match():
 626         sys.exit(1)
 627
 628     # set up and parse arguments
 629     parser = argparse.ArgumentParser(description='generate backport tree')
 630     parser.add_argument('kerneldir', metavar='<kernel tree>', type=str,
 631                         help='Kernel tree to copy drivers from')
 632     parser.add_argument('outdir', metavar='<output directory>', type=str,
 633                         help='Directory to write the generated tree to')
 634     parser.add_argument('--copy-list', metavar='<listfile>', type=argparse.FileType('r'),
 635                         default='copy-list',
 636                         help='File containing list of files/directories to copy, default "copy-list"')
 637     parser.add_argument('--git-revision', metavar='<revision>', type=str,
 638                         help='git commit revision (see gitrevisions(7)) to take objects from.' +
 639                              'If this is specified, the kernel tree is used as git object storage ' +
 640                              'and we use git ls-tree to get the files.')
 641     parser.add_argument('--clean', const=True, default=False, action="store_const",
 642                         help='Clean output directory instead of erroring if it isn\'t empty')
 643     parser.add_argument('--refresh', const=True, default=False, action="store_const",
 644                         help='Refresh patches as they are applied, the source dir will be modified!')
 645     parser.add_argument('--base-name', metavar='<name>', type=str, default='Linux',
 646                         help='name of base tree, default just "Linux"')
 647     parser.add_argument('--gitdebug', const=True, default=False, action="store_const",
 648                         help='Use git, in the output tree, to debug the various transformation steps ' +
 649                              'that the tree generation makes (apply patches, ...)')
 650     parser.add_argument('--verbose', const=True, default=False, action="store_const",
 651                         help='Print more verbose information')
 652     parser.add_argument('--extra-driver', nargs=2, metavar=('<source dir>', '<copy-list>'), type=str,
 653                         action='append', default=[], help='Extra driver directory/copy-list.')
 654     parser.add_argument('--kup', const=True, default=False, action="store_const",
 655                         help='For maintainers: upload a release to kernel.org')
 656     parser.add_argument('--kup-test', const=True, default=False, action="store_const",
 657                         help='For maintainers: do all the work as if you were about to ' +
 658                              'upload to kernel.org but do not do the final `kup put` ' +
 659                              'and also do not run any `kup mkdir` commands. This will ' +
 660                              'however run `kup ls` on the target paths so ' +
 661                              'at the very least we test your kup configuration. ' +
 662                              'If this is your first time uploading use this first!')
 663     parser.add_argument('--test-cocci', metavar='<sp_file>', type=str, default=None,
 664                         help='Only use the cocci file passed for Coccinelle, don\'t do anything else, ' +
 665                              'also creates a git repo on the target directory for easy inspection ' +
 666                              'of changes done by Coccinelle.')
 667     parser.add_argument('--profile-cocci', metavar='<sp_file>', type=str, default=None,
 668                         help='Only use the cocci file passed and pass --profile  to Coccinelle, ' +
 669                              'also creates a git repo on the target directory for easy inspection ' +
 670                              'of changes done by Coccinelle.')
 671     args = parser.parse_args()
 672
 673     # When building a package we use CPTCFG as we can rely on the
 674     # fact that kconfig treats CONFIG_ as an environment variable
 675     # requring less changes on code. For kernel integration we use
 676     # the longer CONFIG_BACKPORT given that we'll be sticking to
 677     # the kernel symbol namespace, to address that we do a final
 678     # search / replace. Technically its possible to rely on the
 679     # same prefix for packaging as with kernel integration but
 680     # there are already some users of the CPTCFG prefix.
 681     bpid = None
 682     integrate = False
 683     if integrate:
 684         bpid = Bp_Identity(integrate = integrate,
 685                            kconfig_prefix = 'CONFIG_',
 686                            project_prefix = 'BACKPORT_',
 687                            project_dir = args.outdir,
 688                            target_dir = os.path.join(args.outdir, 'backports/'),
 689                            target_dir_name = 'backports/',
 690                            kconfig_source_var = '$BACKPORT_DIR',
 691                            )
 692     else:
 693         bpid = Bp_Identity(integrate = integrate,
 694                            kconfig_prefix = 'CPTCFG_',
 695                            project_prefix = '',
 696                            project_dir = args.outdir,
 697                            target_dir = args.outdir,
 698                            target_dir_name = '',
 699                            kconfig_source_var = '$BACKPORT_DIR',
 700                            )
 701
 702     def logwrite(msg):
 703         sys.stdout.write(msg)
 704         sys.stdout.write('\n')
 705         sys.stdout.flush()
 706
 707     return process(args.kerneldir, args.copy_list,
 708                    git_revision=args.git_revision,
 709                    bpid=bpid,
 710                    clean=args.clean,
 711                    refresh=args.refresh, base_name=args.base_name,
 712                    gitdebug=args.gitdebug, verbose=args.verbose,
 713                    extra_driver=args.extra_driver,
 714                    kup=args.kup,
 715                    kup_test=args.kup_test,
 716                    test_cocci=args.test_cocci,
 717                    profile_cocci=args.profile_cocci,
 718                    logwrite=logwrite)
 719
 720 def process(kerneldir, copy_list_file, git_revision=None,
 721             bpid=None,
 722             clean=False, refresh=False, base_name="Linux", gitdebug=False,
 723             verbose=False, extra_driver=[], kup=False,
 724             kup_test=False,
 725             test_cocci=None,
 726             profile_cocci=None,
 727             logwrite=lambda x:None,
 728             git_tracked_version=False):
 729     class Args(object):
 730         def __init__(self, kerneldir, copy_list_file,
 731                      git_revision, bpid, clean, refresh, base_name,
 732                      gitdebug, verbose, extra_driver, kup,
 733                      kup_test,
 734                      test_cocci,
 735                      profile_cocci):
 736             self.kerneldir = kerneldir
 737             self.copy_list = copy_list_file
 738             self.git_revision = git_revision
 739             self.bpid = bpid
 740             self.clean = clean
 741             self.refresh = refresh
 742             self.base_name = base_name
 743             self.gitdebug = gitdebug
 744             self.verbose = verbose
 745             self.extra_driver = extra_driver
 746             self.kup = kup
 747             self.kup_test = kup_test
 748             self.test_cocci = test_cocci
 749             self.profile_cocci = profile_cocci
 750             if self.test_cocci or self.profile_cocci:
 751                 self.gitdebug = True
 752     def git_paranoia(tree=None, logwrite=lambda x:None):
 753         data = git.paranoia(tree)
 754         if (data['r'] != 0):
 755             logwrite('Cannot use %s' % tree)
 756             logwrite('%s' % data['output'])
 757             sys.exit(data['r'])
 758         else:
 759             logwrite('Validated tree: %s' % tree)
 760
 761     args = Args(kerneldir, copy_list_file,
 762                 git_revision, bpid, clean, refresh, base_name,
 763                 gitdebug, verbose, extra_driver, kup, kup_test,
 764                 test_cocci, profile_cocci)
 765     rel_prep = None
 766
 767     # start processing ...
 768     if (args.kup or args.kup_test):
 769         git_paranoia(source_dir, logwrite)
 770         git_paranoia(kerneldir, logwrite)
 771
 772         rel_describe = git.describe(rev=None, tree=source_dir, extra_args=['--dirty'])
 773         release = os.path.basename(bpid.target_dir)
 774         version = release.replace("backports-", "")
 775
 776         rel_prep = get_rel_prep(version)
 777         if (not rel_prep):
 778             logwrite('Invalid backports release name: %s' % release)
 779             logwrite('For rules on the release name see upload_release()')
 780             sys.exit(1)
 781         rel_type = "linux-stable"
 782         if (not rel_prep['stable']):
 783             rel_type = "linux-next"
 784         if (rel_prep['expected_tag'] != rel_describe):
 785             logwrite('Unexpected %s based backports release tag on' % rel_type)
 786             logwrite('the backports tree tree: %s\n' % rel_describe)
 787             logwrite('You asked to make a release with this ')
 788             logwrite('directory name: %s' % release)
 789             logwrite('The actual expected tag we should find on')
 790             logwrite('the backports tree then is: %s\n' % rel_prep['expected_tag'])
 791             logwrite('For rules on the release name see upload_release()')
 792             sys.exit(1)
 793
 794     copy_list = read_copy_list(args.copy_list)
 795     deplist = read_dependencies(os.path.join(source_dir, 'dependencies'))
 796
 797     # validate output directory
 798     check_output_dir(bpid.target_dir, args.clean)
 799
 800     # do the copy
 801     backport_package_files = [(x, x) for x in [
 802         'Makefile',
 803         'kconf/',
 804         'Makefile.real',
 805         'Makefile.kernel',
 806         'Kconfig.package.hacks',
 807         'scripts/',
 808         '.blacklist.map',
 809         '.gitignore',
 810         'Makefile.build'] ]
 811     backport_package_files += [
 812             ('Kconfig.package', 'Kconfig'),
 813             ]
 814     backport_files = [(x, x) for x in [
 815         'Kconfig.sources',
 816         'compat/',
 817         'backport-include/',
 818     ]]
 819
 820     if not bpid.integrate:
 821         backport_files += backport_package_files
 822
 823     if not args.git_revision:
 824         logwrite('Copy original source files ...')
 825     else:
 826         logwrite('Get original source files from git ...')
 827
 828     copy_files(os.path.join(source_dir, 'backport'), backport_files, bpid.target_dir)
 829
 830     git_debug_init(args)
 831
 832     if not args.git_revision:
 833         copy_files(args.kerneldir, copy_list, bpid.target_dir)
 834     else:
 835         copy_git_files(args.kerneldir, copy_list, args.git_revision, bpid.target_dir)
 836
 837     # FIXME: should we add a git version of this (e.g. --git-extra-driver)?
 838     for src, copy_list in args.extra_driver:
 839         if (args.kup or args.kup_test):
 840             git_paranoia(src)
 841         copy_files(src, read_copy_list(open(copy_list, 'r')), bpid.target_dir)
 842
 843     git_debug_snapshot(args, 'Add driver sources')
 844
 845     disable_list = add_automatic_backports(args)
 846     if git_tracked_version:
 847         backports_version = "(see git)"
 848         kernel_version = "(see git)"
 849     else:
 850         backports_version = git.describe(tree=source_dir, extra_args=['--long'])
 851         kernel_version = git.describe(rev=args.git_revision or 'HEAD',
 852                                       tree=args.kerneldir,
 853                                       extra_args=['--long'])
 854
 855     if not bpid.integrate:
 856         f = open(os.path.join(bpid.target_dir, 'versions'), 'w')
 857         f.write('BACKPORTS_VERSION="%s"\n' % backports_version)
 858         f.write('BACKPORTED_KERNEL_VERSION="%s"\n' % kernel_version)
 859         f.write('BACKPORTED_KERNEL_NAME="%s"\n' % args.base_name)
 860         if git_tracked_version:
 861             f.write('BACKPORTS_GIT_TRACKED="backport tracker ID: $(shell git rev-parse HEAD 2>/dev/null || echo \'not built in git tree\')"\n')
 862         f.close()
 863         git_debug_snapshot(args, "add versions files")
 864     else:
 865         kconf_regexes = [
 866                 (re.compile(r'.*(?P<key>%%BACKPORT_DIR%%)'), '%%BACKPORT_DIR%%', 'backports/'),
 867                 (re.compile(r'.*(?P<key>%%BACKPORTS_VERSION%%).*'), '%%BACKPORTS_VERSION%%', backports_version),
 868                 (re.compile(r'.*(?P<key>%%BACKPORTED_KERNEL_VERSION%%).*'), '%%BACKPORTED_KERNEL_VERSION%%', kernel_version),
 869                 (re.compile(r'.*(?P<key>%%BACKPORTED_KERNEL_NAME%%).*'), '%%BACKPORTED_KERNEL_NAME%%', args.base_name),
 870         ]
 871         out = ''
 872         for l in open(os.path.join(bpid.target_dir, 'Kconfig'), 'r'):
 873             for r in kconf_regexes:
 874                 m = r[0].match(l)
 875                 if m:
 876                     l = re.sub(r'(' + r[1] + ')', r'' + r[2] + '', l)
 877             out += l
 878         outf = open(os.path.join(bpid.target_dir, 'Kconfig'), 'w')
 879         outf.write(out)
 880         outf.close()
 881         git_debug_snapshot(args, "modify top level backports/Kconfig with backports identity")
 882
 883     if disable_list:
 884         # No need to verify_sources() as compat's Kconfig has no 'source' call
 885         bpcfg = kconfig.ConfigTree(os.path.join(bpid.target_dir, 'compat', 'Kconfig'), bpid)
 886         bpcfg.disable_symbols(disable_list)
 887     git_debug_snapshot(args, 'Add automatic backports')
 888
 889     apply_patches(args, "backport", source_dir, 'patches', bpid.target_dir, logwrite)
 890
 891     # some post-processing is required
 892     configtree = kconfig.ConfigTree(os.path.join(bpid.target_dir, 'Kconfig'), bpid)
 893     ignore=['Kconfig.kernel', 'Kconfig.versions']
 894
 895     configtree.verify_sources(ignore=ignore)
 896     git_debug_snapshot(args, "verify sources on top level backports Kconfig")
 897
 898     orig_symbols = configtree.symbols()
 899
 900     logwrite('Modify Kconfig tree ...')
 901     configtree.prune_sources(ignore=ignore)
 902     git_debug_snapshot(args, "prune Kconfig tree")
 903
 904     if not bpid.integrate:
 905         configtree.force_tristate_modular()
 906         git_debug_snapshot(args, "force tristate options modular")
 907
 908     configtree.modify_selects()
 909     git_debug_snapshot(args, "convert select to depends on")
 910
 911     symbols = configtree.symbols()
 912
 913     # write local symbol list -- needed during packaging build
 914     if not bpid.integrate:
 915         f = open(os.path.join(bpid.target_dir, '.local-symbols'), 'w')
 916         for sym in symbols:
 917             f.write('%s=\n' % sym)
 918         f.close()
 919         git_debug_snapshot(args, "add symbols files")
 920
 921     # add defconfigs that we want
 922     defconfigs_dir = os.path.join(source_dir, 'backport', 'defconfigs')
 923     os.mkdir(os.path.join(bpid.target_dir, 'defconfigs'))
 924     for dfbase in os.listdir(defconfigs_dir):
 925         copy_defconfig = True
 926         dfsrc = os.path.join(defconfigs_dir, dfbase)
 927         for line in open(dfsrc, 'r'):
 928             if not '=' in line:
 929                 continue
 930             line_ok = False
 931             for sym in symbols:
 932                 if sym + '=' in line:
 933                     line_ok = True
 934                     break
 935             if not line_ok:
 936                 copy_defconfig = False
 937                 break
 938         if copy_defconfig:
 939             shutil.copy(dfsrc, os.path.join(bpid.target_dir, 'defconfigs', dfbase))
 940
 941     git_debug_snapshot(args, "add (useful) defconfig files")
 942
 943     logwrite('Rewrite Makefiles and Kconfig files ...')
 944
 945     # rewrite Makefile and source symbols
 946
 947     regexes = []
 948     for some_symbols in [orig_symbols[i:i + 50] for i in range(0, len(orig_symbols), 50)]:
 949         r = 'CONFIG_((' + '|'.join([s + '(_MODULE)?' for s in some_symbols]) + ')([^A-Za-z0-9_]|$))'
 950         regexes.append(re.compile(r, re.MULTILINE))
 951     for root, dirs, files in os.walk(bpid.target_dir):
 952         # don't go into .git dir (possible debug thing)
 953         if '.git' in dirs:
 954             dirs.remove('.git')
 955         for f in files:
 956             data = open(os.path.join(root, f), 'r').read()
 957             for r in regexes:
 958                 data = r.sub(r'' + bpid.full_prefix + '\\1', data)
 959             data = re.sub(r'\$\(srctree\)', '$(backport_srctree)', data)
 960             data = re.sub(r'-Idrivers', '-I$(backport_srctree)/drivers', data)
 961             if bpid.integrate:
 962                 data = re.sub(r'CPTCFG_', bpid.full_prefix, data)
 963             fo = open(os.path.join(root, f), 'w')
 964             fo.write(data)
 965             fo.close()
 966
 967     git_debug_snapshot(args, "rename config symbol / srctree usage")
 968
 969     # disable unbuildable Kconfig symbols and stuff Makefiles that doesn't exist
 970     maketree = make.MakeTree(os.path.join(bpid.target_dir, 'Makefile.kernel'))
 971     disable_kconfig = []
 972     disable_makefile = []
 973     for sym in maketree.get_impossible_symbols():
 974         disable_kconfig.append(sym[7:])
 975         disable_makefile.append(sym[7:])
 976
 977     configtree.disable_symbols(disable_kconfig)
 978     git_debug_snapshot(args, "disable impossible kconfig symbols")
 979
 980     # add kernel version dependencies to Kconfig, from the dependency list
 981     # we read previously
 982     for sym in tuple(deplist.keys()):
 983         new = []
 984         for dep in deplist[sym]:
 985             if "kconfig:" in dep:
 986                     kconfig_expr = dep.replace('kconfig: ', '')
 987                     new.append(kconfig_expr)
 988             elif (dep == "DISABLE"):
 989                     new.append('BACKPORT_DISABLED_KCONFIG_OPTION')
 990             else:
 991                     new.append('!KERNEL_%s' % dep.replace('.', '_'))
 992         if bpid.integrate:
 993             deplist[sym] = ["BACKPORT_" + x for x in new]
 994         else:
 995             deplist[sym] = new
 996     configtree.add_dependencies(deplist)
 997     git_debug_snapshot(args, "add kernel version dependencies")
 998
 999     # disable things in makefiles that can't be selected and that the
1000     # build shouldn't recurse into because they don't exist -- if we
1001     # don't do that then a symbol from the kernel could cause the build
1002     # to attempt to recurse and fail
1003     #
1004     # Note that we split the regex after 50 symbols, this is because of a
1005     # limitation in the regex implementation (it only supports 100 nested
1006     # groups -- 50 seemed safer and is still fast)
1007     regexes = []
1008     for some_symbols in [disable_makefile[i:i + 50] for i in range(0, len(disable_makefile), 50)]:
1009         r = '^([^#].*((' + bpid.full_prefix_resafe + '|CONFIG_)(' + '|'.join([s for s in some_symbols]) + ')))'
1010         regexes.append(re.compile(r, re.MULTILINE))
1011     for f in maketree.get_makefiles():
1012         data = open(f, 'r').read()
1013         for r in regexes:
1014             data = r.sub(r'#\1', data)
1015         fo = open(f, 'w')
1016         fo.write(data)
1017         fo.close()
1018     git_debug_snapshot(args, "disable unsatisfied Makefile parts")
1019
1020     if (args.kup or args.kup_test):
1021         req = reqs.Req()
1022         req.kup()
1023         if not req.reqs_match():
1024             sys.exit(1)
1025         upload_release(args, rel_prep, logwrite=logwrite)
1026
1027     logwrite('Done!')
1028     return 0
1029
1030 if __name__ == '__main__':
1031     ret = _main()
1032     if ret:
1033         sys.exit(ret)