gentree.py

   1 #!/usr/bin/env python
   2 #
   3 # Generate the output tree, by default in the output/ directory
   4 # but a different one may be specified. The directory must be
   5 # empty already. It's also allowed to not exist.
   6 #
   7
   8 import argparse, sys, os, errno, shutil, re, subprocess
   9
  10 # find self
  11 source_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
  12 sys.path.append(os.path.join(source_dir, 'lib'))
  13 import kconfig, git, patch, make
  14
  15 def read_copy_list(copyfile):
  16     ret = []
  17     for item in copyfile:
  18         # remove leading/trailing whitespace
  19         item = item.strip()
  20         # comments
  21         if not item or item[0] == '#':
  22             continue
  23         if item[0] == '/':
  24             raise Exception("Input path '%s' is absolute path, this isn't allowed" % (item, ))
  25         if ' -> ' in item:
  26             srcitem, dstitem = item.split(' -> ')
  27             if (srcitem[-1] == '/') != (dstitem[-1] == '/'):
  28                 raise Exception("Cannot copy file/dir to dir/file")
  29         else:
  30             srcitem = dstitem = item
  31         ret.append((srcitem, dstitem))
  32     return ret
  33
  34 def read_dependencies(depfilename):
  35     ret = {}
  36     depfile = open(depfilename)
  37     for item in depfile:
  38         item = item.strip()
  39         if not item or item[0] == '#':
  40             continue
  41         sym, dep = item.split()
  42         if not sym in ret:
  43             ret[sym] = [dep, ]
  44         else:
  45             ret[sym].append(dep)
  46     return ret
  47
  48 def check_output_dir(d, clean):
  49     if clean:
  50         shutil.rmtree(d, ignore_errors=True)
  51     try:
  52         os.rmdir(d)
  53     except OSError, e:
  54         if e.errno != errno.ENOENT:
  55             raise
  56
  57 def copytree(src, dst, symlinks=False, ignore=None):
  58     names = os.listdir(src)
  59     if ignore is not None:
  60         ignored_names = ignore(src, names)
  61     else:
  62         ignored_names = set()
  63
  64     if not os.path.isdir(dst):
  65         os.makedirs(dst)
  66     errors = []
  67     for name in names:
  68         if name in ignored_names:
  69             continue
  70         srcname = os.path.join(src, name)
  71         dstname = os.path.join(dst, name)
  72         try:
  73             if symlinks and os.path.islink(srcname):
  74                 linkto = os.readlink(srcname)
  75                 os.symlink(linkto, dstname)
  76             elif os.path.isdir(srcname):
  77                 copytree(srcname, dstname, symlinks, ignore)
  78             else:
  79                 shutil.copy2(srcname, dstname)
  80         except (IOError, os.error) as why:
  81             errors.append((srcname, dstname, str(why)))
  82         # catch the Error from the recursive copytree so that we can
  83         # continue with other files
  84         except shutil.Error as err:
  85             errors.extend(err.args[0])
  86     try:
  87         shutil.copystat(src, dst)
  88     except WindowsError:
  89         # can't copy file access times on Windows
  90         pass
  91     except OSError as why:
  92         errors.extend((src, dst, str(why)))
  93     if errors:
  94         raise shutil.Error(errors)
  95
  96 def copy_files(srcpath, copy_list, outdir):
  97     for srcitem, tgtitem in copy_list:
  98         if tgtitem == '':
  99             copytree(srcpath, outdir, ignore=shutil.ignore_patterns('*~'))
 100         elif tgtitem[-1] == '/':
 101             def copy_ignore(dir, entries):
 102                 r = []
 103                 for i in entries:
 104                     if (not i[-1] in ('c', 'h') and
 105                         i[-4:] != '.awk' and
 106                         not i in ('Kconfig', 'Makefile') and
 107                         not os.path.isdir(os.path.join(dir, i))):
 108                         r.append(i)
 109                 return r
 110             copytree(os.path.join(srcpath, srcitem),
 111                      os.path.join(outdir, tgtitem),
 112                      ignore=copy_ignore)
 113         else:
 114             try:
 115                 os.makedirs(os.path.join(outdir, os.path.dirname(tgtitem)))
 116             except OSError, e:
 117                 # ignore dirs we might have created just now
 118                 if e.errno != errno.EEXIST:
 119                     raise
 120             shutil.copy(os.path.join(srcpath, srcitem),
 121                         os.path.join(outdir, tgtitem))
 122
 123 def copy_git_files(srcpath, copy_list, rev, outdir):
 124     for srcitem, tgtitem in copy_list:
 125         for m, t, h, f in git.ls_tree(rev=rev, files=(srcitem,), tree=srcpath):
 126             assert t == 'blob'
 127             f = os.path.join(outdir, f)
 128             d = os.path.dirname(f)
 129             if not os.path.exists(d):
 130                 os.makedirs(d)
 131             outf = open(f, 'w')
 132             git.get_blob(h, outf, tree=srcpath)
 133             outf.close()
 134             os.chmod(f, int(m, 8))
 135
 136 def git_debug_init(args):
 137     if not args.gitdebug:
 138         return
 139     git.init(tree=args.outdir)
 140     git.commit_all("Copied code", tree=args.outdir)
 141
 142 def git_debug_snapshot(args, name):
 143     if not args.gitdebug:
 144         return
 145     git.commit_all(name, tree=args.outdir)
 146
 147 def main():
 148     # set up and parse arguments
 149     parser = argparse.ArgumentParser(description='generate backport tree')
 150     parser.add_argument('kerneldir', metavar='<kernel tree>', type=str,
 151                         help='Kernel tree to copy drivers from')
 152     parser.add_argument('outdir', metavar='<output directory>', type=str,
 153                         help='Directory to write the generated tree to')
 154     parser.add_argument('--copy-list', metavar='<listfile>', type=argparse.FileType('r'),
 155                         default='copy-list',
 156                         help='File containing list of files/directories to copy, default "copy-list"')
 157     parser.add_argument('--git-revision', metavar='<revision>', type=str,
 158                         help='git commit revision (see gitrevisions(7)) to take objects from.' +
 159                              'If this is specified, the kernel tree is used as git object storage ' +
 160                              'and we use git ls-tree to get the files.')
 161     parser.add_argument('--clean', const=True, default=False, action="store_const",
 162                         help='Clean output directory instead of erroring if it isn\'t empty')
 163     parser.add_argument('--refresh', const=True, default=False, action="store_const",
 164                         help='Refresh patches as they are applied, the source dir will be modified!')
 165     parser.add_argument('--base-name', metavar='<name>', type=str, default='Linux',
 166                         help='name of base tree, default just "Linux"')
 167     parser.add_argument('--gitdebug', const=True, default=False, action="store_const",
 168                         help='Use git, in the output tree, to debug the various transformation steps ' +
 169                              'that the tree generation makes (apply patches, ...)')
 170     parser.add_argument('--verbose', const=True, default=False, action="store_const",
 171                         help='Print more verbose information')
 172     parser.add_argument('--extra-driver', nargs=2, metavar=('<source dir>', '<copy-list>'), type=str,
 173                         action='append', default=[], help='Extra driver directory/copy-list.')
 174     args = parser.parse_args()
 175
 176     # then add stuff from the copy list file
 177     copy_list = read_copy_list(args.copy_list)
 178
 179     deplist = read_dependencies(os.path.join(source_dir, 'dependencies'))
 180
 181     # validate output directory
 182     check_output_dir(args.outdir, args.clean)
 183
 184     # do the copy
 185     if not args.git_revision:
 186         print 'Copy original source files ...'
 187         copy_files(os.path.join(source_dir, 'backport'), [('', '')], args.outdir)
 188         copy_files(args.kerneldir, copy_list, args.outdir)
 189     else:
 190         print 'Get original source files from git ...'
 191         copy_files(os.path.join(source_dir, 'backport'), [('', '')], args.outdir)
 192         copy_git_files(args.kerneldir, copy_list, args.git_revision, args.outdir)
 193
 194     for src, copy_list in args.extra_driver:
 195         copy_files(src, read_copy_list(open(copy_list, 'r')), args.outdir)
 196
 197     git_debug_init(args)
 198
 199     git_debug_snapshot(args, "add versions/symbols files")
 200
 201     print 'Apply patches ...'
 202     patchdirs = []
 203     for root, dirs, files in os.walk(os.path.join(source_dir, 'patches')):
 204         if not dirs:
 205             patchdirs.append(root)
 206     patchdirs.sort()
 207     for pdir in patchdirs:
 208         l = os.listdir(pdir)
 209         printed = False
 210         for pfile in l:
 211             if pfile[-1] == '~':
 212                 continue
 213             pfile = os.path.join(pdir, pfile)
 214             p = patch.fromfile(pfile)
 215             if not p:
 216                 continue
 217             patched_file = '/'.join(p.items[0].source.split('/')[1:])
 218             fullfn = os.path.join(args.outdir, patched_file)
 219             if not os.path.exists(fullfn):
 220                 continue
 221             if not printed:
 222                 if args.verbose:
 223                     print "Applying changes from", os.path.basename(pdir)
 224                 printed = True
 225             if args.refresh:
 226                 for patchitem in p.items:
 227                     patched_file = '/'.join(patchitem.source.split('/')[1:])
 228                     fullfn = os.path.join(args.outdir, patched_file)
 229                     shutil.copyfile(fullfn, fullfn + '.orig_file')
 230             process = subprocess.Popen(['patch', '-p1'], stdout=subprocess.PIPE,
 231                                        stderr=subprocess.STDOUT, stdin=subprocess.PIPE,
 232                                        close_fds=True, universal_newlines=True,
 233                                        cwd=args.outdir)
 234             output = process.communicate(input=open(pfile, 'r').read())[0]
 235             output = output.split('\n')
 236             if output[-1] == '':
 237                 output = output[:-1]
 238             if args.verbose:
 239                 for line in output:
 240                     print '>', line
 241             if process.returncode != 0:
 242                 if not args.verbose:
 243                     print "Failed to apply changes from", os.path.basename(pdir)
 244                     for line in output:
 245                         print '>', line
 246                 sys.exit(2)
 247             if args.refresh:
 248                 pfilef = open(pfile + '.tmp', 'w')
 249                 for patchitem in p.items:
 250                     patched_file = '/'.join(patchitem.source.split('/')[1:])
 251                     fullfn = os.path.join(args.outdir, patched_file)
 252                     process = subprocess.Popen(['diff', '-u', patched_file + '.orig_file', patched_file,
 253                                                 '--label', 'a/' + patched_file,
 254                                                 '--label', 'b/' + patched_file],
 255                                                stdout=pfilef, close_fds=True,
 256                                                universal_newlines=True, cwd=args.outdir)
 257                     process.wait()
 258                     os.unlink(fullfn + '.orig_file')
 259                     if not process.returncode in (0, 1):
 260                         print "Diffing for refresh failed!"
 261                         pfilef.close()
 262                         os.unlink(pfile + '.tmp')
 263                         sys.exit(3)
 264                 pfilef.close()
 265                 os.rename(pfile + '.tmp', pfile)
 266         # remove orig/rej files that patch sometimes creates
 267         for root, dirs, files in os.walk(args.outdir):
 268             for f in files:
 269                 if f[-5:] == '.orig' or f[-4:] == '.rej':
 270                     os.unlink(os.path.join(root, f))
 271         if not printed:
 272             if args.verbose:
 273                 print "Not applying changes from %s, not needed" % (os.path.basename(pdir),)
 274         else:
 275             git_debug_snapshot(args, "apply backport patches from %s" % (os.path.basename(pdir),))
 276
 277     # some post-processing is required
 278     configtree = kconfig.ConfigTree(os.path.join(args.outdir, 'Kconfig'))
 279     print 'Modify Kconfig tree ...'
 280     configtree.prune_sources(ignore=['Kconfig.kernel', 'Kconfig.versions'])
 281     git_debug_snapshot(args, "prune Kconfig tree")
 282     configtree.force_tristate_modular()
 283     git_debug_snapshot(args, "force tristate options modular")
 284     configtree.modify_selects()
 285     git_debug_snapshot(args, "convert select to depends on")
 286
 287     # write the versioning file
 288     backports_version = git.describe(tree=source_dir)
 289     kernel_version = git.describe(tree=args.kerneldir)
 290     f = open(os.path.join(args.outdir, 'versions'), 'w')
 291     f.write('BACKPORTS_VERSION="%s"\n' % backports_version)
 292     f.write('KERNEL_VERSION="%s"\n' % kernel_version)
 293     f.write('KERNEL_NAME="%s"\n' % args.base_name)
 294     f.close()
 295
 296     symbols = configtree.symbols()
 297
 298     # write local symbol list
 299     f = open(os.path.join(args.outdir, '.local-symbols'), 'w')
 300     for sym in symbols:
 301         f.write('%s=\n' % sym)
 302     f.close()
 303
 304     print 'Rewrite Makefiles and Kconfig files ...'
 305
 306     # rewrite Makefile and source symbols
 307     regexes = []
 308     for some_symbols in [symbols[i:i + 50] for i in range(0, len(symbols), 50)]:
 309         r = 'CONFIG_((' + '|'.join([s + '(_MODULE)?' for s in some_symbols]) + ')([^A-Za-z0-9_]|$))'
 310         regexes.append(re.compile(r, re.MULTILINE))
 311     for root, dirs, files in os.walk(args.outdir):
 312         # don't go into .git dir (possible debug thing)
 313         if '.git' in dirs:
 314             dirs.remove('.git')
 315         for f in files:
 316             data = open(os.path.join(root, f), 'r').read()
 317             for r in regexes:
 318                 data = r.sub(r'CPTCFG_\1', data)
 319             fo = open(os.path.join(root, f), 'w')
 320             fo.write(data)
 321             fo.close()
 322
 323     git_debug_snapshot(args, "rename config symbol usage")
 324
 325     # disable unbuildable Kconfig symbols and stuff Makefiles that doesn't exist
 326     maketree = make.MakeTree(os.path.join(args.outdir, 'Makefile.kernel'))
 327     disable_kconfig = []
 328     disable_makefile = []
 329     for sym in maketree.get_impossible_symbols():
 330         if sym[:7] == 'CPTCFG_':
 331             disable_kconfig.append(sym[7:])
 332         else:
 333             disable_makefile.append(sym[7:])
 334
 335     configtree.disable_symbols(disable_kconfig)
 336     git_debug_snapshot(args, "disable impossible kconfig symbols")
 337
 338     for sym in tuple(deplist.keys()):
 339         new = []
 340         for dep in deplist[sym]:
 341             new.append('!BACKPORT_KERNEL_%s' % dep.replace('.', '_'))
 342         deplist[sym] = new
 343     configtree.add_dependencies(deplist)
 344     git_debug_snapshot(args, "add kernel version dependencies")
 345
 346     regexes = []
 347     for some_symbols in [disable_makefile[i:i + 50] for i in range(0, len(disable_makefile), 50)]:
 348         r = '(CONFIG_(' + '|'.join([s for s in some_symbols]) + '))'
 349         regexes.append(re.compile(r, re.MULTILINE))
 350     for f in maketree.get_makefiles():
 351         data = open(f, 'r').read()
 352         for r in regexes:
 353             data = r.sub(r'IMPOSSIBLE_\2', data)
 354         fo = open(f, 'w')
 355         fo.write(data)
 356         fo.close()
 357     git_debug_snapshot(args, "disable unsatisfied Makefile parts")
 358
 359     print 'Done!'
 360
 361 main()