gentree.py

   1 #!/usr/bin/env python
   2 #
   3 # Generate the output tree, by default in the output/ directory
   4 # but a different one may be specified. The directory must be
   5 # empty already. It's also allowed to not exist.
   6 #
   7
   8 import argparse, sys, os, errno, shutil, re, subprocess
   9
  10 # find self
  11 source_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
  12 sys.path.append(os.path.join(source_dir, 'lib'))
  13 import kconfig, git, patch, make
  14
  15 def read_copy_list(kerneldir, copyfile):
  16     ret = []
  17     for item in copyfile:
  18         # remove leading/trailing whitespace
  19         item = item.strip()
  20         # comments
  21         if not item or item[0] == '#':
  22             continue
  23         if item[0] == '/':
  24             raise Exception("Input path '%s' is absolute path, this isn't allowed" % (item, ))
  25         if ' -> ' in item:
  26             srcitem, dstitem = item.split(' -> ')
  27             if (srcitem[-1] == '/') != (dstitem[-1] == '/'):
  28                 raise Exception("Cannot copy file/dir to dir/file")
  29         else:
  30             srcitem = dstitem = item
  31         # check for expected input
  32         src = os.path.join(kerneldir, srcitem)
  33         if item[-1] == '/':
  34             if not os.path.isdir(src):
  35                 raise Exception("Input path '%s' isn't a directory in '%s'" % (srcitem, kerneldir))
  36         else:
  37             if not os.path.isfile(src):
  38                 raise Exception("Input path '%s' isn't a file in '%s'" % (srcitem, kerneldir))
  39         ret.append((kerneldir, srcitem, dstitem))
  40     return ret
  41
  42 def check_output_dir(d, clean):
  43     if clean:
  44         shutil.rmtree(d, ignore_errors=True)
  45     try:
  46         os.rmdir(d)
  47     except OSError, e:
  48         if e.errno != errno.ENOENT:
  49             raise
  50
  51 def copytree(src, dst, symlinks=False, ignore=None):
  52     names = os.listdir(src)
  53     if ignore is not None:
  54         ignored_names = ignore(src, names)
  55     else:
  56         ignored_names = set()
  57
  58     if not os.path.isdir(dst):
  59         os.makedirs(dst)
  60     errors = []
  61     for name in names:
  62         if name in ignored_names:
  63             continue
  64         srcname = os.path.join(src, name)
  65         dstname = os.path.join(dst, name)
  66         try:
  67             if symlinks and os.path.islink(srcname):
  68                 linkto = os.readlink(srcname)
  69                 os.symlink(linkto, dstname)
  70             elif os.path.isdir(srcname):
  71                 copytree(srcname, dstname, symlinks, ignore)
  72             else:
  73                 shutil.copy2(srcname, dstname)
  74         except (IOError, os.error) as why:
  75             errors.append((srcname, dstname, str(why)))
  76         # catch the Error from the recursive copytree so that we can
  77         # continue with other files
  78         except shutil.Error as err:
  79             errors.extend(err.args[0])
  80     try:
  81         shutil.copystat(src, dst)
  82     except WindowsError:
  83         # can't copy file access times on Windows
  84         pass
  85     except OSError as why:
  86         errors.extend((src, dst, str(why)))
  87     if errors:
  88         raise shutil.Error(errors)
  89
  90 def copy_files(copy_list, outdir):
  91     for srcpath, srcitem, tgtitem in copy_list:
  92         if tgtitem == '':
  93             copytree(srcpath, outdir, ignore=shutil.ignore_patterns('*~'))
  94         elif tgtitem[-1] == '/':
  95             def copy_ignore(dir, entries):
  96                 r = []
  97                 for i in entries:
  98                     if (not i[-1] in ('c', 'h') and
  99                         i[-4:] != '.awk' and
 100                         not i in ('Kconfig', 'Makefile') and
 101                         not os.path.isdir(os.path.join(dir, i))):
 102                         r.append(i)
 103                 return r
 104             copytree(os.path.join(srcpath, srcitem),
 105                      os.path.join(outdir, tgtitem),
 106                      ignore=copy_ignore)
 107         else:
 108             try:
 109                 os.makedirs(os.path.join(outdir, os.path.dirname(tgtitem)))
 110             except OSError, e:
 111                 # ignore dirs we might have created just now
 112                 if e.errno != errno.EEXIST:
 113                     raise
 114             shutil.copy(os.path.join(srcpath, srcitem),
 115                         os.path.join(outdir, tgtitem))
 116
 117 def git_debug_init(args):
 118     if not args.gitdebug:
 119         return
 120     git.init(tree=args.outdir)
 121     git.commit_all("Copied code", tree=args.outdir)
 122
 123 def git_debug_snapshot(args, name):
 124     if not args.gitdebug:
 125         return
 126     git.commit_all(name, tree=args.outdir)
 127
 128 def main():
 129     # set up and parse arguments
 130     parser = argparse.ArgumentParser(description='generate backport tree')
 131     parser.add_argument('kerneldir', metavar='<kernel tree>', type=str,
 132                         help='Kernel tree to copy drivers from')
 133     parser.add_argument('outdir', metavar='<output directory>', type=str,
 134                         help='Directory to write the generated tree to')
 135     parser.add_argument('--copy-list', metavar='<listfile>', type=argparse.FileType('r'),
 136                         default='copy-list',
 137                         help='File containing list of files/directories to copy, default "copy-list"')
 138     parser.add_argument('--clean', const=True, default=False, action="store_const",
 139                         help='Clean output directory instead of erroring if it isn\'t empty')
 140     parser.add_argument('--refresh', const=True, default=False, action="store_const",
 141                         help='Refresh patches as they are applied, the source dir will be modified!')
 142     parser.add_argument('--base-name', metavar='<name>', type=str, default='Linux',
 143                         help='name of base tree, default just "Linux"')
 144     parser.add_argument('--gitdebug', const=True, default=False, action="store_const",
 145                         help='Use git, in the output tree, to debug the various transformation steps ' +
 146                              'that the tree generation makes (apply patches, ...)')
 147     parser.add_argument('--verbose', const=True, default=False, action="store_const",
 148                         help='Print more verbose information')
 149     args = parser.parse_args()
 150
 151     # first thing to copy is our own plumbing -- we start from that
 152     copy_list = [(os.path.join(source_dir, 'plumbing'), '', '')]
 153     # then add stuff from the copy list file
 154     copy_list.extend(read_copy_list(args.kerneldir, args.copy_list))
 155     # add compat to the list
 156     copy_list.append((os.path.join(source_dir, 'compat'), 'compat/', 'compat/'))
 157     copy_list.append((os.path.join(source_dir, 'compat'), 'include/', 'include/'))
 158
 159     # validate output directory
 160     check_output_dir(args.outdir, args.clean)
 161
 162     # do the copy
 163     copy_files(copy_list, args.outdir)
 164
 165     git_debug_init(args)
 166
 167     # some post-processing is required
 168     configtree = kconfig.ConfigTree(os.path.join(args.outdir, 'Kconfig'))
 169     configtree.prune_sources(ignore=['Kconfig.kernel', 'Kconfig.versions'])
 170     git_debug_snapshot(args, "prune Kconfig tree")
 171     configtree.force_tristate_modular()
 172     git_debug_snapshot(args, "force tristate options modular")
 173     configtree.modify_selects()
 174     git_debug_snapshot(args, "convert select to depends on")
 175
 176     # write the versioning file
 177     backports_version = git.describe(tree=source_dir)
 178     kernel_version = git.describe(tree=args.kerneldir)
 179     f = open(os.path.join(args.outdir, 'versions'), 'w')
 180     f.write('BACKPORTS_VERSION="%s"\n' % backports_version)
 181     f.write('KERNEL_VERSION="%s"\n' % kernel_version)
 182     f.write('KERNEL_NAME="%s"\n' % args.base_name)
 183     f.close()
 184
 185     symbols = configtree.symbols()
 186
 187     # write local symbol list
 188     f = open(os.path.join(args.outdir, '.local-symbols'), 'w')
 189     for sym in symbols:
 190         f.write('%s=\n' % sym)
 191     f.close()
 192
 193     git_debug_snapshot(args, "add versions/symbols files")
 194
 195     patchdirs = []
 196     for root, dirs, files in os.walk(os.path.join(source_dir, 'patches')):
 197         if not dirs:
 198             patchdirs.append(root)
 199     patchdirs.sort()
 200     for pdir in patchdirs:
 201         l = os.listdir(pdir)
 202         printed = False
 203         for pfile in l:
 204             if pfile[-1] == '~':
 205                 continue
 206             pfile = os.path.join(pdir, pfile)
 207             p = patch.fromfile(pfile)
 208             if not p:
 209                 continue
 210             patched_file = '/'.join(p.items[0].source.split('/')[1:])
 211             fullfn = os.path.join(args.outdir, patched_file)
 212             if not os.path.exists(fullfn):
 213                 continue
 214             if not printed:
 215                 if args.verbose:
 216                     print "Applying changes from", os.path.basename(pdir)
 217                 printed = True
 218             if args.refresh:
 219                 for patchitem in p.items:
 220                     patched_file = '/'.join(patchitem.source.split('/')[1:])
 221                     fullfn = os.path.join(args.outdir, patched_file)
 222                     shutil.copyfile(fullfn, fullfn + '.orig_file')
 223             process = subprocess.Popen(['patch', '-p1'], stdout=subprocess.PIPE,
 224                                        stderr=subprocess.STDOUT, stdin=subprocess.PIPE,
 225                                        close_fds=True, universal_newlines=True,
 226                                        cwd=args.outdir)
 227             output = process.communicate(input=open(pfile, 'r').read())[0]
 228             output = output.split('\n')
 229             if output[-1] == '':
 230                 output = output[:-1]
 231             if args.verbose:
 232                 for line in output:
 233                     print '>', line
 234             if process.returncode != 0:
 235                 if not args.verbose:
 236                     print "Failed to apply changes from", os.path.basename(pdir)
 237                     for line in output:
 238                         print '>', line
 239                 sys.exit(2)
 240             if args.refresh:
 241                 pfilef = open(pfile + '.tmp', 'w')
 242                 for patchitem in p.items:
 243                     patched_file = '/'.join(patchitem.source.split('/')[1:])
 244                     fullfn = os.path.join(args.outdir, patched_file)
 245                     process = subprocess.Popen(['diff', '-u', patched_file + '.orig_file', patched_file,
 246                                                 '--label', 'a/' + patched_file,
 247                                                 '--label', 'b/' + patched_file],
 248                                                stdout=pfilef, close_fds=True,
 249                                                universal_newlines=True, cwd=args.outdir)
 250                     process.wait()
 251                     os.unlink(fullfn + '.orig_file')
 252                     if not process.returncode in (0, 1):
 253                         print "Diffing for refresh failed!"
 254                         pfilef.close()
 255                         os.unlink(pfile + '.tmp')
 256                         sys.exit(3)
 257                 pfilef.close()
 258                 os.rename(pfile + '.tmp', pfile)
 259         # remove orig/rej files that patch sometimes creates
 260         for root, dirs, files in os.walk(args.outdir):
 261             for f in files:
 262                 if f[-5:] == '.orig' or f[-4:] == '.rej':
 263                     os.unlink(os.path.join(root, f))
 264         if not printed:
 265             if args.verbose:
 266                 print "Not applying changes from %s, not needed" % (os.path.basename(pdir),)
 267         else:
 268             git_debug_snapshot(args, "apply backport patches from %s" % (os.path.basename(pdir),))
 269
 270
 271     # rewrite Makefile and source symbols
 272     regexes = []
 273     for some_symbols in [symbols[i:i + 50] for i in range(0, len(symbols), 50)]:
 274         r = 'CONFIG_((' + '|'.join([s + '(_MODULE)?' for s in some_symbols]) + ')([^A-Za-z0-9_]|$))'
 275         regexes.append(re.compile(r, re.MULTILINE))
 276     for root, dirs, files in os.walk(args.outdir):
 277         # don't go into .git dir (possible debug thing)
 278         if '.git' in dirs:
 279             dirs.remove('.git')
 280         for f in files:
 281             data = open(os.path.join(root, f), 'r').read()
 282             for r in regexes:
 283                 data = r.sub(r'CPTCFG_\1', data)
 284             fo = open(os.path.join(root, f), 'w')
 285             fo.write(data)
 286             fo.close()
 287
 288     git_debug_snapshot(args, "rename config symbol usage")
 289
 290     # disable unbuildable Kconfig symbols and stuff Makefiles that doesn't exist
 291     maketree = make.MakeTree(os.path.join(args.outdir, 'Makefile.kernel'))
 292     disable_kconfig = []
 293     disable_makefile = []
 294     for sym in maketree.get_impossible_symbols():
 295         if sym[:7] == 'CPTCFG_':
 296             disable_kconfig.append(sym[7:])
 297         else:
 298             disable_makefile.append(sym[7:])
 299
 300     configtree.disable_symbols(disable_kconfig)
 301     git_debug_snapshot(args, "disable impossible kconfig symbols")
 302
 303     regexes = []
 304     for some_symbols in [disable_makefile[i:i + 50] for i in range(0, len(disable_makefile), 50)]:
 305         r = '(CONFIG_(' + '|'.join([s for s in some_symbols]) + '))'
 306         regexes.append(re.compile(r, re.MULTILINE))
 307     for f in maketree.get_makefiles():
 308         data = open(f, 'r').read()
 309         for r in regexes:
 310             data = r.sub(r'IMPOSSIBLE_\2', data)
 311         fo = open(f, 'w')
 312         fo.write(data)
 313         fo.close()
 314     git_debug_snapshot(args, "disable unsatisfied Makefile parts")
 315
 316 main()