gentree.py

   1 #!/usr/bin/env python
   2 #
   3 # Generate the output tree, by default in the output/ directory
   4 # but a different one may be specified. The directory must be
   5 # empty already. It's also allowed to not exist.
   6 #
   7
   8 import argparse, sys, os, errno, shutil, re, subprocess
   9
  10 # find self
  11 source_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
  12 sys.path.append(os.path.join(source_dir, 'lib'))
  13 import kconfig, git, patch, make
  14
  15 def read_copy_list(kerneldir, copyfile):
  16     ret = []
  17     for item in copyfile:
  18         # remove leading/trailing whitespace
  19         item = item.strip()
  20         # comments
  21         if not item or item[0] == '#':
  22             continue
  23         if item[0] == '/':
  24             raise Exception("Input path '%s' is absolute path, this isn't allowed" % (item, ))
  25         if ' -> ' in item:
  26             srcitem, dstitem = item.split(' -> ')
  27             if (srcitem[-1] == '/') != (dstitem[-1] == '/'):
  28                 raise Exception("Cannot copy file/dir to dir/file")
  29         else:
  30             srcitem = dstitem = item
  31         # check for expected input
  32         src = os.path.join(kerneldir, srcitem)
  33         if item[-1] == '/':
  34             if not os.path.isdir(src):
  35                 raise Exception("Input path '%s' isn't a directory in '%s'" % (srcitem, kerneldir))
  36         else:
  37             if not os.path.isfile(src):
  38                 raise Exception("Input path '%s' isn't a file in '%s'" % (srcitem, kerneldir))
  39         ret.append((kerneldir, srcitem, dstitem))
  40     return ret
  41
  42 def check_output_dir(d, clean):
  43     if clean:
  44         shutil.rmtree(d, ignore_errors=True)
  45     try:
  46         os.rmdir(d)
  47     except OSError, e:
  48         if e.errno != errno.ENOENT:
  49             raise
  50
  51 def copytree(src, dst, symlinks=False, ignore=None):
  52     names = os.listdir(src)
  53     if ignore is not None:
  54         ignored_names = ignore(src, names)
  55     else:
  56         ignored_names = set()
  57
  58     if not os.path.isdir(dst):
  59         os.makedirs(dst)
  60     errors = []
  61     for name in names:
  62         if name in ignored_names:
  63             continue
  64         srcname = os.path.join(src, name)
  65         dstname = os.path.join(dst, name)
  66         try:
  67             if symlinks and os.path.islink(srcname):
  68                 linkto = os.readlink(srcname)
  69                 os.symlink(linkto, dstname)
  70             elif os.path.isdir(srcname):
  71                 copytree(srcname, dstname, symlinks, ignore)
  72             else:
  73                 shutil.copy2(srcname, dstname)
  74         except (IOError, os.error) as why:
  75             errors.append((srcname, dstname, str(why)))
  76         # catch the Error from the recursive copytree so that we can
  77         # continue with other files
  78         except shutil.Error as err:
  79             errors.extend(err.args[0])
  80     try:
  81         shutil.copystat(src, dst)
  82     except WindowsError:
  83         # can't copy file access times on Windows
  84         pass
  85     except OSError as why:
  86         errors.extend((src, dst, str(why)))
  87     if errors:
  88         raise shutil.Error(errors)
  89
  90 def copy_files(copy_list, outdir):
  91     for srcpath, srcitem, tgtitem in copy_list:
  92         if tgtitem == '':
  93             copytree(srcpath, outdir, ignore=shutil.ignore_patterns('*~'))
  94         elif tgtitem[-1] == '/':
  95             def copy_ignore(dir, entries):
  96                 r = []
  97                 for i in entries:
  98                     if (not i[-1] in ('c', 'h') and
  99                         i[-4:] != '.awk' and
 100                         not i in ('Kconfig', 'Makefile') and
 101                         not os.path.isdir(os.path.join(dir, i))):
 102                         r.append(i)
 103                 return r
 104             copytree(os.path.join(srcpath, srcitem),
 105                      os.path.join(outdir, tgtitem),
 106                      ignore=copy_ignore)
 107         else:
 108             try:
 109                 os.makedirs(os.path.join(outdir, os.path.dirname(tgtitem)))
 110             except OSError, e:
 111                 # ignore dirs we might have created just now
 112                 if e.errno != errno.EEXIST:
 113                     raise
 114             shutil.copy(os.path.join(srcpath, srcitem),
 115                         os.path.join(outdir, tgtitem))
 116
 117 def git_debug_init(args):
 118     if not args.gitdebug:
 119         return
 120     git.init(tree=args.outdir)
 121     git.commit_all("Copied code", tree=args.outdir)
 122
 123 def git_debug_snapshot(args, name):
 124     if not args.gitdebug:
 125         return
 126     git.commit_all(name, tree=args.outdir)
 127
 128 def main():
 129     # set up and parse arguments
 130     parser = argparse.ArgumentParser(description='generate backport tree')
 131     parser.add_argument('kerneldir', metavar='<kernel tree>', type=str,
 132                         help='Kernel tree to copy drivers from')
 133     parser.add_argument('outdir', metavar='<output directory>', type=str,
 134                         help='Directory to write the generated tree to')
 135     parser.add_argument('--copy-list', metavar='<listfile>', type=argparse.FileType('r'),
 136                         default='copy-list',
 137                         help='File containing list of files/directories to copy, default "copy-list"')
 138     parser.add_argument('--clean', const=True, default=False, action="store_const",
 139                         help='Clean output directory instead of erroring if it isn\'t empty')
 140     parser.add_argument('--refresh', const=True, default=False, action="store_const",
 141                         help='Refresh patches as they are applied, the source dir will be modified!')
 142     parser.add_argument('--base-name', metavar='<name>', type=str, default='Linux',
 143                         help='name of base tree, default just "Linux"')
 144     parser.add_argument('--gitdebug', const=True, default=False, action="store_const",
 145                         help='Use git, in the output tree, to debug the various transformation steps ' +
 146                              'that the tree generation makes (apply patches, ...)')
 147     args = parser.parse_args()
 148
 149     # first thing to copy is our own plumbing -- we start from that
 150     copy_list = [(os.path.join(source_dir, 'plumbing'), '', '')]
 151     # then add stuff from the copy list file
 152     copy_list.extend(read_copy_list(args.kerneldir, args.copy_list))
 153     # add compat to the list
 154     copy_list.append((os.path.join(source_dir, 'compat'), 'compat/', 'compat/'))
 155     copy_list.append((os.path.join(source_dir, 'compat'), 'include/', 'include/'))
 156
 157     # validate output directory
 158     check_output_dir(args.outdir, args.clean)
 159
 160     # do the copy
 161     copy_files(copy_list, args.outdir)
 162
 163     git_debug_init(args)
 164
 165     # some post-processing is required
 166     configtree = kconfig.ConfigTree(os.path.join(args.outdir, 'Kconfig'))
 167     configtree.prune_sources(ignore=['Kconfig.kernel', 'Kconfig.versions'])
 168     git_debug_snapshot(args, "prune Kconfig tree")
 169     configtree.force_tristate_modular()
 170     git_debug_snapshot(args, "force tristate options modular")
 171     configtree.modify_selects()
 172     git_debug_snapshot(args, "convert select to depends on")
 173
 174     # write the versioning file
 175     backports_version = git.describe(tree=source_dir)
 176     kernel_version = git.describe(tree=args.kerneldir)
 177     f = open(os.path.join(args.outdir, 'versions'), 'w')
 178     f.write('BACKPORTS_VERSION="%s"\n' % backports_version)
 179     f.write('KERNEL_VERSION="%s"\n' % kernel_version)
 180     f.write('KERNEL_NAME="%s"\n' % args.base_name)
 181     f.close()
 182
 183     symbols = configtree.symbols()
 184
 185     # write local symbol list
 186     f = open(os.path.join(args.outdir, '.local-symbols'), 'w')
 187     for sym in symbols:
 188         f.write('%s=\n' % sym)
 189     f.close()
 190
 191     git_debug_snapshot(args, "add versions/symbols files")
 192
 193     patchdirs = []
 194     for root, dirs, files in os.walk(os.path.join(source_dir, 'patches')):
 195         if not dirs:
 196             patchdirs.append(root)
 197     patchdirs.sort()
 198     for pdir in patchdirs:
 199         l = os.listdir(pdir)
 200         printed = False
 201         for pfile in l:
 202             if pfile[-1] == '~':
 203                 continue
 204             pfile = os.path.join(pdir, pfile)
 205             p = patch.fromfile(pfile)
 206             if not p:
 207                 continue
 208             patched_file = '/'.join(p.items[0].source.split('/')[1:])
 209             fullfn = os.path.join(args.outdir, patched_file)
 210             if not os.path.exists(fullfn):
 211                 continue
 212             if not printed:
 213                 print "Applying changes from", os.path.basename(pdir)
 214                 printed = True
 215             if args.refresh:
 216                 for patchitem in p.items:
 217                     patched_file = '/'.join(patchitem.source.split('/')[1:])
 218                     fullfn = os.path.join(args.outdir, patched_file)
 219                     shutil.copyfile(fullfn, fullfn + '.orig_file')
 220             process = subprocess.Popen(['patch', '-p1'], stdout=subprocess.PIPE,
 221                                        stderr=subprocess.STDOUT, stdin=subprocess.PIPE,
 222                                        close_fds=True, universal_newlines=True,
 223                                        cwd=args.outdir)
 224             output = process.communicate(input=open(pfile, 'r').read())[0]
 225             output = output.split('\n')
 226             if output[-1] == '':
 227                 output = output[:-1]
 228             for line in output:
 229                 print '>', line
 230             if process.returncode != 0:
 231                 print "Patch failed!"
 232                 sys.exit(2)
 233             if args.refresh:
 234                 pfilef = open(pfile + '.tmp', 'w')
 235                 for patchitem in p.items:
 236                     patched_file = '/'.join(patchitem.source.split('/')[1:])
 237                     fullfn = os.path.join(args.outdir, patched_file)
 238                     process = subprocess.Popen(['diff', '-u', patched_file + '.orig_file', patched_file,
 239                                                 '--label', 'a/' + patched_file,
 240                                                 '--label', 'b/' + patched_file],
 241                                                stdout=pfilef, close_fds=True,
 242                                                universal_newlines=True, cwd=args.outdir)
 243                     process.wait()
 244                     os.unlink(fullfn + '.orig_file')
 245                     if not process.returncode in (0, 1):
 246                         print "Diffing for refresh failed!"
 247                         pfilef.close()
 248                         os.unlink(pfile + '.tmp')
 249                         sys.exit(3)
 250                 pfilef.close()
 251                 os.rename(pfile + '.tmp', pfile)
 252         # remove orig/rej files that patch sometimes creates
 253         for root, dirs, files in os.walk(args.outdir):
 254             for f in files:
 255                 if f[-5:] == '.orig' or f[-4:] == '.rej':
 256                     os.unlink(os.path.join(root, f))
 257         if not printed:
 258             print "Not applying changes from %s, not needed" % (os.path.basename(pdir),)
 259         else:
 260             git_debug_snapshot(args, "apply backport patches from %s" % (os.path.basename(pdir),))
 261
 262
 263     # rewrite Makefile and source symbols
 264     regexes = []
 265     for some_symbols in [symbols[i:i + 50] for i in range(0, len(symbols), 50)]:
 266         r = 'CONFIG_((' + '|'.join([s + '(_MODULE)?' for s in some_symbols]) + ')([^A-Za-z0-9_]|$))'
 267         regexes.append(re.compile(r, re.MULTILINE))
 268     for root, dirs, files in os.walk(args.outdir):
 269         # don't go into .git dir (possible debug thing)
 270         if '.git' in dirs:
 271             dirs.remove('.git')
 272         for f in files:
 273             data = open(os.path.join(root, f), 'r').read()
 274             for r in regexes:
 275                 data = r.sub(r'CPTCFG_\1', data)
 276             fo = open(os.path.join(root, f), 'w')
 277             fo.write(data)
 278             fo.close()
 279
 280     git_debug_snapshot(args, "rename config symbol usage")
 281
 282     # disable unbuildable Kconfig symbols and stuff Makefiles that doesn't exist
 283     maketree = make.MakeTree(os.path.join(args.outdir, 'Makefile.kernel'))
 284     disable_kconfig = []
 285     disable_makefile = []
 286     for sym in maketree.get_impossible_symbols():
 287         if sym[:7] == 'CPTCFG_':
 288             disable_kconfig.append(sym[7:])
 289         else:
 290             disable_makefile.append(sym[7:])
 291
 292     configtree.disable_symbols(disable_kconfig)
 293     git_debug_snapshot(args, "disable impossible kconfig symbols")
 294
 295     regexes = []
 296     for some_symbols in [disable_makefile[i:i + 50] for i in range(0, len(disable_makefile), 50)]:
 297         r = '(CONFIG_(' + '|'.join([s for s in some_symbols]) + '))'
 298         regexes.append(re.compile(r, re.MULTILINE))
 299     for f in maketree.get_makefiles():
 300         data = open(f, 'r').read()
 301         for r in regexes:
 302             data = r.sub(r'IMPOSSIBLE_\2', data)
 303         fo = open(f, 'w')
 304         fo.write(data)
 305         fo.close()
 306     git_debug_snapshot(args, "disable unsatisfied Makefile parts")
 307
 308 main()