#
import argparse, sys, os, errno, shutil, re, subprocess
+import tarfile, gzip
# find self
source_dir = os.path.abspath(os.path.dirname(__file__))
# and import libraries we have
from lib import kconfig, patch, make
from lib import bpgit as git
+from lib import bpgpg as gpg
+from lib import bpkup as kup
+from lib.tempdir import tempdir
+from lib import bpreqs as reqs
+from lib import bpversion as gen_version
+
+class Bp_Identity(object):
+ """
+ folks considering multiple integrations may want to
+ consider stuffing versioning info here as well but
+ that will need thought/design on sharing compat and
+ module namespaces.
+
+ Use the *_resafe when combining on regexps, although we currently
+ don't support regexps there perhaps later we will and this will
+ just make things safer for the output regardless. Once and if those
+ are added, how we actually use the others for regular printing will
+ need to be considered.
+ """
+ def __init__(self, integrate=False, kconfig_prefix='CPTCFG_',
+ project_prefix='', project_dir='',
+ target_dir='', target_dir_name='',
+ kconfig_source_var=None):
+ self.integrate = integrate
+ self.kconfig_prefix = kconfig_prefix
+ self.kconfig_prefix_resafe = re.escape(kconfig_prefix)
+ self.project_prefix = project_prefix
+ self.project_prefix_resafe = re.escape(project_prefix)
+ self.full_prefix = kconfig_prefix + project_prefix
+ self.full_prefix_resafe = re.escape(self.full_prefix)
+ self.project_dir = project_dir
+ self.target_dir = target_dir
+ self.target_dir_name = target_dir_name
+ self.kconfig_source_var = kconfig_source_var
+ if self.kconfig_source_var:
+ self.kconfig_source_var_resafe = re.escape(self.kconfig_source_var)
+ else:
+ self.kconfig_source_var_resafe = None
def read_copy_list(copyfile):
"""
"""
Read a (the) dependency file and return the list of
dependencies as a dictionary, mapping a Kconfig symbol
- to a list of kernel version dependencies. While reading
- ignore blank/commented lines.
+ to a list of kernel version dependencies.
+
+ If a backported feature that an upstream backported driver
+ depends on had kconfig limitations (ie, debugging feature not
+ available) a built constaint restriction can be expressed
+ by using a kconfig expression. The kconfig expressions can
+ be specified by using the "kconfig: " prefix.
+
+ While reading ignore blank or commented lines.
"""
ret = {}
depfile = open(depfilename, 'r')
for item in depfile:
+ kconfig_exp = ""
item = item.strip()
if not item or item[0] == '#':
continue
- sym, dep = item.split()
- if not sym in ret:
- ret[sym] = [dep, ]
+ if "kconfig:" in item:
+ sym, kconfig_exp = item.split(" ", 1)
+ if not sym in ret:
+ ret[sym] = [kconfig_exp, ]
+ else:
+ ret[sym].append(kconfig_exp)
else:
- ret[sym].append(dep)
+ sym, dep = item.split()
+ if not sym in ret:
+ ret[sym] = [dep, ]
+ else:
+ ret[sym].append(dep)
return ret
def add_automatic_backports(args):
+ disable_list = []
export = re.compile(r'^EXPORT_SYMBOL(_GPL)?\((?P<sym>[^\)]*)\)')
- bpi = kconfig.get_backport_info(os.path.join(args.outdir, 'compat', 'Kconfig'))
- configtree = kconfig.ConfigTree(os.path.join(args.outdir, 'Kconfig'))
+ bpi = kconfig.get_backport_info(os.path.join(args.bpid.target_dir, 'compat', 'Kconfig'))
+ configtree = kconfig.ConfigTree(os.path.join(args.bpid.target_dir, 'Kconfig'), args.bpid)
+ ignore=['Kconfig.kernel', 'Kconfig.versions']
+ configtree.verify_sources(ignore=ignore)
+ git_debug_snapshot(args, "verify sources for automatic backports")
all_selects = configtree.all_selects()
for sym, vals in bpi.items():
- if sym.startswith('BACKPORT_BUILD_'):
- if not sym[15:] in all_selects:
+ if sym.startswith('BPAUTO_BUILD_'):
+ if not sym[13:] in all_selects:
+ disable_list.append(sym)
continue
symtype, module_name, c_files, h_files = vals
files.append((os.path.join('include', f),
os.path.join('include', os.path.dirname(f), 'backport-' + os.path.basename(f))))
if args.git_revision:
- copy_git_files(args.kerneldir, files, args.git_revision, args.outdir)
+ copy_git_files(args.kerneldir, files, args.git_revision, args.bpid.target_dir)
else:
- copy_files(args.kerneldir, files, args.outdir)
+ copy_files(args.kerneldir, files, args.bpid.target_dir)
# now add the Makefile line
- mf = open(os.path.join(args.outdir, 'compat', 'Makefile'), 'a+')
+ mf = open(os.path.join(args.bpid.target_dir, 'compat', 'Makefile'), 'a+')
o_files = [automatic_backport_mangle_c_file(f)[:-1] + 'o' for f in c_files]
if symtype == 'tristate':
if not module_name:
raise Exception('backporting a module requires a #module-name')
for of in o_files:
mf.write('%s-objs += %s\n' % (module_name, of))
- mf.write('obj-$(CPTCFG_%s) += %s.o\n' % (sym, module_name))
+ mf.write('obj-$(%s%s) += %s.o\n' % (args.bpid.full_prefix, sym, module_name))
elif symtype == 'bool':
- mf.write('compat-$(CPTCFG_%s) += %s\n' % (sym, ' '.join(o_files)))
+ mf.write('compat-$(%s%s) += %s\n' % (args.bpid.full_prefix, sym, ' '.join(o_files)))
# finally create the include file
syms = []
for f in c_files:
- for l in open(os.path.join(args.outdir, 'compat',
+ for l in open(os.path.join(args.bpid.target_dir, 'compat',
automatic_backport_mangle_c_file(f)), 'r'):
m = export.match(l)
if m:
syms.append(m.group('sym'))
for f in h_files:
- outf = open(os.path.join(args.outdir, 'include', f), 'w')
+ outf = open(os.path.join(args.bpid.target_dir, 'include', f), 'w')
outf.write('/* Automatically created during backport process */\n')
- outf.write('#ifndef CPTCFG_%s\n' % sym)
+ outf.write('#ifndef %s%s\n' % (args.bpid.full_prefix, sym))
outf.write('#include_next <%s>\n' % f)
outf.write('#else\n');
for s in syms:
outf.write('#undef %s\n' % s)
outf.write('#define %s LINUX_BACKPORT(%s)\n' % (s, s))
outf.write('#include <%s>\n' % (os.path.dirname(f) + '/backport-' + os.path.basename(f), ))
- outf.write('#endif /* CPTCFG_%s */\n' % sym)
+ outf.write('#endif /* %s%s */\n' % (args.bpid.full_prefix, sym))
+ return disable_list
def git_debug_init(args):
"""
"""
if not args.gitdebug:
return
- git.init(tree=args.outdir)
- git.commit_all("Copied backport", tree=args.outdir)
+ # Git supports re-initialization, although not well documented it can
+ # reset config stuff, lets avoid that if the tree already exists.
+ if not os.path.exists(os.path.join(args.bpid.project_dir, '.git')):
+ git.init(tree=args.bpid.project_dir)
+ git.commit_all("Copied backport", tree=args.bpid.project_dir)
def git_debug_snapshot(args, name):
"""
if not args.gitdebug:
return
- git.commit_all(name, tree=args.outdir)
+ git.commit_all(name, tree=args.bpid.project_dir)
+def get_rel_spec_stable(rel):
+ """
+ Returns release specs for a linux-stable backports based release.
+ """
+ if ("rc" in rel):
+ m = re.match(r"(?P<VERSION>\d+)\.+" \
+ "(?P<PATCHLEVEL>\d+)[.]*" \
+ "(?P<SUBLEVEL>\d*)" \
+ "[-rc]+(?P<RC_VERSION>\d+)\-+" \
+ "(?P<RELMOD_UPDATE>\d+)[-]*" \
+ "(?P<RELMOD_TYPE>[usnpc]*)", \
+ rel)
+ else:
+ m = re.match(r"(?P<VERSION>\d+)\.+" \
+ "(?P<PATCHLEVEL>\d+)[.]*" \
+ "(?P<SUBLEVEL>\d*)\-+" \
+ "(?P<RELMOD_UPDATE>\d+)[-]*" \
+ "(?P<RELMOD_TYPE>[usnpc]*)", \
+ rel)
+ if (not m):
+ return m
+ return m.groupdict()
-def _main():
- # set up and parse arguments
- parser = argparse.ArgumentParser(description='generate backport tree')
- parser.add_argument('kerneldir', metavar='<kernel tree>', type=str,
- help='Kernel tree to copy drivers from')
- parser.add_argument('outdir', metavar='<output directory>', type=str,
- help='Directory to write the generated tree to')
- parser.add_argument('--copy-list', metavar='<listfile>', type=argparse.FileType('r'),
- default='copy-list',
- help='File containing list of files/directories to copy, default "copy-list"')
- parser.add_argument('--git-revision', metavar='<revision>', type=str,
- help='git commit revision (see gitrevisions(7)) to take objects from.' +
- 'If this is specified, the kernel tree is used as git object storage ' +
- 'and we use git ls-tree to get the files.')
- parser.add_argument('--clean', const=True, default=False, action="store_const",
- help='Clean output directory instead of erroring if it isn\'t empty')
- parser.add_argument('--refresh', const=True, default=False, action="store_const",
- help='Refresh patches as they are applied, the source dir will be modified!')
- parser.add_argument('--base-name', metavar='<name>', type=str, default='Linux',
- help='name of base tree, default just "Linux"')
- parser.add_argument('--gitdebug', const=True, default=False, action="store_const",
- help='Use git, in the output tree, to debug the various transformation steps ' +
- 'that the tree generation makes (apply patches, ...)')
- parser.add_argument('--verbose', const=True, default=False, action="store_const",
- help='Print more verbose information')
- parser.add_argument('--extra-driver', nargs=2, metavar=('<source dir>', '<copy-list>'), type=str,
- action='append', default=[], help='Extra driver directory/copy-list.')
- args = parser.parse_args()
+def get_rel_spec_next(rel):
+ """
+ Returns release specs for a linux-next backports based release.
+ """
+ m = re.match(r"(?P<DATE_VERSION>\d+)[-]*" \
+ "(?P<RELMOD_UPDATE>\d*)[-]*" \
+ "(?P<RELMOD_TYPE>[usnpc]*)", \
+ rel)
+ if (not m):
+ return m
+ return m.groupdict()
- def logwrite(msg):
- sys.stdout.write(msg)
- sys.stdout.write('\n')
- sys.stdout.flush()
+def get_rel_prep(rel):
+ """
+ Returns a dict with prep work details we need prior to
+ uploading a backports release to kernel.org
+ """
+ rel_specs = get_rel_spec_stable(rel)
+ is_stable = True
+ rel_tag = ""
+ paths = list()
+ if (not rel_specs):
+ rel_specs = get_rel_spec_next(rel)
+ if (not rel_specs):
+ sys.stdout.write("rel: %s\n" % rel)
+ return None
+ if (rel_specs['RELMOD_UPDATE'] == '0' or
+ rel_specs['RELMOD_UPDATE'] == '1'):
+ return None
+ is_stable = False
+ date = rel_specs['DATE_VERSION']
+ year = date[0:4]
+ if (len(year) != 4):
+ return None
+ month = date[4:6]
+ if (len(month) != 2):
+ return None
+ day = date[6:8]
+ if (len(day) != 2):
+ return None
+ paths.append(year)
+ paths.append(month)
+ paths.append(day)
+ rel_tag = "backports-" + rel.replace(rel_specs['RELMOD_TYPE'], "")
+ else:
+ ignore = "-"
+ if (not rel_specs['RELMOD_UPDATE']):
+ return None
+ if (rel_specs['RELMOD_UPDATE'] == '0'):
+ return None
+ ignore += rel_specs['RELMOD_UPDATE']
+ if (rel_specs['RELMOD_TYPE'] != ''):
+ ignore += rel_specs['RELMOD_TYPE']
+ base_rel = rel.replace(ignore, "")
+ paths.append("v" + base_rel)
+ rel_tag = "v" + rel.replace(rel_specs['RELMOD_TYPE'], "")
- return process(args.kerneldir, args.outdir, args.copy_list,
- git_revision=args.git_revision, clean=args.clean,
- refresh=args.refresh, base_name=args.base_name,
- gitdebug=args.gitdebug, verbose=args.verbose,
- extra_driver=args.extra_driver, logwrite=logwrite)
+ rel_prep = dict(stable = is_stable,
+ expected_tag = rel_tag,
+ paths_to_create = paths)
+ return rel_prep
-def process(kerneldir, outdir, copy_list_file, git_revision=None,
- clean=False, refresh=False, base_name="Linux", gitdebug=False,
- verbose=False, extra_driver=[], logwrite=lambda x:None,
- git_tracked_version=False):
- class Args(object):
- def __init__(self, kerneldir, outdir, copy_list_file,
- git_revision, clean, refresh, base_name,
- gitdebug, verbose, extra_driver):
- self.kerneldir = kerneldir
- self.outdir = outdir
- self.copy_list = copy_list_file
- self.git_revision = git_revision
- self.clean = clean
- self.refresh = refresh
- self.base_name = base_name
- self.gitdebug = gitdebug
- self.verbose = verbose
- self.extra_driver = extra_driver
- args = Args(kerneldir, outdir, copy_list_file,
- git_revision, clean, refresh, base_name,
- gitdebug, verbose, extra_driver)
- # start processing ...
+def create_tar_and_gz(tar_name, dir_to_tar):
+ """
+ We need both a tar file and gzip for kernel.org, the tar file
+ gets signed, then we upload the compressed version, kup-server
+ in the backend decompresses and verifies the tarball against
+ our signature.
+ """
+ basename = os.path.basename(dir_to_tar)
+ tar = tarfile.open(tar_name, "w")
+ tar.add(dir_to_tar, basename)
+ tar.close()
- copy_list = read_copy_list(args.copy_list)
- deplist = read_dependencies(os.path.join(source_dir, 'dependencies'))
+ tar_file = open(tar_name, "r")
- # validate output directory
- check_output_dir(args.outdir, args.clean)
+ gz_file = gzip.GzipFile(tar_name + ".gz", 'wb')
+ gz_file.write(tar_file.read())
+ gz_file.close()
- # do the copy
- backport_files = [(x, x) for x in [
- 'Kconfig', 'Makefile', 'Makefile.build', 'Makefile.kernel', '.gitignore',
- 'Makefile.real', 'compat/', 'backport-include/', 'kconf/', 'defconfigs/',
- 'scripts/', '.blacklist.map', 'udev/',
- ]]
- if not args.git_revision:
- logwrite('Copy original source files ...')
- else:
- logwrite('Get original source files from git ...')
-
- copy_files(os.path.join(source_dir, 'backport'), backport_files, args.outdir)
+def upload_release(args, rel_prep, logwrite=lambda x:None):
+ """
+ Given a path of a relase make tarball out of it, PGP sign it, and
+ then upload it to kernel.org using kup.
- git_debug_init(args)
+ The linux-next based release do not require a RELMOD_UPDATE
+ given that typically only one release is made per day. Using
+ RELMOD_UPDATE for these releases is allowed though and if
+ present it must be > 1.
- if not args.git_revision:
- copy_files(args.kerneldir, copy_list, args.outdir)
- else:
- copy_git_files(args.kerneldir, copy_list, args.git_revision, args.outdir)
+ The linux-stable based releases require a RELMOD_UPDATE.
- # FIXME: should we add a git version of this (e.g. --git-extra-driver)?
- for src, copy_list in args.extra_driver:
- copy_files(src, read_copy_list(open(copy_list, 'r')), args.outdir)
+ RELMOD_UPDATE must be numeric and > 0 just as the RC releases
+ of the Linux kernel.
- git_debug_snapshot(args, 'Add driver sources')
+ The tree must also be tagged with the respective release, without
+ the RELMOD_TYPE. For linux-next based releases this consists of
+ backports- followed by DATE_VERSION and if RELMOD_TYPE is present.
+ For linux-stable releases this consists of v followed by the
+ full release version except the RELMOD_TYPE.
- add_automatic_backports(args)
- git_debug_snapshot(args, 'Add automatic backports')
+ Uploads will not be allowed if these rules are not followed.
+ """
+ korg_path = "/pub/linux/kernel/projects/backports"
+
+ if (rel_prep['stable']):
+ korg_path += "/stable"
+
+ parent = os.path.dirname(args.bpid.project_dir)
+ release = os.path.basename(args.bpid.project_dir)
+ tar_name = parent + '/' + release + ".tar"
+ gzip_name = tar_name + ".gz"
- logwrite('Apply patches ...')
+ create_tar_and_gz(tar_name, args.bpid.project_dir)
+
+ logwrite(gpg.sign(tar_name, extra_args=['--armor', '--detach-sign']))
+
+ logwrite("------------------------------------------------------")
+
+ if (not args.kup_test):
+ logwrite("About to upload, current target path contents:")
+ else:
+ logwrite("kup-test: current target path contents:")
+
+ logwrite(kup.ls(path=korg_path))
+
+ for path in rel_prep['paths_to_create']:
+ korg_path += '/' + path
+ if (not args.kup_test):
+ logwrite("create directory: %s" % korg_path)
+ logwrite(kup.mkdir(korg_path))
+ korg_path += '/'
+ if (not args.kup_test):
+ logwrite("upload file %s to %s" % (gzip_name, korg_path))
+ logwrite(kup.put(gzip_name, tar_name + '.asc', korg_path))
+ logwrite("\nFinished upload!\n")
+ logwrite("Target path contents:")
+ logwrite(kup.ls(path=korg_path))
+ else:
+ kup_cmd = "kup put /\n\t\t%s /\n\t\t%s /\n\t\t%s" % (gzip_name, tar_name + '.asc', korg_path)
+ logwrite("kup-test: skipping cmd: %s" % kup_cmd)
+
+def apply_patches(args, desc, source_dir, patch_src, target_dir, logwrite=lambda x:None):
+ """
+ Given a path of a directories of patches and SmPL patches apply
+ them on the target directory. If requested refresh patches, or test
+ a specific SmPL patch.
+ """
+ logwrite('Applying patches from %s to %s ...' % (patch_src, target_dir))
+ test_cocci = args.test_cocci or args.profile_cocci
+ test_cocci_found = False
patches = []
- for root, dirs, files in os.walk(os.path.join(source_dir, 'patches')):
+ sempatches = []
+ for root, dirs, files in os.walk(os.path.join(source_dir, patch_src)):
for f in files:
- if f.endswith('.patch'):
+ if not test_cocci and f.endswith('.patch'):
patches.append(os.path.join(root, f))
+ if f.endswith('.cocci'):
+ if test_cocci:
+ if f not in test_cocci:
+ continue
+ test_cocci_found = True
+ if args.test_cocci:
+ logwrite("Testing Coccinelle SmPL patch: %s" % test_cocci)
+ elif args.profile_cocci:
+ logwrite("Profiling Coccinelle SmPL patch: %s" % test_cocci)
+ sempatches.append(os.path.join(root, f))
patches.sort()
- prefix_len = len(os.path.join(source_dir, 'patches')) + 1
+ prefix_len = len(os.path.join(source_dir, patch_src)) + 1
for pfile in patches:
print_name = pfile[prefix_len:]
# read the patch file
# check if the first file the patch touches exists, if so
# assume the patch needs to be applied -- otherwise continue
patched_file = '/'.join(p.items[0].source.split('/')[1:])
- fullfn = os.path.join(args.outdir, patched_file)
+ fullfn = os.path.join(target_dir, patched_file)
if not os.path.exists(fullfn):
if args.verbose:
logwrite("Not applying %s, not needed" % print_name)
# but for refresh, of course look at all files the patch touches
for patchitem in p.items:
patched_file = '/'.join(patchitem.source.split('/')[1:])
- fullfn = os.path.join(args.outdir, patched_file)
+ fullfn = os.path.join(target_dir, patched_file)
shutil.copyfile(fullfn, fullfn + '.orig_file')
process = subprocess.Popen(['patch', '-p1'], stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, stdin=subprocess.PIPE,
close_fds=True, universal_newlines=True,
- cwd=args.outdir)
+ cwd=target_dir)
output = process.communicate(input=open(pfile, 'r').read())[0]
output = output.split('\n')
if output[-1] == '':
pfilef.flush()
for patchitem in p.items:
patched_file = '/'.join(patchitem.source.split('/')[1:])
- fullfn = os.path.join(args.outdir, patched_file)
+ fullfn = os.path.join(target_dir, patched_file)
process = subprocess.Popen(['diff', '-p', '-u', patched_file + '.orig_file', patched_file,
'--label', 'a/' + patched_file,
'--label', 'b/' + patched_file],
stdout=pfilef, close_fds=True,
- universal_newlines=True, cwd=args.outdir)
+ universal_newlines=True, cwd=target_dir)
process.wait()
os.unlink(fullfn + '.orig_file')
if not process.returncode in (0, 1):
logwrite("Failed to diff to refresh %s" % print_name)
pfilef.close()
os.unlink(pfile + '.tmp')
- return 3
+ return 2
pfilef.close()
os.rename(pfile + '.tmp', pfile)
# remove orig/rej files that patch sometimes creates
- for root, dirs, files in os.walk(args.outdir):
+ for root, dirs, files in os.walk(target_dir):
for f in files:
if f[-5:] == '.orig' or f[-4:] == '.rej':
os.unlink(os.path.join(root, f))
- git_debug_snapshot(args, "apply backport patch %s" % print_name)
+ git_debug_snapshot(args, "apply %s patch %s" % (desc, print_name))
- # some post-processing is required
- configtree = kconfig.ConfigTree(os.path.join(args.outdir, 'Kconfig'))
- logwrite('Modify Kconfig tree ...')
- configtree.prune_sources(ignore=['Kconfig.kernel', 'Kconfig.versions'])
- git_debug_snapshot(args, "prune Kconfig tree")
- configtree.force_tristate_modular()
- git_debug_snapshot(args, "force tristate options modular")
- configtree.modify_selects()
- git_debug_snapshot(args, "convert select to depends on")
+ sempatches.sort()
+ prefix_len = len(os.path.join(source_dir, patch_src)) + 1
+
+ for cocci_file in sempatches:
+ # Until Coccinelle picks this up
+ pycocci = os.path.join(source_dir, 'devel/pycocci')
+ cmd = [pycocci, cocci_file]
+ extra_spatch_args = []
+ if args.profile_cocci:
+ cmd.append('--profile-cocci')
+ cmd.append(os.path.abspath(target_dir))
+ print_name = cocci_file[prefix_len:]
+ if args.verbose:
+ logwrite("Applying SmPL patch %s" % print_name)
+ logwrite(" %s" % cmd)
+ sprocess = subprocess.Popen(cmd,
+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+ close_fds=True, universal_newlines=True,
+ cwd=target_dir)
+ output = sprocess.communicate()[0]
+ sprocess.wait()
+ if sprocess.returncode != 0:
+ logwrite("Failed to process SmPL patch %s with %i" % (print_name, sprocess.returncode))
+ return 2
+ output = output.split('\n')
+ if output[-1] == '':
+ output = output[:-1]
+ if args.verbose:
+ for line in output:
+ logwrite('> %s' % line)
+
+ # remove cocci_backup files
+ for root, dirs, files in os.walk(target_dir):
+ for f in files:
+ if f.endswith('.cocci_backup'):
+ os.unlink(os.path.join(root, f))
+ git_debug_snapshot(args, "apply %s SmPL patch %s" % (desc, print_name))
+
+ if test_cocci and test_cocci_found:
+ logwrite('Done!')
+ sys.exit(0)
+
+def _main():
+ # Our binary requirements go here
+ req = reqs.Req()
+ req.require('git')
+ req.coccinelle('1.0.6')
+ if not req.reqs_match():
+ sys.exit(1)
+
+ # set up and parse arguments
+ parser = argparse.ArgumentParser(description='generate backport tree')
+ parser.add_argument('kerneldir', metavar='<kernel tree>', type=str,
+ help='Kernel tree to copy drivers from')
+ parser.add_argument('outdir', metavar='<output directory>', type=str,
+ help='Directory to write the generated tree to')
+ parser.add_argument('--copy-list', metavar='<listfile>', type=argparse.FileType('r'),
+ default='copy-list',
+ help='File containing list of files/directories to copy, default "copy-list"')
+ parser.add_argument('--git-revision', metavar='<revision>', type=str,
+ help='git commit revision (see gitrevisions(7)) to take objects from.' +
+ 'If this is specified, the kernel tree is used as git object storage ' +
+ 'and we use git ls-tree to get the files.')
+ parser.add_argument('--clean', const=True, default=False, action="store_const",
+ help='Clean output directory instead of erroring if it isn\'t empty')
+ parser.add_argument('--integrate', const=True, default=False, action="store_const",
+ help='Integrate a future backported kernel solution into ' +
+ 'an older kernel tree source directory.')
+ parser.add_argument('--refresh', const=True, default=False, action="store_const",
+ help='Refresh patches as they are applied, the source dir will be modified!')
+ parser.add_argument('--base-name', metavar='<name>', type=str, default='Linux',
+ help='name of base tree, default just "Linux"')
+ parser.add_argument('--gitdebug', '--git-debug', const=True, default=False, action="store_const",
+ help='Use git, in the output tree, to debug the various transformation steps ' +
+ 'that the tree generation makes (apply patches, ...)')
+ parser.add_argument('--verbose', const=True, default=False, action="store_const",
+ help='Print more verbose information')
+ parser.add_argument('--extra-driver', nargs=2, metavar=('<source dir>', '<copy-list>'), type=str,
+ action='append', default=[], help='Extra driver directory/copy-list.')
+ parser.add_argument('--kup', const=True, default=False, action="store_const",
+ help='For maintainers: upload a release to kernel.org')
+ parser.add_argument('--kup-test', const=True, default=False, action="store_const",
+ help='For maintainers: do all the work as if you were about to ' +
+ 'upload to kernel.org but do not do the final `kup put` ' +
+ 'and also do not run any `kup mkdir` commands. This will ' +
+ 'however run `kup ls` on the target paths so ' +
+ 'at the very least we test your kup configuration. ' +
+ 'If this is your first time uploading use this first!')
+ parser.add_argument('--test-cocci', metavar='<sp_file>', type=str, default=None,
+ help='Only use the cocci file passed for Coccinelle, don\'t do anything else, ' +
+ 'also creates a git repo on the target directory for easy inspection ' +
+ 'of changes done by Coccinelle.')
+ parser.add_argument('--profile-cocci', metavar='<sp_file>', type=str, default=None,
+ help='Only use the cocci file passed and pass --profile to Coccinelle, ' +
+ 'also creates a git repo on the target directory for easy inspection ' +
+ 'of changes done by Coccinelle.')
+ args = parser.parse_args()
+
+ # When building a package we use CPTCFG as we can rely on the
+ # fact that kconfig treats CONFIG_ as an environment variable
+ # requring less changes on code. For kernel integration we use
+ # the longer CONFIG_BACKPORT given that we'll be sticking to
+ # the kernel symbol namespace, to address that we do a final
+ # search / replace. Technically its possible to rely on the
+ # same prefix for packaging as with kernel integration but
+ # there are already some users of the CPTCFG prefix.
+ bpid = None
+ if args.integrate:
+ bpid = Bp_Identity(integrate = args.integrate,
+ kconfig_prefix = 'CONFIG_',
+ project_prefix = 'BACKPORT_',
+ project_dir = args.outdir,
+ target_dir = os.path.join(args.outdir, 'backports/'),
+ target_dir_name = 'backports/',
+ kconfig_source_var = '$BACKPORT_DIR',
+ )
+ else:
+ bpid = Bp_Identity(integrate = args.integrate,
+ kconfig_prefix = 'CPTCFG_',
+ project_prefix = '',
+ project_dir = args.outdir,
+ target_dir = args.outdir,
+ target_dir_name = '',
+ kconfig_source_var = '$BACKPORT_DIR',
+ )
+
+ def logwrite(msg):
+ sys.stdout.write(msg)
+ sys.stdout.write('\n')
+ sys.stdout.flush()
+
+ return process(args.kerneldir, args.copy_list,
+ git_revision=args.git_revision,
+ bpid=bpid,
+ clean=args.clean,
+ refresh=args.refresh, base_name=args.base_name,
+ gitdebug=args.gitdebug, verbose=args.verbose,
+ extra_driver=args.extra_driver,
+ kup=args.kup,
+ kup_test=args.kup_test,
+ test_cocci=args.test_cocci,
+ profile_cocci=args.profile_cocci,
+ logwrite=logwrite)
+
+def process(kerneldir, copy_list_file, git_revision=None,
+ bpid=None,
+ clean=False, refresh=False, base_name="Linux", gitdebug=False,
+ verbose=False, extra_driver=[], kup=False,
+ kup_test=False,
+ test_cocci=None,
+ profile_cocci=None,
+ logwrite=lambda x:None,
+ git_tracked_version=False):
+ class Args(object):
+ def __init__(self, kerneldir, copy_list_file,
+ git_revision, bpid, clean, refresh, base_name,
+ gitdebug, verbose, extra_driver, kup,
+ kup_test,
+ test_cocci,
+ profile_cocci):
+ self.kerneldir = kerneldir
+ self.copy_list = copy_list_file
+ self.git_revision = git_revision
+ self.bpid = bpid
+ self.clean = clean
+ self.refresh = refresh
+ self.base_name = base_name
+ self.gitdebug = gitdebug
+ self.verbose = verbose
+ self.extra_driver = extra_driver
+ self.kup = kup
+ self.kup_test = kup_test
+ self.test_cocci = test_cocci
+ self.profile_cocci = profile_cocci
+ if self.test_cocci or self.profile_cocci:
+ self.gitdebug = True
+ def git_paranoia(tree=None, logwrite=lambda x:None):
+ data = git.paranoia(tree)
+ if (data['r'] != 0):
+ logwrite('Cannot use %s' % tree)
+ logwrite('%s' % data['output'])
+ sys.exit(data['r'])
+ else:
+ logwrite('Validated tree: %s' % tree)
+
+ args = Args(kerneldir, copy_list_file,
+ git_revision, bpid, clean, refresh, base_name,
+ gitdebug, verbose, extra_driver, kup, kup_test,
+ test_cocci, profile_cocci)
+ rel_prep = None
+
+ if bpid.integrate:
+ if args.kup_test or args.test_cocci or args.profile_cocci or args.refresh:
+ logwrite('Cannot use integration with:\n\tkup_test\n\ttest_cocci\n\tprofile_cocci\n\trefresh\n');
+ sys.exit(1)
+
+ # start processing ...
+ if (args.kup or args.kup_test):
+ git_paranoia(source_dir, logwrite)
+ git_paranoia(kerneldir, logwrite)
+
+ rel_describe = git.describe(rev=None, tree=source_dir, extra_args=['--dirty'])
+ release = os.path.basename(bpid.target_dir)
+ version = release.replace("backports-", "")
+
+ rel_prep = get_rel_prep(version)
+ if (not rel_prep):
+ logwrite('Invalid backports release name: %s' % release)
+ logwrite('For rules on the release name see upload_release()')
+ sys.exit(1)
+ rel_type = "linux-stable"
+ if (not rel_prep['stable']):
+ rel_type = "linux-next"
+ if (rel_prep['expected_tag'] != rel_describe):
+ logwrite('Unexpected %s based backports release tag on' % rel_type)
+ logwrite('the backports tree tree: %s\n' % rel_describe)
+ logwrite('You asked to make a release with this ')
+ logwrite('directory name: %s' % release)
+ logwrite('The actual expected tag we should find on')
+ logwrite('the backports tree then is: %s\n' % rel_prep['expected_tag'])
+ logwrite('For rules on the release name see upload_release()')
+ sys.exit(1)
- # write the versioning file
+ copy_list = read_copy_list(args.copy_list)
+ deplist = read_dependencies(os.path.join(source_dir, 'dependencies'))
+
+ # validate output directory
+ check_output_dir(bpid.target_dir, args.clean)
+
+ # do the copy
+ backport_integrate_files = [
+ ('Makefile.kernel', 'Makefile'),
+ ('Kconfig.integrate', 'Kconfig'),
+ ]
+ backport_package_files = [(x, x) for x in [
+ 'Makefile',
+ 'kconf/',
+ 'Makefile.real',
+ 'Makefile.kernel',
+ 'Kconfig.package.hacks',
+ 'scripts/',
+ '.blacklist.map',
+ '.gitignore',
+ 'Makefile.build'] ]
+ backport_package_files += [
+ ('Kconfig.package', 'Kconfig'),
+ ]
+ backport_files = [(x, x) for x in [
+ 'Kconfig.sources',
+ 'compat/',
+ 'backport-include/',
+ ]]
+
+ if not bpid.integrate:
+ backport_files += backport_package_files
+ else:
+ backport_files += backport_integrate_files
+
+ if not args.git_revision:
+ logwrite('Copy original source files ...')
+ else:
+ logwrite('Get original source files from git ...')
+
+ copy_files(os.path.join(source_dir, 'backport'), backport_files, bpid.target_dir)
+
+ git_debug_init(args)
+
+ if not args.git_revision:
+ copy_files(args.kerneldir, copy_list, bpid.target_dir)
+ else:
+ copy_git_files(args.kerneldir, copy_list, args.git_revision, bpid.target_dir)
+
+ # FIXME: should we add a git version of this (e.g. --git-extra-driver)?
+ for src, copy_list in args.extra_driver:
+ if (args.kup or args.kup_test):
+ git_paranoia(src)
+ copy_files(src, read_copy_list(open(copy_list, 'r')), bpid.target_dir)
+
+ git_debug_snapshot(args, 'Add driver sources')
+
+ disable_list = add_automatic_backports(args)
if git_tracked_version:
backports_version = "(see git)"
kernel_version = "(see git)"
else:
- backports_version = git.describe(tree=source_dir)
+ backports_version = git.describe(tree=source_dir, extra_args=['--long'])
kernel_version = git.describe(rev=args.git_revision or 'HEAD',
- tree=args.kerneldir)
- f = open(os.path.join(args.outdir, 'versions'), 'w')
- f.write('BACKPORTS_VERSION="%s"\n' % backports_version)
- f.write('BACKPORTED_KERNEL_VERSION="%s"\n' % kernel_version)
- f.write('BACKPORTED_KERNEL_NAME="%s"\n' % args.base_name)
- if git_tracked_version:
- f.write('BACKPORTS_GIT_TRACKED="backport tracker ID: $(shell git rev-parse HEAD 2>/dev/null || echo \'not built in git tree\')"\n')
- f.close()
+ tree=args.kerneldir,
+ extra_args=['--long'])
+
+ if not bpid.integrate:
+ f = open(os.path.join(bpid.target_dir, 'versions'), 'w')
+ f.write('BACKPORTS_VERSION="%s"\n' % backports_version)
+ f.write('BACKPORTED_KERNEL_VERSION="%s"\n' % kernel_version)
+ f.write('BACKPORTED_KERNEL_NAME="%s"\n' % args.base_name)
+ if git_tracked_version:
+ f.write('BACKPORTS_GIT_TRACKED="backport tracker ID: $(shell git rev-parse HEAD 2>/dev/null || echo \'not built in git tree\')"\n')
+ f.close()
+ git_debug_snapshot(args, "add versions files")
+ else:
+ kconf_regexes = [
+ (re.compile(r'.*(?P<key>%%BACKPORT_DIR%%)'), '%%BACKPORT_DIR%%', 'backports/'),
+ (re.compile(r'.*(?P<key>%%BACKPORTS_VERSION%%).*'), '%%BACKPORTS_VERSION%%', backports_version),
+ (re.compile(r'.*(?P<key>%%BACKPORTED_KERNEL_VERSION%%).*'), '%%BACKPORTED_KERNEL_VERSION%%', kernel_version),
+ (re.compile(r'.*(?P<key>%%BACKPORTED_KERNEL_NAME%%).*'), '%%BACKPORTED_KERNEL_NAME%%', args.base_name),
+ ]
+ out = ''
+ for l in open(os.path.join(bpid.target_dir, 'Kconfig'), 'r'):
+ for r in kconf_regexes:
+ m = r[0].match(l)
+ if m:
+ l = re.sub(r'(' + r[1] + ')', r'' + r[2] + '', l)
+ out += l
+ outf = open(os.path.join(bpid.target_dir, 'Kconfig'), 'w')
+ outf.write(out)
+ outf.close()
+ git_debug_snapshot(args, "modify top level backports/Kconfig with backports identity")
+
+ if disable_list:
+ # No need to verify_sources() as compat's Kconfig has no 'source' call
+ bpcfg = kconfig.ConfigTree(os.path.join(bpid.target_dir, 'compat', 'Kconfig'), bpid)
+ bpcfg.disable_symbols(disable_list)
+ git_debug_snapshot(args, 'Add automatic backports')
+
+ failure = apply_patches(args, "backport", source_dir, 'patches', bpid.target_dir, logwrite)
+ if failure:
+ return failure
+
+ # Kernel integration requires Kconfig.versions already generated for you,
+ # we cannot do this for a package as we have no idea what kernel folks
+ # will be using.
+ if bpid.integrate:
+ kver = gen_version.kernelversion(bpid.project_dir)
+ rel_specs = gen_version.get_rel_spec_stable(kver)
+ if not rel_specs:
+ logwrite('Cannot parse source kernel version, update parser')
+ sys.exit(1)
+ data = gen_version.genkconfig_versions(rel_specs)
+ fo = open(os.path.join(bpid.target_dir, 'Kconfig.versions'), 'w')
+ fo.write(data)
+ fo.close()
+ git_debug_snapshot(args, "generate kernel version requirement Kconfig file")
+
+ # some post-processing is required
+ configtree = kconfig.ConfigTree(os.path.join(bpid.target_dir, 'Kconfig'), bpid)
+ ignore=['Kconfig.kernel', 'Kconfig.versions', 'Kconfig.local']
+
+ configtree.verify_sources(ignore=ignore)
+ git_debug_snapshot(args, "verify sources on top level backports Kconfig")
+
+ orig_symbols = configtree.symbols()
+
+ logwrite('Modify Kconfig tree ...')
+ configtree.prune_sources(ignore=ignore)
+ git_debug_snapshot(args, "prune Kconfig tree")
+
+ if not bpid.integrate:
+ configtree.force_tristate_modular()
+ git_debug_snapshot(args, "force tristate options modular")
+
+ ignore = [os.path.join(bpid.target_dir, x) for x in [
+ 'Kconfig.package.hacks',
+ 'Kconfig.versions',
+ 'Kconfig.local',
+ 'Kconfig',
+ ]
+ ]
+ configtree.adjust_backported_configs(ignore=ignore, orig_symbols=orig_symbols)
+ git_debug_snapshot(args, "adjust backports config symbols we port")
+
+ configtree.modify_selects()
+ git_debug_snapshot(args, "convert select to depends on")
symbols = configtree.symbols()
- # write local symbol list -- needed during build
- f = open(os.path.join(args.outdir, '.local-symbols'), 'w')
+ # write local symbol list -- needed during packaging build
+ if not bpid.integrate:
+ f = open(os.path.join(bpid.target_dir, 'local-symbols'), 'w')
+ for sym in symbols:
+ f.write('%s=\n' % sym)
+ f.close()
+ git_debug_snapshot(args, "add symbols files")
+ # also write Kconfig.local, representing all local symbols
+ # with a BACKPORTED_ prefix
+ f = open(os.path.join(bpid.target_dir, 'Kconfig.local'), 'w')
for sym in symbols:
- f.write('%s=\n' % sym)
+ f.write('config BACKPORTED_%s\n' % sym)
+ f.write('\ttristate\n')
+ f.write('\tdefault %s\n' % sym)
f.close()
+ git_debug_snapshot(args, "add Kconfig.local")
+
+ # add defconfigs that we want
+ defconfigs_dir = os.path.join(source_dir, 'backport', 'defconfigs')
+ os.mkdir(os.path.join(bpid.target_dir, 'defconfigs'))
+ for dfbase in os.listdir(defconfigs_dir):
+ copy_defconfig = True
+ dfsrc = os.path.join(defconfigs_dir, dfbase)
+ for line in open(dfsrc, 'r'):
+ if not '=' in line:
+ continue
+ line_ok = False
+ for sym in symbols:
+ if sym + '=' in line:
+ line_ok = True
+ break
+ if not line_ok:
+ copy_defconfig = False
+ break
+ if copy_defconfig:
+ shutil.copy(dfsrc, os.path.join(bpid.target_dir, 'defconfigs', dfbase))
- git_debug_snapshot(args, "add versions/symbols files")
+ git_debug_snapshot(args, "add (useful) defconfig files")
logwrite('Rewrite Makefiles and Kconfig files ...')
# rewrite Makefile and source symbols
+
+ # symbols we know only we can provide under the backport project prefix
+ # for which we need an exception.
+ skip_orig_syms = [ bpid.project_prefix + x for x in [
+ 'INTEGRATE',
+ ]
+ ]
+ parse_orig_syms = [x for x in orig_symbols if x not in skip_orig_syms ]
regexes = []
- for some_symbols in [symbols[i:i + 50] for i in range(0, len(symbols), 50)]:
+ for some_symbols in [parse_orig_syms[i:i + 50] for i in range(0, len(parse_orig_syms), 50)]:
r = 'CONFIG_((' + '|'.join([s + '(_MODULE)?' for s in some_symbols]) + ')([^A-Za-z0-9_]|$))'
regexes.append(re.compile(r, re.MULTILINE))
- for root, dirs, files in os.walk(args.outdir):
+ for root, dirs, files in os.walk(bpid.target_dir):
# don't go into .git dir (possible debug thing)
if '.git' in dirs:
dirs.remove('.git')
for f in files:
data = open(os.path.join(root, f), 'r').read()
for r in regexes:
- data = r.sub(r'CPTCFG_\1', data)
+ data = r.sub(r'' + bpid.full_prefix + '\\1', data)
+ # we have an absolue path in $(src) since we compile out of tree
+ data = re.sub(r'\$\(srctree\)/\$\(src\)', '$(src)', data)
data = re.sub(r'\$\(srctree\)', '$(backport_srctree)', data)
data = re.sub(r'-Idrivers', '-I$(backport_srctree)/drivers', data)
+ if bpid.integrate:
+ data = re.sub(r'CPTCFG_', bpid.full_prefix, data)
fo = open(os.path.join(root, f), 'w')
fo.write(data)
fo.close()
git_debug_snapshot(args, "rename config symbol / srctree usage")
# disable unbuildable Kconfig symbols and stuff Makefiles that doesn't exist
- maketree = make.MakeTree(os.path.join(args.outdir, 'Makefile.kernel'))
+ if bpid.integrate:
+ maketree = make.MakeTree(os.path.join(bpid.target_dir, 'Makefile'))
+ else:
+ maketree = make.MakeTree(os.path.join(bpid.target_dir, 'Makefile.kernel'))
disable_kconfig = []
disable_makefile = []
for sym in maketree.get_impossible_symbols():
for sym in tuple(deplist.keys()):
new = []
for dep in deplist[sym]:
- if dep == "DISABLE":
+ if "kconfig:" in dep:
+ kconfig_expr = dep.replace('kconfig: ', '')
+ new.append(kconfig_expr)
+ elif (dep == "DISABLE"):
new.append('BACKPORT_DISABLED_KCONFIG_OPTION')
else:
- new.append('!BACKPORT_KERNEL_%s' % dep.replace('.', '_'))
- deplist[sym] = new
+ new.append('!KERNEL_%s' % dep.replace('.', '_'))
+ if bpid.integrate:
+ deplist[sym] = ["BACKPORT_" + x for x in new]
+ else:
+ deplist[sym] = new
configtree.add_dependencies(deplist)
git_debug_snapshot(args, "add kernel version dependencies")
# groups -- 50 seemed safer and is still fast)
regexes = []
for some_symbols in [disable_makefile[i:i + 50] for i in range(0, len(disable_makefile), 50)]:
- r = '^([^#].*((CPTCFG|CONFIG)_(' + '|'.join([s for s in some_symbols]) + ')))'
+ r = '^(([^#].*((' + bpid.full_prefix_resafe + '|CONFIG_)(' + '|'.join([s for s in some_symbols]) + ')))\W)'
regexes.append(re.compile(r, re.MULTILINE))
for f in maketree.get_makefiles():
data = open(f, 'r').read()
fo.close()
git_debug_snapshot(args, "disable unsatisfied Makefile parts")
+ if bpid.integrate:
+ f = open(os.path.join(bpid.project_dir, 'Kconfig'), 'a')
+ f.write('source "backports/Kconfig"\n')
+ f.close()
+ git_debug_snapshot(args, "hooked backport to top level Kconfig")
+
+ failure = apply_patches(args, "integration", source_dir, 'integration-patches/',
+ bpid.project_dir, logwrite)
+ if failure:
+ return failure
+
+ if (args.kup or args.kup_test):
+ req = reqs.Req()
+ req.kup()
+ if not req.reqs_match():
+ sys.exit(1)
+ upload_release(args, rel_prep, logwrite=logwrite)
+
logwrite('Done!')
return 0
projects / openwrt / staging / blogic.git / blobdiff