from lib import bpgit as git
from lib import bpgpg as gpg
from lib import bpkup as kup
-from lib import bpcoccinelle as coccinelle
from lib.tempdir import tempdir
+from lib import bpreqs as reqs
+from lib import bpversion as gen_version
+
+class Bp_Identity(object):
+ """
+ folks considering multiple integrations may want to
+ consider stuffing versioning info here as well but
+ that will need thought/design on sharing compat and
+ module namespaces.
+
+ Use the *_resafe when combining on regexps, although we currently
+ don't support regexps there perhaps later we will and this will
+ just make things safer for the output regardless. Once and if those
+ are added, how we actually use the others for regular printing will
+ need to be considered.
+ """
+ def __init__(self, integrate=False, kconfig_prefix='CPTCFG_',
+ project_prefix='', project_dir='',
+ target_dir='', target_dir_name='',
+ kconfig_source_var=None):
+ self.integrate = integrate
+ self.kconfig_prefix = kconfig_prefix
+ self.kconfig_prefix_resafe = re.escape(kconfig_prefix)
+ self.project_prefix = project_prefix
+ self.project_prefix_resafe = re.escape(project_prefix)
+ self.full_prefix = kconfig_prefix + project_prefix
+ self.full_prefix_resafe = re.escape(self.full_prefix)
+ self.project_dir = project_dir
+ self.target_dir = target_dir
+ self.target_dir_name = target_dir_name
+ self.kconfig_source_var = kconfig_source_var
+ if self.kconfig_source_var:
+ self.kconfig_source_var_resafe = re.escape(self.kconfig_source_var)
+ else:
+ self.kconfig_source_var_resafe = None
def read_copy_list(copyfile):
"""
def add_automatic_backports(args):
disable_list = []
export = re.compile(r'^EXPORT_SYMBOL(_GPL)?\((?P<sym>[^\)]*)\)')
- bpi = kconfig.get_backport_info(os.path.join(args.outdir, 'compat', 'Kconfig'))
- configtree = kconfig.ConfigTree(os.path.join(args.outdir, 'Kconfig'))
+ bpi = kconfig.get_backport_info(os.path.join(args.bpid.target_dir, 'compat', 'Kconfig'))
+ configtree = kconfig.ConfigTree(os.path.join(args.bpid.target_dir, 'Kconfig'), args.bpid)
+ ignore=['Kconfig.kernel', 'Kconfig.versions']
+ configtree.verify_sources(ignore=ignore)
+ git_debug_snapshot(args, "verify sources for automatic backports")
all_selects = configtree.all_selects()
for sym, vals in bpi.items():
- if sym.startswith('BACKPORT_BUILD_'):
- if not sym[15:] in all_selects:
+ if sym.startswith('BPAUTO_BUILD_'):
+ if not sym[13:] in all_selects:
disable_list.append(sym)
continue
symtype, module_name, c_files, h_files = vals
files.append((os.path.join('include', f),
os.path.join('include', os.path.dirname(f), 'backport-' + os.path.basename(f))))
if args.git_revision:
- copy_git_files(args.kerneldir, files, args.git_revision, args.outdir)
+ copy_git_files(args.kerneldir, files, args.git_revision, args.bpid.target_dir)
else:
- copy_files(args.kerneldir, files, args.outdir)
+ copy_files(args.kerneldir, files, args.bpid.target_dir)
# now add the Makefile line
- mf = open(os.path.join(args.outdir, 'compat', 'Makefile'), 'a+')
+ mf = open(os.path.join(args.bpid.target_dir, 'compat', 'Makefile'), 'a+')
o_files = [automatic_backport_mangle_c_file(f)[:-1] + 'o' for f in c_files]
if symtype == 'tristate':
if not module_name:
raise Exception('backporting a module requires a #module-name')
for of in o_files:
mf.write('%s-objs += %s\n' % (module_name, of))
- mf.write('obj-$(CPTCFG_%s) += %s.o\n' % (sym, module_name))
+ mf.write('obj-$(%s%s) += %s.o\n' % (args.bpid.full_prefix, sym, module_name))
elif symtype == 'bool':
- mf.write('compat-$(CPTCFG_%s) += %s\n' % (sym, ' '.join(o_files)))
+ mf.write('compat-$(%s%s) += %s\n' % (args.bpid.full_prefix, sym, ' '.join(o_files)))
# finally create the include file
syms = []
for f in c_files:
- for l in open(os.path.join(args.outdir, 'compat',
+ for l in open(os.path.join(args.bpid.target_dir, 'compat',
automatic_backport_mangle_c_file(f)), 'r'):
m = export.match(l)
if m:
syms.append(m.group('sym'))
for f in h_files:
- outf = open(os.path.join(args.outdir, 'include', f), 'w')
+ outf = open(os.path.join(args.bpid.target_dir, 'include', f), 'w')
outf.write('/* Automatically created during backport process */\n')
- outf.write('#ifndef CPTCFG_%s\n' % sym)
+ outf.write('#ifndef %s%s\n' % (args.bpid.full_prefix, sym))
outf.write('#include_next <%s>\n' % f)
outf.write('#else\n');
for s in syms:
outf.write('#undef %s\n' % s)
outf.write('#define %s LINUX_BACKPORT(%s)\n' % (s, s))
outf.write('#include <%s>\n' % (os.path.dirname(f) + '/backport-' + os.path.basename(f), ))
- outf.write('#endif /* CPTCFG_%s */\n' % sym)
+ outf.write('#endif /* %s%s */\n' % (args.bpid.full_prefix, sym))
return disable_list
def git_debug_init(args):
"""
if not args.gitdebug:
return
- git.init(tree=args.outdir)
- git.commit_all("Copied backport", tree=args.outdir)
+ # Git supports re-initialization, although not well documented it can
+ # reset config stuff, lets avoid that if the tree already exists.
+ if not os.path.exists(os.path.join(args.bpid.project_dir, '.git')):
+ git.init(tree=args.bpid.project_dir)
+ git.commit_all("Copied backport", tree=args.bpid.project_dir)
def git_debug_snapshot(args, name):
"""
if not args.gitdebug:
return
- git.commit_all(name, tree=args.outdir)
+ git.commit_all(name, tree=args.bpid.project_dir)
def get_rel_spec_stable(rel):
"""
if (rel_prep['stable']):
korg_path += "/stable"
- parent = os.path.dirname(args.outdir)
- release = os.path.basename(args.outdir)
+ parent = os.path.dirname(args.bpid.project_dir)
+ release = os.path.basename(args.bpid.project_dir)
tar_name = parent + '/' + release + ".tar"
gzip_name = tar_name + ".gz"
- create_tar_and_gz(tar_name, args.outdir)
+ create_tar_and_gz(tar_name, args.bpid.project_dir)
logwrite(gpg.sign(tar_name, extra_args=['--armor', '--detach-sign']))
kup_cmd = "kup put /\n\t\t%s /\n\t\t%s /\n\t\t%s" % (gzip_name, tar_name + '.asc', korg_path)
logwrite("kup-test: skipping cmd: %s" % kup_cmd)
+def apply_patches(args, desc, source_dir, patch_src, target_dir, logwrite=lambda x:None):
+ """
+ Given a path of a directories of patches and SmPL patches apply
+ them on the target directory. If requested refresh patches, or test
+ a specific SmPL patch.
+ """
+ logwrite('Applying patches from %s to %s ...' % (patch_src, target_dir))
+ test_cocci = args.test_cocci or args.profile_cocci
+ test_cocci_found = False
+ patches = []
+ sempatches = []
+ for root, dirs, files in os.walk(os.path.join(source_dir, patch_src)):
+ for f in files:
+ if not test_cocci and f.endswith('.patch'):
+ patches.append(os.path.join(root, f))
+ if f.endswith('.cocci'):
+ if test_cocci:
+ if f not in test_cocci:
+ continue
+ test_cocci_found = True
+ if args.test_cocci:
+ logwrite("Testing Coccinelle SmPL patch: %s" % test_cocci)
+ elif args.profile_cocci:
+ logwrite("Profiling Coccinelle SmPL patch: %s" % test_cocci)
+ sempatches.append(os.path.join(root, f))
+ patches.sort()
+ prefix_len = len(os.path.join(source_dir, patch_src)) + 1
+ for pfile in patches:
+ print_name = pfile[prefix_len:]
+ # read the patch file
+ p = patch.fromfile(pfile)
+ # complain if it's not a patch
+ if not p:
+ raise Exception('No patch content found in %s' % print_name)
+ # leading / seems to be stripped?
+ if 'dev/null' in p.items[0].source:
+ raise Exception('Patches creating files are not supported (in %s)' % print_name)
+ # check if the first file the patch touches exists, if so
+ # assume the patch needs to be applied -- otherwise continue
+ patched_file = '/'.join(p.items[0].source.split('/')[1:])
+ fullfn = os.path.join(target_dir, patched_file)
+ if not os.path.exists(fullfn):
+ if args.verbose:
+ logwrite("Not applying %s, not needed" % print_name)
+ continue
+ if args.verbose:
+ logwrite("Applying patch %s" % print_name)
+
+ if args.refresh:
+ # but for refresh, of course look at all files the patch touches
+ for patchitem in p.items:
+ patched_file = '/'.join(patchitem.source.split('/')[1:])
+ fullfn = os.path.join(target_dir, patched_file)
+ shutil.copyfile(fullfn, fullfn + '.orig_file')
+
+ process = subprocess.Popen(['patch', '-p1'], stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT, stdin=subprocess.PIPE,
+ close_fds=True, universal_newlines=True,
+ cwd=target_dir)
+ output = process.communicate(input=open(pfile, 'r').read())[0]
+ output = output.split('\n')
+ if output[-1] == '':
+ output = output[:-1]
+ if args.verbose:
+ for line in output:
+ logwrite('> %s' % line)
+ if process.returncode != 0:
+ if not args.verbose:
+ logwrite("Failed to apply changes from %s" % print_name)
+ for line in output:
+ logwrite('> %s' % line)
+ return 2
+
+ if args.refresh:
+ pfilef = open(pfile + '.tmp', 'a')
+ pfilef.write(p.top_header)
+ pfilef.flush()
+ for patchitem in p.items:
+ patched_file = '/'.join(patchitem.source.split('/')[1:])
+ fullfn = os.path.join(target_dir, patched_file)
+ process = subprocess.Popen(['diff', '-p', '-u', patched_file + '.orig_file', patched_file,
+ '--label', 'a/' + patched_file,
+ '--label', 'b/' + patched_file],
+ stdout=pfilef, close_fds=True,
+ universal_newlines=True, cwd=target_dir)
+ process.wait()
+ os.unlink(fullfn + '.orig_file')
+ if not process.returncode in (0, 1):
+ logwrite("Failed to diff to refresh %s" % print_name)
+ pfilef.close()
+ os.unlink(pfile + '.tmp')
+ return 2
+ pfilef.close()
+ os.rename(pfile + '.tmp', pfile)
+
+ # remove orig/rej files that patch sometimes creates
+ for root, dirs, files in os.walk(target_dir):
+ for f in files:
+ if f[-5:] == '.orig' or f[-4:] == '.rej':
+ os.unlink(os.path.join(root, f))
+ git_debug_snapshot(args, "apply %s patch %s" % (desc, print_name))
+
+ sempatches.sort()
+ prefix_len = len(os.path.join(source_dir, patch_src)) + 1
+
+ for cocci_file in sempatches:
+ # Until Coccinelle picks this up
+ pycocci = os.path.join(source_dir, 'devel/pycocci')
+ cmd = [pycocci, cocci_file]
+ extra_spatch_args = []
+ if args.profile_cocci:
+ cmd.append('--profile-cocci')
+ cmd.append(os.path.abspath(target_dir))
+ print_name = cocci_file[prefix_len:]
+ if args.verbose:
+ logwrite("Applying SmPL patch %s" % print_name)
+ logwrite(" %s" % cmd)
+ sprocess = subprocess.Popen(cmd,
+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+ close_fds=True, universal_newlines=True,
+ cwd=target_dir)
+ output = sprocess.communicate()[0]
+ sprocess.wait()
+ if sprocess.returncode != 0:
+ logwrite("Failed to process SmPL patch %s with %i" % (print_name, sprocess.returncode))
+ return 2
+ output = output.split('\n')
+ if output[-1] == '':
+ output = output[:-1]
+ if args.verbose:
+ for line in output:
+ logwrite('> %s' % line)
+
+ # remove cocci_backup files
+ for root, dirs, files in os.walk(target_dir):
+ for f in files:
+ if f.endswith('.cocci_backup'):
+ os.unlink(os.path.join(root, f))
+ git_debug_snapshot(args, "apply %s SmPL patch %s" % (desc, print_name))
+
+ if test_cocci and test_cocci_found:
+ logwrite('Done!')
+ sys.exit(0)
+
def _main():
+ # Our binary requirements go here
+ req = reqs.Req()
+ req.require('git')
+ req.coccinelle('1.0.6')
+ if not req.reqs_match():
+ sys.exit(1)
+
# set up and parse arguments
parser = argparse.ArgumentParser(description='generate backport tree')
parser.add_argument('kerneldir', metavar='<kernel tree>', type=str,
'and we use git ls-tree to get the files.')
parser.add_argument('--clean', const=True, default=False, action="store_const",
help='Clean output directory instead of erroring if it isn\'t empty')
+ parser.add_argument('--integrate', const=True, default=False, action="store_const",
+ help='Integrate a future backported kernel solution into ' +
+ 'an older kernel tree source directory.')
parser.add_argument('--refresh', const=True, default=False, action="store_const",
help='Refresh patches as they are applied, the source dir will be modified!')
parser.add_argument('--base-name', metavar='<name>', type=str, default='Linux',
help='name of base tree, default just "Linux"')
- parser.add_argument('--gitdebug', const=True, default=False, action="store_const",
+ parser.add_argument('--gitdebug', '--git-debug', const=True, default=False, action="store_const",
help='Use git, in the output tree, to debug the various transformation steps ' +
'that the tree generation makes (apply patches, ...)')
parser.add_argument('--verbose', const=True, default=False, action="store_const",
help='Only use the cocci file passed for Coccinelle, don\'t do anything else, ' +
'also creates a git repo on the target directory for easy inspection ' +
'of changes done by Coccinelle.')
+ parser.add_argument('--profile-cocci', metavar='<sp_file>', type=str, default=None,
+ help='Only use the cocci file passed and pass --profile to Coccinelle, ' +
+ 'also creates a git repo on the target directory for easy inspection ' +
+ 'of changes done by Coccinelle.')
args = parser.parse_args()
+ # When building a package we use CPTCFG as we can rely on the
+ # fact that kconfig treats CONFIG_ as an environment variable
+ # requring less changes on code. For kernel integration we use
+ # the longer CONFIG_BACKPORT given that we'll be sticking to
+ # the kernel symbol namespace, to address that we do a final
+ # search / replace. Technically its possible to rely on the
+ # same prefix for packaging as with kernel integration but
+ # there are already some users of the CPTCFG prefix.
+ bpid = None
+ if args.integrate:
+ bpid = Bp_Identity(integrate = args.integrate,
+ kconfig_prefix = 'CONFIG_',
+ project_prefix = 'BACKPORT_',
+ project_dir = args.outdir,
+ target_dir = os.path.join(args.outdir, 'backports/'),
+ target_dir_name = 'backports/',
+ kconfig_source_var = '$BACKPORT_DIR',
+ )
+ else:
+ bpid = Bp_Identity(integrate = args.integrate,
+ kconfig_prefix = 'CPTCFG_',
+ project_prefix = '',
+ project_dir = args.outdir,
+ target_dir = args.outdir,
+ target_dir_name = '',
+ kconfig_source_var = '$BACKPORT_DIR',
+ )
+
def logwrite(msg):
sys.stdout.write(msg)
sys.stdout.write('\n')
sys.stdout.flush()
- return process(args.kerneldir, args.outdir, args.copy_list,
- git_revision=args.git_revision, clean=args.clean,
+ return process(args.kerneldir, args.copy_list,
+ git_revision=args.git_revision,
+ bpid=bpid,
+ clean=args.clean,
refresh=args.refresh, base_name=args.base_name,
gitdebug=args.gitdebug, verbose=args.verbose,
extra_driver=args.extra_driver,
kup=args.kup,
kup_test=args.kup_test,
test_cocci=args.test_cocci,
+ profile_cocci=args.profile_cocci,
logwrite=logwrite)
-def process(kerneldir, outdir, copy_list_file, git_revision=None,
+def process(kerneldir, copy_list_file, git_revision=None,
+ bpid=None,
clean=False, refresh=False, base_name="Linux", gitdebug=False,
verbose=False, extra_driver=[], kup=False,
kup_test=False,
test_cocci=None,
+ profile_cocci=None,
logwrite=lambda x:None,
git_tracked_version=False):
class Args(object):
- def __init__(self, kerneldir, outdir, copy_list_file,
- git_revision, clean, refresh, base_name,
+ def __init__(self, kerneldir, copy_list_file,
+ git_revision, bpid, clean, refresh, base_name,
gitdebug, verbose, extra_driver, kup,
kup_test,
- test_cocci):
+ test_cocci,
+ profile_cocci):
self.kerneldir = kerneldir
- self.outdir = outdir
self.copy_list = copy_list_file
self.git_revision = git_revision
+ self.bpid = bpid
self.clean = clean
self.refresh = refresh
self.base_name = base_name
self.kup = kup
self.kup_test = kup_test
self.test_cocci = test_cocci
- if self.test_cocci:
+ self.profile_cocci = profile_cocci
+ if self.test_cocci or self.profile_cocci:
self.gitdebug = True
def git_paranoia(tree=None, logwrite=lambda x:None):
data = git.paranoia(tree)
else:
logwrite('Validated tree: %s' % tree)
- args = Args(kerneldir, outdir, copy_list_file,
- git_revision, clean, refresh, base_name,
+ args = Args(kerneldir, copy_list_file,
+ git_revision, bpid, clean, refresh, base_name,
gitdebug, verbose, extra_driver, kup, kup_test,
- test_cocci)
+ test_cocci, profile_cocci)
rel_prep = None
+ if bpid.integrate:
+ if args.kup_test or args.test_cocci or args.profile_cocci or args.refresh:
+ logwrite('Cannot use integration with:\n\tkup_test\n\ttest_cocci\n\tprofile_cocci\n\trefresh\n');
+ sys.exit(1)
+
# start processing ...
if (args.kup or args.kup_test):
git_paranoia(source_dir, logwrite)
git_paranoia(kerneldir, logwrite)
rel_describe = git.describe(rev=None, tree=source_dir, extra_args=['--dirty'])
- release = os.path.basename(args.outdir)
+ release = os.path.basename(bpid.target_dir)
version = release.replace("backports-", "")
rel_prep = get_rel_prep(version)
deplist = read_dependencies(os.path.join(source_dir, 'dependencies'))
# validate output directory
- check_output_dir(args.outdir, args.clean)
+ check_output_dir(bpid.target_dir, args.clean)
# do the copy
+ backport_integrate_files = [
+ ('Makefile.kernel', 'Makefile'),
+ ('Kconfig.integrate', 'Kconfig'),
+ ]
+ backport_package_files = [(x, x) for x in [
+ 'Makefile',
+ 'kconf/',
+ 'Makefile.real',
+ 'Makefile.kernel',
+ 'Kconfig.package.hacks',
+ 'scripts/',
+ '.blacklist.map',
+ '.gitignore',
+ 'Makefile.build'] ]
+ backport_package_files += [
+ ('Kconfig.package', 'Kconfig'),
+ ]
backport_files = [(x, x) for x in [
- 'Kconfig', 'Makefile', 'Makefile.build', 'Makefile.kernel', '.gitignore',
- 'Makefile.real', 'compat/', 'backport-include/', 'kconf/', 'defconfigs/',
- 'scripts/', '.blacklist.map', 'udev/',
+ 'Kconfig.sources',
+ 'compat/',
+ 'backport-include/',
]]
+
+ if not bpid.integrate:
+ backport_files += backport_package_files
+ else:
+ backport_files += backport_integrate_files
+
if not args.git_revision:
logwrite('Copy original source files ...')
else:
logwrite('Get original source files from git ...')
- copy_files(os.path.join(source_dir, 'backport'), backport_files, args.outdir)
+ copy_files(os.path.join(source_dir, 'backport'), backport_files, bpid.target_dir)
git_debug_init(args)
if not args.git_revision:
- copy_files(args.kerneldir, copy_list, args.outdir)
+ copy_files(args.kerneldir, copy_list, bpid.target_dir)
else:
- copy_git_files(args.kerneldir, copy_list, args.git_revision, args.outdir)
+ copy_git_files(args.kerneldir, copy_list, args.git_revision, bpid.target_dir)
# FIXME: should we add a git version of this (e.g. --git-extra-driver)?
for src, copy_list in args.extra_driver:
if (args.kup or args.kup_test):
git_paranoia(src)
- copy_files(src, read_copy_list(open(copy_list, 'r')), args.outdir)
+ copy_files(src, read_copy_list(open(copy_list, 'r')), bpid.target_dir)
git_debug_snapshot(args, 'Add driver sources')
disable_list = add_automatic_backports(args)
+ if git_tracked_version:
+ backports_version = "(see git)"
+ kernel_version = "(see git)"
+ else:
+ backports_version = git.describe(tree=source_dir, extra_args=['--long'])
+ kernel_version = git.describe(rev=args.git_revision or 'HEAD',
+ tree=args.kerneldir,
+ extra_args=['--long'])
+
+ if not bpid.integrate:
+ f = open(os.path.join(bpid.target_dir, 'versions'), 'w')
+ f.write('BACKPORTS_VERSION="%s"\n' % backports_version)
+ f.write('BACKPORTED_KERNEL_VERSION="%s"\n' % kernel_version)
+ f.write('BACKPORTED_KERNEL_NAME="%s"\n' % args.base_name)
+ if git_tracked_version:
+ f.write('BACKPORTS_GIT_TRACKED="backport tracker ID: $(shell git rev-parse HEAD 2>/dev/null || echo \'not built in git tree\')"\n')
+ f.close()
+ git_debug_snapshot(args, "add versions files")
+ else:
+ kconf_regexes = [
+ (re.compile(r'.*(?P<key>%%BACKPORT_DIR%%)'), '%%BACKPORT_DIR%%', 'backports/'),
+ (re.compile(r'.*(?P<key>%%BACKPORTS_VERSION%%).*'), '%%BACKPORTS_VERSION%%', backports_version),
+ (re.compile(r'.*(?P<key>%%BACKPORTED_KERNEL_VERSION%%).*'), '%%BACKPORTED_KERNEL_VERSION%%', kernel_version),
+ (re.compile(r'.*(?P<key>%%BACKPORTED_KERNEL_NAME%%).*'), '%%BACKPORTED_KERNEL_NAME%%', args.base_name),
+ ]
+ out = ''
+ for l in open(os.path.join(bpid.target_dir, 'Kconfig'), 'r'):
+ for r in kconf_regexes:
+ m = r[0].match(l)
+ if m:
+ l = re.sub(r'(' + r[1] + ')', r'' + r[2] + '', l)
+ out += l
+ outf = open(os.path.join(bpid.target_dir, 'Kconfig'), 'w')
+ outf.write(out)
+ outf.close()
+ git_debug_snapshot(args, "modify top level backports/Kconfig with backports identity")
+
if disable_list:
- bpcfg = kconfig.ConfigTree(os.path.join(args.outdir, 'compat', 'Kconfig'))
+ # No need to verify_sources() as compat's Kconfig has no 'source' call
+ bpcfg = kconfig.ConfigTree(os.path.join(bpid.target_dir, 'compat', 'Kconfig'), bpid)
bpcfg.disable_symbols(disable_list)
git_debug_snapshot(args, 'Add automatic backports')
- # Extend with other tests for Coccinelle
- test_cocci = args.test_cocci
-
- logwrite('Apply patches ...')
- patches = []
- sempatches = []
- for root, dirs, files in os.walk(os.path.join(source_dir, 'patches')):
- for f in files:
- if not test_cocci and f.endswith('.patch'):
- patches.append(os.path.join(root, f))
- if f.endswith('.cocci'):
- if test_cocci:
- if f not in test_cocci:
- continue
- if args.test_cocci:
- logwrite("Testing Coccinelle SmPL patch: %s" % test_cocci)
- sempatches.append(os.path.join(root, f))
- patches.sort()
- prefix_len = len(os.path.join(source_dir, 'patches')) + 1
- for pfile in patches:
- print_name = pfile[prefix_len:]
- # read the patch file
- p = patch.fromfile(pfile)
- # complain if it's not a patch
- if not p:
- raise Exception('No patch content found in %s' % print_name)
- # leading / seems to be stripped?
- if 'dev/null' in p.items[0].source:
- raise Exception('Patches creating files are not supported (in %s)' % print_name)
- # check if the first file the patch touches exists, if so
- # assume the patch needs to be applied -- otherwise continue
- patched_file = '/'.join(p.items[0].source.split('/')[1:])
- fullfn = os.path.join(args.outdir, patched_file)
- if not os.path.exists(fullfn):
- if args.verbose:
- logwrite("Not applying %s, not needed" % print_name)
- continue
- if args.verbose:
- logwrite("Applying patch %s" % print_name)
-
- if args.refresh:
- # but for refresh, of course look at all files the patch touches
- for patchitem in p.items:
- patched_file = '/'.join(patchitem.source.split('/')[1:])
- fullfn = os.path.join(args.outdir, patched_file)
- shutil.copyfile(fullfn, fullfn + '.orig_file')
+ failure = apply_patches(args, "backport", source_dir, 'patches', bpid.target_dir, logwrite)
+ if failure:
+ return failure
+
+ # Kernel integration requires Kconfig.versions already generated for you,
+ # we cannot do this for a package as we have no idea what kernel folks
+ # will be using.
+ if bpid.integrate:
+ kver = gen_version.kernelversion(bpid.project_dir)
+ rel_specs = gen_version.get_rel_spec_stable(kver)
+ if not rel_specs:
+ logwrite('Cannot parse source kernel version, update parser')
+ sys.exit(1)
+ data = gen_version.genkconfig_versions(rel_specs)
+ fo = open(os.path.join(bpid.target_dir, 'Kconfig.versions'), 'w')
+ fo.write(data)
+ fo.close()
+ git_debug_snapshot(args, "generate kernel version requirement Kconfig file")
- process = subprocess.Popen(['patch', '-p1'], stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT, stdin=subprocess.PIPE,
- close_fds=True, universal_newlines=True,
- cwd=args.outdir)
- output = process.communicate(input=open(pfile, 'r').read())[0]
- output = output.split('\n')
- if output[-1] == '':
- output = output[:-1]
- if args.verbose:
- for line in output:
- logwrite('> %s' % line)
- if process.returncode != 0:
- if not args.verbose:
- logwrite("Failed to apply changes from %s" % print_name)
- for line in output:
- logwrite('> %s' % line)
- return 2
+ # some post-processing is required
+ configtree = kconfig.ConfigTree(os.path.join(bpid.target_dir, 'Kconfig'), bpid)
+ ignore=['Kconfig.kernel', 'Kconfig.versions', 'Kconfig.local']
- if args.refresh:
- pfilef = open(pfile + '.tmp', 'a')
- pfilef.write(p.top_header)
- pfilef.flush()
- for patchitem in p.items:
- patched_file = '/'.join(patchitem.source.split('/')[1:])
- fullfn = os.path.join(args.outdir, patched_file)
- process = subprocess.Popen(['diff', '-p', '-u', patched_file + '.orig_file', patched_file,
- '--label', 'a/' + patched_file,
- '--label', 'b/' + patched_file],
- stdout=pfilef, close_fds=True,
- universal_newlines=True, cwd=args.outdir)
- process.wait()
- os.unlink(fullfn + '.orig_file')
- if not process.returncode in (0, 1):
- logwrite("Failed to diff to refresh %s" % print_name)
- pfilef.close()
- os.unlink(pfile + '.tmp')
- return 3
- pfilef.close()
- os.rename(pfile + '.tmp', pfile)
+ configtree.verify_sources(ignore=ignore)
+ git_debug_snapshot(args, "verify sources on top level backports Kconfig")
- # remove orig/rej files that patch sometimes creates
- for root, dirs, files in os.walk(args.outdir):
- for f in files:
- if f[-5:] == '.orig' or f[-4:] == '.rej':
- os.unlink(os.path.join(root, f))
- git_debug_snapshot(args, "apply backport patch %s" % print_name)
+ orig_symbols = configtree.symbols()
- sempatches.sort()
- with tempdir() as t:
- if not args.gitdebug:
- # combine all spatches
- fn = os.path.join(t, 'combined.cocci')
- f = open(fn, 'w')
- for cocci_file in sempatches:
- for l in open(cocci_file, 'r'):
- f.write(l)
- f.write('\n')
- f.close()
- sempatches = [fn]
- prefix_len = 0
- else:
- prefix_len = len(os.path.join(source_dir, 'patches')) + 1
- for cocci_file in sempatches:
- print_name = cocci_file[prefix_len:]
- if args.verbose:
- logwrite("Applying SmPL patch %s" % print_name)
-
- output = coccinelle.threaded_spatch(cocci_file, args.outdir,
- logwrite, print_name,
- test_cocci)
- output = output.split('\n')
- if output[-1] == '':
- output = output[:-1]
- if args.verbose:
- for line in output:
- logwrite('> %s' % line)
+ logwrite('Modify Kconfig tree ...')
+ configtree.prune_sources(ignore=ignore)
+ git_debug_snapshot(args, "prune Kconfig tree")
- # remove cocci_backup files
- for root, dirs, files in os.walk(args.outdir):
- for f in files:
- if f.endswith('.cocci_backup'):
- os.unlink(os.path.join(root, f))
- git_debug_snapshot(args, "apply backport SmPL patch %s" % print_name)
+ if not bpid.integrate:
+ configtree.force_tristate_modular()
+ git_debug_snapshot(args, "force tristate options modular")
- if test_cocci:
- logwrite('Done!')
- return 0
+ ignore = [os.path.join(bpid.target_dir, x) for x in [
+ 'Kconfig.package.hacks',
+ 'Kconfig.versions',
+ 'Kconfig.local',
+ 'Kconfig',
+ ]
+ ]
+ configtree.adjust_backported_configs(ignore=ignore, orig_symbols=orig_symbols)
+ git_debug_snapshot(args, "adjust backports config symbols we port")
- # some post-processing is required
- configtree = kconfig.ConfigTree(os.path.join(args.outdir, 'Kconfig'))
- logwrite('Modify Kconfig tree ...')
- configtree.prune_sources(ignore=['Kconfig.kernel', 'Kconfig.versions'])
- git_debug_snapshot(args, "prune Kconfig tree")
- configtree.force_tristate_modular()
- git_debug_snapshot(args, "force tristate options modular")
configtree.modify_selects()
git_debug_snapshot(args, "convert select to depends on")
- # write the versioning file
- if git_tracked_version:
- backports_version = "(see git)"
- kernel_version = "(see git)"
- else:
- backports_version = git.describe(tree=source_dir, extra_args=['--long'])
- kernel_version = git.describe(rev=args.git_revision or 'HEAD',
- tree=args.kerneldir,
- extra_args=['--long'])
- f = open(os.path.join(args.outdir, 'versions'), 'w')
- f.write('BACKPORTS_VERSION="%s"\n' % backports_version)
- f.write('BACKPORTED_KERNEL_VERSION="%s"\n' % kernel_version)
- f.write('BACKPORTED_KERNEL_NAME="%s"\n' % args.base_name)
- if git_tracked_version:
- f.write('BACKPORTS_GIT_TRACKED="backport tracker ID: $(shell git rev-parse HEAD 2>/dev/null || echo \'not built in git tree\')"\n')
- f.close()
-
symbols = configtree.symbols()
- # write local symbol list -- needed during build
- f = open(os.path.join(args.outdir, '.local-symbols'), 'w')
+ # write local symbol list -- needed during packaging build
+ if not bpid.integrate:
+ f = open(os.path.join(bpid.target_dir, 'local-symbols'), 'w')
+ for sym in symbols:
+ f.write('%s=\n' % sym)
+ f.close()
+ git_debug_snapshot(args, "add symbols files")
+ # also write Kconfig.local, representing all local symbols
+ # with a BACKPORTED_ prefix
+ f = open(os.path.join(bpid.target_dir, 'Kconfig.local'), 'w')
for sym in symbols:
- f.write('%s=\n' % sym)
+ f.write('config BACKPORTED_%s\n' % sym)
+ f.write('\ttristate\n')
+ f.write('\tdefault %s\n' % sym)
f.close()
-
- git_debug_snapshot(args, "add versions/symbols files")
+ git_debug_snapshot(args, "add Kconfig.local")
+
+ # add defconfigs that we want
+ defconfigs_dir = os.path.join(source_dir, 'backport', 'defconfigs')
+ os.mkdir(os.path.join(bpid.target_dir, 'defconfigs'))
+ for dfbase in os.listdir(defconfigs_dir):
+ copy_defconfig = True
+ dfsrc = os.path.join(defconfigs_dir, dfbase)
+ for line in open(dfsrc, 'r'):
+ if not '=' in line:
+ continue
+ line_ok = False
+ for sym in symbols:
+ if sym + '=' in line:
+ line_ok = True
+ break
+ if not line_ok:
+ copy_defconfig = False
+ break
+ if copy_defconfig:
+ shutil.copy(dfsrc, os.path.join(bpid.target_dir, 'defconfigs', dfbase))
+
+ git_debug_snapshot(args, "add (useful) defconfig files")
logwrite('Rewrite Makefiles and Kconfig files ...')
# rewrite Makefile and source symbols
+
+ # symbols we know only we can provide under the backport project prefix
+ # for which we need an exception.
+ skip_orig_syms = [ bpid.project_prefix + x for x in [
+ 'INTEGRATE',
+ ]
+ ]
+ parse_orig_syms = [x for x in orig_symbols if x not in skip_orig_syms ]
regexes = []
- for some_symbols in [symbols[i:i + 50] for i in range(0, len(symbols), 50)]:
+ for some_symbols in [parse_orig_syms[i:i + 50] for i in range(0, len(parse_orig_syms), 50)]:
r = 'CONFIG_((' + '|'.join([s + '(_MODULE)?' for s in some_symbols]) + ')([^A-Za-z0-9_]|$))'
regexes.append(re.compile(r, re.MULTILINE))
- for root, dirs, files in os.walk(args.outdir):
+ for root, dirs, files in os.walk(bpid.target_dir):
# don't go into .git dir (possible debug thing)
if '.git' in dirs:
dirs.remove('.git')
for f in files:
data = open(os.path.join(root, f), 'r').read()
for r in regexes:
- data = r.sub(r'CPTCFG_\1', data)
+ data = r.sub(r'' + bpid.full_prefix + '\\1', data)
+ # we have an absolue path in $(src) since we compile out of tree
+ data = re.sub(r'\$\(srctree\)/\$\(src\)', '$(src)', data)
data = re.sub(r'\$\(srctree\)', '$(backport_srctree)', data)
data = re.sub(r'-Idrivers', '-I$(backport_srctree)/drivers', data)
+ if bpid.integrate:
+ data = re.sub(r'CPTCFG_', bpid.full_prefix, data)
fo = open(os.path.join(root, f), 'w')
fo.write(data)
fo.close()
git_debug_snapshot(args, "rename config symbol / srctree usage")
# disable unbuildable Kconfig symbols and stuff Makefiles that doesn't exist
- maketree = make.MakeTree(os.path.join(args.outdir, 'Makefile.kernel'))
+ if bpid.integrate:
+ maketree = make.MakeTree(os.path.join(bpid.target_dir, 'Makefile'))
+ else:
+ maketree = make.MakeTree(os.path.join(bpid.target_dir, 'Makefile.kernel'))
disable_kconfig = []
disable_makefile = []
for sym in maketree.get_impossible_symbols():
elif (dep == "DISABLE"):
new.append('BACKPORT_DISABLED_KCONFIG_OPTION')
else:
- new.append('!BACKPORT_KERNEL_%s' % dep.replace('.', '_'))
- deplist[sym] = new
+ new.append('!KERNEL_%s' % dep.replace('.', '_'))
+ if bpid.integrate:
+ deplist[sym] = ["BACKPORT_" + x for x in new]
+ else:
+ deplist[sym] = new
configtree.add_dependencies(deplist)
git_debug_snapshot(args, "add kernel version dependencies")
# groups -- 50 seemed safer and is still fast)
regexes = []
for some_symbols in [disable_makefile[i:i + 50] for i in range(0, len(disable_makefile), 50)]:
- r = '^([^#].*((CPTCFG|CONFIG)_(' + '|'.join([s for s in some_symbols]) + ')))'
+ r = '^(([^#].*((' + bpid.full_prefix_resafe + '|CONFIG_)(' + '|'.join([s for s in some_symbols]) + ')))\W)'
regexes.append(re.compile(r, re.MULTILINE))
for f in maketree.get_makefiles():
data = open(f, 'r').read()
fo.close()
git_debug_snapshot(args, "disable unsatisfied Makefile parts")
+ if bpid.integrate:
+ f = open(os.path.join(bpid.project_dir, 'Kconfig'), 'a')
+ f.write('source "backports/Kconfig"\n')
+ f.close()
+ git_debug_snapshot(args, "hooked backport to top level Kconfig")
+
+ failure = apply_patches(args, "integration", source_dir, 'integration-patches/',
+ bpid.project_dir, logwrite)
+ if failure:
+ return failure
+
if (args.kup or args.kup_test):
+ req = reqs.Req()
+ req.kup()
+ if not req.reqs_match():
+ sys.exit(1)
upload_release(args, rel_prep, logwrite=logwrite)
logwrite('Done!')
projects / openwrt / staging / blogic.git / blobdiff