3 # Generate the output tree into a specified directory.
6 import argparse
, sys
, os
, errno
, shutil
, re
, subprocess
10 source_dir
= os
.path
.abspath(os
.path
.dirname(__file__
))
11 sys
.path
.append(source_dir
)
12 # and import libraries we have
13 from lib
import kconfig
, patch
, make
14 from lib
import bpgit
as git
15 from lib
import bpgpg
as gpg
16 from lib
import bpkup
as kup
17 from lib
.tempdir
import tempdir
18 from lib
import bpreqs
as reqs
20 class Bp_Identity(object):
22 folks considering multiple integrations may want to
23 consider stuffing versioning info here as well but
24 that will need thought/design on sharing compat and
27 Use the *_resafe when combining on regexps, although we currently
28 don't support regexps there perhaps later we will and this will
29 just make things safer for the output regardless. Once and if those
30 are added, how we actually use the others for regular printing will
31 need to be considered.
33 def __init__(self
, integrate
=False, kconfig_prefix
='CPTCFG_',
34 project_prefix
='', project_dir
='',
35 target_dir
='', target_dir_name
='',
36 kconfig_source_var
=None):
37 self
.integrate
= integrate
38 self
.kconfig_prefix
= kconfig_prefix
39 self
.kconfig_prefix_resafe
= re
.escape(kconfig_prefix
)
40 self
.project_prefix
= project_prefix
41 self
.project_prefix_resafe
= re
.escape(project_prefix
)
42 self
.full_prefix
= kconfig_prefix
+ project_prefix
43 self
.full_prefix_resafe
= re
.escape(self
.full_prefix
)
44 self
.project_dir
= project_dir
45 self
.target_dir
= target_dir
46 self
.target_dir_name
= target_dir_name
47 self
.kconfig_source_var
= kconfig_source_var
48 if self
.kconfig_source_var
:
49 self
.kconfig_source_var_resafe
= re
.escape(self
.kconfig_source_var
)
51 self
.kconfig_source_var_resafe
= None
53 def read_copy_list(copyfile
):
55 Read a copy-list file and return a list of (source, target)
56 tuples. The source and target are usually the same, but in
57 the copy-list file there may be a rename included.
61 # remove leading/trailing whitespace
64 if not item
or item
[0] == '#':
67 raise Exception("Input path '%s' is absolute path, this isn't allowed" % (item
, ))
69 srcitem
, dstitem
= item
.split(' -> ')
70 if (srcitem
[-1] == '/') != (dstitem
[-1] == '/'):
71 raise Exception("Cannot copy file/dir to dir/file")
73 srcitem
= dstitem
= item
74 ret
.append((srcitem
, dstitem
))
78 def read_dependencies(depfilename
):
80 Read a (the) dependency file and return the list of
81 dependencies as a dictionary, mapping a Kconfig symbol
82 to a list of kernel version dependencies.
84 If a backported feature that an upstream backported driver
85 depends on had kconfig limitations (ie, debugging feature not
86 available) a built constaint restriction can be expressed
87 by using a kconfig expression. The kconfig expressions can
88 be specified by using the "kconfig: " prefix.
90 While reading ignore blank or commented lines.
93 depfile
= open(depfilename
, 'r')
97 if not item
or item
[0] == '#':
99 if "kconfig:" in item
:
100 sym
, kconfig_exp
= item
.split(" ", 1)
102 ret
[sym
] = [kconfig_exp
, ]
104 ret
[sym
].append(kconfig_exp
)
106 sym
, dep
= item
.split()
114 def check_output_dir(d
, clean
):
116 Check that the output directory doesn't exist or is empty,
117 unless clean is True in which case it's nuked. This helps
118 sanity check the output when generating a tree, so usually
119 running with --clean isn't suggested.
122 shutil
.rmtree(d
, ignore_errors
=True)
126 if e
.errno
!= errno
.ENOENT
:
130 def copytree(src
, dst
, symlinks
=False, ignore
=None):
132 Copy a directory tree. This differs from shutil.copytree()
133 in that it allows destination directories to already exist.
135 names
= os
.listdir(src
)
136 if ignore
is not None:
137 ignored_names
= ignore(src
, names
)
139 ignored_names
= set()
141 if not os
.path
.isdir(dst
):
145 if name
in ignored_names
:
147 srcname
= os
.path
.join(src
, name
)
148 dstname
= os
.path
.join(dst
, name
)
150 if symlinks
and os
.path
.islink(srcname
):
151 linkto
= os
.readlink(srcname
)
152 os
.symlink(linkto
, dstname
)
153 elif os
.path
.isdir(srcname
):
154 copytree(srcname
, dstname
, symlinks
, ignore
)
156 shutil
.copy2(srcname
, dstname
)
157 except (IOError, os
.error
) as why
:
158 errors
.append((srcname
, dstname
, str(why
)))
159 # catch the Error from the recursive copytree so that we can
160 # continue with other files
161 except shutil
.Error
as err
:
162 errors
.extend(err
.args
[0])
164 shutil
.copystat(src
, dst
)
166 # can't copy file access times on Windows
168 except OSError as why
:
169 errors
.extend((src
, dst
, str(why
)))
171 raise shutil
.Error(errors
)
174 def copy_files(srcpath
, copy_list
, outdir
):
176 Copy the copy_list files and directories from the srcpath
177 to the outdir. The copy_list contains source and target
180 For now, it also ignores any *~ editor backup files, though
181 this should probably be generalized (maybe using .gitignore?)
182 Similarly the code that only copies some files (*.c, *.h,
183 *.awk, Kconfig, Makefile) to avoid any build remnants in the
184 kernel if they should exist.
186 for srcitem
, tgtitem
in copy_list
:
188 copytree(srcpath
, outdir
, ignore
=shutil
.ignore_patterns('*~'))
189 elif tgtitem
[-1] == '/':
190 def copy_ignore(dir, entries
):
193 if i
[-2:] == '.o' or i
[-1] == '~':
196 copytree(os
.path
.join(srcpath
, srcitem
),
197 os
.path
.join(outdir
, tgtitem
),
201 os
.makedirs(os
.path
.join(outdir
, os
.path
.dirname(tgtitem
)))
203 # ignore dirs we might have created just now
204 if e
.errno
!= errno
.EEXIST
:
206 shutil
.copy(os
.path
.join(srcpath
, srcitem
),
207 os
.path
.join(outdir
, tgtitem
))
210 def copy_git_files(srcpath
, copy_list
, rev
, outdir
):
212 "Copy" files from a git repository. This really means listing them with
213 ls-tree and then using git show to obtain all the blobs.
215 for srcitem
, tgtitem
in copy_list
:
216 for m
, t
, h
, f
in git
.ls_tree(rev
=rev
, files
=(srcitem
,), tree
=srcpath
):
218 f
= os
.path
.join(outdir
, f
.replace(srcitem
, tgtitem
))
219 d
= os
.path
.dirname(f
)
220 if not os
.path
.exists(d
):
223 git
.get_blob(h
, outf
, tree
=srcpath
)
225 os
.chmod(f
, int(m
, 8))
227 def automatic_backport_mangle_c_file(name
):
228 return name
.replace('/', '-')
231 def add_automatic_backports(args
):
233 export
= re
.compile(r
'^EXPORT_SYMBOL(_GPL)?\((?P<sym>[^\)]*)\)')
234 bpi
= kconfig
.get_backport_info(os
.path
.join(args
.bpid
.target_dir
, 'compat', 'Kconfig'))
235 configtree
= kconfig
.ConfigTree(os
.path
.join(args
.bpid
.target_dir
, 'Kconfig'), args
.bpid
)
236 all_selects
= configtree
.all_selects()
237 for sym
, vals
in bpi
.items():
238 if sym
.startswith('BACKPORT_BUILD_'):
239 if not sym
[15:] in all_selects
:
240 disable_list
.append(sym
)
242 symtype
, module_name
, c_files
, h_files
= vals
247 files
.append((f
, os
.path
.join('compat', automatic_backport_mangle_c_file(f
))))
249 files
.append((os
.path
.join('include', f
),
250 os
.path
.join('include', os
.path
.dirname(f
), 'backport-' + os
.path
.basename(f
))))
251 if args
.git_revision
:
252 copy_git_files(args
.kerneldir
, files
, args
.git_revision
, args
.bpid
.target_dir
)
254 copy_files(args
.kerneldir
, files
, args
.bpid
.target_dir
)
256 # now add the Makefile line
257 mf
= open(os
.path
.join(args
.bpid
.target_dir
, 'compat', 'Makefile'), 'a+')
258 o_files
= [automatic_backport_mangle_c_file(f
)[:-1] + 'o' for f
in c_files
]
259 if symtype
== 'tristate':
261 raise Exception('backporting a module requires a #module-name')
263 mf
.write('%s-objs += %s\n' % (module_name
, of
))
264 mf
.write('obj-$(%s%s) += %s.o\n' % (args
.bpid
.full_prefix
, sym
, module_name
))
265 elif symtype
== 'bool':
266 mf
.write('compat-$(%s%s) += %s\n' % (args
.bpid
.full_prefix
, sym
, ' '.join(o_files
)))
268 # finally create the include file
271 for l
in open(os
.path
.join(args
.bpid
.target_dir
, 'compat',
272 automatic_backport_mangle_c_file(f
)), 'r'):
275 syms
.append(m
.group('sym'))
277 outf
= open(os
.path
.join(args
.bpid
.target_dir
, 'include', f
), 'w')
278 outf
.write('/* Automatically created during backport process */\n')
279 outf
.write('#ifndef %s%s\n' % (args
.bpid
.full_prefix
, sym
))
280 outf
.write('#include_next <%s>\n' % f
)
281 outf
.write('#else\n');
283 outf
.write('#undef %s\n' % s
)
284 outf
.write('#define %s LINUX_BACKPORT(%s)\n' % (s
, s
))
285 outf
.write('#include <%s>\n' % (os
.path
.dirname(f
) + '/backport-' + os
.path
.basename(f
), ))
286 outf
.write('#endif /* %s%s */\n' % (args
.bpid
.full_prefix
, sym
))
289 def git_debug_init(args
):
291 Initialize a git repository in the output directory and commit the current
292 code in it. This is only used for debugging the transformations this code
293 will do to the output later.
295 if not args
.gitdebug
:
297 # Git supports re-initialization, although not well documented it can
298 # reset config stuff, lets avoid that if the tree already exists.
299 if not os
.path
.exists(os
.path
.join(args
.bpid
.project_dir
, '.git')):
300 git
.init(tree
=args
.bpid
.project_dir
)
301 git
.commit_all("Copied backport", tree
=args
.bpid
.project_dir
)
304 def git_debug_snapshot(args
, name
):
306 Take a git snapshot for the debugging.
308 if not args
.gitdebug
:
310 git
.commit_all(name
, tree
=args
.bpid
.project_dir
)
312 def get_rel_spec_stable(rel
):
314 Returns release specs for a linux-stable backports based release.
317 m
= re
.match(r
"(?P<VERSION>\d+)\.+" \
318 "(?P<PATCHLEVEL>\d+)[.]*" \
319 "(?P<SUBLEVEL>\d*)" \
320 "[-rc]+(?P<RC_VERSION>\d+)\-+" \
321 "(?P<RELMOD_UPDATE>\d+)[-]*" \
322 "(?P<RELMOD_TYPE>[usnpc]*)", \
325 m
= re
.match(r
"(?P<VERSION>\d+)\.+" \
326 "(?P<PATCHLEVEL>\d+)[.]*" \
327 "(?P<SUBLEVEL>\d*)\-+" \
328 "(?P<RELMOD_UPDATE>\d+)[-]*" \
329 "(?P<RELMOD_TYPE>[usnpc]*)", \
335 def get_rel_spec_next(rel
):
337 Returns release specs for a linux-next backports based release.
339 m
= re
.match(r
"(?P<DATE_VERSION>\d+)[-]*" \
340 "(?P<RELMOD_UPDATE>\d*)[-]*" \
341 "(?P<RELMOD_TYPE>[usnpc]*)", \
347 def get_rel_prep(rel
):
349 Returns a dict with prep work details we need prior to
350 uploading a backports release to kernel.org
352 rel_specs
= get_rel_spec_stable(rel
)
357 rel_specs
= get_rel_spec_next(rel
)
359 sys
.stdout
.write("rel: %s\n" % rel
)
361 if (rel_specs
['RELMOD_UPDATE'] == '0' or
362 rel_specs
['RELMOD_UPDATE'] == '1'):
365 date
= rel_specs
['DATE_VERSION']
370 if (len(month
) != 2):
378 rel_tag
= "backports-" + rel
.replace(rel_specs
['RELMOD_TYPE'], "")
381 if (not rel_specs
['RELMOD_UPDATE']):
383 if (rel_specs
['RELMOD_UPDATE'] == '0'):
385 ignore
+= rel_specs
['RELMOD_UPDATE']
386 if (rel_specs
['RELMOD_TYPE'] != ''):
387 ignore
+= rel_specs
['RELMOD_TYPE']
388 base_rel
= rel
.replace(ignore
, "")
389 paths
.append("v" + base_rel
)
390 rel_tag
= "v" + rel
.replace(rel_specs
['RELMOD_TYPE'], "")
392 rel_prep
= dict(stable
= is_stable
,
393 expected_tag
= rel_tag
,
394 paths_to_create
= paths
)
397 def create_tar_and_gz(tar_name
, dir_to_tar
):
399 We need both a tar file and gzip for kernel.org, the tar file
400 gets signed, then we upload the compressed version, kup-server
401 in the backend decompresses and verifies the tarball against
404 basename
= os
.path
.basename(dir_to_tar
)
405 tar
= tarfile
.open(tar_name
, "w")
406 tar
.add(dir_to_tar
, basename
)
409 tar_file
= open(tar_name
, "r")
411 gz_file
= gzip
.GzipFile(tar_name
+ ".gz", 'wb')
412 gz_file
.write(tar_file
.read())
415 def upload_release(args
, rel_prep
, logwrite
=lambda x
:None):
417 Given a path of a relase make tarball out of it, PGP sign it, and
418 then upload it to kernel.org using kup.
420 The linux-next based release do not require a RELMOD_UPDATE
421 given that typically only one release is made per day. Using
422 RELMOD_UPDATE for these releases is allowed though and if
423 present it must be > 1.
425 The linux-stable based releases require a RELMOD_UPDATE.
427 RELMOD_UPDATE must be numeric and > 0 just as the RC releases
430 The tree must also be tagged with the respective release, without
431 the RELMOD_TYPE. For linux-next based releases this consists of
432 backports- followed by DATE_VERSION and if RELMOD_TYPE is present.
433 For linux-stable releases this consists of v followed by the
434 full release version except the RELMOD_TYPE.
436 Uploads will not be allowed if these rules are not followed.
438 korg_path
= "/pub/linux/kernel/projects/backports"
440 if (rel_prep
['stable']):
441 korg_path
+= "/stable"
443 parent
= os
.path
.dirname(args
.bpid
.project_dir
)
444 release
= os
.path
.basename(args
.bpid
.project_dir
)
445 tar_name
= parent
+ '/' + release
+ ".tar"
446 gzip_name
= tar_name
+ ".gz"
448 create_tar_and_gz(tar_name
, args
.bpid
.project_dir
)
450 logwrite(gpg
.sign(tar_name
, extra_args
=['--armor', '--detach-sign']))
452 logwrite("------------------------------------------------------")
454 if (not args
.kup_test
):
455 logwrite("About to upload, current target path contents:")
457 logwrite("kup-test: current target path contents:")
459 logwrite(kup
.ls(path
=korg_path
))
461 for path
in rel_prep
['paths_to_create']:
462 korg_path
+= '/' + path
463 if (not args
.kup_test
):
464 logwrite("create directory: %s" % korg_path
)
465 logwrite(kup
.mkdir(korg_path
))
467 if (not args
.kup_test
):
468 logwrite("upload file %s to %s" % (gzip_name
, korg_path
))
469 logwrite(kup
.put(gzip_name
, tar_name
+ '.asc', korg_path
))
470 logwrite("\nFinished upload!\n")
471 logwrite("Target path contents:")
472 logwrite(kup
.ls(path
=korg_path
))
474 kup_cmd
= "kup put /\n\t\t%s /\n\t\t%s /\n\t\t%s" % (gzip_name
, tar_name
+ '.asc', korg_path
)
475 logwrite("kup-test: skipping cmd: %s" % kup_cmd
)
477 def apply_patches(args
, desc
, source_dir
, patch_src
, target_dir
, logwrite
=lambda x
:None):
479 Given a path of a directories of patches and SmPL patches apply
480 them on the target directory. If requested refresh patches, or test
481 a specific SmPL patch.
483 logwrite('Applying patches from %s to %s ...' % (patch_src
, target_dir
))
484 test_cocci
= args
.test_cocci
or args
.profile_cocci
485 test_cocci_found
= False
488 for root
, dirs
, files
in os
.walk(os
.path
.join(source_dir
, patch_src
)):
490 if not test_cocci
and f
.endswith('.patch'):
491 patches
.append(os
.path
.join(root
, f
))
492 if f
.endswith('.cocci'):
494 if f
not in test_cocci
:
496 test_cocci_found
= True
498 logwrite("Testing Coccinelle SmPL patch: %s" % test_cocci
)
499 elif args
.profile_cocci
:
500 logwrite("Profiling Coccinelle SmPL patch: %s" % test_cocci
)
501 sempatches
.append(os
.path
.join(root
, f
))
503 prefix_len
= len(os
.path
.join(source_dir
, patch_src
)) + 1
504 for pfile
in patches
:
505 print_name
= pfile
[prefix_len
:]
506 # read the patch file
507 p
= patch
.fromfile(pfile
)
508 # complain if it's not a patch
510 raise Exception('No patch content found in %s' % print_name
)
511 # leading / seems to be stripped?
512 if 'dev/null' in p
.items
[0].source
:
513 raise Exception('Patches creating files are not supported (in %s)' % print_name
)
514 # check if the first file the patch touches exists, if so
515 # assume the patch needs to be applied -- otherwise continue
516 patched_file
= '/'.join(p
.items
[0].source
.split('/')[1:])
517 fullfn
= os
.path
.join(target_dir
, patched_file
)
518 if not os
.path
.exists(fullfn
):
520 logwrite("Not applying %s, not needed" % print_name
)
523 logwrite("Applying patch %s" % print_name
)
526 # but for refresh, of course look at all files the patch touches
527 for patchitem
in p
.items
:
528 patched_file
= '/'.join(patchitem
.source
.split('/')[1:])
529 fullfn
= os
.path
.join(target_dir
, patched_file
)
530 shutil
.copyfile(fullfn
, fullfn
+ '.orig_file')
532 process
= subprocess
.Popen(['patch', '-p1'], stdout
=subprocess
.PIPE
,
533 stderr
=subprocess
.STDOUT
, stdin
=subprocess
.PIPE
,
534 close_fds
=True, universal_newlines
=True,
536 output
= process
.communicate(input=open(pfile
, 'r').read())[0]
537 output
= output
.split('\n')
542 logwrite('> %s' % line
)
543 if process
.returncode
!= 0:
545 logwrite("Failed to apply changes from %s" % print_name
)
547 logwrite('> %s' % line
)
548 raise Exception('Patch failed')
551 pfilef
= open(pfile
+ '.tmp', 'a')
552 pfilef
.write(p
.top_header
)
554 for patchitem
in p
.items
:
555 patched_file
= '/'.join(patchitem
.source
.split('/')[1:])
556 fullfn
= os
.path
.join(target_dir
, patched_file
)
557 process
= subprocess
.Popen(['diff', '-p', '-u', patched_file
+ '.orig_file', patched_file
,
558 '--label', 'a/' + patched_file
,
559 '--label', 'b/' + patched_file
],
560 stdout
=pfilef
, close_fds
=True,
561 universal_newlines
=True, cwd
=target_dir
)
563 os
.unlink(fullfn
+ '.orig_file')
564 if not process
.returncode
in (0, 1):
565 logwrite("Failed to diff to refresh %s" % print_name
)
567 os
.unlink(pfile
+ '.tmp')
568 raise Exception('Refresh failed')
570 os
.rename(pfile
+ '.tmp', pfile
)
572 # remove orig/rej files that patch sometimes creates
573 for root
, dirs
, files
in os
.walk(target_dir
):
575 if f
[-5:] == '.orig' or f
[-4:] == '.rej':
576 os
.unlink(os
.path
.join(root
, f
))
577 git_debug_snapshot(args
, "apply %s patch %s" % (desc
, print_name
))
580 prefix_len
= len(os
.path
.join(source_dir
, patch_src
)) + 1
582 for cocci_file
in sempatches
:
583 # Until Coccinelle picks this up
584 pycocci
= os
.path
.join(source_dir
, 'devel/pycocci')
585 cmd
= [pycocci
, cocci_file
]
586 extra_spatch_args
= []
587 if args
.profile_cocci
:
588 cmd
.append('--profile-cocci')
589 cmd
.append(os
.path
.abspath(target_dir
))
590 print_name
= cocci_file
[prefix_len
:]
592 logwrite("Applying SmPL patch %s" % print_name
)
593 sprocess
= subprocess
.Popen(cmd
,
594 stdout
=subprocess
.PIPE
, stderr
=subprocess
.STDOUT
,
595 close_fds
=True, universal_newlines
=True,
597 output
= sprocess
.communicate()[0]
599 if sprocess
.returncode
!= 0:
600 logwrite("Failed to process SmPL patch %s" % print_name
)
601 raise Exception('SmPL patch failed')
602 output
= output
.split('\n')
607 logwrite('> %s' % line
)
609 # remove cocci_backup files
610 for root
, dirs
, files
in os
.walk(target_dir
):
612 if f
.endswith('.cocci_backup'):
613 os
.unlink(os
.path
.join(root
, f
))
614 git_debug_snapshot(args
, "apply %s SmPL patch %s" % (desc
, print_name
))
616 if test_cocci
and test_cocci_found
:
621 # Our binary requirements go here
624 req
.coccinelle('1.0.0-rc21')
625 if not req
.reqs_match():
628 # set up and parse arguments
629 parser
= argparse
.ArgumentParser(description
='generate backport tree')
630 parser
.add_argument('kerneldir', metavar
='<kernel tree>', type=str,
631 help='Kernel tree to copy drivers from')
632 parser
.add_argument('outdir', metavar
='<output directory>', type=str,
633 help='Directory to write the generated tree to')
634 parser
.add_argument('--copy-list', metavar
='<listfile>', type=argparse
.FileType('r'),
636 help='File containing list of files/directories to copy, default "copy-list"')
637 parser
.add_argument('--git-revision', metavar
='<revision>', type=str,
638 help='git commit revision (see gitrevisions(7)) to take objects from.' +
639 'If this is specified, the kernel tree is used as git object storage ' +
640 'and we use git ls-tree to get the files.')
641 parser
.add_argument('--clean', const
=True, default
=False, action
="store_const",
642 help='Clean output directory instead of erroring if it isn\'t empty')
643 parser
.add_argument('--refresh', const
=True, default
=False, action
="store_const",
644 help='Refresh patches as they are applied, the source dir will be modified!')
645 parser
.add_argument('--base-name', metavar
='<name>', type=str, default
='Linux',
646 help='name of base tree, default just "Linux"')
647 parser
.add_argument('--gitdebug', const
=True, default
=False, action
="store_const",
648 help='Use git, in the output tree, to debug the various transformation steps ' +
649 'that the tree generation makes (apply patches, ...)')
650 parser
.add_argument('--verbose', const
=True, default
=False, action
="store_const",
651 help='Print more verbose information')
652 parser
.add_argument('--extra-driver', nargs
=2, metavar
=('<source dir>', '<copy-list>'), type=str,
653 action
='append', default
=[], help='Extra driver directory/copy-list.')
654 parser
.add_argument('--kup', const
=True, default
=False, action
="store_const",
655 help='For maintainers: upload a release to kernel.org')
656 parser
.add_argument('--kup-test', const
=True, default
=False, action
="store_const",
657 help='For maintainers: do all the work as if you were about to ' +
658 'upload to kernel.org but do not do the final `kup put` ' +
659 'and also do not run any `kup mkdir` commands. This will ' +
660 'however run `kup ls` on the target paths so ' +
661 'at the very least we test your kup configuration. ' +
662 'If this is your first time uploading use this first!')
663 parser
.add_argument('--test-cocci', metavar
='<sp_file>', type=str, default
=None,
664 help='Only use the cocci file passed for Coccinelle, don\'t do anything else, ' +
665 'also creates a git repo on the target directory for easy inspection ' +
666 'of changes done by Coccinelle.')
667 parser
.add_argument('--profile-cocci', metavar
='<sp_file>', type=str, default
=None,
668 help='Only use the cocci file passed and pass --profile to Coccinelle, ' +
669 'also creates a git repo on the target directory for easy inspection ' +
670 'of changes done by Coccinelle.')
671 args
= parser
.parse_args()
673 # When building a package we use CPTCFG as we can rely on the
674 # fact that kconfig treats CONFIG_ as an environment variable
675 # requring less changes on code. For kernel integration we use
676 # the longer CONFIG_BACKPORT given that we'll be sticking to
677 # the kernel symbol namespace, to address that we do a final
678 # search / replace. Technically its possible to rely on the
679 # same prefix for packaging as with kernel integration but
680 # there are already some users of the CPTCFG prefix.
684 bpid
= Bp_Identity(integrate
= integrate
,
685 kconfig_prefix
= 'CONFIG_',
686 project_prefix
= 'BACKPORT_',
687 project_dir
= args
.outdir
,
688 target_dir
= os
.path
.join(args
.outdir
, 'backports/'),
689 target_dir_name
= 'backports/',
690 kconfig_source_var
= '$BACKPORT_DIR',
693 bpid
= Bp_Identity(integrate
= integrate
,
694 kconfig_prefix
= 'CPTCFG_',
696 project_dir
= args
.outdir
,
697 target_dir
= args
.outdir
,
698 target_dir_name
= '',
699 kconfig_source_var
= '$BACKPORT_DIR',
703 sys
.stdout
.write(msg
)
704 sys
.stdout
.write('\n')
707 return process(args
.kerneldir
, args
.copy_list
,
708 git_revision
=args
.git_revision
,
711 refresh
=args
.refresh
, base_name
=args
.base_name
,
712 gitdebug
=args
.gitdebug
, verbose
=args
.verbose
,
713 extra_driver
=args
.extra_driver
,
715 kup_test
=args
.kup_test
,
716 test_cocci
=args
.test_cocci
,
717 profile_cocci
=args
.profile_cocci
,
720 def process(kerneldir
, copy_list_file
, git_revision
=None,
722 clean
=False, refresh
=False, base_name
="Linux", gitdebug
=False,
723 verbose
=False, extra_driver
=[], kup
=False,
727 logwrite
=lambda x
:None,
728 git_tracked_version
=False):
730 def __init__(self
, kerneldir
, copy_list_file
,
731 git_revision
, bpid
, clean
, refresh
, base_name
,
732 gitdebug
, verbose
, extra_driver
, kup
,
736 self
.kerneldir
= kerneldir
737 self
.copy_list
= copy_list_file
738 self
.git_revision
= git_revision
741 self
.refresh
= refresh
742 self
.base_name
= base_name
743 self
.gitdebug
= gitdebug
744 self
.verbose
= verbose
745 self
.extra_driver
= extra_driver
747 self
.kup_test
= kup_test
748 self
.test_cocci
= test_cocci
749 self
.profile_cocci
= profile_cocci
750 if self
.test_cocci
or self
.profile_cocci
:
752 def git_paranoia(tree
=None, logwrite
=lambda x
:None):
753 data
= git
.paranoia(tree
)
755 logwrite('Cannot use %s' % tree
)
756 logwrite('%s' % data
['output'])
759 logwrite('Validated tree: %s' % tree
)
761 args
= Args(kerneldir
, copy_list_file
,
762 git_revision
, bpid
, clean
, refresh
, base_name
,
763 gitdebug
, verbose
, extra_driver
, kup
, kup_test
,
764 test_cocci
, profile_cocci
)
767 # start processing ...
768 if (args
.kup
or args
.kup_test
):
769 git_paranoia(source_dir
, logwrite
)
770 git_paranoia(kerneldir
, logwrite
)
772 rel_describe
= git
.describe(rev
=None, tree
=source_dir
, extra_args
=['--dirty'])
773 release
= os
.path
.basename(bpid
.target_dir
)
774 version
= release
.replace("backports-", "")
776 rel_prep
= get_rel_prep(version
)
778 logwrite('Invalid backports release name: %s' % release
)
779 logwrite('For rules on the release name see upload_release()')
781 rel_type
= "linux-stable"
782 if (not rel_prep
['stable']):
783 rel_type
= "linux-next"
784 if (rel_prep
['expected_tag'] != rel_describe
):
785 logwrite('Unexpected %s based backports release tag on' % rel_type
)
786 logwrite('the backports tree tree: %s\n' % rel_describe
)
787 logwrite('You asked to make a release with this ')
788 logwrite('directory name: %s' % release
)
789 logwrite('The actual expected tag we should find on')
790 logwrite('the backports tree then is: %s\n' % rel_prep
['expected_tag'])
791 logwrite('For rules on the release name see upload_release()')
794 copy_list
= read_copy_list(args
.copy_list
)
795 deplist
= read_dependencies(os
.path
.join(source_dir
, 'dependencies'))
797 # validate output directory
798 check_output_dir(bpid
.target_dir
, args
.clean
)
801 backport_package_files
= [(x
, x
) for x
in [
806 'Kconfig.package.hacks',
811 backport_package_files
+= [
812 ('Kconfig.package', 'Kconfig'),
814 backport_files
= [(x
, x
) for x
in [
820 if not bpid
.integrate
:
821 backport_files
+= backport_package_files
823 if not args
.git_revision
:
824 logwrite('Copy original source files ...')
826 logwrite('Get original source files from git ...')
828 copy_files(os
.path
.join(source_dir
, 'backport'), backport_files
, bpid
.target_dir
)
832 if not args
.git_revision
:
833 copy_files(args
.kerneldir
, copy_list
, bpid
.target_dir
)
835 copy_git_files(args
.kerneldir
, copy_list
, args
.git_revision
, bpid
.target_dir
)
837 # FIXME: should we add a git version of this (e.g. --git-extra-driver)?
838 for src
, copy_list
in args
.extra_driver
:
839 if (args
.kup
or args
.kup_test
):
841 copy_files(src
, read_copy_list(open(copy_list
, 'r')), bpid
.target_dir
)
843 git_debug_snapshot(args
, 'Add driver sources')
845 disable_list
= add_automatic_backports(args
)
848 # No need to verify_sources() as compat's Kconfig has no 'source' call
849 bpcfg
= kconfig
.ConfigTree(os
.path
.join(bpid
.target_dir
, 'compat', 'Kconfig'), bpid
)
850 bpcfg
.disable_symbols(disable_list
)
851 git_debug_snapshot(args
, 'Add automatic backports')
853 apply_patches(args
, "backport", source_dir
, 'patches', bpid
.target_dir
, logwrite
)
855 # some post-processing is required
856 configtree
= kconfig
.ConfigTree(os
.path
.join(bpid
.target_dir
, 'Kconfig'), bpid
)
857 ignore
=['Kconfig.kernel', 'Kconfig.versions']
859 configtree
.verify_sources(ignore
=ignore
)
860 git_debug_snapshot(args
, "verify sources on top level backports Kconfig")
862 orig_symbols
= configtree
.symbols()
864 logwrite('Modify Kconfig tree ...')
865 configtree
.prune_sources(ignore
=ignore
)
866 git_debug_snapshot(args
, "prune Kconfig tree")
868 if not bpid
.integrate
:
869 configtree
.force_tristate_modular()
870 git_debug_snapshot(args
, "force tristate options modular")
872 configtree
.modify_selects()
873 git_debug_snapshot(args
, "convert select to depends on")
875 # write the versioning file
876 if git_tracked_version
:
877 backports_version
= "(see git)"
878 kernel_version
= "(see git)"
880 backports_version
= git
.describe(tree
=source_dir
, extra_args
=['--long'])
881 kernel_version
= git
.describe(rev
=args
.git_revision
or 'HEAD',
883 extra_args
=['--long'])
884 f
= open(os
.path
.join(bpid
.target_dir
, 'versions'), 'w')
885 f
.write('BACKPORTS_VERSION="%s"\n' % backports_version
)
886 f
.write('BACKPORTED_KERNEL_VERSION="%s"\n' % kernel_version
)
887 f
.write('BACKPORTED_KERNEL_NAME="%s"\n' % args
.base_name
)
888 if git_tracked_version
:
889 f
.write('BACKPORTS_GIT_TRACKED="backport tracker ID: $(shell git rev-parse HEAD 2>/dev/null || echo \'not built in git tree\')"\n')
891 git_debug_snapshot(args
, "add versions files")
893 symbols
= configtree
.symbols()
895 # write local symbol list -- needed during packaging build
896 if not bpid
.integrate
:
897 f
= open(os
.path
.join(bpid
.target_dir
, '.local-symbols'), 'w')
899 f
.write('%s=\n' % sym
)
901 git_debug_snapshot(args
, "add symbols files")
903 # add defconfigs that we want
904 defconfigs_dir
= os
.path
.join(source_dir
, 'backport', 'defconfigs')
905 os
.mkdir(os
.path
.join(bpid
.target_dir
, 'defconfigs'))
906 for dfbase
in os
.listdir(defconfigs_dir
):
907 copy_defconfig
= True
908 dfsrc
= os
.path
.join(defconfigs_dir
, dfbase
)
909 for line
in open(dfsrc
, 'r'):
914 if sym
+ '=' in line
:
918 copy_defconfig
= False
921 shutil
.copy(dfsrc
, os
.path
.join(bpid
.target_dir
, 'defconfigs', dfbase
))
923 git_debug_snapshot(args
, "add (useful) defconfig files")
925 logwrite('Rewrite Makefiles and Kconfig files ...')
927 # rewrite Makefile and source symbols
930 for some_symbols
in [orig_symbols
[i
:i
+ 50] for i
in range(0, len(orig_symbols
), 50)]:
931 r
= 'CONFIG_((' + '|'.join([s
+ '(_MODULE)?' for s
in some_symbols
]) + ')([^A-Za-z0-9_]|$))'
932 regexes
.append(re
.compile(r
, re
.MULTILINE
))
933 for root
, dirs
, files
in os
.walk(bpid
.target_dir
):
934 # don't go into .git dir (possible debug thing)
938 data
= open(os
.path
.join(root
, f
), 'r').read()
940 data
= r
.sub(r
'' + bpid
.full_prefix
+ '\\1', data
)
941 data
= re
.sub(r
'\$\(srctree\)', '$(backport_srctree)', data
)
942 data
= re
.sub(r
'-Idrivers', '-I$(backport_srctree)/drivers', data
)
944 data
= re
.sub(r
'CPTCFG_', bpid
.full_prefix
, data
)
945 fo
= open(os
.path
.join(root
, f
), 'w')
949 git_debug_snapshot(args
, "rename config symbol / srctree usage")
951 # disable unbuildable Kconfig symbols and stuff Makefiles that doesn't exist
952 maketree
= make
.MakeTree(os
.path
.join(bpid
.target_dir
, 'Makefile.kernel'))
954 disable_makefile
= []
955 for sym
in maketree
.get_impossible_symbols():
956 disable_kconfig
.append(sym
[7:])
957 disable_makefile
.append(sym
[7:])
959 configtree
.disable_symbols(disable_kconfig
)
960 git_debug_snapshot(args
, "disable impossible kconfig symbols")
962 # add kernel version dependencies to Kconfig, from the dependency list
964 for sym
in tuple(deplist
.keys()):
966 for dep
in deplist
[sym
]:
967 if "kconfig:" in dep
:
968 kconfig_expr
= dep
.replace('kconfig: ', '')
969 new
.append(kconfig_expr
)
970 elif (dep
== "DISABLE"):
971 new
.append('BACKPORT_DISABLED_KCONFIG_OPTION')
973 new
.append('!BACKPORT_KERNEL_%s' % dep
.replace('.', '_'))
975 deplist
[sym
] = ["BACKPORT_" + x
for x
in new
]
978 configtree
.add_dependencies(deplist
)
979 git_debug_snapshot(args
, "add kernel version dependencies")
981 # disable things in makefiles that can't be selected and that the
982 # build shouldn't recurse into because they don't exist -- if we
983 # don't do that then a symbol from the kernel could cause the build
984 # to attempt to recurse and fail
986 # Note that we split the regex after 50 symbols, this is because of a
987 # limitation in the regex implementation (it only supports 100 nested
988 # groups -- 50 seemed safer and is still fast)
990 for some_symbols
in [disable_makefile
[i
:i
+ 50] for i
in range(0, len(disable_makefile
), 50)]:
991 r
= '^([^#].*((' + bpid
.full_prefix_resafe
+ '|CONFIG_)(' + '|'.join([s
for s
in some_symbols
]) + ')))'
992 regexes
.append(re
.compile(r
, re
.MULTILINE
))
993 for f
in maketree
.get_makefiles():
994 data
= open(f
, 'r').read()
996 data
= r
.sub(r
'#\1', data
)
1000 git_debug_snapshot(args
, "disable unsatisfied Makefile parts")
1002 if (args
.kup
or args
.kup_test
):
1005 if not req
.reqs_match():
1007 upload_release(args
, rel_prep
, logwrite
=logwrite
)
1012 if __name__
== '__main__':