3 # Generate the output tree into a specified directory.
6 import argparse
, sys
, os
, errno
, shutil
, re
, subprocess
10 source_dir
= os
.path
.abspath(os
.path
.dirname(__file__
))
11 sys
.path
.append(source_dir
)
12 # and import libraries we have
13 from lib
import kconfig
, patch
, make
14 from lib
import bpgit
as git
15 from lib
import bpgpg
as gpg
16 from lib
import bpkup
as kup
17 from lib
.tempdir
import tempdir
18 from lib
import bpreqs
as reqs
19 from lib
import bpversion
as gen_version
21 class Bp_Identity(object):
23 folks considering multiple integrations may want to
24 consider stuffing versioning info here as well but
25 that will need thought/design on sharing compat and
28 Use the *_resafe when combining on regexps, although we currently
29 don't support regexps there perhaps later we will and this will
30 just make things safer for the output regardless. Once and if those
31 are added, how we actually use the others for regular printing will
32 need to be considered.
34 def __init__(self
, integrate
=False, kconfig_prefix
='CPTCFG_',
35 project_prefix
='', project_dir
='',
36 target_dir
='', target_dir_name
='',
37 kconfig_source_var
=None):
38 self
.integrate
= integrate
39 self
.kconfig_prefix
= kconfig_prefix
40 self
.kconfig_prefix_resafe
= re
.escape(kconfig_prefix
)
41 self
.project_prefix
= project_prefix
42 self
.project_prefix_resafe
= re
.escape(project_prefix
)
43 self
.full_prefix
= kconfig_prefix
+ project_prefix
44 self
.full_prefix_resafe
= re
.escape(self
.full_prefix
)
45 self
.project_dir
= project_dir
46 self
.target_dir
= target_dir
47 self
.target_dir_name
= target_dir_name
48 self
.kconfig_source_var
= kconfig_source_var
49 if self
.kconfig_source_var
:
50 self
.kconfig_source_var_resafe
= re
.escape(self
.kconfig_source_var
)
52 self
.kconfig_source_var_resafe
= None
54 def read_copy_list(copyfile
):
56 Read a copy-list file and return a list of (source, target)
57 tuples. The source and target are usually the same, but in
58 the copy-list file there may be a rename included.
62 # remove leading/trailing whitespace
65 if not item
or item
[0] == '#':
68 raise Exception("Input path '%s' is absolute path, this isn't allowed" % (item
, ))
70 srcitem
, dstitem
= item
.split(' -> ')
71 if (srcitem
[-1] == '/') != (dstitem
[-1] == '/'):
72 raise Exception("Cannot copy file/dir to dir/file")
74 srcitem
= dstitem
= item
75 ret
.append((srcitem
, dstitem
))
79 def read_dependencies(depfilename
):
81 Read a (the) dependency file and return the list of
82 dependencies as a dictionary, mapping a Kconfig symbol
83 to a list of kernel version dependencies.
85 If a backported feature that an upstream backported driver
86 depends on had kconfig limitations (ie, debugging feature not
87 available) a built constaint restriction can be expressed
88 by using a kconfig expression. The kconfig expressions can
89 be specified by using the "kconfig: " prefix.
91 While reading ignore blank or commented lines.
94 depfile
= open(depfilename
, 'r')
98 if not item
or item
[0] == '#':
100 if "kconfig:" in item
:
101 sym
, kconfig_exp
= item
.split(" ", 1)
103 ret
[sym
] = [kconfig_exp
, ]
105 ret
[sym
].append(kconfig_exp
)
107 sym
, dep
= item
.split()
115 def check_output_dir(d
, clean
):
117 Check that the output directory doesn't exist or is empty,
118 unless clean is True in which case it's nuked. This helps
119 sanity check the output when generating a tree, so usually
120 running with --clean isn't suggested.
123 shutil
.rmtree(d
, ignore_errors
=True)
127 if e
.errno
!= errno
.ENOENT
:
131 def copytree(src
, dst
, symlinks
=False, ignore
=None):
133 Copy a directory tree. This differs from shutil.copytree()
134 in that it allows destination directories to already exist.
136 names
= os
.listdir(src
)
137 if ignore
is not None:
138 ignored_names
= ignore(src
, names
)
140 ignored_names
= set()
142 if not os
.path
.isdir(dst
):
146 if name
in ignored_names
:
148 srcname
= os
.path
.join(src
, name
)
149 dstname
= os
.path
.join(dst
, name
)
151 if symlinks
and os
.path
.islink(srcname
):
152 linkto
= os
.readlink(srcname
)
153 os
.symlink(linkto
, dstname
)
154 elif os
.path
.isdir(srcname
):
155 copytree(srcname
, dstname
, symlinks
, ignore
)
157 shutil
.copy2(srcname
, dstname
)
158 except (IOError, os
.error
) as why
:
159 errors
.append((srcname
, dstname
, str(why
)))
160 # catch the Error from the recursive copytree so that we can
161 # continue with other files
162 except shutil
.Error
as err
:
163 errors
.extend(err
.args
[0])
165 shutil
.copystat(src
, dst
)
167 # can't copy file access times on Windows
169 except OSError as why
:
170 errors
.extend((src
, dst
, str(why
)))
172 raise shutil
.Error(errors
)
175 def copy_files(srcpath
, copy_list
, outdir
):
177 Copy the copy_list files and directories from the srcpath
178 to the outdir. The copy_list contains source and target
181 For now, it also ignores any *~ editor backup files, though
182 this should probably be generalized (maybe using .gitignore?)
183 Similarly the code that only copies some files (*.c, *.h,
184 *.awk, Kconfig, Makefile) to avoid any build remnants in the
185 kernel if they should exist.
187 for srcitem
, tgtitem
in copy_list
:
189 copytree(srcpath
, outdir
, ignore
=shutil
.ignore_patterns('*~'))
190 elif tgtitem
[-1] == '/':
191 def copy_ignore(dir, entries
):
194 if i
[-2:] == '.o' or i
[-1] == '~':
197 copytree(os
.path
.join(srcpath
, srcitem
),
198 os
.path
.join(outdir
, tgtitem
),
202 os
.makedirs(os
.path
.join(outdir
, os
.path
.dirname(tgtitem
)))
204 # ignore dirs we might have created just now
205 if e
.errno
!= errno
.EEXIST
:
207 shutil
.copy(os
.path
.join(srcpath
, srcitem
),
208 os
.path
.join(outdir
, tgtitem
))
211 def copy_git_files(srcpath
, copy_list
, rev
, outdir
):
213 "Copy" files from a git repository. This really means listing them with
214 ls-tree and then using git show to obtain all the blobs.
216 for srcitem
, tgtitem
in copy_list
:
217 for m
, t
, h
, f
in git
.ls_tree(rev
=rev
, files
=(srcitem
,), tree
=srcpath
):
219 f
= os
.path
.join(outdir
, f
.replace(srcitem
, tgtitem
))
220 d
= os
.path
.dirname(f
)
221 if not os
.path
.exists(d
):
224 git
.get_blob(h
, outf
, tree
=srcpath
)
226 os
.chmod(f
, int(m
, 8))
228 def automatic_backport_mangle_c_file(name
):
229 return name
.replace('/', '-')
232 def add_automatic_backports(args
):
234 export
= re
.compile(r
'^EXPORT_SYMBOL(_GPL)?\((?P<sym>[^\)]*)\)')
235 bpi
= kconfig
.get_backport_info(os
.path
.join(args
.bpid
.target_dir
, 'compat', 'Kconfig'))
236 configtree
= kconfig
.ConfigTree(os
.path
.join(args
.bpid
.target_dir
, 'Kconfig'), args
.bpid
)
237 ignore
=['Kconfig.kernel', 'Kconfig.versions']
238 configtree
.verify_sources(ignore
=ignore
)
239 git_debug_snapshot(args
, "verify sources for automatic backports")
240 all_selects
= configtree
.all_selects()
241 for sym
, vals
in bpi
.items():
242 if sym
.startswith('BPAUTO_BUILD_'):
243 if not sym
[13:] in all_selects
:
244 disable_list
.append(sym
)
246 symtype
, module_name
, c_files
, h_files
= vals
251 files
.append((f
, os
.path
.join('compat', automatic_backport_mangle_c_file(f
))))
253 files
.append((os
.path
.join('include', f
),
254 os
.path
.join('include', os
.path
.dirname(f
), 'backport-' + os
.path
.basename(f
))))
255 if args
.git_revision
:
256 copy_git_files(args
.kerneldir
, files
, args
.git_revision
, args
.bpid
.target_dir
)
258 copy_files(args
.kerneldir
, files
, args
.bpid
.target_dir
)
260 # now add the Makefile line
261 mf
= open(os
.path
.join(args
.bpid
.target_dir
, 'compat', 'Makefile'), 'a+')
262 o_files
= [automatic_backport_mangle_c_file(f
)[:-1] + 'o' for f
in c_files
]
263 if symtype
== 'tristate':
265 raise Exception('backporting a module requires a #module-name')
267 mf
.write('%s-objs += %s\n' % (module_name
, of
))
268 mf
.write('obj-$(%s%s) += %s.o\n' % (args
.bpid
.full_prefix
, sym
, module_name
))
269 elif symtype
== 'bool':
270 mf
.write('compat-$(%s%s) += %s\n' % (args
.bpid
.full_prefix
, sym
, ' '.join(o_files
)))
272 # finally create the include file
275 for l
in open(os
.path
.join(args
.bpid
.target_dir
, 'compat',
276 automatic_backport_mangle_c_file(f
)), 'r'):
279 syms
.append(m
.group('sym'))
281 outf
= open(os
.path
.join(args
.bpid
.target_dir
, 'include', f
), 'w')
282 outf
.write('/* Automatically created during backport process */\n')
283 outf
.write('#ifndef %s%s\n' % (args
.bpid
.full_prefix
, sym
))
284 outf
.write('#include_next <%s>\n' % f
)
285 outf
.write('#else\n');
287 outf
.write('#undef %s\n' % s
)
288 outf
.write('#define %s LINUX_BACKPORT(%s)\n' % (s
, s
))
289 outf
.write('#include <%s>\n' % (os
.path
.dirname(f
) + '/backport-' + os
.path
.basename(f
), ))
290 outf
.write('#endif /* %s%s */\n' % (args
.bpid
.full_prefix
, sym
))
293 def git_debug_init(args
):
295 Initialize a git repository in the output directory and commit the current
296 code in it. This is only used for debugging the transformations this code
297 will do to the output later.
299 if not args
.gitdebug
:
301 # Git supports re-initialization, although not well documented it can
302 # reset config stuff, lets avoid that if the tree already exists.
303 if not os
.path
.exists(os
.path
.join(args
.bpid
.project_dir
, '.git')):
304 git
.init(tree
=args
.bpid
.project_dir
)
305 git
.commit_all("Copied backport", tree
=args
.bpid
.project_dir
)
308 def git_debug_snapshot(args
, name
):
310 Take a git snapshot for the debugging.
312 if not args
.gitdebug
:
314 git
.commit_all(name
, tree
=args
.bpid
.project_dir
)
316 def get_rel_spec_stable(rel
):
318 Returns release specs for a linux-stable backports based release.
321 m
= re
.match(r
"(?P<VERSION>\d+)\.+" \
322 "(?P<PATCHLEVEL>\d+)[.]*" \
323 "(?P<SUBLEVEL>\d*)" \
324 "[-rc]+(?P<RC_VERSION>\d+)\-+" \
325 "(?P<RELMOD_UPDATE>\d+)[-]*" \
326 "(?P<RELMOD_TYPE>[usnpc]*)", \
329 m
= re
.match(r
"(?P<VERSION>\d+)\.+" \
330 "(?P<PATCHLEVEL>\d+)[.]*" \
331 "(?P<SUBLEVEL>\d*)\-+" \
332 "(?P<RELMOD_UPDATE>\d+)[-]*" \
333 "(?P<RELMOD_TYPE>[usnpc]*)", \
339 def get_rel_spec_next(rel
):
341 Returns release specs for a linux-next backports based release.
343 m
= re
.match(r
"(?P<DATE_VERSION>\d+)[-]*" \
344 "(?P<RELMOD_UPDATE>\d*)[-]*" \
345 "(?P<RELMOD_TYPE>[usnpc]*)", \
351 def get_rel_prep(rel
):
353 Returns a dict with prep work details we need prior to
354 uploading a backports release to kernel.org
356 rel_specs
= get_rel_spec_stable(rel
)
361 rel_specs
= get_rel_spec_next(rel
)
363 sys
.stdout
.write("rel: %s\n" % rel
)
365 if (rel_specs
['RELMOD_UPDATE'] == '0' or
366 rel_specs
['RELMOD_UPDATE'] == '1'):
369 date
= rel_specs
['DATE_VERSION']
374 if (len(month
) != 2):
382 rel_tag
= "backports-" + rel
.replace(rel_specs
['RELMOD_TYPE'], "")
385 if (not rel_specs
['RELMOD_UPDATE']):
387 if (rel_specs
['RELMOD_UPDATE'] == '0'):
389 ignore
+= rel_specs
['RELMOD_UPDATE']
390 if (rel_specs
['RELMOD_TYPE'] != ''):
391 ignore
+= rel_specs
['RELMOD_TYPE']
392 base_rel
= rel
.replace(ignore
, "")
393 paths
.append("v" + base_rel
)
394 rel_tag
= "v" + rel
.replace(rel_specs
['RELMOD_TYPE'], "")
396 rel_prep
= dict(stable
= is_stable
,
397 expected_tag
= rel_tag
,
398 paths_to_create
= paths
)
401 def create_tar_and_gz(tar_name
, dir_to_tar
):
403 We need both a tar file and gzip for kernel.org, the tar file
404 gets signed, then we upload the compressed version, kup-server
405 in the backend decompresses and verifies the tarball against
408 basename
= os
.path
.basename(dir_to_tar
)
409 tar
= tarfile
.open(tar_name
, "w")
410 tar
.add(dir_to_tar
, basename
)
413 tar_file
= open(tar_name
, "r")
415 gz_file
= gzip
.GzipFile(tar_name
+ ".gz", 'wb')
416 gz_file
.write(tar_file
.read())
419 def upload_release(args
, rel_prep
, logwrite
=lambda x
:None):
421 Given a path of a relase make tarball out of it, PGP sign it, and
422 then upload it to kernel.org using kup.
424 The linux-next based release do not require a RELMOD_UPDATE
425 given that typically only one release is made per day. Using
426 RELMOD_UPDATE for these releases is allowed though and if
427 present it must be > 1.
429 The linux-stable based releases require a RELMOD_UPDATE.
431 RELMOD_UPDATE must be numeric and > 0 just as the RC releases
434 The tree must also be tagged with the respective release, without
435 the RELMOD_TYPE. For linux-next based releases this consists of
436 backports- followed by DATE_VERSION and if RELMOD_TYPE is present.
437 For linux-stable releases this consists of v followed by the
438 full release version except the RELMOD_TYPE.
440 Uploads will not be allowed if these rules are not followed.
442 korg_path
= "/pub/linux/kernel/projects/backports"
444 if (rel_prep
['stable']):
445 korg_path
+= "/stable"
447 parent
= os
.path
.dirname(args
.bpid
.project_dir
)
448 release
= os
.path
.basename(args
.bpid
.project_dir
)
449 tar_name
= parent
+ '/' + release
+ ".tar"
450 gzip_name
= tar_name
+ ".gz"
452 create_tar_and_gz(tar_name
, args
.bpid
.project_dir
)
454 logwrite(gpg
.sign(tar_name
, extra_args
=['--armor', '--detach-sign']))
456 logwrite("------------------------------------------------------")
458 if (not args
.kup_test
):
459 logwrite("About to upload, current target path contents:")
461 logwrite("kup-test: current target path contents:")
463 logwrite(kup
.ls(path
=korg_path
))
465 for path
in rel_prep
['paths_to_create']:
466 korg_path
+= '/' + path
467 if (not args
.kup_test
):
468 logwrite("create directory: %s" % korg_path
)
469 logwrite(kup
.mkdir(korg_path
))
471 if (not args
.kup_test
):
472 logwrite("upload file %s to %s" % (gzip_name
, korg_path
))
473 logwrite(kup
.put(gzip_name
, tar_name
+ '.asc', korg_path
))
474 logwrite("\nFinished upload!\n")
475 logwrite("Target path contents:")
476 logwrite(kup
.ls(path
=korg_path
))
478 kup_cmd
= "kup put /\n\t\t%s /\n\t\t%s /\n\t\t%s" % (gzip_name
, tar_name
+ '.asc', korg_path
)
479 logwrite("kup-test: skipping cmd: %s" % kup_cmd
)
481 def apply_patches(args
, desc
, source_dir
, patch_src
, target_dir
, logwrite
=lambda x
:None):
483 Given a path of a directories of patches and SmPL patches apply
484 them on the target directory. If requested refresh patches, or test
485 a specific SmPL patch.
487 logwrite('Applying patches from %s to %s ...' % (patch_src
, target_dir
))
488 test_cocci
= args
.test_cocci
or args
.profile_cocci
489 test_cocci_found
= False
492 for root
, dirs
, files
in os
.walk(os
.path
.join(source_dir
, patch_src
)):
494 if not test_cocci
and f
.endswith('.patch'):
495 patches
.append(os
.path
.join(root
, f
))
496 if f
.endswith('.cocci'):
498 if f
not in test_cocci
:
500 test_cocci_found
= True
502 logwrite("Testing Coccinelle SmPL patch: %s" % test_cocci
)
503 elif args
.profile_cocci
:
504 logwrite("Profiling Coccinelle SmPL patch: %s" % test_cocci
)
505 sempatches
.append(os
.path
.join(root
, f
))
507 prefix_len
= len(os
.path
.join(source_dir
, patch_src
)) + 1
508 for pfile
in patches
:
509 print_name
= pfile
[prefix_len
:]
510 # read the patch file
511 p
= patch
.fromfile(pfile
)
512 # complain if it's not a patch
514 raise Exception('No patch content found in %s' % print_name
)
515 # leading / seems to be stripped?
516 if 'dev/null' in p
.items
[0].source
:
517 raise Exception('Patches creating files are not supported (in %s)' % print_name
)
518 # check if the first file the patch touches exists, if so
519 # assume the patch needs to be applied -- otherwise continue
520 patched_file
= '/'.join(p
.items
[0].source
.split('/')[1:])
521 fullfn
= os
.path
.join(target_dir
, patched_file
)
522 if not os
.path
.exists(fullfn
):
524 logwrite("Not applying %s, not needed" % print_name
)
527 logwrite("Applying patch %s" % print_name
)
530 # but for refresh, of course look at all files the patch touches
531 for patchitem
in p
.items
:
532 patched_file
= '/'.join(patchitem
.source
.split('/')[1:])
533 fullfn
= os
.path
.join(target_dir
, patched_file
)
534 shutil
.copyfile(fullfn
, fullfn
+ '.orig_file')
536 process
= subprocess
.Popen(['patch', '-p1'], stdout
=subprocess
.PIPE
,
537 stderr
=subprocess
.STDOUT
, stdin
=subprocess
.PIPE
,
538 close_fds
=True, universal_newlines
=True,
540 output
= process
.communicate(input=open(pfile
, 'r').read())[0]
541 output
= output
.split('\n')
546 logwrite('> %s' % line
)
547 if process
.returncode
!= 0:
549 logwrite("Failed to apply changes from %s" % print_name
)
551 logwrite('> %s' % line
)
555 pfilef
= open(pfile
+ '.tmp', 'a')
556 pfilef
.write(p
.top_header
)
558 for patchitem
in p
.items
:
559 patched_file
= '/'.join(patchitem
.source
.split('/')[1:])
560 fullfn
= os
.path
.join(target_dir
, patched_file
)
561 process
= subprocess
.Popen(['diff', '-p', '-u', patched_file
+ '.orig_file', patched_file
,
562 '--label', 'a/' + patched_file
,
563 '--label', 'b/' + patched_file
],
564 stdout
=pfilef
, close_fds
=True,
565 universal_newlines
=True, cwd
=target_dir
)
567 os
.unlink(fullfn
+ '.orig_file')
568 if not process
.returncode
in (0, 1):
569 logwrite("Failed to diff to refresh %s" % print_name
)
571 os
.unlink(pfile
+ '.tmp')
574 os
.rename(pfile
+ '.tmp', pfile
)
576 # remove orig/rej files that patch sometimes creates
577 for root
, dirs
, files
in os
.walk(target_dir
):
579 if f
[-5:] == '.orig' or f
[-4:] == '.rej':
580 os
.unlink(os
.path
.join(root
, f
))
581 git_debug_snapshot(args
, "apply %s patch %s" % (desc
, print_name
))
584 prefix_len
= len(os
.path
.join(source_dir
, patch_src
)) + 1
586 for cocci_file
in sempatches
:
587 # Until Coccinelle picks this up
588 pycocci
= os
.path
.join(source_dir
, 'devel/pycocci')
589 cmd
= [pycocci
, cocci_file
]
590 extra_spatch_args
= []
591 if args
.profile_cocci
:
592 cmd
.append('--profile-cocci')
593 cmd
.append(os
.path
.abspath(target_dir
))
594 print_name
= cocci_file
[prefix_len
:]
596 logwrite("Applying SmPL patch %s" % print_name
)
597 sprocess
= subprocess
.Popen(cmd
,
598 stdout
=subprocess
.PIPE
, stderr
=subprocess
.STDOUT
,
599 close_fds
=True, universal_newlines
=True,
601 output
= sprocess
.communicate()[0]
603 if sprocess
.returncode
!= 0:
604 logwrite("Failed to process SmPL patch %s with %i" % (print_name
, sprocess
.returncode
))
606 output
= output
.split('\n')
611 logwrite('> %s' % line
)
613 # remove cocci_backup files
614 for root
, dirs
, files
in os
.walk(target_dir
):
616 if f
.endswith('.cocci_backup'):
617 os
.unlink(os
.path
.join(root
, f
))
618 git_debug_snapshot(args
, "apply %s SmPL patch %s" % (desc
, print_name
))
620 if test_cocci
and test_cocci_found
:
625 # Our binary requirements go here
628 req
.coccinelle('1.0.0-rc24')
629 if not req
.reqs_match():
632 # set up and parse arguments
633 parser
= argparse
.ArgumentParser(description
='generate backport tree')
634 parser
.add_argument('kerneldir', metavar
='<kernel tree>', type=str,
635 help='Kernel tree to copy drivers from')
636 parser
.add_argument('outdir', metavar
='<output directory>', type=str,
637 help='Directory to write the generated tree to')
638 parser
.add_argument('--copy-list', metavar
='<listfile>', type=argparse
.FileType('r'),
640 help='File containing list of files/directories to copy, default "copy-list"')
641 parser
.add_argument('--git-revision', metavar
='<revision>', type=str,
642 help='git commit revision (see gitrevisions(7)) to take objects from.' +
643 'If this is specified, the kernel tree is used as git object storage ' +
644 'and we use git ls-tree to get the files.')
645 parser
.add_argument('--clean', const
=True, default
=False, action
="store_const",
646 help='Clean output directory instead of erroring if it isn\'t empty')
647 parser
.add_argument('--integrate', const
=True, default
=False, action
="store_const",
648 help='Integrate a future backported kernel solution into ' +
649 'an older kernel tree source directory.')
650 parser
.add_argument('--refresh', const
=True, default
=False, action
="store_const",
651 help='Refresh patches as they are applied, the source dir will be modified!')
652 parser
.add_argument('--base-name', metavar
='<name>', type=str, default
='Linux',
653 help='name of base tree, default just "Linux"')
654 parser
.add_argument('--gitdebug', const
=True, default
=False, action
="store_const",
655 help='Use git, in the output tree, to debug the various transformation steps ' +
656 'that the tree generation makes (apply patches, ...)')
657 parser
.add_argument('--verbose', const
=True, default
=False, action
="store_const",
658 help='Print more verbose information')
659 parser
.add_argument('--extra-driver', nargs
=2, metavar
=('<source dir>', '<copy-list>'), type=str,
660 action
='append', default
=[], help='Extra driver directory/copy-list.')
661 parser
.add_argument('--kup', const
=True, default
=False, action
="store_const",
662 help='For maintainers: upload a release to kernel.org')
663 parser
.add_argument('--kup-test', const
=True, default
=False, action
="store_const",
664 help='For maintainers: do all the work as if you were about to ' +
665 'upload to kernel.org but do not do the final `kup put` ' +
666 'and also do not run any `kup mkdir` commands. This will ' +
667 'however run `kup ls` on the target paths so ' +
668 'at the very least we test your kup configuration. ' +
669 'If this is your first time uploading use this first!')
670 parser
.add_argument('--test-cocci', metavar
='<sp_file>', type=str, default
=None,
671 help='Only use the cocci file passed for Coccinelle, don\'t do anything else, ' +
672 'also creates a git repo on the target directory for easy inspection ' +
673 'of changes done by Coccinelle.')
674 parser
.add_argument('--profile-cocci', metavar
='<sp_file>', type=str, default
=None,
675 help='Only use the cocci file passed and pass --profile to Coccinelle, ' +
676 'also creates a git repo on the target directory for easy inspection ' +
677 'of changes done by Coccinelle.')
678 args
= parser
.parse_args()
680 # When building a package we use CPTCFG as we can rely on the
681 # fact that kconfig treats CONFIG_ as an environment variable
682 # requring less changes on code. For kernel integration we use
683 # the longer CONFIG_BACKPORT given that we'll be sticking to
684 # the kernel symbol namespace, to address that we do a final
685 # search / replace. Technically its possible to rely on the
686 # same prefix for packaging as with kernel integration but
687 # there are already some users of the CPTCFG prefix.
690 bpid
= Bp_Identity(integrate
= args
.integrate
,
691 kconfig_prefix
= 'CONFIG_',
692 project_prefix
= 'BACKPORT_',
693 project_dir
= args
.outdir
,
694 target_dir
= os
.path
.join(args
.outdir
, 'backports/'),
695 target_dir_name
= 'backports/',
696 kconfig_source_var
= '$BACKPORT_DIR',
699 bpid
= Bp_Identity(integrate
= args
.integrate
,
700 kconfig_prefix
= 'CPTCFG_',
702 project_dir
= args
.outdir
,
703 target_dir
= args
.outdir
,
704 target_dir_name
= '',
705 kconfig_source_var
= '$BACKPORT_DIR',
709 sys
.stdout
.write(msg
)
710 sys
.stdout
.write('\n')
713 return process(args
.kerneldir
, args
.copy_list
,
714 git_revision
=args
.git_revision
,
717 refresh
=args
.refresh
, base_name
=args
.base_name
,
718 gitdebug
=args
.gitdebug
, verbose
=args
.verbose
,
719 extra_driver
=args
.extra_driver
,
721 kup_test
=args
.kup_test
,
722 test_cocci
=args
.test_cocci
,
723 profile_cocci
=args
.profile_cocci
,
726 def process(kerneldir
, copy_list_file
, git_revision
=None,
728 clean
=False, refresh
=False, base_name
="Linux", gitdebug
=False,
729 verbose
=False, extra_driver
=[], kup
=False,
733 logwrite
=lambda x
:None,
734 git_tracked_version
=False):
736 def __init__(self
, kerneldir
, copy_list_file
,
737 git_revision
, bpid
, clean
, refresh
, base_name
,
738 gitdebug
, verbose
, extra_driver
, kup
,
742 self
.kerneldir
= kerneldir
743 self
.copy_list
= copy_list_file
744 self
.git_revision
= git_revision
747 self
.refresh
= refresh
748 self
.base_name
= base_name
749 self
.gitdebug
= gitdebug
750 self
.verbose
= verbose
751 self
.extra_driver
= extra_driver
753 self
.kup_test
= kup_test
754 self
.test_cocci
= test_cocci
755 self
.profile_cocci
= profile_cocci
756 if self
.test_cocci
or self
.profile_cocci
:
758 def git_paranoia(tree
=None, logwrite
=lambda x
:None):
759 data
= git
.paranoia(tree
)
761 logwrite('Cannot use %s' % tree
)
762 logwrite('%s' % data
['output'])
765 logwrite('Validated tree: %s' % tree
)
767 args
= Args(kerneldir
, copy_list_file
,
768 git_revision
, bpid
, clean
, refresh
, base_name
,
769 gitdebug
, verbose
, extra_driver
, kup
, kup_test
,
770 test_cocci
, profile_cocci
)
774 if args
.kup_test
or args
.test_cocci
or args
.profile_cocci
or args
.refresh
:
775 logwrite('Cannot use integration with:\n\tkup_test\n\ttest_cocci\n\tprofile_cocci\n\trefresh\n');
778 # start processing ...
779 if (args
.kup
or args
.kup_test
):
780 git_paranoia(source_dir
, logwrite
)
781 git_paranoia(kerneldir
, logwrite
)
783 rel_describe
= git
.describe(rev
=None, tree
=source_dir
, extra_args
=['--dirty'])
784 release
= os
.path
.basename(bpid
.target_dir
)
785 version
= release
.replace("backports-", "")
787 rel_prep
= get_rel_prep(version
)
789 logwrite('Invalid backports release name: %s' % release
)
790 logwrite('For rules on the release name see upload_release()')
792 rel_type
= "linux-stable"
793 if (not rel_prep
['stable']):
794 rel_type
= "linux-next"
795 if (rel_prep
['expected_tag'] != rel_describe
):
796 logwrite('Unexpected %s based backports release tag on' % rel_type
)
797 logwrite('the backports tree tree: %s\n' % rel_describe
)
798 logwrite('You asked to make a release with this ')
799 logwrite('directory name: %s' % release
)
800 logwrite('The actual expected tag we should find on')
801 logwrite('the backports tree then is: %s\n' % rel_prep
['expected_tag'])
802 logwrite('For rules on the release name see upload_release()')
805 copy_list
= read_copy_list(args
.copy_list
)
806 deplist
= read_dependencies(os
.path
.join(source_dir
, 'dependencies'))
808 # validate output directory
809 check_output_dir(bpid
.target_dir
, args
.clean
)
812 backport_integrate_files
= [
813 ('Makefile.kernel', 'Makefile'),
814 ('Kconfig.integrate', 'Kconfig'),
816 backport_package_files
= [(x
, x
) for x
in [
821 'Kconfig.package.hacks',
826 backport_package_files
+= [
827 ('Kconfig.package', 'Kconfig'),
829 backport_files
= [(x
, x
) for x
in [
835 if not bpid
.integrate
:
836 backport_files
+= backport_package_files
838 backport_files
+= backport_integrate_files
840 if not args
.git_revision
:
841 logwrite('Copy original source files ...')
843 logwrite('Get original source files from git ...')
845 copy_files(os
.path
.join(source_dir
, 'backport'), backport_files
, bpid
.target_dir
)
849 if not args
.git_revision
:
850 copy_files(args
.kerneldir
, copy_list
, bpid
.target_dir
)
852 copy_git_files(args
.kerneldir
, copy_list
, args
.git_revision
, bpid
.target_dir
)
854 # FIXME: should we add a git version of this (e.g. --git-extra-driver)?
855 for src
, copy_list
in args
.extra_driver
:
856 if (args
.kup
or args
.kup_test
):
858 copy_files(src
, read_copy_list(open(copy_list
, 'r')), bpid
.target_dir
)
860 git_debug_snapshot(args
, 'Add driver sources')
862 disable_list
= add_automatic_backports(args
)
863 if git_tracked_version
:
864 backports_version
= "(see git)"
865 kernel_version
= "(see git)"
867 backports_version
= git
.describe(tree
=source_dir
, extra_args
=['--long'])
868 kernel_version
= git
.describe(rev
=args
.git_revision
or 'HEAD',
870 extra_args
=['--long'])
872 if not bpid
.integrate
:
873 f
= open(os
.path
.join(bpid
.target_dir
, 'versions'), 'w')
874 f
.write('BACKPORTS_VERSION="%s"\n' % backports_version
)
875 f
.write('BACKPORTED_KERNEL_VERSION="%s"\n' % kernel_version
)
876 f
.write('BACKPORTED_KERNEL_NAME="%s"\n' % args
.base_name
)
877 if git_tracked_version
:
878 f
.write('BACKPORTS_GIT_TRACKED="backport tracker ID: $(shell git rev-parse HEAD 2>/dev/null || echo \'not built in git tree\')"\n')
880 git_debug_snapshot(args
, "add versions files")
883 (re
.compile(r
'.*(?P<key>%%BACKPORT_DIR%%)'), '%%BACKPORT_DIR%%', 'backports/'),
884 (re
.compile(r
'.*(?P<key>%%BACKPORTS_VERSION%%).*'), '%%BACKPORTS_VERSION%%', backports_version
),
885 (re
.compile(r
'.*(?P<key>%%BACKPORTED_KERNEL_VERSION%%).*'), '%%BACKPORTED_KERNEL_VERSION%%', kernel_version
),
886 (re
.compile(r
'.*(?P<key>%%BACKPORTED_KERNEL_NAME%%).*'), '%%BACKPORTED_KERNEL_NAME%%', args
.base_name
),
889 for l
in open(os
.path
.join(bpid
.target_dir
, 'Kconfig'), 'r'):
890 for r
in kconf_regexes
:
893 l
= re
.sub(r
'(' + r
[1] + ')', r
'' + r
[2] + '', l
)
895 outf
= open(os
.path
.join(bpid
.target_dir
, 'Kconfig'), 'w')
898 git_debug_snapshot(args
, "modify top level backports/Kconfig with backports identity")
901 # No need to verify_sources() as compat's Kconfig has no 'source' call
902 bpcfg
= kconfig
.ConfigTree(os
.path
.join(bpid
.target_dir
, 'compat', 'Kconfig'), bpid
)
903 bpcfg
.disable_symbols(disable_list
)
904 git_debug_snapshot(args
, 'Add automatic backports')
906 failure
= apply_patches(args
, "backport", source_dir
, 'patches', bpid
.target_dir
, logwrite
)
910 # Kernel integration requires Kconfig.versions already generated for you,
911 # we cannot do this for a package as we have no idea what kernel folks
914 kver
= gen_version
.kernelversion(bpid
.project_dir
)
915 rel_specs
= gen_version
.get_rel_spec_stable(kver
)
917 logwrite('Cannot parse source kernel version, update parser')
919 data
= gen_version
.genkconfig_versions(rel_specs
)
920 fo
= open(os
.path
.join(bpid
.target_dir
, 'Kconfig.versions'), 'w')
923 git_debug_snapshot(args
, "generate kernel version requirement Kconfig file")
925 # some post-processing is required
926 configtree
= kconfig
.ConfigTree(os
.path
.join(bpid
.target_dir
, 'Kconfig'), bpid
)
927 ignore
=['Kconfig.kernel', 'Kconfig.versions', 'Kconfig.local']
929 configtree
.verify_sources(ignore
=ignore
)
930 git_debug_snapshot(args
, "verify sources on top level backports Kconfig")
932 orig_symbols
= configtree
.symbols()
934 logwrite('Modify Kconfig tree ...')
935 configtree
.prune_sources(ignore
=ignore
)
936 git_debug_snapshot(args
, "prune Kconfig tree")
938 if not bpid
.integrate
:
939 configtree
.force_tristate_modular()
940 git_debug_snapshot(args
, "force tristate options modular")
942 ignore
= [os
.path
.join(bpid
.target_dir
, x
) for x
in [
943 'Kconfig.package.hacks',
949 configtree
.adjust_backported_configs(ignore
=ignore
, orig_symbols
=orig_symbols
)
950 git_debug_snapshot(args
, "adjust backports config symbols we port")
952 configtree
.modify_selects()
953 git_debug_snapshot(args
, "convert select to depends on")
955 symbols
= configtree
.symbols()
957 # write local symbol list -- needed during packaging build
958 if not bpid
.integrate
:
959 f
= open(os
.path
.join(bpid
.target_dir
, 'local-symbols'), 'w')
961 f
.write('%s=\n' % sym
)
963 git_debug_snapshot(args
, "add symbols files")
964 # also write Kconfig.local, representing all local symbols
965 # with a BACKPORTED_ prefix
966 f
= open(os
.path
.join(bpid
.target_dir
, 'Kconfig.local'), 'w')
968 f
.write('config BACKPORTED_%s\n' % sym
)
969 f
.write('\ttristate\n')
970 f
.write('\tdefault %s\n' % sym
)
972 git_debug_snapshot(args
, "add Kconfig.local")
974 # add defconfigs that we want
975 defconfigs_dir
= os
.path
.join(source_dir
, 'backport', 'defconfigs')
976 os
.mkdir(os
.path
.join(bpid
.target_dir
, 'defconfigs'))
977 for dfbase
in os
.listdir(defconfigs_dir
):
978 copy_defconfig
= True
979 dfsrc
= os
.path
.join(defconfigs_dir
, dfbase
)
980 for line
in open(dfsrc
, 'r'):
985 if sym
+ '=' in line
:
989 copy_defconfig
= False
992 shutil
.copy(dfsrc
, os
.path
.join(bpid
.target_dir
, 'defconfigs', dfbase
))
994 git_debug_snapshot(args
, "add (useful) defconfig files")
996 logwrite('Rewrite Makefiles and Kconfig files ...')
998 # rewrite Makefile and source symbols
1000 # symbols we know only we can provide under the backport project prefix
1001 # for which we need an exception.
1002 skip_orig_syms
= [ bpid
.project_prefix
+ x
for x
in [
1006 parse_orig_syms
= [x
for x
in orig_symbols
if x
not in skip_orig_syms
]
1008 for some_symbols
in [parse_orig_syms
[i
:i
+ 50] for i
in range(0, len(parse_orig_syms
), 50)]:
1009 r
= 'CONFIG_((' + '|'.join([s
+ '(_MODULE)?' for s
in some_symbols
]) + ')([^A-Za-z0-9_]|$))'
1010 regexes
.append(re
.compile(r
, re
.MULTILINE
))
1011 for root
, dirs
, files
in os
.walk(bpid
.target_dir
):
1012 # don't go into .git dir (possible debug thing)
1016 data
= open(os
.path
.join(root
, f
), 'r').read()
1018 data
= r
.sub(r
'' + bpid
.full_prefix
+ '\\1', data
)
1019 data
= re
.sub(r
'\$\(srctree\)', '$(backport_srctree)', data
)
1020 data
= re
.sub(r
'-Idrivers', '-I$(backport_srctree)/drivers', data
)
1022 data
= re
.sub(r
'CPTCFG_', bpid
.full_prefix
, data
)
1023 fo
= open(os
.path
.join(root
, f
), 'w')
1027 git_debug_snapshot(args
, "rename config symbol / srctree usage")
1029 # disable unbuildable Kconfig symbols and stuff Makefiles that doesn't exist
1031 maketree
= make
.MakeTree(os
.path
.join(bpid
.target_dir
, 'Makefile'))
1033 maketree
= make
.MakeTree(os
.path
.join(bpid
.target_dir
, 'Makefile.kernel'))
1034 disable_kconfig
= []
1035 disable_makefile
= []
1036 for sym
in maketree
.get_impossible_symbols():
1037 disable_kconfig
.append(sym
[7:])
1038 disable_makefile
.append(sym
[7:])
1040 configtree
.disable_symbols(disable_kconfig
)
1041 git_debug_snapshot(args
, "disable impossible kconfig symbols")
1043 # add kernel version dependencies to Kconfig, from the dependency list
1044 # we read previously
1045 for sym
in tuple(deplist
.keys()):
1047 for dep
in deplist
[sym
]:
1048 if "kconfig:" in dep
:
1049 kconfig_expr
= dep
.replace('kconfig: ', '')
1050 new
.append(kconfig_expr
)
1051 elif (dep
== "DISABLE"):
1052 new
.append('BACKPORT_DISABLED_KCONFIG_OPTION')
1054 new
.append('!KERNEL_%s' % dep
.replace('.', '_'))
1056 deplist
[sym
] = ["BACKPORT_" + x
for x
in new
]
1059 configtree
.add_dependencies(deplist
)
1060 git_debug_snapshot(args
, "add kernel version dependencies")
1062 # disable things in makefiles that can't be selected and that the
1063 # build shouldn't recurse into because they don't exist -- if we
1064 # don't do that then a symbol from the kernel could cause the build
1065 # to attempt to recurse and fail
1067 # Note that we split the regex after 50 symbols, this is because of a
1068 # limitation in the regex implementation (it only supports 100 nested
1069 # groups -- 50 seemed safer and is still fast)
1071 for some_symbols
in [disable_makefile
[i
:i
+ 50] for i
in range(0, len(disable_makefile
), 50)]:
1072 r
= '^(([^#].*((' + bpid
.full_prefix_resafe
+ '|CONFIG_)(' + '|'.join([s
for s
in some_symbols
]) + ')))\W)'
1073 regexes
.append(re
.compile(r
, re
.MULTILINE
))
1074 for f
in maketree
.get_makefiles():
1075 data
= open(f
, 'r').read()
1077 data
= r
.sub(r
'#\1', data
)
1081 git_debug_snapshot(args
, "disable unsatisfied Makefile parts")
1084 f
= open(os
.path
.join(bpid
.project_dir
, 'Kconfig'), 'a')
1085 f
.write('source "backports/Kconfig"\n')
1087 git_debug_snapshot(args
, "hooked backport to top level Kconfig")
1089 failure
= apply_patches(args
, "integration", source_dir
, 'integration-patches/',
1090 bpid
.project_dir
, logwrite
)
1094 if (args
.kup
or args
.kup_test
):
1097 if not req
.reqs_match():
1099 upload_release(args
, rel_prep
, logwrite
=logwrite
)
1104 if __name__
== '__main__':