3 # Generate the output tree into a specified directory.
6 import argparse
, sys
, os
, errno
, shutil
, re
, subprocess
9 source_dir
= os
.path
.abspath(os
.path
.dirname(__file__
))
10 sys
.path
.append(source_dir
)
11 # and import libraries we have
12 from lib
import kconfig
, git
, patch
, make
15 def read_copy_list(copyfile
):
17 Read a copy-list file and return a list of (source, target)
18 tuples. The source and target are usually the same, but in
19 the copy-list file there may be a rename included.
23 # remove leading/trailing whitespace
26 if not item
or item
[0] == '#':
29 raise Exception("Input path '%s' is absolute path, this isn't allowed" % (item
, ))
31 srcitem
, dstitem
= item
.split(' -> ')
32 if (srcitem
[-1] == '/') != (dstitem
[-1] == '/'):
33 raise Exception("Cannot copy file/dir to dir/file")
35 srcitem
= dstitem
= item
36 ret
.append((srcitem
, dstitem
))
40 def read_dependencies(depfilename
):
42 Read a (the) dependency file and return the list of
43 dependencies as a dictionary, mapping a Kconfig symbol
44 to a list of kernel version dependencies. While reading
45 ignore blank/commented lines.
48 depfile
= open(depfilename
, 'r')
51 if not item
or item
[0] == '#':
53 sym
, dep
= item
.split()
61 def check_output_dir(d
, clean
):
63 Check that the output directory doesn't exist or is empty,
64 unless clean is True in which case it's nuked. This helps
65 sanity check the output when generating a tree, so usually
66 running with --clean isn't suggested.
69 shutil
.rmtree(d
, ignore_errors
=True)
73 if e
.errno
!= errno
.ENOENT
:
77 def copytree(src
, dst
, symlinks
=False, ignore
=None):
79 Copy a directory tree. This differs from shutil.copytree()
80 in that it allows destination directories to already exist.
82 names
= os
.listdir(src
)
83 if ignore
is not None:
84 ignored_names
= ignore(src
, names
)
88 if not os
.path
.isdir(dst
):
92 if name
in ignored_names
:
94 srcname
= os
.path
.join(src
, name
)
95 dstname
= os
.path
.join(dst
, name
)
97 if symlinks
and os
.path
.islink(srcname
):
98 linkto
= os
.readlink(srcname
)
99 os
.symlink(linkto
, dstname
)
100 elif os
.path
.isdir(srcname
):
101 copytree(srcname
, dstname
, symlinks
, ignore
)
103 shutil
.copy2(srcname
, dstname
)
104 except (IOError, os
.error
) as why
:
105 errors
.append((srcname
, dstname
, str(why
)))
106 # catch the Error from the recursive copytree so that we can
107 # continue with other files
108 except shutil
.Error
as err
:
109 errors
.extend(err
.args
[0])
111 shutil
.copystat(src
, dst
)
113 # can't copy file access times on Windows
115 except OSError as why
:
116 errors
.extend((src
, dst
, str(why
)))
118 raise shutil
.Error(errors
)
121 def copy_files(srcpath
, copy_list
, outdir
):
123 Copy the copy_list files and directories from the srcpath
124 to the outdir. The copy_list contains source and target
127 For now, it also ignores any *~ editor backup files, though
128 this should probably be generalized (maybe using .gitignore?)
129 Similarly the code that only copies some files (*.c, *.h,
130 *.awk, Kconfig, Makefile) to avoid any build remnants in the
131 kernel if they should exist.
133 for srcitem
, tgtitem
in copy_list
:
135 copytree(srcpath
, outdir
, ignore
=shutil
.ignore_patterns('*~'))
136 elif tgtitem
[-1] == '/':
137 def copy_ignore(dir, entries
):
140 if i
[-2] == '.o' or i
[-1] == '~':
143 copytree(os
.path
.join(srcpath
, srcitem
),
144 os
.path
.join(outdir
, tgtitem
),
148 os
.makedirs(os
.path
.join(outdir
, os
.path
.dirname(tgtitem
)))
150 # ignore dirs we might have created just now
151 if e
.errno
!= errno
.EEXIST
:
153 shutil
.copy(os
.path
.join(srcpath
, srcitem
),
154 os
.path
.join(outdir
, tgtitem
))
157 def copy_git_files(srcpath
, copy_list
, rev
, outdir
):
159 "Copy" files from a git repository. This really means listing them with
160 ls-tree and then using git show to obtain all the blobs.
162 for srcitem
, tgtitem
in copy_list
:
163 for m
, t
, h
, f
in git
.ls_tree(rev
=rev
, files
=(srcitem
,), tree
=srcpath
):
165 f
= os
.path
.join(outdir
, f
.replace(srcitem
, tgtitem
))
166 d
= os
.path
.dirname(f
)
167 if not os
.path
.exists(d
):
170 git
.get_blob(h
, outf
, tree
=srcpath
)
172 os
.chmod(f
, int(m
, 8))
175 def add_automatic_backports(args
):
176 export
= re
.compile(r
'^EXPORT_SYMBOL(_GPL)?\((?P<sym>[^\)]*)\)')
177 bpi
= kconfig
.get_backport_info(os
.path
.join(args
.outdir
, 'compat', 'Kconfig'))
178 for sym
, vals
in bpi
.iteritems():
179 symtype
, c_files
, h_files
= vals
184 files
.append((f
, os
.path
.join('compat', os
.path
.basename(f
))))
186 files
.append((os
.path
.join('include', f
),
187 os
.path
.join('include', os
.path
.dirname(f
), 'backport-' + os
.path
.basename(f
))))
188 if args
.git_revision
:
189 copy_git_files(args
.kerneldir
, files
, args
.git_revision
, args
.outdir
)
191 copy_files(args
.kerneldir
, files
, args
.outdir
)
193 # now add the Makefile line
194 mf
= open(os
.path
.join(args
.outdir
, 'compat', 'Makefile'), 'a+')
195 o_files
= [os
.path
.basename(f
)[:-1] + 'o' for f
in c_files
]
196 if symtype
== 'tristate':
197 mf
.write('obj-$(CPTCFG_%s) += %s\n' % (sym
, ' '.join(o_files
)))
198 elif symtype
== 'bool':
199 mf
.write('compat-$(CPTCFG_%s) += %s\n' % (sym
, ' '.join(o_files
)))
201 # finally create the include file
204 for l
in open(os
.path
.join(args
.outdir
, 'compat', os
.path
.basename(f
)), 'r'):
207 syms
.append(m
.group('sym'))
209 outf
= open(os
.path
.join(args
.outdir
, 'include', f
), 'w')
210 outf
.write('/* Automatically created during backport process */\n')
211 outf
.write('#ifndef CPTCFG_%s\n' % sym
)
212 outf
.write('#include_next <%s>\n' % f
)
213 outf
.write('#else\n');
215 outf
.write('#undef %s\n' % s
)
216 outf
.write('#define %s LINUX_BACKPORT(%s)\n' % (s
, s
))
217 outf
.write('#include <%s>\n' % (os
.path
.dirname(f
) + '/backport-' + os
.path
.basename(f
), ))
218 outf
.write('#endif /* CPTCFG_%s */\n' % sym
)
220 def git_debug_init(args
):
222 Initialize a git repository in the output directory and commit the current
223 code in it. This is only used for debugging the transformations this code
224 will do to the output later.
226 if not args
.gitdebug
:
228 git
.init(tree
=args
.outdir
)
229 git
.commit_all("Copied backport", tree
=args
.outdir
)
232 def git_debug_snapshot(args
, name
):
234 Take a git snapshot for the debugging.
236 if not args
.gitdebug
:
238 git
.commit_all(name
, tree
=args
.outdir
)
242 # set up and parse arguments
243 parser
= argparse
.ArgumentParser(description
='generate backport tree')
244 parser
.add_argument('kerneldir', metavar
='<kernel tree>', type=str,
245 help='Kernel tree to copy drivers from')
246 parser
.add_argument('outdir', metavar
='<output directory>', type=str,
247 help='Directory to write the generated tree to')
248 parser
.add_argument('--copy-list', metavar
='<listfile>', type=argparse
.FileType('r'),
250 help='File containing list of files/directories to copy, default "copy-list"')
251 parser
.add_argument('--git-revision', metavar
='<revision>', type=str,
252 help='git commit revision (see gitrevisions(7)) to take objects from.' +
253 'If this is specified, the kernel tree is used as git object storage ' +
254 'and we use git ls-tree to get the files.')
255 parser
.add_argument('--clean', const
=True, default
=False, action
="store_const",
256 help='Clean output directory instead of erroring if it isn\'t empty')
257 parser
.add_argument('--refresh', const
=True, default
=False, action
="store_const",
258 help='Refresh patches as they are applied, the source dir will be modified!')
259 parser
.add_argument('--base-name', metavar
='<name>', type=str, default
='Linux',
260 help='name of base tree, default just "Linux"')
261 parser
.add_argument('--gitdebug', const
=True, default
=False, action
="store_const",
262 help='Use git, in the output tree, to debug the various transformation steps ' +
263 'that the tree generation makes (apply patches, ...)')
264 parser
.add_argument('--verbose', const
=True, default
=False, action
="store_const",
265 help='Print more verbose information')
266 parser
.add_argument('--extra-driver', nargs
=2, metavar
=('<source dir>', '<copy-list>'), type=str,
267 action
='append', default
=[], help='Extra driver directory/copy-list.')
268 args
= parser
.parse_args()
271 sys
.stdout
.write(msg
)
272 sys
.stdout
.write('\n')
275 return process(args
.kerneldir
, args
.outdir
, args
.copy_list
,
276 git_revision
=args
.git_revision
, clean
=args
.clean
,
277 refresh
=args
.refresh
, base_name
=args
.base_name
,
278 gitdebug
=args
.gitdebug
, verbose
=args
.verbose
,
279 extra_driver
=args
.extra_driver
, logwrite
=logwrite
)
281 def process(kerneldir
, outdir
, copy_list_file
, git_revision
=None,
282 clean
=False, refresh
=False, base_name
="Linux", gitdebug
=False,
283 verbose
=False, extra_driver
=[], logwrite
=lambda x
:None,
284 kernel_version_name
=None, backport_version_name
=None):
286 def __init__(self
, kerneldir
, outdir
, copy_list_file
,
287 git_revision
, clean
, refresh
, base_name
,
288 gitdebug
, verbose
, extra_driver
):
289 self
.kerneldir
= kerneldir
291 self
.copy_list
= copy_list_file
292 self
.git_revision
= git_revision
294 self
.refresh
= refresh
295 self
.base_name
= base_name
296 self
.gitdebug
= gitdebug
297 self
.verbose
= verbose
298 self
.extra_driver
= extra_driver
299 args
= Args(kerneldir
, outdir
, copy_list_file
,
300 git_revision
, clean
, refresh
, base_name
,
301 gitdebug
, verbose
, extra_driver
)
302 # start processing ...
304 copy_list
= read_copy_list(args
.copy_list
)
305 deplist
= read_dependencies(os
.path
.join(source_dir
, 'dependencies'))
307 # validate output directory
308 check_output_dir(args
.outdir
, args
.clean
)
311 backport_files
= [(x
, x
) for x
in [
312 'Kconfig', 'Makefile', 'Makefile.build', 'Makefile.kernel', '.gitignore',
313 'Makefile.real', 'compat/', 'include/', 'kconfig/', 'defconfigs/',
315 if not args
.git_revision
:
316 logwrite('Copy original source files ...')
318 logwrite('Get original source files from git ...')
320 copy_files(os
.path
.join(source_dir
, 'backport'), backport_files
, args
.outdir
)
324 add_automatic_backports(args
)
325 git_debug_snapshot(args
, 'Add automatic backports')
327 if not args
.git_revision
:
328 copy_files(args
.kerneldir
, copy_list
, args
.outdir
)
330 copy_git_files(args
.kerneldir
, copy_list
, args
.git_revision
, args
.outdir
)
332 # FIXME: should we add a git version of this (e.g. --git-extra-driver)?
333 for src
, copy_list
in args
.extra_driver
:
334 copy_files(src
, read_copy_list(open(copy_list
, 'r')), args
.outdir
)
336 git_debug_snapshot(args
, 'Add driver sources')
338 logwrite('Apply patches ...')
340 for root
, dirs
, files
in os
.walk(os
.path
.join(source_dir
, 'patches')):
342 if f
.endswith('.patch'):
343 patches
.append(os
.path
.join(root
, f
))
345 prefix_len
= len(os
.path
.join(source_dir
, 'patches')) + 1
346 for pfile
in patches
:
347 print_name
= pfile
[prefix_len
:]
348 # read the patch file
349 p
= patch
.fromfile(pfile
)
350 # complain if it's not a patch
352 raise Exception('No patch content found in %s' % print_name
)
353 # check if the first file the patch touches exists, if so
354 # assume the patch needs to be applied -- otherwise continue
355 patched_file
= '/'.join(p
.items
[0].source
.split('/')[1:])
356 fullfn
= os
.path
.join(args
.outdir
, patched_file
)
357 if not os
.path
.exists(fullfn
):
359 logwrite("Not applying %s, not needed" % print_name
)
362 logwrite("Applying patch %s" % print_name
)
365 # but for refresh, of course look at all files the patch touches
366 for patchitem
in p
.items
:
367 patched_file
= '/'.join(patchitem
.source
.split('/')[1:])
368 fullfn
= os
.path
.join(args
.outdir
, patched_file
)
369 shutil
.copyfile(fullfn
, fullfn
+ '.orig_file')
371 process
= subprocess
.Popen(['patch', '-p1'], stdout
=subprocess
.PIPE
,
372 stderr
=subprocess
.STDOUT
, stdin
=subprocess
.PIPE
,
373 close_fds
=True, universal_newlines
=True,
375 output
= process
.communicate(input=open(pfile
, 'r').read())[0]
376 output
= output
.split('\n')
381 logwrite('> %s' % line
)
382 if process
.returncode
!= 0:
384 logwrite("Failed to apply changes from %s" % print_name
)
386 logwrite('> %s' % line
)
390 pfilef
= open(pfile
+ '.tmp', 'w')
391 for patchitem
in p
.items
:
392 patched_file
= '/'.join(patchitem
.source
.split('/')[1:])
393 fullfn
= os
.path
.join(args
.outdir
, patched_file
)
394 process
= subprocess
.Popen(['diff', '-p', '-u', patched_file
+ '.orig_file', patched_file
,
395 '--label', 'a/' + patched_file
,
396 '--label', 'b/' + patched_file
],
397 stdout
=pfilef
, close_fds
=True,
398 universal_newlines
=True, cwd
=args
.outdir
)
400 os
.unlink(fullfn
+ '.orig_file')
401 if not process
.returncode
in (0, 1):
402 logwrite("Failed to diff to refresh %s" % print_name
)
404 os
.unlink(pfile
+ '.tmp')
407 os
.rename(pfile
+ '.tmp', pfile
)
409 # remove orig/rej files that patch sometimes creates
410 for root
, dirs
, files
in os
.walk(args
.outdir
):
412 if f
[-5:] == '.orig' or f
[-4:] == '.rej':
413 os
.unlink(os
.path
.join(root
, f
))
414 git_debug_snapshot(args
, "apply backport patch %s" % print_name
)
416 # some post-processing is required
417 configtree
= kconfig
.ConfigTree(os
.path
.join(args
.outdir
, 'Kconfig'))
418 logwrite('Modify Kconfig tree ...')
419 configtree
.prune_sources(ignore
=['Kconfig.kernel', 'Kconfig.versions'])
420 git_debug_snapshot(args
, "prune Kconfig tree")
421 configtree
.force_tristate_modular()
422 git_debug_snapshot(args
, "force tristate options modular")
423 configtree
.modify_selects()
424 git_debug_snapshot(args
, "convert select to depends on")
426 # write the versioning file
427 backports_version
= backport_version_name
or git
.describe(tree
=source_dir
)
428 kernel_version
= kernel_version_name
or git
.describe(rev
=args
.git_revision
or 'HEAD',
430 f
= open(os
.path
.join(args
.outdir
, 'versions'), 'w')
431 f
.write('BACKPORTS_VERSION="%s"\n' % backports_version
)
432 f
.write('BACKPORTED_KERNEL_VERSION="%s"\n' % kernel_version
)
433 f
.write('BACKPORTED_KERNEL_NAME="%s"\n' % args
.base_name
)
436 symbols
= configtree
.symbols()
438 # write local symbol list -- needed during build
439 f
= open(os
.path
.join(args
.outdir
, '.local-symbols'), 'w')
441 f
.write('%s=\n' % sym
)
444 git_debug_snapshot(args
, "add versions/symbols files")
446 logwrite('Rewrite Makefiles and Kconfig files ...')
448 # rewrite Makefile and source symbols
450 for some_symbols
in [symbols
[i
:i
+ 50] for i
in range(0, len(symbols
), 50)]:
451 r
= 'CONFIG_((' + '|'.join([s
+ '(_MODULE)?' for s
in some_symbols
]) + ')([^A-Za-z0-9_]|$))'
452 regexes
.append(re
.compile(r
, re
.MULTILINE
))
453 for root
, dirs
, files
in os
.walk(args
.outdir
):
454 # don't go into .git dir (possible debug thing)
458 data
= open(os
.path
.join(root
, f
), 'r').read()
460 data
= r
.sub(r
'CPTCFG_\1', data
)
461 fo
= open(os
.path
.join(root
, f
), 'w')
465 git_debug_snapshot(args
, "rename config symbol usage")
467 # disable unbuildable Kconfig symbols and stuff Makefiles that doesn't exist
468 maketree
= make
.MakeTree(os
.path
.join(args
.outdir
, 'Makefile.kernel'))
470 disable_makefile
= []
471 for sym
in maketree
.get_impossible_symbols():
472 disable_kconfig
.append(sym
[7:])
473 disable_makefile
.append(sym
[7:])
475 configtree
.disable_symbols(disable_kconfig
)
476 git_debug_snapshot(args
, "disable impossible kconfig symbols")
478 # add kernel version dependencies to Kconfig, from the dependency list
480 for sym
in tuple(deplist
.keys()):
482 for dep
in deplist
[sym
]:
483 new
.append('!BACKPORT_KERNEL_%s' % dep
.replace('.', '_'))
485 configtree
.add_dependencies(deplist
)
486 git_debug_snapshot(args
, "add kernel version dependencies")
488 # disable things in makefiles that can't be selected and that the
489 # build shouldn't recurse into because they don't exist -- if we
490 # don't do that then a symbol from the kernel could cause the build
491 # to attempt to recurse and fail
493 # Note that we split the regex after 50 symbols, this is because of a
494 # limitation in the regex implementation (it only supports 100 nested
495 # groups -- 50 seemed safer and is still fast)
497 for some_symbols
in [disable_makefile
[i
:i
+ 50] for i
in range(0, len(disable_makefile
), 50)]:
498 r
= '((CPTCFG|CONFIG)_(' + '|'.join([s
for s
in some_symbols
]) + '))'
499 regexes
.append(re
.compile(r
, re
.MULTILINE
))
500 for f
in maketree
.get_makefiles():
501 data
= open(f
, 'r').read()
503 data
= r
.sub(r
'IMPOSSIBLE_\3', data
)
507 git_debug_snapshot(args
, "disable unsatisfied Makefile parts")
512 if __name__
== '__main__':