3 # Generate the output tree into a specified directory.
6 import argparse
, sys
, os
, errno
, shutil
, re
, subprocess
9 source_dir
= os
.path
.abspath(os
.path
.dirname(__file__
))
10 sys
.path
.append(source_dir
)
11 # and import libraries we have
12 from lib
import kconfig
, git
, patch
, make
15 def read_copy_list(copyfile
):
17 Read a copy-list file and return a list of (source, target)
18 tuples. The source and target are usually the same, but in
19 the copy-list file there may be a rename included.
23 # remove leading/trailing whitespace
26 if not item
or item
[0] == '#':
29 raise Exception("Input path '%s' is absolute path, this isn't allowed" % (item
, ))
31 srcitem
, dstitem
= item
.split(' -> ')
32 if (srcitem
[-1] == '/') != (dstitem
[-1] == '/'):
33 raise Exception("Cannot copy file/dir to dir/file")
35 srcitem
= dstitem
= item
36 ret
.append((srcitem
, dstitem
))
40 def read_dependencies(depfilename
):
42 Read a (the) dependency file and return the list of
43 dependencies as a dictionary, mapping a Kconfig symbol
44 to a list of kernel version dependencies. While reading
45 ignore blank/commented lines.
48 depfile
= open(depfilename
, 'r')
51 if not item
or item
[0] == '#':
53 sym
, dep
= item
.split()
61 def check_output_dir(d
, clean
):
63 Check that the output directory doesn't exist or is empty,
64 unless clean is True in which case it's nuked. This helps
65 sanity check the output when generating a tree, so usually
66 running with --clean isn't suggested.
69 shutil
.rmtree(d
, ignore_errors
=True)
73 if e
.errno
!= errno
.ENOENT
:
77 def copytree(src
, dst
, symlinks
=False, ignore
=None):
79 Copy a directory tree. This differs from shutil.copytree()
80 in that it allows destination directories to already exist.
82 names
= os
.listdir(src
)
83 if ignore
is not None:
84 ignored_names
= ignore(src
, names
)
88 if not os
.path
.isdir(dst
):
92 if name
in ignored_names
:
94 srcname
= os
.path
.join(src
, name
)
95 dstname
= os
.path
.join(dst
, name
)
97 if symlinks
and os
.path
.islink(srcname
):
98 linkto
= os
.readlink(srcname
)
99 os
.symlink(linkto
, dstname
)
100 elif os
.path
.isdir(srcname
):
101 copytree(srcname
, dstname
, symlinks
, ignore
)
103 shutil
.copy2(srcname
, dstname
)
104 except (IOError, os
.error
) as why
:
105 errors
.append((srcname
, dstname
, str(why
)))
106 # catch the Error from the recursive copytree so that we can
107 # continue with other files
108 except shutil
.Error
as err
:
109 errors
.extend(err
.args
[0])
111 shutil
.copystat(src
, dst
)
113 # can't copy file access times on Windows
115 except OSError as why
:
116 errors
.extend((src
, dst
, str(why
)))
118 raise shutil
.Error(errors
)
121 def copy_files(srcpath
, copy_list
, outdir
):
123 Copy the copy_list files and directories from the srcpath
124 to the outdir. The copy_list contains source and target
127 For now, it also ignores any *~ editor backup files, though
128 this should probably be generalized (maybe using .gitignore?)
129 Similarly the code that only copies some files (*.c, *.h,
130 *.awk, Kconfig, Makefile) to avoid any build remnants in the
131 kernel if they should exist.
133 for srcitem
, tgtitem
in copy_list
:
135 copytree(srcpath
, outdir
, ignore
=shutil
.ignore_patterns('*~'))
136 elif tgtitem
[-1] == '/':
137 def copy_ignore(dir, entries
):
140 if i
[-1] in ('o', '~'):
143 copytree(os
.path
.join(srcpath
, srcitem
),
144 os
.path
.join(outdir
, tgtitem
),
148 os
.makedirs(os
.path
.join(outdir
, os
.path
.dirname(tgtitem
)))
150 # ignore dirs we might have created just now
151 if e
.errno
!= errno
.EEXIST
:
153 shutil
.copy(os
.path
.join(srcpath
, srcitem
),
154 os
.path
.join(outdir
, tgtitem
))
157 def copy_git_files(srcpath
, copy_list
, rev
, outdir
):
159 "Copy" files from a git repository. This really means listing them with
160 ls-tree and then using git show to obtain all the blobs.
162 for srcitem
, tgtitem
in copy_list
:
163 for m
, t
, h
, f
in git
.ls_tree(rev
=rev
, files
=(srcitem
,), tree
=srcpath
):
165 f
= os
.path
.join(outdir
, f
)
166 d
= os
.path
.dirname(f
)
167 if not os
.path
.exists(d
):
170 git
.get_blob(h
, outf
, tree
=srcpath
)
172 os
.chmod(f
, int(m
, 8))
175 def git_debug_init(args
):
177 Initialize a git repository in the output directory and commit the current
178 code in it. This is only used for debugging the transformations this code
179 will do to the output later.
181 if not args
.gitdebug
:
183 git
.init(tree
=args
.outdir
)
184 git
.commit_all("Copied code", tree
=args
.outdir
)
187 def git_debug_snapshot(args
, name
):
189 Take a git snapshot for the debugging.
191 if not args
.gitdebug
:
193 git
.commit_all(name
, tree
=args
.outdir
)
197 # set up and parse arguments
198 parser
= argparse
.ArgumentParser(description
='generate backport tree')
199 parser
.add_argument('kerneldir', metavar
='<kernel tree>', type=str,
200 help='Kernel tree to copy drivers from')
201 parser
.add_argument('outdir', metavar
='<output directory>', type=str,
202 help='Directory to write the generated tree to')
203 parser
.add_argument('--copy-list', metavar
='<listfile>', type=argparse
.FileType('r'),
205 help='File containing list of files/directories to copy, default "copy-list"')
206 parser
.add_argument('--git-revision', metavar
='<revision>', type=str,
207 help='git commit revision (see gitrevisions(7)) to take objects from.' +
208 'If this is specified, the kernel tree is used as git object storage ' +
209 'and we use git ls-tree to get the files.')
210 parser
.add_argument('--clean', const
=True, default
=False, action
="store_const",
211 help='Clean output directory instead of erroring if it isn\'t empty')
212 parser
.add_argument('--refresh', const
=True, default
=False, action
="store_const",
213 help='Refresh patches as they are applied, the source dir will be modified!')
214 parser
.add_argument('--base-name', metavar
='<name>', type=str, default
='Linux',
215 help='name of base tree, default just "Linux"')
216 parser
.add_argument('--gitdebug', const
=True, default
=False, action
="store_const",
217 help='Use git, in the output tree, to debug the various transformation steps ' +
218 'that the tree generation makes (apply patches, ...)')
219 parser
.add_argument('--verbose', const
=True, default
=False, action
="store_const",
220 help='Print more verbose information')
221 parser
.add_argument('--extra-driver', nargs
=2, metavar
=('<source dir>', '<copy-list>'), type=str,
222 action
='append', default
=[], help='Extra driver directory/copy-list.')
223 args
= parser
.parse_args()
226 sys
.stdout
.write(msg
)
227 sys
.stdout
.write('\n')
230 return process(args
.kerneldir
, args
.outdir
, args
.copy_list
,
231 git_revision
=args
.git_revision
, clean
=args
.clean
,
232 refresh
=args
.refresh
, base_name
=args
.base_name
,
233 gitdebug
=args
.gitdebug
, verbose
=args
.verbose
,
234 extra_driver
=args
.extra_driver
, logwrite
=logwrite
)
236 def process(kerneldir
, outdir
, copy_list_file
, git_revision
=None,
237 clean
=False, refresh
=False, base_name
="Linux", gitdebug
=False,
238 verbose
=False, extra_driver
=[], logwrite
=lambda x
:None):
240 def __init__(self
, kerneldir
, outdir
, copy_list_file
,
241 git_revision
, clean
, refresh
, base_name
,
242 gitdebug
, verbose
, extra_driver
):
243 self
.kerneldir
= kerneldir
245 self
.copy_list
= copy_list_file
246 self
.git_revision
= git_revision
248 self
.refresh
= refresh
249 self
.base_name
= base_name
250 self
.gitdebug
= gitdebug
251 self
.verbose
= verbose
252 self
.extra_driver
= extra_driver
253 args
= Args(kerneldir
, outdir
, copy_list_file
,
254 git_revision
, clean
, refresh
, base_name
,
255 gitdebug
, verbose
, extra_driver
)
256 # start processing ...
258 copy_list
= read_copy_list(args
.copy_list
)
259 deplist
= read_dependencies(os
.path
.join(source_dir
, 'dependencies'))
261 # validate output directory
262 check_output_dir(args
.outdir
, args
.clean
)
265 backport_files
= [(x
, x
) for x
in [
266 'Kconfig', 'Makefile', 'Makefile.build', 'Makefile.kernel', '.gitignore',
267 'Makefile.real', 'compat/', 'include/', 'kconfig/', 'defconfigs/',
269 if not args
.git_revision
:
270 logwrite('Copy original source files ...')
271 copy_files(os
.path
.join(source_dir
, 'backport'), backport_files
, args
.outdir
)
272 copy_files(args
.kerneldir
, copy_list
, args
.outdir
)
274 logwrite('Get original source files from git ...')
275 copy_files(os
.path
.join(source_dir
, 'backport'), backport_files
, args
.outdir
)
276 copy_git_files(args
.kerneldir
, copy_list
, args
.git_revision
, args
.outdir
)
278 # FIXME: should we add a git version of this (e.g. --git-extra-driver)?
279 for src
, copy_list
in args
.extra_driver
:
280 copy_files(src
, read_copy_list(open(copy_list
, 'r')), args
.outdir
)
284 logwrite('Apply patches ...')
286 for root
, dirs
, files
in os
.walk(os
.path
.join(source_dir
, 'patches')):
288 patchdirs
.append(root
)
290 for pdir
in patchdirs
:
294 # FIXME: again, use .gitignore?
297 pfile
= os
.path
.join(pdir
, pfile
)
298 # read the patch file
299 p
= patch
.fromfile(pfile
)
303 # check if the first file the patch touches exists, if so
304 # assume the patch needs to be applied -- otherwise continue
305 patched_file
= '/'.join(p
.items
[0].source
.split('/')[1:])
306 fullfn
= os
.path
.join(args
.outdir
, patched_file
)
307 if not os
.path
.exists(fullfn
):
311 logwrite("Applying changes from %s" % os
.path
.basename(pdir
))
314 # but for refresh, of course look at all files the patch touches
315 for patchitem
in p
.items
:
316 patched_file
= '/'.join(patchitem
.source
.split('/')[1:])
317 fullfn
= os
.path
.join(args
.outdir
, patched_file
)
318 shutil
.copyfile(fullfn
, fullfn
+ '.orig_file')
320 process
= subprocess
.Popen(['patch', '-p1'], stdout
=subprocess
.PIPE
,
321 stderr
=subprocess
.STDOUT
, stdin
=subprocess
.PIPE
,
322 close_fds
=True, universal_newlines
=True,
324 output
= process
.communicate(input=open(pfile
, 'r').read())[0]
325 output
= output
.split('\n')
330 logwrite('> %s' % line
)
331 if process
.returncode
!= 0:
333 logwrite("Failed to apply changes from %s" % os
.path
.basename(pdir
))
335 logwrite('> %s' % line
)
339 pfilef
= open(pfile
+ '.tmp', 'w')
340 for patchitem
in p
.items
:
341 patched_file
= '/'.join(patchitem
.source
.split('/')[1:])
342 fullfn
= os
.path
.join(args
.outdir
, patched_file
)
343 process
= subprocess
.Popen(['diff', '-u', patched_file
+ '.orig_file', patched_file
,
344 '--label', 'a/' + patched_file
,
345 '--label', 'b/' + patched_file
],
346 stdout
=pfilef
, close_fds
=True,
347 universal_newlines
=True, cwd
=args
.outdir
)
349 os
.unlink(fullfn
+ '.orig_file')
350 if not process
.returncode
in (0, 1):
351 logwrite("Diffing for refresh failed!")
353 os
.unlink(pfile
+ '.tmp')
356 os
.rename(pfile
+ '.tmp', pfile
)
358 # remove orig/rej files that patch sometimes creates
359 for root
, dirs
, files
in os
.walk(args
.outdir
):
361 if f
[-5:] == '.orig' or f
[-4:] == '.rej':
362 os
.unlink(os
.path
.join(root
, f
))
365 logwrite("Not applying changes from %s, not needed" % (os
.path
.basename(pdir
),))
367 git_debug_snapshot(args
, "apply backport patches from %s" % (os
.path
.basename(pdir
),))
369 # some post-processing is required
370 configtree
= kconfig
.ConfigTree(os
.path
.join(args
.outdir
, 'Kconfig'))
371 logwrite('Modify Kconfig tree ...')
372 configtree
.prune_sources(ignore
=['Kconfig.kernel', 'Kconfig.versions'])
373 git_debug_snapshot(args
, "prune Kconfig tree")
374 configtree
.force_tristate_modular()
375 git_debug_snapshot(args
, "force tristate options modular")
376 configtree
.modify_selects()
377 git_debug_snapshot(args
, "convert select to depends on")
379 # write the versioning file
380 backports_version
= git
.describe(tree
=source_dir
)
381 kernel_version
= git
.describe(tree
=args
.kerneldir
)
382 f
= open(os
.path
.join(args
.outdir
, 'versions'), 'w')
383 f
.write('BACKPORTS_VERSION="%s"\n' % backports_version
)
384 f
.write('BACKPORTED_KERNEL_VERSION="%s"\n' % kernel_version
)
385 f
.write('BACKPORTED_KERNEL_NAME="%s"\n' % args
.base_name
)
388 symbols
= configtree
.symbols()
390 # write local symbol list -- needed during build
391 f
= open(os
.path
.join(args
.outdir
, '.local-symbols'), 'w')
393 f
.write('%s=\n' % sym
)
396 git_debug_snapshot(args
, "add versions/symbols files")
398 logwrite('Rewrite Makefiles and Kconfig files ...')
400 # rewrite Makefile and source symbols
402 for some_symbols
in [symbols
[i
:i
+ 50] for i
in range(0, len(symbols
), 50)]:
403 r
= 'CONFIG_((' + '|'.join([s
+ '(_MODULE)?' for s
in some_symbols
]) + ')([^A-Za-z0-9_]|$))'
404 regexes
.append(re
.compile(r
, re
.MULTILINE
))
405 for root
, dirs
, files
in os
.walk(args
.outdir
):
406 # don't go into .git dir (possible debug thing)
410 data
= open(os
.path
.join(root
, f
), 'r').read()
412 data
= r
.sub(r
'CPTCFG_\1', data
)
413 fo
= open(os
.path
.join(root
, f
), 'w')
417 git_debug_snapshot(args
, "rename config symbol usage")
419 # disable unbuildable Kconfig symbols and stuff Makefiles that doesn't exist
420 maketree
= make
.MakeTree(os
.path
.join(args
.outdir
, 'Makefile.kernel'))
422 disable_makefile
= []
423 for sym
in maketree
.get_impossible_symbols():
424 disable_kconfig
.append(sym
[7:])
425 disable_makefile
.append(sym
[7:])
427 configtree
.disable_symbols(disable_kconfig
)
428 git_debug_snapshot(args
, "disable impossible kconfig symbols")
430 # add kernel version dependencies to Kconfig, from the dependency list
432 for sym
in tuple(deplist
.keys()):
434 for dep
in deplist
[sym
]:
435 new
.append('!BACKPORT_KERNEL_%s' % dep
.replace('.', '_'))
437 configtree
.add_dependencies(deplist
)
438 git_debug_snapshot(args
, "add kernel version dependencies")
440 # disable things in makefiles that can't be selected and that the
441 # build shouldn't recurse into because they don't exist -- if we
442 # don't do that then a symbol from the kernel could cause the build
443 # to attempt to recurse and fail
445 # Note that we split the regex after 50 symbols, this is because of a
446 # limitation in the regex implementation (it only supports 100 nested
447 # groups -- 50 seemed safer and is still fast)
449 for some_symbols
in [disable_makefile
[i
:i
+ 50] for i
in range(0, len(disable_makefile
), 50)]:
450 r
= '((CPTCFG|CONFIG)_(' + '|'.join([s
for s
in some_symbols
]) + '))'
451 regexes
.append(re
.compile(r
, re
.MULTILINE
))
452 for f
in maketree
.get_makefiles():
453 data
= open(f
, 'r').read()
455 data
= r
.sub(r
'IMPOSSIBLE_\3', data
)
459 git_debug_snapshot(args
, "disable unsatisfied Makefile parts")
464 if __name__
== '__main__':