3 # Generate the output tree, by default in the output/ directory
4 # but a different one may be specified. The directory must be
5 # empty already. It's also allowed to not exist.
8 import argparse
, sys
, os
, errno
, shutil
, re
, subprocess
11 source_dir
= os
.path
.abspath(os
.path
.dirname(sys
.argv
[0]))
12 sys
.path
.append(os
.path
.join(source_dir
, 'lib'))
13 import kconfig
, git
, patch
, make
15 def read_copy_list(copyfile
):
18 # remove leading/trailing whitespace
21 if not item
or item
[0] == '#':
24 raise Exception("Input path '%s' is absolute path, this isn't allowed" % (item
, ))
26 srcitem
, dstitem
= item
.split(' -> ')
27 if (srcitem
[-1] == '/') != (dstitem
[-1] == '/'):
28 raise Exception("Cannot copy file/dir to dir/file")
30 srcitem
= dstitem
= item
31 ret
.append((srcitem
, dstitem
))
34 def read_dependencies(depfilename
):
36 depfile
= open(depfilename
)
39 if not item
or item
[0] == '#':
41 sym
, dep
= item
.split()
48 def check_output_dir(d
, clean
):
50 shutil
.rmtree(d
, ignore_errors
=True)
54 if e
.errno
!= errno
.ENOENT
:
57 def copytree(src
, dst
, symlinks
=False, ignore
=None):
58 names
= os
.listdir(src
)
59 if ignore
is not None:
60 ignored_names
= ignore(src
, names
)
64 if not os
.path
.isdir(dst
):
68 if name
in ignored_names
:
70 srcname
= os
.path
.join(src
, name
)
71 dstname
= os
.path
.join(dst
, name
)
73 if symlinks
and os
.path
.islink(srcname
):
74 linkto
= os
.readlink(srcname
)
75 os
.symlink(linkto
, dstname
)
76 elif os
.path
.isdir(srcname
):
77 copytree(srcname
, dstname
, symlinks
, ignore
)
79 shutil
.copy2(srcname
, dstname
)
80 except (IOError, os
.error
) as why
:
81 errors
.append((srcname
, dstname
, str(why
)))
82 # catch the Error from the recursive copytree so that we can
83 # continue with other files
84 except shutil
.Error
as err
:
85 errors
.extend(err
.args
[0])
87 shutil
.copystat(src
, dst
)
89 # can't copy file access times on Windows
91 except OSError as why
:
92 errors
.extend((src
, dst
, str(why
)))
94 raise shutil
.Error(errors
)
96 def copy_files(srcpath
, copy_list
, outdir
):
97 for srcitem
, tgtitem
in copy_list
:
99 copytree(srcpath
, outdir
, ignore
=shutil
.ignore_patterns('*~'))
100 elif tgtitem
[-1] == '/':
101 def copy_ignore(dir, entries
):
104 if (not i
[-1] in ('c', 'h') and
106 not i
in ('Kconfig', 'Makefile') and
107 not os
.path
.isdir(os
.path
.join(dir, i
))):
110 copytree(os
.path
.join(srcpath
, srcitem
),
111 os
.path
.join(outdir
, tgtitem
),
115 os
.makedirs(os
.path
.join(outdir
, os
.path
.dirname(tgtitem
)))
117 # ignore dirs we might have created just now
118 if e
.errno
!= errno
.EEXIST
:
120 shutil
.copy(os
.path
.join(srcpath
, srcitem
),
121 os
.path
.join(outdir
, tgtitem
))
123 def copy_git_files(srcpath
, copy_list
, rev
, outdir
):
124 for srcitem
, tgtitem
in copy_list
:
125 for m
, t
, h
, f
in git
.ls_tree(rev
=rev
, files
=(srcitem
,), tree
=srcpath
):
127 f
= os
.path
.join(outdir
, f
)
128 d
= os
.path
.dirname(f
)
129 if not os
.path
.exists(d
):
132 git
.get_blob(h
, outf
, tree
=srcpath
)
134 os
.chmod(f
, int(m
, 8))
136 def git_debug_init(args
):
137 if not args
.gitdebug
:
139 git
.init(tree
=args
.outdir
)
140 git
.commit_all("Copied code", tree
=args
.outdir
)
142 def git_debug_snapshot(args
, name
):
143 if not args
.gitdebug
:
145 git
.commit_all(name
, tree
=args
.outdir
)
148 # set up and parse arguments
149 parser
= argparse
.ArgumentParser(description
='generate backport tree')
150 parser
.add_argument('kerneldir', metavar
='<kernel tree>', type=str,
151 help='Kernel tree to copy drivers from')
152 parser
.add_argument('outdir', metavar
='<output directory>', type=str,
153 help='Directory to write the generated tree to')
154 parser
.add_argument('--copy-list', metavar
='<listfile>', type=argparse
.FileType('r'),
156 help='File containing list of files/directories to copy, default "copy-list"')
157 parser
.add_argument('--git-revision', metavar
='<revision>', type=str,
158 help='git commit revision (see gitrevisions(7)) to take objects from.' +
159 'If this is specified, the kernel tree is used as git object storage ' +
160 'and we use git ls-tree to get the files.')
161 parser
.add_argument('--clean', const
=True, default
=False, action
="store_const",
162 help='Clean output directory instead of erroring if it isn\'t empty')
163 parser
.add_argument('--refresh', const
=True, default
=False, action
="store_const",
164 help='Refresh patches as they are applied, the source dir will be modified!')
165 parser
.add_argument('--base-name', metavar
='<name>', type=str, default
='Linux',
166 help='name of base tree, default just "Linux"')
167 parser
.add_argument('--gitdebug', const
=True, default
=False, action
="store_const",
168 help='Use git, in the output tree, to debug the various transformation steps ' +
169 'that the tree generation makes (apply patches, ...)')
170 parser
.add_argument('--verbose', const
=True, default
=False, action
="store_const",
171 help='Print more verbose information')
172 args
= parser
.parse_args()
174 # then add stuff from the copy list file
175 copy_list
= read_copy_list(args
.copy_list
)
177 deplist
= read_dependencies(os
.path
.join(source_dir
, 'dependencies'))
179 # validate output directory
180 check_output_dir(args
.outdir
, args
.clean
)
183 if not args
.git_revision
:
184 print 'Copy original source files ...'
185 copy_files(os
.path
.join(source_dir
, 'backport'), [('', '')], args
.outdir
)
186 copy_files(args
.kerneldir
, copy_list
, args
.outdir
)
188 print 'Get original source files from git ...'
189 copy_files(os
.path
.join(source_dir
, 'backport'), [('', '')], args
.outdir
)
190 copy_git_files(args
.kerneldir
, copy_list
, args
.git_revision
, args
.outdir
)
194 git_debug_snapshot(args
, "add versions/symbols files")
196 print 'Apply patches ...'
198 for root
, dirs
, files
in os
.walk(os
.path
.join(source_dir
, 'patches')):
200 patchdirs
.append(root
)
202 for pdir
in patchdirs
:
208 pfile
= os
.path
.join(pdir
, pfile
)
209 p
= patch
.fromfile(pfile
)
212 patched_file
= '/'.join(p
.items
[0].source
.split('/')[1:])
213 fullfn
= os
.path
.join(args
.outdir
, patched_file
)
214 if not os
.path
.exists(fullfn
):
218 print "Applying changes from", os
.path
.basename(pdir
)
221 for patchitem
in p
.items
:
222 patched_file
= '/'.join(patchitem
.source
.split('/')[1:])
223 fullfn
= os
.path
.join(args
.outdir
, patched_file
)
224 shutil
.copyfile(fullfn
, fullfn
+ '.orig_file')
225 process
= subprocess
.Popen(['patch', '-p1'], stdout
=subprocess
.PIPE
,
226 stderr
=subprocess
.STDOUT
, stdin
=subprocess
.PIPE
,
227 close_fds
=True, universal_newlines
=True,
229 output
= process
.communicate(input=open(pfile
, 'r').read())[0]
230 output
= output
.split('\n')
236 if process
.returncode
!= 0:
238 print "Failed to apply changes from", os
.path
.basename(pdir
)
243 pfilef
= open(pfile
+ '.tmp', 'w')
244 for patchitem
in p
.items
:
245 patched_file
= '/'.join(patchitem
.source
.split('/')[1:])
246 fullfn
= os
.path
.join(args
.outdir
, patched_file
)
247 process
= subprocess
.Popen(['diff', '-u', patched_file
+ '.orig_file', patched_file
,
248 '--label', 'a/' + patched_file
,
249 '--label', 'b/' + patched_file
],
250 stdout
=pfilef
, close_fds
=True,
251 universal_newlines
=True, cwd
=args
.outdir
)
253 os
.unlink(fullfn
+ '.orig_file')
254 if not process
.returncode
in (0, 1):
255 print "Diffing for refresh failed!"
257 os
.unlink(pfile
+ '.tmp')
260 os
.rename(pfile
+ '.tmp', pfile
)
261 # remove orig/rej files that patch sometimes creates
262 for root
, dirs
, files
in os
.walk(args
.outdir
):
264 if f
[-5:] == '.orig' or f
[-4:] == '.rej':
265 os
.unlink(os
.path
.join(root
, f
))
268 print "Not applying changes from %s, not needed" % (os
.path
.basename(pdir
),)
270 git_debug_snapshot(args
, "apply backport patches from %s" % (os
.path
.basename(pdir
),))
272 # some post-processing is required
273 configtree
= kconfig
.ConfigTree(os
.path
.join(args
.outdir
, 'Kconfig'))
274 print 'Modify Kconfig tree ...'
275 configtree
.prune_sources(ignore
=['Kconfig.kernel', 'Kconfig.versions'])
276 git_debug_snapshot(args
, "prune Kconfig tree")
277 configtree
.force_tristate_modular()
278 git_debug_snapshot(args
, "force tristate options modular")
279 configtree
.modify_selects()
280 git_debug_snapshot(args
, "convert select to depends on")
282 # write the versioning file
283 backports_version
= git
.describe(tree
=source_dir
)
284 kernel_version
= git
.describe(tree
=args
.kerneldir
)
285 f
= open(os
.path
.join(args
.outdir
, 'versions'), 'w')
286 f
.write('BACKPORTS_VERSION="%s"\n' % backports_version
)
287 f
.write('KERNEL_VERSION="%s"\n' % kernel_version
)
288 f
.write('KERNEL_NAME="%s"\n' % args
.base_name
)
291 symbols
= configtree
.symbols()
293 # write local symbol list
294 f
= open(os
.path
.join(args
.outdir
, '.local-symbols'), 'w')
296 f
.write('%s=\n' % sym
)
299 print 'Rewrite Makefiles and Kconfig files ...'
301 # rewrite Makefile and source symbols
303 for some_symbols
in [symbols
[i
:i
+ 50] for i
in range(0, len(symbols
), 50)]:
304 r
= 'CONFIG_((' + '|'.join([s
+ '(_MODULE)?' for s
in some_symbols
]) + ')([^A-Za-z0-9_]|$))'
305 regexes
.append(re
.compile(r
, re
.MULTILINE
))
306 for root
, dirs
, files
in os
.walk(args
.outdir
):
307 # don't go into .git dir (possible debug thing)
311 data
= open(os
.path
.join(root
, f
), 'r').read()
313 data
= r
.sub(r
'CPTCFG_\1', data
)
314 fo
= open(os
.path
.join(root
, f
), 'w')
318 git_debug_snapshot(args
, "rename config symbol usage")
320 # disable unbuildable Kconfig symbols and stuff Makefiles that doesn't exist
321 maketree
= make
.MakeTree(os
.path
.join(args
.outdir
, 'Makefile.kernel'))
323 disable_makefile
= []
324 for sym
in maketree
.get_impossible_symbols():
325 if sym
[:7] == 'CPTCFG_':
326 disable_kconfig
.append(sym
[7:])
328 disable_makefile
.append(sym
[7:])
330 configtree
.disable_symbols(disable_kconfig
)
331 git_debug_snapshot(args
, "disable impossible kconfig symbols")
333 for sym
in tuple(deplist
.keys()):
335 for dep
in deplist
[sym
]:
336 new
.append('!BACKPORT_KERNEL_%s' % dep
.replace('.', '_'))
338 configtree
.add_dependencies(deplist
)
339 git_debug_snapshot(args
, "add kernel version dependencies")
342 for some_symbols
in [disable_makefile
[i
:i
+ 50] for i
in range(0, len(disable_makefile
), 50)]:
343 r
= '(CONFIG_(' + '|'.join([s
for s
in some_symbols
]) + '))'
344 regexes
.append(re
.compile(r
, re
.MULTILINE
))
345 for f
in maketree
.get_makefiles():
346 data
= open(f
, 'r').read()
348 data
= r
.sub(r
'IMPOSSIBLE_\2', data
)
352 git_debug_snapshot(args
, "disable unsatisfied Makefile parts")