gentree.py: fix directory creating
[openwrt/staging/blogic.git] / gentree.py
1 #!/usr/bin/env python
2 #
3 # Generate the output tree into a specified directory.
4 #
5
6 import argparse, sys, os, errno, shutil, re, subprocess
7 import tarfile, bz2
8
9 # find self
10 source_dir = os.path.abspath(os.path.dirname(__file__))
11 sys.path.append(source_dir)
12 # and import libraries we have
13 from lib import kconfig, patch, make
14 from lib import bpgit as git
15 from lib import bpgpg as gpg
16 from lib import bpkup as kup
17
18 def read_copy_list(copyfile):
19 """
20 Read a copy-list file and return a list of (source, target)
21 tuples. The source and target are usually the same, but in
22 the copy-list file there may be a rename included.
23 """
24 ret = []
25 for item in copyfile:
26 # remove leading/trailing whitespace
27 item = item.strip()
28 # comments
29 if not item or item[0] == '#':
30 continue
31 if item[0] == '/':
32 raise Exception("Input path '%s' is absolute path, this isn't allowed" % (item, ))
33 if ' -> ' in item:
34 srcitem, dstitem = item.split(' -> ')
35 if (srcitem[-1] == '/') != (dstitem[-1] == '/'):
36 raise Exception("Cannot copy file/dir to dir/file")
37 else:
38 srcitem = dstitem = item
39 ret.append((srcitem, dstitem))
40 return ret
41
42
43 def read_dependencies(depfilename):
44 """
45 Read a (the) dependency file and return the list of
46 dependencies as a dictionary, mapping a Kconfig symbol
47 to a list of kernel version dependencies.
48
49 If a backported feature that an upstream backported driver
50 depends on had kconfig limitations (ie, debugging feature not
51 available) a built constaint restriction can be expressed
52 by using a kconfig expression. The kconfig expressions can
53 be specified by using the "kconfig: " prefix.
54
55 While reading ignore blank or commented lines.
56 """
57 ret = {}
58 depfile = open(depfilename, 'r')
59 for item in depfile:
60 kconfig_exp = ""
61 item = item.strip()
62 if not item or item[0] == '#':
63 continue
64 if "kconfig:" in item:
65 sym, kconfig_exp = item.split(" ", 1)
66 if not sym in ret:
67 ret[sym] = [kconfig_exp, ]
68 else:
69 ret[sym].append(kconfig_exp)
70 else:
71 sym, dep = item.split()
72 if not sym in ret:
73 ret[sym] = [dep, ]
74 else:
75 ret[sym].append(dep)
76 return ret
77
78
79 def check_output_dir(d, clean):
80 """
81 Check that the output directory doesn't exist or is empty,
82 unless clean is True in which case it's nuked. This helps
83 sanity check the output when generating a tree, so usually
84 running with --clean isn't suggested.
85 """
86 if clean:
87 shutil.rmtree(d, ignore_errors=True)
88 try:
89 os.rmdir(d)
90 except OSError as e:
91 if e.errno != errno.ENOENT:
92 raise
93
94
95 def copytree(src, dst, symlinks=False, ignore=None):
96 """
97 Copy a directory tree. This differs from shutil.copytree()
98 in that it allows destination directories to already exist.
99 """
100 names = os.listdir(src)
101 if ignore is not None:
102 ignored_names = ignore(src, names)
103 else:
104 ignored_names = set()
105
106 if not os.path.isdir(dst):
107 os.makedirs(dst)
108 errors = []
109 for name in names:
110 if name in ignored_names:
111 continue
112 srcname = os.path.join(src, name)
113 dstname = os.path.join(dst, name)
114 try:
115 if symlinks and os.path.islink(srcname):
116 linkto = os.readlink(srcname)
117 os.symlink(linkto, dstname)
118 elif os.path.isdir(srcname):
119 copytree(srcname, dstname, symlinks, ignore)
120 else:
121 shutil.copy2(srcname, dstname)
122 except (IOError, os.error) as why:
123 errors.append((srcname, dstname, str(why)))
124 # catch the Error from the recursive copytree so that we can
125 # continue with other files
126 except shutil.Error as err:
127 errors.extend(err.args[0])
128 try:
129 shutil.copystat(src, dst)
130 except WindowsError:
131 # can't copy file access times on Windows
132 pass
133 except OSError as why:
134 errors.extend((src, dst, str(why)))
135 if errors:
136 raise shutil.Error(errors)
137
138
139 def copy_files(srcpath, copy_list, outdir):
140 """
141 Copy the copy_list files and directories from the srcpath
142 to the outdir. The copy_list contains source and target
143 names.
144
145 For now, it also ignores any *~ editor backup files, though
146 this should probably be generalized (maybe using .gitignore?)
147 Similarly the code that only copies some files (*.c, *.h,
148 *.awk, Kconfig, Makefile) to avoid any build remnants in the
149 kernel if they should exist.
150 """
151 for srcitem, tgtitem in copy_list:
152 if tgtitem == '':
153 copytree(srcpath, outdir, ignore=shutil.ignore_patterns('*~'))
154 elif tgtitem[-1] == '/':
155 def copy_ignore(dir, entries):
156 r = []
157 for i in entries:
158 if i[-2:] == '.o' or i[-1] == '~':
159 r.append(i)
160 return r
161 copytree(os.path.join(srcpath, srcitem),
162 os.path.join(outdir, tgtitem),
163 ignore=copy_ignore)
164 else:
165 try:
166 os.makedirs(os.path.join(outdir, os.path.dirname(tgtitem)))
167 except OSError as e:
168 # ignore dirs we might have created just now
169 if e.errno != errno.EEXIST:
170 raise
171 shutil.copy(os.path.join(srcpath, srcitem),
172 os.path.join(outdir, tgtitem))
173
174
175 def copy_git_files(srcpath, copy_list, rev, outdir):
176 """
177 "Copy" files from a git repository. This really means listing them with
178 ls-tree and then using git show to obtain all the blobs.
179 """
180 for srcitem, tgtitem in copy_list:
181 for m, t, h, f in git.ls_tree(rev=rev, files=(srcitem,), tree=srcpath):
182 assert t == 'blob'
183 f = os.path.join(outdir, f.replace(srcitem, tgtitem))
184 d = os.path.dirname(f)
185 if not os.path.exists(d):
186 os.makedirs(d)
187 outf = open(f, 'w')
188 git.get_blob(h, outf, tree=srcpath)
189 outf.close()
190 os.chmod(f, int(m, 8))
191
192 def automatic_backport_mangle_c_file(name):
193 return name.replace('/', '-')
194
195
196 def add_automatic_backports(args):
197 disable_list = []
198 export = re.compile(r'^EXPORT_SYMBOL(_GPL)?\((?P<sym>[^\)]*)\)')
199 bpi = kconfig.get_backport_info(os.path.join(args.outdir, 'compat', 'Kconfig'))
200 configtree = kconfig.ConfigTree(os.path.join(args.outdir, 'Kconfig'))
201 all_selects = configtree.all_selects()
202 for sym, vals in bpi.items():
203 if sym.startswith('BACKPORT_BUILD_'):
204 if not sym[15:] in all_selects:
205 disable_list.append(sym)
206 continue
207 symtype, module_name, c_files, h_files = vals
208
209 # first copy files
210 files = []
211 for f in c_files:
212 files.append((f, os.path.join('compat', automatic_backport_mangle_c_file(f))))
213 for f in h_files:
214 files.append((os.path.join('include', f),
215 os.path.join('include', os.path.dirname(f), 'backport-' + os.path.basename(f))))
216 if args.git_revision:
217 copy_git_files(args.kerneldir, files, args.git_revision, args.outdir)
218 else:
219 copy_files(args.kerneldir, files, args.outdir)
220
221 # now add the Makefile line
222 mf = open(os.path.join(args.outdir, 'compat', 'Makefile'), 'a+')
223 o_files = [automatic_backport_mangle_c_file(f)[:-1] + 'o' for f in c_files]
224 if symtype == 'tristate':
225 if not module_name:
226 raise Exception('backporting a module requires a #module-name')
227 for of in o_files:
228 mf.write('%s-objs += %s\n' % (module_name, of))
229 mf.write('obj-$(CPTCFG_%s) += %s.o\n' % (sym, module_name))
230 elif symtype == 'bool':
231 mf.write('compat-$(CPTCFG_%s) += %s\n' % (sym, ' '.join(o_files)))
232
233 # finally create the include file
234 syms = []
235 for f in c_files:
236 for l in open(os.path.join(args.outdir, 'compat',
237 automatic_backport_mangle_c_file(f)), 'r'):
238 m = export.match(l)
239 if m:
240 syms.append(m.group('sym'))
241 for f in h_files:
242 outf = open(os.path.join(args.outdir, 'include', f), 'w')
243 outf.write('/* Automatically created during backport process */\n')
244 outf.write('#ifndef CPTCFG_%s\n' % sym)
245 outf.write('#include_next <%s>\n' % f)
246 outf.write('#else\n');
247 for s in syms:
248 outf.write('#undef %s\n' % s)
249 outf.write('#define %s LINUX_BACKPORT(%s)\n' % (s, s))
250 outf.write('#include <%s>\n' % (os.path.dirname(f) + '/backport-' + os.path.basename(f), ))
251 outf.write('#endif /* CPTCFG_%s */\n' % sym)
252 return disable_list
253
254 def git_debug_init(args):
255 """
256 Initialize a git repository in the output directory and commit the current
257 code in it. This is only used for debugging the transformations this code
258 will do to the output later.
259 """
260 if not args.gitdebug:
261 return
262 git.init(tree=args.outdir)
263 git.commit_all("Copied backport", tree=args.outdir)
264
265
266 def git_debug_snapshot(args, name):
267 """
268 Take a git snapshot for the debugging.
269 """
270 if not args.gitdebug:
271 return
272 git.commit_all(name, tree=args.outdir)
273
274 def get_rel_spec_stable(rel):
275 """
276 Returns release specs for a linux-stable backports based release.
277 """
278 if ("rc" in rel):
279 m = re.match(r"(?P<VERSION>\d+)\.+" \
280 "(?P<PATCHLEVEL>\d+)[.]*" \
281 "(?P<SUBLEVEL>\d*)" \
282 "[-rc]+(?P<RC_VERSION>\d+)\-+" \
283 "(?P<RELMOD_UPDATE>\d+)[-]*" \
284 "(?P<RELMOD_TYPE>[usnpc]*)", \
285 rel)
286 else:
287 m = re.match(r"(?P<VERSION>\d+)\.+" \
288 "(?P<PATCHLEVEL>\d+)[.]*" \
289 "(?P<SUBLEVEL>\d*)\-+" \
290 "(?P<RELMOD_UPDATE>\d+)[-]*" \
291 "(?P<RELMOD_TYPE>[usnpc]*)", \
292 rel)
293 if (not m):
294 return m
295 return m.groupdict()
296
297 def get_rel_spec_next(rel):
298 """
299 Returns release specs for a linux-next backports based release.
300 """
301 m = re.match(r"(?P<DATE_VERSION>\d+)[-]*" \
302 "(?P<RELMOD_UPDATE>\d*)[-]*" \
303 "(?P<RELMOD_TYPE>[usnpc]*)", \
304 rel)
305 if (not m):
306 return m
307 return m.groupdict()
308
309 def get_rel_prep(rel):
310 """
311 Returns a dict with prep work details we need prior to
312 uploading a backports release to kernel.org
313 """
314 rel_specs = get_rel_spec_stable(rel)
315 is_stable = True
316 rel_tag = ""
317 paths = list()
318 if (not rel_specs):
319 rel_specs = get_rel_spec_next(rel)
320 if (not rel_specs):
321 sys.stdout.write("rel: %s\n" % rel)
322 return None
323 if (rel_specs['RELMOD_UPDATE'] == '0' or
324 rel_specs['RELMOD_UPDATE'] == '1'):
325 return None
326 is_stable = False
327 date = rel_specs['DATE_VERSION']
328 year = date[0:4]
329 if (len(year) != 4):
330 return None
331 month = date[4:6]
332 if (len(month) != 2):
333 return None
334 day = date[6:8]
335 if (len(day) != 2):
336 return None
337 paths.append(year)
338 paths.append(month)
339 paths.append(day)
340 rel_tag = "backports-" + rel.replace(rel_specs['RELMOD_TYPE'], "")
341 else:
342 ignore = "-"
343 if (not rel_specs['RELMOD_UPDATE']):
344 return None
345 if (rel_specs['RELMOD_UPDATE'] == '0'):
346 return None
347 ignore += rel_specs['RELMOD_UPDATE']
348 if (rel_specs['RELMOD_TYPE'] != ''):
349 ignore += rel_specs['RELMOD_TYPE']
350 base_rel = rel.replace(ignore, "")
351 paths.append(base_rel)
352 rel_tag = "v" + rel.replace(rel_specs['RELMOD_TYPE'], "")
353
354 rel_prep = dict(stable = is_stable,
355 expected_tag = rel_tag,
356 paths_to_create = paths)
357 return rel_prep
358
359 def create_tar_and_bz2(tar_name, dir_to_tar):
360 """
361 We need both a tar file and bzip2 for kernel.org, the tar file
362 gets signed, then we upload the compressed version, kup-server
363 in the backend decompresses and verifies the tarball against
364 our signature.
365 """
366 basename = os.path.basename(dir_to_tar)
367 tar = tarfile.open(tar_name, "w")
368 tar.add(dir_to_tar, basename)
369 tar.close()
370
371 tar_file = open(tar_name, "r")
372
373 bz2_file = bz2.BZ2File(tar_name + ".bz2", 'wb', compresslevel=9)
374 bz2_file.write(tar_file.read())
375 bz2_file.close()
376
377 def upload_release(args, rel_prep, logwrite=lambda x:None):
378 """
379 Given a path of a relase make tarball out of it, PGP sign it, and
380 then upload it to kernel.org using kup.
381
382 The linux-next based release do not require a RELMOD_UPDATE
383 given that typically only one release is made per day. Using
384 RELMOD_UPDATE for these releases is allowed though and if
385 present it must be > 1.
386
387 The linux-stable based releases require a RELMOD_UPDATE.
388
389 RELMOD_UPDATE must be numeric and > 0 just as the RC releases
390 of the Linux kernel.
391
392 The tree must also be tagged with the respective release, without
393 the RELMOD_TYPE. For linux-next based releases this consists of
394 backports- followed by DATE_VERSION and if RELMOD_TYPE is present.
395 For linux-stable releases this consists of v followed by the
396 full release version except the RELMOD_TYPE.
397
398 Uploads will not be allowed if these rules are not followed.
399 """
400 korg_path = "/pub/linux/kernel/projects/backports"
401
402 if (rel_prep['stable']):
403 korg_path += "stable"
404
405 parent = os.path.dirname(args.outdir)
406 release = os.path.basename(args.outdir)
407 tar_name = parent + '/' + release + ".tar"
408 bzip2_name = tar_name + ".bz2"
409
410 create_tar_and_bz2(tar_name, args.outdir)
411
412 logwrite(gpg.sign(tar_name, extra_args=['--armor', '--detach-sign']))
413
414 logwrite("------------------------------------------------------")
415
416 if (not args.kup_test):
417 logwrite("About to upload, current target path contents:")
418 else:
419 logwrite("kup-test: current target path contents:")
420
421 logwrite(kup.ls(path=korg_path))
422
423 for path in rel_prep['paths_to_create']:
424 korg_path += '/' + path
425 if (not args.kup_test):
426 logwrite(kup.mkdir(korg_path))
427 if (not args.kup_test):
428 logwrite(kup.put(bzip2_name, tar_name + '.asc', korg_path))
429 logwrite("\nFinished upload!\n")
430 logwrite("Target path contents:")
431 logwrite(kup.ls(path=korg_path))
432 else:
433 kup_cmd = "kup put /\n\t\t%s /\n\t\t%s /\n\t\t%s" % (bzip2_name, tar_name + '.asc', korg_path)
434 logwrite("kup-test: skipping cmd: %s" % kup_cmd)
435
436 def _main():
437 # set up and parse arguments
438 parser = argparse.ArgumentParser(description='generate backport tree')
439 parser.add_argument('kerneldir', metavar='<kernel tree>', type=str,
440 help='Kernel tree to copy drivers from')
441 parser.add_argument('outdir', metavar='<output directory>', type=str,
442 help='Directory to write the generated tree to')
443 parser.add_argument('--copy-list', metavar='<listfile>', type=argparse.FileType('r'),
444 default='copy-list',
445 help='File containing list of files/directories to copy, default "copy-list"')
446 parser.add_argument('--git-revision', metavar='<revision>', type=str,
447 help='git commit revision (see gitrevisions(7)) to take objects from.' +
448 'If this is specified, the kernel tree is used as git object storage ' +
449 'and we use git ls-tree to get the files.')
450 parser.add_argument('--clean', const=True, default=False, action="store_const",
451 help='Clean output directory instead of erroring if it isn\'t empty')
452 parser.add_argument('--refresh', const=True, default=False, action="store_const",
453 help='Refresh patches as they are applied, the source dir will be modified!')
454 parser.add_argument('--base-name', metavar='<name>', type=str, default='Linux',
455 help='name of base tree, default just "Linux"')
456 parser.add_argument('--gitdebug', const=True, default=False, action="store_const",
457 help='Use git, in the output tree, to debug the various transformation steps ' +
458 'that the tree generation makes (apply patches, ...)')
459 parser.add_argument('--verbose', const=True, default=False, action="store_const",
460 help='Print more verbose information')
461 parser.add_argument('--extra-driver', nargs=2, metavar=('<source dir>', '<copy-list>'), type=str,
462 action='append', default=[], help='Extra driver directory/copy-list.')
463 parser.add_argument('--kup', const=True, default=False, action="store_const",
464 help='For maintainers: upload a release to kernel.org')
465 parser.add_argument('--kup-test', const=True, default=False, action="store_const",
466 help='For maintainers: do all the work as if you were about to ' +
467 'upload to kernel.org but do not do the final `kup put` ' +
468 'and also do not run any `kup mkdir` commands. This will ' +
469 'however run `kup ls` on the target paths so ' +
470 'at the very least we test your kup configuration. ' +
471 'If this is your first time uploading use this first!')
472 args = parser.parse_args()
473
474 def logwrite(msg):
475 sys.stdout.write(msg)
476 sys.stdout.write('\n')
477 sys.stdout.flush()
478
479 return process(args.kerneldir, args.outdir, args.copy_list,
480 git_revision=args.git_revision, clean=args.clean,
481 refresh=args.refresh, base_name=args.base_name,
482 gitdebug=args.gitdebug, verbose=args.verbose,
483 extra_driver=args.extra_driver,
484 kup=args.kup,
485 kup_test=args.kup_test,
486 logwrite=logwrite)
487
488 def process(kerneldir, outdir, copy_list_file, git_revision=None,
489 clean=False, refresh=False, base_name="Linux", gitdebug=False,
490 verbose=False, extra_driver=[], kup=False,
491 kup_test=False,
492 logwrite=lambda x:None,
493 git_tracked_version=False):
494 class Args(object):
495 def __init__(self, kerneldir, outdir, copy_list_file,
496 git_revision, clean, refresh, base_name,
497 gitdebug, verbose, extra_driver, kup,
498 kup_test):
499 self.kerneldir = kerneldir
500 self.outdir = outdir
501 self.copy_list = copy_list_file
502 self.git_revision = git_revision
503 self.clean = clean
504 self.refresh = refresh
505 self.base_name = base_name
506 self.gitdebug = gitdebug
507 self.verbose = verbose
508 self.extra_driver = extra_driver
509 self.kup = kup
510 self.kup_test = kup_test
511 def git_paranoia(tree=None, logwrite=lambda x:None):
512 data = git.paranoia(tree)
513 if (data['r'] != 0):
514 logwrite('Cannot use %s' % tree)
515 logwrite('%s' % data['output'])
516 sys.exit(data['r'])
517 else:
518 logwrite('Validated tree: %s' % tree)
519
520 args = Args(kerneldir, outdir, copy_list_file,
521 git_revision, clean, refresh, base_name,
522 gitdebug, verbose, extra_driver, kup, kup_test)
523 rel_prep = None
524
525 # start processing ...
526 if (args.kup or args.kup_test):
527 git_paranoia(source_dir, logwrite)
528 git_paranoia(kerneldir, logwrite)
529
530 rel_describe = git.describe(rev=None, tree=source_dir, extra_args=['--dirty'])
531 release = os.path.basename(args.outdir)
532 version = release.replace("backports-", "")
533
534 rel_prep = get_rel_prep(version)
535 if (not rel_prep):
536 logwrite('Invalid backports release name: %s' % release)
537 logwrite('For rules on the release name see upload_release()')
538 sys.exit(1)
539 rel_type = "linux-stable"
540 if (not rel_prep['stable']):
541 rel_type = "linux-next"
542 if (rel_prep['expected_tag'] != rel_describe):
543 logwrite('Unexpected %s based backports release tag on' % rel_type)
544 logwrite('the backports tree tree: %s\n' % rel_describe)
545 logwrite('You asked to make a release with this ')
546 logwrite('directory name: %s' % release)
547 logwrite('The actual expected tag we should find on')
548 logwrite('the backports tree then is: %s\n' % rel_prep['expected_tag'])
549 logwrite('For rules on the release name see upload_release()')
550 sys.exit(1)
551
552 copy_list = read_copy_list(args.copy_list)
553 deplist = read_dependencies(os.path.join(source_dir, 'dependencies'))
554
555 # validate output directory
556 check_output_dir(args.outdir, args.clean)
557
558 # do the copy
559 backport_files = [(x, x) for x in [
560 'Kconfig', 'Makefile', 'Makefile.build', 'Makefile.kernel', '.gitignore',
561 'Makefile.real', 'compat/', 'backport-include/', 'kconf/', 'defconfigs/',
562 'scripts/', '.blacklist.map', 'udev/',
563 ]]
564 if not args.git_revision:
565 logwrite('Copy original source files ...')
566 else:
567 logwrite('Get original source files from git ...')
568
569 copy_files(os.path.join(source_dir, 'backport'), backport_files, args.outdir)
570
571 git_debug_init(args)
572
573 if not args.git_revision:
574 copy_files(args.kerneldir, copy_list, args.outdir)
575 else:
576 copy_git_files(args.kerneldir, copy_list, args.git_revision, args.outdir)
577
578 # FIXME: should we add a git version of this (e.g. --git-extra-driver)?
579 for src, copy_list in args.extra_driver:
580 if (args.kup or args.kup_test):
581 git_paranoia(src)
582 copy_files(src, read_copy_list(open(copy_list, 'r')), args.outdir)
583
584 git_debug_snapshot(args, 'Add driver sources')
585
586 disable_list = add_automatic_backports(args)
587 if disable_list:
588 bpcfg = kconfig.ConfigTree(os.path.join(args.outdir, 'compat', 'Kconfig'))
589 bpcfg.disable_symbols(disable_list)
590 git_debug_snapshot(args, 'Add automatic backports')
591
592 logwrite('Apply patches ...')
593 patches = []
594 for root, dirs, files in os.walk(os.path.join(source_dir, 'patches')):
595 for f in files:
596 if f.endswith('.patch'):
597 patches.append(os.path.join(root, f))
598 patches.sort()
599 prefix_len = len(os.path.join(source_dir, 'patches')) + 1
600 for pfile in patches:
601 print_name = pfile[prefix_len:]
602 # read the patch file
603 p = patch.fromfile(pfile)
604 # complain if it's not a patch
605 if not p:
606 raise Exception('No patch content found in %s' % print_name)
607 # leading / seems to be stripped?
608 if 'dev/null' in p.items[0].source:
609 raise Exception('Patches creating files are not supported (in %s)' % print_name)
610 # check if the first file the patch touches exists, if so
611 # assume the patch needs to be applied -- otherwise continue
612 patched_file = '/'.join(p.items[0].source.split('/')[1:])
613 fullfn = os.path.join(args.outdir, patched_file)
614 if not os.path.exists(fullfn):
615 if args.verbose:
616 logwrite("Not applying %s, not needed" % print_name)
617 continue
618 if args.verbose:
619 logwrite("Applying patch %s" % print_name)
620
621 if args.refresh:
622 # but for refresh, of course look at all files the patch touches
623 for patchitem in p.items:
624 patched_file = '/'.join(patchitem.source.split('/')[1:])
625 fullfn = os.path.join(args.outdir, patched_file)
626 shutil.copyfile(fullfn, fullfn + '.orig_file')
627
628 process = subprocess.Popen(['patch', '-p1'], stdout=subprocess.PIPE,
629 stderr=subprocess.STDOUT, stdin=subprocess.PIPE,
630 close_fds=True, universal_newlines=True,
631 cwd=args.outdir)
632 output = process.communicate(input=open(pfile, 'r').read())[0]
633 output = output.split('\n')
634 if output[-1] == '':
635 output = output[:-1]
636 if args.verbose:
637 for line in output:
638 logwrite('> %s' % line)
639 if process.returncode != 0:
640 if not args.verbose:
641 logwrite("Failed to apply changes from %s" % print_name)
642 for line in output:
643 logwrite('> %s' % line)
644 return 2
645
646 if args.refresh:
647 pfilef = open(pfile + '.tmp', 'a')
648 pfilef.write(p.top_header)
649 pfilef.flush()
650 for patchitem in p.items:
651 patched_file = '/'.join(patchitem.source.split('/')[1:])
652 fullfn = os.path.join(args.outdir, patched_file)
653 process = subprocess.Popen(['diff', '-p', '-u', patched_file + '.orig_file', patched_file,
654 '--label', 'a/' + patched_file,
655 '--label', 'b/' + patched_file],
656 stdout=pfilef, close_fds=True,
657 universal_newlines=True, cwd=args.outdir)
658 process.wait()
659 os.unlink(fullfn + '.orig_file')
660 if not process.returncode in (0, 1):
661 logwrite("Failed to diff to refresh %s" % print_name)
662 pfilef.close()
663 os.unlink(pfile + '.tmp')
664 return 3
665 pfilef.close()
666 os.rename(pfile + '.tmp', pfile)
667
668 # remove orig/rej files that patch sometimes creates
669 for root, dirs, files in os.walk(args.outdir):
670 for f in files:
671 if f[-5:] == '.orig' or f[-4:] == '.rej':
672 os.unlink(os.path.join(root, f))
673 git_debug_snapshot(args, "apply backport patch %s" % print_name)
674
675 # some post-processing is required
676 configtree = kconfig.ConfigTree(os.path.join(args.outdir, 'Kconfig'))
677 logwrite('Modify Kconfig tree ...')
678 configtree.prune_sources(ignore=['Kconfig.kernel', 'Kconfig.versions'])
679 git_debug_snapshot(args, "prune Kconfig tree")
680 configtree.force_tristate_modular()
681 git_debug_snapshot(args, "force tristate options modular")
682 configtree.modify_selects()
683 git_debug_snapshot(args, "convert select to depends on")
684
685 # write the versioning file
686 if git_tracked_version:
687 backports_version = "(see git)"
688 kernel_version = "(see git)"
689 else:
690 backports_version = git.describe(tree=source_dir, extra_args=['--long'])
691 kernel_version = git.describe(rev=args.git_revision or 'HEAD',
692 tree=args.kerneldir,
693 extra_args=['--long'])
694 f = open(os.path.join(args.outdir, 'versions'), 'w')
695 f.write('BACKPORTS_VERSION="%s"\n' % backports_version)
696 f.write('BACKPORTED_KERNEL_VERSION="%s"\n' % kernel_version)
697 f.write('BACKPORTED_KERNEL_NAME="%s"\n' % args.base_name)
698 if git_tracked_version:
699 f.write('BACKPORTS_GIT_TRACKED="backport tracker ID: $(shell git rev-parse HEAD 2>/dev/null || echo \'not built in git tree\')"\n')
700 f.close()
701
702 symbols = configtree.symbols()
703
704 # write local symbol list -- needed during build
705 f = open(os.path.join(args.outdir, '.local-symbols'), 'w')
706 for sym in symbols:
707 f.write('%s=\n' % sym)
708 f.close()
709
710 git_debug_snapshot(args, "add versions/symbols files")
711
712 logwrite('Rewrite Makefiles and Kconfig files ...')
713
714 # rewrite Makefile and source symbols
715 regexes = []
716 for some_symbols in [symbols[i:i + 50] for i in range(0, len(symbols), 50)]:
717 r = 'CONFIG_((' + '|'.join([s + '(_MODULE)?' for s in some_symbols]) + ')([^A-Za-z0-9_]|$))'
718 regexes.append(re.compile(r, re.MULTILINE))
719 for root, dirs, files in os.walk(args.outdir):
720 # don't go into .git dir (possible debug thing)
721 if '.git' in dirs:
722 dirs.remove('.git')
723 for f in files:
724 data = open(os.path.join(root, f), 'r').read()
725 for r in regexes:
726 data = r.sub(r'CPTCFG_\1', data)
727 data = re.sub(r'\$\(srctree\)', '$(backport_srctree)', data)
728 data = re.sub(r'-Idrivers', '-I$(backport_srctree)/drivers', data)
729 fo = open(os.path.join(root, f), 'w')
730 fo.write(data)
731 fo.close()
732
733 git_debug_snapshot(args, "rename config symbol / srctree usage")
734
735 # disable unbuildable Kconfig symbols and stuff Makefiles that doesn't exist
736 maketree = make.MakeTree(os.path.join(args.outdir, 'Makefile.kernel'))
737 disable_kconfig = []
738 disable_makefile = []
739 for sym in maketree.get_impossible_symbols():
740 disable_kconfig.append(sym[7:])
741 disable_makefile.append(sym[7:])
742
743 configtree.disable_symbols(disable_kconfig)
744 git_debug_snapshot(args, "disable impossible kconfig symbols")
745
746 # add kernel version dependencies to Kconfig, from the dependency list
747 # we read previously
748 for sym in tuple(deplist.keys()):
749 new = []
750 for dep in deplist[sym]:
751 if "kconfig:" in dep:
752 kconfig_expr = dep.replace('kconfig: ', '')
753 new.append(kconfig_expr)
754 elif (dep == "DISABLE"):
755 new.append('BACKPORT_DISABLED_KCONFIG_OPTION')
756 else:
757 new.append('!BACKPORT_KERNEL_%s' % dep.replace('.', '_'))
758 deplist[sym] = new
759 configtree.add_dependencies(deplist)
760 git_debug_snapshot(args, "add kernel version dependencies")
761
762 # disable things in makefiles that can't be selected and that the
763 # build shouldn't recurse into because they don't exist -- if we
764 # don't do that then a symbol from the kernel could cause the build
765 # to attempt to recurse and fail
766 #
767 # Note that we split the regex after 50 symbols, this is because of a
768 # limitation in the regex implementation (it only supports 100 nested
769 # groups -- 50 seemed safer and is still fast)
770 regexes = []
771 for some_symbols in [disable_makefile[i:i + 50] for i in range(0, len(disable_makefile), 50)]:
772 r = '^([^#].*((CPTCFG|CONFIG)_(' + '|'.join([s for s in some_symbols]) + ')))'
773 regexes.append(re.compile(r, re.MULTILINE))
774 for f in maketree.get_makefiles():
775 data = open(f, 'r').read()
776 for r in regexes:
777 data = r.sub(r'#\1', data)
778 fo = open(f, 'w')
779 fo.write(data)
780 fo.close()
781 git_debug_snapshot(args, "disable unsatisfied Makefile parts")
782
783 if (args.kup or args.kup_test):
784 upload_release(args, rel_prep, logwrite=logwrite)
785
786 logwrite('Done!')
787 return 0
788
789 if __name__ == '__main__':
790 ret = _main()
791 if ret:
792 sys.exit(ret)