2 # ex: set syntax=python:
10 from buildbot import locks
12 ini = ConfigParser.ConfigParser()
13 ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
15 buildbot_url = ini.get("phase2", "buildbot_url")
17 # This is a sample buildmaster config file. It must be installed as
18 # 'master.cfg' in your buildmaster's base directory.
20 # This is the dictionary that the buildmaster pays attention to. We also use
21 # a shorter alias to save typing.
22 c = BuildmasterConfig = {}
26 # The 'slaves' list defines the set of recognized buildslaves. Each element is
27 # a BuildSlave object, specifying a unique slave name and password. The same
28 # slave name and password must be configured on the slave.
29 from buildbot.buildslave import BuildSlave
38 if ini.has_option("phase2", "port"):
39 slave_port = ini.getint("phase2", "port")
41 if ini.has_option("phase2", "persistent"):
42 persistent = ini.getboolean("phase2", "persistent")
44 if ini.has_option("phase2", "other_builds"):
45 other_builds = ini.getint("phase2", "other_builds")
47 if ini.has_option("phase2", "expire"):
48 tree_expire = ini.getint("phase2", "expire")
50 if ini.has_option("general", "git_ssh"):
51 git_ssh = ini.getboolean("general", "git_ssh")
53 if ini.has_option("general", "git_ssh_key"):
54 git_ssh_key = ini.get("general", "git_ssh_key")
61 for section in ini.sections():
62 if section.startswith("slave "):
63 if ini.has_option(section, "name") and ini.has_option(section, "password") and \
64 ini.has_option(section, "phase") and ini.getint(section, "phase") == 2:
65 name = ini.get(section, "name")
66 password = ini.get(section, "password")
68 if ini.has_option(section, "builds"):
69 max_builds[name] = ini.getint(section, "builds")
70 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name]))
72 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
73 # This must match the value configured into the buildslaves (with their
75 c['slavePortnum'] = slave_port
78 c['mergeRequests'] = True
80 # Reduce amount of backlog data
81 c['buildHorizon'] = 30
86 work_dir = os.path.abspath(ini.get("general", "workdir") or ".")
87 scripts_dir = os.path.abspath("../scripts")
89 rsync_bin_url = ini.get("rsync", "binary_url")
90 rsync_bin_key = ini.get("rsync", "binary_password")
95 if ini.has_option("rsync", "source_url"):
96 rsync_src_url = ini.get("rsync", "source_url")
97 rsync_src_key = ini.get("rsync", "source_password")
101 rsync_sdk_pat = "openwrt-sdk-*.tar.xz"
103 if ini.has_option("rsync", "sdk_url"):
104 rsync_sdk_url = ini.get("rsync", "sdk_url")
106 if ini.has_option("rsync", "sdk_password"):
107 rsync_sdk_key = ini.get("rsync", "sdk_password")
109 if ini.has_option("rsync", "sdk_pattern"):
110 rsync_sdk_pat = ini.get("rsync", "sdk_pattern")
112 repo_url = ini.get("repo", "url")
113 repo_branch = "master"
115 if ini.has_option("repo", "branch"):
116 repo_branch = ini.get("repo", "branch")
119 usign_comment = "untrusted comment: " + repo_branch.replace("-", " ").title() + " key"
121 if ini.has_option("usign", "key"):
122 usign_key = ini.get("usign", "key")
124 if ini.has_option("usign", "comment"):
125 usign_comment = ini.get("usign", "comment")
132 if not os.path.isdir(work_dir+'/source.git'):
133 subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, work_dir+'/source.git'])
135 subprocess.call(["git", "pull"], cwd = work_dir+'/source.git')
137 findarches = subprocess.Popen([scripts_dir + '/dumpinfo.pl', 'architectures'],
138 stdout = subprocess.PIPE, cwd = work_dir+'/source.git')
141 line = findarches.stdout.readline()
144 at = line.strip().split()
146 archnames.append(at[0])
151 feedbranches = dict()
153 from buildbot.changes.gitpoller import GitPoller
154 c['change_source'] = []
156 def parse_feed_entry(line):
157 parts = line.strip().split()
158 if parts[0] == "src-git":
160 url = parts[2].strip().split(';')
161 branch = url[1] if len(url) > 1 else 'master'
162 feedbranches[url[0]] = branch
163 c['change_source'].append(GitPoller(url[0], branch=branch, workdir='%s/%s.git' %(os.getcwd(), parts[1]), pollinterval=300))
165 make = subprocess.Popen(['make', '--no-print-directory', '-C', work_dir+'/source.git/target/sdk/', 'val.BASE_FEED'],
166 env = dict(os.environ, TOPDIR=work_dir+'/source.git'), stdout = subprocess.PIPE)
168 line = make.stdout.readline()
170 parse_feed_entry(line)
172 with open(work_dir+'/source.git/feeds.conf.default', 'r') as f:
174 parse_feed_entry(line)
179 # Configure the Schedulers, which decide how to react to incoming changes. In this
180 # case, just kick off a 'basebuild' build
182 def branch_change_filter(change):
183 return change.branch == feedbranches[change.repository]
185 from buildbot.schedulers.basic import SingleBranchScheduler
186 from buildbot.schedulers.forcesched import ForceScheduler
187 from buildbot.changes import filter
189 c['schedulers'].append(SingleBranchScheduler(
191 change_filter=filter.ChangeFilter(filter_fn=branch_change_filter),
193 builderNames=archnames))
195 c['schedulers'].append(ForceScheduler(
197 builderNames=archnames))
201 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
202 # what steps, and which slaves can execute them. Note that any particular build will
203 # only take place on one slave.
205 from buildbot.process.factory import BuildFactory
206 from buildbot.steps.source import Git
207 from buildbot.steps.shell import ShellCommand
208 from buildbot.steps.shell import SetProperty
209 from buildbot.steps.transfer import FileUpload
210 from buildbot.steps.transfer import FileDownload
211 from buildbot.steps.transfer import StringDownload
212 from buildbot.steps.master import MasterShellCommand
213 from buildbot.process.properties import WithProperties
216 def GetDirectorySuffix(props):
217 verpat = re.compile('^([0-9]{2})\.([0-9]{2})(?:\.([0-9]+)(?:-rc([0-9]+))?|-(SNAPSHOT))$')
218 if props.hasProperty("release_version"):
219 m = verpat.match(props["release_version"])
221 return "-%02d.%02d" %(int(m.group(1)), int(m.group(2)))
224 def GetNumJobs(props):
225 if props.hasProperty("slavename") and props.hasProperty("nproc"):
226 return ((int(props["nproc"]) / (max_builds[props["slavename"]] + other_builds)) + 1)
231 if props.hasProperty("builddir"):
232 return props["builddir"]
233 elif props.hasProperty("workdir"):
234 return props["workdir"]
238 def UsignSec2Pub(seckey, comment="untrusted comment: secret key"):
240 seckey = base64.b64decode(seckey)
244 return "{}\n{}".format(re.sub(r"\bsecret key$", "public key", comment),
245 base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]))
250 dlLock = locks.SlaveLock("slave_dl")
254 for slave in c['slaves']:
255 slaveNames.append(slave.slavename)
258 ts = arch[1].split('/')
260 factory = BuildFactory()
262 # find number of cores
263 factory.addStep(SetProperty(
266 description = "Finding number of CPUs",
267 command = ["nproc"]))
270 factory.addStep(FileDownload(
271 mastersrc = scripts_dir + '/cleanup-phase2.sh',
272 slavedest = "cleanup.sh",
276 factory.addStep(ShellCommand(
278 description = "Cleaning previous builds",
279 command = ["./cleanup.sh", buildbot_url, WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"],
280 haltOnFailure = True,
283 factory.addStep(ShellCommand(
285 description = "Cleaning work area",
286 command = ["./cleanup.sh", buildbot_url, WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"],
287 haltOnFailure = True,
290 # expire tree if needed
291 elif tree_expire > 0:
292 factory.addStep(FileDownload(
293 mastersrc = scripts_dir + '/expire.sh',
294 slavedest = "../expire.sh",
297 factory.addStep(ShellCommand(
299 description = "Checking for build tree expiry",
300 command = ["./expire.sh", str(tree_expire)],
302 haltOnFailure = True,
305 factory.addStep(ShellCommand(
307 description = "Preparing SDK directory",
308 command = ["mkdir", "-p", "sdk"],
309 haltOnFailure = True))
311 factory.addStep(ShellCommand(
312 name = "downloadsdk",
313 description = "Downloading SDK archive",
314 command = ["rsync", "-4", "-va", "%s/%s/%s/%s" %(rsync_sdk_url, ts[0], ts[1], rsync_sdk_pat), "sdk.archive"],
315 env={'RSYNC_PASSWORD': rsync_sdk_key},
316 haltOnFailure = True,
319 factory.addStep(ShellCommand(
321 description = "Unpacking SDK archive",
322 command = "rm -rf sdk_update && mkdir sdk_update && tar --strip-components=1 -C sdk_update/ -vxf sdk.archive",
323 haltOnFailure = True))
325 factory.addStep(ShellCommand(
327 description = "Updating SDK",
328 command = "rsync --checksum -av sdk_update/ sdk/ && rm -rf sdk_update",
329 haltOnFailure = True))
331 factory.addStep(StringDownload(
332 name = "writeversionmk",
333 s = 'TOPDIR:=${CURDIR}\n\ninclude $(TOPDIR)/include/version.mk\n\nversion:\n\t@echo $(VERSION_NUMBER)\n',
334 slavedest = "sdk/getversion.mk",
337 factory.addStep(SetProperty(
339 property = "release_version",
340 description = "Finding SDK release version",
341 workdir = "build/sdk",
342 command = ["make", "-f", "getversion.mk"]))
345 if usign_key is not None:
346 factory.addStep(StringDownload(
347 name = "dlkeybuildpub",
348 s = UsignSec2Pub(usign_key, usign_comment),
349 slavedest = "sdk/key-build.pub",
352 factory.addStep(StringDownload(
354 s = "# fake private key",
355 slavedest = "sdk/key-build",
358 factory.addStep(StringDownload(
359 name = "dlkeybuilducert",
360 s = "# fake certificate",
361 slavedest = "sdk/key-build.ucert",
364 factory.addStep(ShellCommand(
366 description = "Preparing download directory",
367 command = ["sh", "-c", "mkdir -p $HOME/dl && rm -rf ./sdk/dl && ln -sf $HOME/dl ./sdk/dl"],
368 haltOnFailure = True))
370 factory.addStep(ShellCommand(
372 description = "Preparing SDK configuration",
373 workdir = "build/sdk",
374 command = ["sh", "-c", "rm -f .config && make defconfig"]))
376 factory.addStep(FileDownload(
377 mastersrc = scripts_dir + '/ccache.sh',
378 slavedest = 'sdk/ccache.sh',
381 factory.addStep(ShellCommand(
383 description = "Preparing ccache",
384 workdir = "build/sdk",
385 command = ["./ccache.sh"],
386 haltOnFailure = True))
389 factory.addStep(StringDownload(
390 name = "dlgitclonekey",
392 slavedest = "../git-clone.key",
395 factory.addStep(ShellCommand(
396 name = "patchfeedsconf",
397 description = "Patching feeds.conf",
398 workdir = "build/sdk",
399 command = "sed -e 's#https://#ssh://git@#g' feeds.conf.default > feeds.conf",
400 haltOnFailure = True))
402 factory.addStep(ShellCommand(
403 name = "updatefeeds",
404 description = "Updating feeds",
405 workdir = "build/sdk",
406 command = ["./scripts/feeds", "update", "-f"],
407 env = {'GIT_SSH_COMMAND': WithProperties("ssh -o IdentitiesOnly=yes -o IdentityFile=%(cwd)s/git-clone.key -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no", cwd=GetCwd)} if git_ssh else {},
408 haltOnFailure = True))
411 factory.addStep(ShellCommand(
412 name = "rmfeedsconf",
413 description = "Removing feeds.conf",
414 workdir = "build/sdk",
415 command=["rm", "feeds.conf"],
416 haltOnFailure = True))
418 factory.addStep(ShellCommand(
419 name = "installfeeds",
420 description = "Installing feeds",
421 workdir = "build/sdk",
422 command = ["./scripts/feeds", "install", "-a"],
423 haltOnFailure = True))
425 factory.addStep(ShellCommand(
427 description = "Clearing failure logs",
428 workdir = "build/sdk",
429 command = ["rm", "-rf", "logs/package/error.txt", "faillogs/"],
430 haltOnFailure = False
433 factory.addStep(ShellCommand(
435 description = "Building packages",
436 workdir = "build/sdk",
438 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_AUTOREMOVE=y", "CONFIG_SIGNED_PACKAGES="],
439 env = {'CCACHE_BASEDIR': WithProperties("%(cwd)s", cwd=GetCwd)},
440 haltOnFailure = True))
442 factory.addStep(ShellCommand(
443 name = "mkfeedsconf",
444 description = "Generating pinned feeds.conf",
445 workdir = "build/sdk",
446 command = "./scripts/feeds list -s -f > bin/packages/%s/feeds.conf" %(arch[0])))
448 if ini.has_option("gpg", "key") or usign_key is not None:
449 factory.addStep(MasterShellCommand(
450 name = "signprepare",
451 description = "Preparing temporary signing directory",
452 command = ["mkdir", "-p", "%s/signing" %(work_dir)],
456 factory.addStep(ShellCommand(
458 description = "Packing files to sign",
459 workdir = "build/sdk",
460 command = "find bin/packages/%s/ -mindepth 2 -maxdepth 2 -type f -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(arch[0]),
464 factory.addStep(FileUpload(
465 slavesrc = "sdk/sign.tar.gz",
466 masterdest = "%s/signing/%s.tar.gz" %(work_dir, arch[0]),
470 factory.addStep(MasterShellCommand(
472 description = "Signing files",
473 command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.tar.gz" %(work_dir, arch[0])],
474 env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
478 factory.addStep(FileDownload(
479 mastersrc = "%s/signing/%s.tar.gz" %(work_dir, arch[0]),
480 slavedest = "sdk/sign.tar.gz",
484 factory.addStep(ShellCommand(
486 description = "Unpacking signed files",
487 workdir = "build/sdk",
488 command = ["tar", "-xzf", "sign.tar.gz"],
492 factory.addStep(ShellCommand(
493 name = "uploadprepare",
494 description = "Preparing package directory",
495 workdir = "build/sdk",
496 command = ["rsync", "-4", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", WithProperties("%s/packages%%(suffix)s/" %(rsync_bin_url), suffix=GetDirectorySuffix)],
497 env={'RSYNC_PASSWORD': rsync_bin_key},
498 haltOnFailure = True,
502 factory.addStep(ShellCommand(
503 name = "packageupload",
504 description = "Uploading package files",
505 workdir = "build/sdk",
506 command = ["rsync", "-4", "--progress", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "bin/packages/%s/" %(arch[0]), WithProperties("%s/packages%%(suffix)s/%s/" %(rsync_bin_url, arch[0]), suffix=GetDirectorySuffix)],
507 env={'RSYNC_PASSWORD': rsync_bin_key},
508 haltOnFailure = True,
512 factory.addStep(ShellCommand(
514 description = "Preparing log directory",
515 workdir = "build/sdk",
516 command = ["rsync", "-4", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", WithProperties("%s/faillogs%%(suffix)s/" %(rsync_bin_url), suffix=GetDirectorySuffix)],
517 env={'RSYNC_PASSWORD': rsync_bin_key},
518 haltOnFailure = True,
522 factory.addStep(ShellCommand(
524 description = "Finding failure logs",
525 workdir = "build/sdk/logs/package/feeds",
526 command = ["sh", "-c", "sed -ne 's!^ *ERROR: package/feeds/\\([^ ]*\\) .*$!\\1!p' ../error.txt | sort -u | xargs -r find > ../../../logs.txt"],
527 haltOnFailure = False
530 factory.addStep(ShellCommand(
532 description = "Collecting failure logs",
533 workdir = "build/sdk",
534 command = ["rsync", "-av", "--files-from=logs.txt", "logs/package/feeds/", "faillogs/"],
535 haltOnFailure = False
538 factory.addStep(ShellCommand(
540 description = "Uploading failure logs",
541 workdir = "build/sdk",
542 command = ["rsync", "-4", "--progress", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "faillogs/", WithProperties("%s/faillogs%%(suffix)s/%s/" %(rsync_bin_url, arch[0]), suffix=GetDirectorySuffix)],
543 env={'RSYNC_PASSWORD': rsync_bin_key},
544 haltOnFailure = False,
548 if rsync_src_url is not None:
549 factory.addStep(ShellCommand(
551 description = "Finding source archives to upload",
552 workdir = "build/sdk",
553 command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -newer ../sdk.archive -printf '%f\\n' > sourcelist",
557 factory.addStep(ShellCommand(
558 name = "sourceupload",
559 description = "Uploading source archives",
560 workdir = "build/sdk",
561 command = ["rsync", "--files-from=sourcelist", "-4", "--progress", "--checksum", "--delay-updates",
562 WithProperties("--partial-dir=.~tmp~%s~%%(slavename)s" %(arch[0])), "-avz", "dl/", "%s/" %(rsync_src_url)],
563 env={'RSYNC_PASSWORD': rsync_src_key},
564 haltOnFailure = False,
568 factory.addStep(ShellCommand(
570 description = "Reporting disk usage",
571 command=["df", "-h", "."],
573 haltOnFailure = False,
577 from buildbot.config import BuilderConfig
579 c['builders'].append(BuilderConfig(name=arch[0], slavenames=slaveNames, factory=factory))
582 ####### STATUS arches
584 # 'status' is a list of Status arches. The results of each build will be
585 # pushed to these arches. buildbot/status/*.py has a variety to choose from,
586 # including web pages, email senders, and IRC bots.
590 from buildbot.status import html
591 from buildbot.status.web import authz, auth
593 if ini.has_option("phase2", "status_bind"):
594 if ini.has_option("phase2", "status_user") and ini.has_option("phase2", "status_password"):
595 authz_cfg=authz.Authz(
596 # change any of these to True to enable; see the manual for more
598 auth=auth.BasicAuth([(ini.get("phase2", "status_user"), ini.get("phase2", "status_password"))]),
599 gracefulShutdown = 'auth',
600 forceBuild = 'auth', # use this to test your slave once it is set up
601 forceAllBuilds = 'auth',
604 stopAllBuilds = 'auth',
605 cancelPendingBuild = 'auth',
607 c['status'].append(html.WebStatus(http_port=ini.get("phase2", "status_bind"), authz=authz_cfg))
609 c['status'].append(html.WebStatus(http_port=ini.get("phase2", "status_bind")))
611 ####### PROJECT IDENTITY
613 # the 'title' string will appear at the top of this buildbot
614 # installation's html.WebStatus home page (linked to the
615 # 'titleURL') and is embedded in the title of the waterfall HTML page.
617 c['title'] = ini.get("general", "title")
618 c['titleURL'] = ini.get("general", "title_url")
620 # the 'buildbotURL' string should point to the location where the buildbot's
621 # internal web server (usually the html.WebStatus page) is visible. This
622 # typically uses the port number set in the Waterfall 'status' entry, but
623 # with an externally-visible host name which the buildbot cannot figure out
626 c['buildbotURL'] = buildbot_url
631 # This specifies what database buildbot uses to store its state. You can leave
632 # this at its default for all but the largest installations.
633 'db_url' : "sqlite:///state.sqlite",