2 # ex: set syntax=python:
10 from buildbot import locks
12 ini = ConfigParser.ConfigParser()
13 ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
15 buildbot_url = ini.get("phase2", "buildbot_url")
17 # This is a sample buildmaster config file. It must be installed as
18 # 'master.cfg' in your buildmaster's base directory.
20 # This is the dictionary that the buildmaster pays attention to. We also use
21 # a shorter alias to save typing.
22 c = BuildmasterConfig = {}
26 # The 'slaves' list defines the set of recognized buildslaves. Each element is
27 # a BuildSlave object, specifying a unique slave name and password. The same
28 # slave name and password must be configured on the slave.
29 from buildbot.buildslave import BuildSlave
38 if ini.has_option("phase2", "port"):
39 slave_port = ini.getint("phase2", "port")
41 if ini.has_option("phase2", "persistent"):
42 persistent = ini.getboolean("phase2", "persistent")
44 if ini.has_option("phase2", "other_builds"):
45 other_builds = ini.getint("phase2", "other_builds")
47 if ini.has_option("phase2", "expire"):
48 tree_expire = ini.getint("phase2", "expire")
50 if ini.has_option("general", "git_ssh"):
51 git_ssh = ini.getboolean("general", "git_ssh")
53 if ini.has_option("general", "git_ssh_key"):
54 git_ssh_key = ini.get("general", "git_ssh_key")
61 for section in ini.sections():
62 if section.startswith("slave "):
63 if ini.has_option(section, "name") and ini.has_option(section, "password") and \
64 ini.has_option(section, "phase") and ini.getint(section, "phase") == 2:
65 name = ini.get(section, "name")
66 password = ini.get(section, "password")
68 if ini.has_option(section, "builds"):
69 max_builds[name] = ini.getint(section, "builds")
70 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name]))
72 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
73 # This must match the value configured into the buildslaves (with their
75 c['slavePortnum'] = slave_port
78 c['mergeRequests'] = True
80 # Reduce amount of backlog data
81 c['buildHorizon'] = 30
86 work_dir = os.path.abspath(ini.get("general", "workdir") or ".")
87 scripts_dir = os.path.abspath("../scripts")
89 rsync_bin_url = ini.get("rsync", "binary_url")
90 rsync_bin_key = ini.get("rsync", "binary_password")
95 if ini.has_option("rsync", "source_url"):
96 rsync_src_url = ini.get("rsync", "source_url")
97 rsync_src_key = ini.get("rsync", "source_password")
101 rsync_sdk_pat = "openwrt-sdk-*.tar.xz"
103 if ini.has_option("rsync", "sdk_url"):
104 rsync_sdk_url = ini.get("rsync", "sdk_url")
106 if ini.has_option("rsync", "sdk_password"):
107 rsync_sdk_key = ini.get("rsync", "sdk_password")
109 if ini.has_option("rsync", "sdk_pattern"):
110 rsync_sdk_pat = ini.get("rsync", "sdk_pattern")
112 repo_url = ini.get("repo", "url")
113 repo_branch = "master"
115 if ini.has_option("repo", "branch"):
116 repo_branch = ini.get("repo", "branch")
119 usign_comment = "untrusted comment: " + repo_branch.replace("-", " ").title() + " key"
121 if ini.has_option("usign", "key"):
122 usign_key = ini.get("usign", "key")
124 if ini.has_option("usign", "comment"):
125 usign_comment = ini.get("usign", "comment")
132 if not os.path.isdir(work_dir+'/source.git'):
133 subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, work_dir+'/source.git'])
135 subprocess.call(["git", "pull"], cwd = work_dir+'/source.git')
137 findarches = subprocess.Popen([scripts_dir + '/dumpinfo.pl', 'architectures'],
138 stdout = subprocess.PIPE, cwd = work_dir+'/source.git')
141 line = findarches.stdout.readline()
144 at = line.strip().split()
146 archnames.append(at[0])
151 feedbranches = dict()
153 from buildbot.changes.gitpoller import GitPoller
154 c['change_source'] = []
156 def parse_feed_entry(line):
157 parts = line.strip().split()
158 if parts[0] == "src-git":
160 url = parts[2].strip().split(';')
161 branch = url[1] if len(url) > 1 else 'master'
162 feedbranches[url[0]] = branch
163 c['change_source'].append(GitPoller(url[0], branch=branch, workdir='%s/%s.git' %(os.getcwd(), parts[1]), pollinterval=300))
165 make = subprocess.Popen(['make', '--no-print-directory', '-C', work_dir+'/source.git/target/sdk/', 'val.BASE_FEED'],
166 env = dict(os.environ, TOPDIR=work_dir+'/source.git'), stdout = subprocess.PIPE)
168 line = make.stdout.readline()
170 parse_feed_entry(line)
172 with open(work_dir+'/source.git/feeds.conf.default', 'r') as f:
174 parse_feed_entry(line)
179 # Configure the Schedulers, which decide how to react to incoming changes. In this
180 # case, just kick off a 'basebuild' build
182 def branch_change_filter(change):
183 return change.branch == feedbranches[change.repository]
185 from buildbot.schedulers.basic import SingleBranchScheduler
186 from buildbot.schedulers.forcesched import ForceScheduler
187 from buildbot.changes import filter
189 c['schedulers'].append(SingleBranchScheduler(
191 change_filter=filter.ChangeFilter(filter_fn=branch_change_filter),
193 builderNames=archnames))
195 c['schedulers'].append(ForceScheduler(
197 builderNames=archnames))
201 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
202 # what steps, and which slaves can execute them. Note that any particular build will
203 # only take place on one slave.
205 from buildbot.process.factory import BuildFactory
206 from buildbot.steps.source import Git
207 from buildbot.steps.shell import ShellCommand
208 from buildbot.steps.shell import SetProperty
209 from buildbot.steps.transfer import FileUpload
210 from buildbot.steps.transfer import FileDownload
211 from buildbot.steps.transfer import StringDownload
212 from buildbot.steps.master import MasterShellCommand
213 from buildbot.process.properties import WithProperties
216 def GetDirectorySuffix(props):
217 verpat = re.compile('^([0-9]{2})\.([0-9]{2})(?:\.([0-9]+)(?:-rc([0-9]+))?|-(SNAPSHOT))$')
218 if props.hasProperty("release_version"):
219 m = verpat.match(props["release_version"])
221 return "-%02d.%02d" %(int(m.group(1)), int(m.group(2)))
224 def GetNumJobs(props):
225 if props.hasProperty("slavename") and props.hasProperty("nproc"):
226 return ((int(props["nproc"]) / (max_builds[props["slavename"]] + other_builds)) + 1)
231 if props.hasProperty("builddir"):
232 return props["builddir"]
233 elif props.hasProperty("workdir"):
234 return props["workdir"]
238 def UsignSec2Pub(seckey, comment="untrusted comment: secret key"):
240 seckey = base64.b64decode(seckey)
244 return "{}\n{}".format(re.sub(r"\bsecret key$", "public key", comment),
245 base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]))
250 dlLock = locks.SlaveLock("slave_dl")
254 for slave in c['slaves']:
255 slaveNames.append(slave.slavename)
258 ts = arch[1].split('/')
260 factory = BuildFactory()
262 # find number of cores
263 factory.addStep(SetProperty(
266 description = "Finding number of CPUs",
267 command = ["nproc"]))
270 factory.addStep(FileDownload(
271 mastersrc = scripts_dir + '/cleanup.sh',
272 slavedest = "../cleanup.sh",
276 factory.addStep(ShellCommand(
278 description = "Cleaning previous builds",
279 command = ["./cleanup.sh", buildbot_url, WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"],
281 haltOnFailure = True,
284 factory.addStep(ShellCommand(
286 description = "Cleaning work area",
287 command = ["./cleanup.sh", buildbot_url, WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"],
289 haltOnFailure = True,
292 # expire tree if needed
293 elif tree_expire > 0:
294 factory.addStep(FileDownload(
295 mastersrc = scripts_dir + '/expire.sh',
296 slavedest = "../expire.sh",
299 factory.addStep(ShellCommand(
301 description = "Checking for build tree expiry",
302 command = ["./expire.sh", str(tree_expire)],
304 haltOnFailure = True,
307 factory.addStep(ShellCommand(
309 description = "Preparing SDK directory",
310 command = ["mkdir", "-p", "sdk"],
311 haltOnFailure = True))
313 factory.addStep(ShellCommand(
314 name = "downloadsdk",
315 description = "Downloading SDK archive",
316 command = ["rsync", "-4", "-va", "%s/%s/%s/%s" %(rsync_sdk_url, ts[0], ts[1], rsync_sdk_pat), "sdk.archive"],
317 env={'RSYNC_PASSWORD': rsync_sdk_key},
318 haltOnFailure = True,
321 factory.addStep(ShellCommand(
323 description = "Unpacking SDK archive",
324 command = "rm -rf sdk_update && mkdir sdk_update && tar --strip-components=1 -C sdk_update/ -vxf sdk.archive",
325 haltOnFailure = True))
327 factory.addStep(ShellCommand(
329 description = "Updating SDK",
330 command = "rsync --checksum -av sdk_update/ sdk/ && rm -rf sdk_update",
331 haltOnFailure = True))
333 factory.addStep(StringDownload(
334 name = "writeversionmk",
335 s = 'TOPDIR:=${CURDIR}\n\ninclude $(TOPDIR)/include/version.mk\n\nversion:\n\t@echo $(VERSION_NUMBER)\n',
336 slavedest = "sdk/getversion.mk",
339 factory.addStep(SetProperty(
341 property = "release_version",
342 description = "Finding SDK release version",
343 workdir = "build/sdk",
344 command = ["make", "-f", "getversion.mk"]))
347 if usign_key is not None:
348 factory.addStep(StringDownload(
349 name = "dlkeybuildpub",
350 s = UsignSec2Pub(usign_key, usign_comment),
351 slavedest = "sdk/key-build.pub",
354 factory.addStep(StringDownload(
356 s = "# fake private key",
357 slavedest = "sdk/key-build",
360 factory.addStep(StringDownload(
361 name = "dlkeybuilducert",
362 s = "# fake certificate",
363 slavedest = "sdk/key-build.ucert",
366 factory.addStep(ShellCommand(
368 description = "Preparing download directory",
369 command = ["sh", "-c", "mkdir -p $HOME/dl && rm -rf ./sdk/dl && ln -sf $HOME/dl ./sdk/dl"],
370 haltOnFailure = True))
372 factory.addStep(ShellCommand(
374 description = "Preparing SDK configuration",
375 workdir = "build/sdk",
376 command = ["sh", "-c", "rm -f .config && make defconfig"]))
378 factory.addStep(FileDownload(
379 mastersrc = scripts_dir + '/ccache.sh',
380 slavedest = 'sdk/ccache.sh',
383 factory.addStep(ShellCommand(
385 description = "Preparing ccache",
386 workdir = "build/sdk",
387 command = ["./ccache.sh"],
388 haltOnFailure = True))
391 factory.addStep(StringDownload(
392 name = "dlgitclonekey",
394 slavedest = "../git-clone.key",
397 factory.addStep(ShellCommand(
398 name = "patchfeedsconf",
399 description = "Patching feeds.conf",
400 workdir = "build/sdk",
401 command = "sed -e 's#https://#ssh://git@#g' feeds.conf.default > feeds.conf",
402 haltOnFailure = True))
404 factory.addStep(ShellCommand(
405 name = "updatefeeds",
406 description = "Updating feeds",
407 workdir = "build/sdk",
408 command = ["./scripts/feeds", "update", "-f"],
409 env = {'GIT_SSH_COMMAND': WithProperties("ssh -o IdentitiesOnly=yes -o IdentityFile=%(cwd)s/git-clone.key -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no", cwd=GetCwd)} if git_ssh else {},
410 haltOnFailure = True))
413 factory.addStep(ShellCommand(
414 name = "rmfeedsconf",
415 description = "Removing feeds.conf",
416 workdir = "build/sdk",
417 command=["rm", "feeds.conf"],
418 haltOnFailure = True))
420 factory.addStep(ShellCommand(
421 name = "installfeeds",
422 description = "Installing feeds",
423 workdir = "build/sdk",
424 command = ["./scripts/feeds", "install", "-a"],
425 haltOnFailure = True))
427 factory.addStep(ShellCommand(
429 description = "Clearing failure logs",
430 workdir = "build/sdk",
431 command = ["rm", "-rf", "logs/package/error.txt", "faillogs/"],
432 haltOnFailure = False
435 factory.addStep(ShellCommand(
437 description = "Building packages",
438 workdir = "build/sdk",
440 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_AUTOREMOVE=y", "CONFIG_SIGNED_PACKAGES="],
441 env = {'CCACHE_BASEDIR': WithProperties("%(cwd)s", cwd=GetCwd)},
442 haltOnFailure = True))
444 factory.addStep(ShellCommand(
445 name = "mkfeedsconf",
446 description = "Generating pinned feeds.conf",
447 workdir = "build/sdk",
448 command = "./scripts/feeds list -s -f > bin/packages/%s/feeds.conf" %(arch[0])))
450 if ini.has_option("gpg", "key") or usign_key is not None:
451 factory.addStep(MasterShellCommand(
452 name = "signprepare",
453 description = "Preparing temporary signing directory",
454 command = ["mkdir", "-p", "%s/signing" %(work_dir)],
458 factory.addStep(ShellCommand(
460 description = "Packing files to sign",
461 workdir = "build/sdk",
462 command = "find bin/packages/%s/ -mindepth 2 -maxdepth 2 -type f -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(arch[0]),
466 factory.addStep(FileUpload(
467 slavesrc = "sdk/sign.tar.gz",
468 masterdest = "%s/signing/%s.tar.gz" %(work_dir, arch[0]),
472 factory.addStep(MasterShellCommand(
474 description = "Signing files",
475 command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.tar.gz" %(work_dir, arch[0])],
476 env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
480 factory.addStep(FileDownload(
481 mastersrc = "%s/signing/%s.tar.gz" %(work_dir, arch[0]),
482 slavedest = "sdk/sign.tar.gz",
486 factory.addStep(ShellCommand(
488 description = "Unpacking signed files",
489 workdir = "build/sdk",
490 command = ["tar", "-xzf", "sign.tar.gz"],
494 factory.addStep(ShellCommand(
495 name = "uploadprepare",
496 description = "Preparing package directory",
497 workdir = "build/sdk",
498 command = ["rsync", "-4", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", WithProperties("%s/packages%%(suffix)s/" %(rsync_bin_url), suffix=GetDirectorySuffix)],
499 env={'RSYNC_PASSWORD': rsync_bin_key},
500 haltOnFailure = True,
504 factory.addStep(ShellCommand(
505 name = "packageupload",
506 description = "Uploading package files",
507 workdir = "build/sdk",
508 command = ["rsync", "-4", "--progress", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "bin/packages/%s/" %(arch[0]), WithProperties("%s/packages%%(suffix)s/%s/" %(rsync_bin_url, arch[0]), suffix=GetDirectorySuffix)],
509 env={'RSYNC_PASSWORD': rsync_bin_key},
510 haltOnFailure = True,
514 factory.addStep(ShellCommand(
516 description = "Preparing log directory",
517 workdir = "build/sdk",
518 command = ["rsync", "-4", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", WithProperties("%s/faillogs%%(suffix)s/" %(rsync_bin_url), suffix=GetDirectorySuffix)],
519 env={'RSYNC_PASSWORD': rsync_bin_key},
520 haltOnFailure = True,
524 factory.addStep(ShellCommand(
526 description = "Finding failure logs",
527 workdir = "build/sdk/logs/package/feeds",
528 command = ["sh", "-c", "sed -ne 's!^ *ERROR: package/feeds/\\([^ ]*\\) .*$!\\1!p' ../error.txt | sort -u | xargs -r find > ../../../logs.txt"],
529 haltOnFailure = False
532 factory.addStep(ShellCommand(
534 description = "Collecting failure logs",
535 workdir = "build/sdk",
536 command = ["rsync", "-av", "--files-from=logs.txt", "logs/package/feeds/", "faillogs/"],
537 haltOnFailure = False
540 factory.addStep(ShellCommand(
542 description = "Uploading failure logs",
543 workdir = "build/sdk",
544 command = ["rsync", "-4", "--progress", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "faillogs/", WithProperties("%s/faillogs%%(suffix)s/%s/" %(rsync_bin_url, arch[0]), suffix=GetDirectorySuffix)],
545 env={'RSYNC_PASSWORD': rsync_bin_key},
546 haltOnFailure = False,
550 if rsync_src_url is not None:
551 factory.addStep(ShellCommand(
553 description = "Finding source archives to upload",
554 workdir = "build/sdk",
555 command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -newer ../sdk.archive -printf '%f\\n' > sourcelist",
559 factory.addStep(ShellCommand(
560 name = "sourceupload",
561 description = "Uploading source archives",
562 workdir = "build/sdk",
563 command = ["rsync", "--files-from=sourcelist", "-4", "--progress", "--checksum", "--delay-updates",
564 WithProperties("--partial-dir=.~tmp~%s~%%(slavename)s" %(arch[0])), "-avz", "dl/", "%s/" %(rsync_src_url)],
565 env={'RSYNC_PASSWORD': rsync_src_key},
566 haltOnFailure = False,
570 factory.addStep(ShellCommand(
572 description = "Reporting disk usage",
573 command=["df", "-h", "."],
575 haltOnFailure = False,
579 from buildbot.config import BuilderConfig
581 c['builders'].append(BuilderConfig(name=arch[0], slavenames=slaveNames, factory=factory))
584 ####### STATUS arches
586 # 'status' is a list of Status arches. The results of each build will be
587 # pushed to these arches. buildbot/status/*.py has a variety to choose from,
588 # including web pages, email senders, and IRC bots.
592 from buildbot.status import html
593 from buildbot.status.web import authz, auth
595 if ini.has_option("phase2", "status_bind"):
596 if ini.has_option("phase2", "status_user") and ini.has_option("phase2", "status_password"):
597 authz_cfg=authz.Authz(
598 # change any of these to True to enable; see the manual for more
600 auth=auth.BasicAuth([(ini.get("phase2", "status_user"), ini.get("phase2", "status_password"))]),
601 gracefulShutdown = 'auth',
602 forceBuild = 'auth', # use this to test your slave once it is set up
603 forceAllBuilds = 'auth',
606 stopAllBuilds = 'auth',
607 cancelPendingBuild = 'auth',
609 c['status'].append(html.WebStatus(http_port=ini.get("phase2", "status_bind"), authz=authz_cfg))
611 c['status'].append(html.WebStatus(http_port=ini.get("phase2", "status_bind")))
613 ####### PROJECT IDENTITY
615 # the 'title' string will appear at the top of this buildbot
616 # installation's html.WebStatus home page (linked to the
617 # 'titleURL') and is embedded in the title of the waterfall HTML page.
619 c['title'] = ini.get("general", "title")
620 c['titleURL'] = ini.get("general", "title_url")
622 # the 'buildbotURL' string should point to the location where the buildbot's
623 # internal web server (usually the html.WebStatus page) is visible. This
624 # typically uses the port number set in the Waterfall 'status' entry, but
625 # with an externally-visible host name which the buildbot cannot figure out
628 c['buildbotURL'] = buildbot_url
633 # This specifies what database buildbot uses to store its state. You can leave
634 # this at its default for all but the largest installations.
635 'db_url' : "sqlite:///state.sqlite",