2 # ex: set syntax=python:
10 from buildbot import locks
12 ini = configparser.ConfigParser()
13 ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
15 buildbot_url = ini.get("phase2", "buildbot_url")
17 # This is a sample buildmaster config file. It must be installed as
18 # 'master.cfg' in your buildmaster's base directory.
20 # This is the dictionary that the buildmaster pays attention to. We also use
21 # a shorter alias to save typing.
22 c = BuildmasterConfig = {}
26 # The 'workers' list defines the set of recognized buildslaves. Each element is
27 # a Worker object, specifying a unique slave name and password. The same
28 # slave name and password must be configured on the slave.
29 from buildbot.worker import Worker
38 if ini.has_option("phase2", "port"):
39 slave_port = ini.getint("phase2", "port")
41 if ini.has_option("phase2", "persistent"):
42 persistent = ini.getboolean("phase2", "persistent")
44 if ini.has_option("phase2", "other_builds"):
45 other_builds = ini.getint("phase2", "other_builds")
47 if ini.has_option("phase2", "expire"):
48 tree_expire = ini.getint("phase2", "expire")
50 if ini.has_option("general", "git_ssh"):
51 git_ssh = ini.getboolean("general", "git_ssh")
53 if ini.has_option("general", "git_ssh_key"):
54 git_ssh_key = ini.get("general", "git_ssh_key")
61 for section in ini.sections():
62 if section.startswith("slave "):
63 if ini.has_option(section, "name") and ini.has_option(section, "password") and \
64 ini.has_option(section, "phase") and ini.getint(section, "phase") == 2:
65 name = ini.get(section, "name")
66 password = ini.get(section, "password")
67 sl_props = { 'shared_wd': False }
70 if ini.has_option(section, "builds"):
71 max_builds[name] = ini.getint(section, "builds")
73 if max_builds[name] == 1:
74 sl_props['shared_wd'] = True
76 if ini.has_option(section, "shared_wd"):
77 sl_props['shared_wd'] = ini.getboolean(section, "shared_wd")
78 if sl_props['shared_wd'] and (max_builds != 1):
79 raise ValueError('max_builds must be 1 with shared workdir!')
81 c['workers'].append(Worker(name, password, max_builds = max_builds[name], properties = sl_props))
83 # 'slavePortnum' defines the TCP port to listen on for connections from workers.
84 # This must match the value configured into the buildslaves (with their
86 c['protocols'] = {'pb': {'port': slave_port}}
89 c['collapseRequests'] = True
91 # Reduce amount of backlog data
92 c['buildHorizon'] = 30
97 work_dir = os.path.abspath(ini.get("general", "workdir") or ".")
98 scripts_dir = os.path.abspath("../scripts")
100 rsync_bin_url = ini.get("rsync", "binary_url")
101 rsync_bin_key = ini.get("rsync", "binary_password")
106 if ini.has_option("rsync", "source_url"):
107 rsync_src_url = ini.get("rsync", "source_url")
108 rsync_src_key = ini.get("rsync", "source_password")
112 rsync_sdk_pat = "openwrt-sdk-*.tar.xz"
114 if ini.has_option("rsync", "sdk_url"):
115 rsync_sdk_url = ini.get("rsync", "sdk_url")
117 if ini.has_option("rsync", "sdk_password"):
118 rsync_sdk_key = ini.get("rsync", "sdk_password")
120 if ini.has_option("rsync", "sdk_pattern"):
121 rsync_sdk_pat = ini.get("rsync", "sdk_pattern")
123 repo_url = ini.get("repo", "url")
124 repo_branch = "master"
126 if ini.has_option("repo", "branch"):
127 repo_branch = ini.get("repo", "branch")
130 usign_comment = "untrusted comment: " + repo_branch.replace("-", " ").title() + " key"
132 if ini.has_option("usign", "key"):
133 usign_key = ini.get("usign", "key")
135 if ini.has_option("usign", "comment"):
136 usign_comment = ini.get("usign", "comment")
143 if not os.path.isdir(work_dir+'/source.git'):
144 subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, work_dir+'/source.git'])
146 subprocess.call(["git", "pull"], cwd = work_dir+'/source.git')
148 findarches = subprocess.Popen([scripts_dir + '/dumpinfo.pl', 'architectures'],
149 stdout = subprocess.PIPE, cwd = work_dir+'/source.git')
152 line = findarches.stdout.readline()
155 at = line.decode().strip().split()
157 archnames.append(at[0])
162 feedbranches = dict()
164 from buildbot.changes.gitpoller import GitPoller
165 c['change_source'] = []
167 def parse_feed_entry(line):
168 parts = line.strip().split()
169 if parts[0] == "src-git":
171 url = parts[2].strip().split(';')
172 branch = url[1] if len(url) > 1 else 'master'
173 feedbranches[url[0]] = branch
174 c['change_source'].append(GitPoller(url[0], branch=branch, workdir='%s/%s.git' %(os.getcwd(), parts[1]), pollinterval=300))
176 make = subprocess.Popen(['make', '--no-print-directory', '-C', work_dir+'/source.git/target/sdk/', 'val.BASE_FEED'],
177 env = dict(os.environ, TOPDIR=work_dir+'/source.git'), stdout = subprocess.PIPE)
179 line = make.stdout.readline()
181 parse_feed_entry(line)
183 with open(work_dir+'/source.git/feeds.conf.default', 'r') as f:
185 parse_feed_entry(line)
190 # Configure the Schedulers, which decide how to react to incoming changes. In this
191 # case, just kick off a 'basebuild' build
193 def branch_change_filter(change):
194 return change.branch == feedbranches[change.repository]
196 from buildbot.schedulers.basic import SingleBranchScheduler
197 from buildbot.schedulers.forcesched import ForceScheduler
198 from buildbot.changes import filter
200 c['schedulers'].append(SingleBranchScheduler(
202 change_filter=filter.ChangeFilter(filter_fn=branch_change_filter),
204 builderNames=archnames))
206 c['schedulers'].append(ForceScheduler(
208 builderNames=archnames))
212 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
213 # what steps, and which workers can execute them. Note that any particular build will
214 # only take place on one slave.
216 from buildbot.process.factory import BuildFactory
217 from buildbot.steps.shell import ShellCommand
218 from buildbot.steps.shell import SetProperty
219 from buildbot.steps.transfer import FileUpload
220 from buildbot.steps.transfer import FileDownload
221 from buildbot.steps.transfer import StringDownload
222 from buildbot.steps.master import MasterShellCommand
223 from buildbot.process.properties import WithProperties
226 def GetDirectorySuffix(props):
227 verpat = re.compile('^([0-9]{2})\.([0-9]{2})(?:\.([0-9]+)(?:-rc([0-9]+))?|-(SNAPSHOT))$')
228 if props.hasProperty("release_version"):
229 m = verpat.match(props["release_version"])
231 return "-%02d.%02d" %(int(m.group(1)), int(m.group(2)))
234 def GetNumJobs(props):
235 if props.hasProperty("workername") and props.hasProperty("nproc"):
236 return ((int(props["nproc"]) / (max_builds[props["workername"]] + other_builds)) + 1)
241 if props.hasProperty("builddir"):
242 return props["builddir"]
243 elif props.hasProperty("workdir"):
244 return props["workdir"]
248 def UsignSec2Pub(seckey, comment="untrusted comment: secret key"):
250 seckey = base64.b64decode(seckey)
254 return "{}\n{}".format(re.sub(r"\bsecret key$", "public key", comment),
255 base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]))
257 def IsSharedWorkdir(step):
258 return bool(step.getProperty("shared_wd"))
263 dlLock = locks.WorkerLock("slave_dl")
267 for slave in c['workers']:
268 slaveNames.append(slave.workername)
271 ts = arch[1].split('/')
273 factory = BuildFactory()
275 # setup shared work directory if required
276 factory.addStep(ShellCommand(
278 description = "Setting up shared work directory",
279 command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
281 haltOnFailure = True,
282 doStepIf = IsSharedWorkdir))
284 # find number of cores
285 factory.addStep(SetProperty(
288 description = "Finding number of CPUs",
289 command = ["nproc"]))
292 factory.addStep(FileDownload(
293 mastersrc = scripts_dir + '/cleanup.sh',
294 workerdest = "../cleanup.sh",
298 factory.addStep(ShellCommand(
300 description = "Cleaning previous builds",
301 command = ["./cleanup.sh", buildbot_url, WithProperties("%(workername)s"), WithProperties("%(buildername)s"), "full"],
303 haltOnFailure = True,
306 factory.addStep(ShellCommand(
308 description = "Cleaning work area",
309 command = ["./cleanup.sh", buildbot_url, WithProperties("%(workername)s"), WithProperties("%(buildername)s"), "single"],
311 haltOnFailure = True,
314 # expire tree if needed
315 elif tree_expire > 0:
316 factory.addStep(FileDownload(
317 mastersrc = scripts_dir + '/expire.sh',
318 workerdest = "../expire.sh",
321 factory.addStep(ShellCommand(
323 description = "Checking for build tree expiry",
324 command = ["./expire.sh", str(tree_expire)],
326 haltOnFailure = True,
329 factory.addStep(ShellCommand(
331 description = "Preparing SDK directory",
332 command = ["mkdir", "-p", "sdk"],
333 haltOnFailure = True))
335 factory.addStep(ShellCommand(
336 name = "downloadsdk",
337 description = "Downloading SDK archive",
338 command = ["rsync", "-4", "-va", "%s/%s/%s/%s" %(rsync_sdk_url, ts[0], ts[1], rsync_sdk_pat), "sdk.archive"],
339 env={'RSYNC_PASSWORD': rsync_sdk_key},
340 haltOnFailure = True,
343 factory.addStep(ShellCommand(
345 description = "Unpacking SDK archive",
346 command = "rm -rf sdk_update && mkdir sdk_update && tar --strip-components=1 -C sdk_update/ -vxf sdk.archive",
347 haltOnFailure = True))
349 factory.addStep(ShellCommand(
351 description = "Updating SDK",
352 command = "rsync --checksum -av sdk_update/ sdk/ && rm -rf sdk_update",
353 haltOnFailure = True))
355 factory.addStep(ShellCommand(
356 name = "cleancmdlinks",
357 description = "Sanitizing host command symlinks",
358 command = "find sdk/staging_dir/host/bin/ -type l -exec sh -c 'case $(readlink {}) in /bin/*|/usr/bin/*) true;; /*) rm -vf {};; esac' \\;",
359 haltOnFailure = True))
361 factory.addStep(StringDownload(
362 name = "writeversionmk",
363 s = 'TOPDIR:=${CURDIR}\n\ninclude $(TOPDIR)/include/version.mk\n\nversion:\n\t@echo $(VERSION_NUMBER)\n',
364 workerdest = "sdk/getversion.mk",
367 factory.addStep(SetProperty(
369 property = "release_version",
370 description = "Finding SDK release version",
371 workdir = "build/sdk",
372 command = ["make", "-f", "getversion.mk"]))
375 if usign_key is not None:
376 factory.addStep(StringDownload(
377 name = "dlkeybuildpub",
378 s = UsignSec2Pub(usign_key, usign_comment),
379 workerdest = "sdk/key-build.pub",
382 factory.addStep(StringDownload(
384 s = "# fake private key",
385 workerdest = "sdk/key-build",
388 factory.addStep(StringDownload(
389 name = "dlkeybuilducert",
390 s = "# fake certificate",
391 workerdest = "sdk/key-build.ucert",
394 factory.addStep(ShellCommand(
396 description = "Preparing download directory",
397 command = ["sh", "-c", "mkdir -p $HOME/dl && rm -rf ./sdk/dl && ln -sf $HOME/dl ./sdk/dl"],
398 haltOnFailure = True))
400 factory.addStep(ShellCommand(
402 description = "Preparing SDK configuration",
403 workdir = "build/sdk",
404 command = ["sh", "-c", "rm -f .config && make defconfig"]))
406 factory.addStep(FileDownload(
407 mastersrc = scripts_dir + '/ccache.sh',
408 workerdest = 'sdk/ccache.sh',
411 factory.addStep(ShellCommand(
413 description = "Preparing ccache",
414 workdir = "build/sdk",
415 command = ["./ccache.sh"],
416 haltOnFailure = True))
419 factory.addStep(StringDownload(
420 name = "dlgitclonekey",
422 workerdest = "../git-clone.key",
425 factory.addStep(ShellCommand(
426 name = "patchfeedsconf",
427 description = "Patching feeds.conf",
428 workdir = "build/sdk",
429 command = "sed -e 's#https://#ssh://git@#g' feeds.conf.default > feeds.conf",
430 haltOnFailure = True))
432 factory.addStep(ShellCommand(
433 name = "updatefeeds",
434 description = "Updating feeds",
435 workdir = "build/sdk",
436 command = ["./scripts/feeds", "update", "-f"],
437 env = {'GIT_SSH_COMMAND': WithProperties("ssh -o IdentitiesOnly=yes -o IdentityFile=%(cwd)s/git-clone.key -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no", cwd=GetCwd)} if git_ssh else {},
438 haltOnFailure = True))
441 factory.addStep(ShellCommand(
442 name = "rmfeedsconf",
443 description = "Removing feeds.conf",
444 workdir = "build/sdk",
445 command=["rm", "feeds.conf"],
446 haltOnFailure = True))
448 factory.addStep(ShellCommand(
449 name = "installfeeds",
450 description = "Installing feeds",
451 workdir = "build/sdk",
452 command = ["./scripts/feeds", "install", "-a"],
453 haltOnFailure = True))
455 factory.addStep(ShellCommand(
457 description = "Clearing failure logs",
458 workdir = "build/sdk",
459 command = ["rm", "-rf", "logs/package/error.txt", "faillogs/"],
460 haltOnFailure = False
463 factory.addStep(ShellCommand(
465 description = "Building packages",
466 workdir = "build/sdk",
468 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_AUTOREMOVE=y", "CONFIG_SIGNED_PACKAGES="],
469 env = {'CCACHE_BASEDIR': WithProperties("%(cwd)s", cwd=GetCwd)},
470 haltOnFailure = True))
472 factory.addStep(ShellCommand(
473 name = "mkfeedsconf",
474 description = "Generating pinned feeds.conf",
475 workdir = "build/sdk",
476 command = "./scripts/feeds list -s -f > bin/packages/%s/feeds.conf" %(arch[0])))
478 if ini.has_option("gpg", "key") or usign_key is not None:
479 factory.addStep(MasterShellCommand(
480 name = "signprepare",
481 description = "Preparing temporary signing directory",
482 command = ["mkdir", "-p", "%s/signing" %(work_dir)],
486 factory.addStep(ShellCommand(
488 description = "Packing files to sign",
489 workdir = "build/sdk",
490 command = "find bin/packages/%s/ -mindepth 2 -maxdepth 2 -type f -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(arch[0]),
494 factory.addStep(FileUpload(
495 workersrc = "sdk/sign.tar.gz",
496 masterdest = "%s/signing/%s.tar.gz" %(work_dir, arch[0]),
500 factory.addStep(MasterShellCommand(
502 description = "Signing files",
503 command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.tar.gz" %(work_dir, arch[0])],
504 env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
508 factory.addStep(FileDownload(
509 mastersrc = "%s/signing/%s.tar.gz" %(work_dir, arch[0]),
510 workerdest = "sdk/sign.tar.gz",
514 factory.addStep(ShellCommand(
516 description = "Unpacking signed files",
517 workdir = "build/sdk",
518 command = ["tar", "-xzf", "sign.tar.gz"],
522 factory.addStep(ShellCommand(
523 name = "uploadprepare",
524 description = "Preparing package directory",
525 workdir = "build/sdk",
526 command = ["rsync", "-4", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", WithProperties("%s/packages%%(suffix)s/" %(rsync_bin_url), suffix=GetDirectorySuffix)],
527 env={'RSYNC_PASSWORD': rsync_bin_key},
528 haltOnFailure = True,
532 factory.addStep(ShellCommand(
533 name = "packageupload",
534 description = "Uploading package files",
535 workdir = "build/sdk",
536 command = ["rsync", "-4", "--progress", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "bin/packages/%s/" %(arch[0]), WithProperties("%s/packages%%(suffix)s/%s/" %(rsync_bin_url, arch[0]), suffix=GetDirectorySuffix)],
537 env={'RSYNC_PASSWORD': rsync_bin_key},
538 haltOnFailure = True,
542 factory.addStep(ShellCommand(
544 description = "Preparing log directory",
545 workdir = "build/sdk",
546 command = ["rsync", "-4", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", WithProperties("%s/faillogs%%(suffix)s/" %(rsync_bin_url), suffix=GetDirectorySuffix)],
547 env={'RSYNC_PASSWORD': rsync_bin_key},
548 haltOnFailure = True,
552 factory.addStep(ShellCommand(
554 description = "Finding failure logs",
555 workdir = "build/sdk/logs/package/feeds",
556 command = ["sh", "-c", "sed -ne 's!^ *ERROR: package/feeds/\\([^ ]*\\) .*$!\\1!p' ../error.txt | sort -u | xargs -r find > ../../../logs.txt"],
557 haltOnFailure = False
560 factory.addStep(ShellCommand(
562 description = "Collecting failure logs",
563 workdir = "build/sdk",
564 command = ["rsync", "-av", "--files-from=logs.txt", "logs/package/feeds/", "faillogs/"],
565 haltOnFailure = False
568 factory.addStep(ShellCommand(
570 description = "Uploading failure logs",
571 workdir = "build/sdk",
572 command = ["rsync", "-4", "--progress", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "faillogs/", WithProperties("%s/faillogs%%(suffix)s/%s/" %(rsync_bin_url, arch[0]), suffix=GetDirectorySuffix)],
573 env={'RSYNC_PASSWORD': rsync_bin_key},
574 haltOnFailure = False,
578 if rsync_src_url is not None:
579 factory.addStep(ShellCommand(
581 description = "Finding source archives to upload",
582 workdir = "build/sdk",
583 command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -newer ../sdk.archive -printf '%f\\n' > sourcelist",
587 factory.addStep(ShellCommand(
588 name = "sourceupload",
589 description = "Uploading source archives",
590 workdir = "build/sdk",
591 command = ["rsync", "--files-from=sourcelist", "-4", "--progress", "--checksum", "--delay-updates",
592 WithProperties("--partial-dir=.~tmp~%s~%%(workername)s" %(arch[0])), "-avz", "dl/", "%s/" %(rsync_src_url)],
593 env={'RSYNC_PASSWORD': rsync_src_key},
594 haltOnFailure = False,
598 factory.addStep(ShellCommand(
600 description = "Reporting disk usage",
601 command=["df", "-h", "."],
603 haltOnFailure = False,
607 from buildbot.config import BuilderConfig
609 c['builders'].append(BuilderConfig(name=arch[0], workernames=slaveNames, factory=factory))
612 ####### STATUS arches
614 # 'status' is a list of Status arches. The results of each build will be
615 # pushed to these arches. buildbot/status/*.py has a variety to choose from,
616 # including web pages, email senders, and IRC bots.
618 from buildbot.plugins import util
620 if ini.has_option("phase2", "status_bind"):
622 'port': ini.get("phase2", "status_bind"),
624 'waterfall_view': True,
625 'console_view': True,
630 if ini.has_option("phase2", "status_user") and ini.has_option("phase2", "status_password"):
631 c['www']['auth'] = util.UserPasswordAuth([
632 (ini.get("phase2", "status_user"), ini.get("phase2", "status_password"))
634 c['www']['authz'] = util.Authz(
635 allowRules=[ util.AnyControlEndpointMatcher(role="admins") ],
636 roleMatchers=[ util.RolesFromUsername(roles=["admins"], usernames=[ini.get("phase2", "status_user")]) ]
639 ####### PROJECT IDENTITY
641 # the 'title' string will appear at the top of this buildbot
642 # installation's html.WebStatus home page (linked to the
643 # 'titleURL') and is embedded in the title of the waterfall HTML page.
645 c['title'] = ini.get("general", "title")
646 c['titleURL'] = ini.get("general", "title_url")
648 # the 'buildbotURL' string should point to the location where the buildbot's
649 # internal web server (usually the html.WebStatus page) is visible. This
650 # typically uses the port number set in the Waterfall 'status' entry, but
651 # with an externally-visible host name which the buildbot cannot figure out
654 c['buildbotURL'] = buildbot_url
659 # This specifies what database buildbot uses to store its state. You can leave
660 # this at its default for all but the largest installations.
661 'db_url' : "sqlite:///state.sqlite",
664 c['buildbotNetUsageData'] = None