2 # ex: set syntax=python:
9 from buildbot import locks
11 ini = ConfigParser.ConfigParser()
12 ini.read("./config.ini")
14 buildbot_url = ini.get("general", "buildbot_url")
16 # This is a sample buildmaster config file. It must be installed as
17 # 'master.cfg' in your buildmaster's base directory.
19 # This is the dictionary that the buildmaster pays attention to. We also use
20 # a shorter alias to save typing.
21 c = BuildmasterConfig = {}
25 # The 'slaves' list defines the set of recognized buildslaves. Each element is
26 # a BuildSlave object, specifying a unique slave name and password. The same
27 # slave name and password must be configured on the slave.
28 from buildbot.buildslave import BuildSlave
35 if ini.has_option("general", "port"):
36 slave_port = ini.getint("general", "port")
38 if ini.has_option("general", "persistent"):
39 persistent = ini.getboolean("general", "persistent")
41 if ini.has_option("general", "other_builds"):
42 other_builds = ini.getint("general", "other_builds")
44 if ini.has_option("general", "expire"):
45 tree_expire = ini.getint("general", "expire")
50 for section in ini.sections():
51 if section.startswith("slave "):
52 if ini.has_option(section, "name") and ini.has_option(section, "password"):
53 name = ini.get(section, "name")
54 password = ini.get(section, "password")
56 if ini.has_option(section, "builds"):
57 max_builds[name] = ini.getint(section, "builds")
58 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name]))
60 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
61 # This must match the value configured into the buildslaves (with their
63 c['slavePortnum'] = slave_port
66 c['mergeRequests'] = True
68 # Reduce amount of backlog data
69 c['buildHorizon'] = 30
74 home_dir = os.path.abspath(ini.get("general", "homedir"))
76 rsync_bin_url = ini.get("rsync", "binary_url")
77 rsync_bin_key = ini.get("rsync", "binary_password")
82 if ini.has_option("rsync", "source_url"):
83 rsync_src_url = ini.get("rsync", "source_url")
84 rsync_src_key = ini.get("rsync", "source_password")
88 rsync_sdk_pat = "lede-sdk-*.tar.xz"
90 if ini.has_option("rsync", "sdk_url"):
91 rsync_sdk_url = ini.get("rsync", "sdk_url")
93 if ini.has_option("rsync", "sdk_password"):
94 rsync_sdk_key = ini.get("rsync", "sdk_password")
96 if ini.has_option("rsync", "sdk_pattern"):
97 rsync_sdk_pat = ini.get("rsync", "sdk_pattern")
101 gpg_comment = "Unattended build signature"
102 gpg_passfile = "/dev/null"
104 if ini.has_option("gpg", "home"):
105 gpg_home = ini.get("gpg", "home")
107 if ini.has_option("gpg", "keyid"):
108 gpg_keyid = ini.get("gpg", "keyid")
110 if ini.has_option("gpg", "comment"):
111 gpg_comment = ini.get("gpg", "comment")
113 if ini.has_option("gpg", "passfile"):
114 gpg_passfile = ini.get("gpg", "passfile")
121 findarches = subprocess.Popen([home_dir+'/dumpinfo.pl', 'architectures'],
122 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
125 line = findarches.stdout.readline()
128 at = line.strip().split()
130 archnames.append(at[0])
135 feedbranches = dict()
137 from buildbot.changes.gitpoller import GitPoller
138 c['change_source'] = []
140 with open(home_dir+'/source.git/feeds.conf.default', 'r') as f:
142 parts = line.strip().split()
143 if parts[0] == "src-git":
145 url = parts[2].strip().split(';')
146 branch = url[1] if len(url) > 1 else 'master'
147 feedbranches[url[0]] = branch
148 c['change_source'].append(GitPoller(url[0], branch=branch, workdir='%s/%s.git' %(os.getcwd(), parts[1]), pollinterval=300))
153 # Configure the Schedulers, which decide how to react to incoming changes. In this
154 # case, just kick off a 'basebuild' build
156 def branch_change_filter(change):
157 return change.branch == feedbranches[change.repository]
159 from buildbot.schedulers.basic import SingleBranchScheduler
160 from buildbot.schedulers.forcesched import ForceScheduler
161 from buildbot.changes import filter
163 c['schedulers'].append(SingleBranchScheduler(
165 change_filter=filter.ChangeFilter(filter_fn=branch_change_filter),
167 builderNames=archnames))
169 c['schedulers'].append(ForceScheduler(
171 builderNames=archnames))
175 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
176 # what steps, and which slaves can execute them. Note that any particular build will
177 # only take place on one slave.
179 from buildbot.process.factory import BuildFactory
180 from buildbot.steps.source import Git
181 from buildbot.steps.shell import ShellCommand
182 from buildbot.steps.shell import SetProperty
183 from buildbot.steps.transfer import FileUpload
184 from buildbot.steps.transfer import FileDownload
185 from buildbot.steps.master import MasterShellCommand
186 from buildbot.process.properties import WithProperties
189 def GetDirectorySuffix(props):
190 if props.hasProperty("slavename") and re.match("^[^-]+-[0-9]+\.[0-9]+-[^-]+$", props["slavename"]):
191 return "-%s" % props["slavename"].split('-')[1]
195 def GetNumJobs(props):
196 if props.hasProperty("slavename") and props.hasProperty("nproc"):
197 return ((int(props["nproc"]) / (max_builds[props["slavename"]] + other_builds)) + 1)
202 if props.hasProperty("builddir"):
203 return props["builddir"]
204 elif props.hasProperty("workdir"):
205 return props["workdir"]
212 dlLock = locks.SlaveLock("slave_dl")
216 for slave in c['slaves']:
217 slaveNames.append(slave.slavename)
220 ts = arch[1].split('/')
222 factory = BuildFactory()
224 # find number of cores
225 factory.addStep(SetProperty(
228 description = "Finding number of CPUs",
229 command = ["nproc"]))
232 factory.addStep(FileDownload(mastersrc="cleanup.sh", slavedest="cleanup.sh", mode=0755))
235 factory.addStep(ShellCommand(
237 description = "Cleaning previous builds",
238 command = ["./cleanup.sh", buildbot_url, WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"],
239 haltOnFailure = True,
242 factory.addStep(ShellCommand(
244 description = "Cleaning work area",
245 command = ["./cleanup.sh", buildbot_url, WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"],
246 haltOnFailure = True,
249 # expire tree if needed
250 elif tree_expire > 0:
251 factory.addStep(FileDownload(
252 mastersrc = home_dir+"/expire.sh",
253 slavedest = "../expire.sh",
256 factory.addStep(ShellCommand(
258 description = "Checking for build tree expiry",
259 command = ["./expire.sh", str(tree_expire)],
261 haltOnFailure = True,
264 factory.addStep(ShellCommand(
266 description = "Preparing SDK directory",
267 command = ["mkdir", "-p", "sdk"],
268 haltOnFailure = True))
270 factory.addStep(ShellCommand(
271 name = "downloadsdk",
272 description = "Downloading SDK archive",
273 command = ["rsync", "-4", "-va", "%s/%s/%s/%s" %(rsync_sdk_url, ts[0], ts[1], rsync_sdk_pat), "sdk.archive"],
274 env={'RSYNC_PASSWORD': rsync_sdk_key},
275 haltOnFailure = True,
278 factory.addStep(ShellCommand(
280 description = "Unpacking SDK archive",
281 command = "rm -rf sdk_update && mkdir sdk_update && tar --strip-components=1 -C sdk_update/ -vxf sdk.archive",
282 haltOnFailure = True))
284 factory.addStep(ShellCommand(
286 description = "Updating SDK",
287 command = "rsync --checksum -av sdk_update/ sdk/ && rm -rf sdk_update",
288 haltOnFailure = True))
290 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="sdk/key-build", mode=0600))
291 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="sdk/key-build.pub", mode=0600))
293 factory.addStep(ShellCommand(
295 description = "Preparing download directory",
296 command = ["sh", "-c", "mkdir -p $HOME/dl && rm -rf ./sdk/dl && ln -sf $HOME/dl ./sdk/dl"]))
298 factory.addStep(ShellCommand(
300 description = "Preparing SDK configuration",
301 workdir = "build/sdk",
302 command = ["sh", "-c", "rm -f .config && make defconfig"]))
304 factory.addStep(FileDownload(
305 mastersrc = home_dir+'/ccache.sh',
306 slavedest = 'sdk/ccache.sh',
309 factory.addStep(ShellCommand(
311 description = "Preparing ccache",
312 workdir = "build/sdk",
313 command = ["./ccache.sh"]))
315 factory.addStep(ShellCommand(
316 name = "updatefeeds",
317 description = "Updating feeds",
318 workdir = "build/sdk",
319 command = ["./scripts/feeds", "update", "-f"]))
321 factory.addStep(ShellCommand(
322 name = "installfeeds",
323 description = "Installing feeds",
324 workdir = "build/sdk",
325 command = ["./scripts/feeds", "install", "-a"]))
327 factory.addStep(ShellCommand(
329 description = "Building packages",
330 workdir = "build/sdk",
332 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y", "CONFIG_AUTOREMOVE=y"],
333 env = {'CCACHE_BASEDIR': WithProperties("%(cwd)s", cwd=GetCwd)},
334 haltOnFailure = True))
336 factory.addStep(ShellCommand(
337 name = "mkfeedsconf",
338 description = "Generating pinned feeds.conf",
339 workdir = "build/sdk",
340 command = "./scripts/feeds list -s -f > bin/packages/%s/feeds.conf" %(arch[0])))
342 if gpg_keyid is not None:
343 factory.addStep(MasterShellCommand(
344 name = "signprepare",
345 description = "Preparing temporary signing directory",
346 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
350 factory.addStep(ShellCommand(
352 description = "Packing files to sign",
353 workdir = "build/sdk",
354 command = "find bin/packages/%s/ -mindepth 2 -maxdepth 2 -type f -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(arch[0]),
358 factory.addStep(FileUpload(
359 slavesrc = "sdk/sign.tar.gz",
360 masterdest = "%s/signing/%s.tar.gz" %(home_dir, arch[0]),
364 factory.addStep(MasterShellCommand(
366 description = "Signing files",
367 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.tar.gz" %(home_dir, arch[0]), gpg_keyid, gpg_comment],
368 env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
372 factory.addStep(FileDownload(
373 mastersrc = "%s/signing/%s.tar.gz" %(home_dir, arch[0]),
374 slavedest = "sdk/sign.tar.gz",
378 factory.addStep(ShellCommand(
380 description = "Unpacking signed files",
381 workdir = "build/sdk",
382 command = ["tar", "-xzf", "sign.tar.gz"],
386 factory.addStep(ShellCommand(
387 name = "uploadprepare",
388 description = "Preparing package directory",
389 workdir = "build/sdk",
390 command = ["rsync", "-4", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", WithProperties("%s/packages%%(suffix)s/" %(rsync_bin_url), suffix=GetDirectorySuffix)],
391 env={'RSYNC_PASSWORD': rsync_bin_key},
392 haltOnFailure = True,
396 factory.addStep(ShellCommand(
397 name = "packageupload",
398 description = "Uploading package files",
399 workdir = "build/sdk",
400 command = ["rsync", "-4", "--progress", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "bin/packages/%s/" %(arch[0]), WithProperties("%s/packages%%(suffix)s/%s/" %(rsync_bin_url, arch[0]), suffix=GetDirectorySuffix)],
401 env={'RSYNC_PASSWORD': rsync_bin_key},
402 haltOnFailure = True,
406 factory.addStep(ShellCommand(
408 description = "Preparing log directory",
409 workdir = "build/sdk",
410 command = ["rsync", "-4", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/faillogs/" %(rsync_bin_url)],
411 env={'RSYNC_PASSWORD': rsync_bin_key},
412 haltOnFailure = True,
416 factory.addStep(ShellCommand(
418 description = "Finding failure logs",
419 workdir = "build/sdk/logs/package/feeds",
420 command = ["sh", "-c", "sed -ne 's!^ *ERROR: package/feeds/\\([^ ]*\\) .*$!\\1!p' ../error.txt | sort -u | xargs -r find > ../../../logs.txt"],
421 haltOnFailure = False
424 factory.addStep(ShellCommand(
426 description = "Collecting failure logs",
427 workdir = "build/sdk",
428 command = ["rsync", "-av", "--files-from=logs.txt", "logs/package/feeds/", "faillogs/"],
429 haltOnFailure = False
432 factory.addStep(ShellCommand(
434 description = "Uploading failure logs",
435 workdir = "build/sdk",
436 command = ["rsync", "-4", "--progress", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "faillogs/", "%s/faillogs/%s/" %(rsync_bin_url, arch[0])],
437 env={'RSYNC_PASSWORD': rsync_bin_key},
438 haltOnFailure = False,
442 if rsync_src_url is not None:
443 factory.addStep(ShellCommand(
444 name = "sourceupload",
445 description = "Uploading source archives",
446 workdir = "build/sdk",
447 command = ["rsync", "-4", "--progress", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "dl/", "%s/" %(rsync_src_url)],
448 env={'RSYNC_PASSWORD': rsync_src_key},
449 haltOnFailure = False,
453 from buildbot.config import BuilderConfig
455 c['builders'].append(BuilderConfig(name=arch[0], slavenames=slaveNames, factory=factory))
458 ####### STATUS arches
460 # 'status' is a list of Status arches. The results of each build will be
461 # pushed to these arches. buildbot/status/*.py has a variety to choose from,
462 # including web pages, email senders, and IRC bots.
466 from buildbot.status import html
467 from buildbot.status.web import authz, auth
469 if ini.has_option("status", "bind"):
470 if ini.has_option("status", "user") and ini.has_option("status", "password"):
471 authz_cfg=authz.Authz(
472 # change any of these to True to enable; see the manual for more
474 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
475 gracefulShutdown = 'auth',
476 forceBuild = 'auth', # use this to test your slave once it is set up
477 forceAllBuilds = 'auth',
480 stopAllBuilds = 'auth',
481 cancelPendingBuild = 'auth',
483 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
485 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
487 ####### PROJECT IDENTITY
489 # the 'title' string will appear at the top of this buildbot
490 # installation's html.WebStatus home page (linked to the
491 # 'titleURL') and is embedded in the title of the waterfall HTML page.
493 c['title'] = ini.get("general", "title")
494 c['titleURL'] = ini.get("general", "title_url")
496 # the 'buildbotURL' string should point to the location where the buildbot's
497 # internal web server (usually the html.WebStatus page) is visible. This
498 # typically uses the port number set in the Waterfall 'status' entry, but
499 # with an externally-visible host name which the buildbot cannot figure out
502 c['buildbotURL'] = buildbot_url
507 # This specifies what database buildbot uses to store its state. You can leave
508 # this at its default for all but the largest installations.
509 'db_url' : "sqlite:///state.sqlite",