2 # ex: set syntax=python:
9 from buildbot import locks
11 ini = ConfigParser.ConfigParser()
12 ini.read("./config.ini")
14 buildbot_url = ini.get("general", "buildbot_url")
16 # This is a sample buildmaster config file. It must be installed as
17 # 'master.cfg' in your buildmaster's base directory.
19 # This is the dictionary that the buildmaster pays attention to. We also use
20 # a shorter alias to save typing.
21 c = BuildmasterConfig = {}
25 # The 'slaves' list defines the set of recognized buildslaves. Each element is
26 # a BuildSlave object, specifying a unique slave name and password. The same
27 # slave name and password must be configured on the slave.
28 from buildbot.buildslave import BuildSlave
32 if ini.has_option("general", "port"):
33 slave_port = ini.getint("general", "port")
37 for section in ini.sections():
38 if section.startswith("slave "):
39 if ini.has_option(section, "name") and ini.has_option(section, "password"):
40 name = ini.get(section, "name")
41 password = ini.get(section, "password")
43 if ini.has_option(section, "builds"):
44 max_builds = ini.getint(section, "builds")
45 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
47 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
48 # This must match the value configured into the buildslaves (with their
50 c['slavePortnum'] = slave_port
53 c['mergeRequests'] = True
57 home_dir = os.path.abspath(ini.get("general", "homedir"))
59 rsync_bin_url = ini.get("rsync", "binary_url")
60 rsync_bin_key = ini.get("rsync", "binary_password")
65 if ini.has_option("rsync", "source_url"):
66 rsync_src_url = ini.get("rsync", "source_url")
67 rsync_src_key = ini.get("rsync", "source_password")
71 rsync_sdk_pat = "lede-sdk-*.tar.xz"
73 if ini.has_option("rsync", "sdk_url"):
74 rsync_sdk_url = ini.get("rsync", "sdk_url")
76 if ini.has_option("rsync", "sdk_password"):
77 rsync_sdk_key = ini.get("rsync", "sdk_password")
79 if ini.has_option("rsync", "sdk_pattern"):
80 rsync_sdk_pat = ini.get("rsync", "sdk_pattern")
83 gpg_comment = "Unattended build signature"
84 gpg_passfile = "/dev/null"
86 if ini.has_option("gpg", "keyid"):
87 gpg_keyid = ini.get("gpg", "keyid")
89 if ini.has_option("gpg", "comment"):
90 gpg_comment = ini.get("gpg", "comment")
92 if ini.has_option("gpg", "passfile"):
93 gpg_passfile = ini.get("gpg", "passfile")
100 findarches = subprocess.Popen([home_dir+'/dumpinfo.pl', 'architectures'],
101 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
104 line = findarches.stdout.readline()
107 at = line.strip().split()
109 archnames.append(at[0])
115 from buildbot.changes.gitpoller import GitPoller
116 c['change_source'] = []
118 with open(home_dir+'/source.git/feeds.conf.default', 'r') as f:
120 parts = line.strip().split()
121 if parts[0] == "src-git":
123 c['change_source'].append(GitPoller(parts[2], workdir='%s/%s.git' %(os.getcwd(), parts[1]), branch='master', pollinterval=300))
128 # Configure the Schedulers, which decide how to react to incoming changes. In this
129 # case, just kick off a 'basebuild' build
131 from buildbot.schedulers.basic import SingleBranchScheduler
132 from buildbot.schedulers.forcesched import ForceScheduler
133 from buildbot.changes import filter
135 c['schedulers'].append(SingleBranchScheduler(
137 change_filter=filter.ChangeFilter(branch='master'),
139 builderNames=archnames))
141 c['schedulers'].append(ForceScheduler(
143 builderNames=archnames))
147 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
148 # what steps, and which slaves can execute them. Note that any particular build will
149 # only take place on one slave.
151 from buildbot.process.factory import BuildFactory
152 from buildbot.steps.source import Git
153 from buildbot.steps.shell import ShellCommand
154 from buildbot.steps.shell import SetProperty
155 from buildbot.steps.transfer import FileUpload
156 from buildbot.steps.transfer import FileDownload
157 from buildbot.steps.master import MasterShellCommand
158 from buildbot.process.properties import WithProperties
162 dlLock = locks.SlaveLock("slave_dl")
166 for slave in c['slaves']:
167 slaveNames.append(slave.slavename)
170 ts = arch[1].split('/')
172 factory = BuildFactory()
174 # find number of cores
175 factory.addStep(SetProperty(
178 description = "Finding number of CPUs",
179 command = ["nproc"]))
182 factory.addStep(FileDownload(mastersrc="cleanup.sh", slavedest="cleanup.sh", mode=0755))
184 factory.addStep(ShellCommand(
186 description = "Cleaning previous builds",
187 command = ["./cleanup.sh", buildbot_url, WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"],
188 haltOnFailure = True,
191 factory.addStep(ShellCommand(
193 description = "Cleaning work area",
194 command = ["./cleanup.sh", buildbot_url, WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"],
195 haltOnFailure = True,
198 factory.addStep(ShellCommand(
200 description = "Preparing SDK directory",
201 command = ["mkdir", "sdk"],
202 haltOnFailure = True))
204 factory.addStep(ShellCommand(
205 name = "downloadsdk",
206 description = "Downloading SDK archive",
207 command = ["rsync", "-va", "%s/%s/%s/%s" %(rsync_sdk_url, ts[0], ts[1], rsync_sdk_pat), "sdk.archive"],
208 env={'RSYNC_PASSWORD': rsync_sdk_key},
209 haltOnFailure = True,
212 factory.addStep(ShellCommand(
214 description = "Unpacking SDK archive",
215 command = ["tar", "--strip-components=1", "-C", "sdk/", "-vxf", "sdk.archive"],
216 haltOnFailure = True))
218 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="sdk/key-build", mode=0600))
219 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="sdk/key-build.pub", mode=0600))
221 factory.addStep(ShellCommand(
223 description = "Preparing download directory",
224 command = ["sh", "-c", "mkdir -p $HOME/dl && rmdir ./sdk/dl && ln -sf $HOME/dl ./sdk/dl"]))
226 factory.addStep(ShellCommand(
228 description = "Preparing SDK configuration",
229 workdir = "build/sdk",
230 command = ["sh", "-c", "rm -f .config && make defconfig"]))
232 factory.addStep(ShellCommand(
233 name = "updatefeeds",
234 description = "Updating feeds",
235 workdir = "build/sdk",
236 command = ["./scripts/feeds", "update"]))
238 factory.addStep(ShellCommand(
239 name = "installfeeds",
240 description = "Installing feeds",
241 workdir = "build/sdk",
242 command = ["./scripts/feeds", "install", "-a"]))
244 factory.addStep(ShellCommand(
246 description = "Building packages",
247 workdir = "build/sdk",
248 command = ["make", WithProperties("-j%(nproc:~4)s"), "V=s", "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y"]))
250 if gpg_keyid is not None:
251 factory.addStep(MasterShellCommand(
252 name = "signprepare",
253 description = "Preparing temporary signing directory",
254 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
258 factory.addStep(ShellCommand(
260 description = "Packing files to sign",
261 workdir = "build/sdk",
262 command = "find bin/packages/%s/ -mindepth 2 -maxdepth 2 -type f -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(arch[0]),
266 factory.addStep(FileUpload(
267 slavesrc = "sdk/sign.tar.gz",
268 masterdest = "%s/signing/%s.tar.gz" %(home_dir, arch[0]),
272 factory.addStep(MasterShellCommand(
274 description = "Signing files",
275 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.tar.gz" %(home_dir, arch[0]), gpg_keyid, gpg_passfile, gpg_comment],
279 factory.addStep(FileDownload(
280 mastersrc = "%s/signing/%s.tar.gz" %(home_dir, arch[0]),
281 slavedest = "sdk/sign.tar.gz",
285 factory.addStep(ShellCommand(
287 description = "Unpacking signed files",
288 workdir = "build/sdk",
289 command = ["tar", "-xzf", "sign.tar.gz"],
293 factory.addStep(ShellCommand(
294 name = "uploadprepare",
295 description = "Preparing package directory",
296 workdir = "build/sdk",
297 command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/packages/" %(rsync_bin_url)],
298 env={'RSYNC_PASSWORD': rsync_bin_key},
299 haltOnFailure = True,
303 factory.addStep(ShellCommand(
304 name = "packageupload",
305 description = "Uploading package files",
306 workdir = "build/sdk",
307 command = ["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "bin/packages/%s/" %(arch[0]), "%s/packages/%s/" %(rsync_bin_url, arch[0])],
308 env={'RSYNC_PASSWORD': rsync_bin_key},
309 haltOnFailure = True,
313 factory.addStep(ShellCommand(
315 description = "Preparing log directory",
316 workdir = "build/sdk",
317 command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/faillogs/" %(rsync_bin_url)],
318 env={'RSYNC_PASSWORD': rsync_bin_key},
319 haltOnFailure = True,
323 factory.addStep(ShellCommand(
325 description = "Finding failure logs",
326 workdir = "build/sdk/logs/package/feeds",
327 command = ["sh", "-c", "sed -ne 's!^ *ERROR: package/feeds/\\([^ ]*\\) .*$!\\1!p' ../error.txt | sort -u | xargs -r find > ../../../logs.txt"],
328 haltOnFailure = False
331 factory.addStep(ShellCommand(
333 description = "Collecting failure logs",
334 workdir = "build/sdk",
335 command = ["rsync", "-av", "--files-from=logs.txt", "logs/package/feeds/", "faillogs/"],
336 haltOnFailure = False
339 factory.addStep(ShellCommand(
341 description = "Uploading failure logs",
342 workdir = "build/sdk",
343 command = ["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "faillogs/", "%s/faillogs/%s/" %(rsync_bin_url, arch[0])],
344 env={'RSYNC_PASSWORD': rsync_bin_key},
345 haltOnFailure = False,
349 if rsync_src_url is not None:
350 factory.addStep(ShellCommand(
351 name = "sourceupload",
352 description = "Uploading source archives",
353 workdir = "build/sdk",
354 command = ["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "dl/", "%s/" %(rsync_src_url)],
355 env={'RSYNC_PASSWORD': rsync_src_key},
356 haltOnFailure = False,
360 from buildbot.config import BuilderConfig
362 c['builders'].append(BuilderConfig(name=arch[0], slavenames=slaveNames, factory=factory))
365 ####### STATUS arches
367 # 'status' is a list of Status arches. The results of each build will be
368 # pushed to these arches. buildbot/status/*.py has a variety to choose from,
369 # including web pages, email senders, and IRC bots.
373 from buildbot.status import html
374 from buildbot.status.web import authz, auth
376 if ini.has_option("status", "bind"):
377 if ini.has_option("status", "user") and ini.has_option("status", "password"):
378 authz_cfg=authz.Authz(
379 # change any of these to True to enable; see the manual for more
381 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
382 gracefulShutdown = 'auth',
383 forceBuild = 'auth', # use this to test your slave once it is set up
384 forceAllBuilds = 'auth',
387 stopAllBuilds = 'auth',
388 cancelPendingBuild = 'auth',
390 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
392 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
394 ####### PROJECT IDENTITY
396 # the 'title' string will appear at the top of this buildbot
397 # installation's html.WebStatus home page (linked to the
398 # 'titleURL') and is embedded in the title of the waterfall HTML page.
400 c['title'] = ini.get("general", "title")
401 c['titleURL'] = ini.get("general", "title_url")
403 # the 'buildbotURL' string should point to the location where the buildbot's
404 # internal web server (usually the html.WebStatus page) is visible. This
405 # typically uses the port number set in the Waterfall 'status' entry, but
406 # with an externally-visible host name which the buildbot cannot figure out
409 c['buildbotURL'] = buildbot_url
414 # This specifies what database buildbot uses to store its state. You can leave
415 # this at its default for all but the largest installations.
416 'db_url' : "sqlite:///state.sqlite",