2 # ex: set syntax=python:
9 from buildbot import locks
11 ini = ConfigParser.ConfigParser()
12 ini.read("./config.ini")
14 buildbot_url = ini.get("general", "buildbot_url")
16 # This is a sample buildmaster config file. It must be installed as
17 # 'master.cfg' in your buildmaster's base directory.
19 # This is the dictionary that the buildmaster pays attention to. We also use
20 # a shorter alias to save typing.
21 c = BuildmasterConfig = {}
25 # The 'slaves' list defines the set of recognized buildslaves. Each element is
26 # a BuildSlave object, specifying a unique slave name and password. The same
27 # slave name and password must be configured on the slave.
28 from buildbot.buildslave import BuildSlave
33 if ini.has_option("general", "port"):
34 slave_port = ini.getint("general", "port")
36 if ini.has_option("general", "persistent"):
37 persistent = ini.getboolean("general", "persistent")
41 for section in ini.sections():
42 if section.startswith("slave "):
43 if ini.has_option(section, "name") and ini.has_option(section, "password"):
44 name = ini.get(section, "name")
45 password = ini.get(section, "password")
47 if ini.has_option(section, "builds"):
48 max_builds = ini.getint(section, "builds")
49 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
51 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
52 # This must match the value configured into the buildslaves (with their
54 c['slavePortnum'] = slave_port
57 c['mergeRequests'] = True
61 home_dir = os.path.abspath(ini.get("general", "homedir"))
63 rsync_bin_url = ini.get("rsync", "binary_url")
64 rsync_bin_key = ini.get("rsync", "binary_password")
69 if ini.has_option("rsync", "source_url"):
70 rsync_src_url = ini.get("rsync", "source_url")
71 rsync_src_key = ini.get("rsync", "source_password")
75 rsync_sdk_pat = "lede-sdk-*.tar.xz"
77 if ini.has_option("rsync", "sdk_url"):
78 rsync_sdk_url = ini.get("rsync", "sdk_url")
80 if ini.has_option("rsync", "sdk_password"):
81 rsync_sdk_key = ini.get("rsync", "sdk_password")
83 if ini.has_option("rsync", "sdk_pattern"):
84 rsync_sdk_pat = ini.get("rsync", "sdk_pattern")
88 gpg_comment = "Unattended build signature"
89 gpg_passfile = "/dev/null"
91 if ini.has_option("gpg", "home"):
92 gpg_home = ini.get("gpg", "home")
94 if ini.has_option("gpg", "keyid"):
95 gpg_keyid = ini.get("gpg", "keyid")
97 if ini.has_option("gpg", "comment"):
98 gpg_comment = ini.get("gpg", "comment")
100 if ini.has_option("gpg", "passfile"):
101 gpg_passfile = ini.get("gpg", "passfile")
108 findarches = subprocess.Popen([home_dir+'/dumpinfo.pl', 'architectures'],
109 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
112 line = findarches.stdout.readline()
115 at = line.strip().split()
117 archnames.append(at[0])
123 from buildbot.changes.gitpoller import GitPoller
124 c['change_source'] = []
126 with open(home_dir+'/source.git/feeds.conf.default', 'r') as f:
128 parts = line.strip().split()
129 if parts[0] == "src-git":
131 url = parts[2].strip().split(';')
132 branch = url[1] if len(url) > 1 else 'master'
133 c['change_source'].append(GitPoller(url[0], branch=branch, workdir='%s/%s.git' %(os.getcwd(), parts[1]), pollinterval=300))
138 # Configure the Schedulers, which decide how to react to incoming changes. In this
139 # case, just kick off a 'basebuild' build
141 from buildbot.schedulers.basic import SingleBranchScheduler
142 from buildbot.schedulers.forcesched import ForceScheduler
143 from buildbot.changes import filter
145 c['schedulers'].append(SingleBranchScheduler(
147 change_filter=filter.ChangeFilter(branch='master'),
149 builderNames=archnames))
151 c['schedulers'].append(ForceScheduler(
153 builderNames=archnames))
157 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
158 # what steps, and which slaves can execute them. Note that any particular build will
159 # only take place on one slave.
161 from buildbot.process.factory import BuildFactory
162 from buildbot.steps.source import Git
163 from buildbot.steps.shell import ShellCommand
164 from buildbot.steps.shell import SetProperty
165 from buildbot.steps.transfer import FileUpload
166 from buildbot.steps.transfer import FileDownload
167 from buildbot.steps.master import MasterShellCommand
168 from buildbot.process.properties import WithProperties
171 def GetDirectorySuffix(props):
172 if props.hasProperty("slavename") and re.match("^[^-]+-[0-9]+\.[0-9]+-[^-]+$", props["slavename"]):
173 return "-%s" % props["slavename"].split('-')[1]
180 dlLock = locks.SlaveLock("slave_dl")
184 for slave in c['slaves']:
185 slaveNames.append(slave.slavename)
188 ts = arch[1].split('/')
190 factory = BuildFactory()
192 # find number of cores
193 factory.addStep(SetProperty(
196 description = "Finding number of CPUs",
197 command = ["nproc"]))
200 factory.addStep(FileDownload(mastersrc="cleanup.sh", slavedest="cleanup.sh", mode=0755))
203 factory.addStep(ShellCommand(
205 description = "Cleaning previous builds",
206 command = ["./cleanup.sh", buildbot_url, WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"],
207 haltOnFailure = True,
210 factory.addStep(ShellCommand(
212 description = "Cleaning work area",
213 command = ["./cleanup.sh", buildbot_url, WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"],
214 haltOnFailure = True,
217 factory.addStep(ShellCommand(
219 description = "Preparing SDK directory",
220 command = ["mkdir", "-p", "sdk"],
221 haltOnFailure = True))
223 factory.addStep(ShellCommand(
224 name = "downloadsdk",
225 description = "Downloading SDK archive",
226 command = ["rsync", "-va", "%s/%s/%s/%s" %(rsync_sdk_url, ts[0], ts[1], rsync_sdk_pat), "sdk.archive"],
227 env={'RSYNC_PASSWORD': rsync_sdk_key},
228 haltOnFailure = True,
231 factory.addStep(ShellCommand(
233 description = "Unpacking SDK archive",
234 command = ["tar", "--strip-components=1", "-C", "sdk/", "-vxf", "sdk.archive"],
235 haltOnFailure = True))
237 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="sdk/key-build", mode=0600))
238 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="sdk/key-build.pub", mode=0600))
240 factory.addStep(ShellCommand(
242 description = "Preparing download directory",
243 command = ["sh", "-c", "mkdir -p $HOME/dl && rm -rf ./sdk/dl && ln -sf $HOME/dl ./sdk/dl"]))
245 factory.addStep(ShellCommand(
247 description = "Preparing SDK configuration",
248 workdir = "build/sdk",
249 command = ["sh", "-c", "rm -f .config && make defconfig"]))
251 factory.addStep(ShellCommand(
252 name = "updatefeeds",
253 description = "Updating feeds",
254 workdir = "build/sdk",
255 command = ["./scripts/feeds", "update"]))
257 factory.addStep(ShellCommand(
258 name = "installfeeds",
259 description = "Installing feeds",
260 workdir = "build/sdk",
261 command = ["./scripts/feeds", "install", "-a"]))
263 factory.addStep(ShellCommand(
265 description = "Building packages",
266 workdir = "build/sdk",
267 command = ["make", WithProperties("-j%(nproc:~4)s"), "V=s", "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y"]))
269 if gpg_keyid is not None:
270 factory.addStep(MasterShellCommand(
271 name = "signprepare",
272 description = "Preparing temporary signing directory",
273 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
277 factory.addStep(ShellCommand(
279 description = "Packing files to sign",
280 workdir = "build/sdk",
281 command = "find bin/packages/%s/ -mindepth 2 -maxdepth 2 -type f -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(arch[0]),
285 factory.addStep(FileUpload(
286 slavesrc = "sdk/sign.tar.gz",
287 masterdest = "%s/signing/%s.tar.gz" %(home_dir, arch[0]),
291 factory.addStep(MasterShellCommand(
293 description = "Signing files",
294 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.tar.gz" %(home_dir, arch[0]), gpg_keyid, gpg_comment],
295 env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
299 factory.addStep(FileDownload(
300 mastersrc = "%s/signing/%s.tar.gz" %(home_dir, arch[0]),
301 slavedest = "sdk/sign.tar.gz",
305 factory.addStep(ShellCommand(
307 description = "Unpacking signed files",
308 workdir = "build/sdk",
309 command = ["tar", "-xzf", "sign.tar.gz"],
313 factory.addStep(ShellCommand(
314 name = "uploadprepare",
315 description = "Preparing package directory",
316 workdir = "build/sdk",
317 command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", WithProperties("%s/packages%%(suffix)s/" %(rsync_bin_url), suffix=GetDirectorySuffix)],
318 env={'RSYNC_PASSWORD': rsync_bin_key},
319 haltOnFailure = True,
323 factory.addStep(ShellCommand(
324 name = "packageupload",
325 description = "Uploading package files",
326 workdir = "build/sdk",
327 command = ["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "bin/packages/%s/" %(arch[0]), WithProperties("%s/packages%%(suffix)s/%s/" %(rsync_bin_url, arch[0]), suffix=GetDirectorySuffix)],
328 env={'RSYNC_PASSWORD': rsync_bin_key},
329 haltOnFailure = True,
333 factory.addStep(ShellCommand(
335 description = "Preparing log directory",
336 workdir = "build/sdk",
337 command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/faillogs/" %(rsync_bin_url)],
338 env={'RSYNC_PASSWORD': rsync_bin_key},
339 haltOnFailure = True,
343 factory.addStep(ShellCommand(
345 description = "Finding failure logs",
346 workdir = "build/sdk/logs/package/feeds",
347 command = ["sh", "-c", "sed -ne 's!^ *ERROR: package/feeds/\\([^ ]*\\) .*$!\\1!p' ../error.txt | sort -u | xargs -r find > ../../../logs.txt"],
348 haltOnFailure = False
351 factory.addStep(ShellCommand(
353 description = "Collecting failure logs",
354 workdir = "build/sdk",
355 command = ["rsync", "-av", "--files-from=logs.txt", "logs/package/feeds/", "faillogs/"],
356 haltOnFailure = False
359 factory.addStep(ShellCommand(
361 description = "Uploading failure logs",
362 workdir = "build/sdk",
363 command = ["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "faillogs/", "%s/faillogs/%s/" %(rsync_bin_url, arch[0])],
364 env={'RSYNC_PASSWORD': rsync_bin_key},
365 haltOnFailure = False,
369 if rsync_src_url is not None:
370 factory.addStep(ShellCommand(
371 name = "sourceupload",
372 description = "Uploading source archives",
373 workdir = "build/sdk",
374 command = ["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "dl/", "%s/" %(rsync_src_url)],
375 env={'RSYNC_PASSWORD': rsync_src_key},
376 haltOnFailure = False,
380 from buildbot.config import BuilderConfig
382 c['builders'].append(BuilderConfig(name=arch[0], slavenames=slaveNames, factory=factory))
385 ####### STATUS arches
387 # 'status' is a list of Status arches. The results of each build will be
388 # pushed to these arches. buildbot/status/*.py has a variety to choose from,
389 # including web pages, email senders, and IRC bots.
393 from buildbot.status import html
394 from buildbot.status.web import authz, auth
396 if ini.has_option("status", "bind"):
397 if ini.has_option("status", "user") and ini.has_option("status", "password"):
398 authz_cfg=authz.Authz(
399 # change any of these to True to enable; see the manual for more
401 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
402 gracefulShutdown = 'auth',
403 forceBuild = 'auth', # use this to test your slave once it is set up
404 forceAllBuilds = 'auth',
407 stopAllBuilds = 'auth',
408 cancelPendingBuild = 'auth',
410 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
412 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
414 ####### PROJECT IDENTITY
416 # the 'title' string will appear at the top of this buildbot
417 # installation's html.WebStatus home page (linked to the
418 # 'titleURL') and is embedded in the title of the waterfall HTML page.
420 c['title'] = ini.get("general", "title")
421 c['titleURL'] = ini.get("general", "title_url")
423 # the 'buildbotURL' string should point to the location where the buildbot's
424 # internal web server (usually the html.WebStatus page) is visible. This
425 # typically uses the port number set in the Waterfall 'status' entry, but
426 # with an externally-visible host name which the buildbot cannot figure out
429 c['buildbotURL'] = buildbot_url
434 # This specifies what database buildbot uses to store its state. You can leave
435 # this at its default for all but the largest installations.
436 'db_url' : "sqlite:///state.sqlite",