Add buildmaster configurations
[buildbot.git] / master / phase1 / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 import os
5 import re
6 import subprocess
7 import ConfigParser
8
9 from buildbot import locks
10
11 # This is a sample buildmaster config file. It must be installed as
12 # 'master.cfg' in your buildmaster's base directory.
13
14 ini = ConfigParser.ConfigParser()
15 ini.read("./config.ini")
16
17 # This is the dictionary that the buildmaster pays attention to. We also use
18 # a shorter alias to save typing.
19 c = BuildmasterConfig = {}
20
21 ####### BUILDSLAVES
22
23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
24 # a BuildSlave object, specifying a unique slave name and password. The same
25 # slave name and password must be configured on the slave.
26 from buildbot.buildslave import BuildSlave
27
28 c['slaves'] = []
29
30 for section in ini.sections():
31 if section.startswith("slave "):
32 if ini.has_option(section, "name") and ini.has_option(section, "password"):
33 name = ini.get(section, "name")
34 password = ini.get(section, "password")
35 max_builds = 1
36 if ini.has_option(section, "builds"):
37 max_builds = ini.getint(section, "builds")
38 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
39
40 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
41 # This must match the value configured into the buildslaves (with their
42 # --master option)
43 c['slavePortnum'] = 9989
44
45 # coalesce builds
46 c['mergeRequests'] = True
47
48 ####### CHANGESOURCES
49
50 home_dir = ini.get("general", "homedir")
51
52 repo_url = ini.get("repo", "url")
53
54 rsync_url = ini.get("rsync", "url")
55 rsync_key = ini.get("rsync", "password")
56
57 # find targets
58 targets = [ ]
59
60 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
61 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
62
63 while True:
64 line = findtargets.stdout.readline()
65 if not line:
66 break
67 ta = line.strip().split(' ')
68 targets.append(ta[0])
69
70
71 # the 'change_source' setting tells the buildmaster how it should find out
72 # about source code changes. Here we point to the buildbot clone of pyflakes.
73
74 from buildbot.changes.gitpoller import GitPoller
75 c['change_source'] = []
76 c['change_source'].append(GitPoller(
77 repo_url,
78 workdir=home_dir+'/source.git', branch='master',
79 pollinterval=300))
80
81 ####### SCHEDULERS
82
83 # Configure the Schedulers, which decide how to react to incoming changes. In this
84 # case, just kick off a 'basebuild' build
85
86 from buildbot.schedulers.basic import SingleBranchScheduler
87 from buildbot.schedulers.forcesched import ForceScheduler
88 from buildbot.changes import filter
89 c['schedulers'] = []
90 c['schedulers'].append(SingleBranchScheduler(
91 name="all",
92 change_filter=filter.ChangeFilter(branch='master'),
93 treeStableTimer=60,
94 builderNames=targets))
95
96 c['schedulers'].append(ForceScheduler(
97 name="force",
98 builderNames=targets))
99
100 ####### BUILDERS
101
102 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
103 # what steps, and which slaves can execute them. Note that any particular build will
104 # only take place on one slave.
105
106 from buildbot.process.factory import BuildFactory
107 from buildbot.steps.source import Git
108 from buildbot.steps.shell import ShellCommand
109 from buildbot.steps.transfer import FileDownload
110
111
112 MakeTargetMap = {
113 "^tools/": "tools/clean",
114 "^toolchain/": "toolchain/clean",
115 "^target/linux/": "target/linux/clean",
116 "^(config|include)/": "dirclean"
117 }
118
119 def IsAffected(pattern):
120 def CheckAffected(change):
121 for request in change.build.requests:
122 for source in request.sources:
123 for change in source.changes:
124 for file in change.files:
125 if re.match(pattern, file):
126 return True
127 return False
128 return CheckAffected
129
130 def isPathBuiltin(path):
131 incl = {}
132 pkgs = {}
133 conf = open(".config", "r")
134
135 while True:
136 line = conf.readline()
137 if line == '':
138 break
139 m = re.match("^(CONFIG_PACKAGE_.+?)=y", line)
140 if m:
141 incl[m.group(1)] = True
142
143 conf.close()
144
145 deps = open("tmp/.packagedeps", "r")
146
147 while True:
148 line = deps.readline()
149 if line == '':
150 break
151 m = re.match("^package-\$\((CONFIG_PACKAGE_.+?)\) \+= (\S+)", line)
152 if m and incl.get(m.group(1)) == True:
153 pkgs["package/%s" % m.group(2)] = True
154
155 deps.close()
156
157 while path != '':
158 if pkgs.get(path) == True:
159 return True
160 path = os.path.dirname(path)
161
162 return False
163
164 def isChangeBuiltin(change):
165 return True
166 # for request in change.build.requests:
167 # for source in request.sources:
168 # for change in source.changes:
169 # for file in change.files:
170 # if isPathBuiltin(file):
171 # return True
172 # return False
173
174
175 c['builders'] = []
176
177 dlLock = locks.SlaveLock("slave_dl")
178
179 checkBuiltin = re.sub('[\t\n ]+', ' ', """
180 checkBuiltin() {
181 local symbol op path file;
182 for file in $CHANGED_FILES; do
183 case "$file" in
184 package/*/*) : ;;
185 *) return 0 ;;
186 esac;
187 done;
188 while read symbol op path; do
189 case "$symbol" in package-*)
190 symbol="${symbol##*(}";
191 symbol="${symbol%)}";
192 for file in $CHANGED_FILES; do
193 case "$file" in "package/$path/"*)
194 grep -qsx "$symbol=y" .config && return 0
195 ;; esac;
196 done;
197 esac;
198 done < tmp/.packagedeps;
199 return 1;
200 }
201 """).strip()
202
203
204 class IfBuiltinShellCommand(ShellCommand):
205 def _quote(self, str):
206 if re.search("[^a-zA-Z0-9/_.-]", str):
207 return "'%s'" %(re.sub("'", "'\"'\"'", str))
208 return str
209
210 def setCommand(self, command):
211 if not isinstance(command, (str, unicode)):
212 command = ' '.join(map(self._quote, command))
213 self.command = [
214 '/bin/sh', '-c',
215 '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
216 ]
217
218 def setupEnvironment(self, cmd):
219 slaveEnv = self.slaveEnvironment
220 if slaveEnv is None:
221 slaveEnv = { }
222 changedFiles = { }
223 for request in self.build.requests:
224 for source in request.sources:
225 for change in source.changes:
226 for file in change.files:
227 changedFiles[file] = True
228 fullSlaveEnv = slaveEnv.copy()
229 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
230 cmd.args['env'] = fullSlaveEnv
231
232 slaveNames = [ ]
233
234 for slave in c['slaves']:
235 slaveNames.append(slave.slavename)
236
237 for target in targets:
238 ts = target.split('/')
239
240 factory = BuildFactory()
241
242 # check out the source
243 factory.addStep(Git(repourl=repo_url, mode='update'))
244
245 factory.addStep(ShellCommand(
246 name = "rmtmp",
247 description = "Remove tmp folder",
248 command=["rm", "-rf", "tmp/"]))
249
250 # feed
251 # factory.addStep(ShellCommand(
252 # name = "feedsconf",
253 # description = "Copy the feeds.conf",
254 # command='''cp ~/feeds.conf ./feeds.conf''' ))
255
256 # feed
257 factory.addStep(ShellCommand(
258 name = "updatefeeds",
259 description = "Updating feeds",
260 command=["./scripts/feeds", "update"]))
261
262 # feed
263 factory.addStep(ShellCommand(
264 name = "installfeeds",
265 description = "Installing feeds",
266 command=["./scripts/feeds", "install", "-a"]))
267
268 # configure
269 factory.addStep(ShellCommand(
270 name = "newconfig",
271 description = "Seeding .config",
272 command='''cat <<EOT > .config
273 CONFIG_TARGET_%s=y
274 CONFIG_TARGET_%s_%s=y
275 CONFIG_ALL_NONSHARED=y
276 CONFIG_SDK=y
277 CONFIG_IB=y
278 # CONFIG_IB_STANDALONE is not set
279 CONFIG_DEVEL=y
280 CONFIG_CCACHE=y
281 CONFIG_SIGNED_PACKAGES=y
282 # CONFIG_PER_FEED_REPO_ADD_COMMENTED is not set
283 CONFIG_KERNEL_KALLSYMS=y
284 CONFIG_COLLECT_KERNEL_DEBUG=y
285 EOT''' %(ts[0], ts[0], ts[1]) ))
286
287 factory.addStep(ShellCommand(
288 name = "delbin",
289 description = "Removing output directory",
290 command = ["rm", "-rf", "bin/"]
291 ))
292
293 factory.addStep(ShellCommand(
294 name = "defconfig",
295 description = "Populating .config",
296 command = ["make", "defconfig"]
297 ))
298
299 # check arch
300 factory.addStep(ShellCommand(
301 name = "checkarch",
302 description = "Checking architecture",
303 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
304 logEnviron = False,
305 want_stdout = False,
306 want_stderr = False,
307 haltOnFailure = True
308 ))
309
310 # install build key
311 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
312
313 # prepare dl
314 factory.addStep(ShellCommand(
315 name = "dldir",
316 description = "Preparing dl/",
317 command = "mkdir -p $HOME/dl && ln -sf $HOME/dl ./dl",
318 logEnviron = False,
319 want_stdout = False
320 ))
321
322 # populate dl
323 factory.addStep(ShellCommand(
324 name = "dlrun",
325 description = "Populating dl/",
326 command = ["make", "-j4", "download", "V=s"],
327 logEnviron = False,
328 locks = [dlLock.access('exclusive')]
329 ))
330
331 factory.addStep(ShellCommand(
332 name = "cleanbase",
333 description = "Cleaning base-files",
334 command=["make", "package/base-files/clean", "V=s"]
335 ))
336
337 # optional clean steps
338 for pattern, maketarget in MakeTargetMap.items():
339 factory.addStep(ShellCommand(
340 name = maketarget,
341 description = maketarget,
342 command=["make", maketarget, "V=s"], doStepIf=IsAffected(pattern)
343 ))
344
345 # build
346 factory.addStep(ShellCommand(
347 name = "tools",
348 description = "Building tools",
349 command = ["make", "-j4", "tools/install", "V=s"],
350 haltOnFailure = True
351 ))
352
353 factory.addStep(ShellCommand(
354 name = "toolchain",
355 description = "Building toolchain",
356 command=["make", "-j4", "toolchain/install", "V=s"],
357 haltOnFailure = True
358 ))
359
360 factory.addStep(ShellCommand(
361 name = "kmods",
362 description = "Building kmods",
363 command=["make", "-j4", "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
364 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
365 haltOnFailure = True
366 ))
367
368 factory.addStep(ShellCommand(
369 name = "pkgbuild",
370 description = "Building packages",
371 command=["make", "-j4", "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
372 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
373 haltOnFailure = True
374 ))
375
376 # factory.addStep(IfBuiltinShellCommand(
377 factory.addStep(ShellCommand(
378 name = "pkginstall",
379 description = "Installing packages",
380 command=["make", "-j4", "package/install", "V=s"],
381 doStepIf = isChangeBuiltin,
382 haltOnFailure = True
383 ))
384
385 factory.addStep(ShellCommand(
386 name = "pkgindex",
387 description = "Indexing packages",
388 command=["make", "-j4", "package/index", "V=s"],
389 haltOnFailure = True
390 ))
391
392 #factory.addStep(IfBuiltinShellCommand(
393 factory.addStep(ShellCommand(
394 name = "images",
395 description = "Building images",
396 command=["make", "-j1", "target/install", "V=s"],
397 doStepIf = isChangeBuiltin,
398 haltOnFailure = True
399 ))
400
401 # upload
402 factory.addStep(ShellCommand(
403 name = "uploadprepare",
404 description = "Preparing target directory",
405 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_url)],
406 env={'RSYNC_PASSWORD': rsync_key},
407 haltOnFailure = True,
408 logEnviron = False
409 ))
410
411 factory.addStep(ShellCommand(
412 name = "targetupload",
413 description = "Uploading target files",
414 command=["rsync", "--delete", "-avz", "bin/targets/%s/%s/" %(ts[0], ts[1]), "%s/targets/%s/%s/" %(rsync_url, ts[0], ts[1])],
415 env={'RSYNC_PASSWORD': rsync_key},
416 haltOnFailure = True,
417 logEnviron = False
418 ))
419
420 if False:
421 factory.addStep(ShellCommand(
422 name = "packageupload",
423 description = "Uploading package files",
424 command=["rsync", "--delete", "-avz", "bin/packages/", "%s/packages/" %(rsync_url)],
425 env={'RSYNC_PASSWORD': rsync_key},
426 haltOnFailure = False,
427 logEnviron = False
428 ))
429
430 # logs
431 if False:
432 factory.addStep(ShellCommand(
433 name = "upload",
434 description = "Uploading logs",
435 command=["rsync", "--delete", "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_url, ts[0], ts[1])],
436 env={'RSYNC_PASSWORD': rsync_key},
437 haltOnFailure = False,
438 alwaysRun = True,
439 logEnviron = False
440 ))
441
442 from buildbot.config import BuilderConfig
443
444 c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
445
446
447 ####### STATUS TARGETS
448
449 # 'status' is a list of Status Targets. The results of each build will be
450 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
451 # including web pages, email senders, and IRC bots.
452
453 c['status'] = []
454
455 from buildbot.status import html
456 from buildbot.status.web import authz, auth
457
458 if ini.has_option("status", "bind"):
459 if ini.has_option("status", "user") and ini.has_option("status", "password"):
460 authz_cfg=authz.Authz(
461 # change any of these to True to enable; see the manual for more
462 # options
463 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
464 gracefulShutdown = False,
465 forceBuild = 'auth', # use this to test your slave once it is set up
466 forceAllBuilds = 'auth',
467 pingBuilder = False,
468 stopBuild = 'auth',
469 stopAllBuilds = 'auth',
470 cancelPendingBuild = 'auth',
471 )
472 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
473 else:
474 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
475
476 ####### PROJECT IDENTITY
477
478 # the 'title' string will appear at the top of this buildbot
479 # installation's html.WebStatus home page (linked to the
480 # 'titleURL') and is embedded in the title of the waterfall HTML page.
481
482 c['title'] = ini.get("general", "title")
483 c['titleURL'] = ini.get("general", "title_url")
484
485 # the 'buildbotURL' string should point to the location where the buildbot's
486 # internal web server (usually the html.WebStatus page) is visible. This
487 # typically uses the port number set in the Waterfall 'status' entry, but
488 # with an externally-visible host name which the buildbot cannot figure out
489 # without some help.
490
491 c['buildbotURL'] = ini.get("general", "buildbot_url")
492
493 ####### DB URL
494
495 c['db'] = {
496 # This specifies what database buildbot uses to store its state. You can leave
497 # this at its default for all but the largest installations.
498 'db_url' : "sqlite:///state.sqlite",
499 }
500