2 # ex: set syntax=python:
9 from buildbot import locks
11 ini = ConfigParser.ConfigParser()
12 ini.read("./config.ini")
14 # This is a sample buildmaster config file. It must be installed as
15 # 'master.cfg' in your buildmaster's base directory.
17 # This is the dictionary that the buildmaster pays attention to. We also use
18 # a shorter alias to save typing.
19 c = BuildmasterConfig = {}
23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
24 # a BuildSlave object, specifying a unique slave name and password. The same
25 # slave name and password must be configured on the slave.
26 from buildbot.buildslave import BuildSlave
30 for section in ini.sections():
31 if section.startswith("slave "):
32 if ini.has_option(section, "name") and ini.has_option(section, "password"):
33 name = ini.get(section, "name")
34 password = ini.get(section, "password")
36 if ini.has_option(section, "builds"):
37 max_builds = ini.getint(section, "builds")
38 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
40 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
41 # This must match the value configured into the buildslaves (with their
43 c['slavePortnum'] = 9990
46 c['mergeRequests'] = True
50 home_dir = os.path.abspath(ini.get("general", "homedir"))
52 repo_url = ini.get("repo", "url")
54 rsync_bin_url = ini.get("rsync", "binary_url")
55 rsync_bin_key = ini.get("rsync", "binary_password")
60 if ini.has_option("rsync", "source_url"):
61 rsync_src_url = ini.get("rsync", "source_url")
62 rsync_src_key = ini.get("rsync", "source_password")
65 gpg_comment = "Unattended build signature"
66 gpg_passfile = "/dev/null"
68 if ini.has_option("gpg", "keyid"):
69 gpg_keyid = ini.get("gpg", "keyid")
71 if ini.has_option("gpg", "comment"):
72 gpg_comment = ini.get("gpg", "comment")
74 if ini.has_option("gpg", "passfile"):
75 gpg_passfile = ini.get("gpg", "passfile")
82 findarches = subprocess.Popen([home_dir+'/dumpinfo.pl', 'architectures'],
83 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
86 line = findarches.stdout.readline()
89 at = line.strip().split()
91 archnames.append(at[0])
97 from buildbot.changes.gitpoller import GitPoller
98 c['change_source'] = []
100 with open(home_dir+'/source.git/feeds.conf.default', 'r') as f:
102 parts = line.strip().split()
103 if parts[0] == "src-git":
105 c['change_source'].append(GitPoller(parts[2], workdir='%s/%s.git' %(os.getcwd(), parts[1]), branch='master', pollinterval=300))
110 # Configure the Schedulers, which decide how to react to incoming changes. In this
111 # case, just kick off a 'basebuild' build
113 from buildbot.schedulers.basic import SingleBranchScheduler
114 from buildbot.schedulers.forcesched import ForceScheduler
115 from buildbot.changes import filter
117 c['schedulers'].append(SingleBranchScheduler(
119 change_filter=filter.ChangeFilter(branch='master'),
121 builderNames=archnames))
123 c['schedulers'].append(ForceScheduler(
125 builderNames=archnames))
129 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
130 # what steps, and which slaves can execute them. Note that any particular build will
131 # only take place on one slave.
133 from buildbot.process.factory import BuildFactory
134 from buildbot.steps.source import Git
135 from buildbot.steps.shell import ShellCommand
136 from buildbot.steps.shell import SetProperty
137 from buildbot.steps.transfer import FileUpload
138 from buildbot.steps.transfer import FileDownload
139 from buildbot.steps.master import MasterShellCommand
140 from buildbot.process.properties import WithProperties
144 dlLock = locks.SlaveLock("slave_dl")
148 for slave in c['slaves']:
149 slaveNames.append(slave.slavename)
152 ts = arch[1].split('/')
154 factory = BuildFactory()
156 # find number of cores
157 factory.addStep(SetProperty(
160 description = "Finding number of CPUs",
161 command = ["nproc"]))
164 factory.addStep(FileDownload(mastersrc="cleanup.sh", slavedest="cleanup.sh", mode=0755))
166 factory.addStep(ShellCommand(
168 description = "Cleaning previous builds",
169 command = ["./cleanup.sh", WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"],
170 haltOnFailure = True,
173 factory.addStep(ShellCommand(
175 description = "Cleaning work area",
176 command = ["./cleanup.sh", WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"],
177 haltOnFailure = True,
180 factory.addStep(ShellCommand(
182 description = "Preparing SDK directory",
183 command = ["mkdir", "sdk"],
184 haltOnFailure = True))
186 factory.addStep(ShellCommand(
187 name = "downloadsdk",
188 description = "Downloading SDK archive",
189 command = ["rsync", "-va", "downloads.lede-project.org::downloads/snapshots/targets/%s/%s/lede-sdk-*.tar.bz2" %(ts[0], ts[1]), "sdk.tar.bz2"],
190 haltOnFailure = True))
192 factory.addStep(ShellCommand(
194 description = "Unpacking SDK archive",
195 command = ["tar", "--strip-components=1", "-C", "sdk/", "-vxjf", "sdk.tar.bz2"],
196 haltOnFailure = True))
198 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="sdk/key-build", mode=0600))
199 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="sdk/key-build.pub", mode=0600))
201 factory.addStep(ShellCommand(
203 description = "Preparing download directory",
204 command = ["sh", "-c", "mkdir -p $HOME/dl && rmdir ./sdk/dl && ln -sf $HOME/dl ./sdk/dl"]))
206 factory.addStep(ShellCommand(
208 description = "Preparing SDK configuration",
209 workdir = "build/sdk",
210 command = ["sh", "-c", "rm -f .config && make defconfig"]))
212 factory.addStep(ShellCommand(
213 name = "updatefeeds",
214 description = "Updating feeds",
215 workdir = "build/sdk",
216 command = ["./scripts/feeds", "update"]))
218 factory.addStep(ShellCommand(
219 name = "installfeeds",
220 description = "Installing feeds",
221 workdir = "build/sdk",
222 command = ["./scripts/feeds", "install", "-a"]))
224 factory.addStep(ShellCommand(
226 description = "Building packages",
227 workdir = "build/sdk",
228 command = ["make", WithProperties("-j%(nproc:~4)s"), "V=s", "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y"]))
230 if gpg_keyid is not None:
231 factory.addStep(MasterShellCommand(
232 name = "signprepare",
233 description = "Preparing temporary signing directory",
234 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
238 factory.addStep(ShellCommand(
240 description = "Packing files to sign",
241 workdir = "build/sdk",
242 command = "find bin/packages/%s/ -mindepth 2 -maxdepth 2 -type f -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(arch[0]),
246 factory.addStep(FileUpload(
247 slavesrc = "sdk/sign.tar.gz",
248 masterdest = "%s/signing/%s.tar.gz" %(home_dir, arch[0]),
252 factory.addStep(MasterShellCommand(
254 description = "Signing files",
255 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.tar.gz" %(home_dir, arch[0]), gpg_keyid, gpg_passfile, gpg_comment],
259 factory.addStep(FileDownload(
260 mastersrc = "%s/signing/%s.tar.gz" %(home_dir, arch[0]),
261 slavedest = "sdk/sign.tar.gz",
265 factory.addStep(ShellCommand(
267 description = "Unpacking signed files",
268 workdir = "build/sdk",
269 command = ["tar", "-xzf", "sign.tar.gz"],
273 factory.addStep(ShellCommand(
274 name = "uploadprepare",
275 description = "Preparing package directory",
276 workdir = "build/sdk",
277 command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/packages/" %(rsync_bin_url)],
278 env={'RSYNC_PASSWORD': rsync_bin_key},
279 haltOnFailure = True,
283 factory.addStep(ShellCommand(
284 name = "packageupload",
285 description = "Uploading package files",
286 workdir = "build/sdk",
287 command = ["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "bin/packages/%s/" %(arch[0]), "%s/packages/%s/" %(rsync_bin_url, arch[0])],
288 env={'RSYNC_PASSWORD': rsync_bin_key},
289 haltOnFailure = True,
293 factory.addStep(ShellCommand(
295 description = "Preparing log directory",
296 workdir = "build/sdk",
297 command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/faillogs/" %(rsync_bin_url)],
298 env={'RSYNC_PASSWORD': rsync_bin_key},
299 haltOnFailure = True,
303 factory.addStep(ShellCommand(
305 description = "Finding failure logs",
306 workdir = "build/sdk/logs/package/feeds",
307 command = ["sh", "-c", "sed -ne 's!^ *ERROR: package/feeds/\\([^ ]*\\) .*$!\\1!p' ../error.txt | sort -u | xargs -r find > ../../../logs.txt"],
308 haltOnFailure = False
311 factory.addStep(ShellCommand(
313 description = "Collecting failure logs",
314 workdir = "build/sdk",
315 command = ["rsync", "-av", "--files-from=logs.txt", "logs/package/feeds/", "faillogs/"],
316 haltOnFailure = False
319 factory.addStep(ShellCommand(
321 description = "Uploading failure logs",
322 workdir = "build/sdk",
323 command = ["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "faillogs/", "%s/faillogs/%s/" %(rsync_bin_url, arch[0])],
324 env={'RSYNC_PASSWORD': rsync_bin_key},
325 haltOnFailure = False,
329 if rsync_src_url is not None:
330 factory.addStep(ShellCommand(
331 name = "sourceupload",
332 description = "Uploading source archives",
333 workdir = "build/sdk",
334 command = ["rsync", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "dl/", "%s/" %(rsync_src_url)],
335 env={'RSYNC_PASSWORD': rsync_src_key},
336 haltOnFailure = False,
340 from buildbot.config import BuilderConfig
342 c['builders'].append(BuilderConfig(name=arch[0], slavenames=slaveNames, factory=factory))
345 ####### STATUS arches
347 # 'status' is a list of Status arches. The results of each build will be
348 # pushed to these arches. buildbot/status/*.py has a variety to choose from,
349 # including web pages, email senders, and IRC bots.
353 from buildbot.status import html
354 from buildbot.status.web import authz, auth
356 if ini.has_option("status", "bind"):
357 if ini.has_option("status", "user") and ini.has_option("status", "password"):
358 authz_cfg=authz.Authz(
359 # change any of these to True to enable; see the manual for more
361 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
362 gracefulShutdown = 'auth',
363 forceBuild = 'auth', # use this to test your slave once it is set up
364 forceAllBuilds = 'auth',
367 stopAllBuilds = 'auth',
368 cancelPendingBuild = 'auth',
370 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
372 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
374 ####### PROJECT IDENTITY
376 # the 'title' string will appear at the top of this buildbot
377 # installation's html.WebStatus home page (linked to the
378 # 'titleURL') and is embedded in the title of the waterfall HTML page.
380 c['title'] = ini.get("general", "title")
381 c['titleURL'] = ini.get("general", "title_url")
383 # the 'buildbotURL' string should point to the location where the buildbot's
384 # internal web server (usually the html.WebStatus page) is visible. This
385 # typically uses the port number set in the Waterfall 'status' entry, but
386 # with an externally-visible host name which the buildbot cannot figure out
389 c['buildbotURL'] = ini.get("general", "buildbot_url")
394 # This specifies what database buildbot uses to store its state. You can leave
395 # this at its default for all but the largest installations.
396 'db_url' : "sqlite:///state.sqlite",