2 # ex: set syntax=python:
9 from buildbot import locks
11 ini = ConfigParser.ConfigParser()
12 ini.read("./config.ini")
14 buildbot_url = ini.get("general", "buildbot_url")
16 # This is a sample buildmaster config file. It must be installed as
17 # 'master.cfg' in your buildmaster's base directory.
19 # This is the dictionary that the buildmaster pays attention to. We also use
20 # a shorter alias to save typing.
21 c = BuildmasterConfig = {}
25 # The 'slaves' list defines the set of recognized buildslaves. Each element is
26 # a BuildSlave object, specifying a unique slave name and password. The same
27 # slave name and password must be configured on the slave.
28 from buildbot.buildslave import BuildSlave
32 if ini.has_option("general", "port"):
33 slave_port = ini.getint("general", "port")
37 for section in ini.sections():
38 if section.startswith("slave "):
39 if ini.has_option(section, "name") and ini.has_option(section, "password"):
40 name = ini.get(section, "name")
41 password = ini.get(section, "password")
43 if ini.has_option(section, "builds"):
44 max_builds = ini.getint(section, "builds")
45 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
47 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
48 # This must match the value configured into the buildslaves (with their
50 c['slavePortnum'] = slave_port
53 c['mergeRequests'] = True
57 home_dir = os.path.abspath(ini.get("general", "homedir"))
59 rsync_bin_url = ini.get("rsync", "binary_url")
60 rsync_bin_key = ini.get("rsync", "binary_password")
65 if ini.has_option("rsync", "source_url"):
66 rsync_src_url = ini.get("rsync", "source_url")
67 rsync_src_key = ini.get("rsync", "source_password")
71 rsync_sdk_pat = "lede-sdk-*.tar.xz"
73 if ini.has_option("rsync", "sdk_url"):
74 rsync_sdk_url = ini.get("rsync", "sdk_url")
76 if ini.has_option("rsync", "sdk_password"):
77 rsync_sdk_key = ini.get("rsync", "sdk_password")
79 if ini.has_option("rsync", "sdk_pattern"):
80 rsync_sdk_pat = ini.get("rsync", "sdk_pattern")
83 gpg_comment = "Unattended build signature"
84 gpg_passfile = "/dev/null"
86 if ini.has_option("gpg", "keyid"):
87 gpg_keyid = ini.get("gpg", "keyid")
89 if ini.has_option("gpg", "comment"):
90 gpg_comment = ini.get("gpg", "comment")
92 if ini.has_option("gpg", "passfile"):
93 gpg_passfile = ini.get("gpg", "passfile")
100 findarches = subprocess.Popen([home_dir+'/dumpinfo.pl', 'architectures'],
101 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
104 line = findarches.stdout.readline()
107 at = line.strip().split()
109 archnames.append(at[0])
115 from buildbot.changes.gitpoller import GitPoller
116 c['change_source'] = []
118 with open(home_dir+'/source.git/feeds.conf.default', 'r') as f:
120 parts = line.strip().split()
121 if parts[0] == "src-git":
123 url = parts[2].strip().split(';')
124 branch = url[1] if len(url) > 1 else 'master'
125 c['change_source'].append(GitPoller(url[0], branch=branch, workdir='%s/%s.git' %(os.getcwd(), parts[1]), branch='master', pollinterval=300))
130 # Configure the Schedulers, which decide how to react to incoming changes. In this
131 # case, just kick off a 'basebuild' build
133 from buildbot.schedulers.basic import SingleBranchScheduler
134 from buildbot.schedulers.forcesched import ForceScheduler
135 from buildbot.changes import filter
137 c['schedulers'].append(SingleBranchScheduler(
139 change_filter=filter.ChangeFilter(branch='master'),
141 builderNames=archnames))
143 c['schedulers'].append(ForceScheduler(
145 builderNames=archnames))
149 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
150 # what steps, and which slaves can execute them. Note that any particular build will
151 # only take place on one slave.
153 from buildbot.process.factory import BuildFactory
154 from buildbot.steps.source import Git
155 from buildbot.steps.shell import ShellCommand
156 from buildbot.steps.shell import SetProperty
157 from buildbot.steps.transfer import FileUpload
158 from buildbot.steps.transfer import FileDownload
159 from buildbot.steps.master import MasterShellCommand
160 from buildbot.process.properties import WithProperties
164 dlLock = locks.SlaveLock("slave_dl")
168 for slave in c['slaves']:
169 slaveNames.append(slave.slavename)
172 ts = arch[1].split('/')
174 factory = BuildFactory()
176 # find number of cores
177 factory.addStep(SetProperty(
180 description = "Finding number of CPUs",
181 command = ["nproc"]))
184 factory.addStep(FileDownload(mastersrc="cleanup.sh", slavedest="cleanup.sh", mode=0755))
186 factory.addStep(ShellCommand(
188 description = "Cleaning previous builds",
189 command = ["./cleanup.sh", buildbot_url, WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"],
190 haltOnFailure = True,
193 factory.addStep(ShellCommand(
195 description = "Cleaning work area",
196 command = ["./cleanup.sh", buildbot_url, WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"],
197 haltOnFailure = True,
200 factory.addStep(ShellCommand(
202 description = "Preparing SDK directory",
203 command = ["mkdir", "sdk"],
204 haltOnFailure = True))
206 factory.addStep(ShellCommand(
207 name = "downloadsdk",
208 description = "Downloading SDK archive",
209 command = ["rsync", "-va", "%s/%s/%s/%s" %(rsync_sdk_url, ts[0], ts[1], rsync_sdk_pat), "sdk.archive"],
210 env={'RSYNC_PASSWORD': rsync_sdk_key},
211 haltOnFailure = True,
214 factory.addStep(ShellCommand(
216 description = "Unpacking SDK archive",
217 command = ["tar", "--strip-components=1", "-C", "sdk/", "-vxf", "sdk.archive"],
218 haltOnFailure = True))
220 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="sdk/key-build", mode=0600))
221 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="sdk/key-build.pub", mode=0600))
223 factory.addStep(ShellCommand(
225 description = "Preparing download directory",
226 command = ["sh", "-c", "mkdir -p $HOME/dl && rm -rf ./sdk/dl && ln -sf $HOME/dl ./sdk/dl"]))
228 factory.addStep(ShellCommand(
230 description = "Preparing SDK configuration",
231 workdir = "build/sdk",
232 command = ["sh", "-c", "rm -f .config && make defconfig"]))
234 factory.addStep(ShellCommand(
235 name = "updatefeeds",
236 description = "Updating feeds",
237 workdir = "build/sdk",
238 command = ["./scripts/feeds", "update"]))
240 factory.addStep(ShellCommand(
241 name = "installfeeds",
242 description = "Installing feeds",
243 workdir = "build/sdk",
244 command = ["./scripts/feeds", "install", "-a"]))
246 factory.addStep(ShellCommand(
248 description = "Building packages",
249 workdir = "build/sdk",
250 command = ["make", WithProperties("-j%(nproc:~4)s"), "V=s", "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y"]))
252 if gpg_keyid is not None:
253 factory.addStep(MasterShellCommand(
254 name = "signprepare",
255 description = "Preparing temporary signing directory",
256 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
260 factory.addStep(ShellCommand(
262 description = "Packing files to sign",
263 workdir = "build/sdk",
264 command = "find bin/packages/%s/ -mindepth 2 -maxdepth 2 -type f -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(arch[0]),
268 factory.addStep(FileUpload(
269 slavesrc = "sdk/sign.tar.gz",
270 masterdest = "%s/signing/%s.tar.gz" %(home_dir, arch[0]),
274 factory.addStep(MasterShellCommand(
276 description = "Signing files",
277 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.tar.gz" %(home_dir, arch[0]), gpg_keyid, gpg_passfile, gpg_comment],
281 factory.addStep(FileDownload(
282 mastersrc = "%s/signing/%s.tar.gz" %(home_dir, arch[0]),
283 slavedest = "sdk/sign.tar.gz",
287 factory.addStep(ShellCommand(
289 description = "Unpacking signed files",
290 workdir = "build/sdk",
291 command = ["tar", "-xzf", "sign.tar.gz"],
295 factory.addStep(ShellCommand(
296 name = "uploadprepare",
297 description = "Preparing package directory",
298 workdir = "build/sdk",
299 command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/packages/" %(rsync_bin_url)],
300 env={'RSYNC_PASSWORD': rsync_bin_key},
301 haltOnFailure = True,
305 factory.addStep(ShellCommand(
306 name = "packageupload",
307 description = "Uploading package files",
308 workdir = "build/sdk",
309 command = ["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "bin/packages/%s/" %(arch[0]), "%s/packages/%s/" %(rsync_bin_url, arch[0])],
310 env={'RSYNC_PASSWORD': rsync_bin_key},
311 haltOnFailure = True,
315 factory.addStep(ShellCommand(
317 description = "Preparing log directory",
318 workdir = "build/sdk",
319 command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/faillogs/" %(rsync_bin_url)],
320 env={'RSYNC_PASSWORD': rsync_bin_key},
321 haltOnFailure = True,
325 factory.addStep(ShellCommand(
327 description = "Finding failure logs",
328 workdir = "build/sdk/logs/package/feeds",
329 command = ["sh", "-c", "sed -ne 's!^ *ERROR: package/feeds/\\([^ ]*\\) .*$!\\1!p' ../error.txt | sort -u | xargs -r find > ../../../logs.txt"],
330 haltOnFailure = False
333 factory.addStep(ShellCommand(
335 description = "Collecting failure logs",
336 workdir = "build/sdk",
337 command = ["rsync", "-av", "--files-from=logs.txt", "logs/package/feeds/", "faillogs/"],
338 haltOnFailure = False
341 factory.addStep(ShellCommand(
343 description = "Uploading failure logs",
344 workdir = "build/sdk",
345 command = ["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "faillogs/", "%s/faillogs/%s/" %(rsync_bin_url, arch[0])],
346 env={'RSYNC_PASSWORD': rsync_bin_key},
347 haltOnFailure = False,
351 if rsync_src_url is not None:
352 factory.addStep(ShellCommand(
353 name = "sourceupload",
354 description = "Uploading source archives",
355 workdir = "build/sdk",
356 command = ["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "dl/", "%s/" %(rsync_src_url)],
357 env={'RSYNC_PASSWORD': rsync_src_key},
358 haltOnFailure = False,
362 from buildbot.config import BuilderConfig
364 c['builders'].append(BuilderConfig(name=arch[0], slavenames=slaveNames, factory=factory))
367 ####### STATUS arches
369 # 'status' is a list of Status arches. The results of each build will be
370 # pushed to these arches. buildbot/status/*.py has a variety to choose from,
371 # including web pages, email senders, and IRC bots.
375 from buildbot.status import html
376 from buildbot.status.web import authz, auth
378 if ini.has_option("status", "bind"):
379 if ini.has_option("status", "user") and ini.has_option("status", "password"):
380 authz_cfg=authz.Authz(
381 # change any of these to True to enable; see the manual for more
383 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
384 gracefulShutdown = 'auth',
385 forceBuild = 'auth', # use this to test your slave once it is set up
386 forceAllBuilds = 'auth',
389 stopAllBuilds = 'auth',
390 cancelPendingBuild = 'auth',
392 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
394 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
396 ####### PROJECT IDENTITY
398 # the 'title' string will appear at the top of this buildbot
399 # installation's html.WebStatus home page (linked to the
400 # 'titleURL') and is embedded in the title of the waterfall HTML page.
402 c['title'] = ini.get("general", "title")
403 c['titleURL'] = ini.get("general", "title_url")
405 # the 'buildbotURL' string should point to the location where the buildbot's
406 # internal web server (usually the html.WebStatus page) is visible. This
407 # typically uses the port number set in the Waterfall 'status' entry, but
408 # with an externally-visible host name which the buildbot cannot figure out
411 c['buildbotURL'] = buildbot_url
416 # This specifies what database buildbot uses to store its state. You can leave
417 # this at its default for all but the largest installations.
418 'db_url' : "sqlite:///state.sqlite",