# 'master.cfg' in your buildmaster's base directory.
ini = ConfigParser.ConfigParser()
-ini.read("./config.ini")
+ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
# with an externally-visible host name which the buildbot cannot figure out
# without some help.
-c['buildbotURL'] = ini.get("general", "buildbot_url")
+c['buildbotURL'] = ini.get("phase1", "buildbot_url")
####### BUILDSLAVES
slave_port = 9989
-if ini.has_option("general", "port"):
- slave_port = ini.getint("general", "port")
+if ini.has_option("phase1", "port"):
+ slave_port = ini.getint("phase1", "port")
c['slaves'] = []
NetLocks = dict()
for section in ini.sections():
if section.startswith("slave "):
- if ini.has_option(section, "name") and ini.has_option(section, "password"):
+ if ini.has_option(section, "name") and ini.has_option(section, "password") and \
+ (not ini.has_option(section, "phase") or ini.getint(section, "phase") == 1):
sl_props = { 'dl_lock':None, 'ul_lock':None, 'do_cleanup':False, 'max_builds':1, 'shared_wd':False }
name = ini.get(section, "name")
password = ini.get(section, "password")
if ini.has_option(section, "builds"):
max_builds = ini.getint(section, "builds")
sl_props['max_builds'] = max_builds
+ if max_builds == 1:
+ sl_props['shared_wd'] = True
if ini.has_option(section, "cleanup"):
sl_props['do_cleanup'] = ini.getboolean(section, "cleanup")
if ini.has_option(section, "dl_lock"):
####### CHANGESOURCES
-home_dir = os.path.abspath(ini.get("general", "homedir"))
+work_dir = os.path.abspath(ini.get("general", "workdir") or ".")
scripts_dir = os.path.abspath("../scripts")
tree_expire = 0
other_builds = 0
cc_command = "gcc"
cxx_command = "g++"
+config_seed = ""
+
git_ssh = False
git_ssh_key = None
-if ini.has_option("general", "expire"):
- tree_expire = ini.getint("general", "expire")
+if ini.has_option("phase1", "expire"):
+ tree_expire = ini.getint("phase1", "expire")
-if ini.has_option("general", "other_builds"):
- other_builds = ini.getint("general", "other_builds")
+if ini.has_option("phase1", "other_builds"):
+ other_builds = ini.getint("phase1", "other_builds")
-if ini.has_option("general", "cc_version"):
- cc_version = ini.get("general", "cc_version").split()
+if ini.has_option("phase1", "cc_version"):
+ cc_version = ini.get("phase1", "cc_version").split()
if len(cc_version) == 1:
cc_version = ["eq", cc_version[0]]
else:
git_ssh = False
+if ini.has_option("phase1", "config_seed"):
+ config_seed = ini.get("phase1", "config_seed")
+
repo_url = ini.get("repo", "url")
repo_branch = "master"
rsync_bin_url = ini.get("rsync", "binary_url")
rsync_bin_key = ini.get("rsync", "binary_password")
+rsync_bin_defopts = ["-v", "-4", "--timeout=120"]
+
+if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
+ rsync_bin_defopts += ["--contimeout=20"]
rsync_src_url = None
rsync_src_key = None
+rsync_src_defopts = ["-v", "-4", "--timeout=120"]
if ini.has_option("rsync", "source_url"):
rsync_src_url = ini.get("rsync", "source_url")
rsync_src_key = ini.get("rsync", "source_password")
-rsync_defopts = ["-4", "-v", "--timeout=120", "--contimeout=20"]
-
-gpg_key = None
-gpg_passphrase = None
-gpg_comment = repo_branch.replace("-", " ").title() + " key"
-
-if ini.has_option("gpg", "key"):
- gpg_key = ini.get("gpg", "key")
-
-if ini.has_option("gpg", "passphrase"):
- gpg_passphrase = ini.get("gpg", "passphrase")
-
-if ini.has_option("gpg", "comment"):
- gpg_comment = ini.get("gpg", "comment")
+ if rsync_src_url.find("::") > 0 or rsync_src_url.find("rsync://") == 0:
+ rsync_src_defopts += ["--contimeout=20"]
usign_key = None
usign_comment = "untrusted comment: " + repo_branch.replace("-", " ").title() + " key"
# find targets
targets = [ ]
-if not os.path.isdir(home_dir+'/source.git'):
- subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, home_dir+'/source.git'])
+if not os.path.isdir(work_dir+'/source.git'):
+ subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, work_dir+'/source.git'])
else:
- subprocess.call(["git", "pull"], cwd = home_dir+'/source.git')
+ subprocess.call(["git", "pull"], cwd = work_dir+'/source.git')
findtargets = subprocess.Popen([scripts_dir + '/dumpinfo.pl', 'targets'],
- stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
+ stdout = subprocess.PIPE, cwd = work_dir+'/source.git')
while True:
line = findtargets.stdout.readline()
c['change_source'] = []
c['change_source'].append(GitPoller(
repo_url,
- workdir=home_dir+'/work.git', branch=repo_branch,
+ workdir=work_dir+'/work.git', branch=repo_branch,
pollinterval=300))
####### SCHEDULERS
return CheckCleanProperty
+def IsSharedWorkdir(step):
+ return bool(step.getProperty("shared_wd"))
+
def IsCleanupRequested(step):
- shared_wd = step.getProperty("shared_wd")
- if shared_wd:
+ if IsSharedWorkdir(step):
return False
do_cleanup = step.getProperty("do_cleanup")
if do_cleanup:
return False
def IsExpireRequested(step):
- shared_wd = step.getProperty("shared_wd")
- if shared_wd:
+ if IsSharedWorkdir(step):
return False
else:
return not IsCleanupRequested(step)
factory = BuildFactory()
+ # setup shared work directory if required
+ factory.addStep(ShellCommand(
+ name = "sharedwd",
+ description = "Setting up shared work directory",
+ command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
+ workdir = ".",
+ haltOnFailure = True,
+ doStepIf = IsSharedWorkdir))
+
# find number of cores
factory.addStep(SetPropertyFromCommand(
name = "nproc",
command = ["nproc"]))
# find gcc and g++ compilers
- if cc_version is not None:
- factory.addStep(FileDownload(
- name = "dlfindbinpl",
- mastersrc = scripts_dir + '/findbin.pl',
- slavedest = "../findbin.pl",
- mode = 0755))
+ factory.addStep(FileDownload(
+ name = "dlfindbinpl",
+ mastersrc = scripts_dir + '/findbin.pl',
+ slavedest = "../findbin.pl",
+ mode = 0755))
- factory.addStep(SetPropertyFromCommand(
- name = "gcc",
- property = "cc_command",
- description = "Finding gcc command",
- command = ["../findbin.pl", "gcc", cc_version[0], cc_version[1]],
- haltOnFailure = True))
+ factory.addStep(SetPropertyFromCommand(
+ name = "gcc",
+ property = "cc_command",
+ description = "Finding gcc command",
+ command = [
+ "../findbin.pl", "gcc",
+ cc_version[0] if cc_version is not None else '',
+ cc_version[1] if cc_version is not None else ''
+ ],
+ haltOnFailure = True))
- factory.addStep(SetPropertyFromCommand(
- name = "g++",
- property = "cxx_command",
- description = "Finding g++ command",
- command = ["../findbin.pl", "g++", cc_version[0], cc_version[1]],
- haltOnFailure = True))
+ factory.addStep(SetPropertyFromCommand(
+ name = "g++",
+ property = "cxx_command",
+ description = "Finding g++ command",
+ command = [
+ "../findbin.pl", "g++",
+ cc_version[0] if cc_version is not None else '',
+ cc_version[1] if cc_version is not None else ''
+ ],
+ haltOnFailure = True))
# see if ccache is available
factory.addStep(SetPropertyFromCommand(
# cleanup.sh if needed
factory.addStep(FileDownload(
- name = "dlcleanupsh",
- mastersrc = "cleanup.sh",
+ name = "dlcleanupsh",
+ mastersrc = scripts_dir + '/cleanup.sh',
slavedest = "../cleanup.sh",
mode = 0755,
doStepIf = IsCleanupRequested))
factory.addStep(ShellCommand(
name = "gitcheckout",
description = "Ensure that Git HEAD is sane",
- command = "if [ -d .git ]; then git checkout master; else exit 0; fi",
+ command = "if [ -d .git ]; then git checkout -f %s; git branch --set-upstream-to origin/%s; else exit 0; fi" %(repo_branch, repo_branch),
haltOnFailure = True))
# check out the source
))
# seed config
- factory.addStep(FileDownload(
- name = "dlconfigseed",
- mastersrc = "config.seed",
- slavedest = ".config",
- mode = 0644
- ))
+ if config_seed is not None:
+ factory.addStep(StringDownload(
+ name = "dlconfigseed",
+ s = config_seed + '\n',
+ slavedest = ".config",
+ mode = 0644
+ ))
# configure
factory.addStep(ShellCommand(
))
factory.addStep(ShellCommand(
- name = "diffconfig",
- description = "Generating config.seed",
- command=["make", "-j1", "diffconfig", "V=s"],
+ name = "buildinfo",
+ description = "Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
+ command=["make", "-j1", "prepare", "V=s"],
env = MakeEnv(),
haltOnFailure = True
))
))
# sign
- if gpg_key is not None or usign_key is not None:
+ if ini.has_option("gpg", "key") or usign_key is not None:
factory.addStep(MasterShellCommand(
name = "signprepare",
description = "Preparing temporary signing directory",
- command = ["mkdir", "-p", "%s/signing" %(home_dir)],
+ command = ["mkdir", "-p", "%s/signing" %(work_dir)],
haltOnFailure = True
))
factory.addStep(FileUpload(
slavesrc = "sign.tar.gz",
- masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
+ masterdest = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
haltOnFailure = True
))
factory.addStep(MasterShellCommand(
name = "signfiles",
description = "Signing files",
- command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1])],
- env = {
- 'GPGKEY': gpg_key,
- 'GPGPASS': gpg_passphrase,
- 'GPGCOMMENT': gpg_comment,
- 'USIGNKEY': usign_key,
- 'USIGNCOMMENT': usign_comment
- },
+ command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1])],
+ env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
haltOnFailure = True
))
factory.addStep(FileDownload(
name = "dlsigntargz",
- mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
+ mastersrc = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
slavedest = "sign.tar.gz",
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "dirupload",
description = "Uploading directory structure",
- command = ["rsync", "-az"] + rsync_defopts + ["tmp/upload/", "%s/" %(rsync_bin_url)],
+ command = ["rsync", "-az"] + rsync_bin_defopts + ["tmp/upload/", "%s/" %(rsync_bin_url)],
env={'RSYNC_PASSWORD': rsync_bin_key},
haltOnFailure = True,
logEnviron = False,
factory.addStep(ShellCommand(
name = "target-sha256sums",
description = "Fetching remote sha256sums for target",
- command = ["rsync", "-z"] + rsync_defopts + [Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
+ command = ["rsync", "-z"] + rsync_bin_defopts + [Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
env={'RSYNC_PASSWORD': rsync_bin_key},
logEnviron = False,
haltOnFailure = False,
factory.addStep(ShellCommand(
name = "targetupload",
description = "Uploading target files",
- command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
+ command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts +
["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
env={'RSYNC_PASSWORD': rsync_bin_key},
factory.addStep(ShellCommand(
name = "targetprune",
description = "Pruning target files",
- command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
+ command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts +
["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
env={'RSYNC_PASSWORD': rsync_bin_key},
factory.addStep(ShellCommand(
name = "kmodupload",
description = "Uploading kmod archive",
- command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
+ command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts +
["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
env={'RSYNC_PASSWORD': rsync_bin_key},
factory.addStep(ShellCommand(
name = "sourceupload",
description = "Uploading source archives",
- command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_defopts +
+ command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_src_defopts +
[Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:slavename)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", "%s/" %(rsync_src_url)],
env={'RSYNC_PASSWORD': rsync_src_key},
haltOnFailure = True,
factory.addStep(ShellCommand(
name = "packageupload",
description = "Uploading package files",
- command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-a"] + rsync_defopts + ["bin/packages/", "%s/packages/" %(rsync_bin_url)],
+ command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-a"] + rsync_bin_defopts + ["bin/packages/", "%s/packages/" %(rsync_bin_url)],
env={'RSYNC_PASSWORD': rsync_bin_key},
haltOnFailure = False,
logEnviron = False,
factory.addStep(ShellCommand(
name = "upload",
description = "Uploading logs",
- command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-az"] + rsync_defopts + ["logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
+ command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-az"] + rsync_bin_defopts + ["logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
env={'RSYNC_PASSWORD': rsync_bin_key},
haltOnFailure = False,
alwaysRun = True,
from buildbot.status import html
from buildbot.status.web import authz, auth
-if ini.has_option("status", "bind"):
- if ini.has_option("status", "user") and ini.has_option("status", "password"):
+if ini.has_option("phase1", "status_bind"):
+ if ini.has_option("phase1", "status_user") and ini.has_option("phase1", "status_password"):
authz_cfg=authz.Authz(
# change any of these to True to enable; see the manual for more
# options
- auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
+ auth=auth.BasicAuth([(ini.get("phase1", "status_user"), ini.get("phase1", "status_password"))]),
gracefulShutdown = 'auth',
forceBuild = 'auth', # use this to test your slave once it is set up
forceAllBuilds = 'auth',
stopAllBuilds = 'auth',
cancelPendingBuild = 'auth',
)
- c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
+ c['status'].append(html.WebStatus(http_port=ini.get("phase1", "status_bind"), authz=authz_cfg))
else:
- c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
+ c['status'].append(html.WebStatus(http_port=ini.get("phase1", "status_bind")))
from buildbot.status import words
projects / buildbot.git / blobdiff