import os
import re
+import base64
import subprocess
import ConfigParser
# 'master.cfg' in your buildmaster's base directory.
ini = ConfigParser.ConfigParser()
-ini.read("./config.ini")
+ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
# with an externally-visible host name which the buildbot cannot figure out
# without some help.
-c['buildbotURL'] = ini.get("general", "buildbot_url")
+c['buildbotURL'] = ini.get("phase1", "buildbot_url")
####### BUILDSLAVES
slave_port = 9989
-if ini.has_option("general", "port"):
- slave_port = ini.getint("general", "port")
+if ini.has_option("phase1", "port"):
+ slave_port = ini.getint("phase1", "port")
c['slaves'] = []
NetLocks = dict()
for section in ini.sections():
if section.startswith("slave "):
- if ini.has_option(section, "name") and ini.has_option(section, "password"):
+ if ini.has_option(section, "name") and ini.has_option(section, "password") and \
+ (not ini.has_option(section, "phase") or ini.getint(section, "phase") == 1):
sl_props = { 'dl_lock':None, 'ul_lock':None, 'do_cleanup':False, 'max_builds':1, 'shared_wd':False }
name = ini.get(section, "name")
password = ini.get(section, "password")
if ini.has_option(section, "builds"):
max_builds = ini.getint(section, "builds")
sl_props['max_builds'] = max_builds
+ if max_builds == 1:
+ sl_props['shared_wd'] = True
if ini.has_option(section, "cleanup"):
sl_props['do_cleanup'] = ini.getboolean(section, "cleanup")
if ini.has_option(section, "dl_lock"):
####### CHANGESOURCES
-home_dir = os.path.abspath(ini.get("general", "homedir"))
+work_dir = os.path.abspath(ini.get("general", "workdir") or ".")
scripts_dir = os.path.abspath("../scripts")
tree_expire = 0
other_builds = 0
cc_command = "gcc"
cxx_command = "g++"
+config_seed = ""
+
git_ssh = False
git_ssh_key = None
-if ini.has_option("general", "expire"):
- tree_expire = ini.getint("general", "expire")
+if ini.has_option("phase1", "expire"):
+ tree_expire = ini.getint("phase1", "expire")
-if ini.has_option("general", "other_builds"):
- other_builds = ini.getint("general", "other_builds")
+if ini.has_option("phase1", "other_builds"):
+ other_builds = ini.getint("phase1", "other_builds")
-if ini.has_option("general", "cc_version"):
- cc_version = ini.get("general", "cc_version").split()
+if ini.has_option("phase1", "cc_version"):
+ cc_version = ini.get("phase1", "cc_version").split()
if len(cc_version) == 1:
cc_version = ["eq", cc_version[0]]
else:
git_ssh = False
+if ini.has_option("phase1", "config_seed"):
+ config_seed = ini.get("phase1", "config_seed")
+
repo_url = ini.get("repo", "url")
repo_branch = "master"
rsync_bin_url = ini.get("rsync", "binary_url")
rsync_bin_key = ini.get("rsync", "binary_password")
+rsync_bin_defopts = ["-v", "-4", "--timeout=120"]
+
+if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
+ rsync_bin_defopts += ["--contimeout=20"]
rsync_src_url = None
rsync_src_key = None
+rsync_src_defopts = ["-v", "-4", "--timeout=120"]
if ini.has_option("rsync", "source_url"):
rsync_src_url = ini.get("rsync", "source_url")
rsync_src_key = ini.get("rsync", "source_password")
-rsync_defopts = ["-4", "-v", "--timeout=120", "--contimeout=20"]
-
-gpg_home = "~/.gnupg"
-gpg_keyid = None
-gpg_comment = "Unattended build signature"
-gpg_passfile = "/dev/null"
-
-if ini.has_option("gpg", "home"):
- gpg_home = ini.get("gpg", "home")
+ if rsync_src_url.find("::") > 0 or rsync_src_url.find("rsync://") == 0:
+ rsync_src_defopts += ["--contimeout=20"]
-if ini.has_option("gpg", "keyid"):
- gpg_keyid = ini.get("gpg", "keyid")
+usign_key = None
+usign_comment = "untrusted comment: " + repo_branch.replace("-", " ").title() + " key"
-if ini.has_option("gpg", "comment"):
- gpg_comment = ini.get("gpg", "comment")
+if ini.has_option("usign", "key"):
+ usign_key = ini.get("usign", "key")
-if ini.has_option("gpg", "passfile"):
- gpg_passfile = ini.get("gpg", "passfile")
+if ini.has_option("usign", "comment"):
+ usign_comment = ini.get("usign", "comment")
enable_kmod_archive = True
# find targets
targets = [ ]
-if not os.path.isdir(home_dir+'/source.git'):
- subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, home_dir+'/source.git'])
+if not os.path.isdir(work_dir+'/source.git'):
+ subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, work_dir+'/source.git'])
else:
- subprocess.call(["git", "pull"], cwd = home_dir+'/source.git')
+ subprocess.call(["git", "pull"], cwd = work_dir+'/source.git')
findtargets = subprocess.Popen([scripts_dir + '/dumpinfo.pl', 'targets'],
- stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
+ stdout = subprocess.PIPE, cwd = work_dir+'/source.git')
while True:
line = findtargets.stdout.readline()
c['change_source'] = []
c['change_source'].append(GitPoller(
repo_url,
- workdir=home_dir+'/work.git', branch=repo_branch,
+ workdir=work_dir+'/work.git', branch=repo_branch,
pollinterval=300))
####### SCHEDULERS
return CheckCleanProperty
+def IsSharedWorkdir(step):
+ return bool(step.getProperty("shared_wd"))
+
def IsCleanupRequested(step):
- shared_wd = step.getProperty("shared_wd")
- if shared_wd:
+ if IsSharedWorkdir(step):
return False
do_cleanup = step.getProperty("do_cleanup")
if do_cleanup:
return False
def IsExpireRequested(step):
- shared_wd = step.getProperty("shared_wd")
- if shared_wd:
+ if IsSharedWorkdir(step):
return False
else:
return not IsCleanupRequested(step)
else:
return []
+def UsignSec2Pub(seckey, comment="untrusted comment: secret key"):
+ try:
+ seckey = base64.b64decode(seckey)
+ except:
+ return None
+
+ return "{}\n{}".format(re.sub(r"\bsecret key$", "public key", comment),
+ base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]))
+
+
c['builders'] = []
dlLock = locks.SlaveLock("slave_dl")
factory = BuildFactory()
+ # setup shared work directory if required
+ factory.addStep(ShellCommand(
+ name = "sharedwd",
+ description = "Setting up shared work directory",
+ command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
+ workdir = ".",
+ haltOnFailure = True,
+ doStepIf = IsSharedWorkdir))
+
# find number of cores
factory.addStep(SetPropertyFromCommand(
name = "nproc",
command = ["nproc"]))
# find gcc and g++ compilers
- if cc_version is not None:
- factory.addStep(FileDownload(
- name = "dlfindbinpl",
- mastersrc = scripts_dir + '/findbin.pl',
- slavedest = "../findbin.pl",
- mode = 0755))
+ factory.addStep(FileDownload(
+ name = "dlfindbinpl",
+ mastersrc = scripts_dir + '/findbin.pl',
+ slavedest = "../findbin.pl",
+ mode = 0755))
- factory.addStep(SetPropertyFromCommand(
- name = "gcc",
- property = "cc_command",
- description = "Finding gcc command",
- command = ["../findbin.pl", "gcc", cc_version[0], cc_version[1]],
- haltOnFailure = True))
+ factory.addStep(SetPropertyFromCommand(
+ name = "gcc",
+ property = "cc_command",
+ description = "Finding gcc command",
+ command = [
+ "../findbin.pl", "gcc",
+ cc_version[0] if cc_version is not None else '',
+ cc_version[1] if cc_version is not None else ''
+ ],
+ haltOnFailure = True))
- factory.addStep(SetPropertyFromCommand(
- name = "g++",
- property = "cxx_command",
- description = "Finding g++ command",
- command = ["../findbin.pl", "g++", cc_version[0], cc_version[1]],
- haltOnFailure = True))
+ factory.addStep(SetPropertyFromCommand(
+ name = "g++",
+ property = "cxx_command",
+ description = "Finding g++ command",
+ command = [
+ "../findbin.pl", "g++",
+ cc_version[0] if cc_version is not None else '',
+ cc_version[1] if cc_version is not None else ''
+ ],
+ haltOnFailure = True))
# see if ccache is available
factory.addStep(SetPropertyFromCommand(
# cleanup.sh if needed
factory.addStep(FileDownload(
- name = "dlcleanupsh",
- mastersrc = "cleanup.sh",
+ name = "dlcleanupsh",
+ mastersrc = scripts_dir + '/cleanup.sh',
slavedest = "../cleanup.sh",
mode = 0755,
doStepIf = IsCleanupRequested))
factory.addStep(ShellCommand(
name = "gitcheckout",
description = "Ensure that Git HEAD is sane",
- command = "if [ -d .git ]; then git checkout master; else exit 0; fi",
+ command = "if [ -d .git ]; then git checkout -f %s; git branch --set-upstream-to origin/%s; else exit 0; fi" %(repo_branch, repo_branch),
haltOnFailure = True))
# check out the source
))
# seed config
- factory.addStep(FileDownload(
- name = "dlconfigseed",
- mastersrc = "config.seed",
- slavedest = ".config",
- mode = 0644
- ))
+ if config_seed is not None:
+ factory.addStep(StringDownload(
+ name = "dlconfigseed",
+ s = config_seed + '\n',
+ slavedest = ".config",
+ mode = 0644
+ ))
# configure
factory.addStep(ShellCommand(
name = "newconfig",
description = "Seeding .config",
- command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\n' >> .config" %(ts[0], ts[0], ts[1])
+ command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\nCONFIG_SIGNED_PACKAGES=%s\\n' >> .config" %(ts[0], ts[0], ts[1], 'y' if usign_key is not None else 'n')
))
factory.addStep(ShellCommand(
command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
# install build key
- factory.addStep(FileDownload(name="dlkeybuild", mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
- factory.addStep(FileDownload(name="dlkeybuildpub", mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
+ if usign_key is not None:
+ factory.addStep(StringDownload(
+ name = "dlkeybuildpub",
+ s = UsignSec2Pub(usign_key, usign_comment),
+ slavedest = "key-build.pub",
+ mode = 0600,
+ ))
+
+ factory.addStep(StringDownload(
+ name = "dlkeybuild",
+ s = "# fake private key",
+ slavedest = "key-build",
+ mode = 0600,
+ ))
+
+ factory.addStep(StringDownload(
+ name = "dlkeybuilducert",
+ s = "# fake certificate",
+ slavedest = "key-build.ucert",
+ mode = 0600,
+ ))
# prepare dl
factory.addStep(ShellCommand(
factory.addStep(ShellCommand(
name = "pkgindex",
description = "Indexing packages",
- command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/index", "V=s"],
+ command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
env = MakeEnv(),
haltOnFailure = True
))
))
factory.addStep(ShellCommand(
- name = "diffconfig",
- description = "Generating config.seed",
- command=["make", "-j1", "diffconfig", "V=s"],
+ name = "buildinfo",
+ description = "Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
+ command=["make", "-j1", "prepare", "V=s"],
env = MakeEnv(),
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "kmodindex",
description = "Indexing kmod archive",
- command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/index", "V=s",
+ command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
env = MakeEnv(),
haltOnFailure = True
))
# sign
- if gpg_keyid is not None:
+ if ini.has_option("gpg", "key") or usign_key is not None:
factory.addStep(MasterShellCommand(
name = "signprepare",
description = "Preparing temporary signing directory",
- command = ["mkdir", "-p", "%s/signing" %(home_dir)],
+ command = ["mkdir", "-p", "%s/signing" %(work_dir)],
haltOnFailure = True
))
factory.addStep(FileUpload(
slavesrc = "sign.tar.gz",
- masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
+ masterdest = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
haltOnFailure = True
))
factory.addStep(MasterShellCommand(
name = "signfiles",
description = "Signing files",
- command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_comment],
- env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
+ command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1])],
+ env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
haltOnFailure = True
))
factory.addStep(FileDownload(
name = "dlsigntargz",
- mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
+ mastersrc = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
slavedest = "sign.tar.gz",
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "dirupload",
description = "Uploading directory structure",
- command = ["rsync", "-az"] + rsync_defopts + ["tmp/upload/", "%s/" %(rsync_bin_url)],
+ command = ["rsync", "-az"] + rsync_bin_defopts + ["tmp/upload/", "%s/" %(rsync_bin_url)],
env={'RSYNC_PASSWORD': rsync_bin_key},
haltOnFailure = True,
logEnviron = False,
factory.addStep(ShellCommand(
name = "target-sha256sums",
description = "Fetching remote sha256sums for target",
- command = ["rsync", "-z"] + rsync_defopts + [Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
+ command = ["rsync", "-z"] + rsync_bin_defopts + [Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
env={'RSYNC_PASSWORD': rsync_bin_key},
logEnviron = False,
haltOnFailure = False,
factory.addStep(ShellCommand(
name = "targetupload",
description = "Uploading target files",
- command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
+ command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts +
["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
env={'RSYNC_PASSWORD': rsync_bin_key},
factory.addStep(ShellCommand(
name = "targetprune",
description = "Pruning target files",
- command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
+ command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts +
["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
env={'RSYNC_PASSWORD': rsync_bin_key},
factory.addStep(ShellCommand(
name = "kmodupload",
description = "Uploading kmod archive",
- command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
+ command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts +
["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
env={'RSYNC_PASSWORD': rsync_bin_key},
factory.addStep(ShellCommand(
name = "sourceupload",
description = "Uploading source archives",
- command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_defopts +
+ command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_src_defopts +
[Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:slavename)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", "%s/" %(rsync_src_url)],
env={'RSYNC_PASSWORD': rsync_src_key},
haltOnFailure = True,
factory.addStep(ShellCommand(
name = "packageupload",
description = "Uploading package files",
- command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-a"] + rsync_defopts + ["bin/packages/", "%s/packages/" %(rsync_bin_url)],
+ command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-a"] + rsync_bin_defopts + ["bin/packages/", "%s/packages/" %(rsync_bin_url)],
env={'RSYNC_PASSWORD': rsync_bin_key},
haltOnFailure = False,
logEnviron = False,
factory.addStep(ShellCommand(
name = "upload",
description = "Uploading logs",
- command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-az"] + rsync_defopts + ["logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
+ command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-az"] + rsync_bin_defopts + ["logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
env={'RSYNC_PASSWORD': rsync_bin_key},
haltOnFailure = False,
alwaysRun = True,
from buildbot.status import html
from buildbot.status.web import authz, auth
-if ini.has_option("status", "bind"):
- if ini.has_option("status", "user") and ini.has_option("status", "password"):
+if ini.has_option("phase1", "status_bind"):
+ if ini.has_option("phase1", "status_user") and ini.has_option("phase1", "status_password"):
authz_cfg=authz.Authz(
# change any of these to True to enable; see the manual for more
# options
- auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
+ auth=auth.BasicAuth([(ini.get("phase1", "status_user"), ini.get("phase1", "status_password"))]),
gracefulShutdown = 'auth',
forceBuild = 'auth', # use this to test your slave once it is set up
forceAllBuilds = 'auth',
stopAllBuilds = 'auth',
cancelPendingBuild = 'auth',
)
- c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
+ c['status'].append(html.WebStatus(http_port=ini.get("phase1", "status_bind"), authz=authz_cfg))
else:
- c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
+ c['status'].append(html.WebStatus(http_port=ini.get("phase1", "status_bind")))
from buildbot.status import words
projects / buildbot.git / blobdiff