# a shorter alias to save typing.
c = BuildmasterConfig = {}
+####### PROJECT IDENTITY
+
+# the 'title' string will appear at the top of this buildbot
+# installation's html.WebStatus home page (linked to the
+# 'titleURL') and is embedded in the title of the waterfall HTML page.
+
+c['title'] = ini.get("general", "title")
+c['titleURL'] = ini.get("general", "title_url")
+
+# the 'buildbotURL' string should point to the location where the buildbot's
+# internal web server (usually the html.WebStatus page) is visible. This
+# typically uses the port number set in the Waterfall 'status' entry, but
+# with an externally-visible host name which the buildbot cannot figure out
+# without some help.
+
+c['buildbotURL'] = ini.get("general", "buildbot_url")
+
####### BUILDSLAVES
# The 'slaves' list defines the set of recognized buildslaves. Each element is
# slave name and password must be configured on the slave.
from buildbot.buildslave import BuildSlave
+slave_port = 9989
+
+if ini.has_option("general", "port"):
+ slave_port = ini.getint("general", "port")
+
c['slaves'] = []
+NetLocks = dict()
for section in ini.sections():
if section.startswith("slave "):
if ini.has_option(section, "name") and ini.has_option(section, "password"):
+ sl_props = { 'dl_lock':None, 'ul_lock':None, 'do_cleanup':False, 'max_builds':1, 'shared_wd':False }
name = ini.get(section, "name")
password = ini.get(section, "password")
max_builds = 1
if ini.has_option(section, "builds"):
max_builds = ini.getint(section, "builds")
- c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
+ sl_props['max_builds'] = max_builds
+ if ini.has_option(section, "cleanup"):
+ sl_props['do_cleanup'] = ini.getboolean(section, "cleanup")
+ if ini.has_option(section, "dl_lock"):
+ lockname = ini.get(section, "dl_lock")
+ sl_props['dl_lock'] = lockname
+ if lockname not in NetLocks:
+ NetLocks[lockname] = locks.MasterLock(lockname)
+ if ini.has_option(section, "ul_lock"):
+ lockname = ini.get(section, "dl_lock")
+ sl_props['ul_lock'] = lockname
+ if lockname not in NetLocks:
+ NetLocks[lockname] = locks.MasterLock(lockname)
+ if ini.has_option(section, "shared_wd"):
+ shared_wd = ini.getboolean(section, "shared_wd")
+ sl_props['shared_wd'] = shared_wd
+ if shared_wd and (max_builds != 1):
+ raise ValueError('max_builds must be 1 with shared workdir!')
+ c['slaves'].append(BuildSlave(name, password, max_builds = max_builds, properties = sl_props))
# 'slavePortnum' defines the TCP port to listen on for connections from slaves.
# This must match the value configured into the buildslaves (with their
# --master option)
-c['slavePortnum'] = 9989
+c['slavePortnum'] = slave_port
# coalesce builds
c['mergeRequests'] = True
+# Reduce amount of backlog data
+c['buildHorizon'] = 30
+c['logHorizon'] = 20
+
####### CHANGESOURCES
home_dir = os.path.abspath(ini.get("general", "homedir"))
tree_expire = 0
+other_builds = 0
+cc_version = None
+
+cc_command = "gcc"
+cxx_command = "g++"
if ini.has_option("general", "expire"):
tree_expire = ini.getint("general", "expire")
+if ini.has_option("general", "other_builds"):
+ other_builds = ini.getint("general", "other_builds")
+
+if ini.has_option("general", "cc_version"):
+ cc_version = ini.get("general", "cc_version").split()
+ if len(cc_version) == 1:
+ cc_version = ["eq", cc_version[0]]
+
repo_url = ini.get("repo", "url")
+repo_branch = "master"
+
+if ini.has_option("repo", "branch"):
+ repo_branch = ini.get("repo", "branch")
rsync_bin_url = ini.get("rsync", "binary_url")
rsync_bin_key = ini.get("rsync", "binary_password")
rsync_src_url = ini.get("rsync", "source_url")
rsync_src_key = ini.get("rsync", "source_password")
+rsync_defopts = ["-4", "-v", "--timeout=120", "--contimeout=20"]
+
+gpg_home = "~/.gnupg"
gpg_keyid = None
gpg_comment = "Unattended build signature"
gpg_passfile = "/dev/null"
+if ini.has_option("gpg", "home"):
+ gpg_home = ini.get("gpg", "home")
+
if ini.has_option("gpg", "keyid"):
gpg_keyid = ini.get("gpg", "keyid")
if ini.has_option("gpg", "passfile"):
gpg_passfile = ini.get("gpg", "passfile")
+enable_kmod_archive = True
+
# find targets
targets = [ ]
+if not os.path.isdir(home_dir+'/source.git'):
+ subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, home_dir+'/source.git'])
+else:
+ subprocess.call(["git", "pull"], cwd = home_dir+'/source.git')
+
findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
c['change_source'] = []
c['change_source'].append(GitPoller(
repo_url,
- workdir=home_dir+'/source.git', branch='master',
+ workdir=home_dir+'/work.git', branch=repo_branch,
pollinterval=300))
####### SCHEDULERS
c['schedulers'] = []
c['schedulers'].append(SingleBranchScheduler(
name="all",
- change_filter=filter.ChangeFilter(branch='master'),
+ change_filter=filter.ChangeFilter(branch=repo_branch),
treeStableTimer=60,
builderNames=targets))
# only take place on one slave.
from buildbot.process.factory import BuildFactory
-from buildbot.steps.source import Git
+from buildbot.steps.source.git import Git
from buildbot.steps.shell import ShellCommand
-from buildbot.steps.shell import SetProperty
+from buildbot.steps.shell import SetPropertyFromCommand
from buildbot.steps.transfer import FileUpload
from buildbot.steps.transfer import FileDownload
+from buildbot.steps.transfer import StringDownload
from buildbot.steps.master import MasterShellCommand
-from buildbot.process.properties import WithProperties
+from buildbot.process.properties import Interpolate
+from buildbot.process import properties
+
+
+CleanTargetMap = [
+ [ "tools", "tools/clean" ],
+ [ "chain", "toolchain/clean" ],
+ [ "linux", "target/linux/clean" ],
+ [ "dir", "dirclean" ],
+ [ "dist", "distclean" ]
+]
+
+def IsMakeCleanRequested(pattern):
+ def CheckCleanProperty(step):
+ val = step.getProperty("clean")
+ if val and re.match(pattern, val):
+ return True
+ else:
+ return False
+ return CheckCleanProperty
-MakeTargetMap = {
- "^tools/": "tools/clean",
- "^toolchain/": "toolchain/clean",
- "^target/linux/": "target/linux/clean",
- "^(config|include)/": "dirclean"
-}
+def IsCleanupRequested(step):
+ shared_wd = step.getProperty("shared_wd")
+ if shared_wd:
+ return False
+ do_cleanup = step.getProperty("do_cleanup")
+ if do_cleanup:
+ return True
+ else:
+ return False
-def IsAffected(pattern):
- def CheckAffected(change):
- for request in change.build.requests:
- for source in request.sources:
- for change in source.changes:
- for file in change.files:
- if re.match(pattern, file):
- return True
+def IsExpireRequested(step):
+ shared_wd = step.getProperty("shared_wd")
+ if shared_wd:
return False
- return CheckAffected
+ else:
+ return not IsCleanupRequested(step)
-def isPathBuiltin(path):
- incl = {}
- pkgs = {}
- conf = open(".config", "r")
+def IsGitFreshRequested(step):
+ do_cleanup = step.getProperty("do_cleanup")
+ if do_cleanup:
+ return True
+ else:
+ return False
- while True:
- line = conf.readline()
- if line == '':
- break
- m = re.match("^(CONFIG_PACKAGE_.+?)=y", line)
- if m:
- incl[m.group(1)] = True
+def IsGitCleanRequested(step):
+ return not IsGitFreshRequested(step)
- conf.close()
+def IsTaggingRequested(step):
+ val = step.getProperty("tag")
+ if val and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val):
+ return True
+ else:
+ return False
- deps = open("tmp/.packagedeps", "r")
+def IsNoTaggingRequested(step):
+ return not IsTaggingRequested(step)
- while True:
- line = deps.readline()
- if line == '':
- break
- m = re.match("^package-\$\((CONFIG_PACKAGE_.+?)\) \+= (\S+)", line)
- if m and incl.get(m.group(1)) == True:
- pkgs["package/%s" % m.group(2)] = True
+def IsNoMasterBuild(step):
+ return repo_branch != "master"
- deps.close()
+def GetBaseVersion():
+ if re.match("^[^-]+-[0-9]+\.[0-9]+$", repo_branch):
+ return repo_branch.split('-')[1]
+ else:
+ return "master"
+
+@properties.renderer
+def GetVersionPrefix(props):
+ basever = GetBaseVersion()
+ if props.hasProperty("tag") and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
+ return "%s/" % props["tag"]
+ elif basever != "master":
+ return "%s-SNAPSHOT/" % basever
+ else:
+ return ""
- while path != '':
- if pkgs.get(path) == True:
- return True
- path = os.path.dirname(path)
+@properties.renderer
+def GetNumJobs(props):
+ if props.hasProperty("max_builds") and props.hasProperty("nproc"):
+ return str(int(props["nproc"]) / (props["max_builds"] + other_builds))
+ else:
+ return "1"
- return False
+@properties.renderer
+def GetCC(props):
+ if props.hasProperty("cc_command"):
+ return props["cc_command"]
+ else:
+ return "gcc"
-def isChangeBuiltin(change):
- return True
-# for request in change.build.requests:
-# for source in request.sources:
-# for change in source.changes:
-# for file in change.files:
-# if isPathBuiltin(file):
-# return True
-# return False
+@properties.renderer
+def GetCXX(props):
+ if props.hasProperty("cxx_command"):
+ return props["cxx_command"]
+ else:
+ return "g++"
+
+@properties.renderer
+def GetCwd(props):
+ if props.hasProperty("builddir"):
+ return props["builddir"]
+ elif props.hasProperty("workdir"):
+ return props["workdir"]
+ else:
+ return "/"
+@properties.renderer
+def GetCCache(props):
+ if props.hasProperty("ccache_command") and "ccache" in props["ccache_command"]:
+ return props["ccache_command"]
+ else:
+ return ""
+
+def GetNextBuild(builder, requests):
+ for r in requests:
+ if r.properties and r.properties.hasProperty("tag"):
+ return r
+ return requests[0]
+
+def MakeEnv(overrides=None, tryccache=False):
+ if tryccache:
+ envcc = Interpolate("%(kw:cwd)s/ccache_cc.sh", cwd=GetCwd)
+ envcxx = Interpolate("%(kw:cwd)s/ccache_cxx.sh", cwd=GetCwd)
+ envccache = Interpolate("%(kw:ccache)s", ccache=GetCCache)
+ envccc = Interpolate("%(kw:cc)s", cc=GetCC)
+ envccxx = Interpolate("%(kw:cxx)s", cxx=GetCXX)
+ else:
+ envcc = Interpolate("%(kw:cc)s", cc=GetCC)
+ envcxx = Interpolate("%(kw:cxx)s", cxx=GetCXX)
+ envccache = ""
+ envccc = ""
+ envccxx = ""
+ env = {
+ 'CC': envcc,
+ 'CXX': envcxx,
+ 'CCACHE': envccache,
+ 'CCC': envccc,
+ 'CCXX': envccxx,
+ }
+ if overrides is not None:
+ env.update(overrides)
+ return env
+
+@properties.renderer
+def NetLockDl(props):
+ lock = None
+ if props.hasProperty("dl_lock"):
+ lock = NetLocks[props["dl_lock"]]
+ if lock is not None:
+ return [lock.access('exclusive')]
+ else:
+ return []
+
+@properties.renderer
+def NetLockUl(props):
+ lock = None
+ if props.hasProperty("ul_lock"):
+ lock = NetLocks[props["ul_lock"]]
+ if lock is not None:
+ return [lock.access('exclusive')]
+ else:
+ return []
c['builders'] = []
factory = BuildFactory()
# find number of cores
- factory.addStep(SetProperty(
+ factory.addStep(SetPropertyFromCommand(
name = "nproc",
property = "nproc",
description = "Finding number of CPUs",
command = ["nproc"]))
+ # find gcc and g++ compilers
+ if cc_version is not None:
+ factory.addStep(FileDownload(
+ name = "dlfindbinpl",
+ mastersrc = "findbin.pl",
+ slavedest = "../findbin.pl",
+ mode = 0755))
+
+ factory.addStep(SetPropertyFromCommand(
+ name = "gcc",
+ property = "cc_command",
+ description = "Finding gcc command",
+ command = ["../findbin.pl", "gcc", cc_version[0], cc_version[1]],
+ haltOnFailure = True))
+
+ factory.addStep(SetPropertyFromCommand(
+ name = "g++",
+ property = "cxx_command",
+ description = "Finding g++ command",
+ command = ["../findbin.pl", "g++", cc_version[0], cc_version[1]],
+ haltOnFailure = True))
+
+ # see if ccache is available
+ factory.addStep(SetPropertyFromCommand(
+ property = "ccache_command",
+ command = ["which", "ccache"],
+ description = "Testing for ccache command",
+ haltOnFailure = False,
+ flunkOnFailure = False,
+ warnOnFailure = False,
+ ))
+
# expire tree if needed
if tree_expire > 0:
factory.addStep(FileDownload(
+ name = "dlexpiresh",
+ doStepIf = IsExpireRequested,
mastersrc = "expire.sh",
- slavedest = "expire.sh",
+ slavedest = "../expire.sh",
mode = 0755))
factory.addStep(ShellCommand(
name = "expire",
description = "Checking for build tree expiry",
- command = ["./expire.sh", tree_expire],
+ command = ["./expire.sh", str(tree_expire)],
workdir = ".",
haltOnFailure = True,
+ doStepIf = IsExpireRequested,
timeout = 2400))
+ # cleanup.sh if needed
+ factory.addStep(FileDownload(
+ name = "dlcleanupsh",
+ mastersrc = "cleanup.sh",
+ slavedest = "../cleanup.sh",
+ mode = 0755,
+ doStepIf = IsCleanupRequested))
+
+ factory.addStep(ShellCommand(
+ name = "cleanold",
+ description = "Cleaning previous builds",
+ command = ["./cleanup.sh", c['buildbotURL'], Interpolate("%(prop:slavename)s"), Interpolate("%(prop:buildername)s"), "full"],
+ workdir = ".",
+ haltOnFailure = True,
+ doStepIf = IsCleanupRequested,
+ timeout = 2400))
+
+ factory.addStep(ShellCommand(
+ name = "cleanup",
+ description = "Cleaning work area",
+ command = ["./cleanup.sh", c['buildbotURL'], Interpolate("%(prop:slavename)s"), Interpolate("%(prop:buildername)s"), "single"],
+ workdir = ".",
+ haltOnFailure = True,
+ doStepIf = IsCleanupRequested,
+ timeout = 2400))
+
+ # user-requested clean targets
+ for tuple in CleanTargetMap:
+ factory.addStep(ShellCommand(
+ name = tuple[1],
+ description = 'User-requested "make %s"' % tuple[1],
+ command = ["make", tuple[1], "V=s"],
+ env = MakeEnv(),
+ doStepIf = IsMakeCleanRequested(tuple[0])
+ ))
+
# check out the source
- factory.addStep(Git(repourl=repo_url, mode='update'))
+ # Git() runs:
+ # if repo doesn't exist: 'git clone repourl'
+ # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -d -f x'. Only works with mode='full'
+ # 'git fetch -t repourl branch; git reset --hard revision'
+ # Git() parameters can't take a renderer until buildbot 0.8.10, so we have to split the fresh and clean cases
+ # if buildbot is updated, one can use: method = Interpolate('%(prop:do_cleanup:#?|fresh|clean)s')
+ factory.addStep(Git(
+ name = "gitclean",
+ repourl = repo_url,
+ branch = repo_branch,
+ mode = 'full',
+ method = 'clean',
+ haltOnFailure = True,
+ doStepIf = IsGitCleanRequested,
+ ))
+
+ factory.addStep(Git(
+ name = "gitfresh",
+ repourl = repo_url,
+ branch = repo_branch,
+ mode = 'full',
+ method = 'fresh',
+ haltOnFailure = True,
+ doStepIf = IsGitFreshRequested,
+ ))
+
+ # update remote refs
+ factory.addStep(ShellCommand(
+ name = "fetchrefs",
+ description = "Fetching Git remote refs",
+ command = ["git", "fetch", "origin", "+refs/heads/%s:refs/remotes/origin/%s" %(repo_branch, repo_branch)],
+ haltOnFailure = True
+ ))
+
+ # switch to tag
+ factory.addStep(ShellCommand(
+ name = "switchtag",
+ description = "Checking out Git tag",
+ command = ["git", "checkout", Interpolate("tags/v%(prop:tag:-)s")],
+ haltOnFailure = True,
+ doStepIf = IsTaggingRequested
+ ))
factory.addStep(ShellCommand(
name = "rmtmp",
factory.addStep(ShellCommand(
name = "updatefeeds",
description = "Updating feeds",
- command=["./scripts/feeds", "update"]))
+ command=["./scripts/feeds", "update"],
+ env = MakeEnv(),
+ ))
# feed
factory.addStep(ShellCommand(
name = "installfeeds",
description = "Installing feeds",
- command=["./scripts/feeds", "install", "-a"]))
+ command=["./scripts/feeds", "install", "-a"],
+ env = MakeEnv()))
+
+ # seed config
+ factory.addStep(FileDownload(
+ name = "dlconfigseed",
+ mastersrc = "config.seed",
+ slavedest = ".config",
+ mode = 0644
+ ))
# configure
factory.addStep(ShellCommand(
name = "newconfig",
description = "Seeding .config",
- command='''cat <<EOT > .config
-CONFIG_TARGET_%s=y
-CONFIG_TARGET_%s_%s=y
-CONFIG_ALL_NONSHARED=y
-CONFIG_SDK=y
-CONFIG_IB=y
-# CONFIG_IB_STANDALONE is not set
-CONFIG_DEVEL=y
-CONFIG_CCACHE=y
-CONFIG_SIGNED_PACKAGES=y
-# CONFIG_PER_FEED_REPO_ADD_COMMENTED is not set
-CONFIG_KERNEL_KALLSYMS=y
-CONFIG_COLLECT_KERNEL_DEBUG=y
-CONFIG_TARGET_ALL_PROFILES=y
-CONFIG_TARGET_MULTI_PROFILE=y
-CONFIG_TARGET_PER_DEVICE_ROOTFS=y
-EOT''' %(ts[0], ts[0], ts[1]) ))
+ command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\n' >> .config" %(ts[0], ts[0], ts[1])
+ ))
factory.addStep(ShellCommand(
name = "delbin",
factory.addStep(ShellCommand(
name = "defconfig",
description = "Populating .config",
- command = ["make", "defconfig"]
+ command = ["make", "defconfig"],
+ env = MakeEnv()
))
# check arch
))
# find libc suffix
- factory.addStep(SetProperty(
+ factory.addStep(SetPropertyFromCommand(
name = "libc",
property = "libc",
description = "Finding libc suffix",
command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
# install build key
- factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
- factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
+ factory.addStep(FileDownload(name="dlkeybuild", mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
+ factory.addStep(FileDownload(name="dlkeybuildpub", mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
# prepare dl
factory.addStep(ShellCommand(
name = "dldir",
description = "Preparing dl/",
- command = "mkdir -p $HOME/dl && ln -sf $HOME/dl ./dl",
+ command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
logEnviron = False,
want_stdout = False
))
+ factory.addStep(StringDownload(
+ name = "ccachecc",
+ s = '#!/bin/sh\nexec ${CCACHE} ${CCC} "$@"\n',
+ slavedest = "../ccache_cc.sh",
+ mode = 0755,
+ ))
+
+ factory.addStep(StringDownload(
+ name = "ccachecxx",
+ s = '#!/bin/sh\nexec ${CCACHE} ${CCXX} "$@"\n',
+ slavedest = "../ccache_cxx.sh",
+ mode = 0755,
+ ))
+
+ # prepare tar
+ factory.addStep(ShellCommand(
+ name = "dltar",
+ description = "Building and installing GNU tar",
+ command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "tools/tar/compile", "V=s"],
+ env = MakeEnv(tryccache=True),
+ haltOnFailure = True
+ ))
+
# populate dl
factory.addStep(ShellCommand(
name = "dlrun",
description = "Populating dl/",
- command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
+ command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "download", "V=s"],
+ env = MakeEnv(),
logEnviron = False,
- locks = [dlLock.access('exclusive')]
+ locks = [dlLock.access('exclusive')],
))
factory.addStep(ShellCommand(
command=["make", "package/base-files/clean", "V=s"]
))
- # optional clean steps
- for pattern, maketarget in MakeTargetMap.items():
- factory.addStep(ShellCommand(
- name = maketarget,
- description = maketarget,
- command=["make", maketarget, "V=s"], doStepIf=IsAffected(pattern)
- ))
-
# build
factory.addStep(ShellCommand(
name = "tools",
- description = "Building tools",
- command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
+ description = "Building and installing tools",
+ command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "tools/install", "V=s"],
+ env = MakeEnv(tryccache=True),
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "toolchain",
- description = "Building toolchain",
- command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
+ description = "Building and installing toolchain",
+ command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "toolchain/install", "V=s"],
+ env = MakeEnv(),
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "kmods",
description = "Building kmods",
- command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
+ command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
+ env = MakeEnv(),
#env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
haltOnFailure = True
))
+ # find kernel version
+ factory.addStep(SetPropertyFromCommand(
+ name = "kernelversion",
+ property = "kernelversion",
+ description = "Finding the effective Kernel version",
+ command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
+ env = { 'TOPDIR': Interpolate("%(kw:cwd)s/build", cwd=GetCwd) }
+ ))
+
+ factory.addStep(ShellCommand(
+ name = "pkgclean",
+ description = "Cleaning up package build",
+ command=["make", "package/cleanup", "V=s"]
+ ))
+
factory.addStep(ShellCommand(
name = "pkgbuild",
description = "Building packages",
- command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
+ command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
+ env = MakeEnv(),
#env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "pkginstall",
description = "Installing packages",
- command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
- doStepIf = isChangeBuiltin,
+ command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/install", "V=s"],
+ env = MakeEnv(),
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "pkgindex",
description = "Indexing packages",
- command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
+ command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/index", "V=s"],
+ env = MakeEnv(),
haltOnFailure = True
))
+ if enable_kmod_archive:
+ # embed kmod repository. Must happen before 'images'
+
+ # find rootfs staging directory
+ factory.addStep(SetPropertyFromCommand(
+ name = "stageroot",
+ property = "stageroot",
+ description = "Finding the rootfs staging directory",
+ command=["make", "--no-print-directory", "val.STAGING_DIR_ROOT"],
+ env = { 'TOPDIR': Interpolate("%(kw:cwd)s/build", cwd=GetCwd) }
+ ))
+
+ factory.addStep(ShellCommand(
+ name = "filesdir",
+ description = "Creating file overlay directory",
+ command=["mkdir", "-p", "files/etc/opkg"],
+ haltOnFailure = True
+ ))
+
+ factory.addStep(ShellCommand(
+ name = "kmodconfig",
+ description = "Embedding kmod repository configuration",
+ command=Interpolate("sed -e 's#^\\(src/gz .*\\)_core \\(.*\\)/packages$#&\\n\\1_kmods \\2/kmods/%(prop:kernelversion)s#' " +
+ "%(prop:stageroot)s/etc/opkg/distfeeds.conf > files/etc/opkg/distfeeds.conf"),
+ haltOnFailure = True
+ ))
+
#factory.addStep(IfBuiltinShellCommand(
factory.addStep(ShellCommand(
name = "images",
- description = "Building images",
- command=["make", "-j1", "target/install", "V=s"],
- doStepIf = isChangeBuiltin,
+ description = "Building and installing images",
+ command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "target/install", "V=s"],
+ env = MakeEnv(),
+ haltOnFailure = True
+ ))
+
+ factory.addStep(ShellCommand(
+ name = "diffconfig",
+ description = "Generating config.seed",
+ command=["make", "-j1", "diffconfig", "V=s"],
+ env = MakeEnv(),
haltOnFailure = True
))
name = "checksums",
description = "Calculating checksums",
command=["make", "-j1", "checksum", "V=s"],
- doStepIf = isChangeBuiltin,
+ env = MakeEnv(),
haltOnFailure = True
))
+ if enable_kmod_archive:
+ factory.addStep(ShellCommand(
+ name = "kmoddir",
+ description = "Creating kmod directory",
+ command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])],
+ haltOnFailure = True
+ ))
+
+ factory.addStep(ShellCommand(
+ name = "kmodprepare",
+ description = "Preparing kmod archive",
+ command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
+ Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
+ Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
+ haltOnFailure = True
+ ))
+
+ factory.addStep(ShellCommand(
+ name = "kmodindex",
+ description = "Indexing kmod archive",
+ command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/index", "V=s",
+ Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
+ env = MakeEnv(),
+ haltOnFailure = True
+ ))
+
# sign
if gpg_keyid is not None:
factory.addStep(MasterShellCommand(
factory.addStep(ShellCommand(
name = "signpack",
description = "Packing files to sign",
- command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
+ command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]),
haltOnFailure = True
))
factory.addStep(MasterShellCommand(
name = "signfiles",
description = "Signing files",
- command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_passfile, gpg_comment],
+ command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_comment],
+ env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
haltOnFailure = True
))
factory.addStep(FileDownload(
+ name = "dlsigntargz",
mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
slavedest = "sign.tar.gz",
haltOnFailure = True
# upload
factory.addStep(ShellCommand(
- name = "uploadprepare",
- description = "Preparing target directory",
- command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)],
+ name = "dirprepare",
+ description = "Preparing upload directory structure",
+ command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
+ haltOnFailure = True
+ ))
+
+ factory.addStep(ShellCommand(
+ name = "linkprepare",
+ description = "Preparing repository symlink",
+ command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=GetBaseVersion()), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
+ doStepIf = IsNoMasterBuild,
+ haltOnFailure = True
+ ))
+
+ if enable_kmod_archive:
+ factory.addStep(ShellCommand(
+ name = "kmoddirprepare",
+ description = "Preparing kmod archive upload directory",
+ command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
+ haltOnFailure = True
+ ))
+
+ factory.addStep(ShellCommand(
+ name = "dirupload",
+ description = "Uploading directory structure",
+ command = ["rsync", "-az"] + rsync_defopts + ["tmp/upload/", "%s/" %(rsync_bin_url)],
env={'RSYNC_PASSWORD': rsync_bin_key},
haltOnFailure = True,
- logEnviron = False
+ logEnviron = False,
))
+ # download remote sha256sums to 'target-sha256sums'
+ factory.addStep(ShellCommand(
+ name = "target-sha256sums",
+ description = "Fetching remote sha256sums for target",
+ command = ["rsync", "-z"] + rsync_defopts + [Interpolate("%(kw:rsyncbinurl)s/targets/%(kw:target)s/%(kw:subtarget)s/sha256sums", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1]), "target-sha256sums"],
+ env={'RSYNC_PASSWORD': rsync_bin_key},
+ logEnviron = False,
+ haltOnFailure = False,
+ flunkOnFailure = False,
+ warnOnFailure = False,
+ ))
+
+ # build list of files to upload
+ factory.addStep(FileDownload(
+ name = "dlsha2rsyncpl",
+ mastersrc = "sha2rsync.pl",
+ slavedest = "../sha2rsync.pl",
+ mode = 0755,
+ ))
+
+ factory.addStep(ShellCommand(
+ name = "buildlist",
+ description = "Building list of files to upload",
+ command = ["../sha2rsync.pl", "target-sha256sums", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", target=ts[0], subtarget=ts[1]), "rsynclist"],
+ haltOnFailure = True,
+ ))
+
+ factory.addStep(FileDownload(
+ name = "dlrsync.sh",
+ mastersrc = "rsync.sh",
+ slavedest = "../rsync.sh",
+ mode = 0755
+ ))
+
+ # upload new files and update existing ones
factory.addStep(ShellCommand(
name = "targetupload",
description = "Uploading target files",
- command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
+ command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
+ ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
+ Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
+ env={'RSYNC_PASSWORD': rsync_bin_key},
+ haltOnFailure = True,
+ logEnviron = False,
+ ))
+
+ # delete files which don't exist locally
+ factory.addStep(ShellCommand(
+ name = "targetprune",
+ description = "Pruning target files",
+ command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
+ ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
+ Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
env={'RSYNC_PASSWORD': rsync_bin_key},
haltOnFailure = True,
- logEnviron = False
+ logEnviron = False,
))
+ if enable_kmod_archive:
+ factory.addStep(ShellCommand(
+ name = "kmodupload",
+ description = "Uploading kmod archive",
+ command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
+ ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
+ Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
+ env={'RSYNC_PASSWORD': rsync_bin_key},
+ haltOnFailure = True,
+ logEnviron = False,
+ ))
+
if rsync_src_url is not None:
factory.addStep(ShellCommand(
name = "sourceupload",
description = "Uploading source archives",
- command=["rsync", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
+ command=["../rsync.sh", "--size-only", "--delay-updates"] + rsync_defopts +
+ [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:slavename)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", "%s/" %(rsync_src_url)],
env={'RSYNC_PASSWORD': rsync_src_key},
haltOnFailure = True,
- logEnviron = False
+ logEnviron = False,
))
if False:
factory.addStep(ShellCommand(
name = "packageupload",
description = "Uploading package files",
- command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
+ command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-a"] + rsync_defopts + ["bin/packages/", "%s/packages/" %(rsync_bin_url)],
env={'RSYNC_PASSWORD': rsync_bin_key},
haltOnFailure = False,
- logEnviron = False
+ logEnviron = False,
))
# logs
factory.addStep(ShellCommand(
name = "upload",
description = "Uploading logs",
- command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
+ command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-az"] + rsync_defopts + ["logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
env={'RSYNC_PASSWORD': rsync_bin_key},
haltOnFailure = False,
alwaysRun = True,
- logEnviron = False
+ logEnviron = False,
))
+ factory.addStep(ShellCommand(
+ name = "df",
+ description = "Reporting disk usage",
+ command=["df", "-h", ".", "~"],
+ env={'LC_ALL': 'C'},
+ haltOnFailure = False,
+ alwaysRun = True
+ ))
+
+ factory.addStep(ShellCommand(
+ name = "ccachestat",
+ description = "Reporting ccache stats",
+ command=["ccache", "-s"],
+ env = MakeEnv(overrides={ 'PATH': ["./staging_dir/host/bin", "${PATH}"] }),
+ want_stderr = False,
+ haltOnFailure = False,
+ flunkOnFailure = False,
+ warnOnFailure = False,
+ alwaysRun = True,
+ ))
+
from buildbot.config import BuilderConfig
- c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
+ c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory, nextBuild=GetNextBuild))
####### STATUS TARGETS
c['status'].append(irc)
-
-####### PROJECT IDENTITY
-
-# the 'title' string will appear at the top of this buildbot
-# installation's html.WebStatus home page (linked to the
-# 'titleURL') and is embedded in the title of the waterfall HTML page.
-
-c['title'] = ini.get("general", "title")
-c['titleURL'] = ini.get("general", "title_url")
-
-# the 'buildbotURL' string should point to the location where the buildbot's
-# internal web server (usually the html.WebStatus page) is visible. This
-# typically uses the port number set in the Waterfall 'status' entry, but
-# with an externally-visible host name which the buildbot cannot figure out
-# without some help.
-
-c['buildbotURL'] = ini.get("general", "buildbot_url")
-
####### DB URL
c['db'] = {
projects / buildbot.git / blobdiff