from buildbot.schedulers.forcesched import BaseParameter
from buildbot.schedulers.forcesched import ForceScheduler
from buildbot.schedulers.forcesched import ValidationError
-from buildbot.steps.master import MasterShellCommand
+from buildbot.steps.master import MasterShellCommand, SetProperty
from buildbot.steps.shell import SetPropertyFromCommand
from buildbot.steps.shell import ShellCommand
from buildbot.steps.source.git import Git
if section.startswith("worker "):
if ini.has_option(section, "name") and ini.has_option(section, "password") and \
(not ini.has_option(section, "phase") or ini.getint(section, "phase") == 1):
- sl_props = { 'dl_lock':None, 'ul_lock':None, 'do_cleanup':False, 'max_builds':1, 'shared_wd':False }
+ sl_props = { 'dl_lock':None, 'ul_lock':None, 'do_cleanup':False, 'shared_wd':True }
name = ini.get(section, "name")
password = ini.get(section, "password")
- max_builds = 1
- if ini.has_option(section, "builds"):
- max_builds = ini.getint(section, "builds")
- sl_props['max_builds'] = max_builds
- if max_builds == 1:
- sl_props['shared_wd'] = True
if ini.has_option(section, "cleanup"):
sl_props['do_cleanup'] = ini.getboolean(section, "cleanup")
if ini.has_option(section, "dl_lock"):
if ini.has_option(section, "shared_wd"):
shared_wd = ini.getboolean(section, "shared_wd")
sl_props['shared_wd'] = shared_wd
- if shared_wd and (max_builds != 1):
- raise ValueError('max_builds must be 1 with shared workdir!')
- c['workers'].append(Worker(name, password, max_builds = max_builds, properties = sl_props))
+ c['workers'].append(Worker(name, password, max_builds = 1, properties = sl_props))
# PB port can be either a numeric port or a connection string
pb_port = inip1.get("port") or 9989
work_dir = os.path.abspath(ini['general'].get("workdir", "."))
scripts_dir = os.path.abspath("../scripts")
-cc_command = "gcc"
-cxx_command = "g++"
-
tree_expire = inip1.getint("expire", 0)
config_seed = inip1.get("config_seed", "")
# what steps, and which workers can execute them. Note that any particular build will
# only take place on one worker.
-CleanTargetMap = [
- [ "tools", "tools/clean" ],
- [ "chain", "toolchain/clean" ],
- [ "linux", "target/linux/clean" ],
- [ "dir", "dirclean" ],
- [ "dist", "distclean" ]
-]
-
-def IsMakeCleanRequested(pattern):
- def CheckCleanProperty(step):
- val = step.getProperty("clean")
- if val and re.match(pattern, val):
- return True
- else:
- return False
-
- return CheckCleanProperty
-
def IsSharedWorkdir(step):
return bool(step.getProperty("shared_wd"))
else:
return not IsCleanupRequested(step)
-def IsGitFreshRequested(step):
- do_cleanup = step.getProperty("do_cleanup")
- if do_cleanup:
- return True
- else:
- return False
-
-def IsGitCleanRequested(step):
- return not IsGitFreshRequested(step)
-
def IsTaggingRequested(step):
val = step.getProperty("tag")
if val and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val):
else:
return False
-def IsNoTaggingRequested(step):
- return not IsTaggingRequested(step)
-
def IsNoMasterBuild(step):
return repo_branch != "master"
else:
return ""
-@properties.renderer
-def GetNumJobs(props):
- if props.hasProperty("max_builds") and props.hasProperty("nproc"):
- return str(int(int(props["nproc"]) / props["max_builds"]))
- else:
- return "1"
-
-@properties.renderer
-def GetCC(props):
- if props.hasProperty("cc_command"):
- return props["cc_command"]
- else:
- return "gcc"
-
-@properties.renderer
-def GetCXX(props):
- if props.hasProperty("cxx_command"):
- return props["cxx_command"]
- else:
- return "g++"
-
-@properties.renderer
-def GetCwd(props):
- if props.hasProperty("builddir"):
- return props["builddir"]
- elif props.hasProperty("workdir"):
- return props["workdir"]
- else:
- return "/"
-
-@properties.renderer
-def GetCCache(props):
- if props.hasProperty("ccache_command") and "ccache" in props["ccache_command"]:
- return props["ccache_command"]
- else:
- return ""
-
def GetNextBuild(builder, requests):
for r in requests:
if r.properties and r.properties.hasProperty("tag"):
def MakeEnv(overrides=None, tryccache=False):
env = {
- 'CCC': Interpolate("%(kw:cc)s", cc=GetCC),
- 'CCXX': Interpolate("%(kw:cxx)s", cxx=GetCXX),
+ 'CCC': Interpolate("%(prop:cc_command:-gcc)s"),
+ 'CCXX': Interpolate("%(prop:cxx_command:-g++)s"),
}
if tryccache:
- env['CC'] = Interpolate("%(kw:cwd)s/ccache_cc.sh", cwd=GetCwd)
- env['CXX'] = Interpolate("%(kw:cwd)s/ccache_cxx.sh", cwd=GetCwd)
- env['CCACHE'] = Interpolate("%(kw:ccache)s", ccache=GetCCache)
+ env['CC'] = Interpolate("%(prop:builddir)s/ccache_cc.sh")
+ env['CXX'] = Interpolate("%(prop:builddir)s/ccache_cxx.sh")
+ env['CCACHE'] = Interpolate("%(prop:ccache_command:-)s")
else:
env['CC'] = env['CCC']
env['CXX'] = env['CCXX']
dlLock = locks.WorkerLock("worker_dl")
-checkBuiltin = re.sub('[\t\n ]+', ' ', """
- checkBuiltin() {
- local symbol op path file;
- for file in $CHANGED_FILES; do
- case "$file" in
- package/*/*) : ;;
- *) return 0 ;;
- esac;
- done;
- while read symbol op path; do
- case "$symbol" in package-*)
- symbol="${symbol##*(}";
- symbol="${symbol%)}";
- for file in $CHANGED_FILES; do
- case "$file" in "package/$path/"*)
- grep -qsx "$symbol=y" .config && return 0
- ;; esac;
- done;
- esac;
- done < tmp/.packagedeps;
- return 1;
- }
-""").strip()
-
-
-class IfBuiltinShellCommand(ShellCommand):
- def _quote(self, str):
- if re.search("[^a-zA-Z0-9/_.-]", str):
- return "'%s'" %(re.sub("'", "'\"'\"'", str))
- return str
-
- def setCommand(self, command):
- if not isinstance(command, (str, unicode)):
- command = ' '.join(map(self._quote, command))
- self.command = [
- '/bin/sh', '-c',
- '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
- ]
-
- def setupEnvironment(self, cmd):
- workerEnv = self.workerEnvironment
- if workerEnv is None:
- workerEnv = { }
- changedFiles = { }
- for request in self.build.requests:
- for source in request.sources:
- for change in source.changes:
- for file in change.files:
- changedFiles[file] = True
- fullSlaveEnv = workerEnv.copy()
- fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
- cmd.args['env'] = fullSlaveEnv
-
workerNames = [ ]
for worker in c['workers']:
description = "Finding number of CPUs",
command = ["nproc"]))
+ # set number of jobs
+ factory.addStep(SetProperty(
+ name = "njobs",
+ property = "njobs",
+ description = "Set max concurrency",
+ value = Interpolate("%(prop:nproc:-1)s")))
+
# find gcc and g++ compilers
factory.addStep(FileDownload(
name = "dlfindbinpl",
doStepIf = IsCleanupRequested,
timeout = 2400))
- # user-requested clean targets
- for tuple in CleanTargetMap:
- factory.addStep(ShellCommand(
- name = tuple[1],
- description = 'User-requested "make %s"' % tuple[1],
- command = ["make", tuple[1], "V=s"],
- env = MakeEnv(),
- doStepIf = IsMakeCleanRequested(tuple[0])
- ))
-
# Workaround bug when switching from a checked out tag back to a branch
# Ref: http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
factory.addStep(ShellCommand(
# if repo doesn't exist: 'git clone repourl'
# method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -d -f x'. Only works with mode='full'
# 'git fetch -t repourl branch; git reset --hard revision'
- # Git() parameters can't take a renderer until buildbot 0.8.10, so we have to split the fresh and clean cases
- # if buildbot is updated, one can use: method = Interpolate('%(prop:do_cleanup:#?|fresh|clean)s')
- factory.addStep(Git(
- name = "gitclean",
- repourl = repo_url,
- branch = repo_branch,
- mode = 'full',
- method = 'clean',
- locks = NetLockDl,
- haltOnFailure = True,
- doStepIf = IsGitCleanRequested,
- ))
-
factory.addStep(Git(
- name = "gitfresh",
+ name = "git",
repourl = repo_url,
branch = repo_branch,
mode = 'full',
- method = 'fresh',
+ method = Interpolate("%(prop:do_cleanup:#?|fresh|clean)s"),
locks = NetLockDl,
haltOnFailure = True,
- doStepIf = IsGitFreshRequested,
))
# update remote refs
command=["rm", "-rf", "tmp/"]))
# feed
-# factory.addStep(ShellCommand(
-# name = "feedsconf",
-# description = "Copy the feeds.conf",
-# command='''cp ~/feeds.conf ./feeds.conf''' ))
-
- # feed
factory.addStep(ShellCommand(
name = "rmfeedlinks",
description = "Remove feed symlinks",
factory.addStep(ShellCommand(
name = "dltar",
description = "Building and installing GNU tar",
- command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "tools/tar/compile", "V=s"],
+ command = ["make", Interpolate("-j%(prop:njobs)s"), "tools/tar/compile", "V=s"],
env = MakeEnv(tryccache=True),
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "dlrun",
description = "Populating dl/",
- command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "download", "V=s"],
+ command = ["make", Interpolate("-j%(prop:njobs)s"), "download", "V=s"],
env = MakeEnv(),
logEnviron = False,
locks = properties.FlattenList(NetLockDl, [dlLock.access('exclusive')]),
factory.addStep(ShellCommand(
name = "tools",
description = "Building and installing tools",
- command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "tools/install", "V=s"],
+ command = ["make", Interpolate("-j%(prop:njobs)s"), "tools/install", "V=s"],
env = MakeEnv(tryccache=True),
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "toolchain",
description = "Building and installing toolchain",
- command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "toolchain/install", "V=s"],
+ command=["make", Interpolate("-j%(prop:njobs)s"), "toolchain/install", "V=s"],
env = MakeEnv(),
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "kmods",
description = "Building kmods",
- command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
+ command=["make", Interpolate("-j%(prop:njobs)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
env = MakeEnv(),
- #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
haltOnFailure = True
))
property = "kernelversion",
description = "Finding the effective Kernel version",
command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
- env = { 'TOPDIR': Interpolate("%(kw:cwd)s/build", cwd=GetCwd) }
+ env = { 'TOPDIR': Interpolate("%(prop:builddir)s/build") }
))
factory.addStep(ShellCommand(
factory.addStep(ShellCommand(
name = "pkgbuild",
description = "Building packages",
- command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
+ command=["make", Interpolate("-j%(prop:njobs)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
env = MakeEnv(),
- #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
haltOnFailure = True
))
- # factory.addStep(IfBuiltinShellCommand(
factory.addStep(ShellCommand(
name = "pkginstall",
description = "Installing packages",
- command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/install", "V=s"],
+ command=["make", Interpolate("-j%(prop:njobs)s"), "package/install", "V=s"],
env = MakeEnv(),
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "pkgindex",
description = "Indexing packages",
- command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
+ command=["make", Interpolate("-j%(prop:njobs)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
env = MakeEnv(),
haltOnFailure = True
))
- #factory.addStep(IfBuiltinShellCommand(
factory.addStep(ShellCommand(
name = "images",
description = "Building and installing images",
- command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "target/install", "V=s"],
+ command=["make", Interpolate("-j%(prop:njobs)s"), "target/install", "V=s"],
env = MakeEnv(),
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "kmodindex",
description = "Indexing kmod archive",
- command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
+ command=["make", Interpolate("-j%(prop:njobs)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
env = MakeEnv(),
haltOnFailure = True
locks = NetLockUl,
))
- if False:
- factory.addStep(ShellCommand(
- name = "packageupload",
- description = "Uploading package files",
- command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-a"] + rsync_bin_defopts + ["bin/packages/", "%s/packages/" %(rsync_bin_url)],
- env={'RSYNC_PASSWORD': rsync_bin_key},
- haltOnFailure = False,
- flunkOnFailure = False,
- warnOnFailure = True,
- logEnviron = False,
- locks = NetLockUl,
- ))
-
- # logs
- if False:
- factory.addStep(ShellCommand(
- name = "upload",
- description = "Uploading logs",
- command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-az"] + rsync_bin_defopts + ["logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
- env={'RSYNC_PASSWORD': rsync_bin_key},
- haltOnFailure = False,
- flunkOnFailure = False,
- warnOnFailure = True,
- alwaysRun = True,
- logEnviron = False,
- locks = NetLockUl,
- ))
-
factory.addStep(ShellCommand(
name = "df",
description = "Reporting disk usage",
projects / buildbot.git / blobdiff