- ts = target.split('/')
-
- factory = BuildFactory()
-
- # setup shared work directory if required
- factory.addStep(ShellCommand(
- name = "sharedwd",
- descriptionDone = "Shared work directory set up",
- command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
- workdir = ".",
- haltOnFailure = True,
- ))
-
- # find number of cores
- factory.addStep(SetPropertyFromCommand(
- name = "nproc",
- property = "nproc",
- description = "Finding number of CPUs",
- command = ["nproc"],
- ))
-
- # find gcc and g++ compilers
- factory.addStep(FileDownload(
- name = "dlfindbinpl",
- mastersrc = scripts_dir + '/findbin.pl',
- workerdest = "../findbin.pl",
- mode = 0o755,
- ))
-
- factory.addStep(SetPropertyFromCommand(
- name = "gcc",
- property = "cc_command",
- description = "Finding gcc command",
- command = ["../findbin.pl", "gcc", "", ""],
- haltOnFailure = True,
- ))
-
- factory.addStep(SetPropertyFromCommand(
- name = "g++",
- property = "cxx_command",
- description = "Finding g++ command",
- command = ["../findbin.pl", "g++", "", ""],
- haltOnFailure = True,
- ))
-
- # see if ccache is available
- factory.addStep(SetPropertyFromCommand(
- name = "ccache",
- property = "ccache_command",
- description = "Testing for ccache command",
- command = ["which", "ccache"],
- haltOnFailure = False,
- flunkOnFailure = False,
- warnOnFailure = False,
- hideStepIf = lambda r, s: r==results.FAILURE,
- ))
-
- # check out the source
- # Git() runs:
- # if repo doesn't exist: 'git clone repourl'
- # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -f -f -d -x'. Only works with mode='full'
- # git cat-file -e <commit>
- # git checkout -f <commit>
- # git checkout -B <branch>
- # git rev-parse HEAD
- factory.addStep(Git(
- name = "git",
- repourl = repo_url,
- mode = 'full',
- method = 'fresh',
- locks = NetLockDl,
- haltOnFailure = True,
- ))
-
- # workaround for https://github.com/openwrt/buildbot/issues/5
- factory.addStep(Git(
- name = "git me once more please",
- repourl = repo_url,
- mode = 'full',
- method = 'fresh',
- locks = NetLockDl,
- haltOnFailure = True,
- ))
-
- # update remote refs
- factory.addStep(ShellCommand(
- name = "fetchrefs",
- description = "Fetching Git remote refs",
- descriptionDone = "Git remote refs fetched",
- command = ["git", "fetch", "origin", Interpolate("+refs/heads/%(prop:branch)s:refs/remotes/origin/%(prop:branch)s")],
- haltOnFailure = True,
- ))
-
- # getver.sh requires local branches to track upstream otherwise version computation fails.
- # Git() does not set tracking branches when cloning or switching, so work around this here
- factory.addStep(ShellCommand(
- name = "trackupstream",
- description = "Setting upstream branch",
- descriptionDone = "getver.sh is happy now",
- command = ["git", "branch", "-u", Interpolate("origin/%(prop:branch)s")],
- haltOnFailure = True,
- ))
-
- # Verify that Git HEAD points to a tag or branch
- # Ref: https://web.archive.org/web/20190729224316/http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
- factory.addStep(ShellCommand(
- name = "gitverify",
- description = "Ensuring that Git HEAD is pointing to a branch or tag",
- descriptionDone = "Git HEAD is sane",
- command = 'git rev-parse --abbrev-ref HEAD | grep -vxqF HEAD || git show-ref --tags --dereference 2>/dev/null | sed -ne "/^$(git rev-parse HEAD) / { s|^.*/||; s|\\^.*||; p }" | grep -qE "^v[0-9][0-9]\\."',
- haltOnFailure = True,
- ))
-
- factory.addStep(StringDownload(
- name = "ccachecc",
- s = '#!/bin/sh\nexec ${CCACHE} ${CCC} "$@"\n',
- workerdest = "../ccache_cc.sh",
- mode = 0o755,
- ))
-
- factory.addStep(StringDownload(
- name = "ccachecxx",
- s = '#!/bin/sh\nexec ${CCACHE} ${CCXX} "$@"\n',
- workerdest = "../ccache_cxx.sh",
- mode = 0o755,
- ))
-
- # feed
- factory.addStep(ShellCommand(
- name = "updatefeeds",
- description = "Updating feeds",
- command=["./scripts/feeds", "update"],
- env = MakeEnv(tryccache=True),
- haltOnFailure = True,
- locks = NetLockDl,
- ))
-
- # feed
- factory.addStep(ShellCommand(
- name = "installfeeds",
- description = "Installing feeds",
- command=["./scripts/feeds", "install", "-a"],
- env = MakeEnv(tryccache=True),
- haltOnFailure = True,
- ))
-
- # seed config
- factory.addStep(StringDownload(
- name = "dlconfigseed",
- s = Interpolate("%(kw:seed)s\n", seed=GetConfigSeed),
- workerdest = ".config",
- mode = 0o644,
- ))
-
- # configure
- factory.addStep(ShellCommand(
- name = "newconfig",
- descriptionDone = ".config seeded",
- command = Interpolate("printf 'CONFIG_TARGET_%(kw:target)s=y\\nCONFIG_TARGET_%(kw:target)s_%(kw:subtarget)s=y\\nCONFIG_SIGNED_PACKAGES=%(kw:usign:#?|y|n)s\\n' >> .config", target=ts[0], subtarget=ts[1], usign=GetUsignKey),
- ))
-
- factory.addStep(ShellCommand(
- name = "defconfig",
- description = "Populating .config",
- command = ["make", "defconfig"],
- env = MakeEnv(),
- ))
-
- # check arch - exit early if does not exist - NB: some targets do not define CONFIG_TARGET_target_subtarget
- factory.addStep(ShellCommand(
- name = "checkarch",
- description = "Checking architecture",
- descriptionDone = "Architecture validated",
- command = 'grep -sq CONFIG_TARGET_%s=y .config && grep -sq CONFIG_TARGET_SUBTARGET=\\"%s\\" .config' %(ts[0], ts[1]),
- logEnviron = False,
- want_stdout = False,
- want_stderr = False,
- haltOnFailure = True,
- flunkOnFailure = False, # this is not a build FAILURE - TODO mark build as SKIPPED
- ))
-
- # find libc suffix
- factory.addStep(SetPropertyFromCommand(
- name = "libc",
- property = "libc",
- description = "Finding libc suffix",
- command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"],
- ))
-
- # install build key
- factory.addStep(StringDownload(
- name = "dlkeybuildpub",
- s = Interpolate("%(kw:sec2pub)s", sec2pub=UsignSec2Pub),
- workerdest = "key-build.pub",
- mode = 0o600,
- doStepIf = IsUsignEnabled,
- ))
-
- factory.addStep(StringDownload(
- name = "dlkeybuild",
- s = "# fake private key",
- workerdest = "key-build",
- mode = 0o600,
- doStepIf = IsUsignEnabled,
- ))
-
- factory.addStep(StringDownload(
- name = "dlkeybuilducert",
- s = "# fake certificate",
- workerdest = "key-build.ucert",
- mode = 0o600,
- doStepIf = IsUsignEnabled,
- ))
-
- # prepare dl
- factory.addStep(ShellCommand(
- name = "dldir",
- description = "Preparing dl/",
- descriptionDone = "dl/ prepared",
- command = 'mkdir -p ../dl && rm -rf "build/dl" && ln -s ../../dl "build/dl"',
- workdir = Property("builddir"),
- logEnviron = False,
- want_stdout = False,
- ))
-
- # cleanup dl
- factory.addStep(ShellCommand(
- name = "dlprune",
- description = "Pruning dl/",
- descriptionDone = "dl/ pruned",
- command = 'find dl/ -mindepth 1 -atime +15 -delete -print',
- logEnviron = False,
- ))
-
- # prepare tar
- factory.addStep(ShellCommand(
- name = "dltar",
- description = "Building and installing GNU tar",
- descriptionDone = "GNU tar built and installed",
- command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/tar/compile", "V=s"],
- env = MakeEnv(tryccache=True),
- haltOnFailure = True,
- ))
-
- # populate dl
- factory.addStep(ShellCommand(
- name = "dlrun",
- description = "Populating dl/",
- descriptionDone = "dl/ populated",
- command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "download", "V=s"],
- env = MakeEnv(),
- logEnviron = False,
- locks = NetLockDl,
- ))
-
- factory.addStep(ShellCommand(
- name = "cleanbase",
- description = "Cleaning base-files",
- command=["make", "package/base-files/clean", "V=s"],
- ))
-
- # build
- factory.addStep(ShellCommand(
- name = "tools",
- description = "Building and installing tools",
- descriptionDone = "Tools built and installed",
- command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/install", "V=s"],
- env = MakeEnv(tryccache=True),
- haltOnFailure = True,
- ))
-
- factory.addStep(ShellCommand(
- name = "toolchain",
- description = "Building and installing toolchain",
- descriptionDone = "Toolchain built and installed",
- command=["make", Interpolate("-j%(prop:nproc:-1)s"), "toolchain/install", "V=s"],
- env = MakeEnv(),
- haltOnFailure = True,
- ))
-
- factory.addStep(ShellCommand(
- name = "kmods",
- description = "Building kmods",
- descriptionDone = "Kmods built",
- command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
- env = MakeEnv(),
- haltOnFailure = True,
- ))
-
- # find kernel version
- factory.addStep(SetPropertyFromCommand(
- name = "kernelversion",
- property = "kernelversion",
- description = "Finding the effective Kernel version",
- command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
- env = { 'TOPDIR': Interpolate("%(prop:builddir)s/build") },
- ))
-
- factory.addStep(ShellCommand(
- name = "pkgclean",
- description = "Cleaning up package build",
- descriptionDone = "Package build cleaned up",
- command=["make", "package/cleanup", "V=s"],
- ))
-
- factory.addStep(ShellCommand(
- name = "pkgbuild",
- description = "Building packages",
- descriptionDone = "Packages built",
- command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
- env = MakeEnv(),
- haltOnFailure = True,
- ))
-
- factory.addStep(ShellCommand(
- name = "pkginstall",
- description = "Installing packages",
- descriptionDone = "Packages installed",
- command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/install", "V=s"],
- env = MakeEnv(),
- haltOnFailure = True,
- ))
-
- factory.addStep(ShellCommand(
- name = "pkgindex",
- description = "Indexing packages",
- descriptionDone = "Packages indexed",
- command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
- env = MakeEnv(),
- haltOnFailure = True,
- ))
-
- factory.addStep(ShellCommand(
- name = "images",
- description = "Building and installing images",
- descriptionDone = "Images built and installed",
- command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/install", "V=s"],
- env = MakeEnv(),
- haltOnFailure = True,
- ))
-
- factory.addStep(ShellCommand(
- name = "buildinfo",
- description = "Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
- command = "make -j1 buildinfo V=s || true",
- env = MakeEnv(),
- haltOnFailure = True,
- ))
-
- factory.addStep(ShellCommand(
- name = "json_overview_image_info",
- description = "Generating profiles.json in target folder",
- command = "make -j1 json_overview_image_info V=s || true",
- env = MakeEnv(),
- haltOnFailure = True,
- ))
-
- factory.addStep(ShellCommand(
- name = "checksums",
- description = "Calculating checksums",
- descriptionDone = "Checksums calculated",
- command=["make", "-j1", "checksum", "V=s"],
- env = MakeEnv(),
- haltOnFailure = True,
- ))
-
- factory.addStep(ShellCommand(
- name = "kmoddir",
- descriptionDone = "Kmod directory created",
- command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])],
- haltOnFailure = True,
- doStepIf = IsKmodArchiveEnabled,
- ))
-
- factory.addStep(ShellCommand(
- name = "kmodprepare",
- description = "Preparing kmod archive",
- descriptionDone = "Kmod archive prepared",
- command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
- Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
- Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
- haltOnFailure = True,
- doStepIf = IsKmodArchiveEnabled,
- ))
-
- factory.addStep(ShellCommand(
- name = "kmodindex",
- description = "Indexing kmod archive",
- descriptionDone = "Kmod archive indexed",
- command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
- Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
- env = MakeEnv(),
- haltOnFailure = True,
- doStepIf = IsKmodArchiveEnabled,
- ))
-
- # sign
- factory.addStep(MasterShellCommand(
- name = "signprepare",
- descriptionDone = "Temporary signing directory prepared",
- command = ["mkdir", "-p", "%s/signing" %(work_dir)],
- haltOnFailure = True,
- doStepIf = IsSignEnabled,
-
- ))
-
- factory.addStep(ShellCommand(
- name = "signpack",
- description = "Packing files to sign",
- descriptionDone = "Files to sign packed",
- command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]),
- haltOnFailure = True,
- doStepIf = IsSignEnabled,
- ))
-
- factory.addStep(FileUpload(
- workersrc = "sign.tar.gz",
- masterdest = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
- haltOnFailure = True,
- doStepIf = IsSignEnabled,
- ))
-
- factory.addStep(MasterShellCommand(
- name = "signfiles",
- description = "Signing files",
- descriptionDone = "Files signed",
- command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]), Interpolate("%(prop:branch)s")],
- env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
- haltOnFailure = True,
- doStepIf = IsSignEnabled,
- ))
-
- factory.addStep(FileDownload(
- name = "dlsigntargz",
- mastersrc = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
- workerdest = "sign.tar.gz",
- haltOnFailure = True,
- doStepIf = IsSignEnabled,
- ))
-
- factory.addStep(ShellCommand(
- name = "signunpack",
- description = "Unpacking signed files",
- descriptionDone = "Signed files unpacked",
- command = ["tar", "-xzf", "sign.tar.gz"],
- haltOnFailure = True,
- doStepIf = IsSignEnabled,
- ))
-
- # upload
- factory.addStep(ShellCommand(
- name = "dirprepare",
- descriptionDone = "Upload directory structure prepared",
- command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
- haltOnFailure = True,
- ))
-
- factory.addStep(ShellCommand(
- name = "linkprepare",
- descriptionDone = "Repository symlink prepared",
- command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=util.Transform(GetBaseVersion, Property("branch"))), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
- doStepIf = IsNoMasterBuild,
- haltOnFailure = True,
- ))
-
- factory.addStep(ShellCommand(
- name = "kmoddirprepare",
- descriptionDone = "Kmod archive upload directory prepared",
- command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
- haltOnFailure = True,
- doStepIf = IsKmodArchiveEnabled,
- ))
-
- factory.addStep(ShellCommand(
- name = "dirupload",
- description = "Uploading directory structure",
- descriptionDone = "Directory structure uploaded",
- command = ["rsync", Interpolate("-az%(prop:rsync_ipv4:+4)s")] + rsync_defopts + ["tmp/upload/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url"))],
- env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
- haltOnFailure = True,
- logEnviron = False,
- locks = NetLockUl,
- doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
- ))
-
- # download remote sha256sums to 'target-sha256sums'
- factory.addStep(ShellCommand(
- name = "target-sha256sums",
- description = "Fetching remote sha256sums for target",
- descriptionDone = "Remote sha256sums for target fetched",
- command = ["rsync", Interpolate("-z%(prop:rsync_ipv4:+4)s")] + rsync_defopts + [Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
- env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
- logEnviron = False,
- haltOnFailure = False,
- flunkOnFailure = False,
- warnOnFailure = False,
- doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
- ))
-
- # build list of files to upload
- factory.addStep(FileDownload(
- name = "dlsha2rsyncpl",
- mastersrc = scripts_dir + '/sha2rsync.pl',
- workerdest = "../sha2rsync.pl",
- mode = 0o755,
- ))
-
- factory.addStep(ShellCommand(
- name = "buildlist",
- description = "Building list of files to upload",
- descriptionDone = "List of files to upload built",
- command = ["../sha2rsync.pl", "target-sha256sums", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", target=ts[0], subtarget=ts[1]), "rsynclist"],
- haltOnFailure = True,
- ))
-
- factory.addStep(FileDownload(
- name = "dlrsync.sh",
- mastersrc = scripts_dir + '/rsync.sh',
- workerdest = "../rsync.sh",
- mode = 0o755,
- ))
-
- # upload new files and update existing ones
- factory.addStep(ShellCommand(
- name = "targetupload",
- description = "Uploading target files",
- descriptionDone = "Target files uploaded",
- command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
- [Interpolate("-a%(prop:rsync_ipv4:+4)s"), Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
- Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
- env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
- haltOnFailure = True,
- logEnviron = False,
- doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
- ))
-
- # delete files which don't exist locally
- factory.addStep(ShellCommand(
- name = "targetprune",
- description = "Pruning target files",
- descriptionDone = "Target files pruned",
- command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
- [Interpolate("-a%(prop:rsync_ipv4:+4)s"), Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
- Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
- env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
- haltOnFailure = True,
- logEnviron = False,
- locks = NetLockUl,
- doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
- ))
-
- factory.addStep(ShellCommand(
- name = "kmodupload",
- description = "Uploading kmod archive",
- descriptionDone = "Kmod archive uploaded",
- command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
- [Interpolate("-a%(prop:rsync_ipv4:+4)s"), Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
- Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
- env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
- haltOnFailure = True,
- logEnviron = False,
- locks = NetLockUl,
- doStepIf = IsKmodArchiveAndRsyncEnabled,
- ))
-
- factory.addStep(ShellCommand(
- name = "sourcelist",
- description = "Finding source archives to upload",
- descriptionDone = "Source archives to upload found",
- command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -not -name '*.hash' -not -name '*.dl' -newer .config -printf '%f\\n' > sourcelist",
- haltOnFailure = True,
- ))
-
- factory.addStep(ShellCommand(
- name = "sourceupload",
- description = "Uploading source archives",
- descriptionDone = "Source archives uploaded",
- command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_defopts +
- [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), Interpolate("-a%(prop:rsync_ipv4:+4)s"), "dl/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url"))],
- env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")) },
- haltOnFailure = True,
- logEnviron = False,
- locks = NetLockUl,
- doStepIf = util.Transform(bool, GetRsyncParams.withArgs("src", "url")),
- ))
-
- factory.addStep(ShellCommand(
- name = "df",
- description = "Reporting disk usage",
- command=["df", "-h", "."],
- env={'LC_ALL': 'C'},
- logEnviron = False,
- haltOnFailure = False,
- flunkOnFailure = False,
- warnOnFailure = False,
- alwaysRun = True,
- ))
-
- factory.addStep(ShellCommand(
- name = "du",
- description = "Reporting estimated file space usage",
- command=["du", "-sh", "."],
- env={'LC_ALL': 'C'},
- logEnviron = False,
- haltOnFailure = False,
- flunkOnFailure = False,
- warnOnFailure = False,
- alwaysRun = True,
- ))
-
- factory.addStep(ShellCommand(
- name = "ccachestat",
- description = "Reporting ccache stats",
- command=["ccache", "-s"],
- logEnviron = False,
- want_stderr = False,
- haltOnFailure = False,
- flunkOnFailure = False,
- warnOnFailure = False,
- doStepIf = util.Transform(bool, Property("ccache_command")),
- ))
-
- for brname in branchNames:
- bldrname = brname + "_" + target
- c['builders'].append(BuilderConfig(name=bldrname, workernames=workerNames, factory=factory, tags=[brname,], nextBuild=GetNextBuild, canStartBuild=canStartBuild))
+ ts = target.split("/")
+
+ factory = BuildFactory()
+
+ # setup shared work directory if required
+ factory.addStep(
+ ShellCommand(
+ name="sharedwd",
+ descriptionDone="Shared work directory set up",
+ command='test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
+ workdir=".",
+ haltOnFailure=True,
+ )
+ )
+
+ # find number of cores
+ factory.addStep(
+ SetPropertyFromCommand(
+ name="nproc",
+ property="nproc",
+ description="Finding number of CPUs",
+ command=["nproc"],
+ )
+ )
+
+ # find gcc and g++ compilers
+ factory.addStep(
+ FileDownload(
+ name="dlfindbinpl",
+ mastersrc=scripts_dir + "/findbin.pl",
+ workerdest="../findbin.pl",
+ mode=0o755,
+ )
+ )
+
+ factory.addStep(
+ SetPropertyFromCommand(
+ name="gcc",
+ property="cc_command",
+ description="Finding gcc command",
+ command=["../findbin.pl", "gcc", "", ""],
+ haltOnFailure=True,
+ )
+ )
+
+ factory.addStep(
+ SetPropertyFromCommand(
+ name="g++",
+ property="cxx_command",
+ description="Finding g++ command",
+ command=["../findbin.pl", "g++", "", ""],
+ haltOnFailure=True,
+ )
+ )
+
+ # see if ccache is available
+ factory.addStep(
+ SetPropertyFromCommand(
+ name="ccache",
+ property="ccache_command",
+ description="Testing for ccache command",
+ command=["which", "ccache"],
+ haltOnFailure=False,
+ flunkOnFailure=False,
+ warnOnFailure=False,
+ hideStepIf=lambda r, s: r == results.FAILURE,
+ )
+ )
+
+ # check out the source
+ # Git() runs:
+ # if repo doesn't exist: 'git clone repourl'
+ # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -f -f -d -x'. Only works with mode='full'
+ # git cat-file -e <commit>
+ # git checkout -f <commit>
+ # git checkout -B <branch>
+ # git rev-parse HEAD
+ factory.addStep(
+ Git(
+ name="git",
+ repourl=repo_url,
+ mode="full",
+ method="fresh",
+ locks=NetLockDl,
+ haltOnFailure=True,
+ )
+ )
+
+ # workaround for https://github.com/openwrt/buildbot/issues/5
+ factory.addStep(
+ Git(
+ name="git me once more please",
+ repourl=repo_url,
+ mode="full",
+ method="fresh",
+ locks=NetLockDl,
+ haltOnFailure=True,
+ )
+ )
+
+ # update remote refs
+ factory.addStep(
+ ShellCommand(
+ name="fetchrefs",
+ description="Fetching Git remote refs",
+ descriptionDone="Git remote refs fetched",
+ command=[
+ "git",
+ "fetch",
+ "origin",
+ Interpolate(
+ "+refs/heads/%(prop:branch)s:refs/remotes/origin/%(prop:branch)s"
+ ),
+ ],
+ haltOnFailure=True,
+ )
+ )
+
+ # getver.sh requires local branches to track upstream otherwise version computation fails.
+ # Git() does not set tracking branches when cloning or switching, so work around this here
+ factory.addStep(
+ ShellCommand(
+ name="trackupstream",
+ description="Setting upstream branch",
+ descriptionDone="getver.sh is happy now",
+ command=["git", "branch", "-u", Interpolate("origin/%(prop:branch)s")],
+ haltOnFailure=True,
+ )
+ )
+
+ # Verify that Git HEAD points to a tag or branch
+ # Ref: https://web.archive.org/web/20190729224316/http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
+ factory.addStep(
+ ShellCommand(
+ name="gitverify",
+ description="Ensuring that Git HEAD is pointing to a branch or tag",
+ descriptionDone="Git HEAD is sane",
+ command='git rev-parse --abbrev-ref HEAD | grep -vxqF HEAD || git show-ref --tags --dereference 2>/dev/null | sed -ne "/^$(git rev-parse HEAD) / { s|^.*/||; s|\\^.*||; p }" | grep -qE "^v[0-9][0-9]\\."',
+ haltOnFailure=True,
+ )
+ )
+
+ factory.addStep(
+ StringDownload(
+ name="ccachecc",
+ s='#!/bin/sh\nexec ${CCACHE} ${CCC} "$@"\n',
+ workerdest="../ccache_cc.sh",
+ mode=0o755,
+ )
+ )
+
+ factory.addStep(
+ StringDownload(
+ name="ccachecxx",
+ s='#!/bin/sh\nexec ${CCACHE} ${CCXX} "$@"\n',
+ workerdest="../ccache_cxx.sh",
+ mode=0o755,
+ )
+ )
+
+ # feed
+ factory.addStep(
+ ShellCommand(
+ name="updatefeeds",
+ description="Updating feeds",
+ command=["./scripts/feeds", "update"],
+ env=MakeEnv(tryccache=True),
+ haltOnFailure=True,
+ locks=NetLockDl,
+ )
+ )
+
+ # feed
+ factory.addStep(
+ ShellCommand(
+ name="installfeeds",
+ description="Installing feeds",
+ command=["./scripts/feeds", "install", "-a"],
+ env=MakeEnv(tryccache=True),
+ haltOnFailure=True,
+ )
+ )
+
+ # seed config
+ factory.addStep(
+ StringDownload(
+ name="dlconfigseed",
+ s=Interpolate("%(kw:seed)s\n", seed=GetConfigSeed),
+ workerdest=".config",
+ mode=0o644,
+ )
+ )
+
+ # configure
+ factory.addStep(
+ ShellCommand(
+ name="newconfig",
+ descriptionDone=".config seeded",
+ command=Interpolate(
+ "printf 'CONFIG_TARGET_%(kw:target)s=y\\nCONFIG_TARGET_%(kw:target)s_%(kw:subtarget)s=y\\nCONFIG_SIGNED_PACKAGES=%(kw:usign:#?|y|n)s\\n' >> .config",
+ target=ts[0],
+ subtarget=ts[1],
+ usign=GetUsignKey,
+ ),
+ )
+ )
+
+ factory.addStep(
+ ShellCommand(
+ name="defconfig",
+ description="Populating .config",
+ command=["make", "defconfig"],
+ env=MakeEnv(),
+ )
+ )
+
+ # check arch - exit early if does not exist - NB: some targets do not define CONFIG_TARGET_target_subtarget
+ factory.addStep(
+ ShellCommand(
+ name="checkarch",
+ description="Checking architecture",
+ descriptionDone="Architecture validated",
+ command='grep -sq CONFIG_TARGET_%s=y .config && grep -sq CONFIG_TARGET_SUBTARGET=\\"%s\\" .config'
+ % (ts[0], ts[1]),
+ logEnviron=False,
+ want_stdout=False,
+ want_stderr=False,
+ haltOnFailure=True,
+ flunkOnFailure=False, # this is not a build FAILURE - TODO mark build as SKIPPED
+ )
+ )
+
+ # find libc suffix
+ factory.addStep(
+ SetPropertyFromCommand(
+ name="libc",
+ property="libc",
+ description="Finding libc suffix",
+ command=[
+ "sed",
+ "-ne",
+ '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }',
+ ".config",
+ ],
+ )
+ )
+
+ # install build key
+ factory.addStep(
+ StringDownload(
+ name="dlkeybuildpub",
+ s=Interpolate("%(kw:sec2pub)s", sec2pub=UsignSec2Pub),
+ workerdest="key-build.pub",
+ mode=0o600,
+ doStepIf=IsUsignEnabled,
+ )
+ )
+
+ factory.addStep(
+ StringDownload(
+ name="dlkeybuild",
+ s="# fake private key",
+ workerdest="key-build",
+ mode=0o600,
+ doStepIf=IsUsignEnabled,
+ )
+ )
+
+ factory.addStep(
+ StringDownload(
+ name="dlkeybuilducert",
+ s="# fake certificate",
+ workerdest="key-build.ucert",
+ mode=0o600,
+ doStepIf=IsUsignEnabled,
+ )
+ )
+
+ # prepare dl
+ factory.addStep(
+ ShellCommand(
+ name="dldir",
+ description="Preparing dl/",
+ descriptionDone="dl/ prepared",
+ command='mkdir -p ../dl && rm -rf "build/dl" && ln -s ../../dl "build/dl"',
+ workdir=Property("builddir"),
+ logEnviron=False,
+ want_stdout=False,
+ )
+ )
+
+ # cleanup dl
+ factory.addStep(
+ ShellCommand(
+ name="dlprune",
+ description="Pruning dl/",
+ descriptionDone="dl/ pruned",
+ command="find dl/ -mindepth 1 -atime +15 -delete -print",
+ logEnviron=False,
+ )
+ )
+
+ # prepare tar
+ factory.addStep(
+ ShellCommand(
+ name="dltar",
+ description="Building and installing GNU tar",
+ descriptionDone="GNU tar built and installed",
+ command=[
+ "make",
+ Interpolate("-j%(prop:nproc:-1)s"),
+ "tools/tar/compile",
+ "V=s",
+ ],
+ env=MakeEnv(tryccache=True),
+ haltOnFailure=True,
+ )
+ )
+
+ # populate dl
+ factory.addStep(
+ ShellCommand(
+ name="dlrun",
+ description="Populating dl/",
+ descriptionDone="dl/ populated",
+ command=["make", Interpolate("-j%(prop:nproc:-1)s"), "download", "V=s"],
+ env=MakeEnv(),
+ logEnviron=False,
+ locks=NetLockDl,
+ )
+ )
+
+ factory.addStep(
+ ShellCommand(
+ name="cleanbase",
+ description="Cleaning base-files",
+ command=["make", "package/base-files/clean", "V=s"],
+ )
+ )
+
+ # build
+ factory.addStep(
+ ShellCommand(
+ name="tools",
+ description="Building and installing tools",
+ descriptionDone="Tools built and installed",
+ command=[
+ "make",
+ Interpolate("-j%(prop:nproc:-1)s"),
+ "tools/install",
+ "V=s",
+ ],
+ env=MakeEnv(tryccache=True),
+ haltOnFailure=True,
+ )
+ )
+
+ factory.addStep(
+ ShellCommand(
+ name="toolchain",
+ description="Building and installing toolchain",
+ descriptionDone="Toolchain built and installed",
+ command=[
+ "make",
+ Interpolate("-j%(prop:nproc:-1)s"),
+ "toolchain/install",
+ "V=s",
+ ],
+ env=MakeEnv(),
+ haltOnFailure=True,
+ )
+ )
+
+ factory.addStep(
+ ShellCommand(
+ name="kmods",
+ description="Building kmods",
+ descriptionDone="Kmods built",
+ command=[
+ "make",
+ Interpolate("-j%(prop:nproc:-1)s"),
+ "target/compile",
+ "V=s",
+ "IGNORE_ERRORS=n m",
+ "BUILD_LOG=1",
+ ],
+ env=MakeEnv(),
+ haltOnFailure=True,
+ )
+ )
+
+ # find kernel version
+ factory.addStep(
+ SetPropertyFromCommand(
+ name="kernelversion",
+ property="kernelversion",
+ description="Finding the effective Kernel version",
+ command="make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
+ env={"TOPDIR": Interpolate("%(prop:builddir)s/build")},
+ )
+ )
+
+ factory.addStep(
+ ShellCommand(
+ name="pkgclean",
+ description="Cleaning up package build",
+ descriptionDone="Package build cleaned up",
+ command=["make", "package/cleanup", "V=s"],
+ )
+ )
+
+ factory.addStep(
+ ShellCommand(
+ name="pkgbuild",
+ description="Building packages",
+ descriptionDone="Packages built",
+ command=[
+ "make",
+ Interpolate("-j%(prop:nproc:-1)s"),
+ "package/compile",
+ "V=s",
+ "IGNORE_ERRORS=n m",
+ "BUILD_LOG=1",
+ ],
+ env=MakeEnv(),
+ haltOnFailure=True,
+ )
+ )
+
+ factory.addStep(
+ ShellCommand(
+ name="pkginstall",
+ description="Installing packages",
+ descriptionDone="Packages installed",
+ command=[
+ "make",
+ Interpolate("-j%(prop:nproc:-1)s"),
+ "package/install",
+ "V=s",
+ ],
+ env=MakeEnv(),
+ haltOnFailure=True,
+ )
+ )
+
+ factory.addStep(
+ ShellCommand(
+ name="pkgindex",
+ description="Indexing packages",
+ descriptionDone="Packages indexed",
+ command=[
+ "make",
+ Interpolate("-j%(prop:nproc:-1)s"),
+ "package/index",
+ "V=s",
+ "CONFIG_SIGNED_PACKAGES=",
+ ],
+ env=MakeEnv(),
+ haltOnFailure=True,
+ )
+ )
+
+ factory.addStep(
+ ShellCommand(
+ name="images",
+ description="Building and installing images",
+ descriptionDone="Images built and installed",
+ command=[
+ "make",
+ Interpolate("-j%(prop:nproc:-1)s"),
+ "target/install",
+ "V=s",
+ ],
+ env=MakeEnv(),
+ haltOnFailure=True,
+ )
+ )
+
+ factory.addStep(
+ ShellCommand(
+ name="buildinfo",
+ description="Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
+ command="make -j1 buildinfo V=s || true",
+ env=MakeEnv(),
+ haltOnFailure=True,
+ )
+ )
+
+ factory.addStep(
+ ShellCommand(
+ name="json_overview_image_info",
+ description="Generating profiles.json in target folder",
+ command="make -j1 json_overview_image_info V=s || true",
+ env=MakeEnv(),
+ haltOnFailure=True,
+ )
+ )
+
+ factory.addStep(
+ ShellCommand(
+ name="checksums",
+ description="Calculating checksums",
+ descriptionDone="Checksums calculated",
+ command=["make", "-j1", "checksum", "V=s"],
+ env=MakeEnv(),
+ haltOnFailure=True,
+ )
+ )
+
+ factory.addStep(
+ ShellCommand(
+ name="kmoddir",
+ descriptionDone="Kmod directory created",
+ command=[
+ "mkdir",
+ "-p",
+ Interpolate(
+ "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s",
+ target=ts[0],
+ subtarget=ts[1],
+ ),
+ ],
+ haltOnFailure=True,
+ doStepIf=IsKmodArchiveEnabled,
+ )
+ )
+
+ factory.addStep(
+ ShellCommand(
+ name="kmodprepare",
+ description="Preparing kmod archive",
+ descriptionDone="Kmod archive prepared",
+ command=[
+ "rsync",
+ "--include=/kmod-*.ipk",
+ "--exclude=*",
+ "-va",
+ Interpolate(
+ "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/",
+ target=ts[0],
+ subtarget=ts[1],
+ ),
+ Interpolate(
+ "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/",
+ target=ts[0],
+ subtarget=ts[1],
+ ),
+ ],
+ haltOnFailure=True,
+ doStepIf=IsKmodArchiveEnabled,
+ )
+ )
+
+ factory.addStep(
+ ShellCommand(
+ name="kmodindex",
+ description="Indexing kmod archive",
+ descriptionDone="Kmod archive indexed",
+ command=[
+ "make",
+ Interpolate("-j%(prop:nproc:-1)s"),
+ "package/index",
+ "V=s",
+ "CONFIG_SIGNED_PACKAGES=",
+ Interpolate(
+ "PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/",
+ target=ts[0],
+ subtarget=ts[1],
+ ),
+ ],
+ env=MakeEnv(),
+ haltOnFailure=True,
+ doStepIf=IsKmodArchiveEnabled,
+ )
+ )
+
+ # sign
+ factory.addStep(
+ MasterShellCommand(
+ name="signprepare",
+ descriptionDone="Temporary signing directory prepared",
+ command=["mkdir", "-p", "%s/signing" % (work_dir)],
+ haltOnFailure=True,
+ doStepIf=IsSignEnabled,
+ )
+ )
+
+ factory.addStep(
+ ShellCommand(
+ name="signpack",
+ description="Packing files to sign",
+ descriptionDone="Files to sign packed",
+ command=Interpolate(
+ "find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz",
+ target=ts[0],
+ subtarget=ts[1],
+ ),
+ haltOnFailure=True,
+ doStepIf=IsSignEnabled,
+ )
+ )
+
+ factory.addStep(
+ FileUpload(
+ workersrc="sign.tar.gz",
+ masterdest="%s/signing/%s.%s.tar.gz" % (work_dir, ts[0], ts[1]),
+ haltOnFailure=True,
+ doStepIf=IsSignEnabled,
+ )
+ )
+
+ factory.addStep(
+ MasterShellCommand(
+ name="signfiles",
+ description="Signing files",
+ descriptionDone="Files signed",
+ command=[
+ "%s/signall.sh" % (scripts_dir),
+ "%s/signing/%s.%s.tar.gz" % (work_dir, ts[0], ts[1]),
+ Interpolate("%(prop:branch)s"),
+ ],
+ env={"CONFIG_INI": os.getenv("BUILDMASTER_CONFIG", "./config.ini")},
+ haltOnFailure=True,
+ doStepIf=IsSignEnabled,
+ )
+ )
+
+ factory.addStep(
+ FileDownload(
+ name="dlsigntargz",
+ mastersrc="%s/signing/%s.%s.tar.gz" % (work_dir, ts[0], ts[1]),
+ workerdest="sign.tar.gz",
+ haltOnFailure=True,
+ doStepIf=IsSignEnabled,
+ )
+ )
+
+ factory.addStep(
+ ShellCommand(
+ name="signunpack",
+ description="Unpacking signed files",
+ descriptionDone="Signed files unpacked",
+ command=["tar", "-xzf", "sign.tar.gz"],
+ haltOnFailure=True,
+ doStepIf=IsSignEnabled,
+ )
+ )
+
+ # upload
+ factory.addStep(
+ ShellCommand(
+ name="dirprepare",
+ descriptionDone="Upload directory structure prepared",
+ command=[
+ "mkdir",
+ "-p",
+ Interpolate(
+ "tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s",
+ target=ts[0],
+ subtarget=ts[1],
+ prefix=GetVersionPrefix,
+ ),
+ ],
+ haltOnFailure=True,
+ )
+ )
+
+ factory.addStep(
+ ShellCommand(
+ name="linkprepare",
+ descriptionDone="Repository symlink prepared",
+ command=[
+ "ln",
+ "-s",
+ "-f",
+ Interpolate(
+ "../packages-%(kw:basever)s",
+ basever=util.Transform(GetBaseVersion, Property("branch")),
+ ),
+ Interpolate(
+ "tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix
+ ),
+ ],
+ doStepIf=IsNoMasterBuild,
+ haltOnFailure=True,
+ )
+ )
+
+ factory.addStep(
+ ShellCommand(
+ name="kmoddirprepare",
+ descriptionDone="Kmod archive upload directory prepared",
+ command=[
+ "mkdir",
+ "-p",
+ Interpolate(
+ "tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s",
+ target=ts[0],
+ subtarget=ts[1],
+ prefix=GetVersionPrefix,
+ ),
+ ],
+ haltOnFailure=True,
+ doStepIf=IsKmodArchiveEnabled,
+ )
+ )
+
+ factory.addStep(
+ ShellCommand(
+ name="dirupload",
+ description="Uploading directory structure",
+ descriptionDone="Directory structure uploaded",
+ command=["rsync", Interpolate("-az%(prop:rsync_ipv4:+4)s")]
+ + rsync_defopts
+ + [
+ "tmp/upload/",
+ Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url")),
+ ],
+ env={
+ "RSYNC_PASSWORD": Interpolate(
+ "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
+ )
+ },
+ haltOnFailure=True,
+ logEnviron=False,
+ locks=NetLockUl,
+ doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
+ )
+ )
+
+ # download remote sha256sums to 'target-sha256sums'
+ factory.addStep(
+ ShellCommand(
+ name="target-sha256sums",
+ description="Fetching remote sha256sums for target",
+ descriptionDone="Remote sha256sums for target fetched",
+ command=["rsync", Interpolate("-z%(prop:rsync_ipv4:+4)s")]
+ + rsync_defopts
+ + [
+ Interpolate(
+ "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums",
+ url=GetRsyncParams.withArgs("bin", "url"),
+ target=ts[0],
+ subtarget=ts[1],
+ prefix=GetVersionPrefix,
+ ),
+ "target-sha256sums",
+ ],
+ env={
+ "RSYNC_PASSWORD": Interpolate(
+ "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
+ )
+ },
+ logEnviron=False,
+ haltOnFailure=False,
+ flunkOnFailure=False,
+ warnOnFailure=False,
+ doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
+ )
+ )
+
+ # build list of files to upload
+ factory.addStep(
+ FileDownload(
+ name="dlsha2rsyncpl",
+ mastersrc=scripts_dir + "/sha2rsync.pl",
+ workerdest="../sha2rsync.pl",
+ mode=0o755,
+ )
+ )
+
+ factory.addStep(
+ ShellCommand(
+ name="buildlist",
+ description="Building list of files to upload",
+ descriptionDone="List of files to upload built",
+ command=[
+ "../sha2rsync.pl",
+ "target-sha256sums",
+ Interpolate(
+ "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums",
+ target=ts[0],
+ subtarget=ts[1],
+ ),
+ "rsynclist",
+ ],
+ haltOnFailure=True,
+ )
+ )
+
+ factory.addStep(
+ FileDownload(
+ name="dlrsync.sh",
+ mastersrc=scripts_dir + "/rsync.sh",
+ workerdest="../rsync.sh",
+ mode=0o755,
+ )
+ )
+
+ # upload new files and update existing ones
+ factory.addStep(
+ ShellCommand(
+ name="targetupload",
+ description="Uploading target files",
+ descriptionDone="Target files uploaded",
+ command=[
+ "../rsync.sh",
+ "--exclude=/kmods/",
+ "--files-from=rsynclist",
+ "--delay-updates",
+ "--partial-dir=.~tmp~%s~%s" % (ts[0], ts[1]),
+ ]
+ + rsync_defopts
+ + [
+ Interpolate("-a%(prop:rsync_ipv4:+4)s"),
+ Interpolate(
+ "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/",
+ target=ts[0],
+ subtarget=ts[1],
+ ),
+ Interpolate(
+ "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/",
+ url=GetRsyncParams.withArgs("bin", "url"),
+ target=ts[0],
+ subtarget=ts[1],
+ prefix=GetVersionPrefix,
+ ),
+ ],
+ env={
+ "RSYNC_PASSWORD": Interpolate(
+ "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
+ )
+ },
+ haltOnFailure=True,
+ logEnviron=False,
+ doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
+ )
+ )
+
+ # delete files which don't exist locally
+ factory.addStep(
+ ShellCommand(
+ name="targetprune",
+ description="Pruning target files",
+ descriptionDone="Target files pruned",
+ command=[
+ "../rsync.sh",
+ "--exclude=/kmods/",
+ "--delete",
+ "--existing",
+ "--ignore-existing",
+ "--delay-updates",
+ "--partial-dir=.~tmp~%s~%s" % (ts[0], ts[1]),
+ ]
+ + rsync_defopts
+ + [
+ Interpolate("-a%(prop:rsync_ipv4:+4)s"),
+ Interpolate(
+ "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/",
+ target=ts[0],
+ subtarget=ts[1],
+ ),
+ Interpolate(
+ "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/",
+ url=GetRsyncParams.withArgs("bin", "url"),
+ target=ts[0],
+ subtarget=ts[1],
+ prefix=GetVersionPrefix,
+ ),
+ ],
+ env={
+ "RSYNC_PASSWORD": Interpolate(
+ "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
+ )
+ },
+ haltOnFailure=True,
+ logEnviron=False,
+ locks=NetLockUl,
+ doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
+ )
+ )
+
+ factory.addStep(
+ ShellCommand(
+ name="kmodupload",
+ description="Uploading kmod archive",
+ descriptionDone="Kmod archive uploaded",
+ command=[
+ "../rsync.sh",
+ "--delete",
+ "--delay-updates",
+ "--partial-dir=.~tmp~%s~%s" % (ts[0], ts[1]),
+ ]
+ + rsync_defopts
+ + [
+ Interpolate("-a%(prop:rsync_ipv4:+4)s"),
+ Interpolate(
+ "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/",
+ target=ts[0],
+ subtarget=ts[1],
+ ),
+ Interpolate(
+ "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/",
+ url=GetRsyncParams.withArgs("bin", "url"),
+ target=ts[0],
+ subtarget=ts[1],
+ prefix=GetVersionPrefix,
+ ),
+ ],
+ env={
+ "RSYNC_PASSWORD": Interpolate(
+ "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
+ )
+ },
+ haltOnFailure=True,
+ logEnviron=False,
+ locks=NetLockUl,
+ doStepIf=IsKmodArchiveAndRsyncEnabled,
+ )
+ )
+
+ factory.addStep(
+ ShellCommand(
+ name="sourcelist",
+ description="Finding source archives to upload",
+ descriptionDone="Source archives to upload found",
+ command="find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -not -name '*.hash' -not -name '*.dl' -newer .config -printf '%f\\n' > sourcelist",
+ haltOnFailure=True,
+ )
+ )
+
+ factory.addStep(
+ ShellCommand(
+ name="sourceupload",
+ description="Uploading source archives",
+ descriptionDone="Source archives uploaded",
+ command=[
+ "../rsync.sh",
+ "--files-from=sourcelist",
+ "--size-only",
+ "--delay-updates",
+ ]
+ + rsync_defopts
+ + [
+ Interpolate(
+ "--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s",
+ target=ts[0],
+ subtarget=ts[1],
+ ),
+ Interpolate("-a%(prop:rsync_ipv4:+4)s"),
+ "dl/",
+ Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url")),
+ ],
+ env={
+ "RSYNC_PASSWORD": Interpolate(
+ "%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")
+ )
+ },
+ haltOnFailure=True,
+ logEnviron=False,
+ locks=NetLockUl,
+ doStepIf=util.Transform(bool, GetRsyncParams.withArgs("src", "url")),
+ )
+ )
+
+ factory.addStep(
+ ShellCommand(
+ name="df",
+ description="Reporting disk usage",
+ command=["df", "-h", "."],
+ env={"LC_ALL": "C"},
+ logEnviron=False,
+ haltOnFailure=False,
+ flunkOnFailure=False,
+ warnOnFailure=False,
+ alwaysRun=True,
+ )
+ )
+
+ factory.addStep(
+ ShellCommand(
+ name="du",
+ description="Reporting estimated file space usage",
+ command=["du", "-sh", "."],
+ env={"LC_ALL": "C"},
+ logEnviron=False,
+ haltOnFailure=False,
+ flunkOnFailure=False,
+ warnOnFailure=False,
+ alwaysRun=True,
+ )
+ )
+
+ factory.addStep(
+ ShellCommand(
+ name="ccachestat",
+ description="Reporting ccache stats",
+ command=["ccache", "-s"],
+ logEnviron=False,
+ want_stderr=False,
+ haltOnFailure=False,
+ flunkOnFailure=False,
+ warnOnFailure=False,
+ doStepIf=util.Transform(bool, Property("ccache_command")),
+ )
+ )
+
+ for brname in branchNames:
+ bldrname = brname + "_" + target
+ c["builders"].append(
+ BuilderConfig(
+ name=bldrname,
+ workernames=workerNames,
+ factory=factory,
+ tags=[
+ brname,
+ ],
+ nextBuild=GetNextBuild,
+ canStartBuild=canStartBuild,
+ )
+ )