phase2: compute checksums

[buildbot.git] / phase2 / master.cfg

diff --git a/phase2/master.cfg b/phase2/master.cfg
index b0aacc54ab0f08d0b234ac1ac80b276e3d0cd87d..774f1a637055c9996ae3862556302fe143bc9899 100644 (file)
--- a/phase2/master.cfg
+++ b/phase2/master.cfg
@@ -150,6 +150,8 @@ if ini.has_option("rsync", "sdk_password"):
 if ini.has_option("rsync", "sdk_pattern"):
        rsync_sdk_pat = ini.get("rsync", "sdk_pattern")
 
+rsync_defopts = ["-4", "-v", "--timeout=120"]
+
 repo_url = ini.get("repo", "url")
 repo_branch = "master"
 
@@ -492,7 +494,7 @@ for arch in arches:
        factory.addStep(ShellCommand(
                name = "downloadsdk",
                description = "Downloading SDK archive",
-               command = ["rsync", "-4", "-va", "%s/%s/%s/%s" %(rsync_sdk_url, ts[0], ts[1], rsync_sdk_pat), "sdk.archive"],
+               command = ["rsync"] + rsync_defopts + ["-a", "%s/%s/%s/%s" %(rsync_sdk_url, ts[0], ts[1], rsync_sdk_pat), "sdk.archive"],
                env={'RSYNC_PASSWORD': rsync_sdk_key},
                haltOnFailure = True,
                logEnviron = False))
@@ -506,7 +508,7 @@ for arch in arches:
        factory.addStep(ShellCommand(
                name = "updatesdk",
                description = "Updating SDK",
-               command = "rsync --checksum -av sdk_update/ sdk/ && rm -rf sdk_update",
+               command = "rsync " + (" ").join(rsync_defopts) + " --checksum -a sdk_update/ sdk/ && rm -rf sdk_update",
                haltOnFailure = True))
 
        factory.addStep(ShellCommand(
@@ -572,13 +574,6 @@ for arch in arches:
                command = ["./ccache.sh"],
                haltOnFailure = True))
 
-       factory.addStep(ShellCommand(
-               name = "patchfeedsconfgitfull",
-               description = "Patching feeds.conf to use src-git-full",
-               workdir = "build/sdk",
-               command = "sed -e 's#^src-git #src-git-full #g' feeds.conf.default > feeds.conf",
-               haltOnFailure = True))
-
        if git_ssh:
                factory.addStep(StringDownload(
                        name = "dlgitclonekey",
@@ -588,9 +583,9 @@ for arch in arches:
 
                factory.addStep(ShellCommand(
                        name = "patchfeedsconf",
-                       description = "Patching feeds.conf to use SSH cloning",
+                       description = "Patching feeds.conf",
                        workdir = "build/sdk",
-                       command = "sed -i -e 's#https://#ssh://git@#g' feeds.conf",
+                       command = "sed -e 's#https://#ssh://git@#g' feeds.conf.default > feeds.conf",
                        haltOnFailure = True))
 
        factory.addStep(ShellCommand(
@@ -641,6 +636,15 @@ for arch in arches:
                workdir = "build/sdk",
                command = "./scripts/feeds list -s -f > bin/packages/%s/feeds.conf" %(arch[0])))
 
+       factory.addStep(ShellCommand(
+               name = "checksums",
+               description = "Calculating checksums",
+               descriptionDone="Checksums calculated",
+               workdir = "build/sdk",
+               command = "cd bin/packages/%s; " %(arch[0]) + "find . -type f -not -name 'sha256sums' -printf \"%P\n\" | sort | xargs -r ../../../staging_dir/host/bin/mkhash -n sha256 | sed -ne 's!^\(.*\) \(.*\)$!\1 *\2!p' > sha256sums)",
+               haltOnFailure = True
+       ))
+
        if ini.has_option("gpg", "key") or usign_key is not None:
                factory.addStep(MasterShellCommand(
                        name = "signprepare",
@@ -685,11 +689,18 @@ for arch in arches:
                        haltOnFailure = True
                ))
 
+       factory.addStep(FileDownload(
+               name="dlrsync.sh",
+               mastersrc = scripts_dir + "/rsync.sh",
+               workerdest = "../rsync.sh",
+               mode = 0o755
+       ))
+
        factory.addStep(ShellCommand(
                name = "uploadprepare",
                description = "Preparing package directory",
                workdir = "build/sdk",
-               command = ["rsync", "-4", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", Interpolate("%(kw:rsyncbinurl)s/packages%(kw:suffix)s/", rsyncbinurl=rsync_bin_url, suffix=GetDirectorySuffix)],
+               command = ["rsync"] + rsync_defopts + ["-a", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", Interpolate("%(kw:rsyncbinurl)s/packages%(kw:suffix)s/", rsyncbinurl=rsync_bin_url, suffix=GetDirectorySuffix)],
                env={'RSYNC_PASSWORD': rsync_bin_key},
                haltOnFailure = True,
                logEnviron = False
@@ -699,7 +710,7 @@ for arch in arches:
                name = "packageupload",
                description = "Uploading package files",
                workdir = "build/sdk",
-               command = ["rsync", "-4", "--progress", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "bin/packages/%s/" %(arch[0]), Interpolate("%(kw:rsyncbinurl)s/packages%(kw:suffix)s/%(kw:archname)s/", rsyncbinurl=rsync_bin_url, suffix=GetDirectorySuffix, archname=arch[0])],
+               command = ["../../../rsync.sh"] + rsync_defopts + ["--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-a", "bin/packages/%s/" %(arch[0]), Interpolate("%(kw:rsyncbinurl)s/packages%(kw:suffix)s/%(kw:archname)s/", rsyncbinurl=rsync_bin_url, suffix=GetDirectorySuffix, archname=arch[0])],
                env={'RSYNC_PASSWORD': rsync_bin_key},
                haltOnFailure = True,
                logEnviron = False
@@ -709,7 +720,7 @@ for arch in arches:
                name = "logprepare",
                description = "Preparing log directory",
                workdir = "build/sdk",
-               command = ["rsync", "-4", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", Interpolate("%(kw:rsyncbinurl)s/faillogs%(kw:suffix)s/", rsyncbinurl=rsync_bin_url, suffix=GetDirectorySuffix)],
+               command = ["rsync"] + rsync_defopts + ["-a", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", Interpolate("%(kw:rsyncbinurl)s/faillogs%(kw:suffix)s/", rsyncbinurl=rsync_bin_url, suffix=GetDirectorySuffix)],
                env={'RSYNC_PASSWORD': rsync_bin_key},
                haltOnFailure = True,
                logEnviron = False
@@ -729,7 +740,7 @@ for arch in arches:
                name = "logcollect",
                description = "Collecting failure logs",
                workdir = "build/sdk",
-               command = ["rsync", "-av", "--files-from=logs.txt", "logs/package/feeds/", "faillogs/"],
+               command = ["rsync"] + rsync_defopts + ["-a", "--files-from=logs.txt", "logs/package/feeds/", "faillogs/"],
                haltOnFailure = False,
                flunkOnFailure = False,
                warnOnFailure = True,
@@ -739,7 +750,7 @@ for arch in arches:
                name = "logupload",
                description = "Uploading failure logs",
                workdir = "build/sdk",
-               command = ["rsync", "-4", "--progress", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "faillogs/", Interpolate("%(kw:rsyncbinurl)s/faillogs%(kw:suffix)s/%(kw:archname)s/", rsyncbinurl=rsync_bin_url, suffix=GetDirectorySuffix, archname=arch[0])],
+               command = ["../../../rsync.sh"] + rsync_defopts + ["--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-az", "faillogs/", Interpolate("%(kw:rsyncbinurl)s/faillogs%(kw:suffix)s/%(kw:archname)s/", rsyncbinurl=rsync_bin_url, suffix=GetDirectorySuffix, archname=arch[0])],
                env={'RSYNC_PASSWORD': rsync_bin_key},
                haltOnFailure = False,
                flunkOnFailure = False,
@@ -760,8 +771,8 @@ for arch in arches:
                        name = "sourceupload",
                        description = "Uploading source archives",
                        workdir = "build/sdk",
-                       command = ["rsync", "--files-from=sourcelist", "-4", "--progress", "--checksum", "--delay-updates",
-                                       Interpolate("--partial-dir=.~tmp~%(kw:archname)s~%(prop:workername)s", archname=arch[0]), "-avz", "dl/", "%s/" %(rsync_src_url)],
+                       command = ["../../../rsync.sh"] + rsync_defopts + ["--files-from=sourcelist", "--checksum", "--delay-updates",
+                                       Interpolate("--partial-dir=.~tmp~%(kw:archname)s~%(prop:workername)s", archname=arch[0]), "-a", "dl/", "%s/" %(rsync_src_url)],
                        env={'RSYNC_PASSWORD': rsync_src_key},
                        haltOnFailure = False,
                        flunkOnFailure = False,
@@ -791,6 +802,17 @@ for arch in arches:
                alwaysRun = True
        ))
 
+       factory.addStep(ShellCommand(
+               name = "ccachestat",
+               description = "Reporting ccache stats",
+               command=["ccache", "-s"],
+               want_stderr = False,
+               haltOnFailure = False,
+               flunkOnFailure = False,
+               warnOnFailure = False,
+               alwaysRun = True,
+       ))
+
        c['builders'].append(BuilderConfig(name=arch[0], workernames=workerNames, factory=factory))
 
        c['schedulers'].append(schedulers.Triggerable(name="trigger_%s" % arch[0], builderNames=[ arch[0] ]))